chore: import upstream snapshot with attribution
CI / benchmark (push) Has been skipped
install-script / posix-syntax (push) Successful in 6m1s
CI / build-onnx (push) Failing after 6m43s
init-smoke / dry-run (push) Failing after 15m57s
security / govulncheck (push) Has been cancelled
security / trivy-fs (push) Has been cancelled
CI / test (1.26, ubuntu-latest) (push) Has been cancelled
Scorecard supply-chain security / Scorecard analysis (push) Has been cancelled
CI / test (1.26, macos-latest) (push) Has been cancelled
CI / build-windows (push) Has been cancelled
CI / lint (push) Has been cancelled
install-script / powershell-syntax (push) Has been cancelled
install-script / install (macos-14) (push) Has been cancelled
install-script / install (ubuntu-latest) (push) Has been cancelled

This commit is contained in:
wehub-resource-sync
2026-07-13 12:33:42 +08:00
commit a06f331eb8
3186 changed files with 689843 additions and 0 deletions
+270
View File
@@ -0,0 +1,270 @@
package graph
import (
"fmt"
"runtime"
"sync"
"testing"
)
// =============================================================================
// Scaled graph benchmarks — simulate RPi memory constraints
// =============================================================================
// graphSizes defines graph sizes for sub-benchmarks.
// "Tiny" approximates a small project on RPi, "Large" approximates a medium monorepo.
var graphSizes = []struct {
name string
nodes int
edges int
}{
{"Tiny_100", 100, 200},
{"Small_1K", 1_000, 3_000},
{"Medium_5K", 5_000, 15_000},
{"Large_10K", 10_000, 30_000},
{"XL_50K", 50_000, 150_000},
{"XXL_100K", 100_000, 300_000},
}
func buildScaledGraph(nodes, edges int) *Graph {
g := New()
for i := range nodes {
g.AddNode(&Node{
ID: fmt.Sprintf("pkg%d/file%d.go::sym%d", i/100, i/10, i),
Kind: KindFunction,
Name: fmt.Sprintf("sym%d", i),
FilePath: fmt.Sprintf("pkg%d/file%d.go", i/100, i/10),
Language: "go",
})
}
for i := range edges {
g.AddEdge(&Edge{
From: fmt.Sprintf("pkg%d/file%d.go::sym%d", (i%nodes)/100, (i%nodes)/10, i%nodes),
To: fmt.Sprintf("pkg%d/file%d.go::sym%d", ((i+1)%nodes)/100, ((i+1)%nodes)/10, (i+1)%nodes),
Kind: EdgeCalls,
})
}
return g
}
func BenchmarkGraph_AddNode_Scaled(b *testing.B) {
for _, sz := range graphSizes {
b.Run(sz.name, func(b *testing.B) {
b.ReportAllocs()
g := New()
b.ResetTimer()
for i := range b.N {
g.AddNode(&Node{
ID: fmt.Sprintf("file%d.go::func%d", i/10, i),
Kind: KindFunction,
Name: fmt.Sprintf("func%d", i),
})
}
})
}
}
func BenchmarkGraph_GetNode_Scaled(b *testing.B) {
for _, sz := range graphSizes {
b.Run(sz.name, func(b *testing.B) {
b.ReportAllocs()
g := buildScaledGraph(sz.nodes, sz.edges)
b.ResetTimer()
for i := range b.N {
g.GetNode(fmt.Sprintf("pkg%d/file%d.go::sym%d", (i%sz.nodes)/100, (i%sz.nodes)/10, i%sz.nodes))
}
})
}
}
func BenchmarkGraph_AllNodes_Scaled(b *testing.B) {
for _, sz := range graphSizes {
b.Run(sz.name, func(b *testing.B) {
b.ReportAllocs()
g := buildScaledGraph(sz.nodes, sz.edges)
b.ResetTimer()
for b.Loop() {
g.AllNodes()
}
})
}
}
func BenchmarkGraph_Stats_Scaled(b *testing.B) {
for _, sz := range graphSizes {
b.Run(sz.name, func(b *testing.B) {
b.ReportAllocs()
g := buildScaledGraph(sz.nodes, sz.edges)
b.ResetTimer()
for b.Loop() {
g.Stats()
}
})
}
}
// =============================================================================
// Eviction benchmarks — critical for memory-constrained devices
// =============================================================================
func BenchmarkGraph_EvictFile(b *testing.B) {
for _, sz := range graphSizes {
b.Run(sz.name, func(b *testing.B) {
b.ReportAllocs()
for b.Loop() {
b.StopTimer()
g := buildScaledGraph(sz.nodes, sz.edges)
b.StartTimer()
// Evict ~10% of files
for i := range sz.nodes / 10 {
g.EvictFile(fmt.Sprintf("pkg%d/file%d.go", i/100, i/10))
}
}
})
}
}
// =============================================================================
// Concurrent access benchmarks — RPi has 4 cores, contention matters
// =============================================================================
func BenchmarkGraph_ConcurrentRead(b *testing.B) {
for _, sz := range graphSizes {
b.Run(sz.name, func(b *testing.B) {
b.ReportAllocs()
g := buildScaledGraph(sz.nodes, sz.edges)
b.ResetTimer()
b.RunParallel(func(pb *testing.PB) {
i := 0
for pb.Next() {
g.GetNode(fmt.Sprintf("pkg%d/file%d.go::sym%d", (i%sz.nodes)/100, (i%sz.nodes)/10, i%sz.nodes))
i++
}
})
})
}
}
func BenchmarkGraph_ConcurrentReadWrite(b *testing.B) {
for _, sz := range graphSizes {
b.Run(sz.name, func(b *testing.B) {
b.ReportAllocs()
g := buildScaledGraph(sz.nodes, sz.edges)
b.ResetTimer()
b.RunParallel(func(pb *testing.PB) {
i := 0
for pb.Next() {
if i%10 == 0 {
// 10% writes
g.AddNode(&Node{
ID: fmt.Sprintf("new%d", i+sz.nodes),
Kind: KindFunction,
Name: fmt.Sprintf("newFunc%d", i),
})
} else {
g.GetNode(fmt.Sprintf("pkg%d/file%d.go::sym%d", (i%sz.nodes)/100, (i%sz.nodes)/10, i%sz.nodes))
}
i++
}
})
})
}
}
// =============================================================================
// Memory footprint measurement
// =============================================================================
func BenchmarkGraph_MemoryFootprint(b *testing.B) {
for _, sz := range graphSizes {
b.Run(sz.name, func(b *testing.B) {
b.ReportAllocs()
for b.Loop() {
b.StopTimer()
runtime.GC()
var before runtime.MemStats
runtime.ReadMemStats(&before)
b.StartTimer()
g := buildScaledGraph(sz.nodes, sz.edges)
_ = g.NodeCount() // prevent optimization
b.StopTimer()
runtime.GC()
var after runtime.MemStats
runtime.ReadMemStats(&after)
b.ReportMetric(float64(after.HeapAlloc-before.HeapAlloc), "heap-bytes")
b.ReportMetric(float64(after.HeapAlloc-before.HeapAlloc)/float64(sz.nodes), "bytes/node")
b.StartTimer()
}
})
}
}
// =============================================================================
// GC pressure benchmark — important for RPi's limited memory bandwidth
// =============================================================================
func BenchmarkGraph_GCPressure(b *testing.B) {
b.ReportAllocs()
g := buildScaledGraph(50_000, 150_000)
// Simulate churn: add and evict files repeatedly
b.ResetTimer()
for i := range b.N {
filePath := fmt.Sprintf("churn/file%d.go", i%100)
g.EvictFile(filePath)
for j := range 10 {
g.AddNode(&Node{
ID: fmt.Sprintf("%s::func%d", filePath, j),
Kind: KindFunction,
Name: fmt.Sprintf("func%d", j),
FilePath: filePath,
})
}
}
}
// =============================================================================
// Lock contention benchmark — simulates RPi's 4-core scenario
// =============================================================================
func BenchmarkGraph_LockContention(b *testing.B) {
for _, goroutines := range []int{2, 4, 8} {
b.Run(fmt.Sprintf("goroutines_%d", goroutines), func(b *testing.B) {
b.ReportAllocs()
g := buildScaledGraph(1000, 3000)
b.ResetTimer()
var wg sync.WaitGroup
opsPerGoroutine := b.N / goroutines
if opsPerGoroutine == 0 {
opsPerGoroutine = 1
}
for gr := range goroutines {
wg.Add(1)
go func(id int) {
defer wg.Done()
for i := range opsPerGoroutine {
switch i % 4 {
case 0:
g.GetNode(fmt.Sprintf("pkg%d/file%d.go::sym%d", (i%1000)/100, (i%1000)/10, i%1000))
case 1:
g.AllNodes()
case 2:
g.GetOutEdges(fmt.Sprintf("pkg%d/file%d.go::sym%d", (i%1000)/100, (i%1000)/10, i%1000))
case 3:
g.AddNode(&Node{
ID: fmt.Sprintf("g%d_n%d", id, i),
Kind: KindVariable,
Name: fmt.Sprintf("v%d", i),
})
}
}
}(gr)
}
wg.Wait()
})
}
}
+105
View File
@@ -0,0 +1,105 @@
package graph
import (
"fmt"
"testing"
)
func BenchmarkGraph_AddNode(b *testing.B) {
g := New()
for i := range b.N {
g.AddNode(&Node{
ID: fmt.Sprintf("file%d.go::func%d", i/10, i),
Kind: KindFunction,
Name: fmt.Sprintf("func%d", i),
})
}
}
func BenchmarkGraph_AddEdge(b *testing.B) {
g := New()
// Pre-populate nodes.
for i := range 1000 {
g.AddNode(&Node{
ID: fmt.Sprintf("node%d", i),
Kind: KindFunction,
Name: fmt.Sprintf("func%d", i),
})
}
b.ResetTimer()
for i := range b.N {
g.AddEdge(&Edge{
From: fmt.Sprintf("node%d", i%1000),
To: fmt.Sprintf("node%d", (i+1)%1000),
Kind: EdgeCalls,
})
}
}
func BenchmarkGraph_GetNode(b *testing.B) {
g := New()
for i := range 1000 {
g.AddNode(&Node{
ID: fmt.Sprintf("node%d", i),
Kind: KindFunction,
Name: fmt.Sprintf("func%d", i),
})
}
b.ResetTimer()
for i := range b.N {
g.GetNode(fmt.Sprintf("node%d", i%1000))
}
}
func BenchmarkGraph_FindNodesByName(b *testing.B) {
g := New()
for i := range 1000 {
g.AddNode(&Node{
ID: fmt.Sprintf("node%d", i),
Kind: KindFunction,
Name: fmt.Sprintf("func%d", i%50), // 50 unique names
})
}
b.ResetTimer()
for i := range b.N {
g.FindNodesByName(fmt.Sprintf("func%d", i%50))
}
}
func BenchmarkGraph_AllNodes(b *testing.B) {
g := New()
for i := range 1000 {
g.AddNode(&Node{
ID: fmt.Sprintf("node%d", i),
Kind: KindFunction,
Name: fmt.Sprintf("func%d", i),
})
}
b.ResetTimer()
for b.Loop() {
g.AllNodes()
}
}
func BenchmarkGraph_Stats(b *testing.B) {
g := New()
for i := range 500 {
g.AddNode(&Node{
ID: fmt.Sprintf("node%d", i),
Kind: KindFunction,
Name: fmt.Sprintf("func%d", i),
})
}
for i := range 500 {
g.AddNode(&Node{
ID: fmt.Sprintf("type%d", i),
Kind: KindType,
Name: fmt.Sprintf("Type%d", i),
Language: "go",
})
}
b.ResetTimer()
for b.Loop() {
g.Stats()
}
}
+85
View File
@@ -0,0 +1,85 @@
package graph
import (
"sort"
"strconv"
"strings"
)
// Edge call-site multiplicity.
//
// The graph natively keys edges by (From, To, Kind, FilePath, Line), so an AST
// extractor that emits one edge per call site preserves in-file multiplicity
// on its own. A *synthesized* producer that can only mint one edge per
// (From, To) — e.g. the LSP references-add pass, which sees N reference sites
// for one declaration — would otherwise collapse those N sites to one. Rather
// than mint N near-identical edges, such a producer keeps one edge (its
// primary site in FilePath/Line) and records the additional sites in
// Meta["call_sites"]; find_usages expands them back into one row per site.
//
// Meta-only: no Edge struct field is added, so no wire-contract / GCX encoder
// churn. Edge.Meta round-trips through the store, so the sites survive a warm
// restart on the sqlite backend (via AddBatch / PersistEdge).
// AppendCallSite records an additional call/reference site on an edge whose
// primary site stays in FilePath/Line. Extra sites are stored in
// Meta["call_sites"] as sorted, deduped "<file>:<line>" strings; the primary
// site is never duplicated there.
func AppendCallSite(e *Edge, filePath string, line int) {
if e == nil || filePath == "" || line <= 0 {
return
}
if filePath == e.FilePath && line == e.Line {
return // the primary site lives in FilePath/Line, not call_sites
}
site := filePath + ":" + strconv.Itoa(line)
sites := CallSites(e)
for _, s := range sites {
if s == site {
return
}
}
sites = append(sites, site)
sort.Strings(sites)
if e.Meta == nil {
e.Meta = map[string]any{}
}
e.Meta["call_sites"] = sites
}
// CallSites returns the extra "<file>:<line>" sites recorded on an edge,
// tolerating both the in-memory []string form and the []any form a JSON meta
// round-trip (disk backend) produces.
func CallSites(e *Edge) []string {
if e == nil || e.Meta == nil {
return nil
}
switch v := e.Meta["call_sites"].(type) {
case []string:
return v
case []any:
out := make([]string, 0, len(v))
for _, x := range v {
if s, ok := x.(string); ok {
out = append(out, s)
}
}
return out
}
return nil
}
// SplitCallSite splits a "<file>:<line>" call-site string into its file and
// 1-based line, returning ("", 0) when malformed. It splits on the last colon
// so a path that itself contains a colon is handled.
func SplitCallSite(site string) (string, int) {
i := strings.LastIndexByte(site, ':')
if i <= 0 || i == len(site)-1 {
return "", 0
}
line, err := strconv.Atoi(site[i+1:])
if err != nil || line <= 0 {
return "", 0
}
return site[:i], line
}
+45
View File
@@ -0,0 +1,45 @@
package graph
import (
"testing"
"github.com/stretchr/testify/assert"
)
func TestAppendCallSite(t *testing.T) {
e := &Edge{From: "a", To: "b", Kind: EdgeCalls, FilePath: "f.go", Line: 12}
// The primary site is never duplicated into call_sites.
AppendCallSite(e, "f.go", 12)
assert.Empty(t, CallSites(e))
// Extra sites accumulate, sorted and deduped.
AppendCallSite(e, "f.go", 20)
AppendCallSite(e, "f.go", 14)
AppendCallSite(e, "f.go", 20) // duplicate — ignored
assert.Equal(t, []string{"f.go:14", "f.go:20"}, CallSites(e))
// Malformed inputs are ignored.
AppendCallSite(e, "", 5)
AppendCallSite(e, "g.go", 0)
AppendCallSite(nil, "g.go", 5)
assert.Equal(t, []string{"f.go:14", "f.go:20"}, CallSites(e))
}
func TestCallSites_JSONRoundTripForm(t *testing.T) {
// A meta round-trip through JSON (disk backend) yields []any, not []string.
e := &Edge{Meta: map[string]any{"call_sites": []any{"f.go:3", "f.go:9"}}}
assert.Equal(t, []string{"f.go:3", "f.go:9"}, CallSites(e))
}
func TestSplitCallSite(t *testing.T) {
f, l := SplitCallSite("dir/f.go:42")
assert.Equal(t, "dir/f.go", f)
assert.Equal(t, 42, l)
for _, bad := range []string{"bad", "f.go:", ":42", "f.go:abc", "f.go:0"} {
f, l := SplitCallSite(bad)
assert.Equal(t, "", f, "malformed %q", bad)
assert.Equal(t, 0, l, "malformed %q", bad)
}
}
+215
View File
@@ -0,0 +1,215 @@
package graph
import "strings"
// ConcurrencyAnnotation carries two cheap, high-signal concurrency
// facts about a symbol — typically a caller surfaced by a navigation
// query. Both flags default to false; the *Why fields are populated
// only when the matching flag is true, so a zero-value annotation is
// safe to attach unconditionally and to omit from a response.
type ConcurrencyAnnotation struct {
// SyncGuarded is true when the symbol is a method (or a closure
// defined on a method-bearing type) whose receiver / parent type
// holds a lock — a sync.Mutex / sync.RWMutex field in Go, a Mutex
// / RwLock field in Rust, a ReentrantLock field in Java, a
// SemaphoreSlim-typed field in C#. Calls made from inside such a
// type are presumptively lock-protected, which matters when
// reasoning about whether a change is concurrency-safe.
SyncGuarded bool `json:"sync_guarded,omitempty"`
SyncGuardedWhy string `json:"sync_guarded_why,omitempty"`
// CrossConcurrent is true when the symbol is launched across a
// concurrency boundary — it is the target of an EdgeSpawns edge,
// i.e. a `go` statement / goroutine closure in Go, an async / Promise
// / worker entry in JS-TS, a threading.Thread / async def in
// Python, a spawned thread in Rust. Any call the symbol makes runs
// on a different goroutine / thread than its lexical parent.
CrossConcurrent bool `json:"cross_concurrent,omitempty"`
CrossConcurrentWhy string `json:"cross_concurrent_why,omitempty"`
}
// Any reports whether either concurrency flag is set. Callers use it
// to decide whether an annotation is worth attaching / serialising.
func (c ConcurrencyAnnotation) Any() bool {
return c.SyncGuarded || c.CrossConcurrent
}
// ClassifyConcurrency derives the concurrency-safety annotation for a
// node. It reads through the Reader contract — so it works unchanged
// against a base graph or a per-session overlay view — and does no
// parser work: it reuses substrate that existing extractors already
// emit:
//
// - EdgeMemberOf links a method / field / closure to its parent
// type or enclosing function.
// - EdgeSpawns links a caller to a function / closure it launches
// asynchronously (goroutine, async, promise, worker pool).
// - A field node carries Meta["field_type"] with the verbatim
// declared type text.
//
// Language coverage:
//
// - sync_guarded relies on typed field nodes. Go, Rust, Java, and C#
// emit KindField nodes with Meta["field_type"], so a lock-holding
// receiver type is detected for those languages. TypeScript and
// PHP model class properties as KindVariable without a typed
// field-type, and Python does not materialise instance attributes
// as nodes at all — sync_guarded is therefore not reported for
// those languages (it stays false rather than guessing).
// - cross_concurrent relies only on EdgeSpawns and so covers every
// language whose extractor emits spawn edges (Go full; TS / Python
// / Rust / Kotlin / C# for the spawn patterns they detect).
//
// An unknown / missing node yields a zero-value annotation.
func ClassifyConcurrency(r Reader, nodeID string) ConcurrencyAnnotation {
var ann ConcurrencyAnnotation
if r == nil || r.GetNode(nodeID) == nil {
return ann
}
if why := spawnedAsConcurrent(r, nodeID); why != "" {
ann.CrossConcurrent = true
ann.CrossConcurrentWhy = why
}
if field, typeName := receiverLockField(r, r.GetNode(nodeID)); field != "" {
ann.SyncGuarded = true
ann.SyncGuardedWhy = "receiver type " + typeName +
" holds a lock (" + field + "); calls from here are presumptively lock-protected"
}
return ann
}
// spawnedAsConcurrent returns a human-readable explanation when the
// node is the target of at least one EdgeSpawns edge, and "" otherwise.
// The explanation names the spawn mode (goroutine / async / promise /
// worker_pool) recorded on the edge's Meta when available.
func spawnedAsConcurrent(r Reader, nodeID string) string {
for _, e := range r.GetInEdges(nodeID) {
if e.Kind != EdgeSpawns {
continue
}
mode, _ := e.Meta["mode"].(string)
switch mode {
case "goroutine":
return "launched as a goroutine — runs on a different goroutine than its caller"
case "async":
return "launched as an async task — runs off the caller's synchronous path"
case "promise":
return "launched inside a promise — runs off the caller's synchronous path"
case "worker_pool":
return "dispatched to a worker pool — runs on a pool thread, not the caller's"
default:
return "launched across a concurrency boundary (spawned), not called synchronously"
}
}
return ""
}
// receiverLockField finds the parent / receiver type of a method (or a
// closure whose enclosing scope is a method-bearing type) and reports
// the first lock-typed field declared on that type. Returns the field
// name and the type name, or ("", "") when the node is not a method,
// has no resolvable receiver type, or the type holds no lock.
func receiverLockField(r Reader, n *Node) (field, typeName string) {
if n == nil || (n.Kind != KindMethod && n.Kind != KindClosure) {
return "", ""
}
// A method (or closure) reaches its owner through EdgeMemberOf.
// For a closure the owner is usually the enclosing function, not a
// type — receiverTypeOf walks one extra hop in that case so a
// closure defined inside a method still resolves to the method's
// receiver type.
typeNode := receiverTypeOf(r, n)
if typeNode == nil {
return "", ""
}
// A type's fields point at it via EdgeMemberOf; walk the inbound
// member_of edges and inspect each field's declared type.
for _, e := range r.GetInEdges(typeNode.ID) {
if e.Kind != EdgeMemberOf {
continue
}
fn := r.GetNode(e.From)
if fn == nil || fn.Kind != KindField {
continue
}
ft, _ := fn.Meta["field_type"].(string)
if isLockTypeName(ft) {
return fn.Name, typeNode.Name
}
}
return "", ""
}
// receiverTypeOf resolves the type a method / closure belongs to. A
// method points straight at its receiver type via EdgeMemberOf. A
// closure points at its enclosing function / method; when that owner
// is itself a method the walk takes one more EdgeMemberOf hop so a
// closure spawned inside a method is still attributed to the method's
// receiver type. Returns nil when no KindType / KindInterface owner is
// reachable within those two hops.
func receiverTypeOf(r Reader, n *Node) *Node {
for _, e := range r.GetOutEdges(n.ID) {
if e.Kind != EdgeMemberOf {
continue
}
owner := r.GetNode(e.To)
if owner == nil {
continue
}
if owner.Kind == KindType || owner.Kind == KindInterface {
return owner
}
// Closure → enclosing method → receiver type (second hop).
if n.Kind == KindClosure && owner.Kind == KindMethod {
for _, e2 := range r.GetOutEdges(owner.ID) {
if e2.Kind != EdgeMemberOf {
continue
}
t := r.GetNode(e2.To)
if t != nil && (t.Kind == KindType || t.Kind == KindInterface) {
return t
}
}
}
}
return nil
}
// isLockTypeName reports whether a declared field-type string names a
// mutual-exclusion primitive. Matching is on the trailing type name so
// package / module qualifiers (`sync.Mutex`, `tokio::sync::Mutex`,
// `std::sync::RwLock`, `java.util.concurrent.locks.ReentrantLock`) and
// a single leading pointer / reference marker do not defeat it.
// Recognised across Go, Rust, Java, and C# — the languages whose
// extractors emit typed KindField nodes.
func isLockTypeName(fieldType string) bool {
t := strings.TrimSpace(fieldType)
if t == "" {
return false
}
t = strings.TrimPrefix(t, "*") // Go pointer-to-mutex
t = strings.TrimPrefix(t, "&") // Rust reference
// Drop a generic parameter list — Rust's Mutex<T> / RwLock<T>,
// C#'s lock wrappers — so the bare type name is what we test.
if i := strings.IndexByte(t, '<'); i >= 0 {
t = t[:i]
}
// Reduce a qualified path to its trailing segment.
for _, sep := range []string{"::", "."} {
if i := strings.LastIndex(t, sep); i >= 0 {
t = t[i+len(sep):]
}
}
switch strings.ToLower(strings.TrimSpace(t)) {
case "mutex", "rwmutex", // Go sync.Mutex / sync.RWMutex
"rwlock", // Rust std::sync::RwLock
"reentrantlock", // Java ReentrantLock
"reentrantreadwritelock", // Java ReentrantReadWriteLock
"readwritelock", "lock", // Java Lock / ReadWriteLock interfaces
"semaphore", // Java / C# Semaphore
"semaphoreslim", // C# SemaphoreSlim
"readerwriterlockslim", // C# ReaderWriterLockSlim
"spinlock": // Rust spin::Mutex-style / C# SpinLock
return true
}
return false
}
+239
View File
@@ -0,0 +1,239 @@
package graph
import "testing"
// addMethodNode / addTypeNode / addFieldNode / addEdge are tiny
// builders so each test case can pin exactly which nodes and edges
// exist. The classifier is graph-only, so a hand-built graph is the
// right fixture — no parser in the loop.
func addNode(g *Graph, id string, kind NodeKind, meta map[string]any) {
g.AddNode(&Node{ID: id, Kind: kind, Name: shortName(id), FilePath: "x.go", Meta: meta})
}
func shortName(id string) string {
for i := len(id) - 1; i >= 0; i-- {
if id[i] == '.' || id[i] == ':' {
return id[i+1:]
}
}
return id
}
func addCEdge(g *Graph, from, to string, kind EdgeKind, meta map[string]any) {
g.AddEdge(&Edge{From: from, To: to, Kind: kind, FilePath: "x.go", Origin: OriginASTResolved, Meta: meta})
}
func TestIsLockTypeName(t *testing.T) {
cases := []struct {
name string
in string
want bool
}{
{"go mutex", "sync.Mutex", true},
{"go rwmutex", "sync.RWMutex", true},
{"go pointer mutex", "*sync.Mutex", true},
{"go embedded mutex", "Mutex", true},
{"rust mutex generic", "Mutex<State>", true},
{"rust qualified rwlock", "std::sync::RwLock<Data>", true},
{"rust ref mutex", "&Mutex<u32>", true},
{"java reentrant lock", "ReentrantLock", true},
{"java qualified rwlock", "java.util.concurrent.locks.ReentrantReadWriteLock", true},
{"java lock iface", "Lock", true},
{"csharp semaphoreslim", "SemaphoreSlim", true},
{"csharp rwlockslim", "ReaderWriterLockSlim", true},
{"plain int not a lock", "int", false},
{"string not a lock", "string", false},
{"map not a lock", "map[string]int", false},
{"empty", "", false},
{"unrelated struct", "bytes.Buffer", false},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
if got := isLockTypeName(tc.in); got != tc.want {
t.Errorf("isLockTypeName(%q) = %v, want %v", tc.in, got, tc.want)
}
})
}
}
// TestClassifyConcurrency_SyncGuarded is the table-driven check that a
// method on a mutex-holding type is flagged sync_guarded and a method
// on a lock-free type is not.
func TestClassifyConcurrency_SyncGuarded(t *testing.T) {
cases := []struct {
name string
fieldType string // declared type of the receiver type's single field
fieldKind NodeKind
want bool
}{
{"go mutex field", "sync.Mutex", KindField, true},
{"go rwmutex field", "sync.RWMutex", KindField, true},
{"go embedded mutex", "Mutex", KindField, true},
{"rust mutex field", "Mutex<State>", KindField, true},
{"java reentrant lock field", "ReentrantLock", KindField, true},
{"lock-free int field", "int", KindField, false},
{"lock-free string field", "string", KindField, false},
// A class property modelled as KindVariable (TS / PHP) is not
// a KindField, so it must not be picked up — sync_guarded is
// honestly not reported for those languages.
{"variable-modelled property ignored", "Mutex", KindVariable, false},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
g := New()
addNode(g, "p.go::Store", KindType, nil)
addNode(g, "p.go::Store.Get", KindMethod, nil)
addCEdge(g, "p.go::Store.Get", "p.go::Store", EdgeMemberOf, nil)
addNode(g, "p.go::Store.f", tc.fieldKind, map[string]any{"field_type": tc.fieldType})
addCEdge(g, "p.go::Store.f", "p.go::Store", EdgeMemberOf, nil)
ann := ClassifyConcurrency(g, "p.go::Store.Get")
if ann.SyncGuarded != tc.want {
t.Errorf("SyncGuarded = %v, want %v", ann.SyncGuarded, tc.want)
}
if tc.want && ann.SyncGuardedWhy == "" {
t.Error("SyncGuarded true but SyncGuardedWhy is empty")
}
if !tc.want && ann.SyncGuardedWhy != "" {
t.Errorf("SyncGuarded false but SyncGuardedWhy = %q", ann.SyncGuardedWhy)
}
})
}
}
// TestClassifyConcurrency_PlainFunctionNeverGuarded confirms a free
// function (no receiver type) is never flagged sync_guarded.
func TestClassifyConcurrency_PlainFunctionNeverGuarded(t *testing.T) {
g := New()
addNode(g, "p.go::doWork", KindFunction, nil)
ann := ClassifyConcurrency(g, "p.go::doWork")
if ann.SyncGuarded {
t.Error("a plain function must not be sync_guarded")
}
}
// TestClassifyConcurrency_CrossConcurrent checks that a symbol that is
// the target of an EdgeSpawns edge is flagged cross_concurrent and a
// plainly-called symbol is not.
func TestClassifyConcurrency_CrossConcurrent(t *testing.T) {
cases := []struct {
name string
spawnMode string // "" means no spawn edge — plain call
spawned bool
want bool
wantWord string // substring expected in the explanation
}{
{"goroutine spawn", "goroutine", true, true, "goroutine"},
{"async spawn", "async", true, true, "async"},
{"promise spawn", "promise", true, true, "promise"},
{"worker pool spawn", "worker_pool", true, true, "worker pool"},
{"spawn without mode meta", "", true, true, "spawned"},
{"plain call only", "", false, false, ""},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
g := New()
addNode(g, "p.go::Caller", KindFunction, nil)
addNode(g, "p.go::Target", KindFunction, nil)
// A plain call edge is always present; it must never on
// its own make the target cross_concurrent.
addCEdge(g, "p.go::Caller", "p.go::Target", EdgeCalls, nil)
if tc.spawned {
var meta map[string]any
if tc.spawnMode != "" {
meta = map[string]any{"mode": tc.spawnMode}
}
addCEdge(g, "p.go::Caller", "p.go::Target", EdgeSpawns, meta)
}
ann := ClassifyConcurrency(g, "p.go::Target")
if ann.CrossConcurrent != tc.want {
t.Errorf("CrossConcurrent = %v, want %v", ann.CrossConcurrent, tc.want)
}
if tc.want {
if ann.CrossConcurrentWhy == "" {
t.Error("CrossConcurrent true but explanation empty")
}
if tc.wantWord != "" && !contains(ann.CrossConcurrentWhy, tc.wantWord) {
t.Errorf("explanation %q missing %q", ann.CrossConcurrentWhy, tc.wantWord)
}
} else if ann.CrossConcurrentWhy != "" {
t.Errorf("CrossConcurrent false but explanation = %q", ann.CrossConcurrentWhy)
}
})
}
}
// TestClassifyConcurrency_ClosureInheritsReceiverType verifies that a
// closure spawned inside a method resolves to the method's receiver
// type, so a goroutine-launched closure on a mutex-holding type is
// flagged sync_guarded.
func TestClassifyConcurrency_ClosureInheritsReceiverType(t *testing.T) {
g := New()
addNode(g, "p.go::Store", KindType, nil)
addNode(g, "p.go::Store.mu", KindField, map[string]any{"field_type": "sync.Mutex"})
addCEdge(g, "p.go::Store.mu", "p.go::Store", EdgeMemberOf, nil)
addNode(g, "p.go::Store.Run", KindMethod, nil)
addCEdge(g, "p.go::Store.Run", "p.go::Store", EdgeMemberOf, nil)
// Closure defined inside the method; member_of points at the method.
addNode(g, "p.go::Store.Run#closure@10", KindClosure, nil)
addCEdge(g, "p.go::Store.Run#closure@10", "p.go::Store.Run", EdgeMemberOf, nil)
// The method launches the closure as a goroutine.
addCEdge(g, "p.go::Store.Run", "p.go::Store.Run#closure@10", EdgeSpawns,
map[string]any{"mode": "goroutine"})
ann := ClassifyConcurrency(g, "p.go::Store.Run#closure@10")
if !ann.SyncGuarded {
t.Error("closure on a mutex-holding receiver type must be sync_guarded")
}
if !ann.CrossConcurrent {
t.Error("goroutine-launched closure must be cross_concurrent")
}
}
// TestClassifyConcurrency_BothFlags confirms the two flags are
// independent — a goroutine-launched method on a mutex type carries
// both.
func TestClassifyConcurrency_BothFlags(t *testing.T) {
g := New()
addNode(g, "p.go::Cache", KindType, nil)
addNode(g, "p.go::Cache.lock", KindField, map[string]any{"field_type": "sync.RWMutex"})
addCEdge(g, "p.go::Cache.lock", "p.go::Cache", EdgeMemberOf, nil)
addNode(g, "p.go::Cache.refresh", KindMethod, nil)
addCEdge(g, "p.go::Cache.refresh", "p.go::Cache", EdgeMemberOf, nil)
addNode(g, "p.go::Manager.Start", KindMethod, nil)
addCEdge(g, "p.go::Manager.Start", "p.go::Cache.refresh", EdgeSpawns,
map[string]any{"mode": "goroutine"})
ann := ClassifyConcurrency(g, "p.go::Cache.refresh")
if !ann.SyncGuarded || !ann.CrossConcurrent {
t.Errorf("expected both flags set, got sync_guarded=%v cross_concurrent=%v",
ann.SyncGuarded, ann.CrossConcurrent)
}
if !ann.Any() {
t.Error("Any() must be true when flags are set")
}
}
// TestClassifyConcurrency_MissingNode returns a zero-value annotation
// for an unknown node, and a nil reader is also tolerated.
func TestClassifyConcurrency_MissingNode(t *testing.T) {
g := New()
if ann := ClassifyConcurrency(g, "p.go::nope"); ann.Any() {
t.Error("unknown node must yield a zero-value annotation")
}
if ann := ClassifyConcurrency(nil, "p.go::nope"); ann.Any() {
t.Error("nil reader must yield a zero-value annotation")
}
}
func contains(haystack, needle string) bool {
if needle == "" {
return true
}
for i := 0; i+len(needle) <= len(haystack); i++ {
if haystack[i:i+len(needle)] == needle {
return true
}
}
return false
}
+66
View File
@@ -0,0 +1,66 @@
package graph
import (
"fmt"
"sync"
"testing"
)
// TestReindexEdge_ConcurrentAddEdge reproduces the "concurrent map read
// and map write" crash seen in production when two MCP handlers both
// trigger ensureFresh → indexFile: one calls AddEdge on shard From, the
// other calls ReindexEdge which mutates From's outEdgeIdx without
// locking From. Run with `-race` for the sharpest detection; even
// without -race the bare runtime map guard panics reliably here.
func TestReindexEdge_ConcurrentAddEdge(t *testing.T) {
g := New()
const n = 200
for i := range n {
g.AddNode(&Node{ID: fmt.Sprintf("from%d::F", i), Name: "F", Kind: KindFunction, FilePath: "f"})
g.AddNode(&Node{ID: fmt.Sprintf("to%d::T", i), Name: "T", Kind: KindFunction, FilePath: "t"})
g.AddNode(&Node{ID: fmt.Sprintf("alt%d::A", i), Name: "A", Kind: KindFunction, FilePath: "a"})
}
var wg sync.WaitGroup
wg.Add(2)
// Writer A: AddEdge against the From shards.
go func() {
defer wg.Done()
for round := range 50 {
for i := range n {
g.AddEdge(&Edge{
From: fmt.Sprintf("from%d::F", i),
To: fmt.Sprintf("to%d::T", i),
Kind: EdgeCalls,
FilePath: "f",
Line: round,
})
}
}
}()
// Writer B: ReindexEdge, retargeting onto a different shard each
// round — this is the resolver path that would collide with A.
go func() {
defer wg.Done()
for round := range 50 {
for i := range n {
e := &Edge{
From: fmt.Sprintf("from%d::F", i),
To: fmt.Sprintf("to%d::T", i),
Kind: EdgeCalls,
FilePath: "f",
Line: round + 1000,
}
g.AddEdge(e)
oldTo := e.To
e.To = fmt.Sprintf("alt%d::A", i)
g.ReindexEdge(e, oldTo)
}
}
}()
wg.Wait()
}
+52
View File
@@ -0,0 +1,52 @@
package graph
// IsContentNode reports whether n is a CONTENT section node — a KindDoc
// chunk tagged data_class="content" (text / pdf / pptx / xlsx section
// bodies). Content bodies are indexed in the dedicated content store
// (ContentSearcher), never the symbol search, and are excluded from the
// code-oriented analysis passes — so this predicate is the single place
// every package agrees on what "content" means. Markdown prose (KindDoc
// without data_class=content) and data assets (data_class="data") are NOT
// content and keep their existing treatment.
func IsContentNode(n *Node) bool {
if n == nil || n.Kind != KindDoc || n.Meta == nil {
return false
}
dc, _ := n.Meta["data_class"].(string)
return dc == "content"
}
// NonContentNodeReader is an optional store capability: a cheap (SQL-level
// on the disk backend) enumeration of a repo's NON-content nodes, so the
// code-oriented passes (search-index build, embedding, language detection)
// never materialise a content-heavy repo's hundreds of thousands of content
// sections just to iterate past them.
type NonContentNodeReader interface {
GetRepoNonContentNodes(repoPrefix string) []*Node
}
// RepoCodeNodes returns repoPrefix's non-content nodes. It uses the store's
// NonContentNodeReader fast path when available (the disk backend filters in
// SQL, so 525k content sections never enter memory); otherwise it falls back
// to materialising the repo's nodes and dropping content in Go — fine for the
// in-memory store, which only backs small repos. An empty repoPrefix means
// "all repos".
func RepoCodeNodes(s Store, repoPrefix string) []*Node {
if r, ok := s.(NonContentNodeReader); ok {
return r.GetRepoNonContentNodes(repoPrefix)
}
var nodes []*Node
if repoPrefix != "" {
nodes = s.GetRepoNodes(repoPrefix)
}
if len(nodes) == 0 {
nodes = s.AllNodes()
}
out := make([]*Node, 0, len(nodes))
for _, n := range nodes {
if !IsContentNode(n) {
out = append(out, n)
}
}
return out
}
+124
View File
@@ -0,0 +1,124 @@
package graph
import "strings"
// NormalizeCppType reduces a C++ type spelling to a stable comparison key used
// by both the extractor (stamping parameter types) and the overload resolver
// (normalizing call-site argument hints), so the two always compare in the same
// space. It strips template arguments, cv-qualifiers, ref/ptr punctuation, and
// namespace qualifiers, and canonicalises a few stdlib aliases — while keeping
// the integer/float ladder distinct (int vs long) so genuinely different
// overloads stay rankable.
func NormalizeCppType(raw string) string {
// Reduce a smart-pointer / optional wrapper to its pointee before stripping
// template arguments — `unique_ptr<Widget>` normalises to `Widget`, not
// `unique_ptr` — so member access through the wrapper lands on the pointee.
raw = UnwrapCppSmartPointer(raw)
s := stripCppTemplateArgs(raw)
s = strings.ReplaceAll(s, "&&", " ")
s = strings.ReplaceAll(s, "&", " ")
s = strings.ReplaceAll(s, "*", " ")
fields := strings.Fields(s)
kept := fields[:0]
for _, f := range fields {
if f == "const" || f == "volatile" {
continue
}
kept = append(kept, f)
}
s = strings.Join(kept, " ")
if i := strings.LastIndex(s, "::"); i >= 0 {
s = s[i+2:]
}
s = strings.TrimSpace(s)
switch s {
case "string", "basic_string", "string_view":
return "string"
case "unsigned", "unsigned int", "uint", "size_t", "uint32_t", "uint64_t":
return "unsigned"
}
return s
}
// cppSmartPointerWrappers names the single-pointee standard wrappers whose
// `<T>` is the type a member access (`p->m()`) or dereference (`*p`) actually
// reaches — each forwards operator-> / operator* (or, for optional, value
// access) to its pointee.
var cppSmartPointerWrappers = map[string]bool{
"unique_ptr": true,
"shared_ptr": true,
"weak_ptr": true,
"optional": true,
}
// UnwrapCppSmartPointer reduces a smart-pointer / optional wrapper to its
// pointee type text — `std::unique_ptr<Widget>` → `Widget`,
// `optional<shared_ptr<const Foo>>` → `const Foo`, `unique_ptr<Widget, Del>` →
// `Widget` — so a member access through the wrapper resolves on the pointee
// rather than the wrapper type. A non-wrapper type, including other generics
// like `vector<T>`, is returned unchanged. const / volatile qualifiers and
// ref / pointer suffixes on the wrapper are tolerated; nested wrappers are
// peeled until a non-wrapper type remains.
func UnwrapCppSmartPointer(s string) string {
for {
t := strings.TrimSpace(s)
t = strings.TrimPrefix(t, "const ")
t = strings.TrimPrefix(t, "volatile ")
t = strings.TrimRight(strings.TrimSpace(t), " &*")
t = strings.TrimSpace(t)
lt := strings.IndexByte(t, '<')
if lt <= 0 || !strings.HasSuffix(t, ">") {
return s
}
head := strings.TrimSpace(t[:lt])
if i := strings.LastIndex(head, "::"); i >= 0 {
head = head[i+2:]
}
if !cppSmartPointerWrappers[head] {
return s
}
s = cppFirstTemplateArg(strings.TrimSpace(t[lt+1 : len(t)-1]))
}
}
// cppFirstTemplateArg returns the first comma-separated template argument,
// respecting nested `<…>` so `Widget, Deleter` yields `Widget` and
// `map<int, Foo>` stays whole.
func cppFirstTemplateArg(args string) string {
depth := 0
for i := 0; i < len(args); i++ {
switch args[i] {
case '<':
depth++
case '>':
if depth > 0 {
depth--
}
case ',':
if depth == 0 {
return strings.TrimSpace(args[:i])
}
}
}
return strings.TrimSpace(args)
}
func stripCppTemplateArgs(s string) string {
depth := 0
var b strings.Builder
for _, r := range s {
switch r {
case '<':
depth++
case '>':
if depth > 0 {
depth--
}
default:
if depth == 0 {
b.WriteRune(r)
}
}
}
return b.String()
}
+40
View File
@@ -0,0 +1,40 @@
package graph
import "testing"
func TestUnwrapCppSmartPointer(t *testing.T) {
cases := map[string]string{
"std::unique_ptr<Widget>": "Widget",
"unique_ptr<Widget>": "Widget",
"std::shared_ptr<Widget>": "Widget",
"std::weak_ptr<Widget>": "Widget",
"std::optional<Widget>": "Widget",
"const std::shared_ptr<Widget> &": "Widget",
"std::unique_ptr<Widget, MyDeleter>": "Widget", // extra deleter arg dropped
"std::optional<std::shared_ptr<Foo>>": "Foo", // nested wrappers peeled
"std::unique_ptr<ns::Widget>": "ns::Widget", // pointee namespace kept
"std::vector<Widget>": "std::vector<Widget>", // non-wrapper unchanged
"Widget": "Widget",
"int": "int",
}
for in, want := range cases {
if got := UnwrapCppSmartPointer(in); got != want {
t.Errorf("UnwrapCppSmartPointer(%q) = %q, want %q", in, got, want)
}
}
}
func TestNormalizeCppType_SmartPointerPointee(t *testing.T) {
cases := map[string]string{
"std::unique_ptr<Widget>": "Widget",
"shared_ptr<Widget>": "Widget",
"std::optional<ns::Widget>": "Widget", // last :: segment, post-unwrap
"std::unique_ptr<std::vector<Widget>>": "vector", // unwrap to vector<Widget>, then strip args
"std::vector<Widget>": "vector", // non-wrapper still strips to container
}
for in, want := range cases {
if got := NormalizeCppType(in); got != want {
t.Errorf("NormalizeCppType(%q) = %q, want %q", in, got, want)
}
}
}
File diff suppressed because it is too large Load Diff
+168
View File
@@ -0,0 +1,168 @@
package graph
import "testing"
func TestOriginRank_Ordering(t *testing.T) {
// The ordering expresses the design contract: LSP-verified evidence
// outranks AST-extracted evidence, which outranks name-only matches.
order := []string{
OriginLSPResolved,
OriginLSPDispatch,
OriginASTResolved,
OriginASTInferred,
OriginTextMatched,
}
for i := 0; i < len(order)-1; i++ {
if OriginRank(order[i]) <= OriginRank(order[i+1]) {
t.Errorf("expected rank(%s)=%d > rank(%s)=%d",
order[i], OriginRank(order[i]),
order[i+1], OriginRank(order[i+1]))
}
}
}
func TestOriginRank_UnknownReturnsZero(t *testing.T) {
if got := OriginRank(""); got != 0 {
t.Errorf("empty origin: got rank %d, want 0", got)
}
if got := OriginRank("bogus_value"); got != 0 {
t.Errorf("unknown origin: got rank %d, want 0", got)
}
}
func TestMeetsMinTier(t *testing.T) {
tests := []struct {
origin, minTier string
want bool
}{
{OriginLSPResolved, OriginLSPResolved, true},
{OriginLSPResolved, OriginTextMatched, true},
{OriginASTResolved, OriginLSPResolved, false},
{OriginTextMatched, OriginASTInferred, false},
{OriginLSPDispatch, OriginASTResolved, true},
{"", OriginLSPResolved, false},
{OriginLSPResolved, "", true}, // no filter = all pass
{"", "", true}, // no filter, no origin = pass
}
for _, tt := range tests {
got := MeetsMinTier(tt.origin, tt.minTier)
if got != tt.want {
t.Errorf("MeetsMinTier(%q, %q) = %v, want %v",
tt.origin, tt.minTier, got, tt.want)
}
}
}
func TestDefaultOriginFor_SemanticSource(t *testing.T) {
// Any non-empty semantic source means a compiler-grade provider
// confirmed the edge — should map to LSP-resolved.
got := DefaultOriginFor(EdgeCalls, 1.0, "go-types")
if got != OriginLSPResolved {
t.Errorf("got %q, want %q", got, OriginLSPResolved)
}
}
func TestDefaultOriginFor_ImplementsWithSource(t *testing.T) {
// Interface → implementation via semantic provider = LSP dispatch.
got := DefaultOriginFor(EdgeImplements, 1.0, "lsp")
if got != OriginLSPDispatch {
t.Errorf("got %q, want %q", got, OriginLSPDispatch)
}
}
func TestDefaultOriginFor_StructuralAST(t *testing.T) {
for _, kind := range []EdgeKind{
EdgeDefines, EdgeImports, EdgeExtends, EdgeMemberOf,
EdgeImplements, EdgeProvides, EdgeConsumes,
} {
got := DefaultOriginFor(kind, 0, "")
if got != OriginASTResolved {
t.Errorf("kind=%s: got %q, want %q", kind, got, OriginASTResolved)
}
}
}
func TestDefaultOriginFor_ConfidenceBuckets(t *testing.T) {
tests := []struct {
conf float64
want string
}{
{1.0, OriginASTResolved},
{0.95, OriginASTResolved},
{0.7, OriginASTInferred},
{0.5, OriginASTInferred},
{0.3, OriginTextMatched},
{0, OriginTextMatched},
}
for _, tt := range tests {
got := DefaultOriginFor(EdgeCalls, tt.conf, "")
if got != tt.want {
t.Errorf("confidence=%v: got %q, want %q", tt.conf, got, tt.want)
}
}
}
// TestEdgeIdentityHash_StableForFixedOrigin proves IdentityHash is a
// pure function of (From, To, Kind, FilePath, Line, Origin): two
// separately-constructed edges with identical fields hash equal.
func TestEdgeIdentityHash_StableForFixedOrigin(t *testing.T) {
a := &Edge{From: "p/a.go::A", To: "p/b.go::B", Kind: EdgeCalls, FilePath: "p/a.go", Line: 12, Origin: OriginASTResolved}
b := &Edge{From: "p/a.go::A", To: "p/b.go::B", Kind: EdgeCalls, FilePath: "p/a.go", Line: 12, Origin: OriginASTResolved}
if a.IdentityHash() != b.IdentityHash() {
t.Fatal("edges with identical fields must share an identity hash")
}
}
// TestEdgeIdentityHash_DiffersOnlyByOrigin proves the deliverable: the
// identity hash changes iff Origin changes when every other field is
// held fixed. Each distinct tier yields a distinct identity, and the
// logical key alone (Origin-free) is NOT enough to pin the identity.
func TestEdgeIdentityHash_DiffersOnlyByOrigin(t *testing.T) {
base := func(origin string) *Edge {
return &Edge{From: "p/a.go::A", To: "p/b.go::B", Kind: EdgeCalls, FilePath: "p/a.go", Line: 12, Origin: origin}
}
origins := []string{
OriginLSPResolved, OriginLSPDispatch, OriginASTResolved,
OriginASTInferred, OriginTextMatched, "",
}
seen := make(map[edgeHash]string, len(origins))
for _, o := range origins {
h := base(o).IdentityHash()
if prev, dup := seen[h]; dup {
t.Fatalf("origins %q and %q collided to the same identity hash", prev, o)
}
seen[h] = o
}
// The Origin-free logical key must NOT determine the identity:
// two edges with the same keyOf but different Origin differ.
lsp := base(OriginLSPResolved)
txt := base(OriginTextMatched)
if keyOf(lsp) != keyOf(txt) {
t.Fatal("test setup: edges should share the logical key")
}
if lsp.IdentityHash() == txt.IdentityHash() {
t.Fatal("identity hash must include Origin — same logical key, different Origin must differ")
}
}
// TestEdgeIdentityHash_DiffersOnLogicalFields confirms IdentityHash
// still discriminates on the logical-key fields, so it is a strict
// superset of edgeKey's discrimination, not a replacement that only
// looks at Origin.
func TestEdgeIdentityHash_DiffersOnLogicalFields(t *testing.T) {
ref := &Edge{From: "p/a.go::A", To: "p/b.go::B", Kind: EdgeCalls, FilePath: "p/a.go", Line: 12, Origin: OriginASTResolved}
variants := []*Edge{
{From: "p/a.go::X", To: "p/b.go::B", Kind: EdgeCalls, FilePath: "p/a.go", Line: 12, Origin: OriginASTResolved},
{From: "p/a.go::A", To: "p/b.go::X", Kind: EdgeCalls, FilePath: "p/a.go", Line: 12, Origin: OriginASTResolved},
{From: "p/a.go::A", To: "p/b.go::B", Kind: EdgeReferences, FilePath: "p/a.go", Line: 12, Origin: OriginASTResolved},
{From: "p/a.go::A", To: "p/b.go::B", Kind: EdgeCalls, FilePath: "p/c.go", Line: 12, Origin: OriginASTResolved},
{From: "p/a.go::A", To: "p/b.go::B", Kind: EdgeCalls, FilePath: "p/a.go", Line: 99, Origin: OriginASTResolved},
}
for i, v := range variants {
if ref.IdentityHash() == v.IdentityHash() {
t.Errorf("variant %d: identity hash must differ when a logical-key field differs", i)
}
}
}
+29
View File
@@ -0,0 +1,29 @@
package graph
import "testing"
func TestViaLabelFor(t *testing.T) {
cases := map[string]string{
"swift.objc.bridge": "Swift↔ObjC bridge",
"observer.channel": "observer channel",
"closure.collection": "closure collection",
"react.setstate": "React setState",
"flutter.setstate": "Flutter setState",
"kmp.expect-actual": "KMP expect/actual",
"": "",
"unknown.synth": "unknown.synth", // unmapped passes through
}
for in, want := range cases {
if got := ViaLabelFor(in); got != want {
t.Errorf("ViaLabelFor(%q) = %q, want %q", in, got, want)
}
}
}
func TestEdgeIdentityHashIgnoresVia(t *testing.T) {
base := &Edge{From: "a", To: "b", Kind: EdgeCalls, Origin: OriginASTInferred}
withVia := &Edge{From: "a", To: "b", Kind: EdgeCalls, Origin: OriginASTInferred, Via: "observer channel"}
if base.IdentityHash() != withVia.IdentityHash() {
t.Errorf("Via must not change the edge identity hash")
}
}
+112
View File
@@ -0,0 +1,112 @@
package graph
import "testing"
func TestEnclosingFromID(t *testing.T) {
cases := []struct {
name string
id string
kind NodeKind
wantID string
wantName string
}{
{
name: "method on type",
id: "pkg/decoder.go::Decoder.Parse",
kind: KindMethod,
wantID: "pkg/decoder.go::Decoder",
wantName: "Decoder",
},
{
name: "field of struct",
id: "pkg/config.go::Config.Timeout",
kind: KindField,
wantID: "pkg/config.go::Config",
wantName: "Config",
},
{
name: "enum member",
id: "pkg/kind.go::NodeKind.Function",
kind: KindEnumMember,
wantID: "pkg/kind.go::NodeKind",
wantName: "NodeKind",
},
{
name: "closure inside function",
id: "pkg/run.go::Execute#closure@42",
kind: KindClosure,
wantID: "pkg/run.go::Execute",
wantName: "Execute",
},
{
name: "nested type method",
id: "pkg/x.go::Outer.Inner.Do",
kind: KindMethod,
wantID: "pkg/x.go::Outer.Inner",
wantName: "Inner",
},
{
name: "top-level function -- no owner",
id: "pkg/x.go::TopLevel",
kind: KindFunction,
wantID: "",
wantName: "",
},
{
name: "plain method ID without owner segment",
id: "pkg/x.go::Bare",
kind: KindMethod,
wantID: "",
wantName: "",
},
{
name: "type is never enclosed",
id: "pkg/x.go::SomeType",
kind: KindType,
wantID: "",
wantName: "",
},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
gotID, gotName := EnclosingFromID(tc.id, tc.kind)
if gotID != tc.wantID || gotName != tc.wantName {
t.Fatalf("EnclosingFromID(%q, %v) = (%q, %q), want (%q, %q)",
tc.id, tc.kind, gotID, gotName, tc.wantID, tc.wantName)
}
})
}
}
func TestNodeBrief_Enclosing(t *testing.T) {
// A method node surfaces its receiver type in Brief.
method := &Node{ID: "pkg/d.go::Decoder.Parse", Kind: KindMethod, Name: "Parse", FilePath: "pkg/d.go"}
b := method.Brief()
if b["enclosing"] != "Decoder" {
t.Errorf("method Brief enclosing = %v, want Decoder", b["enclosing"])
}
if b["enclosing_id"] != "pkg/d.go::Decoder" {
t.Errorf("method Brief enclosing_id = %v, want pkg/d.go::Decoder", b["enclosing_id"])
}
// A top-level function has no enclosing key at all.
fn := &Node{ID: "pkg/d.go::TopLevel", Kind: KindFunction, Name: "TopLevel", FilePath: "pkg/d.go"}
fb := fn.Brief()
if _, ok := fb["enclosing"]; ok {
t.Error("top-level function Brief should carry no enclosing key")
}
}
func TestEnclosingShortName(t *testing.T) {
cases := map[string]string{
"pkg/x.go::Owner": "Owner",
"Outer.Inner": "Inner",
"Plain": "Plain",
"pkg::A.B.C": "C",
}
for in, want := range cases {
if got := EnclosingShortName(in); got != want {
t.Errorf("EnclosingShortName(%q) = %q, want %q", in, got, want)
}
}
}
+201
View File
@@ -0,0 +1,201 @@
package graph
import (
"sort"
"strings"
)
// EpistemicBoundary names one unresolved/dynamic-dispatch site that makes a
// traversal count a *floor* rather than an exact number. It is the honest
// answer to "could the real blast radius / reachable set be larger?" — yes,
// because the resolver could not bind this site, so an unknown number of
// callers/callees hide behind it.
//
// It is an attribute of a *traversal*, not a graph element: no new NodeKind or
// EdgeKind is introduced. Boundaries are recorded at exactly the sites a walk
// would otherwise silently drop (an out-edge to an `unresolved::*` target) or
// where dynamic dispatch is structurally possible (the seed implements an
// interface, so callers may invoke it through that interface).
type EpistemicBoundary struct {
SeedID string `json:"seed_id"`
SeedName string `json:"seed_name,omitempty"`
Target string `json:"target,omitempty"`
EdgeKind string `json:"edge_kind,omitempty"`
Reason BoundaryReason `json:"reason"`
Direction string `json:"direction"` // "callers" | "callees"
}
// DynamicBoundary enriches an EpistemicBoundary with the body-level detail an
// agent needs to cross a runtime-dispatch site without a read-spiral: the SITE
// (file:line), the dispatch FORM, the KEY selector, and a candidate-target
// shortlist. Like EpistemicBoundary it is an attribute of a query result, not a
// graph element — it is computed on demand by scanning the disconnected
// symbol's body, never persisted.
type DynamicBoundary struct {
Site string `json:"site"` // file:line of the dispatch
Form string `json:"form"` // reflection | computed_member | event_bus
Key string `json:"key"` // the selector expression / event name
Candidates []string `json:"candidates,omitempty"`
AgentNamed bool `json:"agent_named,omitempty"`
}
// BoundaryReason classifies why a boundary makes the count a floor. The
// vocabulary aligns with the resolution-outcomes taxonomy's dynamic-dispatch
// concept while staying graph-local (no name-resolution needed to compute it).
type BoundaryReason string
const (
// BoundaryDynamicDispatch: an out call/reference edge whose target the
// resolver left as `unresolved::*` — the callee set could be larger.
BoundaryDynamicDispatch BoundaryReason = "dynamic_dispatch"
// BoundaryInterfaceDispatch: the node implements/overrides an interface
// method, so callers may invoke it through the interface via dispatch that
// is not attributed to this node — the caller set could be larger.
BoundaryInterfaceDispatch BoundaryReason = "interface_dispatch"
// BoundaryExternal: an edge into the `external::` namespace — the chain
// leaves the indexed code. Listed for transparency; not floor-making.
BoundaryExternal BoundaryReason = "external_boundary"
// BoundaryStub: an edge into a stdlib/builtin/module stub. Listed; not
// floor-making (an external stdlib call adds no in-repo callers/callees).
BoundaryStub BoundaryReason = "stub"
)
// maxBoundaries caps the per-result boundary list so a pathological hub cannot
// bloat a response. Mirrors impact.go's maxPerTier.
const maxBoundaries = 50
// ClassifyDroppedTarget classifies an edge target a traversal could not follow.
// ok=false means it is an ordinary in-graph node (follow it normally).
func ClassifyDroppedTarget(targetID string, kind EdgeKind) (BoundaryReason, bool) {
if IsUnresolvedTarget(targetID) {
return BoundaryDynamicDispatch, true
}
if strings.HasPrefix(targetID, "external::") {
return BoundaryExternal, true
}
if IsStub(targetID) {
return BoundaryStub, true
}
return "", false
}
// CalleeBoundaries scans the out-edges of the given nodes for call/reference
// targets a forward (callee-direction) walk could not follow. Each such target
// means the reachable callee set could be larger than what was returned.
func CalleeBoundaries(g Store, nodeIDs []string, limit int) []EpistemicBoundary {
if g == nil {
return nil
}
if limit <= 0 {
limit = maxBoundaries
}
seen := map[string]bool{}
var out []EpistemicBoundary
for _, id := range nodeIDs {
for _, e := range g.GetOutEdges(id) {
if e.Kind != EdgeCalls && e.Kind != EdgeReferences {
continue
}
reason, ok := ClassifyDroppedTarget(e.To, e.Kind)
if !ok {
continue
}
key := id + "\x00" + e.To
if seen[key] {
continue
}
seen[key] = true
out = append(out, EpistemicBoundary{
SeedID: id,
SeedName: nameForID(g, id),
Target: boundaryTargetName(e.To),
EdgeKind: string(e.Kind),
Reason: reason,
Direction: "callees",
})
if len(out) >= limit {
return sortBoundaries(out)
}
}
}
return sortBoundaries(out)
}
// CallerBoundaries flags nodes whose *caller* count is a floor because dynamic
// dispatch into them is structurally possible: each node that implements or
// overrides an interface method may be reached through that interface by
// callers not attributed to it directly. It names the interface so an agent
// can run find_implementations / get_callers on it to widen the picture.
func CallerBoundaries(g Store, nodeIDs []string, limit int) []EpistemicBoundary {
if g == nil {
return nil
}
if limit <= 0 {
limit = maxBoundaries
}
seen := map[string]bool{}
var out []EpistemicBoundary
for _, id := range nodeIDs {
for _, e := range g.GetOutEdges(id) {
if e.Kind != EdgeImplements && e.Kind != EdgeOverrides {
continue
}
key := id + "\x00" + e.To
if seen[key] {
continue
}
seen[key] = true
out = append(out, EpistemicBoundary{
SeedID: id,
SeedName: nameForID(g, id),
Target: boundaryTargetName(e.To),
EdgeKind: string(e.Kind),
Reason: BoundaryInterfaceDispatch,
Direction: "callers",
})
if len(out) >= limit {
return sortBoundaries(out)
}
}
}
return sortBoundaries(out)
}
// LowerBoundCaveat reports whether the boundary set makes the count a genuine
// floor. Only dynamic-dispatch / interface-dispatch boundaries qualify: an
// external/stdlib stub edge is listed for transparency but adds no hidden
// in-repo callers/callees, so by itself it must not raise the flag (otherwise
// nearly every symbol with a stdlib call would be flagged — see the design's
// over-flagging guard).
func LowerBoundCaveat(boundaries []EpistemicBoundary) bool {
for _, b := range boundaries {
if b.Reason == BoundaryDynamicDispatch || b.Reason == BoundaryInterfaceDispatch {
return true
}
}
return false
}
func boundaryTargetName(id string) string {
if IsUnresolvedTarget(id) {
return UnresolvedName(id)
}
return id
}
func nameForID(g Store, id string) string {
if n := g.GetNode(id); n != nil {
return n.Name
}
return ""
}
func sortBoundaries(bs []EpistemicBoundary) []EpistemicBoundary {
sort.SliceStable(bs, func(i, j int) bool {
if bs[i].SeedID != bs[j].SeedID {
return bs[i].SeedID < bs[j].SeedID
}
return bs[i].Target < bs[j].Target
})
return bs
}
+96
View File
@@ -0,0 +1,96 @@
package graph
import "testing"
func boundaryGraph() *Graph {
g := New()
for _, id := range []string{"A", "B", "C"} {
g.AddNode(&Node{ID: id, Kind: KindMethod, Name: id})
}
// A implements an interface method → caller-side dispatch boundary.
g.AddEdge(&Edge{From: "A", To: "iface::I.M", Kind: EdgeImplements, Origin: OriginASTResolved})
// B calls an unresolved (dynamic-dispatch) target → callee-side floor.
g.AddEdge(&Edge{From: "B", To: "unresolved::handler", Kind: EdgeCalls, Origin: OriginASTInferred})
// C calls a stdlib stub → listed but not floor-making.
g.AddEdge(&Edge{From: "C", To: "stdlib::fmt.Println", Kind: EdgeCalls, Origin: OriginASTResolved})
return g
}
func TestCalleeBoundaries(t *testing.T) {
g := boundaryGraph()
bs := CalleeBoundaries(g, []string{"B", "C"}, 0)
if len(bs) != 2 {
t.Fatalf("expected 2 callee boundaries, got %d: %+v", len(bs), bs)
}
byReason := map[BoundaryReason]EpistemicBoundary{}
for _, b := range bs {
byReason[b.Reason] = b
if b.Direction != "callees" {
t.Errorf("expected callees direction, got %s", b.Direction)
}
}
if d, ok := byReason[BoundaryDynamicDispatch]; !ok || d.Target != "handler" {
t.Errorf("expected dynamic_dispatch boundary on 'handler', got %+v", byReason)
}
if _, ok := byReason[BoundaryStub]; !ok {
t.Errorf("expected stub boundary for stdlib call, got %+v", byReason)
}
}
func TestCallerBoundaries(t *testing.T) {
g := boundaryGraph()
bs := CallerBoundaries(g, []string{"A"}, 0)
if len(bs) != 1 {
t.Fatalf("expected 1 caller boundary, got %d: %+v", len(bs), bs)
}
b := bs[0]
if b.Reason != BoundaryInterfaceDispatch || b.Direction != "callers" {
t.Errorf("unexpected boundary: %+v", b)
}
// A method that implements nothing has no caller boundary.
if got := CallerBoundaries(g, []string{"B"}, 0); len(got) != 0 {
t.Errorf("expected no boundary for non-implementing node, got %+v", got)
}
}
func TestLowerBoundCaveat(t *testing.T) {
cases := []struct {
name string
bs []EpistemicBoundary
want bool
}{
{"empty", nil, false},
{"dynamic", []EpistemicBoundary{{Reason: BoundaryDynamicDispatch}}, true},
{"interface", []EpistemicBoundary{{Reason: BoundaryInterfaceDispatch}}, true},
{"stub only", []EpistemicBoundary{{Reason: BoundaryStub}}, false},
{"external only", []EpistemicBoundary{{Reason: BoundaryExternal}}, false},
{"mixed", []EpistemicBoundary{{Reason: BoundaryStub}, {Reason: BoundaryDynamicDispatch}}, true},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
if got := LowerBoundCaveat(tc.bs); got != tc.want {
t.Errorf("LowerBoundCaveat = %v, want %v", got, tc.want)
}
})
}
}
func TestClassifyDroppedTarget(t *testing.T) {
cases := []struct {
target string
kind EdgeKind
reason BoundaryReason
ok bool
}{
{"unresolved::Foo", EdgeCalls, BoundaryDynamicDispatch, true},
{"external::lodash.map", EdgeCalls, BoundaryExternal, true},
{"stdlib::fmt.Println", EdgeCalls, BoundaryStub, true},
{"pkg/x.go::Real", EdgeCalls, "", false},
}
for _, tc := range cases {
reason, ok := ClassifyDroppedTarget(tc.target, tc.kind)
if ok != tc.ok || reason != tc.reason {
t.Errorf("ClassifyDroppedTarget(%q) = (%q,%v), want (%q,%v)", tc.target, reason, ok, tc.reason, tc.ok)
}
}
}
+250
View File
@@ -0,0 +1,250 @@
package graph
// ZeroEdgeClass classifies why a symbol's graph query came back empty.
// An empty result has two very different causes that an agent cannot
// otherwise tell apart, and a pre-edit safety check that trusts a
// false "0 usages" is silently disarmed.
type ZeroEdgeClass string
const (
// ZeroEdgeNone means the symbol has incoming call/reference edges:
// the query was not empty and no caveat is warranted.
ZeroEdgeNone ZeroEdgeClass = "none"
// ZeroEdgeLikelyUnused means the symbol has no incoming
// call/reference edges but DOES carry other graph edges — the
// structural edge from its file (`defines`), a method's
// `member_of`, or outgoing calls / references / type references.
// This is consistent with genuine dead code: the extractor saw
// the symbol, nothing uses it.
ZeroEdgeLikelyUnused ZeroEdgeClass = "likely_unused"
// ZeroEdgePossibleExtractionGap means the symbol has zero edges of
// any kind. A normally indexed function or method always carries
// at least one structural edge — the file `defines` it, a method
// is `member_of` its type — so zero total edges most likely means
// the extractor never processed the symbol or its file. The
// symbol may well be live; the graph just does not know. This is
// the dangerous case for a delete-or-rewrite decision.
ZeroEdgePossibleExtractionGap ZeroEdgeClass = "possible_extraction_gap"
// ZeroEdgeCoverageIncomplete means the symbol has no resolved
// incoming call/reference edges, but the graph DOES carry
// import-level evidence that consumers exist: inbound `imports` /
// `re_exports` edges land on the symbol itself, or (for a public
// JS/TS symbol) on its file. The usage query is incomplete, not
// empty — reference-level resolution failed somewhere upstream —
// so an agent must NOT read the empty result as safe-to-remove.
ZeroEdgeCoverageIncomplete ZeroEdgeClass = "coverage_incomplete"
)
// ZeroEdgeCaveat is the structured caveat attached to an empty graph
// query result. Class is machine-checkable so a safety gate can branch
// on it; Message is a short human-readable explanation.
type ZeroEdgeCaveat struct {
Class ZeroEdgeClass `json:"class" toon:"class"`
Message string `json:"message" toon:"message"`
}
// ZeroImpactCaveat is the per-symbol caveat attached to an empty impact
// analysis result, which is computed over a list of symbols. It carries
// the symbol ID alongside the same machine-checkable classification.
type ZeroImpactCaveat struct {
ID string `json:"id" toon:"id"`
Class ZeroEdgeClass `json:"class" toon:"class"`
Message string `json:"message" toon:"message"`
}
// TierFilteredClass is the machine-checkable class of a min_tier-emptied
// result — distinct from ZeroEdgeClass so a safety gate never confuses
// "filtered out by min_tier" with "genuinely has no usages".
const TierFilteredClass = "tier_filtered"
// TierFilteredCaveat is attached to a find_usages / get_* result whose edge
// list was emptied (or thinned) by a min_tier filter while lower-tier edges
// still exist. Without it a min_tier that filters every edge is
// indistinguishable from "no usages" — the honesty bug it fixes. Class is
// always TierFilteredClass; EdgesBelowMinTier counts the edges dropped by the
// filter and MaxAvailableTier names the best tier actually present.
type TierFilteredCaveat struct {
Class string `json:"class" toon:"class"`
EdgesBelowMinTier int `json:"edges_below_min_tier" toon:"edges_below_min_tier"`
MaxAvailableTier string `json:"max_available_tier" toon:"max_available_tier"`
}
// usageEdgeKinds are the incoming edge kinds that count as a symbol
// being "used" — calls, references, and the type/instantiation edges
// that find_usages itself treats as usages. An incoming edge of any of
// these kinds means the symbol is not dead code.
var usageEdgeKinds = map[EdgeKind]bool{
EdgeCalls: true,
EdgeReferences: true,
EdgeInstantiates: true,
EdgeImplements: true,
EdgeExtends: true,
EdgeReads: true,
EdgeWrites: true,
EdgeTests: true,
}
// UsageInboundEdgeKinds returns the canonical list of incoming edge
// kinds that classify a symbol as "used" by ClassifyZeroEdge. Exposed
// for capability callers (NodeDegreeAggregator) that need to mirror
// the in-graph usage filter server-side. Order is stable so the slice
// is safe to pass directly to a query parameter binding.
func UsageInboundEdgeKinds() []EdgeKind {
return []EdgeKind{
EdgeCalls,
EdgeReferences,
EdgeInstantiates,
EdgeImplements,
EdgeExtends,
EdgeReads,
EdgeWrites,
EdgeTests,
}
}
// ClassifyZeroEdge inspects a symbol's incoming and outgoing edges and
// returns how an empty usage/caller/impact query for it should be read.
//
// - ZeroEdgeNone — the symbol has at least one incoming usage edge.
// - ZeroEdgeLikelyUnused — no incoming usage edge, but the symbol has
// other graph edges (structural defines/member_of, or any outgoing
// edge). Consistent with genuine dead code.
// - ZeroEdgePossibleExtractionGap — the symbol has no edges at all,
// which a normally indexed symbol never has; the extractor most
// likely missed it.
//
// An unknown symbol ID is reported as an extraction gap: a query whose
// target is not even in the graph is exactly as untrustworthy as one
// whose target was never wired up.
func ClassifyZeroEdge(g Store, symbolID string) ZeroEdgeClass {
if g == nil || symbolID == "" {
return ZeroEdgePossibleExtractionGap
}
if g.GetNode(symbolID) == nil {
return ZeroEdgePossibleExtractionGap
}
in := g.GetInEdges(symbolID)
out := g.GetOutEdges(symbolID)
if len(in) == 0 && len(out) == 0 {
return ZeroEdgePossibleExtractionGap
}
for _, e := range in {
if usageEdgeKinds[e.Kind] {
return ZeroEdgeNone
}
}
if importConsumerCount(g, symbolID) > 0 {
return ZeroEdgeCoverageIncomplete
}
// The graph still carries unresolved call candidates that name this
// symbol — `unresolved::*.<name>` call edges the resolver / enrichment
// pass never bound to it. Their existence is direct evidence that call
// sites reference this name; an empty usage set is a resolution/coverage
// gap, not proof the symbol is unused.
if hasUnresolvedSameNameCandidates(g, symbolID) {
return ZeroEdgeCoverageIncomplete
}
return ZeroEdgeLikelyUnused
}
// hasUnresolvedSameNameCandidates reports whether the graph holds any
// unresolved member-call edge (`unresolved::*.<name>`) naming this callable
// symbol. Unresolved call stubs are indexed by their target string, so this is
// a single reverse-edge lookup — no scan. Non-callable symbols never match.
func hasUnresolvedSameNameCandidates(g Store, symbolID string) bool {
n := g.GetNode(symbolID)
if n == nil || n.Name == "" {
return false
}
if n.Kind != KindFunction && n.Kind != KindMethod {
return false
}
return len(g.GetInEdges(UnresolvedMarker+"*."+n.Name)) > 0
}
// importConsumerCount counts the import-level consumer evidence for a
// symbol: inbound `imports` / `re_exports` edges on the symbol itself
// (any language — a per-binding import that resolved onto the symbol is
// direct proof a consumer names it), plus, for a public JS/TS symbol,
// inbound module-level import edges on its file node (a module-level
// `import ... from './file'` lands on the file, but still proves the
// file's exports have consumers).
func importConsumerCount(g Store, symbolID string) int {
count := 0
for _, e := range g.GetInEdges(symbolID) {
if e.Kind == EdgeImports || e.Kind == EdgeReExports {
count++
}
}
if count > 0 {
return count
}
n := g.GetNode(symbolID)
if n == nil || n.FilePath == "" || n.FilePath == symbolID {
return 0
}
switch n.Language {
case "typescript", "tsx", "javascript", "jsx":
default:
return 0 // module-level file imports imply consumers only for JS/TS
}
if vis, _ := n.Meta["visibility"].(string); vis != "public" {
return 0
}
for _, e := range g.GetInEdges(n.FilePath) {
if e.Kind == EdgeImports || e.Kind == EdgeReExports {
count++
}
}
return count
}
// zeroEdgeMessages maps each classification to its human-readable
// caveat text.
var zeroEdgeMessages = map[ZeroEdgeClass]string{
ZeroEdgeLikelyUnused: "no incoming call or reference edges, but the symbol is " +
"indexed (it has structural or outgoing edges) — consistent with genuine " +
"unused code that is safe to remove.",
ZeroEdgePossibleExtractionGap: "the symbol has no graph edges of any kind. A " +
"normally indexed symbol always has at least a structural edge, so the " +
"extractor most likely did not process it — treat this empty result as " +
"unverified, not as proof the symbol is unused.",
ZeroEdgeCoverageIncomplete: "no resolved call or reference edges, but the graph " +
"still carries consumer evidence — import/re-export edges pointing at this " +
"symbol or its file, or unresolved same-name call candidates the resolver / " +
"enrichment pass never bound to it. Reference-level resolution is incomplete, " +
"so treat this empty result as UNVERIFIED coverage, not as proof the symbol is " +
"unused or safe to remove.",
}
// zeroEdgeNotFoundMessage is the caveat text when the queried id is not in
// the graph at all — almost always a mistyped id or one missing its repo
// prefix, rather than a true extraction gap.
const zeroEdgeNotFoundMessage = "no symbol with this id is in the graph — the id is " +
"probably mistyped or missing its repo prefix (ids look like " +
"<repo>/<path>::<symbol>, e.g. gortex/internal/x.go::Foo). Run a symbol search to " +
"get the exact id; treat this empty result as unverified, not as proof of no usages."
// CaveatForZeroEdge builds the structured caveat for an empty graph
// query result on symbolID. It returns nil when the symbol has
// incoming usage edges (ZeroEdgeNone) — a non-empty result carries no
// caveat — so callers can attach the return value unconditionally.
func CaveatForZeroEdge(g Store, symbolID string) *ZeroEdgeCaveat {
// A target that is not even in the graph is the most common cause of a
// "0 usages" surprise — usually a mistyped id or one missing its repo
// prefix. Keep the untrustworthy extraction-gap class so safety gates
// still trip, but point the message at the id rather than the extractor.
if g != nil && symbolID != "" && g.GetNode(symbolID) == nil {
return &ZeroEdgeCaveat{Class: ZeroEdgePossibleExtractionGap, Message: zeroEdgeNotFoundMessage}
}
class := ClassifyZeroEdge(g, symbolID)
if class == ZeroEdgeNone {
return nil
}
return &ZeroEdgeCaveat{Class: class, Message: zeroEdgeMessages[class]}
}
@@ -0,0 +1,32 @@
package graph
import (
"strings"
"testing"
)
// TestCaveatForZeroEdge_NotFoundMessage asserts a queried id that is not in
// the graph (e.g. mistyped or missing its repo prefix) gets a caveat that
// points at the id — while still carrying the untrustworthy class so a
// safety gate trips — and that an existing zero-edge symbol keeps the
// extraction-gap message.
func TestCaveatForZeroEdge_NotFoundMessage(t *testing.T) {
g := New()
g.AddNode(&Node{ID: "repo/a.go::Lonely", Kind: KindFunction, Name: "Lonely"}) // exists, no edges
notFound := CaveatForZeroEdge(g, "a.go::Missing")
if notFound == nil || notFound.Class != ZeroEdgePossibleExtractionGap {
t.Fatalf("a not-found id must still carry the untrustworthy class; got %+v", notFound)
}
if !strings.Contains(notFound.Message, "repo prefix") {
t.Errorf("not-found message should point at the id/prefix; got %q", notFound.Message)
}
existing := CaveatForZeroEdge(g, "repo/a.go::Lonely")
if existing == nil {
t.Fatal("an existing zero-edge symbol should still carry a caveat")
}
if strings.Contains(existing.Message, "repo prefix") {
t.Errorf("an existing zero-edge symbol should get the extraction-gap message, not the not-found one; got %q", existing.Message)
}
}
+121
View File
@@ -0,0 +1,121 @@
package graph
import (
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
// buildClassifyGraph wires three functions in one file, each with a
// distinct edge profile, so ClassifyZeroEdge can be exercised against
// every classification:
//
// - Used — the file defines it AND a caller calls it.
// - Unused — the file defines it, it makes an outgoing call, but
// nothing calls it back. Indexed, dead.
// - Orphaned — added with no edges at all, mimicking a symbol the
// extractor never processed.
func buildClassifyGraph() *Graph {
g := New()
g.AddNode(&Node{ID: "svc.go", Kind: KindFile, Name: "svc.go", FilePath: "svc.go"})
g.AddNode(&Node{ID: "svc.go::Caller", Kind: KindFunction, Name: "Caller", FilePath: "svc.go"})
g.AddNode(&Node{ID: "svc.go::Used", Kind: KindFunction, Name: "Used", FilePath: "svc.go"})
g.AddNode(&Node{ID: "svc.go::Unused", Kind: KindFunction, Name: "Unused", FilePath: "svc.go"})
g.AddNode(&Node{ID: "svc.go::Helper", Kind: KindFunction, Name: "Helper", FilePath: "svc.go"})
// The file defines every function except the orphan.
g.AddEdge(&Edge{From: "svc.go", To: "svc.go::Caller", Kind: EdgeDefines})
g.AddEdge(&Edge{From: "svc.go", To: "svc.go::Used", Kind: EdgeDefines})
g.AddEdge(&Edge{From: "svc.go", To: "svc.go::Unused", Kind: EdgeDefines})
g.AddEdge(&Edge{From: "svc.go", To: "svc.go::Helper", Kind: EdgeDefines})
// Used has an incoming call edge.
g.AddEdge(&Edge{From: "svc.go::Caller", To: "svc.go::Used", Kind: EdgeCalls})
// Unused has only an outgoing call — no incoming usage edge.
g.AddEdge(&Edge{From: "svc.go::Unused", To: "svc.go::Helper", Kind: EdgeCalls})
return g
}
func TestClassifyZeroEdge(t *testing.T) {
g := buildClassifyGraph()
// A method node with only its member_of structural edge — indexed,
// but nothing references it.
g.AddNode(&Node{ID: "svc.go::T", Kind: KindType, Name: "T", FilePath: "svc.go"})
g.AddNode(&Node{ID: "svc.go::T.Method", Kind: KindMethod, Name: "Method", FilePath: "svc.go"})
g.AddEdge(&Edge{From: "svc.go::T.Method", To: "svc.go::T", Kind: EdgeMemberOf})
// An orphan: in the graph, but carrying no edges of any kind.
g.AddNode(&Node{ID: "svc.go::Orphan", Kind: KindFunction, Name: "Orphan", FilePath: "svc.go"})
tests := []struct {
name string
symbolID string
want ZeroEdgeClass
}{
{
name: "incoming call edge yields none",
symbolID: "svc.go::Used",
want: ZeroEdgeNone,
},
{
name: "only structural defines plus outgoing call yields likely_unused",
symbolID: "svc.go::Unused",
want: ZeroEdgeLikelyUnused,
},
{
name: "method with only member_of yields likely_unused",
symbolID: "svc.go::T.Method",
want: ZeroEdgeLikelyUnused,
},
{
name: "zero edges of any kind yields possible_extraction_gap",
symbolID: "svc.go::Orphan",
want: ZeroEdgePossibleExtractionGap,
},
{
name: "unknown symbol id yields possible_extraction_gap",
symbolID: "svc.go::DoesNotExist",
want: ZeroEdgePossibleExtractionGap,
},
{
name: "empty symbol id yields possible_extraction_gap",
symbolID: "",
want: ZeroEdgePossibleExtractionGap,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got := ClassifyZeroEdge(g, tt.symbolID)
assert.Equal(t, tt.want, got)
})
}
}
func TestClassifyZeroEdge_NilGraph(t *testing.T) {
assert.Equal(t, ZeroEdgePossibleExtractionGap, ClassifyZeroEdge(nil, "svc.go::Used"))
}
func TestCaveatForZeroEdge(t *testing.T) {
g := buildClassifyGraph()
g.AddNode(&Node{ID: "svc.go::Orphan", Kind: KindFunction, Name: "Orphan", FilePath: "svc.go"})
// A used symbol carries no caveat — nil is returned so callers can
// attach it unconditionally.
assert.Nil(t, CaveatForZeroEdge(g, "svc.go::Used"))
// Likely-unused: classification plus a non-empty message.
unused := CaveatForZeroEdge(g, "svc.go::Unused")
require.NotNil(t, unused)
assert.Equal(t, ZeroEdgeLikelyUnused, unused.Class)
assert.NotEmpty(t, unused.Message)
// Extraction gap: classification plus a non-empty message.
gap := CaveatForZeroEdge(g, "svc.go::Orphan")
require.NotNil(t, gap)
assert.Equal(t, ZeroEdgePossibleExtractionGap, gap.Class)
assert.NotEmpty(t, gap.Message)
// The two caveat messages must differ — they describe opposite
// situations and an agent has to be able to tell them apart.
assert.NotEqual(t, unused.Message, gap.Message)
}
@@ -0,0 +1,36 @@
package graph
import (
"testing"
"github.com/stretchr/testify/assert"
)
// A callable with no resolved incoming usages, but whose name still has
// unresolved same-name call candidates in the graph, must be classified as
// coverage-incomplete — not likely_unused. The unresolved stubs are direct
// evidence that call sites reference the name; the empty usage set is a
// resolution gap, not proof the symbol is dead.
func TestClassifyZeroEdge_UnresolvedSameNameCandidates(t *testing.T) {
g := New()
g.AddNode(&Node{ID: "Crash.java", Kind: KindFile, Name: "Crash.java", FilePath: "Crash.java", Language: "java"})
g.AddNode(&Node{ID: "Crash.java::CrashController.triggerException", Kind: KindMethod, Name: "triggerException", FilePath: "Crash.java", Language: "java"})
g.AddNode(&Node{ID: "Test.java", Kind: KindFile, Name: "Test.java", FilePath: "Test.java", Language: "java"})
g.AddNode(&Node{ID: "Test.java::T.run", Kind: KindMethod, Name: "run", FilePath: "Test.java", Language: "java"})
// The method is indexed (defined) but has no resolved incoming usage.
g.AddEdge(&Edge{From: "Crash.java", To: "Crash.java::CrashController.triggerException", Kind: EdgeDefines})
// An unresolved same-name call candidate the resolver never bound.
g.AddEdge(&Edge{From: "Test.java::T.run", To: "unresolved::*.triggerException", Kind: EdgeCalls, FilePath: "Test.java", Line: 37})
got := ClassifyZeroEdge(g, "Crash.java::CrashController.triggerException")
assert.Equal(t, ZeroEdgeCoverageIncomplete, got,
"a symbol whose name still has unresolved call candidates must not be reported likely_unused")
// A genuinely unused method (no unresolved candidates naming it) still reads
// as likely_unused.
g.AddNode(&Node{ID: "Crash.java::CrashController.reallyDead", Kind: KindMethod, Name: "reallyDead", FilePath: "Crash.java", Language: "java"})
g.AddEdge(&Edge{From: "Crash.java", To: "Crash.java::CrashController.reallyDead", Kind: EdgeDefines})
assert.Equal(t, ZeroEdgeLikelyUnused, ClassifyZeroEdge(g, "Crash.java::CrashController.reallyDead"),
"a method with no unresolved candidates naming it stays likely_unused")
}
+62
View File
@@ -0,0 +1,62 @@
package graph
import "strings"
// IsProxyNode reports whether n is a cross-daemon proxy-edge node —
// identified by its struct fields, NOT its ID shape. Distinct from
// IsStub(id) (the stdlib/builtin/module string predicate): a proxy node
// is keyed under the "remote:<slug>~..." origin namespace, which IsStub
// does not recognise. Proxy nodes are excluded from graph_stats / BM25 /
// communities / analyzers and never persisted to the durable store.
func IsProxyNode(n *Node) bool {
return n != nil && n.Stub && n.Origin != ""
}
// proxyIDPrefix is the origin-namespace marker for a proxy node id.
const proxyIDPrefix = "remote:"
// proxyIDSep separates the origin segment from the remote's native id.
const proxyIDSep = "~"
// ProxyNodeID composes the origin-namespaced id for a remote symbol so a
// remote node can never alias a local id, even when two daemons share a
// repo prefix: "remote:<slug>~<remoteRepoPrefix>/<file>::<sym>".
func ProxyNodeID(slug, remoteID string) string {
return proxyIDPrefix + slug + proxyIDSep + remoteID
}
// IsProxyID reports whether id is an origin-namespaced proxy id.
func IsProxyID(id string) bool {
if !strings.HasPrefix(id, proxyIDPrefix) {
return false
}
return strings.Contains(id[len(proxyIDPrefix):], proxyIDSep)
}
// ProxyOriginSlug returns the <slug> of a proxy id, or "" if id is not a
// proxy id.
func ProxyOriginSlug(id string) string {
if !strings.HasPrefix(id, proxyIDPrefix) {
return ""
}
rest := id[len(proxyIDPrefix):]
i := strings.Index(rest, proxyIDSep)
if i < 0 {
return ""
}
return rest[:i]
}
// ProxyRemoteID returns the remote's native id encoded in a proxy id, or
// "" if id is not a proxy id.
func ProxyRemoteID(id string) string {
if !strings.HasPrefix(id, proxyIDPrefix) {
return ""
}
rest := id[len(proxyIDPrefix):]
i := strings.Index(rest, proxyIDSep)
if i < 0 {
return ""
}
return rest[i+len(proxyIDSep):]
}
+68
View File
@@ -0,0 +1,68 @@
package graph
import (
"testing"
"time"
)
// TestProxyNodeID_RoundTrip asserts the origin-namespaced id composes and
// decomposes, and that a remote whose repo prefix collides with a local
// prefix still yields a distinct (non-aliasing) proxy id.
func TestProxyNodeID_RoundTrip(t *testing.T) {
remoteID := "gortex/internal/daemon/router.go::Router"
pid := ProxyNodeID("r2", remoteID)
if !IsProxyID(pid) {
t.Fatalf("%q should be a proxy id", pid)
}
if got := ProxyOriginSlug(pid); got != "r2" {
t.Errorf("origin slug = %q, want r2", got)
}
if got := ProxyRemoteID(pid); got != remoteID {
t.Errorf("remote id = %q, want %q", got, remoteID)
}
// A local node with the same native id is never a proxy id, so no
// alias is possible even on a shared prefix.
if IsProxyID(remoteID) {
t.Error("a local native id must not be mistaken for a proxy id")
}
if ProxyNodeID("r3", remoteID) == pid {
t.Error("the same remote symbol under two slugs must yield distinct proxy ids")
}
}
// TestIsProxyNode asserts proxy detection keys on the struct fields, not
// the id shape, and never confuses a stdlib stub.
func TestIsProxyNode(t *testing.T) {
proxy := &Node{ID: ProxyNodeID("r2", "x/y.go::Z"), Origin: "remote:r2", Stub: true, FetchedAt: time.Now()}
if !IsProxyNode(proxy) {
t.Error("a Stub+Origin node is a proxy node")
}
local := &Node{ID: "x/y.go::Z"}
if IsProxyNode(local) {
t.Error("a local node is not a proxy node")
}
// Stub without Origin (some other stub convention) is not a proxy.
if IsProxyNode(&Node{Stub: true}) {
t.Error("Stub alone is not a proxy node")
}
if IsProxyNode(nil) {
t.Error("nil is not a proxy node")
}
// IsStub (the string predicate) must not match a proxy id.
if IsStub(proxy.ID) {
t.Error("IsStub must not recognise a proxy id")
}
}
// TestProxyHelpers_NonProxyInputs asserts the accessors are safe on
// non-proxy ids.
func TestProxyHelpers_NonProxyInputs(t *testing.T) {
for _, id := range []string{"", "x/y.go::Z", "remote:nosep", "stdlib::fmt"} {
if IsProxyID(id) {
t.Errorf("%q should not be a proxy id", id)
}
if ProxyOriginSlug(id) != "" || ProxyRemoteID(id) != "" {
t.Errorf("accessors should be empty for non-proxy id %q", id)
}
}
}
File diff suppressed because it is too large Load Diff
+531
View File
@@ -0,0 +1,531 @@
package graph
import (
"sync"
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"pgregory.net/rapid"
)
func makeNode(id, name string, kind NodeKind, file, lang string) *Node {
return &Node{
ID: id,
Kind: kind,
Name: name,
QualName: "pkg." + name,
FilePath: file,
StartLine: 1,
EndLine: 10,
Language: lang,
}
}
func TestAddAndGetNode(t *testing.T) {
g := New()
n := makeNode("a.go::Foo", "Foo", KindFunction, "a.go", "go")
g.AddNode(n)
assert.Equal(t, n, g.GetNode("a.go::Foo"))
assert.Equal(t, n, g.GetNodeByQualName("pkg.Foo"))
assert.Equal(t, []*Node{n}, g.FindNodesByName("Foo"))
assert.Equal(t, []*Node{n}, g.GetFileNodes("a.go"))
assert.Nil(t, g.GetNode("nonexistent"))
}
func TestAddAndGetEdge(t *testing.T) {
g := New()
n1 := makeNode("a.go::Foo", "Foo", KindFunction, "a.go", "go")
n2 := makeNode("b.go::Bar", "Bar", KindFunction, "b.go", "go")
g.AddNode(n1)
g.AddNode(n2)
e := &Edge{From: n1.ID, To: n2.ID, Kind: EdgeCalls, FilePath: "a.go", Line: 5}
g.AddEdge(e)
out := g.GetOutEdges(n1.ID)
require.Len(t, out, 1)
assert.Equal(t, EdgeCalls, out[0].Kind)
in := g.GetInEdges(n2.ID)
require.Len(t, in, 1)
assert.Equal(t, n1.ID, in[0].From)
}
func TestEvictFile(t *testing.T) {
g := New()
n1 := makeNode("a.go::Foo", "Foo", KindFunction, "a.go", "go")
n2 := makeNode("a.go::Bar", "Bar", KindFunction, "a.go", "go")
n3 := makeNode("b.go::Baz", "Baz", KindFunction, "b.go", "go")
g.AddNode(n1)
g.AddNode(n2)
g.AddNode(n3)
g.AddEdge(&Edge{From: n1.ID, To: n3.ID, Kind: EdgeCalls, FilePath: "a.go", Line: 1})
g.AddEdge(&Edge{From: n3.ID, To: n2.ID, Kind: EdgeCalls, FilePath: "b.go", Line: 2})
nodesRm, edgesRm := g.EvictFile("a.go")
assert.Equal(t, 2, nodesRm)
assert.Equal(t, 2, edgesRm) // both edges reference evicted node IDs
assert.Nil(t, g.GetNode("a.go::Foo"))
assert.Nil(t, g.GetNode("a.go::Bar"))
assert.NotNil(t, g.GetNode("b.go::Baz"))
// Edge from b.go to a.go::Bar should also be cleaned from inEdges.
assert.Empty(t, g.GetOutEdges("a.go::Foo"))
}
func TestEvictFile_NoNodes(t *testing.T) {
g := New()
n, e := g.EvictFile("nonexistent.go")
assert.Equal(t, 0, n)
assert.Equal(t, 0, e)
}
func TestNodeAndEdgeCount(t *testing.T) {
g := New()
g.AddNode(makeNode("a.go::A", "A", KindFunction, "a.go", "go"))
g.AddNode(makeNode("b.go::B", "B", KindType, "b.go", "go"))
g.AddEdge(&Edge{From: "a.go::A", To: "b.go::B", Kind: EdgeReferences, FilePath: "a.go", Line: 1})
assert.Equal(t, 2, g.NodeCount())
assert.Equal(t, 1, g.EdgeCount())
}
func TestStats(t *testing.T) {
g := New()
g.AddNode(makeNode("a.go::A", "A", KindFunction, "a.go", "go"))
g.AddNode(makeNode("b.go::B", "B", KindType, "b.go", "go"))
g.AddNode(makeNode("c.ts::C", "C", KindFunction, "c.ts", "typescript"))
g.AddEdge(&Edge{From: "a.go::A", To: "b.go::B", Kind: EdgeReferences, FilePath: "a.go", Line: 1})
s := g.Stats()
assert.Equal(t, 3, s.TotalNodes)
assert.Equal(t, 1, s.TotalEdges)
assert.Equal(t, 2, s.ByKind["function"])
assert.Equal(t, 1, s.ByKind["type"])
assert.Equal(t, 2, s.ByLanguage["go"])
assert.Equal(t, 1, s.ByLanguage["typescript"])
}
func TestAllNodesAndEdges(t *testing.T) {
g := New()
g.AddNode(makeNode("a.go::A", "A", KindFunction, "a.go", "go"))
g.AddNode(makeNode("b.go::B", "B", KindFunction, "b.go", "go"))
g.AddEdge(&Edge{From: "a.go::A", To: "b.go::B", Kind: EdgeCalls, FilePath: "a.go", Line: 1})
assert.Len(t, g.AllNodes(), 2)
assert.Len(t, g.AllEdges(), 1)
}
func TestConcurrency(t *testing.T) {
g := New()
var wg sync.WaitGroup
// Concurrent writers.
for i := range 50 {
wg.Add(1)
go func(i int) {
defer wg.Done()
id := "file.go::" + string(rune('A'+i))
n := makeNode(id, string(rune('A'+i)), KindFunction, "file.go", "go")
n.QualName = "" // avoid collision
g.AddNode(n)
}(i)
}
// Concurrent readers.
for range 50 {
wg.Add(1)
go func() {
defer wg.Done()
_ = g.NodeCount()
_ = g.GetFileNodes("file.go")
_ = g.Stats()
}()
}
wg.Wait()
}
func TestNodeBrief(t *testing.T) {
n := &Node{
ID: "a.go::Foo", Kind: KindFunction, Name: "Foo",
QualName: "pkg.Foo", FilePath: "a.go", StartLine: 10, EndLine: 20,
Language: "go", Meta: map[string]any{"signature": "func Foo()"},
}
b := n.Brief()
assert.Equal(t, "a.go::Foo", b["id"])
assert.Equal(t, "Foo", b["name"])
assert.Equal(t, NodeKind("function"), b["kind"])
assert.Equal(t, "a.go", b["file_path"])
assert.Equal(t, 10, b["start_line"])
// Should NOT contain meta, qual_name, end_line, language.
_, hasMeta := b["meta"]
assert.False(t, hasMeta)
}
func TestValidNodeKind(t *testing.T) {
assert.True(t, ValidNodeKind(KindFunction))
assert.True(t, ValidNodeKind(KindFile))
assert.False(t, ValidNodeKind(NodeKind("unknown")))
}
func makeRepoNode(id, name string, kind NodeKind, file, lang, repo string) *Node {
return &Node{
ID: id,
Kind: kind,
Name: name,
QualName: repo + "." + name,
FilePath: file,
StartLine: 1,
EndLine: 10,
Language: lang,
RepoPrefix: repo,
}
}
func TestAddNode_ByRepoIndex(t *testing.T) {
g := New()
n1 := makeRepoNode("repoA/a.go::Foo", "Foo", KindFunction, "repoA/a.go", "go", "repoA")
n2 := makeRepoNode("repoB/b.go::Bar", "Bar", KindFunction, "repoB/b.go", "go", "repoB")
n3 := makeRepoNode("repoA/c.go::Baz", "Baz", KindType, "repoA/c.go", "go", "repoA")
g.AddNode(n1)
g.AddNode(n2)
g.AddNode(n3)
repoANodes := g.GetRepoNodes("repoA")
assert.Len(t, repoANodes, 2)
repoBNodes := g.GetRepoNodes("repoB")
assert.Len(t, repoBNodes, 1)
assert.Equal(t, "Bar", repoBNodes[0].Name)
}
func TestAddNode_EmptyRepoPrefix(t *testing.T) {
g := New()
n := makeNode("a.go::Foo", "Foo", KindFunction, "a.go", "go")
g.AddNode(n)
// Nodes without RepoPrefix should not appear in byRepo.
assert.Empty(t, g.GetRepoNodes(""))
assert.Empty(t, g.RepoPrefixes())
}
func TestGetRepoNodes_ReturnsCopy(t *testing.T) {
g := New()
n := makeRepoNode("repoA/a.go::Foo", "Foo", KindFunction, "repoA/a.go", "go", "repoA")
g.AddNode(n)
nodes := g.GetRepoNodes("repoA")
nodes[0] = nil // mutate the returned slice
assert.NotNil(t, g.GetRepoNodes("repoA")[0], "GetRepoNodes should return a copy")
}
func TestGetRepoNodes_NotFound(t *testing.T) {
g := New()
assert.Empty(t, g.GetRepoNodes("nonexistent"))
}
func TestEvictRepo(t *testing.T) {
g := New()
nA1 := makeRepoNode("repoA/a.go::Foo", "Foo", KindFunction, "repoA/a.go", "go", "repoA")
nA2 := makeRepoNode("repoA/a.go::Bar", "Bar", KindFunction, "repoA/a.go", "go", "repoA")
nB1 := makeRepoNode("repoB/b.go::Baz", "Baz", KindFunction, "repoB/b.go", "go", "repoB")
g.AddNode(nA1)
g.AddNode(nA2)
g.AddNode(nB1)
// Edges: intra-repoA, cross-repo A→B, intra-repoB self-ref
g.AddEdge(&Edge{From: nA1.ID, To: nA2.ID, Kind: EdgeCalls, FilePath: "repoA/a.go", Line: 1})
g.AddEdge(&Edge{From: nA1.ID, To: nB1.ID, Kind: EdgeCalls, FilePath: "repoA/a.go", Line: 2, CrossRepo: true})
g.AddEdge(&Edge{From: nB1.ID, To: nA2.ID, Kind: EdgeCalls, FilePath: "repoB/b.go", Line: 3, CrossRepo: true})
nodesRm, edgesRm := g.EvictRepo("repoA")
assert.Equal(t, 2, nodesRm)
assert.Equal(t, 3, edgesRm) // all 3 edges reference repoA nodes
// repoA nodes gone.
assert.Nil(t, g.GetNode("repoA/a.go::Foo"))
assert.Nil(t, g.GetNode("repoA/a.go::Bar"))
assert.Empty(t, g.GetRepoNodes("repoA"))
// repoB node still present.
assert.NotNil(t, g.GetNode("repoB/b.go::Baz"))
assert.Len(t, g.GetRepoNodes("repoB"), 1)
// byName cleaned for evicted nodes.
assert.Empty(t, g.FindNodesByName("Foo"))
assert.Empty(t, g.FindNodesByName("Bar"))
assert.Len(t, g.FindNodesByName("Baz"), 1)
// byFile cleaned for evicted nodes.
assert.Empty(t, g.GetFileNodes("repoA/a.go"))
assert.Len(t, g.GetFileNodes("repoB/b.go"), 1)
}
func TestEvictRepo_NoNodes(t *testing.T) {
g := New()
n, e := g.EvictRepo("nonexistent")
assert.Equal(t, 0, n)
assert.Equal(t, 0, e)
}
func TestEvictRepo_QualNameCleaned(t *testing.T) {
g := New()
n := makeRepoNode("repoA/a.go::Foo", "Foo", KindFunction, "repoA/a.go", "go", "repoA")
g.AddNode(n)
assert.NotNil(t, g.GetNodeByQualName("repoA.Foo"))
g.EvictRepo("repoA")
assert.Nil(t, g.GetNodeByQualName("repoA.Foo"))
}
func TestRepoStats(t *testing.T) {
g := New()
g.AddNode(makeRepoNode("repoA/a.go::Foo", "Foo", KindFunction, "repoA/a.go", "go", "repoA"))
g.AddNode(makeRepoNode("repoA/b.go::Bar", "Bar", KindType, "repoA/b.go", "go", "repoA"))
g.AddNode(makeRepoNode("repoB/c.ts::Baz", "Baz", KindFunction, "repoB/c.ts", "typescript", "repoB"))
// Edge from repoA node.
g.AddEdge(&Edge{From: "repoA/a.go::Foo", To: "repoA/b.go::Bar", Kind: EdgeCalls, FilePath: "repoA/a.go", Line: 1})
// Cross-repo edge from repoA to repoB.
g.AddEdge(&Edge{From: "repoA/a.go::Foo", To: "repoB/c.ts::Baz", Kind: EdgeCalls, FilePath: "repoA/a.go", Line: 2, CrossRepo: true})
stats := g.RepoStats()
require.Len(t, stats, 2)
sA := stats["repoA"]
assert.Equal(t, 2, sA.TotalNodes)
assert.Equal(t, 2, sA.TotalEdges) // both edges originate from repoA
assert.Equal(t, 1, sA.ByKind["function"])
assert.Equal(t, 1, sA.ByKind["type"])
assert.Equal(t, 2, sA.ByLanguage["go"])
sB := stats["repoB"]
assert.Equal(t, 1, sB.TotalNodes)
assert.Equal(t, 0, sB.TotalEdges) // no edges originate from repoB
assert.Equal(t, 1, sB.ByKind["function"])
assert.Equal(t, 1, sB.ByLanguage["typescript"])
}
func TestRepoStats_Empty(t *testing.T) {
g := New()
stats := g.RepoStats()
assert.Empty(t, stats)
}
func TestRepoPrefixes(t *testing.T) {
g := New()
g.AddNode(makeRepoNode("repoA/a.go::Foo", "Foo", KindFunction, "repoA/a.go", "go", "repoA"))
g.AddNode(makeRepoNode("repoB/b.go::Bar", "Bar", KindFunction, "repoB/b.go", "go", "repoB"))
g.AddNode(makeRepoNode("repoA/c.go::Baz", "Baz", KindFunction, "repoA/c.go", "go", "repoA"))
prefixes := g.RepoPrefixes()
assert.Len(t, prefixes, 2)
assert.ElementsMatch(t, []string{"repoA", "repoB"}, prefixes)
}
func TestRepoPrefixes_Empty(t *testing.T) {
g := New()
assert.Empty(t, g.RepoPrefixes())
}
func TestEvictFile_CleansByRepoIndex(t *testing.T) {
g := New()
n1 := makeRepoNode("repoA/a.go::Foo", "Foo", KindFunction, "repoA/a.go", "go", "repoA")
n2 := makeRepoNode("repoA/b.go::Bar", "Bar", KindFunction, "repoA/b.go", "go", "repoA")
g.AddNode(n1)
g.AddNode(n2)
assert.Len(t, g.GetRepoNodes("repoA"), 2)
g.EvictFile("repoA/a.go")
// Only n2 should remain in byRepo.
repoNodes := g.GetRepoNodes("repoA")
require.Len(t, repoNodes, 1)
assert.Equal(t, "Bar", repoNodes[0].Name)
}
func TestEvictFile_CleansByRepoIndex_LastNode(t *testing.T) {
g := New()
n := makeRepoNode("repoA/a.go::Foo", "Foo", KindFunction, "repoA/a.go", "go", "repoA")
g.AddNode(n)
g.EvictFile("repoA/a.go")
assert.Empty(t, g.GetRepoNodes("repoA"))
assert.Empty(t, g.RepoPrefixes())
}
// Feature: multi-repo-support, Property 7: Per-repo index correctness
// genRepoPrefix generates a short repo prefix like "repoA", "repoB", etc.
func genRepoPrefix() *rapid.Generator[string] {
return rapid.Custom(func(t *rapid.T) string {
return "repo" + rapid.StringMatching(`[A-Z][a-z]{0,4}`).Draw(t, "suffix")
})
}
// genMultiRepoGraph builds a graph with 2-5 repo prefixes and 1-10 nodes per repo,
// plus cross-repo edges. Returns the graph and a map of prefix → expected node IDs.
func genMultiRepoGraph(t *rapid.T) (*Graph, map[string][]string) {
g := New()
numRepos := rapid.IntRange(2, 5).Draw(t, "numRepos")
// Generate distinct prefixes.
prefixSet := make(map[string]bool)
var prefixes []string
for len(prefixes) < numRepos {
p := genRepoPrefix().Draw(t, "prefix")
if !prefixSet[p] {
prefixSet[p] = true
prefixes = append(prefixes, p)
}
}
expected := make(map[string][]string) // prefix → node IDs
var allNodeIDs []string
kinds := []NodeKind{KindFunction, KindType, KindMethod, KindVariable}
for _, prefix := range prefixes {
numNodes := rapid.IntRange(1, 10).Draw(t, "numNodes_"+prefix)
for j := 0; j < numNodes; j++ {
name := rapid.StringMatching(`[A-Z][a-zA-Z0-9]{1,10}`).Draw(t, "name")
file := prefix + "/" + rapid.StringMatching(`[a-z]{1,8}`).Draw(t, "file") + ".go"
id := file + "::" + name
kind := kinds[rapid.IntRange(0, len(kinds)-1).Draw(t, "kind")]
n := makeRepoNode(id, name, kind, file, "go", prefix)
g.AddNode(n)
expected[prefix] = append(expected[prefix], id)
allNodeIDs = append(allNodeIDs, id)
}
}
// Add some edges including cross-repo ones.
if len(allNodeIDs) >= 2 {
numEdges := rapid.IntRange(1, len(allNodeIDs)).Draw(t, "numEdges")
for i := 0; i < numEdges; i++ {
fromIdx := rapid.IntRange(0, len(allNodeIDs)-1).Draw(t, "fromIdx")
toIdx := rapid.IntRange(0, len(allNodeIDs)-1).Draw(t, "toIdx")
if fromIdx == toIdx {
continue
}
fromNode := g.GetNode(allNodeIDs[fromIdx])
toNode := g.GetNode(allNodeIDs[toIdx])
crossRepo := fromNode.RepoPrefix != toNode.RepoPrefix
g.AddEdge(&Edge{
From: allNodeIDs[fromIdx],
To: allNodeIDs[toIdx],
Kind: EdgeCalls,
FilePath: fromNode.FilePath,
Line: i + 1,
CrossRepo: crossRepo,
})
}
}
return g, expected
}
// TestPropertyPerRepoIndexCorrectness verifies that GetRepoNodes returns exactly
// the nodes with matching RepoPrefix, and the union of all GetRepoNodes equals
// all nodes that have a RepoPrefix.
func TestPropertyPerRepoIndexCorrectness(t *testing.T) {
rapid.Check(t, func(rt *rapid.T) {
g, expected := genMultiRepoGraph(rt)
// For each prefix, GetRepoNodes must return exactly the matching nodes.
for prefix, wantIDs := range expected {
got := g.GetRepoNodes(prefix)
gotIDs := make([]string, len(got))
for i, n := range got {
gotIDs[i] = n.ID
// Every returned node must have the correct RepoPrefix.
assert.Equal(rt, prefix, n.RepoPrefix,
"GetRepoNodes(%q) returned node %q with wrong RepoPrefix %q", prefix, n.ID, n.RepoPrefix)
}
assert.ElementsMatch(rt, wantIDs, gotIDs,
"GetRepoNodes(%q) returned wrong set of node IDs", prefix)
}
// Union of all GetRepoNodes for all prefixes must equal all nodes with a RepoPrefix.
prefixes := g.RepoPrefixes()
unionIDs := make(map[string]bool)
for _, p := range prefixes {
for _, n := range g.GetRepoNodes(p) {
unionIDs[n.ID] = true
}
}
allNodes := g.AllNodes()
repoNodeIDs := make(map[string]bool)
for _, n := range allNodes {
if n.RepoPrefix != "" {
repoNodeIDs[n.ID] = true
}
}
assert.Equal(rt, repoNodeIDs, unionIDs,
"Union of GetRepoNodes for all prefixes must equal all nodes with a RepoPrefix")
})
}
// Feature: multi-repo-support, Property 8: Repo eviction completeness
// TestPropertyRepoEvictionCompleteness verifies that after EvictRepo, zero nodes/edges
// remain for the evicted prefix, and other repos are unchanged.
func TestPropertyRepoEvictionCompleteness(t *testing.T) {
rapid.Check(t, func(rt *rapid.T) {
g, expected := genMultiRepoGraph(rt)
// Pick a repo to evict.
prefixes := g.RepoPrefixes()
require.NotEmpty(rt, prefixes, "graph must have at least one repo prefix")
evictPrefix := prefixes[rapid.IntRange(0, len(prefixes)-1).Draw(rt, "evictIdx")]
// Record pre-eviction state for other repos.
otherCounts := make(map[string]int)
for _, p := range prefixes {
if p != evictPrefix {
otherCounts[p] = len(g.GetRepoNodes(p))
}
}
// Evict.
nodesRm, _ := g.EvictRepo(evictPrefix)
assert.Equal(rt, len(expected[evictPrefix]), nodesRm,
"EvictRepo should remove exactly the expected number of nodes")
// Verify GetRepoNodes returns empty for evicted prefix.
assert.Empty(rt, g.GetRepoNodes(evictPrefix),
"GetRepoNodes(%q) must be empty after eviction", evictPrefix)
// Verify no nodes in AllNodes have the evicted prefix.
for _, n := range g.AllNodes() {
assert.NotEqual(rt, evictPrefix, n.RepoPrefix,
"AllNodes() still contains node %q with evicted prefix %q", n.ID, evictPrefix)
}
// Verify no edges reference evicted node IDs.
evictedIDs := make(map[string]bool)
for _, id := range expected[evictPrefix] {
evictedIDs[id] = true
}
for _, e := range g.AllEdges() {
assert.False(rt, evictedIDs[e.From],
"Edge from %q → %q still references evicted node", e.From, e.To)
assert.False(rt, evictedIDs[e.To],
"Edge from %q → %q still references evicted node", e.From, e.To)
}
// Verify other repos' node counts are unchanged.
for p, wantCount := range otherCounts {
gotCount := len(g.GetRepoNodes(p))
assert.Equal(rt, wantCount, gotCount,
"Repo %q node count changed after evicting %q", p, evictPrefix)
}
})
}
+661
View File
@@ -0,0 +1,661 @@
package graph
import (
"fmt"
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
// TestAddNode_Idempotent proves the invariant the resilience work added
// to the graph: N duplicate AddNode calls converge to the same Stats()
// and the same secondary-index contents as a single call. Without this,
// a daemon restart that loads a snapshot and then re-runs IndexCtx on
// top of it (which doesn't evict first) produces N× the byFile /
// byName / byRepo slice entries — the B1 symptom.
func TestAddNode_Idempotent(t *testing.T) {
g := New()
n := &Node{
ID: "repo/a.go::Foo",
Name: "Foo",
Kind: KindFunction,
FilePath: "repo/a.go",
QualName: "pkg.Foo",
RepoPrefix: "repo",
}
g.AddNode(n)
base := g.Stats()
require.Equal(t, 1, base.TotalNodes)
for i := 0; i < 10; i++ {
g.AddNode(n)
}
got := g.Stats()
assert.Equal(t, base.TotalNodes, got.TotalNodes,
"duplicate AddNode must not grow node count")
byFile := g.GetFileNodes("repo/a.go")
assert.Len(t, byFile, 1, "byFile must not duplicate")
byName := g.FindNodesByName("Foo")
assert.Len(t, byName, 1, "byName must not duplicate")
byRepo := g.GetRepoNodes("repo")
assert.Len(t, byRepo, 1, "byRepo must not duplicate")
assert.Equal(t, n, g.GetNodeByQualName("pkg.Foo"))
}
// TestAddEdge_Idempotent is the edge counterpart of the node test. With
// the same (From, To, Kind, FilePath, Line), repeated AddEdge calls
// converge to a single adjacency-list entry. This is what made the
// "edges double on every daemon restart" symptom recede.
func TestAddEdge_Idempotent(t *testing.T) {
g := New()
g.AddNode(&Node{ID: "a::A", Name: "A", Kind: KindFunction, FilePath: "a"})
g.AddNode(&Node{ID: "b::B", Name: "B", Kind: KindFunction, FilePath: "b"})
e := &Edge{From: "b::B", To: "a::A", Kind: EdgeCalls, FilePath: "b", Line: 7}
for i := 0; i < 10; i++ {
g.AddEdge(e)
}
assert.Equal(t, 1, g.EdgeCount(), "duplicate AddEdge must not grow edge count")
assert.Len(t, g.GetOutEdges("b::B"), 1, "outEdges must have exactly one entry")
assert.Len(t, g.GetInEdges("a::A"), 1, "inEdges must have exactly one entry")
}
// TestAddEdge_DifferentFromSameTo guards the edgeKey shape: two edges
// with different From but identical (To, Kind, FilePath, Line) must
// both survive, as distinct entries in the target's inEdges bucket.
// An earlier version of the sidecar omitted From from the key, which
// made two such edges collide at the inEdges[to] index — the second
// AddEdge overwrote the first and downstream BFS traversal lost one
// caller. Cross-repo impact analysis regressed until From landed in
// the key.
func TestAddEdge_DifferentFromSameTo(t *testing.T) {
g := New()
g.AddNode(&Node{ID: "target::T", Name: "T", Kind: KindFunction, FilePath: "t"})
g.AddNode(&Node{ID: "caller1::C1", Name: "C1", Kind: KindFunction, FilePath: "c1"})
g.AddNode(&Node{ID: "caller2::C2", Name: "C2", Kind: KindFunction, FilePath: "c2"})
// Both edges lack FilePath/Line — a common shape in tests that
// construct synthetic graphs. Without From in the key they would
// dedup to one inEdges entry.
g.AddEdge(&Edge{From: "caller1::C1", To: "target::T", Kind: EdgeCalls})
g.AddEdge(&Edge{From: "caller2::C2", To: "target::T", Kind: EdgeCalls})
in := g.GetInEdges("target::T")
assert.Len(t, in, 2, "two distinct callers must both appear in inEdges")
}
// TestAddEdge_LineDisambiguates proves that two call-sites from the
// same caller to the same callee at different lines are preserved —
// they're distinct edges, not duplicates. `foo(); foo();` in the same
// function must survive dedup.
func TestAddEdge_LineDisambiguates(t *testing.T) {
g := New()
g.AddNode(&Node{ID: "a::A", Name: "A", Kind: KindFunction, FilePath: "a"})
g.AddNode(&Node{ID: "b::B", Name: "B", Kind: KindFunction, FilePath: "b"})
g.AddEdge(&Edge{From: "b::B", To: "a::A", Kind: EdgeCalls, FilePath: "b", Line: 7})
g.AddEdge(&Edge{From: "b::B", To: "a::A", Kind: EdgeCalls, FilePath: "b", Line: 11})
assert.Equal(t, 2, g.EdgeCount(), "different lines must produce distinct edges")
}
// TestAddNode_Replace verifies that re-adding a node with an updated
// Meta preserves the slice positions and replaces the pointer in place.
// This is the "same ID, new signature / new line" case that happens
// during IncrementalReindex after a file edit.
func TestAddNode_Replace(t *testing.T) {
g := New()
n1 := &Node{ID: "a::X", Name: "X", Kind: KindFunction, FilePath: "a",
Meta: map[string]any{"signature": "X()"}}
g.AddNode(n1)
n2 := &Node{ID: "a::X", Name: "X", Kind: KindFunction, FilePath: "a",
Meta: map[string]any{"signature": "X(arg int)"}}
g.AddNode(n2)
got := g.GetNode("a::X")
require.NotNil(t, got)
assert.Equal(t, "X(arg int)", got.Meta["signature"],
"replacement must install new pointer")
assert.Len(t, g.GetFileNodes("a"), 1, "byFile must not grow on replace")
assert.Len(t, g.FindNodesByName("X"), 1, "byName must not grow on replace")
// The slice entry must be the new pointer — readers iterate byFile
// and rely on it reflecting the current node state.
assert.Same(t, n2, g.GetFileNodes("a")[0])
}
// TestAddNode_MigrateBuckets verifies that when a replacement changes
// the node's FilePath / Name / RepoPrefix, the secondary-index entry
// moves from the old bucket to the new one. Without this, a rename
// (unusual but legal) would leave ghost entries in both buckets.
func TestAddNode_MigrateBuckets(t *testing.T) {
g := New()
g.AddNode(&Node{ID: "x::X", Name: "OldName", Kind: KindFunction,
FilePath: "old.go", RepoPrefix: "oldrepo", QualName: "pkg.Old"})
g.AddNode(&Node{ID: "x::X", Name: "NewName", Kind: KindFunction,
FilePath: "new.go", RepoPrefix: "newrepo", QualName: "pkg.New"})
assert.Empty(t, g.GetFileNodes("old.go"), "old bucket must be emptied")
assert.Len(t, g.GetFileNodes("new.go"), 1, "new bucket must have the entry")
assert.Empty(t, g.FindNodesByName("OldName"))
assert.Len(t, g.FindNodesByName("NewName"), 1)
assert.Empty(t, g.GetRepoNodes("oldrepo"))
assert.Len(t, g.GetRepoNodes("newrepo"), 1)
assert.Nil(t, g.GetNodeByQualName("pkg.Old"))
assert.NotNil(t, g.GetNodeByQualName("pkg.New"))
}
// TestAddNode_PreservesRepoPrefixOnEmptyDowngrade pins the warmup bug
// where some path re-AddNode'd existing repo-stamped nodes with
// RepoPrefix="" — clearing them out of byRepo[prefix] without touching
// the underlying nodes map. The user-visible symptom: per-repo queries
// (RepoStats / GetRepoNodes / RepoMemoryEstimate) returned empty for
// repos whose nodes were still present in the graph. Defensive fix:
// a non-empty prev RepoPrefix is sticky — the empty new value is
// promoted to prev's value rather than allowed to silently strip the
// node from its bucket.
func TestAddNode_PreservesRepoPrefixOnEmptyDowngrade(t *testing.T) {
g := New()
original := &Node{
ID: "myrepo/file.go::Foo", Name: "Foo", Kind: KindFunction,
FilePath: "myrepo/file.go", RepoPrefix: "myrepo",
}
g.AddNode(original)
require.Len(t, g.GetRepoNodes("myrepo"), 1, "node must land in byRepo at first add")
// Re-add with empty RepoPrefix (the buggy caller).
g.AddNode(&Node{
ID: "myrepo/file.go::Foo", Name: "Foo", Kind: KindFunction,
FilePath: "myrepo/file.go",
// RepoPrefix intentionally empty.
})
assert.Len(t, g.GetRepoNodes("myrepo"), 1,
"byRepo[myrepo] must still contain the node after empty-prefix re-add")
assert.NotNil(t, g.GetNode("myrepo/file.go::Foo"),
"node itself must still exist")
assert.Equal(t, "myrepo", g.GetNode("myrepo/file.go::Foo").RepoPrefix,
"RepoPrefix on the stored node must be preserved")
}
// TestEvictFile_SwapWithLast exercises the sidecar-based swap-with-last
// removal path. Uses enough nodes per file that iteration order would
// surface a mis-tracked sidecar position. The assertion is simple: post
// eviction, the graph is empty of entries for that file.
func TestEvictFile_SwapWithLast(t *testing.T) {
g := New()
for i := 0; i < 100; i++ {
g.AddNode(&Node{
ID: fmt.Sprintf("f.go::Sym%d", i),
Name: fmt.Sprintf("Sym%d", i),
Kind: KindFunction,
FilePath: "f.go",
})
}
assert.Len(t, g.GetFileNodes("f.go"), 100)
n, _ := g.EvictFile("f.go")
assert.Equal(t, 100, n)
assert.Empty(t, g.GetFileNodes("f.go"))
assert.Equal(t, 0, g.NodeCount())
}
// TestRestartStability simulates the daemon-restart cycle: snapshot
// into a fresh graph (via AddNode/AddEdge replay, which is what
// loadSnapshot does), and verify Stats() matches the original. Repeat
// many times to catch any state that drifts across restarts.
//
// Before the sidecar landed, Stats().TotalEdges doubled on every cycle;
// after, the invariant holds for arbitrary N.
func TestRestartStability(t *testing.T) {
orig := buildRepresentativeGraph()
want := orig.Stats()
for cycle := 0; cycle < 5; cycle++ {
replay := New()
for _, n := range orig.AllNodes() {
replay.AddNode(n)
}
for _, e := range orig.AllEdges() {
replay.AddEdge(e)
}
// Simulate a second "IndexCtx on top" pass — this is what
// the old warmup did after loadSnapshot. Without idempotent
// writes, this pass doubles every edge.
for _, n := range orig.AllNodes() {
replay.AddNode(n)
}
for _, e := range orig.AllEdges() {
replay.AddEdge(e)
}
got := replay.Stats()
assert.Equal(t, want.TotalNodes, got.TotalNodes,
"cycle %d: node count drifted", cycle)
assert.Equal(t, want.TotalEdges, got.TotalEdges,
"cycle %d: edge count drifted (B1 regression)", cycle)
}
}
func buildRepresentativeGraph() *Graph {
g := New()
// Build a small call graph that stresses every secondary index:
// multiple files, multiple names, multiple repos, calls + imports.
files := []struct {
path, repo string
}{
{"r1/a.go", "r1"},
{"r1/b.go", "r1"},
{"r2/c.go", "r2"},
}
for _, f := range files {
for i := 0; i < 5; i++ {
g.AddNode(&Node{
ID: fmt.Sprintf("%s::Fn%d", f.path, i),
Name: fmt.Sprintf("Fn%d", i),
Kind: KindFunction,
FilePath: f.path,
RepoPrefix: f.repo,
})
}
}
// Add a few call edges between files.
g.AddEdge(&Edge{From: "r1/a.go::Fn0", To: "r1/b.go::Fn1", Kind: EdgeCalls, FilePath: "r1/a.go", Line: 10})
g.AddEdge(&Edge{From: "r1/a.go::Fn0", To: "r2/c.go::Fn2", Kind: EdgeCalls, FilePath: "r1/a.go", Line: 12})
g.AddEdge(&Edge{From: "r1/b.go::Fn3", To: "r2/c.go::Fn4", Kind: EdgeCalls, FilePath: "r1/b.go", Line: 5})
return g
}
// TestReindexEdge_UpdatesSidecar verifies ReindexEdge migrates the
// inEdges bucket + both sidecars when the resolver changes an edge's
// To field (unresolved::X → real::X). A bug here would show up as
// GetInEdges returning zero entries after resolve, or later AddEdge
// refusing to dedup because the key changed out from under the sidecar.
func TestReindexEdge_UpdatesSidecar(t *testing.T) {
g := New()
g.AddNode(&Node{ID: "a::A", Name: "A", Kind: KindFunction, FilePath: "a"})
g.AddNode(&Node{ID: "b::B", Name: "B", Kind: KindFunction, FilePath: "b"})
g.AddNode(&Node{ID: "unresolved::real", Name: "real", Kind: KindFunction, FilePath: "u"})
e := &Edge{From: "a::A", To: "unresolved::real", Kind: EdgeCalls, FilePath: "a", Line: 3}
g.AddEdge(e)
require.Len(t, g.GetInEdges("unresolved::real"), 1)
require.Len(t, g.GetInEdges("b::B"), 0)
// Resolver-style mutation.
oldTo := e.To
e.To = "b::B"
g.ReindexEdge(e, oldTo)
assert.Len(t, g.GetInEdges("unresolved::real"), 0,
"old target bucket must be emptied")
assert.Len(t, g.GetInEdges("b::B"), 1,
"new target bucket must hold the edge")
// Adding the same edge with its NEW identity must dedup via the
// updated sidecar — if ReindexEdge forgot to rewrite the
// outEdgeIdx key, this would append a duplicate.
g.AddEdge(e)
assert.Equal(t, 1, g.EdgeCount(), "AddEdge after ReindexEdge must still dedup")
}
// TestRemoveEdgeFromBucket_SwappedEdgeWithMutatedTo regresses a daemon
// crash:
//
// panic: runtime error: index out of range [N] with length N
// graph.addEdgeToBucket
// graph.(*Graph).ReindexEdge
// resolver.(*Resolver).ResolveAll
//
// The resolver's serial pass mutates `j.edge.To = j.newTo` BEFORE
// taking the shard lock. If the swap-with-last in
// removeEdgeFromBucket lands on an edge whose .To was mutated in the
// same flight (e.g. another job in the same bucket), recomputing
// keyOf(swapped) returns the NEW key while the sidecar still has an
// entry under the ORIGINAL key pointing past the shrunk slice. The
// next AddEdge that collides with the orphaned key panics.
//
// The fix stores each entry's insertion-time edgeKey in a parallel
// slice (outEdgeKeys / inEdgeKeys) so the sidecar update is
// independent of the live Edge struct.
func TestRemoveEdgeFromBucket_SwappedEdgeWithMutatedTo(t *testing.T) {
g := New()
g.AddNode(&Node{ID: "a::A", Name: "A", Kind: KindFunction, FilePath: "a"})
g.AddNode(&Node{ID: "b::B", Name: "B", Kind: KindFunction, FilePath: "b"})
g.AddNode(&Node{ID: "x::X", Name: "X", Kind: KindFunction, FilePath: "x"})
g.AddNode(&Node{ID: "y::Y", Name: "Y", Kind: KindFunction, FilePath: "y"})
// Two edges share an unresolved bucket. We'll mutate eSwapped's To
// out-of-band (mimicking the resolver's pre-lock mutation) before
// removing eHead, forcing eSwapped to be the swap-with-last
// element. With the bug, the sidecar update used keyOf(eSwapped)
// — a different key than the one eSwapped was indexed under —
// leaving a stale entry that pointed past the shrunk slice.
const target = "unresolved::shared"
eHead := &Edge{From: "a::A", To: target, Kind: EdgeCalls, FilePath: "a", Line: 1}
eSwapped := &Edge{From: "b::B", To: target, Kind: EdgeCalls, FilePath: "b", Line: 2}
g.AddEdge(eHead)
g.AddEdge(eSwapped)
require.Len(t, g.GetInEdges(target), 2)
// Out-of-band mutation: eSwapped.To changes BUT we don't yet
// ReindexEdge. This models the in-flight window in
// resolver.go's serial pass.
eSwapped.To = "x::X"
// Now remove eHead via ReindexEdge — this triggers the swap that
// previously corrupted the sidecar.
oldHead := target
eHead.To = "y::Y"
g.ReindexEdge(eHead, oldHead)
// With the bug, inEdgeIdx[target] still held an orphan entry under
// eSwapped's ORIGINAL key (To=target) at position 1 — past the
// now-shrunk slice (length 1, valid index only 0). Any subsequent
// AddEdge whose key collides with that stale entry would do
// `bucket[target][1] = newEdge` and panic with
// "index out of range [1] with length 1".
//
// Construct exactly that collision: a fresh edge sharing
// eSwapped's original (From, To, Kind, FilePath, Line) tuple,
// which is what the resolver does when it pre-stages a duplicate
// pending edge from another file at the same line.
collision := &Edge{From: "b::B", To: target, Kind: EdgeCalls, FilePath: "b", Line: 2}
require.NotPanics(t, func() {
g.AddEdge(collision)
}, "addEdgeToBucket must not panic on a stale sidecar position")
// eHead has been migrated to its new target.
assert.Len(t, g.GetInEdges("y::Y"), 1, "eHead's new target should hold one edge")
}
// TestReindexEdge_OutEdgeKeysStayConsistent regresses the daemon
// warmup panic:
//
// panic: runtime error: index out of range [61] with length 58
// graph.removeEdgeFromBucket
// graph.(*Graph).evictEdgesLocked
// graph.(*Graph).EvictFile
// indexer.(*Indexer).indexFile
// indexer.(*Indexer).IncrementalReindex
//
// The failure mode: ReindexEdge updates outEdgeIdx[oldKey→newKey] but
// previously did NOT update the parallel outEdgeKeys[pos] slice. A
// later swap-with-last removal in the same outEdges bucket reads
// outEdgeKeys[swappedPos] — finds the stale insertion-time key — and
// re-inserts THAT key into outEdgeIdx pointing at the swapped slot.
// outEdgeIdx then holds both the live newKey (still pointing at the
// original pre-swap position) AND a stale-key entry. The next op
// that walks back to the original pos finds the slice has shrunk
// past it and panics.
//
// The fix: ReindexEdge rewrites outEdgeKeys[pos] = newKey alongside
// the outEdgeIdx update so the parallel slice never holds stale keys.
func TestReindexEdge_OutEdgeKeysStayConsistent(t *testing.T) {
g := New()
g.AddNode(&Node{ID: "a::A", Name: "A", Kind: KindFunction, FilePath: "a"})
g.AddNode(&Node{ID: "t1", Name: "t1", Kind: KindFunction, FilePath: "t1"})
g.AddNode(&Node{ID: "t2", Name: "t2", Kind: KindFunction, FilePath: "t2"})
g.AddNode(&Node{ID: "t3", Name: "t3", Kind: KindFunction, FilePath: "t3"})
g.AddNode(&Node{ID: "t2-prime", Name: "t2'", Kind: KindFunction, FilePath: "t2p"})
g.AddNode(&Node{ID: "t2-prime-prime", Name: "t2''", Kind: KindFunction, FilePath: "t2pp"})
// Three edges share the same From, populating one outEdges bucket
// with three slots. Distinct lines so the keys differ.
e1 := &Edge{From: "a::A", To: "t1", Kind: EdgeCalls, FilePath: "a", Line: 1}
e2 := &Edge{From: "a::A", To: "t2", Kind: EdgeCalls, FilePath: "a", Line: 2}
e3 := &Edge{From: "a::A", To: "t3", Kind: EdgeCalls, FilePath: "a", Line: 3}
g.AddEdge(e1)
g.AddEdge(e2)
g.AddEdge(e3)
require.Len(t, g.GetOutEdges("a::A"), 3)
// ReindexEdge e2 — outEdgeKeys[1] would stay stale before the fix.
oldTo := e2.To
e2.To = "t2-prime"
g.ReindexEdge(e2, oldTo)
// Force a swap-with-last in the outEdges["a::A"] bucket by
// removing e1. With the bug, this propagates the stale key for
// slot 1 (e2's original key) into outEdgeIdx.
require.True(t, g.RemoveEdge(e1.From, e1.To, e1.Kind))
// ReindexEdge e2 a second time — drives outEdgeIdx into the
// inconsistent state where it holds both the new key and the
// stale key from the previous swap.
oldTo = e2.To
e2.To = "t2-prime-prime"
g.ReindexEdge(e2, oldTo)
// Removal that touches the bucket must NOT panic. With the bug,
// removing e3 via its resolved key triggered
// `slice[pos] = slice[last]` with pos past the shrunk slice.
require.NotPanics(t, func() {
g.RemoveEdge(e3.From, e3.To, e3.Kind)
}, "swap-with-last after repeated ReindexEdge must not panic")
// e2 still queryable at its final target — sanity check that the
// bucket bookkeeping survived intact.
out := g.GetOutEdges("a::A")
require.Len(t, out, 1)
assert.Equal(t, "t2-prime-prime", out[0].To)
}
// TestEvictFile_AfterReindex regresses the same panic via the actual
// eviction path the daemon hit (EvictFile → evictEdgesLocked) instead
// of going through the public RemoveEdge API. The fixture stages the
// exact corruption window the daemon panic describes:
//
// 1. A multi-edge outEdges bucket on a single From.
// 2. ReindexEdge against a non-last slot in that bucket — outEdgeKeys
// for that slot becomes stale (still holds the pre-mutation key).
// 3. A swap-with-last removal earlier in the bucket pulls the stale
// key into outEdgeIdx pointing at the swapped position.
// 4. The slice subsequently shrinks past that position.
// 5. EvictFile on the reindexed edge's NEW target then walks
// inEdges[that target], grabs the still-correct live key from
// inEdgeKeys, and calls removeEdgeFromBucket(outEdges, ...) on
// the From bucket. With the bug, outEdgeIdx still has the live
// key pointing past the now-shrunk slice → panic.
func TestEvictFile_AfterReindex(t *testing.T) {
g := New()
g.AddNode(&Node{ID: "src/a.go::A", Name: "A", Kind: KindFunction, FilePath: "src/a.go"})
g.AddNode(&Node{ID: "t1.go::T1", Name: "T1", Kind: KindFunction, FilePath: "t1.go"})
g.AddNode(&Node{ID: "t2.go::T2", Name: "T2", Kind: KindFunction, FilePath: "t2.go"})
g.AddNode(&Node{ID: "t3.go::T3", Name: "T3", Kind: KindFunction, FilePath: "t3.go"})
g.AddNode(&Node{ID: "t2p.go::T2P", Name: "T2P", Kind: KindFunction, FilePath: "t2p.go"})
// Three outgoing edges from A — slot 1 is the one we'll reindex.
e1 := &Edge{From: "src/a.go::A", To: "t1.go::T1", Kind: EdgeCalls, FilePath: "src/a.go", Line: 1}
e2 := &Edge{From: "src/a.go::A", To: "t2.go::T2", Kind: EdgeCalls, FilePath: "src/a.go", Line: 2}
e3 := &Edge{From: "src/a.go::A", To: "t3.go::T3", Kind: EdgeCalls, FilePath: "src/a.go", Line: 3}
g.AddEdge(e1)
g.AddEdge(e2)
g.AddEdge(e3)
// Step 1: reindex e2's To. Without the fix, outEdgeKeys[1] keeps
// the pre-mutation key while outEdgeIdx swaps to the new key.
old := e2.To
e2.To = "t2p.go::T2P"
g.ReindexEdge(e2, old)
// Step 2: evict T1 — its inEdges bucket holds e1; Phase 2 of
// evictEdgesLocked calls removeEdgeFromBucket(outEdges["src/a.go::A"], k_for_e1).
// Inside, swap-with-last picks slot 2's key (k_for_e3 — correct)
// because slot 2 is what the swap consumes. So no panic yet, but
// after the swap the bucket is shape [e3, e2] with outEdgeKeys
// = [k_for_e3, STALE_pre-reindex_e2_key].
require.NotPanics(t, func() { g.EvictFile("t1.go") })
// Step 3: evict T3 — its inEdges bucket now points at the
// swapped slot 0 (e3). removeEdgeFromBucket(outEdges, k_for_e3)
// runs, swap-with-last picks up outEdgeKeys[1] which is the
// STALE key. With the bug, that stale key gets re-inserted into
// outEdgeIdx at position 0 alongside the still-live e2 key
// (which now points at position 1, but the slice has shrunk to
// length 1).
require.NotPanics(t, func() { g.EvictFile("t3.go") })
// Step 4: evict T2P. inEdges[T2P] holds e2 with inEdgeKeys
// carrying the LIVE key (insertion via addEdgeToBucket during
// ReindexEdge used the new key). removeEdgeFromBucket(outEdges
// ["src/a.go::A"], LIVE_key) looks up outEdgeIdx[LIVE_key] = 1,
// then tries slice[1] in a slice of length 1 → panic with the
// bug, clean removal with the fix.
require.NotPanics(t, func() {
g.EvictFile("t2p.go")
}, "EvictFile on the reindexed edge's new target must not panic on stale outEdgeIdx")
// All edges removed — bucket should be empty.
assert.Empty(t, g.GetOutEdges("src/a.go::A"), "outEdges bucket must drain after every target was evicted")
}
// edgeIdentityGraph builds a two-node graph with one A→B calls edge at
// the given Origin, returning the graph and the live in-graph edge.
func edgeIdentityGraph(t *testing.T, origin string) (*Graph, *Edge) {
t.Helper()
g := New()
g.AddNode(&Node{ID: "p/a.go::A", Name: "A", Kind: KindFunction, FilePath: "p/a.go"})
g.AddNode(&Node{ID: "p/b.go::B", Name: "B", Kind: KindFunction, FilePath: "p/b.go"})
g.AddEdge(&Edge{From: "p/a.go::A", To: "p/b.go::B", Kind: EdgeCalls, FilePath: "p/a.go", Line: 7, Origin: origin})
out := g.GetOutEdges("p/a.go::A")
require.Len(t, out, 1)
return g, out[0]
}
// TestSetEdgeProvenance_ChangesIdentityAndCounts proves SetEdgeProvenance
// is a delete-then-insert of the edge's identity: a real Origin change
// flips the IdentityHash and bumps the revision counter by exactly one,
// while the logical adjacency-list slot is untouched.
func TestSetEdgeProvenance_ChangesIdentityAndCounts(t *testing.T) {
g, e := edgeIdentityGraph(t, OriginTextMatched)
require.Equal(t, 0, g.EdgeIdentityRevisions(), "fresh graph has no provenance churn")
before := e.IdentityHash()
changed := g.SetEdgeProvenance(e, OriginLSPResolved)
assert.True(t, changed, "upgrading Origin must report an identity change")
assert.Equal(t, OriginLSPResolved, e.Origin, "Origin must be applied")
assert.NotEqual(t, before, e.IdentityHash(), "identity hash must change with Origin")
assert.Equal(t, 1, g.EdgeIdentityRevisions(), "exactly one revision recorded")
// The logical edge is unchanged — same single adjacency entry.
assert.Len(t, g.GetOutEdges("p/a.go::A"), 1, "outEdges count must not change")
assert.Len(t, g.GetInEdges("p/b.go::B"), 1, "inEdges count must not change")
}
// TestSetEdgeProvenance_NoOpWhenOriginUnchanged proves a SetEdgeProvenance
// call that does not actually change Origin is a no-op: identity stable,
// counter untouched, return value false.
func TestSetEdgeProvenance_NoOpWhenOriginUnchanged(t *testing.T) {
g, e := edgeIdentityGraph(t, OriginASTResolved)
before := e.IdentityHash()
changed := g.SetEdgeProvenance(e, OriginASTResolved)
assert.False(t, changed, "setting Origin to its current value is a no-op")
assert.Equal(t, before, e.IdentityHash(), "identity hash must be stable on a no-op")
assert.Equal(t, 0, g.EdgeIdentityRevisions(), "a no-op must not bump the counter")
}
// TestSetEdgeProvenance_RederivesTierWhenSet confirms Tier — the sole
// Origin-derived label on an edge — is recomputed when it was already
// populated, and left empty (the in-memory default) when it was not.
func TestSetEdgeProvenance_RederivesTierWhenSet(t *testing.T) {
// Tier already set: must be re-derived from the new Origin.
g, e := edgeIdentityGraph(t, OriginTextMatched)
e.Tier = ResolvedBy(OriginTextMatched)
g.SetEdgeProvenance(e, OriginLSPResolved)
assert.Equal(t, ResolvedBy(OriginLSPResolved), e.Tier, "populated Tier must track the new Origin")
// Tier left empty: must stay empty rather than start being stamped.
g2, e2 := edgeIdentityGraph(t, OriginTextMatched)
g2.SetEdgeProvenance(e2, OriginLSPResolved)
assert.Equal(t, "", e2.Tier, "an unset Tier must remain unset")
}
// TestAddEdge_ReaddWithUpgradedOriginCounts proves the second mutation
// path: re-adding an edge with the same logical key but an upgraded
// Origin (the resolver's AddEdge-based upgrade path) replaces the slot
// in place AND is counted as an identity revision — without creating a
// duplicate parallel edge.
func TestAddEdge_ReaddWithUpgradedOriginCounts(t *testing.T) {
g, _ := edgeIdentityGraph(t, OriginTextMatched)
require.Equal(t, 0, g.EdgeIdentityRevisions())
// Re-add the same logical edge with a stronger Origin.
g.AddEdge(&Edge{From: "p/a.go::A", To: "p/b.go::B", Kind: EdgeCalls, FilePath: "p/a.go", Line: 7, Origin: OriginLSPResolved})
assert.Equal(t, 1, g.EdgeCount(), "re-add must not create a parallel edge")
assert.Len(t, g.GetOutEdges("p/a.go::A"), 1, "still one outEdge")
assert.Len(t, g.GetInEdges("p/b.go::B"), 1, "still one inEdge")
assert.Equal(t, 1, g.EdgeIdentityRevisions(), "the Origin upgrade on re-add must be counted once")
assert.Equal(t, OriginLSPResolved, g.GetOutEdges("p/a.go::A")[0].Origin, "newer Origin wins")
}
// TestAddEdge_ReaddWithSameOriginDoesNotCount proves an idempotent
// re-add carrying the SAME Origin is not mistaken for provenance churn.
func TestAddEdge_ReaddWithSameOriginDoesNotCount(t *testing.T) {
g, _ := edgeIdentityGraph(t, OriginASTResolved)
for i := 0; i < 5; i++ {
g.AddEdge(&Edge{From: "p/a.go::A", To: "p/b.go::B", Kind: EdgeCalls, FilePath: "p/a.go", Line: 7, Origin: OriginASTResolved})
}
assert.Equal(t, 1, g.EdgeCount(), "idempotent re-add must not grow the edge count")
assert.Equal(t, 0, g.EdgeIdentityRevisions(), "re-add with an unchanged Origin is not a revision")
}
// TestVerifyEdgeIdentities_PassesOnNormalGraph proves a graph built
// only through the sanctioned mutation paths (AddEdge, SetEdgeProvenance)
// is internally consistent — the out-edge and in-edge views agree on
// every edge's provenance-bearing identity.
func TestVerifyEdgeIdentities_PassesOnNormalGraph(t *testing.T) {
g := New()
for _, id := range []string{"p/a.go::A", "p/b.go::B", "p/c.go::C"} {
g.AddNode(&Node{ID: id, Name: id, Kind: KindFunction, FilePath: id})
}
g.AddEdge(&Edge{From: "p/a.go::A", To: "p/b.go::B", Kind: EdgeCalls, FilePath: "p/a.go", Line: 3, Origin: OriginTextMatched})
g.AddEdge(&Edge{From: "p/a.go::A", To: "p/c.go::C", Kind: EdgeCalls, FilePath: "p/a.go", Line: 4, Origin: OriginASTResolved})
g.AddEdge(&Edge{From: "p/b.go::B", To: "p/c.go::C", Kind: EdgeReferences, FilePath: "p/b.go", Line: 9})
require.NoError(t, g.VerifyEdgeIdentities(), "freshly built graph must be identity-consistent")
// A sanctioned provenance change keeps the graph consistent.
out := g.GetOutEdges("p/a.go::A")
require.NotEmpty(t, out)
g.SetEdgeProvenance(out[0], OriginLSPResolved)
require.NoError(t, g.VerifyEdgeIdentities(), "SetEdgeProvenance must preserve identity consistency")
}
// TestVerifyEdgeIdentities_CatchesDivergentOrigin proves the verifier
// is not vacuous: when an edge's Origin is changed on only one
// adjacency view (the failure mode of mutating a copied edge instead
// of routing through SetEdgeProvenance), VerifyEdgeIdentities reports
// the inconsistency.
func TestVerifyEdgeIdentities_CatchesDivergentOrigin(t *testing.T) {
g := New()
g.AddNode(&Node{ID: "p/a.go::A", Name: "A", Kind: KindFunction, FilePath: "p/a.go"})
g.AddNode(&Node{ID: "p/b.go::B", Name: "B", Kind: KindFunction, FilePath: "p/b.go"})
g.AddEdge(&Edge{From: "p/a.go::A", To: "p/b.go::B", Kind: EdgeCalls, FilePath: "p/a.go", Line: 7, Origin: OriginTextMatched})
require.NoError(t, g.VerifyEdgeIdentities())
// Simulate the bug: the in-edge bucket gets a *different* edge
// object whose Origin diverges from the out-edge view. addEdgeToBucket
// keys on the Origin-free logical key, so this overwrites the slot
// with a copy rather than appending.
sTo := g.shardFor("p/b.go::B")
sTo.mu.Lock()
divergent := &Edge{From: "p/a.go::A", To: "p/b.go::B", Kind: EdgeCalls, FilePath: "p/a.go", Line: 7, Origin: OriginLSPResolved}
addEdgeToBucket(sTo.inEdges, sTo.inEdgeKeys, sTo.inEdgeIdx, "p/b.go::B", divergent)
sTo.mu.Unlock()
err := g.VerifyEdgeIdentities()
require.Error(t, err, "a divergent-Origin edge across adjacency views must be caught")
assert.Contains(t, err.Error(), "p/a.go::A", "the error must name the offending edge")
}
+46
View File
@@ -0,0 +1,46 @@
package graph
// RepoIndexState is the per-repo freshness provenance recorded at the
// end of a (re)index: the git revision the graph reflects, whether the
// working tree was dirty at index time, the Merkle workspace
// fingerprint that gates global-pass short-circuiting, node/edge counts
// for the index-plausibility baseline, and a JSON map of the
// per-language extractor versions that produced the graph.
//
// It is the storage half of the FreshnessFact layer; the per-file half
// lives in the Merkle leaf (the salted content hash) and the file_mtimes
// ledger.
type RepoIndexState struct {
RepoPrefix string
IndexedSHA string
Dirty bool
IndexedAt int64 // unix seconds
WorkspaceFP string // Merkle root at index time
NodeCount int
EdgeCount int
ExtractorVersions string // JSON-encoded map[string]int
}
// RepoIndexStateWriter persists the freshness provenance for one repo.
// Backends without durable state simply do not implement it — the
// indexer type-asserts and skips the write when absent, exactly like the
// FileMtime ledger.
type RepoIndexStateWriter interface {
SetRepoIndexState(state RepoIndexState) error
}
// RepoIndexStateReader reads back the freshness provenance for one repo.
// The bool is false when no state has been recorded yet (a never-indexed
// or pre-feature repo), which callers treat as "freshness unknown" — they
// never block on it.
type RepoIndexStateReader interface {
GetRepoIndexState(repoPrefix string) (RepoIndexState, bool, error)
}
// DBStatReporter is an optional capability: report the on-disk size of the
// backing database file and its write-ahead log, in bytes. Surfaced in
// daemon_health so a runaway WAL high-water mark is observable. In-memory
// backends do not implement it.
type DBStatReporter interface {
DBStats() (dbBytes, walBytes int64)
}
+296
View File
@@ -0,0 +1,296 @@
package graph
import (
"testing"
)
func TestRepoMemoryEstimate_Empty(t *testing.T) {
g := New()
est := g.RepoMemoryEstimate("nonexistent")
if est.NodeCount != 0 || est.EdgeCount != 0 || est.Total() != 0 {
t.Errorf("empty repo should estimate zero, got %+v", est)
}
}
func TestRepoMemoryEstimate_NodesAndEdges(t *testing.T) {
g := New()
n1 := &Node{ID: "r/pkg/a.go::Foo", Kind: KindFunction, Name: "Foo",
QualName: "pkg.Foo", FilePath: "pkg/a.go", Language: "go",
RepoPrefix: "r"}
n2 := &Node{ID: "r/pkg/a.go::Bar", Kind: KindFunction, Name: "Bar",
QualName: "pkg.Bar", FilePath: "pkg/a.go", Language: "go",
RepoPrefix: "r"}
g.AddNode(n1)
g.AddNode(n2)
g.AddEdge(&Edge{From: n1.ID, To: n2.ID, Kind: "CALLS", FilePath: "pkg/a.go"})
est := g.RepoMemoryEstimate("r")
if est.NodeCount != 2 {
t.Errorf("expected 2 nodes, got %d", est.NodeCount)
}
if est.EdgeCount != 1 {
t.Errorf("expected 1 edge, got %d", est.EdgeCount)
}
if est.NodeBytes == 0 || est.EdgeBytes == 0 {
t.Errorf("expected non-zero byte estimates, got %+v", est)
}
}
func TestRepoMemoryEstimate_MetaContributes(t *testing.T) {
g := New()
short := &Node{ID: "r/a::s", Kind: KindVariable, Name: "s",
FilePath: "a", RepoPrefix: "r"}
long := &Node{ID: "r/a::l", Kind: KindVariable, Name: "l",
FilePath: "a", RepoPrefix: "r",
Meta: map[string]any{
"signature": "func Foo(ctx context.Context, x, y int) (string, error)",
"docstring": "A long docstring that takes many bytes to store in memory",
"tags": []string{"public", "deprecated", "hot-path"},
}}
g.AddNode(short)
g.AddNode(long)
shortEst := g.RepoMemoryEstimate("r").NodeBytes / 2 // rough avg — need per-node
_ = shortEst
// Direct per-node check via unexported helper is fine from test in same package.
if nodeBytes(long) <= nodeBytes(short) {
t.Errorf("a node with Meta should be bigger than one without: long=%d short=%d",
nodeBytes(long), nodeBytes(short))
}
}
func TestRepoMemoryEstimate_RemoveEdgeDecrements(t *testing.T) {
g := New()
n1 := &Node{ID: "r/a::Foo", Kind: KindFunction, Name: "Foo", FilePath: "a", RepoPrefix: "r"}
n2 := &Node{ID: "r/a::Bar", Kind: KindFunction, Name: "Bar", FilePath: "a", RepoPrefix: "r"}
g.AddNode(n1)
g.AddNode(n2)
e := &Edge{From: n1.ID, To: n2.ID, Kind: "CALLS", FilePath: "a"}
g.AddEdge(e)
before := g.RepoMemoryEstimate("r")
if before.EdgeCount != 1 {
t.Fatalf("expected 1 edge before remove, got %d", before.EdgeCount)
}
g.RemoveEdge(n1.ID, n2.ID, "CALLS")
after := g.RepoMemoryEstimate("r")
if after.EdgeCount != 0 {
t.Errorf("expected 0 edges after RemoveEdge, got %d", after.EdgeCount)
}
if after.EdgeBytes != 0 {
t.Errorf("expected 0 edge bytes after RemoveEdge, got %d", after.EdgeBytes)
}
}
func TestRepoMemoryEstimate_IdempotentAddDoesNotDoubleCount(t *testing.T) {
g := New()
n1 := &Node{ID: "r/a::Foo", Kind: KindFunction, Name: "Foo", FilePath: "a", RepoPrefix: "r"}
n2 := &Node{ID: "r/a::Bar", Kind: KindFunction, Name: "Bar", FilePath: "a", RepoPrefix: "r"}
g.AddNode(n1)
g.AddNode(n2)
e := &Edge{From: n1.ID, To: n2.ID, Kind: "CALLS", FilePath: "a", Line: 10}
g.AddEdge(e)
g.AddEdge(e) // same identity — must not double-count
g.AddNode(n1) // same identity — must not double-count nodes
est := g.RepoMemoryEstimate("r")
if est.NodeCount != 2 {
t.Errorf("expected 2 nodes after duplicate Adds, got %d", est.NodeCount)
}
if est.EdgeCount != 1 {
t.Errorf("expected 1 edge after duplicate AddEdge, got %d", est.EdgeCount)
}
}
func TestRepoMemoryEstimate_EvictFileDecrements(t *testing.T) {
g := New()
n1 := &Node{ID: "r/a::Foo", Kind: KindFunction, Name: "Foo", FilePath: "a", RepoPrefix: "r"}
n2 := &Node{ID: "r/b::Bar", Kind: KindFunction, Name: "Bar", FilePath: "b", RepoPrefix: "r"}
g.AddNode(n1)
g.AddNode(n2)
g.AddEdge(&Edge{From: n1.ID, To: n2.ID, Kind: "CALLS", FilePath: "a"})
g.EvictFile("a")
est := g.RepoMemoryEstimate("r")
if est.NodeCount != 1 {
t.Errorf("expected 1 node after evicting file 'a', got %d", est.NodeCount)
}
// Edge sourced from evicted node 'Foo' must also be gone.
if est.EdgeCount != 0 {
t.Errorf("expected 0 edges after evicting source node's file, got %d", est.EdgeCount)
}
}
func TestRepoMemoryEstimate_EvictRepoZeroesCounter(t *testing.T) {
g := New()
g.AddNode(&Node{ID: "r1/a::A", Kind: KindFunction, Name: "A", FilePath: "a", RepoPrefix: "r1"})
g.AddNode(&Node{ID: "r1/a::B", Kind: KindFunction, Name: "B", FilePath: "a", RepoPrefix: "r1"})
g.AddNode(&Node{ID: "r2/x::X", Kind: KindFunction, Name: "X", FilePath: "x", RepoPrefix: "r2"})
g.AddEdge(&Edge{From: "r1/a::A", To: "r1/a::B", Kind: "CALLS", FilePath: "a"})
g.AddEdge(&Edge{From: "r1/a::A", To: "r2/x::X", Kind: "CALLS", FilePath: "a"})
g.EvictRepo("r1")
r1 := g.RepoMemoryEstimate("r1")
if r1.NodeCount != 0 || r1.EdgeCount != 0 || r1.Total() != 0 {
t.Errorf("evicted repo should estimate zero, got %+v", r1)
}
// r2 has only the surviving X node; the cross-repo edge into X was
// attributed to r1 (source repo) so it shouldn't show up under r2.
r2 := g.RepoMemoryEstimate("r2")
if r2.NodeCount != 1 {
t.Errorf("r2 should still have 1 node, got %d", r2.NodeCount)
}
if r2.EdgeCount != 0 {
t.Errorf("r2 should have 0 outgoing edges, got %d", r2.EdgeCount)
}
}
func TestAllRepoMemoryEstimates_AggregatesAcrossRepos(t *testing.T) {
g := New()
g.AddNode(&Node{ID: "r1/a::A", Kind: KindFunction, Name: "A", FilePath: "a", RepoPrefix: "r1"})
g.AddNode(&Node{ID: "r2/b::B", Kind: KindFunction, Name: "B", FilePath: "b", RepoPrefix: "r2"})
g.AddNode(&Node{ID: "r2/b::C", Kind: KindFunction, Name: "C", FilePath: "b", RepoPrefix: "r2"})
g.AddEdge(&Edge{From: "r2/b::B", To: "r2/b::C", Kind: "CALLS", FilePath: "b"})
all := g.AllRepoMemoryEstimates()
if got := all["r1"].NodeCount; got != 1 {
t.Errorf("r1 node count = %d, want 1", got)
}
if got := all["r2"].NodeCount; got != 2 {
t.Errorf("r2 node count = %d, want 2", got)
}
if got := all["r2"].EdgeCount; got != 1 {
t.Errorf("r2 edge count = %d, want 1", got)
}
if got := all["r1"].EdgeCount; got != 0 {
t.Errorf("r1 edge count = %d, want 0", got)
}
// Sanity: bulk and per-repo agree.
if perRepo := g.RepoMemoryEstimate("r2"); perRepo != all["r2"] {
t.Errorf("per-repo and bulk disagree for r2: per-repo=%+v bulk=%+v", perRepo, all["r2"])
}
}
// walkRepoCounts is a reference implementation: walks every node and
// every edge once to bucket counts by repo. Used to verify the running
// counters stay in sync across a mix of mutations.
func walkRepoCounts(g *Graph) (nodes, edges map[string]int) {
nodes = make(map[string]int)
edges = make(map[string]int)
for _, n := range g.AllNodes() {
if n.RepoPrefix != "" {
nodes[n.RepoPrefix]++
}
}
for _, e := range g.AllEdges() {
src := g.GetNode(e.From)
if src != nil && src.RepoPrefix != "" {
edges[src.RepoPrefix]++
}
}
return
}
func assertCountersMatchWalk(t *testing.T, g *Graph, where string) {
t.Helper()
walkN, walkE := walkRepoCounts(g)
got := g.AllRepoMemoryEstimates()
for prefix, want := range walkN {
if got[prefix].NodeCount != want {
t.Errorf("%s: %q node count: counter=%d walk=%d",
where, prefix, got[prefix].NodeCount, want)
}
}
for prefix, want := range walkE {
if got[prefix].EdgeCount != want {
t.Errorf("%s: %q edge count: counter=%d walk=%d",
where, prefix, got[prefix].EdgeCount, want)
}
}
// Counters that don't exist in the walk are also a bug — leaked
// entries that survived eviction.
for prefix, est := range got {
if walkN[prefix] == 0 && est.NodeCount != 0 {
t.Errorf("%s: %q has counter NodeCount=%d but walk says 0",
where, prefix, est.NodeCount)
}
if walkE[prefix] == 0 && est.EdgeCount != 0 {
t.Errorf("%s: %q has counter EdgeCount=%d but walk says 0",
where, prefix, est.EdgeCount)
}
}
}
func TestRepoMemoryCounters_StayInSyncUnderMixedMutations(t *testing.T) {
g := New()
// Two repos, several nodes each, a mix of intra- and cross-repo edges.
r1Nodes := []*Node{
{ID: "r1/a::A", Kind: KindFunction, Name: "A", FilePath: "a", RepoPrefix: "r1"},
{ID: "r1/a::B", Kind: KindFunction, Name: "B", FilePath: "a", RepoPrefix: "r1"},
{ID: "r1/b::C", Kind: KindFunction, Name: "C", FilePath: "b", RepoPrefix: "r1"},
}
r2Nodes := []*Node{
{ID: "r2/x::X", Kind: KindFunction, Name: "X", FilePath: "x", RepoPrefix: "r2"},
{ID: "r2/x::Y", Kind: KindFunction, Name: "Y", FilePath: "x", RepoPrefix: "r2"},
}
for _, n := range append(append([]*Node{}, r1Nodes...), r2Nodes...) {
g.AddNode(n)
}
assertCountersMatchWalk(t, g, "after AddNode")
edges := []*Edge{
{From: "r1/a::A", To: "r1/a::B", Kind: "CALLS", FilePath: "a"},
{From: "r1/a::A", To: "r1/b::C", Kind: "CALLS", FilePath: "a"},
{From: "r1/b::C", To: "r2/x::X", Kind: "CALLS", FilePath: "b"}, // cross-repo
{From: "r2/x::X", To: "r2/x::Y", Kind: "CALLS", FilePath: "x"},
{From: "r2/x::Y", To: "r1/a::A", Kind: "CALLS", FilePath: "x"}, // cross-repo back
}
for _, e := range edges {
g.AddEdge(e)
}
assertCountersMatchWalk(t, g, "after AddEdge")
// Idempotent re-adds must not double-count.
g.AddNode(r1Nodes[0])
g.AddEdge(edges[0])
assertCountersMatchWalk(t, g, "after idempotent re-adds")
// RemoveEdge.
g.RemoveEdge("r1/a::A", "r1/a::B", "CALLS")
assertCountersMatchWalk(t, g, "after RemoveEdge")
// EvictFile — drops both r1/a::A and r1/a::B's host file, including
// the cross-repo edge r1/b::C → r2/x::X is left alone.
g.EvictFile("a")
assertCountersMatchWalk(t, g, "after EvictFile(a)")
// AddNode an updated version of an existing node (RepoPrefix change).
g.AddNode(&Node{ID: "r1/b::C", Kind: KindFunction, Name: "C", FilePath: "b", RepoPrefix: "r1"})
assertCountersMatchWalk(t, g, "after node update (same prefix)")
// EvictRepo r2 — must zero its counters and decrement r1's edge
// counter for the r1/b::C → r2/x::X edge (source is r1, target was r2).
g.EvictRepo("r2")
assertCountersMatchWalk(t, g, "after EvictRepo(r2)")
}
func TestMetaBytes_HandlesCommonTypes(t *testing.T) {
cases := []map[string]any{
nil,
{},
{"s": "hello"},
{"b": true, "i": 42, "f": 3.14},
{"list": []string{"a", "b", "c"}},
{"nested": map[string]any{"k": "v"}},
}
for i, m := range cases {
// Should never panic; empty map returns non-zero-ish (header).
got := metaBytes(m)
if m == nil && got != 0 {
t.Errorf("case %d: nil map should be 0, got %d", i, got)
}
}
}
+26
View File
@@ -0,0 +1,26 @@
package graph
import (
"testing"
"github.com/stretchr/testify/require"
)
func TestMotivatesKinds(t *testing.T) {
require.Equal(t, NodeKind("rationale"), KindRationale)
require.Equal(t, EdgeKind("motivates"), EdgeMotivates)
require.Equal(t, EdgeKind("cross_repo_motivates"), EdgeCrossRepoMotivates)
}
func TestCrossRepoMotivatesRegistered(t *testing.T) {
cr, ok := CrossRepoKindFor(EdgeMotivates)
require.True(t, ok, "EdgeMotivates must have a cross-repo parallel")
require.Equal(t, EdgeCrossRepoMotivates, cr)
base, ok := BaseKindForCrossRepo(EdgeCrossRepoMotivates)
require.True(t, ok)
require.Equal(t, EdgeMotivates, base)
require.Contains(t, BaseKindsForCrossRepo(), EdgeMotivates,
"DetectCrossRepoEdges must materialise the motivates parallel")
}
+520
View File
@@ -0,0 +1,520 @@
package graph
import (
"strings"
"time"
)
type NodeKind string
const (
KindFile NodeKind = "file"
KindPackage NodeKind = "package"
KindFunction NodeKind = "function"
KindMethod NodeKind = "method"
KindType NodeKind = "type"
KindInterface NodeKind = "interface"
KindVariable NodeKind = "variable"
KindImport NodeKind = "import"
KindContract NodeKind = "contract"
// KindField represents a struct field, class property, or record
// field — anything addressable as `owner.field`. ID convention:
// `<file>::<owner>.<field>`. EdgeMemberOf links the field to its
// owning type. Languages that already emitted class properties as
// KindVariable (TypeScript, PHP) keep doing so for backwards
// compatibility — KindField is reserved for languages that
// previously emitted only type-ref edges from fields (Go, Rust,
// Java, C#).
KindField NodeKind = "field"
// Coverage kinds: each is gated behind a per-domain
// .gortex.yaml::index.coverage.<domain>.enabled. Parsers register a
// kind on first use; the registry is permissive (validNodeKinds
// accepts all known kinds) so an unenabled domain simply produces no
// nodes of that kind, rather than failing extraction.
// KindParam represents a single function/method parameter. ID
// convention: `<func-id>#param:<name>`. EdgeParamOf links the param
// node back to its owner; EdgeTypedAs binds it to its declared
// type. Created when index.function_shape.enabled is true.
KindParam NodeKind = "param"
// KindClosure represents an anonymous function / lambda inside an
// enclosing function. ID convention: `<file>::<enclosing>#closure@<line>`.
// Calls/reads/writes inside the closure attribute to the closure
// node, not its enclosing function. EdgeMemberOf links to the
// enclosing function. EdgeCaptures lists outer bindings closed over.
KindClosure NodeKind = "closure"
// KindLocal represents an intra-function binding — a variable
// declared inside a function body via `x := …` / `var x = …` / a
// range clause / a type-switch / a for-init clause. ID convention:
// `<ownerID>#local:<name>@+<offsetFromOwnerStartLine>` (the
// leading `+` flags the value as a relative offset so the IDs
// stay stable when the enclosing function moves as a whole).
// EdgeMemberOf links each binding to its enclosing function or
// method. KindLocal is excluded from the BM25 search index by
// shouldIndexForSearch — surfacing `err` / `data` / `n` / `i`
// from every function would flood every name lookup. The data-
// flow analysis (flow_between, taint_paths, ...) traverses the
// EdgeValueFlow / EdgeArgOf / EdgeReturnsTo edges that target
// these nodes; consumers that want the locals can ask for them
// by kind explicitly.
KindLocal NodeKind = "local"
// KindBuiltin represents a language intrinsic — a function /
// type / constant that's part of the language itself, not
// declared in any indexed source file. ID convention:
// `builtin::<lang>::<name>` for functions (`builtin::go::append`,
// `builtin::py::len`) and `builtin::<lang>::type::<name>` for
// types (`builtin::go::type::string`). Meta.builtin_kind ∈
// "func" | "type" | "const". KindBuiltin is excluded from the
// BM25 search index — surfacing `string` / `int` / `append`
// would flood every name lookup. They participate in normal
// graph queries: `find_usages(builtin::go::type::float64)`
// answers "every variable typed as float64 in this codebase",
// which is the load-bearing query for type-drift / dataflow
// analyses.
KindBuiltin NodeKind = "builtin"
// KindConstant peels off `const`, `iota`, top-level immutable
// bindings, and language-specific constant declarations from
// KindVariable. Existing variable-kind nodes are re-classified on
// next index; IDs are preserved.
KindConstant NodeKind = "constant"
// KindEnumMember represents one member of an enum-like type. ID
// convention: `<file>::<EnumType>.<Member>`. EdgeMemberOf links to
// the enum's type node.
KindEnumMember NodeKind = "enum_member"
// KindGenericParam represents a type parameter declared by a
// function or type. ID convention: `<owner-id>#tparam:<name>`.
KindGenericParam NodeKind = "generic_param"
// KindModule represents a single (ecosystem, name, version) tuple
// for an external dependency. Shared across files that import it.
// ID convention: `module::<ecosystem>:<name>@<version>`.
// Ecosystems: go, npm, pypi, cargo, maven, composer, gem, hex, nuget.
KindModule NodeKind = "module"
// KindTable represents a database table. ID convention:
// `db::<dialect>::<schema>.<table>`. Sourced from migrations, ORM
// models, and string-literal SQL in priority order.
KindTable NodeKind = "table"
// KindColumn represents a database column. ID convention:
// `db::<dialect>::<schema>.<table>.<column>`. EdgeMemberOf links to
// the owning table.
KindColumn NodeKind = "column"
// KindConfigKey represents a configuration key — env var, viper
// path, CLI flag, struct-tag-driven field, or k8s ConfigMap entry.
// ID convention: `cfg::<source>::<dotted.path>`. Source ∈
// env|viper|flags|k8s_cm|k8s_secret|struct_tag.
KindConfigKey NodeKind = "config_key"
// KindFlag represents a feature flag / experiment. ID convention:
// `flag::<provider>::<name>`. Provider ∈ growthbook|launchdarkly|
// unleash|internal|env.
KindFlag NodeKind = "flag"
// KindEvent represents a log, metric, span, or trace name emitted
// from code, or a pub/sub topic/channel/subject. ID convention:
// `event::<kind>::<name>` for observability events and
// `event::pubsub::<transport>::<name>` for pub/sub topics.
// Meta["event_kind"] ∈ log|metric|trace|span|pubsub. For pubsub
// events Meta["transport"] ∈ nats|kafka|rabbitmq|redis|socketio|
// eventemitter|unknown; publishers link in via EdgeEmits and
// subscribers via EdgeListensOn.
KindEvent NodeKind = "event"
// KindMigration represents a database migration unit. ID
// convention: `migration::<dialect>::<id>`. Provides tables/columns
// it creates; consumes ones it references.
KindMigration NodeKind = "migration"
// KindFixture represents a test data file or golden file. ID
// convention: `fixture::<path>`. Test functions reference it via
// EdgeReferences.
KindFixture NodeKind = "fixture"
// KindTodo represents a TODO/FIXME/HACK/XXX/NOTE comment marker. ID
// convention: `todo::<file>:<line>`. Meta carries tag, assignee,
// due, ticket, and the truncated text.
KindTodo NodeKind = "todo"
// KindTeam represents a CODEOWNERS team or individual. ID
// convention: `team::<name>`. Meta.kind ∈ team|person disambiguates.
KindTeam NodeKind = "team"
// KindRelease represents a tag/version boundary. ID convention:
// `release::<tag>`. Used as a query filter via Node.Meta["added_in"]
// rather than as an edge endpoint in most cases.
KindRelease NodeKind = "release"
// KindLicense represents an SPDX license identifier. ID convention:
// `license::<spdx>`. Files link to it via EdgeLicensedAs.
KindLicense NodeKind = "license"
// KindString represents a string literal that crosses an API
// boundary worth tracking — Datadog/Prometheus metric names,
// errors.New / fmt.Errorf messages, raw HTTP route paths, and
// (later) HTML class/id values. ID convention:
// `string::<context>::<value-or-hash>`. Context ∈
// metric|error_msg|route|html_class|html_id|… EdgeEmits links the
// enclosing function/method to the string node, mirroring KindEvent.
// Per-repo: applyRepoPrefix prefixes every node ID with the repo
// slug so two repos that emit the same string don't collide.
KindString NodeKind = "string"
// KindResource represents a Kubernetes manifest resource —
// Deployment, Service, ConfigMap, Secret, Ingress, CronJob,
// StatefulSet, DaemonSet, Job, ReplicaSet, ServiceAccount,
// Role, RoleBinding, ClusterRole, ClusterRoleBinding, Namespace,
// PersistentVolume, PersistentVolumeClaim, etc. ID convention:
// `k8s::<kind>::<namespace>::<name>` (namespace defaults to
// "_" when not declared in the manifest). Meta carries
// api_version, namespace, labels (truncated). Sourced from YAML
// extractors that detect K8s manifests by `apiVersion:` +
// `kind:` markers.
KindResource NodeKind = "resource"
// KindKustomization represents a Kustomize overlay — one per
// `kustomization.yaml` / `kustomization.yml` file in a repo.
// ID convention: `kustomize::<dir>` where dir is the directory
// holding the kustomization file relative to the repo root.
// Resources, bases, components, and patches are linked via
// EdgeDependsOn (overlay → base) and EdgeReferences
// (overlay → resource files).
KindKustomization NodeKind = "kustomization"
// KindImage represents either a container image or a raster/vector
// image asset — distinguished by ID prefix and Meta. Container images
// come from a Dockerfile FROM (external base or `FROM ... AS <stage>`)
// or a K8s container spec; image assets are picture files ingested by
// the multimodal extractor. ID conventions:
// `image::<name>:<tag>` for external/registry images (tag
// defaults to "latest" when omitted)
// `image::stage::<file>::<stage-name>` for Dockerfile build
// stages
// `image::asset::<path>` for ingested image-file assets
// Meta carries registry/digest/platform for container images, and
// format/width/height/size_bytes/sha256 (asset_kind="image") for
// image-file assets.
KindImage NodeKind = "image"
// KindArtifact represents a non-code knowledge file declared in
// the `.gortex.yaml::artifacts` manifest — a DB schema (SQL /
// Prisma / dbt), an API spec (OpenAPI / GraphQL / protobuf), an
// infra config (Terraform / Kustomize / Helm), or an
// architecture doc (ADR markdown). ID convention:
// `artifact::<repo-relative-path>`. Meta carries artifact_kind
// (schema|api|infra|doc), content_hash (sha256 of the file —
// drives staleness detection), title, and size. The artifact
// node links to every symbol it mentions via EdgeReferences so
// agents can pull the right schema or spec alongside the code.
KindArtifact NodeKind = "artifact"
// KindDoc represents one heading-delimited prose section of a
// documentation file (Markdown). Name is the breadcrumb heading
// path ("README.md > Setup > Build"); Meta["section_text"] holds
// the section's paragraph text with markdown syntax stripped, and
// the BM25 search index is fed that body so a prose query ranks
// the right section. ID convention:
// "<file>::doc:<slug-of-heading-path>" -- derived from the
// heading path, NOT line numbers, so an incremental reindex of an
// edited file keeps stable section identity. The owning file
// links to it via EdgeDefines.
KindDoc NodeKind = "doc"
// KindRationale is a graph projection of a development-memory record
// (a decision / incident / constraint / invariant) — the "why" behind
// code. The store_memory sidecar stays the system of record; this node
// is a derived view re-projected on memory write and reconciled on
// warmup, so a why-query is one hop from the code it explains. ID
// convention: "rationale::<memory-id>". Links to the code it explains
// via EdgeMotivates.
KindRationale NodeKind = "rationale"
// KindTopic represents a message-broker topic / subject / channel /
// exchange — the contract-layer pairing artefact for Kafka,
// RabbitMQ, NATS, and Redis pub-sub. ID convention:
// `topic::<broker>::<name>` where broker ∈ kafka|rabbitmq|nats|
// redis. Meta carries broker (the family) and name (the raw topic
// / subject / channel / exchange string). Producer symbols link
// in via EdgeProducesTopic, consumers via EdgeConsumesTopic, so a
// cross-service event flow is a two-hop path
// producer --produces_topic--> topic <--consumes_topic-- consumer.
// KindTopic is distinct from KindEvent (which the observability /
// pubsub extractor uses for the same calls): KindEvent rides at
// the call-site evidence tier (heuristic / inferred) and is for
// broad "what publishes here" queries, while KindTopic is the
// pairing artefact produced by the contracts matcher and rides at
// the structural ast_resolved tier.
KindTopic NodeKind = "topic"
// KindMacro represents a C/C++ preprocessor macro defined with
// #define — both object-like (`#define PI 3.14`) and function-like
// (`#define SQ(x) ((x)*(x))`). ID convention: `<file>::<NAME>`.
// Meta["macro_kind"] ∈ object|function; function-like macros carry
// Meta["params"] (the parameter names) and Meta["replacement"] (the
// replacement-list text). The owning file links via EdgeDefines, and
// a function-like macro emits EdgeCalls to the symbols its
// replacement list invokes — recovering call edges that would
// otherwise be hidden behind macro expansion (a call site `SQ(2)`
// resolves to the macro, whose body-calls then continue the chain).
KindMacro NodeKind = "macro"
// KindAgent represents a live coding agent participating in a
// multi-agent session — a first-class, queryable presence entity
// distinct from a transport session. ID convention: `agent::<id>`.
// Carries Meta["cursor"] (the symbol/file the agent is focused on),
// Meta["status"], Meta["locked_paths"], and Meta["last_seen"]. Agent
// presence is volatile and lives in an in-memory registry rather than
// the persistent code graph; this kind names the entity so it reads as
// first-class in tool output and query filters.
KindAgent NodeKind = "agent"
// KindContractBridge represents one matched provider↔consumer
// contract group — an HTTP route, a gRPC/Thrift method, or a
// pub/sub topic — materialised as a single graph node that spans
// every repo participating in the group. ID convention:
// `bridge::<contract-id>` where contract-id is the canonical
// contract key (`http::GET::/v1/users`, `grpc::Users::GetUser`,
// `topic::kafka::orders`), so the bridge for any contract is
// addressable from the contract ID alone, across repos. Meta
// carries contract_type, canonical_key, repos (sorted slice of
// participating repo prefixes), provider_count, consumer_count
// and cross_repo. EdgeBridges links the bridge to each
// participating KindContract node. Bridge nodes are re-derived
// from the matcher result on every contract reconcile — all of
// them share the synthetic FilePath "contracts://bridges" so the
// reconcile pass can evict the stale generation with one
// EvictFile call before re-minting.
KindContractBridge NodeKind = "contract_bridge"
)
// IsValidNodeKind reports whether s names a known node kind. Used by
// the search layer to tell a real node-kind clause apart from a
// flavor value that only looks like a kind (e.g. codegraph's
// `kind:class`).
func IsValidNodeKind(s string) bool {
return validNodeKinds[NodeKind(s)]
}
var validNodeKinds = map[NodeKind]bool{
KindFile: true, KindPackage: true, KindFunction: true,
KindMethod: true, KindType: true, KindInterface: true,
KindVariable: true, KindImport: true, KindContract: true,
KindField: true,
// Coverage kinds — see Kind* doc comments above for usage notes.
KindParam: true, KindClosure: true, KindConstant: true,
KindEnumMember: true, KindGenericParam: true, KindModule: true,
KindTable: true, KindColumn: true, KindConfigKey: true,
KindFlag: true, KindEvent: true, KindMigration: true,
KindFixture: true, KindTodo: true, KindTeam: true,
KindRelease: true, KindLicense: true, KindString: true,
KindResource: true, KindKustomization: true, KindImage: true,
KindArtifact: true, KindDoc: true, KindTopic: true,
KindRationale: true,
KindMacro: true, KindAgent: true, KindContractBridge: true,
}
type Node struct {
ID string `json:"id"`
Kind NodeKind `json:"kind"`
Name string `json:"name"`
QualName string `json:"qual_name,omitempty"`
FilePath string `json:"file_path"`
StartLine int `json:"start_line"`
// EndLine is omitted when zero — File-kind nodes don't have ranges.
EndLine int `json:"end_line,omitempty"`
// StartColumn / EndColumn are 0-based source column offsets of the
// symbol's span. Omitted when zero (most extractors record only line
// ranges); promoted to typed nodes columns on the SQLite backend.
StartColumn int `json:"start_column,omitempty"`
EndColumn int `json:"end_column,omitempty"`
Language string `json:"language"`
Meta map[string]any `json:"meta,omitempty"`
RepoPrefix string `json:"repo_prefix,omitempty"`
// WorkspaceID is the hard graph boundary slug. Two nodes with
// different WorkspaceIDs are not allowed to be matched as contract
// provider/consumer pairs and queries scope by it by default.
// Defaults at warmup time to the per-repo `.gortex.yaml::workspace`
// setting; falls back to RepoPrefix when no workspace is declared
// (so old configs keep working) and to "" only for snapshot
// records written before the field existed (gob decodes unknown
// fields as zero — warmup backfills these from config).
WorkspaceID string `json:"workspace_id,omitempty"`
// ProjectID is the soft sub-boundary inside a workspace. One
// project per repo by default; monorepos can declare projects[] in
// .gortex.yaml. Contract pairing is bounded to a single
// (workspace_id, project_id); cross-project contracts become orphans.
// Defaults to the repo name when no projects[] mapping matches.
ProjectID string `json:"project_id,omitempty"`
// AbsoluteFilePath is the on-disk absolute path corresponding to
// FilePath. It is empty on the canonical graph node and is populated
// only on the per-response copies the MCP layer hands to result
// encoders, so an editor or agent can open a result directly without
// reconstructing the path from repo_prefix + file_path.
AbsoluteFilePath string `json:"absolute_file_path,omitempty"`
// Origin marks a node minted by the cross-daemon proxy-edge feature
// as standing in for a symbol another daemon owns. "" on every
// locally-indexed node; "remote:<slug>" on a proxy node. Written ONLY
// by the proxy-edge mint path; the read-only fan-out carries
// provenance in the response, never here. Excluded from
// graph_stats / BM25 / communities / analyzers (see IsProxyNode).
Origin string `json:"origin,omitempty"`
// Stub marks a node as a federation proxy placeholder whose
// neighbour edges hydrate lazily over /v1/subgraph rather than from
// local extraction. Always true together with a non-empty Origin.
Stub bool `json:"stub,omitempty"`
// FetchedAt is the wall-clock time the proxy node (and its hydrated
// ring) was last pulled from the remote — drives the TTL freshness
// gate and the federated last_synced field. Zero on local nodes.
FetchedAt time.Time `json:"fetched_at,omitempty"`
}
// IsReExportNode reports whether n is a barrel re-export binding — a node
// minted at an `export { X } from './mod'` site that forwards another module's
// declaration under the exported (post-alias) name. Marked with
// Meta["reexport"]==true by the JS/TS extractor. These nodes are transparent
// aliases: call resolution skips them (a call binds to the forwarded
// declaration, not the façade), while find_usages delegates their usage set to
// the canonical target.
func IsReExportNode(n *Node) bool {
if n == nil || n.Meta == nil {
return false
}
v, ok := n.Meta["reexport"].(bool)
return ok && v
}
// Brief returns a compact representation with only the fields needed for listing.
func (n *Node) Brief() map[string]any {
b := map[string]any{
"id": n.ID,
"name": n.Name,
"kind": n.Kind,
"file_path": n.FilePath,
"start_line": n.StartLine,
}
if n.RepoPrefix != "" {
b["repo_prefix"] = n.RepoPrefix
}
if n.WorkspaceID != "" {
b["workspace_id"] = n.WorkspaceID
}
if n.ProjectID != "" {
b["project_id"] = n.ProjectID
}
// Surface visibility and a short doc snippet when present — Brief
// is the listing projection used by search_symbols and find_usages,
// where these two fields meaningfully sharpen the result so the
// agent can decide without a follow-up get_symbol_source call.
if v, ok := n.Meta["visibility"].(string); ok && v != "" {
b["visibility"] = v
}
if d, ok := n.Meta["doc"].(string); ok && d != "" {
// Truncate doc to 80 chars in Brief — the full doc is on the
// node, this is just the listing teaser.
const briefDocCap = 80
if len(d) > briefDocCap {
d = d[:briefDocCap] + "…"
}
b["doc"] = d
}
// Test classification — stamped by the indexer's test-edge pass.
// Surfacing it on the listing row lets agents tell production
// callers from test callers without a follow-up call.
if v, ok := n.Meta["is_test"].(bool); ok && v {
b["is_test"] = true
}
if r, ok := n.Meta["test_role"].(string); ok && r != "" {
b["test_role"] = r
}
if r, ok := n.Meta["test_runner"].(string); ok && r != "" {
b["test_runner"] = r
}
if v, ok := n.Meta["is_test_file"].(bool); ok && v {
b["is_test_file"] = true
}
// Structural flavor + UI-component framework — stamped by the
// language extractors. Surfacing them on the listing row lets an
// agent filter / triage by shape (struct vs class, react vs svelte)
// without a follow-up call.
if v, ok := n.Meta["type_flavor"].(string); ok && v != "" {
b["type_flavor"] = v
}
if v, ok := n.Meta["ui_component"].(string); ok && v != "" {
b["ui_component"] = v
}
// A prose-section node carries no signature -- surface a short
// snippet of its body text so a docs search result is
// self-describing without a follow-up read.
if n.Kind == KindDoc {
if txt, ok := n.Meta["section_text"].(string); ok && txt != "" {
const snippetCap = 160
if len(txt) > snippetCap {
txt = txt[:snippetCap] + "\u2026"
}
b["section"] = txt
}
}
// enclosing / enclosing_id name the symbol this node is declared
// inside -- the receiver type of a method, the struct of a field,
// the enum of a member, the function around a closure. Derived
// from the ID convention; absent for top-level symbols. Lets a
// search result say "Parse on type Decoder" without a follow-up
// call.
if eid, ename := EnclosingFromID(n.ID, n.Kind); ename != "" {
b["enclosing"] = ename
b["enclosing_id"] = eid
}
// AbsoluteFilePath is populated only on the per-response copies the
// MCP layer builds (see Server.withAbsPaths); empty on canonical nodes.
if n.AbsoluteFilePath != "" {
b["absolute_file_path"] = n.AbsoluteFilePath
}
return b
}
// EnclosingFromID derives a node's enclosing owner purely from its
// ID and kind -- no graph access. It covers the kinds whose ID
// convention embeds the owner:
//
// - method "<file>::<Owner>.<method>" -> owner "<file>::<Owner>"
// - field "<file>::<owner>.<field>" -> owner "<file>::<owner>"
// - enum "<file>::<EnumType>.<Member>" -> owner "<file>::<EnumType>"
// - closure "<file>::<enclosing>#closure@N" -> owner "<file>::<enclosing>"
//
// For every other kind -- and for a method/field/closure whose ID
// carries no owner segment -- both return values are empty. The
// returned name is the owner's short (last-segment) name.
//
// This is the standalone derivation Node.Brief uses; callers with a
// graph reader should prefer the richer EdgeMemberOf-based lookup,
// which also resolves owners the ID does not name.
func EnclosingFromID(id string, kind NodeKind) (ownerID, ownerName string) {
sep := strings.Index(id, "::")
if sep < 0 {
return "", ""
}
file, symbol := id[:sep], id[sep+2:]
switch kind {
case KindClosure:
// "<enclosing>#closure@<line>" -- the owner is the segment
// before the first '#'.
if h := strings.IndexByte(symbol, '#'); h > 0 {
owner := symbol[:h]
return file + "::" + owner, lastIDSegment(owner)
}
return "", ""
case KindMethod, KindField, KindEnumMember:
// "<Owner>.<member>" -- the owner is everything before the
// last '.'.
if dot := strings.LastIndexByte(symbol, '.'); dot > 0 {
owner := symbol[:dot]
return file + "::" + owner, lastIDSegment(owner)
}
return "", ""
default:
return "", ""
}
}
// lastIDSegment returns the last dotted segment of an identifier --
// its human-facing short name.
func lastIDSegment(s string) string {
if i := strings.LastIndexByte(s, '.'); i >= 0 {
return s[i+1:]
}
return s
}
// EnclosingShortName returns the human-facing short name of an
// owner identifier or node ID -- its last "::"- or "."-separated
// segment. Used when only an owner ID string is in hand and no node
// was resolved.
func EnclosingShortName(s string) string {
if i := strings.LastIndex(s, "::"); i >= 0 {
s = s[i+2:]
}
return lastIDSegment(s)
}
func ValidNodeKind(k NodeKind) bool {
return validNodeKinds[k]
}
+294
View File
@@ -0,0 +1,294 @@
package graph_test
// Node-id stability parity test.
//
// Overlay and cloud paths assume node IDs produced by two different
// indexer invocations of the same source commit are byte-identical.
// If that ever drifts (host-local state in the ID, parse-order leaks,
// RNG, time, etc.), overlay merging silently breaks: the daemon's
// overlay node IDs no longer match the server's base node IDs, edges
// land on dangling endpoints, and queries return half-true answers.
//
// This test runs the live indexer pipeline twice on a freshly-copied
// pair of identical source trees. Different absolute paths simulate
// "two checkouts on one machine" (the cheap proxy for "two machines"
// — the only difference between the two cases is the absolute parent
// directory which the indexer is supposed to strip via repo-prefixing).
import (
"context"
"os"
"path/filepath"
"sort"
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"go.uber.org/zap"
"github.com/zzet/gortex/internal/config"
"github.com/zzet/gortex/internal/graph"
"github.com/zzet/gortex/internal/indexer"
"github.com/zzet/gortex/internal/parser"
"github.com/zzet/gortex/internal/parser/languages"
"github.com/zzet/gortex/internal/search"
)
func newParityRegistry() *parser.Registry {
r := parser.NewRegistry()
r.Register(languages.NewGoExtractor())
return r
}
// fixtureFiles is the source tree planted under each "checkout".
// Mix of:
// - top-level package (Go) — simple types and methods
// - sub-package — exercises path-based ID composition
// - HTTP route via stdlib mux — exercises contract emission
// - import that crosses sub-packages — exercises resolver
//
// Multiple files per package and multiple symbols per file expose any
// parse-order or map-iteration-order leakage in node IDs.
var fixtureFiles = map[string]string{
"go.mod": "module example.com/parity\n\ngo 1.21\n",
"main.go": `package main
import (
"net/http"
"example.com/parity/internal/auth"
)
type Server struct{}
func (s *Server) Start() error {
mux := http.NewServeMux()
mux.HandleFunc("/api/auth/login", auth.LoginHandler)
mux.HandleFunc("/api/health", s.Health)
return http.ListenAndServe(":8080", mux)
}
func (s *Server) Health(w http.ResponseWriter, r *http.Request) {
_, _ = w.Write([]byte("ok"))
}
func main() {
srv := &Server{}
_ = srv.Start()
}
`,
"helpers.go": `package main
import "strings"
func normalize(s string) string { return strings.ToLower(s) }
func reverse(s string) string {
r := []rune(s)
for i, j := 0, len(r)-1; i < j; i, j = i+1, j-1 {
r[i], r[j] = r[j], r[i]
}
return string(r)
}
`,
"internal/auth/login.go": `package auth
import "net/http"
type Credentials struct {
User string
Pass string
}
func LoginHandler(w http.ResponseWriter, r *http.Request) {
_, _ = w.Write([]byte("login"))
}
func Validate(c Credentials) bool {
return c.User != "" && c.Pass != ""
}
`,
"internal/auth/token.go": `package auth
import "time"
type Token struct {
Value string
ExpiresAt time.Time
}
func NewToken(value string) Token {
return Token{Value: value, ExpiresAt: time.Now().Add(time.Hour)}
}
`,
}
// TestNodeIDStability_Parity is the iteration-1 gate test for node-ID
// determinism. Indexes the same source tree from two distinct absolute
// paths (different temp dirs) and asserts the produced node IDs are
// identical sets after stripping the repo prefix. This catches:
//
// - host-local state (working directory, hostname) leaking into IDs
// - parse-order non-determinism (goroutine scheduling) leaking into IDs
// - map-iteration-order leaking into IDs
//
// The repo prefix is allowed to differ because it's deliberately a
// human-readable disambiguator; the rest of the ID after the prefix
// must match byte-for-byte.
//
// For monorepo-shaped IDs the comparison is "same set" — we don't
// require parse-order to be stable, only the final ID set.
func TestNodeIDStability_Parity(t *testing.T) {
idsA := indexFixture(t, "checkout-alpha")
idsB := indexFixture(t, "checkout-beta")
// Strip repo prefix so we're comparing what's structural about the
// ID and not the deliberately-different prefix.
stripped := func(in []string, prefix string) []string {
out := make([]string, 0, len(in))
pre := prefix + "/"
for _, id := range in {
if len(id) > len(pre) && id[:len(pre)] == pre {
out = append(out, id[len(pre):])
continue
}
out = append(out, id)
}
sort.Strings(out)
return out
}
got := stripped(idsA.NodeIDs, idsA.Prefix)
want := stripped(idsB.NodeIDs, idsB.Prefix)
if !assert.Equal(t, want, got, "node IDs must be byte-identical across two indexings of the same source tree (after stripping repo prefix). divergence breaks overlay merging across daemon and cloud.") {
// Surface the first few divergences directly so the failure
// message points at the offending IDs rather than the full
// list-of-thousands diff.
diff := symmetricDifference(got, want)
if len(diff) > 0 {
t.Logf("symmetric difference (up to 20 ids): %v", diff[:minInt(len(diff), 20)])
}
}
}
type fixtureResult struct {
NodeIDs []string
Prefix string
}
// indexFixture writes the fixture into a fresh temp dir under the
// given checkout name, indexes it via MultiIndexer (the warmup path),
// and returns the full set of node IDs in the resulting graph plus
// the repo prefix MultiIndexer assigned.
//
// We use MultiIndexer with two configured repos (the fixture + a
// throwaway sibling) so that willBeMultiRepo is true and the prefix
// path is exercised — that's the production code path the daemon
// runs and the one overlay/cloud merging will rely on.
func indexFixture(t *testing.T, checkoutName string) fixtureResult {
t.Helper()
// Plant the fixture in a unique temp tree.
root := filepath.Join(t.TempDir(), checkoutName)
require.NoError(t, os.MkdirAll(root, 0o755))
for relPath, content := range fixtureFiles {
full := filepath.Join(root, relPath)
require.NoError(t, os.MkdirAll(filepath.Dir(full), 0o755))
require.NoError(t, os.WriteFile(full, []byte(content), 0o644))
}
// Companion repo so multi-repo prefixing kicks in. We keep it
// minimal — no parity comparison runs on it; it only exists to
// flip willBeMultiRepo.
companion := filepath.Join(t.TempDir(), checkoutName+"-companion")
require.NoError(t, os.MkdirAll(companion, 0o755))
require.NoError(t, os.WriteFile(filepath.Join(companion, "noop.go"),
[]byte("package companion\n\nfunc Noop() {}\n"), 0o644))
tmpCfg := filepath.Join(t.TempDir(), "config-"+checkoutName+".yaml")
gc := &config.GlobalConfig{
Repos: []config.RepoEntry{
{Path: root, Name: checkoutName},
{Path: companion, Name: checkoutName + "-companion"},
},
}
gc.SetConfigPath(tmpCfg)
require.NoError(t, gc.Save())
cm, err := config.NewConfigManager(tmpCfg)
require.NoError(t, err)
g := graph.New()
mi := indexer.NewMultiIndexer(g, newParityRegistry(), search.NewBM25(), cm, zap.NewNop())
for _, entry := range cm.Global().Repos {
_, err := mi.TrackRepoCtx(context.Background(), entry)
require.NoError(t, err, "track %s", entry.Name)
}
ids := []string{}
prefix := checkoutName
for _, n := range g.AllNodes() {
// This test is about source-symbol IDs (functions, methods,
// types, files) — the things overlay merging keys on.
// Contract / Module / Builtin nodes are deliberately
// cross-repo singletons (one `dep::foo`, `module::pypi:requests`,
// `builtin::go::len` shared across every repo that uses them)
// and don't carry RepoPrefix; skip them so the parity gate
// stays precise about what it gates. KindFunction nodes
// with meta.external=true are the per-symbol stubs the
// external-call attribution materialises for stdlib/dep
// targets — same rule.
if n.Kind == graph.KindContract || n.Kind == graph.KindModule || n.Kind == graph.KindBuiltin {
continue
}
if ext, _ := n.Meta["external"].(bool); ext {
continue
}
if n.RepoPrefix == "" {
t.Fatalf("node %q has empty RepoPrefix in multi-repo mode", n.ID)
}
if n.RepoPrefix != checkoutName {
continue
}
ids = append(ids, n.ID)
}
require.NotEmpty(t, ids, "no fixture nodes produced — fixture or indexer regression")
return fixtureResult{NodeIDs: ids, Prefix: prefix}
}
// symmetricDifference returns elements present in exactly one of a, b.
// Both must be sorted.
func symmetricDifference(a, b []string) []string {
var diff []string
i, j := 0, 0
for i < len(a) && j < len(b) {
switch {
case a[i] == b[j]:
i++
j++
case a[i] < b[j]:
diff = append(diff, "only-in-A:"+a[i])
i++
default:
diff = append(diff, "only-in-B:"+b[j])
j++
}
}
for ; i < len(a); i++ {
diff = append(diff, "only-in-A:"+a[i])
}
for ; j < len(b); j++ {
diff = append(diff, "only-in-B:"+b[j])
}
return diff
}
func minInt(a, b int) int {
if a < b {
return a
}
return b
}
+27
View File
@@ -0,0 +1,27 @@
package graph
// outEdgesBatcher is implemented by backends that can fetch many nodes'
// out-edges in a single query (the disk-backed stores), collapsing the
// per-node N+1 the single-file resolve path would otherwise issue.
type outEdgesBatcher interface {
GetOutEdgesForNodes(ids []string) map[string][]*Edge
}
// OutEdgesForNodes returns each node's outgoing edges, using the backend's
// batched query when it offers one and falling back to per-node lookups
// otherwise (the in-memory graph, where each lookup is already an O(1) map
// hit). Nodes with no out-edges may be absent from the returned map.
func OutEdgesForNodes(r interface {
GetOutEdges(nodeID string) []*Edge
}, ids []string) map[string][]*Edge {
if b, ok := r.(outEdgesBatcher); ok {
return b.GetOutEdgesForNodes(ids)
}
out := make(map[string][]*Edge, len(ids))
for _, id := range ids {
if e := r.GetOutEdges(id); len(e) > 0 {
out[id] = e
}
}
return out
}
+832
View File
@@ -0,0 +1,832 @@
package graph
import (
"sort"
"strings"
"sync"
)
// OverlayLayer is one MCP session's parsed editor-buffer state. It
// holds the nodes and edges that the overlay introduces (or hides via
// tombstones) on top of an immutable base graph. The layer is built
// once per (session, content-hash) tuple by the MCP overlay middleware
// (`internal/mcp/overlay_view.go::buildOverlayLayer`) and is consulted
// read-only by `OverlaidView`.
//
// **Identity is preserved.** Gortex node IDs are derived from
// `file::symbol` paths, so a symbol that exists in both the on-disk
// and overlay versions of a file ends up with the same ID — the
// view substitutes the overlay's version transparently. New overlay
// symbols (a function the user just typed) get IDs that don't exist
// in base; deleted symbols (removed from the buffer) simply aren't in
// the layer's per-file node list.
//
// The layer is immutable after construction. The middleware never
// mutates it once the View is in flight; the base graph is never
// mutated by overlay flow at all. This is what makes the design
// safe for concurrent multi-session deployments — no shared mutable
// state between sessions or between an overlay-active session and a
// non-overlay session.
type OverlayLayer struct {
// Files covered by the overlay. The key is the file's graph path
// (repo-prefixed in multi-repo mode). Presence in this map means
// "the View should hide base's view of this path" — either to
// replace it with overlay content (entries[path] != nil) or to
// tombstone it (entries[path].Deleted).
entries map[string]*overlayFileEntry
// nodeByID lets GetNode hit a single map lookup. Holds every
// non-tombstoned overlay node across every overlay file.
nodeByID map[string]*Node
// outEdges maps each overlay-introduced source node ID to its
// resolved outgoing edges. Filled by the local resolver pass at
// layer construction.
outEdges map[string][]*Edge
// inEdges is the reverse index of outEdges keyed by target ID,
// so OverlaidView.GetInEdges can merge overlay-originating
// edges with base in-edges in O(1).
inEdges map[string][]*Edge
// nodesByName/Qual index overlay nodes for FindNodesByName /
// GetNodeByQualName fast paths.
nodesByName map[string][]*Node
nodesByQual map[string]*Node
// nameRemoved is the set of (name → IDs from base that are no
// longer present under the View). FindNodesByName uses this to
// filter base hits whose enclosing file is overlaid but whose
// id disappeared from the overlay's node list.
nameRemoved map[string]map[string]bool
}
// overlayFileEntry carries one file's overlay state inside the
// layer. Deleted=true is the tombstone variant — no nodes, no edges.
type overlayFileEntry struct {
Path string
Deleted bool
Nodes []*Node
}
// NewOverlayLayer constructs an empty layer. Callers build it up via
// AddFile / AddNode / AddEdge during the per-request layer-build
// pass, then freeze it by handing it to NewOverlaidView. After that
// point the layer is treated as immutable; the View never writes
// back.
func NewOverlayLayer() *OverlayLayer {
return &OverlayLayer{
entries: make(map[string]*overlayFileEntry),
nodeByID: make(map[string]*Node),
outEdges: make(map[string][]*Edge),
inEdges: make(map[string][]*Edge),
nodesByName: make(map[string][]*Node),
nodesByQual: make(map[string]*Node),
nameRemoved: make(map[string]map[string]bool),
}
}
// MarkFile registers an overlay file. Call once per overlay path
// before AddNode / AddEdge for that file. `deleted` true means the
// path is a tombstone — the View hides base's view of the path
// entirely, returning no nodes from GetFileNodes and treating the
// path's node IDs as non-existent for GetNode.
func (l *OverlayLayer) MarkFile(graphPath string, deleted bool) {
l.entries[graphPath] = &overlayFileEntry{Path: graphPath, Deleted: deleted}
}
// AddNode attaches one parsed overlay node to the layer. Must be
// called after MarkFile for the node's file. Idempotent on (graphPath,
// node ID) — second add silently replaces.
func (l *OverlayLayer) AddNode(graphPath string, n *Node) {
if n == nil {
return
}
entry, ok := l.entries[graphPath]
if !ok {
entry = &overlayFileEntry{Path: graphPath}
l.entries[graphPath] = entry
}
if entry.Deleted {
// Tombstone: silently drop. Caller bug — but cheap to absorb.
return
}
entry.Nodes = append(entry.Nodes, n)
l.nodeByID[n.ID] = n
if n.Name != "" {
l.nodesByName[n.Name] = append(l.nodesByName[n.Name], n)
}
if n.QualName != "" {
l.nodesByQual[n.QualName] = n
}
}
// AddEdge attaches one resolved overlay edge. The local-resolver
// pass at layer construction is expected to have rewritten any
// `unresolved::*` placeholders to point at concrete (overlay or
// base) node IDs before calling this; edges still carrying the
// placeholder are kept verbatim so OverlaidView.GetOutEdges still
// surfaces them — query tools can decide how to handle them, just
// like base's resolver-skipped edges.
func (l *OverlayLayer) AddEdge(e *Edge) {
if e == nil {
return
}
l.outEdges[e.From] = append(l.outEdges[e.From], e)
l.inEdges[e.To] = append(l.inEdges[e.To], e)
}
// MarkRemoved tells the layer that a base node ID is hidden by the
// overlay even though the overlay didn't re-emit it (a symbol the
// user deleted from the buffer). FindNodesByName uses this to filter
// stale base hits.
func (l *OverlayLayer) MarkRemoved(baseName, baseID string) {
if baseName == "" || baseID == "" {
return
}
set, ok := l.nameRemoved[baseName]
if !ok {
set = make(map[string]bool)
l.nameRemoved[baseName] = set
}
set[baseID] = true
}
// HasFile reports whether the overlay covers a particular graph path
// (either with replacement content or as a tombstone). The View uses
// this to decide whether to consult overlay or base for the path's
// reads.
func (l *OverlayLayer) HasFile(graphPath string) bool {
if l == nil {
return false
}
_, ok := l.entries[graphPath]
return ok
}
// IsTombstone reports whether the overlay marks the path as deleted.
func (l *OverlayLayer) IsTombstone(graphPath string) bool {
if l == nil {
return false
}
e := l.entries[graphPath]
return e != nil && e.Deleted
}
// FilePaths returns the sorted list of overlay-covered paths. Used
// by analyzers / the diff tool to enumerate the overlay's footprint.
func (l *OverlayLayer) FilePaths() []string {
if l == nil {
return nil
}
out := make([]string, 0, len(l.entries))
for p := range l.entries {
out = append(out, p)
}
sort.Strings(out)
return out
}
// HasNode reports whether the overlay layer carries a node with this
// ID. Used by the local-resolver pass in the mcp layer to drop base
// hits whose file is overlaid but whose specific ID wasn't kept by
// the overlay (i.e. the user deleted that symbol from the buffer).
func (l *OverlayLayer) HasNode(id string) bool {
if l == nil {
return false
}
_, ok := l.nodeByID[id]
return ok
}
// NodesByName returns the overlay-introduced nodes with the given
// short name. Empty slice when none. Used by the local-resolver
// pass.
func (l *OverlayLayer) NodesByName(name string) []*Node {
if l == nil {
return nil
}
src := l.nodesByName[name]
out := make([]*Node, len(src))
copy(out, src)
return out
}
// OutEdgesByFromAll returns a snapshot of the layer's outgoing-edge
// map keyed by source ID. The resolver pass iterates this to rewrite
// `unresolved::*` placeholders. The returned map shares its slices
// with the layer (resolver mutates Edge.To in place); the map keys
// are stable for the snapshot.
func (l *OverlayLayer) OutEdgesByFromAll() map[string][]*Edge {
if l == nil {
return nil
}
out := make(map[string][]*Edge, len(l.outEdges))
for k, v := range l.outEdges {
out[k] = v
}
return out
}
// RebuildInEdges rebuilds the reverse-index map after the local
// resolver pass mutates Edge.To in place. Cheap: O(#overlay edges).
func (l *OverlayLayer) RebuildInEdges() {
if l == nil {
return
}
l.inEdges = make(map[string][]*Edge, len(l.outEdges))
for _, edges := range l.outEdges {
for _, e := range edges {
l.inEdges[e.To] = append(l.inEdges[e.To], e)
}
}
}
// nodesForFile returns the overlay nodes for a path (empty for
// tombstones). Internal — used by OverlaidView.
func (l *OverlayLayer) nodesForFile(graphPath string) []*Node {
if l == nil {
return nil
}
e := l.entries[graphPath]
if e == nil || e.Deleted {
return nil
}
out := make([]*Node, len(e.Nodes))
copy(out, e.Nodes)
return out
}
// OverlaidView composes an immutable base Reader with a per-session
// overlay layer. Every read path consults the layer first for paths
// the overlay covers; falls through to base otherwise. The base is
// never mutated; the layer is built once per request and discarded
// with the request. This means concurrent sessions — overlay-active
// or not — each see their own consistent view, and the file watcher's
// reindex passes (which mutate base) don't corrupt overlay queries.
type OverlaidView struct {
base Reader
layer *OverlayLayer
// statsOnce caches the (potentially expensive) Stats walk so
// repeated calls within one request don't pay the AllNodes /
// AllEdges cost twice.
statsOnce sync.Once
stats GraphStats
}
// NewOverlaidView builds a view. If layer is nil the view is a pure
// pass-through and consumers pay no overlay overhead.
func NewOverlaidView(base Reader, layer *OverlayLayer) *OverlaidView {
return &OverlaidView{base: base, layer: layer}
}
// Base exposes the underlying base reader. The diff tool reads
// against (view.Base()) and against (view) directly to compute the
// delta induced by the overlay.
func (v *OverlaidView) Base() Reader { return v.base }
// Layer exposes the per-session overlay layer (nil when none).
// Diagnostic / debug tools use it to introspect what the overlay
// covers.
func (v *OverlaidView) Layer() *OverlayLayer { return v.layer }
// IDFile returns the file path encoded in a Gortex node ID, or "" if
// the id isn't file-anchored. Gortex IDs follow the pattern
// `<filepath>::<symbol>[.member][#param:name]` so the file prefix is
// the substring before the first `::`. Module / package / virtual
// nodes use other prefixes that won't match an overlay path.
func IDFile(id string) string {
if id == "" {
return ""
}
if i := strings.Index(id, "::"); i > 0 {
return id[:i]
}
return ""
}
// nodeBelongsToOverlay reports whether an ID's file is covered by
// the layer.
func (v *OverlaidView) nodeBelongsToOverlay(id string) bool {
if v.layer == nil {
return false
}
return v.layer.HasFile(IDFile(id))
}
// GetNode returns the overlay's version of a node when the ID
// belongs to an overlaid file, the base node otherwise. Returns nil
// when the symbol exists in base but was removed in the overlay
// (the per-file overlay node list didn't include it).
func (v *OverlaidView) GetNode(id string) *Node {
if v.layer != nil {
if v.nodeBelongsToOverlay(id) {
return v.layer.nodeByID[id] // may be nil — overlay deleted it
}
}
if v.base == nil {
return nil
}
return v.base.GetNode(id)
}
// GetNodesByIDs returns the overlay-aware *Node for each input ID.
// Overlay-owned IDs short-circuit to the per-session layer (and may
// resolve to nil when the overlay deleted the node); the remainder
// fans out as a single batched lookup against the base store. Missing
// IDs are simply absent from the returned map.
func (v *OverlaidView) GetNodesByIDs(ids []string) map[string]*Node {
if len(ids) == 0 {
return nil
}
out := make(map[string]*Node, len(ids))
baseIDs := ids[:0:0] // fresh backing array — never aliases caller's slice
for _, id := range ids {
if id == "" {
continue
}
if _, dup := out[id]; dup {
continue
}
if v.layer != nil && v.nodeBelongsToOverlay(id) {
if n := v.layer.nodeByID[id]; n != nil {
out[id] = n
}
// Overlay tombstone — ID is hidden, do not fall back to base.
continue
}
// Track for the single base round-trip; reserve a slot in `out`
// only after the batched lookup returns.
baseIDs = append(baseIDs, id)
}
if len(baseIDs) > 0 && v.base != nil {
for id, n := range v.base.GetNodesByIDs(baseIDs) {
if n != nil {
out[id] = n
}
}
}
return out
}
// GetNodeByQualName: overlay first, then base. Base hits are filtered
// to drop entries whose file is overlaid (the overlay's view wins).
func (v *OverlaidView) GetNodeByQualName(qualName string) *Node {
if v.layer != nil {
if n := v.layer.nodesByQual[qualName]; n != nil {
return n
}
}
if v.base == nil {
return nil
}
n := v.base.GetNodeByQualName(qualName)
if n != nil && v.layer != nil && v.layer.HasFile(IDFile(n.ID)) {
// Base hit landed in an overlaid file but the overlay didn't
// re-emit a node with this qualified name → it's gone.
return nil
}
return n
}
// GetNodesByQualNames resolves each name through GetNodeByQualName so the
// overlay's layer-first / shadowed-file filtering applies — an inherited
// base batch would bypass the overlay. Per-name is fine: an interactive
// overlay's working set is small (the batch form exists for the
// cold-warmup scale on the base store, not here). Returns only hits.
func (v *OverlaidView) GetNodesByQualNames(qualNames []string) map[string]*Node {
out := make(map[string]*Node, len(qualNames))
for _, q := range qualNames {
if q == "" {
continue
}
if _, done := out[q]; done {
continue
}
if n := v.GetNodeByQualName(q); n != nil {
out[q] = n
}
}
return out
}
// FindNodesByName merges base hits (filtered to drop nodes in
// overlaid files unless the overlay re-emitted them) with overlay
// hits. Order is overlay-first, then base — callers that picked
// "first match" semantics get the overlay version automatically.
func (v *OverlaidView) FindNodesByName(name string) []*Node {
var out []*Node
if v.layer != nil {
out = append(out, v.layer.nodesByName[name]...)
}
if v.base == nil {
return out
}
for _, n := range v.base.FindNodesByName(name) {
if v.layer != nil {
if v.layer.HasFile(IDFile(n.ID)) {
// Overlaid file: base's node for this name is
// always hidden. If the overlay re-emitted the same
// ID it's already in `out` from the layer's
// nodesByName prepend above; if the overlay deleted
// the symbol it must not surface at all. Either way
// we skip — no need to discriminate.
continue
}
if v.layer.nameRemoved[name] != nil && v.layer.nameRemoved[name][n.ID] {
continue
}
}
out = append(out, n)
}
return out
}
// FindNodesByNameContaining merges overlay-touched name hits with the
// base result, then re-applies the per-overlay-file masking the same
// way FindNodesByName does. Order is overlay-first, then base; the
// limit caps the merged total. Empty substr or both layers nil
// returns nil.
func (v *OverlaidView) FindNodesByNameContaining(substr string, limit int) []*Node {
if substr == "" {
return nil
}
needle := strings.ToLower(substr)
var out []*Node
// Overlay-side: walk the layer's nodesByName index — the same
// bucket FindNodesByName reads from — and accept any name whose
// lowercase form contains the needle.
if v.layer != nil {
for name, bucket := range v.layer.nodesByName {
if strings.Contains(strings.ToLower(name), needle) {
out = append(out, bucket...)
if limit > 0 && len(out) >= limit {
return out[:limit]
}
}
}
}
if v.base == nil {
return out
}
// Base-side: fetch with an inflated limit so overlay-mask drops
// don't leave a short page. Then re-apply the same overlaid-file
// + name-removed mask FindNodesByName uses.
fetch := limit
if fetch > 0 {
fetch *= 2
}
for _, n := range v.base.FindNodesByNameContaining(substr, fetch) {
if v.layer != nil {
if v.layer.HasFile(IDFile(n.ID)) {
continue
}
if v.layer.nameRemoved[n.Name] != nil && v.layer.nameRemoved[n.Name][n.ID] {
continue
}
}
out = append(out, n)
if limit > 0 && len(out) >= limit {
return out[:limit]
}
}
if limit > 0 && len(out) > limit {
out = out[:limit]
}
return out
}
// GetFileNodes: if the path is overlaid, return overlay's nodes
// (empty for tombstones). Otherwise pass through to base.
func (v *OverlaidView) GetFileNodes(filePath string) []*Node {
if v.layer != nil && v.layer.HasFile(filePath) {
return v.layer.nodesForFile(filePath)
}
if v.base == nil {
return nil
}
return v.base.GetFileNodes(filePath)
}
// GetRepoNodes filters base's per-repo node list by dropping nodes
// whose file is overlaid (unless the overlay re-emitted them) and
// appending the overlay's nodes for any overlaid file inside the
// requested repo prefix.
func (v *OverlaidView) GetRepoNodes(repoPrefix string) []*Node {
if v.base == nil {
return nil
}
baseNodes := v.base.GetRepoNodes(repoPrefix)
if v.layer == nil {
return baseNodes
}
out := make([]*Node, 0, len(baseNodes))
for _, n := range baseNodes {
if v.layer.HasFile(IDFile(n.ID)) {
// File is overlaid. Surface only if the overlay
// re-emitted this exact ID; otherwise it's hidden.
if v.layer.nodeByID[n.ID] == nil {
continue
}
}
out = append(out, n)
}
for _, path := range v.layer.FilePaths() {
if !strings.HasPrefix(path, repoPrefix+"/") && path != repoPrefix {
continue
}
out = append(out, v.layer.nodesForFile(path)...)
}
return out
}
// GetOutEdges: when the source node's file is overlaid, use the
// overlay's resolved out-edges. Otherwise return base's edges but
// drop any whose target points into an overlaid file at a node ID
// the overlay no longer carries (target deleted in buffer).
func (v *OverlaidView) GetOutEdges(nodeID string) []*Edge {
if v.layer != nil && v.nodeBelongsToOverlay(nodeID) {
src := v.layer.outEdges[nodeID]
out := make([]*Edge, len(src))
copy(out, src)
return out
}
if v.base == nil {
return nil
}
edges := v.base.GetOutEdges(nodeID)
if v.layer == nil {
return edges
}
out := edges[:0:0]
for _, e := range edges {
if v.layer.HasFile(IDFile(e.To)) {
if v.layer.nodeByID[e.To] == nil {
continue // target deleted in overlay
}
}
out = append(out, e)
}
return out
}
// GetInEdges merges base's incoming edges (filtered to drop those
// originating in overlaid files, since those are replaced by overlay
// versions) with the overlay's in-edges for the same target.
func (v *OverlaidView) GetInEdges(nodeID string) []*Edge {
if v.layer == nil {
if v.base == nil {
return nil
}
return v.base.GetInEdges(nodeID)
}
var out []*Edge
if v.base != nil {
for _, e := range v.base.GetInEdges(nodeID) {
if v.layer.HasFile(IDFile(e.From)) {
// Source is overlaid — the overlay's version of this
// edge wins (or the overlay simply deleted the call).
continue
}
if v.layer.HasFile(IDFile(e.To)) && v.layer.nodeByID[e.To] == nil {
// Target was deleted by the overlay.
continue
}
out = append(out, e)
}
}
out = append(out, v.layer.inEdges[nodeID]...)
return out
}
// GetOutEdgesByNodeIDs returns the overlay-aware outgoing-edge map for
// every input id. Overlay-owned ids short-circuit to the per-session
// layer; the remainder fans out as a single batched lookup against
// the base store. Output mirrors GetOutEdges's per-id semantics
// (target-side overlay deletions filtered out), but in one cgo
// round-trip per direction instead of N.
func (v *OverlaidView) GetOutEdgesByNodeIDs(ids []string) map[string][]*Edge {
if len(ids) == 0 {
return nil
}
out := make(map[string][]*Edge, len(ids))
baseIDs := ids[:0:0]
seen := make(map[string]struct{}, len(ids))
for _, id := range ids {
if id == "" {
continue
}
if _, dup := seen[id]; dup {
continue
}
seen[id] = struct{}{}
if v.layer != nil && v.nodeBelongsToOverlay(id) {
src := v.layer.outEdges[id]
cp := make([]*Edge, len(src))
copy(cp, src)
out[id] = cp
continue
}
baseIDs = append(baseIDs, id)
}
if len(baseIDs) > 0 && v.base != nil {
base := v.base.GetOutEdgesByNodeIDs(baseIDs)
for id, edges := range base {
if v.layer == nil {
out[id] = edges
continue
}
filtered := edges[:0:0]
for _, e := range edges {
if v.layer.HasFile(IDFile(e.To)) {
if v.layer.nodeByID[e.To] == nil {
continue // target deleted in overlay
}
}
filtered = append(filtered, e)
}
out[id] = filtered
}
}
return out
}
// GetInEdgesByNodeIDs is the inbound sibling of GetOutEdgesByNodeIDs.
// Merges base in-edges (filtered to drop edges sourced in overlaid
// files) with overlay-introduced in-edges for each input id, all in a
// single batched base round-trip.
func (v *OverlaidView) GetInEdgesByNodeIDs(ids []string) map[string][]*Edge {
if len(ids) == 0 {
return nil
}
out := make(map[string][]*Edge, len(ids))
seen := make(map[string]struct{}, len(ids))
uniq := ids[:0:0]
for _, id := range ids {
if id == "" {
continue
}
if _, dup := seen[id]; dup {
continue
}
seen[id] = struct{}{}
uniq = append(uniq, id)
}
if len(uniq) == 0 {
return out
}
if v.base != nil {
base := v.base.GetInEdgesByNodeIDs(uniq)
for _, id := range uniq {
edges := base[id]
if v.layer == nil {
out[id] = edges
continue
}
filtered := edges[:0:0]
for _, e := range edges {
if v.layer.HasFile(IDFile(e.From)) {
continue // source is overlaid — overlay's version wins
}
if v.layer.HasFile(IDFile(e.To)) && v.layer.nodeByID[e.To] == nil {
continue // target was deleted by overlay
}
filtered = append(filtered, e)
}
out[id] = filtered
}
}
if v.layer != nil {
for _, id := range uniq {
if extras := v.layer.inEdges[id]; len(extras) > 0 {
out[id] = append(out[id], extras...)
}
}
}
return out
}
// AllNodes returns base's nodes minus nodes in overlaid files, plus
// every node the overlay introduced. Bulk-read consumers (analyzers,
// search reindex, snapshot export) get an overlay-consistent view
// without paying any extra copy beyond the base snapshot's.
func (v *OverlaidView) AllNodes() []*Node {
if v.base == nil {
return nil
}
baseNodes := v.base.AllNodes()
if v.layer == nil {
return baseNodes
}
out := make([]*Node, 0, len(baseNodes))
for _, n := range baseNodes {
if v.layer.HasFile(IDFile(n.ID)) {
if v.layer.nodeByID[n.ID] == nil {
continue
}
// Else: overlay's version was kept under the same ID; the
// layer's slice will include it below, so skip base's copy
// to avoid duplicates.
continue
}
out = append(out, n)
}
for _, n := range v.layer.nodeByID {
out = append(out, n)
}
return out
}
// AllEdges returns base's edges minus those involving overlaid
// files, plus every overlay-introduced edge.
func (v *OverlaidView) AllEdges() []*Edge {
if v.base == nil {
return nil
}
baseEdges := v.base.AllEdges()
if v.layer == nil {
return baseEdges
}
out := make([]*Edge, 0, len(baseEdges))
for _, e := range baseEdges {
if v.layer.HasFile(IDFile(e.From)) || v.layer.HasFile(IDFile(e.To)) {
continue
}
out = append(out, e)
}
for _, edges := range v.layer.outEdges {
out = append(out, edges...)
}
return out
}
// NodeCount / EdgeCount — derived from base counters adjusted by the
// overlay delta. Cheap enough to recompute per call.
func (v *OverlaidView) NodeCount() int {
if v.base == nil {
return 0
}
if v.layer == nil {
return v.base.NodeCount()
}
delta := 0
for path, entry := range v.layer.entries {
baseCount := len(v.base.GetFileNodes(path))
if entry.Deleted {
delta -= baseCount
continue
}
delta += len(entry.Nodes) - baseCount
}
return v.base.NodeCount() + delta
}
func (v *OverlaidView) EdgeCount() int {
if v.base == nil {
return 0
}
if v.layer == nil {
return v.base.EdgeCount()
}
return len(v.AllEdges())
}
// EdgeIdentityRevisions delegates to the base graph: provenance churn
// is a property of the persistent graph, and an overlay layer is a
// non-mutating per-session shadow that never upgrades edge provenance.
func (v *OverlaidView) EdgeIdentityRevisions() int {
if v.base == nil {
return 0
}
return v.base.EdgeIdentityRevisions()
}
// Stats is best-effort under overlay: we report base's stats (the
// analyzer-shaped GraphStats requires per-kind / per-language
// breakdowns that the overlay layer doesn't expose cheaply). Caching
// keeps repeated Stats() calls inside one request to a single base
// lookup.
func (v *OverlaidView) Stats() GraphStats {
if v.base == nil {
return GraphStats{}
}
v.statsOnce.Do(func() {
v.stats = v.base.Stats()
})
return v.stats
}
// RepoStats — same conservatism as Stats; overlay deltas are
// excluded. The handful of tools that read RepoStats are bookkeeping
// rather than load-bearing, and the overlay-affected nodes are still
// reachable through the per-node read paths.
func (v *OverlaidView) RepoStats() map[string]GraphStats {
if v.base == nil {
return nil
}
return v.base.RepoStats()
}
// Compile-time assertion that *OverlaidView satisfies Reader.
var _ Reader = (*OverlaidView)(nil)
+402
View File
@@ -0,0 +1,402 @@
package graph
import (
"go/ast"
"go/parser"
"go/token"
"io/fs"
"os"
"path/filepath"
"regexp"
"sort"
"strings"
"testing"
)
// Schema-vs-extractor parity audit.
//
// The graph package declares ~30 NodeKinds and ~55 EdgeKinds. Without
// a regression fence, the schema drifts: a new kind gets added with a
// detailed comment, but nobody wires the emitter, and the constant
// quietly outlives its purpose. We've seen this exact failure mode
// before — `KindRelease` was declared and documented for many releases
// before any extractor actually instantiated it.
//
// TestSchemaParityAudit walks every Go source file under the repo
// (excluding worktrees, vendor, build artifacts) and asserts that
// every declared NodeKind and EdgeKind constant is referenced from
// at least one file outside the declaring file (node.go / edge.go).
//
// "Referenced" here means the bare identifier appears in production
// source — the resolver, an extractor, an enricher, an analyzer, or
// a downstream consumer. Test-only references count too: a test that
// exercises the consumer is still proof the kind has a real role.
//
// When the audit fails the failure message names the orphan constant
// and points at the file where it was declared so the next sweep
// either wires the emitter or removes the constant. Reserved kinds
// (declared today, emitter coming next sprint) go on the explicit
// allowlist with a documented reason — that turns "we forgot" into
// "we deliberately deferred."
func TestSchemaParityAudit(t *testing.T) {
repoRoot, err := findRepoRoot()
if err != nil {
t.Fatalf("locate repo root: %v", err)
}
declared, err := extractKindDeclarations(filepath.Join(repoRoot, "internal", "graph"))
if err != nil {
t.Fatalf("scan declarations: %v", err)
}
if len(declared) == 0 {
t.Fatal("scan returned zero declarations — audit is broken")
}
usage, err := scanKindUsage(repoRoot, declared)
if err != nil {
t.Fatalf("scan usage: %v", err)
}
var orphans []string
for _, d := range declared {
if reservedKinds[d.Name] {
continue
}
// "Direct" usage means we found the constant referenced
// anywhere outside its own declaration file. Mapping-table
// references inside graph/edge.go (CrossRepoKindFor etc.)
// count because they prove the constant is reachable from
// production query paths — the actual emit site can then sit
// in a resolver or enricher that calls the mapping fn.
if usage[d.Name] == 0 {
orphans = append(orphans, formatOrphan(d))
}
}
if len(orphans) > 0 {
sort.Strings(orphans)
t.Errorf("schema-parity audit found %d orphan kind(s) — wire emitter or remove from schema:\n\n%s\n\n"+
"Add to `reservedKinds` in this file with a documented reason if the gap is intentional.",
len(orphans), strings.Join(orphans, "\n"))
}
}
// reservedKinds is the explicit allowlist of constants that are
// declared today and intentionally lack a production emit site. Every
// entry must carry a comment explaining the reason — "we forgot" is
// not a valid reason; the answer is to either ship the emitter or
// delete the declaration.
var reservedKinds = map[string]bool{
// (empty — every declared kind currently has a production emit
// site. Future deferrals go here with a documented reason.)
}
// kindDecl is a single NodeKind / EdgeKind constant declaration we
// extracted from internal/graph. The Path / Line locate it for the
// failure message so the next sweep knows where to look.
type kindDecl struct {
Name string
Kind string // "NodeKind" or "EdgeKind"
Path string
Line int
}
// extractKindDeclarations parses every .go file under the given
// directory and returns each `const Name SomeKind = ...` whose RHS
// type is NodeKind or EdgeKind. Anything else is ignored — the audit
// is intentionally scoped to schema kinds.
func extractKindDeclarations(dir string) ([]kindDecl, error) {
var out []kindDecl
fset := token.NewFileSet()
err := filepath.WalkDir(dir, func(path string, d fs.DirEntry, err error) error {
if err != nil {
return err
}
if d.IsDir() {
return nil
}
if !strings.HasSuffix(path, ".go") || strings.HasSuffix(path, "_test.go") {
return nil
}
file, err := parser.ParseFile(fset, path, nil, parser.ParseComments)
if err != nil {
return err
}
for _, decl := range file.Decls {
gen, ok := decl.(*ast.GenDecl)
if !ok || gen.Tok != token.CONST {
continue
}
for _, spec := range gen.Specs {
vs, ok := spec.(*ast.ValueSpec)
if !ok {
continue
}
typeName, ok := simpleTypeName(vs.Type)
if !ok {
continue
}
if typeName != "NodeKind" && typeName != "EdgeKind" {
continue
}
for _, ident := range vs.Names {
pos := fset.Position(ident.NamePos)
out = append(out, kindDecl{
Name: ident.Name,
Kind: typeName,
Path: path,
Line: pos.Line,
})
}
}
}
return nil
})
if err != nil {
return nil, err
}
return out, nil
}
// simpleTypeName pulls the identifier name out of a simple type
// expression. Returns false for any expression that isn't a bare
// identifier — qualified names, function types, etc. — because the
// audit only cares about the local NodeKind / EdgeKind types.
func simpleTypeName(expr ast.Expr) (string, bool) {
id, ok := expr.(*ast.Ident)
if !ok {
return "", false
}
return id.Name, true
}
// scanKindUsage counts how many references to each declared constant
// exist outside its declaring file. The scanner is regex-based
// (rather than AST-based) because the audit needs to catch every
// reachable mention — including string-literal mappings and test
// fixtures — without paying the parse cost of every .go file in the
// repo. Word-boundary anchors keep "EdgeReads" and "EdgeReadsCol"
// (and "EdgeReadsConfig") distinct.
func scanKindUsage(repoRoot string, declared []kindDecl) (map[string]int, error) {
declaringFiles := make(map[string]string, len(declared))
for _, d := range declared {
declaringFiles[d.Name] = d.Path
}
// One combined alternation rather than one regex per kind. Matching
// every .go file in the repo against ~100 separate \bName\b patterns
// is O(files × kinds) and grows past the CI timeout as both the tree
// and the schema grow; a single \b(?:Name1|Name2|…)\b scanned once per
// file is O(files). The \b anchors keep the identifiers as distinct as
// before — `EdgeReads` still does NOT match inside `EdgeReadsCol`,
// because the inner \b fails between two word characters and the
// engine falls through to the longer alternative.
names := make([]string, 0, len(declared))
seen := make(map[string]bool, len(declared))
for _, d := range declared {
if seen[d.Name] {
continue
}
seen[d.Name] = true
names = append(names, regexp.QuoteMeta(d.Name))
}
re := regexp.MustCompile(`\b(?:` + strings.Join(names, "|") + `)\b`)
usage := make(map[string]int, len(declared))
err := filepath.WalkDir(repoRoot, func(path string, d fs.DirEntry, err error) error {
if err != nil {
return err
}
if d.IsDir() {
base := d.Name()
// Skip vendor, build caches, worktrees, eval venvs,
// transcript dumps — none of those carry production
// emit sites.
switch base {
case ".git", "vendor", "node_modules", ".claude", ".cache",
".venv", "venv", "build", "dist", "debug":
return filepath.SkipDir
}
return nil
}
if !strings.HasSuffix(path, ".go") {
return nil
}
// Worktrees can hide a duplicate copy of the source tree
// under .claude/worktrees/* — already skipped by the .claude
// directory rule above; defensive double-check.
if strings.Contains(path, "/worktrees/") {
return nil
}
// Don't let the audit's own file count as a reference — it
// names every constant in its failure-message format so a
// naive regex would mark every kind as "used."
if strings.HasSuffix(path, "/parity_audit_test.go") {
return nil
}
data, err := os.ReadFile(path)
if err != nil {
return nil // unreadable files don't fail the audit
}
text := stripGoComments(string(data))
found := make(map[string]bool)
for _, m := range re.FindAllString(text, -1) {
found[m] = true
}
for name := range found {
// A constant referenced only inside its own declaring file
// is not a real consumer — exclude it exactly as the
// per-pattern scan did.
if path == declaringFiles[name] {
continue
}
usage[name]++
}
return nil
})
if err != nil {
return nil, err
}
return usage, nil
}
// findRepoRoot walks up from the test's working directory looking
// for go.mod. Lets the audit run from any package depth — the test
// stays at internal/graph but the scan needs the repo root.
func findRepoRoot() (string, error) {
dir, err := os.Getwd()
if err != nil {
return "", err
}
for {
if _, err := os.Stat(filepath.Join(dir, "go.mod")); err == nil {
return dir, nil
}
parent := filepath.Dir(dir)
if parent == dir {
return "", os.ErrNotExist
}
dir = parent
}
}
func formatOrphan(d kindDecl) string {
rel := d.Path
if root, err := findRepoRoot(); err == nil {
if r, err := filepath.Rel(root, d.Path); err == nil {
rel = r
}
}
return " " + d.Kind + "." + d.Name + " (declared at " + rel + ":" + itoa(d.Line) + ")"
}
// stripGoComments removes line and block comments from Go source so
// the usage scan only counts identifiers that actually appear in
// executable code. Without this a constant mentioned in a docstring
// elsewhere in the tree would falsely satisfy the parity audit.
//
// The implementation is intentionally simple — character-level scan
// with a small state machine — because the alternative (full Go
// parsing) is overkill for the audit's needs and slow when invoked
// across every .go file in the repo. Strings (single- and
// double-quoted, plus raw backtick) are preserved verbatim so a
// string literal containing "//" isn't mistaken for a comment.
func stripGoComments(src string) string {
var b strings.Builder
b.Grow(len(src))
const (
stateCode = iota
stateLineComment
stateBlockComment
stateString
stateRawString
stateRune
)
state := stateCode
for i := 0; i < len(src); i++ {
c := src[i]
switch state {
case stateCode:
switch {
case c == '/' && i+1 < len(src) && src[i+1] == '/':
state = stateLineComment
i++
b.WriteByte(' ')
case c == '/' && i+1 < len(src) && src[i+1] == '*':
state = stateBlockComment
i++
b.WriteByte(' ')
case c == '"':
state = stateString
b.WriteByte(c)
case c == '`':
state = stateRawString
b.WriteByte(c)
case c == '\'':
state = stateRune
b.WriteByte(c)
default:
b.WriteByte(c)
}
case stateLineComment:
if c == '\n' {
state = stateCode
b.WriteByte(c)
}
case stateBlockComment:
if c == '*' && i+1 < len(src) && src[i+1] == '/' {
state = stateCode
i++
b.WriteByte(' ')
}
case stateString:
b.WriteByte(c)
if c == '\\' && i+1 < len(src) {
b.WriteByte(src[i+1])
i++
} else if c == '"' {
state = stateCode
}
case stateRawString:
b.WriteByte(c)
if c == '`' {
state = stateCode
}
case stateRune:
b.WriteByte(c)
if c == '\\' && i+1 < len(src) {
b.WriteByte(src[i+1])
i++
} else if c == '\'' {
state = stateCode
}
}
}
return b.String()
}
func itoa(n int) string {
// Tiny helper so the audit file doesn't need strconv just for one
// formatter. Keeps the import set minimal — easier to read at a
// glance.
if n == 0 {
return "0"
}
neg := n < 0
if neg {
n = -n
}
var buf [20]byte
i := len(buf)
for n > 0 {
i--
buf[i] = byte('0' + n%10)
n /= 10
}
if neg {
i--
buf[i] = '-'
}
return string(buf[i:])
}
+47
View File
@@ -0,0 +1,47 @@
package graph
import "testing"
func TestProvenanceWeight_Tiers(t *testing.T) {
w := func(origin string) float64 {
return ProvenanceWeight(&Edge{From: "a", To: "b", Kind: EdgeCalls, Origin: origin})
}
cases := []struct {
origin string
want float64
}{
{OriginLSPResolved, provWeightLSP},
{OriginLSPDispatch, provWeightLSP},
{OriginASTResolved, ProvenanceWeightMax},
{OriginASTInferred, provWeightASTInferred},
{OriginTextMatched, ProvenanceWeightMin},
}
for _, c := range cases {
if got := w(c.origin); got != c.want {
t.Errorf("ProvenanceWeight(origin=%q) = %v, want %v", c.origin, got, c.want)
}
}
// LSP and text tiers are attenuated below the ast_resolved baseline.
if !(w(OriginLSPResolved) < ProvenanceWeightMax) {
t.Errorf("lsp tier must weight below ast_resolved baseline")
}
if !(w(OriginTextMatched) < ProvenanceWeightMax) {
t.Errorf("text tier must weight below ast_resolved baseline")
}
}
func TestProvenanceWeight_BackfillsAndNilSafe(t *testing.T) {
// A nil edge weights at the trusted baseline.
if got := ProvenanceWeight(nil); got != ProvenanceWeightMax {
t.Errorf("nil edge = %v, want %v", got, ProvenanceWeightMax)
}
// Unset Origin backfills via DefaultOriginFor: a structural import
// edge is ast_resolved (baseline); a bare call edge with no
// confidence falls to text_matched (the minimum).
if got := ProvenanceWeight(&Edge{Kind: EdgeImports}); got != ProvenanceWeightMax {
t.Errorf("structural import edge = %v, want %v", got, ProvenanceWeightMax)
}
if got := ProvenanceWeight(&Edge{Kind: EdgeCalls}); got != ProvenanceWeightMin {
t.Errorf("bare unresolved call edge = %v, want %v", got, ProvenanceWeightMin)
}
}
+83
View File
@@ -0,0 +1,83 @@
package graph
// Reader is the read-only contract every graph consumer (query
// engine, MCP tool handlers, analyzers, resolver introspection) depends
// on. *Graph satisfies it directly; OverlaidView (overlay.go) wraps a
// base Reader plus a per-session overlay layer to deliver a non-
// mutating shadow view for editor-buffer queries.
//
// Mutation methods (AddNode, AddEdge, EvictFile, ReindexEdge, …) live
// on *Graph and are NOT part of this interface. Only the indexer and
// the resolver mutate; everyone else reads, and reads must go through
// Reader so the same call site transparently switches between base
// and overlay views.
//
// New read methods on *Graph should be added here too — keeping the
// surfaces in sync is what guarantees that a tool migrated to read
// through the Reader will keep working for both base and overlay
// queries.
type Reader interface {
// Identity lookups.
GetNode(id string) *Node
GetNodeByQualName(qualName string) *Node
FindNodesByName(name string) []*Node
// FindNodesByNameContaining returns nodes whose Name (case-
// insensitive) contains substr. The filter is pushed into the
// backend so only matching rows cross the boundary on a disk backend;
// the search hot path's substring fallback uses this instead of
// the old AllNodes()-then-filter pattern (which materialised the
// whole node set per call and didn't scale). limit caps the
// result; 0 means "no limit".
FindNodesByNameContaining(substr string, limit int) []*Node
// GetNodesByIDs is the batched sibling of GetNode. The disk-backed
// store collapses N individual point lookups into a
// single bulk query — critical on the search hot path where one
// query materialises 60+ candidate IDs. The in-memory backend
// forwards to per-id GetNode, so the cost matches an inline loop
// there. Missing IDs are simply absent from the map (no nil
// values); duplicates dedupe naturally.
GetNodesByIDs(ids []string) map[string]*Node
// File / repo scopes.
GetFileNodes(filePath string) []*Node
GetRepoNodes(repoPrefix string) []*Node
// Edge walks.
GetOutEdges(nodeID string) []*Edge
GetInEdges(nodeID string) []*Edge
// GetInEdgesByNodeIDs / GetOutEdgesByNodeIDs are the batched
// siblings of GetInEdges / GetOutEdges. The disk-backed store collapses
// N per-id queries into one bulk query over an `id IN $ids`
// filter; the in-memory backend forwards to per-id walks (no
// concurrency win — same algorithmic cost as an inline loop). On
// the rerank hot path this drops ~150 round-trips per
// search_symbols call down to ~4 (prepare collects every
// candidate's ids and fans them out in one inbound + one outbound
// batch). Missing nodes get nil slices in the returned map so
// callers can `for _, e := range m[id]` without an ok-check.
GetInEdgesByNodeIDs(ids []string) map[string][]*Edge
GetOutEdgesByNodeIDs(ids []string) map[string][]*Edge
// Bulk reads — used by analyzers (hotspots, cycles, dead code,
// communities, …) and by the embedded query engine's whole-graph
// passes.
AllNodes() []*Node
AllEdges() []*Edge
// Counters & stats.
NodeCount() int
EdgeCount() int
// EdgeIdentityRevisions is the running count of provenance-bearing
// edge-identity changes — see Graph.EdgeIdentityRevisions.
EdgeIdentityRevisions() int
Stats() GraphStats
RepoStats() map[string]GraphStats
}
// Compile-time assertion that *Graph satisfies Reader. If a new
// Reader method is added without a corresponding *Graph method (or
// the *Graph signature drifts), the build breaks here rather than at
// a far-away callsite.
var _ Reader = (*Graph)(nil)
+41
View File
@@ -0,0 +1,41 @@
package graph
import "testing"
func TestRefContextOf(t *testing.T) {
cases := []struct {
name string
kind EdgeKind
from NodeKind
meta map[string]any
want string
}{
{"return", EdgeReturns, KindFunction, nil, RefContextReturnType},
{"param", EdgeTypedAs, KindParam, nil, RefContextParameterType},
{"field", EdgeTypedAs, KindField, nil, RefContextField},
{"value_var", EdgeTypedAs, KindVariable, nil, RefContextValue},
{"value_local", EdgeTypedAs, KindLocal, nil, RefContextValue},
{"typed_other", EdgeTypedAs, KindType, nil, RefContextType},
{"attribute", EdgeAnnotated, KindFunction, nil, RefContextAttribute},
{"type_ref", EdgeReferences, KindFunction, nil, RefContextType},
{"implements", EdgeImplements, KindType, nil, RefContextType},
{"read", EdgeReads, KindFunction, nil, RefContextValue},
{"call", EdgeCalls, KindFunction, nil, RefContextCall},
{"instantiate", EdgeInstantiates, KindFunction, nil, RefContextCall},
{"meta_override", EdgeReferences, KindFunction, map[string]any{"ref_context": RefContextGenericArg}, RefContextGenericArg},
{"import", EdgeImports, KindFile, nil, RefContextImport},
{"re_export", EdgeReExports, KindFile, nil, RefContextImport},
{"no_context", EdgeContains, KindFile, nil, ""},
}
for _, c := range cases {
t.Run(c.name, func(t *testing.T) {
e := &Edge{Kind: c.kind, Meta: c.meta}
if got := RefContextOf(e, c.from); got != c.want {
t.Errorf("RefContextOf(%s, from=%s) = %q, want %q", c.kind, c.from, got, c.want)
}
})
}
if RefContextOf(nil, KindFunction) != "" {
t.Error("nil edge must classify as empty context")
}
}
+21
View File
@@ -0,0 +1,21 @@
package graph
import "strings"
// RepoPrefixOfID returns the repository prefix encoded in a node ID or
// file path — the leading path segment before the first "/". In
// multi-repo mode every node ID and file path is prefixed with the repo
// it belongs to (e.g. "gortex/internal/mcp/server.go::NewServer" →
// "gortex"), so this recovers the owning repo without a node lookup.
//
// Returns "" for IDs that carry no repo prefix: unresolved sentinels
// like "unresolved::Name", and single-repo-mode IDs that were never
// prefixed. Callers use it only inside a workspace-bound (multi-repo)
// path, where every real node is prefixed, so an empty result simply
// never matches a workspace's repo set.
func RepoPrefixOfID(id string) string {
if i := strings.IndexByte(id, '/'); i > 0 {
return id[:i]
}
return ""
}
+81
View File
@@ -0,0 +1,81 @@
package graph
// Restub provenance handoff.
//
// When a file is re-parsed, its symbols are evicted and every incoming
// reference edge is restubbed to an `unresolved::<name>` target so the
// incoming-resolve pass can rebind it once the symbols come back. The edge
// keeps its identity, but its resolved provenance (origin / tier / confidence)
// is now a lie: it points at an unresolved stub. Two failure modes follow if
// the provenance is left untouched — a genuinely deleted definition leaves a
// stub advertising a compiler-grade tier, and a rebind to a *different* target
// inherits the old tier it never earned.
//
// StashRestubProvenance clears the live provenance and records it in the edge's
// Meta; RestoreRestubProvenance re-applies it only when the stub rebinds to the
// exact same target (an idempotent re-parse: unchanged binding, unchanged
// tier), and always drops the transient keys. The restub pass (indexer) and the
// incoming-resolve pass (resolver) live in different packages, so this handoff
// rides the edge Meta between them.
const (
metaRestubPrevTo = "restub_prev_to"
metaRestubPrevOrigin = "restub_prev_origin"
metaRestubPrevTier = "restub_prev_tier"
metaRestubPrevConf = "restub_prev_conf"
)
// StashRestubProvenance records e's current target + resolved provenance in its
// Meta and clears the live provenance, so a restubbed edge no longer advertises
// the resolved tier it held while bound. Call it just before rewriting e.To to
// the unresolved stub. No-op on a nil edge.
func StashRestubProvenance(e *Edge) {
if e == nil {
return
}
if e.Meta == nil {
e.Meta = map[string]any{}
}
e.Meta[metaRestubPrevTo] = e.To
e.Meta[metaRestubPrevOrigin] = e.Origin
e.Meta[metaRestubPrevTier] = e.Tier
e.Meta[metaRestubPrevConf] = e.Confidence
e.Origin, e.Tier, e.Confidence = "", "", 0
}
// RestoreRestubProvenance re-applies stashed provenance to e when e is now
// resolved to the SAME target it had before the restub — the binding is
// unchanged, so its verified tier is too — and always drops the transient stash
// keys (whether or not it restored). A rebind to a different target, or a stub
// that never rebound, keeps the honest fresh tier the resolver assigned (or
// none). No-op when there is nothing stashed.
//
// Returns true when it actually re-applied the stashed provenance — the caller
// uses that to persist the in-place mutation, since a disk backend hands out
// freshly-decoded edge pointers whose changes are lost unless written back.
func RestoreRestubProvenance(e *Edge) bool {
if e == nil || e.Meta == nil {
return false
}
prevTo, ok := e.Meta[metaRestubPrevTo]
if !ok {
return false
}
restored := false
if to, isStr := prevTo.(string); isStr && to == e.To && !IsUnresolvedTarget(e.To) {
if o, ok := e.Meta[metaRestubPrevOrigin].(string); ok {
e.Origin = o
}
if tr, ok := e.Meta[metaRestubPrevTier].(string); ok {
e.Tier = tr
}
if c, ok := e.Meta[metaRestubPrevConf].(float64); ok {
e.Confidence = c
}
restored = true
}
delete(e.Meta, metaRestubPrevTo)
delete(e.Meta, metaRestubPrevOrigin)
delete(e.Meta, metaRestubPrevTier)
delete(e.Meta, metaRestubPrevConf)
return restored
}
+25
View File
@@ -0,0 +1,25 @@
package graph
import "testing"
// TestStats_ExcludesProxyNodes asserts cross-daemon federation proxy nodes
// are never counted in Graph.Stats.
func TestStats_ExcludesProxyNodes(t *testing.T) {
g := New()
g.AddNode(&Node{ID: "local/a.go::Foo", Kind: KindFunction, Name: "Foo", Language: "go"})
g.AddNode(&Node{
ID: ProxyNodeID("remoteB", "b/x.go::Bar"),
Kind: KindFunction,
Name: "Bar",
Origin: "remote:remoteB",
Stub: true,
})
st := g.Stats()
if st.TotalNodes != 1 {
t.Errorf("Stats.TotalNodes = %d, want 1 (proxy node excluded)", st.TotalNodes)
}
if st.ByKind["function"] != 1 {
t.Errorf("ByKind[function] = %d, want 1 (proxy node excluded)", st.ByKind["function"])
}
}
+183
View File
@@ -0,0 +1,183 @@
package graph
import (
"fmt"
"sync"
"testing"
)
func isPowerOfTwo(n int) bool { return n > 0 && n&(n-1) == 0 }
// TestNextPow2 pins the rounding helper that guarantees the power-of-two
// invariant the shardMask trick depends on.
func TestNextPow2(t *testing.T) {
cases := map[int]int{
-3: 1, 0: 1, 1: 1, 2: 2, 3: 4, 4: 4,
5: 8, 9: 16, 15: 16, 16: 16, 17: 32, 255: 256, 256: 256, 257: 512,
}
for in, want := range cases {
if got := nextPow2(in); got != want {
t.Errorf("nextPow2(%d) = %d, want %d", in, got, want)
}
}
}
// TestCoerceShardCountBounds asserts every coerced count is a power of
// two within [minShards, maxShards] regardless of the input.
func TestCoerceShardCountBounds(t *testing.T) {
for _, in := range []int{-100, 0, 1, 3, 4, 7, 16, 100, 256, 511, 512, 1000, 1 << 20} {
got := coerceShardCount(in)
if !isPowerOfTwo(got) {
t.Errorf("coerceShardCount(%d) = %d, not a power of two", in, got)
}
if got < minShards || got > maxShards {
t.Errorf("coerceShardCount(%d) = %d, outside [%d, %d]", in, got, minShards, maxShards)
}
}
}
// TestDefaultShardCountDerived asserts the CPU-derived default (no
// override) is always a power of two in the derived range.
func TestDefaultShardCountDerived(t *testing.T) {
t.Setenv(shardCountEnv, "") // ensure no override is in effect
got := defaultShardCount()
if !isPowerOfTwo(got) {
t.Fatalf("defaultShardCount() = %d, not a power of two", got)
}
if got < derivedShardFloor || got > derivedShardCeiling {
t.Fatalf("defaultShardCount() = %d, outside derived range [%d, %d]", got, derivedShardFloor, derivedShardCeiling)
}
}
// TestGraphShardFieldsConsistent asserts the constructed graph's shard
// bookkeeping is internally consistent: the slice length equals
// shardCount, shardCount is a power of two, and shardMask == count-1.
func TestGraphShardFieldsConsistent(t *testing.T) {
for _, req := range []int{1, 2, 4, 7, 16, 64, 300, 1000} {
g := newWithShardCount(req)
if !isPowerOfTwo(g.shardCount) {
t.Errorf("newWithShardCount(%d): shardCount = %d, not a power of two", req, g.shardCount)
}
if g.shardCount < minShards || g.shardCount > maxShards {
t.Errorf("newWithShardCount(%d): shardCount = %d, outside [%d, %d]", req, g.shardCount, minShards, maxShards)
}
if g.shardMask != uint32(g.shardCount-1) {
t.Errorf("newWithShardCount(%d): shardMask = %d, want %d", req, g.shardMask, g.shardCount-1)
}
if len(g.shards) != g.shardCount {
t.Errorf("newWithShardCount(%d): len(shards) = %d, want %d", req, len(g.shards), g.shardCount)
}
for i, s := range g.shards {
if s == nil || s.nodes == nil {
t.Errorf("newWithShardCount(%d): shard %d not initialized", req, i)
}
}
}
}
// TestShardCountEnvOverride asserts GORTEX_GRAPH_SHARDS is honored,
// coerced to a power of two within bounds, and reflected in the graph
// built by New().
func TestShardCountEnvOverride(t *testing.T) {
cases := []struct {
env string
want int
}{
{"4", 4}, // exact power of two below the derived floor — still honored
{"5", 8}, // not a power of two — coerced up
{"64", 64}, // exact
{"300", 512}, // not a power of two — coerced up
{"1000", 512}, // above maxShards — clamped
{"1", 1}, // minimum
}
for _, c := range cases {
t.Run(c.env, func(t *testing.T) {
t.Setenv(shardCountEnv, c.env)
if got := defaultShardCount(); got != c.want {
t.Fatalf("defaultShardCount() with %s=%q = %d, want %d", shardCountEnv, c.env, got, c.want)
}
g := New()
if g.shardCount != c.want {
t.Fatalf("New().shardCount = %d, want %d", g.shardCount, c.want)
}
if g.shardMask != uint32(c.want-1) {
t.Fatalf("New().shardMask = %d, want %d", g.shardMask, c.want-1)
}
if len(g.shards) != c.want {
t.Fatalf("len(New().shards) = %d, want %d", len(g.shards), c.want)
}
})
}
}
// TestShardCountInvalidEnvFallsBack asserts a malformed or non-positive
// override is ignored and the graph falls back to the derived default.
func TestShardCountInvalidEnvFallsBack(t *testing.T) {
for _, v := range []string{"abc", "0", "-4", " ", "3.5"} {
t.Run(v, func(t *testing.T) {
t.Setenv(shardCountEnv, v)
got := defaultShardCount()
if !isPowerOfTwo(got) || got < derivedShardFloor || got > derivedShardCeiling {
t.Fatalf("override %q: defaultShardCount() = %d, want power-of-two in [%d, %d]", v, got, derivedShardFloor, derivedShardCeiling)
}
})
}
}
// TestShardedConcurrentInsertNoLoss inserts a large, disjoint set of
// nodes and edges from many goroutines into graphs built with several
// shard counts (including non-default 1, 4, 64) and asserts nothing is
// lost or duplicated — proving every shard is correctly initialized and
// indexed regardless of count.
func TestShardedConcurrentInsertNoLoss(t *testing.T) {
for _, count := range []int{1, 4, 16, 64} {
t.Run(fmt.Sprintf("shards=%d", count), func(t *testing.T) {
g := newWithShardCount(count)
if g.shardCount != count {
t.Fatalf("newWithShardCount(%d).shardCount = %d", count, g.shardCount)
}
const workers = 32
const perWorker = 500
var wg sync.WaitGroup
wg.Add(workers)
for w := range workers {
go func(w int) {
defer wg.Done()
for i := range perWorker {
nid := fmt.Sprintf("w%d-n%d::N", w, i)
g.AddNode(&Node{ID: nid, Name: "N", Kind: KindFunction, FilePath: "f"})
// One out-edge per node to a shared sink, so the From
// and To endpoints frequently land in different shards.
g.AddEdge(&Edge{
From: nid,
To: fmt.Sprintf("sink-%d::S", i%17),
Kind: EdgeCalls,
FilePath: "f",
Line: w,
})
}
}(w)
}
wg.Wait()
wantNodes := workers * perWorker
for w := range workers {
for i := range perWorker {
nid := fmt.Sprintf("w%d-n%d::N", w, i)
if g.GetNode(nid) == nil {
t.Fatalf("node %q missing after concurrent insert (shards=%d)", nid, count)
}
if outs := g.GetOutEdges(nid); len(outs) != 1 {
t.Fatalf("node %q has %d out-edges, want 1 (shards=%d)", nid, len(outs), count)
}
}
}
// Sink targets are never AddNode'd, so NodeCount counts only the
// inserted N nodes — exactly one per (worker, index) pair.
if got := g.NodeCount(); got != wantNodes {
t.Fatalf("NodeCount() = %d, want %d (shards=%d)", got, wantNodes, count)
}
})
}
}
+29
View File
@@ -0,0 +1,29 @@
package graph
import "testing"
func TestOriginRank_Speculative(t *testing.T) {
if OriginRank(OriginSpeculative) >= OriginRank(OriginTextMatched) {
t.Errorf("speculative (%d) must rank below text_matched (%d)",
OriginRank(OriginSpeculative), OriginRank(OriginTextMatched))
}
if OriginRank(OriginSpeculative) <= OriginRank("") {
t.Errorf("speculative must rank above unknown/empty")
}
if MeetsMinTier(OriginSpeculative, OriginTextMatched) {
t.Errorf("min_tier=text_matched must exclude speculative edges")
}
if !MeetsMinTier(OriginSpeculative, "") {
t.Errorf("empty min_tier must pass speculative edges")
}
}
func TestEdge_IsSpeculative(t *testing.T) {
if (&Edge{}).IsSpeculative() {
t.Errorf("plain edge must not be speculative")
}
e := &Edge{Meta: map[string]any{MetaSpeculative: true}}
if !e.IsSpeculative() {
t.Errorf("edge with MetaSpeculative=true must be speculative")
}
}
File diff suppressed because it is too large Load Diff
+218
View File
@@ -0,0 +1,218 @@
package store_sqlite
import (
"context"
"database/sql"
"errors"
"fmt"
"github.com/zzet/gortex/internal/graph"
)
// Compile-time assertion: *Store satisfies graph.BulkLoader.
var _ graph.BulkLoader = (*Store)(nil)
// bulkDroppableIndex is one secondary index the bulk-load fast path drops
// before a first/empty cold index and rebuilds afterward.
type bulkDroppableIndex struct {
name string
ddl string
}
// bulkDroppableIndexes is the single source of truth for these index
// definitions. Open creates them (so the initial DB has them), BeginBulkLoad
// drops them by name, and FlushBulk recreates them from the exact same ddl —
// keeping the initial and post-bulk shapes from drifting.
//
// These are exactly the standalone, NON-UNIQUE CREATE INDEX statements over
// the large nodes / edges tables. Maintaining them per-row across a
// multi-hundred-thousand-row cold load is pure overhead when the rows land
// once, so they are dropped up front and rebuilt in one pass at the end.
//
// Deliberately excluded:
// - nodes_by_qual (UNIQUE): enforces qual_name dedup on every
// INSERT OR REPLACE. Dropping it would change insert conflict semantics
// (collapsed qual_name collisions would diverge from the non-bulk path)
// and a duplicate could make the recreate fail. It stays live.
// - the edges UNIQUE(from_id, …) table constraint and every WITHOUT ROWID
// primary-key index: not standalone indexes; they cannot be dropped while
// the table/constraint exists.
// - edges_external (partial): a tiny index over external-call terminals,
// created from a shared predicate in Open; not worth dropping.
//
// Dropping/recreating these is a runtime operation on identical DDL — it is
// NOT a schema change, so it does not touch the persisted schema version.
var bulkDroppableIndexes = []bulkDroppableIndex{
{"nodes_by_name", `CREATE INDEX IF NOT EXISTS nodes_by_name ON nodes(name)`},
{"nodes_by_kind", `CREATE INDEX IF NOT EXISTS nodes_by_kind ON nodes(kind)`},
{"nodes_by_file", `CREATE INDEX IF NOT EXISTS nodes_by_file ON nodes(file_path)`},
{"nodes_by_repo", `CREATE INDEX IF NOT EXISTS nodes_by_repo ON nodes(repo_prefix) WHERE repo_prefix <> ''`},
{"edges_by_from", `CREATE INDEX IF NOT EXISTS edges_by_from ON edges(from_id, kind)`},
{"edges_by_to", `CREATE INDEX IF NOT EXISTS edges_by_to ON edges(to_id, kind)`},
{"edges_by_kind", `CREATE INDEX IF NOT EXISTS edges_by_kind ON edges(kind)`},
// Backs EdgesWithUnresolvedTarget — the resolver's main pending-edge
// collector, called on every full resolve. is_unresolved is a VIRTUAL
// generated column (see isUnresolvedColumnDDL); indexing it turns a
// full-table scan (the prior to_id-based OR query forced SQLite to
// abandon its index) into an index search whose bookmark lookups land
// in ascending rowid order (see isUnresolvedColumnDDL's doc comment for
// why that beats an equivalent-looking to_id-based index).
{"edges_by_unresolved", `CREATE INDEX IF NOT EXISTS edges_by_unresolved ON edges(is_unresolved)`},
// Partial index over exactly the not-yet-semantically-stamped nodes per
// repo. Stays small in steady state (most nodes end up stamped), so a
// future "unstamped nodes in this repo" query is an index scan over the
// residual few instead of a full-table decode of every node's meta.
{"nodes_semantic_pending", `CREATE INDEX IF NOT EXISTS nodes_semantic_pending ON nodes(repo_prefix) WHERE semantic_type IS NULL`},
}
// bulkCacheSizeKiB is the page cache the fast path requests on its pinned
// connection. SQLite reads a negative cache_size as a KiB budget, so this is
// ~256 MiB — large enough to keep the cold load's working set resident.
const bulkCacheSizeKiB = -262144
// beginWrite starts a write transaction. During a bulk-load fast path it pins
// the single connection that carries synchronous=OFF + the enlarged page
// cache (database/sql PRAGMAs are connection-local, so a pooled connection
// would not see them); otherwise it uses the shared pool. The caller holds
// writeMu, which also guards s.bulkConn.
func (s *Store) beginWrite() (*sql.Tx, error) {
if s.bulkConn != nil {
return s.bulkConn.BeginTx(context.Background(), nil)
}
return s.db.Begin()
}
// BeginBulkLoad enters the bulk-load fast path for a first/empty cold index.
// It pins one connection at synchronous=OFF with an enlarged page cache and
// drops the droppable secondary indexes, so a multi-hundred-thousand-row load
// skips per-row B-tree maintenance and per-commit fsync. FlushBulk reverses
// all of it: restore the pragmas, rebuild the indexes, and checkpoint.
//
// Gated: it engages ONLY when the nodes table is empty. On a populated store
// (incremental reindex, warm restart, or a later repo in a multi-repo cold
// start that shares the disk store) it is a safe no-op — dropping indexes or
// disabling crash durability under live, concurrently-readable rows would be
// unsafe. In-memory stores have no WAL / on-disk B-tree pressure, so it is a
// no-op there too.
func (s *Store) BeginBulkLoad() {
s.writeMu.Lock()
defer s.writeMu.Unlock()
// Re-entrancy / non-disk guard: a second BeginBulkLoad without an
// intervening FlushBulk, or an in-memory store, stays a no-op.
if s.bulkConn != nil || isMemoryPath(s.dbPath) {
return
}
ctx := context.Background()
conn, err := s.db.Conn(ctx)
if err != nil {
return
}
// Gate to a genuinely first/empty index.
if !nodesTableEmpty(ctx, conn) {
_ = conn.Close()
return
}
// Capture prior pragma values so FlushBulk (and every early-return /
// error path) can restore them. If they can't be read, don't engage —
// a slow correct load beats a connection stuck at synchronous=OFF.
prevSync, err := pragmaInt(ctx, conn, "synchronous")
if err != nil {
_ = conn.Close()
return
}
prevCache, err := pragmaInt(ctx, conn, "cache_size")
if err != nil {
_ = conn.Close()
return
}
// synchronous=OFF drops crash durability for the load window —
// acceptable only because a crash on a fresh index just re-indexes.
if _, err := conn.ExecContext(ctx, "PRAGMA synchronous = OFF"); err != nil {
_ = conn.Close()
return
}
if _, err := conn.ExecContext(ctx, fmt.Sprintf("PRAGMA cache_size = %d", bulkCacheSizeKiB)); err != nil {
// Roll the durability change back before bailing.
_, _ = conn.ExecContext(ctx, fmt.Sprintf("PRAGMA synchronous = %d", prevSync))
_ = conn.Close()
return
}
// Drop the droppable secondary indexes; rebuilt in one pass at
// FlushBulk. Best-effort: a failed drop just means that index keeps
// being maintained per-row (slower, still correct), so it is not fatal.
for _, idx := range bulkDroppableIndexes {
_, _ = conn.ExecContext(ctx, "DROP INDEX IF EXISTS "+idx.name)
}
s.bulkConn = conn
s.bulkPrevSync = prevSync
s.bulkPrevCacheSize = prevCache
}
// FlushBulk exits the bulk-load fast path: it rebuilds every index
// BeginBulkLoad dropped, restores synchronous + cache_size on the pinned
// connection, runs one TRUNCATE checkpoint to drain the WAL the no-fsync load
// grew, and returns the connection to the pool. It is a no-op when no fast
// path is active (BeginBulkLoad gated out, or already flushed).
//
// The pragma restore + connection release run unconditionally (defer), so a
// failure mid-rebuild can never leave a connection stuck at synchronous=OFF
// in the pool.
func (s *Store) FlushBulk() error {
s.writeMu.Lock()
defer s.writeMu.Unlock()
conn := s.bulkConn
if conn == nil {
return nil
}
// Detach first: the fast path is over regardless of the outcome below.
s.bulkConn = nil
ctx := context.Background()
defer func() {
// Always restore durability + cache and release the connection,
// even if an index rebuild failed.
_, _ = conn.ExecContext(ctx, fmt.Sprintf("PRAGMA synchronous = %d", s.bulkPrevSync))
_, _ = conn.ExecContext(ctx, fmt.Sprintf("PRAGMA cache_size = %d", s.bulkPrevCacheSize))
_ = conn.Close()
}()
for _, idx := range bulkDroppableIndexes {
if _, err := conn.ExecContext(ctx, idx.ddl); err != nil {
return fmt.Errorf("store_sqlite: rebuild index %s: %w", idx.name, err)
}
}
// Drain the WAL the no-fsync bulk window grew back into the main DB and
// truncate the -wal file. Same TRUNCATE mode as runCheckpointLoop, so it
// cooperates with the journal_size_limit / periodic-checkpoint policy.
if _, err := conn.ExecContext(ctx, "PRAGMA wal_checkpoint(TRUNCATE)"); err != nil {
return fmt.Errorf("store_sqlite: bulk checkpoint: %w", err)
}
return nil
}
// nodesTableEmpty reports whether the nodes table holds no rows. Used to gate
// the bulk-load fast path to a genuinely first/empty cold index.
func nodesTableEmpty(ctx context.Context, conn *sql.Conn) bool {
var one int
err := conn.QueryRowContext(ctx, "SELECT 1 FROM nodes LIMIT 1").Scan(&one)
return errors.Is(err, sql.ErrNoRows)
}
// pragmaInt reads a single-integer PRAGMA (synchronous, cache_size) off the
// given connection.
func pragmaInt(ctx context.Context, conn *sql.Conn, pragma string) (int64, error) {
var v int64
if err := conn.QueryRowContext(ctx, "PRAGMA "+pragma).Scan(&v); err != nil {
return 0, err
}
return v, nil
}
@@ -0,0 +1,388 @@
package store_sqlite
import (
"context"
"database/sql"
"fmt"
"os"
"path/filepath"
"testing"
"time"
"github.com/zzet/gortex/internal/graph"
)
// bulkFixture builds a deterministic node/edge set with distinct ids and
// qual_names (so the UNIQUE nodes_by_qual index never collides) and a mix of
// edge keys (a handful collide → exercise dedup). Input order is intentionally
// not key-sorted.
func bulkFixture(nNodes, nEdges int) ([]*graph.Node, []*graph.Edge) {
nodes := make([]*graph.Node, 0, nNodes)
for i := range nNodes {
nodes = append(nodes, &graph.Node{
ID: fmt.Sprintf("pkg/f%d.go::Sym%d", i%64, i),
Kind: graph.KindFunction,
Name: fmt.Sprintf("Sym%d", i),
QualName: fmt.Sprintf("pkg.f%d.Sym%d", i%64, i),
FilePath: fmt.Sprintf("pkg/f%d.go", i%64),
RepoPrefix: "gortex",
Language: "go",
})
}
edges := make([]*graph.Edge, 0, nEdges)
for i := range nEdges {
from := nodes[i%nNodes]
to := nodes[(i*7+1)%nNodes]
edges = append(edges, &graph.Edge{
From: from.ID,
To: to.ID,
Kind: graph.EdgeCalls,
FilePath: from.FilePath,
Line: i % 500,
Confidence: 1,
})
}
return nodes, edges
}
func openTempStore(t *testing.T) (*Store, string) {
t.Helper()
path := filepath.Join(t.TempDir(), "bulk.sqlite")
s, err := Open(path)
if err != nil {
t.Fatalf("open: %v", err)
}
t.Cleanup(func() { _ = s.Close() })
return s, path
}
// indexNames returns the set of secondary index names present in the schema.
func indexNames(t *testing.T, q interface {
Query(string, ...any) (*sql.Rows, error)
}) map[string]bool {
t.Helper()
rows, err := q.Query("SELECT name FROM sqlite_master WHERE type='index'")
if err != nil {
t.Fatalf("query sqlite_master: %v", err)
}
defer func() { _ = rows.Close() }()
got := map[string]bool{}
for rows.Next() {
var n string
if err := rows.Scan(&n); err != nil {
t.Fatalf("scan index name: %v", err)
}
got[n] = true
}
if err := rows.Err(); err != nil {
t.Fatalf("rows: %v", err)
}
return got
}
func pragmaIntDB(t *testing.T, db *sql.DB, pragma string) int64 {
t.Helper()
var v int64
if err := db.QueryRow("PRAGMA " + pragma).Scan(&v); err != nil {
t.Fatalf("pragma %s: %v", pragma, err)
}
return v
}
func integrityOK(t *testing.T, db *sql.DB) {
t.Helper()
var res string
if err := db.QueryRow("PRAGMA integrity_check").Scan(&res); err != nil {
t.Fatalf("integrity_check: %v", err)
}
if res != "ok" {
t.Fatalf("integrity_check = %q, want \"ok\"", res)
}
}
// TestBulkLoadDropsAndRebuildsIndexes is the core mechanism + restore proof:
// the fast path engages on an empty store, drops the droppable indexes,
// pins a synchronous=OFF connection, then on FlushBulk rebuilds every index,
// restores synchronous, releases the connection, and leaves the DB intact.
func TestBulkLoadDropsAndRebuildsIndexes(t *testing.T) {
s, _ := openTempStore(t)
ctx := context.Background()
// Baseline: all droppable indexes present, synchronous=NORMAL(1).
before := indexNames(t, s.db)
for _, idx := range bulkDroppableIndexes {
if !before[idx.name] {
t.Fatalf("index %s missing before bulk load", idx.name)
}
}
if got := pragmaIntDB(t, s.db, "synchronous"); got != 1 {
t.Fatalf("synchronous before = %d, want 1 (NORMAL)", got)
}
// Engage the fast path on the empty store.
s.BeginBulkLoad()
if s.bulkConn == nil {
t.Fatal("fast path did not engage on empty store")
}
// Read through the pinned connection (it may be the only one when
// GOMAXPROCS/NumCPU is 1) to avoid blocking on connection acquisition.
var sync int64
if err := s.bulkConn.QueryRowContext(ctx, "PRAGMA synchronous").Scan(&sync); err != nil {
t.Fatalf("pinned synchronous: %v", err)
}
if sync != 0 {
t.Fatalf("bulk synchronous = %d, want 0 (OFF)", sync)
}
var cache int64
if err := s.bulkConn.QueryRowContext(ctx, "PRAGMA cache_size").Scan(&cache); err != nil {
t.Fatalf("pinned cache_size: %v", err)
}
if cache != bulkCacheSizeKiB {
t.Fatalf("bulk cache_size = %d, want %d", cache, bulkCacheSizeKiB)
}
// The droppable indexes are gone during the window.
during := indexNames(t, &connQuerier{ctx: ctx, c: s.bulkConn})
for _, idx := range bulkDroppableIndexes {
if during[idx.name] {
t.Fatalf("index %s still present during bulk window", idx.name)
}
}
// nodes_by_qual (UNIQUE, not droppable) must remain live.
if !during["nodes_by_qual"] {
t.Fatal("nodes_by_qual (UNIQUE) must not be dropped")
}
nodes, edges := bulkFixture(2000, 4000)
s.AddBatch(nodes, edges)
if err := s.FlushBulk(); err != nil {
t.Fatalf("FlushBulk: %v", err)
}
if s.bulkConn != nil {
t.Fatal("bulkConn not released after FlushBulk")
}
// Every dropped index is back.
after := indexNames(t, s.db)
for _, idx := range bulkDroppableIndexes {
if !after[idx.name] {
t.Fatalf("index %s not rebuilt after FlushBulk", idx.name)
}
}
// synchronous restored to NORMAL on the pool.
if got := pragmaIntDB(t, s.db, "synchronous"); got != 1 {
t.Fatalf("synchronous after = %d, want 1 (NORMAL)", got)
}
integrityOK(t, s.db)
}
// connQuerier adapts *sql.Conn to the Query signature indexNames expects.
type connQuerier struct {
ctx context.Context
c *sql.Conn
}
func (q *connQuerier) Query(query string, args ...any) (*sql.Rows, error) {
return q.c.QueryContext(q.ctx, query, args...)
}
// TestBulkLoadMatchesNonBulkCounts proves the fast path persists exactly the
// same node/edge counts the plain AddBatch path does.
func TestBulkLoadMatchesNonBulkCounts(t *testing.T) {
nodes, edges := bulkFixture(3000, 6000)
plain, _ := openTempStore(t)
plain.AddBatch(nodes, edges)
wantNodes, wantEdges := plain.NodeCount(), plain.EdgeCount()
bulk, _ := openTempStore(t)
bulk.BeginBulkLoad()
if bulk.bulkConn == nil {
t.Fatal("fast path did not engage on empty store")
}
// Drain in two chunks to mirror the indexer's chunked persist.
bulk.AddBatch(nodes[:1500], nil)
bulk.AddBatch(nodes[1500:], nil)
bulk.AddBatch(nil, edges[:3000])
bulk.AddBatch(nil, edges[3000:])
if err := bulk.FlushBulk(); err != nil {
t.Fatalf("FlushBulk: %v", err)
}
if gotN, gotE := bulk.NodeCount(), bulk.EdgeCount(); gotN != wantNodes || gotE != wantEdges {
t.Fatalf("bulk counts (%d nodes, %d edges) != non-bulk (%d, %d)", gotN, gotE, wantNodes, wantEdges)
}
integrityOK(t, bulk.db)
}
// TestBulkLoadGatedToPopulatedStore confirms the fast path is a safe no-op on
// a store that already holds rows — no indexes are dropped, durability stays.
func TestBulkLoadGatedToPopulatedStore(t *testing.T) {
s, _ := openTempStore(t)
// Populate first (the normal, non-bulk path).
nodes, edges := bulkFixture(50, 100)
s.AddBatch(nodes, edges)
s.BeginBulkLoad()
if s.bulkConn != nil {
t.Fatal("fast path engaged on a populated store; must be a no-op")
}
// Indexes untouched, durability untouched.
present := indexNames(t, s.db)
for _, idx := range bulkDroppableIndexes {
if !present[idx.name] {
t.Fatalf("index %s dropped on a populated store", idx.name)
}
}
if got := pragmaIntDB(t, s.db, "synchronous"); got != 1 {
t.Fatalf("synchronous = %d on populated store, want 1 (NORMAL)", got)
}
if err := s.FlushBulk(); err != nil {
t.Fatalf("FlushBulk no-op returned error: %v", err)
}
}
// TestBulkLoadInMemoryIsNoOp confirms in-memory stores never engage the fast
// path (no WAL / on-disk B-tree to optimise).
func TestBulkLoadInMemoryIsNoOp(t *testing.T) {
s, err := Open(":memory:")
if err != nil {
t.Fatalf("open: %v", err)
}
t.Cleanup(func() { _ = s.Close() })
s.BeginBulkLoad()
if s.bulkConn != nil {
t.Fatal("fast path engaged on an in-memory store")
}
if err := s.FlushBulk(); err != nil {
t.Fatalf("FlushBulk: %v", err)
}
}
// TestBulkLoadWarmRestartLoadsClean bulk-loads, closes, reopens the same file,
// and asserts the persisted graph round-trips: identical counts, indexes
// present, integrity ok.
func TestBulkLoadWarmRestartLoadsClean(t *testing.T) {
path := filepath.Join(t.TempDir(), "warm.sqlite")
nodes, edges := bulkFixture(2500, 5000)
s, err := Open(path)
if err != nil {
t.Fatalf("open: %v", err)
}
s.BeginBulkLoad()
if s.bulkConn == nil {
t.Fatal("fast path did not engage on empty store")
}
s.AddBatch(nodes, edges)
if err := s.FlushBulk(); err != nil {
t.Fatalf("FlushBulk: %v", err)
}
wantNodes, wantEdges := s.NodeCount(), s.EdgeCount()
if err := s.Close(); err != nil {
t.Fatalf("close: %v", err)
}
reopened, err := Open(path)
if err != nil {
t.Fatalf("reopen: %v", err)
}
t.Cleanup(func() { _ = reopened.Close() })
if gotN, gotE := reopened.NodeCount(), reopened.EdgeCount(); gotN != wantNodes || gotE != wantEdges {
t.Fatalf("warm restart counts (%d, %d) != pre-close (%d, %d)", gotN, gotE, wantNodes, wantEdges)
}
present := indexNames(t, reopened.db)
for _, idx := range bulkDroppableIndexes {
if !present[idx.name] {
t.Fatalf("index %s missing after warm restart", idx.name)
}
}
integrityOK(t, reopened.db)
// A populated store on reopen must NOT engage the fast path.
reopened.BeginBulkLoad()
if reopened.bulkConn != nil {
t.Fatal("fast path engaged on warm restart (populated store)")
}
_ = reopened.FlushBulk()
}
// TestBulkLoadPersistSpeed is the persist-speed evidence: it times the plain
// path vs the fast path on the same fixture and logs both. It asserts
// correctness and that the fast path is not pathologically slower; a strict
// speedup ratio is gated behind GORTEX_BULK_PERF_ASSERT so the default run
// stays deterministic on noisy CI.
func TestBulkLoadPersistSpeed(t *testing.T) {
if testing.Short() {
t.Skip("skipping persist-speed timing in -short")
}
n, e := 8000, 16000
nodes, edges := bulkFixture(n, e)
plain, _ := openTempStore(t)
t0 := time.Now()
plain.AddBatch(nodes, edges)
plainDur := time.Since(t0)
bulk, _ := openTempStore(t)
t1 := time.Now()
bulk.BeginBulkLoad()
bulk.AddBatch(nodes, edges)
if err := bulk.FlushBulk(); err != nil {
t.Fatalf("FlushBulk: %v", err)
}
bulkDur := time.Since(t1)
if bulk.NodeCount() != plain.NodeCount() || bulk.EdgeCount() != plain.EdgeCount() {
t.Fatalf("count mismatch: bulk(%d,%d) plain(%d,%d)",
bulk.NodeCount(), bulk.EdgeCount(), plain.NodeCount(), plain.EdgeCount())
}
integrityOK(t, bulk.db)
ratio := float64(plainDur) / float64(bulkDur)
t.Logf("persist %d nodes / %d edges: plain=%s bulk=%s speedup=%.2fx",
n, e, plainDur, bulkDur, ratio)
// Sanity floor: the fast path must never be dramatically slower.
if bulkDur > plainDur*5 {
t.Fatalf("fast path far slower: plain=%s bulk=%s", plainDur, bulkDur)
}
if os.Getenv("GORTEX_BULK_PERF_ASSERT") != "" && ratio < 2.0 {
t.Fatalf("fast path speedup %.2fx below 2x target", ratio)
}
}
// BenchmarkPersistFixture is reproducible persist-speed evidence: run with
//
// go test -run=^$ -bench=BenchmarkPersistFixture ./internal/graph/store_sqlite/
//
// to compare the plain AddBatch path against the bulk-load fast path.
func BenchmarkPersistFixture(b *testing.B) {
nodes, edges := bulkFixture(50000, 100000)
run := func(b *testing.B, bulk bool) {
for i := 0; i < b.N; i++ {
b.StopTimer()
path := filepath.Join(b.TempDir(), fmt.Sprintf("p%d.sqlite", i))
s, err := Open(path)
if err != nil {
b.Fatalf("open: %v", err)
}
b.StartTimer()
if bulk {
s.BeginBulkLoad()
s.AddBatch(nodes, edges)
if err := s.FlushBulk(); err != nil {
b.Fatalf("FlushBulk: %v", err)
}
} else {
s.AddBatch(nodes, edges)
}
b.StopTimer()
_ = s.Close()
}
}
b.Run("nonbulk", func(b *testing.B) { run(b, false) })
b.Run("bulk", func(b *testing.B) { run(b, true) })
}
+320
View File
@@ -0,0 +1,320 @@
package store_sqlite
import (
"os"
"path/filepath"
"strconv"
"strings"
"sync"
"github.com/zzet/gortex/internal/graph"
)
// bundleCacheDefaultMaxBytes bounds the total heap the bundle cache may
// retain across all cached entries. A count ceiling alone is unsafe: an
// entry holds a decoded node plus its full in/out edge lists, and both
// nodes and edges carry meta maps, so entry sizes span ~1 KB for a leaf
// symbol to multiple MB for a hub node with thousands of edges. A cap
// measured in entries therefore admits an unbounded BYTE footprint — a
// few thousand hub bundles can pin gigabytes. This cache serves point
// lookups on the symbol-search hot path; it is a latency optimisation,
// not a working set that needs to be resident, so a modest budget is
// right — 64 MiB holds the hot few thousand ordinary bundles while
// keeping a long-lived daemon's idle heap bounded. Override with
// GORTEX_BUNDLE_CACHE_MAX_MB=<n> (n <= 0 disables the cache entirely).
const bundleCacheDefaultMaxBytes = 64 << 20 // 64 MiB
// bundleCacheMaxEntries is a secondary, generous count ceiling kept
// alongside the byte budget. The byte budget is the primary bound; this
// guards the map's own structural overhead in the degenerate case of a
// flood of tiny entries (a bucket slot and pointers per entry are not
// fully reflected in a per-entry byte estimate), and keeps the
// wholesale-clear allocation predictable. It is deliberately loose: a
// half-million-symbol monorepo's hottest few thousand search hits fit
// far under it, so in normal operation the byte budget always trips
// first.
const bundleCacheMaxEntries = 50000
const (
// bundleEntryOverhead is a coarse fixed charge per cached entry that
// is independent of the bundle's string content: the bundleCacheEntry
// wrapper, the *entry and *Node pointers, the graph.Node value's flat
// struct (its string / slice / map headers, ints, and embedded
// time.Time), and the map bucket the node id occupies. String and map
// *contents* are added on top. Over-estimating here only makes the
// cache clear sooner; it never lets the footprint overshoot the budget.
bundleEntryOverhead = 448
// bundleEdgeOverhead is the coarse fixed charge for one *Edge in an
// in/out slice: the pointer, the slice slot, and the Edge value's flat
// struct. Edge string / map contents are added separately.
bundleEdgeOverhead = 240
// bundleMetaEntryOverhead is the fixed per-key charge for a
// map[string]any entry (bucket slot + interface header); the key
// length and any string value length are added on top.
bundleMetaEntryOverhead = 48
)
// bundleCacheEntry is one node's cached bundle, tagged with the package
// it belongs to and the package fingerprint that was current when the
// bundle was computed. The entry is served only while
// fingerprints[pkgKey] still equals fp — any change to the package's
// content (a node or edge added / removed / reweighted, including a
// cross-file edge that lands on this node from elsewhere) moves the
// fingerprint and forces a recompute, so a cached bundle can never
// carry a stale edge.
type bundleCacheEntry struct {
pkgKey string
fp uint64
bundle graph.SymbolBundle
// bytes is the entry's estimated retained size, recorded at insert so
// the running byte total can be adjusted in O(1) whenever the entry is
// dropped (invalidation or a stale read).
bytes int64
}
// bundleCache is a content-addressed, package-scoped cache over
// SearchSymbolBundles. It is keyed at the node level but validated at
// the package level: an entry is fresh exactly when the package's
// current fingerprint matches the fingerprint the entry was stored at.
//
// Correctness rests entirely on the fingerprint discipline: the daemon
// hands the cache an authoritative per-package fingerprint map after
// every analysis pass (which runs after every incremental reindex and
// every edit_file / fsnotify-driven graph mutation). The fingerprints
// are edge-aware — they fold every package's nodes AND the edges
// touching them — so any mutation that could change a cached bundle's
// in/out edges moves the relevant package fingerprint and invalidates
// the entry. A package whose fingerprint is unchanged is served from
// cache; a package the daemon has never reported a fingerprint for is
// always treated as a miss (conservative: never serve an unvalidated
// bundle).
//
// The cache is bounded by bytes (maxBytes), not by entry count, because
// entry sizes vary by orders of magnitude with a node's edge fan-out and
// meta size. maxEntries is a secondary count ceiling only. When either
// bound would be exceeded the cache is cleared wholesale rather than
// evicting individually: entries are cheap to recompute (one batched
// fetch), and a wholesale clear keeps the bookkeeping O(1) and free of an
// LRU's per-entry ordering overhead. maxBytes <= 0 disables the cache —
// stores become no-ops and every lookup misses (reads still recompute
// live through the caller's fallback path).
type bundleCache struct {
mu sync.Mutex
fingerprints map[string]uint64
entries map[string]*bundleCacheEntry
maxBytes int64 // byte budget (primary bound); <= 0 disables the cache
maxEntries int // count ceiling (secondary bound)
curBytes int64 // running sum of entries' estimated bytes
}
// newBundleCache builds an empty cache with the default budgets. The byte
// budget is overridable with GORTEX_BUNDLE_CACHE_MAX_MB=<n>; n <= 0
// disables the cache. It starts inert (every lookup a miss) until the
// daemon supplies fingerprints.
func newBundleCache() *bundleCache {
return &bundleCache{
fingerprints: map[string]uint64{},
entries: map[string]*bundleCacheEntry{},
maxBytes: bundleCacheMaxBytes(),
maxEntries: bundleCacheMaxEntries,
}
}
// bundleCacheMaxBytes resolves the byte budget from the environment,
// falling back to the default. GORTEX_BUNDLE_CACHE_MAX_MB is read in
// mebibytes; a value <= 0 returns 0 to disable the cache, and an
// unparseable value is ignored (keeps the default).
func bundleCacheMaxBytes() int64 {
if v := strings.TrimSpace(os.Getenv("GORTEX_BUNDLE_CACHE_MAX_MB")); v != "" {
if n, err := strconv.Atoi(v); err == nil {
if n <= 0 {
return 0
}
return int64(n) << 20
}
}
return bundleCacheDefaultMaxBytes
}
// bundleEntryBytes conservatively estimates a bundle's retained heap for
// the byte budget: a fixed per-entry charge plus the node's string and
// meta contents plus each in/out edge's fixed charge and its string and
// meta contents. Computed once at insert so the overflow check is a cheap
// scalar comparison.
func bundleEntryBytes(b graph.SymbolBundle) int64 {
n := int64(bundleEntryOverhead)
if b.Node != nil {
n += nodeStringBytes(b.Node)
n += metaBytes(b.Node.Meta)
}
for _, e := range b.InEdges {
n += edgeBytes(e)
}
for _, e := range b.OutEdges {
n += edgeBytes(e)
}
return n
}
// nodeStringBytes sums the byte lengths of a node's string fields (its
// heap-backed content, on top of the fixed struct overhead counted in
// bundleEntryOverhead).
func nodeStringBytes(nd *graph.Node) int64 {
return int64(len(nd.ID) + len(nd.Name) + len(nd.QualName) + len(nd.FilePath) +
len(string(nd.Kind)) + len(nd.Language) + len(nd.RepoPrefix) +
len(nd.WorkspaceID) + len(nd.ProjectID) + len(nd.AbsoluteFilePath) +
len(nd.Origin))
}
// edgeBytes estimates one edge's retained heap: the fixed per-edge charge
// plus its string fields and meta contents.
func edgeBytes(e *graph.Edge) int64 {
if e == nil {
return bundleEdgeOverhead
}
n := int64(bundleEdgeOverhead)
n += int64(len(e.From) + len(e.To) + len(string(e.Kind)) + len(e.FilePath) +
len(e.ConfidenceLabel) + len(e.Origin) + len(e.Tier) + len(e.Context) +
len(e.ReturnUsage) + len(e.Via) + len(e.Alias))
n += metaBytes(e.Meta)
return n
}
// metaBytes estimates a meta map's retained heap: a fixed charge per key
// plus the key length and, for string values, the value length. Non-string
// values fold into the fixed charge — meta values are overwhelmingly short
// scalars, and a coarse estimate only over-counts, which is safe.
func metaBytes(m map[string]any) int64 {
if len(m) == 0 {
return 0
}
var n int64
for k, v := range m {
n += int64(len(k) + bundleMetaEntryOverhead)
if s, ok := v.(string); ok {
n += int64(len(s))
}
}
return n
}
// SetBundleFingerprints installs the authoritative per-package
// fingerprint map and drops any cached entry whose package fingerprint
// has changed (or whose package is no longer reported). This is the
// invalidation entry point: the daemon calls it after each analysis
// pass with the fresh fingerprints derived from the live graph, so a
// reindex that altered a package's nodes or edges retires exactly the
// affected bundles while leaving untouched packages cached.
//
// fps is keyed by package key (the directory the package's files live
// in, repo-prefixed in multi-repo because the node file paths are).
func (s *Store) SetBundleFingerprints(fps map[string]uint64) {
if s.bundles == nil {
return
}
s.bundles.refresh(fps)
}
// refresh swaps in the new fingerprint map and prunes every entry whose
// package fingerprint no longer matches, decrementing the running byte
// total by each dropped entry's estimated size.
func (c *bundleCache) refresh(fps map[string]uint64) {
c.mu.Lock()
defer c.mu.Unlock()
if fps == nil {
fps = map[string]uint64{}
}
c.fingerprints = fps
for id, e := range c.entries {
cur, ok := fps[e.pkgKey]
if !ok || cur != e.fp {
delete(c.entries, id)
c.curBytes -= e.bytes
}
}
}
// bundlePackageKey derives the package key for a node's file path. It
// mirrors the analysis layer's packageKey so the cache and the
// daemon-supplied fingerprint map agree on package identity: the
// directory the file lives in (repo-prefixed in multi-repo because the
// stored file paths are), or "" for a file at the repo root / a node
// with no path.
func bundlePackageKey(filePath string) string {
if filePath == "" {
return ""
}
dir := filepath.Dir(filepath.ToSlash(filePath))
if dir == "." {
return ""
}
return dir
}
// lookup returns the cached bundle for id when it is fresh — the entry
// exists and its package fingerprint still matches the current one. A
// node whose package has no reported fingerprint is never served (ok is
// false) so an unvalidated bundle can never escape the cache. A stale
// entry is dropped in place and its bytes reclaimed.
func (c *bundleCache) lookup(id string) (graph.SymbolBundle, bool) {
c.mu.Lock()
defer c.mu.Unlock()
e, ok := c.entries[id]
if !ok {
return graph.SymbolBundle{}, false
}
cur, ok := c.fingerprints[e.pkgKey]
if !ok || cur != e.fp {
// Stale or unvalidated — drop it so a later refresh doesn't
// have to, and reclaim its bytes.
delete(c.entries, id)
c.curBytes -= e.bytes
return graph.SymbolBundle{}, false
}
return e.bundle, true
}
// store records a freshly computed bundle, tagged with its package's
// current fingerprint. A node whose package has no reported fingerprint
// is NOT cached (it could not be validated on read-back), keeping the
// cache conservative. The cache is bounded by bytes: when admitting the
// new entry would push the running total over the byte budget (or the
// count over the secondary ceiling) the cache is cleared wholesale
// before the insert. A single bundle that on its own exceeds the whole
// budget — a hub node with thousands of edges, exactly the pathological
// case a byte cap exists to keep out of long-lived memory — is refused
// outright rather than pinned. With maxBytes <= 0 the cache is disabled
// and every store is a no-op.
func (c *bundleCache) store(b graph.SymbolBundle) {
if b.Node == nil {
return
}
pkgKey := bundlePackageKey(b.Node.FilePath)
c.mu.Lock()
defer c.mu.Unlock()
if c.maxBytes <= 0 {
return
}
fp, ok := c.fingerprints[pkgKey]
if !ok {
return
}
sz := bundleEntryBytes(b)
if sz > c.maxBytes {
// One entry larger than the entire budget would blow the bound and
// be evicted by the very next insert's wholesale clear anyway.
return
}
if old, ok := c.entries[b.Node.ID]; ok {
// Replacing an existing entry — discount its bytes and drop it so
// curBytes and the count check track the live set.
c.curBytes -= old.bytes
delete(c.entries, b.Node.ID)
}
if len(c.entries) > 0 && (c.curBytes+sz > c.maxBytes || len(c.entries) >= c.maxEntries) {
c.entries = make(map[string]*bundleCacheEntry)
c.curBytes = 0
}
c.entries[b.Node.ID] = &bundleCacheEntry{pkgKey: pkgKey, fp: fp, bundle: b, bytes: sz}
c.curBytes += sz
}
@@ -0,0 +1,390 @@
package store_sqlite
import (
"fmt"
"path/filepath"
"sync"
"testing"
"github.com/zzet/gortex/internal/graph"
)
func mkFnNode(id, name, file string) *graph.Node {
return &graph.Node{ID: id, Kind: graph.KindFunction, Name: name, FilePath: file, Language: "go"}
}
// newTestBundleCache builds a cache with the default byte budget without
// consulting the environment, so the fingerprint / invalidation unit tests
// stay hermetic regardless of GORTEX_BUNDLE_CACHE_MAX_MB.
func newTestBundleCache() *bundleCache {
return &bundleCache{
fingerprints: map[string]uint64{},
entries: map[string]*bundleCacheEntry{},
maxBytes: bundleCacheDefaultMaxBytes,
maxEntries: bundleCacheMaxEntries,
}
}
// --- unit tests over the cache logic in isolation ---
func TestBundleCache_ServesOnlyValidatedFingerprints(t *testing.T) {
c := newTestBundleCache()
b := graph.SymbolBundle{Node: mkFnNode("pkg/x.go::A", "A", "pkg/x.go")}
// No fingerprint reported for the package yet -> store is a no-op
// (conservative: never cache an unvalidated bundle).
c.store(b)
if _, ok := c.lookup("pkg/x.go::A"); ok {
t.Fatal("bundle was cached despite no package fingerprint")
}
// Report a fingerprint, then store: now it caches and serves.
c.refresh(map[string]uint64{"pkg": 100})
c.store(b)
if _, ok := c.lookup("pkg/x.go::A"); !ok {
t.Fatal("bundle should be served once its package fingerprint is known")
}
}
func TestBundleCache_InvalidatesOnFingerprintChange(t *testing.T) {
c := newTestBundleCache()
c.refresh(map[string]uint64{"pkg": 1})
c.store(graph.SymbolBundle{Node: mkFnNode("pkg/x.go::A", "A", "pkg/x.go")})
if _, ok := c.lookup("pkg/x.go::A"); !ok {
t.Fatal("expected a cache hit on the unchanged fingerprint")
}
// Fingerprint changes -> the entry is invalidated.
c.refresh(map[string]uint64{"pkg": 2})
if _, ok := c.lookup("pkg/x.go::A"); ok {
t.Fatal("entry must be dropped when its package fingerprint changes")
}
}
func TestBundleCache_CrossRepoIsolation(t *testing.T) {
c := newTestBundleCache()
// Two repos with the same inner directory name resolve to DIFFERENT
// package keys because the stored file paths are repo-prefixed.
c.refresh(map[string]uint64{
"repoA/pkg": 10,
"repoB/pkg": 20,
})
c.store(graph.SymbolBundle{Node: mkFnNode("repoA/pkg/x.go::A", "A", "repoA/pkg/x.go")})
c.store(graph.SymbolBundle{Node: mkFnNode("repoB/pkg/x.go::A", "A", "repoB/pkg/x.go")})
// Bumping only repoA's fingerprint must not touch repoB's entry.
c.refresh(map[string]uint64{
"repoA/pkg": 11,
"repoB/pkg": 20,
})
if _, ok := c.lookup("repoA/pkg/x.go::A"); ok {
t.Fatal("repoA entry should have been invalidated")
}
if _, ok := c.lookup("repoB/pkg/x.go::A"); !ok {
t.Fatal("repoB entry must survive a repoA-only fingerprint bump")
}
}
func TestBundlePackageKey(t *testing.T) {
cases := map[string]string{
"pkg/sub/x.go": "pkg/sub",
"x.go": "",
"": "",
"repo/a/b.go": "repo/a",
}
for in, want := range cases {
if got := bundlePackageKey(in); got != want {
t.Errorf("bundlePackageKey(%q) = %q, want %q", in, got, want)
}
}
}
// --- integration tests through the store's SearchSymbolBundles ---
func newBundleTestStore(t *testing.T) *Store {
t.Helper()
s, err := Open(filepath.Join(t.TempDir(), "b.sqlite"))
if err != nil {
t.Fatalf("Open: %v", err)
}
t.Cleanup(func() { _ = s.Close() })
return s
}
func seedBundleStore(t *testing.T, s *Store) {
t.Helper()
s.AddNode(mkFnNode("pkg/x.go::A", "AlphaWidget", "pkg/x.go"))
s.AddNode(mkFnNode("pkg/x.go::B", "BetaWidget", "pkg/x.go"))
s.AddEdge(&graph.Edge{From: "pkg/x.go::A", To: "pkg/x.go::B", Kind: graph.EdgeCalls, FilePath: "pkg/x.go"})
items := []graph.SymbolFTSItem{
{NodeID: "pkg/x.go::A", Tokens: "alpha widget"},
{NodeID: "pkg/x.go::B", Tokens: "beta widget"},
}
if err := s.BulkUpsertSymbolFTS("", items); err != nil {
t.Fatalf("BulkUpsertSymbolFTS: %v", err)
}
if err := s.BuildSymbolIndex(); err != nil {
t.Fatalf("BuildSymbolIndex: %v", err)
}
}
func bundleByID(bundles []graph.SymbolBundle) map[string]graph.SymbolBundle {
out := make(map[string]graph.SymbolBundle, len(bundles))
for _, b := range bundles {
if b.Node != nil {
out[b.Node.ID] = b
}
}
return out
}
func TestSearchSymbolBundles_CacheHitOnUnchangedFingerprint(t *testing.T) {
s := newBundleTestStore(t)
seedBundleStore(t, s)
// Report a fingerprint so the first query populates the cache.
s.SetBundleFingerprints(map[string]uint64{"pkg": 1})
first, err := s.SearchSymbolBundles("widget", 10)
if err != nil {
t.Fatalf("first SearchSymbolBundles: %v", err)
}
got := bundleByID(first)
if b, ok := got["pkg/x.go::A"]; !ok || len(b.OutEdges) != 1 {
t.Fatalf("expected A with 1 out-edge on first query, got %+v", got["pkg/x.go::A"])
}
// Mutate the graph WITHOUT bumping the fingerprint: add a second
// out-edge from A. A correct content-addressed cache serves the
// STALE (1-edge) bundle because the fingerprint is unchanged — proof
// the bundle came from cache, not a fresh fetch.
s.AddNode(mkFnNode("pkg/x.go::C", "GammaWidget", "pkg/x.go"))
s.AddEdge(&graph.Edge{From: "pkg/x.go::A", To: "pkg/x.go::C", Kind: graph.EdgeCalls, FilePath: "pkg/x.go"})
second, err := s.SearchSymbolBundles("widget", 10)
if err != nil {
t.Fatalf("second SearchSymbolBundles: %v", err)
}
cached := bundleByID(second)["pkg/x.go::A"]
if len(cached.OutEdges) != 1 {
t.Fatalf("expected the cached 1-edge bundle to be served on an unchanged fingerprint, got %d edges",
len(cached.OutEdges))
}
}
func TestSearchSymbolBundles_MissAndRecomputeOnFingerprintChange(t *testing.T) {
s := newBundleTestStore(t)
seedBundleStore(t, s)
s.SetBundleFingerprints(map[string]uint64{"pkg": 1})
if _, err := s.SearchSymbolBundles("widget", 10); err != nil {
t.Fatalf("warm-up query: %v", err)
}
// Add a real out-edge, then bump the package fingerprint to signal
// the content changed. The next query must recompute and surface the
// new edge.
s.AddNode(mkFnNode("pkg/x.go::C", "GammaWidget", "pkg/x.go"))
s.AddEdge(&graph.Edge{From: "pkg/x.go::A", To: "pkg/x.go::C", Kind: graph.EdgeCalls, FilePath: "pkg/x.go"})
s.SetBundleFingerprints(map[string]uint64{"pkg": 2})
after, err := s.SearchSymbolBundles("widget", 10)
if err != nil {
t.Fatalf("post-invalidation query: %v", err)
}
fresh := bundleByID(after)["pkg/x.go::A"]
if len(fresh.OutEdges) != 2 {
t.Fatalf("expected the recomputed 2-edge bundle after a fingerprint bump, got %d edges",
len(fresh.OutEdges))
}
}
func TestSearchSymbolBundles_UncachedWithoutFingerprints(t *testing.T) {
s := newBundleTestStore(t)
seedBundleStore(t, s)
// No SetBundleFingerprints call -> the cache stays inert and every
// query recomputes live. Adding an edge must show up immediately.
first, err := s.SearchSymbolBundles("widget", 10)
if err != nil {
t.Fatalf("first query: %v", err)
}
if got := bundleByID(first)["pkg/x.go::A"]; len(got.OutEdges) != 1 {
t.Fatalf("expected 1 edge live, got %d", len(got.OutEdges))
}
s.AddNode(mkFnNode("pkg/x.go::C", "GammaWidget", "pkg/x.go"))
s.AddEdge(&graph.Edge{From: "pkg/x.go::A", To: "pkg/x.go::C", Kind: graph.EdgeCalls, FilePath: "pkg/x.go"})
second, err := s.SearchSymbolBundles("widget", 10)
if err != nil {
t.Fatalf("second query: %v", err)
}
if got := bundleByID(second)["pkg/x.go::A"]; len(got.OutEdges) != 2 {
t.Fatalf("uncached path must reflect the new edge live, got %d", len(got.OutEdges))
}
}
// --- byte-budget tests ---
func TestBundleCache_ByteBudgetEvictionAtBoundary(t *testing.T) {
c := newTestBundleCache()
c.refresh(map[string]uint64{"pkg": 1})
// Fixed-width ids so every entry estimates to the same size.
mk := func(i int) graph.SymbolBundle {
return graph.SymbolBundle{Node: mkFnNode(fmt.Sprintf("pkg/x.go::N%03d", i), "W", "pkg/x.go")}
}
unit := bundleEntryBytes(mk(0))
const k = 4
c.maxBytes = unit * k // budget holds exactly k entries
for i := 0; i < k; i++ {
c.store(mk(i))
}
if len(c.entries) != k {
t.Fatalf("expected %d entries filling the budget, got %d", k, len(c.entries))
}
if c.curBytes != unit*k {
t.Fatalf("curBytes = %d, want %d", c.curBytes, unit*k)
}
// One more entry crosses the budget -> wholesale clear, only the newest
// survives and the byte total resets to a single unit.
c.store(mk(k))
if len(c.entries) != 1 {
t.Fatalf("crossing the budget must clear wholesale to 1 entry, got %d", len(c.entries))
}
if c.curBytes != unit {
t.Fatalf("curBytes after clear = %d, want %d", c.curBytes, unit)
}
if _, ok := c.lookup(fmt.Sprintf("pkg/x.go::N%03d", k)); !ok {
t.Fatal("the entry that triggered the clear must remain served")
}
if _, ok := c.lookup("pkg/x.go::N000"); ok {
t.Fatal("a pre-clear entry must be gone after the wholesale clear")
}
}
func TestBundleCache_RefusesEntryLargerThanBudget(t *testing.T) {
c := newTestBundleCache()
c.refresh(map[string]uint64{"pkg": 1})
b := graph.SymbolBundle{Node: mkFnNode("pkg/x.go::A", "A", "pkg/x.go")}
c.maxBytes = bundleEntryBytes(b) - 1 // budget just below a single entry
c.store(b)
if _, ok := c.lookup("pkg/x.go::A"); ok {
t.Fatal("an entry larger than the whole budget must not be cached")
}
if len(c.entries) != 0 || c.curBytes != 0 {
t.Fatalf("oversized store must leave the cache empty, got %d entries / %d bytes",
len(c.entries), c.curBytes)
}
}
func TestBundleCacheMaxBytes_EnvOverride(t *testing.T) {
t.Setenv("GORTEX_BUNDLE_CACHE_MAX_MB", "128")
if got := bundleCacheMaxBytes(); got != 128<<20 {
t.Fatalf("env override = %d, want %d", got, 128<<20)
}
if c := newBundleCache(); c.maxBytes != 128<<20 {
t.Fatalf("newBundleCache maxBytes = %d, want %d", c.maxBytes, 128<<20)
}
// Empty and unparseable values keep the default.
t.Setenv("GORTEX_BUNDLE_CACHE_MAX_MB", "")
if got := bundleCacheMaxBytes(); got != bundleCacheDefaultMaxBytes {
t.Fatalf("empty override should keep the default, got %d", got)
}
t.Setenv("GORTEX_BUNDLE_CACHE_MAX_MB", "not-a-number")
if got := bundleCacheMaxBytes(); got != bundleCacheDefaultMaxBytes {
t.Fatalf("unparseable override should keep the default, got %d", got)
}
}
func TestBundleCache_DisabledMode(t *testing.T) {
t.Setenv("GORTEX_BUNDLE_CACHE_MAX_MB", "0")
c := newBundleCache()
if c.maxBytes != 0 {
t.Fatalf("expected a disabled cache (maxBytes 0), got %d", c.maxBytes)
}
c.refresh(map[string]uint64{"pkg": 1})
c.store(graph.SymbolBundle{Node: mkFnNode("pkg/x.go::A", "A", "pkg/x.go")})
if len(c.entries) != 0 {
t.Fatalf("a disabled cache must not store, got %d entries", len(c.entries))
}
if _, ok := c.lookup("pkg/x.go::A"); ok {
t.Fatal("a disabled cache must always miss")
}
// A negative budget disables too.
t.Setenv("GORTEX_BUNDLE_CACHE_MAX_MB", "-4")
if got := bundleCacheMaxBytes(); got != 0 {
t.Fatalf("a negative override should disable the cache (0), got %d", got)
}
}
func TestSearchSymbolBundles_DisabledCacheStillServes(t *testing.T) {
t.Setenv("GORTEX_BUNDLE_CACHE_MAX_MB", "0")
s := newBundleTestStore(t)
seedBundleStore(t, s)
s.SetBundleFingerprints(map[string]uint64{"pkg": 1})
res, err := s.SearchSymbolBundles("widget", 10)
if err != nil {
t.Fatalf("SearchSymbolBundles with the cache disabled: %v", err)
}
if b, ok := bundleByID(res)["pkg/x.go::A"]; !ok || len(b.OutEdges) != 1 {
t.Fatalf("a disabled cache must still return live bundles, got %+v", b)
}
if s.bundles.maxBytes != 0 {
t.Fatalf("expected the store's cache disabled, got maxBytes %d", s.bundles.maxBytes)
}
if len(s.bundles.entries) != 0 {
t.Fatalf("a disabled cache must stay empty, got %d entries", len(s.bundles.entries))
}
}
func TestBundleCache_ConcurrentReadInsert(t *testing.T) {
c := newTestBundleCache()
c.maxBytes = 8 << 10 // small budget so wholesale clears fire under contention
c.refresh(map[string]uint64{"pkg": 1})
const workers = 8
const iters = 3000
var wg sync.WaitGroup
wg.Add(workers)
for w := 0; w < workers; w++ {
go func(w int) {
defer wg.Done()
for i := 0; i < iters; i++ {
id := fmt.Sprintf("pkg/x.go::N%d_%d", w, i%64)
switch i % 3 {
case 0:
c.store(graph.SymbolBundle{Node: mkFnNode(id, "W", "pkg/x.go")})
case 1:
_, _ = c.lookup(id)
default:
c.refresh(map[string]uint64{"pkg": uint64(i)})
}
}
}(w)
}
wg.Wait()
// No goroutines remain: the running total must exactly equal the summed
// bytes of the surviving entries (the accounting invariant), which also
// proves it never drifted negative under contention.
var sum int64
for _, e := range c.entries {
sum += e.bytes
}
if c.curBytes != sum {
t.Fatalf("curBytes %d != sum of live entry bytes %d", c.curBytes, sum)
}
if c.curBytes > c.maxBytes {
t.Fatalf("curBytes %d exceeds the byte budget %d", c.curBytes, c.maxBytes)
}
}
@@ -0,0 +1,69 @@
package store_sqlite_test
import (
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/zzet/gortex/internal/graph"
)
// TestClosedStoreReadsDoNotPanic pins the teardown-race fix: after Close()
// has shut the store (daemon shutdown / restart / store swap), an in-flight
// reader — e.g. a deferred parallel-enrich goroutine still holding a cached
// *sql.Stmt — must degrade to an empty result, never panic the whole daemon.
// Before the fix this surfaced as `panic: store_sqlite: sql: statement is
// closed` from GetNode under runDeferredEnrichParallel.
func TestClosedStoreReadsDoNotPanic(t *testing.T) {
s := openTestStore(t)
s.AddNode(&graph.Node{ID: "p/a.go::Foo", Kind: graph.KindType, Name: "Foo", FilePath: "p/a.go"})
require.NotNil(t, s.GetNode("p/a.go::Foo"), "sanity: node readable before close")
require.NoError(t, s.Close())
assert.NotPanics(t, func() {
assert.Nil(t, s.GetNode("p/a.go::Foo"))
assert.Empty(t, s.FindNodesByName("Foo"))
assert.Empty(t, s.GetFileNodes("p/a.go"))
}, "reads after Close must degrade gracefully, not panic")
}
// TestClosedStoreAggregatorsDoNotPanic pins the aggregator teardown-race sweep:
// each aggregator read runs its Query error through panicOnFatal, which
// swallows the "database is closed" race — leaving rows == nil. Every such site
// must early-return its empty value instead of dereferencing nil rows. The live
// crash was NodeIDsByKinds (FindHotspots -> RunAnalysis at watch-start) SIGSEGV
// on nil rows right after a long warmup. Each method is called with non-empty
// args so it actually reaches the Query rather than an early argument guard.
func TestClosedStoreAggregatorsDoNotPanic(t *testing.T) {
s := openTestStore(t)
s.AddNode(&graph.Node{ID: "p/a.go::Foo", Kind: graph.KindType, Name: "Foo", FilePath: "p/a.go"})
s.AddNode(&graph.Node{ID: "p/a.go::bar", Kind: graph.KindFunction, Name: "bar", FilePath: "p/a.go"})
s.AddEdge(&graph.Edge{From: "p/a.go::bar", To: "p/a.go::Foo", Kind: graph.EdgeReferences, FilePath: "p/a.go", Line: 1})
require.NoError(t, s.Close())
nodeKinds := []graph.NodeKind{graph.KindType, graph.KindFunction}
edgeKinds := []graph.EdgeKind{graph.EdgeReferences}
ids := []string{"p/a.go::Foo", "p/a.go::bar"}
assert.NotPanics(t, func() {
assert.Empty(t, s.InEdgeCountsByKind(edgeKinds))
assert.Empty(t, s.NodeIDsByKinds(nodeKinds))
assert.Empty(t, s.EdgeKindCounts())
assert.Empty(t, s.NodeDegreeByKinds(nodeKinds, ""))
assert.Empty(t, s.FileImportCounts(nil)) // exercises aggScanImportCounts
assert.Empty(t, s.InDegreeForNodes(ids))
assert.Empty(t, s.CrossRepoEdgeCounts())
assert.Empty(t, s.FileImporters("p/a.go"))
assert.Empty(t, s.FileSymbolNamesByPaths([]string{"p/a.go"}, nodeKinds))
assert.Empty(t, s.NodeDegreeCounts(ids, edgeKinds))
assert.Empty(t, s.NodeFanCounts(ids, edgeKinds, edgeKinds))
assert.Empty(t, s.CommunityCrossingsByKind(edgeKinds, map[string]string{"p/a.go::bar": "c0"}))
// Iterator-shaped: the Query runs inside the yield closure.
n := 0
for range s.EdgeAdjacencyForKinds(edgeKinds, nodeKinds) {
n++
}
assert.Zero(t, n)
}, "aggregator reads after Close must degrade to empty, not panic")
}
@@ -0,0 +1,46 @@
package store_sqlite
import (
"path/filepath"
"testing"
"github.com/stretchr/testify/require"
"github.com/zzet/gortex/internal/graph"
)
// TestGetRepoNonContentNodes verifies the SQL-level content filter (which
// json_extracts data_class out of the JSON meta blob) drops only content
// section nodes — keeping code, markdown prose, and data assets — so the
// code passes can enumerate without materialising content sections.
func TestGetRepoNonContentNodes(t *testing.T) {
s, err := Open(filepath.Join(t.TempDir(), "n.sqlite"))
require.NoError(t, err)
t.Cleanup(func() { _ = s.Close() })
s.AddBatch([]*graph.Node{
{ID: "code1", Kind: graph.KindFunction, Name: "Foo", RepoPrefix: "r"},
{ID: "content1", Kind: graph.KindDoc, Name: "doc.txt::0", RepoPrefix: "r",
Meta: map[string]any{"data_class": "content", "section_text": "x"}},
{ID: "prose1", Kind: graph.KindDoc, Name: "README.md::0", RepoPrefix: "r",
Meta: map[string]any{"asset_kind": "markdown_section"}},
{ID: "data1", Kind: graph.KindFile, Name: "x.parquet", RepoPrefix: "r",
Meta: map[string]any{"data_class": "data"}},
}, nil)
// Runtime assertion the store satisfies the optional capability.
var cr graph.NonContentNodeReader = s
ids := map[string]bool{}
for _, n := range cr.GetRepoNonContentNodes("r") {
ids[n.ID] = true
}
require.True(t, ids["code1"], "code node kept")
require.True(t, ids["prose1"], "markdown prose kept (not data_class=content)")
require.True(t, ids["data1"], "data asset kept (data_class=data, not content)")
require.False(t, ids["content1"], "content section dropped at the SQL level")
require.Len(t, ids, 3)
// Empty prefix spans all repos (still drops content).
require.Len(t, s.GetRepoNonContentNodes(""), 3)
}
@@ -0,0 +1,61 @@
package store_sqlite_test
import (
"testing"
"github.com/zzet/gortex/internal/graph"
)
// TestExternalCallCandidateEdges asserts the pushdown selects exactly the
// external-package terminals (dep:: / stdlib:: / external::, the
// per-repo-prefixed stdlib form, and already-materialised external-call::
// nodes) and nothing else — no ordinary resolved call/reference edges,
// no non-call edges.
func TestExternalCallCandidateEdges(t *testing.T) {
s := openTestStore(t)
add := func(from, to string, kind graph.EdgeKind) {
s.AddEdge(&graph.Edge{From: from, To: to, Kind: kind, FilePath: "a.go", Line: 1})
}
// External terminals — should be selected.
add("a.go::f", "dep::github.com/x/y::Z", graph.EdgeCalls)
add("a.go::f", "stdlib::fmt::Sprintf", graph.EdgeCalls)
add("a.go::f", "myrepo::stdlib::net/http::Get", graph.EdgeCalls) // per-repo-prefixed stdlib form
add("a.go::f", "external::svc.internal/api", graph.EdgeReferences)
add("a.go::f", "external-call::dep::github.com/a/b", graph.EdgeCalls) // already synthesized
// Non-candidates — must NOT be selected.
add("a.go::f", "a.go::resolvedCallee", graph.EdgeCalls) // ordinary resolved call
add("a.go::f", "unresolved::SomeName", graph.EdgeCalls) // bare unresolved (no import evidence)
add("a.go::f", "a.go::SomeType", graph.EdgeImplements) // not a call/ref edge
add("a.go::f", "dep::github.com/x/y::Z", graph.EdgeTests) // dep target but wrong kind
got := map[string]bool{}
for _, e := range s.ExternalCallCandidateEdges() {
got[e.To] = true
}
want := []string{
"dep::github.com/x/y::Z",
"stdlib::fmt::Sprintf",
"myrepo::stdlib::net/http::Get",
"external::svc.internal/api",
"external-call::dep::github.com/a/b",
}
for _, w := range want {
if !got[w] {
t.Errorf("ExternalCallCandidateEdges missing external terminal %q", w)
}
}
notWant := []string{"a.go::resolvedCallee", "unresolved::SomeName", "a.go::SomeType"}
for _, nw := range notWant {
if got[nw] {
t.Errorf("ExternalCallCandidateEdges wrongly selected non-candidate %q", nw)
}
}
// The EdgeImplements/EdgeTests rows share targets with selected ones
// but must not inflate the count via the wrong kind: exactly the 5
// distinct external targets above, all from calls/references kinds.
if len(got) != len(want) {
t.Errorf("selected %d distinct targets, want %d: %v", len(got), len(want), got)
}
}
@@ -0,0 +1,149 @@
package store_sqlite
import (
"path/filepath"
"testing"
"github.com/zzet/gortex/internal/graph"
)
func ftsRowCount(t *testing.T, s *Store, table, nodeID string) int {
t.Helper()
var n int
q := `SELECT count(*) FROM ` + table + ` WHERE node_id = ?`
if err := s.db.QueryRow(q, nodeID).Scan(&n); err != nil {
t.Fatalf("count %s: %v", table, err)
}
return n
}
func ftsHits(t *testing.T, s *Store, query string) int {
t.Helper()
hits, err := s.SearchSymbols(query, 20)
if err != nil {
t.Fatalf("SearchSymbols(%q): %v", query, err)
}
return len(hits)
}
// TestUpsertSymbolFTS_ReplacesWithoutDuplicates is the core correctness
// guard for the rowid-map delete: re-upserting a symbol must drop its prior
// row (by docid) and leave exactly one FTS row + one map row. A wrong
// LastInsertId / stale map would leave the old tokens searchable and the
// row count at 2 — so this also proves the FTS5 docid round-trips.
func TestUpsertSymbolFTS_ReplacesWithoutDuplicates(t *testing.T) {
s, err := Open(filepath.Join(t.TempDir(), "fts.sqlite"))
if err != nil {
t.Fatalf("Open: %v", err)
}
t.Cleanup(func() { _ = s.Close() })
const id = "pkg/x.go::A"
s.AddNode(mkFnNode(id, "AlphaWidget", "pkg/x.go"))
// Upsert three times: first insert, then two replacements that each
// exercise the mapped-entry docid delete.
for _, tokens := range []string{"alpha widget red", "alpha widget green", "alpha widget blue"} {
if err := s.UpsertSymbolFTS(id, tokens); err != nil {
t.Fatalf("UpsertSymbolFTS(%q): %v", tokens, err)
}
}
if got := ftsRowCount(t, s, "symbol_fts", id); got != 1 {
t.Fatalf("symbol_fts rows for %s = %d, want 1 (duplicate row leaked)", id, got)
}
if got := ftsRowCount(t, s, "symbol_fts_rowid", id); got != 1 {
t.Fatalf("symbol_fts_rowid rows for %s = %d, want 1", id, got)
}
// Only the latest tokens are searchable; the superseded ones are gone.
if got := ftsHits(t, s, "blue"); got != 1 {
t.Fatalf("search 'blue' (current tokens) = %d hits, want 1", got)
}
for _, stale := range []string{"red", "green"} {
if got := ftsHits(t, s, stale); got != 0 {
t.Fatalf("search %q (superseded tokens) = %d hits, want 0 (delete missed)", stale, got)
}
}
}
// TestBulkUpsertSymbolFTS_MaintainsRowidMap proves the bulk path keeps the
// sidecar in lockstep, and that a follow-up incremental upsert on a
// bulk-loaded symbol still replaces cleanly (no duplicate).
func TestBulkUpsertSymbolFTS_MaintainsRowidMap(t *testing.T) {
s, err := Open(filepath.Join(t.TempDir(), "fts.sqlite"))
if err != nil {
t.Fatalf("Open: %v", err)
}
t.Cleanup(func() { _ = s.Close() })
s.AddNode(mkFnNode("pkg/x.go::A", "AlphaWidget", "pkg/x.go"))
s.AddNode(mkFnNode("pkg/x.go::B", "BetaWidget", "pkg/x.go"))
if err := s.BulkUpsertSymbolFTS("", []graph.SymbolFTSItem{
{NodeID: "pkg/x.go::A", Tokens: "alpha widget red"},
{NodeID: "pkg/x.go::B", Tokens: "beta widget red"},
}); err != nil {
t.Fatalf("BulkUpsertSymbolFTS: %v", err)
}
var mapRows int
if err := s.db.QueryRow(`SELECT count(*) FROM symbol_fts_rowid`).Scan(&mapRows); err != nil {
t.Fatalf("count map: %v", err)
}
if mapRows != 2 {
t.Fatalf("symbol_fts_rowid rows = %d, want 2 after bulk", mapRows)
}
// Incremental replace on a bulk-loaded symbol — must not duplicate.
if err := s.UpsertSymbolFTS("pkg/x.go::A", "alpha widget blue"); err != nil {
t.Fatalf("UpsertSymbolFTS: %v", err)
}
if got := ftsRowCount(t, s, "symbol_fts", "pkg/x.go::A"); got != 1 {
t.Fatalf("symbol_fts rows after incremental replace = %d, want 1", got)
}
if got := ftsHits(t, s, "blue"); got != 1 {
t.Fatalf("search 'blue' = %d, want 1", got)
}
}
// TestBackfillSymbolFTSRowidMap simulates a database built before the
// sidecar existed (rows in symbol_fts, none in the map) and proves the
// backfill repopulates it so the next incremental upsert replaces cleanly
// instead of leaking a duplicate.
func TestBackfillSymbolFTSRowidMap(t *testing.T) {
s, err := Open(filepath.Join(t.TempDir(), "fts.sqlite"))
if err != nil {
t.Fatalf("Open: %v", err)
}
t.Cleanup(func() { _ = s.Close() })
const id = "pkg/x.go::A"
s.AddNode(mkFnNode(id, "AlphaWidget", "pkg/x.go"))
// Simulate the legacy state: a row in symbol_fts with no map entry.
if _, err := s.db.Exec(
`INSERT INTO symbol_fts (node_id, repo_prefix, tokens) VALUES (?, '', ?)`,
id, "alpha widget red"); err != nil {
t.Fatalf("seed legacy fts row: %v", err)
}
if _, err := s.db.Exec(`DELETE FROM symbol_fts_rowid`); err != nil {
t.Fatalf("clear map: %v", err)
}
if err := backfillSymbolFTSRowidMap(s.db); err != nil {
t.Fatalf("backfill: %v", err)
}
if got := ftsRowCount(t, s, "symbol_fts_rowid", id); got != 1 {
t.Fatalf("map rows after backfill = %d, want 1", got)
}
// Now an incremental upsert must replace, not duplicate.
if err := s.UpsertSymbolFTS(id, "alpha widget blue"); err != nil {
t.Fatalf("UpsertSymbolFTS: %v", err)
}
if got := ftsRowCount(t, s, "symbol_fts", id); got != 1 {
t.Fatalf("symbol_fts rows after post-backfill replace = %d, want 1 (dup leaked)", got)
}
if got := ftsHits(t, s, "red"); got != 0 {
t.Fatalf("search 'red' (superseded) = %d, want 0", got)
}
}
@@ -0,0 +1,76 @@
package store_sqlite
import (
"path/filepath"
"testing"
"github.com/zzet/gortex/internal/graph"
)
// TestGetRepoNodesLight verifies the graph.LightNodeReader fast path
// matches GetRepoNodes on IDs and promoted-field values, stays scoped to
// repo_prefix, and never surfaces non-promoted meta content — the
// invariant the enrichment hover-candidate refetch depends on for
// correctness (see EnrichRepoContext's use of repoScopedNodesLight).
func TestGetRepoNodesLight(t *testing.T) {
s, err := Open(filepath.Join(t.TempDir(), "light.sqlite"))
if err != nil {
t.Fatal(err)
}
defer s.Close()
s.AddNode(&graph.Node{
ID: "repoA/f.go::Stamped", Kind: graph.KindFunction, Name: "Stamped",
FilePath: "repoA/f.go", RepoPrefix: "repoA",
Meta: map[string]any{
"semantic_type": "string",
"semantic_source": "lsp-gopls",
"doc": "docs",
"complexity": 7, // non-promoted
},
})
s.AddNode(&graph.Node{
ID: "repoA/f.go::Unstamped", Kind: graph.KindFunction, Name: "Unstamped",
FilePath: "repoA/f.go", RepoPrefix: "repoA",
})
s.AddNode(&graph.Node{
ID: "repoB/g.go::Other", Kind: graph.KindFunction, Name: "Other",
FilePath: "repoB/g.go", RepoPrefix: "repoB",
})
var _ graph.LightNodeReader = s // compile-time capability check
full := s.GetRepoNodes("repoA")
light := s.GetRepoNodesLight("repoA")
if len(light) != len(full) {
t.Fatalf("light returned %d nodes, full returned %d", len(light), len(full))
}
byID := make(map[string]*graph.Node, len(light))
for _, n := range light {
byID[n.ID] = n
}
stamped, ok := byID["repoA/f.go::Stamped"]
if !ok {
t.Fatal("light scan missing the stamped node")
}
assertType[string](t, stamped.Meta, "semantic_type", "string")
assertType[string](t, stamped.Meta, "semantic_source", "lsp-gopls")
assertType[string](t, stamped.Meta, "doc", "docs")
if _, ok := stamped.Meta["complexity"]; ok {
t.Errorf("light scan must not surface non-promoted meta, got complexity=%v", stamped.Meta["complexity"])
}
unstamped, ok := byID["repoA/f.go::Unstamped"]
if !ok {
t.Fatal("light scan missing the unstamped node")
}
if _, ok := unstamped.Meta["semantic_type"]; ok {
t.Error("unstamped node must not carry a semantic_type key")
}
if _, ok := byID["repoB/g.go::Other"]; ok {
t.Error("light scan crossed repo_prefix scope")
}
}
@@ -0,0 +1,433 @@
package store_sqlite
import (
"bytes"
"encoding/gob"
"encoding/json"
"path/filepath"
"reflect"
"testing"
"github.com/zzet/gortex/internal/contracts"
"github.com/zzet/gortex/internal/graph"
)
// flatRoundTrip encodes via the flat codec only (asserting the fast path was
// taken) and decodes back through decodeMeta.
func flatRoundTrip(t *testing.T, in map[string]any) map[string]any {
t.Helper()
b, ok := encodeMetaFast(in)
if !ok {
t.Fatalf("encodeMetaFast bailed on a modelled map: %#v", in)
}
if !isFlatMeta(b) {
t.Fatalf("encodeMetaFast did not stamp the flat magic: %q", b)
}
out, err := decodeMetaFast(b)
if err != nil {
t.Fatalf("decodeMetaFast: %v", err)
}
return out
}
// TestFlatCodecEveryValueType round-trips one value of every type the codec
// models and asserts exact Go-type fidelity end to end.
func TestFlatCodecEveryValueType(t *testing.T) {
shape := &contracts.Shape{
Kind: "struct",
Fields: []contracts.ShapeField{{Name: "id", Type: "int64", Required: true}},
Notes: []string{"partial"},
}
in := map[string]any{
"str": "hello",
"unicode": "héllo мир 世界 🚀",
"bool_t": true,
"bool_f": false,
"int": -7,
"int64": int64(1700000000),
"float": 0.875,
"float_int": 2.0, // integral float must stay float64
"strs": []string{"a", "b", "c"},
"nested": map[string]any{"inner": 5, "rate": 1.5, "deep": map[string]any{"x": int64(9)}},
"map_slice": []map[string]any{{"k": "v", "n": 1}, {"k": "w", "n": 2}},
"any_slice": []any{"x", 3, true, 4.5},
"shape": shape,
"nilval": nil,
"empty_map": map[string]any{},
}
got := flatRoundTrip(t, in)
if !reflect.DeepEqual(got, in) {
t.Fatalf("flat round-trip mismatch:\n got: %#v\nwant: %#v", got, in)
}
// Spot-check the load-bearing concrete types explicitly.
mustType[string](t, got, "str")
mustType[bool](t, got, "bool_t")
mustType[int](t, got, "int")
mustType[int64](t, got, "int64")
mustType[float64](t, got, "float")
mustType[float64](t, got, "float_int")
mustType[[]string](t, got, "strs")
mustType[map[string]any](t, got, "nested")
mustType[[]map[string]any](t, got, "map_slice")
mustType[[]any](t, got, "any_slice")
mustType[*contracts.Shape](t, got, "shape")
if got["nilval"] != nil {
t.Errorf("nilval: want nil, got %#v", got["nilval"])
}
}
// TestFlatCodecLargeValues exercises long keys and values that cross the
// single-byte varint boundary (> 127 bytes), proving the length prefixes
// round-trip.
func TestFlatCodecLargeValues(t *testing.T) {
big := string(bytes.Repeat([]byte("x"), 5000))
bigKey := string(bytes.Repeat([]byte("k"), 300))
in := map[string]any{
bigKey: big,
"slice": []string{big, "", big},
}
got := flatRoundTrip(t, in)
if !reflect.DeepEqual(got, in) {
t.Fatalf("large-value round-trip mismatch")
}
}
// TestFlatCodecDeterministic proves the encoding is byte-stable across
// encodes (keys are sorted), which matters for any content-hash / dedup.
func TestFlatCodecDeterministic(t *testing.T) {
in := map[string]any{
"z": 1, "a": "x", "m": []string{"p", "q"},
"nested": map[string]any{"d": 4, "b": 2, "c": 3},
}
var prev []byte
for i := 0; i < 16; i++ {
b, ok := encodeMetaFast(in)
if !ok {
t.Fatalf("encodeMetaFast bailed")
}
if prev != nil && !bytes.Equal(prev, b) {
t.Fatalf("encoding is not deterministic across encodes")
}
prev = b
}
}
// TestEncodeMetaFallbackToJSON: a value whose type the flat codec does not
// model makes encodeMeta fall back to JSON (leading '{'), and decodeMeta
// still reads it. No data is dropped.
func TestEncodeMetaFallbackToJSON(t *testing.T) {
// uint64 is deliberately outside the modelled type set.
in := map[string]any{"weird": uint64(42), "name": "keep"}
if _, ok := encodeMetaFast(in); ok {
t.Fatal("encodeMetaFast should bail on an unmodelled value type")
}
b, err := encodeMeta(in)
if err != nil {
t.Fatalf("encodeMeta: %v", err)
}
if isFlatMeta(b) {
t.Fatalf("encodeMeta should have fallen back to JSON, got a flat blob")
}
if !isJSONObject(b) {
t.Fatalf("encodeMeta fallback did not produce a JSON object: %q", b)
}
got, err := decodeMeta(b)
if err != nil {
t.Fatalf("decodeMeta(json fallback): %v", err)
}
// The JSON fallback widens uint64 -> int (documented, lossy only for the
// exotic tail), but the string survives and no row is lost.
if got["name"] != "keep" {
t.Errorf("name not preserved through JSON fallback: %#v", got["name"])
}
if _, ok := got["weird"]; !ok {
t.Errorf("weird key dropped by JSON fallback")
}
}
// TestDecodeLegacyJSON proves rows written by the previous JSON encoder still
// decode (routed through metaWire for exact types) after the flat-codec
// switch — existing on-disk databases must keep loading.
func TestDecodeLegacyJSON(t *testing.T) {
orig := map[string]any{
"visibility": "private",
"complexity": 9,
"confidence": 1.0, // integral float — metaWire must keep it float64
"path_param_names": []string{"id"},
"last_authored": map[string]any{"timestamp": int64(1700000000)},
}
blob, err := json.Marshal(orig)
if err != nil {
t.Fatalf("json.Marshal: %v", err)
}
if isFlatMeta(blob) {
t.Fatalf("JSON blob unexpectedly looks like a flat blob")
}
got, err := decodeMeta(blob)
if err != nil {
t.Fatalf("decodeMeta(json): %v", err)
}
mustType[string](t, got, "visibility")
mustType[int](t, got, "complexity")
mustType[float64](t, got, "confidence")
mustType[[]string](t, got, "path_param_names")
la, ok := got["last_authored"].(map[string]any)
if !ok {
t.Fatalf("last_authored: want map[string]any, got %T", got["last_authored"])
}
mustType[int64](t, la, "timestamp")
}
// TestDecodeMetaFastMalformed: corrupt / truncated flat blobs return an error
// rather than panicking — a single bad row must not crash a store scan.
func TestDecodeMetaFastMalformed(t *testing.T) {
good, ok := encodeMetaFast(map[string]any{"k": "value", "n": 7, "s": []string{"a", "b"}})
if !ok {
t.Fatalf("encodeMetaFast bailed")
}
cases := map[string][]byte{
"magic only": {metaFlatMagic0, metaFlatVersion},
"count then nothing": {metaFlatMagic0, metaFlatVersion, 0x05},
"truncated mid-blob": good[:len(good)-3],
"unknown value tag": {metaFlatMagic0, metaFlatVersion, 0x01, 0x01, 'k', 0x7E},
"giant key length": {metaFlatMagic0, metaFlatVersion, 0x01, 0xFF, 0xFF, 0xFF, 0xFF, 0x0F},
}
for name, blob := range cases {
t.Run(name, func(t *testing.T) {
_, err := decodeMeta(blob)
if err == nil {
t.Errorf("expected an error for %q, got nil", name)
}
})
}
}
// TestStoreReloadMetaFidelity is the wired-path proof: persist a node and an
// edge with rich Meta through the real store, reopen it (warm restart), and
// assert Meta is byte-for-byte type-identical. Also runs PRAGMA
// integrity_check.
func TestStoreReloadMetaFidelity(t *testing.T) {
path := filepath.Join(t.TempDir(), "store.sqlite")
nodeMeta := map[string]any{
"complexity": 7,
"loop_depth": 2,
"confidence": 0.875,
"coverage_pct": 1.0, // integral float
"candidate_count": 2,
"path_param_names": []string{"id", "org"},
"status_codes": []string{"200", "404"},
"churn": map[string]any{"commit_count": 12, "churn_rate": 2.0, "last_author": "a@b.c"},
"last_authored": map[string]any{"timestamp": int64(1700000000), "email": "x@y.z"},
"response_envelope": []map[string]any{{"name": "data", "n": 1}},
"shape": &contracts.Shape{
Kind: "struct",
Fields: []contracts.ShapeField{{Name: "id", Type: "int64", Required: true}},
Notes: []string{"partial"},
},
"unicode_doc": "héllo 世界 🚀",
"is_generated": false,
}
edgeMeta := map[string]any{
"candidate_count": 3,
"similarity": 0.5,
"score": 1.0, // integral float
"count": 5,
"clone_tokens": 128,
"synthesized_by": "grpc",
"arg_names": []string{"ctx", "req"},
}
func() {
s, err := Open(path)
if err != nil {
t.Fatalf("Open: %v", err)
}
defer s.Close()
s.AddNode(&graph.Node{ID: "n1", Kind: "function", Name: "Foo", FilePath: "f.go", Meta: cloneMeta(nodeMeta)})
s.AddNode(&graph.Node{ID: "n2", Kind: "function", Name: "Bar", FilePath: "f.go"})
s.AddEdge(&graph.Edge{From: "n1", To: "n2", Kind: "calls", FilePath: "f.go", Line: 10, Meta: cloneMeta(edgeMeta)})
}()
// Reopen — the warm-restart path that reads every blob back.
s, err := Open(path)
if err != nil {
t.Fatalf("reopen: %v", err)
}
defer s.Close()
n := s.GetNode("n1")
if n == nil {
t.Fatal("GetNode(n1) = nil after reload")
}
if !reflect.DeepEqual(n.Meta, nodeMeta) {
t.Fatalf("node Meta mismatch after reload:\n got: %#v\nwant: %#v", n.Meta, nodeMeta)
}
edges := s.GetOutEdges("n1")
var got *graph.Edge
for _, e := range edges {
if e.To == "n2" && e.Kind == "calls" {
got = e
break
}
}
if got == nil {
t.Fatalf("edge n1->n2 not found after reload (got %d edges)", len(edges))
}
if !reflect.DeepEqual(got.Meta, edgeMeta) {
t.Fatalf("edge Meta mismatch after reload:\n got: %#v\nwant: %#v", got.Meta, edgeMeta)
}
var res string
if err := s.db.QueryRow(`PRAGMA integrity_check`).Scan(&res); err != nil {
t.Fatalf("integrity_check: %v", err)
}
if res != "ok" {
t.Fatalf("integrity_check = %q, want ok", res)
}
}
func cloneMeta(m map[string]any) map[string]any {
out := make(map[string]any, len(m))
for k, v := range m {
out[k] = v
}
return out
}
func mustType[T any](t *testing.T, m map[string]any, key string) {
t.Helper()
v, ok := m[key]
if !ok {
t.Errorf("%s: missing from decoded map", key)
return
}
if _, ok := v.(T); !ok {
var zero T
t.Errorf("%s: want type %T, got %T (value %v)", key, zero, v, v)
}
}
// -- benchmarks -----------------------------------------------------------
var metaSink any
// benchMetaSample is a representative node/edge meta map restricted to the
// types gob auto-registers (scalars + []string), so all three encoders run on
// identical input for an apples-to-apples comparison. Shape / nested-map /
// map-slice values also ride the flat path (see the round-trip tests); they
// are omitted here only because gob refuses unregistered interface types.
func benchMetaSample() map[string]any {
return map[string]any{
"complexity": 7,
"loop_depth": 2,
"parse_errors": 0,
"position": 3,
"line": 42,
"confidence": 0.875,
"coverage_pct": 83.5,
"candidate_count": 2,
"count": 5,
"clone_tokens": 128,
"timestamp": int64(1700000000),
"path_param_names": []string{"id", "org"},
"status_codes": []string{"200", "404"},
"signature": "func F(ctx context.Context, x int) (T, error)",
"some_plugin_flag": "go_linkname",
"is_generated": false,
"synthesized_by": "grpc",
}
}
func BenchmarkEncodeMetaGob(b *testing.B) {
m := benchMetaSample()
b.ReportAllocs()
b.ResetTimer()
for i := 0; i < b.N; i++ {
var buf bytes.Buffer
if err := gob.NewEncoder(&buf).Encode(m); err != nil {
b.Fatalf("gob encode: %v", err)
}
metaSink = buf.Bytes()
}
}
func BenchmarkEncodeMetaJSON(b *testing.B) {
m := benchMetaSample()
b.ReportAllocs()
b.ResetTimer()
for i := 0; i < b.N; i++ {
out, err := json.Marshal(m)
if err != nil {
b.Fatalf("json marshal: %v", err)
}
metaSink = out
}
}
func BenchmarkEncodeMetaFlat(b *testing.B) {
m := benchMetaSample()
b.ReportAllocs()
b.ResetTimer()
for i := 0; i < b.N; i++ {
out, ok := encodeMetaFast(m)
if !ok {
b.Fatal("encodeMetaFast bailed")
}
metaSink = out
}
}
func BenchmarkDecodeMetaGob(b *testing.B) {
var buf bytes.Buffer
if err := gob.NewEncoder(&buf).Encode(benchMetaSample()); err != nil {
b.Fatalf("gob encode: %v", err)
}
blob := buf.Bytes()
b.ReportAllocs()
b.ResetTimer()
for i := 0; i < b.N; i++ {
m, err := decodeMeta(blob)
if err != nil {
b.Fatalf("decodeMeta: %v", err)
}
metaSink = m
}
}
func BenchmarkDecodeMetaJSON(b *testing.B) {
blob, err := json.Marshal(benchMetaSample())
if err != nil {
b.Fatalf("json marshal: %v", err)
}
b.ReportAllocs()
b.ResetTimer()
for i := 0; i < b.N; i++ {
m, err := decodeMeta(blob)
if err != nil {
b.Fatalf("decodeMeta: %v", err)
}
metaSink = m
}
}
func BenchmarkDecodeMetaFlat(b *testing.B) {
blob, ok := encodeMetaFast(benchMetaSample())
if !ok {
b.Fatal("encodeMetaFast bailed")
}
b.ReportAllocs()
b.ResetTimer()
for i := 0; i < b.N; i++ {
m, err := decodeMeta(blob)
if err != nil {
b.Fatalf("decodeMeta: %v", err)
}
metaSink = m
}
}
File diff suppressed because it is too large Load Diff
@@ -0,0 +1,176 @@
package store_sqlite
import (
"bytes"
"encoding/gob"
"reflect"
"testing"
"github.com/zzet/gortex/internal/contracts"
)
// roundTrip encodes Meta with the flat codec and decodes it back, the
// persist->reload path every reader sees after a daemon restart / store
// hydration.
func roundTrip(t *testing.T, in map[string]any) map[string]any {
t.Helper()
b, err := encodeMeta(in)
if err != nil {
t.Fatalf("encodeMeta: %v", err)
}
if !isFlatMeta(b) {
t.Fatalf("encodeMeta did not produce a flat-codec blob: %q", b)
}
out, err := decodeMeta(b)
if err != nil {
t.Fatalf("decodeMeta: %v", err)
}
return out
}
// TestMetaRoundTripExactTypes is the fidelity canary: every key the audit
// found read with a raw type-assertion must survive a JSON round-trip with
// its exact Go type, or the corresponding reader silently breaks.
func TestMetaRoundTripExactTypes(t *testing.T) {
shape := &contracts.Shape{
Kind: "struct",
Fields: []contracts.ShapeField{{Name: "id", Type: "int64", Required: true}},
Notes: []string{"partial"},
}
node := map[string]any{
"signature": "func F(x int) error",
"visibility": "public",
"doc": "F does a thing.",
"external": true,
"complexity": 7,
"loop_depth": 2,
"parse_errors": 0,
"position": 3,
"line": 42,
"confidence": 1.0, // integral float — must stay float64
"coverage_pct": 83.5,
"shape": shape,
"response_envelope": []map[string]any{{"name": "data", "type": "User"}},
"path_param_names": []string{"id", "org"},
"query_params": []string{"limit"},
"status_codes": []string{"200", "404"},
"churn": map[string]any{"commit_count": 12, "age_days": 365, "churn_rate": 2.0, "last_author": "a@b.c"},
"coverage": map[string]any{"num_stmt": 40, "hit": 33},
"last_authored": map[string]any{"timestamp": int64(1700000000), "email": "x@y.z"},
"some_plugin_flag": "go_linkname", // Extra tail (string)
"is_generated": false, // Extra tail (bool)
}
got := roundTrip(t, node)
assertType[int](t, got, "complexity", 7)
assertType[int](t, got, "loop_depth", 2)
assertType[int](t, got, "parse_errors", 0)
assertType[int](t, got, "position", 3)
assertType[int](t, got, "line", 42)
assertType[float64](t, got, "confidence", 1.0)
assertType[float64](t, got, "coverage_pct", 83.5)
assertType[string](t, got, "signature", "func F(x int) error")
assertType[string](t, got, "visibility", "public")
assertType[bool](t, got, "external", true)
assertType[string](t, got, "some_plugin_flag", "go_linkname")
assertType[bool](t, got, "is_generated", false)
// Shape must rebuild as *contracts.Shape, not map[string]any.
gotShape, ok := got["shape"].(*contracts.Shape)
if !ok {
t.Fatalf("shape: want *contracts.Shape, got %T", got["shape"])
}
if !reflect.DeepEqual(gotShape, shape) {
t.Errorf("shape mismatch: %+v vs %+v", gotShape, shape)
}
// response_envelope must be []map[string]any, not []any.
if _, ok := got["response_envelope"].([]map[string]any); !ok {
t.Errorf("response_envelope: want []map[string]any, got %T", got["response_envelope"])
}
// []string keys.
for _, k := range []string{"path_param_names", "query_params", "status_codes"} {
if _, ok := got[k].([]string); !ok {
t.Errorf("%s: want []string, got %T", k, got[k])
}
}
// Nested map children keep exact types.
churn := got["churn"].(map[string]any)
assertType[int](t, churn, "commit_count", 12)
assertType[int](t, churn, "age_days", 365)
assertType[float64](t, churn, "churn_rate", 2.0) // integral float, nested
assertType[string](t, churn, "last_author", "a@b.c")
cov := got["coverage"].(map[string]any)
assertType[int](t, cov, "num_stmt", 40)
assertType[int](t, cov, "hit", 33)
la := got["last_authored"].(map[string]any)
assertType[int64](t, la, "timestamp", int64(1700000000))
}
func TestEdgeMetaRoundTripExactTypes(t *testing.T) {
edge := map[string]any{
"candidate_count": 2,
"similarity": 0.875,
"score": 1.0, // integral float — must stay float64
"count": 5,
"clone_tokens": 128,
"synthesized_by": "grpc", // Extra tail
}
got := roundTrip(t, edge)
assertType[int](t, got, "candidate_count", 2)
assertType[float64](t, got, "similarity", 0.875)
assertType[float64](t, got, "score", 1.0)
assertType[int](t, got, "count", 5)
assertType[int](t, got, "clone_tokens", 128)
assertType[string](t, got, "synthesized_by", "grpc")
}
// TestDecodeLegacyGob proves existing on-disk gob blobs still decode.
func TestDecodeLegacyGob(t *testing.T) {
orig := map[string]any{"visibility": "private", "complexity": 9, "confidence": 1.0}
var buf bytes.Buffer
if err := gob.NewEncoder(&buf).Encode(orig); err != nil {
t.Fatalf("gob encode: %v", err)
}
got, err := decodeMeta(buf.Bytes())
if err != nil {
t.Fatalf("decodeMeta(gob): %v", err)
}
// gob preserves exact types natively.
assertType[string](t, got, "visibility", "private")
assertType[int](t, got, "complexity", 9)
assertType[float64](t, got, "confidence", 1.0)
}
func TestEncodeMetaEmpty(t *testing.T) {
b, err := encodeMeta(nil)
if err != nil || b != nil {
t.Fatalf("encodeMeta(nil) = %q, %v; want nil, nil", b, err)
}
b, err = encodeMeta(map[string]any{})
if err != nil || b != nil {
t.Fatalf("encodeMeta(empty) = %q, %v; want nil, nil", b, err)
}
m, err := decodeMeta(nil)
if err != nil || m != nil {
t.Fatalf("decodeMeta(nil) = %v, %v; want nil, nil", m, err)
}
}
func assertType[T comparable](t *testing.T, m map[string]any, key string, want T) {
t.Helper()
v, ok := m[key]
if !ok {
t.Errorf("%s: missing from decoded map", key)
return
}
got, ok := v.(T)
if !ok {
t.Errorf("%s: want type %T, got %T (value %v)", key, want, v, v)
return
}
if got != want {
t.Errorf("%s: want %v, got %v", key, want, got)
}
}
@@ -0,0 +1,252 @@
package store_sqlite
import (
"database/sql"
"path/filepath"
"strings"
"testing"
"github.com/zzet/gortex/internal/graph"
)
// TestPromotedColumns_RoundTrip verifies the promoted keys land in their
// columns, are stripped from the JSON blob, and restore into Meta with
// exact types — while non-promoted keys stay in the blob.
func TestPromotedColumns_RoundTrip(t *testing.T) {
s, err := Open(filepath.Join(t.TempDir(), "p.sqlite"))
if err != nil {
t.Fatal(err)
}
defer s.Close()
s.AddNode(&graph.Node{
ID: "f.go::F", Kind: graph.KindFunction, Name: "F", FilePath: "f.go",
Meta: map[string]any{
"signature": "func F()",
"visibility": "public",
"doc": "F docs",
"external": true,
"complexity": 5, // non-promoted — must stay in the blob
},
})
n := s.GetNode("f.go::F")
if n == nil {
t.Fatal("GetNode returned nil")
}
assertType[string](t, n.Meta, "signature", "func F()")
assertType[string](t, n.Meta, "visibility", "public")
assertType[string](t, n.Meta, "doc", "F docs")
assertType[bool](t, n.Meta, "external", true)
assertType[int](t, n.Meta, "complexity", 5)
var sig, vis, doc sql.NullString
var ext sql.NullBool
var blob []byte
row := s.db.QueryRow(`SELECT signature, visibility, doc, external, meta FROM nodes WHERE id=?`, "f.go::F")
if err := row.Scan(&sig, &vis, &doc, &ext, &blob); err != nil {
t.Fatal(err)
}
if !sig.Valid || sig.String != "func F()" {
t.Errorf("signature column = %+v", sig)
}
if !ext.Valid || !ext.Bool {
t.Errorf("external column = %+v", ext)
}
blobStr := string(blob)
for _, k := range []string{"signature", "visibility", "external"} {
if strings.Contains(blobStr, k) {
t.Errorf("blob still contains promoted key %q: %s", k, blobStr)
}
}
if !strings.Contains(blobStr, "complexity") {
t.Errorf("blob missing non-promoted key complexity: %s", blobStr)
}
}
// TestPromotedColumns_NewColumns verifies the added promoted meta columns
// (is_async / is_static / is_abstract / is_exported / return_type / updated_at)
// and the struct-field columns (start_column / end_column) round-trip through
// their typed columns, and that a SQL filter on is_async resolves WITHOUT
// decoding the meta blob — the indexable-column acceptance.
func TestPromotedColumns_NewColumns(t *testing.T) {
s, err := Open(filepath.Join(t.TempDir(), "p.sqlite"))
if err != nil {
t.Fatal(err)
}
defer s.Close()
s.AddNode(&graph.Node{
ID: "f.go::Async", Kind: graph.KindFunction, Name: "Async", FilePath: "f.go",
StartLine: 10, EndLine: 20, StartColumn: 4, EndColumn: 1,
Meta: map[string]any{
"is_async": true,
"is_static": false,
"is_abstract": true,
"is_exported": true,
"return_type": "error",
"updated_at": int64(1700000000),
"complexity": 3, // non-promoted — stays in the blob
},
})
// A second, non-async node to prove the filter is selective.
s.AddNode(&graph.Node{
ID: "f.go::Sync", Kind: graph.KindFunction, Name: "Sync", FilePath: "f.go",
Meta: map[string]any{"is_async": false},
})
// Read-back restores every promoted key into Meta with its exact type,
// and the struct columns into the Node fields.
n := s.GetNode("f.go::Async")
if n == nil {
t.Fatal("GetNode returned nil")
}
assertType[bool](t, n.Meta, "is_async", true)
assertType[bool](t, n.Meta, "is_static", false)
assertType[bool](t, n.Meta, "is_abstract", true)
assertType[bool](t, n.Meta, "is_exported", true)
assertType[string](t, n.Meta, "return_type", "error")
assertType[int64](t, n.Meta, "updated_at", int64(1700000000))
assertType[int](t, n.Meta, "complexity", 3)
if n.StartColumn != 4 || n.EndColumn != 1 {
t.Errorf("column offsets = (%d,%d), want (4,1)", n.StartColumn, n.EndColumn)
}
// The promoted keys are stripped from the JSON blob; complexity is not.
var blob []byte
if err := s.db.QueryRow(`SELECT meta FROM nodes WHERE id=?`, "f.go::Async").Scan(&blob); err != nil {
t.Fatal(err)
}
for _, k := range []string{"is_async", "is_abstract", "return_type", "updated_at"} {
if strings.Contains(string(blob), k) {
t.Errorf("blob still contains promoted key %q: %s", k, blob)
}
}
if !strings.Contains(string(blob), "complexity") {
t.Errorf("blob missing non-promoted key complexity: %s", blob)
}
// Acceptance: a SQL filter on the typed column resolves the node without
// touching the meta blob (only id is selected).
var id string
var startCol, endCol int
if err := s.db.QueryRow(
`SELECT id, start_column, end_column FROM nodes WHERE is_async = 1`,
).Scan(&id, &startCol, &endCol); err != nil {
t.Fatalf("is_async column filter failed: %v", err)
}
if id != "f.go::Async" || startCol != 4 || endCol != 1 {
t.Errorf("filter result = (%q,%d,%d), want (f.go::Async,4,1)", id, startCol, endCol)
}
}
// TestPromotedColumns_SemanticType verifies semantic_type/semantic_source —
// promoted alongside signature/visibility/etc. so enrichment can query the
// unstamped subset by column instead of decoding every node's meta blob —
// round-trip through their columns and out of the JSON blob.
func TestPromotedColumns_SemanticType(t *testing.T) {
s, err := Open(filepath.Join(t.TempDir(), "p.sqlite"))
if err != nil {
t.Fatal(err)
}
defer s.Close()
s.AddNode(&graph.Node{
ID: "f.go::T", Kind: graph.KindFunction, Name: "T", FilePath: "f.go",
Meta: map[string]any{
"semantic_type": "string",
"semantic_source": "lsp-gopls",
"complexity": 5, // non-promoted — must stay in the blob
},
})
n := s.GetNode("f.go::T")
if n == nil {
t.Fatal("GetNode returned nil")
}
assertType[string](t, n.Meta, "semantic_type", "string")
assertType[string](t, n.Meta, "semantic_source", "lsp-gopls")
assertType[int](t, n.Meta, "complexity", 5)
var st, ss sql.NullString
var blob []byte
row := s.db.QueryRow(`SELECT semantic_type, semantic_source, meta FROM nodes WHERE id=?`, "f.go::T")
if err := row.Scan(&st, &ss, &blob); err != nil {
t.Fatal(err)
}
if !st.Valid || st.String != "string" {
t.Errorf("semantic_type column = %+v", st)
}
if !ss.Valid || ss.String != "lsp-gopls" {
t.Errorf("semantic_source column = %+v", ss)
}
blobStr := string(blob)
for _, k := range []string{"semantic_type", "semantic_source"} {
if strings.Contains(blobStr, k) {
t.Errorf("blob still contains promoted key %q: %s", k, blobStr)
}
}
if !strings.Contains(blobStr, "complexity") {
t.Errorf("blob missing non-promoted key complexity: %s", blobStr)
}
}
// TestPromotedColumns_ExternalFalse guards the NULL-vs-false distinction:
// a stored false must round-trip as false, not vanish.
func TestPromotedColumns_ExternalFalse(t *testing.T) {
s, err := Open(filepath.Join(t.TempDir(), "p.sqlite"))
if err != nil {
t.Fatal(err)
}
defer s.Close()
s.AddNode(&graph.Node{
ID: "x", Kind: graph.KindFunction, Name: "x", FilePath: "x.go",
Meta: map[string]any{"external": false},
})
n := s.GetNode("x")
if n == nil {
t.Fatal("nil")
}
v, ok := n.Meta["external"].(bool)
if !ok || v != false {
t.Errorf("external false: got %v (%T)", n.Meta["external"], n.Meta["external"])
}
}
// TestPromotedColumns_Migration verifies ensureNodeColumns adds the
// promoted columns to a database created with the pre-promotion schema.
func TestPromotedColumns_Migration(t *testing.T) {
path := filepath.Join(t.TempDir(), "old.sqlite")
raw, err := sql.Open("sqlite", path)
if err != nil {
t.Fatal(err)
}
_, err = raw.Exec(`CREATE TABLE nodes (
id TEXT PRIMARY KEY, kind TEXT NOT NULL, name TEXT NOT NULL,
qual_name TEXT NOT NULL DEFAULT '', file_path TEXT NOT NULL,
start_line INTEGER NOT NULL DEFAULT 0, end_line INTEGER NOT NULL DEFAULT 0,
language TEXT NOT NULL DEFAULT '', repo_prefix TEXT NOT NULL DEFAULT '',
workspace_id TEXT NOT NULL DEFAULT '', project_id TEXT NOT NULL DEFAULT '',
meta BLOB
) WITHOUT ROWID`)
if err != nil {
t.Fatal(err)
}
_ = raw.Close()
s, err := Open(path)
if err != nil {
t.Fatalf("Open old-schema db: %v", err)
}
defer s.Close()
s.AddNode(&graph.Node{
ID: "m", Kind: graph.KindFunction, Name: "m", FilePath: "m.go",
Meta: map[string]any{"signature": "sig", "external": true},
})
n := s.GetNode("m")
if n == nil {
t.Fatal("nil after migration")
}
assertType[string](t, n.Meta, "signature", "sig")
assertType[bool](t, n.Meta, "external", true)
}
@@ -0,0 +1,45 @@
package store_sqlite_test
import (
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/zzet/gortex/internal/graph"
)
// TestGetOutEdgesLight_SkipsMetaKeepsEndpoints proves the light out-edge
// fetch returns the same endpoints/kind/line as GetOutEdges while leaving
// Meta nil — it must never pay the per-edge meta JSON decode. This is the
// fetch findCallTarget uses on the dataflow hot path.
func TestGetOutEdgesLight_SkipsMetaKeepsEndpoints(t *testing.T) {
s := openTestStore(t)
want := &graph.Edge{
From: "pkg/x.go::Caller",
To: "pkg/x.go::Callee",
Kind: graph.EdgeCalls,
FilePath: "pkg/x.go",
Line: 42,
Meta: map[string]any{"call_line": 42, "callee_target": "unresolved::Callee"},
}
s.AddEdge(want)
full := s.GetOutEdges("pkg/x.go::Caller")
require.Len(t, full, 1)
require.NotNil(t, full[0].Meta, "GetOutEdges must decode Meta")
assert.Equal(t, "unresolved::Callee", full[0].Meta["callee_target"])
light := s.GetOutEdgesLight("pkg/x.go::Caller")
require.Len(t, light, 1)
assert.Equal(t, full[0].From, light[0].From)
assert.Equal(t, full[0].To, light[0].To)
assert.Equal(t, full[0].Kind, light[0].Kind)
assert.Equal(t, full[0].Line, light[0].Line)
assert.Equal(t, full[0].FilePath, light[0].FilePath)
assert.Nil(t, light[0].Meta, "light fetch must not decode the meta blob")
// A node with no out-edges returns nothing on both paths.
assert.Empty(t, s.GetOutEdgesLight("pkg/x.go::Callee"))
}
@@ -0,0 +1,58 @@
package store_sqlite
import (
"path/filepath"
"testing"
"github.com/zzet/gortex/internal/graph"
)
// TestProxyNodes_NeverReachDisk validates that cross-daemon proxy-edge
// nodes (and their edges) are dropped at the single durable write boundary
// (AddNode / AddBatch), so a warm restart over the store never sees them,
// while every real node round-trips intact.
func TestProxyNodes_NeverReachDisk(t *testing.T) {
path := filepath.Join(t.TempDir(), "store.sqlite")
s, err := Open(path)
if err != nil {
t.Fatalf("open: %v", err)
}
real := &graph.Node{ID: "local/a.go::Foo", Kind: graph.KindFunction, Name: "Foo"}
proxy := &graph.Node{
ID: graph.ProxyNodeID("remoteB", "rb/x.go::Bar"),
Kind: graph.KindFunction, Name: "Bar",
Origin: "remote:remoteB", Stub: true,
}
proxyEdge := &graph.Edge{From: real.ID, To: proxy.ID, Kind: graph.EdgeCalls}
// Mix proxy + real through AddBatch...
s.AddBatch([]*graph.Node{real, proxy}, []*graph.Edge{proxyEdge})
// ...and a proxy through the per-node path.
s.AddNode(&graph.Node{
ID: graph.ProxyNodeID("remoteC", "rc/y.go::Baz"),
Kind: graph.KindFunction, Name: "Baz",
Origin: "remote:remoteC", Stub: true,
})
_ = s.Close()
// Reopen — a warm restart sees the durable store only.
s2, err := Open(path)
if err != nil {
t.Fatalf("reopen: %v", err)
}
defer s2.Close()
if s2.GetNode(proxy.ID) != nil {
t.Error("proxy node must not be persisted")
}
if s2.GetNode(graph.ProxyNodeID("remoteC", "rc/y.go::Baz")) != nil {
t.Error("proxy node added via AddNode must not be persisted")
}
if s2.GetNode(real.ID) == nil {
t.Error("the real node must round-trip intact")
}
if outs := s2.GetOutEdges(real.ID); len(outs) != 0 {
t.Errorf("the edge to a proxy node must not be persisted; got %d", len(outs))
}
}
@@ -0,0 +1,74 @@
package store_sqlite
import (
"database/sql"
"fmt"
"os"
"github.com/zzet/gortex/internal/graph"
)
// ReadRepoIndexStates opens the SQLite store at path read-only and returns
// every repo_index_state freshness row keyed by repo_prefix.
//
// It is a deliberately lightweight side door for read-only callers (notably
// `gortex repos`) that must inspect index freshness WITHOUT going through
// Open — which runs schema migrations, alters columns, starts a checkpoint
// goroutine, and (on a version mismatch) can refuse to open or rebuild the
// file. None of that is appropriate for a status command that may run while
// a daemon holds the same store open.
//
// The connection is query-only and inherits the database's existing journal
// mode, so it reads safely alongside a running daemon (WAL permits concurrent
// readers). A missing store file, or a database that predates the
// repo_index_state table, both yield an empty map and a nil error — that is
// "nothing recorded yet", not a failure, so the caller can fall back to other
// freshness sources rather than surfacing an error to the user.
func ReadRepoIndexStates(path string) (map[string]graph.RepoIndexState, error) {
if _, err := os.Stat(path); err != nil {
if os.IsNotExist(err) {
return map[string]graph.RepoIndexState{}, nil
}
return nil, fmt.Errorf("stat sqlite store %q: %w", path, err)
}
// query_only blocks accidental writes; busy_timeout keeps a brief read
// from erroring out if the daemon happens to hold the write lock for a
// moment. We deliberately do NOT set journal_mode — forcing it could try
// to switch the live database's mode; inheriting the on-disk WAL mode is
// exactly what a concurrent reader wants.
dsn := path + "?_pragma=busy_timeout(2000)&_pragma=query_only(1)"
db, err := sql.Open("sqlite", dsn)
if err != nil {
return nil, fmt.Errorf("open sqlite store %q: %w", path, err)
}
defer db.Close()
db.SetMaxOpenConns(1)
rows, err := db.Query(`
SELECT repo_prefix, indexed_sha, dirty, indexed_at, workspace_fp, node_count, edge_count, extractor_versions
FROM repo_index_state`)
if err != nil {
// A store written before the repo_index_state table existed (or any
// other read error) is treated as "no freshness recorded yet" — a
// status command must never hard-fail on a degraded cache.
return map[string]graph.RepoIndexState{}, nil
}
defer rows.Close()
out := map[string]graph.RepoIndexState{}
for rows.Next() {
var st graph.RepoIndexState
var dirty int
if err := rows.Scan(&st.RepoPrefix, &st.IndexedSHA, &dirty, &st.IndexedAt,
&st.WorkspaceFP, &st.NodeCount, &st.EdgeCount, &st.ExtractorVersions); err != nil {
return nil, fmt.Errorf("scan repo_index_state: %w", err)
}
st.Dirty = dirty != 0
out[st.RepoPrefix] = st
}
if err := rows.Err(); err != nil {
return nil, fmt.Errorf("iterate repo_index_state: %w", err)
}
return out, nil
}
@@ -0,0 +1,78 @@
package store_sqlite_test
import (
"path/filepath"
"testing"
"github.com/stretchr/testify/require"
"github.com/zzet/gortex/internal/graph"
"github.com/zzet/gortex/internal/graph/store_sqlite"
)
// A store file that does not exist yet reads back as an empty map, not an
// error — "nothing indexed", so callers fall back to other sources.
func TestReadRepoIndexStates_MissingFile(t *testing.T) {
got, err := store_sqlite.ReadRepoIndexStates(filepath.Join(t.TempDir(), "absent.sqlite"))
require.NoError(t, err)
require.Empty(t, got)
}
// The read-only reader returns exactly the rows SetRepoIndexState wrote,
// keyed by repo prefix, including the empty (lone-repo) prefix.
func TestReadRepoIndexStates_RoundTrip(t *testing.T) {
path := filepath.Join(t.TempDir(), "is.sqlite")
s, err := store_sqlite.Open(path)
require.NoError(t, err)
require.NoError(t, s.SetRepoIndexState(graph.RepoIndexState{
RepoPrefix: "alpha", IndexedSHA: "aaa111", Dirty: false, IndexedAt: 1700000000,
WorkspaceFP: "fp-a", NodeCount: 10, EdgeCount: 20, ExtractorVersions: `{"go":1}`,
}))
require.NoError(t, s.SetRepoIndexState(graph.RepoIndexState{
RepoPrefix: "", IndexedSHA: "bbb222", Dirty: true, IndexedAt: 1700000001,
}))
require.NoError(t, s.Close())
got, err := store_sqlite.ReadRepoIndexStates(path)
require.NoError(t, err)
require.Len(t, got, 2)
alpha, ok := got["alpha"]
require.True(t, ok)
require.Equal(t, "aaa111", alpha.IndexedSHA)
require.False(t, alpha.Dirty)
require.EqualValues(t, 1700000000, alpha.IndexedAt)
require.Equal(t, 10, alpha.NodeCount)
lone, ok := got[""]
require.True(t, ok)
require.Equal(t, "bbb222", lone.IndexedSHA)
require.True(t, lone.Dirty)
}
// Reading is non-destructive and repeatable while the underlying store is
// reopened for writing — the read path must never lock the writer out.
func TestReadRepoIndexStates_ConcurrentWithOpenStore(t *testing.T) {
path := filepath.Join(t.TempDir(), "is.sqlite")
s, err := store_sqlite.Open(path)
require.NoError(t, err)
t.Cleanup(func() { _ = s.Close() })
require.NoError(t, s.SetRepoIndexState(graph.RepoIndexState{
RepoPrefix: "live", IndexedSHA: "c0ffee", IndexedAt: 1700000002,
}))
// Read while the writer store is still open (WAL allows concurrent readers).
got, err := store_sqlite.ReadRepoIndexStates(path)
require.NoError(t, err)
require.Equal(t, "c0ffee", got["live"].IndexedSHA)
// A subsequent write is still possible — the reader did not wedge the writer.
require.NoError(t, s.SetRepoIndexState(graph.RepoIndexState{
RepoPrefix: "live", IndexedSHA: "feedface", IndexedAt: 1700000003,
}))
got, err = store_sqlite.ReadRepoIndexStates(path)
require.NoError(t, err)
require.Equal(t, "feedface", got["live"].IndexedSHA)
}
+518
View File
@@ -0,0 +1,518 @@
package store_sqlite
import "database/sql"
// isUnresolvedColumnDDL is the edges.is_unresolved generated column: a
// VIRTUAL, indexed boolean mirroring graph.IsUnresolvedTarget's two shapes
// (the bare `unresolved::Name` prefix and the multi-repo COPY-rewrite
// `<repoPrefix>::unresolved::Name` infix), computed by SQLite itself from
// to_id — no Go call site has to remember to keep it in sync. VIRTUAL, not
// STORED: SQLite refuses `ALTER TABLE ADD COLUMN ... STORED` on a non-empty
// table ("cannot add a STORED column"), which every real installed store is.
// VIRTUAL has no such restriction and is just as fast here — the read path
// always goes through the index below, and an index always stores its own
// materialised key values regardless of whether the underlying column is
// virtual or stored. Added via ensureEdgeColumns (ALTER TABLE) rather than
// baked into schemaSQL's CREATE TABLE so one code path handles both a fresh
// DB (column missing right after CREATE TABLE) and an existing one (column
// missing from before this was introduced) identically — see
// ensureNodeColumns for the same pattern on the nodes table.
//
// Measured on a real 26-repo store (2.57M edges, 847,684 unresolved, ~33%
// selectivity): replacing the OR'd `to_id` range/LIKE query with
// `is_unresolved = 1` cut EdgesWithUnresolvedTarget from 7.96s to 2.95s
// (2.7x). The prior approach of splitting the OR into two to_id-based
// queries (one indexed range, one LIKE) was WORSE (13.49s) despite a
// better-looking EXPLAIN QUERY PLAN: at ~33% selectivity the to_id index's
// matching rows are ordered by string value, so the mandatory per-row
// bookmark lookup back into the main table is effectively random I/O. The
// boolean column's matching rows are all rowid-tie-broken (identical index
// key), so its bookmark lookups land in ascending rowid order — sequential,
// not random. Same "SEARCH ... USING INDEX" in EXPLAIN QUERY PLAN either way;
// only real measurement told them apart.
const isUnresolvedColumnDDL = `is_unresolved INTEGER GENERATED ALWAYS AS (
CASE WHEN (to_id >= 'unresolved::' AND to_id < 'unresolved:;') OR to_id LIKE '%::unresolved::%' THEN 1 ELSE 0 END
) VIRTUAL`
// edgeGeneratedColumns is the set of edges.* generated columns ensureEdgeColumns
// adds to a table created before they existed — which, since none of them are
// in schemaSQL's CREATE TABLE, includes a freshly created table too.
var edgeGeneratedColumns = []struct {
name string
ddl string
}{
{"is_unresolved", isUnresolvedColumnDDL},
}
// edgePromotedColumns lifts the resolver's resolve_terminal /
// resolve_terminal_reason Meta keys (see resolver/terminal.go) out of the
// meta blob into their own nullable columns — the edge-side sibling of
// promotedMetaColumns on nodes (see meta_json.go's "promoted edge columns"
// section for extractPromotedEdgeMeta/restorePromotedEdgeMeta and why a
// json_extract-derived generated column was tried first and abandoned:
// encodeMeta's common case is a custom flat binary codec, not JSON, so
// json_extract/json_valid against a real store's meta blobs evaluates to
// NULL for effectively every row). Plain (non-generated) columns, so they
// share ensureEdgeColumns' table_xinfo scan but are ordinary ALTER TABLE ADD
// COLUMN statements, not GENERATED ALWAYS AS expressions.
//
// Exists to let a future bulk classification query (replacing per-edge
// Go-side classifyTerminal calls in reconcileTerminalStamps) read the
// CURRENT terminal state as a plain indexed column instead of decoding
// Meta, and compare it against a freshly computed value to find only the
// edges whose state actually changed — reconcileTerminalStamps measured
// only ~1% of examined edges (9,599 of 833,828) ever change state.
var edgePromotedColumns = []struct {
name string
ddl string
}{
{"resolve_terminal", "resolve_terminal INTEGER"},
{"resolve_terminal_reason", "resolve_terminal_reason TEXT"},
}
// ensureEdgeColumns adds edgeGeneratedColumns + edgePromotedColumns to an
// edges table created before they existed. Mirrors ensureNodeColumns'
// PRAGMA + conditional ALTER pattern, but queries table_xinfo rather than
// table_info: table_info silently OMITS generated columns from its result
// set (verified against the pinned modernc.org/sqlite driver — a reopened
// store's is_unresolved column is invisible to table_info, so the existence
// check always came back false and every reopen re-ran the ALTER, failing
// with "duplicate column name"). table_xinfo lists every column, generated
// ones included, with an extra hidden column (3 == generated) table_info
// doesn't have — and works identically for the plain promoted columns too,
// so one scan serves both lists.
func ensureEdgeColumns(db *sql.DB) error {
rows, err := db.Query(`PRAGMA table_xinfo(edges)`)
if err != nil {
return err
}
existing := make(map[string]bool)
for rows.Next() {
var (
cid, notnull, pk, hidden int
name, ctype string
dflt sql.NullString
)
if err := rows.Scan(&cid, &name, &ctype, &notnull, &dflt, &pk, &hidden); err != nil {
_ = rows.Close()
return err
}
existing[name] = true
}
if err := rows.Err(); err != nil {
_ = rows.Close()
return err
}
_ = rows.Close()
for _, c := range edgeGeneratedColumns {
if existing[c.name] {
continue
}
if _, err := db.Exec(`ALTER TABLE edges ADD COLUMN ` + c.ddl); err != nil {
return err
}
}
for _, c := range edgePromotedColumns {
if existing[c.name] {
continue
}
if _, err := db.Exec(`ALTER TABLE edges ADD COLUMN ` + c.ddl); err != nil {
return err
}
}
return nil
}
// isStubColumnDDL is nodes.is_stub: a VIRTUAL generated column mirroring
// graph.IsStub/StubKind's id-prefix logic (stdlib:: / builtin:: /
// external_call:: / module::, bare or repo-prefixed as <repo>::<kind>::...).
// Same rationale as isUnresolvedColumnDDL: computed from the existing id
// column, no Go call site has to keep it in sync. Exists so a future
// SQL-side terminal classification (see resolveTerminalColumnDDL) can check
// "is this candidate a stub" via a plain column instead of a per-row Go
// IsStub(n.ID) call.
const isStubColumnDDL = `is_stub INTEGER GENERATED ALWAYS AS (
CASE WHEN
id LIKE 'stdlib::%' OR id LIKE 'builtin::%' OR id LIKE 'external_call::%' OR id LIKE 'module::%'
OR (
instr(id, '::') > 0 AND (
substr(id, instr(id, '::') + 2) LIKE 'stdlib::%'
OR substr(id, instr(id, '::') + 2) LIKE 'builtin::%'
OR substr(id, instr(id, '::') + 2) LIKE 'external_call::%'
OR substr(id, instr(id, '::') + 2) LIKE 'module::%'
)
)
THEN 1 ELSE 0 END
) VIRTUAL`
// nodeGeneratedColumns is the nodes-table sibling of edgeGeneratedColumns.
// Kept as its own list (and ensureNodeGeneratedColumns as its own function,
// rather than folded into ensureNodeColumns) because ensureNodeColumns
// checks existence via PRAGMA table_info, which — like the edges case
// documented on ensureEdgeColumns — silently omits generated columns.
// Reusing that function's table_info scan for is_stub would hit the exact
// same "always looks missing, ALTER re-runs, duplicate column name" bug.
var nodeGeneratedColumns = []struct {
name string
ddl string
}{
{"is_stub", isStubColumnDDL},
}
// ensureNodeGeneratedColumns adds nodeGeneratedColumns to a nodes table
// created before they existed. See ensureEdgeColumns for the table_xinfo
// vs table_info rationale this mirrors.
func ensureNodeGeneratedColumns(db *sql.DB) error {
rows, err := db.Query(`PRAGMA table_xinfo(nodes)`)
if err != nil {
return err
}
existing := make(map[string]bool)
for rows.Next() {
var (
cid, notnull, pk, hidden int
name, ctype string
dflt sql.NullString
)
if err := rows.Scan(&cid, &name, &ctype, &notnull, &dflt, &pk, &hidden); err != nil {
_ = rows.Close()
return err
}
existing[name] = true
}
if err := rows.Err(); err != nil {
_ = rows.Close()
return err
}
_ = rows.Close()
for _, c := range nodeGeneratedColumns {
if existing[c.name] {
continue
}
if _, err := db.Exec(`ALTER TABLE nodes ADD COLUMN ` + c.ddl); err != nil {
return err
}
}
return nil
}
// schemaSQL is the canonical DDL applied on Open. Statements are
// idempotent (IF NOT EXISTS) so they run cleanly against a fresh DB
// and against an existing one.
//
// Schema choices
//
// - nodes.id is the primary key; INSERT OR REPLACE on the id column
// gives idempotent re-adds with last-write-wins on every other
// column, matching the in-memory store's behaviour.
//
// - edges has a synthetic INTEGER PRIMARY KEY plus a UNIQUE
// constraint over (from_id, to_id, kind, file_path, line) -- the
// logical edge key the in-memory store uses for dedup. INSERT OR
// IGNORE on that constraint matches the in-memory "second AddEdge
// for the same key is a no-op" semantics.
//
// - meta is a JSON document (see meta_json.go). nil / empty Meta is
// stored as NULL. Four universal, hot-read node keys are promoted to
// their own nullable columns (signature / visibility / doc /
// external): they are stripped from the JSON blob on write and
// restored into Meta on read, so the in-memory map is unchanged. A
// NULL column means "not set" (legacy gob rows predate the columns
// and keep their values in the blob). Existing databases gain the
// columns via ALTER on the next Open (ensureNodeColumns).
//
// - Secondary indexes mirror the in-memory store's hot lookup paths:
// nodes_by_name -- FindNodesByName / FindNodesByNameInRepo
// nodes_by_kind -- Stats (group-by-kind)
// nodes_by_file -- GetFileNodes, EvictFile
// nodes_by_repo -- GetRepoNodes, RepoStats, EvictRepo
// (partial index -- empty repo_prefix is
// the common case and indexing it would
// be pure overhead)
// nodes_by_qual -- GetNodeByQualName, unique so duplicate
// qual_names surface as constraint errors
// edges_by_from -- GetOutEdges (kind included so RemoveEdge
// can probe by (from, kind) without a
// second hop)
// edges_by_to -- GetInEdges
const schemaSQL = `
CREATE TABLE IF NOT EXISTS nodes (
id TEXT PRIMARY KEY,
kind TEXT NOT NULL,
name TEXT NOT NULL,
qual_name TEXT NOT NULL DEFAULT '',
file_path TEXT NOT NULL,
start_line INTEGER NOT NULL DEFAULT 0,
end_line INTEGER NOT NULL DEFAULT 0,
start_column INTEGER NOT NULL DEFAULT 0,
end_column INTEGER NOT NULL DEFAULT 0,
language TEXT NOT NULL DEFAULT '',
repo_prefix TEXT NOT NULL DEFAULT '',
workspace_id TEXT NOT NULL DEFAULT '',
project_id TEXT NOT NULL DEFAULT '',
signature TEXT,
visibility TEXT,
doc TEXT,
external INTEGER,
return_type TEXT,
is_async INTEGER,
is_static INTEGER,
is_abstract INTEGER,
is_exported INTEGER,
updated_at INTEGER,
data_class TEXT,
meta BLOB
) WITHOUT ROWID;
-- nodes_by_name / _kind / _file / _repo are created from the shared
-- bulkDroppableIndexes set (see bulk_load.go), not here, so the bulk-load
-- fast path can drop and rebuild the EXACT same DDL without drift.
-- nodes_by_qual is UNIQUE — it enforces qual_name dedup on every
-- INSERT OR REPLACE, so it is never dropped and stays defined here.
CREATE UNIQUE INDEX IF NOT EXISTS nodes_by_qual ON nodes(qual_name) WHERE qual_name <> '';
CREATE TABLE IF NOT EXISTS edges (
id INTEGER PRIMARY KEY AUTOINCREMENT,
from_id TEXT NOT NULL,
to_id TEXT NOT NULL,
kind TEXT NOT NULL,
file_path TEXT NOT NULL DEFAULT '',
line INTEGER NOT NULL DEFAULT 0,
confidence REAL NOT NULL DEFAULT 1.0,
confidence_label TEXT NOT NULL DEFAULT '',
origin TEXT NOT NULL DEFAULT '',
tier TEXT NOT NULL DEFAULT '',
cross_repo INTEGER NOT NULL DEFAULT 0,
meta BLOB,
UNIQUE(from_id, to_id, kind, file_path, line)
);
-- edges_by_from / _to / _kind are created from the shared
-- bulkDroppableIndexes set (see bulk_load.go), not here, so the bulk-load
-- fast path can drop and rebuild the EXACT same DDL without drift.
-- edges_by_kind backs EdgesByKind / EdgesByKinds (resolver whole-graph
-- passes probe single kinds like provides/imports on every file save);
-- without it those are full edges-table scans — edges_by_from/to lead
-- with an id column and the partial edges_external index only covers
-- its own predicate.
CREATE TABLE IF NOT EXISTS file_mtimes (
repo_prefix TEXT NOT NULL,
file_path TEXT NOT NULL,
mtime_ns INTEGER NOT NULL,
PRIMARY KEY (repo_prefix, file_path)
) WITHOUT ROWID;
-- repo_index_state records per-repo freshness provenance written at the
-- end of a (re)index: the git revision + dirty flag the graph reflects,
-- the Merkle workspace fingerprint (Tree.Root) that gates global-pass
-- short-circuiting, node/edge counts for the index-plausibility baseline,
-- and the JSON per-language extractor versions that produced the graph.
-- One row per repo_prefix; WITHOUT ROWID — the PK index IS the table,
-- like file_mtimes / clone_shingles.
CREATE TABLE IF NOT EXISTS repo_index_state (
repo_prefix TEXT PRIMARY KEY,
indexed_sha TEXT NOT NULL DEFAULT '',
dirty INTEGER NOT NULL DEFAULT 0,
indexed_at INTEGER NOT NULL DEFAULT 0,
workspace_fp TEXT NOT NULL DEFAULT '',
node_count INTEGER NOT NULL DEFAULT 0,
edge_count INTEGER NOT NULL DEFAULT 0,
extractor_versions TEXT NOT NULL DEFAULT ''
) WITHOUT ROWID;
-- enrichment_state records, per (repo, semantic provider), the git revision
-- the graph was enriched at plus the coverage that pass reached. Enrichment
-- completion otherwise lives only in an in-memory map, so a restart forgets it
-- and re-runs full LSP hover passes for a repo whose persisted graph already
-- carries the edges. The deferred-enrichment gate reads this row and skips a
-- provider whose IndexedSHA still matches HEAD on a clean tree. One row per
-- (repo_prefix, provider); WITHOUT ROWID — the PK index IS the table, like
-- file_mtimes / repo_index_state.
CREATE TABLE IF NOT EXISTS enrichment_state (
repo_prefix TEXT NOT NULL,
provider TEXT NOT NULL,
indexed_sha TEXT NOT NULL DEFAULT '',
completed_at INTEGER NOT NULL DEFAULT 0,
coverage REAL NOT NULL DEFAULT 0,
PRIMARY KEY (repo_prefix, provider)
) WITHOUT ROWID;
-- clone_shingles is the per-symbol MinHash shingle-set sidecar. Each
-- function/method node's []uint64 shingle set is stored as a little-
-- endian BLOB (8 bytes/elem) keyed by node_id so the maintained clone-
-- detection count-min sketch can be rebuilt after a warm restart from
-- the snapshot instead of re-parsing every body. repo_prefix carries
-- the owning repo so per-repo reseeds (SELECT … WHERE repo_prefix = ?)
-- and per-repo wipes don't clobber other repos' shingle sets. node_id
-- is the PK (the join key back to nodes.id); like file_mtimes this is a
-- WITHOUT ROWID sidecar so the PK index IS the table.
CREATE TABLE IF NOT EXISTS clone_shingles (
node_id TEXT PRIMARY KEY,
repo_prefix TEXT NOT NULL DEFAULT '',
shingles BLOB
) WITHOUT ROWID;
-- constant_values is the per-KindConstant literal-value sidecar: one row
-- per constant whose RHS is a string / numeric literal, keyed by node_id
-- (the join key back to nodes.id). Lifting the value out of the JSON Meta
-- blob keeps it queryable (and out of the every-node-load decode path) so
-- the resolver can dereference a const-identifier dispatch name to its
-- value across files. file_path scopes per-file eviction on reindex;
-- repo_prefix scopes per-repo wipes. WITHOUT ROWID — the PK index IS the
-- table, like file_mtimes / clone_shingles.
CREATE TABLE IF NOT EXISTS constant_values (
node_id TEXT PRIMARY KEY,
repo_prefix TEXT NOT NULL DEFAULT '',
file_path TEXT NOT NULL DEFAULT '',
value TEXT NOT NULL DEFAULT ''
) WITHOUT ROWID;
CREATE INDEX IF NOT EXISTS constant_values_by_file ON constant_values(repo_prefix, file_path);
-- files is the per-file metadata sidecar: one row per indexed file carrying
-- the BLAKE3 content hash (the Merkle leaf), byte size, extracted node count,
-- and a JSON array of parse-error locations. The Merkle tree stays the
-- authoritative change detector; this table is queryable supplementary
-- metadata (index_health reports per-file parse errors + node counts from it).
-- PK is (repo_prefix, file_path) so a reindex replaces the row in place;
-- WITHOUT ROWID — the PK index IS the table, like file_mtimes.
CREATE TABLE IF NOT EXISTS files (
repo_prefix TEXT NOT NULL DEFAULT '',
file_path TEXT NOT NULL,
content_hash TEXT NOT NULL DEFAULT '',
size INTEGER NOT NULL DEFAULT 0,
node_count INTEGER NOT NULL DEFAULT 0,
errors TEXT NOT NULL DEFAULT '',
PRIMARY KEY (repo_prefix, file_path)
) WITHOUT ROWID;
-- files_with_errors backs the index_health "files with parse errors" rollup
-- so it scans only the (usually tiny) set of erroring files, not every row.
CREATE INDEX IF NOT EXISTS files_with_errors ON files(repo_prefix) WHERE errors <> '';
-- ref_facts is the resolved-reference sidecar: one row per reference edge
-- that resolved to a concrete target, recording the target + the provenance
-- tier that resolved it. Denormalized file_path + lang make "all reference
-- facts originating in file X" a single indexed query (the scope unit for
-- incremental re-resolution and the audit/diff surface). repo_prefix scopes
-- per-repo. PK is (repo_prefix, from_id, to_id, kind, line) so re-resolving a
-- file replaces its facts in place; WITHOUT ROWID — the PK index IS the table.
CREATE TABLE IF NOT EXISTS ref_facts (
repo_prefix TEXT NOT NULL DEFAULT '',
from_id TEXT NOT NULL,
to_id TEXT NOT NULL,
kind TEXT NOT NULL,
ref_name TEXT NOT NULL DEFAULT '',
line INTEGER NOT NULL DEFAULT 0,
origin TEXT NOT NULL DEFAULT '',
tier TEXT NOT NULL DEFAULT '',
candidates TEXT NOT NULL DEFAULT '',
file_path TEXT NOT NULL DEFAULT '',
lang TEXT NOT NULL DEFAULT '',
PRIMARY KEY (repo_prefix, from_id, to_id, kind, line)
) WITHOUT ROWID;
CREATE INDEX IF NOT EXISTS ref_facts_by_file ON ref_facts(repo_prefix, file_path);
-- ref_facts_by_target backs the reverse lookup ("which files hold a fact
-- resolving TO these symbols") that affected-by re-resolution runs when a
-- file's symbol signatures change. Without it that query is a full
-- ref_facts scan — the PK leads with from_id, not to_id.
CREATE INDEX IF NOT EXISTS ref_facts_by_target ON ref_facts(repo_prefix, to_id);
CREATE TABLE IF NOT EXISTS vectors (
node_id TEXT PRIMARY KEY,
dims INTEGER NOT NULL,
vec BLOB NOT NULL
) WITHOUT ROWID;
-- churn_enrichment is the per-node git-churn sidecar (change A: move
-- enrichment OUT of nodes.meta so the node hot path stops encoding
-- rarely-read data into the blob and get_churn_rate does an indexed read
-- instead of an AllNodes+meta-decode scan). One typed row per enriched
-- file/function/method node, keyed by node_id (join key back to
-- nodes.id); repo_prefix scopes
-- per-repo reseeds/wipes. head_sha/branch/computed_at are file-level only
-- (empty for symbols). WITHOUT ROWID: the PK index IS the table.
CREATE TABLE IF NOT EXISTS churn_enrichment (
node_id TEXT PRIMARY KEY,
repo_prefix TEXT NOT NULL DEFAULT '',
commit_count INTEGER NOT NULL DEFAULT 0,
age_days INTEGER NOT NULL DEFAULT 0,
churn_rate REAL NOT NULL DEFAULT 0,
last_author TEXT NOT NULL DEFAULT '',
last_commit_at TEXT NOT NULL DEFAULT '',
head_sha TEXT NOT NULL DEFAULT '',
branch TEXT NOT NULL DEFAULT '',
computed_at TEXT NOT NULL DEFAULT ''
) WITHOUT ROWID;
CREATE INDEX IF NOT EXISTS churn_by_repo ON churn_enrichment(repo_prefix) WHERE repo_prefix <> '';
-- coverage_enrichment: per-symbol coverage sidecar (change A). Typed
-- columns keyed by node_id; repo_prefix scopes per-repo wipes.
CREATE TABLE IF NOT EXISTS coverage_enrichment (
node_id TEXT PRIMARY KEY,
repo_prefix TEXT NOT NULL DEFAULT '',
coverage_pct REAL NOT NULL DEFAULT 0,
num_stmt INTEGER NOT NULL DEFAULT 0,
hit INTEGER NOT NULL DEFAULT 0
) WITHOUT ROWID;
CREATE INDEX IF NOT EXISTS coverage_by_repo ON coverage_enrichment(repo_prefix) WHERE repo_prefix <> '';
-- release_enrichment: per-file "added_in <tag>" sidecar (change A).
CREATE TABLE IF NOT EXISTS release_enrichment (
node_id TEXT PRIMARY KEY,
repo_prefix TEXT NOT NULL DEFAULT '',
added_in TEXT NOT NULL DEFAULT ''
) WITHOUT ROWID;
CREATE INDEX IF NOT EXISTS release_by_repo ON release_enrichment(repo_prefix) WHERE repo_prefix <> '';
-- blame_enrichment: per-symbol latest-author sidecar (change A).
CREATE TABLE IF NOT EXISTS blame_enrichment (
node_id TEXT PRIMARY KEY,
repo_prefix TEXT NOT NULL DEFAULT '',
commit_sha TEXT NOT NULL DEFAULT '',
email TEXT NOT NULL DEFAULT '',
ts INTEGER NOT NULL DEFAULT 0
) WITHOUT ROWID;
CREATE INDEX IF NOT EXISTS blame_by_repo ON blame_enrichment(repo_prefix) WHERE repo_prefix <> '';
-- symbol_fts is the FTS5 full-text index over pre-tokenised symbol
-- names. It replaces the multi-GB in-heap Bleve/BM25 index with an
-- on-disk inverted index the SymbolSearcher / SymbolBundleSearcher
-- query through. A standard (NOT contentless) FTS5 table; individual
-- rows are deleted by their FTS5 docid via the symbol_fts_rowid sidecar
-- below (node_id is UNINDEXED, so a DELETE keyed on it would full-scan
-- the index). node_id is the join key back to nodes.id; repo_prefix is
-- carried UNINDEXED so per-repo staleness wipes (DELETE … WHERE
-- repo_prefix = ?) hit a literal column without a separate b-tree.
-- Only "tokens" is indexed for matching. IF NOT EXISTS makes this
-- idempotent on every Open, so an existing .sqlite gains the vtable
-- on its next open + reindex.
CREATE VIRTUAL TABLE IF NOT EXISTS symbol_fts USING fts5(node_id UNINDEXED, repo_prefix UNINDEXED, tokens);
-- symbol_fts_rowid maps a node_id to the rowid (FTS5 docid) of its row in
-- symbol_fts. node_id is UNINDEXED in the FTS5 vtable, so deleting a node's
-- prior row with "DELETE … WHERE node_id = ?" full-scans the entire index
-- once PER symbol — quadratic on the per-edit reindex hot path. This sidecar
-- turns the delete into an O(log n) docid delete ("WHERE rowid = ?", the FTS5
-- docid IS indexed). One row per indexed symbol, keyed by node_id (the join
-- key back to nodes.id); repo_prefix scopes the per-repo wipe that
-- BulkUpsertSymbolFTS performs in lockstep with symbol_fts. WITHOUT ROWID:
-- the PK index IS the table, like file_mtimes / clone_shingles.
CREATE TABLE IF NOT EXISTS symbol_fts_rowid (
node_id TEXT PRIMARY KEY,
repo_prefix TEXT NOT NULL DEFAULT '',
fts_rowid INTEGER NOT NULL
) WITHOUT ROWID;
-- content_fts is the FTS5 full-text index over CONTENT (data_class=
-- "content") section bodies — text / pdf / pptx / xlsx chunks. It is
-- kept SEPARATE from symbol_fts so content text never enters the symbol
-- search or the code-oriented analysis passes: a content-heavy repo of a
-- few hundred large documents explodes into hundreds of thousands of
-- section nodes, and streaming their bodies here (per file, on disk)
-- instead of into symbol_fts + graph nodes keeps the code index and the
-- graph passes bounded. Only "body" is indexed for matching; node_id /
-- repo_prefix / file_path / ordinal ride UNINDEXED so the per-repo and
-- per-file staleness wipes hit literal columns without a b-tree.
CREATE VIRTUAL TABLE IF NOT EXISTS content_fts USING fts5(node_id UNINDEXED, repo_prefix UNINDEXED, file_path UNINDEXED, ordinal UNINDEXED, body);
`
@@ -0,0 +1,104 @@
package store_sqlite
import (
"database/sql"
"path/filepath"
"testing"
"github.com/stretchr/testify/require"
"github.com/zzet/gortex/internal/graph"
)
// hasNodeColumn reports whether the nodes table currently has the named column.
func hasNodeColumn(t *testing.T, db *sql.DB, col string) bool {
t.Helper()
rows, err := db.Query(`PRAGMA table_info(nodes)`)
require.NoError(t, err)
defer rows.Close()
for rows.Next() {
var (
cid, notnull, pk int
name, ctype string
dflt sql.NullString
)
require.NoError(t, rows.Scan(&cid, &name, &ctype, &notnull, &dflt, &pk))
if name == col {
return true
}
}
require.NoError(t, rows.Err())
return false
}
// TestOpenUpgradesPreDataClassStore is the backward-compatibility proof for the
// promoted data_class column: an existing v1 store written before the column
// existed must Open cleanly (ensureNodeColumns ALTERs the column in before the
// node statements are prepared), keep its rows, and immediately get the working
// SQL-level content filter — all WITHOUT a schema_version bump or a reindex.
//
// The simulated old store has data_class dropped while staying at the v1
// baseline, the exact shape of every Gortex sqlite DB already on disk before
// this change. Without the data_class entry in promotedMetaColumns, the reopen
// fails with "no such column: data_class" when prepare() builds the node
// INSERT/SELECT — so this test fails-closed if that wiring regresses.
func TestOpenUpgradesPreDataClassStore(t *testing.T) {
path := filepath.Join(t.TempDir(), "store.sqlite")
// 1. Create a current store and seed a row, then close. A fresh store is
// stamped at the current schema version; step 2 knocks it back to the v1
// baseline to simulate a store written before data_class existed.
s, err := Open(path)
require.NoError(t, err)
s.AddNode(&graph.Node{ID: "old1", Kind: graph.KindFunction, Name: "Legacy", FilePath: "f.go", RepoPrefix: "r"})
require.NoError(t, s.Close())
// 2. Simulate a store written before data_class existed: drop the column
// while leaving user_version at the v1 baseline.
withRawDB(t, path, func(db *sql.DB) {
_, err := db.Exec(`ALTER TABLE nodes DROP COLUMN data_class`)
require.NoError(t, err, "simulate a pre-data_class store")
require.False(t, hasNodeColumn(t, db, "data_class"), "data_class must be absent before the upgrade")
// A fresh Open stamps the current schema version; knock it back to the v1
// baseline so this genuinely simulates a pre-data_class store and the
// reopen exercises the in-place upgrade arm rather than a no-op.
_, err = db.Exec(`PRAGMA user_version = 1`)
require.NoError(t, err, "reset to the v1 baseline")
var v int
require.NoError(t, db.QueryRow(`PRAGMA user_version`).Scan(&v))
require.Equal(t, 1, v, "the simulated old store must sit at the v1 baseline")
})
// 3. Reopen with the current binary. ensureNodeColumns must re-add the
// column before prepare() references it, so Open succeeds without a wipe.
s2, err := Open(path)
require.NoError(t, err, "Open must upgrade a pre-data_class store in place, not fail on the missing column")
t.Cleanup(func() { _ = s2.Close() })
require.True(t, hasNodeColumn(t, s2.db, "data_class"), "ensureNodeColumns must re-add data_class on Open")
require.False(t, s2.NeedsRebuild(), "an additive-column upgrade must not signal a wipe/reindex")
// 4. Existing rows survived (the upgrade is in place, not a rebuild).
require.NotNil(t, s2.GetNode("old1"), "existing rows must survive the in-place upgrade")
// 5. A new content node persists, round-trips through the promoted column,
// and is correctly dropped by the SQL-level content filter.
s2.AddNode(&graph.Node{ID: "content1", Kind: graph.KindDoc, Name: "doc.txt::0", RepoPrefix: "r",
Meta: map[string]any{"data_class": "content", "section_text": "x"}})
s2.AddNode(&graph.Node{ID: "code1", Kind: graph.KindFunction, Name: "Foo", RepoPrefix: "r"})
content := s2.GetNode("content1")
require.NotNil(t, content)
require.Equal(t, "content", content.Meta["data_class"], "data_class round-trips via the promoted column after upgrade")
ids := map[string]bool{}
for _, n := range s2.GetRepoNonContentNodes("r") {
ids[n.ID] = true
}
require.True(t, ids["old1"], "legacy non-content node kept")
require.True(t, ids["code1"], "code node kept")
require.False(t, ids["content1"], "content node filtered at the SQL level after the upgrade")
require.Len(t, ids, 2)
}
@@ -0,0 +1,247 @@
package store_sqlite
import (
"database/sql"
"fmt"
"os"
"strings"
)
// Schema versioning for the graph store.
//
// Unlike the sidecar (which holds irreplaceable user data and must migrate in
// place), the graph store is a DERIVED CACHE: every row is reconstructable by
// re-indexing the source. So the cheapest *always-correct* reaction to a schema
// change an old on-disk DB can't satisfy is to drop the file and let the daemon
// rebuild it on the next index. A migration may therefore declare rebuild=true
// instead of writing an in-place transform that would have to re-derive the new
// data from source anyway. In-place steps remain the cheap path for purely
// mechanical changes (a new index, a denormalisation, a column with a
// computable default) that spare a large repo a multi-minute reindex.
//
// The whole mechanism keys off SQLite's built-in PRAGMA user_version, read on
// Open before schemaSQL runs. There is no separate version table.
//
// Concurrency: the daemon holds an exclusive flock on <store>.lock around Open
// (see serverstack.NewSharedServer), so reading the version, wiping the file,
// and stamping it cannot race another process. That is why — unlike the
// sidecar — this path needs no BEGIN IMMEDIATE / busy-loop handling.
// currentSchemaVersion is the version a fully-reconciled store reports via
// PRAGMA user_version. Bump it whenever schemaSQL's typed-column shape or an
// index changes in a way an old on-disk DB would not already have, and append a
// matching schemaMigrations entry describing how to bring an older store
// forward (in place, or by rebuild).
const currentSchemaVersion = 2
// schemaMigration is one forward step. Exactly one strategy applies:
// - rebuild=true: the change introduces structure/data that can only come
// from re-indexing the source; an older store is wiped and rebuilt.
// - inPlace!=nil: the change is mechanically derivable from the existing
// store and is applied in a transaction with no reindex.
//
// Steps are append-only and ascending; never edit or renumber a shipped one.
// Any inPlace step must be idempotent (IF NOT EXISTS / ADD COLUMN guarded).
type schemaMigration struct {
version int
name string
inPlace func(tx *sql.Tx) error
rebuild bool
}
// schemaMigrations is the ordered, forward-only registry. Version 1 is the
// implicit baseline (no entry): a v1 store is reconciled entirely by schemaSQL's
// idempotent CREATE ... IF NOT EXISTS plus ensureNodeColumns, so any
// pre-versioning database baseline-stamps to v1 without a rebuild. Append
// entries for version 2 and up as the schema evolves.
var schemaMigrations = []schemaMigration{
{version: 2, name: "dedupe fn-value placeholder edges", inPlace: dedupeFnValuePlaceholderEdges},
}
// dedupeFnValuePlaceholderEdges collapses duplicate function-as-value gate
// placeholder edges (graph.FnValuePlaceholderMarker, `unresolved::fnvalue::
// <name>`) to one row per (from_id, to_id), keeping the MIN(id) survivor. The
// capture path now dedups per (from, name) before it emits, but stores written
// earlier accumulated one placeholder per call site — a live store held
// millions — and EdgesWithUnresolvedTarget plus the resolver's terminal
// reconcile materialised every one on each warm restart, the dominant warmup
// heap transient this step drains. The keep set is small (tens of thousands of
// distinct pairs), so the NOT IN materialisation is cheap; the ph filter rides
// the edges_by_to(to_id) range for the bare form and the is_unresolved index for
// the multi-repo infix form. Idempotent: a second run finds no duplicates. Freed
// pages return to the freelist and are reused by later writes; the file itself
// shrinks only under a manual VACUUM, deliberately out of scope for a derived
// cache that reclaims the space on its own.
func dedupeFnValuePlaceholderEdges(tx *sql.Tx) error {
_, err := tx.Exec(`
WITH ph AS (
SELECT id, from_id, to_id FROM edges
WHERE (to_id >= 'unresolved::fnvalue::' AND to_id < 'unresolved::fnvalue:;')
OR (is_unresolved = 1 AND to_id LIKE '%::unresolved::fnvalue::%')
), keep AS (
SELECT MIN(id) AS id FROM ph GROUP BY from_id, to_id
)
DELETE FROM edges WHERE id IN (SELECT id FROM ph) AND id NOT IN (SELECT id FROM keep)`)
return err
}
// schemaPlan is the decision planSchemaMigration derives from the stored
// PRAGMA user_version. It mutates nothing on its own.
type schemaPlan struct {
wipe bool // drop the on-disk DB and rebuild from source
inPlace []schemaMigration // ordered in-place steps to run after schemaSQL
stamp bool // write currentSchemaVersion once reconciled
}
// planSchemaMigrationWith decides how to reconcile a store at the stored
// PRAGMA user_version to current, given the migration registry. It mutates
// nothing. Open passes (currentSchemaVersion, schemaMigrations); tests pass
// fixtures.
func planSchemaMigrationWith(stored, current int, migrations []schemaMigration) schemaPlan {
switch {
case stored == current:
return schemaPlan{} // up to date, nothing to do
case stored > current:
// Written by a newer build than this binary understands; the shape may
// have changed under us. For a cache the safe move is to rebuild.
return schemaPlan{wipe: true, stamp: true}
case stored == 0:
// Fresh DB, or a pre-versioning store of unknown shape. schemaSQL's
// idempotent CREATE ... IF NOT EXISTS plus ensureNodeColumns /
// ensureEdgeColumns reconcile the base shape either way, so a stored==0
// store needs a wipe only when a pending step is a REBUILD whose data can
// only come from re-indexing source. With nothing pending, stamp; with
// only in-place steps pending, run them and stamp — an in-place step is
// idempotent and mechanically derivable, so it upgrades a pre-versioning
// store in place (preserving its rows) exactly as it upgrades a known
// prior version. Wiping a stored==0 store on any migration instead would
// force every non-daemon Open (tests, read-only tools) to pass WithRebuild
// the moment the first migration ships.
pending := pendingBetween(0, current, migrations)
if len(pending) == 0 {
return schemaPlan{stamp: true}
}
if anyRebuild(pending) {
return schemaPlan{wipe: true, stamp: true}
}
return schemaPlan{inPlace: pending, stamp: true}
default: // 0 < stored < current: a known prior version
pending := pendingBetween(stored, current, migrations)
if anyRebuild(pending) {
return schemaPlan{wipe: true, stamp: true}
}
return schemaPlan{inPlace: pending, stamp: true}
}
}
func pendingBetween(stored, current int, migrations []schemaMigration) []schemaMigration {
var out []schemaMigration
for _, m := range migrations {
if m.version > stored && m.version <= current {
out = append(out, m)
}
}
return out
}
func anyRebuild(ms []schemaMigration) bool {
for _, m := range ms {
if m.rebuild {
return true
}
}
return false
}
// validateSchemaMigrations checks the registry is well-formed. A test asserts
// this against the shipped (currentSchemaVersion, schemaMigrations) so the
// dangerous mistake — bumping currentSchemaVersion without appending a matching
// entry — fails CI instead of silently baseline-stamping an un-migrated store
// to the new version at runtime. Rules:
// - versions are >= 2 (v1 is the implicit baseline, never an entry) and
// strictly ascending;
// - each step sets exactly one strategy (inPlace xor rebuild);
// - the highest version equals current, so the registry actually defines how
// to reach it. An empty registry is valid only at version 1.
func validateSchemaMigrations(current int, migs []schemaMigration) error {
if len(migs) == 0 {
if current != 1 {
return fmt.Errorf("schema version %d has no migrations: only v1 may have an empty registry", current)
}
return nil
}
prev := 0
for i, m := range migs {
if m.version < 2 {
return fmt.Errorf("migration %q has version %d: entries must be >= 2 (v1 is the implicit baseline)", m.name, m.version)
}
if i > 0 && m.version <= prev {
return fmt.Errorf("migrations must be strictly ascending: v%d (%s) does not follow v%d", m.version, m.name, prev)
}
if (m.inPlace != nil) == m.rebuild {
return fmt.Errorf("migration v%d (%s) must set exactly one of inPlace / rebuild", m.version, m.name)
}
prev = m.version
}
if prev != current {
return fmt.Errorf("highest migration version %d != currentSchemaVersion %d: a version bump needs a matching migration entry", prev, current)
}
return nil
}
// readUserVersion reads PRAGMA user_version (0 on a fresh database).
func readUserVersion(db *sql.DB) (int, error) {
var v int
if err := db.QueryRow("PRAGMA user_version").Scan(&v); err != nil {
return 0, err
}
return v, nil
}
// setUserVersion stamps the schema version. PRAGMA takes no bound parameters;
// v is an int we control, so the format is safe.
func setUserVersion(db *sql.DB, v int) error {
if _, err := db.Exec(fmt.Sprintf("PRAGMA user_version = %d", v)); err != nil {
return err
}
return nil
}
// applyInPlaceMigrations runs the in-place steps in a single transaction.
func applyInPlaceMigrations(db *sql.DB, steps []schemaMigration) error {
if len(steps) == 0 {
return nil
}
tx, err := db.Begin()
if err != nil {
return err
}
defer func() { _ = tx.Rollback() }() // no-op once Commit succeeds
for _, m := range steps {
if err := m.inPlace(tx); err != nil {
return fmt.Errorf("schema migration v%d (%s): %w", m.version, m.name, err)
}
}
return tx.Commit()
}
// removeStoreFiles deletes the SQLite database and its companions. A missing
// file is not an error. Never called for ":memory:".
//
// The suffix list covers the files the DSN's journal_mode(WAL) produces (-wal,
// -shm) plus the rollback -journal a non-WAL fallback would use; keep it in
// sync if the journal_mode in Open's DSN ever changes.
func removeStoreFiles(path string) error {
for _, suffix := range []string{"", "-wal", "-shm", "-journal"} {
if err := os.Remove(path + suffix); err != nil && !os.IsNotExist(err) {
return fmt.Errorf("remove %s: %w", path+suffix, err)
}
}
return nil
}
// isMemoryPath reports whether path is an in-process SQLite database (no file
// on disk to wipe, always built fresh by schemaSQL).
func isMemoryPath(path string) bool {
return strings.Contains(path, ":memory:")
}
@@ -0,0 +1,545 @@
package store_sqlite
import (
"database/sql"
"errors"
"path/filepath"
"strings"
"testing"
_ "modernc.org/sqlite"
)
// withRawDB opens a bare *sql.DB on path, runs fn, and closes it — used to
// simulate an on-disk store written by an older/newer build (set user_version,
// insert rows) without going through Open's reconciliation.
func withRawDB(t *testing.T, path string, fn func(db *sql.DB)) {
t.Helper()
db, err := sql.Open("sqlite", path+"?_pragma=busy_timeout(5000)")
if err != nil {
t.Fatalf("open raw db: %v", err)
}
defer db.Close()
fn(db)
}
func nodeCount(t *testing.T, db *sql.DB) int {
t.Helper()
var n int
if err := db.QueryRow("SELECT COUNT(*) FROM nodes").Scan(&n); err != nil {
t.Fatalf("count nodes: %v", err)
}
return n
}
// TestOpenStampsFreshDB: a brand-new on-disk store is stamped to the current
// schema version.
func TestOpenStampsFreshDB(t *testing.T) {
path := filepath.Join(t.TempDir(), "store.sqlite")
s, err := Open(path)
if err != nil {
t.Fatalf("Open fresh: %v", err)
}
defer s.Close()
if v, err := readUserVersion(s.db); err != nil || v != currentSchemaVersion {
t.Fatalf("fresh user_version = %d (err %v), want %d", v, err, currentSchemaVersion)
}
}
// TestOpenBaselineStampsOldDBWithoutWipe: a pre-versioning store (user_version
// 0, reconcilable to current by schemaSQL + ensureNodeColumns) is stamped in
// place — its data must survive, not be wiped.
func TestOpenBaselineStampsOldDBWithoutWipe(t *testing.T) {
path := filepath.Join(t.TempDir(), "store.sqlite")
// Create the store, then simulate a pre-versioning DB: a row + user_version 0.
s, err := Open(path)
if err != nil {
t.Fatalf("first open: %v", err)
}
if err := s.Close(); err != nil {
t.Fatalf("close: %v", err)
}
withRawDB(t, path, func(db *sql.DB) {
if _, err := db.Exec(`INSERT INTO nodes (id, kind, name, file_path) VALUES ('n1','func','Foo','f.go')`); err != nil {
t.Fatalf("seed node: %v", err)
}
if _, err := db.Exec(`PRAGMA user_version = 0`); err != nil {
t.Fatalf("reset user_version: %v", err)
}
})
s2, err := Open(path)
if err != nil {
t.Fatalf("reopen old DB: %v", err)
}
defer s2.Close()
if v, _ := readUserVersion(s2.db); v != currentSchemaVersion {
t.Fatalf("user_version after baseline = %d, want %d", v, currentSchemaVersion)
}
if n := nodeCount(t, s2.db); n != 1 {
t.Fatalf("node count after baseline = %d, want 1 (data must NOT be wiped)", n)
}
}
// TestOpenRebuildsNewerDB: a store written by a NEWER build (user_version above
// current) cannot be trusted, so Open drops and rebuilds it — the data is gone
// and the version is re-stamped to current. Proves the wipe path (and that the
// -wal/-shm companions are cleared along with the main file).
func TestOpenRebuildsNewerDB(t *testing.T) {
path := filepath.Join(t.TempDir(), "store.sqlite")
s, err := Open(path)
if err != nil {
t.Fatalf("first open: %v", err)
}
if err := s.Close(); err != nil {
t.Fatalf("close: %v", err)
}
withRawDB(t, path, func(db *sql.DB) {
if _, err := db.Exec(`INSERT INTO nodes (id, kind, name, file_path) VALUES ('n1','func','Foo','f.go')`); err != nil {
t.Fatalf("seed node: %v", err)
}
if _, err := db.Exec(`PRAGMA user_version = 999`); err != nil { // a future version this binary doesn't know
t.Fatalf("set future user_version: %v", err)
}
})
s2, err := Open(path, WithRebuild()) // simulate the daemon: holds the lock, may rebuild
if err != nil {
t.Fatalf("reopen newer DB: %v", err)
}
defer s2.Close()
if v, _ := readUserVersion(s2.db); v != currentSchemaVersion {
t.Fatalf("user_version after rebuild = %d, want %d", v, currentSchemaVersion)
}
if n := nodeCount(t, s2.db); n != 0 {
t.Fatalf("node count after rebuild = %d, want 0 (newer DB must be wiped)", n)
}
}
// TestOpenRefusesWipeWithoutOptIn: the default Open must NOT destroy an
// incompatible on-disk database. Without WithRebuild it returns
// ErrSchemaRebuildRequired and leaves the file (and its rows) intact, so a
// caller that does not hold the store lock cannot silently corrupt a store
// another process may have open.
func TestOpenRefusesWipeWithoutOptIn(t *testing.T) {
path := filepath.Join(t.TempDir(), "store.sqlite")
s, err := Open(path)
if err != nil {
t.Fatalf("first open: %v", err)
}
if _, err := s.db.Exec(`INSERT INTO nodes (id, kind, name, file_path) VALUES ('n1','func','Foo','f.go')`); err != nil {
t.Fatalf("seed node: %v", err)
}
if err := s.Close(); err != nil {
t.Fatalf("close: %v", err)
}
withRawDB(t, path, func(db *sql.DB) {
if _, err := db.Exec(`PRAGMA user_version = 999`); err != nil {
t.Fatalf("set future version: %v", err)
}
})
if _, err := Open(path); !errors.Is(err, ErrSchemaRebuildRequired) {
t.Fatalf("Open without WithRebuild = %v, want ErrSchemaRebuildRequired", err)
}
withRawDB(t, path, func(db *sql.DB) {
if n := nodeCount(t, db); n != 1 {
t.Fatalf("node count = %d after a refused wipe, want 1 (the file must be untouched)", n)
}
var v int
if err := db.QueryRow("PRAGMA user_version").Scan(&v); err != nil {
t.Fatalf("read user_version: %v", err)
}
if v != 999 {
t.Fatalf("user_version = %d after a refused wipe, want 999 (unchanged)", v)
}
})
}
// TestPlanSchemaMigration covers the pure decision logic, including the
// in-place vs rebuild dispatch a future currentSchemaVersion=2 would exercise.
func TestPlanSchemaMigration(t *testing.T) {
inPlace := schemaMigration{version: 2, name: "add-index", inPlace: func(*sql.Tx) error { return nil }}
rebuild := schemaMigration{version: 2, name: "typed-column", rebuild: true}
cases := []struct {
name string
stored, current int
migs []schemaMigration
wantWipe bool
wantStamp bool
wantInPlace int
}{
{"up to date", 1, 1, nil, false, false, 0},
{"fresh at v1 baseline-stamps", 0, 1, nil, false, true, 0},
{"newer DB rebuilds", 2, 1, nil, true, true, 0},
{"v0 with only in-place pending upgrades in place, no wipe", 0, 2, []schemaMigration{inPlace}, false, true, 1},
{"v0 with a pending rebuild wipes", 0, 2, []schemaMigration{rebuild}, true, true, 0},
{"v1->v2 in-place", 1, 2, []schemaMigration{inPlace}, false, true, 1},
{"v1->v2 rebuild", 1, 2, []schemaMigration{rebuild}, true, true, 0},
}
for _, c := range cases {
t.Run(c.name, func(t *testing.T) {
got := planSchemaMigrationWith(c.stored, c.current, c.migs)
if got.wipe != c.wantWipe || got.stamp != c.wantStamp || len(got.inPlace) != c.wantInPlace {
t.Fatalf("plan(%d->%d) = {wipe:%v stamp:%v inPlace:%d}, want {wipe:%v stamp:%v inPlace:%d}",
c.stored, c.current, got.wipe, got.stamp, len(got.inPlace), c.wantWipe, c.wantStamp, c.wantInPlace)
}
})
}
}
// TestApplyInPlaceMigrations: steps run in order and commit; a failing step
// rolls the whole transaction back.
func TestApplyInPlaceMigrations(t *testing.T) {
t.Run("commit", func(t *testing.T) {
path := filepath.Join(t.TempDir(), "m.sqlite")
withRawDB(t, path, func(db *sql.DB) {
step := schemaMigration{version: 2, name: "mk", inPlace: func(tx *sql.Tx) error {
_, err := tx.Exec(`CREATE TABLE marker (x TEXT)`)
return err
}}
if err := applyInPlaceMigrations(db, []schemaMigration{step}); err != nil {
t.Fatalf("apply: %v", err)
}
var name string
if err := db.QueryRow(`SELECT name FROM sqlite_master WHERE type='table' AND name='marker'`).Scan(&name); err != nil {
t.Fatalf("marker table not created: %v", err)
}
})
})
t.Run("rollback on failure preserves cause and rolls back every step", func(t *testing.T) {
path := filepath.Join(t.TempDir(), "m.sqlite")
withRawDB(t, path, func(db *sql.DB) {
// Two steps in one batch: the first creates table A, the second
// creates B then fails. Both must roll back, proving the steps
// share a single transaction.
stepA := schemaMigration{version: 2, name: "make-a", inPlace: func(tx *sql.Tx) error {
_, err := tx.Exec(`CREATE TABLE a (x TEXT)`)
return err
}}
stepB := schemaMigration{version: 3, name: "boom", inPlace: func(tx *sql.Tx) error {
if _, err := tx.Exec(`CREATE TABLE b (x TEXT)`); err != nil {
return err
}
return sql.ErrConnDone // synthetic failure after a partial write
}}
err := applyInPlaceMigrations(db, []schemaMigration{stepA, stepB})
if err == nil {
t.Fatal("expected applyInPlaceMigrations to surface the step error")
}
if !errors.Is(err, sql.ErrConnDone) {
t.Fatalf("error should wrap the step's cause; got %v", err)
}
if !strings.Contains(err.Error(), "v3") || !strings.Contains(err.Error(), "boom") {
t.Fatalf("error should name the failing migration (v3/boom); got %q", err.Error())
}
for _, tbl := range []string{"a", "b"} {
var name string
e := db.QueryRow(`SELECT name FROM sqlite_master WHERE type='table' AND name=?`, tbl).Scan(&name)
if e != sql.ErrNoRows {
t.Fatalf("table %q should have rolled back (shared transaction), got name=%q err=%v", tbl, name, e)
}
}
})
})
}
// TestOpenAtCurrentVersionIsNoOp covers the highest-frequency path — every
// daemon restart reopens an up-to-date store. It must be a no-op that
// preserves data; an off-by-one to wipe here would destroy the cache on every
// restart.
func TestOpenAtCurrentVersionIsNoOp(t *testing.T) {
path := filepath.Join(t.TempDir(), "store.sqlite")
s, err := Open(path)
if err != nil {
t.Fatalf("first open: %v", err)
}
withRawSeed := func(db *sql.DB) {
if _, err := db.Exec(`INSERT INTO nodes (id, kind, name, file_path) VALUES ('n1','func','Foo','f.go')`); err != nil {
t.Fatalf("seed node: %v", err)
}
}
withRawSeed(s.db)
if err := s.Close(); err != nil {
t.Fatalf("close: %v", err)
}
s2, err := Open(path)
if err != nil {
t.Fatalf("reopen at current version: %v", err)
}
defer s2.Close()
if v, _ := readUserVersion(s2.db); v != currentSchemaVersion {
t.Fatalf("user_version = %d, want %d", v, currentSchemaVersion)
}
if n := nodeCount(t, s2.db); n != 1 {
t.Fatalf("node count = %d, want 1 (a no-op reopen must NOT wipe)", n)
}
if s2.NeedsRebuild() {
t.Fatal("a no-op reopen must not signal NeedsRebuild")
}
}
// TestOpenWithInPlaceMigration drives the in-place arm end-to-end through the
// real Open composition (via the openWith seam): an older store at version 1
// is upgraded to version 2 by a registered in-place step that runs AFTER
// schemaSQL, the step's effect is visible, the existing data survives, and the
// version is stamped.
func TestOpenWithInPlaceMigration(t *testing.T) {
path := filepath.Join(t.TempDir(), "store.sqlite")
// Create a store with a row, then knock it back to the v1 baseline so the
// openWith below drives the v1->v2 in-place arm (a fresh Open now stamps the
// current version, which is >= 2).
s, err := Open(path)
if err != nil {
t.Fatalf("first open: %v", err)
}
if _, err := s.db.Exec(`INSERT INTO nodes (id, kind, name, file_path) VALUES ('n1','func','Foo','f.go')`); err != nil {
t.Fatalf("seed node: %v", err)
}
if err := s.Close(); err != nil {
t.Fatalf("close: %v", err)
}
withRawDB(t, path, func(db *sql.DB) {
if _, err := db.Exec(`PRAGMA user_version = 1`); err != nil {
t.Fatalf("reset to v1 baseline: %v", err)
}
})
// An in-place v2 step that depends on the base schema (an index on a
// nodes column) — proving it runs after schemaSQL/ensureNodeColumns.
ran := false
v2 := schemaMigration{version: 2, name: "idx-language", inPlace: func(tx *sql.Tx) error {
ran = true
_, err := tx.Exec(`CREATE INDEX IF NOT EXISTS test_nodes_by_language ON nodes(language)`)
return err
}}
s2, err := openWith(path, 2, []schemaMigration{v2}, false) // in-place never wipes
if err != nil {
t.Fatalf("openWith v2 in-place: %v", err)
}
defer s2.Close()
if !ran {
t.Fatal("the in-place migration step did not run")
}
if v, _ := readUserVersion(s2.db); v != 2 {
t.Fatalf("user_version = %d, want 2", v)
}
if n := nodeCount(t, s2.db); n != 1 {
t.Fatalf("node count = %d, want 1 (in-place upgrade must preserve data)", n)
}
var name string
if err := s2.db.QueryRow(`SELECT name FROM sqlite_master WHERE type='index' AND name='test_nodes_by_language'`).Scan(&name); err != nil {
t.Fatalf("in-place index not created: %v", err)
}
if s2.NeedsRebuild() {
t.Fatal("an in-place upgrade must not signal NeedsRebuild")
}
}
// TestOpenWithInPlaceFailureDoesNotStamp: a failing in-place step makes Open
// return an error and leaves the stored version unchanged, so the next open
// retries the upgrade rather than treating it as done.
func TestOpenWithInPlaceFailureDoesNotStamp(t *testing.T) {
path := filepath.Join(t.TempDir(), "store.sqlite")
s, err := Open(path)
if err != nil {
t.Fatalf("first open: %v", err)
}
if err := s.Close(); err != nil {
t.Fatalf("close: %v", err)
}
// A fresh Open now stamps the current version (>= 2); knock it back to the
// v1 baseline so openWith drives the v1->v2 arm and the failing step runs.
withRawDB(t, path, func(db *sql.DB) {
if _, err := db.Exec(`PRAGMA user_version = 1`); err != nil {
t.Fatalf("reset to v1 baseline: %v", err)
}
})
boom := schemaMigration{version: 2, name: "boom", inPlace: func(*sql.Tx) error {
return sql.ErrConnDone
}}
if _, err := openWith(path, 2, []schemaMigration{boom}, false); err == nil {
t.Fatal("expected openWith to fail when an in-place step errors")
}
withRawDB(t, path, func(db *sql.DB) {
var v int
if err := db.QueryRow("PRAGMA user_version").Scan(&v); err != nil {
t.Fatalf("read user_version: %v", err)
}
if v != 1 {
t.Fatalf("user_version = %d after a failed migration, want 1 (unstamped, so the next open retries)", v)
}
})
}
// TestOpenWithMemoryUnderWipePlanStampsWithoutError: an in-memory store under a
// plan that would wipe an on-disk DB must not attempt a file removal — it is
// always fresh and simply stamps the current version.
func TestOpenWithMemoryUnderWipePlanStampsWithoutError(t *testing.T) {
rebuildV2 := schemaMigration{version: 2, name: "typed-col", rebuild: true}
// stored==0, current==2, a pending rebuild => plan.wipe==true; the memory
// guard must skip the wipe and stamp anyway.
s, err := openWith(":memory:", 2, []schemaMigration{rebuildV2}, false) // memory never wipes
if err != nil {
t.Fatalf("openWith :memory: under wipe plan: %v", err)
}
defer s.Close()
if v, _ := readUserVersion(s.db); v != 2 {
t.Fatalf("user_version = %d, want 2", v)
}
if s.NeedsRebuild() {
t.Fatal(":memory: must never report a wipe (nothing to remove)")
}
}
// TestNeedsRebuildSignalAfterWipe: a store written by a newer build is wiped on
// open and reports NeedsRebuild so the daemon forces a full re-index.
func TestNeedsRebuildSignalAfterWipe(t *testing.T) {
path := filepath.Join(t.TempDir(), "store.sqlite")
s, err := Open(path)
if err != nil {
t.Fatalf("first open: %v", err)
}
if err := s.Close(); err != nil {
t.Fatalf("close: %v", err)
}
withRawDB(t, path, func(db *sql.DB) {
if _, err := db.Exec(`PRAGMA user_version = 999`); err != nil {
t.Fatalf("set future version: %v", err)
}
})
s2, err := Open(path, WithRebuild()) // daemon-equivalent: lock held, rebuild permitted
if err != nil {
t.Fatalf("reopen newer DB: %v", err)
}
defer s2.Close()
if !s2.NeedsRebuild() {
t.Fatal("a wiped store must report NeedsRebuild so the daemon re-indexes")
}
}
// TestSchemaMigrationsWellFormed asserts the shipped registry is valid and that
// the validator rejects the dangerous misconfigurations — above all, bumping
// currentSchemaVersion without appending a matching migration.
func TestSchemaMigrationsWellFormed(t *testing.T) {
if err := validateSchemaMigrations(currentSchemaVersion, schemaMigrations); err != nil {
t.Fatalf("shipped registry is invalid: %v", err)
}
inPlace := func(*sql.Tx) error { return nil }
bad := []struct {
name string
current int
migs []schemaMigration
}{
{"bumped version with no migration", 2, nil},
{"highest below current", 3, []schemaMigration{{version: 2, name: "x", rebuild: true}}},
{"both strategies set", 2, []schemaMigration{{version: 2, name: "x", rebuild: true, inPlace: inPlace}}},
{"neither strategy set", 2, []schemaMigration{{version: 2, name: "x"}}},
{"not strictly ascending", 3, []schemaMigration{{version: 2, name: "a", rebuild: true}, {version: 2, name: "b", rebuild: true}}},
{"v1 entry (baseline is implicit)", 1, []schemaMigration{{version: 1, name: "a", rebuild: true}}},
}
for _, c := range bad {
if err := validateSchemaMigrations(c.current, c.migs); err == nil {
t.Errorf("%s: expected a validation error, got nil", c.name)
}
}
}
// TestOpenDedupesFnValuePlaceholders drives the shipped v2 migration through the
// real Open composition: a store knocked back to the v1 baseline with duplicate
// fn-value placeholder edges is deduped in place on reopen — one survivor per
// (from_id, to_id), the MIN(id) row kept — while a distinct placeholder, a
// resolved edge, and an ordinary unresolved stub are untouched, and the version
// stamps to current. Covers both the bare and the multi-repo COPY-rewrite form.
func TestOpenDedupesFnValuePlaceholders(t *testing.T) {
path := filepath.Join(t.TempDir(), "store.sqlite")
s, err := Open(path)
if err != nil {
t.Fatalf("first open: %v", err)
}
// Seed edges by explicit id so the MIN(id) survivors are predictable. The
// is_unresolved column is generated, so it is omitted from the INSERT.
ins := `INSERT INTO edges (id, from_id, to_id, kind, file_path, line) VALUES (?,?,?,?,?,?)`
seed := []struct {
id int
from, to string
kind string
line int
}{
// Duplicate bare placeholders: same (from,to), distinct lines. Keep id 1.
{1, "a", "unresolved::fnvalue::handler", "references", 10},
{2, "a", "unresolved::fnvalue::handler", "references", 20},
{3, "a", "unresolved::fnvalue::handler", "references", 30},
// A distinct placeholder (different name) — must survive untouched.
{4, "a", "unresolved::fnvalue::other", "references", 10},
// Duplicate multi-repo COPY-rewrite placeholders — exercises the
// is_unresolved infix branch of the migration. Keep id 5.
{5, "b", "r::unresolved::fnvalue::handler", "references", 10},
{6, "b", "r::unresolved::fnvalue::handler", "references", 20},
// A resolved edge and an ordinary unresolved stub — never touched.
{7, "a", "b", "calls", 1},
{8, "a", "unresolved::Foo", "calls", 1},
}
for _, r := range seed {
if _, err := s.db.Exec(ins, r.id, r.from, r.to, r.kind, "f.go", r.line); err != nil {
t.Fatalf("seed edge %d: %v", r.id, err)
}
}
if err := s.Close(); err != nil {
t.Fatalf("close: %v", err)
}
// Knock the store back to the v1 baseline so the reopen runs the v2 dedup.
withRawDB(t, path, func(db *sql.DB) {
if _, err := db.Exec(`PRAGMA user_version = 1`); err != nil {
t.Fatalf("reset to v1 baseline: %v", err)
}
})
s2, err := Open(path)
if err != nil {
t.Fatalf("reopen for dedup: %v", err)
}
defer s2.Close()
if v, _ := readUserVersion(s2.db); v != currentSchemaVersion {
t.Fatalf("user_version after dedup = %d, want %d", v, currentSchemaVersion)
}
present := func(id int) bool {
var n int
if err := s2.db.QueryRow(`SELECT COUNT(*) FROM edges WHERE id = ?`, id).Scan(&n); err != nil {
t.Fatalf("count id %d: %v", id, err)
}
return n == 1
}
// Bare-form dedup keeps the MIN(id) survivor and drops the rest.
if !present(1) || present(2) || present(3) {
t.Fatalf("bare dedup wrong: want keep 1 / drop 2,3; got 1=%v 2=%v 3=%v", present(1), present(2), present(3))
}
// A distinct placeholder pair survives.
if !present(4) {
t.Fatal("distinct fn-value placeholder (id 4) was wrongly deleted")
}
// Multi-repo infix dedup keeps the MIN(id) survivor and drops the rest.
if !present(5) || present(6) {
t.Fatalf("multi-repo dedup wrong: want keep 5 / drop 6; got 5=%v 6=%v", present(5), present(6))
}
// A resolved edge and an ordinary unresolved stub must be untouched.
if !present(7) {
t.Fatal("resolved edge (id 7) must survive the placeholder dedup")
}
if !present(8) {
t.Fatal("ordinary unresolved stub (id 8) must survive the placeholder dedup")
}
}
File diff suppressed because it is too large Load Diff
@@ -0,0 +1,639 @@
package store_sqlite
import (
"iter"
"sort"
"github.com/zzet/gortex/internal/graph"
)
// This file implements the trivial SQL aggregator / scanner optional
// capability interfaces from graph.Store. Each method pushes its
// GROUP BY / WHERE / COUNT into SQLite so the planner drives it through
// the schema's secondary indexes, returning only the aggregate rows
// instead of materialising the whole node / edge table Go-side.
//
// Conventions shared across these methods:
// - Empty / nil input returns nil (parity with the in-memory store).
// - Input id / kind slices are deduped before they reach the IN-list.
// - Large IN-lists are chunked by lookupChunkSize.
// - agg-prefixed helpers are local to this file.
var (
_ graph.InEdgeCounter = (*Store)(nil)
_ graph.NodeIDsByKinds = (*Store)(nil)
_ graph.EdgeKindCounter = (*Store)(nil)
_ graph.NodeDegreeByKinds = (*Store)(nil)
_ graph.NodesInFilesByKindFinder = (*Store)(nil)
_ graph.FileImportAggregator = (*Store)(nil)
_ graph.InDegreeForNodes = (*Store)(nil)
_ graph.CrossRepoEdgeAggregator = (*Store)(nil)
_ graph.FileImporters = (*Store)(nil)
_ graph.FileSymbolNamesByPaths = (*Store)(nil)
_ graph.EdgesByKindsScanner = (*Store)(nil)
_ graph.NodesByKindsScanner = (*Store)(nil)
_ graph.EdgeAdjacencyForKinds = (*Store)(nil)
_ graph.NodeDegreeAggregator = (*Store)(nil)
_ graph.NodeFanAggregator = (*Store)(nil)
)
// aggDedupeEdgeKinds drops empties and duplicates from an edge-kind
// slice, preserving first-seen order; returns the kinds widened to the
// []any an IN-list binds.
func aggDedupeEdgeKinds(kinds []graph.EdgeKind) (uniq []graph.EdgeKind, args []any) {
seen := make(map[graph.EdgeKind]struct{}, len(kinds))
for _, k := range kinds {
if k == "" {
continue
}
if _, ok := seen[k]; ok {
continue
}
seen[k] = struct{}{}
uniq = append(uniq, k)
args = append(args, string(k))
}
return uniq, args
}
// aggDedupeNodeKinds is the node-kind twin of aggDedupeEdgeKinds.
func aggDedupeNodeKinds(kinds []graph.NodeKind) (uniq []graph.NodeKind, args []any) {
seen := make(map[graph.NodeKind]struct{}, len(kinds))
for _, k := range kinds {
if k == "" {
continue
}
if _, ok := seen[k]; ok {
continue
}
seen[k] = struct{}{}
uniq = append(uniq, k)
args = append(args, string(k))
}
return uniq, args
}
// InEdgeCountsByKind returns per-target incoming-edge counts for the
// supplied edge kinds, grouped server-side via edges_by_to.
func (s *Store) InEdgeCountsByKind(kinds []graph.EdgeKind) map[string]int {
_, args := aggDedupeEdgeKinds(kinds)
if len(args) == 0 {
return nil
}
q := `SELECT to_id, COUNT(*) FROM edges WHERE kind IN (` + inPlaceholders(len(args)) + `) GROUP BY to_id`
rows, err := s.db.Query(q, args...)
panicOnFatal(err)
if rows == nil {
// swallowed teardown-race error: read returns empty (see panicOnFatal)
return nil
}
defer rows.Close()
out := make(map[string]int)
for rows.Next() {
var id string
var n int
panicOnFatal(rows.Scan(&id, &n))
out[id] = n
}
panicOnFatal(rows.Err())
return out
}
// NodeIDsByKinds returns the deduplicated IDs of every node whose kind
// is in the supplied set.
func (s *Store) NodeIDsByKinds(kinds []graph.NodeKind) []string {
_, args := aggDedupeNodeKinds(kinds)
if len(args) == 0 {
return nil
}
q := `SELECT id FROM nodes WHERE kind IN (` + inPlaceholders(len(args)) + `) ORDER BY id`
rows, err := s.db.Query(q, args...)
panicOnFatal(err)
if rows == nil {
// swallowed teardown-race error: read returns empty (see panicOnFatal)
return nil
}
defer rows.Close()
var out []string
for rows.Next() {
var id string
panicOnFatal(rows.Scan(&id))
out = append(out, id)
}
panicOnFatal(rows.Err())
return out
}
// EdgeKindCounts returns one entry per distinct edge kind with its
// occurrence count across the whole graph.
func (s *Store) EdgeKindCounts() map[graph.EdgeKind]int {
rows, err := s.db.Query(`SELECT kind, COUNT(*) FROM edges GROUP BY kind`)
panicOnFatal(err)
if rows == nil {
// swallowed teardown-race error: read returns empty (see panicOnFatal)
return nil
}
defer rows.Close()
out := make(map[graph.EdgeKind]int)
for rows.Next() {
var kind string
var n int
panicOnFatal(rows.Scan(&kind, &n))
out[graph.EdgeKind(kind)] = n
}
panicOnFatal(rows.Err())
return out
}
// NodeDegreeByKinds returns total in/out degree for every node whose
// kind is in the set (optionally under pathPrefix); UsageInCount is
// always 0 for this capability.
func (s *Store) NodeDegreeByKinds(kinds []graph.NodeKind, pathPrefix string) []graph.NodeDegreeRow {
_, kindArgs := aggDedupeNodeKinds(kinds)
if len(kindArgs) == 0 {
return nil
}
args := append([]any(nil), kindArgs...)
q := `SELECT n.id,
(SELECT COUNT(*) FROM edges e WHERE e.to_id = n.id) AS in_count,
(SELECT COUNT(*) FROM edges e WHERE e.from_id = n.id) AS out_count
FROM nodes n
WHERE n.kind IN (` + inPlaceholders(len(kindArgs)) + `)`
if pathPrefix != "" {
q += ` AND n.file_path LIKE ? ESCAPE '\'`
args = append(args, escapeLikePattern(pathPrefix)+"%")
}
q += ` ORDER BY n.id`
rows, err := s.db.Query(q, args...)
panicOnFatal(err)
if rows == nil {
// swallowed teardown-race error: read returns empty (see panicOnFatal)
return nil
}
defer rows.Close()
var out []graph.NodeDegreeRow
for rows.Next() {
var r graph.NodeDegreeRow
panicOnFatal(rows.Scan(&r.NodeID, &r.InCount, &r.OutCount))
out = append(out, r)
}
panicOnFatal(rows.Err())
return out
}
// NodesInFilesByKind returns every node living in one of the supplied
// files whose kind is in the supplied set.
func (s *Store) NodesInFilesByKind(files []string, kinds []graph.NodeKind) []*graph.Node {
uniqFiles := dedupeNonEmpty(files)
_, kindArgs := aggDedupeNodeKinds(kinds)
if len(uniqFiles) == 0 || len(kindArgs) == 0 {
return nil
}
var out []*graph.Node
for i := 0; i < len(uniqFiles); i += lookupChunkSize {
end := minInt(i+lookupChunkSize, len(uniqFiles))
chunk := uniqFiles[i:end]
args := append(toAnyArgs(chunk), kindArgs...)
q := `SELECT ` + lookupNodeCols + ` FROM nodes WHERE file_path IN (` +
inPlaceholders(len(chunk)) + `) AND kind IN (` + inPlaceholders(len(kindArgs)) + `) ORDER BY id`
out = append(out, s.queryNodesSQL(q, args...)...)
}
return out
}
// FileImportCounts returns per-target-file incoming-import counts. A
// nil scope counts every import edge; a non-nil scope bounds counts to
// edges whose target node ID lies in the slice (empty non-nil => nil).
func (s *Store) FileImportCounts(scope []string) []graph.FileImportCountRow {
if scope != nil && len(scope) == 0 {
return nil
}
base := `SELECT COALESCE(NULLIF(n.file_path, ''), n.id) AS path, COUNT(*) AS cnt
FROM edges e JOIN nodes n ON e.to_id = n.id
WHERE e.kind = ?`
args := []any{string(graph.EdgeImports)}
fileToCount := make(map[string]int)
if scope == nil {
q := base + ` GROUP BY path`
aggScanImportCounts(s, q, args, fileToCount)
} else {
uniq := dedupeNonEmpty(scope)
if len(uniq) == 0 {
return nil
}
for i := 0; i < len(uniq); i += lookupChunkSize {
end := minInt(i+lookupChunkSize, len(uniq))
chunk := uniq[i:end]
q := base + ` AND e.to_id IN (` + inPlaceholders(len(chunk)) + `) GROUP BY path`
aggScanImportCounts(s, q, append(append([]any(nil), args...), toAnyArgs(chunk)...), fileToCount)
}
}
if len(fileToCount) == 0 {
return nil
}
out := make([]graph.FileImportCountRow, 0, len(fileToCount))
for path, cnt := range fileToCount {
out = append(out, graph.FileImportCountRow{FilePath: path, Count: cnt})
}
return out
}
// aggScanImportCounts runs an import-count query and folds the (path,
// count) rows into the accumulator (chunked scopes can revisit a path).
func aggScanImportCounts(s *Store, q string, args []any, acc map[string]int) {
rows, err := s.db.Query(q, args...)
panicOnFatal(err)
if rows == nil {
// swallowed teardown-race error: read returns empty (see panicOnFatal)
return
}
defer rows.Close()
for rows.Next() {
var path string
var cnt int
panicOnFatal(rows.Scan(&path, &cnt))
acc[path] += cnt
}
panicOnFatal(rows.Err())
}
// InDegreeForNodes returns total incoming-edge counts (any kind) for
// the supplied node id set.
func (s *Store) InDegreeForNodes(ids []string) map[string]int {
uniq := dedupeNonEmpty(ids)
if len(uniq) == 0 {
return nil
}
out := make(map[string]int)
for i := 0; i < len(uniq); i += lookupChunkSize {
end := minInt(i+lookupChunkSize, len(uniq))
chunk := uniq[i:end]
q := `SELECT to_id, COUNT(*) FROM edges WHERE to_id IN (` +
inPlaceholders(len(chunk)) + `) GROUP BY to_id`
rows, err := s.db.Query(q, toAnyArgs(chunk)...)
panicOnFatal(err)
if rows == nil {
// swallowed teardown-race error: read returns empty (see panicOnFatal)
return out
}
for rows.Next() {
var id string
var n int
panicOnFatal(rows.Scan(&id, &n))
out[id] = n
}
panicOnFatal(rows.Err())
_ = rows.Close()
}
return out
}
// CrossRepoEdgeCounts returns pre-grouped cross-repo edge counts keyed
// by (base kind, from-repo, to-repo). Cross-repo kinds are those
// graph.BaseKindForCrossRepo recognises; the count is reported under
// the base kind.
func (s *Store) CrossRepoEdgeCounts() []graph.CrossRepoEdgeRow {
q := `SELECT e.kind, nf.repo_prefix, nt.repo_prefix, COUNT(*)
FROM edges e
JOIN nodes nf ON e.from_id = nf.id
JOIN nodes nt ON e.to_id = nt.id
WHERE nf.repo_prefix <> nt.repo_prefix
GROUP BY e.kind, nf.repo_prefix, nt.repo_prefix`
rows, err := s.db.Query(q)
panicOnFatal(err)
if rows == nil {
// swallowed teardown-race error: read returns empty (see panicOnFatal)
return nil
}
defer rows.Close()
// Aggregate keyed by the edge's OWN kind (cross_repo_*), NOT the base.
// BaseKindForCrossRepo is used only as the recogniser that decides
// whether an edge participates — parity with the in-memory store.
type key struct {
kind graph.EdgeKind
from string
to string
}
acc := make(map[key]int)
for rows.Next() {
var kind, from, to string
var n int
panicOnFatal(rows.Scan(&kind, &from, &to, &n))
ek := graph.EdgeKind(kind)
if _, ok := graph.BaseKindForCrossRepo(ek); !ok {
continue
}
acc[key{kind: ek, from: from, to: to}] += n
}
panicOnFatal(rows.Err())
if len(acc) == 0 {
return nil
}
out := make([]graph.CrossRepoEdgeRow, 0, len(acc))
for k, n := range acc {
out = append(out, graph.CrossRepoEdgeRow{Kind: k.kind, FromRepo: k.from, ToRepo: k.to, Count: n})
}
return out
}
// FileImporters returns the importing-node rows for every EdgeImports
// edge whose target's FilePath OR ID equals filePath.
func (s *Store) FileImporters(filePath string) []graph.FileImporterRow {
if filePath == "" {
return nil
}
q := `SELECT nf.file_path, nf.id, nf.name, nf.kind
FROM edges e
JOIN nodes nt ON e.to_id = nt.id
JOIN nodes nf ON e.from_id = nf.id
WHERE e.kind = ? AND (nt.file_path = ? OR nt.id = ?)
ORDER BY nf.file_path`
rows, err := s.db.Query(q, string(graph.EdgeImports), filePath, filePath)
panicOnFatal(err)
if rows == nil {
// swallowed teardown-race error: read returns empty (see panicOnFatal)
return nil
}
defer rows.Close()
var out []graph.FileImporterRow
for rows.Next() {
var r graph.FileImporterRow
var kind string
panicOnFatal(rows.Scan(&r.FromFile, &r.FromID, &r.FromName, &kind))
r.FromKind = graph.NodeKind(kind)
out = append(out, r)
}
panicOnFatal(rows.Err())
return out
}
// FileSymbolNamesByPaths returns the distinct (file, name) pairs for
// nodes in the supplied paths whose kind is in the set, sorted by
// (file, name).
func (s *Store) FileSymbolNamesByPaths(paths []string, kinds []graph.NodeKind) []graph.FileSymbolNameRow {
uniqPaths := dedupeNonEmpty(paths)
_, kindArgs := aggDedupeNodeKinds(kinds)
if len(uniqPaths) == 0 || len(kindArgs) == 0 {
return nil
}
var out []graph.FileSymbolNameRow
for i := 0; i < len(uniqPaths); i += lookupChunkSize {
end := minInt(i+lookupChunkSize, len(uniqPaths))
chunk := uniqPaths[i:end]
args := append(toAnyArgs(chunk), kindArgs...)
q := `SELECT DISTINCT file_path, name FROM nodes WHERE file_path IN (` +
inPlaceholders(len(chunk)) + `) AND kind IN (` + inPlaceholders(len(kindArgs)) + `)`
rows, err := s.db.Query(q, args...)
panicOnFatal(err)
if rows == nil {
// swallowed teardown-race error: read returns empty (see panicOnFatal)
return out
}
for rows.Next() {
var r graph.FileSymbolNameRow
panicOnFatal(rows.Scan(&r.FilePath, &r.Name))
out = append(out, r)
}
panicOnFatal(rows.Err())
_ = rows.Close()
}
sort.Slice(out, func(i, j int) bool {
if out[i].FilePath != out[j].FilePath {
return out[i].FilePath < out[j].FilePath
}
return out[i].Name < out[j].Name
})
return out
}
// EdgesByKinds streams every edge whose kind is in the supplied set;
// honours early-stop. Empty kinds yields nothing.
func (s *Store) EdgesByKinds(kinds []graph.EdgeKind) iter.Seq[*graph.Edge] {
_, args := aggDedupeEdgeKinds(kinds)
return func(yield func(*graph.Edge) bool) {
if len(args) == 0 {
return
}
q := `SELECT ` + lookupEdgeCols + ` FROM edges WHERE kind IN (` +
inPlaceholders(len(args)) + `) ORDER BY id`
for _, e := range s.queryEdgesSQL(q, args...) {
if e == nil {
continue
}
if !yield(e) {
return
}
}
}
}
// externalCallTargetPredicate selects edges whose target is an
// external-package terminal (dep:: / stdlib:: / external::, including the
// per-repo-prefixed stdlib form) or an already-materialised
// external-call:: node. Shared verbatim by ExternalCallCandidateEdges and
// the edges_external partial index (schema.go) so SQLite matches the
// partial index for the query — keep the two identical.
const externalCallTargetPredicate = `(to_id GLOB 'dep::*' OR to_id GLOB 'external::*' OR to_id GLOB 'stdlib::*' OR to_id GLOB '*::stdlib::*' OR to_id GLOB 'external-call::*')`
// ExternalCallCandidateEdges implements graph.ExternalCallCandidates: it
// returns only the call / reference edges the external-call synthesizer
// might act on, selected server-side so the whole call-edge table never
// crosses into Go just to be prefix-filtered. The GLOB predicate is
// served by the edges_external partial index.
func (s *Store) ExternalCallCandidateEdges() []*graph.Edge {
q := `SELECT ` + lookupEdgeCols + ` FROM edges
WHERE kind IN ('calls','references') AND ` + externalCallTargetPredicate + `
ORDER BY id`
return s.queryEdgesSQL(q)
}
// NodesByKinds returns every node whose kind is in the supplied set.
func (s *Store) NodesByKinds(kinds []graph.NodeKind) []*graph.Node {
_, args := aggDedupeNodeKinds(kinds)
if len(args) == 0 {
return nil
}
q := `SELECT ` + lookupNodeCols + ` FROM nodes WHERE kind IN (` +
inPlaceholders(len(args)) + `) ORDER BY id`
return s.queryNodesSQL(q, args...)
}
// EdgeAdjacencyForKinds streams (from, to) id pairs for edges whose
// kind is in edgeKinds and whose endpoints both have a kind in
// nodeKinds; honours early-stop. Empty kinds yields nothing.
func (s *Store) EdgeAdjacencyForKinds(edgeKinds []graph.EdgeKind, nodeKinds []graph.NodeKind) iter.Seq[[2]string] {
_, eArgs := aggDedupeEdgeKinds(edgeKinds)
_, nArgs := aggDedupeNodeKinds(nodeKinds)
return func(yield func([2]string) bool) {
if len(eArgs) == 0 || len(nArgs) == 0 {
return
}
args := append([]any(nil), eArgs...)
args = append(args, nArgs...)
args = append(args, nArgs...)
q := `SELECT e.from_id, e.to_id
FROM edges e
JOIN nodes nf ON e.from_id = nf.id
JOIN nodes nt ON e.to_id = nt.id
WHERE e.kind IN (` + inPlaceholders(len(eArgs)) + `)
AND nf.kind IN (` + inPlaceholders(len(nArgs)) + `)
AND nt.kind IN (` + inPlaceholders(len(nArgs)) + `)`
rows, err := s.db.Query(q, args...)
panicOnFatal(err)
if rows == nil {
// swallowed teardown-race error: read returns empty (see panicOnFatal)
return
}
defer rows.Close()
for rows.Next() {
var from, to string
panicOnFatal(rows.Scan(&from, &to))
if !yield([2]string{from, to}) {
return
}
}
panicOnFatal(rows.Err())
}
}
// NodeDegreeCounts returns per-node in/out/usage-in edge counts for the
// supplied id set. Unknown ids produce no row; duplicates collapse.
func (s *Store) NodeDegreeCounts(ids []string, usageKinds []graph.EdgeKind) []graph.NodeDegreeRow {
uniq := dedupeNonEmpty(ids)
if len(uniq) == 0 {
return nil
}
_, usageArgs := aggDedupeEdgeKinds(usageKinds)
out := make([]graph.NodeDegreeRow, 0, len(uniq))
for i := 0; i < len(uniq); i += lookupChunkSize {
end := minInt(i+lookupChunkSize, len(uniq))
chunk := uniq[i:end]
// Usage-in subquery: a literal 0 when no usage kinds are given.
usageExpr := `0`
var usageInline []any
if len(usageArgs) > 0 {
usageExpr = `(SELECT COUNT(*) FROM edges e WHERE e.to_id = n.id AND e.kind IN (` +
inPlaceholders(len(usageArgs)) + `))`
usageInline = usageArgs
}
q := `SELECT n.id,
(SELECT COUNT(*) FROM edges e WHERE e.to_id = n.id) AS in_count,
(SELECT COUNT(*) FROM edges e WHERE e.from_id = n.id) AS out_count,
` + usageExpr + ` AS usage_in
FROM nodes n
WHERE n.id IN (` + inPlaceholders(len(chunk)) + `)`
// Bind order matches placeholder order: usage subquery first
// (it appears earlier in the SELECT list), then the id IN-list.
args := append(append([]any(nil), usageInline...), toAnyArgs(chunk)...)
rows, err := s.db.Query(q, args...)
panicOnFatal(err)
if rows == nil {
// swallowed teardown-race error: read returns empty (see panicOnFatal)
return out
}
for rows.Next() {
var r graph.NodeDegreeRow
panicOnFatal(rows.Scan(&r.NodeID, &r.InCount, &r.OutCount, &r.UsageInCount))
out = append(out, r)
}
panicOnFatal(rows.Err())
_ = rows.Close()
}
return out
}
// NodeFanCounts returns per-node fan-in (incoming edges in fanInKinds)
// and fan-out (outgoing edges in fanOutKinds) for the supplied id set.
// Unknown ids produce no row; duplicates collapse.
func (s *Store) NodeFanCounts(ids []string, fanInKinds, fanOutKinds []graph.EdgeKind) []graph.NodeFanRow {
uniq := dedupeNonEmpty(ids)
if len(uniq) == 0 {
return nil
}
_, inArgs := aggDedupeEdgeKinds(fanInKinds)
_, outArgs := aggDedupeEdgeKinds(fanOutKinds)
out := make([]graph.NodeFanRow, 0, len(uniq))
for i := 0; i < len(uniq); i += lookupChunkSize {
end := minInt(i+lookupChunkSize, len(uniq))
chunk := uniq[i:end]
fanInExpr := `0`
var inInline []any
if len(inArgs) > 0 {
fanInExpr = `(SELECT COUNT(*) FROM edges e WHERE e.to_id = n.id AND e.kind IN (` +
inPlaceholders(len(inArgs)) + `))`
inInline = inArgs
}
fanOutExpr := `0`
var outInline []any
if len(outArgs) > 0 {
fanOutExpr = `(SELECT COUNT(*) FROM edges e WHERE e.from_id = n.id AND e.kind IN (` +
inPlaceholders(len(outArgs)) + `))`
outInline = outArgs
}
q := `SELECT n.id, ` + fanInExpr + ` AS fan_in, ` + fanOutExpr + ` AS fan_out
FROM nodes n
WHERE n.id IN (` + inPlaceholders(len(chunk)) + `)`
// Bind order matches placeholder order in the SELECT list:
// fan-in subquery, fan-out subquery, then the id IN-list.
args := append([]any(nil), inInline...)
args = append(args, outInline...)
args = append(args, toAnyArgs(chunk)...)
rows, err := s.db.Query(q, args...)
panicOnFatal(err)
if rows == nil {
// swallowed teardown-race error: read returns empty (see panicOnFatal)
return out
}
for rows.Next() {
var r graph.NodeFanRow
panicOnFatal(rows.Scan(&r.NodeID, &r.FanIn, &r.FanOut))
out = append(out, r)
}
panicOnFatal(rows.Err())
_ = rows.Close()
}
return out
}
// CommunityCrossingsByKind returns per-source crossing counts for edges
// whose kind is in the supplied set, given a node→community map. A
// crossing is an edge whose source community differs from its target
// community; zero-count sources are dropped. Empty kinds or empty
// community map returns nil. The community comparison runs Go-side
// because community membership is not a node column.
func (s *Store) CommunityCrossingsByKind(kinds []graph.EdgeKind, nodeToComm map[string]string) map[string]int {
_, args := aggDedupeEdgeKinds(kinds)
if len(args) == 0 || len(nodeToComm) == 0 {
return nil
}
q := `SELECT from_id, to_id FROM edges WHERE kind IN (` + inPlaceholders(len(args)) + `)`
rows, err := s.db.Query(q, args...)
panicOnFatal(err)
if rows == nil {
// swallowed teardown-race error: read returns empty (see panicOnFatal)
return nil
}
defer rows.Close()
out := make(map[string]int)
for rows.Next() {
var from, to string
panicOnFatal(rows.Scan(&from, &to))
fromComm, ok := nodeToComm[from]
if !ok {
continue
}
toComm, ok := nodeToComm[to]
if !ok {
continue
}
if fromComm != toComm {
out[from]++
}
}
panicOnFatal(rows.Err())
if len(out) == 0 {
return nil
}
return out
}
@@ -0,0 +1,500 @@
package store_sqlite
// This file implements the moderate-SQL analysis capability interfaces
// for the SQLite graph.Store backend. Each method mirrors the in-memory
// reference implementation in internal/graph/graph.go and is verified
// against the same conformance suite (internal/graph/storetest).
//
// Shape: push the structural filter into one indexed SELECT via the raw-
// SQL helpers (queryNodesSQL / s.db.Query), then do any Meta-dependent
// (JSON-decoded) or distinct-counting filtering in Go. No new prepared
// statements are added — every query rides the secondary indexes already
// created in schema.go (edges_by_from / edges_by_to / nodes_by_kind).
import (
"github.com/zzet/gortex/internal/graph"
)
// Compile-time assertions: *Store satisfies each analysis capability.
var _ graph.DeadCodeCandidator = (*Store)(nil)
var _ graph.IfaceImplementsScanner = (*Store)(nil)
var _ graph.MemberMethodsByType = (*Store)(nil)
var _ graph.StructuralParentEdges = (*Store)(nil)
var _ graph.ExtractCandidatesScanner = (*Store)(nil)
var _ graph.CrossRepoCandidates = (*Store)(nil)
var _ graph.ThrowerErrorSurfacer = (*Store)(nil)
// anaDedupeEdgeKinds drops empty / duplicate edge kinds, preserving
// first-seen order — the EdgeKind twin of dedupeNonEmpty.
func anaDedupeEdgeKinds(in []graph.EdgeKind) []graph.EdgeKind {
seen := make(map[graph.EdgeKind]struct{}, len(in))
out := make([]graph.EdgeKind, 0, len(in))
for _, k := range in {
if k == "" {
continue
}
if _, ok := seen[k]; ok {
continue
}
seen[k] = struct{}{}
out = append(out, k)
}
return out
}
// --- DeadCodeCandidator -------------------------------------------------
// DeadCodeCandidates returns nodes of the allowed kinds that have no
// incoming edge of the corresponding allowed in-edge kinds. An empty
// per-kind allowlist (or one that dedupes to nothing) means "any incoming
// edge counts as usage". Mirrors graph.(*Graph).DeadCodeCandidates: the
// candidate set is purely structural (the analysis layer applies the
// exported / test / entry-point / synthetic post-filters in Go), so no
// node-id exclusion happens here. The NOT-EXISTS filter runs server-side
// per node kind.
func (s *Store) DeadCodeCandidates(allowedNodeKinds []graph.NodeKind, allowedInEdgeKinds map[graph.NodeKind][]graph.EdgeKind) []*graph.Node {
if len(allowedNodeKinds) == 0 {
return nil
}
var out []*graph.Node
for _, nk := range allowedNodeKinds {
allowed := anaDedupeEdgeKinds(allowedInEdgeKinds[nk])
anyKindCounts := len(allowed) == 0
var q string
var args []any
if anyKindCounts {
// Any incoming edge disqualifies the node.
q = `SELECT ` + lookupNodeCols + ` FROM nodes n
WHERE n.kind = ?
AND NOT EXISTS (SELECT 1 FROM edges e WHERE e.to_id = n.id)
ORDER BY n.id`
args = []any{string(nk)}
} else {
// Only an incoming edge of one of the allowed kinds counts.
q = `SELECT ` + lookupNodeCols + ` FROM nodes n
WHERE n.kind = ?
AND NOT EXISTS (SELECT 1 FROM edges e WHERE e.to_id = n.id AND e.kind IN (` + inPlaceholders(len(allowed)) + `))
ORDER BY n.id`
args = make([]any, 0, 1+len(allowed))
args = append(args, string(nk))
for _, ek := range allowed {
args = append(args, string(ek))
}
}
for _, n := range s.queryNodesSQL(q, args...) {
if n != nil {
out = append(out, n)
}
}
}
return out
}
// --- IfaceImplementsScanner ---------------------------------------------
// IfaceImplementsRows returns one row per EdgeImplements edge whose
// target is a KindInterface carrying Meta["methods"]. The interface's
// decoded Meta rides on the row (callers pull the "methods" field, which
// round-trips as []string or []any). Interfaces with no Meta or no
// "methods" key are elided server-side.
func (s *Store) IfaceImplementsRows() []graph.IfaceImplementsRow {
q := `SELECT e.from_id, n.id, n.meta
FROM edges e
JOIN nodes n ON n.id = e.to_id
WHERE e.kind = ? AND n.kind = ? AND n.meta IS NOT NULL`
rows, err := s.db.Query(q, string(graph.EdgeImplements), string(graph.KindInterface))
if err != nil {
return nil
}
defer rows.Close()
var out []graph.IfaceImplementsRow
for rows.Next() {
var fromID, ifaceID string
var metaBlob []byte
if err := rows.Scan(&fromID, &ifaceID, &metaBlob); err != nil {
continue
}
meta, derr := decodeMeta(metaBlob)
if derr != nil || meta == nil {
continue
}
if _, ok := meta["methods"]; !ok {
continue
}
out = append(out, graph.IfaceImplementsRow{
TypeID: fromID,
IfaceID: ifaceID,
IfaceMeta: meta,
})
}
return out
}
// --- MemberMethodsByType ------------------------------------------------
// MemberMethodsByType returns typeID → []MemberMethodInfo for every
// EdgeMemberOf edge whose source is a KindMethod. The columns come from
// the METHOD NODE (FilePath / StartLine / RepoPrefix), matching the
// in-memory reference. Per-type lists are deduplicated by MethodID; the
// scan is ordered by the edge PK so the first-seen winner is stable. An
// empty graph (no qualifying rows) returns nil.
func (s *Store) MemberMethodsByType() map[string][]graph.MemberMethodInfo {
q := `SELECT e.to_id, n.id, n.name, n.file_path, n.start_line, n.repo_prefix
FROM edges e
JOIN nodes n ON n.id = e.from_id
WHERE e.kind = ? AND n.kind = ?
ORDER BY e.id`
rows, err := s.db.Query(q, string(graph.EdgeMemberOf), string(graph.KindMethod))
if err != nil {
return nil
}
defer rows.Close()
out := make(map[string][]graph.MemberMethodInfo)
seen := make(map[string]map[string]struct{})
for rows.Next() {
var typeID, methodID, name, filePath, repoPrefix string
var startLine int
if err := rows.Scan(&typeID, &methodID, &name, &filePath, &startLine, &repoPrefix); err != nil {
continue
}
if seen[typeID] == nil {
seen[typeID] = make(map[string]struct{})
}
if _, ok := seen[typeID][methodID]; ok {
continue
}
seen[typeID][methodID] = struct{}{}
out[typeID] = append(out[typeID], graph.MemberMethodInfo{
MethodID: methodID,
Name: name,
FilePath: filePath,
StartLine: startLine,
RepoPrefix: repoPrefix,
})
}
if len(out) == 0 {
// Match the in-memory reference: empty graph returns nil.
return nil
}
return out
}
// --- StructuralParentEdges ----------------------------------------------
// StructuralParentEdges returns every Extends / Implements / Composes
// edge whose endpoints are both Type / Interface, projected as (FromID,
// ToID, FromKind, ToKind, Origin). Endpoints that aren't both type /
// interface are filtered server-side. Empty graph or no matching edges
// returns nil.
func (s *Store) StructuralParentEdges() []graph.StructuralParentEdgeRow {
q := `SELECT e.from_id, e.to_id, nf.kind, nt.kind, e.origin
FROM edges e
JOIN nodes nf ON nf.id = e.from_id
JOIN nodes nt ON nt.id = e.to_id
WHERE e.kind IN (?,?,?)
AND nf.kind IN (?,?) AND nt.kind IN (?,?)
ORDER BY e.id`
rows, err := s.db.Query(q,
string(graph.EdgeExtends), string(graph.EdgeImplements), string(graph.EdgeComposes),
string(graph.KindType), string(graph.KindInterface),
string(graph.KindType), string(graph.KindInterface),
)
if err != nil {
return nil
}
defer rows.Close()
var out []graph.StructuralParentEdgeRow
for rows.Next() {
var fromID, toID, fromKind, toKind, origin string
if err := rows.Scan(&fromID, &toID, &fromKind, &toKind, &origin); err != nil {
continue
}
out = append(out, graph.StructuralParentEdgeRow{
FromID: fromID,
ToID: toID,
FromKind: graph.NodeKind(fromKind),
ToKind: graph.NodeKind(toKind),
Origin: origin,
})
}
return out
}
// --- ExtractCandidatesScanner -------------------------------------------
// ExtractCandidates ranks function / method nodes by extractability: line
// span (EndLine - StartLine + 1), distinct caller fan-in, and distinct
// callee fan-out, counting only edges whose kind is in the supplied set.
// Rows must clear all three thresholds. Nodes with a zero StartLine /
// EndLine are dropped; pathPrefix narrows by file-path prefix. Mirrors
// graph.(*Graph).ExtractCandidates exactly: only KindFunction +
// KindMethod nodes are considered, and the distinct-by-endpoint counting
// runs Go-side over GetInEdges / GetOutEdges.
func (s *Store) ExtractCandidates(kinds []graph.EdgeKind, minLines, minCallers, minFanOut int, pathPrefix string) []graph.ExtractCandidateRow {
if len(kinds) == 0 {
return nil
}
kindSet := make(map[graph.EdgeKind]struct{}, len(kinds))
for _, k := range kinds {
if k == "" {
continue
}
kindSet[k] = struct{}{}
}
if len(kindSet) == 0 {
return nil
}
// Candidate nodes: function / method only, non-zero line span,
// optional path-prefix gate.
q := `SELECT ` + lookupNodeCols + ` FROM nodes
WHERE kind IN (?,?) AND start_line > 0 AND end_line > 0`
args := []any{string(graph.KindFunction), string(graph.KindMethod)}
if pathPrefix != "" {
q += ` AND file_path LIKE ? ESCAPE '\'`
args = append(args, escapeLikePattern(pathPrefix)+"%")
}
q += ` ORDER BY id`
nodes := s.queryNodesSQL(q, args...)
var out []graph.ExtractCandidateRow
for _, n := range nodes {
if n == nil {
continue
}
lineCount := n.EndLine - n.StartLine + 1
if lineCount < minLines {
continue
}
callerSet := make(map[string]struct{})
for _, e := range s.GetInEdges(n.ID) {
if e == nil {
continue
}
if _, ok := kindSet[e.Kind]; !ok {
continue
}
callerSet[e.From] = struct{}{}
}
if len(callerSet) < minCallers {
continue
}
calleeSet := make(map[string]struct{})
for _, e := range s.GetOutEdges(n.ID) {
if e == nil {
continue
}
if _, ok := kindSet[e.Kind]; !ok {
continue
}
calleeSet[e.To] = struct{}{}
}
if len(calleeSet) < minFanOut {
continue
}
out = append(out, graph.ExtractCandidateRow{
NodeID: n.ID,
Name: n.Name,
FilePath: n.FilePath,
StartLine: n.StartLine,
EndLine: n.EndLine,
LineCount: lineCount,
CallerCount: len(callerSet),
FanOut: len(calleeSet),
})
}
return out
}
// --- CrossRepoCandidates ------------------------------------------------
// CrossRepoCandidates returns every edge whose kind is in baseKinds and
// whose endpoints carry two different non-empty RepoPrefix values. The
// edge is returned verbatim (callers rewrite Edge.CrossRepo); FromRepo /
// ToRepo are the endpoint prefixes. Empty baseKinds returns nil; single-
// repo graphs (or graphs whose nodes carry no RepoPrefix) yield nothing.
func (s *Store) CrossRepoCandidates(baseKinds []graph.EdgeKind) []graph.CrossRepoCandidateRow {
uniq := anaDedupeEdgeKinds(baseKinds)
if len(uniq) == 0 {
return nil
}
args := make([]any, 0, len(uniq))
for _, k := range uniq {
args = append(args, string(k))
}
q := `SELECT e.from_id, e.to_id, e.kind, e.file_path, e.line,
e.confidence, e.confidence_label, e.origin, e.tier, e.cross_repo, e.meta,
nf.repo_prefix, nt.repo_prefix
FROM edges e
JOIN nodes nf ON nf.id = e.from_id
JOIN nodes nt ON nt.id = e.to_id
WHERE e.kind IN (` + inPlaceholders(len(uniq)) + `)
AND nf.repo_prefix <> '' AND nt.repo_prefix <> ''
AND nf.repo_prefix <> nt.repo_prefix
ORDER BY e.id`
rows, err := s.db.Query(q, args...)
if err != nil {
return nil
}
defer rows.Close()
var out []graph.CrossRepoCandidateRow
for rows.Next() {
var (
fromRepo, toRepo string
e graph.Edge
metaBlob []byte
crossRepo int64
)
if err := rows.Scan(
&e.From, &e.To, &e.Kind, &e.FilePath, &e.Line,
&e.Confidence, &e.ConfidenceLabel, &e.Origin, &e.Tier,
&crossRepo, &metaBlob,
&fromRepo, &toRepo,
); err != nil {
continue
}
e.CrossRepo = crossRepo != 0
if len(metaBlob) > 0 {
if m, derr := decodeMeta(metaBlob); derr == nil {
e.Meta = m
}
}
edge := e
out = append(out, graph.CrossRepoCandidateRow{
Edge: &edge,
FromRepo: fromRepo,
ToRepo: toRepo,
})
}
return out
}
// --- ThrowerErrorSurfacer -----------------------------------------------
// ThrowerErrorSurface returns one row per thrower (a node with outgoing
// EdgeThrows edges), aggregating the distinct error targets and the
// distinct literal error-message strings it emits (KindString nodes with
// Meta["context"] == "error_msg", linked by EdgeEmits). pathPrefix gates
// the EdgeThrows rows by their stored FilePath prefix. Throws counts the
// underlying EdgeThrows edges; FilePath / Line seed from the first throws
// edge, falling back to the thrower node's own coordinates when the edge
// carries none — matching the in-memory reference.
func (s *Store) ThrowerErrorSurface(pathPrefix string) []graph.ThrowerErrorRow {
type rowAccum struct {
row graph.ThrowerErrorRow
targetSeen map[string]struct{}
msgSeen map[string]struct{}
}
accums := make(map[string]*rowAccum)
var order []string
// Pass 1: EdgeThrows aggregation (count + distinct targets), keyed by
// thrower. The first edge (by PK insertion order) seeds FilePath /
// Line; an empty edge file/line falls back to the thrower node.
tq := `SELECT from_id, to_id, file_path, line FROM edges WHERE kind = ?`
targs := []any{string(graph.EdgeThrows)}
if pathPrefix != "" {
tq += ` AND file_path LIKE ? ESCAPE '\'`
targs = append(targs, escapeLikePattern(pathPrefix)+"%")
}
tq += ` ORDER BY id`
trows, err := s.db.Query(tq, targs...)
if err != nil {
return nil
}
for trows.Next() {
var from, to, filePath string
var line int
if err := trows.Scan(&from, &to, &filePath, &line); err != nil {
continue
}
acc := accums[from]
if acc == nil {
file := filePath
ln := line
if file == "" || ln == 0 {
if n := s.GetNode(from); n != nil {
if file == "" {
file = n.FilePath
}
if ln == 0 {
ln = n.StartLine
}
}
}
acc = &rowAccum{
row: graph.ThrowerErrorRow{
ThrowerID: from,
FilePath: file,
Line: ln,
},
targetSeen: make(map[string]struct{}),
msgSeen: make(map[string]struct{}),
}
accums[from] = acc
order = append(order, from)
}
acc.row.Throws++
if _, ok := acc.targetSeen[to]; !ok {
acc.targetSeen[to] = struct{}{}
acc.row.ErrorTargets = append(acc.row.ErrorTargets, to)
}
}
_ = trows.Close()
if len(accums) == 0 {
return nil
}
// Pass 2: attach the literal error messages each thrower emits. Join
// each thrower's EdgeEmits out-edges to KindString targets and filter
// Meta["context"] == "error_msg" Go-side (the context lives in the
// JSON Meta blob).
for _, id := range order {
acc := accums[id]
mq := `SELECT n.name, n.meta
FROM edges e
JOIN nodes n ON n.id = e.to_id
WHERE e.from_id = ? AND e.kind = ? AND n.kind = ? AND n.meta IS NOT NULL
ORDER BY e.id`
mrows, err := s.db.Query(mq, id, string(graph.EdgeEmits), string(graph.KindString))
if err != nil {
continue
}
for mrows.Next() {
var name string
var metaBlob []byte
if err := mrows.Scan(&name, &metaBlob); err != nil {
continue
}
meta, derr := decodeMeta(metaBlob)
if derr != nil || meta == nil {
continue
}
ctxLabel, _ := meta["context"].(string)
if ctxLabel != "error_msg" {
continue
}
if _, ok := acc.msgSeen[name]; ok {
continue
}
acc.msgSeen[name] = struct{}{}
acc.row.ErrorMsgs = append(acc.row.ErrorMsgs, name)
}
_ = mrows.Close()
}
out := make([]graph.ThrowerErrorRow, 0, len(order))
for _, id := range order {
out = append(out, accums[id].row)
}
return out
}
+172
View File
@@ -0,0 +1,172 @@
package store_sqlite
import (
"sort"
"strings"
"github.com/zzet/gortex/internal/graph"
)
var _ graph.BFSCapable = (*Store)(nil)
// BFS runs a bounded breadth-first traversal in a single round-trip via a
// recursive CTE — the disk-backed sibling of the in-memory
// (*graph.Graph).BFS reference. See graph.BFSCapable for the contract;
// the two are shadow-tested for identical hop-sets in the conformance
// suite (storetest), including a cycle fixture.
//
// The recursive term joins edges on the direction's indexed column —
// edges_by_from(from_id, kind) for a forward walk, edges_by_to(to_id,
// kind) for a backward walk — and the nodes primary key, so it stays
// index-driven instead of scanning the edges table (confirmed via
// EXPLAIN QUERY PLAN in store_bfs_test.go). The nodes join also enforces
// the "node-backed targets only" rule: an edge to an unresolved /
// external stub with no node row is not followed. A cycle terminates on
// the depth bound; the outer ROW_NUMBER picks each node's minimum-depth,
// (parent, kind)-smallest discovery edge so the result is deterministic
// and matches the in-memory walk's bfsHopLess tie-break.
//
// Reads run lock-free, like the store's other read paths (SQLite WAL
// serves readers concurrently with the single serialized writer).
func (s *Store) BFS(seeds []string, dir graph.Direction, kinds []graph.EdgeKind, maxDepth, limit int) ([]graph.BFSHop, error) {
seen := make(map[string]struct{}, len(seeds))
uniqSeeds := make([]string, 0, len(seeds))
for _, sd := range seeds {
if sd == "" {
continue
}
if _, ok := seen[sd]; ok {
continue
}
seen[sd] = struct{}{}
uniqSeeds = append(uniqSeeds, sd)
}
if len(uniqSeeds) == 0 {
return nil, nil
}
uniqKinds := anaDedupeEdgeKinds(kinds)
// Seed-only fast path: with no edge kinds to follow or a non-positive
// depth bound the result is exactly the seeds at depth 0. Seeds enter
// unconditionally (no node-backed gate), matching the in-memory
// reference, which adds them before any traversal.
if len(uniqKinds) == 0 || maxDepth <= 0 {
hops := make([]graph.BFSHop, 0, len(uniqSeeds))
for _, sd := range uniqSeeds {
hops = append(hops, graph.BFSHop{NodeID: sd, Depth: 0})
}
sortBFSHops(hops)
if limit > 0 && len(hops) > limit {
hops = hops[:limit]
}
return hops, nil
}
query := buildBFSQuery(dir, len(uniqSeeds), len(uniqKinds), limit > 0)
args := make([]any, 0, len(uniqSeeds)+1+len(uniqKinds)+1)
for _, sd := range uniqSeeds {
args = append(args, sd)
}
args = append(args, maxDepth)
for _, k := range uniqKinds {
args = append(args, string(k))
}
if limit > 0 {
args = append(args, limit)
}
rows, err := s.db.Query(query, args...)
if err != nil {
return nil, err
}
defer rows.Close()
var out []graph.BFSHop
for rows.Next() {
var (
nodeID, parentID, edgeKind string
depth int
)
if err := rows.Scan(&nodeID, &depth, &parentID, &edgeKind); err != nil {
return nil, err
}
out = append(out, graph.BFSHop{
NodeID: nodeID,
Depth: depth,
ParentID: parentID,
EdgeKind: graph.EdgeKind(edgeKind),
})
}
if err := rows.Err(); err != nil {
return nil, err
}
return out, nil
}
// buildBFSQuery assembles the recursive-CTE BFS statement for the given
// direction, seed count, kind count, and whether a LIMIT is applied. It is
// a pure string builder (no I/O) so a test can EXPLAIN QUERY PLAN the exact
// statement and assert the recursive join stays index-driven.
//
// Direction selects the join columns: forward follows from_id -> to_id (the
// discovered neighbour is the edge target), backward follows to_id ->
// from_id (the neighbour is the edge source). The recursive term joins
// edges on the walked node's id, so a forward walk leads with from_id (the
// edges_by_from(from_id, kind) index) and a backward walk leads with to_id
// (edges_by_to(to_id, kind)); the nodes join uses the nodes primary key.
func buildBFSQuery(dir graph.Direction, nSeeds, nKinds int, withLimit bool) string {
joinCol, nextCol := "e.from_id", "e.to_id"
edgeIdx := "edges_by_from"
if dir == graph.DirectionBackward {
joinCol, nextCol = "e.to_id", "e.from_id"
edgeIdx = "edges_by_to"
}
var b strings.Builder
b.WriteString("WITH RECURSIVE seeds(node_id) AS (VALUES ")
for i := 0; i < nSeeds; i++ {
if i > 0 {
b.WriteString(", ")
}
b.WriteString("(?)")
}
b.WriteString("),\n")
b.WriteString("bfs(node_id, depth, parent_id, edge_kind) AS (\n")
b.WriteString(" SELECT node_id, 0, '', '' FROM seeds\n")
b.WriteString(" UNION\n")
b.WriteString(" SELECT " + nextCol + ", b.depth + 1, b.node_id, e.kind\n")
b.WriteString(" FROM bfs b\n")
// INDEXED BY forces the frontier-node seek (from_id / to_id leading)
// instead of the planner's stats-free preference for edges_by_kind,
// which on a hot kind would scan every edge of that kind per frontier
// node. If the index is ever absent (a bulk-load window drops it) the
// query errors and the engine falls back to the in-memory walk.
b.WriteString(" JOIN edges e INDEXED BY " + edgeIdx + " ON " + joinCol + " = b.node_id\n")
b.WriteString(" JOIN nodes n ON n.id = " + nextCol + "\n")
b.WriteString(" WHERE b.depth < ? AND e.kind IN (" + inPlaceholders(nKinds) + ")\n")
b.WriteString("),\n")
b.WriteString("ranked AS (\n")
b.WriteString(" SELECT node_id, depth, parent_id, edge_kind,\n")
b.WriteString(" ROW_NUMBER() OVER (PARTITION BY node_id ORDER BY depth, parent_id, edge_kind) AS rn\n")
b.WriteString(" FROM bfs\n")
b.WriteString(")\n")
b.WriteString("SELECT node_id, depth, parent_id, edge_kind FROM ranked WHERE rn = 1\n")
b.WriteString("ORDER BY depth, node_id")
if withLimit {
b.WriteString("\nLIMIT ?")
}
return b.String()
}
// sortBFSHops orders hops by (depth, node_id) — the same final ordering
// the recursive-CTE query applies, used by the seed-only fast path.
func sortBFSHops(hops []graph.BFSHop) {
sort.Slice(hops, func(i, j int) bool {
if hops[i].Depth != hops[j].Depth {
return hops[i].Depth < hops[j].Depth
}
return hops[i].NodeID < hops[j].NodeID
})
}
@@ -0,0 +1,74 @@
package store_sqlite
import (
"path/filepath"
"strings"
"testing"
"github.com/zzet/gortex/internal/graph"
)
// queryPlan runs EXPLAIN QUERY PLAN for the given statement + args and
// returns the joined detail lines.
func queryPlan(t *testing.T, s *Store, query string, args ...any) string {
t.Helper()
rows, err := s.db.Query("EXPLAIN QUERY PLAN "+query, args...)
if err != nil {
t.Fatalf("EXPLAIN QUERY PLAN: %v", err)
}
defer rows.Close()
var lines []string
for rows.Next() {
var id, parent, notused int
var detail string
if err := rows.Scan(&id, &parent, &notused, &detail); err != nil {
t.Fatalf("scan plan row: %v", err)
}
lines = append(lines, detail)
}
if err := rows.Err(); err != nil {
t.Fatalf("plan rows: %v", err)
}
return strings.Join(lines, "\n")
}
// TestBFSQueryUsesEdgeIndex asserts the recursive-CTE BFS join reaches the
// edges table through the direction's composite index (edges_by_from for a
// forward walk, edges_by_to for a backward walk) rather than a full table
// scan, and that the node-backed gate join uses the nodes primary key.
func TestBFSQueryUsesEdgeIndex(t *testing.T) {
s, err := Open(filepath.Join(t.TempDir(), "store.sqlite"))
if err != nil {
t.Fatalf("open: %v", err)
}
defer s.Close()
// A small fixture so the planner has real tables/indexes to plan against.
for _, id := range []string{"A", "B", "C"} {
s.AddNode(&graph.Node{ID: id, Kind: graph.KindFunction, Name: id, FilePath: "a.go", Language: "go"})
}
s.AddEdge(&graph.Edge{From: "A", To: "B", Kind: graph.EdgeCalls, Confidence: 1, Origin: graph.OriginASTResolved})
s.AddEdge(&graph.Edge{From: "B", To: "C", Kind: graph.EdgeCalls, Confidence: 1, Origin: graph.OriginASTResolved})
forwardQ := buildBFSQuery(graph.DirectionForward, 1, 1, true)
forwardPlan := queryPlan(t, s, forwardQ, "A", 3, string(graph.EdgeCalls), 50)
if !strings.Contains(forwardPlan, "edges_by_from") {
t.Fatalf("forward BFS plan does not use edges_by_from:\n%s", forwardPlan)
}
// The edges table must not be reached by a full scan (alias e).
if strings.Contains(forwardPlan, "SCAN edges") || strings.Contains(forwardPlan, "SCAN e ") {
t.Fatalf("forward BFS plan scans edges instead of using an index:\n%s", forwardPlan)
}
backwardQ := buildBFSQuery(graph.DirectionBackward, 1, 1, true)
backwardPlan := queryPlan(t, s, backwardQ, "C", 3, string(graph.EdgeCalls), 50)
if !strings.Contains(backwardPlan, "edges_by_to") {
t.Fatalf("backward BFS plan does not use edges_by_to:\n%s", backwardPlan)
}
if strings.Contains(backwardPlan, "SCAN edges") || strings.Contains(backwardPlan, "SCAN e ") {
t.Fatalf("backward BFS plan scans edges instead of using an index:\n%s", backwardPlan)
}
t.Logf("forward BFS query plan:\n%s", forwardPlan)
t.Logf("backward BFS query plan:\n%s", backwardPlan)
}
@@ -0,0 +1,130 @@
package store_sqlite
import (
"database/sql"
"github.com/zzet/gortex/internal/graph"
)
var (
_ graph.BlameEnrichmentWriter = (*Store)(nil)
_ graph.BlameEnrichmentReader = (*Store)(nil)
)
const blameChunk = 180
const blameCols = `node_id, repo_prefix, commit_sha, email, ts`
func (s *Store) BulkSetBlame(repoPrefix string, rows []graph.BlameEnrichment) error {
if len(rows) == 0 {
return nil
}
s.writeMu.Lock()
defer s.writeMu.Unlock()
tx, err := s.db.Begin()
if err != nil {
return err
}
defer tx.Rollback() //nolint:errcheck
for start := 0; start < len(rows); start += blameChunk {
end := start + blameChunk
if end > len(rows) {
end = len(rows)
}
batch := rows[start:end]
args := make([]any, 0, len(batch)*5)
stmt := make([]byte, 0, 96+len(batch)*16)
stmt = append(stmt, "INSERT OR REPLACE INTO blame_enrichment ("...)
stmt = append(stmt, blameCols...)
stmt = append(stmt, ") VALUES "...)
for i, e := range batch {
if i > 0 {
stmt = append(stmt, ',')
}
stmt = append(stmt, "(?,?,?,?,?)"...)
args = append(args, e.NodeID, repoPrefix, e.Commit, e.Email, e.Timestamp)
}
if _, err := tx.Exec(string(stmt), args...); err != nil {
return err
}
}
return tx.Commit()
}
func (s *Store) DeleteBlame(nodeIDs []string) error {
if len(nodeIDs) == 0 {
return nil
}
seen := make(map[string]struct{}, len(nodeIDs))
uniq := make([]string, 0, len(nodeIDs))
for _, id := range nodeIDs {
if id == "" {
continue
}
if _, ok := seen[id]; ok {
continue
}
seen[id] = struct{}{}
uniq = append(uniq, id)
}
if len(uniq) == 0 {
return nil
}
s.writeMu.Lock()
defer s.writeMu.Unlock()
tx, err := s.db.Begin()
if err != nil {
return err
}
defer tx.Rollback() //nolint:errcheck
for start := 0; start < len(uniq); start += blameChunk {
end := start + blameChunk
if end > len(uniq) {
end = len(uniq)
}
chunk := uniq[start:end]
args := make([]any, len(chunk))
stmt := make([]byte, 0, 56+len(chunk)*2)
stmt = append(stmt, "DELETE FROM blame_enrichment WHERE node_id IN ("...)
for i, id := range chunk {
if i > 0 {
stmt = append(stmt, ',')
}
stmt = append(stmt, '?')
args[i] = id
}
stmt = append(stmt, ')')
if _, err := tx.Exec(string(stmt), args...); err != nil {
return err
}
}
return tx.Commit()
}
func (s *Store) BlameRows(repoPrefix string) []graph.BlameEnrichment {
var (
rows *sql.Rows
err error
)
if repoPrefix == "" {
rows, err = s.db.Query(`SELECT ` + blameCols + ` FROM blame_enrichment`)
} else {
rows, err = s.db.Query(`SELECT `+blameCols+` FROM blame_enrichment WHERE repo_prefix = ?`, repoPrefix)
}
if err != nil {
return nil
}
defer rows.Close()
var out []graph.BlameEnrichment
for rows.Next() {
var e graph.BlameEnrichment
if err := rows.Scan(&e.NodeID, &e.RepoPrefix, &e.Commit, &e.Email, &e.Timestamp); err != nil {
return out
}
out = append(out, e)
}
if err := rows.Err(); err != nil {
return out
}
return out
}
@@ -0,0 +1,155 @@
package store_sqlite
import (
"database/sql"
"github.com/zzet/gortex/internal/graph"
)
// Compile-time assertions that the SQLite Store satisfies the optional
// git-churn enrichment sidecar capabilities (change A: enrichment moved
// out of nodes.meta into a typed table so the node hot path stops
// encoding rarely-read data into the meta blob and get_churn_rate reads
// via an index instead of an AllNodes scan).
var (
_ graph.ChurnEnrichmentWriter = (*Store)(nil)
_ graph.ChurnEnrichmentReader = (*Store)(nil)
)
// churnChunk bounds rows per multi-row INSERT. churn_enrichment has 10
// columns, so at 10 params/row the 999 host-param limit caps a statement
// at 99 rows; 90 leaves headroom. Mirrors shingleChunk / mtimeChunk.
const churnChunk = 90
const churnCols = `node_id, repo_prefix, commit_count, age_days, churn_rate, last_author, last_commit_at, head_sha, branch, computed_at`
// BulkSetChurn persists every churn row for one repo prefix in a single
// transaction, chunked under the host-parameter limit. Idempotent on
// node_id (INSERT OR REPLACE). Empty input is a no-op.
func (s *Store) BulkSetChurn(repoPrefix string, rows []graph.ChurnEnrichment) error {
if len(rows) == 0 {
return nil
}
s.writeMu.Lock()
defer s.writeMu.Unlock()
tx, err := s.db.Begin()
if err != nil {
return err
}
defer tx.Rollback() //nolint:errcheck // rollback after Commit is a no-op
for start := 0; start < len(rows); start += churnChunk {
end := start + churnChunk
if end > len(rows) {
end = len(rows)
}
batch := rows[start:end]
args := make([]any, 0, len(batch)*10)
stmt := make([]byte, 0, 128+len(batch)*24)
stmt = append(stmt, "INSERT OR REPLACE INTO churn_enrichment ("...)
stmt = append(stmt, churnCols...)
stmt = append(stmt, ") VALUES "...)
for i, e := range batch {
if i > 0 {
stmt = append(stmt, ',')
}
stmt = append(stmt, "(?,?,?,?,?,?,?,?,?,?)"...)
args = append(args, e.NodeID, repoPrefix, e.CommitCount, e.AgeDays,
e.ChurnRate, e.LastAuthor, e.LastCommitAt, e.HeadSHA, e.Branch, e.ComputedAt)
}
if _, err := tx.Exec(string(stmt), args...); err != nil {
return err
}
}
return tx.Commit()
}
// DeleteChurn drops churn rows for the supplied node ids, chunked into
// `node_id IN (?, …)` DELETEs. Empty input is a no-op.
func (s *Store) DeleteChurn(nodeIDs []string) error {
if len(nodeIDs) == 0 {
return nil
}
seen := make(map[string]struct{}, len(nodeIDs))
uniq := make([]string, 0, len(nodeIDs))
for _, id := range nodeIDs {
if id == "" {
continue
}
if _, ok := seen[id]; ok {
continue
}
seen[id] = struct{}{}
uniq = append(uniq, id)
}
if len(uniq) == 0 {
return nil
}
s.writeMu.Lock()
defer s.writeMu.Unlock()
tx, err := s.db.Begin()
if err != nil {
return err
}
defer tx.Rollback() //nolint:errcheck
for start := 0; start < len(uniq); start += churnChunk {
end := start + churnChunk
if end > len(uniq) {
end = len(uniq)
}
chunk := uniq[start:end]
args := make([]any, len(chunk))
stmt := make([]byte, 0, 48+len(chunk)*2)
stmt = append(stmt, "DELETE FROM churn_enrichment WHERE node_id IN ("...)
for i, id := range chunk {
if i > 0 {
stmt = append(stmt, ',')
}
stmt = append(stmt, '?')
args[i] = id
}
stmt = append(stmt, ')')
if _, err := tx.Exec(string(stmt), args...); err != nil {
return err
}
}
return tx.Commit()
}
// ChurnRows returns every churn row for repoPrefix; an EMPTY repoPrefix
// returns ALL rows across repos. This is an index-only read over the
// (small) enriched set — the whole point of the sidecar, replacing the
// AllNodes()+meta-decode scan get_churn_rate used to do.
func (s *Store) ChurnRows(repoPrefix string) []graph.ChurnEnrichment {
var (
rows *sql.Rows
err error
)
if repoPrefix == "" {
rows, err = s.db.Query(`SELECT ` + churnCols + ` FROM churn_enrichment`)
} else {
rows, err = s.db.Query(`SELECT `+churnCols+` FROM churn_enrichment WHERE repo_prefix = ?`, repoPrefix)
}
if err != nil {
return nil
}
defer rows.Close()
var out []graph.ChurnEnrichment
for rows.Next() {
var e graph.ChurnEnrichment
if err := rows.Scan(&e.NodeID, &e.RepoPrefix, &e.CommitCount, &e.AgeDays,
&e.ChurnRate, &e.LastAuthor, &e.LastCommitAt, &e.HeadSHA, &e.Branch, &e.ComputedAt); err != nil {
return out
}
out = append(out, e)
}
if err := rows.Err(); err != nil {
return out
}
return out
}
@@ -0,0 +1,192 @@
package store_sqlite
import (
"encoding/binary"
"github.com/zzet/gortex/internal/graph"
)
// Compile-time assertions that the SQLite Store satisfies the optional
// per-symbol clone-shingle persistence capabilities. Lifting this state
// into the same backend the graph lives in means warm restarts rebuild
// the clone-detection CMS through one persistence surface instead of a
// second gob snapshot.
var (
_ graph.CloneShingleWriter = (*Store)(nil)
_ graph.CloneShingleReader = (*Store)(nil)
)
// shingleChunk bounds how many (node_id, repo_prefix, shingles) tuples
// ride in a single multi-row INSERT. SQLite's default compiled-in host
// parameter limit is 999; at 3 params per row that caps a statement at
// 333 rows, so 300 leaves headroom. Mirrors mtimeChunk.
const shingleChunk = 300
// encodeShingles serialises a uint64 slice to a little-endian BLOB
// (8 bytes per element). A nil/empty slice encodes to an empty BLOB.
func encodeShingles(shingles []uint64) []byte {
b := make([]byte, len(shingles)*8)
for i, s := range shingles {
binary.LittleEndian.PutUint64(b[i*8:], s)
}
return b
}
// decodeShingles is the inverse of encodeShingles. A BLOB whose length
// is not a multiple of 8 yields nil (corrupt row); callers skip nil
// sets. An empty BLOB decodes to an empty (non-nil) slice.
func decodeShingles(b []byte) []uint64 {
if len(b)%8 != 0 {
return nil
}
out := make([]uint64, len(b)/8)
for i := range out {
out[i] = binary.LittleEndian.Uint64(b[i*8:])
}
return out
}
// BulkSetCloneShingles persists every (nodeID -> shingles) entry for
// one repo prefix in a single transaction, chunked so no statement
// exceeds SQLite's host-parameter limit. Idempotent on node_id:
// re-running with overlapping keys replaces in place. Empty input is a
// no-op.
func (s *Store) BulkSetCloneShingles(repoPrefix string, rows map[string][]uint64) error {
if len(rows) == 0 {
return nil
}
s.writeMu.Lock()
defer s.writeMu.Unlock()
// Stable ordering is not required for correctness, but iterating the
// map directly is fine — we only chunk by count.
type kv struct {
id string
blob []byte
}
pending := make([]kv, 0, len(rows))
for id, sh := range rows {
pending = append(pending, kv{id: id, blob: encodeShingles(sh)})
}
tx, err := s.db.Begin()
if err != nil {
return err
}
defer tx.Rollback() //nolint:errcheck // rollback after Commit is a no-op
for start := 0; start < len(pending); start += shingleChunk {
end := start + shingleChunk
if end > len(pending) {
end = len(pending)
}
batch := pending[start:end]
// Build a multi-row INSERT OR REPLACE: (?, ?, ?), (?, ?, ?), ...
args := make([]any, 0, len(batch)*3)
stmt := make([]byte, 0, 64+len(batch)*16)
stmt = append(stmt, "INSERT OR REPLACE INTO clone_shingles (node_id, repo_prefix, shingles) VALUES "...)
for i, e := range batch {
if i > 0 {
stmt = append(stmt, ',')
}
stmt = append(stmt, "(?, ?, ?)"...)
args = append(args, e.id, repoPrefix, e.blob)
}
if _, err := tx.Exec(string(stmt), args...); err != nil {
return err
}
}
return tx.Commit()
}
// DeleteCloneShingles drops the rows for the supplied node ids, chunked
// into `node_id IN (?, ?, …)` DELETEs so no statement exceeds SQLite's
// host-parameter limit. Empty input is a no-op; missing ids are simply
// not deleted.
func (s *Store) DeleteCloneShingles(nodeIDs []string) error {
if len(nodeIDs) == 0 {
return nil
}
// Dedupe + skip empty up front to keep the chunk loop honest.
seen := make(map[string]struct{}, len(nodeIDs))
uniq := make([]string, 0, len(nodeIDs))
for _, id := range nodeIDs {
if id == "" {
continue
}
if _, ok := seen[id]; ok {
continue
}
seen[id] = struct{}{}
uniq = append(uniq, id)
}
if len(uniq) == 0 {
return nil
}
s.writeMu.Lock()
defer s.writeMu.Unlock()
tx, err := s.db.Begin()
if err != nil {
return err
}
defer tx.Rollback() //nolint:errcheck // rollback after Commit is a no-op
for start := 0; start < len(uniq); start += shingleChunk {
end := start + shingleChunk
if end > len(uniq) {
end = len(uniq)
}
chunk := uniq[start:end]
args := make([]any, len(chunk))
stmt := make([]byte, 0, 48+len(chunk)*2)
stmt = append(stmt, "DELETE FROM clone_shingles WHERE node_id IN ("...)
for i, id := range chunk {
if i > 0 {
stmt = append(stmt, ',')
}
stmt = append(stmt, '?')
args[i] = id
}
stmt = append(stmt, ')')
if _, err := tx.Exec(string(stmt), args...); err != nil {
return err
}
}
return tx.Commit()
}
// LoadCloneShingles returns the recorded shingle sets for one repo
// prefix as a fresh map. It always returns a non-nil (possibly empty)
// map and surfaces any query error. An empty/absent prefix yields an
// empty map, not an error.
func (s *Store) LoadCloneShingles(repoPrefix string) (map[string][]uint64, error) {
rows, err := s.db.Query(
`SELECT node_id, shingles FROM clone_shingles WHERE repo_prefix = ?`,
repoPrefix,
)
if err != nil {
return nil, err
}
defer rows.Close()
out := make(map[string][]uint64)
for rows.Next() {
var id string
var blob []byte
if err := rows.Scan(&id, &blob); err != nil {
return nil, err
}
out[id] = decodeShingles(blob)
}
if err := rows.Err(); err != nil {
return nil, err
}
return out, nil
}
@@ -0,0 +1,63 @@
package store_sqlite
// One-time boot compaction support.
//
// The graph store only ever grows on disk: deleted rows (a purged repo, the
// duplicate-collapse migration, resolver cleanups) return their pages to
// SQLite's freelist, where future writes reuse them — but nothing short of
// VACUUM returns them to the filesystem. A long-lived store that shed a large
// fraction of its rows can therefore pin gigabytes of dead file forever (a
// live store sat at 64% freelist — 4.4 GB reclaimable in a 6.8 GB file).
// These methods give the daemon the numbers to decide whether that one-time
// rewrite is worth it, and the lever to run it. The policy (thresholds, disk
// headroom, kill-switch) deliberately lives with the caller: the store cannot
// know whether minutes of exclusive I/O are acceptable right now.
// Path returns the on-disk database file path. Empty when the store was not
// opened from a file — callers using it to reason about the underlying
// filesystem (disk-headroom checks) must treat "" as "unknown, don't".
func (s *Store) Path() string {
return s.dbPath
}
// CompactStats reports how much of the database file is reclaimable dead
// space: freeBytes is the freelist (freelist_count × page_size), totalBytes
// the whole main file (page_count × page_size). Zeros on any pragma error —
// a read failing here is the same teardown race panicOnFatal swallows, and
// "nothing reclaimable" is the answer that makes every caller do nothing.
// The -wal file is excluded on purpose: the checkpoint loop already bounds
// it, and VACUUM only rewrites the main file.
func (s *Store) CompactStats() (freeBytes, totalBytes int64) {
var pageSize, pageCount, freePages int64
if err := s.db.QueryRow(`PRAGMA page_size`).Scan(&pageSize); err != nil {
return 0, 0
}
if err := s.db.QueryRow(`PRAGMA page_count`).Scan(&pageCount); err != nil {
return 0, 0
}
if err := s.db.QueryRow(`PRAGMA freelist_count`).Scan(&freePages); err != nil {
return 0, 0
}
return freePages * pageSize, pageCount * pageSize
}
// Compact rewrites the database file (VACUUM), returning freelist pages to
// the filesystem, then drains the write-ahead log with a TRUNCATE checkpoint
// so the rewrite's WAL traffic doesn't linger as a second oversized file.
//
// Cost model callers must respect: VACUUM copies the live content into a
// temporary database (up to a full extra copy on the same filesystem) and
// needs exclusive access — it blocks Go-side writers via writeMu here, and a
// concurrent reader on another pooled connection makes SQLite wait out
// busy_timeout and then fail. That failure is clean (the store is untouched,
// freelist pages remain reusable), which is why the daemon treats a Compact
// error as skip-and-continue rather than fatal.
func (s *Store) Compact() error {
s.writeMu.Lock()
_, err := s.db.Exec(`VACUUM`)
s.writeMu.Unlock()
if err != nil {
return err
}
return s.CheckpointWAL()
}
@@ -0,0 +1,76 @@
package store_sqlite_test
import (
"fmt"
"strings"
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/zzet/gortex/internal/graph"
)
// TestCompactReclaimsFreelist pins the boot-compaction capability end to end
// on a real file: shedding most rows leaves the file full of freelist pages
// (CompactStats must say so), Compact() must return them to the filesystem
// (page_count shrinks), and the surviving rows must come through untouched.
// Fractional assertions only — page size and per-row overhead are backend
// details this test must not encode.
func TestCompactReclaimsFreelist(t *testing.T) {
s := openTestStore(t)
// Seed 60 files × 50 nodes with ~2 KiB of meta each (~6 MiB of pages),
// then checkpoint so the rows land in the main file rather than the WAL —
// CompactStats deliberately measures only the main file.
const files, perFile = 60, 50
pad := strings.Repeat("x", 2048)
for f := 0; f < files; f++ {
nodes := make([]*graph.Node, 0, perFile)
path := fmt.Sprintf("p/f%03d.go", f)
for n := 0; n < perFile; n++ {
nodes = append(nodes, &graph.Node{
ID: fmt.Sprintf("%s::N%d", path, n),
Kind: graph.KindFunction,
Name: fmt.Sprintf("N%d", n),
FilePath: path,
Meta: map[string]any{"pad": pad},
})
}
s.AddBatch(nodes, nil)
}
require.NoError(t, s.CheckpointWAL())
_, totalSeeded := s.CompactStats()
require.Greater(t, totalSeeded, int64(3<<20), "sanity: seeding must produce a multi-MiB main file")
// Shed all but one file, checkpoint again so the deletions reach the main
// file's freelist.
for f := 1; f < files; f++ {
s.EvictFile(fmt.Sprintf("p/f%03d.go", f))
}
require.NoError(t, s.CheckpointWAL())
freeBefore, totalBefore := s.CompactStats()
assert.Greater(t, freeBefore*3, totalBefore,
"after shedding ~98%% of rows the freelist must dominate the file (free=%d total=%d)", freeBefore, totalBefore)
assert.Greater(t, freeBefore, int64(1<<20), "freelist must be at least MiB-scale to make the shrink observable")
require.NoError(t, s.Compact())
freeAfter, totalAfter := s.CompactStats()
assert.Less(t, totalAfter, totalBefore/2,
"VACUUM must return the dead majority to the filesystem (before=%d after=%d)", totalBefore, totalAfter)
assert.Less(t, freeAfter, totalBefore/10,
"post-VACUUM freelist must be near empty (free=%d)", freeAfter)
// Survivors intact, evicted rows gone.
assert.Equal(t, perFile, s.NodeCount(), "compaction must not change the row count")
kept := s.GetNode("p/f000.go::N0")
require.NotNil(t, kept, "kept row must survive VACUUM")
assert.Equal(t, pad, kept.Meta["pad"], "kept row's meta blob must round-trip through VACUUM")
assert.Nil(t, s.GetNode("p/f001.go::N0"), "evicted row must stay gone")
// The store stays fully writable after the rewrite.
s.AddNode(&graph.Node{ID: "p/new.go::After", Kind: graph.KindFunction, Name: "After", FilePath: "p/new.go"})
assert.NotNil(t, s.GetNode("p/new.go::After"))
}
@@ -0,0 +1,144 @@
package store_sqlite
import (
"github.com/zzet/gortex/internal/graph"
)
// Compile-time assertions that the SQLite Store satisfies the optional
// constant-value persistence capability. A KindConstant node's literal
// value lives in this queryable sidecar (not the JSON Meta blob)
// so the resolver can dereference a const-identifier dispatch name across
// files without an unindexable per-node blob decode.
var (
_ graph.ConstantValueWriter = (*Store)(nil)
_ graph.ConstantValueReader = (*Store)(nil)
)
// constValueChunk bounds rows per multi-row INSERT (4 params/row; 80 rows
// = 320 host params, well under SQLite's 999 default).
const constValueChunk = 80
// BulkSetConstantValues persists constant values for one repo prefix in a
// single transaction, chunked under the host-parameter limit. Idempotent
// on the node_id primary key. Empty input is a no-op.
func (s *Store) BulkSetConstantValues(repoPrefix string, rows []graph.ConstantValueRow) error {
if len(rows) == 0 {
return nil
}
s.writeMu.Lock()
defer s.writeMu.Unlock()
tx, err := s.db.Begin()
if err != nil {
return err
}
defer tx.Rollback() //nolint:errcheck // rollback after Commit is a no-op
for start := 0; start < len(rows); start += constValueChunk {
end := start + constValueChunk
if end > len(rows) {
end = len(rows)
}
batch := rows[start:end]
args := make([]any, 0, len(batch)*4)
stmt := make([]byte, 0, 96+len(batch)*16)
stmt = append(stmt, "INSERT OR REPLACE INTO constant_values (node_id, repo_prefix, file_path, value) VALUES "...)
for i, r := range batch {
if i > 0 {
stmt = append(stmt, ',')
}
stmt = append(stmt, "(?, ?, ?, ?)"...)
args = append(args, r.NodeID, repoPrefix, r.FilePath, r.Value)
}
if _, err := tx.Exec(string(stmt), args...); err != nil {
return err
}
}
return tx.Commit()
}
// DeleteConstantValuesByFiles drops all constant values sourced in the
// supplied files for one repo prefix, chunked into `file_path IN (…)`
// DELETEs. Empty input is a no-op.
func (s *Store) DeleteConstantValuesByFiles(repoPrefix string, files []string) error {
if len(files) == 0 {
return nil
}
s.writeMu.Lock()
defer s.writeMu.Unlock()
tx, err := s.db.Begin()
if err != nil {
return err
}
defer tx.Rollback() //nolint:errcheck // rollback after Commit is a no-op
for start := 0; start < len(files); start += constValueChunk {
end := start + constValueChunk
if end > len(files) {
end = len(files)
}
chunk := files[start:end]
args := make([]any, 0, len(chunk)+1)
args = append(args, repoPrefix)
stmt := make([]byte, 0, 64+len(chunk)*2)
stmt = append(stmt, "DELETE FROM constant_values WHERE repo_prefix = ? AND file_path IN ("...)
for i, f := range chunk {
if i > 0 {
stmt = append(stmt, ',')
}
stmt = append(stmt, '?')
args = append(args, f)
}
stmt = append(stmt, ')')
if _, err := tx.Exec(string(stmt), args...); err != nil {
return err
}
}
return tx.Commit()
}
// ConstantValuesByNodeIDs returns the persisted values for the supplied
// node ids (omitting ids with no recorded value). Always non-nil.
func (s *Store) ConstantValuesByNodeIDs(nodeIDs []string) (map[string]string, error) {
out := make(map[string]string, len(nodeIDs))
if len(nodeIDs) == 0 {
return out, nil
}
for start := 0; start < len(nodeIDs); start += constValueChunk {
end := start + constValueChunk
if end > len(nodeIDs) {
end = len(nodeIDs)
}
chunk := nodeIDs[start:end]
args := make([]any, 0, len(chunk))
stmt := make([]byte, 0, 64+len(chunk)*2)
stmt = append(stmt, "SELECT node_id, value FROM constant_values WHERE node_id IN ("...)
for i, id := range chunk {
if i > 0 {
stmt = append(stmt, ',')
}
stmt = append(stmt, '?')
args = append(args, id)
}
stmt = append(stmt, ')')
rows, err := s.db.Query(string(stmt), args...)
if err != nil {
return nil, err
}
for rows.Next() {
var id, val string
if err := rows.Scan(&id, &val); err != nil {
_ = rows.Close()
return nil, err
}
out[id] = val
}
if err := rows.Err(); err != nil {
_ = rows.Close()
return nil, err
}
_ = rows.Close()
}
return out, nil
}
@@ -0,0 +1,73 @@
package store_sqlite_test
import (
"path/filepath"
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/zzet/gortex/internal/graph"
"github.com/zzet/gortex/internal/graph/store_sqlite"
)
func openConstValStore(t *testing.T) *store_sqlite.Store {
t.Helper()
s, err := store_sqlite.Open(filepath.Join(t.TempDir(), "cv.sqlite"))
require.NoError(t, err)
t.Cleanup(func() { _ = s.Close() })
return s
}
func TestConstantValues_Roundtrip(t *testing.T) {
s := openConstValStore(t)
rows := []graph.ConstantValueRow{
{NodeID: "a.go::ChargeCardActivity", FilePath: "a.go", Value: "ChargeCard"},
{NodeID: "a.go::RefundActivity", FilePath: "a.go", Value: "Refund"},
}
require.NoError(t, s.BulkSetConstantValues("repo", rows))
got, err := s.ConstantValuesByNodeIDs([]string{"a.go::ChargeCardActivity", "a.go::RefundActivity", "missing"})
require.NoError(t, err)
assert.Equal(t, "ChargeCard", got["a.go::ChargeCardActivity"])
assert.Equal(t, "Refund", got["a.go::RefundActivity"])
_, ok := got["missing"]
assert.False(t, ok)
}
func TestConstantValues_DeleteByFile(t *testing.T) {
s := openConstValStore(t)
require.NoError(t, s.BulkSetConstantValues("repo", []graph.ConstantValueRow{
{NodeID: "a.go::X", FilePath: "a.go", Value: "vx"},
{NodeID: "b.go::Y", FilePath: "b.go", Value: "vy"},
}))
require.NoError(t, s.DeleteConstantValuesByFiles("repo", []string{"a.go"}))
got, err := s.ConstantValuesByNodeIDs([]string{"a.go::X", "b.go::Y"})
require.NoError(t, err)
_, gone := got["a.go::X"]
assert.False(t, gone, "a.go's value must be deleted")
assert.Equal(t, "vy", got["b.go::Y"], "b.go's value must remain")
}
func TestConstantValues_Replace(t *testing.T) {
s := openConstValStore(t)
require.NoError(t, s.BulkSetConstantValues("repo", []graph.ConstantValueRow{
{NodeID: "a.go::X", FilePath: "a.go", Value: "old"},
}))
require.NoError(t, s.BulkSetConstantValues("repo", []graph.ConstantValueRow{
{NodeID: "a.go::X", FilePath: "a.go", Value: "new"},
}))
got, err := s.ConstantValuesByNodeIDs([]string{"a.go::X"})
require.NoError(t, err)
assert.Equal(t, "new", got["a.go::X"], "INSERT OR REPLACE must update by node_id PK")
}
func TestConstantValues_EmptyNoop(t *testing.T) {
s := openConstValStore(t)
require.NoError(t, s.BulkSetConstantValues("repo", nil))
require.NoError(t, s.DeleteConstantValuesByFiles("repo", nil))
got, err := s.ConstantValuesByNodeIDs(nil)
require.NoError(t, err)
assert.Empty(t, got)
}
@@ -0,0 +1,297 @@
package store_sqlite
import (
"database/sql"
"strings"
"github.com/zzet/gortex/internal/graph"
)
// This file implements graph.ContentSearcher on the SQLite backend using
// the content_fts FTS5 virtual table declared in schema.go — the
// dedicated, on-disk full-text index for CONTENT (data_class="content")
// section bodies, kept physically separate from symbol_fts so content
// text never reaches the symbol search or the code-oriented graph passes.
//
// Streamed build: WipeContent(repoPrefix) once at the start of a full
// index, AppendContent each content file's sections as they are parsed
// (no per-file wipe), then BuildContentIndex to merge segments.
// Incremental reindex of one content file is WipeContentFile +
// AppendContent.
// Compile-time assertion: *Store satisfies the content-search capability.
var _ graph.ContentSearcher = (*Store)(nil)
// contentInsertChunkRows bounds rows per multi-row INSERT. Each row binds
// 5 host params (node_id, repo_prefix, file_path, ordinal, body); 180 rows
// is 900 params, comfortably under SQLite's default 999-variable limit.
const contentInsertChunkRows = 180
// WipeContent removes a repo's content rows before a full rebuild. Empty
// repoPrefix wipes the whole table (single-repo / conformance behaviour).
func (s *Store) WipeContent(repoPrefix string) error {
s.writeMu.Lock()
defer s.writeMu.Unlock()
_, err := s.db.Exec(`DELETE FROM content_fts WHERE repo_prefix = ?`, repoPrefix)
return err
}
// WipeContentFile removes one file's content rows — the incremental
// reindex path when a single content file changes.
func (s *Store) WipeContentFile(filePath string) error {
if filePath == "" {
return nil
}
s.writeMu.Lock()
defer s.writeMu.Unlock()
_, err := s.db.Exec(`DELETE FROM content_fts WHERE file_path = ?`, filePath)
return err
}
// WipeContentFileInRepo removes ONE file's content rows scoped to a repo —
// the crash-safe full-index sibling of WipeContentFile (which keys on
// file_path alone and so would clobber a same-named file in another repo).
// A full index streams content per file: delete this file's prior rows,
// then AppendContent its fresh sections — so a mid-parse kill leaves a mix
// of old+new content per file rather than the empty table a repo-wide
// pre-wipe would leave behind. Empty filePath is a no-op.
func (s *Store) WipeContentFileInRepo(repoPrefix, filePath string) error {
if filePath == "" {
return nil
}
s.writeMu.Lock()
defer s.writeMu.Unlock()
_, err := s.db.Exec(`DELETE FROM content_fts WHERE repo_prefix = ? AND file_path = ?`, repoPrefix, filePath)
return err
}
// DeleteContentFilesForRepoNotIn sweeps a repo's content rows down to keep —
// every content row whose file_path is absent from keep is deleted. keep is
// the set of files that actually STREAMED content sections in the walk just
// completed (each recorded as the same file_path form AppendContent wrote),
// NOT the set of files that merely survive on disk: a file can still exist
// yet stop producing content (doc emptied, classification changed), and a
// disk-based keep would protect its stale rows forever. Run once at the end
// of a successful full index (right after the authoritative mtime replace),
// it reaps vanished files and content->no-content transitions in one scan;
// the per-file WipeContentFileInRepo + AppendContent streaming build
// refreshes the files that still produce content. Together they replace the
// old repo-wide pre-wipe: a mid-parse kill no longer empties the content
// index, and stale rows are reaped only on the next clean completion.
// Empty keep is a deliberate no-op safety net — never wipe a whole repo from
// an empty set; a caller that legitimately ends a walk with zero content
// files calls WipeContent explicitly instead.
func (s *Store) DeleteContentFilesForRepoNotIn(repoPrefix string, keep map[string]struct{}) error {
if len(keep) == 0 {
return nil
}
s.writeMu.Lock()
defer s.writeMu.Unlock()
// Enumerate the repo's content file paths, then delete only those not in
// keep. Content files are a small subset (docs / PDFs / office), so the
// DISTINCT scan + targeted deletes stay cheap and dodge a giant NOT IN
// (...) bound-variable list. Rows are drained + closed before the delete
// tx opens (no open read cursor while writing on the same connection).
rows, err := s.db.Query(`SELECT DISTINCT file_path FROM content_fts WHERE repo_prefix = ?`, repoPrefix)
if err != nil {
return err
}
var vanished []string
for rows.Next() {
var fp string
if err := rows.Scan(&fp); err != nil {
_ = rows.Close()
return err
}
if _, ok := keep[fp]; !ok {
vanished = append(vanished, fp)
}
}
if err := rows.Err(); err != nil {
_ = rows.Close()
return err
}
_ = rows.Close()
if len(vanished) == 0 {
return nil
}
tx, err := s.db.Begin()
if err != nil {
return err
}
defer tx.Rollback() //nolint:errcheck // rollback after Commit is a no-op
const chunk = 900
for start := 0; start < len(vanished); start += chunk {
end := minInt(start+chunk, len(vanished))
batch := vanished[start:end]
placeholders := strings.TrimSuffix(strings.Repeat("?,", len(batch)), ",")
args := make([]any, 0, len(batch)+1)
args = append(args, repoPrefix)
for _, fp := range batch {
args = append(args, fp)
}
if _, err := tx.Exec(`DELETE FROM content_fts WHERE repo_prefix = ? AND file_path IN (`+placeholders+`)`, args...); err != nil {
return err
}
}
return tx.Commit()
}
// AppendContent inserts content rows for repoPrefix without wiping — the
// streamed per-file build path. Callers wipe (whole repo or one file)
// first. Rows with an empty NodeID are skipped.
func (s *Store) AppendContent(repoPrefix string, items []graph.ContentFTSItem) error {
if len(items) == 0 {
return nil
}
s.writeMu.Lock()
defer s.writeMu.Unlock()
tx, err := s.db.Begin()
if err != nil {
return err
}
commit := false
defer func() {
if !commit {
_ = tx.Rollback()
}
}()
for start := 0; start < len(items); start += contentInsertChunkRows {
end := minInt(start+contentInsertChunkRows, len(items))
chunk := items[start:end]
var b strings.Builder
b.WriteString(`INSERT INTO content_fts (node_id, repo_prefix, file_path, ordinal, body) VALUES `)
args := make([]any, 0, len(chunk)*5)
for _, it := range chunk {
if it.NodeID == "" {
continue
}
if len(args) > 0 {
b.WriteByte(',')
}
b.WriteString(`(?,?,?,?,?)`)
args = append(args, it.NodeID, repoPrefix, it.FilePath, it.Ordinal, it.Body)
}
if len(args) == 0 {
continue
}
if _, err := tx.Exec(b.String(), args...); err != nil {
return err
}
}
if err := tx.Commit(); err != nil {
return err
}
commit = true
return nil
}
// BuildContentIndex opportunistically merges FTS5 segments (a read-latency
// improvement). Like BuildSymbolIndex it is a no-op for correctness — the
// FTS index is maintained incrementally on every insert — and idempotent.
func (s *Store) BuildContentIndex() error {
s.writeMu.Lock()
defer s.writeMu.Unlock()
_, _ = s.db.Exec(`INSERT INTO content_fts(content_fts) VALUES('optimize')`)
return nil
}
// SearchContent runs a content query scoped to repoPrefix (empty = all
// repos) and returns hits ordered by descending relevance, each carrying a
// short snippet excerpt from the matched body. Reuses the symbol path's
// write-side tokeniser (buildFTSMatch) so the content corpus and queries
// agree on camelCase / path-separator splitting.
func (s *Store) SearchContent(query, repoPrefix string, limit int) ([]graph.ContentHit, error) {
if query == "" {
return nil, nil
}
if limit <= 0 {
limit = 20
}
match := s.buildFTSMatch(query)
if match == "" {
return nil, nil
}
var sb strings.Builder
// snippet() over the body column (index 4): no highlight marks, an
// ellipsis for elision, ~16 tokens of context. CAST(ordinal AS INTEGER)
// forces integer affinity so the FTS5 text column scans cleanly into an
// int.
sb.WriteString(`SELECT node_id, file_path, CAST(ordinal AS INTEGER), snippet(content_fts, 4, '', '', '…', 16), bm25(content_fts) FROM content_fts WHERE content_fts MATCH ?`)
args := []any{match}
if repoPrefix != "" {
sb.WriteString(` AND repo_prefix = ?`)
args = append(args, repoPrefix)
}
sb.WriteString(` ORDER BY bm25(content_fts) LIMIT ?`)
args = append(args, limit)
rows, err := s.db.Query(sb.String(), args...)
if err != nil {
return nil, err
}
defer rows.Close()
var hits []graph.ContentHit
for rows.Next() {
var (
id, fp, snip string
ordinal int
score float64
)
if err := rows.Scan(&id, &fp, &ordinal, &snip, &score); err != nil {
return nil, err
}
if id == "" {
continue
}
// bm25() is negative-better in SQLite; negate so higher = better,
// matching the ContentHit contract. Rows already arrive best-first.
hits = append(hits, graph.ContentHit{
NodeID: id,
FilePath: fp,
Ordinal: ordinal,
Score: -score,
Snippet: snip,
})
}
if err := rows.Err(); err != nil {
return nil, err
}
return hits, nil
}
// ScanContent streams every content row (scoped to repoPrefix; empty = all
// repos) to fn with its FULL body, read incrementally via a cursor so a
// consumer iterating hundreds of thousands of sections stays bounded. fn
// returns false to stop the scan early.
func (s *Store) ScanContent(repoPrefix string, fn func(nodeID, filePath, body string) bool) error {
var rows *sql.Rows
var err error
if repoPrefix == "" {
rows, err = s.db.Query(`SELECT node_id, file_path, body FROM content_fts`)
} else {
rows, err = s.db.Query(`SELECT node_id, file_path, body FROM content_fts WHERE repo_prefix = ?`, repoPrefix)
}
if err != nil {
return err
}
defer rows.Close()
for rows.Next() {
var nodeID, filePath, body string
if err := rows.Scan(&nodeID, &filePath, &body); err != nil {
return err
}
if !fn(nodeID, filePath, body) {
return nil
}
}
return rows.Err()
}
@@ -0,0 +1,82 @@
package store_sqlite
import (
"path/filepath"
"testing"
"github.com/stretchr/testify/require"
"github.com/zzet/gortex/internal/graph"
)
func TestContentFTS_BasicAndFileWipe(t *testing.T) {
s, err := Open(filepath.Join(t.TempDir(), "c.sqlite"))
require.NoError(t, err)
t.Cleanup(func() { _ = s.Close() })
var cs graph.ContentSearcher = s // runtime assertion the store satisfies the capability
require.NoError(t, cs.WipeContent("")) // clean table
items := []graph.ContentFTSItem{
{NodeID: "a.txt::doc:section-0", FilePath: "a.txt", Ordinal: 0, Body: "the quick brown fox jumps over the lazy dog"},
{NodeID: "a.txt::doc:section-1", FilePath: "a.txt", Ordinal: 1, Body: "lorem ipsum dolor sit amet consectetur"},
{NodeID: "b.pdf::doc:pdf_page-0", FilePath: "b.pdf", Ordinal: 0, Body: "quantum entanglement and superposition explained"},
}
require.NoError(t, cs.AppendContent("", items))
require.NoError(t, cs.BuildContentIndex())
// A body term resolves to exactly its section, with a non-empty snippet.
hits, err := cs.SearchContent("quantum", "", 10)
require.NoError(t, err)
require.Len(t, hits, 1)
require.Equal(t, "b.pdf::doc:pdf_page-0", hits[0].NodeID)
require.Equal(t, "b.pdf", hits[0].FilePath)
require.Equal(t, 0, hits[0].Ordinal)
require.NotEmpty(t, hits[0].Snippet)
hits, err = cs.SearchContent("fox", "", 10)
require.NoError(t, err)
require.Len(t, hits, 1)
require.Equal(t, "a.txt::doc:section-0", hits[0].NodeID)
require.Equal(t, 0, hits[0].Ordinal)
// WipeContentFile drops only a.txt's rows; b.pdf survives.
require.NoError(t, cs.WipeContentFile("a.txt"))
hits, err = cs.SearchContent("fox", "", 10)
require.NoError(t, err)
require.Empty(t, hits)
hits, err = cs.SearchContent("quantum", "", 10)
require.NoError(t, err)
require.Len(t, hits, 1)
}
func TestContentFTS_RepoScoping(t *testing.T) {
s, err := Open(filepath.Join(t.TempDir(), "c.sqlite"))
require.NoError(t, err)
t.Cleanup(func() { _ = s.Close() })
require.NoError(t, s.AppendContent("repoA", []graph.ContentFTSItem{
{NodeID: "repoA::x.txt::doc:section-0", FilePath: "x.txt", Body: "alpha beta gamma"},
}))
require.NoError(t, s.AppendContent("repoB", []graph.ContentFTSItem{
{NodeID: "repoB::y.txt::doc:section-0", FilePath: "y.txt", Body: "alpha delta epsilon"},
}))
// Scoped search returns only the matching repo's hit.
hits, err := s.SearchContent("alpha", "repoA", 10)
require.NoError(t, err)
require.Len(t, hits, 1)
require.Equal(t, "repoA::x.txt::doc:section-0", hits[0].NodeID)
// Unscoped search spans both repos.
hits, err = s.SearchContent("alpha", "", 10)
require.NoError(t, err)
require.Len(t, hits, 2)
// WipeContent scopes to one repo, leaving the sibling intact.
require.NoError(t, s.WipeContent("repoA"))
hits, err = s.SearchContent("alpha", "", 10)
require.NoError(t, err)
require.Len(t, hits, 1)
require.Equal(t, "repoB::y.txt::doc:section-0", hits[0].NodeID)
}
@@ -0,0 +1,143 @@
package store_sqlite
import (
"database/sql"
"github.com/zzet/gortex/internal/graph"
)
var (
_ graph.CoverageEnrichmentWriter = (*Store)(nil)
_ graph.CoverageEnrichmentReader = (*Store)(nil)
)
// coverageChunk bounds rows per multi-row INSERT (5 cols → 5 params/row;
// 999/5 ≈ 199 max, 180 leaves headroom).
const coverageChunk = 180
const coverageCols = `node_id, repo_prefix, coverage_pct, num_stmt, hit`
// BulkSetCoverage persists coverage rows for one repo prefix in a single
// chunked transaction. Idempotent on node_id. Empty input is a no-op.
func (s *Store) BulkSetCoverage(repoPrefix string, rows []graph.CoverageEnrichment) error {
if len(rows) == 0 {
return nil
}
s.writeMu.Lock()
defer s.writeMu.Unlock()
tx, err := s.db.Begin()
if err != nil {
return err
}
defer tx.Rollback() //nolint:errcheck
for start := 0; start < len(rows); start += coverageChunk {
end := start + coverageChunk
if end > len(rows) {
end = len(rows)
}
batch := rows[start:end]
args := make([]any, 0, len(batch)*5)
stmt := make([]byte, 0, 96+len(batch)*16)
stmt = append(stmt, "INSERT OR REPLACE INTO coverage_enrichment ("...)
stmt = append(stmt, coverageCols...)
stmt = append(stmt, ") VALUES "...)
for i, e := range batch {
if i > 0 {
stmt = append(stmt, ',')
}
stmt = append(stmt, "(?,?,?,?,?)"...)
args = append(args, e.NodeID, repoPrefix, e.CoveragePct, e.NumStmt, e.Hit)
}
if _, err := tx.Exec(string(stmt), args...); err != nil {
return err
}
}
return tx.Commit()
}
// DeleteCoverage drops coverage rows for the supplied node ids, chunked.
func (s *Store) DeleteCoverage(nodeIDs []string) error {
if len(nodeIDs) == 0 {
return nil
}
seen := make(map[string]struct{}, len(nodeIDs))
uniq := make([]string, 0, len(nodeIDs))
for _, id := range nodeIDs {
if id == "" {
continue
}
if _, ok := seen[id]; ok {
continue
}
seen[id] = struct{}{}
uniq = append(uniq, id)
}
if len(uniq) == 0 {
return nil
}
s.writeMu.Lock()
defer s.writeMu.Unlock()
tx, err := s.db.Begin()
if err != nil {
return err
}
defer tx.Rollback() //nolint:errcheck
for start := 0; start < len(uniq); start += coverageChunk {
end := start + coverageChunk
if end > len(uniq) {
end = len(uniq)
}
chunk := uniq[start:end]
args := make([]any, len(chunk))
stmt := make([]byte, 0, 56+len(chunk)*2)
stmt = append(stmt, "DELETE FROM coverage_enrichment WHERE node_id IN ("...)
for i, id := range chunk {
if i > 0 {
stmt = append(stmt, ',')
}
stmt = append(stmt, '?')
args[i] = id
}
stmt = append(stmt, ')')
if _, err := tx.Exec(string(stmt), args...); err != nil {
return err
}
}
return tx.Commit()
}
// CoverageRows returns coverage rows for repoPrefix; empty repoPrefix
// returns ALL rows across repos. Index-only read over the enriched set.
func (s *Store) CoverageRows(repoPrefix string) []graph.CoverageEnrichment {
var (
rows *sql.Rows
err error
)
if repoPrefix == "" {
rows, err = s.db.Query(`SELECT ` + coverageCols + ` FROM coverage_enrichment`)
} else {
rows, err = s.db.Query(`SELECT `+coverageCols+` FROM coverage_enrichment WHERE repo_prefix = ?`, repoPrefix)
}
if err != nil {
return nil
}
defer rows.Close()
var out []graph.CoverageEnrichment
for rows.Next() {
var e graph.CoverageEnrichment
if err := rows.Scan(&e.NodeID, &e.RepoPrefix, &e.CoveragePct, &e.NumStmt, &e.Hit); err != nil {
return out
}
out = append(out, e)
}
if err := rows.Err(); err != nil {
return out
}
return out
}
@@ -0,0 +1,21 @@
package store_sqlite
import "os"
// DBStats returns the on-disk size of the SQLite database file and its
// write-ahead log, in bytes. A missing file (or a store opened without a
// path) reports 0 for that component. Surfaced in daemon_health so a
// runaway WAL high-water mark is observable instead of silently filling
// the disk.
func (s *Store) DBStats() (dbBytes, walBytes int64) {
if s == nil || s.dbPath == "" {
return 0, 0
}
if fi, err := os.Stat(s.dbPath); err == nil {
dbBytes = fi.Size()
}
if fi, err := os.Stat(s.dbPath + "-wal"); err == nil {
walBytes = fi.Size()
}
return dbBytes, walBytes
}
@@ -0,0 +1,33 @@
package store_sqlite_test
import (
"path/filepath"
"testing"
"github.com/stretchr/testify/require"
"github.com/zzet/gortex/internal/graph"
"github.com/zzet/gortex/internal/graph/store_sqlite"
)
func TestDBStats(t *testing.T) {
path := filepath.Join(t.TempDir(), "g.sqlite")
s, err := store_sqlite.Open(path)
require.NoError(t, err)
t.Cleanup(func() { _ = s.Close() })
// Force a write so the DB (and WAL, in WAL mode) carry real bytes.
s.AddBatch([]*graph.Node{
{ID: "a.go::Foo", Kind: graph.KindFunction, Name: "Foo", FilePath: "a.go"},
}, nil)
dbBytes, walBytes := s.DBStats()
require.Greater(t, dbBytes, int64(0), "the on-disk DB file must have nonzero size")
require.GreaterOrEqual(t, walBytes, int64(0), "WAL size is non-negative (0 after a checkpoint)")
// A store with no path (the zero value) reports zero, never panics.
var empty store_sqlite.Store
db, wal := empty.DBStats()
require.Equal(t, int64(0), db)
require.Equal(t, int64(0), wal)
}
@@ -0,0 +1,45 @@
package store_sqlite
import (
"database/sql"
"github.com/zzet/gortex/internal/graph"
)
// Compile-time assertion that the SQLite Store persists the enrichment
// completion marker. Lifting this state into the same backend the graph
// lives in lets a warm restart skip re-enriching a repo whose persisted
// graph already carries its LSP edges — no second persistence surface.
var _ graph.EnrichmentStateStore = (*Store)(nil)
// SetEnrichmentState upserts the completion marker for one (repo, provider) —
// written when a provider finishes a non-partial enrichment pass. One row per
// (repo_prefix, provider).
func (s *Store) SetEnrichmentState(st graph.EnrichmentState) error {
s.writeMu.Lock()
defer s.writeMu.Unlock()
_, err := s.db.Exec(`
INSERT OR REPLACE INTO enrichment_state
(repo_prefix, provider, indexed_sha, completed_at, coverage)
VALUES (?, ?, ?, ?, ?)`,
st.RepoPrefix, st.Provider, st.IndexedSHA, st.CompletedAt, st.Coverage)
return err
}
// GetEnrichmentState returns the recorded completion marker for a
// (repo, provider). The bool is false when no row exists yet (never-enriched
// or pre-feature).
func (s *Store) GetEnrichmentState(repoPrefix, provider string) (graph.EnrichmentState, bool, error) {
row := s.db.QueryRow(`
SELECT indexed_sha, completed_at, coverage
FROM enrichment_state WHERE repo_prefix = ? AND provider = ?`, repoPrefix, provider)
st := graph.EnrichmentState{RepoPrefix: repoPrefix, Provider: provider}
err := row.Scan(&st.IndexedSHA, &st.CompletedAt, &st.Coverage)
if err == sql.ErrNoRows {
return graph.EnrichmentState{RepoPrefix: repoPrefix, Provider: provider}, false, nil
}
if err != nil {
return graph.EnrichmentState{RepoPrefix: repoPrefix, Provider: provider}, false, err
}
return st, true, nil
}
@@ -0,0 +1,97 @@
package store_sqlite
import (
"path/filepath"
"testing"
"github.com/zzet/gortex/internal/graph"
)
// TestEnrichmentStateRoundTrip: set/get, missing-row, upsert-in-place, and
// per-provider row isolation on the SQLite store.
func TestEnrichmentStateRoundTrip(t *testing.T) {
s, err := Open(filepath.Join(t.TempDir(), "store.sqlite"))
if err != nil {
t.Fatalf("Open: %v", err)
}
defer s.Close()
// Missing row → found=false, no error (the "never enriched" signal).
if _, found, err := s.GetEnrichmentState("repo", "gopls"); err != nil || found {
t.Fatalf("missing row: found=%v err=%v, want found=false, err=nil", found, err)
}
st := graph.EnrichmentState{
RepoPrefix: "repo",
Provider: "gopls",
IndexedSHA: "abc123",
CompletedAt: 1_700_000_000,
Coverage: 91.5,
}
if err := s.SetEnrichmentState(st); err != nil {
t.Fatalf("SetEnrichmentState: %v", err)
}
got, found, err := s.GetEnrichmentState("repo", "gopls")
if err != nil || !found {
t.Fatalf("get after set: found=%v err=%v, want found=true", found, err)
}
if got.RepoPrefix != "repo" || got.Provider != "gopls" ||
got.IndexedSHA != "abc123" || got.CompletedAt != 1_700_000_000 || got.Coverage != 91.5 {
t.Fatalf("round-trip mismatch: %+v", got)
}
// Upsert on (repo_prefix, provider) replaces in place.
st.IndexedSHA = "def456"
st.Coverage = 100
if err := s.SetEnrichmentState(st); err != nil {
t.Fatalf("upsert: %v", err)
}
got, _, _ = s.GetEnrichmentState("repo", "gopls")
if got.IndexedSHA != "def456" || got.Coverage != 100 {
t.Fatalf("upsert did not replace in place: %+v", got)
}
// A different provider under the same repo is a distinct row.
if _, found, _ := s.GetEnrichmentState("repo", "scip-go"); found {
t.Fatalf("a different provider must be its own row, got found=true")
}
}
// TestOpenPreservesEnrichmentStateOnReopen proves the new table is picked up
// by an existing store with no wipe: a marker written on the first open
// survives the second open (which re-runs schemaSQL unconditionally) and the
// schema version does not drift. Mirrors TestOpenAtCurrentVersionIsNoOp.
func TestOpenPreservesEnrichmentStateOnReopen(t *testing.T) {
path := filepath.Join(t.TempDir(), "store.sqlite")
s, err := Open(path)
if err != nil {
t.Fatalf("first open: %v", err)
}
if err := s.SetEnrichmentState(graph.EnrichmentState{
RepoPrefix: "r", Provider: "gopls", IndexedSHA: "sha1", CompletedAt: 42, Coverage: 88,
}); err != nil {
t.Fatalf("seed marker: %v", err)
}
if err := s.Close(); err != nil {
t.Fatalf("close: %v", err)
}
s2, err := Open(path)
if err != nil {
t.Fatalf("reopen: %v", err)
}
defer s2.Close()
got, found, err := s2.GetEnrichmentState("r", "gopls")
if err != nil || !found {
t.Fatalf("marker lost across reopen: found=%v err=%v (the table must survive a no-op reopen)", found, err)
}
if got.IndexedSHA != "sha1" || got.Coverage != 88 {
t.Fatalf("marker corrupted across reopen: %+v", got)
}
if v, _ := readUserVersion(s2.db); v != currentSchemaVersion {
t.Fatalf("schema version drifted to %d after reopen, want %d", v, currentSchemaVersion)
}
}
+118
View File
@@ -0,0 +1,118 @@
package store_sqlite
import (
"github.com/zzet/gortex/internal/graph"
)
// Compile-time assertions that the SQLite Store satisfies the optional
// per-file metadata persistence capability (the files sidecar feeding
// index_health's per-file parse-error / node-count rollup).
var (
_ graph.FileMetaWriter = (*Store)(nil)
_ graph.FileMetaReader = (*Store)(nil)
)
// fileMetaChunk bounds rows per multi-row INSERT (6 params/row; 80 rows =
// 480 host params, well under SQLite's 999 default).
const fileMetaChunk = 80
// SetFileMetas upserts per-file metadata rows for one repo prefix in a single
// transaction, chunked under the host-parameter limit. Idempotent on the
// (repo_prefix, file_path) primary key. Empty input is a no-op.
func (s *Store) SetFileMetas(repoPrefix string, rows []graph.FileMetaRow) error {
if len(rows) == 0 {
return nil
}
s.writeMu.Lock()
defer s.writeMu.Unlock()
tx, err := s.db.Begin()
if err != nil {
return err
}
defer tx.Rollback() //nolint:errcheck // rollback after Commit is a no-op
for start := 0; start < len(rows); start += fileMetaChunk {
end := start + fileMetaChunk
if end > len(rows) {
end = len(rows)
}
batch := rows[start:end]
args := make([]any, 0, len(batch)*6)
stmt := make([]byte, 0, 96+len(batch)*24)
stmt = append(stmt, "INSERT OR REPLACE INTO files (repo_prefix, file_path, content_hash, size, node_count, errors) VALUES "...)
for i, r := range batch {
if i > 0 {
stmt = append(stmt, ',')
}
stmt = append(stmt, "(?, ?, ?, ?, ?, ?)"...)
args = append(args, repoPrefix, r.FilePath, r.ContentHash, r.Size, r.NodeCount, r.Errors)
}
if _, err := tx.Exec(string(stmt), args...); err != nil {
return err
}
}
return tx.Commit()
}
// DeleteFileMetasByFiles drops the metadata rows for the supplied files in one
// repo prefix, chunked into `file_path IN (…)` DELETEs. Empty input is a no-op.
func (s *Store) DeleteFileMetasByFiles(repoPrefix string, files []string) error {
if len(files) == 0 {
return nil
}
s.writeMu.Lock()
defer s.writeMu.Unlock()
tx, err := s.db.Begin()
if err != nil {
return err
}
defer tx.Rollback() //nolint:errcheck // rollback after Commit is a no-op
for start := 0; start < len(files); start += fileMetaChunk {
end := start + fileMetaChunk
if end > len(files) {
end = len(files)
}
chunk := files[start:end]
args := make([]any, 0, len(chunk)+1)
args = append(args, repoPrefix)
stmt := make([]byte, 0, 64+len(chunk)*2)
stmt = append(stmt, "DELETE FROM files WHERE repo_prefix = ? AND file_path IN ("...)
for i, f := range chunk {
if i > 0 {
stmt = append(stmt, ',')
}
stmt = append(stmt, '?')
args = append(args, f)
}
stmt = append(stmt, ')')
if _, err := tx.Exec(string(stmt), args...); err != nil {
return err
}
}
return tx.Commit()
}
// FileMetasForRepo returns every recorded file row for the repo prefix.
// Always non-nil.
func (s *Store) FileMetasForRepo(repoPrefix string) ([]graph.FileMetaRow, error) {
rows, err := s.db.Query(
`SELECT file_path, content_hash, size, node_count, errors FROM files WHERE repo_prefix = ? ORDER BY file_path`,
repoPrefix,
)
if err != nil {
return nil, err
}
defer rows.Close()
out := []graph.FileMetaRow{}
for rows.Next() {
var r graph.FileMetaRow
if err := rows.Scan(&r.FilePath, &r.ContentHash, &r.Size, &r.NodeCount, &r.Errors); err != nil {
return nil, err
}
out = append(out, r)
}
return out, rows.Err()
}
@@ -0,0 +1,69 @@
package store_sqlite
import (
"path/filepath"
"testing"
"github.com/zzet/gortex/internal/graph"
)
// TestFileMetas_RoundTrip pins the per-file metadata sidecar: rows upsert,
// read back per repo, carry their errors JSON, and a per-file delete removes
// just the named file.
func TestFileMetas_RoundTrip(t *testing.T) {
s, err := Open(filepath.Join(t.TempDir(), "f.sqlite"))
if err != nil {
t.Fatal(err)
}
defer s.Close()
rows := []graph.FileMetaRow{
{FilePath: "a/x.go", ContentHash: "h1", Size: 100, NodeCount: 7, Errors: ""},
{FilePath: "a/broken.go", ContentHash: "h2", Size: 50, NodeCount: 1, Errors: `["3:5","4:1"]`},
}
if err := s.SetFileMetas("repoA", rows); err != nil {
t.Fatal(err)
}
// A different repo's row must not bleed in.
if err := s.SetFileMetas("repoB", []graph.FileMetaRow{{FilePath: "b/y.go", NodeCount: 2}}); err != nil {
t.Fatal(err)
}
got, err := s.FileMetasForRepo("repoA")
if err != nil {
t.Fatal(err)
}
if len(got) != 2 {
t.Fatalf("FileMetasForRepo(repoA) = %d rows, want 2", len(got))
}
byFile := map[string]graph.FileMetaRow{}
for _, r := range got {
byFile[r.FilePath] = r
}
if r := byFile["a/x.go"]; r.NodeCount != 7 || r.Size != 100 || r.ContentHash != "h1" || r.Errors != "" {
t.Errorf("x.go row = %+v", r)
}
if r := byFile["a/broken.go"]; r.NodeCount != 1 || r.Errors != `["3:5","4:1"]` {
t.Errorf("broken.go row = %+v", r)
}
// Upsert replaces in place.
if err := s.SetFileMetas("repoA", []graph.FileMetaRow{{FilePath: "a/x.go", NodeCount: 9, Errors: ""}}); err != nil {
t.Fatal(err)
}
got, _ = s.FileMetasForRepo("repoA")
for _, r := range got {
if r.FilePath == "a/x.go" && r.NodeCount != 9 {
t.Errorf("upsert did not replace node_count: %+v", r)
}
}
// Per-file delete removes only the named file.
if err := s.DeleteFileMetasByFiles("repoA", []string{"a/broken.go"}); err != nil {
t.Fatal(err)
}
got, _ = s.FileMetasForRepo("repoA")
if len(got) != 1 || got[0].FilePath != "a/x.go" {
t.Errorf("after delete, rows = %+v, want only a/x.go", got)
}
}
@@ -0,0 +1,38 @@
package store_sqlite
import (
"iter"
"github.com/zzet/gortex/internal/graph"
)
// Compile-time assertion: *Store serves the fn-value placeholder scan.
var _ graph.FnValuePlaceholderScanner = (*Store)(nil)
// FnValuePlaceholderEdges implements graph.FnValuePlaceholderScanner: it yields
// only the fn-value gate's placeholder edges, the exact inverse of the
// fn-value exclusion EdgesWithUnresolvedTarget applies. The predicate is the
// SAME two-form filter the v2 migration's dedupeFnValuePlaceholderEdges uses:
// the bare `unresolved::fnvalue::` range rides edges_by_to(to_id) (the ':;'
// range end is ':'+1, one past the marker); the multi-repo COPY-rewrite infix
// form is caught by is_unresolved = 1 + LIKE. Full column set incl. meta — the
// gate reads Meta["via"] and the captured fn_value_name off each placeholder.
//
// The whole point is that the gate no longer has to scan the entire
// EdgeReferences kind (placeholders + every real reference) and Go-filter on
// every whole-graph synthesizer pass; it pulls the handful of placeholders
// straight off the index instead.
func (s *Store) FnValuePlaceholderEdges() iter.Seq[*graph.Edge] {
return func(yield func(*graph.Edge) bool) {
out := s.queryEdgesSQL(`
SELECT from_id, to_id, kind, file_path, line, confidence, confidence_label, origin, tier, cross_repo, meta, resolve_terminal, resolve_terminal_reason
FROM edges
WHERE (to_id >= 'unresolved::fnvalue::' AND to_id < 'unresolved::fnvalue:;')
OR (is_unresolved = 1 AND to_id LIKE '%::unresolved::fnvalue::%')`)
for _, e := range out {
if !yield(e) {
return
}
}
}
}
+482
View File
@@ -0,0 +1,482 @@
package store_sqlite
import (
"database/sql"
"strings"
"github.com/zzet/gortex/internal/graph"
"github.com/zzet/gortex/internal/search"
)
// This file implements graph.SymbolSearcher + graph.SymbolBundleSearcher
// on the SQLite backend using the FTS5 virtual table declared in
// schema.go (symbol_fts). It is the on-disk replacement for the
// multi-GB in-heap Bleve/BM25 index: the FTS5 inverted index lives in
// the same .sqlite file as the graph, and a tier-0 exact-name boost
// short-circuits identifier queries so
// search quality holds or improves while the heap shrinks.
//
// Semantics:
//
// - BulkUpsertSymbolFTS wipes only the rows owned by repoPrefix
// before re-inserting, so sibling repos sharing one store don't
// clobber each other's corpus. Empty prefix wipes the whole table
// (single-repo / conformance behaviour).
//
// - SearchSymbols tier 0: an identifier query (no whitespace / path
// separators) that resolves to one or more nodes by exact name is
// returned directly with a fixed dominant score, skipping FTS.
// Misses fall through to the FTS5 MATCH path.
//
// - SearchSymbolBundles composes the same hit list with batched
// node + in/out edge fetches the rerank pipeline reads from.
//
// FTS5 maintains its index incrementally on every insert, so the
// Store struct needs no extra state and BuildSymbolIndex is a no-op
// (it only opportunistically merges segments).
// Compile-time assertions: *Store satisfies the symbol-search
// capabilities. The indexer auto-engages these when the active backend
// implements them, routing search_symbols through on-disk FTS5 instead
// of the in-process BM25 index.
var (
_ graph.SymbolSearcher = (*Store)(nil)
_ graph.SymbolBundleSearcher = (*Store)(nil)
_ graph.BundleFingerprintSink = (*Store)(nil)
)
// ftsInsertChunkRows bounds the rows per multi-row INSERT. Each row
// binds 3 host params (node_id, repo_prefix, tokens); 300 rows is 900
// params, comfortably under SQLite's default 999-variable limit so the
// statement stays portable across builds.
const ftsInsertChunkRows = 300
// UpsertSymbolFTS records (or replaces) the pre-tokenised text for
// nodeID. FTS5 offers no UPSERT on a table with UNINDEXED columns, so
// the write is delete-then-insert. The delete targets the prior row's
// FTS5 docid (rowid), looked up from the symbol_fts_rowid sidecar —
// node_id is UNINDEXED, so "DELETE … WHERE node_id = ?" would full-scan
// the whole index once per symbol, which is quadratic over a file's
// symbols on the per-edit reindex hot path. The repo_prefix is derived
// from the owning node (nodes.repo_prefix) so the per-repo staleness
// wipe in BulkUpsertSymbolFTS can scope by prefix; if the node is absent
// the prefix defaults to "".
func (s *Store) UpsertSymbolFTS(nodeID, tokens string) error {
if nodeID == "" {
return nil
}
s.writeMu.Lock()
defer s.writeMu.Unlock()
var repoPrefix string
// A missing node (or a scan error) leaves repoPrefix == "" — the
// row is still indexable, it just won't be reachable by a per-repo
// prefix wipe. The graph.Store contract has no error channel for
// the indexer's incremental writes, so we don't surface this.
_ = s.db.QueryRow(`SELECT repo_prefix FROM nodes WHERE id = ?`, nodeID).Scan(&repoPrefix)
// Delete the prior row by its docid (O(log n)) instead of by node_id
// (full FTS scan). A missing map entry means no prior row to drop —
// the sidecar is kept in lockstep with symbol_fts by every writer and
// backfilled at Open for databases built before it existed, so a miss
// here is a genuinely new symbol, not a stale row we're leaking.
var oldRowid int64
switch err := s.db.QueryRow(
`SELECT fts_rowid FROM symbol_fts_rowid WHERE node_id = ?`, nodeID,
).Scan(&oldRowid); err {
case nil:
if _, err := s.db.Exec(`DELETE FROM symbol_fts WHERE rowid = ?`, oldRowid); err != nil {
return err
}
case sql.ErrNoRows:
// new symbol — nothing to delete
default:
return err
}
res, err := s.db.Exec(
`INSERT INTO symbol_fts (node_id, repo_prefix, tokens) VALUES (?, ?, ?)`,
nodeID, repoPrefix, tokens,
)
if err != nil {
return err
}
newRowid, err := res.LastInsertId()
if err != nil {
return err
}
if _, err := s.db.Exec(
`INSERT OR REPLACE INTO symbol_fts_rowid (node_id, repo_prefix, fts_rowid) VALUES (?, ?, ?)`,
nodeID, repoPrefix, newRowid,
); err != nil {
return err
}
return nil
}
// BulkUpsertSymbolFTS is the cold-start fast path: wipe this repo's
// stale rows, then chunked multi-row INSERT of the deduped items. The
// whole thing runs in one transaction under writeMu so a concurrent
// reader never observes the table mid-wipe.
//
// repoPrefix scopes the pre-insert wipe: a non-empty prefix deletes
// only rows owned by that repo,
// leaving siblings untouched; an empty prefix wipes the whole table
// (single-repo / conformance behaviour — the conformance suite calls
// this with ""). Items are deduped by NodeID with last-write-wins,
// matching UpsertSymbolFTS's replace semantics.
func (s *Store) BulkUpsertSymbolFTS(repoPrefix string, items []graph.SymbolFTSItem) error {
if len(items) == 0 {
return nil
}
s.writeMu.Lock()
defer s.writeMu.Unlock()
// Dedup by ID — last write wins, mirroring UpsertSymbolFTS's
// delete-then-insert. Guards the edge case where a re-parse of a
// file emitted the same ID twice.
pos := make(map[string]int, len(items))
deduped := items[:0]
for _, it := range items {
if it.NodeID == "" {
continue
}
if p, ok := pos[it.NodeID]; ok {
deduped[p] = it
} else {
pos[it.NodeID] = len(deduped)
deduped = append(deduped, it)
}
}
items = deduped
if len(items) == 0 {
return nil
}
tx, err := s.db.Begin()
if err != nil {
return err
}
commit := false
defer func() {
if !commit {
_ = tx.Rollback()
}
}()
// Wipe this repo's prior rows so a clean rebuild of repo A doesn't
// leave phantom hits, while sibling repo B's corpus survives. The
// repo_prefix column is UNINDEXED but still stored, so the equality
// filter is a literal compare over the row set. Empty repoPrefix
// clears the whole table — the legacy single-repo wipe.
if _, err := tx.Exec(`DELETE FROM symbol_fts WHERE repo_prefix = ?`, repoPrefix); err != nil {
return err
}
// Drop this repo's rowid-map entries in lockstep with the symbol_fts
// wipe so the two never diverge; they are rebuilt from the freshly
// inserted rows below.
if _, err := tx.Exec(`DELETE FROM symbol_fts_rowid WHERE repo_prefix = ?`, repoPrefix); err != nil {
return err
}
for start := 0; start < len(items); start += ftsInsertChunkRows {
end := minInt(start+ftsInsertChunkRows, len(items))
chunk := items[start:end]
var b strings.Builder
b.WriteString(`INSERT INTO symbol_fts (node_id, repo_prefix, tokens) VALUES `)
args := make([]any, 0, len(chunk)*3)
for i, it := range chunk {
if i > 0 {
b.WriteByte(',')
}
b.WriteString(`(?,?,?)`)
args = append(args, it.NodeID, repoPrefix, it.Tokens)
}
if _, err := tx.Exec(b.String(), args...); err != nil {
return err
}
}
// Rebuild the rowid map for this repo from the rows just inserted. A
// full multi-row INSERT only exposes the last docid, so we read the
// docids back in one pass (a linear filter over the UNINDEXED
// repo_prefix column — the cold/bulk path, not the per-edit hot path).
if _, err := tx.Exec(
`INSERT OR REPLACE INTO symbol_fts_rowid (node_id, repo_prefix, fts_rowid)
SELECT node_id, repo_prefix, rowid FROM symbol_fts WHERE repo_prefix = ?`,
repoPrefix,
); err != nil {
return err
}
if err := tx.Commit(); err != nil {
return err
}
commit = true
return nil
}
// backfillSymbolFTSRowidMap populates symbol_fts_rowid from symbol_fts for
// a database built before the sidecar existed. Without it, the first
// incremental UpsertSymbolFTS for an already-indexed symbol would find no
// map entry, skip the delete, and leak a duplicate FTS row. It is a
// one-time cost: skipped once the map has any row (steady state) or when
// the FTS index is empty (a fresh DB the bulk path will populate with the
// map maintained inline). Runs at Open, before any reader or writer.
func backfillSymbolFTSRowidMap(db *sql.DB) error {
var mapped bool
if err := db.QueryRow(`SELECT EXISTS(SELECT 1 FROM symbol_fts_rowid)`).Scan(&mapped); err != nil {
return err
}
if mapped {
return nil
}
var hasFTS bool
if err := db.QueryRow(`SELECT EXISTS(SELECT 1 FROM symbol_fts)`).Scan(&hasFTS); err != nil {
return err
}
if !hasFTS {
return nil
}
_, err := db.Exec(
`INSERT OR REPLACE INTO symbol_fts_rowid (node_id, repo_prefix, fts_rowid)
SELECT node_id, repo_prefix, rowid FROM symbol_fts`)
return err
}
// BuildSymbolIndex is a no-op for FTS5: the index is maintained
// incrementally on every insert, so there is nothing to build after the
// bulk parse phase. We opportunistically run the FTS5 'optimize'
// command to merge segments (purely a read-latency improvement); any
// error is ignored because the index is already correct without it.
// Idempotent — safe to call any number of times.
func (s *Store) BuildSymbolIndex() error {
s.writeMu.Lock()
defer s.writeMu.Unlock()
_, _ = s.db.Exec(`INSERT INTO symbol_fts(symbol_fts) VALUES('optimize')`)
return nil
}
// SearchSymbols runs a symbol query and returns hits ordered by
// descending relevance (higher Score = more relevant).
//
// Tier 0 (exact-name boost): when the
// query looks like a literal identifier and resolves to one or more
// nodes by exact name, return those directly with a fixed dominant
// score (100.0) — an O(1)-ish index seek that beats FTS ranking for
// the common "type the symbol name" case. Misses fall through to FTS5.
//
// Otherwise tokenise on the read side with the SAME splitter as the
// write side (search.Tokenize) so a camelCase query lands on the
// split corpus, build a prefix-OR MATCH expression, and rank by BM25.
// SQLite's bm25() returns lower-is-better, so the stored Score is its
// negation (higher-is-better, matching the SymbolHit contract).
func (s *Store) SearchSymbols(query string, limit int) ([]graph.SymbolHit, error) {
if query == "" {
return nil, nil
}
if limit <= 0 {
limit = 20
}
// Tier 0: exact-name lookup. Only engage for identifier-shaped
// queries (no whitespace / path separators); multi-word queries are
// concept searches that need BM25 ranking. We only short-circuit
// when the lookup hits at least one node — misses fall through so a
// partial-identifier query still reaches FTS.
if isIdentifierQuery(query) {
ns := s.FindNodesByName(query)
if len(ns) > 0 {
out := make([]graph.SymbolHit, 0, minInt(len(ns), limit))
for _, n := range ns {
if n == nil || n.ID == "" {
continue
}
out = append(out, graph.SymbolHit{NodeID: n.ID, Score: 100.0})
if len(out) >= limit {
break
}
}
if len(out) > 0 {
return out, nil
}
}
}
match := s.buildFTSMatch(query)
if match == "" {
return nil, nil
}
const q = `SELECT node_id, bm25(symbol_fts) FROM symbol_fts WHERE symbol_fts MATCH ? ORDER BY bm25(symbol_fts) LIMIT ?`
rows, err := s.db.Query(q, match, limit)
if err != nil {
return nil, err
}
defer rows.Close()
var hits []graph.SymbolHit
for rows.Next() {
var (
id string
score float64
)
if err := rows.Scan(&id, &score); err != nil {
return nil, err
}
if id == "" {
continue
}
// bm25() is negative-better in SQLite; negate so higher = better,
// matching the SymbolHit contract. Rows already arrive in bm25
// (best-first) order from the ORDER BY.
hits = append(hits, graph.SymbolHit{NodeID: id, Score: -score})
}
if err := rows.Err(); err != nil {
return nil, err
}
return hits, nil
}
// buildFTSMatch tokenises the query with the write-side splitter and
// builds an FTS5 MATCH expression: each token becomes a quoted prefix
// term ("tok"*) and the terms are OR-joined so any token match counts.
// Returns "" when the query degenerates to no tokens.
func (s *Store) buildFTSMatch(query string) string {
tokens := search.Tokenize(query)
if len(tokens) == 0 {
// Fallback: when Tokenize drops everything (e.g. a single
// sub-2-char token like "go"), use the looser query tokeniser so
// the search still reaches the engine instead of returning empty.
tokens = search.TokenizeQuery(query)
if len(tokens) == 0 {
return ""
}
}
parts := make([]string, 0, len(tokens))
for _, t := range tokens {
if t == "" {
continue
}
parts = append(parts, `"`+escapeFTSQuote(t)+`"*`)
}
if len(parts) == 0 {
return ""
}
return strings.Join(parts, " OR ")
}
// escapeFTSQuote escapes a token for use inside an FTS5 double-quoted
// string literal: a literal double quote is doubled ("" inside "...").
func escapeFTSQuote(t string) string {
return strings.ReplaceAll(t, `"`, `""`)
}
// SearchSymbolBundles is the rerank-shaped fast path: it runs
// SearchSymbols to get the ranked id list (preserving order) plus a
// score-by-id map, then materialises the nodes and their in/out edges
// in batched fetches the rerank pipeline reads from. The engine routes
// through this when the backend implements SymbolBundleSearcher,
// pre-seeding rerank.Context's edge caches.
func (s *Store) SearchSymbolBundles(query string, limit int) ([]graph.SymbolBundle, error) {
hits, err := s.SearchSymbols(query, limit)
if err != nil {
return nil, err
}
if len(hits) == 0 {
return nil, nil
}
ids := make([]string, 0, len(hits))
scoreByID := make(map[string]float64, len(hits))
for _, h := range hits {
if h.NodeID == "" {
continue
}
if _, dup := scoreByID[h.NodeID]; dup {
// First hit keeps the score / position; defend against a
// future ranker that returns an id more than once.
continue
}
scoreByID[h.NodeID] = h.Score
ids = append(ids, h.NodeID)
}
if len(ids) == 0 {
return nil, nil
}
// Content-addressed cache: serve cached bundles for IDs whose
// package fingerprint is unchanged and fetch only the misses. The
// cache is nil until the daemon wires fingerprints, in which case
// every ID is a miss and the path is exactly the legacy fetch.
cached := make(map[string]graph.SymbolBundle, len(ids))
missIDs := ids
if s.bundles != nil {
missIDs = missIDs[:0:0]
for _, id := range ids {
if b, ok := s.bundles.lookup(id); ok {
cached[id] = b
continue
}
missIDs = append(missIDs, id)
}
}
// Fetch the misses' nodes + in/out edges in one batched round-trip
// each. A full cache hit skips all three fetches entirely.
var nodes map[string]*graph.Node
var out, in map[string][]*graph.Edge
if len(missIDs) > 0 {
nodes = s.GetNodesByIDs(missIDs)
out = s.GetOutEdgesByNodeIDs(missIDs)
in = s.GetInEdgesByNodeIDs(missIDs)
}
bundles := make([]graph.SymbolBundle, 0, len(ids))
for _, id := range ids {
if b, ok := cached[id]; ok {
// The cached bundle's score is whatever it was first cached
// with; the live FTS score for THIS query is authoritative,
// so re-stamp it (the score is query-specific, the node +
// edges are not).
b.Score = scoreByID[id]
bundles = append(bundles, b)
continue
}
n := nodes[id]
if n == nil {
// Hit references a node evicted between the search and the
// node fetch — skip; the caller does its own dedup / filter.
continue
}
b := graph.SymbolBundle{
Node: n,
Score: scoreByID[id],
OutEdges: out[id],
InEdges: in[id],
}
if s.bundles != nil {
s.bundles.store(b)
}
bundles = append(bundles, b)
}
return bundles, nil
}
// isIdentifierQuery reports whether a query looks like a literal symbol
// name (no whitespace, no path separators, no dots, no colons, no
// commas). The tier-0 exact-name fast path engages only on such
// queries; multi-token / path / qualified queries always go to FTS.
func isIdentifierQuery(q string) bool {
if q == "" {
return false
}
for _, r := range q {
switch r {
case ' ', '\t', '\n', '/', '.', ':', ',':
return false
}
}
return true
}
@@ -0,0 +1,45 @@
package store_sqlite
import (
"database/sql"
"github.com/zzet/gortex/internal/graph"
)
// SetRepoIndexState upserts the freshness-provenance row for one repo —
// written at the end of every (re)index. One row per repo_prefix.
func (s *Store) SetRepoIndexState(st graph.RepoIndexState) error {
s.writeMu.Lock()
defer s.writeMu.Unlock()
dirty := 0
if st.Dirty {
dirty = 1
}
_, err := s.db.Exec(`
INSERT OR REPLACE INTO repo_index_state
(repo_prefix, indexed_sha, dirty, indexed_at, workspace_fp, node_count, edge_count, extractor_versions)
VALUES (?, ?, ?, ?, ?, ?, ?, ?)`,
st.RepoPrefix, st.IndexedSHA, dirty, st.IndexedAt, st.WorkspaceFP,
st.NodeCount, st.EdgeCount, st.ExtractorVersions)
return err
}
// GetRepoIndexState returns the recorded freshness provenance for a repo.
// The bool is false when no row exists yet (never-indexed / pre-feature).
func (s *Store) GetRepoIndexState(repoPrefix string) (graph.RepoIndexState, bool, error) {
row := s.db.QueryRow(`
SELECT indexed_sha, dirty, indexed_at, workspace_fp, node_count, edge_count, extractor_versions
FROM repo_index_state WHERE repo_prefix = ?`, repoPrefix)
st := graph.RepoIndexState{RepoPrefix: repoPrefix}
var dirty int
err := row.Scan(&st.IndexedSHA, &dirty, &st.IndexedAt, &st.WorkspaceFP,
&st.NodeCount, &st.EdgeCount, &st.ExtractorVersions)
if err == sql.ErrNoRows {
return graph.RepoIndexState{RepoPrefix: repoPrefix}, false, nil
}
if err != nil {
return graph.RepoIndexState{RepoPrefix: repoPrefix}, false, err
}
st.Dirty = dirty != 0
return st, true, nil
}
@@ -0,0 +1,61 @@
package store_sqlite_test
import (
"path/filepath"
"testing"
"github.com/stretchr/testify/require"
"github.com/zzet/gortex/internal/graph"
"github.com/zzet/gortex/internal/graph/store_sqlite"
)
func openIndexStateStore(t *testing.T) *store_sqlite.Store {
t.Helper()
s, err := store_sqlite.Open(filepath.Join(t.TempDir(), "is.sqlite"))
require.NoError(t, err)
t.Cleanup(func() { _ = s.Close() })
return s
}
func TestRepoIndexState_RoundTrip(t *testing.T) {
s := openIndexStateStore(t)
// Absent state reads back as (zero, false, nil).
got, ok, err := s.GetRepoIndexState("gortex")
require.NoError(t, err)
require.False(t, ok)
require.Equal(t, "gortex", got.RepoPrefix)
want := graph.RepoIndexState{
RepoPrefix: "gortex",
IndexedSHA: "abc123",
Dirty: true,
IndexedAt: 1700000000,
WorkspaceFP: "deadbeef",
NodeCount: 42,
EdgeCount: 99,
ExtractorVersions: `{"go":2}`,
}
require.NoError(t, s.SetRepoIndexState(want))
got, ok, err = s.GetRepoIndexState("gortex")
require.NoError(t, err)
require.True(t, ok)
require.Equal(t, want, got)
// Upsert replaces in place (one row per repo_prefix).
want.IndexedSHA = "def456"
want.Dirty = false
require.NoError(t, s.SetRepoIndexState(want))
got, ok, err = s.GetRepoIndexState("gortex")
require.NoError(t, err)
require.True(t, ok)
require.Equal(t, "def456", got.IndexedSHA)
require.False(t, got.Dirty)
// A different repo is isolated.
_, ok, err = s.GetRepoIndexState("other")
require.NoError(t, err)
require.False(t, ok)
}
@@ -0,0 +1,54 @@
package store_sqlite
import "github.com/zzet/gortex/internal/graph"
// Compile-time assertion: *Store serves the meta-less kind-scoped edge scan.
var _ graph.LightEdgeScanner = (*Store)(nil)
// edgeColsLight is the meta-less edge column projection: the promoted struct
// columns WITHOUT the meta blob (and without resolve_terminal, which lives in
// Meta). It is exactly the ten columns scanEdgeLight scans, and is shared with
// the stmtOutEdgesLight prepared statement so the projection can never drift
// from the scanner. Adding meta back here would defeat the whole point — the
// per-row JSON decode this projection exists to skip.
const edgeColsLight = `from_id, to_id, kind, file_path, line, confidence, confidence_label, origin, tier, cross_repo`
// AllEdgesLight implements graph.LightEdgeScanner: a kind-scoped edge scan that
// never decodes the meta blob. An empty kinds list scans every edge; supplying
// only empty-string kinds matches nothing (parity with the aggregators). The
// edges_by_kind index serves the IN filter. Meta is left nil; only the promoted
// fields (origin/tier/confidence/confidence_label/cross_repo/line) are hydrated.
func (s *Store) AllEdgesLight(kinds ...graph.EdgeKind) []*graph.Edge {
_, args := aggDedupeEdgeKinds(kinds)
if len(args) == 0 {
if len(kinds) > 0 {
return nil // caller passed only empty kinds — nothing matches
}
return s.queryEdgesLightSQL(`SELECT ` + edgeColsLight + ` FROM edges ORDER BY id`)
}
q := `SELECT ` + edgeColsLight + ` FROM edges WHERE kind IN (` +
inPlaceholders(len(args)) + `) ORDER BY id`
return s.queryEdgesLightSQL(q, args...)
}
// queryEdgesLightSQL is the meta-less sibling of queryEdgesSQL: it materialises
// the rows into a slice and closes the cursor before returning (releasing the
// single pooled connection), but scans through scanEdgeLight so the meta column
// is never transferred or decoded. Returns nil on any query error, matching
// queryEdgesSQL — a teardown-race read degrades to empty rather than panicking.
func (s *Store) queryEdgesLightSQL(q string, args ...any) []*graph.Edge {
rows, err := s.db.Query(q, args...)
if err != nil {
return nil
}
defer rows.Close()
var out []*graph.Edge
for rows.Next() {
e, err := scanEdgeLight(rows)
if err != nil || e == nil {
continue
}
out = append(out, e)
}
return out
}
@@ -0,0 +1,54 @@
package store_sqlite_test
import (
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/zzet/gortex/internal/graph"
)
// TestAllEdgesLightIsMetaless pins the disk backend's graph.LightEdgeScanner:
// the scan must skip the meta blob (Meta == nil) — the per-edge JSON decode the
// warm-restart analysis passes exist to avoid — while every promoted field
// still equals what the full AllEdges() scan returns.
func TestAllEdgesLightIsMetaless(t *testing.T) {
s := openTestStore(t)
s.AddNode(&graph.Node{ID: "p/a.go::A", Kind: graph.KindFunction, Name: "A", FilePath: "p/a.go"})
s.AddNode(&graph.Node{ID: "p/a.go::B", Kind: graph.KindFunction, Name: "B", FilePath: "p/a.go"})
s.AddEdge(&graph.Edge{
From: "p/a.go::A", To: "p/a.go::B", Kind: graph.EdgeCalls,
FilePath: "p/a.go", Line: 11, Confidence: 0.75, ConfidenceLabel: "high",
Origin: graph.OriginLSPResolved, Tier: "lsp", CrossRepo: true,
Meta: map[string]any{"via": "direct", "blob_only": "x"},
})
// A second kind to prove the IN filter really scopes.
s.AddEdge(&graph.Edge{From: "p/a.go::A", To: "p/a.go::B", Kind: graph.EdgeImports, FilePath: "p/a.go", Line: 1})
light := s.AllEdgesLight(graph.EdgeCalls)
require.Len(t, light, 1, "kind filter must exclude the imports edge")
e := light[0]
assert.Nil(t, e.Meta, "AllEdgesLight must not decode the meta blob")
var full *graph.Edge
for _, fe := range s.AllEdges() {
if fe.Kind == graph.EdgeCalls {
full = fe
}
}
require.NotNil(t, full)
assert.NotNil(t, full.Meta, "sanity: the full scan DOES decode meta")
assert.Equal(t, full.From, e.From)
assert.Equal(t, full.To, e.To)
assert.Equal(t, full.Kind, e.Kind)
assert.Equal(t, full.Line, e.Line)
assert.Equal(t, full.Confidence, e.Confidence)
assert.Equal(t, full.ConfidenceLabel, e.ConfidenceLabel)
assert.Equal(t, full.Origin, e.Origin)
assert.Equal(t, full.Tier, e.Tier)
assert.Equal(t, full.CrossRepo, e.CrossRepo)
// Empty kinds means every edge.
assert.Len(t, s.AllEdgesLight(), 2)
}
@@ -0,0 +1,204 @@
package store_sqlite
import (
"strings"
"github.com/zzet/gortex/internal/graph"
)
// These methods were added to graph.Store after the sqlite backend was
// first removed; they are restored here so *Store satisfies the current
// interface. All reuse the chunked IN-list / raw-SQL helpers in store.go
// (queryNodesSQL / queryEdgesSQL / lookupChunkSize / minInt). SQLite's
// planner drives every one through the existing secondary indexes.
// lookupNodeCols is the canonical node column list (and scan order) for
// every node-shaped SELECT in the package. It must stay in sync with
// scanNode. The struct columns (start_column/end_column) sit with the line
// range; the promoted meta columns (signature/visibility/doc/external/
// return_type/is_async/is_static/is_abstract/is_exported/updated_at/
// data_class/semantic_type/semantic_source) precede meta.
const lookupNodeCols = `id, kind, name, qual_name, file_path, start_line, end_line, start_column, end_column, language, repo_prefix, workspace_id, project_id, signature, visibility, doc, external, return_type, is_async, is_static, is_abstract, is_exported, updated_at, data_class, semantic_type, semantic_source, meta`
// lookupNodeColsLight is lookupNodeCols without the trailing meta column —
// the projection GetRepoNodesLight uses so a repo-scoped scan never
// transfers or decodes a single blob. Derived, not hand-duplicated, so it
// can never drift out of sync with lookupNodeCols / scanNode.
var lookupNodeColsLight = strings.TrimSuffix(lookupNodeCols, ", meta")
const lookupEdgeCols = `from_id, to_id, kind, file_path, line, confidence, confidence_label, origin, tier, cross_repo, meta, resolve_terminal, resolve_terminal_reason`
// Compile-time assertion: *Store satisfies graph.NodeNameClassCounter.
var _ graph.NodeNameClassCounter = (*Store)(nil)
// CountNodesByNameClass implements graph.NodeNameClassCounter: for each
// distinct name, it tallies how many nodes.name matches are Real (is_stub =
// 0 and kind IN definitionKinds) vs Stub (is_stub = 1), server-side via
// nodes_by_name — one aggregate query per chunk instead of one
// FindNodesByName round trip per name. A name absent from the returned map
// has no matching node at all (Real == Stub == 0 either way).
func (s *Store) CountNodesByNameClass(names []string, definitionKinds []graph.NodeKind) map[string]graph.NodeNameClassCount {
_, kindArgs := aggDedupeNodeKinds(definitionKinds)
if len(kindArgs) == 0 {
return nil
}
uniq := dedupeNonEmpty(names)
if len(uniq) == 0 {
return nil
}
out := make(map[string]graph.NodeNameClassCount, len(uniq))
kindPlaceholders := inPlaceholders(len(kindArgs))
for i := 0; i < len(uniq); i += lookupChunkSize {
end := minInt(i+lookupChunkSize, len(uniq))
chunk := uniq[i:end]
q := `SELECT name,
SUM(CASE WHEN is_stub = 0 AND kind IN (` + kindPlaceholders + `) THEN 1 ELSE 0 END),
SUM(CASE WHEN is_stub = 1 THEN 1 ELSE 0 END)
FROM nodes
WHERE name IN (` + inPlaceholders(len(chunk)) + `)
GROUP BY name`
args := make([]any, 0, len(kindArgs)+len(chunk))
args = append(args, kindArgs...)
args = append(args, toAnyArgs(chunk)...)
rows, err := s.db.Query(q, args...)
if err != nil {
panicOnFatal(err)
return out
}
for rows.Next() {
var name string
var c graph.NodeNameClassCount
if err := rows.Scan(&name, &c.Real, &c.Stub); err != nil {
_ = rows.Close()
panicOnFatal(err)
return out
}
out[name] = c
}
if err := rows.Err(); err != nil {
_ = rows.Close()
panicOnFatal(err)
return out
}
_ = rows.Close()
}
return out
}
// FindNodesByNameContaining returns nodes whose Name contains substr,
// case-insensitively (SQLite's LIKE is ASCII case-insensitive). An empty
// substring matches nothing (parity with the in-memory store); a limit > 0
// caps the result set. The leading-wildcard LIKE is a deliberate full scan —
// no index accelerates an unanchored substring — matching the in-memory
// strings.Contains fallback. % and _ in substr are escaped so they match
// literally.
func (s *Store) FindNodesByNameContaining(substr string, limit int) []*graph.Node {
if substr == "" {
return nil
}
pattern := "%" + escapeLikePattern(substr) + "%"
q := `SELECT ` + lookupNodeCols + ` FROM nodes WHERE name LIKE ? ESCAPE '\' ORDER BY id`
if limit > 0 {
return s.queryNodesSQL(q+` LIMIT ?`, pattern, limit)
}
return s.queryNodesSQL(q, pattern)
}
// GetNodesByQualNames returns a map qualName→*Node (first match per
// qual_name) for the batch — the qual-name twin of FindNodesByNames, used to
// pre-warm import resolution. Driven by the unique nodes_by_qual index.
func (s *Store) GetNodesByQualNames(qualNames []string) map[string]*graph.Node {
uniq := dedupeNonEmpty(qualNames)
if len(uniq) == 0 {
return nil
}
out := make(map[string]*graph.Node, len(uniq))
for i := 0; i < len(uniq); i += lookupChunkSize {
end := minInt(i+lookupChunkSize, len(uniq))
chunk := uniq[i:end]
q := `SELECT ` + lookupNodeCols + ` FROM nodes WHERE qual_name IN (` + inPlaceholders(len(chunk)) + `)`
for _, n := range s.queryNodesSQL(q, toAnyArgs(chunk)...) {
if n == nil {
continue
}
if _, ok := out[n.QualName]; !ok {
out[n.QualName] = n
}
}
}
return out
}
// GetOutEdgesByNodeIDs batches per-node out-edge fan-out into one query per
// chunk. Missing IDs are simply absent from the returned map.
func (s *Store) GetOutEdgesByNodeIDs(ids []string) map[string][]*graph.Edge {
return s.edgesByNodeIDs(ids, "from_id", func(e *graph.Edge) string { return e.From })
}
// GetInEdgesByNodeIDs is the incoming-edge twin of GetOutEdgesByNodeIDs.
func (s *Store) GetInEdgesByNodeIDs(ids []string) map[string][]*graph.Edge {
return s.edgesByNodeIDs(ids, "to_id", func(e *graph.Edge) string { return e.To })
}
// edgesByNodeIDs runs the chunked IN-list edge fetch keyed on the given
// column (from_id or to_id), grouping results by the supplied key extractor.
func (s *Store) edgesByNodeIDs(ids []string, col string, key func(*graph.Edge) string) map[string][]*graph.Edge {
uniq := dedupeNonEmpty(ids)
if len(uniq) == 0 {
return nil
}
out := make(map[string][]*graph.Edge, len(uniq))
for i := 0; i < len(uniq); i += lookupChunkSize {
end := minInt(i+lookupChunkSize, len(uniq))
chunk := uniq[i:end]
q := `SELECT ` + lookupEdgeCols + ` FROM edges WHERE ` + col + ` IN (` + inPlaceholders(len(chunk)) + `)`
for _, e := range s.queryEdgesSQL(q, toAnyArgs(chunk)...) {
if e == nil {
continue
}
k := key(e)
out[k] = append(out[k], e)
}
}
return out
}
// dedupeNonEmpty drops empties and duplicates, preserving first-seen order.
func dedupeNonEmpty(in []string) []string {
seen := make(map[string]struct{}, len(in))
out := make([]string, 0, len(in))
for _, v := range in {
if v == "" {
continue
}
if _, ok := seen[v]; ok {
continue
}
seen[v] = struct{}{}
out = append(out, v)
}
return out
}
// inPlaceholders returns "?,?,?" for n bound parameters.
func inPlaceholders(n int) string {
if n <= 0 {
return ""
}
return strings.Repeat(",?", n)[1:]
}
// toAnyArgs widens a string slice for variadic Query/Exec args.
func toAnyArgs(ss []string) []any {
args := make([]any, len(ss))
for i, v := range ss {
args[i] = v
}
return args
}
// escapeLikePattern escapes the LIKE metacharacters so the substring matches
// literally under `... LIKE ? ESCAPE '\'`.
func escapeLikePattern(s string) string {
return strings.NewReplacer(`\`, `\\`, `%`, `\%`, `_`, `\_`).Replace(s)
}
@@ -0,0 +1,54 @@
package store_sqlite_test
import (
"path/filepath"
"testing"
"github.com/stretchr/testify/require"
"github.com/zzet/gortex/internal/graph"
"github.com/zzet/gortex/internal/graph/store_sqlite"
)
// TestAllRepoMemoryEstimates_Memoized verifies the short-TTL memoisation:
// within the TTL a second call serves the cached estimate instead of
// re-running the COUNT … GROUP BY scan, so the status path can poll cheaply
// even while enrichment is writing to the same store.
func TestAllRepoMemoryEstimates_Memoized(t *testing.T) {
path := filepath.Join(t.TempDir(), "g.sqlite")
s, err := store_sqlite.Open(path)
require.NoError(t, err)
t.Cleanup(func() { _ = s.Close() })
totalNodes := func(m map[string]graph.RepoMemoryEstimate) int {
n := 0
for _, e := range m {
n += e.NodeCount
}
return n
}
s.AddBatch([]*graph.Node{
{ID: "a.go::Foo", Kind: graph.KindFunction, Name: "Foo", FilePath: "a.go", RepoPrefix: "r1"},
{ID: "a.go::Bar", Kind: graph.KindFunction, Name: "Bar", FilePath: "a.go", RepoPrefix: "r1"},
}, nil)
first := s.AllRepoMemoryEstimates()
require.Equal(t, 2, totalNodes(first))
// Add another node; within the memoisation TTL the estimate is served
// from cache, so the freshly-added node is intentionally not yet
// reflected — proof the COUNT scan was skipped.
s.AddBatch([]*graph.Node{
{ID: "b.go::Baz", Kind: graph.KindFunction, Name: "Baz", FilePath: "b.go", RepoPrefix: "r1"},
}, nil)
cached := s.AllRepoMemoryEstimates()
require.Equal(t, 2, totalNodes(cached), "within the TTL the result should be the memoised value, not a fresh COUNT")
// The returned map is a copy: mutating it must not corrupt the cache.
for k := range cached {
delete(cached, k)
}
again := s.AllRepoMemoryEstimates()
require.Equal(t, 2, totalNodes(again), "the cache must hand back an independent copy")
}
+243
View File
@@ -0,0 +1,243 @@
package store_sqlite
import (
"database/sql"
"github.com/zzet/gortex/internal/graph"
)
// Compile-time assertions that the SQLite Store satisfies the optional
// per-file mtime persistence capabilities. Lifting this state into the
// same backend the graph lives in means warm restarts read it through
// one persistence surface instead of a second gob snapshot.
var (
_ graph.FileMtimeWriter = (*Store)(nil)
_ graph.FileMtimeReader = (*Store)(nil)
_ graph.FileMtimeReplacer = (*Store)(nil)
_ graph.FileMtimeDeleter = (*Store)(nil)
)
// mtimeChunk bounds how many (repo_prefix, file_path, mtime_ns) tuples
// ride in a single multi-row INSERT. SQLite's default compiled-in host
// parameter limit is 999; at 3 params per row that caps a statement at
// 333 rows, so 300 leaves headroom.
const mtimeChunk = 300
// SetFileMtime records one file's modification time (nanoseconds since
// the epoch) for a repo prefix, replacing any prior value. It is a
// convenience single-row form of BulkSetFileMtimes.
func (s *Store) SetFileMtime(repoPrefix, filePath string, mtimeNs int64) error {
s.writeMu.Lock()
defer s.writeMu.Unlock()
_, err := s.db.Exec(
`INSERT OR REPLACE INTO file_mtimes (repo_prefix, file_path, mtime_ns) VALUES (?, ?, ?)`,
repoPrefix, filePath, mtimeNs,
)
return err
}
// BulkSetFileMtimes persists every (filePath -> mtimeNs) entry for one
// repo prefix in a single transaction, chunked so no statement exceeds
// SQLite's host-parameter limit. Idempotent on (repoPrefix, filePath):
// re-running with overlapping keys replaces in place. Empty input is a
// no-op.
func (s *Store) BulkSetFileMtimes(repoPrefix string, mtimes map[string]int64) error {
if len(mtimes) == 0 {
return nil
}
s.writeMu.Lock()
defer s.writeMu.Unlock()
tx, err := s.db.Begin()
if err != nil {
return err
}
defer tx.Rollback() //nolint:errcheck // rollback after Commit is a no-op
if err := insertMtimesTx(tx, repoPrefix, mtimes); err != nil {
return err
}
return tx.Commit()
}
// ReplaceFileMtimes persists the AUTHORITATIVE full mtime set for one repo
// prefix: every prior row for the prefix is dropped and the supplied set is
// written, all in one transaction. The full-index persist path uses this so
// files deleted since the last index are pruned — BulkSetFileMtimes (upsert)
// would leave their rows behind, and warm-restart reconcile would then
// detect them as phantom deletions on every restart, forcing a full
// re-track that never converges.
//
// Empty input is a deliberate no-op: it never wipes a repo's mtimes from an
// empty snapshot (the indexer guards the call with len(snapshot) > 0).
func (s *Store) ReplaceFileMtimes(repoPrefix string, mtimes map[string]int64) error {
if len(mtimes) == 0 {
return nil
}
s.writeMu.Lock()
defer s.writeMu.Unlock()
tx, err := s.db.Begin()
if err != nil {
return err
}
defer tx.Rollback() //nolint:errcheck // rollback after Commit is a no-op
if _, err := tx.Exec(`DELETE FROM file_mtimes WHERE repo_prefix = ?`, repoPrefix); err != nil {
return err
}
if err := insertMtimesTx(tx, repoPrefix, mtimes); err != nil {
return err
}
return tx.Commit()
}
// DeleteFileMtimes drops the rows for a set of repo-relative file paths
// under one repo prefix — the incremental-reindex sibling of
// ReplaceFileMtimes. The watcher / incremental path calls it when a file is
// deleted so the persisted set stays in step with the live graph and the
// next warm restart does not see the path as a phantom deletion. Empty
// input is a no-op.
func (s *Store) DeleteFileMtimes(repoPrefix string, paths []string) error {
if len(paths) == 0 {
return nil
}
s.writeMu.Lock()
defer s.writeMu.Unlock()
tx, err := s.db.Begin()
if err != nil {
return err
}
defer tx.Rollback() //nolint:errcheck // rollback after Commit is a no-op
// Chunk so the IN-list never exceeds SQLite's host-parameter limit:
// one leading repo_prefix arg + up to mtimeChunk path args per stmt.
for start := 0; start < len(paths); start += mtimeChunk {
end := min(start+mtimeChunk, len(paths))
batch := paths[start:end]
args := make([]any, 0, len(batch)+1)
args = append(args, repoPrefix)
stmt := make([]byte, 0, 64+len(batch)*2)
stmt = append(stmt, "DELETE FROM file_mtimes WHERE repo_prefix = ? AND file_path IN ("...)
for i := range batch {
if i > 0 {
stmt = append(stmt, ',')
}
stmt = append(stmt, '?')
args = append(args, batch[i])
}
stmt = append(stmt, ')')
if _, err := tx.Exec(string(stmt), args...); err != nil {
return err
}
}
return tx.Commit()
}
// insertMtimesTx writes every (path -> ns) entry for repoPrefix into the
// given transaction with chunked multi-row INSERT OR REPLACE statements,
// each kept under SQLite's host-parameter limit. The caller owns the tx
// lifecycle (Begin/Commit/Rollback) and the write lock.
func insertMtimesTx(tx *sql.Tx, repoPrefix string, mtimes map[string]int64) error {
// Stable ordering is not required for correctness, but iterating the
// map directly is fine — we only chunk by count.
type kv struct {
path string
ns int64
}
pending := make([]kv, 0, len(mtimes))
for p, ns := range mtimes {
pending = append(pending, kv{path: p, ns: ns})
}
for start := 0; start < len(pending); start += mtimeChunk {
end := min(start+mtimeChunk, len(pending))
batch := pending[start:end]
// Build a multi-row INSERT OR REPLACE: (?, ?, ?), (?, ?, ?), ...
args := make([]any, 0, len(batch)*3)
stmt := make([]byte, 0, 64+len(batch)*16)
stmt = append(stmt, "INSERT OR REPLACE INTO file_mtimes (repo_prefix, file_path, mtime_ns) VALUES "...)
for i, e := range batch {
if i > 0 {
stmt = append(stmt, ',')
}
stmt = append(stmt, "(?, ?, ?)"...)
args = append(args, repoPrefix, e.path, e.ns)
}
if _, err := tx.Exec(string(stmt), args...); err != nil {
return err
}
}
return nil
}
// LoadFileMtimes returns the recorded mtimes for one repo prefix as a
// fresh map. Returns nil when there is no data for the prefix (the
// "no recorded state" signal warmup expects).
func (s *Store) LoadFileMtimes(repoPrefix string) map[string]int64 {
rows, err := s.db.Query(
`SELECT file_path, mtime_ns FROM file_mtimes WHERE repo_prefix = ?`,
repoPrefix,
)
if err != nil {
return nil
}
defer rows.Close()
var out map[string]int64
for rows.Next() {
var path string
var ns int64
if err := rows.Scan(&path, &ns); err != nil {
return nil
}
if out == nil {
out = make(map[string]int64)
}
out[path] = ns
}
if err := rows.Err(); err != nil {
return nil
}
return out
}
// FileMtimes is a fallible read form of LoadFileMtimes. It always
// returns a non-nil (possibly empty) map for a known/unknown prefix and
// surfaces any query error. The interface method LoadFileMtimes is the
// daemon's entry point; this variant exists for callers (and tests)
// that want the error and an always-materialised map.
func (s *Store) FileMtimes(repoPrefix string) (map[string]int64, error) {
rows, err := s.db.Query(
`SELECT file_path, mtime_ns FROM file_mtimes WHERE repo_prefix = ?`,
repoPrefix,
)
if err != nil {
return nil, err
}
defer rows.Close()
out := make(map[string]int64)
for rows.Next() {
var path string
var ns int64
if err := rows.Scan(&path, &ns); err != nil {
return nil, err
}
out[path] = ns
}
if err := rows.Err(); err != nil {
return nil, err
}
return out, nil
}
@@ -0,0 +1,112 @@
package store_sqlite_test
import (
"reflect"
"testing"
"github.com/zzet/gortex/internal/graph"
)
// TestReplaceFileMtimesPrunesDeleted is the regression for the warm-restart
// "nothing changed but full re-track" bug: the full-index persist path must
// REPLACE a repo's mtime set, not union into it. An upsert-only persist
// leaves rows for files deleted since the last index behind, and warm-restart
// reconcile then detects them as phantom deletions on every restart — forcing
// a full re-track that never converges.
func TestReplaceFileMtimesPrunesDeleted(t *testing.T) {
s := openTestStore(t)
// Assert the store advertises the capability the indexer probes for.
var _ graph.FileMtimeReplacer = s
var _ graph.FileMtimeDeleter = s
// First index: three files persisted.
require := func(err error, what string) {
t.Helper()
if err != nil {
t.Fatalf("%s: %v", what, err)
}
}
require(s.BulkSetFileMtimes("repoA", map[string]int64{
"a/one.go": 100,
"a/two.go": 200,
"a/three.go": 300,
}), "seed BulkSetFileMtimes")
// A different repo whose rows must never be touched by repoA writes.
require(s.BulkSetFileMtimes("repoB", map[string]int64{"b/x.go": 999}), "seed repoB")
// Second index: two.go was deleted on disk, four.go is new, three.go
// changed. The authoritative snapshot is {one, three', four}.
require(s.ReplaceFileMtimes("repoA", map[string]int64{
"a/one.go": 100,
"a/three.go": 350, // changed
"a/four.go": 400, // new
}), "ReplaceFileMtimes")
want := map[string]int64{
"a/one.go": 100,
"a/three.go": 350,
"a/four.go": 400,
}
got := s.LoadFileMtimes("repoA")
if !reflect.DeepEqual(got, want) {
t.Fatalf("after ReplaceFileMtimes = %v, want %v (a/two.go must be pruned)", got, want)
}
if _, stillThere := got["a/two.go"]; stillThere {
t.Fatal("a/two.go was deleted on disk but its mtime row survived the replace — phantom deletion bug")
}
// Repo isolation.
if b := s.LoadFileMtimes("repoB"); !reflect.DeepEqual(b, map[string]int64{"b/x.go": 999}) {
t.Fatalf("repoB rows disturbed by repoA replace: %v", b)
}
// Empty input is a deliberate no-op: it must NEVER wipe a repo's set.
require(s.ReplaceFileMtimes("repoA", nil), "ReplaceFileMtimes(nil)")
if got := s.LoadFileMtimes("repoA"); !reflect.DeepEqual(got, want) {
t.Fatalf("ReplaceFileMtimes(nil) wiped the repo: %v", got)
}
}
// TestDeleteFileMtimes covers the incremental-reindex sibling: the watcher /
// incremental path drops just the deleted paths so the persisted set stays in
// step with the live graph without a full replace.
func TestDeleteFileMtimes(t *testing.T) {
s := openTestStore(t)
if err := s.BulkSetFileMtimes("repoA", map[string]int64{
"a/one.go": 100,
"a/two.go": 200,
"a/three.go": 300,
"a/four.go": 400,
}); err != nil {
t.Fatalf("seed: %v", err)
}
if err := s.BulkSetFileMtimes("repoB", map[string]int64{"b/keep.go": 7}); err != nil {
t.Fatalf("seed repoB: %v", err)
}
// Delete two existing paths and one that was never recorded (harmless).
if err := s.DeleteFileMtimes("repoA", []string{"a/two.go", "a/four.go", "a/never.go"}); err != nil {
t.Fatalf("DeleteFileMtimes: %v", err)
}
want := map[string]int64{"a/one.go": 100, "a/three.go": 300}
if got := s.LoadFileMtimes("repoA"); !reflect.DeepEqual(got, want) {
t.Fatalf("after delete = %v, want %v", got, want)
}
// Repo isolation: same-named delete on repoA must not touch repoB.
if b := s.LoadFileMtimes("repoB"); !reflect.DeepEqual(b, map[string]int64{"b/keep.go": 7}) {
t.Fatalf("repoB disturbed: %v", b)
}
// Empty input is a no-op.
if err := s.DeleteFileMtimes("repoA", nil); err != nil {
t.Fatalf("DeleteFileMtimes(nil): %v", err)
}
if got := s.LoadFileMtimes("repoA"); !reflect.DeepEqual(got, want) {
t.Fatalf("DeleteFileMtimes(nil) changed the set: %v", got)
}
}
+310
View File
@@ -0,0 +1,310 @@
package store_sqlite
import (
"database/sql"
"fmt"
"strings"
)
// This file adds the repo-scoped store hygiene the base eviction path lacks.
// EvictRepo (store.go) deletes ONLY nodes+edges, but a repo owns fifteen
// other repo_prefix-keyed sidecar tables (file_mtimes, repo_index_state,
// enrichment_state, clone_shingles, constant_values, files, ref_facts,
// vectors, churn/coverage/release/blame_enrichment, symbol_fts,
// symbol_fts_rowid, content_fts — see schema.go). Untracking a repo through
// EvictRepo leaks every one of them, so a long-lived store accumulates
// sidecar rows for repos removed from config long ago. PurgeRepo clears a
// repo whole; OrphanRepoPrefixes finds prefixes that outlived their config
// entry; RekeyRepoPrefix moves a lone repo's residue when it earns a prefix.
//
// INVARIANT — the empty repo_prefix is NEVER purged. In a live multi-repo
// store repo_prefix='' identifies SYNTHETIC GLOBAL EXTERNALS (external_call
// ::dep:* / builtin:: / module:: nodes shared across every repo) and, in a
// single-repo store, the sole repo's live data. Deleting '' rows would strip
// the shared externals out from under every repo, or wipe the lone repo.
// Every method here refuses or excludes ''.
// purgeSidecarTables are the repo_prefix-keyed sidecar tables PurgeRepo
// clears for a prefix, alongside nodes+edges. Each carries a repo_prefix
// column a plain `DELETE ... WHERE repo_prefix = ?` keys on. The two FTS5
// vtables (symbol_fts, content_fts) carry repo_prefix UNINDEXED, so their
// delete is a full scan — acceptable for a purge (a rare, whole-repo op),
// unlike the per-edit hot path. `vectors` is deliberately absent: it has NO
// repo_prefix column (keyed by node_id alone), so PurgeRepo deletes its rows
// by node-id membership instead (see deleteByIDColumnsTx below).
var purgeSidecarTables = []string{
"file_mtimes",
"repo_index_state",
"enrichment_state",
"clone_shingles",
"constant_values",
"files",
"ref_facts",
"churn_enrichment",
"coverage_enrichment",
"release_enrichment",
"blame_enrichment",
"symbol_fts",
"symbol_fts_rowid",
"content_fts",
}
// PurgeRepo deletes EVERY row a repo owns — nodes, edges, and all fifteen
// repo_prefix-keyed sidecar tables (purgeSidecarTables + vectors) — in one
// transaction. It is the complete form of EvictRepo (which drops only
// nodes+edges), wired into UntrackRepo so removing a repo from config leaves
// no residue. Refuses prefix=="" (shared global externals / solo-mode live
// data — see the file-level INVARIANT).
func (s *Store) PurgeRepo(prefix string) error {
if prefix == "" {
return fmt.Errorf("store_sqlite: PurgeRepo refuses empty repo prefix (would delete shared global externals / solo-repo data)")
}
s.writeMu.Lock()
defer s.writeMu.Unlock()
tx, err := s.db.Begin()
if err != nil {
return err
}
defer tx.Rollback() //nolint:errcheck // rollback after Commit is a no-op
// Collect this repo's node IDs first: edges and vectors are keyed off
// them (edges by from_id/to_id, vectors by node_id — neither carries a
// repo_prefix column). Edge deletion semantics mirror evictByScopeLocked
// (store.go): delete every edge touching one of these nodes, then the
// nodes themselves.
ids, err := repoNodeIDsTx(tx, prefix)
if err != nil {
return err
}
if err := deleteByIDColumnsTx(tx, "edges", []string{"from_id", "to_id"}, ids); err != nil {
return fmt.Errorf("store_sqlite: PurgeRepo edges: %w", err)
}
if err := deleteByIDColumnsTx(tx, "vectors", []string{"node_id"}, ids); err != nil {
return fmt.Errorf("store_sqlite: PurgeRepo vectors: %w", err)
}
for _, table := range purgeSidecarTables {
if _, err := tx.Exec(`DELETE FROM `+table+` WHERE repo_prefix = ?`, prefix); err != nil {
return fmt.Errorf("store_sqlite: PurgeRepo %s: %w", table, err)
}
}
if _, err := tx.Exec(`DELETE FROM nodes WHERE repo_prefix = ?`, prefix); err != nil {
return fmt.Errorf("store_sqlite: PurgeRepo nodes: %w", err)
}
return tx.Commit()
}
// orphanScanTables are the tables OrphanRepoPrefixes unions DISTINCT
// repo_prefix over. These five span the residue space: nodes (the primary
// keyed store), file_mtimes + repo_index_state (the warm-restart provenance
// that lingers when nodes are gone but sidecars survive — the exact shape a
// leaked untrack leaves), enrichment_state (per-provider provenance), and
// files (per-file metadata). A prefix whose nodes are gone but whose
// sidecars remain is invisible to a nodes-only scan, which is why the
// sidecar tables are unioned in; scanning still more tables would only
// rediscover the same prefixes at higher cost.
var orphanScanTables = []string{
"nodes",
"file_mtimes",
"repo_index_state",
"enrichment_state",
"files",
}
// OrphanRepoPrefixes returns every repo_prefix present in the store but
// absent from known — repos whose rows outlived their config entry (an
// untrack that predated PurgeRepo, or a repo dropped straight from config
// with no untrack at all). The empty prefix is NEVER reported (shared global
// externals / solo data). known is matched case-insensitively as a safety
// net, so a case-only spelling drift on a case-insensitive filesystem can
// never flag a still-tracked repo as an orphan (the #270 failure mode).
// Startup warmup feeds the result to PurgeRepo.
func (s *Store) OrphanRepoPrefixes(known []string) []string {
knownFold := make(map[string]struct{}, len(known))
for _, k := range known {
if k == "" {
continue
}
knownFold[strings.ToLower(k)] = struct{}{}
}
seen := make(map[string]struct{})
var out []string
for _, table := range orphanScanTables {
// WHERE repo_prefix <> '' both excludes the protected empty prefix
// and lets the nodes scan ride the partial nodes_by_repo index
// (defined WHERE repo_prefix <> ''). A table absent on an older
// schema simply contributes nothing.
rows, err := s.db.Query(`SELECT DISTINCT repo_prefix FROM ` + table + ` WHERE repo_prefix <> ''`)
if err != nil {
continue
}
for rows.Next() {
var p string
if err := rows.Scan(&p); err != nil {
break
}
if p == "" {
continue
}
if _, ok := knownFold[strings.ToLower(p)]; ok {
continue
}
if _, dup := seen[p]; dup {
continue
}
seen[p] = struct{}{}
out = append(out, p)
}
_ = rows.Close()
}
return out
}
// rekeyMoveTables are the sidecar tables RekeyRepoPrefix relabels from old
// to new. Every one is keyed by repo_prefix (+ file_path or provider), NOT
// by node_id, so its row content survives a node-id change: file_mtimes /
// files by (repo_prefix, file_path); repo_index_state / enrichment_state by
// repo_prefix (+ provider). At a solo->multi migration every '' row in these
// belongs to the one migrating repo — global externals live in the NODES
// table and hold NO rows here — so moving them wholesale is safe. UPDATE OR
// REPLACE folds any row the re-mint re-index already wrote under new
// (identical content: same files, same mtimes, same commit) instead of
// tripping the primary-key conflict a plain UPDATE would.
var rekeyMoveTables = []string{
"file_mtimes",
"files",
"repo_index_state",
"enrichment_state",
}
// rekeyDropTables are the sidecar tables RekeyRepoPrefix DROPS (rather than
// relabels) for old. Every one is keyed by node_id, and the solo->multi
// re-mint changes every node id (unprefixed `pkg::X` -> `<new>::pkg::X`), so
// these old-id rows are already dangling against the evicted unprefixed
// nodes. Relabeling their repo_prefix would just move dangling rows under
// new — and let, e.g., the clone reseed load a shingle set for a node that
// no longer exists. Dropping them is correct: the re-mint re-index rewrites
// the index-time sidecars (constant_values, ref_facts, clone_shingles) under
// the new node ids, and the enrichment sidecars (churn/coverage/release/
// blame) must re-run for the new ids regardless. The FTS vtables sit here
// too — their rows carry the old node ids, and UPDATE over an FTS5 UNINDEXED
// column is awkward, so delete-then-reindex is the clean path.
var rekeyDropTables = []string{
"clone_shingles",
"constant_values",
"ref_facts",
"churn_enrichment",
"coverage_enrichment",
"release_enrichment",
"blame_enrichment",
"symbol_fts",
"symbol_fts_rowid",
"content_fts",
}
// RekeyRepoPrefix moves a repo's sidecar residue from old to new the moment a
// solo (unprefixed) repo earns a real prefix because a second repo joined —
// the migrateLoneUnprefixedRepoCtx path. The prefix/path-keyed provenance
// tables (rekeyMoveTables) are relabeled so warm restart finds the repo's
// mtimes + freshness under new instead of full-re-tracking it; the
// node_id-keyed tables (rekeyDropTables) are dropped because the re-mint
// changed every node id out from under them (see the two table lists for the
// per-table rationale).
//
// Refuses new=="" (cannot rekey INTO the protected empty prefix). old=="" IS
// allowed — that is the whole point, since solo repos index unprefixed — and
// is safe here because this method touches SIDECAR tables ONLY; the synthetic
// global externals that also carry repo_prefix='' live in the NODES table,
// which RekeyRepoPrefix never writes.
func (s *Store) RekeyRepoPrefix(oldPrefix, newPrefix string) error {
if newPrefix == "" {
return fmt.Errorf("store_sqlite: RekeyRepoPrefix refuses empty destination prefix")
}
if oldPrefix == newPrefix {
return nil
}
s.writeMu.Lock()
defer s.writeMu.Unlock()
tx, err := s.db.Begin()
if err != nil {
return err
}
defer tx.Rollback() //nolint:errcheck // rollback after Commit is a no-op
for _, table := range rekeyMoveTables {
if _, err := tx.Exec(`UPDATE OR REPLACE `+table+` SET repo_prefix = ? WHERE repo_prefix = ?`, newPrefix, oldPrefix); err != nil {
return fmt.Errorf("store_sqlite: RekeyRepoPrefix move %s: %w", table, err)
}
}
for _, table := range rekeyDropTables {
if _, err := tx.Exec(`DELETE FROM `+table+` WHERE repo_prefix = ?`, oldPrefix); err != nil {
return fmt.Errorf("store_sqlite: RekeyRepoPrefix drop %s: %w", table, err)
}
}
// vectors is intentionally omitted: it has NO repo_prefix column (keyed
// by node_id alone), so it cannot be addressed here by prefix. Any ''
// embeddings are node_id-keyed against now-evicted unprefixed ids —
// dangling, and absent in the common case (embeddings are opt-in). They
// are left to a node-membership vector GC rather than guessed at here.
return tx.Commit()
}
// repoNodeIDsTx returns every node id in repoPrefix, read inside tx. The
// caller holds writeMu. Rows are fully drained + closed before the caller
// issues writes on the same tx — SQLite forbids an open read cursor while
// writing on the same connection.
func repoNodeIDsTx(tx *sql.Tx, repoPrefix string) ([]string, error) {
rows, err := tx.Query(`SELECT id FROM nodes WHERE repo_prefix = ?`, repoPrefix)
if err != nil {
return nil, err
}
var ids []string
for rows.Next() {
var id string
if err := rows.Scan(&id); err != nil {
_ = rows.Close()
return nil, err
}
ids = append(ids, id)
}
if err := rows.Err(); err != nil {
_ = rows.Close()
return nil, err
}
_ = rows.Close()
return ids, nil
}
// deleteByIDColumnsTx deletes rows from table where ANY of cols matches one
// of ids, chunked so each statement stays under SQLite's 999 bound-variable
// limit. Mirrors evictByScopeLocked's chunked from_id/to_id edge delete
// (store.go) — the semantics source for edge eviction. Empty ids is a no-op.
func deleteByIDColumnsTx(tx *sql.Tx, table string, cols, ids []string) error {
if len(ids) == 0 {
return nil
}
const chunk = 900
for _, col := range cols {
for start := 0; start < len(ids); start += chunk {
end := minInt(start+chunk, len(ids))
batch := ids[start:end]
placeholders := strings.TrimSuffix(strings.Repeat("?,", len(batch)), ",")
args := make([]any, len(batch))
for i, id := range batch {
args[i] = id
}
if _, err := tx.Exec(`DELETE FROM `+table+` WHERE `+col+` IN (`+placeholders+`)`, args...); err != nil {
return err
}
}
}
return nil
}
@@ -0,0 +1,222 @@
package store_sqlite
import (
"database/sql"
"github.com/zzet/gortex/internal/graph"
"path/filepath"
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
// openPurgeStore opens a throwaway on-disk store for the hygiene tests.
func openPurgeStore(t *testing.T) *Store {
t.Helper()
s, err := Open(filepath.Join(t.TempDir(), "purge.sqlite"))
require.NoError(t, err)
t.Cleanup(func() { _ = s.Close() })
return s
}
// seedRepoRows inserts exactly one row keyed to prefix into nodes, edges,
// vectors, and every repo_prefix-keyed sidecar table, so a PurgeRepo /
// RekeyRepoPrefix test can assert each table is cleared/moved. Node/vector
// ids embed the prefix (`<prefix>::a.go::X`) so the node-id-keyed vectors +
// edges land in the prefix's scope. Uses raw SQL for exhaustiveness: some
// tables have no public setter.
func seedRepoRows(t *testing.T, db *sql.DB, prefix string) {
t.Helper()
nodeID := prefix + "::a.go::X"
exec := func(q string, args ...any) {
t.Helper()
_, err := db.Exec(q, args...)
require.NoError(t, err, q)
}
exec(`INSERT INTO nodes (id, kind, name, file_path, repo_prefix) VALUES (?, 'function', 'X', 'a.go', ?)`, nodeID, prefix)
// An edge from the repo's node to a shared '' global external: PurgeRepo
// must delete this edge (its from_id is a repo node) but NEVER the ''
// target node.
exec(`INSERT INTO edges (from_id, to_id, kind) VALUES (?, 'external_call::dep:shared', 'calls')`, nodeID)
exec(`INSERT INTO vectors (node_id, dims, vec) VALUES (?, 1, X'00')`, nodeID)
exec(`INSERT INTO file_mtimes (repo_prefix, file_path, mtime_ns) VALUES (?, 'a.go', 123)`, prefix)
exec(`INSERT INTO repo_index_state (repo_prefix, indexed_sha) VALUES (?, 'sha')`, prefix)
exec(`INSERT INTO enrichment_state (repo_prefix, provider) VALUES (?, 'lsp')`, prefix)
exec(`INSERT INTO clone_shingles (node_id, repo_prefix, shingles) VALUES (?, ?, X'00')`, nodeID, prefix)
exec(`INSERT INTO constant_values (node_id, repo_prefix, file_path, value) VALUES (?, ?, 'a.go', 'v')`, nodeID, prefix)
exec(`INSERT INTO files (repo_prefix, file_path, content_hash) VALUES (?, 'a.go', 'h')`, prefix)
exec(`INSERT INTO ref_facts (repo_prefix, from_id, to_id, kind, line) VALUES (?, ?, 'a.go::Y', 'ref', 1)`, prefix, nodeID)
exec(`INSERT INTO churn_enrichment (node_id, repo_prefix, commit_count) VALUES (?, ?, 3)`, nodeID, prefix)
exec(`INSERT INTO coverage_enrichment (node_id, repo_prefix, coverage_pct) VALUES (?, ?, 0.5)`, nodeID, prefix)
exec(`INSERT INTO release_enrichment (node_id, repo_prefix, added_in) VALUES (?, ?, 'v1')`, nodeID, prefix)
exec(`INSERT INTO blame_enrichment (node_id, repo_prefix, email) VALUES (?, ?, 'a@b')`, nodeID, prefix)
exec(`INSERT INTO symbol_fts (node_id, repo_prefix, tokens) VALUES (?, ?, 'x')`, nodeID, prefix)
exec(`INSERT INTO symbol_fts_rowid (node_id, repo_prefix, fts_rowid) VALUES (?, ?, 1)`, nodeID, prefix)
exec(`INSERT INTO content_fts (node_id, repo_prefix, file_path, ordinal, body) VALUES (?, ?, 'a.go', 0, 'body')`, nodeID, prefix)
}
// countByPrefix reports how many rows a repo_prefix-keyed table holds for
// prefix. nodes and every sidecar carry a repo_prefix column.
func countByPrefix(t *testing.T, db *sql.DB, table, prefix string) int {
t.Helper()
var n int
require.NoError(t, db.QueryRow(`SELECT COUNT(*) FROM `+table+` WHERE repo_prefix = ?`, prefix).Scan(&n))
return n
}
// countByNodeIDLike reports how many rows a node_id-keyed table (vectors)
// holds whose node_id starts with `<prefix>::`.
func countByNodeIDLike(t *testing.T, db *sql.DB, table, prefix string) int {
t.Helper()
var n int
require.NoError(t, db.QueryRow(`SELECT COUNT(*) FROM `+table+` WHERE node_id LIKE ?`, prefix+"::%").Scan(&n))
return n
}
// prefixKeyedTables is every repo_prefix-keyed table PurgeRepo/Rekey touch,
// minus nodes (asserted separately) — used to loop assertions.
var prefixKeyedTables = []string{
"file_mtimes", "repo_index_state", "enrichment_state", "clone_shingles",
"constant_values", "files", "ref_facts", "churn_enrichment",
"coverage_enrichment", "release_enrichment", "blame_enrichment",
"symbol_fts", "symbol_fts_rowid", "content_fts",
}
func TestPurgeRepo_ClearsEveryTable_LeavesOthersAndGlobals(t *testing.T) {
s := openPurgeStore(t)
// Two real repos plus a shared '' global-external node the purge must
// never touch.
seedRepoRows(t, s.db, "repoA")
seedRepoRows(t, s.db, "repoB")
_, err := s.db.Exec(`INSERT INTO nodes (id, kind, name, file_path, repo_prefix) VALUES ('external_call::dep:shared', 'external', 'shared', '', '')`)
require.NoError(t, err)
require.NoError(t, s.PurgeRepo("repoA"))
// repoA: nodes, edges, vectors, and every sidecar cleared.
assert.Equal(t, 0, countByPrefix(t, s.db, "nodes", "repoA"), "repoA nodes gone")
assert.Equal(t, 0, countByNodeIDLike(t, s.db, "vectors", "repoA"), "repoA vectors gone")
for _, tbl := range prefixKeyedTables {
assert.Equal(t, 0, countByPrefix(t, s.db, tbl, "repoA"), "repoA %s cleared", tbl)
}
var edgesFromA int
require.NoError(t, s.db.QueryRow(`SELECT COUNT(*) FROM edges WHERE from_id LIKE 'repoA::%'`).Scan(&edgesFromA))
assert.Equal(t, 0, edgesFromA, "repoA edges gone")
// repoB untouched across the board.
assert.Equal(t, 1, countByPrefix(t, s.db, "nodes", "repoB"), "repoB nodes intact")
assert.Equal(t, 1, countByNodeIDLike(t, s.db, "vectors", "repoB"), "repoB vectors intact")
for _, tbl := range prefixKeyedTables {
assert.Equal(t, 1, countByPrefix(t, s.db, tbl, "repoB"), "repoB %s intact", tbl)
}
// The shared '' global external survives — nothing may purge ''.
assert.Equal(t, 1, countByPrefix(t, s.db, "nodes", ""), "'' global external survives")
}
func TestPurgeRepo_RefusesEmptyPrefix(t *testing.T) {
s := openPurgeStore(t)
seedRepoRows(t, s.db, "")
require.Error(t, s.PurgeRepo(""), "PurgeRepo must refuse the empty prefix (global externals / solo data)")
// The '' rows are still there.
assert.Equal(t, 1, countByPrefix(t, s.db, "file_mtimes", ""), "'' file_mtimes untouched by refused purge")
}
func TestOrphanRepoPrefixes_SidecarOnlyResidue(t *testing.T) {
s := openPurgeStore(t)
// gone: a repo whose NODES were evicted but whose sidecars linger — the
// exact leaked-untrack shape (residue in file_mtimes + repo_index_state,
// no nodes). live: a fully-present tracked repo.
_, err := s.db.Exec(`INSERT INTO file_mtimes (repo_prefix, file_path, mtime_ns) VALUES ('gone', 'x.go', 1)`)
require.NoError(t, err)
_, err = s.db.Exec(`INSERT INTO repo_index_state (repo_prefix) VALUES ('gone')`)
require.NoError(t, err)
seedRepoRows(t, s.db, "live")
// A '' row must never be reported as an orphan.
_, err = s.db.Exec(`INSERT INTO file_mtimes (repo_prefix, file_path, mtime_ns) VALUES ('', 'g.go', 1)`)
require.NoError(t, err)
orphans := s.OrphanRepoPrefixes([]string{"live"})
assert.Equal(t, []string{"gone"}, orphans, "only the nodes-less residue prefix is an orphan")
// Case-fold safety net: a case-only spelling drift of a tracked repo is
// NOT an orphan.
assert.Empty(t, s.OrphanRepoPrefixes([]string{"LIVE", "GONE"}), "case-insensitive known set covers both prefixes")
}
func TestRekeyRepoPrefix_MovesProvenanceDropsNodeIDKeyed(t *testing.T) {
s := openPurgeStore(t)
seedRepoRows(t, s.db, "") // solo repo: everything under ''
require.NoError(t, s.RekeyRepoPrefix("", "drools"))
// Prefix/path-keyed provenance MOVED '' -> drools (so warm restart finds
// the repo's mtimes under the new prefix instead of full-re-tracking).
moveTables := []string{"file_mtimes", "files", "repo_index_state", "enrichment_state"}
for _, tbl := range moveTables {
assert.Equal(t, 0, countByPrefix(t, s.db, tbl, ""), "%s '' rows moved out", tbl)
assert.Equal(t, 1, countByPrefix(t, s.db, tbl, "drools"), "%s rows now under new prefix", tbl)
}
// node_id-keyed tables DROPPED (their old ids are dangling after the
// re-mint) — the FTS decision included.
dropTables := []string{
"clone_shingles", "constant_values", "ref_facts", "churn_enrichment",
"coverage_enrichment", "release_enrichment", "blame_enrichment",
"symbol_fts", "symbol_fts_rowid", "content_fts",
}
for _, tbl := range dropTables {
assert.Equal(t, 0, countByPrefix(t, s.db, tbl, ""), "%s '' rows dropped", tbl)
assert.Equal(t, 0, countByPrefix(t, s.db, tbl, "drools"), "%s NOT relabeled to new prefix", tbl)
}
assert.Error(t, s.RekeyRepoPrefix("repoA", ""), "rekey INTO the empty prefix is refused")
}
// TestContentCrashWindow simulates the D4 kill-window at the store level:
// per-file delete+append leaves a mix of old+new content instead of an empty
// table, and the end-of-track sweep (keep = files that STREAMED content this
// run) reaps both files that vanished from disk and files that still exist
// but no longer yield content sections.
func TestContentCrashWindow(t *testing.T) {
s := openPurgeStore(t)
item := func(id, file, body string) graph.ContentFTSItem {
return graph.ContentFTSItem{NodeID: id, FilePath: file, Ordinal: 0, Body: body}
}
// Prior full index: three content files present.
require.NoError(t, s.AppendContent("r", []graph.ContentFTSItem{item("r::f1", "f1.md", "old one")}))
require.NoError(t, s.AppendContent("r", []graph.ContentFTSItem{item("r::f2", "f2.md", "old two")}))
require.NoError(t, s.AppendContent("r", []graph.ContentFTSItem{item("r::f3", "f3.md", "old three")}))
// A new full index re-streams only f1 (crash before it reached the
// rest): delete f1's rows then re-append. The other files' OLD rows must
// survive — no empty-table window.
require.NoError(t, s.WipeContentFileInRepo("r", "f1.md"))
require.NoError(t, s.AppendContent("r", []graph.ContentFTSItem{item("r::f1", "f1.md", "new one")}))
countFile := func(file string) int {
var n int
require.NoError(t, s.db.QueryRow(`SELECT COUNT(*) FROM content_fts WHERE repo_prefix='r' AND file_path=?`, file).Scan(&n))
return n
}
assert.Equal(t, 1, countFile("f1.md"), "f1 refreshed (delete+append, not doubled)")
assert.Equal(t, 1, countFile("f2.md"), "f2's old rows survive the mid-parse kill (no empty table)")
assert.Equal(t, 1, countFile("f3.md"), "f3's old rows survive the mid-parse kill (no empty table)")
// The next SUCCESSFUL completion: f2 was deleted from the repo, and f3
// STILL EXISTS on disk but was emptied — it streamed no content sections
// this run, so it is absent from the streamed set. The sweep keeps
// exactly the streamed set {f1}, reaping both the vanished file and the
// content->no-content transition (a disk-survival keep would have
// protected f3's stale rows forever).
require.NoError(t, s.DeleteContentFilesForRepoNotIn("r", map[string]struct{}{"f1.md": {}}))
assert.Equal(t, 1, countFile("f1.md"), "still-streaming file kept")
assert.Equal(t, 0, countFile("f2.md"), "vanished file swept")
assert.Equal(t, 0, countFile("f3.md"), "content->no-content transition swept despite surviving on disk")
// Empty keep is a deliberate no-op (the never-wipe-from-empty safety
// net; a zero-content walk routes to WipeContent instead).
require.NoError(t, s.DeleteContentFilesForRepoNotIn("r", nil))
assert.Equal(t, 1, countFile("f1.md"), "empty keep never wipes")
}
@@ -0,0 +1,219 @@
package store_sqlite
import (
"strings"
"github.com/zzet/gortex/internal/graph"
)
// Compile-time assertions that the SQLite Store satisfies the optional
// reference-facts persistence capability. Persisting resolved-reference facts
// in the same backend the graph lives in makes a reference's resolution an
// auditable, diffable record and a warm-restart seed.
var (
_ graph.RefFactsWriter = (*Store)(nil)
_ graph.RefFactsReader = (*Store)(nil)
)
// refFactChunk bounds rows per multi-row INSERT. 11 params/row; 80 rows = 880
// host params, under SQLite's 999 default. Mirrors shingleChunk.
const refFactChunk = 80
// candidate-list separator (unit separator — never appears in identifiers).
const refFactCandSep = "\x1f"
func encodeCandidates(c []string) string { return strings.Join(c, refFactCandSep) }
func decodeCandidates(s string) []string {
if s == "" {
return nil
}
return strings.Split(s, refFactCandSep)
}
// BulkSetRefFacts persists resolved-reference facts for one repo prefix in a
// single transaction, chunked under the host-parameter limit. Idempotent on
// (repo_prefix, from_id, to_id, kind, line). Empty input is a no-op.
func (s *Store) BulkSetRefFacts(repoPrefix string, facts []graph.RefFact) error {
if len(facts) == 0 {
return nil
}
s.writeMu.Lock()
defer s.writeMu.Unlock()
tx, err := s.db.Begin()
if err != nil {
return err
}
defer tx.Rollback() //nolint:errcheck // rollback after Commit is a no-op
for start := 0; start < len(facts); start += refFactChunk {
end := start + refFactChunk
if end > len(facts) {
end = len(facts)
}
batch := facts[start:end]
args := make([]any, 0, len(batch)*11)
stmt := make([]byte, 0, 96+len(batch)*24)
stmt = append(stmt, "INSERT OR REPLACE INTO ref_facts (repo_prefix, from_id, to_id, kind, ref_name, line, origin, tier, candidates, file_path, lang) VALUES "...)
for i, f := range batch {
if i > 0 {
stmt = append(stmt, ',')
}
stmt = append(stmt, "(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)"...)
args = append(args, repoPrefix, f.FromID, f.ToID, f.Kind, f.RefName, f.Line, f.Origin, f.Tier, encodeCandidates(f.Candidates), f.FilePath, f.Lang)
}
if _, err := tx.Exec(string(stmt), args...); err != nil {
return err
}
}
return tx.Commit()
}
// DeleteRefFactsByFiles drops all facts sourced in the supplied files for one
// repo prefix, chunked into `file_path IN (…)` DELETEs. Empty input is a no-op.
func (s *Store) DeleteRefFactsByFiles(repoPrefix string, files []string) error {
if len(files) == 0 {
return nil
}
s.writeMu.Lock()
defer s.writeMu.Unlock()
tx, err := s.db.Begin()
if err != nil {
return err
}
defer tx.Rollback() //nolint:errcheck // rollback after Commit is a no-op
for start := 0; start < len(files); start += refFactChunk {
end := start + refFactChunk
if end > len(files) {
end = len(files)
}
chunk := files[start:end]
args := make([]any, 0, len(chunk)+1)
args = append(args, repoPrefix)
stmt := make([]byte, 0, 64+len(chunk)*2)
stmt = append(stmt, "DELETE FROM ref_facts WHERE repo_prefix = ? AND file_path IN ("...)
for i, f := range chunk {
if i > 0 {
stmt = append(stmt, ',')
}
stmt = append(stmt, '?')
args = append(args, f)
}
stmt = append(stmt, ')')
if _, err := tx.Exec(string(stmt), args...); err != nil {
return err
}
}
return tx.Commit()
}
// LoadRefFactsByFiles returns the persisted facts for one repo prefix, scoped
// to the given files (all files when files is empty). Always non-nil.
func (s *Store) LoadRefFactsByFiles(repoPrefix string, files []string) ([]graph.RefFact, error) {
out := []graph.RefFact{}
scan := func(query string, args ...any) error {
rows, err := s.db.Query(query, args...)
if err != nil {
return err
}
defer rows.Close()
for rows.Next() {
var f graph.RefFact
var cand string
if err := rows.Scan(&f.FromID, &f.ToID, &f.Kind, &f.RefName, &f.Line, &f.Origin, &f.Tier, &cand, &f.FilePath, &f.Lang); err != nil {
return err
}
f.RepoPrefix = repoPrefix
f.Candidates = decodeCandidates(cand)
out = append(out, f)
}
return rows.Err()
}
const cols = `from_id, to_id, kind, ref_name, line, origin, tier, candidates, file_path, lang`
if len(files) == 0 {
if err := scan(`SELECT `+cols+` FROM ref_facts WHERE repo_prefix = ?`, repoPrefix); err != nil {
return nil, err
}
return out, nil
}
for start := 0; start < len(files); start += refFactChunk {
end := start + refFactChunk
if end > len(files) {
end = len(files)
}
chunk := files[start:end]
args := make([]any, 0, len(chunk)+1)
args = append(args, repoPrefix)
stmt := make([]byte, 0, 96+len(chunk)*2)
stmt = append(stmt, "SELECT "+cols+" FROM ref_facts WHERE repo_prefix = ? AND file_path IN ("...)
for i, f := range chunk {
if i > 0 {
stmt = append(stmt, ',')
}
stmt = append(stmt, '?')
args = append(args, f)
}
stmt = append(stmt, ')')
if err := scan(string(stmt), args...); err != nil {
return nil, err
}
}
return out, nil
}
// LoadRefFactsByTargets returns the persisted facts that resolve TO any of
// the given node IDs for one repo prefix, grouped by source file path — the
// reverse lookup incremental re-resolution uses to find the files that
// referenced a changed symbol after its live in-edges were evicted. Served by
// the ref_facts_by_target index, chunked under the host-parameter limit.
// Always non-nil; empty input is a no-op.
func (s *Store) LoadRefFactsByTargets(repoPrefix string, targetIDs []string) (map[string][]graph.RefFact, error) {
out := map[string][]graph.RefFact{}
if len(targetIDs) == 0 {
return out, nil
}
const cols = `from_id, to_id, kind, ref_name, line, origin, tier, candidates, file_path, lang`
for start := 0; start < len(targetIDs); start += refFactChunk {
end := start + refFactChunk
if end > len(targetIDs) {
end = len(targetIDs)
}
chunk := targetIDs[start:end]
args := make([]any, 0, len(chunk)+1)
args = append(args, repoPrefix)
stmt := make([]byte, 0, 96+len(chunk)*2)
stmt = append(stmt, "SELECT "+cols+" FROM ref_facts WHERE repo_prefix = ? AND to_id IN ("...)
for i, id := range chunk {
if i > 0 {
stmt = append(stmt, ',')
}
stmt = append(stmt, '?')
args = append(args, id)
}
stmt = append(stmt, ')')
rows, err := s.db.Query(string(stmt), args...)
if err != nil {
return nil, err
}
for rows.Next() {
var f graph.RefFact
var cand string
if err := rows.Scan(&f.FromID, &f.ToID, &f.Kind, &f.RefName, &f.Line, &f.Origin, &f.Tier, &cand, &f.FilePath, &f.Lang); err != nil {
_ = rows.Close()
return nil, err
}
f.RepoPrefix = repoPrefix
f.Candidates = decodeCandidates(cand)
out[f.FilePath] = append(out[f.FilePath], f)
}
err = rows.Err()
_ = rows.Close()
if err != nil {
return nil, err
}
}
return out, nil
}
@@ -0,0 +1,172 @@
package store_sqlite_test
import (
"fmt"
"path/filepath"
"testing"
"github.com/stretchr/testify/require"
"github.com/zzet/gortex/internal/graph"
"github.com/zzet/gortex/internal/graph/store_sqlite"
)
func openRefFactStore(t *testing.T) *store_sqlite.Store {
t.Helper()
s, err := store_sqlite.Open(filepath.Join(t.TempDir(), "rf.sqlite"))
require.NoError(t, err)
t.Cleanup(func() { _ = s.Close() })
return s
}
func TestRefFacts_Roundtrip(t *testing.T) {
s := openRefFactStore(t)
facts := []graph.RefFact{
{RepoPrefix: "", FromID: "a.go::A", ToID: "b.go::B", Kind: "calls", RefName: "B", Line: 7, Origin: "ast_resolved", Tier: "ast", FilePath: "a.go", Lang: "go", Candidates: []string{"b.go::B", "c.go::B"}},
{RepoPrefix: "", FromID: "a.go::A", ToID: "d.go::D", Kind: "references", RefName: "D", Line: 9, Origin: "lsp_resolved", Tier: "lsp", FilePath: "a.go", Lang: "go"},
}
require.NoError(t, s.BulkSetRefFacts("", facts))
got, err := s.LoadRefFactsByFiles("", []string{"a.go"})
require.NoError(t, err)
require.Len(t, got, 2)
byTo := map[string]graph.RefFact{}
for _, f := range got {
byTo[f.ToID] = f
}
require.Equal(t, "ast_resolved", byTo["b.go::B"].Origin)
require.Equal(t, []string{"b.go::B", "c.go::B"}, byTo["b.go::B"].Candidates)
require.Equal(t, 7, byTo["b.go::B"].Line)
require.Equal(t, "lsp_resolved", byTo["d.go::D"].Origin)
// LoadRefFactsByFiles with empty file list returns all for the repo.
all, err := s.LoadRefFactsByFiles("", nil)
require.NoError(t, err)
require.Len(t, all, 2)
}
func TestRefFacts_DeleteByFile(t *testing.T) {
s := openRefFactStore(t)
require.NoError(t, s.BulkSetRefFacts("", []graph.RefFact{
{FromID: "a.go::A", ToID: "x", Kind: "calls", FilePath: "a.go"},
{FromID: "b.go::B", ToID: "y", Kind: "calls", FilePath: "b.go"},
}))
require.NoError(t, s.DeleteRefFactsByFiles("", []string{"a.go"}))
got, err := s.LoadRefFactsByFiles("", nil)
require.NoError(t, err)
require.Len(t, got, 1)
require.Equal(t, "b.go", got[0].FilePath)
}
func TestRefFacts_RepoScoping(t *testing.T) {
s := openRefFactStore(t)
require.NoError(t, s.BulkSetRefFacts("repoA", []graph.RefFact{{FromID: "f::A", ToID: "tA", Kind: "calls", FilePath: "f.go"}}))
require.NoError(t, s.BulkSetRefFacts("repoB", []graph.RefFact{{FromID: "f::A", ToID: "tB", Kind: "calls", FilePath: "f.go"}}))
a, err := s.LoadRefFactsByFiles("repoA", []string{"f.go"})
require.NoError(t, err)
require.Len(t, a, 1)
require.Equal(t, "tA", a[0].ToID)
// Deleting repoA's file must not touch repoB.
require.NoError(t, s.DeleteRefFactsByFiles("repoA", []string{"f.go"}))
b, err := s.LoadRefFactsByFiles("repoB", []string{"f.go"})
require.NoError(t, err)
require.Len(t, b, 1)
require.Equal(t, "tB", b[0].ToID)
}
func TestRefFacts_Chunking(t *testing.T) {
s := openRefFactStore(t)
const n = 500 // > refFactChunk (80)
facts := make([]graph.RefFact, n)
for i := range facts {
facts[i] = graph.RefFact{FromID: fmt.Sprintf("a.go::f%d", i), ToID: fmt.Sprintf("t%d", i), Kind: "calls", FilePath: "a.go"}
}
require.NoError(t, s.BulkSetRefFacts("", facts))
got, err := s.LoadRefFactsByFiles("", []string{"a.go"})
require.NoError(t, err)
require.Len(t, got, n)
}
func TestRefFacts_EmptyNoop(t *testing.T) {
s := openRefFactStore(t)
require.NoError(t, s.BulkSetRefFacts("", nil))
require.NoError(t, s.DeleteRefFactsByFiles("", nil))
got, err := s.LoadRefFactsByFiles("", nil)
require.NoError(t, err)
require.Empty(t, got)
}
func TestRefFacts_LoadByTargets(t *testing.T) {
s := openRefFactStore(t)
require.NoError(t, s.BulkSetRefFacts("", []graph.RefFact{
{FromID: "b.go::Caller", ToID: "a.go::F", Kind: "calls", RefName: "F", Line: 3, Origin: "ast_resolved", Tier: "ast", FilePath: "b.go", Lang: "go"},
{FromID: "c.go::Other", ToID: "a.go::F", Kind: "references", RefName: "F", FilePath: "c.go"},
{FromID: "c.go::Other", ToID: "a.go::G", Kind: "calls", RefName: "G", FilePath: "c.go"},
{FromID: "d.go::X", ToID: "z.go::Z", Kind: "calls", RefName: "Z", FilePath: "d.go"},
}))
byFile, err := s.LoadRefFactsByTargets("", []string{"a.go::F", "a.go::G"})
require.NoError(t, err)
require.Len(t, byFile, 2, "facts must be grouped by source file: %v", byFile)
require.Len(t, byFile["b.go"], 1)
require.Equal(t, "a.go::F", byFile["b.go"][0].ToID)
require.Equal(t, "F", byFile["b.go"][0].RefName)
require.Equal(t, "ast_resolved", byFile["b.go"][0].Origin)
require.Len(t, byFile["c.go"], 2, "both of c.go's facts target the queried symbols")
require.NotContains(t, byFile, "d.go", "a fact targeting an unqueried symbol must not match")
}
func TestRefFacts_LoadByTargets_EmptyAndMissing(t *testing.T) {
s := openRefFactStore(t)
require.NoError(t, s.BulkSetRefFacts("", []graph.RefFact{
{FromID: "a.go::A", ToID: "b.go::B", Kind: "calls", FilePath: "a.go"},
}))
// Empty input: empty, non-nil map.
empty, err := s.LoadRefFactsByTargets("", nil)
require.NoError(t, err)
require.NotNil(t, empty)
require.Empty(t, empty)
// A target nothing references: no rows, no error.
miss, err := s.LoadRefFactsByTargets("", []string{"nope::Missing"})
require.NoError(t, err)
require.Empty(t, miss)
}
func TestRefFacts_LoadByTargets_RepoScoping(t *testing.T) {
s := openRefFactStore(t)
require.NoError(t, s.BulkSetRefFacts("repoA", []graph.RefFact{{FromID: "fa::A", ToID: "shared::T", Kind: "calls", FilePath: "fa.go"}}))
require.NoError(t, s.BulkSetRefFacts("repoB", []graph.RefFact{{FromID: "fb::B", ToID: "shared::T", Kind: "calls", FilePath: "fb.go"}}))
a, err := s.LoadRefFactsByTargets("repoA", []string{"shared::T"})
require.NoError(t, err)
require.Len(t, a, 1)
require.Len(t, a["fa.go"], 1)
require.Equal(t, "repoA", a["fa.go"][0].RepoPrefix, "loaded facts must carry the queried repo prefix")
require.NotContains(t, a, "fb.go", "another repo's facts must not leak into the result")
}
func TestRefFacts_LoadByTargets_Chunking(t *testing.T) {
s := openRefFactStore(t)
const n = 500 // > refFactChunk (80)
facts := make([]graph.RefFact, n)
targets := make([]string, n)
for i := range facts {
facts[i] = graph.RefFact{
FromID: fmt.Sprintf("src%d.go::f", i),
ToID: fmt.Sprintf("dst.go::t%d", i),
Kind: "calls",
FilePath: fmt.Sprintf("src%d.go", i),
}
targets[i] = fmt.Sprintf("dst.go::t%d", i)
}
require.NoError(t, s.BulkSetRefFacts("", facts))
byFile, err := s.LoadRefFactsByTargets("", targets)
require.NoError(t, err)
require.Len(t, byFile, n, "every chunked target must come back grouped under its source file")
}
@@ -0,0 +1,140 @@
package store_sqlite
import (
"database/sql"
"github.com/zzet/gortex/internal/graph"
)
var (
_ graph.ReleaseEnrichmentWriter = (*Store)(nil)
_ graph.ReleaseEnrichmentReader = (*Store)(nil)
)
// releaseChunk bounds rows per multi-row INSERT (3 cols → 3 params/row).
const releaseChunk = 300
const releaseCols = `node_id, repo_prefix, added_in`
// BulkSetReleases persists release rows for one repo prefix, chunked.
func (s *Store) BulkSetReleases(repoPrefix string, rows []graph.ReleaseEnrichment) error {
if len(rows) == 0 {
return nil
}
s.writeMu.Lock()
defer s.writeMu.Unlock()
tx, err := s.db.Begin()
if err != nil {
return err
}
defer tx.Rollback() //nolint:errcheck
for start := 0; start < len(rows); start += releaseChunk {
end := start + releaseChunk
if end > len(rows) {
end = len(rows)
}
batch := rows[start:end]
args := make([]any, 0, len(batch)*3)
stmt := make([]byte, 0, 96+len(batch)*12)
stmt = append(stmt, "INSERT OR REPLACE INTO release_enrichment ("...)
stmt = append(stmt, releaseCols...)
stmt = append(stmt, ") VALUES "...)
for i, e := range batch {
if i > 0 {
stmt = append(stmt, ',')
}
stmt = append(stmt, "(?,?,?)"...)
args = append(args, e.NodeID, repoPrefix, e.AddedIn)
}
if _, err := tx.Exec(string(stmt), args...); err != nil {
return err
}
}
return tx.Commit()
}
// DeleteReleases drops release rows for the supplied node ids, chunked.
func (s *Store) DeleteReleases(nodeIDs []string) error {
if len(nodeIDs) == 0 {
return nil
}
seen := make(map[string]struct{}, len(nodeIDs))
uniq := make([]string, 0, len(nodeIDs))
for _, id := range nodeIDs {
if id == "" {
continue
}
if _, ok := seen[id]; ok {
continue
}
seen[id] = struct{}{}
uniq = append(uniq, id)
}
if len(uniq) == 0 {
return nil
}
s.writeMu.Lock()
defer s.writeMu.Unlock()
tx, err := s.db.Begin()
if err != nil {
return err
}
defer tx.Rollback() //nolint:errcheck
for start := 0; start < len(uniq); start += releaseChunk {
end := start + releaseChunk
if end > len(uniq) {
end = len(uniq)
}
chunk := uniq[start:end]
args := make([]any, len(chunk))
stmt := make([]byte, 0, 56+len(chunk)*2)
stmt = append(stmt, "DELETE FROM release_enrichment WHERE node_id IN ("...)
for i, id := range chunk {
if i > 0 {
stmt = append(stmt, ',')
}
stmt = append(stmt, '?')
args[i] = id
}
stmt = append(stmt, ')')
if _, err := tx.Exec(string(stmt), args...); err != nil {
return err
}
}
return tx.Commit()
}
// ReleaseRows returns release rows for repoPrefix; empty → all repos.
func (s *Store) ReleaseRows(repoPrefix string) []graph.ReleaseEnrichment {
var (
rows *sql.Rows
err error
)
if repoPrefix == "" {
rows, err = s.db.Query(`SELECT ` + releaseCols + ` FROM release_enrichment`)
} else {
rows, err = s.db.Query(`SELECT `+releaseCols+` FROM release_enrichment WHERE repo_prefix = ?`, repoPrefix)
}
if err != nil {
return nil
}
defer rows.Close()
var out []graph.ReleaseEnrichment
for rows.Next() {
var e graph.ReleaseEnrichment
if err := rows.Scan(&e.NodeID, &e.RepoPrefix, &e.AddedIn); err != nil {
return out
}
out = append(out, e)
}
if err := rows.Err(); err != nil {
return out
}
return out
}
+22
View File
@@ -0,0 +1,22 @@
package store_sqlite_test
import (
"path/filepath"
"testing"
"github.com/zzet/gortex/internal/graph"
"github.com/zzet/gortex/internal/graph/store_sqlite"
"github.com/zzet/gortex/internal/graph/storetest"
)
func TestSQLiteStoreConformance(t *testing.T) {
storetest.RunConformance(t, func(t *testing.T) graph.Store {
dir := t.TempDir()
s, err := store_sqlite.Open(filepath.Join(dir, "test.sqlite"))
if err != nil {
t.Fatalf("Open: %v", err)
}
t.Cleanup(func() { _ = s.Close() })
return s
})
}

Some files were not shown because too many files have changed in this diff Show More