chore: import upstream snapshot with attribution
CI / benchmark (push) Has been skipped
install-script / posix-syntax (push) Successful in 6m1s
CI / build-onnx (push) Failing after 6m43s
init-smoke / dry-run (push) Failing after 15m57s
security / govulncheck (push) Has been cancelled
security / trivy-fs (push) Has been cancelled
CI / test (1.26, ubuntu-latest) (push) Has been cancelled
Scorecard supply-chain security / Scorecard analysis (push) Has been cancelled
CI / test (1.26, macos-latest) (push) Has been cancelled
CI / build-windows (push) Has been cancelled
CI / lint (push) Has been cancelled
install-script / powershell-syntax (push) Has been cancelled
install-script / install (macos-14) (push) Has been cancelled
install-script / install (ubuntu-latest) (push) Has been cancelled
CI / benchmark (push) Has been skipped
install-script / posix-syntax (push) Successful in 6m1s
CI / build-onnx (push) Failing after 6m43s
init-smoke / dry-run (push) Failing after 15m57s
security / govulncheck (push) Has been cancelled
security / trivy-fs (push) Has been cancelled
CI / test (1.26, ubuntu-latest) (push) Has been cancelled
Scorecard supply-chain security / Scorecard analysis (push) Has been cancelled
CI / test (1.26, macos-latest) (push) Has been cancelled
CI / build-windows (push) Has been cancelled
CI / lint (push) Has been cancelled
install-script / powershell-syntax (push) Has been cancelled
install-script / install (macos-14) (push) Has been cancelled
install-script / install (ubuntu-latest) (push) Has been cancelled
This commit is contained in:
@@ -0,0 +1,270 @@
|
||||
package graph
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"runtime"
|
||||
"sync"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// =============================================================================
|
||||
// Scaled graph benchmarks — simulate RPi memory constraints
|
||||
// =============================================================================
|
||||
|
||||
// graphSizes defines graph sizes for sub-benchmarks.
|
||||
// "Tiny" approximates a small project on RPi, "Large" approximates a medium monorepo.
|
||||
var graphSizes = []struct {
|
||||
name string
|
||||
nodes int
|
||||
edges int
|
||||
}{
|
||||
{"Tiny_100", 100, 200},
|
||||
{"Small_1K", 1_000, 3_000},
|
||||
{"Medium_5K", 5_000, 15_000},
|
||||
{"Large_10K", 10_000, 30_000},
|
||||
{"XL_50K", 50_000, 150_000},
|
||||
{"XXL_100K", 100_000, 300_000},
|
||||
}
|
||||
|
||||
func buildScaledGraph(nodes, edges int) *Graph {
|
||||
g := New()
|
||||
for i := range nodes {
|
||||
g.AddNode(&Node{
|
||||
ID: fmt.Sprintf("pkg%d/file%d.go::sym%d", i/100, i/10, i),
|
||||
Kind: KindFunction,
|
||||
Name: fmt.Sprintf("sym%d", i),
|
||||
FilePath: fmt.Sprintf("pkg%d/file%d.go", i/100, i/10),
|
||||
Language: "go",
|
||||
})
|
||||
}
|
||||
for i := range edges {
|
||||
g.AddEdge(&Edge{
|
||||
From: fmt.Sprintf("pkg%d/file%d.go::sym%d", (i%nodes)/100, (i%nodes)/10, i%nodes),
|
||||
To: fmt.Sprintf("pkg%d/file%d.go::sym%d", ((i+1)%nodes)/100, ((i+1)%nodes)/10, (i+1)%nodes),
|
||||
Kind: EdgeCalls,
|
||||
})
|
||||
}
|
||||
return g
|
||||
}
|
||||
|
||||
func BenchmarkGraph_AddNode_Scaled(b *testing.B) {
|
||||
for _, sz := range graphSizes {
|
||||
b.Run(sz.name, func(b *testing.B) {
|
||||
b.ReportAllocs()
|
||||
g := New()
|
||||
b.ResetTimer()
|
||||
for i := range b.N {
|
||||
g.AddNode(&Node{
|
||||
ID: fmt.Sprintf("file%d.go::func%d", i/10, i),
|
||||
Kind: KindFunction,
|
||||
Name: fmt.Sprintf("func%d", i),
|
||||
})
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkGraph_GetNode_Scaled(b *testing.B) {
|
||||
for _, sz := range graphSizes {
|
||||
b.Run(sz.name, func(b *testing.B) {
|
||||
b.ReportAllocs()
|
||||
g := buildScaledGraph(sz.nodes, sz.edges)
|
||||
b.ResetTimer()
|
||||
for i := range b.N {
|
||||
g.GetNode(fmt.Sprintf("pkg%d/file%d.go::sym%d", (i%sz.nodes)/100, (i%sz.nodes)/10, i%sz.nodes))
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkGraph_AllNodes_Scaled(b *testing.B) {
|
||||
for _, sz := range graphSizes {
|
||||
b.Run(sz.name, func(b *testing.B) {
|
||||
b.ReportAllocs()
|
||||
g := buildScaledGraph(sz.nodes, sz.edges)
|
||||
b.ResetTimer()
|
||||
for b.Loop() {
|
||||
g.AllNodes()
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkGraph_Stats_Scaled(b *testing.B) {
|
||||
for _, sz := range graphSizes {
|
||||
b.Run(sz.name, func(b *testing.B) {
|
||||
b.ReportAllocs()
|
||||
g := buildScaledGraph(sz.nodes, sz.edges)
|
||||
b.ResetTimer()
|
||||
for b.Loop() {
|
||||
g.Stats()
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// =============================================================================
|
||||
// Eviction benchmarks — critical for memory-constrained devices
|
||||
// =============================================================================
|
||||
|
||||
func BenchmarkGraph_EvictFile(b *testing.B) {
|
||||
for _, sz := range graphSizes {
|
||||
b.Run(sz.name, func(b *testing.B) {
|
||||
b.ReportAllocs()
|
||||
for b.Loop() {
|
||||
b.StopTimer()
|
||||
g := buildScaledGraph(sz.nodes, sz.edges)
|
||||
b.StartTimer()
|
||||
// Evict ~10% of files
|
||||
for i := range sz.nodes / 10 {
|
||||
g.EvictFile(fmt.Sprintf("pkg%d/file%d.go", i/100, i/10))
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// =============================================================================
|
||||
// Concurrent access benchmarks — RPi has 4 cores, contention matters
|
||||
// =============================================================================
|
||||
|
||||
func BenchmarkGraph_ConcurrentRead(b *testing.B) {
|
||||
for _, sz := range graphSizes {
|
||||
b.Run(sz.name, func(b *testing.B) {
|
||||
b.ReportAllocs()
|
||||
g := buildScaledGraph(sz.nodes, sz.edges)
|
||||
b.ResetTimer()
|
||||
b.RunParallel(func(pb *testing.PB) {
|
||||
i := 0
|
||||
for pb.Next() {
|
||||
g.GetNode(fmt.Sprintf("pkg%d/file%d.go::sym%d", (i%sz.nodes)/100, (i%sz.nodes)/10, i%sz.nodes))
|
||||
i++
|
||||
}
|
||||
})
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkGraph_ConcurrentReadWrite(b *testing.B) {
|
||||
for _, sz := range graphSizes {
|
||||
b.Run(sz.name, func(b *testing.B) {
|
||||
b.ReportAllocs()
|
||||
g := buildScaledGraph(sz.nodes, sz.edges)
|
||||
b.ResetTimer()
|
||||
b.RunParallel(func(pb *testing.PB) {
|
||||
i := 0
|
||||
for pb.Next() {
|
||||
if i%10 == 0 {
|
||||
// 10% writes
|
||||
g.AddNode(&Node{
|
||||
ID: fmt.Sprintf("new%d", i+sz.nodes),
|
||||
Kind: KindFunction,
|
||||
Name: fmt.Sprintf("newFunc%d", i),
|
||||
})
|
||||
} else {
|
||||
g.GetNode(fmt.Sprintf("pkg%d/file%d.go::sym%d", (i%sz.nodes)/100, (i%sz.nodes)/10, i%sz.nodes))
|
||||
}
|
||||
i++
|
||||
}
|
||||
})
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// =============================================================================
|
||||
// Memory footprint measurement
|
||||
// =============================================================================
|
||||
|
||||
func BenchmarkGraph_MemoryFootprint(b *testing.B) {
|
||||
for _, sz := range graphSizes {
|
||||
b.Run(sz.name, func(b *testing.B) {
|
||||
b.ReportAllocs()
|
||||
for b.Loop() {
|
||||
b.StopTimer()
|
||||
runtime.GC()
|
||||
var before runtime.MemStats
|
||||
runtime.ReadMemStats(&before)
|
||||
b.StartTimer()
|
||||
|
||||
g := buildScaledGraph(sz.nodes, sz.edges)
|
||||
_ = g.NodeCount() // prevent optimization
|
||||
|
||||
b.StopTimer()
|
||||
runtime.GC()
|
||||
var after runtime.MemStats
|
||||
runtime.ReadMemStats(&after)
|
||||
b.ReportMetric(float64(after.HeapAlloc-before.HeapAlloc), "heap-bytes")
|
||||
b.ReportMetric(float64(after.HeapAlloc-before.HeapAlloc)/float64(sz.nodes), "bytes/node")
|
||||
b.StartTimer()
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// =============================================================================
|
||||
// GC pressure benchmark — important for RPi's limited memory bandwidth
|
||||
// =============================================================================
|
||||
|
||||
func BenchmarkGraph_GCPressure(b *testing.B) {
|
||||
b.ReportAllocs()
|
||||
g := buildScaledGraph(50_000, 150_000)
|
||||
|
||||
// Simulate churn: add and evict files repeatedly
|
||||
b.ResetTimer()
|
||||
for i := range b.N {
|
||||
filePath := fmt.Sprintf("churn/file%d.go", i%100)
|
||||
g.EvictFile(filePath)
|
||||
for j := range 10 {
|
||||
g.AddNode(&Node{
|
||||
ID: fmt.Sprintf("%s::func%d", filePath, j),
|
||||
Kind: KindFunction,
|
||||
Name: fmt.Sprintf("func%d", j),
|
||||
FilePath: filePath,
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// =============================================================================
|
||||
// Lock contention benchmark — simulates RPi's 4-core scenario
|
||||
// =============================================================================
|
||||
|
||||
func BenchmarkGraph_LockContention(b *testing.B) {
|
||||
for _, goroutines := range []int{2, 4, 8} {
|
||||
b.Run(fmt.Sprintf("goroutines_%d", goroutines), func(b *testing.B) {
|
||||
b.ReportAllocs()
|
||||
g := buildScaledGraph(1000, 3000)
|
||||
b.ResetTimer()
|
||||
|
||||
var wg sync.WaitGroup
|
||||
opsPerGoroutine := b.N / goroutines
|
||||
if opsPerGoroutine == 0 {
|
||||
opsPerGoroutine = 1
|
||||
}
|
||||
|
||||
for gr := range goroutines {
|
||||
wg.Add(1)
|
||||
go func(id int) {
|
||||
defer wg.Done()
|
||||
for i := range opsPerGoroutine {
|
||||
switch i % 4 {
|
||||
case 0:
|
||||
g.GetNode(fmt.Sprintf("pkg%d/file%d.go::sym%d", (i%1000)/100, (i%1000)/10, i%1000))
|
||||
case 1:
|
||||
g.AllNodes()
|
||||
case 2:
|
||||
g.GetOutEdges(fmt.Sprintf("pkg%d/file%d.go::sym%d", (i%1000)/100, (i%1000)/10, i%1000))
|
||||
case 3:
|
||||
g.AddNode(&Node{
|
||||
ID: fmt.Sprintf("g%d_n%d", id, i),
|
||||
Kind: KindVariable,
|
||||
Name: fmt.Sprintf("v%d", i),
|
||||
})
|
||||
}
|
||||
}
|
||||
}(gr)
|
||||
}
|
||||
wg.Wait()
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,105 @@
|
||||
package graph
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func BenchmarkGraph_AddNode(b *testing.B) {
|
||||
g := New()
|
||||
for i := range b.N {
|
||||
g.AddNode(&Node{
|
||||
ID: fmt.Sprintf("file%d.go::func%d", i/10, i),
|
||||
Kind: KindFunction,
|
||||
Name: fmt.Sprintf("func%d", i),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkGraph_AddEdge(b *testing.B) {
|
||||
g := New()
|
||||
// Pre-populate nodes.
|
||||
for i := range 1000 {
|
||||
g.AddNode(&Node{
|
||||
ID: fmt.Sprintf("node%d", i),
|
||||
Kind: KindFunction,
|
||||
Name: fmt.Sprintf("func%d", i),
|
||||
})
|
||||
}
|
||||
b.ResetTimer()
|
||||
for i := range b.N {
|
||||
g.AddEdge(&Edge{
|
||||
From: fmt.Sprintf("node%d", i%1000),
|
||||
To: fmt.Sprintf("node%d", (i+1)%1000),
|
||||
Kind: EdgeCalls,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkGraph_GetNode(b *testing.B) {
|
||||
g := New()
|
||||
for i := range 1000 {
|
||||
g.AddNode(&Node{
|
||||
ID: fmt.Sprintf("node%d", i),
|
||||
Kind: KindFunction,
|
||||
Name: fmt.Sprintf("func%d", i),
|
||||
})
|
||||
}
|
||||
b.ResetTimer()
|
||||
for i := range b.N {
|
||||
g.GetNode(fmt.Sprintf("node%d", i%1000))
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkGraph_FindNodesByName(b *testing.B) {
|
||||
g := New()
|
||||
for i := range 1000 {
|
||||
g.AddNode(&Node{
|
||||
ID: fmt.Sprintf("node%d", i),
|
||||
Kind: KindFunction,
|
||||
Name: fmt.Sprintf("func%d", i%50), // 50 unique names
|
||||
})
|
||||
}
|
||||
b.ResetTimer()
|
||||
for i := range b.N {
|
||||
g.FindNodesByName(fmt.Sprintf("func%d", i%50))
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkGraph_AllNodes(b *testing.B) {
|
||||
g := New()
|
||||
for i := range 1000 {
|
||||
g.AddNode(&Node{
|
||||
ID: fmt.Sprintf("node%d", i),
|
||||
Kind: KindFunction,
|
||||
Name: fmt.Sprintf("func%d", i),
|
||||
})
|
||||
}
|
||||
b.ResetTimer()
|
||||
for b.Loop() {
|
||||
g.AllNodes()
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkGraph_Stats(b *testing.B) {
|
||||
g := New()
|
||||
for i := range 500 {
|
||||
g.AddNode(&Node{
|
||||
ID: fmt.Sprintf("node%d", i),
|
||||
Kind: KindFunction,
|
||||
Name: fmt.Sprintf("func%d", i),
|
||||
})
|
||||
}
|
||||
for i := range 500 {
|
||||
g.AddNode(&Node{
|
||||
ID: fmt.Sprintf("type%d", i),
|
||||
Kind: KindType,
|
||||
Name: fmt.Sprintf("Type%d", i),
|
||||
Language: "go",
|
||||
})
|
||||
}
|
||||
b.ResetTimer()
|
||||
for b.Loop() {
|
||||
g.Stats()
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,85 @@
|
||||
package graph
|
||||
|
||||
import (
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// Edge call-site multiplicity.
|
||||
//
|
||||
// The graph natively keys edges by (From, To, Kind, FilePath, Line), so an AST
|
||||
// extractor that emits one edge per call site preserves in-file multiplicity
|
||||
// on its own. A *synthesized* producer that can only mint one edge per
|
||||
// (From, To) — e.g. the LSP references-add pass, which sees N reference sites
|
||||
// for one declaration — would otherwise collapse those N sites to one. Rather
|
||||
// than mint N near-identical edges, such a producer keeps one edge (its
|
||||
// primary site in FilePath/Line) and records the additional sites in
|
||||
// Meta["call_sites"]; find_usages expands them back into one row per site.
|
||||
//
|
||||
// Meta-only: no Edge struct field is added, so no wire-contract / GCX encoder
|
||||
// churn. Edge.Meta round-trips through the store, so the sites survive a warm
|
||||
// restart on the sqlite backend (via AddBatch / PersistEdge).
|
||||
|
||||
// AppendCallSite records an additional call/reference site on an edge whose
|
||||
// primary site stays in FilePath/Line. Extra sites are stored in
|
||||
// Meta["call_sites"] as sorted, deduped "<file>:<line>" strings; the primary
|
||||
// site is never duplicated there.
|
||||
func AppendCallSite(e *Edge, filePath string, line int) {
|
||||
if e == nil || filePath == "" || line <= 0 {
|
||||
return
|
||||
}
|
||||
if filePath == e.FilePath && line == e.Line {
|
||||
return // the primary site lives in FilePath/Line, not call_sites
|
||||
}
|
||||
site := filePath + ":" + strconv.Itoa(line)
|
||||
sites := CallSites(e)
|
||||
for _, s := range sites {
|
||||
if s == site {
|
||||
return
|
||||
}
|
||||
}
|
||||
sites = append(sites, site)
|
||||
sort.Strings(sites)
|
||||
if e.Meta == nil {
|
||||
e.Meta = map[string]any{}
|
||||
}
|
||||
e.Meta["call_sites"] = sites
|
||||
}
|
||||
|
||||
// CallSites returns the extra "<file>:<line>" sites recorded on an edge,
|
||||
// tolerating both the in-memory []string form and the []any form a JSON meta
|
||||
// round-trip (disk backend) produces.
|
||||
func CallSites(e *Edge) []string {
|
||||
if e == nil || e.Meta == nil {
|
||||
return nil
|
||||
}
|
||||
switch v := e.Meta["call_sites"].(type) {
|
||||
case []string:
|
||||
return v
|
||||
case []any:
|
||||
out := make([]string, 0, len(v))
|
||||
for _, x := range v {
|
||||
if s, ok := x.(string); ok {
|
||||
out = append(out, s)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// SplitCallSite splits a "<file>:<line>" call-site string into its file and
|
||||
// 1-based line, returning ("", 0) when malformed. It splits on the last colon
|
||||
// so a path that itself contains a colon is handled.
|
||||
func SplitCallSite(site string) (string, int) {
|
||||
i := strings.LastIndexByte(site, ':')
|
||||
if i <= 0 || i == len(site)-1 {
|
||||
return "", 0
|
||||
}
|
||||
line, err := strconv.Atoi(site[i+1:])
|
||||
if err != nil || line <= 0 {
|
||||
return "", 0
|
||||
}
|
||||
return site[:i], line
|
||||
}
|
||||
@@ -0,0 +1,45 @@
|
||||
package graph
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
)
|
||||
|
||||
func TestAppendCallSite(t *testing.T) {
|
||||
e := &Edge{From: "a", To: "b", Kind: EdgeCalls, FilePath: "f.go", Line: 12}
|
||||
|
||||
// The primary site is never duplicated into call_sites.
|
||||
AppendCallSite(e, "f.go", 12)
|
||||
assert.Empty(t, CallSites(e))
|
||||
|
||||
// Extra sites accumulate, sorted and deduped.
|
||||
AppendCallSite(e, "f.go", 20)
|
||||
AppendCallSite(e, "f.go", 14)
|
||||
AppendCallSite(e, "f.go", 20) // duplicate — ignored
|
||||
assert.Equal(t, []string{"f.go:14", "f.go:20"}, CallSites(e))
|
||||
|
||||
// Malformed inputs are ignored.
|
||||
AppendCallSite(e, "", 5)
|
||||
AppendCallSite(e, "g.go", 0)
|
||||
AppendCallSite(nil, "g.go", 5)
|
||||
assert.Equal(t, []string{"f.go:14", "f.go:20"}, CallSites(e))
|
||||
}
|
||||
|
||||
func TestCallSites_JSONRoundTripForm(t *testing.T) {
|
||||
// A meta round-trip through JSON (disk backend) yields []any, not []string.
|
||||
e := &Edge{Meta: map[string]any{"call_sites": []any{"f.go:3", "f.go:9"}}}
|
||||
assert.Equal(t, []string{"f.go:3", "f.go:9"}, CallSites(e))
|
||||
}
|
||||
|
||||
func TestSplitCallSite(t *testing.T) {
|
||||
f, l := SplitCallSite("dir/f.go:42")
|
||||
assert.Equal(t, "dir/f.go", f)
|
||||
assert.Equal(t, 42, l)
|
||||
|
||||
for _, bad := range []string{"bad", "f.go:", ":42", "f.go:abc", "f.go:0"} {
|
||||
f, l := SplitCallSite(bad)
|
||||
assert.Equal(t, "", f, "malformed %q", bad)
|
||||
assert.Equal(t, 0, l, "malformed %q", bad)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,215 @@
|
||||
package graph
|
||||
|
||||
import "strings"
|
||||
|
||||
// ConcurrencyAnnotation carries two cheap, high-signal concurrency
|
||||
// facts about a symbol — typically a caller surfaced by a navigation
|
||||
// query. Both flags default to false; the *Why fields are populated
|
||||
// only when the matching flag is true, so a zero-value annotation is
|
||||
// safe to attach unconditionally and to omit from a response.
|
||||
type ConcurrencyAnnotation struct {
|
||||
// SyncGuarded is true when the symbol is a method (or a closure
|
||||
// defined on a method-bearing type) whose receiver / parent type
|
||||
// holds a lock — a sync.Mutex / sync.RWMutex field in Go, a Mutex
|
||||
// / RwLock field in Rust, a ReentrantLock field in Java, a
|
||||
// SemaphoreSlim-typed field in C#. Calls made from inside such a
|
||||
// type are presumptively lock-protected, which matters when
|
||||
// reasoning about whether a change is concurrency-safe.
|
||||
SyncGuarded bool `json:"sync_guarded,omitempty"`
|
||||
SyncGuardedWhy string `json:"sync_guarded_why,omitempty"`
|
||||
// CrossConcurrent is true when the symbol is launched across a
|
||||
// concurrency boundary — it is the target of an EdgeSpawns edge,
|
||||
// i.e. a `go` statement / goroutine closure in Go, an async / Promise
|
||||
// / worker entry in JS-TS, a threading.Thread / async def in
|
||||
// Python, a spawned thread in Rust. Any call the symbol makes runs
|
||||
// on a different goroutine / thread than its lexical parent.
|
||||
CrossConcurrent bool `json:"cross_concurrent,omitempty"`
|
||||
CrossConcurrentWhy string `json:"cross_concurrent_why,omitempty"`
|
||||
}
|
||||
|
||||
// Any reports whether either concurrency flag is set. Callers use it
|
||||
// to decide whether an annotation is worth attaching / serialising.
|
||||
func (c ConcurrencyAnnotation) Any() bool {
|
||||
return c.SyncGuarded || c.CrossConcurrent
|
||||
}
|
||||
|
||||
// ClassifyConcurrency derives the concurrency-safety annotation for a
|
||||
// node. It reads through the Reader contract — so it works unchanged
|
||||
// against a base graph or a per-session overlay view — and does no
|
||||
// parser work: it reuses substrate that existing extractors already
|
||||
// emit:
|
||||
//
|
||||
// - EdgeMemberOf links a method / field / closure to its parent
|
||||
// type or enclosing function.
|
||||
// - EdgeSpawns links a caller to a function / closure it launches
|
||||
// asynchronously (goroutine, async, promise, worker pool).
|
||||
// - A field node carries Meta["field_type"] with the verbatim
|
||||
// declared type text.
|
||||
//
|
||||
// Language coverage:
|
||||
//
|
||||
// - sync_guarded relies on typed field nodes. Go, Rust, Java, and C#
|
||||
// emit KindField nodes with Meta["field_type"], so a lock-holding
|
||||
// receiver type is detected for those languages. TypeScript and
|
||||
// PHP model class properties as KindVariable without a typed
|
||||
// field-type, and Python does not materialise instance attributes
|
||||
// as nodes at all — sync_guarded is therefore not reported for
|
||||
// those languages (it stays false rather than guessing).
|
||||
// - cross_concurrent relies only on EdgeSpawns and so covers every
|
||||
// language whose extractor emits spawn edges (Go full; TS / Python
|
||||
// / Rust / Kotlin / C# for the spawn patterns they detect).
|
||||
//
|
||||
// An unknown / missing node yields a zero-value annotation.
|
||||
func ClassifyConcurrency(r Reader, nodeID string) ConcurrencyAnnotation {
|
||||
var ann ConcurrencyAnnotation
|
||||
if r == nil || r.GetNode(nodeID) == nil {
|
||||
return ann
|
||||
}
|
||||
if why := spawnedAsConcurrent(r, nodeID); why != "" {
|
||||
ann.CrossConcurrent = true
|
||||
ann.CrossConcurrentWhy = why
|
||||
}
|
||||
if field, typeName := receiverLockField(r, r.GetNode(nodeID)); field != "" {
|
||||
ann.SyncGuarded = true
|
||||
ann.SyncGuardedWhy = "receiver type " + typeName +
|
||||
" holds a lock (" + field + "); calls from here are presumptively lock-protected"
|
||||
}
|
||||
return ann
|
||||
}
|
||||
|
||||
// spawnedAsConcurrent returns a human-readable explanation when the
|
||||
// node is the target of at least one EdgeSpawns edge, and "" otherwise.
|
||||
// The explanation names the spawn mode (goroutine / async / promise /
|
||||
// worker_pool) recorded on the edge's Meta when available.
|
||||
func spawnedAsConcurrent(r Reader, nodeID string) string {
|
||||
for _, e := range r.GetInEdges(nodeID) {
|
||||
if e.Kind != EdgeSpawns {
|
||||
continue
|
||||
}
|
||||
mode, _ := e.Meta["mode"].(string)
|
||||
switch mode {
|
||||
case "goroutine":
|
||||
return "launched as a goroutine — runs on a different goroutine than its caller"
|
||||
case "async":
|
||||
return "launched as an async task — runs off the caller's synchronous path"
|
||||
case "promise":
|
||||
return "launched inside a promise — runs off the caller's synchronous path"
|
||||
case "worker_pool":
|
||||
return "dispatched to a worker pool — runs on a pool thread, not the caller's"
|
||||
default:
|
||||
return "launched across a concurrency boundary (spawned), not called synchronously"
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// receiverLockField finds the parent / receiver type of a method (or a
|
||||
// closure whose enclosing scope is a method-bearing type) and reports
|
||||
// the first lock-typed field declared on that type. Returns the field
|
||||
// name and the type name, or ("", "") when the node is not a method,
|
||||
// has no resolvable receiver type, or the type holds no lock.
|
||||
func receiverLockField(r Reader, n *Node) (field, typeName string) {
|
||||
if n == nil || (n.Kind != KindMethod && n.Kind != KindClosure) {
|
||||
return "", ""
|
||||
}
|
||||
// A method (or closure) reaches its owner through EdgeMemberOf.
|
||||
// For a closure the owner is usually the enclosing function, not a
|
||||
// type — receiverTypeOf walks one extra hop in that case so a
|
||||
// closure defined inside a method still resolves to the method's
|
||||
// receiver type.
|
||||
typeNode := receiverTypeOf(r, n)
|
||||
if typeNode == nil {
|
||||
return "", ""
|
||||
}
|
||||
// A type's fields point at it via EdgeMemberOf; walk the inbound
|
||||
// member_of edges and inspect each field's declared type.
|
||||
for _, e := range r.GetInEdges(typeNode.ID) {
|
||||
if e.Kind != EdgeMemberOf {
|
||||
continue
|
||||
}
|
||||
fn := r.GetNode(e.From)
|
||||
if fn == nil || fn.Kind != KindField {
|
||||
continue
|
||||
}
|
||||
ft, _ := fn.Meta["field_type"].(string)
|
||||
if isLockTypeName(ft) {
|
||||
return fn.Name, typeNode.Name
|
||||
}
|
||||
}
|
||||
return "", ""
|
||||
}
|
||||
|
||||
// receiverTypeOf resolves the type a method / closure belongs to. A
|
||||
// method points straight at its receiver type via EdgeMemberOf. A
|
||||
// closure points at its enclosing function / method; when that owner
|
||||
// is itself a method the walk takes one more EdgeMemberOf hop so a
|
||||
// closure spawned inside a method is still attributed to the method's
|
||||
// receiver type. Returns nil when no KindType / KindInterface owner is
|
||||
// reachable within those two hops.
|
||||
func receiverTypeOf(r Reader, n *Node) *Node {
|
||||
for _, e := range r.GetOutEdges(n.ID) {
|
||||
if e.Kind != EdgeMemberOf {
|
||||
continue
|
||||
}
|
||||
owner := r.GetNode(e.To)
|
||||
if owner == nil {
|
||||
continue
|
||||
}
|
||||
if owner.Kind == KindType || owner.Kind == KindInterface {
|
||||
return owner
|
||||
}
|
||||
// Closure → enclosing method → receiver type (second hop).
|
||||
if n.Kind == KindClosure && owner.Kind == KindMethod {
|
||||
for _, e2 := range r.GetOutEdges(owner.ID) {
|
||||
if e2.Kind != EdgeMemberOf {
|
||||
continue
|
||||
}
|
||||
t := r.GetNode(e2.To)
|
||||
if t != nil && (t.Kind == KindType || t.Kind == KindInterface) {
|
||||
return t
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// isLockTypeName reports whether a declared field-type string names a
|
||||
// mutual-exclusion primitive. Matching is on the trailing type name so
|
||||
// package / module qualifiers (`sync.Mutex`, `tokio::sync::Mutex`,
|
||||
// `std::sync::RwLock`, `java.util.concurrent.locks.ReentrantLock`) and
|
||||
// a single leading pointer / reference marker do not defeat it.
|
||||
// Recognised across Go, Rust, Java, and C# — the languages whose
|
||||
// extractors emit typed KindField nodes.
|
||||
func isLockTypeName(fieldType string) bool {
|
||||
t := strings.TrimSpace(fieldType)
|
||||
if t == "" {
|
||||
return false
|
||||
}
|
||||
t = strings.TrimPrefix(t, "*") // Go pointer-to-mutex
|
||||
t = strings.TrimPrefix(t, "&") // Rust reference
|
||||
// Drop a generic parameter list — Rust's Mutex<T> / RwLock<T>,
|
||||
// C#'s lock wrappers — so the bare type name is what we test.
|
||||
if i := strings.IndexByte(t, '<'); i >= 0 {
|
||||
t = t[:i]
|
||||
}
|
||||
// Reduce a qualified path to its trailing segment.
|
||||
for _, sep := range []string{"::", "."} {
|
||||
if i := strings.LastIndex(t, sep); i >= 0 {
|
||||
t = t[i+len(sep):]
|
||||
}
|
||||
}
|
||||
switch strings.ToLower(strings.TrimSpace(t)) {
|
||||
case "mutex", "rwmutex", // Go sync.Mutex / sync.RWMutex
|
||||
"rwlock", // Rust std::sync::RwLock
|
||||
"reentrantlock", // Java ReentrantLock
|
||||
"reentrantreadwritelock", // Java ReentrantReadWriteLock
|
||||
"readwritelock", "lock", // Java Lock / ReadWriteLock interfaces
|
||||
"semaphore", // Java / C# Semaphore
|
||||
"semaphoreslim", // C# SemaphoreSlim
|
||||
"readerwriterlockslim", // C# ReaderWriterLockSlim
|
||||
"spinlock": // Rust spin::Mutex-style / C# SpinLock
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
@@ -0,0 +1,239 @@
|
||||
package graph
|
||||
|
||||
import "testing"
|
||||
|
||||
// addMethodNode / addTypeNode / addFieldNode / addEdge are tiny
|
||||
// builders so each test case can pin exactly which nodes and edges
|
||||
// exist. The classifier is graph-only, so a hand-built graph is the
|
||||
// right fixture — no parser in the loop.
|
||||
func addNode(g *Graph, id string, kind NodeKind, meta map[string]any) {
|
||||
g.AddNode(&Node{ID: id, Kind: kind, Name: shortName(id), FilePath: "x.go", Meta: meta})
|
||||
}
|
||||
|
||||
func shortName(id string) string {
|
||||
for i := len(id) - 1; i >= 0; i-- {
|
||||
if id[i] == '.' || id[i] == ':' {
|
||||
return id[i+1:]
|
||||
}
|
||||
}
|
||||
return id
|
||||
}
|
||||
|
||||
func addCEdge(g *Graph, from, to string, kind EdgeKind, meta map[string]any) {
|
||||
g.AddEdge(&Edge{From: from, To: to, Kind: kind, FilePath: "x.go", Origin: OriginASTResolved, Meta: meta})
|
||||
}
|
||||
|
||||
func TestIsLockTypeName(t *testing.T) {
|
||||
cases := []struct {
|
||||
name string
|
||||
in string
|
||||
want bool
|
||||
}{
|
||||
{"go mutex", "sync.Mutex", true},
|
||||
{"go rwmutex", "sync.RWMutex", true},
|
||||
{"go pointer mutex", "*sync.Mutex", true},
|
||||
{"go embedded mutex", "Mutex", true},
|
||||
{"rust mutex generic", "Mutex<State>", true},
|
||||
{"rust qualified rwlock", "std::sync::RwLock<Data>", true},
|
||||
{"rust ref mutex", "&Mutex<u32>", true},
|
||||
{"java reentrant lock", "ReentrantLock", true},
|
||||
{"java qualified rwlock", "java.util.concurrent.locks.ReentrantReadWriteLock", true},
|
||||
{"java lock iface", "Lock", true},
|
||||
{"csharp semaphoreslim", "SemaphoreSlim", true},
|
||||
{"csharp rwlockslim", "ReaderWriterLockSlim", true},
|
||||
{"plain int not a lock", "int", false},
|
||||
{"string not a lock", "string", false},
|
||||
{"map not a lock", "map[string]int", false},
|
||||
{"empty", "", false},
|
||||
{"unrelated struct", "bytes.Buffer", false},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
if got := isLockTypeName(tc.in); got != tc.want {
|
||||
t.Errorf("isLockTypeName(%q) = %v, want %v", tc.in, got, tc.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestClassifyConcurrency_SyncGuarded is the table-driven check that a
|
||||
// method on a mutex-holding type is flagged sync_guarded and a method
|
||||
// on a lock-free type is not.
|
||||
func TestClassifyConcurrency_SyncGuarded(t *testing.T) {
|
||||
cases := []struct {
|
||||
name string
|
||||
fieldType string // declared type of the receiver type's single field
|
||||
fieldKind NodeKind
|
||||
want bool
|
||||
}{
|
||||
{"go mutex field", "sync.Mutex", KindField, true},
|
||||
{"go rwmutex field", "sync.RWMutex", KindField, true},
|
||||
{"go embedded mutex", "Mutex", KindField, true},
|
||||
{"rust mutex field", "Mutex<State>", KindField, true},
|
||||
{"java reentrant lock field", "ReentrantLock", KindField, true},
|
||||
{"lock-free int field", "int", KindField, false},
|
||||
{"lock-free string field", "string", KindField, false},
|
||||
// A class property modelled as KindVariable (TS / PHP) is not
|
||||
// a KindField, so it must not be picked up — sync_guarded is
|
||||
// honestly not reported for those languages.
|
||||
{"variable-modelled property ignored", "Mutex", KindVariable, false},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
g := New()
|
||||
addNode(g, "p.go::Store", KindType, nil)
|
||||
addNode(g, "p.go::Store.Get", KindMethod, nil)
|
||||
addCEdge(g, "p.go::Store.Get", "p.go::Store", EdgeMemberOf, nil)
|
||||
addNode(g, "p.go::Store.f", tc.fieldKind, map[string]any{"field_type": tc.fieldType})
|
||||
addCEdge(g, "p.go::Store.f", "p.go::Store", EdgeMemberOf, nil)
|
||||
|
||||
ann := ClassifyConcurrency(g, "p.go::Store.Get")
|
||||
if ann.SyncGuarded != tc.want {
|
||||
t.Errorf("SyncGuarded = %v, want %v", ann.SyncGuarded, tc.want)
|
||||
}
|
||||
if tc.want && ann.SyncGuardedWhy == "" {
|
||||
t.Error("SyncGuarded true but SyncGuardedWhy is empty")
|
||||
}
|
||||
if !tc.want && ann.SyncGuardedWhy != "" {
|
||||
t.Errorf("SyncGuarded false but SyncGuardedWhy = %q", ann.SyncGuardedWhy)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestClassifyConcurrency_PlainFunctionNeverGuarded confirms a free
|
||||
// function (no receiver type) is never flagged sync_guarded.
|
||||
func TestClassifyConcurrency_PlainFunctionNeverGuarded(t *testing.T) {
|
||||
g := New()
|
||||
addNode(g, "p.go::doWork", KindFunction, nil)
|
||||
ann := ClassifyConcurrency(g, "p.go::doWork")
|
||||
if ann.SyncGuarded {
|
||||
t.Error("a plain function must not be sync_guarded")
|
||||
}
|
||||
}
|
||||
|
||||
// TestClassifyConcurrency_CrossConcurrent checks that a symbol that is
|
||||
// the target of an EdgeSpawns edge is flagged cross_concurrent and a
|
||||
// plainly-called symbol is not.
|
||||
func TestClassifyConcurrency_CrossConcurrent(t *testing.T) {
|
||||
cases := []struct {
|
||||
name string
|
||||
spawnMode string // "" means no spawn edge — plain call
|
||||
spawned bool
|
||||
want bool
|
||||
wantWord string // substring expected in the explanation
|
||||
}{
|
||||
{"goroutine spawn", "goroutine", true, true, "goroutine"},
|
||||
{"async spawn", "async", true, true, "async"},
|
||||
{"promise spawn", "promise", true, true, "promise"},
|
||||
{"worker pool spawn", "worker_pool", true, true, "worker pool"},
|
||||
{"spawn without mode meta", "", true, true, "spawned"},
|
||||
{"plain call only", "", false, false, ""},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
g := New()
|
||||
addNode(g, "p.go::Caller", KindFunction, nil)
|
||||
addNode(g, "p.go::Target", KindFunction, nil)
|
||||
// A plain call edge is always present; it must never on
|
||||
// its own make the target cross_concurrent.
|
||||
addCEdge(g, "p.go::Caller", "p.go::Target", EdgeCalls, nil)
|
||||
if tc.spawned {
|
||||
var meta map[string]any
|
||||
if tc.spawnMode != "" {
|
||||
meta = map[string]any{"mode": tc.spawnMode}
|
||||
}
|
||||
addCEdge(g, "p.go::Caller", "p.go::Target", EdgeSpawns, meta)
|
||||
}
|
||||
ann := ClassifyConcurrency(g, "p.go::Target")
|
||||
if ann.CrossConcurrent != tc.want {
|
||||
t.Errorf("CrossConcurrent = %v, want %v", ann.CrossConcurrent, tc.want)
|
||||
}
|
||||
if tc.want {
|
||||
if ann.CrossConcurrentWhy == "" {
|
||||
t.Error("CrossConcurrent true but explanation empty")
|
||||
}
|
||||
if tc.wantWord != "" && !contains(ann.CrossConcurrentWhy, tc.wantWord) {
|
||||
t.Errorf("explanation %q missing %q", ann.CrossConcurrentWhy, tc.wantWord)
|
||||
}
|
||||
} else if ann.CrossConcurrentWhy != "" {
|
||||
t.Errorf("CrossConcurrent false but explanation = %q", ann.CrossConcurrentWhy)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestClassifyConcurrency_ClosureInheritsReceiverType verifies that a
|
||||
// closure spawned inside a method resolves to the method's receiver
|
||||
// type, so a goroutine-launched closure on a mutex-holding type is
|
||||
// flagged sync_guarded.
|
||||
func TestClassifyConcurrency_ClosureInheritsReceiverType(t *testing.T) {
|
||||
g := New()
|
||||
addNode(g, "p.go::Store", KindType, nil)
|
||||
addNode(g, "p.go::Store.mu", KindField, map[string]any{"field_type": "sync.Mutex"})
|
||||
addCEdge(g, "p.go::Store.mu", "p.go::Store", EdgeMemberOf, nil)
|
||||
addNode(g, "p.go::Store.Run", KindMethod, nil)
|
||||
addCEdge(g, "p.go::Store.Run", "p.go::Store", EdgeMemberOf, nil)
|
||||
// Closure defined inside the method; member_of points at the method.
|
||||
addNode(g, "p.go::Store.Run#closure@10", KindClosure, nil)
|
||||
addCEdge(g, "p.go::Store.Run#closure@10", "p.go::Store.Run", EdgeMemberOf, nil)
|
||||
// The method launches the closure as a goroutine.
|
||||
addCEdge(g, "p.go::Store.Run", "p.go::Store.Run#closure@10", EdgeSpawns,
|
||||
map[string]any{"mode": "goroutine"})
|
||||
|
||||
ann := ClassifyConcurrency(g, "p.go::Store.Run#closure@10")
|
||||
if !ann.SyncGuarded {
|
||||
t.Error("closure on a mutex-holding receiver type must be sync_guarded")
|
||||
}
|
||||
if !ann.CrossConcurrent {
|
||||
t.Error("goroutine-launched closure must be cross_concurrent")
|
||||
}
|
||||
}
|
||||
|
||||
// TestClassifyConcurrency_BothFlags confirms the two flags are
|
||||
// independent — a goroutine-launched method on a mutex type carries
|
||||
// both.
|
||||
func TestClassifyConcurrency_BothFlags(t *testing.T) {
|
||||
g := New()
|
||||
addNode(g, "p.go::Cache", KindType, nil)
|
||||
addNode(g, "p.go::Cache.lock", KindField, map[string]any{"field_type": "sync.RWMutex"})
|
||||
addCEdge(g, "p.go::Cache.lock", "p.go::Cache", EdgeMemberOf, nil)
|
||||
addNode(g, "p.go::Cache.refresh", KindMethod, nil)
|
||||
addCEdge(g, "p.go::Cache.refresh", "p.go::Cache", EdgeMemberOf, nil)
|
||||
addNode(g, "p.go::Manager.Start", KindMethod, nil)
|
||||
addCEdge(g, "p.go::Manager.Start", "p.go::Cache.refresh", EdgeSpawns,
|
||||
map[string]any{"mode": "goroutine"})
|
||||
|
||||
ann := ClassifyConcurrency(g, "p.go::Cache.refresh")
|
||||
if !ann.SyncGuarded || !ann.CrossConcurrent {
|
||||
t.Errorf("expected both flags set, got sync_guarded=%v cross_concurrent=%v",
|
||||
ann.SyncGuarded, ann.CrossConcurrent)
|
||||
}
|
||||
if !ann.Any() {
|
||||
t.Error("Any() must be true when flags are set")
|
||||
}
|
||||
}
|
||||
|
||||
// TestClassifyConcurrency_MissingNode returns a zero-value annotation
|
||||
// for an unknown node, and a nil reader is also tolerated.
|
||||
func TestClassifyConcurrency_MissingNode(t *testing.T) {
|
||||
g := New()
|
||||
if ann := ClassifyConcurrency(g, "p.go::nope"); ann.Any() {
|
||||
t.Error("unknown node must yield a zero-value annotation")
|
||||
}
|
||||
if ann := ClassifyConcurrency(nil, "p.go::nope"); ann.Any() {
|
||||
t.Error("nil reader must yield a zero-value annotation")
|
||||
}
|
||||
}
|
||||
|
||||
func contains(haystack, needle string) bool {
|
||||
if needle == "" {
|
||||
return true
|
||||
}
|
||||
for i := 0; i+len(needle) <= len(haystack); i++ {
|
||||
if haystack[i:i+len(needle)] == needle {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
@@ -0,0 +1,66 @@
|
||||
package graph
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"sync"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// TestReindexEdge_ConcurrentAddEdge reproduces the "concurrent map read
|
||||
// and map write" crash seen in production when two MCP handlers both
|
||||
// trigger ensureFresh → indexFile: one calls AddEdge on shard From, the
|
||||
// other calls ReindexEdge which mutates From's outEdgeIdx without
|
||||
// locking From. Run with `-race` for the sharpest detection; even
|
||||
// without -race the bare runtime map guard panics reliably here.
|
||||
func TestReindexEdge_ConcurrentAddEdge(t *testing.T) {
|
||||
g := New()
|
||||
|
||||
const n = 200
|
||||
for i := range n {
|
||||
g.AddNode(&Node{ID: fmt.Sprintf("from%d::F", i), Name: "F", Kind: KindFunction, FilePath: "f"})
|
||||
g.AddNode(&Node{ID: fmt.Sprintf("to%d::T", i), Name: "T", Kind: KindFunction, FilePath: "t"})
|
||||
g.AddNode(&Node{ID: fmt.Sprintf("alt%d::A", i), Name: "A", Kind: KindFunction, FilePath: "a"})
|
||||
}
|
||||
|
||||
var wg sync.WaitGroup
|
||||
wg.Add(2)
|
||||
|
||||
// Writer A: AddEdge against the From shards.
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
for round := range 50 {
|
||||
for i := range n {
|
||||
g.AddEdge(&Edge{
|
||||
From: fmt.Sprintf("from%d::F", i),
|
||||
To: fmt.Sprintf("to%d::T", i),
|
||||
Kind: EdgeCalls,
|
||||
FilePath: "f",
|
||||
Line: round,
|
||||
})
|
||||
}
|
||||
}
|
||||
}()
|
||||
|
||||
// Writer B: ReindexEdge, retargeting onto a different shard each
|
||||
// round — this is the resolver path that would collide with A.
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
for round := range 50 {
|
||||
for i := range n {
|
||||
e := &Edge{
|
||||
From: fmt.Sprintf("from%d::F", i),
|
||||
To: fmt.Sprintf("to%d::T", i),
|
||||
Kind: EdgeCalls,
|
||||
FilePath: "f",
|
||||
Line: round + 1000,
|
||||
}
|
||||
g.AddEdge(e)
|
||||
oldTo := e.To
|
||||
e.To = fmt.Sprintf("alt%d::A", i)
|
||||
g.ReindexEdge(e, oldTo)
|
||||
}
|
||||
}
|
||||
}()
|
||||
|
||||
wg.Wait()
|
||||
}
|
||||
@@ -0,0 +1,52 @@
|
||||
package graph
|
||||
|
||||
// IsContentNode reports whether n is a CONTENT section node — a KindDoc
|
||||
// chunk tagged data_class="content" (text / pdf / pptx / xlsx section
|
||||
// bodies). Content bodies are indexed in the dedicated content store
|
||||
// (ContentSearcher), never the symbol search, and are excluded from the
|
||||
// code-oriented analysis passes — so this predicate is the single place
|
||||
// every package agrees on what "content" means. Markdown prose (KindDoc
|
||||
// without data_class=content) and data assets (data_class="data") are NOT
|
||||
// content and keep their existing treatment.
|
||||
func IsContentNode(n *Node) bool {
|
||||
if n == nil || n.Kind != KindDoc || n.Meta == nil {
|
||||
return false
|
||||
}
|
||||
dc, _ := n.Meta["data_class"].(string)
|
||||
return dc == "content"
|
||||
}
|
||||
|
||||
// NonContentNodeReader is an optional store capability: a cheap (SQL-level
|
||||
// on the disk backend) enumeration of a repo's NON-content nodes, so the
|
||||
// code-oriented passes (search-index build, embedding, language detection)
|
||||
// never materialise a content-heavy repo's hundreds of thousands of content
|
||||
// sections just to iterate past them.
|
||||
type NonContentNodeReader interface {
|
||||
GetRepoNonContentNodes(repoPrefix string) []*Node
|
||||
}
|
||||
|
||||
// RepoCodeNodes returns repoPrefix's non-content nodes. It uses the store's
|
||||
// NonContentNodeReader fast path when available (the disk backend filters in
|
||||
// SQL, so 525k content sections never enter memory); otherwise it falls back
|
||||
// to materialising the repo's nodes and dropping content in Go — fine for the
|
||||
// in-memory store, which only backs small repos. An empty repoPrefix means
|
||||
// "all repos".
|
||||
func RepoCodeNodes(s Store, repoPrefix string) []*Node {
|
||||
if r, ok := s.(NonContentNodeReader); ok {
|
||||
return r.GetRepoNonContentNodes(repoPrefix)
|
||||
}
|
||||
var nodes []*Node
|
||||
if repoPrefix != "" {
|
||||
nodes = s.GetRepoNodes(repoPrefix)
|
||||
}
|
||||
if len(nodes) == 0 {
|
||||
nodes = s.AllNodes()
|
||||
}
|
||||
out := make([]*Node, 0, len(nodes))
|
||||
for _, n := range nodes {
|
||||
if !IsContentNode(n) {
|
||||
out = append(out, n)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
@@ -0,0 +1,124 @@
|
||||
package graph
|
||||
|
||||
import "strings"
|
||||
|
||||
// NormalizeCppType reduces a C++ type spelling to a stable comparison key used
|
||||
// by both the extractor (stamping parameter types) and the overload resolver
|
||||
// (normalizing call-site argument hints), so the two always compare in the same
|
||||
// space. It strips template arguments, cv-qualifiers, ref/ptr punctuation, and
|
||||
// namespace qualifiers, and canonicalises a few stdlib aliases — while keeping
|
||||
// the integer/float ladder distinct (int vs long) so genuinely different
|
||||
// overloads stay rankable.
|
||||
func NormalizeCppType(raw string) string {
|
||||
// Reduce a smart-pointer / optional wrapper to its pointee before stripping
|
||||
// template arguments — `unique_ptr<Widget>` normalises to `Widget`, not
|
||||
// `unique_ptr` — so member access through the wrapper lands on the pointee.
|
||||
raw = UnwrapCppSmartPointer(raw)
|
||||
s := stripCppTemplateArgs(raw)
|
||||
s = strings.ReplaceAll(s, "&&", " ")
|
||||
s = strings.ReplaceAll(s, "&", " ")
|
||||
s = strings.ReplaceAll(s, "*", " ")
|
||||
fields := strings.Fields(s)
|
||||
kept := fields[:0]
|
||||
for _, f := range fields {
|
||||
if f == "const" || f == "volatile" {
|
||||
continue
|
||||
}
|
||||
kept = append(kept, f)
|
||||
}
|
||||
s = strings.Join(kept, " ")
|
||||
if i := strings.LastIndex(s, "::"); i >= 0 {
|
||||
s = s[i+2:]
|
||||
}
|
||||
s = strings.TrimSpace(s)
|
||||
switch s {
|
||||
case "string", "basic_string", "string_view":
|
||||
return "string"
|
||||
case "unsigned", "unsigned int", "uint", "size_t", "uint32_t", "uint64_t":
|
||||
return "unsigned"
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
// cppSmartPointerWrappers names the single-pointee standard wrappers whose
|
||||
// `<T>` is the type a member access (`p->m()`) or dereference (`*p`) actually
|
||||
// reaches — each forwards operator-> / operator* (or, for optional, value
|
||||
// access) to its pointee.
|
||||
var cppSmartPointerWrappers = map[string]bool{
|
||||
"unique_ptr": true,
|
||||
"shared_ptr": true,
|
||||
"weak_ptr": true,
|
||||
"optional": true,
|
||||
}
|
||||
|
||||
// UnwrapCppSmartPointer reduces a smart-pointer / optional wrapper to its
|
||||
// pointee type text — `std::unique_ptr<Widget>` → `Widget`,
|
||||
// `optional<shared_ptr<const Foo>>` → `const Foo`, `unique_ptr<Widget, Del>` →
|
||||
// `Widget` — so a member access through the wrapper resolves on the pointee
|
||||
// rather than the wrapper type. A non-wrapper type, including other generics
|
||||
// like `vector<T>`, is returned unchanged. const / volatile qualifiers and
|
||||
// ref / pointer suffixes on the wrapper are tolerated; nested wrappers are
|
||||
// peeled until a non-wrapper type remains.
|
||||
func UnwrapCppSmartPointer(s string) string {
|
||||
for {
|
||||
t := strings.TrimSpace(s)
|
||||
t = strings.TrimPrefix(t, "const ")
|
||||
t = strings.TrimPrefix(t, "volatile ")
|
||||
t = strings.TrimRight(strings.TrimSpace(t), " &*")
|
||||
t = strings.TrimSpace(t)
|
||||
lt := strings.IndexByte(t, '<')
|
||||
if lt <= 0 || !strings.HasSuffix(t, ">") {
|
||||
return s
|
||||
}
|
||||
head := strings.TrimSpace(t[:lt])
|
||||
if i := strings.LastIndex(head, "::"); i >= 0 {
|
||||
head = head[i+2:]
|
||||
}
|
||||
if !cppSmartPointerWrappers[head] {
|
||||
return s
|
||||
}
|
||||
s = cppFirstTemplateArg(strings.TrimSpace(t[lt+1 : len(t)-1]))
|
||||
}
|
||||
}
|
||||
|
||||
// cppFirstTemplateArg returns the first comma-separated template argument,
|
||||
// respecting nested `<…>` so `Widget, Deleter` yields `Widget` and
|
||||
// `map<int, Foo>` stays whole.
|
||||
func cppFirstTemplateArg(args string) string {
|
||||
depth := 0
|
||||
for i := 0; i < len(args); i++ {
|
||||
switch args[i] {
|
||||
case '<':
|
||||
depth++
|
||||
case '>':
|
||||
if depth > 0 {
|
||||
depth--
|
||||
}
|
||||
case ',':
|
||||
if depth == 0 {
|
||||
return strings.TrimSpace(args[:i])
|
||||
}
|
||||
}
|
||||
}
|
||||
return strings.TrimSpace(args)
|
||||
}
|
||||
|
||||
func stripCppTemplateArgs(s string) string {
|
||||
depth := 0
|
||||
var b strings.Builder
|
||||
for _, r := range s {
|
||||
switch r {
|
||||
case '<':
|
||||
depth++
|
||||
case '>':
|
||||
if depth > 0 {
|
||||
depth--
|
||||
}
|
||||
default:
|
||||
if depth == 0 {
|
||||
b.WriteRune(r)
|
||||
}
|
||||
}
|
||||
}
|
||||
return b.String()
|
||||
}
|
||||
@@ -0,0 +1,40 @@
|
||||
package graph
|
||||
|
||||
import "testing"
|
||||
|
||||
func TestUnwrapCppSmartPointer(t *testing.T) {
|
||||
cases := map[string]string{
|
||||
"std::unique_ptr<Widget>": "Widget",
|
||||
"unique_ptr<Widget>": "Widget",
|
||||
"std::shared_ptr<Widget>": "Widget",
|
||||
"std::weak_ptr<Widget>": "Widget",
|
||||
"std::optional<Widget>": "Widget",
|
||||
"const std::shared_ptr<Widget> &": "Widget",
|
||||
"std::unique_ptr<Widget, MyDeleter>": "Widget", // extra deleter arg dropped
|
||||
"std::optional<std::shared_ptr<Foo>>": "Foo", // nested wrappers peeled
|
||||
"std::unique_ptr<ns::Widget>": "ns::Widget", // pointee namespace kept
|
||||
"std::vector<Widget>": "std::vector<Widget>", // non-wrapper unchanged
|
||||
"Widget": "Widget",
|
||||
"int": "int",
|
||||
}
|
||||
for in, want := range cases {
|
||||
if got := UnwrapCppSmartPointer(in); got != want {
|
||||
t.Errorf("UnwrapCppSmartPointer(%q) = %q, want %q", in, got, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestNormalizeCppType_SmartPointerPointee(t *testing.T) {
|
||||
cases := map[string]string{
|
||||
"std::unique_ptr<Widget>": "Widget",
|
||||
"shared_ptr<Widget>": "Widget",
|
||||
"std::optional<ns::Widget>": "Widget", // last :: segment, post-unwrap
|
||||
"std::unique_ptr<std::vector<Widget>>": "vector", // unwrap to vector<Widget>, then strip args
|
||||
"std::vector<Widget>": "vector", // non-wrapper still strips to container
|
||||
}
|
||||
for in, want := range cases {
|
||||
if got := NormalizeCppType(in); got != want {
|
||||
t.Errorf("NormalizeCppType(%q) = %q, want %q", in, got, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,168 @@
|
||||
package graph
|
||||
|
||||
import "testing"
|
||||
|
||||
func TestOriginRank_Ordering(t *testing.T) {
|
||||
// The ordering expresses the design contract: LSP-verified evidence
|
||||
// outranks AST-extracted evidence, which outranks name-only matches.
|
||||
order := []string{
|
||||
OriginLSPResolved,
|
||||
OriginLSPDispatch,
|
||||
OriginASTResolved,
|
||||
OriginASTInferred,
|
||||
OriginTextMatched,
|
||||
}
|
||||
for i := 0; i < len(order)-1; i++ {
|
||||
if OriginRank(order[i]) <= OriginRank(order[i+1]) {
|
||||
t.Errorf("expected rank(%s)=%d > rank(%s)=%d",
|
||||
order[i], OriginRank(order[i]),
|
||||
order[i+1], OriginRank(order[i+1]))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestOriginRank_UnknownReturnsZero(t *testing.T) {
|
||||
if got := OriginRank(""); got != 0 {
|
||||
t.Errorf("empty origin: got rank %d, want 0", got)
|
||||
}
|
||||
if got := OriginRank("bogus_value"); got != 0 {
|
||||
t.Errorf("unknown origin: got rank %d, want 0", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMeetsMinTier(t *testing.T) {
|
||||
tests := []struct {
|
||||
origin, minTier string
|
||||
want bool
|
||||
}{
|
||||
{OriginLSPResolved, OriginLSPResolved, true},
|
||||
{OriginLSPResolved, OriginTextMatched, true},
|
||||
{OriginASTResolved, OriginLSPResolved, false},
|
||||
{OriginTextMatched, OriginASTInferred, false},
|
||||
{OriginLSPDispatch, OriginASTResolved, true},
|
||||
{"", OriginLSPResolved, false},
|
||||
{OriginLSPResolved, "", true}, // no filter = all pass
|
||||
{"", "", true}, // no filter, no origin = pass
|
||||
}
|
||||
for _, tt := range tests {
|
||||
got := MeetsMinTier(tt.origin, tt.minTier)
|
||||
if got != tt.want {
|
||||
t.Errorf("MeetsMinTier(%q, %q) = %v, want %v",
|
||||
tt.origin, tt.minTier, got, tt.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestDefaultOriginFor_SemanticSource(t *testing.T) {
|
||||
// Any non-empty semantic source means a compiler-grade provider
|
||||
// confirmed the edge — should map to LSP-resolved.
|
||||
got := DefaultOriginFor(EdgeCalls, 1.0, "go-types")
|
||||
if got != OriginLSPResolved {
|
||||
t.Errorf("got %q, want %q", got, OriginLSPResolved)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDefaultOriginFor_ImplementsWithSource(t *testing.T) {
|
||||
// Interface → implementation via semantic provider = LSP dispatch.
|
||||
got := DefaultOriginFor(EdgeImplements, 1.0, "lsp")
|
||||
if got != OriginLSPDispatch {
|
||||
t.Errorf("got %q, want %q", got, OriginLSPDispatch)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDefaultOriginFor_StructuralAST(t *testing.T) {
|
||||
for _, kind := range []EdgeKind{
|
||||
EdgeDefines, EdgeImports, EdgeExtends, EdgeMemberOf,
|
||||
EdgeImplements, EdgeProvides, EdgeConsumes,
|
||||
} {
|
||||
got := DefaultOriginFor(kind, 0, "")
|
||||
if got != OriginASTResolved {
|
||||
t.Errorf("kind=%s: got %q, want %q", kind, got, OriginASTResolved)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestDefaultOriginFor_ConfidenceBuckets(t *testing.T) {
|
||||
tests := []struct {
|
||||
conf float64
|
||||
want string
|
||||
}{
|
||||
{1.0, OriginASTResolved},
|
||||
{0.95, OriginASTResolved},
|
||||
{0.7, OriginASTInferred},
|
||||
{0.5, OriginASTInferred},
|
||||
{0.3, OriginTextMatched},
|
||||
{0, OriginTextMatched},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
got := DefaultOriginFor(EdgeCalls, tt.conf, "")
|
||||
if got != tt.want {
|
||||
t.Errorf("confidence=%v: got %q, want %q", tt.conf, got, tt.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestEdgeIdentityHash_StableForFixedOrigin proves IdentityHash is a
|
||||
// pure function of (From, To, Kind, FilePath, Line, Origin): two
|
||||
// separately-constructed edges with identical fields hash equal.
|
||||
func TestEdgeIdentityHash_StableForFixedOrigin(t *testing.T) {
|
||||
a := &Edge{From: "p/a.go::A", To: "p/b.go::B", Kind: EdgeCalls, FilePath: "p/a.go", Line: 12, Origin: OriginASTResolved}
|
||||
b := &Edge{From: "p/a.go::A", To: "p/b.go::B", Kind: EdgeCalls, FilePath: "p/a.go", Line: 12, Origin: OriginASTResolved}
|
||||
|
||||
if a.IdentityHash() != b.IdentityHash() {
|
||||
t.Fatal("edges with identical fields must share an identity hash")
|
||||
}
|
||||
}
|
||||
|
||||
// TestEdgeIdentityHash_DiffersOnlyByOrigin proves the deliverable: the
|
||||
// identity hash changes iff Origin changes when every other field is
|
||||
// held fixed. Each distinct tier yields a distinct identity, and the
|
||||
// logical key alone (Origin-free) is NOT enough to pin the identity.
|
||||
func TestEdgeIdentityHash_DiffersOnlyByOrigin(t *testing.T) {
|
||||
base := func(origin string) *Edge {
|
||||
return &Edge{From: "p/a.go::A", To: "p/b.go::B", Kind: EdgeCalls, FilePath: "p/a.go", Line: 12, Origin: origin}
|
||||
}
|
||||
origins := []string{
|
||||
OriginLSPResolved, OriginLSPDispatch, OriginASTResolved,
|
||||
OriginASTInferred, OriginTextMatched, "",
|
||||
}
|
||||
seen := make(map[edgeHash]string, len(origins))
|
||||
for _, o := range origins {
|
||||
h := base(o).IdentityHash()
|
||||
if prev, dup := seen[h]; dup {
|
||||
t.Fatalf("origins %q and %q collided to the same identity hash", prev, o)
|
||||
}
|
||||
seen[h] = o
|
||||
}
|
||||
|
||||
// The Origin-free logical key must NOT determine the identity:
|
||||
// two edges with the same keyOf but different Origin differ.
|
||||
lsp := base(OriginLSPResolved)
|
||||
txt := base(OriginTextMatched)
|
||||
if keyOf(lsp) != keyOf(txt) {
|
||||
t.Fatal("test setup: edges should share the logical key")
|
||||
}
|
||||
if lsp.IdentityHash() == txt.IdentityHash() {
|
||||
t.Fatal("identity hash must include Origin — same logical key, different Origin must differ")
|
||||
}
|
||||
}
|
||||
|
||||
// TestEdgeIdentityHash_DiffersOnLogicalFields confirms IdentityHash
|
||||
// still discriminates on the logical-key fields, so it is a strict
|
||||
// superset of edgeKey's discrimination, not a replacement that only
|
||||
// looks at Origin.
|
||||
func TestEdgeIdentityHash_DiffersOnLogicalFields(t *testing.T) {
|
||||
ref := &Edge{From: "p/a.go::A", To: "p/b.go::B", Kind: EdgeCalls, FilePath: "p/a.go", Line: 12, Origin: OriginASTResolved}
|
||||
variants := []*Edge{
|
||||
{From: "p/a.go::X", To: "p/b.go::B", Kind: EdgeCalls, FilePath: "p/a.go", Line: 12, Origin: OriginASTResolved},
|
||||
{From: "p/a.go::A", To: "p/b.go::X", Kind: EdgeCalls, FilePath: "p/a.go", Line: 12, Origin: OriginASTResolved},
|
||||
{From: "p/a.go::A", To: "p/b.go::B", Kind: EdgeReferences, FilePath: "p/a.go", Line: 12, Origin: OriginASTResolved},
|
||||
{From: "p/a.go::A", To: "p/b.go::B", Kind: EdgeCalls, FilePath: "p/c.go", Line: 12, Origin: OriginASTResolved},
|
||||
{From: "p/a.go::A", To: "p/b.go::B", Kind: EdgeCalls, FilePath: "p/a.go", Line: 99, Origin: OriginASTResolved},
|
||||
}
|
||||
for i, v := range variants {
|
||||
if ref.IdentityHash() == v.IdentityHash() {
|
||||
t.Errorf("variant %d: identity hash must differ when a logical-key field differs", i)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,29 @@
|
||||
package graph
|
||||
|
||||
import "testing"
|
||||
|
||||
func TestViaLabelFor(t *testing.T) {
|
||||
cases := map[string]string{
|
||||
"swift.objc.bridge": "Swift↔ObjC bridge",
|
||||
"observer.channel": "observer channel",
|
||||
"closure.collection": "closure collection",
|
||||
"react.setstate": "React setState",
|
||||
"flutter.setstate": "Flutter setState",
|
||||
"kmp.expect-actual": "KMP expect/actual",
|
||||
"": "",
|
||||
"unknown.synth": "unknown.synth", // unmapped passes through
|
||||
}
|
||||
for in, want := range cases {
|
||||
if got := ViaLabelFor(in); got != want {
|
||||
t.Errorf("ViaLabelFor(%q) = %q, want %q", in, got, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestEdgeIdentityHashIgnoresVia(t *testing.T) {
|
||||
base := &Edge{From: "a", To: "b", Kind: EdgeCalls, Origin: OriginASTInferred}
|
||||
withVia := &Edge{From: "a", To: "b", Kind: EdgeCalls, Origin: OriginASTInferred, Via: "observer channel"}
|
||||
if base.IdentityHash() != withVia.IdentityHash() {
|
||||
t.Errorf("Via must not change the edge identity hash")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,112 @@
|
||||
package graph
|
||||
|
||||
import "testing"
|
||||
|
||||
func TestEnclosingFromID(t *testing.T) {
|
||||
cases := []struct {
|
||||
name string
|
||||
id string
|
||||
kind NodeKind
|
||||
wantID string
|
||||
wantName string
|
||||
}{
|
||||
{
|
||||
name: "method on type",
|
||||
id: "pkg/decoder.go::Decoder.Parse",
|
||||
kind: KindMethod,
|
||||
wantID: "pkg/decoder.go::Decoder",
|
||||
wantName: "Decoder",
|
||||
},
|
||||
{
|
||||
name: "field of struct",
|
||||
id: "pkg/config.go::Config.Timeout",
|
||||
kind: KindField,
|
||||
wantID: "pkg/config.go::Config",
|
||||
wantName: "Config",
|
||||
},
|
||||
{
|
||||
name: "enum member",
|
||||
id: "pkg/kind.go::NodeKind.Function",
|
||||
kind: KindEnumMember,
|
||||
wantID: "pkg/kind.go::NodeKind",
|
||||
wantName: "NodeKind",
|
||||
},
|
||||
{
|
||||
name: "closure inside function",
|
||||
id: "pkg/run.go::Execute#closure@42",
|
||||
kind: KindClosure,
|
||||
wantID: "pkg/run.go::Execute",
|
||||
wantName: "Execute",
|
||||
},
|
||||
{
|
||||
name: "nested type method",
|
||||
id: "pkg/x.go::Outer.Inner.Do",
|
||||
kind: KindMethod,
|
||||
wantID: "pkg/x.go::Outer.Inner",
|
||||
wantName: "Inner",
|
||||
},
|
||||
{
|
||||
name: "top-level function -- no owner",
|
||||
id: "pkg/x.go::TopLevel",
|
||||
kind: KindFunction,
|
||||
wantID: "",
|
||||
wantName: "",
|
||||
},
|
||||
{
|
||||
name: "plain method ID without owner segment",
|
||||
id: "pkg/x.go::Bare",
|
||||
kind: KindMethod,
|
||||
wantID: "",
|
||||
wantName: "",
|
||||
},
|
||||
{
|
||||
name: "type is never enclosed",
|
||||
id: "pkg/x.go::SomeType",
|
||||
kind: KindType,
|
||||
wantID: "",
|
||||
wantName: "",
|
||||
},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
gotID, gotName := EnclosingFromID(tc.id, tc.kind)
|
||||
if gotID != tc.wantID || gotName != tc.wantName {
|
||||
t.Fatalf("EnclosingFromID(%q, %v) = (%q, %q), want (%q, %q)",
|
||||
tc.id, tc.kind, gotID, gotName, tc.wantID, tc.wantName)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestNodeBrief_Enclosing(t *testing.T) {
|
||||
// A method node surfaces its receiver type in Brief.
|
||||
method := &Node{ID: "pkg/d.go::Decoder.Parse", Kind: KindMethod, Name: "Parse", FilePath: "pkg/d.go"}
|
||||
b := method.Brief()
|
||||
if b["enclosing"] != "Decoder" {
|
||||
t.Errorf("method Brief enclosing = %v, want Decoder", b["enclosing"])
|
||||
}
|
||||
if b["enclosing_id"] != "pkg/d.go::Decoder" {
|
||||
t.Errorf("method Brief enclosing_id = %v, want pkg/d.go::Decoder", b["enclosing_id"])
|
||||
}
|
||||
|
||||
// A top-level function has no enclosing key at all.
|
||||
fn := &Node{ID: "pkg/d.go::TopLevel", Kind: KindFunction, Name: "TopLevel", FilePath: "pkg/d.go"}
|
||||
fb := fn.Brief()
|
||||
if _, ok := fb["enclosing"]; ok {
|
||||
t.Error("top-level function Brief should carry no enclosing key")
|
||||
}
|
||||
}
|
||||
|
||||
func TestEnclosingShortName(t *testing.T) {
|
||||
cases := map[string]string{
|
||||
"pkg/x.go::Owner": "Owner",
|
||||
"Outer.Inner": "Inner",
|
||||
"Plain": "Plain",
|
||||
"pkg::A.B.C": "C",
|
||||
}
|
||||
for in, want := range cases {
|
||||
if got := EnclosingShortName(in); got != want {
|
||||
t.Errorf("EnclosingShortName(%q) = %q, want %q", in, got, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,201 @@
|
||||
package graph
|
||||
|
||||
import (
|
||||
"sort"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// EpistemicBoundary names one unresolved/dynamic-dispatch site that makes a
|
||||
// traversal count a *floor* rather than an exact number. It is the honest
|
||||
// answer to "could the real blast radius / reachable set be larger?" — yes,
|
||||
// because the resolver could not bind this site, so an unknown number of
|
||||
// callers/callees hide behind it.
|
||||
//
|
||||
// It is an attribute of a *traversal*, not a graph element: no new NodeKind or
|
||||
// EdgeKind is introduced. Boundaries are recorded at exactly the sites a walk
|
||||
// would otherwise silently drop (an out-edge to an `unresolved::*` target) or
|
||||
// where dynamic dispatch is structurally possible (the seed implements an
|
||||
// interface, so callers may invoke it through that interface).
|
||||
type EpistemicBoundary struct {
|
||||
SeedID string `json:"seed_id"`
|
||||
SeedName string `json:"seed_name,omitempty"`
|
||||
Target string `json:"target,omitempty"`
|
||||
EdgeKind string `json:"edge_kind,omitempty"`
|
||||
Reason BoundaryReason `json:"reason"`
|
||||
Direction string `json:"direction"` // "callers" | "callees"
|
||||
}
|
||||
|
||||
// DynamicBoundary enriches an EpistemicBoundary with the body-level detail an
|
||||
// agent needs to cross a runtime-dispatch site without a read-spiral: the SITE
|
||||
// (file:line), the dispatch FORM, the KEY selector, and a candidate-target
|
||||
// shortlist. Like EpistemicBoundary it is an attribute of a query result, not a
|
||||
// graph element — it is computed on demand by scanning the disconnected
|
||||
// symbol's body, never persisted.
|
||||
type DynamicBoundary struct {
|
||||
Site string `json:"site"` // file:line of the dispatch
|
||||
Form string `json:"form"` // reflection | computed_member | event_bus
|
||||
Key string `json:"key"` // the selector expression / event name
|
||||
Candidates []string `json:"candidates,omitempty"`
|
||||
AgentNamed bool `json:"agent_named,omitempty"`
|
||||
}
|
||||
|
||||
// BoundaryReason classifies why a boundary makes the count a floor. The
|
||||
// vocabulary aligns with the resolution-outcomes taxonomy's dynamic-dispatch
|
||||
// concept while staying graph-local (no name-resolution needed to compute it).
|
||||
type BoundaryReason string
|
||||
|
||||
const (
|
||||
// BoundaryDynamicDispatch: an out call/reference edge whose target the
|
||||
// resolver left as `unresolved::*` — the callee set could be larger.
|
||||
BoundaryDynamicDispatch BoundaryReason = "dynamic_dispatch"
|
||||
// BoundaryInterfaceDispatch: the node implements/overrides an interface
|
||||
// method, so callers may invoke it through the interface via dispatch that
|
||||
// is not attributed to this node — the caller set could be larger.
|
||||
BoundaryInterfaceDispatch BoundaryReason = "interface_dispatch"
|
||||
// BoundaryExternal: an edge into the `external::` namespace — the chain
|
||||
// leaves the indexed code. Listed for transparency; not floor-making.
|
||||
BoundaryExternal BoundaryReason = "external_boundary"
|
||||
// BoundaryStub: an edge into a stdlib/builtin/module stub. Listed; not
|
||||
// floor-making (an external stdlib call adds no in-repo callers/callees).
|
||||
BoundaryStub BoundaryReason = "stub"
|
||||
)
|
||||
|
||||
// maxBoundaries caps the per-result boundary list so a pathological hub cannot
|
||||
// bloat a response. Mirrors impact.go's maxPerTier.
|
||||
const maxBoundaries = 50
|
||||
|
||||
// ClassifyDroppedTarget classifies an edge target a traversal could not follow.
|
||||
// ok=false means it is an ordinary in-graph node (follow it normally).
|
||||
func ClassifyDroppedTarget(targetID string, kind EdgeKind) (BoundaryReason, bool) {
|
||||
if IsUnresolvedTarget(targetID) {
|
||||
return BoundaryDynamicDispatch, true
|
||||
}
|
||||
if strings.HasPrefix(targetID, "external::") {
|
||||
return BoundaryExternal, true
|
||||
}
|
||||
if IsStub(targetID) {
|
||||
return BoundaryStub, true
|
||||
}
|
||||
return "", false
|
||||
}
|
||||
|
||||
// CalleeBoundaries scans the out-edges of the given nodes for call/reference
|
||||
// targets a forward (callee-direction) walk could not follow. Each such target
|
||||
// means the reachable callee set could be larger than what was returned.
|
||||
func CalleeBoundaries(g Store, nodeIDs []string, limit int) []EpistemicBoundary {
|
||||
if g == nil {
|
||||
return nil
|
||||
}
|
||||
if limit <= 0 {
|
||||
limit = maxBoundaries
|
||||
}
|
||||
seen := map[string]bool{}
|
||||
var out []EpistemicBoundary
|
||||
for _, id := range nodeIDs {
|
||||
for _, e := range g.GetOutEdges(id) {
|
||||
if e.Kind != EdgeCalls && e.Kind != EdgeReferences {
|
||||
continue
|
||||
}
|
||||
reason, ok := ClassifyDroppedTarget(e.To, e.Kind)
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
key := id + "\x00" + e.To
|
||||
if seen[key] {
|
||||
continue
|
||||
}
|
||||
seen[key] = true
|
||||
out = append(out, EpistemicBoundary{
|
||||
SeedID: id,
|
||||
SeedName: nameForID(g, id),
|
||||
Target: boundaryTargetName(e.To),
|
||||
EdgeKind: string(e.Kind),
|
||||
Reason: reason,
|
||||
Direction: "callees",
|
||||
})
|
||||
if len(out) >= limit {
|
||||
return sortBoundaries(out)
|
||||
}
|
||||
}
|
||||
}
|
||||
return sortBoundaries(out)
|
||||
}
|
||||
|
||||
// CallerBoundaries flags nodes whose *caller* count is a floor because dynamic
|
||||
// dispatch into them is structurally possible: each node that implements or
|
||||
// overrides an interface method may be reached through that interface by
|
||||
// callers not attributed to it directly. It names the interface so an agent
|
||||
// can run find_implementations / get_callers on it to widen the picture.
|
||||
func CallerBoundaries(g Store, nodeIDs []string, limit int) []EpistemicBoundary {
|
||||
if g == nil {
|
||||
return nil
|
||||
}
|
||||
if limit <= 0 {
|
||||
limit = maxBoundaries
|
||||
}
|
||||
seen := map[string]bool{}
|
||||
var out []EpistemicBoundary
|
||||
for _, id := range nodeIDs {
|
||||
for _, e := range g.GetOutEdges(id) {
|
||||
if e.Kind != EdgeImplements && e.Kind != EdgeOverrides {
|
||||
continue
|
||||
}
|
||||
key := id + "\x00" + e.To
|
||||
if seen[key] {
|
||||
continue
|
||||
}
|
||||
seen[key] = true
|
||||
out = append(out, EpistemicBoundary{
|
||||
SeedID: id,
|
||||
SeedName: nameForID(g, id),
|
||||
Target: boundaryTargetName(e.To),
|
||||
EdgeKind: string(e.Kind),
|
||||
Reason: BoundaryInterfaceDispatch,
|
||||
Direction: "callers",
|
||||
})
|
||||
if len(out) >= limit {
|
||||
return sortBoundaries(out)
|
||||
}
|
||||
}
|
||||
}
|
||||
return sortBoundaries(out)
|
||||
}
|
||||
|
||||
// LowerBoundCaveat reports whether the boundary set makes the count a genuine
|
||||
// floor. Only dynamic-dispatch / interface-dispatch boundaries qualify: an
|
||||
// external/stdlib stub edge is listed for transparency but adds no hidden
|
||||
// in-repo callers/callees, so by itself it must not raise the flag (otherwise
|
||||
// nearly every symbol with a stdlib call would be flagged — see the design's
|
||||
// over-flagging guard).
|
||||
func LowerBoundCaveat(boundaries []EpistemicBoundary) bool {
|
||||
for _, b := range boundaries {
|
||||
if b.Reason == BoundaryDynamicDispatch || b.Reason == BoundaryInterfaceDispatch {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func boundaryTargetName(id string) string {
|
||||
if IsUnresolvedTarget(id) {
|
||||
return UnresolvedName(id)
|
||||
}
|
||||
return id
|
||||
}
|
||||
|
||||
func nameForID(g Store, id string) string {
|
||||
if n := g.GetNode(id); n != nil {
|
||||
return n.Name
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func sortBoundaries(bs []EpistemicBoundary) []EpistemicBoundary {
|
||||
sort.SliceStable(bs, func(i, j int) bool {
|
||||
if bs[i].SeedID != bs[j].SeedID {
|
||||
return bs[i].SeedID < bs[j].SeedID
|
||||
}
|
||||
return bs[i].Target < bs[j].Target
|
||||
})
|
||||
return bs
|
||||
}
|
||||
@@ -0,0 +1,96 @@
|
||||
package graph
|
||||
|
||||
import "testing"
|
||||
|
||||
func boundaryGraph() *Graph {
|
||||
g := New()
|
||||
for _, id := range []string{"A", "B", "C"} {
|
||||
g.AddNode(&Node{ID: id, Kind: KindMethod, Name: id})
|
||||
}
|
||||
// A implements an interface method → caller-side dispatch boundary.
|
||||
g.AddEdge(&Edge{From: "A", To: "iface::I.M", Kind: EdgeImplements, Origin: OriginASTResolved})
|
||||
// B calls an unresolved (dynamic-dispatch) target → callee-side floor.
|
||||
g.AddEdge(&Edge{From: "B", To: "unresolved::handler", Kind: EdgeCalls, Origin: OriginASTInferred})
|
||||
// C calls a stdlib stub → listed but not floor-making.
|
||||
g.AddEdge(&Edge{From: "C", To: "stdlib::fmt.Println", Kind: EdgeCalls, Origin: OriginASTResolved})
|
||||
return g
|
||||
}
|
||||
|
||||
func TestCalleeBoundaries(t *testing.T) {
|
||||
g := boundaryGraph()
|
||||
bs := CalleeBoundaries(g, []string{"B", "C"}, 0)
|
||||
if len(bs) != 2 {
|
||||
t.Fatalf("expected 2 callee boundaries, got %d: %+v", len(bs), bs)
|
||||
}
|
||||
byReason := map[BoundaryReason]EpistemicBoundary{}
|
||||
for _, b := range bs {
|
||||
byReason[b.Reason] = b
|
||||
if b.Direction != "callees" {
|
||||
t.Errorf("expected callees direction, got %s", b.Direction)
|
||||
}
|
||||
}
|
||||
if d, ok := byReason[BoundaryDynamicDispatch]; !ok || d.Target != "handler" {
|
||||
t.Errorf("expected dynamic_dispatch boundary on 'handler', got %+v", byReason)
|
||||
}
|
||||
if _, ok := byReason[BoundaryStub]; !ok {
|
||||
t.Errorf("expected stub boundary for stdlib call, got %+v", byReason)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCallerBoundaries(t *testing.T) {
|
||||
g := boundaryGraph()
|
||||
bs := CallerBoundaries(g, []string{"A"}, 0)
|
||||
if len(bs) != 1 {
|
||||
t.Fatalf("expected 1 caller boundary, got %d: %+v", len(bs), bs)
|
||||
}
|
||||
b := bs[0]
|
||||
if b.Reason != BoundaryInterfaceDispatch || b.Direction != "callers" {
|
||||
t.Errorf("unexpected boundary: %+v", b)
|
||||
}
|
||||
// A method that implements nothing has no caller boundary.
|
||||
if got := CallerBoundaries(g, []string{"B"}, 0); len(got) != 0 {
|
||||
t.Errorf("expected no boundary for non-implementing node, got %+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLowerBoundCaveat(t *testing.T) {
|
||||
cases := []struct {
|
||||
name string
|
||||
bs []EpistemicBoundary
|
||||
want bool
|
||||
}{
|
||||
{"empty", nil, false},
|
||||
{"dynamic", []EpistemicBoundary{{Reason: BoundaryDynamicDispatch}}, true},
|
||||
{"interface", []EpistemicBoundary{{Reason: BoundaryInterfaceDispatch}}, true},
|
||||
{"stub only", []EpistemicBoundary{{Reason: BoundaryStub}}, false},
|
||||
{"external only", []EpistemicBoundary{{Reason: BoundaryExternal}}, false},
|
||||
{"mixed", []EpistemicBoundary{{Reason: BoundaryStub}, {Reason: BoundaryDynamicDispatch}}, true},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
if got := LowerBoundCaveat(tc.bs); got != tc.want {
|
||||
t.Errorf("LowerBoundCaveat = %v, want %v", got, tc.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestClassifyDroppedTarget(t *testing.T) {
|
||||
cases := []struct {
|
||||
target string
|
||||
kind EdgeKind
|
||||
reason BoundaryReason
|
||||
ok bool
|
||||
}{
|
||||
{"unresolved::Foo", EdgeCalls, BoundaryDynamicDispatch, true},
|
||||
{"external::lodash.map", EdgeCalls, BoundaryExternal, true},
|
||||
{"stdlib::fmt.Println", EdgeCalls, BoundaryStub, true},
|
||||
{"pkg/x.go::Real", EdgeCalls, "", false},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
reason, ok := ClassifyDroppedTarget(tc.target, tc.kind)
|
||||
if ok != tc.ok || reason != tc.reason {
|
||||
t.Errorf("ClassifyDroppedTarget(%q) = (%q,%v), want (%q,%v)", tc.target, reason, ok, tc.reason, tc.ok)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,250 @@
|
||||
package graph
|
||||
|
||||
// ZeroEdgeClass classifies why a symbol's graph query came back empty.
|
||||
// An empty result has two very different causes that an agent cannot
|
||||
// otherwise tell apart, and a pre-edit safety check that trusts a
|
||||
// false "0 usages" is silently disarmed.
|
||||
type ZeroEdgeClass string
|
||||
|
||||
const (
|
||||
// ZeroEdgeNone means the symbol has incoming call/reference edges:
|
||||
// the query was not empty and no caveat is warranted.
|
||||
ZeroEdgeNone ZeroEdgeClass = "none"
|
||||
|
||||
// ZeroEdgeLikelyUnused means the symbol has no incoming
|
||||
// call/reference edges but DOES carry other graph edges — the
|
||||
// structural edge from its file (`defines`), a method's
|
||||
// `member_of`, or outgoing calls / references / type references.
|
||||
// This is consistent with genuine dead code: the extractor saw
|
||||
// the symbol, nothing uses it.
|
||||
ZeroEdgeLikelyUnused ZeroEdgeClass = "likely_unused"
|
||||
|
||||
// ZeroEdgePossibleExtractionGap means the symbol has zero edges of
|
||||
// any kind. A normally indexed function or method always carries
|
||||
// at least one structural edge — the file `defines` it, a method
|
||||
// is `member_of` its type — so zero total edges most likely means
|
||||
// the extractor never processed the symbol or its file. The
|
||||
// symbol may well be live; the graph just does not know. This is
|
||||
// the dangerous case for a delete-or-rewrite decision.
|
||||
ZeroEdgePossibleExtractionGap ZeroEdgeClass = "possible_extraction_gap"
|
||||
|
||||
// ZeroEdgeCoverageIncomplete means the symbol has no resolved
|
||||
// incoming call/reference edges, but the graph DOES carry
|
||||
// import-level evidence that consumers exist: inbound `imports` /
|
||||
// `re_exports` edges land on the symbol itself, or (for a public
|
||||
// JS/TS symbol) on its file. The usage query is incomplete, not
|
||||
// empty — reference-level resolution failed somewhere upstream —
|
||||
// so an agent must NOT read the empty result as safe-to-remove.
|
||||
ZeroEdgeCoverageIncomplete ZeroEdgeClass = "coverage_incomplete"
|
||||
)
|
||||
|
||||
// ZeroEdgeCaveat is the structured caveat attached to an empty graph
|
||||
// query result. Class is machine-checkable so a safety gate can branch
|
||||
// on it; Message is a short human-readable explanation.
|
||||
type ZeroEdgeCaveat struct {
|
||||
Class ZeroEdgeClass `json:"class" toon:"class"`
|
||||
Message string `json:"message" toon:"message"`
|
||||
}
|
||||
|
||||
// ZeroImpactCaveat is the per-symbol caveat attached to an empty impact
|
||||
// analysis result, which is computed over a list of symbols. It carries
|
||||
// the symbol ID alongside the same machine-checkable classification.
|
||||
type ZeroImpactCaveat struct {
|
||||
ID string `json:"id" toon:"id"`
|
||||
Class ZeroEdgeClass `json:"class" toon:"class"`
|
||||
Message string `json:"message" toon:"message"`
|
||||
}
|
||||
|
||||
// TierFilteredClass is the machine-checkable class of a min_tier-emptied
|
||||
// result — distinct from ZeroEdgeClass so a safety gate never confuses
|
||||
// "filtered out by min_tier" with "genuinely has no usages".
|
||||
const TierFilteredClass = "tier_filtered"
|
||||
|
||||
// TierFilteredCaveat is attached to a find_usages / get_* result whose edge
|
||||
// list was emptied (or thinned) by a min_tier filter while lower-tier edges
|
||||
// still exist. Without it a min_tier that filters every edge is
|
||||
// indistinguishable from "no usages" — the honesty bug it fixes. Class is
|
||||
// always TierFilteredClass; EdgesBelowMinTier counts the edges dropped by the
|
||||
// filter and MaxAvailableTier names the best tier actually present.
|
||||
type TierFilteredCaveat struct {
|
||||
Class string `json:"class" toon:"class"`
|
||||
EdgesBelowMinTier int `json:"edges_below_min_tier" toon:"edges_below_min_tier"`
|
||||
MaxAvailableTier string `json:"max_available_tier" toon:"max_available_tier"`
|
||||
}
|
||||
|
||||
// usageEdgeKinds are the incoming edge kinds that count as a symbol
|
||||
// being "used" — calls, references, and the type/instantiation edges
|
||||
// that find_usages itself treats as usages. An incoming edge of any of
|
||||
// these kinds means the symbol is not dead code.
|
||||
var usageEdgeKinds = map[EdgeKind]bool{
|
||||
EdgeCalls: true,
|
||||
EdgeReferences: true,
|
||||
EdgeInstantiates: true,
|
||||
EdgeImplements: true,
|
||||
EdgeExtends: true,
|
||||
EdgeReads: true,
|
||||
EdgeWrites: true,
|
||||
EdgeTests: true,
|
||||
}
|
||||
|
||||
// UsageInboundEdgeKinds returns the canonical list of incoming edge
|
||||
// kinds that classify a symbol as "used" by ClassifyZeroEdge. Exposed
|
||||
// for capability callers (NodeDegreeAggregator) that need to mirror
|
||||
// the in-graph usage filter server-side. Order is stable so the slice
|
||||
// is safe to pass directly to a query parameter binding.
|
||||
func UsageInboundEdgeKinds() []EdgeKind {
|
||||
return []EdgeKind{
|
||||
EdgeCalls,
|
||||
EdgeReferences,
|
||||
EdgeInstantiates,
|
||||
EdgeImplements,
|
||||
EdgeExtends,
|
||||
EdgeReads,
|
||||
EdgeWrites,
|
||||
EdgeTests,
|
||||
}
|
||||
}
|
||||
|
||||
// ClassifyZeroEdge inspects a symbol's incoming and outgoing edges and
|
||||
// returns how an empty usage/caller/impact query for it should be read.
|
||||
//
|
||||
// - ZeroEdgeNone — the symbol has at least one incoming usage edge.
|
||||
// - ZeroEdgeLikelyUnused — no incoming usage edge, but the symbol has
|
||||
// other graph edges (structural defines/member_of, or any outgoing
|
||||
// edge). Consistent with genuine dead code.
|
||||
// - ZeroEdgePossibleExtractionGap — the symbol has no edges at all,
|
||||
// which a normally indexed symbol never has; the extractor most
|
||||
// likely missed it.
|
||||
//
|
||||
// An unknown symbol ID is reported as an extraction gap: a query whose
|
||||
// target is not even in the graph is exactly as untrustworthy as one
|
||||
// whose target was never wired up.
|
||||
func ClassifyZeroEdge(g Store, symbolID string) ZeroEdgeClass {
|
||||
if g == nil || symbolID == "" {
|
||||
return ZeroEdgePossibleExtractionGap
|
||||
}
|
||||
if g.GetNode(symbolID) == nil {
|
||||
return ZeroEdgePossibleExtractionGap
|
||||
}
|
||||
|
||||
in := g.GetInEdges(symbolID)
|
||||
out := g.GetOutEdges(symbolID)
|
||||
|
||||
if len(in) == 0 && len(out) == 0 {
|
||||
return ZeroEdgePossibleExtractionGap
|
||||
}
|
||||
for _, e := range in {
|
||||
if usageEdgeKinds[e.Kind] {
|
||||
return ZeroEdgeNone
|
||||
}
|
||||
}
|
||||
if importConsumerCount(g, symbolID) > 0 {
|
||||
return ZeroEdgeCoverageIncomplete
|
||||
}
|
||||
// The graph still carries unresolved call candidates that name this
|
||||
// symbol — `unresolved::*.<name>` call edges the resolver / enrichment
|
||||
// pass never bound to it. Their existence is direct evidence that call
|
||||
// sites reference this name; an empty usage set is a resolution/coverage
|
||||
// gap, not proof the symbol is unused.
|
||||
if hasUnresolvedSameNameCandidates(g, symbolID) {
|
||||
return ZeroEdgeCoverageIncomplete
|
||||
}
|
||||
return ZeroEdgeLikelyUnused
|
||||
}
|
||||
|
||||
// hasUnresolvedSameNameCandidates reports whether the graph holds any
|
||||
// unresolved member-call edge (`unresolved::*.<name>`) naming this callable
|
||||
// symbol. Unresolved call stubs are indexed by their target string, so this is
|
||||
// a single reverse-edge lookup — no scan. Non-callable symbols never match.
|
||||
func hasUnresolvedSameNameCandidates(g Store, symbolID string) bool {
|
||||
n := g.GetNode(symbolID)
|
||||
if n == nil || n.Name == "" {
|
||||
return false
|
||||
}
|
||||
if n.Kind != KindFunction && n.Kind != KindMethod {
|
||||
return false
|
||||
}
|
||||
return len(g.GetInEdges(UnresolvedMarker+"*."+n.Name)) > 0
|
||||
}
|
||||
|
||||
// importConsumerCount counts the import-level consumer evidence for a
|
||||
// symbol: inbound `imports` / `re_exports` edges on the symbol itself
|
||||
// (any language — a per-binding import that resolved onto the symbol is
|
||||
// direct proof a consumer names it), plus, for a public JS/TS symbol,
|
||||
// inbound module-level import edges on its file node (a module-level
|
||||
// `import ... from './file'` lands on the file, but still proves the
|
||||
// file's exports have consumers).
|
||||
func importConsumerCount(g Store, symbolID string) int {
|
||||
count := 0
|
||||
for _, e := range g.GetInEdges(symbolID) {
|
||||
if e.Kind == EdgeImports || e.Kind == EdgeReExports {
|
||||
count++
|
||||
}
|
||||
}
|
||||
if count > 0 {
|
||||
return count
|
||||
}
|
||||
n := g.GetNode(symbolID)
|
||||
if n == nil || n.FilePath == "" || n.FilePath == symbolID {
|
||||
return 0
|
||||
}
|
||||
switch n.Language {
|
||||
case "typescript", "tsx", "javascript", "jsx":
|
||||
default:
|
||||
return 0 // module-level file imports imply consumers only for JS/TS
|
||||
}
|
||||
if vis, _ := n.Meta["visibility"].(string); vis != "public" {
|
||||
return 0
|
||||
}
|
||||
for _, e := range g.GetInEdges(n.FilePath) {
|
||||
if e.Kind == EdgeImports || e.Kind == EdgeReExports {
|
||||
count++
|
||||
}
|
||||
}
|
||||
return count
|
||||
}
|
||||
|
||||
// zeroEdgeMessages maps each classification to its human-readable
|
||||
// caveat text.
|
||||
var zeroEdgeMessages = map[ZeroEdgeClass]string{
|
||||
ZeroEdgeLikelyUnused: "no incoming call or reference edges, but the symbol is " +
|
||||
"indexed (it has structural or outgoing edges) — consistent with genuine " +
|
||||
"unused code that is safe to remove.",
|
||||
ZeroEdgePossibleExtractionGap: "the symbol has no graph edges of any kind. A " +
|
||||
"normally indexed symbol always has at least a structural edge, so the " +
|
||||
"extractor most likely did not process it — treat this empty result as " +
|
||||
"unverified, not as proof the symbol is unused.",
|
||||
ZeroEdgeCoverageIncomplete: "no resolved call or reference edges, but the graph " +
|
||||
"still carries consumer evidence — import/re-export edges pointing at this " +
|
||||
"symbol or its file, or unresolved same-name call candidates the resolver / " +
|
||||
"enrichment pass never bound to it. Reference-level resolution is incomplete, " +
|
||||
"so treat this empty result as UNVERIFIED coverage, not as proof the symbol is " +
|
||||
"unused or safe to remove.",
|
||||
}
|
||||
|
||||
// zeroEdgeNotFoundMessage is the caveat text when the queried id is not in
|
||||
// the graph at all — almost always a mistyped id or one missing its repo
|
||||
// prefix, rather than a true extraction gap.
|
||||
const zeroEdgeNotFoundMessage = "no symbol with this id is in the graph — the id is " +
|
||||
"probably mistyped or missing its repo prefix (ids look like " +
|
||||
"<repo>/<path>::<symbol>, e.g. gortex/internal/x.go::Foo). Run a symbol search to " +
|
||||
"get the exact id; treat this empty result as unverified, not as proof of no usages."
|
||||
|
||||
// CaveatForZeroEdge builds the structured caveat for an empty graph
|
||||
// query result on symbolID. It returns nil when the symbol has
|
||||
// incoming usage edges (ZeroEdgeNone) — a non-empty result carries no
|
||||
// caveat — so callers can attach the return value unconditionally.
|
||||
func CaveatForZeroEdge(g Store, symbolID string) *ZeroEdgeCaveat {
|
||||
// A target that is not even in the graph is the most common cause of a
|
||||
// "0 usages" surprise — usually a mistyped id or one missing its repo
|
||||
// prefix. Keep the untrustworthy extraction-gap class so safety gates
|
||||
// still trip, but point the message at the id rather than the extractor.
|
||||
if g != nil && symbolID != "" && g.GetNode(symbolID) == nil {
|
||||
return &ZeroEdgeCaveat{Class: ZeroEdgePossibleExtractionGap, Message: zeroEdgeNotFoundMessage}
|
||||
}
|
||||
class := ClassifyZeroEdge(g, symbolID)
|
||||
if class == ZeroEdgeNone {
|
||||
return nil
|
||||
}
|
||||
return &ZeroEdgeCaveat{Class: class, Message: zeroEdgeMessages[class]}
|
||||
}
|
||||
@@ -0,0 +1,32 @@
|
||||
package graph
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// TestCaveatForZeroEdge_NotFoundMessage asserts a queried id that is not in
|
||||
// the graph (e.g. mistyped or missing its repo prefix) gets a caveat that
|
||||
// points at the id — while still carrying the untrustworthy class so a
|
||||
// safety gate trips — and that an existing zero-edge symbol keeps the
|
||||
// extraction-gap message.
|
||||
func TestCaveatForZeroEdge_NotFoundMessage(t *testing.T) {
|
||||
g := New()
|
||||
g.AddNode(&Node{ID: "repo/a.go::Lonely", Kind: KindFunction, Name: "Lonely"}) // exists, no edges
|
||||
|
||||
notFound := CaveatForZeroEdge(g, "a.go::Missing")
|
||||
if notFound == nil || notFound.Class != ZeroEdgePossibleExtractionGap {
|
||||
t.Fatalf("a not-found id must still carry the untrustworthy class; got %+v", notFound)
|
||||
}
|
||||
if !strings.Contains(notFound.Message, "repo prefix") {
|
||||
t.Errorf("not-found message should point at the id/prefix; got %q", notFound.Message)
|
||||
}
|
||||
|
||||
existing := CaveatForZeroEdge(g, "repo/a.go::Lonely")
|
||||
if existing == nil {
|
||||
t.Fatal("an existing zero-edge symbol should still carry a caveat")
|
||||
}
|
||||
if strings.Contains(existing.Message, "repo prefix") {
|
||||
t.Errorf("an existing zero-edge symbol should get the extraction-gap message, not the not-found one; got %q", existing.Message)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,121 @@
|
||||
package graph
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// buildClassifyGraph wires three functions in one file, each with a
|
||||
// distinct edge profile, so ClassifyZeroEdge can be exercised against
|
||||
// every classification:
|
||||
//
|
||||
// - Used — the file defines it AND a caller calls it.
|
||||
// - Unused — the file defines it, it makes an outgoing call, but
|
||||
// nothing calls it back. Indexed, dead.
|
||||
// - Orphaned — added with no edges at all, mimicking a symbol the
|
||||
// extractor never processed.
|
||||
func buildClassifyGraph() *Graph {
|
||||
g := New()
|
||||
g.AddNode(&Node{ID: "svc.go", Kind: KindFile, Name: "svc.go", FilePath: "svc.go"})
|
||||
g.AddNode(&Node{ID: "svc.go::Caller", Kind: KindFunction, Name: "Caller", FilePath: "svc.go"})
|
||||
g.AddNode(&Node{ID: "svc.go::Used", Kind: KindFunction, Name: "Used", FilePath: "svc.go"})
|
||||
g.AddNode(&Node{ID: "svc.go::Unused", Kind: KindFunction, Name: "Unused", FilePath: "svc.go"})
|
||||
g.AddNode(&Node{ID: "svc.go::Helper", Kind: KindFunction, Name: "Helper", FilePath: "svc.go"})
|
||||
|
||||
// The file defines every function except the orphan.
|
||||
g.AddEdge(&Edge{From: "svc.go", To: "svc.go::Caller", Kind: EdgeDefines})
|
||||
g.AddEdge(&Edge{From: "svc.go", To: "svc.go::Used", Kind: EdgeDefines})
|
||||
g.AddEdge(&Edge{From: "svc.go", To: "svc.go::Unused", Kind: EdgeDefines})
|
||||
g.AddEdge(&Edge{From: "svc.go", To: "svc.go::Helper", Kind: EdgeDefines})
|
||||
|
||||
// Used has an incoming call edge.
|
||||
g.AddEdge(&Edge{From: "svc.go::Caller", To: "svc.go::Used", Kind: EdgeCalls})
|
||||
// Unused has only an outgoing call — no incoming usage edge.
|
||||
g.AddEdge(&Edge{From: "svc.go::Unused", To: "svc.go::Helper", Kind: EdgeCalls})
|
||||
return g
|
||||
}
|
||||
|
||||
func TestClassifyZeroEdge(t *testing.T) {
|
||||
g := buildClassifyGraph()
|
||||
// A method node with only its member_of structural edge — indexed,
|
||||
// but nothing references it.
|
||||
g.AddNode(&Node{ID: "svc.go::T", Kind: KindType, Name: "T", FilePath: "svc.go"})
|
||||
g.AddNode(&Node{ID: "svc.go::T.Method", Kind: KindMethod, Name: "Method", FilePath: "svc.go"})
|
||||
g.AddEdge(&Edge{From: "svc.go::T.Method", To: "svc.go::T", Kind: EdgeMemberOf})
|
||||
// An orphan: in the graph, but carrying no edges of any kind.
|
||||
g.AddNode(&Node{ID: "svc.go::Orphan", Kind: KindFunction, Name: "Orphan", FilePath: "svc.go"})
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
symbolID string
|
||||
want ZeroEdgeClass
|
||||
}{
|
||||
{
|
||||
name: "incoming call edge yields none",
|
||||
symbolID: "svc.go::Used",
|
||||
want: ZeroEdgeNone,
|
||||
},
|
||||
{
|
||||
name: "only structural defines plus outgoing call yields likely_unused",
|
||||
symbolID: "svc.go::Unused",
|
||||
want: ZeroEdgeLikelyUnused,
|
||||
},
|
||||
{
|
||||
name: "method with only member_of yields likely_unused",
|
||||
symbolID: "svc.go::T.Method",
|
||||
want: ZeroEdgeLikelyUnused,
|
||||
},
|
||||
{
|
||||
name: "zero edges of any kind yields possible_extraction_gap",
|
||||
symbolID: "svc.go::Orphan",
|
||||
want: ZeroEdgePossibleExtractionGap,
|
||||
},
|
||||
{
|
||||
name: "unknown symbol id yields possible_extraction_gap",
|
||||
symbolID: "svc.go::DoesNotExist",
|
||||
want: ZeroEdgePossibleExtractionGap,
|
||||
},
|
||||
{
|
||||
name: "empty symbol id yields possible_extraction_gap",
|
||||
symbolID: "",
|
||||
want: ZeroEdgePossibleExtractionGap,
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := ClassifyZeroEdge(g, tt.symbolID)
|
||||
assert.Equal(t, tt.want, got)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestClassifyZeroEdge_NilGraph(t *testing.T) {
|
||||
assert.Equal(t, ZeroEdgePossibleExtractionGap, ClassifyZeroEdge(nil, "svc.go::Used"))
|
||||
}
|
||||
|
||||
func TestCaveatForZeroEdge(t *testing.T) {
|
||||
g := buildClassifyGraph()
|
||||
g.AddNode(&Node{ID: "svc.go::Orphan", Kind: KindFunction, Name: "Orphan", FilePath: "svc.go"})
|
||||
|
||||
// A used symbol carries no caveat — nil is returned so callers can
|
||||
// attach it unconditionally.
|
||||
assert.Nil(t, CaveatForZeroEdge(g, "svc.go::Used"))
|
||||
|
||||
// Likely-unused: classification plus a non-empty message.
|
||||
unused := CaveatForZeroEdge(g, "svc.go::Unused")
|
||||
require.NotNil(t, unused)
|
||||
assert.Equal(t, ZeroEdgeLikelyUnused, unused.Class)
|
||||
assert.NotEmpty(t, unused.Message)
|
||||
|
||||
// Extraction gap: classification plus a non-empty message.
|
||||
gap := CaveatForZeroEdge(g, "svc.go::Orphan")
|
||||
require.NotNil(t, gap)
|
||||
assert.Equal(t, ZeroEdgePossibleExtractionGap, gap.Class)
|
||||
assert.NotEmpty(t, gap.Message)
|
||||
|
||||
// The two caveat messages must differ — they describe opposite
|
||||
// situations and an agent has to be able to tell them apart.
|
||||
assert.NotEqual(t, unused.Message, gap.Message)
|
||||
}
|
||||
@@ -0,0 +1,36 @@
|
||||
package graph
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
)
|
||||
|
||||
// A callable with no resolved incoming usages, but whose name still has
|
||||
// unresolved same-name call candidates in the graph, must be classified as
|
||||
// coverage-incomplete — not likely_unused. The unresolved stubs are direct
|
||||
// evidence that call sites reference the name; the empty usage set is a
|
||||
// resolution gap, not proof the symbol is dead.
|
||||
func TestClassifyZeroEdge_UnresolvedSameNameCandidates(t *testing.T) {
|
||||
g := New()
|
||||
g.AddNode(&Node{ID: "Crash.java", Kind: KindFile, Name: "Crash.java", FilePath: "Crash.java", Language: "java"})
|
||||
g.AddNode(&Node{ID: "Crash.java::CrashController.triggerException", Kind: KindMethod, Name: "triggerException", FilePath: "Crash.java", Language: "java"})
|
||||
g.AddNode(&Node{ID: "Test.java", Kind: KindFile, Name: "Test.java", FilePath: "Test.java", Language: "java"})
|
||||
g.AddNode(&Node{ID: "Test.java::T.run", Kind: KindMethod, Name: "run", FilePath: "Test.java", Language: "java"})
|
||||
|
||||
// The method is indexed (defined) but has no resolved incoming usage.
|
||||
g.AddEdge(&Edge{From: "Crash.java", To: "Crash.java::CrashController.triggerException", Kind: EdgeDefines})
|
||||
// An unresolved same-name call candidate the resolver never bound.
|
||||
g.AddEdge(&Edge{From: "Test.java::T.run", To: "unresolved::*.triggerException", Kind: EdgeCalls, FilePath: "Test.java", Line: 37})
|
||||
|
||||
got := ClassifyZeroEdge(g, "Crash.java::CrashController.triggerException")
|
||||
assert.Equal(t, ZeroEdgeCoverageIncomplete, got,
|
||||
"a symbol whose name still has unresolved call candidates must not be reported likely_unused")
|
||||
|
||||
// A genuinely unused method (no unresolved candidates naming it) still reads
|
||||
// as likely_unused.
|
||||
g.AddNode(&Node{ID: "Crash.java::CrashController.reallyDead", Kind: KindMethod, Name: "reallyDead", FilePath: "Crash.java", Language: "java"})
|
||||
g.AddEdge(&Edge{From: "Crash.java", To: "Crash.java::CrashController.reallyDead", Kind: EdgeDefines})
|
||||
assert.Equal(t, ZeroEdgeLikelyUnused, ClassifyZeroEdge(g, "Crash.java::CrashController.reallyDead"),
|
||||
"a method with no unresolved candidates naming it stays likely_unused")
|
||||
}
|
||||
@@ -0,0 +1,62 @@
|
||||
package graph
|
||||
|
||||
import "strings"
|
||||
|
||||
// IsProxyNode reports whether n is a cross-daemon proxy-edge node —
|
||||
// identified by its struct fields, NOT its ID shape. Distinct from
|
||||
// IsStub(id) (the stdlib/builtin/module string predicate): a proxy node
|
||||
// is keyed under the "remote:<slug>~..." origin namespace, which IsStub
|
||||
// does not recognise. Proxy nodes are excluded from graph_stats / BM25 /
|
||||
// communities / analyzers and never persisted to the durable store.
|
||||
func IsProxyNode(n *Node) bool {
|
||||
return n != nil && n.Stub && n.Origin != ""
|
||||
}
|
||||
|
||||
// proxyIDPrefix is the origin-namespace marker for a proxy node id.
|
||||
const proxyIDPrefix = "remote:"
|
||||
|
||||
// proxyIDSep separates the origin segment from the remote's native id.
|
||||
const proxyIDSep = "~"
|
||||
|
||||
// ProxyNodeID composes the origin-namespaced id for a remote symbol so a
|
||||
// remote node can never alias a local id, even when two daemons share a
|
||||
// repo prefix: "remote:<slug>~<remoteRepoPrefix>/<file>::<sym>".
|
||||
func ProxyNodeID(slug, remoteID string) string {
|
||||
return proxyIDPrefix + slug + proxyIDSep + remoteID
|
||||
}
|
||||
|
||||
// IsProxyID reports whether id is an origin-namespaced proxy id.
|
||||
func IsProxyID(id string) bool {
|
||||
if !strings.HasPrefix(id, proxyIDPrefix) {
|
||||
return false
|
||||
}
|
||||
return strings.Contains(id[len(proxyIDPrefix):], proxyIDSep)
|
||||
}
|
||||
|
||||
// ProxyOriginSlug returns the <slug> of a proxy id, or "" if id is not a
|
||||
// proxy id.
|
||||
func ProxyOriginSlug(id string) string {
|
||||
if !strings.HasPrefix(id, proxyIDPrefix) {
|
||||
return ""
|
||||
}
|
||||
rest := id[len(proxyIDPrefix):]
|
||||
i := strings.Index(rest, proxyIDSep)
|
||||
if i < 0 {
|
||||
return ""
|
||||
}
|
||||
return rest[:i]
|
||||
}
|
||||
|
||||
// ProxyRemoteID returns the remote's native id encoded in a proxy id, or
|
||||
// "" if id is not a proxy id.
|
||||
func ProxyRemoteID(id string) string {
|
||||
if !strings.HasPrefix(id, proxyIDPrefix) {
|
||||
return ""
|
||||
}
|
||||
rest := id[len(proxyIDPrefix):]
|
||||
i := strings.Index(rest, proxyIDSep)
|
||||
if i < 0 {
|
||||
return ""
|
||||
}
|
||||
return rest[i+len(proxyIDSep):]
|
||||
}
|
||||
@@ -0,0 +1,68 @@
|
||||
package graph
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// TestProxyNodeID_RoundTrip asserts the origin-namespaced id composes and
|
||||
// decomposes, and that a remote whose repo prefix collides with a local
|
||||
// prefix still yields a distinct (non-aliasing) proxy id.
|
||||
func TestProxyNodeID_RoundTrip(t *testing.T) {
|
||||
remoteID := "gortex/internal/daemon/router.go::Router"
|
||||
pid := ProxyNodeID("r2", remoteID)
|
||||
if !IsProxyID(pid) {
|
||||
t.Fatalf("%q should be a proxy id", pid)
|
||||
}
|
||||
if got := ProxyOriginSlug(pid); got != "r2" {
|
||||
t.Errorf("origin slug = %q, want r2", got)
|
||||
}
|
||||
if got := ProxyRemoteID(pid); got != remoteID {
|
||||
t.Errorf("remote id = %q, want %q", got, remoteID)
|
||||
}
|
||||
// A local node with the same native id is never a proxy id, so no
|
||||
// alias is possible even on a shared prefix.
|
||||
if IsProxyID(remoteID) {
|
||||
t.Error("a local native id must not be mistaken for a proxy id")
|
||||
}
|
||||
if ProxyNodeID("r3", remoteID) == pid {
|
||||
t.Error("the same remote symbol under two slugs must yield distinct proxy ids")
|
||||
}
|
||||
}
|
||||
|
||||
// TestIsProxyNode asserts proxy detection keys on the struct fields, not
|
||||
// the id shape, and never confuses a stdlib stub.
|
||||
func TestIsProxyNode(t *testing.T) {
|
||||
proxy := &Node{ID: ProxyNodeID("r2", "x/y.go::Z"), Origin: "remote:r2", Stub: true, FetchedAt: time.Now()}
|
||||
if !IsProxyNode(proxy) {
|
||||
t.Error("a Stub+Origin node is a proxy node")
|
||||
}
|
||||
local := &Node{ID: "x/y.go::Z"}
|
||||
if IsProxyNode(local) {
|
||||
t.Error("a local node is not a proxy node")
|
||||
}
|
||||
// Stub without Origin (some other stub convention) is not a proxy.
|
||||
if IsProxyNode(&Node{Stub: true}) {
|
||||
t.Error("Stub alone is not a proxy node")
|
||||
}
|
||||
if IsProxyNode(nil) {
|
||||
t.Error("nil is not a proxy node")
|
||||
}
|
||||
// IsStub (the string predicate) must not match a proxy id.
|
||||
if IsStub(proxy.ID) {
|
||||
t.Error("IsStub must not recognise a proxy id")
|
||||
}
|
||||
}
|
||||
|
||||
// TestProxyHelpers_NonProxyInputs asserts the accessors are safe on
|
||||
// non-proxy ids.
|
||||
func TestProxyHelpers_NonProxyInputs(t *testing.T) {
|
||||
for _, id := range []string{"", "x/y.go::Z", "remote:nosep", "stdlib::fmt"} {
|
||||
if IsProxyID(id) {
|
||||
t.Errorf("%q should not be a proxy id", id)
|
||||
}
|
||||
if ProxyOriginSlug(id) != "" || ProxyRemoteID(id) != "" {
|
||||
t.Errorf("accessors should be empty for non-proxy id %q", id)
|
||||
}
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,531 @@
|
||||
package graph
|
||||
|
||||
import (
|
||||
"sync"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
"pgregory.net/rapid"
|
||||
)
|
||||
|
||||
func makeNode(id, name string, kind NodeKind, file, lang string) *Node {
|
||||
return &Node{
|
||||
ID: id,
|
||||
Kind: kind,
|
||||
Name: name,
|
||||
QualName: "pkg." + name,
|
||||
FilePath: file,
|
||||
StartLine: 1,
|
||||
EndLine: 10,
|
||||
Language: lang,
|
||||
}
|
||||
}
|
||||
|
||||
func TestAddAndGetNode(t *testing.T) {
|
||||
g := New()
|
||||
n := makeNode("a.go::Foo", "Foo", KindFunction, "a.go", "go")
|
||||
g.AddNode(n)
|
||||
|
||||
assert.Equal(t, n, g.GetNode("a.go::Foo"))
|
||||
assert.Equal(t, n, g.GetNodeByQualName("pkg.Foo"))
|
||||
assert.Equal(t, []*Node{n}, g.FindNodesByName("Foo"))
|
||||
assert.Equal(t, []*Node{n}, g.GetFileNodes("a.go"))
|
||||
assert.Nil(t, g.GetNode("nonexistent"))
|
||||
}
|
||||
|
||||
func TestAddAndGetEdge(t *testing.T) {
|
||||
g := New()
|
||||
n1 := makeNode("a.go::Foo", "Foo", KindFunction, "a.go", "go")
|
||||
n2 := makeNode("b.go::Bar", "Bar", KindFunction, "b.go", "go")
|
||||
g.AddNode(n1)
|
||||
g.AddNode(n2)
|
||||
|
||||
e := &Edge{From: n1.ID, To: n2.ID, Kind: EdgeCalls, FilePath: "a.go", Line: 5}
|
||||
g.AddEdge(e)
|
||||
|
||||
out := g.GetOutEdges(n1.ID)
|
||||
require.Len(t, out, 1)
|
||||
assert.Equal(t, EdgeCalls, out[0].Kind)
|
||||
|
||||
in := g.GetInEdges(n2.ID)
|
||||
require.Len(t, in, 1)
|
||||
assert.Equal(t, n1.ID, in[0].From)
|
||||
}
|
||||
|
||||
func TestEvictFile(t *testing.T) {
|
||||
g := New()
|
||||
n1 := makeNode("a.go::Foo", "Foo", KindFunction, "a.go", "go")
|
||||
n2 := makeNode("a.go::Bar", "Bar", KindFunction, "a.go", "go")
|
||||
n3 := makeNode("b.go::Baz", "Baz", KindFunction, "b.go", "go")
|
||||
g.AddNode(n1)
|
||||
g.AddNode(n2)
|
||||
g.AddNode(n3)
|
||||
|
||||
g.AddEdge(&Edge{From: n1.ID, To: n3.ID, Kind: EdgeCalls, FilePath: "a.go", Line: 1})
|
||||
g.AddEdge(&Edge{From: n3.ID, To: n2.ID, Kind: EdgeCalls, FilePath: "b.go", Line: 2})
|
||||
|
||||
nodesRm, edgesRm := g.EvictFile("a.go")
|
||||
assert.Equal(t, 2, nodesRm)
|
||||
assert.Equal(t, 2, edgesRm) // both edges reference evicted node IDs
|
||||
|
||||
assert.Nil(t, g.GetNode("a.go::Foo"))
|
||||
assert.Nil(t, g.GetNode("a.go::Bar"))
|
||||
assert.NotNil(t, g.GetNode("b.go::Baz"))
|
||||
|
||||
// Edge from b.go to a.go::Bar should also be cleaned from inEdges.
|
||||
assert.Empty(t, g.GetOutEdges("a.go::Foo"))
|
||||
}
|
||||
|
||||
func TestEvictFile_NoNodes(t *testing.T) {
|
||||
g := New()
|
||||
n, e := g.EvictFile("nonexistent.go")
|
||||
assert.Equal(t, 0, n)
|
||||
assert.Equal(t, 0, e)
|
||||
}
|
||||
|
||||
func TestNodeAndEdgeCount(t *testing.T) {
|
||||
g := New()
|
||||
g.AddNode(makeNode("a.go::A", "A", KindFunction, "a.go", "go"))
|
||||
g.AddNode(makeNode("b.go::B", "B", KindType, "b.go", "go"))
|
||||
g.AddEdge(&Edge{From: "a.go::A", To: "b.go::B", Kind: EdgeReferences, FilePath: "a.go", Line: 1})
|
||||
|
||||
assert.Equal(t, 2, g.NodeCount())
|
||||
assert.Equal(t, 1, g.EdgeCount())
|
||||
}
|
||||
|
||||
func TestStats(t *testing.T) {
|
||||
g := New()
|
||||
g.AddNode(makeNode("a.go::A", "A", KindFunction, "a.go", "go"))
|
||||
g.AddNode(makeNode("b.go::B", "B", KindType, "b.go", "go"))
|
||||
g.AddNode(makeNode("c.ts::C", "C", KindFunction, "c.ts", "typescript"))
|
||||
g.AddEdge(&Edge{From: "a.go::A", To: "b.go::B", Kind: EdgeReferences, FilePath: "a.go", Line: 1})
|
||||
|
||||
s := g.Stats()
|
||||
assert.Equal(t, 3, s.TotalNodes)
|
||||
assert.Equal(t, 1, s.TotalEdges)
|
||||
assert.Equal(t, 2, s.ByKind["function"])
|
||||
assert.Equal(t, 1, s.ByKind["type"])
|
||||
assert.Equal(t, 2, s.ByLanguage["go"])
|
||||
assert.Equal(t, 1, s.ByLanguage["typescript"])
|
||||
}
|
||||
|
||||
func TestAllNodesAndEdges(t *testing.T) {
|
||||
g := New()
|
||||
g.AddNode(makeNode("a.go::A", "A", KindFunction, "a.go", "go"))
|
||||
g.AddNode(makeNode("b.go::B", "B", KindFunction, "b.go", "go"))
|
||||
g.AddEdge(&Edge{From: "a.go::A", To: "b.go::B", Kind: EdgeCalls, FilePath: "a.go", Line: 1})
|
||||
|
||||
assert.Len(t, g.AllNodes(), 2)
|
||||
assert.Len(t, g.AllEdges(), 1)
|
||||
}
|
||||
|
||||
func TestConcurrency(t *testing.T) {
|
||||
g := New()
|
||||
var wg sync.WaitGroup
|
||||
|
||||
// Concurrent writers.
|
||||
for i := range 50 {
|
||||
wg.Add(1)
|
||||
go func(i int) {
|
||||
defer wg.Done()
|
||||
id := "file.go::" + string(rune('A'+i))
|
||||
n := makeNode(id, string(rune('A'+i)), KindFunction, "file.go", "go")
|
||||
n.QualName = "" // avoid collision
|
||||
g.AddNode(n)
|
||||
}(i)
|
||||
}
|
||||
|
||||
// Concurrent readers.
|
||||
for range 50 {
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
_ = g.NodeCount()
|
||||
_ = g.GetFileNodes("file.go")
|
||||
_ = g.Stats()
|
||||
}()
|
||||
}
|
||||
|
||||
wg.Wait()
|
||||
}
|
||||
|
||||
func TestNodeBrief(t *testing.T) {
|
||||
n := &Node{
|
||||
ID: "a.go::Foo", Kind: KindFunction, Name: "Foo",
|
||||
QualName: "pkg.Foo", FilePath: "a.go", StartLine: 10, EndLine: 20,
|
||||
Language: "go", Meta: map[string]any{"signature": "func Foo()"},
|
||||
}
|
||||
b := n.Brief()
|
||||
assert.Equal(t, "a.go::Foo", b["id"])
|
||||
assert.Equal(t, "Foo", b["name"])
|
||||
assert.Equal(t, NodeKind("function"), b["kind"])
|
||||
assert.Equal(t, "a.go", b["file_path"])
|
||||
assert.Equal(t, 10, b["start_line"])
|
||||
// Should NOT contain meta, qual_name, end_line, language.
|
||||
_, hasMeta := b["meta"]
|
||||
assert.False(t, hasMeta)
|
||||
}
|
||||
|
||||
func TestValidNodeKind(t *testing.T) {
|
||||
assert.True(t, ValidNodeKind(KindFunction))
|
||||
assert.True(t, ValidNodeKind(KindFile))
|
||||
assert.False(t, ValidNodeKind(NodeKind("unknown")))
|
||||
}
|
||||
|
||||
func makeRepoNode(id, name string, kind NodeKind, file, lang, repo string) *Node {
|
||||
return &Node{
|
||||
ID: id,
|
||||
Kind: kind,
|
||||
Name: name,
|
||||
QualName: repo + "." + name,
|
||||
FilePath: file,
|
||||
StartLine: 1,
|
||||
EndLine: 10,
|
||||
Language: lang,
|
||||
RepoPrefix: repo,
|
||||
}
|
||||
}
|
||||
|
||||
func TestAddNode_ByRepoIndex(t *testing.T) {
|
||||
g := New()
|
||||
n1 := makeRepoNode("repoA/a.go::Foo", "Foo", KindFunction, "repoA/a.go", "go", "repoA")
|
||||
n2 := makeRepoNode("repoB/b.go::Bar", "Bar", KindFunction, "repoB/b.go", "go", "repoB")
|
||||
n3 := makeRepoNode("repoA/c.go::Baz", "Baz", KindType, "repoA/c.go", "go", "repoA")
|
||||
g.AddNode(n1)
|
||||
g.AddNode(n2)
|
||||
g.AddNode(n3)
|
||||
|
||||
repoANodes := g.GetRepoNodes("repoA")
|
||||
assert.Len(t, repoANodes, 2)
|
||||
repoBNodes := g.GetRepoNodes("repoB")
|
||||
assert.Len(t, repoBNodes, 1)
|
||||
assert.Equal(t, "Bar", repoBNodes[0].Name)
|
||||
}
|
||||
|
||||
func TestAddNode_EmptyRepoPrefix(t *testing.T) {
|
||||
g := New()
|
||||
n := makeNode("a.go::Foo", "Foo", KindFunction, "a.go", "go")
|
||||
g.AddNode(n)
|
||||
|
||||
// Nodes without RepoPrefix should not appear in byRepo.
|
||||
assert.Empty(t, g.GetRepoNodes(""))
|
||||
assert.Empty(t, g.RepoPrefixes())
|
||||
}
|
||||
|
||||
func TestGetRepoNodes_ReturnsCopy(t *testing.T) {
|
||||
g := New()
|
||||
n := makeRepoNode("repoA/a.go::Foo", "Foo", KindFunction, "repoA/a.go", "go", "repoA")
|
||||
g.AddNode(n)
|
||||
|
||||
nodes := g.GetRepoNodes("repoA")
|
||||
nodes[0] = nil // mutate the returned slice
|
||||
assert.NotNil(t, g.GetRepoNodes("repoA")[0], "GetRepoNodes should return a copy")
|
||||
}
|
||||
|
||||
func TestGetRepoNodes_NotFound(t *testing.T) {
|
||||
g := New()
|
||||
assert.Empty(t, g.GetRepoNodes("nonexistent"))
|
||||
}
|
||||
|
||||
func TestEvictRepo(t *testing.T) {
|
||||
g := New()
|
||||
nA1 := makeRepoNode("repoA/a.go::Foo", "Foo", KindFunction, "repoA/a.go", "go", "repoA")
|
||||
nA2 := makeRepoNode("repoA/a.go::Bar", "Bar", KindFunction, "repoA/a.go", "go", "repoA")
|
||||
nB1 := makeRepoNode("repoB/b.go::Baz", "Baz", KindFunction, "repoB/b.go", "go", "repoB")
|
||||
g.AddNode(nA1)
|
||||
g.AddNode(nA2)
|
||||
g.AddNode(nB1)
|
||||
|
||||
// Edges: intra-repoA, cross-repo A→B, intra-repoB self-ref
|
||||
g.AddEdge(&Edge{From: nA1.ID, To: nA2.ID, Kind: EdgeCalls, FilePath: "repoA/a.go", Line: 1})
|
||||
g.AddEdge(&Edge{From: nA1.ID, To: nB1.ID, Kind: EdgeCalls, FilePath: "repoA/a.go", Line: 2, CrossRepo: true})
|
||||
g.AddEdge(&Edge{From: nB1.ID, To: nA2.ID, Kind: EdgeCalls, FilePath: "repoB/b.go", Line: 3, CrossRepo: true})
|
||||
|
||||
nodesRm, edgesRm := g.EvictRepo("repoA")
|
||||
assert.Equal(t, 2, nodesRm)
|
||||
assert.Equal(t, 3, edgesRm) // all 3 edges reference repoA nodes
|
||||
|
||||
// repoA nodes gone.
|
||||
assert.Nil(t, g.GetNode("repoA/a.go::Foo"))
|
||||
assert.Nil(t, g.GetNode("repoA/a.go::Bar"))
|
||||
assert.Empty(t, g.GetRepoNodes("repoA"))
|
||||
|
||||
// repoB node still present.
|
||||
assert.NotNil(t, g.GetNode("repoB/b.go::Baz"))
|
||||
assert.Len(t, g.GetRepoNodes("repoB"), 1)
|
||||
|
||||
// byName cleaned for evicted nodes.
|
||||
assert.Empty(t, g.FindNodesByName("Foo"))
|
||||
assert.Empty(t, g.FindNodesByName("Bar"))
|
||||
assert.Len(t, g.FindNodesByName("Baz"), 1)
|
||||
|
||||
// byFile cleaned for evicted nodes.
|
||||
assert.Empty(t, g.GetFileNodes("repoA/a.go"))
|
||||
assert.Len(t, g.GetFileNodes("repoB/b.go"), 1)
|
||||
}
|
||||
|
||||
func TestEvictRepo_NoNodes(t *testing.T) {
|
||||
g := New()
|
||||
n, e := g.EvictRepo("nonexistent")
|
||||
assert.Equal(t, 0, n)
|
||||
assert.Equal(t, 0, e)
|
||||
}
|
||||
|
||||
func TestEvictRepo_QualNameCleaned(t *testing.T) {
|
||||
g := New()
|
||||
n := makeRepoNode("repoA/a.go::Foo", "Foo", KindFunction, "repoA/a.go", "go", "repoA")
|
||||
g.AddNode(n)
|
||||
|
||||
assert.NotNil(t, g.GetNodeByQualName("repoA.Foo"))
|
||||
g.EvictRepo("repoA")
|
||||
assert.Nil(t, g.GetNodeByQualName("repoA.Foo"))
|
||||
}
|
||||
|
||||
func TestRepoStats(t *testing.T) {
|
||||
g := New()
|
||||
g.AddNode(makeRepoNode("repoA/a.go::Foo", "Foo", KindFunction, "repoA/a.go", "go", "repoA"))
|
||||
g.AddNode(makeRepoNode("repoA/b.go::Bar", "Bar", KindType, "repoA/b.go", "go", "repoA"))
|
||||
g.AddNode(makeRepoNode("repoB/c.ts::Baz", "Baz", KindFunction, "repoB/c.ts", "typescript", "repoB"))
|
||||
|
||||
// Edge from repoA node.
|
||||
g.AddEdge(&Edge{From: "repoA/a.go::Foo", To: "repoA/b.go::Bar", Kind: EdgeCalls, FilePath: "repoA/a.go", Line: 1})
|
||||
// Cross-repo edge from repoA to repoB.
|
||||
g.AddEdge(&Edge{From: "repoA/a.go::Foo", To: "repoB/c.ts::Baz", Kind: EdgeCalls, FilePath: "repoA/a.go", Line: 2, CrossRepo: true})
|
||||
|
||||
stats := g.RepoStats()
|
||||
require.Len(t, stats, 2)
|
||||
|
||||
sA := stats["repoA"]
|
||||
assert.Equal(t, 2, sA.TotalNodes)
|
||||
assert.Equal(t, 2, sA.TotalEdges) // both edges originate from repoA
|
||||
assert.Equal(t, 1, sA.ByKind["function"])
|
||||
assert.Equal(t, 1, sA.ByKind["type"])
|
||||
assert.Equal(t, 2, sA.ByLanguage["go"])
|
||||
|
||||
sB := stats["repoB"]
|
||||
assert.Equal(t, 1, sB.TotalNodes)
|
||||
assert.Equal(t, 0, sB.TotalEdges) // no edges originate from repoB
|
||||
assert.Equal(t, 1, sB.ByKind["function"])
|
||||
assert.Equal(t, 1, sB.ByLanguage["typescript"])
|
||||
}
|
||||
|
||||
func TestRepoStats_Empty(t *testing.T) {
|
||||
g := New()
|
||||
stats := g.RepoStats()
|
||||
assert.Empty(t, stats)
|
||||
}
|
||||
|
||||
func TestRepoPrefixes(t *testing.T) {
|
||||
g := New()
|
||||
g.AddNode(makeRepoNode("repoA/a.go::Foo", "Foo", KindFunction, "repoA/a.go", "go", "repoA"))
|
||||
g.AddNode(makeRepoNode("repoB/b.go::Bar", "Bar", KindFunction, "repoB/b.go", "go", "repoB"))
|
||||
g.AddNode(makeRepoNode("repoA/c.go::Baz", "Baz", KindFunction, "repoA/c.go", "go", "repoA"))
|
||||
|
||||
prefixes := g.RepoPrefixes()
|
||||
assert.Len(t, prefixes, 2)
|
||||
assert.ElementsMatch(t, []string{"repoA", "repoB"}, prefixes)
|
||||
}
|
||||
|
||||
func TestRepoPrefixes_Empty(t *testing.T) {
|
||||
g := New()
|
||||
assert.Empty(t, g.RepoPrefixes())
|
||||
}
|
||||
|
||||
func TestEvictFile_CleansByRepoIndex(t *testing.T) {
|
||||
g := New()
|
||||
n1 := makeRepoNode("repoA/a.go::Foo", "Foo", KindFunction, "repoA/a.go", "go", "repoA")
|
||||
n2 := makeRepoNode("repoA/b.go::Bar", "Bar", KindFunction, "repoA/b.go", "go", "repoA")
|
||||
g.AddNode(n1)
|
||||
g.AddNode(n2)
|
||||
|
||||
assert.Len(t, g.GetRepoNodes("repoA"), 2)
|
||||
|
||||
g.EvictFile("repoA/a.go")
|
||||
|
||||
// Only n2 should remain in byRepo.
|
||||
repoNodes := g.GetRepoNodes("repoA")
|
||||
require.Len(t, repoNodes, 1)
|
||||
assert.Equal(t, "Bar", repoNodes[0].Name)
|
||||
}
|
||||
|
||||
func TestEvictFile_CleansByRepoIndex_LastNode(t *testing.T) {
|
||||
g := New()
|
||||
n := makeRepoNode("repoA/a.go::Foo", "Foo", KindFunction, "repoA/a.go", "go", "repoA")
|
||||
g.AddNode(n)
|
||||
|
||||
g.EvictFile("repoA/a.go")
|
||||
|
||||
assert.Empty(t, g.GetRepoNodes("repoA"))
|
||||
assert.Empty(t, g.RepoPrefixes())
|
||||
}
|
||||
|
||||
// Feature: multi-repo-support, Property 7: Per-repo index correctness
|
||||
|
||||
// genRepoPrefix generates a short repo prefix like "repoA", "repoB", etc.
|
||||
func genRepoPrefix() *rapid.Generator[string] {
|
||||
return rapid.Custom(func(t *rapid.T) string {
|
||||
return "repo" + rapid.StringMatching(`[A-Z][a-z]{0,4}`).Draw(t, "suffix")
|
||||
})
|
||||
}
|
||||
|
||||
// genMultiRepoGraph builds a graph with 2-5 repo prefixes and 1-10 nodes per repo,
|
||||
// plus cross-repo edges. Returns the graph and a map of prefix → expected node IDs.
|
||||
func genMultiRepoGraph(t *rapid.T) (*Graph, map[string][]string) {
|
||||
g := New()
|
||||
numRepos := rapid.IntRange(2, 5).Draw(t, "numRepos")
|
||||
|
||||
// Generate distinct prefixes.
|
||||
prefixSet := make(map[string]bool)
|
||||
var prefixes []string
|
||||
for len(prefixes) < numRepos {
|
||||
p := genRepoPrefix().Draw(t, "prefix")
|
||||
if !prefixSet[p] {
|
||||
prefixSet[p] = true
|
||||
prefixes = append(prefixes, p)
|
||||
}
|
||||
}
|
||||
|
||||
expected := make(map[string][]string) // prefix → node IDs
|
||||
var allNodeIDs []string
|
||||
kinds := []NodeKind{KindFunction, KindType, KindMethod, KindVariable}
|
||||
|
||||
for _, prefix := range prefixes {
|
||||
numNodes := rapid.IntRange(1, 10).Draw(t, "numNodes_"+prefix)
|
||||
for j := 0; j < numNodes; j++ {
|
||||
name := rapid.StringMatching(`[A-Z][a-zA-Z0-9]{1,10}`).Draw(t, "name")
|
||||
file := prefix + "/" + rapid.StringMatching(`[a-z]{1,8}`).Draw(t, "file") + ".go"
|
||||
id := file + "::" + name
|
||||
kind := kinds[rapid.IntRange(0, len(kinds)-1).Draw(t, "kind")]
|
||||
n := makeRepoNode(id, name, kind, file, "go", prefix)
|
||||
g.AddNode(n)
|
||||
expected[prefix] = append(expected[prefix], id)
|
||||
allNodeIDs = append(allNodeIDs, id)
|
||||
}
|
||||
}
|
||||
|
||||
// Add some edges including cross-repo ones.
|
||||
if len(allNodeIDs) >= 2 {
|
||||
numEdges := rapid.IntRange(1, len(allNodeIDs)).Draw(t, "numEdges")
|
||||
for i := 0; i < numEdges; i++ {
|
||||
fromIdx := rapid.IntRange(0, len(allNodeIDs)-1).Draw(t, "fromIdx")
|
||||
toIdx := rapid.IntRange(0, len(allNodeIDs)-1).Draw(t, "toIdx")
|
||||
if fromIdx == toIdx {
|
||||
continue
|
||||
}
|
||||
fromNode := g.GetNode(allNodeIDs[fromIdx])
|
||||
toNode := g.GetNode(allNodeIDs[toIdx])
|
||||
crossRepo := fromNode.RepoPrefix != toNode.RepoPrefix
|
||||
g.AddEdge(&Edge{
|
||||
From: allNodeIDs[fromIdx],
|
||||
To: allNodeIDs[toIdx],
|
||||
Kind: EdgeCalls,
|
||||
FilePath: fromNode.FilePath,
|
||||
Line: i + 1,
|
||||
CrossRepo: crossRepo,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
return g, expected
|
||||
}
|
||||
|
||||
// TestPropertyPerRepoIndexCorrectness verifies that GetRepoNodes returns exactly
|
||||
// the nodes with matching RepoPrefix, and the union of all GetRepoNodes equals
|
||||
// all nodes that have a RepoPrefix.
|
||||
func TestPropertyPerRepoIndexCorrectness(t *testing.T) {
|
||||
rapid.Check(t, func(rt *rapid.T) {
|
||||
g, expected := genMultiRepoGraph(rt)
|
||||
|
||||
// For each prefix, GetRepoNodes must return exactly the matching nodes.
|
||||
for prefix, wantIDs := range expected {
|
||||
got := g.GetRepoNodes(prefix)
|
||||
gotIDs := make([]string, len(got))
|
||||
for i, n := range got {
|
||||
gotIDs[i] = n.ID
|
||||
// Every returned node must have the correct RepoPrefix.
|
||||
assert.Equal(rt, prefix, n.RepoPrefix,
|
||||
"GetRepoNodes(%q) returned node %q with wrong RepoPrefix %q", prefix, n.ID, n.RepoPrefix)
|
||||
}
|
||||
assert.ElementsMatch(rt, wantIDs, gotIDs,
|
||||
"GetRepoNodes(%q) returned wrong set of node IDs", prefix)
|
||||
}
|
||||
|
||||
// Union of all GetRepoNodes for all prefixes must equal all nodes with a RepoPrefix.
|
||||
prefixes := g.RepoPrefixes()
|
||||
unionIDs := make(map[string]bool)
|
||||
for _, p := range prefixes {
|
||||
for _, n := range g.GetRepoNodes(p) {
|
||||
unionIDs[n.ID] = true
|
||||
}
|
||||
}
|
||||
|
||||
allNodes := g.AllNodes()
|
||||
repoNodeIDs := make(map[string]bool)
|
||||
for _, n := range allNodes {
|
||||
if n.RepoPrefix != "" {
|
||||
repoNodeIDs[n.ID] = true
|
||||
}
|
||||
}
|
||||
|
||||
assert.Equal(rt, repoNodeIDs, unionIDs,
|
||||
"Union of GetRepoNodes for all prefixes must equal all nodes with a RepoPrefix")
|
||||
})
|
||||
}
|
||||
|
||||
// Feature: multi-repo-support, Property 8: Repo eviction completeness
|
||||
|
||||
// TestPropertyRepoEvictionCompleteness verifies that after EvictRepo, zero nodes/edges
|
||||
// remain for the evicted prefix, and other repos are unchanged.
|
||||
func TestPropertyRepoEvictionCompleteness(t *testing.T) {
|
||||
rapid.Check(t, func(rt *rapid.T) {
|
||||
g, expected := genMultiRepoGraph(rt)
|
||||
|
||||
// Pick a repo to evict.
|
||||
prefixes := g.RepoPrefixes()
|
||||
require.NotEmpty(rt, prefixes, "graph must have at least one repo prefix")
|
||||
evictPrefix := prefixes[rapid.IntRange(0, len(prefixes)-1).Draw(rt, "evictIdx")]
|
||||
|
||||
// Record pre-eviction state for other repos.
|
||||
otherCounts := make(map[string]int)
|
||||
for _, p := range prefixes {
|
||||
if p != evictPrefix {
|
||||
otherCounts[p] = len(g.GetRepoNodes(p))
|
||||
}
|
||||
}
|
||||
|
||||
// Evict.
|
||||
nodesRm, _ := g.EvictRepo(evictPrefix)
|
||||
assert.Equal(rt, len(expected[evictPrefix]), nodesRm,
|
||||
"EvictRepo should remove exactly the expected number of nodes")
|
||||
|
||||
// Verify GetRepoNodes returns empty for evicted prefix.
|
||||
assert.Empty(rt, g.GetRepoNodes(evictPrefix),
|
||||
"GetRepoNodes(%q) must be empty after eviction", evictPrefix)
|
||||
|
||||
// Verify no nodes in AllNodes have the evicted prefix.
|
||||
for _, n := range g.AllNodes() {
|
||||
assert.NotEqual(rt, evictPrefix, n.RepoPrefix,
|
||||
"AllNodes() still contains node %q with evicted prefix %q", n.ID, evictPrefix)
|
||||
}
|
||||
|
||||
// Verify no edges reference evicted node IDs.
|
||||
evictedIDs := make(map[string]bool)
|
||||
for _, id := range expected[evictPrefix] {
|
||||
evictedIDs[id] = true
|
||||
}
|
||||
for _, e := range g.AllEdges() {
|
||||
assert.False(rt, evictedIDs[e.From],
|
||||
"Edge from %q → %q still references evicted node", e.From, e.To)
|
||||
assert.False(rt, evictedIDs[e.To],
|
||||
"Edge from %q → %q still references evicted node", e.From, e.To)
|
||||
}
|
||||
|
||||
// Verify other repos' node counts are unchanged.
|
||||
for p, wantCount := range otherCounts {
|
||||
gotCount := len(g.GetRepoNodes(p))
|
||||
assert.Equal(rt, wantCount, gotCount,
|
||||
"Repo %q node count changed after evicting %q", p, evictPrefix)
|
||||
}
|
||||
})
|
||||
}
|
||||
@@ -0,0 +1,661 @@
|
||||
package graph
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// TestAddNode_Idempotent proves the invariant the resilience work added
|
||||
// to the graph: N duplicate AddNode calls converge to the same Stats()
|
||||
// and the same secondary-index contents as a single call. Without this,
|
||||
// a daemon restart that loads a snapshot and then re-runs IndexCtx on
|
||||
// top of it (which doesn't evict first) produces N× the byFile /
|
||||
// byName / byRepo slice entries — the B1 symptom.
|
||||
func TestAddNode_Idempotent(t *testing.T) {
|
||||
g := New()
|
||||
n := &Node{
|
||||
ID: "repo/a.go::Foo",
|
||||
Name: "Foo",
|
||||
Kind: KindFunction,
|
||||
FilePath: "repo/a.go",
|
||||
QualName: "pkg.Foo",
|
||||
RepoPrefix: "repo",
|
||||
}
|
||||
|
||||
g.AddNode(n)
|
||||
base := g.Stats()
|
||||
require.Equal(t, 1, base.TotalNodes)
|
||||
|
||||
for i := 0; i < 10; i++ {
|
||||
g.AddNode(n)
|
||||
}
|
||||
|
||||
got := g.Stats()
|
||||
assert.Equal(t, base.TotalNodes, got.TotalNodes,
|
||||
"duplicate AddNode must not grow node count")
|
||||
|
||||
byFile := g.GetFileNodes("repo/a.go")
|
||||
assert.Len(t, byFile, 1, "byFile must not duplicate")
|
||||
byName := g.FindNodesByName("Foo")
|
||||
assert.Len(t, byName, 1, "byName must not duplicate")
|
||||
byRepo := g.GetRepoNodes("repo")
|
||||
assert.Len(t, byRepo, 1, "byRepo must not duplicate")
|
||||
assert.Equal(t, n, g.GetNodeByQualName("pkg.Foo"))
|
||||
}
|
||||
|
||||
// TestAddEdge_Idempotent is the edge counterpart of the node test. With
|
||||
// the same (From, To, Kind, FilePath, Line), repeated AddEdge calls
|
||||
// converge to a single adjacency-list entry. This is what made the
|
||||
// "edges double on every daemon restart" symptom recede.
|
||||
func TestAddEdge_Idempotent(t *testing.T) {
|
||||
g := New()
|
||||
g.AddNode(&Node{ID: "a::A", Name: "A", Kind: KindFunction, FilePath: "a"})
|
||||
g.AddNode(&Node{ID: "b::B", Name: "B", Kind: KindFunction, FilePath: "b"})
|
||||
|
||||
e := &Edge{From: "b::B", To: "a::A", Kind: EdgeCalls, FilePath: "b", Line: 7}
|
||||
for i := 0; i < 10; i++ {
|
||||
g.AddEdge(e)
|
||||
}
|
||||
|
||||
assert.Equal(t, 1, g.EdgeCount(), "duplicate AddEdge must not grow edge count")
|
||||
assert.Len(t, g.GetOutEdges("b::B"), 1, "outEdges must have exactly one entry")
|
||||
assert.Len(t, g.GetInEdges("a::A"), 1, "inEdges must have exactly one entry")
|
||||
}
|
||||
|
||||
// TestAddEdge_DifferentFromSameTo guards the edgeKey shape: two edges
|
||||
// with different From but identical (To, Kind, FilePath, Line) must
|
||||
// both survive, as distinct entries in the target's inEdges bucket.
|
||||
// An earlier version of the sidecar omitted From from the key, which
|
||||
// made two such edges collide at the inEdges[to] index — the second
|
||||
// AddEdge overwrote the first and downstream BFS traversal lost one
|
||||
// caller. Cross-repo impact analysis regressed until From landed in
|
||||
// the key.
|
||||
func TestAddEdge_DifferentFromSameTo(t *testing.T) {
|
||||
g := New()
|
||||
g.AddNode(&Node{ID: "target::T", Name: "T", Kind: KindFunction, FilePath: "t"})
|
||||
g.AddNode(&Node{ID: "caller1::C1", Name: "C1", Kind: KindFunction, FilePath: "c1"})
|
||||
g.AddNode(&Node{ID: "caller2::C2", Name: "C2", Kind: KindFunction, FilePath: "c2"})
|
||||
|
||||
// Both edges lack FilePath/Line — a common shape in tests that
|
||||
// construct synthetic graphs. Without From in the key they would
|
||||
// dedup to one inEdges entry.
|
||||
g.AddEdge(&Edge{From: "caller1::C1", To: "target::T", Kind: EdgeCalls})
|
||||
g.AddEdge(&Edge{From: "caller2::C2", To: "target::T", Kind: EdgeCalls})
|
||||
|
||||
in := g.GetInEdges("target::T")
|
||||
assert.Len(t, in, 2, "two distinct callers must both appear in inEdges")
|
||||
}
|
||||
|
||||
// TestAddEdge_LineDisambiguates proves that two call-sites from the
|
||||
// same caller to the same callee at different lines are preserved —
|
||||
// they're distinct edges, not duplicates. `foo(); foo();` in the same
|
||||
// function must survive dedup.
|
||||
func TestAddEdge_LineDisambiguates(t *testing.T) {
|
||||
g := New()
|
||||
g.AddNode(&Node{ID: "a::A", Name: "A", Kind: KindFunction, FilePath: "a"})
|
||||
g.AddNode(&Node{ID: "b::B", Name: "B", Kind: KindFunction, FilePath: "b"})
|
||||
|
||||
g.AddEdge(&Edge{From: "b::B", To: "a::A", Kind: EdgeCalls, FilePath: "b", Line: 7})
|
||||
g.AddEdge(&Edge{From: "b::B", To: "a::A", Kind: EdgeCalls, FilePath: "b", Line: 11})
|
||||
|
||||
assert.Equal(t, 2, g.EdgeCount(), "different lines must produce distinct edges")
|
||||
}
|
||||
|
||||
// TestAddNode_Replace verifies that re-adding a node with an updated
|
||||
// Meta preserves the slice positions and replaces the pointer in place.
|
||||
// This is the "same ID, new signature / new line" case that happens
|
||||
// during IncrementalReindex after a file edit.
|
||||
func TestAddNode_Replace(t *testing.T) {
|
||||
g := New()
|
||||
n1 := &Node{ID: "a::X", Name: "X", Kind: KindFunction, FilePath: "a",
|
||||
Meta: map[string]any{"signature": "X()"}}
|
||||
g.AddNode(n1)
|
||||
|
||||
n2 := &Node{ID: "a::X", Name: "X", Kind: KindFunction, FilePath: "a",
|
||||
Meta: map[string]any{"signature": "X(arg int)"}}
|
||||
g.AddNode(n2)
|
||||
|
||||
got := g.GetNode("a::X")
|
||||
require.NotNil(t, got)
|
||||
assert.Equal(t, "X(arg int)", got.Meta["signature"],
|
||||
"replacement must install new pointer")
|
||||
assert.Len(t, g.GetFileNodes("a"), 1, "byFile must not grow on replace")
|
||||
assert.Len(t, g.FindNodesByName("X"), 1, "byName must not grow on replace")
|
||||
// The slice entry must be the new pointer — readers iterate byFile
|
||||
// and rely on it reflecting the current node state.
|
||||
assert.Same(t, n2, g.GetFileNodes("a")[0])
|
||||
}
|
||||
|
||||
// TestAddNode_MigrateBuckets verifies that when a replacement changes
|
||||
// the node's FilePath / Name / RepoPrefix, the secondary-index entry
|
||||
// moves from the old bucket to the new one. Without this, a rename
|
||||
// (unusual but legal) would leave ghost entries in both buckets.
|
||||
func TestAddNode_MigrateBuckets(t *testing.T) {
|
||||
g := New()
|
||||
g.AddNode(&Node{ID: "x::X", Name: "OldName", Kind: KindFunction,
|
||||
FilePath: "old.go", RepoPrefix: "oldrepo", QualName: "pkg.Old"})
|
||||
g.AddNode(&Node{ID: "x::X", Name: "NewName", Kind: KindFunction,
|
||||
FilePath: "new.go", RepoPrefix: "newrepo", QualName: "pkg.New"})
|
||||
|
||||
assert.Empty(t, g.GetFileNodes("old.go"), "old bucket must be emptied")
|
||||
assert.Len(t, g.GetFileNodes("new.go"), 1, "new bucket must have the entry")
|
||||
assert.Empty(t, g.FindNodesByName("OldName"))
|
||||
assert.Len(t, g.FindNodesByName("NewName"), 1)
|
||||
assert.Empty(t, g.GetRepoNodes("oldrepo"))
|
||||
assert.Len(t, g.GetRepoNodes("newrepo"), 1)
|
||||
assert.Nil(t, g.GetNodeByQualName("pkg.Old"))
|
||||
assert.NotNil(t, g.GetNodeByQualName("pkg.New"))
|
||||
}
|
||||
|
||||
// TestAddNode_PreservesRepoPrefixOnEmptyDowngrade pins the warmup bug
|
||||
// where some path re-AddNode'd existing repo-stamped nodes with
|
||||
// RepoPrefix="" — clearing them out of byRepo[prefix] without touching
|
||||
// the underlying nodes map. The user-visible symptom: per-repo queries
|
||||
// (RepoStats / GetRepoNodes / RepoMemoryEstimate) returned empty for
|
||||
// repos whose nodes were still present in the graph. Defensive fix:
|
||||
// a non-empty prev RepoPrefix is sticky — the empty new value is
|
||||
// promoted to prev's value rather than allowed to silently strip the
|
||||
// node from its bucket.
|
||||
func TestAddNode_PreservesRepoPrefixOnEmptyDowngrade(t *testing.T) {
|
||||
g := New()
|
||||
original := &Node{
|
||||
ID: "myrepo/file.go::Foo", Name: "Foo", Kind: KindFunction,
|
||||
FilePath: "myrepo/file.go", RepoPrefix: "myrepo",
|
||||
}
|
||||
g.AddNode(original)
|
||||
require.Len(t, g.GetRepoNodes("myrepo"), 1, "node must land in byRepo at first add")
|
||||
|
||||
// Re-add with empty RepoPrefix (the buggy caller).
|
||||
g.AddNode(&Node{
|
||||
ID: "myrepo/file.go::Foo", Name: "Foo", Kind: KindFunction,
|
||||
FilePath: "myrepo/file.go",
|
||||
// RepoPrefix intentionally empty.
|
||||
})
|
||||
|
||||
assert.Len(t, g.GetRepoNodes("myrepo"), 1,
|
||||
"byRepo[myrepo] must still contain the node after empty-prefix re-add")
|
||||
assert.NotNil(t, g.GetNode("myrepo/file.go::Foo"),
|
||||
"node itself must still exist")
|
||||
assert.Equal(t, "myrepo", g.GetNode("myrepo/file.go::Foo").RepoPrefix,
|
||||
"RepoPrefix on the stored node must be preserved")
|
||||
}
|
||||
|
||||
// TestEvictFile_SwapWithLast exercises the sidecar-based swap-with-last
|
||||
// removal path. Uses enough nodes per file that iteration order would
|
||||
// surface a mis-tracked sidecar position. The assertion is simple: post
|
||||
// eviction, the graph is empty of entries for that file.
|
||||
func TestEvictFile_SwapWithLast(t *testing.T) {
|
||||
g := New()
|
||||
for i := 0; i < 100; i++ {
|
||||
g.AddNode(&Node{
|
||||
ID: fmt.Sprintf("f.go::Sym%d", i),
|
||||
Name: fmt.Sprintf("Sym%d", i),
|
||||
Kind: KindFunction,
|
||||
FilePath: "f.go",
|
||||
})
|
||||
}
|
||||
assert.Len(t, g.GetFileNodes("f.go"), 100)
|
||||
|
||||
n, _ := g.EvictFile("f.go")
|
||||
assert.Equal(t, 100, n)
|
||||
assert.Empty(t, g.GetFileNodes("f.go"))
|
||||
assert.Equal(t, 0, g.NodeCount())
|
||||
}
|
||||
|
||||
// TestRestartStability simulates the daemon-restart cycle: snapshot
|
||||
// into a fresh graph (via AddNode/AddEdge replay, which is what
|
||||
// loadSnapshot does), and verify Stats() matches the original. Repeat
|
||||
// many times to catch any state that drifts across restarts.
|
||||
//
|
||||
// Before the sidecar landed, Stats().TotalEdges doubled on every cycle;
|
||||
// after, the invariant holds for arbitrary N.
|
||||
func TestRestartStability(t *testing.T) {
|
||||
orig := buildRepresentativeGraph()
|
||||
want := orig.Stats()
|
||||
|
||||
for cycle := 0; cycle < 5; cycle++ {
|
||||
replay := New()
|
||||
for _, n := range orig.AllNodes() {
|
||||
replay.AddNode(n)
|
||||
}
|
||||
for _, e := range orig.AllEdges() {
|
||||
replay.AddEdge(e)
|
||||
}
|
||||
|
||||
// Simulate a second "IndexCtx on top" pass — this is what
|
||||
// the old warmup did after loadSnapshot. Without idempotent
|
||||
// writes, this pass doubles every edge.
|
||||
for _, n := range orig.AllNodes() {
|
||||
replay.AddNode(n)
|
||||
}
|
||||
for _, e := range orig.AllEdges() {
|
||||
replay.AddEdge(e)
|
||||
}
|
||||
|
||||
got := replay.Stats()
|
||||
assert.Equal(t, want.TotalNodes, got.TotalNodes,
|
||||
"cycle %d: node count drifted", cycle)
|
||||
assert.Equal(t, want.TotalEdges, got.TotalEdges,
|
||||
"cycle %d: edge count drifted (B1 regression)", cycle)
|
||||
}
|
||||
}
|
||||
|
||||
func buildRepresentativeGraph() *Graph {
|
||||
g := New()
|
||||
// Build a small call graph that stresses every secondary index:
|
||||
// multiple files, multiple names, multiple repos, calls + imports.
|
||||
files := []struct {
|
||||
path, repo string
|
||||
}{
|
||||
{"r1/a.go", "r1"},
|
||||
{"r1/b.go", "r1"},
|
||||
{"r2/c.go", "r2"},
|
||||
}
|
||||
for _, f := range files {
|
||||
for i := 0; i < 5; i++ {
|
||||
g.AddNode(&Node{
|
||||
ID: fmt.Sprintf("%s::Fn%d", f.path, i),
|
||||
Name: fmt.Sprintf("Fn%d", i),
|
||||
Kind: KindFunction,
|
||||
FilePath: f.path,
|
||||
RepoPrefix: f.repo,
|
||||
})
|
||||
}
|
||||
}
|
||||
// Add a few call edges between files.
|
||||
g.AddEdge(&Edge{From: "r1/a.go::Fn0", To: "r1/b.go::Fn1", Kind: EdgeCalls, FilePath: "r1/a.go", Line: 10})
|
||||
g.AddEdge(&Edge{From: "r1/a.go::Fn0", To: "r2/c.go::Fn2", Kind: EdgeCalls, FilePath: "r1/a.go", Line: 12})
|
||||
g.AddEdge(&Edge{From: "r1/b.go::Fn3", To: "r2/c.go::Fn4", Kind: EdgeCalls, FilePath: "r1/b.go", Line: 5})
|
||||
return g
|
||||
}
|
||||
|
||||
// TestReindexEdge_UpdatesSidecar verifies ReindexEdge migrates the
|
||||
// inEdges bucket + both sidecars when the resolver changes an edge's
|
||||
// To field (unresolved::X → real::X). A bug here would show up as
|
||||
// GetInEdges returning zero entries after resolve, or later AddEdge
|
||||
// refusing to dedup because the key changed out from under the sidecar.
|
||||
func TestReindexEdge_UpdatesSidecar(t *testing.T) {
|
||||
g := New()
|
||||
g.AddNode(&Node{ID: "a::A", Name: "A", Kind: KindFunction, FilePath: "a"})
|
||||
g.AddNode(&Node{ID: "b::B", Name: "B", Kind: KindFunction, FilePath: "b"})
|
||||
g.AddNode(&Node{ID: "unresolved::real", Name: "real", Kind: KindFunction, FilePath: "u"})
|
||||
|
||||
e := &Edge{From: "a::A", To: "unresolved::real", Kind: EdgeCalls, FilePath: "a", Line: 3}
|
||||
g.AddEdge(e)
|
||||
|
||||
require.Len(t, g.GetInEdges("unresolved::real"), 1)
|
||||
require.Len(t, g.GetInEdges("b::B"), 0)
|
||||
|
||||
// Resolver-style mutation.
|
||||
oldTo := e.To
|
||||
e.To = "b::B"
|
||||
g.ReindexEdge(e, oldTo)
|
||||
|
||||
assert.Len(t, g.GetInEdges("unresolved::real"), 0,
|
||||
"old target bucket must be emptied")
|
||||
assert.Len(t, g.GetInEdges("b::B"), 1,
|
||||
"new target bucket must hold the edge")
|
||||
|
||||
// Adding the same edge with its NEW identity must dedup via the
|
||||
// updated sidecar — if ReindexEdge forgot to rewrite the
|
||||
// outEdgeIdx key, this would append a duplicate.
|
||||
g.AddEdge(e)
|
||||
assert.Equal(t, 1, g.EdgeCount(), "AddEdge after ReindexEdge must still dedup")
|
||||
}
|
||||
|
||||
// TestRemoveEdgeFromBucket_SwappedEdgeWithMutatedTo regresses a daemon
|
||||
// crash:
|
||||
//
|
||||
// panic: runtime error: index out of range [N] with length N
|
||||
// graph.addEdgeToBucket
|
||||
// graph.(*Graph).ReindexEdge
|
||||
// resolver.(*Resolver).ResolveAll
|
||||
//
|
||||
// The resolver's serial pass mutates `j.edge.To = j.newTo` BEFORE
|
||||
// taking the shard lock. If the swap-with-last in
|
||||
// removeEdgeFromBucket lands on an edge whose .To was mutated in the
|
||||
// same flight (e.g. another job in the same bucket), recomputing
|
||||
// keyOf(swapped) returns the NEW key while the sidecar still has an
|
||||
// entry under the ORIGINAL key pointing past the shrunk slice. The
|
||||
// next AddEdge that collides with the orphaned key panics.
|
||||
//
|
||||
// The fix stores each entry's insertion-time edgeKey in a parallel
|
||||
// slice (outEdgeKeys / inEdgeKeys) so the sidecar update is
|
||||
// independent of the live Edge struct.
|
||||
func TestRemoveEdgeFromBucket_SwappedEdgeWithMutatedTo(t *testing.T) {
|
||||
g := New()
|
||||
g.AddNode(&Node{ID: "a::A", Name: "A", Kind: KindFunction, FilePath: "a"})
|
||||
g.AddNode(&Node{ID: "b::B", Name: "B", Kind: KindFunction, FilePath: "b"})
|
||||
g.AddNode(&Node{ID: "x::X", Name: "X", Kind: KindFunction, FilePath: "x"})
|
||||
g.AddNode(&Node{ID: "y::Y", Name: "Y", Kind: KindFunction, FilePath: "y"})
|
||||
|
||||
// Two edges share an unresolved bucket. We'll mutate eSwapped's To
|
||||
// out-of-band (mimicking the resolver's pre-lock mutation) before
|
||||
// removing eHead, forcing eSwapped to be the swap-with-last
|
||||
// element. With the bug, the sidecar update used keyOf(eSwapped)
|
||||
// — a different key than the one eSwapped was indexed under —
|
||||
// leaving a stale entry that pointed past the shrunk slice.
|
||||
const target = "unresolved::shared"
|
||||
eHead := &Edge{From: "a::A", To: target, Kind: EdgeCalls, FilePath: "a", Line: 1}
|
||||
eSwapped := &Edge{From: "b::B", To: target, Kind: EdgeCalls, FilePath: "b", Line: 2}
|
||||
g.AddEdge(eHead)
|
||||
g.AddEdge(eSwapped)
|
||||
require.Len(t, g.GetInEdges(target), 2)
|
||||
|
||||
// Out-of-band mutation: eSwapped.To changes BUT we don't yet
|
||||
// ReindexEdge. This models the in-flight window in
|
||||
// resolver.go's serial pass.
|
||||
eSwapped.To = "x::X"
|
||||
|
||||
// Now remove eHead via ReindexEdge — this triggers the swap that
|
||||
// previously corrupted the sidecar.
|
||||
oldHead := target
|
||||
eHead.To = "y::Y"
|
||||
g.ReindexEdge(eHead, oldHead)
|
||||
|
||||
// With the bug, inEdgeIdx[target] still held an orphan entry under
|
||||
// eSwapped's ORIGINAL key (To=target) at position 1 — past the
|
||||
// now-shrunk slice (length 1, valid index only 0). Any subsequent
|
||||
// AddEdge whose key collides with that stale entry would do
|
||||
// `bucket[target][1] = newEdge` and panic with
|
||||
// "index out of range [1] with length 1".
|
||||
//
|
||||
// Construct exactly that collision: a fresh edge sharing
|
||||
// eSwapped's original (From, To, Kind, FilePath, Line) tuple,
|
||||
// which is what the resolver does when it pre-stages a duplicate
|
||||
// pending edge from another file at the same line.
|
||||
collision := &Edge{From: "b::B", To: target, Kind: EdgeCalls, FilePath: "b", Line: 2}
|
||||
require.NotPanics(t, func() {
|
||||
g.AddEdge(collision)
|
||||
}, "addEdgeToBucket must not panic on a stale sidecar position")
|
||||
|
||||
// eHead has been migrated to its new target.
|
||||
assert.Len(t, g.GetInEdges("y::Y"), 1, "eHead's new target should hold one edge")
|
||||
}
|
||||
|
||||
// TestReindexEdge_OutEdgeKeysStayConsistent regresses the daemon
|
||||
// warmup panic:
|
||||
//
|
||||
// panic: runtime error: index out of range [61] with length 58
|
||||
// graph.removeEdgeFromBucket
|
||||
// graph.(*Graph).evictEdgesLocked
|
||||
// graph.(*Graph).EvictFile
|
||||
// indexer.(*Indexer).indexFile
|
||||
// indexer.(*Indexer).IncrementalReindex
|
||||
//
|
||||
// The failure mode: ReindexEdge updates outEdgeIdx[oldKey→newKey] but
|
||||
// previously did NOT update the parallel outEdgeKeys[pos] slice. A
|
||||
// later swap-with-last removal in the same outEdges bucket reads
|
||||
// outEdgeKeys[swappedPos] — finds the stale insertion-time key — and
|
||||
// re-inserts THAT key into outEdgeIdx pointing at the swapped slot.
|
||||
// outEdgeIdx then holds both the live newKey (still pointing at the
|
||||
// original pre-swap position) AND a stale-key entry. The next op
|
||||
// that walks back to the original pos finds the slice has shrunk
|
||||
// past it and panics.
|
||||
//
|
||||
// The fix: ReindexEdge rewrites outEdgeKeys[pos] = newKey alongside
|
||||
// the outEdgeIdx update so the parallel slice never holds stale keys.
|
||||
func TestReindexEdge_OutEdgeKeysStayConsistent(t *testing.T) {
|
||||
g := New()
|
||||
g.AddNode(&Node{ID: "a::A", Name: "A", Kind: KindFunction, FilePath: "a"})
|
||||
g.AddNode(&Node{ID: "t1", Name: "t1", Kind: KindFunction, FilePath: "t1"})
|
||||
g.AddNode(&Node{ID: "t2", Name: "t2", Kind: KindFunction, FilePath: "t2"})
|
||||
g.AddNode(&Node{ID: "t3", Name: "t3", Kind: KindFunction, FilePath: "t3"})
|
||||
g.AddNode(&Node{ID: "t2-prime", Name: "t2'", Kind: KindFunction, FilePath: "t2p"})
|
||||
g.AddNode(&Node{ID: "t2-prime-prime", Name: "t2''", Kind: KindFunction, FilePath: "t2pp"})
|
||||
|
||||
// Three edges share the same From, populating one outEdges bucket
|
||||
// with three slots. Distinct lines so the keys differ.
|
||||
e1 := &Edge{From: "a::A", To: "t1", Kind: EdgeCalls, FilePath: "a", Line: 1}
|
||||
e2 := &Edge{From: "a::A", To: "t2", Kind: EdgeCalls, FilePath: "a", Line: 2}
|
||||
e3 := &Edge{From: "a::A", To: "t3", Kind: EdgeCalls, FilePath: "a", Line: 3}
|
||||
g.AddEdge(e1)
|
||||
g.AddEdge(e2)
|
||||
g.AddEdge(e3)
|
||||
require.Len(t, g.GetOutEdges("a::A"), 3)
|
||||
|
||||
// ReindexEdge e2 — outEdgeKeys[1] would stay stale before the fix.
|
||||
oldTo := e2.To
|
||||
e2.To = "t2-prime"
|
||||
g.ReindexEdge(e2, oldTo)
|
||||
|
||||
// Force a swap-with-last in the outEdges["a::A"] bucket by
|
||||
// removing e1. With the bug, this propagates the stale key for
|
||||
// slot 1 (e2's original key) into outEdgeIdx.
|
||||
require.True(t, g.RemoveEdge(e1.From, e1.To, e1.Kind))
|
||||
|
||||
// ReindexEdge e2 a second time — drives outEdgeIdx into the
|
||||
// inconsistent state where it holds both the new key and the
|
||||
// stale key from the previous swap.
|
||||
oldTo = e2.To
|
||||
e2.To = "t2-prime-prime"
|
||||
g.ReindexEdge(e2, oldTo)
|
||||
|
||||
// Removal that touches the bucket must NOT panic. With the bug,
|
||||
// removing e3 via its resolved key triggered
|
||||
// `slice[pos] = slice[last]` with pos past the shrunk slice.
|
||||
require.NotPanics(t, func() {
|
||||
g.RemoveEdge(e3.From, e3.To, e3.Kind)
|
||||
}, "swap-with-last after repeated ReindexEdge must not panic")
|
||||
|
||||
// e2 still queryable at its final target — sanity check that the
|
||||
// bucket bookkeeping survived intact.
|
||||
out := g.GetOutEdges("a::A")
|
||||
require.Len(t, out, 1)
|
||||
assert.Equal(t, "t2-prime-prime", out[0].To)
|
||||
}
|
||||
|
||||
// TestEvictFile_AfterReindex regresses the same panic via the actual
|
||||
// eviction path the daemon hit (EvictFile → evictEdgesLocked) instead
|
||||
// of going through the public RemoveEdge API. The fixture stages the
|
||||
// exact corruption window the daemon panic describes:
|
||||
//
|
||||
// 1. A multi-edge outEdges bucket on a single From.
|
||||
// 2. ReindexEdge against a non-last slot in that bucket — outEdgeKeys
|
||||
// for that slot becomes stale (still holds the pre-mutation key).
|
||||
// 3. A swap-with-last removal earlier in the bucket pulls the stale
|
||||
// key into outEdgeIdx pointing at the swapped position.
|
||||
// 4. The slice subsequently shrinks past that position.
|
||||
// 5. EvictFile on the reindexed edge's NEW target then walks
|
||||
// inEdges[that target], grabs the still-correct live key from
|
||||
// inEdgeKeys, and calls removeEdgeFromBucket(outEdges, ...) on
|
||||
// the From bucket. With the bug, outEdgeIdx still has the live
|
||||
// key pointing past the now-shrunk slice → panic.
|
||||
func TestEvictFile_AfterReindex(t *testing.T) {
|
||||
g := New()
|
||||
g.AddNode(&Node{ID: "src/a.go::A", Name: "A", Kind: KindFunction, FilePath: "src/a.go"})
|
||||
g.AddNode(&Node{ID: "t1.go::T1", Name: "T1", Kind: KindFunction, FilePath: "t1.go"})
|
||||
g.AddNode(&Node{ID: "t2.go::T2", Name: "T2", Kind: KindFunction, FilePath: "t2.go"})
|
||||
g.AddNode(&Node{ID: "t3.go::T3", Name: "T3", Kind: KindFunction, FilePath: "t3.go"})
|
||||
g.AddNode(&Node{ID: "t2p.go::T2P", Name: "T2P", Kind: KindFunction, FilePath: "t2p.go"})
|
||||
|
||||
// Three outgoing edges from A — slot 1 is the one we'll reindex.
|
||||
e1 := &Edge{From: "src/a.go::A", To: "t1.go::T1", Kind: EdgeCalls, FilePath: "src/a.go", Line: 1}
|
||||
e2 := &Edge{From: "src/a.go::A", To: "t2.go::T2", Kind: EdgeCalls, FilePath: "src/a.go", Line: 2}
|
||||
e3 := &Edge{From: "src/a.go::A", To: "t3.go::T3", Kind: EdgeCalls, FilePath: "src/a.go", Line: 3}
|
||||
g.AddEdge(e1)
|
||||
g.AddEdge(e2)
|
||||
g.AddEdge(e3)
|
||||
|
||||
// Step 1: reindex e2's To. Without the fix, outEdgeKeys[1] keeps
|
||||
// the pre-mutation key while outEdgeIdx swaps to the new key.
|
||||
old := e2.To
|
||||
e2.To = "t2p.go::T2P"
|
||||
g.ReindexEdge(e2, old)
|
||||
|
||||
// Step 2: evict T1 — its inEdges bucket holds e1; Phase 2 of
|
||||
// evictEdgesLocked calls removeEdgeFromBucket(outEdges["src/a.go::A"], k_for_e1).
|
||||
// Inside, swap-with-last picks slot 2's key (k_for_e3 — correct)
|
||||
// because slot 2 is what the swap consumes. So no panic yet, but
|
||||
// after the swap the bucket is shape [e3, e2] with outEdgeKeys
|
||||
// = [k_for_e3, STALE_pre-reindex_e2_key].
|
||||
require.NotPanics(t, func() { g.EvictFile("t1.go") })
|
||||
|
||||
// Step 3: evict T3 — its inEdges bucket now points at the
|
||||
// swapped slot 0 (e3). removeEdgeFromBucket(outEdges, k_for_e3)
|
||||
// runs, swap-with-last picks up outEdgeKeys[1] which is the
|
||||
// STALE key. With the bug, that stale key gets re-inserted into
|
||||
// outEdgeIdx at position 0 alongside the still-live e2 key
|
||||
// (which now points at position 1, but the slice has shrunk to
|
||||
// length 1).
|
||||
require.NotPanics(t, func() { g.EvictFile("t3.go") })
|
||||
|
||||
// Step 4: evict T2P. inEdges[T2P] holds e2 with inEdgeKeys
|
||||
// carrying the LIVE key (insertion via addEdgeToBucket during
|
||||
// ReindexEdge used the new key). removeEdgeFromBucket(outEdges
|
||||
// ["src/a.go::A"], LIVE_key) looks up outEdgeIdx[LIVE_key] = 1,
|
||||
// then tries slice[1] in a slice of length 1 → panic with the
|
||||
// bug, clean removal with the fix.
|
||||
require.NotPanics(t, func() {
|
||||
g.EvictFile("t2p.go")
|
||||
}, "EvictFile on the reindexed edge's new target must not panic on stale outEdgeIdx")
|
||||
|
||||
// All edges removed — bucket should be empty.
|
||||
assert.Empty(t, g.GetOutEdges("src/a.go::A"), "outEdges bucket must drain after every target was evicted")
|
||||
}
|
||||
|
||||
// edgeIdentityGraph builds a two-node graph with one A→B calls edge at
|
||||
// the given Origin, returning the graph and the live in-graph edge.
|
||||
func edgeIdentityGraph(t *testing.T, origin string) (*Graph, *Edge) {
|
||||
t.Helper()
|
||||
g := New()
|
||||
g.AddNode(&Node{ID: "p/a.go::A", Name: "A", Kind: KindFunction, FilePath: "p/a.go"})
|
||||
g.AddNode(&Node{ID: "p/b.go::B", Name: "B", Kind: KindFunction, FilePath: "p/b.go"})
|
||||
g.AddEdge(&Edge{From: "p/a.go::A", To: "p/b.go::B", Kind: EdgeCalls, FilePath: "p/a.go", Line: 7, Origin: origin})
|
||||
out := g.GetOutEdges("p/a.go::A")
|
||||
require.Len(t, out, 1)
|
||||
return g, out[0]
|
||||
}
|
||||
|
||||
// TestSetEdgeProvenance_ChangesIdentityAndCounts proves SetEdgeProvenance
|
||||
// is a delete-then-insert of the edge's identity: a real Origin change
|
||||
// flips the IdentityHash and bumps the revision counter by exactly one,
|
||||
// while the logical adjacency-list slot is untouched.
|
||||
func TestSetEdgeProvenance_ChangesIdentityAndCounts(t *testing.T) {
|
||||
g, e := edgeIdentityGraph(t, OriginTextMatched)
|
||||
require.Equal(t, 0, g.EdgeIdentityRevisions(), "fresh graph has no provenance churn")
|
||||
|
||||
before := e.IdentityHash()
|
||||
changed := g.SetEdgeProvenance(e, OriginLSPResolved)
|
||||
|
||||
assert.True(t, changed, "upgrading Origin must report an identity change")
|
||||
assert.Equal(t, OriginLSPResolved, e.Origin, "Origin must be applied")
|
||||
assert.NotEqual(t, before, e.IdentityHash(), "identity hash must change with Origin")
|
||||
assert.Equal(t, 1, g.EdgeIdentityRevisions(), "exactly one revision recorded")
|
||||
|
||||
// The logical edge is unchanged — same single adjacency entry.
|
||||
assert.Len(t, g.GetOutEdges("p/a.go::A"), 1, "outEdges count must not change")
|
||||
assert.Len(t, g.GetInEdges("p/b.go::B"), 1, "inEdges count must not change")
|
||||
}
|
||||
|
||||
// TestSetEdgeProvenance_NoOpWhenOriginUnchanged proves a SetEdgeProvenance
|
||||
// call that does not actually change Origin is a no-op: identity stable,
|
||||
// counter untouched, return value false.
|
||||
func TestSetEdgeProvenance_NoOpWhenOriginUnchanged(t *testing.T) {
|
||||
g, e := edgeIdentityGraph(t, OriginASTResolved)
|
||||
before := e.IdentityHash()
|
||||
|
||||
changed := g.SetEdgeProvenance(e, OriginASTResolved)
|
||||
|
||||
assert.False(t, changed, "setting Origin to its current value is a no-op")
|
||||
assert.Equal(t, before, e.IdentityHash(), "identity hash must be stable on a no-op")
|
||||
assert.Equal(t, 0, g.EdgeIdentityRevisions(), "a no-op must not bump the counter")
|
||||
}
|
||||
|
||||
// TestSetEdgeProvenance_RederivesTierWhenSet confirms Tier — the sole
|
||||
// Origin-derived label on an edge — is recomputed when it was already
|
||||
// populated, and left empty (the in-memory default) when it was not.
|
||||
func TestSetEdgeProvenance_RederivesTierWhenSet(t *testing.T) {
|
||||
// Tier already set: must be re-derived from the new Origin.
|
||||
g, e := edgeIdentityGraph(t, OriginTextMatched)
|
||||
e.Tier = ResolvedBy(OriginTextMatched)
|
||||
g.SetEdgeProvenance(e, OriginLSPResolved)
|
||||
assert.Equal(t, ResolvedBy(OriginLSPResolved), e.Tier, "populated Tier must track the new Origin")
|
||||
|
||||
// Tier left empty: must stay empty rather than start being stamped.
|
||||
g2, e2 := edgeIdentityGraph(t, OriginTextMatched)
|
||||
g2.SetEdgeProvenance(e2, OriginLSPResolved)
|
||||
assert.Equal(t, "", e2.Tier, "an unset Tier must remain unset")
|
||||
}
|
||||
|
||||
// TestAddEdge_ReaddWithUpgradedOriginCounts proves the second mutation
|
||||
// path: re-adding an edge with the same logical key but an upgraded
|
||||
// Origin (the resolver's AddEdge-based upgrade path) replaces the slot
|
||||
// in place AND is counted as an identity revision — without creating a
|
||||
// duplicate parallel edge.
|
||||
func TestAddEdge_ReaddWithUpgradedOriginCounts(t *testing.T) {
|
||||
g, _ := edgeIdentityGraph(t, OriginTextMatched)
|
||||
require.Equal(t, 0, g.EdgeIdentityRevisions())
|
||||
|
||||
// Re-add the same logical edge with a stronger Origin.
|
||||
g.AddEdge(&Edge{From: "p/a.go::A", To: "p/b.go::B", Kind: EdgeCalls, FilePath: "p/a.go", Line: 7, Origin: OriginLSPResolved})
|
||||
|
||||
assert.Equal(t, 1, g.EdgeCount(), "re-add must not create a parallel edge")
|
||||
assert.Len(t, g.GetOutEdges("p/a.go::A"), 1, "still one outEdge")
|
||||
assert.Len(t, g.GetInEdges("p/b.go::B"), 1, "still one inEdge")
|
||||
assert.Equal(t, 1, g.EdgeIdentityRevisions(), "the Origin upgrade on re-add must be counted once")
|
||||
assert.Equal(t, OriginLSPResolved, g.GetOutEdges("p/a.go::A")[0].Origin, "newer Origin wins")
|
||||
}
|
||||
|
||||
// TestAddEdge_ReaddWithSameOriginDoesNotCount proves an idempotent
|
||||
// re-add carrying the SAME Origin is not mistaken for provenance churn.
|
||||
func TestAddEdge_ReaddWithSameOriginDoesNotCount(t *testing.T) {
|
||||
g, _ := edgeIdentityGraph(t, OriginASTResolved)
|
||||
for i := 0; i < 5; i++ {
|
||||
g.AddEdge(&Edge{From: "p/a.go::A", To: "p/b.go::B", Kind: EdgeCalls, FilePath: "p/a.go", Line: 7, Origin: OriginASTResolved})
|
||||
}
|
||||
assert.Equal(t, 1, g.EdgeCount(), "idempotent re-add must not grow the edge count")
|
||||
assert.Equal(t, 0, g.EdgeIdentityRevisions(), "re-add with an unchanged Origin is not a revision")
|
||||
}
|
||||
|
||||
// TestVerifyEdgeIdentities_PassesOnNormalGraph proves a graph built
|
||||
// only through the sanctioned mutation paths (AddEdge, SetEdgeProvenance)
|
||||
// is internally consistent — the out-edge and in-edge views agree on
|
||||
// every edge's provenance-bearing identity.
|
||||
func TestVerifyEdgeIdentities_PassesOnNormalGraph(t *testing.T) {
|
||||
g := New()
|
||||
for _, id := range []string{"p/a.go::A", "p/b.go::B", "p/c.go::C"} {
|
||||
g.AddNode(&Node{ID: id, Name: id, Kind: KindFunction, FilePath: id})
|
||||
}
|
||||
g.AddEdge(&Edge{From: "p/a.go::A", To: "p/b.go::B", Kind: EdgeCalls, FilePath: "p/a.go", Line: 3, Origin: OriginTextMatched})
|
||||
g.AddEdge(&Edge{From: "p/a.go::A", To: "p/c.go::C", Kind: EdgeCalls, FilePath: "p/a.go", Line: 4, Origin: OriginASTResolved})
|
||||
g.AddEdge(&Edge{From: "p/b.go::B", To: "p/c.go::C", Kind: EdgeReferences, FilePath: "p/b.go", Line: 9})
|
||||
|
||||
require.NoError(t, g.VerifyEdgeIdentities(), "freshly built graph must be identity-consistent")
|
||||
|
||||
// A sanctioned provenance change keeps the graph consistent.
|
||||
out := g.GetOutEdges("p/a.go::A")
|
||||
require.NotEmpty(t, out)
|
||||
g.SetEdgeProvenance(out[0], OriginLSPResolved)
|
||||
require.NoError(t, g.VerifyEdgeIdentities(), "SetEdgeProvenance must preserve identity consistency")
|
||||
}
|
||||
|
||||
// TestVerifyEdgeIdentities_CatchesDivergentOrigin proves the verifier
|
||||
// is not vacuous: when an edge's Origin is changed on only one
|
||||
// adjacency view (the failure mode of mutating a copied edge instead
|
||||
// of routing through SetEdgeProvenance), VerifyEdgeIdentities reports
|
||||
// the inconsistency.
|
||||
func TestVerifyEdgeIdentities_CatchesDivergentOrigin(t *testing.T) {
|
||||
g := New()
|
||||
g.AddNode(&Node{ID: "p/a.go::A", Name: "A", Kind: KindFunction, FilePath: "p/a.go"})
|
||||
g.AddNode(&Node{ID: "p/b.go::B", Name: "B", Kind: KindFunction, FilePath: "p/b.go"})
|
||||
g.AddEdge(&Edge{From: "p/a.go::A", To: "p/b.go::B", Kind: EdgeCalls, FilePath: "p/a.go", Line: 7, Origin: OriginTextMatched})
|
||||
require.NoError(t, g.VerifyEdgeIdentities())
|
||||
|
||||
// Simulate the bug: the in-edge bucket gets a *different* edge
|
||||
// object whose Origin diverges from the out-edge view. addEdgeToBucket
|
||||
// keys on the Origin-free logical key, so this overwrites the slot
|
||||
// with a copy rather than appending.
|
||||
sTo := g.shardFor("p/b.go::B")
|
||||
sTo.mu.Lock()
|
||||
divergent := &Edge{From: "p/a.go::A", To: "p/b.go::B", Kind: EdgeCalls, FilePath: "p/a.go", Line: 7, Origin: OriginLSPResolved}
|
||||
addEdgeToBucket(sTo.inEdges, sTo.inEdgeKeys, sTo.inEdgeIdx, "p/b.go::B", divergent)
|
||||
sTo.mu.Unlock()
|
||||
|
||||
err := g.VerifyEdgeIdentities()
|
||||
require.Error(t, err, "a divergent-Origin edge across adjacency views must be caught")
|
||||
assert.Contains(t, err.Error(), "p/a.go::A", "the error must name the offending edge")
|
||||
}
|
||||
@@ -0,0 +1,46 @@
|
||||
package graph
|
||||
|
||||
// RepoIndexState is the per-repo freshness provenance recorded at the
|
||||
// end of a (re)index: the git revision the graph reflects, whether the
|
||||
// working tree was dirty at index time, the Merkle workspace
|
||||
// fingerprint that gates global-pass short-circuiting, node/edge counts
|
||||
// for the index-plausibility baseline, and a JSON map of the
|
||||
// per-language extractor versions that produced the graph.
|
||||
//
|
||||
// It is the storage half of the FreshnessFact layer; the per-file half
|
||||
// lives in the Merkle leaf (the salted content hash) and the file_mtimes
|
||||
// ledger.
|
||||
type RepoIndexState struct {
|
||||
RepoPrefix string
|
||||
IndexedSHA string
|
||||
Dirty bool
|
||||
IndexedAt int64 // unix seconds
|
||||
WorkspaceFP string // Merkle root at index time
|
||||
NodeCount int
|
||||
EdgeCount int
|
||||
ExtractorVersions string // JSON-encoded map[string]int
|
||||
}
|
||||
|
||||
// RepoIndexStateWriter persists the freshness provenance for one repo.
|
||||
// Backends without durable state simply do not implement it — the
|
||||
// indexer type-asserts and skips the write when absent, exactly like the
|
||||
// FileMtime ledger.
|
||||
type RepoIndexStateWriter interface {
|
||||
SetRepoIndexState(state RepoIndexState) error
|
||||
}
|
||||
|
||||
// RepoIndexStateReader reads back the freshness provenance for one repo.
|
||||
// The bool is false when no state has been recorded yet (a never-indexed
|
||||
// or pre-feature repo), which callers treat as "freshness unknown" — they
|
||||
// never block on it.
|
||||
type RepoIndexStateReader interface {
|
||||
GetRepoIndexState(repoPrefix string) (RepoIndexState, bool, error)
|
||||
}
|
||||
|
||||
// DBStatReporter is an optional capability: report the on-disk size of the
|
||||
// backing database file and its write-ahead log, in bytes. Surfaced in
|
||||
// daemon_health so a runaway WAL high-water mark is observable. In-memory
|
||||
// backends do not implement it.
|
||||
type DBStatReporter interface {
|
||||
DBStats() (dbBytes, walBytes int64)
|
||||
}
|
||||
@@ -0,0 +1,296 @@
|
||||
package graph
|
||||
|
||||
import (
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestRepoMemoryEstimate_Empty(t *testing.T) {
|
||||
g := New()
|
||||
est := g.RepoMemoryEstimate("nonexistent")
|
||||
if est.NodeCount != 0 || est.EdgeCount != 0 || est.Total() != 0 {
|
||||
t.Errorf("empty repo should estimate zero, got %+v", est)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRepoMemoryEstimate_NodesAndEdges(t *testing.T) {
|
||||
g := New()
|
||||
n1 := &Node{ID: "r/pkg/a.go::Foo", Kind: KindFunction, Name: "Foo",
|
||||
QualName: "pkg.Foo", FilePath: "pkg/a.go", Language: "go",
|
||||
RepoPrefix: "r"}
|
||||
n2 := &Node{ID: "r/pkg/a.go::Bar", Kind: KindFunction, Name: "Bar",
|
||||
QualName: "pkg.Bar", FilePath: "pkg/a.go", Language: "go",
|
||||
RepoPrefix: "r"}
|
||||
g.AddNode(n1)
|
||||
g.AddNode(n2)
|
||||
g.AddEdge(&Edge{From: n1.ID, To: n2.ID, Kind: "CALLS", FilePath: "pkg/a.go"})
|
||||
|
||||
est := g.RepoMemoryEstimate("r")
|
||||
if est.NodeCount != 2 {
|
||||
t.Errorf("expected 2 nodes, got %d", est.NodeCount)
|
||||
}
|
||||
if est.EdgeCount != 1 {
|
||||
t.Errorf("expected 1 edge, got %d", est.EdgeCount)
|
||||
}
|
||||
if est.NodeBytes == 0 || est.EdgeBytes == 0 {
|
||||
t.Errorf("expected non-zero byte estimates, got %+v", est)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRepoMemoryEstimate_MetaContributes(t *testing.T) {
|
||||
g := New()
|
||||
short := &Node{ID: "r/a::s", Kind: KindVariable, Name: "s",
|
||||
FilePath: "a", RepoPrefix: "r"}
|
||||
long := &Node{ID: "r/a::l", Kind: KindVariable, Name: "l",
|
||||
FilePath: "a", RepoPrefix: "r",
|
||||
Meta: map[string]any{
|
||||
"signature": "func Foo(ctx context.Context, x, y int) (string, error)",
|
||||
"docstring": "A long docstring that takes many bytes to store in memory",
|
||||
"tags": []string{"public", "deprecated", "hot-path"},
|
||||
}}
|
||||
|
||||
g.AddNode(short)
|
||||
g.AddNode(long)
|
||||
|
||||
shortEst := g.RepoMemoryEstimate("r").NodeBytes / 2 // rough avg — need per-node
|
||||
_ = shortEst
|
||||
|
||||
// Direct per-node check via unexported helper is fine from test in same package.
|
||||
if nodeBytes(long) <= nodeBytes(short) {
|
||||
t.Errorf("a node with Meta should be bigger than one without: long=%d short=%d",
|
||||
nodeBytes(long), nodeBytes(short))
|
||||
}
|
||||
}
|
||||
|
||||
func TestRepoMemoryEstimate_RemoveEdgeDecrements(t *testing.T) {
|
||||
g := New()
|
||||
n1 := &Node{ID: "r/a::Foo", Kind: KindFunction, Name: "Foo", FilePath: "a", RepoPrefix: "r"}
|
||||
n2 := &Node{ID: "r/a::Bar", Kind: KindFunction, Name: "Bar", FilePath: "a", RepoPrefix: "r"}
|
||||
g.AddNode(n1)
|
||||
g.AddNode(n2)
|
||||
e := &Edge{From: n1.ID, To: n2.ID, Kind: "CALLS", FilePath: "a"}
|
||||
g.AddEdge(e)
|
||||
before := g.RepoMemoryEstimate("r")
|
||||
if before.EdgeCount != 1 {
|
||||
t.Fatalf("expected 1 edge before remove, got %d", before.EdgeCount)
|
||||
}
|
||||
g.RemoveEdge(n1.ID, n2.ID, "CALLS")
|
||||
after := g.RepoMemoryEstimate("r")
|
||||
if after.EdgeCount != 0 {
|
||||
t.Errorf("expected 0 edges after RemoveEdge, got %d", after.EdgeCount)
|
||||
}
|
||||
if after.EdgeBytes != 0 {
|
||||
t.Errorf("expected 0 edge bytes after RemoveEdge, got %d", after.EdgeBytes)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRepoMemoryEstimate_IdempotentAddDoesNotDoubleCount(t *testing.T) {
|
||||
g := New()
|
||||
n1 := &Node{ID: "r/a::Foo", Kind: KindFunction, Name: "Foo", FilePath: "a", RepoPrefix: "r"}
|
||||
n2 := &Node{ID: "r/a::Bar", Kind: KindFunction, Name: "Bar", FilePath: "a", RepoPrefix: "r"}
|
||||
g.AddNode(n1)
|
||||
g.AddNode(n2)
|
||||
e := &Edge{From: n1.ID, To: n2.ID, Kind: "CALLS", FilePath: "a", Line: 10}
|
||||
g.AddEdge(e)
|
||||
g.AddEdge(e) // same identity — must not double-count
|
||||
g.AddNode(n1) // same identity — must not double-count nodes
|
||||
|
||||
est := g.RepoMemoryEstimate("r")
|
||||
if est.NodeCount != 2 {
|
||||
t.Errorf("expected 2 nodes after duplicate Adds, got %d", est.NodeCount)
|
||||
}
|
||||
if est.EdgeCount != 1 {
|
||||
t.Errorf("expected 1 edge after duplicate AddEdge, got %d", est.EdgeCount)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRepoMemoryEstimate_EvictFileDecrements(t *testing.T) {
|
||||
g := New()
|
||||
n1 := &Node{ID: "r/a::Foo", Kind: KindFunction, Name: "Foo", FilePath: "a", RepoPrefix: "r"}
|
||||
n2 := &Node{ID: "r/b::Bar", Kind: KindFunction, Name: "Bar", FilePath: "b", RepoPrefix: "r"}
|
||||
g.AddNode(n1)
|
||||
g.AddNode(n2)
|
||||
g.AddEdge(&Edge{From: n1.ID, To: n2.ID, Kind: "CALLS", FilePath: "a"})
|
||||
|
||||
g.EvictFile("a")
|
||||
est := g.RepoMemoryEstimate("r")
|
||||
if est.NodeCount != 1 {
|
||||
t.Errorf("expected 1 node after evicting file 'a', got %d", est.NodeCount)
|
||||
}
|
||||
// Edge sourced from evicted node 'Foo' must also be gone.
|
||||
if est.EdgeCount != 0 {
|
||||
t.Errorf("expected 0 edges after evicting source node's file, got %d", est.EdgeCount)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRepoMemoryEstimate_EvictRepoZeroesCounter(t *testing.T) {
|
||||
g := New()
|
||||
g.AddNode(&Node{ID: "r1/a::A", Kind: KindFunction, Name: "A", FilePath: "a", RepoPrefix: "r1"})
|
||||
g.AddNode(&Node{ID: "r1/a::B", Kind: KindFunction, Name: "B", FilePath: "a", RepoPrefix: "r1"})
|
||||
g.AddNode(&Node{ID: "r2/x::X", Kind: KindFunction, Name: "X", FilePath: "x", RepoPrefix: "r2"})
|
||||
g.AddEdge(&Edge{From: "r1/a::A", To: "r1/a::B", Kind: "CALLS", FilePath: "a"})
|
||||
g.AddEdge(&Edge{From: "r1/a::A", To: "r2/x::X", Kind: "CALLS", FilePath: "a"})
|
||||
|
||||
g.EvictRepo("r1")
|
||||
|
||||
r1 := g.RepoMemoryEstimate("r1")
|
||||
if r1.NodeCount != 0 || r1.EdgeCount != 0 || r1.Total() != 0 {
|
||||
t.Errorf("evicted repo should estimate zero, got %+v", r1)
|
||||
}
|
||||
// r2 has only the surviving X node; the cross-repo edge into X was
|
||||
// attributed to r1 (source repo) so it shouldn't show up under r2.
|
||||
r2 := g.RepoMemoryEstimate("r2")
|
||||
if r2.NodeCount != 1 {
|
||||
t.Errorf("r2 should still have 1 node, got %d", r2.NodeCount)
|
||||
}
|
||||
if r2.EdgeCount != 0 {
|
||||
t.Errorf("r2 should have 0 outgoing edges, got %d", r2.EdgeCount)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAllRepoMemoryEstimates_AggregatesAcrossRepos(t *testing.T) {
|
||||
g := New()
|
||||
g.AddNode(&Node{ID: "r1/a::A", Kind: KindFunction, Name: "A", FilePath: "a", RepoPrefix: "r1"})
|
||||
g.AddNode(&Node{ID: "r2/b::B", Kind: KindFunction, Name: "B", FilePath: "b", RepoPrefix: "r2"})
|
||||
g.AddNode(&Node{ID: "r2/b::C", Kind: KindFunction, Name: "C", FilePath: "b", RepoPrefix: "r2"})
|
||||
g.AddEdge(&Edge{From: "r2/b::B", To: "r2/b::C", Kind: "CALLS", FilePath: "b"})
|
||||
|
||||
all := g.AllRepoMemoryEstimates()
|
||||
if got := all["r1"].NodeCount; got != 1 {
|
||||
t.Errorf("r1 node count = %d, want 1", got)
|
||||
}
|
||||
if got := all["r2"].NodeCount; got != 2 {
|
||||
t.Errorf("r2 node count = %d, want 2", got)
|
||||
}
|
||||
if got := all["r2"].EdgeCount; got != 1 {
|
||||
t.Errorf("r2 edge count = %d, want 1", got)
|
||||
}
|
||||
if got := all["r1"].EdgeCount; got != 0 {
|
||||
t.Errorf("r1 edge count = %d, want 0", got)
|
||||
}
|
||||
// Sanity: bulk and per-repo agree.
|
||||
if perRepo := g.RepoMemoryEstimate("r2"); perRepo != all["r2"] {
|
||||
t.Errorf("per-repo and bulk disagree for r2: per-repo=%+v bulk=%+v", perRepo, all["r2"])
|
||||
}
|
||||
}
|
||||
|
||||
// walkRepoCounts is a reference implementation: walks every node and
|
||||
// every edge once to bucket counts by repo. Used to verify the running
|
||||
// counters stay in sync across a mix of mutations.
|
||||
func walkRepoCounts(g *Graph) (nodes, edges map[string]int) {
|
||||
nodes = make(map[string]int)
|
||||
edges = make(map[string]int)
|
||||
for _, n := range g.AllNodes() {
|
||||
if n.RepoPrefix != "" {
|
||||
nodes[n.RepoPrefix]++
|
||||
}
|
||||
}
|
||||
for _, e := range g.AllEdges() {
|
||||
src := g.GetNode(e.From)
|
||||
if src != nil && src.RepoPrefix != "" {
|
||||
edges[src.RepoPrefix]++
|
||||
}
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
func assertCountersMatchWalk(t *testing.T, g *Graph, where string) {
|
||||
t.Helper()
|
||||
walkN, walkE := walkRepoCounts(g)
|
||||
got := g.AllRepoMemoryEstimates()
|
||||
for prefix, want := range walkN {
|
||||
if got[prefix].NodeCount != want {
|
||||
t.Errorf("%s: %q node count: counter=%d walk=%d",
|
||||
where, prefix, got[prefix].NodeCount, want)
|
||||
}
|
||||
}
|
||||
for prefix, want := range walkE {
|
||||
if got[prefix].EdgeCount != want {
|
||||
t.Errorf("%s: %q edge count: counter=%d walk=%d",
|
||||
where, prefix, got[prefix].EdgeCount, want)
|
||||
}
|
||||
}
|
||||
// Counters that don't exist in the walk are also a bug — leaked
|
||||
// entries that survived eviction.
|
||||
for prefix, est := range got {
|
||||
if walkN[prefix] == 0 && est.NodeCount != 0 {
|
||||
t.Errorf("%s: %q has counter NodeCount=%d but walk says 0",
|
||||
where, prefix, est.NodeCount)
|
||||
}
|
||||
if walkE[prefix] == 0 && est.EdgeCount != 0 {
|
||||
t.Errorf("%s: %q has counter EdgeCount=%d but walk says 0",
|
||||
where, prefix, est.EdgeCount)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestRepoMemoryCounters_StayInSyncUnderMixedMutations(t *testing.T) {
|
||||
g := New()
|
||||
|
||||
// Two repos, several nodes each, a mix of intra- and cross-repo edges.
|
||||
r1Nodes := []*Node{
|
||||
{ID: "r1/a::A", Kind: KindFunction, Name: "A", FilePath: "a", RepoPrefix: "r1"},
|
||||
{ID: "r1/a::B", Kind: KindFunction, Name: "B", FilePath: "a", RepoPrefix: "r1"},
|
||||
{ID: "r1/b::C", Kind: KindFunction, Name: "C", FilePath: "b", RepoPrefix: "r1"},
|
||||
}
|
||||
r2Nodes := []*Node{
|
||||
{ID: "r2/x::X", Kind: KindFunction, Name: "X", FilePath: "x", RepoPrefix: "r2"},
|
||||
{ID: "r2/x::Y", Kind: KindFunction, Name: "Y", FilePath: "x", RepoPrefix: "r2"},
|
||||
}
|
||||
for _, n := range append(append([]*Node{}, r1Nodes...), r2Nodes...) {
|
||||
g.AddNode(n)
|
||||
}
|
||||
assertCountersMatchWalk(t, g, "after AddNode")
|
||||
|
||||
edges := []*Edge{
|
||||
{From: "r1/a::A", To: "r1/a::B", Kind: "CALLS", FilePath: "a"},
|
||||
{From: "r1/a::A", To: "r1/b::C", Kind: "CALLS", FilePath: "a"},
|
||||
{From: "r1/b::C", To: "r2/x::X", Kind: "CALLS", FilePath: "b"}, // cross-repo
|
||||
{From: "r2/x::X", To: "r2/x::Y", Kind: "CALLS", FilePath: "x"},
|
||||
{From: "r2/x::Y", To: "r1/a::A", Kind: "CALLS", FilePath: "x"}, // cross-repo back
|
||||
}
|
||||
for _, e := range edges {
|
||||
g.AddEdge(e)
|
||||
}
|
||||
assertCountersMatchWalk(t, g, "after AddEdge")
|
||||
|
||||
// Idempotent re-adds must not double-count.
|
||||
g.AddNode(r1Nodes[0])
|
||||
g.AddEdge(edges[0])
|
||||
assertCountersMatchWalk(t, g, "after idempotent re-adds")
|
||||
|
||||
// RemoveEdge.
|
||||
g.RemoveEdge("r1/a::A", "r1/a::B", "CALLS")
|
||||
assertCountersMatchWalk(t, g, "after RemoveEdge")
|
||||
|
||||
// EvictFile — drops both r1/a::A and r1/a::B's host file, including
|
||||
// the cross-repo edge r1/b::C → r2/x::X is left alone.
|
||||
g.EvictFile("a")
|
||||
assertCountersMatchWalk(t, g, "after EvictFile(a)")
|
||||
|
||||
// AddNode an updated version of an existing node (RepoPrefix change).
|
||||
g.AddNode(&Node{ID: "r1/b::C", Kind: KindFunction, Name: "C", FilePath: "b", RepoPrefix: "r1"})
|
||||
assertCountersMatchWalk(t, g, "after node update (same prefix)")
|
||||
|
||||
// EvictRepo r2 — must zero its counters and decrement r1's edge
|
||||
// counter for the r1/b::C → r2/x::X edge (source is r1, target was r2).
|
||||
g.EvictRepo("r2")
|
||||
assertCountersMatchWalk(t, g, "after EvictRepo(r2)")
|
||||
}
|
||||
|
||||
func TestMetaBytes_HandlesCommonTypes(t *testing.T) {
|
||||
cases := []map[string]any{
|
||||
nil,
|
||||
{},
|
||||
{"s": "hello"},
|
||||
{"b": true, "i": 42, "f": 3.14},
|
||||
{"list": []string{"a", "b", "c"}},
|
||||
{"nested": map[string]any{"k": "v"}},
|
||||
}
|
||||
for i, m := range cases {
|
||||
// Should never panic; empty map returns non-zero-ish (header).
|
||||
got := metaBytes(m)
|
||||
if m == nil && got != 0 {
|
||||
t.Errorf("case %d: nil map should be 0, got %d", i, got)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,26 @@
|
||||
package graph
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestMotivatesKinds(t *testing.T) {
|
||||
require.Equal(t, NodeKind("rationale"), KindRationale)
|
||||
require.Equal(t, EdgeKind("motivates"), EdgeMotivates)
|
||||
require.Equal(t, EdgeKind("cross_repo_motivates"), EdgeCrossRepoMotivates)
|
||||
}
|
||||
|
||||
func TestCrossRepoMotivatesRegistered(t *testing.T) {
|
||||
cr, ok := CrossRepoKindFor(EdgeMotivates)
|
||||
require.True(t, ok, "EdgeMotivates must have a cross-repo parallel")
|
||||
require.Equal(t, EdgeCrossRepoMotivates, cr)
|
||||
|
||||
base, ok := BaseKindForCrossRepo(EdgeCrossRepoMotivates)
|
||||
require.True(t, ok)
|
||||
require.Equal(t, EdgeMotivates, base)
|
||||
|
||||
require.Contains(t, BaseKindsForCrossRepo(), EdgeMotivates,
|
||||
"DetectCrossRepoEdges must materialise the motivates parallel")
|
||||
}
|
||||
@@ -0,0 +1,520 @@
|
||||
package graph
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
type NodeKind string
|
||||
|
||||
const (
|
||||
KindFile NodeKind = "file"
|
||||
KindPackage NodeKind = "package"
|
||||
KindFunction NodeKind = "function"
|
||||
KindMethod NodeKind = "method"
|
||||
KindType NodeKind = "type"
|
||||
KindInterface NodeKind = "interface"
|
||||
KindVariable NodeKind = "variable"
|
||||
KindImport NodeKind = "import"
|
||||
KindContract NodeKind = "contract"
|
||||
// KindField represents a struct field, class property, or record
|
||||
// field — anything addressable as `owner.field`. ID convention:
|
||||
// `<file>::<owner>.<field>`. EdgeMemberOf links the field to its
|
||||
// owning type. Languages that already emitted class properties as
|
||||
// KindVariable (TypeScript, PHP) keep doing so for backwards
|
||||
// compatibility — KindField is reserved for languages that
|
||||
// previously emitted only type-ref edges from fields (Go, Rust,
|
||||
// Java, C#).
|
||||
KindField NodeKind = "field"
|
||||
// Coverage kinds: each is gated behind a per-domain
|
||||
// .gortex.yaml::index.coverage.<domain>.enabled. Parsers register a
|
||||
// kind on first use; the registry is permissive (validNodeKinds
|
||||
// accepts all known kinds) so an unenabled domain simply produces no
|
||||
// nodes of that kind, rather than failing extraction.
|
||||
|
||||
// KindParam represents a single function/method parameter. ID
|
||||
// convention: `<func-id>#param:<name>`. EdgeParamOf links the param
|
||||
// node back to its owner; EdgeTypedAs binds it to its declared
|
||||
// type. Created when index.function_shape.enabled is true.
|
||||
KindParam NodeKind = "param"
|
||||
// KindClosure represents an anonymous function / lambda inside an
|
||||
// enclosing function. ID convention: `<file>::<enclosing>#closure@<line>`.
|
||||
// Calls/reads/writes inside the closure attribute to the closure
|
||||
// node, not its enclosing function. EdgeMemberOf links to the
|
||||
// enclosing function. EdgeCaptures lists outer bindings closed over.
|
||||
KindClosure NodeKind = "closure"
|
||||
// KindLocal represents an intra-function binding — a variable
|
||||
// declared inside a function body via `x := …` / `var x = …` / a
|
||||
// range clause / a type-switch / a for-init clause. ID convention:
|
||||
// `<ownerID>#local:<name>@+<offsetFromOwnerStartLine>` (the
|
||||
// leading `+` flags the value as a relative offset so the IDs
|
||||
// stay stable when the enclosing function moves as a whole).
|
||||
// EdgeMemberOf links each binding to its enclosing function or
|
||||
// method. KindLocal is excluded from the BM25 search index by
|
||||
// shouldIndexForSearch — surfacing `err` / `data` / `n` / `i`
|
||||
// from every function would flood every name lookup. The data-
|
||||
// flow analysis (flow_between, taint_paths, ...) traverses the
|
||||
// EdgeValueFlow / EdgeArgOf / EdgeReturnsTo edges that target
|
||||
// these nodes; consumers that want the locals can ask for them
|
||||
// by kind explicitly.
|
||||
KindLocal NodeKind = "local"
|
||||
// KindBuiltin represents a language intrinsic — a function /
|
||||
// type / constant that's part of the language itself, not
|
||||
// declared in any indexed source file. ID convention:
|
||||
// `builtin::<lang>::<name>` for functions (`builtin::go::append`,
|
||||
// `builtin::py::len`) and `builtin::<lang>::type::<name>` for
|
||||
// types (`builtin::go::type::string`). Meta.builtin_kind ∈
|
||||
// "func" | "type" | "const". KindBuiltin is excluded from the
|
||||
// BM25 search index — surfacing `string` / `int` / `append`
|
||||
// would flood every name lookup. They participate in normal
|
||||
// graph queries: `find_usages(builtin::go::type::float64)`
|
||||
// answers "every variable typed as float64 in this codebase",
|
||||
// which is the load-bearing query for type-drift / dataflow
|
||||
// analyses.
|
||||
KindBuiltin NodeKind = "builtin"
|
||||
// KindConstant peels off `const`, `iota`, top-level immutable
|
||||
// bindings, and language-specific constant declarations from
|
||||
// KindVariable. Existing variable-kind nodes are re-classified on
|
||||
// next index; IDs are preserved.
|
||||
KindConstant NodeKind = "constant"
|
||||
// KindEnumMember represents one member of an enum-like type. ID
|
||||
// convention: `<file>::<EnumType>.<Member>`. EdgeMemberOf links to
|
||||
// the enum's type node.
|
||||
KindEnumMember NodeKind = "enum_member"
|
||||
// KindGenericParam represents a type parameter declared by a
|
||||
// function or type. ID convention: `<owner-id>#tparam:<name>`.
|
||||
KindGenericParam NodeKind = "generic_param"
|
||||
// KindModule represents a single (ecosystem, name, version) tuple
|
||||
// for an external dependency. Shared across files that import it.
|
||||
// ID convention: `module::<ecosystem>:<name>@<version>`.
|
||||
// Ecosystems: go, npm, pypi, cargo, maven, composer, gem, hex, nuget.
|
||||
KindModule NodeKind = "module"
|
||||
// KindTable represents a database table. ID convention:
|
||||
// `db::<dialect>::<schema>.<table>`. Sourced from migrations, ORM
|
||||
// models, and string-literal SQL in priority order.
|
||||
KindTable NodeKind = "table"
|
||||
// KindColumn represents a database column. ID convention:
|
||||
// `db::<dialect>::<schema>.<table>.<column>`. EdgeMemberOf links to
|
||||
// the owning table.
|
||||
KindColumn NodeKind = "column"
|
||||
// KindConfigKey represents a configuration key — env var, viper
|
||||
// path, CLI flag, struct-tag-driven field, or k8s ConfigMap entry.
|
||||
// ID convention: `cfg::<source>::<dotted.path>`. Source ∈
|
||||
// env|viper|flags|k8s_cm|k8s_secret|struct_tag.
|
||||
KindConfigKey NodeKind = "config_key"
|
||||
// KindFlag represents a feature flag / experiment. ID convention:
|
||||
// `flag::<provider>::<name>`. Provider ∈ growthbook|launchdarkly|
|
||||
// unleash|internal|env.
|
||||
KindFlag NodeKind = "flag"
|
||||
// KindEvent represents a log, metric, span, or trace name emitted
|
||||
// from code, or a pub/sub topic/channel/subject. ID convention:
|
||||
// `event::<kind>::<name>` for observability events and
|
||||
// `event::pubsub::<transport>::<name>` for pub/sub topics.
|
||||
// Meta["event_kind"] ∈ log|metric|trace|span|pubsub. For pubsub
|
||||
// events Meta["transport"] ∈ nats|kafka|rabbitmq|redis|socketio|
|
||||
// eventemitter|unknown; publishers link in via EdgeEmits and
|
||||
// subscribers via EdgeListensOn.
|
||||
KindEvent NodeKind = "event"
|
||||
// KindMigration represents a database migration unit. ID
|
||||
// convention: `migration::<dialect>::<id>`. Provides tables/columns
|
||||
// it creates; consumes ones it references.
|
||||
KindMigration NodeKind = "migration"
|
||||
// KindFixture represents a test data file or golden file. ID
|
||||
// convention: `fixture::<path>`. Test functions reference it via
|
||||
// EdgeReferences.
|
||||
KindFixture NodeKind = "fixture"
|
||||
// KindTodo represents a TODO/FIXME/HACK/XXX/NOTE comment marker. ID
|
||||
// convention: `todo::<file>:<line>`. Meta carries tag, assignee,
|
||||
// due, ticket, and the truncated text.
|
||||
KindTodo NodeKind = "todo"
|
||||
// KindTeam represents a CODEOWNERS team or individual. ID
|
||||
// convention: `team::<name>`. Meta.kind ∈ team|person disambiguates.
|
||||
KindTeam NodeKind = "team"
|
||||
// KindRelease represents a tag/version boundary. ID convention:
|
||||
// `release::<tag>`. Used as a query filter via Node.Meta["added_in"]
|
||||
// rather than as an edge endpoint in most cases.
|
||||
KindRelease NodeKind = "release"
|
||||
// KindLicense represents an SPDX license identifier. ID convention:
|
||||
// `license::<spdx>`. Files link to it via EdgeLicensedAs.
|
||||
KindLicense NodeKind = "license"
|
||||
// KindString represents a string literal that crosses an API
|
||||
// boundary worth tracking — Datadog/Prometheus metric names,
|
||||
// errors.New / fmt.Errorf messages, raw HTTP route paths, and
|
||||
// (later) HTML class/id values. ID convention:
|
||||
// `string::<context>::<value-or-hash>`. Context ∈
|
||||
// metric|error_msg|route|html_class|html_id|… EdgeEmits links the
|
||||
// enclosing function/method to the string node, mirroring KindEvent.
|
||||
// Per-repo: applyRepoPrefix prefixes every node ID with the repo
|
||||
// slug so two repos that emit the same string don't collide.
|
||||
KindString NodeKind = "string"
|
||||
// KindResource represents a Kubernetes manifest resource —
|
||||
// Deployment, Service, ConfigMap, Secret, Ingress, CronJob,
|
||||
// StatefulSet, DaemonSet, Job, ReplicaSet, ServiceAccount,
|
||||
// Role, RoleBinding, ClusterRole, ClusterRoleBinding, Namespace,
|
||||
// PersistentVolume, PersistentVolumeClaim, etc. ID convention:
|
||||
// `k8s::<kind>::<namespace>::<name>` (namespace defaults to
|
||||
// "_" when not declared in the manifest). Meta carries
|
||||
// api_version, namespace, labels (truncated). Sourced from YAML
|
||||
// extractors that detect K8s manifests by `apiVersion:` +
|
||||
// `kind:` markers.
|
||||
KindResource NodeKind = "resource"
|
||||
// KindKustomization represents a Kustomize overlay — one per
|
||||
// `kustomization.yaml` / `kustomization.yml` file in a repo.
|
||||
// ID convention: `kustomize::<dir>` where dir is the directory
|
||||
// holding the kustomization file relative to the repo root.
|
||||
// Resources, bases, components, and patches are linked via
|
||||
// EdgeDependsOn (overlay → base) and EdgeReferences
|
||||
// (overlay → resource files).
|
||||
KindKustomization NodeKind = "kustomization"
|
||||
// KindImage represents either a container image or a raster/vector
|
||||
// image asset — distinguished by ID prefix and Meta. Container images
|
||||
// come from a Dockerfile FROM (external base or `FROM ... AS <stage>`)
|
||||
// or a K8s container spec; image assets are picture files ingested by
|
||||
// the multimodal extractor. ID conventions:
|
||||
// `image::<name>:<tag>` for external/registry images (tag
|
||||
// defaults to "latest" when omitted)
|
||||
// `image::stage::<file>::<stage-name>` for Dockerfile build
|
||||
// stages
|
||||
// `image::asset::<path>` for ingested image-file assets
|
||||
// Meta carries registry/digest/platform for container images, and
|
||||
// format/width/height/size_bytes/sha256 (asset_kind="image") for
|
||||
// image-file assets.
|
||||
KindImage NodeKind = "image"
|
||||
// KindArtifact represents a non-code knowledge file declared in
|
||||
// the `.gortex.yaml::artifacts` manifest — a DB schema (SQL /
|
||||
// Prisma / dbt), an API spec (OpenAPI / GraphQL / protobuf), an
|
||||
// infra config (Terraform / Kustomize / Helm), or an
|
||||
// architecture doc (ADR markdown). ID convention:
|
||||
// `artifact::<repo-relative-path>`. Meta carries artifact_kind
|
||||
// (schema|api|infra|doc), content_hash (sha256 of the file —
|
||||
// drives staleness detection), title, and size. The artifact
|
||||
// node links to every symbol it mentions via EdgeReferences so
|
||||
// agents can pull the right schema or spec alongside the code.
|
||||
KindArtifact NodeKind = "artifact"
|
||||
// KindDoc represents one heading-delimited prose section of a
|
||||
// documentation file (Markdown). Name is the breadcrumb heading
|
||||
// path ("README.md > Setup > Build"); Meta["section_text"] holds
|
||||
// the section's paragraph text with markdown syntax stripped, and
|
||||
// the BM25 search index is fed that body so a prose query ranks
|
||||
// the right section. ID convention:
|
||||
// "<file>::doc:<slug-of-heading-path>" -- derived from the
|
||||
// heading path, NOT line numbers, so an incremental reindex of an
|
||||
// edited file keeps stable section identity. The owning file
|
||||
// links to it via EdgeDefines.
|
||||
KindDoc NodeKind = "doc"
|
||||
// KindRationale is a graph projection of a development-memory record
|
||||
// (a decision / incident / constraint / invariant) — the "why" behind
|
||||
// code. The store_memory sidecar stays the system of record; this node
|
||||
// is a derived view re-projected on memory write and reconciled on
|
||||
// warmup, so a why-query is one hop from the code it explains. ID
|
||||
// convention: "rationale::<memory-id>". Links to the code it explains
|
||||
// via EdgeMotivates.
|
||||
KindRationale NodeKind = "rationale"
|
||||
// KindTopic represents a message-broker topic / subject / channel /
|
||||
// exchange — the contract-layer pairing artefact for Kafka,
|
||||
// RabbitMQ, NATS, and Redis pub-sub. ID convention:
|
||||
// `topic::<broker>::<name>` where broker ∈ kafka|rabbitmq|nats|
|
||||
// redis. Meta carries broker (the family) and name (the raw topic
|
||||
// / subject / channel / exchange string). Producer symbols link
|
||||
// in via EdgeProducesTopic, consumers via EdgeConsumesTopic, so a
|
||||
// cross-service event flow is a two-hop path
|
||||
// producer --produces_topic--> topic <--consumes_topic-- consumer.
|
||||
// KindTopic is distinct from KindEvent (which the observability /
|
||||
// pubsub extractor uses for the same calls): KindEvent rides at
|
||||
// the call-site evidence tier (heuristic / inferred) and is for
|
||||
// broad "what publishes here" queries, while KindTopic is the
|
||||
// pairing artefact produced by the contracts matcher and rides at
|
||||
// the structural ast_resolved tier.
|
||||
KindTopic NodeKind = "topic"
|
||||
// KindMacro represents a C/C++ preprocessor macro defined with
|
||||
// #define — both object-like (`#define PI 3.14`) and function-like
|
||||
// (`#define SQ(x) ((x)*(x))`). ID convention: `<file>::<NAME>`.
|
||||
// Meta["macro_kind"] ∈ object|function; function-like macros carry
|
||||
// Meta["params"] (the parameter names) and Meta["replacement"] (the
|
||||
// replacement-list text). The owning file links via EdgeDefines, and
|
||||
// a function-like macro emits EdgeCalls to the symbols its
|
||||
// replacement list invokes — recovering call edges that would
|
||||
// otherwise be hidden behind macro expansion (a call site `SQ(2)`
|
||||
// resolves to the macro, whose body-calls then continue the chain).
|
||||
KindMacro NodeKind = "macro"
|
||||
// KindAgent represents a live coding agent participating in a
|
||||
// multi-agent session — a first-class, queryable presence entity
|
||||
// distinct from a transport session. ID convention: `agent::<id>`.
|
||||
// Carries Meta["cursor"] (the symbol/file the agent is focused on),
|
||||
// Meta["status"], Meta["locked_paths"], and Meta["last_seen"]. Agent
|
||||
// presence is volatile and lives in an in-memory registry rather than
|
||||
// the persistent code graph; this kind names the entity so it reads as
|
||||
// first-class in tool output and query filters.
|
||||
KindAgent NodeKind = "agent"
|
||||
// KindContractBridge represents one matched provider↔consumer
|
||||
// contract group — an HTTP route, a gRPC/Thrift method, or a
|
||||
// pub/sub topic — materialised as a single graph node that spans
|
||||
// every repo participating in the group. ID convention:
|
||||
// `bridge::<contract-id>` where contract-id is the canonical
|
||||
// contract key (`http::GET::/v1/users`, `grpc::Users::GetUser`,
|
||||
// `topic::kafka::orders`), so the bridge for any contract is
|
||||
// addressable from the contract ID alone, across repos. Meta
|
||||
// carries contract_type, canonical_key, repos (sorted slice of
|
||||
// participating repo prefixes), provider_count, consumer_count
|
||||
// and cross_repo. EdgeBridges links the bridge to each
|
||||
// participating KindContract node. Bridge nodes are re-derived
|
||||
// from the matcher result on every contract reconcile — all of
|
||||
// them share the synthetic FilePath "contracts://bridges" so the
|
||||
// reconcile pass can evict the stale generation with one
|
||||
// EvictFile call before re-minting.
|
||||
KindContractBridge NodeKind = "contract_bridge"
|
||||
)
|
||||
|
||||
// IsValidNodeKind reports whether s names a known node kind. Used by
|
||||
// the search layer to tell a real node-kind clause apart from a
|
||||
// flavor value that only looks like a kind (e.g. codegraph's
|
||||
// `kind:class`).
|
||||
func IsValidNodeKind(s string) bool {
|
||||
return validNodeKinds[NodeKind(s)]
|
||||
}
|
||||
|
||||
var validNodeKinds = map[NodeKind]bool{
|
||||
KindFile: true, KindPackage: true, KindFunction: true,
|
||||
KindMethod: true, KindType: true, KindInterface: true,
|
||||
KindVariable: true, KindImport: true, KindContract: true,
|
||||
KindField: true,
|
||||
// Coverage kinds — see Kind* doc comments above for usage notes.
|
||||
KindParam: true, KindClosure: true, KindConstant: true,
|
||||
KindEnumMember: true, KindGenericParam: true, KindModule: true,
|
||||
KindTable: true, KindColumn: true, KindConfigKey: true,
|
||||
KindFlag: true, KindEvent: true, KindMigration: true,
|
||||
KindFixture: true, KindTodo: true, KindTeam: true,
|
||||
KindRelease: true, KindLicense: true, KindString: true,
|
||||
KindResource: true, KindKustomization: true, KindImage: true,
|
||||
KindArtifact: true, KindDoc: true, KindTopic: true,
|
||||
KindRationale: true,
|
||||
KindMacro: true, KindAgent: true, KindContractBridge: true,
|
||||
}
|
||||
|
||||
type Node struct {
|
||||
ID string `json:"id"`
|
||||
Kind NodeKind `json:"kind"`
|
||||
Name string `json:"name"`
|
||||
QualName string `json:"qual_name,omitempty"`
|
||||
FilePath string `json:"file_path"`
|
||||
StartLine int `json:"start_line"`
|
||||
// EndLine is omitted when zero — File-kind nodes don't have ranges.
|
||||
EndLine int `json:"end_line,omitempty"`
|
||||
// StartColumn / EndColumn are 0-based source column offsets of the
|
||||
// symbol's span. Omitted when zero (most extractors record only line
|
||||
// ranges); promoted to typed nodes columns on the SQLite backend.
|
||||
StartColumn int `json:"start_column,omitempty"`
|
||||
EndColumn int `json:"end_column,omitempty"`
|
||||
Language string `json:"language"`
|
||||
Meta map[string]any `json:"meta,omitempty"`
|
||||
RepoPrefix string `json:"repo_prefix,omitempty"`
|
||||
// WorkspaceID is the hard graph boundary slug. Two nodes with
|
||||
// different WorkspaceIDs are not allowed to be matched as contract
|
||||
// provider/consumer pairs and queries scope by it by default.
|
||||
// Defaults at warmup time to the per-repo `.gortex.yaml::workspace`
|
||||
// setting; falls back to RepoPrefix when no workspace is declared
|
||||
// (so old configs keep working) and to "" only for snapshot
|
||||
// records written before the field existed (gob decodes unknown
|
||||
// fields as zero — warmup backfills these from config).
|
||||
WorkspaceID string `json:"workspace_id,omitempty"`
|
||||
// ProjectID is the soft sub-boundary inside a workspace. One
|
||||
// project per repo by default; monorepos can declare projects[] in
|
||||
// .gortex.yaml. Contract pairing is bounded to a single
|
||||
// (workspace_id, project_id); cross-project contracts become orphans.
|
||||
// Defaults to the repo name when no projects[] mapping matches.
|
||||
ProjectID string `json:"project_id,omitempty"`
|
||||
// AbsoluteFilePath is the on-disk absolute path corresponding to
|
||||
// FilePath. It is empty on the canonical graph node and is populated
|
||||
// only on the per-response copies the MCP layer hands to result
|
||||
// encoders, so an editor or agent can open a result directly without
|
||||
// reconstructing the path from repo_prefix + file_path.
|
||||
AbsoluteFilePath string `json:"absolute_file_path,omitempty"`
|
||||
|
||||
// Origin marks a node minted by the cross-daemon proxy-edge feature
|
||||
// as standing in for a symbol another daemon owns. "" on every
|
||||
// locally-indexed node; "remote:<slug>" on a proxy node. Written ONLY
|
||||
// by the proxy-edge mint path; the read-only fan-out carries
|
||||
// provenance in the response, never here. Excluded from
|
||||
// graph_stats / BM25 / communities / analyzers (see IsProxyNode).
|
||||
Origin string `json:"origin,omitempty"`
|
||||
// Stub marks a node as a federation proxy placeholder whose
|
||||
// neighbour edges hydrate lazily over /v1/subgraph rather than from
|
||||
// local extraction. Always true together with a non-empty Origin.
|
||||
Stub bool `json:"stub,omitempty"`
|
||||
// FetchedAt is the wall-clock time the proxy node (and its hydrated
|
||||
// ring) was last pulled from the remote — drives the TTL freshness
|
||||
// gate and the federated last_synced field. Zero on local nodes.
|
||||
FetchedAt time.Time `json:"fetched_at,omitempty"`
|
||||
}
|
||||
|
||||
// IsReExportNode reports whether n is a barrel re-export binding — a node
|
||||
// minted at an `export { X } from './mod'` site that forwards another module's
|
||||
// declaration under the exported (post-alias) name. Marked with
|
||||
// Meta["reexport"]==true by the JS/TS extractor. These nodes are transparent
|
||||
// aliases: call resolution skips them (a call binds to the forwarded
|
||||
// declaration, not the façade), while find_usages delegates their usage set to
|
||||
// the canonical target.
|
||||
func IsReExportNode(n *Node) bool {
|
||||
if n == nil || n.Meta == nil {
|
||||
return false
|
||||
}
|
||||
v, ok := n.Meta["reexport"].(bool)
|
||||
return ok && v
|
||||
}
|
||||
|
||||
// Brief returns a compact representation with only the fields needed for listing.
|
||||
func (n *Node) Brief() map[string]any {
|
||||
b := map[string]any{
|
||||
"id": n.ID,
|
||||
"name": n.Name,
|
||||
"kind": n.Kind,
|
||||
"file_path": n.FilePath,
|
||||
"start_line": n.StartLine,
|
||||
}
|
||||
if n.RepoPrefix != "" {
|
||||
b["repo_prefix"] = n.RepoPrefix
|
||||
}
|
||||
if n.WorkspaceID != "" {
|
||||
b["workspace_id"] = n.WorkspaceID
|
||||
}
|
||||
if n.ProjectID != "" {
|
||||
b["project_id"] = n.ProjectID
|
||||
}
|
||||
// Surface visibility and a short doc snippet when present — Brief
|
||||
// is the listing projection used by search_symbols and find_usages,
|
||||
// where these two fields meaningfully sharpen the result so the
|
||||
// agent can decide without a follow-up get_symbol_source call.
|
||||
if v, ok := n.Meta["visibility"].(string); ok && v != "" {
|
||||
b["visibility"] = v
|
||||
}
|
||||
if d, ok := n.Meta["doc"].(string); ok && d != "" {
|
||||
// Truncate doc to 80 chars in Brief — the full doc is on the
|
||||
// node, this is just the listing teaser.
|
||||
const briefDocCap = 80
|
||||
if len(d) > briefDocCap {
|
||||
d = d[:briefDocCap] + "…"
|
||||
}
|
||||
b["doc"] = d
|
||||
}
|
||||
// Test classification — stamped by the indexer's test-edge pass.
|
||||
// Surfacing it on the listing row lets agents tell production
|
||||
// callers from test callers without a follow-up call.
|
||||
if v, ok := n.Meta["is_test"].(bool); ok && v {
|
||||
b["is_test"] = true
|
||||
}
|
||||
if r, ok := n.Meta["test_role"].(string); ok && r != "" {
|
||||
b["test_role"] = r
|
||||
}
|
||||
if r, ok := n.Meta["test_runner"].(string); ok && r != "" {
|
||||
b["test_runner"] = r
|
||||
}
|
||||
if v, ok := n.Meta["is_test_file"].(bool); ok && v {
|
||||
b["is_test_file"] = true
|
||||
}
|
||||
// Structural flavor + UI-component framework — stamped by the
|
||||
// language extractors. Surfacing them on the listing row lets an
|
||||
// agent filter / triage by shape (struct vs class, react vs svelte)
|
||||
// without a follow-up call.
|
||||
if v, ok := n.Meta["type_flavor"].(string); ok && v != "" {
|
||||
b["type_flavor"] = v
|
||||
}
|
||||
if v, ok := n.Meta["ui_component"].(string); ok && v != "" {
|
||||
b["ui_component"] = v
|
||||
}
|
||||
// A prose-section node carries no signature -- surface a short
|
||||
// snippet of its body text so a docs search result is
|
||||
// self-describing without a follow-up read.
|
||||
if n.Kind == KindDoc {
|
||||
if txt, ok := n.Meta["section_text"].(string); ok && txt != "" {
|
||||
const snippetCap = 160
|
||||
if len(txt) > snippetCap {
|
||||
txt = txt[:snippetCap] + "\u2026"
|
||||
}
|
||||
b["section"] = txt
|
||||
}
|
||||
}
|
||||
// enclosing / enclosing_id name the symbol this node is declared
|
||||
// inside -- the receiver type of a method, the struct of a field,
|
||||
// the enum of a member, the function around a closure. Derived
|
||||
// from the ID convention; absent for top-level symbols. Lets a
|
||||
// search result say "Parse on type Decoder" without a follow-up
|
||||
// call.
|
||||
if eid, ename := EnclosingFromID(n.ID, n.Kind); ename != "" {
|
||||
b["enclosing"] = ename
|
||||
b["enclosing_id"] = eid
|
||||
}
|
||||
// AbsoluteFilePath is populated only on the per-response copies the
|
||||
// MCP layer builds (see Server.withAbsPaths); empty on canonical nodes.
|
||||
if n.AbsoluteFilePath != "" {
|
||||
b["absolute_file_path"] = n.AbsoluteFilePath
|
||||
}
|
||||
return b
|
||||
}
|
||||
|
||||
// EnclosingFromID derives a node's enclosing owner purely from its
|
||||
// ID and kind -- no graph access. It covers the kinds whose ID
|
||||
// convention embeds the owner:
|
||||
//
|
||||
// - method "<file>::<Owner>.<method>" -> owner "<file>::<Owner>"
|
||||
// - field "<file>::<owner>.<field>" -> owner "<file>::<owner>"
|
||||
// - enum "<file>::<EnumType>.<Member>" -> owner "<file>::<EnumType>"
|
||||
// - closure "<file>::<enclosing>#closure@N" -> owner "<file>::<enclosing>"
|
||||
//
|
||||
// For every other kind -- and for a method/field/closure whose ID
|
||||
// carries no owner segment -- both return values are empty. The
|
||||
// returned name is the owner's short (last-segment) name.
|
||||
//
|
||||
// This is the standalone derivation Node.Brief uses; callers with a
|
||||
// graph reader should prefer the richer EdgeMemberOf-based lookup,
|
||||
// which also resolves owners the ID does not name.
|
||||
func EnclosingFromID(id string, kind NodeKind) (ownerID, ownerName string) {
|
||||
sep := strings.Index(id, "::")
|
||||
if sep < 0 {
|
||||
return "", ""
|
||||
}
|
||||
file, symbol := id[:sep], id[sep+2:]
|
||||
switch kind {
|
||||
case KindClosure:
|
||||
// "<enclosing>#closure@<line>" -- the owner is the segment
|
||||
// before the first '#'.
|
||||
if h := strings.IndexByte(symbol, '#'); h > 0 {
|
||||
owner := symbol[:h]
|
||||
return file + "::" + owner, lastIDSegment(owner)
|
||||
}
|
||||
return "", ""
|
||||
case KindMethod, KindField, KindEnumMember:
|
||||
// "<Owner>.<member>" -- the owner is everything before the
|
||||
// last '.'.
|
||||
if dot := strings.LastIndexByte(symbol, '.'); dot > 0 {
|
||||
owner := symbol[:dot]
|
||||
return file + "::" + owner, lastIDSegment(owner)
|
||||
}
|
||||
return "", ""
|
||||
default:
|
||||
return "", ""
|
||||
}
|
||||
}
|
||||
|
||||
// lastIDSegment returns the last dotted segment of an identifier --
|
||||
// its human-facing short name.
|
||||
func lastIDSegment(s string) string {
|
||||
if i := strings.LastIndexByte(s, '.'); i >= 0 {
|
||||
return s[i+1:]
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
// EnclosingShortName returns the human-facing short name of an
|
||||
// owner identifier or node ID -- its last "::"- or "."-separated
|
||||
// segment. Used when only an owner ID string is in hand and no node
|
||||
// was resolved.
|
||||
func EnclosingShortName(s string) string {
|
||||
if i := strings.LastIndex(s, "::"); i >= 0 {
|
||||
s = s[i+2:]
|
||||
}
|
||||
return lastIDSegment(s)
|
||||
}
|
||||
|
||||
func ValidNodeKind(k NodeKind) bool {
|
||||
return validNodeKinds[k]
|
||||
}
|
||||
@@ -0,0 +1,294 @@
|
||||
package graph_test
|
||||
|
||||
// Node-id stability parity test.
|
||||
//
|
||||
// Overlay and cloud paths assume node IDs produced by two different
|
||||
// indexer invocations of the same source commit are byte-identical.
|
||||
// If that ever drifts (host-local state in the ID, parse-order leaks,
|
||||
// RNG, time, etc.), overlay merging silently breaks: the daemon's
|
||||
// overlay node IDs no longer match the server's base node IDs, edges
|
||||
// land on dangling endpoints, and queries return half-true answers.
|
||||
//
|
||||
// This test runs the live indexer pipeline twice on a freshly-copied
|
||||
// pair of identical source trees. Different absolute paths simulate
|
||||
// "two checkouts on one machine" (the cheap proxy for "two machines"
|
||||
// — the only difference between the two cases is the absolute parent
|
||||
// directory which the indexer is supposed to strip via repo-prefixing).
|
||||
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
"go.uber.org/zap"
|
||||
|
||||
"github.com/zzet/gortex/internal/config"
|
||||
"github.com/zzet/gortex/internal/graph"
|
||||
"github.com/zzet/gortex/internal/indexer"
|
||||
"github.com/zzet/gortex/internal/parser"
|
||||
"github.com/zzet/gortex/internal/parser/languages"
|
||||
"github.com/zzet/gortex/internal/search"
|
||||
)
|
||||
|
||||
func newParityRegistry() *parser.Registry {
|
||||
r := parser.NewRegistry()
|
||||
r.Register(languages.NewGoExtractor())
|
||||
return r
|
||||
}
|
||||
|
||||
// fixtureFiles is the source tree planted under each "checkout".
|
||||
// Mix of:
|
||||
// - top-level package (Go) — simple types and methods
|
||||
// - sub-package — exercises path-based ID composition
|
||||
// - HTTP route via stdlib mux — exercises contract emission
|
||||
// - import that crosses sub-packages — exercises resolver
|
||||
//
|
||||
// Multiple files per package and multiple symbols per file expose any
|
||||
// parse-order or map-iteration-order leakage in node IDs.
|
||||
var fixtureFiles = map[string]string{
|
||||
"go.mod": "module example.com/parity\n\ngo 1.21\n",
|
||||
"main.go": `package main
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
|
||||
"example.com/parity/internal/auth"
|
||||
)
|
||||
|
||||
type Server struct{}
|
||||
|
||||
func (s *Server) Start() error {
|
||||
mux := http.NewServeMux()
|
||||
mux.HandleFunc("/api/auth/login", auth.LoginHandler)
|
||||
mux.HandleFunc("/api/health", s.Health)
|
||||
return http.ListenAndServe(":8080", mux)
|
||||
}
|
||||
|
||||
func (s *Server) Health(w http.ResponseWriter, r *http.Request) {
|
||||
_, _ = w.Write([]byte("ok"))
|
||||
}
|
||||
|
||||
func main() {
|
||||
srv := &Server{}
|
||||
_ = srv.Start()
|
||||
}
|
||||
`,
|
||||
"helpers.go": `package main
|
||||
|
||||
import "strings"
|
||||
|
||||
func normalize(s string) string { return strings.ToLower(s) }
|
||||
|
||||
func reverse(s string) string {
|
||||
r := []rune(s)
|
||||
for i, j := 0, len(r)-1; i < j; i, j = i+1, j-1 {
|
||||
r[i], r[j] = r[j], r[i]
|
||||
}
|
||||
return string(r)
|
||||
}
|
||||
`,
|
||||
"internal/auth/login.go": `package auth
|
||||
|
||||
import "net/http"
|
||||
|
||||
type Credentials struct {
|
||||
User string
|
||||
Pass string
|
||||
}
|
||||
|
||||
func LoginHandler(w http.ResponseWriter, r *http.Request) {
|
||||
_, _ = w.Write([]byte("login"))
|
||||
}
|
||||
|
||||
func Validate(c Credentials) bool {
|
||||
return c.User != "" && c.Pass != ""
|
||||
}
|
||||
`,
|
||||
"internal/auth/token.go": `package auth
|
||||
|
||||
import "time"
|
||||
|
||||
type Token struct {
|
||||
Value string
|
||||
ExpiresAt time.Time
|
||||
}
|
||||
|
||||
func NewToken(value string) Token {
|
||||
return Token{Value: value, ExpiresAt: time.Now().Add(time.Hour)}
|
||||
}
|
||||
`,
|
||||
}
|
||||
|
||||
// TestNodeIDStability_Parity is the iteration-1 gate test for node-ID
|
||||
// determinism. Indexes the same source tree from two distinct absolute
|
||||
// paths (different temp dirs) and asserts the produced node IDs are
|
||||
// identical sets after stripping the repo prefix. This catches:
|
||||
//
|
||||
// - host-local state (working directory, hostname) leaking into IDs
|
||||
// - parse-order non-determinism (goroutine scheduling) leaking into IDs
|
||||
// - map-iteration-order leaking into IDs
|
||||
//
|
||||
// The repo prefix is allowed to differ because it's deliberately a
|
||||
// human-readable disambiguator; the rest of the ID after the prefix
|
||||
// must match byte-for-byte.
|
||||
//
|
||||
// For monorepo-shaped IDs the comparison is "same set" — we don't
|
||||
// require parse-order to be stable, only the final ID set.
|
||||
func TestNodeIDStability_Parity(t *testing.T) {
|
||||
idsA := indexFixture(t, "checkout-alpha")
|
||||
idsB := indexFixture(t, "checkout-beta")
|
||||
|
||||
// Strip repo prefix so we're comparing what's structural about the
|
||||
// ID and not the deliberately-different prefix.
|
||||
stripped := func(in []string, prefix string) []string {
|
||||
out := make([]string, 0, len(in))
|
||||
pre := prefix + "/"
|
||||
for _, id := range in {
|
||||
if len(id) > len(pre) && id[:len(pre)] == pre {
|
||||
out = append(out, id[len(pre):])
|
||||
continue
|
||||
}
|
||||
out = append(out, id)
|
||||
}
|
||||
sort.Strings(out)
|
||||
return out
|
||||
}
|
||||
|
||||
got := stripped(idsA.NodeIDs, idsA.Prefix)
|
||||
want := stripped(idsB.NodeIDs, idsB.Prefix)
|
||||
|
||||
if !assert.Equal(t, want, got, "node IDs must be byte-identical across two indexings of the same source tree (after stripping repo prefix). divergence breaks overlay merging across daemon and cloud.") {
|
||||
// Surface the first few divergences directly so the failure
|
||||
// message points at the offending IDs rather than the full
|
||||
// list-of-thousands diff.
|
||||
diff := symmetricDifference(got, want)
|
||||
if len(diff) > 0 {
|
||||
t.Logf("symmetric difference (up to 20 ids): %v", diff[:minInt(len(diff), 20)])
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
type fixtureResult struct {
|
||||
NodeIDs []string
|
||||
Prefix string
|
||||
}
|
||||
|
||||
// indexFixture writes the fixture into a fresh temp dir under the
|
||||
// given checkout name, indexes it via MultiIndexer (the warmup path),
|
||||
// and returns the full set of node IDs in the resulting graph plus
|
||||
// the repo prefix MultiIndexer assigned.
|
||||
//
|
||||
// We use MultiIndexer with two configured repos (the fixture + a
|
||||
// throwaway sibling) so that willBeMultiRepo is true and the prefix
|
||||
// path is exercised — that's the production code path the daemon
|
||||
// runs and the one overlay/cloud merging will rely on.
|
||||
func indexFixture(t *testing.T, checkoutName string) fixtureResult {
|
||||
t.Helper()
|
||||
|
||||
// Plant the fixture in a unique temp tree.
|
||||
root := filepath.Join(t.TempDir(), checkoutName)
|
||||
require.NoError(t, os.MkdirAll(root, 0o755))
|
||||
for relPath, content := range fixtureFiles {
|
||||
full := filepath.Join(root, relPath)
|
||||
require.NoError(t, os.MkdirAll(filepath.Dir(full), 0o755))
|
||||
require.NoError(t, os.WriteFile(full, []byte(content), 0o644))
|
||||
}
|
||||
|
||||
// Companion repo so multi-repo prefixing kicks in. We keep it
|
||||
// minimal — no parity comparison runs on it; it only exists to
|
||||
// flip willBeMultiRepo.
|
||||
companion := filepath.Join(t.TempDir(), checkoutName+"-companion")
|
||||
require.NoError(t, os.MkdirAll(companion, 0o755))
|
||||
require.NoError(t, os.WriteFile(filepath.Join(companion, "noop.go"),
|
||||
[]byte("package companion\n\nfunc Noop() {}\n"), 0o644))
|
||||
|
||||
tmpCfg := filepath.Join(t.TempDir(), "config-"+checkoutName+".yaml")
|
||||
gc := &config.GlobalConfig{
|
||||
Repos: []config.RepoEntry{
|
||||
{Path: root, Name: checkoutName},
|
||||
{Path: companion, Name: checkoutName + "-companion"},
|
||||
},
|
||||
}
|
||||
gc.SetConfigPath(tmpCfg)
|
||||
require.NoError(t, gc.Save())
|
||||
|
||||
cm, err := config.NewConfigManager(tmpCfg)
|
||||
require.NoError(t, err)
|
||||
|
||||
g := graph.New()
|
||||
mi := indexer.NewMultiIndexer(g, newParityRegistry(), search.NewBM25(), cm, zap.NewNop())
|
||||
for _, entry := range cm.Global().Repos {
|
||||
_, err := mi.TrackRepoCtx(context.Background(), entry)
|
||||
require.NoError(t, err, "track %s", entry.Name)
|
||||
}
|
||||
|
||||
ids := []string{}
|
||||
prefix := checkoutName
|
||||
for _, n := range g.AllNodes() {
|
||||
// This test is about source-symbol IDs (functions, methods,
|
||||
// types, files) — the things overlay merging keys on.
|
||||
// Contract / Module / Builtin nodes are deliberately
|
||||
// cross-repo singletons (one `dep::foo`, `module::pypi:requests`,
|
||||
// `builtin::go::len` shared across every repo that uses them)
|
||||
// and don't carry RepoPrefix; skip them so the parity gate
|
||||
// stays precise about what it gates. KindFunction nodes
|
||||
// with meta.external=true are the per-symbol stubs the
|
||||
// external-call attribution materialises for stdlib/dep
|
||||
// targets — same rule.
|
||||
if n.Kind == graph.KindContract || n.Kind == graph.KindModule || n.Kind == graph.KindBuiltin {
|
||||
continue
|
||||
}
|
||||
if ext, _ := n.Meta["external"].(bool); ext {
|
||||
continue
|
||||
}
|
||||
if n.RepoPrefix == "" {
|
||||
t.Fatalf("node %q has empty RepoPrefix in multi-repo mode", n.ID)
|
||||
}
|
||||
if n.RepoPrefix != checkoutName {
|
||||
continue
|
||||
}
|
||||
ids = append(ids, n.ID)
|
||||
}
|
||||
|
||||
require.NotEmpty(t, ids, "no fixture nodes produced — fixture or indexer regression")
|
||||
|
||||
return fixtureResult{NodeIDs: ids, Prefix: prefix}
|
||||
}
|
||||
|
||||
// symmetricDifference returns elements present in exactly one of a, b.
|
||||
// Both must be sorted.
|
||||
func symmetricDifference(a, b []string) []string {
|
||||
var diff []string
|
||||
i, j := 0, 0
|
||||
for i < len(a) && j < len(b) {
|
||||
switch {
|
||||
case a[i] == b[j]:
|
||||
i++
|
||||
j++
|
||||
case a[i] < b[j]:
|
||||
diff = append(diff, "only-in-A:"+a[i])
|
||||
i++
|
||||
default:
|
||||
diff = append(diff, "only-in-B:"+b[j])
|
||||
j++
|
||||
}
|
||||
}
|
||||
for ; i < len(a); i++ {
|
||||
diff = append(diff, "only-in-A:"+a[i])
|
||||
}
|
||||
for ; j < len(b); j++ {
|
||||
diff = append(diff, "only-in-B:"+b[j])
|
||||
}
|
||||
return diff
|
||||
}
|
||||
|
||||
func minInt(a, b int) int {
|
||||
if a < b {
|
||||
return a
|
||||
}
|
||||
return b
|
||||
}
|
||||
@@ -0,0 +1,27 @@
|
||||
package graph
|
||||
|
||||
// outEdgesBatcher is implemented by backends that can fetch many nodes'
|
||||
// out-edges in a single query (the disk-backed stores), collapsing the
|
||||
// per-node N+1 the single-file resolve path would otherwise issue.
|
||||
type outEdgesBatcher interface {
|
||||
GetOutEdgesForNodes(ids []string) map[string][]*Edge
|
||||
}
|
||||
|
||||
// OutEdgesForNodes returns each node's outgoing edges, using the backend's
|
||||
// batched query when it offers one and falling back to per-node lookups
|
||||
// otherwise (the in-memory graph, where each lookup is already an O(1) map
|
||||
// hit). Nodes with no out-edges may be absent from the returned map.
|
||||
func OutEdgesForNodes(r interface {
|
||||
GetOutEdges(nodeID string) []*Edge
|
||||
}, ids []string) map[string][]*Edge {
|
||||
if b, ok := r.(outEdgesBatcher); ok {
|
||||
return b.GetOutEdgesForNodes(ids)
|
||||
}
|
||||
out := make(map[string][]*Edge, len(ids))
|
||||
for _, id := range ids {
|
||||
if e := r.GetOutEdges(id); len(e) > 0 {
|
||||
out[id] = e
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
@@ -0,0 +1,832 @@
|
||||
package graph
|
||||
|
||||
import (
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
)
|
||||
|
||||
// OverlayLayer is one MCP session's parsed editor-buffer state. It
|
||||
// holds the nodes and edges that the overlay introduces (or hides via
|
||||
// tombstones) on top of an immutable base graph. The layer is built
|
||||
// once per (session, content-hash) tuple by the MCP overlay middleware
|
||||
// (`internal/mcp/overlay_view.go::buildOverlayLayer`) and is consulted
|
||||
// read-only by `OverlaidView`.
|
||||
//
|
||||
// **Identity is preserved.** Gortex node IDs are derived from
|
||||
// `file::symbol` paths, so a symbol that exists in both the on-disk
|
||||
// and overlay versions of a file ends up with the same ID — the
|
||||
// view substitutes the overlay's version transparently. New overlay
|
||||
// symbols (a function the user just typed) get IDs that don't exist
|
||||
// in base; deleted symbols (removed from the buffer) simply aren't in
|
||||
// the layer's per-file node list.
|
||||
//
|
||||
// The layer is immutable after construction. The middleware never
|
||||
// mutates it once the View is in flight; the base graph is never
|
||||
// mutated by overlay flow at all. This is what makes the design
|
||||
// safe for concurrent multi-session deployments — no shared mutable
|
||||
// state between sessions or between an overlay-active session and a
|
||||
// non-overlay session.
|
||||
type OverlayLayer struct {
|
||||
// Files covered by the overlay. The key is the file's graph path
|
||||
// (repo-prefixed in multi-repo mode). Presence in this map means
|
||||
// "the View should hide base's view of this path" — either to
|
||||
// replace it with overlay content (entries[path] != nil) or to
|
||||
// tombstone it (entries[path].Deleted).
|
||||
entries map[string]*overlayFileEntry
|
||||
|
||||
// nodeByID lets GetNode hit a single map lookup. Holds every
|
||||
// non-tombstoned overlay node across every overlay file.
|
||||
nodeByID map[string]*Node
|
||||
|
||||
// outEdges maps each overlay-introduced source node ID to its
|
||||
// resolved outgoing edges. Filled by the local resolver pass at
|
||||
// layer construction.
|
||||
outEdges map[string][]*Edge
|
||||
|
||||
// inEdges is the reverse index of outEdges keyed by target ID,
|
||||
// so OverlaidView.GetInEdges can merge overlay-originating
|
||||
// edges with base in-edges in O(1).
|
||||
inEdges map[string][]*Edge
|
||||
|
||||
// nodesByName/Qual index overlay nodes for FindNodesByName /
|
||||
// GetNodeByQualName fast paths.
|
||||
nodesByName map[string][]*Node
|
||||
nodesByQual map[string]*Node
|
||||
|
||||
// nameRemoved is the set of (name → IDs from base that are no
|
||||
// longer present under the View). FindNodesByName uses this to
|
||||
// filter base hits whose enclosing file is overlaid but whose
|
||||
// id disappeared from the overlay's node list.
|
||||
nameRemoved map[string]map[string]bool
|
||||
}
|
||||
|
||||
// overlayFileEntry carries one file's overlay state inside the
|
||||
// layer. Deleted=true is the tombstone variant — no nodes, no edges.
|
||||
type overlayFileEntry struct {
|
||||
Path string
|
||||
Deleted bool
|
||||
Nodes []*Node
|
||||
}
|
||||
|
||||
// NewOverlayLayer constructs an empty layer. Callers build it up via
|
||||
// AddFile / AddNode / AddEdge during the per-request layer-build
|
||||
// pass, then freeze it by handing it to NewOverlaidView. After that
|
||||
// point the layer is treated as immutable; the View never writes
|
||||
// back.
|
||||
func NewOverlayLayer() *OverlayLayer {
|
||||
return &OverlayLayer{
|
||||
entries: make(map[string]*overlayFileEntry),
|
||||
nodeByID: make(map[string]*Node),
|
||||
outEdges: make(map[string][]*Edge),
|
||||
inEdges: make(map[string][]*Edge),
|
||||
nodesByName: make(map[string][]*Node),
|
||||
nodesByQual: make(map[string]*Node),
|
||||
nameRemoved: make(map[string]map[string]bool),
|
||||
}
|
||||
}
|
||||
|
||||
// MarkFile registers an overlay file. Call once per overlay path
|
||||
// before AddNode / AddEdge for that file. `deleted` true means the
|
||||
// path is a tombstone — the View hides base's view of the path
|
||||
// entirely, returning no nodes from GetFileNodes and treating the
|
||||
// path's node IDs as non-existent for GetNode.
|
||||
func (l *OverlayLayer) MarkFile(graphPath string, deleted bool) {
|
||||
l.entries[graphPath] = &overlayFileEntry{Path: graphPath, Deleted: deleted}
|
||||
}
|
||||
|
||||
// AddNode attaches one parsed overlay node to the layer. Must be
|
||||
// called after MarkFile for the node's file. Idempotent on (graphPath,
|
||||
// node ID) — second add silently replaces.
|
||||
func (l *OverlayLayer) AddNode(graphPath string, n *Node) {
|
||||
if n == nil {
|
||||
return
|
||||
}
|
||||
entry, ok := l.entries[graphPath]
|
||||
if !ok {
|
||||
entry = &overlayFileEntry{Path: graphPath}
|
||||
l.entries[graphPath] = entry
|
||||
}
|
||||
if entry.Deleted {
|
||||
// Tombstone: silently drop. Caller bug — but cheap to absorb.
|
||||
return
|
||||
}
|
||||
entry.Nodes = append(entry.Nodes, n)
|
||||
l.nodeByID[n.ID] = n
|
||||
if n.Name != "" {
|
||||
l.nodesByName[n.Name] = append(l.nodesByName[n.Name], n)
|
||||
}
|
||||
if n.QualName != "" {
|
||||
l.nodesByQual[n.QualName] = n
|
||||
}
|
||||
}
|
||||
|
||||
// AddEdge attaches one resolved overlay edge. The local-resolver
|
||||
// pass at layer construction is expected to have rewritten any
|
||||
// `unresolved::*` placeholders to point at concrete (overlay or
|
||||
// base) node IDs before calling this; edges still carrying the
|
||||
// placeholder are kept verbatim so OverlaidView.GetOutEdges still
|
||||
// surfaces them — query tools can decide how to handle them, just
|
||||
// like base's resolver-skipped edges.
|
||||
func (l *OverlayLayer) AddEdge(e *Edge) {
|
||||
if e == nil {
|
||||
return
|
||||
}
|
||||
l.outEdges[e.From] = append(l.outEdges[e.From], e)
|
||||
l.inEdges[e.To] = append(l.inEdges[e.To], e)
|
||||
}
|
||||
|
||||
// MarkRemoved tells the layer that a base node ID is hidden by the
|
||||
// overlay even though the overlay didn't re-emit it (a symbol the
|
||||
// user deleted from the buffer). FindNodesByName uses this to filter
|
||||
// stale base hits.
|
||||
func (l *OverlayLayer) MarkRemoved(baseName, baseID string) {
|
||||
if baseName == "" || baseID == "" {
|
||||
return
|
||||
}
|
||||
set, ok := l.nameRemoved[baseName]
|
||||
if !ok {
|
||||
set = make(map[string]bool)
|
||||
l.nameRemoved[baseName] = set
|
||||
}
|
||||
set[baseID] = true
|
||||
}
|
||||
|
||||
// HasFile reports whether the overlay covers a particular graph path
|
||||
// (either with replacement content or as a tombstone). The View uses
|
||||
// this to decide whether to consult overlay or base for the path's
|
||||
// reads.
|
||||
func (l *OverlayLayer) HasFile(graphPath string) bool {
|
||||
if l == nil {
|
||||
return false
|
||||
}
|
||||
_, ok := l.entries[graphPath]
|
||||
return ok
|
||||
}
|
||||
|
||||
// IsTombstone reports whether the overlay marks the path as deleted.
|
||||
func (l *OverlayLayer) IsTombstone(graphPath string) bool {
|
||||
if l == nil {
|
||||
return false
|
||||
}
|
||||
e := l.entries[graphPath]
|
||||
return e != nil && e.Deleted
|
||||
}
|
||||
|
||||
// FilePaths returns the sorted list of overlay-covered paths. Used
|
||||
// by analyzers / the diff tool to enumerate the overlay's footprint.
|
||||
func (l *OverlayLayer) FilePaths() []string {
|
||||
if l == nil {
|
||||
return nil
|
||||
}
|
||||
out := make([]string, 0, len(l.entries))
|
||||
for p := range l.entries {
|
||||
out = append(out, p)
|
||||
}
|
||||
sort.Strings(out)
|
||||
return out
|
||||
}
|
||||
|
||||
// HasNode reports whether the overlay layer carries a node with this
|
||||
// ID. Used by the local-resolver pass in the mcp layer to drop base
|
||||
// hits whose file is overlaid but whose specific ID wasn't kept by
|
||||
// the overlay (i.e. the user deleted that symbol from the buffer).
|
||||
func (l *OverlayLayer) HasNode(id string) bool {
|
||||
if l == nil {
|
||||
return false
|
||||
}
|
||||
_, ok := l.nodeByID[id]
|
||||
return ok
|
||||
}
|
||||
|
||||
// NodesByName returns the overlay-introduced nodes with the given
|
||||
// short name. Empty slice when none. Used by the local-resolver
|
||||
// pass.
|
||||
func (l *OverlayLayer) NodesByName(name string) []*Node {
|
||||
if l == nil {
|
||||
return nil
|
||||
}
|
||||
src := l.nodesByName[name]
|
||||
out := make([]*Node, len(src))
|
||||
copy(out, src)
|
||||
return out
|
||||
}
|
||||
|
||||
// OutEdgesByFromAll returns a snapshot of the layer's outgoing-edge
|
||||
// map keyed by source ID. The resolver pass iterates this to rewrite
|
||||
// `unresolved::*` placeholders. The returned map shares its slices
|
||||
// with the layer (resolver mutates Edge.To in place); the map keys
|
||||
// are stable for the snapshot.
|
||||
func (l *OverlayLayer) OutEdgesByFromAll() map[string][]*Edge {
|
||||
if l == nil {
|
||||
return nil
|
||||
}
|
||||
out := make(map[string][]*Edge, len(l.outEdges))
|
||||
for k, v := range l.outEdges {
|
||||
out[k] = v
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// RebuildInEdges rebuilds the reverse-index map after the local
|
||||
// resolver pass mutates Edge.To in place. Cheap: O(#overlay edges).
|
||||
func (l *OverlayLayer) RebuildInEdges() {
|
||||
if l == nil {
|
||||
return
|
||||
}
|
||||
l.inEdges = make(map[string][]*Edge, len(l.outEdges))
|
||||
for _, edges := range l.outEdges {
|
||||
for _, e := range edges {
|
||||
l.inEdges[e.To] = append(l.inEdges[e.To], e)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// nodesForFile returns the overlay nodes for a path (empty for
|
||||
// tombstones). Internal — used by OverlaidView.
|
||||
func (l *OverlayLayer) nodesForFile(graphPath string) []*Node {
|
||||
if l == nil {
|
||||
return nil
|
||||
}
|
||||
e := l.entries[graphPath]
|
||||
if e == nil || e.Deleted {
|
||||
return nil
|
||||
}
|
||||
out := make([]*Node, len(e.Nodes))
|
||||
copy(out, e.Nodes)
|
||||
return out
|
||||
}
|
||||
|
||||
// OverlaidView composes an immutable base Reader with a per-session
|
||||
// overlay layer. Every read path consults the layer first for paths
|
||||
// the overlay covers; falls through to base otherwise. The base is
|
||||
// never mutated; the layer is built once per request and discarded
|
||||
// with the request. This means concurrent sessions — overlay-active
|
||||
// or not — each see their own consistent view, and the file watcher's
|
||||
// reindex passes (which mutate base) don't corrupt overlay queries.
|
||||
type OverlaidView struct {
|
||||
base Reader
|
||||
layer *OverlayLayer
|
||||
|
||||
// statsOnce caches the (potentially expensive) Stats walk so
|
||||
// repeated calls within one request don't pay the AllNodes /
|
||||
// AllEdges cost twice.
|
||||
statsOnce sync.Once
|
||||
stats GraphStats
|
||||
}
|
||||
|
||||
// NewOverlaidView builds a view. If layer is nil the view is a pure
|
||||
// pass-through and consumers pay no overlay overhead.
|
||||
func NewOverlaidView(base Reader, layer *OverlayLayer) *OverlaidView {
|
||||
return &OverlaidView{base: base, layer: layer}
|
||||
}
|
||||
|
||||
// Base exposes the underlying base reader. The diff tool reads
|
||||
// against (view.Base()) and against (view) directly to compute the
|
||||
// delta induced by the overlay.
|
||||
func (v *OverlaidView) Base() Reader { return v.base }
|
||||
|
||||
// Layer exposes the per-session overlay layer (nil when none).
|
||||
// Diagnostic / debug tools use it to introspect what the overlay
|
||||
// covers.
|
||||
func (v *OverlaidView) Layer() *OverlayLayer { return v.layer }
|
||||
|
||||
// IDFile returns the file path encoded in a Gortex node ID, or "" if
|
||||
// the id isn't file-anchored. Gortex IDs follow the pattern
|
||||
// `<filepath>::<symbol>[.member][#param:name]` so the file prefix is
|
||||
// the substring before the first `::`. Module / package / virtual
|
||||
// nodes use other prefixes that won't match an overlay path.
|
||||
func IDFile(id string) string {
|
||||
if id == "" {
|
||||
return ""
|
||||
}
|
||||
if i := strings.Index(id, "::"); i > 0 {
|
||||
return id[:i]
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// nodeBelongsToOverlay reports whether an ID's file is covered by
|
||||
// the layer.
|
||||
func (v *OverlaidView) nodeBelongsToOverlay(id string) bool {
|
||||
if v.layer == nil {
|
||||
return false
|
||||
}
|
||||
return v.layer.HasFile(IDFile(id))
|
||||
}
|
||||
|
||||
// GetNode returns the overlay's version of a node when the ID
|
||||
// belongs to an overlaid file, the base node otherwise. Returns nil
|
||||
// when the symbol exists in base but was removed in the overlay
|
||||
// (the per-file overlay node list didn't include it).
|
||||
func (v *OverlaidView) GetNode(id string) *Node {
|
||||
if v.layer != nil {
|
||||
if v.nodeBelongsToOverlay(id) {
|
||||
return v.layer.nodeByID[id] // may be nil — overlay deleted it
|
||||
}
|
||||
}
|
||||
if v.base == nil {
|
||||
return nil
|
||||
}
|
||||
return v.base.GetNode(id)
|
||||
}
|
||||
|
||||
// GetNodesByIDs returns the overlay-aware *Node for each input ID.
|
||||
// Overlay-owned IDs short-circuit to the per-session layer (and may
|
||||
// resolve to nil when the overlay deleted the node); the remainder
|
||||
// fans out as a single batched lookup against the base store. Missing
|
||||
// IDs are simply absent from the returned map.
|
||||
func (v *OverlaidView) GetNodesByIDs(ids []string) map[string]*Node {
|
||||
if len(ids) == 0 {
|
||||
return nil
|
||||
}
|
||||
out := make(map[string]*Node, len(ids))
|
||||
baseIDs := ids[:0:0] // fresh backing array — never aliases caller's slice
|
||||
for _, id := range ids {
|
||||
if id == "" {
|
||||
continue
|
||||
}
|
||||
if _, dup := out[id]; dup {
|
||||
continue
|
||||
}
|
||||
if v.layer != nil && v.nodeBelongsToOverlay(id) {
|
||||
if n := v.layer.nodeByID[id]; n != nil {
|
||||
out[id] = n
|
||||
}
|
||||
// Overlay tombstone — ID is hidden, do not fall back to base.
|
||||
continue
|
||||
}
|
||||
// Track for the single base round-trip; reserve a slot in `out`
|
||||
// only after the batched lookup returns.
|
||||
baseIDs = append(baseIDs, id)
|
||||
}
|
||||
if len(baseIDs) > 0 && v.base != nil {
|
||||
for id, n := range v.base.GetNodesByIDs(baseIDs) {
|
||||
if n != nil {
|
||||
out[id] = n
|
||||
}
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// GetNodeByQualName: overlay first, then base. Base hits are filtered
|
||||
// to drop entries whose file is overlaid (the overlay's view wins).
|
||||
func (v *OverlaidView) GetNodeByQualName(qualName string) *Node {
|
||||
if v.layer != nil {
|
||||
if n := v.layer.nodesByQual[qualName]; n != nil {
|
||||
return n
|
||||
}
|
||||
}
|
||||
if v.base == nil {
|
||||
return nil
|
||||
}
|
||||
n := v.base.GetNodeByQualName(qualName)
|
||||
if n != nil && v.layer != nil && v.layer.HasFile(IDFile(n.ID)) {
|
||||
// Base hit landed in an overlaid file but the overlay didn't
|
||||
// re-emit a node with this qualified name → it's gone.
|
||||
return nil
|
||||
}
|
||||
return n
|
||||
}
|
||||
|
||||
// GetNodesByQualNames resolves each name through GetNodeByQualName so the
|
||||
// overlay's layer-first / shadowed-file filtering applies — an inherited
|
||||
// base batch would bypass the overlay. Per-name is fine: an interactive
|
||||
// overlay's working set is small (the batch form exists for the
|
||||
// cold-warmup scale on the base store, not here). Returns only hits.
|
||||
func (v *OverlaidView) GetNodesByQualNames(qualNames []string) map[string]*Node {
|
||||
out := make(map[string]*Node, len(qualNames))
|
||||
for _, q := range qualNames {
|
||||
if q == "" {
|
||||
continue
|
||||
}
|
||||
if _, done := out[q]; done {
|
||||
continue
|
||||
}
|
||||
if n := v.GetNodeByQualName(q); n != nil {
|
||||
out[q] = n
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// FindNodesByName merges base hits (filtered to drop nodes in
|
||||
// overlaid files unless the overlay re-emitted them) with overlay
|
||||
// hits. Order is overlay-first, then base — callers that picked
|
||||
// "first match" semantics get the overlay version automatically.
|
||||
func (v *OverlaidView) FindNodesByName(name string) []*Node {
|
||||
var out []*Node
|
||||
if v.layer != nil {
|
||||
out = append(out, v.layer.nodesByName[name]...)
|
||||
}
|
||||
if v.base == nil {
|
||||
return out
|
||||
}
|
||||
for _, n := range v.base.FindNodesByName(name) {
|
||||
if v.layer != nil {
|
||||
if v.layer.HasFile(IDFile(n.ID)) {
|
||||
// Overlaid file: base's node for this name is
|
||||
// always hidden. If the overlay re-emitted the same
|
||||
// ID it's already in `out` from the layer's
|
||||
// nodesByName prepend above; if the overlay deleted
|
||||
// the symbol it must not surface at all. Either way
|
||||
// we skip — no need to discriminate.
|
||||
continue
|
||||
}
|
||||
if v.layer.nameRemoved[name] != nil && v.layer.nameRemoved[name][n.ID] {
|
||||
continue
|
||||
}
|
||||
}
|
||||
out = append(out, n)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// FindNodesByNameContaining merges overlay-touched name hits with the
|
||||
// base result, then re-applies the per-overlay-file masking the same
|
||||
// way FindNodesByName does. Order is overlay-first, then base; the
|
||||
// limit caps the merged total. Empty substr or both layers nil
|
||||
// returns nil.
|
||||
func (v *OverlaidView) FindNodesByNameContaining(substr string, limit int) []*Node {
|
||||
if substr == "" {
|
||||
return nil
|
||||
}
|
||||
needle := strings.ToLower(substr)
|
||||
var out []*Node
|
||||
// Overlay-side: walk the layer's nodesByName index — the same
|
||||
// bucket FindNodesByName reads from — and accept any name whose
|
||||
// lowercase form contains the needle.
|
||||
if v.layer != nil {
|
||||
for name, bucket := range v.layer.nodesByName {
|
||||
if strings.Contains(strings.ToLower(name), needle) {
|
||||
out = append(out, bucket...)
|
||||
if limit > 0 && len(out) >= limit {
|
||||
return out[:limit]
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if v.base == nil {
|
||||
return out
|
||||
}
|
||||
// Base-side: fetch with an inflated limit so overlay-mask drops
|
||||
// don't leave a short page. Then re-apply the same overlaid-file
|
||||
// + name-removed mask FindNodesByName uses.
|
||||
fetch := limit
|
||||
if fetch > 0 {
|
||||
fetch *= 2
|
||||
}
|
||||
for _, n := range v.base.FindNodesByNameContaining(substr, fetch) {
|
||||
if v.layer != nil {
|
||||
if v.layer.HasFile(IDFile(n.ID)) {
|
||||
continue
|
||||
}
|
||||
if v.layer.nameRemoved[n.Name] != nil && v.layer.nameRemoved[n.Name][n.ID] {
|
||||
continue
|
||||
}
|
||||
}
|
||||
out = append(out, n)
|
||||
if limit > 0 && len(out) >= limit {
|
||||
return out[:limit]
|
||||
}
|
||||
}
|
||||
if limit > 0 && len(out) > limit {
|
||||
out = out[:limit]
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// GetFileNodes: if the path is overlaid, return overlay's nodes
|
||||
// (empty for tombstones). Otherwise pass through to base.
|
||||
func (v *OverlaidView) GetFileNodes(filePath string) []*Node {
|
||||
if v.layer != nil && v.layer.HasFile(filePath) {
|
||||
return v.layer.nodesForFile(filePath)
|
||||
}
|
||||
if v.base == nil {
|
||||
return nil
|
||||
}
|
||||
return v.base.GetFileNodes(filePath)
|
||||
}
|
||||
|
||||
// GetRepoNodes filters base's per-repo node list by dropping nodes
|
||||
// whose file is overlaid (unless the overlay re-emitted them) and
|
||||
// appending the overlay's nodes for any overlaid file inside the
|
||||
// requested repo prefix.
|
||||
func (v *OverlaidView) GetRepoNodes(repoPrefix string) []*Node {
|
||||
if v.base == nil {
|
||||
return nil
|
||||
}
|
||||
baseNodes := v.base.GetRepoNodes(repoPrefix)
|
||||
if v.layer == nil {
|
||||
return baseNodes
|
||||
}
|
||||
out := make([]*Node, 0, len(baseNodes))
|
||||
for _, n := range baseNodes {
|
||||
if v.layer.HasFile(IDFile(n.ID)) {
|
||||
// File is overlaid. Surface only if the overlay
|
||||
// re-emitted this exact ID; otherwise it's hidden.
|
||||
if v.layer.nodeByID[n.ID] == nil {
|
||||
continue
|
||||
}
|
||||
}
|
||||
out = append(out, n)
|
||||
}
|
||||
for _, path := range v.layer.FilePaths() {
|
||||
if !strings.HasPrefix(path, repoPrefix+"/") && path != repoPrefix {
|
||||
continue
|
||||
}
|
||||
out = append(out, v.layer.nodesForFile(path)...)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// GetOutEdges: when the source node's file is overlaid, use the
|
||||
// overlay's resolved out-edges. Otherwise return base's edges but
|
||||
// drop any whose target points into an overlaid file at a node ID
|
||||
// the overlay no longer carries (target deleted in buffer).
|
||||
func (v *OverlaidView) GetOutEdges(nodeID string) []*Edge {
|
||||
if v.layer != nil && v.nodeBelongsToOverlay(nodeID) {
|
||||
src := v.layer.outEdges[nodeID]
|
||||
out := make([]*Edge, len(src))
|
||||
copy(out, src)
|
||||
return out
|
||||
}
|
||||
if v.base == nil {
|
||||
return nil
|
||||
}
|
||||
edges := v.base.GetOutEdges(nodeID)
|
||||
if v.layer == nil {
|
||||
return edges
|
||||
}
|
||||
out := edges[:0:0]
|
||||
for _, e := range edges {
|
||||
if v.layer.HasFile(IDFile(e.To)) {
|
||||
if v.layer.nodeByID[e.To] == nil {
|
||||
continue // target deleted in overlay
|
||||
}
|
||||
}
|
||||
out = append(out, e)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// GetInEdges merges base's incoming edges (filtered to drop those
|
||||
// originating in overlaid files, since those are replaced by overlay
|
||||
// versions) with the overlay's in-edges for the same target.
|
||||
func (v *OverlaidView) GetInEdges(nodeID string) []*Edge {
|
||||
if v.layer == nil {
|
||||
if v.base == nil {
|
||||
return nil
|
||||
}
|
||||
return v.base.GetInEdges(nodeID)
|
||||
}
|
||||
var out []*Edge
|
||||
if v.base != nil {
|
||||
for _, e := range v.base.GetInEdges(nodeID) {
|
||||
if v.layer.HasFile(IDFile(e.From)) {
|
||||
// Source is overlaid — the overlay's version of this
|
||||
// edge wins (or the overlay simply deleted the call).
|
||||
continue
|
||||
}
|
||||
if v.layer.HasFile(IDFile(e.To)) && v.layer.nodeByID[e.To] == nil {
|
||||
// Target was deleted by the overlay.
|
||||
continue
|
||||
}
|
||||
out = append(out, e)
|
||||
}
|
||||
}
|
||||
out = append(out, v.layer.inEdges[nodeID]...)
|
||||
return out
|
||||
}
|
||||
|
||||
// GetOutEdgesByNodeIDs returns the overlay-aware outgoing-edge map for
|
||||
// every input id. Overlay-owned ids short-circuit to the per-session
|
||||
// layer; the remainder fans out as a single batched lookup against
|
||||
// the base store. Output mirrors GetOutEdges's per-id semantics
|
||||
// (target-side overlay deletions filtered out), but in one cgo
|
||||
// round-trip per direction instead of N.
|
||||
func (v *OverlaidView) GetOutEdgesByNodeIDs(ids []string) map[string][]*Edge {
|
||||
if len(ids) == 0 {
|
||||
return nil
|
||||
}
|
||||
out := make(map[string][]*Edge, len(ids))
|
||||
baseIDs := ids[:0:0]
|
||||
seen := make(map[string]struct{}, len(ids))
|
||||
for _, id := range ids {
|
||||
if id == "" {
|
||||
continue
|
||||
}
|
||||
if _, dup := seen[id]; dup {
|
||||
continue
|
||||
}
|
||||
seen[id] = struct{}{}
|
||||
if v.layer != nil && v.nodeBelongsToOverlay(id) {
|
||||
src := v.layer.outEdges[id]
|
||||
cp := make([]*Edge, len(src))
|
||||
copy(cp, src)
|
||||
out[id] = cp
|
||||
continue
|
||||
}
|
||||
baseIDs = append(baseIDs, id)
|
||||
}
|
||||
if len(baseIDs) > 0 && v.base != nil {
|
||||
base := v.base.GetOutEdgesByNodeIDs(baseIDs)
|
||||
for id, edges := range base {
|
||||
if v.layer == nil {
|
||||
out[id] = edges
|
||||
continue
|
||||
}
|
||||
filtered := edges[:0:0]
|
||||
for _, e := range edges {
|
||||
if v.layer.HasFile(IDFile(e.To)) {
|
||||
if v.layer.nodeByID[e.To] == nil {
|
||||
continue // target deleted in overlay
|
||||
}
|
||||
}
|
||||
filtered = append(filtered, e)
|
||||
}
|
||||
out[id] = filtered
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// GetInEdgesByNodeIDs is the inbound sibling of GetOutEdgesByNodeIDs.
|
||||
// Merges base in-edges (filtered to drop edges sourced in overlaid
|
||||
// files) with overlay-introduced in-edges for each input id, all in a
|
||||
// single batched base round-trip.
|
||||
func (v *OverlaidView) GetInEdgesByNodeIDs(ids []string) map[string][]*Edge {
|
||||
if len(ids) == 0 {
|
||||
return nil
|
||||
}
|
||||
out := make(map[string][]*Edge, len(ids))
|
||||
seen := make(map[string]struct{}, len(ids))
|
||||
uniq := ids[:0:0]
|
||||
for _, id := range ids {
|
||||
if id == "" {
|
||||
continue
|
||||
}
|
||||
if _, dup := seen[id]; dup {
|
||||
continue
|
||||
}
|
||||
seen[id] = struct{}{}
|
||||
uniq = append(uniq, id)
|
||||
}
|
||||
if len(uniq) == 0 {
|
||||
return out
|
||||
}
|
||||
if v.base != nil {
|
||||
base := v.base.GetInEdgesByNodeIDs(uniq)
|
||||
for _, id := range uniq {
|
||||
edges := base[id]
|
||||
if v.layer == nil {
|
||||
out[id] = edges
|
||||
continue
|
||||
}
|
||||
filtered := edges[:0:0]
|
||||
for _, e := range edges {
|
||||
if v.layer.HasFile(IDFile(e.From)) {
|
||||
continue // source is overlaid — overlay's version wins
|
||||
}
|
||||
if v.layer.HasFile(IDFile(e.To)) && v.layer.nodeByID[e.To] == nil {
|
||||
continue // target was deleted by overlay
|
||||
}
|
||||
filtered = append(filtered, e)
|
||||
}
|
||||
out[id] = filtered
|
||||
}
|
||||
}
|
||||
if v.layer != nil {
|
||||
for _, id := range uniq {
|
||||
if extras := v.layer.inEdges[id]; len(extras) > 0 {
|
||||
out[id] = append(out[id], extras...)
|
||||
}
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// AllNodes returns base's nodes minus nodes in overlaid files, plus
|
||||
// every node the overlay introduced. Bulk-read consumers (analyzers,
|
||||
// search reindex, snapshot export) get an overlay-consistent view
|
||||
// without paying any extra copy beyond the base snapshot's.
|
||||
func (v *OverlaidView) AllNodes() []*Node {
|
||||
if v.base == nil {
|
||||
return nil
|
||||
}
|
||||
baseNodes := v.base.AllNodes()
|
||||
if v.layer == nil {
|
||||
return baseNodes
|
||||
}
|
||||
out := make([]*Node, 0, len(baseNodes))
|
||||
for _, n := range baseNodes {
|
||||
if v.layer.HasFile(IDFile(n.ID)) {
|
||||
if v.layer.nodeByID[n.ID] == nil {
|
||||
continue
|
||||
}
|
||||
// Else: overlay's version was kept under the same ID; the
|
||||
// layer's slice will include it below, so skip base's copy
|
||||
// to avoid duplicates.
|
||||
continue
|
||||
}
|
||||
out = append(out, n)
|
||||
}
|
||||
for _, n := range v.layer.nodeByID {
|
||||
out = append(out, n)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// AllEdges returns base's edges minus those involving overlaid
|
||||
// files, plus every overlay-introduced edge.
|
||||
func (v *OverlaidView) AllEdges() []*Edge {
|
||||
if v.base == nil {
|
||||
return nil
|
||||
}
|
||||
baseEdges := v.base.AllEdges()
|
||||
if v.layer == nil {
|
||||
return baseEdges
|
||||
}
|
||||
out := make([]*Edge, 0, len(baseEdges))
|
||||
for _, e := range baseEdges {
|
||||
if v.layer.HasFile(IDFile(e.From)) || v.layer.HasFile(IDFile(e.To)) {
|
||||
continue
|
||||
}
|
||||
out = append(out, e)
|
||||
}
|
||||
for _, edges := range v.layer.outEdges {
|
||||
out = append(out, edges...)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// NodeCount / EdgeCount — derived from base counters adjusted by the
|
||||
// overlay delta. Cheap enough to recompute per call.
|
||||
func (v *OverlaidView) NodeCount() int {
|
||||
if v.base == nil {
|
||||
return 0
|
||||
}
|
||||
if v.layer == nil {
|
||||
return v.base.NodeCount()
|
||||
}
|
||||
delta := 0
|
||||
for path, entry := range v.layer.entries {
|
||||
baseCount := len(v.base.GetFileNodes(path))
|
||||
if entry.Deleted {
|
||||
delta -= baseCount
|
||||
continue
|
||||
}
|
||||
delta += len(entry.Nodes) - baseCount
|
||||
}
|
||||
return v.base.NodeCount() + delta
|
||||
}
|
||||
|
||||
func (v *OverlaidView) EdgeCount() int {
|
||||
if v.base == nil {
|
||||
return 0
|
||||
}
|
||||
if v.layer == nil {
|
||||
return v.base.EdgeCount()
|
||||
}
|
||||
return len(v.AllEdges())
|
||||
}
|
||||
|
||||
// EdgeIdentityRevisions delegates to the base graph: provenance churn
|
||||
// is a property of the persistent graph, and an overlay layer is a
|
||||
// non-mutating per-session shadow that never upgrades edge provenance.
|
||||
func (v *OverlaidView) EdgeIdentityRevisions() int {
|
||||
if v.base == nil {
|
||||
return 0
|
||||
}
|
||||
return v.base.EdgeIdentityRevisions()
|
||||
}
|
||||
|
||||
// Stats is best-effort under overlay: we report base's stats (the
|
||||
// analyzer-shaped GraphStats requires per-kind / per-language
|
||||
// breakdowns that the overlay layer doesn't expose cheaply). Caching
|
||||
// keeps repeated Stats() calls inside one request to a single base
|
||||
// lookup.
|
||||
func (v *OverlaidView) Stats() GraphStats {
|
||||
if v.base == nil {
|
||||
return GraphStats{}
|
||||
}
|
||||
v.statsOnce.Do(func() {
|
||||
v.stats = v.base.Stats()
|
||||
})
|
||||
return v.stats
|
||||
}
|
||||
|
||||
// RepoStats — same conservatism as Stats; overlay deltas are
|
||||
// excluded. The handful of tools that read RepoStats are bookkeeping
|
||||
// rather than load-bearing, and the overlay-affected nodes are still
|
||||
// reachable through the per-node read paths.
|
||||
func (v *OverlaidView) RepoStats() map[string]GraphStats {
|
||||
if v.base == nil {
|
||||
return nil
|
||||
}
|
||||
return v.base.RepoStats()
|
||||
}
|
||||
|
||||
// Compile-time assertion that *OverlaidView satisfies Reader.
|
||||
var _ Reader = (*OverlaidView)(nil)
|
||||
@@ -0,0 +1,402 @@
|
||||
package graph
|
||||
|
||||
import (
|
||||
"go/ast"
|
||||
"go/parser"
|
||||
"go/token"
|
||||
"io/fs"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"sort"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// Schema-vs-extractor parity audit.
|
||||
//
|
||||
// The graph package declares ~30 NodeKinds and ~55 EdgeKinds. Without
|
||||
// a regression fence, the schema drifts: a new kind gets added with a
|
||||
// detailed comment, but nobody wires the emitter, and the constant
|
||||
// quietly outlives its purpose. We've seen this exact failure mode
|
||||
// before — `KindRelease` was declared and documented for many releases
|
||||
// before any extractor actually instantiated it.
|
||||
//
|
||||
// TestSchemaParityAudit walks every Go source file under the repo
|
||||
// (excluding worktrees, vendor, build artifacts) and asserts that
|
||||
// every declared NodeKind and EdgeKind constant is referenced from
|
||||
// at least one file outside the declaring file (node.go / edge.go).
|
||||
//
|
||||
// "Referenced" here means the bare identifier appears in production
|
||||
// source — the resolver, an extractor, an enricher, an analyzer, or
|
||||
// a downstream consumer. Test-only references count too: a test that
|
||||
// exercises the consumer is still proof the kind has a real role.
|
||||
//
|
||||
// When the audit fails the failure message names the orphan constant
|
||||
// and points at the file where it was declared so the next sweep
|
||||
// either wires the emitter or removes the constant. Reserved kinds
|
||||
// (declared today, emitter coming next sprint) go on the explicit
|
||||
// allowlist with a documented reason — that turns "we forgot" into
|
||||
// "we deliberately deferred."
|
||||
func TestSchemaParityAudit(t *testing.T) {
|
||||
repoRoot, err := findRepoRoot()
|
||||
if err != nil {
|
||||
t.Fatalf("locate repo root: %v", err)
|
||||
}
|
||||
|
||||
declared, err := extractKindDeclarations(filepath.Join(repoRoot, "internal", "graph"))
|
||||
if err != nil {
|
||||
t.Fatalf("scan declarations: %v", err)
|
||||
}
|
||||
if len(declared) == 0 {
|
||||
t.Fatal("scan returned zero declarations — audit is broken")
|
||||
}
|
||||
|
||||
usage, err := scanKindUsage(repoRoot, declared)
|
||||
if err != nil {
|
||||
t.Fatalf("scan usage: %v", err)
|
||||
}
|
||||
|
||||
var orphans []string
|
||||
for _, d := range declared {
|
||||
if reservedKinds[d.Name] {
|
||||
continue
|
||||
}
|
||||
// "Direct" usage means we found the constant referenced
|
||||
// anywhere outside its own declaration file. Mapping-table
|
||||
// references inside graph/edge.go (CrossRepoKindFor etc.)
|
||||
// count because they prove the constant is reachable from
|
||||
// production query paths — the actual emit site can then sit
|
||||
// in a resolver or enricher that calls the mapping fn.
|
||||
if usage[d.Name] == 0 {
|
||||
orphans = append(orphans, formatOrphan(d))
|
||||
}
|
||||
}
|
||||
if len(orphans) > 0 {
|
||||
sort.Strings(orphans)
|
||||
t.Errorf("schema-parity audit found %d orphan kind(s) — wire emitter or remove from schema:\n\n%s\n\n"+
|
||||
"Add to `reservedKinds` in this file with a documented reason if the gap is intentional.",
|
||||
len(orphans), strings.Join(orphans, "\n"))
|
||||
}
|
||||
}
|
||||
|
||||
// reservedKinds is the explicit allowlist of constants that are
|
||||
// declared today and intentionally lack a production emit site. Every
|
||||
// entry must carry a comment explaining the reason — "we forgot" is
|
||||
// not a valid reason; the answer is to either ship the emitter or
|
||||
// delete the declaration.
|
||||
var reservedKinds = map[string]bool{
|
||||
// (empty — every declared kind currently has a production emit
|
||||
// site. Future deferrals go here with a documented reason.)
|
||||
}
|
||||
|
||||
// kindDecl is a single NodeKind / EdgeKind constant declaration we
|
||||
// extracted from internal/graph. The Path / Line locate it for the
|
||||
// failure message so the next sweep knows where to look.
|
||||
type kindDecl struct {
|
||||
Name string
|
||||
Kind string // "NodeKind" or "EdgeKind"
|
||||
Path string
|
||||
Line int
|
||||
}
|
||||
|
||||
// extractKindDeclarations parses every .go file under the given
|
||||
// directory and returns each `const Name SomeKind = ...` whose RHS
|
||||
// type is NodeKind or EdgeKind. Anything else is ignored — the audit
|
||||
// is intentionally scoped to schema kinds.
|
||||
func extractKindDeclarations(dir string) ([]kindDecl, error) {
|
||||
var out []kindDecl
|
||||
fset := token.NewFileSet()
|
||||
|
||||
err := filepath.WalkDir(dir, func(path string, d fs.DirEntry, err error) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if d.IsDir() {
|
||||
return nil
|
||||
}
|
||||
if !strings.HasSuffix(path, ".go") || strings.HasSuffix(path, "_test.go") {
|
||||
return nil
|
||||
}
|
||||
file, err := parser.ParseFile(fset, path, nil, parser.ParseComments)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for _, decl := range file.Decls {
|
||||
gen, ok := decl.(*ast.GenDecl)
|
||||
if !ok || gen.Tok != token.CONST {
|
||||
continue
|
||||
}
|
||||
for _, spec := range gen.Specs {
|
||||
vs, ok := spec.(*ast.ValueSpec)
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
typeName, ok := simpleTypeName(vs.Type)
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
if typeName != "NodeKind" && typeName != "EdgeKind" {
|
||||
continue
|
||||
}
|
||||
for _, ident := range vs.Names {
|
||||
pos := fset.Position(ident.NamePos)
|
||||
out = append(out, kindDecl{
|
||||
Name: ident.Name,
|
||||
Kind: typeName,
|
||||
Path: path,
|
||||
Line: pos.Line,
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// simpleTypeName pulls the identifier name out of a simple type
|
||||
// expression. Returns false for any expression that isn't a bare
|
||||
// identifier — qualified names, function types, etc. — because the
|
||||
// audit only cares about the local NodeKind / EdgeKind types.
|
||||
func simpleTypeName(expr ast.Expr) (string, bool) {
|
||||
id, ok := expr.(*ast.Ident)
|
||||
if !ok {
|
||||
return "", false
|
||||
}
|
||||
return id.Name, true
|
||||
}
|
||||
|
||||
// scanKindUsage counts how many references to each declared constant
|
||||
// exist outside its declaring file. The scanner is regex-based
|
||||
// (rather than AST-based) because the audit needs to catch every
|
||||
// reachable mention — including string-literal mappings and test
|
||||
// fixtures — without paying the parse cost of every .go file in the
|
||||
// repo. Word-boundary anchors keep "EdgeReads" and "EdgeReadsCol"
|
||||
// (and "EdgeReadsConfig") distinct.
|
||||
func scanKindUsage(repoRoot string, declared []kindDecl) (map[string]int, error) {
|
||||
declaringFiles := make(map[string]string, len(declared))
|
||||
for _, d := range declared {
|
||||
declaringFiles[d.Name] = d.Path
|
||||
}
|
||||
|
||||
// One combined alternation rather than one regex per kind. Matching
|
||||
// every .go file in the repo against ~100 separate \bName\b patterns
|
||||
// is O(files × kinds) and grows past the CI timeout as both the tree
|
||||
// and the schema grow; a single \b(?:Name1|Name2|…)\b scanned once per
|
||||
// file is O(files). The \b anchors keep the identifiers as distinct as
|
||||
// before — `EdgeReads` still does NOT match inside `EdgeReadsCol`,
|
||||
// because the inner \b fails between two word characters and the
|
||||
// engine falls through to the longer alternative.
|
||||
names := make([]string, 0, len(declared))
|
||||
seen := make(map[string]bool, len(declared))
|
||||
for _, d := range declared {
|
||||
if seen[d.Name] {
|
||||
continue
|
||||
}
|
||||
seen[d.Name] = true
|
||||
names = append(names, regexp.QuoteMeta(d.Name))
|
||||
}
|
||||
re := regexp.MustCompile(`\b(?:` + strings.Join(names, "|") + `)\b`)
|
||||
|
||||
usage := make(map[string]int, len(declared))
|
||||
|
||||
err := filepath.WalkDir(repoRoot, func(path string, d fs.DirEntry, err error) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if d.IsDir() {
|
||||
base := d.Name()
|
||||
// Skip vendor, build caches, worktrees, eval venvs,
|
||||
// transcript dumps — none of those carry production
|
||||
// emit sites.
|
||||
switch base {
|
||||
case ".git", "vendor", "node_modules", ".claude", ".cache",
|
||||
".venv", "venv", "build", "dist", "debug":
|
||||
return filepath.SkipDir
|
||||
}
|
||||
return nil
|
||||
}
|
||||
if !strings.HasSuffix(path, ".go") {
|
||||
return nil
|
||||
}
|
||||
// Worktrees can hide a duplicate copy of the source tree
|
||||
// under .claude/worktrees/* — already skipped by the .claude
|
||||
// directory rule above; defensive double-check.
|
||||
if strings.Contains(path, "/worktrees/") {
|
||||
return nil
|
||||
}
|
||||
// Don't let the audit's own file count as a reference — it
|
||||
// names every constant in its failure-message format so a
|
||||
// naive regex would mark every kind as "used."
|
||||
if strings.HasSuffix(path, "/parity_audit_test.go") {
|
||||
return nil
|
||||
}
|
||||
data, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return nil // unreadable files don't fail the audit
|
||||
}
|
||||
text := stripGoComments(string(data))
|
||||
found := make(map[string]bool)
|
||||
for _, m := range re.FindAllString(text, -1) {
|
||||
found[m] = true
|
||||
}
|
||||
for name := range found {
|
||||
// A constant referenced only inside its own declaring file
|
||||
// is not a real consumer — exclude it exactly as the
|
||||
// per-pattern scan did.
|
||||
if path == declaringFiles[name] {
|
||||
continue
|
||||
}
|
||||
usage[name]++
|
||||
}
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return usage, nil
|
||||
}
|
||||
|
||||
// findRepoRoot walks up from the test's working directory looking
|
||||
// for go.mod. Lets the audit run from any package depth — the test
|
||||
// stays at internal/graph but the scan needs the repo root.
|
||||
func findRepoRoot() (string, error) {
|
||||
dir, err := os.Getwd()
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
for {
|
||||
if _, err := os.Stat(filepath.Join(dir, "go.mod")); err == nil {
|
||||
return dir, nil
|
||||
}
|
||||
parent := filepath.Dir(dir)
|
||||
if parent == dir {
|
||||
return "", os.ErrNotExist
|
||||
}
|
||||
dir = parent
|
||||
}
|
||||
}
|
||||
|
||||
func formatOrphan(d kindDecl) string {
|
||||
rel := d.Path
|
||||
if root, err := findRepoRoot(); err == nil {
|
||||
if r, err := filepath.Rel(root, d.Path); err == nil {
|
||||
rel = r
|
||||
}
|
||||
}
|
||||
return " " + d.Kind + "." + d.Name + " (declared at " + rel + ":" + itoa(d.Line) + ")"
|
||||
}
|
||||
|
||||
// stripGoComments removes line and block comments from Go source so
|
||||
// the usage scan only counts identifiers that actually appear in
|
||||
// executable code. Without this a constant mentioned in a docstring
|
||||
// elsewhere in the tree would falsely satisfy the parity audit.
|
||||
//
|
||||
// The implementation is intentionally simple — character-level scan
|
||||
// with a small state machine — because the alternative (full Go
|
||||
// parsing) is overkill for the audit's needs and slow when invoked
|
||||
// across every .go file in the repo. Strings (single- and
|
||||
// double-quoted, plus raw backtick) are preserved verbatim so a
|
||||
// string literal containing "//" isn't mistaken for a comment.
|
||||
func stripGoComments(src string) string {
|
||||
var b strings.Builder
|
||||
b.Grow(len(src))
|
||||
const (
|
||||
stateCode = iota
|
||||
stateLineComment
|
||||
stateBlockComment
|
||||
stateString
|
||||
stateRawString
|
||||
stateRune
|
||||
)
|
||||
state := stateCode
|
||||
for i := 0; i < len(src); i++ {
|
||||
c := src[i]
|
||||
switch state {
|
||||
case stateCode:
|
||||
switch {
|
||||
case c == '/' && i+1 < len(src) && src[i+1] == '/':
|
||||
state = stateLineComment
|
||||
i++
|
||||
b.WriteByte(' ')
|
||||
case c == '/' && i+1 < len(src) && src[i+1] == '*':
|
||||
state = stateBlockComment
|
||||
i++
|
||||
b.WriteByte(' ')
|
||||
case c == '"':
|
||||
state = stateString
|
||||
b.WriteByte(c)
|
||||
case c == '`':
|
||||
state = stateRawString
|
||||
b.WriteByte(c)
|
||||
case c == '\'':
|
||||
state = stateRune
|
||||
b.WriteByte(c)
|
||||
default:
|
||||
b.WriteByte(c)
|
||||
}
|
||||
case stateLineComment:
|
||||
if c == '\n' {
|
||||
state = stateCode
|
||||
b.WriteByte(c)
|
||||
}
|
||||
case stateBlockComment:
|
||||
if c == '*' && i+1 < len(src) && src[i+1] == '/' {
|
||||
state = stateCode
|
||||
i++
|
||||
b.WriteByte(' ')
|
||||
}
|
||||
case stateString:
|
||||
b.WriteByte(c)
|
||||
if c == '\\' && i+1 < len(src) {
|
||||
b.WriteByte(src[i+1])
|
||||
i++
|
||||
} else if c == '"' {
|
||||
state = stateCode
|
||||
}
|
||||
case stateRawString:
|
||||
b.WriteByte(c)
|
||||
if c == '`' {
|
||||
state = stateCode
|
||||
}
|
||||
case stateRune:
|
||||
b.WriteByte(c)
|
||||
if c == '\\' && i+1 < len(src) {
|
||||
b.WriteByte(src[i+1])
|
||||
i++
|
||||
} else if c == '\'' {
|
||||
state = stateCode
|
||||
}
|
||||
}
|
||||
}
|
||||
return b.String()
|
||||
}
|
||||
|
||||
func itoa(n int) string {
|
||||
// Tiny helper so the audit file doesn't need strconv just for one
|
||||
// formatter. Keeps the import set minimal — easier to read at a
|
||||
// glance.
|
||||
if n == 0 {
|
||||
return "0"
|
||||
}
|
||||
neg := n < 0
|
||||
if neg {
|
||||
n = -n
|
||||
}
|
||||
var buf [20]byte
|
||||
i := len(buf)
|
||||
for n > 0 {
|
||||
i--
|
||||
buf[i] = byte('0' + n%10)
|
||||
n /= 10
|
||||
}
|
||||
if neg {
|
||||
i--
|
||||
buf[i] = '-'
|
||||
}
|
||||
return string(buf[i:])
|
||||
}
|
||||
@@ -0,0 +1,47 @@
|
||||
package graph
|
||||
|
||||
import "testing"
|
||||
|
||||
func TestProvenanceWeight_Tiers(t *testing.T) {
|
||||
w := func(origin string) float64 {
|
||||
return ProvenanceWeight(&Edge{From: "a", To: "b", Kind: EdgeCalls, Origin: origin})
|
||||
}
|
||||
cases := []struct {
|
||||
origin string
|
||||
want float64
|
||||
}{
|
||||
{OriginLSPResolved, provWeightLSP},
|
||||
{OriginLSPDispatch, provWeightLSP},
|
||||
{OriginASTResolved, ProvenanceWeightMax},
|
||||
{OriginASTInferred, provWeightASTInferred},
|
||||
{OriginTextMatched, ProvenanceWeightMin},
|
||||
}
|
||||
for _, c := range cases {
|
||||
if got := w(c.origin); got != c.want {
|
||||
t.Errorf("ProvenanceWeight(origin=%q) = %v, want %v", c.origin, got, c.want)
|
||||
}
|
||||
}
|
||||
// LSP and text tiers are attenuated below the ast_resolved baseline.
|
||||
if !(w(OriginLSPResolved) < ProvenanceWeightMax) {
|
||||
t.Errorf("lsp tier must weight below ast_resolved baseline")
|
||||
}
|
||||
if !(w(OriginTextMatched) < ProvenanceWeightMax) {
|
||||
t.Errorf("text tier must weight below ast_resolved baseline")
|
||||
}
|
||||
}
|
||||
|
||||
func TestProvenanceWeight_BackfillsAndNilSafe(t *testing.T) {
|
||||
// A nil edge weights at the trusted baseline.
|
||||
if got := ProvenanceWeight(nil); got != ProvenanceWeightMax {
|
||||
t.Errorf("nil edge = %v, want %v", got, ProvenanceWeightMax)
|
||||
}
|
||||
// Unset Origin backfills via DefaultOriginFor: a structural import
|
||||
// edge is ast_resolved (baseline); a bare call edge with no
|
||||
// confidence falls to text_matched (the minimum).
|
||||
if got := ProvenanceWeight(&Edge{Kind: EdgeImports}); got != ProvenanceWeightMax {
|
||||
t.Errorf("structural import edge = %v, want %v", got, ProvenanceWeightMax)
|
||||
}
|
||||
if got := ProvenanceWeight(&Edge{Kind: EdgeCalls}); got != ProvenanceWeightMin {
|
||||
t.Errorf("bare unresolved call edge = %v, want %v", got, ProvenanceWeightMin)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,83 @@
|
||||
package graph
|
||||
|
||||
// Reader is the read-only contract every graph consumer (query
|
||||
// engine, MCP tool handlers, analyzers, resolver introspection) depends
|
||||
// on. *Graph satisfies it directly; OverlaidView (overlay.go) wraps a
|
||||
// base Reader plus a per-session overlay layer to deliver a non-
|
||||
// mutating shadow view for editor-buffer queries.
|
||||
//
|
||||
// Mutation methods (AddNode, AddEdge, EvictFile, ReindexEdge, …) live
|
||||
// on *Graph and are NOT part of this interface. Only the indexer and
|
||||
// the resolver mutate; everyone else reads, and reads must go through
|
||||
// Reader so the same call site transparently switches between base
|
||||
// and overlay views.
|
||||
//
|
||||
// New read methods on *Graph should be added here too — keeping the
|
||||
// surfaces in sync is what guarantees that a tool migrated to read
|
||||
// through the Reader will keep working for both base and overlay
|
||||
// queries.
|
||||
type Reader interface {
|
||||
// Identity lookups.
|
||||
GetNode(id string) *Node
|
||||
GetNodeByQualName(qualName string) *Node
|
||||
FindNodesByName(name string) []*Node
|
||||
// FindNodesByNameContaining returns nodes whose Name (case-
|
||||
// insensitive) contains substr. The filter is pushed into the
|
||||
// backend so only matching rows cross the boundary on a disk backend;
|
||||
// the search hot path's substring fallback uses this instead of
|
||||
// the old AllNodes()-then-filter pattern (which materialised the
|
||||
// whole node set per call and didn't scale). limit caps the
|
||||
// result; 0 means "no limit".
|
||||
FindNodesByNameContaining(substr string, limit int) []*Node
|
||||
|
||||
// GetNodesByIDs is the batched sibling of GetNode. The disk-backed
|
||||
// store collapses N individual point lookups into a
|
||||
// single bulk query — critical on the search hot path where one
|
||||
// query materialises 60+ candidate IDs. The in-memory backend
|
||||
// forwards to per-id GetNode, so the cost matches an inline loop
|
||||
// there. Missing IDs are simply absent from the map (no nil
|
||||
// values); duplicates dedupe naturally.
|
||||
GetNodesByIDs(ids []string) map[string]*Node
|
||||
|
||||
// File / repo scopes.
|
||||
GetFileNodes(filePath string) []*Node
|
||||
GetRepoNodes(repoPrefix string) []*Node
|
||||
|
||||
// Edge walks.
|
||||
GetOutEdges(nodeID string) []*Edge
|
||||
GetInEdges(nodeID string) []*Edge
|
||||
|
||||
// GetInEdgesByNodeIDs / GetOutEdgesByNodeIDs are the batched
|
||||
// siblings of GetInEdges / GetOutEdges. The disk-backed store collapses
|
||||
// N per-id queries into one bulk query over an `id IN $ids`
|
||||
// filter; the in-memory backend forwards to per-id walks (no
|
||||
// concurrency win — same algorithmic cost as an inline loop). On
|
||||
// the rerank hot path this drops ~150 round-trips per
|
||||
// search_symbols call down to ~4 (prepare collects every
|
||||
// candidate's ids and fans them out in one inbound + one outbound
|
||||
// batch). Missing nodes get nil slices in the returned map so
|
||||
// callers can `for _, e := range m[id]` without an ok-check.
|
||||
GetInEdgesByNodeIDs(ids []string) map[string][]*Edge
|
||||
GetOutEdgesByNodeIDs(ids []string) map[string][]*Edge
|
||||
|
||||
// Bulk reads — used by analyzers (hotspots, cycles, dead code,
|
||||
// communities, …) and by the embedded query engine's whole-graph
|
||||
// passes.
|
||||
AllNodes() []*Node
|
||||
AllEdges() []*Edge
|
||||
|
||||
// Counters & stats.
|
||||
NodeCount() int
|
||||
EdgeCount() int
|
||||
// EdgeIdentityRevisions is the running count of provenance-bearing
|
||||
// edge-identity changes — see Graph.EdgeIdentityRevisions.
|
||||
EdgeIdentityRevisions() int
|
||||
Stats() GraphStats
|
||||
RepoStats() map[string]GraphStats
|
||||
}
|
||||
|
||||
// Compile-time assertion that *Graph satisfies Reader. If a new
|
||||
// Reader method is added without a corresponding *Graph method (or
|
||||
// the *Graph signature drifts), the build breaks here rather than at
|
||||
// a far-away callsite.
|
||||
var _ Reader = (*Graph)(nil)
|
||||
@@ -0,0 +1,41 @@
|
||||
package graph
|
||||
|
||||
import "testing"
|
||||
|
||||
func TestRefContextOf(t *testing.T) {
|
||||
cases := []struct {
|
||||
name string
|
||||
kind EdgeKind
|
||||
from NodeKind
|
||||
meta map[string]any
|
||||
want string
|
||||
}{
|
||||
{"return", EdgeReturns, KindFunction, nil, RefContextReturnType},
|
||||
{"param", EdgeTypedAs, KindParam, nil, RefContextParameterType},
|
||||
{"field", EdgeTypedAs, KindField, nil, RefContextField},
|
||||
{"value_var", EdgeTypedAs, KindVariable, nil, RefContextValue},
|
||||
{"value_local", EdgeTypedAs, KindLocal, nil, RefContextValue},
|
||||
{"typed_other", EdgeTypedAs, KindType, nil, RefContextType},
|
||||
{"attribute", EdgeAnnotated, KindFunction, nil, RefContextAttribute},
|
||||
{"type_ref", EdgeReferences, KindFunction, nil, RefContextType},
|
||||
{"implements", EdgeImplements, KindType, nil, RefContextType},
|
||||
{"read", EdgeReads, KindFunction, nil, RefContextValue},
|
||||
{"call", EdgeCalls, KindFunction, nil, RefContextCall},
|
||||
{"instantiate", EdgeInstantiates, KindFunction, nil, RefContextCall},
|
||||
{"meta_override", EdgeReferences, KindFunction, map[string]any{"ref_context": RefContextGenericArg}, RefContextGenericArg},
|
||||
{"import", EdgeImports, KindFile, nil, RefContextImport},
|
||||
{"re_export", EdgeReExports, KindFile, nil, RefContextImport},
|
||||
{"no_context", EdgeContains, KindFile, nil, ""},
|
||||
}
|
||||
for _, c := range cases {
|
||||
t.Run(c.name, func(t *testing.T) {
|
||||
e := &Edge{Kind: c.kind, Meta: c.meta}
|
||||
if got := RefContextOf(e, c.from); got != c.want {
|
||||
t.Errorf("RefContextOf(%s, from=%s) = %q, want %q", c.kind, c.from, got, c.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
if RefContextOf(nil, KindFunction) != "" {
|
||||
t.Error("nil edge must classify as empty context")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,21 @@
|
||||
package graph
|
||||
|
||||
import "strings"
|
||||
|
||||
// RepoPrefixOfID returns the repository prefix encoded in a node ID or
|
||||
// file path — the leading path segment before the first "/". In
|
||||
// multi-repo mode every node ID and file path is prefixed with the repo
|
||||
// it belongs to (e.g. "gortex/internal/mcp/server.go::NewServer" →
|
||||
// "gortex"), so this recovers the owning repo without a node lookup.
|
||||
//
|
||||
// Returns "" for IDs that carry no repo prefix: unresolved sentinels
|
||||
// like "unresolved::Name", and single-repo-mode IDs that were never
|
||||
// prefixed. Callers use it only inside a workspace-bound (multi-repo)
|
||||
// path, where every real node is prefixed, so an empty result simply
|
||||
// never matches a workspace's repo set.
|
||||
func RepoPrefixOfID(id string) string {
|
||||
if i := strings.IndexByte(id, '/'); i > 0 {
|
||||
return id[:i]
|
||||
}
|
||||
return ""
|
||||
}
|
||||
@@ -0,0 +1,81 @@
|
||||
package graph
|
||||
|
||||
// Restub provenance handoff.
|
||||
//
|
||||
// When a file is re-parsed, its symbols are evicted and every incoming
|
||||
// reference edge is restubbed to an `unresolved::<name>` target so the
|
||||
// incoming-resolve pass can rebind it once the symbols come back. The edge
|
||||
// keeps its identity, but its resolved provenance (origin / tier / confidence)
|
||||
// is now a lie: it points at an unresolved stub. Two failure modes follow if
|
||||
// the provenance is left untouched — a genuinely deleted definition leaves a
|
||||
// stub advertising a compiler-grade tier, and a rebind to a *different* target
|
||||
// inherits the old tier it never earned.
|
||||
//
|
||||
// StashRestubProvenance clears the live provenance and records it in the edge's
|
||||
// Meta; RestoreRestubProvenance re-applies it only when the stub rebinds to the
|
||||
// exact same target (an idempotent re-parse: unchanged binding, unchanged
|
||||
// tier), and always drops the transient keys. The restub pass (indexer) and the
|
||||
// incoming-resolve pass (resolver) live in different packages, so this handoff
|
||||
// rides the edge Meta between them.
|
||||
const (
|
||||
metaRestubPrevTo = "restub_prev_to"
|
||||
metaRestubPrevOrigin = "restub_prev_origin"
|
||||
metaRestubPrevTier = "restub_prev_tier"
|
||||
metaRestubPrevConf = "restub_prev_conf"
|
||||
)
|
||||
|
||||
// StashRestubProvenance records e's current target + resolved provenance in its
|
||||
// Meta and clears the live provenance, so a restubbed edge no longer advertises
|
||||
// the resolved tier it held while bound. Call it just before rewriting e.To to
|
||||
// the unresolved stub. No-op on a nil edge.
|
||||
func StashRestubProvenance(e *Edge) {
|
||||
if e == nil {
|
||||
return
|
||||
}
|
||||
if e.Meta == nil {
|
||||
e.Meta = map[string]any{}
|
||||
}
|
||||
e.Meta[metaRestubPrevTo] = e.To
|
||||
e.Meta[metaRestubPrevOrigin] = e.Origin
|
||||
e.Meta[metaRestubPrevTier] = e.Tier
|
||||
e.Meta[metaRestubPrevConf] = e.Confidence
|
||||
e.Origin, e.Tier, e.Confidence = "", "", 0
|
||||
}
|
||||
|
||||
// RestoreRestubProvenance re-applies stashed provenance to e when e is now
|
||||
// resolved to the SAME target it had before the restub — the binding is
|
||||
// unchanged, so its verified tier is too — and always drops the transient stash
|
||||
// keys (whether or not it restored). A rebind to a different target, or a stub
|
||||
// that never rebound, keeps the honest fresh tier the resolver assigned (or
|
||||
// none). No-op when there is nothing stashed.
|
||||
//
|
||||
// Returns true when it actually re-applied the stashed provenance — the caller
|
||||
// uses that to persist the in-place mutation, since a disk backend hands out
|
||||
// freshly-decoded edge pointers whose changes are lost unless written back.
|
||||
func RestoreRestubProvenance(e *Edge) bool {
|
||||
if e == nil || e.Meta == nil {
|
||||
return false
|
||||
}
|
||||
prevTo, ok := e.Meta[metaRestubPrevTo]
|
||||
if !ok {
|
||||
return false
|
||||
}
|
||||
restored := false
|
||||
if to, isStr := prevTo.(string); isStr && to == e.To && !IsUnresolvedTarget(e.To) {
|
||||
if o, ok := e.Meta[metaRestubPrevOrigin].(string); ok {
|
||||
e.Origin = o
|
||||
}
|
||||
if tr, ok := e.Meta[metaRestubPrevTier].(string); ok {
|
||||
e.Tier = tr
|
||||
}
|
||||
if c, ok := e.Meta[metaRestubPrevConf].(float64); ok {
|
||||
e.Confidence = c
|
||||
}
|
||||
restored = true
|
||||
}
|
||||
delete(e.Meta, metaRestubPrevTo)
|
||||
delete(e.Meta, metaRestubPrevOrigin)
|
||||
delete(e.Meta, metaRestubPrevTier)
|
||||
delete(e.Meta, metaRestubPrevConf)
|
||||
return restored
|
||||
}
|
||||
@@ -0,0 +1,25 @@
|
||||
package graph
|
||||
|
||||
import "testing"
|
||||
|
||||
// TestStats_ExcludesProxyNodes asserts cross-daemon federation proxy nodes
|
||||
// are never counted in Graph.Stats.
|
||||
func TestStats_ExcludesProxyNodes(t *testing.T) {
|
||||
g := New()
|
||||
g.AddNode(&Node{ID: "local/a.go::Foo", Kind: KindFunction, Name: "Foo", Language: "go"})
|
||||
g.AddNode(&Node{
|
||||
ID: ProxyNodeID("remoteB", "b/x.go::Bar"),
|
||||
Kind: KindFunction,
|
||||
Name: "Bar",
|
||||
Origin: "remote:remoteB",
|
||||
Stub: true,
|
||||
})
|
||||
|
||||
st := g.Stats()
|
||||
if st.TotalNodes != 1 {
|
||||
t.Errorf("Stats.TotalNodes = %d, want 1 (proxy node excluded)", st.TotalNodes)
|
||||
}
|
||||
if st.ByKind["function"] != 1 {
|
||||
t.Errorf("ByKind[function] = %d, want 1 (proxy node excluded)", st.ByKind["function"])
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,183 @@
|
||||
package graph
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"sync"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func isPowerOfTwo(n int) bool { return n > 0 && n&(n-1) == 0 }
|
||||
|
||||
// TestNextPow2 pins the rounding helper that guarantees the power-of-two
|
||||
// invariant the shardMask trick depends on.
|
||||
func TestNextPow2(t *testing.T) {
|
||||
cases := map[int]int{
|
||||
-3: 1, 0: 1, 1: 1, 2: 2, 3: 4, 4: 4,
|
||||
5: 8, 9: 16, 15: 16, 16: 16, 17: 32, 255: 256, 256: 256, 257: 512,
|
||||
}
|
||||
for in, want := range cases {
|
||||
if got := nextPow2(in); got != want {
|
||||
t.Errorf("nextPow2(%d) = %d, want %d", in, got, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestCoerceShardCountBounds asserts every coerced count is a power of
|
||||
// two within [minShards, maxShards] regardless of the input.
|
||||
func TestCoerceShardCountBounds(t *testing.T) {
|
||||
for _, in := range []int{-100, 0, 1, 3, 4, 7, 16, 100, 256, 511, 512, 1000, 1 << 20} {
|
||||
got := coerceShardCount(in)
|
||||
if !isPowerOfTwo(got) {
|
||||
t.Errorf("coerceShardCount(%d) = %d, not a power of two", in, got)
|
||||
}
|
||||
if got < minShards || got > maxShards {
|
||||
t.Errorf("coerceShardCount(%d) = %d, outside [%d, %d]", in, got, minShards, maxShards)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestDefaultShardCountDerived asserts the CPU-derived default (no
|
||||
// override) is always a power of two in the derived range.
|
||||
func TestDefaultShardCountDerived(t *testing.T) {
|
||||
t.Setenv(shardCountEnv, "") // ensure no override is in effect
|
||||
got := defaultShardCount()
|
||||
if !isPowerOfTwo(got) {
|
||||
t.Fatalf("defaultShardCount() = %d, not a power of two", got)
|
||||
}
|
||||
if got < derivedShardFloor || got > derivedShardCeiling {
|
||||
t.Fatalf("defaultShardCount() = %d, outside derived range [%d, %d]", got, derivedShardFloor, derivedShardCeiling)
|
||||
}
|
||||
}
|
||||
|
||||
// TestGraphShardFieldsConsistent asserts the constructed graph's shard
|
||||
// bookkeeping is internally consistent: the slice length equals
|
||||
// shardCount, shardCount is a power of two, and shardMask == count-1.
|
||||
func TestGraphShardFieldsConsistent(t *testing.T) {
|
||||
for _, req := range []int{1, 2, 4, 7, 16, 64, 300, 1000} {
|
||||
g := newWithShardCount(req)
|
||||
if !isPowerOfTwo(g.shardCount) {
|
||||
t.Errorf("newWithShardCount(%d): shardCount = %d, not a power of two", req, g.shardCount)
|
||||
}
|
||||
if g.shardCount < minShards || g.shardCount > maxShards {
|
||||
t.Errorf("newWithShardCount(%d): shardCount = %d, outside [%d, %d]", req, g.shardCount, minShards, maxShards)
|
||||
}
|
||||
if g.shardMask != uint32(g.shardCount-1) {
|
||||
t.Errorf("newWithShardCount(%d): shardMask = %d, want %d", req, g.shardMask, g.shardCount-1)
|
||||
}
|
||||
if len(g.shards) != g.shardCount {
|
||||
t.Errorf("newWithShardCount(%d): len(shards) = %d, want %d", req, len(g.shards), g.shardCount)
|
||||
}
|
||||
for i, s := range g.shards {
|
||||
if s == nil || s.nodes == nil {
|
||||
t.Errorf("newWithShardCount(%d): shard %d not initialized", req, i)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestShardCountEnvOverride asserts GORTEX_GRAPH_SHARDS is honored,
|
||||
// coerced to a power of two within bounds, and reflected in the graph
|
||||
// built by New().
|
||||
func TestShardCountEnvOverride(t *testing.T) {
|
||||
cases := []struct {
|
||||
env string
|
||||
want int
|
||||
}{
|
||||
{"4", 4}, // exact power of two below the derived floor — still honored
|
||||
{"5", 8}, // not a power of two — coerced up
|
||||
{"64", 64}, // exact
|
||||
{"300", 512}, // not a power of two — coerced up
|
||||
{"1000", 512}, // above maxShards — clamped
|
||||
{"1", 1}, // minimum
|
||||
}
|
||||
for _, c := range cases {
|
||||
t.Run(c.env, func(t *testing.T) {
|
||||
t.Setenv(shardCountEnv, c.env)
|
||||
if got := defaultShardCount(); got != c.want {
|
||||
t.Fatalf("defaultShardCount() with %s=%q = %d, want %d", shardCountEnv, c.env, got, c.want)
|
||||
}
|
||||
g := New()
|
||||
if g.shardCount != c.want {
|
||||
t.Fatalf("New().shardCount = %d, want %d", g.shardCount, c.want)
|
||||
}
|
||||
if g.shardMask != uint32(c.want-1) {
|
||||
t.Fatalf("New().shardMask = %d, want %d", g.shardMask, c.want-1)
|
||||
}
|
||||
if len(g.shards) != c.want {
|
||||
t.Fatalf("len(New().shards) = %d, want %d", len(g.shards), c.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestShardCountInvalidEnvFallsBack asserts a malformed or non-positive
|
||||
// override is ignored and the graph falls back to the derived default.
|
||||
func TestShardCountInvalidEnvFallsBack(t *testing.T) {
|
||||
for _, v := range []string{"abc", "0", "-4", " ", "3.5"} {
|
||||
t.Run(v, func(t *testing.T) {
|
||||
t.Setenv(shardCountEnv, v)
|
||||
got := defaultShardCount()
|
||||
if !isPowerOfTwo(got) || got < derivedShardFloor || got > derivedShardCeiling {
|
||||
t.Fatalf("override %q: defaultShardCount() = %d, want power-of-two in [%d, %d]", v, got, derivedShardFloor, derivedShardCeiling)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestShardedConcurrentInsertNoLoss inserts a large, disjoint set of
|
||||
// nodes and edges from many goroutines into graphs built with several
|
||||
// shard counts (including non-default 1, 4, 64) and asserts nothing is
|
||||
// lost or duplicated — proving every shard is correctly initialized and
|
||||
// indexed regardless of count.
|
||||
func TestShardedConcurrentInsertNoLoss(t *testing.T) {
|
||||
for _, count := range []int{1, 4, 16, 64} {
|
||||
t.Run(fmt.Sprintf("shards=%d", count), func(t *testing.T) {
|
||||
g := newWithShardCount(count)
|
||||
if g.shardCount != count {
|
||||
t.Fatalf("newWithShardCount(%d).shardCount = %d", count, g.shardCount)
|
||||
}
|
||||
|
||||
const workers = 32
|
||||
const perWorker = 500
|
||||
var wg sync.WaitGroup
|
||||
wg.Add(workers)
|
||||
for w := range workers {
|
||||
go func(w int) {
|
||||
defer wg.Done()
|
||||
for i := range perWorker {
|
||||
nid := fmt.Sprintf("w%d-n%d::N", w, i)
|
||||
g.AddNode(&Node{ID: nid, Name: "N", Kind: KindFunction, FilePath: "f"})
|
||||
// One out-edge per node to a shared sink, so the From
|
||||
// and To endpoints frequently land in different shards.
|
||||
g.AddEdge(&Edge{
|
||||
From: nid,
|
||||
To: fmt.Sprintf("sink-%d::S", i%17),
|
||||
Kind: EdgeCalls,
|
||||
FilePath: "f",
|
||||
Line: w,
|
||||
})
|
||||
}
|
||||
}(w)
|
||||
}
|
||||
wg.Wait()
|
||||
|
||||
wantNodes := workers * perWorker
|
||||
for w := range workers {
|
||||
for i := range perWorker {
|
||||
nid := fmt.Sprintf("w%d-n%d::N", w, i)
|
||||
if g.GetNode(nid) == nil {
|
||||
t.Fatalf("node %q missing after concurrent insert (shards=%d)", nid, count)
|
||||
}
|
||||
if outs := g.GetOutEdges(nid); len(outs) != 1 {
|
||||
t.Fatalf("node %q has %d out-edges, want 1 (shards=%d)", nid, len(outs), count)
|
||||
}
|
||||
}
|
||||
}
|
||||
// Sink targets are never AddNode'd, so NodeCount counts only the
|
||||
// inserted N nodes — exactly one per (worker, index) pair.
|
||||
if got := g.NodeCount(); got != wantNodes {
|
||||
t.Fatalf("NodeCount() = %d, want %d (shards=%d)", got, wantNodes, count)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,29 @@
|
||||
package graph
|
||||
|
||||
import "testing"
|
||||
|
||||
func TestOriginRank_Speculative(t *testing.T) {
|
||||
if OriginRank(OriginSpeculative) >= OriginRank(OriginTextMatched) {
|
||||
t.Errorf("speculative (%d) must rank below text_matched (%d)",
|
||||
OriginRank(OriginSpeculative), OriginRank(OriginTextMatched))
|
||||
}
|
||||
if OriginRank(OriginSpeculative) <= OriginRank("") {
|
||||
t.Errorf("speculative must rank above unknown/empty")
|
||||
}
|
||||
if MeetsMinTier(OriginSpeculative, OriginTextMatched) {
|
||||
t.Errorf("min_tier=text_matched must exclude speculative edges")
|
||||
}
|
||||
if !MeetsMinTier(OriginSpeculative, "") {
|
||||
t.Errorf("empty min_tier must pass speculative edges")
|
||||
}
|
||||
}
|
||||
|
||||
func TestEdge_IsSpeculative(t *testing.T) {
|
||||
if (&Edge{}).IsSpeculative() {
|
||||
t.Errorf("plain edge must not be speculative")
|
||||
}
|
||||
e := &Edge{Meta: map[string]any{MetaSpeculative: true}}
|
||||
if !e.IsSpeculative() {
|
||||
t.Errorf("edge with MetaSpeculative=true must be speculative")
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,218 @@
|
||||
package store_sqlite
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"errors"
|
||||
"fmt"
|
||||
|
||||
"github.com/zzet/gortex/internal/graph"
|
||||
)
|
||||
|
||||
// Compile-time assertion: *Store satisfies graph.BulkLoader.
|
||||
var _ graph.BulkLoader = (*Store)(nil)
|
||||
|
||||
// bulkDroppableIndex is one secondary index the bulk-load fast path drops
|
||||
// before a first/empty cold index and rebuilds afterward.
|
||||
type bulkDroppableIndex struct {
|
||||
name string
|
||||
ddl string
|
||||
}
|
||||
|
||||
// bulkDroppableIndexes is the single source of truth for these index
|
||||
// definitions. Open creates them (so the initial DB has them), BeginBulkLoad
|
||||
// drops them by name, and FlushBulk recreates them from the exact same ddl —
|
||||
// keeping the initial and post-bulk shapes from drifting.
|
||||
//
|
||||
// These are exactly the standalone, NON-UNIQUE CREATE INDEX statements over
|
||||
// the large nodes / edges tables. Maintaining them per-row across a
|
||||
// multi-hundred-thousand-row cold load is pure overhead when the rows land
|
||||
// once, so they are dropped up front and rebuilt in one pass at the end.
|
||||
//
|
||||
// Deliberately excluded:
|
||||
// - nodes_by_qual (UNIQUE): enforces qual_name dedup on every
|
||||
// INSERT OR REPLACE. Dropping it would change insert conflict semantics
|
||||
// (collapsed qual_name collisions would diverge from the non-bulk path)
|
||||
// and a duplicate could make the recreate fail. It stays live.
|
||||
// - the edges UNIQUE(from_id, …) table constraint and every WITHOUT ROWID
|
||||
// primary-key index: not standalone indexes; they cannot be dropped while
|
||||
// the table/constraint exists.
|
||||
// - edges_external (partial): a tiny index over external-call terminals,
|
||||
// created from a shared predicate in Open; not worth dropping.
|
||||
//
|
||||
// Dropping/recreating these is a runtime operation on identical DDL — it is
|
||||
// NOT a schema change, so it does not touch the persisted schema version.
|
||||
var bulkDroppableIndexes = []bulkDroppableIndex{
|
||||
{"nodes_by_name", `CREATE INDEX IF NOT EXISTS nodes_by_name ON nodes(name)`},
|
||||
{"nodes_by_kind", `CREATE INDEX IF NOT EXISTS nodes_by_kind ON nodes(kind)`},
|
||||
{"nodes_by_file", `CREATE INDEX IF NOT EXISTS nodes_by_file ON nodes(file_path)`},
|
||||
{"nodes_by_repo", `CREATE INDEX IF NOT EXISTS nodes_by_repo ON nodes(repo_prefix) WHERE repo_prefix <> ''`},
|
||||
{"edges_by_from", `CREATE INDEX IF NOT EXISTS edges_by_from ON edges(from_id, kind)`},
|
||||
{"edges_by_to", `CREATE INDEX IF NOT EXISTS edges_by_to ON edges(to_id, kind)`},
|
||||
{"edges_by_kind", `CREATE INDEX IF NOT EXISTS edges_by_kind ON edges(kind)`},
|
||||
// Backs EdgesWithUnresolvedTarget — the resolver's main pending-edge
|
||||
// collector, called on every full resolve. is_unresolved is a VIRTUAL
|
||||
// generated column (see isUnresolvedColumnDDL); indexing it turns a
|
||||
// full-table scan (the prior to_id-based OR query forced SQLite to
|
||||
// abandon its index) into an index search whose bookmark lookups land
|
||||
// in ascending rowid order (see isUnresolvedColumnDDL's doc comment for
|
||||
// why that beats an equivalent-looking to_id-based index).
|
||||
{"edges_by_unresolved", `CREATE INDEX IF NOT EXISTS edges_by_unresolved ON edges(is_unresolved)`},
|
||||
// Partial index over exactly the not-yet-semantically-stamped nodes per
|
||||
// repo. Stays small in steady state (most nodes end up stamped), so a
|
||||
// future "unstamped nodes in this repo" query is an index scan over the
|
||||
// residual few instead of a full-table decode of every node's meta.
|
||||
{"nodes_semantic_pending", `CREATE INDEX IF NOT EXISTS nodes_semantic_pending ON nodes(repo_prefix) WHERE semantic_type IS NULL`},
|
||||
}
|
||||
|
||||
// bulkCacheSizeKiB is the page cache the fast path requests on its pinned
|
||||
// connection. SQLite reads a negative cache_size as a KiB budget, so this is
|
||||
// ~256 MiB — large enough to keep the cold load's working set resident.
|
||||
const bulkCacheSizeKiB = -262144
|
||||
|
||||
// beginWrite starts a write transaction. During a bulk-load fast path it pins
|
||||
// the single connection that carries synchronous=OFF + the enlarged page
|
||||
// cache (database/sql PRAGMAs are connection-local, so a pooled connection
|
||||
// would not see them); otherwise it uses the shared pool. The caller holds
|
||||
// writeMu, which also guards s.bulkConn.
|
||||
func (s *Store) beginWrite() (*sql.Tx, error) {
|
||||
if s.bulkConn != nil {
|
||||
return s.bulkConn.BeginTx(context.Background(), nil)
|
||||
}
|
||||
return s.db.Begin()
|
||||
}
|
||||
|
||||
// BeginBulkLoad enters the bulk-load fast path for a first/empty cold index.
|
||||
// It pins one connection at synchronous=OFF with an enlarged page cache and
|
||||
// drops the droppable secondary indexes, so a multi-hundred-thousand-row load
|
||||
// skips per-row B-tree maintenance and per-commit fsync. FlushBulk reverses
|
||||
// all of it: restore the pragmas, rebuild the indexes, and checkpoint.
|
||||
//
|
||||
// Gated: it engages ONLY when the nodes table is empty. On a populated store
|
||||
// (incremental reindex, warm restart, or a later repo in a multi-repo cold
|
||||
// start that shares the disk store) it is a safe no-op — dropping indexes or
|
||||
// disabling crash durability under live, concurrently-readable rows would be
|
||||
// unsafe. In-memory stores have no WAL / on-disk B-tree pressure, so it is a
|
||||
// no-op there too.
|
||||
func (s *Store) BeginBulkLoad() {
|
||||
s.writeMu.Lock()
|
||||
defer s.writeMu.Unlock()
|
||||
|
||||
// Re-entrancy / non-disk guard: a second BeginBulkLoad without an
|
||||
// intervening FlushBulk, or an in-memory store, stays a no-op.
|
||||
if s.bulkConn != nil || isMemoryPath(s.dbPath) {
|
||||
return
|
||||
}
|
||||
|
||||
ctx := context.Background()
|
||||
conn, err := s.db.Conn(ctx)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
// Gate to a genuinely first/empty index.
|
||||
if !nodesTableEmpty(ctx, conn) {
|
||||
_ = conn.Close()
|
||||
return
|
||||
}
|
||||
|
||||
// Capture prior pragma values so FlushBulk (and every early-return /
|
||||
// error path) can restore them. If they can't be read, don't engage —
|
||||
// a slow correct load beats a connection stuck at synchronous=OFF.
|
||||
prevSync, err := pragmaInt(ctx, conn, "synchronous")
|
||||
if err != nil {
|
||||
_ = conn.Close()
|
||||
return
|
||||
}
|
||||
prevCache, err := pragmaInt(ctx, conn, "cache_size")
|
||||
if err != nil {
|
||||
_ = conn.Close()
|
||||
return
|
||||
}
|
||||
|
||||
// synchronous=OFF drops crash durability for the load window —
|
||||
// acceptable only because a crash on a fresh index just re-indexes.
|
||||
if _, err := conn.ExecContext(ctx, "PRAGMA synchronous = OFF"); err != nil {
|
||||
_ = conn.Close()
|
||||
return
|
||||
}
|
||||
if _, err := conn.ExecContext(ctx, fmt.Sprintf("PRAGMA cache_size = %d", bulkCacheSizeKiB)); err != nil {
|
||||
// Roll the durability change back before bailing.
|
||||
_, _ = conn.ExecContext(ctx, fmt.Sprintf("PRAGMA synchronous = %d", prevSync))
|
||||
_ = conn.Close()
|
||||
return
|
||||
}
|
||||
|
||||
// Drop the droppable secondary indexes; rebuilt in one pass at
|
||||
// FlushBulk. Best-effort: a failed drop just means that index keeps
|
||||
// being maintained per-row (slower, still correct), so it is not fatal.
|
||||
for _, idx := range bulkDroppableIndexes {
|
||||
_, _ = conn.ExecContext(ctx, "DROP INDEX IF EXISTS "+idx.name)
|
||||
}
|
||||
|
||||
s.bulkConn = conn
|
||||
s.bulkPrevSync = prevSync
|
||||
s.bulkPrevCacheSize = prevCache
|
||||
}
|
||||
|
||||
// FlushBulk exits the bulk-load fast path: it rebuilds every index
|
||||
// BeginBulkLoad dropped, restores synchronous + cache_size on the pinned
|
||||
// connection, runs one TRUNCATE checkpoint to drain the WAL the no-fsync load
|
||||
// grew, and returns the connection to the pool. It is a no-op when no fast
|
||||
// path is active (BeginBulkLoad gated out, or already flushed).
|
||||
//
|
||||
// The pragma restore + connection release run unconditionally (defer), so a
|
||||
// failure mid-rebuild can never leave a connection stuck at synchronous=OFF
|
||||
// in the pool.
|
||||
func (s *Store) FlushBulk() error {
|
||||
s.writeMu.Lock()
|
||||
defer s.writeMu.Unlock()
|
||||
|
||||
conn := s.bulkConn
|
||||
if conn == nil {
|
||||
return nil
|
||||
}
|
||||
// Detach first: the fast path is over regardless of the outcome below.
|
||||
s.bulkConn = nil
|
||||
|
||||
ctx := context.Background()
|
||||
defer func() {
|
||||
// Always restore durability + cache and release the connection,
|
||||
// even if an index rebuild failed.
|
||||
_, _ = conn.ExecContext(ctx, fmt.Sprintf("PRAGMA synchronous = %d", s.bulkPrevSync))
|
||||
_, _ = conn.ExecContext(ctx, fmt.Sprintf("PRAGMA cache_size = %d", s.bulkPrevCacheSize))
|
||||
_ = conn.Close()
|
||||
}()
|
||||
|
||||
for _, idx := range bulkDroppableIndexes {
|
||||
if _, err := conn.ExecContext(ctx, idx.ddl); err != nil {
|
||||
return fmt.Errorf("store_sqlite: rebuild index %s: %w", idx.name, err)
|
||||
}
|
||||
}
|
||||
|
||||
// Drain the WAL the no-fsync bulk window grew back into the main DB and
|
||||
// truncate the -wal file. Same TRUNCATE mode as runCheckpointLoop, so it
|
||||
// cooperates with the journal_size_limit / periodic-checkpoint policy.
|
||||
if _, err := conn.ExecContext(ctx, "PRAGMA wal_checkpoint(TRUNCATE)"); err != nil {
|
||||
return fmt.Errorf("store_sqlite: bulk checkpoint: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// nodesTableEmpty reports whether the nodes table holds no rows. Used to gate
|
||||
// the bulk-load fast path to a genuinely first/empty cold index.
|
||||
func nodesTableEmpty(ctx context.Context, conn *sql.Conn) bool {
|
||||
var one int
|
||||
err := conn.QueryRowContext(ctx, "SELECT 1 FROM nodes LIMIT 1").Scan(&one)
|
||||
return errors.Is(err, sql.ErrNoRows)
|
||||
}
|
||||
|
||||
// pragmaInt reads a single-integer PRAGMA (synchronous, cache_size) off the
|
||||
// given connection.
|
||||
func pragmaInt(ctx context.Context, conn *sql.Conn, pragma string) (int64, error) {
|
||||
var v int64
|
||||
if err := conn.QueryRowContext(ctx, "PRAGMA "+pragma).Scan(&v); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return v, nil
|
||||
}
|
||||
@@ -0,0 +1,388 @@
|
||||
package store_sqlite
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/zzet/gortex/internal/graph"
|
||||
)
|
||||
|
||||
// bulkFixture builds a deterministic node/edge set with distinct ids and
|
||||
// qual_names (so the UNIQUE nodes_by_qual index never collides) and a mix of
|
||||
// edge keys (a handful collide → exercise dedup). Input order is intentionally
|
||||
// not key-sorted.
|
||||
func bulkFixture(nNodes, nEdges int) ([]*graph.Node, []*graph.Edge) {
|
||||
nodes := make([]*graph.Node, 0, nNodes)
|
||||
for i := range nNodes {
|
||||
nodes = append(nodes, &graph.Node{
|
||||
ID: fmt.Sprintf("pkg/f%d.go::Sym%d", i%64, i),
|
||||
Kind: graph.KindFunction,
|
||||
Name: fmt.Sprintf("Sym%d", i),
|
||||
QualName: fmt.Sprintf("pkg.f%d.Sym%d", i%64, i),
|
||||
FilePath: fmt.Sprintf("pkg/f%d.go", i%64),
|
||||
RepoPrefix: "gortex",
|
||||
Language: "go",
|
||||
})
|
||||
}
|
||||
edges := make([]*graph.Edge, 0, nEdges)
|
||||
for i := range nEdges {
|
||||
from := nodes[i%nNodes]
|
||||
to := nodes[(i*7+1)%nNodes]
|
||||
edges = append(edges, &graph.Edge{
|
||||
From: from.ID,
|
||||
To: to.ID,
|
||||
Kind: graph.EdgeCalls,
|
||||
FilePath: from.FilePath,
|
||||
Line: i % 500,
|
||||
Confidence: 1,
|
||||
})
|
||||
}
|
||||
return nodes, edges
|
||||
}
|
||||
|
||||
func openTempStore(t *testing.T) (*Store, string) {
|
||||
t.Helper()
|
||||
path := filepath.Join(t.TempDir(), "bulk.sqlite")
|
||||
s, err := Open(path)
|
||||
if err != nil {
|
||||
t.Fatalf("open: %v", err)
|
||||
}
|
||||
t.Cleanup(func() { _ = s.Close() })
|
||||
return s, path
|
||||
}
|
||||
|
||||
// indexNames returns the set of secondary index names present in the schema.
|
||||
func indexNames(t *testing.T, q interface {
|
||||
Query(string, ...any) (*sql.Rows, error)
|
||||
}) map[string]bool {
|
||||
t.Helper()
|
||||
rows, err := q.Query("SELECT name FROM sqlite_master WHERE type='index'")
|
||||
if err != nil {
|
||||
t.Fatalf("query sqlite_master: %v", err)
|
||||
}
|
||||
defer func() { _ = rows.Close() }()
|
||||
got := map[string]bool{}
|
||||
for rows.Next() {
|
||||
var n string
|
||||
if err := rows.Scan(&n); err != nil {
|
||||
t.Fatalf("scan index name: %v", err)
|
||||
}
|
||||
got[n] = true
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
t.Fatalf("rows: %v", err)
|
||||
}
|
||||
return got
|
||||
}
|
||||
|
||||
func pragmaIntDB(t *testing.T, db *sql.DB, pragma string) int64 {
|
||||
t.Helper()
|
||||
var v int64
|
||||
if err := db.QueryRow("PRAGMA " + pragma).Scan(&v); err != nil {
|
||||
t.Fatalf("pragma %s: %v", pragma, err)
|
||||
}
|
||||
return v
|
||||
}
|
||||
|
||||
func integrityOK(t *testing.T, db *sql.DB) {
|
||||
t.Helper()
|
||||
var res string
|
||||
if err := db.QueryRow("PRAGMA integrity_check").Scan(&res); err != nil {
|
||||
t.Fatalf("integrity_check: %v", err)
|
||||
}
|
||||
if res != "ok" {
|
||||
t.Fatalf("integrity_check = %q, want \"ok\"", res)
|
||||
}
|
||||
}
|
||||
|
||||
// TestBulkLoadDropsAndRebuildsIndexes is the core mechanism + restore proof:
|
||||
// the fast path engages on an empty store, drops the droppable indexes,
|
||||
// pins a synchronous=OFF connection, then on FlushBulk rebuilds every index,
|
||||
// restores synchronous, releases the connection, and leaves the DB intact.
|
||||
func TestBulkLoadDropsAndRebuildsIndexes(t *testing.T) {
|
||||
s, _ := openTempStore(t)
|
||||
ctx := context.Background()
|
||||
|
||||
// Baseline: all droppable indexes present, synchronous=NORMAL(1).
|
||||
before := indexNames(t, s.db)
|
||||
for _, idx := range bulkDroppableIndexes {
|
||||
if !before[idx.name] {
|
||||
t.Fatalf("index %s missing before bulk load", idx.name)
|
||||
}
|
||||
}
|
||||
if got := pragmaIntDB(t, s.db, "synchronous"); got != 1 {
|
||||
t.Fatalf("synchronous before = %d, want 1 (NORMAL)", got)
|
||||
}
|
||||
|
||||
// Engage the fast path on the empty store.
|
||||
s.BeginBulkLoad()
|
||||
if s.bulkConn == nil {
|
||||
t.Fatal("fast path did not engage on empty store")
|
||||
}
|
||||
// Read through the pinned connection (it may be the only one when
|
||||
// GOMAXPROCS/NumCPU is 1) to avoid blocking on connection acquisition.
|
||||
var sync int64
|
||||
if err := s.bulkConn.QueryRowContext(ctx, "PRAGMA synchronous").Scan(&sync); err != nil {
|
||||
t.Fatalf("pinned synchronous: %v", err)
|
||||
}
|
||||
if sync != 0 {
|
||||
t.Fatalf("bulk synchronous = %d, want 0 (OFF)", sync)
|
||||
}
|
||||
var cache int64
|
||||
if err := s.bulkConn.QueryRowContext(ctx, "PRAGMA cache_size").Scan(&cache); err != nil {
|
||||
t.Fatalf("pinned cache_size: %v", err)
|
||||
}
|
||||
if cache != bulkCacheSizeKiB {
|
||||
t.Fatalf("bulk cache_size = %d, want %d", cache, bulkCacheSizeKiB)
|
||||
}
|
||||
// The droppable indexes are gone during the window.
|
||||
during := indexNames(t, &connQuerier{ctx: ctx, c: s.bulkConn})
|
||||
for _, idx := range bulkDroppableIndexes {
|
||||
if during[idx.name] {
|
||||
t.Fatalf("index %s still present during bulk window", idx.name)
|
||||
}
|
||||
}
|
||||
// nodes_by_qual (UNIQUE, not droppable) must remain live.
|
||||
if !during["nodes_by_qual"] {
|
||||
t.Fatal("nodes_by_qual (UNIQUE) must not be dropped")
|
||||
}
|
||||
|
||||
nodes, edges := bulkFixture(2000, 4000)
|
||||
s.AddBatch(nodes, edges)
|
||||
|
||||
if err := s.FlushBulk(); err != nil {
|
||||
t.Fatalf("FlushBulk: %v", err)
|
||||
}
|
||||
if s.bulkConn != nil {
|
||||
t.Fatal("bulkConn not released after FlushBulk")
|
||||
}
|
||||
|
||||
// Every dropped index is back.
|
||||
after := indexNames(t, s.db)
|
||||
for _, idx := range bulkDroppableIndexes {
|
||||
if !after[idx.name] {
|
||||
t.Fatalf("index %s not rebuilt after FlushBulk", idx.name)
|
||||
}
|
||||
}
|
||||
// synchronous restored to NORMAL on the pool.
|
||||
if got := pragmaIntDB(t, s.db, "synchronous"); got != 1 {
|
||||
t.Fatalf("synchronous after = %d, want 1 (NORMAL)", got)
|
||||
}
|
||||
integrityOK(t, s.db)
|
||||
}
|
||||
|
||||
// connQuerier adapts *sql.Conn to the Query signature indexNames expects.
|
||||
type connQuerier struct {
|
||||
ctx context.Context
|
||||
c *sql.Conn
|
||||
}
|
||||
|
||||
func (q *connQuerier) Query(query string, args ...any) (*sql.Rows, error) {
|
||||
return q.c.QueryContext(q.ctx, query, args...)
|
||||
}
|
||||
|
||||
// TestBulkLoadMatchesNonBulkCounts proves the fast path persists exactly the
|
||||
// same node/edge counts the plain AddBatch path does.
|
||||
func TestBulkLoadMatchesNonBulkCounts(t *testing.T) {
|
||||
nodes, edges := bulkFixture(3000, 6000)
|
||||
|
||||
plain, _ := openTempStore(t)
|
||||
plain.AddBatch(nodes, edges)
|
||||
wantNodes, wantEdges := plain.NodeCount(), plain.EdgeCount()
|
||||
|
||||
bulk, _ := openTempStore(t)
|
||||
bulk.BeginBulkLoad()
|
||||
if bulk.bulkConn == nil {
|
||||
t.Fatal("fast path did not engage on empty store")
|
||||
}
|
||||
// Drain in two chunks to mirror the indexer's chunked persist.
|
||||
bulk.AddBatch(nodes[:1500], nil)
|
||||
bulk.AddBatch(nodes[1500:], nil)
|
||||
bulk.AddBatch(nil, edges[:3000])
|
||||
bulk.AddBatch(nil, edges[3000:])
|
||||
if err := bulk.FlushBulk(); err != nil {
|
||||
t.Fatalf("FlushBulk: %v", err)
|
||||
}
|
||||
|
||||
if gotN, gotE := bulk.NodeCount(), bulk.EdgeCount(); gotN != wantNodes || gotE != wantEdges {
|
||||
t.Fatalf("bulk counts (%d nodes, %d edges) != non-bulk (%d, %d)", gotN, gotE, wantNodes, wantEdges)
|
||||
}
|
||||
integrityOK(t, bulk.db)
|
||||
}
|
||||
|
||||
// TestBulkLoadGatedToPopulatedStore confirms the fast path is a safe no-op on
|
||||
// a store that already holds rows — no indexes are dropped, durability stays.
|
||||
func TestBulkLoadGatedToPopulatedStore(t *testing.T) {
|
||||
s, _ := openTempStore(t)
|
||||
// Populate first (the normal, non-bulk path).
|
||||
nodes, edges := bulkFixture(50, 100)
|
||||
s.AddBatch(nodes, edges)
|
||||
|
||||
s.BeginBulkLoad()
|
||||
if s.bulkConn != nil {
|
||||
t.Fatal("fast path engaged on a populated store; must be a no-op")
|
||||
}
|
||||
// Indexes untouched, durability untouched.
|
||||
present := indexNames(t, s.db)
|
||||
for _, idx := range bulkDroppableIndexes {
|
||||
if !present[idx.name] {
|
||||
t.Fatalf("index %s dropped on a populated store", idx.name)
|
||||
}
|
||||
}
|
||||
if got := pragmaIntDB(t, s.db, "synchronous"); got != 1 {
|
||||
t.Fatalf("synchronous = %d on populated store, want 1 (NORMAL)", got)
|
||||
}
|
||||
if err := s.FlushBulk(); err != nil {
|
||||
t.Fatalf("FlushBulk no-op returned error: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// TestBulkLoadInMemoryIsNoOp confirms in-memory stores never engage the fast
|
||||
// path (no WAL / on-disk B-tree to optimise).
|
||||
func TestBulkLoadInMemoryIsNoOp(t *testing.T) {
|
||||
s, err := Open(":memory:")
|
||||
if err != nil {
|
||||
t.Fatalf("open: %v", err)
|
||||
}
|
||||
t.Cleanup(func() { _ = s.Close() })
|
||||
s.BeginBulkLoad()
|
||||
if s.bulkConn != nil {
|
||||
t.Fatal("fast path engaged on an in-memory store")
|
||||
}
|
||||
if err := s.FlushBulk(); err != nil {
|
||||
t.Fatalf("FlushBulk: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// TestBulkLoadWarmRestartLoadsClean bulk-loads, closes, reopens the same file,
|
||||
// and asserts the persisted graph round-trips: identical counts, indexes
|
||||
// present, integrity ok.
|
||||
func TestBulkLoadWarmRestartLoadsClean(t *testing.T) {
|
||||
path := filepath.Join(t.TempDir(), "warm.sqlite")
|
||||
nodes, edges := bulkFixture(2500, 5000)
|
||||
|
||||
s, err := Open(path)
|
||||
if err != nil {
|
||||
t.Fatalf("open: %v", err)
|
||||
}
|
||||
s.BeginBulkLoad()
|
||||
if s.bulkConn == nil {
|
||||
t.Fatal("fast path did not engage on empty store")
|
||||
}
|
||||
s.AddBatch(nodes, edges)
|
||||
if err := s.FlushBulk(); err != nil {
|
||||
t.Fatalf("FlushBulk: %v", err)
|
||||
}
|
||||
wantNodes, wantEdges := s.NodeCount(), s.EdgeCount()
|
||||
if err := s.Close(); err != nil {
|
||||
t.Fatalf("close: %v", err)
|
||||
}
|
||||
|
||||
reopened, err := Open(path)
|
||||
if err != nil {
|
||||
t.Fatalf("reopen: %v", err)
|
||||
}
|
||||
t.Cleanup(func() { _ = reopened.Close() })
|
||||
|
||||
if gotN, gotE := reopened.NodeCount(), reopened.EdgeCount(); gotN != wantNodes || gotE != wantEdges {
|
||||
t.Fatalf("warm restart counts (%d, %d) != pre-close (%d, %d)", gotN, gotE, wantNodes, wantEdges)
|
||||
}
|
||||
present := indexNames(t, reopened.db)
|
||||
for _, idx := range bulkDroppableIndexes {
|
||||
if !present[idx.name] {
|
||||
t.Fatalf("index %s missing after warm restart", idx.name)
|
||||
}
|
||||
}
|
||||
integrityOK(t, reopened.db)
|
||||
// A populated store on reopen must NOT engage the fast path.
|
||||
reopened.BeginBulkLoad()
|
||||
if reopened.bulkConn != nil {
|
||||
t.Fatal("fast path engaged on warm restart (populated store)")
|
||||
}
|
||||
_ = reopened.FlushBulk()
|
||||
}
|
||||
|
||||
// TestBulkLoadPersistSpeed is the persist-speed evidence: it times the plain
|
||||
// path vs the fast path on the same fixture and logs both. It asserts
|
||||
// correctness and that the fast path is not pathologically slower; a strict
|
||||
// speedup ratio is gated behind GORTEX_BULK_PERF_ASSERT so the default run
|
||||
// stays deterministic on noisy CI.
|
||||
func TestBulkLoadPersistSpeed(t *testing.T) {
|
||||
if testing.Short() {
|
||||
t.Skip("skipping persist-speed timing in -short")
|
||||
}
|
||||
n, e := 8000, 16000
|
||||
nodes, edges := bulkFixture(n, e)
|
||||
|
||||
plain, _ := openTempStore(t)
|
||||
t0 := time.Now()
|
||||
plain.AddBatch(nodes, edges)
|
||||
plainDur := time.Since(t0)
|
||||
|
||||
bulk, _ := openTempStore(t)
|
||||
t1 := time.Now()
|
||||
bulk.BeginBulkLoad()
|
||||
bulk.AddBatch(nodes, edges)
|
||||
if err := bulk.FlushBulk(); err != nil {
|
||||
t.Fatalf("FlushBulk: %v", err)
|
||||
}
|
||||
bulkDur := time.Since(t1)
|
||||
|
||||
if bulk.NodeCount() != plain.NodeCount() || bulk.EdgeCount() != plain.EdgeCount() {
|
||||
t.Fatalf("count mismatch: bulk(%d,%d) plain(%d,%d)",
|
||||
bulk.NodeCount(), bulk.EdgeCount(), plain.NodeCount(), plain.EdgeCount())
|
||||
}
|
||||
integrityOK(t, bulk.db)
|
||||
|
||||
ratio := float64(plainDur) / float64(bulkDur)
|
||||
t.Logf("persist %d nodes / %d edges: plain=%s bulk=%s speedup=%.2fx",
|
||||
n, e, plainDur, bulkDur, ratio)
|
||||
|
||||
// Sanity floor: the fast path must never be dramatically slower.
|
||||
if bulkDur > plainDur*5 {
|
||||
t.Fatalf("fast path far slower: plain=%s bulk=%s", plainDur, bulkDur)
|
||||
}
|
||||
if os.Getenv("GORTEX_BULK_PERF_ASSERT") != "" && ratio < 2.0 {
|
||||
t.Fatalf("fast path speedup %.2fx below 2x target", ratio)
|
||||
}
|
||||
}
|
||||
|
||||
// BenchmarkPersistFixture is reproducible persist-speed evidence: run with
|
||||
//
|
||||
// go test -run=^$ -bench=BenchmarkPersistFixture ./internal/graph/store_sqlite/
|
||||
//
|
||||
// to compare the plain AddBatch path against the bulk-load fast path.
|
||||
func BenchmarkPersistFixture(b *testing.B) {
|
||||
nodes, edges := bulkFixture(50000, 100000)
|
||||
|
||||
run := func(b *testing.B, bulk bool) {
|
||||
for i := 0; i < b.N; i++ {
|
||||
b.StopTimer()
|
||||
path := filepath.Join(b.TempDir(), fmt.Sprintf("p%d.sqlite", i))
|
||||
s, err := Open(path)
|
||||
if err != nil {
|
||||
b.Fatalf("open: %v", err)
|
||||
}
|
||||
b.StartTimer()
|
||||
if bulk {
|
||||
s.BeginBulkLoad()
|
||||
s.AddBatch(nodes, edges)
|
||||
if err := s.FlushBulk(); err != nil {
|
||||
b.Fatalf("FlushBulk: %v", err)
|
||||
}
|
||||
} else {
|
||||
s.AddBatch(nodes, edges)
|
||||
}
|
||||
b.StopTimer()
|
||||
_ = s.Close()
|
||||
}
|
||||
}
|
||||
|
||||
b.Run("nonbulk", func(b *testing.B) { run(b, false) })
|
||||
b.Run("bulk", func(b *testing.B) { run(b, true) })
|
||||
}
|
||||
@@ -0,0 +1,320 @@
|
||||
package store_sqlite
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
|
||||
"github.com/zzet/gortex/internal/graph"
|
||||
)
|
||||
|
||||
// bundleCacheDefaultMaxBytes bounds the total heap the bundle cache may
|
||||
// retain across all cached entries. A count ceiling alone is unsafe: an
|
||||
// entry holds a decoded node plus its full in/out edge lists, and both
|
||||
// nodes and edges carry meta maps, so entry sizes span ~1 KB for a leaf
|
||||
// symbol to multiple MB for a hub node with thousands of edges. A cap
|
||||
// measured in entries therefore admits an unbounded BYTE footprint — a
|
||||
// few thousand hub bundles can pin gigabytes. This cache serves point
|
||||
// lookups on the symbol-search hot path; it is a latency optimisation,
|
||||
// not a working set that needs to be resident, so a modest budget is
|
||||
// right — 64 MiB holds the hot few thousand ordinary bundles while
|
||||
// keeping a long-lived daemon's idle heap bounded. Override with
|
||||
// GORTEX_BUNDLE_CACHE_MAX_MB=<n> (n <= 0 disables the cache entirely).
|
||||
const bundleCacheDefaultMaxBytes = 64 << 20 // 64 MiB
|
||||
|
||||
// bundleCacheMaxEntries is a secondary, generous count ceiling kept
|
||||
// alongside the byte budget. The byte budget is the primary bound; this
|
||||
// guards the map's own structural overhead in the degenerate case of a
|
||||
// flood of tiny entries (a bucket slot and pointers per entry are not
|
||||
// fully reflected in a per-entry byte estimate), and keeps the
|
||||
// wholesale-clear allocation predictable. It is deliberately loose: a
|
||||
// half-million-symbol monorepo's hottest few thousand search hits fit
|
||||
// far under it, so in normal operation the byte budget always trips
|
||||
// first.
|
||||
const bundleCacheMaxEntries = 50000
|
||||
|
||||
const (
|
||||
// bundleEntryOverhead is a coarse fixed charge per cached entry that
|
||||
// is independent of the bundle's string content: the bundleCacheEntry
|
||||
// wrapper, the *entry and *Node pointers, the graph.Node value's flat
|
||||
// struct (its string / slice / map headers, ints, and embedded
|
||||
// time.Time), and the map bucket the node id occupies. String and map
|
||||
// *contents* are added on top. Over-estimating here only makes the
|
||||
// cache clear sooner; it never lets the footprint overshoot the budget.
|
||||
bundleEntryOverhead = 448
|
||||
// bundleEdgeOverhead is the coarse fixed charge for one *Edge in an
|
||||
// in/out slice: the pointer, the slice slot, and the Edge value's flat
|
||||
// struct. Edge string / map contents are added separately.
|
||||
bundleEdgeOverhead = 240
|
||||
// bundleMetaEntryOverhead is the fixed per-key charge for a
|
||||
// map[string]any entry (bucket slot + interface header); the key
|
||||
// length and any string value length are added on top.
|
||||
bundleMetaEntryOverhead = 48
|
||||
)
|
||||
|
||||
// bundleCacheEntry is one node's cached bundle, tagged with the package
|
||||
// it belongs to and the package fingerprint that was current when the
|
||||
// bundle was computed. The entry is served only while
|
||||
// fingerprints[pkgKey] still equals fp — any change to the package's
|
||||
// content (a node or edge added / removed / reweighted, including a
|
||||
// cross-file edge that lands on this node from elsewhere) moves the
|
||||
// fingerprint and forces a recompute, so a cached bundle can never
|
||||
// carry a stale edge.
|
||||
type bundleCacheEntry struct {
|
||||
pkgKey string
|
||||
fp uint64
|
||||
bundle graph.SymbolBundle
|
||||
// bytes is the entry's estimated retained size, recorded at insert so
|
||||
// the running byte total can be adjusted in O(1) whenever the entry is
|
||||
// dropped (invalidation or a stale read).
|
||||
bytes int64
|
||||
}
|
||||
|
||||
// bundleCache is a content-addressed, package-scoped cache over
|
||||
// SearchSymbolBundles. It is keyed at the node level but validated at
|
||||
// the package level: an entry is fresh exactly when the package's
|
||||
// current fingerprint matches the fingerprint the entry was stored at.
|
||||
//
|
||||
// Correctness rests entirely on the fingerprint discipline: the daemon
|
||||
// hands the cache an authoritative per-package fingerprint map after
|
||||
// every analysis pass (which runs after every incremental reindex and
|
||||
// every edit_file / fsnotify-driven graph mutation). The fingerprints
|
||||
// are edge-aware — they fold every package's nodes AND the edges
|
||||
// touching them — so any mutation that could change a cached bundle's
|
||||
// in/out edges moves the relevant package fingerprint and invalidates
|
||||
// the entry. A package whose fingerprint is unchanged is served from
|
||||
// cache; a package the daemon has never reported a fingerprint for is
|
||||
// always treated as a miss (conservative: never serve an unvalidated
|
||||
// bundle).
|
||||
//
|
||||
// The cache is bounded by bytes (maxBytes), not by entry count, because
|
||||
// entry sizes vary by orders of magnitude with a node's edge fan-out and
|
||||
// meta size. maxEntries is a secondary count ceiling only. When either
|
||||
// bound would be exceeded the cache is cleared wholesale rather than
|
||||
// evicting individually: entries are cheap to recompute (one batched
|
||||
// fetch), and a wholesale clear keeps the bookkeeping O(1) and free of an
|
||||
// LRU's per-entry ordering overhead. maxBytes <= 0 disables the cache —
|
||||
// stores become no-ops and every lookup misses (reads still recompute
|
||||
// live through the caller's fallback path).
|
||||
type bundleCache struct {
|
||||
mu sync.Mutex
|
||||
fingerprints map[string]uint64
|
||||
entries map[string]*bundleCacheEntry
|
||||
maxBytes int64 // byte budget (primary bound); <= 0 disables the cache
|
||||
maxEntries int // count ceiling (secondary bound)
|
||||
curBytes int64 // running sum of entries' estimated bytes
|
||||
}
|
||||
|
||||
// newBundleCache builds an empty cache with the default budgets. The byte
|
||||
// budget is overridable with GORTEX_BUNDLE_CACHE_MAX_MB=<n>; n <= 0
|
||||
// disables the cache. It starts inert (every lookup a miss) until the
|
||||
// daemon supplies fingerprints.
|
||||
func newBundleCache() *bundleCache {
|
||||
return &bundleCache{
|
||||
fingerprints: map[string]uint64{},
|
||||
entries: map[string]*bundleCacheEntry{},
|
||||
maxBytes: bundleCacheMaxBytes(),
|
||||
maxEntries: bundleCacheMaxEntries,
|
||||
}
|
||||
}
|
||||
|
||||
// bundleCacheMaxBytes resolves the byte budget from the environment,
|
||||
// falling back to the default. GORTEX_BUNDLE_CACHE_MAX_MB is read in
|
||||
// mebibytes; a value <= 0 returns 0 to disable the cache, and an
|
||||
// unparseable value is ignored (keeps the default).
|
||||
func bundleCacheMaxBytes() int64 {
|
||||
if v := strings.TrimSpace(os.Getenv("GORTEX_BUNDLE_CACHE_MAX_MB")); v != "" {
|
||||
if n, err := strconv.Atoi(v); err == nil {
|
||||
if n <= 0 {
|
||||
return 0
|
||||
}
|
||||
return int64(n) << 20
|
||||
}
|
||||
}
|
||||
return bundleCacheDefaultMaxBytes
|
||||
}
|
||||
|
||||
// bundleEntryBytes conservatively estimates a bundle's retained heap for
|
||||
// the byte budget: a fixed per-entry charge plus the node's string and
|
||||
// meta contents plus each in/out edge's fixed charge and its string and
|
||||
// meta contents. Computed once at insert so the overflow check is a cheap
|
||||
// scalar comparison.
|
||||
func bundleEntryBytes(b graph.SymbolBundle) int64 {
|
||||
n := int64(bundleEntryOverhead)
|
||||
if b.Node != nil {
|
||||
n += nodeStringBytes(b.Node)
|
||||
n += metaBytes(b.Node.Meta)
|
||||
}
|
||||
for _, e := range b.InEdges {
|
||||
n += edgeBytes(e)
|
||||
}
|
||||
for _, e := range b.OutEdges {
|
||||
n += edgeBytes(e)
|
||||
}
|
||||
return n
|
||||
}
|
||||
|
||||
// nodeStringBytes sums the byte lengths of a node's string fields (its
|
||||
// heap-backed content, on top of the fixed struct overhead counted in
|
||||
// bundleEntryOverhead).
|
||||
func nodeStringBytes(nd *graph.Node) int64 {
|
||||
return int64(len(nd.ID) + len(nd.Name) + len(nd.QualName) + len(nd.FilePath) +
|
||||
len(string(nd.Kind)) + len(nd.Language) + len(nd.RepoPrefix) +
|
||||
len(nd.WorkspaceID) + len(nd.ProjectID) + len(nd.AbsoluteFilePath) +
|
||||
len(nd.Origin))
|
||||
}
|
||||
|
||||
// edgeBytes estimates one edge's retained heap: the fixed per-edge charge
|
||||
// plus its string fields and meta contents.
|
||||
func edgeBytes(e *graph.Edge) int64 {
|
||||
if e == nil {
|
||||
return bundleEdgeOverhead
|
||||
}
|
||||
n := int64(bundleEdgeOverhead)
|
||||
n += int64(len(e.From) + len(e.To) + len(string(e.Kind)) + len(e.FilePath) +
|
||||
len(e.ConfidenceLabel) + len(e.Origin) + len(e.Tier) + len(e.Context) +
|
||||
len(e.ReturnUsage) + len(e.Via) + len(e.Alias))
|
||||
n += metaBytes(e.Meta)
|
||||
return n
|
||||
}
|
||||
|
||||
// metaBytes estimates a meta map's retained heap: a fixed charge per key
|
||||
// plus the key length and, for string values, the value length. Non-string
|
||||
// values fold into the fixed charge — meta values are overwhelmingly short
|
||||
// scalars, and a coarse estimate only over-counts, which is safe.
|
||||
func metaBytes(m map[string]any) int64 {
|
||||
if len(m) == 0 {
|
||||
return 0
|
||||
}
|
||||
var n int64
|
||||
for k, v := range m {
|
||||
n += int64(len(k) + bundleMetaEntryOverhead)
|
||||
if s, ok := v.(string); ok {
|
||||
n += int64(len(s))
|
||||
}
|
||||
}
|
||||
return n
|
||||
}
|
||||
|
||||
// SetBundleFingerprints installs the authoritative per-package
|
||||
// fingerprint map and drops any cached entry whose package fingerprint
|
||||
// has changed (or whose package is no longer reported). This is the
|
||||
// invalidation entry point: the daemon calls it after each analysis
|
||||
// pass with the fresh fingerprints derived from the live graph, so a
|
||||
// reindex that altered a package's nodes or edges retires exactly the
|
||||
// affected bundles while leaving untouched packages cached.
|
||||
//
|
||||
// fps is keyed by package key (the directory the package's files live
|
||||
// in, repo-prefixed in multi-repo because the node file paths are).
|
||||
func (s *Store) SetBundleFingerprints(fps map[string]uint64) {
|
||||
if s.bundles == nil {
|
||||
return
|
||||
}
|
||||
s.bundles.refresh(fps)
|
||||
}
|
||||
|
||||
// refresh swaps in the new fingerprint map and prunes every entry whose
|
||||
// package fingerprint no longer matches, decrementing the running byte
|
||||
// total by each dropped entry's estimated size.
|
||||
func (c *bundleCache) refresh(fps map[string]uint64) {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
if fps == nil {
|
||||
fps = map[string]uint64{}
|
||||
}
|
||||
c.fingerprints = fps
|
||||
for id, e := range c.entries {
|
||||
cur, ok := fps[e.pkgKey]
|
||||
if !ok || cur != e.fp {
|
||||
delete(c.entries, id)
|
||||
c.curBytes -= e.bytes
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// bundlePackageKey derives the package key for a node's file path. It
|
||||
// mirrors the analysis layer's packageKey so the cache and the
|
||||
// daemon-supplied fingerprint map agree on package identity: the
|
||||
// directory the file lives in (repo-prefixed in multi-repo because the
|
||||
// stored file paths are), or "" for a file at the repo root / a node
|
||||
// with no path.
|
||||
func bundlePackageKey(filePath string) string {
|
||||
if filePath == "" {
|
||||
return ""
|
||||
}
|
||||
dir := filepath.Dir(filepath.ToSlash(filePath))
|
||||
if dir == "." {
|
||||
return ""
|
||||
}
|
||||
return dir
|
||||
}
|
||||
|
||||
// lookup returns the cached bundle for id when it is fresh — the entry
|
||||
// exists and its package fingerprint still matches the current one. A
|
||||
// node whose package has no reported fingerprint is never served (ok is
|
||||
// false) so an unvalidated bundle can never escape the cache. A stale
|
||||
// entry is dropped in place and its bytes reclaimed.
|
||||
func (c *bundleCache) lookup(id string) (graph.SymbolBundle, bool) {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
e, ok := c.entries[id]
|
||||
if !ok {
|
||||
return graph.SymbolBundle{}, false
|
||||
}
|
||||
cur, ok := c.fingerprints[e.pkgKey]
|
||||
if !ok || cur != e.fp {
|
||||
// Stale or unvalidated — drop it so a later refresh doesn't
|
||||
// have to, and reclaim its bytes.
|
||||
delete(c.entries, id)
|
||||
c.curBytes -= e.bytes
|
||||
return graph.SymbolBundle{}, false
|
||||
}
|
||||
return e.bundle, true
|
||||
}
|
||||
|
||||
// store records a freshly computed bundle, tagged with its package's
|
||||
// current fingerprint. A node whose package has no reported fingerprint
|
||||
// is NOT cached (it could not be validated on read-back), keeping the
|
||||
// cache conservative. The cache is bounded by bytes: when admitting the
|
||||
// new entry would push the running total over the byte budget (or the
|
||||
// count over the secondary ceiling) the cache is cleared wholesale
|
||||
// before the insert. A single bundle that on its own exceeds the whole
|
||||
// budget — a hub node with thousands of edges, exactly the pathological
|
||||
// case a byte cap exists to keep out of long-lived memory — is refused
|
||||
// outright rather than pinned. With maxBytes <= 0 the cache is disabled
|
||||
// and every store is a no-op.
|
||||
func (c *bundleCache) store(b graph.SymbolBundle) {
|
||||
if b.Node == nil {
|
||||
return
|
||||
}
|
||||
pkgKey := bundlePackageKey(b.Node.FilePath)
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
if c.maxBytes <= 0 {
|
||||
return
|
||||
}
|
||||
fp, ok := c.fingerprints[pkgKey]
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
sz := bundleEntryBytes(b)
|
||||
if sz > c.maxBytes {
|
||||
// One entry larger than the entire budget would blow the bound and
|
||||
// be evicted by the very next insert's wholesale clear anyway.
|
||||
return
|
||||
}
|
||||
if old, ok := c.entries[b.Node.ID]; ok {
|
||||
// Replacing an existing entry — discount its bytes and drop it so
|
||||
// curBytes and the count check track the live set.
|
||||
c.curBytes -= old.bytes
|
||||
delete(c.entries, b.Node.ID)
|
||||
}
|
||||
if len(c.entries) > 0 && (c.curBytes+sz > c.maxBytes || len(c.entries) >= c.maxEntries) {
|
||||
c.entries = make(map[string]*bundleCacheEntry)
|
||||
c.curBytes = 0
|
||||
}
|
||||
c.entries[b.Node.ID] = &bundleCacheEntry{pkgKey: pkgKey, fp: fp, bundle: b, bytes: sz}
|
||||
c.curBytes += sz
|
||||
}
|
||||
@@ -0,0 +1,390 @@
|
||||
package store_sqlite
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"path/filepath"
|
||||
"sync"
|
||||
"testing"
|
||||
|
||||
"github.com/zzet/gortex/internal/graph"
|
||||
)
|
||||
|
||||
func mkFnNode(id, name, file string) *graph.Node {
|
||||
return &graph.Node{ID: id, Kind: graph.KindFunction, Name: name, FilePath: file, Language: "go"}
|
||||
}
|
||||
|
||||
// newTestBundleCache builds a cache with the default byte budget without
|
||||
// consulting the environment, so the fingerprint / invalidation unit tests
|
||||
// stay hermetic regardless of GORTEX_BUNDLE_CACHE_MAX_MB.
|
||||
func newTestBundleCache() *bundleCache {
|
||||
return &bundleCache{
|
||||
fingerprints: map[string]uint64{},
|
||||
entries: map[string]*bundleCacheEntry{},
|
||||
maxBytes: bundleCacheDefaultMaxBytes,
|
||||
maxEntries: bundleCacheMaxEntries,
|
||||
}
|
||||
}
|
||||
|
||||
// --- unit tests over the cache logic in isolation ---
|
||||
|
||||
func TestBundleCache_ServesOnlyValidatedFingerprints(t *testing.T) {
|
||||
c := newTestBundleCache()
|
||||
|
||||
b := graph.SymbolBundle{Node: mkFnNode("pkg/x.go::A", "A", "pkg/x.go")}
|
||||
|
||||
// No fingerprint reported for the package yet -> store is a no-op
|
||||
// (conservative: never cache an unvalidated bundle).
|
||||
c.store(b)
|
||||
if _, ok := c.lookup("pkg/x.go::A"); ok {
|
||||
t.Fatal("bundle was cached despite no package fingerprint")
|
||||
}
|
||||
|
||||
// Report a fingerprint, then store: now it caches and serves.
|
||||
c.refresh(map[string]uint64{"pkg": 100})
|
||||
c.store(b)
|
||||
if _, ok := c.lookup("pkg/x.go::A"); !ok {
|
||||
t.Fatal("bundle should be served once its package fingerprint is known")
|
||||
}
|
||||
}
|
||||
|
||||
func TestBundleCache_InvalidatesOnFingerprintChange(t *testing.T) {
|
||||
c := newTestBundleCache()
|
||||
c.refresh(map[string]uint64{"pkg": 1})
|
||||
c.store(graph.SymbolBundle{Node: mkFnNode("pkg/x.go::A", "A", "pkg/x.go")})
|
||||
|
||||
if _, ok := c.lookup("pkg/x.go::A"); !ok {
|
||||
t.Fatal("expected a cache hit on the unchanged fingerprint")
|
||||
}
|
||||
|
||||
// Fingerprint changes -> the entry is invalidated.
|
||||
c.refresh(map[string]uint64{"pkg": 2})
|
||||
if _, ok := c.lookup("pkg/x.go::A"); ok {
|
||||
t.Fatal("entry must be dropped when its package fingerprint changes")
|
||||
}
|
||||
}
|
||||
|
||||
func TestBundleCache_CrossRepoIsolation(t *testing.T) {
|
||||
c := newTestBundleCache()
|
||||
// Two repos with the same inner directory name resolve to DIFFERENT
|
||||
// package keys because the stored file paths are repo-prefixed.
|
||||
c.refresh(map[string]uint64{
|
||||
"repoA/pkg": 10,
|
||||
"repoB/pkg": 20,
|
||||
})
|
||||
c.store(graph.SymbolBundle{Node: mkFnNode("repoA/pkg/x.go::A", "A", "repoA/pkg/x.go")})
|
||||
c.store(graph.SymbolBundle{Node: mkFnNode("repoB/pkg/x.go::A", "A", "repoB/pkg/x.go")})
|
||||
|
||||
// Bumping only repoA's fingerprint must not touch repoB's entry.
|
||||
c.refresh(map[string]uint64{
|
||||
"repoA/pkg": 11,
|
||||
"repoB/pkg": 20,
|
||||
})
|
||||
if _, ok := c.lookup("repoA/pkg/x.go::A"); ok {
|
||||
t.Fatal("repoA entry should have been invalidated")
|
||||
}
|
||||
if _, ok := c.lookup("repoB/pkg/x.go::A"); !ok {
|
||||
t.Fatal("repoB entry must survive a repoA-only fingerprint bump")
|
||||
}
|
||||
}
|
||||
|
||||
func TestBundlePackageKey(t *testing.T) {
|
||||
cases := map[string]string{
|
||||
"pkg/sub/x.go": "pkg/sub",
|
||||
"x.go": "",
|
||||
"": "",
|
||||
"repo/a/b.go": "repo/a",
|
||||
}
|
||||
for in, want := range cases {
|
||||
if got := bundlePackageKey(in); got != want {
|
||||
t.Errorf("bundlePackageKey(%q) = %q, want %q", in, got, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// --- integration tests through the store's SearchSymbolBundles ---
|
||||
|
||||
func newBundleTestStore(t *testing.T) *Store {
|
||||
t.Helper()
|
||||
s, err := Open(filepath.Join(t.TempDir(), "b.sqlite"))
|
||||
if err != nil {
|
||||
t.Fatalf("Open: %v", err)
|
||||
}
|
||||
t.Cleanup(func() { _ = s.Close() })
|
||||
return s
|
||||
}
|
||||
|
||||
func seedBundleStore(t *testing.T, s *Store) {
|
||||
t.Helper()
|
||||
s.AddNode(mkFnNode("pkg/x.go::A", "AlphaWidget", "pkg/x.go"))
|
||||
s.AddNode(mkFnNode("pkg/x.go::B", "BetaWidget", "pkg/x.go"))
|
||||
s.AddEdge(&graph.Edge{From: "pkg/x.go::A", To: "pkg/x.go::B", Kind: graph.EdgeCalls, FilePath: "pkg/x.go"})
|
||||
|
||||
items := []graph.SymbolFTSItem{
|
||||
{NodeID: "pkg/x.go::A", Tokens: "alpha widget"},
|
||||
{NodeID: "pkg/x.go::B", Tokens: "beta widget"},
|
||||
}
|
||||
if err := s.BulkUpsertSymbolFTS("", items); err != nil {
|
||||
t.Fatalf("BulkUpsertSymbolFTS: %v", err)
|
||||
}
|
||||
if err := s.BuildSymbolIndex(); err != nil {
|
||||
t.Fatalf("BuildSymbolIndex: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func bundleByID(bundles []graph.SymbolBundle) map[string]graph.SymbolBundle {
|
||||
out := make(map[string]graph.SymbolBundle, len(bundles))
|
||||
for _, b := range bundles {
|
||||
if b.Node != nil {
|
||||
out[b.Node.ID] = b
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func TestSearchSymbolBundles_CacheHitOnUnchangedFingerprint(t *testing.T) {
|
||||
s := newBundleTestStore(t)
|
||||
seedBundleStore(t, s)
|
||||
|
||||
// Report a fingerprint so the first query populates the cache.
|
||||
s.SetBundleFingerprints(map[string]uint64{"pkg": 1})
|
||||
|
||||
first, err := s.SearchSymbolBundles("widget", 10)
|
||||
if err != nil {
|
||||
t.Fatalf("first SearchSymbolBundles: %v", err)
|
||||
}
|
||||
got := bundleByID(first)
|
||||
if b, ok := got["pkg/x.go::A"]; !ok || len(b.OutEdges) != 1 {
|
||||
t.Fatalf("expected A with 1 out-edge on first query, got %+v", got["pkg/x.go::A"])
|
||||
}
|
||||
|
||||
// Mutate the graph WITHOUT bumping the fingerprint: add a second
|
||||
// out-edge from A. A correct content-addressed cache serves the
|
||||
// STALE (1-edge) bundle because the fingerprint is unchanged — proof
|
||||
// the bundle came from cache, not a fresh fetch.
|
||||
s.AddNode(mkFnNode("pkg/x.go::C", "GammaWidget", "pkg/x.go"))
|
||||
s.AddEdge(&graph.Edge{From: "pkg/x.go::A", To: "pkg/x.go::C", Kind: graph.EdgeCalls, FilePath: "pkg/x.go"})
|
||||
|
||||
second, err := s.SearchSymbolBundles("widget", 10)
|
||||
if err != nil {
|
||||
t.Fatalf("second SearchSymbolBundles: %v", err)
|
||||
}
|
||||
cached := bundleByID(second)["pkg/x.go::A"]
|
||||
if len(cached.OutEdges) != 1 {
|
||||
t.Fatalf("expected the cached 1-edge bundle to be served on an unchanged fingerprint, got %d edges",
|
||||
len(cached.OutEdges))
|
||||
}
|
||||
}
|
||||
|
||||
func TestSearchSymbolBundles_MissAndRecomputeOnFingerprintChange(t *testing.T) {
|
||||
s := newBundleTestStore(t)
|
||||
seedBundleStore(t, s)
|
||||
s.SetBundleFingerprints(map[string]uint64{"pkg": 1})
|
||||
|
||||
if _, err := s.SearchSymbolBundles("widget", 10); err != nil {
|
||||
t.Fatalf("warm-up query: %v", err)
|
||||
}
|
||||
|
||||
// Add a real out-edge, then bump the package fingerprint to signal
|
||||
// the content changed. The next query must recompute and surface the
|
||||
// new edge.
|
||||
s.AddNode(mkFnNode("pkg/x.go::C", "GammaWidget", "pkg/x.go"))
|
||||
s.AddEdge(&graph.Edge{From: "pkg/x.go::A", To: "pkg/x.go::C", Kind: graph.EdgeCalls, FilePath: "pkg/x.go"})
|
||||
s.SetBundleFingerprints(map[string]uint64{"pkg": 2})
|
||||
|
||||
after, err := s.SearchSymbolBundles("widget", 10)
|
||||
if err != nil {
|
||||
t.Fatalf("post-invalidation query: %v", err)
|
||||
}
|
||||
fresh := bundleByID(after)["pkg/x.go::A"]
|
||||
if len(fresh.OutEdges) != 2 {
|
||||
t.Fatalf("expected the recomputed 2-edge bundle after a fingerprint bump, got %d edges",
|
||||
len(fresh.OutEdges))
|
||||
}
|
||||
}
|
||||
|
||||
func TestSearchSymbolBundles_UncachedWithoutFingerprints(t *testing.T) {
|
||||
s := newBundleTestStore(t)
|
||||
seedBundleStore(t, s)
|
||||
// No SetBundleFingerprints call -> the cache stays inert and every
|
||||
// query recomputes live. Adding an edge must show up immediately.
|
||||
first, err := s.SearchSymbolBundles("widget", 10)
|
||||
if err != nil {
|
||||
t.Fatalf("first query: %v", err)
|
||||
}
|
||||
if got := bundleByID(first)["pkg/x.go::A"]; len(got.OutEdges) != 1 {
|
||||
t.Fatalf("expected 1 edge live, got %d", len(got.OutEdges))
|
||||
}
|
||||
|
||||
s.AddNode(mkFnNode("pkg/x.go::C", "GammaWidget", "pkg/x.go"))
|
||||
s.AddEdge(&graph.Edge{From: "pkg/x.go::A", To: "pkg/x.go::C", Kind: graph.EdgeCalls, FilePath: "pkg/x.go"})
|
||||
|
||||
second, err := s.SearchSymbolBundles("widget", 10)
|
||||
if err != nil {
|
||||
t.Fatalf("second query: %v", err)
|
||||
}
|
||||
if got := bundleByID(second)["pkg/x.go::A"]; len(got.OutEdges) != 2 {
|
||||
t.Fatalf("uncached path must reflect the new edge live, got %d", len(got.OutEdges))
|
||||
}
|
||||
}
|
||||
|
||||
// --- byte-budget tests ---
|
||||
|
||||
func TestBundleCache_ByteBudgetEvictionAtBoundary(t *testing.T) {
|
||||
c := newTestBundleCache()
|
||||
c.refresh(map[string]uint64{"pkg": 1})
|
||||
|
||||
// Fixed-width ids so every entry estimates to the same size.
|
||||
mk := func(i int) graph.SymbolBundle {
|
||||
return graph.SymbolBundle{Node: mkFnNode(fmt.Sprintf("pkg/x.go::N%03d", i), "W", "pkg/x.go")}
|
||||
}
|
||||
unit := bundleEntryBytes(mk(0))
|
||||
const k = 4
|
||||
c.maxBytes = unit * k // budget holds exactly k entries
|
||||
|
||||
for i := 0; i < k; i++ {
|
||||
c.store(mk(i))
|
||||
}
|
||||
if len(c.entries) != k {
|
||||
t.Fatalf("expected %d entries filling the budget, got %d", k, len(c.entries))
|
||||
}
|
||||
if c.curBytes != unit*k {
|
||||
t.Fatalf("curBytes = %d, want %d", c.curBytes, unit*k)
|
||||
}
|
||||
|
||||
// One more entry crosses the budget -> wholesale clear, only the newest
|
||||
// survives and the byte total resets to a single unit.
|
||||
c.store(mk(k))
|
||||
if len(c.entries) != 1 {
|
||||
t.Fatalf("crossing the budget must clear wholesale to 1 entry, got %d", len(c.entries))
|
||||
}
|
||||
if c.curBytes != unit {
|
||||
t.Fatalf("curBytes after clear = %d, want %d", c.curBytes, unit)
|
||||
}
|
||||
if _, ok := c.lookup(fmt.Sprintf("pkg/x.go::N%03d", k)); !ok {
|
||||
t.Fatal("the entry that triggered the clear must remain served")
|
||||
}
|
||||
if _, ok := c.lookup("pkg/x.go::N000"); ok {
|
||||
t.Fatal("a pre-clear entry must be gone after the wholesale clear")
|
||||
}
|
||||
}
|
||||
|
||||
func TestBundleCache_RefusesEntryLargerThanBudget(t *testing.T) {
|
||||
c := newTestBundleCache()
|
||||
c.refresh(map[string]uint64{"pkg": 1})
|
||||
b := graph.SymbolBundle{Node: mkFnNode("pkg/x.go::A", "A", "pkg/x.go")}
|
||||
c.maxBytes = bundleEntryBytes(b) - 1 // budget just below a single entry
|
||||
|
||||
c.store(b)
|
||||
if _, ok := c.lookup("pkg/x.go::A"); ok {
|
||||
t.Fatal("an entry larger than the whole budget must not be cached")
|
||||
}
|
||||
if len(c.entries) != 0 || c.curBytes != 0 {
|
||||
t.Fatalf("oversized store must leave the cache empty, got %d entries / %d bytes",
|
||||
len(c.entries), c.curBytes)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBundleCacheMaxBytes_EnvOverride(t *testing.T) {
|
||||
t.Setenv("GORTEX_BUNDLE_CACHE_MAX_MB", "128")
|
||||
if got := bundleCacheMaxBytes(); got != 128<<20 {
|
||||
t.Fatalf("env override = %d, want %d", got, 128<<20)
|
||||
}
|
||||
if c := newBundleCache(); c.maxBytes != 128<<20 {
|
||||
t.Fatalf("newBundleCache maxBytes = %d, want %d", c.maxBytes, 128<<20)
|
||||
}
|
||||
|
||||
// Empty and unparseable values keep the default.
|
||||
t.Setenv("GORTEX_BUNDLE_CACHE_MAX_MB", "")
|
||||
if got := bundleCacheMaxBytes(); got != bundleCacheDefaultMaxBytes {
|
||||
t.Fatalf("empty override should keep the default, got %d", got)
|
||||
}
|
||||
t.Setenv("GORTEX_BUNDLE_CACHE_MAX_MB", "not-a-number")
|
||||
if got := bundleCacheMaxBytes(); got != bundleCacheDefaultMaxBytes {
|
||||
t.Fatalf("unparseable override should keep the default, got %d", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBundleCache_DisabledMode(t *testing.T) {
|
||||
t.Setenv("GORTEX_BUNDLE_CACHE_MAX_MB", "0")
|
||||
c := newBundleCache()
|
||||
if c.maxBytes != 0 {
|
||||
t.Fatalf("expected a disabled cache (maxBytes 0), got %d", c.maxBytes)
|
||||
}
|
||||
c.refresh(map[string]uint64{"pkg": 1})
|
||||
c.store(graph.SymbolBundle{Node: mkFnNode("pkg/x.go::A", "A", "pkg/x.go")})
|
||||
if len(c.entries) != 0 {
|
||||
t.Fatalf("a disabled cache must not store, got %d entries", len(c.entries))
|
||||
}
|
||||
if _, ok := c.lookup("pkg/x.go::A"); ok {
|
||||
t.Fatal("a disabled cache must always miss")
|
||||
}
|
||||
|
||||
// A negative budget disables too.
|
||||
t.Setenv("GORTEX_BUNDLE_CACHE_MAX_MB", "-4")
|
||||
if got := bundleCacheMaxBytes(); got != 0 {
|
||||
t.Fatalf("a negative override should disable the cache (0), got %d", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSearchSymbolBundles_DisabledCacheStillServes(t *testing.T) {
|
||||
t.Setenv("GORTEX_BUNDLE_CACHE_MAX_MB", "0")
|
||||
s := newBundleTestStore(t)
|
||||
seedBundleStore(t, s)
|
||||
s.SetBundleFingerprints(map[string]uint64{"pkg": 1})
|
||||
|
||||
res, err := s.SearchSymbolBundles("widget", 10)
|
||||
if err != nil {
|
||||
t.Fatalf("SearchSymbolBundles with the cache disabled: %v", err)
|
||||
}
|
||||
if b, ok := bundleByID(res)["pkg/x.go::A"]; !ok || len(b.OutEdges) != 1 {
|
||||
t.Fatalf("a disabled cache must still return live bundles, got %+v", b)
|
||||
}
|
||||
if s.bundles.maxBytes != 0 {
|
||||
t.Fatalf("expected the store's cache disabled, got maxBytes %d", s.bundles.maxBytes)
|
||||
}
|
||||
if len(s.bundles.entries) != 0 {
|
||||
t.Fatalf("a disabled cache must stay empty, got %d entries", len(s.bundles.entries))
|
||||
}
|
||||
}
|
||||
|
||||
func TestBundleCache_ConcurrentReadInsert(t *testing.T) {
|
||||
c := newTestBundleCache()
|
||||
c.maxBytes = 8 << 10 // small budget so wholesale clears fire under contention
|
||||
c.refresh(map[string]uint64{"pkg": 1})
|
||||
|
||||
const workers = 8
|
||||
const iters = 3000
|
||||
var wg sync.WaitGroup
|
||||
wg.Add(workers)
|
||||
for w := 0; w < workers; w++ {
|
||||
go func(w int) {
|
||||
defer wg.Done()
|
||||
for i := 0; i < iters; i++ {
|
||||
id := fmt.Sprintf("pkg/x.go::N%d_%d", w, i%64)
|
||||
switch i % 3 {
|
||||
case 0:
|
||||
c.store(graph.SymbolBundle{Node: mkFnNode(id, "W", "pkg/x.go")})
|
||||
case 1:
|
||||
_, _ = c.lookup(id)
|
||||
default:
|
||||
c.refresh(map[string]uint64{"pkg": uint64(i)})
|
||||
}
|
||||
}
|
||||
}(w)
|
||||
}
|
||||
wg.Wait()
|
||||
|
||||
// No goroutines remain: the running total must exactly equal the summed
|
||||
// bytes of the surviving entries (the accounting invariant), which also
|
||||
// proves it never drifted negative under contention.
|
||||
var sum int64
|
||||
for _, e := range c.entries {
|
||||
sum += e.bytes
|
||||
}
|
||||
if c.curBytes != sum {
|
||||
t.Fatalf("curBytes %d != sum of live entry bytes %d", c.curBytes, sum)
|
||||
}
|
||||
if c.curBytes > c.maxBytes {
|
||||
t.Fatalf("curBytes %d exceeds the byte budget %d", c.curBytes, c.maxBytes)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,69 @@
|
||||
package store_sqlite_test
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"github.com/zzet/gortex/internal/graph"
|
||||
)
|
||||
|
||||
// TestClosedStoreReadsDoNotPanic pins the teardown-race fix: after Close()
|
||||
// has shut the store (daemon shutdown / restart / store swap), an in-flight
|
||||
// reader — e.g. a deferred parallel-enrich goroutine still holding a cached
|
||||
// *sql.Stmt — must degrade to an empty result, never panic the whole daemon.
|
||||
// Before the fix this surfaced as `panic: store_sqlite: sql: statement is
|
||||
// closed` from GetNode under runDeferredEnrichParallel.
|
||||
func TestClosedStoreReadsDoNotPanic(t *testing.T) {
|
||||
s := openTestStore(t)
|
||||
s.AddNode(&graph.Node{ID: "p/a.go::Foo", Kind: graph.KindType, Name: "Foo", FilePath: "p/a.go"})
|
||||
require.NotNil(t, s.GetNode("p/a.go::Foo"), "sanity: node readable before close")
|
||||
|
||||
require.NoError(t, s.Close())
|
||||
|
||||
assert.NotPanics(t, func() {
|
||||
assert.Nil(t, s.GetNode("p/a.go::Foo"))
|
||||
assert.Empty(t, s.FindNodesByName("Foo"))
|
||||
assert.Empty(t, s.GetFileNodes("p/a.go"))
|
||||
}, "reads after Close must degrade gracefully, not panic")
|
||||
}
|
||||
|
||||
// TestClosedStoreAggregatorsDoNotPanic pins the aggregator teardown-race sweep:
|
||||
// each aggregator read runs its Query error through panicOnFatal, which
|
||||
// swallows the "database is closed" race — leaving rows == nil. Every such site
|
||||
// must early-return its empty value instead of dereferencing nil rows. The live
|
||||
// crash was NodeIDsByKinds (FindHotspots -> RunAnalysis at watch-start) SIGSEGV
|
||||
// on nil rows right after a long warmup. Each method is called with non-empty
|
||||
// args so it actually reaches the Query rather than an early argument guard.
|
||||
func TestClosedStoreAggregatorsDoNotPanic(t *testing.T) {
|
||||
s := openTestStore(t)
|
||||
s.AddNode(&graph.Node{ID: "p/a.go::Foo", Kind: graph.KindType, Name: "Foo", FilePath: "p/a.go"})
|
||||
s.AddNode(&graph.Node{ID: "p/a.go::bar", Kind: graph.KindFunction, Name: "bar", FilePath: "p/a.go"})
|
||||
s.AddEdge(&graph.Edge{From: "p/a.go::bar", To: "p/a.go::Foo", Kind: graph.EdgeReferences, FilePath: "p/a.go", Line: 1})
|
||||
require.NoError(t, s.Close())
|
||||
|
||||
nodeKinds := []graph.NodeKind{graph.KindType, graph.KindFunction}
|
||||
edgeKinds := []graph.EdgeKind{graph.EdgeReferences}
|
||||
ids := []string{"p/a.go::Foo", "p/a.go::bar"}
|
||||
assert.NotPanics(t, func() {
|
||||
assert.Empty(t, s.InEdgeCountsByKind(edgeKinds))
|
||||
assert.Empty(t, s.NodeIDsByKinds(nodeKinds))
|
||||
assert.Empty(t, s.EdgeKindCounts())
|
||||
assert.Empty(t, s.NodeDegreeByKinds(nodeKinds, ""))
|
||||
assert.Empty(t, s.FileImportCounts(nil)) // exercises aggScanImportCounts
|
||||
assert.Empty(t, s.InDegreeForNodes(ids))
|
||||
assert.Empty(t, s.CrossRepoEdgeCounts())
|
||||
assert.Empty(t, s.FileImporters("p/a.go"))
|
||||
assert.Empty(t, s.FileSymbolNamesByPaths([]string{"p/a.go"}, nodeKinds))
|
||||
assert.Empty(t, s.NodeDegreeCounts(ids, edgeKinds))
|
||||
assert.Empty(t, s.NodeFanCounts(ids, edgeKinds, edgeKinds))
|
||||
assert.Empty(t, s.CommunityCrossingsByKind(edgeKinds, map[string]string{"p/a.go::bar": "c0"}))
|
||||
// Iterator-shaped: the Query runs inside the yield closure.
|
||||
n := 0
|
||||
for range s.EdgeAdjacencyForKinds(edgeKinds, nodeKinds) {
|
||||
n++
|
||||
}
|
||||
assert.Zero(t, n)
|
||||
}, "aggregator reads after Close must degrade to empty, not panic")
|
||||
}
|
||||
@@ -0,0 +1,46 @@
|
||||
package store_sqlite
|
||||
|
||||
import (
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"github.com/zzet/gortex/internal/graph"
|
||||
)
|
||||
|
||||
// TestGetRepoNonContentNodes verifies the SQL-level content filter (which
|
||||
// json_extracts data_class out of the JSON meta blob) drops only content
|
||||
// section nodes — keeping code, markdown prose, and data assets — so the
|
||||
// code passes can enumerate without materialising content sections.
|
||||
func TestGetRepoNonContentNodes(t *testing.T) {
|
||||
s, err := Open(filepath.Join(t.TempDir(), "n.sqlite"))
|
||||
require.NoError(t, err)
|
||||
t.Cleanup(func() { _ = s.Close() })
|
||||
|
||||
s.AddBatch([]*graph.Node{
|
||||
{ID: "code1", Kind: graph.KindFunction, Name: "Foo", RepoPrefix: "r"},
|
||||
{ID: "content1", Kind: graph.KindDoc, Name: "doc.txt::0", RepoPrefix: "r",
|
||||
Meta: map[string]any{"data_class": "content", "section_text": "x"}},
|
||||
{ID: "prose1", Kind: graph.KindDoc, Name: "README.md::0", RepoPrefix: "r",
|
||||
Meta: map[string]any{"asset_kind": "markdown_section"}},
|
||||
{ID: "data1", Kind: graph.KindFile, Name: "x.parquet", RepoPrefix: "r",
|
||||
Meta: map[string]any{"data_class": "data"}},
|
||||
}, nil)
|
||||
|
||||
// Runtime assertion the store satisfies the optional capability.
|
||||
var cr graph.NonContentNodeReader = s
|
||||
|
||||
ids := map[string]bool{}
|
||||
for _, n := range cr.GetRepoNonContentNodes("r") {
|
||||
ids[n.ID] = true
|
||||
}
|
||||
require.True(t, ids["code1"], "code node kept")
|
||||
require.True(t, ids["prose1"], "markdown prose kept (not data_class=content)")
|
||||
require.True(t, ids["data1"], "data asset kept (data_class=data, not content)")
|
||||
require.False(t, ids["content1"], "content section dropped at the SQL level")
|
||||
require.Len(t, ids, 3)
|
||||
|
||||
// Empty prefix spans all repos (still drops content).
|
||||
require.Len(t, s.GetRepoNonContentNodes(""), 3)
|
||||
}
|
||||
@@ -0,0 +1,61 @@
|
||||
package store_sqlite_test
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/zzet/gortex/internal/graph"
|
||||
)
|
||||
|
||||
// TestExternalCallCandidateEdges asserts the pushdown selects exactly the
|
||||
// external-package terminals (dep:: / stdlib:: / external::, the
|
||||
// per-repo-prefixed stdlib form, and already-materialised external-call::
|
||||
// nodes) and nothing else — no ordinary resolved call/reference edges,
|
||||
// no non-call edges.
|
||||
func TestExternalCallCandidateEdges(t *testing.T) {
|
||||
s := openTestStore(t)
|
||||
|
||||
add := func(from, to string, kind graph.EdgeKind) {
|
||||
s.AddEdge(&graph.Edge{From: from, To: to, Kind: kind, FilePath: "a.go", Line: 1})
|
||||
}
|
||||
// External terminals — should be selected.
|
||||
add("a.go::f", "dep::github.com/x/y::Z", graph.EdgeCalls)
|
||||
add("a.go::f", "stdlib::fmt::Sprintf", graph.EdgeCalls)
|
||||
add("a.go::f", "myrepo::stdlib::net/http::Get", graph.EdgeCalls) // per-repo-prefixed stdlib form
|
||||
add("a.go::f", "external::svc.internal/api", graph.EdgeReferences)
|
||||
add("a.go::f", "external-call::dep::github.com/a/b", graph.EdgeCalls) // already synthesized
|
||||
// Non-candidates — must NOT be selected.
|
||||
add("a.go::f", "a.go::resolvedCallee", graph.EdgeCalls) // ordinary resolved call
|
||||
add("a.go::f", "unresolved::SomeName", graph.EdgeCalls) // bare unresolved (no import evidence)
|
||||
add("a.go::f", "a.go::SomeType", graph.EdgeImplements) // not a call/ref edge
|
||||
add("a.go::f", "dep::github.com/x/y::Z", graph.EdgeTests) // dep target but wrong kind
|
||||
|
||||
got := map[string]bool{}
|
||||
for _, e := range s.ExternalCallCandidateEdges() {
|
||||
got[e.To] = true
|
||||
}
|
||||
|
||||
want := []string{
|
||||
"dep::github.com/x/y::Z",
|
||||
"stdlib::fmt::Sprintf",
|
||||
"myrepo::stdlib::net/http::Get",
|
||||
"external::svc.internal/api",
|
||||
"external-call::dep::github.com/a/b",
|
||||
}
|
||||
for _, w := range want {
|
||||
if !got[w] {
|
||||
t.Errorf("ExternalCallCandidateEdges missing external terminal %q", w)
|
||||
}
|
||||
}
|
||||
notWant := []string{"a.go::resolvedCallee", "unresolved::SomeName", "a.go::SomeType"}
|
||||
for _, nw := range notWant {
|
||||
if got[nw] {
|
||||
t.Errorf("ExternalCallCandidateEdges wrongly selected non-candidate %q", nw)
|
||||
}
|
||||
}
|
||||
// The EdgeImplements/EdgeTests rows share targets with selected ones
|
||||
// but must not inflate the count via the wrong kind: exactly the 5
|
||||
// distinct external targets above, all from calls/references kinds.
|
||||
if len(got) != len(want) {
|
||||
t.Errorf("selected %d distinct targets, want %d: %v", len(got), len(want), got)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,149 @@
|
||||
package store_sqlite
|
||||
|
||||
import (
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"github.com/zzet/gortex/internal/graph"
|
||||
)
|
||||
|
||||
func ftsRowCount(t *testing.T, s *Store, table, nodeID string) int {
|
||||
t.Helper()
|
||||
var n int
|
||||
q := `SELECT count(*) FROM ` + table + ` WHERE node_id = ?`
|
||||
if err := s.db.QueryRow(q, nodeID).Scan(&n); err != nil {
|
||||
t.Fatalf("count %s: %v", table, err)
|
||||
}
|
||||
return n
|
||||
}
|
||||
|
||||
func ftsHits(t *testing.T, s *Store, query string) int {
|
||||
t.Helper()
|
||||
hits, err := s.SearchSymbols(query, 20)
|
||||
if err != nil {
|
||||
t.Fatalf("SearchSymbols(%q): %v", query, err)
|
||||
}
|
||||
return len(hits)
|
||||
}
|
||||
|
||||
// TestUpsertSymbolFTS_ReplacesWithoutDuplicates is the core correctness
|
||||
// guard for the rowid-map delete: re-upserting a symbol must drop its prior
|
||||
// row (by docid) and leave exactly one FTS row + one map row. A wrong
|
||||
// LastInsertId / stale map would leave the old tokens searchable and the
|
||||
// row count at 2 — so this also proves the FTS5 docid round-trips.
|
||||
func TestUpsertSymbolFTS_ReplacesWithoutDuplicates(t *testing.T) {
|
||||
s, err := Open(filepath.Join(t.TempDir(), "fts.sqlite"))
|
||||
if err != nil {
|
||||
t.Fatalf("Open: %v", err)
|
||||
}
|
||||
t.Cleanup(func() { _ = s.Close() })
|
||||
|
||||
const id = "pkg/x.go::A"
|
||||
s.AddNode(mkFnNode(id, "AlphaWidget", "pkg/x.go"))
|
||||
|
||||
// Upsert three times: first insert, then two replacements that each
|
||||
// exercise the mapped-entry docid delete.
|
||||
for _, tokens := range []string{"alpha widget red", "alpha widget green", "alpha widget blue"} {
|
||||
if err := s.UpsertSymbolFTS(id, tokens); err != nil {
|
||||
t.Fatalf("UpsertSymbolFTS(%q): %v", tokens, err)
|
||||
}
|
||||
}
|
||||
|
||||
if got := ftsRowCount(t, s, "symbol_fts", id); got != 1 {
|
||||
t.Fatalf("symbol_fts rows for %s = %d, want 1 (duplicate row leaked)", id, got)
|
||||
}
|
||||
if got := ftsRowCount(t, s, "symbol_fts_rowid", id); got != 1 {
|
||||
t.Fatalf("symbol_fts_rowid rows for %s = %d, want 1", id, got)
|
||||
}
|
||||
// Only the latest tokens are searchable; the superseded ones are gone.
|
||||
if got := ftsHits(t, s, "blue"); got != 1 {
|
||||
t.Fatalf("search 'blue' (current tokens) = %d hits, want 1", got)
|
||||
}
|
||||
for _, stale := range []string{"red", "green"} {
|
||||
if got := ftsHits(t, s, stale); got != 0 {
|
||||
t.Fatalf("search %q (superseded tokens) = %d hits, want 0 (delete missed)", stale, got)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestBulkUpsertSymbolFTS_MaintainsRowidMap proves the bulk path keeps the
|
||||
// sidecar in lockstep, and that a follow-up incremental upsert on a
|
||||
// bulk-loaded symbol still replaces cleanly (no duplicate).
|
||||
func TestBulkUpsertSymbolFTS_MaintainsRowidMap(t *testing.T) {
|
||||
s, err := Open(filepath.Join(t.TempDir(), "fts.sqlite"))
|
||||
if err != nil {
|
||||
t.Fatalf("Open: %v", err)
|
||||
}
|
||||
t.Cleanup(func() { _ = s.Close() })
|
||||
|
||||
s.AddNode(mkFnNode("pkg/x.go::A", "AlphaWidget", "pkg/x.go"))
|
||||
s.AddNode(mkFnNode("pkg/x.go::B", "BetaWidget", "pkg/x.go"))
|
||||
if err := s.BulkUpsertSymbolFTS("", []graph.SymbolFTSItem{
|
||||
{NodeID: "pkg/x.go::A", Tokens: "alpha widget red"},
|
||||
{NodeID: "pkg/x.go::B", Tokens: "beta widget red"},
|
||||
}); err != nil {
|
||||
t.Fatalf("BulkUpsertSymbolFTS: %v", err)
|
||||
}
|
||||
|
||||
var mapRows int
|
||||
if err := s.db.QueryRow(`SELECT count(*) FROM symbol_fts_rowid`).Scan(&mapRows); err != nil {
|
||||
t.Fatalf("count map: %v", err)
|
||||
}
|
||||
if mapRows != 2 {
|
||||
t.Fatalf("symbol_fts_rowid rows = %d, want 2 after bulk", mapRows)
|
||||
}
|
||||
|
||||
// Incremental replace on a bulk-loaded symbol — must not duplicate.
|
||||
if err := s.UpsertSymbolFTS("pkg/x.go::A", "alpha widget blue"); err != nil {
|
||||
t.Fatalf("UpsertSymbolFTS: %v", err)
|
||||
}
|
||||
if got := ftsRowCount(t, s, "symbol_fts", "pkg/x.go::A"); got != 1 {
|
||||
t.Fatalf("symbol_fts rows after incremental replace = %d, want 1", got)
|
||||
}
|
||||
if got := ftsHits(t, s, "blue"); got != 1 {
|
||||
t.Fatalf("search 'blue' = %d, want 1", got)
|
||||
}
|
||||
}
|
||||
|
||||
// TestBackfillSymbolFTSRowidMap simulates a database built before the
|
||||
// sidecar existed (rows in symbol_fts, none in the map) and proves the
|
||||
// backfill repopulates it so the next incremental upsert replaces cleanly
|
||||
// instead of leaking a duplicate.
|
||||
func TestBackfillSymbolFTSRowidMap(t *testing.T) {
|
||||
s, err := Open(filepath.Join(t.TempDir(), "fts.sqlite"))
|
||||
if err != nil {
|
||||
t.Fatalf("Open: %v", err)
|
||||
}
|
||||
t.Cleanup(func() { _ = s.Close() })
|
||||
|
||||
const id = "pkg/x.go::A"
|
||||
s.AddNode(mkFnNode(id, "AlphaWidget", "pkg/x.go"))
|
||||
|
||||
// Simulate the legacy state: a row in symbol_fts with no map entry.
|
||||
if _, err := s.db.Exec(
|
||||
`INSERT INTO symbol_fts (node_id, repo_prefix, tokens) VALUES (?, '', ?)`,
|
||||
id, "alpha widget red"); err != nil {
|
||||
t.Fatalf("seed legacy fts row: %v", err)
|
||||
}
|
||||
if _, err := s.db.Exec(`DELETE FROM symbol_fts_rowid`); err != nil {
|
||||
t.Fatalf("clear map: %v", err)
|
||||
}
|
||||
|
||||
if err := backfillSymbolFTSRowidMap(s.db); err != nil {
|
||||
t.Fatalf("backfill: %v", err)
|
||||
}
|
||||
if got := ftsRowCount(t, s, "symbol_fts_rowid", id); got != 1 {
|
||||
t.Fatalf("map rows after backfill = %d, want 1", got)
|
||||
}
|
||||
|
||||
// Now an incremental upsert must replace, not duplicate.
|
||||
if err := s.UpsertSymbolFTS(id, "alpha widget blue"); err != nil {
|
||||
t.Fatalf("UpsertSymbolFTS: %v", err)
|
||||
}
|
||||
if got := ftsRowCount(t, s, "symbol_fts", id); got != 1 {
|
||||
t.Fatalf("symbol_fts rows after post-backfill replace = %d, want 1 (dup leaked)", got)
|
||||
}
|
||||
if got := ftsHits(t, s, "red"); got != 0 {
|
||||
t.Fatalf("search 'red' (superseded) = %d, want 0", got)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,76 @@
|
||||
package store_sqlite
|
||||
|
||||
import (
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"github.com/zzet/gortex/internal/graph"
|
||||
)
|
||||
|
||||
// TestGetRepoNodesLight verifies the graph.LightNodeReader fast path
|
||||
// matches GetRepoNodes on IDs and promoted-field values, stays scoped to
|
||||
// repo_prefix, and never surfaces non-promoted meta content — the
|
||||
// invariant the enrichment hover-candidate refetch depends on for
|
||||
// correctness (see EnrichRepoContext's use of repoScopedNodesLight).
|
||||
func TestGetRepoNodesLight(t *testing.T) {
|
||||
s, err := Open(filepath.Join(t.TempDir(), "light.sqlite"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer s.Close()
|
||||
|
||||
s.AddNode(&graph.Node{
|
||||
ID: "repoA/f.go::Stamped", Kind: graph.KindFunction, Name: "Stamped",
|
||||
FilePath: "repoA/f.go", RepoPrefix: "repoA",
|
||||
Meta: map[string]any{
|
||||
"semantic_type": "string",
|
||||
"semantic_source": "lsp-gopls",
|
||||
"doc": "docs",
|
||||
"complexity": 7, // non-promoted
|
||||
},
|
||||
})
|
||||
s.AddNode(&graph.Node{
|
||||
ID: "repoA/f.go::Unstamped", Kind: graph.KindFunction, Name: "Unstamped",
|
||||
FilePath: "repoA/f.go", RepoPrefix: "repoA",
|
||||
})
|
||||
s.AddNode(&graph.Node{
|
||||
ID: "repoB/g.go::Other", Kind: graph.KindFunction, Name: "Other",
|
||||
FilePath: "repoB/g.go", RepoPrefix: "repoB",
|
||||
})
|
||||
|
||||
var _ graph.LightNodeReader = s // compile-time capability check
|
||||
|
||||
full := s.GetRepoNodes("repoA")
|
||||
light := s.GetRepoNodesLight("repoA")
|
||||
if len(light) != len(full) {
|
||||
t.Fatalf("light returned %d nodes, full returned %d", len(light), len(full))
|
||||
}
|
||||
|
||||
byID := make(map[string]*graph.Node, len(light))
|
||||
for _, n := range light {
|
||||
byID[n.ID] = n
|
||||
}
|
||||
|
||||
stamped, ok := byID["repoA/f.go::Stamped"]
|
||||
if !ok {
|
||||
t.Fatal("light scan missing the stamped node")
|
||||
}
|
||||
assertType[string](t, stamped.Meta, "semantic_type", "string")
|
||||
assertType[string](t, stamped.Meta, "semantic_source", "lsp-gopls")
|
||||
assertType[string](t, stamped.Meta, "doc", "docs")
|
||||
if _, ok := stamped.Meta["complexity"]; ok {
|
||||
t.Errorf("light scan must not surface non-promoted meta, got complexity=%v", stamped.Meta["complexity"])
|
||||
}
|
||||
|
||||
unstamped, ok := byID["repoA/f.go::Unstamped"]
|
||||
if !ok {
|
||||
t.Fatal("light scan missing the unstamped node")
|
||||
}
|
||||
if _, ok := unstamped.Meta["semantic_type"]; ok {
|
||||
t.Error("unstamped node must not carry a semantic_type key")
|
||||
}
|
||||
|
||||
if _, ok := byID["repoB/g.go::Other"]; ok {
|
||||
t.Error("light scan crossed repo_prefix scope")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,433 @@
|
||||
package store_sqlite
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/gob"
|
||||
"encoding/json"
|
||||
"path/filepath"
|
||||
"reflect"
|
||||
"testing"
|
||||
|
||||
"github.com/zzet/gortex/internal/contracts"
|
||||
"github.com/zzet/gortex/internal/graph"
|
||||
)
|
||||
|
||||
// flatRoundTrip encodes via the flat codec only (asserting the fast path was
|
||||
// taken) and decodes back through decodeMeta.
|
||||
func flatRoundTrip(t *testing.T, in map[string]any) map[string]any {
|
||||
t.Helper()
|
||||
b, ok := encodeMetaFast(in)
|
||||
if !ok {
|
||||
t.Fatalf("encodeMetaFast bailed on a modelled map: %#v", in)
|
||||
}
|
||||
if !isFlatMeta(b) {
|
||||
t.Fatalf("encodeMetaFast did not stamp the flat magic: %q", b)
|
||||
}
|
||||
out, err := decodeMetaFast(b)
|
||||
if err != nil {
|
||||
t.Fatalf("decodeMetaFast: %v", err)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// TestFlatCodecEveryValueType round-trips one value of every type the codec
|
||||
// models and asserts exact Go-type fidelity end to end.
|
||||
func TestFlatCodecEveryValueType(t *testing.T) {
|
||||
shape := &contracts.Shape{
|
||||
Kind: "struct",
|
||||
Fields: []contracts.ShapeField{{Name: "id", Type: "int64", Required: true}},
|
||||
Notes: []string{"partial"},
|
||||
}
|
||||
in := map[string]any{
|
||||
"str": "hello",
|
||||
"unicode": "héllo – мир – 世界 – 🚀",
|
||||
"bool_t": true,
|
||||
"bool_f": false,
|
||||
"int": -7,
|
||||
"int64": int64(1700000000),
|
||||
"float": 0.875,
|
||||
"float_int": 2.0, // integral float must stay float64
|
||||
"strs": []string{"a", "b", "c"},
|
||||
"nested": map[string]any{"inner": 5, "rate": 1.5, "deep": map[string]any{"x": int64(9)}},
|
||||
"map_slice": []map[string]any{{"k": "v", "n": 1}, {"k": "w", "n": 2}},
|
||||
"any_slice": []any{"x", 3, true, 4.5},
|
||||
"shape": shape,
|
||||
"nilval": nil,
|
||||
"empty_map": map[string]any{},
|
||||
}
|
||||
got := flatRoundTrip(t, in)
|
||||
|
||||
if !reflect.DeepEqual(got, in) {
|
||||
t.Fatalf("flat round-trip mismatch:\n got: %#v\nwant: %#v", got, in)
|
||||
}
|
||||
|
||||
// Spot-check the load-bearing concrete types explicitly.
|
||||
mustType[string](t, got, "str")
|
||||
mustType[bool](t, got, "bool_t")
|
||||
mustType[int](t, got, "int")
|
||||
mustType[int64](t, got, "int64")
|
||||
mustType[float64](t, got, "float")
|
||||
mustType[float64](t, got, "float_int")
|
||||
mustType[[]string](t, got, "strs")
|
||||
mustType[map[string]any](t, got, "nested")
|
||||
mustType[[]map[string]any](t, got, "map_slice")
|
||||
mustType[[]any](t, got, "any_slice")
|
||||
mustType[*contracts.Shape](t, got, "shape")
|
||||
if got["nilval"] != nil {
|
||||
t.Errorf("nilval: want nil, got %#v", got["nilval"])
|
||||
}
|
||||
}
|
||||
|
||||
// TestFlatCodecLargeValues exercises long keys and values that cross the
|
||||
// single-byte varint boundary (> 127 bytes), proving the length prefixes
|
||||
// round-trip.
|
||||
func TestFlatCodecLargeValues(t *testing.T) {
|
||||
big := string(bytes.Repeat([]byte("x"), 5000))
|
||||
bigKey := string(bytes.Repeat([]byte("k"), 300))
|
||||
in := map[string]any{
|
||||
bigKey: big,
|
||||
"slice": []string{big, "", big},
|
||||
}
|
||||
got := flatRoundTrip(t, in)
|
||||
if !reflect.DeepEqual(got, in) {
|
||||
t.Fatalf("large-value round-trip mismatch")
|
||||
}
|
||||
}
|
||||
|
||||
// TestFlatCodecDeterministic proves the encoding is byte-stable across
|
||||
// encodes (keys are sorted), which matters for any content-hash / dedup.
|
||||
func TestFlatCodecDeterministic(t *testing.T) {
|
||||
in := map[string]any{
|
||||
"z": 1, "a": "x", "m": []string{"p", "q"},
|
||||
"nested": map[string]any{"d": 4, "b": 2, "c": 3},
|
||||
}
|
||||
var prev []byte
|
||||
for i := 0; i < 16; i++ {
|
||||
b, ok := encodeMetaFast(in)
|
||||
if !ok {
|
||||
t.Fatalf("encodeMetaFast bailed")
|
||||
}
|
||||
if prev != nil && !bytes.Equal(prev, b) {
|
||||
t.Fatalf("encoding is not deterministic across encodes")
|
||||
}
|
||||
prev = b
|
||||
}
|
||||
}
|
||||
|
||||
// TestEncodeMetaFallbackToJSON: a value whose type the flat codec does not
|
||||
// model makes encodeMeta fall back to JSON (leading '{'), and decodeMeta
|
||||
// still reads it. No data is dropped.
|
||||
func TestEncodeMetaFallbackToJSON(t *testing.T) {
|
||||
// uint64 is deliberately outside the modelled type set.
|
||||
in := map[string]any{"weird": uint64(42), "name": "keep"}
|
||||
|
||||
if _, ok := encodeMetaFast(in); ok {
|
||||
t.Fatal("encodeMetaFast should bail on an unmodelled value type")
|
||||
}
|
||||
|
||||
b, err := encodeMeta(in)
|
||||
if err != nil {
|
||||
t.Fatalf("encodeMeta: %v", err)
|
||||
}
|
||||
if isFlatMeta(b) {
|
||||
t.Fatalf("encodeMeta should have fallen back to JSON, got a flat blob")
|
||||
}
|
||||
if !isJSONObject(b) {
|
||||
t.Fatalf("encodeMeta fallback did not produce a JSON object: %q", b)
|
||||
}
|
||||
got, err := decodeMeta(b)
|
||||
if err != nil {
|
||||
t.Fatalf("decodeMeta(json fallback): %v", err)
|
||||
}
|
||||
// The JSON fallback widens uint64 -> int (documented, lossy only for the
|
||||
// exotic tail), but the string survives and no row is lost.
|
||||
if got["name"] != "keep" {
|
||||
t.Errorf("name not preserved through JSON fallback: %#v", got["name"])
|
||||
}
|
||||
if _, ok := got["weird"]; !ok {
|
||||
t.Errorf("weird key dropped by JSON fallback")
|
||||
}
|
||||
}
|
||||
|
||||
// TestDecodeLegacyJSON proves rows written by the previous JSON encoder still
|
||||
// decode (routed through metaWire for exact types) after the flat-codec
|
||||
// switch — existing on-disk databases must keep loading.
|
||||
func TestDecodeLegacyJSON(t *testing.T) {
|
||||
orig := map[string]any{
|
||||
"visibility": "private",
|
||||
"complexity": 9,
|
||||
"confidence": 1.0, // integral float — metaWire must keep it float64
|
||||
"path_param_names": []string{"id"},
|
||||
"last_authored": map[string]any{"timestamp": int64(1700000000)},
|
||||
}
|
||||
blob, err := json.Marshal(orig)
|
||||
if err != nil {
|
||||
t.Fatalf("json.Marshal: %v", err)
|
||||
}
|
||||
if isFlatMeta(blob) {
|
||||
t.Fatalf("JSON blob unexpectedly looks like a flat blob")
|
||||
}
|
||||
got, err := decodeMeta(blob)
|
||||
if err != nil {
|
||||
t.Fatalf("decodeMeta(json): %v", err)
|
||||
}
|
||||
mustType[string](t, got, "visibility")
|
||||
mustType[int](t, got, "complexity")
|
||||
mustType[float64](t, got, "confidence")
|
||||
mustType[[]string](t, got, "path_param_names")
|
||||
la, ok := got["last_authored"].(map[string]any)
|
||||
if !ok {
|
||||
t.Fatalf("last_authored: want map[string]any, got %T", got["last_authored"])
|
||||
}
|
||||
mustType[int64](t, la, "timestamp")
|
||||
}
|
||||
|
||||
// TestDecodeMetaFastMalformed: corrupt / truncated flat blobs return an error
|
||||
// rather than panicking — a single bad row must not crash a store scan.
|
||||
func TestDecodeMetaFastMalformed(t *testing.T) {
|
||||
good, ok := encodeMetaFast(map[string]any{"k": "value", "n": 7, "s": []string{"a", "b"}})
|
||||
if !ok {
|
||||
t.Fatalf("encodeMetaFast bailed")
|
||||
}
|
||||
cases := map[string][]byte{
|
||||
"magic only": {metaFlatMagic0, metaFlatVersion},
|
||||
"count then nothing": {metaFlatMagic0, metaFlatVersion, 0x05},
|
||||
"truncated mid-blob": good[:len(good)-3],
|
||||
"unknown value tag": {metaFlatMagic0, metaFlatVersion, 0x01, 0x01, 'k', 0x7E},
|
||||
"giant key length": {metaFlatMagic0, metaFlatVersion, 0x01, 0xFF, 0xFF, 0xFF, 0xFF, 0x0F},
|
||||
}
|
||||
for name, blob := range cases {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
_, err := decodeMeta(blob)
|
||||
if err == nil {
|
||||
t.Errorf("expected an error for %q, got nil", name)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestStoreReloadMetaFidelity is the wired-path proof: persist a node and an
|
||||
// edge with rich Meta through the real store, reopen it (warm restart), and
|
||||
// assert Meta is byte-for-byte type-identical. Also runs PRAGMA
|
||||
// integrity_check.
|
||||
func TestStoreReloadMetaFidelity(t *testing.T) {
|
||||
path := filepath.Join(t.TempDir(), "store.sqlite")
|
||||
|
||||
nodeMeta := map[string]any{
|
||||
"complexity": 7,
|
||||
"loop_depth": 2,
|
||||
"confidence": 0.875,
|
||||
"coverage_pct": 1.0, // integral float
|
||||
"candidate_count": 2,
|
||||
"path_param_names": []string{"id", "org"},
|
||||
"status_codes": []string{"200", "404"},
|
||||
"churn": map[string]any{"commit_count": 12, "churn_rate": 2.0, "last_author": "a@b.c"},
|
||||
"last_authored": map[string]any{"timestamp": int64(1700000000), "email": "x@y.z"},
|
||||
"response_envelope": []map[string]any{{"name": "data", "n": 1}},
|
||||
"shape": &contracts.Shape{
|
||||
Kind: "struct",
|
||||
Fields: []contracts.ShapeField{{Name: "id", Type: "int64", Required: true}},
|
||||
Notes: []string{"partial"},
|
||||
},
|
||||
"unicode_doc": "héllo 世界 🚀",
|
||||
"is_generated": false,
|
||||
}
|
||||
edgeMeta := map[string]any{
|
||||
"candidate_count": 3,
|
||||
"similarity": 0.5,
|
||||
"score": 1.0, // integral float
|
||||
"count": 5,
|
||||
"clone_tokens": 128,
|
||||
"synthesized_by": "grpc",
|
||||
"arg_names": []string{"ctx", "req"},
|
||||
}
|
||||
|
||||
func() {
|
||||
s, err := Open(path)
|
||||
if err != nil {
|
||||
t.Fatalf("Open: %v", err)
|
||||
}
|
||||
defer s.Close()
|
||||
s.AddNode(&graph.Node{ID: "n1", Kind: "function", Name: "Foo", FilePath: "f.go", Meta: cloneMeta(nodeMeta)})
|
||||
s.AddNode(&graph.Node{ID: "n2", Kind: "function", Name: "Bar", FilePath: "f.go"})
|
||||
s.AddEdge(&graph.Edge{From: "n1", To: "n2", Kind: "calls", FilePath: "f.go", Line: 10, Meta: cloneMeta(edgeMeta)})
|
||||
}()
|
||||
|
||||
// Reopen — the warm-restart path that reads every blob back.
|
||||
s, err := Open(path)
|
||||
if err != nil {
|
||||
t.Fatalf("reopen: %v", err)
|
||||
}
|
||||
defer s.Close()
|
||||
|
||||
n := s.GetNode("n1")
|
||||
if n == nil {
|
||||
t.Fatal("GetNode(n1) = nil after reload")
|
||||
}
|
||||
if !reflect.DeepEqual(n.Meta, nodeMeta) {
|
||||
t.Fatalf("node Meta mismatch after reload:\n got: %#v\nwant: %#v", n.Meta, nodeMeta)
|
||||
}
|
||||
|
||||
edges := s.GetOutEdges("n1")
|
||||
var got *graph.Edge
|
||||
for _, e := range edges {
|
||||
if e.To == "n2" && e.Kind == "calls" {
|
||||
got = e
|
||||
break
|
||||
}
|
||||
}
|
||||
if got == nil {
|
||||
t.Fatalf("edge n1->n2 not found after reload (got %d edges)", len(edges))
|
||||
}
|
||||
if !reflect.DeepEqual(got.Meta, edgeMeta) {
|
||||
t.Fatalf("edge Meta mismatch after reload:\n got: %#v\nwant: %#v", got.Meta, edgeMeta)
|
||||
}
|
||||
|
||||
var res string
|
||||
if err := s.db.QueryRow(`PRAGMA integrity_check`).Scan(&res); err != nil {
|
||||
t.Fatalf("integrity_check: %v", err)
|
||||
}
|
||||
if res != "ok" {
|
||||
t.Fatalf("integrity_check = %q, want ok", res)
|
||||
}
|
||||
}
|
||||
|
||||
func cloneMeta(m map[string]any) map[string]any {
|
||||
out := make(map[string]any, len(m))
|
||||
for k, v := range m {
|
||||
out[k] = v
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func mustType[T any](t *testing.T, m map[string]any, key string) {
|
||||
t.Helper()
|
||||
v, ok := m[key]
|
||||
if !ok {
|
||||
t.Errorf("%s: missing from decoded map", key)
|
||||
return
|
||||
}
|
||||
if _, ok := v.(T); !ok {
|
||||
var zero T
|
||||
t.Errorf("%s: want type %T, got %T (value %v)", key, zero, v, v)
|
||||
}
|
||||
}
|
||||
|
||||
// -- benchmarks -----------------------------------------------------------
|
||||
|
||||
var metaSink any
|
||||
|
||||
// benchMetaSample is a representative node/edge meta map restricted to the
|
||||
// types gob auto-registers (scalars + []string), so all three encoders run on
|
||||
// identical input for an apples-to-apples comparison. Shape / nested-map /
|
||||
// map-slice values also ride the flat path (see the round-trip tests); they
|
||||
// are omitted here only because gob refuses unregistered interface types.
|
||||
func benchMetaSample() map[string]any {
|
||||
return map[string]any{
|
||||
"complexity": 7,
|
||||
"loop_depth": 2,
|
||||
"parse_errors": 0,
|
||||
"position": 3,
|
||||
"line": 42,
|
||||
"confidence": 0.875,
|
||||
"coverage_pct": 83.5,
|
||||
"candidate_count": 2,
|
||||
"count": 5,
|
||||
"clone_tokens": 128,
|
||||
"timestamp": int64(1700000000),
|
||||
"path_param_names": []string{"id", "org"},
|
||||
"status_codes": []string{"200", "404"},
|
||||
"signature": "func F(ctx context.Context, x int) (T, error)",
|
||||
"some_plugin_flag": "go_linkname",
|
||||
"is_generated": false,
|
||||
"synthesized_by": "grpc",
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkEncodeMetaGob(b *testing.B) {
|
||||
m := benchMetaSample()
|
||||
b.ReportAllocs()
|
||||
b.ResetTimer()
|
||||
for i := 0; i < b.N; i++ {
|
||||
var buf bytes.Buffer
|
||||
if err := gob.NewEncoder(&buf).Encode(m); err != nil {
|
||||
b.Fatalf("gob encode: %v", err)
|
||||
}
|
||||
metaSink = buf.Bytes()
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkEncodeMetaJSON(b *testing.B) {
|
||||
m := benchMetaSample()
|
||||
b.ReportAllocs()
|
||||
b.ResetTimer()
|
||||
for i := 0; i < b.N; i++ {
|
||||
out, err := json.Marshal(m)
|
||||
if err != nil {
|
||||
b.Fatalf("json marshal: %v", err)
|
||||
}
|
||||
metaSink = out
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkEncodeMetaFlat(b *testing.B) {
|
||||
m := benchMetaSample()
|
||||
b.ReportAllocs()
|
||||
b.ResetTimer()
|
||||
for i := 0; i < b.N; i++ {
|
||||
out, ok := encodeMetaFast(m)
|
||||
if !ok {
|
||||
b.Fatal("encodeMetaFast bailed")
|
||||
}
|
||||
metaSink = out
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkDecodeMetaGob(b *testing.B) {
|
||||
var buf bytes.Buffer
|
||||
if err := gob.NewEncoder(&buf).Encode(benchMetaSample()); err != nil {
|
||||
b.Fatalf("gob encode: %v", err)
|
||||
}
|
||||
blob := buf.Bytes()
|
||||
b.ReportAllocs()
|
||||
b.ResetTimer()
|
||||
for i := 0; i < b.N; i++ {
|
||||
m, err := decodeMeta(blob)
|
||||
if err != nil {
|
||||
b.Fatalf("decodeMeta: %v", err)
|
||||
}
|
||||
metaSink = m
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkDecodeMetaJSON(b *testing.B) {
|
||||
blob, err := json.Marshal(benchMetaSample())
|
||||
if err != nil {
|
||||
b.Fatalf("json marshal: %v", err)
|
||||
}
|
||||
b.ReportAllocs()
|
||||
b.ResetTimer()
|
||||
for i := 0; i < b.N; i++ {
|
||||
m, err := decodeMeta(blob)
|
||||
if err != nil {
|
||||
b.Fatalf("decodeMeta: %v", err)
|
||||
}
|
||||
metaSink = m
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkDecodeMetaFlat(b *testing.B) {
|
||||
blob, ok := encodeMetaFast(benchMetaSample())
|
||||
if !ok {
|
||||
b.Fatal("encodeMetaFast bailed")
|
||||
}
|
||||
b.ReportAllocs()
|
||||
b.ResetTimer()
|
||||
for i := 0; i < b.N; i++ {
|
||||
m, err := decodeMeta(blob)
|
||||
if err != nil {
|
||||
b.Fatalf("decodeMeta: %v", err)
|
||||
}
|
||||
metaSink = m
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,176 @@
|
||||
package store_sqlite
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/gob"
|
||||
"reflect"
|
||||
"testing"
|
||||
|
||||
"github.com/zzet/gortex/internal/contracts"
|
||||
)
|
||||
|
||||
// roundTrip encodes Meta with the flat codec and decodes it back, the
|
||||
// persist->reload path every reader sees after a daemon restart / store
|
||||
// hydration.
|
||||
func roundTrip(t *testing.T, in map[string]any) map[string]any {
|
||||
t.Helper()
|
||||
b, err := encodeMeta(in)
|
||||
if err != nil {
|
||||
t.Fatalf("encodeMeta: %v", err)
|
||||
}
|
||||
if !isFlatMeta(b) {
|
||||
t.Fatalf("encodeMeta did not produce a flat-codec blob: %q", b)
|
||||
}
|
||||
out, err := decodeMeta(b)
|
||||
if err != nil {
|
||||
t.Fatalf("decodeMeta: %v", err)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// TestMetaRoundTripExactTypes is the fidelity canary: every key the audit
|
||||
// found read with a raw type-assertion must survive a JSON round-trip with
|
||||
// its exact Go type, or the corresponding reader silently breaks.
|
||||
func TestMetaRoundTripExactTypes(t *testing.T) {
|
||||
shape := &contracts.Shape{
|
||||
Kind: "struct",
|
||||
Fields: []contracts.ShapeField{{Name: "id", Type: "int64", Required: true}},
|
||||
Notes: []string{"partial"},
|
||||
}
|
||||
node := map[string]any{
|
||||
"signature": "func F(x int) error",
|
||||
"visibility": "public",
|
||||
"doc": "F does a thing.",
|
||||
"external": true,
|
||||
"complexity": 7,
|
||||
"loop_depth": 2,
|
||||
"parse_errors": 0,
|
||||
"position": 3,
|
||||
"line": 42,
|
||||
"confidence": 1.0, // integral float — must stay float64
|
||||
"coverage_pct": 83.5,
|
||||
"shape": shape,
|
||||
"response_envelope": []map[string]any{{"name": "data", "type": "User"}},
|
||||
"path_param_names": []string{"id", "org"},
|
||||
"query_params": []string{"limit"},
|
||||
"status_codes": []string{"200", "404"},
|
||||
"churn": map[string]any{"commit_count": 12, "age_days": 365, "churn_rate": 2.0, "last_author": "a@b.c"},
|
||||
"coverage": map[string]any{"num_stmt": 40, "hit": 33},
|
||||
"last_authored": map[string]any{"timestamp": int64(1700000000), "email": "x@y.z"},
|
||||
"some_plugin_flag": "go_linkname", // Extra tail (string)
|
||||
"is_generated": false, // Extra tail (bool)
|
||||
}
|
||||
got := roundTrip(t, node)
|
||||
|
||||
assertType[int](t, got, "complexity", 7)
|
||||
assertType[int](t, got, "loop_depth", 2)
|
||||
assertType[int](t, got, "parse_errors", 0)
|
||||
assertType[int](t, got, "position", 3)
|
||||
assertType[int](t, got, "line", 42)
|
||||
assertType[float64](t, got, "confidence", 1.0)
|
||||
assertType[float64](t, got, "coverage_pct", 83.5)
|
||||
assertType[string](t, got, "signature", "func F(x int) error")
|
||||
assertType[string](t, got, "visibility", "public")
|
||||
assertType[bool](t, got, "external", true)
|
||||
assertType[string](t, got, "some_plugin_flag", "go_linkname")
|
||||
assertType[bool](t, got, "is_generated", false)
|
||||
|
||||
// Shape must rebuild as *contracts.Shape, not map[string]any.
|
||||
gotShape, ok := got["shape"].(*contracts.Shape)
|
||||
if !ok {
|
||||
t.Fatalf("shape: want *contracts.Shape, got %T", got["shape"])
|
||||
}
|
||||
if !reflect.DeepEqual(gotShape, shape) {
|
||||
t.Errorf("shape mismatch: %+v vs %+v", gotShape, shape)
|
||||
}
|
||||
|
||||
// response_envelope must be []map[string]any, not []any.
|
||||
if _, ok := got["response_envelope"].([]map[string]any); !ok {
|
||||
t.Errorf("response_envelope: want []map[string]any, got %T", got["response_envelope"])
|
||||
}
|
||||
// []string keys.
|
||||
for _, k := range []string{"path_param_names", "query_params", "status_codes"} {
|
||||
if _, ok := got[k].([]string); !ok {
|
||||
t.Errorf("%s: want []string, got %T", k, got[k])
|
||||
}
|
||||
}
|
||||
|
||||
// Nested map children keep exact types.
|
||||
churn := got["churn"].(map[string]any)
|
||||
assertType[int](t, churn, "commit_count", 12)
|
||||
assertType[int](t, churn, "age_days", 365)
|
||||
assertType[float64](t, churn, "churn_rate", 2.0) // integral float, nested
|
||||
assertType[string](t, churn, "last_author", "a@b.c")
|
||||
cov := got["coverage"].(map[string]any)
|
||||
assertType[int](t, cov, "num_stmt", 40)
|
||||
assertType[int](t, cov, "hit", 33)
|
||||
la := got["last_authored"].(map[string]any)
|
||||
assertType[int64](t, la, "timestamp", int64(1700000000))
|
||||
}
|
||||
|
||||
func TestEdgeMetaRoundTripExactTypes(t *testing.T) {
|
||||
edge := map[string]any{
|
||||
"candidate_count": 2,
|
||||
"similarity": 0.875,
|
||||
"score": 1.0, // integral float — must stay float64
|
||||
"count": 5,
|
||||
"clone_tokens": 128,
|
||||
"synthesized_by": "grpc", // Extra tail
|
||||
}
|
||||
got := roundTrip(t, edge)
|
||||
assertType[int](t, got, "candidate_count", 2)
|
||||
assertType[float64](t, got, "similarity", 0.875)
|
||||
assertType[float64](t, got, "score", 1.0)
|
||||
assertType[int](t, got, "count", 5)
|
||||
assertType[int](t, got, "clone_tokens", 128)
|
||||
assertType[string](t, got, "synthesized_by", "grpc")
|
||||
}
|
||||
|
||||
// TestDecodeLegacyGob proves existing on-disk gob blobs still decode.
|
||||
func TestDecodeLegacyGob(t *testing.T) {
|
||||
orig := map[string]any{"visibility": "private", "complexity": 9, "confidence": 1.0}
|
||||
var buf bytes.Buffer
|
||||
if err := gob.NewEncoder(&buf).Encode(orig); err != nil {
|
||||
t.Fatalf("gob encode: %v", err)
|
||||
}
|
||||
got, err := decodeMeta(buf.Bytes())
|
||||
if err != nil {
|
||||
t.Fatalf("decodeMeta(gob): %v", err)
|
||||
}
|
||||
// gob preserves exact types natively.
|
||||
assertType[string](t, got, "visibility", "private")
|
||||
assertType[int](t, got, "complexity", 9)
|
||||
assertType[float64](t, got, "confidence", 1.0)
|
||||
}
|
||||
|
||||
func TestEncodeMetaEmpty(t *testing.T) {
|
||||
b, err := encodeMeta(nil)
|
||||
if err != nil || b != nil {
|
||||
t.Fatalf("encodeMeta(nil) = %q, %v; want nil, nil", b, err)
|
||||
}
|
||||
b, err = encodeMeta(map[string]any{})
|
||||
if err != nil || b != nil {
|
||||
t.Fatalf("encodeMeta(empty) = %q, %v; want nil, nil", b, err)
|
||||
}
|
||||
m, err := decodeMeta(nil)
|
||||
if err != nil || m != nil {
|
||||
t.Fatalf("decodeMeta(nil) = %v, %v; want nil, nil", m, err)
|
||||
}
|
||||
}
|
||||
|
||||
func assertType[T comparable](t *testing.T, m map[string]any, key string, want T) {
|
||||
t.Helper()
|
||||
v, ok := m[key]
|
||||
if !ok {
|
||||
t.Errorf("%s: missing from decoded map", key)
|
||||
return
|
||||
}
|
||||
got, ok := v.(T)
|
||||
if !ok {
|
||||
t.Errorf("%s: want type %T, got %T (value %v)", key, want, v, v)
|
||||
return
|
||||
}
|
||||
if got != want {
|
||||
t.Errorf("%s: want %v, got %v", key, want, got)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,252 @@
|
||||
package store_sqlite
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/zzet/gortex/internal/graph"
|
||||
)
|
||||
|
||||
// TestPromotedColumns_RoundTrip verifies the promoted keys land in their
|
||||
// columns, are stripped from the JSON blob, and restore into Meta with
|
||||
// exact types — while non-promoted keys stay in the blob.
|
||||
func TestPromotedColumns_RoundTrip(t *testing.T) {
|
||||
s, err := Open(filepath.Join(t.TempDir(), "p.sqlite"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer s.Close()
|
||||
|
||||
s.AddNode(&graph.Node{
|
||||
ID: "f.go::F", Kind: graph.KindFunction, Name: "F", FilePath: "f.go",
|
||||
Meta: map[string]any{
|
||||
"signature": "func F()",
|
||||
"visibility": "public",
|
||||
"doc": "F docs",
|
||||
"external": true,
|
||||
"complexity": 5, // non-promoted — must stay in the blob
|
||||
},
|
||||
})
|
||||
|
||||
n := s.GetNode("f.go::F")
|
||||
if n == nil {
|
||||
t.Fatal("GetNode returned nil")
|
||||
}
|
||||
assertType[string](t, n.Meta, "signature", "func F()")
|
||||
assertType[string](t, n.Meta, "visibility", "public")
|
||||
assertType[string](t, n.Meta, "doc", "F docs")
|
||||
assertType[bool](t, n.Meta, "external", true)
|
||||
assertType[int](t, n.Meta, "complexity", 5)
|
||||
|
||||
var sig, vis, doc sql.NullString
|
||||
var ext sql.NullBool
|
||||
var blob []byte
|
||||
row := s.db.QueryRow(`SELECT signature, visibility, doc, external, meta FROM nodes WHERE id=?`, "f.go::F")
|
||||
if err := row.Scan(&sig, &vis, &doc, &ext, &blob); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !sig.Valid || sig.String != "func F()" {
|
||||
t.Errorf("signature column = %+v", sig)
|
||||
}
|
||||
if !ext.Valid || !ext.Bool {
|
||||
t.Errorf("external column = %+v", ext)
|
||||
}
|
||||
blobStr := string(blob)
|
||||
for _, k := range []string{"signature", "visibility", "external"} {
|
||||
if strings.Contains(blobStr, k) {
|
||||
t.Errorf("blob still contains promoted key %q: %s", k, blobStr)
|
||||
}
|
||||
}
|
||||
if !strings.Contains(blobStr, "complexity") {
|
||||
t.Errorf("blob missing non-promoted key complexity: %s", blobStr)
|
||||
}
|
||||
}
|
||||
|
||||
// TestPromotedColumns_NewColumns verifies the added promoted meta columns
|
||||
// (is_async / is_static / is_abstract / is_exported / return_type / updated_at)
|
||||
// and the struct-field columns (start_column / end_column) round-trip through
|
||||
// their typed columns, and that a SQL filter on is_async resolves WITHOUT
|
||||
// decoding the meta blob — the indexable-column acceptance.
|
||||
func TestPromotedColumns_NewColumns(t *testing.T) {
|
||||
s, err := Open(filepath.Join(t.TempDir(), "p.sqlite"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer s.Close()
|
||||
|
||||
s.AddNode(&graph.Node{
|
||||
ID: "f.go::Async", Kind: graph.KindFunction, Name: "Async", FilePath: "f.go",
|
||||
StartLine: 10, EndLine: 20, StartColumn: 4, EndColumn: 1,
|
||||
Meta: map[string]any{
|
||||
"is_async": true,
|
||||
"is_static": false,
|
||||
"is_abstract": true,
|
||||
"is_exported": true,
|
||||
"return_type": "error",
|
||||
"updated_at": int64(1700000000),
|
||||
"complexity": 3, // non-promoted — stays in the blob
|
||||
},
|
||||
})
|
||||
// A second, non-async node to prove the filter is selective.
|
||||
s.AddNode(&graph.Node{
|
||||
ID: "f.go::Sync", Kind: graph.KindFunction, Name: "Sync", FilePath: "f.go",
|
||||
Meta: map[string]any{"is_async": false},
|
||||
})
|
||||
|
||||
// Read-back restores every promoted key into Meta with its exact type,
|
||||
// and the struct columns into the Node fields.
|
||||
n := s.GetNode("f.go::Async")
|
||||
if n == nil {
|
||||
t.Fatal("GetNode returned nil")
|
||||
}
|
||||
assertType[bool](t, n.Meta, "is_async", true)
|
||||
assertType[bool](t, n.Meta, "is_static", false)
|
||||
assertType[bool](t, n.Meta, "is_abstract", true)
|
||||
assertType[bool](t, n.Meta, "is_exported", true)
|
||||
assertType[string](t, n.Meta, "return_type", "error")
|
||||
assertType[int64](t, n.Meta, "updated_at", int64(1700000000))
|
||||
assertType[int](t, n.Meta, "complexity", 3)
|
||||
if n.StartColumn != 4 || n.EndColumn != 1 {
|
||||
t.Errorf("column offsets = (%d,%d), want (4,1)", n.StartColumn, n.EndColumn)
|
||||
}
|
||||
|
||||
// The promoted keys are stripped from the JSON blob; complexity is not.
|
||||
var blob []byte
|
||||
if err := s.db.QueryRow(`SELECT meta FROM nodes WHERE id=?`, "f.go::Async").Scan(&blob); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, k := range []string{"is_async", "is_abstract", "return_type", "updated_at"} {
|
||||
if strings.Contains(string(blob), k) {
|
||||
t.Errorf("blob still contains promoted key %q: %s", k, blob)
|
||||
}
|
||||
}
|
||||
if !strings.Contains(string(blob), "complexity") {
|
||||
t.Errorf("blob missing non-promoted key complexity: %s", blob)
|
||||
}
|
||||
|
||||
// Acceptance: a SQL filter on the typed column resolves the node without
|
||||
// touching the meta blob (only id is selected).
|
||||
var id string
|
||||
var startCol, endCol int
|
||||
if err := s.db.QueryRow(
|
||||
`SELECT id, start_column, end_column FROM nodes WHERE is_async = 1`,
|
||||
).Scan(&id, &startCol, &endCol); err != nil {
|
||||
t.Fatalf("is_async column filter failed: %v", err)
|
||||
}
|
||||
if id != "f.go::Async" || startCol != 4 || endCol != 1 {
|
||||
t.Errorf("filter result = (%q,%d,%d), want (f.go::Async,4,1)", id, startCol, endCol)
|
||||
}
|
||||
}
|
||||
|
||||
// TestPromotedColumns_SemanticType verifies semantic_type/semantic_source —
|
||||
// promoted alongside signature/visibility/etc. so enrichment can query the
|
||||
// unstamped subset by column instead of decoding every node's meta blob —
|
||||
// round-trip through their columns and out of the JSON blob.
|
||||
func TestPromotedColumns_SemanticType(t *testing.T) {
|
||||
s, err := Open(filepath.Join(t.TempDir(), "p.sqlite"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer s.Close()
|
||||
|
||||
s.AddNode(&graph.Node{
|
||||
ID: "f.go::T", Kind: graph.KindFunction, Name: "T", FilePath: "f.go",
|
||||
Meta: map[string]any{
|
||||
"semantic_type": "string",
|
||||
"semantic_source": "lsp-gopls",
|
||||
"complexity": 5, // non-promoted — must stay in the blob
|
||||
},
|
||||
})
|
||||
|
||||
n := s.GetNode("f.go::T")
|
||||
if n == nil {
|
||||
t.Fatal("GetNode returned nil")
|
||||
}
|
||||
assertType[string](t, n.Meta, "semantic_type", "string")
|
||||
assertType[string](t, n.Meta, "semantic_source", "lsp-gopls")
|
||||
assertType[int](t, n.Meta, "complexity", 5)
|
||||
|
||||
var st, ss sql.NullString
|
||||
var blob []byte
|
||||
row := s.db.QueryRow(`SELECT semantic_type, semantic_source, meta FROM nodes WHERE id=?`, "f.go::T")
|
||||
if err := row.Scan(&st, &ss, &blob); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !st.Valid || st.String != "string" {
|
||||
t.Errorf("semantic_type column = %+v", st)
|
||||
}
|
||||
if !ss.Valid || ss.String != "lsp-gopls" {
|
||||
t.Errorf("semantic_source column = %+v", ss)
|
||||
}
|
||||
blobStr := string(blob)
|
||||
for _, k := range []string{"semantic_type", "semantic_source"} {
|
||||
if strings.Contains(blobStr, k) {
|
||||
t.Errorf("blob still contains promoted key %q: %s", k, blobStr)
|
||||
}
|
||||
}
|
||||
if !strings.Contains(blobStr, "complexity") {
|
||||
t.Errorf("blob missing non-promoted key complexity: %s", blobStr)
|
||||
}
|
||||
}
|
||||
|
||||
// TestPromotedColumns_ExternalFalse guards the NULL-vs-false distinction:
|
||||
// a stored false must round-trip as false, not vanish.
|
||||
func TestPromotedColumns_ExternalFalse(t *testing.T) {
|
||||
s, err := Open(filepath.Join(t.TempDir(), "p.sqlite"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer s.Close()
|
||||
s.AddNode(&graph.Node{
|
||||
ID: "x", Kind: graph.KindFunction, Name: "x", FilePath: "x.go",
|
||||
Meta: map[string]any{"external": false},
|
||||
})
|
||||
n := s.GetNode("x")
|
||||
if n == nil {
|
||||
t.Fatal("nil")
|
||||
}
|
||||
v, ok := n.Meta["external"].(bool)
|
||||
if !ok || v != false {
|
||||
t.Errorf("external false: got %v (%T)", n.Meta["external"], n.Meta["external"])
|
||||
}
|
||||
}
|
||||
|
||||
// TestPromotedColumns_Migration verifies ensureNodeColumns adds the
|
||||
// promoted columns to a database created with the pre-promotion schema.
|
||||
func TestPromotedColumns_Migration(t *testing.T) {
|
||||
path := filepath.Join(t.TempDir(), "old.sqlite")
|
||||
raw, err := sql.Open("sqlite", path)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
_, err = raw.Exec(`CREATE TABLE nodes (
|
||||
id TEXT PRIMARY KEY, kind TEXT NOT NULL, name TEXT NOT NULL,
|
||||
qual_name TEXT NOT NULL DEFAULT '', file_path TEXT NOT NULL,
|
||||
start_line INTEGER NOT NULL DEFAULT 0, end_line INTEGER NOT NULL DEFAULT 0,
|
||||
language TEXT NOT NULL DEFAULT '', repo_prefix TEXT NOT NULL DEFAULT '',
|
||||
workspace_id TEXT NOT NULL DEFAULT '', project_id TEXT NOT NULL DEFAULT '',
|
||||
meta BLOB
|
||||
) WITHOUT ROWID`)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
_ = raw.Close()
|
||||
|
||||
s, err := Open(path)
|
||||
if err != nil {
|
||||
t.Fatalf("Open old-schema db: %v", err)
|
||||
}
|
||||
defer s.Close()
|
||||
s.AddNode(&graph.Node{
|
||||
ID: "m", Kind: graph.KindFunction, Name: "m", FilePath: "m.go",
|
||||
Meta: map[string]any{"signature": "sig", "external": true},
|
||||
})
|
||||
n := s.GetNode("m")
|
||||
if n == nil {
|
||||
t.Fatal("nil after migration")
|
||||
}
|
||||
assertType[string](t, n.Meta, "signature", "sig")
|
||||
assertType[bool](t, n.Meta, "external", true)
|
||||
}
|
||||
@@ -0,0 +1,45 @@
|
||||
package store_sqlite_test
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"github.com/zzet/gortex/internal/graph"
|
||||
)
|
||||
|
||||
// TestGetOutEdgesLight_SkipsMetaKeepsEndpoints proves the light out-edge
|
||||
// fetch returns the same endpoints/kind/line as GetOutEdges while leaving
|
||||
// Meta nil — it must never pay the per-edge meta JSON decode. This is the
|
||||
// fetch findCallTarget uses on the dataflow hot path.
|
||||
func TestGetOutEdgesLight_SkipsMetaKeepsEndpoints(t *testing.T) {
|
||||
s := openTestStore(t)
|
||||
|
||||
want := &graph.Edge{
|
||||
From: "pkg/x.go::Caller",
|
||||
To: "pkg/x.go::Callee",
|
||||
Kind: graph.EdgeCalls,
|
||||
FilePath: "pkg/x.go",
|
||||
Line: 42,
|
||||
Meta: map[string]any{"call_line": 42, "callee_target": "unresolved::Callee"},
|
||||
}
|
||||
s.AddEdge(want)
|
||||
|
||||
full := s.GetOutEdges("pkg/x.go::Caller")
|
||||
require.Len(t, full, 1)
|
||||
require.NotNil(t, full[0].Meta, "GetOutEdges must decode Meta")
|
||||
assert.Equal(t, "unresolved::Callee", full[0].Meta["callee_target"])
|
||||
|
||||
light := s.GetOutEdgesLight("pkg/x.go::Caller")
|
||||
require.Len(t, light, 1)
|
||||
assert.Equal(t, full[0].From, light[0].From)
|
||||
assert.Equal(t, full[0].To, light[0].To)
|
||||
assert.Equal(t, full[0].Kind, light[0].Kind)
|
||||
assert.Equal(t, full[0].Line, light[0].Line)
|
||||
assert.Equal(t, full[0].FilePath, light[0].FilePath)
|
||||
assert.Nil(t, light[0].Meta, "light fetch must not decode the meta blob")
|
||||
|
||||
// A node with no out-edges returns nothing on both paths.
|
||||
assert.Empty(t, s.GetOutEdgesLight("pkg/x.go::Callee"))
|
||||
}
|
||||
@@ -0,0 +1,58 @@
|
||||
package store_sqlite
|
||||
|
||||
import (
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"github.com/zzet/gortex/internal/graph"
|
||||
)
|
||||
|
||||
// TestProxyNodes_NeverReachDisk validates that cross-daemon proxy-edge
|
||||
// nodes (and their edges) are dropped at the single durable write boundary
|
||||
// (AddNode / AddBatch), so a warm restart over the store never sees them,
|
||||
// while every real node round-trips intact.
|
||||
func TestProxyNodes_NeverReachDisk(t *testing.T) {
|
||||
path := filepath.Join(t.TempDir(), "store.sqlite")
|
||||
s, err := Open(path)
|
||||
if err != nil {
|
||||
t.Fatalf("open: %v", err)
|
||||
}
|
||||
|
||||
real := &graph.Node{ID: "local/a.go::Foo", Kind: graph.KindFunction, Name: "Foo"}
|
||||
proxy := &graph.Node{
|
||||
ID: graph.ProxyNodeID("remoteB", "rb/x.go::Bar"),
|
||||
Kind: graph.KindFunction, Name: "Bar",
|
||||
Origin: "remote:remoteB", Stub: true,
|
||||
}
|
||||
proxyEdge := &graph.Edge{From: real.ID, To: proxy.ID, Kind: graph.EdgeCalls}
|
||||
|
||||
// Mix proxy + real through AddBatch...
|
||||
s.AddBatch([]*graph.Node{real, proxy}, []*graph.Edge{proxyEdge})
|
||||
// ...and a proxy through the per-node path.
|
||||
s.AddNode(&graph.Node{
|
||||
ID: graph.ProxyNodeID("remoteC", "rc/y.go::Baz"),
|
||||
Kind: graph.KindFunction, Name: "Baz",
|
||||
Origin: "remote:remoteC", Stub: true,
|
||||
})
|
||||
_ = s.Close()
|
||||
|
||||
// Reopen — a warm restart sees the durable store only.
|
||||
s2, err := Open(path)
|
||||
if err != nil {
|
||||
t.Fatalf("reopen: %v", err)
|
||||
}
|
||||
defer s2.Close()
|
||||
|
||||
if s2.GetNode(proxy.ID) != nil {
|
||||
t.Error("proxy node must not be persisted")
|
||||
}
|
||||
if s2.GetNode(graph.ProxyNodeID("remoteC", "rc/y.go::Baz")) != nil {
|
||||
t.Error("proxy node added via AddNode must not be persisted")
|
||||
}
|
||||
if s2.GetNode(real.ID) == nil {
|
||||
t.Error("the real node must round-trip intact")
|
||||
}
|
||||
if outs := s2.GetOutEdges(real.ID); len(outs) != 0 {
|
||||
t.Errorf("the edge to a proxy node must not be persisted; got %d", len(outs))
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,74 @@
|
||||
package store_sqlite
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"fmt"
|
||||
"os"
|
||||
|
||||
"github.com/zzet/gortex/internal/graph"
|
||||
)
|
||||
|
||||
// ReadRepoIndexStates opens the SQLite store at path read-only and returns
|
||||
// every repo_index_state freshness row keyed by repo_prefix.
|
||||
//
|
||||
// It is a deliberately lightweight side door for read-only callers (notably
|
||||
// `gortex repos`) that must inspect index freshness WITHOUT going through
|
||||
// Open — which runs schema migrations, alters columns, starts a checkpoint
|
||||
// goroutine, and (on a version mismatch) can refuse to open or rebuild the
|
||||
// file. None of that is appropriate for a status command that may run while
|
||||
// a daemon holds the same store open.
|
||||
//
|
||||
// The connection is query-only and inherits the database's existing journal
|
||||
// mode, so it reads safely alongside a running daemon (WAL permits concurrent
|
||||
// readers). A missing store file, or a database that predates the
|
||||
// repo_index_state table, both yield an empty map and a nil error — that is
|
||||
// "nothing recorded yet", not a failure, so the caller can fall back to other
|
||||
// freshness sources rather than surfacing an error to the user.
|
||||
func ReadRepoIndexStates(path string) (map[string]graph.RepoIndexState, error) {
|
||||
if _, err := os.Stat(path); err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
return map[string]graph.RepoIndexState{}, nil
|
||||
}
|
||||
return nil, fmt.Errorf("stat sqlite store %q: %w", path, err)
|
||||
}
|
||||
|
||||
// query_only blocks accidental writes; busy_timeout keeps a brief read
|
||||
// from erroring out if the daemon happens to hold the write lock for a
|
||||
// moment. We deliberately do NOT set journal_mode — forcing it could try
|
||||
// to switch the live database's mode; inheriting the on-disk WAL mode is
|
||||
// exactly what a concurrent reader wants.
|
||||
dsn := path + "?_pragma=busy_timeout(2000)&_pragma=query_only(1)"
|
||||
db, err := sql.Open("sqlite", dsn)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("open sqlite store %q: %w", path, err)
|
||||
}
|
||||
defer db.Close()
|
||||
db.SetMaxOpenConns(1)
|
||||
|
||||
rows, err := db.Query(`
|
||||
SELECT repo_prefix, indexed_sha, dirty, indexed_at, workspace_fp, node_count, edge_count, extractor_versions
|
||||
FROM repo_index_state`)
|
||||
if err != nil {
|
||||
// A store written before the repo_index_state table existed (or any
|
||||
// other read error) is treated as "no freshness recorded yet" — a
|
||||
// status command must never hard-fail on a degraded cache.
|
||||
return map[string]graph.RepoIndexState{}, nil
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
out := map[string]graph.RepoIndexState{}
|
||||
for rows.Next() {
|
||||
var st graph.RepoIndexState
|
||||
var dirty int
|
||||
if err := rows.Scan(&st.RepoPrefix, &st.IndexedSHA, &dirty, &st.IndexedAt,
|
||||
&st.WorkspaceFP, &st.NodeCount, &st.EdgeCount, &st.ExtractorVersions); err != nil {
|
||||
return nil, fmt.Errorf("scan repo_index_state: %w", err)
|
||||
}
|
||||
st.Dirty = dirty != 0
|
||||
out[st.RepoPrefix] = st
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return nil, fmt.Errorf("iterate repo_index_state: %w", err)
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
@@ -0,0 +1,78 @@
|
||||
package store_sqlite_test
|
||||
|
||||
import (
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"github.com/zzet/gortex/internal/graph"
|
||||
"github.com/zzet/gortex/internal/graph/store_sqlite"
|
||||
)
|
||||
|
||||
// A store file that does not exist yet reads back as an empty map, not an
|
||||
// error — "nothing indexed", so callers fall back to other sources.
|
||||
func TestReadRepoIndexStates_MissingFile(t *testing.T) {
|
||||
got, err := store_sqlite.ReadRepoIndexStates(filepath.Join(t.TempDir(), "absent.sqlite"))
|
||||
require.NoError(t, err)
|
||||
require.Empty(t, got)
|
||||
}
|
||||
|
||||
// The read-only reader returns exactly the rows SetRepoIndexState wrote,
|
||||
// keyed by repo prefix, including the empty (lone-repo) prefix.
|
||||
func TestReadRepoIndexStates_RoundTrip(t *testing.T) {
|
||||
path := filepath.Join(t.TempDir(), "is.sqlite")
|
||||
|
||||
s, err := store_sqlite.Open(path)
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, s.SetRepoIndexState(graph.RepoIndexState{
|
||||
RepoPrefix: "alpha", IndexedSHA: "aaa111", Dirty: false, IndexedAt: 1700000000,
|
||||
WorkspaceFP: "fp-a", NodeCount: 10, EdgeCount: 20, ExtractorVersions: `{"go":1}`,
|
||||
}))
|
||||
require.NoError(t, s.SetRepoIndexState(graph.RepoIndexState{
|
||||
RepoPrefix: "", IndexedSHA: "bbb222", Dirty: true, IndexedAt: 1700000001,
|
||||
}))
|
||||
require.NoError(t, s.Close())
|
||||
|
||||
got, err := store_sqlite.ReadRepoIndexStates(path)
|
||||
require.NoError(t, err)
|
||||
require.Len(t, got, 2)
|
||||
|
||||
alpha, ok := got["alpha"]
|
||||
require.True(t, ok)
|
||||
require.Equal(t, "aaa111", alpha.IndexedSHA)
|
||||
require.False(t, alpha.Dirty)
|
||||
require.EqualValues(t, 1700000000, alpha.IndexedAt)
|
||||
require.Equal(t, 10, alpha.NodeCount)
|
||||
|
||||
lone, ok := got[""]
|
||||
require.True(t, ok)
|
||||
require.Equal(t, "bbb222", lone.IndexedSHA)
|
||||
require.True(t, lone.Dirty)
|
||||
}
|
||||
|
||||
// Reading is non-destructive and repeatable while the underlying store is
|
||||
// reopened for writing — the read path must never lock the writer out.
|
||||
func TestReadRepoIndexStates_ConcurrentWithOpenStore(t *testing.T) {
|
||||
path := filepath.Join(t.TempDir(), "is.sqlite")
|
||||
|
||||
s, err := store_sqlite.Open(path)
|
||||
require.NoError(t, err)
|
||||
t.Cleanup(func() { _ = s.Close() })
|
||||
require.NoError(t, s.SetRepoIndexState(graph.RepoIndexState{
|
||||
RepoPrefix: "live", IndexedSHA: "c0ffee", IndexedAt: 1700000002,
|
||||
}))
|
||||
|
||||
// Read while the writer store is still open (WAL allows concurrent readers).
|
||||
got, err := store_sqlite.ReadRepoIndexStates(path)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, "c0ffee", got["live"].IndexedSHA)
|
||||
|
||||
// A subsequent write is still possible — the reader did not wedge the writer.
|
||||
require.NoError(t, s.SetRepoIndexState(graph.RepoIndexState{
|
||||
RepoPrefix: "live", IndexedSHA: "feedface", IndexedAt: 1700000003,
|
||||
}))
|
||||
got, err = store_sqlite.ReadRepoIndexStates(path)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, "feedface", got["live"].IndexedSHA)
|
||||
}
|
||||
@@ -0,0 +1,518 @@
|
||||
package store_sqlite
|
||||
|
||||
import "database/sql"
|
||||
|
||||
// isUnresolvedColumnDDL is the edges.is_unresolved generated column: a
|
||||
// VIRTUAL, indexed boolean mirroring graph.IsUnresolvedTarget's two shapes
|
||||
// (the bare `unresolved::Name` prefix and the multi-repo COPY-rewrite
|
||||
// `<repoPrefix>::unresolved::Name` infix), computed by SQLite itself from
|
||||
// to_id — no Go call site has to remember to keep it in sync. VIRTUAL, not
|
||||
// STORED: SQLite refuses `ALTER TABLE ADD COLUMN ... STORED` on a non-empty
|
||||
// table ("cannot add a STORED column"), which every real installed store is.
|
||||
// VIRTUAL has no such restriction and is just as fast here — the read path
|
||||
// always goes through the index below, and an index always stores its own
|
||||
// materialised key values regardless of whether the underlying column is
|
||||
// virtual or stored. Added via ensureEdgeColumns (ALTER TABLE) rather than
|
||||
// baked into schemaSQL's CREATE TABLE so one code path handles both a fresh
|
||||
// DB (column missing right after CREATE TABLE) and an existing one (column
|
||||
// missing from before this was introduced) identically — see
|
||||
// ensureNodeColumns for the same pattern on the nodes table.
|
||||
//
|
||||
// Measured on a real 26-repo store (2.57M edges, 847,684 unresolved, ~33%
|
||||
// selectivity): replacing the OR'd `to_id` range/LIKE query with
|
||||
// `is_unresolved = 1` cut EdgesWithUnresolvedTarget from 7.96s to 2.95s
|
||||
// (2.7x). The prior approach of splitting the OR into two to_id-based
|
||||
// queries (one indexed range, one LIKE) was WORSE (13.49s) despite a
|
||||
// better-looking EXPLAIN QUERY PLAN: at ~33% selectivity the to_id index's
|
||||
// matching rows are ordered by string value, so the mandatory per-row
|
||||
// bookmark lookup back into the main table is effectively random I/O. The
|
||||
// boolean column's matching rows are all rowid-tie-broken (identical index
|
||||
// key), so its bookmark lookups land in ascending rowid order — sequential,
|
||||
// not random. Same "SEARCH ... USING INDEX" in EXPLAIN QUERY PLAN either way;
|
||||
// only real measurement told them apart.
|
||||
const isUnresolvedColumnDDL = `is_unresolved INTEGER GENERATED ALWAYS AS (
|
||||
CASE WHEN (to_id >= 'unresolved::' AND to_id < 'unresolved:;') OR to_id LIKE '%::unresolved::%' THEN 1 ELSE 0 END
|
||||
) VIRTUAL`
|
||||
|
||||
// edgeGeneratedColumns is the set of edges.* generated columns ensureEdgeColumns
|
||||
// adds to a table created before they existed — which, since none of them are
|
||||
// in schemaSQL's CREATE TABLE, includes a freshly created table too.
|
||||
var edgeGeneratedColumns = []struct {
|
||||
name string
|
||||
ddl string
|
||||
}{
|
||||
{"is_unresolved", isUnresolvedColumnDDL},
|
||||
}
|
||||
|
||||
// edgePromotedColumns lifts the resolver's resolve_terminal /
|
||||
// resolve_terminal_reason Meta keys (see resolver/terminal.go) out of the
|
||||
// meta blob into their own nullable columns — the edge-side sibling of
|
||||
// promotedMetaColumns on nodes (see meta_json.go's "promoted edge columns"
|
||||
// section for extractPromotedEdgeMeta/restorePromotedEdgeMeta and why a
|
||||
// json_extract-derived generated column was tried first and abandoned:
|
||||
// encodeMeta's common case is a custom flat binary codec, not JSON, so
|
||||
// json_extract/json_valid against a real store's meta blobs evaluates to
|
||||
// NULL for effectively every row). Plain (non-generated) columns, so they
|
||||
// share ensureEdgeColumns' table_xinfo scan but are ordinary ALTER TABLE ADD
|
||||
// COLUMN statements, not GENERATED ALWAYS AS expressions.
|
||||
//
|
||||
// Exists to let a future bulk classification query (replacing per-edge
|
||||
// Go-side classifyTerminal calls in reconcileTerminalStamps) read the
|
||||
// CURRENT terminal state as a plain indexed column instead of decoding
|
||||
// Meta, and compare it against a freshly computed value to find only the
|
||||
// edges whose state actually changed — reconcileTerminalStamps measured
|
||||
// only ~1% of examined edges (9,599 of 833,828) ever change state.
|
||||
var edgePromotedColumns = []struct {
|
||||
name string
|
||||
ddl string
|
||||
}{
|
||||
{"resolve_terminal", "resolve_terminal INTEGER"},
|
||||
{"resolve_terminal_reason", "resolve_terminal_reason TEXT"},
|
||||
}
|
||||
|
||||
// ensureEdgeColumns adds edgeGeneratedColumns + edgePromotedColumns to an
|
||||
// edges table created before they existed. Mirrors ensureNodeColumns'
|
||||
// PRAGMA + conditional ALTER pattern, but queries table_xinfo rather than
|
||||
// table_info: table_info silently OMITS generated columns from its result
|
||||
// set (verified against the pinned modernc.org/sqlite driver — a reopened
|
||||
// store's is_unresolved column is invisible to table_info, so the existence
|
||||
// check always came back false and every reopen re-ran the ALTER, failing
|
||||
// with "duplicate column name"). table_xinfo lists every column, generated
|
||||
// ones included, with an extra hidden column (3 == generated) table_info
|
||||
// doesn't have — and works identically for the plain promoted columns too,
|
||||
// so one scan serves both lists.
|
||||
func ensureEdgeColumns(db *sql.DB) error {
|
||||
rows, err := db.Query(`PRAGMA table_xinfo(edges)`)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
existing := make(map[string]bool)
|
||||
for rows.Next() {
|
||||
var (
|
||||
cid, notnull, pk, hidden int
|
||||
name, ctype string
|
||||
dflt sql.NullString
|
||||
)
|
||||
if err := rows.Scan(&cid, &name, &ctype, ¬null, &dflt, &pk, &hidden); err != nil {
|
||||
_ = rows.Close()
|
||||
return err
|
||||
}
|
||||
existing[name] = true
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
_ = rows.Close()
|
||||
return err
|
||||
}
|
||||
_ = rows.Close()
|
||||
for _, c := range edgeGeneratedColumns {
|
||||
if existing[c.name] {
|
||||
continue
|
||||
}
|
||||
if _, err := db.Exec(`ALTER TABLE edges ADD COLUMN ` + c.ddl); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
for _, c := range edgePromotedColumns {
|
||||
if existing[c.name] {
|
||||
continue
|
||||
}
|
||||
if _, err := db.Exec(`ALTER TABLE edges ADD COLUMN ` + c.ddl); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// isStubColumnDDL is nodes.is_stub: a VIRTUAL generated column mirroring
|
||||
// graph.IsStub/StubKind's id-prefix logic (stdlib:: / builtin:: /
|
||||
// external_call:: / module::, bare or repo-prefixed as <repo>::<kind>::...).
|
||||
// Same rationale as isUnresolvedColumnDDL: computed from the existing id
|
||||
// column, no Go call site has to keep it in sync. Exists so a future
|
||||
// SQL-side terminal classification (see resolveTerminalColumnDDL) can check
|
||||
// "is this candidate a stub" via a plain column instead of a per-row Go
|
||||
// IsStub(n.ID) call.
|
||||
const isStubColumnDDL = `is_stub INTEGER GENERATED ALWAYS AS (
|
||||
CASE WHEN
|
||||
id LIKE 'stdlib::%' OR id LIKE 'builtin::%' OR id LIKE 'external_call::%' OR id LIKE 'module::%'
|
||||
OR (
|
||||
instr(id, '::') > 0 AND (
|
||||
substr(id, instr(id, '::') + 2) LIKE 'stdlib::%'
|
||||
OR substr(id, instr(id, '::') + 2) LIKE 'builtin::%'
|
||||
OR substr(id, instr(id, '::') + 2) LIKE 'external_call::%'
|
||||
OR substr(id, instr(id, '::') + 2) LIKE 'module::%'
|
||||
)
|
||||
)
|
||||
THEN 1 ELSE 0 END
|
||||
) VIRTUAL`
|
||||
|
||||
// nodeGeneratedColumns is the nodes-table sibling of edgeGeneratedColumns.
|
||||
// Kept as its own list (and ensureNodeGeneratedColumns as its own function,
|
||||
// rather than folded into ensureNodeColumns) because ensureNodeColumns
|
||||
// checks existence via PRAGMA table_info, which — like the edges case
|
||||
// documented on ensureEdgeColumns — silently omits generated columns.
|
||||
// Reusing that function's table_info scan for is_stub would hit the exact
|
||||
// same "always looks missing, ALTER re-runs, duplicate column name" bug.
|
||||
var nodeGeneratedColumns = []struct {
|
||||
name string
|
||||
ddl string
|
||||
}{
|
||||
{"is_stub", isStubColumnDDL},
|
||||
}
|
||||
|
||||
// ensureNodeGeneratedColumns adds nodeGeneratedColumns to a nodes table
|
||||
// created before they existed. See ensureEdgeColumns for the table_xinfo
|
||||
// vs table_info rationale this mirrors.
|
||||
func ensureNodeGeneratedColumns(db *sql.DB) error {
|
||||
rows, err := db.Query(`PRAGMA table_xinfo(nodes)`)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
existing := make(map[string]bool)
|
||||
for rows.Next() {
|
||||
var (
|
||||
cid, notnull, pk, hidden int
|
||||
name, ctype string
|
||||
dflt sql.NullString
|
||||
)
|
||||
if err := rows.Scan(&cid, &name, &ctype, ¬null, &dflt, &pk, &hidden); err != nil {
|
||||
_ = rows.Close()
|
||||
return err
|
||||
}
|
||||
existing[name] = true
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
_ = rows.Close()
|
||||
return err
|
||||
}
|
||||
_ = rows.Close()
|
||||
for _, c := range nodeGeneratedColumns {
|
||||
if existing[c.name] {
|
||||
continue
|
||||
}
|
||||
if _, err := db.Exec(`ALTER TABLE nodes ADD COLUMN ` + c.ddl); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// schemaSQL is the canonical DDL applied on Open. Statements are
|
||||
// idempotent (IF NOT EXISTS) so they run cleanly against a fresh DB
|
||||
// and against an existing one.
|
||||
//
|
||||
// Schema choices
|
||||
//
|
||||
// - nodes.id is the primary key; INSERT OR REPLACE on the id column
|
||||
// gives idempotent re-adds with last-write-wins on every other
|
||||
// column, matching the in-memory store's behaviour.
|
||||
//
|
||||
// - edges has a synthetic INTEGER PRIMARY KEY plus a UNIQUE
|
||||
// constraint over (from_id, to_id, kind, file_path, line) -- the
|
||||
// logical edge key the in-memory store uses for dedup. INSERT OR
|
||||
// IGNORE on that constraint matches the in-memory "second AddEdge
|
||||
// for the same key is a no-op" semantics.
|
||||
//
|
||||
// - meta is a JSON document (see meta_json.go). nil / empty Meta is
|
||||
// stored as NULL. Four universal, hot-read node keys are promoted to
|
||||
// their own nullable columns (signature / visibility / doc /
|
||||
// external): they are stripped from the JSON blob on write and
|
||||
// restored into Meta on read, so the in-memory map is unchanged. A
|
||||
// NULL column means "not set" (legacy gob rows predate the columns
|
||||
// and keep their values in the blob). Existing databases gain the
|
||||
// columns via ALTER on the next Open (ensureNodeColumns).
|
||||
//
|
||||
// - Secondary indexes mirror the in-memory store's hot lookup paths:
|
||||
// nodes_by_name -- FindNodesByName / FindNodesByNameInRepo
|
||||
// nodes_by_kind -- Stats (group-by-kind)
|
||||
// nodes_by_file -- GetFileNodes, EvictFile
|
||||
// nodes_by_repo -- GetRepoNodes, RepoStats, EvictRepo
|
||||
// (partial index -- empty repo_prefix is
|
||||
// the common case and indexing it would
|
||||
// be pure overhead)
|
||||
// nodes_by_qual -- GetNodeByQualName, unique so duplicate
|
||||
// qual_names surface as constraint errors
|
||||
// edges_by_from -- GetOutEdges (kind included so RemoveEdge
|
||||
// can probe by (from, kind) without a
|
||||
// second hop)
|
||||
// edges_by_to -- GetInEdges
|
||||
const schemaSQL = `
|
||||
CREATE TABLE IF NOT EXISTS nodes (
|
||||
id TEXT PRIMARY KEY,
|
||||
kind TEXT NOT NULL,
|
||||
name TEXT NOT NULL,
|
||||
qual_name TEXT NOT NULL DEFAULT '',
|
||||
file_path TEXT NOT NULL,
|
||||
start_line INTEGER NOT NULL DEFAULT 0,
|
||||
end_line INTEGER NOT NULL DEFAULT 0,
|
||||
start_column INTEGER NOT NULL DEFAULT 0,
|
||||
end_column INTEGER NOT NULL DEFAULT 0,
|
||||
language TEXT NOT NULL DEFAULT '',
|
||||
repo_prefix TEXT NOT NULL DEFAULT '',
|
||||
workspace_id TEXT NOT NULL DEFAULT '',
|
||||
project_id TEXT NOT NULL DEFAULT '',
|
||||
signature TEXT,
|
||||
visibility TEXT,
|
||||
doc TEXT,
|
||||
external INTEGER,
|
||||
return_type TEXT,
|
||||
is_async INTEGER,
|
||||
is_static INTEGER,
|
||||
is_abstract INTEGER,
|
||||
is_exported INTEGER,
|
||||
updated_at INTEGER,
|
||||
data_class TEXT,
|
||||
meta BLOB
|
||||
) WITHOUT ROWID;
|
||||
|
||||
-- nodes_by_name / _kind / _file / _repo are created from the shared
|
||||
-- bulkDroppableIndexes set (see bulk_load.go), not here, so the bulk-load
|
||||
-- fast path can drop and rebuild the EXACT same DDL without drift.
|
||||
-- nodes_by_qual is UNIQUE — it enforces qual_name dedup on every
|
||||
-- INSERT OR REPLACE, so it is never dropped and stays defined here.
|
||||
CREATE UNIQUE INDEX IF NOT EXISTS nodes_by_qual ON nodes(qual_name) WHERE qual_name <> '';
|
||||
|
||||
CREATE TABLE IF NOT EXISTS edges (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
from_id TEXT NOT NULL,
|
||||
to_id TEXT NOT NULL,
|
||||
kind TEXT NOT NULL,
|
||||
file_path TEXT NOT NULL DEFAULT '',
|
||||
line INTEGER NOT NULL DEFAULT 0,
|
||||
confidence REAL NOT NULL DEFAULT 1.0,
|
||||
confidence_label TEXT NOT NULL DEFAULT '',
|
||||
origin TEXT NOT NULL DEFAULT '',
|
||||
tier TEXT NOT NULL DEFAULT '',
|
||||
cross_repo INTEGER NOT NULL DEFAULT 0,
|
||||
meta BLOB,
|
||||
UNIQUE(from_id, to_id, kind, file_path, line)
|
||||
);
|
||||
|
||||
-- edges_by_from / _to / _kind are created from the shared
|
||||
-- bulkDroppableIndexes set (see bulk_load.go), not here, so the bulk-load
|
||||
-- fast path can drop and rebuild the EXACT same DDL without drift.
|
||||
-- edges_by_kind backs EdgesByKind / EdgesByKinds (resolver whole-graph
|
||||
-- passes probe single kinds like provides/imports on every file save);
|
||||
-- without it those are full edges-table scans — edges_by_from/to lead
|
||||
-- with an id column and the partial edges_external index only covers
|
||||
-- its own predicate.
|
||||
|
||||
CREATE TABLE IF NOT EXISTS file_mtimes (
|
||||
repo_prefix TEXT NOT NULL,
|
||||
file_path TEXT NOT NULL,
|
||||
mtime_ns INTEGER NOT NULL,
|
||||
PRIMARY KEY (repo_prefix, file_path)
|
||||
) WITHOUT ROWID;
|
||||
|
||||
-- repo_index_state records per-repo freshness provenance written at the
|
||||
-- end of a (re)index: the git revision + dirty flag the graph reflects,
|
||||
-- the Merkle workspace fingerprint (Tree.Root) that gates global-pass
|
||||
-- short-circuiting, node/edge counts for the index-plausibility baseline,
|
||||
-- and the JSON per-language extractor versions that produced the graph.
|
||||
-- One row per repo_prefix; WITHOUT ROWID — the PK index IS the table,
|
||||
-- like file_mtimes / clone_shingles.
|
||||
CREATE TABLE IF NOT EXISTS repo_index_state (
|
||||
repo_prefix TEXT PRIMARY KEY,
|
||||
indexed_sha TEXT NOT NULL DEFAULT '',
|
||||
dirty INTEGER NOT NULL DEFAULT 0,
|
||||
indexed_at INTEGER NOT NULL DEFAULT 0,
|
||||
workspace_fp TEXT NOT NULL DEFAULT '',
|
||||
node_count INTEGER NOT NULL DEFAULT 0,
|
||||
edge_count INTEGER NOT NULL DEFAULT 0,
|
||||
extractor_versions TEXT NOT NULL DEFAULT ''
|
||||
) WITHOUT ROWID;
|
||||
|
||||
-- enrichment_state records, per (repo, semantic provider), the git revision
|
||||
-- the graph was enriched at plus the coverage that pass reached. Enrichment
|
||||
-- completion otherwise lives only in an in-memory map, so a restart forgets it
|
||||
-- and re-runs full LSP hover passes for a repo whose persisted graph already
|
||||
-- carries the edges. The deferred-enrichment gate reads this row and skips a
|
||||
-- provider whose IndexedSHA still matches HEAD on a clean tree. One row per
|
||||
-- (repo_prefix, provider); WITHOUT ROWID — the PK index IS the table, like
|
||||
-- file_mtimes / repo_index_state.
|
||||
CREATE TABLE IF NOT EXISTS enrichment_state (
|
||||
repo_prefix TEXT NOT NULL,
|
||||
provider TEXT NOT NULL,
|
||||
indexed_sha TEXT NOT NULL DEFAULT '',
|
||||
completed_at INTEGER NOT NULL DEFAULT 0,
|
||||
coverage REAL NOT NULL DEFAULT 0,
|
||||
PRIMARY KEY (repo_prefix, provider)
|
||||
) WITHOUT ROWID;
|
||||
|
||||
-- clone_shingles is the per-symbol MinHash shingle-set sidecar. Each
|
||||
-- function/method node's []uint64 shingle set is stored as a little-
|
||||
-- endian BLOB (8 bytes/elem) keyed by node_id so the maintained clone-
|
||||
-- detection count-min sketch can be rebuilt after a warm restart from
|
||||
-- the snapshot instead of re-parsing every body. repo_prefix carries
|
||||
-- the owning repo so per-repo reseeds (SELECT … WHERE repo_prefix = ?)
|
||||
-- and per-repo wipes don't clobber other repos' shingle sets. node_id
|
||||
-- is the PK (the join key back to nodes.id); like file_mtimes this is a
|
||||
-- WITHOUT ROWID sidecar so the PK index IS the table.
|
||||
CREATE TABLE IF NOT EXISTS clone_shingles (
|
||||
node_id TEXT PRIMARY KEY,
|
||||
repo_prefix TEXT NOT NULL DEFAULT '',
|
||||
shingles BLOB
|
||||
) WITHOUT ROWID;
|
||||
|
||||
-- constant_values is the per-KindConstant literal-value sidecar: one row
|
||||
-- per constant whose RHS is a string / numeric literal, keyed by node_id
|
||||
-- (the join key back to nodes.id). Lifting the value out of the JSON Meta
|
||||
-- blob keeps it queryable (and out of the every-node-load decode path) so
|
||||
-- the resolver can dereference a const-identifier dispatch name to its
|
||||
-- value across files. file_path scopes per-file eviction on reindex;
|
||||
-- repo_prefix scopes per-repo wipes. WITHOUT ROWID — the PK index IS the
|
||||
-- table, like file_mtimes / clone_shingles.
|
||||
CREATE TABLE IF NOT EXISTS constant_values (
|
||||
node_id TEXT PRIMARY KEY,
|
||||
repo_prefix TEXT NOT NULL DEFAULT '',
|
||||
file_path TEXT NOT NULL DEFAULT '',
|
||||
value TEXT NOT NULL DEFAULT ''
|
||||
) WITHOUT ROWID;
|
||||
|
||||
CREATE INDEX IF NOT EXISTS constant_values_by_file ON constant_values(repo_prefix, file_path);
|
||||
|
||||
-- files is the per-file metadata sidecar: one row per indexed file carrying
|
||||
-- the BLAKE3 content hash (the Merkle leaf), byte size, extracted node count,
|
||||
-- and a JSON array of parse-error locations. The Merkle tree stays the
|
||||
-- authoritative change detector; this table is queryable supplementary
|
||||
-- metadata (index_health reports per-file parse errors + node counts from it).
|
||||
-- PK is (repo_prefix, file_path) so a reindex replaces the row in place;
|
||||
-- WITHOUT ROWID — the PK index IS the table, like file_mtimes.
|
||||
CREATE TABLE IF NOT EXISTS files (
|
||||
repo_prefix TEXT NOT NULL DEFAULT '',
|
||||
file_path TEXT NOT NULL,
|
||||
content_hash TEXT NOT NULL DEFAULT '',
|
||||
size INTEGER NOT NULL DEFAULT 0,
|
||||
node_count INTEGER NOT NULL DEFAULT 0,
|
||||
errors TEXT NOT NULL DEFAULT '',
|
||||
PRIMARY KEY (repo_prefix, file_path)
|
||||
) WITHOUT ROWID;
|
||||
-- files_with_errors backs the index_health "files with parse errors" rollup
|
||||
-- so it scans only the (usually tiny) set of erroring files, not every row.
|
||||
CREATE INDEX IF NOT EXISTS files_with_errors ON files(repo_prefix) WHERE errors <> '';
|
||||
|
||||
-- ref_facts is the resolved-reference sidecar: one row per reference edge
|
||||
-- that resolved to a concrete target, recording the target + the provenance
|
||||
-- tier that resolved it. Denormalized file_path + lang make "all reference
|
||||
-- facts originating in file X" a single indexed query (the scope unit for
|
||||
-- incremental re-resolution and the audit/diff surface). repo_prefix scopes
|
||||
-- per-repo. PK is (repo_prefix, from_id, to_id, kind, line) so re-resolving a
|
||||
-- file replaces its facts in place; WITHOUT ROWID — the PK index IS the table.
|
||||
CREATE TABLE IF NOT EXISTS ref_facts (
|
||||
repo_prefix TEXT NOT NULL DEFAULT '',
|
||||
from_id TEXT NOT NULL,
|
||||
to_id TEXT NOT NULL,
|
||||
kind TEXT NOT NULL,
|
||||
ref_name TEXT NOT NULL DEFAULT '',
|
||||
line INTEGER NOT NULL DEFAULT 0,
|
||||
origin TEXT NOT NULL DEFAULT '',
|
||||
tier TEXT NOT NULL DEFAULT '',
|
||||
candidates TEXT NOT NULL DEFAULT '',
|
||||
file_path TEXT NOT NULL DEFAULT '',
|
||||
lang TEXT NOT NULL DEFAULT '',
|
||||
PRIMARY KEY (repo_prefix, from_id, to_id, kind, line)
|
||||
) WITHOUT ROWID;
|
||||
CREATE INDEX IF NOT EXISTS ref_facts_by_file ON ref_facts(repo_prefix, file_path);
|
||||
-- ref_facts_by_target backs the reverse lookup ("which files hold a fact
|
||||
-- resolving TO these symbols") that affected-by re-resolution runs when a
|
||||
-- file's symbol signatures change. Without it that query is a full
|
||||
-- ref_facts scan — the PK leads with from_id, not to_id.
|
||||
CREATE INDEX IF NOT EXISTS ref_facts_by_target ON ref_facts(repo_prefix, to_id);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS vectors (
|
||||
node_id TEXT PRIMARY KEY,
|
||||
dims INTEGER NOT NULL,
|
||||
vec BLOB NOT NULL
|
||||
) WITHOUT ROWID;
|
||||
|
||||
-- churn_enrichment is the per-node git-churn sidecar (change A: move
|
||||
-- enrichment OUT of nodes.meta so the node hot path stops encoding
|
||||
-- rarely-read data into the blob and get_churn_rate does an indexed read
|
||||
-- instead of an AllNodes+meta-decode scan). One typed row per enriched
|
||||
-- file/function/method node, keyed by node_id (join key back to
|
||||
-- nodes.id); repo_prefix scopes
|
||||
-- per-repo reseeds/wipes. head_sha/branch/computed_at are file-level only
|
||||
-- (empty for symbols). WITHOUT ROWID: the PK index IS the table.
|
||||
CREATE TABLE IF NOT EXISTS churn_enrichment (
|
||||
node_id TEXT PRIMARY KEY,
|
||||
repo_prefix TEXT NOT NULL DEFAULT '',
|
||||
commit_count INTEGER NOT NULL DEFAULT 0,
|
||||
age_days INTEGER NOT NULL DEFAULT 0,
|
||||
churn_rate REAL NOT NULL DEFAULT 0,
|
||||
last_author TEXT NOT NULL DEFAULT '',
|
||||
last_commit_at TEXT NOT NULL DEFAULT '',
|
||||
head_sha TEXT NOT NULL DEFAULT '',
|
||||
branch TEXT NOT NULL DEFAULT '',
|
||||
computed_at TEXT NOT NULL DEFAULT ''
|
||||
) WITHOUT ROWID;
|
||||
CREATE INDEX IF NOT EXISTS churn_by_repo ON churn_enrichment(repo_prefix) WHERE repo_prefix <> '';
|
||||
|
||||
-- coverage_enrichment: per-symbol coverage sidecar (change A). Typed
|
||||
-- columns keyed by node_id; repo_prefix scopes per-repo wipes.
|
||||
CREATE TABLE IF NOT EXISTS coverage_enrichment (
|
||||
node_id TEXT PRIMARY KEY,
|
||||
repo_prefix TEXT NOT NULL DEFAULT '',
|
||||
coverage_pct REAL NOT NULL DEFAULT 0,
|
||||
num_stmt INTEGER NOT NULL DEFAULT 0,
|
||||
hit INTEGER NOT NULL DEFAULT 0
|
||||
) WITHOUT ROWID;
|
||||
CREATE INDEX IF NOT EXISTS coverage_by_repo ON coverage_enrichment(repo_prefix) WHERE repo_prefix <> '';
|
||||
|
||||
-- release_enrichment: per-file "added_in <tag>" sidecar (change A).
|
||||
CREATE TABLE IF NOT EXISTS release_enrichment (
|
||||
node_id TEXT PRIMARY KEY,
|
||||
repo_prefix TEXT NOT NULL DEFAULT '',
|
||||
added_in TEXT NOT NULL DEFAULT ''
|
||||
) WITHOUT ROWID;
|
||||
CREATE INDEX IF NOT EXISTS release_by_repo ON release_enrichment(repo_prefix) WHERE repo_prefix <> '';
|
||||
|
||||
-- blame_enrichment: per-symbol latest-author sidecar (change A).
|
||||
CREATE TABLE IF NOT EXISTS blame_enrichment (
|
||||
node_id TEXT PRIMARY KEY,
|
||||
repo_prefix TEXT NOT NULL DEFAULT '',
|
||||
commit_sha TEXT NOT NULL DEFAULT '',
|
||||
email TEXT NOT NULL DEFAULT '',
|
||||
ts INTEGER NOT NULL DEFAULT 0
|
||||
) WITHOUT ROWID;
|
||||
CREATE INDEX IF NOT EXISTS blame_by_repo ON blame_enrichment(repo_prefix) WHERE repo_prefix <> '';
|
||||
|
||||
-- symbol_fts is the FTS5 full-text index over pre-tokenised symbol
|
||||
-- names. It replaces the multi-GB in-heap Bleve/BM25 index with an
|
||||
-- on-disk inverted index the SymbolSearcher / SymbolBundleSearcher
|
||||
-- query through. A standard (NOT contentless) FTS5 table; individual
|
||||
-- rows are deleted by their FTS5 docid via the symbol_fts_rowid sidecar
|
||||
-- below (node_id is UNINDEXED, so a DELETE keyed on it would full-scan
|
||||
-- the index). node_id is the join key back to nodes.id; repo_prefix is
|
||||
-- carried UNINDEXED so per-repo staleness wipes (DELETE … WHERE
|
||||
-- repo_prefix = ?) hit a literal column without a separate b-tree.
|
||||
-- Only "tokens" is indexed for matching. IF NOT EXISTS makes this
|
||||
-- idempotent on every Open, so an existing .sqlite gains the vtable
|
||||
-- on its next open + reindex.
|
||||
CREATE VIRTUAL TABLE IF NOT EXISTS symbol_fts USING fts5(node_id UNINDEXED, repo_prefix UNINDEXED, tokens);
|
||||
|
||||
-- symbol_fts_rowid maps a node_id to the rowid (FTS5 docid) of its row in
|
||||
-- symbol_fts. node_id is UNINDEXED in the FTS5 vtable, so deleting a node's
|
||||
-- prior row with "DELETE … WHERE node_id = ?" full-scans the entire index
|
||||
-- once PER symbol — quadratic on the per-edit reindex hot path. This sidecar
|
||||
-- turns the delete into an O(log n) docid delete ("WHERE rowid = ?", the FTS5
|
||||
-- docid IS indexed). One row per indexed symbol, keyed by node_id (the join
|
||||
-- key back to nodes.id); repo_prefix scopes the per-repo wipe that
|
||||
-- BulkUpsertSymbolFTS performs in lockstep with symbol_fts. WITHOUT ROWID:
|
||||
-- the PK index IS the table, like file_mtimes / clone_shingles.
|
||||
CREATE TABLE IF NOT EXISTS symbol_fts_rowid (
|
||||
node_id TEXT PRIMARY KEY,
|
||||
repo_prefix TEXT NOT NULL DEFAULT '',
|
||||
fts_rowid INTEGER NOT NULL
|
||||
) WITHOUT ROWID;
|
||||
|
||||
-- content_fts is the FTS5 full-text index over CONTENT (data_class=
|
||||
-- "content") section bodies — text / pdf / pptx / xlsx chunks. It is
|
||||
-- kept SEPARATE from symbol_fts so content text never enters the symbol
|
||||
-- search or the code-oriented analysis passes: a content-heavy repo of a
|
||||
-- few hundred large documents explodes into hundreds of thousands of
|
||||
-- section nodes, and streaming their bodies here (per file, on disk)
|
||||
-- instead of into symbol_fts + graph nodes keeps the code index and the
|
||||
-- graph passes bounded. Only "body" is indexed for matching; node_id /
|
||||
-- repo_prefix / file_path / ordinal ride UNINDEXED so the per-repo and
|
||||
-- per-file staleness wipes hit literal columns without a b-tree.
|
||||
CREATE VIRTUAL TABLE IF NOT EXISTS content_fts USING fts5(node_id UNINDEXED, repo_prefix UNINDEXED, file_path UNINDEXED, ordinal UNINDEXED, body);
|
||||
`
|
||||
@@ -0,0 +1,104 @@
|
||||
package store_sqlite
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"github.com/zzet/gortex/internal/graph"
|
||||
)
|
||||
|
||||
// hasNodeColumn reports whether the nodes table currently has the named column.
|
||||
func hasNodeColumn(t *testing.T, db *sql.DB, col string) bool {
|
||||
t.Helper()
|
||||
rows, err := db.Query(`PRAGMA table_info(nodes)`)
|
||||
require.NoError(t, err)
|
||||
defer rows.Close()
|
||||
for rows.Next() {
|
||||
var (
|
||||
cid, notnull, pk int
|
||||
name, ctype string
|
||||
dflt sql.NullString
|
||||
)
|
||||
require.NoError(t, rows.Scan(&cid, &name, &ctype, ¬null, &dflt, &pk))
|
||||
if name == col {
|
||||
return true
|
||||
}
|
||||
}
|
||||
require.NoError(t, rows.Err())
|
||||
return false
|
||||
}
|
||||
|
||||
// TestOpenUpgradesPreDataClassStore is the backward-compatibility proof for the
|
||||
// promoted data_class column: an existing v1 store written before the column
|
||||
// existed must Open cleanly (ensureNodeColumns ALTERs the column in before the
|
||||
// node statements are prepared), keep its rows, and immediately get the working
|
||||
// SQL-level content filter — all WITHOUT a schema_version bump or a reindex.
|
||||
//
|
||||
// The simulated old store has data_class dropped while staying at the v1
|
||||
// baseline, the exact shape of every Gortex sqlite DB already on disk before
|
||||
// this change. Without the data_class entry in promotedMetaColumns, the reopen
|
||||
// fails with "no such column: data_class" when prepare() builds the node
|
||||
// INSERT/SELECT — so this test fails-closed if that wiring regresses.
|
||||
func TestOpenUpgradesPreDataClassStore(t *testing.T) {
|
||||
path := filepath.Join(t.TempDir(), "store.sqlite")
|
||||
|
||||
// 1. Create a current store and seed a row, then close. A fresh store is
|
||||
// stamped at the current schema version; step 2 knocks it back to the v1
|
||||
// baseline to simulate a store written before data_class existed.
|
||||
s, err := Open(path)
|
||||
require.NoError(t, err)
|
||||
s.AddNode(&graph.Node{ID: "old1", Kind: graph.KindFunction, Name: "Legacy", FilePath: "f.go", RepoPrefix: "r"})
|
||||
require.NoError(t, s.Close())
|
||||
|
||||
// 2. Simulate a store written before data_class existed: drop the column
|
||||
// while leaving user_version at the v1 baseline.
|
||||
withRawDB(t, path, func(db *sql.DB) {
|
||||
_, err := db.Exec(`ALTER TABLE nodes DROP COLUMN data_class`)
|
||||
require.NoError(t, err, "simulate a pre-data_class store")
|
||||
require.False(t, hasNodeColumn(t, db, "data_class"), "data_class must be absent before the upgrade")
|
||||
|
||||
// A fresh Open stamps the current schema version; knock it back to the v1
|
||||
// baseline so this genuinely simulates a pre-data_class store and the
|
||||
// reopen exercises the in-place upgrade arm rather than a no-op.
|
||||
_, err = db.Exec(`PRAGMA user_version = 1`)
|
||||
require.NoError(t, err, "reset to the v1 baseline")
|
||||
|
||||
var v int
|
||||
require.NoError(t, db.QueryRow(`PRAGMA user_version`).Scan(&v))
|
||||
require.Equal(t, 1, v, "the simulated old store must sit at the v1 baseline")
|
||||
})
|
||||
|
||||
// 3. Reopen with the current binary. ensureNodeColumns must re-add the
|
||||
// column before prepare() references it, so Open succeeds without a wipe.
|
||||
s2, err := Open(path)
|
||||
require.NoError(t, err, "Open must upgrade a pre-data_class store in place, not fail on the missing column")
|
||||
t.Cleanup(func() { _ = s2.Close() })
|
||||
|
||||
require.True(t, hasNodeColumn(t, s2.db, "data_class"), "ensureNodeColumns must re-add data_class on Open")
|
||||
require.False(t, s2.NeedsRebuild(), "an additive-column upgrade must not signal a wipe/reindex")
|
||||
|
||||
// 4. Existing rows survived (the upgrade is in place, not a rebuild).
|
||||
require.NotNil(t, s2.GetNode("old1"), "existing rows must survive the in-place upgrade")
|
||||
|
||||
// 5. A new content node persists, round-trips through the promoted column,
|
||||
// and is correctly dropped by the SQL-level content filter.
|
||||
s2.AddNode(&graph.Node{ID: "content1", Kind: graph.KindDoc, Name: "doc.txt::0", RepoPrefix: "r",
|
||||
Meta: map[string]any{"data_class": "content", "section_text": "x"}})
|
||||
s2.AddNode(&graph.Node{ID: "code1", Kind: graph.KindFunction, Name: "Foo", RepoPrefix: "r"})
|
||||
|
||||
content := s2.GetNode("content1")
|
||||
require.NotNil(t, content)
|
||||
require.Equal(t, "content", content.Meta["data_class"], "data_class round-trips via the promoted column after upgrade")
|
||||
|
||||
ids := map[string]bool{}
|
||||
for _, n := range s2.GetRepoNonContentNodes("r") {
|
||||
ids[n.ID] = true
|
||||
}
|
||||
require.True(t, ids["old1"], "legacy non-content node kept")
|
||||
require.True(t, ids["code1"], "code node kept")
|
||||
require.False(t, ids["content1"], "content node filtered at the SQL level after the upgrade")
|
||||
require.Len(t, ids, 2)
|
||||
}
|
||||
@@ -0,0 +1,247 @@
|
||||
package store_sqlite
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"fmt"
|
||||
"os"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// Schema versioning for the graph store.
|
||||
//
|
||||
// Unlike the sidecar (which holds irreplaceable user data and must migrate in
|
||||
// place), the graph store is a DERIVED CACHE: every row is reconstructable by
|
||||
// re-indexing the source. So the cheapest *always-correct* reaction to a schema
|
||||
// change an old on-disk DB can't satisfy is to drop the file and let the daemon
|
||||
// rebuild it on the next index. A migration may therefore declare rebuild=true
|
||||
// instead of writing an in-place transform that would have to re-derive the new
|
||||
// data from source anyway. In-place steps remain the cheap path for purely
|
||||
// mechanical changes (a new index, a denormalisation, a column with a
|
||||
// computable default) that spare a large repo a multi-minute reindex.
|
||||
//
|
||||
// The whole mechanism keys off SQLite's built-in PRAGMA user_version, read on
|
||||
// Open before schemaSQL runs. There is no separate version table.
|
||||
//
|
||||
// Concurrency: the daemon holds an exclusive flock on <store>.lock around Open
|
||||
// (see serverstack.NewSharedServer), so reading the version, wiping the file,
|
||||
// and stamping it cannot race another process. That is why — unlike the
|
||||
// sidecar — this path needs no BEGIN IMMEDIATE / busy-loop handling.
|
||||
|
||||
// currentSchemaVersion is the version a fully-reconciled store reports via
|
||||
// PRAGMA user_version. Bump it whenever schemaSQL's typed-column shape or an
|
||||
// index changes in a way an old on-disk DB would not already have, and append a
|
||||
// matching schemaMigrations entry describing how to bring an older store
|
||||
// forward (in place, or by rebuild).
|
||||
const currentSchemaVersion = 2
|
||||
|
||||
// schemaMigration is one forward step. Exactly one strategy applies:
|
||||
// - rebuild=true: the change introduces structure/data that can only come
|
||||
// from re-indexing the source; an older store is wiped and rebuilt.
|
||||
// - inPlace!=nil: the change is mechanically derivable from the existing
|
||||
// store and is applied in a transaction with no reindex.
|
||||
//
|
||||
// Steps are append-only and ascending; never edit or renumber a shipped one.
|
||||
// Any inPlace step must be idempotent (IF NOT EXISTS / ADD COLUMN guarded).
|
||||
type schemaMigration struct {
|
||||
version int
|
||||
name string
|
||||
inPlace func(tx *sql.Tx) error
|
||||
rebuild bool
|
||||
}
|
||||
|
||||
// schemaMigrations is the ordered, forward-only registry. Version 1 is the
|
||||
// implicit baseline (no entry): a v1 store is reconciled entirely by schemaSQL's
|
||||
// idempotent CREATE ... IF NOT EXISTS plus ensureNodeColumns, so any
|
||||
// pre-versioning database baseline-stamps to v1 without a rebuild. Append
|
||||
// entries for version 2 and up as the schema evolves.
|
||||
var schemaMigrations = []schemaMigration{
|
||||
{version: 2, name: "dedupe fn-value placeholder edges", inPlace: dedupeFnValuePlaceholderEdges},
|
||||
}
|
||||
|
||||
// dedupeFnValuePlaceholderEdges collapses duplicate function-as-value gate
|
||||
// placeholder edges (graph.FnValuePlaceholderMarker, `unresolved::fnvalue::
|
||||
// <name>`) to one row per (from_id, to_id), keeping the MIN(id) survivor. The
|
||||
// capture path now dedups per (from, name) before it emits, but stores written
|
||||
// earlier accumulated one placeholder per call site — a live store held
|
||||
// millions — and EdgesWithUnresolvedTarget plus the resolver's terminal
|
||||
// reconcile materialised every one on each warm restart, the dominant warmup
|
||||
// heap transient this step drains. The keep set is small (tens of thousands of
|
||||
// distinct pairs), so the NOT IN materialisation is cheap; the ph filter rides
|
||||
// the edges_by_to(to_id) range for the bare form and the is_unresolved index for
|
||||
// the multi-repo infix form. Idempotent: a second run finds no duplicates. Freed
|
||||
// pages return to the freelist and are reused by later writes; the file itself
|
||||
// shrinks only under a manual VACUUM, deliberately out of scope for a derived
|
||||
// cache that reclaims the space on its own.
|
||||
func dedupeFnValuePlaceholderEdges(tx *sql.Tx) error {
|
||||
_, err := tx.Exec(`
|
||||
WITH ph AS (
|
||||
SELECT id, from_id, to_id FROM edges
|
||||
WHERE (to_id >= 'unresolved::fnvalue::' AND to_id < 'unresolved::fnvalue:;')
|
||||
OR (is_unresolved = 1 AND to_id LIKE '%::unresolved::fnvalue::%')
|
||||
), keep AS (
|
||||
SELECT MIN(id) AS id FROM ph GROUP BY from_id, to_id
|
||||
)
|
||||
DELETE FROM edges WHERE id IN (SELECT id FROM ph) AND id NOT IN (SELECT id FROM keep)`)
|
||||
return err
|
||||
}
|
||||
|
||||
// schemaPlan is the decision planSchemaMigration derives from the stored
|
||||
// PRAGMA user_version. It mutates nothing on its own.
|
||||
type schemaPlan struct {
|
||||
wipe bool // drop the on-disk DB and rebuild from source
|
||||
inPlace []schemaMigration // ordered in-place steps to run after schemaSQL
|
||||
stamp bool // write currentSchemaVersion once reconciled
|
||||
}
|
||||
|
||||
// planSchemaMigrationWith decides how to reconcile a store at the stored
|
||||
// PRAGMA user_version to current, given the migration registry. It mutates
|
||||
// nothing. Open passes (currentSchemaVersion, schemaMigrations); tests pass
|
||||
// fixtures.
|
||||
func planSchemaMigrationWith(stored, current int, migrations []schemaMigration) schemaPlan {
|
||||
switch {
|
||||
case stored == current:
|
||||
return schemaPlan{} // up to date, nothing to do
|
||||
case stored > current:
|
||||
// Written by a newer build than this binary understands; the shape may
|
||||
// have changed under us. For a cache the safe move is to rebuild.
|
||||
return schemaPlan{wipe: true, stamp: true}
|
||||
case stored == 0:
|
||||
// Fresh DB, or a pre-versioning store of unknown shape. schemaSQL's
|
||||
// idempotent CREATE ... IF NOT EXISTS plus ensureNodeColumns /
|
||||
// ensureEdgeColumns reconcile the base shape either way, so a stored==0
|
||||
// store needs a wipe only when a pending step is a REBUILD whose data can
|
||||
// only come from re-indexing source. With nothing pending, stamp; with
|
||||
// only in-place steps pending, run them and stamp — an in-place step is
|
||||
// idempotent and mechanically derivable, so it upgrades a pre-versioning
|
||||
// store in place (preserving its rows) exactly as it upgrades a known
|
||||
// prior version. Wiping a stored==0 store on any migration instead would
|
||||
// force every non-daemon Open (tests, read-only tools) to pass WithRebuild
|
||||
// the moment the first migration ships.
|
||||
pending := pendingBetween(0, current, migrations)
|
||||
if len(pending) == 0 {
|
||||
return schemaPlan{stamp: true}
|
||||
}
|
||||
if anyRebuild(pending) {
|
||||
return schemaPlan{wipe: true, stamp: true}
|
||||
}
|
||||
return schemaPlan{inPlace: pending, stamp: true}
|
||||
default: // 0 < stored < current: a known prior version
|
||||
pending := pendingBetween(stored, current, migrations)
|
||||
if anyRebuild(pending) {
|
||||
return schemaPlan{wipe: true, stamp: true}
|
||||
}
|
||||
return schemaPlan{inPlace: pending, stamp: true}
|
||||
}
|
||||
}
|
||||
|
||||
func pendingBetween(stored, current int, migrations []schemaMigration) []schemaMigration {
|
||||
var out []schemaMigration
|
||||
for _, m := range migrations {
|
||||
if m.version > stored && m.version <= current {
|
||||
out = append(out, m)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func anyRebuild(ms []schemaMigration) bool {
|
||||
for _, m := range ms {
|
||||
if m.rebuild {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// validateSchemaMigrations checks the registry is well-formed. A test asserts
|
||||
// this against the shipped (currentSchemaVersion, schemaMigrations) so the
|
||||
// dangerous mistake — bumping currentSchemaVersion without appending a matching
|
||||
// entry — fails CI instead of silently baseline-stamping an un-migrated store
|
||||
// to the new version at runtime. Rules:
|
||||
// - versions are >= 2 (v1 is the implicit baseline, never an entry) and
|
||||
// strictly ascending;
|
||||
// - each step sets exactly one strategy (inPlace xor rebuild);
|
||||
// - the highest version equals current, so the registry actually defines how
|
||||
// to reach it. An empty registry is valid only at version 1.
|
||||
func validateSchemaMigrations(current int, migs []schemaMigration) error {
|
||||
if len(migs) == 0 {
|
||||
if current != 1 {
|
||||
return fmt.Errorf("schema version %d has no migrations: only v1 may have an empty registry", current)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
prev := 0
|
||||
for i, m := range migs {
|
||||
if m.version < 2 {
|
||||
return fmt.Errorf("migration %q has version %d: entries must be >= 2 (v1 is the implicit baseline)", m.name, m.version)
|
||||
}
|
||||
if i > 0 && m.version <= prev {
|
||||
return fmt.Errorf("migrations must be strictly ascending: v%d (%s) does not follow v%d", m.version, m.name, prev)
|
||||
}
|
||||
if (m.inPlace != nil) == m.rebuild {
|
||||
return fmt.Errorf("migration v%d (%s) must set exactly one of inPlace / rebuild", m.version, m.name)
|
||||
}
|
||||
prev = m.version
|
||||
}
|
||||
if prev != current {
|
||||
return fmt.Errorf("highest migration version %d != currentSchemaVersion %d: a version bump needs a matching migration entry", prev, current)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// readUserVersion reads PRAGMA user_version (0 on a fresh database).
|
||||
func readUserVersion(db *sql.DB) (int, error) {
|
||||
var v int
|
||||
if err := db.QueryRow("PRAGMA user_version").Scan(&v); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return v, nil
|
||||
}
|
||||
|
||||
// setUserVersion stamps the schema version. PRAGMA takes no bound parameters;
|
||||
// v is an int we control, so the format is safe.
|
||||
func setUserVersion(db *sql.DB, v int) error {
|
||||
if _, err := db.Exec(fmt.Sprintf("PRAGMA user_version = %d", v)); err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// applyInPlaceMigrations runs the in-place steps in a single transaction.
|
||||
func applyInPlaceMigrations(db *sql.DB, steps []schemaMigration) error {
|
||||
if len(steps) == 0 {
|
||||
return nil
|
||||
}
|
||||
tx, err := db.Begin()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer func() { _ = tx.Rollback() }() // no-op once Commit succeeds
|
||||
for _, m := range steps {
|
||||
if err := m.inPlace(tx); err != nil {
|
||||
return fmt.Errorf("schema migration v%d (%s): %w", m.version, m.name, err)
|
||||
}
|
||||
}
|
||||
return tx.Commit()
|
||||
}
|
||||
|
||||
// removeStoreFiles deletes the SQLite database and its companions. A missing
|
||||
// file is not an error. Never called for ":memory:".
|
||||
//
|
||||
// The suffix list covers the files the DSN's journal_mode(WAL) produces (-wal,
|
||||
// -shm) plus the rollback -journal a non-WAL fallback would use; keep it in
|
||||
// sync if the journal_mode in Open's DSN ever changes.
|
||||
func removeStoreFiles(path string) error {
|
||||
for _, suffix := range []string{"", "-wal", "-shm", "-journal"} {
|
||||
if err := os.Remove(path + suffix); err != nil && !os.IsNotExist(err) {
|
||||
return fmt.Errorf("remove %s: %w", path+suffix, err)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// isMemoryPath reports whether path is an in-process SQLite database (no file
|
||||
// on disk to wipe, always built fresh by schemaSQL).
|
||||
func isMemoryPath(path string) bool {
|
||||
return strings.Contains(path, ":memory:")
|
||||
}
|
||||
@@ -0,0 +1,545 @@
|
||||
package store_sqlite
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"errors"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
_ "modernc.org/sqlite"
|
||||
)
|
||||
|
||||
// withRawDB opens a bare *sql.DB on path, runs fn, and closes it — used to
|
||||
// simulate an on-disk store written by an older/newer build (set user_version,
|
||||
// insert rows) without going through Open's reconciliation.
|
||||
func withRawDB(t *testing.T, path string, fn func(db *sql.DB)) {
|
||||
t.Helper()
|
||||
db, err := sql.Open("sqlite", path+"?_pragma=busy_timeout(5000)")
|
||||
if err != nil {
|
||||
t.Fatalf("open raw db: %v", err)
|
||||
}
|
||||
defer db.Close()
|
||||
fn(db)
|
||||
}
|
||||
|
||||
func nodeCount(t *testing.T, db *sql.DB) int {
|
||||
t.Helper()
|
||||
var n int
|
||||
if err := db.QueryRow("SELECT COUNT(*) FROM nodes").Scan(&n); err != nil {
|
||||
t.Fatalf("count nodes: %v", err)
|
||||
}
|
||||
return n
|
||||
}
|
||||
|
||||
// TestOpenStampsFreshDB: a brand-new on-disk store is stamped to the current
|
||||
// schema version.
|
||||
func TestOpenStampsFreshDB(t *testing.T) {
|
||||
path := filepath.Join(t.TempDir(), "store.sqlite")
|
||||
s, err := Open(path)
|
||||
if err != nil {
|
||||
t.Fatalf("Open fresh: %v", err)
|
||||
}
|
||||
defer s.Close()
|
||||
if v, err := readUserVersion(s.db); err != nil || v != currentSchemaVersion {
|
||||
t.Fatalf("fresh user_version = %d (err %v), want %d", v, err, currentSchemaVersion)
|
||||
}
|
||||
}
|
||||
|
||||
// TestOpenBaselineStampsOldDBWithoutWipe: a pre-versioning store (user_version
|
||||
// 0, reconcilable to current by schemaSQL + ensureNodeColumns) is stamped in
|
||||
// place — its data must survive, not be wiped.
|
||||
func TestOpenBaselineStampsOldDBWithoutWipe(t *testing.T) {
|
||||
path := filepath.Join(t.TempDir(), "store.sqlite")
|
||||
|
||||
// Create the store, then simulate a pre-versioning DB: a row + user_version 0.
|
||||
s, err := Open(path)
|
||||
if err != nil {
|
||||
t.Fatalf("first open: %v", err)
|
||||
}
|
||||
if err := s.Close(); err != nil {
|
||||
t.Fatalf("close: %v", err)
|
||||
}
|
||||
withRawDB(t, path, func(db *sql.DB) {
|
||||
if _, err := db.Exec(`INSERT INTO nodes (id, kind, name, file_path) VALUES ('n1','func','Foo','f.go')`); err != nil {
|
||||
t.Fatalf("seed node: %v", err)
|
||||
}
|
||||
if _, err := db.Exec(`PRAGMA user_version = 0`); err != nil {
|
||||
t.Fatalf("reset user_version: %v", err)
|
||||
}
|
||||
})
|
||||
|
||||
s2, err := Open(path)
|
||||
if err != nil {
|
||||
t.Fatalf("reopen old DB: %v", err)
|
||||
}
|
||||
defer s2.Close()
|
||||
if v, _ := readUserVersion(s2.db); v != currentSchemaVersion {
|
||||
t.Fatalf("user_version after baseline = %d, want %d", v, currentSchemaVersion)
|
||||
}
|
||||
if n := nodeCount(t, s2.db); n != 1 {
|
||||
t.Fatalf("node count after baseline = %d, want 1 (data must NOT be wiped)", n)
|
||||
}
|
||||
}
|
||||
|
||||
// TestOpenRebuildsNewerDB: a store written by a NEWER build (user_version above
|
||||
// current) cannot be trusted, so Open drops and rebuilds it — the data is gone
|
||||
// and the version is re-stamped to current. Proves the wipe path (and that the
|
||||
// -wal/-shm companions are cleared along with the main file).
|
||||
func TestOpenRebuildsNewerDB(t *testing.T) {
|
||||
path := filepath.Join(t.TempDir(), "store.sqlite")
|
||||
|
||||
s, err := Open(path)
|
||||
if err != nil {
|
||||
t.Fatalf("first open: %v", err)
|
||||
}
|
||||
if err := s.Close(); err != nil {
|
||||
t.Fatalf("close: %v", err)
|
||||
}
|
||||
withRawDB(t, path, func(db *sql.DB) {
|
||||
if _, err := db.Exec(`INSERT INTO nodes (id, kind, name, file_path) VALUES ('n1','func','Foo','f.go')`); err != nil {
|
||||
t.Fatalf("seed node: %v", err)
|
||||
}
|
||||
if _, err := db.Exec(`PRAGMA user_version = 999`); err != nil { // a future version this binary doesn't know
|
||||
t.Fatalf("set future user_version: %v", err)
|
||||
}
|
||||
})
|
||||
|
||||
s2, err := Open(path, WithRebuild()) // simulate the daemon: holds the lock, may rebuild
|
||||
if err != nil {
|
||||
t.Fatalf("reopen newer DB: %v", err)
|
||||
}
|
||||
defer s2.Close()
|
||||
if v, _ := readUserVersion(s2.db); v != currentSchemaVersion {
|
||||
t.Fatalf("user_version after rebuild = %d, want %d", v, currentSchemaVersion)
|
||||
}
|
||||
if n := nodeCount(t, s2.db); n != 0 {
|
||||
t.Fatalf("node count after rebuild = %d, want 0 (newer DB must be wiped)", n)
|
||||
}
|
||||
}
|
||||
|
||||
// TestOpenRefusesWipeWithoutOptIn: the default Open must NOT destroy an
|
||||
// incompatible on-disk database. Without WithRebuild it returns
|
||||
// ErrSchemaRebuildRequired and leaves the file (and its rows) intact, so a
|
||||
// caller that does not hold the store lock cannot silently corrupt a store
|
||||
// another process may have open.
|
||||
func TestOpenRefusesWipeWithoutOptIn(t *testing.T) {
|
||||
path := filepath.Join(t.TempDir(), "store.sqlite")
|
||||
s, err := Open(path)
|
||||
if err != nil {
|
||||
t.Fatalf("first open: %v", err)
|
||||
}
|
||||
if _, err := s.db.Exec(`INSERT INTO nodes (id, kind, name, file_path) VALUES ('n1','func','Foo','f.go')`); err != nil {
|
||||
t.Fatalf("seed node: %v", err)
|
||||
}
|
||||
if err := s.Close(); err != nil {
|
||||
t.Fatalf("close: %v", err)
|
||||
}
|
||||
withRawDB(t, path, func(db *sql.DB) {
|
||||
if _, err := db.Exec(`PRAGMA user_version = 999`); err != nil {
|
||||
t.Fatalf("set future version: %v", err)
|
||||
}
|
||||
})
|
||||
|
||||
if _, err := Open(path); !errors.Is(err, ErrSchemaRebuildRequired) {
|
||||
t.Fatalf("Open without WithRebuild = %v, want ErrSchemaRebuildRequired", err)
|
||||
}
|
||||
withRawDB(t, path, func(db *sql.DB) {
|
||||
if n := nodeCount(t, db); n != 1 {
|
||||
t.Fatalf("node count = %d after a refused wipe, want 1 (the file must be untouched)", n)
|
||||
}
|
||||
var v int
|
||||
if err := db.QueryRow("PRAGMA user_version").Scan(&v); err != nil {
|
||||
t.Fatalf("read user_version: %v", err)
|
||||
}
|
||||
if v != 999 {
|
||||
t.Fatalf("user_version = %d after a refused wipe, want 999 (unchanged)", v)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// TestPlanSchemaMigration covers the pure decision logic, including the
|
||||
// in-place vs rebuild dispatch a future currentSchemaVersion=2 would exercise.
|
||||
func TestPlanSchemaMigration(t *testing.T) {
|
||||
inPlace := schemaMigration{version: 2, name: "add-index", inPlace: func(*sql.Tx) error { return nil }}
|
||||
rebuild := schemaMigration{version: 2, name: "typed-column", rebuild: true}
|
||||
|
||||
cases := []struct {
|
||||
name string
|
||||
stored, current int
|
||||
migs []schemaMigration
|
||||
wantWipe bool
|
||||
wantStamp bool
|
||||
wantInPlace int
|
||||
}{
|
||||
{"up to date", 1, 1, nil, false, false, 0},
|
||||
{"fresh at v1 baseline-stamps", 0, 1, nil, false, true, 0},
|
||||
{"newer DB rebuilds", 2, 1, nil, true, true, 0},
|
||||
{"v0 with only in-place pending upgrades in place, no wipe", 0, 2, []schemaMigration{inPlace}, false, true, 1},
|
||||
{"v0 with a pending rebuild wipes", 0, 2, []schemaMigration{rebuild}, true, true, 0},
|
||||
{"v1->v2 in-place", 1, 2, []schemaMigration{inPlace}, false, true, 1},
|
||||
{"v1->v2 rebuild", 1, 2, []schemaMigration{rebuild}, true, true, 0},
|
||||
}
|
||||
for _, c := range cases {
|
||||
t.Run(c.name, func(t *testing.T) {
|
||||
got := planSchemaMigrationWith(c.stored, c.current, c.migs)
|
||||
if got.wipe != c.wantWipe || got.stamp != c.wantStamp || len(got.inPlace) != c.wantInPlace {
|
||||
t.Fatalf("plan(%d->%d) = {wipe:%v stamp:%v inPlace:%d}, want {wipe:%v stamp:%v inPlace:%d}",
|
||||
c.stored, c.current, got.wipe, got.stamp, len(got.inPlace), c.wantWipe, c.wantStamp, c.wantInPlace)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestApplyInPlaceMigrations: steps run in order and commit; a failing step
|
||||
// rolls the whole transaction back.
|
||||
func TestApplyInPlaceMigrations(t *testing.T) {
|
||||
t.Run("commit", func(t *testing.T) {
|
||||
path := filepath.Join(t.TempDir(), "m.sqlite")
|
||||
withRawDB(t, path, func(db *sql.DB) {
|
||||
step := schemaMigration{version: 2, name: "mk", inPlace: func(tx *sql.Tx) error {
|
||||
_, err := tx.Exec(`CREATE TABLE marker (x TEXT)`)
|
||||
return err
|
||||
}}
|
||||
if err := applyInPlaceMigrations(db, []schemaMigration{step}); err != nil {
|
||||
t.Fatalf("apply: %v", err)
|
||||
}
|
||||
var name string
|
||||
if err := db.QueryRow(`SELECT name FROM sqlite_master WHERE type='table' AND name='marker'`).Scan(&name); err != nil {
|
||||
t.Fatalf("marker table not created: %v", err)
|
||||
}
|
||||
})
|
||||
})
|
||||
|
||||
t.Run("rollback on failure preserves cause and rolls back every step", func(t *testing.T) {
|
||||
path := filepath.Join(t.TempDir(), "m.sqlite")
|
||||
withRawDB(t, path, func(db *sql.DB) {
|
||||
// Two steps in one batch: the first creates table A, the second
|
||||
// creates B then fails. Both must roll back, proving the steps
|
||||
// share a single transaction.
|
||||
stepA := schemaMigration{version: 2, name: "make-a", inPlace: func(tx *sql.Tx) error {
|
||||
_, err := tx.Exec(`CREATE TABLE a (x TEXT)`)
|
||||
return err
|
||||
}}
|
||||
stepB := schemaMigration{version: 3, name: "boom", inPlace: func(tx *sql.Tx) error {
|
||||
if _, err := tx.Exec(`CREATE TABLE b (x TEXT)`); err != nil {
|
||||
return err
|
||||
}
|
||||
return sql.ErrConnDone // synthetic failure after a partial write
|
||||
}}
|
||||
err := applyInPlaceMigrations(db, []schemaMigration{stepA, stepB})
|
||||
if err == nil {
|
||||
t.Fatal("expected applyInPlaceMigrations to surface the step error")
|
||||
}
|
||||
if !errors.Is(err, sql.ErrConnDone) {
|
||||
t.Fatalf("error should wrap the step's cause; got %v", err)
|
||||
}
|
||||
if !strings.Contains(err.Error(), "v3") || !strings.Contains(err.Error(), "boom") {
|
||||
t.Fatalf("error should name the failing migration (v3/boom); got %q", err.Error())
|
||||
}
|
||||
for _, tbl := range []string{"a", "b"} {
|
||||
var name string
|
||||
e := db.QueryRow(`SELECT name FROM sqlite_master WHERE type='table' AND name=?`, tbl).Scan(&name)
|
||||
if e != sql.ErrNoRows {
|
||||
t.Fatalf("table %q should have rolled back (shared transaction), got name=%q err=%v", tbl, name, e)
|
||||
}
|
||||
}
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
// TestOpenAtCurrentVersionIsNoOp covers the highest-frequency path — every
|
||||
// daemon restart reopens an up-to-date store. It must be a no-op that
|
||||
// preserves data; an off-by-one to wipe here would destroy the cache on every
|
||||
// restart.
|
||||
func TestOpenAtCurrentVersionIsNoOp(t *testing.T) {
|
||||
path := filepath.Join(t.TempDir(), "store.sqlite")
|
||||
s, err := Open(path)
|
||||
if err != nil {
|
||||
t.Fatalf("first open: %v", err)
|
||||
}
|
||||
withRawSeed := func(db *sql.DB) {
|
||||
if _, err := db.Exec(`INSERT INTO nodes (id, kind, name, file_path) VALUES ('n1','func','Foo','f.go')`); err != nil {
|
||||
t.Fatalf("seed node: %v", err)
|
||||
}
|
||||
}
|
||||
withRawSeed(s.db)
|
||||
if err := s.Close(); err != nil {
|
||||
t.Fatalf("close: %v", err)
|
||||
}
|
||||
|
||||
s2, err := Open(path)
|
||||
if err != nil {
|
||||
t.Fatalf("reopen at current version: %v", err)
|
||||
}
|
||||
defer s2.Close()
|
||||
if v, _ := readUserVersion(s2.db); v != currentSchemaVersion {
|
||||
t.Fatalf("user_version = %d, want %d", v, currentSchemaVersion)
|
||||
}
|
||||
if n := nodeCount(t, s2.db); n != 1 {
|
||||
t.Fatalf("node count = %d, want 1 (a no-op reopen must NOT wipe)", n)
|
||||
}
|
||||
if s2.NeedsRebuild() {
|
||||
t.Fatal("a no-op reopen must not signal NeedsRebuild")
|
||||
}
|
||||
}
|
||||
|
||||
// TestOpenWithInPlaceMigration drives the in-place arm end-to-end through the
|
||||
// real Open composition (via the openWith seam): an older store at version 1
|
||||
// is upgraded to version 2 by a registered in-place step that runs AFTER
|
||||
// schemaSQL, the step's effect is visible, the existing data survives, and the
|
||||
// version is stamped.
|
||||
func TestOpenWithInPlaceMigration(t *testing.T) {
|
||||
path := filepath.Join(t.TempDir(), "store.sqlite")
|
||||
|
||||
// Create a store with a row, then knock it back to the v1 baseline so the
|
||||
// openWith below drives the v1->v2 in-place arm (a fresh Open now stamps the
|
||||
// current version, which is >= 2).
|
||||
s, err := Open(path)
|
||||
if err != nil {
|
||||
t.Fatalf("first open: %v", err)
|
||||
}
|
||||
if _, err := s.db.Exec(`INSERT INTO nodes (id, kind, name, file_path) VALUES ('n1','func','Foo','f.go')`); err != nil {
|
||||
t.Fatalf("seed node: %v", err)
|
||||
}
|
||||
if err := s.Close(); err != nil {
|
||||
t.Fatalf("close: %v", err)
|
||||
}
|
||||
withRawDB(t, path, func(db *sql.DB) {
|
||||
if _, err := db.Exec(`PRAGMA user_version = 1`); err != nil {
|
||||
t.Fatalf("reset to v1 baseline: %v", err)
|
||||
}
|
||||
})
|
||||
|
||||
// An in-place v2 step that depends on the base schema (an index on a
|
||||
// nodes column) — proving it runs after schemaSQL/ensureNodeColumns.
|
||||
ran := false
|
||||
v2 := schemaMigration{version: 2, name: "idx-language", inPlace: func(tx *sql.Tx) error {
|
||||
ran = true
|
||||
_, err := tx.Exec(`CREATE INDEX IF NOT EXISTS test_nodes_by_language ON nodes(language)`)
|
||||
return err
|
||||
}}
|
||||
|
||||
s2, err := openWith(path, 2, []schemaMigration{v2}, false) // in-place never wipes
|
||||
if err != nil {
|
||||
t.Fatalf("openWith v2 in-place: %v", err)
|
||||
}
|
||||
defer s2.Close()
|
||||
if !ran {
|
||||
t.Fatal("the in-place migration step did not run")
|
||||
}
|
||||
if v, _ := readUserVersion(s2.db); v != 2 {
|
||||
t.Fatalf("user_version = %d, want 2", v)
|
||||
}
|
||||
if n := nodeCount(t, s2.db); n != 1 {
|
||||
t.Fatalf("node count = %d, want 1 (in-place upgrade must preserve data)", n)
|
||||
}
|
||||
var name string
|
||||
if err := s2.db.QueryRow(`SELECT name FROM sqlite_master WHERE type='index' AND name='test_nodes_by_language'`).Scan(&name); err != nil {
|
||||
t.Fatalf("in-place index not created: %v", err)
|
||||
}
|
||||
if s2.NeedsRebuild() {
|
||||
t.Fatal("an in-place upgrade must not signal NeedsRebuild")
|
||||
}
|
||||
}
|
||||
|
||||
// TestOpenWithInPlaceFailureDoesNotStamp: a failing in-place step makes Open
|
||||
// return an error and leaves the stored version unchanged, so the next open
|
||||
// retries the upgrade rather than treating it as done.
|
||||
func TestOpenWithInPlaceFailureDoesNotStamp(t *testing.T) {
|
||||
path := filepath.Join(t.TempDir(), "store.sqlite")
|
||||
s, err := Open(path)
|
||||
if err != nil {
|
||||
t.Fatalf("first open: %v", err)
|
||||
}
|
||||
if err := s.Close(); err != nil {
|
||||
t.Fatalf("close: %v", err)
|
||||
}
|
||||
// A fresh Open now stamps the current version (>= 2); knock it back to the
|
||||
// v1 baseline so openWith drives the v1->v2 arm and the failing step runs.
|
||||
withRawDB(t, path, func(db *sql.DB) {
|
||||
if _, err := db.Exec(`PRAGMA user_version = 1`); err != nil {
|
||||
t.Fatalf("reset to v1 baseline: %v", err)
|
||||
}
|
||||
})
|
||||
|
||||
boom := schemaMigration{version: 2, name: "boom", inPlace: func(*sql.Tx) error {
|
||||
return sql.ErrConnDone
|
||||
}}
|
||||
if _, err := openWith(path, 2, []schemaMigration{boom}, false); err == nil {
|
||||
t.Fatal("expected openWith to fail when an in-place step errors")
|
||||
}
|
||||
|
||||
withRawDB(t, path, func(db *sql.DB) {
|
||||
var v int
|
||||
if err := db.QueryRow("PRAGMA user_version").Scan(&v); err != nil {
|
||||
t.Fatalf("read user_version: %v", err)
|
||||
}
|
||||
if v != 1 {
|
||||
t.Fatalf("user_version = %d after a failed migration, want 1 (unstamped, so the next open retries)", v)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// TestOpenWithMemoryUnderWipePlanStampsWithoutError: an in-memory store under a
|
||||
// plan that would wipe an on-disk DB must not attempt a file removal — it is
|
||||
// always fresh and simply stamps the current version.
|
||||
func TestOpenWithMemoryUnderWipePlanStampsWithoutError(t *testing.T) {
|
||||
rebuildV2 := schemaMigration{version: 2, name: "typed-col", rebuild: true}
|
||||
// stored==0, current==2, a pending rebuild => plan.wipe==true; the memory
|
||||
// guard must skip the wipe and stamp anyway.
|
||||
s, err := openWith(":memory:", 2, []schemaMigration{rebuildV2}, false) // memory never wipes
|
||||
if err != nil {
|
||||
t.Fatalf("openWith :memory: under wipe plan: %v", err)
|
||||
}
|
||||
defer s.Close()
|
||||
if v, _ := readUserVersion(s.db); v != 2 {
|
||||
t.Fatalf("user_version = %d, want 2", v)
|
||||
}
|
||||
if s.NeedsRebuild() {
|
||||
t.Fatal(":memory: must never report a wipe (nothing to remove)")
|
||||
}
|
||||
}
|
||||
|
||||
// TestNeedsRebuildSignalAfterWipe: a store written by a newer build is wiped on
|
||||
// open and reports NeedsRebuild so the daemon forces a full re-index.
|
||||
func TestNeedsRebuildSignalAfterWipe(t *testing.T) {
|
||||
path := filepath.Join(t.TempDir(), "store.sqlite")
|
||||
s, err := Open(path)
|
||||
if err != nil {
|
||||
t.Fatalf("first open: %v", err)
|
||||
}
|
||||
if err := s.Close(); err != nil {
|
||||
t.Fatalf("close: %v", err)
|
||||
}
|
||||
withRawDB(t, path, func(db *sql.DB) {
|
||||
if _, err := db.Exec(`PRAGMA user_version = 999`); err != nil {
|
||||
t.Fatalf("set future version: %v", err)
|
||||
}
|
||||
})
|
||||
s2, err := Open(path, WithRebuild()) // daemon-equivalent: lock held, rebuild permitted
|
||||
if err != nil {
|
||||
t.Fatalf("reopen newer DB: %v", err)
|
||||
}
|
||||
defer s2.Close()
|
||||
if !s2.NeedsRebuild() {
|
||||
t.Fatal("a wiped store must report NeedsRebuild so the daemon re-indexes")
|
||||
}
|
||||
}
|
||||
|
||||
// TestSchemaMigrationsWellFormed asserts the shipped registry is valid and that
|
||||
// the validator rejects the dangerous misconfigurations — above all, bumping
|
||||
// currentSchemaVersion without appending a matching migration.
|
||||
func TestSchemaMigrationsWellFormed(t *testing.T) {
|
||||
if err := validateSchemaMigrations(currentSchemaVersion, schemaMigrations); err != nil {
|
||||
t.Fatalf("shipped registry is invalid: %v", err)
|
||||
}
|
||||
|
||||
inPlace := func(*sql.Tx) error { return nil }
|
||||
bad := []struct {
|
||||
name string
|
||||
current int
|
||||
migs []schemaMigration
|
||||
}{
|
||||
{"bumped version with no migration", 2, nil},
|
||||
{"highest below current", 3, []schemaMigration{{version: 2, name: "x", rebuild: true}}},
|
||||
{"both strategies set", 2, []schemaMigration{{version: 2, name: "x", rebuild: true, inPlace: inPlace}}},
|
||||
{"neither strategy set", 2, []schemaMigration{{version: 2, name: "x"}}},
|
||||
{"not strictly ascending", 3, []schemaMigration{{version: 2, name: "a", rebuild: true}, {version: 2, name: "b", rebuild: true}}},
|
||||
{"v1 entry (baseline is implicit)", 1, []schemaMigration{{version: 1, name: "a", rebuild: true}}},
|
||||
}
|
||||
for _, c := range bad {
|
||||
if err := validateSchemaMigrations(c.current, c.migs); err == nil {
|
||||
t.Errorf("%s: expected a validation error, got nil", c.name)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestOpenDedupesFnValuePlaceholders drives the shipped v2 migration through the
|
||||
// real Open composition: a store knocked back to the v1 baseline with duplicate
|
||||
// fn-value placeholder edges is deduped in place on reopen — one survivor per
|
||||
// (from_id, to_id), the MIN(id) row kept — while a distinct placeholder, a
|
||||
// resolved edge, and an ordinary unresolved stub are untouched, and the version
|
||||
// stamps to current. Covers both the bare and the multi-repo COPY-rewrite form.
|
||||
func TestOpenDedupesFnValuePlaceholders(t *testing.T) {
|
||||
path := filepath.Join(t.TempDir(), "store.sqlite")
|
||||
|
||||
s, err := Open(path)
|
||||
if err != nil {
|
||||
t.Fatalf("first open: %v", err)
|
||||
}
|
||||
// Seed edges by explicit id so the MIN(id) survivors are predictable. The
|
||||
// is_unresolved column is generated, so it is omitted from the INSERT.
|
||||
ins := `INSERT INTO edges (id, from_id, to_id, kind, file_path, line) VALUES (?,?,?,?,?,?)`
|
||||
seed := []struct {
|
||||
id int
|
||||
from, to string
|
||||
kind string
|
||||
line int
|
||||
}{
|
||||
// Duplicate bare placeholders: same (from,to), distinct lines. Keep id 1.
|
||||
{1, "a", "unresolved::fnvalue::handler", "references", 10},
|
||||
{2, "a", "unresolved::fnvalue::handler", "references", 20},
|
||||
{3, "a", "unresolved::fnvalue::handler", "references", 30},
|
||||
// A distinct placeholder (different name) — must survive untouched.
|
||||
{4, "a", "unresolved::fnvalue::other", "references", 10},
|
||||
// Duplicate multi-repo COPY-rewrite placeholders — exercises the
|
||||
// is_unresolved infix branch of the migration. Keep id 5.
|
||||
{5, "b", "r::unresolved::fnvalue::handler", "references", 10},
|
||||
{6, "b", "r::unresolved::fnvalue::handler", "references", 20},
|
||||
// A resolved edge and an ordinary unresolved stub — never touched.
|
||||
{7, "a", "b", "calls", 1},
|
||||
{8, "a", "unresolved::Foo", "calls", 1},
|
||||
}
|
||||
for _, r := range seed {
|
||||
if _, err := s.db.Exec(ins, r.id, r.from, r.to, r.kind, "f.go", r.line); err != nil {
|
||||
t.Fatalf("seed edge %d: %v", r.id, err)
|
||||
}
|
||||
}
|
||||
if err := s.Close(); err != nil {
|
||||
t.Fatalf("close: %v", err)
|
||||
}
|
||||
// Knock the store back to the v1 baseline so the reopen runs the v2 dedup.
|
||||
withRawDB(t, path, func(db *sql.DB) {
|
||||
if _, err := db.Exec(`PRAGMA user_version = 1`); err != nil {
|
||||
t.Fatalf("reset to v1 baseline: %v", err)
|
||||
}
|
||||
})
|
||||
|
||||
s2, err := Open(path)
|
||||
if err != nil {
|
||||
t.Fatalf("reopen for dedup: %v", err)
|
||||
}
|
||||
defer s2.Close()
|
||||
|
||||
if v, _ := readUserVersion(s2.db); v != currentSchemaVersion {
|
||||
t.Fatalf("user_version after dedup = %d, want %d", v, currentSchemaVersion)
|
||||
}
|
||||
|
||||
present := func(id int) bool {
|
||||
var n int
|
||||
if err := s2.db.QueryRow(`SELECT COUNT(*) FROM edges WHERE id = ?`, id).Scan(&n); err != nil {
|
||||
t.Fatalf("count id %d: %v", id, err)
|
||||
}
|
||||
return n == 1
|
||||
}
|
||||
// Bare-form dedup keeps the MIN(id) survivor and drops the rest.
|
||||
if !present(1) || present(2) || present(3) {
|
||||
t.Fatalf("bare dedup wrong: want keep 1 / drop 2,3; got 1=%v 2=%v 3=%v", present(1), present(2), present(3))
|
||||
}
|
||||
// A distinct placeholder pair survives.
|
||||
if !present(4) {
|
||||
t.Fatal("distinct fn-value placeholder (id 4) was wrongly deleted")
|
||||
}
|
||||
// Multi-repo infix dedup keeps the MIN(id) survivor and drops the rest.
|
||||
if !present(5) || present(6) {
|
||||
t.Fatalf("multi-repo dedup wrong: want keep 5 / drop 6; got 5=%v 6=%v", present(5), present(6))
|
||||
}
|
||||
// A resolved edge and an ordinary unresolved stub must be untouched.
|
||||
if !present(7) {
|
||||
t.Fatal("resolved edge (id 7) must survive the placeholder dedup")
|
||||
}
|
||||
if !present(8) {
|
||||
t.Fatal("ordinary unresolved stub (id 8) must survive the placeholder dedup")
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,639 @@
|
||||
package store_sqlite
|
||||
|
||||
import (
|
||||
"iter"
|
||||
"sort"
|
||||
|
||||
"github.com/zzet/gortex/internal/graph"
|
||||
)
|
||||
|
||||
// This file implements the trivial SQL aggregator / scanner optional
|
||||
// capability interfaces from graph.Store. Each method pushes its
|
||||
// GROUP BY / WHERE / COUNT into SQLite so the planner drives it through
|
||||
// the schema's secondary indexes, returning only the aggregate rows
|
||||
// instead of materialising the whole node / edge table Go-side.
|
||||
//
|
||||
// Conventions shared across these methods:
|
||||
// - Empty / nil input returns nil (parity with the in-memory store).
|
||||
// - Input id / kind slices are deduped before they reach the IN-list.
|
||||
// - Large IN-lists are chunked by lookupChunkSize.
|
||||
// - agg-prefixed helpers are local to this file.
|
||||
|
||||
var (
|
||||
_ graph.InEdgeCounter = (*Store)(nil)
|
||||
_ graph.NodeIDsByKinds = (*Store)(nil)
|
||||
_ graph.EdgeKindCounter = (*Store)(nil)
|
||||
_ graph.NodeDegreeByKinds = (*Store)(nil)
|
||||
_ graph.NodesInFilesByKindFinder = (*Store)(nil)
|
||||
_ graph.FileImportAggregator = (*Store)(nil)
|
||||
_ graph.InDegreeForNodes = (*Store)(nil)
|
||||
_ graph.CrossRepoEdgeAggregator = (*Store)(nil)
|
||||
_ graph.FileImporters = (*Store)(nil)
|
||||
_ graph.FileSymbolNamesByPaths = (*Store)(nil)
|
||||
_ graph.EdgesByKindsScanner = (*Store)(nil)
|
||||
_ graph.NodesByKindsScanner = (*Store)(nil)
|
||||
_ graph.EdgeAdjacencyForKinds = (*Store)(nil)
|
||||
_ graph.NodeDegreeAggregator = (*Store)(nil)
|
||||
_ graph.NodeFanAggregator = (*Store)(nil)
|
||||
)
|
||||
|
||||
// aggDedupeEdgeKinds drops empties and duplicates from an edge-kind
|
||||
// slice, preserving first-seen order; returns the kinds widened to the
|
||||
// []any an IN-list binds.
|
||||
func aggDedupeEdgeKinds(kinds []graph.EdgeKind) (uniq []graph.EdgeKind, args []any) {
|
||||
seen := make(map[graph.EdgeKind]struct{}, len(kinds))
|
||||
for _, k := range kinds {
|
||||
if k == "" {
|
||||
continue
|
||||
}
|
||||
if _, ok := seen[k]; ok {
|
||||
continue
|
||||
}
|
||||
seen[k] = struct{}{}
|
||||
uniq = append(uniq, k)
|
||||
args = append(args, string(k))
|
||||
}
|
||||
return uniq, args
|
||||
}
|
||||
|
||||
// aggDedupeNodeKinds is the node-kind twin of aggDedupeEdgeKinds.
|
||||
func aggDedupeNodeKinds(kinds []graph.NodeKind) (uniq []graph.NodeKind, args []any) {
|
||||
seen := make(map[graph.NodeKind]struct{}, len(kinds))
|
||||
for _, k := range kinds {
|
||||
if k == "" {
|
||||
continue
|
||||
}
|
||||
if _, ok := seen[k]; ok {
|
||||
continue
|
||||
}
|
||||
seen[k] = struct{}{}
|
||||
uniq = append(uniq, k)
|
||||
args = append(args, string(k))
|
||||
}
|
||||
return uniq, args
|
||||
}
|
||||
|
||||
// InEdgeCountsByKind returns per-target incoming-edge counts for the
|
||||
// supplied edge kinds, grouped server-side via edges_by_to.
|
||||
func (s *Store) InEdgeCountsByKind(kinds []graph.EdgeKind) map[string]int {
|
||||
_, args := aggDedupeEdgeKinds(kinds)
|
||||
if len(args) == 0 {
|
||||
return nil
|
||||
}
|
||||
q := `SELECT to_id, COUNT(*) FROM edges WHERE kind IN (` + inPlaceholders(len(args)) + `) GROUP BY to_id`
|
||||
rows, err := s.db.Query(q, args...)
|
||||
panicOnFatal(err)
|
||||
if rows == nil {
|
||||
// swallowed teardown-race error: read returns empty (see panicOnFatal)
|
||||
return nil
|
||||
}
|
||||
defer rows.Close()
|
||||
out := make(map[string]int)
|
||||
for rows.Next() {
|
||||
var id string
|
||||
var n int
|
||||
panicOnFatal(rows.Scan(&id, &n))
|
||||
out[id] = n
|
||||
}
|
||||
panicOnFatal(rows.Err())
|
||||
return out
|
||||
}
|
||||
|
||||
// NodeIDsByKinds returns the deduplicated IDs of every node whose kind
|
||||
// is in the supplied set.
|
||||
func (s *Store) NodeIDsByKinds(kinds []graph.NodeKind) []string {
|
||||
_, args := aggDedupeNodeKinds(kinds)
|
||||
if len(args) == 0 {
|
||||
return nil
|
||||
}
|
||||
q := `SELECT id FROM nodes WHERE kind IN (` + inPlaceholders(len(args)) + `) ORDER BY id`
|
||||
rows, err := s.db.Query(q, args...)
|
||||
panicOnFatal(err)
|
||||
if rows == nil {
|
||||
// swallowed teardown-race error: read returns empty (see panicOnFatal)
|
||||
return nil
|
||||
}
|
||||
defer rows.Close()
|
||||
var out []string
|
||||
for rows.Next() {
|
||||
var id string
|
||||
panicOnFatal(rows.Scan(&id))
|
||||
out = append(out, id)
|
||||
}
|
||||
panicOnFatal(rows.Err())
|
||||
return out
|
||||
}
|
||||
|
||||
// EdgeKindCounts returns one entry per distinct edge kind with its
|
||||
// occurrence count across the whole graph.
|
||||
func (s *Store) EdgeKindCounts() map[graph.EdgeKind]int {
|
||||
rows, err := s.db.Query(`SELECT kind, COUNT(*) FROM edges GROUP BY kind`)
|
||||
panicOnFatal(err)
|
||||
if rows == nil {
|
||||
// swallowed teardown-race error: read returns empty (see panicOnFatal)
|
||||
return nil
|
||||
}
|
||||
defer rows.Close()
|
||||
out := make(map[graph.EdgeKind]int)
|
||||
for rows.Next() {
|
||||
var kind string
|
||||
var n int
|
||||
panicOnFatal(rows.Scan(&kind, &n))
|
||||
out[graph.EdgeKind(kind)] = n
|
||||
}
|
||||
panicOnFatal(rows.Err())
|
||||
return out
|
||||
}
|
||||
|
||||
// NodeDegreeByKinds returns total in/out degree for every node whose
|
||||
// kind is in the set (optionally under pathPrefix); UsageInCount is
|
||||
// always 0 for this capability.
|
||||
func (s *Store) NodeDegreeByKinds(kinds []graph.NodeKind, pathPrefix string) []graph.NodeDegreeRow {
|
||||
_, kindArgs := aggDedupeNodeKinds(kinds)
|
||||
if len(kindArgs) == 0 {
|
||||
return nil
|
||||
}
|
||||
args := append([]any(nil), kindArgs...)
|
||||
q := `SELECT n.id,
|
||||
(SELECT COUNT(*) FROM edges e WHERE e.to_id = n.id) AS in_count,
|
||||
(SELECT COUNT(*) FROM edges e WHERE e.from_id = n.id) AS out_count
|
||||
FROM nodes n
|
||||
WHERE n.kind IN (` + inPlaceholders(len(kindArgs)) + `)`
|
||||
if pathPrefix != "" {
|
||||
q += ` AND n.file_path LIKE ? ESCAPE '\'`
|
||||
args = append(args, escapeLikePattern(pathPrefix)+"%")
|
||||
}
|
||||
q += ` ORDER BY n.id`
|
||||
rows, err := s.db.Query(q, args...)
|
||||
panicOnFatal(err)
|
||||
if rows == nil {
|
||||
// swallowed teardown-race error: read returns empty (see panicOnFatal)
|
||||
return nil
|
||||
}
|
||||
defer rows.Close()
|
||||
var out []graph.NodeDegreeRow
|
||||
for rows.Next() {
|
||||
var r graph.NodeDegreeRow
|
||||
panicOnFatal(rows.Scan(&r.NodeID, &r.InCount, &r.OutCount))
|
||||
out = append(out, r)
|
||||
}
|
||||
panicOnFatal(rows.Err())
|
||||
return out
|
||||
}
|
||||
|
||||
// NodesInFilesByKind returns every node living in one of the supplied
|
||||
// files whose kind is in the supplied set.
|
||||
func (s *Store) NodesInFilesByKind(files []string, kinds []graph.NodeKind) []*graph.Node {
|
||||
uniqFiles := dedupeNonEmpty(files)
|
||||
_, kindArgs := aggDedupeNodeKinds(kinds)
|
||||
if len(uniqFiles) == 0 || len(kindArgs) == 0 {
|
||||
return nil
|
||||
}
|
||||
var out []*graph.Node
|
||||
for i := 0; i < len(uniqFiles); i += lookupChunkSize {
|
||||
end := minInt(i+lookupChunkSize, len(uniqFiles))
|
||||
chunk := uniqFiles[i:end]
|
||||
args := append(toAnyArgs(chunk), kindArgs...)
|
||||
q := `SELECT ` + lookupNodeCols + ` FROM nodes WHERE file_path IN (` +
|
||||
inPlaceholders(len(chunk)) + `) AND kind IN (` + inPlaceholders(len(kindArgs)) + `) ORDER BY id`
|
||||
out = append(out, s.queryNodesSQL(q, args...)...)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// FileImportCounts returns per-target-file incoming-import counts. A
|
||||
// nil scope counts every import edge; a non-nil scope bounds counts to
|
||||
// edges whose target node ID lies in the slice (empty non-nil => nil).
|
||||
func (s *Store) FileImportCounts(scope []string) []graph.FileImportCountRow {
|
||||
if scope != nil && len(scope) == 0 {
|
||||
return nil
|
||||
}
|
||||
base := `SELECT COALESCE(NULLIF(n.file_path, ''), n.id) AS path, COUNT(*) AS cnt
|
||||
FROM edges e JOIN nodes n ON e.to_id = n.id
|
||||
WHERE e.kind = ?`
|
||||
args := []any{string(graph.EdgeImports)}
|
||||
fileToCount := make(map[string]int)
|
||||
if scope == nil {
|
||||
q := base + ` GROUP BY path`
|
||||
aggScanImportCounts(s, q, args, fileToCount)
|
||||
} else {
|
||||
uniq := dedupeNonEmpty(scope)
|
||||
if len(uniq) == 0 {
|
||||
return nil
|
||||
}
|
||||
for i := 0; i < len(uniq); i += lookupChunkSize {
|
||||
end := minInt(i+lookupChunkSize, len(uniq))
|
||||
chunk := uniq[i:end]
|
||||
q := base + ` AND e.to_id IN (` + inPlaceholders(len(chunk)) + `) GROUP BY path`
|
||||
aggScanImportCounts(s, q, append(append([]any(nil), args...), toAnyArgs(chunk)...), fileToCount)
|
||||
}
|
||||
}
|
||||
if len(fileToCount) == 0 {
|
||||
return nil
|
||||
}
|
||||
out := make([]graph.FileImportCountRow, 0, len(fileToCount))
|
||||
for path, cnt := range fileToCount {
|
||||
out = append(out, graph.FileImportCountRow{FilePath: path, Count: cnt})
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// aggScanImportCounts runs an import-count query and folds the (path,
|
||||
// count) rows into the accumulator (chunked scopes can revisit a path).
|
||||
func aggScanImportCounts(s *Store, q string, args []any, acc map[string]int) {
|
||||
rows, err := s.db.Query(q, args...)
|
||||
panicOnFatal(err)
|
||||
if rows == nil {
|
||||
// swallowed teardown-race error: read returns empty (see panicOnFatal)
|
||||
return
|
||||
}
|
||||
defer rows.Close()
|
||||
for rows.Next() {
|
||||
var path string
|
||||
var cnt int
|
||||
panicOnFatal(rows.Scan(&path, &cnt))
|
||||
acc[path] += cnt
|
||||
}
|
||||
panicOnFatal(rows.Err())
|
||||
}
|
||||
|
||||
// InDegreeForNodes returns total incoming-edge counts (any kind) for
|
||||
// the supplied node id set.
|
||||
func (s *Store) InDegreeForNodes(ids []string) map[string]int {
|
||||
uniq := dedupeNonEmpty(ids)
|
||||
if len(uniq) == 0 {
|
||||
return nil
|
||||
}
|
||||
out := make(map[string]int)
|
||||
for i := 0; i < len(uniq); i += lookupChunkSize {
|
||||
end := minInt(i+lookupChunkSize, len(uniq))
|
||||
chunk := uniq[i:end]
|
||||
q := `SELECT to_id, COUNT(*) FROM edges WHERE to_id IN (` +
|
||||
inPlaceholders(len(chunk)) + `) GROUP BY to_id`
|
||||
rows, err := s.db.Query(q, toAnyArgs(chunk)...)
|
||||
panicOnFatal(err)
|
||||
if rows == nil {
|
||||
// swallowed teardown-race error: read returns empty (see panicOnFatal)
|
||||
return out
|
||||
}
|
||||
for rows.Next() {
|
||||
var id string
|
||||
var n int
|
||||
panicOnFatal(rows.Scan(&id, &n))
|
||||
out[id] = n
|
||||
}
|
||||
panicOnFatal(rows.Err())
|
||||
_ = rows.Close()
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// CrossRepoEdgeCounts returns pre-grouped cross-repo edge counts keyed
|
||||
// by (base kind, from-repo, to-repo). Cross-repo kinds are those
|
||||
// graph.BaseKindForCrossRepo recognises; the count is reported under
|
||||
// the base kind.
|
||||
func (s *Store) CrossRepoEdgeCounts() []graph.CrossRepoEdgeRow {
|
||||
q := `SELECT e.kind, nf.repo_prefix, nt.repo_prefix, COUNT(*)
|
||||
FROM edges e
|
||||
JOIN nodes nf ON e.from_id = nf.id
|
||||
JOIN nodes nt ON e.to_id = nt.id
|
||||
WHERE nf.repo_prefix <> nt.repo_prefix
|
||||
GROUP BY e.kind, nf.repo_prefix, nt.repo_prefix`
|
||||
rows, err := s.db.Query(q)
|
||||
panicOnFatal(err)
|
||||
if rows == nil {
|
||||
// swallowed teardown-race error: read returns empty (see panicOnFatal)
|
||||
return nil
|
||||
}
|
||||
defer rows.Close()
|
||||
// Aggregate keyed by the edge's OWN kind (cross_repo_*), NOT the base.
|
||||
// BaseKindForCrossRepo is used only as the recogniser that decides
|
||||
// whether an edge participates — parity with the in-memory store.
|
||||
type key struct {
|
||||
kind graph.EdgeKind
|
||||
from string
|
||||
to string
|
||||
}
|
||||
acc := make(map[key]int)
|
||||
for rows.Next() {
|
||||
var kind, from, to string
|
||||
var n int
|
||||
panicOnFatal(rows.Scan(&kind, &from, &to, &n))
|
||||
ek := graph.EdgeKind(kind)
|
||||
if _, ok := graph.BaseKindForCrossRepo(ek); !ok {
|
||||
continue
|
||||
}
|
||||
acc[key{kind: ek, from: from, to: to}] += n
|
||||
}
|
||||
panicOnFatal(rows.Err())
|
||||
if len(acc) == 0 {
|
||||
return nil
|
||||
}
|
||||
out := make([]graph.CrossRepoEdgeRow, 0, len(acc))
|
||||
for k, n := range acc {
|
||||
out = append(out, graph.CrossRepoEdgeRow{Kind: k.kind, FromRepo: k.from, ToRepo: k.to, Count: n})
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// FileImporters returns the importing-node rows for every EdgeImports
|
||||
// edge whose target's FilePath OR ID equals filePath.
|
||||
func (s *Store) FileImporters(filePath string) []graph.FileImporterRow {
|
||||
if filePath == "" {
|
||||
return nil
|
||||
}
|
||||
q := `SELECT nf.file_path, nf.id, nf.name, nf.kind
|
||||
FROM edges e
|
||||
JOIN nodes nt ON e.to_id = nt.id
|
||||
JOIN nodes nf ON e.from_id = nf.id
|
||||
WHERE e.kind = ? AND (nt.file_path = ? OR nt.id = ?)
|
||||
ORDER BY nf.file_path`
|
||||
rows, err := s.db.Query(q, string(graph.EdgeImports), filePath, filePath)
|
||||
panicOnFatal(err)
|
||||
if rows == nil {
|
||||
// swallowed teardown-race error: read returns empty (see panicOnFatal)
|
||||
return nil
|
||||
}
|
||||
defer rows.Close()
|
||||
var out []graph.FileImporterRow
|
||||
for rows.Next() {
|
||||
var r graph.FileImporterRow
|
||||
var kind string
|
||||
panicOnFatal(rows.Scan(&r.FromFile, &r.FromID, &r.FromName, &kind))
|
||||
r.FromKind = graph.NodeKind(kind)
|
||||
out = append(out, r)
|
||||
}
|
||||
panicOnFatal(rows.Err())
|
||||
return out
|
||||
}
|
||||
|
||||
// FileSymbolNamesByPaths returns the distinct (file, name) pairs for
|
||||
// nodes in the supplied paths whose kind is in the set, sorted by
|
||||
// (file, name).
|
||||
func (s *Store) FileSymbolNamesByPaths(paths []string, kinds []graph.NodeKind) []graph.FileSymbolNameRow {
|
||||
uniqPaths := dedupeNonEmpty(paths)
|
||||
_, kindArgs := aggDedupeNodeKinds(kinds)
|
||||
if len(uniqPaths) == 0 || len(kindArgs) == 0 {
|
||||
return nil
|
||||
}
|
||||
var out []graph.FileSymbolNameRow
|
||||
for i := 0; i < len(uniqPaths); i += lookupChunkSize {
|
||||
end := minInt(i+lookupChunkSize, len(uniqPaths))
|
||||
chunk := uniqPaths[i:end]
|
||||
args := append(toAnyArgs(chunk), kindArgs...)
|
||||
q := `SELECT DISTINCT file_path, name FROM nodes WHERE file_path IN (` +
|
||||
inPlaceholders(len(chunk)) + `) AND kind IN (` + inPlaceholders(len(kindArgs)) + `)`
|
||||
rows, err := s.db.Query(q, args...)
|
||||
panicOnFatal(err)
|
||||
if rows == nil {
|
||||
// swallowed teardown-race error: read returns empty (see panicOnFatal)
|
||||
return out
|
||||
}
|
||||
for rows.Next() {
|
||||
var r graph.FileSymbolNameRow
|
||||
panicOnFatal(rows.Scan(&r.FilePath, &r.Name))
|
||||
out = append(out, r)
|
||||
}
|
||||
panicOnFatal(rows.Err())
|
||||
_ = rows.Close()
|
||||
}
|
||||
sort.Slice(out, func(i, j int) bool {
|
||||
if out[i].FilePath != out[j].FilePath {
|
||||
return out[i].FilePath < out[j].FilePath
|
||||
}
|
||||
return out[i].Name < out[j].Name
|
||||
})
|
||||
return out
|
||||
}
|
||||
|
||||
// EdgesByKinds streams every edge whose kind is in the supplied set;
|
||||
// honours early-stop. Empty kinds yields nothing.
|
||||
func (s *Store) EdgesByKinds(kinds []graph.EdgeKind) iter.Seq[*graph.Edge] {
|
||||
_, args := aggDedupeEdgeKinds(kinds)
|
||||
return func(yield func(*graph.Edge) bool) {
|
||||
if len(args) == 0 {
|
||||
return
|
||||
}
|
||||
q := `SELECT ` + lookupEdgeCols + ` FROM edges WHERE kind IN (` +
|
||||
inPlaceholders(len(args)) + `) ORDER BY id`
|
||||
for _, e := range s.queryEdgesSQL(q, args...) {
|
||||
if e == nil {
|
||||
continue
|
||||
}
|
||||
if !yield(e) {
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// externalCallTargetPredicate selects edges whose target is an
|
||||
// external-package terminal (dep:: / stdlib:: / external::, including the
|
||||
// per-repo-prefixed stdlib form) or an already-materialised
|
||||
// external-call:: node. Shared verbatim by ExternalCallCandidateEdges and
|
||||
// the edges_external partial index (schema.go) so SQLite matches the
|
||||
// partial index for the query — keep the two identical.
|
||||
const externalCallTargetPredicate = `(to_id GLOB 'dep::*' OR to_id GLOB 'external::*' OR to_id GLOB 'stdlib::*' OR to_id GLOB '*::stdlib::*' OR to_id GLOB 'external-call::*')`
|
||||
|
||||
// ExternalCallCandidateEdges implements graph.ExternalCallCandidates: it
|
||||
// returns only the call / reference edges the external-call synthesizer
|
||||
// might act on, selected server-side so the whole call-edge table never
|
||||
// crosses into Go just to be prefix-filtered. The GLOB predicate is
|
||||
// served by the edges_external partial index.
|
||||
func (s *Store) ExternalCallCandidateEdges() []*graph.Edge {
|
||||
q := `SELECT ` + lookupEdgeCols + ` FROM edges
|
||||
WHERE kind IN ('calls','references') AND ` + externalCallTargetPredicate + `
|
||||
ORDER BY id`
|
||||
return s.queryEdgesSQL(q)
|
||||
}
|
||||
|
||||
// NodesByKinds returns every node whose kind is in the supplied set.
|
||||
func (s *Store) NodesByKinds(kinds []graph.NodeKind) []*graph.Node {
|
||||
_, args := aggDedupeNodeKinds(kinds)
|
||||
if len(args) == 0 {
|
||||
return nil
|
||||
}
|
||||
q := `SELECT ` + lookupNodeCols + ` FROM nodes WHERE kind IN (` +
|
||||
inPlaceholders(len(args)) + `) ORDER BY id`
|
||||
return s.queryNodesSQL(q, args...)
|
||||
}
|
||||
|
||||
// EdgeAdjacencyForKinds streams (from, to) id pairs for edges whose
|
||||
// kind is in edgeKinds and whose endpoints both have a kind in
|
||||
// nodeKinds; honours early-stop. Empty kinds yields nothing.
|
||||
func (s *Store) EdgeAdjacencyForKinds(edgeKinds []graph.EdgeKind, nodeKinds []graph.NodeKind) iter.Seq[[2]string] {
|
||||
_, eArgs := aggDedupeEdgeKinds(edgeKinds)
|
||||
_, nArgs := aggDedupeNodeKinds(nodeKinds)
|
||||
return func(yield func([2]string) bool) {
|
||||
if len(eArgs) == 0 || len(nArgs) == 0 {
|
||||
return
|
||||
}
|
||||
args := append([]any(nil), eArgs...)
|
||||
args = append(args, nArgs...)
|
||||
args = append(args, nArgs...)
|
||||
q := `SELECT e.from_id, e.to_id
|
||||
FROM edges e
|
||||
JOIN nodes nf ON e.from_id = nf.id
|
||||
JOIN nodes nt ON e.to_id = nt.id
|
||||
WHERE e.kind IN (` + inPlaceholders(len(eArgs)) + `)
|
||||
AND nf.kind IN (` + inPlaceholders(len(nArgs)) + `)
|
||||
AND nt.kind IN (` + inPlaceholders(len(nArgs)) + `)`
|
||||
rows, err := s.db.Query(q, args...)
|
||||
panicOnFatal(err)
|
||||
if rows == nil {
|
||||
// swallowed teardown-race error: read returns empty (see panicOnFatal)
|
||||
return
|
||||
}
|
||||
defer rows.Close()
|
||||
for rows.Next() {
|
||||
var from, to string
|
||||
panicOnFatal(rows.Scan(&from, &to))
|
||||
if !yield([2]string{from, to}) {
|
||||
return
|
||||
}
|
||||
}
|
||||
panicOnFatal(rows.Err())
|
||||
}
|
||||
}
|
||||
|
||||
// NodeDegreeCounts returns per-node in/out/usage-in edge counts for the
|
||||
// supplied id set. Unknown ids produce no row; duplicates collapse.
|
||||
func (s *Store) NodeDegreeCounts(ids []string, usageKinds []graph.EdgeKind) []graph.NodeDegreeRow {
|
||||
uniq := dedupeNonEmpty(ids)
|
||||
if len(uniq) == 0 {
|
||||
return nil
|
||||
}
|
||||
_, usageArgs := aggDedupeEdgeKinds(usageKinds)
|
||||
out := make([]graph.NodeDegreeRow, 0, len(uniq))
|
||||
for i := 0; i < len(uniq); i += lookupChunkSize {
|
||||
end := minInt(i+lookupChunkSize, len(uniq))
|
||||
chunk := uniq[i:end]
|
||||
// Usage-in subquery: a literal 0 when no usage kinds are given.
|
||||
usageExpr := `0`
|
||||
var usageInline []any
|
||||
if len(usageArgs) > 0 {
|
||||
usageExpr = `(SELECT COUNT(*) FROM edges e WHERE e.to_id = n.id AND e.kind IN (` +
|
||||
inPlaceholders(len(usageArgs)) + `))`
|
||||
usageInline = usageArgs
|
||||
}
|
||||
q := `SELECT n.id,
|
||||
(SELECT COUNT(*) FROM edges e WHERE e.to_id = n.id) AS in_count,
|
||||
(SELECT COUNT(*) FROM edges e WHERE e.from_id = n.id) AS out_count,
|
||||
` + usageExpr + ` AS usage_in
|
||||
FROM nodes n
|
||||
WHERE n.id IN (` + inPlaceholders(len(chunk)) + `)`
|
||||
// Bind order matches placeholder order: usage subquery first
|
||||
// (it appears earlier in the SELECT list), then the id IN-list.
|
||||
args := append(append([]any(nil), usageInline...), toAnyArgs(chunk)...)
|
||||
rows, err := s.db.Query(q, args...)
|
||||
panicOnFatal(err)
|
||||
if rows == nil {
|
||||
// swallowed teardown-race error: read returns empty (see panicOnFatal)
|
||||
return out
|
||||
}
|
||||
for rows.Next() {
|
||||
var r graph.NodeDegreeRow
|
||||
panicOnFatal(rows.Scan(&r.NodeID, &r.InCount, &r.OutCount, &r.UsageInCount))
|
||||
out = append(out, r)
|
||||
}
|
||||
panicOnFatal(rows.Err())
|
||||
_ = rows.Close()
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// NodeFanCounts returns per-node fan-in (incoming edges in fanInKinds)
|
||||
// and fan-out (outgoing edges in fanOutKinds) for the supplied id set.
|
||||
// Unknown ids produce no row; duplicates collapse.
|
||||
func (s *Store) NodeFanCounts(ids []string, fanInKinds, fanOutKinds []graph.EdgeKind) []graph.NodeFanRow {
|
||||
uniq := dedupeNonEmpty(ids)
|
||||
if len(uniq) == 0 {
|
||||
return nil
|
||||
}
|
||||
_, inArgs := aggDedupeEdgeKinds(fanInKinds)
|
||||
_, outArgs := aggDedupeEdgeKinds(fanOutKinds)
|
||||
out := make([]graph.NodeFanRow, 0, len(uniq))
|
||||
for i := 0; i < len(uniq); i += lookupChunkSize {
|
||||
end := minInt(i+lookupChunkSize, len(uniq))
|
||||
chunk := uniq[i:end]
|
||||
|
||||
fanInExpr := `0`
|
||||
var inInline []any
|
||||
if len(inArgs) > 0 {
|
||||
fanInExpr = `(SELECT COUNT(*) FROM edges e WHERE e.to_id = n.id AND e.kind IN (` +
|
||||
inPlaceholders(len(inArgs)) + `))`
|
||||
inInline = inArgs
|
||||
}
|
||||
fanOutExpr := `0`
|
||||
var outInline []any
|
||||
if len(outArgs) > 0 {
|
||||
fanOutExpr = `(SELECT COUNT(*) FROM edges e WHERE e.from_id = n.id AND e.kind IN (` +
|
||||
inPlaceholders(len(outArgs)) + `))`
|
||||
outInline = outArgs
|
||||
}
|
||||
q := `SELECT n.id, ` + fanInExpr + ` AS fan_in, ` + fanOutExpr + ` AS fan_out
|
||||
FROM nodes n
|
||||
WHERE n.id IN (` + inPlaceholders(len(chunk)) + `)`
|
||||
// Bind order matches placeholder order in the SELECT list:
|
||||
// fan-in subquery, fan-out subquery, then the id IN-list.
|
||||
args := append([]any(nil), inInline...)
|
||||
args = append(args, outInline...)
|
||||
args = append(args, toAnyArgs(chunk)...)
|
||||
rows, err := s.db.Query(q, args...)
|
||||
panicOnFatal(err)
|
||||
if rows == nil {
|
||||
// swallowed teardown-race error: read returns empty (see panicOnFatal)
|
||||
return out
|
||||
}
|
||||
for rows.Next() {
|
||||
var r graph.NodeFanRow
|
||||
panicOnFatal(rows.Scan(&r.NodeID, &r.FanIn, &r.FanOut))
|
||||
out = append(out, r)
|
||||
}
|
||||
panicOnFatal(rows.Err())
|
||||
_ = rows.Close()
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// CommunityCrossingsByKind returns per-source crossing counts for edges
|
||||
// whose kind is in the supplied set, given a node→community map. A
|
||||
// crossing is an edge whose source community differs from its target
|
||||
// community; zero-count sources are dropped. Empty kinds or empty
|
||||
// community map returns nil. The community comparison runs Go-side
|
||||
// because community membership is not a node column.
|
||||
func (s *Store) CommunityCrossingsByKind(kinds []graph.EdgeKind, nodeToComm map[string]string) map[string]int {
|
||||
_, args := aggDedupeEdgeKinds(kinds)
|
||||
if len(args) == 0 || len(nodeToComm) == 0 {
|
||||
return nil
|
||||
}
|
||||
q := `SELECT from_id, to_id FROM edges WHERE kind IN (` + inPlaceholders(len(args)) + `)`
|
||||
rows, err := s.db.Query(q, args...)
|
||||
panicOnFatal(err)
|
||||
if rows == nil {
|
||||
// swallowed teardown-race error: read returns empty (see panicOnFatal)
|
||||
return nil
|
||||
}
|
||||
defer rows.Close()
|
||||
out := make(map[string]int)
|
||||
for rows.Next() {
|
||||
var from, to string
|
||||
panicOnFatal(rows.Scan(&from, &to))
|
||||
fromComm, ok := nodeToComm[from]
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
toComm, ok := nodeToComm[to]
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
if fromComm != toComm {
|
||||
out[from]++
|
||||
}
|
||||
}
|
||||
panicOnFatal(rows.Err())
|
||||
if len(out) == 0 {
|
||||
return nil
|
||||
}
|
||||
return out
|
||||
}
|
||||
@@ -0,0 +1,500 @@
|
||||
package store_sqlite
|
||||
|
||||
// This file implements the moderate-SQL analysis capability interfaces
|
||||
// for the SQLite graph.Store backend. Each method mirrors the in-memory
|
||||
// reference implementation in internal/graph/graph.go and is verified
|
||||
// against the same conformance suite (internal/graph/storetest).
|
||||
//
|
||||
// Shape: push the structural filter into one indexed SELECT via the raw-
|
||||
// SQL helpers (queryNodesSQL / s.db.Query), then do any Meta-dependent
|
||||
// (JSON-decoded) or distinct-counting filtering in Go. No new prepared
|
||||
// statements are added — every query rides the secondary indexes already
|
||||
// created in schema.go (edges_by_from / edges_by_to / nodes_by_kind).
|
||||
|
||||
import (
|
||||
"github.com/zzet/gortex/internal/graph"
|
||||
)
|
||||
|
||||
// Compile-time assertions: *Store satisfies each analysis capability.
|
||||
var _ graph.DeadCodeCandidator = (*Store)(nil)
|
||||
var _ graph.IfaceImplementsScanner = (*Store)(nil)
|
||||
var _ graph.MemberMethodsByType = (*Store)(nil)
|
||||
var _ graph.StructuralParentEdges = (*Store)(nil)
|
||||
var _ graph.ExtractCandidatesScanner = (*Store)(nil)
|
||||
var _ graph.CrossRepoCandidates = (*Store)(nil)
|
||||
var _ graph.ThrowerErrorSurfacer = (*Store)(nil)
|
||||
|
||||
// anaDedupeEdgeKinds drops empty / duplicate edge kinds, preserving
|
||||
// first-seen order — the EdgeKind twin of dedupeNonEmpty.
|
||||
func anaDedupeEdgeKinds(in []graph.EdgeKind) []graph.EdgeKind {
|
||||
seen := make(map[graph.EdgeKind]struct{}, len(in))
|
||||
out := make([]graph.EdgeKind, 0, len(in))
|
||||
for _, k := range in {
|
||||
if k == "" {
|
||||
continue
|
||||
}
|
||||
if _, ok := seen[k]; ok {
|
||||
continue
|
||||
}
|
||||
seen[k] = struct{}{}
|
||||
out = append(out, k)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// --- DeadCodeCandidator -------------------------------------------------
|
||||
|
||||
// DeadCodeCandidates returns nodes of the allowed kinds that have no
|
||||
// incoming edge of the corresponding allowed in-edge kinds. An empty
|
||||
// per-kind allowlist (or one that dedupes to nothing) means "any incoming
|
||||
// edge counts as usage". Mirrors graph.(*Graph).DeadCodeCandidates: the
|
||||
// candidate set is purely structural (the analysis layer applies the
|
||||
// exported / test / entry-point / synthetic post-filters in Go), so no
|
||||
// node-id exclusion happens here. The NOT-EXISTS filter runs server-side
|
||||
// per node kind.
|
||||
func (s *Store) DeadCodeCandidates(allowedNodeKinds []graph.NodeKind, allowedInEdgeKinds map[graph.NodeKind][]graph.EdgeKind) []*graph.Node {
|
||||
if len(allowedNodeKinds) == 0 {
|
||||
return nil
|
||||
}
|
||||
var out []*graph.Node
|
||||
for _, nk := range allowedNodeKinds {
|
||||
allowed := anaDedupeEdgeKinds(allowedInEdgeKinds[nk])
|
||||
anyKindCounts := len(allowed) == 0
|
||||
|
||||
var q string
|
||||
var args []any
|
||||
if anyKindCounts {
|
||||
// Any incoming edge disqualifies the node.
|
||||
q = `SELECT ` + lookupNodeCols + ` FROM nodes n
|
||||
WHERE n.kind = ?
|
||||
AND NOT EXISTS (SELECT 1 FROM edges e WHERE e.to_id = n.id)
|
||||
ORDER BY n.id`
|
||||
args = []any{string(nk)}
|
||||
} else {
|
||||
// Only an incoming edge of one of the allowed kinds counts.
|
||||
q = `SELECT ` + lookupNodeCols + ` FROM nodes n
|
||||
WHERE n.kind = ?
|
||||
AND NOT EXISTS (SELECT 1 FROM edges e WHERE e.to_id = n.id AND e.kind IN (` + inPlaceholders(len(allowed)) + `))
|
||||
ORDER BY n.id`
|
||||
args = make([]any, 0, 1+len(allowed))
|
||||
args = append(args, string(nk))
|
||||
for _, ek := range allowed {
|
||||
args = append(args, string(ek))
|
||||
}
|
||||
}
|
||||
|
||||
for _, n := range s.queryNodesSQL(q, args...) {
|
||||
if n != nil {
|
||||
out = append(out, n)
|
||||
}
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// --- IfaceImplementsScanner ---------------------------------------------
|
||||
|
||||
// IfaceImplementsRows returns one row per EdgeImplements edge whose
|
||||
// target is a KindInterface carrying Meta["methods"]. The interface's
|
||||
// decoded Meta rides on the row (callers pull the "methods" field, which
|
||||
// round-trips as []string or []any). Interfaces with no Meta or no
|
||||
// "methods" key are elided server-side.
|
||||
func (s *Store) IfaceImplementsRows() []graph.IfaceImplementsRow {
|
||||
q := `SELECT e.from_id, n.id, n.meta
|
||||
FROM edges e
|
||||
JOIN nodes n ON n.id = e.to_id
|
||||
WHERE e.kind = ? AND n.kind = ? AND n.meta IS NOT NULL`
|
||||
rows, err := s.db.Query(q, string(graph.EdgeImplements), string(graph.KindInterface))
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
var out []graph.IfaceImplementsRow
|
||||
for rows.Next() {
|
||||
var fromID, ifaceID string
|
||||
var metaBlob []byte
|
||||
if err := rows.Scan(&fromID, &ifaceID, &metaBlob); err != nil {
|
||||
continue
|
||||
}
|
||||
meta, derr := decodeMeta(metaBlob)
|
||||
if derr != nil || meta == nil {
|
||||
continue
|
||||
}
|
||||
if _, ok := meta["methods"]; !ok {
|
||||
continue
|
||||
}
|
||||
out = append(out, graph.IfaceImplementsRow{
|
||||
TypeID: fromID,
|
||||
IfaceID: ifaceID,
|
||||
IfaceMeta: meta,
|
||||
})
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// --- MemberMethodsByType ------------------------------------------------
|
||||
|
||||
// MemberMethodsByType returns typeID → []MemberMethodInfo for every
|
||||
// EdgeMemberOf edge whose source is a KindMethod. The columns come from
|
||||
// the METHOD NODE (FilePath / StartLine / RepoPrefix), matching the
|
||||
// in-memory reference. Per-type lists are deduplicated by MethodID; the
|
||||
// scan is ordered by the edge PK so the first-seen winner is stable. An
|
||||
// empty graph (no qualifying rows) returns nil.
|
||||
func (s *Store) MemberMethodsByType() map[string][]graph.MemberMethodInfo {
|
||||
q := `SELECT e.to_id, n.id, n.name, n.file_path, n.start_line, n.repo_prefix
|
||||
FROM edges e
|
||||
JOIN nodes n ON n.id = e.from_id
|
||||
WHERE e.kind = ? AND n.kind = ?
|
||||
ORDER BY e.id`
|
||||
rows, err := s.db.Query(q, string(graph.EdgeMemberOf), string(graph.KindMethod))
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
out := make(map[string][]graph.MemberMethodInfo)
|
||||
seen := make(map[string]map[string]struct{})
|
||||
for rows.Next() {
|
||||
var typeID, methodID, name, filePath, repoPrefix string
|
||||
var startLine int
|
||||
if err := rows.Scan(&typeID, &methodID, &name, &filePath, &startLine, &repoPrefix); err != nil {
|
||||
continue
|
||||
}
|
||||
if seen[typeID] == nil {
|
||||
seen[typeID] = make(map[string]struct{})
|
||||
}
|
||||
if _, ok := seen[typeID][methodID]; ok {
|
||||
continue
|
||||
}
|
||||
seen[typeID][methodID] = struct{}{}
|
||||
out[typeID] = append(out[typeID], graph.MemberMethodInfo{
|
||||
MethodID: methodID,
|
||||
Name: name,
|
||||
FilePath: filePath,
|
||||
StartLine: startLine,
|
||||
RepoPrefix: repoPrefix,
|
||||
})
|
||||
}
|
||||
if len(out) == 0 {
|
||||
// Match the in-memory reference: empty graph returns nil.
|
||||
return nil
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// --- StructuralParentEdges ----------------------------------------------
|
||||
|
||||
// StructuralParentEdges returns every Extends / Implements / Composes
|
||||
// edge whose endpoints are both Type / Interface, projected as (FromID,
|
||||
// ToID, FromKind, ToKind, Origin). Endpoints that aren't both type /
|
||||
// interface are filtered server-side. Empty graph or no matching edges
|
||||
// returns nil.
|
||||
func (s *Store) StructuralParentEdges() []graph.StructuralParentEdgeRow {
|
||||
q := `SELECT e.from_id, e.to_id, nf.kind, nt.kind, e.origin
|
||||
FROM edges e
|
||||
JOIN nodes nf ON nf.id = e.from_id
|
||||
JOIN nodes nt ON nt.id = e.to_id
|
||||
WHERE e.kind IN (?,?,?)
|
||||
AND nf.kind IN (?,?) AND nt.kind IN (?,?)
|
||||
ORDER BY e.id`
|
||||
rows, err := s.db.Query(q,
|
||||
string(graph.EdgeExtends), string(graph.EdgeImplements), string(graph.EdgeComposes),
|
||||
string(graph.KindType), string(graph.KindInterface),
|
||||
string(graph.KindType), string(graph.KindInterface),
|
||||
)
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
var out []graph.StructuralParentEdgeRow
|
||||
for rows.Next() {
|
||||
var fromID, toID, fromKind, toKind, origin string
|
||||
if err := rows.Scan(&fromID, &toID, &fromKind, &toKind, &origin); err != nil {
|
||||
continue
|
||||
}
|
||||
out = append(out, graph.StructuralParentEdgeRow{
|
||||
FromID: fromID,
|
||||
ToID: toID,
|
||||
FromKind: graph.NodeKind(fromKind),
|
||||
ToKind: graph.NodeKind(toKind),
|
||||
Origin: origin,
|
||||
})
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// --- ExtractCandidatesScanner -------------------------------------------
|
||||
|
||||
// ExtractCandidates ranks function / method nodes by extractability: line
|
||||
// span (EndLine - StartLine + 1), distinct caller fan-in, and distinct
|
||||
// callee fan-out, counting only edges whose kind is in the supplied set.
|
||||
// Rows must clear all three thresholds. Nodes with a zero StartLine /
|
||||
// EndLine are dropped; pathPrefix narrows by file-path prefix. Mirrors
|
||||
// graph.(*Graph).ExtractCandidates exactly: only KindFunction +
|
||||
// KindMethod nodes are considered, and the distinct-by-endpoint counting
|
||||
// runs Go-side over GetInEdges / GetOutEdges.
|
||||
func (s *Store) ExtractCandidates(kinds []graph.EdgeKind, minLines, minCallers, minFanOut int, pathPrefix string) []graph.ExtractCandidateRow {
|
||||
if len(kinds) == 0 {
|
||||
return nil
|
||||
}
|
||||
kindSet := make(map[graph.EdgeKind]struct{}, len(kinds))
|
||||
for _, k := range kinds {
|
||||
if k == "" {
|
||||
continue
|
||||
}
|
||||
kindSet[k] = struct{}{}
|
||||
}
|
||||
if len(kindSet) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
// Candidate nodes: function / method only, non-zero line span,
|
||||
// optional path-prefix gate.
|
||||
q := `SELECT ` + lookupNodeCols + ` FROM nodes
|
||||
WHERE kind IN (?,?) AND start_line > 0 AND end_line > 0`
|
||||
args := []any{string(graph.KindFunction), string(graph.KindMethod)}
|
||||
if pathPrefix != "" {
|
||||
q += ` AND file_path LIKE ? ESCAPE '\'`
|
||||
args = append(args, escapeLikePattern(pathPrefix)+"%")
|
||||
}
|
||||
q += ` ORDER BY id`
|
||||
nodes := s.queryNodesSQL(q, args...)
|
||||
|
||||
var out []graph.ExtractCandidateRow
|
||||
for _, n := range nodes {
|
||||
if n == nil {
|
||||
continue
|
||||
}
|
||||
lineCount := n.EndLine - n.StartLine + 1
|
||||
if lineCount < minLines {
|
||||
continue
|
||||
}
|
||||
|
||||
callerSet := make(map[string]struct{})
|
||||
for _, e := range s.GetInEdges(n.ID) {
|
||||
if e == nil {
|
||||
continue
|
||||
}
|
||||
if _, ok := kindSet[e.Kind]; !ok {
|
||||
continue
|
||||
}
|
||||
callerSet[e.From] = struct{}{}
|
||||
}
|
||||
if len(callerSet) < minCallers {
|
||||
continue
|
||||
}
|
||||
|
||||
calleeSet := make(map[string]struct{})
|
||||
for _, e := range s.GetOutEdges(n.ID) {
|
||||
if e == nil {
|
||||
continue
|
||||
}
|
||||
if _, ok := kindSet[e.Kind]; !ok {
|
||||
continue
|
||||
}
|
||||
calleeSet[e.To] = struct{}{}
|
||||
}
|
||||
if len(calleeSet) < minFanOut {
|
||||
continue
|
||||
}
|
||||
|
||||
out = append(out, graph.ExtractCandidateRow{
|
||||
NodeID: n.ID,
|
||||
Name: n.Name,
|
||||
FilePath: n.FilePath,
|
||||
StartLine: n.StartLine,
|
||||
EndLine: n.EndLine,
|
||||
LineCount: lineCount,
|
||||
CallerCount: len(callerSet),
|
||||
FanOut: len(calleeSet),
|
||||
})
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// --- CrossRepoCandidates ------------------------------------------------
|
||||
|
||||
// CrossRepoCandidates returns every edge whose kind is in baseKinds and
|
||||
// whose endpoints carry two different non-empty RepoPrefix values. The
|
||||
// edge is returned verbatim (callers rewrite Edge.CrossRepo); FromRepo /
|
||||
// ToRepo are the endpoint prefixes. Empty baseKinds returns nil; single-
|
||||
// repo graphs (or graphs whose nodes carry no RepoPrefix) yield nothing.
|
||||
func (s *Store) CrossRepoCandidates(baseKinds []graph.EdgeKind) []graph.CrossRepoCandidateRow {
|
||||
uniq := anaDedupeEdgeKinds(baseKinds)
|
||||
if len(uniq) == 0 {
|
||||
return nil
|
||||
}
|
||||
args := make([]any, 0, len(uniq))
|
||||
for _, k := range uniq {
|
||||
args = append(args, string(k))
|
||||
}
|
||||
q := `SELECT e.from_id, e.to_id, e.kind, e.file_path, e.line,
|
||||
e.confidence, e.confidence_label, e.origin, e.tier, e.cross_repo, e.meta,
|
||||
nf.repo_prefix, nt.repo_prefix
|
||||
FROM edges e
|
||||
JOIN nodes nf ON nf.id = e.from_id
|
||||
JOIN nodes nt ON nt.id = e.to_id
|
||||
WHERE e.kind IN (` + inPlaceholders(len(uniq)) + `)
|
||||
AND nf.repo_prefix <> '' AND nt.repo_prefix <> ''
|
||||
AND nf.repo_prefix <> nt.repo_prefix
|
||||
ORDER BY e.id`
|
||||
rows, err := s.db.Query(q, args...)
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
var out []graph.CrossRepoCandidateRow
|
||||
for rows.Next() {
|
||||
var (
|
||||
fromRepo, toRepo string
|
||||
e graph.Edge
|
||||
metaBlob []byte
|
||||
crossRepo int64
|
||||
)
|
||||
if err := rows.Scan(
|
||||
&e.From, &e.To, &e.Kind, &e.FilePath, &e.Line,
|
||||
&e.Confidence, &e.ConfidenceLabel, &e.Origin, &e.Tier,
|
||||
&crossRepo, &metaBlob,
|
||||
&fromRepo, &toRepo,
|
||||
); err != nil {
|
||||
continue
|
||||
}
|
||||
e.CrossRepo = crossRepo != 0
|
||||
if len(metaBlob) > 0 {
|
||||
if m, derr := decodeMeta(metaBlob); derr == nil {
|
||||
e.Meta = m
|
||||
}
|
||||
}
|
||||
edge := e
|
||||
out = append(out, graph.CrossRepoCandidateRow{
|
||||
Edge: &edge,
|
||||
FromRepo: fromRepo,
|
||||
ToRepo: toRepo,
|
||||
})
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// --- ThrowerErrorSurfacer -----------------------------------------------
|
||||
|
||||
// ThrowerErrorSurface returns one row per thrower (a node with outgoing
|
||||
// EdgeThrows edges), aggregating the distinct error targets and the
|
||||
// distinct literal error-message strings it emits (KindString nodes with
|
||||
// Meta["context"] == "error_msg", linked by EdgeEmits). pathPrefix gates
|
||||
// the EdgeThrows rows by their stored FilePath prefix. Throws counts the
|
||||
// underlying EdgeThrows edges; FilePath / Line seed from the first throws
|
||||
// edge, falling back to the thrower node's own coordinates when the edge
|
||||
// carries none — matching the in-memory reference.
|
||||
func (s *Store) ThrowerErrorSurface(pathPrefix string) []graph.ThrowerErrorRow {
|
||||
type rowAccum struct {
|
||||
row graph.ThrowerErrorRow
|
||||
targetSeen map[string]struct{}
|
||||
msgSeen map[string]struct{}
|
||||
}
|
||||
accums := make(map[string]*rowAccum)
|
||||
var order []string
|
||||
|
||||
// Pass 1: EdgeThrows aggregation (count + distinct targets), keyed by
|
||||
// thrower. The first edge (by PK insertion order) seeds FilePath /
|
||||
// Line; an empty edge file/line falls back to the thrower node.
|
||||
tq := `SELECT from_id, to_id, file_path, line FROM edges WHERE kind = ?`
|
||||
targs := []any{string(graph.EdgeThrows)}
|
||||
if pathPrefix != "" {
|
||||
tq += ` AND file_path LIKE ? ESCAPE '\'`
|
||||
targs = append(targs, escapeLikePattern(pathPrefix)+"%")
|
||||
}
|
||||
tq += ` ORDER BY id`
|
||||
trows, err := s.db.Query(tq, targs...)
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
for trows.Next() {
|
||||
var from, to, filePath string
|
||||
var line int
|
||||
if err := trows.Scan(&from, &to, &filePath, &line); err != nil {
|
||||
continue
|
||||
}
|
||||
acc := accums[from]
|
||||
if acc == nil {
|
||||
file := filePath
|
||||
ln := line
|
||||
if file == "" || ln == 0 {
|
||||
if n := s.GetNode(from); n != nil {
|
||||
if file == "" {
|
||||
file = n.FilePath
|
||||
}
|
||||
if ln == 0 {
|
||||
ln = n.StartLine
|
||||
}
|
||||
}
|
||||
}
|
||||
acc = &rowAccum{
|
||||
row: graph.ThrowerErrorRow{
|
||||
ThrowerID: from,
|
||||
FilePath: file,
|
||||
Line: ln,
|
||||
},
|
||||
targetSeen: make(map[string]struct{}),
|
||||
msgSeen: make(map[string]struct{}),
|
||||
}
|
||||
accums[from] = acc
|
||||
order = append(order, from)
|
||||
}
|
||||
acc.row.Throws++
|
||||
if _, ok := acc.targetSeen[to]; !ok {
|
||||
acc.targetSeen[to] = struct{}{}
|
||||
acc.row.ErrorTargets = append(acc.row.ErrorTargets, to)
|
||||
}
|
||||
}
|
||||
_ = trows.Close()
|
||||
if len(accums) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
// Pass 2: attach the literal error messages each thrower emits. Join
|
||||
// each thrower's EdgeEmits out-edges to KindString targets and filter
|
||||
// Meta["context"] == "error_msg" Go-side (the context lives in the
|
||||
// JSON Meta blob).
|
||||
for _, id := range order {
|
||||
acc := accums[id]
|
||||
mq := `SELECT n.name, n.meta
|
||||
FROM edges e
|
||||
JOIN nodes n ON n.id = e.to_id
|
||||
WHERE e.from_id = ? AND e.kind = ? AND n.kind = ? AND n.meta IS NOT NULL
|
||||
ORDER BY e.id`
|
||||
mrows, err := s.db.Query(mq, id, string(graph.EdgeEmits), string(graph.KindString))
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
for mrows.Next() {
|
||||
var name string
|
||||
var metaBlob []byte
|
||||
if err := mrows.Scan(&name, &metaBlob); err != nil {
|
||||
continue
|
||||
}
|
||||
meta, derr := decodeMeta(metaBlob)
|
||||
if derr != nil || meta == nil {
|
||||
continue
|
||||
}
|
||||
ctxLabel, _ := meta["context"].(string)
|
||||
if ctxLabel != "error_msg" {
|
||||
continue
|
||||
}
|
||||
if _, ok := acc.msgSeen[name]; ok {
|
||||
continue
|
||||
}
|
||||
acc.msgSeen[name] = struct{}{}
|
||||
acc.row.ErrorMsgs = append(acc.row.ErrorMsgs, name)
|
||||
}
|
||||
_ = mrows.Close()
|
||||
}
|
||||
|
||||
out := make([]graph.ThrowerErrorRow, 0, len(order))
|
||||
for _, id := range order {
|
||||
out = append(out, accums[id].row)
|
||||
}
|
||||
return out
|
||||
}
|
||||
@@ -0,0 +1,172 @@
|
||||
package store_sqlite
|
||||
|
||||
import (
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
"github.com/zzet/gortex/internal/graph"
|
||||
)
|
||||
|
||||
var _ graph.BFSCapable = (*Store)(nil)
|
||||
|
||||
// BFS runs a bounded breadth-first traversal in a single round-trip via a
|
||||
// recursive CTE — the disk-backed sibling of the in-memory
|
||||
// (*graph.Graph).BFS reference. See graph.BFSCapable for the contract;
|
||||
// the two are shadow-tested for identical hop-sets in the conformance
|
||||
// suite (storetest), including a cycle fixture.
|
||||
//
|
||||
// The recursive term joins edges on the direction's indexed column —
|
||||
// edges_by_from(from_id, kind) for a forward walk, edges_by_to(to_id,
|
||||
// kind) for a backward walk — and the nodes primary key, so it stays
|
||||
// index-driven instead of scanning the edges table (confirmed via
|
||||
// EXPLAIN QUERY PLAN in store_bfs_test.go). The nodes join also enforces
|
||||
// the "node-backed targets only" rule: an edge to an unresolved /
|
||||
// external stub with no node row is not followed. A cycle terminates on
|
||||
// the depth bound; the outer ROW_NUMBER picks each node's minimum-depth,
|
||||
// (parent, kind)-smallest discovery edge so the result is deterministic
|
||||
// and matches the in-memory walk's bfsHopLess tie-break.
|
||||
//
|
||||
// Reads run lock-free, like the store's other read paths (SQLite WAL
|
||||
// serves readers concurrently with the single serialized writer).
|
||||
func (s *Store) BFS(seeds []string, dir graph.Direction, kinds []graph.EdgeKind, maxDepth, limit int) ([]graph.BFSHop, error) {
|
||||
seen := make(map[string]struct{}, len(seeds))
|
||||
uniqSeeds := make([]string, 0, len(seeds))
|
||||
for _, sd := range seeds {
|
||||
if sd == "" {
|
||||
continue
|
||||
}
|
||||
if _, ok := seen[sd]; ok {
|
||||
continue
|
||||
}
|
||||
seen[sd] = struct{}{}
|
||||
uniqSeeds = append(uniqSeeds, sd)
|
||||
}
|
||||
if len(uniqSeeds) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
uniqKinds := anaDedupeEdgeKinds(kinds)
|
||||
|
||||
// Seed-only fast path: with no edge kinds to follow or a non-positive
|
||||
// depth bound the result is exactly the seeds at depth 0. Seeds enter
|
||||
// unconditionally (no node-backed gate), matching the in-memory
|
||||
// reference, which adds them before any traversal.
|
||||
if len(uniqKinds) == 0 || maxDepth <= 0 {
|
||||
hops := make([]graph.BFSHop, 0, len(uniqSeeds))
|
||||
for _, sd := range uniqSeeds {
|
||||
hops = append(hops, graph.BFSHop{NodeID: sd, Depth: 0})
|
||||
}
|
||||
sortBFSHops(hops)
|
||||
if limit > 0 && len(hops) > limit {
|
||||
hops = hops[:limit]
|
||||
}
|
||||
return hops, nil
|
||||
}
|
||||
|
||||
query := buildBFSQuery(dir, len(uniqSeeds), len(uniqKinds), limit > 0)
|
||||
|
||||
args := make([]any, 0, len(uniqSeeds)+1+len(uniqKinds)+1)
|
||||
for _, sd := range uniqSeeds {
|
||||
args = append(args, sd)
|
||||
}
|
||||
args = append(args, maxDepth)
|
||||
for _, k := range uniqKinds {
|
||||
args = append(args, string(k))
|
||||
}
|
||||
if limit > 0 {
|
||||
args = append(args, limit)
|
||||
}
|
||||
|
||||
rows, err := s.db.Query(query, args...)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
var out []graph.BFSHop
|
||||
for rows.Next() {
|
||||
var (
|
||||
nodeID, parentID, edgeKind string
|
||||
depth int
|
||||
)
|
||||
if err := rows.Scan(&nodeID, &depth, &parentID, &edgeKind); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out = append(out, graph.BFSHop{
|
||||
NodeID: nodeID,
|
||||
Depth: depth,
|
||||
ParentID: parentID,
|
||||
EdgeKind: graph.EdgeKind(edgeKind),
|
||||
})
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// buildBFSQuery assembles the recursive-CTE BFS statement for the given
|
||||
// direction, seed count, kind count, and whether a LIMIT is applied. It is
|
||||
// a pure string builder (no I/O) so a test can EXPLAIN QUERY PLAN the exact
|
||||
// statement and assert the recursive join stays index-driven.
|
||||
//
|
||||
// Direction selects the join columns: forward follows from_id -> to_id (the
|
||||
// discovered neighbour is the edge target), backward follows to_id ->
|
||||
// from_id (the neighbour is the edge source). The recursive term joins
|
||||
// edges on the walked node's id, so a forward walk leads with from_id (the
|
||||
// edges_by_from(from_id, kind) index) and a backward walk leads with to_id
|
||||
// (edges_by_to(to_id, kind)); the nodes join uses the nodes primary key.
|
||||
func buildBFSQuery(dir graph.Direction, nSeeds, nKinds int, withLimit bool) string {
|
||||
joinCol, nextCol := "e.from_id", "e.to_id"
|
||||
edgeIdx := "edges_by_from"
|
||||
if dir == graph.DirectionBackward {
|
||||
joinCol, nextCol = "e.to_id", "e.from_id"
|
||||
edgeIdx = "edges_by_to"
|
||||
}
|
||||
|
||||
var b strings.Builder
|
||||
b.WriteString("WITH RECURSIVE seeds(node_id) AS (VALUES ")
|
||||
for i := 0; i < nSeeds; i++ {
|
||||
if i > 0 {
|
||||
b.WriteString(", ")
|
||||
}
|
||||
b.WriteString("(?)")
|
||||
}
|
||||
b.WriteString("),\n")
|
||||
b.WriteString("bfs(node_id, depth, parent_id, edge_kind) AS (\n")
|
||||
b.WriteString(" SELECT node_id, 0, '', '' FROM seeds\n")
|
||||
b.WriteString(" UNION\n")
|
||||
b.WriteString(" SELECT " + nextCol + ", b.depth + 1, b.node_id, e.kind\n")
|
||||
b.WriteString(" FROM bfs b\n")
|
||||
// INDEXED BY forces the frontier-node seek (from_id / to_id leading)
|
||||
// instead of the planner's stats-free preference for edges_by_kind,
|
||||
// which on a hot kind would scan every edge of that kind per frontier
|
||||
// node. If the index is ever absent (a bulk-load window drops it) the
|
||||
// query errors and the engine falls back to the in-memory walk.
|
||||
b.WriteString(" JOIN edges e INDEXED BY " + edgeIdx + " ON " + joinCol + " = b.node_id\n")
|
||||
b.WriteString(" JOIN nodes n ON n.id = " + nextCol + "\n")
|
||||
b.WriteString(" WHERE b.depth < ? AND e.kind IN (" + inPlaceholders(nKinds) + ")\n")
|
||||
b.WriteString("),\n")
|
||||
b.WriteString("ranked AS (\n")
|
||||
b.WriteString(" SELECT node_id, depth, parent_id, edge_kind,\n")
|
||||
b.WriteString(" ROW_NUMBER() OVER (PARTITION BY node_id ORDER BY depth, parent_id, edge_kind) AS rn\n")
|
||||
b.WriteString(" FROM bfs\n")
|
||||
b.WriteString(")\n")
|
||||
b.WriteString("SELECT node_id, depth, parent_id, edge_kind FROM ranked WHERE rn = 1\n")
|
||||
b.WriteString("ORDER BY depth, node_id")
|
||||
if withLimit {
|
||||
b.WriteString("\nLIMIT ?")
|
||||
}
|
||||
return b.String()
|
||||
}
|
||||
|
||||
// sortBFSHops orders hops by (depth, node_id) — the same final ordering
|
||||
// the recursive-CTE query applies, used by the seed-only fast path.
|
||||
func sortBFSHops(hops []graph.BFSHop) {
|
||||
sort.Slice(hops, func(i, j int) bool {
|
||||
if hops[i].Depth != hops[j].Depth {
|
||||
return hops[i].Depth < hops[j].Depth
|
||||
}
|
||||
return hops[i].NodeID < hops[j].NodeID
|
||||
})
|
||||
}
|
||||
@@ -0,0 +1,74 @@
|
||||
package store_sqlite
|
||||
|
||||
import (
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/zzet/gortex/internal/graph"
|
||||
)
|
||||
|
||||
// queryPlan runs EXPLAIN QUERY PLAN for the given statement + args and
|
||||
// returns the joined detail lines.
|
||||
func queryPlan(t *testing.T, s *Store, query string, args ...any) string {
|
||||
t.Helper()
|
||||
rows, err := s.db.Query("EXPLAIN QUERY PLAN "+query, args...)
|
||||
if err != nil {
|
||||
t.Fatalf("EXPLAIN QUERY PLAN: %v", err)
|
||||
}
|
||||
defer rows.Close()
|
||||
var lines []string
|
||||
for rows.Next() {
|
||||
var id, parent, notused int
|
||||
var detail string
|
||||
if err := rows.Scan(&id, &parent, ¬used, &detail); err != nil {
|
||||
t.Fatalf("scan plan row: %v", err)
|
||||
}
|
||||
lines = append(lines, detail)
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
t.Fatalf("plan rows: %v", err)
|
||||
}
|
||||
return strings.Join(lines, "\n")
|
||||
}
|
||||
|
||||
// TestBFSQueryUsesEdgeIndex asserts the recursive-CTE BFS join reaches the
|
||||
// edges table through the direction's composite index (edges_by_from for a
|
||||
// forward walk, edges_by_to for a backward walk) rather than a full table
|
||||
// scan, and that the node-backed gate join uses the nodes primary key.
|
||||
func TestBFSQueryUsesEdgeIndex(t *testing.T) {
|
||||
s, err := Open(filepath.Join(t.TempDir(), "store.sqlite"))
|
||||
if err != nil {
|
||||
t.Fatalf("open: %v", err)
|
||||
}
|
||||
defer s.Close()
|
||||
|
||||
// A small fixture so the planner has real tables/indexes to plan against.
|
||||
for _, id := range []string{"A", "B", "C"} {
|
||||
s.AddNode(&graph.Node{ID: id, Kind: graph.KindFunction, Name: id, FilePath: "a.go", Language: "go"})
|
||||
}
|
||||
s.AddEdge(&graph.Edge{From: "A", To: "B", Kind: graph.EdgeCalls, Confidence: 1, Origin: graph.OriginASTResolved})
|
||||
s.AddEdge(&graph.Edge{From: "B", To: "C", Kind: graph.EdgeCalls, Confidence: 1, Origin: graph.OriginASTResolved})
|
||||
|
||||
forwardQ := buildBFSQuery(graph.DirectionForward, 1, 1, true)
|
||||
forwardPlan := queryPlan(t, s, forwardQ, "A", 3, string(graph.EdgeCalls), 50)
|
||||
if !strings.Contains(forwardPlan, "edges_by_from") {
|
||||
t.Fatalf("forward BFS plan does not use edges_by_from:\n%s", forwardPlan)
|
||||
}
|
||||
// The edges table must not be reached by a full scan (alias e).
|
||||
if strings.Contains(forwardPlan, "SCAN edges") || strings.Contains(forwardPlan, "SCAN e ") {
|
||||
t.Fatalf("forward BFS plan scans edges instead of using an index:\n%s", forwardPlan)
|
||||
}
|
||||
|
||||
backwardQ := buildBFSQuery(graph.DirectionBackward, 1, 1, true)
|
||||
backwardPlan := queryPlan(t, s, backwardQ, "C", 3, string(graph.EdgeCalls), 50)
|
||||
if !strings.Contains(backwardPlan, "edges_by_to") {
|
||||
t.Fatalf("backward BFS plan does not use edges_by_to:\n%s", backwardPlan)
|
||||
}
|
||||
if strings.Contains(backwardPlan, "SCAN edges") || strings.Contains(backwardPlan, "SCAN e ") {
|
||||
t.Fatalf("backward BFS plan scans edges instead of using an index:\n%s", backwardPlan)
|
||||
}
|
||||
|
||||
t.Logf("forward BFS query plan:\n%s", forwardPlan)
|
||||
t.Logf("backward BFS query plan:\n%s", backwardPlan)
|
||||
}
|
||||
@@ -0,0 +1,130 @@
|
||||
package store_sqlite
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
|
||||
"github.com/zzet/gortex/internal/graph"
|
||||
)
|
||||
|
||||
var (
|
||||
_ graph.BlameEnrichmentWriter = (*Store)(nil)
|
||||
_ graph.BlameEnrichmentReader = (*Store)(nil)
|
||||
)
|
||||
|
||||
const blameChunk = 180
|
||||
|
||||
const blameCols = `node_id, repo_prefix, commit_sha, email, ts`
|
||||
|
||||
func (s *Store) BulkSetBlame(repoPrefix string, rows []graph.BlameEnrichment) error {
|
||||
if len(rows) == 0 {
|
||||
return nil
|
||||
}
|
||||
s.writeMu.Lock()
|
||||
defer s.writeMu.Unlock()
|
||||
tx, err := s.db.Begin()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer tx.Rollback() //nolint:errcheck
|
||||
for start := 0; start < len(rows); start += blameChunk {
|
||||
end := start + blameChunk
|
||||
if end > len(rows) {
|
||||
end = len(rows)
|
||||
}
|
||||
batch := rows[start:end]
|
||||
args := make([]any, 0, len(batch)*5)
|
||||
stmt := make([]byte, 0, 96+len(batch)*16)
|
||||
stmt = append(stmt, "INSERT OR REPLACE INTO blame_enrichment ("...)
|
||||
stmt = append(stmt, blameCols...)
|
||||
stmt = append(stmt, ") VALUES "...)
|
||||
for i, e := range batch {
|
||||
if i > 0 {
|
||||
stmt = append(stmt, ',')
|
||||
}
|
||||
stmt = append(stmt, "(?,?,?,?,?)"...)
|
||||
args = append(args, e.NodeID, repoPrefix, e.Commit, e.Email, e.Timestamp)
|
||||
}
|
||||
if _, err := tx.Exec(string(stmt), args...); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return tx.Commit()
|
||||
}
|
||||
|
||||
func (s *Store) DeleteBlame(nodeIDs []string) error {
|
||||
if len(nodeIDs) == 0 {
|
||||
return nil
|
||||
}
|
||||
seen := make(map[string]struct{}, len(nodeIDs))
|
||||
uniq := make([]string, 0, len(nodeIDs))
|
||||
for _, id := range nodeIDs {
|
||||
if id == "" {
|
||||
continue
|
||||
}
|
||||
if _, ok := seen[id]; ok {
|
||||
continue
|
||||
}
|
||||
seen[id] = struct{}{}
|
||||
uniq = append(uniq, id)
|
||||
}
|
||||
if len(uniq) == 0 {
|
||||
return nil
|
||||
}
|
||||
s.writeMu.Lock()
|
||||
defer s.writeMu.Unlock()
|
||||
tx, err := s.db.Begin()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer tx.Rollback() //nolint:errcheck
|
||||
for start := 0; start < len(uniq); start += blameChunk {
|
||||
end := start + blameChunk
|
||||
if end > len(uniq) {
|
||||
end = len(uniq)
|
||||
}
|
||||
chunk := uniq[start:end]
|
||||
args := make([]any, len(chunk))
|
||||
stmt := make([]byte, 0, 56+len(chunk)*2)
|
||||
stmt = append(stmt, "DELETE FROM blame_enrichment WHERE node_id IN ("...)
|
||||
for i, id := range chunk {
|
||||
if i > 0 {
|
||||
stmt = append(stmt, ',')
|
||||
}
|
||||
stmt = append(stmt, '?')
|
||||
args[i] = id
|
||||
}
|
||||
stmt = append(stmt, ')')
|
||||
if _, err := tx.Exec(string(stmt), args...); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return tx.Commit()
|
||||
}
|
||||
|
||||
func (s *Store) BlameRows(repoPrefix string) []graph.BlameEnrichment {
|
||||
var (
|
||||
rows *sql.Rows
|
||||
err error
|
||||
)
|
||||
if repoPrefix == "" {
|
||||
rows, err = s.db.Query(`SELECT ` + blameCols + ` FROM blame_enrichment`)
|
||||
} else {
|
||||
rows, err = s.db.Query(`SELECT `+blameCols+` FROM blame_enrichment WHERE repo_prefix = ?`, repoPrefix)
|
||||
}
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
defer rows.Close()
|
||||
var out []graph.BlameEnrichment
|
||||
for rows.Next() {
|
||||
var e graph.BlameEnrichment
|
||||
if err := rows.Scan(&e.NodeID, &e.RepoPrefix, &e.Commit, &e.Email, &e.Timestamp); err != nil {
|
||||
return out
|
||||
}
|
||||
out = append(out, e)
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return out
|
||||
}
|
||||
return out
|
||||
}
|
||||
@@ -0,0 +1,155 @@
|
||||
package store_sqlite
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
|
||||
"github.com/zzet/gortex/internal/graph"
|
||||
)
|
||||
|
||||
// Compile-time assertions that the SQLite Store satisfies the optional
|
||||
// git-churn enrichment sidecar capabilities (change A: enrichment moved
|
||||
// out of nodes.meta into a typed table so the node hot path stops
|
||||
// encoding rarely-read data into the meta blob and get_churn_rate reads
|
||||
// via an index instead of an AllNodes scan).
|
||||
var (
|
||||
_ graph.ChurnEnrichmentWriter = (*Store)(nil)
|
||||
_ graph.ChurnEnrichmentReader = (*Store)(nil)
|
||||
)
|
||||
|
||||
// churnChunk bounds rows per multi-row INSERT. churn_enrichment has 10
|
||||
// columns, so at 10 params/row the 999 host-param limit caps a statement
|
||||
// at 99 rows; 90 leaves headroom. Mirrors shingleChunk / mtimeChunk.
|
||||
const churnChunk = 90
|
||||
|
||||
const churnCols = `node_id, repo_prefix, commit_count, age_days, churn_rate, last_author, last_commit_at, head_sha, branch, computed_at`
|
||||
|
||||
// BulkSetChurn persists every churn row for one repo prefix in a single
|
||||
// transaction, chunked under the host-parameter limit. Idempotent on
|
||||
// node_id (INSERT OR REPLACE). Empty input is a no-op.
|
||||
func (s *Store) BulkSetChurn(repoPrefix string, rows []graph.ChurnEnrichment) error {
|
||||
if len(rows) == 0 {
|
||||
return nil
|
||||
}
|
||||
s.writeMu.Lock()
|
||||
defer s.writeMu.Unlock()
|
||||
|
||||
tx, err := s.db.Begin()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer tx.Rollback() //nolint:errcheck // rollback after Commit is a no-op
|
||||
|
||||
for start := 0; start < len(rows); start += churnChunk {
|
||||
end := start + churnChunk
|
||||
if end > len(rows) {
|
||||
end = len(rows)
|
||||
}
|
||||
batch := rows[start:end]
|
||||
args := make([]any, 0, len(batch)*10)
|
||||
stmt := make([]byte, 0, 128+len(batch)*24)
|
||||
stmt = append(stmt, "INSERT OR REPLACE INTO churn_enrichment ("...)
|
||||
stmt = append(stmt, churnCols...)
|
||||
stmt = append(stmt, ") VALUES "...)
|
||||
for i, e := range batch {
|
||||
if i > 0 {
|
||||
stmt = append(stmt, ',')
|
||||
}
|
||||
stmt = append(stmt, "(?,?,?,?,?,?,?,?,?,?)"...)
|
||||
args = append(args, e.NodeID, repoPrefix, e.CommitCount, e.AgeDays,
|
||||
e.ChurnRate, e.LastAuthor, e.LastCommitAt, e.HeadSHA, e.Branch, e.ComputedAt)
|
||||
}
|
||||
if _, err := tx.Exec(string(stmt), args...); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return tx.Commit()
|
||||
}
|
||||
|
||||
// DeleteChurn drops churn rows for the supplied node ids, chunked into
|
||||
// `node_id IN (?, …)` DELETEs. Empty input is a no-op.
|
||||
func (s *Store) DeleteChurn(nodeIDs []string) error {
|
||||
if len(nodeIDs) == 0 {
|
||||
return nil
|
||||
}
|
||||
seen := make(map[string]struct{}, len(nodeIDs))
|
||||
uniq := make([]string, 0, len(nodeIDs))
|
||||
for _, id := range nodeIDs {
|
||||
if id == "" {
|
||||
continue
|
||||
}
|
||||
if _, ok := seen[id]; ok {
|
||||
continue
|
||||
}
|
||||
seen[id] = struct{}{}
|
||||
uniq = append(uniq, id)
|
||||
}
|
||||
if len(uniq) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
s.writeMu.Lock()
|
||||
defer s.writeMu.Unlock()
|
||||
|
||||
tx, err := s.db.Begin()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer tx.Rollback() //nolint:errcheck
|
||||
|
||||
for start := 0; start < len(uniq); start += churnChunk {
|
||||
end := start + churnChunk
|
||||
if end > len(uniq) {
|
||||
end = len(uniq)
|
||||
}
|
||||
chunk := uniq[start:end]
|
||||
args := make([]any, len(chunk))
|
||||
stmt := make([]byte, 0, 48+len(chunk)*2)
|
||||
stmt = append(stmt, "DELETE FROM churn_enrichment WHERE node_id IN ("...)
|
||||
for i, id := range chunk {
|
||||
if i > 0 {
|
||||
stmt = append(stmt, ',')
|
||||
}
|
||||
stmt = append(stmt, '?')
|
||||
args[i] = id
|
||||
}
|
||||
stmt = append(stmt, ')')
|
||||
if _, err := tx.Exec(string(stmt), args...); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return tx.Commit()
|
||||
}
|
||||
|
||||
// ChurnRows returns every churn row for repoPrefix; an EMPTY repoPrefix
|
||||
// returns ALL rows across repos. This is an index-only read over the
|
||||
// (small) enriched set — the whole point of the sidecar, replacing the
|
||||
// AllNodes()+meta-decode scan get_churn_rate used to do.
|
||||
func (s *Store) ChurnRows(repoPrefix string) []graph.ChurnEnrichment {
|
||||
var (
|
||||
rows *sql.Rows
|
||||
err error
|
||||
)
|
||||
if repoPrefix == "" {
|
||||
rows, err = s.db.Query(`SELECT ` + churnCols + ` FROM churn_enrichment`)
|
||||
} else {
|
||||
rows, err = s.db.Query(`SELECT `+churnCols+` FROM churn_enrichment WHERE repo_prefix = ?`, repoPrefix)
|
||||
}
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
var out []graph.ChurnEnrichment
|
||||
for rows.Next() {
|
||||
var e graph.ChurnEnrichment
|
||||
if err := rows.Scan(&e.NodeID, &e.RepoPrefix, &e.CommitCount, &e.AgeDays,
|
||||
&e.ChurnRate, &e.LastAuthor, &e.LastCommitAt, &e.HeadSHA, &e.Branch, &e.ComputedAt); err != nil {
|
||||
return out
|
||||
}
|
||||
out = append(out, e)
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return out
|
||||
}
|
||||
return out
|
||||
}
|
||||
@@ -0,0 +1,192 @@
|
||||
package store_sqlite
|
||||
|
||||
import (
|
||||
"encoding/binary"
|
||||
|
||||
"github.com/zzet/gortex/internal/graph"
|
||||
)
|
||||
|
||||
// Compile-time assertions that the SQLite Store satisfies the optional
|
||||
// per-symbol clone-shingle persistence capabilities. Lifting this state
|
||||
// into the same backend the graph lives in means warm restarts rebuild
|
||||
// the clone-detection CMS through one persistence surface instead of a
|
||||
// second gob snapshot.
|
||||
var (
|
||||
_ graph.CloneShingleWriter = (*Store)(nil)
|
||||
_ graph.CloneShingleReader = (*Store)(nil)
|
||||
)
|
||||
|
||||
// shingleChunk bounds how many (node_id, repo_prefix, shingles) tuples
|
||||
// ride in a single multi-row INSERT. SQLite's default compiled-in host
|
||||
// parameter limit is 999; at 3 params per row that caps a statement at
|
||||
// 333 rows, so 300 leaves headroom. Mirrors mtimeChunk.
|
||||
const shingleChunk = 300
|
||||
|
||||
// encodeShingles serialises a uint64 slice to a little-endian BLOB
|
||||
// (8 bytes per element). A nil/empty slice encodes to an empty BLOB.
|
||||
func encodeShingles(shingles []uint64) []byte {
|
||||
b := make([]byte, len(shingles)*8)
|
||||
for i, s := range shingles {
|
||||
binary.LittleEndian.PutUint64(b[i*8:], s)
|
||||
}
|
||||
return b
|
||||
}
|
||||
|
||||
// decodeShingles is the inverse of encodeShingles. A BLOB whose length
|
||||
// is not a multiple of 8 yields nil (corrupt row); callers skip nil
|
||||
// sets. An empty BLOB decodes to an empty (non-nil) slice.
|
||||
func decodeShingles(b []byte) []uint64 {
|
||||
if len(b)%8 != 0 {
|
||||
return nil
|
||||
}
|
||||
out := make([]uint64, len(b)/8)
|
||||
for i := range out {
|
||||
out[i] = binary.LittleEndian.Uint64(b[i*8:])
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// BulkSetCloneShingles persists every (nodeID -> shingles) entry for
|
||||
// one repo prefix in a single transaction, chunked so no statement
|
||||
// exceeds SQLite's host-parameter limit. Idempotent on node_id:
|
||||
// re-running with overlapping keys replaces in place. Empty input is a
|
||||
// no-op.
|
||||
func (s *Store) BulkSetCloneShingles(repoPrefix string, rows map[string][]uint64) error {
|
||||
if len(rows) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
s.writeMu.Lock()
|
||||
defer s.writeMu.Unlock()
|
||||
|
||||
// Stable ordering is not required for correctness, but iterating the
|
||||
// map directly is fine — we only chunk by count.
|
||||
type kv struct {
|
||||
id string
|
||||
blob []byte
|
||||
}
|
||||
pending := make([]kv, 0, len(rows))
|
||||
for id, sh := range rows {
|
||||
pending = append(pending, kv{id: id, blob: encodeShingles(sh)})
|
||||
}
|
||||
|
||||
tx, err := s.db.Begin()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer tx.Rollback() //nolint:errcheck // rollback after Commit is a no-op
|
||||
|
||||
for start := 0; start < len(pending); start += shingleChunk {
|
||||
end := start + shingleChunk
|
||||
if end > len(pending) {
|
||||
end = len(pending)
|
||||
}
|
||||
batch := pending[start:end]
|
||||
|
||||
// Build a multi-row INSERT OR REPLACE: (?, ?, ?), (?, ?, ?), ...
|
||||
args := make([]any, 0, len(batch)*3)
|
||||
stmt := make([]byte, 0, 64+len(batch)*16)
|
||||
stmt = append(stmt, "INSERT OR REPLACE INTO clone_shingles (node_id, repo_prefix, shingles) VALUES "...)
|
||||
for i, e := range batch {
|
||||
if i > 0 {
|
||||
stmt = append(stmt, ',')
|
||||
}
|
||||
stmt = append(stmt, "(?, ?, ?)"...)
|
||||
args = append(args, e.id, repoPrefix, e.blob)
|
||||
}
|
||||
if _, err := tx.Exec(string(stmt), args...); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
return tx.Commit()
|
||||
}
|
||||
|
||||
// DeleteCloneShingles drops the rows for the supplied node ids, chunked
|
||||
// into `node_id IN (?, ?, …)` DELETEs so no statement exceeds SQLite's
|
||||
// host-parameter limit. Empty input is a no-op; missing ids are simply
|
||||
// not deleted.
|
||||
func (s *Store) DeleteCloneShingles(nodeIDs []string) error {
|
||||
if len(nodeIDs) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
// Dedupe + skip empty up front to keep the chunk loop honest.
|
||||
seen := make(map[string]struct{}, len(nodeIDs))
|
||||
uniq := make([]string, 0, len(nodeIDs))
|
||||
for _, id := range nodeIDs {
|
||||
if id == "" {
|
||||
continue
|
||||
}
|
||||
if _, ok := seen[id]; ok {
|
||||
continue
|
||||
}
|
||||
seen[id] = struct{}{}
|
||||
uniq = append(uniq, id)
|
||||
}
|
||||
if len(uniq) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
s.writeMu.Lock()
|
||||
defer s.writeMu.Unlock()
|
||||
|
||||
tx, err := s.db.Begin()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer tx.Rollback() //nolint:errcheck // rollback after Commit is a no-op
|
||||
|
||||
for start := 0; start < len(uniq); start += shingleChunk {
|
||||
end := start + shingleChunk
|
||||
if end > len(uniq) {
|
||||
end = len(uniq)
|
||||
}
|
||||
chunk := uniq[start:end]
|
||||
args := make([]any, len(chunk))
|
||||
stmt := make([]byte, 0, 48+len(chunk)*2)
|
||||
stmt = append(stmt, "DELETE FROM clone_shingles WHERE node_id IN ("...)
|
||||
for i, id := range chunk {
|
||||
if i > 0 {
|
||||
stmt = append(stmt, ',')
|
||||
}
|
||||
stmt = append(stmt, '?')
|
||||
args[i] = id
|
||||
}
|
||||
stmt = append(stmt, ')')
|
||||
if _, err := tx.Exec(string(stmt), args...); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
return tx.Commit()
|
||||
}
|
||||
|
||||
// LoadCloneShingles returns the recorded shingle sets for one repo
|
||||
// prefix as a fresh map. It always returns a non-nil (possibly empty)
|
||||
// map and surfaces any query error. An empty/absent prefix yields an
|
||||
// empty map, not an error.
|
||||
func (s *Store) LoadCloneShingles(repoPrefix string) (map[string][]uint64, error) {
|
||||
rows, err := s.db.Query(
|
||||
`SELECT node_id, shingles FROM clone_shingles WHERE repo_prefix = ?`,
|
||||
repoPrefix,
|
||||
)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
out := make(map[string][]uint64)
|
||||
for rows.Next() {
|
||||
var id string
|
||||
var blob []byte
|
||||
if err := rows.Scan(&id, &blob); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out[id] = decodeShingles(blob)
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
@@ -0,0 +1,63 @@
|
||||
package store_sqlite
|
||||
|
||||
// One-time boot compaction support.
|
||||
//
|
||||
// The graph store only ever grows on disk: deleted rows (a purged repo, the
|
||||
// duplicate-collapse migration, resolver cleanups) return their pages to
|
||||
// SQLite's freelist, where future writes reuse them — but nothing short of
|
||||
// VACUUM returns them to the filesystem. A long-lived store that shed a large
|
||||
// fraction of its rows can therefore pin gigabytes of dead file forever (a
|
||||
// live store sat at 64% freelist — 4.4 GB reclaimable in a 6.8 GB file).
|
||||
// These methods give the daemon the numbers to decide whether that one-time
|
||||
// rewrite is worth it, and the lever to run it. The policy (thresholds, disk
|
||||
// headroom, kill-switch) deliberately lives with the caller: the store cannot
|
||||
// know whether minutes of exclusive I/O are acceptable right now.
|
||||
|
||||
// Path returns the on-disk database file path. Empty when the store was not
|
||||
// opened from a file — callers using it to reason about the underlying
|
||||
// filesystem (disk-headroom checks) must treat "" as "unknown, don't".
|
||||
func (s *Store) Path() string {
|
||||
return s.dbPath
|
||||
}
|
||||
|
||||
// CompactStats reports how much of the database file is reclaimable dead
|
||||
// space: freeBytes is the freelist (freelist_count × page_size), totalBytes
|
||||
// the whole main file (page_count × page_size). Zeros on any pragma error —
|
||||
// a read failing here is the same teardown race panicOnFatal swallows, and
|
||||
// "nothing reclaimable" is the answer that makes every caller do nothing.
|
||||
// The -wal file is excluded on purpose: the checkpoint loop already bounds
|
||||
// it, and VACUUM only rewrites the main file.
|
||||
func (s *Store) CompactStats() (freeBytes, totalBytes int64) {
|
||||
var pageSize, pageCount, freePages int64
|
||||
if err := s.db.QueryRow(`PRAGMA page_size`).Scan(&pageSize); err != nil {
|
||||
return 0, 0
|
||||
}
|
||||
if err := s.db.QueryRow(`PRAGMA page_count`).Scan(&pageCount); err != nil {
|
||||
return 0, 0
|
||||
}
|
||||
if err := s.db.QueryRow(`PRAGMA freelist_count`).Scan(&freePages); err != nil {
|
||||
return 0, 0
|
||||
}
|
||||
return freePages * pageSize, pageCount * pageSize
|
||||
}
|
||||
|
||||
// Compact rewrites the database file (VACUUM), returning freelist pages to
|
||||
// the filesystem, then drains the write-ahead log with a TRUNCATE checkpoint
|
||||
// so the rewrite's WAL traffic doesn't linger as a second oversized file.
|
||||
//
|
||||
// Cost model callers must respect: VACUUM copies the live content into a
|
||||
// temporary database (up to a full extra copy on the same filesystem) and
|
||||
// needs exclusive access — it blocks Go-side writers via writeMu here, and a
|
||||
// concurrent reader on another pooled connection makes SQLite wait out
|
||||
// busy_timeout and then fail. That failure is clean (the store is untouched,
|
||||
// freelist pages remain reusable), which is why the daemon treats a Compact
|
||||
// error as skip-and-continue rather than fatal.
|
||||
func (s *Store) Compact() error {
|
||||
s.writeMu.Lock()
|
||||
_, err := s.db.Exec(`VACUUM`)
|
||||
s.writeMu.Unlock()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return s.CheckpointWAL()
|
||||
}
|
||||
@@ -0,0 +1,76 @@
|
||||
package store_sqlite_test
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"github.com/zzet/gortex/internal/graph"
|
||||
)
|
||||
|
||||
// TestCompactReclaimsFreelist pins the boot-compaction capability end to end
|
||||
// on a real file: shedding most rows leaves the file full of freelist pages
|
||||
// (CompactStats must say so), Compact() must return them to the filesystem
|
||||
// (page_count shrinks), and the surviving rows must come through untouched.
|
||||
// Fractional assertions only — page size and per-row overhead are backend
|
||||
// details this test must not encode.
|
||||
func TestCompactReclaimsFreelist(t *testing.T) {
|
||||
s := openTestStore(t)
|
||||
|
||||
// Seed 60 files × 50 nodes with ~2 KiB of meta each (~6 MiB of pages),
|
||||
// then checkpoint so the rows land in the main file rather than the WAL —
|
||||
// CompactStats deliberately measures only the main file.
|
||||
const files, perFile = 60, 50
|
||||
pad := strings.Repeat("x", 2048)
|
||||
for f := 0; f < files; f++ {
|
||||
nodes := make([]*graph.Node, 0, perFile)
|
||||
path := fmt.Sprintf("p/f%03d.go", f)
|
||||
for n := 0; n < perFile; n++ {
|
||||
nodes = append(nodes, &graph.Node{
|
||||
ID: fmt.Sprintf("%s::N%d", path, n),
|
||||
Kind: graph.KindFunction,
|
||||
Name: fmt.Sprintf("N%d", n),
|
||||
FilePath: path,
|
||||
Meta: map[string]any{"pad": pad},
|
||||
})
|
||||
}
|
||||
s.AddBatch(nodes, nil)
|
||||
}
|
||||
require.NoError(t, s.CheckpointWAL())
|
||||
_, totalSeeded := s.CompactStats()
|
||||
require.Greater(t, totalSeeded, int64(3<<20), "sanity: seeding must produce a multi-MiB main file")
|
||||
|
||||
// Shed all but one file, checkpoint again so the deletions reach the main
|
||||
// file's freelist.
|
||||
for f := 1; f < files; f++ {
|
||||
s.EvictFile(fmt.Sprintf("p/f%03d.go", f))
|
||||
}
|
||||
require.NoError(t, s.CheckpointWAL())
|
||||
|
||||
freeBefore, totalBefore := s.CompactStats()
|
||||
assert.Greater(t, freeBefore*3, totalBefore,
|
||||
"after shedding ~98%% of rows the freelist must dominate the file (free=%d total=%d)", freeBefore, totalBefore)
|
||||
assert.Greater(t, freeBefore, int64(1<<20), "freelist must be at least MiB-scale to make the shrink observable")
|
||||
|
||||
require.NoError(t, s.Compact())
|
||||
|
||||
freeAfter, totalAfter := s.CompactStats()
|
||||
assert.Less(t, totalAfter, totalBefore/2,
|
||||
"VACUUM must return the dead majority to the filesystem (before=%d after=%d)", totalBefore, totalAfter)
|
||||
assert.Less(t, freeAfter, totalBefore/10,
|
||||
"post-VACUUM freelist must be near empty (free=%d)", freeAfter)
|
||||
|
||||
// Survivors intact, evicted rows gone.
|
||||
assert.Equal(t, perFile, s.NodeCount(), "compaction must not change the row count")
|
||||
kept := s.GetNode("p/f000.go::N0")
|
||||
require.NotNil(t, kept, "kept row must survive VACUUM")
|
||||
assert.Equal(t, pad, kept.Meta["pad"], "kept row's meta blob must round-trip through VACUUM")
|
||||
assert.Nil(t, s.GetNode("p/f001.go::N0"), "evicted row must stay gone")
|
||||
|
||||
// The store stays fully writable after the rewrite.
|
||||
s.AddNode(&graph.Node{ID: "p/new.go::After", Kind: graph.KindFunction, Name: "After", FilePath: "p/new.go"})
|
||||
assert.NotNil(t, s.GetNode("p/new.go::After"))
|
||||
}
|
||||
@@ -0,0 +1,144 @@
|
||||
package store_sqlite
|
||||
|
||||
import (
|
||||
"github.com/zzet/gortex/internal/graph"
|
||||
)
|
||||
|
||||
// Compile-time assertions that the SQLite Store satisfies the optional
|
||||
// constant-value persistence capability. A KindConstant node's literal
|
||||
// value lives in this queryable sidecar (not the JSON Meta blob)
|
||||
// so the resolver can dereference a const-identifier dispatch name across
|
||||
// files without an unindexable per-node blob decode.
|
||||
var (
|
||||
_ graph.ConstantValueWriter = (*Store)(nil)
|
||||
_ graph.ConstantValueReader = (*Store)(nil)
|
||||
)
|
||||
|
||||
// constValueChunk bounds rows per multi-row INSERT (4 params/row; 80 rows
|
||||
// = 320 host params, well under SQLite's 999 default).
|
||||
const constValueChunk = 80
|
||||
|
||||
// BulkSetConstantValues persists constant values for one repo prefix in a
|
||||
// single transaction, chunked under the host-parameter limit. Idempotent
|
||||
// on the node_id primary key. Empty input is a no-op.
|
||||
func (s *Store) BulkSetConstantValues(repoPrefix string, rows []graph.ConstantValueRow) error {
|
||||
if len(rows) == 0 {
|
||||
return nil
|
||||
}
|
||||
s.writeMu.Lock()
|
||||
defer s.writeMu.Unlock()
|
||||
|
||||
tx, err := s.db.Begin()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer tx.Rollback() //nolint:errcheck // rollback after Commit is a no-op
|
||||
|
||||
for start := 0; start < len(rows); start += constValueChunk {
|
||||
end := start + constValueChunk
|
||||
if end > len(rows) {
|
||||
end = len(rows)
|
||||
}
|
||||
batch := rows[start:end]
|
||||
args := make([]any, 0, len(batch)*4)
|
||||
stmt := make([]byte, 0, 96+len(batch)*16)
|
||||
stmt = append(stmt, "INSERT OR REPLACE INTO constant_values (node_id, repo_prefix, file_path, value) VALUES "...)
|
||||
for i, r := range batch {
|
||||
if i > 0 {
|
||||
stmt = append(stmt, ',')
|
||||
}
|
||||
stmt = append(stmt, "(?, ?, ?, ?)"...)
|
||||
args = append(args, r.NodeID, repoPrefix, r.FilePath, r.Value)
|
||||
}
|
||||
if _, err := tx.Exec(string(stmt), args...); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return tx.Commit()
|
||||
}
|
||||
|
||||
// DeleteConstantValuesByFiles drops all constant values sourced in the
|
||||
// supplied files for one repo prefix, chunked into `file_path IN (…)`
|
||||
// DELETEs. Empty input is a no-op.
|
||||
func (s *Store) DeleteConstantValuesByFiles(repoPrefix string, files []string) error {
|
||||
if len(files) == 0 {
|
||||
return nil
|
||||
}
|
||||
s.writeMu.Lock()
|
||||
defer s.writeMu.Unlock()
|
||||
|
||||
tx, err := s.db.Begin()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer tx.Rollback() //nolint:errcheck // rollback after Commit is a no-op
|
||||
|
||||
for start := 0; start < len(files); start += constValueChunk {
|
||||
end := start + constValueChunk
|
||||
if end > len(files) {
|
||||
end = len(files)
|
||||
}
|
||||
chunk := files[start:end]
|
||||
args := make([]any, 0, len(chunk)+1)
|
||||
args = append(args, repoPrefix)
|
||||
stmt := make([]byte, 0, 64+len(chunk)*2)
|
||||
stmt = append(stmt, "DELETE FROM constant_values WHERE repo_prefix = ? AND file_path IN ("...)
|
||||
for i, f := range chunk {
|
||||
if i > 0 {
|
||||
stmt = append(stmt, ',')
|
||||
}
|
||||
stmt = append(stmt, '?')
|
||||
args = append(args, f)
|
||||
}
|
||||
stmt = append(stmt, ')')
|
||||
if _, err := tx.Exec(string(stmt), args...); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return tx.Commit()
|
||||
}
|
||||
|
||||
// ConstantValuesByNodeIDs returns the persisted values for the supplied
|
||||
// node ids (omitting ids with no recorded value). Always non-nil.
|
||||
func (s *Store) ConstantValuesByNodeIDs(nodeIDs []string) (map[string]string, error) {
|
||||
out := make(map[string]string, len(nodeIDs))
|
||||
if len(nodeIDs) == 0 {
|
||||
return out, nil
|
||||
}
|
||||
for start := 0; start < len(nodeIDs); start += constValueChunk {
|
||||
end := start + constValueChunk
|
||||
if end > len(nodeIDs) {
|
||||
end = len(nodeIDs)
|
||||
}
|
||||
chunk := nodeIDs[start:end]
|
||||
args := make([]any, 0, len(chunk))
|
||||
stmt := make([]byte, 0, 64+len(chunk)*2)
|
||||
stmt = append(stmt, "SELECT node_id, value FROM constant_values WHERE node_id IN ("...)
|
||||
for i, id := range chunk {
|
||||
if i > 0 {
|
||||
stmt = append(stmt, ',')
|
||||
}
|
||||
stmt = append(stmt, '?')
|
||||
args = append(args, id)
|
||||
}
|
||||
stmt = append(stmt, ')')
|
||||
rows, err := s.db.Query(string(stmt), args...)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for rows.Next() {
|
||||
var id, val string
|
||||
if err := rows.Scan(&id, &val); err != nil {
|
||||
_ = rows.Close()
|
||||
return nil, err
|
||||
}
|
||||
out[id] = val
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
_ = rows.Close()
|
||||
return nil, err
|
||||
}
|
||||
_ = rows.Close()
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
@@ -0,0 +1,73 @@
|
||||
package store_sqlite_test
|
||||
|
||||
import (
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"github.com/zzet/gortex/internal/graph"
|
||||
"github.com/zzet/gortex/internal/graph/store_sqlite"
|
||||
)
|
||||
|
||||
func openConstValStore(t *testing.T) *store_sqlite.Store {
|
||||
t.Helper()
|
||||
s, err := store_sqlite.Open(filepath.Join(t.TempDir(), "cv.sqlite"))
|
||||
require.NoError(t, err)
|
||||
t.Cleanup(func() { _ = s.Close() })
|
||||
return s
|
||||
}
|
||||
|
||||
func TestConstantValues_Roundtrip(t *testing.T) {
|
||||
s := openConstValStore(t)
|
||||
rows := []graph.ConstantValueRow{
|
||||
{NodeID: "a.go::ChargeCardActivity", FilePath: "a.go", Value: "ChargeCard"},
|
||||
{NodeID: "a.go::RefundActivity", FilePath: "a.go", Value: "Refund"},
|
||||
}
|
||||
require.NoError(t, s.BulkSetConstantValues("repo", rows))
|
||||
|
||||
got, err := s.ConstantValuesByNodeIDs([]string{"a.go::ChargeCardActivity", "a.go::RefundActivity", "missing"})
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "ChargeCard", got["a.go::ChargeCardActivity"])
|
||||
assert.Equal(t, "Refund", got["a.go::RefundActivity"])
|
||||
_, ok := got["missing"]
|
||||
assert.False(t, ok)
|
||||
}
|
||||
|
||||
func TestConstantValues_DeleteByFile(t *testing.T) {
|
||||
s := openConstValStore(t)
|
||||
require.NoError(t, s.BulkSetConstantValues("repo", []graph.ConstantValueRow{
|
||||
{NodeID: "a.go::X", FilePath: "a.go", Value: "vx"},
|
||||
{NodeID: "b.go::Y", FilePath: "b.go", Value: "vy"},
|
||||
}))
|
||||
require.NoError(t, s.DeleteConstantValuesByFiles("repo", []string{"a.go"}))
|
||||
|
||||
got, err := s.ConstantValuesByNodeIDs([]string{"a.go::X", "b.go::Y"})
|
||||
require.NoError(t, err)
|
||||
_, gone := got["a.go::X"]
|
||||
assert.False(t, gone, "a.go's value must be deleted")
|
||||
assert.Equal(t, "vy", got["b.go::Y"], "b.go's value must remain")
|
||||
}
|
||||
|
||||
func TestConstantValues_Replace(t *testing.T) {
|
||||
s := openConstValStore(t)
|
||||
require.NoError(t, s.BulkSetConstantValues("repo", []graph.ConstantValueRow{
|
||||
{NodeID: "a.go::X", FilePath: "a.go", Value: "old"},
|
||||
}))
|
||||
require.NoError(t, s.BulkSetConstantValues("repo", []graph.ConstantValueRow{
|
||||
{NodeID: "a.go::X", FilePath: "a.go", Value: "new"},
|
||||
}))
|
||||
got, err := s.ConstantValuesByNodeIDs([]string{"a.go::X"})
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "new", got["a.go::X"], "INSERT OR REPLACE must update by node_id PK")
|
||||
}
|
||||
|
||||
func TestConstantValues_EmptyNoop(t *testing.T) {
|
||||
s := openConstValStore(t)
|
||||
require.NoError(t, s.BulkSetConstantValues("repo", nil))
|
||||
require.NoError(t, s.DeleteConstantValuesByFiles("repo", nil))
|
||||
got, err := s.ConstantValuesByNodeIDs(nil)
|
||||
require.NoError(t, err)
|
||||
assert.Empty(t, got)
|
||||
}
|
||||
@@ -0,0 +1,297 @@
|
||||
package store_sqlite
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"strings"
|
||||
|
||||
"github.com/zzet/gortex/internal/graph"
|
||||
)
|
||||
|
||||
// This file implements graph.ContentSearcher on the SQLite backend using
|
||||
// the content_fts FTS5 virtual table declared in schema.go — the
|
||||
// dedicated, on-disk full-text index for CONTENT (data_class="content")
|
||||
// section bodies, kept physically separate from symbol_fts so content
|
||||
// text never reaches the symbol search or the code-oriented graph passes.
|
||||
//
|
||||
// Streamed build: WipeContent(repoPrefix) once at the start of a full
|
||||
// index, AppendContent each content file's sections as they are parsed
|
||||
// (no per-file wipe), then BuildContentIndex to merge segments.
|
||||
// Incremental reindex of one content file is WipeContentFile +
|
||||
// AppendContent.
|
||||
|
||||
// Compile-time assertion: *Store satisfies the content-search capability.
|
||||
var _ graph.ContentSearcher = (*Store)(nil)
|
||||
|
||||
// contentInsertChunkRows bounds rows per multi-row INSERT. Each row binds
|
||||
// 5 host params (node_id, repo_prefix, file_path, ordinal, body); 180 rows
|
||||
// is 900 params, comfortably under SQLite's default 999-variable limit.
|
||||
const contentInsertChunkRows = 180
|
||||
|
||||
// WipeContent removes a repo's content rows before a full rebuild. Empty
|
||||
// repoPrefix wipes the whole table (single-repo / conformance behaviour).
|
||||
func (s *Store) WipeContent(repoPrefix string) error {
|
||||
s.writeMu.Lock()
|
||||
defer s.writeMu.Unlock()
|
||||
_, err := s.db.Exec(`DELETE FROM content_fts WHERE repo_prefix = ?`, repoPrefix)
|
||||
return err
|
||||
}
|
||||
|
||||
// WipeContentFile removes one file's content rows — the incremental
|
||||
// reindex path when a single content file changes.
|
||||
func (s *Store) WipeContentFile(filePath string) error {
|
||||
if filePath == "" {
|
||||
return nil
|
||||
}
|
||||
s.writeMu.Lock()
|
||||
defer s.writeMu.Unlock()
|
||||
_, err := s.db.Exec(`DELETE FROM content_fts WHERE file_path = ?`, filePath)
|
||||
return err
|
||||
}
|
||||
|
||||
// WipeContentFileInRepo removes ONE file's content rows scoped to a repo —
|
||||
// the crash-safe full-index sibling of WipeContentFile (which keys on
|
||||
// file_path alone and so would clobber a same-named file in another repo).
|
||||
// A full index streams content per file: delete this file's prior rows,
|
||||
// then AppendContent its fresh sections — so a mid-parse kill leaves a mix
|
||||
// of old+new content per file rather than the empty table a repo-wide
|
||||
// pre-wipe would leave behind. Empty filePath is a no-op.
|
||||
func (s *Store) WipeContentFileInRepo(repoPrefix, filePath string) error {
|
||||
if filePath == "" {
|
||||
return nil
|
||||
}
|
||||
s.writeMu.Lock()
|
||||
defer s.writeMu.Unlock()
|
||||
_, err := s.db.Exec(`DELETE FROM content_fts WHERE repo_prefix = ? AND file_path = ?`, repoPrefix, filePath)
|
||||
return err
|
||||
}
|
||||
|
||||
// DeleteContentFilesForRepoNotIn sweeps a repo's content rows down to keep —
|
||||
// every content row whose file_path is absent from keep is deleted. keep is
|
||||
// the set of files that actually STREAMED content sections in the walk just
|
||||
// completed (each recorded as the same file_path form AppendContent wrote),
|
||||
// NOT the set of files that merely survive on disk: a file can still exist
|
||||
// yet stop producing content (doc emptied, classification changed), and a
|
||||
// disk-based keep would protect its stale rows forever. Run once at the end
|
||||
// of a successful full index (right after the authoritative mtime replace),
|
||||
// it reaps vanished files and content->no-content transitions in one scan;
|
||||
// the per-file WipeContentFileInRepo + AppendContent streaming build
|
||||
// refreshes the files that still produce content. Together they replace the
|
||||
// old repo-wide pre-wipe: a mid-parse kill no longer empties the content
|
||||
// index, and stale rows are reaped only on the next clean completion.
|
||||
// Empty keep is a deliberate no-op safety net — never wipe a whole repo from
|
||||
// an empty set; a caller that legitimately ends a walk with zero content
|
||||
// files calls WipeContent explicitly instead.
|
||||
func (s *Store) DeleteContentFilesForRepoNotIn(repoPrefix string, keep map[string]struct{}) error {
|
||||
if len(keep) == 0 {
|
||||
return nil
|
||||
}
|
||||
s.writeMu.Lock()
|
||||
defer s.writeMu.Unlock()
|
||||
|
||||
// Enumerate the repo's content file paths, then delete only those not in
|
||||
// keep. Content files are a small subset (docs / PDFs / office), so the
|
||||
// DISTINCT scan + targeted deletes stay cheap and dodge a giant NOT IN
|
||||
// (...) bound-variable list. Rows are drained + closed before the delete
|
||||
// tx opens (no open read cursor while writing on the same connection).
|
||||
rows, err := s.db.Query(`SELECT DISTINCT file_path FROM content_fts WHERE repo_prefix = ?`, repoPrefix)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
var vanished []string
|
||||
for rows.Next() {
|
||||
var fp string
|
||||
if err := rows.Scan(&fp); err != nil {
|
||||
_ = rows.Close()
|
||||
return err
|
||||
}
|
||||
if _, ok := keep[fp]; !ok {
|
||||
vanished = append(vanished, fp)
|
||||
}
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
_ = rows.Close()
|
||||
return err
|
||||
}
|
||||
_ = rows.Close()
|
||||
if len(vanished) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
tx, err := s.db.Begin()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer tx.Rollback() //nolint:errcheck // rollback after Commit is a no-op
|
||||
const chunk = 900
|
||||
for start := 0; start < len(vanished); start += chunk {
|
||||
end := minInt(start+chunk, len(vanished))
|
||||
batch := vanished[start:end]
|
||||
placeholders := strings.TrimSuffix(strings.Repeat("?,", len(batch)), ",")
|
||||
args := make([]any, 0, len(batch)+1)
|
||||
args = append(args, repoPrefix)
|
||||
for _, fp := range batch {
|
||||
args = append(args, fp)
|
||||
}
|
||||
if _, err := tx.Exec(`DELETE FROM content_fts WHERE repo_prefix = ? AND file_path IN (`+placeholders+`)`, args...); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return tx.Commit()
|
||||
}
|
||||
|
||||
// AppendContent inserts content rows for repoPrefix without wiping — the
|
||||
// streamed per-file build path. Callers wipe (whole repo or one file)
|
||||
// first. Rows with an empty NodeID are skipped.
|
||||
func (s *Store) AppendContent(repoPrefix string, items []graph.ContentFTSItem) error {
|
||||
if len(items) == 0 {
|
||||
return nil
|
||||
}
|
||||
s.writeMu.Lock()
|
||||
defer s.writeMu.Unlock()
|
||||
|
||||
tx, err := s.db.Begin()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
commit := false
|
||||
defer func() {
|
||||
if !commit {
|
||||
_ = tx.Rollback()
|
||||
}
|
||||
}()
|
||||
|
||||
for start := 0; start < len(items); start += contentInsertChunkRows {
|
||||
end := minInt(start+contentInsertChunkRows, len(items))
|
||||
chunk := items[start:end]
|
||||
|
||||
var b strings.Builder
|
||||
b.WriteString(`INSERT INTO content_fts (node_id, repo_prefix, file_path, ordinal, body) VALUES `)
|
||||
args := make([]any, 0, len(chunk)*5)
|
||||
for _, it := range chunk {
|
||||
if it.NodeID == "" {
|
||||
continue
|
||||
}
|
||||
if len(args) > 0 {
|
||||
b.WriteByte(',')
|
||||
}
|
||||
b.WriteString(`(?,?,?,?,?)`)
|
||||
args = append(args, it.NodeID, repoPrefix, it.FilePath, it.Ordinal, it.Body)
|
||||
}
|
||||
if len(args) == 0 {
|
||||
continue
|
||||
}
|
||||
if _, err := tx.Exec(b.String(), args...); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
if err := tx.Commit(); err != nil {
|
||||
return err
|
||||
}
|
||||
commit = true
|
||||
return nil
|
||||
}
|
||||
|
||||
// BuildContentIndex opportunistically merges FTS5 segments (a read-latency
|
||||
// improvement). Like BuildSymbolIndex it is a no-op for correctness — the
|
||||
// FTS index is maintained incrementally on every insert — and idempotent.
|
||||
func (s *Store) BuildContentIndex() error {
|
||||
s.writeMu.Lock()
|
||||
defer s.writeMu.Unlock()
|
||||
_, _ = s.db.Exec(`INSERT INTO content_fts(content_fts) VALUES('optimize')`)
|
||||
return nil
|
||||
}
|
||||
|
||||
// SearchContent runs a content query scoped to repoPrefix (empty = all
|
||||
// repos) and returns hits ordered by descending relevance, each carrying a
|
||||
// short snippet excerpt from the matched body. Reuses the symbol path's
|
||||
// write-side tokeniser (buildFTSMatch) so the content corpus and queries
|
||||
// agree on camelCase / path-separator splitting.
|
||||
func (s *Store) SearchContent(query, repoPrefix string, limit int) ([]graph.ContentHit, error) {
|
||||
if query == "" {
|
||||
return nil, nil
|
||||
}
|
||||
if limit <= 0 {
|
||||
limit = 20
|
||||
}
|
||||
match := s.buildFTSMatch(query)
|
||||
if match == "" {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
var sb strings.Builder
|
||||
// snippet() over the body column (index 4): no highlight marks, an
|
||||
// ellipsis for elision, ~16 tokens of context. CAST(ordinal AS INTEGER)
|
||||
// forces integer affinity so the FTS5 text column scans cleanly into an
|
||||
// int.
|
||||
sb.WriteString(`SELECT node_id, file_path, CAST(ordinal AS INTEGER), snippet(content_fts, 4, '', '', '…', 16), bm25(content_fts) FROM content_fts WHERE content_fts MATCH ?`)
|
||||
args := []any{match}
|
||||
if repoPrefix != "" {
|
||||
sb.WriteString(` AND repo_prefix = ?`)
|
||||
args = append(args, repoPrefix)
|
||||
}
|
||||
sb.WriteString(` ORDER BY bm25(content_fts) LIMIT ?`)
|
||||
args = append(args, limit)
|
||||
|
||||
rows, err := s.db.Query(sb.String(), args...)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
var hits []graph.ContentHit
|
||||
for rows.Next() {
|
||||
var (
|
||||
id, fp, snip string
|
||||
ordinal int
|
||||
score float64
|
||||
)
|
||||
if err := rows.Scan(&id, &fp, &ordinal, &snip, &score); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if id == "" {
|
||||
continue
|
||||
}
|
||||
// bm25() is negative-better in SQLite; negate so higher = better,
|
||||
// matching the ContentHit contract. Rows already arrive best-first.
|
||||
hits = append(hits, graph.ContentHit{
|
||||
NodeID: id,
|
||||
FilePath: fp,
|
||||
Ordinal: ordinal,
|
||||
Score: -score,
|
||||
Snippet: snip,
|
||||
})
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return hits, nil
|
||||
}
|
||||
|
||||
// ScanContent streams every content row (scoped to repoPrefix; empty = all
|
||||
// repos) to fn with its FULL body, read incrementally via a cursor so a
|
||||
// consumer iterating hundreds of thousands of sections stays bounded. fn
|
||||
// returns false to stop the scan early.
|
||||
func (s *Store) ScanContent(repoPrefix string, fn func(nodeID, filePath, body string) bool) error {
|
||||
var rows *sql.Rows
|
||||
var err error
|
||||
if repoPrefix == "" {
|
||||
rows, err = s.db.Query(`SELECT node_id, file_path, body FROM content_fts`)
|
||||
} else {
|
||||
rows, err = s.db.Query(`SELECT node_id, file_path, body FROM content_fts WHERE repo_prefix = ?`, repoPrefix)
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer rows.Close()
|
||||
for rows.Next() {
|
||||
var nodeID, filePath, body string
|
||||
if err := rows.Scan(&nodeID, &filePath, &body); err != nil {
|
||||
return err
|
||||
}
|
||||
if !fn(nodeID, filePath, body) {
|
||||
return nil
|
||||
}
|
||||
}
|
||||
return rows.Err()
|
||||
}
|
||||
@@ -0,0 +1,82 @@
|
||||
package store_sqlite
|
||||
|
||||
import (
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"github.com/zzet/gortex/internal/graph"
|
||||
)
|
||||
|
||||
func TestContentFTS_BasicAndFileWipe(t *testing.T) {
|
||||
s, err := Open(filepath.Join(t.TempDir(), "c.sqlite"))
|
||||
require.NoError(t, err)
|
||||
t.Cleanup(func() { _ = s.Close() })
|
||||
|
||||
var cs graph.ContentSearcher = s // runtime assertion the store satisfies the capability
|
||||
|
||||
require.NoError(t, cs.WipeContent("")) // clean table
|
||||
items := []graph.ContentFTSItem{
|
||||
{NodeID: "a.txt::doc:section-0", FilePath: "a.txt", Ordinal: 0, Body: "the quick brown fox jumps over the lazy dog"},
|
||||
{NodeID: "a.txt::doc:section-1", FilePath: "a.txt", Ordinal: 1, Body: "lorem ipsum dolor sit amet consectetur"},
|
||||
{NodeID: "b.pdf::doc:pdf_page-0", FilePath: "b.pdf", Ordinal: 0, Body: "quantum entanglement and superposition explained"},
|
||||
}
|
||||
require.NoError(t, cs.AppendContent("", items))
|
||||
require.NoError(t, cs.BuildContentIndex())
|
||||
|
||||
// A body term resolves to exactly its section, with a non-empty snippet.
|
||||
hits, err := cs.SearchContent("quantum", "", 10)
|
||||
require.NoError(t, err)
|
||||
require.Len(t, hits, 1)
|
||||
require.Equal(t, "b.pdf::doc:pdf_page-0", hits[0].NodeID)
|
||||
require.Equal(t, "b.pdf", hits[0].FilePath)
|
||||
require.Equal(t, 0, hits[0].Ordinal)
|
||||
require.NotEmpty(t, hits[0].Snippet)
|
||||
|
||||
hits, err = cs.SearchContent("fox", "", 10)
|
||||
require.NoError(t, err)
|
||||
require.Len(t, hits, 1)
|
||||
require.Equal(t, "a.txt::doc:section-0", hits[0].NodeID)
|
||||
require.Equal(t, 0, hits[0].Ordinal)
|
||||
|
||||
// WipeContentFile drops only a.txt's rows; b.pdf survives.
|
||||
require.NoError(t, cs.WipeContentFile("a.txt"))
|
||||
hits, err = cs.SearchContent("fox", "", 10)
|
||||
require.NoError(t, err)
|
||||
require.Empty(t, hits)
|
||||
hits, err = cs.SearchContent("quantum", "", 10)
|
||||
require.NoError(t, err)
|
||||
require.Len(t, hits, 1)
|
||||
}
|
||||
|
||||
func TestContentFTS_RepoScoping(t *testing.T) {
|
||||
s, err := Open(filepath.Join(t.TempDir(), "c.sqlite"))
|
||||
require.NoError(t, err)
|
||||
t.Cleanup(func() { _ = s.Close() })
|
||||
|
||||
require.NoError(t, s.AppendContent("repoA", []graph.ContentFTSItem{
|
||||
{NodeID: "repoA::x.txt::doc:section-0", FilePath: "x.txt", Body: "alpha beta gamma"},
|
||||
}))
|
||||
require.NoError(t, s.AppendContent("repoB", []graph.ContentFTSItem{
|
||||
{NodeID: "repoB::y.txt::doc:section-0", FilePath: "y.txt", Body: "alpha delta epsilon"},
|
||||
}))
|
||||
|
||||
// Scoped search returns only the matching repo's hit.
|
||||
hits, err := s.SearchContent("alpha", "repoA", 10)
|
||||
require.NoError(t, err)
|
||||
require.Len(t, hits, 1)
|
||||
require.Equal(t, "repoA::x.txt::doc:section-0", hits[0].NodeID)
|
||||
|
||||
// Unscoped search spans both repos.
|
||||
hits, err = s.SearchContent("alpha", "", 10)
|
||||
require.NoError(t, err)
|
||||
require.Len(t, hits, 2)
|
||||
|
||||
// WipeContent scopes to one repo, leaving the sibling intact.
|
||||
require.NoError(t, s.WipeContent("repoA"))
|
||||
hits, err = s.SearchContent("alpha", "", 10)
|
||||
require.NoError(t, err)
|
||||
require.Len(t, hits, 1)
|
||||
require.Equal(t, "repoB::y.txt::doc:section-0", hits[0].NodeID)
|
||||
}
|
||||
@@ -0,0 +1,143 @@
|
||||
package store_sqlite
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
|
||||
"github.com/zzet/gortex/internal/graph"
|
||||
)
|
||||
|
||||
var (
|
||||
_ graph.CoverageEnrichmentWriter = (*Store)(nil)
|
||||
_ graph.CoverageEnrichmentReader = (*Store)(nil)
|
||||
)
|
||||
|
||||
// coverageChunk bounds rows per multi-row INSERT (5 cols → 5 params/row;
|
||||
// 999/5 ≈ 199 max, 180 leaves headroom).
|
||||
const coverageChunk = 180
|
||||
|
||||
const coverageCols = `node_id, repo_prefix, coverage_pct, num_stmt, hit`
|
||||
|
||||
// BulkSetCoverage persists coverage rows for one repo prefix in a single
|
||||
// chunked transaction. Idempotent on node_id. Empty input is a no-op.
|
||||
func (s *Store) BulkSetCoverage(repoPrefix string, rows []graph.CoverageEnrichment) error {
|
||||
if len(rows) == 0 {
|
||||
return nil
|
||||
}
|
||||
s.writeMu.Lock()
|
||||
defer s.writeMu.Unlock()
|
||||
|
||||
tx, err := s.db.Begin()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer tx.Rollback() //nolint:errcheck
|
||||
|
||||
for start := 0; start < len(rows); start += coverageChunk {
|
||||
end := start + coverageChunk
|
||||
if end > len(rows) {
|
||||
end = len(rows)
|
||||
}
|
||||
batch := rows[start:end]
|
||||
args := make([]any, 0, len(batch)*5)
|
||||
stmt := make([]byte, 0, 96+len(batch)*16)
|
||||
stmt = append(stmt, "INSERT OR REPLACE INTO coverage_enrichment ("...)
|
||||
stmt = append(stmt, coverageCols...)
|
||||
stmt = append(stmt, ") VALUES "...)
|
||||
for i, e := range batch {
|
||||
if i > 0 {
|
||||
stmt = append(stmt, ',')
|
||||
}
|
||||
stmt = append(stmt, "(?,?,?,?,?)"...)
|
||||
args = append(args, e.NodeID, repoPrefix, e.CoveragePct, e.NumStmt, e.Hit)
|
||||
}
|
||||
if _, err := tx.Exec(string(stmt), args...); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return tx.Commit()
|
||||
}
|
||||
|
||||
// DeleteCoverage drops coverage rows for the supplied node ids, chunked.
|
||||
func (s *Store) DeleteCoverage(nodeIDs []string) error {
|
||||
if len(nodeIDs) == 0 {
|
||||
return nil
|
||||
}
|
||||
seen := make(map[string]struct{}, len(nodeIDs))
|
||||
uniq := make([]string, 0, len(nodeIDs))
|
||||
for _, id := range nodeIDs {
|
||||
if id == "" {
|
||||
continue
|
||||
}
|
||||
if _, ok := seen[id]; ok {
|
||||
continue
|
||||
}
|
||||
seen[id] = struct{}{}
|
||||
uniq = append(uniq, id)
|
||||
}
|
||||
if len(uniq) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
s.writeMu.Lock()
|
||||
defer s.writeMu.Unlock()
|
||||
|
||||
tx, err := s.db.Begin()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer tx.Rollback() //nolint:errcheck
|
||||
|
||||
for start := 0; start < len(uniq); start += coverageChunk {
|
||||
end := start + coverageChunk
|
||||
if end > len(uniq) {
|
||||
end = len(uniq)
|
||||
}
|
||||
chunk := uniq[start:end]
|
||||
args := make([]any, len(chunk))
|
||||
stmt := make([]byte, 0, 56+len(chunk)*2)
|
||||
stmt = append(stmt, "DELETE FROM coverage_enrichment WHERE node_id IN ("...)
|
||||
for i, id := range chunk {
|
||||
if i > 0 {
|
||||
stmt = append(stmt, ',')
|
||||
}
|
||||
stmt = append(stmt, '?')
|
||||
args[i] = id
|
||||
}
|
||||
stmt = append(stmt, ')')
|
||||
if _, err := tx.Exec(string(stmt), args...); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return tx.Commit()
|
||||
}
|
||||
|
||||
// CoverageRows returns coverage rows for repoPrefix; empty repoPrefix
|
||||
// returns ALL rows across repos. Index-only read over the enriched set.
|
||||
func (s *Store) CoverageRows(repoPrefix string) []graph.CoverageEnrichment {
|
||||
var (
|
||||
rows *sql.Rows
|
||||
err error
|
||||
)
|
||||
if repoPrefix == "" {
|
||||
rows, err = s.db.Query(`SELECT ` + coverageCols + ` FROM coverage_enrichment`)
|
||||
} else {
|
||||
rows, err = s.db.Query(`SELECT `+coverageCols+` FROM coverage_enrichment WHERE repo_prefix = ?`, repoPrefix)
|
||||
}
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
var out []graph.CoverageEnrichment
|
||||
for rows.Next() {
|
||||
var e graph.CoverageEnrichment
|
||||
if err := rows.Scan(&e.NodeID, &e.RepoPrefix, &e.CoveragePct, &e.NumStmt, &e.Hit); err != nil {
|
||||
return out
|
||||
}
|
||||
out = append(out, e)
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return out
|
||||
}
|
||||
return out
|
||||
}
|
||||
@@ -0,0 +1,21 @@
|
||||
package store_sqlite
|
||||
|
||||
import "os"
|
||||
|
||||
// DBStats returns the on-disk size of the SQLite database file and its
|
||||
// write-ahead log, in bytes. A missing file (or a store opened without a
|
||||
// path) reports 0 for that component. Surfaced in daemon_health so a
|
||||
// runaway WAL high-water mark is observable instead of silently filling
|
||||
// the disk.
|
||||
func (s *Store) DBStats() (dbBytes, walBytes int64) {
|
||||
if s == nil || s.dbPath == "" {
|
||||
return 0, 0
|
||||
}
|
||||
if fi, err := os.Stat(s.dbPath); err == nil {
|
||||
dbBytes = fi.Size()
|
||||
}
|
||||
if fi, err := os.Stat(s.dbPath + "-wal"); err == nil {
|
||||
walBytes = fi.Size()
|
||||
}
|
||||
return dbBytes, walBytes
|
||||
}
|
||||
@@ -0,0 +1,33 @@
|
||||
package store_sqlite_test
|
||||
|
||||
import (
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"github.com/zzet/gortex/internal/graph"
|
||||
"github.com/zzet/gortex/internal/graph/store_sqlite"
|
||||
)
|
||||
|
||||
func TestDBStats(t *testing.T) {
|
||||
path := filepath.Join(t.TempDir(), "g.sqlite")
|
||||
s, err := store_sqlite.Open(path)
|
||||
require.NoError(t, err)
|
||||
t.Cleanup(func() { _ = s.Close() })
|
||||
|
||||
// Force a write so the DB (and WAL, in WAL mode) carry real bytes.
|
||||
s.AddBatch([]*graph.Node{
|
||||
{ID: "a.go::Foo", Kind: graph.KindFunction, Name: "Foo", FilePath: "a.go"},
|
||||
}, nil)
|
||||
|
||||
dbBytes, walBytes := s.DBStats()
|
||||
require.Greater(t, dbBytes, int64(0), "the on-disk DB file must have nonzero size")
|
||||
require.GreaterOrEqual(t, walBytes, int64(0), "WAL size is non-negative (0 after a checkpoint)")
|
||||
|
||||
// A store with no path (the zero value) reports zero, never panics.
|
||||
var empty store_sqlite.Store
|
||||
db, wal := empty.DBStats()
|
||||
require.Equal(t, int64(0), db)
|
||||
require.Equal(t, int64(0), wal)
|
||||
}
|
||||
@@ -0,0 +1,45 @@
|
||||
package store_sqlite
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
|
||||
"github.com/zzet/gortex/internal/graph"
|
||||
)
|
||||
|
||||
// Compile-time assertion that the SQLite Store persists the enrichment
|
||||
// completion marker. Lifting this state into the same backend the graph
|
||||
// lives in lets a warm restart skip re-enriching a repo whose persisted
|
||||
// graph already carries its LSP edges — no second persistence surface.
|
||||
var _ graph.EnrichmentStateStore = (*Store)(nil)
|
||||
|
||||
// SetEnrichmentState upserts the completion marker for one (repo, provider) —
|
||||
// written when a provider finishes a non-partial enrichment pass. One row per
|
||||
// (repo_prefix, provider).
|
||||
func (s *Store) SetEnrichmentState(st graph.EnrichmentState) error {
|
||||
s.writeMu.Lock()
|
||||
defer s.writeMu.Unlock()
|
||||
_, err := s.db.Exec(`
|
||||
INSERT OR REPLACE INTO enrichment_state
|
||||
(repo_prefix, provider, indexed_sha, completed_at, coverage)
|
||||
VALUES (?, ?, ?, ?, ?)`,
|
||||
st.RepoPrefix, st.Provider, st.IndexedSHA, st.CompletedAt, st.Coverage)
|
||||
return err
|
||||
}
|
||||
|
||||
// GetEnrichmentState returns the recorded completion marker for a
|
||||
// (repo, provider). The bool is false when no row exists yet (never-enriched
|
||||
// or pre-feature).
|
||||
func (s *Store) GetEnrichmentState(repoPrefix, provider string) (graph.EnrichmentState, bool, error) {
|
||||
row := s.db.QueryRow(`
|
||||
SELECT indexed_sha, completed_at, coverage
|
||||
FROM enrichment_state WHERE repo_prefix = ? AND provider = ?`, repoPrefix, provider)
|
||||
st := graph.EnrichmentState{RepoPrefix: repoPrefix, Provider: provider}
|
||||
err := row.Scan(&st.IndexedSHA, &st.CompletedAt, &st.Coverage)
|
||||
if err == sql.ErrNoRows {
|
||||
return graph.EnrichmentState{RepoPrefix: repoPrefix, Provider: provider}, false, nil
|
||||
}
|
||||
if err != nil {
|
||||
return graph.EnrichmentState{RepoPrefix: repoPrefix, Provider: provider}, false, err
|
||||
}
|
||||
return st, true, nil
|
||||
}
|
||||
@@ -0,0 +1,97 @@
|
||||
package store_sqlite
|
||||
|
||||
import (
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"github.com/zzet/gortex/internal/graph"
|
||||
)
|
||||
|
||||
// TestEnrichmentStateRoundTrip: set/get, missing-row, upsert-in-place, and
|
||||
// per-provider row isolation on the SQLite store.
|
||||
func TestEnrichmentStateRoundTrip(t *testing.T) {
|
||||
s, err := Open(filepath.Join(t.TempDir(), "store.sqlite"))
|
||||
if err != nil {
|
||||
t.Fatalf("Open: %v", err)
|
||||
}
|
||||
defer s.Close()
|
||||
|
||||
// Missing row → found=false, no error (the "never enriched" signal).
|
||||
if _, found, err := s.GetEnrichmentState("repo", "gopls"); err != nil || found {
|
||||
t.Fatalf("missing row: found=%v err=%v, want found=false, err=nil", found, err)
|
||||
}
|
||||
|
||||
st := graph.EnrichmentState{
|
||||
RepoPrefix: "repo",
|
||||
Provider: "gopls",
|
||||
IndexedSHA: "abc123",
|
||||
CompletedAt: 1_700_000_000,
|
||||
Coverage: 91.5,
|
||||
}
|
||||
if err := s.SetEnrichmentState(st); err != nil {
|
||||
t.Fatalf("SetEnrichmentState: %v", err)
|
||||
}
|
||||
|
||||
got, found, err := s.GetEnrichmentState("repo", "gopls")
|
||||
if err != nil || !found {
|
||||
t.Fatalf("get after set: found=%v err=%v, want found=true", found, err)
|
||||
}
|
||||
if got.RepoPrefix != "repo" || got.Provider != "gopls" ||
|
||||
got.IndexedSHA != "abc123" || got.CompletedAt != 1_700_000_000 || got.Coverage != 91.5 {
|
||||
t.Fatalf("round-trip mismatch: %+v", got)
|
||||
}
|
||||
|
||||
// Upsert on (repo_prefix, provider) replaces in place.
|
||||
st.IndexedSHA = "def456"
|
||||
st.Coverage = 100
|
||||
if err := s.SetEnrichmentState(st); err != nil {
|
||||
t.Fatalf("upsert: %v", err)
|
||||
}
|
||||
got, _, _ = s.GetEnrichmentState("repo", "gopls")
|
||||
if got.IndexedSHA != "def456" || got.Coverage != 100 {
|
||||
t.Fatalf("upsert did not replace in place: %+v", got)
|
||||
}
|
||||
|
||||
// A different provider under the same repo is a distinct row.
|
||||
if _, found, _ := s.GetEnrichmentState("repo", "scip-go"); found {
|
||||
t.Fatalf("a different provider must be its own row, got found=true")
|
||||
}
|
||||
}
|
||||
|
||||
// TestOpenPreservesEnrichmentStateOnReopen proves the new table is picked up
|
||||
// by an existing store with no wipe: a marker written on the first open
|
||||
// survives the second open (which re-runs schemaSQL unconditionally) and the
|
||||
// schema version does not drift. Mirrors TestOpenAtCurrentVersionIsNoOp.
|
||||
func TestOpenPreservesEnrichmentStateOnReopen(t *testing.T) {
|
||||
path := filepath.Join(t.TempDir(), "store.sqlite")
|
||||
|
||||
s, err := Open(path)
|
||||
if err != nil {
|
||||
t.Fatalf("first open: %v", err)
|
||||
}
|
||||
if err := s.SetEnrichmentState(graph.EnrichmentState{
|
||||
RepoPrefix: "r", Provider: "gopls", IndexedSHA: "sha1", CompletedAt: 42, Coverage: 88,
|
||||
}); err != nil {
|
||||
t.Fatalf("seed marker: %v", err)
|
||||
}
|
||||
if err := s.Close(); err != nil {
|
||||
t.Fatalf("close: %v", err)
|
||||
}
|
||||
|
||||
s2, err := Open(path)
|
||||
if err != nil {
|
||||
t.Fatalf("reopen: %v", err)
|
||||
}
|
||||
defer s2.Close()
|
||||
|
||||
got, found, err := s2.GetEnrichmentState("r", "gopls")
|
||||
if err != nil || !found {
|
||||
t.Fatalf("marker lost across reopen: found=%v err=%v (the table must survive a no-op reopen)", found, err)
|
||||
}
|
||||
if got.IndexedSHA != "sha1" || got.Coverage != 88 {
|
||||
t.Fatalf("marker corrupted across reopen: %+v", got)
|
||||
}
|
||||
if v, _ := readUserVersion(s2.db); v != currentSchemaVersion {
|
||||
t.Fatalf("schema version drifted to %d after reopen, want %d", v, currentSchemaVersion)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,118 @@
|
||||
package store_sqlite
|
||||
|
||||
import (
|
||||
"github.com/zzet/gortex/internal/graph"
|
||||
)
|
||||
|
||||
// Compile-time assertions that the SQLite Store satisfies the optional
|
||||
// per-file metadata persistence capability (the files sidecar feeding
|
||||
// index_health's per-file parse-error / node-count rollup).
|
||||
var (
|
||||
_ graph.FileMetaWriter = (*Store)(nil)
|
||||
_ graph.FileMetaReader = (*Store)(nil)
|
||||
)
|
||||
|
||||
// fileMetaChunk bounds rows per multi-row INSERT (6 params/row; 80 rows =
|
||||
// 480 host params, well under SQLite's 999 default).
|
||||
const fileMetaChunk = 80
|
||||
|
||||
// SetFileMetas upserts per-file metadata rows for one repo prefix in a single
|
||||
// transaction, chunked under the host-parameter limit. Idempotent on the
|
||||
// (repo_prefix, file_path) primary key. Empty input is a no-op.
|
||||
func (s *Store) SetFileMetas(repoPrefix string, rows []graph.FileMetaRow) error {
|
||||
if len(rows) == 0 {
|
||||
return nil
|
||||
}
|
||||
s.writeMu.Lock()
|
||||
defer s.writeMu.Unlock()
|
||||
|
||||
tx, err := s.db.Begin()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer tx.Rollback() //nolint:errcheck // rollback after Commit is a no-op
|
||||
|
||||
for start := 0; start < len(rows); start += fileMetaChunk {
|
||||
end := start + fileMetaChunk
|
||||
if end > len(rows) {
|
||||
end = len(rows)
|
||||
}
|
||||
batch := rows[start:end]
|
||||
args := make([]any, 0, len(batch)*6)
|
||||
stmt := make([]byte, 0, 96+len(batch)*24)
|
||||
stmt = append(stmt, "INSERT OR REPLACE INTO files (repo_prefix, file_path, content_hash, size, node_count, errors) VALUES "...)
|
||||
for i, r := range batch {
|
||||
if i > 0 {
|
||||
stmt = append(stmt, ',')
|
||||
}
|
||||
stmt = append(stmt, "(?, ?, ?, ?, ?, ?)"...)
|
||||
args = append(args, repoPrefix, r.FilePath, r.ContentHash, r.Size, r.NodeCount, r.Errors)
|
||||
}
|
||||
if _, err := tx.Exec(string(stmt), args...); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return tx.Commit()
|
||||
}
|
||||
|
||||
// DeleteFileMetasByFiles drops the metadata rows for the supplied files in one
|
||||
// repo prefix, chunked into `file_path IN (…)` DELETEs. Empty input is a no-op.
|
||||
func (s *Store) DeleteFileMetasByFiles(repoPrefix string, files []string) error {
|
||||
if len(files) == 0 {
|
||||
return nil
|
||||
}
|
||||
s.writeMu.Lock()
|
||||
defer s.writeMu.Unlock()
|
||||
|
||||
tx, err := s.db.Begin()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer tx.Rollback() //nolint:errcheck // rollback after Commit is a no-op
|
||||
|
||||
for start := 0; start < len(files); start += fileMetaChunk {
|
||||
end := start + fileMetaChunk
|
||||
if end > len(files) {
|
||||
end = len(files)
|
||||
}
|
||||
chunk := files[start:end]
|
||||
args := make([]any, 0, len(chunk)+1)
|
||||
args = append(args, repoPrefix)
|
||||
stmt := make([]byte, 0, 64+len(chunk)*2)
|
||||
stmt = append(stmt, "DELETE FROM files WHERE repo_prefix = ? AND file_path IN ("...)
|
||||
for i, f := range chunk {
|
||||
if i > 0 {
|
||||
stmt = append(stmt, ',')
|
||||
}
|
||||
stmt = append(stmt, '?')
|
||||
args = append(args, f)
|
||||
}
|
||||
stmt = append(stmt, ')')
|
||||
if _, err := tx.Exec(string(stmt), args...); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return tx.Commit()
|
||||
}
|
||||
|
||||
// FileMetasForRepo returns every recorded file row for the repo prefix.
|
||||
// Always non-nil.
|
||||
func (s *Store) FileMetasForRepo(repoPrefix string) ([]graph.FileMetaRow, error) {
|
||||
rows, err := s.db.Query(
|
||||
`SELECT file_path, content_hash, size, node_count, errors FROM files WHERE repo_prefix = ? ORDER BY file_path`,
|
||||
repoPrefix,
|
||||
)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
out := []graph.FileMetaRow{}
|
||||
for rows.Next() {
|
||||
var r graph.FileMetaRow
|
||||
if err := rows.Scan(&r.FilePath, &r.ContentHash, &r.Size, &r.NodeCount, &r.Errors); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out = append(out, r)
|
||||
}
|
||||
return out, rows.Err()
|
||||
}
|
||||
@@ -0,0 +1,69 @@
|
||||
package store_sqlite
|
||||
|
||||
import (
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"github.com/zzet/gortex/internal/graph"
|
||||
)
|
||||
|
||||
// TestFileMetas_RoundTrip pins the per-file metadata sidecar: rows upsert,
|
||||
// read back per repo, carry their errors JSON, and a per-file delete removes
|
||||
// just the named file.
|
||||
func TestFileMetas_RoundTrip(t *testing.T) {
|
||||
s, err := Open(filepath.Join(t.TempDir(), "f.sqlite"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer s.Close()
|
||||
|
||||
rows := []graph.FileMetaRow{
|
||||
{FilePath: "a/x.go", ContentHash: "h1", Size: 100, NodeCount: 7, Errors: ""},
|
||||
{FilePath: "a/broken.go", ContentHash: "h2", Size: 50, NodeCount: 1, Errors: `["3:5","4:1"]`},
|
||||
}
|
||||
if err := s.SetFileMetas("repoA", rows); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// A different repo's row must not bleed in.
|
||||
if err := s.SetFileMetas("repoB", []graph.FileMetaRow{{FilePath: "b/y.go", NodeCount: 2}}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
got, err := s.FileMetasForRepo("repoA")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(got) != 2 {
|
||||
t.Fatalf("FileMetasForRepo(repoA) = %d rows, want 2", len(got))
|
||||
}
|
||||
byFile := map[string]graph.FileMetaRow{}
|
||||
for _, r := range got {
|
||||
byFile[r.FilePath] = r
|
||||
}
|
||||
if r := byFile["a/x.go"]; r.NodeCount != 7 || r.Size != 100 || r.ContentHash != "h1" || r.Errors != "" {
|
||||
t.Errorf("x.go row = %+v", r)
|
||||
}
|
||||
if r := byFile["a/broken.go"]; r.NodeCount != 1 || r.Errors != `["3:5","4:1"]` {
|
||||
t.Errorf("broken.go row = %+v", r)
|
||||
}
|
||||
|
||||
// Upsert replaces in place.
|
||||
if err := s.SetFileMetas("repoA", []graph.FileMetaRow{{FilePath: "a/x.go", NodeCount: 9, Errors: ""}}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
got, _ = s.FileMetasForRepo("repoA")
|
||||
for _, r := range got {
|
||||
if r.FilePath == "a/x.go" && r.NodeCount != 9 {
|
||||
t.Errorf("upsert did not replace node_count: %+v", r)
|
||||
}
|
||||
}
|
||||
|
||||
// Per-file delete removes only the named file.
|
||||
if err := s.DeleteFileMetasByFiles("repoA", []string{"a/broken.go"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
got, _ = s.FileMetasForRepo("repoA")
|
||||
if len(got) != 1 || got[0].FilePath != "a/x.go" {
|
||||
t.Errorf("after delete, rows = %+v, want only a/x.go", got)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,38 @@
|
||||
package store_sqlite
|
||||
|
||||
import (
|
||||
"iter"
|
||||
|
||||
"github.com/zzet/gortex/internal/graph"
|
||||
)
|
||||
|
||||
// Compile-time assertion: *Store serves the fn-value placeholder scan.
|
||||
var _ graph.FnValuePlaceholderScanner = (*Store)(nil)
|
||||
|
||||
// FnValuePlaceholderEdges implements graph.FnValuePlaceholderScanner: it yields
|
||||
// only the fn-value gate's placeholder edges, the exact inverse of the
|
||||
// fn-value exclusion EdgesWithUnresolvedTarget applies. The predicate is the
|
||||
// SAME two-form filter the v2 migration's dedupeFnValuePlaceholderEdges uses:
|
||||
// the bare `unresolved::fnvalue::` range rides edges_by_to(to_id) (the ':;'
|
||||
// range end is ':'+1, one past the marker); the multi-repo COPY-rewrite infix
|
||||
// form is caught by is_unresolved = 1 + LIKE. Full column set incl. meta — the
|
||||
// gate reads Meta["via"] and the captured fn_value_name off each placeholder.
|
||||
//
|
||||
// The whole point is that the gate no longer has to scan the entire
|
||||
// EdgeReferences kind (placeholders + every real reference) and Go-filter on
|
||||
// every whole-graph synthesizer pass; it pulls the handful of placeholders
|
||||
// straight off the index instead.
|
||||
func (s *Store) FnValuePlaceholderEdges() iter.Seq[*graph.Edge] {
|
||||
return func(yield func(*graph.Edge) bool) {
|
||||
out := s.queryEdgesSQL(`
|
||||
SELECT from_id, to_id, kind, file_path, line, confidence, confidence_label, origin, tier, cross_repo, meta, resolve_terminal, resolve_terminal_reason
|
||||
FROM edges
|
||||
WHERE (to_id >= 'unresolved::fnvalue::' AND to_id < 'unresolved::fnvalue:;')
|
||||
OR (is_unresolved = 1 AND to_id LIKE '%::unresolved::fnvalue::%')`)
|
||||
for _, e := range out {
|
||||
if !yield(e) {
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,482 @@
|
||||
package store_sqlite
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"strings"
|
||||
|
||||
"github.com/zzet/gortex/internal/graph"
|
||||
"github.com/zzet/gortex/internal/search"
|
||||
)
|
||||
|
||||
// This file implements graph.SymbolSearcher + graph.SymbolBundleSearcher
|
||||
// on the SQLite backend using the FTS5 virtual table declared in
|
||||
// schema.go (symbol_fts). It is the on-disk replacement for the
|
||||
// multi-GB in-heap Bleve/BM25 index: the FTS5 inverted index lives in
|
||||
// the same .sqlite file as the graph, and a tier-0 exact-name boost
|
||||
// short-circuits identifier queries so
|
||||
// search quality holds or improves while the heap shrinks.
|
||||
//
|
||||
// Semantics:
|
||||
//
|
||||
// - BulkUpsertSymbolFTS wipes only the rows owned by repoPrefix
|
||||
// before re-inserting, so sibling repos sharing one store don't
|
||||
// clobber each other's corpus. Empty prefix wipes the whole table
|
||||
// (single-repo / conformance behaviour).
|
||||
//
|
||||
// - SearchSymbols tier 0: an identifier query (no whitespace / path
|
||||
// separators) that resolves to one or more nodes by exact name is
|
||||
// returned directly with a fixed dominant score, skipping FTS.
|
||||
// Misses fall through to the FTS5 MATCH path.
|
||||
//
|
||||
// - SearchSymbolBundles composes the same hit list with batched
|
||||
// node + in/out edge fetches the rerank pipeline reads from.
|
||||
//
|
||||
// FTS5 maintains its index incrementally on every insert, so the
|
||||
// Store struct needs no extra state and BuildSymbolIndex is a no-op
|
||||
// (it only opportunistically merges segments).
|
||||
|
||||
// Compile-time assertions: *Store satisfies the symbol-search
|
||||
// capabilities. The indexer auto-engages these when the active backend
|
||||
// implements them, routing search_symbols through on-disk FTS5 instead
|
||||
// of the in-process BM25 index.
|
||||
var (
|
||||
_ graph.SymbolSearcher = (*Store)(nil)
|
||||
_ graph.SymbolBundleSearcher = (*Store)(nil)
|
||||
_ graph.BundleFingerprintSink = (*Store)(nil)
|
||||
)
|
||||
|
||||
// ftsInsertChunkRows bounds the rows per multi-row INSERT. Each row
|
||||
// binds 3 host params (node_id, repo_prefix, tokens); 300 rows is 900
|
||||
// params, comfortably under SQLite's default 999-variable limit so the
|
||||
// statement stays portable across builds.
|
||||
const ftsInsertChunkRows = 300
|
||||
|
||||
// UpsertSymbolFTS records (or replaces) the pre-tokenised text for
|
||||
// nodeID. FTS5 offers no UPSERT on a table with UNINDEXED columns, so
|
||||
// the write is delete-then-insert. The delete targets the prior row's
|
||||
// FTS5 docid (rowid), looked up from the symbol_fts_rowid sidecar —
|
||||
// node_id is UNINDEXED, so "DELETE … WHERE node_id = ?" would full-scan
|
||||
// the whole index once per symbol, which is quadratic over a file's
|
||||
// symbols on the per-edit reindex hot path. The repo_prefix is derived
|
||||
// from the owning node (nodes.repo_prefix) so the per-repo staleness
|
||||
// wipe in BulkUpsertSymbolFTS can scope by prefix; if the node is absent
|
||||
// the prefix defaults to "".
|
||||
func (s *Store) UpsertSymbolFTS(nodeID, tokens string) error {
|
||||
if nodeID == "" {
|
||||
return nil
|
||||
}
|
||||
s.writeMu.Lock()
|
||||
defer s.writeMu.Unlock()
|
||||
|
||||
var repoPrefix string
|
||||
// A missing node (or a scan error) leaves repoPrefix == "" — the
|
||||
// row is still indexable, it just won't be reachable by a per-repo
|
||||
// prefix wipe. The graph.Store contract has no error channel for
|
||||
// the indexer's incremental writes, so we don't surface this.
|
||||
_ = s.db.QueryRow(`SELECT repo_prefix FROM nodes WHERE id = ?`, nodeID).Scan(&repoPrefix)
|
||||
|
||||
// Delete the prior row by its docid (O(log n)) instead of by node_id
|
||||
// (full FTS scan). A missing map entry means no prior row to drop —
|
||||
// the sidecar is kept in lockstep with symbol_fts by every writer and
|
||||
// backfilled at Open for databases built before it existed, so a miss
|
||||
// here is a genuinely new symbol, not a stale row we're leaking.
|
||||
var oldRowid int64
|
||||
switch err := s.db.QueryRow(
|
||||
`SELECT fts_rowid FROM symbol_fts_rowid WHERE node_id = ?`, nodeID,
|
||||
).Scan(&oldRowid); err {
|
||||
case nil:
|
||||
if _, err := s.db.Exec(`DELETE FROM symbol_fts WHERE rowid = ?`, oldRowid); err != nil {
|
||||
return err
|
||||
}
|
||||
case sql.ErrNoRows:
|
||||
// new symbol — nothing to delete
|
||||
default:
|
||||
return err
|
||||
}
|
||||
|
||||
res, err := s.db.Exec(
|
||||
`INSERT INTO symbol_fts (node_id, repo_prefix, tokens) VALUES (?, ?, ?)`,
|
||||
nodeID, repoPrefix, tokens,
|
||||
)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
newRowid, err := res.LastInsertId()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := s.db.Exec(
|
||||
`INSERT OR REPLACE INTO symbol_fts_rowid (node_id, repo_prefix, fts_rowid) VALUES (?, ?, ?)`,
|
||||
nodeID, repoPrefix, newRowid,
|
||||
); err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// BulkUpsertSymbolFTS is the cold-start fast path: wipe this repo's
|
||||
// stale rows, then chunked multi-row INSERT of the deduped items. The
|
||||
// whole thing runs in one transaction under writeMu so a concurrent
|
||||
// reader never observes the table mid-wipe.
|
||||
//
|
||||
// repoPrefix scopes the pre-insert wipe: a non-empty prefix deletes
|
||||
// only rows owned by that repo,
|
||||
// leaving siblings untouched; an empty prefix wipes the whole table
|
||||
// (single-repo / conformance behaviour — the conformance suite calls
|
||||
// this with ""). Items are deduped by NodeID with last-write-wins,
|
||||
// matching UpsertSymbolFTS's replace semantics.
|
||||
func (s *Store) BulkUpsertSymbolFTS(repoPrefix string, items []graph.SymbolFTSItem) error {
|
||||
if len(items) == 0 {
|
||||
return nil
|
||||
}
|
||||
s.writeMu.Lock()
|
||||
defer s.writeMu.Unlock()
|
||||
|
||||
// Dedup by ID — last write wins, mirroring UpsertSymbolFTS's
|
||||
// delete-then-insert. Guards the edge case where a re-parse of a
|
||||
// file emitted the same ID twice.
|
||||
pos := make(map[string]int, len(items))
|
||||
deduped := items[:0]
|
||||
for _, it := range items {
|
||||
if it.NodeID == "" {
|
||||
continue
|
||||
}
|
||||
if p, ok := pos[it.NodeID]; ok {
|
||||
deduped[p] = it
|
||||
} else {
|
||||
pos[it.NodeID] = len(deduped)
|
||||
deduped = append(deduped, it)
|
||||
}
|
||||
}
|
||||
items = deduped
|
||||
if len(items) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
tx, err := s.db.Begin()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
commit := false
|
||||
defer func() {
|
||||
if !commit {
|
||||
_ = tx.Rollback()
|
||||
}
|
||||
}()
|
||||
|
||||
// Wipe this repo's prior rows so a clean rebuild of repo A doesn't
|
||||
// leave phantom hits, while sibling repo B's corpus survives. The
|
||||
// repo_prefix column is UNINDEXED but still stored, so the equality
|
||||
// filter is a literal compare over the row set. Empty repoPrefix
|
||||
// clears the whole table — the legacy single-repo wipe.
|
||||
if _, err := tx.Exec(`DELETE FROM symbol_fts WHERE repo_prefix = ?`, repoPrefix); err != nil {
|
||||
return err
|
||||
}
|
||||
// Drop this repo's rowid-map entries in lockstep with the symbol_fts
|
||||
// wipe so the two never diverge; they are rebuilt from the freshly
|
||||
// inserted rows below.
|
||||
if _, err := tx.Exec(`DELETE FROM symbol_fts_rowid WHERE repo_prefix = ?`, repoPrefix); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
for start := 0; start < len(items); start += ftsInsertChunkRows {
|
||||
end := minInt(start+ftsInsertChunkRows, len(items))
|
||||
chunk := items[start:end]
|
||||
|
||||
var b strings.Builder
|
||||
b.WriteString(`INSERT INTO symbol_fts (node_id, repo_prefix, tokens) VALUES `)
|
||||
args := make([]any, 0, len(chunk)*3)
|
||||
for i, it := range chunk {
|
||||
if i > 0 {
|
||||
b.WriteByte(',')
|
||||
}
|
||||
b.WriteString(`(?,?,?)`)
|
||||
args = append(args, it.NodeID, repoPrefix, it.Tokens)
|
||||
}
|
||||
if _, err := tx.Exec(b.String(), args...); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
// Rebuild the rowid map for this repo from the rows just inserted. A
|
||||
// full multi-row INSERT only exposes the last docid, so we read the
|
||||
// docids back in one pass (a linear filter over the UNINDEXED
|
||||
// repo_prefix column — the cold/bulk path, not the per-edit hot path).
|
||||
if _, err := tx.Exec(
|
||||
`INSERT OR REPLACE INTO symbol_fts_rowid (node_id, repo_prefix, fts_rowid)
|
||||
SELECT node_id, repo_prefix, rowid FROM symbol_fts WHERE repo_prefix = ?`,
|
||||
repoPrefix,
|
||||
); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if err := tx.Commit(); err != nil {
|
||||
return err
|
||||
}
|
||||
commit = true
|
||||
return nil
|
||||
}
|
||||
|
||||
// backfillSymbolFTSRowidMap populates symbol_fts_rowid from symbol_fts for
|
||||
// a database built before the sidecar existed. Without it, the first
|
||||
// incremental UpsertSymbolFTS for an already-indexed symbol would find no
|
||||
// map entry, skip the delete, and leak a duplicate FTS row. It is a
|
||||
// one-time cost: skipped once the map has any row (steady state) or when
|
||||
// the FTS index is empty (a fresh DB the bulk path will populate with the
|
||||
// map maintained inline). Runs at Open, before any reader or writer.
|
||||
func backfillSymbolFTSRowidMap(db *sql.DB) error {
|
||||
var mapped bool
|
||||
if err := db.QueryRow(`SELECT EXISTS(SELECT 1 FROM symbol_fts_rowid)`).Scan(&mapped); err != nil {
|
||||
return err
|
||||
}
|
||||
if mapped {
|
||||
return nil
|
||||
}
|
||||
var hasFTS bool
|
||||
if err := db.QueryRow(`SELECT EXISTS(SELECT 1 FROM symbol_fts)`).Scan(&hasFTS); err != nil {
|
||||
return err
|
||||
}
|
||||
if !hasFTS {
|
||||
return nil
|
||||
}
|
||||
_, err := db.Exec(
|
||||
`INSERT OR REPLACE INTO symbol_fts_rowid (node_id, repo_prefix, fts_rowid)
|
||||
SELECT node_id, repo_prefix, rowid FROM symbol_fts`)
|
||||
return err
|
||||
}
|
||||
|
||||
// BuildSymbolIndex is a no-op for FTS5: the index is maintained
|
||||
// incrementally on every insert, so there is nothing to build after the
|
||||
// bulk parse phase. We opportunistically run the FTS5 'optimize'
|
||||
// command to merge segments (purely a read-latency improvement); any
|
||||
// error is ignored because the index is already correct without it.
|
||||
// Idempotent — safe to call any number of times.
|
||||
func (s *Store) BuildSymbolIndex() error {
|
||||
s.writeMu.Lock()
|
||||
defer s.writeMu.Unlock()
|
||||
_, _ = s.db.Exec(`INSERT INTO symbol_fts(symbol_fts) VALUES('optimize')`)
|
||||
return nil
|
||||
}
|
||||
|
||||
// SearchSymbols runs a symbol query and returns hits ordered by
|
||||
// descending relevance (higher Score = more relevant).
|
||||
//
|
||||
// Tier 0 (exact-name boost): when the
|
||||
// query looks like a literal identifier and resolves to one or more
|
||||
// nodes by exact name, return those directly with a fixed dominant
|
||||
// score (100.0) — an O(1)-ish index seek that beats FTS ranking for
|
||||
// the common "type the symbol name" case. Misses fall through to FTS5.
|
||||
//
|
||||
// Otherwise tokenise on the read side with the SAME splitter as the
|
||||
// write side (search.Tokenize) so a camelCase query lands on the
|
||||
// split corpus, build a prefix-OR MATCH expression, and rank by BM25.
|
||||
// SQLite's bm25() returns lower-is-better, so the stored Score is its
|
||||
// negation (higher-is-better, matching the SymbolHit contract).
|
||||
func (s *Store) SearchSymbols(query string, limit int) ([]graph.SymbolHit, error) {
|
||||
if query == "" {
|
||||
return nil, nil
|
||||
}
|
||||
if limit <= 0 {
|
||||
limit = 20
|
||||
}
|
||||
|
||||
// Tier 0: exact-name lookup. Only engage for identifier-shaped
|
||||
// queries (no whitespace / path separators); multi-word queries are
|
||||
// concept searches that need BM25 ranking. We only short-circuit
|
||||
// when the lookup hits at least one node — misses fall through so a
|
||||
// partial-identifier query still reaches FTS.
|
||||
if isIdentifierQuery(query) {
|
||||
ns := s.FindNodesByName(query)
|
||||
if len(ns) > 0 {
|
||||
out := make([]graph.SymbolHit, 0, minInt(len(ns), limit))
|
||||
for _, n := range ns {
|
||||
if n == nil || n.ID == "" {
|
||||
continue
|
||||
}
|
||||
out = append(out, graph.SymbolHit{NodeID: n.ID, Score: 100.0})
|
||||
if len(out) >= limit {
|
||||
break
|
||||
}
|
||||
}
|
||||
if len(out) > 0 {
|
||||
return out, nil
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
match := s.buildFTSMatch(query)
|
||||
if match == "" {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
const q = `SELECT node_id, bm25(symbol_fts) FROM symbol_fts WHERE symbol_fts MATCH ? ORDER BY bm25(symbol_fts) LIMIT ?`
|
||||
rows, err := s.db.Query(q, match, limit)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
var hits []graph.SymbolHit
|
||||
for rows.Next() {
|
||||
var (
|
||||
id string
|
||||
score float64
|
||||
)
|
||||
if err := rows.Scan(&id, &score); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if id == "" {
|
||||
continue
|
||||
}
|
||||
// bm25() is negative-better in SQLite; negate so higher = better,
|
||||
// matching the SymbolHit contract. Rows already arrive in bm25
|
||||
// (best-first) order from the ORDER BY.
|
||||
hits = append(hits, graph.SymbolHit{NodeID: id, Score: -score})
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return hits, nil
|
||||
}
|
||||
|
||||
// buildFTSMatch tokenises the query with the write-side splitter and
|
||||
// builds an FTS5 MATCH expression: each token becomes a quoted prefix
|
||||
// term ("tok"*) and the terms are OR-joined so any token match counts.
|
||||
// Returns "" when the query degenerates to no tokens.
|
||||
func (s *Store) buildFTSMatch(query string) string {
|
||||
tokens := search.Tokenize(query)
|
||||
if len(tokens) == 0 {
|
||||
// Fallback: when Tokenize drops everything (e.g. a single
|
||||
// sub-2-char token like "go"), use the looser query tokeniser so
|
||||
// the search still reaches the engine instead of returning empty.
|
||||
tokens = search.TokenizeQuery(query)
|
||||
if len(tokens) == 0 {
|
||||
return ""
|
||||
}
|
||||
}
|
||||
parts := make([]string, 0, len(tokens))
|
||||
for _, t := range tokens {
|
||||
if t == "" {
|
||||
continue
|
||||
}
|
||||
parts = append(parts, `"`+escapeFTSQuote(t)+`"*`)
|
||||
}
|
||||
if len(parts) == 0 {
|
||||
return ""
|
||||
}
|
||||
return strings.Join(parts, " OR ")
|
||||
}
|
||||
|
||||
// escapeFTSQuote escapes a token for use inside an FTS5 double-quoted
|
||||
// string literal: a literal double quote is doubled ("" inside "...").
|
||||
func escapeFTSQuote(t string) string {
|
||||
return strings.ReplaceAll(t, `"`, `""`)
|
||||
}
|
||||
|
||||
// SearchSymbolBundles is the rerank-shaped fast path: it runs
|
||||
// SearchSymbols to get the ranked id list (preserving order) plus a
|
||||
// score-by-id map, then materialises the nodes and their in/out edges
|
||||
// in batched fetches the rerank pipeline reads from. The engine routes
|
||||
// through this when the backend implements SymbolBundleSearcher,
|
||||
// pre-seeding rerank.Context's edge caches.
|
||||
func (s *Store) SearchSymbolBundles(query string, limit int) ([]graph.SymbolBundle, error) {
|
||||
hits, err := s.SearchSymbols(query, limit)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if len(hits) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
ids := make([]string, 0, len(hits))
|
||||
scoreByID := make(map[string]float64, len(hits))
|
||||
for _, h := range hits {
|
||||
if h.NodeID == "" {
|
||||
continue
|
||||
}
|
||||
if _, dup := scoreByID[h.NodeID]; dup {
|
||||
// First hit keeps the score / position; defend against a
|
||||
// future ranker that returns an id more than once.
|
||||
continue
|
||||
}
|
||||
scoreByID[h.NodeID] = h.Score
|
||||
ids = append(ids, h.NodeID)
|
||||
}
|
||||
if len(ids) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
// Content-addressed cache: serve cached bundles for IDs whose
|
||||
// package fingerprint is unchanged and fetch only the misses. The
|
||||
// cache is nil until the daemon wires fingerprints, in which case
|
||||
// every ID is a miss and the path is exactly the legacy fetch.
|
||||
cached := make(map[string]graph.SymbolBundle, len(ids))
|
||||
missIDs := ids
|
||||
if s.bundles != nil {
|
||||
missIDs = missIDs[:0:0]
|
||||
for _, id := range ids {
|
||||
if b, ok := s.bundles.lookup(id); ok {
|
||||
cached[id] = b
|
||||
continue
|
||||
}
|
||||
missIDs = append(missIDs, id)
|
||||
}
|
||||
}
|
||||
|
||||
// Fetch the misses' nodes + in/out edges in one batched round-trip
|
||||
// each. A full cache hit skips all three fetches entirely.
|
||||
var nodes map[string]*graph.Node
|
||||
var out, in map[string][]*graph.Edge
|
||||
if len(missIDs) > 0 {
|
||||
nodes = s.GetNodesByIDs(missIDs)
|
||||
out = s.GetOutEdgesByNodeIDs(missIDs)
|
||||
in = s.GetInEdgesByNodeIDs(missIDs)
|
||||
}
|
||||
|
||||
bundles := make([]graph.SymbolBundle, 0, len(ids))
|
||||
for _, id := range ids {
|
||||
if b, ok := cached[id]; ok {
|
||||
// The cached bundle's score is whatever it was first cached
|
||||
// with; the live FTS score for THIS query is authoritative,
|
||||
// so re-stamp it (the score is query-specific, the node +
|
||||
// edges are not).
|
||||
b.Score = scoreByID[id]
|
||||
bundles = append(bundles, b)
|
||||
continue
|
||||
}
|
||||
n := nodes[id]
|
||||
if n == nil {
|
||||
// Hit references a node evicted between the search and the
|
||||
// node fetch — skip; the caller does its own dedup / filter.
|
||||
continue
|
||||
}
|
||||
b := graph.SymbolBundle{
|
||||
Node: n,
|
||||
Score: scoreByID[id],
|
||||
OutEdges: out[id],
|
||||
InEdges: in[id],
|
||||
}
|
||||
if s.bundles != nil {
|
||||
s.bundles.store(b)
|
||||
}
|
||||
bundles = append(bundles, b)
|
||||
}
|
||||
return bundles, nil
|
||||
}
|
||||
|
||||
// isIdentifierQuery reports whether a query looks like a literal symbol
|
||||
// name (no whitespace, no path separators, no dots, no colons, no
|
||||
// commas). The tier-0 exact-name fast path engages only on such
|
||||
// queries; multi-token / path / qualified queries always go to FTS.
|
||||
func isIdentifierQuery(q string) bool {
|
||||
if q == "" {
|
||||
return false
|
||||
}
|
||||
for _, r := range q {
|
||||
switch r {
|
||||
case ' ', '\t', '\n', '/', '.', ':', ',':
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
@@ -0,0 +1,45 @@
|
||||
package store_sqlite
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
|
||||
"github.com/zzet/gortex/internal/graph"
|
||||
)
|
||||
|
||||
// SetRepoIndexState upserts the freshness-provenance row for one repo —
|
||||
// written at the end of every (re)index. One row per repo_prefix.
|
||||
func (s *Store) SetRepoIndexState(st graph.RepoIndexState) error {
|
||||
s.writeMu.Lock()
|
||||
defer s.writeMu.Unlock()
|
||||
dirty := 0
|
||||
if st.Dirty {
|
||||
dirty = 1
|
||||
}
|
||||
_, err := s.db.Exec(`
|
||||
INSERT OR REPLACE INTO repo_index_state
|
||||
(repo_prefix, indexed_sha, dirty, indexed_at, workspace_fp, node_count, edge_count, extractor_versions)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?)`,
|
||||
st.RepoPrefix, st.IndexedSHA, dirty, st.IndexedAt, st.WorkspaceFP,
|
||||
st.NodeCount, st.EdgeCount, st.ExtractorVersions)
|
||||
return err
|
||||
}
|
||||
|
||||
// GetRepoIndexState returns the recorded freshness provenance for a repo.
|
||||
// The bool is false when no row exists yet (never-indexed / pre-feature).
|
||||
func (s *Store) GetRepoIndexState(repoPrefix string) (graph.RepoIndexState, bool, error) {
|
||||
row := s.db.QueryRow(`
|
||||
SELECT indexed_sha, dirty, indexed_at, workspace_fp, node_count, edge_count, extractor_versions
|
||||
FROM repo_index_state WHERE repo_prefix = ?`, repoPrefix)
|
||||
st := graph.RepoIndexState{RepoPrefix: repoPrefix}
|
||||
var dirty int
|
||||
err := row.Scan(&st.IndexedSHA, &dirty, &st.IndexedAt, &st.WorkspaceFP,
|
||||
&st.NodeCount, &st.EdgeCount, &st.ExtractorVersions)
|
||||
if err == sql.ErrNoRows {
|
||||
return graph.RepoIndexState{RepoPrefix: repoPrefix}, false, nil
|
||||
}
|
||||
if err != nil {
|
||||
return graph.RepoIndexState{RepoPrefix: repoPrefix}, false, err
|
||||
}
|
||||
st.Dirty = dirty != 0
|
||||
return st, true, nil
|
||||
}
|
||||
@@ -0,0 +1,61 @@
|
||||
package store_sqlite_test
|
||||
|
||||
import (
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"github.com/zzet/gortex/internal/graph"
|
||||
"github.com/zzet/gortex/internal/graph/store_sqlite"
|
||||
)
|
||||
|
||||
func openIndexStateStore(t *testing.T) *store_sqlite.Store {
|
||||
t.Helper()
|
||||
s, err := store_sqlite.Open(filepath.Join(t.TempDir(), "is.sqlite"))
|
||||
require.NoError(t, err)
|
||||
t.Cleanup(func() { _ = s.Close() })
|
||||
return s
|
||||
}
|
||||
|
||||
func TestRepoIndexState_RoundTrip(t *testing.T) {
|
||||
s := openIndexStateStore(t)
|
||||
|
||||
// Absent state reads back as (zero, false, nil).
|
||||
got, ok, err := s.GetRepoIndexState("gortex")
|
||||
require.NoError(t, err)
|
||||
require.False(t, ok)
|
||||
require.Equal(t, "gortex", got.RepoPrefix)
|
||||
|
||||
want := graph.RepoIndexState{
|
||||
RepoPrefix: "gortex",
|
||||
IndexedSHA: "abc123",
|
||||
Dirty: true,
|
||||
IndexedAt: 1700000000,
|
||||
WorkspaceFP: "deadbeef",
|
||||
NodeCount: 42,
|
||||
EdgeCount: 99,
|
||||
ExtractorVersions: `{"go":2}`,
|
||||
}
|
||||
require.NoError(t, s.SetRepoIndexState(want))
|
||||
|
||||
got, ok, err = s.GetRepoIndexState("gortex")
|
||||
require.NoError(t, err)
|
||||
require.True(t, ok)
|
||||
require.Equal(t, want, got)
|
||||
|
||||
// Upsert replaces in place (one row per repo_prefix).
|
||||
want.IndexedSHA = "def456"
|
||||
want.Dirty = false
|
||||
require.NoError(t, s.SetRepoIndexState(want))
|
||||
got, ok, err = s.GetRepoIndexState("gortex")
|
||||
require.NoError(t, err)
|
||||
require.True(t, ok)
|
||||
require.Equal(t, "def456", got.IndexedSHA)
|
||||
require.False(t, got.Dirty)
|
||||
|
||||
// A different repo is isolated.
|
||||
_, ok, err = s.GetRepoIndexState("other")
|
||||
require.NoError(t, err)
|
||||
require.False(t, ok)
|
||||
}
|
||||
@@ -0,0 +1,54 @@
|
||||
package store_sqlite
|
||||
|
||||
import "github.com/zzet/gortex/internal/graph"
|
||||
|
||||
// Compile-time assertion: *Store serves the meta-less kind-scoped edge scan.
|
||||
var _ graph.LightEdgeScanner = (*Store)(nil)
|
||||
|
||||
// edgeColsLight is the meta-less edge column projection: the promoted struct
|
||||
// columns WITHOUT the meta blob (and without resolve_terminal, which lives in
|
||||
// Meta). It is exactly the ten columns scanEdgeLight scans, and is shared with
|
||||
// the stmtOutEdgesLight prepared statement so the projection can never drift
|
||||
// from the scanner. Adding meta back here would defeat the whole point — the
|
||||
// per-row JSON decode this projection exists to skip.
|
||||
const edgeColsLight = `from_id, to_id, kind, file_path, line, confidence, confidence_label, origin, tier, cross_repo`
|
||||
|
||||
// AllEdgesLight implements graph.LightEdgeScanner: a kind-scoped edge scan that
|
||||
// never decodes the meta blob. An empty kinds list scans every edge; supplying
|
||||
// only empty-string kinds matches nothing (parity with the aggregators). The
|
||||
// edges_by_kind index serves the IN filter. Meta is left nil; only the promoted
|
||||
// fields (origin/tier/confidence/confidence_label/cross_repo/line) are hydrated.
|
||||
func (s *Store) AllEdgesLight(kinds ...graph.EdgeKind) []*graph.Edge {
|
||||
_, args := aggDedupeEdgeKinds(kinds)
|
||||
if len(args) == 0 {
|
||||
if len(kinds) > 0 {
|
||||
return nil // caller passed only empty kinds — nothing matches
|
||||
}
|
||||
return s.queryEdgesLightSQL(`SELECT ` + edgeColsLight + ` FROM edges ORDER BY id`)
|
||||
}
|
||||
q := `SELECT ` + edgeColsLight + ` FROM edges WHERE kind IN (` +
|
||||
inPlaceholders(len(args)) + `) ORDER BY id`
|
||||
return s.queryEdgesLightSQL(q, args...)
|
||||
}
|
||||
|
||||
// queryEdgesLightSQL is the meta-less sibling of queryEdgesSQL: it materialises
|
||||
// the rows into a slice and closes the cursor before returning (releasing the
|
||||
// single pooled connection), but scans through scanEdgeLight so the meta column
|
||||
// is never transferred or decoded. Returns nil on any query error, matching
|
||||
// queryEdgesSQL — a teardown-race read degrades to empty rather than panicking.
|
||||
func (s *Store) queryEdgesLightSQL(q string, args ...any) []*graph.Edge {
|
||||
rows, err := s.db.Query(q, args...)
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
defer rows.Close()
|
||||
var out []*graph.Edge
|
||||
for rows.Next() {
|
||||
e, err := scanEdgeLight(rows)
|
||||
if err != nil || e == nil {
|
||||
continue
|
||||
}
|
||||
out = append(out, e)
|
||||
}
|
||||
return out
|
||||
}
|
||||
@@ -0,0 +1,54 @@
|
||||
package store_sqlite_test
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"github.com/zzet/gortex/internal/graph"
|
||||
)
|
||||
|
||||
// TestAllEdgesLightIsMetaless pins the disk backend's graph.LightEdgeScanner:
|
||||
// the scan must skip the meta blob (Meta == nil) — the per-edge JSON decode the
|
||||
// warm-restart analysis passes exist to avoid — while every promoted field
|
||||
// still equals what the full AllEdges() scan returns.
|
||||
func TestAllEdgesLightIsMetaless(t *testing.T) {
|
||||
s := openTestStore(t)
|
||||
s.AddNode(&graph.Node{ID: "p/a.go::A", Kind: graph.KindFunction, Name: "A", FilePath: "p/a.go"})
|
||||
s.AddNode(&graph.Node{ID: "p/a.go::B", Kind: graph.KindFunction, Name: "B", FilePath: "p/a.go"})
|
||||
s.AddEdge(&graph.Edge{
|
||||
From: "p/a.go::A", To: "p/a.go::B", Kind: graph.EdgeCalls,
|
||||
FilePath: "p/a.go", Line: 11, Confidence: 0.75, ConfidenceLabel: "high",
|
||||
Origin: graph.OriginLSPResolved, Tier: "lsp", CrossRepo: true,
|
||||
Meta: map[string]any{"via": "direct", "blob_only": "x"},
|
||||
})
|
||||
// A second kind to prove the IN filter really scopes.
|
||||
s.AddEdge(&graph.Edge{From: "p/a.go::A", To: "p/a.go::B", Kind: graph.EdgeImports, FilePath: "p/a.go", Line: 1})
|
||||
|
||||
light := s.AllEdgesLight(graph.EdgeCalls)
|
||||
require.Len(t, light, 1, "kind filter must exclude the imports edge")
|
||||
e := light[0]
|
||||
assert.Nil(t, e.Meta, "AllEdgesLight must not decode the meta blob")
|
||||
|
||||
var full *graph.Edge
|
||||
for _, fe := range s.AllEdges() {
|
||||
if fe.Kind == graph.EdgeCalls {
|
||||
full = fe
|
||||
}
|
||||
}
|
||||
require.NotNil(t, full)
|
||||
assert.NotNil(t, full.Meta, "sanity: the full scan DOES decode meta")
|
||||
assert.Equal(t, full.From, e.From)
|
||||
assert.Equal(t, full.To, e.To)
|
||||
assert.Equal(t, full.Kind, e.Kind)
|
||||
assert.Equal(t, full.Line, e.Line)
|
||||
assert.Equal(t, full.Confidence, e.Confidence)
|
||||
assert.Equal(t, full.ConfidenceLabel, e.ConfidenceLabel)
|
||||
assert.Equal(t, full.Origin, e.Origin)
|
||||
assert.Equal(t, full.Tier, e.Tier)
|
||||
assert.Equal(t, full.CrossRepo, e.CrossRepo)
|
||||
|
||||
// Empty kinds means every edge.
|
||||
assert.Len(t, s.AllEdgesLight(), 2)
|
||||
}
|
||||
@@ -0,0 +1,204 @@
|
||||
package store_sqlite
|
||||
|
||||
import (
|
||||
"strings"
|
||||
|
||||
"github.com/zzet/gortex/internal/graph"
|
||||
)
|
||||
|
||||
// These methods were added to graph.Store after the sqlite backend was
|
||||
// first removed; they are restored here so *Store satisfies the current
|
||||
// interface. All reuse the chunked IN-list / raw-SQL helpers in store.go
|
||||
// (queryNodesSQL / queryEdgesSQL / lookupChunkSize / minInt). SQLite's
|
||||
// planner drives every one through the existing secondary indexes.
|
||||
|
||||
// lookupNodeCols is the canonical node column list (and scan order) for
|
||||
// every node-shaped SELECT in the package. It must stay in sync with
|
||||
// scanNode. The struct columns (start_column/end_column) sit with the line
|
||||
// range; the promoted meta columns (signature/visibility/doc/external/
|
||||
// return_type/is_async/is_static/is_abstract/is_exported/updated_at/
|
||||
// data_class/semantic_type/semantic_source) precede meta.
|
||||
const lookupNodeCols = `id, kind, name, qual_name, file_path, start_line, end_line, start_column, end_column, language, repo_prefix, workspace_id, project_id, signature, visibility, doc, external, return_type, is_async, is_static, is_abstract, is_exported, updated_at, data_class, semantic_type, semantic_source, meta`
|
||||
|
||||
// lookupNodeColsLight is lookupNodeCols without the trailing meta column —
|
||||
// the projection GetRepoNodesLight uses so a repo-scoped scan never
|
||||
// transfers or decodes a single blob. Derived, not hand-duplicated, so it
|
||||
// can never drift out of sync with lookupNodeCols / scanNode.
|
||||
var lookupNodeColsLight = strings.TrimSuffix(lookupNodeCols, ", meta")
|
||||
|
||||
const lookupEdgeCols = `from_id, to_id, kind, file_path, line, confidence, confidence_label, origin, tier, cross_repo, meta, resolve_terminal, resolve_terminal_reason`
|
||||
|
||||
// Compile-time assertion: *Store satisfies graph.NodeNameClassCounter.
|
||||
var _ graph.NodeNameClassCounter = (*Store)(nil)
|
||||
|
||||
// CountNodesByNameClass implements graph.NodeNameClassCounter: for each
|
||||
// distinct name, it tallies how many nodes.name matches are Real (is_stub =
|
||||
// 0 and kind IN definitionKinds) vs Stub (is_stub = 1), server-side via
|
||||
// nodes_by_name — one aggregate query per chunk instead of one
|
||||
// FindNodesByName round trip per name. A name absent from the returned map
|
||||
// has no matching node at all (Real == Stub == 0 either way).
|
||||
func (s *Store) CountNodesByNameClass(names []string, definitionKinds []graph.NodeKind) map[string]graph.NodeNameClassCount {
|
||||
_, kindArgs := aggDedupeNodeKinds(definitionKinds)
|
||||
if len(kindArgs) == 0 {
|
||||
return nil
|
||||
}
|
||||
uniq := dedupeNonEmpty(names)
|
||||
if len(uniq) == 0 {
|
||||
return nil
|
||||
}
|
||||
out := make(map[string]graph.NodeNameClassCount, len(uniq))
|
||||
kindPlaceholders := inPlaceholders(len(kindArgs))
|
||||
for i := 0; i < len(uniq); i += lookupChunkSize {
|
||||
end := minInt(i+lookupChunkSize, len(uniq))
|
||||
chunk := uniq[i:end]
|
||||
q := `SELECT name,
|
||||
SUM(CASE WHEN is_stub = 0 AND kind IN (` + kindPlaceholders + `) THEN 1 ELSE 0 END),
|
||||
SUM(CASE WHEN is_stub = 1 THEN 1 ELSE 0 END)
|
||||
FROM nodes
|
||||
WHERE name IN (` + inPlaceholders(len(chunk)) + `)
|
||||
GROUP BY name`
|
||||
args := make([]any, 0, len(kindArgs)+len(chunk))
|
||||
args = append(args, kindArgs...)
|
||||
args = append(args, toAnyArgs(chunk)...)
|
||||
rows, err := s.db.Query(q, args...)
|
||||
if err != nil {
|
||||
panicOnFatal(err)
|
||||
return out
|
||||
}
|
||||
for rows.Next() {
|
||||
var name string
|
||||
var c graph.NodeNameClassCount
|
||||
if err := rows.Scan(&name, &c.Real, &c.Stub); err != nil {
|
||||
_ = rows.Close()
|
||||
panicOnFatal(err)
|
||||
return out
|
||||
}
|
||||
out[name] = c
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
_ = rows.Close()
|
||||
panicOnFatal(err)
|
||||
return out
|
||||
}
|
||||
_ = rows.Close()
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// FindNodesByNameContaining returns nodes whose Name contains substr,
|
||||
// case-insensitively (SQLite's LIKE is ASCII case-insensitive). An empty
|
||||
// substring matches nothing (parity with the in-memory store); a limit > 0
|
||||
// caps the result set. The leading-wildcard LIKE is a deliberate full scan —
|
||||
// no index accelerates an unanchored substring — matching the in-memory
|
||||
// strings.Contains fallback. % and _ in substr are escaped so they match
|
||||
// literally.
|
||||
func (s *Store) FindNodesByNameContaining(substr string, limit int) []*graph.Node {
|
||||
if substr == "" {
|
||||
return nil
|
||||
}
|
||||
pattern := "%" + escapeLikePattern(substr) + "%"
|
||||
q := `SELECT ` + lookupNodeCols + ` FROM nodes WHERE name LIKE ? ESCAPE '\' ORDER BY id`
|
||||
if limit > 0 {
|
||||
return s.queryNodesSQL(q+` LIMIT ?`, pattern, limit)
|
||||
}
|
||||
return s.queryNodesSQL(q, pattern)
|
||||
}
|
||||
|
||||
// GetNodesByQualNames returns a map qualName→*Node (first match per
|
||||
// qual_name) for the batch — the qual-name twin of FindNodesByNames, used to
|
||||
// pre-warm import resolution. Driven by the unique nodes_by_qual index.
|
||||
func (s *Store) GetNodesByQualNames(qualNames []string) map[string]*graph.Node {
|
||||
uniq := dedupeNonEmpty(qualNames)
|
||||
if len(uniq) == 0 {
|
||||
return nil
|
||||
}
|
||||
out := make(map[string]*graph.Node, len(uniq))
|
||||
for i := 0; i < len(uniq); i += lookupChunkSize {
|
||||
end := minInt(i+lookupChunkSize, len(uniq))
|
||||
chunk := uniq[i:end]
|
||||
q := `SELECT ` + lookupNodeCols + ` FROM nodes WHERE qual_name IN (` + inPlaceholders(len(chunk)) + `)`
|
||||
for _, n := range s.queryNodesSQL(q, toAnyArgs(chunk)...) {
|
||||
if n == nil {
|
||||
continue
|
||||
}
|
||||
if _, ok := out[n.QualName]; !ok {
|
||||
out[n.QualName] = n
|
||||
}
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// GetOutEdgesByNodeIDs batches per-node out-edge fan-out into one query per
|
||||
// chunk. Missing IDs are simply absent from the returned map.
|
||||
func (s *Store) GetOutEdgesByNodeIDs(ids []string) map[string][]*graph.Edge {
|
||||
return s.edgesByNodeIDs(ids, "from_id", func(e *graph.Edge) string { return e.From })
|
||||
}
|
||||
|
||||
// GetInEdgesByNodeIDs is the incoming-edge twin of GetOutEdgesByNodeIDs.
|
||||
func (s *Store) GetInEdgesByNodeIDs(ids []string) map[string][]*graph.Edge {
|
||||
return s.edgesByNodeIDs(ids, "to_id", func(e *graph.Edge) string { return e.To })
|
||||
}
|
||||
|
||||
// edgesByNodeIDs runs the chunked IN-list edge fetch keyed on the given
|
||||
// column (from_id or to_id), grouping results by the supplied key extractor.
|
||||
func (s *Store) edgesByNodeIDs(ids []string, col string, key func(*graph.Edge) string) map[string][]*graph.Edge {
|
||||
uniq := dedupeNonEmpty(ids)
|
||||
if len(uniq) == 0 {
|
||||
return nil
|
||||
}
|
||||
out := make(map[string][]*graph.Edge, len(uniq))
|
||||
for i := 0; i < len(uniq); i += lookupChunkSize {
|
||||
end := minInt(i+lookupChunkSize, len(uniq))
|
||||
chunk := uniq[i:end]
|
||||
q := `SELECT ` + lookupEdgeCols + ` FROM edges WHERE ` + col + ` IN (` + inPlaceholders(len(chunk)) + `)`
|
||||
for _, e := range s.queryEdgesSQL(q, toAnyArgs(chunk)...) {
|
||||
if e == nil {
|
||||
continue
|
||||
}
|
||||
k := key(e)
|
||||
out[k] = append(out[k], e)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// dedupeNonEmpty drops empties and duplicates, preserving first-seen order.
|
||||
func dedupeNonEmpty(in []string) []string {
|
||||
seen := make(map[string]struct{}, len(in))
|
||||
out := make([]string, 0, len(in))
|
||||
for _, v := range in {
|
||||
if v == "" {
|
||||
continue
|
||||
}
|
||||
if _, ok := seen[v]; ok {
|
||||
continue
|
||||
}
|
||||
seen[v] = struct{}{}
|
||||
out = append(out, v)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// inPlaceholders returns "?,?,?" for n bound parameters.
|
||||
func inPlaceholders(n int) string {
|
||||
if n <= 0 {
|
||||
return ""
|
||||
}
|
||||
return strings.Repeat(",?", n)[1:]
|
||||
}
|
||||
|
||||
// toAnyArgs widens a string slice for variadic Query/Exec args.
|
||||
func toAnyArgs(ss []string) []any {
|
||||
args := make([]any, len(ss))
|
||||
for i, v := range ss {
|
||||
args[i] = v
|
||||
}
|
||||
return args
|
||||
}
|
||||
|
||||
// escapeLikePattern escapes the LIKE metacharacters so the substring matches
|
||||
// literally under `... LIKE ? ESCAPE '\'`.
|
||||
func escapeLikePattern(s string) string {
|
||||
return strings.NewReplacer(`\`, `\\`, `%`, `\%`, `_`, `\_`).Replace(s)
|
||||
}
|
||||
@@ -0,0 +1,54 @@
|
||||
package store_sqlite_test
|
||||
|
||||
import (
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"github.com/zzet/gortex/internal/graph"
|
||||
"github.com/zzet/gortex/internal/graph/store_sqlite"
|
||||
)
|
||||
|
||||
// TestAllRepoMemoryEstimates_Memoized verifies the short-TTL memoisation:
|
||||
// within the TTL a second call serves the cached estimate instead of
|
||||
// re-running the COUNT … GROUP BY scan, so the status path can poll cheaply
|
||||
// even while enrichment is writing to the same store.
|
||||
func TestAllRepoMemoryEstimates_Memoized(t *testing.T) {
|
||||
path := filepath.Join(t.TempDir(), "g.sqlite")
|
||||
s, err := store_sqlite.Open(path)
|
||||
require.NoError(t, err)
|
||||
t.Cleanup(func() { _ = s.Close() })
|
||||
|
||||
totalNodes := func(m map[string]graph.RepoMemoryEstimate) int {
|
||||
n := 0
|
||||
for _, e := range m {
|
||||
n += e.NodeCount
|
||||
}
|
||||
return n
|
||||
}
|
||||
|
||||
s.AddBatch([]*graph.Node{
|
||||
{ID: "a.go::Foo", Kind: graph.KindFunction, Name: "Foo", FilePath: "a.go", RepoPrefix: "r1"},
|
||||
{ID: "a.go::Bar", Kind: graph.KindFunction, Name: "Bar", FilePath: "a.go", RepoPrefix: "r1"},
|
||||
}, nil)
|
||||
|
||||
first := s.AllRepoMemoryEstimates()
|
||||
require.Equal(t, 2, totalNodes(first))
|
||||
|
||||
// Add another node; within the memoisation TTL the estimate is served
|
||||
// from cache, so the freshly-added node is intentionally not yet
|
||||
// reflected — proof the COUNT scan was skipped.
|
||||
s.AddBatch([]*graph.Node{
|
||||
{ID: "b.go::Baz", Kind: graph.KindFunction, Name: "Baz", FilePath: "b.go", RepoPrefix: "r1"},
|
||||
}, nil)
|
||||
cached := s.AllRepoMemoryEstimates()
|
||||
require.Equal(t, 2, totalNodes(cached), "within the TTL the result should be the memoised value, not a fresh COUNT")
|
||||
|
||||
// The returned map is a copy: mutating it must not corrupt the cache.
|
||||
for k := range cached {
|
||||
delete(cached, k)
|
||||
}
|
||||
again := s.AllRepoMemoryEstimates()
|
||||
require.Equal(t, 2, totalNodes(again), "the cache must hand back an independent copy")
|
||||
}
|
||||
@@ -0,0 +1,243 @@
|
||||
package store_sqlite
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
|
||||
"github.com/zzet/gortex/internal/graph"
|
||||
)
|
||||
|
||||
// Compile-time assertions that the SQLite Store satisfies the optional
|
||||
// per-file mtime persistence capabilities. Lifting this state into the
|
||||
// same backend the graph lives in means warm restarts read it through
|
||||
// one persistence surface instead of a second gob snapshot.
|
||||
var (
|
||||
_ graph.FileMtimeWriter = (*Store)(nil)
|
||||
_ graph.FileMtimeReader = (*Store)(nil)
|
||||
_ graph.FileMtimeReplacer = (*Store)(nil)
|
||||
_ graph.FileMtimeDeleter = (*Store)(nil)
|
||||
)
|
||||
|
||||
// mtimeChunk bounds how many (repo_prefix, file_path, mtime_ns) tuples
|
||||
// ride in a single multi-row INSERT. SQLite's default compiled-in host
|
||||
// parameter limit is 999; at 3 params per row that caps a statement at
|
||||
// 333 rows, so 300 leaves headroom.
|
||||
const mtimeChunk = 300
|
||||
|
||||
// SetFileMtime records one file's modification time (nanoseconds since
|
||||
// the epoch) for a repo prefix, replacing any prior value. It is a
|
||||
// convenience single-row form of BulkSetFileMtimes.
|
||||
func (s *Store) SetFileMtime(repoPrefix, filePath string, mtimeNs int64) error {
|
||||
s.writeMu.Lock()
|
||||
defer s.writeMu.Unlock()
|
||||
_, err := s.db.Exec(
|
||||
`INSERT OR REPLACE INTO file_mtimes (repo_prefix, file_path, mtime_ns) VALUES (?, ?, ?)`,
|
||||
repoPrefix, filePath, mtimeNs,
|
||||
)
|
||||
return err
|
||||
}
|
||||
|
||||
// BulkSetFileMtimes persists every (filePath -> mtimeNs) entry for one
|
||||
// repo prefix in a single transaction, chunked so no statement exceeds
|
||||
// SQLite's host-parameter limit. Idempotent on (repoPrefix, filePath):
|
||||
// re-running with overlapping keys replaces in place. Empty input is a
|
||||
// no-op.
|
||||
func (s *Store) BulkSetFileMtimes(repoPrefix string, mtimes map[string]int64) error {
|
||||
if len(mtimes) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
s.writeMu.Lock()
|
||||
defer s.writeMu.Unlock()
|
||||
|
||||
tx, err := s.db.Begin()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer tx.Rollback() //nolint:errcheck // rollback after Commit is a no-op
|
||||
|
||||
if err := insertMtimesTx(tx, repoPrefix, mtimes); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return tx.Commit()
|
||||
}
|
||||
|
||||
// ReplaceFileMtimes persists the AUTHORITATIVE full mtime set for one repo
|
||||
// prefix: every prior row for the prefix is dropped and the supplied set is
|
||||
// written, all in one transaction. The full-index persist path uses this so
|
||||
// files deleted since the last index are pruned — BulkSetFileMtimes (upsert)
|
||||
// would leave their rows behind, and warm-restart reconcile would then
|
||||
// detect them as phantom deletions on every restart, forcing a full
|
||||
// re-track that never converges.
|
||||
//
|
||||
// Empty input is a deliberate no-op: it never wipes a repo's mtimes from an
|
||||
// empty snapshot (the indexer guards the call with len(snapshot) > 0).
|
||||
func (s *Store) ReplaceFileMtimes(repoPrefix string, mtimes map[string]int64) error {
|
||||
if len(mtimes) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
s.writeMu.Lock()
|
||||
defer s.writeMu.Unlock()
|
||||
|
||||
tx, err := s.db.Begin()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer tx.Rollback() //nolint:errcheck // rollback after Commit is a no-op
|
||||
|
||||
if _, err := tx.Exec(`DELETE FROM file_mtimes WHERE repo_prefix = ?`, repoPrefix); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := insertMtimesTx(tx, repoPrefix, mtimes); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return tx.Commit()
|
||||
}
|
||||
|
||||
// DeleteFileMtimes drops the rows for a set of repo-relative file paths
|
||||
// under one repo prefix — the incremental-reindex sibling of
|
||||
// ReplaceFileMtimes. The watcher / incremental path calls it when a file is
|
||||
// deleted so the persisted set stays in step with the live graph and the
|
||||
// next warm restart does not see the path as a phantom deletion. Empty
|
||||
// input is a no-op.
|
||||
func (s *Store) DeleteFileMtimes(repoPrefix string, paths []string) error {
|
||||
if len(paths) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
s.writeMu.Lock()
|
||||
defer s.writeMu.Unlock()
|
||||
|
||||
tx, err := s.db.Begin()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer tx.Rollback() //nolint:errcheck // rollback after Commit is a no-op
|
||||
|
||||
// Chunk so the IN-list never exceeds SQLite's host-parameter limit:
|
||||
// one leading repo_prefix arg + up to mtimeChunk path args per stmt.
|
||||
for start := 0; start < len(paths); start += mtimeChunk {
|
||||
end := min(start+mtimeChunk, len(paths))
|
||||
batch := paths[start:end]
|
||||
|
||||
args := make([]any, 0, len(batch)+1)
|
||||
args = append(args, repoPrefix)
|
||||
stmt := make([]byte, 0, 64+len(batch)*2)
|
||||
stmt = append(stmt, "DELETE FROM file_mtimes WHERE repo_prefix = ? AND file_path IN ("...)
|
||||
for i := range batch {
|
||||
if i > 0 {
|
||||
stmt = append(stmt, ',')
|
||||
}
|
||||
stmt = append(stmt, '?')
|
||||
args = append(args, batch[i])
|
||||
}
|
||||
stmt = append(stmt, ')')
|
||||
if _, err := tx.Exec(string(stmt), args...); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
return tx.Commit()
|
||||
}
|
||||
|
||||
// insertMtimesTx writes every (path -> ns) entry for repoPrefix into the
|
||||
// given transaction with chunked multi-row INSERT OR REPLACE statements,
|
||||
// each kept under SQLite's host-parameter limit. The caller owns the tx
|
||||
// lifecycle (Begin/Commit/Rollback) and the write lock.
|
||||
func insertMtimesTx(tx *sql.Tx, repoPrefix string, mtimes map[string]int64) error {
|
||||
// Stable ordering is not required for correctness, but iterating the
|
||||
// map directly is fine — we only chunk by count.
|
||||
type kv struct {
|
||||
path string
|
||||
ns int64
|
||||
}
|
||||
pending := make([]kv, 0, len(mtimes))
|
||||
for p, ns := range mtimes {
|
||||
pending = append(pending, kv{path: p, ns: ns})
|
||||
}
|
||||
|
||||
for start := 0; start < len(pending); start += mtimeChunk {
|
||||
end := min(start+mtimeChunk, len(pending))
|
||||
batch := pending[start:end]
|
||||
|
||||
// Build a multi-row INSERT OR REPLACE: (?, ?, ?), (?, ?, ?), ...
|
||||
args := make([]any, 0, len(batch)*3)
|
||||
stmt := make([]byte, 0, 64+len(batch)*16)
|
||||
stmt = append(stmt, "INSERT OR REPLACE INTO file_mtimes (repo_prefix, file_path, mtime_ns) VALUES "...)
|
||||
for i, e := range batch {
|
||||
if i > 0 {
|
||||
stmt = append(stmt, ',')
|
||||
}
|
||||
stmt = append(stmt, "(?, ?, ?)"...)
|
||||
args = append(args, repoPrefix, e.path, e.ns)
|
||||
}
|
||||
if _, err := tx.Exec(string(stmt), args...); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// LoadFileMtimes returns the recorded mtimes for one repo prefix as a
|
||||
// fresh map. Returns nil when there is no data for the prefix (the
|
||||
// "no recorded state" signal warmup expects).
|
||||
func (s *Store) LoadFileMtimes(repoPrefix string) map[string]int64 {
|
||||
rows, err := s.db.Query(
|
||||
`SELECT file_path, mtime_ns FROM file_mtimes WHERE repo_prefix = ?`,
|
||||
repoPrefix,
|
||||
)
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
var out map[string]int64
|
||||
for rows.Next() {
|
||||
var path string
|
||||
var ns int64
|
||||
if err := rows.Scan(&path, &ns); err != nil {
|
||||
return nil
|
||||
}
|
||||
if out == nil {
|
||||
out = make(map[string]int64)
|
||||
}
|
||||
out[path] = ns
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return nil
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// FileMtimes is a fallible read form of LoadFileMtimes. It always
|
||||
// returns a non-nil (possibly empty) map for a known/unknown prefix and
|
||||
// surfaces any query error. The interface method LoadFileMtimes is the
|
||||
// daemon's entry point; this variant exists for callers (and tests)
|
||||
// that want the error and an always-materialised map.
|
||||
func (s *Store) FileMtimes(repoPrefix string) (map[string]int64, error) {
|
||||
rows, err := s.db.Query(
|
||||
`SELECT file_path, mtime_ns FROM file_mtimes WHERE repo_prefix = ?`,
|
||||
repoPrefix,
|
||||
)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
out := make(map[string]int64)
|
||||
for rows.Next() {
|
||||
var path string
|
||||
var ns int64
|
||||
if err := rows.Scan(&path, &ns); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out[path] = ns
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
@@ -0,0 +1,112 @@
|
||||
package store_sqlite_test
|
||||
|
||||
import (
|
||||
"reflect"
|
||||
"testing"
|
||||
|
||||
"github.com/zzet/gortex/internal/graph"
|
||||
)
|
||||
|
||||
// TestReplaceFileMtimesPrunesDeleted is the regression for the warm-restart
|
||||
// "nothing changed but full re-track" bug: the full-index persist path must
|
||||
// REPLACE a repo's mtime set, not union into it. An upsert-only persist
|
||||
// leaves rows for files deleted since the last index behind, and warm-restart
|
||||
// reconcile then detects them as phantom deletions on every restart — forcing
|
||||
// a full re-track that never converges.
|
||||
func TestReplaceFileMtimesPrunesDeleted(t *testing.T) {
|
||||
s := openTestStore(t)
|
||||
|
||||
// Assert the store advertises the capability the indexer probes for.
|
||||
var _ graph.FileMtimeReplacer = s
|
||||
var _ graph.FileMtimeDeleter = s
|
||||
|
||||
// First index: three files persisted.
|
||||
require := func(err error, what string) {
|
||||
t.Helper()
|
||||
if err != nil {
|
||||
t.Fatalf("%s: %v", what, err)
|
||||
}
|
||||
}
|
||||
require(s.BulkSetFileMtimes("repoA", map[string]int64{
|
||||
"a/one.go": 100,
|
||||
"a/two.go": 200,
|
||||
"a/three.go": 300,
|
||||
}), "seed BulkSetFileMtimes")
|
||||
|
||||
// A different repo whose rows must never be touched by repoA writes.
|
||||
require(s.BulkSetFileMtimes("repoB", map[string]int64{"b/x.go": 999}), "seed repoB")
|
||||
|
||||
// Second index: two.go was deleted on disk, four.go is new, three.go
|
||||
// changed. The authoritative snapshot is {one, three', four}.
|
||||
require(s.ReplaceFileMtimes("repoA", map[string]int64{
|
||||
"a/one.go": 100,
|
||||
"a/three.go": 350, // changed
|
||||
"a/four.go": 400, // new
|
||||
}), "ReplaceFileMtimes")
|
||||
|
||||
want := map[string]int64{
|
||||
"a/one.go": 100,
|
||||
"a/three.go": 350,
|
||||
"a/four.go": 400,
|
||||
}
|
||||
got := s.LoadFileMtimes("repoA")
|
||||
if !reflect.DeepEqual(got, want) {
|
||||
t.Fatalf("after ReplaceFileMtimes = %v, want %v (a/two.go must be pruned)", got, want)
|
||||
}
|
||||
if _, stillThere := got["a/two.go"]; stillThere {
|
||||
t.Fatal("a/two.go was deleted on disk but its mtime row survived the replace — phantom deletion bug")
|
||||
}
|
||||
|
||||
// Repo isolation.
|
||||
if b := s.LoadFileMtimes("repoB"); !reflect.DeepEqual(b, map[string]int64{"b/x.go": 999}) {
|
||||
t.Fatalf("repoB rows disturbed by repoA replace: %v", b)
|
||||
}
|
||||
|
||||
// Empty input is a deliberate no-op: it must NEVER wipe a repo's set.
|
||||
require(s.ReplaceFileMtimes("repoA", nil), "ReplaceFileMtimes(nil)")
|
||||
if got := s.LoadFileMtimes("repoA"); !reflect.DeepEqual(got, want) {
|
||||
t.Fatalf("ReplaceFileMtimes(nil) wiped the repo: %v", got)
|
||||
}
|
||||
}
|
||||
|
||||
// TestDeleteFileMtimes covers the incremental-reindex sibling: the watcher /
|
||||
// incremental path drops just the deleted paths so the persisted set stays in
|
||||
// step with the live graph without a full replace.
|
||||
func TestDeleteFileMtimes(t *testing.T) {
|
||||
s := openTestStore(t)
|
||||
|
||||
if err := s.BulkSetFileMtimes("repoA", map[string]int64{
|
||||
"a/one.go": 100,
|
||||
"a/two.go": 200,
|
||||
"a/three.go": 300,
|
||||
"a/four.go": 400,
|
||||
}); err != nil {
|
||||
t.Fatalf("seed: %v", err)
|
||||
}
|
||||
if err := s.BulkSetFileMtimes("repoB", map[string]int64{"b/keep.go": 7}); err != nil {
|
||||
t.Fatalf("seed repoB: %v", err)
|
||||
}
|
||||
|
||||
// Delete two existing paths and one that was never recorded (harmless).
|
||||
if err := s.DeleteFileMtimes("repoA", []string{"a/two.go", "a/four.go", "a/never.go"}); err != nil {
|
||||
t.Fatalf("DeleteFileMtimes: %v", err)
|
||||
}
|
||||
|
||||
want := map[string]int64{"a/one.go": 100, "a/three.go": 300}
|
||||
if got := s.LoadFileMtimes("repoA"); !reflect.DeepEqual(got, want) {
|
||||
t.Fatalf("after delete = %v, want %v", got, want)
|
||||
}
|
||||
|
||||
// Repo isolation: same-named delete on repoA must not touch repoB.
|
||||
if b := s.LoadFileMtimes("repoB"); !reflect.DeepEqual(b, map[string]int64{"b/keep.go": 7}) {
|
||||
t.Fatalf("repoB disturbed: %v", b)
|
||||
}
|
||||
|
||||
// Empty input is a no-op.
|
||||
if err := s.DeleteFileMtimes("repoA", nil); err != nil {
|
||||
t.Fatalf("DeleteFileMtimes(nil): %v", err)
|
||||
}
|
||||
if got := s.LoadFileMtimes("repoA"); !reflect.DeepEqual(got, want) {
|
||||
t.Fatalf("DeleteFileMtimes(nil) changed the set: %v", got)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,310 @@
|
||||
package store_sqlite
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"fmt"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// This file adds the repo-scoped store hygiene the base eviction path lacks.
|
||||
// EvictRepo (store.go) deletes ONLY nodes+edges, but a repo owns fifteen
|
||||
// other repo_prefix-keyed sidecar tables (file_mtimes, repo_index_state,
|
||||
// enrichment_state, clone_shingles, constant_values, files, ref_facts,
|
||||
// vectors, churn/coverage/release/blame_enrichment, symbol_fts,
|
||||
// symbol_fts_rowid, content_fts — see schema.go). Untracking a repo through
|
||||
// EvictRepo leaks every one of them, so a long-lived store accumulates
|
||||
// sidecar rows for repos removed from config long ago. PurgeRepo clears a
|
||||
// repo whole; OrphanRepoPrefixes finds prefixes that outlived their config
|
||||
// entry; RekeyRepoPrefix moves a lone repo's residue when it earns a prefix.
|
||||
//
|
||||
// INVARIANT — the empty repo_prefix is NEVER purged. In a live multi-repo
|
||||
// store repo_prefix='' identifies SYNTHETIC GLOBAL EXTERNALS (external_call
|
||||
// ::dep:* / builtin:: / module:: nodes shared across every repo) and, in a
|
||||
// single-repo store, the sole repo's live data. Deleting '' rows would strip
|
||||
// the shared externals out from under every repo, or wipe the lone repo.
|
||||
// Every method here refuses or excludes ''.
|
||||
|
||||
// purgeSidecarTables are the repo_prefix-keyed sidecar tables PurgeRepo
|
||||
// clears for a prefix, alongside nodes+edges. Each carries a repo_prefix
|
||||
// column a plain `DELETE ... WHERE repo_prefix = ?` keys on. The two FTS5
|
||||
// vtables (symbol_fts, content_fts) carry repo_prefix UNINDEXED, so their
|
||||
// delete is a full scan — acceptable for a purge (a rare, whole-repo op),
|
||||
// unlike the per-edit hot path. `vectors` is deliberately absent: it has NO
|
||||
// repo_prefix column (keyed by node_id alone), so PurgeRepo deletes its rows
|
||||
// by node-id membership instead (see deleteByIDColumnsTx below).
|
||||
var purgeSidecarTables = []string{
|
||||
"file_mtimes",
|
||||
"repo_index_state",
|
||||
"enrichment_state",
|
||||
"clone_shingles",
|
||||
"constant_values",
|
||||
"files",
|
||||
"ref_facts",
|
||||
"churn_enrichment",
|
||||
"coverage_enrichment",
|
||||
"release_enrichment",
|
||||
"blame_enrichment",
|
||||
"symbol_fts",
|
||||
"symbol_fts_rowid",
|
||||
"content_fts",
|
||||
}
|
||||
|
||||
// PurgeRepo deletes EVERY row a repo owns — nodes, edges, and all fifteen
|
||||
// repo_prefix-keyed sidecar tables (purgeSidecarTables + vectors) — in one
|
||||
// transaction. It is the complete form of EvictRepo (which drops only
|
||||
// nodes+edges), wired into UntrackRepo so removing a repo from config leaves
|
||||
// no residue. Refuses prefix=="" (shared global externals / solo-mode live
|
||||
// data — see the file-level INVARIANT).
|
||||
func (s *Store) PurgeRepo(prefix string) error {
|
||||
if prefix == "" {
|
||||
return fmt.Errorf("store_sqlite: PurgeRepo refuses empty repo prefix (would delete shared global externals / solo-repo data)")
|
||||
}
|
||||
|
||||
s.writeMu.Lock()
|
||||
defer s.writeMu.Unlock()
|
||||
|
||||
tx, err := s.db.Begin()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer tx.Rollback() //nolint:errcheck // rollback after Commit is a no-op
|
||||
|
||||
// Collect this repo's node IDs first: edges and vectors are keyed off
|
||||
// them (edges by from_id/to_id, vectors by node_id — neither carries a
|
||||
// repo_prefix column). Edge deletion semantics mirror evictByScopeLocked
|
||||
// (store.go): delete every edge touching one of these nodes, then the
|
||||
// nodes themselves.
|
||||
ids, err := repoNodeIDsTx(tx, prefix)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := deleteByIDColumnsTx(tx, "edges", []string{"from_id", "to_id"}, ids); err != nil {
|
||||
return fmt.Errorf("store_sqlite: PurgeRepo edges: %w", err)
|
||||
}
|
||||
if err := deleteByIDColumnsTx(tx, "vectors", []string{"node_id"}, ids); err != nil {
|
||||
return fmt.Errorf("store_sqlite: PurgeRepo vectors: %w", err)
|
||||
}
|
||||
|
||||
for _, table := range purgeSidecarTables {
|
||||
if _, err := tx.Exec(`DELETE FROM `+table+` WHERE repo_prefix = ?`, prefix); err != nil {
|
||||
return fmt.Errorf("store_sqlite: PurgeRepo %s: %w", table, err)
|
||||
}
|
||||
}
|
||||
|
||||
if _, err := tx.Exec(`DELETE FROM nodes WHERE repo_prefix = ?`, prefix); err != nil {
|
||||
return fmt.Errorf("store_sqlite: PurgeRepo nodes: %w", err)
|
||||
}
|
||||
|
||||
return tx.Commit()
|
||||
}
|
||||
|
||||
// orphanScanTables are the tables OrphanRepoPrefixes unions DISTINCT
|
||||
// repo_prefix over. These five span the residue space: nodes (the primary
|
||||
// keyed store), file_mtimes + repo_index_state (the warm-restart provenance
|
||||
// that lingers when nodes are gone but sidecars survive — the exact shape a
|
||||
// leaked untrack leaves), enrichment_state (per-provider provenance), and
|
||||
// files (per-file metadata). A prefix whose nodes are gone but whose
|
||||
// sidecars remain is invisible to a nodes-only scan, which is why the
|
||||
// sidecar tables are unioned in; scanning still more tables would only
|
||||
// rediscover the same prefixes at higher cost.
|
||||
var orphanScanTables = []string{
|
||||
"nodes",
|
||||
"file_mtimes",
|
||||
"repo_index_state",
|
||||
"enrichment_state",
|
||||
"files",
|
||||
}
|
||||
|
||||
// OrphanRepoPrefixes returns every repo_prefix present in the store but
|
||||
// absent from known — repos whose rows outlived their config entry (an
|
||||
// untrack that predated PurgeRepo, or a repo dropped straight from config
|
||||
// with no untrack at all). The empty prefix is NEVER reported (shared global
|
||||
// externals / solo data). known is matched case-insensitively as a safety
|
||||
// net, so a case-only spelling drift on a case-insensitive filesystem can
|
||||
// never flag a still-tracked repo as an orphan (the #270 failure mode).
|
||||
// Startup warmup feeds the result to PurgeRepo.
|
||||
func (s *Store) OrphanRepoPrefixes(known []string) []string {
|
||||
knownFold := make(map[string]struct{}, len(known))
|
||||
for _, k := range known {
|
||||
if k == "" {
|
||||
continue
|
||||
}
|
||||
knownFold[strings.ToLower(k)] = struct{}{}
|
||||
}
|
||||
|
||||
seen := make(map[string]struct{})
|
||||
var out []string
|
||||
for _, table := range orphanScanTables {
|
||||
// WHERE repo_prefix <> '' both excludes the protected empty prefix
|
||||
// and lets the nodes scan ride the partial nodes_by_repo index
|
||||
// (defined WHERE repo_prefix <> ''). A table absent on an older
|
||||
// schema simply contributes nothing.
|
||||
rows, err := s.db.Query(`SELECT DISTINCT repo_prefix FROM ` + table + ` WHERE repo_prefix <> ''`)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
for rows.Next() {
|
||||
var p string
|
||||
if err := rows.Scan(&p); err != nil {
|
||||
break
|
||||
}
|
||||
if p == "" {
|
||||
continue
|
||||
}
|
||||
if _, ok := knownFold[strings.ToLower(p)]; ok {
|
||||
continue
|
||||
}
|
||||
if _, dup := seen[p]; dup {
|
||||
continue
|
||||
}
|
||||
seen[p] = struct{}{}
|
||||
out = append(out, p)
|
||||
}
|
||||
_ = rows.Close()
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// rekeyMoveTables are the sidecar tables RekeyRepoPrefix relabels from old
|
||||
// to new. Every one is keyed by repo_prefix (+ file_path or provider), NOT
|
||||
// by node_id, so its row content survives a node-id change: file_mtimes /
|
||||
// files by (repo_prefix, file_path); repo_index_state / enrichment_state by
|
||||
// repo_prefix (+ provider). At a solo->multi migration every '' row in these
|
||||
// belongs to the one migrating repo — global externals live in the NODES
|
||||
// table and hold NO rows here — so moving them wholesale is safe. UPDATE OR
|
||||
// REPLACE folds any row the re-mint re-index already wrote under new
|
||||
// (identical content: same files, same mtimes, same commit) instead of
|
||||
// tripping the primary-key conflict a plain UPDATE would.
|
||||
var rekeyMoveTables = []string{
|
||||
"file_mtimes",
|
||||
"files",
|
||||
"repo_index_state",
|
||||
"enrichment_state",
|
||||
}
|
||||
|
||||
// rekeyDropTables are the sidecar tables RekeyRepoPrefix DROPS (rather than
|
||||
// relabels) for old. Every one is keyed by node_id, and the solo->multi
|
||||
// re-mint changes every node id (unprefixed `pkg::X` -> `<new>::pkg::X`), so
|
||||
// these old-id rows are already dangling against the evicted unprefixed
|
||||
// nodes. Relabeling their repo_prefix would just move dangling rows under
|
||||
// new — and let, e.g., the clone reseed load a shingle set for a node that
|
||||
// no longer exists. Dropping them is correct: the re-mint re-index rewrites
|
||||
// the index-time sidecars (constant_values, ref_facts, clone_shingles) under
|
||||
// the new node ids, and the enrichment sidecars (churn/coverage/release/
|
||||
// blame) must re-run for the new ids regardless. The FTS vtables sit here
|
||||
// too — their rows carry the old node ids, and UPDATE over an FTS5 UNINDEXED
|
||||
// column is awkward, so delete-then-reindex is the clean path.
|
||||
var rekeyDropTables = []string{
|
||||
"clone_shingles",
|
||||
"constant_values",
|
||||
"ref_facts",
|
||||
"churn_enrichment",
|
||||
"coverage_enrichment",
|
||||
"release_enrichment",
|
||||
"blame_enrichment",
|
||||
"symbol_fts",
|
||||
"symbol_fts_rowid",
|
||||
"content_fts",
|
||||
}
|
||||
|
||||
// RekeyRepoPrefix moves a repo's sidecar residue from old to new the moment a
|
||||
// solo (unprefixed) repo earns a real prefix because a second repo joined —
|
||||
// the migrateLoneUnprefixedRepoCtx path. The prefix/path-keyed provenance
|
||||
// tables (rekeyMoveTables) are relabeled so warm restart finds the repo's
|
||||
// mtimes + freshness under new instead of full-re-tracking it; the
|
||||
// node_id-keyed tables (rekeyDropTables) are dropped because the re-mint
|
||||
// changed every node id out from under them (see the two table lists for the
|
||||
// per-table rationale).
|
||||
//
|
||||
// Refuses new=="" (cannot rekey INTO the protected empty prefix). old=="" IS
|
||||
// allowed — that is the whole point, since solo repos index unprefixed — and
|
||||
// is safe here because this method touches SIDECAR tables ONLY; the synthetic
|
||||
// global externals that also carry repo_prefix='' live in the NODES table,
|
||||
// which RekeyRepoPrefix never writes.
|
||||
func (s *Store) RekeyRepoPrefix(oldPrefix, newPrefix string) error {
|
||||
if newPrefix == "" {
|
||||
return fmt.Errorf("store_sqlite: RekeyRepoPrefix refuses empty destination prefix")
|
||||
}
|
||||
if oldPrefix == newPrefix {
|
||||
return nil
|
||||
}
|
||||
|
||||
s.writeMu.Lock()
|
||||
defer s.writeMu.Unlock()
|
||||
|
||||
tx, err := s.db.Begin()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer tx.Rollback() //nolint:errcheck // rollback after Commit is a no-op
|
||||
|
||||
for _, table := range rekeyMoveTables {
|
||||
if _, err := tx.Exec(`UPDATE OR REPLACE `+table+` SET repo_prefix = ? WHERE repo_prefix = ?`, newPrefix, oldPrefix); err != nil {
|
||||
return fmt.Errorf("store_sqlite: RekeyRepoPrefix move %s: %w", table, err)
|
||||
}
|
||||
}
|
||||
for _, table := range rekeyDropTables {
|
||||
if _, err := tx.Exec(`DELETE FROM `+table+` WHERE repo_prefix = ?`, oldPrefix); err != nil {
|
||||
return fmt.Errorf("store_sqlite: RekeyRepoPrefix drop %s: %w", table, err)
|
||||
}
|
||||
}
|
||||
// vectors is intentionally omitted: it has NO repo_prefix column (keyed
|
||||
// by node_id alone), so it cannot be addressed here by prefix. Any ''
|
||||
// embeddings are node_id-keyed against now-evicted unprefixed ids —
|
||||
// dangling, and absent in the common case (embeddings are opt-in). They
|
||||
// are left to a node-membership vector GC rather than guessed at here.
|
||||
|
||||
return tx.Commit()
|
||||
}
|
||||
|
||||
// repoNodeIDsTx returns every node id in repoPrefix, read inside tx. The
|
||||
// caller holds writeMu. Rows are fully drained + closed before the caller
|
||||
// issues writes on the same tx — SQLite forbids an open read cursor while
|
||||
// writing on the same connection.
|
||||
func repoNodeIDsTx(tx *sql.Tx, repoPrefix string) ([]string, error) {
|
||||
rows, err := tx.Query(`SELECT id FROM nodes WHERE repo_prefix = ?`, repoPrefix)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var ids []string
|
||||
for rows.Next() {
|
||||
var id string
|
||||
if err := rows.Scan(&id); err != nil {
|
||||
_ = rows.Close()
|
||||
return nil, err
|
||||
}
|
||||
ids = append(ids, id)
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
_ = rows.Close()
|
||||
return nil, err
|
||||
}
|
||||
_ = rows.Close()
|
||||
return ids, nil
|
||||
}
|
||||
|
||||
// deleteByIDColumnsTx deletes rows from table where ANY of cols matches one
|
||||
// of ids, chunked so each statement stays under SQLite's 999 bound-variable
|
||||
// limit. Mirrors evictByScopeLocked's chunked from_id/to_id edge delete
|
||||
// (store.go) — the semantics source for edge eviction. Empty ids is a no-op.
|
||||
func deleteByIDColumnsTx(tx *sql.Tx, table string, cols, ids []string) error {
|
||||
if len(ids) == 0 {
|
||||
return nil
|
||||
}
|
||||
const chunk = 900
|
||||
for _, col := range cols {
|
||||
for start := 0; start < len(ids); start += chunk {
|
||||
end := minInt(start+chunk, len(ids))
|
||||
batch := ids[start:end]
|
||||
placeholders := strings.TrimSuffix(strings.Repeat("?,", len(batch)), ",")
|
||||
args := make([]any, len(batch))
|
||||
for i, id := range batch {
|
||||
args[i] = id
|
||||
}
|
||||
if _, err := tx.Exec(`DELETE FROM `+table+` WHERE `+col+` IN (`+placeholders+`)`, args...); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,222 @@
|
||||
package store_sqlite
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"github.com/zzet/gortex/internal/graph"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// openPurgeStore opens a throwaway on-disk store for the hygiene tests.
|
||||
func openPurgeStore(t *testing.T) *Store {
|
||||
t.Helper()
|
||||
s, err := Open(filepath.Join(t.TempDir(), "purge.sqlite"))
|
||||
require.NoError(t, err)
|
||||
t.Cleanup(func() { _ = s.Close() })
|
||||
return s
|
||||
}
|
||||
|
||||
// seedRepoRows inserts exactly one row keyed to prefix into nodes, edges,
|
||||
// vectors, and every repo_prefix-keyed sidecar table, so a PurgeRepo /
|
||||
// RekeyRepoPrefix test can assert each table is cleared/moved. Node/vector
|
||||
// ids embed the prefix (`<prefix>::a.go::X`) so the node-id-keyed vectors +
|
||||
// edges land in the prefix's scope. Uses raw SQL for exhaustiveness: some
|
||||
// tables have no public setter.
|
||||
func seedRepoRows(t *testing.T, db *sql.DB, prefix string) {
|
||||
t.Helper()
|
||||
nodeID := prefix + "::a.go::X"
|
||||
exec := func(q string, args ...any) {
|
||||
t.Helper()
|
||||
_, err := db.Exec(q, args...)
|
||||
require.NoError(t, err, q)
|
||||
}
|
||||
exec(`INSERT INTO nodes (id, kind, name, file_path, repo_prefix) VALUES (?, 'function', 'X', 'a.go', ?)`, nodeID, prefix)
|
||||
// An edge from the repo's node to a shared '' global external: PurgeRepo
|
||||
// must delete this edge (its from_id is a repo node) but NEVER the ''
|
||||
// target node.
|
||||
exec(`INSERT INTO edges (from_id, to_id, kind) VALUES (?, 'external_call::dep:shared', 'calls')`, nodeID)
|
||||
exec(`INSERT INTO vectors (node_id, dims, vec) VALUES (?, 1, X'00')`, nodeID)
|
||||
|
||||
exec(`INSERT INTO file_mtimes (repo_prefix, file_path, mtime_ns) VALUES (?, 'a.go', 123)`, prefix)
|
||||
exec(`INSERT INTO repo_index_state (repo_prefix, indexed_sha) VALUES (?, 'sha')`, prefix)
|
||||
exec(`INSERT INTO enrichment_state (repo_prefix, provider) VALUES (?, 'lsp')`, prefix)
|
||||
exec(`INSERT INTO clone_shingles (node_id, repo_prefix, shingles) VALUES (?, ?, X'00')`, nodeID, prefix)
|
||||
exec(`INSERT INTO constant_values (node_id, repo_prefix, file_path, value) VALUES (?, ?, 'a.go', 'v')`, nodeID, prefix)
|
||||
exec(`INSERT INTO files (repo_prefix, file_path, content_hash) VALUES (?, 'a.go', 'h')`, prefix)
|
||||
exec(`INSERT INTO ref_facts (repo_prefix, from_id, to_id, kind, line) VALUES (?, ?, 'a.go::Y', 'ref', 1)`, prefix, nodeID)
|
||||
exec(`INSERT INTO churn_enrichment (node_id, repo_prefix, commit_count) VALUES (?, ?, 3)`, nodeID, prefix)
|
||||
exec(`INSERT INTO coverage_enrichment (node_id, repo_prefix, coverage_pct) VALUES (?, ?, 0.5)`, nodeID, prefix)
|
||||
exec(`INSERT INTO release_enrichment (node_id, repo_prefix, added_in) VALUES (?, ?, 'v1')`, nodeID, prefix)
|
||||
exec(`INSERT INTO blame_enrichment (node_id, repo_prefix, email) VALUES (?, ?, 'a@b')`, nodeID, prefix)
|
||||
exec(`INSERT INTO symbol_fts (node_id, repo_prefix, tokens) VALUES (?, ?, 'x')`, nodeID, prefix)
|
||||
exec(`INSERT INTO symbol_fts_rowid (node_id, repo_prefix, fts_rowid) VALUES (?, ?, 1)`, nodeID, prefix)
|
||||
exec(`INSERT INTO content_fts (node_id, repo_prefix, file_path, ordinal, body) VALUES (?, ?, 'a.go', 0, 'body')`, nodeID, prefix)
|
||||
}
|
||||
|
||||
// countByPrefix reports how many rows a repo_prefix-keyed table holds for
|
||||
// prefix. nodes and every sidecar carry a repo_prefix column.
|
||||
func countByPrefix(t *testing.T, db *sql.DB, table, prefix string) int {
|
||||
t.Helper()
|
||||
var n int
|
||||
require.NoError(t, db.QueryRow(`SELECT COUNT(*) FROM `+table+` WHERE repo_prefix = ?`, prefix).Scan(&n))
|
||||
return n
|
||||
}
|
||||
|
||||
// countByNodeIDLike reports how many rows a node_id-keyed table (vectors)
|
||||
// holds whose node_id starts with `<prefix>::`.
|
||||
func countByNodeIDLike(t *testing.T, db *sql.DB, table, prefix string) int {
|
||||
t.Helper()
|
||||
var n int
|
||||
require.NoError(t, db.QueryRow(`SELECT COUNT(*) FROM `+table+` WHERE node_id LIKE ?`, prefix+"::%").Scan(&n))
|
||||
return n
|
||||
}
|
||||
|
||||
// prefixKeyedTables is every repo_prefix-keyed table PurgeRepo/Rekey touch,
|
||||
// minus nodes (asserted separately) — used to loop assertions.
|
||||
var prefixKeyedTables = []string{
|
||||
"file_mtimes", "repo_index_state", "enrichment_state", "clone_shingles",
|
||||
"constant_values", "files", "ref_facts", "churn_enrichment",
|
||||
"coverage_enrichment", "release_enrichment", "blame_enrichment",
|
||||
"symbol_fts", "symbol_fts_rowid", "content_fts",
|
||||
}
|
||||
|
||||
func TestPurgeRepo_ClearsEveryTable_LeavesOthersAndGlobals(t *testing.T) {
|
||||
s := openPurgeStore(t)
|
||||
// Two real repos plus a shared '' global-external node the purge must
|
||||
// never touch.
|
||||
seedRepoRows(t, s.db, "repoA")
|
||||
seedRepoRows(t, s.db, "repoB")
|
||||
_, err := s.db.Exec(`INSERT INTO nodes (id, kind, name, file_path, repo_prefix) VALUES ('external_call::dep:shared', 'external', 'shared', '', '')`)
|
||||
require.NoError(t, err)
|
||||
|
||||
require.NoError(t, s.PurgeRepo("repoA"))
|
||||
|
||||
// repoA: nodes, edges, vectors, and every sidecar cleared.
|
||||
assert.Equal(t, 0, countByPrefix(t, s.db, "nodes", "repoA"), "repoA nodes gone")
|
||||
assert.Equal(t, 0, countByNodeIDLike(t, s.db, "vectors", "repoA"), "repoA vectors gone")
|
||||
for _, tbl := range prefixKeyedTables {
|
||||
assert.Equal(t, 0, countByPrefix(t, s.db, tbl, "repoA"), "repoA %s cleared", tbl)
|
||||
}
|
||||
var edgesFromA int
|
||||
require.NoError(t, s.db.QueryRow(`SELECT COUNT(*) FROM edges WHERE from_id LIKE 'repoA::%'`).Scan(&edgesFromA))
|
||||
assert.Equal(t, 0, edgesFromA, "repoA edges gone")
|
||||
|
||||
// repoB untouched across the board.
|
||||
assert.Equal(t, 1, countByPrefix(t, s.db, "nodes", "repoB"), "repoB nodes intact")
|
||||
assert.Equal(t, 1, countByNodeIDLike(t, s.db, "vectors", "repoB"), "repoB vectors intact")
|
||||
for _, tbl := range prefixKeyedTables {
|
||||
assert.Equal(t, 1, countByPrefix(t, s.db, tbl, "repoB"), "repoB %s intact", tbl)
|
||||
}
|
||||
|
||||
// The shared '' global external survives — nothing may purge ''.
|
||||
assert.Equal(t, 1, countByPrefix(t, s.db, "nodes", ""), "'' global external survives")
|
||||
}
|
||||
|
||||
func TestPurgeRepo_RefusesEmptyPrefix(t *testing.T) {
|
||||
s := openPurgeStore(t)
|
||||
seedRepoRows(t, s.db, "")
|
||||
require.Error(t, s.PurgeRepo(""), "PurgeRepo must refuse the empty prefix (global externals / solo data)")
|
||||
// The '' rows are still there.
|
||||
assert.Equal(t, 1, countByPrefix(t, s.db, "file_mtimes", ""), "'' file_mtimes untouched by refused purge")
|
||||
}
|
||||
|
||||
func TestOrphanRepoPrefixes_SidecarOnlyResidue(t *testing.T) {
|
||||
s := openPurgeStore(t)
|
||||
// gone: a repo whose NODES were evicted but whose sidecars linger — the
|
||||
// exact leaked-untrack shape (residue in file_mtimes + repo_index_state,
|
||||
// no nodes). live: a fully-present tracked repo.
|
||||
_, err := s.db.Exec(`INSERT INTO file_mtimes (repo_prefix, file_path, mtime_ns) VALUES ('gone', 'x.go', 1)`)
|
||||
require.NoError(t, err)
|
||||
_, err = s.db.Exec(`INSERT INTO repo_index_state (repo_prefix) VALUES ('gone')`)
|
||||
require.NoError(t, err)
|
||||
seedRepoRows(t, s.db, "live")
|
||||
// A '' row must never be reported as an orphan.
|
||||
_, err = s.db.Exec(`INSERT INTO file_mtimes (repo_prefix, file_path, mtime_ns) VALUES ('', 'g.go', 1)`)
|
||||
require.NoError(t, err)
|
||||
|
||||
orphans := s.OrphanRepoPrefixes([]string{"live"})
|
||||
assert.Equal(t, []string{"gone"}, orphans, "only the nodes-less residue prefix is an orphan")
|
||||
|
||||
// Case-fold safety net: a case-only spelling drift of a tracked repo is
|
||||
// NOT an orphan.
|
||||
assert.Empty(t, s.OrphanRepoPrefixes([]string{"LIVE", "GONE"}), "case-insensitive known set covers both prefixes")
|
||||
}
|
||||
|
||||
func TestRekeyRepoPrefix_MovesProvenanceDropsNodeIDKeyed(t *testing.T) {
|
||||
s := openPurgeStore(t)
|
||||
seedRepoRows(t, s.db, "") // solo repo: everything under ''
|
||||
|
||||
require.NoError(t, s.RekeyRepoPrefix("", "drools"))
|
||||
|
||||
// Prefix/path-keyed provenance MOVED '' -> drools (so warm restart finds
|
||||
// the repo's mtimes under the new prefix instead of full-re-tracking).
|
||||
moveTables := []string{"file_mtimes", "files", "repo_index_state", "enrichment_state"}
|
||||
for _, tbl := range moveTables {
|
||||
assert.Equal(t, 0, countByPrefix(t, s.db, tbl, ""), "%s '' rows moved out", tbl)
|
||||
assert.Equal(t, 1, countByPrefix(t, s.db, tbl, "drools"), "%s rows now under new prefix", tbl)
|
||||
}
|
||||
|
||||
// node_id-keyed tables DROPPED (their old ids are dangling after the
|
||||
// re-mint) — the FTS decision included.
|
||||
dropTables := []string{
|
||||
"clone_shingles", "constant_values", "ref_facts", "churn_enrichment",
|
||||
"coverage_enrichment", "release_enrichment", "blame_enrichment",
|
||||
"symbol_fts", "symbol_fts_rowid", "content_fts",
|
||||
}
|
||||
for _, tbl := range dropTables {
|
||||
assert.Equal(t, 0, countByPrefix(t, s.db, tbl, ""), "%s '' rows dropped", tbl)
|
||||
assert.Equal(t, 0, countByPrefix(t, s.db, tbl, "drools"), "%s NOT relabeled to new prefix", tbl)
|
||||
}
|
||||
|
||||
assert.Error(t, s.RekeyRepoPrefix("repoA", ""), "rekey INTO the empty prefix is refused")
|
||||
}
|
||||
|
||||
// TestContentCrashWindow simulates the D4 kill-window at the store level:
|
||||
// per-file delete+append leaves a mix of old+new content instead of an empty
|
||||
// table, and the end-of-track sweep (keep = files that STREAMED content this
|
||||
// run) reaps both files that vanished from disk and files that still exist
|
||||
// but no longer yield content sections.
|
||||
func TestContentCrashWindow(t *testing.T) {
|
||||
s := openPurgeStore(t)
|
||||
item := func(id, file, body string) graph.ContentFTSItem {
|
||||
return graph.ContentFTSItem{NodeID: id, FilePath: file, Ordinal: 0, Body: body}
|
||||
}
|
||||
// Prior full index: three content files present.
|
||||
require.NoError(t, s.AppendContent("r", []graph.ContentFTSItem{item("r::f1", "f1.md", "old one")}))
|
||||
require.NoError(t, s.AppendContent("r", []graph.ContentFTSItem{item("r::f2", "f2.md", "old two")}))
|
||||
require.NoError(t, s.AppendContent("r", []graph.ContentFTSItem{item("r::f3", "f3.md", "old three")}))
|
||||
|
||||
// A new full index re-streams only f1 (crash before it reached the
|
||||
// rest): delete f1's rows then re-append. The other files' OLD rows must
|
||||
// survive — no empty-table window.
|
||||
require.NoError(t, s.WipeContentFileInRepo("r", "f1.md"))
|
||||
require.NoError(t, s.AppendContent("r", []graph.ContentFTSItem{item("r::f1", "f1.md", "new one")}))
|
||||
|
||||
countFile := func(file string) int {
|
||||
var n int
|
||||
require.NoError(t, s.db.QueryRow(`SELECT COUNT(*) FROM content_fts WHERE repo_prefix='r' AND file_path=?`, file).Scan(&n))
|
||||
return n
|
||||
}
|
||||
assert.Equal(t, 1, countFile("f1.md"), "f1 refreshed (delete+append, not doubled)")
|
||||
assert.Equal(t, 1, countFile("f2.md"), "f2's old rows survive the mid-parse kill (no empty table)")
|
||||
assert.Equal(t, 1, countFile("f3.md"), "f3's old rows survive the mid-parse kill (no empty table)")
|
||||
|
||||
// The next SUCCESSFUL completion: f2 was deleted from the repo, and f3
|
||||
// STILL EXISTS on disk but was emptied — it streamed no content sections
|
||||
// this run, so it is absent from the streamed set. The sweep keeps
|
||||
// exactly the streamed set {f1}, reaping both the vanished file and the
|
||||
// content->no-content transition (a disk-survival keep would have
|
||||
// protected f3's stale rows forever).
|
||||
require.NoError(t, s.DeleteContentFilesForRepoNotIn("r", map[string]struct{}{"f1.md": {}}))
|
||||
assert.Equal(t, 1, countFile("f1.md"), "still-streaming file kept")
|
||||
assert.Equal(t, 0, countFile("f2.md"), "vanished file swept")
|
||||
assert.Equal(t, 0, countFile("f3.md"), "content->no-content transition swept despite surviving on disk")
|
||||
|
||||
// Empty keep is a deliberate no-op (the never-wipe-from-empty safety
|
||||
// net; a zero-content walk routes to WipeContent instead).
|
||||
require.NoError(t, s.DeleteContentFilesForRepoNotIn("r", nil))
|
||||
assert.Equal(t, 1, countFile("f1.md"), "empty keep never wipes")
|
||||
}
|
||||
@@ -0,0 +1,219 @@
|
||||
package store_sqlite
|
||||
|
||||
import (
|
||||
"strings"
|
||||
|
||||
"github.com/zzet/gortex/internal/graph"
|
||||
)
|
||||
|
||||
// Compile-time assertions that the SQLite Store satisfies the optional
|
||||
// reference-facts persistence capability. Persisting resolved-reference facts
|
||||
// in the same backend the graph lives in makes a reference's resolution an
|
||||
// auditable, diffable record and a warm-restart seed.
|
||||
var (
|
||||
_ graph.RefFactsWriter = (*Store)(nil)
|
||||
_ graph.RefFactsReader = (*Store)(nil)
|
||||
)
|
||||
|
||||
// refFactChunk bounds rows per multi-row INSERT. 11 params/row; 80 rows = 880
|
||||
// host params, under SQLite's 999 default. Mirrors shingleChunk.
|
||||
const refFactChunk = 80
|
||||
|
||||
// candidate-list separator (unit separator — never appears in identifiers).
|
||||
const refFactCandSep = "\x1f"
|
||||
|
||||
func encodeCandidates(c []string) string { return strings.Join(c, refFactCandSep) }
|
||||
|
||||
func decodeCandidates(s string) []string {
|
||||
if s == "" {
|
||||
return nil
|
||||
}
|
||||
return strings.Split(s, refFactCandSep)
|
||||
}
|
||||
|
||||
// BulkSetRefFacts persists resolved-reference facts for one repo prefix in a
|
||||
// single transaction, chunked under the host-parameter limit. Idempotent on
|
||||
// (repo_prefix, from_id, to_id, kind, line). Empty input is a no-op.
|
||||
func (s *Store) BulkSetRefFacts(repoPrefix string, facts []graph.RefFact) error {
|
||||
if len(facts) == 0 {
|
||||
return nil
|
||||
}
|
||||
s.writeMu.Lock()
|
||||
defer s.writeMu.Unlock()
|
||||
|
||||
tx, err := s.db.Begin()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer tx.Rollback() //nolint:errcheck // rollback after Commit is a no-op
|
||||
|
||||
for start := 0; start < len(facts); start += refFactChunk {
|
||||
end := start + refFactChunk
|
||||
if end > len(facts) {
|
||||
end = len(facts)
|
||||
}
|
||||
batch := facts[start:end]
|
||||
args := make([]any, 0, len(batch)*11)
|
||||
stmt := make([]byte, 0, 96+len(batch)*24)
|
||||
stmt = append(stmt, "INSERT OR REPLACE INTO ref_facts (repo_prefix, from_id, to_id, kind, ref_name, line, origin, tier, candidates, file_path, lang) VALUES "...)
|
||||
for i, f := range batch {
|
||||
if i > 0 {
|
||||
stmt = append(stmt, ',')
|
||||
}
|
||||
stmt = append(stmt, "(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)"...)
|
||||
args = append(args, repoPrefix, f.FromID, f.ToID, f.Kind, f.RefName, f.Line, f.Origin, f.Tier, encodeCandidates(f.Candidates), f.FilePath, f.Lang)
|
||||
}
|
||||
if _, err := tx.Exec(string(stmt), args...); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return tx.Commit()
|
||||
}
|
||||
|
||||
// DeleteRefFactsByFiles drops all facts sourced in the supplied files for one
|
||||
// repo prefix, chunked into `file_path IN (…)` DELETEs. Empty input is a no-op.
|
||||
func (s *Store) DeleteRefFactsByFiles(repoPrefix string, files []string) error {
|
||||
if len(files) == 0 {
|
||||
return nil
|
||||
}
|
||||
s.writeMu.Lock()
|
||||
defer s.writeMu.Unlock()
|
||||
|
||||
tx, err := s.db.Begin()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer tx.Rollback() //nolint:errcheck // rollback after Commit is a no-op
|
||||
|
||||
for start := 0; start < len(files); start += refFactChunk {
|
||||
end := start + refFactChunk
|
||||
if end > len(files) {
|
||||
end = len(files)
|
||||
}
|
||||
chunk := files[start:end]
|
||||
args := make([]any, 0, len(chunk)+1)
|
||||
args = append(args, repoPrefix)
|
||||
stmt := make([]byte, 0, 64+len(chunk)*2)
|
||||
stmt = append(stmt, "DELETE FROM ref_facts WHERE repo_prefix = ? AND file_path IN ("...)
|
||||
for i, f := range chunk {
|
||||
if i > 0 {
|
||||
stmt = append(stmt, ',')
|
||||
}
|
||||
stmt = append(stmt, '?')
|
||||
args = append(args, f)
|
||||
}
|
||||
stmt = append(stmt, ')')
|
||||
if _, err := tx.Exec(string(stmt), args...); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return tx.Commit()
|
||||
}
|
||||
|
||||
// LoadRefFactsByFiles returns the persisted facts for one repo prefix, scoped
|
||||
// to the given files (all files when files is empty). Always non-nil.
|
||||
func (s *Store) LoadRefFactsByFiles(repoPrefix string, files []string) ([]graph.RefFact, error) {
|
||||
out := []graph.RefFact{}
|
||||
scan := func(query string, args ...any) error {
|
||||
rows, err := s.db.Query(query, args...)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer rows.Close()
|
||||
for rows.Next() {
|
||||
var f graph.RefFact
|
||||
var cand string
|
||||
if err := rows.Scan(&f.FromID, &f.ToID, &f.Kind, &f.RefName, &f.Line, &f.Origin, &f.Tier, &cand, &f.FilePath, &f.Lang); err != nil {
|
||||
return err
|
||||
}
|
||||
f.RepoPrefix = repoPrefix
|
||||
f.Candidates = decodeCandidates(cand)
|
||||
out = append(out, f)
|
||||
}
|
||||
return rows.Err()
|
||||
}
|
||||
const cols = `from_id, to_id, kind, ref_name, line, origin, tier, candidates, file_path, lang`
|
||||
if len(files) == 0 {
|
||||
if err := scan(`SELECT `+cols+` FROM ref_facts WHERE repo_prefix = ?`, repoPrefix); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
for start := 0; start < len(files); start += refFactChunk {
|
||||
end := start + refFactChunk
|
||||
if end > len(files) {
|
||||
end = len(files)
|
||||
}
|
||||
chunk := files[start:end]
|
||||
args := make([]any, 0, len(chunk)+1)
|
||||
args = append(args, repoPrefix)
|
||||
stmt := make([]byte, 0, 96+len(chunk)*2)
|
||||
stmt = append(stmt, "SELECT "+cols+" FROM ref_facts WHERE repo_prefix = ? AND file_path IN ("...)
|
||||
for i, f := range chunk {
|
||||
if i > 0 {
|
||||
stmt = append(stmt, ',')
|
||||
}
|
||||
stmt = append(stmt, '?')
|
||||
args = append(args, f)
|
||||
}
|
||||
stmt = append(stmt, ')')
|
||||
if err := scan(string(stmt), args...); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// LoadRefFactsByTargets returns the persisted facts that resolve TO any of
|
||||
// the given node IDs for one repo prefix, grouped by source file path — the
|
||||
// reverse lookup incremental re-resolution uses to find the files that
|
||||
// referenced a changed symbol after its live in-edges were evicted. Served by
|
||||
// the ref_facts_by_target index, chunked under the host-parameter limit.
|
||||
// Always non-nil; empty input is a no-op.
|
||||
func (s *Store) LoadRefFactsByTargets(repoPrefix string, targetIDs []string) (map[string][]graph.RefFact, error) {
|
||||
out := map[string][]graph.RefFact{}
|
||||
if len(targetIDs) == 0 {
|
||||
return out, nil
|
||||
}
|
||||
const cols = `from_id, to_id, kind, ref_name, line, origin, tier, candidates, file_path, lang`
|
||||
for start := 0; start < len(targetIDs); start += refFactChunk {
|
||||
end := start + refFactChunk
|
||||
if end > len(targetIDs) {
|
||||
end = len(targetIDs)
|
||||
}
|
||||
chunk := targetIDs[start:end]
|
||||
args := make([]any, 0, len(chunk)+1)
|
||||
args = append(args, repoPrefix)
|
||||
stmt := make([]byte, 0, 96+len(chunk)*2)
|
||||
stmt = append(stmt, "SELECT "+cols+" FROM ref_facts WHERE repo_prefix = ? AND to_id IN ("...)
|
||||
for i, id := range chunk {
|
||||
if i > 0 {
|
||||
stmt = append(stmt, ',')
|
||||
}
|
||||
stmt = append(stmt, '?')
|
||||
args = append(args, id)
|
||||
}
|
||||
stmt = append(stmt, ')')
|
||||
rows, err := s.db.Query(string(stmt), args...)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for rows.Next() {
|
||||
var f graph.RefFact
|
||||
var cand string
|
||||
if err := rows.Scan(&f.FromID, &f.ToID, &f.Kind, &f.RefName, &f.Line, &f.Origin, &f.Tier, &cand, &f.FilePath, &f.Lang); err != nil {
|
||||
_ = rows.Close()
|
||||
return nil, err
|
||||
}
|
||||
f.RepoPrefix = repoPrefix
|
||||
f.Candidates = decodeCandidates(cand)
|
||||
out[f.FilePath] = append(out[f.FilePath], f)
|
||||
}
|
||||
err = rows.Err()
|
||||
_ = rows.Close()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
@@ -0,0 +1,172 @@
|
||||
package store_sqlite_test
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"github.com/zzet/gortex/internal/graph"
|
||||
"github.com/zzet/gortex/internal/graph/store_sqlite"
|
||||
)
|
||||
|
||||
func openRefFactStore(t *testing.T) *store_sqlite.Store {
|
||||
t.Helper()
|
||||
s, err := store_sqlite.Open(filepath.Join(t.TempDir(), "rf.sqlite"))
|
||||
require.NoError(t, err)
|
||||
t.Cleanup(func() { _ = s.Close() })
|
||||
return s
|
||||
}
|
||||
|
||||
func TestRefFacts_Roundtrip(t *testing.T) {
|
||||
s := openRefFactStore(t)
|
||||
facts := []graph.RefFact{
|
||||
{RepoPrefix: "", FromID: "a.go::A", ToID: "b.go::B", Kind: "calls", RefName: "B", Line: 7, Origin: "ast_resolved", Tier: "ast", FilePath: "a.go", Lang: "go", Candidates: []string{"b.go::B", "c.go::B"}},
|
||||
{RepoPrefix: "", FromID: "a.go::A", ToID: "d.go::D", Kind: "references", RefName: "D", Line: 9, Origin: "lsp_resolved", Tier: "lsp", FilePath: "a.go", Lang: "go"},
|
||||
}
|
||||
require.NoError(t, s.BulkSetRefFacts("", facts))
|
||||
|
||||
got, err := s.LoadRefFactsByFiles("", []string{"a.go"})
|
||||
require.NoError(t, err)
|
||||
require.Len(t, got, 2)
|
||||
|
||||
byTo := map[string]graph.RefFact{}
|
||||
for _, f := range got {
|
||||
byTo[f.ToID] = f
|
||||
}
|
||||
require.Equal(t, "ast_resolved", byTo["b.go::B"].Origin)
|
||||
require.Equal(t, []string{"b.go::B", "c.go::B"}, byTo["b.go::B"].Candidates)
|
||||
require.Equal(t, 7, byTo["b.go::B"].Line)
|
||||
require.Equal(t, "lsp_resolved", byTo["d.go::D"].Origin)
|
||||
|
||||
// LoadRefFactsByFiles with empty file list returns all for the repo.
|
||||
all, err := s.LoadRefFactsByFiles("", nil)
|
||||
require.NoError(t, err)
|
||||
require.Len(t, all, 2)
|
||||
}
|
||||
|
||||
func TestRefFacts_DeleteByFile(t *testing.T) {
|
||||
s := openRefFactStore(t)
|
||||
require.NoError(t, s.BulkSetRefFacts("", []graph.RefFact{
|
||||
{FromID: "a.go::A", ToID: "x", Kind: "calls", FilePath: "a.go"},
|
||||
{FromID: "b.go::B", ToID: "y", Kind: "calls", FilePath: "b.go"},
|
||||
}))
|
||||
require.NoError(t, s.DeleteRefFactsByFiles("", []string{"a.go"}))
|
||||
got, err := s.LoadRefFactsByFiles("", nil)
|
||||
require.NoError(t, err)
|
||||
require.Len(t, got, 1)
|
||||
require.Equal(t, "b.go", got[0].FilePath)
|
||||
}
|
||||
|
||||
func TestRefFacts_RepoScoping(t *testing.T) {
|
||||
s := openRefFactStore(t)
|
||||
require.NoError(t, s.BulkSetRefFacts("repoA", []graph.RefFact{{FromID: "f::A", ToID: "tA", Kind: "calls", FilePath: "f.go"}}))
|
||||
require.NoError(t, s.BulkSetRefFacts("repoB", []graph.RefFact{{FromID: "f::A", ToID: "tB", Kind: "calls", FilePath: "f.go"}}))
|
||||
|
||||
a, err := s.LoadRefFactsByFiles("repoA", []string{"f.go"})
|
||||
require.NoError(t, err)
|
||||
require.Len(t, a, 1)
|
||||
require.Equal(t, "tA", a[0].ToID)
|
||||
|
||||
// Deleting repoA's file must not touch repoB.
|
||||
require.NoError(t, s.DeleteRefFactsByFiles("repoA", []string{"f.go"}))
|
||||
b, err := s.LoadRefFactsByFiles("repoB", []string{"f.go"})
|
||||
require.NoError(t, err)
|
||||
require.Len(t, b, 1)
|
||||
require.Equal(t, "tB", b[0].ToID)
|
||||
}
|
||||
|
||||
func TestRefFacts_Chunking(t *testing.T) {
|
||||
s := openRefFactStore(t)
|
||||
const n = 500 // > refFactChunk (80)
|
||||
facts := make([]graph.RefFact, n)
|
||||
for i := range facts {
|
||||
facts[i] = graph.RefFact{FromID: fmt.Sprintf("a.go::f%d", i), ToID: fmt.Sprintf("t%d", i), Kind: "calls", FilePath: "a.go"}
|
||||
}
|
||||
require.NoError(t, s.BulkSetRefFacts("", facts))
|
||||
got, err := s.LoadRefFactsByFiles("", []string{"a.go"})
|
||||
require.NoError(t, err)
|
||||
require.Len(t, got, n)
|
||||
}
|
||||
|
||||
func TestRefFacts_EmptyNoop(t *testing.T) {
|
||||
s := openRefFactStore(t)
|
||||
require.NoError(t, s.BulkSetRefFacts("", nil))
|
||||
require.NoError(t, s.DeleteRefFactsByFiles("", nil))
|
||||
got, err := s.LoadRefFactsByFiles("", nil)
|
||||
require.NoError(t, err)
|
||||
require.Empty(t, got)
|
||||
}
|
||||
|
||||
func TestRefFacts_LoadByTargets(t *testing.T) {
|
||||
s := openRefFactStore(t)
|
||||
require.NoError(t, s.BulkSetRefFacts("", []graph.RefFact{
|
||||
{FromID: "b.go::Caller", ToID: "a.go::F", Kind: "calls", RefName: "F", Line: 3, Origin: "ast_resolved", Tier: "ast", FilePath: "b.go", Lang: "go"},
|
||||
{FromID: "c.go::Other", ToID: "a.go::F", Kind: "references", RefName: "F", FilePath: "c.go"},
|
||||
{FromID: "c.go::Other", ToID: "a.go::G", Kind: "calls", RefName: "G", FilePath: "c.go"},
|
||||
{FromID: "d.go::X", ToID: "z.go::Z", Kind: "calls", RefName: "Z", FilePath: "d.go"},
|
||||
}))
|
||||
|
||||
byFile, err := s.LoadRefFactsByTargets("", []string{"a.go::F", "a.go::G"})
|
||||
require.NoError(t, err)
|
||||
require.Len(t, byFile, 2, "facts must be grouped by source file: %v", byFile)
|
||||
require.Len(t, byFile["b.go"], 1)
|
||||
require.Equal(t, "a.go::F", byFile["b.go"][0].ToID)
|
||||
require.Equal(t, "F", byFile["b.go"][0].RefName)
|
||||
require.Equal(t, "ast_resolved", byFile["b.go"][0].Origin)
|
||||
require.Len(t, byFile["c.go"], 2, "both of c.go's facts target the queried symbols")
|
||||
require.NotContains(t, byFile, "d.go", "a fact targeting an unqueried symbol must not match")
|
||||
}
|
||||
|
||||
func TestRefFacts_LoadByTargets_EmptyAndMissing(t *testing.T) {
|
||||
s := openRefFactStore(t)
|
||||
require.NoError(t, s.BulkSetRefFacts("", []graph.RefFact{
|
||||
{FromID: "a.go::A", ToID: "b.go::B", Kind: "calls", FilePath: "a.go"},
|
||||
}))
|
||||
|
||||
// Empty input: empty, non-nil map.
|
||||
empty, err := s.LoadRefFactsByTargets("", nil)
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, empty)
|
||||
require.Empty(t, empty)
|
||||
|
||||
// A target nothing references: no rows, no error.
|
||||
miss, err := s.LoadRefFactsByTargets("", []string{"nope::Missing"})
|
||||
require.NoError(t, err)
|
||||
require.Empty(t, miss)
|
||||
}
|
||||
|
||||
func TestRefFacts_LoadByTargets_RepoScoping(t *testing.T) {
|
||||
s := openRefFactStore(t)
|
||||
require.NoError(t, s.BulkSetRefFacts("repoA", []graph.RefFact{{FromID: "fa::A", ToID: "shared::T", Kind: "calls", FilePath: "fa.go"}}))
|
||||
require.NoError(t, s.BulkSetRefFacts("repoB", []graph.RefFact{{FromID: "fb::B", ToID: "shared::T", Kind: "calls", FilePath: "fb.go"}}))
|
||||
|
||||
a, err := s.LoadRefFactsByTargets("repoA", []string{"shared::T"})
|
||||
require.NoError(t, err)
|
||||
require.Len(t, a, 1)
|
||||
require.Len(t, a["fa.go"], 1)
|
||||
require.Equal(t, "repoA", a["fa.go"][0].RepoPrefix, "loaded facts must carry the queried repo prefix")
|
||||
require.NotContains(t, a, "fb.go", "another repo's facts must not leak into the result")
|
||||
}
|
||||
|
||||
func TestRefFacts_LoadByTargets_Chunking(t *testing.T) {
|
||||
s := openRefFactStore(t)
|
||||
const n = 500 // > refFactChunk (80)
|
||||
facts := make([]graph.RefFact, n)
|
||||
targets := make([]string, n)
|
||||
for i := range facts {
|
||||
facts[i] = graph.RefFact{
|
||||
FromID: fmt.Sprintf("src%d.go::f", i),
|
||||
ToID: fmt.Sprintf("dst.go::t%d", i),
|
||||
Kind: "calls",
|
||||
FilePath: fmt.Sprintf("src%d.go", i),
|
||||
}
|
||||
targets[i] = fmt.Sprintf("dst.go::t%d", i)
|
||||
}
|
||||
require.NoError(t, s.BulkSetRefFacts("", facts))
|
||||
|
||||
byFile, err := s.LoadRefFactsByTargets("", targets)
|
||||
require.NoError(t, err)
|
||||
require.Len(t, byFile, n, "every chunked target must come back grouped under its source file")
|
||||
}
|
||||
@@ -0,0 +1,140 @@
|
||||
package store_sqlite
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
|
||||
"github.com/zzet/gortex/internal/graph"
|
||||
)
|
||||
|
||||
var (
|
||||
_ graph.ReleaseEnrichmentWriter = (*Store)(nil)
|
||||
_ graph.ReleaseEnrichmentReader = (*Store)(nil)
|
||||
)
|
||||
|
||||
// releaseChunk bounds rows per multi-row INSERT (3 cols → 3 params/row).
|
||||
const releaseChunk = 300
|
||||
|
||||
const releaseCols = `node_id, repo_prefix, added_in`
|
||||
|
||||
// BulkSetReleases persists release rows for one repo prefix, chunked.
|
||||
func (s *Store) BulkSetReleases(repoPrefix string, rows []graph.ReleaseEnrichment) error {
|
||||
if len(rows) == 0 {
|
||||
return nil
|
||||
}
|
||||
s.writeMu.Lock()
|
||||
defer s.writeMu.Unlock()
|
||||
|
||||
tx, err := s.db.Begin()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer tx.Rollback() //nolint:errcheck
|
||||
|
||||
for start := 0; start < len(rows); start += releaseChunk {
|
||||
end := start + releaseChunk
|
||||
if end > len(rows) {
|
||||
end = len(rows)
|
||||
}
|
||||
batch := rows[start:end]
|
||||
args := make([]any, 0, len(batch)*3)
|
||||
stmt := make([]byte, 0, 96+len(batch)*12)
|
||||
stmt = append(stmt, "INSERT OR REPLACE INTO release_enrichment ("...)
|
||||
stmt = append(stmt, releaseCols...)
|
||||
stmt = append(stmt, ") VALUES "...)
|
||||
for i, e := range batch {
|
||||
if i > 0 {
|
||||
stmt = append(stmt, ',')
|
||||
}
|
||||
stmt = append(stmt, "(?,?,?)"...)
|
||||
args = append(args, e.NodeID, repoPrefix, e.AddedIn)
|
||||
}
|
||||
if _, err := tx.Exec(string(stmt), args...); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return tx.Commit()
|
||||
}
|
||||
|
||||
// DeleteReleases drops release rows for the supplied node ids, chunked.
|
||||
func (s *Store) DeleteReleases(nodeIDs []string) error {
|
||||
if len(nodeIDs) == 0 {
|
||||
return nil
|
||||
}
|
||||
seen := make(map[string]struct{}, len(nodeIDs))
|
||||
uniq := make([]string, 0, len(nodeIDs))
|
||||
for _, id := range nodeIDs {
|
||||
if id == "" {
|
||||
continue
|
||||
}
|
||||
if _, ok := seen[id]; ok {
|
||||
continue
|
||||
}
|
||||
seen[id] = struct{}{}
|
||||
uniq = append(uniq, id)
|
||||
}
|
||||
if len(uniq) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
s.writeMu.Lock()
|
||||
defer s.writeMu.Unlock()
|
||||
|
||||
tx, err := s.db.Begin()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer tx.Rollback() //nolint:errcheck
|
||||
|
||||
for start := 0; start < len(uniq); start += releaseChunk {
|
||||
end := start + releaseChunk
|
||||
if end > len(uniq) {
|
||||
end = len(uniq)
|
||||
}
|
||||
chunk := uniq[start:end]
|
||||
args := make([]any, len(chunk))
|
||||
stmt := make([]byte, 0, 56+len(chunk)*2)
|
||||
stmt = append(stmt, "DELETE FROM release_enrichment WHERE node_id IN ("...)
|
||||
for i, id := range chunk {
|
||||
if i > 0 {
|
||||
stmt = append(stmt, ',')
|
||||
}
|
||||
stmt = append(stmt, '?')
|
||||
args[i] = id
|
||||
}
|
||||
stmt = append(stmt, ')')
|
||||
if _, err := tx.Exec(string(stmt), args...); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return tx.Commit()
|
||||
}
|
||||
|
||||
// ReleaseRows returns release rows for repoPrefix; empty → all repos.
|
||||
func (s *Store) ReleaseRows(repoPrefix string) []graph.ReleaseEnrichment {
|
||||
var (
|
||||
rows *sql.Rows
|
||||
err error
|
||||
)
|
||||
if repoPrefix == "" {
|
||||
rows, err = s.db.Query(`SELECT ` + releaseCols + ` FROM release_enrichment`)
|
||||
} else {
|
||||
rows, err = s.db.Query(`SELECT `+releaseCols+` FROM release_enrichment WHERE repo_prefix = ?`, repoPrefix)
|
||||
}
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
var out []graph.ReleaseEnrichment
|
||||
for rows.Next() {
|
||||
var e graph.ReleaseEnrichment
|
||||
if err := rows.Scan(&e.NodeID, &e.RepoPrefix, &e.AddedIn); err != nil {
|
||||
return out
|
||||
}
|
||||
out = append(out, e)
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return out
|
||||
}
|
||||
return out
|
||||
}
|
||||
@@ -0,0 +1,22 @@
|
||||
package store_sqlite_test
|
||||
|
||||
import (
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"github.com/zzet/gortex/internal/graph"
|
||||
"github.com/zzet/gortex/internal/graph/store_sqlite"
|
||||
"github.com/zzet/gortex/internal/graph/storetest"
|
||||
)
|
||||
|
||||
func TestSQLiteStoreConformance(t *testing.T) {
|
||||
storetest.RunConformance(t, func(t *testing.T) graph.Store {
|
||||
dir := t.TempDir()
|
||||
s, err := store_sqlite.Open(filepath.Join(dir, "test.sqlite"))
|
||||
if err != nil {
|
||||
t.Fatalf("Open: %v", err)
|
||||
}
|
||||
t.Cleanup(func() { _ = s.Close() })
|
||||
return s
|
||||
})
|
||||
}
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user