chore: import upstream snapshot with attribution
CI / benchmark (push) Has been skipped
install-script / posix-syntax (push) Successful in 6m1s
CI / build-onnx (push) Failing after 6m43s
init-smoke / dry-run (push) Failing after 15m57s
security / govulncheck (push) Has been cancelled
security / trivy-fs (push) Has been cancelled
CI / test (1.26, ubuntu-latest) (push) Has been cancelled
Scorecard supply-chain security / Scorecard analysis (push) Has been cancelled
CI / test (1.26, macos-latest) (push) Has been cancelled
CI / build-windows (push) Has been cancelled
CI / lint (push) Has been cancelled
install-script / powershell-syntax (push) Has been cancelled
install-script / install (macos-14) (push) Has been cancelled
install-script / install (ubuntu-latest) (push) Has been cancelled

This commit is contained in:
wehub-resource-sync
2026-07-13 12:33:42 +08:00
commit a06f331eb8
3186 changed files with 689843 additions and 0 deletions
+218
View File
@@ -0,0 +1,218 @@
package store_sqlite
import (
"context"
"database/sql"
"errors"
"fmt"
"github.com/zzet/gortex/internal/graph"
)
// Compile-time assertion: *Store satisfies graph.BulkLoader.
var _ graph.BulkLoader = (*Store)(nil)
// bulkDroppableIndex is one secondary index the bulk-load fast path drops
// before a first/empty cold index and rebuilds afterward.
type bulkDroppableIndex struct {
name string
ddl string
}
// bulkDroppableIndexes is the single source of truth for these index
// definitions. Open creates them (so the initial DB has them), BeginBulkLoad
// drops them by name, and FlushBulk recreates them from the exact same ddl —
// keeping the initial and post-bulk shapes from drifting.
//
// These are exactly the standalone, NON-UNIQUE CREATE INDEX statements over
// the large nodes / edges tables. Maintaining them per-row across a
// multi-hundred-thousand-row cold load is pure overhead when the rows land
// once, so they are dropped up front and rebuilt in one pass at the end.
//
// Deliberately excluded:
// - nodes_by_qual (UNIQUE): enforces qual_name dedup on every
// INSERT OR REPLACE. Dropping it would change insert conflict semantics
// (collapsed qual_name collisions would diverge from the non-bulk path)
// and a duplicate could make the recreate fail. It stays live.
// - the edges UNIQUE(from_id, …) table constraint and every WITHOUT ROWID
// primary-key index: not standalone indexes; they cannot be dropped while
// the table/constraint exists.
// - edges_external (partial): a tiny index over external-call terminals,
// created from a shared predicate in Open; not worth dropping.
//
// Dropping/recreating these is a runtime operation on identical DDL — it is
// NOT a schema change, so it does not touch the persisted schema version.
var bulkDroppableIndexes = []bulkDroppableIndex{
{"nodes_by_name", `CREATE INDEX IF NOT EXISTS nodes_by_name ON nodes(name)`},
{"nodes_by_kind", `CREATE INDEX IF NOT EXISTS nodes_by_kind ON nodes(kind)`},
{"nodes_by_file", `CREATE INDEX IF NOT EXISTS nodes_by_file ON nodes(file_path)`},
{"nodes_by_repo", `CREATE INDEX IF NOT EXISTS nodes_by_repo ON nodes(repo_prefix) WHERE repo_prefix <> ''`},
{"edges_by_from", `CREATE INDEX IF NOT EXISTS edges_by_from ON edges(from_id, kind)`},
{"edges_by_to", `CREATE INDEX IF NOT EXISTS edges_by_to ON edges(to_id, kind)`},
{"edges_by_kind", `CREATE INDEX IF NOT EXISTS edges_by_kind ON edges(kind)`},
// Backs EdgesWithUnresolvedTarget — the resolver's main pending-edge
// collector, called on every full resolve. is_unresolved is a VIRTUAL
// generated column (see isUnresolvedColumnDDL); indexing it turns a
// full-table scan (the prior to_id-based OR query forced SQLite to
// abandon its index) into an index search whose bookmark lookups land
// in ascending rowid order (see isUnresolvedColumnDDL's doc comment for
// why that beats an equivalent-looking to_id-based index).
{"edges_by_unresolved", `CREATE INDEX IF NOT EXISTS edges_by_unresolved ON edges(is_unresolved)`},
// Partial index over exactly the not-yet-semantically-stamped nodes per
// repo. Stays small in steady state (most nodes end up stamped), so a
// future "unstamped nodes in this repo" query is an index scan over the
// residual few instead of a full-table decode of every node's meta.
{"nodes_semantic_pending", `CREATE INDEX IF NOT EXISTS nodes_semantic_pending ON nodes(repo_prefix) WHERE semantic_type IS NULL`},
}
// bulkCacheSizeKiB is the page cache the fast path requests on its pinned
// connection. SQLite reads a negative cache_size as a KiB budget, so this is
// ~256 MiB — large enough to keep the cold load's working set resident.
const bulkCacheSizeKiB = -262144
// beginWrite starts a write transaction. During a bulk-load fast path it pins
// the single connection that carries synchronous=OFF + the enlarged page
// cache (database/sql PRAGMAs are connection-local, so a pooled connection
// would not see them); otherwise it uses the shared pool. The caller holds
// writeMu, which also guards s.bulkConn.
func (s *Store) beginWrite() (*sql.Tx, error) {
if s.bulkConn != nil {
return s.bulkConn.BeginTx(context.Background(), nil)
}
return s.db.Begin()
}
// BeginBulkLoad enters the bulk-load fast path for a first/empty cold index.
// It pins one connection at synchronous=OFF with an enlarged page cache and
// drops the droppable secondary indexes, so a multi-hundred-thousand-row load
// skips per-row B-tree maintenance and per-commit fsync. FlushBulk reverses
// all of it: restore the pragmas, rebuild the indexes, and checkpoint.
//
// Gated: it engages ONLY when the nodes table is empty. On a populated store
// (incremental reindex, warm restart, or a later repo in a multi-repo cold
// start that shares the disk store) it is a safe no-op — dropping indexes or
// disabling crash durability under live, concurrently-readable rows would be
// unsafe. In-memory stores have no WAL / on-disk B-tree pressure, so it is a
// no-op there too.
func (s *Store) BeginBulkLoad() {
s.writeMu.Lock()
defer s.writeMu.Unlock()
// Re-entrancy / non-disk guard: a second BeginBulkLoad without an
// intervening FlushBulk, or an in-memory store, stays a no-op.
if s.bulkConn != nil || isMemoryPath(s.dbPath) {
return
}
ctx := context.Background()
conn, err := s.db.Conn(ctx)
if err != nil {
return
}
// Gate to a genuinely first/empty index.
if !nodesTableEmpty(ctx, conn) {
_ = conn.Close()
return
}
// Capture prior pragma values so FlushBulk (and every early-return /
// error path) can restore them. If they can't be read, don't engage —
// a slow correct load beats a connection stuck at synchronous=OFF.
prevSync, err := pragmaInt(ctx, conn, "synchronous")
if err != nil {
_ = conn.Close()
return
}
prevCache, err := pragmaInt(ctx, conn, "cache_size")
if err != nil {
_ = conn.Close()
return
}
// synchronous=OFF drops crash durability for the load window —
// acceptable only because a crash on a fresh index just re-indexes.
if _, err := conn.ExecContext(ctx, "PRAGMA synchronous = OFF"); err != nil {
_ = conn.Close()
return
}
if _, err := conn.ExecContext(ctx, fmt.Sprintf("PRAGMA cache_size = %d", bulkCacheSizeKiB)); err != nil {
// Roll the durability change back before bailing.
_, _ = conn.ExecContext(ctx, fmt.Sprintf("PRAGMA synchronous = %d", prevSync))
_ = conn.Close()
return
}
// Drop the droppable secondary indexes; rebuilt in one pass at
// FlushBulk. Best-effort: a failed drop just means that index keeps
// being maintained per-row (slower, still correct), so it is not fatal.
for _, idx := range bulkDroppableIndexes {
_, _ = conn.ExecContext(ctx, "DROP INDEX IF EXISTS "+idx.name)
}
s.bulkConn = conn
s.bulkPrevSync = prevSync
s.bulkPrevCacheSize = prevCache
}
// FlushBulk exits the bulk-load fast path: it rebuilds every index
// BeginBulkLoad dropped, restores synchronous + cache_size on the pinned
// connection, runs one TRUNCATE checkpoint to drain the WAL the no-fsync load
// grew, and returns the connection to the pool. It is a no-op when no fast
// path is active (BeginBulkLoad gated out, or already flushed).
//
// The pragma restore + connection release run unconditionally (defer), so a
// failure mid-rebuild can never leave a connection stuck at synchronous=OFF
// in the pool.
func (s *Store) FlushBulk() error {
s.writeMu.Lock()
defer s.writeMu.Unlock()
conn := s.bulkConn
if conn == nil {
return nil
}
// Detach first: the fast path is over regardless of the outcome below.
s.bulkConn = nil
ctx := context.Background()
defer func() {
// Always restore durability + cache and release the connection,
// even if an index rebuild failed.
_, _ = conn.ExecContext(ctx, fmt.Sprintf("PRAGMA synchronous = %d", s.bulkPrevSync))
_, _ = conn.ExecContext(ctx, fmt.Sprintf("PRAGMA cache_size = %d", s.bulkPrevCacheSize))
_ = conn.Close()
}()
for _, idx := range bulkDroppableIndexes {
if _, err := conn.ExecContext(ctx, idx.ddl); err != nil {
return fmt.Errorf("store_sqlite: rebuild index %s: %w", idx.name, err)
}
}
// Drain the WAL the no-fsync bulk window grew back into the main DB and
// truncate the -wal file. Same TRUNCATE mode as runCheckpointLoop, so it
// cooperates with the journal_size_limit / periodic-checkpoint policy.
if _, err := conn.ExecContext(ctx, "PRAGMA wal_checkpoint(TRUNCATE)"); err != nil {
return fmt.Errorf("store_sqlite: bulk checkpoint: %w", err)
}
return nil
}
// nodesTableEmpty reports whether the nodes table holds no rows. Used to gate
// the bulk-load fast path to a genuinely first/empty cold index.
func nodesTableEmpty(ctx context.Context, conn *sql.Conn) bool {
var one int
err := conn.QueryRowContext(ctx, "SELECT 1 FROM nodes LIMIT 1").Scan(&one)
return errors.Is(err, sql.ErrNoRows)
}
// pragmaInt reads a single-integer PRAGMA (synchronous, cache_size) off the
// given connection.
func pragmaInt(ctx context.Context, conn *sql.Conn, pragma string) (int64, error) {
var v int64
if err := conn.QueryRowContext(ctx, "PRAGMA "+pragma).Scan(&v); err != nil {
return 0, err
}
return v, nil
}
@@ -0,0 +1,388 @@
package store_sqlite
import (
"context"
"database/sql"
"fmt"
"os"
"path/filepath"
"testing"
"time"
"github.com/zzet/gortex/internal/graph"
)
// bulkFixture builds a deterministic node/edge set with distinct ids and
// qual_names (so the UNIQUE nodes_by_qual index never collides) and a mix of
// edge keys (a handful collide → exercise dedup). Input order is intentionally
// not key-sorted.
func bulkFixture(nNodes, nEdges int) ([]*graph.Node, []*graph.Edge) {
nodes := make([]*graph.Node, 0, nNodes)
for i := range nNodes {
nodes = append(nodes, &graph.Node{
ID: fmt.Sprintf("pkg/f%d.go::Sym%d", i%64, i),
Kind: graph.KindFunction,
Name: fmt.Sprintf("Sym%d", i),
QualName: fmt.Sprintf("pkg.f%d.Sym%d", i%64, i),
FilePath: fmt.Sprintf("pkg/f%d.go", i%64),
RepoPrefix: "gortex",
Language: "go",
})
}
edges := make([]*graph.Edge, 0, nEdges)
for i := range nEdges {
from := nodes[i%nNodes]
to := nodes[(i*7+1)%nNodes]
edges = append(edges, &graph.Edge{
From: from.ID,
To: to.ID,
Kind: graph.EdgeCalls,
FilePath: from.FilePath,
Line: i % 500,
Confidence: 1,
})
}
return nodes, edges
}
func openTempStore(t *testing.T) (*Store, string) {
t.Helper()
path := filepath.Join(t.TempDir(), "bulk.sqlite")
s, err := Open(path)
if err != nil {
t.Fatalf("open: %v", err)
}
t.Cleanup(func() { _ = s.Close() })
return s, path
}
// indexNames returns the set of secondary index names present in the schema.
func indexNames(t *testing.T, q interface {
Query(string, ...any) (*sql.Rows, error)
}) map[string]bool {
t.Helper()
rows, err := q.Query("SELECT name FROM sqlite_master WHERE type='index'")
if err != nil {
t.Fatalf("query sqlite_master: %v", err)
}
defer func() { _ = rows.Close() }()
got := map[string]bool{}
for rows.Next() {
var n string
if err := rows.Scan(&n); err != nil {
t.Fatalf("scan index name: %v", err)
}
got[n] = true
}
if err := rows.Err(); err != nil {
t.Fatalf("rows: %v", err)
}
return got
}
func pragmaIntDB(t *testing.T, db *sql.DB, pragma string) int64 {
t.Helper()
var v int64
if err := db.QueryRow("PRAGMA " + pragma).Scan(&v); err != nil {
t.Fatalf("pragma %s: %v", pragma, err)
}
return v
}
func integrityOK(t *testing.T, db *sql.DB) {
t.Helper()
var res string
if err := db.QueryRow("PRAGMA integrity_check").Scan(&res); err != nil {
t.Fatalf("integrity_check: %v", err)
}
if res != "ok" {
t.Fatalf("integrity_check = %q, want \"ok\"", res)
}
}
// TestBulkLoadDropsAndRebuildsIndexes is the core mechanism + restore proof:
// the fast path engages on an empty store, drops the droppable indexes,
// pins a synchronous=OFF connection, then on FlushBulk rebuilds every index,
// restores synchronous, releases the connection, and leaves the DB intact.
func TestBulkLoadDropsAndRebuildsIndexes(t *testing.T) {
s, _ := openTempStore(t)
ctx := context.Background()
// Baseline: all droppable indexes present, synchronous=NORMAL(1).
before := indexNames(t, s.db)
for _, idx := range bulkDroppableIndexes {
if !before[idx.name] {
t.Fatalf("index %s missing before bulk load", idx.name)
}
}
if got := pragmaIntDB(t, s.db, "synchronous"); got != 1 {
t.Fatalf("synchronous before = %d, want 1 (NORMAL)", got)
}
// Engage the fast path on the empty store.
s.BeginBulkLoad()
if s.bulkConn == nil {
t.Fatal("fast path did not engage on empty store")
}
// Read through the pinned connection (it may be the only one when
// GOMAXPROCS/NumCPU is 1) to avoid blocking on connection acquisition.
var sync int64
if err := s.bulkConn.QueryRowContext(ctx, "PRAGMA synchronous").Scan(&sync); err != nil {
t.Fatalf("pinned synchronous: %v", err)
}
if sync != 0 {
t.Fatalf("bulk synchronous = %d, want 0 (OFF)", sync)
}
var cache int64
if err := s.bulkConn.QueryRowContext(ctx, "PRAGMA cache_size").Scan(&cache); err != nil {
t.Fatalf("pinned cache_size: %v", err)
}
if cache != bulkCacheSizeKiB {
t.Fatalf("bulk cache_size = %d, want %d", cache, bulkCacheSizeKiB)
}
// The droppable indexes are gone during the window.
during := indexNames(t, &connQuerier{ctx: ctx, c: s.bulkConn})
for _, idx := range bulkDroppableIndexes {
if during[idx.name] {
t.Fatalf("index %s still present during bulk window", idx.name)
}
}
// nodes_by_qual (UNIQUE, not droppable) must remain live.
if !during["nodes_by_qual"] {
t.Fatal("nodes_by_qual (UNIQUE) must not be dropped")
}
nodes, edges := bulkFixture(2000, 4000)
s.AddBatch(nodes, edges)
if err := s.FlushBulk(); err != nil {
t.Fatalf("FlushBulk: %v", err)
}
if s.bulkConn != nil {
t.Fatal("bulkConn not released after FlushBulk")
}
// Every dropped index is back.
after := indexNames(t, s.db)
for _, idx := range bulkDroppableIndexes {
if !after[idx.name] {
t.Fatalf("index %s not rebuilt after FlushBulk", idx.name)
}
}
// synchronous restored to NORMAL on the pool.
if got := pragmaIntDB(t, s.db, "synchronous"); got != 1 {
t.Fatalf("synchronous after = %d, want 1 (NORMAL)", got)
}
integrityOK(t, s.db)
}
// connQuerier adapts *sql.Conn to the Query signature indexNames expects.
type connQuerier struct {
ctx context.Context
c *sql.Conn
}
func (q *connQuerier) Query(query string, args ...any) (*sql.Rows, error) {
return q.c.QueryContext(q.ctx, query, args...)
}
// TestBulkLoadMatchesNonBulkCounts proves the fast path persists exactly the
// same node/edge counts the plain AddBatch path does.
func TestBulkLoadMatchesNonBulkCounts(t *testing.T) {
nodes, edges := bulkFixture(3000, 6000)
plain, _ := openTempStore(t)
plain.AddBatch(nodes, edges)
wantNodes, wantEdges := plain.NodeCount(), plain.EdgeCount()
bulk, _ := openTempStore(t)
bulk.BeginBulkLoad()
if bulk.bulkConn == nil {
t.Fatal("fast path did not engage on empty store")
}
// Drain in two chunks to mirror the indexer's chunked persist.
bulk.AddBatch(nodes[:1500], nil)
bulk.AddBatch(nodes[1500:], nil)
bulk.AddBatch(nil, edges[:3000])
bulk.AddBatch(nil, edges[3000:])
if err := bulk.FlushBulk(); err != nil {
t.Fatalf("FlushBulk: %v", err)
}
if gotN, gotE := bulk.NodeCount(), bulk.EdgeCount(); gotN != wantNodes || gotE != wantEdges {
t.Fatalf("bulk counts (%d nodes, %d edges) != non-bulk (%d, %d)", gotN, gotE, wantNodes, wantEdges)
}
integrityOK(t, bulk.db)
}
// TestBulkLoadGatedToPopulatedStore confirms the fast path is a safe no-op on
// a store that already holds rows — no indexes are dropped, durability stays.
func TestBulkLoadGatedToPopulatedStore(t *testing.T) {
s, _ := openTempStore(t)
// Populate first (the normal, non-bulk path).
nodes, edges := bulkFixture(50, 100)
s.AddBatch(nodes, edges)
s.BeginBulkLoad()
if s.bulkConn != nil {
t.Fatal("fast path engaged on a populated store; must be a no-op")
}
// Indexes untouched, durability untouched.
present := indexNames(t, s.db)
for _, idx := range bulkDroppableIndexes {
if !present[idx.name] {
t.Fatalf("index %s dropped on a populated store", idx.name)
}
}
if got := pragmaIntDB(t, s.db, "synchronous"); got != 1 {
t.Fatalf("synchronous = %d on populated store, want 1 (NORMAL)", got)
}
if err := s.FlushBulk(); err != nil {
t.Fatalf("FlushBulk no-op returned error: %v", err)
}
}
// TestBulkLoadInMemoryIsNoOp confirms in-memory stores never engage the fast
// path (no WAL / on-disk B-tree to optimise).
func TestBulkLoadInMemoryIsNoOp(t *testing.T) {
s, err := Open(":memory:")
if err != nil {
t.Fatalf("open: %v", err)
}
t.Cleanup(func() { _ = s.Close() })
s.BeginBulkLoad()
if s.bulkConn != nil {
t.Fatal("fast path engaged on an in-memory store")
}
if err := s.FlushBulk(); err != nil {
t.Fatalf("FlushBulk: %v", err)
}
}
// TestBulkLoadWarmRestartLoadsClean bulk-loads, closes, reopens the same file,
// and asserts the persisted graph round-trips: identical counts, indexes
// present, integrity ok.
func TestBulkLoadWarmRestartLoadsClean(t *testing.T) {
path := filepath.Join(t.TempDir(), "warm.sqlite")
nodes, edges := bulkFixture(2500, 5000)
s, err := Open(path)
if err != nil {
t.Fatalf("open: %v", err)
}
s.BeginBulkLoad()
if s.bulkConn == nil {
t.Fatal("fast path did not engage on empty store")
}
s.AddBatch(nodes, edges)
if err := s.FlushBulk(); err != nil {
t.Fatalf("FlushBulk: %v", err)
}
wantNodes, wantEdges := s.NodeCount(), s.EdgeCount()
if err := s.Close(); err != nil {
t.Fatalf("close: %v", err)
}
reopened, err := Open(path)
if err != nil {
t.Fatalf("reopen: %v", err)
}
t.Cleanup(func() { _ = reopened.Close() })
if gotN, gotE := reopened.NodeCount(), reopened.EdgeCount(); gotN != wantNodes || gotE != wantEdges {
t.Fatalf("warm restart counts (%d, %d) != pre-close (%d, %d)", gotN, gotE, wantNodes, wantEdges)
}
present := indexNames(t, reopened.db)
for _, idx := range bulkDroppableIndexes {
if !present[idx.name] {
t.Fatalf("index %s missing after warm restart", idx.name)
}
}
integrityOK(t, reopened.db)
// A populated store on reopen must NOT engage the fast path.
reopened.BeginBulkLoad()
if reopened.bulkConn != nil {
t.Fatal("fast path engaged on warm restart (populated store)")
}
_ = reopened.FlushBulk()
}
// TestBulkLoadPersistSpeed is the persist-speed evidence: it times the plain
// path vs the fast path on the same fixture and logs both. It asserts
// correctness and that the fast path is not pathologically slower; a strict
// speedup ratio is gated behind GORTEX_BULK_PERF_ASSERT so the default run
// stays deterministic on noisy CI.
func TestBulkLoadPersistSpeed(t *testing.T) {
if testing.Short() {
t.Skip("skipping persist-speed timing in -short")
}
n, e := 8000, 16000
nodes, edges := bulkFixture(n, e)
plain, _ := openTempStore(t)
t0 := time.Now()
plain.AddBatch(nodes, edges)
plainDur := time.Since(t0)
bulk, _ := openTempStore(t)
t1 := time.Now()
bulk.BeginBulkLoad()
bulk.AddBatch(nodes, edges)
if err := bulk.FlushBulk(); err != nil {
t.Fatalf("FlushBulk: %v", err)
}
bulkDur := time.Since(t1)
if bulk.NodeCount() != plain.NodeCount() || bulk.EdgeCount() != plain.EdgeCount() {
t.Fatalf("count mismatch: bulk(%d,%d) plain(%d,%d)",
bulk.NodeCount(), bulk.EdgeCount(), plain.NodeCount(), plain.EdgeCount())
}
integrityOK(t, bulk.db)
ratio := float64(plainDur) / float64(bulkDur)
t.Logf("persist %d nodes / %d edges: plain=%s bulk=%s speedup=%.2fx",
n, e, plainDur, bulkDur, ratio)
// Sanity floor: the fast path must never be dramatically slower.
if bulkDur > plainDur*5 {
t.Fatalf("fast path far slower: plain=%s bulk=%s", plainDur, bulkDur)
}
if os.Getenv("GORTEX_BULK_PERF_ASSERT") != "" && ratio < 2.0 {
t.Fatalf("fast path speedup %.2fx below 2x target", ratio)
}
}
// BenchmarkPersistFixture is reproducible persist-speed evidence: run with
//
// go test -run=^$ -bench=BenchmarkPersistFixture ./internal/graph/store_sqlite/
//
// to compare the plain AddBatch path against the bulk-load fast path.
func BenchmarkPersistFixture(b *testing.B) {
nodes, edges := bulkFixture(50000, 100000)
run := func(b *testing.B, bulk bool) {
for i := 0; i < b.N; i++ {
b.StopTimer()
path := filepath.Join(b.TempDir(), fmt.Sprintf("p%d.sqlite", i))
s, err := Open(path)
if err != nil {
b.Fatalf("open: %v", err)
}
b.StartTimer()
if bulk {
s.BeginBulkLoad()
s.AddBatch(nodes, edges)
if err := s.FlushBulk(); err != nil {
b.Fatalf("FlushBulk: %v", err)
}
} else {
s.AddBatch(nodes, edges)
}
b.StopTimer()
_ = s.Close()
}
}
b.Run("nonbulk", func(b *testing.B) { run(b, false) })
b.Run("bulk", func(b *testing.B) { run(b, true) })
}
+320
View File
@@ -0,0 +1,320 @@
package store_sqlite
import (
"os"
"path/filepath"
"strconv"
"strings"
"sync"
"github.com/zzet/gortex/internal/graph"
)
// bundleCacheDefaultMaxBytes bounds the total heap the bundle cache may
// retain across all cached entries. A count ceiling alone is unsafe: an
// entry holds a decoded node plus its full in/out edge lists, and both
// nodes and edges carry meta maps, so entry sizes span ~1 KB for a leaf
// symbol to multiple MB for a hub node with thousands of edges. A cap
// measured in entries therefore admits an unbounded BYTE footprint — a
// few thousand hub bundles can pin gigabytes. This cache serves point
// lookups on the symbol-search hot path; it is a latency optimisation,
// not a working set that needs to be resident, so a modest budget is
// right — 64 MiB holds the hot few thousand ordinary bundles while
// keeping a long-lived daemon's idle heap bounded. Override with
// GORTEX_BUNDLE_CACHE_MAX_MB=<n> (n <= 0 disables the cache entirely).
const bundleCacheDefaultMaxBytes = 64 << 20 // 64 MiB
// bundleCacheMaxEntries is a secondary, generous count ceiling kept
// alongside the byte budget. The byte budget is the primary bound; this
// guards the map's own structural overhead in the degenerate case of a
// flood of tiny entries (a bucket slot and pointers per entry are not
// fully reflected in a per-entry byte estimate), and keeps the
// wholesale-clear allocation predictable. It is deliberately loose: a
// half-million-symbol monorepo's hottest few thousand search hits fit
// far under it, so in normal operation the byte budget always trips
// first.
const bundleCacheMaxEntries = 50000
const (
// bundleEntryOverhead is a coarse fixed charge per cached entry that
// is independent of the bundle's string content: the bundleCacheEntry
// wrapper, the *entry and *Node pointers, the graph.Node value's flat
// struct (its string / slice / map headers, ints, and embedded
// time.Time), and the map bucket the node id occupies. String and map
// *contents* are added on top. Over-estimating here only makes the
// cache clear sooner; it never lets the footprint overshoot the budget.
bundleEntryOverhead = 448
// bundleEdgeOverhead is the coarse fixed charge for one *Edge in an
// in/out slice: the pointer, the slice slot, and the Edge value's flat
// struct. Edge string / map contents are added separately.
bundleEdgeOverhead = 240
// bundleMetaEntryOverhead is the fixed per-key charge for a
// map[string]any entry (bucket slot + interface header); the key
// length and any string value length are added on top.
bundleMetaEntryOverhead = 48
)
// bundleCacheEntry is one node's cached bundle, tagged with the package
// it belongs to and the package fingerprint that was current when the
// bundle was computed. The entry is served only while
// fingerprints[pkgKey] still equals fp — any change to the package's
// content (a node or edge added / removed / reweighted, including a
// cross-file edge that lands on this node from elsewhere) moves the
// fingerprint and forces a recompute, so a cached bundle can never
// carry a stale edge.
type bundleCacheEntry struct {
pkgKey string
fp uint64
bundle graph.SymbolBundle
// bytes is the entry's estimated retained size, recorded at insert so
// the running byte total can be adjusted in O(1) whenever the entry is
// dropped (invalidation or a stale read).
bytes int64
}
// bundleCache is a content-addressed, package-scoped cache over
// SearchSymbolBundles. It is keyed at the node level but validated at
// the package level: an entry is fresh exactly when the package's
// current fingerprint matches the fingerprint the entry was stored at.
//
// Correctness rests entirely on the fingerprint discipline: the daemon
// hands the cache an authoritative per-package fingerprint map after
// every analysis pass (which runs after every incremental reindex and
// every edit_file / fsnotify-driven graph mutation). The fingerprints
// are edge-aware — they fold every package's nodes AND the edges
// touching them — so any mutation that could change a cached bundle's
// in/out edges moves the relevant package fingerprint and invalidates
// the entry. A package whose fingerprint is unchanged is served from
// cache; a package the daemon has never reported a fingerprint for is
// always treated as a miss (conservative: never serve an unvalidated
// bundle).
//
// The cache is bounded by bytes (maxBytes), not by entry count, because
// entry sizes vary by orders of magnitude with a node's edge fan-out and
// meta size. maxEntries is a secondary count ceiling only. When either
// bound would be exceeded the cache is cleared wholesale rather than
// evicting individually: entries are cheap to recompute (one batched
// fetch), and a wholesale clear keeps the bookkeeping O(1) and free of an
// LRU's per-entry ordering overhead. maxBytes <= 0 disables the cache —
// stores become no-ops and every lookup misses (reads still recompute
// live through the caller's fallback path).
type bundleCache struct {
mu sync.Mutex
fingerprints map[string]uint64
entries map[string]*bundleCacheEntry
maxBytes int64 // byte budget (primary bound); <= 0 disables the cache
maxEntries int // count ceiling (secondary bound)
curBytes int64 // running sum of entries' estimated bytes
}
// newBundleCache builds an empty cache with the default budgets. The byte
// budget is overridable with GORTEX_BUNDLE_CACHE_MAX_MB=<n>; n <= 0
// disables the cache. It starts inert (every lookup a miss) until the
// daemon supplies fingerprints.
func newBundleCache() *bundleCache {
return &bundleCache{
fingerprints: map[string]uint64{},
entries: map[string]*bundleCacheEntry{},
maxBytes: bundleCacheMaxBytes(),
maxEntries: bundleCacheMaxEntries,
}
}
// bundleCacheMaxBytes resolves the byte budget from the environment,
// falling back to the default. GORTEX_BUNDLE_CACHE_MAX_MB is read in
// mebibytes; a value <= 0 returns 0 to disable the cache, and an
// unparseable value is ignored (keeps the default).
func bundleCacheMaxBytes() int64 {
if v := strings.TrimSpace(os.Getenv("GORTEX_BUNDLE_CACHE_MAX_MB")); v != "" {
if n, err := strconv.Atoi(v); err == nil {
if n <= 0 {
return 0
}
return int64(n) << 20
}
}
return bundleCacheDefaultMaxBytes
}
// bundleEntryBytes conservatively estimates a bundle's retained heap for
// the byte budget: a fixed per-entry charge plus the node's string and
// meta contents plus each in/out edge's fixed charge and its string and
// meta contents. Computed once at insert so the overflow check is a cheap
// scalar comparison.
func bundleEntryBytes(b graph.SymbolBundle) int64 {
n := int64(bundleEntryOverhead)
if b.Node != nil {
n += nodeStringBytes(b.Node)
n += metaBytes(b.Node.Meta)
}
for _, e := range b.InEdges {
n += edgeBytes(e)
}
for _, e := range b.OutEdges {
n += edgeBytes(e)
}
return n
}
// nodeStringBytes sums the byte lengths of a node's string fields (its
// heap-backed content, on top of the fixed struct overhead counted in
// bundleEntryOverhead).
func nodeStringBytes(nd *graph.Node) int64 {
return int64(len(nd.ID) + len(nd.Name) + len(nd.QualName) + len(nd.FilePath) +
len(string(nd.Kind)) + len(nd.Language) + len(nd.RepoPrefix) +
len(nd.WorkspaceID) + len(nd.ProjectID) + len(nd.AbsoluteFilePath) +
len(nd.Origin))
}
// edgeBytes estimates one edge's retained heap: the fixed per-edge charge
// plus its string fields and meta contents.
func edgeBytes(e *graph.Edge) int64 {
if e == nil {
return bundleEdgeOverhead
}
n := int64(bundleEdgeOverhead)
n += int64(len(e.From) + len(e.To) + len(string(e.Kind)) + len(e.FilePath) +
len(e.ConfidenceLabel) + len(e.Origin) + len(e.Tier) + len(e.Context) +
len(e.ReturnUsage) + len(e.Via) + len(e.Alias))
n += metaBytes(e.Meta)
return n
}
// metaBytes estimates a meta map's retained heap: a fixed charge per key
// plus the key length and, for string values, the value length. Non-string
// values fold into the fixed charge — meta values are overwhelmingly short
// scalars, and a coarse estimate only over-counts, which is safe.
func metaBytes(m map[string]any) int64 {
if len(m) == 0 {
return 0
}
var n int64
for k, v := range m {
n += int64(len(k) + bundleMetaEntryOverhead)
if s, ok := v.(string); ok {
n += int64(len(s))
}
}
return n
}
// SetBundleFingerprints installs the authoritative per-package
// fingerprint map and drops any cached entry whose package fingerprint
// has changed (or whose package is no longer reported). This is the
// invalidation entry point: the daemon calls it after each analysis
// pass with the fresh fingerprints derived from the live graph, so a
// reindex that altered a package's nodes or edges retires exactly the
// affected bundles while leaving untouched packages cached.
//
// fps is keyed by package key (the directory the package's files live
// in, repo-prefixed in multi-repo because the node file paths are).
func (s *Store) SetBundleFingerprints(fps map[string]uint64) {
if s.bundles == nil {
return
}
s.bundles.refresh(fps)
}
// refresh swaps in the new fingerprint map and prunes every entry whose
// package fingerprint no longer matches, decrementing the running byte
// total by each dropped entry's estimated size.
func (c *bundleCache) refresh(fps map[string]uint64) {
c.mu.Lock()
defer c.mu.Unlock()
if fps == nil {
fps = map[string]uint64{}
}
c.fingerprints = fps
for id, e := range c.entries {
cur, ok := fps[e.pkgKey]
if !ok || cur != e.fp {
delete(c.entries, id)
c.curBytes -= e.bytes
}
}
}
// bundlePackageKey derives the package key for a node's file path. It
// mirrors the analysis layer's packageKey so the cache and the
// daemon-supplied fingerprint map agree on package identity: the
// directory the file lives in (repo-prefixed in multi-repo because the
// stored file paths are), or "" for a file at the repo root / a node
// with no path.
func bundlePackageKey(filePath string) string {
if filePath == "" {
return ""
}
dir := filepath.Dir(filepath.ToSlash(filePath))
if dir == "." {
return ""
}
return dir
}
// lookup returns the cached bundle for id when it is fresh — the entry
// exists and its package fingerprint still matches the current one. A
// node whose package has no reported fingerprint is never served (ok is
// false) so an unvalidated bundle can never escape the cache. A stale
// entry is dropped in place and its bytes reclaimed.
func (c *bundleCache) lookup(id string) (graph.SymbolBundle, bool) {
c.mu.Lock()
defer c.mu.Unlock()
e, ok := c.entries[id]
if !ok {
return graph.SymbolBundle{}, false
}
cur, ok := c.fingerprints[e.pkgKey]
if !ok || cur != e.fp {
// Stale or unvalidated — drop it so a later refresh doesn't
// have to, and reclaim its bytes.
delete(c.entries, id)
c.curBytes -= e.bytes
return graph.SymbolBundle{}, false
}
return e.bundle, true
}
// store records a freshly computed bundle, tagged with its package's
// current fingerprint. A node whose package has no reported fingerprint
// is NOT cached (it could not be validated on read-back), keeping the
// cache conservative. The cache is bounded by bytes: when admitting the
// new entry would push the running total over the byte budget (or the
// count over the secondary ceiling) the cache is cleared wholesale
// before the insert. A single bundle that on its own exceeds the whole
// budget — a hub node with thousands of edges, exactly the pathological
// case a byte cap exists to keep out of long-lived memory — is refused
// outright rather than pinned. With maxBytes <= 0 the cache is disabled
// and every store is a no-op.
func (c *bundleCache) store(b graph.SymbolBundle) {
if b.Node == nil {
return
}
pkgKey := bundlePackageKey(b.Node.FilePath)
c.mu.Lock()
defer c.mu.Unlock()
if c.maxBytes <= 0 {
return
}
fp, ok := c.fingerprints[pkgKey]
if !ok {
return
}
sz := bundleEntryBytes(b)
if sz > c.maxBytes {
// One entry larger than the entire budget would blow the bound and
// be evicted by the very next insert's wholesale clear anyway.
return
}
if old, ok := c.entries[b.Node.ID]; ok {
// Replacing an existing entry — discount its bytes and drop it so
// curBytes and the count check track the live set.
c.curBytes -= old.bytes
delete(c.entries, b.Node.ID)
}
if len(c.entries) > 0 && (c.curBytes+sz > c.maxBytes || len(c.entries) >= c.maxEntries) {
c.entries = make(map[string]*bundleCacheEntry)
c.curBytes = 0
}
c.entries[b.Node.ID] = &bundleCacheEntry{pkgKey: pkgKey, fp: fp, bundle: b, bytes: sz}
c.curBytes += sz
}
@@ -0,0 +1,390 @@
package store_sqlite
import (
"fmt"
"path/filepath"
"sync"
"testing"
"github.com/zzet/gortex/internal/graph"
)
func mkFnNode(id, name, file string) *graph.Node {
return &graph.Node{ID: id, Kind: graph.KindFunction, Name: name, FilePath: file, Language: "go"}
}
// newTestBundleCache builds a cache with the default byte budget without
// consulting the environment, so the fingerprint / invalidation unit tests
// stay hermetic regardless of GORTEX_BUNDLE_CACHE_MAX_MB.
func newTestBundleCache() *bundleCache {
return &bundleCache{
fingerprints: map[string]uint64{},
entries: map[string]*bundleCacheEntry{},
maxBytes: bundleCacheDefaultMaxBytes,
maxEntries: bundleCacheMaxEntries,
}
}
// --- unit tests over the cache logic in isolation ---
func TestBundleCache_ServesOnlyValidatedFingerprints(t *testing.T) {
c := newTestBundleCache()
b := graph.SymbolBundle{Node: mkFnNode("pkg/x.go::A", "A", "pkg/x.go")}
// No fingerprint reported for the package yet -> store is a no-op
// (conservative: never cache an unvalidated bundle).
c.store(b)
if _, ok := c.lookup("pkg/x.go::A"); ok {
t.Fatal("bundle was cached despite no package fingerprint")
}
// Report a fingerprint, then store: now it caches and serves.
c.refresh(map[string]uint64{"pkg": 100})
c.store(b)
if _, ok := c.lookup("pkg/x.go::A"); !ok {
t.Fatal("bundle should be served once its package fingerprint is known")
}
}
func TestBundleCache_InvalidatesOnFingerprintChange(t *testing.T) {
c := newTestBundleCache()
c.refresh(map[string]uint64{"pkg": 1})
c.store(graph.SymbolBundle{Node: mkFnNode("pkg/x.go::A", "A", "pkg/x.go")})
if _, ok := c.lookup("pkg/x.go::A"); !ok {
t.Fatal("expected a cache hit on the unchanged fingerprint")
}
// Fingerprint changes -> the entry is invalidated.
c.refresh(map[string]uint64{"pkg": 2})
if _, ok := c.lookup("pkg/x.go::A"); ok {
t.Fatal("entry must be dropped when its package fingerprint changes")
}
}
func TestBundleCache_CrossRepoIsolation(t *testing.T) {
c := newTestBundleCache()
// Two repos with the same inner directory name resolve to DIFFERENT
// package keys because the stored file paths are repo-prefixed.
c.refresh(map[string]uint64{
"repoA/pkg": 10,
"repoB/pkg": 20,
})
c.store(graph.SymbolBundle{Node: mkFnNode("repoA/pkg/x.go::A", "A", "repoA/pkg/x.go")})
c.store(graph.SymbolBundle{Node: mkFnNode("repoB/pkg/x.go::A", "A", "repoB/pkg/x.go")})
// Bumping only repoA's fingerprint must not touch repoB's entry.
c.refresh(map[string]uint64{
"repoA/pkg": 11,
"repoB/pkg": 20,
})
if _, ok := c.lookup("repoA/pkg/x.go::A"); ok {
t.Fatal("repoA entry should have been invalidated")
}
if _, ok := c.lookup("repoB/pkg/x.go::A"); !ok {
t.Fatal("repoB entry must survive a repoA-only fingerprint bump")
}
}
func TestBundlePackageKey(t *testing.T) {
cases := map[string]string{
"pkg/sub/x.go": "pkg/sub",
"x.go": "",
"": "",
"repo/a/b.go": "repo/a",
}
for in, want := range cases {
if got := bundlePackageKey(in); got != want {
t.Errorf("bundlePackageKey(%q) = %q, want %q", in, got, want)
}
}
}
// --- integration tests through the store's SearchSymbolBundles ---
func newBundleTestStore(t *testing.T) *Store {
t.Helper()
s, err := Open(filepath.Join(t.TempDir(), "b.sqlite"))
if err != nil {
t.Fatalf("Open: %v", err)
}
t.Cleanup(func() { _ = s.Close() })
return s
}
func seedBundleStore(t *testing.T, s *Store) {
t.Helper()
s.AddNode(mkFnNode("pkg/x.go::A", "AlphaWidget", "pkg/x.go"))
s.AddNode(mkFnNode("pkg/x.go::B", "BetaWidget", "pkg/x.go"))
s.AddEdge(&graph.Edge{From: "pkg/x.go::A", To: "pkg/x.go::B", Kind: graph.EdgeCalls, FilePath: "pkg/x.go"})
items := []graph.SymbolFTSItem{
{NodeID: "pkg/x.go::A", Tokens: "alpha widget"},
{NodeID: "pkg/x.go::B", Tokens: "beta widget"},
}
if err := s.BulkUpsertSymbolFTS("", items); err != nil {
t.Fatalf("BulkUpsertSymbolFTS: %v", err)
}
if err := s.BuildSymbolIndex(); err != nil {
t.Fatalf("BuildSymbolIndex: %v", err)
}
}
func bundleByID(bundles []graph.SymbolBundle) map[string]graph.SymbolBundle {
out := make(map[string]graph.SymbolBundle, len(bundles))
for _, b := range bundles {
if b.Node != nil {
out[b.Node.ID] = b
}
}
return out
}
func TestSearchSymbolBundles_CacheHitOnUnchangedFingerprint(t *testing.T) {
s := newBundleTestStore(t)
seedBundleStore(t, s)
// Report a fingerprint so the first query populates the cache.
s.SetBundleFingerprints(map[string]uint64{"pkg": 1})
first, err := s.SearchSymbolBundles("widget", 10)
if err != nil {
t.Fatalf("first SearchSymbolBundles: %v", err)
}
got := bundleByID(first)
if b, ok := got["pkg/x.go::A"]; !ok || len(b.OutEdges) != 1 {
t.Fatalf("expected A with 1 out-edge on first query, got %+v", got["pkg/x.go::A"])
}
// Mutate the graph WITHOUT bumping the fingerprint: add a second
// out-edge from A. A correct content-addressed cache serves the
// STALE (1-edge) bundle because the fingerprint is unchanged — proof
// the bundle came from cache, not a fresh fetch.
s.AddNode(mkFnNode("pkg/x.go::C", "GammaWidget", "pkg/x.go"))
s.AddEdge(&graph.Edge{From: "pkg/x.go::A", To: "pkg/x.go::C", Kind: graph.EdgeCalls, FilePath: "pkg/x.go"})
second, err := s.SearchSymbolBundles("widget", 10)
if err != nil {
t.Fatalf("second SearchSymbolBundles: %v", err)
}
cached := bundleByID(second)["pkg/x.go::A"]
if len(cached.OutEdges) != 1 {
t.Fatalf("expected the cached 1-edge bundle to be served on an unchanged fingerprint, got %d edges",
len(cached.OutEdges))
}
}
func TestSearchSymbolBundles_MissAndRecomputeOnFingerprintChange(t *testing.T) {
s := newBundleTestStore(t)
seedBundleStore(t, s)
s.SetBundleFingerprints(map[string]uint64{"pkg": 1})
if _, err := s.SearchSymbolBundles("widget", 10); err != nil {
t.Fatalf("warm-up query: %v", err)
}
// Add a real out-edge, then bump the package fingerprint to signal
// the content changed. The next query must recompute and surface the
// new edge.
s.AddNode(mkFnNode("pkg/x.go::C", "GammaWidget", "pkg/x.go"))
s.AddEdge(&graph.Edge{From: "pkg/x.go::A", To: "pkg/x.go::C", Kind: graph.EdgeCalls, FilePath: "pkg/x.go"})
s.SetBundleFingerprints(map[string]uint64{"pkg": 2})
after, err := s.SearchSymbolBundles("widget", 10)
if err != nil {
t.Fatalf("post-invalidation query: %v", err)
}
fresh := bundleByID(after)["pkg/x.go::A"]
if len(fresh.OutEdges) != 2 {
t.Fatalf("expected the recomputed 2-edge bundle after a fingerprint bump, got %d edges",
len(fresh.OutEdges))
}
}
func TestSearchSymbolBundles_UncachedWithoutFingerprints(t *testing.T) {
s := newBundleTestStore(t)
seedBundleStore(t, s)
// No SetBundleFingerprints call -> the cache stays inert and every
// query recomputes live. Adding an edge must show up immediately.
first, err := s.SearchSymbolBundles("widget", 10)
if err != nil {
t.Fatalf("first query: %v", err)
}
if got := bundleByID(first)["pkg/x.go::A"]; len(got.OutEdges) != 1 {
t.Fatalf("expected 1 edge live, got %d", len(got.OutEdges))
}
s.AddNode(mkFnNode("pkg/x.go::C", "GammaWidget", "pkg/x.go"))
s.AddEdge(&graph.Edge{From: "pkg/x.go::A", To: "pkg/x.go::C", Kind: graph.EdgeCalls, FilePath: "pkg/x.go"})
second, err := s.SearchSymbolBundles("widget", 10)
if err != nil {
t.Fatalf("second query: %v", err)
}
if got := bundleByID(second)["pkg/x.go::A"]; len(got.OutEdges) != 2 {
t.Fatalf("uncached path must reflect the new edge live, got %d", len(got.OutEdges))
}
}
// --- byte-budget tests ---
func TestBundleCache_ByteBudgetEvictionAtBoundary(t *testing.T) {
c := newTestBundleCache()
c.refresh(map[string]uint64{"pkg": 1})
// Fixed-width ids so every entry estimates to the same size.
mk := func(i int) graph.SymbolBundle {
return graph.SymbolBundle{Node: mkFnNode(fmt.Sprintf("pkg/x.go::N%03d", i), "W", "pkg/x.go")}
}
unit := bundleEntryBytes(mk(0))
const k = 4
c.maxBytes = unit * k // budget holds exactly k entries
for i := 0; i < k; i++ {
c.store(mk(i))
}
if len(c.entries) != k {
t.Fatalf("expected %d entries filling the budget, got %d", k, len(c.entries))
}
if c.curBytes != unit*k {
t.Fatalf("curBytes = %d, want %d", c.curBytes, unit*k)
}
// One more entry crosses the budget -> wholesale clear, only the newest
// survives and the byte total resets to a single unit.
c.store(mk(k))
if len(c.entries) != 1 {
t.Fatalf("crossing the budget must clear wholesale to 1 entry, got %d", len(c.entries))
}
if c.curBytes != unit {
t.Fatalf("curBytes after clear = %d, want %d", c.curBytes, unit)
}
if _, ok := c.lookup(fmt.Sprintf("pkg/x.go::N%03d", k)); !ok {
t.Fatal("the entry that triggered the clear must remain served")
}
if _, ok := c.lookup("pkg/x.go::N000"); ok {
t.Fatal("a pre-clear entry must be gone after the wholesale clear")
}
}
func TestBundleCache_RefusesEntryLargerThanBudget(t *testing.T) {
c := newTestBundleCache()
c.refresh(map[string]uint64{"pkg": 1})
b := graph.SymbolBundle{Node: mkFnNode("pkg/x.go::A", "A", "pkg/x.go")}
c.maxBytes = bundleEntryBytes(b) - 1 // budget just below a single entry
c.store(b)
if _, ok := c.lookup("pkg/x.go::A"); ok {
t.Fatal("an entry larger than the whole budget must not be cached")
}
if len(c.entries) != 0 || c.curBytes != 0 {
t.Fatalf("oversized store must leave the cache empty, got %d entries / %d bytes",
len(c.entries), c.curBytes)
}
}
func TestBundleCacheMaxBytes_EnvOverride(t *testing.T) {
t.Setenv("GORTEX_BUNDLE_CACHE_MAX_MB", "128")
if got := bundleCacheMaxBytes(); got != 128<<20 {
t.Fatalf("env override = %d, want %d", got, 128<<20)
}
if c := newBundleCache(); c.maxBytes != 128<<20 {
t.Fatalf("newBundleCache maxBytes = %d, want %d", c.maxBytes, 128<<20)
}
// Empty and unparseable values keep the default.
t.Setenv("GORTEX_BUNDLE_CACHE_MAX_MB", "")
if got := bundleCacheMaxBytes(); got != bundleCacheDefaultMaxBytes {
t.Fatalf("empty override should keep the default, got %d", got)
}
t.Setenv("GORTEX_BUNDLE_CACHE_MAX_MB", "not-a-number")
if got := bundleCacheMaxBytes(); got != bundleCacheDefaultMaxBytes {
t.Fatalf("unparseable override should keep the default, got %d", got)
}
}
func TestBundleCache_DisabledMode(t *testing.T) {
t.Setenv("GORTEX_BUNDLE_CACHE_MAX_MB", "0")
c := newBundleCache()
if c.maxBytes != 0 {
t.Fatalf("expected a disabled cache (maxBytes 0), got %d", c.maxBytes)
}
c.refresh(map[string]uint64{"pkg": 1})
c.store(graph.SymbolBundle{Node: mkFnNode("pkg/x.go::A", "A", "pkg/x.go")})
if len(c.entries) != 0 {
t.Fatalf("a disabled cache must not store, got %d entries", len(c.entries))
}
if _, ok := c.lookup("pkg/x.go::A"); ok {
t.Fatal("a disabled cache must always miss")
}
// A negative budget disables too.
t.Setenv("GORTEX_BUNDLE_CACHE_MAX_MB", "-4")
if got := bundleCacheMaxBytes(); got != 0 {
t.Fatalf("a negative override should disable the cache (0), got %d", got)
}
}
func TestSearchSymbolBundles_DisabledCacheStillServes(t *testing.T) {
t.Setenv("GORTEX_BUNDLE_CACHE_MAX_MB", "0")
s := newBundleTestStore(t)
seedBundleStore(t, s)
s.SetBundleFingerprints(map[string]uint64{"pkg": 1})
res, err := s.SearchSymbolBundles("widget", 10)
if err != nil {
t.Fatalf("SearchSymbolBundles with the cache disabled: %v", err)
}
if b, ok := bundleByID(res)["pkg/x.go::A"]; !ok || len(b.OutEdges) != 1 {
t.Fatalf("a disabled cache must still return live bundles, got %+v", b)
}
if s.bundles.maxBytes != 0 {
t.Fatalf("expected the store's cache disabled, got maxBytes %d", s.bundles.maxBytes)
}
if len(s.bundles.entries) != 0 {
t.Fatalf("a disabled cache must stay empty, got %d entries", len(s.bundles.entries))
}
}
func TestBundleCache_ConcurrentReadInsert(t *testing.T) {
c := newTestBundleCache()
c.maxBytes = 8 << 10 // small budget so wholesale clears fire under contention
c.refresh(map[string]uint64{"pkg": 1})
const workers = 8
const iters = 3000
var wg sync.WaitGroup
wg.Add(workers)
for w := 0; w < workers; w++ {
go func(w int) {
defer wg.Done()
for i := 0; i < iters; i++ {
id := fmt.Sprintf("pkg/x.go::N%d_%d", w, i%64)
switch i % 3 {
case 0:
c.store(graph.SymbolBundle{Node: mkFnNode(id, "W", "pkg/x.go")})
case 1:
_, _ = c.lookup(id)
default:
c.refresh(map[string]uint64{"pkg": uint64(i)})
}
}
}(w)
}
wg.Wait()
// No goroutines remain: the running total must exactly equal the summed
// bytes of the surviving entries (the accounting invariant), which also
// proves it never drifted negative under contention.
var sum int64
for _, e := range c.entries {
sum += e.bytes
}
if c.curBytes != sum {
t.Fatalf("curBytes %d != sum of live entry bytes %d", c.curBytes, sum)
}
if c.curBytes > c.maxBytes {
t.Fatalf("curBytes %d exceeds the byte budget %d", c.curBytes, c.maxBytes)
}
}
@@ -0,0 +1,69 @@
package store_sqlite_test
import (
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/zzet/gortex/internal/graph"
)
// TestClosedStoreReadsDoNotPanic pins the teardown-race fix: after Close()
// has shut the store (daemon shutdown / restart / store swap), an in-flight
// reader — e.g. a deferred parallel-enrich goroutine still holding a cached
// *sql.Stmt — must degrade to an empty result, never panic the whole daemon.
// Before the fix this surfaced as `panic: store_sqlite: sql: statement is
// closed` from GetNode under runDeferredEnrichParallel.
func TestClosedStoreReadsDoNotPanic(t *testing.T) {
s := openTestStore(t)
s.AddNode(&graph.Node{ID: "p/a.go::Foo", Kind: graph.KindType, Name: "Foo", FilePath: "p/a.go"})
require.NotNil(t, s.GetNode("p/a.go::Foo"), "sanity: node readable before close")
require.NoError(t, s.Close())
assert.NotPanics(t, func() {
assert.Nil(t, s.GetNode("p/a.go::Foo"))
assert.Empty(t, s.FindNodesByName("Foo"))
assert.Empty(t, s.GetFileNodes("p/a.go"))
}, "reads after Close must degrade gracefully, not panic")
}
// TestClosedStoreAggregatorsDoNotPanic pins the aggregator teardown-race sweep:
// each aggregator read runs its Query error through panicOnFatal, which
// swallows the "database is closed" race — leaving rows == nil. Every such site
// must early-return its empty value instead of dereferencing nil rows. The live
// crash was NodeIDsByKinds (FindHotspots -> RunAnalysis at watch-start) SIGSEGV
// on nil rows right after a long warmup. Each method is called with non-empty
// args so it actually reaches the Query rather than an early argument guard.
func TestClosedStoreAggregatorsDoNotPanic(t *testing.T) {
s := openTestStore(t)
s.AddNode(&graph.Node{ID: "p/a.go::Foo", Kind: graph.KindType, Name: "Foo", FilePath: "p/a.go"})
s.AddNode(&graph.Node{ID: "p/a.go::bar", Kind: graph.KindFunction, Name: "bar", FilePath: "p/a.go"})
s.AddEdge(&graph.Edge{From: "p/a.go::bar", To: "p/a.go::Foo", Kind: graph.EdgeReferences, FilePath: "p/a.go", Line: 1})
require.NoError(t, s.Close())
nodeKinds := []graph.NodeKind{graph.KindType, graph.KindFunction}
edgeKinds := []graph.EdgeKind{graph.EdgeReferences}
ids := []string{"p/a.go::Foo", "p/a.go::bar"}
assert.NotPanics(t, func() {
assert.Empty(t, s.InEdgeCountsByKind(edgeKinds))
assert.Empty(t, s.NodeIDsByKinds(nodeKinds))
assert.Empty(t, s.EdgeKindCounts())
assert.Empty(t, s.NodeDegreeByKinds(nodeKinds, ""))
assert.Empty(t, s.FileImportCounts(nil)) // exercises aggScanImportCounts
assert.Empty(t, s.InDegreeForNodes(ids))
assert.Empty(t, s.CrossRepoEdgeCounts())
assert.Empty(t, s.FileImporters("p/a.go"))
assert.Empty(t, s.FileSymbolNamesByPaths([]string{"p/a.go"}, nodeKinds))
assert.Empty(t, s.NodeDegreeCounts(ids, edgeKinds))
assert.Empty(t, s.NodeFanCounts(ids, edgeKinds, edgeKinds))
assert.Empty(t, s.CommunityCrossingsByKind(edgeKinds, map[string]string{"p/a.go::bar": "c0"}))
// Iterator-shaped: the Query runs inside the yield closure.
n := 0
for range s.EdgeAdjacencyForKinds(edgeKinds, nodeKinds) {
n++
}
assert.Zero(t, n)
}, "aggregator reads after Close must degrade to empty, not panic")
}
@@ -0,0 +1,46 @@
package store_sqlite
import (
"path/filepath"
"testing"
"github.com/stretchr/testify/require"
"github.com/zzet/gortex/internal/graph"
)
// TestGetRepoNonContentNodes verifies the SQL-level content filter (which
// json_extracts data_class out of the JSON meta blob) drops only content
// section nodes — keeping code, markdown prose, and data assets — so the
// code passes can enumerate without materialising content sections.
func TestGetRepoNonContentNodes(t *testing.T) {
s, err := Open(filepath.Join(t.TempDir(), "n.sqlite"))
require.NoError(t, err)
t.Cleanup(func() { _ = s.Close() })
s.AddBatch([]*graph.Node{
{ID: "code1", Kind: graph.KindFunction, Name: "Foo", RepoPrefix: "r"},
{ID: "content1", Kind: graph.KindDoc, Name: "doc.txt::0", RepoPrefix: "r",
Meta: map[string]any{"data_class": "content", "section_text": "x"}},
{ID: "prose1", Kind: graph.KindDoc, Name: "README.md::0", RepoPrefix: "r",
Meta: map[string]any{"asset_kind": "markdown_section"}},
{ID: "data1", Kind: graph.KindFile, Name: "x.parquet", RepoPrefix: "r",
Meta: map[string]any{"data_class": "data"}},
}, nil)
// Runtime assertion the store satisfies the optional capability.
var cr graph.NonContentNodeReader = s
ids := map[string]bool{}
for _, n := range cr.GetRepoNonContentNodes("r") {
ids[n.ID] = true
}
require.True(t, ids["code1"], "code node kept")
require.True(t, ids["prose1"], "markdown prose kept (not data_class=content)")
require.True(t, ids["data1"], "data asset kept (data_class=data, not content)")
require.False(t, ids["content1"], "content section dropped at the SQL level")
require.Len(t, ids, 3)
// Empty prefix spans all repos (still drops content).
require.Len(t, s.GetRepoNonContentNodes(""), 3)
}
@@ -0,0 +1,61 @@
package store_sqlite_test
import (
"testing"
"github.com/zzet/gortex/internal/graph"
)
// TestExternalCallCandidateEdges asserts the pushdown selects exactly the
// external-package terminals (dep:: / stdlib:: / external::, the
// per-repo-prefixed stdlib form, and already-materialised external-call::
// nodes) and nothing else — no ordinary resolved call/reference edges,
// no non-call edges.
func TestExternalCallCandidateEdges(t *testing.T) {
s := openTestStore(t)
add := func(from, to string, kind graph.EdgeKind) {
s.AddEdge(&graph.Edge{From: from, To: to, Kind: kind, FilePath: "a.go", Line: 1})
}
// External terminals — should be selected.
add("a.go::f", "dep::github.com/x/y::Z", graph.EdgeCalls)
add("a.go::f", "stdlib::fmt::Sprintf", graph.EdgeCalls)
add("a.go::f", "myrepo::stdlib::net/http::Get", graph.EdgeCalls) // per-repo-prefixed stdlib form
add("a.go::f", "external::svc.internal/api", graph.EdgeReferences)
add("a.go::f", "external-call::dep::github.com/a/b", graph.EdgeCalls) // already synthesized
// Non-candidates — must NOT be selected.
add("a.go::f", "a.go::resolvedCallee", graph.EdgeCalls) // ordinary resolved call
add("a.go::f", "unresolved::SomeName", graph.EdgeCalls) // bare unresolved (no import evidence)
add("a.go::f", "a.go::SomeType", graph.EdgeImplements) // not a call/ref edge
add("a.go::f", "dep::github.com/x/y::Z", graph.EdgeTests) // dep target but wrong kind
got := map[string]bool{}
for _, e := range s.ExternalCallCandidateEdges() {
got[e.To] = true
}
want := []string{
"dep::github.com/x/y::Z",
"stdlib::fmt::Sprintf",
"myrepo::stdlib::net/http::Get",
"external::svc.internal/api",
"external-call::dep::github.com/a/b",
}
for _, w := range want {
if !got[w] {
t.Errorf("ExternalCallCandidateEdges missing external terminal %q", w)
}
}
notWant := []string{"a.go::resolvedCallee", "unresolved::SomeName", "a.go::SomeType"}
for _, nw := range notWant {
if got[nw] {
t.Errorf("ExternalCallCandidateEdges wrongly selected non-candidate %q", nw)
}
}
// The EdgeImplements/EdgeTests rows share targets with selected ones
// but must not inflate the count via the wrong kind: exactly the 5
// distinct external targets above, all from calls/references kinds.
if len(got) != len(want) {
t.Errorf("selected %d distinct targets, want %d: %v", len(got), len(want), got)
}
}
@@ -0,0 +1,149 @@
package store_sqlite
import (
"path/filepath"
"testing"
"github.com/zzet/gortex/internal/graph"
)
func ftsRowCount(t *testing.T, s *Store, table, nodeID string) int {
t.Helper()
var n int
q := `SELECT count(*) FROM ` + table + ` WHERE node_id = ?`
if err := s.db.QueryRow(q, nodeID).Scan(&n); err != nil {
t.Fatalf("count %s: %v", table, err)
}
return n
}
func ftsHits(t *testing.T, s *Store, query string) int {
t.Helper()
hits, err := s.SearchSymbols(query, 20)
if err != nil {
t.Fatalf("SearchSymbols(%q): %v", query, err)
}
return len(hits)
}
// TestUpsertSymbolFTS_ReplacesWithoutDuplicates is the core correctness
// guard for the rowid-map delete: re-upserting a symbol must drop its prior
// row (by docid) and leave exactly one FTS row + one map row. A wrong
// LastInsertId / stale map would leave the old tokens searchable and the
// row count at 2 — so this also proves the FTS5 docid round-trips.
func TestUpsertSymbolFTS_ReplacesWithoutDuplicates(t *testing.T) {
s, err := Open(filepath.Join(t.TempDir(), "fts.sqlite"))
if err != nil {
t.Fatalf("Open: %v", err)
}
t.Cleanup(func() { _ = s.Close() })
const id = "pkg/x.go::A"
s.AddNode(mkFnNode(id, "AlphaWidget", "pkg/x.go"))
// Upsert three times: first insert, then two replacements that each
// exercise the mapped-entry docid delete.
for _, tokens := range []string{"alpha widget red", "alpha widget green", "alpha widget blue"} {
if err := s.UpsertSymbolFTS(id, tokens); err != nil {
t.Fatalf("UpsertSymbolFTS(%q): %v", tokens, err)
}
}
if got := ftsRowCount(t, s, "symbol_fts", id); got != 1 {
t.Fatalf("symbol_fts rows for %s = %d, want 1 (duplicate row leaked)", id, got)
}
if got := ftsRowCount(t, s, "symbol_fts_rowid", id); got != 1 {
t.Fatalf("symbol_fts_rowid rows for %s = %d, want 1", id, got)
}
// Only the latest tokens are searchable; the superseded ones are gone.
if got := ftsHits(t, s, "blue"); got != 1 {
t.Fatalf("search 'blue' (current tokens) = %d hits, want 1", got)
}
for _, stale := range []string{"red", "green"} {
if got := ftsHits(t, s, stale); got != 0 {
t.Fatalf("search %q (superseded tokens) = %d hits, want 0 (delete missed)", stale, got)
}
}
}
// TestBulkUpsertSymbolFTS_MaintainsRowidMap proves the bulk path keeps the
// sidecar in lockstep, and that a follow-up incremental upsert on a
// bulk-loaded symbol still replaces cleanly (no duplicate).
func TestBulkUpsertSymbolFTS_MaintainsRowidMap(t *testing.T) {
s, err := Open(filepath.Join(t.TempDir(), "fts.sqlite"))
if err != nil {
t.Fatalf("Open: %v", err)
}
t.Cleanup(func() { _ = s.Close() })
s.AddNode(mkFnNode("pkg/x.go::A", "AlphaWidget", "pkg/x.go"))
s.AddNode(mkFnNode("pkg/x.go::B", "BetaWidget", "pkg/x.go"))
if err := s.BulkUpsertSymbolFTS("", []graph.SymbolFTSItem{
{NodeID: "pkg/x.go::A", Tokens: "alpha widget red"},
{NodeID: "pkg/x.go::B", Tokens: "beta widget red"},
}); err != nil {
t.Fatalf("BulkUpsertSymbolFTS: %v", err)
}
var mapRows int
if err := s.db.QueryRow(`SELECT count(*) FROM symbol_fts_rowid`).Scan(&mapRows); err != nil {
t.Fatalf("count map: %v", err)
}
if mapRows != 2 {
t.Fatalf("symbol_fts_rowid rows = %d, want 2 after bulk", mapRows)
}
// Incremental replace on a bulk-loaded symbol — must not duplicate.
if err := s.UpsertSymbolFTS("pkg/x.go::A", "alpha widget blue"); err != nil {
t.Fatalf("UpsertSymbolFTS: %v", err)
}
if got := ftsRowCount(t, s, "symbol_fts", "pkg/x.go::A"); got != 1 {
t.Fatalf("symbol_fts rows after incremental replace = %d, want 1", got)
}
if got := ftsHits(t, s, "blue"); got != 1 {
t.Fatalf("search 'blue' = %d, want 1", got)
}
}
// TestBackfillSymbolFTSRowidMap simulates a database built before the
// sidecar existed (rows in symbol_fts, none in the map) and proves the
// backfill repopulates it so the next incremental upsert replaces cleanly
// instead of leaking a duplicate.
func TestBackfillSymbolFTSRowidMap(t *testing.T) {
s, err := Open(filepath.Join(t.TempDir(), "fts.sqlite"))
if err != nil {
t.Fatalf("Open: %v", err)
}
t.Cleanup(func() { _ = s.Close() })
const id = "pkg/x.go::A"
s.AddNode(mkFnNode(id, "AlphaWidget", "pkg/x.go"))
// Simulate the legacy state: a row in symbol_fts with no map entry.
if _, err := s.db.Exec(
`INSERT INTO symbol_fts (node_id, repo_prefix, tokens) VALUES (?, '', ?)`,
id, "alpha widget red"); err != nil {
t.Fatalf("seed legacy fts row: %v", err)
}
if _, err := s.db.Exec(`DELETE FROM symbol_fts_rowid`); err != nil {
t.Fatalf("clear map: %v", err)
}
if err := backfillSymbolFTSRowidMap(s.db); err != nil {
t.Fatalf("backfill: %v", err)
}
if got := ftsRowCount(t, s, "symbol_fts_rowid", id); got != 1 {
t.Fatalf("map rows after backfill = %d, want 1", got)
}
// Now an incremental upsert must replace, not duplicate.
if err := s.UpsertSymbolFTS(id, "alpha widget blue"); err != nil {
t.Fatalf("UpsertSymbolFTS: %v", err)
}
if got := ftsRowCount(t, s, "symbol_fts", id); got != 1 {
t.Fatalf("symbol_fts rows after post-backfill replace = %d, want 1 (dup leaked)", got)
}
if got := ftsHits(t, s, "red"); got != 0 {
t.Fatalf("search 'red' (superseded) = %d, want 0", got)
}
}
@@ -0,0 +1,76 @@
package store_sqlite
import (
"path/filepath"
"testing"
"github.com/zzet/gortex/internal/graph"
)
// TestGetRepoNodesLight verifies the graph.LightNodeReader fast path
// matches GetRepoNodes on IDs and promoted-field values, stays scoped to
// repo_prefix, and never surfaces non-promoted meta content — the
// invariant the enrichment hover-candidate refetch depends on for
// correctness (see EnrichRepoContext's use of repoScopedNodesLight).
func TestGetRepoNodesLight(t *testing.T) {
s, err := Open(filepath.Join(t.TempDir(), "light.sqlite"))
if err != nil {
t.Fatal(err)
}
defer s.Close()
s.AddNode(&graph.Node{
ID: "repoA/f.go::Stamped", Kind: graph.KindFunction, Name: "Stamped",
FilePath: "repoA/f.go", RepoPrefix: "repoA",
Meta: map[string]any{
"semantic_type": "string",
"semantic_source": "lsp-gopls",
"doc": "docs",
"complexity": 7, // non-promoted
},
})
s.AddNode(&graph.Node{
ID: "repoA/f.go::Unstamped", Kind: graph.KindFunction, Name: "Unstamped",
FilePath: "repoA/f.go", RepoPrefix: "repoA",
})
s.AddNode(&graph.Node{
ID: "repoB/g.go::Other", Kind: graph.KindFunction, Name: "Other",
FilePath: "repoB/g.go", RepoPrefix: "repoB",
})
var _ graph.LightNodeReader = s // compile-time capability check
full := s.GetRepoNodes("repoA")
light := s.GetRepoNodesLight("repoA")
if len(light) != len(full) {
t.Fatalf("light returned %d nodes, full returned %d", len(light), len(full))
}
byID := make(map[string]*graph.Node, len(light))
for _, n := range light {
byID[n.ID] = n
}
stamped, ok := byID["repoA/f.go::Stamped"]
if !ok {
t.Fatal("light scan missing the stamped node")
}
assertType[string](t, stamped.Meta, "semantic_type", "string")
assertType[string](t, stamped.Meta, "semantic_source", "lsp-gopls")
assertType[string](t, stamped.Meta, "doc", "docs")
if _, ok := stamped.Meta["complexity"]; ok {
t.Errorf("light scan must not surface non-promoted meta, got complexity=%v", stamped.Meta["complexity"])
}
unstamped, ok := byID["repoA/f.go::Unstamped"]
if !ok {
t.Fatal("light scan missing the unstamped node")
}
if _, ok := unstamped.Meta["semantic_type"]; ok {
t.Error("unstamped node must not carry a semantic_type key")
}
if _, ok := byID["repoB/g.go::Other"]; ok {
t.Error("light scan crossed repo_prefix scope")
}
}
@@ -0,0 +1,433 @@
package store_sqlite
import (
"bytes"
"encoding/gob"
"encoding/json"
"path/filepath"
"reflect"
"testing"
"github.com/zzet/gortex/internal/contracts"
"github.com/zzet/gortex/internal/graph"
)
// flatRoundTrip encodes via the flat codec only (asserting the fast path was
// taken) and decodes back through decodeMeta.
func flatRoundTrip(t *testing.T, in map[string]any) map[string]any {
t.Helper()
b, ok := encodeMetaFast(in)
if !ok {
t.Fatalf("encodeMetaFast bailed on a modelled map: %#v", in)
}
if !isFlatMeta(b) {
t.Fatalf("encodeMetaFast did not stamp the flat magic: %q", b)
}
out, err := decodeMetaFast(b)
if err != nil {
t.Fatalf("decodeMetaFast: %v", err)
}
return out
}
// TestFlatCodecEveryValueType round-trips one value of every type the codec
// models and asserts exact Go-type fidelity end to end.
func TestFlatCodecEveryValueType(t *testing.T) {
shape := &contracts.Shape{
Kind: "struct",
Fields: []contracts.ShapeField{{Name: "id", Type: "int64", Required: true}},
Notes: []string{"partial"},
}
in := map[string]any{
"str": "hello",
"unicode": "héllo мир 世界 🚀",
"bool_t": true,
"bool_f": false,
"int": -7,
"int64": int64(1700000000),
"float": 0.875,
"float_int": 2.0, // integral float must stay float64
"strs": []string{"a", "b", "c"},
"nested": map[string]any{"inner": 5, "rate": 1.5, "deep": map[string]any{"x": int64(9)}},
"map_slice": []map[string]any{{"k": "v", "n": 1}, {"k": "w", "n": 2}},
"any_slice": []any{"x", 3, true, 4.5},
"shape": shape,
"nilval": nil,
"empty_map": map[string]any{},
}
got := flatRoundTrip(t, in)
if !reflect.DeepEqual(got, in) {
t.Fatalf("flat round-trip mismatch:\n got: %#v\nwant: %#v", got, in)
}
// Spot-check the load-bearing concrete types explicitly.
mustType[string](t, got, "str")
mustType[bool](t, got, "bool_t")
mustType[int](t, got, "int")
mustType[int64](t, got, "int64")
mustType[float64](t, got, "float")
mustType[float64](t, got, "float_int")
mustType[[]string](t, got, "strs")
mustType[map[string]any](t, got, "nested")
mustType[[]map[string]any](t, got, "map_slice")
mustType[[]any](t, got, "any_slice")
mustType[*contracts.Shape](t, got, "shape")
if got["nilval"] != nil {
t.Errorf("nilval: want nil, got %#v", got["nilval"])
}
}
// TestFlatCodecLargeValues exercises long keys and values that cross the
// single-byte varint boundary (> 127 bytes), proving the length prefixes
// round-trip.
func TestFlatCodecLargeValues(t *testing.T) {
big := string(bytes.Repeat([]byte("x"), 5000))
bigKey := string(bytes.Repeat([]byte("k"), 300))
in := map[string]any{
bigKey: big,
"slice": []string{big, "", big},
}
got := flatRoundTrip(t, in)
if !reflect.DeepEqual(got, in) {
t.Fatalf("large-value round-trip mismatch")
}
}
// TestFlatCodecDeterministic proves the encoding is byte-stable across
// encodes (keys are sorted), which matters for any content-hash / dedup.
func TestFlatCodecDeterministic(t *testing.T) {
in := map[string]any{
"z": 1, "a": "x", "m": []string{"p", "q"},
"nested": map[string]any{"d": 4, "b": 2, "c": 3},
}
var prev []byte
for i := 0; i < 16; i++ {
b, ok := encodeMetaFast(in)
if !ok {
t.Fatalf("encodeMetaFast bailed")
}
if prev != nil && !bytes.Equal(prev, b) {
t.Fatalf("encoding is not deterministic across encodes")
}
prev = b
}
}
// TestEncodeMetaFallbackToJSON: a value whose type the flat codec does not
// model makes encodeMeta fall back to JSON (leading '{'), and decodeMeta
// still reads it. No data is dropped.
func TestEncodeMetaFallbackToJSON(t *testing.T) {
// uint64 is deliberately outside the modelled type set.
in := map[string]any{"weird": uint64(42), "name": "keep"}
if _, ok := encodeMetaFast(in); ok {
t.Fatal("encodeMetaFast should bail on an unmodelled value type")
}
b, err := encodeMeta(in)
if err != nil {
t.Fatalf("encodeMeta: %v", err)
}
if isFlatMeta(b) {
t.Fatalf("encodeMeta should have fallen back to JSON, got a flat blob")
}
if !isJSONObject(b) {
t.Fatalf("encodeMeta fallback did not produce a JSON object: %q", b)
}
got, err := decodeMeta(b)
if err != nil {
t.Fatalf("decodeMeta(json fallback): %v", err)
}
// The JSON fallback widens uint64 -> int (documented, lossy only for the
// exotic tail), but the string survives and no row is lost.
if got["name"] != "keep" {
t.Errorf("name not preserved through JSON fallback: %#v", got["name"])
}
if _, ok := got["weird"]; !ok {
t.Errorf("weird key dropped by JSON fallback")
}
}
// TestDecodeLegacyJSON proves rows written by the previous JSON encoder still
// decode (routed through metaWire for exact types) after the flat-codec
// switch — existing on-disk databases must keep loading.
func TestDecodeLegacyJSON(t *testing.T) {
orig := map[string]any{
"visibility": "private",
"complexity": 9,
"confidence": 1.0, // integral float — metaWire must keep it float64
"path_param_names": []string{"id"},
"last_authored": map[string]any{"timestamp": int64(1700000000)},
}
blob, err := json.Marshal(orig)
if err != nil {
t.Fatalf("json.Marshal: %v", err)
}
if isFlatMeta(blob) {
t.Fatalf("JSON blob unexpectedly looks like a flat blob")
}
got, err := decodeMeta(blob)
if err != nil {
t.Fatalf("decodeMeta(json): %v", err)
}
mustType[string](t, got, "visibility")
mustType[int](t, got, "complexity")
mustType[float64](t, got, "confidence")
mustType[[]string](t, got, "path_param_names")
la, ok := got["last_authored"].(map[string]any)
if !ok {
t.Fatalf("last_authored: want map[string]any, got %T", got["last_authored"])
}
mustType[int64](t, la, "timestamp")
}
// TestDecodeMetaFastMalformed: corrupt / truncated flat blobs return an error
// rather than panicking — a single bad row must not crash a store scan.
func TestDecodeMetaFastMalformed(t *testing.T) {
good, ok := encodeMetaFast(map[string]any{"k": "value", "n": 7, "s": []string{"a", "b"}})
if !ok {
t.Fatalf("encodeMetaFast bailed")
}
cases := map[string][]byte{
"magic only": {metaFlatMagic0, metaFlatVersion},
"count then nothing": {metaFlatMagic0, metaFlatVersion, 0x05},
"truncated mid-blob": good[:len(good)-3],
"unknown value tag": {metaFlatMagic0, metaFlatVersion, 0x01, 0x01, 'k', 0x7E},
"giant key length": {metaFlatMagic0, metaFlatVersion, 0x01, 0xFF, 0xFF, 0xFF, 0xFF, 0x0F},
}
for name, blob := range cases {
t.Run(name, func(t *testing.T) {
_, err := decodeMeta(blob)
if err == nil {
t.Errorf("expected an error for %q, got nil", name)
}
})
}
}
// TestStoreReloadMetaFidelity is the wired-path proof: persist a node and an
// edge with rich Meta through the real store, reopen it (warm restart), and
// assert Meta is byte-for-byte type-identical. Also runs PRAGMA
// integrity_check.
func TestStoreReloadMetaFidelity(t *testing.T) {
path := filepath.Join(t.TempDir(), "store.sqlite")
nodeMeta := map[string]any{
"complexity": 7,
"loop_depth": 2,
"confidence": 0.875,
"coverage_pct": 1.0, // integral float
"candidate_count": 2,
"path_param_names": []string{"id", "org"},
"status_codes": []string{"200", "404"},
"churn": map[string]any{"commit_count": 12, "churn_rate": 2.0, "last_author": "a@b.c"},
"last_authored": map[string]any{"timestamp": int64(1700000000), "email": "x@y.z"},
"response_envelope": []map[string]any{{"name": "data", "n": 1}},
"shape": &contracts.Shape{
Kind: "struct",
Fields: []contracts.ShapeField{{Name: "id", Type: "int64", Required: true}},
Notes: []string{"partial"},
},
"unicode_doc": "héllo 世界 🚀",
"is_generated": false,
}
edgeMeta := map[string]any{
"candidate_count": 3,
"similarity": 0.5,
"score": 1.0, // integral float
"count": 5,
"clone_tokens": 128,
"synthesized_by": "grpc",
"arg_names": []string{"ctx", "req"},
}
func() {
s, err := Open(path)
if err != nil {
t.Fatalf("Open: %v", err)
}
defer s.Close()
s.AddNode(&graph.Node{ID: "n1", Kind: "function", Name: "Foo", FilePath: "f.go", Meta: cloneMeta(nodeMeta)})
s.AddNode(&graph.Node{ID: "n2", Kind: "function", Name: "Bar", FilePath: "f.go"})
s.AddEdge(&graph.Edge{From: "n1", To: "n2", Kind: "calls", FilePath: "f.go", Line: 10, Meta: cloneMeta(edgeMeta)})
}()
// Reopen — the warm-restart path that reads every blob back.
s, err := Open(path)
if err != nil {
t.Fatalf("reopen: %v", err)
}
defer s.Close()
n := s.GetNode("n1")
if n == nil {
t.Fatal("GetNode(n1) = nil after reload")
}
if !reflect.DeepEqual(n.Meta, nodeMeta) {
t.Fatalf("node Meta mismatch after reload:\n got: %#v\nwant: %#v", n.Meta, nodeMeta)
}
edges := s.GetOutEdges("n1")
var got *graph.Edge
for _, e := range edges {
if e.To == "n2" && e.Kind == "calls" {
got = e
break
}
}
if got == nil {
t.Fatalf("edge n1->n2 not found after reload (got %d edges)", len(edges))
}
if !reflect.DeepEqual(got.Meta, edgeMeta) {
t.Fatalf("edge Meta mismatch after reload:\n got: %#v\nwant: %#v", got.Meta, edgeMeta)
}
var res string
if err := s.db.QueryRow(`PRAGMA integrity_check`).Scan(&res); err != nil {
t.Fatalf("integrity_check: %v", err)
}
if res != "ok" {
t.Fatalf("integrity_check = %q, want ok", res)
}
}
func cloneMeta(m map[string]any) map[string]any {
out := make(map[string]any, len(m))
for k, v := range m {
out[k] = v
}
return out
}
func mustType[T any](t *testing.T, m map[string]any, key string) {
t.Helper()
v, ok := m[key]
if !ok {
t.Errorf("%s: missing from decoded map", key)
return
}
if _, ok := v.(T); !ok {
var zero T
t.Errorf("%s: want type %T, got %T (value %v)", key, zero, v, v)
}
}
// -- benchmarks -----------------------------------------------------------
var metaSink any
// benchMetaSample is a representative node/edge meta map restricted to the
// types gob auto-registers (scalars + []string), so all three encoders run on
// identical input for an apples-to-apples comparison. Shape / nested-map /
// map-slice values also ride the flat path (see the round-trip tests); they
// are omitted here only because gob refuses unregistered interface types.
func benchMetaSample() map[string]any {
return map[string]any{
"complexity": 7,
"loop_depth": 2,
"parse_errors": 0,
"position": 3,
"line": 42,
"confidence": 0.875,
"coverage_pct": 83.5,
"candidate_count": 2,
"count": 5,
"clone_tokens": 128,
"timestamp": int64(1700000000),
"path_param_names": []string{"id", "org"},
"status_codes": []string{"200", "404"},
"signature": "func F(ctx context.Context, x int) (T, error)",
"some_plugin_flag": "go_linkname",
"is_generated": false,
"synthesized_by": "grpc",
}
}
func BenchmarkEncodeMetaGob(b *testing.B) {
m := benchMetaSample()
b.ReportAllocs()
b.ResetTimer()
for i := 0; i < b.N; i++ {
var buf bytes.Buffer
if err := gob.NewEncoder(&buf).Encode(m); err != nil {
b.Fatalf("gob encode: %v", err)
}
metaSink = buf.Bytes()
}
}
func BenchmarkEncodeMetaJSON(b *testing.B) {
m := benchMetaSample()
b.ReportAllocs()
b.ResetTimer()
for i := 0; i < b.N; i++ {
out, err := json.Marshal(m)
if err != nil {
b.Fatalf("json marshal: %v", err)
}
metaSink = out
}
}
func BenchmarkEncodeMetaFlat(b *testing.B) {
m := benchMetaSample()
b.ReportAllocs()
b.ResetTimer()
for i := 0; i < b.N; i++ {
out, ok := encodeMetaFast(m)
if !ok {
b.Fatal("encodeMetaFast bailed")
}
metaSink = out
}
}
func BenchmarkDecodeMetaGob(b *testing.B) {
var buf bytes.Buffer
if err := gob.NewEncoder(&buf).Encode(benchMetaSample()); err != nil {
b.Fatalf("gob encode: %v", err)
}
blob := buf.Bytes()
b.ReportAllocs()
b.ResetTimer()
for i := 0; i < b.N; i++ {
m, err := decodeMeta(blob)
if err != nil {
b.Fatalf("decodeMeta: %v", err)
}
metaSink = m
}
}
func BenchmarkDecodeMetaJSON(b *testing.B) {
blob, err := json.Marshal(benchMetaSample())
if err != nil {
b.Fatalf("json marshal: %v", err)
}
b.ReportAllocs()
b.ResetTimer()
for i := 0; i < b.N; i++ {
m, err := decodeMeta(blob)
if err != nil {
b.Fatalf("decodeMeta: %v", err)
}
metaSink = m
}
}
func BenchmarkDecodeMetaFlat(b *testing.B) {
blob, ok := encodeMetaFast(benchMetaSample())
if !ok {
b.Fatal("encodeMetaFast bailed")
}
b.ReportAllocs()
b.ResetTimer()
for i := 0; i < b.N; i++ {
m, err := decodeMeta(blob)
if err != nil {
b.Fatalf("decodeMeta: %v", err)
}
metaSink = m
}
}
File diff suppressed because it is too large Load Diff
@@ -0,0 +1,176 @@
package store_sqlite
import (
"bytes"
"encoding/gob"
"reflect"
"testing"
"github.com/zzet/gortex/internal/contracts"
)
// roundTrip encodes Meta with the flat codec and decodes it back, the
// persist->reload path every reader sees after a daemon restart / store
// hydration.
func roundTrip(t *testing.T, in map[string]any) map[string]any {
t.Helper()
b, err := encodeMeta(in)
if err != nil {
t.Fatalf("encodeMeta: %v", err)
}
if !isFlatMeta(b) {
t.Fatalf("encodeMeta did not produce a flat-codec blob: %q", b)
}
out, err := decodeMeta(b)
if err != nil {
t.Fatalf("decodeMeta: %v", err)
}
return out
}
// TestMetaRoundTripExactTypes is the fidelity canary: every key the audit
// found read with a raw type-assertion must survive a JSON round-trip with
// its exact Go type, or the corresponding reader silently breaks.
func TestMetaRoundTripExactTypes(t *testing.T) {
shape := &contracts.Shape{
Kind: "struct",
Fields: []contracts.ShapeField{{Name: "id", Type: "int64", Required: true}},
Notes: []string{"partial"},
}
node := map[string]any{
"signature": "func F(x int) error",
"visibility": "public",
"doc": "F does a thing.",
"external": true,
"complexity": 7,
"loop_depth": 2,
"parse_errors": 0,
"position": 3,
"line": 42,
"confidence": 1.0, // integral float — must stay float64
"coverage_pct": 83.5,
"shape": shape,
"response_envelope": []map[string]any{{"name": "data", "type": "User"}},
"path_param_names": []string{"id", "org"},
"query_params": []string{"limit"},
"status_codes": []string{"200", "404"},
"churn": map[string]any{"commit_count": 12, "age_days": 365, "churn_rate": 2.0, "last_author": "a@b.c"},
"coverage": map[string]any{"num_stmt": 40, "hit": 33},
"last_authored": map[string]any{"timestamp": int64(1700000000), "email": "x@y.z"},
"some_plugin_flag": "go_linkname", // Extra tail (string)
"is_generated": false, // Extra tail (bool)
}
got := roundTrip(t, node)
assertType[int](t, got, "complexity", 7)
assertType[int](t, got, "loop_depth", 2)
assertType[int](t, got, "parse_errors", 0)
assertType[int](t, got, "position", 3)
assertType[int](t, got, "line", 42)
assertType[float64](t, got, "confidence", 1.0)
assertType[float64](t, got, "coverage_pct", 83.5)
assertType[string](t, got, "signature", "func F(x int) error")
assertType[string](t, got, "visibility", "public")
assertType[bool](t, got, "external", true)
assertType[string](t, got, "some_plugin_flag", "go_linkname")
assertType[bool](t, got, "is_generated", false)
// Shape must rebuild as *contracts.Shape, not map[string]any.
gotShape, ok := got["shape"].(*contracts.Shape)
if !ok {
t.Fatalf("shape: want *contracts.Shape, got %T", got["shape"])
}
if !reflect.DeepEqual(gotShape, shape) {
t.Errorf("shape mismatch: %+v vs %+v", gotShape, shape)
}
// response_envelope must be []map[string]any, not []any.
if _, ok := got["response_envelope"].([]map[string]any); !ok {
t.Errorf("response_envelope: want []map[string]any, got %T", got["response_envelope"])
}
// []string keys.
for _, k := range []string{"path_param_names", "query_params", "status_codes"} {
if _, ok := got[k].([]string); !ok {
t.Errorf("%s: want []string, got %T", k, got[k])
}
}
// Nested map children keep exact types.
churn := got["churn"].(map[string]any)
assertType[int](t, churn, "commit_count", 12)
assertType[int](t, churn, "age_days", 365)
assertType[float64](t, churn, "churn_rate", 2.0) // integral float, nested
assertType[string](t, churn, "last_author", "a@b.c")
cov := got["coverage"].(map[string]any)
assertType[int](t, cov, "num_stmt", 40)
assertType[int](t, cov, "hit", 33)
la := got["last_authored"].(map[string]any)
assertType[int64](t, la, "timestamp", int64(1700000000))
}
func TestEdgeMetaRoundTripExactTypes(t *testing.T) {
edge := map[string]any{
"candidate_count": 2,
"similarity": 0.875,
"score": 1.0, // integral float — must stay float64
"count": 5,
"clone_tokens": 128,
"synthesized_by": "grpc", // Extra tail
}
got := roundTrip(t, edge)
assertType[int](t, got, "candidate_count", 2)
assertType[float64](t, got, "similarity", 0.875)
assertType[float64](t, got, "score", 1.0)
assertType[int](t, got, "count", 5)
assertType[int](t, got, "clone_tokens", 128)
assertType[string](t, got, "synthesized_by", "grpc")
}
// TestDecodeLegacyGob proves existing on-disk gob blobs still decode.
func TestDecodeLegacyGob(t *testing.T) {
orig := map[string]any{"visibility": "private", "complexity": 9, "confidence": 1.0}
var buf bytes.Buffer
if err := gob.NewEncoder(&buf).Encode(orig); err != nil {
t.Fatalf("gob encode: %v", err)
}
got, err := decodeMeta(buf.Bytes())
if err != nil {
t.Fatalf("decodeMeta(gob): %v", err)
}
// gob preserves exact types natively.
assertType[string](t, got, "visibility", "private")
assertType[int](t, got, "complexity", 9)
assertType[float64](t, got, "confidence", 1.0)
}
func TestEncodeMetaEmpty(t *testing.T) {
b, err := encodeMeta(nil)
if err != nil || b != nil {
t.Fatalf("encodeMeta(nil) = %q, %v; want nil, nil", b, err)
}
b, err = encodeMeta(map[string]any{})
if err != nil || b != nil {
t.Fatalf("encodeMeta(empty) = %q, %v; want nil, nil", b, err)
}
m, err := decodeMeta(nil)
if err != nil || m != nil {
t.Fatalf("decodeMeta(nil) = %v, %v; want nil, nil", m, err)
}
}
func assertType[T comparable](t *testing.T, m map[string]any, key string, want T) {
t.Helper()
v, ok := m[key]
if !ok {
t.Errorf("%s: missing from decoded map", key)
return
}
got, ok := v.(T)
if !ok {
t.Errorf("%s: want type %T, got %T (value %v)", key, want, v, v)
return
}
if got != want {
t.Errorf("%s: want %v, got %v", key, want, got)
}
}
@@ -0,0 +1,252 @@
package store_sqlite
import (
"database/sql"
"path/filepath"
"strings"
"testing"
"github.com/zzet/gortex/internal/graph"
)
// TestPromotedColumns_RoundTrip verifies the promoted keys land in their
// columns, are stripped from the JSON blob, and restore into Meta with
// exact types — while non-promoted keys stay in the blob.
func TestPromotedColumns_RoundTrip(t *testing.T) {
s, err := Open(filepath.Join(t.TempDir(), "p.sqlite"))
if err != nil {
t.Fatal(err)
}
defer s.Close()
s.AddNode(&graph.Node{
ID: "f.go::F", Kind: graph.KindFunction, Name: "F", FilePath: "f.go",
Meta: map[string]any{
"signature": "func F()",
"visibility": "public",
"doc": "F docs",
"external": true,
"complexity": 5, // non-promoted — must stay in the blob
},
})
n := s.GetNode("f.go::F")
if n == nil {
t.Fatal("GetNode returned nil")
}
assertType[string](t, n.Meta, "signature", "func F()")
assertType[string](t, n.Meta, "visibility", "public")
assertType[string](t, n.Meta, "doc", "F docs")
assertType[bool](t, n.Meta, "external", true)
assertType[int](t, n.Meta, "complexity", 5)
var sig, vis, doc sql.NullString
var ext sql.NullBool
var blob []byte
row := s.db.QueryRow(`SELECT signature, visibility, doc, external, meta FROM nodes WHERE id=?`, "f.go::F")
if err := row.Scan(&sig, &vis, &doc, &ext, &blob); err != nil {
t.Fatal(err)
}
if !sig.Valid || sig.String != "func F()" {
t.Errorf("signature column = %+v", sig)
}
if !ext.Valid || !ext.Bool {
t.Errorf("external column = %+v", ext)
}
blobStr := string(blob)
for _, k := range []string{"signature", "visibility", "external"} {
if strings.Contains(blobStr, k) {
t.Errorf("blob still contains promoted key %q: %s", k, blobStr)
}
}
if !strings.Contains(blobStr, "complexity") {
t.Errorf("blob missing non-promoted key complexity: %s", blobStr)
}
}
// TestPromotedColumns_NewColumns verifies the added promoted meta columns
// (is_async / is_static / is_abstract / is_exported / return_type / updated_at)
// and the struct-field columns (start_column / end_column) round-trip through
// their typed columns, and that a SQL filter on is_async resolves WITHOUT
// decoding the meta blob — the indexable-column acceptance.
func TestPromotedColumns_NewColumns(t *testing.T) {
s, err := Open(filepath.Join(t.TempDir(), "p.sqlite"))
if err != nil {
t.Fatal(err)
}
defer s.Close()
s.AddNode(&graph.Node{
ID: "f.go::Async", Kind: graph.KindFunction, Name: "Async", FilePath: "f.go",
StartLine: 10, EndLine: 20, StartColumn: 4, EndColumn: 1,
Meta: map[string]any{
"is_async": true,
"is_static": false,
"is_abstract": true,
"is_exported": true,
"return_type": "error",
"updated_at": int64(1700000000),
"complexity": 3, // non-promoted — stays in the blob
},
})
// A second, non-async node to prove the filter is selective.
s.AddNode(&graph.Node{
ID: "f.go::Sync", Kind: graph.KindFunction, Name: "Sync", FilePath: "f.go",
Meta: map[string]any{"is_async": false},
})
// Read-back restores every promoted key into Meta with its exact type,
// and the struct columns into the Node fields.
n := s.GetNode("f.go::Async")
if n == nil {
t.Fatal("GetNode returned nil")
}
assertType[bool](t, n.Meta, "is_async", true)
assertType[bool](t, n.Meta, "is_static", false)
assertType[bool](t, n.Meta, "is_abstract", true)
assertType[bool](t, n.Meta, "is_exported", true)
assertType[string](t, n.Meta, "return_type", "error")
assertType[int64](t, n.Meta, "updated_at", int64(1700000000))
assertType[int](t, n.Meta, "complexity", 3)
if n.StartColumn != 4 || n.EndColumn != 1 {
t.Errorf("column offsets = (%d,%d), want (4,1)", n.StartColumn, n.EndColumn)
}
// The promoted keys are stripped from the JSON blob; complexity is not.
var blob []byte
if err := s.db.QueryRow(`SELECT meta FROM nodes WHERE id=?`, "f.go::Async").Scan(&blob); err != nil {
t.Fatal(err)
}
for _, k := range []string{"is_async", "is_abstract", "return_type", "updated_at"} {
if strings.Contains(string(blob), k) {
t.Errorf("blob still contains promoted key %q: %s", k, blob)
}
}
if !strings.Contains(string(blob), "complexity") {
t.Errorf("blob missing non-promoted key complexity: %s", blob)
}
// Acceptance: a SQL filter on the typed column resolves the node without
// touching the meta blob (only id is selected).
var id string
var startCol, endCol int
if err := s.db.QueryRow(
`SELECT id, start_column, end_column FROM nodes WHERE is_async = 1`,
).Scan(&id, &startCol, &endCol); err != nil {
t.Fatalf("is_async column filter failed: %v", err)
}
if id != "f.go::Async" || startCol != 4 || endCol != 1 {
t.Errorf("filter result = (%q,%d,%d), want (f.go::Async,4,1)", id, startCol, endCol)
}
}
// TestPromotedColumns_SemanticType verifies semantic_type/semantic_source —
// promoted alongside signature/visibility/etc. so enrichment can query the
// unstamped subset by column instead of decoding every node's meta blob —
// round-trip through their columns and out of the JSON blob.
func TestPromotedColumns_SemanticType(t *testing.T) {
s, err := Open(filepath.Join(t.TempDir(), "p.sqlite"))
if err != nil {
t.Fatal(err)
}
defer s.Close()
s.AddNode(&graph.Node{
ID: "f.go::T", Kind: graph.KindFunction, Name: "T", FilePath: "f.go",
Meta: map[string]any{
"semantic_type": "string",
"semantic_source": "lsp-gopls",
"complexity": 5, // non-promoted — must stay in the blob
},
})
n := s.GetNode("f.go::T")
if n == nil {
t.Fatal("GetNode returned nil")
}
assertType[string](t, n.Meta, "semantic_type", "string")
assertType[string](t, n.Meta, "semantic_source", "lsp-gopls")
assertType[int](t, n.Meta, "complexity", 5)
var st, ss sql.NullString
var blob []byte
row := s.db.QueryRow(`SELECT semantic_type, semantic_source, meta FROM nodes WHERE id=?`, "f.go::T")
if err := row.Scan(&st, &ss, &blob); err != nil {
t.Fatal(err)
}
if !st.Valid || st.String != "string" {
t.Errorf("semantic_type column = %+v", st)
}
if !ss.Valid || ss.String != "lsp-gopls" {
t.Errorf("semantic_source column = %+v", ss)
}
blobStr := string(blob)
for _, k := range []string{"semantic_type", "semantic_source"} {
if strings.Contains(blobStr, k) {
t.Errorf("blob still contains promoted key %q: %s", k, blobStr)
}
}
if !strings.Contains(blobStr, "complexity") {
t.Errorf("blob missing non-promoted key complexity: %s", blobStr)
}
}
// TestPromotedColumns_ExternalFalse guards the NULL-vs-false distinction:
// a stored false must round-trip as false, not vanish.
func TestPromotedColumns_ExternalFalse(t *testing.T) {
s, err := Open(filepath.Join(t.TempDir(), "p.sqlite"))
if err != nil {
t.Fatal(err)
}
defer s.Close()
s.AddNode(&graph.Node{
ID: "x", Kind: graph.KindFunction, Name: "x", FilePath: "x.go",
Meta: map[string]any{"external": false},
})
n := s.GetNode("x")
if n == nil {
t.Fatal("nil")
}
v, ok := n.Meta["external"].(bool)
if !ok || v != false {
t.Errorf("external false: got %v (%T)", n.Meta["external"], n.Meta["external"])
}
}
// TestPromotedColumns_Migration verifies ensureNodeColumns adds the
// promoted columns to a database created with the pre-promotion schema.
func TestPromotedColumns_Migration(t *testing.T) {
path := filepath.Join(t.TempDir(), "old.sqlite")
raw, err := sql.Open("sqlite", path)
if err != nil {
t.Fatal(err)
}
_, err = raw.Exec(`CREATE TABLE nodes (
id TEXT PRIMARY KEY, kind TEXT NOT NULL, name TEXT NOT NULL,
qual_name TEXT NOT NULL DEFAULT '', file_path TEXT NOT NULL,
start_line INTEGER NOT NULL DEFAULT 0, end_line INTEGER NOT NULL DEFAULT 0,
language TEXT NOT NULL DEFAULT '', repo_prefix TEXT NOT NULL DEFAULT '',
workspace_id TEXT NOT NULL DEFAULT '', project_id TEXT NOT NULL DEFAULT '',
meta BLOB
) WITHOUT ROWID`)
if err != nil {
t.Fatal(err)
}
_ = raw.Close()
s, err := Open(path)
if err != nil {
t.Fatalf("Open old-schema db: %v", err)
}
defer s.Close()
s.AddNode(&graph.Node{
ID: "m", Kind: graph.KindFunction, Name: "m", FilePath: "m.go",
Meta: map[string]any{"signature": "sig", "external": true},
})
n := s.GetNode("m")
if n == nil {
t.Fatal("nil after migration")
}
assertType[string](t, n.Meta, "signature", "sig")
assertType[bool](t, n.Meta, "external", true)
}
@@ -0,0 +1,45 @@
package store_sqlite_test
import (
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/zzet/gortex/internal/graph"
)
// TestGetOutEdgesLight_SkipsMetaKeepsEndpoints proves the light out-edge
// fetch returns the same endpoints/kind/line as GetOutEdges while leaving
// Meta nil — it must never pay the per-edge meta JSON decode. This is the
// fetch findCallTarget uses on the dataflow hot path.
func TestGetOutEdgesLight_SkipsMetaKeepsEndpoints(t *testing.T) {
s := openTestStore(t)
want := &graph.Edge{
From: "pkg/x.go::Caller",
To: "pkg/x.go::Callee",
Kind: graph.EdgeCalls,
FilePath: "pkg/x.go",
Line: 42,
Meta: map[string]any{"call_line": 42, "callee_target": "unresolved::Callee"},
}
s.AddEdge(want)
full := s.GetOutEdges("pkg/x.go::Caller")
require.Len(t, full, 1)
require.NotNil(t, full[0].Meta, "GetOutEdges must decode Meta")
assert.Equal(t, "unresolved::Callee", full[0].Meta["callee_target"])
light := s.GetOutEdgesLight("pkg/x.go::Caller")
require.Len(t, light, 1)
assert.Equal(t, full[0].From, light[0].From)
assert.Equal(t, full[0].To, light[0].To)
assert.Equal(t, full[0].Kind, light[0].Kind)
assert.Equal(t, full[0].Line, light[0].Line)
assert.Equal(t, full[0].FilePath, light[0].FilePath)
assert.Nil(t, light[0].Meta, "light fetch must not decode the meta blob")
// A node with no out-edges returns nothing on both paths.
assert.Empty(t, s.GetOutEdgesLight("pkg/x.go::Callee"))
}
@@ -0,0 +1,58 @@
package store_sqlite
import (
"path/filepath"
"testing"
"github.com/zzet/gortex/internal/graph"
)
// TestProxyNodes_NeverReachDisk validates that cross-daemon proxy-edge
// nodes (and their edges) are dropped at the single durable write boundary
// (AddNode / AddBatch), so a warm restart over the store never sees them,
// while every real node round-trips intact.
func TestProxyNodes_NeverReachDisk(t *testing.T) {
path := filepath.Join(t.TempDir(), "store.sqlite")
s, err := Open(path)
if err != nil {
t.Fatalf("open: %v", err)
}
real := &graph.Node{ID: "local/a.go::Foo", Kind: graph.KindFunction, Name: "Foo"}
proxy := &graph.Node{
ID: graph.ProxyNodeID("remoteB", "rb/x.go::Bar"),
Kind: graph.KindFunction, Name: "Bar",
Origin: "remote:remoteB", Stub: true,
}
proxyEdge := &graph.Edge{From: real.ID, To: proxy.ID, Kind: graph.EdgeCalls}
// Mix proxy + real through AddBatch...
s.AddBatch([]*graph.Node{real, proxy}, []*graph.Edge{proxyEdge})
// ...and a proxy through the per-node path.
s.AddNode(&graph.Node{
ID: graph.ProxyNodeID("remoteC", "rc/y.go::Baz"),
Kind: graph.KindFunction, Name: "Baz",
Origin: "remote:remoteC", Stub: true,
})
_ = s.Close()
// Reopen — a warm restart sees the durable store only.
s2, err := Open(path)
if err != nil {
t.Fatalf("reopen: %v", err)
}
defer s2.Close()
if s2.GetNode(proxy.ID) != nil {
t.Error("proxy node must not be persisted")
}
if s2.GetNode(graph.ProxyNodeID("remoteC", "rc/y.go::Baz")) != nil {
t.Error("proxy node added via AddNode must not be persisted")
}
if s2.GetNode(real.ID) == nil {
t.Error("the real node must round-trip intact")
}
if outs := s2.GetOutEdges(real.ID); len(outs) != 0 {
t.Errorf("the edge to a proxy node must not be persisted; got %d", len(outs))
}
}
@@ -0,0 +1,74 @@
package store_sqlite
import (
"database/sql"
"fmt"
"os"
"github.com/zzet/gortex/internal/graph"
)
// ReadRepoIndexStates opens the SQLite store at path read-only and returns
// every repo_index_state freshness row keyed by repo_prefix.
//
// It is a deliberately lightweight side door for read-only callers (notably
// `gortex repos`) that must inspect index freshness WITHOUT going through
// Open — which runs schema migrations, alters columns, starts a checkpoint
// goroutine, and (on a version mismatch) can refuse to open or rebuild the
// file. None of that is appropriate for a status command that may run while
// a daemon holds the same store open.
//
// The connection is query-only and inherits the database's existing journal
// mode, so it reads safely alongside a running daemon (WAL permits concurrent
// readers). A missing store file, or a database that predates the
// repo_index_state table, both yield an empty map and a nil error — that is
// "nothing recorded yet", not a failure, so the caller can fall back to other
// freshness sources rather than surfacing an error to the user.
func ReadRepoIndexStates(path string) (map[string]graph.RepoIndexState, error) {
if _, err := os.Stat(path); err != nil {
if os.IsNotExist(err) {
return map[string]graph.RepoIndexState{}, nil
}
return nil, fmt.Errorf("stat sqlite store %q: %w", path, err)
}
// query_only blocks accidental writes; busy_timeout keeps a brief read
// from erroring out if the daemon happens to hold the write lock for a
// moment. We deliberately do NOT set journal_mode — forcing it could try
// to switch the live database's mode; inheriting the on-disk WAL mode is
// exactly what a concurrent reader wants.
dsn := path + "?_pragma=busy_timeout(2000)&_pragma=query_only(1)"
db, err := sql.Open("sqlite", dsn)
if err != nil {
return nil, fmt.Errorf("open sqlite store %q: %w", path, err)
}
defer db.Close()
db.SetMaxOpenConns(1)
rows, err := db.Query(`
SELECT repo_prefix, indexed_sha, dirty, indexed_at, workspace_fp, node_count, edge_count, extractor_versions
FROM repo_index_state`)
if err != nil {
// A store written before the repo_index_state table existed (or any
// other read error) is treated as "no freshness recorded yet" — a
// status command must never hard-fail on a degraded cache.
return map[string]graph.RepoIndexState{}, nil
}
defer rows.Close()
out := map[string]graph.RepoIndexState{}
for rows.Next() {
var st graph.RepoIndexState
var dirty int
if err := rows.Scan(&st.RepoPrefix, &st.IndexedSHA, &dirty, &st.IndexedAt,
&st.WorkspaceFP, &st.NodeCount, &st.EdgeCount, &st.ExtractorVersions); err != nil {
return nil, fmt.Errorf("scan repo_index_state: %w", err)
}
st.Dirty = dirty != 0
out[st.RepoPrefix] = st
}
if err := rows.Err(); err != nil {
return nil, fmt.Errorf("iterate repo_index_state: %w", err)
}
return out, nil
}
@@ -0,0 +1,78 @@
package store_sqlite_test
import (
"path/filepath"
"testing"
"github.com/stretchr/testify/require"
"github.com/zzet/gortex/internal/graph"
"github.com/zzet/gortex/internal/graph/store_sqlite"
)
// A store file that does not exist yet reads back as an empty map, not an
// error — "nothing indexed", so callers fall back to other sources.
func TestReadRepoIndexStates_MissingFile(t *testing.T) {
got, err := store_sqlite.ReadRepoIndexStates(filepath.Join(t.TempDir(), "absent.sqlite"))
require.NoError(t, err)
require.Empty(t, got)
}
// The read-only reader returns exactly the rows SetRepoIndexState wrote,
// keyed by repo prefix, including the empty (lone-repo) prefix.
func TestReadRepoIndexStates_RoundTrip(t *testing.T) {
path := filepath.Join(t.TempDir(), "is.sqlite")
s, err := store_sqlite.Open(path)
require.NoError(t, err)
require.NoError(t, s.SetRepoIndexState(graph.RepoIndexState{
RepoPrefix: "alpha", IndexedSHA: "aaa111", Dirty: false, IndexedAt: 1700000000,
WorkspaceFP: "fp-a", NodeCount: 10, EdgeCount: 20, ExtractorVersions: `{"go":1}`,
}))
require.NoError(t, s.SetRepoIndexState(graph.RepoIndexState{
RepoPrefix: "", IndexedSHA: "bbb222", Dirty: true, IndexedAt: 1700000001,
}))
require.NoError(t, s.Close())
got, err := store_sqlite.ReadRepoIndexStates(path)
require.NoError(t, err)
require.Len(t, got, 2)
alpha, ok := got["alpha"]
require.True(t, ok)
require.Equal(t, "aaa111", alpha.IndexedSHA)
require.False(t, alpha.Dirty)
require.EqualValues(t, 1700000000, alpha.IndexedAt)
require.Equal(t, 10, alpha.NodeCount)
lone, ok := got[""]
require.True(t, ok)
require.Equal(t, "bbb222", lone.IndexedSHA)
require.True(t, lone.Dirty)
}
// Reading is non-destructive and repeatable while the underlying store is
// reopened for writing — the read path must never lock the writer out.
func TestReadRepoIndexStates_ConcurrentWithOpenStore(t *testing.T) {
path := filepath.Join(t.TempDir(), "is.sqlite")
s, err := store_sqlite.Open(path)
require.NoError(t, err)
t.Cleanup(func() { _ = s.Close() })
require.NoError(t, s.SetRepoIndexState(graph.RepoIndexState{
RepoPrefix: "live", IndexedSHA: "c0ffee", IndexedAt: 1700000002,
}))
// Read while the writer store is still open (WAL allows concurrent readers).
got, err := store_sqlite.ReadRepoIndexStates(path)
require.NoError(t, err)
require.Equal(t, "c0ffee", got["live"].IndexedSHA)
// A subsequent write is still possible — the reader did not wedge the writer.
require.NoError(t, s.SetRepoIndexState(graph.RepoIndexState{
RepoPrefix: "live", IndexedSHA: "feedface", IndexedAt: 1700000003,
}))
got, err = store_sqlite.ReadRepoIndexStates(path)
require.NoError(t, err)
require.Equal(t, "feedface", got["live"].IndexedSHA)
}
+518
View File
@@ -0,0 +1,518 @@
package store_sqlite
import "database/sql"
// isUnresolvedColumnDDL is the edges.is_unresolved generated column: a
// VIRTUAL, indexed boolean mirroring graph.IsUnresolvedTarget's two shapes
// (the bare `unresolved::Name` prefix and the multi-repo COPY-rewrite
// `<repoPrefix>::unresolved::Name` infix), computed by SQLite itself from
// to_id — no Go call site has to remember to keep it in sync. VIRTUAL, not
// STORED: SQLite refuses `ALTER TABLE ADD COLUMN ... STORED` on a non-empty
// table ("cannot add a STORED column"), which every real installed store is.
// VIRTUAL has no such restriction and is just as fast here — the read path
// always goes through the index below, and an index always stores its own
// materialised key values regardless of whether the underlying column is
// virtual or stored. Added via ensureEdgeColumns (ALTER TABLE) rather than
// baked into schemaSQL's CREATE TABLE so one code path handles both a fresh
// DB (column missing right after CREATE TABLE) and an existing one (column
// missing from before this was introduced) identically — see
// ensureNodeColumns for the same pattern on the nodes table.
//
// Measured on a real 26-repo store (2.57M edges, 847,684 unresolved, ~33%
// selectivity): replacing the OR'd `to_id` range/LIKE query with
// `is_unresolved = 1` cut EdgesWithUnresolvedTarget from 7.96s to 2.95s
// (2.7x). The prior approach of splitting the OR into two to_id-based
// queries (one indexed range, one LIKE) was WORSE (13.49s) despite a
// better-looking EXPLAIN QUERY PLAN: at ~33% selectivity the to_id index's
// matching rows are ordered by string value, so the mandatory per-row
// bookmark lookup back into the main table is effectively random I/O. The
// boolean column's matching rows are all rowid-tie-broken (identical index
// key), so its bookmark lookups land in ascending rowid order — sequential,
// not random. Same "SEARCH ... USING INDEX" in EXPLAIN QUERY PLAN either way;
// only real measurement told them apart.
const isUnresolvedColumnDDL = `is_unresolved INTEGER GENERATED ALWAYS AS (
CASE WHEN (to_id >= 'unresolved::' AND to_id < 'unresolved:;') OR to_id LIKE '%::unresolved::%' THEN 1 ELSE 0 END
) VIRTUAL`
// edgeGeneratedColumns is the set of edges.* generated columns ensureEdgeColumns
// adds to a table created before they existed — which, since none of them are
// in schemaSQL's CREATE TABLE, includes a freshly created table too.
var edgeGeneratedColumns = []struct {
name string
ddl string
}{
{"is_unresolved", isUnresolvedColumnDDL},
}
// edgePromotedColumns lifts the resolver's resolve_terminal /
// resolve_terminal_reason Meta keys (see resolver/terminal.go) out of the
// meta blob into their own nullable columns — the edge-side sibling of
// promotedMetaColumns on nodes (see meta_json.go's "promoted edge columns"
// section for extractPromotedEdgeMeta/restorePromotedEdgeMeta and why a
// json_extract-derived generated column was tried first and abandoned:
// encodeMeta's common case is a custom flat binary codec, not JSON, so
// json_extract/json_valid against a real store's meta blobs evaluates to
// NULL for effectively every row). Plain (non-generated) columns, so they
// share ensureEdgeColumns' table_xinfo scan but are ordinary ALTER TABLE ADD
// COLUMN statements, not GENERATED ALWAYS AS expressions.
//
// Exists to let a future bulk classification query (replacing per-edge
// Go-side classifyTerminal calls in reconcileTerminalStamps) read the
// CURRENT terminal state as a plain indexed column instead of decoding
// Meta, and compare it against a freshly computed value to find only the
// edges whose state actually changed — reconcileTerminalStamps measured
// only ~1% of examined edges (9,599 of 833,828) ever change state.
var edgePromotedColumns = []struct {
name string
ddl string
}{
{"resolve_terminal", "resolve_terminal INTEGER"},
{"resolve_terminal_reason", "resolve_terminal_reason TEXT"},
}
// ensureEdgeColumns adds edgeGeneratedColumns + edgePromotedColumns to an
// edges table created before they existed. Mirrors ensureNodeColumns'
// PRAGMA + conditional ALTER pattern, but queries table_xinfo rather than
// table_info: table_info silently OMITS generated columns from its result
// set (verified against the pinned modernc.org/sqlite driver — a reopened
// store's is_unresolved column is invisible to table_info, so the existence
// check always came back false and every reopen re-ran the ALTER, failing
// with "duplicate column name"). table_xinfo lists every column, generated
// ones included, with an extra hidden column (3 == generated) table_info
// doesn't have — and works identically for the plain promoted columns too,
// so one scan serves both lists.
func ensureEdgeColumns(db *sql.DB) error {
rows, err := db.Query(`PRAGMA table_xinfo(edges)`)
if err != nil {
return err
}
existing := make(map[string]bool)
for rows.Next() {
var (
cid, notnull, pk, hidden int
name, ctype string
dflt sql.NullString
)
if err := rows.Scan(&cid, &name, &ctype, &notnull, &dflt, &pk, &hidden); err != nil {
_ = rows.Close()
return err
}
existing[name] = true
}
if err := rows.Err(); err != nil {
_ = rows.Close()
return err
}
_ = rows.Close()
for _, c := range edgeGeneratedColumns {
if existing[c.name] {
continue
}
if _, err := db.Exec(`ALTER TABLE edges ADD COLUMN ` + c.ddl); err != nil {
return err
}
}
for _, c := range edgePromotedColumns {
if existing[c.name] {
continue
}
if _, err := db.Exec(`ALTER TABLE edges ADD COLUMN ` + c.ddl); err != nil {
return err
}
}
return nil
}
// isStubColumnDDL is nodes.is_stub: a VIRTUAL generated column mirroring
// graph.IsStub/StubKind's id-prefix logic (stdlib:: / builtin:: /
// external_call:: / module::, bare or repo-prefixed as <repo>::<kind>::...).
// Same rationale as isUnresolvedColumnDDL: computed from the existing id
// column, no Go call site has to keep it in sync. Exists so a future
// SQL-side terminal classification (see resolveTerminalColumnDDL) can check
// "is this candidate a stub" via a plain column instead of a per-row Go
// IsStub(n.ID) call.
const isStubColumnDDL = `is_stub INTEGER GENERATED ALWAYS AS (
CASE WHEN
id LIKE 'stdlib::%' OR id LIKE 'builtin::%' OR id LIKE 'external_call::%' OR id LIKE 'module::%'
OR (
instr(id, '::') > 0 AND (
substr(id, instr(id, '::') + 2) LIKE 'stdlib::%'
OR substr(id, instr(id, '::') + 2) LIKE 'builtin::%'
OR substr(id, instr(id, '::') + 2) LIKE 'external_call::%'
OR substr(id, instr(id, '::') + 2) LIKE 'module::%'
)
)
THEN 1 ELSE 0 END
) VIRTUAL`
// nodeGeneratedColumns is the nodes-table sibling of edgeGeneratedColumns.
// Kept as its own list (and ensureNodeGeneratedColumns as its own function,
// rather than folded into ensureNodeColumns) because ensureNodeColumns
// checks existence via PRAGMA table_info, which — like the edges case
// documented on ensureEdgeColumns — silently omits generated columns.
// Reusing that function's table_info scan for is_stub would hit the exact
// same "always looks missing, ALTER re-runs, duplicate column name" bug.
var nodeGeneratedColumns = []struct {
name string
ddl string
}{
{"is_stub", isStubColumnDDL},
}
// ensureNodeGeneratedColumns adds nodeGeneratedColumns to a nodes table
// created before they existed. See ensureEdgeColumns for the table_xinfo
// vs table_info rationale this mirrors.
func ensureNodeGeneratedColumns(db *sql.DB) error {
rows, err := db.Query(`PRAGMA table_xinfo(nodes)`)
if err != nil {
return err
}
existing := make(map[string]bool)
for rows.Next() {
var (
cid, notnull, pk, hidden int
name, ctype string
dflt sql.NullString
)
if err := rows.Scan(&cid, &name, &ctype, &notnull, &dflt, &pk, &hidden); err != nil {
_ = rows.Close()
return err
}
existing[name] = true
}
if err := rows.Err(); err != nil {
_ = rows.Close()
return err
}
_ = rows.Close()
for _, c := range nodeGeneratedColumns {
if existing[c.name] {
continue
}
if _, err := db.Exec(`ALTER TABLE nodes ADD COLUMN ` + c.ddl); err != nil {
return err
}
}
return nil
}
// schemaSQL is the canonical DDL applied on Open. Statements are
// idempotent (IF NOT EXISTS) so they run cleanly against a fresh DB
// and against an existing one.
//
// Schema choices
//
// - nodes.id is the primary key; INSERT OR REPLACE on the id column
// gives idempotent re-adds with last-write-wins on every other
// column, matching the in-memory store's behaviour.
//
// - edges has a synthetic INTEGER PRIMARY KEY plus a UNIQUE
// constraint over (from_id, to_id, kind, file_path, line) -- the
// logical edge key the in-memory store uses for dedup. INSERT OR
// IGNORE on that constraint matches the in-memory "second AddEdge
// for the same key is a no-op" semantics.
//
// - meta is a JSON document (see meta_json.go). nil / empty Meta is
// stored as NULL. Four universal, hot-read node keys are promoted to
// their own nullable columns (signature / visibility / doc /
// external): they are stripped from the JSON blob on write and
// restored into Meta on read, so the in-memory map is unchanged. A
// NULL column means "not set" (legacy gob rows predate the columns
// and keep their values in the blob). Existing databases gain the
// columns via ALTER on the next Open (ensureNodeColumns).
//
// - Secondary indexes mirror the in-memory store's hot lookup paths:
// nodes_by_name -- FindNodesByName / FindNodesByNameInRepo
// nodes_by_kind -- Stats (group-by-kind)
// nodes_by_file -- GetFileNodes, EvictFile
// nodes_by_repo -- GetRepoNodes, RepoStats, EvictRepo
// (partial index -- empty repo_prefix is
// the common case and indexing it would
// be pure overhead)
// nodes_by_qual -- GetNodeByQualName, unique so duplicate
// qual_names surface as constraint errors
// edges_by_from -- GetOutEdges (kind included so RemoveEdge
// can probe by (from, kind) without a
// second hop)
// edges_by_to -- GetInEdges
const schemaSQL = `
CREATE TABLE IF NOT EXISTS nodes (
id TEXT PRIMARY KEY,
kind TEXT NOT NULL,
name TEXT NOT NULL,
qual_name TEXT NOT NULL DEFAULT '',
file_path TEXT NOT NULL,
start_line INTEGER NOT NULL DEFAULT 0,
end_line INTEGER NOT NULL DEFAULT 0,
start_column INTEGER NOT NULL DEFAULT 0,
end_column INTEGER NOT NULL DEFAULT 0,
language TEXT NOT NULL DEFAULT '',
repo_prefix TEXT NOT NULL DEFAULT '',
workspace_id TEXT NOT NULL DEFAULT '',
project_id TEXT NOT NULL DEFAULT '',
signature TEXT,
visibility TEXT,
doc TEXT,
external INTEGER,
return_type TEXT,
is_async INTEGER,
is_static INTEGER,
is_abstract INTEGER,
is_exported INTEGER,
updated_at INTEGER,
data_class TEXT,
meta BLOB
) WITHOUT ROWID;
-- nodes_by_name / _kind / _file / _repo are created from the shared
-- bulkDroppableIndexes set (see bulk_load.go), not here, so the bulk-load
-- fast path can drop and rebuild the EXACT same DDL without drift.
-- nodes_by_qual is UNIQUE — it enforces qual_name dedup on every
-- INSERT OR REPLACE, so it is never dropped and stays defined here.
CREATE UNIQUE INDEX IF NOT EXISTS nodes_by_qual ON nodes(qual_name) WHERE qual_name <> '';
CREATE TABLE IF NOT EXISTS edges (
id INTEGER PRIMARY KEY AUTOINCREMENT,
from_id TEXT NOT NULL,
to_id TEXT NOT NULL,
kind TEXT NOT NULL,
file_path TEXT NOT NULL DEFAULT '',
line INTEGER NOT NULL DEFAULT 0,
confidence REAL NOT NULL DEFAULT 1.0,
confidence_label TEXT NOT NULL DEFAULT '',
origin TEXT NOT NULL DEFAULT '',
tier TEXT NOT NULL DEFAULT '',
cross_repo INTEGER NOT NULL DEFAULT 0,
meta BLOB,
UNIQUE(from_id, to_id, kind, file_path, line)
);
-- edges_by_from / _to / _kind are created from the shared
-- bulkDroppableIndexes set (see bulk_load.go), not here, so the bulk-load
-- fast path can drop and rebuild the EXACT same DDL without drift.
-- edges_by_kind backs EdgesByKind / EdgesByKinds (resolver whole-graph
-- passes probe single kinds like provides/imports on every file save);
-- without it those are full edges-table scans — edges_by_from/to lead
-- with an id column and the partial edges_external index only covers
-- its own predicate.
CREATE TABLE IF NOT EXISTS file_mtimes (
repo_prefix TEXT NOT NULL,
file_path TEXT NOT NULL,
mtime_ns INTEGER NOT NULL,
PRIMARY KEY (repo_prefix, file_path)
) WITHOUT ROWID;
-- repo_index_state records per-repo freshness provenance written at the
-- end of a (re)index: the git revision + dirty flag the graph reflects,
-- the Merkle workspace fingerprint (Tree.Root) that gates global-pass
-- short-circuiting, node/edge counts for the index-plausibility baseline,
-- and the JSON per-language extractor versions that produced the graph.
-- One row per repo_prefix; WITHOUT ROWID — the PK index IS the table,
-- like file_mtimes / clone_shingles.
CREATE TABLE IF NOT EXISTS repo_index_state (
repo_prefix TEXT PRIMARY KEY,
indexed_sha TEXT NOT NULL DEFAULT '',
dirty INTEGER NOT NULL DEFAULT 0,
indexed_at INTEGER NOT NULL DEFAULT 0,
workspace_fp TEXT NOT NULL DEFAULT '',
node_count INTEGER NOT NULL DEFAULT 0,
edge_count INTEGER NOT NULL DEFAULT 0,
extractor_versions TEXT NOT NULL DEFAULT ''
) WITHOUT ROWID;
-- enrichment_state records, per (repo, semantic provider), the git revision
-- the graph was enriched at plus the coverage that pass reached. Enrichment
-- completion otherwise lives only in an in-memory map, so a restart forgets it
-- and re-runs full LSP hover passes for a repo whose persisted graph already
-- carries the edges. The deferred-enrichment gate reads this row and skips a
-- provider whose IndexedSHA still matches HEAD on a clean tree. One row per
-- (repo_prefix, provider); WITHOUT ROWID — the PK index IS the table, like
-- file_mtimes / repo_index_state.
CREATE TABLE IF NOT EXISTS enrichment_state (
repo_prefix TEXT NOT NULL,
provider TEXT NOT NULL,
indexed_sha TEXT NOT NULL DEFAULT '',
completed_at INTEGER NOT NULL DEFAULT 0,
coverage REAL NOT NULL DEFAULT 0,
PRIMARY KEY (repo_prefix, provider)
) WITHOUT ROWID;
-- clone_shingles is the per-symbol MinHash shingle-set sidecar. Each
-- function/method node's []uint64 shingle set is stored as a little-
-- endian BLOB (8 bytes/elem) keyed by node_id so the maintained clone-
-- detection count-min sketch can be rebuilt after a warm restart from
-- the snapshot instead of re-parsing every body. repo_prefix carries
-- the owning repo so per-repo reseeds (SELECT … WHERE repo_prefix = ?)
-- and per-repo wipes don't clobber other repos' shingle sets. node_id
-- is the PK (the join key back to nodes.id); like file_mtimes this is a
-- WITHOUT ROWID sidecar so the PK index IS the table.
CREATE TABLE IF NOT EXISTS clone_shingles (
node_id TEXT PRIMARY KEY,
repo_prefix TEXT NOT NULL DEFAULT '',
shingles BLOB
) WITHOUT ROWID;
-- constant_values is the per-KindConstant literal-value sidecar: one row
-- per constant whose RHS is a string / numeric literal, keyed by node_id
-- (the join key back to nodes.id). Lifting the value out of the JSON Meta
-- blob keeps it queryable (and out of the every-node-load decode path) so
-- the resolver can dereference a const-identifier dispatch name to its
-- value across files. file_path scopes per-file eviction on reindex;
-- repo_prefix scopes per-repo wipes. WITHOUT ROWID — the PK index IS the
-- table, like file_mtimes / clone_shingles.
CREATE TABLE IF NOT EXISTS constant_values (
node_id TEXT PRIMARY KEY,
repo_prefix TEXT NOT NULL DEFAULT '',
file_path TEXT NOT NULL DEFAULT '',
value TEXT NOT NULL DEFAULT ''
) WITHOUT ROWID;
CREATE INDEX IF NOT EXISTS constant_values_by_file ON constant_values(repo_prefix, file_path);
-- files is the per-file metadata sidecar: one row per indexed file carrying
-- the BLAKE3 content hash (the Merkle leaf), byte size, extracted node count,
-- and a JSON array of parse-error locations. The Merkle tree stays the
-- authoritative change detector; this table is queryable supplementary
-- metadata (index_health reports per-file parse errors + node counts from it).
-- PK is (repo_prefix, file_path) so a reindex replaces the row in place;
-- WITHOUT ROWID — the PK index IS the table, like file_mtimes.
CREATE TABLE IF NOT EXISTS files (
repo_prefix TEXT NOT NULL DEFAULT '',
file_path TEXT NOT NULL,
content_hash TEXT NOT NULL DEFAULT '',
size INTEGER NOT NULL DEFAULT 0,
node_count INTEGER NOT NULL DEFAULT 0,
errors TEXT NOT NULL DEFAULT '',
PRIMARY KEY (repo_prefix, file_path)
) WITHOUT ROWID;
-- files_with_errors backs the index_health "files with parse errors" rollup
-- so it scans only the (usually tiny) set of erroring files, not every row.
CREATE INDEX IF NOT EXISTS files_with_errors ON files(repo_prefix) WHERE errors <> '';
-- ref_facts is the resolved-reference sidecar: one row per reference edge
-- that resolved to a concrete target, recording the target + the provenance
-- tier that resolved it. Denormalized file_path + lang make "all reference
-- facts originating in file X" a single indexed query (the scope unit for
-- incremental re-resolution and the audit/diff surface). repo_prefix scopes
-- per-repo. PK is (repo_prefix, from_id, to_id, kind, line) so re-resolving a
-- file replaces its facts in place; WITHOUT ROWID — the PK index IS the table.
CREATE TABLE IF NOT EXISTS ref_facts (
repo_prefix TEXT NOT NULL DEFAULT '',
from_id TEXT NOT NULL,
to_id TEXT NOT NULL,
kind TEXT NOT NULL,
ref_name TEXT NOT NULL DEFAULT '',
line INTEGER NOT NULL DEFAULT 0,
origin TEXT NOT NULL DEFAULT '',
tier TEXT NOT NULL DEFAULT '',
candidates TEXT NOT NULL DEFAULT '',
file_path TEXT NOT NULL DEFAULT '',
lang TEXT NOT NULL DEFAULT '',
PRIMARY KEY (repo_prefix, from_id, to_id, kind, line)
) WITHOUT ROWID;
CREATE INDEX IF NOT EXISTS ref_facts_by_file ON ref_facts(repo_prefix, file_path);
-- ref_facts_by_target backs the reverse lookup ("which files hold a fact
-- resolving TO these symbols") that affected-by re-resolution runs when a
-- file's symbol signatures change. Without it that query is a full
-- ref_facts scan — the PK leads with from_id, not to_id.
CREATE INDEX IF NOT EXISTS ref_facts_by_target ON ref_facts(repo_prefix, to_id);
CREATE TABLE IF NOT EXISTS vectors (
node_id TEXT PRIMARY KEY,
dims INTEGER NOT NULL,
vec BLOB NOT NULL
) WITHOUT ROWID;
-- churn_enrichment is the per-node git-churn sidecar (change A: move
-- enrichment OUT of nodes.meta so the node hot path stops encoding
-- rarely-read data into the blob and get_churn_rate does an indexed read
-- instead of an AllNodes+meta-decode scan). One typed row per enriched
-- file/function/method node, keyed by node_id (join key back to
-- nodes.id); repo_prefix scopes
-- per-repo reseeds/wipes. head_sha/branch/computed_at are file-level only
-- (empty for symbols). WITHOUT ROWID: the PK index IS the table.
CREATE TABLE IF NOT EXISTS churn_enrichment (
node_id TEXT PRIMARY KEY,
repo_prefix TEXT NOT NULL DEFAULT '',
commit_count INTEGER NOT NULL DEFAULT 0,
age_days INTEGER NOT NULL DEFAULT 0,
churn_rate REAL NOT NULL DEFAULT 0,
last_author TEXT NOT NULL DEFAULT '',
last_commit_at TEXT NOT NULL DEFAULT '',
head_sha TEXT NOT NULL DEFAULT '',
branch TEXT NOT NULL DEFAULT '',
computed_at TEXT NOT NULL DEFAULT ''
) WITHOUT ROWID;
CREATE INDEX IF NOT EXISTS churn_by_repo ON churn_enrichment(repo_prefix) WHERE repo_prefix <> '';
-- coverage_enrichment: per-symbol coverage sidecar (change A). Typed
-- columns keyed by node_id; repo_prefix scopes per-repo wipes.
CREATE TABLE IF NOT EXISTS coverage_enrichment (
node_id TEXT PRIMARY KEY,
repo_prefix TEXT NOT NULL DEFAULT '',
coverage_pct REAL NOT NULL DEFAULT 0,
num_stmt INTEGER NOT NULL DEFAULT 0,
hit INTEGER NOT NULL DEFAULT 0
) WITHOUT ROWID;
CREATE INDEX IF NOT EXISTS coverage_by_repo ON coverage_enrichment(repo_prefix) WHERE repo_prefix <> '';
-- release_enrichment: per-file "added_in <tag>" sidecar (change A).
CREATE TABLE IF NOT EXISTS release_enrichment (
node_id TEXT PRIMARY KEY,
repo_prefix TEXT NOT NULL DEFAULT '',
added_in TEXT NOT NULL DEFAULT ''
) WITHOUT ROWID;
CREATE INDEX IF NOT EXISTS release_by_repo ON release_enrichment(repo_prefix) WHERE repo_prefix <> '';
-- blame_enrichment: per-symbol latest-author sidecar (change A).
CREATE TABLE IF NOT EXISTS blame_enrichment (
node_id TEXT PRIMARY KEY,
repo_prefix TEXT NOT NULL DEFAULT '',
commit_sha TEXT NOT NULL DEFAULT '',
email TEXT NOT NULL DEFAULT '',
ts INTEGER NOT NULL DEFAULT 0
) WITHOUT ROWID;
CREATE INDEX IF NOT EXISTS blame_by_repo ON blame_enrichment(repo_prefix) WHERE repo_prefix <> '';
-- symbol_fts is the FTS5 full-text index over pre-tokenised symbol
-- names. It replaces the multi-GB in-heap Bleve/BM25 index with an
-- on-disk inverted index the SymbolSearcher / SymbolBundleSearcher
-- query through. A standard (NOT contentless) FTS5 table; individual
-- rows are deleted by their FTS5 docid via the symbol_fts_rowid sidecar
-- below (node_id is UNINDEXED, so a DELETE keyed on it would full-scan
-- the index). node_id is the join key back to nodes.id; repo_prefix is
-- carried UNINDEXED so per-repo staleness wipes (DELETE … WHERE
-- repo_prefix = ?) hit a literal column without a separate b-tree.
-- Only "tokens" is indexed for matching. IF NOT EXISTS makes this
-- idempotent on every Open, so an existing .sqlite gains the vtable
-- on its next open + reindex.
CREATE VIRTUAL TABLE IF NOT EXISTS symbol_fts USING fts5(node_id UNINDEXED, repo_prefix UNINDEXED, tokens);
-- symbol_fts_rowid maps a node_id to the rowid (FTS5 docid) of its row in
-- symbol_fts. node_id is UNINDEXED in the FTS5 vtable, so deleting a node's
-- prior row with "DELETE … WHERE node_id = ?" full-scans the entire index
-- once PER symbol — quadratic on the per-edit reindex hot path. This sidecar
-- turns the delete into an O(log n) docid delete ("WHERE rowid = ?", the FTS5
-- docid IS indexed). One row per indexed symbol, keyed by node_id (the join
-- key back to nodes.id); repo_prefix scopes the per-repo wipe that
-- BulkUpsertSymbolFTS performs in lockstep with symbol_fts. WITHOUT ROWID:
-- the PK index IS the table, like file_mtimes / clone_shingles.
CREATE TABLE IF NOT EXISTS symbol_fts_rowid (
node_id TEXT PRIMARY KEY,
repo_prefix TEXT NOT NULL DEFAULT '',
fts_rowid INTEGER NOT NULL
) WITHOUT ROWID;
-- content_fts is the FTS5 full-text index over CONTENT (data_class=
-- "content") section bodies — text / pdf / pptx / xlsx chunks. It is
-- kept SEPARATE from symbol_fts so content text never enters the symbol
-- search or the code-oriented analysis passes: a content-heavy repo of a
-- few hundred large documents explodes into hundreds of thousands of
-- section nodes, and streaming their bodies here (per file, on disk)
-- instead of into symbol_fts + graph nodes keeps the code index and the
-- graph passes bounded. Only "body" is indexed for matching; node_id /
-- repo_prefix / file_path / ordinal ride UNINDEXED so the per-repo and
-- per-file staleness wipes hit literal columns without a b-tree.
CREATE VIRTUAL TABLE IF NOT EXISTS content_fts USING fts5(node_id UNINDEXED, repo_prefix UNINDEXED, file_path UNINDEXED, ordinal UNINDEXED, body);
`
@@ -0,0 +1,104 @@
package store_sqlite
import (
"database/sql"
"path/filepath"
"testing"
"github.com/stretchr/testify/require"
"github.com/zzet/gortex/internal/graph"
)
// hasNodeColumn reports whether the nodes table currently has the named column.
func hasNodeColumn(t *testing.T, db *sql.DB, col string) bool {
t.Helper()
rows, err := db.Query(`PRAGMA table_info(nodes)`)
require.NoError(t, err)
defer rows.Close()
for rows.Next() {
var (
cid, notnull, pk int
name, ctype string
dflt sql.NullString
)
require.NoError(t, rows.Scan(&cid, &name, &ctype, &notnull, &dflt, &pk))
if name == col {
return true
}
}
require.NoError(t, rows.Err())
return false
}
// TestOpenUpgradesPreDataClassStore is the backward-compatibility proof for the
// promoted data_class column: an existing v1 store written before the column
// existed must Open cleanly (ensureNodeColumns ALTERs the column in before the
// node statements are prepared), keep its rows, and immediately get the working
// SQL-level content filter — all WITHOUT a schema_version bump or a reindex.
//
// The simulated old store has data_class dropped while staying at the v1
// baseline, the exact shape of every Gortex sqlite DB already on disk before
// this change. Without the data_class entry in promotedMetaColumns, the reopen
// fails with "no such column: data_class" when prepare() builds the node
// INSERT/SELECT — so this test fails-closed if that wiring regresses.
func TestOpenUpgradesPreDataClassStore(t *testing.T) {
path := filepath.Join(t.TempDir(), "store.sqlite")
// 1. Create a current store and seed a row, then close. A fresh store is
// stamped at the current schema version; step 2 knocks it back to the v1
// baseline to simulate a store written before data_class existed.
s, err := Open(path)
require.NoError(t, err)
s.AddNode(&graph.Node{ID: "old1", Kind: graph.KindFunction, Name: "Legacy", FilePath: "f.go", RepoPrefix: "r"})
require.NoError(t, s.Close())
// 2. Simulate a store written before data_class existed: drop the column
// while leaving user_version at the v1 baseline.
withRawDB(t, path, func(db *sql.DB) {
_, err := db.Exec(`ALTER TABLE nodes DROP COLUMN data_class`)
require.NoError(t, err, "simulate a pre-data_class store")
require.False(t, hasNodeColumn(t, db, "data_class"), "data_class must be absent before the upgrade")
// A fresh Open stamps the current schema version; knock it back to the v1
// baseline so this genuinely simulates a pre-data_class store and the
// reopen exercises the in-place upgrade arm rather than a no-op.
_, err = db.Exec(`PRAGMA user_version = 1`)
require.NoError(t, err, "reset to the v1 baseline")
var v int
require.NoError(t, db.QueryRow(`PRAGMA user_version`).Scan(&v))
require.Equal(t, 1, v, "the simulated old store must sit at the v1 baseline")
})
// 3. Reopen with the current binary. ensureNodeColumns must re-add the
// column before prepare() references it, so Open succeeds without a wipe.
s2, err := Open(path)
require.NoError(t, err, "Open must upgrade a pre-data_class store in place, not fail on the missing column")
t.Cleanup(func() { _ = s2.Close() })
require.True(t, hasNodeColumn(t, s2.db, "data_class"), "ensureNodeColumns must re-add data_class on Open")
require.False(t, s2.NeedsRebuild(), "an additive-column upgrade must not signal a wipe/reindex")
// 4. Existing rows survived (the upgrade is in place, not a rebuild).
require.NotNil(t, s2.GetNode("old1"), "existing rows must survive the in-place upgrade")
// 5. A new content node persists, round-trips through the promoted column,
// and is correctly dropped by the SQL-level content filter.
s2.AddNode(&graph.Node{ID: "content1", Kind: graph.KindDoc, Name: "doc.txt::0", RepoPrefix: "r",
Meta: map[string]any{"data_class": "content", "section_text": "x"}})
s2.AddNode(&graph.Node{ID: "code1", Kind: graph.KindFunction, Name: "Foo", RepoPrefix: "r"})
content := s2.GetNode("content1")
require.NotNil(t, content)
require.Equal(t, "content", content.Meta["data_class"], "data_class round-trips via the promoted column after upgrade")
ids := map[string]bool{}
for _, n := range s2.GetRepoNonContentNodes("r") {
ids[n.ID] = true
}
require.True(t, ids["old1"], "legacy non-content node kept")
require.True(t, ids["code1"], "code node kept")
require.False(t, ids["content1"], "content node filtered at the SQL level after the upgrade")
require.Len(t, ids, 2)
}
@@ -0,0 +1,247 @@
package store_sqlite
import (
"database/sql"
"fmt"
"os"
"strings"
)
// Schema versioning for the graph store.
//
// Unlike the sidecar (which holds irreplaceable user data and must migrate in
// place), the graph store is a DERIVED CACHE: every row is reconstructable by
// re-indexing the source. So the cheapest *always-correct* reaction to a schema
// change an old on-disk DB can't satisfy is to drop the file and let the daemon
// rebuild it on the next index. A migration may therefore declare rebuild=true
// instead of writing an in-place transform that would have to re-derive the new
// data from source anyway. In-place steps remain the cheap path for purely
// mechanical changes (a new index, a denormalisation, a column with a
// computable default) that spare a large repo a multi-minute reindex.
//
// The whole mechanism keys off SQLite's built-in PRAGMA user_version, read on
// Open before schemaSQL runs. There is no separate version table.
//
// Concurrency: the daemon holds an exclusive flock on <store>.lock around Open
// (see serverstack.NewSharedServer), so reading the version, wiping the file,
// and stamping it cannot race another process. That is why — unlike the
// sidecar — this path needs no BEGIN IMMEDIATE / busy-loop handling.
// currentSchemaVersion is the version a fully-reconciled store reports via
// PRAGMA user_version. Bump it whenever schemaSQL's typed-column shape or an
// index changes in a way an old on-disk DB would not already have, and append a
// matching schemaMigrations entry describing how to bring an older store
// forward (in place, or by rebuild).
const currentSchemaVersion = 2
// schemaMigration is one forward step. Exactly one strategy applies:
// - rebuild=true: the change introduces structure/data that can only come
// from re-indexing the source; an older store is wiped and rebuilt.
// - inPlace!=nil: the change is mechanically derivable from the existing
// store and is applied in a transaction with no reindex.
//
// Steps are append-only and ascending; never edit or renumber a shipped one.
// Any inPlace step must be idempotent (IF NOT EXISTS / ADD COLUMN guarded).
type schemaMigration struct {
version int
name string
inPlace func(tx *sql.Tx) error
rebuild bool
}
// schemaMigrations is the ordered, forward-only registry. Version 1 is the
// implicit baseline (no entry): a v1 store is reconciled entirely by schemaSQL's
// idempotent CREATE ... IF NOT EXISTS plus ensureNodeColumns, so any
// pre-versioning database baseline-stamps to v1 without a rebuild. Append
// entries for version 2 and up as the schema evolves.
var schemaMigrations = []schemaMigration{
{version: 2, name: "dedupe fn-value placeholder edges", inPlace: dedupeFnValuePlaceholderEdges},
}
// dedupeFnValuePlaceholderEdges collapses duplicate function-as-value gate
// placeholder edges (graph.FnValuePlaceholderMarker, `unresolved::fnvalue::
// <name>`) to one row per (from_id, to_id), keeping the MIN(id) survivor. The
// capture path now dedups per (from, name) before it emits, but stores written
// earlier accumulated one placeholder per call site — a live store held
// millions — and EdgesWithUnresolvedTarget plus the resolver's terminal
// reconcile materialised every one on each warm restart, the dominant warmup
// heap transient this step drains. The keep set is small (tens of thousands of
// distinct pairs), so the NOT IN materialisation is cheap; the ph filter rides
// the edges_by_to(to_id) range for the bare form and the is_unresolved index for
// the multi-repo infix form. Idempotent: a second run finds no duplicates. Freed
// pages return to the freelist and are reused by later writes; the file itself
// shrinks only under a manual VACUUM, deliberately out of scope for a derived
// cache that reclaims the space on its own.
func dedupeFnValuePlaceholderEdges(tx *sql.Tx) error {
_, err := tx.Exec(`
WITH ph AS (
SELECT id, from_id, to_id FROM edges
WHERE (to_id >= 'unresolved::fnvalue::' AND to_id < 'unresolved::fnvalue:;')
OR (is_unresolved = 1 AND to_id LIKE '%::unresolved::fnvalue::%')
), keep AS (
SELECT MIN(id) AS id FROM ph GROUP BY from_id, to_id
)
DELETE FROM edges WHERE id IN (SELECT id FROM ph) AND id NOT IN (SELECT id FROM keep)`)
return err
}
// schemaPlan is the decision planSchemaMigration derives from the stored
// PRAGMA user_version. It mutates nothing on its own.
type schemaPlan struct {
wipe bool // drop the on-disk DB and rebuild from source
inPlace []schemaMigration // ordered in-place steps to run after schemaSQL
stamp bool // write currentSchemaVersion once reconciled
}
// planSchemaMigrationWith decides how to reconcile a store at the stored
// PRAGMA user_version to current, given the migration registry. It mutates
// nothing. Open passes (currentSchemaVersion, schemaMigrations); tests pass
// fixtures.
func planSchemaMigrationWith(stored, current int, migrations []schemaMigration) schemaPlan {
switch {
case stored == current:
return schemaPlan{} // up to date, nothing to do
case stored > current:
// Written by a newer build than this binary understands; the shape may
// have changed under us. For a cache the safe move is to rebuild.
return schemaPlan{wipe: true, stamp: true}
case stored == 0:
// Fresh DB, or a pre-versioning store of unknown shape. schemaSQL's
// idempotent CREATE ... IF NOT EXISTS plus ensureNodeColumns /
// ensureEdgeColumns reconcile the base shape either way, so a stored==0
// store needs a wipe only when a pending step is a REBUILD whose data can
// only come from re-indexing source. With nothing pending, stamp; with
// only in-place steps pending, run them and stamp — an in-place step is
// idempotent and mechanically derivable, so it upgrades a pre-versioning
// store in place (preserving its rows) exactly as it upgrades a known
// prior version. Wiping a stored==0 store on any migration instead would
// force every non-daemon Open (tests, read-only tools) to pass WithRebuild
// the moment the first migration ships.
pending := pendingBetween(0, current, migrations)
if len(pending) == 0 {
return schemaPlan{stamp: true}
}
if anyRebuild(pending) {
return schemaPlan{wipe: true, stamp: true}
}
return schemaPlan{inPlace: pending, stamp: true}
default: // 0 < stored < current: a known prior version
pending := pendingBetween(stored, current, migrations)
if anyRebuild(pending) {
return schemaPlan{wipe: true, stamp: true}
}
return schemaPlan{inPlace: pending, stamp: true}
}
}
func pendingBetween(stored, current int, migrations []schemaMigration) []schemaMigration {
var out []schemaMigration
for _, m := range migrations {
if m.version > stored && m.version <= current {
out = append(out, m)
}
}
return out
}
func anyRebuild(ms []schemaMigration) bool {
for _, m := range ms {
if m.rebuild {
return true
}
}
return false
}
// validateSchemaMigrations checks the registry is well-formed. A test asserts
// this against the shipped (currentSchemaVersion, schemaMigrations) so the
// dangerous mistake — bumping currentSchemaVersion without appending a matching
// entry — fails CI instead of silently baseline-stamping an un-migrated store
// to the new version at runtime. Rules:
// - versions are >= 2 (v1 is the implicit baseline, never an entry) and
// strictly ascending;
// - each step sets exactly one strategy (inPlace xor rebuild);
// - the highest version equals current, so the registry actually defines how
// to reach it. An empty registry is valid only at version 1.
func validateSchemaMigrations(current int, migs []schemaMigration) error {
if len(migs) == 0 {
if current != 1 {
return fmt.Errorf("schema version %d has no migrations: only v1 may have an empty registry", current)
}
return nil
}
prev := 0
for i, m := range migs {
if m.version < 2 {
return fmt.Errorf("migration %q has version %d: entries must be >= 2 (v1 is the implicit baseline)", m.name, m.version)
}
if i > 0 && m.version <= prev {
return fmt.Errorf("migrations must be strictly ascending: v%d (%s) does not follow v%d", m.version, m.name, prev)
}
if (m.inPlace != nil) == m.rebuild {
return fmt.Errorf("migration v%d (%s) must set exactly one of inPlace / rebuild", m.version, m.name)
}
prev = m.version
}
if prev != current {
return fmt.Errorf("highest migration version %d != currentSchemaVersion %d: a version bump needs a matching migration entry", prev, current)
}
return nil
}
// readUserVersion reads PRAGMA user_version (0 on a fresh database).
func readUserVersion(db *sql.DB) (int, error) {
var v int
if err := db.QueryRow("PRAGMA user_version").Scan(&v); err != nil {
return 0, err
}
return v, nil
}
// setUserVersion stamps the schema version. PRAGMA takes no bound parameters;
// v is an int we control, so the format is safe.
func setUserVersion(db *sql.DB, v int) error {
if _, err := db.Exec(fmt.Sprintf("PRAGMA user_version = %d", v)); err != nil {
return err
}
return nil
}
// applyInPlaceMigrations runs the in-place steps in a single transaction.
func applyInPlaceMigrations(db *sql.DB, steps []schemaMigration) error {
if len(steps) == 0 {
return nil
}
tx, err := db.Begin()
if err != nil {
return err
}
defer func() { _ = tx.Rollback() }() // no-op once Commit succeeds
for _, m := range steps {
if err := m.inPlace(tx); err != nil {
return fmt.Errorf("schema migration v%d (%s): %w", m.version, m.name, err)
}
}
return tx.Commit()
}
// removeStoreFiles deletes the SQLite database and its companions. A missing
// file is not an error. Never called for ":memory:".
//
// The suffix list covers the files the DSN's journal_mode(WAL) produces (-wal,
// -shm) plus the rollback -journal a non-WAL fallback would use; keep it in
// sync if the journal_mode in Open's DSN ever changes.
func removeStoreFiles(path string) error {
for _, suffix := range []string{"", "-wal", "-shm", "-journal"} {
if err := os.Remove(path + suffix); err != nil && !os.IsNotExist(err) {
return fmt.Errorf("remove %s: %w", path+suffix, err)
}
}
return nil
}
// isMemoryPath reports whether path is an in-process SQLite database (no file
// on disk to wipe, always built fresh by schemaSQL).
func isMemoryPath(path string) bool {
return strings.Contains(path, ":memory:")
}
@@ -0,0 +1,545 @@
package store_sqlite
import (
"database/sql"
"errors"
"path/filepath"
"strings"
"testing"
_ "modernc.org/sqlite"
)
// withRawDB opens a bare *sql.DB on path, runs fn, and closes it — used to
// simulate an on-disk store written by an older/newer build (set user_version,
// insert rows) without going through Open's reconciliation.
func withRawDB(t *testing.T, path string, fn func(db *sql.DB)) {
t.Helper()
db, err := sql.Open("sqlite", path+"?_pragma=busy_timeout(5000)")
if err != nil {
t.Fatalf("open raw db: %v", err)
}
defer db.Close()
fn(db)
}
func nodeCount(t *testing.T, db *sql.DB) int {
t.Helper()
var n int
if err := db.QueryRow("SELECT COUNT(*) FROM nodes").Scan(&n); err != nil {
t.Fatalf("count nodes: %v", err)
}
return n
}
// TestOpenStampsFreshDB: a brand-new on-disk store is stamped to the current
// schema version.
func TestOpenStampsFreshDB(t *testing.T) {
path := filepath.Join(t.TempDir(), "store.sqlite")
s, err := Open(path)
if err != nil {
t.Fatalf("Open fresh: %v", err)
}
defer s.Close()
if v, err := readUserVersion(s.db); err != nil || v != currentSchemaVersion {
t.Fatalf("fresh user_version = %d (err %v), want %d", v, err, currentSchemaVersion)
}
}
// TestOpenBaselineStampsOldDBWithoutWipe: a pre-versioning store (user_version
// 0, reconcilable to current by schemaSQL + ensureNodeColumns) is stamped in
// place — its data must survive, not be wiped.
func TestOpenBaselineStampsOldDBWithoutWipe(t *testing.T) {
path := filepath.Join(t.TempDir(), "store.sqlite")
// Create the store, then simulate a pre-versioning DB: a row + user_version 0.
s, err := Open(path)
if err != nil {
t.Fatalf("first open: %v", err)
}
if err := s.Close(); err != nil {
t.Fatalf("close: %v", err)
}
withRawDB(t, path, func(db *sql.DB) {
if _, err := db.Exec(`INSERT INTO nodes (id, kind, name, file_path) VALUES ('n1','func','Foo','f.go')`); err != nil {
t.Fatalf("seed node: %v", err)
}
if _, err := db.Exec(`PRAGMA user_version = 0`); err != nil {
t.Fatalf("reset user_version: %v", err)
}
})
s2, err := Open(path)
if err != nil {
t.Fatalf("reopen old DB: %v", err)
}
defer s2.Close()
if v, _ := readUserVersion(s2.db); v != currentSchemaVersion {
t.Fatalf("user_version after baseline = %d, want %d", v, currentSchemaVersion)
}
if n := nodeCount(t, s2.db); n != 1 {
t.Fatalf("node count after baseline = %d, want 1 (data must NOT be wiped)", n)
}
}
// TestOpenRebuildsNewerDB: a store written by a NEWER build (user_version above
// current) cannot be trusted, so Open drops and rebuilds it — the data is gone
// and the version is re-stamped to current. Proves the wipe path (and that the
// -wal/-shm companions are cleared along with the main file).
func TestOpenRebuildsNewerDB(t *testing.T) {
path := filepath.Join(t.TempDir(), "store.sqlite")
s, err := Open(path)
if err != nil {
t.Fatalf("first open: %v", err)
}
if err := s.Close(); err != nil {
t.Fatalf("close: %v", err)
}
withRawDB(t, path, func(db *sql.DB) {
if _, err := db.Exec(`INSERT INTO nodes (id, kind, name, file_path) VALUES ('n1','func','Foo','f.go')`); err != nil {
t.Fatalf("seed node: %v", err)
}
if _, err := db.Exec(`PRAGMA user_version = 999`); err != nil { // a future version this binary doesn't know
t.Fatalf("set future user_version: %v", err)
}
})
s2, err := Open(path, WithRebuild()) // simulate the daemon: holds the lock, may rebuild
if err != nil {
t.Fatalf("reopen newer DB: %v", err)
}
defer s2.Close()
if v, _ := readUserVersion(s2.db); v != currentSchemaVersion {
t.Fatalf("user_version after rebuild = %d, want %d", v, currentSchemaVersion)
}
if n := nodeCount(t, s2.db); n != 0 {
t.Fatalf("node count after rebuild = %d, want 0 (newer DB must be wiped)", n)
}
}
// TestOpenRefusesWipeWithoutOptIn: the default Open must NOT destroy an
// incompatible on-disk database. Without WithRebuild it returns
// ErrSchemaRebuildRequired and leaves the file (and its rows) intact, so a
// caller that does not hold the store lock cannot silently corrupt a store
// another process may have open.
func TestOpenRefusesWipeWithoutOptIn(t *testing.T) {
path := filepath.Join(t.TempDir(), "store.sqlite")
s, err := Open(path)
if err != nil {
t.Fatalf("first open: %v", err)
}
if _, err := s.db.Exec(`INSERT INTO nodes (id, kind, name, file_path) VALUES ('n1','func','Foo','f.go')`); err != nil {
t.Fatalf("seed node: %v", err)
}
if err := s.Close(); err != nil {
t.Fatalf("close: %v", err)
}
withRawDB(t, path, func(db *sql.DB) {
if _, err := db.Exec(`PRAGMA user_version = 999`); err != nil {
t.Fatalf("set future version: %v", err)
}
})
if _, err := Open(path); !errors.Is(err, ErrSchemaRebuildRequired) {
t.Fatalf("Open without WithRebuild = %v, want ErrSchemaRebuildRequired", err)
}
withRawDB(t, path, func(db *sql.DB) {
if n := nodeCount(t, db); n != 1 {
t.Fatalf("node count = %d after a refused wipe, want 1 (the file must be untouched)", n)
}
var v int
if err := db.QueryRow("PRAGMA user_version").Scan(&v); err != nil {
t.Fatalf("read user_version: %v", err)
}
if v != 999 {
t.Fatalf("user_version = %d after a refused wipe, want 999 (unchanged)", v)
}
})
}
// TestPlanSchemaMigration covers the pure decision logic, including the
// in-place vs rebuild dispatch a future currentSchemaVersion=2 would exercise.
func TestPlanSchemaMigration(t *testing.T) {
inPlace := schemaMigration{version: 2, name: "add-index", inPlace: func(*sql.Tx) error { return nil }}
rebuild := schemaMigration{version: 2, name: "typed-column", rebuild: true}
cases := []struct {
name string
stored, current int
migs []schemaMigration
wantWipe bool
wantStamp bool
wantInPlace int
}{
{"up to date", 1, 1, nil, false, false, 0},
{"fresh at v1 baseline-stamps", 0, 1, nil, false, true, 0},
{"newer DB rebuilds", 2, 1, nil, true, true, 0},
{"v0 with only in-place pending upgrades in place, no wipe", 0, 2, []schemaMigration{inPlace}, false, true, 1},
{"v0 with a pending rebuild wipes", 0, 2, []schemaMigration{rebuild}, true, true, 0},
{"v1->v2 in-place", 1, 2, []schemaMigration{inPlace}, false, true, 1},
{"v1->v2 rebuild", 1, 2, []schemaMigration{rebuild}, true, true, 0},
}
for _, c := range cases {
t.Run(c.name, func(t *testing.T) {
got := planSchemaMigrationWith(c.stored, c.current, c.migs)
if got.wipe != c.wantWipe || got.stamp != c.wantStamp || len(got.inPlace) != c.wantInPlace {
t.Fatalf("plan(%d->%d) = {wipe:%v stamp:%v inPlace:%d}, want {wipe:%v stamp:%v inPlace:%d}",
c.stored, c.current, got.wipe, got.stamp, len(got.inPlace), c.wantWipe, c.wantStamp, c.wantInPlace)
}
})
}
}
// TestApplyInPlaceMigrations: steps run in order and commit; a failing step
// rolls the whole transaction back.
func TestApplyInPlaceMigrations(t *testing.T) {
t.Run("commit", func(t *testing.T) {
path := filepath.Join(t.TempDir(), "m.sqlite")
withRawDB(t, path, func(db *sql.DB) {
step := schemaMigration{version: 2, name: "mk", inPlace: func(tx *sql.Tx) error {
_, err := tx.Exec(`CREATE TABLE marker (x TEXT)`)
return err
}}
if err := applyInPlaceMigrations(db, []schemaMigration{step}); err != nil {
t.Fatalf("apply: %v", err)
}
var name string
if err := db.QueryRow(`SELECT name FROM sqlite_master WHERE type='table' AND name='marker'`).Scan(&name); err != nil {
t.Fatalf("marker table not created: %v", err)
}
})
})
t.Run("rollback on failure preserves cause and rolls back every step", func(t *testing.T) {
path := filepath.Join(t.TempDir(), "m.sqlite")
withRawDB(t, path, func(db *sql.DB) {
// Two steps in one batch: the first creates table A, the second
// creates B then fails. Both must roll back, proving the steps
// share a single transaction.
stepA := schemaMigration{version: 2, name: "make-a", inPlace: func(tx *sql.Tx) error {
_, err := tx.Exec(`CREATE TABLE a (x TEXT)`)
return err
}}
stepB := schemaMigration{version: 3, name: "boom", inPlace: func(tx *sql.Tx) error {
if _, err := tx.Exec(`CREATE TABLE b (x TEXT)`); err != nil {
return err
}
return sql.ErrConnDone // synthetic failure after a partial write
}}
err := applyInPlaceMigrations(db, []schemaMigration{stepA, stepB})
if err == nil {
t.Fatal("expected applyInPlaceMigrations to surface the step error")
}
if !errors.Is(err, sql.ErrConnDone) {
t.Fatalf("error should wrap the step's cause; got %v", err)
}
if !strings.Contains(err.Error(), "v3") || !strings.Contains(err.Error(), "boom") {
t.Fatalf("error should name the failing migration (v3/boom); got %q", err.Error())
}
for _, tbl := range []string{"a", "b"} {
var name string
e := db.QueryRow(`SELECT name FROM sqlite_master WHERE type='table' AND name=?`, tbl).Scan(&name)
if e != sql.ErrNoRows {
t.Fatalf("table %q should have rolled back (shared transaction), got name=%q err=%v", tbl, name, e)
}
}
})
})
}
// TestOpenAtCurrentVersionIsNoOp covers the highest-frequency path — every
// daemon restart reopens an up-to-date store. It must be a no-op that
// preserves data; an off-by-one to wipe here would destroy the cache on every
// restart.
func TestOpenAtCurrentVersionIsNoOp(t *testing.T) {
path := filepath.Join(t.TempDir(), "store.sqlite")
s, err := Open(path)
if err != nil {
t.Fatalf("first open: %v", err)
}
withRawSeed := func(db *sql.DB) {
if _, err := db.Exec(`INSERT INTO nodes (id, kind, name, file_path) VALUES ('n1','func','Foo','f.go')`); err != nil {
t.Fatalf("seed node: %v", err)
}
}
withRawSeed(s.db)
if err := s.Close(); err != nil {
t.Fatalf("close: %v", err)
}
s2, err := Open(path)
if err != nil {
t.Fatalf("reopen at current version: %v", err)
}
defer s2.Close()
if v, _ := readUserVersion(s2.db); v != currentSchemaVersion {
t.Fatalf("user_version = %d, want %d", v, currentSchemaVersion)
}
if n := nodeCount(t, s2.db); n != 1 {
t.Fatalf("node count = %d, want 1 (a no-op reopen must NOT wipe)", n)
}
if s2.NeedsRebuild() {
t.Fatal("a no-op reopen must not signal NeedsRebuild")
}
}
// TestOpenWithInPlaceMigration drives the in-place arm end-to-end through the
// real Open composition (via the openWith seam): an older store at version 1
// is upgraded to version 2 by a registered in-place step that runs AFTER
// schemaSQL, the step's effect is visible, the existing data survives, and the
// version is stamped.
func TestOpenWithInPlaceMigration(t *testing.T) {
path := filepath.Join(t.TempDir(), "store.sqlite")
// Create a store with a row, then knock it back to the v1 baseline so the
// openWith below drives the v1->v2 in-place arm (a fresh Open now stamps the
// current version, which is >= 2).
s, err := Open(path)
if err != nil {
t.Fatalf("first open: %v", err)
}
if _, err := s.db.Exec(`INSERT INTO nodes (id, kind, name, file_path) VALUES ('n1','func','Foo','f.go')`); err != nil {
t.Fatalf("seed node: %v", err)
}
if err := s.Close(); err != nil {
t.Fatalf("close: %v", err)
}
withRawDB(t, path, func(db *sql.DB) {
if _, err := db.Exec(`PRAGMA user_version = 1`); err != nil {
t.Fatalf("reset to v1 baseline: %v", err)
}
})
// An in-place v2 step that depends on the base schema (an index on a
// nodes column) — proving it runs after schemaSQL/ensureNodeColumns.
ran := false
v2 := schemaMigration{version: 2, name: "idx-language", inPlace: func(tx *sql.Tx) error {
ran = true
_, err := tx.Exec(`CREATE INDEX IF NOT EXISTS test_nodes_by_language ON nodes(language)`)
return err
}}
s2, err := openWith(path, 2, []schemaMigration{v2}, false) // in-place never wipes
if err != nil {
t.Fatalf("openWith v2 in-place: %v", err)
}
defer s2.Close()
if !ran {
t.Fatal("the in-place migration step did not run")
}
if v, _ := readUserVersion(s2.db); v != 2 {
t.Fatalf("user_version = %d, want 2", v)
}
if n := nodeCount(t, s2.db); n != 1 {
t.Fatalf("node count = %d, want 1 (in-place upgrade must preserve data)", n)
}
var name string
if err := s2.db.QueryRow(`SELECT name FROM sqlite_master WHERE type='index' AND name='test_nodes_by_language'`).Scan(&name); err != nil {
t.Fatalf("in-place index not created: %v", err)
}
if s2.NeedsRebuild() {
t.Fatal("an in-place upgrade must not signal NeedsRebuild")
}
}
// TestOpenWithInPlaceFailureDoesNotStamp: a failing in-place step makes Open
// return an error and leaves the stored version unchanged, so the next open
// retries the upgrade rather than treating it as done.
func TestOpenWithInPlaceFailureDoesNotStamp(t *testing.T) {
path := filepath.Join(t.TempDir(), "store.sqlite")
s, err := Open(path)
if err != nil {
t.Fatalf("first open: %v", err)
}
if err := s.Close(); err != nil {
t.Fatalf("close: %v", err)
}
// A fresh Open now stamps the current version (>= 2); knock it back to the
// v1 baseline so openWith drives the v1->v2 arm and the failing step runs.
withRawDB(t, path, func(db *sql.DB) {
if _, err := db.Exec(`PRAGMA user_version = 1`); err != nil {
t.Fatalf("reset to v1 baseline: %v", err)
}
})
boom := schemaMigration{version: 2, name: "boom", inPlace: func(*sql.Tx) error {
return sql.ErrConnDone
}}
if _, err := openWith(path, 2, []schemaMigration{boom}, false); err == nil {
t.Fatal("expected openWith to fail when an in-place step errors")
}
withRawDB(t, path, func(db *sql.DB) {
var v int
if err := db.QueryRow("PRAGMA user_version").Scan(&v); err != nil {
t.Fatalf("read user_version: %v", err)
}
if v != 1 {
t.Fatalf("user_version = %d after a failed migration, want 1 (unstamped, so the next open retries)", v)
}
})
}
// TestOpenWithMemoryUnderWipePlanStampsWithoutError: an in-memory store under a
// plan that would wipe an on-disk DB must not attempt a file removal — it is
// always fresh and simply stamps the current version.
func TestOpenWithMemoryUnderWipePlanStampsWithoutError(t *testing.T) {
rebuildV2 := schemaMigration{version: 2, name: "typed-col", rebuild: true}
// stored==0, current==2, a pending rebuild => plan.wipe==true; the memory
// guard must skip the wipe and stamp anyway.
s, err := openWith(":memory:", 2, []schemaMigration{rebuildV2}, false) // memory never wipes
if err != nil {
t.Fatalf("openWith :memory: under wipe plan: %v", err)
}
defer s.Close()
if v, _ := readUserVersion(s.db); v != 2 {
t.Fatalf("user_version = %d, want 2", v)
}
if s.NeedsRebuild() {
t.Fatal(":memory: must never report a wipe (nothing to remove)")
}
}
// TestNeedsRebuildSignalAfterWipe: a store written by a newer build is wiped on
// open and reports NeedsRebuild so the daemon forces a full re-index.
func TestNeedsRebuildSignalAfterWipe(t *testing.T) {
path := filepath.Join(t.TempDir(), "store.sqlite")
s, err := Open(path)
if err != nil {
t.Fatalf("first open: %v", err)
}
if err := s.Close(); err != nil {
t.Fatalf("close: %v", err)
}
withRawDB(t, path, func(db *sql.DB) {
if _, err := db.Exec(`PRAGMA user_version = 999`); err != nil {
t.Fatalf("set future version: %v", err)
}
})
s2, err := Open(path, WithRebuild()) // daemon-equivalent: lock held, rebuild permitted
if err != nil {
t.Fatalf("reopen newer DB: %v", err)
}
defer s2.Close()
if !s2.NeedsRebuild() {
t.Fatal("a wiped store must report NeedsRebuild so the daemon re-indexes")
}
}
// TestSchemaMigrationsWellFormed asserts the shipped registry is valid and that
// the validator rejects the dangerous misconfigurations — above all, bumping
// currentSchemaVersion without appending a matching migration.
func TestSchemaMigrationsWellFormed(t *testing.T) {
if err := validateSchemaMigrations(currentSchemaVersion, schemaMigrations); err != nil {
t.Fatalf("shipped registry is invalid: %v", err)
}
inPlace := func(*sql.Tx) error { return nil }
bad := []struct {
name string
current int
migs []schemaMigration
}{
{"bumped version with no migration", 2, nil},
{"highest below current", 3, []schemaMigration{{version: 2, name: "x", rebuild: true}}},
{"both strategies set", 2, []schemaMigration{{version: 2, name: "x", rebuild: true, inPlace: inPlace}}},
{"neither strategy set", 2, []schemaMigration{{version: 2, name: "x"}}},
{"not strictly ascending", 3, []schemaMigration{{version: 2, name: "a", rebuild: true}, {version: 2, name: "b", rebuild: true}}},
{"v1 entry (baseline is implicit)", 1, []schemaMigration{{version: 1, name: "a", rebuild: true}}},
}
for _, c := range bad {
if err := validateSchemaMigrations(c.current, c.migs); err == nil {
t.Errorf("%s: expected a validation error, got nil", c.name)
}
}
}
// TestOpenDedupesFnValuePlaceholders drives the shipped v2 migration through the
// real Open composition: a store knocked back to the v1 baseline with duplicate
// fn-value placeholder edges is deduped in place on reopen — one survivor per
// (from_id, to_id), the MIN(id) row kept — while a distinct placeholder, a
// resolved edge, and an ordinary unresolved stub are untouched, and the version
// stamps to current. Covers both the bare and the multi-repo COPY-rewrite form.
func TestOpenDedupesFnValuePlaceholders(t *testing.T) {
path := filepath.Join(t.TempDir(), "store.sqlite")
s, err := Open(path)
if err != nil {
t.Fatalf("first open: %v", err)
}
// Seed edges by explicit id so the MIN(id) survivors are predictable. The
// is_unresolved column is generated, so it is omitted from the INSERT.
ins := `INSERT INTO edges (id, from_id, to_id, kind, file_path, line) VALUES (?,?,?,?,?,?)`
seed := []struct {
id int
from, to string
kind string
line int
}{
// Duplicate bare placeholders: same (from,to), distinct lines. Keep id 1.
{1, "a", "unresolved::fnvalue::handler", "references", 10},
{2, "a", "unresolved::fnvalue::handler", "references", 20},
{3, "a", "unresolved::fnvalue::handler", "references", 30},
// A distinct placeholder (different name) — must survive untouched.
{4, "a", "unresolved::fnvalue::other", "references", 10},
// Duplicate multi-repo COPY-rewrite placeholders — exercises the
// is_unresolved infix branch of the migration. Keep id 5.
{5, "b", "r::unresolved::fnvalue::handler", "references", 10},
{6, "b", "r::unresolved::fnvalue::handler", "references", 20},
// A resolved edge and an ordinary unresolved stub — never touched.
{7, "a", "b", "calls", 1},
{8, "a", "unresolved::Foo", "calls", 1},
}
for _, r := range seed {
if _, err := s.db.Exec(ins, r.id, r.from, r.to, r.kind, "f.go", r.line); err != nil {
t.Fatalf("seed edge %d: %v", r.id, err)
}
}
if err := s.Close(); err != nil {
t.Fatalf("close: %v", err)
}
// Knock the store back to the v1 baseline so the reopen runs the v2 dedup.
withRawDB(t, path, func(db *sql.DB) {
if _, err := db.Exec(`PRAGMA user_version = 1`); err != nil {
t.Fatalf("reset to v1 baseline: %v", err)
}
})
s2, err := Open(path)
if err != nil {
t.Fatalf("reopen for dedup: %v", err)
}
defer s2.Close()
if v, _ := readUserVersion(s2.db); v != currentSchemaVersion {
t.Fatalf("user_version after dedup = %d, want %d", v, currentSchemaVersion)
}
present := func(id int) bool {
var n int
if err := s2.db.QueryRow(`SELECT COUNT(*) FROM edges WHERE id = ?`, id).Scan(&n); err != nil {
t.Fatalf("count id %d: %v", id, err)
}
return n == 1
}
// Bare-form dedup keeps the MIN(id) survivor and drops the rest.
if !present(1) || present(2) || present(3) {
t.Fatalf("bare dedup wrong: want keep 1 / drop 2,3; got 1=%v 2=%v 3=%v", present(1), present(2), present(3))
}
// A distinct placeholder pair survives.
if !present(4) {
t.Fatal("distinct fn-value placeholder (id 4) was wrongly deleted")
}
// Multi-repo infix dedup keeps the MIN(id) survivor and drops the rest.
if !present(5) || present(6) {
t.Fatalf("multi-repo dedup wrong: want keep 5 / drop 6; got 5=%v 6=%v", present(5), present(6))
}
// A resolved edge and an ordinary unresolved stub must be untouched.
if !present(7) {
t.Fatal("resolved edge (id 7) must survive the placeholder dedup")
}
if !present(8) {
t.Fatal("ordinary unresolved stub (id 8) must survive the placeholder dedup")
}
}
File diff suppressed because it is too large Load Diff
@@ -0,0 +1,639 @@
package store_sqlite
import (
"iter"
"sort"
"github.com/zzet/gortex/internal/graph"
)
// This file implements the trivial SQL aggregator / scanner optional
// capability interfaces from graph.Store. Each method pushes its
// GROUP BY / WHERE / COUNT into SQLite so the planner drives it through
// the schema's secondary indexes, returning only the aggregate rows
// instead of materialising the whole node / edge table Go-side.
//
// Conventions shared across these methods:
// - Empty / nil input returns nil (parity with the in-memory store).
// - Input id / kind slices are deduped before they reach the IN-list.
// - Large IN-lists are chunked by lookupChunkSize.
// - agg-prefixed helpers are local to this file.
var (
_ graph.InEdgeCounter = (*Store)(nil)
_ graph.NodeIDsByKinds = (*Store)(nil)
_ graph.EdgeKindCounter = (*Store)(nil)
_ graph.NodeDegreeByKinds = (*Store)(nil)
_ graph.NodesInFilesByKindFinder = (*Store)(nil)
_ graph.FileImportAggregator = (*Store)(nil)
_ graph.InDegreeForNodes = (*Store)(nil)
_ graph.CrossRepoEdgeAggregator = (*Store)(nil)
_ graph.FileImporters = (*Store)(nil)
_ graph.FileSymbolNamesByPaths = (*Store)(nil)
_ graph.EdgesByKindsScanner = (*Store)(nil)
_ graph.NodesByKindsScanner = (*Store)(nil)
_ graph.EdgeAdjacencyForKinds = (*Store)(nil)
_ graph.NodeDegreeAggregator = (*Store)(nil)
_ graph.NodeFanAggregator = (*Store)(nil)
)
// aggDedupeEdgeKinds drops empties and duplicates from an edge-kind
// slice, preserving first-seen order; returns the kinds widened to the
// []any an IN-list binds.
func aggDedupeEdgeKinds(kinds []graph.EdgeKind) (uniq []graph.EdgeKind, args []any) {
seen := make(map[graph.EdgeKind]struct{}, len(kinds))
for _, k := range kinds {
if k == "" {
continue
}
if _, ok := seen[k]; ok {
continue
}
seen[k] = struct{}{}
uniq = append(uniq, k)
args = append(args, string(k))
}
return uniq, args
}
// aggDedupeNodeKinds is the node-kind twin of aggDedupeEdgeKinds.
func aggDedupeNodeKinds(kinds []graph.NodeKind) (uniq []graph.NodeKind, args []any) {
seen := make(map[graph.NodeKind]struct{}, len(kinds))
for _, k := range kinds {
if k == "" {
continue
}
if _, ok := seen[k]; ok {
continue
}
seen[k] = struct{}{}
uniq = append(uniq, k)
args = append(args, string(k))
}
return uniq, args
}
// InEdgeCountsByKind returns per-target incoming-edge counts for the
// supplied edge kinds, grouped server-side via edges_by_to.
func (s *Store) InEdgeCountsByKind(kinds []graph.EdgeKind) map[string]int {
_, args := aggDedupeEdgeKinds(kinds)
if len(args) == 0 {
return nil
}
q := `SELECT to_id, COUNT(*) FROM edges WHERE kind IN (` + inPlaceholders(len(args)) + `) GROUP BY to_id`
rows, err := s.db.Query(q, args...)
panicOnFatal(err)
if rows == nil {
// swallowed teardown-race error: read returns empty (see panicOnFatal)
return nil
}
defer rows.Close()
out := make(map[string]int)
for rows.Next() {
var id string
var n int
panicOnFatal(rows.Scan(&id, &n))
out[id] = n
}
panicOnFatal(rows.Err())
return out
}
// NodeIDsByKinds returns the deduplicated IDs of every node whose kind
// is in the supplied set.
func (s *Store) NodeIDsByKinds(kinds []graph.NodeKind) []string {
_, args := aggDedupeNodeKinds(kinds)
if len(args) == 0 {
return nil
}
q := `SELECT id FROM nodes WHERE kind IN (` + inPlaceholders(len(args)) + `) ORDER BY id`
rows, err := s.db.Query(q, args...)
panicOnFatal(err)
if rows == nil {
// swallowed teardown-race error: read returns empty (see panicOnFatal)
return nil
}
defer rows.Close()
var out []string
for rows.Next() {
var id string
panicOnFatal(rows.Scan(&id))
out = append(out, id)
}
panicOnFatal(rows.Err())
return out
}
// EdgeKindCounts returns one entry per distinct edge kind with its
// occurrence count across the whole graph.
func (s *Store) EdgeKindCounts() map[graph.EdgeKind]int {
rows, err := s.db.Query(`SELECT kind, COUNT(*) FROM edges GROUP BY kind`)
panicOnFatal(err)
if rows == nil {
// swallowed teardown-race error: read returns empty (see panicOnFatal)
return nil
}
defer rows.Close()
out := make(map[graph.EdgeKind]int)
for rows.Next() {
var kind string
var n int
panicOnFatal(rows.Scan(&kind, &n))
out[graph.EdgeKind(kind)] = n
}
panicOnFatal(rows.Err())
return out
}
// NodeDegreeByKinds returns total in/out degree for every node whose
// kind is in the set (optionally under pathPrefix); UsageInCount is
// always 0 for this capability.
func (s *Store) NodeDegreeByKinds(kinds []graph.NodeKind, pathPrefix string) []graph.NodeDegreeRow {
_, kindArgs := aggDedupeNodeKinds(kinds)
if len(kindArgs) == 0 {
return nil
}
args := append([]any(nil), kindArgs...)
q := `SELECT n.id,
(SELECT COUNT(*) FROM edges e WHERE e.to_id = n.id) AS in_count,
(SELECT COUNT(*) FROM edges e WHERE e.from_id = n.id) AS out_count
FROM nodes n
WHERE n.kind IN (` + inPlaceholders(len(kindArgs)) + `)`
if pathPrefix != "" {
q += ` AND n.file_path LIKE ? ESCAPE '\'`
args = append(args, escapeLikePattern(pathPrefix)+"%")
}
q += ` ORDER BY n.id`
rows, err := s.db.Query(q, args...)
panicOnFatal(err)
if rows == nil {
// swallowed teardown-race error: read returns empty (see panicOnFatal)
return nil
}
defer rows.Close()
var out []graph.NodeDegreeRow
for rows.Next() {
var r graph.NodeDegreeRow
panicOnFatal(rows.Scan(&r.NodeID, &r.InCount, &r.OutCount))
out = append(out, r)
}
panicOnFatal(rows.Err())
return out
}
// NodesInFilesByKind returns every node living in one of the supplied
// files whose kind is in the supplied set.
func (s *Store) NodesInFilesByKind(files []string, kinds []graph.NodeKind) []*graph.Node {
uniqFiles := dedupeNonEmpty(files)
_, kindArgs := aggDedupeNodeKinds(kinds)
if len(uniqFiles) == 0 || len(kindArgs) == 0 {
return nil
}
var out []*graph.Node
for i := 0; i < len(uniqFiles); i += lookupChunkSize {
end := minInt(i+lookupChunkSize, len(uniqFiles))
chunk := uniqFiles[i:end]
args := append(toAnyArgs(chunk), kindArgs...)
q := `SELECT ` + lookupNodeCols + ` FROM nodes WHERE file_path IN (` +
inPlaceholders(len(chunk)) + `) AND kind IN (` + inPlaceholders(len(kindArgs)) + `) ORDER BY id`
out = append(out, s.queryNodesSQL(q, args...)...)
}
return out
}
// FileImportCounts returns per-target-file incoming-import counts. A
// nil scope counts every import edge; a non-nil scope bounds counts to
// edges whose target node ID lies in the slice (empty non-nil => nil).
func (s *Store) FileImportCounts(scope []string) []graph.FileImportCountRow {
if scope != nil && len(scope) == 0 {
return nil
}
base := `SELECT COALESCE(NULLIF(n.file_path, ''), n.id) AS path, COUNT(*) AS cnt
FROM edges e JOIN nodes n ON e.to_id = n.id
WHERE e.kind = ?`
args := []any{string(graph.EdgeImports)}
fileToCount := make(map[string]int)
if scope == nil {
q := base + ` GROUP BY path`
aggScanImportCounts(s, q, args, fileToCount)
} else {
uniq := dedupeNonEmpty(scope)
if len(uniq) == 0 {
return nil
}
for i := 0; i < len(uniq); i += lookupChunkSize {
end := minInt(i+lookupChunkSize, len(uniq))
chunk := uniq[i:end]
q := base + ` AND e.to_id IN (` + inPlaceholders(len(chunk)) + `) GROUP BY path`
aggScanImportCounts(s, q, append(append([]any(nil), args...), toAnyArgs(chunk)...), fileToCount)
}
}
if len(fileToCount) == 0 {
return nil
}
out := make([]graph.FileImportCountRow, 0, len(fileToCount))
for path, cnt := range fileToCount {
out = append(out, graph.FileImportCountRow{FilePath: path, Count: cnt})
}
return out
}
// aggScanImportCounts runs an import-count query and folds the (path,
// count) rows into the accumulator (chunked scopes can revisit a path).
func aggScanImportCounts(s *Store, q string, args []any, acc map[string]int) {
rows, err := s.db.Query(q, args...)
panicOnFatal(err)
if rows == nil {
// swallowed teardown-race error: read returns empty (see panicOnFatal)
return
}
defer rows.Close()
for rows.Next() {
var path string
var cnt int
panicOnFatal(rows.Scan(&path, &cnt))
acc[path] += cnt
}
panicOnFatal(rows.Err())
}
// InDegreeForNodes returns total incoming-edge counts (any kind) for
// the supplied node id set.
func (s *Store) InDegreeForNodes(ids []string) map[string]int {
uniq := dedupeNonEmpty(ids)
if len(uniq) == 0 {
return nil
}
out := make(map[string]int)
for i := 0; i < len(uniq); i += lookupChunkSize {
end := minInt(i+lookupChunkSize, len(uniq))
chunk := uniq[i:end]
q := `SELECT to_id, COUNT(*) FROM edges WHERE to_id IN (` +
inPlaceholders(len(chunk)) + `) GROUP BY to_id`
rows, err := s.db.Query(q, toAnyArgs(chunk)...)
panicOnFatal(err)
if rows == nil {
// swallowed teardown-race error: read returns empty (see panicOnFatal)
return out
}
for rows.Next() {
var id string
var n int
panicOnFatal(rows.Scan(&id, &n))
out[id] = n
}
panicOnFatal(rows.Err())
_ = rows.Close()
}
return out
}
// CrossRepoEdgeCounts returns pre-grouped cross-repo edge counts keyed
// by (base kind, from-repo, to-repo). Cross-repo kinds are those
// graph.BaseKindForCrossRepo recognises; the count is reported under
// the base kind.
func (s *Store) CrossRepoEdgeCounts() []graph.CrossRepoEdgeRow {
q := `SELECT e.kind, nf.repo_prefix, nt.repo_prefix, COUNT(*)
FROM edges e
JOIN nodes nf ON e.from_id = nf.id
JOIN nodes nt ON e.to_id = nt.id
WHERE nf.repo_prefix <> nt.repo_prefix
GROUP BY e.kind, nf.repo_prefix, nt.repo_prefix`
rows, err := s.db.Query(q)
panicOnFatal(err)
if rows == nil {
// swallowed teardown-race error: read returns empty (see panicOnFatal)
return nil
}
defer rows.Close()
// Aggregate keyed by the edge's OWN kind (cross_repo_*), NOT the base.
// BaseKindForCrossRepo is used only as the recogniser that decides
// whether an edge participates — parity with the in-memory store.
type key struct {
kind graph.EdgeKind
from string
to string
}
acc := make(map[key]int)
for rows.Next() {
var kind, from, to string
var n int
panicOnFatal(rows.Scan(&kind, &from, &to, &n))
ek := graph.EdgeKind(kind)
if _, ok := graph.BaseKindForCrossRepo(ek); !ok {
continue
}
acc[key{kind: ek, from: from, to: to}] += n
}
panicOnFatal(rows.Err())
if len(acc) == 0 {
return nil
}
out := make([]graph.CrossRepoEdgeRow, 0, len(acc))
for k, n := range acc {
out = append(out, graph.CrossRepoEdgeRow{Kind: k.kind, FromRepo: k.from, ToRepo: k.to, Count: n})
}
return out
}
// FileImporters returns the importing-node rows for every EdgeImports
// edge whose target's FilePath OR ID equals filePath.
func (s *Store) FileImporters(filePath string) []graph.FileImporterRow {
if filePath == "" {
return nil
}
q := `SELECT nf.file_path, nf.id, nf.name, nf.kind
FROM edges e
JOIN nodes nt ON e.to_id = nt.id
JOIN nodes nf ON e.from_id = nf.id
WHERE e.kind = ? AND (nt.file_path = ? OR nt.id = ?)
ORDER BY nf.file_path`
rows, err := s.db.Query(q, string(graph.EdgeImports), filePath, filePath)
panicOnFatal(err)
if rows == nil {
// swallowed teardown-race error: read returns empty (see panicOnFatal)
return nil
}
defer rows.Close()
var out []graph.FileImporterRow
for rows.Next() {
var r graph.FileImporterRow
var kind string
panicOnFatal(rows.Scan(&r.FromFile, &r.FromID, &r.FromName, &kind))
r.FromKind = graph.NodeKind(kind)
out = append(out, r)
}
panicOnFatal(rows.Err())
return out
}
// FileSymbolNamesByPaths returns the distinct (file, name) pairs for
// nodes in the supplied paths whose kind is in the set, sorted by
// (file, name).
func (s *Store) FileSymbolNamesByPaths(paths []string, kinds []graph.NodeKind) []graph.FileSymbolNameRow {
uniqPaths := dedupeNonEmpty(paths)
_, kindArgs := aggDedupeNodeKinds(kinds)
if len(uniqPaths) == 0 || len(kindArgs) == 0 {
return nil
}
var out []graph.FileSymbolNameRow
for i := 0; i < len(uniqPaths); i += lookupChunkSize {
end := minInt(i+lookupChunkSize, len(uniqPaths))
chunk := uniqPaths[i:end]
args := append(toAnyArgs(chunk), kindArgs...)
q := `SELECT DISTINCT file_path, name FROM nodes WHERE file_path IN (` +
inPlaceholders(len(chunk)) + `) AND kind IN (` + inPlaceholders(len(kindArgs)) + `)`
rows, err := s.db.Query(q, args...)
panicOnFatal(err)
if rows == nil {
// swallowed teardown-race error: read returns empty (see panicOnFatal)
return out
}
for rows.Next() {
var r graph.FileSymbolNameRow
panicOnFatal(rows.Scan(&r.FilePath, &r.Name))
out = append(out, r)
}
panicOnFatal(rows.Err())
_ = rows.Close()
}
sort.Slice(out, func(i, j int) bool {
if out[i].FilePath != out[j].FilePath {
return out[i].FilePath < out[j].FilePath
}
return out[i].Name < out[j].Name
})
return out
}
// EdgesByKinds streams every edge whose kind is in the supplied set;
// honours early-stop. Empty kinds yields nothing.
func (s *Store) EdgesByKinds(kinds []graph.EdgeKind) iter.Seq[*graph.Edge] {
_, args := aggDedupeEdgeKinds(kinds)
return func(yield func(*graph.Edge) bool) {
if len(args) == 0 {
return
}
q := `SELECT ` + lookupEdgeCols + ` FROM edges WHERE kind IN (` +
inPlaceholders(len(args)) + `) ORDER BY id`
for _, e := range s.queryEdgesSQL(q, args...) {
if e == nil {
continue
}
if !yield(e) {
return
}
}
}
}
// externalCallTargetPredicate selects edges whose target is an
// external-package terminal (dep:: / stdlib:: / external::, including the
// per-repo-prefixed stdlib form) or an already-materialised
// external-call:: node. Shared verbatim by ExternalCallCandidateEdges and
// the edges_external partial index (schema.go) so SQLite matches the
// partial index for the query — keep the two identical.
const externalCallTargetPredicate = `(to_id GLOB 'dep::*' OR to_id GLOB 'external::*' OR to_id GLOB 'stdlib::*' OR to_id GLOB '*::stdlib::*' OR to_id GLOB 'external-call::*')`
// ExternalCallCandidateEdges implements graph.ExternalCallCandidates: it
// returns only the call / reference edges the external-call synthesizer
// might act on, selected server-side so the whole call-edge table never
// crosses into Go just to be prefix-filtered. The GLOB predicate is
// served by the edges_external partial index.
func (s *Store) ExternalCallCandidateEdges() []*graph.Edge {
q := `SELECT ` + lookupEdgeCols + ` FROM edges
WHERE kind IN ('calls','references') AND ` + externalCallTargetPredicate + `
ORDER BY id`
return s.queryEdgesSQL(q)
}
// NodesByKinds returns every node whose kind is in the supplied set.
func (s *Store) NodesByKinds(kinds []graph.NodeKind) []*graph.Node {
_, args := aggDedupeNodeKinds(kinds)
if len(args) == 0 {
return nil
}
q := `SELECT ` + lookupNodeCols + ` FROM nodes WHERE kind IN (` +
inPlaceholders(len(args)) + `) ORDER BY id`
return s.queryNodesSQL(q, args...)
}
// EdgeAdjacencyForKinds streams (from, to) id pairs for edges whose
// kind is in edgeKinds and whose endpoints both have a kind in
// nodeKinds; honours early-stop. Empty kinds yields nothing.
func (s *Store) EdgeAdjacencyForKinds(edgeKinds []graph.EdgeKind, nodeKinds []graph.NodeKind) iter.Seq[[2]string] {
_, eArgs := aggDedupeEdgeKinds(edgeKinds)
_, nArgs := aggDedupeNodeKinds(nodeKinds)
return func(yield func([2]string) bool) {
if len(eArgs) == 0 || len(nArgs) == 0 {
return
}
args := append([]any(nil), eArgs...)
args = append(args, nArgs...)
args = append(args, nArgs...)
q := `SELECT e.from_id, e.to_id
FROM edges e
JOIN nodes nf ON e.from_id = nf.id
JOIN nodes nt ON e.to_id = nt.id
WHERE e.kind IN (` + inPlaceholders(len(eArgs)) + `)
AND nf.kind IN (` + inPlaceholders(len(nArgs)) + `)
AND nt.kind IN (` + inPlaceholders(len(nArgs)) + `)`
rows, err := s.db.Query(q, args...)
panicOnFatal(err)
if rows == nil {
// swallowed teardown-race error: read returns empty (see panicOnFatal)
return
}
defer rows.Close()
for rows.Next() {
var from, to string
panicOnFatal(rows.Scan(&from, &to))
if !yield([2]string{from, to}) {
return
}
}
panicOnFatal(rows.Err())
}
}
// NodeDegreeCounts returns per-node in/out/usage-in edge counts for the
// supplied id set. Unknown ids produce no row; duplicates collapse.
func (s *Store) NodeDegreeCounts(ids []string, usageKinds []graph.EdgeKind) []graph.NodeDegreeRow {
uniq := dedupeNonEmpty(ids)
if len(uniq) == 0 {
return nil
}
_, usageArgs := aggDedupeEdgeKinds(usageKinds)
out := make([]graph.NodeDegreeRow, 0, len(uniq))
for i := 0; i < len(uniq); i += lookupChunkSize {
end := minInt(i+lookupChunkSize, len(uniq))
chunk := uniq[i:end]
// Usage-in subquery: a literal 0 when no usage kinds are given.
usageExpr := `0`
var usageInline []any
if len(usageArgs) > 0 {
usageExpr = `(SELECT COUNT(*) FROM edges e WHERE e.to_id = n.id AND e.kind IN (` +
inPlaceholders(len(usageArgs)) + `))`
usageInline = usageArgs
}
q := `SELECT n.id,
(SELECT COUNT(*) FROM edges e WHERE e.to_id = n.id) AS in_count,
(SELECT COUNT(*) FROM edges e WHERE e.from_id = n.id) AS out_count,
` + usageExpr + ` AS usage_in
FROM nodes n
WHERE n.id IN (` + inPlaceholders(len(chunk)) + `)`
// Bind order matches placeholder order: usage subquery first
// (it appears earlier in the SELECT list), then the id IN-list.
args := append(append([]any(nil), usageInline...), toAnyArgs(chunk)...)
rows, err := s.db.Query(q, args...)
panicOnFatal(err)
if rows == nil {
// swallowed teardown-race error: read returns empty (see panicOnFatal)
return out
}
for rows.Next() {
var r graph.NodeDegreeRow
panicOnFatal(rows.Scan(&r.NodeID, &r.InCount, &r.OutCount, &r.UsageInCount))
out = append(out, r)
}
panicOnFatal(rows.Err())
_ = rows.Close()
}
return out
}
// NodeFanCounts returns per-node fan-in (incoming edges in fanInKinds)
// and fan-out (outgoing edges in fanOutKinds) for the supplied id set.
// Unknown ids produce no row; duplicates collapse.
func (s *Store) NodeFanCounts(ids []string, fanInKinds, fanOutKinds []graph.EdgeKind) []graph.NodeFanRow {
uniq := dedupeNonEmpty(ids)
if len(uniq) == 0 {
return nil
}
_, inArgs := aggDedupeEdgeKinds(fanInKinds)
_, outArgs := aggDedupeEdgeKinds(fanOutKinds)
out := make([]graph.NodeFanRow, 0, len(uniq))
for i := 0; i < len(uniq); i += lookupChunkSize {
end := minInt(i+lookupChunkSize, len(uniq))
chunk := uniq[i:end]
fanInExpr := `0`
var inInline []any
if len(inArgs) > 0 {
fanInExpr = `(SELECT COUNT(*) FROM edges e WHERE e.to_id = n.id AND e.kind IN (` +
inPlaceholders(len(inArgs)) + `))`
inInline = inArgs
}
fanOutExpr := `0`
var outInline []any
if len(outArgs) > 0 {
fanOutExpr = `(SELECT COUNT(*) FROM edges e WHERE e.from_id = n.id AND e.kind IN (` +
inPlaceholders(len(outArgs)) + `))`
outInline = outArgs
}
q := `SELECT n.id, ` + fanInExpr + ` AS fan_in, ` + fanOutExpr + ` AS fan_out
FROM nodes n
WHERE n.id IN (` + inPlaceholders(len(chunk)) + `)`
// Bind order matches placeholder order in the SELECT list:
// fan-in subquery, fan-out subquery, then the id IN-list.
args := append([]any(nil), inInline...)
args = append(args, outInline...)
args = append(args, toAnyArgs(chunk)...)
rows, err := s.db.Query(q, args...)
panicOnFatal(err)
if rows == nil {
// swallowed teardown-race error: read returns empty (see panicOnFatal)
return out
}
for rows.Next() {
var r graph.NodeFanRow
panicOnFatal(rows.Scan(&r.NodeID, &r.FanIn, &r.FanOut))
out = append(out, r)
}
panicOnFatal(rows.Err())
_ = rows.Close()
}
return out
}
// CommunityCrossingsByKind returns per-source crossing counts for edges
// whose kind is in the supplied set, given a node→community map. A
// crossing is an edge whose source community differs from its target
// community; zero-count sources are dropped. Empty kinds or empty
// community map returns nil. The community comparison runs Go-side
// because community membership is not a node column.
func (s *Store) CommunityCrossingsByKind(kinds []graph.EdgeKind, nodeToComm map[string]string) map[string]int {
_, args := aggDedupeEdgeKinds(kinds)
if len(args) == 0 || len(nodeToComm) == 0 {
return nil
}
q := `SELECT from_id, to_id FROM edges WHERE kind IN (` + inPlaceholders(len(args)) + `)`
rows, err := s.db.Query(q, args...)
panicOnFatal(err)
if rows == nil {
// swallowed teardown-race error: read returns empty (see panicOnFatal)
return nil
}
defer rows.Close()
out := make(map[string]int)
for rows.Next() {
var from, to string
panicOnFatal(rows.Scan(&from, &to))
fromComm, ok := nodeToComm[from]
if !ok {
continue
}
toComm, ok := nodeToComm[to]
if !ok {
continue
}
if fromComm != toComm {
out[from]++
}
}
panicOnFatal(rows.Err())
if len(out) == 0 {
return nil
}
return out
}
@@ -0,0 +1,500 @@
package store_sqlite
// This file implements the moderate-SQL analysis capability interfaces
// for the SQLite graph.Store backend. Each method mirrors the in-memory
// reference implementation in internal/graph/graph.go and is verified
// against the same conformance suite (internal/graph/storetest).
//
// Shape: push the structural filter into one indexed SELECT via the raw-
// SQL helpers (queryNodesSQL / s.db.Query), then do any Meta-dependent
// (JSON-decoded) or distinct-counting filtering in Go. No new prepared
// statements are added — every query rides the secondary indexes already
// created in schema.go (edges_by_from / edges_by_to / nodes_by_kind).
import (
"github.com/zzet/gortex/internal/graph"
)
// Compile-time assertions: *Store satisfies each analysis capability.
var _ graph.DeadCodeCandidator = (*Store)(nil)
var _ graph.IfaceImplementsScanner = (*Store)(nil)
var _ graph.MemberMethodsByType = (*Store)(nil)
var _ graph.StructuralParentEdges = (*Store)(nil)
var _ graph.ExtractCandidatesScanner = (*Store)(nil)
var _ graph.CrossRepoCandidates = (*Store)(nil)
var _ graph.ThrowerErrorSurfacer = (*Store)(nil)
// anaDedupeEdgeKinds drops empty / duplicate edge kinds, preserving
// first-seen order — the EdgeKind twin of dedupeNonEmpty.
func anaDedupeEdgeKinds(in []graph.EdgeKind) []graph.EdgeKind {
seen := make(map[graph.EdgeKind]struct{}, len(in))
out := make([]graph.EdgeKind, 0, len(in))
for _, k := range in {
if k == "" {
continue
}
if _, ok := seen[k]; ok {
continue
}
seen[k] = struct{}{}
out = append(out, k)
}
return out
}
// --- DeadCodeCandidator -------------------------------------------------
// DeadCodeCandidates returns nodes of the allowed kinds that have no
// incoming edge of the corresponding allowed in-edge kinds. An empty
// per-kind allowlist (or one that dedupes to nothing) means "any incoming
// edge counts as usage". Mirrors graph.(*Graph).DeadCodeCandidates: the
// candidate set is purely structural (the analysis layer applies the
// exported / test / entry-point / synthetic post-filters in Go), so no
// node-id exclusion happens here. The NOT-EXISTS filter runs server-side
// per node kind.
func (s *Store) DeadCodeCandidates(allowedNodeKinds []graph.NodeKind, allowedInEdgeKinds map[graph.NodeKind][]graph.EdgeKind) []*graph.Node {
if len(allowedNodeKinds) == 0 {
return nil
}
var out []*graph.Node
for _, nk := range allowedNodeKinds {
allowed := anaDedupeEdgeKinds(allowedInEdgeKinds[nk])
anyKindCounts := len(allowed) == 0
var q string
var args []any
if anyKindCounts {
// Any incoming edge disqualifies the node.
q = `SELECT ` + lookupNodeCols + ` FROM nodes n
WHERE n.kind = ?
AND NOT EXISTS (SELECT 1 FROM edges e WHERE e.to_id = n.id)
ORDER BY n.id`
args = []any{string(nk)}
} else {
// Only an incoming edge of one of the allowed kinds counts.
q = `SELECT ` + lookupNodeCols + ` FROM nodes n
WHERE n.kind = ?
AND NOT EXISTS (SELECT 1 FROM edges e WHERE e.to_id = n.id AND e.kind IN (` + inPlaceholders(len(allowed)) + `))
ORDER BY n.id`
args = make([]any, 0, 1+len(allowed))
args = append(args, string(nk))
for _, ek := range allowed {
args = append(args, string(ek))
}
}
for _, n := range s.queryNodesSQL(q, args...) {
if n != nil {
out = append(out, n)
}
}
}
return out
}
// --- IfaceImplementsScanner ---------------------------------------------
// IfaceImplementsRows returns one row per EdgeImplements edge whose
// target is a KindInterface carrying Meta["methods"]. The interface's
// decoded Meta rides on the row (callers pull the "methods" field, which
// round-trips as []string or []any). Interfaces with no Meta or no
// "methods" key are elided server-side.
func (s *Store) IfaceImplementsRows() []graph.IfaceImplementsRow {
q := `SELECT e.from_id, n.id, n.meta
FROM edges e
JOIN nodes n ON n.id = e.to_id
WHERE e.kind = ? AND n.kind = ? AND n.meta IS NOT NULL`
rows, err := s.db.Query(q, string(graph.EdgeImplements), string(graph.KindInterface))
if err != nil {
return nil
}
defer rows.Close()
var out []graph.IfaceImplementsRow
for rows.Next() {
var fromID, ifaceID string
var metaBlob []byte
if err := rows.Scan(&fromID, &ifaceID, &metaBlob); err != nil {
continue
}
meta, derr := decodeMeta(metaBlob)
if derr != nil || meta == nil {
continue
}
if _, ok := meta["methods"]; !ok {
continue
}
out = append(out, graph.IfaceImplementsRow{
TypeID: fromID,
IfaceID: ifaceID,
IfaceMeta: meta,
})
}
return out
}
// --- MemberMethodsByType ------------------------------------------------
// MemberMethodsByType returns typeID → []MemberMethodInfo for every
// EdgeMemberOf edge whose source is a KindMethod. The columns come from
// the METHOD NODE (FilePath / StartLine / RepoPrefix), matching the
// in-memory reference. Per-type lists are deduplicated by MethodID; the
// scan is ordered by the edge PK so the first-seen winner is stable. An
// empty graph (no qualifying rows) returns nil.
func (s *Store) MemberMethodsByType() map[string][]graph.MemberMethodInfo {
q := `SELECT e.to_id, n.id, n.name, n.file_path, n.start_line, n.repo_prefix
FROM edges e
JOIN nodes n ON n.id = e.from_id
WHERE e.kind = ? AND n.kind = ?
ORDER BY e.id`
rows, err := s.db.Query(q, string(graph.EdgeMemberOf), string(graph.KindMethod))
if err != nil {
return nil
}
defer rows.Close()
out := make(map[string][]graph.MemberMethodInfo)
seen := make(map[string]map[string]struct{})
for rows.Next() {
var typeID, methodID, name, filePath, repoPrefix string
var startLine int
if err := rows.Scan(&typeID, &methodID, &name, &filePath, &startLine, &repoPrefix); err != nil {
continue
}
if seen[typeID] == nil {
seen[typeID] = make(map[string]struct{})
}
if _, ok := seen[typeID][methodID]; ok {
continue
}
seen[typeID][methodID] = struct{}{}
out[typeID] = append(out[typeID], graph.MemberMethodInfo{
MethodID: methodID,
Name: name,
FilePath: filePath,
StartLine: startLine,
RepoPrefix: repoPrefix,
})
}
if len(out) == 0 {
// Match the in-memory reference: empty graph returns nil.
return nil
}
return out
}
// --- StructuralParentEdges ----------------------------------------------
// StructuralParentEdges returns every Extends / Implements / Composes
// edge whose endpoints are both Type / Interface, projected as (FromID,
// ToID, FromKind, ToKind, Origin). Endpoints that aren't both type /
// interface are filtered server-side. Empty graph or no matching edges
// returns nil.
func (s *Store) StructuralParentEdges() []graph.StructuralParentEdgeRow {
q := `SELECT e.from_id, e.to_id, nf.kind, nt.kind, e.origin
FROM edges e
JOIN nodes nf ON nf.id = e.from_id
JOIN nodes nt ON nt.id = e.to_id
WHERE e.kind IN (?,?,?)
AND nf.kind IN (?,?) AND nt.kind IN (?,?)
ORDER BY e.id`
rows, err := s.db.Query(q,
string(graph.EdgeExtends), string(graph.EdgeImplements), string(graph.EdgeComposes),
string(graph.KindType), string(graph.KindInterface),
string(graph.KindType), string(graph.KindInterface),
)
if err != nil {
return nil
}
defer rows.Close()
var out []graph.StructuralParentEdgeRow
for rows.Next() {
var fromID, toID, fromKind, toKind, origin string
if err := rows.Scan(&fromID, &toID, &fromKind, &toKind, &origin); err != nil {
continue
}
out = append(out, graph.StructuralParentEdgeRow{
FromID: fromID,
ToID: toID,
FromKind: graph.NodeKind(fromKind),
ToKind: graph.NodeKind(toKind),
Origin: origin,
})
}
return out
}
// --- ExtractCandidatesScanner -------------------------------------------
// ExtractCandidates ranks function / method nodes by extractability: line
// span (EndLine - StartLine + 1), distinct caller fan-in, and distinct
// callee fan-out, counting only edges whose kind is in the supplied set.
// Rows must clear all three thresholds. Nodes with a zero StartLine /
// EndLine are dropped; pathPrefix narrows by file-path prefix. Mirrors
// graph.(*Graph).ExtractCandidates exactly: only KindFunction +
// KindMethod nodes are considered, and the distinct-by-endpoint counting
// runs Go-side over GetInEdges / GetOutEdges.
func (s *Store) ExtractCandidates(kinds []graph.EdgeKind, minLines, minCallers, minFanOut int, pathPrefix string) []graph.ExtractCandidateRow {
if len(kinds) == 0 {
return nil
}
kindSet := make(map[graph.EdgeKind]struct{}, len(kinds))
for _, k := range kinds {
if k == "" {
continue
}
kindSet[k] = struct{}{}
}
if len(kindSet) == 0 {
return nil
}
// Candidate nodes: function / method only, non-zero line span,
// optional path-prefix gate.
q := `SELECT ` + lookupNodeCols + ` FROM nodes
WHERE kind IN (?,?) AND start_line > 0 AND end_line > 0`
args := []any{string(graph.KindFunction), string(graph.KindMethod)}
if pathPrefix != "" {
q += ` AND file_path LIKE ? ESCAPE '\'`
args = append(args, escapeLikePattern(pathPrefix)+"%")
}
q += ` ORDER BY id`
nodes := s.queryNodesSQL(q, args...)
var out []graph.ExtractCandidateRow
for _, n := range nodes {
if n == nil {
continue
}
lineCount := n.EndLine - n.StartLine + 1
if lineCount < minLines {
continue
}
callerSet := make(map[string]struct{})
for _, e := range s.GetInEdges(n.ID) {
if e == nil {
continue
}
if _, ok := kindSet[e.Kind]; !ok {
continue
}
callerSet[e.From] = struct{}{}
}
if len(callerSet) < minCallers {
continue
}
calleeSet := make(map[string]struct{})
for _, e := range s.GetOutEdges(n.ID) {
if e == nil {
continue
}
if _, ok := kindSet[e.Kind]; !ok {
continue
}
calleeSet[e.To] = struct{}{}
}
if len(calleeSet) < minFanOut {
continue
}
out = append(out, graph.ExtractCandidateRow{
NodeID: n.ID,
Name: n.Name,
FilePath: n.FilePath,
StartLine: n.StartLine,
EndLine: n.EndLine,
LineCount: lineCount,
CallerCount: len(callerSet),
FanOut: len(calleeSet),
})
}
return out
}
// --- CrossRepoCandidates ------------------------------------------------
// CrossRepoCandidates returns every edge whose kind is in baseKinds and
// whose endpoints carry two different non-empty RepoPrefix values. The
// edge is returned verbatim (callers rewrite Edge.CrossRepo); FromRepo /
// ToRepo are the endpoint prefixes. Empty baseKinds returns nil; single-
// repo graphs (or graphs whose nodes carry no RepoPrefix) yield nothing.
func (s *Store) CrossRepoCandidates(baseKinds []graph.EdgeKind) []graph.CrossRepoCandidateRow {
uniq := anaDedupeEdgeKinds(baseKinds)
if len(uniq) == 0 {
return nil
}
args := make([]any, 0, len(uniq))
for _, k := range uniq {
args = append(args, string(k))
}
q := `SELECT e.from_id, e.to_id, e.kind, e.file_path, e.line,
e.confidence, e.confidence_label, e.origin, e.tier, e.cross_repo, e.meta,
nf.repo_prefix, nt.repo_prefix
FROM edges e
JOIN nodes nf ON nf.id = e.from_id
JOIN nodes nt ON nt.id = e.to_id
WHERE e.kind IN (` + inPlaceholders(len(uniq)) + `)
AND nf.repo_prefix <> '' AND nt.repo_prefix <> ''
AND nf.repo_prefix <> nt.repo_prefix
ORDER BY e.id`
rows, err := s.db.Query(q, args...)
if err != nil {
return nil
}
defer rows.Close()
var out []graph.CrossRepoCandidateRow
for rows.Next() {
var (
fromRepo, toRepo string
e graph.Edge
metaBlob []byte
crossRepo int64
)
if err := rows.Scan(
&e.From, &e.To, &e.Kind, &e.FilePath, &e.Line,
&e.Confidence, &e.ConfidenceLabel, &e.Origin, &e.Tier,
&crossRepo, &metaBlob,
&fromRepo, &toRepo,
); err != nil {
continue
}
e.CrossRepo = crossRepo != 0
if len(metaBlob) > 0 {
if m, derr := decodeMeta(metaBlob); derr == nil {
e.Meta = m
}
}
edge := e
out = append(out, graph.CrossRepoCandidateRow{
Edge: &edge,
FromRepo: fromRepo,
ToRepo: toRepo,
})
}
return out
}
// --- ThrowerErrorSurfacer -----------------------------------------------
// ThrowerErrorSurface returns one row per thrower (a node with outgoing
// EdgeThrows edges), aggregating the distinct error targets and the
// distinct literal error-message strings it emits (KindString nodes with
// Meta["context"] == "error_msg", linked by EdgeEmits). pathPrefix gates
// the EdgeThrows rows by their stored FilePath prefix. Throws counts the
// underlying EdgeThrows edges; FilePath / Line seed from the first throws
// edge, falling back to the thrower node's own coordinates when the edge
// carries none — matching the in-memory reference.
func (s *Store) ThrowerErrorSurface(pathPrefix string) []graph.ThrowerErrorRow {
type rowAccum struct {
row graph.ThrowerErrorRow
targetSeen map[string]struct{}
msgSeen map[string]struct{}
}
accums := make(map[string]*rowAccum)
var order []string
// Pass 1: EdgeThrows aggregation (count + distinct targets), keyed by
// thrower. The first edge (by PK insertion order) seeds FilePath /
// Line; an empty edge file/line falls back to the thrower node.
tq := `SELECT from_id, to_id, file_path, line FROM edges WHERE kind = ?`
targs := []any{string(graph.EdgeThrows)}
if pathPrefix != "" {
tq += ` AND file_path LIKE ? ESCAPE '\'`
targs = append(targs, escapeLikePattern(pathPrefix)+"%")
}
tq += ` ORDER BY id`
trows, err := s.db.Query(tq, targs...)
if err != nil {
return nil
}
for trows.Next() {
var from, to, filePath string
var line int
if err := trows.Scan(&from, &to, &filePath, &line); err != nil {
continue
}
acc := accums[from]
if acc == nil {
file := filePath
ln := line
if file == "" || ln == 0 {
if n := s.GetNode(from); n != nil {
if file == "" {
file = n.FilePath
}
if ln == 0 {
ln = n.StartLine
}
}
}
acc = &rowAccum{
row: graph.ThrowerErrorRow{
ThrowerID: from,
FilePath: file,
Line: ln,
},
targetSeen: make(map[string]struct{}),
msgSeen: make(map[string]struct{}),
}
accums[from] = acc
order = append(order, from)
}
acc.row.Throws++
if _, ok := acc.targetSeen[to]; !ok {
acc.targetSeen[to] = struct{}{}
acc.row.ErrorTargets = append(acc.row.ErrorTargets, to)
}
}
_ = trows.Close()
if len(accums) == 0 {
return nil
}
// Pass 2: attach the literal error messages each thrower emits. Join
// each thrower's EdgeEmits out-edges to KindString targets and filter
// Meta["context"] == "error_msg" Go-side (the context lives in the
// JSON Meta blob).
for _, id := range order {
acc := accums[id]
mq := `SELECT n.name, n.meta
FROM edges e
JOIN nodes n ON n.id = e.to_id
WHERE e.from_id = ? AND e.kind = ? AND n.kind = ? AND n.meta IS NOT NULL
ORDER BY e.id`
mrows, err := s.db.Query(mq, id, string(graph.EdgeEmits), string(graph.KindString))
if err != nil {
continue
}
for mrows.Next() {
var name string
var metaBlob []byte
if err := mrows.Scan(&name, &metaBlob); err != nil {
continue
}
meta, derr := decodeMeta(metaBlob)
if derr != nil || meta == nil {
continue
}
ctxLabel, _ := meta["context"].(string)
if ctxLabel != "error_msg" {
continue
}
if _, ok := acc.msgSeen[name]; ok {
continue
}
acc.msgSeen[name] = struct{}{}
acc.row.ErrorMsgs = append(acc.row.ErrorMsgs, name)
}
_ = mrows.Close()
}
out := make([]graph.ThrowerErrorRow, 0, len(order))
for _, id := range order {
out = append(out, accums[id].row)
}
return out
}
+172
View File
@@ -0,0 +1,172 @@
package store_sqlite
import (
"sort"
"strings"
"github.com/zzet/gortex/internal/graph"
)
var _ graph.BFSCapable = (*Store)(nil)
// BFS runs a bounded breadth-first traversal in a single round-trip via a
// recursive CTE — the disk-backed sibling of the in-memory
// (*graph.Graph).BFS reference. See graph.BFSCapable for the contract;
// the two are shadow-tested for identical hop-sets in the conformance
// suite (storetest), including a cycle fixture.
//
// The recursive term joins edges on the direction's indexed column —
// edges_by_from(from_id, kind) for a forward walk, edges_by_to(to_id,
// kind) for a backward walk — and the nodes primary key, so it stays
// index-driven instead of scanning the edges table (confirmed via
// EXPLAIN QUERY PLAN in store_bfs_test.go). The nodes join also enforces
// the "node-backed targets only" rule: an edge to an unresolved /
// external stub with no node row is not followed. A cycle terminates on
// the depth bound; the outer ROW_NUMBER picks each node's minimum-depth,
// (parent, kind)-smallest discovery edge so the result is deterministic
// and matches the in-memory walk's bfsHopLess tie-break.
//
// Reads run lock-free, like the store's other read paths (SQLite WAL
// serves readers concurrently with the single serialized writer).
func (s *Store) BFS(seeds []string, dir graph.Direction, kinds []graph.EdgeKind, maxDepth, limit int) ([]graph.BFSHop, error) {
seen := make(map[string]struct{}, len(seeds))
uniqSeeds := make([]string, 0, len(seeds))
for _, sd := range seeds {
if sd == "" {
continue
}
if _, ok := seen[sd]; ok {
continue
}
seen[sd] = struct{}{}
uniqSeeds = append(uniqSeeds, sd)
}
if len(uniqSeeds) == 0 {
return nil, nil
}
uniqKinds := anaDedupeEdgeKinds(kinds)
// Seed-only fast path: with no edge kinds to follow or a non-positive
// depth bound the result is exactly the seeds at depth 0. Seeds enter
// unconditionally (no node-backed gate), matching the in-memory
// reference, which adds them before any traversal.
if len(uniqKinds) == 0 || maxDepth <= 0 {
hops := make([]graph.BFSHop, 0, len(uniqSeeds))
for _, sd := range uniqSeeds {
hops = append(hops, graph.BFSHop{NodeID: sd, Depth: 0})
}
sortBFSHops(hops)
if limit > 0 && len(hops) > limit {
hops = hops[:limit]
}
return hops, nil
}
query := buildBFSQuery(dir, len(uniqSeeds), len(uniqKinds), limit > 0)
args := make([]any, 0, len(uniqSeeds)+1+len(uniqKinds)+1)
for _, sd := range uniqSeeds {
args = append(args, sd)
}
args = append(args, maxDepth)
for _, k := range uniqKinds {
args = append(args, string(k))
}
if limit > 0 {
args = append(args, limit)
}
rows, err := s.db.Query(query, args...)
if err != nil {
return nil, err
}
defer rows.Close()
var out []graph.BFSHop
for rows.Next() {
var (
nodeID, parentID, edgeKind string
depth int
)
if err := rows.Scan(&nodeID, &depth, &parentID, &edgeKind); err != nil {
return nil, err
}
out = append(out, graph.BFSHop{
NodeID: nodeID,
Depth: depth,
ParentID: parentID,
EdgeKind: graph.EdgeKind(edgeKind),
})
}
if err := rows.Err(); err != nil {
return nil, err
}
return out, nil
}
// buildBFSQuery assembles the recursive-CTE BFS statement for the given
// direction, seed count, kind count, and whether a LIMIT is applied. It is
// a pure string builder (no I/O) so a test can EXPLAIN QUERY PLAN the exact
// statement and assert the recursive join stays index-driven.
//
// Direction selects the join columns: forward follows from_id -> to_id (the
// discovered neighbour is the edge target), backward follows to_id ->
// from_id (the neighbour is the edge source). The recursive term joins
// edges on the walked node's id, so a forward walk leads with from_id (the
// edges_by_from(from_id, kind) index) and a backward walk leads with to_id
// (edges_by_to(to_id, kind)); the nodes join uses the nodes primary key.
func buildBFSQuery(dir graph.Direction, nSeeds, nKinds int, withLimit bool) string {
joinCol, nextCol := "e.from_id", "e.to_id"
edgeIdx := "edges_by_from"
if dir == graph.DirectionBackward {
joinCol, nextCol = "e.to_id", "e.from_id"
edgeIdx = "edges_by_to"
}
var b strings.Builder
b.WriteString("WITH RECURSIVE seeds(node_id) AS (VALUES ")
for i := 0; i < nSeeds; i++ {
if i > 0 {
b.WriteString(", ")
}
b.WriteString("(?)")
}
b.WriteString("),\n")
b.WriteString("bfs(node_id, depth, parent_id, edge_kind) AS (\n")
b.WriteString(" SELECT node_id, 0, '', '' FROM seeds\n")
b.WriteString(" UNION\n")
b.WriteString(" SELECT " + nextCol + ", b.depth + 1, b.node_id, e.kind\n")
b.WriteString(" FROM bfs b\n")
// INDEXED BY forces the frontier-node seek (from_id / to_id leading)
// instead of the planner's stats-free preference for edges_by_kind,
// which on a hot kind would scan every edge of that kind per frontier
// node. If the index is ever absent (a bulk-load window drops it) the
// query errors and the engine falls back to the in-memory walk.
b.WriteString(" JOIN edges e INDEXED BY " + edgeIdx + " ON " + joinCol + " = b.node_id\n")
b.WriteString(" JOIN nodes n ON n.id = " + nextCol + "\n")
b.WriteString(" WHERE b.depth < ? AND e.kind IN (" + inPlaceholders(nKinds) + ")\n")
b.WriteString("),\n")
b.WriteString("ranked AS (\n")
b.WriteString(" SELECT node_id, depth, parent_id, edge_kind,\n")
b.WriteString(" ROW_NUMBER() OVER (PARTITION BY node_id ORDER BY depth, parent_id, edge_kind) AS rn\n")
b.WriteString(" FROM bfs\n")
b.WriteString(")\n")
b.WriteString("SELECT node_id, depth, parent_id, edge_kind FROM ranked WHERE rn = 1\n")
b.WriteString("ORDER BY depth, node_id")
if withLimit {
b.WriteString("\nLIMIT ?")
}
return b.String()
}
// sortBFSHops orders hops by (depth, node_id) — the same final ordering
// the recursive-CTE query applies, used by the seed-only fast path.
func sortBFSHops(hops []graph.BFSHop) {
sort.Slice(hops, func(i, j int) bool {
if hops[i].Depth != hops[j].Depth {
return hops[i].Depth < hops[j].Depth
}
return hops[i].NodeID < hops[j].NodeID
})
}
@@ -0,0 +1,74 @@
package store_sqlite
import (
"path/filepath"
"strings"
"testing"
"github.com/zzet/gortex/internal/graph"
)
// queryPlan runs EXPLAIN QUERY PLAN for the given statement + args and
// returns the joined detail lines.
func queryPlan(t *testing.T, s *Store, query string, args ...any) string {
t.Helper()
rows, err := s.db.Query("EXPLAIN QUERY PLAN "+query, args...)
if err != nil {
t.Fatalf("EXPLAIN QUERY PLAN: %v", err)
}
defer rows.Close()
var lines []string
for rows.Next() {
var id, parent, notused int
var detail string
if err := rows.Scan(&id, &parent, &notused, &detail); err != nil {
t.Fatalf("scan plan row: %v", err)
}
lines = append(lines, detail)
}
if err := rows.Err(); err != nil {
t.Fatalf("plan rows: %v", err)
}
return strings.Join(lines, "\n")
}
// TestBFSQueryUsesEdgeIndex asserts the recursive-CTE BFS join reaches the
// edges table through the direction's composite index (edges_by_from for a
// forward walk, edges_by_to for a backward walk) rather than a full table
// scan, and that the node-backed gate join uses the nodes primary key.
func TestBFSQueryUsesEdgeIndex(t *testing.T) {
s, err := Open(filepath.Join(t.TempDir(), "store.sqlite"))
if err != nil {
t.Fatalf("open: %v", err)
}
defer s.Close()
// A small fixture so the planner has real tables/indexes to plan against.
for _, id := range []string{"A", "B", "C"} {
s.AddNode(&graph.Node{ID: id, Kind: graph.KindFunction, Name: id, FilePath: "a.go", Language: "go"})
}
s.AddEdge(&graph.Edge{From: "A", To: "B", Kind: graph.EdgeCalls, Confidence: 1, Origin: graph.OriginASTResolved})
s.AddEdge(&graph.Edge{From: "B", To: "C", Kind: graph.EdgeCalls, Confidence: 1, Origin: graph.OriginASTResolved})
forwardQ := buildBFSQuery(graph.DirectionForward, 1, 1, true)
forwardPlan := queryPlan(t, s, forwardQ, "A", 3, string(graph.EdgeCalls), 50)
if !strings.Contains(forwardPlan, "edges_by_from") {
t.Fatalf("forward BFS plan does not use edges_by_from:\n%s", forwardPlan)
}
// The edges table must not be reached by a full scan (alias e).
if strings.Contains(forwardPlan, "SCAN edges") || strings.Contains(forwardPlan, "SCAN e ") {
t.Fatalf("forward BFS plan scans edges instead of using an index:\n%s", forwardPlan)
}
backwardQ := buildBFSQuery(graph.DirectionBackward, 1, 1, true)
backwardPlan := queryPlan(t, s, backwardQ, "C", 3, string(graph.EdgeCalls), 50)
if !strings.Contains(backwardPlan, "edges_by_to") {
t.Fatalf("backward BFS plan does not use edges_by_to:\n%s", backwardPlan)
}
if strings.Contains(backwardPlan, "SCAN edges") || strings.Contains(backwardPlan, "SCAN e ") {
t.Fatalf("backward BFS plan scans edges instead of using an index:\n%s", backwardPlan)
}
t.Logf("forward BFS query plan:\n%s", forwardPlan)
t.Logf("backward BFS query plan:\n%s", backwardPlan)
}
@@ -0,0 +1,130 @@
package store_sqlite
import (
"database/sql"
"github.com/zzet/gortex/internal/graph"
)
var (
_ graph.BlameEnrichmentWriter = (*Store)(nil)
_ graph.BlameEnrichmentReader = (*Store)(nil)
)
const blameChunk = 180
const blameCols = `node_id, repo_prefix, commit_sha, email, ts`
func (s *Store) BulkSetBlame(repoPrefix string, rows []graph.BlameEnrichment) error {
if len(rows) == 0 {
return nil
}
s.writeMu.Lock()
defer s.writeMu.Unlock()
tx, err := s.db.Begin()
if err != nil {
return err
}
defer tx.Rollback() //nolint:errcheck
for start := 0; start < len(rows); start += blameChunk {
end := start + blameChunk
if end > len(rows) {
end = len(rows)
}
batch := rows[start:end]
args := make([]any, 0, len(batch)*5)
stmt := make([]byte, 0, 96+len(batch)*16)
stmt = append(stmt, "INSERT OR REPLACE INTO blame_enrichment ("...)
stmt = append(stmt, blameCols...)
stmt = append(stmt, ") VALUES "...)
for i, e := range batch {
if i > 0 {
stmt = append(stmt, ',')
}
stmt = append(stmt, "(?,?,?,?,?)"...)
args = append(args, e.NodeID, repoPrefix, e.Commit, e.Email, e.Timestamp)
}
if _, err := tx.Exec(string(stmt), args...); err != nil {
return err
}
}
return tx.Commit()
}
func (s *Store) DeleteBlame(nodeIDs []string) error {
if len(nodeIDs) == 0 {
return nil
}
seen := make(map[string]struct{}, len(nodeIDs))
uniq := make([]string, 0, len(nodeIDs))
for _, id := range nodeIDs {
if id == "" {
continue
}
if _, ok := seen[id]; ok {
continue
}
seen[id] = struct{}{}
uniq = append(uniq, id)
}
if len(uniq) == 0 {
return nil
}
s.writeMu.Lock()
defer s.writeMu.Unlock()
tx, err := s.db.Begin()
if err != nil {
return err
}
defer tx.Rollback() //nolint:errcheck
for start := 0; start < len(uniq); start += blameChunk {
end := start + blameChunk
if end > len(uniq) {
end = len(uniq)
}
chunk := uniq[start:end]
args := make([]any, len(chunk))
stmt := make([]byte, 0, 56+len(chunk)*2)
stmt = append(stmt, "DELETE FROM blame_enrichment WHERE node_id IN ("...)
for i, id := range chunk {
if i > 0 {
stmt = append(stmt, ',')
}
stmt = append(stmt, '?')
args[i] = id
}
stmt = append(stmt, ')')
if _, err := tx.Exec(string(stmt), args...); err != nil {
return err
}
}
return tx.Commit()
}
func (s *Store) BlameRows(repoPrefix string) []graph.BlameEnrichment {
var (
rows *sql.Rows
err error
)
if repoPrefix == "" {
rows, err = s.db.Query(`SELECT ` + blameCols + ` FROM blame_enrichment`)
} else {
rows, err = s.db.Query(`SELECT `+blameCols+` FROM blame_enrichment WHERE repo_prefix = ?`, repoPrefix)
}
if err != nil {
return nil
}
defer rows.Close()
var out []graph.BlameEnrichment
for rows.Next() {
var e graph.BlameEnrichment
if err := rows.Scan(&e.NodeID, &e.RepoPrefix, &e.Commit, &e.Email, &e.Timestamp); err != nil {
return out
}
out = append(out, e)
}
if err := rows.Err(); err != nil {
return out
}
return out
}
@@ -0,0 +1,155 @@
package store_sqlite
import (
"database/sql"
"github.com/zzet/gortex/internal/graph"
)
// Compile-time assertions that the SQLite Store satisfies the optional
// git-churn enrichment sidecar capabilities (change A: enrichment moved
// out of nodes.meta into a typed table so the node hot path stops
// encoding rarely-read data into the meta blob and get_churn_rate reads
// via an index instead of an AllNodes scan).
var (
_ graph.ChurnEnrichmentWriter = (*Store)(nil)
_ graph.ChurnEnrichmentReader = (*Store)(nil)
)
// churnChunk bounds rows per multi-row INSERT. churn_enrichment has 10
// columns, so at 10 params/row the 999 host-param limit caps a statement
// at 99 rows; 90 leaves headroom. Mirrors shingleChunk / mtimeChunk.
const churnChunk = 90
const churnCols = `node_id, repo_prefix, commit_count, age_days, churn_rate, last_author, last_commit_at, head_sha, branch, computed_at`
// BulkSetChurn persists every churn row for one repo prefix in a single
// transaction, chunked under the host-parameter limit. Idempotent on
// node_id (INSERT OR REPLACE). Empty input is a no-op.
func (s *Store) BulkSetChurn(repoPrefix string, rows []graph.ChurnEnrichment) error {
if len(rows) == 0 {
return nil
}
s.writeMu.Lock()
defer s.writeMu.Unlock()
tx, err := s.db.Begin()
if err != nil {
return err
}
defer tx.Rollback() //nolint:errcheck // rollback after Commit is a no-op
for start := 0; start < len(rows); start += churnChunk {
end := start + churnChunk
if end > len(rows) {
end = len(rows)
}
batch := rows[start:end]
args := make([]any, 0, len(batch)*10)
stmt := make([]byte, 0, 128+len(batch)*24)
stmt = append(stmt, "INSERT OR REPLACE INTO churn_enrichment ("...)
stmt = append(stmt, churnCols...)
stmt = append(stmt, ") VALUES "...)
for i, e := range batch {
if i > 0 {
stmt = append(stmt, ',')
}
stmt = append(stmt, "(?,?,?,?,?,?,?,?,?,?)"...)
args = append(args, e.NodeID, repoPrefix, e.CommitCount, e.AgeDays,
e.ChurnRate, e.LastAuthor, e.LastCommitAt, e.HeadSHA, e.Branch, e.ComputedAt)
}
if _, err := tx.Exec(string(stmt), args...); err != nil {
return err
}
}
return tx.Commit()
}
// DeleteChurn drops churn rows for the supplied node ids, chunked into
// `node_id IN (?, …)` DELETEs. Empty input is a no-op.
func (s *Store) DeleteChurn(nodeIDs []string) error {
if len(nodeIDs) == 0 {
return nil
}
seen := make(map[string]struct{}, len(nodeIDs))
uniq := make([]string, 0, len(nodeIDs))
for _, id := range nodeIDs {
if id == "" {
continue
}
if _, ok := seen[id]; ok {
continue
}
seen[id] = struct{}{}
uniq = append(uniq, id)
}
if len(uniq) == 0 {
return nil
}
s.writeMu.Lock()
defer s.writeMu.Unlock()
tx, err := s.db.Begin()
if err != nil {
return err
}
defer tx.Rollback() //nolint:errcheck
for start := 0; start < len(uniq); start += churnChunk {
end := start + churnChunk
if end > len(uniq) {
end = len(uniq)
}
chunk := uniq[start:end]
args := make([]any, len(chunk))
stmt := make([]byte, 0, 48+len(chunk)*2)
stmt = append(stmt, "DELETE FROM churn_enrichment WHERE node_id IN ("...)
for i, id := range chunk {
if i > 0 {
stmt = append(stmt, ',')
}
stmt = append(stmt, '?')
args[i] = id
}
stmt = append(stmt, ')')
if _, err := tx.Exec(string(stmt), args...); err != nil {
return err
}
}
return tx.Commit()
}
// ChurnRows returns every churn row for repoPrefix; an EMPTY repoPrefix
// returns ALL rows across repos. This is an index-only read over the
// (small) enriched set — the whole point of the sidecar, replacing the
// AllNodes()+meta-decode scan get_churn_rate used to do.
func (s *Store) ChurnRows(repoPrefix string) []graph.ChurnEnrichment {
var (
rows *sql.Rows
err error
)
if repoPrefix == "" {
rows, err = s.db.Query(`SELECT ` + churnCols + ` FROM churn_enrichment`)
} else {
rows, err = s.db.Query(`SELECT `+churnCols+` FROM churn_enrichment WHERE repo_prefix = ?`, repoPrefix)
}
if err != nil {
return nil
}
defer rows.Close()
var out []graph.ChurnEnrichment
for rows.Next() {
var e graph.ChurnEnrichment
if err := rows.Scan(&e.NodeID, &e.RepoPrefix, &e.CommitCount, &e.AgeDays,
&e.ChurnRate, &e.LastAuthor, &e.LastCommitAt, &e.HeadSHA, &e.Branch, &e.ComputedAt); err != nil {
return out
}
out = append(out, e)
}
if err := rows.Err(); err != nil {
return out
}
return out
}
@@ -0,0 +1,192 @@
package store_sqlite
import (
"encoding/binary"
"github.com/zzet/gortex/internal/graph"
)
// Compile-time assertions that the SQLite Store satisfies the optional
// per-symbol clone-shingle persistence capabilities. Lifting this state
// into the same backend the graph lives in means warm restarts rebuild
// the clone-detection CMS through one persistence surface instead of a
// second gob snapshot.
var (
_ graph.CloneShingleWriter = (*Store)(nil)
_ graph.CloneShingleReader = (*Store)(nil)
)
// shingleChunk bounds how many (node_id, repo_prefix, shingles) tuples
// ride in a single multi-row INSERT. SQLite's default compiled-in host
// parameter limit is 999; at 3 params per row that caps a statement at
// 333 rows, so 300 leaves headroom. Mirrors mtimeChunk.
const shingleChunk = 300
// encodeShingles serialises a uint64 slice to a little-endian BLOB
// (8 bytes per element). A nil/empty slice encodes to an empty BLOB.
func encodeShingles(shingles []uint64) []byte {
b := make([]byte, len(shingles)*8)
for i, s := range shingles {
binary.LittleEndian.PutUint64(b[i*8:], s)
}
return b
}
// decodeShingles is the inverse of encodeShingles. A BLOB whose length
// is not a multiple of 8 yields nil (corrupt row); callers skip nil
// sets. An empty BLOB decodes to an empty (non-nil) slice.
func decodeShingles(b []byte) []uint64 {
if len(b)%8 != 0 {
return nil
}
out := make([]uint64, len(b)/8)
for i := range out {
out[i] = binary.LittleEndian.Uint64(b[i*8:])
}
return out
}
// BulkSetCloneShingles persists every (nodeID -> shingles) entry for
// one repo prefix in a single transaction, chunked so no statement
// exceeds SQLite's host-parameter limit. Idempotent on node_id:
// re-running with overlapping keys replaces in place. Empty input is a
// no-op.
func (s *Store) BulkSetCloneShingles(repoPrefix string, rows map[string][]uint64) error {
if len(rows) == 0 {
return nil
}
s.writeMu.Lock()
defer s.writeMu.Unlock()
// Stable ordering is not required for correctness, but iterating the
// map directly is fine — we only chunk by count.
type kv struct {
id string
blob []byte
}
pending := make([]kv, 0, len(rows))
for id, sh := range rows {
pending = append(pending, kv{id: id, blob: encodeShingles(sh)})
}
tx, err := s.db.Begin()
if err != nil {
return err
}
defer tx.Rollback() //nolint:errcheck // rollback after Commit is a no-op
for start := 0; start < len(pending); start += shingleChunk {
end := start + shingleChunk
if end > len(pending) {
end = len(pending)
}
batch := pending[start:end]
// Build a multi-row INSERT OR REPLACE: (?, ?, ?), (?, ?, ?), ...
args := make([]any, 0, len(batch)*3)
stmt := make([]byte, 0, 64+len(batch)*16)
stmt = append(stmt, "INSERT OR REPLACE INTO clone_shingles (node_id, repo_prefix, shingles) VALUES "...)
for i, e := range batch {
if i > 0 {
stmt = append(stmt, ',')
}
stmt = append(stmt, "(?, ?, ?)"...)
args = append(args, e.id, repoPrefix, e.blob)
}
if _, err := tx.Exec(string(stmt), args...); err != nil {
return err
}
}
return tx.Commit()
}
// DeleteCloneShingles drops the rows for the supplied node ids, chunked
// into `node_id IN (?, ?, …)` DELETEs so no statement exceeds SQLite's
// host-parameter limit. Empty input is a no-op; missing ids are simply
// not deleted.
func (s *Store) DeleteCloneShingles(nodeIDs []string) error {
if len(nodeIDs) == 0 {
return nil
}
// Dedupe + skip empty up front to keep the chunk loop honest.
seen := make(map[string]struct{}, len(nodeIDs))
uniq := make([]string, 0, len(nodeIDs))
for _, id := range nodeIDs {
if id == "" {
continue
}
if _, ok := seen[id]; ok {
continue
}
seen[id] = struct{}{}
uniq = append(uniq, id)
}
if len(uniq) == 0 {
return nil
}
s.writeMu.Lock()
defer s.writeMu.Unlock()
tx, err := s.db.Begin()
if err != nil {
return err
}
defer tx.Rollback() //nolint:errcheck // rollback after Commit is a no-op
for start := 0; start < len(uniq); start += shingleChunk {
end := start + shingleChunk
if end > len(uniq) {
end = len(uniq)
}
chunk := uniq[start:end]
args := make([]any, len(chunk))
stmt := make([]byte, 0, 48+len(chunk)*2)
stmt = append(stmt, "DELETE FROM clone_shingles WHERE node_id IN ("...)
for i, id := range chunk {
if i > 0 {
stmt = append(stmt, ',')
}
stmt = append(stmt, '?')
args[i] = id
}
stmt = append(stmt, ')')
if _, err := tx.Exec(string(stmt), args...); err != nil {
return err
}
}
return tx.Commit()
}
// LoadCloneShingles returns the recorded shingle sets for one repo
// prefix as a fresh map. It always returns a non-nil (possibly empty)
// map and surfaces any query error. An empty/absent prefix yields an
// empty map, not an error.
func (s *Store) LoadCloneShingles(repoPrefix string) (map[string][]uint64, error) {
rows, err := s.db.Query(
`SELECT node_id, shingles FROM clone_shingles WHERE repo_prefix = ?`,
repoPrefix,
)
if err != nil {
return nil, err
}
defer rows.Close()
out := make(map[string][]uint64)
for rows.Next() {
var id string
var blob []byte
if err := rows.Scan(&id, &blob); err != nil {
return nil, err
}
out[id] = decodeShingles(blob)
}
if err := rows.Err(); err != nil {
return nil, err
}
return out, nil
}
@@ -0,0 +1,63 @@
package store_sqlite
// One-time boot compaction support.
//
// The graph store only ever grows on disk: deleted rows (a purged repo, the
// duplicate-collapse migration, resolver cleanups) return their pages to
// SQLite's freelist, where future writes reuse them — but nothing short of
// VACUUM returns them to the filesystem. A long-lived store that shed a large
// fraction of its rows can therefore pin gigabytes of dead file forever (a
// live store sat at 64% freelist — 4.4 GB reclaimable in a 6.8 GB file).
// These methods give the daemon the numbers to decide whether that one-time
// rewrite is worth it, and the lever to run it. The policy (thresholds, disk
// headroom, kill-switch) deliberately lives with the caller: the store cannot
// know whether minutes of exclusive I/O are acceptable right now.
// Path returns the on-disk database file path. Empty when the store was not
// opened from a file — callers using it to reason about the underlying
// filesystem (disk-headroom checks) must treat "" as "unknown, don't".
func (s *Store) Path() string {
return s.dbPath
}
// CompactStats reports how much of the database file is reclaimable dead
// space: freeBytes is the freelist (freelist_count × page_size), totalBytes
// the whole main file (page_count × page_size). Zeros on any pragma error —
// a read failing here is the same teardown race panicOnFatal swallows, and
// "nothing reclaimable" is the answer that makes every caller do nothing.
// The -wal file is excluded on purpose: the checkpoint loop already bounds
// it, and VACUUM only rewrites the main file.
func (s *Store) CompactStats() (freeBytes, totalBytes int64) {
var pageSize, pageCount, freePages int64
if err := s.db.QueryRow(`PRAGMA page_size`).Scan(&pageSize); err != nil {
return 0, 0
}
if err := s.db.QueryRow(`PRAGMA page_count`).Scan(&pageCount); err != nil {
return 0, 0
}
if err := s.db.QueryRow(`PRAGMA freelist_count`).Scan(&freePages); err != nil {
return 0, 0
}
return freePages * pageSize, pageCount * pageSize
}
// Compact rewrites the database file (VACUUM), returning freelist pages to
// the filesystem, then drains the write-ahead log with a TRUNCATE checkpoint
// so the rewrite's WAL traffic doesn't linger as a second oversized file.
//
// Cost model callers must respect: VACUUM copies the live content into a
// temporary database (up to a full extra copy on the same filesystem) and
// needs exclusive access — it blocks Go-side writers via writeMu here, and a
// concurrent reader on another pooled connection makes SQLite wait out
// busy_timeout and then fail. That failure is clean (the store is untouched,
// freelist pages remain reusable), which is why the daemon treats a Compact
// error as skip-and-continue rather than fatal.
func (s *Store) Compact() error {
s.writeMu.Lock()
_, err := s.db.Exec(`VACUUM`)
s.writeMu.Unlock()
if err != nil {
return err
}
return s.CheckpointWAL()
}
@@ -0,0 +1,76 @@
package store_sqlite_test
import (
"fmt"
"strings"
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/zzet/gortex/internal/graph"
)
// TestCompactReclaimsFreelist pins the boot-compaction capability end to end
// on a real file: shedding most rows leaves the file full of freelist pages
// (CompactStats must say so), Compact() must return them to the filesystem
// (page_count shrinks), and the surviving rows must come through untouched.
// Fractional assertions only — page size and per-row overhead are backend
// details this test must not encode.
func TestCompactReclaimsFreelist(t *testing.T) {
s := openTestStore(t)
// Seed 60 files × 50 nodes with ~2 KiB of meta each (~6 MiB of pages),
// then checkpoint so the rows land in the main file rather than the WAL —
// CompactStats deliberately measures only the main file.
const files, perFile = 60, 50
pad := strings.Repeat("x", 2048)
for f := 0; f < files; f++ {
nodes := make([]*graph.Node, 0, perFile)
path := fmt.Sprintf("p/f%03d.go", f)
for n := 0; n < perFile; n++ {
nodes = append(nodes, &graph.Node{
ID: fmt.Sprintf("%s::N%d", path, n),
Kind: graph.KindFunction,
Name: fmt.Sprintf("N%d", n),
FilePath: path,
Meta: map[string]any{"pad": pad},
})
}
s.AddBatch(nodes, nil)
}
require.NoError(t, s.CheckpointWAL())
_, totalSeeded := s.CompactStats()
require.Greater(t, totalSeeded, int64(3<<20), "sanity: seeding must produce a multi-MiB main file")
// Shed all but one file, checkpoint again so the deletions reach the main
// file's freelist.
for f := 1; f < files; f++ {
s.EvictFile(fmt.Sprintf("p/f%03d.go", f))
}
require.NoError(t, s.CheckpointWAL())
freeBefore, totalBefore := s.CompactStats()
assert.Greater(t, freeBefore*3, totalBefore,
"after shedding ~98%% of rows the freelist must dominate the file (free=%d total=%d)", freeBefore, totalBefore)
assert.Greater(t, freeBefore, int64(1<<20), "freelist must be at least MiB-scale to make the shrink observable")
require.NoError(t, s.Compact())
freeAfter, totalAfter := s.CompactStats()
assert.Less(t, totalAfter, totalBefore/2,
"VACUUM must return the dead majority to the filesystem (before=%d after=%d)", totalBefore, totalAfter)
assert.Less(t, freeAfter, totalBefore/10,
"post-VACUUM freelist must be near empty (free=%d)", freeAfter)
// Survivors intact, evicted rows gone.
assert.Equal(t, perFile, s.NodeCount(), "compaction must not change the row count")
kept := s.GetNode("p/f000.go::N0")
require.NotNil(t, kept, "kept row must survive VACUUM")
assert.Equal(t, pad, kept.Meta["pad"], "kept row's meta blob must round-trip through VACUUM")
assert.Nil(t, s.GetNode("p/f001.go::N0"), "evicted row must stay gone")
// The store stays fully writable after the rewrite.
s.AddNode(&graph.Node{ID: "p/new.go::After", Kind: graph.KindFunction, Name: "After", FilePath: "p/new.go"})
assert.NotNil(t, s.GetNode("p/new.go::After"))
}
@@ -0,0 +1,144 @@
package store_sqlite
import (
"github.com/zzet/gortex/internal/graph"
)
// Compile-time assertions that the SQLite Store satisfies the optional
// constant-value persistence capability. A KindConstant node's literal
// value lives in this queryable sidecar (not the JSON Meta blob)
// so the resolver can dereference a const-identifier dispatch name across
// files without an unindexable per-node blob decode.
var (
_ graph.ConstantValueWriter = (*Store)(nil)
_ graph.ConstantValueReader = (*Store)(nil)
)
// constValueChunk bounds rows per multi-row INSERT (4 params/row; 80 rows
// = 320 host params, well under SQLite's 999 default).
const constValueChunk = 80
// BulkSetConstantValues persists constant values for one repo prefix in a
// single transaction, chunked under the host-parameter limit. Idempotent
// on the node_id primary key. Empty input is a no-op.
func (s *Store) BulkSetConstantValues(repoPrefix string, rows []graph.ConstantValueRow) error {
if len(rows) == 0 {
return nil
}
s.writeMu.Lock()
defer s.writeMu.Unlock()
tx, err := s.db.Begin()
if err != nil {
return err
}
defer tx.Rollback() //nolint:errcheck // rollback after Commit is a no-op
for start := 0; start < len(rows); start += constValueChunk {
end := start + constValueChunk
if end > len(rows) {
end = len(rows)
}
batch := rows[start:end]
args := make([]any, 0, len(batch)*4)
stmt := make([]byte, 0, 96+len(batch)*16)
stmt = append(stmt, "INSERT OR REPLACE INTO constant_values (node_id, repo_prefix, file_path, value) VALUES "...)
for i, r := range batch {
if i > 0 {
stmt = append(stmt, ',')
}
stmt = append(stmt, "(?, ?, ?, ?)"...)
args = append(args, r.NodeID, repoPrefix, r.FilePath, r.Value)
}
if _, err := tx.Exec(string(stmt), args...); err != nil {
return err
}
}
return tx.Commit()
}
// DeleteConstantValuesByFiles drops all constant values sourced in the
// supplied files for one repo prefix, chunked into `file_path IN (…)`
// DELETEs. Empty input is a no-op.
func (s *Store) DeleteConstantValuesByFiles(repoPrefix string, files []string) error {
if len(files) == 0 {
return nil
}
s.writeMu.Lock()
defer s.writeMu.Unlock()
tx, err := s.db.Begin()
if err != nil {
return err
}
defer tx.Rollback() //nolint:errcheck // rollback after Commit is a no-op
for start := 0; start < len(files); start += constValueChunk {
end := start + constValueChunk
if end > len(files) {
end = len(files)
}
chunk := files[start:end]
args := make([]any, 0, len(chunk)+1)
args = append(args, repoPrefix)
stmt := make([]byte, 0, 64+len(chunk)*2)
stmt = append(stmt, "DELETE FROM constant_values WHERE repo_prefix = ? AND file_path IN ("...)
for i, f := range chunk {
if i > 0 {
stmt = append(stmt, ',')
}
stmt = append(stmt, '?')
args = append(args, f)
}
stmt = append(stmt, ')')
if _, err := tx.Exec(string(stmt), args...); err != nil {
return err
}
}
return tx.Commit()
}
// ConstantValuesByNodeIDs returns the persisted values for the supplied
// node ids (omitting ids with no recorded value). Always non-nil.
func (s *Store) ConstantValuesByNodeIDs(nodeIDs []string) (map[string]string, error) {
out := make(map[string]string, len(nodeIDs))
if len(nodeIDs) == 0 {
return out, nil
}
for start := 0; start < len(nodeIDs); start += constValueChunk {
end := start + constValueChunk
if end > len(nodeIDs) {
end = len(nodeIDs)
}
chunk := nodeIDs[start:end]
args := make([]any, 0, len(chunk))
stmt := make([]byte, 0, 64+len(chunk)*2)
stmt = append(stmt, "SELECT node_id, value FROM constant_values WHERE node_id IN ("...)
for i, id := range chunk {
if i > 0 {
stmt = append(stmt, ',')
}
stmt = append(stmt, '?')
args = append(args, id)
}
stmt = append(stmt, ')')
rows, err := s.db.Query(string(stmt), args...)
if err != nil {
return nil, err
}
for rows.Next() {
var id, val string
if err := rows.Scan(&id, &val); err != nil {
_ = rows.Close()
return nil, err
}
out[id] = val
}
if err := rows.Err(); err != nil {
_ = rows.Close()
return nil, err
}
_ = rows.Close()
}
return out, nil
}
@@ -0,0 +1,73 @@
package store_sqlite_test
import (
"path/filepath"
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/zzet/gortex/internal/graph"
"github.com/zzet/gortex/internal/graph/store_sqlite"
)
func openConstValStore(t *testing.T) *store_sqlite.Store {
t.Helper()
s, err := store_sqlite.Open(filepath.Join(t.TempDir(), "cv.sqlite"))
require.NoError(t, err)
t.Cleanup(func() { _ = s.Close() })
return s
}
func TestConstantValues_Roundtrip(t *testing.T) {
s := openConstValStore(t)
rows := []graph.ConstantValueRow{
{NodeID: "a.go::ChargeCardActivity", FilePath: "a.go", Value: "ChargeCard"},
{NodeID: "a.go::RefundActivity", FilePath: "a.go", Value: "Refund"},
}
require.NoError(t, s.BulkSetConstantValues("repo", rows))
got, err := s.ConstantValuesByNodeIDs([]string{"a.go::ChargeCardActivity", "a.go::RefundActivity", "missing"})
require.NoError(t, err)
assert.Equal(t, "ChargeCard", got["a.go::ChargeCardActivity"])
assert.Equal(t, "Refund", got["a.go::RefundActivity"])
_, ok := got["missing"]
assert.False(t, ok)
}
func TestConstantValues_DeleteByFile(t *testing.T) {
s := openConstValStore(t)
require.NoError(t, s.BulkSetConstantValues("repo", []graph.ConstantValueRow{
{NodeID: "a.go::X", FilePath: "a.go", Value: "vx"},
{NodeID: "b.go::Y", FilePath: "b.go", Value: "vy"},
}))
require.NoError(t, s.DeleteConstantValuesByFiles("repo", []string{"a.go"}))
got, err := s.ConstantValuesByNodeIDs([]string{"a.go::X", "b.go::Y"})
require.NoError(t, err)
_, gone := got["a.go::X"]
assert.False(t, gone, "a.go's value must be deleted")
assert.Equal(t, "vy", got["b.go::Y"], "b.go's value must remain")
}
func TestConstantValues_Replace(t *testing.T) {
s := openConstValStore(t)
require.NoError(t, s.BulkSetConstantValues("repo", []graph.ConstantValueRow{
{NodeID: "a.go::X", FilePath: "a.go", Value: "old"},
}))
require.NoError(t, s.BulkSetConstantValues("repo", []graph.ConstantValueRow{
{NodeID: "a.go::X", FilePath: "a.go", Value: "new"},
}))
got, err := s.ConstantValuesByNodeIDs([]string{"a.go::X"})
require.NoError(t, err)
assert.Equal(t, "new", got["a.go::X"], "INSERT OR REPLACE must update by node_id PK")
}
func TestConstantValues_EmptyNoop(t *testing.T) {
s := openConstValStore(t)
require.NoError(t, s.BulkSetConstantValues("repo", nil))
require.NoError(t, s.DeleteConstantValuesByFiles("repo", nil))
got, err := s.ConstantValuesByNodeIDs(nil)
require.NoError(t, err)
assert.Empty(t, got)
}
@@ -0,0 +1,297 @@
package store_sqlite
import (
"database/sql"
"strings"
"github.com/zzet/gortex/internal/graph"
)
// This file implements graph.ContentSearcher on the SQLite backend using
// the content_fts FTS5 virtual table declared in schema.go — the
// dedicated, on-disk full-text index for CONTENT (data_class="content")
// section bodies, kept physically separate from symbol_fts so content
// text never reaches the symbol search or the code-oriented graph passes.
//
// Streamed build: WipeContent(repoPrefix) once at the start of a full
// index, AppendContent each content file's sections as they are parsed
// (no per-file wipe), then BuildContentIndex to merge segments.
// Incremental reindex of one content file is WipeContentFile +
// AppendContent.
// Compile-time assertion: *Store satisfies the content-search capability.
var _ graph.ContentSearcher = (*Store)(nil)
// contentInsertChunkRows bounds rows per multi-row INSERT. Each row binds
// 5 host params (node_id, repo_prefix, file_path, ordinal, body); 180 rows
// is 900 params, comfortably under SQLite's default 999-variable limit.
const contentInsertChunkRows = 180
// WipeContent removes a repo's content rows before a full rebuild. Empty
// repoPrefix wipes the whole table (single-repo / conformance behaviour).
func (s *Store) WipeContent(repoPrefix string) error {
s.writeMu.Lock()
defer s.writeMu.Unlock()
_, err := s.db.Exec(`DELETE FROM content_fts WHERE repo_prefix = ?`, repoPrefix)
return err
}
// WipeContentFile removes one file's content rows — the incremental
// reindex path when a single content file changes.
func (s *Store) WipeContentFile(filePath string) error {
if filePath == "" {
return nil
}
s.writeMu.Lock()
defer s.writeMu.Unlock()
_, err := s.db.Exec(`DELETE FROM content_fts WHERE file_path = ?`, filePath)
return err
}
// WipeContentFileInRepo removes ONE file's content rows scoped to a repo —
// the crash-safe full-index sibling of WipeContentFile (which keys on
// file_path alone and so would clobber a same-named file in another repo).
// A full index streams content per file: delete this file's prior rows,
// then AppendContent its fresh sections — so a mid-parse kill leaves a mix
// of old+new content per file rather than the empty table a repo-wide
// pre-wipe would leave behind. Empty filePath is a no-op.
func (s *Store) WipeContentFileInRepo(repoPrefix, filePath string) error {
if filePath == "" {
return nil
}
s.writeMu.Lock()
defer s.writeMu.Unlock()
_, err := s.db.Exec(`DELETE FROM content_fts WHERE repo_prefix = ? AND file_path = ?`, repoPrefix, filePath)
return err
}
// DeleteContentFilesForRepoNotIn sweeps a repo's content rows down to keep —
// every content row whose file_path is absent from keep is deleted. keep is
// the set of files that actually STREAMED content sections in the walk just
// completed (each recorded as the same file_path form AppendContent wrote),
// NOT the set of files that merely survive on disk: a file can still exist
// yet stop producing content (doc emptied, classification changed), and a
// disk-based keep would protect its stale rows forever. Run once at the end
// of a successful full index (right after the authoritative mtime replace),
// it reaps vanished files and content->no-content transitions in one scan;
// the per-file WipeContentFileInRepo + AppendContent streaming build
// refreshes the files that still produce content. Together they replace the
// old repo-wide pre-wipe: a mid-parse kill no longer empties the content
// index, and stale rows are reaped only on the next clean completion.
// Empty keep is a deliberate no-op safety net — never wipe a whole repo from
// an empty set; a caller that legitimately ends a walk with zero content
// files calls WipeContent explicitly instead.
func (s *Store) DeleteContentFilesForRepoNotIn(repoPrefix string, keep map[string]struct{}) error {
if len(keep) == 0 {
return nil
}
s.writeMu.Lock()
defer s.writeMu.Unlock()
// Enumerate the repo's content file paths, then delete only those not in
// keep. Content files are a small subset (docs / PDFs / office), so the
// DISTINCT scan + targeted deletes stay cheap and dodge a giant NOT IN
// (...) bound-variable list. Rows are drained + closed before the delete
// tx opens (no open read cursor while writing on the same connection).
rows, err := s.db.Query(`SELECT DISTINCT file_path FROM content_fts WHERE repo_prefix = ?`, repoPrefix)
if err != nil {
return err
}
var vanished []string
for rows.Next() {
var fp string
if err := rows.Scan(&fp); err != nil {
_ = rows.Close()
return err
}
if _, ok := keep[fp]; !ok {
vanished = append(vanished, fp)
}
}
if err := rows.Err(); err != nil {
_ = rows.Close()
return err
}
_ = rows.Close()
if len(vanished) == 0 {
return nil
}
tx, err := s.db.Begin()
if err != nil {
return err
}
defer tx.Rollback() //nolint:errcheck // rollback after Commit is a no-op
const chunk = 900
for start := 0; start < len(vanished); start += chunk {
end := minInt(start+chunk, len(vanished))
batch := vanished[start:end]
placeholders := strings.TrimSuffix(strings.Repeat("?,", len(batch)), ",")
args := make([]any, 0, len(batch)+1)
args = append(args, repoPrefix)
for _, fp := range batch {
args = append(args, fp)
}
if _, err := tx.Exec(`DELETE FROM content_fts WHERE repo_prefix = ? AND file_path IN (`+placeholders+`)`, args...); err != nil {
return err
}
}
return tx.Commit()
}
// AppendContent inserts content rows for repoPrefix without wiping — the
// streamed per-file build path. Callers wipe (whole repo or one file)
// first. Rows with an empty NodeID are skipped.
func (s *Store) AppendContent(repoPrefix string, items []graph.ContentFTSItem) error {
if len(items) == 0 {
return nil
}
s.writeMu.Lock()
defer s.writeMu.Unlock()
tx, err := s.db.Begin()
if err != nil {
return err
}
commit := false
defer func() {
if !commit {
_ = tx.Rollback()
}
}()
for start := 0; start < len(items); start += contentInsertChunkRows {
end := minInt(start+contentInsertChunkRows, len(items))
chunk := items[start:end]
var b strings.Builder
b.WriteString(`INSERT INTO content_fts (node_id, repo_prefix, file_path, ordinal, body) VALUES `)
args := make([]any, 0, len(chunk)*5)
for _, it := range chunk {
if it.NodeID == "" {
continue
}
if len(args) > 0 {
b.WriteByte(',')
}
b.WriteString(`(?,?,?,?,?)`)
args = append(args, it.NodeID, repoPrefix, it.FilePath, it.Ordinal, it.Body)
}
if len(args) == 0 {
continue
}
if _, err := tx.Exec(b.String(), args...); err != nil {
return err
}
}
if err := tx.Commit(); err != nil {
return err
}
commit = true
return nil
}
// BuildContentIndex opportunistically merges FTS5 segments (a read-latency
// improvement). Like BuildSymbolIndex it is a no-op for correctness — the
// FTS index is maintained incrementally on every insert — and idempotent.
func (s *Store) BuildContentIndex() error {
s.writeMu.Lock()
defer s.writeMu.Unlock()
_, _ = s.db.Exec(`INSERT INTO content_fts(content_fts) VALUES('optimize')`)
return nil
}
// SearchContent runs a content query scoped to repoPrefix (empty = all
// repos) and returns hits ordered by descending relevance, each carrying a
// short snippet excerpt from the matched body. Reuses the symbol path's
// write-side tokeniser (buildFTSMatch) so the content corpus and queries
// agree on camelCase / path-separator splitting.
func (s *Store) SearchContent(query, repoPrefix string, limit int) ([]graph.ContentHit, error) {
if query == "" {
return nil, nil
}
if limit <= 0 {
limit = 20
}
match := s.buildFTSMatch(query)
if match == "" {
return nil, nil
}
var sb strings.Builder
// snippet() over the body column (index 4): no highlight marks, an
// ellipsis for elision, ~16 tokens of context. CAST(ordinal AS INTEGER)
// forces integer affinity so the FTS5 text column scans cleanly into an
// int.
sb.WriteString(`SELECT node_id, file_path, CAST(ordinal AS INTEGER), snippet(content_fts, 4, '', '', '…', 16), bm25(content_fts) FROM content_fts WHERE content_fts MATCH ?`)
args := []any{match}
if repoPrefix != "" {
sb.WriteString(` AND repo_prefix = ?`)
args = append(args, repoPrefix)
}
sb.WriteString(` ORDER BY bm25(content_fts) LIMIT ?`)
args = append(args, limit)
rows, err := s.db.Query(sb.String(), args...)
if err != nil {
return nil, err
}
defer rows.Close()
var hits []graph.ContentHit
for rows.Next() {
var (
id, fp, snip string
ordinal int
score float64
)
if err := rows.Scan(&id, &fp, &ordinal, &snip, &score); err != nil {
return nil, err
}
if id == "" {
continue
}
// bm25() is negative-better in SQLite; negate so higher = better,
// matching the ContentHit contract. Rows already arrive best-first.
hits = append(hits, graph.ContentHit{
NodeID: id,
FilePath: fp,
Ordinal: ordinal,
Score: -score,
Snippet: snip,
})
}
if err := rows.Err(); err != nil {
return nil, err
}
return hits, nil
}
// ScanContent streams every content row (scoped to repoPrefix; empty = all
// repos) to fn with its FULL body, read incrementally via a cursor so a
// consumer iterating hundreds of thousands of sections stays bounded. fn
// returns false to stop the scan early.
func (s *Store) ScanContent(repoPrefix string, fn func(nodeID, filePath, body string) bool) error {
var rows *sql.Rows
var err error
if repoPrefix == "" {
rows, err = s.db.Query(`SELECT node_id, file_path, body FROM content_fts`)
} else {
rows, err = s.db.Query(`SELECT node_id, file_path, body FROM content_fts WHERE repo_prefix = ?`, repoPrefix)
}
if err != nil {
return err
}
defer rows.Close()
for rows.Next() {
var nodeID, filePath, body string
if err := rows.Scan(&nodeID, &filePath, &body); err != nil {
return err
}
if !fn(nodeID, filePath, body) {
return nil
}
}
return rows.Err()
}
@@ -0,0 +1,82 @@
package store_sqlite
import (
"path/filepath"
"testing"
"github.com/stretchr/testify/require"
"github.com/zzet/gortex/internal/graph"
)
func TestContentFTS_BasicAndFileWipe(t *testing.T) {
s, err := Open(filepath.Join(t.TempDir(), "c.sqlite"))
require.NoError(t, err)
t.Cleanup(func() { _ = s.Close() })
var cs graph.ContentSearcher = s // runtime assertion the store satisfies the capability
require.NoError(t, cs.WipeContent("")) // clean table
items := []graph.ContentFTSItem{
{NodeID: "a.txt::doc:section-0", FilePath: "a.txt", Ordinal: 0, Body: "the quick brown fox jumps over the lazy dog"},
{NodeID: "a.txt::doc:section-1", FilePath: "a.txt", Ordinal: 1, Body: "lorem ipsum dolor sit amet consectetur"},
{NodeID: "b.pdf::doc:pdf_page-0", FilePath: "b.pdf", Ordinal: 0, Body: "quantum entanglement and superposition explained"},
}
require.NoError(t, cs.AppendContent("", items))
require.NoError(t, cs.BuildContentIndex())
// A body term resolves to exactly its section, with a non-empty snippet.
hits, err := cs.SearchContent("quantum", "", 10)
require.NoError(t, err)
require.Len(t, hits, 1)
require.Equal(t, "b.pdf::doc:pdf_page-0", hits[0].NodeID)
require.Equal(t, "b.pdf", hits[0].FilePath)
require.Equal(t, 0, hits[0].Ordinal)
require.NotEmpty(t, hits[0].Snippet)
hits, err = cs.SearchContent("fox", "", 10)
require.NoError(t, err)
require.Len(t, hits, 1)
require.Equal(t, "a.txt::doc:section-0", hits[0].NodeID)
require.Equal(t, 0, hits[0].Ordinal)
// WipeContentFile drops only a.txt's rows; b.pdf survives.
require.NoError(t, cs.WipeContentFile("a.txt"))
hits, err = cs.SearchContent("fox", "", 10)
require.NoError(t, err)
require.Empty(t, hits)
hits, err = cs.SearchContent("quantum", "", 10)
require.NoError(t, err)
require.Len(t, hits, 1)
}
func TestContentFTS_RepoScoping(t *testing.T) {
s, err := Open(filepath.Join(t.TempDir(), "c.sqlite"))
require.NoError(t, err)
t.Cleanup(func() { _ = s.Close() })
require.NoError(t, s.AppendContent("repoA", []graph.ContentFTSItem{
{NodeID: "repoA::x.txt::doc:section-0", FilePath: "x.txt", Body: "alpha beta gamma"},
}))
require.NoError(t, s.AppendContent("repoB", []graph.ContentFTSItem{
{NodeID: "repoB::y.txt::doc:section-0", FilePath: "y.txt", Body: "alpha delta epsilon"},
}))
// Scoped search returns only the matching repo's hit.
hits, err := s.SearchContent("alpha", "repoA", 10)
require.NoError(t, err)
require.Len(t, hits, 1)
require.Equal(t, "repoA::x.txt::doc:section-0", hits[0].NodeID)
// Unscoped search spans both repos.
hits, err = s.SearchContent("alpha", "", 10)
require.NoError(t, err)
require.Len(t, hits, 2)
// WipeContent scopes to one repo, leaving the sibling intact.
require.NoError(t, s.WipeContent("repoA"))
hits, err = s.SearchContent("alpha", "", 10)
require.NoError(t, err)
require.Len(t, hits, 1)
require.Equal(t, "repoB::y.txt::doc:section-0", hits[0].NodeID)
}
@@ -0,0 +1,143 @@
package store_sqlite
import (
"database/sql"
"github.com/zzet/gortex/internal/graph"
)
var (
_ graph.CoverageEnrichmentWriter = (*Store)(nil)
_ graph.CoverageEnrichmentReader = (*Store)(nil)
)
// coverageChunk bounds rows per multi-row INSERT (5 cols → 5 params/row;
// 999/5 ≈ 199 max, 180 leaves headroom).
const coverageChunk = 180
const coverageCols = `node_id, repo_prefix, coverage_pct, num_stmt, hit`
// BulkSetCoverage persists coverage rows for one repo prefix in a single
// chunked transaction. Idempotent on node_id. Empty input is a no-op.
func (s *Store) BulkSetCoverage(repoPrefix string, rows []graph.CoverageEnrichment) error {
if len(rows) == 0 {
return nil
}
s.writeMu.Lock()
defer s.writeMu.Unlock()
tx, err := s.db.Begin()
if err != nil {
return err
}
defer tx.Rollback() //nolint:errcheck
for start := 0; start < len(rows); start += coverageChunk {
end := start + coverageChunk
if end > len(rows) {
end = len(rows)
}
batch := rows[start:end]
args := make([]any, 0, len(batch)*5)
stmt := make([]byte, 0, 96+len(batch)*16)
stmt = append(stmt, "INSERT OR REPLACE INTO coverage_enrichment ("...)
stmt = append(stmt, coverageCols...)
stmt = append(stmt, ") VALUES "...)
for i, e := range batch {
if i > 0 {
stmt = append(stmt, ',')
}
stmt = append(stmt, "(?,?,?,?,?)"...)
args = append(args, e.NodeID, repoPrefix, e.CoveragePct, e.NumStmt, e.Hit)
}
if _, err := tx.Exec(string(stmt), args...); err != nil {
return err
}
}
return tx.Commit()
}
// DeleteCoverage drops coverage rows for the supplied node ids, chunked.
func (s *Store) DeleteCoverage(nodeIDs []string) error {
if len(nodeIDs) == 0 {
return nil
}
seen := make(map[string]struct{}, len(nodeIDs))
uniq := make([]string, 0, len(nodeIDs))
for _, id := range nodeIDs {
if id == "" {
continue
}
if _, ok := seen[id]; ok {
continue
}
seen[id] = struct{}{}
uniq = append(uniq, id)
}
if len(uniq) == 0 {
return nil
}
s.writeMu.Lock()
defer s.writeMu.Unlock()
tx, err := s.db.Begin()
if err != nil {
return err
}
defer tx.Rollback() //nolint:errcheck
for start := 0; start < len(uniq); start += coverageChunk {
end := start + coverageChunk
if end > len(uniq) {
end = len(uniq)
}
chunk := uniq[start:end]
args := make([]any, len(chunk))
stmt := make([]byte, 0, 56+len(chunk)*2)
stmt = append(stmt, "DELETE FROM coverage_enrichment WHERE node_id IN ("...)
for i, id := range chunk {
if i > 0 {
stmt = append(stmt, ',')
}
stmt = append(stmt, '?')
args[i] = id
}
stmt = append(stmt, ')')
if _, err := tx.Exec(string(stmt), args...); err != nil {
return err
}
}
return tx.Commit()
}
// CoverageRows returns coverage rows for repoPrefix; empty repoPrefix
// returns ALL rows across repos. Index-only read over the enriched set.
func (s *Store) CoverageRows(repoPrefix string) []graph.CoverageEnrichment {
var (
rows *sql.Rows
err error
)
if repoPrefix == "" {
rows, err = s.db.Query(`SELECT ` + coverageCols + ` FROM coverage_enrichment`)
} else {
rows, err = s.db.Query(`SELECT `+coverageCols+` FROM coverage_enrichment WHERE repo_prefix = ?`, repoPrefix)
}
if err != nil {
return nil
}
defer rows.Close()
var out []graph.CoverageEnrichment
for rows.Next() {
var e graph.CoverageEnrichment
if err := rows.Scan(&e.NodeID, &e.RepoPrefix, &e.CoveragePct, &e.NumStmt, &e.Hit); err != nil {
return out
}
out = append(out, e)
}
if err := rows.Err(); err != nil {
return out
}
return out
}
@@ -0,0 +1,21 @@
package store_sqlite
import "os"
// DBStats returns the on-disk size of the SQLite database file and its
// write-ahead log, in bytes. A missing file (or a store opened without a
// path) reports 0 for that component. Surfaced in daemon_health so a
// runaway WAL high-water mark is observable instead of silently filling
// the disk.
func (s *Store) DBStats() (dbBytes, walBytes int64) {
if s == nil || s.dbPath == "" {
return 0, 0
}
if fi, err := os.Stat(s.dbPath); err == nil {
dbBytes = fi.Size()
}
if fi, err := os.Stat(s.dbPath + "-wal"); err == nil {
walBytes = fi.Size()
}
return dbBytes, walBytes
}
@@ -0,0 +1,33 @@
package store_sqlite_test
import (
"path/filepath"
"testing"
"github.com/stretchr/testify/require"
"github.com/zzet/gortex/internal/graph"
"github.com/zzet/gortex/internal/graph/store_sqlite"
)
func TestDBStats(t *testing.T) {
path := filepath.Join(t.TempDir(), "g.sqlite")
s, err := store_sqlite.Open(path)
require.NoError(t, err)
t.Cleanup(func() { _ = s.Close() })
// Force a write so the DB (and WAL, in WAL mode) carry real bytes.
s.AddBatch([]*graph.Node{
{ID: "a.go::Foo", Kind: graph.KindFunction, Name: "Foo", FilePath: "a.go"},
}, nil)
dbBytes, walBytes := s.DBStats()
require.Greater(t, dbBytes, int64(0), "the on-disk DB file must have nonzero size")
require.GreaterOrEqual(t, walBytes, int64(0), "WAL size is non-negative (0 after a checkpoint)")
// A store with no path (the zero value) reports zero, never panics.
var empty store_sqlite.Store
db, wal := empty.DBStats()
require.Equal(t, int64(0), db)
require.Equal(t, int64(0), wal)
}
@@ -0,0 +1,45 @@
package store_sqlite
import (
"database/sql"
"github.com/zzet/gortex/internal/graph"
)
// Compile-time assertion that the SQLite Store persists the enrichment
// completion marker. Lifting this state into the same backend the graph
// lives in lets a warm restart skip re-enriching a repo whose persisted
// graph already carries its LSP edges — no second persistence surface.
var _ graph.EnrichmentStateStore = (*Store)(nil)
// SetEnrichmentState upserts the completion marker for one (repo, provider) —
// written when a provider finishes a non-partial enrichment pass. One row per
// (repo_prefix, provider).
func (s *Store) SetEnrichmentState(st graph.EnrichmentState) error {
s.writeMu.Lock()
defer s.writeMu.Unlock()
_, err := s.db.Exec(`
INSERT OR REPLACE INTO enrichment_state
(repo_prefix, provider, indexed_sha, completed_at, coverage)
VALUES (?, ?, ?, ?, ?)`,
st.RepoPrefix, st.Provider, st.IndexedSHA, st.CompletedAt, st.Coverage)
return err
}
// GetEnrichmentState returns the recorded completion marker for a
// (repo, provider). The bool is false when no row exists yet (never-enriched
// or pre-feature).
func (s *Store) GetEnrichmentState(repoPrefix, provider string) (graph.EnrichmentState, bool, error) {
row := s.db.QueryRow(`
SELECT indexed_sha, completed_at, coverage
FROM enrichment_state WHERE repo_prefix = ? AND provider = ?`, repoPrefix, provider)
st := graph.EnrichmentState{RepoPrefix: repoPrefix, Provider: provider}
err := row.Scan(&st.IndexedSHA, &st.CompletedAt, &st.Coverage)
if err == sql.ErrNoRows {
return graph.EnrichmentState{RepoPrefix: repoPrefix, Provider: provider}, false, nil
}
if err != nil {
return graph.EnrichmentState{RepoPrefix: repoPrefix, Provider: provider}, false, err
}
return st, true, nil
}
@@ -0,0 +1,97 @@
package store_sqlite
import (
"path/filepath"
"testing"
"github.com/zzet/gortex/internal/graph"
)
// TestEnrichmentStateRoundTrip: set/get, missing-row, upsert-in-place, and
// per-provider row isolation on the SQLite store.
func TestEnrichmentStateRoundTrip(t *testing.T) {
s, err := Open(filepath.Join(t.TempDir(), "store.sqlite"))
if err != nil {
t.Fatalf("Open: %v", err)
}
defer s.Close()
// Missing row → found=false, no error (the "never enriched" signal).
if _, found, err := s.GetEnrichmentState("repo", "gopls"); err != nil || found {
t.Fatalf("missing row: found=%v err=%v, want found=false, err=nil", found, err)
}
st := graph.EnrichmentState{
RepoPrefix: "repo",
Provider: "gopls",
IndexedSHA: "abc123",
CompletedAt: 1_700_000_000,
Coverage: 91.5,
}
if err := s.SetEnrichmentState(st); err != nil {
t.Fatalf("SetEnrichmentState: %v", err)
}
got, found, err := s.GetEnrichmentState("repo", "gopls")
if err != nil || !found {
t.Fatalf("get after set: found=%v err=%v, want found=true", found, err)
}
if got.RepoPrefix != "repo" || got.Provider != "gopls" ||
got.IndexedSHA != "abc123" || got.CompletedAt != 1_700_000_000 || got.Coverage != 91.5 {
t.Fatalf("round-trip mismatch: %+v", got)
}
// Upsert on (repo_prefix, provider) replaces in place.
st.IndexedSHA = "def456"
st.Coverage = 100
if err := s.SetEnrichmentState(st); err != nil {
t.Fatalf("upsert: %v", err)
}
got, _, _ = s.GetEnrichmentState("repo", "gopls")
if got.IndexedSHA != "def456" || got.Coverage != 100 {
t.Fatalf("upsert did not replace in place: %+v", got)
}
// A different provider under the same repo is a distinct row.
if _, found, _ := s.GetEnrichmentState("repo", "scip-go"); found {
t.Fatalf("a different provider must be its own row, got found=true")
}
}
// TestOpenPreservesEnrichmentStateOnReopen proves the new table is picked up
// by an existing store with no wipe: a marker written on the first open
// survives the second open (which re-runs schemaSQL unconditionally) and the
// schema version does not drift. Mirrors TestOpenAtCurrentVersionIsNoOp.
func TestOpenPreservesEnrichmentStateOnReopen(t *testing.T) {
path := filepath.Join(t.TempDir(), "store.sqlite")
s, err := Open(path)
if err != nil {
t.Fatalf("first open: %v", err)
}
if err := s.SetEnrichmentState(graph.EnrichmentState{
RepoPrefix: "r", Provider: "gopls", IndexedSHA: "sha1", CompletedAt: 42, Coverage: 88,
}); err != nil {
t.Fatalf("seed marker: %v", err)
}
if err := s.Close(); err != nil {
t.Fatalf("close: %v", err)
}
s2, err := Open(path)
if err != nil {
t.Fatalf("reopen: %v", err)
}
defer s2.Close()
got, found, err := s2.GetEnrichmentState("r", "gopls")
if err != nil || !found {
t.Fatalf("marker lost across reopen: found=%v err=%v (the table must survive a no-op reopen)", found, err)
}
if got.IndexedSHA != "sha1" || got.Coverage != 88 {
t.Fatalf("marker corrupted across reopen: %+v", got)
}
if v, _ := readUserVersion(s2.db); v != currentSchemaVersion {
t.Fatalf("schema version drifted to %d after reopen, want %d", v, currentSchemaVersion)
}
}
+118
View File
@@ -0,0 +1,118 @@
package store_sqlite
import (
"github.com/zzet/gortex/internal/graph"
)
// Compile-time assertions that the SQLite Store satisfies the optional
// per-file metadata persistence capability (the files sidecar feeding
// index_health's per-file parse-error / node-count rollup).
var (
_ graph.FileMetaWriter = (*Store)(nil)
_ graph.FileMetaReader = (*Store)(nil)
)
// fileMetaChunk bounds rows per multi-row INSERT (6 params/row; 80 rows =
// 480 host params, well under SQLite's 999 default).
const fileMetaChunk = 80
// SetFileMetas upserts per-file metadata rows for one repo prefix in a single
// transaction, chunked under the host-parameter limit. Idempotent on the
// (repo_prefix, file_path) primary key. Empty input is a no-op.
func (s *Store) SetFileMetas(repoPrefix string, rows []graph.FileMetaRow) error {
if len(rows) == 0 {
return nil
}
s.writeMu.Lock()
defer s.writeMu.Unlock()
tx, err := s.db.Begin()
if err != nil {
return err
}
defer tx.Rollback() //nolint:errcheck // rollback after Commit is a no-op
for start := 0; start < len(rows); start += fileMetaChunk {
end := start + fileMetaChunk
if end > len(rows) {
end = len(rows)
}
batch := rows[start:end]
args := make([]any, 0, len(batch)*6)
stmt := make([]byte, 0, 96+len(batch)*24)
stmt = append(stmt, "INSERT OR REPLACE INTO files (repo_prefix, file_path, content_hash, size, node_count, errors) VALUES "...)
for i, r := range batch {
if i > 0 {
stmt = append(stmt, ',')
}
stmt = append(stmt, "(?, ?, ?, ?, ?, ?)"...)
args = append(args, repoPrefix, r.FilePath, r.ContentHash, r.Size, r.NodeCount, r.Errors)
}
if _, err := tx.Exec(string(stmt), args...); err != nil {
return err
}
}
return tx.Commit()
}
// DeleteFileMetasByFiles drops the metadata rows for the supplied files in one
// repo prefix, chunked into `file_path IN (…)` DELETEs. Empty input is a no-op.
func (s *Store) DeleteFileMetasByFiles(repoPrefix string, files []string) error {
if len(files) == 0 {
return nil
}
s.writeMu.Lock()
defer s.writeMu.Unlock()
tx, err := s.db.Begin()
if err != nil {
return err
}
defer tx.Rollback() //nolint:errcheck // rollback after Commit is a no-op
for start := 0; start < len(files); start += fileMetaChunk {
end := start + fileMetaChunk
if end > len(files) {
end = len(files)
}
chunk := files[start:end]
args := make([]any, 0, len(chunk)+1)
args = append(args, repoPrefix)
stmt := make([]byte, 0, 64+len(chunk)*2)
stmt = append(stmt, "DELETE FROM files WHERE repo_prefix = ? AND file_path IN ("...)
for i, f := range chunk {
if i > 0 {
stmt = append(stmt, ',')
}
stmt = append(stmt, '?')
args = append(args, f)
}
stmt = append(stmt, ')')
if _, err := tx.Exec(string(stmt), args...); err != nil {
return err
}
}
return tx.Commit()
}
// FileMetasForRepo returns every recorded file row for the repo prefix.
// Always non-nil.
func (s *Store) FileMetasForRepo(repoPrefix string) ([]graph.FileMetaRow, error) {
rows, err := s.db.Query(
`SELECT file_path, content_hash, size, node_count, errors FROM files WHERE repo_prefix = ? ORDER BY file_path`,
repoPrefix,
)
if err != nil {
return nil, err
}
defer rows.Close()
out := []graph.FileMetaRow{}
for rows.Next() {
var r graph.FileMetaRow
if err := rows.Scan(&r.FilePath, &r.ContentHash, &r.Size, &r.NodeCount, &r.Errors); err != nil {
return nil, err
}
out = append(out, r)
}
return out, rows.Err()
}
@@ -0,0 +1,69 @@
package store_sqlite
import (
"path/filepath"
"testing"
"github.com/zzet/gortex/internal/graph"
)
// TestFileMetas_RoundTrip pins the per-file metadata sidecar: rows upsert,
// read back per repo, carry their errors JSON, and a per-file delete removes
// just the named file.
func TestFileMetas_RoundTrip(t *testing.T) {
s, err := Open(filepath.Join(t.TempDir(), "f.sqlite"))
if err != nil {
t.Fatal(err)
}
defer s.Close()
rows := []graph.FileMetaRow{
{FilePath: "a/x.go", ContentHash: "h1", Size: 100, NodeCount: 7, Errors: ""},
{FilePath: "a/broken.go", ContentHash: "h2", Size: 50, NodeCount: 1, Errors: `["3:5","4:1"]`},
}
if err := s.SetFileMetas("repoA", rows); err != nil {
t.Fatal(err)
}
// A different repo's row must not bleed in.
if err := s.SetFileMetas("repoB", []graph.FileMetaRow{{FilePath: "b/y.go", NodeCount: 2}}); err != nil {
t.Fatal(err)
}
got, err := s.FileMetasForRepo("repoA")
if err != nil {
t.Fatal(err)
}
if len(got) != 2 {
t.Fatalf("FileMetasForRepo(repoA) = %d rows, want 2", len(got))
}
byFile := map[string]graph.FileMetaRow{}
for _, r := range got {
byFile[r.FilePath] = r
}
if r := byFile["a/x.go"]; r.NodeCount != 7 || r.Size != 100 || r.ContentHash != "h1" || r.Errors != "" {
t.Errorf("x.go row = %+v", r)
}
if r := byFile["a/broken.go"]; r.NodeCount != 1 || r.Errors != `["3:5","4:1"]` {
t.Errorf("broken.go row = %+v", r)
}
// Upsert replaces in place.
if err := s.SetFileMetas("repoA", []graph.FileMetaRow{{FilePath: "a/x.go", NodeCount: 9, Errors: ""}}); err != nil {
t.Fatal(err)
}
got, _ = s.FileMetasForRepo("repoA")
for _, r := range got {
if r.FilePath == "a/x.go" && r.NodeCount != 9 {
t.Errorf("upsert did not replace node_count: %+v", r)
}
}
// Per-file delete removes only the named file.
if err := s.DeleteFileMetasByFiles("repoA", []string{"a/broken.go"}); err != nil {
t.Fatal(err)
}
got, _ = s.FileMetasForRepo("repoA")
if len(got) != 1 || got[0].FilePath != "a/x.go" {
t.Errorf("after delete, rows = %+v, want only a/x.go", got)
}
}
@@ -0,0 +1,38 @@
package store_sqlite
import (
"iter"
"github.com/zzet/gortex/internal/graph"
)
// Compile-time assertion: *Store serves the fn-value placeholder scan.
var _ graph.FnValuePlaceholderScanner = (*Store)(nil)
// FnValuePlaceholderEdges implements graph.FnValuePlaceholderScanner: it yields
// only the fn-value gate's placeholder edges, the exact inverse of the
// fn-value exclusion EdgesWithUnresolvedTarget applies. The predicate is the
// SAME two-form filter the v2 migration's dedupeFnValuePlaceholderEdges uses:
// the bare `unresolved::fnvalue::` range rides edges_by_to(to_id) (the ':;'
// range end is ':'+1, one past the marker); the multi-repo COPY-rewrite infix
// form is caught by is_unresolved = 1 + LIKE. Full column set incl. meta — the
// gate reads Meta["via"] and the captured fn_value_name off each placeholder.
//
// The whole point is that the gate no longer has to scan the entire
// EdgeReferences kind (placeholders + every real reference) and Go-filter on
// every whole-graph synthesizer pass; it pulls the handful of placeholders
// straight off the index instead.
func (s *Store) FnValuePlaceholderEdges() iter.Seq[*graph.Edge] {
return func(yield func(*graph.Edge) bool) {
out := s.queryEdgesSQL(`
SELECT from_id, to_id, kind, file_path, line, confidence, confidence_label, origin, tier, cross_repo, meta, resolve_terminal, resolve_terminal_reason
FROM edges
WHERE (to_id >= 'unresolved::fnvalue::' AND to_id < 'unresolved::fnvalue:;')
OR (is_unresolved = 1 AND to_id LIKE '%::unresolved::fnvalue::%')`)
for _, e := range out {
if !yield(e) {
return
}
}
}
}
+482
View File
@@ -0,0 +1,482 @@
package store_sqlite
import (
"database/sql"
"strings"
"github.com/zzet/gortex/internal/graph"
"github.com/zzet/gortex/internal/search"
)
// This file implements graph.SymbolSearcher + graph.SymbolBundleSearcher
// on the SQLite backend using the FTS5 virtual table declared in
// schema.go (symbol_fts). It is the on-disk replacement for the
// multi-GB in-heap Bleve/BM25 index: the FTS5 inverted index lives in
// the same .sqlite file as the graph, and a tier-0 exact-name boost
// short-circuits identifier queries so
// search quality holds or improves while the heap shrinks.
//
// Semantics:
//
// - BulkUpsertSymbolFTS wipes only the rows owned by repoPrefix
// before re-inserting, so sibling repos sharing one store don't
// clobber each other's corpus. Empty prefix wipes the whole table
// (single-repo / conformance behaviour).
//
// - SearchSymbols tier 0: an identifier query (no whitespace / path
// separators) that resolves to one or more nodes by exact name is
// returned directly with a fixed dominant score, skipping FTS.
// Misses fall through to the FTS5 MATCH path.
//
// - SearchSymbolBundles composes the same hit list with batched
// node + in/out edge fetches the rerank pipeline reads from.
//
// FTS5 maintains its index incrementally on every insert, so the
// Store struct needs no extra state and BuildSymbolIndex is a no-op
// (it only opportunistically merges segments).
// Compile-time assertions: *Store satisfies the symbol-search
// capabilities. The indexer auto-engages these when the active backend
// implements them, routing search_symbols through on-disk FTS5 instead
// of the in-process BM25 index.
var (
_ graph.SymbolSearcher = (*Store)(nil)
_ graph.SymbolBundleSearcher = (*Store)(nil)
_ graph.BundleFingerprintSink = (*Store)(nil)
)
// ftsInsertChunkRows bounds the rows per multi-row INSERT. Each row
// binds 3 host params (node_id, repo_prefix, tokens); 300 rows is 900
// params, comfortably under SQLite's default 999-variable limit so the
// statement stays portable across builds.
const ftsInsertChunkRows = 300
// UpsertSymbolFTS records (or replaces) the pre-tokenised text for
// nodeID. FTS5 offers no UPSERT on a table with UNINDEXED columns, so
// the write is delete-then-insert. The delete targets the prior row's
// FTS5 docid (rowid), looked up from the symbol_fts_rowid sidecar —
// node_id is UNINDEXED, so "DELETE … WHERE node_id = ?" would full-scan
// the whole index once per symbol, which is quadratic over a file's
// symbols on the per-edit reindex hot path. The repo_prefix is derived
// from the owning node (nodes.repo_prefix) so the per-repo staleness
// wipe in BulkUpsertSymbolFTS can scope by prefix; if the node is absent
// the prefix defaults to "".
func (s *Store) UpsertSymbolFTS(nodeID, tokens string) error {
if nodeID == "" {
return nil
}
s.writeMu.Lock()
defer s.writeMu.Unlock()
var repoPrefix string
// A missing node (or a scan error) leaves repoPrefix == "" — the
// row is still indexable, it just won't be reachable by a per-repo
// prefix wipe. The graph.Store contract has no error channel for
// the indexer's incremental writes, so we don't surface this.
_ = s.db.QueryRow(`SELECT repo_prefix FROM nodes WHERE id = ?`, nodeID).Scan(&repoPrefix)
// Delete the prior row by its docid (O(log n)) instead of by node_id
// (full FTS scan). A missing map entry means no prior row to drop —
// the sidecar is kept in lockstep with symbol_fts by every writer and
// backfilled at Open for databases built before it existed, so a miss
// here is a genuinely new symbol, not a stale row we're leaking.
var oldRowid int64
switch err := s.db.QueryRow(
`SELECT fts_rowid FROM symbol_fts_rowid WHERE node_id = ?`, nodeID,
).Scan(&oldRowid); err {
case nil:
if _, err := s.db.Exec(`DELETE FROM symbol_fts WHERE rowid = ?`, oldRowid); err != nil {
return err
}
case sql.ErrNoRows:
// new symbol — nothing to delete
default:
return err
}
res, err := s.db.Exec(
`INSERT INTO symbol_fts (node_id, repo_prefix, tokens) VALUES (?, ?, ?)`,
nodeID, repoPrefix, tokens,
)
if err != nil {
return err
}
newRowid, err := res.LastInsertId()
if err != nil {
return err
}
if _, err := s.db.Exec(
`INSERT OR REPLACE INTO symbol_fts_rowid (node_id, repo_prefix, fts_rowid) VALUES (?, ?, ?)`,
nodeID, repoPrefix, newRowid,
); err != nil {
return err
}
return nil
}
// BulkUpsertSymbolFTS is the cold-start fast path: wipe this repo's
// stale rows, then chunked multi-row INSERT of the deduped items. The
// whole thing runs in one transaction under writeMu so a concurrent
// reader never observes the table mid-wipe.
//
// repoPrefix scopes the pre-insert wipe: a non-empty prefix deletes
// only rows owned by that repo,
// leaving siblings untouched; an empty prefix wipes the whole table
// (single-repo / conformance behaviour — the conformance suite calls
// this with ""). Items are deduped by NodeID with last-write-wins,
// matching UpsertSymbolFTS's replace semantics.
func (s *Store) BulkUpsertSymbolFTS(repoPrefix string, items []graph.SymbolFTSItem) error {
if len(items) == 0 {
return nil
}
s.writeMu.Lock()
defer s.writeMu.Unlock()
// Dedup by ID — last write wins, mirroring UpsertSymbolFTS's
// delete-then-insert. Guards the edge case where a re-parse of a
// file emitted the same ID twice.
pos := make(map[string]int, len(items))
deduped := items[:0]
for _, it := range items {
if it.NodeID == "" {
continue
}
if p, ok := pos[it.NodeID]; ok {
deduped[p] = it
} else {
pos[it.NodeID] = len(deduped)
deduped = append(deduped, it)
}
}
items = deduped
if len(items) == 0 {
return nil
}
tx, err := s.db.Begin()
if err != nil {
return err
}
commit := false
defer func() {
if !commit {
_ = tx.Rollback()
}
}()
// Wipe this repo's prior rows so a clean rebuild of repo A doesn't
// leave phantom hits, while sibling repo B's corpus survives. The
// repo_prefix column is UNINDEXED but still stored, so the equality
// filter is a literal compare over the row set. Empty repoPrefix
// clears the whole table — the legacy single-repo wipe.
if _, err := tx.Exec(`DELETE FROM symbol_fts WHERE repo_prefix = ?`, repoPrefix); err != nil {
return err
}
// Drop this repo's rowid-map entries in lockstep with the symbol_fts
// wipe so the two never diverge; they are rebuilt from the freshly
// inserted rows below.
if _, err := tx.Exec(`DELETE FROM symbol_fts_rowid WHERE repo_prefix = ?`, repoPrefix); err != nil {
return err
}
for start := 0; start < len(items); start += ftsInsertChunkRows {
end := minInt(start+ftsInsertChunkRows, len(items))
chunk := items[start:end]
var b strings.Builder
b.WriteString(`INSERT INTO symbol_fts (node_id, repo_prefix, tokens) VALUES `)
args := make([]any, 0, len(chunk)*3)
for i, it := range chunk {
if i > 0 {
b.WriteByte(',')
}
b.WriteString(`(?,?,?)`)
args = append(args, it.NodeID, repoPrefix, it.Tokens)
}
if _, err := tx.Exec(b.String(), args...); err != nil {
return err
}
}
// Rebuild the rowid map for this repo from the rows just inserted. A
// full multi-row INSERT only exposes the last docid, so we read the
// docids back in one pass (a linear filter over the UNINDEXED
// repo_prefix column — the cold/bulk path, not the per-edit hot path).
if _, err := tx.Exec(
`INSERT OR REPLACE INTO symbol_fts_rowid (node_id, repo_prefix, fts_rowid)
SELECT node_id, repo_prefix, rowid FROM symbol_fts WHERE repo_prefix = ?`,
repoPrefix,
); err != nil {
return err
}
if err := tx.Commit(); err != nil {
return err
}
commit = true
return nil
}
// backfillSymbolFTSRowidMap populates symbol_fts_rowid from symbol_fts for
// a database built before the sidecar existed. Without it, the first
// incremental UpsertSymbolFTS for an already-indexed symbol would find no
// map entry, skip the delete, and leak a duplicate FTS row. It is a
// one-time cost: skipped once the map has any row (steady state) or when
// the FTS index is empty (a fresh DB the bulk path will populate with the
// map maintained inline). Runs at Open, before any reader or writer.
func backfillSymbolFTSRowidMap(db *sql.DB) error {
var mapped bool
if err := db.QueryRow(`SELECT EXISTS(SELECT 1 FROM symbol_fts_rowid)`).Scan(&mapped); err != nil {
return err
}
if mapped {
return nil
}
var hasFTS bool
if err := db.QueryRow(`SELECT EXISTS(SELECT 1 FROM symbol_fts)`).Scan(&hasFTS); err != nil {
return err
}
if !hasFTS {
return nil
}
_, err := db.Exec(
`INSERT OR REPLACE INTO symbol_fts_rowid (node_id, repo_prefix, fts_rowid)
SELECT node_id, repo_prefix, rowid FROM symbol_fts`)
return err
}
// BuildSymbolIndex is a no-op for FTS5: the index is maintained
// incrementally on every insert, so there is nothing to build after the
// bulk parse phase. We opportunistically run the FTS5 'optimize'
// command to merge segments (purely a read-latency improvement); any
// error is ignored because the index is already correct without it.
// Idempotent — safe to call any number of times.
func (s *Store) BuildSymbolIndex() error {
s.writeMu.Lock()
defer s.writeMu.Unlock()
_, _ = s.db.Exec(`INSERT INTO symbol_fts(symbol_fts) VALUES('optimize')`)
return nil
}
// SearchSymbols runs a symbol query and returns hits ordered by
// descending relevance (higher Score = more relevant).
//
// Tier 0 (exact-name boost): when the
// query looks like a literal identifier and resolves to one or more
// nodes by exact name, return those directly with a fixed dominant
// score (100.0) — an O(1)-ish index seek that beats FTS ranking for
// the common "type the symbol name" case. Misses fall through to FTS5.
//
// Otherwise tokenise on the read side with the SAME splitter as the
// write side (search.Tokenize) so a camelCase query lands on the
// split corpus, build a prefix-OR MATCH expression, and rank by BM25.
// SQLite's bm25() returns lower-is-better, so the stored Score is its
// negation (higher-is-better, matching the SymbolHit contract).
func (s *Store) SearchSymbols(query string, limit int) ([]graph.SymbolHit, error) {
if query == "" {
return nil, nil
}
if limit <= 0 {
limit = 20
}
// Tier 0: exact-name lookup. Only engage for identifier-shaped
// queries (no whitespace / path separators); multi-word queries are
// concept searches that need BM25 ranking. We only short-circuit
// when the lookup hits at least one node — misses fall through so a
// partial-identifier query still reaches FTS.
if isIdentifierQuery(query) {
ns := s.FindNodesByName(query)
if len(ns) > 0 {
out := make([]graph.SymbolHit, 0, minInt(len(ns), limit))
for _, n := range ns {
if n == nil || n.ID == "" {
continue
}
out = append(out, graph.SymbolHit{NodeID: n.ID, Score: 100.0})
if len(out) >= limit {
break
}
}
if len(out) > 0 {
return out, nil
}
}
}
match := s.buildFTSMatch(query)
if match == "" {
return nil, nil
}
const q = `SELECT node_id, bm25(symbol_fts) FROM symbol_fts WHERE symbol_fts MATCH ? ORDER BY bm25(symbol_fts) LIMIT ?`
rows, err := s.db.Query(q, match, limit)
if err != nil {
return nil, err
}
defer rows.Close()
var hits []graph.SymbolHit
for rows.Next() {
var (
id string
score float64
)
if err := rows.Scan(&id, &score); err != nil {
return nil, err
}
if id == "" {
continue
}
// bm25() is negative-better in SQLite; negate so higher = better,
// matching the SymbolHit contract. Rows already arrive in bm25
// (best-first) order from the ORDER BY.
hits = append(hits, graph.SymbolHit{NodeID: id, Score: -score})
}
if err := rows.Err(); err != nil {
return nil, err
}
return hits, nil
}
// buildFTSMatch tokenises the query with the write-side splitter and
// builds an FTS5 MATCH expression: each token becomes a quoted prefix
// term ("tok"*) and the terms are OR-joined so any token match counts.
// Returns "" when the query degenerates to no tokens.
func (s *Store) buildFTSMatch(query string) string {
tokens := search.Tokenize(query)
if len(tokens) == 0 {
// Fallback: when Tokenize drops everything (e.g. a single
// sub-2-char token like "go"), use the looser query tokeniser so
// the search still reaches the engine instead of returning empty.
tokens = search.TokenizeQuery(query)
if len(tokens) == 0 {
return ""
}
}
parts := make([]string, 0, len(tokens))
for _, t := range tokens {
if t == "" {
continue
}
parts = append(parts, `"`+escapeFTSQuote(t)+`"*`)
}
if len(parts) == 0 {
return ""
}
return strings.Join(parts, " OR ")
}
// escapeFTSQuote escapes a token for use inside an FTS5 double-quoted
// string literal: a literal double quote is doubled ("" inside "...").
func escapeFTSQuote(t string) string {
return strings.ReplaceAll(t, `"`, `""`)
}
// SearchSymbolBundles is the rerank-shaped fast path: it runs
// SearchSymbols to get the ranked id list (preserving order) plus a
// score-by-id map, then materialises the nodes and their in/out edges
// in batched fetches the rerank pipeline reads from. The engine routes
// through this when the backend implements SymbolBundleSearcher,
// pre-seeding rerank.Context's edge caches.
func (s *Store) SearchSymbolBundles(query string, limit int) ([]graph.SymbolBundle, error) {
hits, err := s.SearchSymbols(query, limit)
if err != nil {
return nil, err
}
if len(hits) == 0 {
return nil, nil
}
ids := make([]string, 0, len(hits))
scoreByID := make(map[string]float64, len(hits))
for _, h := range hits {
if h.NodeID == "" {
continue
}
if _, dup := scoreByID[h.NodeID]; dup {
// First hit keeps the score / position; defend against a
// future ranker that returns an id more than once.
continue
}
scoreByID[h.NodeID] = h.Score
ids = append(ids, h.NodeID)
}
if len(ids) == 0 {
return nil, nil
}
// Content-addressed cache: serve cached bundles for IDs whose
// package fingerprint is unchanged and fetch only the misses. The
// cache is nil until the daemon wires fingerprints, in which case
// every ID is a miss and the path is exactly the legacy fetch.
cached := make(map[string]graph.SymbolBundle, len(ids))
missIDs := ids
if s.bundles != nil {
missIDs = missIDs[:0:0]
for _, id := range ids {
if b, ok := s.bundles.lookup(id); ok {
cached[id] = b
continue
}
missIDs = append(missIDs, id)
}
}
// Fetch the misses' nodes + in/out edges in one batched round-trip
// each. A full cache hit skips all three fetches entirely.
var nodes map[string]*graph.Node
var out, in map[string][]*graph.Edge
if len(missIDs) > 0 {
nodes = s.GetNodesByIDs(missIDs)
out = s.GetOutEdgesByNodeIDs(missIDs)
in = s.GetInEdgesByNodeIDs(missIDs)
}
bundles := make([]graph.SymbolBundle, 0, len(ids))
for _, id := range ids {
if b, ok := cached[id]; ok {
// The cached bundle's score is whatever it was first cached
// with; the live FTS score for THIS query is authoritative,
// so re-stamp it (the score is query-specific, the node +
// edges are not).
b.Score = scoreByID[id]
bundles = append(bundles, b)
continue
}
n := nodes[id]
if n == nil {
// Hit references a node evicted between the search and the
// node fetch — skip; the caller does its own dedup / filter.
continue
}
b := graph.SymbolBundle{
Node: n,
Score: scoreByID[id],
OutEdges: out[id],
InEdges: in[id],
}
if s.bundles != nil {
s.bundles.store(b)
}
bundles = append(bundles, b)
}
return bundles, nil
}
// isIdentifierQuery reports whether a query looks like a literal symbol
// name (no whitespace, no path separators, no dots, no colons, no
// commas). The tier-0 exact-name fast path engages only on such
// queries; multi-token / path / qualified queries always go to FTS.
func isIdentifierQuery(q string) bool {
if q == "" {
return false
}
for _, r := range q {
switch r {
case ' ', '\t', '\n', '/', '.', ':', ',':
return false
}
}
return true
}
@@ -0,0 +1,45 @@
package store_sqlite
import (
"database/sql"
"github.com/zzet/gortex/internal/graph"
)
// SetRepoIndexState upserts the freshness-provenance row for one repo —
// written at the end of every (re)index. One row per repo_prefix.
func (s *Store) SetRepoIndexState(st graph.RepoIndexState) error {
s.writeMu.Lock()
defer s.writeMu.Unlock()
dirty := 0
if st.Dirty {
dirty = 1
}
_, err := s.db.Exec(`
INSERT OR REPLACE INTO repo_index_state
(repo_prefix, indexed_sha, dirty, indexed_at, workspace_fp, node_count, edge_count, extractor_versions)
VALUES (?, ?, ?, ?, ?, ?, ?, ?)`,
st.RepoPrefix, st.IndexedSHA, dirty, st.IndexedAt, st.WorkspaceFP,
st.NodeCount, st.EdgeCount, st.ExtractorVersions)
return err
}
// GetRepoIndexState returns the recorded freshness provenance for a repo.
// The bool is false when no row exists yet (never-indexed / pre-feature).
func (s *Store) GetRepoIndexState(repoPrefix string) (graph.RepoIndexState, bool, error) {
row := s.db.QueryRow(`
SELECT indexed_sha, dirty, indexed_at, workspace_fp, node_count, edge_count, extractor_versions
FROM repo_index_state WHERE repo_prefix = ?`, repoPrefix)
st := graph.RepoIndexState{RepoPrefix: repoPrefix}
var dirty int
err := row.Scan(&st.IndexedSHA, &dirty, &st.IndexedAt, &st.WorkspaceFP,
&st.NodeCount, &st.EdgeCount, &st.ExtractorVersions)
if err == sql.ErrNoRows {
return graph.RepoIndexState{RepoPrefix: repoPrefix}, false, nil
}
if err != nil {
return graph.RepoIndexState{RepoPrefix: repoPrefix}, false, err
}
st.Dirty = dirty != 0
return st, true, nil
}
@@ -0,0 +1,61 @@
package store_sqlite_test
import (
"path/filepath"
"testing"
"github.com/stretchr/testify/require"
"github.com/zzet/gortex/internal/graph"
"github.com/zzet/gortex/internal/graph/store_sqlite"
)
func openIndexStateStore(t *testing.T) *store_sqlite.Store {
t.Helper()
s, err := store_sqlite.Open(filepath.Join(t.TempDir(), "is.sqlite"))
require.NoError(t, err)
t.Cleanup(func() { _ = s.Close() })
return s
}
func TestRepoIndexState_RoundTrip(t *testing.T) {
s := openIndexStateStore(t)
// Absent state reads back as (zero, false, nil).
got, ok, err := s.GetRepoIndexState("gortex")
require.NoError(t, err)
require.False(t, ok)
require.Equal(t, "gortex", got.RepoPrefix)
want := graph.RepoIndexState{
RepoPrefix: "gortex",
IndexedSHA: "abc123",
Dirty: true,
IndexedAt: 1700000000,
WorkspaceFP: "deadbeef",
NodeCount: 42,
EdgeCount: 99,
ExtractorVersions: `{"go":2}`,
}
require.NoError(t, s.SetRepoIndexState(want))
got, ok, err = s.GetRepoIndexState("gortex")
require.NoError(t, err)
require.True(t, ok)
require.Equal(t, want, got)
// Upsert replaces in place (one row per repo_prefix).
want.IndexedSHA = "def456"
want.Dirty = false
require.NoError(t, s.SetRepoIndexState(want))
got, ok, err = s.GetRepoIndexState("gortex")
require.NoError(t, err)
require.True(t, ok)
require.Equal(t, "def456", got.IndexedSHA)
require.False(t, got.Dirty)
// A different repo is isolated.
_, ok, err = s.GetRepoIndexState("other")
require.NoError(t, err)
require.False(t, ok)
}
@@ -0,0 +1,54 @@
package store_sqlite
import "github.com/zzet/gortex/internal/graph"
// Compile-time assertion: *Store serves the meta-less kind-scoped edge scan.
var _ graph.LightEdgeScanner = (*Store)(nil)
// edgeColsLight is the meta-less edge column projection: the promoted struct
// columns WITHOUT the meta blob (and without resolve_terminal, which lives in
// Meta). It is exactly the ten columns scanEdgeLight scans, and is shared with
// the stmtOutEdgesLight prepared statement so the projection can never drift
// from the scanner. Adding meta back here would defeat the whole point — the
// per-row JSON decode this projection exists to skip.
const edgeColsLight = `from_id, to_id, kind, file_path, line, confidence, confidence_label, origin, tier, cross_repo`
// AllEdgesLight implements graph.LightEdgeScanner: a kind-scoped edge scan that
// never decodes the meta blob. An empty kinds list scans every edge; supplying
// only empty-string kinds matches nothing (parity with the aggregators). The
// edges_by_kind index serves the IN filter. Meta is left nil; only the promoted
// fields (origin/tier/confidence/confidence_label/cross_repo/line) are hydrated.
func (s *Store) AllEdgesLight(kinds ...graph.EdgeKind) []*graph.Edge {
_, args := aggDedupeEdgeKinds(kinds)
if len(args) == 0 {
if len(kinds) > 0 {
return nil // caller passed only empty kinds — nothing matches
}
return s.queryEdgesLightSQL(`SELECT ` + edgeColsLight + ` FROM edges ORDER BY id`)
}
q := `SELECT ` + edgeColsLight + ` FROM edges WHERE kind IN (` +
inPlaceholders(len(args)) + `) ORDER BY id`
return s.queryEdgesLightSQL(q, args...)
}
// queryEdgesLightSQL is the meta-less sibling of queryEdgesSQL: it materialises
// the rows into a slice and closes the cursor before returning (releasing the
// single pooled connection), but scans through scanEdgeLight so the meta column
// is never transferred or decoded. Returns nil on any query error, matching
// queryEdgesSQL — a teardown-race read degrades to empty rather than panicking.
func (s *Store) queryEdgesLightSQL(q string, args ...any) []*graph.Edge {
rows, err := s.db.Query(q, args...)
if err != nil {
return nil
}
defer rows.Close()
var out []*graph.Edge
for rows.Next() {
e, err := scanEdgeLight(rows)
if err != nil || e == nil {
continue
}
out = append(out, e)
}
return out
}
@@ -0,0 +1,54 @@
package store_sqlite_test
import (
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/zzet/gortex/internal/graph"
)
// TestAllEdgesLightIsMetaless pins the disk backend's graph.LightEdgeScanner:
// the scan must skip the meta blob (Meta == nil) — the per-edge JSON decode the
// warm-restart analysis passes exist to avoid — while every promoted field
// still equals what the full AllEdges() scan returns.
func TestAllEdgesLightIsMetaless(t *testing.T) {
s := openTestStore(t)
s.AddNode(&graph.Node{ID: "p/a.go::A", Kind: graph.KindFunction, Name: "A", FilePath: "p/a.go"})
s.AddNode(&graph.Node{ID: "p/a.go::B", Kind: graph.KindFunction, Name: "B", FilePath: "p/a.go"})
s.AddEdge(&graph.Edge{
From: "p/a.go::A", To: "p/a.go::B", Kind: graph.EdgeCalls,
FilePath: "p/a.go", Line: 11, Confidence: 0.75, ConfidenceLabel: "high",
Origin: graph.OriginLSPResolved, Tier: "lsp", CrossRepo: true,
Meta: map[string]any{"via": "direct", "blob_only": "x"},
})
// A second kind to prove the IN filter really scopes.
s.AddEdge(&graph.Edge{From: "p/a.go::A", To: "p/a.go::B", Kind: graph.EdgeImports, FilePath: "p/a.go", Line: 1})
light := s.AllEdgesLight(graph.EdgeCalls)
require.Len(t, light, 1, "kind filter must exclude the imports edge")
e := light[0]
assert.Nil(t, e.Meta, "AllEdgesLight must not decode the meta blob")
var full *graph.Edge
for _, fe := range s.AllEdges() {
if fe.Kind == graph.EdgeCalls {
full = fe
}
}
require.NotNil(t, full)
assert.NotNil(t, full.Meta, "sanity: the full scan DOES decode meta")
assert.Equal(t, full.From, e.From)
assert.Equal(t, full.To, e.To)
assert.Equal(t, full.Kind, e.Kind)
assert.Equal(t, full.Line, e.Line)
assert.Equal(t, full.Confidence, e.Confidence)
assert.Equal(t, full.ConfidenceLabel, e.ConfidenceLabel)
assert.Equal(t, full.Origin, e.Origin)
assert.Equal(t, full.Tier, e.Tier)
assert.Equal(t, full.CrossRepo, e.CrossRepo)
// Empty kinds means every edge.
assert.Len(t, s.AllEdgesLight(), 2)
}
@@ -0,0 +1,204 @@
package store_sqlite
import (
"strings"
"github.com/zzet/gortex/internal/graph"
)
// These methods were added to graph.Store after the sqlite backend was
// first removed; they are restored here so *Store satisfies the current
// interface. All reuse the chunked IN-list / raw-SQL helpers in store.go
// (queryNodesSQL / queryEdgesSQL / lookupChunkSize / minInt). SQLite's
// planner drives every one through the existing secondary indexes.
// lookupNodeCols is the canonical node column list (and scan order) for
// every node-shaped SELECT in the package. It must stay in sync with
// scanNode. The struct columns (start_column/end_column) sit with the line
// range; the promoted meta columns (signature/visibility/doc/external/
// return_type/is_async/is_static/is_abstract/is_exported/updated_at/
// data_class/semantic_type/semantic_source) precede meta.
const lookupNodeCols = `id, kind, name, qual_name, file_path, start_line, end_line, start_column, end_column, language, repo_prefix, workspace_id, project_id, signature, visibility, doc, external, return_type, is_async, is_static, is_abstract, is_exported, updated_at, data_class, semantic_type, semantic_source, meta`
// lookupNodeColsLight is lookupNodeCols without the trailing meta column —
// the projection GetRepoNodesLight uses so a repo-scoped scan never
// transfers or decodes a single blob. Derived, not hand-duplicated, so it
// can never drift out of sync with lookupNodeCols / scanNode.
var lookupNodeColsLight = strings.TrimSuffix(lookupNodeCols, ", meta")
const lookupEdgeCols = `from_id, to_id, kind, file_path, line, confidence, confidence_label, origin, tier, cross_repo, meta, resolve_terminal, resolve_terminal_reason`
// Compile-time assertion: *Store satisfies graph.NodeNameClassCounter.
var _ graph.NodeNameClassCounter = (*Store)(nil)
// CountNodesByNameClass implements graph.NodeNameClassCounter: for each
// distinct name, it tallies how many nodes.name matches are Real (is_stub =
// 0 and kind IN definitionKinds) vs Stub (is_stub = 1), server-side via
// nodes_by_name — one aggregate query per chunk instead of one
// FindNodesByName round trip per name. A name absent from the returned map
// has no matching node at all (Real == Stub == 0 either way).
func (s *Store) CountNodesByNameClass(names []string, definitionKinds []graph.NodeKind) map[string]graph.NodeNameClassCount {
_, kindArgs := aggDedupeNodeKinds(definitionKinds)
if len(kindArgs) == 0 {
return nil
}
uniq := dedupeNonEmpty(names)
if len(uniq) == 0 {
return nil
}
out := make(map[string]graph.NodeNameClassCount, len(uniq))
kindPlaceholders := inPlaceholders(len(kindArgs))
for i := 0; i < len(uniq); i += lookupChunkSize {
end := minInt(i+lookupChunkSize, len(uniq))
chunk := uniq[i:end]
q := `SELECT name,
SUM(CASE WHEN is_stub = 0 AND kind IN (` + kindPlaceholders + `) THEN 1 ELSE 0 END),
SUM(CASE WHEN is_stub = 1 THEN 1 ELSE 0 END)
FROM nodes
WHERE name IN (` + inPlaceholders(len(chunk)) + `)
GROUP BY name`
args := make([]any, 0, len(kindArgs)+len(chunk))
args = append(args, kindArgs...)
args = append(args, toAnyArgs(chunk)...)
rows, err := s.db.Query(q, args...)
if err != nil {
panicOnFatal(err)
return out
}
for rows.Next() {
var name string
var c graph.NodeNameClassCount
if err := rows.Scan(&name, &c.Real, &c.Stub); err != nil {
_ = rows.Close()
panicOnFatal(err)
return out
}
out[name] = c
}
if err := rows.Err(); err != nil {
_ = rows.Close()
panicOnFatal(err)
return out
}
_ = rows.Close()
}
return out
}
// FindNodesByNameContaining returns nodes whose Name contains substr,
// case-insensitively (SQLite's LIKE is ASCII case-insensitive). An empty
// substring matches nothing (parity with the in-memory store); a limit > 0
// caps the result set. The leading-wildcard LIKE is a deliberate full scan —
// no index accelerates an unanchored substring — matching the in-memory
// strings.Contains fallback. % and _ in substr are escaped so they match
// literally.
func (s *Store) FindNodesByNameContaining(substr string, limit int) []*graph.Node {
if substr == "" {
return nil
}
pattern := "%" + escapeLikePattern(substr) + "%"
q := `SELECT ` + lookupNodeCols + ` FROM nodes WHERE name LIKE ? ESCAPE '\' ORDER BY id`
if limit > 0 {
return s.queryNodesSQL(q+` LIMIT ?`, pattern, limit)
}
return s.queryNodesSQL(q, pattern)
}
// GetNodesByQualNames returns a map qualName→*Node (first match per
// qual_name) for the batch — the qual-name twin of FindNodesByNames, used to
// pre-warm import resolution. Driven by the unique nodes_by_qual index.
func (s *Store) GetNodesByQualNames(qualNames []string) map[string]*graph.Node {
uniq := dedupeNonEmpty(qualNames)
if len(uniq) == 0 {
return nil
}
out := make(map[string]*graph.Node, len(uniq))
for i := 0; i < len(uniq); i += lookupChunkSize {
end := minInt(i+lookupChunkSize, len(uniq))
chunk := uniq[i:end]
q := `SELECT ` + lookupNodeCols + ` FROM nodes WHERE qual_name IN (` + inPlaceholders(len(chunk)) + `)`
for _, n := range s.queryNodesSQL(q, toAnyArgs(chunk)...) {
if n == nil {
continue
}
if _, ok := out[n.QualName]; !ok {
out[n.QualName] = n
}
}
}
return out
}
// GetOutEdgesByNodeIDs batches per-node out-edge fan-out into one query per
// chunk. Missing IDs are simply absent from the returned map.
func (s *Store) GetOutEdgesByNodeIDs(ids []string) map[string][]*graph.Edge {
return s.edgesByNodeIDs(ids, "from_id", func(e *graph.Edge) string { return e.From })
}
// GetInEdgesByNodeIDs is the incoming-edge twin of GetOutEdgesByNodeIDs.
func (s *Store) GetInEdgesByNodeIDs(ids []string) map[string][]*graph.Edge {
return s.edgesByNodeIDs(ids, "to_id", func(e *graph.Edge) string { return e.To })
}
// edgesByNodeIDs runs the chunked IN-list edge fetch keyed on the given
// column (from_id or to_id), grouping results by the supplied key extractor.
func (s *Store) edgesByNodeIDs(ids []string, col string, key func(*graph.Edge) string) map[string][]*graph.Edge {
uniq := dedupeNonEmpty(ids)
if len(uniq) == 0 {
return nil
}
out := make(map[string][]*graph.Edge, len(uniq))
for i := 0; i < len(uniq); i += lookupChunkSize {
end := minInt(i+lookupChunkSize, len(uniq))
chunk := uniq[i:end]
q := `SELECT ` + lookupEdgeCols + ` FROM edges WHERE ` + col + ` IN (` + inPlaceholders(len(chunk)) + `)`
for _, e := range s.queryEdgesSQL(q, toAnyArgs(chunk)...) {
if e == nil {
continue
}
k := key(e)
out[k] = append(out[k], e)
}
}
return out
}
// dedupeNonEmpty drops empties and duplicates, preserving first-seen order.
func dedupeNonEmpty(in []string) []string {
seen := make(map[string]struct{}, len(in))
out := make([]string, 0, len(in))
for _, v := range in {
if v == "" {
continue
}
if _, ok := seen[v]; ok {
continue
}
seen[v] = struct{}{}
out = append(out, v)
}
return out
}
// inPlaceholders returns "?,?,?" for n bound parameters.
func inPlaceholders(n int) string {
if n <= 0 {
return ""
}
return strings.Repeat(",?", n)[1:]
}
// toAnyArgs widens a string slice for variadic Query/Exec args.
func toAnyArgs(ss []string) []any {
args := make([]any, len(ss))
for i, v := range ss {
args[i] = v
}
return args
}
// escapeLikePattern escapes the LIKE metacharacters so the substring matches
// literally under `... LIKE ? ESCAPE '\'`.
func escapeLikePattern(s string) string {
return strings.NewReplacer(`\`, `\\`, `%`, `\%`, `_`, `\_`).Replace(s)
}
@@ -0,0 +1,54 @@
package store_sqlite_test
import (
"path/filepath"
"testing"
"github.com/stretchr/testify/require"
"github.com/zzet/gortex/internal/graph"
"github.com/zzet/gortex/internal/graph/store_sqlite"
)
// TestAllRepoMemoryEstimates_Memoized verifies the short-TTL memoisation:
// within the TTL a second call serves the cached estimate instead of
// re-running the COUNT … GROUP BY scan, so the status path can poll cheaply
// even while enrichment is writing to the same store.
func TestAllRepoMemoryEstimates_Memoized(t *testing.T) {
path := filepath.Join(t.TempDir(), "g.sqlite")
s, err := store_sqlite.Open(path)
require.NoError(t, err)
t.Cleanup(func() { _ = s.Close() })
totalNodes := func(m map[string]graph.RepoMemoryEstimate) int {
n := 0
for _, e := range m {
n += e.NodeCount
}
return n
}
s.AddBatch([]*graph.Node{
{ID: "a.go::Foo", Kind: graph.KindFunction, Name: "Foo", FilePath: "a.go", RepoPrefix: "r1"},
{ID: "a.go::Bar", Kind: graph.KindFunction, Name: "Bar", FilePath: "a.go", RepoPrefix: "r1"},
}, nil)
first := s.AllRepoMemoryEstimates()
require.Equal(t, 2, totalNodes(first))
// Add another node; within the memoisation TTL the estimate is served
// from cache, so the freshly-added node is intentionally not yet
// reflected — proof the COUNT scan was skipped.
s.AddBatch([]*graph.Node{
{ID: "b.go::Baz", Kind: graph.KindFunction, Name: "Baz", FilePath: "b.go", RepoPrefix: "r1"},
}, nil)
cached := s.AllRepoMemoryEstimates()
require.Equal(t, 2, totalNodes(cached), "within the TTL the result should be the memoised value, not a fresh COUNT")
// The returned map is a copy: mutating it must not corrupt the cache.
for k := range cached {
delete(cached, k)
}
again := s.AllRepoMemoryEstimates()
require.Equal(t, 2, totalNodes(again), "the cache must hand back an independent copy")
}
+243
View File
@@ -0,0 +1,243 @@
package store_sqlite
import (
"database/sql"
"github.com/zzet/gortex/internal/graph"
)
// Compile-time assertions that the SQLite Store satisfies the optional
// per-file mtime persistence capabilities. Lifting this state into the
// same backend the graph lives in means warm restarts read it through
// one persistence surface instead of a second gob snapshot.
var (
_ graph.FileMtimeWriter = (*Store)(nil)
_ graph.FileMtimeReader = (*Store)(nil)
_ graph.FileMtimeReplacer = (*Store)(nil)
_ graph.FileMtimeDeleter = (*Store)(nil)
)
// mtimeChunk bounds how many (repo_prefix, file_path, mtime_ns) tuples
// ride in a single multi-row INSERT. SQLite's default compiled-in host
// parameter limit is 999; at 3 params per row that caps a statement at
// 333 rows, so 300 leaves headroom.
const mtimeChunk = 300
// SetFileMtime records one file's modification time (nanoseconds since
// the epoch) for a repo prefix, replacing any prior value. It is a
// convenience single-row form of BulkSetFileMtimes.
func (s *Store) SetFileMtime(repoPrefix, filePath string, mtimeNs int64) error {
s.writeMu.Lock()
defer s.writeMu.Unlock()
_, err := s.db.Exec(
`INSERT OR REPLACE INTO file_mtimes (repo_prefix, file_path, mtime_ns) VALUES (?, ?, ?)`,
repoPrefix, filePath, mtimeNs,
)
return err
}
// BulkSetFileMtimes persists every (filePath -> mtimeNs) entry for one
// repo prefix in a single transaction, chunked so no statement exceeds
// SQLite's host-parameter limit. Idempotent on (repoPrefix, filePath):
// re-running with overlapping keys replaces in place. Empty input is a
// no-op.
func (s *Store) BulkSetFileMtimes(repoPrefix string, mtimes map[string]int64) error {
if len(mtimes) == 0 {
return nil
}
s.writeMu.Lock()
defer s.writeMu.Unlock()
tx, err := s.db.Begin()
if err != nil {
return err
}
defer tx.Rollback() //nolint:errcheck // rollback after Commit is a no-op
if err := insertMtimesTx(tx, repoPrefix, mtimes); err != nil {
return err
}
return tx.Commit()
}
// ReplaceFileMtimes persists the AUTHORITATIVE full mtime set for one repo
// prefix: every prior row for the prefix is dropped and the supplied set is
// written, all in one transaction. The full-index persist path uses this so
// files deleted since the last index are pruned — BulkSetFileMtimes (upsert)
// would leave their rows behind, and warm-restart reconcile would then
// detect them as phantom deletions on every restart, forcing a full
// re-track that never converges.
//
// Empty input is a deliberate no-op: it never wipes a repo's mtimes from an
// empty snapshot (the indexer guards the call with len(snapshot) > 0).
func (s *Store) ReplaceFileMtimes(repoPrefix string, mtimes map[string]int64) error {
if len(mtimes) == 0 {
return nil
}
s.writeMu.Lock()
defer s.writeMu.Unlock()
tx, err := s.db.Begin()
if err != nil {
return err
}
defer tx.Rollback() //nolint:errcheck // rollback after Commit is a no-op
if _, err := tx.Exec(`DELETE FROM file_mtimes WHERE repo_prefix = ?`, repoPrefix); err != nil {
return err
}
if err := insertMtimesTx(tx, repoPrefix, mtimes); err != nil {
return err
}
return tx.Commit()
}
// DeleteFileMtimes drops the rows for a set of repo-relative file paths
// under one repo prefix — the incremental-reindex sibling of
// ReplaceFileMtimes. The watcher / incremental path calls it when a file is
// deleted so the persisted set stays in step with the live graph and the
// next warm restart does not see the path as a phantom deletion. Empty
// input is a no-op.
func (s *Store) DeleteFileMtimes(repoPrefix string, paths []string) error {
if len(paths) == 0 {
return nil
}
s.writeMu.Lock()
defer s.writeMu.Unlock()
tx, err := s.db.Begin()
if err != nil {
return err
}
defer tx.Rollback() //nolint:errcheck // rollback after Commit is a no-op
// Chunk so the IN-list never exceeds SQLite's host-parameter limit:
// one leading repo_prefix arg + up to mtimeChunk path args per stmt.
for start := 0; start < len(paths); start += mtimeChunk {
end := min(start+mtimeChunk, len(paths))
batch := paths[start:end]
args := make([]any, 0, len(batch)+1)
args = append(args, repoPrefix)
stmt := make([]byte, 0, 64+len(batch)*2)
stmt = append(stmt, "DELETE FROM file_mtimes WHERE repo_prefix = ? AND file_path IN ("...)
for i := range batch {
if i > 0 {
stmt = append(stmt, ',')
}
stmt = append(stmt, '?')
args = append(args, batch[i])
}
stmt = append(stmt, ')')
if _, err := tx.Exec(string(stmt), args...); err != nil {
return err
}
}
return tx.Commit()
}
// insertMtimesTx writes every (path -> ns) entry for repoPrefix into the
// given transaction with chunked multi-row INSERT OR REPLACE statements,
// each kept under SQLite's host-parameter limit. The caller owns the tx
// lifecycle (Begin/Commit/Rollback) and the write lock.
func insertMtimesTx(tx *sql.Tx, repoPrefix string, mtimes map[string]int64) error {
// Stable ordering is not required for correctness, but iterating the
// map directly is fine — we only chunk by count.
type kv struct {
path string
ns int64
}
pending := make([]kv, 0, len(mtimes))
for p, ns := range mtimes {
pending = append(pending, kv{path: p, ns: ns})
}
for start := 0; start < len(pending); start += mtimeChunk {
end := min(start+mtimeChunk, len(pending))
batch := pending[start:end]
// Build a multi-row INSERT OR REPLACE: (?, ?, ?), (?, ?, ?), ...
args := make([]any, 0, len(batch)*3)
stmt := make([]byte, 0, 64+len(batch)*16)
stmt = append(stmt, "INSERT OR REPLACE INTO file_mtimes (repo_prefix, file_path, mtime_ns) VALUES "...)
for i, e := range batch {
if i > 0 {
stmt = append(stmt, ',')
}
stmt = append(stmt, "(?, ?, ?)"...)
args = append(args, repoPrefix, e.path, e.ns)
}
if _, err := tx.Exec(string(stmt), args...); err != nil {
return err
}
}
return nil
}
// LoadFileMtimes returns the recorded mtimes for one repo prefix as a
// fresh map. Returns nil when there is no data for the prefix (the
// "no recorded state" signal warmup expects).
func (s *Store) LoadFileMtimes(repoPrefix string) map[string]int64 {
rows, err := s.db.Query(
`SELECT file_path, mtime_ns FROM file_mtimes WHERE repo_prefix = ?`,
repoPrefix,
)
if err != nil {
return nil
}
defer rows.Close()
var out map[string]int64
for rows.Next() {
var path string
var ns int64
if err := rows.Scan(&path, &ns); err != nil {
return nil
}
if out == nil {
out = make(map[string]int64)
}
out[path] = ns
}
if err := rows.Err(); err != nil {
return nil
}
return out
}
// FileMtimes is a fallible read form of LoadFileMtimes. It always
// returns a non-nil (possibly empty) map for a known/unknown prefix and
// surfaces any query error. The interface method LoadFileMtimes is the
// daemon's entry point; this variant exists for callers (and tests)
// that want the error and an always-materialised map.
func (s *Store) FileMtimes(repoPrefix string) (map[string]int64, error) {
rows, err := s.db.Query(
`SELECT file_path, mtime_ns FROM file_mtimes WHERE repo_prefix = ?`,
repoPrefix,
)
if err != nil {
return nil, err
}
defer rows.Close()
out := make(map[string]int64)
for rows.Next() {
var path string
var ns int64
if err := rows.Scan(&path, &ns); err != nil {
return nil, err
}
out[path] = ns
}
if err := rows.Err(); err != nil {
return nil, err
}
return out, nil
}
@@ -0,0 +1,112 @@
package store_sqlite_test
import (
"reflect"
"testing"
"github.com/zzet/gortex/internal/graph"
)
// TestReplaceFileMtimesPrunesDeleted is the regression for the warm-restart
// "nothing changed but full re-track" bug: the full-index persist path must
// REPLACE a repo's mtime set, not union into it. An upsert-only persist
// leaves rows for files deleted since the last index behind, and warm-restart
// reconcile then detects them as phantom deletions on every restart — forcing
// a full re-track that never converges.
func TestReplaceFileMtimesPrunesDeleted(t *testing.T) {
s := openTestStore(t)
// Assert the store advertises the capability the indexer probes for.
var _ graph.FileMtimeReplacer = s
var _ graph.FileMtimeDeleter = s
// First index: three files persisted.
require := func(err error, what string) {
t.Helper()
if err != nil {
t.Fatalf("%s: %v", what, err)
}
}
require(s.BulkSetFileMtimes("repoA", map[string]int64{
"a/one.go": 100,
"a/two.go": 200,
"a/three.go": 300,
}), "seed BulkSetFileMtimes")
// A different repo whose rows must never be touched by repoA writes.
require(s.BulkSetFileMtimes("repoB", map[string]int64{"b/x.go": 999}), "seed repoB")
// Second index: two.go was deleted on disk, four.go is new, three.go
// changed. The authoritative snapshot is {one, three', four}.
require(s.ReplaceFileMtimes("repoA", map[string]int64{
"a/one.go": 100,
"a/three.go": 350, // changed
"a/four.go": 400, // new
}), "ReplaceFileMtimes")
want := map[string]int64{
"a/one.go": 100,
"a/three.go": 350,
"a/four.go": 400,
}
got := s.LoadFileMtimes("repoA")
if !reflect.DeepEqual(got, want) {
t.Fatalf("after ReplaceFileMtimes = %v, want %v (a/two.go must be pruned)", got, want)
}
if _, stillThere := got["a/two.go"]; stillThere {
t.Fatal("a/two.go was deleted on disk but its mtime row survived the replace — phantom deletion bug")
}
// Repo isolation.
if b := s.LoadFileMtimes("repoB"); !reflect.DeepEqual(b, map[string]int64{"b/x.go": 999}) {
t.Fatalf("repoB rows disturbed by repoA replace: %v", b)
}
// Empty input is a deliberate no-op: it must NEVER wipe a repo's set.
require(s.ReplaceFileMtimes("repoA", nil), "ReplaceFileMtimes(nil)")
if got := s.LoadFileMtimes("repoA"); !reflect.DeepEqual(got, want) {
t.Fatalf("ReplaceFileMtimes(nil) wiped the repo: %v", got)
}
}
// TestDeleteFileMtimes covers the incremental-reindex sibling: the watcher /
// incremental path drops just the deleted paths so the persisted set stays in
// step with the live graph without a full replace.
func TestDeleteFileMtimes(t *testing.T) {
s := openTestStore(t)
if err := s.BulkSetFileMtimes("repoA", map[string]int64{
"a/one.go": 100,
"a/two.go": 200,
"a/three.go": 300,
"a/four.go": 400,
}); err != nil {
t.Fatalf("seed: %v", err)
}
if err := s.BulkSetFileMtimes("repoB", map[string]int64{"b/keep.go": 7}); err != nil {
t.Fatalf("seed repoB: %v", err)
}
// Delete two existing paths and one that was never recorded (harmless).
if err := s.DeleteFileMtimes("repoA", []string{"a/two.go", "a/four.go", "a/never.go"}); err != nil {
t.Fatalf("DeleteFileMtimes: %v", err)
}
want := map[string]int64{"a/one.go": 100, "a/three.go": 300}
if got := s.LoadFileMtimes("repoA"); !reflect.DeepEqual(got, want) {
t.Fatalf("after delete = %v, want %v", got, want)
}
// Repo isolation: same-named delete on repoA must not touch repoB.
if b := s.LoadFileMtimes("repoB"); !reflect.DeepEqual(b, map[string]int64{"b/keep.go": 7}) {
t.Fatalf("repoB disturbed: %v", b)
}
// Empty input is a no-op.
if err := s.DeleteFileMtimes("repoA", nil); err != nil {
t.Fatalf("DeleteFileMtimes(nil): %v", err)
}
if got := s.LoadFileMtimes("repoA"); !reflect.DeepEqual(got, want) {
t.Fatalf("DeleteFileMtimes(nil) changed the set: %v", got)
}
}
+310
View File
@@ -0,0 +1,310 @@
package store_sqlite
import (
"database/sql"
"fmt"
"strings"
)
// This file adds the repo-scoped store hygiene the base eviction path lacks.
// EvictRepo (store.go) deletes ONLY nodes+edges, but a repo owns fifteen
// other repo_prefix-keyed sidecar tables (file_mtimes, repo_index_state,
// enrichment_state, clone_shingles, constant_values, files, ref_facts,
// vectors, churn/coverage/release/blame_enrichment, symbol_fts,
// symbol_fts_rowid, content_fts — see schema.go). Untracking a repo through
// EvictRepo leaks every one of them, so a long-lived store accumulates
// sidecar rows for repos removed from config long ago. PurgeRepo clears a
// repo whole; OrphanRepoPrefixes finds prefixes that outlived their config
// entry; RekeyRepoPrefix moves a lone repo's residue when it earns a prefix.
//
// INVARIANT — the empty repo_prefix is NEVER purged. In a live multi-repo
// store repo_prefix='' identifies SYNTHETIC GLOBAL EXTERNALS (external_call
// ::dep:* / builtin:: / module:: nodes shared across every repo) and, in a
// single-repo store, the sole repo's live data. Deleting '' rows would strip
// the shared externals out from under every repo, or wipe the lone repo.
// Every method here refuses or excludes ''.
// purgeSidecarTables are the repo_prefix-keyed sidecar tables PurgeRepo
// clears for a prefix, alongside nodes+edges. Each carries a repo_prefix
// column a plain `DELETE ... WHERE repo_prefix = ?` keys on. The two FTS5
// vtables (symbol_fts, content_fts) carry repo_prefix UNINDEXED, so their
// delete is a full scan — acceptable for a purge (a rare, whole-repo op),
// unlike the per-edit hot path. `vectors` is deliberately absent: it has NO
// repo_prefix column (keyed by node_id alone), so PurgeRepo deletes its rows
// by node-id membership instead (see deleteByIDColumnsTx below).
var purgeSidecarTables = []string{
"file_mtimes",
"repo_index_state",
"enrichment_state",
"clone_shingles",
"constant_values",
"files",
"ref_facts",
"churn_enrichment",
"coverage_enrichment",
"release_enrichment",
"blame_enrichment",
"symbol_fts",
"symbol_fts_rowid",
"content_fts",
}
// PurgeRepo deletes EVERY row a repo owns — nodes, edges, and all fifteen
// repo_prefix-keyed sidecar tables (purgeSidecarTables + vectors) — in one
// transaction. It is the complete form of EvictRepo (which drops only
// nodes+edges), wired into UntrackRepo so removing a repo from config leaves
// no residue. Refuses prefix=="" (shared global externals / solo-mode live
// data — see the file-level INVARIANT).
func (s *Store) PurgeRepo(prefix string) error {
if prefix == "" {
return fmt.Errorf("store_sqlite: PurgeRepo refuses empty repo prefix (would delete shared global externals / solo-repo data)")
}
s.writeMu.Lock()
defer s.writeMu.Unlock()
tx, err := s.db.Begin()
if err != nil {
return err
}
defer tx.Rollback() //nolint:errcheck // rollback after Commit is a no-op
// Collect this repo's node IDs first: edges and vectors are keyed off
// them (edges by from_id/to_id, vectors by node_id — neither carries a
// repo_prefix column). Edge deletion semantics mirror evictByScopeLocked
// (store.go): delete every edge touching one of these nodes, then the
// nodes themselves.
ids, err := repoNodeIDsTx(tx, prefix)
if err != nil {
return err
}
if err := deleteByIDColumnsTx(tx, "edges", []string{"from_id", "to_id"}, ids); err != nil {
return fmt.Errorf("store_sqlite: PurgeRepo edges: %w", err)
}
if err := deleteByIDColumnsTx(tx, "vectors", []string{"node_id"}, ids); err != nil {
return fmt.Errorf("store_sqlite: PurgeRepo vectors: %w", err)
}
for _, table := range purgeSidecarTables {
if _, err := tx.Exec(`DELETE FROM `+table+` WHERE repo_prefix = ?`, prefix); err != nil {
return fmt.Errorf("store_sqlite: PurgeRepo %s: %w", table, err)
}
}
if _, err := tx.Exec(`DELETE FROM nodes WHERE repo_prefix = ?`, prefix); err != nil {
return fmt.Errorf("store_sqlite: PurgeRepo nodes: %w", err)
}
return tx.Commit()
}
// orphanScanTables are the tables OrphanRepoPrefixes unions DISTINCT
// repo_prefix over. These five span the residue space: nodes (the primary
// keyed store), file_mtimes + repo_index_state (the warm-restart provenance
// that lingers when nodes are gone but sidecars survive — the exact shape a
// leaked untrack leaves), enrichment_state (per-provider provenance), and
// files (per-file metadata). A prefix whose nodes are gone but whose
// sidecars remain is invisible to a nodes-only scan, which is why the
// sidecar tables are unioned in; scanning still more tables would only
// rediscover the same prefixes at higher cost.
var orphanScanTables = []string{
"nodes",
"file_mtimes",
"repo_index_state",
"enrichment_state",
"files",
}
// OrphanRepoPrefixes returns every repo_prefix present in the store but
// absent from known — repos whose rows outlived their config entry (an
// untrack that predated PurgeRepo, or a repo dropped straight from config
// with no untrack at all). The empty prefix is NEVER reported (shared global
// externals / solo data). known is matched case-insensitively as a safety
// net, so a case-only spelling drift on a case-insensitive filesystem can
// never flag a still-tracked repo as an orphan (the #270 failure mode).
// Startup warmup feeds the result to PurgeRepo.
func (s *Store) OrphanRepoPrefixes(known []string) []string {
knownFold := make(map[string]struct{}, len(known))
for _, k := range known {
if k == "" {
continue
}
knownFold[strings.ToLower(k)] = struct{}{}
}
seen := make(map[string]struct{})
var out []string
for _, table := range orphanScanTables {
// WHERE repo_prefix <> '' both excludes the protected empty prefix
// and lets the nodes scan ride the partial nodes_by_repo index
// (defined WHERE repo_prefix <> ''). A table absent on an older
// schema simply contributes nothing.
rows, err := s.db.Query(`SELECT DISTINCT repo_prefix FROM ` + table + ` WHERE repo_prefix <> ''`)
if err != nil {
continue
}
for rows.Next() {
var p string
if err := rows.Scan(&p); err != nil {
break
}
if p == "" {
continue
}
if _, ok := knownFold[strings.ToLower(p)]; ok {
continue
}
if _, dup := seen[p]; dup {
continue
}
seen[p] = struct{}{}
out = append(out, p)
}
_ = rows.Close()
}
return out
}
// rekeyMoveTables are the sidecar tables RekeyRepoPrefix relabels from old
// to new. Every one is keyed by repo_prefix (+ file_path or provider), NOT
// by node_id, so its row content survives a node-id change: file_mtimes /
// files by (repo_prefix, file_path); repo_index_state / enrichment_state by
// repo_prefix (+ provider). At a solo->multi migration every '' row in these
// belongs to the one migrating repo — global externals live in the NODES
// table and hold NO rows here — so moving them wholesale is safe. UPDATE OR
// REPLACE folds any row the re-mint re-index already wrote under new
// (identical content: same files, same mtimes, same commit) instead of
// tripping the primary-key conflict a plain UPDATE would.
var rekeyMoveTables = []string{
"file_mtimes",
"files",
"repo_index_state",
"enrichment_state",
}
// rekeyDropTables are the sidecar tables RekeyRepoPrefix DROPS (rather than
// relabels) for old. Every one is keyed by node_id, and the solo->multi
// re-mint changes every node id (unprefixed `pkg::X` -> `<new>::pkg::X`), so
// these old-id rows are already dangling against the evicted unprefixed
// nodes. Relabeling their repo_prefix would just move dangling rows under
// new — and let, e.g., the clone reseed load a shingle set for a node that
// no longer exists. Dropping them is correct: the re-mint re-index rewrites
// the index-time sidecars (constant_values, ref_facts, clone_shingles) under
// the new node ids, and the enrichment sidecars (churn/coverage/release/
// blame) must re-run for the new ids regardless. The FTS vtables sit here
// too — their rows carry the old node ids, and UPDATE over an FTS5 UNINDEXED
// column is awkward, so delete-then-reindex is the clean path.
var rekeyDropTables = []string{
"clone_shingles",
"constant_values",
"ref_facts",
"churn_enrichment",
"coverage_enrichment",
"release_enrichment",
"blame_enrichment",
"symbol_fts",
"symbol_fts_rowid",
"content_fts",
}
// RekeyRepoPrefix moves a repo's sidecar residue from old to new the moment a
// solo (unprefixed) repo earns a real prefix because a second repo joined —
// the migrateLoneUnprefixedRepoCtx path. The prefix/path-keyed provenance
// tables (rekeyMoveTables) are relabeled so warm restart finds the repo's
// mtimes + freshness under new instead of full-re-tracking it; the
// node_id-keyed tables (rekeyDropTables) are dropped because the re-mint
// changed every node id out from under them (see the two table lists for the
// per-table rationale).
//
// Refuses new=="" (cannot rekey INTO the protected empty prefix). old=="" IS
// allowed — that is the whole point, since solo repos index unprefixed — and
// is safe here because this method touches SIDECAR tables ONLY; the synthetic
// global externals that also carry repo_prefix='' live in the NODES table,
// which RekeyRepoPrefix never writes.
func (s *Store) RekeyRepoPrefix(oldPrefix, newPrefix string) error {
if newPrefix == "" {
return fmt.Errorf("store_sqlite: RekeyRepoPrefix refuses empty destination prefix")
}
if oldPrefix == newPrefix {
return nil
}
s.writeMu.Lock()
defer s.writeMu.Unlock()
tx, err := s.db.Begin()
if err != nil {
return err
}
defer tx.Rollback() //nolint:errcheck // rollback after Commit is a no-op
for _, table := range rekeyMoveTables {
if _, err := tx.Exec(`UPDATE OR REPLACE `+table+` SET repo_prefix = ? WHERE repo_prefix = ?`, newPrefix, oldPrefix); err != nil {
return fmt.Errorf("store_sqlite: RekeyRepoPrefix move %s: %w", table, err)
}
}
for _, table := range rekeyDropTables {
if _, err := tx.Exec(`DELETE FROM `+table+` WHERE repo_prefix = ?`, oldPrefix); err != nil {
return fmt.Errorf("store_sqlite: RekeyRepoPrefix drop %s: %w", table, err)
}
}
// vectors is intentionally omitted: it has NO repo_prefix column (keyed
// by node_id alone), so it cannot be addressed here by prefix. Any ''
// embeddings are node_id-keyed against now-evicted unprefixed ids —
// dangling, and absent in the common case (embeddings are opt-in). They
// are left to a node-membership vector GC rather than guessed at here.
return tx.Commit()
}
// repoNodeIDsTx returns every node id in repoPrefix, read inside tx. The
// caller holds writeMu. Rows are fully drained + closed before the caller
// issues writes on the same tx — SQLite forbids an open read cursor while
// writing on the same connection.
func repoNodeIDsTx(tx *sql.Tx, repoPrefix string) ([]string, error) {
rows, err := tx.Query(`SELECT id FROM nodes WHERE repo_prefix = ?`, repoPrefix)
if err != nil {
return nil, err
}
var ids []string
for rows.Next() {
var id string
if err := rows.Scan(&id); err != nil {
_ = rows.Close()
return nil, err
}
ids = append(ids, id)
}
if err := rows.Err(); err != nil {
_ = rows.Close()
return nil, err
}
_ = rows.Close()
return ids, nil
}
// deleteByIDColumnsTx deletes rows from table where ANY of cols matches one
// of ids, chunked so each statement stays under SQLite's 999 bound-variable
// limit. Mirrors evictByScopeLocked's chunked from_id/to_id edge delete
// (store.go) — the semantics source for edge eviction. Empty ids is a no-op.
func deleteByIDColumnsTx(tx *sql.Tx, table string, cols, ids []string) error {
if len(ids) == 0 {
return nil
}
const chunk = 900
for _, col := range cols {
for start := 0; start < len(ids); start += chunk {
end := minInt(start+chunk, len(ids))
batch := ids[start:end]
placeholders := strings.TrimSuffix(strings.Repeat("?,", len(batch)), ",")
args := make([]any, len(batch))
for i, id := range batch {
args[i] = id
}
if _, err := tx.Exec(`DELETE FROM `+table+` WHERE `+col+` IN (`+placeholders+`)`, args...); err != nil {
return err
}
}
}
return nil
}
@@ -0,0 +1,222 @@
package store_sqlite
import (
"database/sql"
"github.com/zzet/gortex/internal/graph"
"path/filepath"
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
// openPurgeStore opens a throwaway on-disk store for the hygiene tests.
func openPurgeStore(t *testing.T) *Store {
t.Helper()
s, err := Open(filepath.Join(t.TempDir(), "purge.sqlite"))
require.NoError(t, err)
t.Cleanup(func() { _ = s.Close() })
return s
}
// seedRepoRows inserts exactly one row keyed to prefix into nodes, edges,
// vectors, and every repo_prefix-keyed sidecar table, so a PurgeRepo /
// RekeyRepoPrefix test can assert each table is cleared/moved. Node/vector
// ids embed the prefix (`<prefix>::a.go::X`) so the node-id-keyed vectors +
// edges land in the prefix's scope. Uses raw SQL for exhaustiveness: some
// tables have no public setter.
func seedRepoRows(t *testing.T, db *sql.DB, prefix string) {
t.Helper()
nodeID := prefix + "::a.go::X"
exec := func(q string, args ...any) {
t.Helper()
_, err := db.Exec(q, args...)
require.NoError(t, err, q)
}
exec(`INSERT INTO nodes (id, kind, name, file_path, repo_prefix) VALUES (?, 'function', 'X', 'a.go', ?)`, nodeID, prefix)
// An edge from the repo's node to a shared '' global external: PurgeRepo
// must delete this edge (its from_id is a repo node) but NEVER the ''
// target node.
exec(`INSERT INTO edges (from_id, to_id, kind) VALUES (?, 'external_call::dep:shared', 'calls')`, nodeID)
exec(`INSERT INTO vectors (node_id, dims, vec) VALUES (?, 1, X'00')`, nodeID)
exec(`INSERT INTO file_mtimes (repo_prefix, file_path, mtime_ns) VALUES (?, 'a.go', 123)`, prefix)
exec(`INSERT INTO repo_index_state (repo_prefix, indexed_sha) VALUES (?, 'sha')`, prefix)
exec(`INSERT INTO enrichment_state (repo_prefix, provider) VALUES (?, 'lsp')`, prefix)
exec(`INSERT INTO clone_shingles (node_id, repo_prefix, shingles) VALUES (?, ?, X'00')`, nodeID, prefix)
exec(`INSERT INTO constant_values (node_id, repo_prefix, file_path, value) VALUES (?, ?, 'a.go', 'v')`, nodeID, prefix)
exec(`INSERT INTO files (repo_prefix, file_path, content_hash) VALUES (?, 'a.go', 'h')`, prefix)
exec(`INSERT INTO ref_facts (repo_prefix, from_id, to_id, kind, line) VALUES (?, ?, 'a.go::Y', 'ref', 1)`, prefix, nodeID)
exec(`INSERT INTO churn_enrichment (node_id, repo_prefix, commit_count) VALUES (?, ?, 3)`, nodeID, prefix)
exec(`INSERT INTO coverage_enrichment (node_id, repo_prefix, coverage_pct) VALUES (?, ?, 0.5)`, nodeID, prefix)
exec(`INSERT INTO release_enrichment (node_id, repo_prefix, added_in) VALUES (?, ?, 'v1')`, nodeID, prefix)
exec(`INSERT INTO blame_enrichment (node_id, repo_prefix, email) VALUES (?, ?, 'a@b')`, nodeID, prefix)
exec(`INSERT INTO symbol_fts (node_id, repo_prefix, tokens) VALUES (?, ?, 'x')`, nodeID, prefix)
exec(`INSERT INTO symbol_fts_rowid (node_id, repo_prefix, fts_rowid) VALUES (?, ?, 1)`, nodeID, prefix)
exec(`INSERT INTO content_fts (node_id, repo_prefix, file_path, ordinal, body) VALUES (?, ?, 'a.go', 0, 'body')`, nodeID, prefix)
}
// countByPrefix reports how many rows a repo_prefix-keyed table holds for
// prefix. nodes and every sidecar carry a repo_prefix column.
func countByPrefix(t *testing.T, db *sql.DB, table, prefix string) int {
t.Helper()
var n int
require.NoError(t, db.QueryRow(`SELECT COUNT(*) FROM `+table+` WHERE repo_prefix = ?`, prefix).Scan(&n))
return n
}
// countByNodeIDLike reports how many rows a node_id-keyed table (vectors)
// holds whose node_id starts with `<prefix>::`.
func countByNodeIDLike(t *testing.T, db *sql.DB, table, prefix string) int {
t.Helper()
var n int
require.NoError(t, db.QueryRow(`SELECT COUNT(*) FROM `+table+` WHERE node_id LIKE ?`, prefix+"::%").Scan(&n))
return n
}
// prefixKeyedTables is every repo_prefix-keyed table PurgeRepo/Rekey touch,
// minus nodes (asserted separately) — used to loop assertions.
var prefixKeyedTables = []string{
"file_mtimes", "repo_index_state", "enrichment_state", "clone_shingles",
"constant_values", "files", "ref_facts", "churn_enrichment",
"coverage_enrichment", "release_enrichment", "blame_enrichment",
"symbol_fts", "symbol_fts_rowid", "content_fts",
}
func TestPurgeRepo_ClearsEveryTable_LeavesOthersAndGlobals(t *testing.T) {
s := openPurgeStore(t)
// Two real repos plus a shared '' global-external node the purge must
// never touch.
seedRepoRows(t, s.db, "repoA")
seedRepoRows(t, s.db, "repoB")
_, err := s.db.Exec(`INSERT INTO nodes (id, kind, name, file_path, repo_prefix) VALUES ('external_call::dep:shared', 'external', 'shared', '', '')`)
require.NoError(t, err)
require.NoError(t, s.PurgeRepo("repoA"))
// repoA: nodes, edges, vectors, and every sidecar cleared.
assert.Equal(t, 0, countByPrefix(t, s.db, "nodes", "repoA"), "repoA nodes gone")
assert.Equal(t, 0, countByNodeIDLike(t, s.db, "vectors", "repoA"), "repoA vectors gone")
for _, tbl := range prefixKeyedTables {
assert.Equal(t, 0, countByPrefix(t, s.db, tbl, "repoA"), "repoA %s cleared", tbl)
}
var edgesFromA int
require.NoError(t, s.db.QueryRow(`SELECT COUNT(*) FROM edges WHERE from_id LIKE 'repoA::%'`).Scan(&edgesFromA))
assert.Equal(t, 0, edgesFromA, "repoA edges gone")
// repoB untouched across the board.
assert.Equal(t, 1, countByPrefix(t, s.db, "nodes", "repoB"), "repoB nodes intact")
assert.Equal(t, 1, countByNodeIDLike(t, s.db, "vectors", "repoB"), "repoB vectors intact")
for _, tbl := range prefixKeyedTables {
assert.Equal(t, 1, countByPrefix(t, s.db, tbl, "repoB"), "repoB %s intact", tbl)
}
// The shared '' global external survives — nothing may purge ''.
assert.Equal(t, 1, countByPrefix(t, s.db, "nodes", ""), "'' global external survives")
}
func TestPurgeRepo_RefusesEmptyPrefix(t *testing.T) {
s := openPurgeStore(t)
seedRepoRows(t, s.db, "")
require.Error(t, s.PurgeRepo(""), "PurgeRepo must refuse the empty prefix (global externals / solo data)")
// The '' rows are still there.
assert.Equal(t, 1, countByPrefix(t, s.db, "file_mtimes", ""), "'' file_mtimes untouched by refused purge")
}
func TestOrphanRepoPrefixes_SidecarOnlyResidue(t *testing.T) {
s := openPurgeStore(t)
// gone: a repo whose NODES were evicted but whose sidecars linger — the
// exact leaked-untrack shape (residue in file_mtimes + repo_index_state,
// no nodes). live: a fully-present tracked repo.
_, err := s.db.Exec(`INSERT INTO file_mtimes (repo_prefix, file_path, mtime_ns) VALUES ('gone', 'x.go', 1)`)
require.NoError(t, err)
_, err = s.db.Exec(`INSERT INTO repo_index_state (repo_prefix) VALUES ('gone')`)
require.NoError(t, err)
seedRepoRows(t, s.db, "live")
// A '' row must never be reported as an orphan.
_, err = s.db.Exec(`INSERT INTO file_mtimes (repo_prefix, file_path, mtime_ns) VALUES ('', 'g.go', 1)`)
require.NoError(t, err)
orphans := s.OrphanRepoPrefixes([]string{"live"})
assert.Equal(t, []string{"gone"}, orphans, "only the nodes-less residue prefix is an orphan")
// Case-fold safety net: a case-only spelling drift of a tracked repo is
// NOT an orphan.
assert.Empty(t, s.OrphanRepoPrefixes([]string{"LIVE", "GONE"}), "case-insensitive known set covers both prefixes")
}
func TestRekeyRepoPrefix_MovesProvenanceDropsNodeIDKeyed(t *testing.T) {
s := openPurgeStore(t)
seedRepoRows(t, s.db, "") // solo repo: everything under ''
require.NoError(t, s.RekeyRepoPrefix("", "drools"))
// Prefix/path-keyed provenance MOVED '' -> drools (so warm restart finds
// the repo's mtimes under the new prefix instead of full-re-tracking).
moveTables := []string{"file_mtimes", "files", "repo_index_state", "enrichment_state"}
for _, tbl := range moveTables {
assert.Equal(t, 0, countByPrefix(t, s.db, tbl, ""), "%s '' rows moved out", tbl)
assert.Equal(t, 1, countByPrefix(t, s.db, tbl, "drools"), "%s rows now under new prefix", tbl)
}
// node_id-keyed tables DROPPED (their old ids are dangling after the
// re-mint) — the FTS decision included.
dropTables := []string{
"clone_shingles", "constant_values", "ref_facts", "churn_enrichment",
"coverage_enrichment", "release_enrichment", "blame_enrichment",
"symbol_fts", "symbol_fts_rowid", "content_fts",
}
for _, tbl := range dropTables {
assert.Equal(t, 0, countByPrefix(t, s.db, tbl, ""), "%s '' rows dropped", tbl)
assert.Equal(t, 0, countByPrefix(t, s.db, tbl, "drools"), "%s NOT relabeled to new prefix", tbl)
}
assert.Error(t, s.RekeyRepoPrefix("repoA", ""), "rekey INTO the empty prefix is refused")
}
// TestContentCrashWindow simulates the D4 kill-window at the store level:
// per-file delete+append leaves a mix of old+new content instead of an empty
// table, and the end-of-track sweep (keep = files that STREAMED content this
// run) reaps both files that vanished from disk and files that still exist
// but no longer yield content sections.
func TestContentCrashWindow(t *testing.T) {
s := openPurgeStore(t)
item := func(id, file, body string) graph.ContentFTSItem {
return graph.ContentFTSItem{NodeID: id, FilePath: file, Ordinal: 0, Body: body}
}
// Prior full index: three content files present.
require.NoError(t, s.AppendContent("r", []graph.ContentFTSItem{item("r::f1", "f1.md", "old one")}))
require.NoError(t, s.AppendContent("r", []graph.ContentFTSItem{item("r::f2", "f2.md", "old two")}))
require.NoError(t, s.AppendContent("r", []graph.ContentFTSItem{item("r::f3", "f3.md", "old three")}))
// A new full index re-streams only f1 (crash before it reached the
// rest): delete f1's rows then re-append. The other files' OLD rows must
// survive — no empty-table window.
require.NoError(t, s.WipeContentFileInRepo("r", "f1.md"))
require.NoError(t, s.AppendContent("r", []graph.ContentFTSItem{item("r::f1", "f1.md", "new one")}))
countFile := func(file string) int {
var n int
require.NoError(t, s.db.QueryRow(`SELECT COUNT(*) FROM content_fts WHERE repo_prefix='r' AND file_path=?`, file).Scan(&n))
return n
}
assert.Equal(t, 1, countFile("f1.md"), "f1 refreshed (delete+append, not doubled)")
assert.Equal(t, 1, countFile("f2.md"), "f2's old rows survive the mid-parse kill (no empty table)")
assert.Equal(t, 1, countFile("f3.md"), "f3's old rows survive the mid-parse kill (no empty table)")
// The next SUCCESSFUL completion: f2 was deleted from the repo, and f3
// STILL EXISTS on disk but was emptied — it streamed no content sections
// this run, so it is absent from the streamed set. The sweep keeps
// exactly the streamed set {f1}, reaping both the vanished file and the
// content->no-content transition (a disk-survival keep would have
// protected f3's stale rows forever).
require.NoError(t, s.DeleteContentFilesForRepoNotIn("r", map[string]struct{}{"f1.md": {}}))
assert.Equal(t, 1, countFile("f1.md"), "still-streaming file kept")
assert.Equal(t, 0, countFile("f2.md"), "vanished file swept")
assert.Equal(t, 0, countFile("f3.md"), "content->no-content transition swept despite surviving on disk")
// Empty keep is a deliberate no-op (the never-wipe-from-empty safety
// net; a zero-content walk routes to WipeContent instead).
require.NoError(t, s.DeleteContentFilesForRepoNotIn("r", nil))
assert.Equal(t, 1, countFile("f1.md"), "empty keep never wipes")
}
@@ -0,0 +1,219 @@
package store_sqlite
import (
"strings"
"github.com/zzet/gortex/internal/graph"
)
// Compile-time assertions that the SQLite Store satisfies the optional
// reference-facts persistence capability. Persisting resolved-reference facts
// in the same backend the graph lives in makes a reference's resolution an
// auditable, diffable record and a warm-restart seed.
var (
_ graph.RefFactsWriter = (*Store)(nil)
_ graph.RefFactsReader = (*Store)(nil)
)
// refFactChunk bounds rows per multi-row INSERT. 11 params/row; 80 rows = 880
// host params, under SQLite's 999 default. Mirrors shingleChunk.
const refFactChunk = 80
// candidate-list separator (unit separator — never appears in identifiers).
const refFactCandSep = "\x1f"
func encodeCandidates(c []string) string { return strings.Join(c, refFactCandSep) }
func decodeCandidates(s string) []string {
if s == "" {
return nil
}
return strings.Split(s, refFactCandSep)
}
// BulkSetRefFacts persists resolved-reference facts for one repo prefix in a
// single transaction, chunked under the host-parameter limit. Idempotent on
// (repo_prefix, from_id, to_id, kind, line). Empty input is a no-op.
func (s *Store) BulkSetRefFacts(repoPrefix string, facts []graph.RefFact) error {
if len(facts) == 0 {
return nil
}
s.writeMu.Lock()
defer s.writeMu.Unlock()
tx, err := s.db.Begin()
if err != nil {
return err
}
defer tx.Rollback() //nolint:errcheck // rollback after Commit is a no-op
for start := 0; start < len(facts); start += refFactChunk {
end := start + refFactChunk
if end > len(facts) {
end = len(facts)
}
batch := facts[start:end]
args := make([]any, 0, len(batch)*11)
stmt := make([]byte, 0, 96+len(batch)*24)
stmt = append(stmt, "INSERT OR REPLACE INTO ref_facts (repo_prefix, from_id, to_id, kind, ref_name, line, origin, tier, candidates, file_path, lang) VALUES "...)
for i, f := range batch {
if i > 0 {
stmt = append(stmt, ',')
}
stmt = append(stmt, "(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)"...)
args = append(args, repoPrefix, f.FromID, f.ToID, f.Kind, f.RefName, f.Line, f.Origin, f.Tier, encodeCandidates(f.Candidates), f.FilePath, f.Lang)
}
if _, err := tx.Exec(string(stmt), args...); err != nil {
return err
}
}
return tx.Commit()
}
// DeleteRefFactsByFiles drops all facts sourced in the supplied files for one
// repo prefix, chunked into `file_path IN (…)` DELETEs. Empty input is a no-op.
func (s *Store) DeleteRefFactsByFiles(repoPrefix string, files []string) error {
if len(files) == 0 {
return nil
}
s.writeMu.Lock()
defer s.writeMu.Unlock()
tx, err := s.db.Begin()
if err != nil {
return err
}
defer tx.Rollback() //nolint:errcheck // rollback after Commit is a no-op
for start := 0; start < len(files); start += refFactChunk {
end := start + refFactChunk
if end > len(files) {
end = len(files)
}
chunk := files[start:end]
args := make([]any, 0, len(chunk)+1)
args = append(args, repoPrefix)
stmt := make([]byte, 0, 64+len(chunk)*2)
stmt = append(stmt, "DELETE FROM ref_facts WHERE repo_prefix = ? AND file_path IN ("...)
for i, f := range chunk {
if i > 0 {
stmt = append(stmt, ',')
}
stmt = append(stmt, '?')
args = append(args, f)
}
stmt = append(stmt, ')')
if _, err := tx.Exec(string(stmt), args...); err != nil {
return err
}
}
return tx.Commit()
}
// LoadRefFactsByFiles returns the persisted facts for one repo prefix, scoped
// to the given files (all files when files is empty). Always non-nil.
func (s *Store) LoadRefFactsByFiles(repoPrefix string, files []string) ([]graph.RefFact, error) {
out := []graph.RefFact{}
scan := func(query string, args ...any) error {
rows, err := s.db.Query(query, args...)
if err != nil {
return err
}
defer rows.Close()
for rows.Next() {
var f graph.RefFact
var cand string
if err := rows.Scan(&f.FromID, &f.ToID, &f.Kind, &f.RefName, &f.Line, &f.Origin, &f.Tier, &cand, &f.FilePath, &f.Lang); err != nil {
return err
}
f.RepoPrefix = repoPrefix
f.Candidates = decodeCandidates(cand)
out = append(out, f)
}
return rows.Err()
}
const cols = `from_id, to_id, kind, ref_name, line, origin, tier, candidates, file_path, lang`
if len(files) == 0 {
if err := scan(`SELECT `+cols+` FROM ref_facts WHERE repo_prefix = ?`, repoPrefix); err != nil {
return nil, err
}
return out, nil
}
for start := 0; start < len(files); start += refFactChunk {
end := start + refFactChunk
if end > len(files) {
end = len(files)
}
chunk := files[start:end]
args := make([]any, 0, len(chunk)+1)
args = append(args, repoPrefix)
stmt := make([]byte, 0, 96+len(chunk)*2)
stmt = append(stmt, "SELECT "+cols+" FROM ref_facts WHERE repo_prefix = ? AND file_path IN ("...)
for i, f := range chunk {
if i > 0 {
stmt = append(stmt, ',')
}
stmt = append(stmt, '?')
args = append(args, f)
}
stmt = append(stmt, ')')
if err := scan(string(stmt), args...); err != nil {
return nil, err
}
}
return out, nil
}
// LoadRefFactsByTargets returns the persisted facts that resolve TO any of
// the given node IDs for one repo prefix, grouped by source file path — the
// reverse lookup incremental re-resolution uses to find the files that
// referenced a changed symbol after its live in-edges were evicted. Served by
// the ref_facts_by_target index, chunked under the host-parameter limit.
// Always non-nil; empty input is a no-op.
func (s *Store) LoadRefFactsByTargets(repoPrefix string, targetIDs []string) (map[string][]graph.RefFact, error) {
out := map[string][]graph.RefFact{}
if len(targetIDs) == 0 {
return out, nil
}
const cols = `from_id, to_id, kind, ref_name, line, origin, tier, candidates, file_path, lang`
for start := 0; start < len(targetIDs); start += refFactChunk {
end := start + refFactChunk
if end > len(targetIDs) {
end = len(targetIDs)
}
chunk := targetIDs[start:end]
args := make([]any, 0, len(chunk)+1)
args = append(args, repoPrefix)
stmt := make([]byte, 0, 96+len(chunk)*2)
stmt = append(stmt, "SELECT "+cols+" FROM ref_facts WHERE repo_prefix = ? AND to_id IN ("...)
for i, id := range chunk {
if i > 0 {
stmt = append(stmt, ',')
}
stmt = append(stmt, '?')
args = append(args, id)
}
stmt = append(stmt, ')')
rows, err := s.db.Query(string(stmt), args...)
if err != nil {
return nil, err
}
for rows.Next() {
var f graph.RefFact
var cand string
if err := rows.Scan(&f.FromID, &f.ToID, &f.Kind, &f.RefName, &f.Line, &f.Origin, &f.Tier, &cand, &f.FilePath, &f.Lang); err != nil {
_ = rows.Close()
return nil, err
}
f.RepoPrefix = repoPrefix
f.Candidates = decodeCandidates(cand)
out[f.FilePath] = append(out[f.FilePath], f)
}
err = rows.Err()
_ = rows.Close()
if err != nil {
return nil, err
}
}
return out, nil
}
@@ -0,0 +1,172 @@
package store_sqlite_test
import (
"fmt"
"path/filepath"
"testing"
"github.com/stretchr/testify/require"
"github.com/zzet/gortex/internal/graph"
"github.com/zzet/gortex/internal/graph/store_sqlite"
)
func openRefFactStore(t *testing.T) *store_sqlite.Store {
t.Helper()
s, err := store_sqlite.Open(filepath.Join(t.TempDir(), "rf.sqlite"))
require.NoError(t, err)
t.Cleanup(func() { _ = s.Close() })
return s
}
func TestRefFacts_Roundtrip(t *testing.T) {
s := openRefFactStore(t)
facts := []graph.RefFact{
{RepoPrefix: "", FromID: "a.go::A", ToID: "b.go::B", Kind: "calls", RefName: "B", Line: 7, Origin: "ast_resolved", Tier: "ast", FilePath: "a.go", Lang: "go", Candidates: []string{"b.go::B", "c.go::B"}},
{RepoPrefix: "", FromID: "a.go::A", ToID: "d.go::D", Kind: "references", RefName: "D", Line: 9, Origin: "lsp_resolved", Tier: "lsp", FilePath: "a.go", Lang: "go"},
}
require.NoError(t, s.BulkSetRefFacts("", facts))
got, err := s.LoadRefFactsByFiles("", []string{"a.go"})
require.NoError(t, err)
require.Len(t, got, 2)
byTo := map[string]graph.RefFact{}
for _, f := range got {
byTo[f.ToID] = f
}
require.Equal(t, "ast_resolved", byTo["b.go::B"].Origin)
require.Equal(t, []string{"b.go::B", "c.go::B"}, byTo["b.go::B"].Candidates)
require.Equal(t, 7, byTo["b.go::B"].Line)
require.Equal(t, "lsp_resolved", byTo["d.go::D"].Origin)
// LoadRefFactsByFiles with empty file list returns all for the repo.
all, err := s.LoadRefFactsByFiles("", nil)
require.NoError(t, err)
require.Len(t, all, 2)
}
func TestRefFacts_DeleteByFile(t *testing.T) {
s := openRefFactStore(t)
require.NoError(t, s.BulkSetRefFacts("", []graph.RefFact{
{FromID: "a.go::A", ToID: "x", Kind: "calls", FilePath: "a.go"},
{FromID: "b.go::B", ToID: "y", Kind: "calls", FilePath: "b.go"},
}))
require.NoError(t, s.DeleteRefFactsByFiles("", []string{"a.go"}))
got, err := s.LoadRefFactsByFiles("", nil)
require.NoError(t, err)
require.Len(t, got, 1)
require.Equal(t, "b.go", got[0].FilePath)
}
func TestRefFacts_RepoScoping(t *testing.T) {
s := openRefFactStore(t)
require.NoError(t, s.BulkSetRefFacts("repoA", []graph.RefFact{{FromID: "f::A", ToID: "tA", Kind: "calls", FilePath: "f.go"}}))
require.NoError(t, s.BulkSetRefFacts("repoB", []graph.RefFact{{FromID: "f::A", ToID: "tB", Kind: "calls", FilePath: "f.go"}}))
a, err := s.LoadRefFactsByFiles("repoA", []string{"f.go"})
require.NoError(t, err)
require.Len(t, a, 1)
require.Equal(t, "tA", a[0].ToID)
// Deleting repoA's file must not touch repoB.
require.NoError(t, s.DeleteRefFactsByFiles("repoA", []string{"f.go"}))
b, err := s.LoadRefFactsByFiles("repoB", []string{"f.go"})
require.NoError(t, err)
require.Len(t, b, 1)
require.Equal(t, "tB", b[0].ToID)
}
func TestRefFacts_Chunking(t *testing.T) {
s := openRefFactStore(t)
const n = 500 // > refFactChunk (80)
facts := make([]graph.RefFact, n)
for i := range facts {
facts[i] = graph.RefFact{FromID: fmt.Sprintf("a.go::f%d", i), ToID: fmt.Sprintf("t%d", i), Kind: "calls", FilePath: "a.go"}
}
require.NoError(t, s.BulkSetRefFacts("", facts))
got, err := s.LoadRefFactsByFiles("", []string{"a.go"})
require.NoError(t, err)
require.Len(t, got, n)
}
func TestRefFacts_EmptyNoop(t *testing.T) {
s := openRefFactStore(t)
require.NoError(t, s.BulkSetRefFacts("", nil))
require.NoError(t, s.DeleteRefFactsByFiles("", nil))
got, err := s.LoadRefFactsByFiles("", nil)
require.NoError(t, err)
require.Empty(t, got)
}
func TestRefFacts_LoadByTargets(t *testing.T) {
s := openRefFactStore(t)
require.NoError(t, s.BulkSetRefFacts("", []graph.RefFact{
{FromID: "b.go::Caller", ToID: "a.go::F", Kind: "calls", RefName: "F", Line: 3, Origin: "ast_resolved", Tier: "ast", FilePath: "b.go", Lang: "go"},
{FromID: "c.go::Other", ToID: "a.go::F", Kind: "references", RefName: "F", FilePath: "c.go"},
{FromID: "c.go::Other", ToID: "a.go::G", Kind: "calls", RefName: "G", FilePath: "c.go"},
{FromID: "d.go::X", ToID: "z.go::Z", Kind: "calls", RefName: "Z", FilePath: "d.go"},
}))
byFile, err := s.LoadRefFactsByTargets("", []string{"a.go::F", "a.go::G"})
require.NoError(t, err)
require.Len(t, byFile, 2, "facts must be grouped by source file: %v", byFile)
require.Len(t, byFile["b.go"], 1)
require.Equal(t, "a.go::F", byFile["b.go"][0].ToID)
require.Equal(t, "F", byFile["b.go"][0].RefName)
require.Equal(t, "ast_resolved", byFile["b.go"][0].Origin)
require.Len(t, byFile["c.go"], 2, "both of c.go's facts target the queried symbols")
require.NotContains(t, byFile, "d.go", "a fact targeting an unqueried symbol must not match")
}
func TestRefFacts_LoadByTargets_EmptyAndMissing(t *testing.T) {
s := openRefFactStore(t)
require.NoError(t, s.BulkSetRefFacts("", []graph.RefFact{
{FromID: "a.go::A", ToID: "b.go::B", Kind: "calls", FilePath: "a.go"},
}))
// Empty input: empty, non-nil map.
empty, err := s.LoadRefFactsByTargets("", nil)
require.NoError(t, err)
require.NotNil(t, empty)
require.Empty(t, empty)
// A target nothing references: no rows, no error.
miss, err := s.LoadRefFactsByTargets("", []string{"nope::Missing"})
require.NoError(t, err)
require.Empty(t, miss)
}
func TestRefFacts_LoadByTargets_RepoScoping(t *testing.T) {
s := openRefFactStore(t)
require.NoError(t, s.BulkSetRefFacts("repoA", []graph.RefFact{{FromID: "fa::A", ToID: "shared::T", Kind: "calls", FilePath: "fa.go"}}))
require.NoError(t, s.BulkSetRefFacts("repoB", []graph.RefFact{{FromID: "fb::B", ToID: "shared::T", Kind: "calls", FilePath: "fb.go"}}))
a, err := s.LoadRefFactsByTargets("repoA", []string{"shared::T"})
require.NoError(t, err)
require.Len(t, a, 1)
require.Len(t, a["fa.go"], 1)
require.Equal(t, "repoA", a["fa.go"][0].RepoPrefix, "loaded facts must carry the queried repo prefix")
require.NotContains(t, a, "fb.go", "another repo's facts must not leak into the result")
}
func TestRefFacts_LoadByTargets_Chunking(t *testing.T) {
s := openRefFactStore(t)
const n = 500 // > refFactChunk (80)
facts := make([]graph.RefFact, n)
targets := make([]string, n)
for i := range facts {
facts[i] = graph.RefFact{
FromID: fmt.Sprintf("src%d.go::f", i),
ToID: fmt.Sprintf("dst.go::t%d", i),
Kind: "calls",
FilePath: fmt.Sprintf("src%d.go", i),
}
targets[i] = fmt.Sprintf("dst.go::t%d", i)
}
require.NoError(t, s.BulkSetRefFacts("", facts))
byFile, err := s.LoadRefFactsByTargets("", targets)
require.NoError(t, err)
require.Len(t, byFile, n, "every chunked target must come back grouped under its source file")
}
@@ -0,0 +1,140 @@
package store_sqlite
import (
"database/sql"
"github.com/zzet/gortex/internal/graph"
)
var (
_ graph.ReleaseEnrichmentWriter = (*Store)(nil)
_ graph.ReleaseEnrichmentReader = (*Store)(nil)
)
// releaseChunk bounds rows per multi-row INSERT (3 cols → 3 params/row).
const releaseChunk = 300
const releaseCols = `node_id, repo_prefix, added_in`
// BulkSetReleases persists release rows for one repo prefix, chunked.
func (s *Store) BulkSetReleases(repoPrefix string, rows []graph.ReleaseEnrichment) error {
if len(rows) == 0 {
return nil
}
s.writeMu.Lock()
defer s.writeMu.Unlock()
tx, err := s.db.Begin()
if err != nil {
return err
}
defer tx.Rollback() //nolint:errcheck
for start := 0; start < len(rows); start += releaseChunk {
end := start + releaseChunk
if end > len(rows) {
end = len(rows)
}
batch := rows[start:end]
args := make([]any, 0, len(batch)*3)
stmt := make([]byte, 0, 96+len(batch)*12)
stmt = append(stmt, "INSERT OR REPLACE INTO release_enrichment ("...)
stmt = append(stmt, releaseCols...)
stmt = append(stmt, ") VALUES "...)
for i, e := range batch {
if i > 0 {
stmt = append(stmt, ',')
}
stmt = append(stmt, "(?,?,?)"...)
args = append(args, e.NodeID, repoPrefix, e.AddedIn)
}
if _, err := tx.Exec(string(stmt), args...); err != nil {
return err
}
}
return tx.Commit()
}
// DeleteReleases drops release rows for the supplied node ids, chunked.
func (s *Store) DeleteReleases(nodeIDs []string) error {
if len(nodeIDs) == 0 {
return nil
}
seen := make(map[string]struct{}, len(nodeIDs))
uniq := make([]string, 0, len(nodeIDs))
for _, id := range nodeIDs {
if id == "" {
continue
}
if _, ok := seen[id]; ok {
continue
}
seen[id] = struct{}{}
uniq = append(uniq, id)
}
if len(uniq) == 0 {
return nil
}
s.writeMu.Lock()
defer s.writeMu.Unlock()
tx, err := s.db.Begin()
if err != nil {
return err
}
defer tx.Rollback() //nolint:errcheck
for start := 0; start < len(uniq); start += releaseChunk {
end := start + releaseChunk
if end > len(uniq) {
end = len(uniq)
}
chunk := uniq[start:end]
args := make([]any, len(chunk))
stmt := make([]byte, 0, 56+len(chunk)*2)
stmt = append(stmt, "DELETE FROM release_enrichment WHERE node_id IN ("...)
for i, id := range chunk {
if i > 0 {
stmt = append(stmt, ',')
}
stmt = append(stmt, '?')
args[i] = id
}
stmt = append(stmt, ')')
if _, err := tx.Exec(string(stmt), args...); err != nil {
return err
}
}
return tx.Commit()
}
// ReleaseRows returns release rows for repoPrefix; empty → all repos.
func (s *Store) ReleaseRows(repoPrefix string) []graph.ReleaseEnrichment {
var (
rows *sql.Rows
err error
)
if repoPrefix == "" {
rows, err = s.db.Query(`SELECT ` + releaseCols + ` FROM release_enrichment`)
} else {
rows, err = s.db.Query(`SELECT `+releaseCols+` FROM release_enrichment WHERE repo_prefix = ?`, repoPrefix)
}
if err != nil {
return nil
}
defer rows.Close()
var out []graph.ReleaseEnrichment
for rows.Next() {
var e graph.ReleaseEnrichment
if err := rows.Scan(&e.NodeID, &e.RepoPrefix, &e.AddedIn); err != nil {
return out
}
out = append(out, e)
}
if err := rows.Err(); err != nil {
return out
}
return out
}
+22
View File
@@ -0,0 +1,22 @@
package store_sqlite_test
import (
"path/filepath"
"testing"
"github.com/zzet/gortex/internal/graph"
"github.com/zzet/gortex/internal/graph/store_sqlite"
"github.com/zzet/gortex/internal/graph/storetest"
)
func TestSQLiteStoreConformance(t *testing.T) {
storetest.RunConformance(t, func(t *testing.T) graph.Store {
dir := t.TempDir()
s, err := store_sqlite.Open(filepath.Join(dir, "test.sqlite"))
if err != nil {
t.Fatalf("Open: %v", err)
}
t.Cleanup(func() { _ = s.Close() })
return s
})
}
@@ -0,0 +1,362 @@
package store_sqlite
import (
"github.com/zzet/gortex/internal/graph"
)
// The graph-traversal and subgraph-reader optional capabilities for the
// SQLite backend. Each method mirrors the in-memory *graph.Graph
// reference implementation exactly so both satisfy the same conformance
// suite (internal/graph/storetest). The walks use the same per-node /
// batched edge readers the in-memory store uses (GetOutEdges /
// GetInEdges / GetFileNodes / GetNodesByIDs / GetIn|OutEdgesByNodeIDs),
// which on SQLite hit the (from_id,kind) / (to_id,kind) / file_path
// indexes — no new prepared statements needed.
var (
_ graph.ReachableForwardByKinds = (*Store)(nil)
_ graph.ClassHierarchyTraverser = (*Store)(nil)
_ graph.FrontierExpander = (*Store)(nil)
_ graph.FileEditingContext = (*Store)(nil)
_ graph.FileSubGraphReader = (*Store)(nil)
_ graph.FileSubGraphCountReader = (*Store)(nil)
)
// ReachableForwardByKinds computes the set of node IDs reachable from
// the seed frontier via outgoing edges whose Kind is in kinds, via a
// layer-by-layer forward BFS. Empty seeds returns nil; empty kinds
// returns the seed set unchanged. The returned map keys are the
// reachable IDs (seeds included); every value is true.
func (s *Store) ReachableForwardByKinds(seeds []string, kinds []graph.EdgeKind) map[string]bool {
if len(seeds) == 0 {
return nil
}
covered := make(map[string]bool, len(seeds))
frontier := make([]string, 0, len(seeds))
for _, id := range seeds {
if id == "" || covered[id] {
continue
}
covered[id] = true
frontier = append(frontier, id)
}
if len(kinds) == 0 {
return covered
}
allowed := make(map[graph.EdgeKind]struct{}, len(kinds))
for _, k := range kinds {
allowed[k] = struct{}{}
}
for len(frontier) > 0 {
next := frontier[:0:0]
for _, id := range frontier {
for _, e := range s.GetOutEdges(id) {
if e == nil {
continue
}
if _, ok := allowed[e.Kind]; !ok {
continue
}
if !covered[e.To] {
covered[e.To] = true
next = append(next, e.To)
}
}
}
frontier = next
}
return covered
}
// ClassHierarchyTraverse walks the inheritance subgraph rooted at
// seedID, following only edges whose Kind is in kinds, up to depth hops.
// direction "up" follows outgoing edges; "down" follows incoming. Empty
// kinds, depth <= 0, an unknown direction, or an unknown seed return
// nil. Each returned row carries the full Path (node IDs from the seed,
// exclusive) and per-hop EdgeKinds for one terminal node.
func (s *Store) ClassHierarchyTraverse(
seedID string,
direction string,
kinds []graph.EdgeKind,
depth int,
) []graph.ClassHierarchyRow {
if seedID == "" || depth <= 0 || len(kinds) == 0 {
return nil
}
kset := make(map[graph.EdgeKind]struct{}, len(kinds))
for _, k := range kinds {
if k == "" {
continue
}
kset[k] = struct{}{}
}
if len(kset) == 0 {
return nil
}
if s.GetNode(seedID) == nil {
return nil
}
walkUp := direction == "up"
walkDown := direction == "down"
if !walkUp && !walkDown {
return nil
}
type travQueued struct {
id string
path []string
edgeKinds []graph.EdgeKind
hops int
}
visited := map[string]struct{}{seedID: {}}
queue := []travQueued{{id: seedID, path: nil, edgeKinds: nil, hops: 0}}
var out []graph.ClassHierarchyRow
for len(queue) > 0 {
cur := queue[0]
queue = queue[1:]
if cur.hops >= depth {
continue
}
var edges []*graph.Edge
if walkUp {
edges = s.GetOutEdges(cur.id)
} else {
edges = s.GetInEdges(cur.id)
}
for _, e := range edges {
if e == nil {
continue
}
if _, ok := kset[e.Kind]; !ok {
continue
}
var nb string
if walkUp {
nb = e.To
} else {
nb = e.From
}
if nb == "" {
continue
}
if _, ok := visited[nb]; ok {
continue
}
visited[nb] = struct{}{}
newPath := append([]string(nil), cur.path...)
newPath = append(newPath, nb)
newKinds := append([]graph.EdgeKind(nil), cur.edgeKinds...)
newKinds = append(newKinds, e.Kind)
out = append(out, graph.ClassHierarchyRow{
Path: newPath,
EdgeKinds: newKinds,
})
queue = append(queue, travQueued{id: nb, path: newPath, edgeKinds: newKinds, hops: cur.hops + 1})
}
}
return out
}
// ExpandFrontier returns, for the given source IDs, their adjacent edges
// of the requested kinds plus the neighbour node at each edge's far end.
// forward=true follows outgoing edges (neighbour = edge target);
// forward=false follows incoming (neighbour = edge source). Empty ids or
// empty kinds return nil; limit > 0 caps the total number of hops.
func (s *Store) ExpandFrontier(ids []string, forward bool, kinds []graph.EdgeKind, limit int) []graph.FrontierHop {
if len(ids) == 0 || len(kinds) == 0 {
return nil
}
kset := make(map[graph.EdgeKind]struct{}, len(kinds))
for _, k := range kinds {
kset[k] = struct{}{}
}
var out []graph.FrontierHop
for _, id := range ids {
var edges []*graph.Edge
if forward {
edges = s.GetOutEdges(id)
} else {
edges = s.GetInEdges(id)
}
for _, e := range edges {
if e == nil {
continue
}
if _, ok := kset[e.Kind]; !ok {
continue
}
var nbID string
if forward {
nbID = e.To
} else {
nbID = e.From
}
nb := s.GetNode(nbID)
if nb == nil {
continue
}
out = append(out, graph.FrontierHop{Edge: e, Neighbor: nb})
if limit > 0 && len(out) >= limit {
return out
}
}
}
return out
}
// FileEditingContext returns the get_editing_context payload for
// filePath: the file node, the symbols defined in it, the file node's
// import out-edges, and the 1-hop callers / callees (via EdgeCalls) of
// the defined call-target symbols, filtered to symbols outside the file.
// kinds is the set of node kinds treated as call targets (function +
// method). Empty path or a file with no nodes returns nil.
func (s *Store) FileEditingContext(filePath string, kinds []graph.NodeKind) *graph.FileEditingContextResult {
if filePath == "" {
return nil
}
nodes := s.GetFileNodes(filePath)
if len(nodes) == 0 {
return nil
}
kset := make(map[graph.NodeKind]struct{}, len(kinds))
for _, k := range kinds {
if k == "" {
continue
}
kset[k] = struct{}{}
}
res := &graph.FileEditingContextResult{}
var fileNodeID string
var defNodeIDs []string
for _, n := range nodes {
if n == nil {
continue
}
if n.Kind == graph.KindFile {
res.FileNode = n
fileNodeID = n.ID
continue
}
res.Defines = append(res.Defines, n)
if _, ok := kset[n.Kind]; ok {
defNodeIDs = append(defNodeIDs, n.ID)
}
}
if fileNodeID != "" {
for _, e := range s.GetOutEdges(fileNodeID) {
if e == nil {
continue
}
if e.Kind == graph.EdgeImports {
res.Imports = append(res.Imports, e)
}
}
}
if len(defNodeIDs) == 0 {
return res
}
inEdges := s.GetInEdgesByNodeIDs(defNodeIDs)
outEdges := s.GetOutEdgesByNodeIDs(defNodeIDs)
callerIDSet := make(map[string]struct{})
calleeIDSet := make(map[string]struct{})
for _, id := range defNodeIDs {
for _, e := range inEdges[id] {
if e == nil || e.Kind != graph.EdgeCalls {
continue
}
if e.From == "" {
continue
}
callerIDSet[e.From] = struct{}{}
}
for _, e := range outEdges[id] {
if e == nil || e.Kind != graph.EdgeCalls {
continue
}
if e.To == "" {
continue
}
calleeIDSet[e.To] = struct{}{}
}
}
callerIDs := make([]string, 0, len(callerIDSet))
for id := range callerIDSet {
callerIDs = append(callerIDs, id)
}
calleeIDs := make([]string, 0, len(calleeIDSet))
for id := range calleeIDSet {
calleeIDs = append(calleeIDs, id)
}
callerNodes := s.GetNodesByIDs(callerIDs)
calleeNodes := s.GetNodesByIDs(calleeIDs)
for _, id := range callerIDs {
n := callerNodes[id]
if n == nil || n.FilePath == filePath {
continue
}
res.CalledBy = append(res.CalledBy, n)
}
for _, id := range calleeIDs {
n := calleeNodes[id]
if n == nil || n.FilePath == filePath {
continue
}
res.Calls = append(res.Calls, n)
}
return res
}
// GetFileSubGraph returns every node anchored to filePath plus every
// edge adjacent to one of those nodes, deduplicated by (from, to, kind).
// A missing / empty file returns (nil, nil).
func (s *Store) GetFileSubGraph(filePath string) ([]*graph.Node, []*graph.Edge) {
if filePath == "" {
return nil, nil
}
nodes := s.GetFileNodes(filePath)
if len(nodes) == 0 {
return nil, nil
}
ids := make([]string, 0, len(nodes))
for _, n := range nodes {
if n != nil && n.ID != "" {
ids = append(ids, n.ID)
}
}
outByID := s.GetOutEdgesByNodeIDs(ids)
inByID := s.GetInEdgesByNodeIDs(ids)
type travEdgeKey struct {
from string
to string
kind graph.EdgeKind
}
seen := make(map[travEdgeKey]struct{}, 2*len(ids))
edges := make([]*graph.Edge, 0, 2*len(ids))
add := func(e *graph.Edge) {
if e == nil {
return
}
k := travEdgeKey{from: e.From, to: e.To, kind: e.Kind}
if _, ok := seen[k]; ok {
return
}
seen[k] = struct{}{}
edges = append(edges, e)
}
for _, id := range ids {
for _, e := range outByID[id] {
add(e)
}
for _, e := range inByID[id] {
add(e)
}
}
return nodes, edges
}
// GetFileSubGraphCounts is the count-only sibling of GetFileSubGraph:
// it returns the file's nodes plus the number of distinct adjacent
// edges, without materialising the edge slice for the caller.
func (s *Store) GetFileSubGraphCounts(filePath string) ([]*graph.Node, int) {
nodes, edges := s.GetFileSubGraph(filePath)
return nodes, len(edges)
}
+295
View File
@@ -0,0 +1,295 @@
package store_sqlite
import (
"container/heap"
"encoding/binary"
"errors"
"math"
"github.com/zzet/gortex/internal/graph"
)
// Compile-time assertion that the SQLite Store satisfies the optional
// engine-native vector-search capability.
var _ graph.VectorSearcher = (*Store)(nil)
// errInvalidDims is returned by BuildVectorIndex for a negative width.
var errInvalidDims = errors.New("store_sqlite: invalid vector dims")
// Vector design (pure-Go, zero CGo)
//
// modernc.org/sqlite is a pure-Go SQLite that cannot load C extensions,
// so sqlite-vec / sqlite-vector are off the table — and staying CGo-free
// is the whole point of this backend. Embeddings are persisted as a
// little-endian float32 BLOB in the `vectors` table; the win over the
// daemon's in-process HNSW fallback is durability: vectors survive a
// restart instead of being recomputed.
//
// Queries use an exact brute-force cosine top-k: SimilarTo streams every
// stored vector, scores it against the query, and keeps the best `limit`
// in a bounded max-heap. This is O(N) per query but fully correct,
// deterministic, and holds no extra Store state (the Store struct lives
// in store.go and cannot be edited here). An on-Store HNSW cache is a
// future optimisation; for the corpus sizes this backend targets the
// exact path is the simplest thing that is verifiably right.
//
// BuildVectorIndex only validates/records intent — there is no separate
// index structure to build, since SimilarTo computes over the table
// directly.
// vectorChunk bounds rows per multi-row INSERT in BulkUpsertEmbeddings.
// 3 host params per row, SQLite's default limit is 999 → 333 max; 300
// leaves headroom.
const vectorChunk = 300
// encodeVec serialises a float32 slice to a little-endian BLOB
// (4 bytes per element).
func encodeVec(vec []float32) []byte {
b := make([]byte, len(vec)*4)
for i, f := range vec {
binary.LittleEndian.PutUint32(b[i*4:], math.Float32bits(f))
}
return b
}
// decodeVec is the inverse of encodeVec. A BLOB whose length is not a
// multiple of 4 yields nil (corrupt row); callers skip nil vectors.
func decodeVec(b []byte) []float32 {
if len(b)%4 != 0 {
return nil
}
out := make([]float32, len(b)/4)
for i := range out {
out[i] = math.Float32frombits(binary.LittleEndian.Uint32(b[i*4:]))
}
return out
}
// UpsertEmbedding persists one node's embedding, replacing any prior
// vector for that node ID.
func (s *Store) UpsertEmbedding(nodeID string, vec []float32) error {
s.writeMu.Lock()
defer s.writeMu.Unlock()
_, err := s.db.Exec(
`INSERT OR REPLACE INTO vectors (node_id, dims, vec) VALUES (?, ?, ?)`,
nodeID, len(vec), encodeVec(vec),
)
return err
}
// BulkUpsertEmbeddings persists many embeddings in a single transaction,
// chunked under SQLite's host-parameter limit. Idempotent on NodeID.
// Empty input is a no-op.
func (s *Store) BulkUpsertEmbeddings(items []graph.VectorItem) error {
if len(items) == 0 {
return nil
}
s.writeMu.Lock()
defer s.writeMu.Unlock()
tx, err := s.db.Begin()
if err != nil {
return err
}
defer tx.Rollback() //nolint:errcheck // rollback after Commit is a no-op
for start := 0; start < len(items); start += vectorChunk {
end := start + vectorChunk
if end > len(items) {
end = len(items)
}
batch := items[start:end]
args := make([]any, 0, len(batch)*3)
stmt := make([]byte, 0, 64+len(batch)*16)
stmt = append(stmt, "INSERT OR REPLACE INTO vectors (node_id, dims, vec) VALUES "...)
for i, it := range batch {
if i > 0 {
stmt = append(stmt, ',')
}
stmt = append(stmt, "(?, ?, ?)"...)
args = append(args, it.NodeID, len(it.Vec), encodeVec(it.Vec))
}
if _, err := tx.Exec(string(stmt), args...); err != nil {
return err
}
}
return tx.Commit()
}
// BuildVectorIndex finalises the vector index. Because SimilarTo scores
// over the `vectors` table directly there is no separate structure to
// populate; this validates the declared width is positive and is
// otherwise a no-op (idempotent, safe to call repeatedly).
func (s *Store) BuildVectorIndex(dims int) error {
if dims < 0 {
return errInvalidDims
}
return nil
}
// GetEmbeddings reads back the stored vectors for an explicit set of
// node IDs in one batched scan. It does not rank — it is the read side
// of the post-rerank cosine-refinement stage, which needs the raw
// vectors (not a distance) to score against a freshly embedded query.
//
// IDs with no stored vector (or a corrupt BLOB) are simply absent from
// the returned map; empty input yields an empty map. A query error
// yields whatever was decoded so far rather than failing the caller —
// the refinement stage treats a thin result as "score what we have",
// never as a hard error, so a transient read can never regress search.
// The IN-list is chunked under SQLite's host-parameter limit.
func (s *Store) GetEmbeddings(ids []string) map[string][]float32 {
out := make(map[string][]float32, len(ids))
if len(ids) == 0 {
return out
}
for start := 0; start < len(ids); start += vectorChunk {
end := start + vectorChunk
if end > len(ids) {
end = len(ids)
}
batch := ids[start:end]
stmt := make([]byte, 0, 48+len(batch)*2)
stmt = append(stmt, "SELECT node_id, vec FROM vectors WHERE node_id IN ("...)
args := make([]any, 0, len(batch))
for i, id := range batch {
if i > 0 {
stmt = append(stmt, ',')
}
stmt = append(stmt, '?')
args = append(args, id)
}
stmt = append(stmt, ')')
rows, err := s.db.Query(string(stmt), args...)
if err != nil {
// Return what we have; the refinement stage is best-effort.
return out
}
for rows.Next() {
var (
id string
blob []byte
)
if err := rows.Scan(&id, &blob); err != nil {
_ = rows.Close()
return out
}
if vec := decodeVec(blob); vec != nil {
out[id] = vec
}
}
_ = rows.Close()
}
return out
}
// SimilarTo returns up to `limit` stored vectors closest to the query
// under cosine distance, ordered by ascending distance (most similar
// first). Vectors whose length differs from the query are skipped — a
// dimension mismatch can't be meaningfully scored.
func (s *Store) SimilarTo(vec []float32, limit int) ([]graph.VectorHit, error) {
if limit <= 0 || len(vec) == 0 {
return nil, nil
}
qNorm := norm(vec)
if qNorm == 0 {
return nil, nil
}
rows, err := s.db.Query(`SELECT node_id, vec FROM vectors`)
if err != nil {
return nil, err
}
defer rows.Close()
// Max-heap keyed on distance: the root is the *worst* kept hit, so a
// candidate better than the root evicts it. This keeps the heap at
// `limit` and yields an exact top-k.
h := &hitHeap{}
for rows.Next() {
var id string
var blob []byte
if err := rows.Scan(&id, &blob); err != nil {
return nil, err
}
cand := decodeVec(blob)
if len(cand) != len(vec) {
continue
}
cNorm := norm(cand)
if cNorm == 0 {
continue
}
dist := cosineDistance(vec, cand, qNorm, cNorm)
if h.Len() < limit {
heap.Push(h, graph.VectorHit{NodeID: id, Distance: dist})
} else if dist < (*h)[0].Distance {
(*h)[0] = graph.VectorHit{NodeID: id, Distance: dist}
heap.Fix(h, 0)
}
}
if err := rows.Err(); err != nil {
return nil, err
}
// Drain the max-heap (largest distance first) then reverse so the
// result is ascending by distance (most similar first).
out := make([]graph.VectorHit, h.Len())
for i := len(out) - 1; i >= 0; i-- {
out[i] = heap.Pop(h).(graph.VectorHit)
}
return out, nil
}
// norm returns the Euclidean norm (L2) of v as a float64.
func norm(v []float32) float64 {
var sum float64
for _, f := range v {
d := float64(f)
sum += d * d
}
return math.Sqrt(sum)
}
// cosineDistance returns 1 - cosine_similarity(a, b), given precomputed
// norms. Lower = more similar; identical direction → ~0, orthogonal → 1,
// opposite → 2. a and b are assumed equal length and non-zero norm.
func cosineDistance(a, b []float32, aNorm, bNorm float64) float64 {
var dot float64
for i := range a {
dot += float64(a[i]) * float64(b[i])
}
sim := dot / (aNorm * bNorm)
// Guard against tiny floating-point overshoot past ±1.
if sim > 1 {
sim = 1
} else if sim < -1 {
sim = -1
}
return 1 - sim
}
// hitHeap is a max-heap of VectorHit ordered by Distance: Less reports
// the *larger* distance as "less" so the root is the worst-kept hit.
type hitHeap []graph.VectorHit
func (h hitHeap) Len() int { return len(h) }
func (h hitHeap) Less(i, j int) bool { return h[i].Distance > h[j].Distance }
func (h hitHeap) Swap(i, j int) { h[i], h[j] = h[j], h[i] }
func (h *hitHeap) Push(x any) { *h = append(*h, x.(graph.VectorHit)) }
func (h *hitHeap) Pop() any {
old := *h
n := len(old)
it := old[n-1]
*h = old[:n-1]
return it
}
@@ -0,0 +1,340 @@
package store_sqlite_test
import (
"math"
"math/rand"
"path/filepath"
"reflect"
"sort"
"testing"
"github.com/zzet/gortex/internal/graph"
"github.com/zzet/gortex/internal/graph/store_sqlite"
)
// openTestStore opens a fresh on-disk SQLite store in a temp dir and
// registers Close as cleanup. (modernc.org/sqlite's ":memory:" gives
// each pooled connection its OWN private database, so the conformance
// suite — and these tests — use an on-disk file shared across the pool.)
func openTestStore(t *testing.T) *store_sqlite.Store {
t.Helper()
s, err := store_sqlite.Open(filepath.Join(t.TempDir(), "test.sqlite"))
if err != nil {
t.Fatalf("Open: %v", err)
}
t.Cleanup(func() { _ = s.Close() })
return s
}
// --- FileMtime persistence -------------------------------------------
func TestSQLiteFileMtimeRoundTrip(t *testing.T) {
s := openTestStore(t)
// Single-row writes.
if err := s.SetFileMtime("repoA", "a/one.go", 100); err != nil {
t.Fatalf("SetFileMtime: %v", err)
}
if err := s.SetFileMtime("repoA", "a/two.go", 200); err != nil {
t.Fatalf("SetFileMtime: %v", err)
}
// Batch write (includes an overwrite of an existing key).
batch := map[string]int64{
"a/two.go": 250, // overwrite
"a/three.go": 300,
"a/four.go": 400,
}
if err := s.BulkSetFileMtimes("repoA", batch); err != nil {
t.Fatalf("BulkSetFileMtimes: %v", err)
}
want := map[string]int64{
"a/one.go": 100,
"a/two.go": 250,
"a/three.go": 300,
"a/four.go": 400,
}
got, err := s.FileMtimes("repoA")
if err != nil {
t.Fatalf("FileMtimes: %v", err)
}
if !reflect.DeepEqual(got, want) {
t.Fatalf("FileMtimes(repoA) = %v, want %v", got, want)
}
// LoadFileMtimes (the interface method) must agree.
if loaded := s.LoadFileMtimes("repoA"); !reflect.DeepEqual(loaded, want) {
t.Fatalf("LoadFileMtimes(repoA) = %v, want %v", loaded, want)
}
// Repo isolation: a different prefix is unaffected.
if err := s.SetFileMtime("repoB", "b/x.go", 999); err != nil {
t.Fatalf("SetFileMtime repoB: %v", err)
}
if got, _ := s.FileMtimes("repoA"); !reflect.DeepEqual(got, want) {
t.Fatalf("repoA changed after repoB write: %v", got)
}
// Unknown repo: FileMtimes returns an empty (non-nil) map;
// LoadFileMtimes returns nil (the "no data" signal).
empty, err := s.FileMtimes("nope")
if err != nil {
t.Fatalf("FileMtimes(unknown): %v", err)
}
if len(empty) != 0 {
t.Fatalf("FileMtimes(unknown) = %v, want empty", empty)
}
if loaded := s.LoadFileMtimes("nope"); loaded != nil {
t.Fatalf("LoadFileMtimes(unknown) = %v, want nil", loaded)
}
// Empty batch is a no-op.
if err := s.BulkSetFileMtimes("repoA", nil); err != nil {
t.Fatalf("BulkSetFileMtimes(nil): %v", err)
}
}
// --- Vector search ---------------------------------------------------
// bruteForceCosine ranks corpus against query the long way (exact cosine
// distance, ascending) so the test verifies SimilarTo independently of
// the implementation under test.
func bruteForceCosine(query []float32, corpus map[string][]float32, k int) []string {
type sc struct {
id string
dist float64
}
scored := make([]sc, 0, len(corpus))
qn := l2(query)
for id, v := range corpus {
vn := l2(v)
if qn == 0 || vn == 0 {
continue
}
var dot float64
for i := range query {
dot += float64(query[i]) * float64(v[i])
}
scored = append(scored, sc{id: id, dist: 1 - dot/(qn*vn)})
}
sort.Slice(scored, func(i, j int) bool {
if scored[i].dist == scored[j].dist {
return scored[i].id < scored[j].id // stable tie-break
}
return scored[i].dist < scored[j].dist
})
out := make([]string, 0, k)
for i := 0; i < k && i < len(scored); i++ {
out = append(out, scored[i].id)
}
return out
}
func l2(v []float32) float64 {
var s float64
for _, f := range v {
s += float64(f) * float64(f)
}
return math.Sqrt(s)
}
func TestSQLiteVectorSimilarTo(t *testing.T) {
s := openTestStore(t)
const (
n = 50
dims = 16
)
rng := rand.New(rand.NewSource(42))
corpus := make(map[string][]float32, n)
items := make([]graph.VectorItem, 0, n)
var ids []string
for i := 0; i < n; i++ {
id := nodeID(i)
ids = append(ids, id)
v := make([]float32, dims)
for d := 0; d < dims; d++ {
v[d] = float32(rng.NormFloat64())
}
corpus[id] = v
items = append(items, graph.VectorItem{NodeID: id, Vec: v})
}
if err := s.BulkUpsertEmbeddings(items); err != nil {
t.Fatalf("BulkUpsertEmbeddings: %v", err)
}
if err := s.BuildVectorIndex(dims); err != nil {
t.Fatalf("BuildVectorIndex: %v", err)
}
// Query == a stored vector → it must rank first at distance ~0.
queryID := ids[7]
query := corpus[queryID]
hits, err := s.SimilarTo(query, 5)
if err != nil {
t.Fatalf("SimilarTo: %v", err)
}
if len(hits) != 5 {
t.Fatalf("SimilarTo returned %d hits, want 5", len(hits))
}
if hits[0].NodeID != queryID {
t.Fatalf("top hit = %q, want the query vector %q", hits[0].NodeID, queryID)
}
if hits[0].Distance > 1e-6 {
t.Fatalf("top hit distance = %g, want ~0", hits[0].Distance)
}
// Distances must be ascending.
for i := 1; i < len(hits); i++ {
if hits[i].Distance < hits[i-1].Distance {
t.Fatalf("hits not ascending by distance: %v", hits)
}
}
// Independent brute-force ranking must match the returned top-5 ids.
want := bruteForceCosine(query, corpus, 5)
gotIDs := make([]string, len(hits))
for i, h := range hits {
gotIDs[i] = h.NodeID
}
if !reflect.DeepEqual(gotIDs, want) {
t.Fatalf("SimilarTo top-5 = %v, brute-force = %v", gotIDs, want)
}
// Single-add path: a new vector identical to ids[3]'s should be
// retrievable and rank at distance ~0 for its own query.
extra := make([]float32, dims)
copy(extra, corpus[ids[3]])
if err := s.UpsertEmbedding("extra::node", extra); err != nil {
t.Fatalf("UpsertEmbedding: %v", err)
}
exHits, err := s.SimilarTo(extra, 3)
if err != nil {
t.Fatalf("SimilarTo (extra): %v", err)
}
if len(exHits) == 0 {
t.Fatalf("SimilarTo(extra) returned nothing")
}
// Either the original ids[3] or the new extra::node (both identical
// vectors, distance ~0) may sort first; the new one must be present
// at distance ~0.
foundExtra := false
for _, h := range exHits {
if h.NodeID == "extra::node" {
foundExtra = true
if h.Distance > 1e-6 {
t.Fatalf("extra::node distance = %g, want ~0", h.Distance)
}
}
}
if !foundExtra {
t.Fatalf("UpsertEmbedding'd vector not found in SimilarTo results: %v", exHits)
}
}
func TestSQLiteVectorPersistence(t *testing.T) {
path := filepath.Join(t.TempDir(), "v.sqlite")
corpus := map[string][]float32{
"n::1": {1, 0, 0, 0, 0, 0, 0, 0},
"n::2": {0, 1, 0, 0, 0, 0, 0, 0},
"n::3": {0, 0, 1, 0, 0, 0, 0, 0},
}
// First session: write and close.
{
s, err := store_sqlite.Open(path)
if err != nil {
t.Fatalf("open: %v", err)
}
items := make([]graph.VectorItem, 0, len(corpus))
for id, v := range corpus {
items = append(items, graph.VectorItem{NodeID: id, Vec: v})
}
if err := s.BulkUpsertEmbeddings(items); err != nil {
t.Fatalf("BulkUpsertEmbeddings: %v", err)
}
if err := s.Close(); err != nil {
t.Fatalf("close: %v", err)
}
}
// Second session: reopen, vectors must still be queryable.
{
s, err := store_sqlite.Open(path)
if err != nil {
t.Fatalf("reopen: %v", err)
}
t.Cleanup(func() { _ = s.Close() })
query := []float32{1, 0, 0, 0, 0, 0, 0, 0}
hits, err := s.SimilarTo(query, 3)
if err != nil {
t.Fatalf("SimilarTo after reopen: %v", err)
}
if len(hits) != 3 {
t.Fatalf("after reopen got %d hits, want 3 (persistence failed)", len(hits))
}
if hits[0].NodeID != "n::1" {
t.Fatalf("after reopen top hit = %q, want n::1", hits[0].NodeID)
}
if hits[0].Distance > 1e-6 {
t.Fatalf("after reopen top distance = %g, want ~0", hits[0].Distance)
}
}
}
func TestSQLiteGetEmbeddings(t *testing.T) {
s := openTestStore(t)
corpus := map[string][]float32{
"n::1": {1, 0, 0, 0},
"n::2": {0, 1, 0, 0},
"n::3": {0, 0, 1, 0},
}
items := make([]graph.VectorItem, 0, len(corpus))
for id, v := range corpus {
items = append(items, graph.VectorItem{NodeID: id, Vec: v})
}
if err := s.BulkUpsertEmbeddings(items); err != nil {
t.Fatalf("BulkUpsertEmbeddings: %v", err)
}
// Batch read of a mix of present + absent IDs: present IDs come back
// with their exact stored vectors; absent IDs are simply omitted.
got := s.GetEmbeddings([]string{"n::1", "n::3", "missing::x"})
if len(got) != 2 {
t.Fatalf("GetEmbeddings returned %d vectors, want 2 (absent id must be omitted)", len(got))
}
if !reflect.DeepEqual(got["n::1"], corpus["n::1"]) {
t.Fatalf("GetEmbeddings[n::1] = %v, want %v", got["n::1"], corpus["n::1"])
}
if !reflect.DeepEqual(got["n::3"], corpus["n::3"]) {
t.Fatalf("GetEmbeddings[n::3] = %v, want %v", got["n::3"], corpus["n::3"])
}
if _, present := got["missing::x"]; present {
t.Fatalf("GetEmbeddings must omit ids with no stored vector")
}
// Empty input yields a non-nil empty map, never an error or panic.
if empty := s.GetEmbeddings(nil); empty == nil || len(empty) != 0 {
t.Fatalf("GetEmbeddings(nil) = %v, want empty non-nil map", empty)
}
}
func nodeID(i int) string {
const digits = "0123456789"
if i == 0 {
return "node::0"
}
var b []byte
for i > 0 {
b = append([]byte{digits[i%10]}, b...)
i /= 10
}
return "node::" + string(b)
}
@@ -0,0 +1,92 @@
package store_sqlite
import (
"fmt"
"os"
"path/filepath"
"testing"
"time"
"github.com/zzet/gortex/internal/graph"
)
// TestCheckpointWALBoundsFileAndPreservesData writes enough rows to push pages
// into the -wal file, then forces a TRUNCATE checkpoint and asserts the WAL is
// drained (bounded well under journal_size_limit) without losing any data.
func TestCheckpointWALBoundsFileAndPreservesData(t *testing.T) {
path := filepath.Join(t.TempDir(), "wal.sqlite")
s, err := Open(path)
if err != nil {
t.Fatalf("open: %v", err)
}
t.Cleanup(func() { _ = s.Close() })
// On-disk stores must arm the background checkpoint loop.
if s.stopCheckpoint == nil || s.checkpointDone == nil {
t.Fatal("on-disk store did not start the WAL-checkpoint loop")
}
const n = 4000
nodes := make([]*graph.Node, 0, n)
for i := range n {
nodes = append(nodes, &graph.Node{
ID: fmt.Sprintf("pkg/f.go::Sym%d", i),
Kind: graph.KindFunction,
Name: fmt.Sprintf("Sym%d", i),
FilePath: "pkg/f.go",
Language: "go",
})
}
s.AddBatch(nodes, nil)
if err := s.CheckpointWAL(); err != nil {
t.Fatalf("checkpoint: %v", err)
}
// journal_size_limit caps the WAL at 64 MiB; after a TRUNCATE checkpoint
// with no concurrent reader it should be far smaller still.
if fi, err := os.Stat(path + "-wal"); err == nil && fi.Size() > 64<<20 {
t.Fatalf("wal not bounded after checkpoint: %d bytes", fi.Size())
}
if got := s.NodeCount(); got != n {
t.Fatalf("checkpoint lost data: NodeCount = %d, want %d", got, n)
}
}
// TestCloseStopsCheckpointLoop verifies Close signals the loop and waits for it
// to exit, and that calling Close twice does not panic on the stop channel.
func TestCloseStopsCheckpointLoop(t *testing.T) {
path := filepath.Join(t.TempDir(), "wal.sqlite")
s, err := Open(path)
if err != nil {
t.Fatalf("open: %v", err)
}
done := s.checkpointDone
if err := s.Close(); err != nil {
t.Fatalf("close: %v", err)
}
select {
case <-done:
case <-time.After(2 * time.Second):
t.Fatal("checkpoint loop did not stop within 2s of Close")
}
// stopCheckpointLoop is guarded by sync.Once, so a second stop is a no-op
// rather than a close-of-closed-channel panic.
s.stopCheckpointLoop()
}
// TestInMemoryStoreSkipsCheckpointLoop confirms ":memory:" stores, which have
// no WAL, never spawn the checkpoint goroutine.
func TestInMemoryStoreSkipsCheckpointLoop(t *testing.T) {
s, err := Open(":memory:")
if err != nil {
t.Fatalf("open: %v", err)
}
t.Cleanup(func() { _ = s.Close() })
if s.stopCheckpoint != nil || s.checkpointDone != nil {
t.Fatal("in-memory store should not arm the WAL-checkpoint loop")
}
}