chore: import upstream snapshot with attribution
CI / benchmark (push) Has been skipped
install-script / posix-syntax (push) Successful in 6m1s
CI / build-onnx (push) Failing after 6m43s
init-smoke / dry-run (push) Failing after 15m57s
security / govulncheck (push) Has been cancelled
security / trivy-fs (push) Has been cancelled
CI / test (1.26, ubuntu-latest) (push) Has been cancelled
Scorecard supply-chain security / Scorecard analysis (push) Has been cancelled
CI / test (1.26, macos-latest) (push) Has been cancelled
CI / build-windows (push) Has been cancelled
CI / lint (push) Has been cancelled
install-script / powershell-syntax (push) Has been cancelled
install-script / install (macos-14) (push) Has been cancelled
install-script / install (ubuntu-latest) (push) Has been cancelled
CI / benchmark (push) Has been skipped
install-script / posix-syntax (push) Successful in 6m1s
CI / build-onnx (push) Failing after 6m43s
init-smoke / dry-run (push) Failing after 15m57s
security / govulncheck (push) Has been cancelled
security / trivy-fs (push) Has been cancelled
CI / test (1.26, ubuntu-latest) (push) Has been cancelled
Scorecard supply-chain security / Scorecard analysis (push) Has been cancelled
CI / test (1.26, macos-latest) (push) Has been cancelled
CI / build-windows (push) Has been cancelled
CI / lint (push) Has been cancelled
install-script / powershell-syntax (push) Has been cancelled
install-script / install (macos-14) (push) Has been cancelled
install-script / install (ubuntu-latest) (push) Has been cancelled
This commit is contained in:
@@ -0,0 +1,218 @@
|
||||
package store_sqlite
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"errors"
|
||||
"fmt"
|
||||
|
||||
"github.com/zzet/gortex/internal/graph"
|
||||
)
|
||||
|
||||
// Compile-time assertion: *Store satisfies graph.BulkLoader.
|
||||
var _ graph.BulkLoader = (*Store)(nil)
|
||||
|
||||
// bulkDroppableIndex is one secondary index the bulk-load fast path drops
|
||||
// before a first/empty cold index and rebuilds afterward.
|
||||
type bulkDroppableIndex struct {
|
||||
name string
|
||||
ddl string
|
||||
}
|
||||
|
||||
// bulkDroppableIndexes is the single source of truth for these index
|
||||
// definitions. Open creates them (so the initial DB has them), BeginBulkLoad
|
||||
// drops them by name, and FlushBulk recreates them from the exact same ddl —
|
||||
// keeping the initial and post-bulk shapes from drifting.
|
||||
//
|
||||
// These are exactly the standalone, NON-UNIQUE CREATE INDEX statements over
|
||||
// the large nodes / edges tables. Maintaining them per-row across a
|
||||
// multi-hundred-thousand-row cold load is pure overhead when the rows land
|
||||
// once, so they are dropped up front and rebuilt in one pass at the end.
|
||||
//
|
||||
// Deliberately excluded:
|
||||
// - nodes_by_qual (UNIQUE): enforces qual_name dedup on every
|
||||
// INSERT OR REPLACE. Dropping it would change insert conflict semantics
|
||||
// (collapsed qual_name collisions would diverge from the non-bulk path)
|
||||
// and a duplicate could make the recreate fail. It stays live.
|
||||
// - the edges UNIQUE(from_id, …) table constraint and every WITHOUT ROWID
|
||||
// primary-key index: not standalone indexes; they cannot be dropped while
|
||||
// the table/constraint exists.
|
||||
// - edges_external (partial): a tiny index over external-call terminals,
|
||||
// created from a shared predicate in Open; not worth dropping.
|
||||
//
|
||||
// Dropping/recreating these is a runtime operation on identical DDL — it is
|
||||
// NOT a schema change, so it does not touch the persisted schema version.
|
||||
var bulkDroppableIndexes = []bulkDroppableIndex{
|
||||
{"nodes_by_name", `CREATE INDEX IF NOT EXISTS nodes_by_name ON nodes(name)`},
|
||||
{"nodes_by_kind", `CREATE INDEX IF NOT EXISTS nodes_by_kind ON nodes(kind)`},
|
||||
{"nodes_by_file", `CREATE INDEX IF NOT EXISTS nodes_by_file ON nodes(file_path)`},
|
||||
{"nodes_by_repo", `CREATE INDEX IF NOT EXISTS nodes_by_repo ON nodes(repo_prefix) WHERE repo_prefix <> ''`},
|
||||
{"edges_by_from", `CREATE INDEX IF NOT EXISTS edges_by_from ON edges(from_id, kind)`},
|
||||
{"edges_by_to", `CREATE INDEX IF NOT EXISTS edges_by_to ON edges(to_id, kind)`},
|
||||
{"edges_by_kind", `CREATE INDEX IF NOT EXISTS edges_by_kind ON edges(kind)`},
|
||||
// Backs EdgesWithUnresolvedTarget — the resolver's main pending-edge
|
||||
// collector, called on every full resolve. is_unresolved is a VIRTUAL
|
||||
// generated column (see isUnresolvedColumnDDL); indexing it turns a
|
||||
// full-table scan (the prior to_id-based OR query forced SQLite to
|
||||
// abandon its index) into an index search whose bookmark lookups land
|
||||
// in ascending rowid order (see isUnresolvedColumnDDL's doc comment for
|
||||
// why that beats an equivalent-looking to_id-based index).
|
||||
{"edges_by_unresolved", `CREATE INDEX IF NOT EXISTS edges_by_unresolved ON edges(is_unresolved)`},
|
||||
// Partial index over exactly the not-yet-semantically-stamped nodes per
|
||||
// repo. Stays small in steady state (most nodes end up stamped), so a
|
||||
// future "unstamped nodes in this repo" query is an index scan over the
|
||||
// residual few instead of a full-table decode of every node's meta.
|
||||
{"nodes_semantic_pending", `CREATE INDEX IF NOT EXISTS nodes_semantic_pending ON nodes(repo_prefix) WHERE semantic_type IS NULL`},
|
||||
}
|
||||
|
||||
// bulkCacheSizeKiB is the page cache the fast path requests on its pinned
|
||||
// connection. SQLite reads a negative cache_size as a KiB budget, so this is
|
||||
// ~256 MiB — large enough to keep the cold load's working set resident.
|
||||
const bulkCacheSizeKiB = -262144
|
||||
|
||||
// beginWrite starts a write transaction. During a bulk-load fast path it pins
|
||||
// the single connection that carries synchronous=OFF + the enlarged page
|
||||
// cache (database/sql PRAGMAs are connection-local, so a pooled connection
|
||||
// would not see them); otherwise it uses the shared pool. The caller holds
|
||||
// writeMu, which also guards s.bulkConn.
|
||||
func (s *Store) beginWrite() (*sql.Tx, error) {
|
||||
if s.bulkConn != nil {
|
||||
return s.bulkConn.BeginTx(context.Background(), nil)
|
||||
}
|
||||
return s.db.Begin()
|
||||
}
|
||||
|
||||
// BeginBulkLoad enters the bulk-load fast path for a first/empty cold index.
|
||||
// It pins one connection at synchronous=OFF with an enlarged page cache and
|
||||
// drops the droppable secondary indexes, so a multi-hundred-thousand-row load
|
||||
// skips per-row B-tree maintenance and per-commit fsync. FlushBulk reverses
|
||||
// all of it: restore the pragmas, rebuild the indexes, and checkpoint.
|
||||
//
|
||||
// Gated: it engages ONLY when the nodes table is empty. On a populated store
|
||||
// (incremental reindex, warm restart, or a later repo in a multi-repo cold
|
||||
// start that shares the disk store) it is a safe no-op — dropping indexes or
|
||||
// disabling crash durability under live, concurrently-readable rows would be
|
||||
// unsafe. In-memory stores have no WAL / on-disk B-tree pressure, so it is a
|
||||
// no-op there too.
|
||||
func (s *Store) BeginBulkLoad() {
|
||||
s.writeMu.Lock()
|
||||
defer s.writeMu.Unlock()
|
||||
|
||||
// Re-entrancy / non-disk guard: a second BeginBulkLoad without an
|
||||
// intervening FlushBulk, or an in-memory store, stays a no-op.
|
||||
if s.bulkConn != nil || isMemoryPath(s.dbPath) {
|
||||
return
|
||||
}
|
||||
|
||||
ctx := context.Background()
|
||||
conn, err := s.db.Conn(ctx)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
// Gate to a genuinely first/empty index.
|
||||
if !nodesTableEmpty(ctx, conn) {
|
||||
_ = conn.Close()
|
||||
return
|
||||
}
|
||||
|
||||
// Capture prior pragma values so FlushBulk (and every early-return /
|
||||
// error path) can restore them. If they can't be read, don't engage —
|
||||
// a slow correct load beats a connection stuck at synchronous=OFF.
|
||||
prevSync, err := pragmaInt(ctx, conn, "synchronous")
|
||||
if err != nil {
|
||||
_ = conn.Close()
|
||||
return
|
||||
}
|
||||
prevCache, err := pragmaInt(ctx, conn, "cache_size")
|
||||
if err != nil {
|
||||
_ = conn.Close()
|
||||
return
|
||||
}
|
||||
|
||||
// synchronous=OFF drops crash durability for the load window —
|
||||
// acceptable only because a crash on a fresh index just re-indexes.
|
||||
if _, err := conn.ExecContext(ctx, "PRAGMA synchronous = OFF"); err != nil {
|
||||
_ = conn.Close()
|
||||
return
|
||||
}
|
||||
if _, err := conn.ExecContext(ctx, fmt.Sprintf("PRAGMA cache_size = %d", bulkCacheSizeKiB)); err != nil {
|
||||
// Roll the durability change back before bailing.
|
||||
_, _ = conn.ExecContext(ctx, fmt.Sprintf("PRAGMA synchronous = %d", prevSync))
|
||||
_ = conn.Close()
|
||||
return
|
||||
}
|
||||
|
||||
// Drop the droppable secondary indexes; rebuilt in one pass at
|
||||
// FlushBulk. Best-effort: a failed drop just means that index keeps
|
||||
// being maintained per-row (slower, still correct), so it is not fatal.
|
||||
for _, idx := range bulkDroppableIndexes {
|
||||
_, _ = conn.ExecContext(ctx, "DROP INDEX IF EXISTS "+idx.name)
|
||||
}
|
||||
|
||||
s.bulkConn = conn
|
||||
s.bulkPrevSync = prevSync
|
||||
s.bulkPrevCacheSize = prevCache
|
||||
}
|
||||
|
||||
// FlushBulk exits the bulk-load fast path: it rebuilds every index
|
||||
// BeginBulkLoad dropped, restores synchronous + cache_size on the pinned
|
||||
// connection, runs one TRUNCATE checkpoint to drain the WAL the no-fsync load
|
||||
// grew, and returns the connection to the pool. It is a no-op when no fast
|
||||
// path is active (BeginBulkLoad gated out, or already flushed).
|
||||
//
|
||||
// The pragma restore + connection release run unconditionally (defer), so a
|
||||
// failure mid-rebuild can never leave a connection stuck at synchronous=OFF
|
||||
// in the pool.
|
||||
func (s *Store) FlushBulk() error {
|
||||
s.writeMu.Lock()
|
||||
defer s.writeMu.Unlock()
|
||||
|
||||
conn := s.bulkConn
|
||||
if conn == nil {
|
||||
return nil
|
||||
}
|
||||
// Detach first: the fast path is over regardless of the outcome below.
|
||||
s.bulkConn = nil
|
||||
|
||||
ctx := context.Background()
|
||||
defer func() {
|
||||
// Always restore durability + cache and release the connection,
|
||||
// even if an index rebuild failed.
|
||||
_, _ = conn.ExecContext(ctx, fmt.Sprintf("PRAGMA synchronous = %d", s.bulkPrevSync))
|
||||
_, _ = conn.ExecContext(ctx, fmt.Sprintf("PRAGMA cache_size = %d", s.bulkPrevCacheSize))
|
||||
_ = conn.Close()
|
||||
}()
|
||||
|
||||
for _, idx := range bulkDroppableIndexes {
|
||||
if _, err := conn.ExecContext(ctx, idx.ddl); err != nil {
|
||||
return fmt.Errorf("store_sqlite: rebuild index %s: %w", idx.name, err)
|
||||
}
|
||||
}
|
||||
|
||||
// Drain the WAL the no-fsync bulk window grew back into the main DB and
|
||||
// truncate the -wal file. Same TRUNCATE mode as runCheckpointLoop, so it
|
||||
// cooperates with the journal_size_limit / periodic-checkpoint policy.
|
||||
if _, err := conn.ExecContext(ctx, "PRAGMA wal_checkpoint(TRUNCATE)"); err != nil {
|
||||
return fmt.Errorf("store_sqlite: bulk checkpoint: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// nodesTableEmpty reports whether the nodes table holds no rows. Used to gate
|
||||
// the bulk-load fast path to a genuinely first/empty cold index.
|
||||
func nodesTableEmpty(ctx context.Context, conn *sql.Conn) bool {
|
||||
var one int
|
||||
err := conn.QueryRowContext(ctx, "SELECT 1 FROM nodes LIMIT 1").Scan(&one)
|
||||
return errors.Is(err, sql.ErrNoRows)
|
||||
}
|
||||
|
||||
// pragmaInt reads a single-integer PRAGMA (synchronous, cache_size) off the
|
||||
// given connection.
|
||||
func pragmaInt(ctx context.Context, conn *sql.Conn, pragma string) (int64, error) {
|
||||
var v int64
|
||||
if err := conn.QueryRowContext(ctx, "PRAGMA "+pragma).Scan(&v); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return v, nil
|
||||
}
|
||||
@@ -0,0 +1,388 @@
|
||||
package store_sqlite
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/zzet/gortex/internal/graph"
|
||||
)
|
||||
|
||||
// bulkFixture builds a deterministic node/edge set with distinct ids and
|
||||
// qual_names (so the UNIQUE nodes_by_qual index never collides) and a mix of
|
||||
// edge keys (a handful collide → exercise dedup). Input order is intentionally
|
||||
// not key-sorted.
|
||||
func bulkFixture(nNodes, nEdges int) ([]*graph.Node, []*graph.Edge) {
|
||||
nodes := make([]*graph.Node, 0, nNodes)
|
||||
for i := range nNodes {
|
||||
nodes = append(nodes, &graph.Node{
|
||||
ID: fmt.Sprintf("pkg/f%d.go::Sym%d", i%64, i),
|
||||
Kind: graph.KindFunction,
|
||||
Name: fmt.Sprintf("Sym%d", i),
|
||||
QualName: fmt.Sprintf("pkg.f%d.Sym%d", i%64, i),
|
||||
FilePath: fmt.Sprintf("pkg/f%d.go", i%64),
|
||||
RepoPrefix: "gortex",
|
||||
Language: "go",
|
||||
})
|
||||
}
|
||||
edges := make([]*graph.Edge, 0, nEdges)
|
||||
for i := range nEdges {
|
||||
from := nodes[i%nNodes]
|
||||
to := nodes[(i*7+1)%nNodes]
|
||||
edges = append(edges, &graph.Edge{
|
||||
From: from.ID,
|
||||
To: to.ID,
|
||||
Kind: graph.EdgeCalls,
|
||||
FilePath: from.FilePath,
|
||||
Line: i % 500,
|
||||
Confidence: 1,
|
||||
})
|
||||
}
|
||||
return nodes, edges
|
||||
}
|
||||
|
||||
func openTempStore(t *testing.T) (*Store, string) {
|
||||
t.Helper()
|
||||
path := filepath.Join(t.TempDir(), "bulk.sqlite")
|
||||
s, err := Open(path)
|
||||
if err != nil {
|
||||
t.Fatalf("open: %v", err)
|
||||
}
|
||||
t.Cleanup(func() { _ = s.Close() })
|
||||
return s, path
|
||||
}
|
||||
|
||||
// indexNames returns the set of secondary index names present in the schema.
|
||||
func indexNames(t *testing.T, q interface {
|
||||
Query(string, ...any) (*sql.Rows, error)
|
||||
}) map[string]bool {
|
||||
t.Helper()
|
||||
rows, err := q.Query("SELECT name FROM sqlite_master WHERE type='index'")
|
||||
if err != nil {
|
||||
t.Fatalf("query sqlite_master: %v", err)
|
||||
}
|
||||
defer func() { _ = rows.Close() }()
|
||||
got := map[string]bool{}
|
||||
for rows.Next() {
|
||||
var n string
|
||||
if err := rows.Scan(&n); err != nil {
|
||||
t.Fatalf("scan index name: %v", err)
|
||||
}
|
||||
got[n] = true
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
t.Fatalf("rows: %v", err)
|
||||
}
|
||||
return got
|
||||
}
|
||||
|
||||
func pragmaIntDB(t *testing.T, db *sql.DB, pragma string) int64 {
|
||||
t.Helper()
|
||||
var v int64
|
||||
if err := db.QueryRow("PRAGMA " + pragma).Scan(&v); err != nil {
|
||||
t.Fatalf("pragma %s: %v", pragma, err)
|
||||
}
|
||||
return v
|
||||
}
|
||||
|
||||
func integrityOK(t *testing.T, db *sql.DB) {
|
||||
t.Helper()
|
||||
var res string
|
||||
if err := db.QueryRow("PRAGMA integrity_check").Scan(&res); err != nil {
|
||||
t.Fatalf("integrity_check: %v", err)
|
||||
}
|
||||
if res != "ok" {
|
||||
t.Fatalf("integrity_check = %q, want \"ok\"", res)
|
||||
}
|
||||
}
|
||||
|
||||
// TestBulkLoadDropsAndRebuildsIndexes is the core mechanism + restore proof:
|
||||
// the fast path engages on an empty store, drops the droppable indexes,
|
||||
// pins a synchronous=OFF connection, then on FlushBulk rebuilds every index,
|
||||
// restores synchronous, releases the connection, and leaves the DB intact.
|
||||
func TestBulkLoadDropsAndRebuildsIndexes(t *testing.T) {
|
||||
s, _ := openTempStore(t)
|
||||
ctx := context.Background()
|
||||
|
||||
// Baseline: all droppable indexes present, synchronous=NORMAL(1).
|
||||
before := indexNames(t, s.db)
|
||||
for _, idx := range bulkDroppableIndexes {
|
||||
if !before[idx.name] {
|
||||
t.Fatalf("index %s missing before bulk load", idx.name)
|
||||
}
|
||||
}
|
||||
if got := pragmaIntDB(t, s.db, "synchronous"); got != 1 {
|
||||
t.Fatalf("synchronous before = %d, want 1 (NORMAL)", got)
|
||||
}
|
||||
|
||||
// Engage the fast path on the empty store.
|
||||
s.BeginBulkLoad()
|
||||
if s.bulkConn == nil {
|
||||
t.Fatal("fast path did not engage on empty store")
|
||||
}
|
||||
// Read through the pinned connection (it may be the only one when
|
||||
// GOMAXPROCS/NumCPU is 1) to avoid blocking on connection acquisition.
|
||||
var sync int64
|
||||
if err := s.bulkConn.QueryRowContext(ctx, "PRAGMA synchronous").Scan(&sync); err != nil {
|
||||
t.Fatalf("pinned synchronous: %v", err)
|
||||
}
|
||||
if sync != 0 {
|
||||
t.Fatalf("bulk synchronous = %d, want 0 (OFF)", sync)
|
||||
}
|
||||
var cache int64
|
||||
if err := s.bulkConn.QueryRowContext(ctx, "PRAGMA cache_size").Scan(&cache); err != nil {
|
||||
t.Fatalf("pinned cache_size: %v", err)
|
||||
}
|
||||
if cache != bulkCacheSizeKiB {
|
||||
t.Fatalf("bulk cache_size = %d, want %d", cache, bulkCacheSizeKiB)
|
||||
}
|
||||
// The droppable indexes are gone during the window.
|
||||
during := indexNames(t, &connQuerier{ctx: ctx, c: s.bulkConn})
|
||||
for _, idx := range bulkDroppableIndexes {
|
||||
if during[idx.name] {
|
||||
t.Fatalf("index %s still present during bulk window", idx.name)
|
||||
}
|
||||
}
|
||||
// nodes_by_qual (UNIQUE, not droppable) must remain live.
|
||||
if !during["nodes_by_qual"] {
|
||||
t.Fatal("nodes_by_qual (UNIQUE) must not be dropped")
|
||||
}
|
||||
|
||||
nodes, edges := bulkFixture(2000, 4000)
|
||||
s.AddBatch(nodes, edges)
|
||||
|
||||
if err := s.FlushBulk(); err != nil {
|
||||
t.Fatalf("FlushBulk: %v", err)
|
||||
}
|
||||
if s.bulkConn != nil {
|
||||
t.Fatal("bulkConn not released after FlushBulk")
|
||||
}
|
||||
|
||||
// Every dropped index is back.
|
||||
after := indexNames(t, s.db)
|
||||
for _, idx := range bulkDroppableIndexes {
|
||||
if !after[idx.name] {
|
||||
t.Fatalf("index %s not rebuilt after FlushBulk", idx.name)
|
||||
}
|
||||
}
|
||||
// synchronous restored to NORMAL on the pool.
|
||||
if got := pragmaIntDB(t, s.db, "synchronous"); got != 1 {
|
||||
t.Fatalf("synchronous after = %d, want 1 (NORMAL)", got)
|
||||
}
|
||||
integrityOK(t, s.db)
|
||||
}
|
||||
|
||||
// connQuerier adapts *sql.Conn to the Query signature indexNames expects.
|
||||
type connQuerier struct {
|
||||
ctx context.Context
|
||||
c *sql.Conn
|
||||
}
|
||||
|
||||
func (q *connQuerier) Query(query string, args ...any) (*sql.Rows, error) {
|
||||
return q.c.QueryContext(q.ctx, query, args...)
|
||||
}
|
||||
|
||||
// TestBulkLoadMatchesNonBulkCounts proves the fast path persists exactly the
|
||||
// same node/edge counts the plain AddBatch path does.
|
||||
func TestBulkLoadMatchesNonBulkCounts(t *testing.T) {
|
||||
nodes, edges := bulkFixture(3000, 6000)
|
||||
|
||||
plain, _ := openTempStore(t)
|
||||
plain.AddBatch(nodes, edges)
|
||||
wantNodes, wantEdges := plain.NodeCount(), plain.EdgeCount()
|
||||
|
||||
bulk, _ := openTempStore(t)
|
||||
bulk.BeginBulkLoad()
|
||||
if bulk.bulkConn == nil {
|
||||
t.Fatal("fast path did not engage on empty store")
|
||||
}
|
||||
// Drain in two chunks to mirror the indexer's chunked persist.
|
||||
bulk.AddBatch(nodes[:1500], nil)
|
||||
bulk.AddBatch(nodes[1500:], nil)
|
||||
bulk.AddBatch(nil, edges[:3000])
|
||||
bulk.AddBatch(nil, edges[3000:])
|
||||
if err := bulk.FlushBulk(); err != nil {
|
||||
t.Fatalf("FlushBulk: %v", err)
|
||||
}
|
||||
|
||||
if gotN, gotE := bulk.NodeCount(), bulk.EdgeCount(); gotN != wantNodes || gotE != wantEdges {
|
||||
t.Fatalf("bulk counts (%d nodes, %d edges) != non-bulk (%d, %d)", gotN, gotE, wantNodes, wantEdges)
|
||||
}
|
||||
integrityOK(t, bulk.db)
|
||||
}
|
||||
|
||||
// TestBulkLoadGatedToPopulatedStore confirms the fast path is a safe no-op on
|
||||
// a store that already holds rows — no indexes are dropped, durability stays.
|
||||
func TestBulkLoadGatedToPopulatedStore(t *testing.T) {
|
||||
s, _ := openTempStore(t)
|
||||
// Populate first (the normal, non-bulk path).
|
||||
nodes, edges := bulkFixture(50, 100)
|
||||
s.AddBatch(nodes, edges)
|
||||
|
||||
s.BeginBulkLoad()
|
||||
if s.bulkConn != nil {
|
||||
t.Fatal("fast path engaged on a populated store; must be a no-op")
|
||||
}
|
||||
// Indexes untouched, durability untouched.
|
||||
present := indexNames(t, s.db)
|
||||
for _, idx := range bulkDroppableIndexes {
|
||||
if !present[idx.name] {
|
||||
t.Fatalf("index %s dropped on a populated store", idx.name)
|
||||
}
|
||||
}
|
||||
if got := pragmaIntDB(t, s.db, "synchronous"); got != 1 {
|
||||
t.Fatalf("synchronous = %d on populated store, want 1 (NORMAL)", got)
|
||||
}
|
||||
if err := s.FlushBulk(); err != nil {
|
||||
t.Fatalf("FlushBulk no-op returned error: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// TestBulkLoadInMemoryIsNoOp confirms in-memory stores never engage the fast
|
||||
// path (no WAL / on-disk B-tree to optimise).
|
||||
func TestBulkLoadInMemoryIsNoOp(t *testing.T) {
|
||||
s, err := Open(":memory:")
|
||||
if err != nil {
|
||||
t.Fatalf("open: %v", err)
|
||||
}
|
||||
t.Cleanup(func() { _ = s.Close() })
|
||||
s.BeginBulkLoad()
|
||||
if s.bulkConn != nil {
|
||||
t.Fatal("fast path engaged on an in-memory store")
|
||||
}
|
||||
if err := s.FlushBulk(); err != nil {
|
||||
t.Fatalf("FlushBulk: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// TestBulkLoadWarmRestartLoadsClean bulk-loads, closes, reopens the same file,
|
||||
// and asserts the persisted graph round-trips: identical counts, indexes
|
||||
// present, integrity ok.
|
||||
func TestBulkLoadWarmRestartLoadsClean(t *testing.T) {
|
||||
path := filepath.Join(t.TempDir(), "warm.sqlite")
|
||||
nodes, edges := bulkFixture(2500, 5000)
|
||||
|
||||
s, err := Open(path)
|
||||
if err != nil {
|
||||
t.Fatalf("open: %v", err)
|
||||
}
|
||||
s.BeginBulkLoad()
|
||||
if s.bulkConn == nil {
|
||||
t.Fatal("fast path did not engage on empty store")
|
||||
}
|
||||
s.AddBatch(nodes, edges)
|
||||
if err := s.FlushBulk(); err != nil {
|
||||
t.Fatalf("FlushBulk: %v", err)
|
||||
}
|
||||
wantNodes, wantEdges := s.NodeCount(), s.EdgeCount()
|
||||
if err := s.Close(); err != nil {
|
||||
t.Fatalf("close: %v", err)
|
||||
}
|
||||
|
||||
reopened, err := Open(path)
|
||||
if err != nil {
|
||||
t.Fatalf("reopen: %v", err)
|
||||
}
|
||||
t.Cleanup(func() { _ = reopened.Close() })
|
||||
|
||||
if gotN, gotE := reopened.NodeCount(), reopened.EdgeCount(); gotN != wantNodes || gotE != wantEdges {
|
||||
t.Fatalf("warm restart counts (%d, %d) != pre-close (%d, %d)", gotN, gotE, wantNodes, wantEdges)
|
||||
}
|
||||
present := indexNames(t, reopened.db)
|
||||
for _, idx := range bulkDroppableIndexes {
|
||||
if !present[idx.name] {
|
||||
t.Fatalf("index %s missing after warm restart", idx.name)
|
||||
}
|
||||
}
|
||||
integrityOK(t, reopened.db)
|
||||
// A populated store on reopen must NOT engage the fast path.
|
||||
reopened.BeginBulkLoad()
|
||||
if reopened.bulkConn != nil {
|
||||
t.Fatal("fast path engaged on warm restart (populated store)")
|
||||
}
|
||||
_ = reopened.FlushBulk()
|
||||
}
|
||||
|
||||
// TestBulkLoadPersistSpeed is the persist-speed evidence: it times the plain
|
||||
// path vs the fast path on the same fixture and logs both. It asserts
|
||||
// correctness and that the fast path is not pathologically slower; a strict
|
||||
// speedup ratio is gated behind GORTEX_BULK_PERF_ASSERT so the default run
|
||||
// stays deterministic on noisy CI.
|
||||
func TestBulkLoadPersistSpeed(t *testing.T) {
|
||||
if testing.Short() {
|
||||
t.Skip("skipping persist-speed timing in -short")
|
||||
}
|
||||
n, e := 8000, 16000
|
||||
nodes, edges := bulkFixture(n, e)
|
||||
|
||||
plain, _ := openTempStore(t)
|
||||
t0 := time.Now()
|
||||
plain.AddBatch(nodes, edges)
|
||||
plainDur := time.Since(t0)
|
||||
|
||||
bulk, _ := openTempStore(t)
|
||||
t1 := time.Now()
|
||||
bulk.BeginBulkLoad()
|
||||
bulk.AddBatch(nodes, edges)
|
||||
if err := bulk.FlushBulk(); err != nil {
|
||||
t.Fatalf("FlushBulk: %v", err)
|
||||
}
|
||||
bulkDur := time.Since(t1)
|
||||
|
||||
if bulk.NodeCount() != plain.NodeCount() || bulk.EdgeCount() != plain.EdgeCount() {
|
||||
t.Fatalf("count mismatch: bulk(%d,%d) plain(%d,%d)",
|
||||
bulk.NodeCount(), bulk.EdgeCount(), plain.NodeCount(), plain.EdgeCount())
|
||||
}
|
||||
integrityOK(t, bulk.db)
|
||||
|
||||
ratio := float64(plainDur) / float64(bulkDur)
|
||||
t.Logf("persist %d nodes / %d edges: plain=%s bulk=%s speedup=%.2fx",
|
||||
n, e, plainDur, bulkDur, ratio)
|
||||
|
||||
// Sanity floor: the fast path must never be dramatically slower.
|
||||
if bulkDur > plainDur*5 {
|
||||
t.Fatalf("fast path far slower: plain=%s bulk=%s", plainDur, bulkDur)
|
||||
}
|
||||
if os.Getenv("GORTEX_BULK_PERF_ASSERT") != "" && ratio < 2.0 {
|
||||
t.Fatalf("fast path speedup %.2fx below 2x target", ratio)
|
||||
}
|
||||
}
|
||||
|
||||
// BenchmarkPersistFixture is reproducible persist-speed evidence: run with
|
||||
//
|
||||
// go test -run=^$ -bench=BenchmarkPersistFixture ./internal/graph/store_sqlite/
|
||||
//
|
||||
// to compare the plain AddBatch path against the bulk-load fast path.
|
||||
func BenchmarkPersistFixture(b *testing.B) {
|
||||
nodes, edges := bulkFixture(50000, 100000)
|
||||
|
||||
run := func(b *testing.B, bulk bool) {
|
||||
for i := 0; i < b.N; i++ {
|
||||
b.StopTimer()
|
||||
path := filepath.Join(b.TempDir(), fmt.Sprintf("p%d.sqlite", i))
|
||||
s, err := Open(path)
|
||||
if err != nil {
|
||||
b.Fatalf("open: %v", err)
|
||||
}
|
||||
b.StartTimer()
|
||||
if bulk {
|
||||
s.BeginBulkLoad()
|
||||
s.AddBatch(nodes, edges)
|
||||
if err := s.FlushBulk(); err != nil {
|
||||
b.Fatalf("FlushBulk: %v", err)
|
||||
}
|
||||
} else {
|
||||
s.AddBatch(nodes, edges)
|
||||
}
|
||||
b.StopTimer()
|
||||
_ = s.Close()
|
||||
}
|
||||
}
|
||||
|
||||
b.Run("nonbulk", func(b *testing.B) { run(b, false) })
|
||||
b.Run("bulk", func(b *testing.B) { run(b, true) })
|
||||
}
|
||||
@@ -0,0 +1,320 @@
|
||||
package store_sqlite
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
|
||||
"github.com/zzet/gortex/internal/graph"
|
||||
)
|
||||
|
||||
// bundleCacheDefaultMaxBytes bounds the total heap the bundle cache may
|
||||
// retain across all cached entries. A count ceiling alone is unsafe: an
|
||||
// entry holds a decoded node plus its full in/out edge lists, and both
|
||||
// nodes and edges carry meta maps, so entry sizes span ~1 KB for a leaf
|
||||
// symbol to multiple MB for a hub node with thousands of edges. A cap
|
||||
// measured in entries therefore admits an unbounded BYTE footprint — a
|
||||
// few thousand hub bundles can pin gigabytes. This cache serves point
|
||||
// lookups on the symbol-search hot path; it is a latency optimisation,
|
||||
// not a working set that needs to be resident, so a modest budget is
|
||||
// right — 64 MiB holds the hot few thousand ordinary bundles while
|
||||
// keeping a long-lived daemon's idle heap bounded. Override with
|
||||
// GORTEX_BUNDLE_CACHE_MAX_MB=<n> (n <= 0 disables the cache entirely).
|
||||
const bundleCacheDefaultMaxBytes = 64 << 20 // 64 MiB
|
||||
|
||||
// bundleCacheMaxEntries is a secondary, generous count ceiling kept
|
||||
// alongside the byte budget. The byte budget is the primary bound; this
|
||||
// guards the map's own structural overhead in the degenerate case of a
|
||||
// flood of tiny entries (a bucket slot and pointers per entry are not
|
||||
// fully reflected in a per-entry byte estimate), and keeps the
|
||||
// wholesale-clear allocation predictable. It is deliberately loose: a
|
||||
// half-million-symbol monorepo's hottest few thousand search hits fit
|
||||
// far under it, so in normal operation the byte budget always trips
|
||||
// first.
|
||||
const bundleCacheMaxEntries = 50000
|
||||
|
||||
const (
|
||||
// bundleEntryOverhead is a coarse fixed charge per cached entry that
|
||||
// is independent of the bundle's string content: the bundleCacheEntry
|
||||
// wrapper, the *entry and *Node pointers, the graph.Node value's flat
|
||||
// struct (its string / slice / map headers, ints, and embedded
|
||||
// time.Time), and the map bucket the node id occupies. String and map
|
||||
// *contents* are added on top. Over-estimating here only makes the
|
||||
// cache clear sooner; it never lets the footprint overshoot the budget.
|
||||
bundleEntryOverhead = 448
|
||||
// bundleEdgeOverhead is the coarse fixed charge for one *Edge in an
|
||||
// in/out slice: the pointer, the slice slot, and the Edge value's flat
|
||||
// struct. Edge string / map contents are added separately.
|
||||
bundleEdgeOverhead = 240
|
||||
// bundleMetaEntryOverhead is the fixed per-key charge for a
|
||||
// map[string]any entry (bucket slot + interface header); the key
|
||||
// length and any string value length are added on top.
|
||||
bundleMetaEntryOverhead = 48
|
||||
)
|
||||
|
||||
// bundleCacheEntry is one node's cached bundle, tagged with the package
|
||||
// it belongs to and the package fingerprint that was current when the
|
||||
// bundle was computed. The entry is served only while
|
||||
// fingerprints[pkgKey] still equals fp — any change to the package's
|
||||
// content (a node or edge added / removed / reweighted, including a
|
||||
// cross-file edge that lands on this node from elsewhere) moves the
|
||||
// fingerprint and forces a recompute, so a cached bundle can never
|
||||
// carry a stale edge.
|
||||
type bundleCacheEntry struct {
|
||||
pkgKey string
|
||||
fp uint64
|
||||
bundle graph.SymbolBundle
|
||||
// bytes is the entry's estimated retained size, recorded at insert so
|
||||
// the running byte total can be adjusted in O(1) whenever the entry is
|
||||
// dropped (invalidation or a stale read).
|
||||
bytes int64
|
||||
}
|
||||
|
||||
// bundleCache is a content-addressed, package-scoped cache over
|
||||
// SearchSymbolBundles. It is keyed at the node level but validated at
|
||||
// the package level: an entry is fresh exactly when the package's
|
||||
// current fingerprint matches the fingerprint the entry was stored at.
|
||||
//
|
||||
// Correctness rests entirely on the fingerprint discipline: the daemon
|
||||
// hands the cache an authoritative per-package fingerprint map after
|
||||
// every analysis pass (which runs after every incremental reindex and
|
||||
// every edit_file / fsnotify-driven graph mutation). The fingerprints
|
||||
// are edge-aware — they fold every package's nodes AND the edges
|
||||
// touching them — so any mutation that could change a cached bundle's
|
||||
// in/out edges moves the relevant package fingerprint and invalidates
|
||||
// the entry. A package whose fingerprint is unchanged is served from
|
||||
// cache; a package the daemon has never reported a fingerprint for is
|
||||
// always treated as a miss (conservative: never serve an unvalidated
|
||||
// bundle).
|
||||
//
|
||||
// The cache is bounded by bytes (maxBytes), not by entry count, because
|
||||
// entry sizes vary by orders of magnitude with a node's edge fan-out and
|
||||
// meta size. maxEntries is a secondary count ceiling only. When either
|
||||
// bound would be exceeded the cache is cleared wholesale rather than
|
||||
// evicting individually: entries are cheap to recompute (one batched
|
||||
// fetch), and a wholesale clear keeps the bookkeeping O(1) and free of an
|
||||
// LRU's per-entry ordering overhead. maxBytes <= 0 disables the cache —
|
||||
// stores become no-ops and every lookup misses (reads still recompute
|
||||
// live through the caller's fallback path).
|
||||
type bundleCache struct {
|
||||
mu sync.Mutex
|
||||
fingerprints map[string]uint64
|
||||
entries map[string]*bundleCacheEntry
|
||||
maxBytes int64 // byte budget (primary bound); <= 0 disables the cache
|
||||
maxEntries int // count ceiling (secondary bound)
|
||||
curBytes int64 // running sum of entries' estimated bytes
|
||||
}
|
||||
|
||||
// newBundleCache builds an empty cache with the default budgets. The byte
|
||||
// budget is overridable with GORTEX_BUNDLE_CACHE_MAX_MB=<n>; n <= 0
|
||||
// disables the cache. It starts inert (every lookup a miss) until the
|
||||
// daemon supplies fingerprints.
|
||||
func newBundleCache() *bundleCache {
|
||||
return &bundleCache{
|
||||
fingerprints: map[string]uint64{},
|
||||
entries: map[string]*bundleCacheEntry{},
|
||||
maxBytes: bundleCacheMaxBytes(),
|
||||
maxEntries: bundleCacheMaxEntries,
|
||||
}
|
||||
}
|
||||
|
||||
// bundleCacheMaxBytes resolves the byte budget from the environment,
|
||||
// falling back to the default. GORTEX_BUNDLE_CACHE_MAX_MB is read in
|
||||
// mebibytes; a value <= 0 returns 0 to disable the cache, and an
|
||||
// unparseable value is ignored (keeps the default).
|
||||
func bundleCacheMaxBytes() int64 {
|
||||
if v := strings.TrimSpace(os.Getenv("GORTEX_BUNDLE_CACHE_MAX_MB")); v != "" {
|
||||
if n, err := strconv.Atoi(v); err == nil {
|
||||
if n <= 0 {
|
||||
return 0
|
||||
}
|
||||
return int64(n) << 20
|
||||
}
|
||||
}
|
||||
return bundleCacheDefaultMaxBytes
|
||||
}
|
||||
|
||||
// bundleEntryBytes conservatively estimates a bundle's retained heap for
|
||||
// the byte budget: a fixed per-entry charge plus the node's string and
|
||||
// meta contents plus each in/out edge's fixed charge and its string and
|
||||
// meta contents. Computed once at insert so the overflow check is a cheap
|
||||
// scalar comparison.
|
||||
func bundleEntryBytes(b graph.SymbolBundle) int64 {
|
||||
n := int64(bundleEntryOverhead)
|
||||
if b.Node != nil {
|
||||
n += nodeStringBytes(b.Node)
|
||||
n += metaBytes(b.Node.Meta)
|
||||
}
|
||||
for _, e := range b.InEdges {
|
||||
n += edgeBytes(e)
|
||||
}
|
||||
for _, e := range b.OutEdges {
|
||||
n += edgeBytes(e)
|
||||
}
|
||||
return n
|
||||
}
|
||||
|
||||
// nodeStringBytes sums the byte lengths of a node's string fields (its
|
||||
// heap-backed content, on top of the fixed struct overhead counted in
|
||||
// bundleEntryOverhead).
|
||||
func nodeStringBytes(nd *graph.Node) int64 {
|
||||
return int64(len(nd.ID) + len(nd.Name) + len(nd.QualName) + len(nd.FilePath) +
|
||||
len(string(nd.Kind)) + len(nd.Language) + len(nd.RepoPrefix) +
|
||||
len(nd.WorkspaceID) + len(nd.ProjectID) + len(nd.AbsoluteFilePath) +
|
||||
len(nd.Origin))
|
||||
}
|
||||
|
||||
// edgeBytes estimates one edge's retained heap: the fixed per-edge charge
|
||||
// plus its string fields and meta contents.
|
||||
func edgeBytes(e *graph.Edge) int64 {
|
||||
if e == nil {
|
||||
return bundleEdgeOverhead
|
||||
}
|
||||
n := int64(bundleEdgeOverhead)
|
||||
n += int64(len(e.From) + len(e.To) + len(string(e.Kind)) + len(e.FilePath) +
|
||||
len(e.ConfidenceLabel) + len(e.Origin) + len(e.Tier) + len(e.Context) +
|
||||
len(e.ReturnUsage) + len(e.Via) + len(e.Alias))
|
||||
n += metaBytes(e.Meta)
|
||||
return n
|
||||
}
|
||||
|
||||
// metaBytes estimates a meta map's retained heap: a fixed charge per key
|
||||
// plus the key length and, for string values, the value length. Non-string
|
||||
// values fold into the fixed charge — meta values are overwhelmingly short
|
||||
// scalars, and a coarse estimate only over-counts, which is safe.
|
||||
func metaBytes(m map[string]any) int64 {
|
||||
if len(m) == 0 {
|
||||
return 0
|
||||
}
|
||||
var n int64
|
||||
for k, v := range m {
|
||||
n += int64(len(k) + bundleMetaEntryOverhead)
|
||||
if s, ok := v.(string); ok {
|
||||
n += int64(len(s))
|
||||
}
|
||||
}
|
||||
return n
|
||||
}
|
||||
|
||||
// SetBundleFingerprints installs the authoritative per-package
|
||||
// fingerprint map and drops any cached entry whose package fingerprint
|
||||
// has changed (or whose package is no longer reported). This is the
|
||||
// invalidation entry point: the daemon calls it after each analysis
|
||||
// pass with the fresh fingerprints derived from the live graph, so a
|
||||
// reindex that altered a package's nodes or edges retires exactly the
|
||||
// affected bundles while leaving untouched packages cached.
|
||||
//
|
||||
// fps is keyed by package key (the directory the package's files live
|
||||
// in, repo-prefixed in multi-repo because the node file paths are).
|
||||
func (s *Store) SetBundleFingerprints(fps map[string]uint64) {
|
||||
if s.bundles == nil {
|
||||
return
|
||||
}
|
||||
s.bundles.refresh(fps)
|
||||
}
|
||||
|
||||
// refresh swaps in the new fingerprint map and prunes every entry whose
|
||||
// package fingerprint no longer matches, decrementing the running byte
|
||||
// total by each dropped entry's estimated size.
|
||||
func (c *bundleCache) refresh(fps map[string]uint64) {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
if fps == nil {
|
||||
fps = map[string]uint64{}
|
||||
}
|
||||
c.fingerprints = fps
|
||||
for id, e := range c.entries {
|
||||
cur, ok := fps[e.pkgKey]
|
||||
if !ok || cur != e.fp {
|
||||
delete(c.entries, id)
|
||||
c.curBytes -= e.bytes
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// bundlePackageKey derives the package key for a node's file path. It
|
||||
// mirrors the analysis layer's packageKey so the cache and the
|
||||
// daemon-supplied fingerprint map agree on package identity: the
|
||||
// directory the file lives in (repo-prefixed in multi-repo because the
|
||||
// stored file paths are), or "" for a file at the repo root / a node
|
||||
// with no path.
|
||||
func bundlePackageKey(filePath string) string {
|
||||
if filePath == "" {
|
||||
return ""
|
||||
}
|
||||
dir := filepath.Dir(filepath.ToSlash(filePath))
|
||||
if dir == "." {
|
||||
return ""
|
||||
}
|
||||
return dir
|
||||
}
|
||||
|
||||
// lookup returns the cached bundle for id when it is fresh — the entry
|
||||
// exists and its package fingerprint still matches the current one. A
|
||||
// node whose package has no reported fingerprint is never served (ok is
|
||||
// false) so an unvalidated bundle can never escape the cache. A stale
|
||||
// entry is dropped in place and its bytes reclaimed.
|
||||
func (c *bundleCache) lookup(id string) (graph.SymbolBundle, bool) {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
e, ok := c.entries[id]
|
||||
if !ok {
|
||||
return graph.SymbolBundle{}, false
|
||||
}
|
||||
cur, ok := c.fingerprints[e.pkgKey]
|
||||
if !ok || cur != e.fp {
|
||||
// Stale or unvalidated — drop it so a later refresh doesn't
|
||||
// have to, and reclaim its bytes.
|
||||
delete(c.entries, id)
|
||||
c.curBytes -= e.bytes
|
||||
return graph.SymbolBundle{}, false
|
||||
}
|
||||
return e.bundle, true
|
||||
}
|
||||
|
||||
// store records a freshly computed bundle, tagged with its package's
|
||||
// current fingerprint. A node whose package has no reported fingerprint
|
||||
// is NOT cached (it could not be validated on read-back), keeping the
|
||||
// cache conservative. The cache is bounded by bytes: when admitting the
|
||||
// new entry would push the running total over the byte budget (or the
|
||||
// count over the secondary ceiling) the cache is cleared wholesale
|
||||
// before the insert. A single bundle that on its own exceeds the whole
|
||||
// budget — a hub node with thousands of edges, exactly the pathological
|
||||
// case a byte cap exists to keep out of long-lived memory — is refused
|
||||
// outright rather than pinned. With maxBytes <= 0 the cache is disabled
|
||||
// and every store is a no-op.
|
||||
func (c *bundleCache) store(b graph.SymbolBundle) {
|
||||
if b.Node == nil {
|
||||
return
|
||||
}
|
||||
pkgKey := bundlePackageKey(b.Node.FilePath)
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
if c.maxBytes <= 0 {
|
||||
return
|
||||
}
|
||||
fp, ok := c.fingerprints[pkgKey]
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
sz := bundleEntryBytes(b)
|
||||
if sz > c.maxBytes {
|
||||
// One entry larger than the entire budget would blow the bound and
|
||||
// be evicted by the very next insert's wholesale clear anyway.
|
||||
return
|
||||
}
|
||||
if old, ok := c.entries[b.Node.ID]; ok {
|
||||
// Replacing an existing entry — discount its bytes and drop it so
|
||||
// curBytes and the count check track the live set.
|
||||
c.curBytes -= old.bytes
|
||||
delete(c.entries, b.Node.ID)
|
||||
}
|
||||
if len(c.entries) > 0 && (c.curBytes+sz > c.maxBytes || len(c.entries) >= c.maxEntries) {
|
||||
c.entries = make(map[string]*bundleCacheEntry)
|
||||
c.curBytes = 0
|
||||
}
|
||||
c.entries[b.Node.ID] = &bundleCacheEntry{pkgKey: pkgKey, fp: fp, bundle: b, bytes: sz}
|
||||
c.curBytes += sz
|
||||
}
|
||||
@@ -0,0 +1,390 @@
|
||||
package store_sqlite
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"path/filepath"
|
||||
"sync"
|
||||
"testing"
|
||||
|
||||
"github.com/zzet/gortex/internal/graph"
|
||||
)
|
||||
|
||||
func mkFnNode(id, name, file string) *graph.Node {
|
||||
return &graph.Node{ID: id, Kind: graph.KindFunction, Name: name, FilePath: file, Language: "go"}
|
||||
}
|
||||
|
||||
// newTestBundleCache builds a cache with the default byte budget without
|
||||
// consulting the environment, so the fingerprint / invalidation unit tests
|
||||
// stay hermetic regardless of GORTEX_BUNDLE_CACHE_MAX_MB.
|
||||
func newTestBundleCache() *bundleCache {
|
||||
return &bundleCache{
|
||||
fingerprints: map[string]uint64{},
|
||||
entries: map[string]*bundleCacheEntry{},
|
||||
maxBytes: bundleCacheDefaultMaxBytes,
|
||||
maxEntries: bundleCacheMaxEntries,
|
||||
}
|
||||
}
|
||||
|
||||
// --- unit tests over the cache logic in isolation ---
|
||||
|
||||
func TestBundleCache_ServesOnlyValidatedFingerprints(t *testing.T) {
|
||||
c := newTestBundleCache()
|
||||
|
||||
b := graph.SymbolBundle{Node: mkFnNode("pkg/x.go::A", "A", "pkg/x.go")}
|
||||
|
||||
// No fingerprint reported for the package yet -> store is a no-op
|
||||
// (conservative: never cache an unvalidated bundle).
|
||||
c.store(b)
|
||||
if _, ok := c.lookup("pkg/x.go::A"); ok {
|
||||
t.Fatal("bundle was cached despite no package fingerprint")
|
||||
}
|
||||
|
||||
// Report a fingerprint, then store: now it caches and serves.
|
||||
c.refresh(map[string]uint64{"pkg": 100})
|
||||
c.store(b)
|
||||
if _, ok := c.lookup("pkg/x.go::A"); !ok {
|
||||
t.Fatal("bundle should be served once its package fingerprint is known")
|
||||
}
|
||||
}
|
||||
|
||||
func TestBundleCache_InvalidatesOnFingerprintChange(t *testing.T) {
|
||||
c := newTestBundleCache()
|
||||
c.refresh(map[string]uint64{"pkg": 1})
|
||||
c.store(graph.SymbolBundle{Node: mkFnNode("pkg/x.go::A", "A", "pkg/x.go")})
|
||||
|
||||
if _, ok := c.lookup("pkg/x.go::A"); !ok {
|
||||
t.Fatal("expected a cache hit on the unchanged fingerprint")
|
||||
}
|
||||
|
||||
// Fingerprint changes -> the entry is invalidated.
|
||||
c.refresh(map[string]uint64{"pkg": 2})
|
||||
if _, ok := c.lookup("pkg/x.go::A"); ok {
|
||||
t.Fatal("entry must be dropped when its package fingerprint changes")
|
||||
}
|
||||
}
|
||||
|
||||
func TestBundleCache_CrossRepoIsolation(t *testing.T) {
|
||||
c := newTestBundleCache()
|
||||
// Two repos with the same inner directory name resolve to DIFFERENT
|
||||
// package keys because the stored file paths are repo-prefixed.
|
||||
c.refresh(map[string]uint64{
|
||||
"repoA/pkg": 10,
|
||||
"repoB/pkg": 20,
|
||||
})
|
||||
c.store(graph.SymbolBundle{Node: mkFnNode("repoA/pkg/x.go::A", "A", "repoA/pkg/x.go")})
|
||||
c.store(graph.SymbolBundle{Node: mkFnNode("repoB/pkg/x.go::A", "A", "repoB/pkg/x.go")})
|
||||
|
||||
// Bumping only repoA's fingerprint must not touch repoB's entry.
|
||||
c.refresh(map[string]uint64{
|
||||
"repoA/pkg": 11,
|
||||
"repoB/pkg": 20,
|
||||
})
|
||||
if _, ok := c.lookup("repoA/pkg/x.go::A"); ok {
|
||||
t.Fatal("repoA entry should have been invalidated")
|
||||
}
|
||||
if _, ok := c.lookup("repoB/pkg/x.go::A"); !ok {
|
||||
t.Fatal("repoB entry must survive a repoA-only fingerprint bump")
|
||||
}
|
||||
}
|
||||
|
||||
func TestBundlePackageKey(t *testing.T) {
|
||||
cases := map[string]string{
|
||||
"pkg/sub/x.go": "pkg/sub",
|
||||
"x.go": "",
|
||||
"": "",
|
||||
"repo/a/b.go": "repo/a",
|
||||
}
|
||||
for in, want := range cases {
|
||||
if got := bundlePackageKey(in); got != want {
|
||||
t.Errorf("bundlePackageKey(%q) = %q, want %q", in, got, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// --- integration tests through the store's SearchSymbolBundles ---
|
||||
|
||||
func newBundleTestStore(t *testing.T) *Store {
|
||||
t.Helper()
|
||||
s, err := Open(filepath.Join(t.TempDir(), "b.sqlite"))
|
||||
if err != nil {
|
||||
t.Fatalf("Open: %v", err)
|
||||
}
|
||||
t.Cleanup(func() { _ = s.Close() })
|
||||
return s
|
||||
}
|
||||
|
||||
func seedBundleStore(t *testing.T, s *Store) {
|
||||
t.Helper()
|
||||
s.AddNode(mkFnNode("pkg/x.go::A", "AlphaWidget", "pkg/x.go"))
|
||||
s.AddNode(mkFnNode("pkg/x.go::B", "BetaWidget", "pkg/x.go"))
|
||||
s.AddEdge(&graph.Edge{From: "pkg/x.go::A", To: "pkg/x.go::B", Kind: graph.EdgeCalls, FilePath: "pkg/x.go"})
|
||||
|
||||
items := []graph.SymbolFTSItem{
|
||||
{NodeID: "pkg/x.go::A", Tokens: "alpha widget"},
|
||||
{NodeID: "pkg/x.go::B", Tokens: "beta widget"},
|
||||
}
|
||||
if err := s.BulkUpsertSymbolFTS("", items); err != nil {
|
||||
t.Fatalf("BulkUpsertSymbolFTS: %v", err)
|
||||
}
|
||||
if err := s.BuildSymbolIndex(); err != nil {
|
||||
t.Fatalf("BuildSymbolIndex: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func bundleByID(bundles []graph.SymbolBundle) map[string]graph.SymbolBundle {
|
||||
out := make(map[string]graph.SymbolBundle, len(bundles))
|
||||
for _, b := range bundles {
|
||||
if b.Node != nil {
|
||||
out[b.Node.ID] = b
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func TestSearchSymbolBundles_CacheHitOnUnchangedFingerprint(t *testing.T) {
|
||||
s := newBundleTestStore(t)
|
||||
seedBundleStore(t, s)
|
||||
|
||||
// Report a fingerprint so the first query populates the cache.
|
||||
s.SetBundleFingerprints(map[string]uint64{"pkg": 1})
|
||||
|
||||
first, err := s.SearchSymbolBundles("widget", 10)
|
||||
if err != nil {
|
||||
t.Fatalf("first SearchSymbolBundles: %v", err)
|
||||
}
|
||||
got := bundleByID(first)
|
||||
if b, ok := got["pkg/x.go::A"]; !ok || len(b.OutEdges) != 1 {
|
||||
t.Fatalf("expected A with 1 out-edge on first query, got %+v", got["pkg/x.go::A"])
|
||||
}
|
||||
|
||||
// Mutate the graph WITHOUT bumping the fingerprint: add a second
|
||||
// out-edge from A. A correct content-addressed cache serves the
|
||||
// STALE (1-edge) bundle because the fingerprint is unchanged — proof
|
||||
// the bundle came from cache, not a fresh fetch.
|
||||
s.AddNode(mkFnNode("pkg/x.go::C", "GammaWidget", "pkg/x.go"))
|
||||
s.AddEdge(&graph.Edge{From: "pkg/x.go::A", To: "pkg/x.go::C", Kind: graph.EdgeCalls, FilePath: "pkg/x.go"})
|
||||
|
||||
second, err := s.SearchSymbolBundles("widget", 10)
|
||||
if err != nil {
|
||||
t.Fatalf("second SearchSymbolBundles: %v", err)
|
||||
}
|
||||
cached := bundleByID(second)["pkg/x.go::A"]
|
||||
if len(cached.OutEdges) != 1 {
|
||||
t.Fatalf("expected the cached 1-edge bundle to be served on an unchanged fingerprint, got %d edges",
|
||||
len(cached.OutEdges))
|
||||
}
|
||||
}
|
||||
|
||||
func TestSearchSymbolBundles_MissAndRecomputeOnFingerprintChange(t *testing.T) {
|
||||
s := newBundleTestStore(t)
|
||||
seedBundleStore(t, s)
|
||||
s.SetBundleFingerprints(map[string]uint64{"pkg": 1})
|
||||
|
||||
if _, err := s.SearchSymbolBundles("widget", 10); err != nil {
|
||||
t.Fatalf("warm-up query: %v", err)
|
||||
}
|
||||
|
||||
// Add a real out-edge, then bump the package fingerprint to signal
|
||||
// the content changed. The next query must recompute and surface the
|
||||
// new edge.
|
||||
s.AddNode(mkFnNode("pkg/x.go::C", "GammaWidget", "pkg/x.go"))
|
||||
s.AddEdge(&graph.Edge{From: "pkg/x.go::A", To: "pkg/x.go::C", Kind: graph.EdgeCalls, FilePath: "pkg/x.go"})
|
||||
s.SetBundleFingerprints(map[string]uint64{"pkg": 2})
|
||||
|
||||
after, err := s.SearchSymbolBundles("widget", 10)
|
||||
if err != nil {
|
||||
t.Fatalf("post-invalidation query: %v", err)
|
||||
}
|
||||
fresh := bundleByID(after)["pkg/x.go::A"]
|
||||
if len(fresh.OutEdges) != 2 {
|
||||
t.Fatalf("expected the recomputed 2-edge bundle after a fingerprint bump, got %d edges",
|
||||
len(fresh.OutEdges))
|
||||
}
|
||||
}
|
||||
|
||||
func TestSearchSymbolBundles_UncachedWithoutFingerprints(t *testing.T) {
|
||||
s := newBundleTestStore(t)
|
||||
seedBundleStore(t, s)
|
||||
// No SetBundleFingerprints call -> the cache stays inert and every
|
||||
// query recomputes live. Adding an edge must show up immediately.
|
||||
first, err := s.SearchSymbolBundles("widget", 10)
|
||||
if err != nil {
|
||||
t.Fatalf("first query: %v", err)
|
||||
}
|
||||
if got := bundleByID(first)["pkg/x.go::A"]; len(got.OutEdges) != 1 {
|
||||
t.Fatalf("expected 1 edge live, got %d", len(got.OutEdges))
|
||||
}
|
||||
|
||||
s.AddNode(mkFnNode("pkg/x.go::C", "GammaWidget", "pkg/x.go"))
|
||||
s.AddEdge(&graph.Edge{From: "pkg/x.go::A", To: "pkg/x.go::C", Kind: graph.EdgeCalls, FilePath: "pkg/x.go"})
|
||||
|
||||
second, err := s.SearchSymbolBundles("widget", 10)
|
||||
if err != nil {
|
||||
t.Fatalf("second query: %v", err)
|
||||
}
|
||||
if got := bundleByID(second)["pkg/x.go::A"]; len(got.OutEdges) != 2 {
|
||||
t.Fatalf("uncached path must reflect the new edge live, got %d", len(got.OutEdges))
|
||||
}
|
||||
}
|
||||
|
||||
// --- byte-budget tests ---
|
||||
|
||||
func TestBundleCache_ByteBudgetEvictionAtBoundary(t *testing.T) {
|
||||
c := newTestBundleCache()
|
||||
c.refresh(map[string]uint64{"pkg": 1})
|
||||
|
||||
// Fixed-width ids so every entry estimates to the same size.
|
||||
mk := func(i int) graph.SymbolBundle {
|
||||
return graph.SymbolBundle{Node: mkFnNode(fmt.Sprintf("pkg/x.go::N%03d", i), "W", "pkg/x.go")}
|
||||
}
|
||||
unit := bundleEntryBytes(mk(0))
|
||||
const k = 4
|
||||
c.maxBytes = unit * k // budget holds exactly k entries
|
||||
|
||||
for i := 0; i < k; i++ {
|
||||
c.store(mk(i))
|
||||
}
|
||||
if len(c.entries) != k {
|
||||
t.Fatalf("expected %d entries filling the budget, got %d", k, len(c.entries))
|
||||
}
|
||||
if c.curBytes != unit*k {
|
||||
t.Fatalf("curBytes = %d, want %d", c.curBytes, unit*k)
|
||||
}
|
||||
|
||||
// One more entry crosses the budget -> wholesale clear, only the newest
|
||||
// survives and the byte total resets to a single unit.
|
||||
c.store(mk(k))
|
||||
if len(c.entries) != 1 {
|
||||
t.Fatalf("crossing the budget must clear wholesale to 1 entry, got %d", len(c.entries))
|
||||
}
|
||||
if c.curBytes != unit {
|
||||
t.Fatalf("curBytes after clear = %d, want %d", c.curBytes, unit)
|
||||
}
|
||||
if _, ok := c.lookup(fmt.Sprintf("pkg/x.go::N%03d", k)); !ok {
|
||||
t.Fatal("the entry that triggered the clear must remain served")
|
||||
}
|
||||
if _, ok := c.lookup("pkg/x.go::N000"); ok {
|
||||
t.Fatal("a pre-clear entry must be gone after the wholesale clear")
|
||||
}
|
||||
}
|
||||
|
||||
func TestBundleCache_RefusesEntryLargerThanBudget(t *testing.T) {
|
||||
c := newTestBundleCache()
|
||||
c.refresh(map[string]uint64{"pkg": 1})
|
||||
b := graph.SymbolBundle{Node: mkFnNode("pkg/x.go::A", "A", "pkg/x.go")}
|
||||
c.maxBytes = bundleEntryBytes(b) - 1 // budget just below a single entry
|
||||
|
||||
c.store(b)
|
||||
if _, ok := c.lookup("pkg/x.go::A"); ok {
|
||||
t.Fatal("an entry larger than the whole budget must not be cached")
|
||||
}
|
||||
if len(c.entries) != 0 || c.curBytes != 0 {
|
||||
t.Fatalf("oversized store must leave the cache empty, got %d entries / %d bytes",
|
||||
len(c.entries), c.curBytes)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBundleCacheMaxBytes_EnvOverride(t *testing.T) {
|
||||
t.Setenv("GORTEX_BUNDLE_CACHE_MAX_MB", "128")
|
||||
if got := bundleCacheMaxBytes(); got != 128<<20 {
|
||||
t.Fatalf("env override = %d, want %d", got, 128<<20)
|
||||
}
|
||||
if c := newBundleCache(); c.maxBytes != 128<<20 {
|
||||
t.Fatalf("newBundleCache maxBytes = %d, want %d", c.maxBytes, 128<<20)
|
||||
}
|
||||
|
||||
// Empty and unparseable values keep the default.
|
||||
t.Setenv("GORTEX_BUNDLE_CACHE_MAX_MB", "")
|
||||
if got := bundleCacheMaxBytes(); got != bundleCacheDefaultMaxBytes {
|
||||
t.Fatalf("empty override should keep the default, got %d", got)
|
||||
}
|
||||
t.Setenv("GORTEX_BUNDLE_CACHE_MAX_MB", "not-a-number")
|
||||
if got := bundleCacheMaxBytes(); got != bundleCacheDefaultMaxBytes {
|
||||
t.Fatalf("unparseable override should keep the default, got %d", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBundleCache_DisabledMode(t *testing.T) {
|
||||
t.Setenv("GORTEX_BUNDLE_CACHE_MAX_MB", "0")
|
||||
c := newBundleCache()
|
||||
if c.maxBytes != 0 {
|
||||
t.Fatalf("expected a disabled cache (maxBytes 0), got %d", c.maxBytes)
|
||||
}
|
||||
c.refresh(map[string]uint64{"pkg": 1})
|
||||
c.store(graph.SymbolBundle{Node: mkFnNode("pkg/x.go::A", "A", "pkg/x.go")})
|
||||
if len(c.entries) != 0 {
|
||||
t.Fatalf("a disabled cache must not store, got %d entries", len(c.entries))
|
||||
}
|
||||
if _, ok := c.lookup("pkg/x.go::A"); ok {
|
||||
t.Fatal("a disabled cache must always miss")
|
||||
}
|
||||
|
||||
// A negative budget disables too.
|
||||
t.Setenv("GORTEX_BUNDLE_CACHE_MAX_MB", "-4")
|
||||
if got := bundleCacheMaxBytes(); got != 0 {
|
||||
t.Fatalf("a negative override should disable the cache (0), got %d", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSearchSymbolBundles_DisabledCacheStillServes(t *testing.T) {
|
||||
t.Setenv("GORTEX_BUNDLE_CACHE_MAX_MB", "0")
|
||||
s := newBundleTestStore(t)
|
||||
seedBundleStore(t, s)
|
||||
s.SetBundleFingerprints(map[string]uint64{"pkg": 1})
|
||||
|
||||
res, err := s.SearchSymbolBundles("widget", 10)
|
||||
if err != nil {
|
||||
t.Fatalf("SearchSymbolBundles with the cache disabled: %v", err)
|
||||
}
|
||||
if b, ok := bundleByID(res)["pkg/x.go::A"]; !ok || len(b.OutEdges) != 1 {
|
||||
t.Fatalf("a disabled cache must still return live bundles, got %+v", b)
|
||||
}
|
||||
if s.bundles.maxBytes != 0 {
|
||||
t.Fatalf("expected the store's cache disabled, got maxBytes %d", s.bundles.maxBytes)
|
||||
}
|
||||
if len(s.bundles.entries) != 0 {
|
||||
t.Fatalf("a disabled cache must stay empty, got %d entries", len(s.bundles.entries))
|
||||
}
|
||||
}
|
||||
|
||||
func TestBundleCache_ConcurrentReadInsert(t *testing.T) {
|
||||
c := newTestBundleCache()
|
||||
c.maxBytes = 8 << 10 // small budget so wholesale clears fire under contention
|
||||
c.refresh(map[string]uint64{"pkg": 1})
|
||||
|
||||
const workers = 8
|
||||
const iters = 3000
|
||||
var wg sync.WaitGroup
|
||||
wg.Add(workers)
|
||||
for w := 0; w < workers; w++ {
|
||||
go func(w int) {
|
||||
defer wg.Done()
|
||||
for i := 0; i < iters; i++ {
|
||||
id := fmt.Sprintf("pkg/x.go::N%d_%d", w, i%64)
|
||||
switch i % 3 {
|
||||
case 0:
|
||||
c.store(graph.SymbolBundle{Node: mkFnNode(id, "W", "pkg/x.go")})
|
||||
case 1:
|
||||
_, _ = c.lookup(id)
|
||||
default:
|
||||
c.refresh(map[string]uint64{"pkg": uint64(i)})
|
||||
}
|
||||
}
|
||||
}(w)
|
||||
}
|
||||
wg.Wait()
|
||||
|
||||
// No goroutines remain: the running total must exactly equal the summed
|
||||
// bytes of the surviving entries (the accounting invariant), which also
|
||||
// proves it never drifted negative under contention.
|
||||
var sum int64
|
||||
for _, e := range c.entries {
|
||||
sum += e.bytes
|
||||
}
|
||||
if c.curBytes != sum {
|
||||
t.Fatalf("curBytes %d != sum of live entry bytes %d", c.curBytes, sum)
|
||||
}
|
||||
if c.curBytes > c.maxBytes {
|
||||
t.Fatalf("curBytes %d exceeds the byte budget %d", c.curBytes, c.maxBytes)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,69 @@
|
||||
package store_sqlite_test
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"github.com/zzet/gortex/internal/graph"
|
||||
)
|
||||
|
||||
// TestClosedStoreReadsDoNotPanic pins the teardown-race fix: after Close()
|
||||
// has shut the store (daemon shutdown / restart / store swap), an in-flight
|
||||
// reader — e.g. a deferred parallel-enrich goroutine still holding a cached
|
||||
// *sql.Stmt — must degrade to an empty result, never panic the whole daemon.
|
||||
// Before the fix this surfaced as `panic: store_sqlite: sql: statement is
|
||||
// closed` from GetNode under runDeferredEnrichParallel.
|
||||
func TestClosedStoreReadsDoNotPanic(t *testing.T) {
|
||||
s := openTestStore(t)
|
||||
s.AddNode(&graph.Node{ID: "p/a.go::Foo", Kind: graph.KindType, Name: "Foo", FilePath: "p/a.go"})
|
||||
require.NotNil(t, s.GetNode("p/a.go::Foo"), "sanity: node readable before close")
|
||||
|
||||
require.NoError(t, s.Close())
|
||||
|
||||
assert.NotPanics(t, func() {
|
||||
assert.Nil(t, s.GetNode("p/a.go::Foo"))
|
||||
assert.Empty(t, s.FindNodesByName("Foo"))
|
||||
assert.Empty(t, s.GetFileNodes("p/a.go"))
|
||||
}, "reads after Close must degrade gracefully, not panic")
|
||||
}
|
||||
|
||||
// TestClosedStoreAggregatorsDoNotPanic pins the aggregator teardown-race sweep:
|
||||
// each aggregator read runs its Query error through panicOnFatal, which
|
||||
// swallows the "database is closed" race — leaving rows == nil. Every such site
|
||||
// must early-return its empty value instead of dereferencing nil rows. The live
|
||||
// crash was NodeIDsByKinds (FindHotspots -> RunAnalysis at watch-start) SIGSEGV
|
||||
// on nil rows right after a long warmup. Each method is called with non-empty
|
||||
// args so it actually reaches the Query rather than an early argument guard.
|
||||
func TestClosedStoreAggregatorsDoNotPanic(t *testing.T) {
|
||||
s := openTestStore(t)
|
||||
s.AddNode(&graph.Node{ID: "p/a.go::Foo", Kind: graph.KindType, Name: "Foo", FilePath: "p/a.go"})
|
||||
s.AddNode(&graph.Node{ID: "p/a.go::bar", Kind: graph.KindFunction, Name: "bar", FilePath: "p/a.go"})
|
||||
s.AddEdge(&graph.Edge{From: "p/a.go::bar", To: "p/a.go::Foo", Kind: graph.EdgeReferences, FilePath: "p/a.go", Line: 1})
|
||||
require.NoError(t, s.Close())
|
||||
|
||||
nodeKinds := []graph.NodeKind{graph.KindType, graph.KindFunction}
|
||||
edgeKinds := []graph.EdgeKind{graph.EdgeReferences}
|
||||
ids := []string{"p/a.go::Foo", "p/a.go::bar"}
|
||||
assert.NotPanics(t, func() {
|
||||
assert.Empty(t, s.InEdgeCountsByKind(edgeKinds))
|
||||
assert.Empty(t, s.NodeIDsByKinds(nodeKinds))
|
||||
assert.Empty(t, s.EdgeKindCounts())
|
||||
assert.Empty(t, s.NodeDegreeByKinds(nodeKinds, ""))
|
||||
assert.Empty(t, s.FileImportCounts(nil)) // exercises aggScanImportCounts
|
||||
assert.Empty(t, s.InDegreeForNodes(ids))
|
||||
assert.Empty(t, s.CrossRepoEdgeCounts())
|
||||
assert.Empty(t, s.FileImporters("p/a.go"))
|
||||
assert.Empty(t, s.FileSymbolNamesByPaths([]string{"p/a.go"}, nodeKinds))
|
||||
assert.Empty(t, s.NodeDegreeCounts(ids, edgeKinds))
|
||||
assert.Empty(t, s.NodeFanCounts(ids, edgeKinds, edgeKinds))
|
||||
assert.Empty(t, s.CommunityCrossingsByKind(edgeKinds, map[string]string{"p/a.go::bar": "c0"}))
|
||||
// Iterator-shaped: the Query runs inside the yield closure.
|
||||
n := 0
|
||||
for range s.EdgeAdjacencyForKinds(edgeKinds, nodeKinds) {
|
||||
n++
|
||||
}
|
||||
assert.Zero(t, n)
|
||||
}, "aggregator reads after Close must degrade to empty, not panic")
|
||||
}
|
||||
@@ -0,0 +1,46 @@
|
||||
package store_sqlite
|
||||
|
||||
import (
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"github.com/zzet/gortex/internal/graph"
|
||||
)
|
||||
|
||||
// TestGetRepoNonContentNodes verifies the SQL-level content filter (which
|
||||
// json_extracts data_class out of the JSON meta blob) drops only content
|
||||
// section nodes — keeping code, markdown prose, and data assets — so the
|
||||
// code passes can enumerate without materialising content sections.
|
||||
func TestGetRepoNonContentNodes(t *testing.T) {
|
||||
s, err := Open(filepath.Join(t.TempDir(), "n.sqlite"))
|
||||
require.NoError(t, err)
|
||||
t.Cleanup(func() { _ = s.Close() })
|
||||
|
||||
s.AddBatch([]*graph.Node{
|
||||
{ID: "code1", Kind: graph.KindFunction, Name: "Foo", RepoPrefix: "r"},
|
||||
{ID: "content1", Kind: graph.KindDoc, Name: "doc.txt::0", RepoPrefix: "r",
|
||||
Meta: map[string]any{"data_class": "content", "section_text": "x"}},
|
||||
{ID: "prose1", Kind: graph.KindDoc, Name: "README.md::0", RepoPrefix: "r",
|
||||
Meta: map[string]any{"asset_kind": "markdown_section"}},
|
||||
{ID: "data1", Kind: graph.KindFile, Name: "x.parquet", RepoPrefix: "r",
|
||||
Meta: map[string]any{"data_class": "data"}},
|
||||
}, nil)
|
||||
|
||||
// Runtime assertion the store satisfies the optional capability.
|
||||
var cr graph.NonContentNodeReader = s
|
||||
|
||||
ids := map[string]bool{}
|
||||
for _, n := range cr.GetRepoNonContentNodes("r") {
|
||||
ids[n.ID] = true
|
||||
}
|
||||
require.True(t, ids["code1"], "code node kept")
|
||||
require.True(t, ids["prose1"], "markdown prose kept (not data_class=content)")
|
||||
require.True(t, ids["data1"], "data asset kept (data_class=data, not content)")
|
||||
require.False(t, ids["content1"], "content section dropped at the SQL level")
|
||||
require.Len(t, ids, 3)
|
||||
|
||||
// Empty prefix spans all repos (still drops content).
|
||||
require.Len(t, s.GetRepoNonContentNodes(""), 3)
|
||||
}
|
||||
@@ -0,0 +1,61 @@
|
||||
package store_sqlite_test
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/zzet/gortex/internal/graph"
|
||||
)
|
||||
|
||||
// TestExternalCallCandidateEdges asserts the pushdown selects exactly the
|
||||
// external-package terminals (dep:: / stdlib:: / external::, the
|
||||
// per-repo-prefixed stdlib form, and already-materialised external-call::
|
||||
// nodes) and nothing else — no ordinary resolved call/reference edges,
|
||||
// no non-call edges.
|
||||
func TestExternalCallCandidateEdges(t *testing.T) {
|
||||
s := openTestStore(t)
|
||||
|
||||
add := func(from, to string, kind graph.EdgeKind) {
|
||||
s.AddEdge(&graph.Edge{From: from, To: to, Kind: kind, FilePath: "a.go", Line: 1})
|
||||
}
|
||||
// External terminals — should be selected.
|
||||
add("a.go::f", "dep::github.com/x/y::Z", graph.EdgeCalls)
|
||||
add("a.go::f", "stdlib::fmt::Sprintf", graph.EdgeCalls)
|
||||
add("a.go::f", "myrepo::stdlib::net/http::Get", graph.EdgeCalls) // per-repo-prefixed stdlib form
|
||||
add("a.go::f", "external::svc.internal/api", graph.EdgeReferences)
|
||||
add("a.go::f", "external-call::dep::github.com/a/b", graph.EdgeCalls) // already synthesized
|
||||
// Non-candidates — must NOT be selected.
|
||||
add("a.go::f", "a.go::resolvedCallee", graph.EdgeCalls) // ordinary resolved call
|
||||
add("a.go::f", "unresolved::SomeName", graph.EdgeCalls) // bare unresolved (no import evidence)
|
||||
add("a.go::f", "a.go::SomeType", graph.EdgeImplements) // not a call/ref edge
|
||||
add("a.go::f", "dep::github.com/x/y::Z", graph.EdgeTests) // dep target but wrong kind
|
||||
|
||||
got := map[string]bool{}
|
||||
for _, e := range s.ExternalCallCandidateEdges() {
|
||||
got[e.To] = true
|
||||
}
|
||||
|
||||
want := []string{
|
||||
"dep::github.com/x/y::Z",
|
||||
"stdlib::fmt::Sprintf",
|
||||
"myrepo::stdlib::net/http::Get",
|
||||
"external::svc.internal/api",
|
||||
"external-call::dep::github.com/a/b",
|
||||
}
|
||||
for _, w := range want {
|
||||
if !got[w] {
|
||||
t.Errorf("ExternalCallCandidateEdges missing external terminal %q", w)
|
||||
}
|
||||
}
|
||||
notWant := []string{"a.go::resolvedCallee", "unresolved::SomeName", "a.go::SomeType"}
|
||||
for _, nw := range notWant {
|
||||
if got[nw] {
|
||||
t.Errorf("ExternalCallCandidateEdges wrongly selected non-candidate %q", nw)
|
||||
}
|
||||
}
|
||||
// The EdgeImplements/EdgeTests rows share targets with selected ones
|
||||
// but must not inflate the count via the wrong kind: exactly the 5
|
||||
// distinct external targets above, all from calls/references kinds.
|
||||
if len(got) != len(want) {
|
||||
t.Errorf("selected %d distinct targets, want %d: %v", len(got), len(want), got)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,149 @@
|
||||
package store_sqlite
|
||||
|
||||
import (
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"github.com/zzet/gortex/internal/graph"
|
||||
)
|
||||
|
||||
func ftsRowCount(t *testing.T, s *Store, table, nodeID string) int {
|
||||
t.Helper()
|
||||
var n int
|
||||
q := `SELECT count(*) FROM ` + table + ` WHERE node_id = ?`
|
||||
if err := s.db.QueryRow(q, nodeID).Scan(&n); err != nil {
|
||||
t.Fatalf("count %s: %v", table, err)
|
||||
}
|
||||
return n
|
||||
}
|
||||
|
||||
func ftsHits(t *testing.T, s *Store, query string) int {
|
||||
t.Helper()
|
||||
hits, err := s.SearchSymbols(query, 20)
|
||||
if err != nil {
|
||||
t.Fatalf("SearchSymbols(%q): %v", query, err)
|
||||
}
|
||||
return len(hits)
|
||||
}
|
||||
|
||||
// TestUpsertSymbolFTS_ReplacesWithoutDuplicates is the core correctness
|
||||
// guard for the rowid-map delete: re-upserting a symbol must drop its prior
|
||||
// row (by docid) and leave exactly one FTS row + one map row. A wrong
|
||||
// LastInsertId / stale map would leave the old tokens searchable and the
|
||||
// row count at 2 — so this also proves the FTS5 docid round-trips.
|
||||
func TestUpsertSymbolFTS_ReplacesWithoutDuplicates(t *testing.T) {
|
||||
s, err := Open(filepath.Join(t.TempDir(), "fts.sqlite"))
|
||||
if err != nil {
|
||||
t.Fatalf("Open: %v", err)
|
||||
}
|
||||
t.Cleanup(func() { _ = s.Close() })
|
||||
|
||||
const id = "pkg/x.go::A"
|
||||
s.AddNode(mkFnNode(id, "AlphaWidget", "pkg/x.go"))
|
||||
|
||||
// Upsert three times: first insert, then two replacements that each
|
||||
// exercise the mapped-entry docid delete.
|
||||
for _, tokens := range []string{"alpha widget red", "alpha widget green", "alpha widget blue"} {
|
||||
if err := s.UpsertSymbolFTS(id, tokens); err != nil {
|
||||
t.Fatalf("UpsertSymbolFTS(%q): %v", tokens, err)
|
||||
}
|
||||
}
|
||||
|
||||
if got := ftsRowCount(t, s, "symbol_fts", id); got != 1 {
|
||||
t.Fatalf("symbol_fts rows for %s = %d, want 1 (duplicate row leaked)", id, got)
|
||||
}
|
||||
if got := ftsRowCount(t, s, "symbol_fts_rowid", id); got != 1 {
|
||||
t.Fatalf("symbol_fts_rowid rows for %s = %d, want 1", id, got)
|
||||
}
|
||||
// Only the latest tokens are searchable; the superseded ones are gone.
|
||||
if got := ftsHits(t, s, "blue"); got != 1 {
|
||||
t.Fatalf("search 'blue' (current tokens) = %d hits, want 1", got)
|
||||
}
|
||||
for _, stale := range []string{"red", "green"} {
|
||||
if got := ftsHits(t, s, stale); got != 0 {
|
||||
t.Fatalf("search %q (superseded tokens) = %d hits, want 0 (delete missed)", stale, got)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestBulkUpsertSymbolFTS_MaintainsRowidMap proves the bulk path keeps the
|
||||
// sidecar in lockstep, and that a follow-up incremental upsert on a
|
||||
// bulk-loaded symbol still replaces cleanly (no duplicate).
|
||||
func TestBulkUpsertSymbolFTS_MaintainsRowidMap(t *testing.T) {
|
||||
s, err := Open(filepath.Join(t.TempDir(), "fts.sqlite"))
|
||||
if err != nil {
|
||||
t.Fatalf("Open: %v", err)
|
||||
}
|
||||
t.Cleanup(func() { _ = s.Close() })
|
||||
|
||||
s.AddNode(mkFnNode("pkg/x.go::A", "AlphaWidget", "pkg/x.go"))
|
||||
s.AddNode(mkFnNode("pkg/x.go::B", "BetaWidget", "pkg/x.go"))
|
||||
if err := s.BulkUpsertSymbolFTS("", []graph.SymbolFTSItem{
|
||||
{NodeID: "pkg/x.go::A", Tokens: "alpha widget red"},
|
||||
{NodeID: "pkg/x.go::B", Tokens: "beta widget red"},
|
||||
}); err != nil {
|
||||
t.Fatalf("BulkUpsertSymbolFTS: %v", err)
|
||||
}
|
||||
|
||||
var mapRows int
|
||||
if err := s.db.QueryRow(`SELECT count(*) FROM symbol_fts_rowid`).Scan(&mapRows); err != nil {
|
||||
t.Fatalf("count map: %v", err)
|
||||
}
|
||||
if mapRows != 2 {
|
||||
t.Fatalf("symbol_fts_rowid rows = %d, want 2 after bulk", mapRows)
|
||||
}
|
||||
|
||||
// Incremental replace on a bulk-loaded symbol — must not duplicate.
|
||||
if err := s.UpsertSymbolFTS("pkg/x.go::A", "alpha widget blue"); err != nil {
|
||||
t.Fatalf("UpsertSymbolFTS: %v", err)
|
||||
}
|
||||
if got := ftsRowCount(t, s, "symbol_fts", "pkg/x.go::A"); got != 1 {
|
||||
t.Fatalf("symbol_fts rows after incremental replace = %d, want 1", got)
|
||||
}
|
||||
if got := ftsHits(t, s, "blue"); got != 1 {
|
||||
t.Fatalf("search 'blue' = %d, want 1", got)
|
||||
}
|
||||
}
|
||||
|
||||
// TestBackfillSymbolFTSRowidMap simulates a database built before the
|
||||
// sidecar existed (rows in symbol_fts, none in the map) and proves the
|
||||
// backfill repopulates it so the next incremental upsert replaces cleanly
|
||||
// instead of leaking a duplicate.
|
||||
func TestBackfillSymbolFTSRowidMap(t *testing.T) {
|
||||
s, err := Open(filepath.Join(t.TempDir(), "fts.sqlite"))
|
||||
if err != nil {
|
||||
t.Fatalf("Open: %v", err)
|
||||
}
|
||||
t.Cleanup(func() { _ = s.Close() })
|
||||
|
||||
const id = "pkg/x.go::A"
|
||||
s.AddNode(mkFnNode(id, "AlphaWidget", "pkg/x.go"))
|
||||
|
||||
// Simulate the legacy state: a row in symbol_fts with no map entry.
|
||||
if _, err := s.db.Exec(
|
||||
`INSERT INTO symbol_fts (node_id, repo_prefix, tokens) VALUES (?, '', ?)`,
|
||||
id, "alpha widget red"); err != nil {
|
||||
t.Fatalf("seed legacy fts row: %v", err)
|
||||
}
|
||||
if _, err := s.db.Exec(`DELETE FROM symbol_fts_rowid`); err != nil {
|
||||
t.Fatalf("clear map: %v", err)
|
||||
}
|
||||
|
||||
if err := backfillSymbolFTSRowidMap(s.db); err != nil {
|
||||
t.Fatalf("backfill: %v", err)
|
||||
}
|
||||
if got := ftsRowCount(t, s, "symbol_fts_rowid", id); got != 1 {
|
||||
t.Fatalf("map rows after backfill = %d, want 1", got)
|
||||
}
|
||||
|
||||
// Now an incremental upsert must replace, not duplicate.
|
||||
if err := s.UpsertSymbolFTS(id, "alpha widget blue"); err != nil {
|
||||
t.Fatalf("UpsertSymbolFTS: %v", err)
|
||||
}
|
||||
if got := ftsRowCount(t, s, "symbol_fts", id); got != 1 {
|
||||
t.Fatalf("symbol_fts rows after post-backfill replace = %d, want 1 (dup leaked)", got)
|
||||
}
|
||||
if got := ftsHits(t, s, "red"); got != 0 {
|
||||
t.Fatalf("search 'red' (superseded) = %d, want 0", got)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,76 @@
|
||||
package store_sqlite
|
||||
|
||||
import (
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"github.com/zzet/gortex/internal/graph"
|
||||
)
|
||||
|
||||
// TestGetRepoNodesLight verifies the graph.LightNodeReader fast path
|
||||
// matches GetRepoNodes on IDs and promoted-field values, stays scoped to
|
||||
// repo_prefix, and never surfaces non-promoted meta content — the
|
||||
// invariant the enrichment hover-candidate refetch depends on for
|
||||
// correctness (see EnrichRepoContext's use of repoScopedNodesLight).
|
||||
func TestGetRepoNodesLight(t *testing.T) {
|
||||
s, err := Open(filepath.Join(t.TempDir(), "light.sqlite"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer s.Close()
|
||||
|
||||
s.AddNode(&graph.Node{
|
||||
ID: "repoA/f.go::Stamped", Kind: graph.KindFunction, Name: "Stamped",
|
||||
FilePath: "repoA/f.go", RepoPrefix: "repoA",
|
||||
Meta: map[string]any{
|
||||
"semantic_type": "string",
|
||||
"semantic_source": "lsp-gopls",
|
||||
"doc": "docs",
|
||||
"complexity": 7, // non-promoted
|
||||
},
|
||||
})
|
||||
s.AddNode(&graph.Node{
|
||||
ID: "repoA/f.go::Unstamped", Kind: graph.KindFunction, Name: "Unstamped",
|
||||
FilePath: "repoA/f.go", RepoPrefix: "repoA",
|
||||
})
|
||||
s.AddNode(&graph.Node{
|
||||
ID: "repoB/g.go::Other", Kind: graph.KindFunction, Name: "Other",
|
||||
FilePath: "repoB/g.go", RepoPrefix: "repoB",
|
||||
})
|
||||
|
||||
var _ graph.LightNodeReader = s // compile-time capability check
|
||||
|
||||
full := s.GetRepoNodes("repoA")
|
||||
light := s.GetRepoNodesLight("repoA")
|
||||
if len(light) != len(full) {
|
||||
t.Fatalf("light returned %d nodes, full returned %d", len(light), len(full))
|
||||
}
|
||||
|
||||
byID := make(map[string]*graph.Node, len(light))
|
||||
for _, n := range light {
|
||||
byID[n.ID] = n
|
||||
}
|
||||
|
||||
stamped, ok := byID["repoA/f.go::Stamped"]
|
||||
if !ok {
|
||||
t.Fatal("light scan missing the stamped node")
|
||||
}
|
||||
assertType[string](t, stamped.Meta, "semantic_type", "string")
|
||||
assertType[string](t, stamped.Meta, "semantic_source", "lsp-gopls")
|
||||
assertType[string](t, stamped.Meta, "doc", "docs")
|
||||
if _, ok := stamped.Meta["complexity"]; ok {
|
||||
t.Errorf("light scan must not surface non-promoted meta, got complexity=%v", stamped.Meta["complexity"])
|
||||
}
|
||||
|
||||
unstamped, ok := byID["repoA/f.go::Unstamped"]
|
||||
if !ok {
|
||||
t.Fatal("light scan missing the unstamped node")
|
||||
}
|
||||
if _, ok := unstamped.Meta["semantic_type"]; ok {
|
||||
t.Error("unstamped node must not carry a semantic_type key")
|
||||
}
|
||||
|
||||
if _, ok := byID["repoB/g.go::Other"]; ok {
|
||||
t.Error("light scan crossed repo_prefix scope")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,433 @@
|
||||
package store_sqlite
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/gob"
|
||||
"encoding/json"
|
||||
"path/filepath"
|
||||
"reflect"
|
||||
"testing"
|
||||
|
||||
"github.com/zzet/gortex/internal/contracts"
|
||||
"github.com/zzet/gortex/internal/graph"
|
||||
)
|
||||
|
||||
// flatRoundTrip encodes via the flat codec only (asserting the fast path was
|
||||
// taken) and decodes back through decodeMeta.
|
||||
func flatRoundTrip(t *testing.T, in map[string]any) map[string]any {
|
||||
t.Helper()
|
||||
b, ok := encodeMetaFast(in)
|
||||
if !ok {
|
||||
t.Fatalf("encodeMetaFast bailed on a modelled map: %#v", in)
|
||||
}
|
||||
if !isFlatMeta(b) {
|
||||
t.Fatalf("encodeMetaFast did not stamp the flat magic: %q", b)
|
||||
}
|
||||
out, err := decodeMetaFast(b)
|
||||
if err != nil {
|
||||
t.Fatalf("decodeMetaFast: %v", err)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// TestFlatCodecEveryValueType round-trips one value of every type the codec
|
||||
// models and asserts exact Go-type fidelity end to end.
|
||||
func TestFlatCodecEveryValueType(t *testing.T) {
|
||||
shape := &contracts.Shape{
|
||||
Kind: "struct",
|
||||
Fields: []contracts.ShapeField{{Name: "id", Type: "int64", Required: true}},
|
||||
Notes: []string{"partial"},
|
||||
}
|
||||
in := map[string]any{
|
||||
"str": "hello",
|
||||
"unicode": "héllo – мир – 世界 – 🚀",
|
||||
"bool_t": true,
|
||||
"bool_f": false,
|
||||
"int": -7,
|
||||
"int64": int64(1700000000),
|
||||
"float": 0.875,
|
||||
"float_int": 2.0, // integral float must stay float64
|
||||
"strs": []string{"a", "b", "c"},
|
||||
"nested": map[string]any{"inner": 5, "rate": 1.5, "deep": map[string]any{"x": int64(9)}},
|
||||
"map_slice": []map[string]any{{"k": "v", "n": 1}, {"k": "w", "n": 2}},
|
||||
"any_slice": []any{"x", 3, true, 4.5},
|
||||
"shape": shape,
|
||||
"nilval": nil,
|
||||
"empty_map": map[string]any{},
|
||||
}
|
||||
got := flatRoundTrip(t, in)
|
||||
|
||||
if !reflect.DeepEqual(got, in) {
|
||||
t.Fatalf("flat round-trip mismatch:\n got: %#v\nwant: %#v", got, in)
|
||||
}
|
||||
|
||||
// Spot-check the load-bearing concrete types explicitly.
|
||||
mustType[string](t, got, "str")
|
||||
mustType[bool](t, got, "bool_t")
|
||||
mustType[int](t, got, "int")
|
||||
mustType[int64](t, got, "int64")
|
||||
mustType[float64](t, got, "float")
|
||||
mustType[float64](t, got, "float_int")
|
||||
mustType[[]string](t, got, "strs")
|
||||
mustType[map[string]any](t, got, "nested")
|
||||
mustType[[]map[string]any](t, got, "map_slice")
|
||||
mustType[[]any](t, got, "any_slice")
|
||||
mustType[*contracts.Shape](t, got, "shape")
|
||||
if got["nilval"] != nil {
|
||||
t.Errorf("nilval: want nil, got %#v", got["nilval"])
|
||||
}
|
||||
}
|
||||
|
||||
// TestFlatCodecLargeValues exercises long keys and values that cross the
|
||||
// single-byte varint boundary (> 127 bytes), proving the length prefixes
|
||||
// round-trip.
|
||||
func TestFlatCodecLargeValues(t *testing.T) {
|
||||
big := string(bytes.Repeat([]byte("x"), 5000))
|
||||
bigKey := string(bytes.Repeat([]byte("k"), 300))
|
||||
in := map[string]any{
|
||||
bigKey: big,
|
||||
"slice": []string{big, "", big},
|
||||
}
|
||||
got := flatRoundTrip(t, in)
|
||||
if !reflect.DeepEqual(got, in) {
|
||||
t.Fatalf("large-value round-trip mismatch")
|
||||
}
|
||||
}
|
||||
|
||||
// TestFlatCodecDeterministic proves the encoding is byte-stable across
|
||||
// encodes (keys are sorted), which matters for any content-hash / dedup.
|
||||
func TestFlatCodecDeterministic(t *testing.T) {
|
||||
in := map[string]any{
|
||||
"z": 1, "a": "x", "m": []string{"p", "q"},
|
||||
"nested": map[string]any{"d": 4, "b": 2, "c": 3},
|
||||
}
|
||||
var prev []byte
|
||||
for i := 0; i < 16; i++ {
|
||||
b, ok := encodeMetaFast(in)
|
||||
if !ok {
|
||||
t.Fatalf("encodeMetaFast bailed")
|
||||
}
|
||||
if prev != nil && !bytes.Equal(prev, b) {
|
||||
t.Fatalf("encoding is not deterministic across encodes")
|
||||
}
|
||||
prev = b
|
||||
}
|
||||
}
|
||||
|
||||
// TestEncodeMetaFallbackToJSON: a value whose type the flat codec does not
|
||||
// model makes encodeMeta fall back to JSON (leading '{'), and decodeMeta
|
||||
// still reads it. No data is dropped.
|
||||
func TestEncodeMetaFallbackToJSON(t *testing.T) {
|
||||
// uint64 is deliberately outside the modelled type set.
|
||||
in := map[string]any{"weird": uint64(42), "name": "keep"}
|
||||
|
||||
if _, ok := encodeMetaFast(in); ok {
|
||||
t.Fatal("encodeMetaFast should bail on an unmodelled value type")
|
||||
}
|
||||
|
||||
b, err := encodeMeta(in)
|
||||
if err != nil {
|
||||
t.Fatalf("encodeMeta: %v", err)
|
||||
}
|
||||
if isFlatMeta(b) {
|
||||
t.Fatalf("encodeMeta should have fallen back to JSON, got a flat blob")
|
||||
}
|
||||
if !isJSONObject(b) {
|
||||
t.Fatalf("encodeMeta fallback did not produce a JSON object: %q", b)
|
||||
}
|
||||
got, err := decodeMeta(b)
|
||||
if err != nil {
|
||||
t.Fatalf("decodeMeta(json fallback): %v", err)
|
||||
}
|
||||
// The JSON fallback widens uint64 -> int (documented, lossy only for the
|
||||
// exotic tail), but the string survives and no row is lost.
|
||||
if got["name"] != "keep" {
|
||||
t.Errorf("name not preserved through JSON fallback: %#v", got["name"])
|
||||
}
|
||||
if _, ok := got["weird"]; !ok {
|
||||
t.Errorf("weird key dropped by JSON fallback")
|
||||
}
|
||||
}
|
||||
|
||||
// TestDecodeLegacyJSON proves rows written by the previous JSON encoder still
|
||||
// decode (routed through metaWire for exact types) after the flat-codec
|
||||
// switch — existing on-disk databases must keep loading.
|
||||
func TestDecodeLegacyJSON(t *testing.T) {
|
||||
orig := map[string]any{
|
||||
"visibility": "private",
|
||||
"complexity": 9,
|
||||
"confidence": 1.0, // integral float — metaWire must keep it float64
|
||||
"path_param_names": []string{"id"},
|
||||
"last_authored": map[string]any{"timestamp": int64(1700000000)},
|
||||
}
|
||||
blob, err := json.Marshal(orig)
|
||||
if err != nil {
|
||||
t.Fatalf("json.Marshal: %v", err)
|
||||
}
|
||||
if isFlatMeta(blob) {
|
||||
t.Fatalf("JSON blob unexpectedly looks like a flat blob")
|
||||
}
|
||||
got, err := decodeMeta(blob)
|
||||
if err != nil {
|
||||
t.Fatalf("decodeMeta(json): %v", err)
|
||||
}
|
||||
mustType[string](t, got, "visibility")
|
||||
mustType[int](t, got, "complexity")
|
||||
mustType[float64](t, got, "confidence")
|
||||
mustType[[]string](t, got, "path_param_names")
|
||||
la, ok := got["last_authored"].(map[string]any)
|
||||
if !ok {
|
||||
t.Fatalf("last_authored: want map[string]any, got %T", got["last_authored"])
|
||||
}
|
||||
mustType[int64](t, la, "timestamp")
|
||||
}
|
||||
|
||||
// TestDecodeMetaFastMalformed: corrupt / truncated flat blobs return an error
|
||||
// rather than panicking — a single bad row must not crash a store scan.
|
||||
func TestDecodeMetaFastMalformed(t *testing.T) {
|
||||
good, ok := encodeMetaFast(map[string]any{"k": "value", "n": 7, "s": []string{"a", "b"}})
|
||||
if !ok {
|
||||
t.Fatalf("encodeMetaFast bailed")
|
||||
}
|
||||
cases := map[string][]byte{
|
||||
"magic only": {metaFlatMagic0, metaFlatVersion},
|
||||
"count then nothing": {metaFlatMagic0, metaFlatVersion, 0x05},
|
||||
"truncated mid-blob": good[:len(good)-3],
|
||||
"unknown value tag": {metaFlatMagic0, metaFlatVersion, 0x01, 0x01, 'k', 0x7E},
|
||||
"giant key length": {metaFlatMagic0, metaFlatVersion, 0x01, 0xFF, 0xFF, 0xFF, 0xFF, 0x0F},
|
||||
}
|
||||
for name, blob := range cases {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
_, err := decodeMeta(blob)
|
||||
if err == nil {
|
||||
t.Errorf("expected an error for %q, got nil", name)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestStoreReloadMetaFidelity is the wired-path proof: persist a node and an
|
||||
// edge with rich Meta through the real store, reopen it (warm restart), and
|
||||
// assert Meta is byte-for-byte type-identical. Also runs PRAGMA
|
||||
// integrity_check.
|
||||
func TestStoreReloadMetaFidelity(t *testing.T) {
|
||||
path := filepath.Join(t.TempDir(), "store.sqlite")
|
||||
|
||||
nodeMeta := map[string]any{
|
||||
"complexity": 7,
|
||||
"loop_depth": 2,
|
||||
"confidence": 0.875,
|
||||
"coverage_pct": 1.0, // integral float
|
||||
"candidate_count": 2,
|
||||
"path_param_names": []string{"id", "org"},
|
||||
"status_codes": []string{"200", "404"},
|
||||
"churn": map[string]any{"commit_count": 12, "churn_rate": 2.0, "last_author": "a@b.c"},
|
||||
"last_authored": map[string]any{"timestamp": int64(1700000000), "email": "x@y.z"},
|
||||
"response_envelope": []map[string]any{{"name": "data", "n": 1}},
|
||||
"shape": &contracts.Shape{
|
||||
Kind: "struct",
|
||||
Fields: []contracts.ShapeField{{Name: "id", Type: "int64", Required: true}},
|
||||
Notes: []string{"partial"},
|
||||
},
|
||||
"unicode_doc": "héllo 世界 🚀",
|
||||
"is_generated": false,
|
||||
}
|
||||
edgeMeta := map[string]any{
|
||||
"candidate_count": 3,
|
||||
"similarity": 0.5,
|
||||
"score": 1.0, // integral float
|
||||
"count": 5,
|
||||
"clone_tokens": 128,
|
||||
"synthesized_by": "grpc",
|
||||
"arg_names": []string{"ctx", "req"},
|
||||
}
|
||||
|
||||
func() {
|
||||
s, err := Open(path)
|
||||
if err != nil {
|
||||
t.Fatalf("Open: %v", err)
|
||||
}
|
||||
defer s.Close()
|
||||
s.AddNode(&graph.Node{ID: "n1", Kind: "function", Name: "Foo", FilePath: "f.go", Meta: cloneMeta(nodeMeta)})
|
||||
s.AddNode(&graph.Node{ID: "n2", Kind: "function", Name: "Bar", FilePath: "f.go"})
|
||||
s.AddEdge(&graph.Edge{From: "n1", To: "n2", Kind: "calls", FilePath: "f.go", Line: 10, Meta: cloneMeta(edgeMeta)})
|
||||
}()
|
||||
|
||||
// Reopen — the warm-restart path that reads every blob back.
|
||||
s, err := Open(path)
|
||||
if err != nil {
|
||||
t.Fatalf("reopen: %v", err)
|
||||
}
|
||||
defer s.Close()
|
||||
|
||||
n := s.GetNode("n1")
|
||||
if n == nil {
|
||||
t.Fatal("GetNode(n1) = nil after reload")
|
||||
}
|
||||
if !reflect.DeepEqual(n.Meta, nodeMeta) {
|
||||
t.Fatalf("node Meta mismatch after reload:\n got: %#v\nwant: %#v", n.Meta, nodeMeta)
|
||||
}
|
||||
|
||||
edges := s.GetOutEdges("n1")
|
||||
var got *graph.Edge
|
||||
for _, e := range edges {
|
||||
if e.To == "n2" && e.Kind == "calls" {
|
||||
got = e
|
||||
break
|
||||
}
|
||||
}
|
||||
if got == nil {
|
||||
t.Fatalf("edge n1->n2 not found after reload (got %d edges)", len(edges))
|
||||
}
|
||||
if !reflect.DeepEqual(got.Meta, edgeMeta) {
|
||||
t.Fatalf("edge Meta mismatch after reload:\n got: %#v\nwant: %#v", got.Meta, edgeMeta)
|
||||
}
|
||||
|
||||
var res string
|
||||
if err := s.db.QueryRow(`PRAGMA integrity_check`).Scan(&res); err != nil {
|
||||
t.Fatalf("integrity_check: %v", err)
|
||||
}
|
||||
if res != "ok" {
|
||||
t.Fatalf("integrity_check = %q, want ok", res)
|
||||
}
|
||||
}
|
||||
|
||||
func cloneMeta(m map[string]any) map[string]any {
|
||||
out := make(map[string]any, len(m))
|
||||
for k, v := range m {
|
||||
out[k] = v
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func mustType[T any](t *testing.T, m map[string]any, key string) {
|
||||
t.Helper()
|
||||
v, ok := m[key]
|
||||
if !ok {
|
||||
t.Errorf("%s: missing from decoded map", key)
|
||||
return
|
||||
}
|
||||
if _, ok := v.(T); !ok {
|
||||
var zero T
|
||||
t.Errorf("%s: want type %T, got %T (value %v)", key, zero, v, v)
|
||||
}
|
||||
}
|
||||
|
||||
// -- benchmarks -----------------------------------------------------------
|
||||
|
||||
var metaSink any
|
||||
|
||||
// benchMetaSample is a representative node/edge meta map restricted to the
|
||||
// types gob auto-registers (scalars + []string), so all three encoders run on
|
||||
// identical input for an apples-to-apples comparison. Shape / nested-map /
|
||||
// map-slice values also ride the flat path (see the round-trip tests); they
|
||||
// are omitted here only because gob refuses unregistered interface types.
|
||||
func benchMetaSample() map[string]any {
|
||||
return map[string]any{
|
||||
"complexity": 7,
|
||||
"loop_depth": 2,
|
||||
"parse_errors": 0,
|
||||
"position": 3,
|
||||
"line": 42,
|
||||
"confidence": 0.875,
|
||||
"coverage_pct": 83.5,
|
||||
"candidate_count": 2,
|
||||
"count": 5,
|
||||
"clone_tokens": 128,
|
||||
"timestamp": int64(1700000000),
|
||||
"path_param_names": []string{"id", "org"},
|
||||
"status_codes": []string{"200", "404"},
|
||||
"signature": "func F(ctx context.Context, x int) (T, error)",
|
||||
"some_plugin_flag": "go_linkname",
|
||||
"is_generated": false,
|
||||
"synthesized_by": "grpc",
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkEncodeMetaGob(b *testing.B) {
|
||||
m := benchMetaSample()
|
||||
b.ReportAllocs()
|
||||
b.ResetTimer()
|
||||
for i := 0; i < b.N; i++ {
|
||||
var buf bytes.Buffer
|
||||
if err := gob.NewEncoder(&buf).Encode(m); err != nil {
|
||||
b.Fatalf("gob encode: %v", err)
|
||||
}
|
||||
metaSink = buf.Bytes()
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkEncodeMetaJSON(b *testing.B) {
|
||||
m := benchMetaSample()
|
||||
b.ReportAllocs()
|
||||
b.ResetTimer()
|
||||
for i := 0; i < b.N; i++ {
|
||||
out, err := json.Marshal(m)
|
||||
if err != nil {
|
||||
b.Fatalf("json marshal: %v", err)
|
||||
}
|
||||
metaSink = out
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkEncodeMetaFlat(b *testing.B) {
|
||||
m := benchMetaSample()
|
||||
b.ReportAllocs()
|
||||
b.ResetTimer()
|
||||
for i := 0; i < b.N; i++ {
|
||||
out, ok := encodeMetaFast(m)
|
||||
if !ok {
|
||||
b.Fatal("encodeMetaFast bailed")
|
||||
}
|
||||
metaSink = out
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkDecodeMetaGob(b *testing.B) {
|
||||
var buf bytes.Buffer
|
||||
if err := gob.NewEncoder(&buf).Encode(benchMetaSample()); err != nil {
|
||||
b.Fatalf("gob encode: %v", err)
|
||||
}
|
||||
blob := buf.Bytes()
|
||||
b.ReportAllocs()
|
||||
b.ResetTimer()
|
||||
for i := 0; i < b.N; i++ {
|
||||
m, err := decodeMeta(blob)
|
||||
if err != nil {
|
||||
b.Fatalf("decodeMeta: %v", err)
|
||||
}
|
||||
metaSink = m
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkDecodeMetaJSON(b *testing.B) {
|
||||
blob, err := json.Marshal(benchMetaSample())
|
||||
if err != nil {
|
||||
b.Fatalf("json marshal: %v", err)
|
||||
}
|
||||
b.ReportAllocs()
|
||||
b.ResetTimer()
|
||||
for i := 0; i < b.N; i++ {
|
||||
m, err := decodeMeta(blob)
|
||||
if err != nil {
|
||||
b.Fatalf("decodeMeta: %v", err)
|
||||
}
|
||||
metaSink = m
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkDecodeMetaFlat(b *testing.B) {
|
||||
blob, ok := encodeMetaFast(benchMetaSample())
|
||||
if !ok {
|
||||
b.Fatal("encodeMetaFast bailed")
|
||||
}
|
||||
b.ReportAllocs()
|
||||
b.ResetTimer()
|
||||
for i := 0; i < b.N; i++ {
|
||||
m, err := decodeMeta(blob)
|
||||
if err != nil {
|
||||
b.Fatalf("decodeMeta: %v", err)
|
||||
}
|
||||
metaSink = m
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,176 @@
|
||||
package store_sqlite
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/gob"
|
||||
"reflect"
|
||||
"testing"
|
||||
|
||||
"github.com/zzet/gortex/internal/contracts"
|
||||
)
|
||||
|
||||
// roundTrip encodes Meta with the flat codec and decodes it back, the
|
||||
// persist->reload path every reader sees after a daemon restart / store
|
||||
// hydration.
|
||||
func roundTrip(t *testing.T, in map[string]any) map[string]any {
|
||||
t.Helper()
|
||||
b, err := encodeMeta(in)
|
||||
if err != nil {
|
||||
t.Fatalf("encodeMeta: %v", err)
|
||||
}
|
||||
if !isFlatMeta(b) {
|
||||
t.Fatalf("encodeMeta did not produce a flat-codec blob: %q", b)
|
||||
}
|
||||
out, err := decodeMeta(b)
|
||||
if err != nil {
|
||||
t.Fatalf("decodeMeta: %v", err)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// TestMetaRoundTripExactTypes is the fidelity canary: every key the audit
|
||||
// found read with a raw type-assertion must survive a JSON round-trip with
|
||||
// its exact Go type, or the corresponding reader silently breaks.
|
||||
func TestMetaRoundTripExactTypes(t *testing.T) {
|
||||
shape := &contracts.Shape{
|
||||
Kind: "struct",
|
||||
Fields: []contracts.ShapeField{{Name: "id", Type: "int64", Required: true}},
|
||||
Notes: []string{"partial"},
|
||||
}
|
||||
node := map[string]any{
|
||||
"signature": "func F(x int) error",
|
||||
"visibility": "public",
|
||||
"doc": "F does a thing.",
|
||||
"external": true,
|
||||
"complexity": 7,
|
||||
"loop_depth": 2,
|
||||
"parse_errors": 0,
|
||||
"position": 3,
|
||||
"line": 42,
|
||||
"confidence": 1.0, // integral float — must stay float64
|
||||
"coverage_pct": 83.5,
|
||||
"shape": shape,
|
||||
"response_envelope": []map[string]any{{"name": "data", "type": "User"}},
|
||||
"path_param_names": []string{"id", "org"},
|
||||
"query_params": []string{"limit"},
|
||||
"status_codes": []string{"200", "404"},
|
||||
"churn": map[string]any{"commit_count": 12, "age_days": 365, "churn_rate": 2.0, "last_author": "a@b.c"},
|
||||
"coverage": map[string]any{"num_stmt": 40, "hit": 33},
|
||||
"last_authored": map[string]any{"timestamp": int64(1700000000), "email": "x@y.z"},
|
||||
"some_plugin_flag": "go_linkname", // Extra tail (string)
|
||||
"is_generated": false, // Extra tail (bool)
|
||||
}
|
||||
got := roundTrip(t, node)
|
||||
|
||||
assertType[int](t, got, "complexity", 7)
|
||||
assertType[int](t, got, "loop_depth", 2)
|
||||
assertType[int](t, got, "parse_errors", 0)
|
||||
assertType[int](t, got, "position", 3)
|
||||
assertType[int](t, got, "line", 42)
|
||||
assertType[float64](t, got, "confidence", 1.0)
|
||||
assertType[float64](t, got, "coverage_pct", 83.5)
|
||||
assertType[string](t, got, "signature", "func F(x int) error")
|
||||
assertType[string](t, got, "visibility", "public")
|
||||
assertType[bool](t, got, "external", true)
|
||||
assertType[string](t, got, "some_plugin_flag", "go_linkname")
|
||||
assertType[bool](t, got, "is_generated", false)
|
||||
|
||||
// Shape must rebuild as *contracts.Shape, not map[string]any.
|
||||
gotShape, ok := got["shape"].(*contracts.Shape)
|
||||
if !ok {
|
||||
t.Fatalf("shape: want *contracts.Shape, got %T", got["shape"])
|
||||
}
|
||||
if !reflect.DeepEqual(gotShape, shape) {
|
||||
t.Errorf("shape mismatch: %+v vs %+v", gotShape, shape)
|
||||
}
|
||||
|
||||
// response_envelope must be []map[string]any, not []any.
|
||||
if _, ok := got["response_envelope"].([]map[string]any); !ok {
|
||||
t.Errorf("response_envelope: want []map[string]any, got %T", got["response_envelope"])
|
||||
}
|
||||
// []string keys.
|
||||
for _, k := range []string{"path_param_names", "query_params", "status_codes"} {
|
||||
if _, ok := got[k].([]string); !ok {
|
||||
t.Errorf("%s: want []string, got %T", k, got[k])
|
||||
}
|
||||
}
|
||||
|
||||
// Nested map children keep exact types.
|
||||
churn := got["churn"].(map[string]any)
|
||||
assertType[int](t, churn, "commit_count", 12)
|
||||
assertType[int](t, churn, "age_days", 365)
|
||||
assertType[float64](t, churn, "churn_rate", 2.0) // integral float, nested
|
||||
assertType[string](t, churn, "last_author", "a@b.c")
|
||||
cov := got["coverage"].(map[string]any)
|
||||
assertType[int](t, cov, "num_stmt", 40)
|
||||
assertType[int](t, cov, "hit", 33)
|
||||
la := got["last_authored"].(map[string]any)
|
||||
assertType[int64](t, la, "timestamp", int64(1700000000))
|
||||
}
|
||||
|
||||
func TestEdgeMetaRoundTripExactTypes(t *testing.T) {
|
||||
edge := map[string]any{
|
||||
"candidate_count": 2,
|
||||
"similarity": 0.875,
|
||||
"score": 1.0, // integral float — must stay float64
|
||||
"count": 5,
|
||||
"clone_tokens": 128,
|
||||
"synthesized_by": "grpc", // Extra tail
|
||||
}
|
||||
got := roundTrip(t, edge)
|
||||
assertType[int](t, got, "candidate_count", 2)
|
||||
assertType[float64](t, got, "similarity", 0.875)
|
||||
assertType[float64](t, got, "score", 1.0)
|
||||
assertType[int](t, got, "count", 5)
|
||||
assertType[int](t, got, "clone_tokens", 128)
|
||||
assertType[string](t, got, "synthesized_by", "grpc")
|
||||
}
|
||||
|
||||
// TestDecodeLegacyGob proves existing on-disk gob blobs still decode.
|
||||
func TestDecodeLegacyGob(t *testing.T) {
|
||||
orig := map[string]any{"visibility": "private", "complexity": 9, "confidence": 1.0}
|
||||
var buf bytes.Buffer
|
||||
if err := gob.NewEncoder(&buf).Encode(orig); err != nil {
|
||||
t.Fatalf("gob encode: %v", err)
|
||||
}
|
||||
got, err := decodeMeta(buf.Bytes())
|
||||
if err != nil {
|
||||
t.Fatalf("decodeMeta(gob): %v", err)
|
||||
}
|
||||
// gob preserves exact types natively.
|
||||
assertType[string](t, got, "visibility", "private")
|
||||
assertType[int](t, got, "complexity", 9)
|
||||
assertType[float64](t, got, "confidence", 1.0)
|
||||
}
|
||||
|
||||
func TestEncodeMetaEmpty(t *testing.T) {
|
||||
b, err := encodeMeta(nil)
|
||||
if err != nil || b != nil {
|
||||
t.Fatalf("encodeMeta(nil) = %q, %v; want nil, nil", b, err)
|
||||
}
|
||||
b, err = encodeMeta(map[string]any{})
|
||||
if err != nil || b != nil {
|
||||
t.Fatalf("encodeMeta(empty) = %q, %v; want nil, nil", b, err)
|
||||
}
|
||||
m, err := decodeMeta(nil)
|
||||
if err != nil || m != nil {
|
||||
t.Fatalf("decodeMeta(nil) = %v, %v; want nil, nil", m, err)
|
||||
}
|
||||
}
|
||||
|
||||
func assertType[T comparable](t *testing.T, m map[string]any, key string, want T) {
|
||||
t.Helper()
|
||||
v, ok := m[key]
|
||||
if !ok {
|
||||
t.Errorf("%s: missing from decoded map", key)
|
||||
return
|
||||
}
|
||||
got, ok := v.(T)
|
||||
if !ok {
|
||||
t.Errorf("%s: want type %T, got %T (value %v)", key, want, v, v)
|
||||
return
|
||||
}
|
||||
if got != want {
|
||||
t.Errorf("%s: want %v, got %v", key, want, got)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,252 @@
|
||||
package store_sqlite
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/zzet/gortex/internal/graph"
|
||||
)
|
||||
|
||||
// TestPromotedColumns_RoundTrip verifies the promoted keys land in their
|
||||
// columns, are stripped from the JSON blob, and restore into Meta with
|
||||
// exact types — while non-promoted keys stay in the blob.
|
||||
func TestPromotedColumns_RoundTrip(t *testing.T) {
|
||||
s, err := Open(filepath.Join(t.TempDir(), "p.sqlite"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer s.Close()
|
||||
|
||||
s.AddNode(&graph.Node{
|
||||
ID: "f.go::F", Kind: graph.KindFunction, Name: "F", FilePath: "f.go",
|
||||
Meta: map[string]any{
|
||||
"signature": "func F()",
|
||||
"visibility": "public",
|
||||
"doc": "F docs",
|
||||
"external": true,
|
||||
"complexity": 5, // non-promoted — must stay in the blob
|
||||
},
|
||||
})
|
||||
|
||||
n := s.GetNode("f.go::F")
|
||||
if n == nil {
|
||||
t.Fatal("GetNode returned nil")
|
||||
}
|
||||
assertType[string](t, n.Meta, "signature", "func F()")
|
||||
assertType[string](t, n.Meta, "visibility", "public")
|
||||
assertType[string](t, n.Meta, "doc", "F docs")
|
||||
assertType[bool](t, n.Meta, "external", true)
|
||||
assertType[int](t, n.Meta, "complexity", 5)
|
||||
|
||||
var sig, vis, doc sql.NullString
|
||||
var ext sql.NullBool
|
||||
var blob []byte
|
||||
row := s.db.QueryRow(`SELECT signature, visibility, doc, external, meta FROM nodes WHERE id=?`, "f.go::F")
|
||||
if err := row.Scan(&sig, &vis, &doc, &ext, &blob); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !sig.Valid || sig.String != "func F()" {
|
||||
t.Errorf("signature column = %+v", sig)
|
||||
}
|
||||
if !ext.Valid || !ext.Bool {
|
||||
t.Errorf("external column = %+v", ext)
|
||||
}
|
||||
blobStr := string(blob)
|
||||
for _, k := range []string{"signature", "visibility", "external"} {
|
||||
if strings.Contains(blobStr, k) {
|
||||
t.Errorf("blob still contains promoted key %q: %s", k, blobStr)
|
||||
}
|
||||
}
|
||||
if !strings.Contains(blobStr, "complexity") {
|
||||
t.Errorf("blob missing non-promoted key complexity: %s", blobStr)
|
||||
}
|
||||
}
|
||||
|
||||
// TestPromotedColumns_NewColumns verifies the added promoted meta columns
|
||||
// (is_async / is_static / is_abstract / is_exported / return_type / updated_at)
|
||||
// and the struct-field columns (start_column / end_column) round-trip through
|
||||
// their typed columns, and that a SQL filter on is_async resolves WITHOUT
|
||||
// decoding the meta blob — the indexable-column acceptance.
|
||||
func TestPromotedColumns_NewColumns(t *testing.T) {
|
||||
s, err := Open(filepath.Join(t.TempDir(), "p.sqlite"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer s.Close()
|
||||
|
||||
s.AddNode(&graph.Node{
|
||||
ID: "f.go::Async", Kind: graph.KindFunction, Name: "Async", FilePath: "f.go",
|
||||
StartLine: 10, EndLine: 20, StartColumn: 4, EndColumn: 1,
|
||||
Meta: map[string]any{
|
||||
"is_async": true,
|
||||
"is_static": false,
|
||||
"is_abstract": true,
|
||||
"is_exported": true,
|
||||
"return_type": "error",
|
||||
"updated_at": int64(1700000000),
|
||||
"complexity": 3, // non-promoted — stays in the blob
|
||||
},
|
||||
})
|
||||
// A second, non-async node to prove the filter is selective.
|
||||
s.AddNode(&graph.Node{
|
||||
ID: "f.go::Sync", Kind: graph.KindFunction, Name: "Sync", FilePath: "f.go",
|
||||
Meta: map[string]any{"is_async": false},
|
||||
})
|
||||
|
||||
// Read-back restores every promoted key into Meta with its exact type,
|
||||
// and the struct columns into the Node fields.
|
||||
n := s.GetNode("f.go::Async")
|
||||
if n == nil {
|
||||
t.Fatal("GetNode returned nil")
|
||||
}
|
||||
assertType[bool](t, n.Meta, "is_async", true)
|
||||
assertType[bool](t, n.Meta, "is_static", false)
|
||||
assertType[bool](t, n.Meta, "is_abstract", true)
|
||||
assertType[bool](t, n.Meta, "is_exported", true)
|
||||
assertType[string](t, n.Meta, "return_type", "error")
|
||||
assertType[int64](t, n.Meta, "updated_at", int64(1700000000))
|
||||
assertType[int](t, n.Meta, "complexity", 3)
|
||||
if n.StartColumn != 4 || n.EndColumn != 1 {
|
||||
t.Errorf("column offsets = (%d,%d), want (4,1)", n.StartColumn, n.EndColumn)
|
||||
}
|
||||
|
||||
// The promoted keys are stripped from the JSON blob; complexity is not.
|
||||
var blob []byte
|
||||
if err := s.db.QueryRow(`SELECT meta FROM nodes WHERE id=?`, "f.go::Async").Scan(&blob); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, k := range []string{"is_async", "is_abstract", "return_type", "updated_at"} {
|
||||
if strings.Contains(string(blob), k) {
|
||||
t.Errorf("blob still contains promoted key %q: %s", k, blob)
|
||||
}
|
||||
}
|
||||
if !strings.Contains(string(blob), "complexity") {
|
||||
t.Errorf("blob missing non-promoted key complexity: %s", blob)
|
||||
}
|
||||
|
||||
// Acceptance: a SQL filter on the typed column resolves the node without
|
||||
// touching the meta blob (only id is selected).
|
||||
var id string
|
||||
var startCol, endCol int
|
||||
if err := s.db.QueryRow(
|
||||
`SELECT id, start_column, end_column FROM nodes WHERE is_async = 1`,
|
||||
).Scan(&id, &startCol, &endCol); err != nil {
|
||||
t.Fatalf("is_async column filter failed: %v", err)
|
||||
}
|
||||
if id != "f.go::Async" || startCol != 4 || endCol != 1 {
|
||||
t.Errorf("filter result = (%q,%d,%d), want (f.go::Async,4,1)", id, startCol, endCol)
|
||||
}
|
||||
}
|
||||
|
||||
// TestPromotedColumns_SemanticType verifies semantic_type/semantic_source —
|
||||
// promoted alongside signature/visibility/etc. so enrichment can query the
|
||||
// unstamped subset by column instead of decoding every node's meta blob —
|
||||
// round-trip through their columns and out of the JSON blob.
|
||||
func TestPromotedColumns_SemanticType(t *testing.T) {
|
||||
s, err := Open(filepath.Join(t.TempDir(), "p.sqlite"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer s.Close()
|
||||
|
||||
s.AddNode(&graph.Node{
|
||||
ID: "f.go::T", Kind: graph.KindFunction, Name: "T", FilePath: "f.go",
|
||||
Meta: map[string]any{
|
||||
"semantic_type": "string",
|
||||
"semantic_source": "lsp-gopls",
|
||||
"complexity": 5, // non-promoted — must stay in the blob
|
||||
},
|
||||
})
|
||||
|
||||
n := s.GetNode("f.go::T")
|
||||
if n == nil {
|
||||
t.Fatal("GetNode returned nil")
|
||||
}
|
||||
assertType[string](t, n.Meta, "semantic_type", "string")
|
||||
assertType[string](t, n.Meta, "semantic_source", "lsp-gopls")
|
||||
assertType[int](t, n.Meta, "complexity", 5)
|
||||
|
||||
var st, ss sql.NullString
|
||||
var blob []byte
|
||||
row := s.db.QueryRow(`SELECT semantic_type, semantic_source, meta FROM nodes WHERE id=?`, "f.go::T")
|
||||
if err := row.Scan(&st, &ss, &blob); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !st.Valid || st.String != "string" {
|
||||
t.Errorf("semantic_type column = %+v", st)
|
||||
}
|
||||
if !ss.Valid || ss.String != "lsp-gopls" {
|
||||
t.Errorf("semantic_source column = %+v", ss)
|
||||
}
|
||||
blobStr := string(blob)
|
||||
for _, k := range []string{"semantic_type", "semantic_source"} {
|
||||
if strings.Contains(blobStr, k) {
|
||||
t.Errorf("blob still contains promoted key %q: %s", k, blobStr)
|
||||
}
|
||||
}
|
||||
if !strings.Contains(blobStr, "complexity") {
|
||||
t.Errorf("blob missing non-promoted key complexity: %s", blobStr)
|
||||
}
|
||||
}
|
||||
|
||||
// TestPromotedColumns_ExternalFalse guards the NULL-vs-false distinction:
|
||||
// a stored false must round-trip as false, not vanish.
|
||||
func TestPromotedColumns_ExternalFalse(t *testing.T) {
|
||||
s, err := Open(filepath.Join(t.TempDir(), "p.sqlite"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer s.Close()
|
||||
s.AddNode(&graph.Node{
|
||||
ID: "x", Kind: graph.KindFunction, Name: "x", FilePath: "x.go",
|
||||
Meta: map[string]any{"external": false},
|
||||
})
|
||||
n := s.GetNode("x")
|
||||
if n == nil {
|
||||
t.Fatal("nil")
|
||||
}
|
||||
v, ok := n.Meta["external"].(bool)
|
||||
if !ok || v != false {
|
||||
t.Errorf("external false: got %v (%T)", n.Meta["external"], n.Meta["external"])
|
||||
}
|
||||
}
|
||||
|
||||
// TestPromotedColumns_Migration verifies ensureNodeColumns adds the
|
||||
// promoted columns to a database created with the pre-promotion schema.
|
||||
func TestPromotedColumns_Migration(t *testing.T) {
|
||||
path := filepath.Join(t.TempDir(), "old.sqlite")
|
||||
raw, err := sql.Open("sqlite", path)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
_, err = raw.Exec(`CREATE TABLE nodes (
|
||||
id TEXT PRIMARY KEY, kind TEXT NOT NULL, name TEXT NOT NULL,
|
||||
qual_name TEXT NOT NULL DEFAULT '', file_path TEXT NOT NULL,
|
||||
start_line INTEGER NOT NULL DEFAULT 0, end_line INTEGER NOT NULL DEFAULT 0,
|
||||
language TEXT NOT NULL DEFAULT '', repo_prefix TEXT NOT NULL DEFAULT '',
|
||||
workspace_id TEXT NOT NULL DEFAULT '', project_id TEXT NOT NULL DEFAULT '',
|
||||
meta BLOB
|
||||
) WITHOUT ROWID`)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
_ = raw.Close()
|
||||
|
||||
s, err := Open(path)
|
||||
if err != nil {
|
||||
t.Fatalf("Open old-schema db: %v", err)
|
||||
}
|
||||
defer s.Close()
|
||||
s.AddNode(&graph.Node{
|
||||
ID: "m", Kind: graph.KindFunction, Name: "m", FilePath: "m.go",
|
||||
Meta: map[string]any{"signature": "sig", "external": true},
|
||||
})
|
||||
n := s.GetNode("m")
|
||||
if n == nil {
|
||||
t.Fatal("nil after migration")
|
||||
}
|
||||
assertType[string](t, n.Meta, "signature", "sig")
|
||||
assertType[bool](t, n.Meta, "external", true)
|
||||
}
|
||||
@@ -0,0 +1,45 @@
|
||||
package store_sqlite_test
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"github.com/zzet/gortex/internal/graph"
|
||||
)
|
||||
|
||||
// TestGetOutEdgesLight_SkipsMetaKeepsEndpoints proves the light out-edge
|
||||
// fetch returns the same endpoints/kind/line as GetOutEdges while leaving
|
||||
// Meta nil — it must never pay the per-edge meta JSON decode. This is the
|
||||
// fetch findCallTarget uses on the dataflow hot path.
|
||||
func TestGetOutEdgesLight_SkipsMetaKeepsEndpoints(t *testing.T) {
|
||||
s := openTestStore(t)
|
||||
|
||||
want := &graph.Edge{
|
||||
From: "pkg/x.go::Caller",
|
||||
To: "pkg/x.go::Callee",
|
||||
Kind: graph.EdgeCalls,
|
||||
FilePath: "pkg/x.go",
|
||||
Line: 42,
|
||||
Meta: map[string]any{"call_line": 42, "callee_target": "unresolved::Callee"},
|
||||
}
|
||||
s.AddEdge(want)
|
||||
|
||||
full := s.GetOutEdges("pkg/x.go::Caller")
|
||||
require.Len(t, full, 1)
|
||||
require.NotNil(t, full[0].Meta, "GetOutEdges must decode Meta")
|
||||
assert.Equal(t, "unresolved::Callee", full[0].Meta["callee_target"])
|
||||
|
||||
light := s.GetOutEdgesLight("pkg/x.go::Caller")
|
||||
require.Len(t, light, 1)
|
||||
assert.Equal(t, full[0].From, light[0].From)
|
||||
assert.Equal(t, full[0].To, light[0].To)
|
||||
assert.Equal(t, full[0].Kind, light[0].Kind)
|
||||
assert.Equal(t, full[0].Line, light[0].Line)
|
||||
assert.Equal(t, full[0].FilePath, light[0].FilePath)
|
||||
assert.Nil(t, light[0].Meta, "light fetch must not decode the meta blob")
|
||||
|
||||
// A node with no out-edges returns nothing on both paths.
|
||||
assert.Empty(t, s.GetOutEdgesLight("pkg/x.go::Callee"))
|
||||
}
|
||||
@@ -0,0 +1,58 @@
|
||||
package store_sqlite
|
||||
|
||||
import (
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"github.com/zzet/gortex/internal/graph"
|
||||
)
|
||||
|
||||
// TestProxyNodes_NeverReachDisk validates that cross-daemon proxy-edge
|
||||
// nodes (and their edges) are dropped at the single durable write boundary
|
||||
// (AddNode / AddBatch), so a warm restart over the store never sees them,
|
||||
// while every real node round-trips intact.
|
||||
func TestProxyNodes_NeverReachDisk(t *testing.T) {
|
||||
path := filepath.Join(t.TempDir(), "store.sqlite")
|
||||
s, err := Open(path)
|
||||
if err != nil {
|
||||
t.Fatalf("open: %v", err)
|
||||
}
|
||||
|
||||
real := &graph.Node{ID: "local/a.go::Foo", Kind: graph.KindFunction, Name: "Foo"}
|
||||
proxy := &graph.Node{
|
||||
ID: graph.ProxyNodeID("remoteB", "rb/x.go::Bar"),
|
||||
Kind: graph.KindFunction, Name: "Bar",
|
||||
Origin: "remote:remoteB", Stub: true,
|
||||
}
|
||||
proxyEdge := &graph.Edge{From: real.ID, To: proxy.ID, Kind: graph.EdgeCalls}
|
||||
|
||||
// Mix proxy + real through AddBatch...
|
||||
s.AddBatch([]*graph.Node{real, proxy}, []*graph.Edge{proxyEdge})
|
||||
// ...and a proxy through the per-node path.
|
||||
s.AddNode(&graph.Node{
|
||||
ID: graph.ProxyNodeID("remoteC", "rc/y.go::Baz"),
|
||||
Kind: graph.KindFunction, Name: "Baz",
|
||||
Origin: "remote:remoteC", Stub: true,
|
||||
})
|
||||
_ = s.Close()
|
||||
|
||||
// Reopen — a warm restart sees the durable store only.
|
||||
s2, err := Open(path)
|
||||
if err != nil {
|
||||
t.Fatalf("reopen: %v", err)
|
||||
}
|
||||
defer s2.Close()
|
||||
|
||||
if s2.GetNode(proxy.ID) != nil {
|
||||
t.Error("proxy node must not be persisted")
|
||||
}
|
||||
if s2.GetNode(graph.ProxyNodeID("remoteC", "rc/y.go::Baz")) != nil {
|
||||
t.Error("proxy node added via AddNode must not be persisted")
|
||||
}
|
||||
if s2.GetNode(real.ID) == nil {
|
||||
t.Error("the real node must round-trip intact")
|
||||
}
|
||||
if outs := s2.GetOutEdges(real.ID); len(outs) != 0 {
|
||||
t.Errorf("the edge to a proxy node must not be persisted; got %d", len(outs))
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,74 @@
|
||||
package store_sqlite
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"fmt"
|
||||
"os"
|
||||
|
||||
"github.com/zzet/gortex/internal/graph"
|
||||
)
|
||||
|
||||
// ReadRepoIndexStates opens the SQLite store at path read-only and returns
|
||||
// every repo_index_state freshness row keyed by repo_prefix.
|
||||
//
|
||||
// It is a deliberately lightweight side door for read-only callers (notably
|
||||
// `gortex repos`) that must inspect index freshness WITHOUT going through
|
||||
// Open — which runs schema migrations, alters columns, starts a checkpoint
|
||||
// goroutine, and (on a version mismatch) can refuse to open or rebuild the
|
||||
// file. None of that is appropriate for a status command that may run while
|
||||
// a daemon holds the same store open.
|
||||
//
|
||||
// The connection is query-only and inherits the database's existing journal
|
||||
// mode, so it reads safely alongside a running daemon (WAL permits concurrent
|
||||
// readers). A missing store file, or a database that predates the
|
||||
// repo_index_state table, both yield an empty map and a nil error — that is
|
||||
// "nothing recorded yet", not a failure, so the caller can fall back to other
|
||||
// freshness sources rather than surfacing an error to the user.
|
||||
func ReadRepoIndexStates(path string) (map[string]graph.RepoIndexState, error) {
|
||||
if _, err := os.Stat(path); err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
return map[string]graph.RepoIndexState{}, nil
|
||||
}
|
||||
return nil, fmt.Errorf("stat sqlite store %q: %w", path, err)
|
||||
}
|
||||
|
||||
// query_only blocks accidental writes; busy_timeout keeps a brief read
|
||||
// from erroring out if the daemon happens to hold the write lock for a
|
||||
// moment. We deliberately do NOT set journal_mode — forcing it could try
|
||||
// to switch the live database's mode; inheriting the on-disk WAL mode is
|
||||
// exactly what a concurrent reader wants.
|
||||
dsn := path + "?_pragma=busy_timeout(2000)&_pragma=query_only(1)"
|
||||
db, err := sql.Open("sqlite", dsn)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("open sqlite store %q: %w", path, err)
|
||||
}
|
||||
defer db.Close()
|
||||
db.SetMaxOpenConns(1)
|
||||
|
||||
rows, err := db.Query(`
|
||||
SELECT repo_prefix, indexed_sha, dirty, indexed_at, workspace_fp, node_count, edge_count, extractor_versions
|
||||
FROM repo_index_state`)
|
||||
if err != nil {
|
||||
// A store written before the repo_index_state table existed (or any
|
||||
// other read error) is treated as "no freshness recorded yet" — a
|
||||
// status command must never hard-fail on a degraded cache.
|
||||
return map[string]graph.RepoIndexState{}, nil
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
out := map[string]graph.RepoIndexState{}
|
||||
for rows.Next() {
|
||||
var st graph.RepoIndexState
|
||||
var dirty int
|
||||
if err := rows.Scan(&st.RepoPrefix, &st.IndexedSHA, &dirty, &st.IndexedAt,
|
||||
&st.WorkspaceFP, &st.NodeCount, &st.EdgeCount, &st.ExtractorVersions); err != nil {
|
||||
return nil, fmt.Errorf("scan repo_index_state: %w", err)
|
||||
}
|
||||
st.Dirty = dirty != 0
|
||||
out[st.RepoPrefix] = st
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return nil, fmt.Errorf("iterate repo_index_state: %w", err)
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
@@ -0,0 +1,78 @@
|
||||
package store_sqlite_test
|
||||
|
||||
import (
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"github.com/zzet/gortex/internal/graph"
|
||||
"github.com/zzet/gortex/internal/graph/store_sqlite"
|
||||
)
|
||||
|
||||
// A store file that does not exist yet reads back as an empty map, not an
|
||||
// error — "nothing indexed", so callers fall back to other sources.
|
||||
func TestReadRepoIndexStates_MissingFile(t *testing.T) {
|
||||
got, err := store_sqlite.ReadRepoIndexStates(filepath.Join(t.TempDir(), "absent.sqlite"))
|
||||
require.NoError(t, err)
|
||||
require.Empty(t, got)
|
||||
}
|
||||
|
||||
// The read-only reader returns exactly the rows SetRepoIndexState wrote,
|
||||
// keyed by repo prefix, including the empty (lone-repo) prefix.
|
||||
func TestReadRepoIndexStates_RoundTrip(t *testing.T) {
|
||||
path := filepath.Join(t.TempDir(), "is.sqlite")
|
||||
|
||||
s, err := store_sqlite.Open(path)
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, s.SetRepoIndexState(graph.RepoIndexState{
|
||||
RepoPrefix: "alpha", IndexedSHA: "aaa111", Dirty: false, IndexedAt: 1700000000,
|
||||
WorkspaceFP: "fp-a", NodeCount: 10, EdgeCount: 20, ExtractorVersions: `{"go":1}`,
|
||||
}))
|
||||
require.NoError(t, s.SetRepoIndexState(graph.RepoIndexState{
|
||||
RepoPrefix: "", IndexedSHA: "bbb222", Dirty: true, IndexedAt: 1700000001,
|
||||
}))
|
||||
require.NoError(t, s.Close())
|
||||
|
||||
got, err := store_sqlite.ReadRepoIndexStates(path)
|
||||
require.NoError(t, err)
|
||||
require.Len(t, got, 2)
|
||||
|
||||
alpha, ok := got["alpha"]
|
||||
require.True(t, ok)
|
||||
require.Equal(t, "aaa111", alpha.IndexedSHA)
|
||||
require.False(t, alpha.Dirty)
|
||||
require.EqualValues(t, 1700000000, alpha.IndexedAt)
|
||||
require.Equal(t, 10, alpha.NodeCount)
|
||||
|
||||
lone, ok := got[""]
|
||||
require.True(t, ok)
|
||||
require.Equal(t, "bbb222", lone.IndexedSHA)
|
||||
require.True(t, lone.Dirty)
|
||||
}
|
||||
|
||||
// Reading is non-destructive and repeatable while the underlying store is
|
||||
// reopened for writing — the read path must never lock the writer out.
|
||||
func TestReadRepoIndexStates_ConcurrentWithOpenStore(t *testing.T) {
|
||||
path := filepath.Join(t.TempDir(), "is.sqlite")
|
||||
|
||||
s, err := store_sqlite.Open(path)
|
||||
require.NoError(t, err)
|
||||
t.Cleanup(func() { _ = s.Close() })
|
||||
require.NoError(t, s.SetRepoIndexState(graph.RepoIndexState{
|
||||
RepoPrefix: "live", IndexedSHA: "c0ffee", IndexedAt: 1700000002,
|
||||
}))
|
||||
|
||||
// Read while the writer store is still open (WAL allows concurrent readers).
|
||||
got, err := store_sqlite.ReadRepoIndexStates(path)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, "c0ffee", got["live"].IndexedSHA)
|
||||
|
||||
// A subsequent write is still possible — the reader did not wedge the writer.
|
||||
require.NoError(t, s.SetRepoIndexState(graph.RepoIndexState{
|
||||
RepoPrefix: "live", IndexedSHA: "feedface", IndexedAt: 1700000003,
|
||||
}))
|
||||
got, err = store_sqlite.ReadRepoIndexStates(path)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, "feedface", got["live"].IndexedSHA)
|
||||
}
|
||||
@@ -0,0 +1,518 @@
|
||||
package store_sqlite
|
||||
|
||||
import "database/sql"
|
||||
|
||||
// isUnresolvedColumnDDL is the edges.is_unresolved generated column: a
|
||||
// VIRTUAL, indexed boolean mirroring graph.IsUnresolvedTarget's two shapes
|
||||
// (the bare `unresolved::Name` prefix and the multi-repo COPY-rewrite
|
||||
// `<repoPrefix>::unresolved::Name` infix), computed by SQLite itself from
|
||||
// to_id — no Go call site has to remember to keep it in sync. VIRTUAL, not
|
||||
// STORED: SQLite refuses `ALTER TABLE ADD COLUMN ... STORED` on a non-empty
|
||||
// table ("cannot add a STORED column"), which every real installed store is.
|
||||
// VIRTUAL has no such restriction and is just as fast here — the read path
|
||||
// always goes through the index below, and an index always stores its own
|
||||
// materialised key values regardless of whether the underlying column is
|
||||
// virtual or stored. Added via ensureEdgeColumns (ALTER TABLE) rather than
|
||||
// baked into schemaSQL's CREATE TABLE so one code path handles both a fresh
|
||||
// DB (column missing right after CREATE TABLE) and an existing one (column
|
||||
// missing from before this was introduced) identically — see
|
||||
// ensureNodeColumns for the same pattern on the nodes table.
|
||||
//
|
||||
// Measured on a real 26-repo store (2.57M edges, 847,684 unresolved, ~33%
|
||||
// selectivity): replacing the OR'd `to_id` range/LIKE query with
|
||||
// `is_unresolved = 1` cut EdgesWithUnresolvedTarget from 7.96s to 2.95s
|
||||
// (2.7x). The prior approach of splitting the OR into two to_id-based
|
||||
// queries (one indexed range, one LIKE) was WORSE (13.49s) despite a
|
||||
// better-looking EXPLAIN QUERY PLAN: at ~33% selectivity the to_id index's
|
||||
// matching rows are ordered by string value, so the mandatory per-row
|
||||
// bookmark lookup back into the main table is effectively random I/O. The
|
||||
// boolean column's matching rows are all rowid-tie-broken (identical index
|
||||
// key), so its bookmark lookups land in ascending rowid order — sequential,
|
||||
// not random. Same "SEARCH ... USING INDEX" in EXPLAIN QUERY PLAN either way;
|
||||
// only real measurement told them apart.
|
||||
const isUnresolvedColumnDDL = `is_unresolved INTEGER GENERATED ALWAYS AS (
|
||||
CASE WHEN (to_id >= 'unresolved::' AND to_id < 'unresolved:;') OR to_id LIKE '%::unresolved::%' THEN 1 ELSE 0 END
|
||||
) VIRTUAL`
|
||||
|
||||
// edgeGeneratedColumns is the set of edges.* generated columns ensureEdgeColumns
|
||||
// adds to a table created before they existed — which, since none of them are
|
||||
// in schemaSQL's CREATE TABLE, includes a freshly created table too.
|
||||
var edgeGeneratedColumns = []struct {
|
||||
name string
|
||||
ddl string
|
||||
}{
|
||||
{"is_unresolved", isUnresolvedColumnDDL},
|
||||
}
|
||||
|
||||
// edgePromotedColumns lifts the resolver's resolve_terminal /
|
||||
// resolve_terminal_reason Meta keys (see resolver/terminal.go) out of the
|
||||
// meta blob into their own nullable columns — the edge-side sibling of
|
||||
// promotedMetaColumns on nodes (see meta_json.go's "promoted edge columns"
|
||||
// section for extractPromotedEdgeMeta/restorePromotedEdgeMeta and why a
|
||||
// json_extract-derived generated column was tried first and abandoned:
|
||||
// encodeMeta's common case is a custom flat binary codec, not JSON, so
|
||||
// json_extract/json_valid against a real store's meta blobs evaluates to
|
||||
// NULL for effectively every row). Plain (non-generated) columns, so they
|
||||
// share ensureEdgeColumns' table_xinfo scan but are ordinary ALTER TABLE ADD
|
||||
// COLUMN statements, not GENERATED ALWAYS AS expressions.
|
||||
//
|
||||
// Exists to let a future bulk classification query (replacing per-edge
|
||||
// Go-side classifyTerminal calls in reconcileTerminalStamps) read the
|
||||
// CURRENT terminal state as a plain indexed column instead of decoding
|
||||
// Meta, and compare it against a freshly computed value to find only the
|
||||
// edges whose state actually changed — reconcileTerminalStamps measured
|
||||
// only ~1% of examined edges (9,599 of 833,828) ever change state.
|
||||
var edgePromotedColumns = []struct {
|
||||
name string
|
||||
ddl string
|
||||
}{
|
||||
{"resolve_terminal", "resolve_terminal INTEGER"},
|
||||
{"resolve_terminal_reason", "resolve_terminal_reason TEXT"},
|
||||
}
|
||||
|
||||
// ensureEdgeColumns adds edgeGeneratedColumns + edgePromotedColumns to an
|
||||
// edges table created before they existed. Mirrors ensureNodeColumns'
|
||||
// PRAGMA + conditional ALTER pattern, but queries table_xinfo rather than
|
||||
// table_info: table_info silently OMITS generated columns from its result
|
||||
// set (verified against the pinned modernc.org/sqlite driver — a reopened
|
||||
// store's is_unresolved column is invisible to table_info, so the existence
|
||||
// check always came back false and every reopen re-ran the ALTER, failing
|
||||
// with "duplicate column name"). table_xinfo lists every column, generated
|
||||
// ones included, with an extra hidden column (3 == generated) table_info
|
||||
// doesn't have — and works identically for the plain promoted columns too,
|
||||
// so one scan serves both lists.
|
||||
func ensureEdgeColumns(db *sql.DB) error {
|
||||
rows, err := db.Query(`PRAGMA table_xinfo(edges)`)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
existing := make(map[string]bool)
|
||||
for rows.Next() {
|
||||
var (
|
||||
cid, notnull, pk, hidden int
|
||||
name, ctype string
|
||||
dflt sql.NullString
|
||||
)
|
||||
if err := rows.Scan(&cid, &name, &ctype, ¬null, &dflt, &pk, &hidden); err != nil {
|
||||
_ = rows.Close()
|
||||
return err
|
||||
}
|
||||
existing[name] = true
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
_ = rows.Close()
|
||||
return err
|
||||
}
|
||||
_ = rows.Close()
|
||||
for _, c := range edgeGeneratedColumns {
|
||||
if existing[c.name] {
|
||||
continue
|
||||
}
|
||||
if _, err := db.Exec(`ALTER TABLE edges ADD COLUMN ` + c.ddl); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
for _, c := range edgePromotedColumns {
|
||||
if existing[c.name] {
|
||||
continue
|
||||
}
|
||||
if _, err := db.Exec(`ALTER TABLE edges ADD COLUMN ` + c.ddl); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// isStubColumnDDL is nodes.is_stub: a VIRTUAL generated column mirroring
|
||||
// graph.IsStub/StubKind's id-prefix logic (stdlib:: / builtin:: /
|
||||
// external_call:: / module::, bare or repo-prefixed as <repo>::<kind>::...).
|
||||
// Same rationale as isUnresolvedColumnDDL: computed from the existing id
|
||||
// column, no Go call site has to keep it in sync. Exists so a future
|
||||
// SQL-side terminal classification (see resolveTerminalColumnDDL) can check
|
||||
// "is this candidate a stub" via a plain column instead of a per-row Go
|
||||
// IsStub(n.ID) call.
|
||||
const isStubColumnDDL = `is_stub INTEGER GENERATED ALWAYS AS (
|
||||
CASE WHEN
|
||||
id LIKE 'stdlib::%' OR id LIKE 'builtin::%' OR id LIKE 'external_call::%' OR id LIKE 'module::%'
|
||||
OR (
|
||||
instr(id, '::') > 0 AND (
|
||||
substr(id, instr(id, '::') + 2) LIKE 'stdlib::%'
|
||||
OR substr(id, instr(id, '::') + 2) LIKE 'builtin::%'
|
||||
OR substr(id, instr(id, '::') + 2) LIKE 'external_call::%'
|
||||
OR substr(id, instr(id, '::') + 2) LIKE 'module::%'
|
||||
)
|
||||
)
|
||||
THEN 1 ELSE 0 END
|
||||
) VIRTUAL`
|
||||
|
||||
// nodeGeneratedColumns is the nodes-table sibling of edgeGeneratedColumns.
|
||||
// Kept as its own list (and ensureNodeGeneratedColumns as its own function,
|
||||
// rather than folded into ensureNodeColumns) because ensureNodeColumns
|
||||
// checks existence via PRAGMA table_info, which — like the edges case
|
||||
// documented on ensureEdgeColumns — silently omits generated columns.
|
||||
// Reusing that function's table_info scan for is_stub would hit the exact
|
||||
// same "always looks missing, ALTER re-runs, duplicate column name" bug.
|
||||
var nodeGeneratedColumns = []struct {
|
||||
name string
|
||||
ddl string
|
||||
}{
|
||||
{"is_stub", isStubColumnDDL},
|
||||
}
|
||||
|
||||
// ensureNodeGeneratedColumns adds nodeGeneratedColumns to a nodes table
|
||||
// created before they existed. See ensureEdgeColumns for the table_xinfo
|
||||
// vs table_info rationale this mirrors.
|
||||
func ensureNodeGeneratedColumns(db *sql.DB) error {
|
||||
rows, err := db.Query(`PRAGMA table_xinfo(nodes)`)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
existing := make(map[string]bool)
|
||||
for rows.Next() {
|
||||
var (
|
||||
cid, notnull, pk, hidden int
|
||||
name, ctype string
|
||||
dflt sql.NullString
|
||||
)
|
||||
if err := rows.Scan(&cid, &name, &ctype, ¬null, &dflt, &pk, &hidden); err != nil {
|
||||
_ = rows.Close()
|
||||
return err
|
||||
}
|
||||
existing[name] = true
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
_ = rows.Close()
|
||||
return err
|
||||
}
|
||||
_ = rows.Close()
|
||||
for _, c := range nodeGeneratedColumns {
|
||||
if existing[c.name] {
|
||||
continue
|
||||
}
|
||||
if _, err := db.Exec(`ALTER TABLE nodes ADD COLUMN ` + c.ddl); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// schemaSQL is the canonical DDL applied on Open. Statements are
|
||||
// idempotent (IF NOT EXISTS) so they run cleanly against a fresh DB
|
||||
// and against an existing one.
|
||||
//
|
||||
// Schema choices
|
||||
//
|
||||
// - nodes.id is the primary key; INSERT OR REPLACE on the id column
|
||||
// gives idempotent re-adds with last-write-wins on every other
|
||||
// column, matching the in-memory store's behaviour.
|
||||
//
|
||||
// - edges has a synthetic INTEGER PRIMARY KEY plus a UNIQUE
|
||||
// constraint over (from_id, to_id, kind, file_path, line) -- the
|
||||
// logical edge key the in-memory store uses for dedup. INSERT OR
|
||||
// IGNORE on that constraint matches the in-memory "second AddEdge
|
||||
// for the same key is a no-op" semantics.
|
||||
//
|
||||
// - meta is a JSON document (see meta_json.go). nil / empty Meta is
|
||||
// stored as NULL. Four universal, hot-read node keys are promoted to
|
||||
// their own nullable columns (signature / visibility / doc /
|
||||
// external): they are stripped from the JSON blob on write and
|
||||
// restored into Meta on read, so the in-memory map is unchanged. A
|
||||
// NULL column means "not set" (legacy gob rows predate the columns
|
||||
// and keep their values in the blob). Existing databases gain the
|
||||
// columns via ALTER on the next Open (ensureNodeColumns).
|
||||
//
|
||||
// - Secondary indexes mirror the in-memory store's hot lookup paths:
|
||||
// nodes_by_name -- FindNodesByName / FindNodesByNameInRepo
|
||||
// nodes_by_kind -- Stats (group-by-kind)
|
||||
// nodes_by_file -- GetFileNodes, EvictFile
|
||||
// nodes_by_repo -- GetRepoNodes, RepoStats, EvictRepo
|
||||
// (partial index -- empty repo_prefix is
|
||||
// the common case and indexing it would
|
||||
// be pure overhead)
|
||||
// nodes_by_qual -- GetNodeByQualName, unique so duplicate
|
||||
// qual_names surface as constraint errors
|
||||
// edges_by_from -- GetOutEdges (kind included so RemoveEdge
|
||||
// can probe by (from, kind) without a
|
||||
// second hop)
|
||||
// edges_by_to -- GetInEdges
|
||||
const schemaSQL = `
|
||||
CREATE TABLE IF NOT EXISTS nodes (
|
||||
id TEXT PRIMARY KEY,
|
||||
kind TEXT NOT NULL,
|
||||
name TEXT NOT NULL,
|
||||
qual_name TEXT NOT NULL DEFAULT '',
|
||||
file_path TEXT NOT NULL,
|
||||
start_line INTEGER NOT NULL DEFAULT 0,
|
||||
end_line INTEGER NOT NULL DEFAULT 0,
|
||||
start_column INTEGER NOT NULL DEFAULT 0,
|
||||
end_column INTEGER NOT NULL DEFAULT 0,
|
||||
language TEXT NOT NULL DEFAULT '',
|
||||
repo_prefix TEXT NOT NULL DEFAULT '',
|
||||
workspace_id TEXT NOT NULL DEFAULT '',
|
||||
project_id TEXT NOT NULL DEFAULT '',
|
||||
signature TEXT,
|
||||
visibility TEXT,
|
||||
doc TEXT,
|
||||
external INTEGER,
|
||||
return_type TEXT,
|
||||
is_async INTEGER,
|
||||
is_static INTEGER,
|
||||
is_abstract INTEGER,
|
||||
is_exported INTEGER,
|
||||
updated_at INTEGER,
|
||||
data_class TEXT,
|
||||
meta BLOB
|
||||
) WITHOUT ROWID;
|
||||
|
||||
-- nodes_by_name / _kind / _file / _repo are created from the shared
|
||||
-- bulkDroppableIndexes set (see bulk_load.go), not here, so the bulk-load
|
||||
-- fast path can drop and rebuild the EXACT same DDL without drift.
|
||||
-- nodes_by_qual is UNIQUE — it enforces qual_name dedup on every
|
||||
-- INSERT OR REPLACE, so it is never dropped and stays defined here.
|
||||
CREATE UNIQUE INDEX IF NOT EXISTS nodes_by_qual ON nodes(qual_name) WHERE qual_name <> '';
|
||||
|
||||
CREATE TABLE IF NOT EXISTS edges (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
from_id TEXT NOT NULL,
|
||||
to_id TEXT NOT NULL,
|
||||
kind TEXT NOT NULL,
|
||||
file_path TEXT NOT NULL DEFAULT '',
|
||||
line INTEGER NOT NULL DEFAULT 0,
|
||||
confidence REAL NOT NULL DEFAULT 1.0,
|
||||
confidence_label TEXT NOT NULL DEFAULT '',
|
||||
origin TEXT NOT NULL DEFAULT '',
|
||||
tier TEXT NOT NULL DEFAULT '',
|
||||
cross_repo INTEGER NOT NULL DEFAULT 0,
|
||||
meta BLOB,
|
||||
UNIQUE(from_id, to_id, kind, file_path, line)
|
||||
);
|
||||
|
||||
-- edges_by_from / _to / _kind are created from the shared
|
||||
-- bulkDroppableIndexes set (see bulk_load.go), not here, so the bulk-load
|
||||
-- fast path can drop and rebuild the EXACT same DDL without drift.
|
||||
-- edges_by_kind backs EdgesByKind / EdgesByKinds (resolver whole-graph
|
||||
-- passes probe single kinds like provides/imports on every file save);
|
||||
-- without it those are full edges-table scans — edges_by_from/to lead
|
||||
-- with an id column and the partial edges_external index only covers
|
||||
-- its own predicate.
|
||||
|
||||
CREATE TABLE IF NOT EXISTS file_mtimes (
|
||||
repo_prefix TEXT NOT NULL,
|
||||
file_path TEXT NOT NULL,
|
||||
mtime_ns INTEGER NOT NULL,
|
||||
PRIMARY KEY (repo_prefix, file_path)
|
||||
) WITHOUT ROWID;
|
||||
|
||||
-- repo_index_state records per-repo freshness provenance written at the
|
||||
-- end of a (re)index: the git revision + dirty flag the graph reflects,
|
||||
-- the Merkle workspace fingerprint (Tree.Root) that gates global-pass
|
||||
-- short-circuiting, node/edge counts for the index-plausibility baseline,
|
||||
-- and the JSON per-language extractor versions that produced the graph.
|
||||
-- One row per repo_prefix; WITHOUT ROWID — the PK index IS the table,
|
||||
-- like file_mtimes / clone_shingles.
|
||||
CREATE TABLE IF NOT EXISTS repo_index_state (
|
||||
repo_prefix TEXT PRIMARY KEY,
|
||||
indexed_sha TEXT NOT NULL DEFAULT '',
|
||||
dirty INTEGER NOT NULL DEFAULT 0,
|
||||
indexed_at INTEGER NOT NULL DEFAULT 0,
|
||||
workspace_fp TEXT NOT NULL DEFAULT '',
|
||||
node_count INTEGER NOT NULL DEFAULT 0,
|
||||
edge_count INTEGER NOT NULL DEFAULT 0,
|
||||
extractor_versions TEXT NOT NULL DEFAULT ''
|
||||
) WITHOUT ROWID;
|
||||
|
||||
-- enrichment_state records, per (repo, semantic provider), the git revision
|
||||
-- the graph was enriched at plus the coverage that pass reached. Enrichment
|
||||
-- completion otherwise lives only in an in-memory map, so a restart forgets it
|
||||
-- and re-runs full LSP hover passes for a repo whose persisted graph already
|
||||
-- carries the edges. The deferred-enrichment gate reads this row and skips a
|
||||
-- provider whose IndexedSHA still matches HEAD on a clean tree. One row per
|
||||
-- (repo_prefix, provider); WITHOUT ROWID — the PK index IS the table, like
|
||||
-- file_mtimes / repo_index_state.
|
||||
CREATE TABLE IF NOT EXISTS enrichment_state (
|
||||
repo_prefix TEXT NOT NULL,
|
||||
provider TEXT NOT NULL,
|
||||
indexed_sha TEXT NOT NULL DEFAULT '',
|
||||
completed_at INTEGER NOT NULL DEFAULT 0,
|
||||
coverage REAL NOT NULL DEFAULT 0,
|
||||
PRIMARY KEY (repo_prefix, provider)
|
||||
) WITHOUT ROWID;
|
||||
|
||||
-- clone_shingles is the per-symbol MinHash shingle-set sidecar. Each
|
||||
-- function/method node's []uint64 shingle set is stored as a little-
|
||||
-- endian BLOB (8 bytes/elem) keyed by node_id so the maintained clone-
|
||||
-- detection count-min sketch can be rebuilt after a warm restart from
|
||||
-- the snapshot instead of re-parsing every body. repo_prefix carries
|
||||
-- the owning repo so per-repo reseeds (SELECT … WHERE repo_prefix = ?)
|
||||
-- and per-repo wipes don't clobber other repos' shingle sets. node_id
|
||||
-- is the PK (the join key back to nodes.id); like file_mtimes this is a
|
||||
-- WITHOUT ROWID sidecar so the PK index IS the table.
|
||||
CREATE TABLE IF NOT EXISTS clone_shingles (
|
||||
node_id TEXT PRIMARY KEY,
|
||||
repo_prefix TEXT NOT NULL DEFAULT '',
|
||||
shingles BLOB
|
||||
) WITHOUT ROWID;
|
||||
|
||||
-- constant_values is the per-KindConstant literal-value sidecar: one row
|
||||
-- per constant whose RHS is a string / numeric literal, keyed by node_id
|
||||
-- (the join key back to nodes.id). Lifting the value out of the JSON Meta
|
||||
-- blob keeps it queryable (and out of the every-node-load decode path) so
|
||||
-- the resolver can dereference a const-identifier dispatch name to its
|
||||
-- value across files. file_path scopes per-file eviction on reindex;
|
||||
-- repo_prefix scopes per-repo wipes. WITHOUT ROWID — the PK index IS the
|
||||
-- table, like file_mtimes / clone_shingles.
|
||||
CREATE TABLE IF NOT EXISTS constant_values (
|
||||
node_id TEXT PRIMARY KEY,
|
||||
repo_prefix TEXT NOT NULL DEFAULT '',
|
||||
file_path TEXT NOT NULL DEFAULT '',
|
||||
value TEXT NOT NULL DEFAULT ''
|
||||
) WITHOUT ROWID;
|
||||
|
||||
CREATE INDEX IF NOT EXISTS constant_values_by_file ON constant_values(repo_prefix, file_path);
|
||||
|
||||
-- files is the per-file metadata sidecar: one row per indexed file carrying
|
||||
-- the BLAKE3 content hash (the Merkle leaf), byte size, extracted node count,
|
||||
-- and a JSON array of parse-error locations. The Merkle tree stays the
|
||||
-- authoritative change detector; this table is queryable supplementary
|
||||
-- metadata (index_health reports per-file parse errors + node counts from it).
|
||||
-- PK is (repo_prefix, file_path) so a reindex replaces the row in place;
|
||||
-- WITHOUT ROWID — the PK index IS the table, like file_mtimes.
|
||||
CREATE TABLE IF NOT EXISTS files (
|
||||
repo_prefix TEXT NOT NULL DEFAULT '',
|
||||
file_path TEXT NOT NULL,
|
||||
content_hash TEXT NOT NULL DEFAULT '',
|
||||
size INTEGER NOT NULL DEFAULT 0,
|
||||
node_count INTEGER NOT NULL DEFAULT 0,
|
||||
errors TEXT NOT NULL DEFAULT '',
|
||||
PRIMARY KEY (repo_prefix, file_path)
|
||||
) WITHOUT ROWID;
|
||||
-- files_with_errors backs the index_health "files with parse errors" rollup
|
||||
-- so it scans only the (usually tiny) set of erroring files, not every row.
|
||||
CREATE INDEX IF NOT EXISTS files_with_errors ON files(repo_prefix) WHERE errors <> '';
|
||||
|
||||
-- ref_facts is the resolved-reference sidecar: one row per reference edge
|
||||
-- that resolved to a concrete target, recording the target + the provenance
|
||||
-- tier that resolved it. Denormalized file_path + lang make "all reference
|
||||
-- facts originating in file X" a single indexed query (the scope unit for
|
||||
-- incremental re-resolution and the audit/diff surface). repo_prefix scopes
|
||||
-- per-repo. PK is (repo_prefix, from_id, to_id, kind, line) so re-resolving a
|
||||
-- file replaces its facts in place; WITHOUT ROWID — the PK index IS the table.
|
||||
CREATE TABLE IF NOT EXISTS ref_facts (
|
||||
repo_prefix TEXT NOT NULL DEFAULT '',
|
||||
from_id TEXT NOT NULL,
|
||||
to_id TEXT NOT NULL,
|
||||
kind TEXT NOT NULL,
|
||||
ref_name TEXT NOT NULL DEFAULT '',
|
||||
line INTEGER NOT NULL DEFAULT 0,
|
||||
origin TEXT NOT NULL DEFAULT '',
|
||||
tier TEXT NOT NULL DEFAULT '',
|
||||
candidates TEXT NOT NULL DEFAULT '',
|
||||
file_path TEXT NOT NULL DEFAULT '',
|
||||
lang TEXT NOT NULL DEFAULT '',
|
||||
PRIMARY KEY (repo_prefix, from_id, to_id, kind, line)
|
||||
) WITHOUT ROWID;
|
||||
CREATE INDEX IF NOT EXISTS ref_facts_by_file ON ref_facts(repo_prefix, file_path);
|
||||
-- ref_facts_by_target backs the reverse lookup ("which files hold a fact
|
||||
-- resolving TO these symbols") that affected-by re-resolution runs when a
|
||||
-- file's symbol signatures change. Without it that query is a full
|
||||
-- ref_facts scan — the PK leads with from_id, not to_id.
|
||||
CREATE INDEX IF NOT EXISTS ref_facts_by_target ON ref_facts(repo_prefix, to_id);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS vectors (
|
||||
node_id TEXT PRIMARY KEY,
|
||||
dims INTEGER NOT NULL,
|
||||
vec BLOB NOT NULL
|
||||
) WITHOUT ROWID;
|
||||
|
||||
-- churn_enrichment is the per-node git-churn sidecar (change A: move
|
||||
-- enrichment OUT of nodes.meta so the node hot path stops encoding
|
||||
-- rarely-read data into the blob and get_churn_rate does an indexed read
|
||||
-- instead of an AllNodes+meta-decode scan). One typed row per enriched
|
||||
-- file/function/method node, keyed by node_id (join key back to
|
||||
-- nodes.id); repo_prefix scopes
|
||||
-- per-repo reseeds/wipes. head_sha/branch/computed_at are file-level only
|
||||
-- (empty for symbols). WITHOUT ROWID: the PK index IS the table.
|
||||
CREATE TABLE IF NOT EXISTS churn_enrichment (
|
||||
node_id TEXT PRIMARY KEY,
|
||||
repo_prefix TEXT NOT NULL DEFAULT '',
|
||||
commit_count INTEGER NOT NULL DEFAULT 0,
|
||||
age_days INTEGER NOT NULL DEFAULT 0,
|
||||
churn_rate REAL NOT NULL DEFAULT 0,
|
||||
last_author TEXT NOT NULL DEFAULT '',
|
||||
last_commit_at TEXT NOT NULL DEFAULT '',
|
||||
head_sha TEXT NOT NULL DEFAULT '',
|
||||
branch TEXT NOT NULL DEFAULT '',
|
||||
computed_at TEXT NOT NULL DEFAULT ''
|
||||
) WITHOUT ROWID;
|
||||
CREATE INDEX IF NOT EXISTS churn_by_repo ON churn_enrichment(repo_prefix) WHERE repo_prefix <> '';
|
||||
|
||||
-- coverage_enrichment: per-symbol coverage sidecar (change A). Typed
|
||||
-- columns keyed by node_id; repo_prefix scopes per-repo wipes.
|
||||
CREATE TABLE IF NOT EXISTS coverage_enrichment (
|
||||
node_id TEXT PRIMARY KEY,
|
||||
repo_prefix TEXT NOT NULL DEFAULT '',
|
||||
coverage_pct REAL NOT NULL DEFAULT 0,
|
||||
num_stmt INTEGER NOT NULL DEFAULT 0,
|
||||
hit INTEGER NOT NULL DEFAULT 0
|
||||
) WITHOUT ROWID;
|
||||
CREATE INDEX IF NOT EXISTS coverage_by_repo ON coverage_enrichment(repo_prefix) WHERE repo_prefix <> '';
|
||||
|
||||
-- release_enrichment: per-file "added_in <tag>" sidecar (change A).
|
||||
CREATE TABLE IF NOT EXISTS release_enrichment (
|
||||
node_id TEXT PRIMARY KEY,
|
||||
repo_prefix TEXT NOT NULL DEFAULT '',
|
||||
added_in TEXT NOT NULL DEFAULT ''
|
||||
) WITHOUT ROWID;
|
||||
CREATE INDEX IF NOT EXISTS release_by_repo ON release_enrichment(repo_prefix) WHERE repo_prefix <> '';
|
||||
|
||||
-- blame_enrichment: per-symbol latest-author sidecar (change A).
|
||||
CREATE TABLE IF NOT EXISTS blame_enrichment (
|
||||
node_id TEXT PRIMARY KEY,
|
||||
repo_prefix TEXT NOT NULL DEFAULT '',
|
||||
commit_sha TEXT NOT NULL DEFAULT '',
|
||||
email TEXT NOT NULL DEFAULT '',
|
||||
ts INTEGER NOT NULL DEFAULT 0
|
||||
) WITHOUT ROWID;
|
||||
CREATE INDEX IF NOT EXISTS blame_by_repo ON blame_enrichment(repo_prefix) WHERE repo_prefix <> '';
|
||||
|
||||
-- symbol_fts is the FTS5 full-text index over pre-tokenised symbol
|
||||
-- names. It replaces the multi-GB in-heap Bleve/BM25 index with an
|
||||
-- on-disk inverted index the SymbolSearcher / SymbolBundleSearcher
|
||||
-- query through. A standard (NOT contentless) FTS5 table; individual
|
||||
-- rows are deleted by their FTS5 docid via the symbol_fts_rowid sidecar
|
||||
-- below (node_id is UNINDEXED, so a DELETE keyed on it would full-scan
|
||||
-- the index). node_id is the join key back to nodes.id; repo_prefix is
|
||||
-- carried UNINDEXED so per-repo staleness wipes (DELETE … WHERE
|
||||
-- repo_prefix = ?) hit a literal column without a separate b-tree.
|
||||
-- Only "tokens" is indexed for matching. IF NOT EXISTS makes this
|
||||
-- idempotent on every Open, so an existing .sqlite gains the vtable
|
||||
-- on its next open + reindex.
|
||||
CREATE VIRTUAL TABLE IF NOT EXISTS symbol_fts USING fts5(node_id UNINDEXED, repo_prefix UNINDEXED, tokens);
|
||||
|
||||
-- symbol_fts_rowid maps a node_id to the rowid (FTS5 docid) of its row in
|
||||
-- symbol_fts. node_id is UNINDEXED in the FTS5 vtable, so deleting a node's
|
||||
-- prior row with "DELETE … WHERE node_id = ?" full-scans the entire index
|
||||
-- once PER symbol — quadratic on the per-edit reindex hot path. This sidecar
|
||||
-- turns the delete into an O(log n) docid delete ("WHERE rowid = ?", the FTS5
|
||||
-- docid IS indexed). One row per indexed symbol, keyed by node_id (the join
|
||||
-- key back to nodes.id); repo_prefix scopes the per-repo wipe that
|
||||
-- BulkUpsertSymbolFTS performs in lockstep with symbol_fts. WITHOUT ROWID:
|
||||
-- the PK index IS the table, like file_mtimes / clone_shingles.
|
||||
CREATE TABLE IF NOT EXISTS symbol_fts_rowid (
|
||||
node_id TEXT PRIMARY KEY,
|
||||
repo_prefix TEXT NOT NULL DEFAULT '',
|
||||
fts_rowid INTEGER NOT NULL
|
||||
) WITHOUT ROWID;
|
||||
|
||||
-- content_fts is the FTS5 full-text index over CONTENT (data_class=
|
||||
-- "content") section bodies — text / pdf / pptx / xlsx chunks. It is
|
||||
-- kept SEPARATE from symbol_fts so content text never enters the symbol
|
||||
-- search or the code-oriented analysis passes: a content-heavy repo of a
|
||||
-- few hundred large documents explodes into hundreds of thousands of
|
||||
-- section nodes, and streaming their bodies here (per file, on disk)
|
||||
-- instead of into symbol_fts + graph nodes keeps the code index and the
|
||||
-- graph passes bounded. Only "body" is indexed for matching; node_id /
|
||||
-- repo_prefix / file_path / ordinal ride UNINDEXED so the per-repo and
|
||||
-- per-file staleness wipes hit literal columns without a b-tree.
|
||||
CREATE VIRTUAL TABLE IF NOT EXISTS content_fts USING fts5(node_id UNINDEXED, repo_prefix UNINDEXED, file_path UNINDEXED, ordinal UNINDEXED, body);
|
||||
`
|
||||
@@ -0,0 +1,104 @@
|
||||
package store_sqlite
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"github.com/zzet/gortex/internal/graph"
|
||||
)
|
||||
|
||||
// hasNodeColumn reports whether the nodes table currently has the named column.
|
||||
func hasNodeColumn(t *testing.T, db *sql.DB, col string) bool {
|
||||
t.Helper()
|
||||
rows, err := db.Query(`PRAGMA table_info(nodes)`)
|
||||
require.NoError(t, err)
|
||||
defer rows.Close()
|
||||
for rows.Next() {
|
||||
var (
|
||||
cid, notnull, pk int
|
||||
name, ctype string
|
||||
dflt sql.NullString
|
||||
)
|
||||
require.NoError(t, rows.Scan(&cid, &name, &ctype, ¬null, &dflt, &pk))
|
||||
if name == col {
|
||||
return true
|
||||
}
|
||||
}
|
||||
require.NoError(t, rows.Err())
|
||||
return false
|
||||
}
|
||||
|
||||
// TestOpenUpgradesPreDataClassStore is the backward-compatibility proof for the
|
||||
// promoted data_class column: an existing v1 store written before the column
|
||||
// existed must Open cleanly (ensureNodeColumns ALTERs the column in before the
|
||||
// node statements are prepared), keep its rows, and immediately get the working
|
||||
// SQL-level content filter — all WITHOUT a schema_version bump or a reindex.
|
||||
//
|
||||
// The simulated old store has data_class dropped while staying at the v1
|
||||
// baseline, the exact shape of every Gortex sqlite DB already on disk before
|
||||
// this change. Without the data_class entry in promotedMetaColumns, the reopen
|
||||
// fails with "no such column: data_class" when prepare() builds the node
|
||||
// INSERT/SELECT — so this test fails-closed if that wiring regresses.
|
||||
func TestOpenUpgradesPreDataClassStore(t *testing.T) {
|
||||
path := filepath.Join(t.TempDir(), "store.sqlite")
|
||||
|
||||
// 1. Create a current store and seed a row, then close. A fresh store is
|
||||
// stamped at the current schema version; step 2 knocks it back to the v1
|
||||
// baseline to simulate a store written before data_class existed.
|
||||
s, err := Open(path)
|
||||
require.NoError(t, err)
|
||||
s.AddNode(&graph.Node{ID: "old1", Kind: graph.KindFunction, Name: "Legacy", FilePath: "f.go", RepoPrefix: "r"})
|
||||
require.NoError(t, s.Close())
|
||||
|
||||
// 2. Simulate a store written before data_class existed: drop the column
|
||||
// while leaving user_version at the v1 baseline.
|
||||
withRawDB(t, path, func(db *sql.DB) {
|
||||
_, err := db.Exec(`ALTER TABLE nodes DROP COLUMN data_class`)
|
||||
require.NoError(t, err, "simulate a pre-data_class store")
|
||||
require.False(t, hasNodeColumn(t, db, "data_class"), "data_class must be absent before the upgrade")
|
||||
|
||||
// A fresh Open stamps the current schema version; knock it back to the v1
|
||||
// baseline so this genuinely simulates a pre-data_class store and the
|
||||
// reopen exercises the in-place upgrade arm rather than a no-op.
|
||||
_, err = db.Exec(`PRAGMA user_version = 1`)
|
||||
require.NoError(t, err, "reset to the v1 baseline")
|
||||
|
||||
var v int
|
||||
require.NoError(t, db.QueryRow(`PRAGMA user_version`).Scan(&v))
|
||||
require.Equal(t, 1, v, "the simulated old store must sit at the v1 baseline")
|
||||
})
|
||||
|
||||
// 3. Reopen with the current binary. ensureNodeColumns must re-add the
|
||||
// column before prepare() references it, so Open succeeds without a wipe.
|
||||
s2, err := Open(path)
|
||||
require.NoError(t, err, "Open must upgrade a pre-data_class store in place, not fail on the missing column")
|
||||
t.Cleanup(func() { _ = s2.Close() })
|
||||
|
||||
require.True(t, hasNodeColumn(t, s2.db, "data_class"), "ensureNodeColumns must re-add data_class on Open")
|
||||
require.False(t, s2.NeedsRebuild(), "an additive-column upgrade must not signal a wipe/reindex")
|
||||
|
||||
// 4. Existing rows survived (the upgrade is in place, not a rebuild).
|
||||
require.NotNil(t, s2.GetNode("old1"), "existing rows must survive the in-place upgrade")
|
||||
|
||||
// 5. A new content node persists, round-trips through the promoted column,
|
||||
// and is correctly dropped by the SQL-level content filter.
|
||||
s2.AddNode(&graph.Node{ID: "content1", Kind: graph.KindDoc, Name: "doc.txt::0", RepoPrefix: "r",
|
||||
Meta: map[string]any{"data_class": "content", "section_text": "x"}})
|
||||
s2.AddNode(&graph.Node{ID: "code1", Kind: graph.KindFunction, Name: "Foo", RepoPrefix: "r"})
|
||||
|
||||
content := s2.GetNode("content1")
|
||||
require.NotNil(t, content)
|
||||
require.Equal(t, "content", content.Meta["data_class"], "data_class round-trips via the promoted column after upgrade")
|
||||
|
||||
ids := map[string]bool{}
|
||||
for _, n := range s2.GetRepoNonContentNodes("r") {
|
||||
ids[n.ID] = true
|
||||
}
|
||||
require.True(t, ids["old1"], "legacy non-content node kept")
|
||||
require.True(t, ids["code1"], "code node kept")
|
||||
require.False(t, ids["content1"], "content node filtered at the SQL level after the upgrade")
|
||||
require.Len(t, ids, 2)
|
||||
}
|
||||
@@ -0,0 +1,247 @@
|
||||
package store_sqlite
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"fmt"
|
||||
"os"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// Schema versioning for the graph store.
|
||||
//
|
||||
// Unlike the sidecar (which holds irreplaceable user data and must migrate in
|
||||
// place), the graph store is a DERIVED CACHE: every row is reconstructable by
|
||||
// re-indexing the source. So the cheapest *always-correct* reaction to a schema
|
||||
// change an old on-disk DB can't satisfy is to drop the file and let the daemon
|
||||
// rebuild it on the next index. A migration may therefore declare rebuild=true
|
||||
// instead of writing an in-place transform that would have to re-derive the new
|
||||
// data from source anyway. In-place steps remain the cheap path for purely
|
||||
// mechanical changes (a new index, a denormalisation, a column with a
|
||||
// computable default) that spare a large repo a multi-minute reindex.
|
||||
//
|
||||
// The whole mechanism keys off SQLite's built-in PRAGMA user_version, read on
|
||||
// Open before schemaSQL runs. There is no separate version table.
|
||||
//
|
||||
// Concurrency: the daemon holds an exclusive flock on <store>.lock around Open
|
||||
// (see serverstack.NewSharedServer), so reading the version, wiping the file,
|
||||
// and stamping it cannot race another process. That is why — unlike the
|
||||
// sidecar — this path needs no BEGIN IMMEDIATE / busy-loop handling.
|
||||
|
||||
// currentSchemaVersion is the version a fully-reconciled store reports via
|
||||
// PRAGMA user_version. Bump it whenever schemaSQL's typed-column shape or an
|
||||
// index changes in a way an old on-disk DB would not already have, and append a
|
||||
// matching schemaMigrations entry describing how to bring an older store
|
||||
// forward (in place, or by rebuild).
|
||||
const currentSchemaVersion = 2
|
||||
|
||||
// schemaMigration is one forward step. Exactly one strategy applies:
|
||||
// - rebuild=true: the change introduces structure/data that can only come
|
||||
// from re-indexing the source; an older store is wiped and rebuilt.
|
||||
// - inPlace!=nil: the change is mechanically derivable from the existing
|
||||
// store and is applied in a transaction with no reindex.
|
||||
//
|
||||
// Steps are append-only and ascending; never edit or renumber a shipped one.
|
||||
// Any inPlace step must be idempotent (IF NOT EXISTS / ADD COLUMN guarded).
|
||||
type schemaMigration struct {
|
||||
version int
|
||||
name string
|
||||
inPlace func(tx *sql.Tx) error
|
||||
rebuild bool
|
||||
}
|
||||
|
||||
// schemaMigrations is the ordered, forward-only registry. Version 1 is the
|
||||
// implicit baseline (no entry): a v1 store is reconciled entirely by schemaSQL's
|
||||
// idempotent CREATE ... IF NOT EXISTS plus ensureNodeColumns, so any
|
||||
// pre-versioning database baseline-stamps to v1 without a rebuild. Append
|
||||
// entries for version 2 and up as the schema evolves.
|
||||
var schemaMigrations = []schemaMigration{
|
||||
{version: 2, name: "dedupe fn-value placeholder edges", inPlace: dedupeFnValuePlaceholderEdges},
|
||||
}
|
||||
|
||||
// dedupeFnValuePlaceholderEdges collapses duplicate function-as-value gate
|
||||
// placeholder edges (graph.FnValuePlaceholderMarker, `unresolved::fnvalue::
|
||||
// <name>`) to one row per (from_id, to_id), keeping the MIN(id) survivor. The
|
||||
// capture path now dedups per (from, name) before it emits, but stores written
|
||||
// earlier accumulated one placeholder per call site — a live store held
|
||||
// millions — and EdgesWithUnresolvedTarget plus the resolver's terminal
|
||||
// reconcile materialised every one on each warm restart, the dominant warmup
|
||||
// heap transient this step drains. The keep set is small (tens of thousands of
|
||||
// distinct pairs), so the NOT IN materialisation is cheap; the ph filter rides
|
||||
// the edges_by_to(to_id) range for the bare form and the is_unresolved index for
|
||||
// the multi-repo infix form. Idempotent: a second run finds no duplicates. Freed
|
||||
// pages return to the freelist and are reused by later writes; the file itself
|
||||
// shrinks only under a manual VACUUM, deliberately out of scope for a derived
|
||||
// cache that reclaims the space on its own.
|
||||
func dedupeFnValuePlaceholderEdges(tx *sql.Tx) error {
|
||||
_, err := tx.Exec(`
|
||||
WITH ph AS (
|
||||
SELECT id, from_id, to_id FROM edges
|
||||
WHERE (to_id >= 'unresolved::fnvalue::' AND to_id < 'unresolved::fnvalue:;')
|
||||
OR (is_unresolved = 1 AND to_id LIKE '%::unresolved::fnvalue::%')
|
||||
), keep AS (
|
||||
SELECT MIN(id) AS id FROM ph GROUP BY from_id, to_id
|
||||
)
|
||||
DELETE FROM edges WHERE id IN (SELECT id FROM ph) AND id NOT IN (SELECT id FROM keep)`)
|
||||
return err
|
||||
}
|
||||
|
||||
// schemaPlan is the decision planSchemaMigration derives from the stored
|
||||
// PRAGMA user_version. It mutates nothing on its own.
|
||||
type schemaPlan struct {
|
||||
wipe bool // drop the on-disk DB and rebuild from source
|
||||
inPlace []schemaMigration // ordered in-place steps to run after schemaSQL
|
||||
stamp bool // write currentSchemaVersion once reconciled
|
||||
}
|
||||
|
||||
// planSchemaMigrationWith decides how to reconcile a store at the stored
|
||||
// PRAGMA user_version to current, given the migration registry. It mutates
|
||||
// nothing. Open passes (currentSchemaVersion, schemaMigrations); tests pass
|
||||
// fixtures.
|
||||
func planSchemaMigrationWith(stored, current int, migrations []schemaMigration) schemaPlan {
|
||||
switch {
|
||||
case stored == current:
|
||||
return schemaPlan{} // up to date, nothing to do
|
||||
case stored > current:
|
||||
// Written by a newer build than this binary understands; the shape may
|
||||
// have changed under us. For a cache the safe move is to rebuild.
|
||||
return schemaPlan{wipe: true, stamp: true}
|
||||
case stored == 0:
|
||||
// Fresh DB, or a pre-versioning store of unknown shape. schemaSQL's
|
||||
// idempotent CREATE ... IF NOT EXISTS plus ensureNodeColumns /
|
||||
// ensureEdgeColumns reconcile the base shape either way, so a stored==0
|
||||
// store needs a wipe only when a pending step is a REBUILD whose data can
|
||||
// only come from re-indexing source. With nothing pending, stamp; with
|
||||
// only in-place steps pending, run them and stamp — an in-place step is
|
||||
// idempotent and mechanically derivable, so it upgrades a pre-versioning
|
||||
// store in place (preserving its rows) exactly as it upgrades a known
|
||||
// prior version. Wiping a stored==0 store on any migration instead would
|
||||
// force every non-daemon Open (tests, read-only tools) to pass WithRebuild
|
||||
// the moment the first migration ships.
|
||||
pending := pendingBetween(0, current, migrations)
|
||||
if len(pending) == 0 {
|
||||
return schemaPlan{stamp: true}
|
||||
}
|
||||
if anyRebuild(pending) {
|
||||
return schemaPlan{wipe: true, stamp: true}
|
||||
}
|
||||
return schemaPlan{inPlace: pending, stamp: true}
|
||||
default: // 0 < stored < current: a known prior version
|
||||
pending := pendingBetween(stored, current, migrations)
|
||||
if anyRebuild(pending) {
|
||||
return schemaPlan{wipe: true, stamp: true}
|
||||
}
|
||||
return schemaPlan{inPlace: pending, stamp: true}
|
||||
}
|
||||
}
|
||||
|
||||
func pendingBetween(stored, current int, migrations []schemaMigration) []schemaMigration {
|
||||
var out []schemaMigration
|
||||
for _, m := range migrations {
|
||||
if m.version > stored && m.version <= current {
|
||||
out = append(out, m)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func anyRebuild(ms []schemaMigration) bool {
|
||||
for _, m := range ms {
|
||||
if m.rebuild {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// validateSchemaMigrations checks the registry is well-formed. A test asserts
|
||||
// this against the shipped (currentSchemaVersion, schemaMigrations) so the
|
||||
// dangerous mistake — bumping currentSchemaVersion without appending a matching
|
||||
// entry — fails CI instead of silently baseline-stamping an un-migrated store
|
||||
// to the new version at runtime. Rules:
|
||||
// - versions are >= 2 (v1 is the implicit baseline, never an entry) and
|
||||
// strictly ascending;
|
||||
// - each step sets exactly one strategy (inPlace xor rebuild);
|
||||
// - the highest version equals current, so the registry actually defines how
|
||||
// to reach it. An empty registry is valid only at version 1.
|
||||
func validateSchemaMigrations(current int, migs []schemaMigration) error {
|
||||
if len(migs) == 0 {
|
||||
if current != 1 {
|
||||
return fmt.Errorf("schema version %d has no migrations: only v1 may have an empty registry", current)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
prev := 0
|
||||
for i, m := range migs {
|
||||
if m.version < 2 {
|
||||
return fmt.Errorf("migration %q has version %d: entries must be >= 2 (v1 is the implicit baseline)", m.name, m.version)
|
||||
}
|
||||
if i > 0 && m.version <= prev {
|
||||
return fmt.Errorf("migrations must be strictly ascending: v%d (%s) does not follow v%d", m.version, m.name, prev)
|
||||
}
|
||||
if (m.inPlace != nil) == m.rebuild {
|
||||
return fmt.Errorf("migration v%d (%s) must set exactly one of inPlace / rebuild", m.version, m.name)
|
||||
}
|
||||
prev = m.version
|
||||
}
|
||||
if prev != current {
|
||||
return fmt.Errorf("highest migration version %d != currentSchemaVersion %d: a version bump needs a matching migration entry", prev, current)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// readUserVersion reads PRAGMA user_version (0 on a fresh database).
|
||||
func readUserVersion(db *sql.DB) (int, error) {
|
||||
var v int
|
||||
if err := db.QueryRow("PRAGMA user_version").Scan(&v); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return v, nil
|
||||
}
|
||||
|
||||
// setUserVersion stamps the schema version. PRAGMA takes no bound parameters;
|
||||
// v is an int we control, so the format is safe.
|
||||
func setUserVersion(db *sql.DB, v int) error {
|
||||
if _, err := db.Exec(fmt.Sprintf("PRAGMA user_version = %d", v)); err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// applyInPlaceMigrations runs the in-place steps in a single transaction.
|
||||
func applyInPlaceMigrations(db *sql.DB, steps []schemaMigration) error {
|
||||
if len(steps) == 0 {
|
||||
return nil
|
||||
}
|
||||
tx, err := db.Begin()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer func() { _ = tx.Rollback() }() // no-op once Commit succeeds
|
||||
for _, m := range steps {
|
||||
if err := m.inPlace(tx); err != nil {
|
||||
return fmt.Errorf("schema migration v%d (%s): %w", m.version, m.name, err)
|
||||
}
|
||||
}
|
||||
return tx.Commit()
|
||||
}
|
||||
|
||||
// removeStoreFiles deletes the SQLite database and its companions. A missing
|
||||
// file is not an error. Never called for ":memory:".
|
||||
//
|
||||
// The suffix list covers the files the DSN's journal_mode(WAL) produces (-wal,
|
||||
// -shm) plus the rollback -journal a non-WAL fallback would use; keep it in
|
||||
// sync if the journal_mode in Open's DSN ever changes.
|
||||
func removeStoreFiles(path string) error {
|
||||
for _, suffix := range []string{"", "-wal", "-shm", "-journal"} {
|
||||
if err := os.Remove(path + suffix); err != nil && !os.IsNotExist(err) {
|
||||
return fmt.Errorf("remove %s: %w", path+suffix, err)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// isMemoryPath reports whether path is an in-process SQLite database (no file
|
||||
// on disk to wipe, always built fresh by schemaSQL).
|
||||
func isMemoryPath(path string) bool {
|
||||
return strings.Contains(path, ":memory:")
|
||||
}
|
||||
@@ -0,0 +1,545 @@
|
||||
package store_sqlite
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"errors"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
_ "modernc.org/sqlite"
|
||||
)
|
||||
|
||||
// withRawDB opens a bare *sql.DB on path, runs fn, and closes it — used to
|
||||
// simulate an on-disk store written by an older/newer build (set user_version,
|
||||
// insert rows) without going through Open's reconciliation.
|
||||
func withRawDB(t *testing.T, path string, fn func(db *sql.DB)) {
|
||||
t.Helper()
|
||||
db, err := sql.Open("sqlite", path+"?_pragma=busy_timeout(5000)")
|
||||
if err != nil {
|
||||
t.Fatalf("open raw db: %v", err)
|
||||
}
|
||||
defer db.Close()
|
||||
fn(db)
|
||||
}
|
||||
|
||||
func nodeCount(t *testing.T, db *sql.DB) int {
|
||||
t.Helper()
|
||||
var n int
|
||||
if err := db.QueryRow("SELECT COUNT(*) FROM nodes").Scan(&n); err != nil {
|
||||
t.Fatalf("count nodes: %v", err)
|
||||
}
|
||||
return n
|
||||
}
|
||||
|
||||
// TestOpenStampsFreshDB: a brand-new on-disk store is stamped to the current
|
||||
// schema version.
|
||||
func TestOpenStampsFreshDB(t *testing.T) {
|
||||
path := filepath.Join(t.TempDir(), "store.sqlite")
|
||||
s, err := Open(path)
|
||||
if err != nil {
|
||||
t.Fatalf("Open fresh: %v", err)
|
||||
}
|
||||
defer s.Close()
|
||||
if v, err := readUserVersion(s.db); err != nil || v != currentSchemaVersion {
|
||||
t.Fatalf("fresh user_version = %d (err %v), want %d", v, err, currentSchemaVersion)
|
||||
}
|
||||
}
|
||||
|
||||
// TestOpenBaselineStampsOldDBWithoutWipe: a pre-versioning store (user_version
|
||||
// 0, reconcilable to current by schemaSQL + ensureNodeColumns) is stamped in
|
||||
// place — its data must survive, not be wiped.
|
||||
func TestOpenBaselineStampsOldDBWithoutWipe(t *testing.T) {
|
||||
path := filepath.Join(t.TempDir(), "store.sqlite")
|
||||
|
||||
// Create the store, then simulate a pre-versioning DB: a row + user_version 0.
|
||||
s, err := Open(path)
|
||||
if err != nil {
|
||||
t.Fatalf("first open: %v", err)
|
||||
}
|
||||
if err := s.Close(); err != nil {
|
||||
t.Fatalf("close: %v", err)
|
||||
}
|
||||
withRawDB(t, path, func(db *sql.DB) {
|
||||
if _, err := db.Exec(`INSERT INTO nodes (id, kind, name, file_path) VALUES ('n1','func','Foo','f.go')`); err != nil {
|
||||
t.Fatalf("seed node: %v", err)
|
||||
}
|
||||
if _, err := db.Exec(`PRAGMA user_version = 0`); err != nil {
|
||||
t.Fatalf("reset user_version: %v", err)
|
||||
}
|
||||
})
|
||||
|
||||
s2, err := Open(path)
|
||||
if err != nil {
|
||||
t.Fatalf("reopen old DB: %v", err)
|
||||
}
|
||||
defer s2.Close()
|
||||
if v, _ := readUserVersion(s2.db); v != currentSchemaVersion {
|
||||
t.Fatalf("user_version after baseline = %d, want %d", v, currentSchemaVersion)
|
||||
}
|
||||
if n := nodeCount(t, s2.db); n != 1 {
|
||||
t.Fatalf("node count after baseline = %d, want 1 (data must NOT be wiped)", n)
|
||||
}
|
||||
}
|
||||
|
||||
// TestOpenRebuildsNewerDB: a store written by a NEWER build (user_version above
|
||||
// current) cannot be trusted, so Open drops and rebuilds it — the data is gone
|
||||
// and the version is re-stamped to current. Proves the wipe path (and that the
|
||||
// -wal/-shm companions are cleared along with the main file).
|
||||
func TestOpenRebuildsNewerDB(t *testing.T) {
|
||||
path := filepath.Join(t.TempDir(), "store.sqlite")
|
||||
|
||||
s, err := Open(path)
|
||||
if err != nil {
|
||||
t.Fatalf("first open: %v", err)
|
||||
}
|
||||
if err := s.Close(); err != nil {
|
||||
t.Fatalf("close: %v", err)
|
||||
}
|
||||
withRawDB(t, path, func(db *sql.DB) {
|
||||
if _, err := db.Exec(`INSERT INTO nodes (id, kind, name, file_path) VALUES ('n1','func','Foo','f.go')`); err != nil {
|
||||
t.Fatalf("seed node: %v", err)
|
||||
}
|
||||
if _, err := db.Exec(`PRAGMA user_version = 999`); err != nil { // a future version this binary doesn't know
|
||||
t.Fatalf("set future user_version: %v", err)
|
||||
}
|
||||
})
|
||||
|
||||
s2, err := Open(path, WithRebuild()) // simulate the daemon: holds the lock, may rebuild
|
||||
if err != nil {
|
||||
t.Fatalf("reopen newer DB: %v", err)
|
||||
}
|
||||
defer s2.Close()
|
||||
if v, _ := readUserVersion(s2.db); v != currentSchemaVersion {
|
||||
t.Fatalf("user_version after rebuild = %d, want %d", v, currentSchemaVersion)
|
||||
}
|
||||
if n := nodeCount(t, s2.db); n != 0 {
|
||||
t.Fatalf("node count after rebuild = %d, want 0 (newer DB must be wiped)", n)
|
||||
}
|
||||
}
|
||||
|
||||
// TestOpenRefusesWipeWithoutOptIn: the default Open must NOT destroy an
|
||||
// incompatible on-disk database. Without WithRebuild it returns
|
||||
// ErrSchemaRebuildRequired and leaves the file (and its rows) intact, so a
|
||||
// caller that does not hold the store lock cannot silently corrupt a store
|
||||
// another process may have open.
|
||||
func TestOpenRefusesWipeWithoutOptIn(t *testing.T) {
|
||||
path := filepath.Join(t.TempDir(), "store.sqlite")
|
||||
s, err := Open(path)
|
||||
if err != nil {
|
||||
t.Fatalf("first open: %v", err)
|
||||
}
|
||||
if _, err := s.db.Exec(`INSERT INTO nodes (id, kind, name, file_path) VALUES ('n1','func','Foo','f.go')`); err != nil {
|
||||
t.Fatalf("seed node: %v", err)
|
||||
}
|
||||
if err := s.Close(); err != nil {
|
||||
t.Fatalf("close: %v", err)
|
||||
}
|
||||
withRawDB(t, path, func(db *sql.DB) {
|
||||
if _, err := db.Exec(`PRAGMA user_version = 999`); err != nil {
|
||||
t.Fatalf("set future version: %v", err)
|
||||
}
|
||||
})
|
||||
|
||||
if _, err := Open(path); !errors.Is(err, ErrSchemaRebuildRequired) {
|
||||
t.Fatalf("Open without WithRebuild = %v, want ErrSchemaRebuildRequired", err)
|
||||
}
|
||||
withRawDB(t, path, func(db *sql.DB) {
|
||||
if n := nodeCount(t, db); n != 1 {
|
||||
t.Fatalf("node count = %d after a refused wipe, want 1 (the file must be untouched)", n)
|
||||
}
|
||||
var v int
|
||||
if err := db.QueryRow("PRAGMA user_version").Scan(&v); err != nil {
|
||||
t.Fatalf("read user_version: %v", err)
|
||||
}
|
||||
if v != 999 {
|
||||
t.Fatalf("user_version = %d after a refused wipe, want 999 (unchanged)", v)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// TestPlanSchemaMigration covers the pure decision logic, including the
|
||||
// in-place vs rebuild dispatch a future currentSchemaVersion=2 would exercise.
|
||||
func TestPlanSchemaMigration(t *testing.T) {
|
||||
inPlace := schemaMigration{version: 2, name: "add-index", inPlace: func(*sql.Tx) error { return nil }}
|
||||
rebuild := schemaMigration{version: 2, name: "typed-column", rebuild: true}
|
||||
|
||||
cases := []struct {
|
||||
name string
|
||||
stored, current int
|
||||
migs []schemaMigration
|
||||
wantWipe bool
|
||||
wantStamp bool
|
||||
wantInPlace int
|
||||
}{
|
||||
{"up to date", 1, 1, nil, false, false, 0},
|
||||
{"fresh at v1 baseline-stamps", 0, 1, nil, false, true, 0},
|
||||
{"newer DB rebuilds", 2, 1, nil, true, true, 0},
|
||||
{"v0 with only in-place pending upgrades in place, no wipe", 0, 2, []schemaMigration{inPlace}, false, true, 1},
|
||||
{"v0 with a pending rebuild wipes", 0, 2, []schemaMigration{rebuild}, true, true, 0},
|
||||
{"v1->v2 in-place", 1, 2, []schemaMigration{inPlace}, false, true, 1},
|
||||
{"v1->v2 rebuild", 1, 2, []schemaMigration{rebuild}, true, true, 0},
|
||||
}
|
||||
for _, c := range cases {
|
||||
t.Run(c.name, func(t *testing.T) {
|
||||
got := planSchemaMigrationWith(c.stored, c.current, c.migs)
|
||||
if got.wipe != c.wantWipe || got.stamp != c.wantStamp || len(got.inPlace) != c.wantInPlace {
|
||||
t.Fatalf("plan(%d->%d) = {wipe:%v stamp:%v inPlace:%d}, want {wipe:%v stamp:%v inPlace:%d}",
|
||||
c.stored, c.current, got.wipe, got.stamp, len(got.inPlace), c.wantWipe, c.wantStamp, c.wantInPlace)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestApplyInPlaceMigrations: steps run in order and commit; a failing step
|
||||
// rolls the whole transaction back.
|
||||
func TestApplyInPlaceMigrations(t *testing.T) {
|
||||
t.Run("commit", func(t *testing.T) {
|
||||
path := filepath.Join(t.TempDir(), "m.sqlite")
|
||||
withRawDB(t, path, func(db *sql.DB) {
|
||||
step := schemaMigration{version: 2, name: "mk", inPlace: func(tx *sql.Tx) error {
|
||||
_, err := tx.Exec(`CREATE TABLE marker (x TEXT)`)
|
||||
return err
|
||||
}}
|
||||
if err := applyInPlaceMigrations(db, []schemaMigration{step}); err != nil {
|
||||
t.Fatalf("apply: %v", err)
|
||||
}
|
||||
var name string
|
||||
if err := db.QueryRow(`SELECT name FROM sqlite_master WHERE type='table' AND name='marker'`).Scan(&name); err != nil {
|
||||
t.Fatalf("marker table not created: %v", err)
|
||||
}
|
||||
})
|
||||
})
|
||||
|
||||
t.Run("rollback on failure preserves cause and rolls back every step", func(t *testing.T) {
|
||||
path := filepath.Join(t.TempDir(), "m.sqlite")
|
||||
withRawDB(t, path, func(db *sql.DB) {
|
||||
// Two steps in one batch: the first creates table A, the second
|
||||
// creates B then fails. Both must roll back, proving the steps
|
||||
// share a single transaction.
|
||||
stepA := schemaMigration{version: 2, name: "make-a", inPlace: func(tx *sql.Tx) error {
|
||||
_, err := tx.Exec(`CREATE TABLE a (x TEXT)`)
|
||||
return err
|
||||
}}
|
||||
stepB := schemaMigration{version: 3, name: "boom", inPlace: func(tx *sql.Tx) error {
|
||||
if _, err := tx.Exec(`CREATE TABLE b (x TEXT)`); err != nil {
|
||||
return err
|
||||
}
|
||||
return sql.ErrConnDone // synthetic failure after a partial write
|
||||
}}
|
||||
err := applyInPlaceMigrations(db, []schemaMigration{stepA, stepB})
|
||||
if err == nil {
|
||||
t.Fatal("expected applyInPlaceMigrations to surface the step error")
|
||||
}
|
||||
if !errors.Is(err, sql.ErrConnDone) {
|
||||
t.Fatalf("error should wrap the step's cause; got %v", err)
|
||||
}
|
||||
if !strings.Contains(err.Error(), "v3") || !strings.Contains(err.Error(), "boom") {
|
||||
t.Fatalf("error should name the failing migration (v3/boom); got %q", err.Error())
|
||||
}
|
||||
for _, tbl := range []string{"a", "b"} {
|
||||
var name string
|
||||
e := db.QueryRow(`SELECT name FROM sqlite_master WHERE type='table' AND name=?`, tbl).Scan(&name)
|
||||
if e != sql.ErrNoRows {
|
||||
t.Fatalf("table %q should have rolled back (shared transaction), got name=%q err=%v", tbl, name, e)
|
||||
}
|
||||
}
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
// TestOpenAtCurrentVersionIsNoOp covers the highest-frequency path — every
|
||||
// daemon restart reopens an up-to-date store. It must be a no-op that
|
||||
// preserves data; an off-by-one to wipe here would destroy the cache on every
|
||||
// restart.
|
||||
func TestOpenAtCurrentVersionIsNoOp(t *testing.T) {
|
||||
path := filepath.Join(t.TempDir(), "store.sqlite")
|
||||
s, err := Open(path)
|
||||
if err != nil {
|
||||
t.Fatalf("first open: %v", err)
|
||||
}
|
||||
withRawSeed := func(db *sql.DB) {
|
||||
if _, err := db.Exec(`INSERT INTO nodes (id, kind, name, file_path) VALUES ('n1','func','Foo','f.go')`); err != nil {
|
||||
t.Fatalf("seed node: %v", err)
|
||||
}
|
||||
}
|
||||
withRawSeed(s.db)
|
||||
if err := s.Close(); err != nil {
|
||||
t.Fatalf("close: %v", err)
|
||||
}
|
||||
|
||||
s2, err := Open(path)
|
||||
if err != nil {
|
||||
t.Fatalf("reopen at current version: %v", err)
|
||||
}
|
||||
defer s2.Close()
|
||||
if v, _ := readUserVersion(s2.db); v != currentSchemaVersion {
|
||||
t.Fatalf("user_version = %d, want %d", v, currentSchemaVersion)
|
||||
}
|
||||
if n := nodeCount(t, s2.db); n != 1 {
|
||||
t.Fatalf("node count = %d, want 1 (a no-op reopen must NOT wipe)", n)
|
||||
}
|
||||
if s2.NeedsRebuild() {
|
||||
t.Fatal("a no-op reopen must not signal NeedsRebuild")
|
||||
}
|
||||
}
|
||||
|
||||
// TestOpenWithInPlaceMigration drives the in-place arm end-to-end through the
|
||||
// real Open composition (via the openWith seam): an older store at version 1
|
||||
// is upgraded to version 2 by a registered in-place step that runs AFTER
|
||||
// schemaSQL, the step's effect is visible, the existing data survives, and the
|
||||
// version is stamped.
|
||||
func TestOpenWithInPlaceMigration(t *testing.T) {
|
||||
path := filepath.Join(t.TempDir(), "store.sqlite")
|
||||
|
||||
// Create a store with a row, then knock it back to the v1 baseline so the
|
||||
// openWith below drives the v1->v2 in-place arm (a fresh Open now stamps the
|
||||
// current version, which is >= 2).
|
||||
s, err := Open(path)
|
||||
if err != nil {
|
||||
t.Fatalf("first open: %v", err)
|
||||
}
|
||||
if _, err := s.db.Exec(`INSERT INTO nodes (id, kind, name, file_path) VALUES ('n1','func','Foo','f.go')`); err != nil {
|
||||
t.Fatalf("seed node: %v", err)
|
||||
}
|
||||
if err := s.Close(); err != nil {
|
||||
t.Fatalf("close: %v", err)
|
||||
}
|
||||
withRawDB(t, path, func(db *sql.DB) {
|
||||
if _, err := db.Exec(`PRAGMA user_version = 1`); err != nil {
|
||||
t.Fatalf("reset to v1 baseline: %v", err)
|
||||
}
|
||||
})
|
||||
|
||||
// An in-place v2 step that depends on the base schema (an index on a
|
||||
// nodes column) — proving it runs after schemaSQL/ensureNodeColumns.
|
||||
ran := false
|
||||
v2 := schemaMigration{version: 2, name: "idx-language", inPlace: func(tx *sql.Tx) error {
|
||||
ran = true
|
||||
_, err := tx.Exec(`CREATE INDEX IF NOT EXISTS test_nodes_by_language ON nodes(language)`)
|
||||
return err
|
||||
}}
|
||||
|
||||
s2, err := openWith(path, 2, []schemaMigration{v2}, false) // in-place never wipes
|
||||
if err != nil {
|
||||
t.Fatalf("openWith v2 in-place: %v", err)
|
||||
}
|
||||
defer s2.Close()
|
||||
if !ran {
|
||||
t.Fatal("the in-place migration step did not run")
|
||||
}
|
||||
if v, _ := readUserVersion(s2.db); v != 2 {
|
||||
t.Fatalf("user_version = %d, want 2", v)
|
||||
}
|
||||
if n := nodeCount(t, s2.db); n != 1 {
|
||||
t.Fatalf("node count = %d, want 1 (in-place upgrade must preserve data)", n)
|
||||
}
|
||||
var name string
|
||||
if err := s2.db.QueryRow(`SELECT name FROM sqlite_master WHERE type='index' AND name='test_nodes_by_language'`).Scan(&name); err != nil {
|
||||
t.Fatalf("in-place index not created: %v", err)
|
||||
}
|
||||
if s2.NeedsRebuild() {
|
||||
t.Fatal("an in-place upgrade must not signal NeedsRebuild")
|
||||
}
|
||||
}
|
||||
|
||||
// TestOpenWithInPlaceFailureDoesNotStamp: a failing in-place step makes Open
|
||||
// return an error and leaves the stored version unchanged, so the next open
|
||||
// retries the upgrade rather than treating it as done.
|
||||
func TestOpenWithInPlaceFailureDoesNotStamp(t *testing.T) {
|
||||
path := filepath.Join(t.TempDir(), "store.sqlite")
|
||||
s, err := Open(path)
|
||||
if err != nil {
|
||||
t.Fatalf("first open: %v", err)
|
||||
}
|
||||
if err := s.Close(); err != nil {
|
||||
t.Fatalf("close: %v", err)
|
||||
}
|
||||
// A fresh Open now stamps the current version (>= 2); knock it back to the
|
||||
// v1 baseline so openWith drives the v1->v2 arm and the failing step runs.
|
||||
withRawDB(t, path, func(db *sql.DB) {
|
||||
if _, err := db.Exec(`PRAGMA user_version = 1`); err != nil {
|
||||
t.Fatalf("reset to v1 baseline: %v", err)
|
||||
}
|
||||
})
|
||||
|
||||
boom := schemaMigration{version: 2, name: "boom", inPlace: func(*sql.Tx) error {
|
||||
return sql.ErrConnDone
|
||||
}}
|
||||
if _, err := openWith(path, 2, []schemaMigration{boom}, false); err == nil {
|
||||
t.Fatal("expected openWith to fail when an in-place step errors")
|
||||
}
|
||||
|
||||
withRawDB(t, path, func(db *sql.DB) {
|
||||
var v int
|
||||
if err := db.QueryRow("PRAGMA user_version").Scan(&v); err != nil {
|
||||
t.Fatalf("read user_version: %v", err)
|
||||
}
|
||||
if v != 1 {
|
||||
t.Fatalf("user_version = %d after a failed migration, want 1 (unstamped, so the next open retries)", v)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// TestOpenWithMemoryUnderWipePlanStampsWithoutError: an in-memory store under a
|
||||
// plan that would wipe an on-disk DB must not attempt a file removal — it is
|
||||
// always fresh and simply stamps the current version.
|
||||
func TestOpenWithMemoryUnderWipePlanStampsWithoutError(t *testing.T) {
|
||||
rebuildV2 := schemaMigration{version: 2, name: "typed-col", rebuild: true}
|
||||
// stored==0, current==2, a pending rebuild => plan.wipe==true; the memory
|
||||
// guard must skip the wipe and stamp anyway.
|
||||
s, err := openWith(":memory:", 2, []schemaMigration{rebuildV2}, false) // memory never wipes
|
||||
if err != nil {
|
||||
t.Fatalf("openWith :memory: under wipe plan: %v", err)
|
||||
}
|
||||
defer s.Close()
|
||||
if v, _ := readUserVersion(s.db); v != 2 {
|
||||
t.Fatalf("user_version = %d, want 2", v)
|
||||
}
|
||||
if s.NeedsRebuild() {
|
||||
t.Fatal(":memory: must never report a wipe (nothing to remove)")
|
||||
}
|
||||
}
|
||||
|
||||
// TestNeedsRebuildSignalAfterWipe: a store written by a newer build is wiped on
|
||||
// open and reports NeedsRebuild so the daemon forces a full re-index.
|
||||
func TestNeedsRebuildSignalAfterWipe(t *testing.T) {
|
||||
path := filepath.Join(t.TempDir(), "store.sqlite")
|
||||
s, err := Open(path)
|
||||
if err != nil {
|
||||
t.Fatalf("first open: %v", err)
|
||||
}
|
||||
if err := s.Close(); err != nil {
|
||||
t.Fatalf("close: %v", err)
|
||||
}
|
||||
withRawDB(t, path, func(db *sql.DB) {
|
||||
if _, err := db.Exec(`PRAGMA user_version = 999`); err != nil {
|
||||
t.Fatalf("set future version: %v", err)
|
||||
}
|
||||
})
|
||||
s2, err := Open(path, WithRebuild()) // daemon-equivalent: lock held, rebuild permitted
|
||||
if err != nil {
|
||||
t.Fatalf("reopen newer DB: %v", err)
|
||||
}
|
||||
defer s2.Close()
|
||||
if !s2.NeedsRebuild() {
|
||||
t.Fatal("a wiped store must report NeedsRebuild so the daemon re-indexes")
|
||||
}
|
||||
}
|
||||
|
||||
// TestSchemaMigrationsWellFormed asserts the shipped registry is valid and that
|
||||
// the validator rejects the dangerous misconfigurations — above all, bumping
|
||||
// currentSchemaVersion without appending a matching migration.
|
||||
func TestSchemaMigrationsWellFormed(t *testing.T) {
|
||||
if err := validateSchemaMigrations(currentSchemaVersion, schemaMigrations); err != nil {
|
||||
t.Fatalf("shipped registry is invalid: %v", err)
|
||||
}
|
||||
|
||||
inPlace := func(*sql.Tx) error { return nil }
|
||||
bad := []struct {
|
||||
name string
|
||||
current int
|
||||
migs []schemaMigration
|
||||
}{
|
||||
{"bumped version with no migration", 2, nil},
|
||||
{"highest below current", 3, []schemaMigration{{version: 2, name: "x", rebuild: true}}},
|
||||
{"both strategies set", 2, []schemaMigration{{version: 2, name: "x", rebuild: true, inPlace: inPlace}}},
|
||||
{"neither strategy set", 2, []schemaMigration{{version: 2, name: "x"}}},
|
||||
{"not strictly ascending", 3, []schemaMigration{{version: 2, name: "a", rebuild: true}, {version: 2, name: "b", rebuild: true}}},
|
||||
{"v1 entry (baseline is implicit)", 1, []schemaMigration{{version: 1, name: "a", rebuild: true}}},
|
||||
}
|
||||
for _, c := range bad {
|
||||
if err := validateSchemaMigrations(c.current, c.migs); err == nil {
|
||||
t.Errorf("%s: expected a validation error, got nil", c.name)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestOpenDedupesFnValuePlaceholders drives the shipped v2 migration through the
|
||||
// real Open composition: a store knocked back to the v1 baseline with duplicate
|
||||
// fn-value placeholder edges is deduped in place on reopen — one survivor per
|
||||
// (from_id, to_id), the MIN(id) row kept — while a distinct placeholder, a
|
||||
// resolved edge, and an ordinary unresolved stub are untouched, and the version
|
||||
// stamps to current. Covers both the bare and the multi-repo COPY-rewrite form.
|
||||
func TestOpenDedupesFnValuePlaceholders(t *testing.T) {
|
||||
path := filepath.Join(t.TempDir(), "store.sqlite")
|
||||
|
||||
s, err := Open(path)
|
||||
if err != nil {
|
||||
t.Fatalf("first open: %v", err)
|
||||
}
|
||||
// Seed edges by explicit id so the MIN(id) survivors are predictable. The
|
||||
// is_unresolved column is generated, so it is omitted from the INSERT.
|
||||
ins := `INSERT INTO edges (id, from_id, to_id, kind, file_path, line) VALUES (?,?,?,?,?,?)`
|
||||
seed := []struct {
|
||||
id int
|
||||
from, to string
|
||||
kind string
|
||||
line int
|
||||
}{
|
||||
// Duplicate bare placeholders: same (from,to), distinct lines. Keep id 1.
|
||||
{1, "a", "unresolved::fnvalue::handler", "references", 10},
|
||||
{2, "a", "unresolved::fnvalue::handler", "references", 20},
|
||||
{3, "a", "unresolved::fnvalue::handler", "references", 30},
|
||||
// A distinct placeholder (different name) — must survive untouched.
|
||||
{4, "a", "unresolved::fnvalue::other", "references", 10},
|
||||
// Duplicate multi-repo COPY-rewrite placeholders — exercises the
|
||||
// is_unresolved infix branch of the migration. Keep id 5.
|
||||
{5, "b", "r::unresolved::fnvalue::handler", "references", 10},
|
||||
{6, "b", "r::unresolved::fnvalue::handler", "references", 20},
|
||||
// A resolved edge and an ordinary unresolved stub — never touched.
|
||||
{7, "a", "b", "calls", 1},
|
||||
{8, "a", "unresolved::Foo", "calls", 1},
|
||||
}
|
||||
for _, r := range seed {
|
||||
if _, err := s.db.Exec(ins, r.id, r.from, r.to, r.kind, "f.go", r.line); err != nil {
|
||||
t.Fatalf("seed edge %d: %v", r.id, err)
|
||||
}
|
||||
}
|
||||
if err := s.Close(); err != nil {
|
||||
t.Fatalf("close: %v", err)
|
||||
}
|
||||
// Knock the store back to the v1 baseline so the reopen runs the v2 dedup.
|
||||
withRawDB(t, path, func(db *sql.DB) {
|
||||
if _, err := db.Exec(`PRAGMA user_version = 1`); err != nil {
|
||||
t.Fatalf("reset to v1 baseline: %v", err)
|
||||
}
|
||||
})
|
||||
|
||||
s2, err := Open(path)
|
||||
if err != nil {
|
||||
t.Fatalf("reopen for dedup: %v", err)
|
||||
}
|
||||
defer s2.Close()
|
||||
|
||||
if v, _ := readUserVersion(s2.db); v != currentSchemaVersion {
|
||||
t.Fatalf("user_version after dedup = %d, want %d", v, currentSchemaVersion)
|
||||
}
|
||||
|
||||
present := func(id int) bool {
|
||||
var n int
|
||||
if err := s2.db.QueryRow(`SELECT COUNT(*) FROM edges WHERE id = ?`, id).Scan(&n); err != nil {
|
||||
t.Fatalf("count id %d: %v", id, err)
|
||||
}
|
||||
return n == 1
|
||||
}
|
||||
// Bare-form dedup keeps the MIN(id) survivor and drops the rest.
|
||||
if !present(1) || present(2) || present(3) {
|
||||
t.Fatalf("bare dedup wrong: want keep 1 / drop 2,3; got 1=%v 2=%v 3=%v", present(1), present(2), present(3))
|
||||
}
|
||||
// A distinct placeholder pair survives.
|
||||
if !present(4) {
|
||||
t.Fatal("distinct fn-value placeholder (id 4) was wrongly deleted")
|
||||
}
|
||||
// Multi-repo infix dedup keeps the MIN(id) survivor and drops the rest.
|
||||
if !present(5) || present(6) {
|
||||
t.Fatalf("multi-repo dedup wrong: want keep 5 / drop 6; got 5=%v 6=%v", present(5), present(6))
|
||||
}
|
||||
// A resolved edge and an ordinary unresolved stub must be untouched.
|
||||
if !present(7) {
|
||||
t.Fatal("resolved edge (id 7) must survive the placeholder dedup")
|
||||
}
|
||||
if !present(8) {
|
||||
t.Fatal("ordinary unresolved stub (id 8) must survive the placeholder dedup")
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,639 @@
|
||||
package store_sqlite
|
||||
|
||||
import (
|
||||
"iter"
|
||||
"sort"
|
||||
|
||||
"github.com/zzet/gortex/internal/graph"
|
||||
)
|
||||
|
||||
// This file implements the trivial SQL aggregator / scanner optional
|
||||
// capability interfaces from graph.Store. Each method pushes its
|
||||
// GROUP BY / WHERE / COUNT into SQLite so the planner drives it through
|
||||
// the schema's secondary indexes, returning only the aggregate rows
|
||||
// instead of materialising the whole node / edge table Go-side.
|
||||
//
|
||||
// Conventions shared across these methods:
|
||||
// - Empty / nil input returns nil (parity with the in-memory store).
|
||||
// - Input id / kind slices are deduped before they reach the IN-list.
|
||||
// - Large IN-lists are chunked by lookupChunkSize.
|
||||
// - agg-prefixed helpers are local to this file.
|
||||
|
||||
var (
|
||||
_ graph.InEdgeCounter = (*Store)(nil)
|
||||
_ graph.NodeIDsByKinds = (*Store)(nil)
|
||||
_ graph.EdgeKindCounter = (*Store)(nil)
|
||||
_ graph.NodeDegreeByKinds = (*Store)(nil)
|
||||
_ graph.NodesInFilesByKindFinder = (*Store)(nil)
|
||||
_ graph.FileImportAggregator = (*Store)(nil)
|
||||
_ graph.InDegreeForNodes = (*Store)(nil)
|
||||
_ graph.CrossRepoEdgeAggregator = (*Store)(nil)
|
||||
_ graph.FileImporters = (*Store)(nil)
|
||||
_ graph.FileSymbolNamesByPaths = (*Store)(nil)
|
||||
_ graph.EdgesByKindsScanner = (*Store)(nil)
|
||||
_ graph.NodesByKindsScanner = (*Store)(nil)
|
||||
_ graph.EdgeAdjacencyForKinds = (*Store)(nil)
|
||||
_ graph.NodeDegreeAggregator = (*Store)(nil)
|
||||
_ graph.NodeFanAggregator = (*Store)(nil)
|
||||
)
|
||||
|
||||
// aggDedupeEdgeKinds drops empties and duplicates from an edge-kind
|
||||
// slice, preserving first-seen order; returns the kinds widened to the
|
||||
// []any an IN-list binds.
|
||||
func aggDedupeEdgeKinds(kinds []graph.EdgeKind) (uniq []graph.EdgeKind, args []any) {
|
||||
seen := make(map[graph.EdgeKind]struct{}, len(kinds))
|
||||
for _, k := range kinds {
|
||||
if k == "" {
|
||||
continue
|
||||
}
|
||||
if _, ok := seen[k]; ok {
|
||||
continue
|
||||
}
|
||||
seen[k] = struct{}{}
|
||||
uniq = append(uniq, k)
|
||||
args = append(args, string(k))
|
||||
}
|
||||
return uniq, args
|
||||
}
|
||||
|
||||
// aggDedupeNodeKinds is the node-kind twin of aggDedupeEdgeKinds.
|
||||
func aggDedupeNodeKinds(kinds []graph.NodeKind) (uniq []graph.NodeKind, args []any) {
|
||||
seen := make(map[graph.NodeKind]struct{}, len(kinds))
|
||||
for _, k := range kinds {
|
||||
if k == "" {
|
||||
continue
|
||||
}
|
||||
if _, ok := seen[k]; ok {
|
||||
continue
|
||||
}
|
||||
seen[k] = struct{}{}
|
||||
uniq = append(uniq, k)
|
||||
args = append(args, string(k))
|
||||
}
|
||||
return uniq, args
|
||||
}
|
||||
|
||||
// InEdgeCountsByKind returns per-target incoming-edge counts for the
|
||||
// supplied edge kinds, grouped server-side via edges_by_to.
|
||||
func (s *Store) InEdgeCountsByKind(kinds []graph.EdgeKind) map[string]int {
|
||||
_, args := aggDedupeEdgeKinds(kinds)
|
||||
if len(args) == 0 {
|
||||
return nil
|
||||
}
|
||||
q := `SELECT to_id, COUNT(*) FROM edges WHERE kind IN (` + inPlaceholders(len(args)) + `) GROUP BY to_id`
|
||||
rows, err := s.db.Query(q, args...)
|
||||
panicOnFatal(err)
|
||||
if rows == nil {
|
||||
// swallowed teardown-race error: read returns empty (see panicOnFatal)
|
||||
return nil
|
||||
}
|
||||
defer rows.Close()
|
||||
out := make(map[string]int)
|
||||
for rows.Next() {
|
||||
var id string
|
||||
var n int
|
||||
panicOnFatal(rows.Scan(&id, &n))
|
||||
out[id] = n
|
||||
}
|
||||
panicOnFatal(rows.Err())
|
||||
return out
|
||||
}
|
||||
|
||||
// NodeIDsByKinds returns the deduplicated IDs of every node whose kind
|
||||
// is in the supplied set.
|
||||
func (s *Store) NodeIDsByKinds(kinds []graph.NodeKind) []string {
|
||||
_, args := aggDedupeNodeKinds(kinds)
|
||||
if len(args) == 0 {
|
||||
return nil
|
||||
}
|
||||
q := `SELECT id FROM nodes WHERE kind IN (` + inPlaceholders(len(args)) + `) ORDER BY id`
|
||||
rows, err := s.db.Query(q, args...)
|
||||
panicOnFatal(err)
|
||||
if rows == nil {
|
||||
// swallowed teardown-race error: read returns empty (see panicOnFatal)
|
||||
return nil
|
||||
}
|
||||
defer rows.Close()
|
||||
var out []string
|
||||
for rows.Next() {
|
||||
var id string
|
||||
panicOnFatal(rows.Scan(&id))
|
||||
out = append(out, id)
|
||||
}
|
||||
panicOnFatal(rows.Err())
|
||||
return out
|
||||
}
|
||||
|
||||
// EdgeKindCounts returns one entry per distinct edge kind with its
|
||||
// occurrence count across the whole graph.
|
||||
func (s *Store) EdgeKindCounts() map[graph.EdgeKind]int {
|
||||
rows, err := s.db.Query(`SELECT kind, COUNT(*) FROM edges GROUP BY kind`)
|
||||
panicOnFatal(err)
|
||||
if rows == nil {
|
||||
// swallowed teardown-race error: read returns empty (see panicOnFatal)
|
||||
return nil
|
||||
}
|
||||
defer rows.Close()
|
||||
out := make(map[graph.EdgeKind]int)
|
||||
for rows.Next() {
|
||||
var kind string
|
||||
var n int
|
||||
panicOnFatal(rows.Scan(&kind, &n))
|
||||
out[graph.EdgeKind(kind)] = n
|
||||
}
|
||||
panicOnFatal(rows.Err())
|
||||
return out
|
||||
}
|
||||
|
||||
// NodeDegreeByKinds returns total in/out degree for every node whose
|
||||
// kind is in the set (optionally under pathPrefix); UsageInCount is
|
||||
// always 0 for this capability.
|
||||
func (s *Store) NodeDegreeByKinds(kinds []graph.NodeKind, pathPrefix string) []graph.NodeDegreeRow {
|
||||
_, kindArgs := aggDedupeNodeKinds(kinds)
|
||||
if len(kindArgs) == 0 {
|
||||
return nil
|
||||
}
|
||||
args := append([]any(nil), kindArgs...)
|
||||
q := `SELECT n.id,
|
||||
(SELECT COUNT(*) FROM edges e WHERE e.to_id = n.id) AS in_count,
|
||||
(SELECT COUNT(*) FROM edges e WHERE e.from_id = n.id) AS out_count
|
||||
FROM nodes n
|
||||
WHERE n.kind IN (` + inPlaceholders(len(kindArgs)) + `)`
|
||||
if pathPrefix != "" {
|
||||
q += ` AND n.file_path LIKE ? ESCAPE '\'`
|
||||
args = append(args, escapeLikePattern(pathPrefix)+"%")
|
||||
}
|
||||
q += ` ORDER BY n.id`
|
||||
rows, err := s.db.Query(q, args...)
|
||||
panicOnFatal(err)
|
||||
if rows == nil {
|
||||
// swallowed teardown-race error: read returns empty (see panicOnFatal)
|
||||
return nil
|
||||
}
|
||||
defer rows.Close()
|
||||
var out []graph.NodeDegreeRow
|
||||
for rows.Next() {
|
||||
var r graph.NodeDegreeRow
|
||||
panicOnFatal(rows.Scan(&r.NodeID, &r.InCount, &r.OutCount))
|
||||
out = append(out, r)
|
||||
}
|
||||
panicOnFatal(rows.Err())
|
||||
return out
|
||||
}
|
||||
|
||||
// NodesInFilesByKind returns every node living in one of the supplied
|
||||
// files whose kind is in the supplied set.
|
||||
func (s *Store) NodesInFilesByKind(files []string, kinds []graph.NodeKind) []*graph.Node {
|
||||
uniqFiles := dedupeNonEmpty(files)
|
||||
_, kindArgs := aggDedupeNodeKinds(kinds)
|
||||
if len(uniqFiles) == 0 || len(kindArgs) == 0 {
|
||||
return nil
|
||||
}
|
||||
var out []*graph.Node
|
||||
for i := 0; i < len(uniqFiles); i += lookupChunkSize {
|
||||
end := minInt(i+lookupChunkSize, len(uniqFiles))
|
||||
chunk := uniqFiles[i:end]
|
||||
args := append(toAnyArgs(chunk), kindArgs...)
|
||||
q := `SELECT ` + lookupNodeCols + ` FROM nodes WHERE file_path IN (` +
|
||||
inPlaceholders(len(chunk)) + `) AND kind IN (` + inPlaceholders(len(kindArgs)) + `) ORDER BY id`
|
||||
out = append(out, s.queryNodesSQL(q, args...)...)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// FileImportCounts returns per-target-file incoming-import counts. A
|
||||
// nil scope counts every import edge; a non-nil scope bounds counts to
|
||||
// edges whose target node ID lies in the slice (empty non-nil => nil).
|
||||
func (s *Store) FileImportCounts(scope []string) []graph.FileImportCountRow {
|
||||
if scope != nil && len(scope) == 0 {
|
||||
return nil
|
||||
}
|
||||
base := `SELECT COALESCE(NULLIF(n.file_path, ''), n.id) AS path, COUNT(*) AS cnt
|
||||
FROM edges e JOIN nodes n ON e.to_id = n.id
|
||||
WHERE e.kind = ?`
|
||||
args := []any{string(graph.EdgeImports)}
|
||||
fileToCount := make(map[string]int)
|
||||
if scope == nil {
|
||||
q := base + ` GROUP BY path`
|
||||
aggScanImportCounts(s, q, args, fileToCount)
|
||||
} else {
|
||||
uniq := dedupeNonEmpty(scope)
|
||||
if len(uniq) == 0 {
|
||||
return nil
|
||||
}
|
||||
for i := 0; i < len(uniq); i += lookupChunkSize {
|
||||
end := minInt(i+lookupChunkSize, len(uniq))
|
||||
chunk := uniq[i:end]
|
||||
q := base + ` AND e.to_id IN (` + inPlaceholders(len(chunk)) + `) GROUP BY path`
|
||||
aggScanImportCounts(s, q, append(append([]any(nil), args...), toAnyArgs(chunk)...), fileToCount)
|
||||
}
|
||||
}
|
||||
if len(fileToCount) == 0 {
|
||||
return nil
|
||||
}
|
||||
out := make([]graph.FileImportCountRow, 0, len(fileToCount))
|
||||
for path, cnt := range fileToCount {
|
||||
out = append(out, graph.FileImportCountRow{FilePath: path, Count: cnt})
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// aggScanImportCounts runs an import-count query and folds the (path,
|
||||
// count) rows into the accumulator (chunked scopes can revisit a path).
|
||||
func aggScanImportCounts(s *Store, q string, args []any, acc map[string]int) {
|
||||
rows, err := s.db.Query(q, args...)
|
||||
panicOnFatal(err)
|
||||
if rows == nil {
|
||||
// swallowed teardown-race error: read returns empty (see panicOnFatal)
|
||||
return
|
||||
}
|
||||
defer rows.Close()
|
||||
for rows.Next() {
|
||||
var path string
|
||||
var cnt int
|
||||
panicOnFatal(rows.Scan(&path, &cnt))
|
||||
acc[path] += cnt
|
||||
}
|
||||
panicOnFatal(rows.Err())
|
||||
}
|
||||
|
||||
// InDegreeForNodes returns total incoming-edge counts (any kind) for
|
||||
// the supplied node id set.
|
||||
func (s *Store) InDegreeForNodes(ids []string) map[string]int {
|
||||
uniq := dedupeNonEmpty(ids)
|
||||
if len(uniq) == 0 {
|
||||
return nil
|
||||
}
|
||||
out := make(map[string]int)
|
||||
for i := 0; i < len(uniq); i += lookupChunkSize {
|
||||
end := minInt(i+lookupChunkSize, len(uniq))
|
||||
chunk := uniq[i:end]
|
||||
q := `SELECT to_id, COUNT(*) FROM edges WHERE to_id IN (` +
|
||||
inPlaceholders(len(chunk)) + `) GROUP BY to_id`
|
||||
rows, err := s.db.Query(q, toAnyArgs(chunk)...)
|
||||
panicOnFatal(err)
|
||||
if rows == nil {
|
||||
// swallowed teardown-race error: read returns empty (see panicOnFatal)
|
||||
return out
|
||||
}
|
||||
for rows.Next() {
|
||||
var id string
|
||||
var n int
|
||||
panicOnFatal(rows.Scan(&id, &n))
|
||||
out[id] = n
|
||||
}
|
||||
panicOnFatal(rows.Err())
|
||||
_ = rows.Close()
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// CrossRepoEdgeCounts returns pre-grouped cross-repo edge counts keyed
|
||||
// by (base kind, from-repo, to-repo). Cross-repo kinds are those
|
||||
// graph.BaseKindForCrossRepo recognises; the count is reported under
|
||||
// the base kind.
|
||||
func (s *Store) CrossRepoEdgeCounts() []graph.CrossRepoEdgeRow {
|
||||
q := `SELECT e.kind, nf.repo_prefix, nt.repo_prefix, COUNT(*)
|
||||
FROM edges e
|
||||
JOIN nodes nf ON e.from_id = nf.id
|
||||
JOIN nodes nt ON e.to_id = nt.id
|
||||
WHERE nf.repo_prefix <> nt.repo_prefix
|
||||
GROUP BY e.kind, nf.repo_prefix, nt.repo_prefix`
|
||||
rows, err := s.db.Query(q)
|
||||
panicOnFatal(err)
|
||||
if rows == nil {
|
||||
// swallowed teardown-race error: read returns empty (see panicOnFatal)
|
||||
return nil
|
||||
}
|
||||
defer rows.Close()
|
||||
// Aggregate keyed by the edge's OWN kind (cross_repo_*), NOT the base.
|
||||
// BaseKindForCrossRepo is used only as the recogniser that decides
|
||||
// whether an edge participates — parity with the in-memory store.
|
||||
type key struct {
|
||||
kind graph.EdgeKind
|
||||
from string
|
||||
to string
|
||||
}
|
||||
acc := make(map[key]int)
|
||||
for rows.Next() {
|
||||
var kind, from, to string
|
||||
var n int
|
||||
panicOnFatal(rows.Scan(&kind, &from, &to, &n))
|
||||
ek := graph.EdgeKind(kind)
|
||||
if _, ok := graph.BaseKindForCrossRepo(ek); !ok {
|
||||
continue
|
||||
}
|
||||
acc[key{kind: ek, from: from, to: to}] += n
|
||||
}
|
||||
panicOnFatal(rows.Err())
|
||||
if len(acc) == 0 {
|
||||
return nil
|
||||
}
|
||||
out := make([]graph.CrossRepoEdgeRow, 0, len(acc))
|
||||
for k, n := range acc {
|
||||
out = append(out, graph.CrossRepoEdgeRow{Kind: k.kind, FromRepo: k.from, ToRepo: k.to, Count: n})
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// FileImporters returns the importing-node rows for every EdgeImports
|
||||
// edge whose target's FilePath OR ID equals filePath.
|
||||
func (s *Store) FileImporters(filePath string) []graph.FileImporterRow {
|
||||
if filePath == "" {
|
||||
return nil
|
||||
}
|
||||
q := `SELECT nf.file_path, nf.id, nf.name, nf.kind
|
||||
FROM edges e
|
||||
JOIN nodes nt ON e.to_id = nt.id
|
||||
JOIN nodes nf ON e.from_id = nf.id
|
||||
WHERE e.kind = ? AND (nt.file_path = ? OR nt.id = ?)
|
||||
ORDER BY nf.file_path`
|
||||
rows, err := s.db.Query(q, string(graph.EdgeImports), filePath, filePath)
|
||||
panicOnFatal(err)
|
||||
if rows == nil {
|
||||
// swallowed teardown-race error: read returns empty (see panicOnFatal)
|
||||
return nil
|
||||
}
|
||||
defer rows.Close()
|
||||
var out []graph.FileImporterRow
|
||||
for rows.Next() {
|
||||
var r graph.FileImporterRow
|
||||
var kind string
|
||||
panicOnFatal(rows.Scan(&r.FromFile, &r.FromID, &r.FromName, &kind))
|
||||
r.FromKind = graph.NodeKind(kind)
|
||||
out = append(out, r)
|
||||
}
|
||||
panicOnFatal(rows.Err())
|
||||
return out
|
||||
}
|
||||
|
||||
// FileSymbolNamesByPaths returns the distinct (file, name) pairs for
|
||||
// nodes in the supplied paths whose kind is in the set, sorted by
|
||||
// (file, name).
|
||||
func (s *Store) FileSymbolNamesByPaths(paths []string, kinds []graph.NodeKind) []graph.FileSymbolNameRow {
|
||||
uniqPaths := dedupeNonEmpty(paths)
|
||||
_, kindArgs := aggDedupeNodeKinds(kinds)
|
||||
if len(uniqPaths) == 0 || len(kindArgs) == 0 {
|
||||
return nil
|
||||
}
|
||||
var out []graph.FileSymbolNameRow
|
||||
for i := 0; i < len(uniqPaths); i += lookupChunkSize {
|
||||
end := minInt(i+lookupChunkSize, len(uniqPaths))
|
||||
chunk := uniqPaths[i:end]
|
||||
args := append(toAnyArgs(chunk), kindArgs...)
|
||||
q := `SELECT DISTINCT file_path, name FROM nodes WHERE file_path IN (` +
|
||||
inPlaceholders(len(chunk)) + `) AND kind IN (` + inPlaceholders(len(kindArgs)) + `)`
|
||||
rows, err := s.db.Query(q, args...)
|
||||
panicOnFatal(err)
|
||||
if rows == nil {
|
||||
// swallowed teardown-race error: read returns empty (see panicOnFatal)
|
||||
return out
|
||||
}
|
||||
for rows.Next() {
|
||||
var r graph.FileSymbolNameRow
|
||||
panicOnFatal(rows.Scan(&r.FilePath, &r.Name))
|
||||
out = append(out, r)
|
||||
}
|
||||
panicOnFatal(rows.Err())
|
||||
_ = rows.Close()
|
||||
}
|
||||
sort.Slice(out, func(i, j int) bool {
|
||||
if out[i].FilePath != out[j].FilePath {
|
||||
return out[i].FilePath < out[j].FilePath
|
||||
}
|
||||
return out[i].Name < out[j].Name
|
||||
})
|
||||
return out
|
||||
}
|
||||
|
||||
// EdgesByKinds streams every edge whose kind is in the supplied set;
|
||||
// honours early-stop. Empty kinds yields nothing.
|
||||
func (s *Store) EdgesByKinds(kinds []graph.EdgeKind) iter.Seq[*graph.Edge] {
|
||||
_, args := aggDedupeEdgeKinds(kinds)
|
||||
return func(yield func(*graph.Edge) bool) {
|
||||
if len(args) == 0 {
|
||||
return
|
||||
}
|
||||
q := `SELECT ` + lookupEdgeCols + ` FROM edges WHERE kind IN (` +
|
||||
inPlaceholders(len(args)) + `) ORDER BY id`
|
||||
for _, e := range s.queryEdgesSQL(q, args...) {
|
||||
if e == nil {
|
||||
continue
|
||||
}
|
||||
if !yield(e) {
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// externalCallTargetPredicate selects edges whose target is an
|
||||
// external-package terminal (dep:: / stdlib:: / external::, including the
|
||||
// per-repo-prefixed stdlib form) or an already-materialised
|
||||
// external-call:: node. Shared verbatim by ExternalCallCandidateEdges and
|
||||
// the edges_external partial index (schema.go) so SQLite matches the
|
||||
// partial index for the query — keep the two identical.
|
||||
const externalCallTargetPredicate = `(to_id GLOB 'dep::*' OR to_id GLOB 'external::*' OR to_id GLOB 'stdlib::*' OR to_id GLOB '*::stdlib::*' OR to_id GLOB 'external-call::*')`
|
||||
|
||||
// ExternalCallCandidateEdges implements graph.ExternalCallCandidates: it
|
||||
// returns only the call / reference edges the external-call synthesizer
|
||||
// might act on, selected server-side so the whole call-edge table never
|
||||
// crosses into Go just to be prefix-filtered. The GLOB predicate is
|
||||
// served by the edges_external partial index.
|
||||
func (s *Store) ExternalCallCandidateEdges() []*graph.Edge {
|
||||
q := `SELECT ` + lookupEdgeCols + ` FROM edges
|
||||
WHERE kind IN ('calls','references') AND ` + externalCallTargetPredicate + `
|
||||
ORDER BY id`
|
||||
return s.queryEdgesSQL(q)
|
||||
}
|
||||
|
||||
// NodesByKinds returns every node whose kind is in the supplied set.
|
||||
func (s *Store) NodesByKinds(kinds []graph.NodeKind) []*graph.Node {
|
||||
_, args := aggDedupeNodeKinds(kinds)
|
||||
if len(args) == 0 {
|
||||
return nil
|
||||
}
|
||||
q := `SELECT ` + lookupNodeCols + ` FROM nodes WHERE kind IN (` +
|
||||
inPlaceholders(len(args)) + `) ORDER BY id`
|
||||
return s.queryNodesSQL(q, args...)
|
||||
}
|
||||
|
||||
// EdgeAdjacencyForKinds streams (from, to) id pairs for edges whose
|
||||
// kind is in edgeKinds and whose endpoints both have a kind in
|
||||
// nodeKinds; honours early-stop. Empty kinds yields nothing.
|
||||
func (s *Store) EdgeAdjacencyForKinds(edgeKinds []graph.EdgeKind, nodeKinds []graph.NodeKind) iter.Seq[[2]string] {
|
||||
_, eArgs := aggDedupeEdgeKinds(edgeKinds)
|
||||
_, nArgs := aggDedupeNodeKinds(nodeKinds)
|
||||
return func(yield func([2]string) bool) {
|
||||
if len(eArgs) == 0 || len(nArgs) == 0 {
|
||||
return
|
||||
}
|
||||
args := append([]any(nil), eArgs...)
|
||||
args = append(args, nArgs...)
|
||||
args = append(args, nArgs...)
|
||||
q := `SELECT e.from_id, e.to_id
|
||||
FROM edges e
|
||||
JOIN nodes nf ON e.from_id = nf.id
|
||||
JOIN nodes nt ON e.to_id = nt.id
|
||||
WHERE e.kind IN (` + inPlaceholders(len(eArgs)) + `)
|
||||
AND nf.kind IN (` + inPlaceholders(len(nArgs)) + `)
|
||||
AND nt.kind IN (` + inPlaceholders(len(nArgs)) + `)`
|
||||
rows, err := s.db.Query(q, args...)
|
||||
panicOnFatal(err)
|
||||
if rows == nil {
|
||||
// swallowed teardown-race error: read returns empty (see panicOnFatal)
|
||||
return
|
||||
}
|
||||
defer rows.Close()
|
||||
for rows.Next() {
|
||||
var from, to string
|
||||
panicOnFatal(rows.Scan(&from, &to))
|
||||
if !yield([2]string{from, to}) {
|
||||
return
|
||||
}
|
||||
}
|
||||
panicOnFatal(rows.Err())
|
||||
}
|
||||
}
|
||||
|
||||
// NodeDegreeCounts returns per-node in/out/usage-in edge counts for the
|
||||
// supplied id set. Unknown ids produce no row; duplicates collapse.
|
||||
func (s *Store) NodeDegreeCounts(ids []string, usageKinds []graph.EdgeKind) []graph.NodeDegreeRow {
|
||||
uniq := dedupeNonEmpty(ids)
|
||||
if len(uniq) == 0 {
|
||||
return nil
|
||||
}
|
||||
_, usageArgs := aggDedupeEdgeKinds(usageKinds)
|
||||
out := make([]graph.NodeDegreeRow, 0, len(uniq))
|
||||
for i := 0; i < len(uniq); i += lookupChunkSize {
|
||||
end := minInt(i+lookupChunkSize, len(uniq))
|
||||
chunk := uniq[i:end]
|
||||
// Usage-in subquery: a literal 0 when no usage kinds are given.
|
||||
usageExpr := `0`
|
||||
var usageInline []any
|
||||
if len(usageArgs) > 0 {
|
||||
usageExpr = `(SELECT COUNT(*) FROM edges e WHERE e.to_id = n.id AND e.kind IN (` +
|
||||
inPlaceholders(len(usageArgs)) + `))`
|
||||
usageInline = usageArgs
|
||||
}
|
||||
q := `SELECT n.id,
|
||||
(SELECT COUNT(*) FROM edges e WHERE e.to_id = n.id) AS in_count,
|
||||
(SELECT COUNT(*) FROM edges e WHERE e.from_id = n.id) AS out_count,
|
||||
` + usageExpr + ` AS usage_in
|
||||
FROM nodes n
|
||||
WHERE n.id IN (` + inPlaceholders(len(chunk)) + `)`
|
||||
// Bind order matches placeholder order: usage subquery first
|
||||
// (it appears earlier in the SELECT list), then the id IN-list.
|
||||
args := append(append([]any(nil), usageInline...), toAnyArgs(chunk)...)
|
||||
rows, err := s.db.Query(q, args...)
|
||||
panicOnFatal(err)
|
||||
if rows == nil {
|
||||
// swallowed teardown-race error: read returns empty (see panicOnFatal)
|
||||
return out
|
||||
}
|
||||
for rows.Next() {
|
||||
var r graph.NodeDegreeRow
|
||||
panicOnFatal(rows.Scan(&r.NodeID, &r.InCount, &r.OutCount, &r.UsageInCount))
|
||||
out = append(out, r)
|
||||
}
|
||||
panicOnFatal(rows.Err())
|
||||
_ = rows.Close()
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// NodeFanCounts returns per-node fan-in (incoming edges in fanInKinds)
|
||||
// and fan-out (outgoing edges in fanOutKinds) for the supplied id set.
|
||||
// Unknown ids produce no row; duplicates collapse.
|
||||
func (s *Store) NodeFanCounts(ids []string, fanInKinds, fanOutKinds []graph.EdgeKind) []graph.NodeFanRow {
|
||||
uniq := dedupeNonEmpty(ids)
|
||||
if len(uniq) == 0 {
|
||||
return nil
|
||||
}
|
||||
_, inArgs := aggDedupeEdgeKinds(fanInKinds)
|
||||
_, outArgs := aggDedupeEdgeKinds(fanOutKinds)
|
||||
out := make([]graph.NodeFanRow, 0, len(uniq))
|
||||
for i := 0; i < len(uniq); i += lookupChunkSize {
|
||||
end := minInt(i+lookupChunkSize, len(uniq))
|
||||
chunk := uniq[i:end]
|
||||
|
||||
fanInExpr := `0`
|
||||
var inInline []any
|
||||
if len(inArgs) > 0 {
|
||||
fanInExpr = `(SELECT COUNT(*) FROM edges e WHERE e.to_id = n.id AND e.kind IN (` +
|
||||
inPlaceholders(len(inArgs)) + `))`
|
||||
inInline = inArgs
|
||||
}
|
||||
fanOutExpr := `0`
|
||||
var outInline []any
|
||||
if len(outArgs) > 0 {
|
||||
fanOutExpr = `(SELECT COUNT(*) FROM edges e WHERE e.from_id = n.id AND e.kind IN (` +
|
||||
inPlaceholders(len(outArgs)) + `))`
|
||||
outInline = outArgs
|
||||
}
|
||||
q := `SELECT n.id, ` + fanInExpr + ` AS fan_in, ` + fanOutExpr + ` AS fan_out
|
||||
FROM nodes n
|
||||
WHERE n.id IN (` + inPlaceholders(len(chunk)) + `)`
|
||||
// Bind order matches placeholder order in the SELECT list:
|
||||
// fan-in subquery, fan-out subquery, then the id IN-list.
|
||||
args := append([]any(nil), inInline...)
|
||||
args = append(args, outInline...)
|
||||
args = append(args, toAnyArgs(chunk)...)
|
||||
rows, err := s.db.Query(q, args...)
|
||||
panicOnFatal(err)
|
||||
if rows == nil {
|
||||
// swallowed teardown-race error: read returns empty (see panicOnFatal)
|
||||
return out
|
||||
}
|
||||
for rows.Next() {
|
||||
var r graph.NodeFanRow
|
||||
panicOnFatal(rows.Scan(&r.NodeID, &r.FanIn, &r.FanOut))
|
||||
out = append(out, r)
|
||||
}
|
||||
panicOnFatal(rows.Err())
|
||||
_ = rows.Close()
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// CommunityCrossingsByKind returns per-source crossing counts for edges
|
||||
// whose kind is in the supplied set, given a node→community map. A
|
||||
// crossing is an edge whose source community differs from its target
|
||||
// community; zero-count sources are dropped. Empty kinds or empty
|
||||
// community map returns nil. The community comparison runs Go-side
|
||||
// because community membership is not a node column.
|
||||
func (s *Store) CommunityCrossingsByKind(kinds []graph.EdgeKind, nodeToComm map[string]string) map[string]int {
|
||||
_, args := aggDedupeEdgeKinds(kinds)
|
||||
if len(args) == 0 || len(nodeToComm) == 0 {
|
||||
return nil
|
||||
}
|
||||
q := `SELECT from_id, to_id FROM edges WHERE kind IN (` + inPlaceholders(len(args)) + `)`
|
||||
rows, err := s.db.Query(q, args...)
|
||||
panicOnFatal(err)
|
||||
if rows == nil {
|
||||
// swallowed teardown-race error: read returns empty (see panicOnFatal)
|
||||
return nil
|
||||
}
|
||||
defer rows.Close()
|
||||
out := make(map[string]int)
|
||||
for rows.Next() {
|
||||
var from, to string
|
||||
panicOnFatal(rows.Scan(&from, &to))
|
||||
fromComm, ok := nodeToComm[from]
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
toComm, ok := nodeToComm[to]
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
if fromComm != toComm {
|
||||
out[from]++
|
||||
}
|
||||
}
|
||||
panicOnFatal(rows.Err())
|
||||
if len(out) == 0 {
|
||||
return nil
|
||||
}
|
||||
return out
|
||||
}
|
||||
@@ -0,0 +1,500 @@
|
||||
package store_sqlite
|
||||
|
||||
// This file implements the moderate-SQL analysis capability interfaces
|
||||
// for the SQLite graph.Store backend. Each method mirrors the in-memory
|
||||
// reference implementation in internal/graph/graph.go and is verified
|
||||
// against the same conformance suite (internal/graph/storetest).
|
||||
//
|
||||
// Shape: push the structural filter into one indexed SELECT via the raw-
|
||||
// SQL helpers (queryNodesSQL / s.db.Query), then do any Meta-dependent
|
||||
// (JSON-decoded) or distinct-counting filtering in Go. No new prepared
|
||||
// statements are added — every query rides the secondary indexes already
|
||||
// created in schema.go (edges_by_from / edges_by_to / nodes_by_kind).
|
||||
|
||||
import (
|
||||
"github.com/zzet/gortex/internal/graph"
|
||||
)
|
||||
|
||||
// Compile-time assertions: *Store satisfies each analysis capability.
|
||||
var _ graph.DeadCodeCandidator = (*Store)(nil)
|
||||
var _ graph.IfaceImplementsScanner = (*Store)(nil)
|
||||
var _ graph.MemberMethodsByType = (*Store)(nil)
|
||||
var _ graph.StructuralParentEdges = (*Store)(nil)
|
||||
var _ graph.ExtractCandidatesScanner = (*Store)(nil)
|
||||
var _ graph.CrossRepoCandidates = (*Store)(nil)
|
||||
var _ graph.ThrowerErrorSurfacer = (*Store)(nil)
|
||||
|
||||
// anaDedupeEdgeKinds drops empty / duplicate edge kinds, preserving
|
||||
// first-seen order — the EdgeKind twin of dedupeNonEmpty.
|
||||
func anaDedupeEdgeKinds(in []graph.EdgeKind) []graph.EdgeKind {
|
||||
seen := make(map[graph.EdgeKind]struct{}, len(in))
|
||||
out := make([]graph.EdgeKind, 0, len(in))
|
||||
for _, k := range in {
|
||||
if k == "" {
|
||||
continue
|
||||
}
|
||||
if _, ok := seen[k]; ok {
|
||||
continue
|
||||
}
|
||||
seen[k] = struct{}{}
|
||||
out = append(out, k)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// --- DeadCodeCandidator -------------------------------------------------
|
||||
|
||||
// DeadCodeCandidates returns nodes of the allowed kinds that have no
|
||||
// incoming edge of the corresponding allowed in-edge kinds. An empty
|
||||
// per-kind allowlist (or one that dedupes to nothing) means "any incoming
|
||||
// edge counts as usage". Mirrors graph.(*Graph).DeadCodeCandidates: the
|
||||
// candidate set is purely structural (the analysis layer applies the
|
||||
// exported / test / entry-point / synthetic post-filters in Go), so no
|
||||
// node-id exclusion happens here. The NOT-EXISTS filter runs server-side
|
||||
// per node kind.
|
||||
func (s *Store) DeadCodeCandidates(allowedNodeKinds []graph.NodeKind, allowedInEdgeKinds map[graph.NodeKind][]graph.EdgeKind) []*graph.Node {
|
||||
if len(allowedNodeKinds) == 0 {
|
||||
return nil
|
||||
}
|
||||
var out []*graph.Node
|
||||
for _, nk := range allowedNodeKinds {
|
||||
allowed := anaDedupeEdgeKinds(allowedInEdgeKinds[nk])
|
||||
anyKindCounts := len(allowed) == 0
|
||||
|
||||
var q string
|
||||
var args []any
|
||||
if anyKindCounts {
|
||||
// Any incoming edge disqualifies the node.
|
||||
q = `SELECT ` + lookupNodeCols + ` FROM nodes n
|
||||
WHERE n.kind = ?
|
||||
AND NOT EXISTS (SELECT 1 FROM edges e WHERE e.to_id = n.id)
|
||||
ORDER BY n.id`
|
||||
args = []any{string(nk)}
|
||||
} else {
|
||||
// Only an incoming edge of one of the allowed kinds counts.
|
||||
q = `SELECT ` + lookupNodeCols + ` FROM nodes n
|
||||
WHERE n.kind = ?
|
||||
AND NOT EXISTS (SELECT 1 FROM edges e WHERE e.to_id = n.id AND e.kind IN (` + inPlaceholders(len(allowed)) + `))
|
||||
ORDER BY n.id`
|
||||
args = make([]any, 0, 1+len(allowed))
|
||||
args = append(args, string(nk))
|
||||
for _, ek := range allowed {
|
||||
args = append(args, string(ek))
|
||||
}
|
||||
}
|
||||
|
||||
for _, n := range s.queryNodesSQL(q, args...) {
|
||||
if n != nil {
|
||||
out = append(out, n)
|
||||
}
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// --- IfaceImplementsScanner ---------------------------------------------
|
||||
|
||||
// IfaceImplementsRows returns one row per EdgeImplements edge whose
|
||||
// target is a KindInterface carrying Meta["methods"]. The interface's
|
||||
// decoded Meta rides on the row (callers pull the "methods" field, which
|
||||
// round-trips as []string or []any). Interfaces with no Meta or no
|
||||
// "methods" key are elided server-side.
|
||||
func (s *Store) IfaceImplementsRows() []graph.IfaceImplementsRow {
|
||||
q := `SELECT e.from_id, n.id, n.meta
|
||||
FROM edges e
|
||||
JOIN nodes n ON n.id = e.to_id
|
||||
WHERE e.kind = ? AND n.kind = ? AND n.meta IS NOT NULL`
|
||||
rows, err := s.db.Query(q, string(graph.EdgeImplements), string(graph.KindInterface))
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
var out []graph.IfaceImplementsRow
|
||||
for rows.Next() {
|
||||
var fromID, ifaceID string
|
||||
var metaBlob []byte
|
||||
if err := rows.Scan(&fromID, &ifaceID, &metaBlob); err != nil {
|
||||
continue
|
||||
}
|
||||
meta, derr := decodeMeta(metaBlob)
|
||||
if derr != nil || meta == nil {
|
||||
continue
|
||||
}
|
||||
if _, ok := meta["methods"]; !ok {
|
||||
continue
|
||||
}
|
||||
out = append(out, graph.IfaceImplementsRow{
|
||||
TypeID: fromID,
|
||||
IfaceID: ifaceID,
|
||||
IfaceMeta: meta,
|
||||
})
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// --- MemberMethodsByType ------------------------------------------------
|
||||
|
||||
// MemberMethodsByType returns typeID → []MemberMethodInfo for every
|
||||
// EdgeMemberOf edge whose source is a KindMethod. The columns come from
|
||||
// the METHOD NODE (FilePath / StartLine / RepoPrefix), matching the
|
||||
// in-memory reference. Per-type lists are deduplicated by MethodID; the
|
||||
// scan is ordered by the edge PK so the first-seen winner is stable. An
|
||||
// empty graph (no qualifying rows) returns nil.
|
||||
func (s *Store) MemberMethodsByType() map[string][]graph.MemberMethodInfo {
|
||||
q := `SELECT e.to_id, n.id, n.name, n.file_path, n.start_line, n.repo_prefix
|
||||
FROM edges e
|
||||
JOIN nodes n ON n.id = e.from_id
|
||||
WHERE e.kind = ? AND n.kind = ?
|
||||
ORDER BY e.id`
|
||||
rows, err := s.db.Query(q, string(graph.EdgeMemberOf), string(graph.KindMethod))
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
out := make(map[string][]graph.MemberMethodInfo)
|
||||
seen := make(map[string]map[string]struct{})
|
||||
for rows.Next() {
|
||||
var typeID, methodID, name, filePath, repoPrefix string
|
||||
var startLine int
|
||||
if err := rows.Scan(&typeID, &methodID, &name, &filePath, &startLine, &repoPrefix); err != nil {
|
||||
continue
|
||||
}
|
||||
if seen[typeID] == nil {
|
||||
seen[typeID] = make(map[string]struct{})
|
||||
}
|
||||
if _, ok := seen[typeID][methodID]; ok {
|
||||
continue
|
||||
}
|
||||
seen[typeID][methodID] = struct{}{}
|
||||
out[typeID] = append(out[typeID], graph.MemberMethodInfo{
|
||||
MethodID: methodID,
|
||||
Name: name,
|
||||
FilePath: filePath,
|
||||
StartLine: startLine,
|
||||
RepoPrefix: repoPrefix,
|
||||
})
|
||||
}
|
||||
if len(out) == 0 {
|
||||
// Match the in-memory reference: empty graph returns nil.
|
||||
return nil
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// --- StructuralParentEdges ----------------------------------------------
|
||||
|
||||
// StructuralParentEdges returns every Extends / Implements / Composes
|
||||
// edge whose endpoints are both Type / Interface, projected as (FromID,
|
||||
// ToID, FromKind, ToKind, Origin). Endpoints that aren't both type /
|
||||
// interface are filtered server-side. Empty graph or no matching edges
|
||||
// returns nil.
|
||||
func (s *Store) StructuralParentEdges() []graph.StructuralParentEdgeRow {
|
||||
q := `SELECT e.from_id, e.to_id, nf.kind, nt.kind, e.origin
|
||||
FROM edges e
|
||||
JOIN nodes nf ON nf.id = e.from_id
|
||||
JOIN nodes nt ON nt.id = e.to_id
|
||||
WHERE e.kind IN (?,?,?)
|
||||
AND nf.kind IN (?,?) AND nt.kind IN (?,?)
|
||||
ORDER BY e.id`
|
||||
rows, err := s.db.Query(q,
|
||||
string(graph.EdgeExtends), string(graph.EdgeImplements), string(graph.EdgeComposes),
|
||||
string(graph.KindType), string(graph.KindInterface),
|
||||
string(graph.KindType), string(graph.KindInterface),
|
||||
)
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
var out []graph.StructuralParentEdgeRow
|
||||
for rows.Next() {
|
||||
var fromID, toID, fromKind, toKind, origin string
|
||||
if err := rows.Scan(&fromID, &toID, &fromKind, &toKind, &origin); err != nil {
|
||||
continue
|
||||
}
|
||||
out = append(out, graph.StructuralParentEdgeRow{
|
||||
FromID: fromID,
|
||||
ToID: toID,
|
||||
FromKind: graph.NodeKind(fromKind),
|
||||
ToKind: graph.NodeKind(toKind),
|
||||
Origin: origin,
|
||||
})
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// --- ExtractCandidatesScanner -------------------------------------------
|
||||
|
||||
// ExtractCandidates ranks function / method nodes by extractability: line
|
||||
// span (EndLine - StartLine + 1), distinct caller fan-in, and distinct
|
||||
// callee fan-out, counting only edges whose kind is in the supplied set.
|
||||
// Rows must clear all three thresholds. Nodes with a zero StartLine /
|
||||
// EndLine are dropped; pathPrefix narrows by file-path prefix. Mirrors
|
||||
// graph.(*Graph).ExtractCandidates exactly: only KindFunction +
|
||||
// KindMethod nodes are considered, and the distinct-by-endpoint counting
|
||||
// runs Go-side over GetInEdges / GetOutEdges.
|
||||
func (s *Store) ExtractCandidates(kinds []graph.EdgeKind, minLines, minCallers, minFanOut int, pathPrefix string) []graph.ExtractCandidateRow {
|
||||
if len(kinds) == 0 {
|
||||
return nil
|
||||
}
|
||||
kindSet := make(map[graph.EdgeKind]struct{}, len(kinds))
|
||||
for _, k := range kinds {
|
||||
if k == "" {
|
||||
continue
|
||||
}
|
||||
kindSet[k] = struct{}{}
|
||||
}
|
||||
if len(kindSet) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
// Candidate nodes: function / method only, non-zero line span,
|
||||
// optional path-prefix gate.
|
||||
q := `SELECT ` + lookupNodeCols + ` FROM nodes
|
||||
WHERE kind IN (?,?) AND start_line > 0 AND end_line > 0`
|
||||
args := []any{string(graph.KindFunction), string(graph.KindMethod)}
|
||||
if pathPrefix != "" {
|
||||
q += ` AND file_path LIKE ? ESCAPE '\'`
|
||||
args = append(args, escapeLikePattern(pathPrefix)+"%")
|
||||
}
|
||||
q += ` ORDER BY id`
|
||||
nodes := s.queryNodesSQL(q, args...)
|
||||
|
||||
var out []graph.ExtractCandidateRow
|
||||
for _, n := range nodes {
|
||||
if n == nil {
|
||||
continue
|
||||
}
|
||||
lineCount := n.EndLine - n.StartLine + 1
|
||||
if lineCount < minLines {
|
||||
continue
|
||||
}
|
||||
|
||||
callerSet := make(map[string]struct{})
|
||||
for _, e := range s.GetInEdges(n.ID) {
|
||||
if e == nil {
|
||||
continue
|
||||
}
|
||||
if _, ok := kindSet[e.Kind]; !ok {
|
||||
continue
|
||||
}
|
||||
callerSet[e.From] = struct{}{}
|
||||
}
|
||||
if len(callerSet) < minCallers {
|
||||
continue
|
||||
}
|
||||
|
||||
calleeSet := make(map[string]struct{})
|
||||
for _, e := range s.GetOutEdges(n.ID) {
|
||||
if e == nil {
|
||||
continue
|
||||
}
|
||||
if _, ok := kindSet[e.Kind]; !ok {
|
||||
continue
|
||||
}
|
||||
calleeSet[e.To] = struct{}{}
|
||||
}
|
||||
if len(calleeSet) < minFanOut {
|
||||
continue
|
||||
}
|
||||
|
||||
out = append(out, graph.ExtractCandidateRow{
|
||||
NodeID: n.ID,
|
||||
Name: n.Name,
|
||||
FilePath: n.FilePath,
|
||||
StartLine: n.StartLine,
|
||||
EndLine: n.EndLine,
|
||||
LineCount: lineCount,
|
||||
CallerCount: len(callerSet),
|
||||
FanOut: len(calleeSet),
|
||||
})
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// --- CrossRepoCandidates ------------------------------------------------
|
||||
|
||||
// CrossRepoCandidates returns every edge whose kind is in baseKinds and
|
||||
// whose endpoints carry two different non-empty RepoPrefix values. The
|
||||
// edge is returned verbatim (callers rewrite Edge.CrossRepo); FromRepo /
|
||||
// ToRepo are the endpoint prefixes. Empty baseKinds returns nil; single-
|
||||
// repo graphs (or graphs whose nodes carry no RepoPrefix) yield nothing.
|
||||
func (s *Store) CrossRepoCandidates(baseKinds []graph.EdgeKind) []graph.CrossRepoCandidateRow {
|
||||
uniq := anaDedupeEdgeKinds(baseKinds)
|
||||
if len(uniq) == 0 {
|
||||
return nil
|
||||
}
|
||||
args := make([]any, 0, len(uniq))
|
||||
for _, k := range uniq {
|
||||
args = append(args, string(k))
|
||||
}
|
||||
q := `SELECT e.from_id, e.to_id, e.kind, e.file_path, e.line,
|
||||
e.confidence, e.confidence_label, e.origin, e.tier, e.cross_repo, e.meta,
|
||||
nf.repo_prefix, nt.repo_prefix
|
||||
FROM edges e
|
||||
JOIN nodes nf ON nf.id = e.from_id
|
||||
JOIN nodes nt ON nt.id = e.to_id
|
||||
WHERE e.kind IN (` + inPlaceholders(len(uniq)) + `)
|
||||
AND nf.repo_prefix <> '' AND nt.repo_prefix <> ''
|
||||
AND nf.repo_prefix <> nt.repo_prefix
|
||||
ORDER BY e.id`
|
||||
rows, err := s.db.Query(q, args...)
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
var out []graph.CrossRepoCandidateRow
|
||||
for rows.Next() {
|
||||
var (
|
||||
fromRepo, toRepo string
|
||||
e graph.Edge
|
||||
metaBlob []byte
|
||||
crossRepo int64
|
||||
)
|
||||
if err := rows.Scan(
|
||||
&e.From, &e.To, &e.Kind, &e.FilePath, &e.Line,
|
||||
&e.Confidence, &e.ConfidenceLabel, &e.Origin, &e.Tier,
|
||||
&crossRepo, &metaBlob,
|
||||
&fromRepo, &toRepo,
|
||||
); err != nil {
|
||||
continue
|
||||
}
|
||||
e.CrossRepo = crossRepo != 0
|
||||
if len(metaBlob) > 0 {
|
||||
if m, derr := decodeMeta(metaBlob); derr == nil {
|
||||
e.Meta = m
|
||||
}
|
||||
}
|
||||
edge := e
|
||||
out = append(out, graph.CrossRepoCandidateRow{
|
||||
Edge: &edge,
|
||||
FromRepo: fromRepo,
|
||||
ToRepo: toRepo,
|
||||
})
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// --- ThrowerErrorSurfacer -----------------------------------------------
|
||||
|
||||
// ThrowerErrorSurface returns one row per thrower (a node with outgoing
|
||||
// EdgeThrows edges), aggregating the distinct error targets and the
|
||||
// distinct literal error-message strings it emits (KindString nodes with
|
||||
// Meta["context"] == "error_msg", linked by EdgeEmits). pathPrefix gates
|
||||
// the EdgeThrows rows by their stored FilePath prefix. Throws counts the
|
||||
// underlying EdgeThrows edges; FilePath / Line seed from the first throws
|
||||
// edge, falling back to the thrower node's own coordinates when the edge
|
||||
// carries none — matching the in-memory reference.
|
||||
func (s *Store) ThrowerErrorSurface(pathPrefix string) []graph.ThrowerErrorRow {
|
||||
type rowAccum struct {
|
||||
row graph.ThrowerErrorRow
|
||||
targetSeen map[string]struct{}
|
||||
msgSeen map[string]struct{}
|
||||
}
|
||||
accums := make(map[string]*rowAccum)
|
||||
var order []string
|
||||
|
||||
// Pass 1: EdgeThrows aggregation (count + distinct targets), keyed by
|
||||
// thrower. The first edge (by PK insertion order) seeds FilePath /
|
||||
// Line; an empty edge file/line falls back to the thrower node.
|
||||
tq := `SELECT from_id, to_id, file_path, line FROM edges WHERE kind = ?`
|
||||
targs := []any{string(graph.EdgeThrows)}
|
||||
if pathPrefix != "" {
|
||||
tq += ` AND file_path LIKE ? ESCAPE '\'`
|
||||
targs = append(targs, escapeLikePattern(pathPrefix)+"%")
|
||||
}
|
||||
tq += ` ORDER BY id`
|
||||
trows, err := s.db.Query(tq, targs...)
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
for trows.Next() {
|
||||
var from, to, filePath string
|
||||
var line int
|
||||
if err := trows.Scan(&from, &to, &filePath, &line); err != nil {
|
||||
continue
|
||||
}
|
||||
acc := accums[from]
|
||||
if acc == nil {
|
||||
file := filePath
|
||||
ln := line
|
||||
if file == "" || ln == 0 {
|
||||
if n := s.GetNode(from); n != nil {
|
||||
if file == "" {
|
||||
file = n.FilePath
|
||||
}
|
||||
if ln == 0 {
|
||||
ln = n.StartLine
|
||||
}
|
||||
}
|
||||
}
|
||||
acc = &rowAccum{
|
||||
row: graph.ThrowerErrorRow{
|
||||
ThrowerID: from,
|
||||
FilePath: file,
|
||||
Line: ln,
|
||||
},
|
||||
targetSeen: make(map[string]struct{}),
|
||||
msgSeen: make(map[string]struct{}),
|
||||
}
|
||||
accums[from] = acc
|
||||
order = append(order, from)
|
||||
}
|
||||
acc.row.Throws++
|
||||
if _, ok := acc.targetSeen[to]; !ok {
|
||||
acc.targetSeen[to] = struct{}{}
|
||||
acc.row.ErrorTargets = append(acc.row.ErrorTargets, to)
|
||||
}
|
||||
}
|
||||
_ = trows.Close()
|
||||
if len(accums) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
// Pass 2: attach the literal error messages each thrower emits. Join
|
||||
// each thrower's EdgeEmits out-edges to KindString targets and filter
|
||||
// Meta["context"] == "error_msg" Go-side (the context lives in the
|
||||
// JSON Meta blob).
|
||||
for _, id := range order {
|
||||
acc := accums[id]
|
||||
mq := `SELECT n.name, n.meta
|
||||
FROM edges e
|
||||
JOIN nodes n ON n.id = e.to_id
|
||||
WHERE e.from_id = ? AND e.kind = ? AND n.kind = ? AND n.meta IS NOT NULL
|
||||
ORDER BY e.id`
|
||||
mrows, err := s.db.Query(mq, id, string(graph.EdgeEmits), string(graph.KindString))
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
for mrows.Next() {
|
||||
var name string
|
||||
var metaBlob []byte
|
||||
if err := mrows.Scan(&name, &metaBlob); err != nil {
|
||||
continue
|
||||
}
|
||||
meta, derr := decodeMeta(metaBlob)
|
||||
if derr != nil || meta == nil {
|
||||
continue
|
||||
}
|
||||
ctxLabel, _ := meta["context"].(string)
|
||||
if ctxLabel != "error_msg" {
|
||||
continue
|
||||
}
|
||||
if _, ok := acc.msgSeen[name]; ok {
|
||||
continue
|
||||
}
|
||||
acc.msgSeen[name] = struct{}{}
|
||||
acc.row.ErrorMsgs = append(acc.row.ErrorMsgs, name)
|
||||
}
|
||||
_ = mrows.Close()
|
||||
}
|
||||
|
||||
out := make([]graph.ThrowerErrorRow, 0, len(order))
|
||||
for _, id := range order {
|
||||
out = append(out, accums[id].row)
|
||||
}
|
||||
return out
|
||||
}
|
||||
@@ -0,0 +1,172 @@
|
||||
package store_sqlite
|
||||
|
||||
import (
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
"github.com/zzet/gortex/internal/graph"
|
||||
)
|
||||
|
||||
var _ graph.BFSCapable = (*Store)(nil)
|
||||
|
||||
// BFS runs a bounded breadth-first traversal in a single round-trip via a
|
||||
// recursive CTE — the disk-backed sibling of the in-memory
|
||||
// (*graph.Graph).BFS reference. See graph.BFSCapable for the contract;
|
||||
// the two are shadow-tested for identical hop-sets in the conformance
|
||||
// suite (storetest), including a cycle fixture.
|
||||
//
|
||||
// The recursive term joins edges on the direction's indexed column —
|
||||
// edges_by_from(from_id, kind) for a forward walk, edges_by_to(to_id,
|
||||
// kind) for a backward walk — and the nodes primary key, so it stays
|
||||
// index-driven instead of scanning the edges table (confirmed via
|
||||
// EXPLAIN QUERY PLAN in store_bfs_test.go). The nodes join also enforces
|
||||
// the "node-backed targets only" rule: an edge to an unresolved /
|
||||
// external stub with no node row is not followed. A cycle terminates on
|
||||
// the depth bound; the outer ROW_NUMBER picks each node's minimum-depth,
|
||||
// (parent, kind)-smallest discovery edge so the result is deterministic
|
||||
// and matches the in-memory walk's bfsHopLess tie-break.
|
||||
//
|
||||
// Reads run lock-free, like the store's other read paths (SQLite WAL
|
||||
// serves readers concurrently with the single serialized writer).
|
||||
func (s *Store) BFS(seeds []string, dir graph.Direction, kinds []graph.EdgeKind, maxDepth, limit int) ([]graph.BFSHop, error) {
|
||||
seen := make(map[string]struct{}, len(seeds))
|
||||
uniqSeeds := make([]string, 0, len(seeds))
|
||||
for _, sd := range seeds {
|
||||
if sd == "" {
|
||||
continue
|
||||
}
|
||||
if _, ok := seen[sd]; ok {
|
||||
continue
|
||||
}
|
||||
seen[sd] = struct{}{}
|
||||
uniqSeeds = append(uniqSeeds, sd)
|
||||
}
|
||||
if len(uniqSeeds) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
uniqKinds := anaDedupeEdgeKinds(kinds)
|
||||
|
||||
// Seed-only fast path: with no edge kinds to follow or a non-positive
|
||||
// depth bound the result is exactly the seeds at depth 0. Seeds enter
|
||||
// unconditionally (no node-backed gate), matching the in-memory
|
||||
// reference, which adds them before any traversal.
|
||||
if len(uniqKinds) == 0 || maxDepth <= 0 {
|
||||
hops := make([]graph.BFSHop, 0, len(uniqSeeds))
|
||||
for _, sd := range uniqSeeds {
|
||||
hops = append(hops, graph.BFSHop{NodeID: sd, Depth: 0})
|
||||
}
|
||||
sortBFSHops(hops)
|
||||
if limit > 0 && len(hops) > limit {
|
||||
hops = hops[:limit]
|
||||
}
|
||||
return hops, nil
|
||||
}
|
||||
|
||||
query := buildBFSQuery(dir, len(uniqSeeds), len(uniqKinds), limit > 0)
|
||||
|
||||
args := make([]any, 0, len(uniqSeeds)+1+len(uniqKinds)+1)
|
||||
for _, sd := range uniqSeeds {
|
||||
args = append(args, sd)
|
||||
}
|
||||
args = append(args, maxDepth)
|
||||
for _, k := range uniqKinds {
|
||||
args = append(args, string(k))
|
||||
}
|
||||
if limit > 0 {
|
||||
args = append(args, limit)
|
||||
}
|
||||
|
||||
rows, err := s.db.Query(query, args...)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
var out []graph.BFSHop
|
||||
for rows.Next() {
|
||||
var (
|
||||
nodeID, parentID, edgeKind string
|
||||
depth int
|
||||
)
|
||||
if err := rows.Scan(&nodeID, &depth, &parentID, &edgeKind); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out = append(out, graph.BFSHop{
|
||||
NodeID: nodeID,
|
||||
Depth: depth,
|
||||
ParentID: parentID,
|
||||
EdgeKind: graph.EdgeKind(edgeKind),
|
||||
})
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// buildBFSQuery assembles the recursive-CTE BFS statement for the given
|
||||
// direction, seed count, kind count, and whether a LIMIT is applied. It is
|
||||
// a pure string builder (no I/O) so a test can EXPLAIN QUERY PLAN the exact
|
||||
// statement and assert the recursive join stays index-driven.
|
||||
//
|
||||
// Direction selects the join columns: forward follows from_id -> to_id (the
|
||||
// discovered neighbour is the edge target), backward follows to_id ->
|
||||
// from_id (the neighbour is the edge source). The recursive term joins
|
||||
// edges on the walked node's id, so a forward walk leads with from_id (the
|
||||
// edges_by_from(from_id, kind) index) and a backward walk leads with to_id
|
||||
// (edges_by_to(to_id, kind)); the nodes join uses the nodes primary key.
|
||||
func buildBFSQuery(dir graph.Direction, nSeeds, nKinds int, withLimit bool) string {
|
||||
joinCol, nextCol := "e.from_id", "e.to_id"
|
||||
edgeIdx := "edges_by_from"
|
||||
if dir == graph.DirectionBackward {
|
||||
joinCol, nextCol = "e.to_id", "e.from_id"
|
||||
edgeIdx = "edges_by_to"
|
||||
}
|
||||
|
||||
var b strings.Builder
|
||||
b.WriteString("WITH RECURSIVE seeds(node_id) AS (VALUES ")
|
||||
for i := 0; i < nSeeds; i++ {
|
||||
if i > 0 {
|
||||
b.WriteString(", ")
|
||||
}
|
||||
b.WriteString("(?)")
|
||||
}
|
||||
b.WriteString("),\n")
|
||||
b.WriteString("bfs(node_id, depth, parent_id, edge_kind) AS (\n")
|
||||
b.WriteString(" SELECT node_id, 0, '', '' FROM seeds\n")
|
||||
b.WriteString(" UNION\n")
|
||||
b.WriteString(" SELECT " + nextCol + ", b.depth + 1, b.node_id, e.kind\n")
|
||||
b.WriteString(" FROM bfs b\n")
|
||||
// INDEXED BY forces the frontier-node seek (from_id / to_id leading)
|
||||
// instead of the planner's stats-free preference for edges_by_kind,
|
||||
// which on a hot kind would scan every edge of that kind per frontier
|
||||
// node. If the index is ever absent (a bulk-load window drops it) the
|
||||
// query errors and the engine falls back to the in-memory walk.
|
||||
b.WriteString(" JOIN edges e INDEXED BY " + edgeIdx + " ON " + joinCol + " = b.node_id\n")
|
||||
b.WriteString(" JOIN nodes n ON n.id = " + nextCol + "\n")
|
||||
b.WriteString(" WHERE b.depth < ? AND e.kind IN (" + inPlaceholders(nKinds) + ")\n")
|
||||
b.WriteString("),\n")
|
||||
b.WriteString("ranked AS (\n")
|
||||
b.WriteString(" SELECT node_id, depth, parent_id, edge_kind,\n")
|
||||
b.WriteString(" ROW_NUMBER() OVER (PARTITION BY node_id ORDER BY depth, parent_id, edge_kind) AS rn\n")
|
||||
b.WriteString(" FROM bfs\n")
|
||||
b.WriteString(")\n")
|
||||
b.WriteString("SELECT node_id, depth, parent_id, edge_kind FROM ranked WHERE rn = 1\n")
|
||||
b.WriteString("ORDER BY depth, node_id")
|
||||
if withLimit {
|
||||
b.WriteString("\nLIMIT ?")
|
||||
}
|
||||
return b.String()
|
||||
}
|
||||
|
||||
// sortBFSHops orders hops by (depth, node_id) — the same final ordering
|
||||
// the recursive-CTE query applies, used by the seed-only fast path.
|
||||
func sortBFSHops(hops []graph.BFSHop) {
|
||||
sort.Slice(hops, func(i, j int) bool {
|
||||
if hops[i].Depth != hops[j].Depth {
|
||||
return hops[i].Depth < hops[j].Depth
|
||||
}
|
||||
return hops[i].NodeID < hops[j].NodeID
|
||||
})
|
||||
}
|
||||
@@ -0,0 +1,74 @@
|
||||
package store_sqlite
|
||||
|
||||
import (
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/zzet/gortex/internal/graph"
|
||||
)
|
||||
|
||||
// queryPlan runs EXPLAIN QUERY PLAN for the given statement + args and
|
||||
// returns the joined detail lines.
|
||||
func queryPlan(t *testing.T, s *Store, query string, args ...any) string {
|
||||
t.Helper()
|
||||
rows, err := s.db.Query("EXPLAIN QUERY PLAN "+query, args...)
|
||||
if err != nil {
|
||||
t.Fatalf("EXPLAIN QUERY PLAN: %v", err)
|
||||
}
|
||||
defer rows.Close()
|
||||
var lines []string
|
||||
for rows.Next() {
|
||||
var id, parent, notused int
|
||||
var detail string
|
||||
if err := rows.Scan(&id, &parent, ¬used, &detail); err != nil {
|
||||
t.Fatalf("scan plan row: %v", err)
|
||||
}
|
||||
lines = append(lines, detail)
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
t.Fatalf("plan rows: %v", err)
|
||||
}
|
||||
return strings.Join(lines, "\n")
|
||||
}
|
||||
|
||||
// TestBFSQueryUsesEdgeIndex asserts the recursive-CTE BFS join reaches the
|
||||
// edges table through the direction's composite index (edges_by_from for a
|
||||
// forward walk, edges_by_to for a backward walk) rather than a full table
|
||||
// scan, and that the node-backed gate join uses the nodes primary key.
|
||||
func TestBFSQueryUsesEdgeIndex(t *testing.T) {
|
||||
s, err := Open(filepath.Join(t.TempDir(), "store.sqlite"))
|
||||
if err != nil {
|
||||
t.Fatalf("open: %v", err)
|
||||
}
|
||||
defer s.Close()
|
||||
|
||||
// A small fixture so the planner has real tables/indexes to plan against.
|
||||
for _, id := range []string{"A", "B", "C"} {
|
||||
s.AddNode(&graph.Node{ID: id, Kind: graph.KindFunction, Name: id, FilePath: "a.go", Language: "go"})
|
||||
}
|
||||
s.AddEdge(&graph.Edge{From: "A", To: "B", Kind: graph.EdgeCalls, Confidence: 1, Origin: graph.OriginASTResolved})
|
||||
s.AddEdge(&graph.Edge{From: "B", To: "C", Kind: graph.EdgeCalls, Confidence: 1, Origin: graph.OriginASTResolved})
|
||||
|
||||
forwardQ := buildBFSQuery(graph.DirectionForward, 1, 1, true)
|
||||
forwardPlan := queryPlan(t, s, forwardQ, "A", 3, string(graph.EdgeCalls), 50)
|
||||
if !strings.Contains(forwardPlan, "edges_by_from") {
|
||||
t.Fatalf("forward BFS plan does not use edges_by_from:\n%s", forwardPlan)
|
||||
}
|
||||
// The edges table must not be reached by a full scan (alias e).
|
||||
if strings.Contains(forwardPlan, "SCAN edges") || strings.Contains(forwardPlan, "SCAN e ") {
|
||||
t.Fatalf("forward BFS plan scans edges instead of using an index:\n%s", forwardPlan)
|
||||
}
|
||||
|
||||
backwardQ := buildBFSQuery(graph.DirectionBackward, 1, 1, true)
|
||||
backwardPlan := queryPlan(t, s, backwardQ, "C", 3, string(graph.EdgeCalls), 50)
|
||||
if !strings.Contains(backwardPlan, "edges_by_to") {
|
||||
t.Fatalf("backward BFS plan does not use edges_by_to:\n%s", backwardPlan)
|
||||
}
|
||||
if strings.Contains(backwardPlan, "SCAN edges") || strings.Contains(backwardPlan, "SCAN e ") {
|
||||
t.Fatalf("backward BFS plan scans edges instead of using an index:\n%s", backwardPlan)
|
||||
}
|
||||
|
||||
t.Logf("forward BFS query plan:\n%s", forwardPlan)
|
||||
t.Logf("backward BFS query plan:\n%s", backwardPlan)
|
||||
}
|
||||
@@ -0,0 +1,130 @@
|
||||
package store_sqlite
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
|
||||
"github.com/zzet/gortex/internal/graph"
|
||||
)
|
||||
|
||||
var (
|
||||
_ graph.BlameEnrichmentWriter = (*Store)(nil)
|
||||
_ graph.BlameEnrichmentReader = (*Store)(nil)
|
||||
)
|
||||
|
||||
const blameChunk = 180
|
||||
|
||||
const blameCols = `node_id, repo_prefix, commit_sha, email, ts`
|
||||
|
||||
func (s *Store) BulkSetBlame(repoPrefix string, rows []graph.BlameEnrichment) error {
|
||||
if len(rows) == 0 {
|
||||
return nil
|
||||
}
|
||||
s.writeMu.Lock()
|
||||
defer s.writeMu.Unlock()
|
||||
tx, err := s.db.Begin()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer tx.Rollback() //nolint:errcheck
|
||||
for start := 0; start < len(rows); start += blameChunk {
|
||||
end := start + blameChunk
|
||||
if end > len(rows) {
|
||||
end = len(rows)
|
||||
}
|
||||
batch := rows[start:end]
|
||||
args := make([]any, 0, len(batch)*5)
|
||||
stmt := make([]byte, 0, 96+len(batch)*16)
|
||||
stmt = append(stmt, "INSERT OR REPLACE INTO blame_enrichment ("...)
|
||||
stmt = append(stmt, blameCols...)
|
||||
stmt = append(stmt, ") VALUES "...)
|
||||
for i, e := range batch {
|
||||
if i > 0 {
|
||||
stmt = append(stmt, ',')
|
||||
}
|
||||
stmt = append(stmt, "(?,?,?,?,?)"...)
|
||||
args = append(args, e.NodeID, repoPrefix, e.Commit, e.Email, e.Timestamp)
|
||||
}
|
||||
if _, err := tx.Exec(string(stmt), args...); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return tx.Commit()
|
||||
}
|
||||
|
||||
func (s *Store) DeleteBlame(nodeIDs []string) error {
|
||||
if len(nodeIDs) == 0 {
|
||||
return nil
|
||||
}
|
||||
seen := make(map[string]struct{}, len(nodeIDs))
|
||||
uniq := make([]string, 0, len(nodeIDs))
|
||||
for _, id := range nodeIDs {
|
||||
if id == "" {
|
||||
continue
|
||||
}
|
||||
if _, ok := seen[id]; ok {
|
||||
continue
|
||||
}
|
||||
seen[id] = struct{}{}
|
||||
uniq = append(uniq, id)
|
||||
}
|
||||
if len(uniq) == 0 {
|
||||
return nil
|
||||
}
|
||||
s.writeMu.Lock()
|
||||
defer s.writeMu.Unlock()
|
||||
tx, err := s.db.Begin()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer tx.Rollback() //nolint:errcheck
|
||||
for start := 0; start < len(uniq); start += blameChunk {
|
||||
end := start + blameChunk
|
||||
if end > len(uniq) {
|
||||
end = len(uniq)
|
||||
}
|
||||
chunk := uniq[start:end]
|
||||
args := make([]any, len(chunk))
|
||||
stmt := make([]byte, 0, 56+len(chunk)*2)
|
||||
stmt = append(stmt, "DELETE FROM blame_enrichment WHERE node_id IN ("...)
|
||||
for i, id := range chunk {
|
||||
if i > 0 {
|
||||
stmt = append(stmt, ',')
|
||||
}
|
||||
stmt = append(stmt, '?')
|
||||
args[i] = id
|
||||
}
|
||||
stmt = append(stmt, ')')
|
||||
if _, err := tx.Exec(string(stmt), args...); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return tx.Commit()
|
||||
}
|
||||
|
||||
func (s *Store) BlameRows(repoPrefix string) []graph.BlameEnrichment {
|
||||
var (
|
||||
rows *sql.Rows
|
||||
err error
|
||||
)
|
||||
if repoPrefix == "" {
|
||||
rows, err = s.db.Query(`SELECT ` + blameCols + ` FROM blame_enrichment`)
|
||||
} else {
|
||||
rows, err = s.db.Query(`SELECT `+blameCols+` FROM blame_enrichment WHERE repo_prefix = ?`, repoPrefix)
|
||||
}
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
defer rows.Close()
|
||||
var out []graph.BlameEnrichment
|
||||
for rows.Next() {
|
||||
var e graph.BlameEnrichment
|
||||
if err := rows.Scan(&e.NodeID, &e.RepoPrefix, &e.Commit, &e.Email, &e.Timestamp); err != nil {
|
||||
return out
|
||||
}
|
||||
out = append(out, e)
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return out
|
||||
}
|
||||
return out
|
||||
}
|
||||
@@ -0,0 +1,155 @@
|
||||
package store_sqlite
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
|
||||
"github.com/zzet/gortex/internal/graph"
|
||||
)
|
||||
|
||||
// Compile-time assertions that the SQLite Store satisfies the optional
|
||||
// git-churn enrichment sidecar capabilities (change A: enrichment moved
|
||||
// out of nodes.meta into a typed table so the node hot path stops
|
||||
// encoding rarely-read data into the meta blob and get_churn_rate reads
|
||||
// via an index instead of an AllNodes scan).
|
||||
var (
|
||||
_ graph.ChurnEnrichmentWriter = (*Store)(nil)
|
||||
_ graph.ChurnEnrichmentReader = (*Store)(nil)
|
||||
)
|
||||
|
||||
// churnChunk bounds rows per multi-row INSERT. churn_enrichment has 10
|
||||
// columns, so at 10 params/row the 999 host-param limit caps a statement
|
||||
// at 99 rows; 90 leaves headroom. Mirrors shingleChunk / mtimeChunk.
|
||||
const churnChunk = 90
|
||||
|
||||
const churnCols = `node_id, repo_prefix, commit_count, age_days, churn_rate, last_author, last_commit_at, head_sha, branch, computed_at`
|
||||
|
||||
// BulkSetChurn persists every churn row for one repo prefix in a single
|
||||
// transaction, chunked under the host-parameter limit. Idempotent on
|
||||
// node_id (INSERT OR REPLACE). Empty input is a no-op.
|
||||
func (s *Store) BulkSetChurn(repoPrefix string, rows []graph.ChurnEnrichment) error {
|
||||
if len(rows) == 0 {
|
||||
return nil
|
||||
}
|
||||
s.writeMu.Lock()
|
||||
defer s.writeMu.Unlock()
|
||||
|
||||
tx, err := s.db.Begin()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer tx.Rollback() //nolint:errcheck // rollback after Commit is a no-op
|
||||
|
||||
for start := 0; start < len(rows); start += churnChunk {
|
||||
end := start + churnChunk
|
||||
if end > len(rows) {
|
||||
end = len(rows)
|
||||
}
|
||||
batch := rows[start:end]
|
||||
args := make([]any, 0, len(batch)*10)
|
||||
stmt := make([]byte, 0, 128+len(batch)*24)
|
||||
stmt = append(stmt, "INSERT OR REPLACE INTO churn_enrichment ("...)
|
||||
stmt = append(stmt, churnCols...)
|
||||
stmt = append(stmt, ") VALUES "...)
|
||||
for i, e := range batch {
|
||||
if i > 0 {
|
||||
stmt = append(stmt, ',')
|
||||
}
|
||||
stmt = append(stmt, "(?,?,?,?,?,?,?,?,?,?)"...)
|
||||
args = append(args, e.NodeID, repoPrefix, e.CommitCount, e.AgeDays,
|
||||
e.ChurnRate, e.LastAuthor, e.LastCommitAt, e.HeadSHA, e.Branch, e.ComputedAt)
|
||||
}
|
||||
if _, err := tx.Exec(string(stmt), args...); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return tx.Commit()
|
||||
}
|
||||
|
||||
// DeleteChurn drops churn rows for the supplied node ids, chunked into
|
||||
// `node_id IN (?, …)` DELETEs. Empty input is a no-op.
|
||||
func (s *Store) DeleteChurn(nodeIDs []string) error {
|
||||
if len(nodeIDs) == 0 {
|
||||
return nil
|
||||
}
|
||||
seen := make(map[string]struct{}, len(nodeIDs))
|
||||
uniq := make([]string, 0, len(nodeIDs))
|
||||
for _, id := range nodeIDs {
|
||||
if id == "" {
|
||||
continue
|
||||
}
|
||||
if _, ok := seen[id]; ok {
|
||||
continue
|
||||
}
|
||||
seen[id] = struct{}{}
|
||||
uniq = append(uniq, id)
|
||||
}
|
||||
if len(uniq) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
s.writeMu.Lock()
|
||||
defer s.writeMu.Unlock()
|
||||
|
||||
tx, err := s.db.Begin()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer tx.Rollback() //nolint:errcheck
|
||||
|
||||
for start := 0; start < len(uniq); start += churnChunk {
|
||||
end := start + churnChunk
|
||||
if end > len(uniq) {
|
||||
end = len(uniq)
|
||||
}
|
||||
chunk := uniq[start:end]
|
||||
args := make([]any, len(chunk))
|
||||
stmt := make([]byte, 0, 48+len(chunk)*2)
|
||||
stmt = append(stmt, "DELETE FROM churn_enrichment WHERE node_id IN ("...)
|
||||
for i, id := range chunk {
|
||||
if i > 0 {
|
||||
stmt = append(stmt, ',')
|
||||
}
|
||||
stmt = append(stmt, '?')
|
||||
args[i] = id
|
||||
}
|
||||
stmt = append(stmt, ')')
|
||||
if _, err := tx.Exec(string(stmt), args...); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return tx.Commit()
|
||||
}
|
||||
|
||||
// ChurnRows returns every churn row for repoPrefix; an EMPTY repoPrefix
|
||||
// returns ALL rows across repos. This is an index-only read over the
|
||||
// (small) enriched set — the whole point of the sidecar, replacing the
|
||||
// AllNodes()+meta-decode scan get_churn_rate used to do.
|
||||
func (s *Store) ChurnRows(repoPrefix string) []graph.ChurnEnrichment {
|
||||
var (
|
||||
rows *sql.Rows
|
||||
err error
|
||||
)
|
||||
if repoPrefix == "" {
|
||||
rows, err = s.db.Query(`SELECT ` + churnCols + ` FROM churn_enrichment`)
|
||||
} else {
|
||||
rows, err = s.db.Query(`SELECT `+churnCols+` FROM churn_enrichment WHERE repo_prefix = ?`, repoPrefix)
|
||||
}
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
var out []graph.ChurnEnrichment
|
||||
for rows.Next() {
|
||||
var e graph.ChurnEnrichment
|
||||
if err := rows.Scan(&e.NodeID, &e.RepoPrefix, &e.CommitCount, &e.AgeDays,
|
||||
&e.ChurnRate, &e.LastAuthor, &e.LastCommitAt, &e.HeadSHA, &e.Branch, &e.ComputedAt); err != nil {
|
||||
return out
|
||||
}
|
||||
out = append(out, e)
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return out
|
||||
}
|
||||
return out
|
||||
}
|
||||
@@ -0,0 +1,192 @@
|
||||
package store_sqlite
|
||||
|
||||
import (
|
||||
"encoding/binary"
|
||||
|
||||
"github.com/zzet/gortex/internal/graph"
|
||||
)
|
||||
|
||||
// Compile-time assertions that the SQLite Store satisfies the optional
|
||||
// per-symbol clone-shingle persistence capabilities. Lifting this state
|
||||
// into the same backend the graph lives in means warm restarts rebuild
|
||||
// the clone-detection CMS through one persistence surface instead of a
|
||||
// second gob snapshot.
|
||||
var (
|
||||
_ graph.CloneShingleWriter = (*Store)(nil)
|
||||
_ graph.CloneShingleReader = (*Store)(nil)
|
||||
)
|
||||
|
||||
// shingleChunk bounds how many (node_id, repo_prefix, shingles) tuples
|
||||
// ride in a single multi-row INSERT. SQLite's default compiled-in host
|
||||
// parameter limit is 999; at 3 params per row that caps a statement at
|
||||
// 333 rows, so 300 leaves headroom. Mirrors mtimeChunk.
|
||||
const shingleChunk = 300
|
||||
|
||||
// encodeShingles serialises a uint64 slice to a little-endian BLOB
|
||||
// (8 bytes per element). A nil/empty slice encodes to an empty BLOB.
|
||||
func encodeShingles(shingles []uint64) []byte {
|
||||
b := make([]byte, len(shingles)*8)
|
||||
for i, s := range shingles {
|
||||
binary.LittleEndian.PutUint64(b[i*8:], s)
|
||||
}
|
||||
return b
|
||||
}
|
||||
|
||||
// decodeShingles is the inverse of encodeShingles. A BLOB whose length
|
||||
// is not a multiple of 8 yields nil (corrupt row); callers skip nil
|
||||
// sets. An empty BLOB decodes to an empty (non-nil) slice.
|
||||
func decodeShingles(b []byte) []uint64 {
|
||||
if len(b)%8 != 0 {
|
||||
return nil
|
||||
}
|
||||
out := make([]uint64, len(b)/8)
|
||||
for i := range out {
|
||||
out[i] = binary.LittleEndian.Uint64(b[i*8:])
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// BulkSetCloneShingles persists every (nodeID -> shingles) entry for
|
||||
// one repo prefix in a single transaction, chunked so no statement
|
||||
// exceeds SQLite's host-parameter limit. Idempotent on node_id:
|
||||
// re-running with overlapping keys replaces in place. Empty input is a
|
||||
// no-op.
|
||||
func (s *Store) BulkSetCloneShingles(repoPrefix string, rows map[string][]uint64) error {
|
||||
if len(rows) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
s.writeMu.Lock()
|
||||
defer s.writeMu.Unlock()
|
||||
|
||||
// Stable ordering is not required for correctness, but iterating the
|
||||
// map directly is fine — we only chunk by count.
|
||||
type kv struct {
|
||||
id string
|
||||
blob []byte
|
||||
}
|
||||
pending := make([]kv, 0, len(rows))
|
||||
for id, sh := range rows {
|
||||
pending = append(pending, kv{id: id, blob: encodeShingles(sh)})
|
||||
}
|
||||
|
||||
tx, err := s.db.Begin()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer tx.Rollback() //nolint:errcheck // rollback after Commit is a no-op
|
||||
|
||||
for start := 0; start < len(pending); start += shingleChunk {
|
||||
end := start + shingleChunk
|
||||
if end > len(pending) {
|
||||
end = len(pending)
|
||||
}
|
||||
batch := pending[start:end]
|
||||
|
||||
// Build a multi-row INSERT OR REPLACE: (?, ?, ?), (?, ?, ?), ...
|
||||
args := make([]any, 0, len(batch)*3)
|
||||
stmt := make([]byte, 0, 64+len(batch)*16)
|
||||
stmt = append(stmt, "INSERT OR REPLACE INTO clone_shingles (node_id, repo_prefix, shingles) VALUES "...)
|
||||
for i, e := range batch {
|
||||
if i > 0 {
|
||||
stmt = append(stmt, ',')
|
||||
}
|
||||
stmt = append(stmt, "(?, ?, ?)"...)
|
||||
args = append(args, e.id, repoPrefix, e.blob)
|
||||
}
|
||||
if _, err := tx.Exec(string(stmt), args...); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
return tx.Commit()
|
||||
}
|
||||
|
||||
// DeleteCloneShingles drops the rows for the supplied node ids, chunked
|
||||
// into `node_id IN (?, ?, …)` DELETEs so no statement exceeds SQLite's
|
||||
// host-parameter limit. Empty input is a no-op; missing ids are simply
|
||||
// not deleted.
|
||||
func (s *Store) DeleteCloneShingles(nodeIDs []string) error {
|
||||
if len(nodeIDs) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
// Dedupe + skip empty up front to keep the chunk loop honest.
|
||||
seen := make(map[string]struct{}, len(nodeIDs))
|
||||
uniq := make([]string, 0, len(nodeIDs))
|
||||
for _, id := range nodeIDs {
|
||||
if id == "" {
|
||||
continue
|
||||
}
|
||||
if _, ok := seen[id]; ok {
|
||||
continue
|
||||
}
|
||||
seen[id] = struct{}{}
|
||||
uniq = append(uniq, id)
|
||||
}
|
||||
if len(uniq) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
s.writeMu.Lock()
|
||||
defer s.writeMu.Unlock()
|
||||
|
||||
tx, err := s.db.Begin()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer tx.Rollback() //nolint:errcheck // rollback after Commit is a no-op
|
||||
|
||||
for start := 0; start < len(uniq); start += shingleChunk {
|
||||
end := start + shingleChunk
|
||||
if end > len(uniq) {
|
||||
end = len(uniq)
|
||||
}
|
||||
chunk := uniq[start:end]
|
||||
args := make([]any, len(chunk))
|
||||
stmt := make([]byte, 0, 48+len(chunk)*2)
|
||||
stmt = append(stmt, "DELETE FROM clone_shingles WHERE node_id IN ("...)
|
||||
for i, id := range chunk {
|
||||
if i > 0 {
|
||||
stmt = append(stmt, ',')
|
||||
}
|
||||
stmt = append(stmt, '?')
|
||||
args[i] = id
|
||||
}
|
||||
stmt = append(stmt, ')')
|
||||
if _, err := tx.Exec(string(stmt), args...); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
return tx.Commit()
|
||||
}
|
||||
|
||||
// LoadCloneShingles returns the recorded shingle sets for one repo
|
||||
// prefix as a fresh map. It always returns a non-nil (possibly empty)
|
||||
// map and surfaces any query error. An empty/absent prefix yields an
|
||||
// empty map, not an error.
|
||||
func (s *Store) LoadCloneShingles(repoPrefix string) (map[string][]uint64, error) {
|
||||
rows, err := s.db.Query(
|
||||
`SELECT node_id, shingles FROM clone_shingles WHERE repo_prefix = ?`,
|
||||
repoPrefix,
|
||||
)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
out := make(map[string][]uint64)
|
||||
for rows.Next() {
|
||||
var id string
|
||||
var blob []byte
|
||||
if err := rows.Scan(&id, &blob); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out[id] = decodeShingles(blob)
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
@@ -0,0 +1,63 @@
|
||||
package store_sqlite
|
||||
|
||||
// One-time boot compaction support.
|
||||
//
|
||||
// The graph store only ever grows on disk: deleted rows (a purged repo, the
|
||||
// duplicate-collapse migration, resolver cleanups) return their pages to
|
||||
// SQLite's freelist, where future writes reuse them — but nothing short of
|
||||
// VACUUM returns them to the filesystem. A long-lived store that shed a large
|
||||
// fraction of its rows can therefore pin gigabytes of dead file forever (a
|
||||
// live store sat at 64% freelist — 4.4 GB reclaimable in a 6.8 GB file).
|
||||
// These methods give the daemon the numbers to decide whether that one-time
|
||||
// rewrite is worth it, and the lever to run it. The policy (thresholds, disk
|
||||
// headroom, kill-switch) deliberately lives with the caller: the store cannot
|
||||
// know whether minutes of exclusive I/O are acceptable right now.
|
||||
|
||||
// Path returns the on-disk database file path. Empty when the store was not
|
||||
// opened from a file — callers using it to reason about the underlying
|
||||
// filesystem (disk-headroom checks) must treat "" as "unknown, don't".
|
||||
func (s *Store) Path() string {
|
||||
return s.dbPath
|
||||
}
|
||||
|
||||
// CompactStats reports how much of the database file is reclaimable dead
|
||||
// space: freeBytes is the freelist (freelist_count × page_size), totalBytes
|
||||
// the whole main file (page_count × page_size). Zeros on any pragma error —
|
||||
// a read failing here is the same teardown race panicOnFatal swallows, and
|
||||
// "nothing reclaimable" is the answer that makes every caller do nothing.
|
||||
// The -wal file is excluded on purpose: the checkpoint loop already bounds
|
||||
// it, and VACUUM only rewrites the main file.
|
||||
func (s *Store) CompactStats() (freeBytes, totalBytes int64) {
|
||||
var pageSize, pageCount, freePages int64
|
||||
if err := s.db.QueryRow(`PRAGMA page_size`).Scan(&pageSize); err != nil {
|
||||
return 0, 0
|
||||
}
|
||||
if err := s.db.QueryRow(`PRAGMA page_count`).Scan(&pageCount); err != nil {
|
||||
return 0, 0
|
||||
}
|
||||
if err := s.db.QueryRow(`PRAGMA freelist_count`).Scan(&freePages); err != nil {
|
||||
return 0, 0
|
||||
}
|
||||
return freePages * pageSize, pageCount * pageSize
|
||||
}
|
||||
|
||||
// Compact rewrites the database file (VACUUM), returning freelist pages to
|
||||
// the filesystem, then drains the write-ahead log with a TRUNCATE checkpoint
|
||||
// so the rewrite's WAL traffic doesn't linger as a second oversized file.
|
||||
//
|
||||
// Cost model callers must respect: VACUUM copies the live content into a
|
||||
// temporary database (up to a full extra copy on the same filesystem) and
|
||||
// needs exclusive access — it blocks Go-side writers via writeMu here, and a
|
||||
// concurrent reader on another pooled connection makes SQLite wait out
|
||||
// busy_timeout and then fail. That failure is clean (the store is untouched,
|
||||
// freelist pages remain reusable), which is why the daemon treats a Compact
|
||||
// error as skip-and-continue rather than fatal.
|
||||
func (s *Store) Compact() error {
|
||||
s.writeMu.Lock()
|
||||
_, err := s.db.Exec(`VACUUM`)
|
||||
s.writeMu.Unlock()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return s.CheckpointWAL()
|
||||
}
|
||||
@@ -0,0 +1,76 @@
|
||||
package store_sqlite_test
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"github.com/zzet/gortex/internal/graph"
|
||||
)
|
||||
|
||||
// TestCompactReclaimsFreelist pins the boot-compaction capability end to end
|
||||
// on a real file: shedding most rows leaves the file full of freelist pages
|
||||
// (CompactStats must say so), Compact() must return them to the filesystem
|
||||
// (page_count shrinks), and the surviving rows must come through untouched.
|
||||
// Fractional assertions only — page size and per-row overhead are backend
|
||||
// details this test must not encode.
|
||||
func TestCompactReclaimsFreelist(t *testing.T) {
|
||||
s := openTestStore(t)
|
||||
|
||||
// Seed 60 files × 50 nodes with ~2 KiB of meta each (~6 MiB of pages),
|
||||
// then checkpoint so the rows land in the main file rather than the WAL —
|
||||
// CompactStats deliberately measures only the main file.
|
||||
const files, perFile = 60, 50
|
||||
pad := strings.Repeat("x", 2048)
|
||||
for f := 0; f < files; f++ {
|
||||
nodes := make([]*graph.Node, 0, perFile)
|
||||
path := fmt.Sprintf("p/f%03d.go", f)
|
||||
for n := 0; n < perFile; n++ {
|
||||
nodes = append(nodes, &graph.Node{
|
||||
ID: fmt.Sprintf("%s::N%d", path, n),
|
||||
Kind: graph.KindFunction,
|
||||
Name: fmt.Sprintf("N%d", n),
|
||||
FilePath: path,
|
||||
Meta: map[string]any{"pad": pad},
|
||||
})
|
||||
}
|
||||
s.AddBatch(nodes, nil)
|
||||
}
|
||||
require.NoError(t, s.CheckpointWAL())
|
||||
_, totalSeeded := s.CompactStats()
|
||||
require.Greater(t, totalSeeded, int64(3<<20), "sanity: seeding must produce a multi-MiB main file")
|
||||
|
||||
// Shed all but one file, checkpoint again so the deletions reach the main
|
||||
// file's freelist.
|
||||
for f := 1; f < files; f++ {
|
||||
s.EvictFile(fmt.Sprintf("p/f%03d.go", f))
|
||||
}
|
||||
require.NoError(t, s.CheckpointWAL())
|
||||
|
||||
freeBefore, totalBefore := s.CompactStats()
|
||||
assert.Greater(t, freeBefore*3, totalBefore,
|
||||
"after shedding ~98%% of rows the freelist must dominate the file (free=%d total=%d)", freeBefore, totalBefore)
|
||||
assert.Greater(t, freeBefore, int64(1<<20), "freelist must be at least MiB-scale to make the shrink observable")
|
||||
|
||||
require.NoError(t, s.Compact())
|
||||
|
||||
freeAfter, totalAfter := s.CompactStats()
|
||||
assert.Less(t, totalAfter, totalBefore/2,
|
||||
"VACUUM must return the dead majority to the filesystem (before=%d after=%d)", totalBefore, totalAfter)
|
||||
assert.Less(t, freeAfter, totalBefore/10,
|
||||
"post-VACUUM freelist must be near empty (free=%d)", freeAfter)
|
||||
|
||||
// Survivors intact, evicted rows gone.
|
||||
assert.Equal(t, perFile, s.NodeCount(), "compaction must not change the row count")
|
||||
kept := s.GetNode("p/f000.go::N0")
|
||||
require.NotNil(t, kept, "kept row must survive VACUUM")
|
||||
assert.Equal(t, pad, kept.Meta["pad"], "kept row's meta blob must round-trip through VACUUM")
|
||||
assert.Nil(t, s.GetNode("p/f001.go::N0"), "evicted row must stay gone")
|
||||
|
||||
// The store stays fully writable after the rewrite.
|
||||
s.AddNode(&graph.Node{ID: "p/new.go::After", Kind: graph.KindFunction, Name: "After", FilePath: "p/new.go"})
|
||||
assert.NotNil(t, s.GetNode("p/new.go::After"))
|
||||
}
|
||||
@@ -0,0 +1,144 @@
|
||||
package store_sqlite
|
||||
|
||||
import (
|
||||
"github.com/zzet/gortex/internal/graph"
|
||||
)
|
||||
|
||||
// Compile-time assertions that the SQLite Store satisfies the optional
|
||||
// constant-value persistence capability. A KindConstant node's literal
|
||||
// value lives in this queryable sidecar (not the JSON Meta blob)
|
||||
// so the resolver can dereference a const-identifier dispatch name across
|
||||
// files without an unindexable per-node blob decode.
|
||||
var (
|
||||
_ graph.ConstantValueWriter = (*Store)(nil)
|
||||
_ graph.ConstantValueReader = (*Store)(nil)
|
||||
)
|
||||
|
||||
// constValueChunk bounds rows per multi-row INSERT (4 params/row; 80 rows
|
||||
// = 320 host params, well under SQLite's 999 default).
|
||||
const constValueChunk = 80
|
||||
|
||||
// BulkSetConstantValues persists constant values for one repo prefix in a
|
||||
// single transaction, chunked under the host-parameter limit. Idempotent
|
||||
// on the node_id primary key. Empty input is a no-op.
|
||||
func (s *Store) BulkSetConstantValues(repoPrefix string, rows []graph.ConstantValueRow) error {
|
||||
if len(rows) == 0 {
|
||||
return nil
|
||||
}
|
||||
s.writeMu.Lock()
|
||||
defer s.writeMu.Unlock()
|
||||
|
||||
tx, err := s.db.Begin()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer tx.Rollback() //nolint:errcheck // rollback after Commit is a no-op
|
||||
|
||||
for start := 0; start < len(rows); start += constValueChunk {
|
||||
end := start + constValueChunk
|
||||
if end > len(rows) {
|
||||
end = len(rows)
|
||||
}
|
||||
batch := rows[start:end]
|
||||
args := make([]any, 0, len(batch)*4)
|
||||
stmt := make([]byte, 0, 96+len(batch)*16)
|
||||
stmt = append(stmt, "INSERT OR REPLACE INTO constant_values (node_id, repo_prefix, file_path, value) VALUES "...)
|
||||
for i, r := range batch {
|
||||
if i > 0 {
|
||||
stmt = append(stmt, ',')
|
||||
}
|
||||
stmt = append(stmt, "(?, ?, ?, ?)"...)
|
||||
args = append(args, r.NodeID, repoPrefix, r.FilePath, r.Value)
|
||||
}
|
||||
if _, err := tx.Exec(string(stmt), args...); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return tx.Commit()
|
||||
}
|
||||
|
||||
// DeleteConstantValuesByFiles drops all constant values sourced in the
|
||||
// supplied files for one repo prefix, chunked into `file_path IN (…)`
|
||||
// DELETEs. Empty input is a no-op.
|
||||
func (s *Store) DeleteConstantValuesByFiles(repoPrefix string, files []string) error {
|
||||
if len(files) == 0 {
|
||||
return nil
|
||||
}
|
||||
s.writeMu.Lock()
|
||||
defer s.writeMu.Unlock()
|
||||
|
||||
tx, err := s.db.Begin()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer tx.Rollback() //nolint:errcheck // rollback after Commit is a no-op
|
||||
|
||||
for start := 0; start < len(files); start += constValueChunk {
|
||||
end := start + constValueChunk
|
||||
if end > len(files) {
|
||||
end = len(files)
|
||||
}
|
||||
chunk := files[start:end]
|
||||
args := make([]any, 0, len(chunk)+1)
|
||||
args = append(args, repoPrefix)
|
||||
stmt := make([]byte, 0, 64+len(chunk)*2)
|
||||
stmt = append(stmt, "DELETE FROM constant_values WHERE repo_prefix = ? AND file_path IN ("...)
|
||||
for i, f := range chunk {
|
||||
if i > 0 {
|
||||
stmt = append(stmt, ',')
|
||||
}
|
||||
stmt = append(stmt, '?')
|
||||
args = append(args, f)
|
||||
}
|
||||
stmt = append(stmt, ')')
|
||||
if _, err := tx.Exec(string(stmt), args...); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return tx.Commit()
|
||||
}
|
||||
|
||||
// ConstantValuesByNodeIDs returns the persisted values for the supplied
|
||||
// node ids (omitting ids with no recorded value). Always non-nil.
|
||||
func (s *Store) ConstantValuesByNodeIDs(nodeIDs []string) (map[string]string, error) {
|
||||
out := make(map[string]string, len(nodeIDs))
|
||||
if len(nodeIDs) == 0 {
|
||||
return out, nil
|
||||
}
|
||||
for start := 0; start < len(nodeIDs); start += constValueChunk {
|
||||
end := start + constValueChunk
|
||||
if end > len(nodeIDs) {
|
||||
end = len(nodeIDs)
|
||||
}
|
||||
chunk := nodeIDs[start:end]
|
||||
args := make([]any, 0, len(chunk))
|
||||
stmt := make([]byte, 0, 64+len(chunk)*2)
|
||||
stmt = append(stmt, "SELECT node_id, value FROM constant_values WHERE node_id IN ("...)
|
||||
for i, id := range chunk {
|
||||
if i > 0 {
|
||||
stmt = append(stmt, ',')
|
||||
}
|
||||
stmt = append(stmt, '?')
|
||||
args = append(args, id)
|
||||
}
|
||||
stmt = append(stmt, ')')
|
||||
rows, err := s.db.Query(string(stmt), args...)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for rows.Next() {
|
||||
var id, val string
|
||||
if err := rows.Scan(&id, &val); err != nil {
|
||||
_ = rows.Close()
|
||||
return nil, err
|
||||
}
|
||||
out[id] = val
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
_ = rows.Close()
|
||||
return nil, err
|
||||
}
|
||||
_ = rows.Close()
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
@@ -0,0 +1,73 @@
|
||||
package store_sqlite_test
|
||||
|
||||
import (
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"github.com/zzet/gortex/internal/graph"
|
||||
"github.com/zzet/gortex/internal/graph/store_sqlite"
|
||||
)
|
||||
|
||||
func openConstValStore(t *testing.T) *store_sqlite.Store {
|
||||
t.Helper()
|
||||
s, err := store_sqlite.Open(filepath.Join(t.TempDir(), "cv.sqlite"))
|
||||
require.NoError(t, err)
|
||||
t.Cleanup(func() { _ = s.Close() })
|
||||
return s
|
||||
}
|
||||
|
||||
func TestConstantValues_Roundtrip(t *testing.T) {
|
||||
s := openConstValStore(t)
|
||||
rows := []graph.ConstantValueRow{
|
||||
{NodeID: "a.go::ChargeCardActivity", FilePath: "a.go", Value: "ChargeCard"},
|
||||
{NodeID: "a.go::RefundActivity", FilePath: "a.go", Value: "Refund"},
|
||||
}
|
||||
require.NoError(t, s.BulkSetConstantValues("repo", rows))
|
||||
|
||||
got, err := s.ConstantValuesByNodeIDs([]string{"a.go::ChargeCardActivity", "a.go::RefundActivity", "missing"})
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "ChargeCard", got["a.go::ChargeCardActivity"])
|
||||
assert.Equal(t, "Refund", got["a.go::RefundActivity"])
|
||||
_, ok := got["missing"]
|
||||
assert.False(t, ok)
|
||||
}
|
||||
|
||||
func TestConstantValues_DeleteByFile(t *testing.T) {
|
||||
s := openConstValStore(t)
|
||||
require.NoError(t, s.BulkSetConstantValues("repo", []graph.ConstantValueRow{
|
||||
{NodeID: "a.go::X", FilePath: "a.go", Value: "vx"},
|
||||
{NodeID: "b.go::Y", FilePath: "b.go", Value: "vy"},
|
||||
}))
|
||||
require.NoError(t, s.DeleteConstantValuesByFiles("repo", []string{"a.go"}))
|
||||
|
||||
got, err := s.ConstantValuesByNodeIDs([]string{"a.go::X", "b.go::Y"})
|
||||
require.NoError(t, err)
|
||||
_, gone := got["a.go::X"]
|
||||
assert.False(t, gone, "a.go's value must be deleted")
|
||||
assert.Equal(t, "vy", got["b.go::Y"], "b.go's value must remain")
|
||||
}
|
||||
|
||||
func TestConstantValues_Replace(t *testing.T) {
|
||||
s := openConstValStore(t)
|
||||
require.NoError(t, s.BulkSetConstantValues("repo", []graph.ConstantValueRow{
|
||||
{NodeID: "a.go::X", FilePath: "a.go", Value: "old"},
|
||||
}))
|
||||
require.NoError(t, s.BulkSetConstantValues("repo", []graph.ConstantValueRow{
|
||||
{NodeID: "a.go::X", FilePath: "a.go", Value: "new"},
|
||||
}))
|
||||
got, err := s.ConstantValuesByNodeIDs([]string{"a.go::X"})
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "new", got["a.go::X"], "INSERT OR REPLACE must update by node_id PK")
|
||||
}
|
||||
|
||||
func TestConstantValues_EmptyNoop(t *testing.T) {
|
||||
s := openConstValStore(t)
|
||||
require.NoError(t, s.BulkSetConstantValues("repo", nil))
|
||||
require.NoError(t, s.DeleteConstantValuesByFiles("repo", nil))
|
||||
got, err := s.ConstantValuesByNodeIDs(nil)
|
||||
require.NoError(t, err)
|
||||
assert.Empty(t, got)
|
||||
}
|
||||
@@ -0,0 +1,297 @@
|
||||
package store_sqlite
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"strings"
|
||||
|
||||
"github.com/zzet/gortex/internal/graph"
|
||||
)
|
||||
|
||||
// This file implements graph.ContentSearcher on the SQLite backend using
|
||||
// the content_fts FTS5 virtual table declared in schema.go — the
|
||||
// dedicated, on-disk full-text index for CONTENT (data_class="content")
|
||||
// section bodies, kept physically separate from symbol_fts so content
|
||||
// text never reaches the symbol search or the code-oriented graph passes.
|
||||
//
|
||||
// Streamed build: WipeContent(repoPrefix) once at the start of a full
|
||||
// index, AppendContent each content file's sections as they are parsed
|
||||
// (no per-file wipe), then BuildContentIndex to merge segments.
|
||||
// Incremental reindex of one content file is WipeContentFile +
|
||||
// AppendContent.
|
||||
|
||||
// Compile-time assertion: *Store satisfies the content-search capability.
|
||||
var _ graph.ContentSearcher = (*Store)(nil)
|
||||
|
||||
// contentInsertChunkRows bounds rows per multi-row INSERT. Each row binds
|
||||
// 5 host params (node_id, repo_prefix, file_path, ordinal, body); 180 rows
|
||||
// is 900 params, comfortably under SQLite's default 999-variable limit.
|
||||
const contentInsertChunkRows = 180
|
||||
|
||||
// WipeContent removes a repo's content rows before a full rebuild. Empty
|
||||
// repoPrefix wipes the whole table (single-repo / conformance behaviour).
|
||||
func (s *Store) WipeContent(repoPrefix string) error {
|
||||
s.writeMu.Lock()
|
||||
defer s.writeMu.Unlock()
|
||||
_, err := s.db.Exec(`DELETE FROM content_fts WHERE repo_prefix = ?`, repoPrefix)
|
||||
return err
|
||||
}
|
||||
|
||||
// WipeContentFile removes one file's content rows — the incremental
|
||||
// reindex path when a single content file changes.
|
||||
func (s *Store) WipeContentFile(filePath string) error {
|
||||
if filePath == "" {
|
||||
return nil
|
||||
}
|
||||
s.writeMu.Lock()
|
||||
defer s.writeMu.Unlock()
|
||||
_, err := s.db.Exec(`DELETE FROM content_fts WHERE file_path = ?`, filePath)
|
||||
return err
|
||||
}
|
||||
|
||||
// WipeContentFileInRepo removes ONE file's content rows scoped to a repo —
|
||||
// the crash-safe full-index sibling of WipeContentFile (which keys on
|
||||
// file_path alone and so would clobber a same-named file in another repo).
|
||||
// A full index streams content per file: delete this file's prior rows,
|
||||
// then AppendContent its fresh sections — so a mid-parse kill leaves a mix
|
||||
// of old+new content per file rather than the empty table a repo-wide
|
||||
// pre-wipe would leave behind. Empty filePath is a no-op.
|
||||
func (s *Store) WipeContentFileInRepo(repoPrefix, filePath string) error {
|
||||
if filePath == "" {
|
||||
return nil
|
||||
}
|
||||
s.writeMu.Lock()
|
||||
defer s.writeMu.Unlock()
|
||||
_, err := s.db.Exec(`DELETE FROM content_fts WHERE repo_prefix = ? AND file_path = ?`, repoPrefix, filePath)
|
||||
return err
|
||||
}
|
||||
|
||||
// DeleteContentFilesForRepoNotIn sweeps a repo's content rows down to keep —
|
||||
// every content row whose file_path is absent from keep is deleted. keep is
|
||||
// the set of files that actually STREAMED content sections in the walk just
|
||||
// completed (each recorded as the same file_path form AppendContent wrote),
|
||||
// NOT the set of files that merely survive on disk: a file can still exist
|
||||
// yet stop producing content (doc emptied, classification changed), and a
|
||||
// disk-based keep would protect its stale rows forever. Run once at the end
|
||||
// of a successful full index (right after the authoritative mtime replace),
|
||||
// it reaps vanished files and content->no-content transitions in one scan;
|
||||
// the per-file WipeContentFileInRepo + AppendContent streaming build
|
||||
// refreshes the files that still produce content. Together they replace the
|
||||
// old repo-wide pre-wipe: a mid-parse kill no longer empties the content
|
||||
// index, and stale rows are reaped only on the next clean completion.
|
||||
// Empty keep is a deliberate no-op safety net — never wipe a whole repo from
|
||||
// an empty set; a caller that legitimately ends a walk with zero content
|
||||
// files calls WipeContent explicitly instead.
|
||||
func (s *Store) DeleteContentFilesForRepoNotIn(repoPrefix string, keep map[string]struct{}) error {
|
||||
if len(keep) == 0 {
|
||||
return nil
|
||||
}
|
||||
s.writeMu.Lock()
|
||||
defer s.writeMu.Unlock()
|
||||
|
||||
// Enumerate the repo's content file paths, then delete only those not in
|
||||
// keep. Content files are a small subset (docs / PDFs / office), so the
|
||||
// DISTINCT scan + targeted deletes stay cheap and dodge a giant NOT IN
|
||||
// (...) bound-variable list. Rows are drained + closed before the delete
|
||||
// tx opens (no open read cursor while writing on the same connection).
|
||||
rows, err := s.db.Query(`SELECT DISTINCT file_path FROM content_fts WHERE repo_prefix = ?`, repoPrefix)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
var vanished []string
|
||||
for rows.Next() {
|
||||
var fp string
|
||||
if err := rows.Scan(&fp); err != nil {
|
||||
_ = rows.Close()
|
||||
return err
|
||||
}
|
||||
if _, ok := keep[fp]; !ok {
|
||||
vanished = append(vanished, fp)
|
||||
}
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
_ = rows.Close()
|
||||
return err
|
||||
}
|
||||
_ = rows.Close()
|
||||
if len(vanished) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
tx, err := s.db.Begin()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer tx.Rollback() //nolint:errcheck // rollback after Commit is a no-op
|
||||
const chunk = 900
|
||||
for start := 0; start < len(vanished); start += chunk {
|
||||
end := minInt(start+chunk, len(vanished))
|
||||
batch := vanished[start:end]
|
||||
placeholders := strings.TrimSuffix(strings.Repeat("?,", len(batch)), ",")
|
||||
args := make([]any, 0, len(batch)+1)
|
||||
args = append(args, repoPrefix)
|
||||
for _, fp := range batch {
|
||||
args = append(args, fp)
|
||||
}
|
||||
if _, err := tx.Exec(`DELETE FROM content_fts WHERE repo_prefix = ? AND file_path IN (`+placeholders+`)`, args...); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return tx.Commit()
|
||||
}
|
||||
|
||||
// AppendContent inserts content rows for repoPrefix without wiping — the
|
||||
// streamed per-file build path. Callers wipe (whole repo or one file)
|
||||
// first. Rows with an empty NodeID are skipped.
|
||||
func (s *Store) AppendContent(repoPrefix string, items []graph.ContentFTSItem) error {
|
||||
if len(items) == 0 {
|
||||
return nil
|
||||
}
|
||||
s.writeMu.Lock()
|
||||
defer s.writeMu.Unlock()
|
||||
|
||||
tx, err := s.db.Begin()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
commit := false
|
||||
defer func() {
|
||||
if !commit {
|
||||
_ = tx.Rollback()
|
||||
}
|
||||
}()
|
||||
|
||||
for start := 0; start < len(items); start += contentInsertChunkRows {
|
||||
end := minInt(start+contentInsertChunkRows, len(items))
|
||||
chunk := items[start:end]
|
||||
|
||||
var b strings.Builder
|
||||
b.WriteString(`INSERT INTO content_fts (node_id, repo_prefix, file_path, ordinal, body) VALUES `)
|
||||
args := make([]any, 0, len(chunk)*5)
|
||||
for _, it := range chunk {
|
||||
if it.NodeID == "" {
|
||||
continue
|
||||
}
|
||||
if len(args) > 0 {
|
||||
b.WriteByte(',')
|
||||
}
|
||||
b.WriteString(`(?,?,?,?,?)`)
|
||||
args = append(args, it.NodeID, repoPrefix, it.FilePath, it.Ordinal, it.Body)
|
||||
}
|
||||
if len(args) == 0 {
|
||||
continue
|
||||
}
|
||||
if _, err := tx.Exec(b.String(), args...); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
if err := tx.Commit(); err != nil {
|
||||
return err
|
||||
}
|
||||
commit = true
|
||||
return nil
|
||||
}
|
||||
|
||||
// BuildContentIndex opportunistically merges FTS5 segments (a read-latency
|
||||
// improvement). Like BuildSymbolIndex it is a no-op for correctness — the
|
||||
// FTS index is maintained incrementally on every insert — and idempotent.
|
||||
func (s *Store) BuildContentIndex() error {
|
||||
s.writeMu.Lock()
|
||||
defer s.writeMu.Unlock()
|
||||
_, _ = s.db.Exec(`INSERT INTO content_fts(content_fts) VALUES('optimize')`)
|
||||
return nil
|
||||
}
|
||||
|
||||
// SearchContent runs a content query scoped to repoPrefix (empty = all
|
||||
// repos) and returns hits ordered by descending relevance, each carrying a
|
||||
// short snippet excerpt from the matched body. Reuses the symbol path's
|
||||
// write-side tokeniser (buildFTSMatch) so the content corpus and queries
|
||||
// agree on camelCase / path-separator splitting.
|
||||
func (s *Store) SearchContent(query, repoPrefix string, limit int) ([]graph.ContentHit, error) {
|
||||
if query == "" {
|
||||
return nil, nil
|
||||
}
|
||||
if limit <= 0 {
|
||||
limit = 20
|
||||
}
|
||||
match := s.buildFTSMatch(query)
|
||||
if match == "" {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
var sb strings.Builder
|
||||
// snippet() over the body column (index 4): no highlight marks, an
|
||||
// ellipsis for elision, ~16 tokens of context. CAST(ordinal AS INTEGER)
|
||||
// forces integer affinity so the FTS5 text column scans cleanly into an
|
||||
// int.
|
||||
sb.WriteString(`SELECT node_id, file_path, CAST(ordinal AS INTEGER), snippet(content_fts, 4, '', '', '…', 16), bm25(content_fts) FROM content_fts WHERE content_fts MATCH ?`)
|
||||
args := []any{match}
|
||||
if repoPrefix != "" {
|
||||
sb.WriteString(` AND repo_prefix = ?`)
|
||||
args = append(args, repoPrefix)
|
||||
}
|
||||
sb.WriteString(` ORDER BY bm25(content_fts) LIMIT ?`)
|
||||
args = append(args, limit)
|
||||
|
||||
rows, err := s.db.Query(sb.String(), args...)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
var hits []graph.ContentHit
|
||||
for rows.Next() {
|
||||
var (
|
||||
id, fp, snip string
|
||||
ordinal int
|
||||
score float64
|
||||
)
|
||||
if err := rows.Scan(&id, &fp, &ordinal, &snip, &score); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if id == "" {
|
||||
continue
|
||||
}
|
||||
// bm25() is negative-better in SQLite; negate so higher = better,
|
||||
// matching the ContentHit contract. Rows already arrive best-first.
|
||||
hits = append(hits, graph.ContentHit{
|
||||
NodeID: id,
|
||||
FilePath: fp,
|
||||
Ordinal: ordinal,
|
||||
Score: -score,
|
||||
Snippet: snip,
|
||||
})
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return hits, nil
|
||||
}
|
||||
|
||||
// ScanContent streams every content row (scoped to repoPrefix; empty = all
|
||||
// repos) to fn with its FULL body, read incrementally via a cursor so a
|
||||
// consumer iterating hundreds of thousands of sections stays bounded. fn
|
||||
// returns false to stop the scan early.
|
||||
func (s *Store) ScanContent(repoPrefix string, fn func(nodeID, filePath, body string) bool) error {
|
||||
var rows *sql.Rows
|
||||
var err error
|
||||
if repoPrefix == "" {
|
||||
rows, err = s.db.Query(`SELECT node_id, file_path, body FROM content_fts`)
|
||||
} else {
|
||||
rows, err = s.db.Query(`SELECT node_id, file_path, body FROM content_fts WHERE repo_prefix = ?`, repoPrefix)
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer rows.Close()
|
||||
for rows.Next() {
|
||||
var nodeID, filePath, body string
|
||||
if err := rows.Scan(&nodeID, &filePath, &body); err != nil {
|
||||
return err
|
||||
}
|
||||
if !fn(nodeID, filePath, body) {
|
||||
return nil
|
||||
}
|
||||
}
|
||||
return rows.Err()
|
||||
}
|
||||
@@ -0,0 +1,82 @@
|
||||
package store_sqlite
|
||||
|
||||
import (
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"github.com/zzet/gortex/internal/graph"
|
||||
)
|
||||
|
||||
func TestContentFTS_BasicAndFileWipe(t *testing.T) {
|
||||
s, err := Open(filepath.Join(t.TempDir(), "c.sqlite"))
|
||||
require.NoError(t, err)
|
||||
t.Cleanup(func() { _ = s.Close() })
|
||||
|
||||
var cs graph.ContentSearcher = s // runtime assertion the store satisfies the capability
|
||||
|
||||
require.NoError(t, cs.WipeContent("")) // clean table
|
||||
items := []graph.ContentFTSItem{
|
||||
{NodeID: "a.txt::doc:section-0", FilePath: "a.txt", Ordinal: 0, Body: "the quick brown fox jumps over the lazy dog"},
|
||||
{NodeID: "a.txt::doc:section-1", FilePath: "a.txt", Ordinal: 1, Body: "lorem ipsum dolor sit amet consectetur"},
|
||||
{NodeID: "b.pdf::doc:pdf_page-0", FilePath: "b.pdf", Ordinal: 0, Body: "quantum entanglement and superposition explained"},
|
||||
}
|
||||
require.NoError(t, cs.AppendContent("", items))
|
||||
require.NoError(t, cs.BuildContentIndex())
|
||||
|
||||
// A body term resolves to exactly its section, with a non-empty snippet.
|
||||
hits, err := cs.SearchContent("quantum", "", 10)
|
||||
require.NoError(t, err)
|
||||
require.Len(t, hits, 1)
|
||||
require.Equal(t, "b.pdf::doc:pdf_page-0", hits[0].NodeID)
|
||||
require.Equal(t, "b.pdf", hits[0].FilePath)
|
||||
require.Equal(t, 0, hits[0].Ordinal)
|
||||
require.NotEmpty(t, hits[0].Snippet)
|
||||
|
||||
hits, err = cs.SearchContent("fox", "", 10)
|
||||
require.NoError(t, err)
|
||||
require.Len(t, hits, 1)
|
||||
require.Equal(t, "a.txt::doc:section-0", hits[0].NodeID)
|
||||
require.Equal(t, 0, hits[0].Ordinal)
|
||||
|
||||
// WipeContentFile drops only a.txt's rows; b.pdf survives.
|
||||
require.NoError(t, cs.WipeContentFile("a.txt"))
|
||||
hits, err = cs.SearchContent("fox", "", 10)
|
||||
require.NoError(t, err)
|
||||
require.Empty(t, hits)
|
||||
hits, err = cs.SearchContent("quantum", "", 10)
|
||||
require.NoError(t, err)
|
||||
require.Len(t, hits, 1)
|
||||
}
|
||||
|
||||
func TestContentFTS_RepoScoping(t *testing.T) {
|
||||
s, err := Open(filepath.Join(t.TempDir(), "c.sqlite"))
|
||||
require.NoError(t, err)
|
||||
t.Cleanup(func() { _ = s.Close() })
|
||||
|
||||
require.NoError(t, s.AppendContent("repoA", []graph.ContentFTSItem{
|
||||
{NodeID: "repoA::x.txt::doc:section-0", FilePath: "x.txt", Body: "alpha beta gamma"},
|
||||
}))
|
||||
require.NoError(t, s.AppendContent("repoB", []graph.ContentFTSItem{
|
||||
{NodeID: "repoB::y.txt::doc:section-0", FilePath: "y.txt", Body: "alpha delta epsilon"},
|
||||
}))
|
||||
|
||||
// Scoped search returns only the matching repo's hit.
|
||||
hits, err := s.SearchContent("alpha", "repoA", 10)
|
||||
require.NoError(t, err)
|
||||
require.Len(t, hits, 1)
|
||||
require.Equal(t, "repoA::x.txt::doc:section-0", hits[0].NodeID)
|
||||
|
||||
// Unscoped search spans both repos.
|
||||
hits, err = s.SearchContent("alpha", "", 10)
|
||||
require.NoError(t, err)
|
||||
require.Len(t, hits, 2)
|
||||
|
||||
// WipeContent scopes to one repo, leaving the sibling intact.
|
||||
require.NoError(t, s.WipeContent("repoA"))
|
||||
hits, err = s.SearchContent("alpha", "", 10)
|
||||
require.NoError(t, err)
|
||||
require.Len(t, hits, 1)
|
||||
require.Equal(t, "repoB::y.txt::doc:section-0", hits[0].NodeID)
|
||||
}
|
||||
@@ -0,0 +1,143 @@
|
||||
package store_sqlite
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
|
||||
"github.com/zzet/gortex/internal/graph"
|
||||
)
|
||||
|
||||
var (
|
||||
_ graph.CoverageEnrichmentWriter = (*Store)(nil)
|
||||
_ graph.CoverageEnrichmentReader = (*Store)(nil)
|
||||
)
|
||||
|
||||
// coverageChunk bounds rows per multi-row INSERT (5 cols → 5 params/row;
|
||||
// 999/5 ≈ 199 max, 180 leaves headroom).
|
||||
const coverageChunk = 180
|
||||
|
||||
const coverageCols = `node_id, repo_prefix, coverage_pct, num_stmt, hit`
|
||||
|
||||
// BulkSetCoverage persists coverage rows for one repo prefix in a single
|
||||
// chunked transaction. Idempotent on node_id. Empty input is a no-op.
|
||||
func (s *Store) BulkSetCoverage(repoPrefix string, rows []graph.CoverageEnrichment) error {
|
||||
if len(rows) == 0 {
|
||||
return nil
|
||||
}
|
||||
s.writeMu.Lock()
|
||||
defer s.writeMu.Unlock()
|
||||
|
||||
tx, err := s.db.Begin()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer tx.Rollback() //nolint:errcheck
|
||||
|
||||
for start := 0; start < len(rows); start += coverageChunk {
|
||||
end := start + coverageChunk
|
||||
if end > len(rows) {
|
||||
end = len(rows)
|
||||
}
|
||||
batch := rows[start:end]
|
||||
args := make([]any, 0, len(batch)*5)
|
||||
stmt := make([]byte, 0, 96+len(batch)*16)
|
||||
stmt = append(stmt, "INSERT OR REPLACE INTO coverage_enrichment ("...)
|
||||
stmt = append(stmt, coverageCols...)
|
||||
stmt = append(stmt, ") VALUES "...)
|
||||
for i, e := range batch {
|
||||
if i > 0 {
|
||||
stmt = append(stmt, ',')
|
||||
}
|
||||
stmt = append(stmt, "(?,?,?,?,?)"...)
|
||||
args = append(args, e.NodeID, repoPrefix, e.CoveragePct, e.NumStmt, e.Hit)
|
||||
}
|
||||
if _, err := tx.Exec(string(stmt), args...); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return tx.Commit()
|
||||
}
|
||||
|
||||
// DeleteCoverage drops coverage rows for the supplied node ids, chunked.
|
||||
func (s *Store) DeleteCoverage(nodeIDs []string) error {
|
||||
if len(nodeIDs) == 0 {
|
||||
return nil
|
||||
}
|
||||
seen := make(map[string]struct{}, len(nodeIDs))
|
||||
uniq := make([]string, 0, len(nodeIDs))
|
||||
for _, id := range nodeIDs {
|
||||
if id == "" {
|
||||
continue
|
||||
}
|
||||
if _, ok := seen[id]; ok {
|
||||
continue
|
||||
}
|
||||
seen[id] = struct{}{}
|
||||
uniq = append(uniq, id)
|
||||
}
|
||||
if len(uniq) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
s.writeMu.Lock()
|
||||
defer s.writeMu.Unlock()
|
||||
|
||||
tx, err := s.db.Begin()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer tx.Rollback() //nolint:errcheck
|
||||
|
||||
for start := 0; start < len(uniq); start += coverageChunk {
|
||||
end := start + coverageChunk
|
||||
if end > len(uniq) {
|
||||
end = len(uniq)
|
||||
}
|
||||
chunk := uniq[start:end]
|
||||
args := make([]any, len(chunk))
|
||||
stmt := make([]byte, 0, 56+len(chunk)*2)
|
||||
stmt = append(stmt, "DELETE FROM coverage_enrichment WHERE node_id IN ("...)
|
||||
for i, id := range chunk {
|
||||
if i > 0 {
|
||||
stmt = append(stmt, ',')
|
||||
}
|
||||
stmt = append(stmt, '?')
|
||||
args[i] = id
|
||||
}
|
||||
stmt = append(stmt, ')')
|
||||
if _, err := tx.Exec(string(stmt), args...); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return tx.Commit()
|
||||
}
|
||||
|
||||
// CoverageRows returns coverage rows for repoPrefix; empty repoPrefix
|
||||
// returns ALL rows across repos. Index-only read over the enriched set.
|
||||
func (s *Store) CoverageRows(repoPrefix string) []graph.CoverageEnrichment {
|
||||
var (
|
||||
rows *sql.Rows
|
||||
err error
|
||||
)
|
||||
if repoPrefix == "" {
|
||||
rows, err = s.db.Query(`SELECT ` + coverageCols + ` FROM coverage_enrichment`)
|
||||
} else {
|
||||
rows, err = s.db.Query(`SELECT `+coverageCols+` FROM coverage_enrichment WHERE repo_prefix = ?`, repoPrefix)
|
||||
}
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
var out []graph.CoverageEnrichment
|
||||
for rows.Next() {
|
||||
var e graph.CoverageEnrichment
|
||||
if err := rows.Scan(&e.NodeID, &e.RepoPrefix, &e.CoveragePct, &e.NumStmt, &e.Hit); err != nil {
|
||||
return out
|
||||
}
|
||||
out = append(out, e)
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return out
|
||||
}
|
||||
return out
|
||||
}
|
||||
@@ -0,0 +1,21 @@
|
||||
package store_sqlite
|
||||
|
||||
import "os"
|
||||
|
||||
// DBStats returns the on-disk size of the SQLite database file and its
|
||||
// write-ahead log, in bytes. A missing file (or a store opened without a
|
||||
// path) reports 0 for that component. Surfaced in daemon_health so a
|
||||
// runaway WAL high-water mark is observable instead of silently filling
|
||||
// the disk.
|
||||
func (s *Store) DBStats() (dbBytes, walBytes int64) {
|
||||
if s == nil || s.dbPath == "" {
|
||||
return 0, 0
|
||||
}
|
||||
if fi, err := os.Stat(s.dbPath); err == nil {
|
||||
dbBytes = fi.Size()
|
||||
}
|
||||
if fi, err := os.Stat(s.dbPath + "-wal"); err == nil {
|
||||
walBytes = fi.Size()
|
||||
}
|
||||
return dbBytes, walBytes
|
||||
}
|
||||
@@ -0,0 +1,33 @@
|
||||
package store_sqlite_test
|
||||
|
||||
import (
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"github.com/zzet/gortex/internal/graph"
|
||||
"github.com/zzet/gortex/internal/graph/store_sqlite"
|
||||
)
|
||||
|
||||
func TestDBStats(t *testing.T) {
|
||||
path := filepath.Join(t.TempDir(), "g.sqlite")
|
||||
s, err := store_sqlite.Open(path)
|
||||
require.NoError(t, err)
|
||||
t.Cleanup(func() { _ = s.Close() })
|
||||
|
||||
// Force a write so the DB (and WAL, in WAL mode) carry real bytes.
|
||||
s.AddBatch([]*graph.Node{
|
||||
{ID: "a.go::Foo", Kind: graph.KindFunction, Name: "Foo", FilePath: "a.go"},
|
||||
}, nil)
|
||||
|
||||
dbBytes, walBytes := s.DBStats()
|
||||
require.Greater(t, dbBytes, int64(0), "the on-disk DB file must have nonzero size")
|
||||
require.GreaterOrEqual(t, walBytes, int64(0), "WAL size is non-negative (0 after a checkpoint)")
|
||||
|
||||
// A store with no path (the zero value) reports zero, never panics.
|
||||
var empty store_sqlite.Store
|
||||
db, wal := empty.DBStats()
|
||||
require.Equal(t, int64(0), db)
|
||||
require.Equal(t, int64(0), wal)
|
||||
}
|
||||
@@ -0,0 +1,45 @@
|
||||
package store_sqlite
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
|
||||
"github.com/zzet/gortex/internal/graph"
|
||||
)
|
||||
|
||||
// Compile-time assertion that the SQLite Store persists the enrichment
|
||||
// completion marker. Lifting this state into the same backend the graph
|
||||
// lives in lets a warm restart skip re-enriching a repo whose persisted
|
||||
// graph already carries its LSP edges — no second persistence surface.
|
||||
var _ graph.EnrichmentStateStore = (*Store)(nil)
|
||||
|
||||
// SetEnrichmentState upserts the completion marker for one (repo, provider) —
|
||||
// written when a provider finishes a non-partial enrichment pass. One row per
|
||||
// (repo_prefix, provider).
|
||||
func (s *Store) SetEnrichmentState(st graph.EnrichmentState) error {
|
||||
s.writeMu.Lock()
|
||||
defer s.writeMu.Unlock()
|
||||
_, err := s.db.Exec(`
|
||||
INSERT OR REPLACE INTO enrichment_state
|
||||
(repo_prefix, provider, indexed_sha, completed_at, coverage)
|
||||
VALUES (?, ?, ?, ?, ?)`,
|
||||
st.RepoPrefix, st.Provider, st.IndexedSHA, st.CompletedAt, st.Coverage)
|
||||
return err
|
||||
}
|
||||
|
||||
// GetEnrichmentState returns the recorded completion marker for a
|
||||
// (repo, provider). The bool is false when no row exists yet (never-enriched
|
||||
// or pre-feature).
|
||||
func (s *Store) GetEnrichmentState(repoPrefix, provider string) (graph.EnrichmentState, bool, error) {
|
||||
row := s.db.QueryRow(`
|
||||
SELECT indexed_sha, completed_at, coverage
|
||||
FROM enrichment_state WHERE repo_prefix = ? AND provider = ?`, repoPrefix, provider)
|
||||
st := graph.EnrichmentState{RepoPrefix: repoPrefix, Provider: provider}
|
||||
err := row.Scan(&st.IndexedSHA, &st.CompletedAt, &st.Coverage)
|
||||
if err == sql.ErrNoRows {
|
||||
return graph.EnrichmentState{RepoPrefix: repoPrefix, Provider: provider}, false, nil
|
||||
}
|
||||
if err != nil {
|
||||
return graph.EnrichmentState{RepoPrefix: repoPrefix, Provider: provider}, false, err
|
||||
}
|
||||
return st, true, nil
|
||||
}
|
||||
@@ -0,0 +1,97 @@
|
||||
package store_sqlite
|
||||
|
||||
import (
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"github.com/zzet/gortex/internal/graph"
|
||||
)
|
||||
|
||||
// TestEnrichmentStateRoundTrip: set/get, missing-row, upsert-in-place, and
|
||||
// per-provider row isolation on the SQLite store.
|
||||
func TestEnrichmentStateRoundTrip(t *testing.T) {
|
||||
s, err := Open(filepath.Join(t.TempDir(), "store.sqlite"))
|
||||
if err != nil {
|
||||
t.Fatalf("Open: %v", err)
|
||||
}
|
||||
defer s.Close()
|
||||
|
||||
// Missing row → found=false, no error (the "never enriched" signal).
|
||||
if _, found, err := s.GetEnrichmentState("repo", "gopls"); err != nil || found {
|
||||
t.Fatalf("missing row: found=%v err=%v, want found=false, err=nil", found, err)
|
||||
}
|
||||
|
||||
st := graph.EnrichmentState{
|
||||
RepoPrefix: "repo",
|
||||
Provider: "gopls",
|
||||
IndexedSHA: "abc123",
|
||||
CompletedAt: 1_700_000_000,
|
||||
Coverage: 91.5,
|
||||
}
|
||||
if err := s.SetEnrichmentState(st); err != nil {
|
||||
t.Fatalf("SetEnrichmentState: %v", err)
|
||||
}
|
||||
|
||||
got, found, err := s.GetEnrichmentState("repo", "gopls")
|
||||
if err != nil || !found {
|
||||
t.Fatalf("get after set: found=%v err=%v, want found=true", found, err)
|
||||
}
|
||||
if got.RepoPrefix != "repo" || got.Provider != "gopls" ||
|
||||
got.IndexedSHA != "abc123" || got.CompletedAt != 1_700_000_000 || got.Coverage != 91.5 {
|
||||
t.Fatalf("round-trip mismatch: %+v", got)
|
||||
}
|
||||
|
||||
// Upsert on (repo_prefix, provider) replaces in place.
|
||||
st.IndexedSHA = "def456"
|
||||
st.Coverage = 100
|
||||
if err := s.SetEnrichmentState(st); err != nil {
|
||||
t.Fatalf("upsert: %v", err)
|
||||
}
|
||||
got, _, _ = s.GetEnrichmentState("repo", "gopls")
|
||||
if got.IndexedSHA != "def456" || got.Coverage != 100 {
|
||||
t.Fatalf("upsert did not replace in place: %+v", got)
|
||||
}
|
||||
|
||||
// A different provider under the same repo is a distinct row.
|
||||
if _, found, _ := s.GetEnrichmentState("repo", "scip-go"); found {
|
||||
t.Fatalf("a different provider must be its own row, got found=true")
|
||||
}
|
||||
}
|
||||
|
||||
// TestOpenPreservesEnrichmentStateOnReopen proves the new table is picked up
|
||||
// by an existing store with no wipe: a marker written on the first open
|
||||
// survives the second open (which re-runs schemaSQL unconditionally) and the
|
||||
// schema version does not drift. Mirrors TestOpenAtCurrentVersionIsNoOp.
|
||||
func TestOpenPreservesEnrichmentStateOnReopen(t *testing.T) {
|
||||
path := filepath.Join(t.TempDir(), "store.sqlite")
|
||||
|
||||
s, err := Open(path)
|
||||
if err != nil {
|
||||
t.Fatalf("first open: %v", err)
|
||||
}
|
||||
if err := s.SetEnrichmentState(graph.EnrichmentState{
|
||||
RepoPrefix: "r", Provider: "gopls", IndexedSHA: "sha1", CompletedAt: 42, Coverage: 88,
|
||||
}); err != nil {
|
||||
t.Fatalf("seed marker: %v", err)
|
||||
}
|
||||
if err := s.Close(); err != nil {
|
||||
t.Fatalf("close: %v", err)
|
||||
}
|
||||
|
||||
s2, err := Open(path)
|
||||
if err != nil {
|
||||
t.Fatalf("reopen: %v", err)
|
||||
}
|
||||
defer s2.Close()
|
||||
|
||||
got, found, err := s2.GetEnrichmentState("r", "gopls")
|
||||
if err != nil || !found {
|
||||
t.Fatalf("marker lost across reopen: found=%v err=%v (the table must survive a no-op reopen)", found, err)
|
||||
}
|
||||
if got.IndexedSHA != "sha1" || got.Coverage != 88 {
|
||||
t.Fatalf("marker corrupted across reopen: %+v", got)
|
||||
}
|
||||
if v, _ := readUserVersion(s2.db); v != currentSchemaVersion {
|
||||
t.Fatalf("schema version drifted to %d after reopen, want %d", v, currentSchemaVersion)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,118 @@
|
||||
package store_sqlite
|
||||
|
||||
import (
|
||||
"github.com/zzet/gortex/internal/graph"
|
||||
)
|
||||
|
||||
// Compile-time assertions that the SQLite Store satisfies the optional
|
||||
// per-file metadata persistence capability (the files sidecar feeding
|
||||
// index_health's per-file parse-error / node-count rollup).
|
||||
var (
|
||||
_ graph.FileMetaWriter = (*Store)(nil)
|
||||
_ graph.FileMetaReader = (*Store)(nil)
|
||||
)
|
||||
|
||||
// fileMetaChunk bounds rows per multi-row INSERT (6 params/row; 80 rows =
|
||||
// 480 host params, well under SQLite's 999 default).
|
||||
const fileMetaChunk = 80
|
||||
|
||||
// SetFileMetas upserts per-file metadata rows for one repo prefix in a single
|
||||
// transaction, chunked under the host-parameter limit. Idempotent on the
|
||||
// (repo_prefix, file_path) primary key. Empty input is a no-op.
|
||||
func (s *Store) SetFileMetas(repoPrefix string, rows []graph.FileMetaRow) error {
|
||||
if len(rows) == 0 {
|
||||
return nil
|
||||
}
|
||||
s.writeMu.Lock()
|
||||
defer s.writeMu.Unlock()
|
||||
|
||||
tx, err := s.db.Begin()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer tx.Rollback() //nolint:errcheck // rollback after Commit is a no-op
|
||||
|
||||
for start := 0; start < len(rows); start += fileMetaChunk {
|
||||
end := start + fileMetaChunk
|
||||
if end > len(rows) {
|
||||
end = len(rows)
|
||||
}
|
||||
batch := rows[start:end]
|
||||
args := make([]any, 0, len(batch)*6)
|
||||
stmt := make([]byte, 0, 96+len(batch)*24)
|
||||
stmt = append(stmt, "INSERT OR REPLACE INTO files (repo_prefix, file_path, content_hash, size, node_count, errors) VALUES "...)
|
||||
for i, r := range batch {
|
||||
if i > 0 {
|
||||
stmt = append(stmt, ',')
|
||||
}
|
||||
stmt = append(stmt, "(?, ?, ?, ?, ?, ?)"...)
|
||||
args = append(args, repoPrefix, r.FilePath, r.ContentHash, r.Size, r.NodeCount, r.Errors)
|
||||
}
|
||||
if _, err := tx.Exec(string(stmt), args...); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return tx.Commit()
|
||||
}
|
||||
|
||||
// DeleteFileMetasByFiles drops the metadata rows for the supplied files in one
|
||||
// repo prefix, chunked into `file_path IN (…)` DELETEs. Empty input is a no-op.
|
||||
func (s *Store) DeleteFileMetasByFiles(repoPrefix string, files []string) error {
|
||||
if len(files) == 0 {
|
||||
return nil
|
||||
}
|
||||
s.writeMu.Lock()
|
||||
defer s.writeMu.Unlock()
|
||||
|
||||
tx, err := s.db.Begin()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer tx.Rollback() //nolint:errcheck // rollback after Commit is a no-op
|
||||
|
||||
for start := 0; start < len(files); start += fileMetaChunk {
|
||||
end := start + fileMetaChunk
|
||||
if end > len(files) {
|
||||
end = len(files)
|
||||
}
|
||||
chunk := files[start:end]
|
||||
args := make([]any, 0, len(chunk)+1)
|
||||
args = append(args, repoPrefix)
|
||||
stmt := make([]byte, 0, 64+len(chunk)*2)
|
||||
stmt = append(stmt, "DELETE FROM files WHERE repo_prefix = ? AND file_path IN ("...)
|
||||
for i, f := range chunk {
|
||||
if i > 0 {
|
||||
stmt = append(stmt, ',')
|
||||
}
|
||||
stmt = append(stmt, '?')
|
||||
args = append(args, f)
|
||||
}
|
||||
stmt = append(stmt, ')')
|
||||
if _, err := tx.Exec(string(stmt), args...); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return tx.Commit()
|
||||
}
|
||||
|
||||
// FileMetasForRepo returns every recorded file row for the repo prefix.
|
||||
// Always non-nil.
|
||||
func (s *Store) FileMetasForRepo(repoPrefix string) ([]graph.FileMetaRow, error) {
|
||||
rows, err := s.db.Query(
|
||||
`SELECT file_path, content_hash, size, node_count, errors FROM files WHERE repo_prefix = ? ORDER BY file_path`,
|
||||
repoPrefix,
|
||||
)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
out := []graph.FileMetaRow{}
|
||||
for rows.Next() {
|
||||
var r graph.FileMetaRow
|
||||
if err := rows.Scan(&r.FilePath, &r.ContentHash, &r.Size, &r.NodeCount, &r.Errors); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out = append(out, r)
|
||||
}
|
||||
return out, rows.Err()
|
||||
}
|
||||
@@ -0,0 +1,69 @@
|
||||
package store_sqlite
|
||||
|
||||
import (
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"github.com/zzet/gortex/internal/graph"
|
||||
)
|
||||
|
||||
// TestFileMetas_RoundTrip pins the per-file metadata sidecar: rows upsert,
|
||||
// read back per repo, carry their errors JSON, and a per-file delete removes
|
||||
// just the named file.
|
||||
func TestFileMetas_RoundTrip(t *testing.T) {
|
||||
s, err := Open(filepath.Join(t.TempDir(), "f.sqlite"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer s.Close()
|
||||
|
||||
rows := []graph.FileMetaRow{
|
||||
{FilePath: "a/x.go", ContentHash: "h1", Size: 100, NodeCount: 7, Errors: ""},
|
||||
{FilePath: "a/broken.go", ContentHash: "h2", Size: 50, NodeCount: 1, Errors: `["3:5","4:1"]`},
|
||||
}
|
||||
if err := s.SetFileMetas("repoA", rows); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// A different repo's row must not bleed in.
|
||||
if err := s.SetFileMetas("repoB", []graph.FileMetaRow{{FilePath: "b/y.go", NodeCount: 2}}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
got, err := s.FileMetasForRepo("repoA")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(got) != 2 {
|
||||
t.Fatalf("FileMetasForRepo(repoA) = %d rows, want 2", len(got))
|
||||
}
|
||||
byFile := map[string]graph.FileMetaRow{}
|
||||
for _, r := range got {
|
||||
byFile[r.FilePath] = r
|
||||
}
|
||||
if r := byFile["a/x.go"]; r.NodeCount != 7 || r.Size != 100 || r.ContentHash != "h1" || r.Errors != "" {
|
||||
t.Errorf("x.go row = %+v", r)
|
||||
}
|
||||
if r := byFile["a/broken.go"]; r.NodeCount != 1 || r.Errors != `["3:5","4:1"]` {
|
||||
t.Errorf("broken.go row = %+v", r)
|
||||
}
|
||||
|
||||
// Upsert replaces in place.
|
||||
if err := s.SetFileMetas("repoA", []graph.FileMetaRow{{FilePath: "a/x.go", NodeCount: 9, Errors: ""}}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
got, _ = s.FileMetasForRepo("repoA")
|
||||
for _, r := range got {
|
||||
if r.FilePath == "a/x.go" && r.NodeCount != 9 {
|
||||
t.Errorf("upsert did not replace node_count: %+v", r)
|
||||
}
|
||||
}
|
||||
|
||||
// Per-file delete removes only the named file.
|
||||
if err := s.DeleteFileMetasByFiles("repoA", []string{"a/broken.go"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
got, _ = s.FileMetasForRepo("repoA")
|
||||
if len(got) != 1 || got[0].FilePath != "a/x.go" {
|
||||
t.Errorf("after delete, rows = %+v, want only a/x.go", got)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,38 @@
|
||||
package store_sqlite
|
||||
|
||||
import (
|
||||
"iter"
|
||||
|
||||
"github.com/zzet/gortex/internal/graph"
|
||||
)
|
||||
|
||||
// Compile-time assertion: *Store serves the fn-value placeholder scan.
|
||||
var _ graph.FnValuePlaceholderScanner = (*Store)(nil)
|
||||
|
||||
// FnValuePlaceholderEdges implements graph.FnValuePlaceholderScanner: it yields
|
||||
// only the fn-value gate's placeholder edges, the exact inverse of the
|
||||
// fn-value exclusion EdgesWithUnresolvedTarget applies. The predicate is the
|
||||
// SAME two-form filter the v2 migration's dedupeFnValuePlaceholderEdges uses:
|
||||
// the bare `unresolved::fnvalue::` range rides edges_by_to(to_id) (the ':;'
|
||||
// range end is ':'+1, one past the marker); the multi-repo COPY-rewrite infix
|
||||
// form is caught by is_unresolved = 1 + LIKE. Full column set incl. meta — the
|
||||
// gate reads Meta["via"] and the captured fn_value_name off each placeholder.
|
||||
//
|
||||
// The whole point is that the gate no longer has to scan the entire
|
||||
// EdgeReferences kind (placeholders + every real reference) and Go-filter on
|
||||
// every whole-graph synthesizer pass; it pulls the handful of placeholders
|
||||
// straight off the index instead.
|
||||
func (s *Store) FnValuePlaceholderEdges() iter.Seq[*graph.Edge] {
|
||||
return func(yield func(*graph.Edge) bool) {
|
||||
out := s.queryEdgesSQL(`
|
||||
SELECT from_id, to_id, kind, file_path, line, confidence, confidence_label, origin, tier, cross_repo, meta, resolve_terminal, resolve_terminal_reason
|
||||
FROM edges
|
||||
WHERE (to_id >= 'unresolved::fnvalue::' AND to_id < 'unresolved::fnvalue:;')
|
||||
OR (is_unresolved = 1 AND to_id LIKE '%::unresolved::fnvalue::%')`)
|
||||
for _, e := range out {
|
||||
if !yield(e) {
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,482 @@
|
||||
package store_sqlite
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"strings"
|
||||
|
||||
"github.com/zzet/gortex/internal/graph"
|
||||
"github.com/zzet/gortex/internal/search"
|
||||
)
|
||||
|
||||
// This file implements graph.SymbolSearcher + graph.SymbolBundleSearcher
|
||||
// on the SQLite backend using the FTS5 virtual table declared in
|
||||
// schema.go (symbol_fts). It is the on-disk replacement for the
|
||||
// multi-GB in-heap Bleve/BM25 index: the FTS5 inverted index lives in
|
||||
// the same .sqlite file as the graph, and a tier-0 exact-name boost
|
||||
// short-circuits identifier queries so
|
||||
// search quality holds or improves while the heap shrinks.
|
||||
//
|
||||
// Semantics:
|
||||
//
|
||||
// - BulkUpsertSymbolFTS wipes only the rows owned by repoPrefix
|
||||
// before re-inserting, so sibling repos sharing one store don't
|
||||
// clobber each other's corpus. Empty prefix wipes the whole table
|
||||
// (single-repo / conformance behaviour).
|
||||
//
|
||||
// - SearchSymbols tier 0: an identifier query (no whitespace / path
|
||||
// separators) that resolves to one or more nodes by exact name is
|
||||
// returned directly with a fixed dominant score, skipping FTS.
|
||||
// Misses fall through to the FTS5 MATCH path.
|
||||
//
|
||||
// - SearchSymbolBundles composes the same hit list with batched
|
||||
// node + in/out edge fetches the rerank pipeline reads from.
|
||||
//
|
||||
// FTS5 maintains its index incrementally on every insert, so the
|
||||
// Store struct needs no extra state and BuildSymbolIndex is a no-op
|
||||
// (it only opportunistically merges segments).
|
||||
|
||||
// Compile-time assertions: *Store satisfies the symbol-search
|
||||
// capabilities. The indexer auto-engages these when the active backend
|
||||
// implements them, routing search_symbols through on-disk FTS5 instead
|
||||
// of the in-process BM25 index.
|
||||
var (
|
||||
_ graph.SymbolSearcher = (*Store)(nil)
|
||||
_ graph.SymbolBundleSearcher = (*Store)(nil)
|
||||
_ graph.BundleFingerprintSink = (*Store)(nil)
|
||||
)
|
||||
|
||||
// ftsInsertChunkRows bounds the rows per multi-row INSERT. Each row
|
||||
// binds 3 host params (node_id, repo_prefix, tokens); 300 rows is 900
|
||||
// params, comfortably under SQLite's default 999-variable limit so the
|
||||
// statement stays portable across builds.
|
||||
const ftsInsertChunkRows = 300
|
||||
|
||||
// UpsertSymbolFTS records (or replaces) the pre-tokenised text for
|
||||
// nodeID. FTS5 offers no UPSERT on a table with UNINDEXED columns, so
|
||||
// the write is delete-then-insert. The delete targets the prior row's
|
||||
// FTS5 docid (rowid), looked up from the symbol_fts_rowid sidecar —
|
||||
// node_id is UNINDEXED, so "DELETE … WHERE node_id = ?" would full-scan
|
||||
// the whole index once per symbol, which is quadratic over a file's
|
||||
// symbols on the per-edit reindex hot path. The repo_prefix is derived
|
||||
// from the owning node (nodes.repo_prefix) so the per-repo staleness
|
||||
// wipe in BulkUpsertSymbolFTS can scope by prefix; if the node is absent
|
||||
// the prefix defaults to "".
|
||||
func (s *Store) UpsertSymbolFTS(nodeID, tokens string) error {
|
||||
if nodeID == "" {
|
||||
return nil
|
||||
}
|
||||
s.writeMu.Lock()
|
||||
defer s.writeMu.Unlock()
|
||||
|
||||
var repoPrefix string
|
||||
// A missing node (or a scan error) leaves repoPrefix == "" — the
|
||||
// row is still indexable, it just won't be reachable by a per-repo
|
||||
// prefix wipe. The graph.Store contract has no error channel for
|
||||
// the indexer's incremental writes, so we don't surface this.
|
||||
_ = s.db.QueryRow(`SELECT repo_prefix FROM nodes WHERE id = ?`, nodeID).Scan(&repoPrefix)
|
||||
|
||||
// Delete the prior row by its docid (O(log n)) instead of by node_id
|
||||
// (full FTS scan). A missing map entry means no prior row to drop —
|
||||
// the sidecar is kept in lockstep with symbol_fts by every writer and
|
||||
// backfilled at Open for databases built before it existed, so a miss
|
||||
// here is a genuinely new symbol, not a stale row we're leaking.
|
||||
var oldRowid int64
|
||||
switch err := s.db.QueryRow(
|
||||
`SELECT fts_rowid FROM symbol_fts_rowid WHERE node_id = ?`, nodeID,
|
||||
).Scan(&oldRowid); err {
|
||||
case nil:
|
||||
if _, err := s.db.Exec(`DELETE FROM symbol_fts WHERE rowid = ?`, oldRowid); err != nil {
|
||||
return err
|
||||
}
|
||||
case sql.ErrNoRows:
|
||||
// new symbol — nothing to delete
|
||||
default:
|
||||
return err
|
||||
}
|
||||
|
||||
res, err := s.db.Exec(
|
||||
`INSERT INTO symbol_fts (node_id, repo_prefix, tokens) VALUES (?, ?, ?)`,
|
||||
nodeID, repoPrefix, tokens,
|
||||
)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
newRowid, err := res.LastInsertId()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := s.db.Exec(
|
||||
`INSERT OR REPLACE INTO symbol_fts_rowid (node_id, repo_prefix, fts_rowid) VALUES (?, ?, ?)`,
|
||||
nodeID, repoPrefix, newRowid,
|
||||
); err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// BulkUpsertSymbolFTS is the cold-start fast path: wipe this repo's
|
||||
// stale rows, then chunked multi-row INSERT of the deduped items. The
|
||||
// whole thing runs in one transaction under writeMu so a concurrent
|
||||
// reader never observes the table mid-wipe.
|
||||
//
|
||||
// repoPrefix scopes the pre-insert wipe: a non-empty prefix deletes
|
||||
// only rows owned by that repo,
|
||||
// leaving siblings untouched; an empty prefix wipes the whole table
|
||||
// (single-repo / conformance behaviour — the conformance suite calls
|
||||
// this with ""). Items are deduped by NodeID with last-write-wins,
|
||||
// matching UpsertSymbolFTS's replace semantics.
|
||||
func (s *Store) BulkUpsertSymbolFTS(repoPrefix string, items []graph.SymbolFTSItem) error {
|
||||
if len(items) == 0 {
|
||||
return nil
|
||||
}
|
||||
s.writeMu.Lock()
|
||||
defer s.writeMu.Unlock()
|
||||
|
||||
// Dedup by ID — last write wins, mirroring UpsertSymbolFTS's
|
||||
// delete-then-insert. Guards the edge case where a re-parse of a
|
||||
// file emitted the same ID twice.
|
||||
pos := make(map[string]int, len(items))
|
||||
deduped := items[:0]
|
||||
for _, it := range items {
|
||||
if it.NodeID == "" {
|
||||
continue
|
||||
}
|
||||
if p, ok := pos[it.NodeID]; ok {
|
||||
deduped[p] = it
|
||||
} else {
|
||||
pos[it.NodeID] = len(deduped)
|
||||
deduped = append(deduped, it)
|
||||
}
|
||||
}
|
||||
items = deduped
|
||||
if len(items) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
tx, err := s.db.Begin()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
commit := false
|
||||
defer func() {
|
||||
if !commit {
|
||||
_ = tx.Rollback()
|
||||
}
|
||||
}()
|
||||
|
||||
// Wipe this repo's prior rows so a clean rebuild of repo A doesn't
|
||||
// leave phantom hits, while sibling repo B's corpus survives. The
|
||||
// repo_prefix column is UNINDEXED but still stored, so the equality
|
||||
// filter is a literal compare over the row set. Empty repoPrefix
|
||||
// clears the whole table — the legacy single-repo wipe.
|
||||
if _, err := tx.Exec(`DELETE FROM symbol_fts WHERE repo_prefix = ?`, repoPrefix); err != nil {
|
||||
return err
|
||||
}
|
||||
// Drop this repo's rowid-map entries in lockstep with the symbol_fts
|
||||
// wipe so the two never diverge; they are rebuilt from the freshly
|
||||
// inserted rows below.
|
||||
if _, err := tx.Exec(`DELETE FROM symbol_fts_rowid WHERE repo_prefix = ?`, repoPrefix); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
for start := 0; start < len(items); start += ftsInsertChunkRows {
|
||||
end := minInt(start+ftsInsertChunkRows, len(items))
|
||||
chunk := items[start:end]
|
||||
|
||||
var b strings.Builder
|
||||
b.WriteString(`INSERT INTO symbol_fts (node_id, repo_prefix, tokens) VALUES `)
|
||||
args := make([]any, 0, len(chunk)*3)
|
||||
for i, it := range chunk {
|
||||
if i > 0 {
|
||||
b.WriteByte(',')
|
||||
}
|
||||
b.WriteString(`(?,?,?)`)
|
||||
args = append(args, it.NodeID, repoPrefix, it.Tokens)
|
||||
}
|
||||
if _, err := tx.Exec(b.String(), args...); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
// Rebuild the rowid map for this repo from the rows just inserted. A
|
||||
// full multi-row INSERT only exposes the last docid, so we read the
|
||||
// docids back in one pass (a linear filter over the UNINDEXED
|
||||
// repo_prefix column — the cold/bulk path, not the per-edit hot path).
|
||||
if _, err := tx.Exec(
|
||||
`INSERT OR REPLACE INTO symbol_fts_rowid (node_id, repo_prefix, fts_rowid)
|
||||
SELECT node_id, repo_prefix, rowid FROM symbol_fts WHERE repo_prefix = ?`,
|
||||
repoPrefix,
|
||||
); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if err := tx.Commit(); err != nil {
|
||||
return err
|
||||
}
|
||||
commit = true
|
||||
return nil
|
||||
}
|
||||
|
||||
// backfillSymbolFTSRowidMap populates symbol_fts_rowid from symbol_fts for
|
||||
// a database built before the sidecar existed. Without it, the first
|
||||
// incremental UpsertSymbolFTS for an already-indexed symbol would find no
|
||||
// map entry, skip the delete, and leak a duplicate FTS row. It is a
|
||||
// one-time cost: skipped once the map has any row (steady state) or when
|
||||
// the FTS index is empty (a fresh DB the bulk path will populate with the
|
||||
// map maintained inline). Runs at Open, before any reader or writer.
|
||||
func backfillSymbolFTSRowidMap(db *sql.DB) error {
|
||||
var mapped bool
|
||||
if err := db.QueryRow(`SELECT EXISTS(SELECT 1 FROM symbol_fts_rowid)`).Scan(&mapped); err != nil {
|
||||
return err
|
||||
}
|
||||
if mapped {
|
||||
return nil
|
||||
}
|
||||
var hasFTS bool
|
||||
if err := db.QueryRow(`SELECT EXISTS(SELECT 1 FROM symbol_fts)`).Scan(&hasFTS); err != nil {
|
||||
return err
|
||||
}
|
||||
if !hasFTS {
|
||||
return nil
|
||||
}
|
||||
_, err := db.Exec(
|
||||
`INSERT OR REPLACE INTO symbol_fts_rowid (node_id, repo_prefix, fts_rowid)
|
||||
SELECT node_id, repo_prefix, rowid FROM symbol_fts`)
|
||||
return err
|
||||
}
|
||||
|
||||
// BuildSymbolIndex is a no-op for FTS5: the index is maintained
|
||||
// incrementally on every insert, so there is nothing to build after the
|
||||
// bulk parse phase. We opportunistically run the FTS5 'optimize'
|
||||
// command to merge segments (purely a read-latency improvement); any
|
||||
// error is ignored because the index is already correct without it.
|
||||
// Idempotent — safe to call any number of times.
|
||||
func (s *Store) BuildSymbolIndex() error {
|
||||
s.writeMu.Lock()
|
||||
defer s.writeMu.Unlock()
|
||||
_, _ = s.db.Exec(`INSERT INTO symbol_fts(symbol_fts) VALUES('optimize')`)
|
||||
return nil
|
||||
}
|
||||
|
||||
// SearchSymbols runs a symbol query and returns hits ordered by
|
||||
// descending relevance (higher Score = more relevant).
|
||||
//
|
||||
// Tier 0 (exact-name boost): when the
|
||||
// query looks like a literal identifier and resolves to one or more
|
||||
// nodes by exact name, return those directly with a fixed dominant
|
||||
// score (100.0) — an O(1)-ish index seek that beats FTS ranking for
|
||||
// the common "type the symbol name" case. Misses fall through to FTS5.
|
||||
//
|
||||
// Otherwise tokenise on the read side with the SAME splitter as the
|
||||
// write side (search.Tokenize) so a camelCase query lands on the
|
||||
// split corpus, build a prefix-OR MATCH expression, and rank by BM25.
|
||||
// SQLite's bm25() returns lower-is-better, so the stored Score is its
|
||||
// negation (higher-is-better, matching the SymbolHit contract).
|
||||
func (s *Store) SearchSymbols(query string, limit int) ([]graph.SymbolHit, error) {
|
||||
if query == "" {
|
||||
return nil, nil
|
||||
}
|
||||
if limit <= 0 {
|
||||
limit = 20
|
||||
}
|
||||
|
||||
// Tier 0: exact-name lookup. Only engage for identifier-shaped
|
||||
// queries (no whitespace / path separators); multi-word queries are
|
||||
// concept searches that need BM25 ranking. We only short-circuit
|
||||
// when the lookup hits at least one node — misses fall through so a
|
||||
// partial-identifier query still reaches FTS.
|
||||
if isIdentifierQuery(query) {
|
||||
ns := s.FindNodesByName(query)
|
||||
if len(ns) > 0 {
|
||||
out := make([]graph.SymbolHit, 0, minInt(len(ns), limit))
|
||||
for _, n := range ns {
|
||||
if n == nil || n.ID == "" {
|
||||
continue
|
||||
}
|
||||
out = append(out, graph.SymbolHit{NodeID: n.ID, Score: 100.0})
|
||||
if len(out) >= limit {
|
||||
break
|
||||
}
|
||||
}
|
||||
if len(out) > 0 {
|
||||
return out, nil
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
match := s.buildFTSMatch(query)
|
||||
if match == "" {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
const q = `SELECT node_id, bm25(symbol_fts) FROM symbol_fts WHERE symbol_fts MATCH ? ORDER BY bm25(symbol_fts) LIMIT ?`
|
||||
rows, err := s.db.Query(q, match, limit)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
var hits []graph.SymbolHit
|
||||
for rows.Next() {
|
||||
var (
|
||||
id string
|
||||
score float64
|
||||
)
|
||||
if err := rows.Scan(&id, &score); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if id == "" {
|
||||
continue
|
||||
}
|
||||
// bm25() is negative-better in SQLite; negate so higher = better,
|
||||
// matching the SymbolHit contract. Rows already arrive in bm25
|
||||
// (best-first) order from the ORDER BY.
|
||||
hits = append(hits, graph.SymbolHit{NodeID: id, Score: -score})
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return hits, nil
|
||||
}
|
||||
|
||||
// buildFTSMatch tokenises the query with the write-side splitter and
|
||||
// builds an FTS5 MATCH expression: each token becomes a quoted prefix
|
||||
// term ("tok"*) and the terms are OR-joined so any token match counts.
|
||||
// Returns "" when the query degenerates to no tokens.
|
||||
func (s *Store) buildFTSMatch(query string) string {
|
||||
tokens := search.Tokenize(query)
|
||||
if len(tokens) == 0 {
|
||||
// Fallback: when Tokenize drops everything (e.g. a single
|
||||
// sub-2-char token like "go"), use the looser query tokeniser so
|
||||
// the search still reaches the engine instead of returning empty.
|
||||
tokens = search.TokenizeQuery(query)
|
||||
if len(tokens) == 0 {
|
||||
return ""
|
||||
}
|
||||
}
|
||||
parts := make([]string, 0, len(tokens))
|
||||
for _, t := range tokens {
|
||||
if t == "" {
|
||||
continue
|
||||
}
|
||||
parts = append(parts, `"`+escapeFTSQuote(t)+`"*`)
|
||||
}
|
||||
if len(parts) == 0 {
|
||||
return ""
|
||||
}
|
||||
return strings.Join(parts, " OR ")
|
||||
}
|
||||
|
||||
// escapeFTSQuote escapes a token for use inside an FTS5 double-quoted
|
||||
// string literal: a literal double quote is doubled ("" inside "...").
|
||||
func escapeFTSQuote(t string) string {
|
||||
return strings.ReplaceAll(t, `"`, `""`)
|
||||
}
|
||||
|
||||
// SearchSymbolBundles is the rerank-shaped fast path: it runs
|
||||
// SearchSymbols to get the ranked id list (preserving order) plus a
|
||||
// score-by-id map, then materialises the nodes and their in/out edges
|
||||
// in batched fetches the rerank pipeline reads from. The engine routes
|
||||
// through this when the backend implements SymbolBundleSearcher,
|
||||
// pre-seeding rerank.Context's edge caches.
|
||||
func (s *Store) SearchSymbolBundles(query string, limit int) ([]graph.SymbolBundle, error) {
|
||||
hits, err := s.SearchSymbols(query, limit)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if len(hits) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
ids := make([]string, 0, len(hits))
|
||||
scoreByID := make(map[string]float64, len(hits))
|
||||
for _, h := range hits {
|
||||
if h.NodeID == "" {
|
||||
continue
|
||||
}
|
||||
if _, dup := scoreByID[h.NodeID]; dup {
|
||||
// First hit keeps the score / position; defend against a
|
||||
// future ranker that returns an id more than once.
|
||||
continue
|
||||
}
|
||||
scoreByID[h.NodeID] = h.Score
|
||||
ids = append(ids, h.NodeID)
|
||||
}
|
||||
if len(ids) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
// Content-addressed cache: serve cached bundles for IDs whose
|
||||
// package fingerprint is unchanged and fetch only the misses. The
|
||||
// cache is nil until the daemon wires fingerprints, in which case
|
||||
// every ID is a miss and the path is exactly the legacy fetch.
|
||||
cached := make(map[string]graph.SymbolBundle, len(ids))
|
||||
missIDs := ids
|
||||
if s.bundles != nil {
|
||||
missIDs = missIDs[:0:0]
|
||||
for _, id := range ids {
|
||||
if b, ok := s.bundles.lookup(id); ok {
|
||||
cached[id] = b
|
||||
continue
|
||||
}
|
||||
missIDs = append(missIDs, id)
|
||||
}
|
||||
}
|
||||
|
||||
// Fetch the misses' nodes + in/out edges in one batched round-trip
|
||||
// each. A full cache hit skips all three fetches entirely.
|
||||
var nodes map[string]*graph.Node
|
||||
var out, in map[string][]*graph.Edge
|
||||
if len(missIDs) > 0 {
|
||||
nodes = s.GetNodesByIDs(missIDs)
|
||||
out = s.GetOutEdgesByNodeIDs(missIDs)
|
||||
in = s.GetInEdgesByNodeIDs(missIDs)
|
||||
}
|
||||
|
||||
bundles := make([]graph.SymbolBundle, 0, len(ids))
|
||||
for _, id := range ids {
|
||||
if b, ok := cached[id]; ok {
|
||||
// The cached bundle's score is whatever it was first cached
|
||||
// with; the live FTS score for THIS query is authoritative,
|
||||
// so re-stamp it (the score is query-specific, the node +
|
||||
// edges are not).
|
||||
b.Score = scoreByID[id]
|
||||
bundles = append(bundles, b)
|
||||
continue
|
||||
}
|
||||
n := nodes[id]
|
||||
if n == nil {
|
||||
// Hit references a node evicted between the search and the
|
||||
// node fetch — skip; the caller does its own dedup / filter.
|
||||
continue
|
||||
}
|
||||
b := graph.SymbolBundle{
|
||||
Node: n,
|
||||
Score: scoreByID[id],
|
||||
OutEdges: out[id],
|
||||
InEdges: in[id],
|
||||
}
|
||||
if s.bundles != nil {
|
||||
s.bundles.store(b)
|
||||
}
|
||||
bundles = append(bundles, b)
|
||||
}
|
||||
return bundles, nil
|
||||
}
|
||||
|
||||
// isIdentifierQuery reports whether a query looks like a literal symbol
|
||||
// name (no whitespace, no path separators, no dots, no colons, no
|
||||
// commas). The tier-0 exact-name fast path engages only on such
|
||||
// queries; multi-token / path / qualified queries always go to FTS.
|
||||
func isIdentifierQuery(q string) bool {
|
||||
if q == "" {
|
||||
return false
|
||||
}
|
||||
for _, r := range q {
|
||||
switch r {
|
||||
case ' ', '\t', '\n', '/', '.', ':', ',':
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
@@ -0,0 +1,45 @@
|
||||
package store_sqlite
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
|
||||
"github.com/zzet/gortex/internal/graph"
|
||||
)
|
||||
|
||||
// SetRepoIndexState upserts the freshness-provenance row for one repo —
|
||||
// written at the end of every (re)index. One row per repo_prefix.
|
||||
func (s *Store) SetRepoIndexState(st graph.RepoIndexState) error {
|
||||
s.writeMu.Lock()
|
||||
defer s.writeMu.Unlock()
|
||||
dirty := 0
|
||||
if st.Dirty {
|
||||
dirty = 1
|
||||
}
|
||||
_, err := s.db.Exec(`
|
||||
INSERT OR REPLACE INTO repo_index_state
|
||||
(repo_prefix, indexed_sha, dirty, indexed_at, workspace_fp, node_count, edge_count, extractor_versions)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?)`,
|
||||
st.RepoPrefix, st.IndexedSHA, dirty, st.IndexedAt, st.WorkspaceFP,
|
||||
st.NodeCount, st.EdgeCount, st.ExtractorVersions)
|
||||
return err
|
||||
}
|
||||
|
||||
// GetRepoIndexState returns the recorded freshness provenance for a repo.
|
||||
// The bool is false when no row exists yet (never-indexed / pre-feature).
|
||||
func (s *Store) GetRepoIndexState(repoPrefix string) (graph.RepoIndexState, bool, error) {
|
||||
row := s.db.QueryRow(`
|
||||
SELECT indexed_sha, dirty, indexed_at, workspace_fp, node_count, edge_count, extractor_versions
|
||||
FROM repo_index_state WHERE repo_prefix = ?`, repoPrefix)
|
||||
st := graph.RepoIndexState{RepoPrefix: repoPrefix}
|
||||
var dirty int
|
||||
err := row.Scan(&st.IndexedSHA, &dirty, &st.IndexedAt, &st.WorkspaceFP,
|
||||
&st.NodeCount, &st.EdgeCount, &st.ExtractorVersions)
|
||||
if err == sql.ErrNoRows {
|
||||
return graph.RepoIndexState{RepoPrefix: repoPrefix}, false, nil
|
||||
}
|
||||
if err != nil {
|
||||
return graph.RepoIndexState{RepoPrefix: repoPrefix}, false, err
|
||||
}
|
||||
st.Dirty = dirty != 0
|
||||
return st, true, nil
|
||||
}
|
||||
@@ -0,0 +1,61 @@
|
||||
package store_sqlite_test
|
||||
|
||||
import (
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"github.com/zzet/gortex/internal/graph"
|
||||
"github.com/zzet/gortex/internal/graph/store_sqlite"
|
||||
)
|
||||
|
||||
func openIndexStateStore(t *testing.T) *store_sqlite.Store {
|
||||
t.Helper()
|
||||
s, err := store_sqlite.Open(filepath.Join(t.TempDir(), "is.sqlite"))
|
||||
require.NoError(t, err)
|
||||
t.Cleanup(func() { _ = s.Close() })
|
||||
return s
|
||||
}
|
||||
|
||||
func TestRepoIndexState_RoundTrip(t *testing.T) {
|
||||
s := openIndexStateStore(t)
|
||||
|
||||
// Absent state reads back as (zero, false, nil).
|
||||
got, ok, err := s.GetRepoIndexState("gortex")
|
||||
require.NoError(t, err)
|
||||
require.False(t, ok)
|
||||
require.Equal(t, "gortex", got.RepoPrefix)
|
||||
|
||||
want := graph.RepoIndexState{
|
||||
RepoPrefix: "gortex",
|
||||
IndexedSHA: "abc123",
|
||||
Dirty: true,
|
||||
IndexedAt: 1700000000,
|
||||
WorkspaceFP: "deadbeef",
|
||||
NodeCount: 42,
|
||||
EdgeCount: 99,
|
||||
ExtractorVersions: `{"go":2}`,
|
||||
}
|
||||
require.NoError(t, s.SetRepoIndexState(want))
|
||||
|
||||
got, ok, err = s.GetRepoIndexState("gortex")
|
||||
require.NoError(t, err)
|
||||
require.True(t, ok)
|
||||
require.Equal(t, want, got)
|
||||
|
||||
// Upsert replaces in place (one row per repo_prefix).
|
||||
want.IndexedSHA = "def456"
|
||||
want.Dirty = false
|
||||
require.NoError(t, s.SetRepoIndexState(want))
|
||||
got, ok, err = s.GetRepoIndexState("gortex")
|
||||
require.NoError(t, err)
|
||||
require.True(t, ok)
|
||||
require.Equal(t, "def456", got.IndexedSHA)
|
||||
require.False(t, got.Dirty)
|
||||
|
||||
// A different repo is isolated.
|
||||
_, ok, err = s.GetRepoIndexState("other")
|
||||
require.NoError(t, err)
|
||||
require.False(t, ok)
|
||||
}
|
||||
@@ -0,0 +1,54 @@
|
||||
package store_sqlite
|
||||
|
||||
import "github.com/zzet/gortex/internal/graph"
|
||||
|
||||
// Compile-time assertion: *Store serves the meta-less kind-scoped edge scan.
|
||||
var _ graph.LightEdgeScanner = (*Store)(nil)
|
||||
|
||||
// edgeColsLight is the meta-less edge column projection: the promoted struct
|
||||
// columns WITHOUT the meta blob (and without resolve_terminal, which lives in
|
||||
// Meta). It is exactly the ten columns scanEdgeLight scans, and is shared with
|
||||
// the stmtOutEdgesLight prepared statement so the projection can never drift
|
||||
// from the scanner. Adding meta back here would defeat the whole point — the
|
||||
// per-row JSON decode this projection exists to skip.
|
||||
const edgeColsLight = `from_id, to_id, kind, file_path, line, confidence, confidence_label, origin, tier, cross_repo`
|
||||
|
||||
// AllEdgesLight implements graph.LightEdgeScanner: a kind-scoped edge scan that
|
||||
// never decodes the meta blob. An empty kinds list scans every edge; supplying
|
||||
// only empty-string kinds matches nothing (parity with the aggregators). The
|
||||
// edges_by_kind index serves the IN filter. Meta is left nil; only the promoted
|
||||
// fields (origin/tier/confidence/confidence_label/cross_repo/line) are hydrated.
|
||||
func (s *Store) AllEdgesLight(kinds ...graph.EdgeKind) []*graph.Edge {
|
||||
_, args := aggDedupeEdgeKinds(kinds)
|
||||
if len(args) == 0 {
|
||||
if len(kinds) > 0 {
|
||||
return nil // caller passed only empty kinds — nothing matches
|
||||
}
|
||||
return s.queryEdgesLightSQL(`SELECT ` + edgeColsLight + ` FROM edges ORDER BY id`)
|
||||
}
|
||||
q := `SELECT ` + edgeColsLight + ` FROM edges WHERE kind IN (` +
|
||||
inPlaceholders(len(args)) + `) ORDER BY id`
|
||||
return s.queryEdgesLightSQL(q, args...)
|
||||
}
|
||||
|
||||
// queryEdgesLightSQL is the meta-less sibling of queryEdgesSQL: it materialises
|
||||
// the rows into a slice and closes the cursor before returning (releasing the
|
||||
// single pooled connection), but scans through scanEdgeLight so the meta column
|
||||
// is never transferred or decoded. Returns nil on any query error, matching
|
||||
// queryEdgesSQL — a teardown-race read degrades to empty rather than panicking.
|
||||
func (s *Store) queryEdgesLightSQL(q string, args ...any) []*graph.Edge {
|
||||
rows, err := s.db.Query(q, args...)
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
defer rows.Close()
|
||||
var out []*graph.Edge
|
||||
for rows.Next() {
|
||||
e, err := scanEdgeLight(rows)
|
||||
if err != nil || e == nil {
|
||||
continue
|
||||
}
|
||||
out = append(out, e)
|
||||
}
|
||||
return out
|
||||
}
|
||||
@@ -0,0 +1,54 @@
|
||||
package store_sqlite_test
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"github.com/zzet/gortex/internal/graph"
|
||||
)
|
||||
|
||||
// TestAllEdgesLightIsMetaless pins the disk backend's graph.LightEdgeScanner:
|
||||
// the scan must skip the meta blob (Meta == nil) — the per-edge JSON decode the
|
||||
// warm-restart analysis passes exist to avoid — while every promoted field
|
||||
// still equals what the full AllEdges() scan returns.
|
||||
func TestAllEdgesLightIsMetaless(t *testing.T) {
|
||||
s := openTestStore(t)
|
||||
s.AddNode(&graph.Node{ID: "p/a.go::A", Kind: graph.KindFunction, Name: "A", FilePath: "p/a.go"})
|
||||
s.AddNode(&graph.Node{ID: "p/a.go::B", Kind: graph.KindFunction, Name: "B", FilePath: "p/a.go"})
|
||||
s.AddEdge(&graph.Edge{
|
||||
From: "p/a.go::A", To: "p/a.go::B", Kind: graph.EdgeCalls,
|
||||
FilePath: "p/a.go", Line: 11, Confidence: 0.75, ConfidenceLabel: "high",
|
||||
Origin: graph.OriginLSPResolved, Tier: "lsp", CrossRepo: true,
|
||||
Meta: map[string]any{"via": "direct", "blob_only": "x"},
|
||||
})
|
||||
// A second kind to prove the IN filter really scopes.
|
||||
s.AddEdge(&graph.Edge{From: "p/a.go::A", To: "p/a.go::B", Kind: graph.EdgeImports, FilePath: "p/a.go", Line: 1})
|
||||
|
||||
light := s.AllEdgesLight(graph.EdgeCalls)
|
||||
require.Len(t, light, 1, "kind filter must exclude the imports edge")
|
||||
e := light[0]
|
||||
assert.Nil(t, e.Meta, "AllEdgesLight must not decode the meta blob")
|
||||
|
||||
var full *graph.Edge
|
||||
for _, fe := range s.AllEdges() {
|
||||
if fe.Kind == graph.EdgeCalls {
|
||||
full = fe
|
||||
}
|
||||
}
|
||||
require.NotNil(t, full)
|
||||
assert.NotNil(t, full.Meta, "sanity: the full scan DOES decode meta")
|
||||
assert.Equal(t, full.From, e.From)
|
||||
assert.Equal(t, full.To, e.To)
|
||||
assert.Equal(t, full.Kind, e.Kind)
|
||||
assert.Equal(t, full.Line, e.Line)
|
||||
assert.Equal(t, full.Confidence, e.Confidence)
|
||||
assert.Equal(t, full.ConfidenceLabel, e.ConfidenceLabel)
|
||||
assert.Equal(t, full.Origin, e.Origin)
|
||||
assert.Equal(t, full.Tier, e.Tier)
|
||||
assert.Equal(t, full.CrossRepo, e.CrossRepo)
|
||||
|
||||
// Empty kinds means every edge.
|
||||
assert.Len(t, s.AllEdgesLight(), 2)
|
||||
}
|
||||
@@ -0,0 +1,204 @@
|
||||
package store_sqlite
|
||||
|
||||
import (
|
||||
"strings"
|
||||
|
||||
"github.com/zzet/gortex/internal/graph"
|
||||
)
|
||||
|
||||
// These methods were added to graph.Store after the sqlite backend was
|
||||
// first removed; they are restored here so *Store satisfies the current
|
||||
// interface. All reuse the chunked IN-list / raw-SQL helpers in store.go
|
||||
// (queryNodesSQL / queryEdgesSQL / lookupChunkSize / minInt). SQLite's
|
||||
// planner drives every one through the existing secondary indexes.
|
||||
|
||||
// lookupNodeCols is the canonical node column list (and scan order) for
|
||||
// every node-shaped SELECT in the package. It must stay in sync with
|
||||
// scanNode. The struct columns (start_column/end_column) sit with the line
|
||||
// range; the promoted meta columns (signature/visibility/doc/external/
|
||||
// return_type/is_async/is_static/is_abstract/is_exported/updated_at/
|
||||
// data_class/semantic_type/semantic_source) precede meta.
|
||||
const lookupNodeCols = `id, kind, name, qual_name, file_path, start_line, end_line, start_column, end_column, language, repo_prefix, workspace_id, project_id, signature, visibility, doc, external, return_type, is_async, is_static, is_abstract, is_exported, updated_at, data_class, semantic_type, semantic_source, meta`
|
||||
|
||||
// lookupNodeColsLight is lookupNodeCols without the trailing meta column —
|
||||
// the projection GetRepoNodesLight uses so a repo-scoped scan never
|
||||
// transfers or decodes a single blob. Derived, not hand-duplicated, so it
|
||||
// can never drift out of sync with lookupNodeCols / scanNode.
|
||||
var lookupNodeColsLight = strings.TrimSuffix(lookupNodeCols, ", meta")
|
||||
|
||||
const lookupEdgeCols = `from_id, to_id, kind, file_path, line, confidence, confidence_label, origin, tier, cross_repo, meta, resolve_terminal, resolve_terminal_reason`
|
||||
|
||||
// Compile-time assertion: *Store satisfies graph.NodeNameClassCounter.
|
||||
var _ graph.NodeNameClassCounter = (*Store)(nil)
|
||||
|
||||
// CountNodesByNameClass implements graph.NodeNameClassCounter: for each
|
||||
// distinct name, it tallies how many nodes.name matches are Real (is_stub =
|
||||
// 0 and kind IN definitionKinds) vs Stub (is_stub = 1), server-side via
|
||||
// nodes_by_name — one aggregate query per chunk instead of one
|
||||
// FindNodesByName round trip per name. A name absent from the returned map
|
||||
// has no matching node at all (Real == Stub == 0 either way).
|
||||
func (s *Store) CountNodesByNameClass(names []string, definitionKinds []graph.NodeKind) map[string]graph.NodeNameClassCount {
|
||||
_, kindArgs := aggDedupeNodeKinds(definitionKinds)
|
||||
if len(kindArgs) == 0 {
|
||||
return nil
|
||||
}
|
||||
uniq := dedupeNonEmpty(names)
|
||||
if len(uniq) == 0 {
|
||||
return nil
|
||||
}
|
||||
out := make(map[string]graph.NodeNameClassCount, len(uniq))
|
||||
kindPlaceholders := inPlaceholders(len(kindArgs))
|
||||
for i := 0; i < len(uniq); i += lookupChunkSize {
|
||||
end := minInt(i+lookupChunkSize, len(uniq))
|
||||
chunk := uniq[i:end]
|
||||
q := `SELECT name,
|
||||
SUM(CASE WHEN is_stub = 0 AND kind IN (` + kindPlaceholders + `) THEN 1 ELSE 0 END),
|
||||
SUM(CASE WHEN is_stub = 1 THEN 1 ELSE 0 END)
|
||||
FROM nodes
|
||||
WHERE name IN (` + inPlaceholders(len(chunk)) + `)
|
||||
GROUP BY name`
|
||||
args := make([]any, 0, len(kindArgs)+len(chunk))
|
||||
args = append(args, kindArgs...)
|
||||
args = append(args, toAnyArgs(chunk)...)
|
||||
rows, err := s.db.Query(q, args...)
|
||||
if err != nil {
|
||||
panicOnFatal(err)
|
||||
return out
|
||||
}
|
||||
for rows.Next() {
|
||||
var name string
|
||||
var c graph.NodeNameClassCount
|
||||
if err := rows.Scan(&name, &c.Real, &c.Stub); err != nil {
|
||||
_ = rows.Close()
|
||||
panicOnFatal(err)
|
||||
return out
|
||||
}
|
||||
out[name] = c
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
_ = rows.Close()
|
||||
panicOnFatal(err)
|
||||
return out
|
||||
}
|
||||
_ = rows.Close()
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// FindNodesByNameContaining returns nodes whose Name contains substr,
|
||||
// case-insensitively (SQLite's LIKE is ASCII case-insensitive). An empty
|
||||
// substring matches nothing (parity with the in-memory store); a limit > 0
|
||||
// caps the result set. The leading-wildcard LIKE is a deliberate full scan —
|
||||
// no index accelerates an unanchored substring — matching the in-memory
|
||||
// strings.Contains fallback. % and _ in substr are escaped so they match
|
||||
// literally.
|
||||
func (s *Store) FindNodesByNameContaining(substr string, limit int) []*graph.Node {
|
||||
if substr == "" {
|
||||
return nil
|
||||
}
|
||||
pattern := "%" + escapeLikePattern(substr) + "%"
|
||||
q := `SELECT ` + lookupNodeCols + ` FROM nodes WHERE name LIKE ? ESCAPE '\' ORDER BY id`
|
||||
if limit > 0 {
|
||||
return s.queryNodesSQL(q+` LIMIT ?`, pattern, limit)
|
||||
}
|
||||
return s.queryNodesSQL(q, pattern)
|
||||
}
|
||||
|
||||
// GetNodesByQualNames returns a map qualName→*Node (first match per
|
||||
// qual_name) for the batch — the qual-name twin of FindNodesByNames, used to
|
||||
// pre-warm import resolution. Driven by the unique nodes_by_qual index.
|
||||
func (s *Store) GetNodesByQualNames(qualNames []string) map[string]*graph.Node {
|
||||
uniq := dedupeNonEmpty(qualNames)
|
||||
if len(uniq) == 0 {
|
||||
return nil
|
||||
}
|
||||
out := make(map[string]*graph.Node, len(uniq))
|
||||
for i := 0; i < len(uniq); i += lookupChunkSize {
|
||||
end := minInt(i+lookupChunkSize, len(uniq))
|
||||
chunk := uniq[i:end]
|
||||
q := `SELECT ` + lookupNodeCols + ` FROM nodes WHERE qual_name IN (` + inPlaceholders(len(chunk)) + `)`
|
||||
for _, n := range s.queryNodesSQL(q, toAnyArgs(chunk)...) {
|
||||
if n == nil {
|
||||
continue
|
||||
}
|
||||
if _, ok := out[n.QualName]; !ok {
|
||||
out[n.QualName] = n
|
||||
}
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// GetOutEdgesByNodeIDs batches per-node out-edge fan-out into one query per
|
||||
// chunk. Missing IDs are simply absent from the returned map.
|
||||
func (s *Store) GetOutEdgesByNodeIDs(ids []string) map[string][]*graph.Edge {
|
||||
return s.edgesByNodeIDs(ids, "from_id", func(e *graph.Edge) string { return e.From })
|
||||
}
|
||||
|
||||
// GetInEdgesByNodeIDs is the incoming-edge twin of GetOutEdgesByNodeIDs.
|
||||
func (s *Store) GetInEdgesByNodeIDs(ids []string) map[string][]*graph.Edge {
|
||||
return s.edgesByNodeIDs(ids, "to_id", func(e *graph.Edge) string { return e.To })
|
||||
}
|
||||
|
||||
// edgesByNodeIDs runs the chunked IN-list edge fetch keyed on the given
|
||||
// column (from_id or to_id), grouping results by the supplied key extractor.
|
||||
func (s *Store) edgesByNodeIDs(ids []string, col string, key func(*graph.Edge) string) map[string][]*graph.Edge {
|
||||
uniq := dedupeNonEmpty(ids)
|
||||
if len(uniq) == 0 {
|
||||
return nil
|
||||
}
|
||||
out := make(map[string][]*graph.Edge, len(uniq))
|
||||
for i := 0; i < len(uniq); i += lookupChunkSize {
|
||||
end := minInt(i+lookupChunkSize, len(uniq))
|
||||
chunk := uniq[i:end]
|
||||
q := `SELECT ` + lookupEdgeCols + ` FROM edges WHERE ` + col + ` IN (` + inPlaceholders(len(chunk)) + `)`
|
||||
for _, e := range s.queryEdgesSQL(q, toAnyArgs(chunk)...) {
|
||||
if e == nil {
|
||||
continue
|
||||
}
|
||||
k := key(e)
|
||||
out[k] = append(out[k], e)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// dedupeNonEmpty drops empties and duplicates, preserving first-seen order.
|
||||
func dedupeNonEmpty(in []string) []string {
|
||||
seen := make(map[string]struct{}, len(in))
|
||||
out := make([]string, 0, len(in))
|
||||
for _, v := range in {
|
||||
if v == "" {
|
||||
continue
|
||||
}
|
||||
if _, ok := seen[v]; ok {
|
||||
continue
|
||||
}
|
||||
seen[v] = struct{}{}
|
||||
out = append(out, v)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// inPlaceholders returns "?,?,?" for n bound parameters.
|
||||
func inPlaceholders(n int) string {
|
||||
if n <= 0 {
|
||||
return ""
|
||||
}
|
||||
return strings.Repeat(",?", n)[1:]
|
||||
}
|
||||
|
||||
// toAnyArgs widens a string slice for variadic Query/Exec args.
|
||||
func toAnyArgs(ss []string) []any {
|
||||
args := make([]any, len(ss))
|
||||
for i, v := range ss {
|
||||
args[i] = v
|
||||
}
|
||||
return args
|
||||
}
|
||||
|
||||
// escapeLikePattern escapes the LIKE metacharacters so the substring matches
|
||||
// literally under `... LIKE ? ESCAPE '\'`.
|
||||
func escapeLikePattern(s string) string {
|
||||
return strings.NewReplacer(`\`, `\\`, `%`, `\%`, `_`, `\_`).Replace(s)
|
||||
}
|
||||
@@ -0,0 +1,54 @@
|
||||
package store_sqlite_test
|
||||
|
||||
import (
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"github.com/zzet/gortex/internal/graph"
|
||||
"github.com/zzet/gortex/internal/graph/store_sqlite"
|
||||
)
|
||||
|
||||
// TestAllRepoMemoryEstimates_Memoized verifies the short-TTL memoisation:
|
||||
// within the TTL a second call serves the cached estimate instead of
|
||||
// re-running the COUNT … GROUP BY scan, so the status path can poll cheaply
|
||||
// even while enrichment is writing to the same store.
|
||||
func TestAllRepoMemoryEstimates_Memoized(t *testing.T) {
|
||||
path := filepath.Join(t.TempDir(), "g.sqlite")
|
||||
s, err := store_sqlite.Open(path)
|
||||
require.NoError(t, err)
|
||||
t.Cleanup(func() { _ = s.Close() })
|
||||
|
||||
totalNodes := func(m map[string]graph.RepoMemoryEstimate) int {
|
||||
n := 0
|
||||
for _, e := range m {
|
||||
n += e.NodeCount
|
||||
}
|
||||
return n
|
||||
}
|
||||
|
||||
s.AddBatch([]*graph.Node{
|
||||
{ID: "a.go::Foo", Kind: graph.KindFunction, Name: "Foo", FilePath: "a.go", RepoPrefix: "r1"},
|
||||
{ID: "a.go::Bar", Kind: graph.KindFunction, Name: "Bar", FilePath: "a.go", RepoPrefix: "r1"},
|
||||
}, nil)
|
||||
|
||||
first := s.AllRepoMemoryEstimates()
|
||||
require.Equal(t, 2, totalNodes(first))
|
||||
|
||||
// Add another node; within the memoisation TTL the estimate is served
|
||||
// from cache, so the freshly-added node is intentionally not yet
|
||||
// reflected — proof the COUNT scan was skipped.
|
||||
s.AddBatch([]*graph.Node{
|
||||
{ID: "b.go::Baz", Kind: graph.KindFunction, Name: "Baz", FilePath: "b.go", RepoPrefix: "r1"},
|
||||
}, nil)
|
||||
cached := s.AllRepoMemoryEstimates()
|
||||
require.Equal(t, 2, totalNodes(cached), "within the TTL the result should be the memoised value, not a fresh COUNT")
|
||||
|
||||
// The returned map is a copy: mutating it must not corrupt the cache.
|
||||
for k := range cached {
|
||||
delete(cached, k)
|
||||
}
|
||||
again := s.AllRepoMemoryEstimates()
|
||||
require.Equal(t, 2, totalNodes(again), "the cache must hand back an independent copy")
|
||||
}
|
||||
@@ -0,0 +1,243 @@
|
||||
package store_sqlite
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
|
||||
"github.com/zzet/gortex/internal/graph"
|
||||
)
|
||||
|
||||
// Compile-time assertions that the SQLite Store satisfies the optional
|
||||
// per-file mtime persistence capabilities. Lifting this state into the
|
||||
// same backend the graph lives in means warm restarts read it through
|
||||
// one persistence surface instead of a second gob snapshot.
|
||||
var (
|
||||
_ graph.FileMtimeWriter = (*Store)(nil)
|
||||
_ graph.FileMtimeReader = (*Store)(nil)
|
||||
_ graph.FileMtimeReplacer = (*Store)(nil)
|
||||
_ graph.FileMtimeDeleter = (*Store)(nil)
|
||||
)
|
||||
|
||||
// mtimeChunk bounds how many (repo_prefix, file_path, mtime_ns) tuples
|
||||
// ride in a single multi-row INSERT. SQLite's default compiled-in host
|
||||
// parameter limit is 999; at 3 params per row that caps a statement at
|
||||
// 333 rows, so 300 leaves headroom.
|
||||
const mtimeChunk = 300
|
||||
|
||||
// SetFileMtime records one file's modification time (nanoseconds since
|
||||
// the epoch) for a repo prefix, replacing any prior value. It is a
|
||||
// convenience single-row form of BulkSetFileMtimes.
|
||||
func (s *Store) SetFileMtime(repoPrefix, filePath string, mtimeNs int64) error {
|
||||
s.writeMu.Lock()
|
||||
defer s.writeMu.Unlock()
|
||||
_, err := s.db.Exec(
|
||||
`INSERT OR REPLACE INTO file_mtimes (repo_prefix, file_path, mtime_ns) VALUES (?, ?, ?)`,
|
||||
repoPrefix, filePath, mtimeNs,
|
||||
)
|
||||
return err
|
||||
}
|
||||
|
||||
// BulkSetFileMtimes persists every (filePath -> mtimeNs) entry for one
|
||||
// repo prefix in a single transaction, chunked so no statement exceeds
|
||||
// SQLite's host-parameter limit. Idempotent on (repoPrefix, filePath):
|
||||
// re-running with overlapping keys replaces in place. Empty input is a
|
||||
// no-op.
|
||||
func (s *Store) BulkSetFileMtimes(repoPrefix string, mtimes map[string]int64) error {
|
||||
if len(mtimes) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
s.writeMu.Lock()
|
||||
defer s.writeMu.Unlock()
|
||||
|
||||
tx, err := s.db.Begin()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer tx.Rollback() //nolint:errcheck // rollback after Commit is a no-op
|
||||
|
||||
if err := insertMtimesTx(tx, repoPrefix, mtimes); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return tx.Commit()
|
||||
}
|
||||
|
||||
// ReplaceFileMtimes persists the AUTHORITATIVE full mtime set for one repo
|
||||
// prefix: every prior row for the prefix is dropped and the supplied set is
|
||||
// written, all in one transaction. The full-index persist path uses this so
|
||||
// files deleted since the last index are pruned — BulkSetFileMtimes (upsert)
|
||||
// would leave their rows behind, and warm-restart reconcile would then
|
||||
// detect them as phantom deletions on every restart, forcing a full
|
||||
// re-track that never converges.
|
||||
//
|
||||
// Empty input is a deliberate no-op: it never wipes a repo's mtimes from an
|
||||
// empty snapshot (the indexer guards the call with len(snapshot) > 0).
|
||||
func (s *Store) ReplaceFileMtimes(repoPrefix string, mtimes map[string]int64) error {
|
||||
if len(mtimes) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
s.writeMu.Lock()
|
||||
defer s.writeMu.Unlock()
|
||||
|
||||
tx, err := s.db.Begin()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer tx.Rollback() //nolint:errcheck // rollback after Commit is a no-op
|
||||
|
||||
if _, err := tx.Exec(`DELETE FROM file_mtimes WHERE repo_prefix = ?`, repoPrefix); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := insertMtimesTx(tx, repoPrefix, mtimes); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return tx.Commit()
|
||||
}
|
||||
|
||||
// DeleteFileMtimes drops the rows for a set of repo-relative file paths
|
||||
// under one repo prefix — the incremental-reindex sibling of
|
||||
// ReplaceFileMtimes. The watcher / incremental path calls it when a file is
|
||||
// deleted so the persisted set stays in step with the live graph and the
|
||||
// next warm restart does not see the path as a phantom deletion. Empty
|
||||
// input is a no-op.
|
||||
func (s *Store) DeleteFileMtimes(repoPrefix string, paths []string) error {
|
||||
if len(paths) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
s.writeMu.Lock()
|
||||
defer s.writeMu.Unlock()
|
||||
|
||||
tx, err := s.db.Begin()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer tx.Rollback() //nolint:errcheck // rollback after Commit is a no-op
|
||||
|
||||
// Chunk so the IN-list never exceeds SQLite's host-parameter limit:
|
||||
// one leading repo_prefix arg + up to mtimeChunk path args per stmt.
|
||||
for start := 0; start < len(paths); start += mtimeChunk {
|
||||
end := min(start+mtimeChunk, len(paths))
|
||||
batch := paths[start:end]
|
||||
|
||||
args := make([]any, 0, len(batch)+1)
|
||||
args = append(args, repoPrefix)
|
||||
stmt := make([]byte, 0, 64+len(batch)*2)
|
||||
stmt = append(stmt, "DELETE FROM file_mtimes WHERE repo_prefix = ? AND file_path IN ("...)
|
||||
for i := range batch {
|
||||
if i > 0 {
|
||||
stmt = append(stmt, ',')
|
||||
}
|
||||
stmt = append(stmt, '?')
|
||||
args = append(args, batch[i])
|
||||
}
|
||||
stmt = append(stmt, ')')
|
||||
if _, err := tx.Exec(string(stmt), args...); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
return tx.Commit()
|
||||
}
|
||||
|
||||
// insertMtimesTx writes every (path -> ns) entry for repoPrefix into the
|
||||
// given transaction with chunked multi-row INSERT OR REPLACE statements,
|
||||
// each kept under SQLite's host-parameter limit. The caller owns the tx
|
||||
// lifecycle (Begin/Commit/Rollback) and the write lock.
|
||||
func insertMtimesTx(tx *sql.Tx, repoPrefix string, mtimes map[string]int64) error {
|
||||
// Stable ordering is not required for correctness, but iterating the
|
||||
// map directly is fine — we only chunk by count.
|
||||
type kv struct {
|
||||
path string
|
||||
ns int64
|
||||
}
|
||||
pending := make([]kv, 0, len(mtimes))
|
||||
for p, ns := range mtimes {
|
||||
pending = append(pending, kv{path: p, ns: ns})
|
||||
}
|
||||
|
||||
for start := 0; start < len(pending); start += mtimeChunk {
|
||||
end := min(start+mtimeChunk, len(pending))
|
||||
batch := pending[start:end]
|
||||
|
||||
// Build a multi-row INSERT OR REPLACE: (?, ?, ?), (?, ?, ?), ...
|
||||
args := make([]any, 0, len(batch)*3)
|
||||
stmt := make([]byte, 0, 64+len(batch)*16)
|
||||
stmt = append(stmt, "INSERT OR REPLACE INTO file_mtimes (repo_prefix, file_path, mtime_ns) VALUES "...)
|
||||
for i, e := range batch {
|
||||
if i > 0 {
|
||||
stmt = append(stmt, ',')
|
||||
}
|
||||
stmt = append(stmt, "(?, ?, ?)"...)
|
||||
args = append(args, repoPrefix, e.path, e.ns)
|
||||
}
|
||||
if _, err := tx.Exec(string(stmt), args...); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// LoadFileMtimes returns the recorded mtimes for one repo prefix as a
|
||||
// fresh map. Returns nil when there is no data for the prefix (the
|
||||
// "no recorded state" signal warmup expects).
|
||||
func (s *Store) LoadFileMtimes(repoPrefix string) map[string]int64 {
|
||||
rows, err := s.db.Query(
|
||||
`SELECT file_path, mtime_ns FROM file_mtimes WHERE repo_prefix = ?`,
|
||||
repoPrefix,
|
||||
)
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
var out map[string]int64
|
||||
for rows.Next() {
|
||||
var path string
|
||||
var ns int64
|
||||
if err := rows.Scan(&path, &ns); err != nil {
|
||||
return nil
|
||||
}
|
||||
if out == nil {
|
||||
out = make(map[string]int64)
|
||||
}
|
||||
out[path] = ns
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return nil
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// FileMtimes is a fallible read form of LoadFileMtimes. It always
|
||||
// returns a non-nil (possibly empty) map for a known/unknown prefix and
|
||||
// surfaces any query error. The interface method LoadFileMtimes is the
|
||||
// daemon's entry point; this variant exists for callers (and tests)
|
||||
// that want the error and an always-materialised map.
|
||||
func (s *Store) FileMtimes(repoPrefix string) (map[string]int64, error) {
|
||||
rows, err := s.db.Query(
|
||||
`SELECT file_path, mtime_ns FROM file_mtimes WHERE repo_prefix = ?`,
|
||||
repoPrefix,
|
||||
)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
out := make(map[string]int64)
|
||||
for rows.Next() {
|
||||
var path string
|
||||
var ns int64
|
||||
if err := rows.Scan(&path, &ns); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out[path] = ns
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
@@ -0,0 +1,112 @@
|
||||
package store_sqlite_test
|
||||
|
||||
import (
|
||||
"reflect"
|
||||
"testing"
|
||||
|
||||
"github.com/zzet/gortex/internal/graph"
|
||||
)
|
||||
|
||||
// TestReplaceFileMtimesPrunesDeleted is the regression for the warm-restart
|
||||
// "nothing changed but full re-track" bug: the full-index persist path must
|
||||
// REPLACE a repo's mtime set, not union into it. An upsert-only persist
|
||||
// leaves rows for files deleted since the last index behind, and warm-restart
|
||||
// reconcile then detects them as phantom deletions on every restart — forcing
|
||||
// a full re-track that never converges.
|
||||
func TestReplaceFileMtimesPrunesDeleted(t *testing.T) {
|
||||
s := openTestStore(t)
|
||||
|
||||
// Assert the store advertises the capability the indexer probes for.
|
||||
var _ graph.FileMtimeReplacer = s
|
||||
var _ graph.FileMtimeDeleter = s
|
||||
|
||||
// First index: three files persisted.
|
||||
require := func(err error, what string) {
|
||||
t.Helper()
|
||||
if err != nil {
|
||||
t.Fatalf("%s: %v", what, err)
|
||||
}
|
||||
}
|
||||
require(s.BulkSetFileMtimes("repoA", map[string]int64{
|
||||
"a/one.go": 100,
|
||||
"a/two.go": 200,
|
||||
"a/three.go": 300,
|
||||
}), "seed BulkSetFileMtimes")
|
||||
|
||||
// A different repo whose rows must never be touched by repoA writes.
|
||||
require(s.BulkSetFileMtimes("repoB", map[string]int64{"b/x.go": 999}), "seed repoB")
|
||||
|
||||
// Second index: two.go was deleted on disk, four.go is new, three.go
|
||||
// changed. The authoritative snapshot is {one, three', four}.
|
||||
require(s.ReplaceFileMtimes("repoA", map[string]int64{
|
||||
"a/one.go": 100,
|
||||
"a/three.go": 350, // changed
|
||||
"a/four.go": 400, // new
|
||||
}), "ReplaceFileMtimes")
|
||||
|
||||
want := map[string]int64{
|
||||
"a/one.go": 100,
|
||||
"a/three.go": 350,
|
||||
"a/four.go": 400,
|
||||
}
|
||||
got := s.LoadFileMtimes("repoA")
|
||||
if !reflect.DeepEqual(got, want) {
|
||||
t.Fatalf("after ReplaceFileMtimes = %v, want %v (a/two.go must be pruned)", got, want)
|
||||
}
|
||||
if _, stillThere := got["a/two.go"]; stillThere {
|
||||
t.Fatal("a/two.go was deleted on disk but its mtime row survived the replace — phantom deletion bug")
|
||||
}
|
||||
|
||||
// Repo isolation.
|
||||
if b := s.LoadFileMtimes("repoB"); !reflect.DeepEqual(b, map[string]int64{"b/x.go": 999}) {
|
||||
t.Fatalf("repoB rows disturbed by repoA replace: %v", b)
|
||||
}
|
||||
|
||||
// Empty input is a deliberate no-op: it must NEVER wipe a repo's set.
|
||||
require(s.ReplaceFileMtimes("repoA", nil), "ReplaceFileMtimes(nil)")
|
||||
if got := s.LoadFileMtimes("repoA"); !reflect.DeepEqual(got, want) {
|
||||
t.Fatalf("ReplaceFileMtimes(nil) wiped the repo: %v", got)
|
||||
}
|
||||
}
|
||||
|
||||
// TestDeleteFileMtimes covers the incremental-reindex sibling: the watcher /
|
||||
// incremental path drops just the deleted paths so the persisted set stays in
|
||||
// step with the live graph without a full replace.
|
||||
func TestDeleteFileMtimes(t *testing.T) {
|
||||
s := openTestStore(t)
|
||||
|
||||
if err := s.BulkSetFileMtimes("repoA", map[string]int64{
|
||||
"a/one.go": 100,
|
||||
"a/two.go": 200,
|
||||
"a/three.go": 300,
|
||||
"a/four.go": 400,
|
||||
}); err != nil {
|
||||
t.Fatalf("seed: %v", err)
|
||||
}
|
||||
if err := s.BulkSetFileMtimes("repoB", map[string]int64{"b/keep.go": 7}); err != nil {
|
||||
t.Fatalf("seed repoB: %v", err)
|
||||
}
|
||||
|
||||
// Delete two existing paths and one that was never recorded (harmless).
|
||||
if err := s.DeleteFileMtimes("repoA", []string{"a/two.go", "a/four.go", "a/never.go"}); err != nil {
|
||||
t.Fatalf("DeleteFileMtimes: %v", err)
|
||||
}
|
||||
|
||||
want := map[string]int64{"a/one.go": 100, "a/three.go": 300}
|
||||
if got := s.LoadFileMtimes("repoA"); !reflect.DeepEqual(got, want) {
|
||||
t.Fatalf("after delete = %v, want %v", got, want)
|
||||
}
|
||||
|
||||
// Repo isolation: same-named delete on repoA must not touch repoB.
|
||||
if b := s.LoadFileMtimes("repoB"); !reflect.DeepEqual(b, map[string]int64{"b/keep.go": 7}) {
|
||||
t.Fatalf("repoB disturbed: %v", b)
|
||||
}
|
||||
|
||||
// Empty input is a no-op.
|
||||
if err := s.DeleteFileMtimes("repoA", nil); err != nil {
|
||||
t.Fatalf("DeleteFileMtimes(nil): %v", err)
|
||||
}
|
||||
if got := s.LoadFileMtimes("repoA"); !reflect.DeepEqual(got, want) {
|
||||
t.Fatalf("DeleteFileMtimes(nil) changed the set: %v", got)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,310 @@
|
||||
package store_sqlite
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"fmt"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// This file adds the repo-scoped store hygiene the base eviction path lacks.
|
||||
// EvictRepo (store.go) deletes ONLY nodes+edges, but a repo owns fifteen
|
||||
// other repo_prefix-keyed sidecar tables (file_mtimes, repo_index_state,
|
||||
// enrichment_state, clone_shingles, constant_values, files, ref_facts,
|
||||
// vectors, churn/coverage/release/blame_enrichment, symbol_fts,
|
||||
// symbol_fts_rowid, content_fts — see schema.go). Untracking a repo through
|
||||
// EvictRepo leaks every one of them, so a long-lived store accumulates
|
||||
// sidecar rows for repos removed from config long ago. PurgeRepo clears a
|
||||
// repo whole; OrphanRepoPrefixes finds prefixes that outlived their config
|
||||
// entry; RekeyRepoPrefix moves a lone repo's residue when it earns a prefix.
|
||||
//
|
||||
// INVARIANT — the empty repo_prefix is NEVER purged. In a live multi-repo
|
||||
// store repo_prefix='' identifies SYNTHETIC GLOBAL EXTERNALS (external_call
|
||||
// ::dep:* / builtin:: / module:: nodes shared across every repo) and, in a
|
||||
// single-repo store, the sole repo's live data. Deleting '' rows would strip
|
||||
// the shared externals out from under every repo, or wipe the lone repo.
|
||||
// Every method here refuses or excludes ''.
|
||||
|
||||
// purgeSidecarTables are the repo_prefix-keyed sidecar tables PurgeRepo
|
||||
// clears for a prefix, alongside nodes+edges. Each carries a repo_prefix
|
||||
// column a plain `DELETE ... WHERE repo_prefix = ?` keys on. The two FTS5
|
||||
// vtables (symbol_fts, content_fts) carry repo_prefix UNINDEXED, so their
|
||||
// delete is a full scan — acceptable for a purge (a rare, whole-repo op),
|
||||
// unlike the per-edit hot path. `vectors` is deliberately absent: it has NO
|
||||
// repo_prefix column (keyed by node_id alone), so PurgeRepo deletes its rows
|
||||
// by node-id membership instead (see deleteByIDColumnsTx below).
|
||||
var purgeSidecarTables = []string{
|
||||
"file_mtimes",
|
||||
"repo_index_state",
|
||||
"enrichment_state",
|
||||
"clone_shingles",
|
||||
"constant_values",
|
||||
"files",
|
||||
"ref_facts",
|
||||
"churn_enrichment",
|
||||
"coverage_enrichment",
|
||||
"release_enrichment",
|
||||
"blame_enrichment",
|
||||
"symbol_fts",
|
||||
"symbol_fts_rowid",
|
||||
"content_fts",
|
||||
}
|
||||
|
||||
// PurgeRepo deletes EVERY row a repo owns — nodes, edges, and all fifteen
|
||||
// repo_prefix-keyed sidecar tables (purgeSidecarTables + vectors) — in one
|
||||
// transaction. It is the complete form of EvictRepo (which drops only
|
||||
// nodes+edges), wired into UntrackRepo so removing a repo from config leaves
|
||||
// no residue. Refuses prefix=="" (shared global externals / solo-mode live
|
||||
// data — see the file-level INVARIANT).
|
||||
func (s *Store) PurgeRepo(prefix string) error {
|
||||
if prefix == "" {
|
||||
return fmt.Errorf("store_sqlite: PurgeRepo refuses empty repo prefix (would delete shared global externals / solo-repo data)")
|
||||
}
|
||||
|
||||
s.writeMu.Lock()
|
||||
defer s.writeMu.Unlock()
|
||||
|
||||
tx, err := s.db.Begin()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer tx.Rollback() //nolint:errcheck // rollback after Commit is a no-op
|
||||
|
||||
// Collect this repo's node IDs first: edges and vectors are keyed off
|
||||
// them (edges by from_id/to_id, vectors by node_id — neither carries a
|
||||
// repo_prefix column). Edge deletion semantics mirror evictByScopeLocked
|
||||
// (store.go): delete every edge touching one of these nodes, then the
|
||||
// nodes themselves.
|
||||
ids, err := repoNodeIDsTx(tx, prefix)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := deleteByIDColumnsTx(tx, "edges", []string{"from_id", "to_id"}, ids); err != nil {
|
||||
return fmt.Errorf("store_sqlite: PurgeRepo edges: %w", err)
|
||||
}
|
||||
if err := deleteByIDColumnsTx(tx, "vectors", []string{"node_id"}, ids); err != nil {
|
||||
return fmt.Errorf("store_sqlite: PurgeRepo vectors: %w", err)
|
||||
}
|
||||
|
||||
for _, table := range purgeSidecarTables {
|
||||
if _, err := tx.Exec(`DELETE FROM `+table+` WHERE repo_prefix = ?`, prefix); err != nil {
|
||||
return fmt.Errorf("store_sqlite: PurgeRepo %s: %w", table, err)
|
||||
}
|
||||
}
|
||||
|
||||
if _, err := tx.Exec(`DELETE FROM nodes WHERE repo_prefix = ?`, prefix); err != nil {
|
||||
return fmt.Errorf("store_sqlite: PurgeRepo nodes: %w", err)
|
||||
}
|
||||
|
||||
return tx.Commit()
|
||||
}
|
||||
|
||||
// orphanScanTables are the tables OrphanRepoPrefixes unions DISTINCT
|
||||
// repo_prefix over. These five span the residue space: nodes (the primary
|
||||
// keyed store), file_mtimes + repo_index_state (the warm-restart provenance
|
||||
// that lingers when nodes are gone but sidecars survive — the exact shape a
|
||||
// leaked untrack leaves), enrichment_state (per-provider provenance), and
|
||||
// files (per-file metadata). A prefix whose nodes are gone but whose
|
||||
// sidecars remain is invisible to a nodes-only scan, which is why the
|
||||
// sidecar tables are unioned in; scanning still more tables would only
|
||||
// rediscover the same prefixes at higher cost.
|
||||
var orphanScanTables = []string{
|
||||
"nodes",
|
||||
"file_mtimes",
|
||||
"repo_index_state",
|
||||
"enrichment_state",
|
||||
"files",
|
||||
}
|
||||
|
||||
// OrphanRepoPrefixes returns every repo_prefix present in the store but
|
||||
// absent from known — repos whose rows outlived their config entry (an
|
||||
// untrack that predated PurgeRepo, or a repo dropped straight from config
|
||||
// with no untrack at all). The empty prefix is NEVER reported (shared global
|
||||
// externals / solo data). known is matched case-insensitively as a safety
|
||||
// net, so a case-only spelling drift on a case-insensitive filesystem can
|
||||
// never flag a still-tracked repo as an orphan (the #270 failure mode).
|
||||
// Startup warmup feeds the result to PurgeRepo.
|
||||
func (s *Store) OrphanRepoPrefixes(known []string) []string {
|
||||
knownFold := make(map[string]struct{}, len(known))
|
||||
for _, k := range known {
|
||||
if k == "" {
|
||||
continue
|
||||
}
|
||||
knownFold[strings.ToLower(k)] = struct{}{}
|
||||
}
|
||||
|
||||
seen := make(map[string]struct{})
|
||||
var out []string
|
||||
for _, table := range orphanScanTables {
|
||||
// WHERE repo_prefix <> '' both excludes the protected empty prefix
|
||||
// and lets the nodes scan ride the partial nodes_by_repo index
|
||||
// (defined WHERE repo_prefix <> ''). A table absent on an older
|
||||
// schema simply contributes nothing.
|
||||
rows, err := s.db.Query(`SELECT DISTINCT repo_prefix FROM ` + table + ` WHERE repo_prefix <> ''`)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
for rows.Next() {
|
||||
var p string
|
||||
if err := rows.Scan(&p); err != nil {
|
||||
break
|
||||
}
|
||||
if p == "" {
|
||||
continue
|
||||
}
|
||||
if _, ok := knownFold[strings.ToLower(p)]; ok {
|
||||
continue
|
||||
}
|
||||
if _, dup := seen[p]; dup {
|
||||
continue
|
||||
}
|
||||
seen[p] = struct{}{}
|
||||
out = append(out, p)
|
||||
}
|
||||
_ = rows.Close()
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// rekeyMoveTables are the sidecar tables RekeyRepoPrefix relabels from old
|
||||
// to new. Every one is keyed by repo_prefix (+ file_path or provider), NOT
|
||||
// by node_id, so its row content survives a node-id change: file_mtimes /
|
||||
// files by (repo_prefix, file_path); repo_index_state / enrichment_state by
|
||||
// repo_prefix (+ provider). At a solo->multi migration every '' row in these
|
||||
// belongs to the one migrating repo — global externals live in the NODES
|
||||
// table and hold NO rows here — so moving them wholesale is safe. UPDATE OR
|
||||
// REPLACE folds any row the re-mint re-index already wrote under new
|
||||
// (identical content: same files, same mtimes, same commit) instead of
|
||||
// tripping the primary-key conflict a plain UPDATE would.
|
||||
var rekeyMoveTables = []string{
|
||||
"file_mtimes",
|
||||
"files",
|
||||
"repo_index_state",
|
||||
"enrichment_state",
|
||||
}
|
||||
|
||||
// rekeyDropTables are the sidecar tables RekeyRepoPrefix DROPS (rather than
|
||||
// relabels) for old. Every one is keyed by node_id, and the solo->multi
|
||||
// re-mint changes every node id (unprefixed `pkg::X` -> `<new>::pkg::X`), so
|
||||
// these old-id rows are already dangling against the evicted unprefixed
|
||||
// nodes. Relabeling their repo_prefix would just move dangling rows under
|
||||
// new — and let, e.g., the clone reseed load a shingle set for a node that
|
||||
// no longer exists. Dropping them is correct: the re-mint re-index rewrites
|
||||
// the index-time sidecars (constant_values, ref_facts, clone_shingles) under
|
||||
// the new node ids, and the enrichment sidecars (churn/coverage/release/
|
||||
// blame) must re-run for the new ids regardless. The FTS vtables sit here
|
||||
// too — their rows carry the old node ids, and UPDATE over an FTS5 UNINDEXED
|
||||
// column is awkward, so delete-then-reindex is the clean path.
|
||||
var rekeyDropTables = []string{
|
||||
"clone_shingles",
|
||||
"constant_values",
|
||||
"ref_facts",
|
||||
"churn_enrichment",
|
||||
"coverage_enrichment",
|
||||
"release_enrichment",
|
||||
"blame_enrichment",
|
||||
"symbol_fts",
|
||||
"symbol_fts_rowid",
|
||||
"content_fts",
|
||||
}
|
||||
|
||||
// RekeyRepoPrefix moves a repo's sidecar residue from old to new the moment a
|
||||
// solo (unprefixed) repo earns a real prefix because a second repo joined —
|
||||
// the migrateLoneUnprefixedRepoCtx path. The prefix/path-keyed provenance
|
||||
// tables (rekeyMoveTables) are relabeled so warm restart finds the repo's
|
||||
// mtimes + freshness under new instead of full-re-tracking it; the
|
||||
// node_id-keyed tables (rekeyDropTables) are dropped because the re-mint
|
||||
// changed every node id out from under them (see the two table lists for the
|
||||
// per-table rationale).
|
||||
//
|
||||
// Refuses new=="" (cannot rekey INTO the protected empty prefix). old=="" IS
|
||||
// allowed — that is the whole point, since solo repos index unprefixed — and
|
||||
// is safe here because this method touches SIDECAR tables ONLY; the synthetic
|
||||
// global externals that also carry repo_prefix='' live in the NODES table,
|
||||
// which RekeyRepoPrefix never writes.
|
||||
func (s *Store) RekeyRepoPrefix(oldPrefix, newPrefix string) error {
|
||||
if newPrefix == "" {
|
||||
return fmt.Errorf("store_sqlite: RekeyRepoPrefix refuses empty destination prefix")
|
||||
}
|
||||
if oldPrefix == newPrefix {
|
||||
return nil
|
||||
}
|
||||
|
||||
s.writeMu.Lock()
|
||||
defer s.writeMu.Unlock()
|
||||
|
||||
tx, err := s.db.Begin()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer tx.Rollback() //nolint:errcheck // rollback after Commit is a no-op
|
||||
|
||||
for _, table := range rekeyMoveTables {
|
||||
if _, err := tx.Exec(`UPDATE OR REPLACE `+table+` SET repo_prefix = ? WHERE repo_prefix = ?`, newPrefix, oldPrefix); err != nil {
|
||||
return fmt.Errorf("store_sqlite: RekeyRepoPrefix move %s: %w", table, err)
|
||||
}
|
||||
}
|
||||
for _, table := range rekeyDropTables {
|
||||
if _, err := tx.Exec(`DELETE FROM `+table+` WHERE repo_prefix = ?`, oldPrefix); err != nil {
|
||||
return fmt.Errorf("store_sqlite: RekeyRepoPrefix drop %s: %w", table, err)
|
||||
}
|
||||
}
|
||||
// vectors is intentionally omitted: it has NO repo_prefix column (keyed
|
||||
// by node_id alone), so it cannot be addressed here by prefix. Any ''
|
||||
// embeddings are node_id-keyed against now-evicted unprefixed ids —
|
||||
// dangling, and absent in the common case (embeddings are opt-in). They
|
||||
// are left to a node-membership vector GC rather than guessed at here.
|
||||
|
||||
return tx.Commit()
|
||||
}
|
||||
|
||||
// repoNodeIDsTx returns every node id in repoPrefix, read inside tx. The
|
||||
// caller holds writeMu. Rows are fully drained + closed before the caller
|
||||
// issues writes on the same tx — SQLite forbids an open read cursor while
|
||||
// writing on the same connection.
|
||||
func repoNodeIDsTx(tx *sql.Tx, repoPrefix string) ([]string, error) {
|
||||
rows, err := tx.Query(`SELECT id FROM nodes WHERE repo_prefix = ?`, repoPrefix)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var ids []string
|
||||
for rows.Next() {
|
||||
var id string
|
||||
if err := rows.Scan(&id); err != nil {
|
||||
_ = rows.Close()
|
||||
return nil, err
|
||||
}
|
||||
ids = append(ids, id)
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
_ = rows.Close()
|
||||
return nil, err
|
||||
}
|
||||
_ = rows.Close()
|
||||
return ids, nil
|
||||
}
|
||||
|
||||
// deleteByIDColumnsTx deletes rows from table where ANY of cols matches one
|
||||
// of ids, chunked so each statement stays under SQLite's 999 bound-variable
|
||||
// limit. Mirrors evictByScopeLocked's chunked from_id/to_id edge delete
|
||||
// (store.go) — the semantics source for edge eviction. Empty ids is a no-op.
|
||||
func deleteByIDColumnsTx(tx *sql.Tx, table string, cols, ids []string) error {
|
||||
if len(ids) == 0 {
|
||||
return nil
|
||||
}
|
||||
const chunk = 900
|
||||
for _, col := range cols {
|
||||
for start := 0; start < len(ids); start += chunk {
|
||||
end := minInt(start+chunk, len(ids))
|
||||
batch := ids[start:end]
|
||||
placeholders := strings.TrimSuffix(strings.Repeat("?,", len(batch)), ",")
|
||||
args := make([]any, len(batch))
|
||||
for i, id := range batch {
|
||||
args[i] = id
|
||||
}
|
||||
if _, err := tx.Exec(`DELETE FROM `+table+` WHERE `+col+` IN (`+placeholders+`)`, args...); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,222 @@
|
||||
package store_sqlite
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"github.com/zzet/gortex/internal/graph"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// openPurgeStore opens a throwaway on-disk store for the hygiene tests.
|
||||
func openPurgeStore(t *testing.T) *Store {
|
||||
t.Helper()
|
||||
s, err := Open(filepath.Join(t.TempDir(), "purge.sqlite"))
|
||||
require.NoError(t, err)
|
||||
t.Cleanup(func() { _ = s.Close() })
|
||||
return s
|
||||
}
|
||||
|
||||
// seedRepoRows inserts exactly one row keyed to prefix into nodes, edges,
|
||||
// vectors, and every repo_prefix-keyed sidecar table, so a PurgeRepo /
|
||||
// RekeyRepoPrefix test can assert each table is cleared/moved. Node/vector
|
||||
// ids embed the prefix (`<prefix>::a.go::X`) so the node-id-keyed vectors +
|
||||
// edges land in the prefix's scope. Uses raw SQL for exhaustiveness: some
|
||||
// tables have no public setter.
|
||||
func seedRepoRows(t *testing.T, db *sql.DB, prefix string) {
|
||||
t.Helper()
|
||||
nodeID := prefix + "::a.go::X"
|
||||
exec := func(q string, args ...any) {
|
||||
t.Helper()
|
||||
_, err := db.Exec(q, args...)
|
||||
require.NoError(t, err, q)
|
||||
}
|
||||
exec(`INSERT INTO nodes (id, kind, name, file_path, repo_prefix) VALUES (?, 'function', 'X', 'a.go', ?)`, nodeID, prefix)
|
||||
// An edge from the repo's node to a shared '' global external: PurgeRepo
|
||||
// must delete this edge (its from_id is a repo node) but NEVER the ''
|
||||
// target node.
|
||||
exec(`INSERT INTO edges (from_id, to_id, kind) VALUES (?, 'external_call::dep:shared', 'calls')`, nodeID)
|
||||
exec(`INSERT INTO vectors (node_id, dims, vec) VALUES (?, 1, X'00')`, nodeID)
|
||||
|
||||
exec(`INSERT INTO file_mtimes (repo_prefix, file_path, mtime_ns) VALUES (?, 'a.go', 123)`, prefix)
|
||||
exec(`INSERT INTO repo_index_state (repo_prefix, indexed_sha) VALUES (?, 'sha')`, prefix)
|
||||
exec(`INSERT INTO enrichment_state (repo_prefix, provider) VALUES (?, 'lsp')`, prefix)
|
||||
exec(`INSERT INTO clone_shingles (node_id, repo_prefix, shingles) VALUES (?, ?, X'00')`, nodeID, prefix)
|
||||
exec(`INSERT INTO constant_values (node_id, repo_prefix, file_path, value) VALUES (?, ?, 'a.go', 'v')`, nodeID, prefix)
|
||||
exec(`INSERT INTO files (repo_prefix, file_path, content_hash) VALUES (?, 'a.go', 'h')`, prefix)
|
||||
exec(`INSERT INTO ref_facts (repo_prefix, from_id, to_id, kind, line) VALUES (?, ?, 'a.go::Y', 'ref', 1)`, prefix, nodeID)
|
||||
exec(`INSERT INTO churn_enrichment (node_id, repo_prefix, commit_count) VALUES (?, ?, 3)`, nodeID, prefix)
|
||||
exec(`INSERT INTO coverage_enrichment (node_id, repo_prefix, coverage_pct) VALUES (?, ?, 0.5)`, nodeID, prefix)
|
||||
exec(`INSERT INTO release_enrichment (node_id, repo_prefix, added_in) VALUES (?, ?, 'v1')`, nodeID, prefix)
|
||||
exec(`INSERT INTO blame_enrichment (node_id, repo_prefix, email) VALUES (?, ?, 'a@b')`, nodeID, prefix)
|
||||
exec(`INSERT INTO symbol_fts (node_id, repo_prefix, tokens) VALUES (?, ?, 'x')`, nodeID, prefix)
|
||||
exec(`INSERT INTO symbol_fts_rowid (node_id, repo_prefix, fts_rowid) VALUES (?, ?, 1)`, nodeID, prefix)
|
||||
exec(`INSERT INTO content_fts (node_id, repo_prefix, file_path, ordinal, body) VALUES (?, ?, 'a.go', 0, 'body')`, nodeID, prefix)
|
||||
}
|
||||
|
||||
// countByPrefix reports how many rows a repo_prefix-keyed table holds for
|
||||
// prefix. nodes and every sidecar carry a repo_prefix column.
|
||||
func countByPrefix(t *testing.T, db *sql.DB, table, prefix string) int {
|
||||
t.Helper()
|
||||
var n int
|
||||
require.NoError(t, db.QueryRow(`SELECT COUNT(*) FROM `+table+` WHERE repo_prefix = ?`, prefix).Scan(&n))
|
||||
return n
|
||||
}
|
||||
|
||||
// countByNodeIDLike reports how many rows a node_id-keyed table (vectors)
|
||||
// holds whose node_id starts with `<prefix>::`.
|
||||
func countByNodeIDLike(t *testing.T, db *sql.DB, table, prefix string) int {
|
||||
t.Helper()
|
||||
var n int
|
||||
require.NoError(t, db.QueryRow(`SELECT COUNT(*) FROM `+table+` WHERE node_id LIKE ?`, prefix+"::%").Scan(&n))
|
||||
return n
|
||||
}
|
||||
|
||||
// prefixKeyedTables is every repo_prefix-keyed table PurgeRepo/Rekey touch,
|
||||
// minus nodes (asserted separately) — used to loop assertions.
|
||||
var prefixKeyedTables = []string{
|
||||
"file_mtimes", "repo_index_state", "enrichment_state", "clone_shingles",
|
||||
"constant_values", "files", "ref_facts", "churn_enrichment",
|
||||
"coverage_enrichment", "release_enrichment", "blame_enrichment",
|
||||
"symbol_fts", "symbol_fts_rowid", "content_fts",
|
||||
}
|
||||
|
||||
func TestPurgeRepo_ClearsEveryTable_LeavesOthersAndGlobals(t *testing.T) {
|
||||
s := openPurgeStore(t)
|
||||
// Two real repos plus a shared '' global-external node the purge must
|
||||
// never touch.
|
||||
seedRepoRows(t, s.db, "repoA")
|
||||
seedRepoRows(t, s.db, "repoB")
|
||||
_, err := s.db.Exec(`INSERT INTO nodes (id, kind, name, file_path, repo_prefix) VALUES ('external_call::dep:shared', 'external', 'shared', '', '')`)
|
||||
require.NoError(t, err)
|
||||
|
||||
require.NoError(t, s.PurgeRepo("repoA"))
|
||||
|
||||
// repoA: nodes, edges, vectors, and every sidecar cleared.
|
||||
assert.Equal(t, 0, countByPrefix(t, s.db, "nodes", "repoA"), "repoA nodes gone")
|
||||
assert.Equal(t, 0, countByNodeIDLike(t, s.db, "vectors", "repoA"), "repoA vectors gone")
|
||||
for _, tbl := range prefixKeyedTables {
|
||||
assert.Equal(t, 0, countByPrefix(t, s.db, tbl, "repoA"), "repoA %s cleared", tbl)
|
||||
}
|
||||
var edgesFromA int
|
||||
require.NoError(t, s.db.QueryRow(`SELECT COUNT(*) FROM edges WHERE from_id LIKE 'repoA::%'`).Scan(&edgesFromA))
|
||||
assert.Equal(t, 0, edgesFromA, "repoA edges gone")
|
||||
|
||||
// repoB untouched across the board.
|
||||
assert.Equal(t, 1, countByPrefix(t, s.db, "nodes", "repoB"), "repoB nodes intact")
|
||||
assert.Equal(t, 1, countByNodeIDLike(t, s.db, "vectors", "repoB"), "repoB vectors intact")
|
||||
for _, tbl := range prefixKeyedTables {
|
||||
assert.Equal(t, 1, countByPrefix(t, s.db, tbl, "repoB"), "repoB %s intact", tbl)
|
||||
}
|
||||
|
||||
// The shared '' global external survives — nothing may purge ''.
|
||||
assert.Equal(t, 1, countByPrefix(t, s.db, "nodes", ""), "'' global external survives")
|
||||
}
|
||||
|
||||
func TestPurgeRepo_RefusesEmptyPrefix(t *testing.T) {
|
||||
s := openPurgeStore(t)
|
||||
seedRepoRows(t, s.db, "")
|
||||
require.Error(t, s.PurgeRepo(""), "PurgeRepo must refuse the empty prefix (global externals / solo data)")
|
||||
// The '' rows are still there.
|
||||
assert.Equal(t, 1, countByPrefix(t, s.db, "file_mtimes", ""), "'' file_mtimes untouched by refused purge")
|
||||
}
|
||||
|
||||
func TestOrphanRepoPrefixes_SidecarOnlyResidue(t *testing.T) {
|
||||
s := openPurgeStore(t)
|
||||
// gone: a repo whose NODES were evicted but whose sidecars linger — the
|
||||
// exact leaked-untrack shape (residue in file_mtimes + repo_index_state,
|
||||
// no nodes). live: a fully-present tracked repo.
|
||||
_, err := s.db.Exec(`INSERT INTO file_mtimes (repo_prefix, file_path, mtime_ns) VALUES ('gone', 'x.go', 1)`)
|
||||
require.NoError(t, err)
|
||||
_, err = s.db.Exec(`INSERT INTO repo_index_state (repo_prefix) VALUES ('gone')`)
|
||||
require.NoError(t, err)
|
||||
seedRepoRows(t, s.db, "live")
|
||||
// A '' row must never be reported as an orphan.
|
||||
_, err = s.db.Exec(`INSERT INTO file_mtimes (repo_prefix, file_path, mtime_ns) VALUES ('', 'g.go', 1)`)
|
||||
require.NoError(t, err)
|
||||
|
||||
orphans := s.OrphanRepoPrefixes([]string{"live"})
|
||||
assert.Equal(t, []string{"gone"}, orphans, "only the nodes-less residue prefix is an orphan")
|
||||
|
||||
// Case-fold safety net: a case-only spelling drift of a tracked repo is
|
||||
// NOT an orphan.
|
||||
assert.Empty(t, s.OrphanRepoPrefixes([]string{"LIVE", "GONE"}), "case-insensitive known set covers both prefixes")
|
||||
}
|
||||
|
||||
func TestRekeyRepoPrefix_MovesProvenanceDropsNodeIDKeyed(t *testing.T) {
|
||||
s := openPurgeStore(t)
|
||||
seedRepoRows(t, s.db, "") // solo repo: everything under ''
|
||||
|
||||
require.NoError(t, s.RekeyRepoPrefix("", "drools"))
|
||||
|
||||
// Prefix/path-keyed provenance MOVED '' -> drools (so warm restart finds
|
||||
// the repo's mtimes under the new prefix instead of full-re-tracking).
|
||||
moveTables := []string{"file_mtimes", "files", "repo_index_state", "enrichment_state"}
|
||||
for _, tbl := range moveTables {
|
||||
assert.Equal(t, 0, countByPrefix(t, s.db, tbl, ""), "%s '' rows moved out", tbl)
|
||||
assert.Equal(t, 1, countByPrefix(t, s.db, tbl, "drools"), "%s rows now under new prefix", tbl)
|
||||
}
|
||||
|
||||
// node_id-keyed tables DROPPED (their old ids are dangling after the
|
||||
// re-mint) — the FTS decision included.
|
||||
dropTables := []string{
|
||||
"clone_shingles", "constant_values", "ref_facts", "churn_enrichment",
|
||||
"coverage_enrichment", "release_enrichment", "blame_enrichment",
|
||||
"symbol_fts", "symbol_fts_rowid", "content_fts",
|
||||
}
|
||||
for _, tbl := range dropTables {
|
||||
assert.Equal(t, 0, countByPrefix(t, s.db, tbl, ""), "%s '' rows dropped", tbl)
|
||||
assert.Equal(t, 0, countByPrefix(t, s.db, tbl, "drools"), "%s NOT relabeled to new prefix", tbl)
|
||||
}
|
||||
|
||||
assert.Error(t, s.RekeyRepoPrefix("repoA", ""), "rekey INTO the empty prefix is refused")
|
||||
}
|
||||
|
||||
// TestContentCrashWindow simulates the D4 kill-window at the store level:
|
||||
// per-file delete+append leaves a mix of old+new content instead of an empty
|
||||
// table, and the end-of-track sweep (keep = files that STREAMED content this
|
||||
// run) reaps both files that vanished from disk and files that still exist
|
||||
// but no longer yield content sections.
|
||||
func TestContentCrashWindow(t *testing.T) {
|
||||
s := openPurgeStore(t)
|
||||
item := func(id, file, body string) graph.ContentFTSItem {
|
||||
return graph.ContentFTSItem{NodeID: id, FilePath: file, Ordinal: 0, Body: body}
|
||||
}
|
||||
// Prior full index: three content files present.
|
||||
require.NoError(t, s.AppendContent("r", []graph.ContentFTSItem{item("r::f1", "f1.md", "old one")}))
|
||||
require.NoError(t, s.AppendContent("r", []graph.ContentFTSItem{item("r::f2", "f2.md", "old two")}))
|
||||
require.NoError(t, s.AppendContent("r", []graph.ContentFTSItem{item("r::f3", "f3.md", "old three")}))
|
||||
|
||||
// A new full index re-streams only f1 (crash before it reached the
|
||||
// rest): delete f1's rows then re-append. The other files' OLD rows must
|
||||
// survive — no empty-table window.
|
||||
require.NoError(t, s.WipeContentFileInRepo("r", "f1.md"))
|
||||
require.NoError(t, s.AppendContent("r", []graph.ContentFTSItem{item("r::f1", "f1.md", "new one")}))
|
||||
|
||||
countFile := func(file string) int {
|
||||
var n int
|
||||
require.NoError(t, s.db.QueryRow(`SELECT COUNT(*) FROM content_fts WHERE repo_prefix='r' AND file_path=?`, file).Scan(&n))
|
||||
return n
|
||||
}
|
||||
assert.Equal(t, 1, countFile("f1.md"), "f1 refreshed (delete+append, not doubled)")
|
||||
assert.Equal(t, 1, countFile("f2.md"), "f2's old rows survive the mid-parse kill (no empty table)")
|
||||
assert.Equal(t, 1, countFile("f3.md"), "f3's old rows survive the mid-parse kill (no empty table)")
|
||||
|
||||
// The next SUCCESSFUL completion: f2 was deleted from the repo, and f3
|
||||
// STILL EXISTS on disk but was emptied — it streamed no content sections
|
||||
// this run, so it is absent from the streamed set. The sweep keeps
|
||||
// exactly the streamed set {f1}, reaping both the vanished file and the
|
||||
// content->no-content transition (a disk-survival keep would have
|
||||
// protected f3's stale rows forever).
|
||||
require.NoError(t, s.DeleteContentFilesForRepoNotIn("r", map[string]struct{}{"f1.md": {}}))
|
||||
assert.Equal(t, 1, countFile("f1.md"), "still-streaming file kept")
|
||||
assert.Equal(t, 0, countFile("f2.md"), "vanished file swept")
|
||||
assert.Equal(t, 0, countFile("f3.md"), "content->no-content transition swept despite surviving on disk")
|
||||
|
||||
// Empty keep is a deliberate no-op (the never-wipe-from-empty safety
|
||||
// net; a zero-content walk routes to WipeContent instead).
|
||||
require.NoError(t, s.DeleteContentFilesForRepoNotIn("r", nil))
|
||||
assert.Equal(t, 1, countFile("f1.md"), "empty keep never wipes")
|
||||
}
|
||||
@@ -0,0 +1,219 @@
|
||||
package store_sqlite
|
||||
|
||||
import (
|
||||
"strings"
|
||||
|
||||
"github.com/zzet/gortex/internal/graph"
|
||||
)
|
||||
|
||||
// Compile-time assertions that the SQLite Store satisfies the optional
|
||||
// reference-facts persistence capability. Persisting resolved-reference facts
|
||||
// in the same backend the graph lives in makes a reference's resolution an
|
||||
// auditable, diffable record and a warm-restart seed.
|
||||
var (
|
||||
_ graph.RefFactsWriter = (*Store)(nil)
|
||||
_ graph.RefFactsReader = (*Store)(nil)
|
||||
)
|
||||
|
||||
// refFactChunk bounds rows per multi-row INSERT. 11 params/row; 80 rows = 880
|
||||
// host params, under SQLite's 999 default. Mirrors shingleChunk.
|
||||
const refFactChunk = 80
|
||||
|
||||
// candidate-list separator (unit separator — never appears in identifiers).
|
||||
const refFactCandSep = "\x1f"
|
||||
|
||||
func encodeCandidates(c []string) string { return strings.Join(c, refFactCandSep) }
|
||||
|
||||
func decodeCandidates(s string) []string {
|
||||
if s == "" {
|
||||
return nil
|
||||
}
|
||||
return strings.Split(s, refFactCandSep)
|
||||
}
|
||||
|
||||
// BulkSetRefFacts persists resolved-reference facts for one repo prefix in a
|
||||
// single transaction, chunked under the host-parameter limit. Idempotent on
|
||||
// (repo_prefix, from_id, to_id, kind, line). Empty input is a no-op.
|
||||
func (s *Store) BulkSetRefFacts(repoPrefix string, facts []graph.RefFact) error {
|
||||
if len(facts) == 0 {
|
||||
return nil
|
||||
}
|
||||
s.writeMu.Lock()
|
||||
defer s.writeMu.Unlock()
|
||||
|
||||
tx, err := s.db.Begin()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer tx.Rollback() //nolint:errcheck // rollback after Commit is a no-op
|
||||
|
||||
for start := 0; start < len(facts); start += refFactChunk {
|
||||
end := start + refFactChunk
|
||||
if end > len(facts) {
|
||||
end = len(facts)
|
||||
}
|
||||
batch := facts[start:end]
|
||||
args := make([]any, 0, len(batch)*11)
|
||||
stmt := make([]byte, 0, 96+len(batch)*24)
|
||||
stmt = append(stmt, "INSERT OR REPLACE INTO ref_facts (repo_prefix, from_id, to_id, kind, ref_name, line, origin, tier, candidates, file_path, lang) VALUES "...)
|
||||
for i, f := range batch {
|
||||
if i > 0 {
|
||||
stmt = append(stmt, ',')
|
||||
}
|
||||
stmt = append(stmt, "(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)"...)
|
||||
args = append(args, repoPrefix, f.FromID, f.ToID, f.Kind, f.RefName, f.Line, f.Origin, f.Tier, encodeCandidates(f.Candidates), f.FilePath, f.Lang)
|
||||
}
|
||||
if _, err := tx.Exec(string(stmt), args...); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return tx.Commit()
|
||||
}
|
||||
|
||||
// DeleteRefFactsByFiles drops all facts sourced in the supplied files for one
|
||||
// repo prefix, chunked into `file_path IN (…)` DELETEs. Empty input is a no-op.
|
||||
func (s *Store) DeleteRefFactsByFiles(repoPrefix string, files []string) error {
|
||||
if len(files) == 0 {
|
||||
return nil
|
||||
}
|
||||
s.writeMu.Lock()
|
||||
defer s.writeMu.Unlock()
|
||||
|
||||
tx, err := s.db.Begin()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer tx.Rollback() //nolint:errcheck // rollback after Commit is a no-op
|
||||
|
||||
for start := 0; start < len(files); start += refFactChunk {
|
||||
end := start + refFactChunk
|
||||
if end > len(files) {
|
||||
end = len(files)
|
||||
}
|
||||
chunk := files[start:end]
|
||||
args := make([]any, 0, len(chunk)+1)
|
||||
args = append(args, repoPrefix)
|
||||
stmt := make([]byte, 0, 64+len(chunk)*2)
|
||||
stmt = append(stmt, "DELETE FROM ref_facts WHERE repo_prefix = ? AND file_path IN ("...)
|
||||
for i, f := range chunk {
|
||||
if i > 0 {
|
||||
stmt = append(stmt, ',')
|
||||
}
|
||||
stmt = append(stmt, '?')
|
||||
args = append(args, f)
|
||||
}
|
||||
stmt = append(stmt, ')')
|
||||
if _, err := tx.Exec(string(stmt), args...); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return tx.Commit()
|
||||
}
|
||||
|
||||
// LoadRefFactsByFiles returns the persisted facts for one repo prefix, scoped
|
||||
// to the given files (all files when files is empty). Always non-nil.
|
||||
func (s *Store) LoadRefFactsByFiles(repoPrefix string, files []string) ([]graph.RefFact, error) {
|
||||
out := []graph.RefFact{}
|
||||
scan := func(query string, args ...any) error {
|
||||
rows, err := s.db.Query(query, args...)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer rows.Close()
|
||||
for rows.Next() {
|
||||
var f graph.RefFact
|
||||
var cand string
|
||||
if err := rows.Scan(&f.FromID, &f.ToID, &f.Kind, &f.RefName, &f.Line, &f.Origin, &f.Tier, &cand, &f.FilePath, &f.Lang); err != nil {
|
||||
return err
|
||||
}
|
||||
f.RepoPrefix = repoPrefix
|
||||
f.Candidates = decodeCandidates(cand)
|
||||
out = append(out, f)
|
||||
}
|
||||
return rows.Err()
|
||||
}
|
||||
const cols = `from_id, to_id, kind, ref_name, line, origin, tier, candidates, file_path, lang`
|
||||
if len(files) == 0 {
|
||||
if err := scan(`SELECT `+cols+` FROM ref_facts WHERE repo_prefix = ?`, repoPrefix); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
for start := 0; start < len(files); start += refFactChunk {
|
||||
end := start + refFactChunk
|
||||
if end > len(files) {
|
||||
end = len(files)
|
||||
}
|
||||
chunk := files[start:end]
|
||||
args := make([]any, 0, len(chunk)+1)
|
||||
args = append(args, repoPrefix)
|
||||
stmt := make([]byte, 0, 96+len(chunk)*2)
|
||||
stmt = append(stmt, "SELECT "+cols+" FROM ref_facts WHERE repo_prefix = ? AND file_path IN ("...)
|
||||
for i, f := range chunk {
|
||||
if i > 0 {
|
||||
stmt = append(stmt, ',')
|
||||
}
|
||||
stmt = append(stmt, '?')
|
||||
args = append(args, f)
|
||||
}
|
||||
stmt = append(stmt, ')')
|
||||
if err := scan(string(stmt), args...); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// LoadRefFactsByTargets returns the persisted facts that resolve TO any of
|
||||
// the given node IDs for one repo prefix, grouped by source file path — the
|
||||
// reverse lookup incremental re-resolution uses to find the files that
|
||||
// referenced a changed symbol after its live in-edges were evicted. Served by
|
||||
// the ref_facts_by_target index, chunked under the host-parameter limit.
|
||||
// Always non-nil; empty input is a no-op.
|
||||
func (s *Store) LoadRefFactsByTargets(repoPrefix string, targetIDs []string) (map[string][]graph.RefFact, error) {
|
||||
out := map[string][]graph.RefFact{}
|
||||
if len(targetIDs) == 0 {
|
||||
return out, nil
|
||||
}
|
||||
const cols = `from_id, to_id, kind, ref_name, line, origin, tier, candidates, file_path, lang`
|
||||
for start := 0; start < len(targetIDs); start += refFactChunk {
|
||||
end := start + refFactChunk
|
||||
if end > len(targetIDs) {
|
||||
end = len(targetIDs)
|
||||
}
|
||||
chunk := targetIDs[start:end]
|
||||
args := make([]any, 0, len(chunk)+1)
|
||||
args = append(args, repoPrefix)
|
||||
stmt := make([]byte, 0, 96+len(chunk)*2)
|
||||
stmt = append(stmt, "SELECT "+cols+" FROM ref_facts WHERE repo_prefix = ? AND to_id IN ("...)
|
||||
for i, id := range chunk {
|
||||
if i > 0 {
|
||||
stmt = append(stmt, ',')
|
||||
}
|
||||
stmt = append(stmt, '?')
|
||||
args = append(args, id)
|
||||
}
|
||||
stmt = append(stmt, ')')
|
||||
rows, err := s.db.Query(string(stmt), args...)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for rows.Next() {
|
||||
var f graph.RefFact
|
||||
var cand string
|
||||
if err := rows.Scan(&f.FromID, &f.ToID, &f.Kind, &f.RefName, &f.Line, &f.Origin, &f.Tier, &cand, &f.FilePath, &f.Lang); err != nil {
|
||||
_ = rows.Close()
|
||||
return nil, err
|
||||
}
|
||||
f.RepoPrefix = repoPrefix
|
||||
f.Candidates = decodeCandidates(cand)
|
||||
out[f.FilePath] = append(out[f.FilePath], f)
|
||||
}
|
||||
err = rows.Err()
|
||||
_ = rows.Close()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
@@ -0,0 +1,172 @@
|
||||
package store_sqlite_test
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"github.com/zzet/gortex/internal/graph"
|
||||
"github.com/zzet/gortex/internal/graph/store_sqlite"
|
||||
)
|
||||
|
||||
func openRefFactStore(t *testing.T) *store_sqlite.Store {
|
||||
t.Helper()
|
||||
s, err := store_sqlite.Open(filepath.Join(t.TempDir(), "rf.sqlite"))
|
||||
require.NoError(t, err)
|
||||
t.Cleanup(func() { _ = s.Close() })
|
||||
return s
|
||||
}
|
||||
|
||||
func TestRefFacts_Roundtrip(t *testing.T) {
|
||||
s := openRefFactStore(t)
|
||||
facts := []graph.RefFact{
|
||||
{RepoPrefix: "", FromID: "a.go::A", ToID: "b.go::B", Kind: "calls", RefName: "B", Line: 7, Origin: "ast_resolved", Tier: "ast", FilePath: "a.go", Lang: "go", Candidates: []string{"b.go::B", "c.go::B"}},
|
||||
{RepoPrefix: "", FromID: "a.go::A", ToID: "d.go::D", Kind: "references", RefName: "D", Line: 9, Origin: "lsp_resolved", Tier: "lsp", FilePath: "a.go", Lang: "go"},
|
||||
}
|
||||
require.NoError(t, s.BulkSetRefFacts("", facts))
|
||||
|
||||
got, err := s.LoadRefFactsByFiles("", []string{"a.go"})
|
||||
require.NoError(t, err)
|
||||
require.Len(t, got, 2)
|
||||
|
||||
byTo := map[string]graph.RefFact{}
|
||||
for _, f := range got {
|
||||
byTo[f.ToID] = f
|
||||
}
|
||||
require.Equal(t, "ast_resolved", byTo["b.go::B"].Origin)
|
||||
require.Equal(t, []string{"b.go::B", "c.go::B"}, byTo["b.go::B"].Candidates)
|
||||
require.Equal(t, 7, byTo["b.go::B"].Line)
|
||||
require.Equal(t, "lsp_resolved", byTo["d.go::D"].Origin)
|
||||
|
||||
// LoadRefFactsByFiles with empty file list returns all for the repo.
|
||||
all, err := s.LoadRefFactsByFiles("", nil)
|
||||
require.NoError(t, err)
|
||||
require.Len(t, all, 2)
|
||||
}
|
||||
|
||||
func TestRefFacts_DeleteByFile(t *testing.T) {
|
||||
s := openRefFactStore(t)
|
||||
require.NoError(t, s.BulkSetRefFacts("", []graph.RefFact{
|
||||
{FromID: "a.go::A", ToID: "x", Kind: "calls", FilePath: "a.go"},
|
||||
{FromID: "b.go::B", ToID: "y", Kind: "calls", FilePath: "b.go"},
|
||||
}))
|
||||
require.NoError(t, s.DeleteRefFactsByFiles("", []string{"a.go"}))
|
||||
got, err := s.LoadRefFactsByFiles("", nil)
|
||||
require.NoError(t, err)
|
||||
require.Len(t, got, 1)
|
||||
require.Equal(t, "b.go", got[0].FilePath)
|
||||
}
|
||||
|
||||
func TestRefFacts_RepoScoping(t *testing.T) {
|
||||
s := openRefFactStore(t)
|
||||
require.NoError(t, s.BulkSetRefFacts("repoA", []graph.RefFact{{FromID: "f::A", ToID: "tA", Kind: "calls", FilePath: "f.go"}}))
|
||||
require.NoError(t, s.BulkSetRefFacts("repoB", []graph.RefFact{{FromID: "f::A", ToID: "tB", Kind: "calls", FilePath: "f.go"}}))
|
||||
|
||||
a, err := s.LoadRefFactsByFiles("repoA", []string{"f.go"})
|
||||
require.NoError(t, err)
|
||||
require.Len(t, a, 1)
|
||||
require.Equal(t, "tA", a[0].ToID)
|
||||
|
||||
// Deleting repoA's file must not touch repoB.
|
||||
require.NoError(t, s.DeleteRefFactsByFiles("repoA", []string{"f.go"}))
|
||||
b, err := s.LoadRefFactsByFiles("repoB", []string{"f.go"})
|
||||
require.NoError(t, err)
|
||||
require.Len(t, b, 1)
|
||||
require.Equal(t, "tB", b[0].ToID)
|
||||
}
|
||||
|
||||
func TestRefFacts_Chunking(t *testing.T) {
|
||||
s := openRefFactStore(t)
|
||||
const n = 500 // > refFactChunk (80)
|
||||
facts := make([]graph.RefFact, n)
|
||||
for i := range facts {
|
||||
facts[i] = graph.RefFact{FromID: fmt.Sprintf("a.go::f%d", i), ToID: fmt.Sprintf("t%d", i), Kind: "calls", FilePath: "a.go"}
|
||||
}
|
||||
require.NoError(t, s.BulkSetRefFacts("", facts))
|
||||
got, err := s.LoadRefFactsByFiles("", []string{"a.go"})
|
||||
require.NoError(t, err)
|
||||
require.Len(t, got, n)
|
||||
}
|
||||
|
||||
func TestRefFacts_EmptyNoop(t *testing.T) {
|
||||
s := openRefFactStore(t)
|
||||
require.NoError(t, s.BulkSetRefFacts("", nil))
|
||||
require.NoError(t, s.DeleteRefFactsByFiles("", nil))
|
||||
got, err := s.LoadRefFactsByFiles("", nil)
|
||||
require.NoError(t, err)
|
||||
require.Empty(t, got)
|
||||
}
|
||||
|
||||
func TestRefFacts_LoadByTargets(t *testing.T) {
|
||||
s := openRefFactStore(t)
|
||||
require.NoError(t, s.BulkSetRefFacts("", []graph.RefFact{
|
||||
{FromID: "b.go::Caller", ToID: "a.go::F", Kind: "calls", RefName: "F", Line: 3, Origin: "ast_resolved", Tier: "ast", FilePath: "b.go", Lang: "go"},
|
||||
{FromID: "c.go::Other", ToID: "a.go::F", Kind: "references", RefName: "F", FilePath: "c.go"},
|
||||
{FromID: "c.go::Other", ToID: "a.go::G", Kind: "calls", RefName: "G", FilePath: "c.go"},
|
||||
{FromID: "d.go::X", ToID: "z.go::Z", Kind: "calls", RefName: "Z", FilePath: "d.go"},
|
||||
}))
|
||||
|
||||
byFile, err := s.LoadRefFactsByTargets("", []string{"a.go::F", "a.go::G"})
|
||||
require.NoError(t, err)
|
||||
require.Len(t, byFile, 2, "facts must be grouped by source file: %v", byFile)
|
||||
require.Len(t, byFile["b.go"], 1)
|
||||
require.Equal(t, "a.go::F", byFile["b.go"][0].ToID)
|
||||
require.Equal(t, "F", byFile["b.go"][0].RefName)
|
||||
require.Equal(t, "ast_resolved", byFile["b.go"][0].Origin)
|
||||
require.Len(t, byFile["c.go"], 2, "both of c.go's facts target the queried symbols")
|
||||
require.NotContains(t, byFile, "d.go", "a fact targeting an unqueried symbol must not match")
|
||||
}
|
||||
|
||||
func TestRefFacts_LoadByTargets_EmptyAndMissing(t *testing.T) {
|
||||
s := openRefFactStore(t)
|
||||
require.NoError(t, s.BulkSetRefFacts("", []graph.RefFact{
|
||||
{FromID: "a.go::A", ToID: "b.go::B", Kind: "calls", FilePath: "a.go"},
|
||||
}))
|
||||
|
||||
// Empty input: empty, non-nil map.
|
||||
empty, err := s.LoadRefFactsByTargets("", nil)
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, empty)
|
||||
require.Empty(t, empty)
|
||||
|
||||
// A target nothing references: no rows, no error.
|
||||
miss, err := s.LoadRefFactsByTargets("", []string{"nope::Missing"})
|
||||
require.NoError(t, err)
|
||||
require.Empty(t, miss)
|
||||
}
|
||||
|
||||
func TestRefFacts_LoadByTargets_RepoScoping(t *testing.T) {
|
||||
s := openRefFactStore(t)
|
||||
require.NoError(t, s.BulkSetRefFacts("repoA", []graph.RefFact{{FromID: "fa::A", ToID: "shared::T", Kind: "calls", FilePath: "fa.go"}}))
|
||||
require.NoError(t, s.BulkSetRefFacts("repoB", []graph.RefFact{{FromID: "fb::B", ToID: "shared::T", Kind: "calls", FilePath: "fb.go"}}))
|
||||
|
||||
a, err := s.LoadRefFactsByTargets("repoA", []string{"shared::T"})
|
||||
require.NoError(t, err)
|
||||
require.Len(t, a, 1)
|
||||
require.Len(t, a["fa.go"], 1)
|
||||
require.Equal(t, "repoA", a["fa.go"][0].RepoPrefix, "loaded facts must carry the queried repo prefix")
|
||||
require.NotContains(t, a, "fb.go", "another repo's facts must not leak into the result")
|
||||
}
|
||||
|
||||
func TestRefFacts_LoadByTargets_Chunking(t *testing.T) {
|
||||
s := openRefFactStore(t)
|
||||
const n = 500 // > refFactChunk (80)
|
||||
facts := make([]graph.RefFact, n)
|
||||
targets := make([]string, n)
|
||||
for i := range facts {
|
||||
facts[i] = graph.RefFact{
|
||||
FromID: fmt.Sprintf("src%d.go::f", i),
|
||||
ToID: fmt.Sprintf("dst.go::t%d", i),
|
||||
Kind: "calls",
|
||||
FilePath: fmt.Sprintf("src%d.go", i),
|
||||
}
|
||||
targets[i] = fmt.Sprintf("dst.go::t%d", i)
|
||||
}
|
||||
require.NoError(t, s.BulkSetRefFacts("", facts))
|
||||
|
||||
byFile, err := s.LoadRefFactsByTargets("", targets)
|
||||
require.NoError(t, err)
|
||||
require.Len(t, byFile, n, "every chunked target must come back grouped under its source file")
|
||||
}
|
||||
@@ -0,0 +1,140 @@
|
||||
package store_sqlite
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
|
||||
"github.com/zzet/gortex/internal/graph"
|
||||
)
|
||||
|
||||
var (
|
||||
_ graph.ReleaseEnrichmentWriter = (*Store)(nil)
|
||||
_ graph.ReleaseEnrichmentReader = (*Store)(nil)
|
||||
)
|
||||
|
||||
// releaseChunk bounds rows per multi-row INSERT (3 cols → 3 params/row).
|
||||
const releaseChunk = 300
|
||||
|
||||
const releaseCols = `node_id, repo_prefix, added_in`
|
||||
|
||||
// BulkSetReleases persists release rows for one repo prefix, chunked.
|
||||
func (s *Store) BulkSetReleases(repoPrefix string, rows []graph.ReleaseEnrichment) error {
|
||||
if len(rows) == 0 {
|
||||
return nil
|
||||
}
|
||||
s.writeMu.Lock()
|
||||
defer s.writeMu.Unlock()
|
||||
|
||||
tx, err := s.db.Begin()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer tx.Rollback() //nolint:errcheck
|
||||
|
||||
for start := 0; start < len(rows); start += releaseChunk {
|
||||
end := start + releaseChunk
|
||||
if end > len(rows) {
|
||||
end = len(rows)
|
||||
}
|
||||
batch := rows[start:end]
|
||||
args := make([]any, 0, len(batch)*3)
|
||||
stmt := make([]byte, 0, 96+len(batch)*12)
|
||||
stmt = append(stmt, "INSERT OR REPLACE INTO release_enrichment ("...)
|
||||
stmt = append(stmt, releaseCols...)
|
||||
stmt = append(stmt, ") VALUES "...)
|
||||
for i, e := range batch {
|
||||
if i > 0 {
|
||||
stmt = append(stmt, ',')
|
||||
}
|
||||
stmt = append(stmt, "(?,?,?)"...)
|
||||
args = append(args, e.NodeID, repoPrefix, e.AddedIn)
|
||||
}
|
||||
if _, err := tx.Exec(string(stmt), args...); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return tx.Commit()
|
||||
}
|
||||
|
||||
// DeleteReleases drops release rows for the supplied node ids, chunked.
|
||||
func (s *Store) DeleteReleases(nodeIDs []string) error {
|
||||
if len(nodeIDs) == 0 {
|
||||
return nil
|
||||
}
|
||||
seen := make(map[string]struct{}, len(nodeIDs))
|
||||
uniq := make([]string, 0, len(nodeIDs))
|
||||
for _, id := range nodeIDs {
|
||||
if id == "" {
|
||||
continue
|
||||
}
|
||||
if _, ok := seen[id]; ok {
|
||||
continue
|
||||
}
|
||||
seen[id] = struct{}{}
|
||||
uniq = append(uniq, id)
|
||||
}
|
||||
if len(uniq) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
s.writeMu.Lock()
|
||||
defer s.writeMu.Unlock()
|
||||
|
||||
tx, err := s.db.Begin()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer tx.Rollback() //nolint:errcheck
|
||||
|
||||
for start := 0; start < len(uniq); start += releaseChunk {
|
||||
end := start + releaseChunk
|
||||
if end > len(uniq) {
|
||||
end = len(uniq)
|
||||
}
|
||||
chunk := uniq[start:end]
|
||||
args := make([]any, len(chunk))
|
||||
stmt := make([]byte, 0, 56+len(chunk)*2)
|
||||
stmt = append(stmt, "DELETE FROM release_enrichment WHERE node_id IN ("...)
|
||||
for i, id := range chunk {
|
||||
if i > 0 {
|
||||
stmt = append(stmt, ',')
|
||||
}
|
||||
stmt = append(stmt, '?')
|
||||
args[i] = id
|
||||
}
|
||||
stmt = append(stmt, ')')
|
||||
if _, err := tx.Exec(string(stmt), args...); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return tx.Commit()
|
||||
}
|
||||
|
||||
// ReleaseRows returns release rows for repoPrefix; empty → all repos.
|
||||
func (s *Store) ReleaseRows(repoPrefix string) []graph.ReleaseEnrichment {
|
||||
var (
|
||||
rows *sql.Rows
|
||||
err error
|
||||
)
|
||||
if repoPrefix == "" {
|
||||
rows, err = s.db.Query(`SELECT ` + releaseCols + ` FROM release_enrichment`)
|
||||
} else {
|
||||
rows, err = s.db.Query(`SELECT `+releaseCols+` FROM release_enrichment WHERE repo_prefix = ?`, repoPrefix)
|
||||
}
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
var out []graph.ReleaseEnrichment
|
||||
for rows.Next() {
|
||||
var e graph.ReleaseEnrichment
|
||||
if err := rows.Scan(&e.NodeID, &e.RepoPrefix, &e.AddedIn); err != nil {
|
||||
return out
|
||||
}
|
||||
out = append(out, e)
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return out
|
||||
}
|
||||
return out
|
||||
}
|
||||
@@ -0,0 +1,22 @@
|
||||
package store_sqlite_test
|
||||
|
||||
import (
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"github.com/zzet/gortex/internal/graph"
|
||||
"github.com/zzet/gortex/internal/graph/store_sqlite"
|
||||
"github.com/zzet/gortex/internal/graph/storetest"
|
||||
)
|
||||
|
||||
func TestSQLiteStoreConformance(t *testing.T) {
|
||||
storetest.RunConformance(t, func(t *testing.T) graph.Store {
|
||||
dir := t.TempDir()
|
||||
s, err := store_sqlite.Open(filepath.Join(dir, "test.sqlite"))
|
||||
if err != nil {
|
||||
t.Fatalf("Open: %v", err)
|
||||
}
|
||||
t.Cleanup(func() { _ = s.Close() })
|
||||
return s
|
||||
})
|
||||
}
|
||||
@@ -0,0 +1,362 @@
|
||||
package store_sqlite
|
||||
|
||||
import (
|
||||
"github.com/zzet/gortex/internal/graph"
|
||||
)
|
||||
|
||||
// The graph-traversal and subgraph-reader optional capabilities for the
|
||||
// SQLite backend. Each method mirrors the in-memory *graph.Graph
|
||||
// reference implementation exactly so both satisfy the same conformance
|
||||
// suite (internal/graph/storetest). The walks use the same per-node /
|
||||
// batched edge readers the in-memory store uses (GetOutEdges /
|
||||
// GetInEdges / GetFileNodes / GetNodesByIDs / GetIn|OutEdgesByNodeIDs),
|
||||
// which on SQLite hit the (from_id,kind) / (to_id,kind) / file_path
|
||||
// indexes — no new prepared statements needed.
|
||||
|
||||
var (
|
||||
_ graph.ReachableForwardByKinds = (*Store)(nil)
|
||||
_ graph.ClassHierarchyTraverser = (*Store)(nil)
|
||||
_ graph.FrontierExpander = (*Store)(nil)
|
||||
_ graph.FileEditingContext = (*Store)(nil)
|
||||
_ graph.FileSubGraphReader = (*Store)(nil)
|
||||
_ graph.FileSubGraphCountReader = (*Store)(nil)
|
||||
)
|
||||
|
||||
// ReachableForwardByKinds computes the set of node IDs reachable from
|
||||
// the seed frontier via outgoing edges whose Kind is in kinds, via a
|
||||
// layer-by-layer forward BFS. Empty seeds returns nil; empty kinds
|
||||
// returns the seed set unchanged. The returned map keys are the
|
||||
// reachable IDs (seeds included); every value is true.
|
||||
func (s *Store) ReachableForwardByKinds(seeds []string, kinds []graph.EdgeKind) map[string]bool {
|
||||
if len(seeds) == 0 {
|
||||
return nil
|
||||
}
|
||||
covered := make(map[string]bool, len(seeds))
|
||||
frontier := make([]string, 0, len(seeds))
|
||||
for _, id := range seeds {
|
||||
if id == "" || covered[id] {
|
||||
continue
|
||||
}
|
||||
covered[id] = true
|
||||
frontier = append(frontier, id)
|
||||
}
|
||||
if len(kinds) == 0 {
|
||||
return covered
|
||||
}
|
||||
allowed := make(map[graph.EdgeKind]struct{}, len(kinds))
|
||||
for _, k := range kinds {
|
||||
allowed[k] = struct{}{}
|
||||
}
|
||||
for len(frontier) > 0 {
|
||||
next := frontier[:0:0]
|
||||
for _, id := range frontier {
|
||||
for _, e := range s.GetOutEdges(id) {
|
||||
if e == nil {
|
||||
continue
|
||||
}
|
||||
if _, ok := allowed[e.Kind]; !ok {
|
||||
continue
|
||||
}
|
||||
if !covered[e.To] {
|
||||
covered[e.To] = true
|
||||
next = append(next, e.To)
|
||||
}
|
||||
}
|
||||
}
|
||||
frontier = next
|
||||
}
|
||||
return covered
|
||||
}
|
||||
|
||||
// ClassHierarchyTraverse walks the inheritance subgraph rooted at
|
||||
// seedID, following only edges whose Kind is in kinds, up to depth hops.
|
||||
// direction "up" follows outgoing edges; "down" follows incoming. Empty
|
||||
// kinds, depth <= 0, an unknown direction, or an unknown seed return
|
||||
// nil. Each returned row carries the full Path (node IDs from the seed,
|
||||
// exclusive) and per-hop EdgeKinds for one terminal node.
|
||||
func (s *Store) ClassHierarchyTraverse(
|
||||
seedID string,
|
||||
direction string,
|
||||
kinds []graph.EdgeKind,
|
||||
depth int,
|
||||
) []graph.ClassHierarchyRow {
|
||||
if seedID == "" || depth <= 0 || len(kinds) == 0 {
|
||||
return nil
|
||||
}
|
||||
kset := make(map[graph.EdgeKind]struct{}, len(kinds))
|
||||
for _, k := range kinds {
|
||||
if k == "" {
|
||||
continue
|
||||
}
|
||||
kset[k] = struct{}{}
|
||||
}
|
||||
if len(kset) == 0 {
|
||||
return nil
|
||||
}
|
||||
if s.GetNode(seedID) == nil {
|
||||
return nil
|
||||
}
|
||||
walkUp := direction == "up"
|
||||
walkDown := direction == "down"
|
||||
if !walkUp && !walkDown {
|
||||
return nil
|
||||
}
|
||||
type travQueued struct {
|
||||
id string
|
||||
path []string
|
||||
edgeKinds []graph.EdgeKind
|
||||
hops int
|
||||
}
|
||||
visited := map[string]struct{}{seedID: {}}
|
||||
queue := []travQueued{{id: seedID, path: nil, edgeKinds: nil, hops: 0}}
|
||||
var out []graph.ClassHierarchyRow
|
||||
for len(queue) > 0 {
|
||||
cur := queue[0]
|
||||
queue = queue[1:]
|
||||
if cur.hops >= depth {
|
||||
continue
|
||||
}
|
||||
var edges []*graph.Edge
|
||||
if walkUp {
|
||||
edges = s.GetOutEdges(cur.id)
|
||||
} else {
|
||||
edges = s.GetInEdges(cur.id)
|
||||
}
|
||||
for _, e := range edges {
|
||||
if e == nil {
|
||||
continue
|
||||
}
|
||||
if _, ok := kset[e.Kind]; !ok {
|
||||
continue
|
||||
}
|
||||
var nb string
|
||||
if walkUp {
|
||||
nb = e.To
|
||||
} else {
|
||||
nb = e.From
|
||||
}
|
||||
if nb == "" {
|
||||
continue
|
||||
}
|
||||
if _, ok := visited[nb]; ok {
|
||||
continue
|
||||
}
|
||||
visited[nb] = struct{}{}
|
||||
newPath := append([]string(nil), cur.path...)
|
||||
newPath = append(newPath, nb)
|
||||
newKinds := append([]graph.EdgeKind(nil), cur.edgeKinds...)
|
||||
newKinds = append(newKinds, e.Kind)
|
||||
out = append(out, graph.ClassHierarchyRow{
|
||||
Path: newPath,
|
||||
EdgeKinds: newKinds,
|
||||
})
|
||||
queue = append(queue, travQueued{id: nb, path: newPath, edgeKinds: newKinds, hops: cur.hops + 1})
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// ExpandFrontier returns, for the given source IDs, their adjacent edges
|
||||
// of the requested kinds plus the neighbour node at each edge's far end.
|
||||
// forward=true follows outgoing edges (neighbour = edge target);
|
||||
// forward=false follows incoming (neighbour = edge source). Empty ids or
|
||||
// empty kinds return nil; limit > 0 caps the total number of hops.
|
||||
func (s *Store) ExpandFrontier(ids []string, forward bool, kinds []graph.EdgeKind, limit int) []graph.FrontierHop {
|
||||
if len(ids) == 0 || len(kinds) == 0 {
|
||||
return nil
|
||||
}
|
||||
kset := make(map[graph.EdgeKind]struct{}, len(kinds))
|
||||
for _, k := range kinds {
|
||||
kset[k] = struct{}{}
|
||||
}
|
||||
var out []graph.FrontierHop
|
||||
for _, id := range ids {
|
||||
var edges []*graph.Edge
|
||||
if forward {
|
||||
edges = s.GetOutEdges(id)
|
||||
} else {
|
||||
edges = s.GetInEdges(id)
|
||||
}
|
||||
for _, e := range edges {
|
||||
if e == nil {
|
||||
continue
|
||||
}
|
||||
if _, ok := kset[e.Kind]; !ok {
|
||||
continue
|
||||
}
|
||||
var nbID string
|
||||
if forward {
|
||||
nbID = e.To
|
||||
} else {
|
||||
nbID = e.From
|
||||
}
|
||||
nb := s.GetNode(nbID)
|
||||
if nb == nil {
|
||||
continue
|
||||
}
|
||||
out = append(out, graph.FrontierHop{Edge: e, Neighbor: nb})
|
||||
if limit > 0 && len(out) >= limit {
|
||||
return out
|
||||
}
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// FileEditingContext returns the get_editing_context payload for
|
||||
// filePath: the file node, the symbols defined in it, the file node's
|
||||
// import out-edges, and the 1-hop callers / callees (via EdgeCalls) of
|
||||
// the defined call-target symbols, filtered to symbols outside the file.
|
||||
// kinds is the set of node kinds treated as call targets (function +
|
||||
// method). Empty path or a file with no nodes returns nil.
|
||||
func (s *Store) FileEditingContext(filePath string, kinds []graph.NodeKind) *graph.FileEditingContextResult {
|
||||
if filePath == "" {
|
||||
return nil
|
||||
}
|
||||
nodes := s.GetFileNodes(filePath)
|
||||
if len(nodes) == 0 {
|
||||
return nil
|
||||
}
|
||||
kset := make(map[graph.NodeKind]struct{}, len(kinds))
|
||||
for _, k := range kinds {
|
||||
if k == "" {
|
||||
continue
|
||||
}
|
||||
kset[k] = struct{}{}
|
||||
}
|
||||
res := &graph.FileEditingContextResult{}
|
||||
var fileNodeID string
|
||||
var defNodeIDs []string
|
||||
for _, n := range nodes {
|
||||
if n == nil {
|
||||
continue
|
||||
}
|
||||
if n.Kind == graph.KindFile {
|
||||
res.FileNode = n
|
||||
fileNodeID = n.ID
|
||||
continue
|
||||
}
|
||||
res.Defines = append(res.Defines, n)
|
||||
if _, ok := kset[n.Kind]; ok {
|
||||
defNodeIDs = append(defNodeIDs, n.ID)
|
||||
}
|
||||
}
|
||||
if fileNodeID != "" {
|
||||
for _, e := range s.GetOutEdges(fileNodeID) {
|
||||
if e == nil {
|
||||
continue
|
||||
}
|
||||
if e.Kind == graph.EdgeImports {
|
||||
res.Imports = append(res.Imports, e)
|
||||
}
|
||||
}
|
||||
}
|
||||
if len(defNodeIDs) == 0 {
|
||||
return res
|
||||
}
|
||||
inEdges := s.GetInEdgesByNodeIDs(defNodeIDs)
|
||||
outEdges := s.GetOutEdgesByNodeIDs(defNodeIDs)
|
||||
callerIDSet := make(map[string]struct{})
|
||||
calleeIDSet := make(map[string]struct{})
|
||||
for _, id := range defNodeIDs {
|
||||
for _, e := range inEdges[id] {
|
||||
if e == nil || e.Kind != graph.EdgeCalls {
|
||||
continue
|
||||
}
|
||||
if e.From == "" {
|
||||
continue
|
||||
}
|
||||
callerIDSet[e.From] = struct{}{}
|
||||
}
|
||||
for _, e := range outEdges[id] {
|
||||
if e == nil || e.Kind != graph.EdgeCalls {
|
||||
continue
|
||||
}
|
||||
if e.To == "" {
|
||||
continue
|
||||
}
|
||||
calleeIDSet[e.To] = struct{}{}
|
||||
}
|
||||
}
|
||||
callerIDs := make([]string, 0, len(callerIDSet))
|
||||
for id := range callerIDSet {
|
||||
callerIDs = append(callerIDs, id)
|
||||
}
|
||||
calleeIDs := make([]string, 0, len(calleeIDSet))
|
||||
for id := range calleeIDSet {
|
||||
calleeIDs = append(calleeIDs, id)
|
||||
}
|
||||
callerNodes := s.GetNodesByIDs(callerIDs)
|
||||
calleeNodes := s.GetNodesByIDs(calleeIDs)
|
||||
for _, id := range callerIDs {
|
||||
n := callerNodes[id]
|
||||
if n == nil || n.FilePath == filePath {
|
||||
continue
|
||||
}
|
||||
res.CalledBy = append(res.CalledBy, n)
|
||||
}
|
||||
for _, id := range calleeIDs {
|
||||
n := calleeNodes[id]
|
||||
if n == nil || n.FilePath == filePath {
|
||||
continue
|
||||
}
|
||||
res.Calls = append(res.Calls, n)
|
||||
}
|
||||
return res
|
||||
}
|
||||
|
||||
// GetFileSubGraph returns every node anchored to filePath plus every
|
||||
// edge adjacent to one of those nodes, deduplicated by (from, to, kind).
|
||||
// A missing / empty file returns (nil, nil).
|
||||
func (s *Store) GetFileSubGraph(filePath string) ([]*graph.Node, []*graph.Edge) {
|
||||
if filePath == "" {
|
||||
return nil, nil
|
||||
}
|
||||
nodes := s.GetFileNodes(filePath)
|
||||
if len(nodes) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
ids := make([]string, 0, len(nodes))
|
||||
for _, n := range nodes {
|
||||
if n != nil && n.ID != "" {
|
||||
ids = append(ids, n.ID)
|
||||
}
|
||||
}
|
||||
outByID := s.GetOutEdgesByNodeIDs(ids)
|
||||
inByID := s.GetInEdgesByNodeIDs(ids)
|
||||
type travEdgeKey struct {
|
||||
from string
|
||||
to string
|
||||
kind graph.EdgeKind
|
||||
}
|
||||
seen := make(map[travEdgeKey]struct{}, 2*len(ids))
|
||||
edges := make([]*graph.Edge, 0, 2*len(ids))
|
||||
add := func(e *graph.Edge) {
|
||||
if e == nil {
|
||||
return
|
||||
}
|
||||
k := travEdgeKey{from: e.From, to: e.To, kind: e.Kind}
|
||||
if _, ok := seen[k]; ok {
|
||||
return
|
||||
}
|
||||
seen[k] = struct{}{}
|
||||
edges = append(edges, e)
|
||||
}
|
||||
for _, id := range ids {
|
||||
for _, e := range outByID[id] {
|
||||
add(e)
|
||||
}
|
||||
for _, e := range inByID[id] {
|
||||
add(e)
|
||||
}
|
||||
}
|
||||
return nodes, edges
|
||||
}
|
||||
|
||||
// GetFileSubGraphCounts is the count-only sibling of GetFileSubGraph:
|
||||
// it returns the file's nodes plus the number of distinct adjacent
|
||||
// edges, without materialising the edge slice for the caller.
|
||||
func (s *Store) GetFileSubGraphCounts(filePath string) ([]*graph.Node, int) {
|
||||
nodes, edges := s.GetFileSubGraph(filePath)
|
||||
return nodes, len(edges)
|
||||
}
|
||||
@@ -0,0 +1,295 @@
|
||||
package store_sqlite
|
||||
|
||||
import (
|
||||
"container/heap"
|
||||
"encoding/binary"
|
||||
"errors"
|
||||
"math"
|
||||
|
||||
"github.com/zzet/gortex/internal/graph"
|
||||
)
|
||||
|
||||
// Compile-time assertion that the SQLite Store satisfies the optional
|
||||
// engine-native vector-search capability.
|
||||
var _ graph.VectorSearcher = (*Store)(nil)
|
||||
|
||||
// errInvalidDims is returned by BuildVectorIndex for a negative width.
|
||||
var errInvalidDims = errors.New("store_sqlite: invalid vector dims")
|
||||
|
||||
// Vector design (pure-Go, zero CGo)
|
||||
//
|
||||
// modernc.org/sqlite is a pure-Go SQLite that cannot load C extensions,
|
||||
// so sqlite-vec / sqlite-vector are off the table — and staying CGo-free
|
||||
// is the whole point of this backend. Embeddings are persisted as a
|
||||
// little-endian float32 BLOB in the `vectors` table; the win over the
|
||||
// daemon's in-process HNSW fallback is durability: vectors survive a
|
||||
// restart instead of being recomputed.
|
||||
//
|
||||
// Queries use an exact brute-force cosine top-k: SimilarTo streams every
|
||||
// stored vector, scores it against the query, and keeps the best `limit`
|
||||
// in a bounded max-heap. This is O(N) per query but fully correct,
|
||||
// deterministic, and holds no extra Store state (the Store struct lives
|
||||
// in store.go and cannot be edited here). An on-Store HNSW cache is a
|
||||
// future optimisation; for the corpus sizes this backend targets the
|
||||
// exact path is the simplest thing that is verifiably right.
|
||||
//
|
||||
// BuildVectorIndex only validates/records intent — there is no separate
|
||||
// index structure to build, since SimilarTo computes over the table
|
||||
// directly.
|
||||
|
||||
// vectorChunk bounds rows per multi-row INSERT in BulkUpsertEmbeddings.
|
||||
// 3 host params per row, SQLite's default limit is 999 → 333 max; 300
|
||||
// leaves headroom.
|
||||
const vectorChunk = 300
|
||||
|
||||
// encodeVec serialises a float32 slice to a little-endian BLOB
|
||||
// (4 bytes per element).
|
||||
func encodeVec(vec []float32) []byte {
|
||||
b := make([]byte, len(vec)*4)
|
||||
for i, f := range vec {
|
||||
binary.LittleEndian.PutUint32(b[i*4:], math.Float32bits(f))
|
||||
}
|
||||
return b
|
||||
}
|
||||
|
||||
// decodeVec is the inverse of encodeVec. A BLOB whose length is not a
|
||||
// multiple of 4 yields nil (corrupt row); callers skip nil vectors.
|
||||
func decodeVec(b []byte) []float32 {
|
||||
if len(b)%4 != 0 {
|
||||
return nil
|
||||
}
|
||||
out := make([]float32, len(b)/4)
|
||||
for i := range out {
|
||||
out[i] = math.Float32frombits(binary.LittleEndian.Uint32(b[i*4:]))
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// UpsertEmbedding persists one node's embedding, replacing any prior
|
||||
// vector for that node ID.
|
||||
func (s *Store) UpsertEmbedding(nodeID string, vec []float32) error {
|
||||
s.writeMu.Lock()
|
||||
defer s.writeMu.Unlock()
|
||||
_, err := s.db.Exec(
|
||||
`INSERT OR REPLACE INTO vectors (node_id, dims, vec) VALUES (?, ?, ?)`,
|
||||
nodeID, len(vec), encodeVec(vec),
|
||||
)
|
||||
return err
|
||||
}
|
||||
|
||||
// BulkUpsertEmbeddings persists many embeddings in a single transaction,
|
||||
// chunked under SQLite's host-parameter limit. Idempotent on NodeID.
|
||||
// Empty input is a no-op.
|
||||
func (s *Store) BulkUpsertEmbeddings(items []graph.VectorItem) error {
|
||||
if len(items) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
s.writeMu.Lock()
|
||||
defer s.writeMu.Unlock()
|
||||
|
||||
tx, err := s.db.Begin()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer tx.Rollback() //nolint:errcheck // rollback after Commit is a no-op
|
||||
|
||||
for start := 0; start < len(items); start += vectorChunk {
|
||||
end := start + vectorChunk
|
||||
if end > len(items) {
|
||||
end = len(items)
|
||||
}
|
||||
batch := items[start:end]
|
||||
|
||||
args := make([]any, 0, len(batch)*3)
|
||||
stmt := make([]byte, 0, 64+len(batch)*16)
|
||||
stmt = append(stmt, "INSERT OR REPLACE INTO vectors (node_id, dims, vec) VALUES "...)
|
||||
for i, it := range batch {
|
||||
if i > 0 {
|
||||
stmt = append(stmt, ',')
|
||||
}
|
||||
stmt = append(stmt, "(?, ?, ?)"...)
|
||||
args = append(args, it.NodeID, len(it.Vec), encodeVec(it.Vec))
|
||||
}
|
||||
if _, err := tx.Exec(string(stmt), args...); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
return tx.Commit()
|
||||
}
|
||||
|
||||
// BuildVectorIndex finalises the vector index. Because SimilarTo scores
|
||||
// over the `vectors` table directly there is no separate structure to
|
||||
// populate; this validates the declared width is positive and is
|
||||
// otherwise a no-op (idempotent, safe to call repeatedly).
|
||||
func (s *Store) BuildVectorIndex(dims int) error {
|
||||
if dims < 0 {
|
||||
return errInvalidDims
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// GetEmbeddings reads back the stored vectors for an explicit set of
|
||||
// node IDs in one batched scan. It does not rank — it is the read side
|
||||
// of the post-rerank cosine-refinement stage, which needs the raw
|
||||
// vectors (not a distance) to score against a freshly embedded query.
|
||||
//
|
||||
// IDs with no stored vector (or a corrupt BLOB) are simply absent from
|
||||
// the returned map; empty input yields an empty map. A query error
|
||||
// yields whatever was decoded so far rather than failing the caller —
|
||||
// the refinement stage treats a thin result as "score what we have",
|
||||
// never as a hard error, so a transient read can never regress search.
|
||||
// The IN-list is chunked under SQLite's host-parameter limit.
|
||||
func (s *Store) GetEmbeddings(ids []string) map[string][]float32 {
|
||||
out := make(map[string][]float32, len(ids))
|
||||
if len(ids) == 0 {
|
||||
return out
|
||||
}
|
||||
|
||||
for start := 0; start < len(ids); start += vectorChunk {
|
||||
end := start + vectorChunk
|
||||
if end > len(ids) {
|
||||
end = len(ids)
|
||||
}
|
||||
batch := ids[start:end]
|
||||
|
||||
stmt := make([]byte, 0, 48+len(batch)*2)
|
||||
stmt = append(stmt, "SELECT node_id, vec FROM vectors WHERE node_id IN ("...)
|
||||
args := make([]any, 0, len(batch))
|
||||
for i, id := range batch {
|
||||
if i > 0 {
|
||||
stmt = append(stmt, ',')
|
||||
}
|
||||
stmt = append(stmt, '?')
|
||||
args = append(args, id)
|
||||
}
|
||||
stmt = append(stmt, ')')
|
||||
|
||||
rows, err := s.db.Query(string(stmt), args...)
|
||||
if err != nil {
|
||||
// Return what we have; the refinement stage is best-effort.
|
||||
return out
|
||||
}
|
||||
for rows.Next() {
|
||||
var (
|
||||
id string
|
||||
blob []byte
|
||||
)
|
||||
if err := rows.Scan(&id, &blob); err != nil {
|
||||
_ = rows.Close()
|
||||
return out
|
||||
}
|
||||
if vec := decodeVec(blob); vec != nil {
|
||||
out[id] = vec
|
||||
}
|
||||
}
|
||||
_ = rows.Close()
|
||||
}
|
||||
|
||||
return out
|
||||
}
|
||||
|
||||
// SimilarTo returns up to `limit` stored vectors closest to the query
|
||||
// under cosine distance, ordered by ascending distance (most similar
|
||||
// first). Vectors whose length differs from the query are skipped — a
|
||||
// dimension mismatch can't be meaningfully scored.
|
||||
func (s *Store) SimilarTo(vec []float32, limit int) ([]graph.VectorHit, error) {
|
||||
if limit <= 0 || len(vec) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
qNorm := norm(vec)
|
||||
if qNorm == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
rows, err := s.db.Query(`SELECT node_id, vec FROM vectors`)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
// Max-heap keyed on distance: the root is the *worst* kept hit, so a
|
||||
// candidate better than the root evicts it. This keeps the heap at
|
||||
// `limit` and yields an exact top-k.
|
||||
h := &hitHeap{}
|
||||
for rows.Next() {
|
||||
var id string
|
||||
var blob []byte
|
||||
if err := rows.Scan(&id, &blob); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
cand := decodeVec(blob)
|
||||
if len(cand) != len(vec) {
|
||||
continue
|
||||
}
|
||||
cNorm := norm(cand)
|
||||
if cNorm == 0 {
|
||||
continue
|
||||
}
|
||||
dist := cosineDistance(vec, cand, qNorm, cNorm)
|
||||
|
||||
if h.Len() < limit {
|
||||
heap.Push(h, graph.VectorHit{NodeID: id, Distance: dist})
|
||||
} else if dist < (*h)[0].Distance {
|
||||
(*h)[0] = graph.VectorHit{NodeID: id, Distance: dist}
|
||||
heap.Fix(h, 0)
|
||||
}
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Drain the max-heap (largest distance first) then reverse so the
|
||||
// result is ascending by distance (most similar first).
|
||||
out := make([]graph.VectorHit, h.Len())
|
||||
for i := len(out) - 1; i >= 0; i-- {
|
||||
out[i] = heap.Pop(h).(graph.VectorHit)
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// norm returns the Euclidean norm (L2) of v as a float64.
|
||||
func norm(v []float32) float64 {
|
||||
var sum float64
|
||||
for _, f := range v {
|
||||
d := float64(f)
|
||||
sum += d * d
|
||||
}
|
||||
return math.Sqrt(sum)
|
||||
}
|
||||
|
||||
// cosineDistance returns 1 - cosine_similarity(a, b), given precomputed
|
||||
// norms. Lower = more similar; identical direction → ~0, orthogonal → 1,
|
||||
// opposite → 2. a and b are assumed equal length and non-zero norm.
|
||||
func cosineDistance(a, b []float32, aNorm, bNorm float64) float64 {
|
||||
var dot float64
|
||||
for i := range a {
|
||||
dot += float64(a[i]) * float64(b[i])
|
||||
}
|
||||
sim := dot / (aNorm * bNorm)
|
||||
// Guard against tiny floating-point overshoot past ±1.
|
||||
if sim > 1 {
|
||||
sim = 1
|
||||
} else if sim < -1 {
|
||||
sim = -1
|
||||
}
|
||||
return 1 - sim
|
||||
}
|
||||
|
||||
// hitHeap is a max-heap of VectorHit ordered by Distance: Less reports
|
||||
// the *larger* distance as "less" so the root is the worst-kept hit.
|
||||
type hitHeap []graph.VectorHit
|
||||
|
||||
func (h hitHeap) Len() int { return len(h) }
|
||||
func (h hitHeap) Less(i, j int) bool { return h[i].Distance > h[j].Distance }
|
||||
func (h hitHeap) Swap(i, j int) { h[i], h[j] = h[j], h[i] }
|
||||
func (h *hitHeap) Push(x any) { *h = append(*h, x.(graph.VectorHit)) }
|
||||
func (h *hitHeap) Pop() any {
|
||||
old := *h
|
||||
n := len(old)
|
||||
it := old[n-1]
|
||||
*h = old[:n-1]
|
||||
return it
|
||||
}
|
||||
@@ -0,0 +1,340 @@
|
||||
package store_sqlite_test
|
||||
|
||||
import (
|
||||
"math"
|
||||
"math/rand"
|
||||
"path/filepath"
|
||||
"reflect"
|
||||
"sort"
|
||||
"testing"
|
||||
|
||||
"github.com/zzet/gortex/internal/graph"
|
||||
"github.com/zzet/gortex/internal/graph/store_sqlite"
|
||||
)
|
||||
|
||||
// openTestStore opens a fresh on-disk SQLite store in a temp dir and
|
||||
// registers Close as cleanup. (modernc.org/sqlite's ":memory:" gives
|
||||
// each pooled connection its OWN private database, so the conformance
|
||||
// suite — and these tests — use an on-disk file shared across the pool.)
|
||||
func openTestStore(t *testing.T) *store_sqlite.Store {
|
||||
t.Helper()
|
||||
s, err := store_sqlite.Open(filepath.Join(t.TempDir(), "test.sqlite"))
|
||||
if err != nil {
|
||||
t.Fatalf("Open: %v", err)
|
||||
}
|
||||
t.Cleanup(func() { _ = s.Close() })
|
||||
return s
|
||||
}
|
||||
|
||||
// --- FileMtime persistence -------------------------------------------
|
||||
|
||||
func TestSQLiteFileMtimeRoundTrip(t *testing.T) {
|
||||
s := openTestStore(t)
|
||||
|
||||
// Single-row writes.
|
||||
if err := s.SetFileMtime("repoA", "a/one.go", 100); err != nil {
|
||||
t.Fatalf("SetFileMtime: %v", err)
|
||||
}
|
||||
if err := s.SetFileMtime("repoA", "a/two.go", 200); err != nil {
|
||||
t.Fatalf("SetFileMtime: %v", err)
|
||||
}
|
||||
|
||||
// Batch write (includes an overwrite of an existing key).
|
||||
batch := map[string]int64{
|
||||
"a/two.go": 250, // overwrite
|
||||
"a/three.go": 300,
|
||||
"a/four.go": 400,
|
||||
}
|
||||
if err := s.BulkSetFileMtimes("repoA", batch); err != nil {
|
||||
t.Fatalf("BulkSetFileMtimes: %v", err)
|
||||
}
|
||||
|
||||
want := map[string]int64{
|
||||
"a/one.go": 100,
|
||||
"a/two.go": 250,
|
||||
"a/three.go": 300,
|
||||
"a/four.go": 400,
|
||||
}
|
||||
|
||||
got, err := s.FileMtimes("repoA")
|
||||
if err != nil {
|
||||
t.Fatalf("FileMtimes: %v", err)
|
||||
}
|
||||
if !reflect.DeepEqual(got, want) {
|
||||
t.Fatalf("FileMtimes(repoA) = %v, want %v", got, want)
|
||||
}
|
||||
|
||||
// LoadFileMtimes (the interface method) must agree.
|
||||
if loaded := s.LoadFileMtimes("repoA"); !reflect.DeepEqual(loaded, want) {
|
||||
t.Fatalf("LoadFileMtimes(repoA) = %v, want %v", loaded, want)
|
||||
}
|
||||
|
||||
// Repo isolation: a different prefix is unaffected.
|
||||
if err := s.SetFileMtime("repoB", "b/x.go", 999); err != nil {
|
||||
t.Fatalf("SetFileMtime repoB: %v", err)
|
||||
}
|
||||
if got, _ := s.FileMtimes("repoA"); !reflect.DeepEqual(got, want) {
|
||||
t.Fatalf("repoA changed after repoB write: %v", got)
|
||||
}
|
||||
|
||||
// Unknown repo: FileMtimes returns an empty (non-nil) map;
|
||||
// LoadFileMtimes returns nil (the "no data" signal).
|
||||
empty, err := s.FileMtimes("nope")
|
||||
if err != nil {
|
||||
t.Fatalf("FileMtimes(unknown): %v", err)
|
||||
}
|
||||
if len(empty) != 0 {
|
||||
t.Fatalf("FileMtimes(unknown) = %v, want empty", empty)
|
||||
}
|
||||
if loaded := s.LoadFileMtimes("nope"); loaded != nil {
|
||||
t.Fatalf("LoadFileMtimes(unknown) = %v, want nil", loaded)
|
||||
}
|
||||
|
||||
// Empty batch is a no-op.
|
||||
if err := s.BulkSetFileMtimes("repoA", nil); err != nil {
|
||||
t.Fatalf("BulkSetFileMtimes(nil): %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// --- Vector search ---------------------------------------------------
|
||||
|
||||
// bruteForceCosine ranks corpus against query the long way (exact cosine
|
||||
// distance, ascending) so the test verifies SimilarTo independently of
|
||||
// the implementation under test.
|
||||
func bruteForceCosine(query []float32, corpus map[string][]float32, k int) []string {
|
||||
type sc struct {
|
||||
id string
|
||||
dist float64
|
||||
}
|
||||
scored := make([]sc, 0, len(corpus))
|
||||
qn := l2(query)
|
||||
for id, v := range corpus {
|
||||
vn := l2(v)
|
||||
if qn == 0 || vn == 0 {
|
||||
continue
|
||||
}
|
||||
var dot float64
|
||||
for i := range query {
|
||||
dot += float64(query[i]) * float64(v[i])
|
||||
}
|
||||
scored = append(scored, sc{id: id, dist: 1 - dot/(qn*vn)})
|
||||
}
|
||||
sort.Slice(scored, func(i, j int) bool {
|
||||
if scored[i].dist == scored[j].dist {
|
||||
return scored[i].id < scored[j].id // stable tie-break
|
||||
}
|
||||
return scored[i].dist < scored[j].dist
|
||||
})
|
||||
out := make([]string, 0, k)
|
||||
for i := 0; i < k && i < len(scored); i++ {
|
||||
out = append(out, scored[i].id)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func l2(v []float32) float64 {
|
||||
var s float64
|
||||
for _, f := range v {
|
||||
s += float64(f) * float64(f)
|
||||
}
|
||||
return math.Sqrt(s)
|
||||
}
|
||||
|
||||
func TestSQLiteVectorSimilarTo(t *testing.T) {
|
||||
s := openTestStore(t)
|
||||
|
||||
const (
|
||||
n = 50
|
||||
dims = 16
|
||||
)
|
||||
rng := rand.New(rand.NewSource(42))
|
||||
|
||||
corpus := make(map[string][]float32, n)
|
||||
items := make([]graph.VectorItem, 0, n)
|
||||
var ids []string
|
||||
for i := 0; i < n; i++ {
|
||||
id := nodeID(i)
|
||||
ids = append(ids, id)
|
||||
v := make([]float32, dims)
|
||||
for d := 0; d < dims; d++ {
|
||||
v[d] = float32(rng.NormFloat64())
|
||||
}
|
||||
corpus[id] = v
|
||||
items = append(items, graph.VectorItem{NodeID: id, Vec: v})
|
||||
}
|
||||
|
||||
if err := s.BulkUpsertEmbeddings(items); err != nil {
|
||||
t.Fatalf("BulkUpsertEmbeddings: %v", err)
|
||||
}
|
||||
if err := s.BuildVectorIndex(dims); err != nil {
|
||||
t.Fatalf("BuildVectorIndex: %v", err)
|
||||
}
|
||||
|
||||
// Query == a stored vector → it must rank first at distance ~0.
|
||||
queryID := ids[7]
|
||||
query := corpus[queryID]
|
||||
|
||||
hits, err := s.SimilarTo(query, 5)
|
||||
if err != nil {
|
||||
t.Fatalf("SimilarTo: %v", err)
|
||||
}
|
||||
if len(hits) != 5 {
|
||||
t.Fatalf("SimilarTo returned %d hits, want 5", len(hits))
|
||||
}
|
||||
if hits[0].NodeID != queryID {
|
||||
t.Fatalf("top hit = %q, want the query vector %q", hits[0].NodeID, queryID)
|
||||
}
|
||||
if hits[0].Distance > 1e-6 {
|
||||
t.Fatalf("top hit distance = %g, want ~0", hits[0].Distance)
|
||||
}
|
||||
|
||||
// Distances must be ascending.
|
||||
for i := 1; i < len(hits); i++ {
|
||||
if hits[i].Distance < hits[i-1].Distance {
|
||||
t.Fatalf("hits not ascending by distance: %v", hits)
|
||||
}
|
||||
}
|
||||
|
||||
// Independent brute-force ranking must match the returned top-5 ids.
|
||||
want := bruteForceCosine(query, corpus, 5)
|
||||
gotIDs := make([]string, len(hits))
|
||||
for i, h := range hits {
|
||||
gotIDs[i] = h.NodeID
|
||||
}
|
||||
if !reflect.DeepEqual(gotIDs, want) {
|
||||
t.Fatalf("SimilarTo top-5 = %v, brute-force = %v", gotIDs, want)
|
||||
}
|
||||
|
||||
// Single-add path: a new vector identical to ids[3]'s should be
|
||||
// retrievable and rank at distance ~0 for its own query.
|
||||
extra := make([]float32, dims)
|
||||
copy(extra, corpus[ids[3]])
|
||||
if err := s.UpsertEmbedding("extra::node", extra); err != nil {
|
||||
t.Fatalf("UpsertEmbedding: %v", err)
|
||||
}
|
||||
exHits, err := s.SimilarTo(extra, 3)
|
||||
if err != nil {
|
||||
t.Fatalf("SimilarTo (extra): %v", err)
|
||||
}
|
||||
if len(exHits) == 0 {
|
||||
t.Fatalf("SimilarTo(extra) returned nothing")
|
||||
}
|
||||
// Either the original ids[3] or the new extra::node (both identical
|
||||
// vectors, distance ~0) may sort first; the new one must be present
|
||||
// at distance ~0.
|
||||
foundExtra := false
|
||||
for _, h := range exHits {
|
||||
if h.NodeID == "extra::node" {
|
||||
foundExtra = true
|
||||
if h.Distance > 1e-6 {
|
||||
t.Fatalf("extra::node distance = %g, want ~0", h.Distance)
|
||||
}
|
||||
}
|
||||
}
|
||||
if !foundExtra {
|
||||
t.Fatalf("UpsertEmbedding'd vector not found in SimilarTo results: %v", exHits)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSQLiteVectorPersistence(t *testing.T) {
|
||||
path := filepath.Join(t.TempDir(), "v.sqlite")
|
||||
|
||||
corpus := map[string][]float32{
|
||||
"n::1": {1, 0, 0, 0, 0, 0, 0, 0},
|
||||
"n::2": {0, 1, 0, 0, 0, 0, 0, 0},
|
||||
"n::3": {0, 0, 1, 0, 0, 0, 0, 0},
|
||||
}
|
||||
|
||||
// First session: write and close.
|
||||
{
|
||||
s, err := store_sqlite.Open(path)
|
||||
if err != nil {
|
||||
t.Fatalf("open: %v", err)
|
||||
}
|
||||
items := make([]graph.VectorItem, 0, len(corpus))
|
||||
for id, v := range corpus {
|
||||
items = append(items, graph.VectorItem{NodeID: id, Vec: v})
|
||||
}
|
||||
if err := s.BulkUpsertEmbeddings(items); err != nil {
|
||||
t.Fatalf("BulkUpsertEmbeddings: %v", err)
|
||||
}
|
||||
if err := s.Close(); err != nil {
|
||||
t.Fatalf("close: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// Second session: reopen, vectors must still be queryable.
|
||||
{
|
||||
s, err := store_sqlite.Open(path)
|
||||
if err != nil {
|
||||
t.Fatalf("reopen: %v", err)
|
||||
}
|
||||
t.Cleanup(func() { _ = s.Close() })
|
||||
|
||||
query := []float32{1, 0, 0, 0, 0, 0, 0, 0}
|
||||
hits, err := s.SimilarTo(query, 3)
|
||||
if err != nil {
|
||||
t.Fatalf("SimilarTo after reopen: %v", err)
|
||||
}
|
||||
if len(hits) != 3 {
|
||||
t.Fatalf("after reopen got %d hits, want 3 (persistence failed)", len(hits))
|
||||
}
|
||||
if hits[0].NodeID != "n::1" {
|
||||
t.Fatalf("after reopen top hit = %q, want n::1", hits[0].NodeID)
|
||||
}
|
||||
if hits[0].Distance > 1e-6 {
|
||||
t.Fatalf("after reopen top distance = %g, want ~0", hits[0].Distance)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestSQLiteGetEmbeddings(t *testing.T) {
|
||||
s := openTestStore(t)
|
||||
|
||||
corpus := map[string][]float32{
|
||||
"n::1": {1, 0, 0, 0},
|
||||
"n::2": {0, 1, 0, 0},
|
||||
"n::3": {0, 0, 1, 0},
|
||||
}
|
||||
items := make([]graph.VectorItem, 0, len(corpus))
|
||||
for id, v := range corpus {
|
||||
items = append(items, graph.VectorItem{NodeID: id, Vec: v})
|
||||
}
|
||||
if err := s.BulkUpsertEmbeddings(items); err != nil {
|
||||
t.Fatalf("BulkUpsertEmbeddings: %v", err)
|
||||
}
|
||||
|
||||
// Batch read of a mix of present + absent IDs: present IDs come back
|
||||
// with their exact stored vectors; absent IDs are simply omitted.
|
||||
got := s.GetEmbeddings([]string{"n::1", "n::3", "missing::x"})
|
||||
if len(got) != 2 {
|
||||
t.Fatalf("GetEmbeddings returned %d vectors, want 2 (absent id must be omitted)", len(got))
|
||||
}
|
||||
if !reflect.DeepEqual(got["n::1"], corpus["n::1"]) {
|
||||
t.Fatalf("GetEmbeddings[n::1] = %v, want %v", got["n::1"], corpus["n::1"])
|
||||
}
|
||||
if !reflect.DeepEqual(got["n::3"], corpus["n::3"]) {
|
||||
t.Fatalf("GetEmbeddings[n::3] = %v, want %v", got["n::3"], corpus["n::3"])
|
||||
}
|
||||
if _, present := got["missing::x"]; present {
|
||||
t.Fatalf("GetEmbeddings must omit ids with no stored vector")
|
||||
}
|
||||
|
||||
// Empty input yields a non-nil empty map, never an error or panic.
|
||||
if empty := s.GetEmbeddings(nil); empty == nil || len(empty) != 0 {
|
||||
t.Fatalf("GetEmbeddings(nil) = %v, want empty non-nil map", empty)
|
||||
}
|
||||
}
|
||||
|
||||
func nodeID(i int) string {
|
||||
const digits = "0123456789"
|
||||
if i == 0 {
|
||||
return "node::0"
|
||||
}
|
||||
var b []byte
|
||||
for i > 0 {
|
||||
b = append([]byte{digits[i%10]}, b...)
|
||||
i /= 10
|
||||
}
|
||||
return "node::" + string(b)
|
||||
}
|
||||
@@ -0,0 +1,92 @@
|
||||
package store_sqlite
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/zzet/gortex/internal/graph"
|
||||
)
|
||||
|
||||
// TestCheckpointWALBoundsFileAndPreservesData writes enough rows to push pages
|
||||
// into the -wal file, then forces a TRUNCATE checkpoint and asserts the WAL is
|
||||
// drained (bounded well under journal_size_limit) without losing any data.
|
||||
func TestCheckpointWALBoundsFileAndPreservesData(t *testing.T) {
|
||||
path := filepath.Join(t.TempDir(), "wal.sqlite")
|
||||
s, err := Open(path)
|
||||
if err != nil {
|
||||
t.Fatalf("open: %v", err)
|
||||
}
|
||||
t.Cleanup(func() { _ = s.Close() })
|
||||
|
||||
// On-disk stores must arm the background checkpoint loop.
|
||||
if s.stopCheckpoint == nil || s.checkpointDone == nil {
|
||||
t.Fatal("on-disk store did not start the WAL-checkpoint loop")
|
||||
}
|
||||
|
||||
const n = 4000
|
||||
nodes := make([]*graph.Node, 0, n)
|
||||
for i := range n {
|
||||
nodes = append(nodes, &graph.Node{
|
||||
ID: fmt.Sprintf("pkg/f.go::Sym%d", i),
|
||||
Kind: graph.KindFunction,
|
||||
Name: fmt.Sprintf("Sym%d", i),
|
||||
FilePath: "pkg/f.go",
|
||||
Language: "go",
|
||||
})
|
||||
}
|
||||
s.AddBatch(nodes, nil)
|
||||
|
||||
if err := s.CheckpointWAL(); err != nil {
|
||||
t.Fatalf("checkpoint: %v", err)
|
||||
}
|
||||
|
||||
// journal_size_limit caps the WAL at 64 MiB; after a TRUNCATE checkpoint
|
||||
// with no concurrent reader it should be far smaller still.
|
||||
if fi, err := os.Stat(path + "-wal"); err == nil && fi.Size() > 64<<20 {
|
||||
t.Fatalf("wal not bounded after checkpoint: %d bytes", fi.Size())
|
||||
}
|
||||
|
||||
if got := s.NodeCount(); got != n {
|
||||
t.Fatalf("checkpoint lost data: NodeCount = %d, want %d", got, n)
|
||||
}
|
||||
}
|
||||
|
||||
// TestCloseStopsCheckpointLoop verifies Close signals the loop and waits for it
|
||||
// to exit, and that calling Close twice does not panic on the stop channel.
|
||||
func TestCloseStopsCheckpointLoop(t *testing.T) {
|
||||
path := filepath.Join(t.TempDir(), "wal.sqlite")
|
||||
s, err := Open(path)
|
||||
if err != nil {
|
||||
t.Fatalf("open: %v", err)
|
||||
}
|
||||
done := s.checkpointDone
|
||||
|
||||
if err := s.Close(); err != nil {
|
||||
t.Fatalf("close: %v", err)
|
||||
}
|
||||
select {
|
||||
case <-done:
|
||||
case <-time.After(2 * time.Second):
|
||||
t.Fatal("checkpoint loop did not stop within 2s of Close")
|
||||
}
|
||||
|
||||
// stopCheckpointLoop is guarded by sync.Once, so a second stop is a no-op
|
||||
// rather than a close-of-closed-channel panic.
|
||||
s.stopCheckpointLoop()
|
||||
}
|
||||
|
||||
// TestInMemoryStoreSkipsCheckpointLoop confirms ":memory:" stores, which have
|
||||
// no WAL, never spawn the checkpoint goroutine.
|
||||
func TestInMemoryStoreSkipsCheckpointLoop(t *testing.T) {
|
||||
s, err := Open(":memory:")
|
||||
if err != nil {
|
||||
t.Fatalf("open: %v", err)
|
||||
}
|
||||
t.Cleanup(func() { _ = s.Close() })
|
||||
if s.stopCheckpoint != nil || s.checkpointDone != nil {
|
||||
t.Fatal("in-memory store should not arm the WAL-checkpoint loop")
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user