chore: import upstream snapshot with attribution
CI / benchmark (push) Has been skipped
install-script / posix-syntax (push) Successful in 6m1s
CI / build-onnx (push) Failing after 6m43s
init-smoke / dry-run (push) Failing after 15m57s
security / govulncheck (push) Has been cancelled
security / trivy-fs (push) Has been cancelled
CI / test (1.26, ubuntu-latest) (push) Has been cancelled
Scorecard supply-chain security / Scorecard analysis (push) Has been cancelled
CI / test (1.26, macos-latest) (push) Has been cancelled
CI / build-windows (push) Has been cancelled
CI / lint (push) Has been cancelled
install-script / powershell-syntax (push) Has been cancelled
install-script / install (macos-14) (push) Has been cancelled
install-script / install (ubuntu-latest) (push) Has been cancelled

This commit is contained in:
wehub-resource-sync
2026-07-13 12:33:42 +08:00
commit a06f331eb8
3186 changed files with 689843 additions and 0 deletions
+113
View File
@@ -0,0 +1,113 @@
package persistence
import (
"compress/gzip"
"encoding/gob"
"fmt"
"os"
"path/filepath"
"time"
)
const (
comboFile = "combo.gob.gz"
maxComboQueries = 2000
maxComboEntriesPerQuery = 20
)
// ComboMatch records one (query → symbol) association. HitCount is how many
// times the agent picked this symbol following the same normalized query;
// LastUsed is a unix timestamp (seconds) for decay and reaping.
type ComboMatch struct {
SymbolID string
HitCount uint32
LastUsed int64
}
// ComboQuery holds all recorded matches for one normalized query string
// within a single repo. Ordered most-hit-first after any record.
type ComboQuery struct {
Query string
Matches []ComboMatch
}
// ComboStore is the persisted state of the query→symbol combo tracker for
// one repo. Separate file from feedback so each subsystem can evolve its
// schema independently.
type ComboStore struct {
Version string
RepoPath string
Queries []ComboQuery
}
// ComboDir returns the on-disk directory for combo storage. Shares the
// repo cache key with feedback so all repo-scoped state lives together.
func ComboDir(cacheDir, repoPath string) string {
return filepath.Join(cacheDir, RepoCacheKey(repoPath))
}
// LoadCombo reads the combo store from disk. Missing file is not an error.
func LoadCombo(dir string) (*ComboStore, error) {
path := filepath.Join(dir, comboFile)
f, err := os.Open(path)
if err != nil {
if os.IsNotExist(err) {
return &ComboStore{}, nil
}
return nil, fmt.Errorf("persistence: open combo: %w", err)
}
defer f.Close()
gz, err := gzip.NewReader(f)
if err != nil {
return nil, fmt.Errorf("persistence: gzip reader combo: %w", err)
}
defer gz.Close()
var store ComboStore
if err := gob.NewDecoder(gz).Decode(&store); err != nil {
return nil, fmt.Errorf("persistence: gob decode combo: %w", err)
}
return &store, nil
}
// SaveCombo writes the combo store to disk with gob+gzip compression. Trims
// the oldest-used queries if over cap so the file can't grow unboundedly on
// a long-running daemon.
func SaveCombo(dir string, store *ComboStore) error {
if len(store.Queries) > maxComboQueries {
// Keep the queries touched most recently by scanning their matches.
// Stable order (no sort) preserves the common case where callers
// already maintain MRU; only trigger a full scan when we'd otherwise
// drop unbounded.
trim := len(store.Queries) - maxComboQueries
store.Queries = store.Queries[trim:]
}
if err := os.MkdirAll(dir, 0o755); err != nil {
return fmt.Errorf("persistence: mkdir combo: %w", err)
}
path := filepath.Join(dir, comboFile)
f, err := os.Create(path)
if err != nil {
return fmt.Errorf("persistence: create combo: %w", err)
}
gz := gzip.NewWriter(f)
enc := gob.NewEncoder(gz)
if err := enc.Encode(store); err != nil {
_ = gz.Close()
_ = f.Close()
return fmt.Errorf("persistence: gob encode combo: %w", err)
}
if err := gz.Close(); err != nil {
_ = f.Close()
return fmt.Errorf("persistence: gzip close combo: %w", err)
}
return f.Close()
}
// MaxComboEntries returns the cap on matches per query. Exported so the
// manager can enforce the same limit in-memory before flushing.
func MaxComboEntries() int { return maxComboEntriesPerQuery }
// Now is overridable for tests; used when SaveCombo needs to timestamp.
var Now = func() time.Time { return time.Now() }
+125
View File
@@ -0,0 +1,125 @@
package persistence
import (
"compress/gzip"
"crypto/sha256"
"encoding/gob"
"encoding/hex"
"fmt"
"os"
"path/filepath"
"time"
"github.com/zzet/gortex/internal/pathkey"
)
const (
feedbackFile = "feedback.gob.gz"
maxFeedbackCap = 500
)
// FeedbackEntry records one agent feedback event about a context call.
type FeedbackEntry struct {
Timestamp time.Time
Task string // task description from the context call
Useful []string // symbol IDs the agent found useful
NotNeeded []string // symbol IDs returned but not needed
Missing []string // symbol IDs that should have been included
Source string // "smart_context" or "prefetch_context"
// Keywords is the task's keyword cluster (derived from Task at
// record time). Feedback is scored only against entries whose
// keyword cluster overlaps the querying task's, so a symbol marked
// useful for one task does not contaminate an unrelated one. A nil
// Keywords (legacy entry, or a keyword-less task) is treated as
// matching any query for backward compatibility.
Keywords []string
}
// FeedbackStore holds all feedback entries for a single repo.
// Persisted separately from the graph snapshot (repo-scoped, not commit-scoped).
type FeedbackStore struct {
Version string
RepoPath string
Entries []FeedbackEntry
}
// RepoCacheKey produces a filesystem-safe directory name from repo path alone
// (no commit hash). Used for data that persists across commits, like feedback.
//
// The path is folded to Unicode NFC before hashing so a repo whose path
// contains non-ASCII characters keys to the same directory whether the
// caller passes a decomposed (macOS) or precomposed (Linux / git) form.
func RepoCacheKey(repoPath string) string {
abs, err := filepath.Abs(repoPath)
if err != nil {
abs = repoPath
}
h := sha256.Sum256([]byte(pathkey.Normalize(abs)))
return hex.EncodeToString(h[:6]) + "_latest"
}
// FeedbackDir returns the full directory path for feedback storage.
func FeedbackDir(cacheDir, repoPath string) string {
return filepath.Join(cacheDir, RepoCacheKey(repoPath))
}
// LoadFeedback reads a feedback store from disk. Returns an empty store if
// the file does not exist (not an error — cold start is normal).
func LoadFeedback(dir string) (*FeedbackStore, error) {
path := filepath.Join(dir, feedbackFile)
f, err := os.Open(path)
if err != nil {
if os.IsNotExist(err) {
return &FeedbackStore{}, nil
}
return nil, fmt.Errorf("persistence: open feedback: %w", err)
}
defer f.Close()
gz, err := gzip.NewReader(f)
if err != nil {
return nil, fmt.Errorf("persistence: gzip reader: %w", err)
}
defer gz.Close()
var store FeedbackStore
if err := gob.NewDecoder(gz).Decode(&store); err != nil {
return nil, fmt.Errorf("persistence: gob decode feedback: %w", err)
}
return &store, nil
}
// SaveFeedback writes a feedback store to disk with gob+gzip compression.
// Entries are trimmed to maxFeedbackCap before writing (oldest removed).
func SaveFeedback(dir string, store *FeedbackStore) error {
// Trim oldest entries if over cap.
if len(store.Entries) > maxFeedbackCap {
store.Entries = store.Entries[len(store.Entries)-maxFeedbackCap:]
}
if err := os.MkdirAll(dir, 0o755); err != nil {
return fmt.Errorf("persistence: mkdir feedback: %w", err)
}
path := filepath.Join(dir, feedbackFile)
f, err := os.Create(path)
if err != nil {
return fmt.Errorf("persistence: create feedback: %w", err)
}
gz := gzip.NewWriter(f)
enc := gob.NewEncoder(gz)
if err := enc.Encode(store); err != nil {
_ = gz.Close()
_ = f.Close()
return fmt.Errorf("persistence: gob encode feedback: %w", err)
}
if err := gz.Close(); err != nil {
_ = f.Close()
return fmt.Errorf("persistence: gzip close feedback: %w", err)
}
return f.Close()
}
+106
View File
@@ -0,0 +1,106 @@
package persistence
import (
"testing"
"time"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func testFeedbackStore() *FeedbackStore {
return &FeedbackStore{
Version: "0.1.0-test",
RepoPath: "/tmp/test-repo",
Entries: []FeedbackEntry{
{
Timestamp: time.Now().Truncate(time.Second),
Task: "add new MCP tool",
Useful: []string{"server.go::Server", "tools_core.go::registerCoreTools"},
NotNeeded: []string{"types.go::NodeKind"},
Missing: []string{"tools_enhancements.go::registerEnhancementTools"},
Source: "smart_context",
},
{
Timestamp: time.Now().Add(-time.Hour).Truncate(time.Second),
Task: "fix bug in search",
Useful: []string{"search.go::Search"},
Source: "prefetch_context",
},
},
}
}
func TestFeedback_RoundTrip(t *testing.T) {
dir := t.TempDir()
store := testFeedbackStore()
require.NoError(t, SaveFeedback(dir, store))
loaded, err := LoadFeedback(dir)
require.NoError(t, err)
assert.Equal(t, store.Version, loaded.Version)
assert.Equal(t, store.RepoPath, loaded.RepoPath)
require.Len(t, loaded.Entries, 2)
assert.Equal(t, "add new MCP tool", loaded.Entries[0].Task)
assert.Equal(t, []string{"server.go::Server", "tools_core.go::registerCoreTools"}, loaded.Entries[0].Useful)
assert.Equal(t, []string{"types.go::NodeKind"}, loaded.Entries[0].NotNeeded)
assert.Equal(t, []string{"tools_enhancements.go::registerEnhancementTools"}, loaded.Entries[0].Missing)
assert.Equal(t, "smart_context", loaded.Entries[0].Source)
assert.Equal(t, "fix bug in search", loaded.Entries[1].Task)
assert.Equal(t, "prefetch_context", loaded.Entries[1].Source)
}
func TestFeedback_ColdStart(t *testing.T) {
dir := t.TempDir()
loaded, err := LoadFeedback(dir)
require.NoError(t, err)
assert.NotNil(t, loaded)
assert.Empty(t, loaded.Entries)
}
func TestFeedback_TrimOldEntries(t *testing.T) {
dir := t.TempDir()
store := &FeedbackStore{
Version: "0.1.0",
RepoPath: "/tmp/test",
}
// Create 510 entries.
for range 510 {
store.Entries = append(store.Entries, FeedbackEntry{
Timestamp: time.Now().Truncate(time.Second),
Task: "task",
Useful: []string{"sym"},
Source: "smart_context",
})
}
require.NoError(t, SaveFeedback(dir, store))
loaded, err := LoadFeedback(dir)
require.NoError(t, err)
assert.Len(t, loaded.Entries, 500)
}
func TestRepoCacheKey_Stable(t *testing.T) {
key1 := RepoCacheKey("/tmp/my-repo")
key2 := RepoCacheKey("/tmp/my-repo")
assert.Equal(t, key1, key2)
assert.Contains(t, key1, "_latest")
}
func TestRepoCacheKey_DifferentRepos(t *testing.T) {
key1 := RepoCacheKey("/tmp/repo-a")
key2 := RepoCacheKey("/tmp/repo-b")
assert.NotEqual(t, key1, key2)
}
func TestFeedbackDir(t *testing.T) {
dir := FeedbackDir("/home/user/.cache/gortex", "/tmp/my-repo")
assert.Contains(t, dir, "_latest")
assert.Contains(t, dir, ".cache/gortex")
}
+212
View File
@@ -0,0 +1,212 @@
package persistence
import (
"compress/gzip"
"encoding/gob"
"fmt"
"os"
"path/filepath"
"strconv"
"strings"
"github.com/gofrs/flock"
"github.com/zzet/gortex/internal/platform"
)
func init() {
// Register concrete types that appear in Node.Meta / Edge.Meta map[string]any.
gob.Register(map[string]any{})
gob.Register([]any{})
gob.Register([]string{})
gob.Register([]int{})
gob.Register([]map[string]string{})
gob.Register([]map[string]any{})
}
const (
snapshotFile = "snapshot.gob.gz"
versionFile = ".version"
extractionVersionFile = ".extraction_version"
)
// FileStore persists snapshots as gob+gzip files in a directory hierarchy.
// Layout: {dir}/{cacheKey}/snapshot.gob.gz + .version
type FileStore struct {
dir string
version string
}
// NewFileStore creates a file-based persistence store.
// If dir is empty, defaults to the Gortex cache dir (~/.gortex/cache/,
// or the $XDG_CACHE_HOME equivalent when that variable is set).
func NewFileStore(dir, version string) (*FileStore, error) {
if dir == "" {
dir = platform.CacheDir()
}
return &FileStore{dir: dir, version: version}, nil
}
func (fs *FileStore) entryDir(repoPath, branch, commitHash string) string {
return filepath.Join(fs.dir, CacheKey(repoPath, branch, commitHash))
}
// lockPath is the advisory-lock file for one snapshot entry. It is a
// sibling of the entry directory ({dir}/{cacheKey}.lock), deliberately
// outside it so the lock survives the os.RemoveAll a writer runs against
// the entry directory on Save/Evict.
func (fs *FileStore) lockPath(repoPath, branch, commitHash string) string {
return fs.entryDir(repoPath, branch, commitHash) + ".lock"
}
// acquireWrite takes an exclusive cross-process advisory lock for one
// snapshot entry. A second gortex process writing the same snapshot
// blocks here instead of racing the RemoveAll/MkdirAll/write sequence,
// which would otherwise leave a torn or empty index on disk.
func (fs *FileStore) acquireWrite(repoPath, branch, commitHash string) (*flock.Flock, error) {
if err := os.MkdirAll(fs.dir, 0o755); err != nil {
return nil, fmt.Errorf("persistence: mkdir cache dir: %w", err)
}
fl := flock.New(fs.lockPath(repoPath, branch, commitHash))
if err := fl.Lock(); err != nil {
return nil, fmt.Errorf("persistence: acquire index write lock: %w", err)
}
return fl, nil
}
// acquireRead takes a shared cross-process advisory lock for one snapshot
// entry, so a reader waits out an in-progress write instead of decoding a
// half-written snapshot.
func (fs *FileStore) acquireRead(repoPath, branch, commitHash string) (*flock.Flock, error) {
if err := os.MkdirAll(fs.dir, 0o755); err != nil {
return nil, fmt.Errorf("persistence: mkdir cache dir: %w", err)
}
fl := flock.New(fs.lockPath(repoPath, branch, commitHash))
if err := fl.RLock(); err != nil {
return nil, fmt.Errorf("persistence: acquire index read lock: %w", err)
}
return fl, nil
}
func (fs *FileStore) Check(repoPath, branch, commitHash string) bool {
dir := fs.entryDir(repoPath, branch, commitHash)
info, err := os.Stat(dir)
if err != nil || !info.IsDir() {
return false
}
_, err = os.Stat(filepath.Join(dir, versionFile))
return err == nil
}
func (fs *FileStore) Validate(repoPath, branch, commitHash string) bool {
dir := fs.entryDir(repoPath, branch, commitHash)
// Extraction-version gate: a warm snapshot is reusable across binary
// releases that produce the SAME extraction output, so a no-op version
// bump no longer forces a full cold rebuild.
if data, err := os.ReadFile(filepath.Join(dir, extractionVersionFile)); err == nil {
v, perr := strconv.Atoi(strings.TrimSpace(string(data)))
return perr == nil && v == CurrentExtractionVersion
}
// Legacy slot with no extraction-version file: fall back to the exact
// binary-version match so pre-existing snapshots still validate (and get
// rewritten with an extraction-version marker on the next save).
data, err := os.ReadFile(filepath.Join(dir, versionFile))
if err != nil {
return false
}
return strings.TrimSpace(string(data)) == fs.version
}
func (fs *FileStore) Load(repoPath, branch, commitHash string) (*Snapshot, error) {
fl, err := fs.acquireRead(repoPath, branch, commitHash)
if err != nil {
return nil, err
}
defer func() { _ = fl.Unlock() }()
dir := fs.entryDir(repoPath, branch, commitHash)
f, err := os.Open(filepath.Join(dir, snapshotFile))
if err != nil {
if os.IsNotExist(err) {
return nil, ErrNotFound
}
return nil, fmt.Errorf("persistence: open snapshot: %w", err)
}
defer f.Close()
gz, err := gzip.NewReader(f)
if err != nil {
return nil, fmt.Errorf("persistence: gzip reader: %w", err)
}
defer gz.Close()
var snap Snapshot
if err := gob.NewDecoder(gz).Decode(&snap); err != nil {
return nil, fmt.Errorf("persistence: gob decode: %w", err)
}
return &snap, nil
}
func (fs *FileStore) Save(snap *Snapshot) error {
fl, err := fs.acquireWrite(snap.RepoPath, snap.Branch, snap.CommitHash)
if err != nil {
return err
}
defer func() { _ = fl.Unlock() }()
dir := fs.entryDir(snap.RepoPath, snap.Branch, snap.CommitHash)
// Remove old entry if it exists.
_ = os.RemoveAll(dir)
if err := os.MkdirAll(dir, 0o755); err != nil {
return fmt.Errorf("persistence: mkdir: %w", err)
}
// Write snapshot.
f, err := os.Create(filepath.Join(dir, snapshotFile))
if err != nil {
return fmt.Errorf("persistence: create snapshot: %w", err)
}
gz := gzip.NewWriter(f)
enc := gob.NewEncoder(gz)
if err := enc.Encode(snap); err != nil {
_ = gz.Close()
_ = f.Close()
return fmt.Errorf("persistence: gob encode: %w", err)
}
if err := gz.Close(); err != nil {
_ = f.Close()
return fmt.Errorf("persistence: gzip close: %w", err)
}
if err := f.Close(); err != nil {
return fmt.Errorf("persistence: file close: %w", err)
}
// Write version file (binary version — kept for display / diagnostics).
if err := os.WriteFile(filepath.Join(dir, versionFile), []byte(fs.version), 0o644); err != nil {
return fmt.Errorf("persistence: write version: %w", err)
}
// Write the extraction-version marker — the value Validate gates reuse on.
if err := os.WriteFile(filepath.Join(dir, extractionVersionFile), []byte(strconv.Itoa(CurrentExtractionVersion)), 0o644); err != nil {
return fmt.Errorf("persistence: write extraction version: %w", err)
}
return nil
}
func (fs *FileStore) Evict(repoPath, branch, commitHash string) error {
fl, err := fs.acquireWrite(repoPath, branch, commitHash)
if err != nil {
return err
}
defer func() { _ = fl.Unlock() }()
return os.RemoveAll(fs.entryDir(repoPath, branch, commitHash))
}
func (fs *FileStore) Close() error { return nil }
+365
View File
@@ -0,0 +1,365 @@
package persistence
import (
"fmt"
"os"
"path/filepath"
"sync"
"testing"
"time"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/zzet/gortex/internal/graph"
)
func testSnapshot() *Snapshot {
return &Snapshot{
Version: "0.1.0-test",
RepoPath: "/tmp/test-repo",
CommitHash: "abc123def456",
Branch: "main",
IndexedAt: time.Now().Truncate(time.Second),
Nodes: []*graph.Node{
{
ID: "main.go::Foo", Kind: graph.KindFunction, Name: "Foo",
FilePath: "main.go", StartLine: 1, EndLine: 5, Language: "go",
Meta: map[string]any{"signature": "func Foo(x int) error"},
},
{
ID: "main.go::Bar", Kind: graph.KindMethod, Name: "Bar",
FilePath: "main.go", StartLine: 7, EndLine: 12, Language: "go",
Meta: map[string]any{"receiver": "Server", "signature": "func (s *Server) Bar()"},
},
},
Edges: []*graph.Edge{
{
From: "main.go::Foo", To: "main.go::Bar", Kind: graph.EdgeCalls,
FilePath: "main.go", Line: 3, Confidence: 0.95,
Meta: map[string]any{"receiver_type": "Server"},
},
},
FileMtimes: map[string]int64{
"main.go": 1700000000000000000,
"util.go": 1700000001000000000,
},
}
}
func TestFileStore_RoundTrip(t *testing.T) {
dir := t.TempDir()
fs, err := NewFileStore(dir, "0.1.0-test")
require.NoError(t, err)
snap := testSnapshot()
require.NoError(t, fs.Save(snap))
assert.True(t, fs.Check(snap.RepoPath, snap.Branch, snap.CommitHash))
assert.True(t, fs.Validate(snap.RepoPath, snap.Branch, snap.CommitHash))
loaded, err := fs.Load(snap.RepoPath, snap.Branch, snap.CommitHash)
require.NoError(t, err)
assert.Equal(t, snap.Version, loaded.Version)
assert.Equal(t, snap.RepoPath, loaded.RepoPath)
assert.Equal(t, snap.CommitHash, loaded.CommitHash)
assert.Equal(t, snap.Branch, loaded.Branch)
assert.Equal(t, snap.IndexedAt, loaded.IndexedAt)
require.Len(t, loaded.Nodes, 2)
assert.Equal(t, "main.go::Foo", loaded.Nodes[0].ID)
assert.Equal(t, "Foo", loaded.Nodes[0].Name)
assert.Equal(t, "func Foo(x int) error", loaded.Nodes[0].Meta["signature"])
assert.Equal(t, "Server", loaded.Nodes[1].Meta["receiver"])
require.Len(t, loaded.Edges, 1)
assert.Equal(t, "main.go::Foo", loaded.Edges[0].From)
assert.Equal(t, "main.go::Bar", loaded.Edges[0].To)
assert.Equal(t, 0.95, loaded.Edges[0].Confidence)
assert.Equal(t, "Server", loaded.Edges[0].Meta["receiver_type"])
assert.Equal(t, snap.FileMtimes, loaded.FileMtimes)
}
func TestFileStore_Validate_VersionMismatch(t *testing.T) {
dir := t.TempDir()
fsV1, err := NewFileStore(dir, "0.1.0")
require.NoError(t, err)
snap := testSnapshot()
snap.Version = "0.1.0"
require.NoError(t, fsV1.Save(snap))
// Same version validates.
assert.True(t, fsV1.Validate(snap.RepoPath, snap.Branch, snap.CommitHash))
// A different BINARY version now REUSES the slot — the extraction version
// (not the binary string) gates reuse, so a no-op release skips the rebuild.
fsV2, err := NewFileStore(dir, "0.2.0")
require.NoError(t, err)
assert.True(t, fsV2.Validate(snap.RepoPath, snap.Branch, snap.CommitHash))
// A LEGACY slot with no extraction-version marker falls back to the exact
// binary-version match — so it correctly invalidates across the bump.
entry := fsV2.entryDir(snap.RepoPath, snap.Branch, snap.CommitHash)
require.NoError(t, os.Remove(filepath.Join(entry, extractionVersionFile)))
assert.False(t, fsV2.Validate(snap.RepoPath, snap.Branch, snap.CommitHash))
}
func TestFileStore_Evict(t *testing.T) {
dir := t.TempDir()
fs, err := NewFileStore(dir, "0.1.0")
require.NoError(t, err)
snap := testSnapshot()
require.NoError(t, fs.Save(snap))
assert.True(t, fs.Check(snap.RepoPath, snap.Branch, snap.CommitHash))
require.NoError(t, fs.Evict(snap.RepoPath, snap.Branch, snap.CommitHash))
assert.False(t, fs.Check(snap.RepoPath, snap.Branch, snap.CommitHash))
}
func TestFileStore_Load_NotFound(t *testing.T) {
dir := t.TempDir()
fs, err := NewFileStore(dir, "0.1.0")
require.NoError(t, err)
_, err = fs.Load("/nonexistent", "main", "abc123")
assert.ErrorIs(t, err, ErrNotFound)
}
func TestFileStore_MetaWithSliceTypes(t *testing.T) {
dir := t.TempDir()
fs, err := NewFileStore(dir, "0.1.0")
require.NoError(t, err)
snap := &Snapshot{
Version: "0.1.0",
RepoPath: "/tmp/test",
CommitHash: "def789",
Branch: "main",
IndexedAt: time.Now().Truncate(time.Second),
Nodes: []*graph.Node{
{
ID: "iface.go::Reader", Kind: graph.KindInterface, Name: "Reader",
FilePath: "iface.go", Language: "go",
Meta: map[string]any{"methods": []string{"Read", "Close"}},
},
},
FileMtimes: map[string]int64{"iface.go": 1700000000},
}
require.NoError(t, fs.Save(snap))
loaded, err := fs.Load(snap.RepoPath, snap.Branch, snap.CommitHash)
require.NoError(t, err)
methods, ok := loaded.Nodes[0].Meta["methods"].([]string)
require.True(t, ok, "methods should deserialize as []string")
assert.Equal(t, []string{"Read", "Close"}, methods)
}
// TestFileStore_BranchKeyedSlots proves snapshots are keyed by
// (repo, branch): two branches of the same repo, even at the same
// commit, occupy distinct slots, so switching branches never clobbers
// the other branch's cached index.
func TestFileStore_BranchKeyedSlots(t *testing.T) {
dir := t.TempDir()
fs, err := NewFileStore(dir, "0.1.0-test")
require.NoError(t, err)
main := testSnapshot()
main.Branch = "main"
feature := testSnapshot()
feature.Branch = "feature/login"
feature.Nodes[0].Name = "FeatureFoo"
require.NoError(t, fs.Save(main))
require.NoError(t, fs.Save(feature))
gotMain, err := fs.Load(main.RepoPath, "main", main.CommitHash)
require.NoError(t, err)
assert.Equal(t, "Foo", gotMain.Nodes[0].Name)
gotFeature, err := fs.Load(feature.RepoPath, "feature/login", feature.CommitHash)
require.NoError(t, err)
assert.Equal(t, "FeatureFoo", gotFeature.Nodes[0].Name)
}
// TestFileStore_DetachedHeadKeyedByCommit checks the detached-HEAD
// fallback: with no branch the slot keys on the commit hash, so two
// checked-out commits keep separate snapshots.
func TestFileStore_DetachedHeadKeyedByCommit(t *testing.T) {
dir := t.TempDir()
fs, err := NewFileStore(dir, "0.1.0-test")
require.NoError(t, err)
a := testSnapshot()
a.Branch = ""
a.CommitHash = "aaaaaaaaaaaa"
b := testSnapshot()
b.Branch = ""
b.CommitHash = "bbbbbbbbbbbb"
require.NoError(t, fs.Save(a))
require.NoError(t, fs.Save(b))
assert.True(t, fs.Check(a.RepoPath, "", "aaaaaaaaaaaa"))
assert.True(t, fs.Check(b.RepoPath, "", "bbbbbbbbbbbb"))
}
func TestNopStore(t *testing.T) {
var s NopStore
assert.False(t, s.Check("x", "main", "y"))
_, err := s.Load("x", "main", "y")
assert.ErrorIs(t, err, ErrNotFound)
assert.NoError(t, s.Save(testSnapshot()))
assert.False(t, s.Validate("x", "main", "y"))
assert.NoError(t, s.Evict("x", "main", "y"))
assert.NoError(t, s.Close())
}
// TestFileStore_ConcurrentSave exercises the cross-process advisory lock:
// every writer targets the same cache key, so without serialization
// one writer's os.RemoveAll would race another's MkdirAll/write sequence
// and leave a torn entry. flock(2) contends across file descriptors even
// within one process, so concurrent goroutines reproduce the cross-process
// hazard. After all writers complete, the entry must load as exactly one
// writer's complete payload.
func TestFileStore_ConcurrentSave(t *testing.T) {
dir := t.TempDir()
fs, err := NewFileStore(dir, "0.1.0-test")
require.NoError(t, err)
const writers = 12
markers := make(map[string]bool, writers)
var wg sync.WaitGroup
errs := make(chan error, writers)
for i := range writers {
marker := fmt.Sprintf("writer-%d", i)
markers[marker] = true
wg.Add(1)
go func(m string) {
defer wg.Done()
snap := testSnapshot()
snap.Nodes[0].Meta["writer"] = m
errs <- fs.Save(snap)
}(marker)
}
wg.Wait()
close(errs)
for e := range errs {
require.NoError(t, e)
}
loaded, err := fs.Load("/tmp/test-repo", "main", "abc123def456")
require.NoError(t, err)
require.Len(t, loaded.Nodes, 2)
got, _ := loaded.Nodes[0].Meta["writer"].(string)
assert.True(t, markers[got], "loaded snapshot must be one writer's complete payload, got %q", got)
}
// TestFileStore_ConcurrentReadWrite runs readers against a writer churning
// the same entry. The shared read lock must hand every reader either a
// fully decodable snapshot or a clean ErrNotFound — never a gob/gzip
// decode error from a half-written file.
func TestFileStore_ConcurrentReadWrite(t *testing.T) {
dir := t.TempDir()
fs, err := NewFileStore(dir, "0.1.0-test")
require.NoError(t, err)
snap := testSnapshot()
require.NoError(t, fs.Save(snap))
var wg sync.WaitGroup
errs := make(chan error, 64)
stop := make(chan struct{})
wg.Add(1)
go func() {
defer wg.Done()
for i := 0; ; i++ {
select {
case <-stop:
return
default:
}
var e error
if i%2 == 0 {
e = fs.Save(snap)
} else {
e = fs.Evict(snap.RepoPath, snap.Branch, snap.CommitHash)
}
// Honour stop while sending: errs is buffered, and the
// writer outruns the buffer in microseconds. Without the
// stop arm here the writer blocks on a full errs channel,
// never re-checks stop, and wg.Wait() deadlocks (the buffer
// only drains after wg.Wait()).
select {
case errs <- e:
case <-stop:
return
}
}
}()
for r := 0; r < 6; r++ {
wg.Add(1)
go func() {
defer wg.Done()
for i := 0; i < 40; i++ {
_, err := fs.Load(snap.RepoPath, snap.Branch, snap.CommitHash)
if err != nil && err != ErrNotFound {
errs <- err
}
}
}()
}
time.Sleep(50 * time.Millisecond)
close(stop)
wg.Wait()
close(errs)
for e := range errs {
require.NoError(t, e, "no reader may observe a torn snapshot")
}
}
// TestSnapshotReuseAcrossBinaryBump proves the warm snapshot is reused across a
// binary version bump that did NOT change extraction output (the extraction
// version is the gate, not the binary-version string) — avoiding the needless
// full cold rebuild a binary-string gate would force on a no-op release — while
// a genuine extraction-version change still invalidates the slot.
func TestSnapshotReuseAcrossBinaryBump(t *testing.T) {
dir := t.TempDir()
s1, err := NewFileStore(dir, "0.48.0")
if err != nil {
t.Fatal(err)
}
snap := testSnapshot()
snap.Version = "0.48.0"
if err := s1.Save(snap); err != nil {
t.Fatal(err)
}
// A newer binary with the SAME extraction version reuses the slot.
s2, err := NewFileStore(dir, "0.48.1")
if err != nil {
t.Fatal(err)
}
if !s2.Validate(snap.RepoPath, snap.Branch, snap.CommitHash) {
t.Error("snapshot should be reused across a binary bump with an unchanged extraction version")
}
// Corrupt the extraction-version marker to simulate an extraction-output
// change: the slot must now be rejected (cold rebuild).
entry := s2.entryDir(snap.RepoPath, snap.Branch, snap.CommitHash)
if err := os.WriteFile(filepath.Join(entry, extractionVersionFile), []byte("999"), 0o644); err != nil {
t.Fatal(err)
}
if s2.Validate(snap.RepoPath, snap.Branch, snap.CommitHash) {
t.Error("a changed extraction version must invalidate the snapshot")
}
}
+93
View File
@@ -0,0 +1,93 @@
package persistence
import (
"compress/gzip"
"encoding/gob"
"fmt"
"os"
"path/filepath"
)
const (
frecencyFile = "frecency.gob.gz"
maxFrecencyAccesses = 16 // entries per symbol; matches FFF's bounded deque
maxFrecencySymbols = 10000
)
// FrecencyAccesses is the bounded, ordered (oldest→newest) list of unix
// timestamps (seconds) at which one symbol was consumed. Bounded because
// the decay formula already weights recent accesses far more heavily than
// old ones — beyond ~16 entries, additional history contributes noise.
type FrecencyAccesses struct {
SymbolID string
Times []int64
}
// FrecencyStore holds all per-symbol access histories for a single repo.
type FrecencyStore struct {
Version string
RepoPath string
Symbols []FrecencyAccesses
}
// MaxFrecencyAccesses returns the per-symbol access-history cap.
func MaxFrecencyAccesses() int { return maxFrecencyAccesses }
// FrecencyDir returns the on-disk directory for frecency storage.
func FrecencyDir(cacheDir, repoPath string) string {
return filepath.Join(cacheDir, RepoCacheKey(repoPath))
}
// LoadFrecency reads the frecency store from disk.
func LoadFrecency(dir string) (*FrecencyStore, error) {
path := filepath.Join(dir, frecencyFile)
f, err := os.Open(path)
if err != nil {
if os.IsNotExist(err) {
return &FrecencyStore{}, nil
}
return nil, fmt.Errorf("persistence: open frecency: %w", err)
}
defer f.Close()
gz, err := gzip.NewReader(f)
if err != nil {
return nil, fmt.Errorf("persistence: gzip reader frecency: %w", err)
}
defer gz.Close()
var store FrecencyStore
if err := gob.NewDecoder(gz).Decode(&store); err != nil {
return nil, fmt.Errorf("persistence: gob decode frecency: %w", err)
}
return &store, nil
}
// SaveFrecency writes the frecency store to disk. Trims the oldest-touched
// symbols if over cap so the file can't grow unboundedly.
func SaveFrecency(dir string, store *FrecencyStore) error {
if len(store.Symbols) > maxFrecencySymbols {
trim := len(store.Symbols) - maxFrecencySymbols
store.Symbols = store.Symbols[trim:]
}
if err := os.MkdirAll(dir, 0o755); err != nil {
return fmt.Errorf("persistence: mkdir frecency: %w", err)
}
path := filepath.Join(dir, frecencyFile)
f, err := os.Create(path)
if err != nil {
return fmt.Errorf("persistence: create frecency: %w", err)
}
gz := gzip.NewWriter(f)
enc := gob.NewEncoder(gz)
if err := enc.Encode(store); err != nil {
_ = gz.Close()
_ = f.Close()
return fmt.Errorf("persistence: gob encode frecency: %w", err)
}
if err := gz.Close(); err != nil {
_ = f.Close()
return fmt.Errorf("persistence: gzip close frecency: %w", err)
}
return f.Close()
}
+67
View File
@@ -0,0 +1,67 @@
package persistence
import (
"crypto/sha256"
"encoding/hex"
"path/filepath"
"strings"
"github.com/zzet/gortex/internal/pathkey"
)
// CacheKey produces a filesystem-safe directory name identifying one
// snapshot slot. Snapshots are keyed by (repo, branch): one slot per
// branch, overwritten as the branch advances. A daemon restart after
// new commits then loads the branch's last snapshot and incrementally
// reconciles, instead of cold-indexing because the commit hash moved.
// A detached HEAD (empty branch) falls back to the commit hash so each
// checked-out commit still gets a stable slot.
//
// Both the repo path and the ref are folded to Unicode NFC before
// hashing: a non-ASCII path or branch name presents with different
// bytes on macOS (NFD) than on Linux / git (NFC), and without the
// fold the same repo would key into two distinct snapshot slots.
func CacheKey(repoPath, branch, commitHash string) string {
abs, err := filepath.Abs(repoPath)
if err != nil {
abs = repoPath
}
h := sha256.Sum256([]byte(pathkey.Normalize(abs)))
pathPart := hex.EncodeToString(h[:6])
ref := strings.TrimSpace(branch)
if ref == "" || ref == "HEAD" {
ref = strings.TrimSpace(commitHash)
}
return pathPart + "_" + refSlug(ref)
}
// refSlug renders a git ref — a branch name or a commit hash — as a
// stable, filesystem-safe path segment: a readable sanitized prefix
// plus a short hash of the full ref, so two refs that sanitize or
// truncate to the same prefix (e.g. feature/x vs feature-x) still get
// distinct slots.
func refSlug(ref string) string {
if ref == "" {
return "none"
}
// Fold to NFC first: a non-ASCII branch name read in decomposed
// form on one platform and precomposed form on another would
// otherwise hash to two different slugs for the same branch.
ref = pathkey.Normalize(ref)
var b strings.Builder
for _, r := range ref {
switch {
case r >= 'a' && r <= 'z', r >= 'A' && r <= 'Z',
r >= '0' && r <= '9', r == '.', r == '-', r == '_':
b.WriteRune(r)
default:
b.WriteByte('-')
}
if b.Len() >= 32 {
break
}
}
h := sha256.Sum256([]byte(ref))
return b.String() + "_" + hex.EncodeToString(h[:4])
}
+155
View File
@@ -0,0 +1,155 @@
package persistence
import (
"strings"
"testing"
"golang.org/x/text/unicode/norm"
)
// nfcDir / nfdDir are the precomposed and decomposed byte forms of the
// same accented repo path. They are derived in code with norm.NFC /
// norm.NFD from a single base so the two byte sequences are guaranteed
// distinct and deterministic — never dependent on how this source file
// happens to be saved. On macOS APFS a repo cloned into such a
// directory is reported in NFD by the OS, while the same path written
// into a config file or read on Linux is typically NFC; a snapshot
// keyed under one form must resolve to the same slot under the other.
//
// The non-ASCII characters in the base strings are written as explicit
// \u / \U escapes so this source file stays pure-ASCII.
var (
repoDirBase = "/home/dev/café/repo" // .../café/repo
nfcDir = norm.NFC.String(repoDirBase)
nfdDir = norm.NFD.String(repoDirBase)
)
const (
cjkBranch = "機能/日本語" // 機能/日本語
cyrBranch = "функция-ветка" // функция-ветка
asciiPath = "/home/dev/plain/repo"
asciiBr = "feat/some-branch"
commitHash = "0123456789abcdef0123456789abcdef01234567"
)
// TestCacheKey_NFCvsNFDPathSameSlot is the core round-trip guarantee:
// the same repo path supplied in decomposed and precomposed Unicode
// forms must hash to one snapshot slot. Without the NFC fold in
// CacheKey the two byte sequences hash differently and the daemon
// loses its cache across an OS / form boundary.
func TestCacheKey_NFCvsNFDPathSameSlot(t *testing.T) {
if nfcDir == nfdDir {
t.Fatal("test fixture invalid: NFC and NFD repo paths are byte-identical")
}
keyNFC := CacheKey(nfcDir, "main", commitHash)
keyNFD := CacheKey(nfdDir, "main", commitHash)
if keyNFC != keyNFD {
t.Fatalf("CacheKey not normalisation-stable for repo path:\n NFC -> %q\n NFD -> %q", keyNFC, keyNFD)
}
}
// TestCacheKey_NFCvsNFDBranchSameSlot guards the branch half of the
// key: a non-ASCII branch name in two Unicode forms must land in one
// slot, so the daemon does not split a single branch's snapshot in two.
func TestCacheKey_NFCvsNFDBranchSameSlot(t *testing.T) {
branchNFD := norm.NFD.String(cjkBranch)
branchNFC := norm.NFC.String(cjkBranch)
keyNFD := CacheKey(asciiPath, branchNFD, commitHash)
keyNFC := CacheKey(asciiPath, branchNFC, commitHash)
if keyNFD != keyNFC {
t.Fatalf("CacheKey not normalisation-stable for branch:\n NFD -> %q\n NFC -> %q", keyNFD, keyNFC)
}
}
// TestCacheKey_DistinctReposDistinctSlots confirms the fold does not
// over-collapse: two genuinely different non-ASCII repo paths must
// still get distinct slots.
func TestCacheKey_DistinctReposDistinctSlots(t *testing.T) {
a := CacheKey(nfcDir, "main", commitHash)
b := CacheKey(asciiPath, "main", commitHash)
if a == b {
t.Fatalf("CacheKey collided two different repos onto slot %q", a)
}
}
// TestCacheKey_DistinctBranchesDistinctSlots confirms two different
// non-ASCII branch names on the same repo do not collide — the
// collision-safety hash inside refSlug must survive normalisation.
func TestCacheKey_DistinctBranchesDistinctSlots(t *testing.T) {
a := CacheKey(asciiPath, cjkBranch, commitHash)
b := CacheKey(asciiPath, cyrBranch, commitHash)
if a == b {
t.Fatalf("CacheKey collided two different branches onto slot %q", a)
}
}
// TestCacheKey_FilesystemSafe checks the produced key is usable as a
// directory name: no path separators, no NUL, non-empty — true even
// when the branch is entirely non-ASCII (refSlug replaces every such
// rune, leaving the hash suffix to carry identity).
func TestCacheKey_FilesystemSafe(t *testing.T) {
for _, branch := range []string{cjkBranch, cyrBranch, "main", ""} {
key := CacheKey(nfdDir, branch, commitHash)
if key == "" {
t.Fatalf("CacheKey returned empty string for branch %q", branch)
}
if strings.ContainsAny(key, "/\\\x00") {
t.Fatalf("CacheKey %q for branch %q contains a path-unsafe byte", key, branch)
}
}
}
// TestCacheKey_ASCIIUnchangedByFold pins that the NFC fold is a no-op
// for the common all-ASCII case — the keys for ASCII inputs must not
// shift, so existing on-disk snapshot directories stay valid.
func TestCacheKey_ASCIIUnchangedByFold(t *testing.T) {
// Recomputed twice must be stable, and an ASCII path/branch must
// survive the fold byte-for-byte (the slug prefix is visible in
// the key, so a regression here would change the directory name).
key1 := CacheKey(asciiPath, asciiBr, commitHash)
key2 := CacheKey(asciiPath, asciiBr, commitHash)
if key1 != key2 {
t.Fatalf("CacheKey not deterministic for ASCII input: %q vs %q", key1, key2)
}
if !strings.Contains(key1, "feat-some-branch") {
t.Fatalf("CacheKey %q lost its readable ASCII branch slug", key1)
}
}
// TestCacheKey_DetachedHeadFallsBackToCommit checks the detached-HEAD
// path still works once the fold is in place: an empty branch keys by
// commit hash.
func TestCacheKey_DetachedHeadFallsBackToCommit(t *testing.T) {
withBranch := CacheKey(asciiPath, "main", commitHash)
detached := CacheKey(asciiPath, "", commitHash)
headLiteral := CacheKey(asciiPath, "HEAD", commitHash)
if detached == withBranch {
t.Fatal("detached-HEAD key collided with a real branch key")
}
if detached != headLiteral {
t.Fatalf("empty branch and literal HEAD must key alike: %q vs %q", detached, headLiteral)
}
}
// TestRepoCacheKey_NFCvsNFDSameSlot mirrors the CacheKey round-trip
// guarantee for the commit-independent feedback key.
func TestRepoCacheKey_NFCvsNFDSameSlot(t *testing.T) {
if nfcDir == nfdDir {
t.Fatal("test fixture invalid: NFC and NFD repo paths are byte-identical")
}
keyNFC := RepoCacheKey(nfcDir)
keyNFD := RepoCacheKey(nfdDir)
if keyNFC != keyNFD {
t.Fatalf("RepoCacheKey not normalisation-stable:\n NFC -> %q\n NFD -> %q", keyNFC, keyNFD)
}
}
// TestRepoCacheKey_DistinctReposDistinctSlots confirms the feedback
// key does not over-collapse distinct non-ASCII repos.
func TestRepoCacheKey_DistinctReposDistinctSlots(t *testing.T) {
a := RepoCacheKey(nfcDir)
b := RepoCacheKey(asciiPath)
if a == b {
t.Fatalf("RepoCacheKey collided two different repos onto slot %q", a)
}
}
+120
View File
@@ -0,0 +1,120 @@
package persistence
import (
"compress/gzip"
"encoding/gob"
"fmt"
"os"
"path/filepath"
)
const (
keywordFile = "keyword.gob.gz"
maxKeywords = 4000
maxKeywordEntriesPerKey = 10
)
// KeywordMatch records one (keyword -> symbol) association. HitCount
// is how many times the agent picked this symbol following a query
// that contained the keyword; LastUsed is a unix timestamp (seconds)
// for decay and reaping. Structurally identical to ComboMatch but
// kept as a distinct type so the two stores' schemas can diverge.
type KeywordMatch struct {
SymbolID string
HitCount uint32
LastUsed int64
// MissCount is the implicit-negative tally: how many times the
// agent was shown this symbol for a query carrying the keyword but
// skipped over it to pick a lower-ranked result. The per-keyword
// boost nets HitCount-MissCount, so a symbol the agent keeps passing
// over loses its learned boost. A zero value (legacy data) is the
// pre-negative-signal behaviour. Decays on the same clock as hits.
MissCount uint32
}
// KeywordAssoc holds all recorded matches for one query keyword
// within a single repo. Ordered most-hit-first after any record.
type KeywordAssoc struct {
Keyword string
Matches []KeywordMatch
}
// KeywordStore is the persisted per-keyword association index for one
// repo. Where ComboStore keys on the whole normalized query,
// KeywordStore keys on each surviving query token -- so a new task
// with overlapping keywords but different phrasing still inherits the
// associations its keywords earned. Separate file from combo and
// feedback so each subsystem's schema evolves independently.
type KeywordStore struct {
Version string
RepoPath string
Keywords []KeywordAssoc
}
// KeywordDir returns the on-disk directory for keyword storage.
// Shares the repo cache key with combo / feedback so all repo-scoped
// state lives together.
func KeywordDir(cacheDir, repoPath string) string {
return filepath.Join(cacheDir, RepoCacheKey(repoPath))
}
// LoadKeyword reads the keyword store from disk. A missing file is
// not an error -- it yields an empty store.
func LoadKeyword(dir string) (*KeywordStore, error) {
path := filepath.Join(dir, keywordFile)
f, err := os.Open(path)
if err != nil {
if os.IsNotExist(err) {
return &KeywordStore{}, nil
}
return nil, fmt.Errorf("persistence: open keyword: %w", err)
}
defer f.Close()
gz, err := gzip.NewReader(f)
if err != nil {
return nil, fmt.Errorf("persistence: gzip reader keyword: %w", err)
}
defer gz.Close()
var store KeywordStore
if err := gob.NewDecoder(gz).Decode(&store); err != nil {
return nil, fmt.Errorf("persistence: gob decode keyword: %w", err)
}
return &store, nil
}
// SaveKeyword writes the keyword store with gob+gzip compression.
// Trims the oldest keywords if over cap so the file cannot grow
// unboundedly on a long-running daemon.
func SaveKeyword(dir string, store *KeywordStore) error {
if len(store.Keywords) > maxKeywords {
trim := len(store.Keywords) - maxKeywords
store.Keywords = store.Keywords[trim:]
}
if err := os.MkdirAll(dir, 0o755); err != nil {
return fmt.Errorf("persistence: mkdir keyword: %w", err)
}
path := filepath.Join(dir, keywordFile)
f, err := os.Create(path)
if err != nil {
return fmt.Errorf("persistence: create keyword: %w", err)
}
gz := gzip.NewWriter(f)
enc := gob.NewEncoder(gz)
if err := enc.Encode(store); err != nil {
_ = gz.Close()
_ = f.Close()
return fmt.Errorf("persistence: gob encode keyword: %w", err)
}
if err := gz.Close(); err != nil {
_ = f.Close()
return fmt.Errorf("persistence: gzip close keyword: %w", err)
}
return f.Close()
}
// MaxKeywordEntries returns the cap on matches per keyword. Tighter
// than MaxComboEntries -- a single keyword is a coarser key than a
// whole query, so its match list is held shorter to stay precise.
func MaxKeywordEntries() int { return maxKeywordEntriesPerKey }
+68
View File
@@ -0,0 +1,68 @@
package persistence
import (
"path/filepath"
"testing"
)
func TestKeywordStore_RoundTrip(t *testing.T) {
dir := filepath.Join(t.TempDir(), "kw")
// A missing file loads as an empty store, not an error.
loaded, err := LoadKeyword(dir)
if err != nil {
t.Fatalf("LoadKeyword on a missing dir errored: %v", err)
}
if loaded == nil || len(loaded.Keywords) != 0 {
t.Fatalf("missing-file load should yield an empty store")
}
store := &KeywordStore{
Version: "1",
RepoPath: "/repo",
Keywords: []KeywordAssoc{
{Keyword: "auth", Matches: []KeywordMatch{
{SymbolID: "pkg::LoginService", HitCount: 5, LastUsed: 1700},
}},
{Keyword: "token", Matches: []KeywordMatch{
{SymbolID: "pkg::ParseJWT", HitCount: 3, LastUsed: 1701},
}},
},
}
if err := SaveKeyword(dir, store); err != nil {
t.Fatalf("SaveKeyword errored: %v", err)
}
back, err := LoadKeyword(dir)
if err != nil {
t.Fatalf("LoadKeyword after save errored: %v", err)
}
if len(back.Keywords) != 2 {
t.Fatalf("round-trip lost keywords: got %d, want 2", len(back.Keywords))
}
if back.Keywords[0].Keyword != "auth" ||
back.Keywords[0].Matches[0].SymbolID != "pkg::LoginService" ||
back.Keywords[0].Matches[0].HitCount != 5 {
t.Errorf("round-trip corrupted the first keyword: %+v", back.Keywords[0])
}
}
func TestKeywordStore_TrimsOverCap(t *testing.T) {
dir := filepath.Join(t.TempDir(), "kw")
store := &KeywordStore{}
for i := 0; i < maxKeywords+50; i++ {
store.Keywords = append(store.Keywords, KeywordAssoc{Keyword: string(rune('a' + i%26))})
}
if err := SaveKeyword(dir, store); err != nil {
t.Fatalf("SaveKeyword errored: %v", err)
}
if len(store.Keywords) != maxKeywords {
t.Errorf("SaveKeyword should trim to %d keywords, got %d", maxKeywords, len(store.Keywords))
}
}
func TestMaxKeywordEntries_TighterThanCombo(t *testing.T) {
if MaxKeywordEntries() >= MaxComboEntries() {
t.Errorf("MaxKeywordEntries (%d) should be tighter than MaxComboEntries (%d)",
MaxKeywordEntries(), MaxComboEntries())
}
}
+169
View File
@@ -0,0 +1,169 @@
package persistence
import (
"compress/gzip"
"encoding/gob"
"fmt"
"os"
"path/filepath"
"time"
)
const (
memoriesFile = "memories.gob.gz"
maxMemoriesCap = 10000
)
// MemoryEntry is a single cross-session development memory. Unlike
// NoteEntry (per-session scratchpad), MemoryEntry has no SessionID
// — every memory is workspace-wide and durable across sessions.
//
// Memories accumulate over time and compound the longer a team uses
// Gortex: every recorded invariant, gotcha, decision, or convention
// becomes discoverable by every future agent in the same workspace.
//
// Memories are surfaced when:
// - an anchor symbol / file enters the agent's working set
// (via surface_memories)
// - the agent explicitly queries by symbol / file / tag / text
// (via query_memories)
type MemoryEntry struct {
ID string
Timestamp time.Time
UpdatedAt time.Time
LastAccessed time.Time
AccessCount uint64
Body string // free-form text
Title string // short caption (one-liner)
Kind string // invariant | constraint | convention | gotcha | decision | incident | reference
Source string // manual | distilled | incident | review
Confidence float32 // 0..1 — how sure we are this still holds
Importance int // 1..5 — operator-assigned weight
AuthorAgent string // mcp clientInfo.name
SymbolIDs []string // primary symbol anchors
FilePaths []string // primary file anchors
AutoLinks []string // additional referenced symbol IDs (auto-detected)
Tags []string // free-form labels
WorkspaceID string
ProjectID string
RepoPrefix string
Pinned bool // pinned memories are never evicted and float to top
SupersededBy string // ID of newer memory that replaces this one
}
// MemoryStore is the persisted shape: a versioned, repo-scoped list
// of entries. Mirrors NoteStore so the on-disk layout stays
// consistent.
type MemoryStore struct {
Version string
RepoPath string
Entries []MemoryEntry
}
// MemoriesDir resolves the cache directory holding the memories
// file for the given repo. Shares the per-repo cache directory
// with notes / feedback / combo / frecency.
func MemoriesDir(cacheDir, repoPath string) string {
return filepath.Join(cacheDir, RepoCacheKey(repoPath))
}
// LoadMemories reads a memory store from disk. Returns an empty
// store when the file does not exist (cold start is normal).
func LoadMemories(dir string) (*MemoryStore, error) {
path := filepath.Join(dir, memoriesFile)
f, err := os.Open(path)
if err != nil {
if os.IsNotExist(err) {
return &MemoryStore{}, nil
}
return nil, fmt.Errorf("persistence: open memories: %w", err)
}
defer f.Close()
gz, err := gzip.NewReader(f)
if err != nil {
return nil, fmt.Errorf("persistence: gzip reader memories: %w", err)
}
defer gz.Close()
var store MemoryStore
if err := gob.NewDecoder(gz).Decode(&store); err != nil {
return nil, fmt.Errorf("persistence: gob decode memories: %w", err)
}
return &store, nil
}
// SaveMemories writes the store to disk with gob+gzip. Trimming
// honours pinned memories: when the entry count exceeds
// maxMemoriesCap, the lowest-importance / non-pinned entries are
// dropped first; if more shedding is still needed, the oldest
// non-pinned entries go next.
func SaveMemories(dir string, store *MemoryStore) error {
if len(store.Entries) > maxMemoriesCap {
store.Entries = trimMemories(store.Entries, maxMemoriesCap)
}
if err := os.MkdirAll(dir, 0o755); err != nil {
return fmt.Errorf("persistence: mkdir memories: %w", err)
}
path := filepath.Join(dir, memoriesFile)
f, err := os.Create(path)
if err != nil {
return fmt.Errorf("persistence: create memories: %w", err)
}
gz := gzip.NewWriter(f)
enc := gob.NewEncoder(gz)
if err := enc.Encode(store); err != nil {
_ = gz.Close()
_ = f.Close()
return fmt.Errorf("persistence: gob encode memories: %w", err)
}
if err := gz.Close(); err != nil {
_ = f.Close()
return fmt.Errorf("persistence: gzip close memories: %w", err)
}
return f.Close()
}
// trimMemories drops entries until len(out) <= cap. Two passes:
// - first, shed non-pinned entries with importance <= 2
// - then, if still over cap, shed oldest non-pinned regardless
//
// Pinned entries are always retained, even if the resulting slice
// exceeds cap (the cap is a soft ceiling for the prunable tail).
func trimMemories(in []MemoryEntry, cap int) []MemoryEntry {
if len(in) <= cap {
return in
}
excess := len(in) - cap
out := make([]MemoryEntry, 0, len(in))
dropped := 0
for _, e := range in {
if dropped < excess && !e.Pinned && e.Importance <= 2 {
dropped++
continue
}
out = append(out, e)
}
if len(out) > cap {
excess = len(out) - cap
next := make([]MemoryEntry, 0, len(out))
dropped = 0
for _, e := range out {
if dropped < excess && !e.Pinned {
dropped++
continue
}
next = append(next, e)
}
out = next
}
return out
}
+113
View File
@@ -0,0 +1,113 @@
package persistence
import (
"path/filepath"
"testing"
"time"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestMemoriesPersistence_RoundTrip(t *testing.T) {
cache := t.TempDir()
dir := MemoriesDir(cache, "/tmp/repo-a")
store := &MemoryStore{
Version: "test",
RepoPath: "/tmp/repo-a",
Entries: []MemoryEntry{
{
ID: "mem-1",
Timestamp: time.Now().UTC(),
UpdatedAt: time.Now().UTC(),
Body: "lock invariant for Bar",
Title: "Bar lock invariant",
Kind: "invariant",
Source: "manual",
Importance: 5,
Confidence: 1.0,
SymbolIDs: []string{"pkg/foo.go::Bar"},
FilePaths: []string{"pkg/foo.go"},
Tags: []string{"invariant", "lock"},
WorkspaceID: "ws-a",
Pinned: true,
},
},
}
require.NoError(t, SaveMemories(dir, store))
got, err := LoadMemories(dir)
require.NoError(t, err)
require.NotNil(t, got)
require.Len(t, got.Entries, 1)
assert.Equal(t, "mem-1", got.Entries[0].ID)
assert.Equal(t, "Bar lock invariant", got.Entries[0].Title)
assert.Equal(t, "invariant", got.Entries[0].Kind)
assert.Equal(t, 5, got.Entries[0].Importance)
assert.True(t, got.Entries[0].Pinned)
assert.Equal(t, []string{"pkg/foo.go::Bar"}, got.Entries[0].SymbolIDs)
}
func TestMemoriesPersistence_EmptyOnMissingFile(t *testing.T) {
dir := filepath.Join(t.TempDir(), "does-not-exist")
got, err := LoadMemories(dir)
require.NoError(t, err)
require.NotNil(t, got)
assert.Empty(t, got.Entries)
}
func TestMemoriesPersistence_TrimDropsLowImportanceFirst(t *testing.T) {
in := make([]MemoryEntry, 0, 10)
for i := range 10 {
e := MemoryEntry{ID: memID(i), Importance: 4}
// Mark a couple as importance=1 — those should be dropped first.
if i == 2 || i == 4 {
e.Importance = 1
}
// Pin one — must survive.
if i == 7 {
e.Pinned = true
e.Importance = 1
}
in = append(in, e)
}
out := trimMemories(in, 6)
require.Len(t, out, 6)
pinnedOrHi := map[string]bool{}
for _, e := range out {
pinnedOrHi[e.ID] = true
}
assert.True(t, pinnedOrHi[memID(7)], "pinned[7] must survive even with importance=1")
// Low-importance non-pinned (i=2, i=4) should both be dropped.
assert.False(t, pinnedOrHi[memID(2)], "low-imp[2] must be dropped")
assert.False(t, pinnedOrHi[memID(4)], "low-imp[4] must be dropped")
}
func TestMemoriesPersistence_TrimFallsBackToOldestNonPinned(t *testing.T) {
// All entries have importance > 2 — the first pass can shed none,
// so the fallback pass must shed the oldest non-pinned.
in := make([]MemoryEntry, 0, 5)
for i := range 5 {
in = append(in, MemoryEntry{ID: memID(i), Importance: 5, Pinned: i == 4})
}
out := trimMemories(in, 3)
require.Len(t, out, 3)
// The pinned entry (i=4) and the newest non-pinned tail must survive.
survived := map[string]bool{}
for _, e := range out {
survived[e.ID] = true
}
assert.True(t, survived[memID(4)], "pinned[4] must survive")
}
func TestMemoriesPersistence_TrimNoopUnderCap(t *testing.T) {
in := []MemoryEntry{{ID: "a"}, {ID: "b"}, {ID: "c"}}
out := trimMemories(in, 10)
assert.Equal(t, in, out)
}
func memID(i int) string {
return "mem-" + string(rune('a'+i))
}
+11
View File
@@ -0,0 +1,11 @@
package persistence
// NopStore is a no-op persistence backend used when caching is disabled.
type NopStore struct{}
func (NopStore) Check(_, _, _ string) bool { return false }
func (NopStore) Load(_, _, _ string) (*Snapshot, error) { return nil, ErrNotFound }
func (NopStore) Save(_ *Snapshot) error { return nil }
func (NopStore) Validate(_, _, _ string) bool { return false }
func (NopStore) Evict(_, _, _ string) error { return nil }
func (NopStore) Close() error { return nil }
+143
View File
@@ -0,0 +1,143 @@
package persistence
import (
"compress/gzip"
"encoding/gob"
"fmt"
"os"
"path/filepath"
"time"
)
const (
notesFile = "notes.gob.gz"
maxNotesCap = 5000
)
// NoteEntry is a single session-memory note. Notes are not graph
// nodes; they live alongside the graph as a separate, persistent
// side-store. A note is created via the `save_note` tool, surfaced
// via `query_notes`, and folded into the per-session digest emitted
// by `distill_session`.
//
// Auto-links capture the symbol IDs the agent (or the auto-linker)
// determined to be referenced by the note body. Because they are
// stored explicitly, queries by symbol stay O(notes) without
// re-tokenising every body on every call.
type NoteEntry struct {
ID string
Timestamp time.Time
UpdatedAt time.Time
SessionID string // MCP session that created the note ("" for shared/embedded session)
ClientName string // MCP clientInfo.name at create time (claude-code / cursor / ...)
Body string // free-form text the agent wrote
SymbolID string // primary attached symbol (optional)
FilePath string // primary attached file (optional)
RepoPrefix string // repo prefix derived from session scope or attached symbol/file
WorkspaceID string // workspace boundary; queries scope by this
ProjectID string // project sub-boundary
Tags []string // free-form labels — "decision", "bug", "todo", ...
AutoLinks []string // symbol IDs referenced by the body (auto-detected + explicit links)
Pinned bool // pinned notes are never evicted by the cap
}
// NoteStore is the persisted shape: a versioned, repo-scoped list of
// entries. Same persistence shape as FeedbackStore so the cache
// directory layout stays consistent.
type NoteStore struct {
Version string
RepoPath string
Entries []NoteEntry
}
// NotesDir resolves the cache directory holding the notes file for
// the given repo. Mirrors FeedbackDir so the two side-stores share
// a per-repo cache subdirectory.
func NotesDir(cacheDir, repoPath string) string {
return filepath.Join(cacheDir, RepoCacheKey(repoPath))
}
// LoadNotes reads a note store from disk. Returns an empty store
// when the file does not exist (cold start is normal).
func LoadNotes(dir string) (*NoteStore, error) {
path := filepath.Join(dir, notesFile)
f, err := os.Open(path)
if err != nil {
if os.IsNotExist(err) {
return &NoteStore{}, nil
}
return nil, fmt.Errorf("persistence: open notes: %w", err)
}
defer f.Close()
gz, err := gzip.NewReader(f)
if err != nil {
return nil, fmt.Errorf("persistence: gzip reader notes: %w", err)
}
defer gz.Close()
var store NoteStore
if err := gob.NewDecoder(gz).Decode(&store); err != nil {
return nil, fmt.Errorf("persistence: gob decode notes: %w", err)
}
return &store, nil
}
// SaveNotes writes the store to disk with gob+gzip. Trimming
// honours pinned notes: when the entry count exceeds maxNotesCap,
// the oldest non-pinned entries are dropped first.
func SaveNotes(dir string, store *NoteStore) error {
if len(store.Entries) > maxNotesCap {
store.Entries = trimNotes(store.Entries, maxNotesCap)
}
if err := os.MkdirAll(dir, 0o755); err != nil {
return fmt.Errorf("persistence: mkdir notes: %w", err)
}
path := filepath.Join(dir, notesFile)
f, err := os.Create(path)
if err != nil {
return fmt.Errorf("persistence: create notes: %w", err)
}
gz := gzip.NewWriter(f)
enc := gob.NewEncoder(gz)
if err := enc.Encode(store); err != nil {
_ = gz.Close()
_ = f.Close()
return fmt.Errorf("persistence: gob encode notes: %w", err)
}
if err := gz.Close(); err != nil {
_ = f.Close()
return fmt.Errorf("persistence: gzip close notes: %w", err)
}
return f.Close()
}
// trimNotes drops the oldest non-pinned entries until len(out) <= cap.
// Order is preserved so callers iterate chronologically. If pinned
// entries alone exceed cap, every pinned entry is still retained
// — the cap is a soft ceiling for the unpinned tail.
func trimNotes(in []NoteEntry, cap int) []NoteEntry {
if len(in) <= cap {
return in
}
excess := len(in) - cap
// Walk forward dropping non-pinned entries until we have shed
// `excess` of them, then return the tail.
out := make([]NoteEntry, 0, cap)
dropped := 0
for _, e := range in {
if dropped < excess && !e.Pinned {
dropped++
continue
}
out = append(out, e)
}
return out
}
+82
View File
@@ -0,0 +1,82 @@
package persistence
import (
"path/filepath"
"testing"
"time"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestNotesPersistence_RoundTrip(t *testing.T) {
cache := t.TempDir()
dir := NotesDir(cache, "/tmp/repo-a")
store := &NoteStore{
Version: "test",
RepoPath: "/tmp/repo-a",
Entries: []NoteEntry{
{
ID: "nt-1",
Timestamp: time.Now().UTC(),
UpdatedAt: time.Now().UTC(),
SessionID: "sess-1",
Body: "decision: switch to fastpath",
SymbolID: "pkg/foo.go::Bar",
Tags: []string{"decision"},
AutoLinks: []string{"pkg/foo.go::Bar"},
Pinned: true,
},
},
}
require.NoError(t, SaveNotes(dir, store))
got, err := LoadNotes(dir)
require.NoError(t, err)
require.NotNil(t, got)
require.Len(t, got.Entries, 1)
assert.Equal(t, "nt-1", got.Entries[0].ID)
assert.Equal(t, "pkg/foo.go::Bar", got.Entries[0].SymbolID)
assert.True(t, got.Entries[0].Pinned)
}
func TestNotesPersistence_EmptyOnMissingFile(t *testing.T) {
dir := filepath.Join(t.TempDir(), "does-not-exist")
got, err := LoadNotes(dir)
require.NoError(t, err)
require.NotNil(t, got)
assert.Empty(t, got.Entries)
}
func TestNotesPersistence_TrimDropsOldestUnpinned(t *testing.T) {
in := make([]NoteEntry, 0, 10)
for i := range 10 {
in = append(in, NoteEntry{ID: noteID(i), Pinned: i%5 == 0})
}
out := trimNotes(in, 6)
require.Len(t, out, 6)
// Both pinned entries (i=0, i=5) must survive.
pinnedIDs := map[string]bool{}
for _, e := range out {
if e.Pinned {
pinnedIDs[e.ID] = true
}
}
assert.True(t, pinnedIDs[noteID(0)], "pinned[0] must survive")
assert.True(t, pinnedIDs[noteID(5)], "pinned[5] must survive")
// The newest entries should be present (LIFO preserves the tail).
assert.Equal(t, noteID(9), out[len(out)-1].ID)
}
func TestNotesPersistence_TrimNoopUnderCap(t *testing.T) {
in := []NoteEntry{{ID: "a"}, {ID: "b"}, {ID: "c"}}
out := trimNotes(in, 10)
assert.Equal(t, in, out)
}
func noteID(i int) string {
return "nt-" + string(rune('a'+i))
}
+102
View File
@@ -0,0 +1,102 @@
package persistence
import (
"fmt"
"time"
)
// CLI-verb ledger. The cli_events table records one row per CLI verb that
// ran under a correlation session id. Unlike the opt-in savings ledger it
// carries no consent gate and no daily-aggregate dimension: it is a thin,
// consent-free, per-event log scoped by session_id so a context-budget
// receipt can name the exact verbs (and the safety steps derived from them)
// a single agent session drove through the CLI.
//
// Modeled on savings_events: a single INSERT per event (durable at the call,
// nothing batched), and time- / session-queryable reads. The table is keyed
// on session_id so the receipt can read back exactly one session's verbs.
// CLIEvent is one recorded CLI-verb invocation.
type CLIEvent struct {
TS time.Time
SessionID string
Verb string
}
// AddCLIEvent books one CLI-verb invocation as a single INSERT. Durable at
// the call — a SIGKILLed process loses nothing. A zero ts is stamped with
// the current time.
func (s *SidecarStore) AddCLIEvent(ts time.Time, sessionID, verb string) error {
if s == nil {
return nil
}
s.writeMu.Lock()
defer s.writeMu.Unlock()
if ts.IsZero() {
ts = time.Now()
}
if _, err := s.db.Exec(
`INSERT INTO cli_events (ts, session_id, verb) VALUES (?,?,?)`,
ts.UTC().UnixNano(), sessionID, verb,
); err != nil {
return fmt.Errorf("persistence: cli event: %w", err)
}
return nil
}
// CLIEventsSince returns events with ts >= since, oldest first.
// since=zero returns everything.
func (s *SidecarStore) CLIEventsSince(since time.Time) ([]CLIEvent, error) {
if s == nil {
return nil, nil
}
rows, err := s.db.Query(
`SELECT ts, session_id, verb FROM cli_events WHERE ts >= ? ORDER BY ts, id`,
unixOrZero(since),
)
if err != nil {
return nil, fmt.Errorf("persistence: cli events since: %w", err)
}
defer rows.Close()
var out []CLIEvent
for rows.Next() {
var ev CLIEvent
var tsN int64
if err := rows.Scan(&tsN, &ev.SessionID, &ev.Verb); err != nil {
return nil, err
}
ev.TS = time.Unix(0, tsN).UTC()
out = append(out, ev)
}
return out, rows.Err()
}
// CLIEventsBySession returns every event for one correlation session id,
// oldest first.
func (s *SidecarStore) CLIEventsBySession(sessionID string) ([]CLIEvent, error) {
if s == nil {
return nil, nil
}
rows, err := s.db.Query(
`SELECT ts, session_id, verb FROM cli_events WHERE session_id = ? ORDER BY ts, id`,
sessionID,
)
if err != nil {
return nil, fmt.Errorf("persistence: cli events by session: %w", err)
}
defer rows.Close()
var out []CLIEvent
for rows.Next() {
var ev CLIEvent
var tsN int64
if err := rows.Scan(&tsN, &ev.SessionID, &ev.Verb); err != nil {
return nil, err
}
ev.TS = time.Unix(0, tsN).UTC()
out = append(out, ev)
}
return out, rows.Err()
}
@@ -0,0 +1,106 @@
package persistence
import (
"testing"
"time"
)
// TestCLIEvents_RoundTrip books a few CLI-verb events under two sessions
// and proves both read paths: time-windowed (Since) and per-session
// (BySession). Events survive a close + reopen — durable at the call.
func TestCLIEvents_RoundTrip(t *testing.T) {
sc, path := openTestSidecar(t)
t0 := time.Date(2026, 6, 1, 10, 0, 0, 0, time.UTC)
events := []CLIEvent{
{TS: t0, SessionID: "sess-1", Verb: "edit.verify"},
{TS: t0.Add(time.Minute), SessionID: "sess-1", Verb: "edit.guards"},
{TS: t0.Add(2 * time.Minute), SessionID: "sess-2", Verb: "query.stats"},
}
for _, ev := range events {
if err := sc.AddCLIEvent(ev.TS, ev.SessionID, ev.Verb); err != nil {
t.Fatalf("AddCLIEvent: %v", err)
}
}
if err := sc.Close(); err != nil {
t.Fatal(err)
}
sc2, err := OpenSidecar(path)
if err != nil {
t.Fatal(err)
}
defer sc2.Close()
// Since(zero) returns everything, oldest first.
all, err := sc2.CLIEventsSince(time.Time{})
if err != nil {
t.Fatal(err)
}
if len(all) != 3 {
t.Fatalf("CLIEventsSince(zero) = %d events, want 3", len(all))
}
if all[0].Verb != "edit.verify" || all[2].Verb != "query.stats" {
t.Errorf("order = %q…%q, want edit.verify…query.stats", all[0].Verb, all[2].Verb)
}
if !all[0].TS.Equal(t0) || all[0].SessionID != "sess-1" {
t.Errorf("first event = %+v, want ts=%v session=sess-1", all[0], t0)
}
// Since a cutoff after the first two events: only sess-2 remains.
recent, err := sc2.CLIEventsSince(t0.Add(90 * time.Second))
if err != nil {
t.Fatal(err)
}
if len(recent) != 1 || recent[0].Verb != "query.stats" {
t.Errorf("CLIEventsSince(cutoff) = %+v, want only query.stats", recent)
}
// BySession returns just that session's verbs, oldest first.
s1, err := sc2.CLIEventsBySession("sess-1")
if err != nil {
t.Fatal(err)
}
if len(s1) != 2 || s1[0].Verb != "edit.verify" || s1[1].Verb != "edit.guards" {
t.Errorf("CLIEventsBySession(sess-1) = %+v, want [edit.verify edit.guards]", s1)
}
s2, err := sc2.CLIEventsBySession("sess-2")
if err != nil {
t.Fatal(err)
}
if len(s2) != 1 || s2[0].Verb != "query.stats" {
t.Errorf("CLIEventsBySession(sess-2) = %+v, want [query.stats]", s2)
}
// An unseen session is a clean empty, never an error.
none, err := sc2.CLIEventsBySession("nope")
if err != nil {
t.Fatal(err)
}
if len(none) != 0 {
t.Errorf("CLIEventsBySession(unknown) = %+v, want empty", none)
}
}
// TestCLIEvents_ZeroTSStamped: a zero ts is stamped with the current time
// rather than persisted as the unix epoch, so a "since recent" read still
// finds the row.
func TestCLIEvents_ZeroTSStamped(t *testing.T) {
sc, _ := openTestSidecar(t)
defer sc.Close()
before := time.Now().Add(-time.Second)
if err := sc.AddCLIEvent(time.Time{}, "sess-z", "edit.tests"); err != nil {
t.Fatal(err)
}
got, err := sc.CLIEventsBySession("sess-z")
if err != nil {
t.Fatal(err)
}
if len(got) != 1 {
t.Fatalf("want 1 event, got %d", len(got))
}
if got[0].TS.Before(before) {
t.Errorf("zero ts was not stamped to now: %v", got[0].TS)
}
}
+220
View File
@@ -0,0 +1,220 @@
package persistence
import (
"database/sql"
"errors"
"fmt"
"strings"
"time"
sqlite "modernc.org/sqlite"
)
// Sidecar schema migrations.
//
// The sidecar DB has no migration framework historically: sidecarSchema was
// run with CREATE TABLE / CREATE INDEX IF NOT EXISTS, and columns added after
// a table's original shape were retrofitted with best-effort ALTERs. That
// breaks when an index (or any statement) inside sidecarSchema references a
// column the IF-NOT-EXISTS create never adds to a pre-existing table — the
// whole schema batch aborts with "no such column" and the install can no
// longer open its ledger.
//
// runMigrations replaces that with a forward-only, version-stamped sequence
// keyed on SQLite's built-in PRAGMA user_version. sidecarSchema keeps only the
// idempotent base shape (tables + original-shape indexes); every later column
// and every column-dependent index lives in a migration that runs after the
// column's ALTER. Existing installs upgrade in place on the next OpenSidecar —
// no user action, no data loss (migrations are additive only).
//
// Concurrency: applyOne relies on the sidecar DSN's _txlock=immediate so
// db.Begin() takes the write lock at BEGIN, making the in-transaction
// user_version check authoritative. Two processes opening a stale DB at once
// serialise on busy_timeout; the loser re-reads the bumped version and skips.
// migration is one forward step. Steps are append-only and ascending: never
// edit or renumber a migration that has shipped — add a new higher version.
type migration struct {
version int
name string
fn func(tx *sql.Tx) error
}
// currentSidecarVersion is the schema version a fully-migrated DB reports via
// PRAGMA user_version. It must equal the highest version in sidecarMigrations.
const currentSidecarVersion = 1
// sidecarMigrations is the ordered, forward-only migration list.
var sidecarMigrations = []migration{
{version: 1, name: "baseline-reconcile", fn: migrateV1Baseline},
}
// runMigrations applies every pending migration in order. Each runs in its own
// IMMEDIATE-locked transaction and bumps user_version on success, so a failure
// at version N leaves versions < N committed and stamped and is safe to retry
// on the next open.
func runMigrations(db *sql.DB) error {
for _, m := range sidecarMigrations {
if err := applyOne(db, m); err != nil {
return fmt.Errorf("persistence: sidecar migration v%d (%s): %w", m.version, m.name, err)
}
}
return nil
}
// isSidecarBusyErr reports whether err is a SQLite BUSY/LOCKED result code
// from the modernc driver (matching extended codes by their base value).
func isSidecarBusyErr(err error) bool {
var se *sqlite.Error
if !errors.As(err, &se) {
return false
}
// Mask the high byte so extended codes (SQLITE_BUSY_SNAPSHOT, ...) match
// their base SQLITE_BUSY (5) / SQLITE_LOCKED (6).
switch se.Code() & 0xff {
case 5, 6:
return true
}
return false
}
// withSidecarBusyRetry runs fn, retrying on a SQLite BUSY/LOCKED error with
// bounded exponential backoff.
//
// busy_timeout (set in the sidecar DSN) covers ordinary write-lock contention
// once the file is in WAL mode, but it does NOT cover the rollback-journal ->
// WAL conversion the very first opener performs: that conversion takes a brief
// EXCLUSIVE lock whose acquisition SQLite answers with an immediate, un-retried
// SQLITE_BUSY (the busy handler is not consulted for a journal-mode change)
// when another process has the file open. The daemon, every per-repo
// `gortex mcp` subprocess, and the CLI can all open a fresh/stale sidecar at
// the same instant, so a bounded application-level retry is required on top of
// busy_timeout. fn must be idempotent — runBaseSchema (CREATE ... IF NOT
// EXISTS) and runMigrations (user_version-gated) both are.
func withSidecarBusyRetry(fn func() error) error {
const (
maxAttempts = 40
baseDelay = 5 * time.Millisecond
maxDelay = 250 * time.Millisecond
)
delay := baseDelay
var err error
for attempt := 0; attempt < maxAttempts; attempt++ {
if err = fn(); err == nil || !isSidecarBusyErr(err) {
return err
}
time.Sleep(delay)
if delay *= 2; delay > maxDelay {
delay = maxDelay
}
}
return err
}
// runBaseSchema applies the idempotent base shape (sidecarSchema) inside one
// IMMEDIATE-locked transaction. The sidecar DSN's _txlock=immediate makes
// db.Begin() emit BEGIN IMMEDIATE, so the reserved write lock is held from the
// start of the batch.
//
// This matters for the same reason applyOne wraps its work: the base schema's
// CREATE TABLE / CREATE INDEX IF NOT EXISTS statements write, and run in
// autocommit each one first takes a read lock and then tries to promote to a
// write lock. SQLite answers that promotion with an un-retryable SQLITE_BUSY
// (busy_timeout is not consulted on a lock upgrade) when another process is
// concurrently creating the same objects — so several gortex processes opening
// a fresh or stale DB at once would race here, before runMigrations' own
// IMMEDIATE transactions ever run, and the losers would fail the open. Taking
// the write lock at BEGIN instead makes a concurrent opener block on
// busy_timeout and then re-run the IF NOT EXISTS statements as clean no-ops, so
// every opener succeeds.
func runBaseSchema(db *sql.DB) error {
tx, err := db.Begin()
if err != nil {
return fmt.Errorf("begin: %w", err)
}
defer func() { _ = tx.Rollback() }() // no-op once Commit succeeds
if _, err := tx.Exec(sidecarSchema); err != nil {
return err
}
return tx.Commit()
}
// applyOne applies a single migration if the DB's user_version is below it.
//
// The sidecar DSN sets _txlock=immediate, so db.Begin() emits BEGIN IMMEDIATE
// and the reserved write lock is held before the user_version read below —
// the gate is therefore authoritative across processes. A concurrent opener
// blocks on busy_timeout at BEGIN, then (this winner having committed) reads
// the bumped version and returns without writing. PRAGMA user_version is
// transactional, so on any failure the deferred Rollback reverts both the
// partial DDL and the version bump.
func applyOne(db *sql.DB, m migration) error {
tx, err := db.Begin()
if err != nil {
return fmt.Errorf("begin: %w", err)
}
defer func() { _ = tx.Rollback() }() // no-op once Commit succeeds
var cur int
if err := tx.QueryRow("PRAGMA user_version").Scan(&cur); err != nil {
return fmt.Errorf("read user_version: %w", err)
}
if cur >= m.version {
return nil // already applied (this or another process)
}
if err := m.fn(tx); err != nil {
return err
}
// PRAGMA takes no bound parameters; m.version is an int constant we own.
// Bumped last so it rolls back with the transaction on any earlier error.
if _, err := tx.Exec(fmt.Sprintf("PRAGMA user_version = %d", m.version)); err != nil {
return fmt.Errorf("bump user_version: %w", err)
}
return tx.Commit()
}
// migrateV1Baseline reconciles any historical sidecar shape (which all report
// user_version 0) to the current schema. It is purely additive and idempotent:
// CREATE TABLE IF NOT EXISTS for every table runs earlier in sidecarSchema, so
// this only adds the columns and column-dependent indexes that a pre-existing
// table cannot gain from CREATE TABLE.
//
// savings_events.model / .client are the only column-level drift in the whole
// sidecar history; every other shape difference is a whole-table addition
// already covered by the base schema.
func migrateV1Baseline(tx *sql.Tx) error {
if err := addColumnIfMissingTx(tx, "savings_events", "model", "TEXT NOT NULL DEFAULT ''"); err != nil {
return err
}
if err := addColumnIfMissingTx(tx, "savings_events", "client", "TEXT NOT NULL DEFAULT ''"); err != nil {
return err
}
// Column-dependent indexes LAST — only after the columns are guaranteed to
// exist. These are exactly the statements that aborted sidecarSchema on a
// database created before model/client existed.
if _, err := tx.Exec(`CREATE INDEX IF NOT EXISTS idx_savings_events_model ON savings_events (model, ts)`); err != nil {
return fmt.Errorf("index idx_savings_events_model: %w", err)
}
if _, err := tx.Exec(`CREATE INDEX IF NOT EXISTS idx_savings_events_client ON savings_events (client, ts)`); err != nil {
return fmt.Errorf("index idx_savings_events_client: %w", err)
}
return nil
}
// addColumnIfMissingTx runs ALTER TABLE ... ADD COLUMN inside a transaction,
// swallowing ONLY the "duplicate column name" error that a table which already
// has the column returns (idempotency for fresh / already-upgraded databases).
// Every other error propagates — surfacing a genuine failure with its real
// cause instead of letting a later column-dependent statement fail with a
// misleading "no such column".
func addColumnIfMissingTx(tx *sql.Tx, table, column, decl string) error {
if _, err := tx.Exec("ALTER TABLE " + table + " ADD COLUMN " + column + " " + decl); err != nil {
if strings.Contains(err.Error(), "duplicate column name") {
return nil
}
return fmt.Errorf("add column %s.%s: %w", table, column, err)
}
return nil
}
@@ -0,0 +1,326 @@
package persistence
import (
"bytes"
"database/sql"
"fmt"
"os"
"os/exec"
"path/filepath"
"testing"
)
// openRaw opens a bare *sql.DB on path (driver registered by the package's
// modernc.org/sqlite blank import) for building fixtures and reading state
// without going through OpenSidecar's cache.
func openRaw(t *testing.T, path string) *sql.DB {
t.Helper()
db, err := sql.Open("sqlite", path)
if err != nil {
t.Fatalf("open raw %s: %v", path, err)
}
return db
}
func userVersion(t *testing.T, db *sql.DB) int {
t.Helper()
var v int
if err := db.QueryRow("PRAGMA user_version").Scan(&v); err != nil {
t.Fatalf("read user_version: %v", err)
}
return v
}
func hasColumn(t *testing.T, db *sql.DB, table, col string) bool {
t.Helper()
rows, err := db.Query("PRAGMA table_info(" + table + ")")
if err != nil {
t.Fatalf("table_info(%s): %v", table, err)
}
defer rows.Close()
for rows.Next() {
var (
cid int
name, ctype string
notnull, pk int
dflt sql.NullString
)
if err := rows.Scan(&cid, &name, &ctype, &notnull, &dflt, &pk); err != nil {
t.Fatalf("scan table_info(%s): %v", table, err)
}
if name == col {
return true
}
}
return false
}
func hasObject(t *testing.T, db *sql.DB, objType, name string) bool {
t.Helper()
var got string
err := db.QueryRow("SELECT name FROM sqlite_master WHERE type=? AND name=?", objType, name).Scan(&got)
if err == sql.ErrNoRows {
return false
}
if err != nil {
t.Fatalf("lookup %s %s: %v", objType, name, err)
}
return got == name
}
// makeShapeC writes a pre-v0.46 sidecar: savings_events with its ORIGINAL
// 8 columns (no model, no client) and only the original indexes, with one
// row. user_version stays 0 — the state every install created before the
// migration framework reports. Opening this DB with the buggy code aborted
// with "no such column: model".
func makeShapeC(t *testing.T, path string) {
t.Helper()
db := openRaw(t, path)
defer db.Close()
_, err := db.Exec(`
CREATE TABLE savings_events (
id INTEGER PRIMARY KEY AUTOINCREMENT,
ts INTEGER NOT NULL DEFAULT 0,
session_id TEXT NOT NULL DEFAULT '',
tool TEXT NOT NULL DEFAULT '',
repo TEXT NOT NULL DEFAULT '',
language TEXT NOT NULL DEFAULT '',
returned INTEGER NOT NULL DEFAULT 0,
saved INTEGER NOT NULL DEFAULT 0
);
CREATE INDEX idx_savings_events_ts ON savings_events (ts);
CREATE INDEX idx_savings_events_tool ON savings_events (tool, ts);
CREATE TABLE savings_totals (bucket TEXT NOT NULL PRIMARY KEY, saved INTEGER NOT NULL DEFAULT 0, returned INTEGER NOT NULL DEFAULT 0, calls INTEGER NOT NULL DEFAULT 0) WITHOUT ROWID;
CREATE TABLE savings_meta (key TEXT NOT NULL PRIMARY KEY, value INTEGER NOT NULL DEFAULT 0) WITHOUT ROWID;
INSERT INTO savings_events (ts, tool, returned, saved) VALUES (123, 'get_symbol_source', 1000, 720);
`)
if err != nil {
t.Fatalf("build shape-C fixture: %v", err)
}
}
// TestOpenSidecarFreshDB: a brand-new DB gets the full current shape and is
// stamped to the current version. The v1 migration's ALTERs hit the
// duplicate-column path (columns already present from CREATE TABLE) and must
// be swallowed inside the transaction without wedging the open.
func TestOpenSidecarFreshDB(t *testing.T) {
path := filepath.Join(t.TempDir(), "sidecar.sqlite")
st, err := OpenSidecar(path)
if err != nil {
t.Fatalf("OpenSidecar fresh: %v", err)
}
defer st.Close()
if !hasColumn(t, st.db, "savings_events", "model") || !hasColumn(t, st.db, "savings_events", "client") {
t.Fatal("fresh DB missing model/client columns")
}
if !hasObject(t, st.db, "index", "idx_savings_events_model") || !hasObject(t, st.db, "index", "idx_savings_events_client") {
t.Fatal("fresh DB missing model/client indexes")
}
if v := userVersion(t, st.db); v != currentSidecarVersion {
t.Fatalf("user_version = %d, want %d", v, currentSidecarVersion)
}
}
// TestOpenSidecarMigratesShapeC reproduces the user-reported crash and proves
// the migration fixes it in place: the old 8-column DB opens cleanly, gains
// model/client + their indexes, is stamped to v1, and its existing row
// survives.
func TestOpenSidecarMigratesShapeC(t *testing.T) {
path := filepath.Join(t.TempDir(), "sidecar.sqlite")
makeShapeC(t, path)
st, err := OpenSidecar(path)
if err != nil {
t.Fatalf("OpenSidecar shape-C (the reported crash): %v", err)
}
defer st.Close()
for _, col := range []string{"model", "client"} {
if !hasColumn(t, st.db, "savings_events", col) {
t.Fatalf("column %q not added by migration", col)
}
}
for _, idx := range []string{"idx_savings_events_model", "idx_savings_events_client"} {
if !hasObject(t, st.db, "index", idx) {
t.Fatalf("index %q not created by migration", idx)
}
}
if v := userVersion(t, st.db); v != currentSidecarVersion {
t.Fatalf("user_version = %d, want %d", v, currentSidecarVersion)
}
var n int
if err := st.db.QueryRow("SELECT COUNT(*) FROM savings_events").Scan(&n); err != nil {
t.Fatalf("count savings_events: %v", err)
}
if n != 1 {
t.Fatalf("savings row count = %d after migration, want 1 (data must survive)", n)
}
}
// TestOpenSidecarShapeBCreatesMissingTables locks the load-bearing invariant
// that sidecarSchema runs on EVERY open (never gated on user_version): an old
// DB missing newer tables must gain them, and its data must survive.
func TestOpenSidecarShapeBCreatesMissingTables(t *testing.T) {
path := filepath.Join(t.TempDir(), "sidecar.sqlite")
func() {
db := openRaw(t, path)
defer db.Close()
// A v0.36-era DB: it has scopes (an early table with no secondary
// indexes, so it survives the base-schema rerun untouched) but not
// the later savings_* / suppressions tables.
if _, err := db.Exec(`CREATE TABLE scopes (name TEXT NOT NULL PRIMARY KEY, description TEXT NOT NULL DEFAULT '', repos TEXT NOT NULL DEFAULT '[]', paths TEXT NOT NULL DEFAULT '[]') WITHOUT ROWID;
INSERT INTO scopes (name, description) VALUES ('old-scope', 'keep me');`); err != nil {
t.Fatalf("build shape-B fixture: %v", err)
}
}()
st, err := OpenSidecar(path)
if err != nil {
t.Fatalf("OpenSidecar shape-B: %v", err)
}
defer st.Close()
for _, tbl := range []string{"savings_events", "suppressions", "memories", "notebooks"} {
if !hasObject(t, st.db, "table", tbl) {
t.Fatalf("table %q not created on open of an old DB", tbl)
}
}
if v := userVersion(t, st.db); v != currentSidecarVersion {
t.Fatalf("user_version = %d, want %d", v, currentSidecarVersion)
}
var desc string
if err := st.db.QueryRow("SELECT description FROM scopes WHERE name='old-scope'").Scan(&desc); err != nil {
t.Fatalf("pre-existing scope lost: %v", err)
}
if desc != "keep me" {
t.Fatalf("scope description = %q, want 'keep me'", desc)
}
}
// TestOpenSidecarIdempotentReopen: a second open of a migrated DB is a clean
// no-op that leaves the version unchanged.
func TestOpenSidecarIdempotentReopen(t *testing.T) {
path := filepath.Join(t.TempDir(), "sidecar.sqlite")
makeShapeC(t, path)
st, err := OpenSidecar(path)
if err != nil {
t.Fatalf("first open: %v", err)
}
if err := st.Close(); err != nil { // drops the cache entry so the reopen re-runs the path
t.Fatalf("close: %v", err)
}
st2, err := OpenSidecar(path)
if err != nil {
t.Fatalf("reopen: %v", err)
}
defer st2.Close()
if v := userVersion(t, st2.db); v != currentSidecarVersion {
t.Fatalf("user_version after reopen = %d, want %d", v, currentSidecarVersion)
}
}
// TestApplyOneFailureLeavesVersionUnchanged: a failing migration rolls back
// and does not advance user_version, so the next open retries it.
func TestApplyOneFailureLeavesVersionUnchanged(t *testing.T) {
path := filepath.Join(t.TempDir(), "s.sqlite")
db := openRaw(t, path)
defer db.Close()
boom := migration{version: 1, name: "boom", fn: func(tx *sql.Tx) error {
if _, err := tx.Exec("CREATE TABLE partial (x TEXT)"); err != nil {
return err
}
return fmt.Errorf("synthetic failure after a partial write")
}}
if err := applyOne(db, boom); err == nil {
t.Fatal("expected applyOne to surface the migration error")
}
if v := userVersion(t, db); v != 0 {
t.Fatalf("user_version = %d after failed migration, want 0", v)
}
if hasObject(t, db, "table", "partial") {
t.Fatal("partial table from the failed migration was not rolled back")
}
}
// TestAddColumnIfMissingTxScope: only "duplicate column name" is swallowed;
// every other ALTER error propagates (so a real failure is not masked).
func TestAddColumnIfMissingTxScope(t *testing.T) {
path := filepath.Join(t.TempDir(), "s.sqlite")
db := openRaw(t, path)
defer db.Close()
if _, err := db.Exec(`CREATE TABLE t (a TEXT)`); err != nil {
t.Fatalf("create table: %v", err)
}
tx, err := db.Begin()
if err != nil {
t.Fatalf("begin: %v", err)
}
defer func() { _ = tx.Rollback() }()
if err := addColumnIfMissingTx(tx, "t", "b", "TEXT NOT NULL DEFAULT ''"); err != nil {
t.Fatalf("adding a new column should succeed: %v", err)
}
if err := addColumnIfMissingTx(tx, "t", "b", "TEXT NOT NULL DEFAULT ''"); err != nil {
t.Fatalf("duplicate column should be swallowed: %v", err)
}
if err := addColumnIfMissingTx(tx, "does_not_exist", "x", "TEXT"); err == nil {
t.Fatal("a non-duplicate error (no such table) must propagate")
}
}
// TestOpenSidecarConcurrentProcesses is the regression guard for the headline
// concurrency hazard: several processes opening a stale DB at once must ALL
// succeed (one migrates under the IMMEDIATE write lock, the others wait on
// busy_timeout and skip). With a plain DEFERRED begin the losers hit an
// un-retryable SQLITE_BUSY — this test fails without _txlock=immediate.
func TestOpenSidecarConcurrentProcesses(t *testing.T) {
if path := os.Getenv("GORTEX_SIDECAR_MIGRATE_CHILD"); path != "" {
// Child process: open once, report via exit code, never spawn.
st, err := OpenSidecar(path)
if err != nil {
fmt.Fprintln(os.Stderr, "child OpenSidecar failed:", err)
os.Exit(3)
}
_ = st.Close()
os.Exit(0)
}
path := filepath.Join(t.TempDir(), "sidecar.sqlite")
makeShapeC(t, path) // stale DB whose first open must migrate
const procs = 6
cmds := make([]*exec.Cmd, procs)
outs := make([]*bytes.Buffer, procs)
for i := range cmds {
buf := &bytes.Buffer{}
cmd := exec.Command(os.Args[0], "-test.run=^TestOpenSidecarConcurrentProcesses$", "-test.count=1")
cmd.Env = append(os.Environ(), "GORTEX_SIDECAR_MIGRATE_CHILD="+path)
cmd.Stderr = buf
cmds[i], outs[i] = cmd, buf
}
for i, cmd := range cmds {
if err := cmd.Start(); err != nil {
t.Fatalf("start child %d: %v", i, err)
}
}
for i, cmd := range cmds {
if err := cmd.Wait(); err != nil {
t.Fatalf("concurrent opener %d errored (none may): %v\nstderr: %s", i, err, outs[i].String())
}
}
db := openRaw(t, path)
defer db.Close()
if v := userVersion(t, db); v != currentSidecarVersion {
t.Fatalf("user_version = %d after concurrent opens, want %d", v, currentSidecarVersion)
}
if !hasColumn(t, db, "savings_events", "model") || !hasColumn(t, db, "savings_events", "client") {
t.Fatal("model/client columns missing after concurrent migration")
}
}
@@ -0,0 +1,87 @@
package persistence
import (
"fmt"
"time"
)
// PromotedToolEntry is one workspace-learned tool promotion: a tool that
// was deferred behind tools_search but has been promoted into the eager
// surface for this repo/workspace after use. Persisted so the learned
// surface survives daemon restarts. last_used_epoch drives demotion — a
// promotion unused for N session epochs is dropped back to deferred.
type PromotedToolEntry struct {
Tool string
WorkspaceID string
PromotedEpoch int64
LastUsedEpoch int64
UseCount int64
UpdatedAt time.Time
}
// LoadPromotedTools reads every learned promotion for a repo_key.
func (s *SidecarStore) LoadPromotedTools(repoKey string) ([]PromotedToolEntry, error) {
rows, err := s.db.Query(`
SELECT tool, workspace_id, promoted_epoch, last_used_epoch, use_count, updated_at
FROM promoted_tools WHERE repo_key = ?
ORDER BY tool ASC`, repoKey)
if err != nil {
return nil, fmt.Errorf("persistence: query promoted_tools: %w", err)
}
defer rows.Close()
var out []PromotedToolEntry
for rows.Next() {
var e PromotedToolEntry
var updatedAt int64
if err := rows.Scan(&e.Tool, &e.WorkspaceID, &e.PromotedEpoch,
&e.LastUsedEpoch, &e.UseCount, &updatedAt); err != nil {
return out, fmt.Errorf("persistence: scan promoted_tool: %w", err)
}
e.UpdatedAt = fromUnix(updatedAt)
out = append(out, e)
}
return out, rows.Err()
}
// UpsertPromotedTool writes (or replaces) one learned promotion.
func (s *SidecarStore) UpsertPromotedTool(repoKey string, e PromotedToolEntry) error {
s.writeMu.Lock()
defer s.writeMu.Unlock()
_, err := s.db.Exec(`
INSERT OR REPLACE INTO promoted_tools
(repo_key, tool, workspace_id, promoted_epoch, last_used_epoch, use_count, updated_at)
VALUES (?,?,?,?,?,?,?)`,
repoKey, e.Tool, e.WorkspaceID, e.PromotedEpoch, e.LastUsedEpoch,
e.UseCount, unixOrZero(e.UpdatedAt))
if err != nil {
return fmt.Errorf("persistence: upsert promoted_tool: %w", err)
}
return nil
}
// DeletePromotedTool drops one learned promotion (demotion). Missing rows
// are not errors.
func (s *SidecarStore) DeletePromotedTool(repoKey, tool string) error {
s.writeMu.Lock()
defer s.writeMu.Unlock()
_, err := s.db.Exec(`DELETE FROM promoted_tools WHERE repo_key = ? AND tool = ?`, repoKey, tool)
return err
}
// BumpSessionEpoch increments and returns the repo_key's session epoch — a
// monotonic counter advanced once per server startup, so "unused for N
// sessions" can be measured against last_used_epoch.
func (s *SidecarStore) BumpSessionEpoch(repoKey string) (int64, error) {
s.writeMu.Lock()
defer s.writeMu.Unlock()
if _, err := s.db.Exec(`
INSERT INTO tool_surface_meta (repo_key, session_epoch) VALUES (?, 1)
ON CONFLICT(repo_key) DO UPDATE SET session_epoch = session_epoch + 1`, repoKey); err != nil {
return 0, fmt.Errorf("persistence: bump session epoch: %w", err)
}
var epoch int64
if err := s.db.QueryRow(`SELECT session_epoch FROM tool_surface_meta WHERE repo_key = ?`, repoKey).Scan(&epoch); err != nil {
return 0, fmt.Errorf("persistence: read session epoch: %w", err)
}
return epoch, nil
}
+384
View File
@@ -0,0 +1,384 @@
package persistence
import (
"database/sql"
"fmt"
"time"
)
// Token-savings ledger tables. The savings ledger is machine-global —
// unlike notes/memories it carries no repo_key partition; per-repo
// attribution rides on the bucket key of savings_totals and the repo
// column of savings_events instead.
//
// Three tables, one job each:
// - savings_events: one row per recorded source-reading tool call.
// Durable at the call (single INSERT inside the observation tx), so
// a SIGKILLed server loses nothing — the property the flat-file
// ledger's batched flush could not give.
// - savings_totals: running aggregates keyed by bucket ('' top-line,
// 'repo:<prefix>', 'lang:<code>'), updated transactionally with the
// event insert so reads are point lookups instead of full scans.
// - savings_meta: first_seen / last_updated unix-nano stamps.
// SavingsEvent is one recorded source-reading observation.
type SavingsEvent struct {
TS time.Time
SessionID string
Tool string
Repo string
Language string
// Model is the LLM model that drove the call when known (resolved
// from the host's hook-supplied model hint); Client is the MCP
// client app from the initialize handshake. Both may be empty.
Model string
Client string
Returned int64
Saved int64
}
// SavingsTotalsRow is the aggregate for one savings_totals bucket.
type SavingsTotalsRow struct {
Saved int64
Returned int64
Calls int64
}
const savingsLegacyMigrationKind = "savings_files"
// AddSavingsObservation books one observation: the event row, the
// affected totals buckets, and the meta stamps, in a single transaction.
func (s *SidecarStore) AddSavingsObservation(ev SavingsEvent) error {
if s == nil {
return nil
}
s.writeMu.Lock()
defer s.writeMu.Unlock()
tx, err := s.db.Begin()
if err != nil {
return fmt.Errorf("persistence: savings tx: %w", err)
}
defer func() { _ = tx.Rollback() }()
ts := ev.TS
if ts.IsZero() {
ts = time.Now()
}
tsN := ts.UTC().UnixNano()
if _, err := tx.Exec(
`INSERT INTO savings_events (ts, session_id, tool, repo, language, model, client, returned, saved) VALUES (?,?,?,?,?,?,?,?,?)`,
tsN, ev.SessionID, ev.Tool, ev.Repo, ev.Language, ev.Model, ev.Client, ev.Returned, ev.Saved,
); err != nil {
return fmt.Errorf("persistence: savings event: %w", err)
}
buckets := []string{""}
if ev.Repo != "" {
buckets = append(buckets, "repo:"+ev.Repo)
}
if ev.Language != "" {
buckets = append(buckets, "lang:"+ev.Language)
}
for _, bucket := range buckets {
if err := upsertSavingsBucket(tx, bucket, ev.Saved, ev.Returned, 1); err != nil {
return err
}
}
if _, err := tx.Exec(
`INSERT INTO savings_meta (key, value) VALUES ('first_seen', ?) ON CONFLICT(key) DO NOTHING`, tsN,
); err != nil {
return fmt.Errorf("persistence: savings meta: %w", err)
}
if _, err := tx.Exec(
`INSERT INTO savings_meta (key, value) VALUES ('last_updated', ?) ON CONFLICT(key) DO UPDATE SET value = excluded.value`, tsN,
); err != nil {
return fmt.Errorf("persistence: savings meta: %w", err)
}
return tx.Commit()
}
func upsertSavingsBucket(tx *sql.Tx, bucket string, saved, returned, calls int64) error {
_, err := tx.Exec(
`INSERT INTO savings_totals (bucket, saved, returned, calls) VALUES (?,?,?,?)
ON CONFLICT(bucket) DO UPDATE SET
saved = savings_totals.saved + excluded.saved,
returned = savings_totals.returned + excluded.returned,
calls = savings_totals.calls + excluded.calls`,
bucket, saved, returned, calls,
)
if err != nil {
return fmt.Errorf("persistence: savings totals: %w", err)
}
return nil
}
// SavingsTotals returns every totals bucket plus the first_seen /
// last_updated stamps. Buckets map keys are ” (top-line),
// 'repo:<prefix>', and 'lang:<code>'. The zero time means "never".
func (s *SidecarStore) SavingsTotals() (map[string]SavingsTotalsRow, time.Time, time.Time, error) {
if s == nil {
return map[string]SavingsTotalsRow{}, time.Time{}, time.Time{}, nil
}
rows, err := s.db.Query(`SELECT bucket, saved, returned, calls FROM savings_totals`)
if err != nil {
return nil, time.Time{}, time.Time{}, fmt.Errorf("persistence: savings totals: %w", err)
}
defer rows.Close()
out := make(map[string]SavingsTotalsRow)
for rows.Next() {
var bucket string
var r SavingsTotalsRow
if err := rows.Scan(&bucket, &r.Saved, &r.Returned, &r.Calls); err != nil {
return nil, time.Time{}, time.Time{}, err
}
out[bucket] = r
}
if err := rows.Err(); err != nil {
return nil, time.Time{}, time.Time{}, err
}
return out, s.savingsMetaTime("first_seen"), s.savingsMetaTime("last_updated"), nil
}
func (s *SidecarStore) savingsMetaTime(key string) time.Time {
var n int64
row := s.db.QueryRow(`SELECT value FROM savings_meta WHERE key = ?`, key)
if err := row.Scan(&n); err != nil || n == 0 {
return time.Time{}
}
return time.Unix(0, n).UTC()
}
// SavingsEventsSince returns events with ts >= since, oldest first.
// since=zero returns everything.
func (s *SidecarStore) SavingsEventsSince(since time.Time) ([]SavingsEvent, error) {
if s == nil {
return nil, nil
}
rows, err := s.db.Query(
`SELECT ts, session_id, tool, repo, language, model, client, returned, saved
FROM savings_events WHERE ts >= ? ORDER BY ts, id`,
unixOrZero(since),
)
if err != nil {
return nil, fmt.Errorf("persistence: savings events: %w", err)
}
defer rows.Close()
var out []SavingsEvent
for rows.Next() {
var ev SavingsEvent
var tsN int64
if err := rows.Scan(&tsN, &ev.SessionID, &ev.Tool, &ev.Repo, &ev.Language, &ev.Model, &ev.Client, &ev.Returned, &ev.Saved); err != nil {
return nil, err
}
ev.TS = time.Unix(0, tsN).UTC()
out = append(out, ev)
}
return out, rows.Err()
}
// SavingsLegacyImportDone reports whether the one-shot flat-file
// (savings.json + savings.jsonl) import has already run.
func (s *SidecarStore) SavingsLegacyImportDone() bool {
if s == nil {
return true
}
return s.migrationDone("", savingsLegacyMigrationKind)
}
// ImportLegacySavings seeds the ledger from the flat-file era: bucket
// totals from the cumulative savings.json and event rows from the
// savings.jsonl log. Idempotent — guarded by a migration mark, which is
// set even for an empty import so the file probing never repeats. The
// mark is checked and written inside the import transaction, so two
// processes racing the first start (daemon + CLI) cannot both seed the
// ledger: the loser either sees the winner's mark or aborts on the
// write conflict. The caller owns reading (and afterwards renaming)
// the legacy files.
func (s *SidecarStore) ImportLegacySavings(buckets map[string]SavingsTotalsRow, firstSeen, lastUpdated time.Time, events []SavingsEvent) error {
if s == nil || s.migrationDone("", savingsLegacyMigrationKind) {
return nil
}
s.writeMu.Lock()
defer s.writeMu.Unlock()
tx, err := s.db.Begin()
if err != nil {
return fmt.Errorf("persistence: savings import tx: %w", err)
}
defer func() { _ = tx.Rollback() }()
// Re-check under the transaction — the cheap pre-check above races
// with other writers (in-process via writeMu it cannot, but another
// process opening the same database can).
var marked int
if err := tx.QueryRow(
`SELECT COUNT(1) FROM migration_marks WHERE repo_key = '' AND kind = ?`, savingsLegacyMigrationKind,
).Scan(&marked); err != nil {
return fmt.Errorf("persistence: savings import mark check: %w", err)
}
if marked > 0 {
return nil
}
for bucket, r := range buckets {
if err := upsertSavingsBucket(tx, bucket, r.Saved, r.Returned, r.Calls); err != nil {
return err
}
}
for _, ev := range events {
if _, err := tx.Exec(
`INSERT INTO savings_events (ts, session_id, tool, repo, language, model, client, returned, saved) VALUES (?,?,?,?,?,?,?,?,?)`,
unixOrZero(ev.TS), ev.SessionID, ev.Tool, ev.Repo, ev.Language, ev.Model, ev.Client, ev.Returned, ev.Saved,
); err != nil {
return fmt.Errorf("persistence: savings import event: %w", err)
}
}
if !firstSeen.IsZero() {
if _, err := tx.Exec(
`INSERT INTO savings_meta (key, value) VALUES ('first_seen', ?)
ON CONFLICT(key) DO UPDATE SET value = MIN(savings_meta.value, excluded.value)`,
unixOrZero(firstSeen),
); err != nil {
return fmt.Errorf("persistence: savings import meta: %w", err)
}
}
if !lastUpdated.IsZero() {
if _, err := tx.Exec(
`INSERT INTO savings_meta (key, value) VALUES ('last_updated', ?)
ON CONFLICT(key) DO UPDATE SET value = MAX(savings_meta.value, excluded.value)`,
unixOrZero(lastUpdated),
); err != nil {
return fmt.Errorf("persistence: savings import meta: %w", err)
}
}
if _, err := tx.Exec(
`INSERT OR REPLACE INTO migration_marks (repo_key, kind, done_at) VALUES ('', ?, ?)`,
savingsLegacyMigrationKind, time.Now().UTC().UnixNano(),
); err != nil {
return fmt.Errorf("persistence: savings import mark: %w", err)
}
return tx.Commit()
}
// SavingsToolTotals aggregates events per tool over ts >= since
// (zero = all time), tokens-saved descending — the dashboard breakdown
// without materializing the event history into Go.
func (s *SidecarStore) SavingsToolTotals(since time.Time) ([]SavingsToolRow, error) {
if s == nil {
return nil, nil
}
rows, err := s.db.Query(
`SELECT tool, SUM(saved), SUM(returned), COUNT(1)
FROM savings_events WHERE ts >= ?
GROUP BY tool ORDER BY SUM(saved) DESC, tool`,
unixOrZero(since),
)
if err != nil {
return nil, fmt.Errorf("persistence: savings tool totals: %w", err)
}
defer rows.Close()
var out []SavingsToolRow
for rows.Next() {
var r SavingsToolRow
if err := rows.Scan(&r.Tool, &r.Saved, &r.Returned, &r.Calls); err != nil {
return nil, err
}
out = append(out, r)
}
return out, rows.Err()
}
// SavingsToolRow is one per-tool aggregate row.
type SavingsToolRow struct {
Tool string
Saved int64
Returned int64
Calls int64
}
// SavingsDimRow is one row of a per-dimension (model / client) aggregate.
type SavingsDimRow struct {
Name string
Saved int64
Returned int64
Calls int64
}
// SavingsModelTotals aggregates events per attributed model over
// ts >= since (zero = all time), tokens-saved descending. Rows with no
// model attribution are excluded — this is the "per known model" view.
func (s *SidecarStore) SavingsModelTotals(since time.Time) ([]SavingsDimRow, error) {
return s.savingsDimTotals("model", since)
}
// SavingsClientTotals aggregates events per MCP client over ts >= since
// (zero = all time), tokens-saved descending. Rows with no client are
// excluded.
func (s *SidecarStore) SavingsClientTotals(since time.Time) ([]SavingsDimRow, error) {
return s.savingsDimTotals("client", since)
}
// savingsDimTotals is the shared GROUP BY for the model / client
// breakdowns. column is a fixed, caller-controlled identifier (never
// user input), so interpolating it into the statement is safe.
func (s *SidecarStore) savingsDimTotals(column string, since time.Time) ([]SavingsDimRow, error) {
if s == nil {
return nil, nil
}
rows, err := s.db.Query(
`SELECT `+column+`, SUM(saved), SUM(returned), COUNT(1)
FROM savings_events WHERE ts >= ? AND `+column+` <> ''
GROUP BY `+column+` ORDER BY SUM(saved) DESC, `+column,
unixOrZero(since),
)
if err != nil {
return nil, fmt.Errorf("persistence: savings %s totals: %w", column, err)
}
defer rows.Close()
var out []SavingsDimRow
for rows.Next() {
var r SavingsDimRow
if err := rows.Scan(&r.Name, &r.Saved, &r.Returned, &r.Calls); err != nil {
return nil, err
}
out = append(out, r)
}
return out, rows.Err()
}
// ResetSavings wipes the savings ledger (events, totals, meta) in one
// transaction, so a concurrent writer's observation either survives
// whole or is wiped whole — never an orphan event without totals. The
// legacy-import migration mark survives so renamed flat files are not
// re-imported after a reset.
func (s *SidecarStore) ResetSavings() error {
if s == nil {
return nil
}
s.writeMu.Lock()
defer s.writeMu.Unlock()
tx, err := s.db.Begin()
if err != nil {
return fmt.Errorf("persistence: savings reset tx: %w", err)
}
defer func() { _ = tx.Rollback() }()
for _, stmt := range []string{
`DELETE FROM savings_events`,
`DELETE FROM savings_totals`,
`DELETE FROM savings_meta`,
} {
if _, err := tx.Exec(stmt); err != nil {
return fmt.Errorf("persistence: savings reset: %w", err)
}
}
return tx.Commit()
}
@@ -0,0 +1,145 @@
package persistence
import (
"path/filepath"
"testing"
"time"
)
func openTestSidecar(t *testing.T) (*SidecarStore, string) {
t.Helper()
path := filepath.Join(t.TempDir(), "sidecar.sqlite")
sc, err := OpenSidecar(path)
if err != nil {
t.Fatal(err)
}
return sc, path
}
// The durability contract behind the savings ledger: an observation
// survives a full close + reopen of the database — no flush step exists
// to forget.
func TestSavings_DurableAcrossReopen(t *testing.T) {
sc, path := openTestSidecar(t)
ev := SavingsEvent{
TS: time.Date(2026, 6, 1, 10, 0, 0, 0, time.UTC),
SessionID: "sess-1",
Tool: "get_symbol_source",
Repo: "repo-a",
Language: "go",
Returned: 23,
Saved: 77,
}
if err := sc.AddSavingsObservation(ev); err != nil {
t.Fatal(err)
}
if err := sc.Close(); err != nil {
t.Fatal(err)
}
sc2, err := OpenSidecar(path)
if err != nil {
t.Fatal(err)
}
defer sc2.Close()
buckets, firstSeen, lastUpdated, err := sc2.SavingsTotals()
if err != nil {
t.Fatal(err)
}
top := buckets[""]
if top.Calls != 1 || top.Saved != 77 || top.Returned != 23 {
t.Errorf("top-line bucket = %+v, want calls=1 saved=77 returned=23", top)
}
if r := buckets["repo:repo-a"]; r.Calls != 1 {
t.Errorf("repo bucket = %+v, want calls=1", r)
}
if l := buckets["lang:go"]; l.Calls != 1 {
t.Errorf("lang bucket = %+v, want calls=1", l)
}
if !firstSeen.Equal(ev.TS) || !lastUpdated.Equal(ev.TS) {
t.Errorf("meta stamps = (%v, %v), want both %v", firstSeen, lastUpdated, ev.TS)
}
evs, err := sc2.SavingsEventsSince(time.Time{})
if err != nil {
t.Fatal(err)
}
if len(evs) != 1 || evs[0].SessionID != "sess-1" || !evs[0].TS.Equal(ev.TS) {
t.Errorf("reloaded events = %+v", evs)
}
}
func TestSavings_MetaStampsMinMax(t *testing.T) {
sc, _ := openTestSidecar(t)
defer sc.Close()
t1 := time.Date(2026, 6, 1, 10, 0, 0, 0, time.UTC)
t2 := t1.Add(time.Hour)
if err := sc.AddSavingsObservation(SavingsEvent{TS: t1, Tool: "a"}); err != nil {
t.Fatal(err)
}
if err := sc.AddSavingsObservation(SavingsEvent{TS: t2, Tool: "b"}); err != nil {
t.Fatal(err)
}
_, firstSeen, lastUpdated, err := sc.SavingsTotals()
if err != nil {
t.Fatal(err)
}
if !firstSeen.Equal(t1) {
t.Errorf("first_seen = %v, want %v (first observation wins)", firstSeen, t1)
}
if !lastUpdated.Equal(t2) {
t.Errorf("last_updated = %v, want %v (latest observation wins)", lastUpdated, t2)
}
}
func TestSavings_ResetClearsButKeepsImportMark(t *testing.T) {
sc, _ := openTestSidecar(t)
defer sc.Close()
if err := sc.ImportLegacySavings(
map[string]SavingsTotalsRow{"": {Saved: 100, Returned: 10, Calls: 1}},
time.Now().UTC(), time.Now().UTC(), nil,
); err != nil {
t.Fatal(err)
}
if !sc.SavingsLegacyImportDone() {
t.Fatal("import mark must be set after ImportLegacySavings")
}
if err := sc.ResetSavings(); err != nil {
t.Fatal(err)
}
buckets, firstSeen, _, err := sc.SavingsTotals()
if err != nil {
t.Fatal(err)
}
if len(buckets) != 0 || !firstSeen.IsZero() {
t.Errorf("reset must clear totals + meta, got buckets=%v firstSeen=%v", buckets, firstSeen)
}
if !sc.SavingsLegacyImportDone() {
t.Error("reset must NOT clear the legacy-import mark (renamed files would re-import)")
}
}
func TestSavings_ImportIsIdempotent(t *testing.T) {
sc, _ := openTestSidecar(t)
defer sc.Close()
rows := map[string]SavingsTotalsRow{"": {Saved: 100, Returned: 10, Calls: 1}}
if err := sc.ImportLegacySavings(rows, time.Time{}, time.Time{}, nil); err != nil {
t.Fatal(err)
}
if err := sc.ImportLegacySavings(rows, time.Time{}, time.Time{}, nil); err != nil {
t.Fatal(err)
}
buckets, _, _, err := sc.SavingsTotals()
if err != nil {
t.Fatal(err)
}
if got := buckets[""].Calls; got != 1 {
t.Errorf("calls after double import = %d, want 1", got)
}
}
File diff suppressed because it is too large Load Diff
+440
View File
@@ -0,0 +1,440 @@
package persistence
import (
"os"
"path/filepath"
"testing"
"time"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func openTempSidecar(t *testing.T) *SidecarStore {
t.Helper()
path := filepath.Join(t.TempDir(), "sidecar.sqlite")
st, err := OpenSidecar(path)
require.NoError(t, err)
require.NotNil(t, st)
t.Cleanup(func() { _ = st.Close() })
return st
}
func TestSidecar_OpenEmptyPathIsNoOp(t *testing.T) {
st, err := OpenSidecar("")
require.NoError(t, err)
require.Nil(t, st)
}
func TestSidecar_SameAbsPathReusesHandle(t *testing.T) {
path := filepath.Join(t.TempDir(), "sidecar.sqlite")
a, err := OpenSidecar(path)
require.NoError(t, err)
b, err := OpenSidecar(path)
require.NoError(t, err)
require.Same(t, a, b, "same absolute path must return the cached handle")
t.Cleanup(func() { _ = a.Close() })
}
func TestSidecar_NotesRoundTrip(t *testing.T) {
st := openTempSidecar(t)
now := time.Now().UTC().Truncate(time.Nanosecond)
in := NoteEntry{
ID: "nt-1",
Timestamp: now,
UpdatedAt: now,
SessionID: "sess-1",
ClientName: "claude-code",
Body: "decision: switch to fastpath",
SymbolID: "pkg/foo.go::Bar",
FilePath: "pkg/foo.go",
RepoPrefix: "core",
WorkspaceID: "ws-a",
ProjectID: "proj-a",
Tags: []string{"decision", "perf"},
AutoLinks: []string{"pkg/foo.go::Bar"},
Pinned: true,
}
require.NoError(t, st.UpsertNote("rk", in))
rows, err := st.LoadNotesRows("rk")
require.NoError(t, err)
require.Len(t, rows, 1)
got := rows[0]
assert.Equal(t, in.ID, got.ID)
assert.Equal(t, in.SessionID, got.SessionID)
assert.Equal(t, in.ClientName, got.ClientName)
assert.Equal(t, in.Body, got.Body)
assert.Equal(t, in.SymbolID, got.SymbolID)
assert.Equal(t, in.FilePath, got.FilePath)
assert.Equal(t, in.WorkspaceID, got.WorkspaceID)
assert.Equal(t, in.Tags, got.Tags)
assert.Equal(t, in.AutoLinks, got.AutoLinks)
assert.True(t, got.Pinned)
assert.WithinDuration(t, in.UpdatedAt, got.UpdatedAt, time.Microsecond)
// Scope isolation: another repo_key sees nothing.
other, err := st.LoadNotesRows("other")
require.NoError(t, err)
require.Empty(t, other)
// Delete.
require.NoError(t, st.DeleteNote("rk", "nt-1"))
rows, err = st.LoadNotesRows("rk")
require.NoError(t, err)
require.Empty(t, rows)
}
func TestSidecar_NotesTrimKeepsPinnedAndNewest(t *testing.T) {
st := openTempSidecar(t)
base := time.Now().UTC()
for i := 0; i < 10; i++ {
require.NoError(t, st.UpsertNote("rk", NoteEntry{
ID: noteID(i),
Timestamp: base.Add(time.Duration(i) * time.Second),
UpdatedAt: base.Add(time.Duration(i) * time.Second),
Pinned: i == 0 || i == 5,
}))
}
require.NoError(t, st.TrimNotes("rk", 6))
rows, err := st.LoadNotesRows("rk")
require.NoError(t, err)
require.Len(t, rows, 6)
ids := map[string]bool{}
for _, r := range rows {
ids[r.ID] = true
}
assert.True(t, ids[noteID(0)], "pinned[0] survives")
assert.True(t, ids[noteID(5)], "pinned[5] survives")
assert.True(t, ids[noteID(9)], "newest survives")
assert.False(t, ids[noteID(1)], "oldest non-pinned dropped")
}
func TestSidecar_MemoriesRoundTrip(t *testing.T) {
st := openTempSidecar(t)
now := time.Now().UTC()
in := MemoryEntry{
ID: "mem-1",
Timestamp: now,
UpdatedAt: now,
LastAccessed: now,
AccessCount: 7,
Body: "lock invariant for Bar",
Title: "Bar lock invariant",
Kind: "invariant",
Source: "manual",
Confidence: 0.8,
Importance: 5,
AuthorAgent: "claude-code",
SymbolIDs: []string{"pkg/foo.go::Bar"},
FilePaths: []string{"pkg/foo.go"},
AutoLinks: []string{"pkg/foo.go::Baz"},
Tags: []string{"invariant", "lock"},
WorkspaceID: "ws-a",
ProjectID: "proj-a",
RepoPrefix: "core",
Pinned: true,
SupersededBy: "mem-2",
}
require.NoError(t, st.UpsertMemory("rk", in))
rows, err := st.LoadMemoriesRows("rk")
require.NoError(t, err)
require.Len(t, rows, 1)
got := rows[0]
assert.Equal(t, in.ID, got.ID)
assert.Equal(t, in.Title, got.Title)
assert.Equal(t, in.Kind, got.Kind)
assert.Equal(t, in.Source, got.Source)
assert.InDelta(t, in.Confidence, got.Confidence, 1e-6)
assert.Equal(t, in.Importance, got.Importance)
assert.Equal(t, in.AuthorAgent, got.AuthorAgent)
assert.Equal(t, in.SymbolIDs, got.SymbolIDs)
assert.Equal(t, in.FilePaths, got.FilePaths)
assert.Equal(t, in.AutoLinks, got.AutoLinks)
assert.Equal(t, in.Tags, got.Tags)
assert.Equal(t, uint64(7), got.AccessCount)
assert.Equal(t, "mem-2", got.SupersededBy)
assert.True(t, got.Pinned)
require.NoError(t, st.DeleteMemory("rk", "mem-1"))
rows, err = st.LoadMemoriesRows("rk")
require.NoError(t, err)
require.Empty(t, rows)
}
func TestSidecar_MemoriesTrimTwoPass(t *testing.T) {
st := openTempSidecar(t)
base := time.Now().UTC()
for i := 0; i < 10; i++ {
e := MemoryEntry{
ID: memID(i),
Timestamp: base.Add(time.Duration(i) * time.Second),
UpdatedAt: base.Add(time.Duration(i) * time.Second),
Importance: 4,
}
if i == 2 || i == 4 {
e.Importance = 1
}
if i == 7 {
e.Pinned = true
e.Importance = 1
}
require.NoError(t, st.UpsertMemory("rk", e))
}
require.NoError(t, st.TrimMemories("rk", 6))
rows, err := st.LoadMemoriesRows("rk")
require.NoError(t, err)
require.Len(t, rows, 6)
ids := map[string]bool{}
for _, r := range rows {
ids[r.ID] = true
}
assert.True(t, ids[memID(7)], "pinned low-imp survives")
assert.False(t, ids[memID(2)], "low-imp dropped")
assert.False(t, ids[memID(4)], "low-imp dropped")
}
func TestSidecar_ScopesRoundTrip(t *testing.T) {
st := openTempSidecar(t)
require.NoError(t, st.UpsertScope(ScopeRow{
Name: "backend", Description: "be", Repos: []string{"api", "core"}, Paths: []string{"services/x"},
}))
require.NoError(t, st.UpsertScope(ScopeRow{Name: "frontend", Repos: []string{"web"}}))
rows, err := st.LoadScopes()
require.NoError(t, err)
require.Len(t, rows, 2)
assert.Equal(t, "backend", rows[0].Name)
assert.Equal(t, []string{"api", "core"}, rows[0].Repos)
assert.Equal(t, []string{"services/x"}, rows[0].Paths)
assert.Equal(t, 2, st.ScopeCount())
require.NoError(t, st.DeleteScope("backend"))
rows, err = st.LoadScopes()
require.NoError(t, err)
require.Len(t, rows, 1)
assert.Equal(t, "frontend", rows[0].Name)
}
func TestSidecar_NotebookRoundTrip(t *testing.T) {
st := openTempSidecar(t)
now := time.Now().UTC()
in := NotebookRow{
ID: "nb-1",
Title: "design: sidecar",
Body: "use sqlite\nfor durability",
Tags: []string{"design", "storage"},
SymbolIDs: []string{"pkg/p.go::Q"},
UsedCount: 3,
LastUsed: now,
Created: now,
Updated: now,
}
require.NoError(t, st.UpsertNotebook("rk", in))
got, ok := st.GetNotebookRow("rk", "nb-1")
require.True(t, ok)
assert.Equal(t, in.Title, got.Title)
assert.Equal(t, in.Body, got.Body)
assert.Equal(t, in.Tags, got.Tags)
assert.Equal(t, in.SymbolIDs, got.SymbolIDs)
assert.Equal(t, uint64(3), got.UsedCount)
rows, err := st.LoadNotebookRows("rk")
require.NoError(t, err)
require.Len(t, rows, 1)
require.NoError(t, st.DeleteNotebook("rk", "nb-1"))
_, ok = st.GetNotebookRow("rk", "nb-1")
require.False(t, ok)
}
func TestSidecar_NotebookPrune(t *testing.T) {
st := openTempSidecar(t)
old := time.Now().UTC().Add(-2 * time.Hour)
fresh := time.Now().UTC()
require.NoError(t, st.UpsertNotebook("rk", NotebookRow{ID: "stale", Updated: old}))
require.NoError(t, st.UpsertNotebook("rk", NotebookRow{ID: "fresh", Updated: fresh, LastUsed: fresh}))
require.NoError(t, st.NotebookPrune("rk", time.Now().UTC().Add(-time.Hour)))
rows, err := st.LoadNotebookRows("rk")
require.NoError(t, err)
require.Len(t, rows, 1)
assert.Equal(t, "fresh", rows[0].ID)
}
// ---------------------------------------------------------------------------
// Migration: legacy gob.gz / json → sqlite.
// ---------------------------------------------------------------------------
func TestSidecar_MigrateLegacyNotes(t *testing.T) {
legacyDir := t.TempDir()
require.NoError(t, SaveNotes(legacyDir, &NoteStore{Entries: []NoteEntry{
{ID: "nt-old", Body: "legacy note", SessionID: "s1", Pinned: true},
}}))
st := openTempSidecar(t)
require.NoError(t, st.MigrateLegacyNotes("rk", legacyDir))
rows, err := st.LoadNotesRows("rk")
require.NoError(t, err)
require.Len(t, rows, 1)
assert.Equal(t, "nt-old", rows[0].ID)
assert.Equal(t, "legacy note", rows[0].Body)
assert.True(t, rows[0].Pinned)
// Legacy file renamed to .bak.
_, errOrig := os.Stat(filepath.Join(legacyDir, notesFile))
assert.Error(t, errOrig, "original gob.gz must be renamed away")
_, errBak := os.Stat(filepath.Join(legacyDir, notesFile+".bak"))
assert.NoError(t, errBak, ".bak must exist")
// Idempotent: a second migrate is a no-op (no duplicate rows).
require.NoError(t, st.MigrateLegacyNotes("rk", legacyDir))
rows, err = st.LoadNotesRows("rk")
require.NoError(t, err)
require.Len(t, rows, 1)
}
func TestSidecar_MigrateLegacyMemories(t *testing.T) {
legacyDir := t.TempDir()
require.NoError(t, SaveMemories(legacyDir, &MemoryStore{Entries: []MemoryEntry{
{ID: "mem-old", Body: "legacy memory", Kind: "invariant", Importance: 5},
}}))
st := openTempSidecar(t)
require.NoError(t, st.MigrateLegacyMemories("rk", legacyDir))
rows, err := st.LoadMemoriesRows("rk")
require.NoError(t, err)
require.Len(t, rows, 1)
assert.Equal(t, "mem-old", rows[0].ID)
assert.Equal(t, "invariant", rows[0].Kind)
_, errBak := os.Stat(filepath.Join(legacyDir, memoriesFile+".bak"))
assert.NoError(t, errBak, ".bak must exist")
}
func TestSidecar_MigrateLegacyScopes(t *testing.T) {
legacyPath := filepath.Join(t.TempDir(), "scopes.json")
require.NoError(t, os.WriteFile(legacyPath, []byte(`[{"name":"be","description":"backend","repos":["api"],"paths":["svc/x"]}]`), 0o644))
st := openTempSidecar(t)
require.NoError(t, st.MigrateLegacyScopes(legacyPath))
rows, err := st.LoadScopes()
require.NoError(t, err)
require.Len(t, rows, 1)
assert.Equal(t, "be", rows[0].Name)
assert.Equal(t, []string{"api"}, rows[0].Repos)
assert.Equal(t, []string{"svc/x"}, rows[0].Paths)
_, errBak := os.Stat(legacyPath + ".bak")
assert.NoError(t, errBak)
// Idempotent.
require.NoError(t, st.MigrateLegacyScopes(legacyPath))
assert.Equal(t, 1, st.ScopeCount())
}
func TestSidecar_MigrateLegacyNotebook(t *testing.T) {
legacyDir := t.TempDir()
md := "---\ntitle: old entry\ntags: [a, b]\nused_count: 4\n---\n\nbody text\n"
require.NoError(t, os.WriteFile(filepath.Join(legacyDir, "nbold.md"), []byte(md), 0o644))
st := openTempSidecar(t)
importMD := func(id, contents string) (NotebookRow, bool) {
// Minimal frontmatter parse for the test importer.
return NotebookRow{ID: id, Title: "old entry", Body: "body text\n", Tags: []string{"a", "b"}, UsedCount: 4}, true
}
require.NoError(t, st.MigrateLegacyNotebook("rk", legacyDir, importMD))
rows, err := st.LoadNotebookRows("rk")
require.NoError(t, err)
require.Len(t, rows, 1)
assert.Equal(t, "nbold", rows[0].ID)
assert.Equal(t, "old entry", rows[0].Title)
assert.Equal(t, uint64(4), rows[0].UsedCount)
_, errBak := os.Stat(filepath.Join(legacyDir, "nbold.md.bak"))
assert.NoError(t, errBak)
}
func TestSidecar_MigrateSkippedWhenTableNonEmpty(t *testing.T) {
legacyDir := t.TempDir()
require.NoError(t, SaveNotes(legacyDir, &NoteStore{Entries: []NoteEntry{{ID: "nt-old", Body: "legacy"}}}))
st := openTempSidecar(t)
// Pre-seed the table so the import is skipped (guard on existing rows).
require.NoError(t, st.UpsertNote("rk", NoteEntry{ID: "nt-existing", Body: "already here"}))
require.NoError(t, st.MigrateLegacyNotes("rk", legacyDir))
rows, err := st.LoadNotesRows("rk")
require.NoError(t, err)
require.Len(t, rows, 1, "import must be skipped when the table already has rows")
assert.Equal(t, "nt-existing", rows[0].ID)
}
func TestSidecar_SuppressionsRoundTrip(t *testing.T) {
st := openTempSidecar(t)
now := time.Now().UTC().Truncate(time.Nanosecond)
in := SuppressionEntry{
IdentityKey: "ik-abc",
Rule: "nil-deref",
Category: "nil-deref",
File: "pkg/x.go",
SymbolID: "pkg/x.go::Foo",
Reason: "guarded by a prior nil check",
Author: "me@zzet.org",
HitCount: 0,
Created: now,
}
require.NoError(t, st.UpsertSuppression("rk", in))
// LoadSuppression hits.
got, ok := st.LoadSuppression("rk", "ik-abc")
require.True(t, ok)
assert.Equal(t, in.IdentityKey, got.IdentityKey)
assert.Equal(t, in.Rule, got.Rule)
assert.Equal(t, in.Category, got.Category)
assert.Equal(t, in.File, got.File)
assert.Equal(t, in.SymbolID, got.SymbolID)
assert.Equal(t, in.Reason, got.Reason)
assert.Equal(t, in.Author, got.Author)
assert.WithinDuration(t, now, got.Created, time.Microsecond)
// Missing identity → clean miss, not an error.
_, ok = st.LoadSuppression("rk", "nope")
assert.False(t, ok)
// Scope isolation: another repo_key sees nothing.
_, ok = st.LoadSuppression("other", "ik-abc")
assert.False(t, ok)
// LoadSuppressions for the repo.
all, err := st.LoadSuppressions("rk")
require.NoError(t, err)
require.Len(t, all, 1)
assert.Equal(t, "ik-abc", all[0].IdentityKey)
// Bump hit twice and confirm counters advance.
require.NoError(t, st.BumpSuppressionHit("rk", "ik-abc", now.Add(time.Second)))
require.NoError(t, st.BumpSuppressionHit("rk", "ik-abc", now.Add(2*time.Second)))
bumped, ok := st.LoadSuppression("rk", "ik-abc")
require.True(t, ok)
assert.Equal(t, int64(2), bumped.HitCount)
assert.WithinDuration(t, now.Add(2*time.Second), bumped.LastHit, time.Microsecond)
// Bumping a missing row is a no-op, not an error.
require.NoError(t, st.BumpSuppressionHit("rk", "nope", now))
// Delete.
require.NoError(t, st.DeleteSuppression("rk", "ik-abc"))
_, ok = st.LoadSuppression("rk", "ik-abc")
assert.False(t, ok)
// Deleting a missing row is not an error.
require.NoError(t, st.DeleteSuppression("rk", "ik-abc"))
}
+71
View File
@@ -0,0 +1,71 @@
// Package persistence provides snapshot-based graph persistence with pluggable backends.
package persistence
import (
"errors"
"time"
"github.com/zzet/gortex/internal/graph"
)
// ErrNotFound is returned when no snapshot exists for the given key.
var ErrNotFound = errors.New("persistence: snapshot not found")
// Snapshot is the unit of persistence — a complete graph state at a point in time.
// CurrentExtractionVersion is the version of the extraction OUTPUT schema. It
// is bumped ONLY when a binary actually changes what extraction produces (a new
// node/edge shape, a corrected graph) — never on a routine version bump. A warm
// snapshot stays reusable across binary releases that share this version, so a
// no-op release does not force the full cold rebuild a binary-version-string
// gate would. Raise it in lockstep with any change that alters indexed output.
const CurrentExtractionVersion = 1
type Snapshot struct {
Version string `json:"version"`
// ExtractionVersion is the CurrentExtractionVersion the snapshot was
// produced with; warm reuse is gated on it, not the binary version string.
ExtractionVersion int `json:"extraction_version,omitempty"`
RepoPath string `json:"repo_path"`
CommitHash string `json:"commit_hash"`
// Branch is the git branch the snapshot was taken on. Snapshots
// are keyed by (repo, branch) so a slot survives commits on the
// branch; empty for a detached HEAD (keyed by commit instead).
Branch string `json:"branch,omitempty"`
IndexedAt time.Time `json:"indexed_at"`
Nodes []*graph.Node `json:"nodes"`
Edges []*graph.Edge `json:"edges"`
FileMtimes map[string]int64 `json:"file_mtimes"`
// VectorIndex is the serialized HNSW vector index (nil when embeddings are disabled).
VectorIndex []byte `json:"vector_index,omitempty"`
// VectorDims is the embedding dimensionality (0 when embeddings are disabled).
VectorDims int `json:"vector_dims,omitempty"`
// VectorCount is the number of vectors in the index.
VectorCount int `json:"vector_count,omitempty"`
}
// Store is the pluggable persistence backend interface.
// Implementations must be safe for sequential use (not required to be concurrent).
type Store interface {
// Check returns true if a snapshot exists for the (repo, branch)
// slot. commitHash is used only as the slot key for a detached
// HEAD (empty branch).
Check(repoPath, branch, commitHash string) bool
// Load deserializes and returns the snapshot for the (repo,
// branch) slot. Returns ErrNotFound if no snapshot exists.
Load(repoPath, branch, commitHash string) (*Snapshot, error)
// Save serializes the snapshot to persistent storage.
Save(snap *Snapshot) error
// Validate checks that an existing snapshot is compatible with
// the current gortex version. Returns false on version mismatch.
Validate(repoPath, branch, commitHash string) bool
// Evict removes the snapshot for the (repo, branch) slot.
Evict(repoPath, branch, commitHash string) error
// Close releases any resources held by the backend.
Close() error
}