chore: import upstream snapshot with attribution
CI / benchmark (push) Has been skipped
install-script / posix-syntax (push) Successful in 6m1s
CI / build-onnx (push) Failing after 6m43s
init-smoke / dry-run (push) Failing after 15m57s
security / govulncheck (push) Has been cancelled
security / trivy-fs (push) Has been cancelled
CI / test (1.26, ubuntu-latest) (push) Has been cancelled
Scorecard supply-chain security / Scorecard analysis (push) Has been cancelled
CI / test (1.26, macos-latest) (push) Has been cancelled
CI / build-windows (push) Has been cancelled
CI / lint (push) Has been cancelled
install-script / powershell-syntax (push) Has been cancelled
install-script / install (macos-14) (push) Has been cancelled
install-script / install (ubuntu-latest) (push) Has been cancelled
CI / benchmark (push) Has been skipped
install-script / posix-syntax (push) Successful in 6m1s
CI / build-onnx (push) Failing after 6m43s
init-smoke / dry-run (push) Failing after 15m57s
security / govulncheck (push) Has been cancelled
security / trivy-fs (push) Has been cancelled
CI / test (1.26, ubuntu-latest) (push) Has been cancelled
Scorecard supply-chain security / Scorecard analysis (push) Has been cancelled
CI / test (1.26, macos-latest) (push) Has been cancelled
CI / build-windows (push) Has been cancelled
CI / lint (push) Has been cancelled
install-script / powershell-syntax (push) Has been cancelled
install-script / install (macos-14) (push) Has been cancelled
install-script / install (ubuntu-latest) (push) Has been cancelled
This commit is contained in:
@@ -0,0 +1,113 @@
|
||||
package persistence
|
||||
|
||||
import (
|
||||
"compress/gzip"
|
||||
"encoding/gob"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"time"
|
||||
)
|
||||
|
||||
const (
|
||||
comboFile = "combo.gob.gz"
|
||||
maxComboQueries = 2000
|
||||
maxComboEntriesPerQuery = 20
|
||||
)
|
||||
|
||||
// ComboMatch records one (query → symbol) association. HitCount is how many
|
||||
// times the agent picked this symbol following the same normalized query;
|
||||
// LastUsed is a unix timestamp (seconds) for decay and reaping.
|
||||
type ComboMatch struct {
|
||||
SymbolID string
|
||||
HitCount uint32
|
||||
LastUsed int64
|
||||
}
|
||||
|
||||
// ComboQuery holds all recorded matches for one normalized query string
|
||||
// within a single repo. Ordered most-hit-first after any record.
|
||||
type ComboQuery struct {
|
||||
Query string
|
||||
Matches []ComboMatch
|
||||
}
|
||||
|
||||
// ComboStore is the persisted state of the query→symbol combo tracker for
|
||||
// one repo. Separate file from feedback so each subsystem can evolve its
|
||||
// schema independently.
|
||||
type ComboStore struct {
|
||||
Version string
|
||||
RepoPath string
|
||||
Queries []ComboQuery
|
||||
}
|
||||
|
||||
// ComboDir returns the on-disk directory for combo storage. Shares the
|
||||
// repo cache key with feedback so all repo-scoped state lives together.
|
||||
func ComboDir(cacheDir, repoPath string) string {
|
||||
return filepath.Join(cacheDir, RepoCacheKey(repoPath))
|
||||
}
|
||||
|
||||
// LoadCombo reads the combo store from disk. Missing file is not an error.
|
||||
func LoadCombo(dir string) (*ComboStore, error) {
|
||||
path := filepath.Join(dir, comboFile)
|
||||
f, err := os.Open(path)
|
||||
if err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
return &ComboStore{}, nil
|
||||
}
|
||||
return nil, fmt.Errorf("persistence: open combo: %w", err)
|
||||
}
|
||||
defer f.Close()
|
||||
|
||||
gz, err := gzip.NewReader(f)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("persistence: gzip reader combo: %w", err)
|
||||
}
|
||||
defer gz.Close()
|
||||
|
||||
var store ComboStore
|
||||
if err := gob.NewDecoder(gz).Decode(&store); err != nil {
|
||||
return nil, fmt.Errorf("persistence: gob decode combo: %w", err)
|
||||
}
|
||||
return &store, nil
|
||||
}
|
||||
|
||||
// SaveCombo writes the combo store to disk with gob+gzip compression. Trims
|
||||
// the oldest-used queries if over cap so the file can't grow unboundedly on
|
||||
// a long-running daemon.
|
||||
func SaveCombo(dir string, store *ComboStore) error {
|
||||
if len(store.Queries) > maxComboQueries {
|
||||
// Keep the queries touched most recently by scanning their matches.
|
||||
// Stable order (no sort) preserves the common case where callers
|
||||
// already maintain MRU; only trigger a full scan when we'd otherwise
|
||||
// drop unbounded.
|
||||
trim := len(store.Queries) - maxComboQueries
|
||||
store.Queries = store.Queries[trim:]
|
||||
}
|
||||
if err := os.MkdirAll(dir, 0o755); err != nil {
|
||||
return fmt.Errorf("persistence: mkdir combo: %w", err)
|
||||
}
|
||||
path := filepath.Join(dir, comboFile)
|
||||
f, err := os.Create(path)
|
||||
if err != nil {
|
||||
return fmt.Errorf("persistence: create combo: %w", err)
|
||||
}
|
||||
gz := gzip.NewWriter(f)
|
||||
enc := gob.NewEncoder(gz)
|
||||
if err := enc.Encode(store); err != nil {
|
||||
_ = gz.Close()
|
||||
_ = f.Close()
|
||||
return fmt.Errorf("persistence: gob encode combo: %w", err)
|
||||
}
|
||||
if err := gz.Close(); err != nil {
|
||||
_ = f.Close()
|
||||
return fmt.Errorf("persistence: gzip close combo: %w", err)
|
||||
}
|
||||
return f.Close()
|
||||
}
|
||||
|
||||
// MaxComboEntries returns the cap on matches per query. Exported so the
|
||||
// manager can enforce the same limit in-memory before flushing.
|
||||
func MaxComboEntries() int { return maxComboEntriesPerQuery }
|
||||
|
||||
// Now is overridable for tests; used when SaveCombo needs to timestamp.
|
||||
var Now = func() time.Time { return time.Now() }
|
||||
@@ -0,0 +1,125 @@
|
||||
package persistence
|
||||
|
||||
import (
|
||||
"compress/gzip"
|
||||
"crypto/sha256"
|
||||
"encoding/gob"
|
||||
"encoding/hex"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"time"
|
||||
|
||||
"github.com/zzet/gortex/internal/pathkey"
|
||||
)
|
||||
|
||||
const (
|
||||
feedbackFile = "feedback.gob.gz"
|
||||
maxFeedbackCap = 500
|
||||
)
|
||||
|
||||
// FeedbackEntry records one agent feedback event about a context call.
|
||||
type FeedbackEntry struct {
|
||||
Timestamp time.Time
|
||||
Task string // task description from the context call
|
||||
Useful []string // symbol IDs the agent found useful
|
||||
NotNeeded []string // symbol IDs returned but not needed
|
||||
Missing []string // symbol IDs that should have been included
|
||||
Source string // "smart_context" or "prefetch_context"
|
||||
// Keywords is the task's keyword cluster (derived from Task at
|
||||
// record time). Feedback is scored only against entries whose
|
||||
// keyword cluster overlaps the querying task's, so a symbol marked
|
||||
// useful for one task does not contaminate an unrelated one. A nil
|
||||
// Keywords (legacy entry, or a keyword-less task) is treated as
|
||||
// matching any query for backward compatibility.
|
||||
Keywords []string
|
||||
}
|
||||
|
||||
// FeedbackStore holds all feedback entries for a single repo.
|
||||
// Persisted separately from the graph snapshot (repo-scoped, not commit-scoped).
|
||||
type FeedbackStore struct {
|
||||
Version string
|
||||
RepoPath string
|
||||
Entries []FeedbackEntry
|
||||
}
|
||||
|
||||
// RepoCacheKey produces a filesystem-safe directory name from repo path alone
|
||||
// (no commit hash). Used for data that persists across commits, like feedback.
|
||||
//
|
||||
// The path is folded to Unicode NFC before hashing so a repo whose path
|
||||
// contains non-ASCII characters keys to the same directory whether the
|
||||
// caller passes a decomposed (macOS) or precomposed (Linux / git) form.
|
||||
func RepoCacheKey(repoPath string) string {
|
||||
abs, err := filepath.Abs(repoPath)
|
||||
if err != nil {
|
||||
abs = repoPath
|
||||
}
|
||||
h := sha256.Sum256([]byte(pathkey.Normalize(abs)))
|
||||
return hex.EncodeToString(h[:6]) + "_latest"
|
||||
}
|
||||
|
||||
// FeedbackDir returns the full directory path for feedback storage.
|
||||
func FeedbackDir(cacheDir, repoPath string) string {
|
||||
return filepath.Join(cacheDir, RepoCacheKey(repoPath))
|
||||
}
|
||||
|
||||
// LoadFeedback reads a feedback store from disk. Returns an empty store if
|
||||
// the file does not exist (not an error — cold start is normal).
|
||||
func LoadFeedback(dir string) (*FeedbackStore, error) {
|
||||
path := filepath.Join(dir, feedbackFile)
|
||||
f, err := os.Open(path)
|
||||
if err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
return &FeedbackStore{}, nil
|
||||
}
|
||||
return nil, fmt.Errorf("persistence: open feedback: %w", err)
|
||||
}
|
||||
defer f.Close()
|
||||
|
||||
gz, err := gzip.NewReader(f)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("persistence: gzip reader: %w", err)
|
||||
}
|
||||
defer gz.Close()
|
||||
|
||||
var store FeedbackStore
|
||||
if err := gob.NewDecoder(gz).Decode(&store); err != nil {
|
||||
return nil, fmt.Errorf("persistence: gob decode feedback: %w", err)
|
||||
}
|
||||
return &store, nil
|
||||
}
|
||||
|
||||
// SaveFeedback writes a feedback store to disk with gob+gzip compression.
|
||||
// Entries are trimmed to maxFeedbackCap before writing (oldest removed).
|
||||
func SaveFeedback(dir string, store *FeedbackStore) error {
|
||||
// Trim oldest entries if over cap.
|
||||
if len(store.Entries) > maxFeedbackCap {
|
||||
store.Entries = store.Entries[len(store.Entries)-maxFeedbackCap:]
|
||||
}
|
||||
|
||||
if err := os.MkdirAll(dir, 0o755); err != nil {
|
||||
return fmt.Errorf("persistence: mkdir feedback: %w", err)
|
||||
}
|
||||
|
||||
path := filepath.Join(dir, feedbackFile)
|
||||
f, err := os.Create(path)
|
||||
if err != nil {
|
||||
return fmt.Errorf("persistence: create feedback: %w", err)
|
||||
}
|
||||
|
||||
gz := gzip.NewWriter(f)
|
||||
enc := gob.NewEncoder(gz)
|
||||
|
||||
if err := enc.Encode(store); err != nil {
|
||||
_ = gz.Close()
|
||||
_ = f.Close()
|
||||
return fmt.Errorf("persistence: gob encode feedback: %w", err)
|
||||
}
|
||||
|
||||
if err := gz.Close(); err != nil {
|
||||
_ = f.Close()
|
||||
return fmt.Errorf("persistence: gzip close feedback: %w", err)
|
||||
}
|
||||
|
||||
return f.Close()
|
||||
}
|
||||
@@ -0,0 +1,106 @@
|
||||
package persistence
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func testFeedbackStore() *FeedbackStore {
|
||||
return &FeedbackStore{
|
||||
Version: "0.1.0-test",
|
||||
RepoPath: "/tmp/test-repo",
|
||||
Entries: []FeedbackEntry{
|
||||
{
|
||||
Timestamp: time.Now().Truncate(time.Second),
|
||||
Task: "add new MCP tool",
|
||||
Useful: []string{"server.go::Server", "tools_core.go::registerCoreTools"},
|
||||
NotNeeded: []string{"types.go::NodeKind"},
|
||||
Missing: []string{"tools_enhancements.go::registerEnhancementTools"},
|
||||
Source: "smart_context",
|
||||
},
|
||||
{
|
||||
Timestamp: time.Now().Add(-time.Hour).Truncate(time.Second),
|
||||
Task: "fix bug in search",
|
||||
Useful: []string{"search.go::Search"},
|
||||
Source: "prefetch_context",
|
||||
},
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func TestFeedback_RoundTrip(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
store := testFeedbackStore()
|
||||
|
||||
require.NoError(t, SaveFeedback(dir, store))
|
||||
|
||||
loaded, err := LoadFeedback(dir)
|
||||
require.NoError(t, err)
|
||||
|
||||
assert.Equal(t, store.Version, loaded.Version)
|
||||
assert.Equal(t, store.RepoPath, loaded.RepoPath)
|
||||
require.Len(t, loaded.Entries, 2)
|
||||
|
||||
assert.Equal(t, "add new MCP tool", loaded.Entries[0].Task)
|
||||
assert.Equal(t, []string{"server.go::Server", "tools_core.go::registerCoreTools"}, loaded.Entries[0].Useful)
|
||||
assert.Equal(t, []string{"types.go::NodeKind"}, loaded.Entries[0].NotNeeded)
|
||||
assert.Equal(t, []string{"tools_enhancements.go::registerEnhancementTools"}, loaded.Entries[0].Missing)
|
||||
assert.Equal(t, "smart_context", loaded.Entries[0].Source)
|
||||
|
||||
assert.Equal(t, "fix bug in search", loaded.Entries[1].Task)
|
||||
assert.Equal(t, "prefetch_context", loaded.Entries[1].Source)
|
||||
}
|
||||
|
||||
func TestFeedback_ColdStart(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
loaded, err := LoadFeedback(dir)
|
||||
require.NoError(t, err)
|
||||
assert.NotNil(t, loaded)
|
||||
assert.Empty(t, loaded.Entries)
|
||||
}
|
||||
|
||||
func TestFeedback_TrimOldEntries(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
store := &FeedbackStore{
|
||||
Version: "0.1.0",
|
||||
RepoPath: "/tmp/test",
|
||||
}
|
||||
|
||||
// Create 510 entries.
|
||||
for range 510 {
|
||||
store.Entries = append(store.Entries, FeedbackEntry{
|
||||
Timestamp: time.Now().Truncate(time.Second),
|
||||
Task: "task",
|
||||
Useful: []string{"sym"},
|
||||
Source: "smart_context",
|
||||
})
|
||||
}
|
||||
|
||||
require.NoError(t, SaveFeedback(dir, store))
|
||||
|
||||
loaded, err := LoadFeedback(dir)
|
||||
require.NoError(t, err)
|
||||
assert.Len(t, loaded.Entries, 500)
|
||||
}
|
||||
|
||||
func TestRepoCacheKey_Stable(t *testing.T) {
|
||||
key1 := RepoCacheKey("/tmp/my-repo")
|
||||
key2 := RepoCacheKey("/tmp/my-repo")
|
||||
assert.Equal(t, key1, key2)
|
||||
assert.Contains(t, key1, "_latest")
|
||||
}
|
||||
|
||||
func TestRepoCacheKey_DifferentRepos(t *testing.T) {
|
||||
key1 := RepoCacheKey("/tmp/repo-a")
|
||||
key2 := RepoCacheKey("/tmp/repo-b")
|
||||
assert.NotEqual(t, key1, key2)
|
||||
}
|
||||
|
||||
func TestFeedbackDir(t *testing.T) {
|
||||
dir := FeedbackDir("/home/user/.cache/gortex", "/tmp/my-repo")
|
||||
assert.Contains(t, dir, "_latest")
|
||||
assert.Contains(t, dir, ".cache/gortex")
|
||||
}
|
||||
@@ -0,0 +1,212 @@
|
||||
package persistence
|
||||
|
||||
import (
|
||||
"compress/gzip"
|
||||
"encoding/gob"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"github.com/gofrs/flock"
|
||||
|
||||
"github.com/zzet/gortex/internal/platform"
|
||||
)
|
||||
|
||||
func init() {
|
||||
// Register concrete types that appear in Node.Meta / Edge.Meta map[string]any.
|
||||
gob.Register(map[string]any{})
|
||||
gob.Register([]any{})
|
||||
gob.Register([]string{})
|
||||
gob.Register([]int{})
|
||||
gob.Register([]map[string]string{})
|
||||
gob.Register([]map[string]any{})
|
||||
}
|
||||
|
||||
const (
|
||||
snapshotFile = "snapshot.gob.gz"
|
||||
versionFile = ".version"
|
||||
extractionVersionFile = ".extraction_version"
|
||||
)
|
||||
|
||||
// FileStore persists snapshots as gob+gzip files in a directory hierarchy.
|
||||
// Layout: {dir}/{cacheKey}/snapshot.gob.gz + .version
|
||||
type FileStore struct {
|
||||
dir string
|
||||
version string
|
||||
}
|
||||
|
||||
// NewFileStore creates a file-based persistence store.
|
||||
// If dir is empty, defaults to the Gortex cache dir (~/.gortex/cache/,
|
||||
// or the $XDG_CACHE_HOME equivalent when that variable is set).
|
||||
func NewFileStore(dir, version string) (*FileStore, error) {
|
||||
if dir == "" {
|
||||
dir = platform.CacheDir()
|
||||
}
|
||||
return &FileStore{dir: dir, version: version}, nil
|
||||
}
|
||||
|
||||
func (fs *FileStore) entryDir(repoPath, branch, commitHash string) string {
|
||||
return filepath.Join(fs.dir, CacheKey(repoPath, branch, commitHash))
|
||||
}
|
||||
|
||||
// lockPath is the advisory-lock file for one snapshot entry. It is a
|
||||
// sibling of the entry directory ({dir}/{cacheKey}.lock), deliberately
|
||||
// outside it so the lock survives the os.RemoveAll a writer runs against
|
||||
// the entry directory on Save/Evict.
|
||||
func (fs *FileStore) lockPath(repoPath, branch, commitHash string) string {
|
||||
return fs.entryDir(repoPath, branch, commitHash) + ".lock"
|
||||
}
|
||||
|
||||
// acquireWrite takes an exclusive cross-process advisory lock for one
|
||||
// snapshot entry. A second gortex process writing the same snapshot
|
||||
// blocks here instead of racing the RemoveAll/MkdirAll/write sequence,
|
||||
// which would otherwise leave a torn or empty index on disk.
|
||||
func (fs *FileStore) acquireWrite(repoPath, branch, commitHash string) (*flock.Flock, error) {
|
||||
if err := os.MkdirAll(fs.dir, 0o755); err != nil {
|
||||
return nil, fmt.Errorf("persistence: mkdir cache dir: %w", err)
|
||||
}
|
||||
fl := flock.New(fs.lockPath(repoPath, branch, commitHash))
|
||||
if err := fl.Lock(); err != nil {
|
||||
return nil, fmt.Errorf("persistence: acquire index write lock: %w", err)
|
||||
}
|
||||
return fl, nil
|
||||
}
|
||||
|
||||
// acquireRead takes a shared cross-process advisory lock for one snapshot
|
||||
// entry, so a reader waits out an in-progress write instead of decoding a
|
||||
// half-written snapshot.
|
||||
func (fs *FileStore) acquireRead(repoPath, branch, commitHash string) (*flock.Flock, error) {
|
||||
if err := os.MkdirAll(fs.dir, 0o755); err != nil {
|
||||
return nil, fmt.Errorf("persistence: mkdir cache dir: %w", err)
|
||||
}
|
||||
fl := flock.New(fs.lockPath(repoPath, branch, commitHash))
|
||||
if err := fl.RLock(); err != nil {
|
||||
return nil, fmt.Errorf("persistence: acquire index read lock: %w", err)
|
||||
}
|
||||
return fl, nil
|
||||
}
|
||||
|
||||
func (fs *FileStore) Check(repoPath, branch, commitHash string) bool {
|
||||
dir := fs.entryDir(repoPath, branch, commitHash)
|
||||
info, err := os.Stat(dir)
|
||||
if err != nil || !info.IsDir() {
|
||||
return false
|
||||
}
|
||||
_, err = os.Stat(filepath.Join(dir, versionFile))
|
||||
return err == nil
|
||||
}
|
||||
|
||||
func (fs *FileStore) Validate(repoPath, branch, commitHash string) bool {
|
||||
dir := fs.entryDir(repoPath, branch, commitHash)
|
||||
// Extraction-version gate: a warm snapshot is reusable across binary
|
||||
// releases that produce the SAME extraction output, so a no-op version
|
||||
// bump no longer forces a full cold rebuild.
|
||||
if data, err := os.ReadFile(filepath.Join(dir, extractionVersionFile)); err == nil {
|
||||
v, perr := strconv.Atoi(strings.TrimSpace(string(data)))
|
||||
return perr == nil && v == CurrentExtractionVersion
|
||||
}
|
||||
// Legacy slot with no extraction-version file: fall back to the exact
|
||||
// binary-version match so pre-existing snapshots still validate (and get
|
||||
// rewritten with an extraction-version marker on the next save).
|
||||
data, err := os.ReadFile(filepath.Join(dir, versionFile))
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
return strings.TrimSpace(string(data)) == fs.version
|
||||
}
|
||||
|
||||
func (fs *FileStore) Load(repoPath, branch, commitHash string) (*Snapshot, error) {
|
||||
fl, err := fs.acquireRead(repoPath, branch, commitHash)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer func() { _ = fl.Unlock() }()
|
||||
|
||||
dir := fs.entryDir(repoPath, branch, commitHash)
|
||||
f, err := os.Open(filepath.Join(dir, snapshotFile))
|
||||
if err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
return nil, ErrNotFound
|
||||
}
|
||||
return nil, fmt.Errorf("persistence: open snapshot: %w", err)
|
||||
}
|
||||
defer f.Close()
|
||||
|
||||
gz, err := gzip.NewReader(f)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("persistence: gzip reader: %w", err)
|
||||
}
|
||||
defer gz.Close()
|
||||
|
||||
var snap Snapshot
|
||||
if err := gob.NewDecoder(gz).Decode(&snap); err != nil {
|
||||
return nil, fmt.Errorf("persistence: gob decode: %w", err)
|
||||
}
|
||||
|
||||
return &snap, nil
|
||||
}
|
||||
|
||||
func (fs *FileStore) Save(snap *Snapshot) error {
|
||||
fl, err := fs.acquireWrite(snap.RepoPath, snap.Branch, snap.CommitHash)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer func() { _ = fl.Unlock() }()
|
||||
|
||||
dir := fs.entryDir(snap.RepoPath, snap.Branch, snap.CommitHash)
|
||||
|
||||
// Remove old entry if it exists.
|
||||
_ = os.RemoveAll(dir)
|
||||
|
||||
if err := os.MkdirAll(dir, 0o755); err != nil {
|
||||
return fmt.Errorf("persistence: mkdir: %w", err)
|
||||
}
|
||||
|
||||
// Write snapshot.
|
||||
f, err := os.Create(filepath.Join(dir, snapshotFile))
|
||||
if err != nil {
|
||||
return fmt.Errorf("persistence: create snapshot: %w", err)
|
||||
}
|
||||
|
||||
gz := gzip.NewWriter(f)
|
||||
enc := gob.NewEncoder(gz)
|
||||
|
||||
if err := enc.Encode(snap); err != nil {
|
||||
_ = gz.Close()
|
||||
_ = f.Close()
|
||||
return fmt.Errorf("persistence: gob encode: %w", err)
|
||||
}
|
||||
|
||||
if err := gz.Close(); err != nil {
|
||||
_ = f.Close()
|
||||
return fmt.Errorf("persistence: gzip close: %w", err)
|
||||
}
|
||||
|
||||
if err := f.Close(); err != nil {
|
||||
return fmt.Errorf("persistence: file close: %w", err)
|
||||
}
|
||||
|
||||
// Write version file (binary version — kept for display / diagnostics).
|
||||
if err := os.WriteFile(filepath.Join(dir, versionFile), []byte(fs.version), 0o644); err != nil {
|
||||
return fmt.Errorf("persistence: write version: %w", err)
|
||||
}
|
||||
// Write the extraction-version marker — the value Validate gates reuse on.
|
||||
if err := os.WriteFile(filepath.Join(dir, extractionVersionFile), []byte(strconv.Itoa(CurrentExtractionVersion)), 0o644); err != nil {
|
||||
return fmt.Errorf("persistence: write extraction version: %w", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (fs *FileStore) Evict(repoPath, branch, commitHash string) error {
|
||||
fl, err := fs.acquireWrite(repoPath, branch, commitHash)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer func() { _ = fl.Unlock() }()
|
||||
return os.RemoveAll(fs.entryDir(repoPath, branch, commitHash))
|
||||
}
|
||||
|
||||
func (fs *FileStore) Close() error { return nil }
|
||||
@@ -0,0 +1,365 @@
|
||||
package persistence
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
"github.com/zzet/gortex/internal/graph"
|
||||
)
|
||||
|
||||
func testSnapshot() *Snapshot {
|
||||
return &Snapshot{
|
||||
Version: "0.1.0-test",
|
||||
RepoPath: "/tmp/test-repo",
|
||||
CommitHash: "abc123def456",
|
||||
Branch: "main",
|
||||
IndexedAt: time.Now().Truncate(time.Second),
|
||||
Nodes: []*graph.Node{
|
||||
{
|
||||
ID: "main.go::Foo", Kind: graph.KindFunction, Name: "Foo",
|
||||
FilePath: "main.go", StartLine: 1, EndLine: 5, Language: "go",
|
||||
Meta: map[string]any{"signature": "func Foo(x int) error"},
|
||||
},
|
||||
{
|
||||
ID: "main.go::Bar", Kind: graph.KindMethod, Name: "Bar",
|
||||
FilePath: "main.go", StartLine: 7, EndLine: 12, Language: "go",
|
||||
Meta: map[string]any{"receiver": "Server", "signature": "func (s *Server) Bar()"},
|
||||
},
|
||||
},
|
||||
Edges: []*graph.Edge{
|
||||
{
|
||||
From: "main.go::Foo", To: "main.go::Bar", Kind: graph.EdgeCalls,
|
||||
FilePath: "main.go", Line: 3, Confidence: 0.95,
|
||||
Meta: map[string]any{"receiver_type": "Server"},
|
||||
},
|
||||
},
|
||||
FileMtimes: map[string]int64{
|
||||
"main.go": 1700000000000000000,
|
||||
"util.go": 1700000001000000000,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func TestFileStore_RoundTrip(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
fs, err := NewFileStore(dir, "0.1.0-test")
|
||||
require.NoError(t, err)
|
||||
|
||||
snap := testSnapshot()
|
||||
|
||||
require.NoError(t, fs.Save(snap))
|
||||
assert.True(t, fs.Check(snap.RepoPath, snap.Branch, snap.CommitHash))
|
||||
assert.True(t, fs.Validate(snap.RepoPath, snap.Branch, snap.CommitHash))
|
||||
|
||||
loaded, err := fs.Load(snap.RepoPath, snap.Branch, snap.CommitHash)
|
||||
require.NoError(t, err)
|
||||
|
||||
assert.Equal(t, snap.Version, loaded.Version)
|
||||
assert.Equal(t, snap.RepoPath, loaded.RepoPath)
|
||||
assert.Equal(t, snap.CommitHash, loaded.CommitHash)
|
||||
assert.Equal(t, snap.Branch, loaded.Branch)
|
||||
assert.Equal(t, snap.IndexedAt, loaded.IndexedAt)
|
||||
|
||||
require.Len(t, loaded.Nodes, 2)
|
||||
assert.Equal(t, "main.go::Foo", loaded.Nodes[0].ID)
|
||||
assert.Equal(t, "Foo", loaded.Nodes[0].Name)
|
||||
assert.Equal(t, "func Foo(x int) error", loaded.Nodes[0].Meta["signature"])
|
||||
|
||||
assert.Equal(t, "Server", loaded.Nodes[1].Meta["receiver"])
|
||||
|
||||
require.Len(t, loaded.Edges, 1)
|
||||
assert.Equal(t, "main.go::Foo", loaded.Edges[0].From)
|
||||
assert.Equal(t, "main.go::Bar", loaded.Edges[0].To)
|
||||
assert.Equal(t, 0.95, loaded.Edges[0].Confidence)
|
||||
assert.Equal(t, "Server", loaded.Edges[0].Meta["receiver_type"])
|
||||
|
||||
assert.Equal(t, snap.FileMtimes, loaded.FileMtimes)
|
||||
}
|
||||
|
||||
func TestFileStore_Validate_VersionMismatch(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
fsV1, err := NewFileStore(dir, "0.1.0")
|
||||
require.NoError(t, err)
|
||||
|
||||
snap := testSnapshot()
|
||||
snap.Version = "0.1.0"
|
||||
require.NoError(t, fsV1.Save(snap))
|
||||
|
||||
// Same version validates.
|
||||
assert.True(t, fsV1.Validate(snap.RepoPath, snap.Branch, snap.CommitHash))
|
||||
|
||||
// A different BINARY version now REUSES the slot — the extraction version
|
||||
// (not the binary string) gates reuse, so a no-op release skips the rebuild.
|
||||
fsV2, err := NewFileStore(dir, "0.2.0")
|
||||
require.NoError(t, err)
|
||||
assert.True(t, fsV2.Validate(snap.RepoPath, snap.Branch, snap.CommitHash))
|
||||
|
||||
// A LEGACY slot with no extraction-version marker falls back to the exact
|
||||
// binary-version match — so it correctly invalidates across the bump.
|
||||
entry := fsV2.entryDir(snap.RepoPath, snap.Branch, snap.CommitHash)
|
||||
require.NoError(t, os.Remove(filepath.Join(entry, extractionVersionFile)))
|
||||
assert.False(t, fsV2.Validate(snap.RepoPath, snap.Branch, snap.CommitHash))
|
||||
}
|
||||
|
||||
func TestFileStore_Evict(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
fs, err := NewFileStore(dir, "0.1.0")
|
||||
require.NoError(t, err)
|
||||
|
||||
snap := testSnapshot()
|
||||
require.NoError(t, fs.Save(snap))
|
||||
assert.True(t, fs.Check(snap.RepoPath, snap.Branch, snap.CommitHash))
|
||||
|
||||
require.NoError(t, fs.Evict(snap.RepoPath, snap.Branch, snap.CommitHash))
|
||||
assert.False(t, fs.Check(snap.RepoPath, snap.Branch, snap.CommitHash))
|
||||
}
|
||||
|
||||
func TestFileStore_Load_NotFound(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
fs, err := NewFileStore(dir, "0.1.0")
|
||||
require.NoError(t, err)
|
||||
|
||||
_, err = fs.Load("/nonexistent", "main", "abc123")
|
||||
assert.ErrorIs(t, err, ErrNotFound)
|
||||
}
|
||||
|
||||
func TestFileStore_MetaWithSliceTypes(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
fs, err := NewFileStore(dir, "0.1.0")
|
||||
require.NoError(t, err)
|
||||
|
||||
snap := &Snapshot{
|
||||
Version: "0.1.0",
|
||||
RepoPath: "/tmp/test",
|
||||
CommitHash: "def789",
|
||||
Branch: "main",
|
||||
IndexedAt: time.Now().Truncate(time.Second),
|
||||
Nodes: []*graph.Node{
|
||||
{
|
||||
ID: "iface.go::Reader", Kind: graph.KindInterface, Name: "Reader",
|
||||
FilePath: "iface.go", Language: "go",
|
||||
Meta: map[string]any{"methods": []string{"Read", "Close"}},
|
||||
},
|
||||
},
|
||||
FileMtimes: map[string]int64{"iface.go": 1700000000},
|
||||
}
|
||||
|
||||
require.NoError(t, fs.Save(snap))
|
||||
|
||||
loaded, err := fs.Load(snap.RepoPath, snap.Branch, snap.CommitHash)
|
||||
require.NoError(t, err)
|
||||
|
||||
methods, ok := loaded.Nodes[0].Meta["methods"].([]string)
|
||||
require.True(t, ok, "methods should deserialize as []string")
|
||||
assert.Equal(t, []string{"Read", "Close"}, methods)
|
||||
}
|
||||
|
||||
// TestFileStore_BranchKeyedSlots proves snapshots are keyed by
|
||||
// (repo, branch): two branches of the same repo, even at the same
|
||||
// commit, occupy distinct slots, so switching branches never clobbers
|
||||
// the other branch's cached index.
|
||||
func TestFileStore_BranchKeyedSlots(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
fs, err := NewFileStore(dir, "0.1.0-test")
|
||||
require.NoError(t, err)
|
||||
|
||||
main := testSnapshot()
|
||||
main.Branch = "main"
|
||||
feature := testSnapshot()
|
||||
feature.Branch = "feature/login"
|
||||
feature.Nodes[0].Name = "FeatureFoo"
|
||||
|
||||
require.NoError(t, fs.Save(main))
|
||||
require.NoError(t, fs.Save(feature))
|
||||
|
||||
gotMain, err := fs.Load(main.RepoPath, "main", main.CommitHash)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "Foo", gotMain.Nodes[0].Name)
|
||||
|
||||
gotFeature, err := fs.Load(feature.RepoPath, "feature/login", feature.CommitHash)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "FeatureFoo", gotFeature.Nodes[0].Name)
|
||||
}
|
||||
|
||||
// TestFileStore_DetachedHeadKeyedByCommit checks the detached-HEAD
|
||||
// fallback: with no branch the slot keys on the commit hash, so two
|
||||
// checked-out commits keep separate snapshots.
|
||||
func TestFileStore_DetachedHeadKeyedByCommit(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
fs, err := NewFileStore(dir, "0.1.0-test")
|
||||
require.NoError(t, err)
|
||||
|
||||
a := testSnapshot()
|
||||
a.Branch = ""
|
||||
a.CommitHash = "aaaaaaaaaaaa"
|
||||
b := testSnapshot()
|
||||
b.Branch = ""
|
||||
b.CommitHash = "bbbbbbbbbbbb"
|
||||
|
||||
require.NoError(t, fs.Save(a))
|
||||
require.NoError(t, fs.Save(b))
|
||||
|
||||
assert.True(t, fs.Check(a.RepoPath, "", "aaaaaaaaaaaa"))
|
||||
assert.True(t, fs.Check(b.RepoPath, "", "bbbbbbbbbbbb"))
|
||||
}
|
||||
|
||||
func TestNopStore(t *testing.T) {
|
||||
var s NopStore
|
||||
assert.False(t, s.Check("x", "main", "y"))
|
||||
_, err := s.Load("x", "main", "y")
|
||||
assert.ErrorIs(t, err, ErrNotFound)
|
||||
assert.NoError(t, s.Save(testSnapshot()))
|
||||
assert.False(t, s.Validate("x", "main", "y"))
|
||||
assert.NoError(t, s.Evict("x", "main", "y"))
|
||||
assert.NoError(t, s.Close())
|
||||
}
|
||||
|
||||
// TestFileStore_ConcurrentSave exercises the cross-process advisory lock:
|
||||
// every writer targets the same cache key, so without serialization
|
||||
// one writer's os.RemoveAll would race another's MkdirAll/write sequence
|
||||
// and leave a torn entry. flock(2) contends across file descriptors even
|
||||
// within one process, so concurrent goroutines reproduce the cross-process
|
||||
// hazard. After all writers complete, the entry must load as exactly one
|
||||
// writer's complete payload.
|
||||
func TestFileStore_ConcurrentSave(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
fs, err := NewFileStore(dir, "0.1.0-test")
|
||||
require.NoError(t, err)
|
||||
|
||||
const writers = 12
|
||||
markers := make(map[string]bool, writers)
|
||||
var wg sync.WaitGroup
|
||||
errs := make(chan error, writers)
|
||||
for i := range writers {
|
||||
marker := fmt.Sprintf("writer-%d", i)
|
||||
markers[marker] = true
|
||||
wg.Add(1)
|
||||
go func(m string) {
|
||||
defer wg.Done()
|
||||
snap := testSnapshot()
|
||||
snap.Nodes[0].Meta["writer"] = m
|
||||
errs <- fs.Save(snap)
|
||||
}(marker)
|
||||
}
|
||||
wg.Wait()
|
||||
close(errs)
|
||||
for e := range errs {
|
||||
require.NoError(t, e)
|
||||
}
|
||||
|
||||
loaded, err := fs.Load("/tmp/test-repo", "main", "abc123def456")
|
||||
require.NoError(t, err)
|
||||
require.Len(t, loaded.Nodes, 2)
|
||||
got, _ := loaded.Nodes[0].Meta["writer"].(string)
|
||||
assert.True(t, markers[got], "loaded snapshot must be one writer's complete payload, got %q", got)
|
||||
}
|
||||
|
||||
// TestFileStore_ConcurrentReadWrite runs readers against a writer churning
|
||||
// the same entry. The shared read lock must hand every reader either a
|
||||
// fully decodable snapshot or a clean ErrNotFound — never a gob/gzip
|
||||
// decode error from a half-written file.
|
||||
func TestFileStore_ConcurrentReadWrite(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
fs, err := NewFileStore(dir, "0.1.0-test")
|
||||
require.NoError(t, err)
|
||||
|
||||
snap := testSnapshot()
|
||||
require.NoError(t, fs.Save(snap))
|
||||
|
||||
var wg sync.WaitGroup
|
||||
errs := make(chan error, 64)
|
||||
stop := make(chan struct{})
|
||||
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
for i := 0; ; i++ {
|
||||
select {
|
||||
case <-stop:
|
||||
return
|
||||
default:
|
||||
}
|
||||
var e error
|
||||
if i%2 == 0 {
|
||||
e = fs.Save(snap)
|
||||
} else {
|
||||
e = fs.Evict(snap.RepoPath, snap.Branch, snap.CommitHash)
|
||||
}
|
||||
// Honour stop while sending: errs is buffered, and the
|
||||
// writer outruns the buffer in microseconds. Without the
|
||||
// stop arm here the writer blocks on a full errs channel,
|
||||
// never re-checks stop, and wg.Wait() deadlocks (the buffer
|
||||
// only drains after wg.Wait()).
|
||||
select {
|
||||
case errs <- e:
|
||||
case <-stop:
|
||||
return
|
||||
}
|
||||
}
|
||||
}()
|
||||
|
||||
for r := 0; r < 6; r++ {
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
for i := 0; i < 40; i++ {
|
||||
_, err := fs.Load(snap.RepoPath, snap.Branch, snap.CommitHash)
|
||||
if err != nil && err != ErrNotFound {
|
||||
errs <- err
|
||||
}
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
time.Sleep(50 * time.Millisecond)
|
||||
close(stop)
|
||||
wg.Wait()
|
||||
close(errs)
|
||||
for e := range errs {
|
||||
require.NoError(t, e, "no reader may observe a torn snapshot")
|
||||
}
|
||||
}
|
||||
|
||||
// TestSnapshotReuseAcrossBinaryBump proves the warm snapshot is reused across a
|
||||
// binary version bump that did NOT change extraction output (the extraction
|
||||
// version is the gate, not the binary-version string) — avoiding the needless
|
||||
// full cold rebuild a binary-string gate would force on a no-op release — while
|
||||
// a genuine extraction-version change still invalidates the slot.
|
||||
func TestSnapshotReuseAcrossBinaryBump(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
|
||||
s1, err := NewFileStore(dir, "0.48.0")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
snap := testSnapshot()
|
||||
snap.Version = "0.48.0"
|
||||
if err := s1.Save(snap); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// A newer binary with the SAME extraction version reuses the slot.
|
||||
s2, err := NewFileStore(dir, "0.48.1")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !s2.Validate(snap.RepoPath, snap.Branch, snap.CommitHash) {
|
||||
t.Error("snapshot should be reused across a binary bump with an unchanged extraction version")
|
||||
}
|
||||
|
||||
// Corrupt the extraction-version marker to simulate an extraction-output
|
||||
// change: the slot must now be rejected (cold rebuild).
|
||||
entry := s2.entryDir(snap.RepoPath, snap.Branch, snap.CommitHash)
|
||||
if err := os.WriteFile(filepath.Join(entry, extractionVersionFile), []byte("999"), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if s2.Validate(snap.RepoPath, snap.Branch, snap.CommitHash) {
|
||||
t.Error("a changed extraction version must invalidate the snapshot")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,93 @@
|
||||
package persistence
|
||||
|
||||
import (
|
||||
"compress/gzip"
|
||||
"encoding/gob"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
)
|
||||
|
||||
const (
|
||||
frecencyFile = "frecency.gob.gz"
|
||||
maxFrecencyAccesses = 16 // entries per symbol; matches FFF's bounded deque
|
||||
maxFrecencySymbols = 10000
|
||||
)
|
||||
|
||||
// FrecencyAccesses is the bounded, ordered (oldest→newest) list of unix
|
||||
// timestamps (seconds) at which one symbol was consumed. Bounded because
|
||||
// the decay formula already weights recent accesses far more heavily than
|
||||
// old ones — beyond ~16 entries, additional history contributes noise.
|
||||
type FrecencyAccesses struct {
|
||||
SymbolID string
|
||||
Times []int64
|
||||
}
|
||||
|
||||
// FrecencyStore holds all per-symbol access histories for a single repo.
|
||||
type FrecencyStore struct {
|
||||
Version string
|
||||
RepoPath string
|
||||
Symbols []FrecencyAccesses
|
||||
}
|
||||
|
||||
// MaxFrecencyAccesses returns the per-symbol access-history cap.
|
||||
func MaxFrecencyAccesses() int { return maxFrecencyAccesses }
|
||||
|
||||
// FrecencyDir returns the on-disk directory for frecency storage.
|
||||
func FrecencyDir(cacheDir, repoPath string) string {
|
||||
return filepath.Join(cacheDir, RepoCacheKey(repoPath))
|
||||
}
|
||||
|
||||
// LoadFrecency reads the frecency store from disk.
|
||||
func LoadFrecency(dir string) (*FrecencyStore, error) {
|
||||
path := filepath.Join(dir, frecencyFile)
|
||||
f, err := os.Open(path)
|
||||
if err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
return &FrecencyStore{}, nil
|
||||
}
|
||||
return nil, fmt.Errorf("persistence: open frecency: %w", err)
|
||||
}
|
||||
defer f.Close()
|
||||
|
||||
gz, err := gzip.NewReader(f)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("persistence: gzip reader frecency: %w", err)
|
||||
}
|
||||
defer gz.Close()
|
||||
|
||||
var store FrecencyStore
|
||||
if err := gob.NewDecoder(gz).Decode(&store); err != nil {
|
||||
return nil, fmt.Errorf("persistence: gob decode frecency: %w", err)
|
||||
}
|
||||
return &store, nil
|
||||
}
|
||||
|
||||
// SaveFrecency writes the frecency store to disk. Trims the oldest-touched
|
||||
// symbols if over cap so the file can't grow unboundedly.
|
||||
func SaveFrecency(dir string, store *FrecencyStore) error {
|
||||
if len(store.Symbols) > maxFrecencySymbols {
|
||||
trim := len(store.Symbols) - maxFrecencySymbols
|
||||
store.Symbols = store.Symbols[trim:]
|
||||
}
|
||||
if err := os.MkdirAll(dir, 0o755); err != nil {
|
||||
return fmt.Errorf("persistence: mkdir frecency: %w", err)
|
||||
}
|
||||
path := filepath.Join(dir, frecencyFile)
|
||||
f, err := os.Create(path)
|
||||
if err != nil {
|
||||
return fmt.Errorf("persistence: create frecency: %w", err)
|
||||
}
|
||||
gz := gzip.NewWriter(f)
|
||||
enc := gob.NewEncoder(gz)
|
||||
if err := enc.Encode(store); err != nil {
|
||||
_ = gz.Close()
|
||||
_ = f.Close()
|
||||
return fmt.Errorf("persistence: gob encode frecency: %w", err)
|
||||
}
|
||||
if err := gz.Close(); err != nil {
|
||||
_ = f.Close()
|
||||
return fmt.Errorf("persistence: gzip close frecency: %w", err)
|
||||
}
|
||||
return f.Close()
|
||||
}
|
||||
@@ -0,0 +1,67 @@
|
||||
package persistence
|
||||
|
||||
import (
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
|
||||
"github.com/zzet/gortex/internal/pathkey"
|
||||
)
|
||||
|
||||
// CacheKey produces a filesystem-safe directory name identifying one
|
||||
// snapshot slot. Snapshots are keyed by (repo, branch): one slot per
|
||||
// branch, overwritten as the branch advances. A daemon restart after
|
||||
// new commits then loads the branch's last snapshot and incrementally
|
||||
// reconciles, instead of cold-indexing because the commit hash moved.
|
||||
// A detached HEAD (empty branch) falls back to the commit hash so each
|
||||
// checked-out commit still gets a stable slot.
|
||||
//
|
||||
// Both the repo path and the ref are folded to Unicode NFC before
|
||||
// hashing: a non-ASCII path or branch name presents with different
|
||||
// bytes on macOS (NFD) than on Linux / git (NFC), and without the
|
||||
// fold the same repo would key into two distinct snapshot slots.
|
||||
func CacheKey(repoPath, branch, commitHash string) string {
|
||||
abs, err := filepath.Abs(repoPath)
|
||||
if err != nil {
|
||||
abs = repoPath
|
||||
}
|
||||
h := sha256.Sum256([]byte(pathkey.Normalize(abs)))
|
||||
pathPart := hex.EncodeToString(h[:6])
|
||||
|
||||
ref := strings.TrimSpace(branch)
|
||||
if ref == "" || ref == "HEAD" {
|
||||
ref = strings.TrimSpace(commitHash)
|
||||
}
|
||||
return pathPart + "_" + refSlug(ref)
|
||||
}
|
||||
|
||||
// refSlug renders a git ref — a branch name or a commit hash — as a
|
||||
// stable, filesystem-safe path segment: a readable sanitized prefix
|
||||
// plus a short hash of the full ref, so two refs that sanitize or
|
||||
// truncate to the same prefix (e.g. feature/x vs feature-x) still get
|
||||
// distinct slots.
|
||||
func refSlug(ref string) string {
|
||||
if ref == "" {
|
||||
return "none"
|
||||
}
|
||||
// Fold to NFC first: a non-ASCII branch name read in decomposed
|
||||
// form on one platform and precomposed form on another would
|
||||
// otherwise hash to two different slugs for the same branch.
|
||||
ref = pathkey.Normalize(ref)
|
||||
var b strings.Builder
|
||||
for _, r := range ref {
|
||||
switch {
|
||||
case r >= 'a' && r <= 'z', r >= 'A' && r <= 'Z',
|
||||
r >= '0' && r <= '9', r == '.', r == '-', r == '_':
|
||||
b.WriteRune(r)
|
||||
default:
|
||||
b.WriteByte('-')
|
||||
}
|
||||
if b.Len() >= 32 {
|
||||
break
|
||||
}
|
||||
}
|
||||
h := sha256.Sum256([]byte(ref))
|
||||
return b.String() + "_" + hex.EncodeToString(h[:4])
|
||||
}
|
||||
@@ -0,0 +1,155 @@
|
||||
package persistence
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"golang.org/x/text/unicode/norm"
|
||||
)
|
||||
|
||||
// nfcDir / nfdDir are the precomposed and decomposed byte forms of the
|
||||
// same accented repo path. They are derived in code with norm.NFC /
|
||||
// norm.NFD from a single base so the two byte sequences are guaranteed
|
||||
// distinct and deterministic — never dependent on how this source file
|
||||
// happens to be saved. On macOS APFS a repo cloned into such a
|
||||
// directory is reported in NFD by the OS, while the same path written
|
||||
// into a config file or read on Linux is typically NFC; a snapshot
|
||||
// keyed under one form must resolve to the same slot under the other.
|
||||
//
|
||||
// The non-ASCII characters in the base strings are written as explicit
|
||||
// \u / \U escapes so this source file stays pure-ASCII.
|
||||
var (
|
||||
repoDirBase = "/home/dev/café/repo" // .../café/repo
|
||||
nfcDir = norm.NFC.String(repoDirBase)
|
||||
nfdDir = norm.NFD.String(repoDirBase)
|
||||
)
|
||||
|
||||
const (
|
||||
cjkBranch = "機能/日本語" // 機能/日本語
|
||||
cyrBranch = "функция-ветка" // функция-ветка
|
||||
asciiPath = "/home/dev/plain/repo"
|
||||
asciiBr = "feat/some-branch"
|
||||
commitHash = "0123456789abcdef0123456789abcdef01234567"
|
||||
)
|
||||
|
||||
// TestCacheKey_NFCvsNFDPathSameSlot is the core round-trip guarantee:
|
||||
// the same repo path supplied in decomposed and precomposed Unicode
|
||||
// forms must hash to one snapshot slot. Without the NFC fold in
|
||||
// CacheKey the two byte sequences hash differently and the daemon
|
||||
// loses its cache across an OS / form boundary.
|
||||
func TestCacheKey_NFCvsNFDPathSameSlot(t *testing.T) {
|
||||
if nfcDir == nfdDir {
|
||||
t.Fatal("test fixture invalid: NFC and NFD repo paths are byte-identical")
|
||||
}
|
||||
keyNFC := CacheKey(nfcDir, "main", commitHash)
|
||||
keyNFD := CacheKey(nfdDir, "main", commitHash)
|
||||
if keyNFC != keyNFD {
|
||||
t.Fatalf("CacheKey not normalisation-stable for repo path:\n NFC -> %q\n NFD -> %q", keyNFC, keyNFD)
|
||||
}
|
||||
}
|
||||
|
||||
// TestCacheKey_NFCvsNFDBranchSameSlot guards the branch half of the
|
||||
// key: a non-ASCII branch name in two Unicode forms must land in one
|
||||
// slot, so the daemon does not split a single branch's snapshot in two.
|
||||
func TestCacheKey_NFCvsNFDBranchSameSlot(t *testing.T) {
|
||||
branchNFD := norm.NFD.String(cjkBranch)
|
||||
branchNFC := norm.NFC.String(cjkBranch)
|
||||
keyNFD := CacheKey(asciiPath, branchNFD, commitHash)
|
||||
keyNFC := CacheKey(asciiPath, branchNFC, commitHash)
|
||||
if keyNFD != keyNFC {
|
||||
t.Fatalf("CacheKey not normalisation-stable for branch:\n NFD -> %q\n NFC -> %q", keyNFD, keyNFC)
|
||||
}
|
||||
}
|
||||
|
||||
// TestCacheKey_DistinctReposDistinctSlots confirms the fold does not
|
||||
// over-collapse: two genuinely different non-ASCII repo paths must
|
||||
// still get distinct slots.
|
||||
func TestCacheKey_DistinctReposDistinctSlots(t *testing.T) {
|
||||
a := CacheKey(nfcDir, "main", commitHash)
|
||||
b := CacheKey(asciiPath, "main", commitHash)
|
||||
if a == b {
|
||||
t.Fatalf("CacheKey collided two different repos onto slot %q", a)
|
||||
}
|
||||
}
|
||||
|
||||
// TestCacheKey_DistinctBranchesDistinctSlots confirms two different
|
||||
// non-ASCII branch names on the same repo do not collide — the
|
||||
// collision-safety hash inside refSlug must survive normalisation.
|
||||
func TestCacheKey_DistinctBranchesDistinctSlots(t *testing.T) {
|
||||
a := CacheKey(asciiPath, cjkBranch, commitHash)
|
||||
b := CacheKey(asciiPath, cyrBranch, commitHash)
|
||||
if a == b {
|
||||
t.Fatalf("CacheKey collided two different branches onto slot %q", a)
|
||||
}
|
||||
}
|
||||
|
||||
// TestCacheKey_FilesystemSafe checks the produced key is usable as a
|
||||
// directory name: no path separators, no NUL, non-empty — true even
|
||||
// when the branch is entirely non-ASCII (refSlug replaces every such
|
||||
// rune, leaving the hash suffix to carry identity).
|
||||
func TestCacheKey_FilesystemSafe(t *testing.T) {
|
||||
for _, branch := range []string{cjkBranch, cyrBranch, "main", ""} {
|
||||
key := CacheKey(nfdDir, branch, commitHash)
|
||||
if key == "" {
|
||||
t.Fatalf("CacheKey returned empty string for branch %q", branch)
|
||||
}
|
||||
if strings.ContainsAny(key, "/\\\x00") {
|
||||
t.Fatalf("CacheKey %q for branch %q contains a path-unsafe byte", key, branch)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestCacheKey_ASCIIUnchangedByFold pins that the NFC fold is a no-op
|
||||
// for the common all-ASCII case — the keys for ASCII inputs must not
|
||||
// shift, so existing on-disk snapshot directories stay valid.
|
||||
func TestCacheKey_ASCIIUnchangedByFold(t *testing.T) {
|
||||
// Recomputed twice must be stable, and an ASCII path/branch must
|
||||
// survive the fold byte-for-byte (the slug prefix is visible in
|
||||
// the key, so a regression here would change the directory name).
|
||||
key1 := CacheKey(asciiPath, asciiBr, commitHash)
|
||||
key2 := CacheKey(asciiPath, asciiBr, commitHash)
|
||||
if key1 != key2 {
|
||||
t.Fatalf("CacheKey not deterministic for ASCII input: %q vs %q", key1, key2)
|
||||
}
|
||||
if !strings.Contains(key1, "feat-some-branch") {
|
||||
t.Fatalf("CacheKey %q lost its readable ASCII branch slug", key1)
|
||||
}
|
||||
}
|
||||
|
||||
// TestCacheKey_DetachedHeadFallsBackToCommit checks the detached-HEAD
|
||||
// path still works once the fold is in place: an empty branch keys by
|
||||
// commit hash.
|
||||
func TestCacheKey_DetachedHeadFallsBackToCommit(t *testing.T) {
|
||||
withBranch := CacheKey(asciiPath, "main", commitHash)
|
||||
detached := CacheKey(asciiPath, "", commitHash)
|
||||
headLiteral := CacheKey(asciiPath, "HEAD", commitHash)
|
||||
if detached == withBranch {
|
||||
t.Fatal("detached-HEAD key collided with a real branch key")
|
||||
}
|
||||
if detached != headLiteral {
|
||||
t.Fatalf("empty branch and literal HEAD must key alike: %q vs %q", detached, headLiteral)
|
||||
}
|
||||
}
|
||||
|
||||
// TestRepoCacheKey_NFCvsNFDSameSlot mirrors the CacheKey round-trip
|
||||
// guarantee for the commit-independent feedback key.
|
||||
func TestRepoCacheKey_NFCvsNFDSameSlot(t *testing.T) {
|
||||
if nfcDir == nfdDir {
|
||||
t.Fatal("test fixture invalid: NFC and NFD repo paths are byte-identical")
|
||||
}
|
||||
keyNFC := RepoCacheKey(nfcDir)
|
||||
keyNFD := RepoCacheKey(nfdDir)
|
||||
if keyNFC != keyNFD {
|
||||
t.Fatalf("RepoCacheKey not normalisation-stable:\n NFC -> %q\n NFD -> %q", keyNFC, keyNFD)
|
||||
}
|
||||
}
|
||||
|
||||
// TestRepoCacheKey_DistinctReposDistinctSlots confirms the feedback
|
||||
// key does not over-collapse distinct non-ASCII repos.
|
||||
func TestRepoCacheKey_DistinctReposDistinctSlots(t *testing.T) {
|
||||
a := RepoCacheKey(nfcDir)
|
||||
b := RepoCacheKey(asciiPath)
|
||||
if a == b {
|
||||
t.Fatalf("RepoCacheKey collided two different repos onto slot %q", a)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,120 @@
|
||||
package persistence
|
||||
|
||||
import (
|
||||
"compress/gzip"
|
||||
"encoding/gob"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
)
|
||||
|
||||
const (
|
||||
keywordFile = "keyword.gob.gz"
|
||||
maxKeywords = 4000
|
||||
maxKeywordEntriesPerKey = 10
|
||||
)
|
||||
|
||||
// KeywordMatch records one (keyword -> symbol) association. HitCount
|
||||
// is how many times the agent picked this symbol following a query
|
||||
// that contained the keyword; LastUsed is a unix timestamp (seconds)
|
||||
// for decay and reaping. Structurally identical to ComboMatch but
|
||||
// kept as a distinct type so the two stores' schemas can diverge.
|
||||
type KeywordMatch struct {
|
||||
SymbolID string
|
||||
HitCount uint32
|
||||
LastUsed int64
|
||||
// MissCount is the implicit-negative tally: how many times the
|
||||
// agent was shown this symbol for a query carrying the keyword but
|
||||
// skipped over it to pick a lower-ranked result. The per-keyword
|
||||
// boost nets HitCount-MissCount, so a symbol the agent keeps passing
|
||||
// over loses its learned boost. A zero value (legacy data) is the
|
||||
// pre-negative-signal behaviour. Decays on the same clock as hits.
|
||||
MissCount uint32
|
||||
}
|
||||
|
||||
// KeywordAssoc holds all recorded matches for one query keyword
|
||||
// within a single repo. Ordered most-hit-first after any record.
|
||||
type KeywordAssoc struct {
|
||||
Keyword string
|
||||
Matches []KeywordMatch
|
||||
}
|
||||
|
||||
// KeywordStore is the persisted per-keyword association index for one
|
||||
// repo. Where ComboStore keys on the whole normalized query,
|
||||
// KeywordStore keys on each surviving query token -- so a new task
|
||||
// with overlapping keywords but different phrasing still inherits the
|
||||
// associations its keywords earned. Separate file from combo and
|
||||
// feedback so each subsystem's schema evolves independently.
|
||||
type KeywordStore struct {
|
||||
Version string
|
||||
RepoPath string
|
||||
Keywords []KeywordAssoc
|
||||
}
|
||||
|
||||
// KeywordDir returns the on-disk directory for keyword storage.
|
||||
// Shares the repo cache key with combo / feedback so all repo-scoped
|
||||
// state lives together.
|
||||
func KeywordDir(cacheDir, repoPath string) string {
|
||||
return filepath.Join(cacheDir, RepoCacheKey(repoPath))
|
||||
}
|
||||
|
||||
// LoadKeyword reads the keyword store from disk. A missing file is
|
||||
// not an error -- it yields an empty store.
|
||||
func LoadKeyword(dir string) (*KeywordStore, error) {
|
||||
path := filepath.Join(dir, keywordFile)
|
||||
f, err := os.Open(path)
|
||||
if err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
return &KeywordStore{}, nil
|
||||
}
|
||||
return nil, fmt.Errorf("persistence: open keyword: %w", err)
|
||||
}
|
||||
defer f.Close()
|
||||
|
||||
gz, err := gzip.NewReader(f)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("persistence: gzip reader keyword: %w", err)
|
||||
}
|
||||
defer gz.Close()
|
||||
|
||||
var store KeywordStore
|
||||
if err := gob.NewDecoder(gz).Decode(&store); err != nil {
|
||||
return nil, fmt.Errorf("persistence: gob decode keyword: %w", err)
|
||||
}
|
||||
return &store, nil
|
||||
}
|
||||
|
||||
// SaveKeyword writes the keyword store with gob+gzip compression.
|
||||
// Trims the oldest keywords if over cap so the file cannot grow
|
||||
// unboundedly on a long-running daemon.
|
||||
func SaveKeyword(dir string, store *KeywordStore) error {
|
||||
if len(store.Keywords) > maxKeywords {
|
||||
trim := len(store.Keywords) - maxKeywords
|
||||
store.Keywords = store.Keywords[trim:]
|
||||
}
|
||||
if err := os.MkdirAll(dir, 0o755); err != nil {
|
||||
return fmt.Errorf("persistence: mkdir keyword: %w", err)
|
||||
}
|
||||
path := filepath.Join(dir, keywordFile)
|
||||
f, err := os.Create(path)
|
||||
if err != nil {
|
||||
return fmt.Errorf("persistence: create keyword: %w", err)
|
||||
}
|
||||
gz := gzip.NewWriter(f)
|
||||
enc := gob.NewEncoder(gz)
|
||||
if err := enc.Encode(store); err != nil {
|
||||
_ = gz.Close()
|
||||
_ = f.Close()
|
||||
return fmt.Errorf("persistence: gob encode keyword: %w", err)
|
||||
}
|
||||
if err := gz.Close(); err != nil {
|
||||
_ = f.Close()
|
||||
return fmt.Errorf("persistence: gzip close keyword: %w", err)
|
||||
}
|
||||
return f.Close()
|
||||
}
|
||||
|
||||
// MaxKeywordEntries returns the cap on matches per keyword. Tighter
|
||||
// than MaxComboEntries -- a single keyword is a coarser key than a
|
||||
// whole query, so its match list is held shorter to stay precise.
|
||||
func MaxKeywordEntries() int { return maxKeywordEntriesPerKey }
|
||||
@@ -0,0 +1,68 @@
|
||||
package persistence
|
||||
|
||||
import (
|
||||
"path/filepath"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestKeywordStore_RoundTrip(t *testing.T) {
|
||||
dir := filepath.Join(t.TempDir(), "kw")
|
||||
|
||||
// A missing file loads as an empty store, not an error.
|
||||
loaded, err := LoadKeyword(dir)
|
||||
if err != nil {
|
||||
t.Fatalf("LoadKeyword on a missing dir errored: %v", err)
|
||||
}
|
||||
if loaded == nil || len(loaded.Keywords) != 0 {
|
||||
t.Fatalf("missing-file load should yield an empty store")
|
||||
}
|
||||
|
||||
store := &KeywordStore{
|
||||
Version: "1",
|
||||
RepoPath: "/repo",
|
||||
Keywords: []KeywordAssoc{
|
||||
{Keyword: "auth", Matches: []KeywordMatch{
|
||||
{SymbolID: "pkg::LoginService", HitCount: 5, LastUsed: 1700},
|
||||
}},
|
||||
{Keyword: "token", Matches: []KeywordMatch{
|
||||
{SymbolID: "pkg::ParseJWT", HitCount: 3, LastUsed: 1701},
|
||||
}},
|
||||
},
|
||||
}
|
||||
if err := SaveKeyword(dir, store); err != nil {
|
||||
t.Fatalf("SaveKeyword errored: %v", err)
|
||||
}
|
||||
back, err := LoadKeyword(dir)
|
||||
if err != nil {
|
||||
t.Fatalf("LoadKeyword after save errored: %v", err)
|
||||
}
|
||||
if len(back.Keywords) != 2 {
|
||||
t.Fatalf("round-trip lost keywords: got %d, want 2", len(back.Keywords))
|
||||
}
|
||||
if back.Keywords[0].Keyword != "auth" ||
|
||||
back.Keywords[0].Matches[0].SymbolID != "pkg::LoginService" ||
|
||||
back.Keywords[0].Matches[0].HitCount != 5 {
|
||||
t.Errorf("round-trip corrupted the first keyword: %+v", back.Keywords[0])
|
||||
}
|
||||
}
|
||||
|
||||
func TestKeywordStore_TrimsOverCap(t *testing.T) {
|
||||
dir := filepath.Join(t.TempDir(), "kw")
|
||||
store := &KeywordStore{}
|
||||
for i := 0; i < maxKeywords+50; i++ {
|
||||
store.Keywords = append(store.Keywords, KeywordAssoc{Keyword: string(rune('a' + i%26))})
|
||||
}
|
||||
if err := SaveKeyword(dir, store); err != nil {
|
||||
t.Fatalf("SaveKeyword errored: %v", err)
|
||||
}
|
||||
if len(store.Keywords) != maxKeywords {
|
||||
t.Errorf("SaveKeyword should trim to %d keywords, got %d", maxKeywords, len(store.Keywords))
|
||||
}
|
||||
}
|
||||
|
||||
func TestMaxKeywordEntries_TighterThanCombo(t *testing.T) {
|
||||
if MaxKeywordEntries() >= MaxComboEntries() {
|
||||
t.Errorf("MaxKeywordEntries (%d) should be tighter than MaxComboEntries (%d)",
|
||||
MaxKeywordEntries(), MaxComboEntries())
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,169 @@
|
||||
package persistence
|
||||
|
||||
import (
|
||||
"compress/gzip"
|
||||
"encoding/gob"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"time"
|
||||
)
|
||||
|
||||
const (
|
||||
memoriesFile = "memories.gob.gz"
|
||||
maxMemoriesCap = 10000
|
||||
)
|
||||
|
||||
// MemoryEntry is a single cross-session development memory. Unlike
|
||||
// NoteEntry (per-session scratchpad), MemoryEntry has no SessionID
|
||||
// — every memory is workspace-wide and durable across sessions.
|
||||
//
|
||||
// Memories accumulate over time and compound the longer a team uses
|
||||
// Gortex: every recorded invariant, gotcha, decision, or convention
|
||||
// becomes discoverable by every future agent in the same workspace.
|
||||
//
|
||||
// Memories are surfaced when:
|
||||
// - an anchor symbol / file enters the agent's working set
|
||||
// (via surface_memories)
|
||||
// - the agent explicitly queries by symbol / file / tag / text
|
||||
// (via query_memories)
|
||||
type MemoryEntry struct {
|
||||
ID string
|
||||
Timestamp time.Time
|
||||
UpdatedAt time.Time
|
||||
LastAccessed time.Time
|
||||
AccessCount uint64
|
||||
Body string // free-form text
|
||||
Title string // short caption (one-liner)
|
||||
Kind string // invariant | constraint | convention | gotcha | decision | incident | reference
|
||||
Source string // manual | distilled | incident | review
|
||||
Confidence float32 // 0..1 — how sure we are this still holds
|
||||
Importance int // 1..5 — operator-assigned weight
|
||||
AuthorAgent string // mcp clientInfo.name
|
||||
SymbolIDs []string // primary symbol anchors
|
||||
FilePaths []string // primary file anchors
|
||||
AutoLinks []string // additional referenced symbol IDs (auto-detected)
|
||||
Tags []string // free-form labels
|
||||
WorkspaceID string
|
||||
ProjectID string
|
||||
RepoPrefix string
|
||||
Pinned bool // pinned memories are never evicted and float to top
|
||||
SupersededBy string // ID of newer memory that replaces this one
|
||||
}
|
||||
|
||||
// MemoryStore is the persisted shape: a versioned, repo-scoped list
|
||||
// of entries. Mirrors NoteStore so the on-disk layout stays
|
||||
// consistent.
|
||||
type MemoryStore struct {
|
||||
Version string
|
||||
RepoPath string
|
||||
Entries []MemoryEntry
|
||||
}
|
||||
|
||||
// MemoriesDir resolves the cache directory holding the memories
|
||||
// file for the given repo. Shares the per-repo cache directory
|
||||
// with notes / feedback / combo / frecency.
|
||||
func MemoriesDir(cacheDir, repoPath string) string {
|
||||
return filepath.Join(cacheDir, RepoCacheKey(repoPath))
|
||||
}
|
||||
|
||||
// LoadMemories reads a memory store from disk. Returns an empty
|
||||
// store when the file does not exist (cold start is normal).
|
||||
func LoadMemories(dir string) (*MemoryStore, error) {
|
||||
path := filepath.Join(dir, memoriesFile)
|
||||
f, err := os.Open(path)
|
||||
if err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
return &MemoryStore{}, nil
|
||||
}
|
||||
return nil, fmt.Errorf("persistence: open memories: %w", err)
|
||||
}
|
||||
defer f.Close()
|
||||
|
||||
gz, err := gzip.NewReader(f)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("persistence: gzip reader memories: %w", err)
|
||||
}
|
||||
defer gz.Close()
|
||||
|
||||
var store MemoryStore
|
||||
if err := gob.NewDecoder(gz).Decode(&store); err != nil {
|
||||
return nil, fmt.Errorf("persistence: gob decode memories: %w", err)
|
||||
}
|
||||
return &store, nil
|
||||
}
|
||||
|
||||
// SaveMemories writes the store to disk with gob+gzip. Trimming
|
||||
// honours pinned memories: when the entry count exceeds
|
||||
// maxMemoriesCap, the lowest-importance / non-pinned entries are
|
||||
// dropped first; if more shedding is still needed, the oldest
|
||||
// non-pinned entries go next.
|
||||
func SaveMemories(dir string, store *MemoryStore) error {
|
||||
if len(store.Entries) > maxMemoriesCap {
|
||||
store.Entries = trimMemories(store.Entries, maxMemoriesCap)
|
||||
}
|
||||
|
||||
if err := os.MkdirAll(dir, 0o755); err != nil {
|
||||
return fmt.Errorf("persistence: mkdir memories: %w", err)
|
||||
}
|
||||
|
||||
path := filepath.Join(dir, memoriesFile)
|
||||
f, err := os.Create(path)
|
||||
if err != nil {
|
||||
return fmt.Errorf("persistence: create memories: %w", err)
|
||||
}
|
||||
|
||||
gz := gzip.NewWriter(f)
|
||||
enc := gob.NewEncoder(gz)
|
||||
|
||||
if err := enc.Encode(store); err != nil {
|
||||
_ = gz.Close()
|
||||
_ = f.Close()
|
||||
return fmt.Errorf("persistence: gob encode memories: %w", err)
|
||||
}
|
||||
|
||||
if err := gz.Close(); err != nil {
|
||||
_ = f.Close()
|
||||
return fmt.Errorf("persistence: gzip close memories: %w", err)
|
||||
}
|
||||
|
||||
return f.Close()
|
||||
}
|
||||
|
||||
// trimMemories drops entries until len(out) <= cap. Two passes:
|
||||
// - first, shed non-pinned entries with importance <= 2
|
||||
// - then, if still over cap, shed oldest non-pinned regardless
|
||||
//
|
||||
// Pinned entries are always retained, even if the resulting slice
|
||||
// exceeds cap (the cap is a soft ceiling for the prunable tail).
|
||||
func trimMemories(in []MemoryEntry, cap int) []MemoryEntry {
|
||||
if len(in) <= cap {
|
||||
return in
|
||||
}
|
||||
excess := len(in) - cap
|
||||
|
||||
out := make([]MemoryEntry, 0, len(in))
|
||||
dropped := 0
|
||||
for _, e := range in {
|
||||
if dropped < excess && !e.Pinned && e.Importance <= 2 {
|
||||
dropped++
|
||||
continue
|
||||
}
|
||||
out = append(out, e)
|
||||
}
|
||||
|
||||
if len(out) > cap {
|
||||
excess = len(out) - cap
|
||||
next := make([]MemoryEntry, 0, len(out))
|
||||
dropped = 0
|
||||
for _, e := range out {
|
||||
if dropped < excess && !e.Pinned {
|
||||
dropped++
|
||||
continue
|
||||
}
|
||||
next = append(next, e)
|
||||
}
|
||||
out = next
|
||||
}
|
||||
return out
|
||||
}
|
||||
@@ -0,0 +1,113 @@
|
||||
package persistence
|
||||
|
||||
import (
|
||||
"path/filepath"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestMemoriesPersistence_RoundTrip(t *testing.T) {
|
||||
cache := t.TempDir()
|
||||
dir := MemoriesDir(cache, "/tmp/repo-a")
|
||||
|
||||
store := &MemoryStore{
|
||||
Version: "test",
|
||||
RepoPath: "/tmp/repo-a",
|
||||
Entries: []MemoryEntry{
|
||||
{
|
||||
ID: "mem-1",
|
||||
Timestamp: time.Now().UTC(),
|
||||
UpdatedAt: time.Now().UTC(),
|
||||
Body: "lock invariant for Bar",
|
||||
Title: "Bar lock invariant",
|
||||
Kind: "invariant",
|
||||
Source: "manual",
|
||||
Importance: 5,
|
||||
Confidence: 1.0,
|
||||
SymbolIDs: []string{"pkg/foo.go::Bar"},
|
||||
FilePaths: []string{"pkg/foo.go"},
|
||||
Tags: []string{"invariant", "lock"},
|
||||
WorkspaceID: "ws-a",
|
||||
Pinned: true,
|
||||
},
|
||||
},
|
||||
}
|
||||
require.NoError(t, SaveMemories(dir, store))
|
||||
|
||||
got, err := LoadMemories(dir)
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, got)
|
||||
require.Len(t, got.Entries, 1)
|
||||
assert.Equal(t, "mem-1", got.Entries[0].ID)
|
||||
assert.Equal(t, "Bar lock invariant", got.Entries[0].Title)
|
||||
assert.Equal(t, "invariant", got.Entries[0].Kind)
|
||||
assert.Equal(t, 5, got.Entries[0].Importance)
|
||||
assert.True(t, got.Entries[0].Pinned)
|
||||
assert.Equal(t, []string{"pkg/foo.go::Bar"}, got.Entries[0].SymbolIDs)
|
||||
}
|
||||
|
||||
func TestMemoriesPersistence_EmptyOnMissingFile(t *testing.T) {
|
||||
dir := filepath.Join(t.TempDir(), "does-not-exist")
|
||||
got, err := LoadMemories(dir)
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, got)
|
||||
assert.Empty(t, got.Entries)
|
||||
}
|
||||
|
||||
func TestMemoriesPersistence_TrimDropsLowImportanceFirst(t *testing.T) {
|
||||
in := make([]MemoryEntry, 0, 10)
|
||||
for i := range 10 {
|
||||
e := MemoryEntry{ID: memID(i), Importance: 4}
|
||||
// Mark a couple as importance=1 — those should be dropped first.
|
||||
if i == 2 || i == 4 {
|
||||
e.Importance = 1
|
||||
}
|
||||
// Pin one — must survive.
|
||||
if i == 7 {
|
||||
e.Pinned = true
|
||||
e.Importance = 1
|
||||
}
|
||||
in = append(in, e)
|
||||
}
|
||||
out := trimMemories(in, 6)
|
||||
require.Len(t, out, 6)
|
||||
|
||||
pinnedOrHi := map[string]bool{}
|
||||
for _, e := range out {
|
||||
pinnedOrHi[e.ID] = true
|
||||
}
|
||||
assert.True(t, pinnedOrHi[memID(7)], "pinned[7] must survive even with importance=1")
|
||||
// Low-importance non-pinned (i=2, i=4) should both be dropped.
|
||||
assert.False(t, pinnedOrHi[memID(2)], "low-imp[2] must be dropped")
|
||||
assert.False(t, pinnedOrHi[memID(4)], "low-imp[4] must be dropped")
|
||||
}
|
||||
|
||||
func TestMemoriesPersistence_TrimFallsBackToOldestNonPinned(t *testing.T) {
|
||||
// All entries have importance > 2 — the first pass can shed none,
|
||||
// so the fallback pass must shed the oldest non-pinned.
|
||||
in := make([]MemoryEntry, 0, 5)
|
||||
for i := range 5 {
|
||||
in = append(in, MemoryEntry{ID: memID(i), Importance: 5, Pinned: i == 4})
|
||||
}
|
||||
out := trimMemories(in, 3)
|
||||
require.Len(t, out, 3)
|
||||
// The pinned entry (i=4) and the newest non-pinned tail must survive.
|
||||
survived := map[string]bool{}
|
||||
for _, e := range out {
|
||||
survived[e.ID] = true
|
||||
}
|
||||
assert.True(t, survived[memID(4)], "pinned[4] must survive")
|
||||
}
|
||||
|
||||
func TestMemoriesPersistence_TrimNoopUnderCap(t *testing.T) {
|
||||
in := []MemoryEntry{{ID: "a"}, {ID: "b"}, {ID: "c"}}
|
||||
out := trimMemories(in, 10)
|
||||
assert.Equal(t, in, out)
|
||||
}
|
||||
|
||||
func memID(i int) string {
|
||||
return "mem-" + string(rune('a'+i))
|
||||
}
|
||||
@@ -0,0 +1,11 @@
|
||||
package persistence
|
||||
|
||||
// NopStore is a no-op persistence backend used when caching is disabled.
|
||||
type NopStore struct{}
|
||||
|
||||
func (NopStore) Check(_, _, _ string) bool { return false }
|
||||
func (NopStore) Load(_, _, _ string) (*Snapshot, error) { return nil, ErrNotFound }
|
||||
func (NopStore) Save(_ *Snapshot) error { return nil }
|
||||
func (NopStore) Validate(_, _, _ string) bool { return false }
|
||||
func (NopStore) Evict(_, _, _ string) error { return nil }
|
||||
func (NopStore) Close() error { return nil }
|
||||
@@ -0,0 +1,143 @@
|
||||
package persistence
|
||||
|
||||
import (
|
||||
"compress/gzip"
|
||||
"encoding/gob"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"time"
|
||||
)
|
||||
|
||||
const (
|
||||
notesFile = "notes.gob.gz"
|
||||
maxNotesCap = 5000
|
||||
)
|
||||
|
||||
// NoteEntry is a single session-memory note. Notes are not graph
|
||||
// nodes; they live alongside the graph as a separate, persistent
|
||||
// side-store. A note is created via the `save_note` tool, surfaced
|
||||
// via `query_notes`, and folded into the per-session digest emitted
|
||||
// by `distill_session`.
|
||||
//
|
||||
// Auto-links capture the symbol IDs the agent (or the auto-linker)
|
||||
// determined to be referenced by the note body. Because they are
|
||||
// stored explicitly, queries by symbol stay O(notes) without
|
||||
// re-tokenising every body on every call.
|
||||
type NoteEntry struct {
|
||||
ID string
|
||||
Timestamp time.Time
|
||||
UpdatedAt time.Time
|
||||
SessionID string // MCP session that created the note ("" for shared/embedded session)
|
||||
ClientName string // MCP clientInfo.name at create time (claude-code / cursor / ...)
|
||||
Body string // free-form text the agent wrote
|
||||
SymbolID string // primary attached symbol (optional)
|
||||
FilePath string // primary attached file (optional)
|
||||
RepoPrefix string // repo prefix derived from session scope or attached symbol/file
|
||||
WorkspaceID string // workspace boundary; queries scope by this
|
||||
ProjectID string // project sub-boundary
|
||||
Tags []string // free-form labels — "decision", "bug", "todo", ...
|
||||
AutoLinks []string // symbol IDs referenced by the body (auto-detected + explicit links)
|
||||
Pinned bool // pinned notes are never evicted by the cap
|
||||
}
|
||||
|
||||
// NoteStore is the persisted shape: a versioned, repo-scoped list of
|
||||
// entries. Same persistence shape as FeedbackStore so the cache
|
||||
// directory layout stays consistent.
|
||||
type NoteStore struct {
|
||||
Version string
|
||||
RepoPath string
|
||||
Entries []NoteEntry
|
||||
}
|
||||
|
||||
// NotesDir resolves the cache directory holding the notes file for
|
||||
// the given repo. Mirrors FeedbackDir so the two side-stores share
|
||||
// a per-repo cache subdirectory.
|
||||
func NotesDir(cacheDir, repoPath string) string {
|
||||
return filepath.Join(cacheDir, RepoCacheKey(repoPath))
|
||||
}
|
||||
|
||||
// LoadNotes reads a note store from disk. Returns an empty store
|
||||
// when the file does not exist (cold start is normal).
|
||||
func LoadNotes(dir string) (*NoteStore, error) {
|
||||
path := filepath.Join(dir, notesFile)
|
||||
f, err := os.Open(path)
|
||||
if err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
return &NoteStore{}, nil
|
||||
}
|
||||
return nil, fmt.Errorf("persistence: open notes: %w", err)
|
||||
}
|
||||
defer f.Close()
|
||||
|
||||
gz, err := gzip.NewReader(f)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("persistence: gzip reader notes: %w", err)
|
||||
}
|
||||
defer gz.Close()
|
||||
|
||||
var store NoteStore
|
||||
if err := gob.NewDecoder(gz).Decode(&store); err != nil {
|
||||
return nil, fmt.Errorf("persistence: gob decode notes: %w", err)
|
||||
}
|
||||
return &store, nil
|
||||
}
|
||||
|
||||
// SaveNotes writes the store to disk with gob+gzip. Trimming
|
||||
// honours pinned notes: when the entry count exceeds maxNotesCap,
|
||||
// the oldest non-pinned entries are dropped first.
|
||||
func SaveNotes(dir string, store *NoteStore) error {
|
||||
if len(store.Entries) > maxNotesCap {
|
||||
store.Entries = trimNotes(store.Entries, maxNotesCap)
|
||||
}
|
||||
|
||||
if err := os.MkdirAll(dir, 0o755); err != nil {
|
||||
return fmt.Errorf("persistence: mkdir notes: %w", err)
|
||||
}
|
||||
|
||||
path := filepath.Join(dir, notesFile)
|
||||
f, err := os.Create(path)
|
||||
if err != nil {
|
||||
return fmt.Errorf("persistence: create notes: %w", err)
|
||||
}
|
||||
|
||||
gz := gzip.NewWriter(f)
|
||||
enc := gob.NewEncoder(gz)
|
||||
|
||||
if err := enc.Encode(store); err != nil {
|
||||
_ = gz.Close()
|
||||
_ = f.Close()
|
||||
return fmt.Errorf("persistence: gob encode notes: %w", err)
|
||||
}
|
||||
|
||||
if err := gz.Close(); err != nil {
|
||||
_ = f.Close()
|
||||
return fmt.Errorf("persistence: gzip close notes: %w", err)
|
||||
}
|
||||
|
||||
return f.Close()
|
||||
}
|
||||
|
||||
// trimNotes drops the oldest non-pinned entries until len(out) <= cap.
|
||||
// Order is preserved so callers iterate chronologically. If pinned
|
||||
// entries alone exceed cap, every pinned entry is still retained
|
||||
// — the cap is a soft ceiling for the unpinned tail.
|
||||
func trimNotes(in []NoteEntry, cap int) []NoteEntry {
|
||||
if len(in) <= cap {
|
||||
return in
|
||||
}
|
||||
excess := len(in) - cap
|
||||
|
||||
// Walk forward dropping non-pinned entries until we have shed
|
||||
// `excess` of them, then return the tail.
|
||||
out := make([]NoteEntry, 0, cap)
|
||||
dropped := 0
|
||||
for _, e := range in {
|
||||
if dropped < excess && !e.Pinned {
|
||||
dropped++
|
||||
continue
|
||||
}
|
||||
out = append(out, e)
|
||||
}
|
||||
return out
|
||||
}
|
||||
@@ -0,0 +1,82 @@
|
||||
package persistence
|
||||
|
||||
import (
|
||||
"path/filepath"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestNotesPersistence_RoundTrip(t *testing.T) {
|
||||
cache := t.TempDir()
|
||||
dir := NotesDir(cache, "/tmp/repo-a")
|
||||
|
||||
store := &NoteStore{
|
||||
Version: "test",
|
||||
RepoPath: "/tmp/repo-a",
|
||||
Entries: []NoteEntry{
|
||||
{
|
||||
ID: "nt-1",
|
||||
Timestamp: time.Now().UTC(),
|
||||
UpdatedAt: time.Now().UTC(),
|
||||
SessionID: "sess-1",
|
||||
Body: "decision: switch to fastpath",
|
||||
SymbolID: "pkg/foo.go::Bar",
|
||||
Tags: []string{"decision"},
|
||||
AutoLinks: []string{"pkg/foo.go::Bar"},
|
||||
Pinned: true,
|
||||
},
|
||||
},
|
||||
}
|
||||
require.NoError(t, SaveNotes(dir, store))
|
||||
|
||||
got, err := LoadNotes(dir)
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, got)
|
||||
require.Len(t, got.Entries, 1)
|
||||
assert.Equal(t, "nt-1", got.Entries[0].ID)
|
||||
assert.Equal(t, "pkg/foo.go::Bar", got.Entries[0].SymbolID)
|
||||
assert.True(t, got.Entries[0].Pinned)
|
||||
}
|
||||
|
||||
func TestNotesPersistence_EmptyOnMissingFile(t *testing.T) {
|
||||
dir := filepath.Join(t.TempDir(), "does-not-exist")
|
||||
got, err := LoadNotes(dir)
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, got)
|
||||
assert.Empty(t, got.Entries)
|
||||
}
|
||||
|
||||
func TestNotesPersistence_TrimDropsOldestUnpinned(t *testing.T) {
|
||||
in := make([]NoteEntry, 0, 10)
|
||||
for i := range 10 {
|
||||
in = append(in, NoteEntry{ID: noteID(i), Pinned: i%5 == 0})
|
||||
}
|
||||
out := trimNotes(in, 6)
|
||||
require.Len(t, out, 6)
|
||||
|
||||
// Both pinned entries (i=0, i=5) must survive.
|
||||
pinnedIDs := map[string]bool{}
|
||||
for _, e := range out {
|
||||
if e.Pinned {
|
||||
pinnedIDs[e.ID] = true
|
||||
}
|
||||
}
|
||||
assert.True(t, pinnedIDs[noteID(0)], "pinned[0] must survive")
|
||||
assert.True(t, pinnedIDs[noteID(5)], "pinned[5] must survive")
|
||||
|
||||
// The newest entries should be present (LIFO preserves the tail).
|
||||
assert.Equal(t, noteID(9), out[len(out)-1].ID)
|
||||
}
|
||||
|
||||
func TestNotesPersistence_TrimNoopUnderCap(t *testing.T) {
|
||||
in := []NoteEntry{{ID: "a"}, {ID: "b"}, {ID: "c"}}
|
||||
out := trimNotes(in, 10)
|
||||
assert.Equal(t, in, out)
|
||||
}
|
||||
|
||||
func noteID(i int) string {
|
||||
return "nt-" + string(rune('a'+i))
|
||||
}
|
||||
@@ -0,0 +1,102 @@
|
||||
package persistence
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"time"
|
||||
)
|
||||
|
||||
// CLI-verb ledger. The cli_events table records one row per CLI verb that
|
||||
// ran under a correlation session id. Unlike the opt-in savings ledger it
|
||||
// carries no consent gate and no daily-aggregate dimension: it is a thin,
|
||||
// consent-free, per-event log scoped by session_id so a context-budget
|
||||
// receipt can name the exact verbs (and the safety steps derived from them)
|
||||
// a single agent session drove through the CLI.
|
||||
//
|
||||
// Modeled on savings_events: a single INSERT per event (durable at the call,
|
||||
// nothing batched), and time- / session-queryable reads. The table is keyed
|
||||
// on session_id so the receipt can read back exactly one session's verbs.
|
||||
|
||||
// CLIEvent is one recorded CLI-verb invocation.
|
||||
type CLIEvent struct {
|
||||
TS time.Time
|
||||
SessionID string
|
||||
Verb string
|
||||
}
|
||||
|
||||
// AddCLIEvent books one CLI-verb invocation as a single INSERT. Durable at
|
||||
// the call — a SIGKILLed process loses nothing. A zero ts is stamped with
|
||||
// the current time.
|
||||
func (s *SidecarStore) AddCLIEvent(ts time.Time, sessionID, verb string) error {
|
||||
if s == nil {
|
||||
return nil
|
||||
}
|
||||
s.writeMu.Lock()
|
||||
defer s.writeMu.Unlock()
|
||||
|
||||
if ts.IsZero() {
|
||||
ts = time.Now()
|
||||
}
|
||||
if _, err := s.db.Exec(
|
||||
`INSERT INTO cli_events (ts, session_id, verb) VALUES (?,?,?)`,
|
||||
ts.UTC().UnixNano(), sessionID, verb,
|
||||
); err != nil {
|
||||
return fmt.Errorf("persistence: cli event: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// CLIEventsSince returns events with ts >= since, oldest first.
|
||||
// since=zero returns everything.
|
||||
func (s *SidecarStore) CLIEventsSince(since time.Time) ([]CLIEvent, error) {
|
||||
if s == nil {
|
||||
return nil, nil
|
||||
}
|
||||
rows, err := s.db.Query(
|
||||
`SELECT ts, session_id, verb FROM cli_events WHERE ts >= ? ORDER BY ts, id`,
|
||||
unixOrZero(since),
|
||||
)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("persistence: cli events since: %w", err)
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
var out []CLIEvent
|
||||
for rows.Next() {
|
||||
var ev CLIEvent
|
||||
var tsN int64
|
||||
if err := rows.Scan(&tsN, &ev.SessionID, &ev.Verb); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
ev.TS = time.Unix(0, tsN).UTC()
|
||||
out = append(out, ev)
|
||||
}
|
||||
return out, rows.Err()
|
||||
}
|
||||
|
||||
// CLIEventsBySession returns every event for one correlation session id,
|
||||
// oldest first.
|
||||
func (s *SidecarStore) CLIEventsBySession(sessionID string) ([]CLIEvent, error) {
|
||||
if s == nil {
|
||||
return nil, nil
|
||||
}
|
||||
rows, err := s.db.Query(
|
||||
`SELECT ts, session_id, verb FROM cli_events WHERE session_id = ? ORDER BY ts, id`,
|
||||
sessionID,
|
||||
)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("persistence: cli events by session: %w", err)
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
var out []CLIEvent
|
||||
for rows.Next() {
|
||||
var ev CLIEvent
|
||||
var tsN int64
|
||||
if err := rows.Scan(&tsN, &ev.SessionID, &ev.Verb); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
ev.TS = time.Unix(0, tsN).UTC()
|
||||
out = append(out, ev)
|
||||
}
|
||||
return out, rows.Err()
|
||||
}
|
||||
@@ -0,0 +1,106 @@
|
||||
package persistence
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// TestCLIEvents_RoundTrip books a few CLI-verb events under two sessions
|
||||
// and proves both read paths: time-windowed (Since) and per-session
|
||||
// (BySession). Events survive a close + reopen — durable at the call.
|
||||
func TestCLIEvents_RoundTrip(t *testing.T) {
|
||||
sc, path := openTestSidecar(t)
|
||||
|
||||
t0 := time.Date(2026, 6, 1, 10, 0, 0, 0, time.UTC)
|
||||
events := []CLIEvent{
|
||||
{TS: t0, SessionID: "sess-1", Verb: "edit.verify"},
|
||||
{TS: t0.Add(time.Minute), SessionID: "sess-1", Verb: "edit.guards"},
|
||||
{TS: t0.Add(2 * time.Minute), SessionID: "sess-2", Verb: "query.stats"},
|
||||
}
|
||||
for _, ev := range events {
|
||||
if err := sc.AddCLIEvent(ev.TS, ev.SessionID, ev.Verb); err != nil {
|
||||
t.Fatalf("AddCLIEvent: %v", err)
|
||||
}
|
||||
}
|
||||
if err := sc.Close(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
sc2, err := OpenSidecar(path)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer sc2.Close()
|
||||
|
||||
// Since(zero) returns everything, oldest first.
|
||||
all, err := sc2.CLIEventsSince(time.Time{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(all) != 3 {
|
||||
t.Fatalf("CLIEventsSince(zero) = %d events, want 3", len(all))
|
||||
}
|
||||
if all[0].Verb != "edit.verify" || all[2].Verb != "query.stats" {
|
||||
t.Errorf("order = %q…%q, want edit.verify…query.stats", all[0].Verb, all[2].Verb)
|
||||
}
|
||||
if !all[0].TS.Equal(t0) || all[0].SessionID != "sess-1" {
|
||||
t.Errorf("first event = %+v, want ts=%v session=sess-1", all[0], t0)
|
||||
}
|
||||
|
||||
// Since a cutoff after the first two events: only sess-2 remains.
|
||||
recent, err := sc2.CLIEventsSince(t0.Add(90 * time.Second))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(recent) != 1 || recent[0].Verb != "query.stats" {
|
||||
t.Errorf("CLIEventsSince(cutoff) = %+v, want only query.stats", recent)
|
||||
}
|
||||
|
||||
// BySession returns just that session's verbs, oldest first.
|
||||
s1, err := sc2.CLIEventsBySession("sess-1")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(s1) != 2 || s1[0].Verb != "edit.verify" || s1[1].Verb != "edit.guards" {
|
||||
t.Errorf("CLIEventsBySession(sess-1) = %+v, want [edit.verify edit.guards]", s1)
|
||||
}
|
||||
s2, err := sc2.CLIEventsBySession("sess-2")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(s2) != 1 || s2[0].Verb != "query.stats" {
|
||||
t.Errorf("CLIEventsBySession(sess-2) = %+v, want [query.stats]", s2)
|
||||
}
|
||||
|
||||
// An unseen session is a clean empty, never an error.
|
||||
none, err := sc2.CLIEventsBySession("nope")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(none) != 0 {
|
||||
t.Errorf("CLIEventsBySession(unknown) = %+v, want empty", none)
|
||||
}
|
||||
}
|
||||
|
||||
// TestCLIEvents_ZeroTSStamped: a zero ts is stamped with the current time
|
||||
// rather than persisted as the unix epoch, so a "since recent" read still
|
||||
// finds the row.
|
||||
func TestCLIEvents_ZeroTSStamped(t *testing.T) {
|
||||
sc, _ := openTestSidecar(t)
|
||||
defer sc.Close()
|
||||
|
||||
before := time.Now().Add(-time.Second)
|
||||
if err := sc.AddCLIEvent(time.Time{}, "sess-z", "edit.tests"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
got, err := sc.CLIEventsBySession("sess-z")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(got) != 1 {
|
||||
t.Fatalf("want 1 event, got %d", len(got))
|
||||
}
|
||||
if got[0].TS.Before(before) {
|
||||
t.Errorf("zero ts was not stamped to now: %v", got[0].TS)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,220 @@
|
||||
package persistence
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
sqlite "modernc.org/sqlite"
|
||||
)
|
||||
|
||||
// Sidecar schema migrations.
|
||||
//
|
||||
// The sidecar DB has no migration framework historically: sidecarSchema was
|
||||
// run with CREATE TABLE / CREATE INDEX IF NOT EXISTS, and columns added after
|
||||
// a table's original shape were retrofitted with best-effort ALTERs. That
|
||||
// breaks when an index (or any statement) inside sidecarSchema references a
|
||||
// column the IF-NOT-EXISTS create never adds to a pre-existing table — the
|
||||
// whole schema batch aborts with "no such column" and the install can no
|
||||
// longer open its ledger.
|
||||
//
|
||||
// runMigrations replaces that with a forward-only, version-stamped sequence
|
||||
// keyed on SQLite's built-in PRAGMA user_version. sidecarSchema keeps only the
|
||||
// idempotent base shape (tables + original-shape indexes); every later column
|
||||
// and every column-dependent index lives in a migration that runs after the
|
||||
// column's ALTER. Existing installs upgrade in place on the next OpenSidecar —
|
||||
// no user action, no data loss (migrations are additive only).
|
||||
//
|
||||
// Concurrency: applyOne relies on the sidecar DSN's _txlock=immediate so
|
||||
// db.Begin() takes the write lock at BEGIN, making the in-transaction
|
||||
// user_version check authoritative. Two processes opening a stale DB at once
|
||||
// serialise on busy_timeout; the loser re-reads the bumped version and skips.
|
||||
|
||||
// migration is one forward step. Steps are append-only and ascending: never
|
||||
// edit or renumber a migration that has shipped — add a new higher version.
|
||||
type migration struct {
|
||||
version int
|
||||
name string
|
||||
fn func(tx *sql.Tx) error
|
||||
}
|
||||
|
||||
// currentSidecarVersion is the schema version a fully-migrated DB reports via
|
||||
// PRAGMA user_version. It must equal the highest version in sidecarMigrations.
|
||||
const currentSidecarVersion = 1
|
||||
|
||||
// sidecarMigrations is the ordered, forward-only migration list.
|
||||
var sidecarMigrations = []migration{
|
||||
{version: 1, name: "baseline-reconcile", fn: migrateV1Baseline},
|
||||
}
|
||||
|
||||
// runMigrations applies every pending migration in order. Each runs in its own
|
||||
// IMMEDIATE-locked transaction and bumps user_version on success, so a failure
|
||||
// at version N leaves versions < N committed and stamped and is safe to retry
|
||||
// on the next open.
|
||||
func runMigrations(db *sql.DB) error {
|
||||
for _, m := range sidecarMigrations {
|
||||
if err := applyOne(db, m); err != nil {
|
||||
return fmt.Errorf("persistence: sidecar migration v%d (%s): %w", m.version, m.name, err)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// isSidecarBusyErr reports whether err is a SQLite BUSY/LOCKED result code
|
||||
// from the modernc driver (matching extended codes by their base value).
|
||||
func isSidecarBusyErr(err error) bool {
|
||||
var se *sqlite.Error
|
||||
if !errors.As(err, &se) {
|
||||
return false
|
||||
}
|
||||
// Mask the high byte so extended codes (SQLITE_BUSY_SNAPSHOT, ...) match
|
||||
// their base SQLITE_BUSY (5) / SQLITE_LOCKED (6).
|
||||
switch se.Code() & 0xff {
|
||||
case 5, 6:
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// withSidecarBusyRetry runs fn, retrying on a SQLite BUSY/LOCKED error with
|
||||
// bounded exponential backoff.
|
||||
//
|
||||
// busy_timeout (set in the sidecar DSN) covers ordinary write-lock contention
|
||||
// once the file is in WAL mode, but it does NOT cover the rollback-journal ->
|
||||
// WAL conversion the very first opener performs: that conversion takes a brief
|
||||
// EXCLUSIVE lock whose acquisition SQLite answers with an immediate, un-retried
|
||||
// SQLITE_BUSY (the busy handler is not consulted for a journal-mode change)
|
||||
// when another process has the file open. The daemon, every per-repo
|
||||
// `gortex mcp` subprocess, and the CLI can all open a fresh/stale sidecar at
|
||||
// the same instant, so a bounded application-level retry is required on top of
|
||||
// busy_timeout. fn must be idempotent — runBaseSchema (CREATE ... IF NOT
|
||||
// EXISTS) and runMigrations (user_version-gated) both are.
|
||||
func withSidecarBusyRetry(fn func() error) error {
|
||||
const (
|
||||
maxAttempts = 40
|
||||
baseDelay = 5 * time.Millisecond
|
||||
maxDelay = 250 * time.Millisecond
|
||||
)
|
||||
delay := baseDelay
|
||||
var err error
|
||||
for attempt := 0; attempt < maxAttempts; attempt++ {
|
||||
if err = fn(); err == nil || !isSidecarBusyErr(err) {
|
||||
return err
|
||||
}
|
||||
time.Sleep(delay)
|
||||
if delay *= 2; delay > maxDelay {
|
||||
delay = maxDelay
|
||||
}
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
// runBaseSchema applies the idempotent base shape (sidecarSchema) inside one
|
||||
// IMMEDIATE-locked transaction. The sidecar DSN's _txlock=immediate makes
|
||||
// db.Begin() emit BEGIN IMMEDIATE, so the reserved write lock is held from the
|
||||
// start of the batch.
|
||||
//
|
||||
// This matters for the same reason applyOne wraps its work: the base schema's
|
||||
// CREATE TABLE / CREATE INDEX IF NOT EXISTS statements write, and run in
|
||||
// autocommit each one first takes a read lock and then tries to promote to a
|
||||
// write lock. SQLite answers that promotion with an un-retryable SQLITE_BUSY
|
||||
// (busy_timeout is not consulted on a lock upgrade) when another process is
|
||||
// concurrently creating the same objects — so several gortex processes opening
|
||||
// a fresh or stale DB at once would race here, before runMigrations' own
|
||||
// IMMEDIATE transactions ever run, and the losers would fail the open. Taking
|
||||
// the write lock at BEGIN instead makes a concurrent opener block on
|
||||
// busy_timeout and then re-run the IF NOT EXISTS statements as clean no-ops, so
|
||||
// every opener succeeds.
|
||||
func runBaseSchema(db *sql.DB) error {
|
||||
tx, err := db.Begin()
|
||||
if err != nil {
|
||||
return fmt.Errorf("begin: %w", err)
|
||||
}
|
||||
defer func() { _ = tx.Rollback() }() // no-op once Commit succeeds
|
||||
if _, err := tx.Exec(sidecarSchema); err != nil {
|
||||
return err
|
||||
}
|
||||
return tx.Commit()
|
||||
}
|
||||
|
||||
// applyOne applies a single migration if the DB's user_version is below it.
|
||||
//
|
||||
// The sidecar DSN sets _txlock=immediate, so db.Begin() emits BEGIN IMMEDIATE
|
||||
// and the reserved write lock is held before the user_version read below —
|
||||
// the gate is therefore authoritative across processes. A concurrent opener
|
||||
// blocks on busy_timeout at BEGIN, then (this winner having committed) reads
|
||||
// the bumped version and returns without writing. PRAGMA user_version is
|
||||
// transactional, so on any failure the deferred Rollback reverts both the
|
||||
// partial DDL and the version bump.
|
||||
func applyOne(db *sql.DB, m migration) error {
|
||||
tx, err := db.Begin()
|
||||
if err != nil {
|
||||
return fmt.Errorf("begin: %w", err)
|
||||
}
|
||||
defer func() { _ = tx.Rollback() }() // no-op once Commit succeeds
|
||||
|
||||
var cur int
|
||||
if err := tx.QueryRow("PRAGMA user_version").Scan(&cur); err != nil {
|
||||
return fmt.Errorf("read user_version: %w", err)
|
||||
}
|
||||
if cur >= m.version {
|
||||
return nil // already applied (this or another process)
|
||||
}
|
||||
|
||||
if err := m.fn(tx); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// PRAGMA takes no bound parameters; m.version is an int constant we own.
|
||||
// Bumped last so it rolls back with the transaction on any earlier error.
|
||||
if _, err := tx.Exec(fmt.Sprintf("PRAGMA user_version = %d", m.version)); err != nil {
|
||||
return fmt.Errorf("bump user_version: %w", err)
|
||||
}
|
||||
return tx.Commit()
|
||||
}
|
||||
|
||||
// migrateV1Baseline reconciles any historical sidecar shape (which all report
|
||||
// user_version 0) to the current schema. It is purely additive and idempotent:
|
||||
// CREATE TABLE IF NOT EXISTS for every table runs earlier in sidecarSchema, so
|
||||
// this only adds the columns and column-dependent indexes that a pre-existing
|
||||
// table cannot gain from CREATE TABLE.
|
||||
//
|
||||
// savings_events.model / .client are the only column-level drift in the whole
|
||||
// sidecar history; every other shape difference is a whole-table addition
|
||||
// already covered by the base schema.
|
||||
func migrateV1Baseline(tx *sql.Tx) error {
|
||||
if err := addColumnIfMissingTx(tx, "savings_events", "model", "TEXT NOT NULL DEFAULT ''"); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := addColumnIfMissingTx(tx, "savings_events", "client", "TEXT NOT NULL DEFAULT ''"); err != nil {
|
||||
return err
|
||||
}
|
||||
// Column-dependent indexes LAST — only after the columns are guaranteed to
|
||||
// exist. These are exactly the statements that aborted sidecarSchema on a
|
||||
// database created before model/client existed.
|
||||
if _, err := tx.Exec(`CREATE INDEX IF NOT EXISTS idx_savings_events_model ON savings_events (model, ts)`); err != nil {
|
||||
return fmt.Errorf("index idx_savings_events_model: %w", err)
|
||||
}
|
||||
if _, err := tx.Exec(`CREATE INDEX IF NOT EXISTS idx_savings_events_client ON savings_events (client, ts)`); err != nil {
|
||||
return fmt.Errorf("index idx_savings_events_client: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// addColumnIfMissingTx runs ALTER TABLE ... ADD COLUMN inside a transaction,
|
||||
// swallowing ONLY the "duplicate column name" error that a table which already
|
||||
// has the column returns (idempotency for fresh / already-upgraded databases).
|
||||
// Every other error propagates — surfacing a genuine failure with its real
|
||||
// cause instead of letting a later column-dependent statement fail with a
|
||||
// misleading "no such column".
|
||||
func addColumnIfMissingTx(tx *sql.Tx, table, column, decl string) error {
|
||||
if _, err := tx.Exec("ALTER TABLE " + table + " ADD COLUMN " + column + " " + decl); err != nil {
|
||||
if strings.Contains(err.Error(), "duplicate column name") {
|
||||
return nil
|
||||
}
|
||||
return fmt.Errorf("add column %s.%s: %w", table, column, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,326 @@
|
||||
package persistence
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"database/sql"
|
||||
"fmt"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// openRaw opens a bare *sql.DB on path (driver registered by the package's
|
||||
// modernc.org/sqlite blank import) for building fixtures and reading state
|
||||
// without going through OpenSidecar's cache.
|
||||
func openRaw(t *testing.T, path string) *sql.DB {
|
||||
t.Helper()
|
||||
db, err := sql.Open("sqlite", path)
|
||||
if err != nil {
|
||||
t.Fatalf("open raw %s: %v", path, err)
|
||||
}
|
||||
return db
|
||||
}
|
||||
|
||||
func userVersion(t *testing.T, db *sql.DB) int {
|
||||
t.Helper()
|
||||
var v int
|
||||
if err := db.QueryRow("PRAGMA user_version").Scan(&v); err != nil {
|
||||
t.Fatalf("read user_version: %v", err)
|
||||
}
|
||||
return v
|
||||
}
|
||||
|
||||
func hasColumn(t *testing.T, db *sql.DB, table, col string) bool {
|
||||
t.Helper()
|
||||
rows, err := db.Query("PRAGMA table_info(" + table + ")")
|
||||
if err != nil {
|
||||
t.Fatalf("table_info(%s): %v", table, err)
|
||||
}
|
||||
defer rows.Close()
|
||||
for rows.Next() {
|
||||
var (
|
||||
cid int
|
||||
name, ctype string
|
||||
notnull, pk int
|
||||
dflt sql.NullString
|
||||
)
|
||||
if err := rows.Scan(&cid, &name, &ctype, ¬null, &dflt, &pk); err != nil {
|
||||
t.Fatalf("scan table_info(%s): %v", table, err)
|
||||
}
|
||||
if name == col {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func hasObject(t *testing.T, db *sql.DB, objType, name string) bool {
|
||||
t.Helper()
|
||||
var got string
|
||||
err := db.QueryRow("SELECT name FROM sqlite_master WHERE type=? AND name=?", objType, name).Scan(&got)
|
||||
if err == sql.ErrNoRows {
|
||||
return false
|
||||
}
|
||||
if err != nil {
|
||||
t.Fatalf("lookup %s %s: %v", objType, name, err)
|
||||
}
|
||||
return got == name
|
||||
}
|
||||
|
||||
// makeShapeC writes a pre-v0.46 sidecar: savings_events with its ORIGINAL
|
||||
// 8 columns (no model, no client) and only the original indexes, with one
|
||||
// row. user_version stays 0 — the state every install created before the
|
||||
// migration framework reports. Opening this DB with the buggy code aborted
|
||||
// with "no such column: model".
|
||||
func makeShapeC(t *testing.T, path string) {
|
||||
t.Helper()
|
||||
db := openRaw(t, path)
|
||||
defer db.Close()
|
||||
_, err := db.Exec(`
|
||||
CREATE TABLE savings_events (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
ts INTEGER NOT NULL DEFAULT 0,
|
||||
session_id TEXT NOT NULL DEFAULT '',
|
||||
tool TEXT NOT NULL DEFAULT '',
|
||||
repo TEXT NOT NULL DEFAULT '',
|
||||
language TEXT NOT NULL DEFAULT '',
|
||||
returned INTEGER NOT NULL DEFAULT 0,
|
||||
saved INTEGER NOT NULL DEFAULT 0
|
||||
);
|
||||
CREATE INDEX idx_savings_events_ts ON savings_events (ts);
|
||||
CREATE INDEX idx_savings_events_tool ON savings_events (tool, ts);
|
||||
CREATE TABLE savings_totals (bucket TEXT NOT NULL PRIMARY KEY, saved INTEGER NOT NULL DEFAULT 0, returned INTEGER NOT NULL DEFAULT 0, calls INTEGER NOT NULL DEFAULT 0) WITHOUT ROWID;
|
||||
CREATE TABLE savings_meta (key TEXT NOT NULL PRIMARY KEY, value INTEGER NOT NULL DEFAULT 0) WITHOUT ROWID;
|
||||
INSERT INTO savings_events (ts, tool, returned, saved) VALUES (123, 'get_symbol_source', 1000, 720);
|
||||
`)
|
||||
if err != nil {
|
||||
t.Fatalf("build shape-C fixture: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// TestOpenSidecarFreshDB: a brand-new DB gets the full current shape and is
|
||||
// stamped to the current version. The v1 migration's ALTERs hit the
|
||||
// duplicate-column path (columns already present from CREATE TABLE) and must
|
||||
// be swallowed inside the transaction without wedging the open.
|
||||
func TestOpenSidecarFreshDB(t *testing.T) {
|
||||
path := filepath.Join(t.TempDir(), "sidecar.sqlite")
|
||||
st, err := OpenSidecar(path)
|
||||
if err != nil {
|
||||
t.Fatalf("OpenSidecar fresh: %v", err)
|
||||
}
|
||||
defer st.Close()
|
||||
|
||||
if !hasColumn(t, st.db, "savings_events", "model") || !hasColumn(t, st.db, "savings_events", "client") {
|
||||
t.Fatal("fresh DB missing model/client columns")
|
||||
}
|
||||
if !hasObject(t, st.db, "index", "idx_savings_events_model") || !hasObject(t, st.db, "index", "idx_savings_events_client") {
|
||||
t.Fatal("fresh DB missing model/client indexes")
|
||||
}
|
||||
if v := userVersion(t, st.db); v != currentSidecarVersion {
|
||||
t.Fatalf("user_version = %d, want %d", v, currentSidecarVersion)
|
||||
}
|
||||
}
|
||||
|
||||
// TestOpenSidecarMigratesShapeC reproduces the user-reported crash and proves
|
||||
// the migration fixes it in place: the old 8-column DB opens cleanly, gains
|
||||
// model/client + their indexes, is stamped to v1, and its existing row
|
||||
// survives.
|
||||
func TestOpenSidecarMigratesShapeC(t *testing.T) {
|
||||
path := filepath.Join(t.TempDir(), "sidecar.sqlite")
|
||||
makeShapeC(t, path)
|
||||
|
||||
st, err := OpenSidecar(path)
|
||||
if err != nil {
|
||||
t.Fatalf("OpenSidecar shape-C (the reported crash): %v", err)
|
||||
}
|
||||
defer st.Close()
|
||||
|
||||
for _, col := range []string{"model", "client"} {
|
||||
if !hasColumn(t, st.db, "savings_events", col) {
|
||||
t.Fatalf("column %q not added by migration", col)
|
||||
}
|
||||
}
|
||||
for _, idx := range []string{"idx_savings_events_model", "idx_savings_events_client"} {
|
||||
if !hasObject(t, st.db, "index", idx) {
|
||||
t.Fatalf("index %q not created by migration", idx)
|
||||
}
|
||||
}
|
||||
if v := userVersion(t, st.db); v != currentSidecarVersion {
|
||||
t.Fatalf("user_version = %d, want %d", v, currentSidecarVersion)
|
||||
}
|
||||
|
||||
var n int
|
||||
if err := st.db.QueryRow("SELECT COUNT(*) FROM savings_events").Scan(&n); err != nil {
|
||||
t.Fatalf("count savings_events: %v", err)
|
||||
}
|
||||
if n != 1 {
|
||||
t.Fatalf("savings row count = %d after migration, want 1 (data must survive)", n)
|
||||
}
|
||||
}
|
||||
|
||||
// TestOpenSidecarShapeBCreatesMissingTables locks the load-bearing invariant
|
||||
// that sidecarSchema runs on EVERY open (never gated on user_version): an old
|
||||
// DB missing newer tables must gain them, and its data must survive.
|
||||
func TestOpenSidecarShapeBCreatesMissingTables(t *testing.T) {
|
||||
path := filepath.Join(t.TempDir(), "sidecar.sqlite")
|
||||
func() {
|
||||
db := openRaw(t, path)
|
||||
defer db.Close()
|
||||
// A v0.36-era DB: it has scopes (an early table with no secondary
|
||||
// indexes, so it survives the base-schema rerun untouched) but not
|
||||
// the later savings_* / suppressions tables.
|
||||
if _, err := db.Exec(`CREATE TABLE scopes (name TEXT NOT NULL PRIMARY KEY, description TEXT NOT NULL DEFAULT '', repos TEXT NOT NULL DEFAULT '[]', paths TEXT NOT NULL DEFAULT '[]') WITHOUT ROWID;
|
||||
INSERT INTO scopes (name, description) VALUES ('old-scope', 'keep me');`); err != nil {
|
||||
t.Fatalf("build shape-B fixture: %v", err)
|
||||
}
|
||||
}()
|
||||
|
||||
st, err := OpenSidecar(path)
|
||||
if err != nil {
|
||||
t.Fatalf("OpenSidecar shape-B: %v", err)
|
||||
}
|
||||
defer st.Close()
|
||||
|
||||
for _, tbl := range []string{"savings_events", "suppressions", "memories", "notebooks"} {
|
||||
if !hasObject(t, st.db, "table", tbl) {
|
||||
t.Fatalf("table %q not created on open of an old DB", tbl)
|
||||
}
|
||||
}
|
||||
if v := userVersion(t, st.db); v != currentSidecarVersion {
|
||||
t.Fatalf("user_version = %d, want %d", v, currentSidecarVersion)
|
||||
}
|
||||
var desc string
|
||||
if err := st.db.QueryRow("SELECT description FROM scopes WHERE name='old-scope'").Scan(&desc); err != nil {
|
||||
t.Fatalf("pre-existing scope lost: %v", err)
|
||||
}
|
||||
if desc != "keep me" {
|
||||
t.Fatalf("scope description = %q, want 'keep me'", desc)
|
||||
}
|
||||
}
|
||||
|
||||
// TestOpenSidecarIdempotentReopen: a second open of a migrated DB is a clean
|
||||
// no-op that leaves the version unchanged.
|
||||
func TestOpenSidecarIdempotentReopen(t *testing.T) {
|
||||
path := filepath.Join(t.TempDir(), "sidecar.sqlite")
|
||||
makeShapeC(t, path)
|
||||
|
||||
st, err := OpenSidecar(path)
|
||||
if err != nil {
|
||||
t.Fatalf("first open: %v", err)
|
||||
}
|
||||
if err := st.Close(); err != nil { // drops the cache entry so the reopen re-runs the path
|
||||
t.Fatalf("close: %v", err)
|
||||
}
|
||||
|
||||
st2, err := OpenSidecar(path)
|
||||
if err != nil {
|
||||
t.Fatalf("reopen: %v", err)
|
||||
}
|
||||
defer st2.Close()
|
||||
if v := userVersion(t, st2.db); v != currentSidecarVersion {
|
||||
t.Fatalf("user_version after reopen = %d, want %d", v, currentSidecarVersion)
|
||||
}
|
||||
}
|
||||
|
||||
// TestApplyOneFailureLeavesVersionUnchanged: a failing migration rolls back
|
||||
// and does not advance user_version, so the next open retries it.
|
||||
func TestApplyOneFailureLeavesVersionUnchanged(t *testing.T) {
|
||||
path := filepath.Join(t.TempDir(), "s.sqlite")
|
||||
db := openRaw(t, path)
|
||||
defer db.Close()
|
||||
|
||||
boom := migration{version: 1, name: "boom", fn: func(tx *sql.Tx) error {
|
||||
if _, err := tx.Exec("CREATE TABLE partial (x TEXT)"); err != nil {
|
||||
return err
|
||||
}
|
||||
return fmt.Errorf("synthetic failure after a partial write")
|
||||
}}
|
||||
if err := applyOne(db, boom); err == nil {
|
||||
t.Fatal("expected applyOne to surface the migration error")
|
||||
}
|
||||
if v := userVersion(t, db); v != 0 {
|
||||
t.Fatalf("user_version = %d after failed migration, want 0", v)
|
||||
}
|
||||
if hasObject(t, db, "table", "partial") {
|
||||
t.Fatal("partial table from the failed migration was not rolled back")
|
||||
}
|
||||
}
|
||||
|
||||
// TestAddColumnIfMissingTxScope: only "duplicate column name" is swallowed;
|
||||
// every other ALTER error propagates (so a real failure is not masked).
|
||||
func TestAddColumnIfMissingTxScope(t *testing.T) {
|
||||
path := filepath.Join(t.TempDir(), "s.sqlite")
|
||||
db := openRaw(t, path)
|
||||
defer db.Close()
|
||||
if _, err := db.Exec(`CREATE TABLE t (a TEXT)`); err != nil {
|
||||
t.Fatalf("create table: %v", err)
|
||||
}
|
||||
|
||||
tx, err := db.Begin()
|
||||
if err != nil {
|
||||
t.Fatalf("begin: %v", err)
|
||||
}
|
||||
defer func() { _ = tx.Rollback() }()
|
||||
|
||||
if err := addColumnIfMissingTx(tx, "t", "b", "TEXT NOT NULL DEFAULT ''"); err != nil {
|
||||
t.Fatalf("adding a new column should succeed: %v", err)
|
||||
}
|
||||
if err := addColumnIfMissingTx(tx, "t", "b", "TEXT NOT NULL DEFAULT ''"); err != nil {
|
||||
t.Fatalf("duplicate column should be swallowed: %v", err)
|
||||
}
|
||||
if err := addColumnIfMissingTx(tx, "does_not_exist", "x", "TEXT"); err == nil {
|
||||
t.Fatal("a non-duplicate error (no such table) must propagate")
|
||||
}
|
||||
}
|
||||
|
||||
// TestOpenSidecarConcurrentProcesses is the regression guard for the headline
|
||||
// concurrency hazard: several processes opening a stale DB at once must ALL
|
||||
// succeed (one migrates under the IMMEDIATE write lock, the others wait on
|
||||
// busy_timeout and skip). With a plain DEFERRED begin the losers hit an
|
||||
// un-retryable SQLITE_BUSY — this test fails without _txlock=immediate.
|
||||
func TestOpenSidecarConcurrentProcesses(t *testing.T) {
|
||||
if path := os.Getenv("GORTEX_SIDECAR_MIGRATE_CHILD"); path != "" {
|
||||
// Child process: open once, report via exit code, never spawn.
|
||||
st, err := OpenSidecar(path)
|
||||
if err != nil {
|
||||
fmt.Fprintln(os.Stderr, "child OpenSidecar failed:", err)
|
||||
os.Exit(3)
|
||||
}
|
||||
_ = st.Close()
|
||||
os.Exit(0)
|
||||
}
|
||||
|
||||
path := filepath.Join(t.TempDir(), "sidecar.sqlite")
|
||||
makeShapeC(t, path) // stale DB whose first open must migrate
|
||||
|
||||
const procs = 6
|
||||
cmds := make([]*exec.Cmd, procs)
|
||||
outs := make([]*bytes.Buffer, procs)
|
||||
for i := range cmds {
|
||||
buf := &bytes.Buffer{}
|
||||
cmd := exec.Command(os.Args[0], "-test.run=^TestOpenSidecarConcurrentProcesses$", "-test.count=1")
|
||||
cmd.Env = append(os.Environ(), "GORTEX_SIDECAR_MIGRATE_CHILD="+path)
|
||||
cmd.Stderr = buf
|
||||
cmds[i], outs[i] = cmd, buf
|
||||
}
|
||||
for i, cmd := range cmds {
|
||||
if err := cmd.Start(); err != nil {
|
||||
t.Fatalf("start child %d: %v", i, err)
|
||||
}
|
||||
}
|
||||
for i, cmd := range cmds {
|
||||
if err := cmd.Wait(); err != nil {
|
||||
t.Fatalf("concurrent opener %d errored (none may): %v\nstderr: %s", i, err, outs[i].String())
|
||||
}
|
||||
}
|
||||
|
||||
db := openRaw(t, path)
|
||||
defer db.Close()
|
||||
if v := userVersion(t, db); v != currentSidecarVersion {
|
||||
t.Fatalf("user_version = %d after concurrent opens, want %d", v, currentSidecarVersion)
|
||||
}
|
||||
if !hasColumn(t, db, "savings_events", "model") || !hasColumn(t, db, "savings_events", "client") {
|
||||
t.Fatal("model/client columns missing after concurrent migration")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,87 @@
|
||||
package persistence
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"time"
|
||||
)
|
||||
|
||||
// PromotedToolEntry is one workspace-learned tool promotion: a tool that
|
||||
// was deferred behind tools_search but has been promoted into the eager
|
||||
// surface for this repo/workspace after use. Persisted so the learned
|
||||
// surface survives daemon restarts. last_used_epoch drives demotion — a
|
||||
// promotion unused for N session epochs is dropped back to deferred.
|
||||
type PromotedToolEntry struct {
|
||||
Tool string
|
||||
WorkspaceID string
|
||||
PromotedEpoch int64
|
||||
LastUsedEpoch int64
|
||||
UseCount int64
|
||||
UpdatedAt time.Time
|
||||
}
|
||||
|
||||
// LoadPromotedTools reads every learned promotion for a repo_key.
|
||||
func (s *SidecarStore) LoadPromotedTools(repoKey string) ([]PromotedToolEntry, error) {
|
||||
rows, err := s.db.Query(`
|
||||
SELECT tool, workspace_id, promoted_epoch, last_used_epoch, use_count, updated_at
|
||||
FROM promoted_tools WHERE repo_key = ?
|
||||
ORDER BY tool ASC`, repoKey)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("persistence: query promoted_tools: %w", err)
|
||||
}
|
||||
defer rows.Close()
|
||||
var out []PromotedToolEntry
|
||||
for rows.Next() {
|
||||
var e PromotedToolEntry
|
||||
var updatedAt int64
|
||||
if err := rows.Scan(&e.Tool, &e.WorkspaceID, &e.PromotedEpoch,
|
||||
&e.LastUsedEpoch, &e.UseCount, &updatedAt); err != nil {
|
||||
return out, fmt.Errorf("persistence: scan promoted_tool: %w", err)
|
||||
}
|
||||
e.UpdatedAt = fromUnix(updatedAt)
|
||||
out = append(out, e)
|
||||
}
|
||||
return out, rows.Err()
|
||||
}
|
||||
|
||||
// UpsertPromotedTool writes (or replaces) one learned promotion.
|
||||
func (s *SidecarStore) UpsertPromotedTool(repoKey string, e PromotedToolEntry) error {
|
||||
s.writeMu.Lock()
|
||||
defer s.writeMu.Unlock()
|
||||
_, err := s.db.Exec(`
|
||||
INSERT OR REPLACE INTO promoted_tools
|
||||
(repo_key, tool, workspace_id, promoted_epoch, last_used_epoch, use_count, updated_at)
|
||||
VALUES (?,?,?,?,?,?,?)`,
|
||||
repoKey, e.Tool, e.WorkspaceID, e.PromotedEpoch, e.LastUsedEpoch,
|
||||
e.UseCount, unixOrZero(e.UpdatedAt))
|
||||
if err != nil {
|
||||
return fmt.Errorf("persistence: upsert promoted_tool: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// DeletePromotedTool drops one learned promotion (demotion). Missing rows
|
||||
// are not errors.
|
||||
func (s *SidecarStore) DeletePromotedTool(repoKey, tool string) error {
|
||||
s.writeMu.Lock()
|
||||
defer s.writeMu.Unlock()
|
||||
_, err := s.db.Exec(`DELETE FROM promoted_tools WHERE repo_key = ? AND tool = ?`, repoKey, tool)
|
||||
return err
|
||||
}
|
||||
|
||||
// BumpSessionEpoch increments and returns the repo_key's session epoch — a
|
||||
// monotonic counter advanced once per server startup, so "unused for N
|
||||
// sessions" can be measured against last_used_epoch.
|
||||
func (s *SidecarStore) BumpSessionEpoch(repoKey string) (int64, error) {
|
||||
s.writeMu.Lock()
|
||||
defer s.writeMu.Unlock()
|
||||
if _, err := s.db.Exec(`
|
||||
INSERT INTO tool_surface_meta (repo_key, session_epoch) VALUES (?, 1)
|
||||
ON CONFLICT(repo_key) DO UPDATE SET session_epoch = session_epoch + 1`, repoKey); err != nil {
|
||||
return 0, fmt.Errorf("persistence: bump session epoch: %w", err)
|
||||
}
|
||||
var epoch int64
|
||||
if err := s.db.QueryRow(`SELECT session_epoch FROM tool_surface_meta WHERE repo_key = ?`, repoKey).Scan(&epoch); err != nil {
|
||||
return 0, fmt.Errorf("persistence: read session epoch: %w", err)
|
||||
}
|
||||
return epoch, nil
|
||||
}
|
||||
@@ -0,0 +1,384 @@
|
||||
package persistence
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"fmt"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Token-savings ledger tables. The savings ledger is machine-global —
|
||||
// unlike notes/memories it carries no repo_key partition; per-repo
|
||||
// attribution rides on the bucket key of savings_totals and the repo
|
||||
// column of savings_events instead.
|
||||
//
|
||||
// Three tables, one job each:
|
||||
// - savings_events: one row per recorded source-reading tool call.
|
||||
// Durable at the call (single INSERT inside the observation tx), so
|
||||
// a SIGKILLed server loses nothing — the property the flat-file
|
||||
// ledger's batched flush could not give.
|
||||
// - savings_totals: running aggregates keyed by bucket ('' top-line,
|
||||
// 'repo:<prefix>', 'lang:<code>'), updated transactionally with the
|
||||
// event insert so reads are point lookups instead of full scans.
|
||||
// - savings_meta: first_seen / last_updated unix-nano stamps.
|
||||
|
||||
// SavingsEvent is one recorded source-reading observation.
|
||||
type SavingsEvent struct {
|
||||
TS time.Time
|
||||
SessionID string
|
||||
Tool string
|
||||
Repo string
|
||||
Language string
|
||||
// Model is the LLM model that drove the call when known (resolved
|
||||
// from the host's hook-supplied model hint); Client is the MCP
|
||||
// client app from the initialize handshake. Both may be empty.
|
||||
Model string
|
||||
Client string
|
||||
Returned int64
|
||||
Saved int64
|
||||
}
|
||||
|
||||
// SavingsTotalsRow is the aggregate for one savings_totals bucket.
|
||||
type SavingsTotalsRow struct {
|
||||
Saved int64
|
||||
Returned int64
|
||||
Calls int64
|
||||
}
|
||||
|
||||
const savingsLegacyMigrationKind = "savings_files"
|
||||
|
||||
// AddSavingsObservation books one observation: the event row, the
|
||||
// affected totals buckets, and the meta stamps, in a single transaction.
|
||||
func (s *SidecarStore) AddSavingsObservation(ev SavingsEvent) error {
|
||||
if s == nil {
|
||||
return nil
|
||||
}
|
||||
s.writeMu.Lock()
|
||||
defer s.writeMu.Unlock()
|
||||
|
||||
tx, err := s.db.Begin()
|
||||
if err != nil {
|
||||
return fmt.Errorf("persistence: savings tx: %w", err)
|
||||
}
|
||||
defer func() { _ = tx.Rollback() }()
|
||||
|
||||
ts := ev.TS
|
||||
if ts.IsZero() {
|
||||
ts = time.Now()
|
||||
}
|
||||
tsN := ts.UTC().UnixNano()
|
||||
|
||||
if _, err := tx.Exec(
|
||||
`INSERT INTO savings_events (ts, session_id, tool, repo, language, model, client, returned, saved) VALUES (?,?,?,?,?,?,?,?,?)`,
|
||||
tsN, ev.SessionID, ev.Tool, ev.Repo, ev.Language, ev.Model, ev.Client, ev.Returned, ev.Saved,
|
||||
); err != nil {
|
||||
return fmt.Errorf("persistence: savings event: %w", err)
|
||||
}
|
||||
|
||||
buckets := []string{""}
|
||||
if ev.Repo != "" {
|
||||
buckets = append(buckets, "repo:"+ev.Repo)
|
||||
}
|
||||
if ev.Language != "" {
|
||||
buckets = append(buckets, "lang:"+ev.Language)
|
||||
}
|
||||
for _, bucket := range buckets {
|
||||
if err := upsertSavingsBucket(tx, bucket, ev.Saved, ev.Returned, 1); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
if _, err := tx.Exec(
|
||||
`INSERT INTO savings_meta (key, value) VALUES ('first_seen', ?) ON CONFLICT(key) DO NOTHING`, tsN,
|
||||
); err != nil {
|
||||
return fmt.Errorf("persistence: savings meta: %w", err)
|
||||
}
|
||||
if _, err := tx.Exec(
|
||||
`INSERT INTO savings_meta (key, value) VALUES ('last_updated', ?) ON CONFLICT(key) DO UPDATE SET value = excluded.value`, tsN,
|
||||
); err != nil {
|
||||
return fmt.Errorf("persistence: savings meta: %w", err)
|
||||
}
|
||||
|
||||
return tx.Commit()
|
||||
}
|
||||
|
||||
func upsertSavingsBucket(tx *sql.Tx, bucket string, saved, returned, calls int64) error {
|
||||
_, err := tx.Exec(
|
||||
`INSERT INTO savings_totals (bucket, saved, returned, calls) VALUES (?,?,?,?)
|
||||
ON CONFLICT(bucket) DO UPDATE SET
|
||||
saved = savings_totals.saved + excluded.saved,
|
||||
returned = savings_totals.returned + excluded.returned,
|
||||
calls = savings_totals.calls + excluded.calls`,
|
||||
bucket, saved, returned, calls,
|
||||
)
|
||||
if err != nil {
|
||||
return fmt.Errorf("persistence: savings totals: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// SavingsTotals returns every totals bucket plus the first_seen /
|
||||
// last_updated stamps. Buckets map keys are ” (top-line),
|
||||
// 'repo:<prefix>', and 'lang:<code>'. The zero time means "never".
|
||||
func (s *SidecarStore) SavingsTotals() (map[string]SavingsTotalsRow, time.Time, time.Time, error) {
|
||||
if s == nil {
|
||||
return map[string]SavingsTotalsRow{}, time.Time{}, time.Time{}, nil
|
||||
}
|
||||
rows, err := s.db.Query(`SELECT bucket, saved, returned, calls FROM savings_totals`)
|
||||
if err != nil {
|
||||
return nil, time.Time{}, time.Time{}, fmt.Errorf("persistence: savings totals: %w", err)
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
out := make(map[string]SavingsTotalsRow)
|
||||
for rows.Next() {
|
||||
var bucket string
|
||||
var r SavingsTotalsRow
|
||||
if err := rows.Scan(&bucket, &r.Saved, &r.Returned, &r.Calls); err != nil {
|
||||
return nil, time.Time{}, time.Time{}, err
|
||||
}
|
||||
out[bucket] = r
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return nil, time.Time{}, time.Time{}, err
|
||||
}
|
||||
|
||||
return out, s.savingsMetaTime("first_seen"), s.savingsMetaTime("last_updated"), nil
|
||||
}
|
||||
|
||||
func (s *SidecarStore) savingsMetaTime(key string) time.Time {
|
||||
var n int64
|
||||
row := s.db.QueryRow(`SELECT value FROM savings_meta WHERE key = ?`, key)
|
||||
if err := row.Scan(&n); err != nil || n == 0 {
|
||||
return time.Time{}
|
||||
}
|
||||
return time.Unix(0, n).UTC()
|
||||
}
|
||||
|
||||
// SavingsEventsSince returns events with ts >= since, oldest first.
|
||||
// since=zero returns everything.
|
||||
func (s *SidecarStore) SavingsEventsSince(since time.Time) ([]SavingsEvent, error) {
|
||||
if s == nil {
|
||||
return nil, nil
|
||||
}
|
||||
rows, err := s.db.Query(
|
||||
`SELECT ts, session_id, tool, repo, language, model, client, returned, saved
|
||||
FROM savings_events WHERE ts >= ? ORDER BY ts, id`,
|
||||
unixOrZero(since),
|
||||
)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("persistence: savings events: %w", err)
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
var out []SavingsEvent
|
||||
for rows.Next() {
|
||||
var ev SavingsEvent
|
||||
var tsN int64
|
||||
if err := rows.Scan(&tsN, &ev.SessionID, &ev.Tool, &ev.Repo, &ev.Language, &ev.Model, &ev.Client, &ev.Returned, &ev.Saved); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
ev.TS = time.Unix(0, tsN).UTC()
|
||||
out = append(out, ev)
|
||||
}
|
||||
return out, rows.Err()
|
||||
}
|
||||
|
||||
// SavingsLegacyImportDone reports whether the one-shot flat-file
|
||||
// (savings.json + savings.jsonl) import has already run.
|
||||
func (s *SidecarStore) SavingsLegacyImportDone() bool {
|
||||
if s == nil {
|
||||
return true
|
||||
}
|
||||
return s.migrationDone("", savingsLegacyMigrationKind)
|
||||
}
|
||||
|
||||
// ImportLegacySavings seeds the ledger from the flat-file era: bucket
|
||||
// totals from the cumulative savings.json and event rows from the
|
||||
// savings.jsonl log. Idempotent — guarded by a migration mark, which is
|
||||
// set even for an empty import so the file probing never repeats. The
|
||||
// mark is checked and written inside the import transaction, so two
|
||||
// processes racing the first start (daemon + CLI) cannot both seed the
|
||||
// ledger: the loser either sees the winner's mark or aborts on the
|
||||
// write conflict. The caller owns reading (and afterwards renaming)
|
||||
// the legacy files.
|
||||
func (s *SidecarStore) ImportLegacySavings(buckets map[string]SavingsTotalsRow, firstSeen, lastUpdated time.Time, events []SavingsEvent) error {
|
||||
if s == nil || s.migrationDone("", savingsLegacyMigrationKind) {
|
||||
return nil
|
||||
}
|
||||
s.writeMu.Lock()
|
||||
defer s.writeMu.Unlock()
|
||||
|
||||
tx, err := s.db.Begin()
|
||||
if err != nil {
|
||||
return fmt.Errorf("persistence: savings import tx: %w", err)
|
||||
}
|
||||
defer func() { _ = tx.Rollback() }()
|
||||
|
||||
// Re-check under the transaction — the cheap pre-check above races
|
||||
// with other writers (in-process via writeMu it cannot, but another
|
||||
// process opening the same database can).
|
||||
var marked int
|
||||
if err := tx.QueryRow(
|
||||
`SELECT COUNT(1) FROM migration_marks WHERE repo_key = '' AND kind = ?`, savingsLegacyMigrationKind,
|
||||
).Scan(&marked); err != nil {
|
||||
return fmt.Errorf("persistence: savings import mark check: %w", err)
|
||||
}
|
||||
if marked > 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
for bucket, r := range buckets {
|
||||
if err := upsertSavingsBucket(tx, bucket, r.Saved, r.Returned, r.Calls); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
for _, ev := range events {
|
||||
if _, err := tx.Exec(
|
||||
`INSERT INTO savings_events (ts, session_id, tool, repo, language, model, client, returned, saved) VALUES (?,?,?,?,?,?,?,?,?)`,
|
||||
unixOrZero(ev.TS), ev.SessionID, ev.Tool, ev.Repo, ev.Language, ev.Model, ev.Client, ev.Returned, ev.Saved,
|
||||
); err != nil {
|
||||
return fmt.Errorf("persistence: savings import event: %w", err)
|
||||
}
|
||||
}
|
||||
if !firstSeen.IsZero() {
|
||||
if _, err := tx.Exec(
|
||||
`INSERT INTO savings_meta (key, value) VALUES ('first_seen', ?)
|
||||
ON CONFLICT(key) DO UPDATE SET value = MIN(savings_meta.value, excluded.value)`,
|
||||
unixOrZero(firstSeen),
|
||||
); err != nil {
|
||||
return fmt.Errorf("persistence: savings import meta: %w", err)
|
||||
}
|
||||
}
|
||||
if !lastUpdated.IsZero() {
|
||||
if _, err := tx.Exec(
|
||||
`INSERT INTO savings_meta (key, value) VALUES ('last_updated', ?)
|
||||
ON CONFLICT(key) DO UPDATE SET value = MAX(savings_meta.value, excluded.value)`,
|
||||
unixOrZero(lastUpdated),
|
||||
); err != nil {
|
||||
return fmt.Errorf("persistence: savings import meta: %w", err)
|
||||
}
|
||||
}
|
||||
if _, err := tx.Exec(
|
||||
`INSERT OR REPLACE INTO migration_marks (repo_key, kind, done_at) VALUES ('', ?, ?)`,
|
||||
savingsLegacyMigrationKind, time.Now().UTC().UnixNano(),
|
||||
); err != nil {
|
||||
return fmt.Errorf("persistence: savings import mark: %w", err)
|
||||
}
|
||||
return tx.Commit()
|
||||
}
|
||||
|
||||
// SavingsToolTotals aggregates events per tool over ts >= since
|
||||
// (zero = all time), tokens-saved descending — the dashboard breakdown
|
||||
// without materializing the event history into Go.
|
||||
func (s *SidecarStore) SavingsToolTotals(since time.Time) ([]SavingsToolRow, error) {
|
||||
if s == nil {
|
||||
return nil, nil
|
||||
}
|
||||
rows, err := s.db.Query(
|
||||
`SELECT tool, SUM(saved), SUM(returned), COUNT(1)
|
||||
FROM savings_events WHERE ts >= ?
|
||||
GROUP BY tool ORDER BY SUM(saved) DESC, tool`,
|
||||
unixOrZero(since),
|
||||
)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("persistence: savings tool totals: %w", err)
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
var out []SavingsToolRow
|
||||
for rows.Next() {
|
||||
var r SavingsToolRow
|
||||
if err := rows.Scan(&r.Tool, &r.Saved, &r.Returned, &r.Calls); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out = append(out, r)
|
||||
}
|
||||
return out, rows.Err()
|
||||
}
|
||||
|
||||
// SavingsToolRow is one per-tool aggregate row.
|
||||
type SavingsToolRow struct {
|
||||
Tool string
|
||||
Saved int64
|
||||
Returned int64
|
||||
Calls int64
|
||||
}
|
||||
|
||||
// SavingsDimRow is one row of a per-dimension (model / client) aggregate.
|
||||
type SavingsDimRow struct {
|
||||
Name string
|
||||
Saved int64
|
||||
Returned int64
|
||||
Calls int64
|
||||
}
|
||||
|
||||
// SavingsModelTotals aggregates events per attributed model over
|
||||
// ts >= since (zero = all time), tokens-saved descending. Rows with no
|
||||
// model attribution are excluded — this is the "per known model" view.
|
||||
func (s *SidecarStore) SavingsModelTotals(since time.Time) ([]SavingsDimRow, error) {
|
||||
return s.savingsDimTotals("model", since)
|
||||
}
|
||||
|
||||
// SavingsClientTotals aggregates events per MCP client over ts >= since
|
||||
// (zero = all time), tokens-saved descending. Rows with no client are
|
||||
// excluded.
|
||||
func (s *SidecarStore) SavingsClientTotals(since time.Time) ([]SavingsDimRow, error) {
|
||||
return s.savingsDimTotals("client", since)
|
||||
}
|
||||
|
||||
// savingsDimTotals is the shared GROUP BY for the model / client
|
||||
// breakdowns. column is a fixed, caller-controlled identifier (never
|
||||
// user input), so interpolating it into the statement is safe.
|
||||
func (s *SidecarStore) savingsDimTotals(column string, since time.Time) ([]SavingsDimRow, error) {
|
||||
if s == nil {
|
||||
return nil, nil
|
||||
}
|
||||
rows, err := s.db.Query(
|
||||
`SELECT `+column+`, SUM(saved), SUM(returned), COUNT(1)
|
||||
FROM savings_events WHERE ts >= ? AND `+column+` <> ''
|
||||
GROUP BY `+column+` ORDER BY SUM(saved) DESC, `+column,
|
||||
unixOrZero(since),
|
||||
)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("persistence: savings %s totals: %w", column, err)
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
var out []SavingsDimRow
|
||||
for rows.Next() {
|
||||
var r SavingsDimRow
|
||||
if err := rows.Scan(&r.Name, &r.Saved, &r.Returned, &r.Calls); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out = append(out, r)
|
||||
}
|
||||
return out, rows.Err()
|
||||
}
|
||||
|
||||
// ResetSavings wipes the savings ledger (events, totals, meta) in one
|
||||
// transaction, so a concurrent writer's observation either survives
|
||||
// whole or is wiped whole — never an orphan event without totals. The
|
||||
// legacy-import migration mark survives so renamed flat files are not
|
||||
// re-imported after a reset.
|
||||
func (s *SidecarStore) ResetSavings() error {
|
||||
if s == nil {
|
||||
return nil
|
||||
}
|
||||
s.writeMu.Lock()
|
||||
defer s.writeMu.Unlock()
|
||||
tx, err := s.db.Begin()
|
||||
if err != nil {
|
||||
return fmt.Errorf("persistence: savings reset tx: %w", err)
|
||||
}
|
||||
defer func() { _ = tx.Rollback() }()
|
||||
for _, stmt := range []string{
|
||||
`DELETE FROM savings_events`,
|
||||
`DELETE FROM savings_totals`,
|
||||
`DELETE FROM savings_meta`,
|
||||
} {
|
||||
if _, err := tx.Exec(stmt); err != nil {
|
||||
return fmt.Errorf("persistence: savings reset: %w", err)
|
||||
}
|
||||
}
|
||||
return tx.Commit()
|
||||
}
|
||||
@@ -0,0 +1,145 @@
|
||||
package persistence
|
||||
|
||||
import (
|
||||
"path/filepath"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func openTestSidecar(t *testing.T) (*SidecarStore, string) {
|
||||
t.Helper()
|
||||
path := filepath.Join(t.TempDir(), "sidecar.sqlite")
|
||||
sc, err := OpenSidecar(path)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return sc, path
|
||||
}
|
||||
|
||||
// The durability contract behind the savings ledger: an observation
|
||||
// survives a full close + reopen of the database — no flush step exists
|
||||
// to forget.
|
||||
func TestSavings_DurableAcrossReopen(t *testing.T) {
|
||||
sc, path := openTestSidecar(t)
|
||||
|
||||
ev := SavingsEvent{
|
||||
TS: time.Date(2026, 6, 1, 10, 0, 0, 0, time.UTC),
|
||||
SessionID: "sess-1",
|
||||
Tool: "get_symbol_source",
|
||||
Repo: "repo-a",
|
||||
Language: "go",
|
||||
Returned: 23,
|
||||
Saved: 77,
|
||||
}
|
||||
if err := sc.AddSavingsObservation(ev); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := sc.Close(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
sc2, err := OpenSidecar(path)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer sc2.Close()
|
||||
|
||||
buckets, firstSeen, lastUpdated, err := sc2.SavingsTotals()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
top := buckets[""]
|
||||
if top.Calls != 1 || top.Saved != 77 || top.Returned != 23 {
|
||||
t.Errorf("top-line bucket = %+v, want calls=1 saved=77 returned=23", top)
|
||||
}
|
||||
if r := buckets["repo:repo-a"]; r.Calls != 1 {
|
||||
t.Errorf("repo bucket = %+v, want calls=1", r)
|
||||
}
|
||||
if l := buckets["lang:go"]; l.Calls != 1 {
|
||||
t.Errorf("lang bucket = %+v, want calls=1", l)
|
||||
}
|
||||
if !firstSeen.Equal(ev.TS) || !lastUpdated.Equal(ev.TS) {
|
||||
t.Errorf("meta stamps = (%v, %v), want both %v", firstSeen, lastUpdated, ev.TS)
|
||||
}
|
||||
|
||||
evs, err := sc2.SavingsEventsSince(time.Time{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(evs) != 1 || evs[0].SessionID != "sess-1" || !evs[0].TS.Equal(ev.TS) {
|
||||
t.Errorf("reloaded events = %+v", evs)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSavings_MetaStampsMinMax(t *testing.T) {
|
||||
sc, _ := openTestSidecar(t)
|
||||
defer sc.Close()
|
||||
|
||||
t1 := time.Date(2026, 6, 1, 10, 0, 0, 0, time.UTC)
|
||||
t2 := t1.Add(time.Hour)
|
||||
if err := sc.AddSavingsObservation(SavingsEvent{TS: t1, Tool: "a"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := sc.AddSavingsObservation(SavingsEvent{TS: t2, Tool: "b"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
_, firstSeen, lastUpdated, err := sc.SavingsTotals()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !firstSeen.Equal(t1) {
|
||||
t.Errorf("first_seen = %v, want %v (first observation wins)", firstSeen, t1)
|
||||
}
|
||||
if !lastUpdated.Equal(t2) {
|
||||
t.Errorf("last_updated = %v, want %v (latest observation wins)", lastUpdated, t2)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSavings_ResetClearsButKeepsImportMark(t *testing.T) {
|
||||
sc, _ := openTestSidecar(t)
|
||||
defer sc.Close()
|
||||
|
||||
if err := sc.ImportLegacySavings(
|
||||
map[string]SavingsTotalsRow{"": {Saved: 100, Returned: 10, Calls: 1}},
|
||||
time.Now().UTC(), time.Now().UTC(), nil,
|
||||
); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !sc.SavingsLegacyImportDone() {
|
||||
t.Fatal("import mark must be set after ImportLegacySavings")
|
||||
}
|
||||
if err := sc.ResetSavings(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
buckets, firstSeen, _, err := sc.SavingsTotals()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(buckets) != 0 || !firstSeen.IsZero() {
|
||||
t.Errorf("reset must clear totals + meta, got buckets=%v firstSeen=%v", buckets, firstSeen)
|
||||
}
|
||||
if !sc.SavingsLegacyImportDone() {
|
||||
t.Error("reset must NOT clear the legacy-import mark (renamed files would re-import)")
|
||||
}
|
||||
}
|
||||
|
||||
func TestSavings_ImportIsIdempotent(t *testing.T) {
|
||||
sc, _ := openTestSidecar(t)
|
||||
defer sc.Close()
|
||||
|
||||
rows := map[string]SavingsTotalsRow{"": {Saved: 100, Returned: 10, Calls: 1}}
|
||||
if err := sc.ImportLegacySavings(rows, time.Time{}, time.Time{}, nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := sc.ImportLegacySavings(rows, time.Time{}, time.Time{}, nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
buckets, _, _, err := sc.SavingsTotals()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got := buckets[""].Calls; got != 1 {
|
||||
t.Errorf("calls after double import = %d, want 1", got)
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,440 @@
|
||||
package persistence
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func openTempSidecar(t *testing.T) *SidecarStore {
|
||||
t.Helper()
|
||||
path := filepath.Join(t.TempDir(), "sidecar.sqlite")
|
||||
st, err := OpenSidecar(path)
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, st)
|
||||
t.Cleanup(func() { _ = st.Close() })
|
||||
return st
|
||||
}
|
||||
|
||||
func TestSidecar_OpenEmptyPathIsNoOp(t *testing.T) {
|
||||
st, err := OpenSidecar("")
|
||||
require.NoError(t, err)
|
||||
require.Nil(t, st)
|
||||
}
|
||||
|
||||
func TestSidecar_SameAbsPathReusesHandle(t *testing.T) {
|
||||
path := filepath.Join(t.TempDir(), "sidecar.sqlite")
|
||||
a, err := OpenSidecar(path)
|
||||
require.NoError(t, err)
|
||||
b, err := OpenSidecar(path)
|
||||
require.NoError(t, err)
|
||||
require.Same(t, a, b, "same absolute path must return the cached handle")
|
||||
t.Cleanup(func() { _ = a.Close() })
|
||||
}
|
||||
|
||||
func TestSidecar_NotesRoundTrip(t *testing.T) {
|
||||
st := openTempSidecar(t)
|
||||
now := time.Now().UTC().Truncate(time.Nanosecond)
|
||||
in := NoteEntry{
|
||||
ID: "nt-1",
|
||||
Timestamp: now,
|
||||
UpdatedAt: now,
|
||||
SessionID: "sess-1",
|
||||
ClientName: "claude-code",
|
||||
Body: "decision: switch to fastpath",
|
||||
SymbolID: "pkg/foo.go::Bar",
|
||||
FilePath: "pkg/foo.go",
|
||||
RepoPrefix: "core",
|
||||
WorkspaceID: "ws-a",
|
||||
ProjectID: "proj-a",
|
||||
Tags: []string{"decision", "perf"},
|
||||
AutoLinks: []string{"pkg/foo.go::Bar"},
|
||||
Pinned: true,
|
||||
}
|
||||
require.NoError(t, st.UpsertNote("rk", in))
|
||||
|
||||
rows, err := st.LoadNotesRows("rk")
|
||||
require.NoError(t, err)
|
||||
require.Len(t, rows, 1)
|
||||
got := rows[0]
|
||||
assert.Equal(t, in.ID, got.ID)
|
||||
assert.Equal(t, in.SessionID, got.SessionID)
|
||||
assert.Equal(t, in.ClientName, got.ClientName)
|
||||
assert.Equal(t, in.Body, got.Body)
|
||||
assert.Equal(t, in.SymbolID, got.SymbolID)
|
||||
assert.Equal(t, in.FilePath, got.FilePath)
|
||||
assert.Equal(t, in.WorkspaceID, got.WorkspaceID)
|
||||
assert.Equal(t, in.Tags, got.Tags)
|
||||
assert.Equal(t, in.AutoLinks, got.AutoLinks)
|
||||
assert.True(t, got.Pinned)
|
||||
assert.WithinDuration(t, in.UpdatedAt, got.UpdatedAt, time.Microsecond)
|
||||
|
||||
// Scope isolation: another repo_key sees nothing.
|
||||
other, err := st.LoadNotesRows("other")
|
||||
require.NoError(t, err)
|
||||
require.Empty(t, other)
|
||||
|
||||
// Delete.
|
||||
require.NoError(t, st.DeleteNote("rk", "nt-1"))
|
||||
rows, err = st.LoadNotesRows("rk")
|
||||
require.NoError(t, err)
|
||||
require.Empty(t, rows)
|
||||
}
|
||||
|
||||
func TestSidecar_NotesTrimKeepsPinnedAndNewest(t *testing.T) {
|
||||
st := openTempSidecar(t)
|
||||
base := time.Now().UTC()
|
||||
for i := 0; i < 10; i++ {
|
||||
require.NoError(t, st.UpsertNote("rk", NoteEntry{
|
||||
ID: noteID(i),
|
||||
Timestamp: base.Add(time.Duration(i) * time.Second),
|
||||
UpdatedAt: base.Add(time.Duration(i) * time.Second),
|
||||
Pinned: i == 0 || i == 5,
|
||||
}))
|
||||
}
|
||||
require.NoError(t, st.TrimNotes("rk", 6))
|
||||
|
||||
rows, err := st.LoadNotesRows("rk")
|
||||
require.NoError(t, err)
|
||||
require.Len(t, rows, 6)
|
||||
ids := map[string]bool{}
|
||||
for _, r := range rows {
|
||||
ids[r.ID] = true
|
||||
}
|
||||
assert.True(t, ids[noteID(0)], "pinned[0] survives")
|
||||
assert.True(t, ids[noteID(5)], "pinned[5] survives")
|
||||
assert.True(t, ids[noteID(9)], "newest survives")
|
||||
assert.False(t, ids[noteID(1)], "oldest non-pinned dropped")
|
||||
}
|
||||
|
||||
func TestSidecar_MemoriesRoundTrip(t *testing.T) {
|
||||
st := openTempSidecar(t)
|
||||
now := time.Now().UTC()
|
||||
in := MemoryEntry{
|
||||
ID: "mem-1",
|
||||
Timestamp: now,
|
||||
UpdatedAt: now,
|
||||
LastAccessed: now,
|
||||
AccessCount: 7,
|
||||
Body: "lock invariant for Bar",
|
||||
Title: "Bar lock invariant",
|
||||
Kind: "invariant",
|
||||
Source: "manual",
|
||||
Confidence: 0.8,
|
||||
Importance: 5,
|
||||
AuthorAgent: "claude-code",
|
||||
SymbolIDs: []string{"pkg/foo.go::Bar"},
|
||||
FilePaths: []string{"pkg/foo.go"},
|
||||
AutoLinks: []string{"pkg/foo.go::Baz"},
|
||||
Tags: []string{"invariant", "lock"},
|
||||
WorkspaceID: "ws-a",
|
||||
ProjectID: "proj-a",
|
||||
RepoPrefix: "core",
|
||||
Pinned: true,
|
||||
SupersededBy: "mem-2",
|
||||
}
|
||||
require.NoError(t, st.UpsertMemory("rk", in))
|
||||
|
||||
rows, err := st.LoadMemoriesRows("rk")
|
||||
require.NoError(t, err)
|
||||
require.Len(t, rows, 1)
|
||||
got := rows[0]
|
||||
assert.Equal(t, in.ID, got.ID)
|
||||
assert.Equal(t, in.Title, got.Title)
|
||||
assert.Equal(t, in.Kind, got.Kind)
|
||||
assert.Equal(t, in.Source, got.Source)
|
||||
assert.InDelta(t, in.Confidence, got.Confidence, 1e-6)
|
||||
assert.Equal(t, in.Importance, got.Importance)
|
||||
assert.Equal(t, in.AuthorAgent, got.AuthorAgent)
|
||||
assert.Equal(t, in.SymbolIDs, got.SymbolIDs)
|
||||
assert.Equal(t, in.FilePaths, got.FilePaths)
|
||||
assert.Equal(t, in.AutoLinks, got.AutoLinks)
|
||||
assert.Equal(t, in.Tags, got.Tags)
|
||||
assert.Equal(t, uint64(7), got.AccessCount)
|
||||
assert.Equal(t, "mem-2", got.SupersededBy)
|
||||
assert.True(t, got.Pinned)
|
||||
|
||||
require.NoError(t, st.DeleteMemory("rk", "mem-1"))
|
||||
rows, err = st.LoadMemoriesRows("rk")
|
||||
require.NoError(t, err)
|
||||
require.Empty(t, rows)
|
||||
}
|
||||
|
||||
func TestSidecar_MemoriesTrimTwoPass(t *testing.T) {
|
||||
st := openTempSidecar(t)
|
||||
base := time.Now().UTC()
|
||||
for i := 0; i < 10; i++ {
|
||||
e := MemoryEntry{
|
||||
ID: memID(i),
|
||||
Timestamp: base.Add(time.Duration(i) * time.Second),
|
||||
UpdatedAt: base.Add(time.Duration(i) * time.Second),
|
||||
Importance: 4,
|
||||
}
|
||||
if i == 2 || i == 4 {
|
||||
e.Importance = 1
|
||||
}
|
||||
if i == 7 {
|
||||
e.Pinned = true
|
||||
e.Importance = 1
|
||||
}
|
||||
require.NoError(t, st.UpsertMemory("rk", e))
|
||||
}
|
||||
require.NoError(t, st.TrimMemories("rk", 6))
|
||||
|
||||
rows, err := st.LoadMemoriesRows("rk")
|
||||
require.NoError(t, err)
|
||||
require.Len(t, rows, 6)
|
||||
ids := map[string]bool{}
|
||||
for _, r := range rows {
|
||||
ids[r.ID] = true
|
||||
}
|
||||
assert.True(t, ids[memID(7)], "pinned low-imp survives")
|
||||
assert.False(t, ids[memID(2)], "low-imp dropped")
|
||||
assert.False(t, ids[memID(4)], "low-imp dropped")
|
||||
}
|
||||
|
||||
func TestSidecar_ScopesRoundTrip(t *testing.T) {
|
||||
st := openTempSidecar(t)
|
||||
require.NoError(t, st.UpsertScope(ScopeRow{
|
||||
Name: "backend", Description: "be", Repos: []string{"api", "core"}, Paths: []string{"services/x"},
|
||||
}))
|
||||
require.NoError(t, st.UpsertScope(ScopeRow{Name: "frontend", Repos: []string{"web"}}))
|
||||
|
||||
rows, err := st.LoadScopes()
|
||||
require.NoError(t, err)
|
||||
require.Len(t, rows, 2)
|
||||
assert.Equal(t, "backend", rows[0].Name)
|
||||
assert.Equal(t, []string{"api", "core"}, rows[0].Repos)
|
||||
assert.Equal(t, []string{"services/x"}, rows[0].Paths)
|
||||
assert.Equal(t, 2, st.ScopeCount())
|
||||
|
||||
require.NoError(t, st.DeleteScope("backend"))
|
||||
rows, err = st.LoadScopes()
|
||||
require.NoError(t, err)
|
||||
require.Len(t, rows, 1)
|
||||
assert.Equal(t, "frontend", rows[0].Name)
|
||||
}
|
||||
|
||||
func TestSidecar_NotebookRoundTrip(t *testing.T) {
|
||||
st := openTempSidecar(t)
|
||||
now := time.Now().UTC()
|
||||
in := NotebookRow{
|
||||
ID: "nb-1",
|
||||
Title: "design: sidecar",
|
||||
Body: "use sqlite\nfor durability",
|
||||
Tags: []string{"design", "storage"},
|
||||
SymbolIDs: []string{"pkg/p.go::Q"},
|
||||
UsedCount: 3,
|
||||
LastUsed: now,
|
||||
Created: now,
|
||||
Updated: now,
|
||||
}
|
||||
require.NoError(t, st.UpsertNotebook("rk", in))
|
||||
|
||||
got, ok := st.GetNotebookRow("rk", "nb-1")
|
||||
require.True(t, ok)
|
||||
assert.Equal(t, in.Title, got.Title)
|
||||
assert.Equal(t, in.Body, got.Body)
|
||||
assert.Equal(t, in.Tags, got.Tags)
|
||||
assert.Equal(t, in.SymbolIDs, got.SymbolIDs)
|
||||
assert.Equal(t, uint64(3), got.UsedCount)
|
||||
|
||||
rows, err := st.LoadNotebookRows("rk")
|
||||
require.NoError(t, err)
|
||||
require.Len(t, rows, 1)
|
||||
|
||||
require.NoError(t, st.DeleteNotebook("rk", "nb-1"))
|
||||
_, ok = st.GetNotebookRow("rk", "nb-1")
|
||||
require.False(t, ok)
|
||||
}
|
||||
|
||||
func TestSidecar_NotebookPrune(t *testing.T) {
|
||||
st := openTempSidecar(t)
|
||||
old := time.Now().UTC().Add(-2 * time.Hour)
|
||||
fresh := time.Now().UTC()
|
||||
require.NoError(t, st.UpsertNotebook("rk", NotebookRow{ID: "stale", Updated: old}))
|
||||
require.NoError(t, st.UpsertNotebook("rk", NotebookRow{ID: "fresh", Updated: fresh, LastUsed: fresh}))
|
||||
|
||||
require.NoError(t, st.NotebookPrune("rk", time.Now().UTC().Add(-time.Hour)))
|
||||
rows, err := st.LoadNotebookRows("rk")
|
||||
require.NoError(t, err)
|
||||
require.Len(t, rows, 1)
|
||||
assert.Equal(t, "fresh", rows[0].ID)
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Migration: legacy gob.gz / json → sqlite.
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
func TestSidecar_MigrateLegacyNotes(t *testing.T) {
|
||||
legacyDir := t.TempDir()
|
||||
require.NoError(t, SaveNotes(legacyDir, &NoteStore{Entries: []NoteEntry{
|
||||
{ID: "nt-old", Body: "legacy note", SessionID: "s1", Pinned: true},
|
||||
}}))
|
||||
|
||||
st := openTempSidecar(t)
|
||||
require.NoError(t, st.MigrateLegacyNotes("rk", legacyDir))
|
||||
|
||||
rows, err := st.LoadNotesRows("rk")
|
||||
require.NoError(t, err)
|
||||
require.Len(t, rows, 1)
|
||||
assert.Equal(t, "nt-old", rows[0].ID)
|
||||
assert.Equal(t, "legacy note", rows[0].Body)
|
||||
assert.True(t, rows[0].Pinned)
|
||||
|
||||
// Legacy file renamed to .bak.
|
||||
_, errOrig := os.Stat(filepath.Join(legacyDir, notesFile))
|
||||
assert.Error(t, errOrig, "original gob.gz must be renamed away")
|
||||
_, errBak := os.Stat(filepath.Join(legacyDir, notesFile+".bak"))
|
||||
assert.NoError(t, errBak, ".bak must exist")
|
||||
|
||||
// Idempotent: a second migrate is a no-op (no duplicate rows).
|
||||
require.NoError(t, st.MigrateLegacyNotes("rk", legacyDir))
|
||||
rows, err = st.LoadNotesRows("rk")
|
||||
require.NoError(t, err)
|
||||
require.Len(t, rows, 1)
|
||||
}
|
||||
|
||||
func TestSidecar_MigrateLegacyMemories(t *testing.T) {
|
||||
legacyDir := t.TempDir()
|
||||
require.NoError(t, SaveMemories(legacyDir, &MemoryStore{Entries: []MemoryEntry{
|
||||
{ID: "mem-old", Body: "legacy memory", Kind: "invariant", Importance: 5},
|
||||
}}))
|
||||
|
||||
st := openTempSidecar(t)
|
||||
require.NoError(t, st.MigrateLegacyMemories("rk", legacyDir))
|
||||
|
||||
rows, err := st.LoadMemoriesRows("rk")
|
||||
require.NoError(t, err)
|
||||
require.Len(t, rows, 1)
|
||||
assert.Equal(t, "mem-old", rows[0].ID)
|
||||
assert.Equal(t, "invariant", rows[0].Kind)
|
||||
|
||||
_, errBak := os.Stat(filepath.Join(legacyDir, memoriesFile+".bak"))
|
||||
assert.NoError(t, errBak, ".bak must exist")
|
||||
}
|
||||
|
||||
func TestSidecar_MigrateLegacyScopes(t *testing.T) {
|
||||
legacyPath := filepath.Join(t.TempDir(), "scopes.json")
|
||||
require.NoError(t, os.WriteFile(legacyPath, []byte(`[{"name":"be","description":"backend","repos":["api"],"paths":["svc/x"]}]`), 0o644))
|
||||
|
||||
st := openTempSidecar(t)
|
||||
require.NoError(t, st.MigrateLegacyScopes(legacyPath))
|
||||
|
||||
rows, err := st.LoadScopes()
|
||||
require.NoError(t, err)
|
||||
require.Len(t, rows, 1)
|
||||
assert.Equal(t, "be", rows[0].Name)
|
||||
assert.Equal(t, []string{"api"}, rows[0].Repos)
|
||||
assert.Equal(t, []string{"svc/x"}, rows[0].Paths)
|
||||
|
||||
_, errBak := os.Stat(legacyPath + ".bak")
|
||||
assert.NoError(t, errBak)
|
||||
|
||||
// Idempotent.
|
||||
require.NoError(t, st.MigrateLegacyScopes(legacyPath))
|
||||
assert.Equal(t, 1, st.ScopeCount())
|
||||
}
|
||||
|
||||
func TestSidecar_MigrateLegacyNotebook(t *testing.T) {
|
||||
legacyDir := t.TempDir()
|
||||
md := "---\ntitle: old entry\ntags: [a, b]\nused_count: 4\n---\n\nbody text\n"
|
||||
require.NoError(t, os.WriteFile(filepath.Join(legacyDir, "nbold.md"), []byte(md), 0o644))
|
||||
|
||||
st := openTempSidecar(t)
|
||||
importMD := func(id, contents string) (NotebookRow, bool) {
|
||||
// Minimal frontmatter parse for the test importer.
|
||||
return NotebookRow{ID: id, Title: "old entry", Body: "body text\n", Tags: []string{"a", "b"}, UsedCount: 4}, true
|
||||
}
|
||||
require.NoError(t, st.MigrateLegacyNotebook("rk", legacyDir, importMD))
|
||||
|
||||
rows, err := st.LoadNotebookRows("rk")
|
||||
require.NoError(t, err)
|
||||
require.Len(t, rows, 1)
|
||||
assert.Equal(t, "nbold", rows[0].ID)
|
||||
assert.Equal(t, "old entry", rows[0].Title)
|
||||
assert.Equal(t, uint64(4), rows[0].UsedCount)
|
||||
|
||||
_, errBak := os.Stat(filepath.Join(legacyDir, "nbold.md.bak"))
|
||||
assert.NoError(t, errBak)
|
||||
}
|
||||
|
||||
func TestSidecar_MigrateSkippedWhenTableNonEmpty(t *testing.T) {
|
||||
legacyDir := t.TempDir()
|
||||
require.NoError(t, SaveNotes(legacyDir, &NoteStore{Entries: []NoteEntry{{ID: "nt-old", Body: "legacy"}}}))
|
||||
|
||||
st := openTempSidecar(t)
|
||||
// Pre-seed the table so the import is skipped (guard on existing rows).
|
||||
require.NoError(t, st.UpsertNote("rk", NoteEntry{ID: "nt-existing", Body: "already here"}))
|
||||
require.NoError(t, st.MigrateLegacyNotes("rk", legacyDir))
|
||||
|
||||
rows, err := st.LoadNotesRows("rk")
|
||||
require.NoError(t, err)
|
||||
require.Len(t, rows, 1, "import must be skipped when the table already has rows")
|
||||
assert.Equal(t, "nt-existing", rows[0].ID)
|
||||
}
|
||||
|
||||
func TestSidecar_SuppressionsRoundTrip(t *testing.T) {
|
||||
st := openTempSidecar(t)
|
||||
now := time.Now().UTC().Truncate(time.Nanosecond)
|
||||
in := SuppressionEntry{
|
||||
IdentityKey: "ik-abc",
|
||||
Rule: "nil-deref",
|
||||
Category: "nil-deref",
|
||||
File: "pkg/x.go",
|
||||
SymbolID: "pkg/x.go::Foo",
|
||||
Reason: "guarded by a prior nil check",
|
||||
Author: "me@zzet.org",
|
||||
HitCount: 0,
|
||||
Created: now,
|
||||
}
|
||||
require.NoError(t, st.UpsertSuppression("rk", in))
|
||||
|
||||
// LoadSuppression hits.
|
||||
got, ok := st.LoadSuppression("rk", "ik-abc")
|
||||
require.True(t, ok)
|
||||
assert.Equal(t, in.IdentityKey, got.IdentityKey)
|
||||
assert.Equal(t, in.Rule, got.Rule)
|
||||
assert.Equal(t, in.Category, got.Category)
|
||||
assert.Equal(t, in.File, got.File)
|
||||
assert.Equal(t, in.SymbolID, got.SymbolID)
|
||||
assert.Equal(t, in.Reason, got.Reason)
|
||||
assert.Equal(t, in.Author, got.Author)
|
||||
assert.WithinDuration(t, now, got.Created, time.Microsecond)
|
||||
|
||||
// Missing identity → clean miss, not an error.
|
||||
_, ok = st.LoadSuppression("rk", "nope")
|
||||
assert.False(t, ok)
|
||||
|
||||
// Scope isolation: another repo_key sees nothing.
|
||||
_, ok = st.LoadSuppression("other", "ik-abc")
|
||||
assert.False(t, ok)
|
||||
|
||||
// LoadSuppressions for the repo.
|
||||
all, err := st.LoadSuppressions("rk")
|
||||
require.NoError(t, err)
|
||||
require.Len(t, all, 1)
|
||||
assert.Equal(t, "ik-abc", all[0].IdentityKey)
|
||||
|
||||
// Bump hit twice and confirm counters advance.
|
||||
require.NoError(t, st.BumpSuppressionHit("rk", "ik-abc", now.Add(time.Second)))
|
||||
require.NoError(t, st.BumpSuppressionHit("rk", "ik-abc", now.Add(2*time.Second)))
|
||||
bumped, ok := st.LoadSuppression("rk", "ik-abc")
|
||||
require.True(t, ok)
|
||||
assert.Equal(t, int64(2), bumped.HitCount)
|
||||
assert.WithinDuration(t, now.Add(2*time.Second), bumped.LastHit, time.Microsecond)
|
||||
|
||||
// Bumping a missing row is a no-op, not an error.
|
||||
require.NoError(t, st.BumpSuppressionHit("rk", "nope", now))
|
||||
|
||||
// Delete.
|
||||
require.NoError(t, st.DeleteSuppression("rk", "ik-abc"))
|
||||
_, ok = st.LoadSuppression("rk", "ik-abc")
|
||||
assert.False(t, ok)
|
||||
// Deleting a missing row is not an error.
|
||||
require.NoError(t, st.DeleteSuppression("rk", "ik-abc"))
|
||||
}
|
||||
@@ -0,0 +1,71 @@
|
||||
// Package persistence provides snapshot-based graph persistence with pluggable backends.
|
||||
package persistence
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"time"
|
||||
|
||||
"github.com/zzet/gortex/internal/graph"
|
||||
)
|
||||
|
||||
// ErrNotFound is returned when no snapshot exists for the given key.
|
||||
var ErrNotFound = errors.New("persistence: snapshot not found")
|
||||
|
||||
// Snapshot is the unit of persistence — a complete graph state at a point in time.
|
||||
// CurrentExtractionVersion is the version of the extraction OUTPUT schema. It
|
||||
// is bumped ONLY when a binary actually changes what extraction produces (a new
|
||||
// node/edge shape, a corrected graph) — never on a routine version bump. A warm
|
||||
// snapshot stays reusable across binary releases that share this version, so a
|
||||
// no-op release does not force the full cold rebuild a binary-version-string
|
||||
// gate would. Raise it in lockstep with any change that alters indexed output.
|
||||
const CurrentExtractionVersion = 1
|
||||
|
||||
type Snapshot struct {
|
||||
Version string `json:"version"`
|
||||
// ExtractionVersion is the CurrentExtractionVersion the snapshot was
|
||||
// produced with; warm reuse is gated on it, not the binary version string.
|
||||
ExtractionVersion int `json:"extraction_version,omitempty"`
|
||||
RepoPath string `json:"repo_path"`
|
||||
CommitHash string `json:"commit_hash"`
|
||||
// Branch is the git branch the snapshot was taken on. Snapshots
|
||||
// are keyed by (repo, branch) so a slot survives commits on the
|
||||
// branch; empty for a detached HEAD (keyed by commit instead).
|
||||
Branch string `json:"branch,omitempty"`
|
||||
IndexedAt time.Time `json:"indexed_at"`
|
||||
Nodes []*graph.Node `json:"nodes"`
|
||||
Edges []*graph.Edge `json:"edges"`
|
||||
FileMtimes map[string]int64 `json:"file_mtimes"`
|
||||
|
||||
// VectorIndex is the serialized HNSW vector index (nil when embeddings are disabled).
|
||||
VectorIndex []byte `json:"vector_index,omitempty"`
|
||||
// VectorDims is the embedding dimensionality (0 when embeddings are disabled).
|
||||
VectorDims int `json:"vector_dims,omitempty"`
|
||||
// VectorCount is the number of vectors in the index.
|
||||
VectorCount int `json:"vector_count,omitempty"`
|
||||
}
|
||||
|
||||
// Store is the pluggable persistence backend interface.
|
||||
// Implementations must be safe for sequential use (not required to be concurrent).
|
||||
type Store interface {
|
||||
// Check returns true if a snapshot exists for the (repo, branch)
|
||||
// slot. commitHash is used only as the slot key for a detached
|
||||
// HEAD (empty branch).
|
||||
Check(repoPath, branch, commitHash string) bool
|
||||
|
||||
// Load deserializes and returns the snapshot for the (repo,
|
||||
// branch) slot. Returns ErrNotFound if no snapshot exists.
|
||||
Load(repoPath, branch, commitHash string) (*Snapshot, error)
|
||||
|
||||
// Save serializes the snapshot to persistent storage.
|
||||
Save(snap *Snapshot) error
|
||||
|
||||
// Validate checks that an existing snapshot is compatible with
|
||||
// the current gortex version. Returns false on version mismatch.
|
||||
Validate(repoPath, branch, commitHash string) bool
|
||||
|
||||
// Evict removes the snapshot for the (repo, branch) slot.
|
||||
Evict(repoPath, branch, commitHash string) error
|
||||
|
||||
// Close releases any resources held by the backend.
|
||||
Close() error
|
||||
}
|
||||
Reference in New Issue
Block a user