chore: import upstream snapshot with attribution

internal/eval/cache.go

@@ -0,0 +1,186 @@
+package eval
+
+import (
+	"fmt"
+	"io"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"github.com/zzet/gortex/internal/platform"
+)
+
+// Cache provides filesystem-based index caching keyed by (repo_name, commit_hash).
+// Cache entries are stored under {cacheDir}/{repo_name}_{commit_hash}/ and contain
+// a .version file, graph.bin, and search.bleve/ directory.
+type Cache struct {
+	dir     string // root cache directory
+	version string // current gortex version for compatibility checks
+}
+
+// NewCache creates a Cache rooted at dir with the given gortex version string.
+//
+// If dir is empty the location is resolved by env: when $XDG_CACHE_HOME is
+// set it is honoured ($XDG_CACHE_HOME/gortex/eval-cache); otherwise the
+// historical default ~/.gortex-eval-cache/ is kept so an existing eval
+// cache is not orphaned.
+func NewCache(dir, version string) (*Cache, error) {
+	if dir == "" {
+		if v := os.Getenv("XDG_CACHE_HOME"); v != "" && filepath.IsAbs(v) {
+			dir = filepath.Join(platform.CacheDir(), "eval-cache")
+		} else {
+			home, err := os.UserHomeDir()
+			if err != nil {
+				return nil, fmt.Errorf("cache: resolve home dir: %w", err)
+			}
+			dir = filepath.Join(home, ".gortex-eval-cache")
+		}
+	}
+	return &Cache{dir: dir, version: version}, nil
+}
+
+// CacheKey generates the cache directory name for a (repo, commit) pair.
+// The key format is {repo}_{commit}.
+func CacheKey(repo, commit string) string {
+	return repo + "_" + commit
+}
+
+// entryDir returns the full path to the cache entry directory.
+func (c *Cache) entryDir(repo, commit string) string {
+	return filepath.Join(c.dir, CacheKey(repo, commit))
+}
+
+// versionFile returns the path to the .version file inside a cache entry.
+func (c *Cache) versionFile(repo, commit string) string {
+	return filepath.Join(c.entryDir(repo, commit), ".version")
+}
+
+// Check returns true if a cached index exists for the (repo, commit) pair.
+// It verifies the entry directory exists and contains the expected files.
+func (c *Cache) Check(repo, commit string) bool {
+	dir := c.entryDir(repo, commit)
+	info, err := os.Stat(dir)
+	if err != nil || !info.IsDir() {
+		return false
+	}
+	// Verify .version file exists.
+	if _, err := os.Stat(c.versionFile(repo, commit)); err != nil {
+		return false
+	}
+	return true
+}
+
+// Load returns the path to the cached index directory for the (repo, commit) pair.
+// Returns an error if the cache entry does not exist.
+func (c *Cache) Load(repo, commit string) (string, error) {
+	dir := c.entryDir(repo, commit)
+	info, err := os.Stat(dir)
+	if err != nil {
+		return "", fmt.Errorf("cache: entry not found for %s: %w", CacheKey(repo, commit), err)
+	}
+	if !info.IsDir() {
+		return "", fmt.Errorf("cache: entry %s is not a directory", CacheKey(repo, commit))
+	}
+	return dir, nil
+}
+
+// Store persists an index directory into the cache for the (repo, commit) pair.
+// It copies the contents of indexPath (graph.bin, search.bleve/) into the cache
+// entry directory and writes a .version file with the current gortex version.
+func (c *Cache) Store(repo, commit, indexPath string) error {
+	dir := c.entryDir(repo, commit)
+
+	// Remove any existing entry to ensure a clean store.
+	if err := os.RemoveAll(dir); err != nil {
+		return fmt.Errorf("cache: remove existing entry: %w", err)
+	}
+
+	if err := os.MkdirAll(dir, 0o755); err != nil {
+		return fmt.Errorf("cache: create entry dir: %w", err)
+	}
+
+	// Copy contents from indexPath into the cache entry.
+	if err := copyDir(indexPath, dir); err != nil {
+		// Clean up on failure.
+		_ = os.RemoveAll(dir)
+		return fmt.Errorf("cache: copy index: %w", err)
+	}
+
+	// Write .version file.
+	if err := os.WriteFile(c.versionFile(repo, commit), []byte(c.version), 0o644); err != nil {
+		_ = os.RemoveAll(dir)
+		return fmt.Errorf("cache: write version file: %w", err)
+	}
+
+	return nil
+}
+
+// Validate checks whether the cached index for (repo, commit) is compatible
+// with the current gortex version by comparing the .version file contents.
+func (c *Cache) Validate(repo, commit string) bool {
+	data, err := os.ReadFile(c.versionFile(repo, commit))
+	if err != nil {
+		return false
+	}
+	return strings.TrimSpace(string(data)) == c.version
+}
+
+// Evict removes the cached index entry for the (repo, commit) pair.
+func (c *Cache) Evict(repo, commit string) error {
+	dir := c.entryDir(repo, commit)
+	if err := os.RemoveAll(dir); err != nil {
+		return fmt.Errorf("cache: evict %s: %w", CacheKey(repo, commit), err)
+	}
+	return nil
+}
+
+// copyDir recursively copies the contents of src into dst.
+// dst must already exist. Only regular files and directories are copied.
+func copyDir(src, dst string) error {
+	entries, err := os.ReadDir(src)
+	if err != nil {
+		return err
+	}
+
+	for _, entry := range entries {
+		srcPath := filepath.Join(src, entry.Name())
+		dstPath := filepath.Join(dst, entry.Name())
+
+		if entry.IsDir() {
+			if err := os.MkdirAll(dstPath, 0o755); err != nil {
+				return err
+			}
+			if err := copyDir(srcPath, dstPath); err != nil {
+				return err
+			}
+		} else {
+			if err := copyFile(srcPath, dstPath); err != nil {
+				return err
+			}
+		}
+	}
+	return nil
+}
+
+// copyFile copies a single file from src to dst, preserving permissions.
+func copyFile(src, dst string) error {
+	srcFile, err := os.Open(src)
+	if err != nil {
+		return err
+	}
+	defer srcFile.Close()
+
+	srcInfo, err := srcFile.Stat()
+	if err != nil {
+		return err
+	}
+
+	dstFile, err := os.OpenFile(dst, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, srcInfo.Mode())
+	if err != nil {
+		return err
+	}
+	defer dstFile.Close()
+
+	_, err = io.Copy(dstFile, srcFile)
+	return err
+}

internal/eval/cache_property_test.go

@@ -0,0 +1,75 @@
+package eval
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"pgregory.net/rapid"
+)
+
+// Feature: eval-framework, Property 6: Cache key determinism and uniqueness
+
+// --- Generators ---
+
+// genRepoCommitPair generates a (repo, commit) pair with non-empty strings
+// that don't contain underscores (to avoid ambiguity in the key format).
+func genRepoCommitPair() *rapid.Generator[[2]string] {
+	return rapid.Custom(func(t *rapid.T) [2]string {
+		repo := rapid.StringMatching(`[a-zA-Z0-9\-\.\/]{1,50}`).Draw(t, "repo")
+		commit := rapid.StringMatching(`[a-f0-9]{7,40}`).Draw(t, "commit")
+		return [2]string{repo, commit}
+	})
+}
+
+// genDistinctRepoCommitPairs generates two distinct (repo, commit) pairs.
+func genDistinctRepoCommitPairs() *rapid.Generator[[2][2]string] {
+	return rapid.Custom(func(t *rapid.T) [2][2]string {
+		pair1 := genRepoCommitPair().Draw(t, "pair1")
+		pair2 := genRepoCommitPair().Draw(t, "pair2")
+
+		// Ensure the pairs are actually distinct
+		for pair1[0] == pair2[0] && pair1[1] == pair2[1] {
+			pair2 = genRepoCommitPair().Draw(t, "pair2_retry")
+		}
+
+		return [2][2]string{pair1, pair2}
+	})
+}
+
+// --- Property Tests ---
+
+// TestProperty6_CacheKeyDeterminism verifies that CacheKey always returns the
+// same result for the same (repo, commit) inputs.
+// **Validates: Requirements 4.4**
+func TestProperty6_CacheKeyDeterminism(t *testing.T) {
+	rapid.Check(t, func(t *rapid.T) {
+		pair := genRepoCommitPair().Draw(t, "pair")
+		repo, commit := pair[0], pair[1]
+
+		key1 := CacheKey(repo, commit)
+		key2 := CacheKey(repo, commit)
+		key3 := CacheKey(repo, commit)
+
+		assert.Equal(t, key1, key2, "CacheKey must be deterministic: first and second calls differ")
+		assert.Equal(t, key1, key3, "CacheKey must be deterministic: first and third calls differ")
+		assert.NotEmpty(t, key1, "CacheKey must produce a non-empty string")
+	})
+}
+
+// TestProperty6_CacheKeyUniqueness verifies that two distinct (repo, commit)
+// pairs always produce different cache keys.
+// **Validates: Requirements 4.4**
+func TestProperty6_CacheKeyUniqueness(t *testing.T) {
+	rapid.Check(t, func(t *rapid.T) {
+		pairs := genDistinctRepoCommitPairs().Draw(t, "pairs")
+		repo1, commit1 := pairs[0][0], pairs[0][1]
+		repo2, commit2 := pairs[1][0], pairs[1][1]
+
+		key1 := CacheKey(repo1, commit1)
+		key2 := CacheKey(repo2, commit2)
+
+		assert.NotEqual(t, key1, key2,
+			"CacheKey must produce different keys for distinct pairs: (%q, %q) vs (%q, %q)",
+			repo1, commit1, repo2, commit2)
+	})
+}

internal/eval/cache_test.go

@@ -0,0 +1,159 @@
+package eval
+
+import (
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestNewCache_EmptyDirDefaultsToHome(t *testing.T) {
+	c, err := NewCache("", "v1.0.0")
+	require.NoError(t, err)
+
+	home, err := os.UserHomeDir()
+	require.NoError(t, err)
+
+	expected := filepath.Join(home, ".gortex-eval-cache")
+	assert.Equal(t, expected, c.dir)
+}
+
+func TestCheck_ReturnsFalseForNonExistentEntry(t *testing.T) {
+	c, err := NewCache(t.TempDir(), "v1.0.0")
+	require.NoError(t, err)
+
+	assert.False(t, c.Check("myrepo", "abc123"))
+}
+
+func TestStoreAndCheck(t *testing.T) {
+	cacheDir := t.TempDir()
+	c, err := NewCache(cacheDir, "v1.0.0")
+	require.NoError(t, err)
+
+	// Create a fake index directory with some content.
+	indexDir := t.TempDir()
+	require.NoError(t, os.WriteFile(filepath.Join(indexDir, "graph.bin"), []byte("graph-data"), 0o644))
+
+	require.NoError(t, c.Store("myrepo", "abc123", indexDir))
+	assert.True(t, c.Check("myrepo", "abc123"))
+}
+
+func TestStoreAndLoad(t *testing.T) {
+	cacheDir := t.TempDir()
+	c, err := NewCache(cacheDir, "v1.0.0")
+	require.NoError(t, err)
+
+	indexDir := t.TempDir()
+	require.NoError(t, os.WriteFile(filepath.Join(indexDir, "graph.bin"), []byte("graph-data"), 0o644))
+
+	require.NoError(t, c.Store("myrepo", "abc123", indexDir))
+
+	path, err := c.Load("myrepo", "abc123")
+	require.NoError(t, err)
+
+	expected := filepath.Join(cacheDir, "myrepo_abc123")
+	assert.Equal(t, expected, path)
+
+	// Verify the copied content is intact.
+	data, err := os.ReadFile(filepath.Join(path, "graph.bin"))
+	require.NoError(t, err)
+	assert.Equal(t, "graph-data", string(data))
+}
+
+func TestStoreAndValidate_MatchingVersion(t *testing.T) {
+	cacheDir := t.TempDir()
+	c, err := NewCache(cacheDir, "v1.0.0")
+	require.NoError(t, err)
+
+	indexDir := t.TempDir()
+	require.NoError(t, os.WriteFile(filepath.Join(indexDir, "graph.bin"), []byte("data"), 0o644))
+
+	require.NoError(t, c.Store("myrepo", "abc123", indexDir))
+	assert.True(t, c.Validate("myrepo", "abc123"))
+}
+
+func TestValidate_ReturnsFalseForMismatchedVersion(t *testing.T) {
+	cacheDir := t.TempDir()
+	c1, err := NewCache(cacheDir, "v1.0.0")
+	require.NoError(t, err)
+
+	indexDir := t.TempDir()
+	require.NoError(t, os.WriteFile(filepath.Join(indexDir, "graph.bin"), []byte("data"), 0o644))
+
+	require.NoError(t, c1.Store("myrepo", "abc123", indexDir))
+
+	// Create a new cache instance with a different version.
+	c2, err := NewCache(cacheDir, "v2.0.0")
+	require.NoError(t, err)
+
+	assert.False(t, c2.Validate("myrepo", "abc123"))
+}
+
+func TestEvict_RemovesEntry(t *testing.T) {
+	cacheDir := t.TempDir()
+	c, err := NewCache(cacheDir, "v1.0.0")
+	require.NoError(t, err)
+
+	indexDir := t.TempDir()
+	require.NoError(t, os.WriteFile(filepath.Join(indexDir, "graph.bin"), []byte("data"), 0o644))
+
+	require.NoError(t, c.Store("myrepo", "abc123", indexDir))
+	assert.True(t, c.Check("myrepo", "abc123"))
+
+	require.NoError(t, c.Evict("myrepo", "abc123"))
+	assert.False(t, c.Check("myrepo", "abc123"))
+}
+
+func TestStore_OverwritesExistingEntry(t *testing.T) {
+	cacheDir := t.TempDir()
+	c, err := NewCache(cacheDir, "v1.0.0")
+	require.NoError(t, err)
+
+	// Store first version.
+	indexDir1 := t.TempDir()
+	require.NoError(t, os.WriteFile(filepath.Join(indexDir1, "graph.bin"), []byte("old-data"), 0o644))
+	require.NoError(t, c.Store("myrepo", "abc123", indexDir1))
+
+	// Store second version — should overwrite.
+	indexDir2 := t.TempDir()
+	require.NoError(t, os.WriteFile(filepath.Join(indexDir2, "graph.bin"), []byte("new-data"), 0o644))
+	require.NoError(t, c.Store("myrepo", "abc123", indexDir2))
+
+	path, err := c.Load("myrepo", "abc123")
+	require.NoError(t, err)
+
+	data, err := os.ReadFile(filepath.Join(path, "graph.bin"))
+	require.NoError(t, err)
+	assert.Equal(t, "new-data", string(data))
+}
+
+func TestVersionMismatch_StoreV1_ValidateWithV2(t *testing.T) {
+	cacheDir := t.TempDir()
+
+	// Store with v1.
+	c1, err := NewCache(cacheDir, "v1.0.0")
+	require.NoError(t, err)
+
+	indexDir := t.TempDir()
+	require.NoError(t, os.WriteFile(filepath.Join(indexDir, "graph.bin"), []byte("data"), 0o644))
+	require.NoError(t, c1.Store("myrepo", "abc123", indexDir))
+
+	// Create cache with v2 — validate should return false.
+	c2, err := NewCache(cacheDir, "v2.0.0")
+	require.NoError(t, err)
+
+	assert.False(t, c2.Validate("myrepo", "abc123"))
+
+	// Entry still exists (Check is version-agnostic).
+	assert.True(t, c2.Check("myrepo", "abc123"))
+
+	// Evict the stale entry.
+	require.NoError(t, c2.Evict("myrepo", "abc123"))
+	assert.False(t, c2.Check("myrepo", "abc123"))
+
+	// Re-store with v2.
+	require.NoError(t, c2.Store("myrepo", "abc123", indexDir))
+	assert.True(t, c2.Validate("myrepo", "abc123"))
+}

internal/eval/http_handler.go

@@ -0,0 +1,162 @@
+package eval
+
+import (
+	"encoding/json"
+	"io"
+	"net/http"
+	"strings"
+
+	mcpserver "github.com/mark3labs/mcp-go/server"
+	"github.com/zzet/gortex/internal/graph"
+	"github.com/zzet/gortex/internal/server"
+	"go.uber.org/zap"
+)
+
+// Handler extends server.Handler with eval-specific endpoints.
+// It inherits every /v1/* route from the base handler and adds
+// POST /v1/augment for the eval augment pipeline.
+type Handler struct {
+	*server.Handler
+}
+
+// NewHandler creates an eval HTTP handler that dispatches to MCP tools.
+// It provides all server /v1/* endpoints plus POST /v1/augment.
+func NewHandler(mcpServer *mcpserver.MCPServer, g *graph.Graph, version string, logger *zap.Logger) *Handler {
+	base := server.NewHandler(mcpServer, g, version, logger)
+	h := &Handler{Handler: base}
+	base.Mux().HandleFunc("POST /v1/augment", h.handleAugment)
+	return h
+}
+
+// --- /augment (eval-specific) ---
+
+type augmentRequest struct {
+	Pattern string `json:"pattern"`
+}
+
+type augmentResponse struct {
+	Pattern string          `json:"pattern"`
+	Symbols []augmentSymbol `json:"symbols"`
+}
+
+type augmentSymbol struct {
+	ID        string   `json:"id"`
+	Name      string   `json:"name"`
+	File      string   `json:"file"`
+	Kind      string   `json:"kind"`
+	Callers   []string `json:"callers,omitempty"`
+	Callees   []string `json:"callees,omitempty"`
+	CallChain []string `json:"call_chain,omitempty"`
+}
+
+func (h *Handler) handleAugment(w http.ResponseWriter, r *http.Request) {
+	body, err := io.ReadAll(io.LimitReader(r.Body, 1<<20))
+	if err != nil {
+		server.WriteJSONError(w, http.StatusBadRequest, "failed to read request body")
+		return
+	}
+
+	var req augmentRequest
+	if err := json.Unmarshal(body, &req); err != nil {
+		server.WriteJSONError(w, http.StatusBadRequest, "malformed JSON: "+err.Error())
+		return
+	}
+
+	if req.Pattern == "" {
+		server.WriteJSONError(w, http.StatusBadRequest, "missing 'pattern' field")
+		return
+	}
+
+	ctx := r.Context()
+
+	searchResults := h.CallTool(ctx, "search_symbols", map[string]any{
+		"query":   req.Pattern,
+		"compact": true,
+	})
+
+	symbolIDs := extractSymbolIDs(searchResults)
+
+	var symbols []augmentSymbol
+	for _, id := range symbolIDs {
+		sym := augmentSymbol{ID: id}
+
+		if parts := strings.SplitN(id, "::", 2); len(parts) == 2 {
+			sym.File = parts[0]
+			sym.Name = parts[1]
+		} else {
+			sym.Name = id
+		}
+
+		if node := h.Graph().GetNode(id); node != nil {
+			sym.Kind = string(node.Kind)
+		}
+
+		usageResults := h.CallTool(ctx, "find_usages", map[string]any{
+			"id":      id,
+			"compact": true,
+		})
+		sym.Callers = extractLines(usageResults)
+
+		chainResults := h.CallTool(ctx, "get_call_chain", map[string]any{
+			"function_id": id,
+			"compact":     true,
+			"depth":       float64(2),
+		})
+		sym.CallChain = extractLines(chainResults)
+
+		symbols = append(symbols, sym)
+	}
+
+	resp := augmentResponse{
+		Pattern: req.Pattern,
+		Symbols: symbols,
+	}
+	server.WriteJSON(w, http.StatusOK, resp)
+}
+
+// --- Helpers ---
+
+// extractSymbolIDs parses symbol IDs from compact search_symbols output.
+func extractSymbolIDs(text string) []string {
+	if text == "" {
+		return nil
+	}
+	var ids []string
+	seen := make(map[string]bool)
+	for _, line := range strings.Split(text, "\n") {
+		line = strings.TrimSpace(line)
+		if line == "" {
+			continue
+		}
+		parts := strings.Fields(line)
+		if len(parts) >= 3 {
+			name := parts[1]
+			fileLine := parts[2]
+			file := fileLine
+			if idx := strings.LastIndex(fileLine, ":"); idx > 0 {
+				file = fileLine[:idx]
+			}
+			id := file + "::" + name
+			if !seen[id] {
+				seen[id] = true
+				ids = append(ids, id)
+			}
+		}
+	}
+	return ids
+}
+
+// extractLines splits text into non-empty trimmed lines.
+func extractLines(text string) []string {
+	if text == "" {
+		return nil
+	}
+	var lines []string
+	for _, line := range strings.Split(text, "\n") {
+		line = strings.TrimSpace(line)
+		if line != "" {
+			lines = append(lines, line)
+		}
+	}
+	return lines
+}

internal/eval/http_handler_test.go

@@ -0,0 +1,198 @@
+package eval
+
+import (
+	"context"
+	"encoding/json"
+	"net/http"
+	"net/http/httptest"
+	"strings"
+	"testing"
+
+	"github.com/mark3labs/mcp-go/mcp"
+	mcpserver "github.com/mark3labs/mcp-go/server"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"github.com/zzet/gortex/internal/graph"
+	"github.com/zzet/gortex/internal/server"
+	"go.uber.org/zap"
+)
+
+func newTestHandler(t *testing.T) *Handler {
+	t.Helper()
+
+	g := graph.New()
+	g.AddNode(&graph.Node{
+		ID: "test.go::Foo", Kind: graph.KindFunction,
+		Name: "Foo", FilePath: "test.go", Language: "go",
+	})
+	g.AddNode(&graph.Node{
+		ID: "test.go::Bar", Kind: graph.KindFunction,
+		Name: "Bar", FilePath: "test.go", Language: "go",
+	})
+
+	srv := mcpserver.NewMCPServer("gortex-test", "0.0.1-test",
+		mcpserver.WithToolCapabilities(false),
+		mcpserver.WithRecovery(),
+	)
+	srv.AddTool(
+		mcp.NewTool("echo",
+			mcp.WithDescription("Echo tool for testing"),
+			mcp.WithString("message", mcp.Description("Message to echo")),
+		),
+		func(_ context.Context, req mcp.CallToolRequest) (*mcp.CallToolResult, error) {
+			args := req.GetArguments()
+			msg, _ := args["message"].(string)
+			if msg == "" {
+				msg = "no message"
+			}
+			return mcp.NewToolResultText(msg), nil
+		},
+	)
+
+	return NewHandler(srv, g, "0.0.1-test", zap.NewNop())
+}
+
+func TestHealthEndpoint(t *testing.T) {
+	h := newTestHandler(t)
+	req := httptest.NewRequest(http.MethodGet, "/v1/health", nil)
+	rec := httptest.NewRecorder()
+	h.ServeHTTP(rec, req)
+
+	assert.Equal(t, http.StatusOK, rec.Code)
+	assert.Equal(t, "application/json", rec.Header().Get("Content-Type"))
+
+	var resp server.HealthResponse
+	require.NoError(t, json.NewDecoder(rec.Body).Decode(&resp))
+	assert.Equal(t, "ok", resp.Status)
+	assert.True(t, resp.Indexed)
+	assert.Equal(t, 2, resp.Nodes)
+	assert.Equal(t, 0, resp.Edges)
+	assert.Equal(t, "0.0.1-test", resp.Version)
+	assert.Greater(t, resp.UptimeSeconds, float64(0))
+}
+
+func TestStatsEndpoint(t *testing.T) {
+	h := newTestHandler(t)
+	req := httptest.NewRequest(http.MethodGet, "/v1/stats", nil)
+	rec := httptest.NewRecorder()
+	h.ServeHTTP(rec, req)
+
+	assert.Equal(t, http.StatusOK, rec.Code)
+
+	var resp server.StatsResponse
+	require.NoError(t, json.NewDecoder(rec.Body).Decode(&resp))
+	assert.Equal(t, 2, resp.TotalNodes)
+	assert.Equal(t, 0, resp.TotalEdges)
+	assert.Equal(t, 2, resp.ByKind["function"])
+	assert.Equal(t, 2, resp.ByLanguage["go"])
+}
+
+func TestToolCallValid(t *testing.T) {
+	h := newTestHandler(t)
+	body := `{"arguments":{"message":"hello world"}}`
+	req := httptest.NewRequest(http.MethodPost, "/v1/tools/echo", strings.NewReader(body))
+	req.Header.Set("Content-Type", "application/json")
+	rec := httptest.NewRecorder()
+	h.ServeHTTP(rec, req)
+
+	assert.Equal(t, http.StatusOK, rec.Code)
+
+	var resp server.ToolResponse
+	require.NoError(t, json.NewDecoder(rec.Body).Decode(&resp))
+	assert.False(t, resp.IsError)
+	require.Len(t, resp.Content, 1)
+	assert.Equal(t, "text", resp.Content[0].Type)
+	assert.Equal(t, "hello world", resp.Content[0].Text)
+}
+
+func TestToolCallUnknownTool(t *testing.T) {
+	h := newTestHandler(t)
+	body := `{"arguments":{}}`
+	req := httptest.NewRequest(http.MethodPost, "/v1/tools/nonexistent", strings.NewReader(body))
+	req.Header.Set("Content-Type", "application/json")
+	rec := httptest.NewRecorder()
+	h.ServeHTTP(rec, req)
+
+	assert.Equal(t, http.StatusNotFound, rec.Code)
+
+	var resp map[string]any
+	require.NoError(t, json.NewDecoder(rec.Body).Decode(&resp))
+	assert.Equal(t, "tool_not_found", resp["error"])
+	assert.Contains(t, resp["message"], "nonexistent")
+	available, ok := resp["available_tools"].([]any)
+	require.True(t, ok)
+	assert.Contains(t, available, "echo")
+}
+
+func TestToolCallMalformedJSON(t *testing.T) {
+	h := newTestHandler(t)
+	req := httptest.NewRequest(http.MethodPost, "/v1/tools/echo", strings.NewReader("{invalid json"))
+	req.Header.Set("Content-Type", "application/json")
+	rec := httptest.NewRecorder()
+	h.ServeHTTP(rec, req)
+
+	assert.Equal(t, http.StatusBadRequest, rec.Code)
+
+	var resp map[string]string
+	require.NoError(t, json.NewDecoder(rec.Body).Decode(&resp))
+	assert.Contains(t, resp["message"], "malformed JSON")
+}
+
+func TestToolCallEmptyToolName(t *testing.T) {
+	h := newTestHandler(t)
+	req := httptest.NewRequest(http.MethodPost, "/v1/tools/", strings.NewReader("{}"))
+	req.Header.Set("Content-Type", "application/json")
+	rec := httptest.NewRecorder()
+	h.ServeHTTP(rec, req)
+
+	assert.Equal(t, http.StatusBadRequest, rec.Code)
+
+	var resp map[string]string
+	require.NoError(t, json.NewDecoder(rec.Body).Decode(&resp))
+	assert.Contains(t, resp["message"], "missing tool name")
+}
+
+func TestPanicRecovery(t *testing.T) {
+	g := graph.New()
+	srv := mcpserver.NewMCPServer("gortex-test", "0.0.1-test",
+		mcpserver.WithToolCapabilities(false),
+	)
+	srv.AddTool(
+		mcp.NewTool("panic_tool",
+			mcp.WithDescription("Tool that panics for testing"),
+		),
+		func(_ context.Context, _ mcp.CallToolRequest) (*mcp.CallToolResult, error) {
+			panic("test panic")
+		},
+	)
+
+	h := NewHandler(srv, g, "0.0.1-test", zap.NewNop())
+
+	req := httptest.NewRequest(http.MethodPost, "/v1/tools/panic_tool", strings.NewReader("{}"))
+	req.Header.Set("Content-Type", "application/json")
+	rec := httptest.NewRecorder()
+
+	assert.NotPanics(t, func() {
+		h.ServeHTTP(rec, req)
+	})
+
+	assert.Equal(t, http.StatusInternalServerError, rec.Code)
+
+	var resp map[string]string
+	require.NoError(t, json.NewDecoder(rec.Body).Decode(&resp))
+	assert.Contains(t, resp["message"], "internal server error")
+}
+
+func TestListToolsEndpoint(t *testing.T) {
+	h := newTestHandler(t)
+	req := httptest.NewRequest(http.MethodGet, "/v1/tools", nil)
+	rec := httptest.NewRecorder()
+	h.ServeHTTP(rec, req)
+
+	assert.Equal(t, http.StatusOK, rec.Code)
+
+	var tools []map[string]any
+	require.NoError(t, json.NewDecoder(rec.Body).Decode(&tools))
+	require.Len(t, tools, 1)
+	assert.Equal(t, "echo", tools[0]["name"])
+}

internal/eval/integration_test.go

@@ -0,0 +1,241 @@
+package eval
+
+import (
+	"context"
+	"encoding/json"
+	"net/http"
+	"net/http/httptest"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+
+	"github.com/mark3labs/mcp-go/mcp"
+	mcpserver "github.com/mark3labs/mcp-go/server"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"github.com/zzet/gortex/internal/graph"
+	"github.com/zzet/gortex/internal/server"
+	"go.uber.org/zap"
+)
+
+// TestEvalServerLifecycle is an integration test that exercises the full
+// eval-server HTTP lifecycle: start → health check → tool call → stats → shutdown.
+func TestEvalServerLifecycle(t *testing.T) {
+	// --- Setup: build a handler with a graph and a registered tool ---
+	g := graph.New()
+	g.AddNode(&graph.Node{
+		ID:       "main.go::Main",
+		Kind:     graph.KindFunction,
+		Name:     "Main",
+		FilePath: "main.go",
+		Language: "go",
+	})
+	g.AddNode(&graph.Node{
+		ID:       "main.go::Helper",
+		Kind:     graph.KindFunction,
+		Name:     "Helper",
+		FilePath: "main.go",
+		Language: "go",
+	})
+	g.AddEdge(&graph.Edge{
+		From: "main.go::Main",
+		To:   "main.go::Helper",
+		Kind: graph.EdgeCalls,
+	})
+
+	srv := mcpserver.NewMCPServer("gortex-integration", "0.1.0-test",
+		mcpserver.WithToolCapabilities(false),
+		mcpserver.WithRecovery(),
+	)
+	srv.AddTool(
+		mcp.NewTool("echo",
+			mcp.WithDescription("Echo tool for integration testing"),
+			mcp.WithString("message", mcp.Description("Message to echo")),
+		),
+		func(_ context.Context, req mcp.CallToolRequest) (*mcp.CallToolResult, error) {
+			args := req.GetArguments()
+			msg, _ := args["message"].(string)
+			if msg == "" {
+				msg = "empty"
+			}
+			return mcp.NewToolResultText("echo: " + msg), nil
+		},
+	)
+
+	logger := zap.NewNop()
+	handler := NewHandler(srv, g, "0.1.0-test", logger)
+
+	// --- Start: use httptest.NewServer for a real HTTP server ---
+	ts := httptest.NewServer(handler)
+	defer ts.Close()
+
+	client := ts.Client()
+
+	// --- Step 1: Health check ---
+	t.Run("health_check", func(t *testing.T) {
+		resp, err := client.Get(ts.URL + "/v1/health")
+		require.NoError(t, err)
+		defer resp.Body.Close()
+
+		assert.Equal(t, http.StatusOK, resp.StatusCode)
+		assert.Equal(t, "application/json", resp.Header.Get("Content-Type"))
+
+		var health server.HealthResponse
+		require.NoError(t, json.NewDecoder(resp.Body).Decode(&health))
+
+		assert.Equal(t, "ok", health.Status)
+		assert.True(t, health.Indexed, "graph has nodes so indexed should be true")
+		assert.Equal(t, 2, health.Nodes)
+		assert.Equal(t, 1, health.Edges)
+		assert.Equal(t, "0.1.0-test", health.Version)
+		assert.Greater(t, health.UptimeSeconds, float64(0))
+	})
+
+	// --- Step 2: Tool call (echo) ---
+	t.Run("tool_call_echo", func(t *testing.T) {
+		body := `{"arguments":{"message":"integration test"}}`
+		resp, err := client.Post(
+			ts.URL+"/v1/tools/echo",
+			"application/json",
+			strings.NewReader(body),
+		)
+		require.NoError(t, err)
+		defer resp.Body.Close()
+
+		assert.Equal(t, http.StatusOK, resp.StatusCode)
+
+		var toolResp server.ToolResponse
+		require.NoError(t, json.NewDecoder(resp.Body).Decode(&toolResp))
+
+		assert.False(t, toolResp.IsError)
+		require.Len(t, toolResp.Content, 1)
+		assert.Equal(t, "text", toolResp.Content[0].Type)
+		assert.Contains(t, toolResp.Content[0].Text, "integration test")
+	})
+
+	// --- Step 3: Stats endpoint ---
+	t.Run("stats", func(t *testing.T) {
+		resp, err := client.Get(ts.URL + "/v1/stats")
+		require.NoError(t, err)
+		defer resp.Body.Close()
+
+		assert.Equal(t, http.StatusOK, resp.StatusCode)
+
+		var stats server.StatsResponse
+		require.NoError(t, json.NewDecoder(resp.Body).Decode(&stats))
+
+		assert.Equal(t, 2, stats.TotalNodes)
+		assert.Equal(t, 1, stats.TotalEdges)
+		assert.NotNil(t, stats.ByKind)
+		assert.NotNil(t, stats.ByLanguage)
+	})
+
+	// --- Step 4: Unknown tool returns 404 ---
+	t.Run("unknown_tool_404", func(t *testing.T) {
+		body := `{"arguments":{}}`
+		resp, err := client.Post(
+			ts.URL+"/v1/tools/nonexistent",
+			"application/json",
+			strings.NewReader(body),
+		)
+		require.NoError(t, err)
+		defer resp.Body.Close()
+
+		assert.Equal(t, http.StatusNotFound, resp.StatusCode)
+	})
+
+	// --- Shutdown is implicit: ts.Close() in defer ---
+}
+
+// TestIndexCacheRoundTrip is an integration test that exercises the full
+// cache lifecycle: create files → store → load → verify integrity → evict.
+func TestIndexCacheRoundTrip(t *testing.T) {
+	cacheDir := t.TempDir()
+	version := "0.2.0-test"
+
+	cache, err := NewCache(cacheDir, version)
+	require.NoError(t, err)
+
+	// --- Step 1: Create a realistic index directory with multiple files ---
+	indexDir := t.TempDir()
+	require.NoError(t, os.WriteFile(
+		filepath.Join(indexDir, "graph.bin"),
+		[]byte("serialized-graph-data-with-nodes-and-edges"),
+		0o644,
+	))
+	// Create a nested directory simulating search.bleve/
+	bleveDir := filepath.Join(indexDir, "search.bleve")
+	require.NoError(t, os.MkdirAll(bleveDir, 0o755))
+	require.NoError(t, os.WriteFile(
+		filepath.Join(bleveDir, "store"),
+		[]byte("bleve-store-data"),
+		0o644,
+	))
+	require.NoError(t, os.WriteFile(
+		filepath.Join(bleveDir, "index_meta.json"),
+		[]byte(`{"version":1}`),
+		0o644,
+	))
+
+	repo := "django/django"
+	commit := "a1b2c3d4e5f6"
+
+	// --- Step 2: Verify cache is empty initially ---
+	assert.False(t, cache.Check(repo, commit), "cache should be empty initially")
+
+	// --- Step 3: Store the index ---
+	require.NoError(t, cache.Store(repo, commit, indexDir))
+
+	// --- Step 4: Verify cache entry exists ---
+	assert.True(t, cache.Check(repo, commit), "cache should have entry after store")
+
+	// --- Step 5: Load and verify path ---
+	loadedPath, err := cache.Load(repo, commit)
+	require.NoError(t, err)
+	expectedPath := filepath.Join(cacheDir, CacheKey(repo, commit))
+	assert.Equal(t, expectedPath, loadedPath)
+
+	// --- Step 6: Verify all files are intact ---
+	graphData, err := os.ReadFile(filepath.Join(loadedPath, "graph.bin"))
+	require.NoError(t, err)
+	assert.Equal(t, "serialized-graph-data-with-nodes-and-edges", string(graphData))
+
+	bleveStore, err := os.ReadFile(filepath.Join(loadedPath, "search.bleve", "store"))
+	require.NoError(t, err)
+	assert.Equal(t, "bleve-store-data", string(bleveStore))
+
+	bleveMeta, err := os.ReadFile(filepath.Join(loadedPath, "search.bleve", "index_meta.json"))
+	require.NoError(t, err)
+	assert.Equal(t, `{"version":1}`, string(bleveMeta))
+
+	// --- Step 7: Validate version compatibility ---
+	assert.True(t, cache.Validate(repo, commit), "version should match")
+
+	// --- Step 8: Version mismatch detection ---
+	cacheV2, err := NewCache(cacheDir, "0.3.0-different")
+	require.NoError(t, err)
+	assert.False(t, cacheV2.Validate(repo, commit), "different version should fail validation")
+
+	// --- Step 9: Re-store with updated content ---
+	indexDir2 := t.TempDir()
+	require.NoError(t, os.WriteFile(
+		filepath.Join(indexDir2, "graph.bin"),
+		[]byte("updated-graph-data"),
+		0o644,
+	))
+	require.NoError(t, cache.Store(repo, commit, indexDir2))
+
+	loadedPath2, err := cache.Load(repo, commit)
+	require.NoError(t, err)
+	updatedData, err := os.ReadFile(filepath.Join(loadedPath2, "graph.bin"))
+	require.NoError(t, err)
+	assert.Equal(t, "updated-graph-data", string(updatedData))
+
+	// --- Step 10: Evict and verify removal ---
+	require.NoError(t, cache.Evict(repo, commit))
+	assert.False(t, cache.Check(repo, commit), "cache should be empty after eviction")
+
+	_, err = cache.Load(repo, commit)
+	assert.Error(t, err, "load after eviction should fail")
+}

internal/eval/packeval/format_comprehension.go

@@ -0,0 +1,147 @@
+package packeval
+
+import (
+	"fmt"
+	"sort"
+	"strings"
+)
+
+// LLM-format-comprehension: a packed context is only useful if the model
+// can actually read it. The same symbols rendered as JSON, GCX1, TOON,
+// or Markdown cost different tokens and read differently; this measures
+// whether an LLM can still answer a grounded question from each format.
+// It is the comprehension half of the packing eval — density without
+// comprehension is a false economy.
+
+// Asker runs one prompt against an LLM and returns its raw answer. It is
+// injected so the harness stays provider-agnostic: the CLI wires it to
+// the configured llm.Service; tests pass a deterministic stub. A nil
+// Asker means "no provider" and RunFormatComprehension reports skipped.
+type Asker func(prompt string) (string, error)
+
+// FormatRenderer renders a packed context (a list of symbol entries)
+// into a wire format's textual form. Keyed by format name.
+type FormatRenderer func(entries []ContextEntry) string
+
+// ContextEntry is one symbol in a packed context used for the
+// comprehension probe: enough to render and to ground a question.
+type ContextEntry struct {
+	ID        string
+	Name      string
+	FilePath  string
+	Signature string
+	Source    string
+}
+
+// ComprehensionQuestion is a grounded Q/A whose answer is derivable from
+// the packed context. Correctness is a case-insensitive substring match
+// of any Accept string in the model's answer — robust to phrasing.
+type ComprehensionQuestion struct {
+	Question string
+	Accept   []string
+}
+
+// FormatResult is one format's comprehension score.
+type FormatResult struct {
+	Format    string  `json:"format"`
+	Asked     int     `json:"asked"`
+	Correct   int     `json:"correct"`
+	Accuracy  float64 `json:"accuracy"`
+	Tokens    int     `json:"tokens"`
+	Skipped   string  `json:"skipped,omitempty"`
+}
+
+// ComprehensionReport bundles per-format results.
+type ComprehensionReport struct {
+	Questions int            `json:"questions"`
+	Formats   []FormatResult `json:"formats"`
+}
+
+// RunFormatComprehension renders the packed context in each format and
+// asks the model every question, scoring comprehension per format. A nil
+// or failing asker yields a skipped result per format (never an error),
+// so the probe degrades cleanly when no LLM provider is configured.
+func RunFormatComprehension(
+	entries []ContextEntry,
+	questions []ComprehensionQuestion,
+	renderers map[string]FormatRenderer,
+	tokenCount func(string) int,
+	ask Asker,
+) ComprehensionReport {
+	rep := ComprehensionReport{Questions: len(questions)}
+	formats := make([]string, 0, len(renderers))
+	for f := range renderers {
+		formats = append(formats, f)
+	}
+	sort.Strings(formats)
+
+	for _, format := range formats {
+		rendered := renderers[format](entries)
+		fr := FormatResult{Format: format}
+		if tokenCount != nil {
+			fr.Tokens = tokenCount(rendered)
+		}
+		if ask == nil {
+			fr.Skipped = "no LLM provider"
+			rep.Formats = append(rep.Formats, fr)
+			continue
+		}
+		for _, q := range questions {
+			fr.Asked++
+			prompt := buildComprehensionPrompt(format, rendered, q.Question)
+			answer, err := ask(prompt)
+			if err != nil {
+				continue // a provider error counts as an unanswered (incorrect) probe
+			}
+			if answerAccepts(answer, q.Accept) {
+				fr.Correct++
+			}
+		}
+		if fr.Asked > 0 {
+			fr.Accuracy = float64(fr.Correct) / float64(fr.Asked)
+		}
+		rep.Formats = append(rep.Formats, fr)
+	}
+	return rep
+}
+
+func buildComprehensionPrompt(format, rendered, question string) string {
+	var b strings.Builder
+	fmt.Fprintf(&b, "You are given a code-context pack in %s format.\n", format)
+	b.WriteString("Answer the question using ONLY the pack. Reply with just the answer, no preamble.\n\n")
+	b.WriteString("=== CONTEXT PACK ===\n")
+	b.WriteString(rendered)
+	b.WriteString("\n=== END PACK ===\n\n")
+	fmt.Fprintf(&b, "Question: %s\nAnswer:", question)
+	return b.String()
+}
+
+func answerAccepts(answer string, accept []string) bool {
+	a := strings.ToLower(answer)
+	for _, want := range accept {
+		if want == "" {
+			continue
+		}
+		if strings.Contains(a, strings.ToLower(want)) {
+			return true
+		}
+	}
+	return false
+}
+
+// ComprehensionMarkdown renders the comprehension report.
+func ComprehensionMarkdown(rep ComprehensionReport) string {
+	var b strings.Builder
+	fmt.Fprintf(&b, "## Format comprehension (%d questions)\n\n", rep.Questions)
+	b.WriteString("| format | accuracy | correct/asked | tokens |\n")
+	b.WriteString("|--------|----------|---------------|--------|\n")
+	for _, f := range rep.Formats {
+		if f.Skipped != "" {
+			fmt.Fprintf(&b, "| %s | — | — | %d (skipped: %s) |\n", f.Format, f.Tokens, f.Skipped)
+			continue
+		}
+		fmt.Fprintf(&b, "| %s | %5.1f%% | %d/%d | %d |\n",
+			f.Format, f.Accuracy*100, f.Correct, f.Asked, f.Tokens)
+	}
+	return b.String()
+}

internal/eval/packeval/packeval.go

@@ -0,0 +1,239 @@
+// Package packeval is the held-out retrieval-precision harness for
+// context packing. It scores the real retrieval stack against curated
+// gold fixtures on Precision@K / Recall@K / MRR, and A/Bs the pluggable
+// pack strategies (top-k / density / file-grouped) under a fixed token
+// budget — the offline measurement Gortex previously lacked (it tuned
+// rerank weights only from online feedback telemetry).
+//
+// The harness is provider-driven: a RankedProvider returns the ranked,
+// token-costed candidate set for a query (wired to the live engine +
+// rerank pipeline by the `gortex eval pack` CLI). For each strategy the
+// harness packs the candidates into the budget, scores the delivered
+// top-K against the fixture gold, and aggregates overall and per-tier.
+package packeval
+
+import (
+	"fmt"
+	"sort"
+	"strings"
+
+	"github.com/zzet/gortex/internal/eval/recall"
+	"github.com/zzet/gortex/internal/search/packstrategy"
+)
+
+// RankedProvider returns the ranked candidate items for a query, best
+// first, each carrying its file and token cost so a strategy can pack
+// it. limit caps how many candidates to gather before packing.
+type RankedProvider func(query string, limit int) []packstrategy.Item
+
+// Metrics holds the standard retrieval-precision numbers for one slice
+// of cases.
+type Metrics struct {
+	Cases       int     `json:"cases"`
+	PrecisionAtK float64 `json:"precision_at_k"`
+	RecallAtK    float64 `json:"recall_at_k"`
+	MRR          float64 `json:"mrr"`
+}
+
+// StrategyResult aggregates a strategy's run over the whole fixture.
+type StrategyResult struct {
+	Strategy     string                 `json:"strategy"`
+	Overall      Metrics                `json:"overall"`
+	PerTier      map[recall.Tier]Metrics `json:"per_tier"`
+	MeanSelected float64                `json:"mean_selected"`   // avg symbols packed
+	MeanTokens   float64                `json:"mean_tokens"`     // avg tokens packed
+}
+
+// Report bundles the sweep.
+type Report struct {
+	Fixture     string           `json:"fixture"`
+	K           int              `json:"k"`
+	TokenBudget int              `json:"token_budget"`
+	FetchLimit  int              `json:"fetch_limit"`
+	Cases       int              `json:"cases"`
+	Strategies  []StrategyResult `json:"strategies"`
+}
+
+// Options configure a sweep.
+type Options struct {
+	Strategies  []packstrategy.Strategy // default: packstrategy.All()
+	K           int                     // precision/recall cutoff (default 10)
+	TokenBudget int                     // pack budget (default 8000)
+	FetchLimit  int                     // candidates gathered before packing (default 50)
+}
+
+func (o Options) withDefaults() Options {
+	if len(o.Strategies) == 0 {
+		o.Strategies = packstrategy.All()
+	}
+	if o.K <= 0 {
+		o.K = 10
+	}
+	if o.TokenBudget <= 0 {
+		o.TokenBudget = 8000
+	}
+	if o.FetchLimit <= 0 {
+		o.FetchLimit = 50
+	}
+	return o
+}
+
+// Run sweeps every strategy over the fixture and returns the report.
+// The provider is invoked once per case per — its result is reused
+// across strategies (packing is pure), so retrieval cost is paid once.
+func Run(fixture recall.Fixture, provider RankedProvider, opts Options) Report {
+	opts = opts.withDefaults()
+	rep := Report{
+		Fixture:     fixture.Name,
+		K:           opts.K,
+		TokenBudget: opts.TokenBudget,
+		FetchLimit:  opts.FetchLimit,
+		Cases:       len(fixture.Cases),
+	}
+
+	// Gather candidates once per case (retrieval is the expensive part;
+	// packing each strategy over the cached candidates is cheap).
+	cands := make([][]packstrategy.Item, len(fixture.Cases))
+	for i, c := range fixture.Cases {
+		cands[i] = provider(c.Query, opts.FetchLimit)
+	}
+
+	for _, strat := range opts.Strategies {
+		rep.Strategies = append(rep.Strategies, runStrategy(fixture, cands, strat, opts))
+	}
+	return rep
+}
+
+func runStrategy(fixture recall.Fixture, cands [][]packstrategy.Item, strat packstrategy.Strategy, opts Options) StrategyResult {
+	type acc struct {
+		cases             int
+		sumP, sumR, sumMR float64
+	}
+	overall := &acc{}
+	perTier := map[recall.Tier]*acc{}
+	var selectedSum, tokensSum float64
+
+	for i, c := range fixture.Cases {
+		selected := packstrategy.Select(strat, cands[i], opts.TokenBudget)
+		selectedSum += float64(len(selected))
+		toks := 0
+		for _, it := range selected {
+			toks += it.Tokens
+		}
+		tokensSum += float64(toks)
+
+		p, r, mrr := scoreCase(selected, c.Expected, opts.K)
+		overall.cases++
+		overall.sumP += p
+		overall.sumR += r
+		overall.sumMR += mrr
+
+		tier := c.Tier
+		if tier == "" {
+			tier = recall.TierExact
+		}
+		a := perTier[tier]
+		if a == nil {
+			a = &acc{}
+			perTier[tier] = a
+		}
+		a.cases++
+		a.sumP += p
+		a.sumR += r
+		a.sumMR += mrr
+	}
+
+	res := StrategyResult{
+		Strategy: string(packstrategy.Normalize(string(strat))),
+		Overall:  finalize(overall.cases, overall.sumP, overall.sumR, overall.sumMR),
+		PerTier:  make(map[recall.Tier]Metrics, len(perTier)),
+	}
+	for tier, a := range perTier {
+		res.PerTier[tier] = finalize(a.cases, a.sumP, a.sumR, a.sumMR)
+	}
+	if n := float64(len(fixture.Cases)); n > 0 {
+		res.MeanSelected = selectedSum / n
+		res.MeanTokens = tokensSum / n
+	}
+	return res
+}
+
+func finalize(cases int, sumP, sumR, sumMR float64) Metrics {
+	m := Metrics{Cases: cases}
+	if cases > 0 {
+		n := float64(cases)
+		m.PrecisionAtK = sumP / n
+		m.RecallAtK = sumR / n
+		m.MRR = sumMR / n
+	}
+	return m
+}
+
+// scoreCase computes Precision@K, Recall@K, and reciprocal rank for one
+// case against its gold expected set. A case is relevant-at-rank when a
+// delivered item's ID appears in the gold set.
+func scoreCase(selected []packstrategy.Item, expected []string, k int) (precision, recallV, mrr float64) {
+	gold := make(map[string]struct{}, len(expected))
+	for _, id := range expected {
+		gold[id] = struct{}{}
+	}
+	if len(gold) == 0 {
+		return 0, 0, 0
+	}
+	hits := 0
+	firstHit := 0
+	for i, it := range selected {
+		if i >= k {
+			break
+		}
+		if _, ok := gold[it.ID]; ok {
+			hits++
+			if firstHit == 0 {
+				firstHit = i + 1
+			}
+		}
+	}
+	precision = float64(hits) / float64(k)
+	recallV = float64(hits) / float64(len(gold))
+	if recallV > 1 {
+		recallV = 1
+	}
+	if firstHit > 0 {
+		mrr = 1.0 / float64(firstHit)
+	}
+	return precision, recallV, mrr
+}
+
+// Markdown renders the sweep as a diffable report.
+func Markdown(rep Report) string {
+	var b strings.Builder
+	fmt.Fprintf(&b, "# Pack-strategy retrieval eval\n\n")
+	fmt.Fprintf(&b, "_Fixture: `%s` · %d cases · K=%d · token_budget=%d · fetch_limit=%d_\n\n",
+		rep.Fixture, rep.Cases, rep.K, rep.TokenBudget, rep.FetchLimit)
+
+	b.WriteString("## Overall\n\n")
+	fmt.Fprintf(&b, "| strategy | P@%d | R@%d | MRR | mean symbols | mean tokens |\n", rep.K, rep.K)
+	b.WriteString("|----------|------|------|-----|--------------|-------------|\n")
+	strats := append([]StrategyResult(nil), rep.Strategies...)
+	sort.SliceStable(strats, func(i, j int) bool {
+		return strats[i].Overall.PrecisionAtK > strats[j].Overall.PrecisionAtK
+	})
+	for _, s := range strats {
+		fmt.Fprintf(&b, "| %s | %5.1f%% | %5.1f%% | %.3f | %.1f | %.0f |\n",
+			s.Strategy, s.Overall.PrecisionAtK*100, s.Overall.RecallAtK*100, s.Overall.MRR,
+			s.MeanSelected, s.MeanTokens)
+	}
+
+	b.WriteString("\n## Per tier (P@K)\n\n")
+	b.WriteString("| strategy | exact | concept | multi_hop |\n")
+	b.WriteString("|----------|-------|---------|-----------|\n")
+	for _, s := range strats {
+		fmt.Fprintf(&b, "| %s | %5.1f%% | %5.1f%% | %5.1f%% |\n",
+			s.Strategy,
+			s.PerTier[recall.TierExact].PrecisionAtK*100,
+			s.PerTier[recall.TierConcept].PrecisionAtK*100,
+			s.PerTier[recall.TierMultiHop].PrecisionAtK*100,
+		)
+	}
+	return b.String()
+}

internal/eval/packeval/packeval_test.go

@@ -0,0 +1,139 @@
+package packeval
+
+import (
+	"strings"
+	"testing"
+
+	"github.com/zzet/gortex/internal/eval/recall"
+	"github.com/zzet/gortex/internal/search/packstrategy"
+)
+
+// syntheticProvider returns a fixed ranked candidate set per query so
+// the harness logic (packing + scoring + sweep) is tested without
+// indexing a real repo.
+func syntheticProvider(items map[string][]packstrategy.Item) RankedProvider {
+	return func(query string, limit int) []packstrategy.Item {
+		got := items[query]
+		if limit > 0 && len(got) > limit {
+			got = got[:limit]
+		}
+		return got
+	}
+}
+
+func TestScoreCase(t *testing.T) {
+	selected := []packstrategy.Item{
+		{ID: "a"}, {ID: "gold1"}, {ID: "b"}, {ID: "gold2"},
+	}
+	p, r, mrr := scoreCase(selected, []string{"gold1", "gold2"}, 10)
+	if p != 2.0/10.0 {
+		t.Fatalf("P@10 = %v, want 0.2", p)
+	}
+	if r != 1.0 {
+		t.Fatalf("R@10 = %v, want 1.0 (both gold found)", r)
+	}
+	if mrr != 1.0/2.0 {
+		t.Fatalf("MRR = %v, want 0.5 (first gold at rank 2)", mrr)
+	}
+}
+
+func TestScoreCaseCutoff(t *testing.T) {
+	selected := []packstrategy.Item{{ID: "x"}, {ID: "y"}, {ID: "gold"}}
+	// gold is at rank 3 but K=2 — outside the cutoff.
+	p, r, mrr := scoreCase(selected, []string{"gold"}, 2)
+	if p != 0 || r != 0 || mrr != 0 {
+		t.Fatalf("gold outside K must score 0/0/0, got %v %v %v", p, r, mrr)
+	}
+}
+
+func TestRunSweepDistinguishesStrategies(t *testing.T) {
+	// One concept case: the gold symbol is token-cheap but ranked below
+	// a fat irrelevant one. Under a tight budget, density packs the
+	// lean gold; top-k spends the budget on the fat rank-1 miss.
+	q := "find the thing"
+	cands := map[string][]packstrategy.Item{
+		q: {
+			{ID: "fat-miss", FilePath: "a.go", Score: 10, Tokens: 800},
+			{ID: "lean-gold", FilePath: "b.go", Score: 6, Tokens: 50},
+			{ID: "lean-miss", FilePath: "b.go", Score: 5, Tokens: 50},
+		},
+	}
+	fixture := recall.Fixture{
+		Name: "synthetic",
+		Cases: []recall.Case{
+			{ID: "c1", Tier: recall.TierConcept, Query: q, Expected: []string{"lean-gold"}},
+		},
+	}
+	rep := Run(fixture, syntheticProvider(cands), Options{
+		Strategies:  []packstrategy.Strategy{packstrategy.StrategyTopK, packstrategy.StrategyDensity},
+		K:           10,
+		TokenBudget: 800, // fits the fat one alone, or both lean ones
+		FetchLimit:  50,
+	})
+
+	byName := map[string]StrategyResult{}
+	for _, s := range rep.Strategies {
+		byName[s.Strategy] = s
+	}
+	topk := byName[string(packstrategy.StrategyTopK)]
+	density := byName[string(packstrategy.StrategyDensity)]
+
+	// top-k spends 800 on fat-miss (rank 1), then can't fit the lean ones
+	// -> gold missed -> P@10 = 0.
+	if topk.Overall.PrecisionAtK != 0 {
+		t.Fatalf("top-k should miss the gold under a tight budget, P@10=%v", topk.Overall.PrecisionAtK)
+	}
+	// density orders lean-gold (0.12) and lean-miss (0.10) above fat (0.0125),
+	// packs both lean -> gold hit -> P@10 > 0.
+	if density.Overall.PrecisionAtK <= 0 {
+		t.Fatalf("density should pack the lean gold, P@10=%v", density.Overall.PrecisionAtK)
+	}
+}
+
+func TestMarkdownRenders(t *testing.T) {
+	rep := Report{
+		Fixture: "x", K: 10, TokenBudget: 8000, Cases: 1,
+		Strategies: []StrategyResult{
+			{Strategy: "density", Overall: Metrics{PrecisionAtK: 0.5, RecallAtK: 0.8, MRR: 0.6}, PerTier: map[recall.Tier]Metrics{}},
+		},
+	}
+	md := Markdown(rep)
+	if !strings.Contains(md, "Pack-strategy retrieval eval") || !strings.Contains(md, "density") {
+		t.Fatalf("markdown missing content:\n%s", md)
+	}
+}
+
+func TestFormatComprehensionStubAsker(t *testing.T) {
+	entries := []ContextEntry{{ID: "f.go::Foo", Name: "Foo", Signature: "func Foo() error"}}
+	questions := []ComprehensionQuestion{
+		{Question: "What does Foo return?", Accept: []string{"error"}},
+	}
+	renderers := map[string]FormatRenderer{
+		"plain": func(es []ContextEntry) string {
+			var b strings.Builder
+			for _, e := range es {
+				b.WriteString(e.Name + " " + e.Signature + "\n")
+			}
+			return b.String()
+		},
+	}
+	// Stub asker that "reads" the prompt and answers from it.
+	ask := func(prompt string) (string, error) {
+		if strings.Contains(prompt, "func Foo() error") {
+			return "It returns an error.", nil
+		}
+		return "unknown", nil
+	}
+	rep := RunFormatComprehension(entries, questions, renderers, func(s string) int { return len(s) / 4 }, ask)
+	if len(rep.Formats) != 1 || rep.Formats[0].Correct != 1 {
+		t.Fatalf("expected 1 correct, got %+v", rep.Formats)
+	}
+}
+
+func TestFormatComprehensionNilAskerSkips(t *testing.T) {
+	renderers := map[string]FormatRenderer{"plain": func(es []ContextEntry) string { return "" }}
+	rep := RunFormatComprehension(nil, nil, renderers, nil, nil)
+	if len(rep.Formats) != 1 || rep.Formats[0].Skipped == "" {
+		t.Fatalf("nil asker should skip, got %+v", rep.Formats)
+	}
+}

internal/eval/parity/baseline.json

@@ -0,0 +1,19 @@
+{
+  "c": 0.4504881450488145,
+  "cpp": 0.5294117647058824,
+  "csharp": 0.4161073825503356,
+  "dart": 0.44890620317650587,
+  "go": 0.8070175438596491,
+  "java": 0.3805668016194332,
+  "kotlin": 0.36312849162011174,
+  "lua": 0.2602739726027397,
+  "php": 0.2682926829268293,
+  "python": 0.6666666666666666,
+  "ruby": 0.3188405797101449,
+  "rust": 0.7126436781609196,
+  "scala": 0.3243606998654105,
+  "svelte": 0.3333333333333333,
+  "swift": 0.22988505747126436,
+  "typescript": 0.1196319018404908,
+  "vue": 0.047619047619047616
+}

internal/eval/parity/baselines.go

@@ -0,0 +1,65 @@
+package parity
+
+import (
+	_ "embed"
+	"encoding/json"
+	"sort"
+)
+
+//go:embed baseline.json
+var baselineJSON []byte
+
+// Baseline maps a language to its committed minimum resolved-cross-file-dependent
+// coverage. The parity run refuses to let any language regress below its
+// baseline, which is how COVERED stays COVERED over time.
+type Baseline map[string]float64
+
+// LoadBaseline returns the committed baseline. An empty baseline ("{}") means
+// none has been captured yet — informational, not a failure.
+func LoadBaseline() (Baseline, error) {
+	b := Baseline{}
+	if len(baselineJSON) == 0 {
+		return b, nil
+	}
+	if err := json.Unmarshal(baselineJSON, &b); err != nil {
+		return nil, err
+	}
+	return b, nil
+}
+
+// Regression records a language whose measured coverage fell below its baseline.
+type Regression struct {
+	Language string
+	Baseline float64
+	Measured float64
+}
+
+// Check returns the languages whose measured coverage is below their baseline by
+// more than epsilon. A language carried in the baseline but absent from the
+// measured set is treated as 0 (it dropped out entirely). Languages with no
+// baseline are not enforced.
+func (b Baseline) Check(measured []LanguageCoverage, epsilon float64) []Regression {
+	got := make(map[string]float64, len(measured))
+	for _, m := range measured {
+		got[m.Language] = m.Coverage
+	}
+	var regs []Regression
+	for lang, base := range b {
+		m := got[lang] // absent → 0
+		if m < base-epsilon {
+			regs = append(regs, Regression{Language: lang, Baseline: base, Measured: m})
+		}
+	}
+	sort.Slice(regs, func(i, j int) bool { return regs[i].Language < regs[j].Language })
+	return regs
+}
+
+// MarshalBaseline renders a measured coverage set as a baseline JSON document
+// (stable, indented) suitable for committing to baseline.json.
+func MarshalBaseline(measured []LanguageCoverage) ([]byte, error) {
+	b := Baseline{}
+	for _, m := range measured {
+		b[m.Language] = m.Coverage
+	}
+	return json.MarshalIndent(b, "", "  ")
+}

internal/eval/parity/coverage.go

@@ -0,0 +1,128 @@
+// Package parity computes cross-file dependency-graph coverage metrics used to
+// measure — and lock in — the quality of Gortex's resolved graph per language.
+package parity
+
+import (
+	"sort"
+
+	"github.com/zzet/gortex/internal/graph"
+)
+
+// LanguageCoverage is the per-language share of symbol-bearing source files that
+// have at least one resolved cross-file dependent — something in another file
+// that imports, calls, references, routes to, or otherwise depends on a symbol
+// defined in the file.
+type LanguageCoverage struct {
+	Language     string  `json:"language"`
+	SymbolFiles  int     `json:"symbol_files"`
+	CoveredFiles int     `json:"covered_files"`
+	Coverage     float64 `json:"coverage"`
+}
+
+// coverageSymbolKinds are the node kinds whose presence makes a file a
+// symbol-bearing source file (the denominator of the coverage metric).
+var coverageSymbolKinds = map[graph.NodeKind]bool{
+	graph.KindFunction:   true,
+	graph.KindMethod:     true,
+	graph.KindType:       true,
+	graph.KindInterface:  true,
+	graph.KindVariable:   true,
+	graph.KindConstant:   true,
+	graph.KindField:      true,
+	graph.KindEnumMember: true,
+}
+
+// dependencyEdgeKinds are the edge kinds that count as a cross-file dependent: a
+// resolved edge of one of these kinds crossing a file boundary marks its target
+// file as covered.
+var dependencyEdgeKinds = map[graph.EdgeKind]bool{
+	graph.EdgeImports:      true,
+	graph.EdgeCalls:        true,
+	graph.EdgeReferences:   true,
+	graph.EdgeExtends:      true,
+	graph.EdgeImplements:   true,
+	graph.EdgeInstantiates: true,
+	graph.EdgeTypedAs:      true,
+	graph.EdgeReturns:      true,
+	graph.EdgeOverrides:    true,
+	graph.EdgeHandlesRoute: true,
+}
+
+// CoverageOf computes per-language resolved-cross-file-dependent coverage over
+// the whole graph. Languages are returned sorted by name; a language with no
+// symbol-bearing source file is omitted.
+func CoverageOf(g graph.Store) []LanguageCoverage {
+	nodes := make(map[string]*graph.Node)
+	fileLang := make(map[string]string)
+	symbolFiles := make(map[string]bool)
+
+	for _, n := range g.AllNodes() {
+		if n == nil {
+			continue
+		}
+		nodes[n.ID] = n
+		switch {
+		case n.Kind == graph.KindFile:
+			if path := nodeFile(n); path != "" && n.Language != "" {
+				fileLang[path] = n.Language
+			}
+		case coverageSymbolKinds[n.Kind] && n.FilePath != "":
+			symbolFiles[n.FilePath] = true
+			if fileLang[n.FilePath] == "" && n.Language != "" {
+				fileLang[n.FilePath] = n.Language
+			}
+		}
+	}
+
+	covered := make(map[string]bool)
+	for _, e := range g.AllEdges() {
+		if e == nil || !dependencyEdgeKinds[e.Kind] || graph.IsUnresolvedTarget(e.To) {
+			continue
+		}
+		src, dst := nodes[e.From], nodes[e.To]
+		if src == nil || dst == nil {
+			continue
+		}
+		srcFile, dstFile := nodeFile(src), nodeFile(dst)
+		if srcFile == "" || dstFile == "" || srcFile == dstFile {
+			continue
+		}
+		covered[dstFile] = true
+	}
+
+	type acc struct{ sym, cov int }
+	byLang := make(map[string]*acc)
+	for f := range symbolFiles {
+		a := byLang[fileLang[f]]
+		if a == nil {
+			a = &acc{}
+			byLang[fileLang[f]] = a
+		}
+		a.sym++
+		if covered[f] {
+			a.cov++
+		}
+	}
+
+	out := make([]LanguageCoverage, 0, len(byLang))
+	for lang, a := range byLang {
+		cov := 0.0
+		if a.sym > 0 {
+			cov = float64(a.cov) / float64(a.sym)
+		}
+		out = append(out, LanguageCoverage{Language: lang, SymbolFiles: a.sym, CoveredFiles: a.cov, Coverage: cov})
+	}
+	sort.Slice(out, func(i, j int) bool { return out[i].Language < out[j].Language })
+	return out
+}
+
+// nodeFile returns the source-file path a node belongs to.
+func nodeFile(n *graph.Node) string {
+	if n.FilePath != "" {
+		return n.FilePath
+	}
+	if n.Kind == graph.KindFile {
+		return n.ID
+	}
+	return ""
+}

internal/eval/parity/coverage_test.go

@@ -0,0 +1,67 @@
+package parity
+
+import (
+	"math"
+	"testing"
+
+	"github.com/zzet/gortex/internal/graph"
+)
+
+func TestCoverageOf(t *testing.T) {
+	g := graph.New()
+
+	fn := func(id, file, lang string) {
+		g.AddNode(&graph.Node{ID: id, Name: id, Kind: graph.KindFunction, FilePath: file, Language: lang})
+	}
+	file := func(path, lang string) {
+		g.AddNode(&graph.Node{ID: path, Name: path, Kind: graph.KindFile, FilePath: path, Language: lang})
+	}
+	call := func(from, to string) {
+		g.AddEdge(&graph.Edge{From: from, To: to, Kind: graph.EdgeCalls})
+	}
+
+	// Three Go source files, each with one function.
+	file("a.go", "go")
+	file("b.go", "go")
+	file("c.go", "go")
+	fn("a.go::Foo", "a.go", "go")
+	fn("b.go::Bar", "b.go", "go")
+	fn("c.go::Baz", "c.go", "go")
+
+	// b depends on a, c depends on b → a and b are covered, c is not.
+	call("b.go::Bar", "a.go::Foo")
+	call("c.go::Baz", "b.go::Bar")
+	// Same-file call must not count toward coverage.
+	call("a.go::Foo", "a.go::Foo")
+	// An unresolved target must not cover anything.
+	call("a.go::Foo", "unresolved::Qux")
+
+	// A Python file nothing depends on → 0% covered.
+	file("p.py", "python")
+	fn("p.py::pyf", "p.py", "python")
+
+	cov := CoverageOf(g)
+	byLang := map[string]LanguageCoverage{}
+	for _, c := range cov {
+		byLang[c.Language] = c
+	}
+
+	got, ok := byLang["go"]
+	if !ok {
+		t.Fatalf("no go coverage row in %+v", cov)
+	}
+	if got.SymbolFiles != 3 || got.CoveredFiles != 2 {
+		t.Errorf("go: SymbolFiles=%d CoveredFiles=%d, want 3 and 2", got.SymbolFiles, got.CoveredFiles)
+	}
+	if math.Abs(got.Coverage-2.0/3.0) > 1e-9 {
+		t.Errorf("go: Coverage=%v, want %v", got.Coverage, 2.0/3.0)
+	}
+
+	py, ok := byLang["python"]
+	if !ok {
+		t.Fatalf("no python coverage row in %+v", cov)
+	}
+	if py.SymbolFiles != 1 || py.CoveredFiles != 0 || py.Coverage != 0 {
+		t.Errorf("python: %+v, want SymbolFiles=1 CoveredFiles=0 Coverage=0", py)
+	}
+}

internal/eval/parity/fence_test.go

@@ -0,0 +1,63 @@
+package parity
+
+import "testing"
+
+// parityFenceCount is the frozen number of languages held at or beyond their
+// captured parity coverage in baseline.json. It is a CI-enforced golden, the
+// same discipline as a wire-contract fingerprint: a DROP means a language was
+// silently removed from the fence (un-protecting it from regression) and a RISE
+// means a new benchmark language was fenced — either way an intentional change
+// must bump this constant deliberately, which is the audit trail.
+const parityFenceCount = 17
+
+// TestParityCount freezes the at-or-beyond-parity language count. The committed
+// baseline is the set of languages whose coverage CI refuses to let regress;
+// pinning its size here makes any change to that set a deliberate, reviewed act
+// rather than a silent erosion of the fence.
+func TestParityCount(t *testing.T) {
+	b, err := LoadBaseline()
+	if err != nil {
+		t.Fatalf("load baseline: %v", err)
+	}
+	if got := len(b); got != parityFenceCount {
+		t.Fatalf("parity fence count = %d, want %d (frozen golden).\n"+
+			"If a benchmark language was intentionally added or removed, update parityFenceCount.",
+			got, parityFenceCount)
+	}
+	// Every fenced language must carry a real coverage floor in (0, 1]. A zero
+	// or negative floor fences nothing; a floor above 1 is not a valid ratio.
+	for lang, cov := range b {
+		if cov <= 0 {
+			t.Errorf("language %q baseline %.4f is non-positive — it fences nothing", lang, cov)
+		}
+		if cov > 1 {
+			t.Errorf("language %q baseline %.4f exceeds 1.0 (coverage is a ratio)", lang, cov)
+		}
+	}
+}
+
+// TestBaselineRepoExhaustive ensures the fence is exactly the benchmark corpus:
+// every benchmark language has a baseline floor (none is measured but unfenced),
+// and no baseline entry lacks a benchmark repo (no dead fence that is never
+// measured). Together with TestParityCount this keeps the fence complete and
+// honest as the corpus evolves.
+func TestBaselineRepoExhaustive(t *testing.T) {
+	b, err := LoadBaseline()
+	if err != nil {
+		t.Fatalf("load baseline: %v", err)
+	}
+
+	repoLangs := map[string]bool{}
+	for _, repo := range BenchRepos() {
+		repoLangs[repo.Language] = true
+		if _, ok := b[repo.Language]; !ok {
+			t.Errorf("benchmark language %q (%s) has no baseline entry — unfenced, could regress undetected",
+				repo.Language, repo.URL)
+		}
+	}
+	for lang := range b {
+		if !repoLangs[lang] {
+			t.Errorf("baseline carries %q but no benchmark repo measures it — dead fence entry", lang)
+		}
+	}
+}

internal/eval/parity/golden/capabilities.go

@@ -0,0 +1,111 @@
+package golden
+
+import "github.com/zzet/gortex/internal/graph"
+
+// capabilities is the registry of ported extraction features under golden
+// regression. Each entry is self-contained — a snippet plus the nodes/edges the
+// feature must produce — so a failure points straight at the capability that
+// regressed. Extraction-layer only: every assertion is satisfiable from the raw
+// extractor output, no resolver or daemon required.
+var capabilities = []Capability{
+	{
+		Name:     "java/annotation-type-as-interface",
+		Language: "java",
+		FileName: "Audited.java",
+		Source: `package com.app;
+public @interface Audited {
+    String value() default "";
+}
+`,
+		WantNodes: []nodeWant{
+			{Kind: graph.KindInterface, Name: "Audited"},
+		},
+	},
+	{
+		Name:     "javascript/arrow-class-field-as-method",
+		Language: "javascript",
+		FileName: "comp.js",
+		Source: `class Counter {
+  inc = () => { this.n++; };
+}
+`,
+		WantNodes: []nodeWant{
+			{Kind: graph.KindMethod, Name: "inc"},
+		},
+	},
+	{
+		Name:     "java/anonymous-class-synthetic-type",
+		Language: "java",
+		FileName: "Host.java",
+		Source: `package com.app;
+class Host {
+    void wire() {
+        Runnable r = new Runnable() {
+            public void run() {}
+        };
+    }
+}
+`,
+		WantNodes: []nodeWant{
+			{Kind: graph.KindType, Name: "Runnable$anon@4", MetaKey: "anonymous", MetaVal: true},
+		},
+		WantEdges: []edgeWant{
+			{Kind: graph.EdgeExtends, ToSub: "unresolved::Runnable"},
+		},
+	},
+	{
+		Name:     "csharp/anonymous-type-synthetic-type",
+		Language: "csharp",
+		FileName: "Host.cs",
+		Source: `namespace App;
+class Host {
+    void Wire() {
+        var p = new { Name = "x", Age = 5 };
+    }
+}
+`,
+		WantNodes: []nodeWant{
+			{Kind: graph.KindType, Name: "anon@4", MetaKey: "anonymous", MetaVal: true},
+		},
+		WantEdges: []edgeWant{
+			{Kind: graph.EdgeExtends, ToSub: "unresolved::object"},
+		},
+	},
+	{
+		Name:     "typescript/per-binding-import",
+		Language: "typescript",
+		FileName: "app.ts",
+		Source: `import { Router, json as parseJson } from "express";
+`,
+		WantEdges: []edgeWant{
+			{Kind: graph.EdgeImports, ToSub: "unresolved::import::express::Router"},
+			{Kind: graph.EdgeImports, ToSub: "unresolved::import::express::json", Alias: "parseJson"},
+		},
+	},
+	{
+		Name:     "typescript/alias-aware-re-export",
+		Language: "typescript",
+		FileName: "barrel.ts",
+		Source: `export { a, b as c } from "up";
+export * as ns from "nsmod";
+`,
+		WantEdges: []edgeWant{
+			{Kind: graph.EdgeReExports, ToSub: "unresolved::import::up::a"},
+			{Kind: graph.EdgeReExports, ToSub: "unresolved::import::up::b", Alias: "c"},
+			{Kind: graph.EdgeReExports, ToSub: "unresolved::import::nsmod", Alias: "ns"},
+		},
+	},
+	{
+		Name:     "javascript/per-binding-import-and-re-export",
+		Language: "javascript",
+		FileName: "barrel.js",
+		Source: `import { foo, bar as baz } from "mod";
+export { x as y } from "down";
+`,
+		WantEdges: []edgeWant{
+			{Kind: graph.EdgeImports, ToSub: "unresolved::import::mod::foo"},
+			{Kind: graph.EdgeImports, ToSub: "unresolved::import::mod::bar", Alias: "baz"},
+			{Kind: graph.EdgeReExports, ToSub: "unresolved::import::down::x", Alias: "y"},
+		},
+	},
+}

internal/eval/parity/golden/golden.go

@@ -0,0 +1,147 @@
+// Package golden locks in the per-feature extraction output of the
+// capabilities ported to match — and exceed — the reference benchmark suite.
+//
+// Each Capability feeds a fixed source snippet to the matching language
+// extractor and asserts the nodes and edges the feature must produce. Unlike
+// the per-language coverage metric (which measures breadth), these are narrow
+// regression fences: if a future refactor silently drops a ported capability —
+// stops emitting an anonymous-class type, a per-binding import edge, an
+// annotation interface — the corresponding golden fails immediately, naming
+// the exact node or edge that went missing.
+package golden
+
+import (
+	"fmt"
+	"strings"
+
+	"github.com/zzet/gortex/internal/graph"
+	"github.com/zzet/gortex/internal/parser"
+	"github.com/zzet/gortex/internal/parser/languages"
+)
+
+// extractorFor returns the extractor for a capability's language, or nil for
+// an unknown language.
+func extractorFor(lang string) parser.Extractor {
+	switch lang {
+	case "java":
+		return languages.NewJavaExtractor()
+	case "csharp":
+		return languages.NewCSharpExtractor()
+	case "typescript":
+		return languages.NewTypeScriptExtractor()
+	case "javascript":
+		return languages.NewJavaScriptExtractor()
+	case "go":
+		return languages.NewGoExtractor()
+	default:
+		return nil
+	}
+}
+
+// nodeWant asserts that a node of the given Kind and Name was extracted. A
+// non-empty MetaKey additionally requires Meta[MetaKey] to equal MetaVal.
+type nodeWant struct {
+	Kind    graph.NodeKind
+	Name    string
+	MetaKey string
+	MetaVal any
+}
+
+func (w nodeWant) String() string {
+	if w.MetaKey != "" {
+		return fmt.Sprintf("node{kind=%s name=%q meta[%s]=%v}", w.Kind, w.Name, w.MetaKey, w.MetaVal)
+	}
+	return fmt.Sprintf("node{kind=%s name=%q}", w.Kind, w.Name)
+}
+
+func (w nodeWant) matches(n *graph.Node) bool {
+	if n.Kind != w.Kind || n.Name != w.Name {
+		return false
+	}
+	if w.MetaKey == "" {
+		return true
+	}
+	if n.Meta == nil {
+		return false
+	}
+	return n.Meta[w.MetaKey] == w.MetaVal
+}
+
+// edgeWant asserts that an edge of the given Kind exists whose To target
+// contains ToSub (To is frequently an unresolved::… path, so a substring match
+// keeps the golden robust to id-format details). A non-empty Alias additionally
+// requires the edge's Alias to match.
+type edgeWant struct {
+	Kind  graph.EdgeKind
+	ToSub string
+	Alias string
+}
+
+func (w edgeWant) String() string {
+	if w.Alias != "" {
+		return fmt.Sprintf("edge{kind=%s to~%q alias=%q}", w.Kind, w.ToSub, w.Alias)
+	}
+	return fmt.Sprintf("edge{kind=%s to~%q}", w.Kind, w.ToSub)
+}
+
+func (w edgeWant) matches(e *graph.Edge) bool {
+	if e.Kind != w.Kind || !strings.Contains(e.To, w.ToSub) {
+		return false
+	}
+	if w.Alias != "" && e.Alias != w.Alias {
+		return false
+	}
+	return true
+}
+
+// Capability is one ported extraction feature with its golden assertions.
+type Capability struct {
+	Name      string
+	Language  string
+	FileName  string
+	Source    string
+	WantNodes []nodeWant
+	WantEdges []edgeWant
+}
+
+// check extracts the snippet and returns the wanted nodes / edges that were not
+// produced. A nil error with empty slices means the capability holds.
+func (c Capability) check() (missingNodes []nodeWant, missingEdges []edgeWant, err error) {
+	ext := extractorFor(c.Language)
+	if ext == nil {
+		return nil, nil, fmt.Errorf("no extractor for language %q", c.Language)
+	}
+	res, err := ext.Extract(c.FileName, []byte(c.Source))
+	if err != nil {
+		return nil, nil, err
+	}
+	for _, wn := range c.WantNodes {
+		if !anyNode(res.Nodes, wn.matches) {
+			missingNodes = append(missingNodes, wn)
+		}
+	}
+	for _, we := range c.WantEdges {
+		if !anyEdge(res.Edges, we.matches) {
+			missingEdges = append(missingEdges, we)
+		}
+	}
+	return missingNodes, missingEdges, nil
+}
+
+func anyNode(nodes []*graph.Node, pred func(*graph.Node) bool) bool {
+	for _, n := range nodes {
+		if n != nil && pred(n) {
+			return true
+		}
+	}
+	return false
+}
+
+func anyEdge(edges []*graph.Edge, pred func(*graph.Edge) bool) bool {
+	for _, e := range edges {
+		if e != nil && pred(e) {
+			return true
+		}
+	}
+	return false
+}

internal/eval/parity/golden/golden_test.go

@@ -0,0 +1,32 @@
+package golden
+
+import "testing"
+
+// TestPortedExtractionCapabilities runs every registered ported-capability
+// golden. Each sub-test names the capability so a regression report reads like
+// a feature checklist.
+func TestPortedExtractionCapabilities(t *testing.T) {
+	if len(capabilities) == 0 {
+		t.Fatal("no ported capabilities registered — the golden fence would pass vacuously")
+	}
+	seen := map[string]bool{}
+	for _, c := range capabilities {
+		c := c
+		if seen[c.Name] {
+			t.Errorf("duplicate capability name %q", c.Name)
+		}
+		seen[c.Name] = true
+		t.Run(c.Name, func(t *testing.T) {
+			missingNodes, missingEdges, err := c.check()
+			if err != nil {
+				t.Fatalf("extract failed: %v", err)
+			}
+			for _, n := range missingNodes {
+				t.Errorf("capability regressed — missing %s", n)
+			}
+			for _, e := range missingEdges {
+				t.Errorf("capability regressed — missing %s", e)
+			}
+		})
+	}
+}

internal/eval/parity/repos.go

@@ -0,0 +1,86 @@
+package parity
+
+import (
+	"fmt"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"strings"
+)
+
+// BenchRepo is one per-language benchmark repository the parity run indexes and
+// measures coverage over.
+type BenchRepo struct {
+	Language string
+	URL      string
+	// Ref optionally pins a branch/tag for reproducibility (empty = default).
+	Ref string
+}
+
+// benchRepos is the frozen per-language corpus used for parity coverage. The
+// list is the canonical, license-clean public repository per language; it is
+// cloned and cached on demand, never vendored.
+var benchRepos = []BenchRepo{
+	{Language: "python", URL: "https://github.com/psf/requests"},
+	{Language: "go", URL: "https://github.com/gin-gonic/gin"},
+	{Language: "rust", URL: "https://github.com/BurntSushi/ripgrep"},
+	{Language: "java", URL: "https://github.com/google/gson"},
+	{Language: "csharp", URL: "https://github.com/jbogard/MediatR"},
+	{Language: "php", URL: "https://github.com/guzzle/guzzle"},
+	{Language: "ruby", URL: "https://github.com/sidekiq/sidekiq"},
+	{Language: "c", URL: "https://github.com/redis/redis"},
+	{Language: "cpp", URL: "https://github.com/google/leveldb"},
+	{Language: "swift", URL: "https://github.com/Alamofire/Alamofire"},
+	{Language: "kotlin", URL: "https://github.com/square/okhttp"},
+	{Language: "scala", URL: "https://github.com/gatling/gatling"},
+	{Language: "dart", URL: "https://github.com/flutter/packages"},
+	{Language: "lua", URL: "https://github.com/nvim-telescope/telescope.nvim"},
+	{Language: "typescript", URL: "https://github.com/colinhacks/zod"},
+	{Language: "svelte", URL: "https://github.com/sveltejs/realworld"},
+	{Language: "vue", URL: "https://github.com/nuxt/movies"},
+}
+
+// BenchRepos returns the frozen benchmark corpus.
+func BenchRepos() []BenchRepo {
+	out := make([]BenchRepo, len(benchRepos))
+	copy(out, benchRepos)
+	return out
+}
+
+// EnsureRepo returns the local checkout path for repo, shallow-cloning it into
+// cacheDir on a cache miss. A repo already checked out under its derived
+// directory name is reused without re-cloning.
+func EnsureRepo(cacheDir string, repo BenchRepo) (string, error) {
+	dir := filepath.Join(cacheDir, repoDirName(repo))
+	if isGitCheckout(dir) {
+		return dir, nil
+	}
+	if err := os.MkdirAll(cacheDir, 0o755); err != nil {
+		return "", err
+	}
+	args := []string{"clone", "--depth", "1", "--quiet"}
+	if repo.Ref != "" {
+		args = append(args, "--branch", repo.Ref)
+	}
+	args = append(args, repo.URL, dir)
+	if out, err := exec.Command("git", args...).CombinedOutput(); err != nil {
+		return "", fmt.Errorf("git clone %s: %w: %s", repo.URL, err, strings.TrimSpace(string(out)))
+	}
+	return dir, nil
+}
+
+// repoDirName derives a filesystem-safe cache directory name from a repo URL,
+// e.g. https://github.com/psf/requests -> psf__requests.
+func repoDirName(repo BenchRepo) string {
+	u := strings.TrimSuffix(repo.URL, ".git")
+	u = strings.TrimPrefix(u, "https://github.com/")
+	u = strings.TrimPrefix(u, "http://github.com/")
+	u = strings.Trim(u, "/")
+	return strings.ReplaceAll(u, "/", "__")
+}
+
+// isGitCheckout reports whether dir is an existing git checkout.
+func isGitCheckout(dir string) bool {
+	info, err := os.Stat(filepath.Join(dir, ".git"))
+	return err == nil && info.IsDir()
+}

internal/eval/parity/repos_test.go

@@ -0,0 +1,54 @@
+package parity
+
+import (
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+)
+
+func TestBenchRepos(t *testing.T) {
+	repos := BenchRepos()
+	if len(repos) == 0 {
+		t.Fatal("benchmark corpus is empty")
+	}
+	seen := map[string]bool{}
+	for _, r := range repos {
+		if r.Language == "" || r.URL == "" {
+			t.Errorf("malformed bench repo: %+v", r)
+		}
+		if !strings.HasPrefix(r.URL, "https://") {
+			t.Errorf("bench repo URL is not https: %q", r.URL)
+		}
+		if seen[r.Language] {
+			t.Errorf("duplicate language in corpus: %q", r.Language)
+		}
+		seen[r.Language] = true
+		if repoDirName(r) == "" || strings.ContainsAny(repoDirName(r), "/\\") {
+			t.Errorf("unsafe cache dir name %q for %+v", repoDirName(r), r)
+		}
+	}
+	// BenchRepos returns a copy — mutating it must not affect the source list.
+	repos[0].URL = "mutated"
+	if BenchRepos()[0].URL == "mutated" {
+		t.Error("BenchRepos leaked a reference to the frozen corpus")
+	}
+}
+
+func TestEnsureRepoCacheHit(t *testing.T) {
+	cacheDir := t.TempDir()
+	repo := BenchRepo{Language: "go", URL: "https://github.com/example/widget"}
+
+	// Simulate a prior checkout — EnsureRepo must reuse it without cloning.
+	dir := filepath.Join(cacheDir, repoDirName(repo))
+	if err := os.MkdirAll(filepath.Join(dir, ".git"), 0o755); err != nil {
+		t.Fatal(err)
+	}
+	got, err := EnsureRepo(cacheDir, repo)
+	if err != nil {
+		t.Fatalf("EnsureRepo cache hit errored (did it try to clone?): %v", err)
+	}
+	if got != dir {
+		t.Errorf("EnsureRepo = %q, want cached %q", got, dir)
+	}
+}

internal/eval/quality/confidence.go

@@ -0,0 +1,204 @@
+package quality
+
+import (
+	"bufio"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io"
+	"math"
+	"os"
+	"path/filepath"
+	"sort"
+	"time"
+
+	"github.com/zzet/gortex/internal/platform"
+)
+
+// ConfidenceRecord captures the candidate-score distribution for
+// one search call. The distribution shape (top-1 vs top-2 gap,
+// std-dev, ratio) tells you how confident the ranker was — a sharp
+// top-1 with a long tail is a high-confidence answer; a flat
+// distribution is the ranker shrugging.
+type ConfidenceRecord struct {
+	TS       time.Time `json:"ts"`
+	Query    string    `json:"query"`
+	Top1     float64   `json:"top1"`
+	Top2     float64   `json:"top2,omitempty"`
+	Mean     float64   `json:"mean"`
+	StdDev   float64   `json:"std_dev"`
+	Ratio12  float64   `json:"ratio_top1_top2,omitempty"` // top1 / top2; >>1 = confident
+	K        int       `json:"k"`                          // number of scored candidates
+}
+
+// ConfidenceFromScores derives a ConfidenceRecord from a slice of
+// per-candidate scores. Negative / zero K returns a zero record so
+// the caller can opt out by passing an empty slice.
+func ConfidenceFromScores(query string, scores []float64) ConfidenceRecord {
+	r := ConfidenceRecord{
+		TS:    time.Now().UTC(),
+		Query: query,
+		K:     len(scores),
+	}
+	if len(scores) == 0 {
+		return r
+	}
+	sorted := make([]float64, len(scores))
+	copy(sorted, scores)
+	sort.Sort(sort.Reverse(sort.Float64Slice(sorted)))
+	r.Top1 = sorted[0]
+	if len(sorted) > 1 {
+		r.Top2 = sorted[1]
+		if r.Top2 != 0 {
+			r.Ratio12 = r.Top1 / r.Top2
+		}
+	}
+	var sum float64
+	for _, v := range scores {
+		sum += v
+	}
+	r.Mean = sum / float64(len(scores))
+	var ss float64
+	for _, v := range scores {
+		ss += (v - r.Mean) * (v - r.Mean)
+	}
+	r.StdDev = math.Sqrt(ss / float64(len(scores)))
+	return r
+}
+
+// ConfidenceTracker appends records to a JSONL log on disk. Safe
+// for in-process concurrent appends because the file is opened
+// with O_APPEND.
+type ConfidenceTracker struct {
+	Path string
+}
+
+// NewConfidenceTracker returns a tracker bound to a log path.
+// Empty path is allowed — Record then no-ops without erroring.
+func NewConfidenceTracker(path string) *ConfidenceTracker {
+	return &ConfidenceTracker{Path: path}
+}
+
+// DefaultConfidenceLogPath is the canonical persistence location.
+// An absolute $XDG_CACHE_HOME is honoured; otherwise the log stays
+// under os.UserCacheDir() as before. Returns empty when no cache dir
+// can be resolved.
+func DefaultConfidenceLogPath() string {
+	if v := os.Getenv("XDG_CACHE_HOME"); v == "" || !filepath.IsAbs(v) {
+		if base, err := os.UserCacheDir(); err != nil || base == "" {
+			return ""
+		}
+	}
+	return filepath.Join(platform.OSCacheDir(), "confidence.jsonl")
+}
+
+// Record appends one record to the log. No-op when Path is empty.
+func (t *ConfidenceTracker) Record(r ConfidenceRecord) error {
+	if t.Path == "" {
+		return nil
+	}
+	if err := os.MkdirAll(filepath.Dir(t.Path), 0o755); err != nil {
+		return err
+	}
+	f, err := os.OpenFile(t.Path, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0o644)
+	if err != nil {
+		return err
+	}
+	defer f.Close()
+	body, err := json.Marshal(r)
+	if err != nil {
+		return err
+	}
+	body = append(body, '\n')
+	_, err = f.Write(body)
+	return err
+}
+
+// LoadConfidenceLog reads the log at path and returns records with
+// ts >= since (zero since returns all). Malformed lines are
+// skipped so a previous crash mid-write doesn't break readers.
+func LoadConfidenceLog(path string, since time.Time) ([]ConfidenceRecord, error) {
+	if path == "" {
+		return nil, nil
+	}
+	f, err := os.Open(path)
+	if err != nil {
+		if errors.Is(err, os.ErrNotExist) {
+			return nil, nil
+		}
+		return nil, fmt.Errorf("open confidence log: %w", err)
+	}
+	defer f.Close()
+	r := bufio.NewReaderSize(f, 64*1024)
+	out := []ConfidenceRecord{}
+	for {
+		line, err := r.ReadBytes('\n')
+		if len(line) > 0 {
+			if n := len(line); n > 0 && line[n-1] == '\n' {
+				line = line[:n-1]
+			}
+			if len(line) > 0 {
+				var rec ConfidenceRecord
+				if json.Unmarshal(line, &rec) == nil {
+					if since.IsZero() || !rec.TS.Before(since) {
+						out = append(out, rec)
+					}
+				}
+			}
+		}
+		if err != nil {
+			if errors.Is(err, io.EOF) {
+				break
+			}
+			return out, fmt.Errorf("read confidence log: %w", err)
+		}
+	}
+	return out, nil
+}
+
+// ConfidenceSummary aggregates a slice of records into the headline
+// statistics the `gortex eval quality confidence` subcommand emits.
+type ConfidenceSummary struct {
+	Count            int     `json:"count"`
+	MedianTop1       float64 `json:"median_top1"`
+	MedianRatio12    float64 `json:"median_ratio_top1_top2"`
+	MedianStdDev     float64 `json:"median_std_dev"`
+	LowConfidenceCount int   `json:"low_confidence_count"` // records with Ratio12 < 1.25
+}
+
+// SummarizeConfidence reduces a slice of records into the summary.
+// Median across records; "low confidence" = Ratio12 < 1.25 (the
+// ranker barely separated top-1 from top-2).
+func SummarizeConfidence(records []ConfidenceRecord) ConfidenceSummary {
+	s := ConfidenceSummary{Count: len(records)}
+	if len(records) == 0 {
+		return s
+	}
+	top1s := make([]float64, 0, len(records))
+	ratios := make([]float64, 0, len(records))
+	stds := make([]float64, 0, len(records))
+	for _, r := range records {
+		top1s = append(top1s, r.Top1)
+		stds = append(stds, r.StdDev)
+		if r.Ratio12 > 0 {
+			ratios = append(ratios, r.Ratio12)
+		}
+		if r.Ratio12 > 0 && r.Ratio12 < 1.25 {
+			s.LowConfidenceCount++
+		}
+	}
+	s.MedianTop1 = medianFloats(top1s)
+	s.MedianRatio12 = medianFloats(ratios)
+	s.MedianStdDev = medianFloats(stds)
+	return s
+}
+
+func medianFloats(vs []float64) float64 {
+	if len(vs) == 0 {
+		return 0
+	}
+	c := make([]float64, len(vs))
+	copy(c, vs)
+	sort.Float64s(c)
+	return c[len(c)/2]
+}

internal/eval/quality/confidence_test.go

@@ -0,0 +1,153 @@
+package quality
+
+import (
+	"math"
+	"os"
+	"path/filepath"
+	"testing"
+	"time"
+)
+
+func TestConfidenceFromScores_Basic(t *testing.T) {
+	scores := []float64{0.95, 0.7, 0.5, 0.3}
+	r := ConfidenceFromScores("q", scores)
+	if r.Top1 != 0.95 || r.Top2 != 0.7 {
+		t.Errorf("top1/top2 = %.2f/%.2f, want 0.95/0.7", r.Top1, r.Top2)
+	}
+	wantRatio := 0.95 / 0.7
+	if math.Abs(r.Ratio12-wantRatio) > 1e-9 {
+		t.Errorf("Ratio12 = %.4f, want %.4f", r.Ratio12, wantRatio)
+	}
+	if r.K != 4 {
+		t.Errorf("K = %d, want 4", r.K)
+	}
+}
+
+func TestConfidenceFromScores_Empty(t *testing.T) {
+	r := ConfidenceFromScores("q", nil)
+	if r.K != 0 || r.Top1 != 0 || r.Top2 != 0 {
+		t.Errorf("empty input should yield zero record, got %+v", r)
+	}
+}
+
+func TestConfidenceFromScores_SingleScore(t *testing.T) {
+	r := ConfidenceFromScores("q", []float64{0.5})
+	if r.Top1 != 0.5 || r.Top2 != 0 || r.Ratio12 != 0 {
+		t.Errorf("single-score = %+v, want top1=0.5 others=0", r)
+	}
+}
+
+func TestConfidenceTracker_RoundTrip(t *testing.T) {
+	dir := t.TempDir()
+	path := filepath.Join(dir, "conf.jsonl")
+	tr := NewConfidenceTracker(path)
+	for i := range 3 {
+		rec := ConfidenceFromScores(
+			"q",
+			[]float64{0.9, 0.8 - 0.1*float64(i), 0.5, 0.4},
+		)
+		if err := tr.Record(rec); err != nil {
+			t.Fatal(err)
+		}
+	}
+	got, err := LoadConfidenceLog(path, time.Time{})
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(got) != 3 {
+		t.Errorf("loaded %d records, want 3", len(got))
+	}
+}
+
+func TestConfidenceTracker_EmptyPathNoop(t *testing.T) {
+	tr := NewConfidenceTracker("")
+	if err := tr.Record(ConfidenceFromScores("q", []float64{1})); err != nil {
+		t.Errorf("empty-path tracker should no-op, got %v", err)
+	}
+}
+
+func TestLoadConfidenceLog_FiltersSince(t *testing.T) {
+	dir := t.TempDir()
+	path := filepath.Join(dir, "conf.jsonl")
+	tr := NewConfidenceTracker(path)
+	now := time.Now().UTC()
+
+	old := ConfidenceFromScores("old", []float64{0.5})
+	old.TS = now.Add(-48 * time.Hour)
+	_ = tr.Record(old)
+
+	recent := ConfidenceFromScores("recent", []float64{0.7})
+	recent.TS = now
+	_ = tr.Record(recent)
+
+	got, _ := LoadConfidenceLog(path, now.Add(-1*time.Hour))
+	if len(got) != 1 {
+		t.Errorf("since-filter = %d records, want 1", len(got))
+	}
+	if got[0].Query != "recent" {
+		t.Errorf("kept wrong record: %+v", got[0])
+	}
+}
+
+func TestLoadConfidenceLog_MissingFile(t *testing.T) {
+	got, err := LoadConfidenceLog(filepath.Join(t.TempDir(), "missing.jsonl"), time.Time{})
+	if err != nil {
+		t.Errorf("missing file should not error, got %v", err)
+	}
+	if len(got) != 0 {
+		t.Errorf("missing file should yield empty, got %d", len(got))
+	}
+}
+
+func TestLoadConfidenceLog_TolerateMalformedLines(t *testing.T) {
+	dir := t.TempDir()
+	path := filepath.Join(dir, "conf.jsonl")
+	body := `{"ts":"2026-05-18T10:00:00Z","query":"good","top1":0.9,"k":1}` + "\n" +
+		`{not json}` + "\n" +
+		`{"ts":"2026-05-18T11:00:00Z","query":"also-good","top1":0.8,"k":1}` + "\n"
+	if err := os.WriteFile(path, []byte(body), 0o644); err != nil {
+		t.Fatal(err)
+	}
+	got, err := LoadConfidenceLog(path, time.Time{})
+	if err != nil {
+		t.Fatalf("malformed-line tolerance failed: %v", err)
+	}
+	if len(got) != 2 {
+		t.Errorf("got %d records, want 2 (malformed dropped)", len(got))
+	}
+}
+
+func TestSummarizeConfidence(t *testing.T) {
+	records := []ConfidenceRecord{
+		{Top1: 0.9, Top2: 0.7, Ratio12: 1.286, StdDev: 0.2},
+		{Top1: 0.5, Top2: 0.45, Ratio12: 1.11, StdDev: 0.1}, // low confidence
+		{Top1: 0.8, Top2: 0.3, Ratio12: 2.67, StdDev: 0.3},
+	}
+	s := SummarizeConfidence(records)
+	if s.Count != 3 {
+		t.Errorf("count = %d, want 3", s.Count)
+	}
+	if s.LowConfidenceCount != 1 {
+		t.Errorf("low confidence = %d, want 1 (Ratio12 < 1.25)", s.LowConfidenceCount)
+	}
+	if s.MedianTop1 != 0.8 {
+		t.Errorf("median top1 = %.2f, want 0.8", s.MedianTop1)
+	}
+}
+
+func TestSummarizeConfidence_Empty(t *testing.T) {
+	s := SummarizeConfidence(nil)
+	if s.Count != 0 || s.MedianTop1 != 0 {
+		t.Errorf("empty input should yield zero summary, got %+v", s)
+	}
+}
+
+func TestDefaultConfidenceLogPath(t *testing.T) {
+	got := DefaultConfidenceLogPath()
+	if got == "" {
+		return // UserCacheDir unavailable
+	}
+	if filepath.Base(got) != "confidence.jsonl" {
+		t.Errorf("default path basename = %q, want confidence.jsonl", filepath.Base(got))
+	}
+}

internal/eval/quality/drift.go

@@ -0,0 +1,156 @@
+// Package quality provides measurement infrastructure for the
+// retrieval pipeline: embedder drift detection, retrieval-confidence
+// tracking, query-log replay (top-k churn between two ranker
+// configurations), and weight-tuning analysis.
+//
+// All four are surface-level analyzers — they read substrate already
+// shipped (savings store, search index, rerank pipeline) and produce
+// markdown / JSON artifacts. No new state in the hot path.
+package quality
+
+import (
+	"encoding/json"
+	"errors"
+	"fmt"
+	"os"
+	"path/filepath"
+	"time"
+
+	"github.com/zzet/gortex/internal/platform"
+)
+
+// EmbedderFingerprint captures the identity of the active embedder
+// at one point in time. Persisting it lets the drift detector flag
+// silent provider / model / dimension changes that would otherwise
+// produce confusing recall regressions.
+type EmbedderFingerprint struct {
+	Provider       string    `json:"provider"`
+	Model          string    `json:"model"`
+	ModelRevision  string    `json:"model_revision,omitempty"`
+	EmbeddingDim   int       `json:"embedding_dim"`
+	SampleVecSHA256 string   `json:"sample_vec_sha256,omitempty"`
+	RecordedAt     time.Time `json:"recorded_at"`
+}
+
+// DriftWarning is the structured signal the detector emits when the
+// current fingerprint differs from the stored one. Empty Changes
+// slice means no drift.
+type DriftWarning struct {
+	Previous EmbedderFingerprint `json:"previous"`
+	Current  EmbedderFingerprint `json:"current"`
+	Changes  []string            `json:"changes"`
+}
+
+// HasDrift reports whether the warning is non-empty — convenience
+// for callers that just want a boolean (e.g. CI gates).
+func (w DriftWarning) HasDrift() bool { return len(w.Changes) > 0 }
+
+// DefaultFingerprintPath returns the canonical persistence location.
+// An absolute $XDG_CACHE_HOME is honoured; otherwise the file stays
+// under os.UserCacheDir() as before. Returns empty when the cache dir
+// is unavailable; callers should treat empty as "don't persist, just
+// compare in-memory".
+func DefaultFingerprintPath() string {
+	if v := os.Getenv("XDG_CACHE_HOME"); v == "" || !filepath.IsAbs(v) {
+		if base, err := os.UserCacheDir(); err != nil || base == "" {
+			return ""
+		}
+	}
+	return filepath.Join(platform.OSCacheDir(), "embedding-fingerprint.json")
+}
+
+// DriftDetector wraps the fingerprint persistence and the
+// comparison logic. Concurrency: not safe — call from one goroutine
+// per detector instance.
+type DriftDetector struct {
+	Path string
+}
+
+// NewDriftDetector returns a detector bound to a persistence path.
+// Empty path is allowed — the detector still compares in-memory but
+// never writes to disk.
+func NewDriftDetector(path string) *DriftDetector {
+	return &DriftDetector{Path: path}
+}
+
+// LoadPrevious reads the most recent fingerprint, or returns
+// (zero-value, nil) when none exists. Real I/O errors propagate so a
+// permission problem surfaces.
+func (d *DriftDetector) LoadPrevious() (EmbedderFingerprint, error) {
+	if d.Path == "" {
+		return EmbedderFingerprint{}, nil
+	}
+	raw, err := os.ReadFile(d.Path)
+	if errors.Is(err, os.ErrNotExist) {
+		return EmbedderFingerprint{}, nil
+	}
+	if err != nil {
+		return EmbedderFingerprint{}, fmt.Errorf("read fingerprint: %w", err)
+	}
+	var fp EmbedderFingerprint
+	if err := json.Unmarshal(raw, &fp); err != nil {
+		return EmbedderFingerprint{}, fmt.Errorf("parse fingerprint: %w", err)
+	}
+	return fp, nil
+}
+
+// Save writes the current fingerprint atomically (temp + rename).
+// Silently no-ops when the detector has no persistence path.
+func (d *DriftDetector) Save(fp EmbedderFingerprint) error {
+	if d.Path == "" {
+		return nil
+	}
+	if err := os.MkdirAll(filepath.Dir(d.Path), 0o755); err != nil {
+		return err
+	}
+	body, err := json.MarshalIndent(fp, "", "  ")
+	if err != nil {
+		return err
+	}
+	tmp := d.Path + ".tmp"
+	if err := os.WriteFile(tmp, body, 0o644); err != nil {
+		return err
+	}
+	return os.Rename(tmp, d.Path)
+}
+
+// Compare returns the drift warning for current vs the stored
+// fingerprint. The persistence file is NOT updated by Compare —
+// callers decide when to promote (e.g. only after operator
+// confirmation, or always in CI).
+func (d *DriftDetector) Compare(current EmbedderFingerprint) (DriftWarning, error) {
+	prev, err := d.LoadPrevious()
+	if err != nil {
+		return DriftWarning{}, err
+	}
+	return DiffFingerprints(prev, current), nil
+}
+
+// DiffFingerprints is the pure comparison logic — exported so tests
+// and standalone consumers can use it without touching disk.
+func DiffFingerprints(prev, current EmbedderFingerprint) DriftWarning {
+	w := DriftWarning{Previous: prev, Current: current}
+	if prev == (EmbedderFingerprint{}) {
+		// No previous record — first run; not drift.
+		return w
+	}
+	if prev.Provider != current.Provider {
+		w.Changes = append(w.Changes, fmt.Sprintf("provider: %q → %q", prev.Provider, current.Provider))
+	}
+	if prev.Model != current.Model {
+		w.Changes = append(w.Changes, fmt.Sprintf("model: %q → %q", prev.Model, current.Model))
+	}
+	if prev.ModelRevision != current.ModelRevision && (prev.ModelRevision != "" || current.ModelRevision != "") {
+		w.Changes = append(w.Changes, fmt.Sprintf("model_revision: %q → %q", prev.ModelRevision, current.ModelRevision))
+	}
+	if prev.EmbeddingDim != current.EmbeddingDim {
+		w.Changes = append(w.Changes, fmt.Sprintf("embedding_dim: %d → %d", prev.EmbeddingDim, current.EmbeddingDim))
+	}
+	if prev.SampleVecSHA256 != current.SampleVecSHA256 && (prev.SampleVecSHA256 != "" || current.SampleVecSHA256 != "") {
+		// Vector-shape change without dim change usually means the
+		// model was re-quantized or the input pre-processor
+		// changed. Worth flagging.
+		w.Changes = append(w.Changes, "sample_vec_sha256 changed (re-quantized or pre-processor swap?)")
+	}
+	return w
+}

internal/eval/quality/drift_test.go

@@ -0,0 +1,168 @@
+package quality
+
+import (
+	"os"
+	"path/filepath"
+	"testing"
+	"time"
+)
+
+func TestDiffFingerprints_NoPreviousIsNoDrift(t *testing.T) {
+	cur := EmbedderFingerprint{Provider: "hugot", Model: "MiniLM-L6-v2", EmbeddingDim: 384}
+	w := DiffFingerprints(EmbedderFingerprint{}, cur)
+	if w.HasDrift() {
+		t.Errorf("no previous fingerprint should not flag drift; got %v", w.Changes)
+	}
+}
+
+func TestDiffFingerprints_ProviderChangeFlagged(t *testing.T) {
+	prev := EmbedderFingerprint{Provider: "hugot", Model: "MiniLM-L6-v2", EmbeddingDim: 384}
+	cur := EmbedderFingerprint{Provider: "openai", Model: "text-embedding-3-small", EmbeddingDim: 384}
+	w := DiffFingerprints(prev, cur)
+	if !w.HasDrift() {
+		t.Fatal("provider change should flag drift")
+	}
+	hasProvider := false
+	hasModel := false
+	for _, c := range w.Changes {
+		if contains(c, "provider:") {
+			hasProvider = true
+		}
+		if contains(c, "model:") {
+			hasModel = true
+		}
+	}
+	if !hasProvider || !hasModel {
+		t.Errorf("expected provider + model in changes, got %v", w.Changes)
+	}
+}
+
+func TestDiffFingerprints_DimChangeFlagged(t *testing.T) {
+	prev := EmbedderFingerprint{Provider: "hugot", Model: "MiniLM-L6-v2", EmbeddingDim: 384}
+	cur := EmbedderFingerprint{Provider: "hugot", Model: "MiniLM-L6-v2", EmbeddingDim: 768}
+	w := DiffFingerprints(prev, cur)
+	if !w.HasDrift() {
+		t.Fatal("dim change should flag drift")
+	}
+}
+
+func TestDiffFingerprints_IdenticalIsNoDrift(t *testing.T) {
+	fp := EmbedderFingerprint{Provider: "hugot", Model: "MiniLM-L6-v2", EmbeddingDim: 384, SampleVecSHA256: "abc"}
+	w := DiffFingerprints(fp, fp)
+	if w.HasDrift() {
+		t.Errorf("identical fingerprints should not flag drift; got %v", w.Changes)
+	}
+}
+
+func TestDiffFingerprints_SampleVecChangeFlagged(t *testing.T) {
+	prev := EmbedderFingerprint{Provider: "hugot", Model: "M", EmbeddingDim: 384, SampleVecSHA256: "abc"}
+	cur := EmbedderFingerprint{Provider: "hugot", Model: "M", EmbeddingDim: 384, SampleVecSHA256: "xyz"}
+	w := DiffFingerprints(prev, cur)
+	if !w.HasDrift() {
+		t.Fatal("sample-vec change should flag drift")
+	}
+}
+
+func TestDriftDetector_RoundTripFingerprint(t *testing.T) {
+	dir := t.TempDir()
+	path := filepath.Join(dir, "fp.json")
+	d := NewDriftDetector(path)
+
+	want := EmbedderFingerprint{
+		Provider:        "hugot",
+		Model:           "MiniLM-L6-v2",
+		ModelRevision:   "fp32",
+		EmbeddingDim:    384,
+		SampleVecSHA256: "deadbeef",
+		RecordedAt:      time.Now().UTC().Round(time.Second),
+	}
+	if err := d.Save(want); err != nil {
+		t.Fatalf("Save: %v", err)
+	}
+	got, err := d.LoadPrevious()
+	if err != nil {
+		t.Fatalf("LoadPrevious: %v", err)
+	}
+	if got.Provider != want.Provider || got.EmbeddingDim != want.EmbeddingDim {
+		t.Errorf("round-trip lost data: got %+v want %+v", got, want)
+	}
+}
+
+func TestDriftDetector_LoadMissingReturnsZero(t *testing.T) {
+	d := NewDriftDetector(filepath.Join(t.TempDir(), "missing.json"))
+	got, err := d.LoadPrevious()
+	if err != nil {
+		t.Errorf("missing file should not error, got %v", err)
+	}
+	if got != (EmbedderFingerprint{}) {
+		t.Errorf("missing file should yield zero, got %+v", got)
+	}
+}
+
+func TestDriftDetector_EmptyPathSilent(t *testing.T) {
+	d := NewDriftDetector("")
+	if err := d.Save(EmbedderFingerprint{Provider: "x"}); err != nil {
+		t.Errorf("empty path Save should be no-op, got %v", err)
+	}
+	got, err := d.LoadPrevious()
+	if err != nil || got != (EmbedderFingerprint{}) {
+		t.Errorf("empty path LoadPrevious should return zero / nil, got %+v / %v", got, err)
+	}
+}
+
+func TestDriftDetector_Compare_EndToEnd(t *testing.T) {
+	dir := t.TempDir()
+	d := NewDriftDetector(filepath.Join(dir, "fp.json"))
+
+	// First compare: no previous record → no drift.
+	cur := EmbedderFingerprint{Provider: "hugot", Model: "A", EmbeddingDim: 384}
+	w, err := d.Compare(cur)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if w.HasDrift() {
+		t.Error("first comparison should report no drift")
+	}
+	_ = d.Save(cur)
+
+	// Second compare with same fingerprint: still no drift.
+	w, _ = d.Compare(cur)
+	if w.HasDrift() {
+		t.Errorf("identical fingerprint should not drift, got %v", w.Changes)
+	}
+
+	// Change the model: should drift.
+	cur.Model = "B"
+	w, _ = d.Compare(cur)
+	if !w.HasDrift() {
+		t.Error("model change should drift")
+	}
+}
+
+func TestDefaultFingerprintPath(t *testing.T) {
+	got := DefaultFingerprintPath()
+	if got == "" {
+		// Acceptable when UserCacheDir is unavailable; just don't panic.
+		return
+	}
+	if filepath.Base(got) != "embedding-fingerprint.json" {
+		t.Errorf("default path basename = %q, want embedding-fingerprint.json", filepath.Base(got))
+	}
+}
+
+func contains(s, sub string) bool {
+	return len(s) >= len(sub) && (s == sub || stringsHasSubstring(s, sub))
+}
+
+func stringsHasSubstring(s, sub string) bool {
+	for i := 0; i+len(sub) <= len(s); i++ {
+		if s[i:i+len(sub)] == sub {
+			return true
+		}
+	}
+	return false
+}
+
+// Quiet "imported and not used" when the harness iterates the
+// fingerprint file directly without going through DriftDetector.
+var _ = os.Stat

internal/eval/quality/replay.go

@@ -0,0 +1,211 @@
+package quality
+
+import (
+	"fmt"
+	"sort"
+)
+
+// ReplayQuery is one (query, expected) pair sourced from a query
+// log. The expected list is optional — when present the replay
+// computes recall@k against it; when absent the replay only
+// produces ranking-delta metrics (Kendall τ, top-k churn) between
+// baseline and candidate.
+type ReplayQuery struct {
+	Query    string   `json:"query"`
+	Expected []string `json:"expected,omitempty"`
+}
+
+// RankerFunc is the contract a candidate / baseline implementation
+// satisfies. Returns the ordered list of result IDs (file paths /
+// symbol IDs — what shape the caller uses) for one query.
+type RankerFunc func(query string, topK int) []string
+
+// PerQueryDelta is one query's outcome from a replay run.
+type PerQueryDelta struct {
+	Query        string   `json:"query"`
+	Baseline     []string `json:"baseline"`
+	Candidate    []string `json:"candidate"`
+	Kendall      float64  `json:"kendall_tau"`     // -1..+1; 1 = identical order
+	Top1Changed  bool     `json:"top1_changed"`
+	Top5Changes  int      `json:"top5_changes"`    // |baseline[:5] △ candidate[:5]|
+	RecallBase   float64  `json:"recall_baseline,omitempty"`
+	RecallCand   float64  `json:"recall_candidate,omitempty"`
+}
+
+// ReplayResult is the aggregate output of a replay run.
+type ReplayResult struct {
+	PerQuery       []PerQueryDelta `json:"per_query"`
+	MeanKendall    float64         `json:"mean_kendall_tau"`
+	Top1ChurnPct   float64         `json:"top1_churn_pct"`   // % of queries with top1 changed
+	MeanTop5Churn  float64         `json:"mean_top5_changes"`
+	RecallDelta    float64         `json:"recall_delta"`     // candidate - baseline mean recall (when ground truth exists)
+}
+
+// Replay walks the query log, scores each query against baseline +
+// candidate, and aggregates the deltas. K is the top-K depth for
+// the comparison (10 is the standard NDCG@10 / top-10 churn shape).
+func Replay(queries []ReplayQuery, baseline, candidate RankerFunc, k int) (ReplayResult, error) {
+	if baseline == nil || candidate == nil {
+		return ReplayResult{}, fmt.Errorf("baseline and candidate rankers are required")
+	}
+	if k <= 0 {
+		k = 10
+	}
+	out := ReplayResult{PerQuery: make([]PerQueryDelta, 0, len(queries))}
+	var totKendall, totTop5 float64
+	var top1Changes, withGT int
+	var totRecallBase, totRecallCand float64
+	for _, q := range queries {
+		b := baseline(q.Query, k)
+		c := candidate(q.Query, k)
+		row := PerQueryDelta{
+			Query:       q.Query,
+			Baseline:    truncate(b, k),
+			Candidate:   truncate(c, k),
+			Kendall:     kendallTauTopK(b, c, k),
+			Top1Changed: top1Differs(b, c),
+			Top5Changes: setSymDiffSize(prefix(b, 5), prefix(c, 5)),
+		}
+		if len(q.Expected) > 0 {
+			row.RecallBase = recallAtK(b, q.Expected, k)
+			row.RecallCand = recallAtK(c, q.Expected, k)
+			totRecallBase += row.RecallBase
+			totRecallCand += row.RecallCand
+			withGT++
+		}
+		totKendall += row.Kendall
+		totTop5 += float64(row.Top5Changes)
+		if row.Top1Changed {
+			top1Changes++
+		}
+		out.PerQuery = append(out.PerQuery, row)
+	}
+	if n := len(queries); n > 0 {
+		out.MeanKendall = totKendall / float64(n)
+		out.MeanTop5Churn = totTop5 / float64(n)
+		out.Top1ChurnPct = float64(top1Changes) / float64(n) * 100.0
+	}
+	if withGT > 0 {
+		out.RecallDelta = (totRecallCand - totRecallBase) / float64(withGT)
+	}
+	return out, nil
+}
+
+// --- ranking-delta math ---------------------------------------------
+
+// kendallTauTopK computes Kendall's τ over the top-K shared
+// elements of two ranked lists. Returns 1 when both lists are
+// empty or share fewer than 2 elements (τ is undefined; treat as
+// "no measurable disagreement").
+func kendallTauTopK(a, b []string, k int) float64 {
+	aPref := prefix(a, k)
+	bPref := prefix(b, k)
+	rankA := indexMap(aPref)
+	rankB := indexMap(bPref)
+	// Intersection of the two prefixes.
+	var shared []string
+	for _, id := range aPref {
+		if _, ok := rankB[id]; ok {
+			shared = append(shared, id)
+		}
+	}
+	n := len(shared)
+	if n < 2 {
+		return 1
+	}
+	var concordant, discordant int
+	for i := range n {
+		for j := i + 1; j < n; j++ {
+			a, b := shared[i], shared[j]
+			aOrder := rankA[a] < rankA[b]
+			bOrder := rankB[a] < rankB[b]
+			if aOrder == bOrder {
+				concordant++
+			} else {
+				discordant++
+			}
+		}
+	}
+	pairs := concordant + discordant
+	if pairs == 0 {
+		return 1
+	}
+	return float64(concordant-discordant) / float64(pairs)
+}
+
+func top1Differs(a, b []string) bool {
+	if len(a) == 0 && len(b) == 0 {
+		return false
+	}
+	if len(a) == 0 || len(b) == 0 {
+		return true
+	}
+	return a[0] != b[0]
+}
+
+func setSymDiffSize(a, b []string) int {
+	as := map[string]struct{}{}
+	for _, x := range a {
+		as[x] = struct{}{}
+	}
+	bs := map[string]struct{}{}
+	for _, x := range b {
+		bs[x] = struct{}{}
+	}
+	count := 0
+	for x := range as {
+		if _, ok := bs[x]; !ok {
+			count++
+		}
+	}
+	for x := range bs {
+		if _, ok := as[x]; !ok {
+			count++
+		}
+	}
+	return count
+}
+
+func recallAtK(returned, expected []string, k int) float64 {
+	if len(expected) == 0 {
+		return 0
+	}
+	expSet := map[string]bool{}
+	for _, e := range expected {
+		expSet[e] = true
+	}
+	hits := 0
+	limit := min(k, len(returned))
+	for i := range limit {
+		if expSet[returned[i]] {
+			hits++
+		}
+	}
+	return float64(hits) / float64(len(expected))
+}
+
+func prefix(s []string, k int) []string {
+	if k <= 0 || len(s) <= k {
+		return s
+	}
+	return s[:k]
+}
+
+func truncate(s []string, k int) []string {
+	out := make([]string, 0, k)
+	out = append(out, prefix(s, k)...)
+	return out
+}
+
+func indexMap(s []string) map[string]int {
+	m := make(map[string]int, len(s))
+	for i, v := range s {
+		if _, ok := m[v]; !ok {
+			m[v] = i
+		}
+	}
+	return m
+}
+
+// Used only to keep imports stable when iterating.
+var _ = sort.Sort

internal/eval/quality/replay_test.go

@@ -0,0 +1,179 @@
+package quality
+
+import (
+	"math"
+	"strings"
+	"testing"
+)
+
+// Test rankers: stub ranker funcs that return fixed lists per query
+// — keeps the replay tests deterministic and avoids pulling in the
+// real engine.
+
+func staticRanker(byQuery map[string][]string) RankerFunc {
+	return func(q string, topK int) []string {
+		out := byQuery[q]
+		if topK > 0 && len(out) > topK {
+			return out[:topK]
+		}
+		return out
+	}
+}
+
+func TestReplay_IdenticalRankersZeroChurn(t *testing.T) {
+	baseline := staticRanker(map[string][]string{"q1": {"a", "b", "c"}})
+	candidate := baseline
+
+	queries := []ReplayQuery{{Query: "q1"}}
+	got, err := Replay(queries, baseline, candidate, 10)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if got.Top1ChurnPct != 0 {
+		t.Errorf("identical rankers should have 0%% top1 churn, got %.2f%%", got.Top1ChurnPct)
+	}
+	if got.MeanKendall < 0.99 {
+		t.Errorf("identical rankers should have kendall ≈ 1, got %.4f", got.MeanKendall)
+	}
+	if got.MeanTop5Churn != 0 {
+		t.Errorf("identical rankers should have 0 top5 changes, got %.2f", got.MeanTop5Churn)
+	}
+}
+
+func TestReplay_OppositeRankersHighChurn(t *testing.T) {
+	baseline := staticRanker(map[string][]string{"q1": {"a", "b", "c", "d", "e"}})
+	candidate := staticRanker(map[string][]string{"q1": {"e", "d", "c", "b", "a"}})
+
+	queries := []ReplayQuery{{Query: "q1"}}
+	got, err := Replay(queries, baseline, candidate, 10)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if got.Top1ChurnPct != 100 {
+		t.Errorf("reversed ranker should have 100%% top1 churn, got %.2f%%", got.Top1ChurnPct)
+	}
+	// Kendall τ for reversed order over 5 elements: all pairs are
+	// discordant → τ = -1.
+	if got.MeanKendall > -0.99 {
+		t.Errorf("reversed rankers should have kendall ≈ -1, got %.4f", got.MeanKendall)
+	}
+}
+
+func TestReplay_RecallDelta(t *testing.T) {
+	baseline := staticRanker(map[string][]string{"q1": {"miss1", "miss2", "a"}})
+	candidate := staticRanker(map[string][]string{"q1": {"a", "miss1", "miss2"}})
+
+	queries := []ReplayQuery{
+		{Query: "q1", Expected: []string{"a"}},
+	}
+	got, err := Replay(queries, baseline, candidate, 10)
+	if err != nil {
+		t.Fatal(err)
+	}
+	// Both find "a" within top 10 → both recall = 1.0 → delta = 0.
+	if math.Abs(got.RecallDelta) > 0.01 {
+		t.Errorf("both-find recall delta = %.4f, want 0", got.RecallDelta)
+	}
+}
+
+func TestReplay_RecallDelta_CandidateBetter(t *testing.T) {
+	// Baseline finds nothing in top-K, candidate finds the expected.
+	baseline := staticRanker(map[string][]string{"q1": {"x", "y", "z"}})
+	candidate := staticRanker(map[string][]string{"q1": {"a"}})
+
+	queries := []ReplayQuery{{Query: "q1", Expected: []string{"a"}}}
+	got, _ := Replay(queries, baseline, candidate, 10)
+	if got.RecallDelta <= 0 {
+		t.Errorf("candidate-better should yield positive recall delta, got %.4f", got.RecallDelta)
+	}
+}
+
+func TestReplay_NilRankerErrors(t *testing.T) {
+	_, err := Replay(nil, nil, staticRanker(nil), 10)
+	if err == nil {
+		t.Error("nil baseline should error")
+	}
+}
+
+func TestKendallTauTopK_PerfectMatch(t *testing.T) {
+	tau := kendallTauTopK([]string{"a", "b", "c"}, []string{"a", "b", "c"}, 10)
+	if tau != 1 {
+		t.Errorf("identical lists τ = %.4f, want 1", tau)
+	}
+}
+
+func TestKendallTauTopK_FullyReversed(t *testing.T) {
+	tau := kendallTauTopK([]string{"a", "b", "c"}, []string{"c", "b", "a"}, 10)
+	if tau != -1 {
+		t.Errorf("reversed lists τ = %.4f, want -1", tau)
+	}
+}
+
+func TestKendallTauTopK_PartialOverlap(t *testing.T) {
+	// Only "b" and "c" overlap; in baseline they're in order (b,c);
+	// in candidate they're reversed (c,b). Single discordant pair
+	// out of 1 total → τ = -1.
+	tau := kendallTauTopK(
+		[]string{"a", "b", "c", "d"},
+		[]string{"e", "c", "b", "f"},
+		10,
+	)
+	if tau != -1 {
+		t.Errorf("partial-overlap τ = %.4f, want -1", tau)
+	}
+}
+
+func TestKendallTauTopK_TooFewSharedReturnsOne(t *testing.T) {
+	// Less than 2 shared elements → τ undefined; we return 1 to
+	// mean "no measurable disagreement".
+	tau := kendallTauTopK([]string{"a", "b"}, []string{"c", "d"}, 10)
+	if tau != 1 {
+		t.Errorf("disjoint lists τ = %.4f, want 1 (no measurable disagreement)", tau)
+	}
+}
+
+func TestSetSymDiffSize(t *testing.T) {
+	if got := setSymDiffSize([]string{"a", "b"}, []string{"b", "c"}); got != 2 {
+		t.Errorf("symdiff = %d, want 2 (a and c are unique)", got)
+	}
+	if got := setSymDiffSize([]string{"a"}, []string{"a"}); got != 0 {
+		t.Errorf("symdiff identical = %d, want 0", got)
+	}
+}
+
+func TestRecallAtK_BoundaryCases(t *testing.T) {
+	// No expected → 0.
+	if got := recallAtK([]string{"a"}, nil, 10); got != 0 {
+		t.Errorf("empty expected = %.4f, want 0", got)
+	}
+	// All expected found.
+	if got := recallAtK([]string{"a", "b"}, []string{"a", "b"}, 10); got != 1 {
+		t.Errorf("full recall = %.4f, want 1", got)
+	}
+	// Half found.
+	if got := recallAtK([]string{"a"}, []string{"a", "b"}, 10); got != 0.5 {
+		t.Errorf("half recall = %.4f, want 0.5", got)
+	}
+	// Cut by k.
+	if got := recallAtK([]string{"x", "y", "a"}, []string{"a"}, 2); got != 0 {
+		t.Errorf("k-cut recall = %.4f, want 0 (a is outside top-2)", got)
+	}
+}
+
+func TestTop1Differs(t *testing.T) {
+	if !top1Differs([]string{"a"}, []string{"b"}) {
+		t.Error("different top1 should report changed")
+	}
+	if top1Differs([]string{"a"}, []string{"a"}) {
+		t.Error("same top1 should not report changed")
+	}
+	if !top1Differs([]string{}, []string{"a"}) {
+		t.Error("empty vs non-empty top1 should report changed")
+	}
+	if top1Differs(nil, nil) {
+		t.Error("both empty should not report changed")
+	}
+}
+
+// Quiet "imported and not used" if a future helper needs it.
+var _ = strings.HasPrefix

internal/eval/quality/tuning.go

@@ -0,0 +1,126 @@
+package quality
+
+import (
+	"fmt"
+	"sort"
+	"strings"
+)
+
+// SignalFeedback ties one rerank signal to its observed effect on
+// agent-confirmed-useful results. Source: the existing `feedback`
+// tool's per-symbol useful / not-needed / missing counters.
+type SignalFeedback struct {
+	Signal     string  `json:"signal"`
+	Weight     float64 `json:"current_weight"`
+	UsefulHits int     `json:"useful_hits"`
+	MissedHits int     `json:"missed_hits"`
+	// SuggestedWeight is the tuner's recommendation; the operator
+	// decides whether to apply it. >Weight = the signal is producing
+	// useful results and should be amplified; <Weight = it's
+	// adding noise.
+	SuggestedWeight float64 `json:"suggested_weight"`
+	Reasoning       string  `json:"reasoning"`
+}
+
+// SuggestWeights consumes the per-signal feedback rows + a global
+// nudge factor and emits a per-signal suggestion. Pure analysis —
+// the rerank pipeline isn't mutated; the caller applies via
+// `.gortex.yaml::search.weights`.
+//
+// Algorithm:
+//   - signals with useful_hits > missed_hits get nudged UP by
+//     min(0.1, (useful - missed) / 100 * nudge)
+//   - signals with missed_hits > useful_hits get nudged DOWN by
+//     the same formula
+//   - signals with neither (no data) keep their current weight and
+//     report "insufficient data"
+//
+// Nudge is the per-call cap; 1.0 means "max ±0.1 per call",
+// 0.5 means "half of that". Keeping nudges small makes tuning a
+// gradient process rather than a wholesale weight swap.
+func SuggestWeights(rows []SignalFeedback, nudge float64) []SignalFeedback {
+	if nudge <= 0 {
+		nudge = 1.0
+	}
+	out := make([]SignalFeedback, 0, len(rows))
+	for _, r := range rows {
+		delta := 0.0
+		reason := ""
+		switch {
+		case r.UsefulHits == 0 && r.MissedHits == 0:
+			reason = "insufficient data (no useful / missed records this window)"
+		case r.UsefulHits > r.MissedHits:
+			diff := float64(r.UsefulHits - r.MissedHits)
+			delta = clamp(diff/100.0*nudge, 0, 0.1)
+			reason = fmt.Sprintf("useful_hits=%d > missed=%d → nudge up by %.3f",
+				r.UsefulHits, r.MissedHits, delta)
+		case r.MissedHits > r.UsefulHits:
+			diff := float64(r.MissedHits - r.UsefulHits)
+			delta = -clamp(diff/100.0*nudge, 0, 0.1)
+			reason = fmt.Sprintf("missed_hits=%d > useful=%d → nudge down by %.3f",
+				r.MissedHits, r.UsefulHits, -delta)
+		default:
+			reason = fmt.Sprintf("useful_hits == missed_hits (%d) → no change", r.UsefulHits)
+		}
+		r.SuggestedWeight = r.Weight + delta
+		if r.SuggestedWeight < 0 {
+			r.SuggestedWeight = 0
+		}
+		r.Reasoning = reason
+		out = append(out, r)
+	}
+	// Stable display order: biggest absolute suggested change first
+	// so the operator sees the action items at the top.
+	sort.SliceStable(out, func(i, j int) bool {
+		return absVal(out[i].SuggestedWeight-out[i].Weight) > absVal(out[j].SuggestedWeight-out[j].Weight)
+	})
+	return out
+}
+
+// RenderTuningMarkdown produces the operator-facing report. Columns:
+// signal, current weight, useful / missed counts, suggested weight,
+// reasoning. Tail summary: count of nudges with abs(delta) > 0.05
+// so the operator knows whether anything substantial is on the
+// table.
+func RenderTuningMarkdown(rows []SignalFeedback) string {
+	var b strings.Builder
+	fmt.Fprintln(&b, "# Rerank weight-tuning suggestion")
+	fmt.Fprintln(&b)
+	fmt.Fprintln(&b, "_Each row's `suggested_weight` is a calculated nudge from the current `.gortex.yaml::search.weights` value, based on the feedback log. **Operator decides whether to apply** — no automatic mutation._")
+	fmt.Fprintln(&b)
+	fmt.Fprintln(&b, "| signal | current | useful | missed | suggested | Δ | reasoning |")
+	fmt.Fprintln(&b, "|--------|--------:|-------:|-------:|----------:|--:|-----------|")
+	substantial := 0
+	for _, r := range rows {
+		delta := r.SuggestedWeight - r.Weight
+		if absVal(delta) >= 0.05 {
+			substantial++
+		}
+		fmt.Fprintf(&b, "| %s | %.3f | %d | %d | %.3f | %+.3f | %s |\n",
+			r.Signal, r.Weight, r.UsefulHits, r.MissedHits,
+			r.SuggestedWeight, delta, r.Reasoning)
+	}
+	fmt.Fprintln(&b)
+	fmt.Fprintf(&b, "**Substantial nudges (|Δ| ≥ 0.05): %d / %d signals.**\n",
+		substantial, len(rows))
+	return b.String()
+}
+
+// --- math helpers ---------------------------------------------------
+
+func clamp(v, lo, hi float64) float64 {
+	if v < lo {
+		return lo
+	}
+	if v > hi {
+		return hi
+	}
+	return v
+}
+
+func absVal(v float64) float64 {
+	if v < 0 {
+		return -v
+	}
+	return v
+}

internal/eval/quality/tuning_test.go

@@ -0,0 +1,119 @@
+package quality
+
+import (
+	"strings"
+	"testing"
+)
+
+func TestSuggestWeights_UsefulPushesUp(t *testing.T) {
+	in := []SignalFeedback{
+		{Signal: "bm25", Weight: 1.0, UsefulHits: 80, MissedHits: 20},
+	}
+	out := SuggestWeights(in, 1.0)
+	if len(out) != 1 {
+		t.Fatalf("got %d rows, want 1", len(out))
+	}
+	if out[0].SuggestedWeight <= out[0].Weight {
+		t.Errorf("useful > missed should push weight UP; got %.3f → %.3f", out[0].Weight, out[0].SuggestedWeight)
+	}
+}
+
+func TestSuggestWeights_MissedPushesDown(t *testing.T) {
+	in := []SignalFeedback{
+		{Signal: "churn", Weight: 0.5, UsefulHits: 5, MissedHits: 50},
+	}
+	out := SuggestWeights(in, 1.0)
+	if out[0].SuggestedWeight >= out[0].Weight {
+		t.Errorf("missed > useful should push weight DOWN; got %.3f → %.3f", out[0].Weight, out[0].SuggestedWeight)
+	}
+}
+
+func TestSuggestWeights_NoDataKeepsWeight(t *testing.T) {
+	in := []SignalFeedback{
+		{Signal: "minhash", Weight: 0.3, UsefulHits: 0, MissedHits: 0},
+	}
+	out := SuggestWeights(in, 1.0)
+	if out[0].SuggestedWeight != out[0].Weight {
+		t.Errorf("no-data should keep weight unchanged; got %.3f → %.3f", out[0].Weight, out[0].SuggestedWeight)
+	}
+	if !strings.Contains(out[0].Reasoning, "insufficient data") {
+		t.Errorf("no-data reasoning should mention insufficient data; got %q", out[0].Reasoning)
+	}
+}
+
+func TestSuggestWeights_TiePicksNoChange(t *testing.T) {
+	in := []SignalFeedback{
+		{Signal: "fan_in", Weight: 0.6, UsefulHits: 10, MissedHits: 10},
+	}
+	out := SuggestWeights(in, 1.0)
+	if out[0].SuggestedWeight != out[0].Weight {
+		t.Errorf("tie should keep weight; got %.3f → %.3f", out[0].Weight, out[0].SuggestedWeight)
+	}
+}
+
+func TestSuggestWeights_NudgeCapped(t *testing.T) {
+	// Even with absurdly high useful counts, the nudge per call is
+	// capped at 0.1 so a single run can't wildly swing weights.
+	in := []SignalFeedback{
+		{Signal: "fan_in", Weight: 0.5, UsefulHits: 1_000_000, MissedHits: 0},
+	}
+	out := SuggestWeights(in, 1.0)
+	delta := out[0].SuggestedWeight - out[0].Weight
+	if delta > 0.101 {
+		t.Errorf("nudge should be capped at 0.1, got %+.3f", delta)
+	}
+}
+
+func TestSuggestWeights_DownNudgeCanNotGoNegative(t *testing.T) {
+	in := []SignalFeedback{
+		{Signal: "rare", Weight: 0.05, UsefulHits: 0, MissedHits: 1_000_000},
+	}
+	out := SuggestWeights(in, 1.0)
+	if out[0].SuggestedWeight < 0 {
+		t.Errorf("suggested weight should never go negative, got %.3f", out[0].SuggestedWeight)
+	}
+}
+
+func TestSuggestWeights_SortedByAbsoluteDelta(t *testing.T) {
+	in := []SignalFeedback{
+		{Signal: "small_change", Weight: 1.0, UsefulHits: 5, MissedHits: 4},
+		{Signal: "big_change", Weight: 1.0, UsefulHits: 90, MissedHits: 10},
+		{Signal: "no_change", Weight: 1.0, UsefulHits: 0, MissedHits: 0},
+	}
+	out := SuggestWeights(in, 1.0)
+	if out[0].Signal != "big_change" {
+		t.Errorf("first row should be the biggest change; got %s", out[0].Signal)
+	}
+}
+
+func TestRenderTuningMarkdown_HasHeaderAndRows(t *testing.T) {
+	rows := []SignalFeedback{
+		{Signal: "bm25", Weight: 1.0, UsefulHits: 80, MissedHits: 20, SuggestedWeight: 1.06, Reasoning: "useful > missed"},
+	}
+	md := RenderTuningMarkdown(rows)
+	for _, want := range []string{
+		"# Rerank weight-tuning suggestion",
+		"| bm25 |",
+		"1.000",
+		"1.060",
+		"useful > missed",
+		"**Substantial nudges",
+	} {
+		if !strings.Contains(md, want) {
+			t.Errorf("markdown missing %q\n----\n%s", want, md)
+		}
+	}
+}
+
+func TestRenderTuningMarkdown_SubstantialCount(t *testing.T) {
+	rows := []SignalFeedback{
+		{Signal: "tiny",    Weight: 1.0, SuggestedWeight: 1.01}, // not substantial
+		{Signal: "big1",    Weight: 1.0, SuggestedWeight: 1.10}, // substantial
+		{Signal: "big2",    Weight: 1.0, SuggestedWeight: 0.90}, // substantial (negative)
+		{Signal: "nothing", Weight: 1.0, SuggestedWeight: 1.0},  // not substantial
+	}
+	md := RenderTuningMarkdown(rows)
+	if !strings.Contains(md, "Substantial nudges (|Δ| ≥ 0.05): 2 / 4") {
+		t.Errorf("substantial count wrong:\n%s", md)
+	}
+}

internal/eval/recall/judge.go

@@ -0,0 +1,251 @@
+package recall
+
+import (
+	"bytes"
+	"crypto/sha256"
+	"encoding/hex"
+	"encoding/json"
+	"fmt"
+	"io"
+	"net/http"
+	"os"
+	"path/filepath"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/zzet/gortex/internal/platform"
+)
+
+// Judge rescues recall misses by asking a cheap LLM whether any symbol
+// in the ranker's top-K plausibly answers the query, modelling CQS-
+// style dual-judge evaluation. Verdicts are cached on disk per
+// (query, top-K set) so re-runs don't re-pay.
+type Judge struct {
+	Model      string // Anthropic model ID, e.g. "claude-haiku-4-5"
+	APIKey     string // from ANTHROPIC_API_KEY
+	CachePath  string // file path for verdict cache
+	HTTPClient *http.Client
+
+	cacheOnce sync.Once
+	cache     map[string]bool
+	cacheMu   sync.Mutex
+}
+
+// NewJudge constructs a Judge. Returns nil if api key is missing —
+// caller treats nil as "judging disabled."
+func NewJudge(model string) *Judge {
+	key := os.Getenv("ANTHROPIC_API_KEY")
+	if key == "" || model == "" {
+		return nil
+	}
+	// $XDG_CACHE_HOME is honoured; otherwise the cache stays under
+	// os.UserCacheDir() as before.
+	cachePath := filepath.Join(platform.OSCacheDir(), "eval_judge_cache.json")
+	return &Judge{
+		Model:      model,
+		APIKey:     key,
+		CachePath:  cachePath,
+		HTTPClient: &http.Client{Timeout: 30 * time.Second},
+	}
+}
+
+// Verdict asks whether any ID in top plausibly answers query. Returns
+// true/false plus the string verdict. Cached by (model, query, sorted
+// top list).
+func (j *Judge) Verdict(query string, top []string) (bool, error) {
+	j.loadCache()
+	key := verdictKey(j.Model, query, top)
+
+	j.cacheMu.Lock()
+	if v, ok := j.cache[key]; ok {
+		j.cacheMu.Unlock()
+		return v, nil
+	}
+	j.cacheMu.Unlock()
+
+	verdict, err := j.ask(query, top)
+	if err != nil {
+		return false, err
+	}
+
+	j.cacheMu.Lock()
+	j.cache[key] = verdict
+	j.cacheMu.Unlock()
+	_ = j.saveCache()
+	return verdict, nil
+}
+
+// ask POSTs to Anthropic /v1/messages with a YES/NO prompt.
+func (j *Judge) ask(query string, top []string) (bool, error) {
+	if len(top) == 0 {
+		return false, nil
+	}
+	var b strings.Builder
+	fmt.Fprintf(&b, "You are judging whether a code-search result list contains a valid answer to a natural-language query about a codebase.\n\n")
+	fmt.Fprintf(&b, "QUERY: %s\n\n", query)
+	fmt.Fprintf(&b, "TOP RESULTS (ranked, ID format `relative/path.go::SymbolName`):\n")
+	for i, id := range top {
+		fmt.Fprintf(&b, "%d. %s\n", i+1, id)
+	}
+	fmt.Fprintf(&b, "\nIf ANY listed symbol is a reasonable answer to the query — i.e. a programmer asking this query would accept it as the right thing to look at — reply with the single word YES. Otherwise reply with the single word NO. No preamble, no punctuation.\n")
+
+	payload := map[string]any{
+		"model":      j.Model,
+		"max_tokens": 4,
+		"messages": []map[string]any{
+			{"role": "user", "content": b.String()},
+		},
+	}
+	body, _ := json.Marshal(payload)
+	req, err := http.NewRequest("POST", "https://api.anthropic.com/v1/messages", bytes.NewReader(body))
+	if err != nil {
+		return false, err
+	}
+	req.Header.Set("x-api-key", j.APIKey)
+	req.Header.Set("anthropic-version", "2023-06-01")
+	req.Header.Set("content-type", "application/json")
+
+	resp, err := j.HTTPClient.Do(req)
+	if err != nil {
+		return false, err
+	}
+	defer resp.Body.Close()
+	respBody, _ := io.ReadAll(resp.Body)
+	if resp.StatusCode >= 300 {
+		return false, fmt.Errorf("judge http %d: %s", resp.StatusCode, string(respBody))
+	}
+
+	var parsed struct {
+		Content []struct {
+			Type string `json:"type"`
+			Text string `json:"text"`
+		} `json:"content"`
+	}
+	if err := json.Unmarshal(respBody, &parsed); err != nil {
+		return false, err
+	}
+	var text string
+	for _, c := range parsed.Content {
+		if c.Type == "text" {
+			text += c.Text
+		}
+	}
+	text = strings.TrimSpace(strings.ToUpper(text))
+	return strings.HasPrefix(text, "YES"), nil
+}
+
+func verdictKey(model, query string, top []string) string {
+	sorted := append([]string(nil), top...)
+	// Order-independent to let RRF / BM25 share cache entries when
+	// they return the same set in different orders. Comment on the
+	// tradeoff: if order materially changes the verdict, flip this to
+	// preserve order. For gortex this is benign.
+	for i := range sorted {
+		for j := i + 1; j < len(sorted); j++ {
+			if sorted[i] > sorted[j] {
+				sorted[i], sorted[j] = sorted[j], sorted[i]
+			}
+		}
+	}
+	// hash.Hash.Write is documented to never return an error, so we
+	// feed it through Write directly instead of fmt.Fprintln to keep
+	// errcheck quiet without _, _ = noise.
+	h := sha256.New()
+	h.Write([]byte(model + "\n"))
+	h.Write([]byte(query + "\n"))
+	for _, id := range sorted {
+		h.Write([]byte(id + "\n"))
+	}
+	return hex.EncodeToString(h.Sum(nil))
+}
+
+// loadCache reads the on-disk cache once per Judge lifetime.
+func (j *Judge) loadCache() {
+	j.cacheOnce.Do(func() {
+		j.cache = make(map[string]bool)
+		data, err := os.ReadFile(j.CachePath)
+		if err != nil {
+			return
+		}
+		_ = json.Unmarshal(data, &j.cache)
+	})
+}
+
+// saveCache writes atomically via temp+rename.
+func (j *Judge) saveCache() error {
+	j.cacheMu.Lock()
+	data, err := json.MarshalIndent(j.cache, "", "  ")
+	j.cacheMu.Unlock()
+	if err != nil {
+		return err
+	}
+	if err := os.MkdirAll(filepath.Dir(j.CachePath), 0o755); err != nil {
+		return err
+	}
+	tmp := j.CachePath + ".tmp"
+	if err := os.WriteFile(tmp, data, 0o644); err != nil {
+		return err
+	}
+	return os.Rename(tmp, j.CachePath)
+}
+
+// ApplyJudge rescues misses in the report: for each RankerResult's
+// miss list, asks the judge if any top-K entry is a valid answer;
+// when yes, bumps Hits[K]+1 for every k where the top-K was within
+// bounds (judge operates on the biggest K only — that's the Miss.Top
+// length). Returns (rescued count, judge API errors).
+func ApplyJudge(report *Report, judge *Judge) (int, []error) {
+	if judge == nil {
+		return 0, nil
+	}
+	maxK := 0
+	for _, k := range Ks {
+		if k > maxK {
+			maxK = k
+		}
+	}
+
+	var rescued int
+	var errs []error
+	for i := range report.Rankers {
+		r := &report.Rankers[i]
+		if r.Skipped != "" {
+			continue
+		}
+		for mi := range r.Misses {
+			m := &r.Misses[mi]
+			// Judge the biggest-K slice the ranker actually returned.
+			top := m.Top
+			if len(top) > maxK {
+				top = top[:maxK]
+			}
+			ok, err := judge.Verdict(m.Query, top)
+			if err != nil {
+				errs = append(errs, fmt.Errorf("judge %s/%s: %w", r.Name, m.CaseID, err))
+				continue
+			}
+			m.JudgedHit = &ok
+			if !ok {
+				continue
+			}
+			rescued++
+			// A judged hit counts at every k where the top slice was
+			// non-empty — we don't know the judged rank, so attribute
+			// it to the max-K bucket only. This is the safer read
+			// vs CQS's R@1 claims; note it in the methodology section.
+			for _, k := range Ks {
+				if k >= maxK {
+					r.Hits[k]++
+				}
+			}
+		}
+		// Recompute recall after rescues.
+		if r.Cases > 0 {
+			for _, k := range Ks {
+				r.Recall[k] = float64(r.Hits[k]) / float64(r.Cases)
+			}
+		}
+	}
+	return rescued, errs
+}

internal/eval/recall/judge_test.go

@@ -0,0 +1,137 @@
+package recall
+
+import (
+	"encoding/json"
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestVerdictKey_OrderIndependent(t *testing.T) {
+	k1 := verdictKey("claude", "find X", []string{"a/b.go::Foo", "c/d.go::Bar"})
+	k2 := verdictKey("claude", "find X", []string{"c/d.go::Bar", "a/b.go::Foo"})
+	assert.Equal(t, k1, k2, "verdict key must be independent of top-K order")
+}
+
+func TestVerdictKey_ChangesWithModel(t *testing.T) {
+	k1 := verdictKey("claude-haiku-4-5", "q", []string{"a"})
+	k2 := verdictKey("claude-opus-4", "q", []string{"a"})
+	assert.NotEqual(t, k1, k2, "verdict key must differ when the model differs")
+}
+
+func TestJudgeCacheRoundTrip(t *testing.T) {
+	tmp := t.TempDir()
+	path := filepath.Join(tmp, "judge_cache.json")
+	j := &Judge{
+		Model:     "claude-haiku-4-5",
+		APIKey:    "test-key",
+		CachePath: path,
+	}
+	j.loadCache()
+	j.cacheMu.Lock()
+	j.cache["abc"] = true
+	j.cache["def"] = false
+	j.cacheMu.Unlock()
+	require.NoError(t, j.saveCache())
+
+	// Reload from disk.
+	j2 := &Judge{CachePath: path}
+	j2.loadCache()
+	assert.Equal(t, true, j2.cache["abc"])
+	assert.Equal(t, false, j2.cache["def"])
+
+	// Sanity: on-disk format is valid JSON.
+	raw, err := os.ReadFile(path)
+	require.NoError(t, err)
+	var decoded map[string]bool
+	require.NoError(t, json.Unmarshal(raw, &decoded))
+	assert.Len(t, decoded, 2)
+}
+
+func TestNewJudge_NoAPIKeyReturnsNil(t *testing.T) {
+	t.Setenv("ANTHROPIC_API_KEY", "")
+	assert.Nil(t, NewJudge("claude-haiku-4-5"))
+}
+
+func TestNewJudge_EmptyModelReturnsNil(t *testing.T) {
+	t.Setenv("ANTHROPIC_API_KEY", "fake")
+	assert.Nil(t, NewJudge(""))
+}
+
+func TestApplyJudge_NilIsNoop(t *testing.T) {
+	r := Report{Rankers: []RankerResult{
+		{Name: "x", Hits: map[int]int{20: 5}, Recall: map[int]float64{20: 0.5}, Cases: 10},
+	}}
+	before := r.Rankers[0].Hits[20]
+	rescued, errs := ApplyJudge(&r, nil)
+	assert.Equal(t, 0, rescued)
+	assert.Nil(t, errs)
+	assert.Equal(t, before, r.Rankers[0].Hits[20])
+}
+
+func TestRun_CapturesMisses(t *testing.T) {
+	fixture := Fixture{
+		Name: "t",
+		Cases: []Case{
+			{ID: "hit", Query: "q1", Tier: TierExact, Expected: []string{"a"}},
+			{ID: "miss", Query: "q2", Tier: TierExact, Expected: []string{"z"}},
+		},
+	}
+	r := staticRanker("x", []string{"a", "b", "c"})
+	report := Run(fixture, []Ranker{r}, nil)
+	require.Len(t, report.Rankers, 1)
+	assert.Len(t, report.Rankers[0].Misses, 1)
+	miss := report.Rankers[0].Misses[0]
+	assert.Equal(t, "miss", miss.CaseID)
+	assert.Equal(t, []string{"z"}, miss.Expected)
+	assert.Equal(t, []string{"a", "b", "c"}, miss.Top)
+	assert.Nil(t, miss.JudgedHit)
+}
+
+func TestApplyJudge_RescuesMissWithStubbedVerdict(t *testing.T) {
+	// Shape-level test: we can't hit the real API, so pre-seed the
+	// judge cache with a "yes" verdict for a known query+top.
+	tmp := t.TempDir()
+	cachePath := filepath.Join(tmp, "cache.json")
+	query := "find me a writer"
+	top := []string{"pkg/x.go::WriteThing"}
+
+	judge := &Judge{
+		Model:     "claude-stub",
+		APIKey:    "stub",
+		CachePath: cachePath,
+	}
+	judge.loadCache()
+	judge.cacheMu.Lock()
+	judge.cache[verdictKey("claude-stub", query, top)] = true
+	judge.cacheMu.Unlock()
+
+	report := Report{
+		Rankers: []RankerResult{{
+			Name:   "r",
+			Cases:  1,
+			Hits:   map[int]int{1: 0, 5: 0, 20: 0},
+			Recall: map[int]float64{1: 0, 5: 0, 20: 0},
+			Misses: []Miss{{
+				CaseID:   "c1",
+				Query:    query,
+				Expected: []string{"pkg/x.go::OtherWriter"},
+				Top:      top,
+			}},
+		}},
+	}
+	rescued, errs := ApplyJudge(&report, judge)
+	assert.Empty(t, errs)
+	assert.Equal(t, 1, rescued)
+	// Judged hits bump the biggest-K bucket (20).
+	assert.Equal(t, 1, report.Rankers[0].Hits[20])
+	assert.Equal(t, 1.0, report.Rankers[0].Recall[20])
+	// R@1 / R@5 are NOT credited by the judge — we don't know the
+	// judged rank, so the conservative call is max-K only.
+	assert.Equal(t, 0, report.Rankers[0].Hits[1])
+	require.NotNil(t, report.Rankers[0].Misses[0].JudgedHit)
+	assert.True(t, *report.Rankers[0].Misses[0].JudgedHit)
+}

internal/eval/recall/rankers.go

@@ -0,0 +1,236 @@
+package recall
+
+import (
+	"bufio"
+	"bytes"
+	"context"
+	"os/exec"
+	"path/filepath"
+	"sort"
+	"strings"
+	"time"
+
+	"github.com/zzet/gortex/internal/embedding"
+	"github.com/zzet/gortex/internal/search"
+)
+
+// BM25Ranker adapts a plain search.Backend to the Ranker shape. Works
+// for either a raw BM25/Bleve backend or a HybridBackend's text side
+// extracted via HybridBackend.TextBackend().
+//
+// Note: Gortex's indexer tokenizes symbol names at ingest time
+// (Tokenize — camelCase-aware), but the query side (TokenizeQuery) does
+// NOT split camelCase — so `backend.Search("NewServer", ...)` matches
+// zero documents because the inverted index has `new` and `server`
+// separately. The user-facing MCP search_symbols path avoids this by
+// running through Engine.SearchSymbols, which adds a substring
+// fallback. Use EngineRanker (below) to measure that full call path;
+// use BM25Ranker only when you want the raw backend's behaviour.
+func BM25Ranker(name string, backend search.Backend) Ranker {
+	return Ranker{
+		Name: name,
+		Search: func(query string, limit int) []string {
+			hits := backend.Search(query, limit)
+			out := make([]string, len(hits))
+			for i, h := range hits {
+				out[i] = h.ID
+			}
+			return out
+		},
+	}
+}
+
+// EngineRanker measures what a real MCP caller sees via
+// Engine.SearchSymbols — BM25/Bleve results + camelCase-friendly
+// substring fallback. This is the recommended default for "bm25"-
+// style evaluation; it reflects production behaviour.
+func EngineRanker(name string, searchFn func(query string, limit int) []string) Ranker {
+	return Ranker{Name: name, Search: searchFn}
+}
+
+// GraphRanker wraps graph-traversal queries (get_callers, get_call_chain,
+// find_usages) as Rankers so DI-dependent recall can be measured next to
+// plain text retrieval. The fixture encodes the retrieval shape in the
+// Case.Query field as "<tool>:<symbol_id>" — the Ranker parses the prefix
+// and dispatches to the appropriate engine method. Unknown prefixes and
+// unparseable queries return empty, which the scorer counts as a miss.
+//
+// Shapes supported (tool names match the equivalent MCP surface):
+//
+//	callers:<id>     → Engine.GetCallers (reverse EdgeCalls/EdgeMatches)
+//	call_chain:<id>  → Engine.GetCallChain (forward EdgeCalls/EdgeMatches)
+//	usages:<id>      → Engine.FindUsages (all references)
+//	dependents:<id>  → Engine.GetDependents (imports+calls+references, reverse)
+func GraphRanker(name string, provider GraphProvider) Ranker {
+	return Ranker{
+		Name: name,
+		Search: func(q string, limit int) []string {
+			idx := strings.IndexByte(q, ':')
+			if idx <= 0 || idx == len(q)-1 {
+				return nil
+			}
+			tool, id := q[:idx], q[idx+1:]
+			return provider.Traverse(tool, id, limit)
+		},
+	}
+}
+
+// GraphProvider is the thin interface GraphRanker needs. Implemented in
+// cmd/gortex (where the query engine is already wired) to keep this
+// package free of engine imports — keeps recall reusable from CLI eval,
+// unit tests, or any other caller that can provide the traversal.
+type GraphProvider interface {
+	Traverse(tool, id string, limit int) []string
+}
+
+// SemanticRanker adapts a vector backend + embedder to the Ranker
+// shape. Returns ranker-level skip (empty slice + registered skip
+// reason) when the vector backend is empty — callers can still emit
+// a stable row for the report.
+func SemanticRanker(name string, vector *search.VectorBackend, embedder embedding.Provider) Ranker {
+	return Ranker{
+		Name: name,
+		Search: func(query string, limit int) []string {
+			if vector == nil || embedder == nil || vector.Count() == 0 {
+				return nil
+			}
+			ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+			defer cancel()
+			vec, err := embedder.Embed(ctx, query)
+			if err != nil || vec == nil {
+				return nil
+			}
+			return vector.Search(vec, limit)
+		},
+	}
+}
+
+// RRFRanker adapts a HybridBackend (BM25 + vector fused via RRF) to the
+// Ranker shape. Uses Search() which runs both sides and fuses when the
+// vector backend has data — otherwise it degrades to BM25 gracefully.
+func RRFRanker(name string, hybrid *search.HybridBackend) Ranker {
+	return Ranker{
+		Name: name,
+		Search: func(query string, limit int) []string {
+			hits := hybrid.Search(query, limit)
+			out := make([]string, len(hits))
+			for i, h := range hits {
+				out[i] = h.ID
+			}
+			return out
+		},
+	}
+}
+
+// WinnowProvider is the minimal surface the WinnowRanker needs from the
+// caller — an opaque hook that wraps the MCP server's WinnowForEval so
+// we don't import internal/mcp from this retrieval-only package.
+type WinnowProvider func(query string, extras map[string]any, limit int) []string
+
+// WinnowRanker adapts a winnow provider to the Ranker shape. The
+// provider runs the MCP server's graph-aware constraint scorer; per-
+// case constraints flow through via Case.WinnowConstraints.
+func WinnowRanker(name string, provide WinnowProvider, caseExtras func(query string) map[string]any) Ranker {
+	return Ranker{
+		Name: name,
+		Search: func(query string, limit int) []string {
+			var extras map[string]any
+			if caseExtras != nil {
+				extras = caseExtras(query)
+			}
+			return provide(query, extras, limit)
+		},
+	}
+}
+
+// RipgrepRanker shells out to `rg --files-with-matches` for each query
+// and returns file paths as a ranked list. For any-hit recall we treat
+// a hit as correct at rank K when any of the top-K file paths matches
+// the file component of any Expected symbol ID.
+//
+// Gracefully skips if rg isn't on PATH — the Ranker will return an
+// empty slice and the report surfaces it via the skip channel.
+func RipgrepRanker(name, root string) Ranker {
+	if _, err := exec.LookPath("rg"); err != nil {
+		return Ranker{
+			Name: name,
+			Search: func(_ string, _ int) []string {
+				return nil
+			},
+		}
+	}
+	// Map rg file paths into the graph's symbol-ID path prefix so
+	// any-hit comparison works: an ID like "internal/foo.go::Bar"
+	// matches a file hit like "internal/foo.go".
+	return Ranker{
+		Name: name,
+		Search: func(query string, limit int) []string {
+			ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+			defer cancel()
+			cmd := exec.CommandContext(ctx, "rg",
+				"--files-with-matches",
+				"--no-messages",
+				"--max-count", "1",
+				"--", query, root)
+			var outBuf bytes.Buffer
+			cmd.Stdout = &outBuf
+			cmd.Stderr = nil
+			_ = cmd.Run() // non-zero exit means "no matches" — treat as empty
+			files := make([]string, 0, 32)
+			sc := bufio.NewScanner(&outBuf)
+			for sc.Scan() {
+				p := sc.Text()
+				if p == "" {
+					continue
+				}
+				rel, err := filepath.Rel(root, p)
+				if err == nil {
+					p = rel
+				}
+				files = append(files, filepath.ToSlash(p))
+				if len(files) >= limit {
+					break
+				}
+			}
+			return files
+		},
+	}
+}
+
+// RipgrepAnyHit adapts a Ranker whose results are file paths (as
+// produced by RipgrepRanker) to the any-hit recall model by checking
+// whether the file prefix of any expected symbol ID matches any of
+// the top-K ranked files. Returns a new Ranker whose Search still
+// returns the raw file list — matching is done by rewriting the
+// fixture Expected set before Run. Keep this as a helper so Run's
+// code path stays uniform.
+//
+// Callers typically invoke AdaptCasesForFileRanker on their fixture
+// before running the rg ranker.
+func AdaptCasesForFileRanker(cases []Case) []Case {
+	out := make([]Case, len(cases))
+	for i, c := range cases {
+		newExpected := make([]string, 0, len(c.Expected))
+		seen := map[string]bool{}
+		for _, id := range c.Expected {
+			// Symbol IDs have the form "<file-path>::<symbol>". Split
+			// on "::" and keep the file path; if there's no "::" the
+			// ID is already a file path (or something we can't adapt).
+			f := id
+			if idx := strings.Index(id, "::"); idx >= 0 {
+				f = id[:idx]
+			}
+			if !seen[f] {
+				newExpected = append(newExpected, f)
+				seen[f] = true
+			}
+		}
+		out[i] = c
+		out[i].Expected = newExpected
+	}
+	// Sort each expected list deterministically so test diffs are stable.
+	for i := range out {
+		sort.Strings(out[i].Expected)
+	}
+	return out
+}

internal/eval/recall/recall.go

@@ -0,0 +1,381 @@
+// Package recall computes retrieval recall@K, latency, and token-return
+// metrics for the Gortex retrieval stack against a curated fixture of
+// {query, expected_ids} pairs.
+//
+// # Methodology
+//
+// Recall is reported as any-hit set-level recall: a retrieval counts as
+// correct at rank K if *any* of the Expected IDs for a case appears in
+// the ranker's top-K results. Multiple Expected IDs per case are OK —
+// they represent valid alternative targets (e.g. a type and its
+// constructor both being reasonable answers to "BM25 backend").
+//
+// Cases are tiered so per-tier weakness is visible:
+//
+//   - exact:     symbol-name queries. Tests the basic "can you find a
+//     named symbol I already know about" case. BM25 should
+//     dominate here; a retrieval tool that can't ace exact
+//     tier is broken.
+//   - concept:   natural-language paraphrase queries. Tests semantic
+//     understanding. This is where BM25 starts losing to
+//     semantic / RRF.
+//   - multi_hop: relational queries accepting several valid expected
+//     IDs (any-hit). Tests graph-aware retrieval.
+//
+// # Gold vs judged
+//
+// All fixture labels in this package are hand-curated gold labels, not
+// LLM-judged. This is a stricter measurement than a dual-judge setup
+// (CQS style) — the numbers you get here are lower than what a
+// per-query LLM judge would award, because paraphrased-but-still-correct
+// ranker output is scored as a miss unless it matches the gold ID.
+//
+// Used by the `gortex eval recall` CLI verb (roadmap I6).
+package recall
+
+import (
+	"fmt"
+	"sort"
+	"strings"
+	"time"
+)
+
+// Tier classifies a case by query difficulty.
+type Tier string
+
+const (
+	TierExact    Tier = "exact"
+	TierConcept  Tier = "concept"
+	TierMultiHop Tier = "multi_hop"
+)
+
+// Ranker names a retrieval strategy used in the output report.
+type Ranker struct {
+	Name string
+	// Search returns a ranked list of symbol IDs for the query.
+	Search func(query string, limit int) []string
+}
+
+// Case is a single fixture entry.
+type Case struct {
+	ID       string   `yaml:"id" json:"id,omitempty"`
+	Tier     Tier     `yaml:"tier" json:"tier"`
+	Query    string   `yaml:"query" json:"query"`
+	Expected []string `yaml:"expected" json:"expected"`
+	// WinnowConstraints is passed through to winnow-style rankers so a
+	// case can pre-filter the candidate pool (e.g. {"language": "go"}).
+	WinnowConstraints map[string]any `yaml:"winnow_constraints,omitempty" json:"winnow_constraints,omitempty"`
+}
+
+// Fixture is a collection of cases plus metadata.
+type Fixture struct {
+	Name  string `yaml:"name" json:"name"`
+	Cases []Case `yaml:"cases" json:"cases"`
+}
+
+// Ks lists the rank cutoffs the report computes.
+var Ks = []int{1, 5, 20}
+
+// TokenCounter returns the tokenized length of an arbitrary string.
+// Typically wired to tokens.Count (tiktoken cl100k_base) so the number
+// agents actually pay is reported.
+type TokenCounter func(string) int
+
+// defaultTokenCounter falls back to a rough chars/4 estimate when no
+// real counter is wired.
+func defaultTokenCounter(s string) int { return (len(s) + 3) / 4 }
+
+// RankerResult holds aggregated metrics for a single ranker over a
+// fixture run. All maps are keyed by k or tier.
+type RankerResult struct {
+	Name       string               `json:"name"`
+	Cases      int                  `json:"cases"`
+	Hits       map[int]int          `json:"hits"`
+	Recall     map[int]float64      `json:"recall"`
+	MeanRRank  float64              `json:"mean_reciprocal_rank"`
+	PerTier    map[Tier]TierMetrics `json:"per_tier,omitempty"`
+	P50Micros  int64                `json:"p50_micros"`
+	P95Micros  int64                `json:"p95_micros"`
+	MaxMicros  int64                `json:"max_micros"`
+	MeanTokens float64              `json:"mean_tokens_returned"`
+	// Misses records cases where none of Expected appeared in top-K
+	// at the biggest K in Ks. Populated for diagnostics / judge mode.
+	Misses []Miss `json:"misses,omitempty"`
+	// Note: skipped is true when the ranker was registered but had no
+	// usable backend (e.g. semantic ranker with no embedder). Zero-
+	// filled results help keep the report table shape stable.
+	Skipped string `json:"skipped,omitempty"`
+}
+
+// Miss is a single per-case diagnostic row for when a ranker didn't
+// surface any expected ID in top-K. Rank holds the 1-based position of
+// the first-hit expected ID, or 0 when no expected ID appeared at all
+// — lets callers distinguish "found at position 3" from "never found".
+type Miss struct {
+	CaseID   string   `json:"case_id"`
+	Query    string   `json:"query"`
+	Expected []string `json:"expected"`
+	Top      []string `json:"top"` // top-K IDs the ranker returned
+	Rank     int      `json:"rank,omitempty"`
+	// JudgedHit is populated by a post-hoc LLM-judge pass: true when
+	// the judge accepts any entry in Top as answering the Query. Nil
+	// pointer means "not judged."
+	JudgedHit *bool `json:"judged_hit,omitempty"`
+}
+
+// TierMetrics is a per-tier recall slice.
+type TierMetrics struct {
+	Cases  int             `json:"cases"`
+	Hits   map[int]int     `json:"hits"`
+	Recall map[int]float64 `json:"recall"`
+}
+
+// Report bundles the results of a fixture run across every ranker.
+type Report struct {
+	Fixture   string         `json:"fixture"`
+	Cases     int            `json:"cases"`
+	PerTier   map[Tier]int   `json:"per_tier_cases"`
+	Rankers   []RankerResult `json:"rankers"`
+	GortexRev string         `json:"gortex_rev,omitempty"`
+}
+
+// Run evaluates every ranker against every case in the fixture and
+// returns an aggregated report. Limits retrieval to the largest K in Ks.
+func Run(fixture Fixture, rankers []Ranker, tc TokenCounter) Report {
+	if tc == nil {
+		tc = defaultTokenCounter
+	}
+
+	maxK := 0
+	for _, k := range Ks {
+		if k > maxK {
+			maxK = k
+		}
+	}
+
+	report := Report{
+		Fixture: fixture.Name,
+		Cases:   len(fixture.Cases),
+		PerTier: make(map[Tier]int),
+	}
+	for _, c := range fixture.Cases {
+		report.PerTier[c.Tier]++
+	}
+
+	for _, r := range rankers {
+		report.Rankers = append(report.Rankers, evalOne(fixture, r, tc, maxK))
+	}
+	return report
+}
+
+// evalOne runs a single ranker across the fixture and aggregates.
+func evalOne(fixture Fixture, r Ranker, tc TokenCounter, maxK int) RankerResult {
+	res := RankerResult{
+		Name:    r.Name,
+		Cases:   len(fixture.Cases),
+		Hits:    make(map[int]int, len(Ks)),
+		Recall:  make(map[int]float64, len(Ks)),
+		PerTier: make(map[Tier]TierMetrics),
+	}
+	// Initialise per-tier buckets so the JSON shape is stable. The stock
+	// tiers are always included for schema consistency; any additional
+	// tier seen in the fixture (e.g. "di") is added on the fly so the
+	// harness doesn't panic when fixtures grow new categories.
+	for _, tier := range []Tier{TierExact, TierConcept, TierMultiHop} {
+		res.PerTier[tier] = TierMetrics{
+			Hits:   make(map[int]int, len(Ks)),
+			Recall: make(map[int]float64, len(Ks)),
+		}
+	}
+	for _, c := range fixture.Cases {
+		if _, ok := res.PerTier[c.Tier]; c.Tier != "" && !ok {
+			res.PerTier[c.Tier] = TierMetrics{
+				Hits:   make(map[int]int, len(Ks)),
+				Recall: make(map[int]float64, len(Ks)),
+			}
+		}
+	}
+
+	var rrSum, tokensSum float64
+	latencies := make([]time.Duration, 0, len(fixture.Cases))
+
+	for _, c := range fixture.Cases {
+		expected := stringSet(c.Expected)
+
+		start := time.Now()
+		ranked := r.Search(c.Query, maxK)
+		elapsed := time.Since(start)
+		latencies = append(latencies, elapsed)
+
+		// Token-return metric: tokenize the ranked ID list as a proxy
+		// for "how many tokens does this ranker's top-K cost to return?"
+		// Gives a rough Pareto knob (recall vs cost).
+		var payload string
+		for _, id := range ranked {
+			payload += id + "\n"
+		}
+		tokensSum += float64(tc(payload))
+
+		firstHit := -1
+		for i, id := range ranked {
+			if expected[id] {
+				firstHit = i
+				break
+			}
+		}
+		for _, k := range Ks {
+			if firstHit >= 0 && firstHit < k {
+				res.Hits[k]++
+			}
+		}
+		if firstHit >= 0 {
+			rrSum += 1.0 / float64(firstHit+1)
+		}
+		// Capture a row when the case missed outright OR landed outside
+		// the top slot. Rank=0 means "no expected ID in top-K at all"
+		// (the hard miss we've always tracked); Rank>=2 means "found
+		// but not at rank 1" — surfaces ordering bugs the aggregate
+		// R@1 number hides.
+		if firstHit < 0 || firstHit > 0 {
+			topCopy := append([]string(nil), ranked...)
+			expCopy := append([]string(nil), c.Expected...)
+			rank := 0
+			if firstHit >= 0 {
+				rank = firstHit + 1
+			}
+			res.Misses = append(res.Misses, Miss{
+				CaseID:   c.ID,
+				Query:    c.Query,
+				Expected: expCopy,
+				Top:      topCopy,
+				Rank:     rank,
+			})
+		}
+
+		// Per-tier bookkeeping.
+		tier := c.Tier
+		if tier == "" {
+			tier = TierExact
+		}
+		tm := res.PerTier[tier]
+		tm.Cases++
+		for _, k := range Ks {
+			if firstHit >= 0 && firstHit < k {
+				tm.Hits[k]++
+			}
+		}
+		res.PerTier[tier] = tm
+	}
+
+	if res.Cases > 0 {
+		for _, k := range Ks {
+			res.Recall[k] = float64(res.Hits[k]) / float64(res.Cases)
+		}
+		res.MeanRRank = rrSum / float64(res.Cases)
+		res.MeanTokens = tokensSum / float64(res.Cases)
+	}
+	for tier, tm := range res.PerTier {
+		if tm.Cases > 0 {
+			for _, k := range Ks {
+				tm.Recall[k] = float64(tm.Hits[k]) / float64(tm.Cases)
+			}
+		}
+		res.PerTier[tier] = tm
+	}
+
+	res.P50Micros, res.P95Micros, res.MaxMicros = latencyPercentiles(latencies)
+	return res
+}
+
+// latencyPercentiles sorts a copy and returns p50/p95/max in microseconds.
+func latencyPercentiles(lats []time.Duration) (int64, int64, int64) {
+	if len(lats) == 0 {
+		return 0, 0, 0
+	}
+	sorted := append([]time.Duration(nil), lats...)
+	sort.Slice(sorted, func(i, j int) bool { return sorted[i] < sorted[j] })
+	p := func(pct float64) time.Duration {
+		idx := int(float64(len(sorted)-1) * pct)
+		return sorted[idx]
+	}
+	toMicros := func(d time.Duration) int64 { return d.Microseconds() }
+	return toMicros(p(0.50)), toMicros(p(0.95)), toMicros(sorted[len(sorted)-1])
+}
+
+// stringSet turns a slice into a membership map.
+func stringSet(s []string) map[string]bool {
+	m := make(map[string]bool, len(s))
+	for _, v := range s {
+		m[v] = true
+	}
+	return m
+}
+
+// Markdown renders the report as a Markdown document. Stable ordering so
+// the output is diffable across runs.
+func Markdown(report Report) string {
+	var b strings.Builder
+	b.WriteString("# Gortex retrieval recall\n\n")
+	if report.GortexRev != "" {
+		fmt.Fprintf(&b, "_Fixture: `%s` · rev: `%s` · %d cases_\n\n",
+			report.Fixture, report.GortexRev, report.Cases)
+	} else {
+		fmt.Fprintf(&b, "_Fixture: `%s` · %d cases_\n\n",
+			report.Fixture, report.Cases)
+	}
+	if len(report.PerTier) > 0 {
+		tierKeys := make([]Tier, 0, len(report.PerTier))
+		for t := range report.PerTier {
+			tierKeys = append(tierKeys, t)
+		}
+		sort.Slice(tierKeys, func(i, j int) bool { return tierKeys[i] < tierKeys[j] })
+		b.WriteString("Tier distribution:")
+		for _, t := range tierKeys {
+			fmt.Fprintf(&b, " `%s`=%d", t, report.PerTier[t])
+		}
+		b.WriteString("\n\n")
+	}
+
+	b.WriteString("## Overall\n\n")
+	b.WriteString("| ranker | R@1 | R@5 | R@20 | MRR | p50 µs | p95 µs | tokens/q |\n")
+	b.WriteString("|--------|-----|-----|------|-----|--------|--------|----------|\n")
+
+	rankers := append([]RankerResult(nil), report.Rankers...)
+	sort.Slice(rankers, func(i, j int) bool { return rankers[i].Name < rankers[j].Name })
+	for _, r := range rankers {
+		if r.Skipped != "" {
+			fmt.Fprintf(&b, "| %s | — | — | — | — | — | — | — (skipped: %s) |\n", r.Name, r.Skipped)
+			continue
+		}
+		fmt.Fprintf(&b, "| %s | %s | %s | %s | %.3f | %d | %d | %.0f |\n",
+			r.Name,
+			pct(r.Recall[1]),
+			pct(r.Recall[5]),
+			pct(r.Recall[20]),
+			r.MeanRRank,
+			r.P50Micros,
+			r.P95Micros,
+			r.MeanTokens,
+		)
+	}
+
+	b.WriteString("\n## Per tier (R@5)\n\n")
+	b.WriteString("| ranker | exact | concept | multi_hop |\n")
+	b.WriteString("|--------|-------|---------|-----------|\n")
+	for _, r := range rankers {
+		if r.Skipped != "" {
+			fmt.Fprintf(&b, "| %s | — | — | — |\n", r.Name)
+			continue
+		}
+		fmt.Fprintf(&b, "| %s | %s | %s | %s |\n",
+			r.Name,
+			pct(r.PerTier[TierExact].Recall[5]),
+			pct(r.PerTier[TierConcept].Recall[5]),
+			pct(r.PerTier[TierMultiHop].Recall[5]),
+		)
+	}
+
+	return b.String()
+}
+
+func pct(f float64) string { return fmt.Sprintf("%5.1f%%", f*100) }

internal/eval/recall/recall_test.go

@@ -0,0 +1,164 @@
+package recall
+
+import (
+	"strings"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+
+	"github.com/zzet/gortex/internal/search"
+)
+
+// staticRanker returns a predetermined ranked list regardless of query.
+// Lets us pin exact metrics without standing up the full indexer.
+func staticRanker(name string, ids []string) Ranker {
+	return Ranker{
+		Name: name,
+		Search: func(_ string, limit int) []string {
+			if limit >= len(ids) {
+				return ids
+			}
+			return ids[:limit]
+		},
+	}
+}
+
+func TestRun_AllHitsAtRankOne(t *testing.T) {
+	fixture := Fixture{
+		Name: "t",
+		Cases: []Case{
+			{Query: "q1", Tier: TierExact, Expected: []string{"a"}},
+			{Query: "q2", Tier: TierExact, Expected: []string{"a"}},
+		},
+	}
+	r := staticRanker("perfect", []string{"a", "b", "c"})
+	report := Run(fixture, []Ranker{r}, nil)
+	assert.Equal(t, 2, report.Cases)
+	res := report.Rankers[0]
+	assert.Equal(t, 2, res.Hits[1])
+	assert.Equal(t, 2, res.Hits[5])
+	assert.Equal(t, 2, res.Hits[20])
+	assert.Equal(t, 1.0, res.Recall[1])
+	assert.Equal(t, 1.0, res.MeanRRank)
+	// Per-tier bookkeeping: exact tier should get both hits.
+	assert.Equal(t, 2, res.PerTier[TierExact].Hits[1])
+	assert.Equal(t, 0, res.PerTier[TierConcept].Hits[1])
+}
+
+func TestRun_MissAndRankTwo(t *testing.T) {
+	fixture := Fixture{
+		Name: "t",
+		Cases: []Case{
+			{Query: "q1", Tier: TierExact, Expected: []string{"b"}}, // rank 2
+			{Query: "q2", Tier: TierExact, Expected: []string{"z"}}, // miss
+		},
+	}
+	r := staticRanker("partial", []string{"a", "b", "c"})
+	report := Run(fixture, []Ranker{r}, nil)
+	res := report.Rankers[0]
+	assert.Equal(t, 0, res.Hits[1])
+	assert.Equal(t, 1, res.Hits[5])
+	assert.Equal(t, 1, res.Hits[20])
+	assert.Equal(t, 0.5, res.Recall[5])
+	assert.InDelta(t, 0.25, res.MeanRRank, 1e-9)
+}
+
+func TestRun_AnyExpectedIsHit(t *testing.T) {
+	fixture := Fixture{
+		Name: "t",
+		Cases: []Case{
+			{Query: "q1", Tier: TierMultiHop, Expected: []string{"z", "b", "y"}},
+		},
+	}
+	r := staticRanker("multi-expected", []string{"a", "b", "c"})
+	report := Run(fixture, []Ranker{r}, nil)
+	res := report.Rankers[0]
+	assert.Equal(t, 1, res.Hits[5])
+	assert.InDelta(t, 0.5, res.MeanRRank, 1e-9)
+	assert.Equal(t, 1, res.PerTier[TierMultiHop].Hits[5])
+}
+
+func TestRun_LatencyPercentiles(t *testing.T) {
+	fixture := Fixture{
+		Name: "t",
+		Cases: []Case{
+			{Query: "q", Tier: TierExact, Expected: []string{"a"}},
+			{Query: "q", Tier: TierExact, Expected: []string{"a"}},
+			{Query: "q", Tier: TierExact, Expected: []string{"a"}},
+		},
+	}
+	r := staticRanker("x", []string{"a"})
+	report := Run(fixture, []Ranker{r}, nil)
+	// Latency buckets exist and are non-negative.
+	assert.GreaterOrEqual(t, report.Rankers[0].P50Micros, int64(0))
+	assert.GreaterOrEqual(t, report.Rankers[0].P95Micros, report.Rankers[0].P50Micros)
+	assert.GreaterOrEqual(t, report.Rankers[0].MaxMicros, report.Rankers[0].P95Micros)
+}
+
+func TestRun_TokensReturnedCounted(t *testing.T) {
+	fixture := Fixture{
+		Name:  "t",
+		Cases: []Case{{Query: "q", Tier: TierExact, Expected: []string{"a"}}},
+	}
+	r := staticRanker("x", []string{"aaaaaaaa", "bbbbbbbb"})
+	// Token counter that counts bytes to make the assertion deterministic.
+	report := Run(fixture, []Ranker{r}, func(s string) int { return len(s) })
+	assert.Greater(t, report.Rankers[0].MeanTokens, 0.0)
+}
+
+func TestMarkdownStableOrder(t *testing.T) {
+	fixture := Fixture{
+		Name:  "t",
+		Cases: []Case{{Query: "q", Tier: TierExact, Expected: []string{"a"}}},
+	}
+	a := staticRanker("zeta", []string{"a"})
+	b := staticRanker("alpha", []string{"a"})
+	report := Run(fixture, []Ranker{a, b}, nil)
+	md := Markdown(report)
+	alphaIdx := strings.Index(md, "| alpha ")
+	zetaIdx := strings.Index(md, "| zeta ")
+	assert.Greater(t, alphaIdx, 0)
+	assert.Greater(t, zetaIdx, alphaIdx)
+	// Per-tier section present.
+	assert.Contains(t, md, "## Per tier (R@5)")
+}
+
+func TestMarkdownSkippedRankerRow(t *testing.T) {
+	fixture := Fixture{
+		Name:  "t",
+		Cases: []Case{{Query: "q", Tier: TierExact, Expected: []string{"a"}}},
+	}
+	r := Ranker{Name: "semantic", Search: func(_ string, _ int) []string { return nil }}
+	report := Run(fixture, []Ranker{r}, nil)
+	report.Rankers[0].Skipped = "no embedder"
+	md := Markdown(report)
+	assert.Contains(t, md, "skipped: no embedder")
+}
+
+// TestBM25Ranker_AgainstRealBackend wires the adapter to a live BM25
+// backend and spot-checks ranked output.
+func TestBM25Ranker_AgainstRealBackend(t *testing.T) {
+	backend := search.NewBM25()
+	backend.Add("pkg/a.go::Foo", "Foo", "pkg/a.go", "")
+	backend.Add("pkg/b.go::Bar", "Bar", "pkg/b.go", "")
+
+	r := BM25Ranker("bm25", backend)
+	hits := r.Search("Foo", 5)
+	assert.NotEmpty(t, hits)
+	assert.Equal(t, "pkg/a.go::Foo", hits[0])
+
+	fixture := Fixture{Cases: []Case{
+		{Query: "Bar", Tier: TierExact, Expected: []string{"pkg/b.go::Bar"}},
+		{Query: "Foo", Tier: TierExact, Expected: []string{"pkg/a.go::Foo"}},
+	}}
+	report := Run(fixture, []Ranker{r}, nil)
+	assert.Equal(t, 2, report.Rankers[0].Hits[1])
+}
+
+func TestAdaptCasesForFileRanker(t *testing.T) {
+	in := []Case{
+		{Query: "q", Expected: []string{"a/b.go::Foo", "a/b.go::Bar", "c/d.go::Baz"}},
+	}
+	out := AdaptCasesForFileRanker(in)
+	assert.Equal(t, []string{"a/b.go", "c/d.go"}, out[0].Expected)
+}

internal/eval/stdbench/dataset.go

@@ -0,0 +1,49 @@
+// Package stdbench loads standardized code-retrieval benchmarks —
+// CoIR, SWE-ContextBench, and ContextBench — into a single normalized
+// {corpus, queries, qrels} model and scores a retriever against them
+// with the textbook Recall@K / Precision@K / NDCG@K / MRR metrics.
+//
+// The loaders parse the on-disk formats; the actual retrieval is left
+// to the caller (the `gortex eval stdbench` verb wires Gortex's BM25
+// backend in), so the same harness measures whatever retriever is
+// handed to Evaluate.
+package stdbench
+
+// Doc is one corpus document — a code snippet, file, or symbol the
+// retriever ranks. ID is the identifier the benchmark's relevance
+// judgements reference.
+type Doc struct {
+	ID   string
+	Text string
+}
+
+// Query is one benchmark query plus its graded relevance judgements.
+// Relevant maps a corpus Doc.ID to its relevance grade: 1 means
+// relevant, higher grades mean more relevant (CoIR / BEIR qrels carry
+// graded labels; the JSONL task benchmarks default every gold ID to
+// grade 1).
+type Query struct {
+	ID       string
+	Text     string
+	Relevant map[string]int
+}
+
+// Dataset is a loaded benchmark: a corpus to index plus the queries to
+// run against it.
+type Dataset struct {
+	Name    string
+	Corpus  []Doc
+	Queries []Query
+}
+
+// RelevantCount returns the number of queries that carry at least one
+// relevance judgement — the denominator Evaluate averages over.
+func (d Dataset) RelevantCount() int {
+	n := 0
+	for _, q := range d.Queries {
+		if len(q.Relevant) > 0 {
+			n++
+		}
+	}
+	return n
+}

internal/eval/stdbench/eval.go

@@ -0,0 +1,151 @@
+package stdbench
+
+import (
+	"fmt"
+	"math"
+	"sort"
+	"strings"
+)
+
+// DefaultKs are the rank cutoffs Recall@K / Precision@K report.
+var DefaultKs = []int{1, 5, 10, 20}
+
+// Retriever ranks corpus Doc IDs for a query, best first, capped at k.
+type Retriever func(query string, k int) []string
+
+// Metrics is the aggregate score of a retriever over a Dataset.
+type Metrics struct {
+	Dataset   string          `json:"dataset"`
+	Queries   int             `json:"queries"`
+	Scored    int             `json:"scored"` // queries with a relevance judgement
+	RecallAtK map[int]float64 `json:"recall_at_k"`
+	PrecAtK   map[int]float64 `json:"precision_at_k"`
+	NDCGAt10  float64         `json:"ndcg_at_10"`
+	MRR       float64         `json:"mrr"`
+}
+
+// Evaluate runs retrieve against every query in ds and aggregates the
+// standard retrieval metrics. Queries with no relevance judgement are
+// counted in Queries but excluded from the metric averages. ks is the
+// Recall@K / Precision@K cutoff set; pass nil for DefaultKs.
+func Evaluate(ds Dataset, retrieve Retriever, ks []int) Metrics {
+	if len(ks) == 0 {
+		ks = DefaultKs
+	}
+	maxK := 10 // NDCG@10 always needs at least the top 10.
+	for _, k := range ks {
+		if k > maxK {
+			maxK = k
+		}
+	}
+
+	m := Metrics{
+		Dataset:   ds.Name,
+		Queries:   len(ds.Queries),
+		RecallAtK: make(map[int]float64, len(ks)),
+		PrecAtK:   make(map[int]float64, len(ks)),
+	}
+	recallSum := make(map[int]float64, len(ks))
+	precSum := make(map[int]float64, len(ks))
+	var ndcgSum, rrSum float64
+
+	for _, q := range ds.Queries {
+		if len(q.Relevant) == 0 {
+			continue
+		}
+		m.Scored++
+		ranked := retrieve(q.Text, maxK)
+		for _, k := range ks {
+			hit := 0
+			for i, id := range ranked {
+				if i >= k {
+					break
+				}
+				if q.Relevant[id] > 0 {
+					hit++
+				}
+			}
+			recallSum[k] += float64(hit) / float64(len(q.Relevant))
+			precSum[k] += float64(hit) / float64(k)
+		}
+		ndcgSum += ndcg(ranked, q.Relevant, 10)
+		rrSum += reciprocalRank(ranked, q.Relevant)
+	}
+
+	if m.Scored > 0 {
+		for _, k := range ks {
+			m.RecallAtK[k] = recallSum[k] / float64(m.Scored)
+			m.PrecAtK[k] = precSum[k] / float64(m.Scored)
+		}
+		m.NDCGAt10 = ndcgSum / float64(m.Scored)
+		m.MRR = rrSum / float64(m.Scored)
+	}
+	return m
+}
+
+// ndcg computes normalized discounted cumulative gain at cutoff k using
+// the graded relevance labels in rel. Returns 0 when no relevant doc
+// exists (ideal DCG would be zero).
+func ndcg(ranked []string, rel map[string]int, k int) float64 {
+	dcg := 0.0
+	for i, id := range ranked {
+		if i >= k {
+			break
+		}
+		if g := rel[id]; g > 0 {
+			dcg += float64(g) / math.Log2(float64(i+2))
+		}
+	}
+	grades := make([]int, 0, len(rel))
+	for _, g := range rel {
+		if g > 0 {
+			grades = append(grades, g)
+		}
+	}
+	sort.Sort(sort.Reverse(sort.IntSlice(grades)))
+	idcg := 0.0
+	for i, g := range grades {
+		if i >= k {
+			break
+		}
+		idcg += float64(g) / math.Log2(float64(i+2))
+	}
+	if idcg == 0 {
+		return 0
+	}
+	return dcg / idcg
+}
+
+// reciprocalRank returns 1/rank of the first relevant hit, or 0 when no
+// relevant doc appears in the ranked list.
+func reciprocalRank(ranked []string, rel map[string]int) float64 {
+	for i, id := range ranked {
+		if rel[id] > 0 {
+			return 1.0 / float64(i+1)
+		}
+	}
+	return 0
+}
+
+// Markdown renders the metrics as a Markdown section.
+func (m Metrics) Markdown() string {
+	var b strings.Builder
+	fmt.Fprintf(&b, "### %s\n\n", m.Dataset)
+	fmt.Fprintf(&b, "_%d queries · %d scored against relevance judgements_\n\n", m.Queries, m.Scored)
+	b.WriteString("| metric | value |\n|--------|-------|\n")
+
+	ks := make([]int, 0, len(m.RecallAtK))
+	for k := range m.RecallAtK {
+		ks = append(ks, k)
+	}
+	sort.Ints(ks)
+	for _, k := range ks {
+		fmt.Fprintf(&b, "| Recall@%d | %.3f |\n", k, m.RecallAtK[k])
+	}
+	for _, k := range ks {
+		fmt.Fprintf(&b, "| Precision@%d | %.3f |\n", k, m.PrecAtK[k])
+	}
+	fmt.Fprintf(&b, "| NDCG@10 | %.3f |\n", m.NDCGAt10)
+	fmt.Fprintf(&b, "| MRR | %.3f |\n", m.MRR)
+	return b.String()
+}

internal/eval/stdbench/eval_test.go

@@ -0,0 +1,80 @@
+package stdbench
+
+import (
+	"math"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+
+	"github.com/zzet/gortex/internal/search"
+)
+
+func TestEvaluate_KnownRetriever(t *testing.T) {
+	ds := Dataset{
+		Name: "synthetic",
+		Queries: []Query{
+			{ID: "q1", Text: "x", Relevant: map[string]int{"d1": 1}},
+		},
+	}
+	// Retriever puts the one relevant doc at rank 2.
+	retrieve := func(_ string, _ int) []string { return []string{"d2", "d1", "d3"} }
+
+	m := Evaluate(ds, retrieve, []int{1, 5})
+	require.Equal(t, 1, m.Scored)
+	require.InDelta(t, 0.0, m.RecallAtK[1], 1e-9, "d1 is not in the top 1")
+	require.InDelta(t, 1.0, m.RecallAtK[5], 1e-9, "d1 is in the top 5")
+	require.InDelta(t, 0.5, m.MRR, 1e-9, "first hit is at rank 2")
+	// NDCG@10: grade 1 at rank 2 → 1/log2(3); ideal DCG → 1/log2(2)=1.
+	require.InDelta(t, 1.0/math.Log2(3), m.NDCGAt10, 1e-9)
+}
+
+func TestEvaluate_SkipsUnjudgedQueries(t *testing.T) {
+	ds := Dataset{
+		Queries: []Query{
+			{ID: "q1", Text: "x", Relevant: map[string]int{"d1": 1}},
+			{ID: "q2", Text: "y"}, // no relevance judgement
+		},
+	}
+	m := Evaluate(ds, func(string, int) []string { return []string{"d1"} }, nil)
+	require.Equal(t, 2, m.Queries)
+	require.Equal(t, 1, m.Scored, "the unjudged query is excluded from the averages")
+	require.InDelta(t, 1.0, m.RecallAtK[1], 1e-9)
+}
+
+func TestEvaluate_PerfectRanking(t *testing.T) {
+	ds := Dataset{
+		Queries: []Query{
+			{ID: "q1", Text: "x", Relevant: map[string]int{"d1": 3}},
+		},
+	}
+	m := Evaluate(ds, func(string, int) []string { return []string{"d1", "d2"} }, []int{1})
+	require.InDelta(t, 1.0, m.RecallAtK[1], 1e-9)
+	require.InDelta(t, 1.0, m.MRR, 1e-9)
+	require.InDelta(t, 1.0, m.NDCGAt10, 1e-9)
+}
+
+// TestEvaluate_EndToEndBM25 runs the full evaluation path: load a JSONL
+// benchmark, index its corpus into Gortex's real BM25 backend, and
+// score retrieval — the same wiring `gortex eval stdbench` uses.
+func TestEvaluate_EndToEndBM25(t *testing.T) {
+	ds, err := LoadContextBench("testdata/contextbench.jsonl")
+	require.NoError(t, err)
+
+	bm := search.NewBM25()
+	for _, d := range ds.Corpus {
+		bm.Add(d.ID, d.Text)
+	}
+	retrieve := func(query string, k int) []string {
+		hits := bm.Search(query, k)
+		ids := make([]string, 0, len(hits))
+		for _, h := range hits {
+			ids = append(ids, h.ID)
+		}
+		return ids
+	}
+
+	m := Evaluate(ds, retrieve, nil)
+	require.Equal(t, 2, m.Scored)
+	require.Greater(t, m.RecallAtK[5], 0.0, "BM25 should surface the gold candidate")
+	require.Greater(t, m.MRR, 0.0)
+}

internal/eval/stdbench/loaders.go

@@ -0,0 +1,293 @@
+package stdbench
+
+import (
+	"bufio"
+	"encoding/json"
+	"fmt"
+	"os"
+	"path/filepath"
+	"sort"
+	"strconv"
+	"strings"
+)
+
+// maxJSONLLine caps the scanner buffer — benchmark corpus lines can be
+// large code blobs, well past bufio's 64 KiB default.
+const maxJSONLLine = 16 * 1024 * 1024
+
+// scanJSONL streams a JSON-lines file, decoding each non-blank line
+// into a map and handing it to fn. Blank lines are skipped; a decode
+// error is reported with its line number.
+func scanJSONL(path string, fn func(rec map[string]any) error) error {
+	f, err := os.Open(path)
+	if err != nil {
+		return err
+	}
+	defer f.Close()
+
+	sc := bufio.NewScanner(f)
+	sc.Buffer(make([]byte, 0, 64*1024), maxJSONLLine)
+	line := 0
+	for sc.Scan() {
+		line++
+		text := strings.TrimSpace(sc.Text())
+		if text == "" {
+			continue
+		}
+		var rec map[string]any
+		if err := json.Unmarshal([]byte(text), &rec); err != nil {
+			return fmt.Errorf("%s line %d: %w", path, line, err)
+		}
+		if err := fn(rec); err != nil {
+			return fmt.Errorf("%s line %d: %w", path, line, err)
+		}
+	}
+	return sc.Err()
+}
+
+// firstString returns the first string-valued key present in m.
+func firstString(m map[string]any, keys ...string) string {
+	for _, k := range keys {
+		if v, ok := m[k].(string); ok && v != "" {
+			return v
+		}
+	}
+	return ""
+}
+
+// LoadCoIR loads a CoIR / BEIR-format benchmark from a directory laid
+// out as corpus.jsonl + queries.jsonl + qrels/<split>.tsv. CoIR (Code
+// Information Retrieval, ACL 2025) ships every task in this canonical
+// BEIR triple. Only queries that appear in the qrels file are kept —
+// queries.jsonl typically holds every split at once.
+func LoadCoIR(dir string) (Dataset, error) {
+	ds := Dataset{Name: "CoIR"}
+
+	corpus, err := loadBEIRCorpus(filepath.Join(dir, "corpus.jsonl"))
+	if err != nil {
+		return Dataset{}, fmt.Errorf("CoIR corpus: %w", err)
+	}
+	ds.Corpus = corpus
+
+	queryText, err := loadBEIRQueries(filepath.Join(dir, "queries.jsonl"))
+	if err != nil {
+		return Dataset{}, fmt.Errorf("CoIR queries: %w", err)
+	}
+
+	qrels, err := loadBEIRQrels(dir)
+	if err != nil {
+		return Dataset{}, fmt.Errorf("CoIR qrels: %w", err)
+	}
+
+	ids := make([]string, 0, len(qrels))
+	for id := range qrels {
+		ids = append(ids, id)
+	}
+	sort.Strings(ids)
+	for _, id := range ids {
+		ds.Queries = append(ds.Queries, Query{
+			ID:       id,
+			Text:     queryText[id],
+			Relevant: qrels[id],
+		})
+	}
+	return ds, nil
+}
+
+func loadBEIRCorpus(path string) ([]Doc, error) {
+	var corpus []Doc
+	err := scanJSONL(path, func(rec map[string]any) error {
+		id := firstString(rec, "_id", "id", "doc_id")
+		if id == "" {
+			return nil
+		}
+		title := firstString(rec, "title")
+		text := firstString(rec, "text", "content", "body", "code")
+		corpus = append(corpus, Doc{ID: id, Text: strings.TrimSpace(title + " " + text)})
+		return nil
+	})
+	return corpus, err
+}
+
+func loadBEIRQueries(path string) (map[string]string, error) {
+	out := make(map[string]string)
+	err := scanJSONL(path, func(rec map[string]any) error {
+		id := firstString(rec, "_id", "id", "query_id")
+		if id == "" {
+			return nil
+		}
+		out[id] = firstString(rec, "text", "query", "question")
+		return nil
+	})
+	return out, err
+}
+
+// loadBEIRQrels reads the first qrels file it finds — qrels/test.tsv,
+// qrels/dev.tsv, qrels/train.tsv, or a bare qrels.tsv. The TSV is
+// `query-id<TAB>corpus-id<TAB>score`, with an optional header row.
+func loadBEIRQrels(dir string) (map[string]map[string]int, error) {
+	candidates := []string{
+		filepath.Join(dir, "qrels", "test.tsv"),
+		filepath.Join(dir, "qrels", "dev.tsv"),
+		filepath.Join(dir, "qrels", "train.tsv"),
+		filepath.Join(dir, "qrels.tsv"),
+	}
+	var path string
+	for _, c := range candidates {
+		if _, err := os.Stat(c); err == nil {
+			path = c
+			break
+		}
+	}
+	if path == "" {
+		return nil, fmt.Errorf("no qrels file under %s", dir)
+	}
+
+	f, err := os.Open(path)
+	if err != nil {
+		return nil, err
+	}
+	defer f.Close()
+
+	qrels := make(map[string]map[string]int)
+	sc := bufio.NewScanner(f)
+	sc.Buffer(make([]byte, 0, 64*1024), maxJSONLLine)
+	for sc.Scan() {
+		fields := strings.Split(strings.TrimSpace(sc.Text()), "\t")
+		if len(fields) < 3 {
+			continue
+		}
+		score, err := strconv.Atoi(strings.TrimSpace(fields[2]))
+		if err != nil {
+			continue // header row ("query-id corpus-id score") or junk.
+		}
+		qid, cid := fields[0], fields[1]
+		if qrels[qid] == nil {
+			qrels[qid] = make(map[string]int)
+		}
+		qrels[qid][cid] = score
+	}
+	return qrels, sc.Err()
+}
+
+// LoadSWEContextBench loads SWE-ContextBench (arXiv 2602.08316) from a
+// JSON-lines file — one context-retrieval task per line.
+func LoadSWEContextBench(path string) (Dataset, error) {
+	return loadJSONLBench("SWE-ContextBench", path)
+}
+
+// LoadContextBench loads ContextBench (arXiv 2602.05892) from a
+// JSON-lines file — one context-retrieval task per line.
+func LoadContextBench(path string) (Dataset, error) {
+	return loadJSONLBench("ContextBench", path)
+}
+
+// loadJSONLBench loads a JSON-lines context-retrieval benchmark. Each
+// line is one task object:
+//
+//	{
+//	  "id":         "<task id>",            // also task_id / instance_id
+//	  "query":      "<natural-language>",   // also question / problem_statement
+//	  "relevant":   ["<doc id>", ...],      // also gold / context / expected;
+//	                                        //   entries may be {"id","score"}
+//	  "candidates": [{"id","text"}, ...]    // optional per-task corpus pool;
+//	                                        //   also documents / pool
+//	}
+//
+// Per-task candidate pools are merged (deduplicated by ID) into one
+// corpus. A task with no candidates still contributes its query — the
+// caller indexes its own corpus in that case.
+func loadJSONLBench(name, path string) (Dataset, error) {
+	ds := Dataset{Name: name}
+	corpusSeen := make(map[string]bool)
+	line := 0
+
+	err := scanJSONL(path, func(rec map[string]any) error {
+		line++
+		q := Query{
+			ID:       firstString(rec, "id", "task_id", "instance_id", "_id"),
+			Text:     firstString(rec, "query", "question", "problem_statement", "text"),
+			Relevant: make(map[string]int),
+		}
+		if q.ID == "" {
+			q.ID = fmt.Sprintf("%s-%d", name, line)
+		}
+		for _, rs := range relevantIDs(rec, "relevant", "gold", "context", "expected", "gold_files") {
+			q.Relevant[rs.id] = rs.score
+		}
+		for _, doc := range candidateDocs(rec, "candidates", "documents", "pool") {
+			if !corpusSeen[doc.ID] {
+				corpusSeen[doc.ID] = true
+				ds.Corpus = append(ds.Corpus, doc)
+			}
+		}
+		ds.Queries = append(ds.Queries, q)
+		return nil
+	})
+	if err != nil {
+		return Dataset{}, err
+	}
+	return ds, nil
+}
+
+// idScore is one relevance judgement extracted from a JSONL task.
+type idScore struct {
+	id    string
+	score int
+}
+
+// relevantIDs pulls the relevance judgement list from the first
+// matching key. Each list entry is either a bare ID string or an
+// {"id","score"} object; a bare string defaults to grade 1.
+func relevantIDs(rec map[string]any, keys ...string) []idScore {
+	list := firstList(rec, keys...)
+	out := make([]idScore, 0, len(list))
+	for _, el := range list {
+		switch v := el.(type) {
+		case string:
+			if v != "" {
+				out = append(out, idScore{id: v, score: 1})
+			}
+		case map[string]any:
+			id := firstString(v, "id", "_id", "doc_id", "corpus_id")
+			if id == "" {
+				continue
+			}
+			score := 1
+			if s, ok := v["score"].(float64); ok && int(s) > 0 {
+				score = int(s)
+			}
+			out = append(out, idScore{id: id, score: score})
+		}
+	}
+	return out
+}
+
+// candidateDocs pulls the per-task candidate pool from the first
+// matching key. Each entry is an {"id","text"} object.
+func candidateDocs(rec map[string]any, keys ...string) []Doc {
+	list := firstList(rec, keys...)
+	out := make([]Doc, 0, len(list))
+	for _, el := range list {
+		obj, ok := el.(map[string]any)
+		if !ok {
+			continue
+		}
+		id := firstString(obj, "id", "_id", "doc_id")
+		if id == "" {
+			continue
+		}
+		out = append(out, Doc{ID: id, Text: firstString(obj, "text", "content", "body", "code")})
+	}
+	return out
+}
+
+// firstList returns the first key whose value is a JSON array.
+func firstList(m map[string]any, keys ...string) []any {
+	for _, k := range keys {
+		if v, ok := m[k].([]any); ok {
+			return v
+		}
+	}
+	return nil
+}

internal/eval/stdbench/loaders_test.go

@@ -0,0 +1,62 @@
+package stdbench
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func queriesByID(ds Dataset) map[string]Query {
+	out := make(map[string]Query, len(ds.Queries))
+	for _, q := range ds.Queries {
+		out[q.ID] = q
+	}
+	return out
+}
+
+func TestLoadCoIR(t *testing.T) {
+	ds, err := LoadCoIR("testdata/coir")
+	require.NoError(t, err)
+	require.Equal(t, "CoIR", ds.Name)
+	require.Len(t, ds.Corpus, 3)
+	// Only queries present in qrels are kept — q3 has no judgement.
+	require.Len(t, ds.Queries, 2)
+
+	q := queriesByID(ds)
+	require.Equal(t, "search a sorted array for an element", q["q1"].Text)
+	require.Equal(t, map[string]int{"d1": 2}, q["q1"].Relevant)
+	require.Equal(t, map[string]int{"d3": 1}, q["q2"].Relevant)
+	// Corpus text folds the title into the body.
+	require.Contains(t, ds.Corpus[0].Text, "binary search")
+}
+
+func TestLoadContextBench(t *testing.T) {
+	ds, err := LoadContextBench("testdata/contextbench.jsonl")
+	require.NoError(t, err)
+	require.Equal(t, "ContextBench", ds.Name)
+	// Both tasks carry the same 2-candidate pool — deduplicated to 2.
+	require.Len(t, ds.Corpus, 2)
+	require.Len(t, ds.Queries, 2)
+	require.Equal(t, 2, ds.RelevantCount())
+
+	q := queriesByID(ds)
+	// A bare string relevance entry defaults to grade 1.
+	require.Equal(t, map[string]int{"c1": 1}, q["t1"].Relevant)
+	// An {id,score} relevance object keeps its graded score.
+	require.Equal(t, map[string]int{"c2": 3}, q["t2"].Relevant)
+}
+
+func TestLoadSWEContextBench_SharesJSONLLoader(t *testing.T) {
+	// SWE-ContextBench reuses the JSONL task loader — only the dataset
+	// name differs from ContextBench.
+	ds, err := LoadSWEContextBench("testdata/contextbench.jsonl")
+	require.NoError(t, err)
+	require.Equal(t, "SWE-ContextBench", ds.Name)
+	require.Len(t, ds.Queries, 2)
+	require.Len(t, ds.Corpus, 2)
+}
+
+func TestLoadCoIR_MissingDirectory(t *testing.T) {
+	_, err := LoadCoIR("testdata/no-such-dir")
+	require.Error(t, err)
+}

internal/eval/stdbench/testdata/coir/corpus.jsonl

@@ -0,0 +1,3 @@
+{"_id":"d1","title":"binary search","text":"func binarySearch(a []int, x int) int finds an element in a sorted slice"}
+{"_id":"d2","title":"quick sort","text":"func quickSort(a []int) sorts a slice in place"}
+{"_id":"d3","title":"hash map lookup","text":"func (m *HashMap) Get(key string) any returns the value stored under key"}

internal/eval/stdbench/testdata/coir/qrels/test.tsv

@@ -0,0 +1,3 @@
+query-id	corpus-id	score
+q1	d1	2
+q2	d3	1

internal/eval/stdbench/testdata/coir/queries.jsonl

@@ -0,0 +1,3 @@
+{"_id":"q1","text":"search a sorted array for an element"}
+{"_id":"q2","text":"look up a value in a map by its key"}
+{"_id":"q3","text":"a query with no relevance judgement"}

internal/eval/stdbench/testdata/contextbench.jsonl

@@ -0,0 +1,2 @@
+{"id":"t1","query":"parse the authentication token from a request","relevant":["c1"],"candidates":[{"id":"c1","text":"func parseAuthToken(req Request) Token validates and decodes the bearer token"},{"id":"c2","text":"func renderTemplate(name string, data any) string expands an HTML template"}]}
+{"id":"t2","query":"render an HTML template with data","relevant":[{"id":"c2","score":3}],"candidates":[{"id":"c1","text":"func parseAuthToken(req Request) Token validates and decodes the bearer token"},{"id":"c2","text":"func renderTemplate(name string, data any) string expands an HTML template"}]}