chore: import upstream snapshot with attribution

bench/baselines/README.md

@@ -0,0 +1,76 @@
+# Retrieval baselines bench
+
+Reproducible per-baseline NDCG@10 + latency table across six
+candidate retrieval systems. The harness ships **six** adapters
+behind a single uniform Go-side surface; per-adapter `Available()`
+checks let `--smoke` verify wiring without paying for the heavy
+ones.
+
+Supported adapters:
+
+| Adapter        | Type          | Install                                                            |
+|----------------|---------------|--------------------------------------------------------------------|
+| ripgrep        | Go-native     | `brew install ripgrep` (or distro equivalent)                      |
+| probe          | Go-native     | `cargo install probe-ai`                                           |
+| colgrep        | Python        | `pip install colgrep` (requires CUDA-capable GPU)                  |
+| grepai         | Python        | `pip install grepai-cli`                                           |
+| coderankembed  | Python+model  | `pip install sentence-transformers transformers torch` (model ~440MB on first run) |
+| semble         | Python        | `pip install semble`                                               |
+
+## Running
+
+```sh
+# Smoke check: report which adapters are available right now
+go run ./bench/baselines -smoke
+
+# Full table against the gortex repo itself, all adapters
+go run ./bench/baselines
+
+# Just the locally-installable ones (ripgrep + probe)
+go run ./bench/baselines -against ripgrep,probe
+
+# JSON output for downstream tooling
+go run ./bench/baselines -format json -json bench/results/baselines.json
+```
+
+Flags:
+
+- `-repo PATH` — corpus to query (default `.`)
+- `-queries PATH` — JSON query set (default `queries.json` here)
+- `-groundtruth PATH` — JSON per-query expected file paths (default
+  `groundtruth.json` here)
+- `-against LIST` — comma-separated adapter names; unknown names
+  error out so a typo doesn't silently drop a baseline
+- `-top-k N` — top-K per query (default 10, matches NDCG@10)
+- `-out PATH` — primary output (default stdout)
+- `-json PATH` — companion JSON metrics output
+- `-format markdown|json` — primary format
+- `-smoke` — skip runs; only probe `Available()` for each adapter
+- `-timeout DURATION` — per-adapter wall-clock cap (default 5m)
+
+## How NDCG@10 is computed
+
+For each query the harness computes:
+
+```
+DCG@10  = Σ (gain_i / log2(i + 2))   for i in [0, min(K, |hits|))
+IDCG@10 = Σ (1     / log2(i + 2))    for i in [0, min(K, |expected|))
+NDCG@10 = DCG@10 / IDCG@10
+```
+
+with `gain_i = 1` when `hits[i]` is in the ground-truth set,
+`0` otherwise. The reported number is the **mean** across queries
+with a non-empty ground-truth entry. Queries the adapter failed
+on (Error set) are skipped from the mean — they're shown in the
+per-query JSON for debugging but don't penalize the headline.
+
+## Adding an adapter
+
+1. Add a struct in `adapters.go` that satisfies the `adapter`
+   interface (`Name`, `Available`, `Run`).
+2. Register it in `allAdapters()` — that single list drives the CLI,
+   the smoke output, and `adapterByName`.
+3. For Python baselines: embed `pythonBaselineAdapter` and call
+   `pythonModuleAvailable` / `runPythonModule` (or
+   `runPythonScript` if you need a custom wrapper). Document the
+   install in the table above.

bench/baselines/adapters.go

@@ -0,0 +1,333 @@
+// Package main defines the baseline adapter framework: a single Go
+// surface that uniformly invokes external retrieval baselines
+// (ripgrep, probe, ColGREP, grepai, CodeRankEmbed Hybrid, semble)
+// against a shared query set, then reports per-baseline NDCG@10 +
+// latency in a comparable table.
+//
+// Per-adapter contract:
+//
+//   - Name()       canonical baseline name, used in CLI / table cols
+//   - Available()  cheap probe: does this baseline run on this box?
+//   - Run(queries) per-query result list; honest empty-on-failure
+//
+// Go-native baselines (ripgrep, probe) shell to their respective
+// binaries. Python-heavy ones (ColGREP, grepai, CodeRankEmbed) shell
+// to `python3 -m <module>` and require an explicit per-package install
+// that the harness documents in bench/baselines/README.md but does
+// NOT attempt automatically. This keeps `gortex eval baselines
+// --smoke` cheap on a fresh CI runner — it reports each baseline's
+// availability without trying to install anything.
+package main
+
+import (
+	"bytes"
+	"context"
+	"fmt"
+	"io"
+	"os/exec"
+	"sort"
+	"strings"
+	"time"
+)
+
+// adapter is the interface every baseline implementation satisfies.
+type adapter interface {
+	Name() string
+	Available() (bool, string)
+	Run(ctx context.Context, queries []string, repoRoot string, topK int) []queryResult
+}
+
+// queryResult is the per-query outcome from one adapter.
+type queryResult struct {
+	Query    string        `json:"query"`
+	Hits     []string      `json:"hits"`
+	Latency  time.Duration `json:"-"`
+	LatencyMs float64      `json:"latency_ms"`
+	Error    string        `json:"error,omitempty"`
+}
+
+// allAdapters returns the canonical adapter set, in stable order.
+// Adding a new baseline = adding one line here + one adapter
+// implementation. The smoke check uses this list directly so the
+// CLI never drifts from the registry.
+func allAdapters() []adapter {
+	return []adapter{
+		&ripgrepAdapter{},
+		&probeAdapter{},
+		&colgrepAdapter{},
+		&grepaiAdapter{},
+		&coderankAdapter{},
+		&sembleAdapter{},
+	}
+}
+
+// adapterByName looks up one adapter by canonical name; nil when no
+// match (the caller should check before invoking).
+func adapterByName(name string) adapter {
+	for _, a := range allAdapters() {
+		if strings.EqualFold(a.Name(), name) {
+			return a
+		}
+	}
+	return nil
+}
+
+// --- ripgrep --------------------------------------------------------
+
+type ripgrepAdapter struct{}
+
+func (*ripgrepAdapter) Name() string { return "ripgrep" }
+
+func (*ripgrepAdapter) Available() (bool, string) {
+	if _, err := exec.LookPath("rg"); err != nil {
+		return false, "rg not on PATH"
+	}
+	return true, ""
+}
+
+func (*ripgrepAdapter) Run(ctx context.Context, queries []string, repoRoot string, topK int) []queryResult {
+	out := make([]queryResult, 0, len(queries))
+	for _, q := range queries {
+		start := time.Now()
+		cmd := exec.CommandContext(ctx, "rg", "--files-with-matches", q, repoRoot)
+		var buf bytes.Buffer
+		cmd.Stdout = &buf
+		cmd.Stderr = io.Discard
+		err := cmd.Run()
+		latency := time.Since(start)
+		r := queryResult{Query: q, Latency: latency, LatencyMs: msFrom(latency)}
+		if err != nil {
+			if ee, ok := err.(*exec.ExitError); ok && ee.ExitCode() == 1 {
+				// rg exit code 1 = no matches; not an error.
+				out = append(out, r)
+				continue
+			}
+			r.Error = err.Error()
+			out = append(out, r)
+			continue
+		}
+		hits := parseLines(buf.String(), repoRoot, topK)
+		r.Hits = hits
+		out = append(out, r)
+	}
+	return out
+}
+
+// --- probe ----------------------------------------------------------
+
+type probeAdapter struct{}
+
+func (*probeAdapter) Name() string { return "probe" }
+
+func (*probeAdapter) Available() (bool, string) {
+	if _, err := exec.LookPath("probe"); err != nil {
+		return false, "probe binary not on PATH (install: cargo install probe-ai)"
+	}
+	return true, ""
+}
+
+func (*probeAdapter) Run(ctx context.Context, queries []string, repoRoot string, topK int) []queryResult {
+	out := make([]queryResult, 0, len(queries))
+	for _, q := range queries {
+		start := time.Now()
+		cmd := exec.CommandContext(ctx, "probe", "search", "--paths-only", q, repoRoot)
+		var buf bytes.Buffer
+		cmd.Stdout = &buf
+		cmd.Stderr = io.Discard
+		err := cmd.Run()
+		latency := time.Since(start)
+		r := queryResult{Query: q, Latency: latency, LatencyMs: msFrom(latency)}
+		if err != nil {
+			r.Error = err.Error()
+			out = append(out, r)
+			continue
+		}
+		r.Hits = parseLines(buf.String(), repoRoot, topK)
+		out = append(out, r)
+	}
+	return out
+}
+
+// --- ColGREP --------------------------------------------------------
+
+type colgrepAdapter struct{ pythonBaselineAdapter }
+
+func (*colgrepAdapter) Name() string { return "colgrep" }
+
+func (a *colgrepAdapter) Available() (bool, string) {
+	return a.pythonModuleAvailable("colgrep",
+		"pip install colgrep (requires CUDA-capable GPU for the bundled ONNX model)")
+}
+
+func (a *colgrepAdapter) Run(ctx context.Context, queries []string, repoRoot string, topK int) []queryResult {
+	return a.runPythonModule(ctx, "colgrep", queries, repoRoot, topK)
+}
+
+// --- grepai ---------------------------------------------------------
+
+type grepaiAdapter struct{ pythonBaselineAdapter }
+
+func (*grepaiAdapter) Name() string { return "grepai" }
+
+func (a *grepaiAdapter) Available() (bool, string) {
+	return a.pythonModuleAvailable("grepai_cli",
+		"pip install grepai-cli")
+}
+
+func (a *grepaiAdapter) Run(ctx context.Context, queries []string, repoRoot string, topK int) []queryResult {
+	return a.runPythonModule(ctx, "grepai_cli", queries, repoRoot, topK)
+}
+
+// --- CodeRankEmbed Hybrid -------------------------------------------
+
+type coderankAdapter struct{ pythonBaselineAdapter }
+
+func (*coderankAdapter) Name() string { return "coderankembed" }
+
+func (a *coderankAdapter) Available() (bool, string) {
+	// CodeRankEmbed ships as a model on Hugging Face; the bench
+	// invoker is a small Python script we ship in
+	// bench/baselines/python/coderankembed_runner.py. The script
+	// requires `pip install sentence-transformers transformers
+	// torch` and is documented in bench/baselines/README.md.
+	return a.pythonModuleAvailable("sentence_transformers",
+		"pip install sentence-transformers transformers torch (multi-GB model download on first run)")
+}
+
+func (a *coderankAdapter) Run(ctx context.Context, queries []string, repoRoot string, topK int) []queryResult {
+	return a.runPythonScript(ctx, "bench/baselines/python/coderankembed_runner.py", queries, repoRoot, topK)
+}
+
+// --- semble ---------------------------------------------------------
+
+type sembleAdapter struct{ pythonBaselineAdapter }
+
+func (*sembleAdapter) Name() string { return "semble" }
+
+func (a *sembleAdapter) Available() (bool, string) {
+	return a.pythonModuleAvailable("semble",
+		"pip install semble")
+}
+
+func (a *sembleAdapter) Run(ctx context.Context, queries []string, repoRoot string, topK int) []queryResult {
+	return a.runPythonModule(ctx, "semble.cli", queries, repoRoot, topK)
+}
+
+// --- shared Python-adapter plumbing ---------------------------------
+
+// pythonBaselineAdapter embeds the common shell-out logic that all
+// Python-based baselines reuse. Per-baseline behaviour comes from
+// the module name + (optional) wrapper script path.
+type pythonBaselineAdapter struct{}
+
+// pythonModuleAvailable runs `python3 -c "import <mod>"`. Returns
+// true when the import succeeds; otherwise returns false with the
+// install hint so the smoke output is actionable.
+func (pythonBaselineAdapter) pythonModuleAvailable(module, installHint string) (bool, string) {
+	pythons := []string{"python3", "python"}
+	var lastErr string
+	for _, py := range pythons {
+		if _, err := exec.LookPath(py); err != nil {
+			continue
+		}
+		cmd := exec.Command(py, "-c", "import "+module)
+		cmd.Stderr = io.Discard
+		if err := cmd.Run(); err == nil {
+			return true, ""
+		} else {
+			lastErr = err.Error()
+		}
+	}
+	if lastErr == "" {
+		return false, "python3 not on PATH"
+	}
+	return false, fmt.Sprintf("python module %q not importable (%s)", module, installHint)
+}
+
+// runPythonModule invokes the module's CLI via `python3 -m <module>`,
+// forwarding the queries on stdin one per line. Mirrors how most
+// Python retrieval baselines we wrap accept input.
+func (pythonBaselineAdapter) runPythonModule(ctx context.Context, module string, queries []string, repoRoot string, topK int) []queryResult {
+	return pythonBaselineRun(ctx, []string{"-m", module}, queries, repoRoot, topK)
+}
+
+// runPythonScript invokes a wrapper script with the same I/O
+// convention. The script lives under bench/baselines/python/ and is
+// shipped with the harness so each adapter has a known-good entry
+// point.
+func (pythonBaselineAdapter) runPythonScript(ctx context.Context, scriptPath string, queries []string, repoRoot string, topK int) []queryResult {
+	return pythonBaselineRun(ctx, []string{scriptPath}, queries, repoRoot, topK)
+}
+
+// pythonBaselineRun is the shared dispatcher: pipes queries to the
+// Python process on stdin and parses one-hit-per-line JSON from
+// stdout. Each baseline's wrapper is responsible for emitting the
+// canonical JSON shape: {"query": "...", "hits": ["path1", ...]}.
+func pythonBaselineRun(ctx context.Context, args []string, queries []string, repoRoot string, topK int) []queryResult {
+	out := make([]queryResult, 0, len(queries))
+	for _, q := range queries {
+		start := time.Now()
+		// We invoke the script once per query so a per-query
+		// failure doesn't drag the whole run. Higher overhead than
+		// streaming but fundamentally honest.
+		baseArgs := append([]string{}, args...)
+		baseArgs = append(baseArgs, "--repo", repoRoot, "--top-k", fmt.Sprintf("%d", topK), "--query", q)
+		cmd := exec.CommandContext(ctx, "python3", baseArgs...)
+		var stdout, stderr bytes.Buffer
+		cmd.Stdout = &stdout
+		cmd.Stderr = &stderr
+		err := cmd.Run()
+		latency := time.Since(start)
+		r := queryResult{Query: q, Latency: latency, LatencyMs: msFrom(latency)}
+		if err != nil {
+			r.Error = fmt.Sprintf("%v: %s", err, strings.TrimSpace(stderr.String()))
+			out = append(out, r)
+			continue
+		}
+		r.Hits = parseLines(stdout.String(), repoRoot, topK)
+		out = append(out, r)
+	}
+	return out
+}
+
+// --- helpers --------------------------------------------------------
+
+// parseLines splits the adapter's stdout into one hit per line,
+// strips the repo-root prefix so all adapters return repo-relative
+// paths, dedups while preserving order, and caps at topK.
+func parseLines(s, repoRoot string, topK int) []string {
+	seen := map[string]bool{}
+	out := []string{}
+	for line := range strings.SplitSeq(s, "\n") {
+		line = strings.TrimSpace(line)
+		if line == "" {
+			continue
+		}
+		rel := strings.TrimPrefix(line, repoRoot+"/")
+		if seen[rel] {
+			continue
+		}
+		seen[rel] = true
+		out = append(out, rel)
+		if topK > 0 && len(out) >= topK {
+			break
+		}
+	}
+	return out
+}
+
+func msFrom(d time.Duration) float64 {
+	return float64(d.Microseconds()) / 1000.0
+}
+
+// adapterNames returns the names in canonical order for table
+// rendering / smoke output.
+func adapterNames() []string {
+	all := allAdapters()
+	names := make([]string, len(all))
+	for i, a := range all {
+		names[i] = a.Name()
+	}
+	sort.Strings(names)
+	return names
+}

bench/baselines/groundtruth.json

@@ -0,0 +1,37 @@
+{
+  "comment": "Per-query expected file paths against the gortex repo. NDCG@10 = mean over queries of DCG_at_10 / IDCG_at_10 where gain=1 for hits in this list and 0 otherwise. Mirrors the token-efficiency ground truth so the two benches are directly comparable.",
+  "queries": {
+    "AddObservation": [
+      "internal/savings/store.go"
+    ],
+    "IsSymbolQuery": [
+      "internal/search/rerank/query_kind.go"
+    ],
+    "FileCoherenceSignal": [
+      "internal/search/rerank/signals_file_coherence.go"
+    ],
+    "alphaFuse": [
+      "internal/search/hybrid.go"
+    ],
+    "savings dashboard rendering": [
+      "cmd/gortex/savings.go",
+      "internal/savings/events.go"
+    ],
+    "rerank pipeline default signals": [
+      "internal/search/rerank/pipeline.go"
+    ],
+    "Indexer Index method": [
+      "internal/indexer/indexer.go"
+    ],
+    "graph build edges": [
+      "internal/graph/graph.go"
+    ],
+    "MCP server start": [
+      "internal/mcp/server.go",
+      "cmd/gortex/mcp.go"
+    ],
+    "token counting tiktoken": [
+      "internal/tokens/tokens.go"
+    ]
+  }
+}

bench/baselines/main.go

@@ -0,0 +1,367 @@
+// Command baselines runs the per-baseline retrieval comparison:
+// dispatch the same query set through each registered adapter,
+// compare the per-query hit lists against a shared ground truth,
+// and emit a per-adapter NDCG@10 + latency table.
+//
+// Smoke mode (`--smoke`) skips actual runs and reports which adapters
+// are available on the local box — useful from CI to verify wiring
+// without paying for the heavy Python baselines.
+package main
+
+import (
+	"context"
+	"encoding/json"
+	"flag"
+	"fmt"
+	"math"
+	"os"
+	"path/filepath"
+	"sort"
+	"strings"
+	"time"
+)
+
+func main() {
+	repo := flag.String("repo", ".", "indexed corpus path")
+	queriesPath := flag.String("queries", "bench/baselines/queries.json", "JSON query set")
+	truthPath := flag.String("groundtruth", "bench/baselines/groundtruth.json", "JSON per-query expected file paths")
+	against := flag.String("against", "ripgrep,probe,colgrep,grepai,coderankembed,semble", "comma-separated adapter names to run (default: all)")
+	topK := flag.Int("top-k", 10, "top-K candidates per query (NDCG@10 uses K=10)")
+	out := flag.String("out", "", "output table path (default stdout)")
+	jsonOut := flag.String("json", "", "optional JSON metrics output")
+	format := flag.String("format", "markdown", "markdown | json")
+	smoke := flag.Bool("smoke", false, "skip runs; only probe availability of each requested adapter")
+	timeout := flag.Duration("timeout", 5*time.Minute, "per-adapter wall-clock cap")
+	flag.Parse()
+
+	requested, err := resolveAdapters(*against)
+	if err != nil {
+		die("%v", err)
+	}
+
+	if *smoke {
+		smokeReport(requested, *format, *out)
+		return
+	}
+
+	absRepo, err := filepath.Abs(*repo)
+	if err != nil {
+		die("repo path: %v", err)
+	}
+	if _, err := os.Stat(absRepo); err != nil {
+		die("repo path: %v", err)
+	}
+
+	queries, err := loadQueries(*queriesPath)
+	if err != nil {
+		die("queries: %v", err)
+	}
+	truth, err := loadGroundTruth(*truthPath)
+	if err != nil {
+		die("groundtruth: %v", err)
+	}
+
+	rows := make([]adapterRow, 0, len(requested))
+	for _, a := range requested {
+		fmt.Fprintf(os.Stderr, "[baselines] %s ... ", a.Name())
+		avail, why := a.Available()
+		if !avail {
+			fmt.Fprintf(os.Stderr, "skipped (%s)\n", why)
+			rows = append(rows, adapterRow{
+				Adapter: a.Name(),
+				Skipped: why,
+			})
+			continue
+		}
+		t0 := time.Now()
+		ctx, cancel := context.WithTimeout(context.Background(), *timeout)
+		results := a.Run(ctx, queries, absRepo, *topK)
+		cancel()
+		elapsed := time.Since(t0)
+
+		row := adapterRow{
+			Adapter:       a.Name(),
+			PerQuery:      results,
+			TotalDuration: elapsed.String(),
+		}
+		row.MedianLatencyMs = medianLatency(results)
+		row.NDCGAt10 = ndcgAt10(results, truth, *topK)
+		fmt.Fprintf(os.Stderr, "NDCG@10 %.3f · median %.1fms · total %.1fs\n",
+			row.NDCGAt10, row.MedianLatencyMs, elapsed.Seconds())
+		rows = append(rows, row)
+	}
+
+	var primary []byte
+	switch strings.ToLower(*format) {
+	case "markdown", "md":
+		primary = []byte(renderMarkdown(rows))
+	case "json":
+		primary = mustMarshalJSON(rows)
+	default:
+		die("unknown --format %q", *format)
+	}
+	if err := writeOutput(*out, primary); err != nil {
+		die("write output: %v", err)
+	}
+	if *jsonOut != "" {
+		if err := writeOutput(*jsonOut, mustMarshalJSON(rows)); err != nil {
+			die("write json: %v", err)
+		}
+	}
+}
+
+// adapterRow is the per-baseline outcome with the columns the
+// published table cares about.
+type adapterRow struct {
+	Adapter         string        `json:"adapter"`
+	NDCGAt10        float64       `json:"ndcg_at_10"`
+	MedianLatencyMs float64       `json:"median_latency_ms"`
+	TotalDuration   string        `json:"total_duration"`
+	PerQuery        []queryResult `json:"per_query,omitempty"`
+	Skipped         string        `json:"skipped,omitempty"`
+}
+
+// resolveAdapters expands the --against flag into the requested
+// adapter set, preserving the caller's order. Unknown names error
+// out so a typo doesn't silently drop a baseline.
+func resolveAdapters(spec string) ([]adapter, error) {
+	if strings.TrimSpace(spec) == "" {
+		return allAdapters(), nil
+	}
+	var out []adapter
+	for tok := range strings.SplitSeq(spec, ",") {
+		name := strings.TrimSpace(tok)
+		if name == "" {
+			continue
+		}
+		a := adapterByName(name)
+		if a == nil {
+			return nil, fmt.Errorf("unknown adapter %q (known: %s)",
+				name, strings.Join(adapterNames(), ", "))
+		}
+		out = append(out, a)
+	}
+	if len(out) == 0 {
+		return allAdapters(), nil
+	}
+	return out, nil
+}
+
+// smokeReport prints one row per requested adapter showing
+// available / not-available with the install hint. Output respects
+// --format; default markdown.
+func smokeReport(adapters []adapter, format, out string) {
+	if strings.ToLower(format) == "json" {
+		rows := make([]map[string]any, 0, len(adapters))
+		for _, a := range adapters {
+			avail, why := a.Available()
+			row := map[string]any{
+				"adapter":   a.Name(),
+				"available": avail,
+			}
+			if why != "" {
+				row["why"] = why
+			}
+			rows = append(rows, row)
+		}
+		raw, _ := json.MarshalIndent(rows, "", "  ")
+		_ = writeOutput(out, append(raw, '\n'))
+		return
+	}
+	var b strings.Builder
+	fmt.Fprintln(&b, "# Baseline-adapter smoke check")
+	fmt.Fprintln(&b)
+	fmt.Fprintln(&b, "| adapter | available | reason |")
+	fmt.Fprintln(&b, "|---------|:---------:|--------|")
+	for _, a := range adapters {
+		avail, why := a.Available()
+		mark := "✗"
+		if avail {
+			mark = "✓"
+		}
+		if why == "" {
+			why = "—"
+		}
+		fmt.Fprintf(&b, "| %s | %s | %s |\n", a.Name(), mark, why)
+	}
+	_ = writeOutput(out, []byte(b.String()))
+}
+
+// ndcgAt10 computes NDCG@10 across the result set. Per query: gain
+// is 1 for ground-truth hits, 0 otherwise; DCG = sum gain_i /
+// log2(i+2); IDCG = DCG of the perfect ordering. Mean across
+// queries; 0 when the result set is empty.
+func ndcgAt10(results []queryResult, truth map[string][]string, k int) float64 {
+	if len(results) == 0 {
+		return 0
+	}
+	var sum float64
+	counted := 0
+	for _, r := range results {
+		if r.Error != "" {
+			continue
+		}
+		expected := truth[r.Query]
+		if len(expected) == 0 {
+			continue
+		}
+		expSet := map[string]bool{}
+		for _, e := range expected {
+			expSet[e] = true
+		}
+		// DCG@k
+		var dcg float64
+		hits := r.Hits
+		if len(hits) > k {
+			hits = hits[:k]
+		}
+		for i, h := range hits {
+			if expSet[h] {
+				dcg += 1.0 / math.Log2(float64(i+2))
+			}
+		}
+		// IDCG@k: best case is min(len(expected), k) hits at the top.
+		idealHits := min(len(expected), k)
+		var idcg float64
+		for i := range idealHits {
+			idcg += 1.0 / math.Log2(float64(i+2))
+		}
+		if idcg == 0 {
+			continue
+		}
+		sum += dcg / idcg
+		counted++
+	}
+	if counted == 0 {
+		return 0
+	}
+	return sum / float64(counted)
+}
+
+func medianLatency(results []queryResult) float64 {
+	vs := make([]float64, 0, len(results))
+	for _, r := range results {
+		if r.Error == "" {
+			vs = append(vs, r.LatencyMs)
+		}
+	}
+	if len(vs) == 0 {
+		return 0
+	}
+	sort.Float64s(vs)
+	return vs[len(vs)/2]
+}
+
+// --- rendering ------------------------------------------------------
+
+func renderMarkdown(rows []adapterRow) string {
+	var b strings.Builder
+	fmt.Fprintln(&b, "# Retrieval-baselines NDCG@10 + speed")
+	fmt.Fprintln(&b)
+	fmt.Fprintln(&b, "_NDCG@10 against the ground-truth set + median per-query latency for each baseline. Adapters reporting `skipped` weren't available on this box; see the install hints in `bench/baselines/README.md`._")
+	fmt.Fprintln(&b)
+	fmt.Fprintln(&b, "| adapter | NDCG@10 | median latency | total | status |")
+	fmt.Fprintln(&b, "|---------|--------:|---------------:|------:|--------|")
+	for _, r := range rows {
+		if r.Skipped != "" {
+			fmt.Fprintf(&b, "| %s | — | — | — | skipped: %s |\n", r.Adapter, r.Skipped)
+			continue
+		}
+		fmt.Fprintf(&b, "| %s | %.3f | %s | %s | ✓ |\n",
+			r.Adapter, r.NDCGAt10, fmtMs(r.MedianLatencyMs), r.TotalDuration)
+	}
+	fmt.Fprintln(&b)
+	fmt.Fprintln(&b, summaryLine(rows))
+	return b.String()
+}
+
+func summaryLine(rows []adapterRow) string {
+	ran := 0
+	bestName := ""
+	var bestScore float64
+	for _, r := range rows {
+		if r.Skipped != "" {
+			continue
+		}
+		ran++
+		if r.NDCGAt10 > bestScore {
+			bestScore = r.NDCGAt10
+			bestName = r.Adapter
+		}
+	}
+	if ran == 0 {
+		return "_no adapters available — install at least one baseline (see bench/baselines/README.md)_"
+	}
+	return fmt.Sprintf("**Summary:** %d/%d adapter(s) ran. Best NDCG@10: %s @ %.3f.",
+		ran, len(rows), bestName, bestScore)
+}
+
+func fmtMs(v float64) string {
+	if v == 0 {
+		return "—"
+	}
+	if v < 1 {
+		return fmt.Sprintf("%.2fms", v)
+	}
+	if v < 1000 {
+		return fmt.Sprintf("%.1fms", v)
+	}
+	return fmt.Sprintf("%.2fs", v/1000.0)
+}
+
+// --- helpers --------------------------------------------------------
+
+func die(format string, args ...any) {
+	fmt.Fprintf(os.Stderr, "baselines: "+format+"\n", args...)
+	os.Exit(1)
+}
+
+func writeOutput(path string, body []byte) error {
+	if path == "" {
+		_, err := os.Stdout.Write(body)
+		return err
+	}
+	if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
+		return err
+	}
+	return os.WriteFile(path, body, 0o644)
+}
+
+func mustMarshalJSON(rows []adapterRow) []byte {
+	b, err := json.MarshalIndent(rows, "", "  ")
+	if err != nil {
+		die("marshal json: %v", err)
+	}
+	return append(b, '\n')
+}
+
+func loadQueries(path string) ([]string, error) {
+	raw, err := os.ReadFile(path)
+	if err != nil {
+		return nil, err
+	}
+	var doc struct {
+		Queries []string `json:"queries"`
+	}
+	if err := json.Unmarshal(raw, &doc); err != nil {
+		return nil, fmt.Errorf("parse %s: %w", path, err)
+	}
+	return doc.Queries, nil
+}
+
+func loadGroundTruth(path string) (map[string][]string, error) {
+	raw, err := os.ReadFile(path)
+	if err != nil {
+		return nil, err
+	}
+	var doc struct {
+		Queries map[string][]string `json:"queries"`
+	}
+	if err := json.Unmarshal(raw, &doc); err != nil {
+		return nil, fmt.Errorf("parse %s: %w", path, err)
+	}
+	if doc.Queries == nil {
+		doc.Queries = map[string][]string{}
+	}
+	return doc.Queries, nil
+}

bench/baselines/main_test.go

@@ -0,0 +1,300 @@
+package main
+
+import (
+	"context"
+	"encoding/json"
+	"math"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+	"time"
+)
+
+func TestAllAdapters_RegisteredInStableOrder(t *testing.T) {
+	got := allAdapters()
+	want := []string{"ripgrep", "probe", "colgrep", "grepai", "coderankembed", "semble"}
+	if len(got) != len(want) {
+		t.Fatalf("adapter count = %d, want %d", len(got), len(want))
+	}
+	for i, a := range got {
+		if a.Name() != want[i] {
+			t.Errorf("adapter[%d] = %q, want %q", i, a.Name(), want[i])
+		}
+	}
+}
+
+func TestAdapterByName(t *testing.T) {
+	if a := adapterByName("ripgrep"); a == nil || a.Name() != "ripgrep" {
+		t.Errorf("adapterByName(ripgrep) = %v", a)
+	}
+	if a := adapterByName("RIPGREP"); a == nil { // case-insensitive
+		t.Error("adapterByName should be case-insensitive")
+	}
+	if a := adapterByName("nonexistent"); a != nil {
+		t.Errorf("adapterByName(nonexistent) should be nil, got %v", a)
+	}
+}
+
+func TestResolveAdapters_EmptyMeansAll(t *testing.T) {
+	got, err := resolveAdapters("")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(got) != len(allAdapters()) {
+		t.Errorf("empty spec = %d adapters, want %d", len(got), len(allAdapters()))
+	}
+}
+
+func TestResolveAdapters_Subset(t *testing.T) {
+	got, err := resolveAdapters("ripgrep,probe")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(got) != 2 || got[0].Name() != "ripgrep" || got[1].Name() != "probe" {
+		t.Errorf("subset = %v, want [ripgrep, probe] in order", names(got))
+	}
+}
+
+func TestResolveAdapters_UnknownErrors(t *testing.T) {
+	if _, err := resolveAdapters("ripgrep,bogus"); err == nil {
+		t.Error("expected error for unknown adapter")
+	}
+}
+
+func TestNDCGAt10_PerfectRanking(t *testing.T) {
+	results := []queryResult{
+		{Query: "q", Hits: []string{"a", "b", "c"}},
+	}
+	truth := map[string][]string{"q": {"a", "b"}}
+	got := ndcgAt10(results, truth, 10)
+	// IDCG = 1/log2(2) + 1/log2(3) = 1 + 0.6309 = 1.6309
+	// DCG  = 1/log2(2) + 1/log2(3) + 0 = same = 1.6309
+	// NDCG = 1.0
+	if got < 0.99 || got > 1.01 {
+		t.Errorf("perfect ranking NDCG@10 = %.4f, want 1.0", got)
+	}
+}
+
+func TestNDCGAt10_WrongOrderPenalized(t *testing.T) {
+	results := []queryResult{
+		{Query: "q", Hits: []string{"miss1", "miss2", "a", "b"}},
+	}
+	truth := map[string][]string{"q": {"a", "b"}}
+	got := ndcgAt10(results, truth, 10)
+	// DCG  = 0 + 0 + 1/log2(4) + 1/log2(5) ≈ 0.5 + 0.431 = 0.931
+	// IDCG = 1/log2(2) + 1/log2(3) ≈ 1.0 + 0.631 = 1.631
+	// NDCG = 0.931/1.631 ≈ 0.571
+	if got > 0.7 {
+		t.Errorf("wrong-order NDCG@10 = %.4f, expected <0.7", got)
+	}
+	if got < 0.4 {
+		t.Errorf("wrong-order NDCG@10 = %.4f, expected >0.4", got)
+	}
+}
+
+func TestNDCGAt10_AllMissesScoreZero(t *testing.T) {
+	results := []queryResult{{Query: "q", Hits: []string{"x", "y"}}}
+	truth := map[string][]string{"q": {"a", "b"}}
+	if got := ndcgAt10(results, truth, 10); got != 0 {
+		t.Errorf("all-miss NDCG@10 = %.4f, want 0", got)
+	}
+}
+
+func TestNDCGAt10_EmptyTruthSkipped(t *testing.T) {
+	results := []queryResult{
+		{Query: "q1", Hits: []string{"a"}},
+		{Query: "q2", Hits: []string{"a"}},
+	}
+	truth := map[string][]string{"q2": {"a"}}
+	// q1 has no truth → skipped; q2 has 1 hit at position 0 → NDCG=1.
+	got := ndcgAt10(results, truth, 10)
+	if math.Abs(got-1.0) > 0.01 {
+		t.Errorf("mixed-truth NDCG@10 = %.4f, want ~1.0 (q1 should be skipped)", got)
+	}
+}
+
+func TestNDCGAt10_ErroredQuerySkipped(t *testing.T) {
+	results := []queryResult{
+		{Query: "q1", Error: "boom"},
+		{Query: "q2", Hits: []string{"a"}},
+	}
+	truth := map[string][]string{"q1": {"a"}, "q2": {"a"}}
+	// q1 errored → skipped; only q2 counts → NDCG=1.
+	got := ndcgAt10(results, truth, 10)
+	if math.Abs(got-1.0) > 0.01 {
+		t.Errorf("errored-skipped NDCG@10 = %.4f, want ~1.0", got)
+	}
+}
+
+func TestMedianLatency(t *testing.T) {
+	results := []queryResult{
+		{LatencyMs: 30},
+		{LatencyMs: 10},
+		{LatencyMs: 20},
+	}
+	if got := medianLatency(results); got != 20 {
+		t.Errorf("median = %.2f, want 20", got)
+	}
+}
+
+func TestMedianLatency_ErrorsExcluded(t *testing.T) {
+	results := []queryResult{
+		{LatencyMs: 30, Error: "boom"},
+		{LatencyMs: 10},
+		{LatencyMs: 20},
+	}
+	if got := medianLatency(results); got != 20 {
+		t.Errorf("median (errors excluded) = %.2f, want 20", got)
+	}
+}
+
+func TestParseLines_DedupAndCap(t *testing.T) {
+	body := `/repo/a.go
+/repo/b.go
+/repo/a.go
+/repo/c.go
+/repo/d.go
+`
+	got := parseLines(body, "/repo", 3)
+	want := []string{"a.go", "b.go", "c.go"}
+	if len(got) != 3 {
+		t.Fatalf("got %v, want %v", got, want)
+	}
+	for i := range want {
+		if got[i] != want[i] {
+			t.Errorf("got[%d] = %q, want %q", i, got[i], want[i])
+		}
+	}
+}
+
+func TestParseLines_TopKZeroMeansUnlimited(t *testing.T) {
+	body := "a\nb\nc\nd\ne\nf\ng\n"
+	got := parseLines(body, "", 0)
+	if len(got) != 7 {
+		t.Errorf("got %d lines, want 7 (topK=0 unlimited)", len(got))
+	}
+}
+
+func TestRenderMarkdown_HasHeaderAndRows(t *testing.T) {
+	rows := []adapterRow{
+		{Adapter: "ripgrep", NDCGAt10: 0.45, MedianLatencyMs: 12.3, TotalDuration: "1.2s"},
+		{Adapter: "colgrep", Skipped: "python module not importable"},
+	}
+	md := renderMarkdown(rows)
+	for _, want := range []string{
+		"# Retrieval-baselines NDCG@10",
+		"| ripgrep |",
+		"0.450",
+		"| colgrep |",
+		"skipped:",
+		"**Summary:**",
+	} {
+		if !strings.Contains(md, want) {
+			t.Errorf("markdown missing %q\n----\n%s", want, md)
+		}
+	}
+}
+
+func TestSmokeReport_Markdown(t *testing.T) {
+	dir := t.TempDir()
+	out := filepath.Join(dir, "smoke.md")
+	smokeReport([]adapter{&ripgrepAdapter{}, &colgrepAdapter{}}, "markdown", out)
+	raw, err := os.ReadFile(out)
+	if err != nil {
+		t.Fatal(err)
+	}
+	body := string(raw)
+	for _, want := range []string{
+		"# Baseline-adapter smoke check",
+		"| ripgrep |",
+		"| colgrep |",
+	} {
+		if !strings.Contains(body, want) {
+			t.Errorf("smoke markdown missing %q\n----\n%s", want, body)
+		}
+	}
+}
+
+func TestSmokeReport_JSON(t *testing.T) {
+	dir := t.TempDir()
+	out := filepath.Join(dir, "smoke.json")
+	smokeReport([]adapter{&ripgrepAdapter{}}, "json", out)
+	raw, err := os.ReadFile(out)
+	if err != nil {
+		t.Fatal(err)
+	}
+	var got []map[string]any
+	if err := json.Unmarshal(raw, &got); err != nil {
+		t.Fatalf("smoke JSON unparseable: %v\n%s", err, raw)
+	}
+	if len(got) != 1 || got[0]["adapter"] != "ripgrep" {
+		t.Errorf("smoke JSON shape wrong: %v", got)
+	}
+	if _, ok := got[0]["available"]; !ok {
+		t.Errorf("smoke JSON missing 'available' field: %v", got)
+	}
+}
+
+func TestRipgrepAdapter_NoMatchesReturnsEmpty(t *testing.T) {
+	a := &ripgrepAdapter{}
+	avail, _ := a.Available()
+	if !avail {
+		t.Skip("rg not installed on this box")
+	}
+	dir := t.TempDir()
+	if err := os.WriteFile(filepath.Join(dir, "f.go"), []byte("package x\n"), 0o644); err != nil {
+		t.Fatal(err)
+	}
+	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+	defer cancel()
+	out := a.Run(ctx, []string{"definitely_not_in_the_corpus_XYZ"}, dir, 10)
+	if len(out) != 1 {
+		t.Fatalf("got %d results, want 1", len(out))
+	}
+	if out[0].Error != "" {
+		t.Errorf("no-match should not error, got %q", out[0].Error)
+	}
+	if len(out[0].Hits) != 0 {
+		t.Errorf("no-match should yield empty hits, got %v", out[0].Hits)
+	}
+}
+
+func TestLoadQueries(t *testing.T) {
+	dir := t.TempDir()
+	path := filepath.Join(dir, "q.json")
+	if err := os.WriteFile(path, []byte(`{"queries":["a","b","c"]}`), 0o644); err != nil {
+		t.Fatal(err)
+	}
+	got, err := loadQueries(path)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(got) != 3 {
+		t.Errorf("got %d, want 3", len(got))
+	}
+}
+
+func TestLoadGroundTruth(t *testing.T) {
+	dir := t.TempDir()
+	path := filepath.Join(dir, "gt.json")
+	body := `{"queries":{"q":["a.go"]}}`
+	if err := os.WriteFile(path, []byte(body), 0o644); err != nil {
+		t.Fatal(err)
+	}
+	got, err := loadGroundTruth(path)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(got["q"]) != 1 || got["q"][0] != "a.go" {
+		t.Errorf("got %v, want q:[a.go]", got)
+	}
+}
+
+func names(adapters []adapter) []string {
+	out := make([]string, len(adapters))
+	for i, a := range adapters {
+		out[i] = a.Name()
+	}
+	return out
+}

bench/baselines/python/coderankembed_runner.py

@@ -0,0 +1,92 @@
+#!/usr/bin/env python3
+"""CodeRankEmbed Hybrid baseline runner.
+
+Wrapper that lets the Go-side bench/baselines harness invoke
+CodeRankEmbed Hybrid without per-baseline Go code growing model-
+download logic. Usage:
+
+    python3 bench/baselines/python/coderankembed_runner.py \\
+        --repo PATH --query "validateToken" --top-k 10
+
+Emits one repo-relative path per line on stdout. Errors go to
+stderr; non-zero exit when the model isn't available.
+
+Install: `pip install sentence-transformers transformers torch`.
+First run downloads the CodeRankEmbed model (~440 MB).
+"""
+
+import argparse
+import os
+import sys
+from pathlib import Path
+
+
+def main() -> int:
+    ap = argparse.ArgumentParser()
+    ap.add_argument("--repo", required=True, help="indexed corpus path")
+    ap.add_argument("--query", required=True, help="single query string")
+    ap.add_argument("--top-k", type=int, default=10)
+    args = ap.parse_args()
+
+    try:
+        from sentence_transformers import SentenceTransformer
+    except ImportError as e:
+        print(
+            f"coderankembed_runner: missing dependency ({e}). "
+            "pip install sentence-transformers transformers torch",
+            file=sys.stderr,
+        )
+        return 2
+
+    model = SentenceTransformer("nomic-ai/CodeRankEmbed", trust_remote_code=True)
+
+    # Index every file under repo (cheap for sub-million LoC; the
+    # ground-truth fixture is the gortex repo itself).
+    repo = Path(args.repo).resolve()
+    paths: list[Path] = []
+    texts: list[str] = []
+    for p in repo.rglob("*"):
+        if not p.is_file():
+            continue
+        if any(seg.startswith(".") for seg in p.relative_to(repo).parts):
+            continue
+        if p.suffix.lower() not in {
+            ".go", ".py", ".ts", ".tsx", ".js", ".jsx", ".rs",
+            ".java", ".kt", ".swift", ".rb", ".cs", ".cpp", ".c",
+            ".h", ".hpp", ".md", ".yaml", ".yml", ".json",
+        }:
+            continue
+        try:
+            text = p.read_text(errors="ignore")
+        except OSError:
+            continue
+        if not text.strip():
+            continue
+        paths.append(p)
+        texts.append(text[:8000])  # truncate to keep the embed cheap
+
+    if not texts:
+        print(
+            "coderankembed_runner: no indexable files under repo",
+            file=sys.stderr,
+        )
+        return 1
+
+    embeds = model.encode(texts, show_progress_bar=False, convert_to_numpy=True)
+    qe = model.encode([args.query], show_progress_bar=False, convert_to_numpy=True)[0]
+
+    # Cosine similarity → rank.
+    import numpy as np
+
+    sims = embeds @ qe / (
+        (np.linalg.norm(embeds, axis=1) * np.linalg.norm(qe)) + 1e-12
+    )
+    order = np.argsort(-sims)[: args.top_k]
+    for idx in order:
+        rel = paths[idx].relative_to(repo)
+        print(rel.as_posix())
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

bench/baselines/queries.json

@@ -0,0 +1,15 @@
+{
+  "comment": "Query set for the retrieval-baselines NDCG@10 bench. Same shape as the token-efficiency set so the two benches stay comparable: an adapter that scores well here should also produce low tokens-per-response there.",
+  "queries": [
+    "AddObservation",
+    "IsSymbolQuery",
+    "FileCoherenceSignal",
+    "alphaFuse",
+    "savings dashboard rendering",
+    "rerank pipeline default signals",
+    "Indexer Index method",
+    "graph build edges",
+    "MCP server start",
+    "token counting tiktoken"
+  ]
+}