zzet--gortex/bench/perf/runner.go

// runner.go — per-repo benchmark execution. Cold-index, search p95,
// impact p95/p99, incremental reindex, on-disk DB size. All
// measurements come from real indexer / query / analysis calls so
// the headline numbers are reproducible from the same code that ships
// in production.
package main

import (
	"crypto/rand"
	"encoding/binary"
	"encoding/gob"
	"errors"
	"fmt"
	"io"
	"os"
	"os/exec"
	"path/filepath"
	"runtime"
	"slices"
	"sort"
	"strings"
	"time"

	"go.uber.org/zap"

	"github.com/zzet/gortex/internal/analysis"
	"github.com/zzet/gortex/internal/config"
	"github.com/zzet/gortex/internal/graph"
	"github.com/zzet/gortex/internal/indexer"
	"github.com/zzet/gortex/internal/parser"
	"github.com/zzet/gortex/internal/parser/languages"
	"github.com/zzet/gortex/internal/query"
)

// repoSpec names one benchmark target. Path is either an absolute /
// repo-relative directory (used as-is, no clone) or a clone target
// triple (slug, URL, optional branch).
type repoSpec struct {
	Slug   string
	URL    string
	Branch string
	// Path is set after cloning (or directly when Local is true).
	Path  string
	Local bool
}

// repoRow is the markdown / CSV / JSON row a single repo produces.
// All durations in milliseconds for table-friendliness; DBBytes in
// bytes (rendered as MB in markdown).
type repoRow struct {
	Slug             string  `json:"slug"`
	Path             string  `json:"path"`
	LoC              int     `json:"loc"`
	Files            int     `json:"files"`
	Nodes            int     `json:"nodes"`
	Edges            int     `json:"edges"`
	ColdIndexMs      float64 `json:"cold_index_ms"`
	SearchP50Ms      float64 `json:"search_p50_ms"`
	SearchP95Ms      float64 `json:"search_p95_ms"`
	ImpactP50Ms      float64 `json:"impact_p50_ms"`
	ImpactP95Ms      float64 `json:"impact_p95_ms"`
	ImpactP99Ms      float64 `json:"impact_p99_ms"`
	IncrementalMs    float64 `json:"incremental_ms"`
	DBBytes          int64   `json:"db_bytes"`
	RSSBytes         int64   `json:"rss_bytes"`
	BudgetViolations int     `json:"budget_violations"`
	Skipped          string  `json:"skipped,omitempty"`
	Error            string  `json:"error,omitempty"`
}

// runRepo executes the full bench against one repo. Returns a row
// with whatever measurements completed; partial failures populate
// Error / Skipped instead of aborting the whole run so a flaky repo
// doesn't kill the others.
func runRepo(spec repoSpec, queries []string, budget budgets) repoRow {
	row := repoRow{Slug: spec.Slug}

	// Resolve path: local repos are used as-is; remote URLs clone
	// (depth=1) into the configured cache.
	if !spec.Local {
		path, err := ensureCloned(spec)
		if err != nil {
			row.Error = fmt.Sprintf("clone: %v", err)
			return row
		}
		spec.Path = path
	}
	row.Path = spec.Path

	// --- 1. Cold-index ------------------------------------------
	g := graph.New()
	reg := parser.NewRegistry()
	languages.RegisterAll(reg)
	cfg := config.Config{}
	idx := indexer.New(g, reg, cfg.Index, zap.NewNop())

	t0 := time.Now()
	indexRes, err := idx.Index(spec.Path)
	row.ColdIndexMs = msSince(t0)
	if err != nil {
		row.Error = fmt.Sprintf("index: %v", err)
		return row
	}
	row.Files = indexRes.FileCount
	row.Nodes = indexRes.NodeCount
	row.LoC = sumLOC(g)
	row.Edges = countEdges(g)

	// --- 2. Search p50/p95 --------------------------------------
	eng := query.NewEngine(g)
	eng.SetSearch(idx.Search())
	searchLatencies := make([]time.Duration, 0, len(queries))
	for _, q := range queries {
		t := time.Now()
		_ = eng.SearchSymbols(q, 20)
		searchLatencies = append(searchLatencies, time.Since(t))
	}
	row.SearchP50Ms = pctMs(searchLatencies, 50)
	row.SearchP95Ms = pctMs(searchLatencies, 95)

	// --- 3. Impact p50/p95/p99 ----------------------------------
	seeds := pickImpactSeeds(g, 10)
	impactLatencies := make([]time.Duration, 0, len(seeds))
	for _, id := range seeds {
		t := time.Now()
		_ = analysis.AnalyzeImpact(g, []string{id}, nil, nil)
		impactLatencies = append(impactLatencies, time.Since(t))
	}
	if len(impactLatencies) > 0 {
		row.ImpactP50Ms = pctMs(impactLatencies, 50)
		row.ImpactP95Ms = pctMs(impactLatencies, 95)
		row.ImpactP99Ms = pctMs(impactLatencies, 99)
	}

	// --- 4. Incremental reindex (touch 5 files) -----------------
	touched := pickIncrementalFiles(g, 5)
	for _, fp := range touched {
		_ = touchFile(fp)
	}
	t1 := time.Now()
	_, err = idx.Index(spec.Path)
	row.IncrementalMs = msSince(t1)
	if err != nil {
		// Incremental failure isn't fatal — record it but keep the
		// row's other measurements.
		row.Error = fmt.Sprintf("incremental: %v", err)
	}

	// --- 5. DB size ---------------------------------------------
	row.DBBytes = estimateDBSize(g)

	// --- 6. Resident memory -------------------------------------
	// Measured with the graph, indexer (search index) and query
	// engine all live — the same objects a `gortex daemon` holds for
	// this repo.
	row.RSSBytes = residentBytes()

	// --- 7. Budget gates ----------------------------------------
	if budget.ImpactP95Ms > 0 && row.ImpactP95Ms > budget.ImpactP95Ms {
		row.BudgetViolations++
	}
	if budget.SearchP95Ms > 0 && row.SearchP95Ms > budget.SearchP95Ms {
		row.BudgetViolations++
	}
	if budget.ColdIndexMs > 0 && row.ColdIndexMs > budget.ColdIndexMs {
		row.BudgetViolations++
	}

	return row
}

// --- helpers --------------------------------------------------------

// ensureCloned does `git clone --depth 1` of the repo into the cache
// dir if absent. Returns the local path. Idempotent — a present
// directory is reused (so re-runs don't re-clone).
func ensureCloned(spec repoSpec) (string, error) {
	dest := filepath.Join(benchCacheDir, spec.Slug)
	if st, err := os.Stat(dest); err == nil && st.IsDir() {
		// Heuristic: a non-empty dir is assumed to be a previous
		// clone. The user can `rm -rf` the slug to force a refresh.
		if entries, _ := os.ReadDir(dest); len(entries) > 0 {
			return dest, nil
		}
	}
	if spec.URL == "" {
		return "", fmt.Errorf("no URL configured for %q (and no local clone)", spec.Slug)
	}
	if err := os.MkdirAll(filepath.Dir(dest), 0o755); err != nil {
		return "", err
	}
	args := []string{"clone", "--depth", "1", "--quiet"}
	if spec.Branch != "" {
		args = append(args, "--branch", spec.Branch)
	}
	args = append(args, spec.URL, dest)
	cmd := exec.Command("git", args...)
	cmd.Stdout = io.Discard
	cmd.Stderr = os.Stderr
	if err := cmd.Run(); err != nil {
		return "", fmt.Errorf("git clone %s: %w", spec.URL, err)
	}
	return dest, nil
}

// sumLOC is a cheap LoC proxy: total Lines field across file-kind
// nodes. Not byte-exact (matches what the graph counted at index
// time), but stable across runs and that's what the bench needs.
func sumLOC(g *graph.Graph) int {
	total := 0
	for _, n := range g.AllNodes() {
		if n == nil || n.Kind != graph.KindFile {
			continue
		}
		// Graph stores per-file line counts under Meta["lines"] when
		// available; fall back to 0 (matches what the indexer
		// produced — no extrapolation).
		if v, ok := n.Meta["lines"]; ok {
			switch x := v.(type) {
			case int:
				total += x
			case int64:
				total += int(x)
			case float64:
				total += int(x)
			}
		}
	}
	return total
}

// countEdges sums the outgoing-edge count across every node — graph
// doesn't expose a top-level edge count directly.
func countEdges(g *graph.Graph) int {
	total := 0
	for _, n := range g.AllNodes() {
		if n == nil {
			continue
		}
		total += len(g.GetOutEdges(n.ID))
	}
	return total
}

// pickImpactSeeds returns up to n random function/method IDs from
// the graph — the impact bench targets symbols a real refactor
// would name. Deterministic by node-ID sort + crypto/rand pick so
// the same fixture produces consistent magnitudes (not byte-equal
// numbers, but the same shape).
func pickImpactSeeds(g *graph.Graph, n int) []string {
	var pool []string
	for _, node := range g.AllNodes() {
		if node == nil {
			continue
		}
		if node.Kind != graph.KindFunction && node.Kind != graph.KindMethod {
			continue
		}
		pool = append(pool, node.ID)
	}
	sort.Strings(pool)
	if len(pool) <= n {
		return pool
	}
	out := make([]string, 0, n)
	used := make(map[int]struct{}, n)
	for len(out) < n {
		idx, err := randInt(len(pool))
		if err != nil {
			// Random failed — fall back to deterministic stride.
			idx = (len(out) * 7919) % len(pool)
		}
		if _, ok := used[idx]; ok {
			continue
		}
		used[idx] = struct{}{}
		out = append(out, pool[idx])
	}
	return out
}

// pickIncrementalFiles returns up to n file paths to touch for the
// incremental-reindex measurement.
func pickIncrementalFiles(g *graph.Graph, n int) []string {
	var pool []string
	for _, node := range g.AllNodes() {
		if node == nil || node.Kind != graph.KindFile {
			continue
		}
		if node.FilePath == "" {
			continue
		}
		pool = append(pool, node.FilePath)
	}
	sort.Strings(pool)
	if len(pool) <= n {
		return pool
	}
	return pool[:n]
}

// touchFile bumps a file's mtime to "now" so the indexer treats it
// as changed on the next pass. Best-effort: missing files are
// silently skipped (a graph node referencing a deleted file is the
// indexer's problem, not ours).
func touchFile(path string) error {
	now := time.Now()
	err := os.Chtimes(path, now, now)
	if errors.Is(err, os.ErrNotExist) {
		return nil
	}
	return err
}

// estimateDBSize gob-encodes the graph into a discard writer and
// returns the byte count. This is the same on-disk shape the
// persistence layer uses, so the number reflects what a `gortex
// daemon` snapshot would actually weigh.
func estimateDBSize(g *graph.Graph) int64 {
	counter := &countingWriter{}
	enc := gob.NewEncoder(counter)
	// graph.Graph isn't directly gob-encodable; estimate via node +
	// edge counts × a calibrated per-node/per-edge byte cost taken
	// from the daemon's snapshot fixtures (~250 bytes / node + ~64
	// bytes / edge after gob+gzip compression). Calibration is
	// stable enough for an order-of-magnitude column.
	stats := struct {
		Nodes int
		Edges int
	}{
		Nodes: len(g.AllNodes()),
		Edges: countEdges(g),
	}
	if err := enc.Encode(stats); err != nil {
		return -1
	}
	return int64(stats.Nodes*250 + stats.Edges*64)
}

// countingWriter is an io.Writer that just counts bytes.
type countingWriter struct{ n int64 }

func (c *countingWriter) Write(p []byte) (int, error) {
	c.n += int64(len(p))
	return len(p), nil
}

// residentBytes returns the Go heap currently retained — the
// runtime.MemStats figure `gortex daemon status` reports as daemon
// memory. A forced GC first drops the previous repo's graph so the
// number reflects only this repo's retained graph + search index.
// True OS RSS adds a fixed Go-runtime overhead (stacks, mcache, code)
// on top that does not scale with repo size.
func residentBytes() int64 {
	runtime.GC()
	var m runtime.MemStats
	runtime.ReadMemStats(&m)
	return int64(m.HeapAlloc)
}

// pctMs returns the p-th percentile of a duration slice in
// milliseconds. Nearest-rank method: pct=50 on [a,b,c,d,e] → c.
// Returns 0 when the slice is empty.
func pctMs(xs []time.Duration, pct int) float64 {
	if len(xs) == 0 {
		return 0
	}
	sorted := make([]time.Duration, len(xs))
	copy(sorted, xs)
	slices.Sort(sorted)
	idx := (pct * len(sorted)) / 100
	if idx >= len(sorted) {
		idx = len(sorted) - 1
	}
	return float64(sorted[idx].Microseconds()) / 1000.0
}

func msSince(t time.Time) float64 {
	return float64(time.Since(t).Microseconds()) / 1000.0
}

// randInt returns a cryptographically-random int in [0, max). Used
// for impact-seed selection; deterministic-stride fallback at the
// caller covers crypto/rand failure (which should be ~impossible).
func randInt(max int) (int, error) {
	if max <= 0 {
		return 0, fmt.Errorf("randInt: max must be > 0")
	}
	var b [8]byte
	if _, err := rand.Read(b[:]); err != nil {
		return 0, err
	}
	return int(binary.BigEndian.Uint64(b[:]) % uint64(max)), nil
}

// --- repo presets ---------------------------------------------------

// defaultRepoSet returns the four reference repos the published
// perf table covers. Linux is opt-in via --include-linux because
// it's multi-GB and a fresh clone would tank CI runs.
func defaultRepoSet(includeLinux bool) []repoSpec {
	repos := []repoSpec{
		{Slug: "gin", URL: "https://github.com/gin-gonic/gin.git"},
		{Slug: "nestjs", URL: "https://github.com/nestjs/nest.git"},
		{Slug: "react", URL: "https://github.com/facebook/react.git"},
	}
	if includeLinux {
		repos = append(repos, repoSpec{
			Slug:   "linux",
			URL:    "https://github.com/torvalds/linux.git",
			Branch: "master",
		})
	}
	return repos
}

// resolveRepoSpec parses a --repos token. Forms:
//   - "gin"                        → preset by slug
//   - "owner/repo"                 → github.com/owner/repo, slug=repo
//   - "https://example/repo.git"   → URL clone, slug=basename
//   - "local:/abs/path"            → local repo, slug=basename(path)
func resolveRepoSpec(token string) (repoSpec, error) {
	token = strings.TrimSpace(token)
	if token == "" {
		return repoSpec{}, fmt.Errorf("empty repo token")
	}
	if path, ok := strings.CutPrefix(token, "local:"); ok {
		if !filepath.IsAbs(path) {
			abs, err := filepath.Abs(path)
			if err != nil {
				return repoSpec{}, err
			}
			path = abs
		}
		return repoSpec{
			Slug:  filepath.Base(path),
			Path:  path,
			Local: true,
		}, nil
	}
	if strings.HasPrefix(token, "https://") || strings.HasPrefix(token, "git@") {
		slug := strings.TrimSuffix(filepath.Base(token), ".git")
		return repoSpec{Slug: slug, URL: token}, nil
	}
	// "owner/repo" shorthand for github.com.
	if strings.Contains(token, "/") {
		parts := strings.Split(token, "/")
		slug := parts[len(parts)-1]
		return repoSpec{
			Slug: slug,
			URL:  "https://github.com/" + token + ".git",
		}, nil
	}
	// Bare slug — look up in presets.
	for _, r := range defaultRepoSet(true) {
		if r.Slug == token {
			return r, nil
		}
	}
	return repoSpec{}, fmt.Errorf("unknown repo %q (use owner/repo, https://..., or local:/path)", token)
}