zzet--gortex/bench/daemon-latency/main.go

// Command daemon-latency measures per-tool MCP dispatch latency.
// Builds an in-process MCP server against a target corpus, fires N
// `CallTool` invocations per tool, reports p50 / p95 / p99 per tool
// and a top-line summary.
//
// What it measures: tool-handler latency end-to-end through the
// real MCP dispatch path (`Handler.CallTool` invoked via the same
// `server.MCPServer` the production stdio / HTTP / daemon
// front-ends use). What it does NOT measure: stdio framing,
// daemon socket dispatch, JSON-RPC envelope overhead. Those add a
// small constant per call (typically <1 ms on a warm pipe); the
// handler latency dominates the user-perceived response time.
//
// The bench therefore reflects "daemon-mode handler cost", which
// is the load-bearing number for the daemon-latency publication.
package main

import (
	"context"
	"encoding/csv"
	"encoding/json"
	"flag"
	"fmt"
	"os"
	"path/filepath"
	"slices"
	"sort"
	"strings"
	"time"

	"go.uber.org/zap"

	"github.com/zzet/gortex/internal/config"
	"github.com/zzet/gortex/internal/graph"
	"github.com/zzet/gortex/internal/indexer"
	gortexmcp "github.com/zzet/gortex/internal/mcp"
	"github.com/zzet/gortex/internal/parser"
	"github.com/zzet/gortex/internal/parser/languages"
	"github.com/zzet/gortex/internal/query"
	internalserver "github.com/zzet/gortex/internal/server"
)

// toolCall is one synthetic MCP request the bench fires per
// iteration. ArgsFn lets the bench vary the args across iterations
// (e.g. different query strings) so the dispatch path isn't
// trivially memoised by an upstream cache.
type toolCall struct {
	Tool     string
	ArgsFn   func(iter int) map[string]any
	WarmupN  int
	IterN    int
	// SkipIfMissing lets a tool opt out when its substrate isn't
	// in the indexed graph (e.g. nothing to call get_callers on).
	SkipIfMissing func(g *graph.Graph) bool
}

// result captures the per-tool aggregate the bench publishes.
type result struct {
	Tool      string    `json:"tool"`
	Iters     int       `json:"iters"`
	P50Ms     float64   `json:"p50_ms"`
	P95Ms     float64   `json:"p95_ms"`
	P99Ms     float64   `json:"p99_ms"`
	MeanMs    float64   `json:"mean_ms"`
	MaxMs     float64   `json:"max_ms"`
	ErrorRate float64   `json:"error_rate"`
	Skipped   string    `json:"skipped,omitempty"`
	Started   time.Time `json:"-"`
}

func main() {
	repo := flag.String("repo", ".", "corpus to index for the bench")
	iter := flag.Int("iter", 200, "iterations per tool (warm-up of iter/10 is added on top)")
	out := flag.String("out", "", "primary output path (default stdout)")
	jsonOut := flag.String("json", "", "companion JSON metrics output")
	csvOut := flag.String("csv", "", "companion CSV output")
	format := flag.String("format", "markdown", "markdown | json | csv")
	tools := flag.String("tools", "", "comma-separated subset (default: all known tools)")
	flag.Parse()

	absRepo, err := filepath.Abs(*repo)
	if err != nil {
		die("repo path: %v", err)
	}

	fmt.Fprintf(os.Stderr, "[daemon-latency] indexing %s...\n", absRepo)
	g, srv := buildInProcessServer(absRepo)
	fmt.Fprintf(os.Stderr, "[daemon-latency] indexed %d nodes\n", len(g.AllNodes()))

	handler := internalserver.NewHandler(srv.MCPServer(), g, "bench", zap.NewNop())

	// Build the call set against the freshly indexed graph so each
	// synthetic request has at least some structural validity (a
	// real symbol id, an extant file path).
	calls := defaultCalls(g, *iter)
	if *tools != "" {
		calls = filterCalls(calls, strings.Split(*tools, ","))
	}

	rows := make([]result, 0, len(calls))
	for _, c := range calls {
		if c.SkipIfMissing != nil && c.SkipIfMissing(g) {
			rows = append(rows, result{Tool: c.Tool, Iters: 0, Skipped: "no eligible substrate in indexed graph"})
			fmt.Fprintf(os.Stderr, "[daemon-latency] %-22s skipped (no substrate)\n", c.Tool)
			continue
		}
		row := runOne(handler, c)
		rows = append(rows, row)
		fmt.Fprintf(os.Stderr, "[daemon-latency] %-22s p50=%6.2fms p95=%6.2fms p99=%6.2fms iters=%d errs=%.0f%%\n",
			c.Tool, row.P50Ms, row.P95Ms, row.P99Ms, row.Iters, row.ErrorRate*100)
	}

	var primary []byte
	switch strings.ToLower(*format) {
	case "markdown", "md":
		primary = []byte(renderMarkdown(rows, absRepo, g))
	case "csv":
		primary = []byte(renderCSV(rows))
	case "json":
		primary = mustMarshalJSON(rows)
	default:
		die("unknown --format %q", *format)
	}
	if err := writeOutput(*out, primary); err != nil {
		die("write output: %v", err)
	}
	if *csvOut != "" {
		if err := writeOutput(*csvOut, []byte(renderCSV(rows))); err != nil {
			die("write csv: %v", err)
		}
	}
	if *jsonOut != "" {
		if err := writeOutput(*jsonOut, mustMarshalJSON(rows)); err != nil {
			die("write json: %v", err)
		}
	}
}

// --- in-process server ---------------------------------------------

// buildInProcessServer wires the same Server the production stdio /
// daemon front-ends use, against a fresh in-process graph of repoRoot.
// Identical wiring to `cmd/gortex/eval_recall.go`'s indexed-server
// path so the bench reflects production handler arithmetic.
func buildInProcessServer(repoRoot string) (*graph.Graph, *gortexmcp.Server) {
	g := graph.New()
	reg := parser.NewRegistry()
	languages.RegisterAll(reg)
	cfg := config.Config{}
	idx := indexer.New(g, reg, cfg.Index, zap.NewNop())
	if _, err := idx.Index(repoRoot); err != nil {
		die("index %s: %v", repoRoot, err)
	}
	eng := query.NewEngine(g)
	eng.SetSearch(idx.Search())
	srv := gortexmcp.NewServer(eng, g, idx, nil, zap.NewNop(), cfg.Guards.Rules)
	srv.RunAnalysis()
	return g, srv
}

// --- call set -------------------------------------------------------

// defaultCalls returns the canonical bench surface. We focus on
// tools agents actually call in production (the headline savings
// drivers) — covering both cheap (graph_stats) and expensive
// (smart_context) shapes so the published table shows the spread.
func defaultCalls(g *graph.Graph, iter int) []toolCall {
	if iter <= 0 {
		iter = 200
	}
	warmup := max(iter/10, 5)

	// Pick representative symbol IDs / file paths from the indexed
	// graph so the synthetic requests have real targets.
	var sampleFnID, sampleFilePath string
	for _, n := range g.AllNodes() {
		if n == nil {
			continue
		}
		if sampleFnID == "" && (n.Kind == graph.KindFunction || n.Kind == graph.KindMethod) {
			sampleFnID = n.ID
		}
		if sampleFilePath == "" && n.Kind == graph.KindFile && n.FilePath != "" {
			sampleFilePath = n.FilePath
		}
		if sampleFnID != "" && sampleFilePath != "" {
			break
		}
	}

	queries := []string{
		"validateToken", "Indexer", "search", "newServer",
		"handler", "config", "graph", "rerank", "query", "savings",
	}

	return []toolCall{
		{
			Tool:    "graph_stats",
			ArgsFn:  func(_ int) map[string]any { return map[string]any{} },
			WarmupN: warmup, IterN: iter,
		},
		{
			Tool: "search_symbols",
			ArgsFn: func(i int) map[string]any {
				return map[string]any{
					"query": queries[i%len(queries)],
					"limit": float64(20),
				}
			},
			WarmupN: warmup, IterN: iter,
		},
		{
			Tool: "get_symbol_source",
			ArgsFn: func(_ int) map[string]any {
				return map[string]any{"id": sampleFnID}
			},
			WarmupN: warmup, IterN: iter,
			SkipIfMissing: func(g *graph.Graph) bool { return sampleFnID == "" },
		},
		{
			Tool: "get_callers",
			ArgsFn: func(_ int) map[string]any {
				return map[string]any{"id": sampleFnID, "limit": float64(50)}
			},
			WarmupN: warmup, IterN: iter,
			SkipIfMissing: func(g *graph.Graph) bool { return sampleFnID == "" },
		},
		{
			Tool: "find_usages",
			ArgsFn: func(_ int) map[string]any {
				return map[string]any{"id": sampleFnID}
			},
			WarmupN: warmup, IterN: iter,
			SkipIfMissing: func(g *graph.Graph) bool { return sampleFnID == "" },
		},
		{
			Tool: "get_file_summary",
			ArgsFn: func(_ int) map[string]any {
				return map[string]any{"path": sampleFilePath}
			},
			WarmupN: warmup, IterN: iter,
			SkipIfMissing: func(g *graph.Graph) bool { return sampleFilePath == "" },
		},
		{
			Tool: "smart_context",
			ArgsFn: func(i int) map[string]any {
				return map[string]any{"task": "find " + queries[i%len(queries)]}
			},
			// smart_context is heavy — fewer iterations so the
			// whole bench stays reasonable. Still produces a
			// credible p50/p95 with 30-50 samples.
			WarmupN: 3, IterN: iter / 5,
		},
		{
			Tool: "get_repo_outline",
			ArgsFn: func(_ int) map[string]any {
				return map[string]any{}
			},
			WarmupN: warmup, IterN: iter,
		},
	}
}

func filterCalls(calls []toolCall, names []string) []toolCall {
	want := map[string]bool{}
	for _, n := range names {
		want[strings.TrimSpace(n)] = true
	}
	out := make([]toolCall, 0, len(calls))
	for _, c := range calls {
		if want[c.Tool] {
			out = append(out, c)
		}
	}
	return out
}

// --- run loop -------------------------------------------------------

func runOne(handler *internalserver.Handler, c toolCall) result {
	ctx := context.Background()
	// Warm-up: prime any lazy initialisation in the handler /
	// graph so the measured iterations are steady-state.
	for i := range c.WarmupN {
		_, _ = handler.CallToolStrict(ctx, c.Tool, c.ArgsFn(i))
	}

	latencies := make([]time.Duration, 0, c.IterN)
	errors := 0
	for i := range c.IterN {
		t := time.Now()
		_, err := handler.CallToolStrict(ctx, c.Tool, c.ArgsFn(i))
		latencies = append(latencies, time.Since(t))
		if err != nil {
			errors++
		}
	}
	r := result{
		Tool:  c.Tool,
		Iters: c.IterN,
	}
	if len(latencies) > 0 {
		r.P50Ms = pctMs(latencies, 50)
		r.P95Ms = pctMs(latencies, 95)
		r.P99Ms = pctMs(latencies, 99)
		r.MaxMs = pctMs(latencies, 100)
		r.MeanMs = meanMs(latencies)
		r.ErrorRate = float64(errors) / float64(len(latencies))
	}
	return r
}

func pctMs(xs []time.Duration, pct int) float64 {
	if len(xs) == 0 {
		return 0
	}
	sorted := make([]time.Duration, len(xs))
	copy(sorted, xs)
	slices.Sort(sorted)
	idx := (pct * len(sorted)) / 100
	if idx >= len(sorted) {
		idx = len(sorted) - 1
	}
	return float64(sorted[idx].Microseconds()) / 1000.0
}

func meanMs(xs []time.Duration) float64 {
	if len(xs) == 0 {
		return 0
	}
	var sum time.Duration
	for _, x := range xs {
		sum += x
	}
	avg := sum / time.Duration(len(xs))
	return float64(avg.Microseconds()) / 1000.0
}

// --- rendering ------------------------------------------------------

func renderMarkdown(rows []result, repoRoot string, g *graph.Graph) string {
	var b strings.Builder
	fmt.Fprintln(&b, "# Daemon-mode MCP-tool latency")
	fmt.Fprintln(&b)
	fmt.Fprintf(&b, "_Corpus: `%s` (%d nodes). In-process handler dispatch — measures `Handler.CallToolStrict` end-to-end. Daemon socket overhead adds typically <1 ms on a warm pipe; the handler latency below dominates user-perceived response time._\n",
		repoRoot, len(g.AllNodes()))
	fmt.Fprintln(&b)
	fmt.Fprintln(&b, "| tool | iters | p50 | p95 | p99 | mean | max | errors |")
	fmt.Fprintln(&b, "|------|------:|----:|----:|----:|-----:|----:|-------:|")
	for _, r := range rows {
		if r.Skipped != "" {
			fmt.Fprintf(&b, "| %s | — | — | — | — | — | — | skipped: %s |\n", r.Tool, r.Skipped)
			continue
		}
		fmt.Fprintf(&b, "| %s | %d | %s | %s | %s | %s | %s | %.0f%% |\n",
			r.Tool, r.Iters,
			fmtMs(r.P50Ms), fmtMs(r.P95Ms), fmtMs(r.P99Ms),
			fmtMs(r.MeanMs), fmtMs(r.MaxMs),
			r.ErrorRate*100,
		)
	}
	fmt.Fprintln(&b)
	fmt.Fprintln(&b, summary(rows))
	return b.String()
}

func renderCSV(rows []result) string {
	var b strings.Builder
	w := csv.NewWriter(&b)
	_ = w.Write([]string{"tool", "iters", "p50_ms", "p95_ms", "p99_ms", "mean_ms", "max_ms", "error_rate", "skipped"})
	for _, r := range rows {
		_ = w.Write([]string{
			r.Tool,
			fmt.Sprintf("%d", r.Iters),
			fmt.Sprintf("%.3f", r.P50Ms),
			fmt.Sprintf("%.3f", r.P95Ms),
			fmt.Sprintf("%.3f", r.P99Ms),
			fmt.Sprintf("%.3f", r.MeanMs),
			fmt.Sprintf("%.3f", r.MaxMs),
			fmt.Sprintf("%.4f", r.ErrorRate),
			r.Skipped,
		})
	}
	w.Flush()
	return b.String()
}

func mustMarshalJSON(rows []result) []byte {
	b, err := json.MarshalIndent(rows, "", "  ")
	if err != nil {
		die("marshal json: %v", err)
	}
	return append(b, '\n')
}

func summary(rows []result) string {
	ran := 0
	var p95s, p99s []float64
	for _, r := range rows {
		if r.Skipped != "" {
			continue
		}
		ran++
		p95s = append(p95s, r.P95Ms)
		p99s = append(p99s, r.P99Ms)
	}
	if ran == 0 {
		return "_no tools ran (all skipped)_"
	}
	sort.Float64s(p95s)
	sort.Float64s(p99s)
	medianP95 := p95s[len(p95s)/2]
	medianP99 := p99s[len(p99s)/2]
	return fmt.Sprintf("**Summary:** %d/%d tools ran. Median p95 across tools: %s. Median p99: %s.",
		ran, len(rows), fmtMs(medianP95), fmtMs(medianP99))
}

func fmtMs(v float64) string {
	switch {
	case v == 0:
		return "—"
	case v < 1.0:
		return fmt.Sprintf("%.2fms", v)
	case v < 1000:
		return fmt.Sprintf("%.1fms", v)
	default:
		return fmt.Sprintf("%.2fs", v/1000.0)
	}
}

// --- helpers --------------------------------------------------------

func die(format string, args ...any) {
	fmt.Fprintf(os.Stderr, "daemon-latency: "+format+"\n", args...)
	os.Exit(1)
}

func writeOutput(path string, body []byte) error {
	if path == "" {
		_, err := os.Stdout.Write(body)
		return err
	}
	if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
		return err
	}
	return os.WriteFile(path, body, 0o644)
}