chore: import upstream snapshot with attribution

code_review_graph/eval/__init__.py

@@ -0,0 +1,33 @@
+"""Evaluation framework for code-review-graph.
+
+Provides scoring metrics (token efficiency, MRR, precision/recall),
+benchmark runners, and report generators for benchmarking graph-based code reviews.
+"""
+
+from __future__ import annotations
+
+from .reporter import generate_full_report, generate_markdown_report, generate_readme_tables
+from .scorer import compute_mrr, compute_precision_recall, compute_token_efficiency
+
+
+def __getattr__(name: str):
+    """Lazy-import runner functions (require pyyaml)."""
+    _runner_names = {"load_all_configs", "load_config", "run_eval", "write_csv"}
+    if name in _runner_names:
+        from . import runner
+        return getattr(runner, name)
+    raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
+
+
+__all__ = [
+    "compute_mrr",
+    "compute_precision_recall",
+    "compute_token_efficiency",
+    "generate_full_report",
+    "generate_markdown_report",
+    "generate_readme_tables",
+    "load_all_configs",
+    "load_config",
+    "run_eval",
+    "write_csv",
+]

code_review_graph/eval/benchmarks/__init__.py

@@ -0,0 +1 @@
+"""Benchmark modules for the evaluation framework."""

code_review_graph/eval/benchmarks/agent_baseline.py

@@ -0,0 +1,193 @@
+"""Agent baseline benchmark: grep-and-read-top-k versus a graph query.
+
+The whole-corpus baseline in the standalone token benchmark is an upper
+bound no real agent pays: a competent agent greps for identifiers from the
+question and reads only the best-matching files. This benchmark measures
+that realistic baseline:
+
+1. Derive search terms from the question (identifier-shaped tokens via
+   ``search.extract_query_identifiers`` plus plain keywords).
+2. Pure-python grep over the corpus (no external ``rg``/``grep`` binary),
+   ranking files by total case-insensitive match count.
+3. Read the top-k files (k=3) and token-count them with the chars/4 utility
+   (``token_benchmark.estimate_tokens``) as ``baseline_tokens``.
+4. Compare against the graph-query cost for the same question — hybrid
+   search hits plus one hop of neighbor edges, the same accounting used by
+   ``code_review_graph/token_benchmark.py``.
+
+Questions come from ``agent_questions:`` in the repo config, falling back to
+the ``search_queries`` query strings when absent.
+
+Failure semantics match the other benchmarks: a thrown search is recorded
+with ``status="error"`` and excluded from aggregates; rows where either side
+of the ratio is zero get ``status="no_graph_results"`` /
+``status="no_baseline_match"`` and are likewise excluded.
+"""
+
+from __future__ import annotations
+
+import logging
+import statistics
+from collections.abc import Iterator
+from pathlib import Path
+
+from code_review_graph.token_benchmark import estimate_tokens
+
+logger = logging.getLogger(__name__)
+
+DEFAULT_TOP_K = 3
+
+_SOURCE_EXTS = (
+    ".py", ".js", ".ts", ".tsx", ".go", ".rs", ".java",
+    ".c", ".cpp", ".h", ".rb", ".php", ".swift", ".kt",
+)
+
+_SKIP_DIRS = {
+    ".git", ".hg", ".svn", "node_modules", "__pycache__",
+    ".code-review-graph", ".venv", "venv", "dist", "build",
+}
+
+_STOPWORDS = {
+    "how", "does", "do", "the", "a", "an", "is", "are", "was", "what",
+    "where", "when", "which", "who", "why", "and", "or", "in", "on", "of",
+    "to", "for", "with", "via", "into", "from", "this", "that", "it", "its",
+}
+
+
+def derive_search_terms(question: str) -> list[str]:
+    """Derive lowercase grep terms: identifiers first, then plain keywords.
+
+    Identifier-shaped tokens (``Client.request``, ``get_users``, ``APIRoute``)
+    are extracted via ``search.extract_query_identifiers``; remaining words of
+    3+ characters that are not stopwords are appended. Order is deterministic.
+    """
+    from code_review_graph.search import extract_query_identifiers
+
+    terms: list[str] = []
+    seen: set[str] = set()
+    for ident in extract_query_identifiers(question):
+        if ident not in seen:
+            seen.add(ident)
+            terms.append(ident)
+    for word in question.split():
+        w = word.strip(".,;:!?\"'()[]{}`").lower()
+        if len(w) >= 3 and w not in _STOPWORDS and w not in seen:
+            seen.add(w)
+            terms.append(w)
+    return terms
+
+
+def iter_source_files(repo_path: Path) -> Iterator[Path]:
+    """Yield source files under *repo_path*, skipping vendored/VCS dirs."""
+    for path in sorted(repo_path.rglob("*")):
+        if path.suffix not in _SOURCE_EXTS or not path.is_file():
+            continue
+        if any(part in _SKIP_DIRS for part in path.parts):
+            continue
+        yield path
+
+
+def grep_rank(
+    repo_path: Path, terms: list[str], k: int = DEFAULT_TOP_K,
+) -> list[tuple[str, int]]:
+    """Rank source files by total case-insensitive term matches; take top-k.
+
+    Pure python — no external grep/rg dependency. Deterministic: ties break
+    on the relative path. Files with zero matches are dropped.
+    """
+    lowered = [t.lower() for t in terms if t]
+    if not lowered:
+        return []
+    scores: list[tuple[str, int]] = []
+    for path in iter_source_files(repo_path):
+        try:
+            text = path.read_text(encoding="utf-8", errors="replace").lower()
+        except OSError:
+            continue
+        count = sum(text.count(term) for term in lowered)
+        if count > 0:
+            scores.append((str(path.relative_to(repo_path)), count))
+    scores.sort(key=lambda item: (-item[1], item[0]))
+    return scores[:k]
+
+
+def run(repo_path: Path, store, config: dict) -> list[dict]:
+    """Run the agent baseline benchmark for one repo."""
+    questions = list(config.get("agent_questions") or [])
+    if not questions:
+        questions = [sq["query"] for sq in config.get("search_queries", [])]
+
+    k = int(config.get("agent_baseline_top_k", DEFAULT_TOP_K))
+    results: list[dict] = []
+
+    for question in questions:
+        terms = derive_search_terms(question)
+        top = grep_rank(repo_path, terms, k=k)
+        baseline_tokens = 0
+        for rel, _count in top:
+            try:
+                baseline_tokens += estimate_tokens(
+                    (repo_path / rel).read_text(encoding="utf-8", errors="replace")
+                )
+            except OSError:
+                continue
+
+        row: dict = {
+            "repo": config["name"],
+            "question": question,
+            "terms": " ".join(terms),
+            "files_matched": len(top),
+            "top_files": ";".join(rel for rel, _ in top),
+            "baseline_tokens": baseline_tokens,
+            "graph_tokens": "",
+            "baseline_to_graph_ratio": "",
+            "status": "ok",
+            "error": "",
+        }
+
+        try:
+            from code_review_graph.search import hybrid_search
+            hits = hybrid_search(store, question, limit=5)
+        except Exception as exc:
+            logger.warning("hybrid_search failed on %r: %s", question, exc)
+            row["status"] = "error"
+            row["error"] = str(exc)[:200]
+            results.append(row)
+            continue
+
+        # Same accounting as the standalone token benchmark: search hits
+        # plus up to 5 outgoing edges of neighbor context per hit.
+        graph_tokens = 0
+        for hit in hits:
+            graph_tokens += estimate_tokens(str(hit))
+            qn = hit.get("qualified_name", "")
+            for edge in store.get_edges_by_source(qn)[:5]:
+                graph_tokens += estimate_tokens(str(edge))
+
+        row["graph_tokens"] = graph_tokens
+        if baseline_tokens > 0 and graph_tokens > 0:
+            row["baseline_to_graph_ratio"] = round(baseline_tokens / graph_tokens, 1)
+        elif graph_tokens == 0:
+            row["status"] = "no_graph_results"
+        else:
+            row["status"] = "no_baseline_match"
+        results.append(row)
+
+    return results
+
+
+def aggregate(results: list[dict]) -> dict:
+    """Aggregate over rows where both sides of the comparison exist."""
+    ok = [r for r in results if r.get("status") == "ok"]
+    ratios = [float(r["baseline_to_graph_ratio"]) for r in ok]
+    return {
+        "total_rows": len(results),
+        "ok_rows": len(ok),
+        "error_rows": sum(1 for r in results if r.get("status") == "error"),
+        "median_baseline_to_graph_ratio": (
+            round(statistics.median(ratios), 1) if ratios else None
+        ),
+        "mean_baseline_to_graph_ratio": (
+            round(statistics.mean(ratios), 1) if ratios else None
+        ),
+    }

code_review_graph/eval/benchmarks/build_performance.py

@@ -0,0 +1,60 @@
+"""Build performance benchmark: measures timing of graph operations."""
+
+from __future__ import annotations
+
+import logging
+import time
+from pathlib import Path
+
+logger = logging.getLogger(__name__)
+
+
+def run(repo_path: Path, store, config: dict) -> list[dict]:
+    """Run build performance benchmark."""
+    stats = store.get_stats()
+
+    # Time flow detection
+    try:
+        from code_review_graph.flows import store_flows, trace_flows
+        t0 = time.perf_counter()
+        flows = trace_flows(store)
+        store_flows(store, flows)
+        flow_time = time.perf_counter() - t0
+    except Exception as exc:
+        logger.warning("Flow detection failed: %s", exc)
+        flow_time = 0.0
+
+    # Time community detection
+    try:
+        from code_review_graph.communities import detect_communities, store_communities
+        t0 = time.perf_counter()
+        comms = detect_communities(store)
+        store_communities(store, comms)
+        community_time = time.perf_counter() - t0
+    except Exception as exc:
+        logger.warning("Community detection failed: %s", exc)
+        community_time = 0.0
+
+    # Time search (average of queries)
+    search_times: list[float] = []
+    for sq in config.get("search_queries", [])[:10]:
+        t0 = time.perf_counter()
+        store.search_nodes(sq["query"], limit=20)
+        search_times.append(time.perf_counter() - t0)
+
+    avg_search_ms = round(
+        sum(search_times) / max(len(search_times), 1) * 1000, 1
+    )
+
+    return [{
+        "repo": config["name"],
+        "file_count": stats.files_count,
+        "node_count": stats.total_nodes,
+        "edge_count": stats.total_edges,
+        "flow_detection_seconds": round(flow_time, 3),
+        "community_detection_seconds": round(community_time, 3),
+        "search_avg_ms": avg_search_ms,
+        "nodes_per_second": round(
+            stats.total_nodes / max(flow_time, 0.001)
+        ),
+    }]

code_review_graph/eval/benchmarks/flow_completeness.py

@@ -0,0 +1,36 @@
+"""Flow completeness benchmark: evaluates entry point detection and flow tracing."""
+
+from __future__ import annotations
+
+import logging
+from pathlib import Path
+
+logger = logging.getLogger(__name__)
+
+
+def run(repo_path: Path, store, config: dict) -> list[dict]:
+    """Run flow completeness benchmark."""
+    from code_review_graph.flows import store_flows, trace_flows
+
+    flows = trace_flows(store)
+    count = store_flows(store, flows)
+
+    # Get detected entry point names
+    detected_entries = set()
+    for flow in flows:
+        detected_entries.add(flow.get("entry_point") or flow.get("name", ""))
+
+    known = set(config.get("entry_points", []))
+    found = sum(1 for ep in known if any(ep in d for d in detected_entries))
+
+    depths = [f.get("depth", 0) for f in flows]
+
+    return [{
+        "repo": config["name"],
+        "known_entry_points": len(known),
+        "detected_entry_points": found,
+        "recall": round(found / max(len(known), 1), 3),
+        "detected_flows": count,
+        "avg_flow_depth": round(sum(depths) / max(len(depths), 1), 1),
+        "max_flow_depth": max(depths, default=0),
+    }]

code_review_graph/eval/benchmarks/impact_accuracy.py

@@ -0,0 +1,220 @@
+"""Impact accuracy benchmark: measures precision/recall of change impact analysis.
+
+Two ground-truth modes are emitted side by side (``ground_truth_mode`` column):
+
+- **graph-derived (circular — upper bound)** — the historical mode. Ground
+  truth is the changed files plus files with CALLS/IMPORTS_FROM edges into
+  them, i.e. derived from the same graph the predictor traverses. Recall in
+  this mode is an upper bound by construction, not independent evidence.
+- **co-change (same commit, seed excluded)** — the honest mode. The predictor
+  is seeded with a single changed file and graded against the *other* files
+  the author actually touched in the same commit. The ground truth comes from
+  git history, not from the graph.
+
+Failure semantics: if ``analyze_changes`` throws, the row is recorded with
+``status="error"`` and empty metric fields — it stays in the CSV but is
+excluded from aggregates. (Previously a failure silently set
+``predicted = set(changed)``, guaranteeing a fake recall of 1.0.)
+"""
+
+from __future__ import annotations
+
+import logging
+import statistics
+import subprocess
+from pathlib import Path
+
+logger = logging.getLogger(__name__)
+
+MODE_GRAPH_DERIVED = "graph-derived (circular — upper bound)"
+MODE_CO_CHANGE = "co-change (same commit, seed excluded)"
+
+
+def _get_changed_files(repo_path: Path, sha: str) -> list[str]:
+    """Get list of changed files for a commit."""
+    result = subprocess.run(
+        ["git", "diff", "--name-only", f"{sha}~1", sha],
+        cwd=str(repo_path),
+        capture_output=True,
+        text=True,
+    )
+    if result.returncode != 0:
+        result = subprocess.run(
+            ["git", "diff", "--name-only", "HEAD~1", "HEAD"],
+            cwd=str(repo_path),
+            capture_output=True,
+            text=True,
+        )
+    return [f.strip() for f in result.stdout.strip().splitlines() if f.strip()]
+
+
+def _files_from_analysis(analysis: dict) -> set[str]:
+    """Extract predicted file paths from an ``analyze_changes`` result."""
+    predicted: set[str] = set()
+    for f in analysis.get("changed_functions", []):
+        if isinstance(f, dict) and "file_path" in f:
+            predicted.add(f["file_path"])
+        elif isinstance(f, dict) and "file" in f:
+            predicted.add(f["file"])
+    for flow in analysis.get("affected_flows", []):
+        if isinstance(flow, dict):
+            for node in flow.get("nodes", []):
+                if isinstance(node, dict) and "file_path" in node:
+                    predicted.add(node["file_path"])
+    return predicted
+
+
+def _graph_neighbor_files(store, files: list[str]) -> set[str]:
+    """Files with CALLS/IMPORTS_FROM edges into any node of *files* (one hop)."""
+    out: set[str] = set()
+    for f in files:
+        for node in store.get_nodes_by_file(f):
+            for edge in store.get_edges_by_target(node.qualified_name):
+                if edge.kind in ("CALLS", "IMPORTS_FROM"):
+                    src_qual = edge.source_qualified
+                    src_file = src_qual.split("::")[0] if "::" in src_qual else ""
+                    if src_file:
+                        out.add(src_file)
+    return out
+
+
+def _base_row(repo: str, sha: str, mode: str, seed: str) -> dict:
+    return {
+        "repo": repo,
+        "commit": sha,
+        "ground_truth_mode": mode,
+        "seed_file": seed,
+        "predicted_files": "",
+        "actual_files": "",
+        "true_positives": "",
+        "precision": "",
+        "recall": "",
+        "f1": "",
+        "status": "ok",
+        "error": "",
+    }
+
+
+def _scored_row(
+    repo: str, sha: str, mode: str, seed: str,
+    predicted: set[str], actual: set[str],
+) -> dict:
+    tp = len(predicted & actual)
+    precision = tp / max(len(predicted), 1)
+    recall = tp / max(len(actual), 1)
+    f1 = 2 * precision * recall / max(precision + recall, 0.001)
+    row = _base_row(repo, sha, mode, seed)
+    row.update({
+        "predicted_files": len(predicted),
+        "actual_files": len(actual),
+        "true_positives": tp,
+        "precision": round(precision, 3),
+        "recall": round(recall, 3),
+        "f1": round(f1, 3),
+    })
+    return row
+
+
+def _error_row(repo: str, sha: str, mode: str, seed: str, exc: Exception) -> dict:
+    row = _base_row(repo, sha, mode, seed)
+    row["status"] = "error"
+    row["error"] = str(exc)[:200]
+    return row
+
+
+def run(repo_path: Path, store, config: dict) -> list[dict]:
+    """Run impact accuracy benchmark (both ground-truth modes)."""
+    from code_review_graph.changes import analyze_changes
+
+    results = []
+    repo = config["name"]
+    for tc in config.get("test_commits", []):
+        sha = tc["sha"]
+        changed = _get_changed_files(repo_path, sha)
+        if not changed:
+            continue
+
+        # --- Mode 1: graph-derived ground truth (circular — upper bound) ---
+        try:
+            analysis = analyze_changes(
+                store, changed, repo_root=str(repo_path), base=sha + "~1",
+            )
+        except Exception as exc:
+            # Old behaviour set predicted = set(changed) here, which
+            # guarantees recall 1.0 on a *failed* run. Mark failed instead.
+            logger.warning("analyze_changes failed on %s: %s", sha, exc)
+            results.append(_error_row(repo, sha, MODE_GRAPH_DERIVED, "", exc))
+            analysis = None
+
+        if analysis is not None:
+            predicted = set(changed) | _files_from_analysis(analysis)
+            actual = set(changed) | _graph_neighbor_files(store, changed)
+            results.append(
+                _scored_row(repo, sha, MODE_GRAPH_DERIVED, "", predicted, actual)
+            )
+
+        # --- Mode 2: co-change ground truth (honest) ---
+        # Seed the predictor with a single changed file and grade against
+        # the other files the author touched in the same commit. Note the
+        # seed analysis deliberately gets no repo_root/diff: it must only
+        # see the seed file, never the full commit diff.
+        seed = sorted(changed)[0]
+        co_actual = set(changed) - {seed}
+        if not co_actual:
+            row = _base_row(repo, sha, MODE_CO_CHANGE, seed)
+            row["status"] = "skipped"
+            row["error"] = "single-file commit: no co-changed files to grade against"
+            results.append(row)
+            continue
+
+        try:
+            seed_analysis = analyze_changes(store, [seed])
+        except Exception as exc:
+            logger.warning("analyze_changes (seed=%s) failed on %s: %s", seed, sha, exc)
+            results.append(_error_row(repo, sha, MODE_CO_CHANGE, seed, exc))
+            continue
+
+        co_predicted = _files_from_analysis(seed_analysis)
+        co_predicted |= _graph_neighbor_files(store, [seed])
+        co_predicted.discard(seed)
+        results.append(
+            _scored_row(repo, sha, MODE_CO_CHANGE, seed, co_predicted, co_actual)
+        )
+
+    return results
+
+
+def aggregate(results: list[dict]) -> dict:
+    """Per-mode means over successful rows only.
+
+    Error/skipped rows stay in the CSV but never contribute to a number.
+    """
+    out: dict = {
+        "total_rows": len(results),
+        "error_rows": sum(1 for r in results if r.get("status") == "error"),
+        "skipped_rows": sum(1 for r in results if r.get("status") == "skipped"),
+    }
+    for key, mode in (
+        ("graph_derived", MODE_GRAPH_DERIVED),
+        ("co_change", MODE_CO_CHANGE),
+    ):
+        rows = [
+            r for r in results
+            if r.get("ground_truth_mode") == mode and r.get("status") == "ok"
+        ]
+        out[key] = {
+            "ok_rows": len(rows),
+            "mean_precision": (
+                round(statistics.mean(float(r["precision"]) for r in rows), 3)
+                if rows else None
+            ),
+            "mean_recall": (
+                round(statistics.mean(float(r["recall"]) for r in rows), 3)
+                if rows else None
+            ),
+            "mean_f1": (
+                round(statistics.mean(float(r["f1"]) for r in rows), 3)
+                if rows else None
+            ),
+        }
+    return out

code_review_graph/eval/benchmarks/multi_hop_retrieval.py

@@ -0,0 +1,125 @@
+"""Multi-hop retrieval benchmark.
+
+Tests a two-step tool chain that mimics how an LLM agent actually uses the
+graph for complex tasks:
+
+  1. ``hybrid_search(nl_query)`` to find a starting anchor from a natural-
+     language question.
+  2. ``query_graph(pattern, target=anchor)`` to traverse one hop along the
+     requested edge kind (callers_of / callees_of / tests_for / ...).
+
+For each task the benchmark records:
+
+- ``anchor_found`` — did semantic search return a node whose qualified_name
+  ends with the expected suffix in the top-K?
+- ``anchor_rank`` — index in the search result list (lower is better).
+- ``neighbor_count`` — number of neighbors returned by the traversal.
+- ``neighbor_recall`` — fraction of ``expected_neighbor_names`` that appear
+  among the neighbor names.
+- ``score`` — ``int(anchor_found) * neighbor_recall``. Range 0–1.
+
+Tasks are defined per-config under ``multi_hop_tasks:`` in
+``code_review_graph/eval/configs/*.yaml``. See
+``docs/REPRODUCING.md`` for the schema and the curated canonical task set.
+"""
+
+from __future__ import annotations
+
+import logging
+from pathlib import Path
+from typing import Any
+
+logger = logging.getLogger(__name__)
+
+
+def _name_set(rows: list[dict[str, Any]]) -> set[str]:
+    out: set[str] = set()
+    for r in rows:
+        name = (r.get("name") or "").lower()
+        if name:
+            out.add(name)
+    return out
+
+
+def run(repo_path: Path, store, config: dict) -> list[dict]:
+    """Run the multi-hop retrieval benchmark for one repo."""
+    # Imports are local so an import-time failure in one optional benchmark
+    # does not poison the whole runner.
+    from code_review_graph.search import hybrid_search
+    from code_review_graph.tools.query import query_graph
+
+    repo_root = str(repo_path)
+    results: list[dict] = []
+
+    for task in config.get("multi_hop_tasks", []):
+        task_id = task["id"]
+        nl_query = task["nl_query"]
+        suffix = task["anchor_qualified_suffix"].lower()
+        traversal = task.get("traversal_pattern", "callers_of")
+        expected = [e.lower() for e in task.get("expected_neighbor_names", [])]
+        k = int(task.get("k", 10))
+
+        # Step 1 — semantic search
+        try:
+            hits = hybrid_search(store, nl_query, limit=k)
+        except Exception as exc:  # noqa: BLE001 — benchmark must not abort the runner
+            logger.warning("hybrid_search failed on %s: %s", task_id, exc)
+            hits = []
+
+        anchor = None
+        anchor_rank = -1
+        for i, h in enumerate(hits):
+            qn = (h.get("qualified_name") or "").lower()
+            if qn.endswith(suffix):
+                anchor = h
+                anchor_rank = i
+                break
+
+        if anchor is None:
+            results.append({
+                "repo": config["name"],
+                "task_id": task_id,
+                "nl_query": nl_query,
+                "anchor_found": False,
+                "anchor_rank": -1,
+                "neighbor_count": 0,
+                "expected_count": len(expected),
+                "matched_count": 0,
+                "neighbor_recall": 0.0,
+                "score": 0.0,
+            })
+            continue
+
+        # Step 2 — single-hop graph traversal from the anchor
+        try:
+            trav = query_graph(
+                pattern=traversal,
+                target=anchor["qualified_name"],
+                repo_root=repo_root,
+                detail_level="standard",
+            )
+        except Exception as exc:  # noqa: BLE001
+            logger.warning(
+                "query_graph(%s) failed on %s: %s", traversal, task_id, exc,
+            )
+            trav = {}
+
+        rows = trav.get("data") or trav.get("results") or []
+        names = _name_set(rows)
+        matched = sum(1 for e in expected if e in names)
+        recall = matched / len(expected) if expected else 0.0
+
+        results.append({
+            "repo": config["name"],
+            "task_id": task_id,
+            "nl_query": nl_query,
+            "anchor_found": True,
+            "anchor_rank": anchor_rank,
+            "neighbor_count": len(rows),
+            "expected_count": len(expected),
+            "matched_count": matched,
+            "neighbor_recall": round(recall, 3),
+            "score": round(recall, 3),
+        })
+
+    return results

code_review_graph/eval/benchmarks/search_quality.py

@@ -0,0 +1,59 @@
+"""Search quality benchmark: measures search result ranking via MRR."""
+
+from __future__ import annotations
+
+import logging
+import sqlite3
+from pathlib import Path
+
+logger = logging.getLogger(__name__)
+
+
+def run(repo_path: Path, store, config: dict) -> list[dict]:
+    """Run search quality benchmark."""
+    results = []
+    for sq in config.get("search_queries", []):
+        query = sq["query"]
+        expected = sq["expected"]
+
+        try:
+            from code_review_graph.search import hybrid_search
+            search_results = hybrid_search(store, query, limit=20)
+        except (ImportError, sqlite3.OperationalError) as exc:
+            logger.debug("hybrid_search unavailable, using fallback: %s", exc)
+            # Fallback to basic search
+            search_results = [
+                {"qualified_name": n.qualified_name}
+                for n in store.search_nodes(query, limit=20)
+            ]
+
+        rank = 0
+        for i, r in enumerate(search_results):
+            if isinstance(r, dict):
+                qn = r.get("qualified_name", "")
+            elif hasattr(r, "qualified_name"):
+                qn = r.qualified_name
+            else:
+                qn = ""
+            qn_lower = qn.lower()
+            exp_lower = expected.lower()
+            # Match if expected is substring of qn, qn is substring of expected,
+            # or the name part after :: matches
+            exp_name = expected.rsplit("::", 1)[-1] if "::" in expected else expected
+            qn_name = qn.rsplit("::", 1)[-1] if "::" in qn else qn
+            if (
+                exp_lower in qn_lower
+                or qn_lower in exp_lower
+                or exp_name.lower() == qn_name.lower()
+            ):
+                rank = i + 1
+                break
+
+        results.append({
+            "repo": config["name"],
+            "query": query,
+            "expected": expected,
+            "rank": rank,
+            "reciprocal_rank": round(1.0 / rank if rank > 0 else 0.0, 3),
+        })
+    return results

code_review_graph/eval/benchmarks/token_efficiency.py

@@ -0,0 +1,143 @@
+"""Token efficiency benchmark: compares naive, standard, and graph-based token counts.
+
+Failure semantics: if ``get_review_context`` throws, the row is recorded with
+``status="error"`` and empty metric fields. It stays in the CSV for forensics
+but is excluded from every aggregate — a failed tool call is not a
+measurement. (Previously a failure silently produced ``graph_tokens=0`` and
+``ratio = naive / 1``, inflating the results.)
+"""
+
+from __future__ import annotations
+
+import json
+import logging
+import statistics
+import subprocess
+from pathlib import Path
+
+logger = logging.getLogger(__name__)
+
+
+def _count_tokens(text: str) -> int:
+    """Approximate token count (1 token ~ 4 chars)."""
+    return len(text) // 4
+
+
+def _get_changed_files(repo_path: Path, sha: str) -> list[str]:
+    """Get list of changed files for a commit."""
+    result = subprocess.run(
+        ["git", "diff", "--name-only", f"{sha}~1", sha],
+        cwd=str(repo_path),
+        capture_output=True,
+        text=True,
+    )
+    if result.returncode != 0:
+        # Fallback: diff against parent
+        result = subprocess.run(
+            ["git", "diff", "--name-only", "HEAD~1", "HEAD"],
+            cwd=str(repo_path),
+            capture_output=True,
+            text=True,
+        )
+    return [f.strip() for f in result.stdout.strip().splitlines() if f.strip()]
+
+
+def _count_file_tokens(repo_path: Path, files: list[str]) -> int:
+    """Count tokens from full file contents (naive approach)."""
+    total = 0
+    for f in files:
+        fp = repo_path / f
+        if fp.is_file():
+            try:
+                total += _count_tokens(fp.read_text(encoding="utf-8", errors="replace"))
+            except OSError:
+                pass
+    return total
+
+
+def _count_diff_tokens(repo_path: Path, sha: str) -> int:
+    """Count tokens from git diff output (standard approach)."""
+    result = subprocess.run(
+        ["git", "diff", f"{sha}~1", sha],
+        cwd=str(repo_path),
+        capture_output=True,
+        text=True,
+    )
+    if result.returncode != 0:
+        result = subprocess.run(
+            ["git", "diff", "HEAD~1", "HEAD"],
+            cwd=str(repo_path),
+            capture_output=True,
+            text=True,
+        )
+    return _count_tokens(result.stdout)
+
+
+def run(repo_path: Path, store, config: dict) -> list[dict]:
+    """Run token efficiency benchmark."""
+    results = []
+    for tc in config.get("test_commits", []):
+        changed = _get_changed_files(repo_path, tc["sha"])
+        if not changed:
+            continue
+
+        naive_tokens = _count_file_tokens(repo_path, changed)
+        standard_tokens = _count_diff_tokens(repo_path, tc["sha"])
+
+        row: dict = {
+            "repo": config["name"],
+            "commit": tc["sha"],
+            "description": tc.get("description", ""),
+            "changed_files": len(changed),
+            "naive_tokens": naive_tokens,
+            "standard_tokens": standard_tokens,
+            "graph_tokens": "",
+            "naive_to_graph_ratio": "",
+            "standard_to_graph_ratio": "",
+            "status": "ok",
+            "error": "",
+        }
+
+        # Graph-based: use get_review_context
+        try:
+            from code_review_graph.tools import get_review_context
+            ctx = get_review_context(
+                changed_files=changed, repo_root=str(repo_path)
+            )
+            graph_tokens = _count_tokens(json.dumps(ctx))
+        except Exception as exc:
+            # A failed tool call is not a measurement. Recording
+            # graph_tokens=0 used to turn this into ratio = naive/1 — a
+            # huge fake win. Mark the row failed; aggregate() excludes it.
+            logger.warning("get_review_context failed on %s: %s", tc["sha"], exc)
+            row["status"] = "error"
+            row["error"] = str(exc)[:200]
+            results.append(row)
+            continue
+
+        row["graph_tokens"] = graph_tokens
+        row["naive_to_graph_ratio"] = round(naive_tokens / max(graph_tokens, 1), 1)
+        row["standard_to_graph_ratio"] = round(standard_tokens / max(graph_tokens, 1), 1)
+        results.append(row)
+    return results
+
+
+def aggregate(results: list[dict]) -> dict:
+    """Aggregate token-efficiency rows, excluding failed measurements.
+
+    Rows with ``status != "ok"`` stay in the CSV for forensics but must not
+    contribute to any headline number.
+    """
+    ok = [r for r in results if r.get("status") == "ok"]
+    ratios = [float(r["naive_to_graph_ratio"]) for r in ok]
+    return {
+        "total_rows": len(results),
+        "ok_rows": len(ok),
+        "error_rows": sum(1 for r in results if r.get("status") == "error"),
+        "median_naive_to_graph_ratio": (
+            round(statistics.median(ratios), 1) if ratios else None
+        ),
+        "mean_naive_to_graph_ratio": (
+            round(statistics.mean(ratios), 1) if ratios else None
+        ),
+    }

code_review_graph/eval/configs/code-review-graph.yaml

@@ -0,0 +1,50 @@
+name: code-review-graph
+url: https://github.com/tirth8205/code-review-graph
+# Pinned to the latest test_commit SHA so the snapshot is deterministic and
+# every test_commit below is reachable as an ancestor. (This config replaces
+# the historical "nextjs" entry, which used the same URL but mis-labelled the
+# target as a Next.js monorepo.)
+commit: 84bde35459c52e1e0c4b25c6c4799743021e0fc7
+language: python
+size_category: medium
+
+test_commits:
+  - sha: 528801f841e519567ef54d6e52e9b9831d162e1b
+    description: "feat: add multi-platform MCP server installation support"
+    changed_files: 3
+  - sha: 84bde35459c52e1e0c4b25c6c4799743021e0fc7
+    description: "feat: add Google Antigravity platform support for MCP install"
+    changed_files: 2
+
+entry_points:
+  - "code_review_graph/cli.py::cli"
+  - "code_review_graph/main.py::main"
+
+search_queries:
+  - query: "GraphStore nodes"
+    expected: "code_review_graph/graph.py::GraphStore"
+  - query: "parse AST"
+    expected: "code_review_graph/parser.py::CodeParser"
+  - query: "full build"
+    expected: "code_review_graph/incremental.py::full_build"
+
+multi_hop_tasks:
+  - id: crg-parse-file-callers
+    nl_query: "Who invokes the parser entry point on a single source file"
+    anchor_qualified_suffix: "code_review_graph/parser.py::codeparser.parse_file"
+    traversal_pattern: callers_of
+    expected_neighbor_names: ["setup_method"]
+    k: 10
+  - id: crg-upsert-node-callers
+    nl_query: "Where the graph store inserts or updates a node"
+    anchor_qualified_suffix: "code_review_graph/graph.py::graphstore.upsert_node"
+    traversal_pattern: callers_of
+    expected_neighbor_names: ["store_file_nodes_edges"]
+    k: 10
+
+# Questions for the agent_baseline benchmark (pure-python grep top-k vs graph
+# query). See docs/REPRODUCING.md for the methodology.
+agent_questions:
+  - "How does GraphStore upsert_node store a node"
+  - "Where does full_build parse the repository"
+  - "How does hybrid_search rank search results"

code_review_graph/eval/configs/express.yaml

@@ -0,0 +1,45 @@
+name: express
+url: https://github.com/expressjs/express
+# Pinned to the latest test_commit SHA so the snapshot is deterministic and
+# every test_commit below is reachable as an ancestor.
+commit: b4ab7d65d7724d9309b6faaaf82ad492da2a6d35
+language: javascript
+size_category: small
+
+test_commits:
+  - sha: 925a1dff1e42f1b393c977b8b77757fcf633e09f
+    description: "fix: bump qs minimum to ^6.14.2 for CVE-2026-2391"
+    changed_files: 1
+  - sha: b4ab7d65d7724d9309b6faaaf82ad492da2a6d35
+    description: "test: include edge case tests for res.type()"
+    changed_files: 1
+
+entry_points:
+  - "lib/application.js::app.handle"
+  - "lib/express.js::createApplication"
+
+search_queries:
+  - query: "app handle"
+    expected: "lib/application.js::app"
+  - query: "response send"
+    expected: "lib/response.js::res"
+  - query: "request"
+    expected: "lib/request.js::req"
+
+# Express has only one task — JS modules use prototypes + module.exports
+# heavily, so most "method" callers are not represented as proper Function
+# edges in the graph. createApplication is the cleanest anchor.
+multi_hop_tasks:
+  - id: express-create-application-callees
+    nl_query: "What express does when constructing an application"
+    anchor_qualified_suffix: "lib/express.js::createapplication"
+    traversal_pattern: callees_of
+    expected_neighbor_names: ["mixin", "create", "init"]
+    k: 10
+
+# Questions for the agent_baseline benchmark (pure-python grep top-k vs graph
+# query). See docs/REPRODUCING.md for the methodology.
+agent_questions:
+  - "How does app.handle process the middleware stack"
+  - "Where does res.send write the response body"
+  - "How does createApplication initialize an app"

code_review_graph/eval/configs/fastapi.yaml

@@ -0,0 +1,48 @@
+name: fastapi
+url: https://github.com/tiangolo/fastapi
+# Pinned to the latest test_commit SHA so the snapshot is deterministic and
+# every test_commit below is reachable as an ancestor.
+commit: 0227991a01e61bf5cdd93cc00e9e243f52b47a4a
+language: python
+size_category: medium
+
+test_commits:
+  - sha: fa3588c38c7473aca7536b12d686102de4b0f407
+    description: "Fix typo for client_secret in OAuth2 form docstrings"
+    changed_files: 1
+  - sha: 0227991a01e61bf5cdd93cc00e9e243f52b47a4a
+    description: "Exclude spam comments from statistics in scripts/people.py"
+    changed_files: 1
+
+entry_points:
+  - "fastapi/applications.py::FastAPI"
+  - "fastapi/routing.py::APIRouter"
+
+search_queries:
+  - query: "FastAPI application"
+    expected: "fastapi/applications.py::FastAPI"
+  - query: "APIRoute routing"
+    expected: "fastapi/routing.py::APIRoute"
+  - query: "Depends injection"
+    expected: "fastapi/params.py::Depends"
+
+multi_hop_tasks:
+  - id: fastapi-route-handler-callers
+    nl_query: "How fastapi binds a route handler to an APIRoute"
+    anchor_qualified_suffix: "fastapi/routing.py::apiroute.get_route_handler"
+    traversal_pattern: callers_of
+    expected_neighbor_names: ["__init__"]
+    k: 10
+  - id: fastapi-get-dependant-callers
+    nl_query: "Where fastapi resolves dependency declarations into a tree"
+    anchor_qualified_suffix: "fastapi/dependencies/utils.py::get_dependant"
+    traversal_pattern: callers_of
+    expected_neighbor_names: ["get_parameterless_sub_dependant", "solve_dependencies"]
+    k: 10
+
+# Questions for the agent_baseline benchmark (pure-python grep top-k vs graph
+# query). See docs/REPRODUCING.md for the methodology.
+agent_questions:
+  - "How does include_router register routes on the application"
+  - "Where does APIRoute build its route handler"
+  - "How does solve_dependencies resolve Depends parameters"

code_review_graph/eval/configs/flask.yaml

@@ -0,0 +1,50 @@
+name: flask
+url: https://github.com/pallets/flask
+# Pinned to the latest test_commit SHA so the snapshot is deterministic and
+# every test_commit below is reachable as an ancestor.
+commit: a29f88ce6f2f9843bd6fcbbfce1390a2071965d6
+language: python
+size_category: small
+
+test_commits:
+  - sha: fbb6f0bc4c60a0bada0e03c3480d0ccf30a3c1df
+    description: "all teardown callbacks are called despite errors"
+    changed_files: 10
+  - sha: a29f88ce6f2f9843bd6fcbbfce1390a2071965d6
+    description: "document that headers must be set before streaming"
+    changed_files: 4
+
+entry_points:
+  - "src/flask/app.py::Flask.wsgi_app"
+  - "src/flask/sansio/app.py::App.add_url_rule"
+
+search_queries:
+  - query: "Flask wsgi"
+    expected: "src/flask/app.py::Flask"
+  - query: "AppContext globals"
+    expected: "src/flask/ctx.py::AppContext"
+  - query: "create logger"
+    expected: "src/flask/logging.py::create_logger"
+
+# Multi-hop retrieval tasks (semantic_search → query_graph one-hop)
+# See docs/REPRODUCING.md for the schema.
+multi_hop_tasks:
+  - id: flask-dispatch-callers
+    nl_query: "Where Flask dispatches HTTP requests"
+    anchor_qualified_suffix: "src/flask/app.py::flask.dispatch_request"
+    traversal_pattern: callers_of
+    expected_neighbor_names: ["full_dispatch_request"]
+    k: 10
+  - id: flask-exception-callers
+    nl_query: "Where Flask handles uncaught exceptions"
+    anchor_qualified_suffix: "src/flask/app.py::flask.handle_exception"
+    traversal_pattern: callers_of
+    expected_neighbor_names: ["wsgi_app"]
+    k: 10
+
+# Questions for the agent_baseline benchmark (pure-python grep top-k vs graph
+# query). See docs/REPRODUCING.md for the methodology.
+agent_questions:
+  - "How does dispatch_request route an incoming HTTP request"
+  - "Where is the AppContext pushed and popped"
+  - "How does create_logger configure application logging"

code_review_graph/eval/configs/gin.yaml

@@ -0,0 +1,51 @@
+name: gin
+url: https://github.com/gin-gonic/gin
+# Pinned to the latest test_commit SHA so the snapshot is deterministic and
+# every test_commit below is reachable as an ancestor.
+commit: 5c00df8afadd06cc5be530dde00fe6d9fa4a2e4a
+language: go
+size_category: small
+
+test_commits:
+  - sha: 052d1a79aafe3f04078a2716f8e77d4340308383
+    description: "feat(render): add PDF renderer and tests"
+    changed_files: 5
+  - sha: 472d086af2acd924cb4b9d7be0525f7d790f69bc
+    description: "fix(tree): panic in findCaseInsensitivePathRec with RedirectFixedPath"
+    changed_files: 2
+  - sha: 5c00df8afadd06cc5be530dde00fe6d9fa4a2e4a
+    description: "fix(render): write content length in Data.Render"
+    changed_files: 2
+
+entry_points:
+  - "gin.go::Engine"
+  - "routergroup.go::RouterGroup"
+
+search_queries:
+  - query: "Engine ServeHTTP"
+    expected: "gin.go::Engine"
+  - query: "Context request"
+    expected: "context.go::Context"
+  - query: "node tree"
+    expected: "tree.go::node"
+
+multi_hop_tasks:
+  - id: gin-serve-http-callees
+    nl_query: "What does the gin engine do when serving an HTTP request"
+    anchor_qualified_suffix: "gin.go::engine.servehttp"
+    traversal_pattern: callees_of
+    expected_neighbor_names: ["reset"]
+    k: 10
+  - id: gin-context-next-callers
+    nl_query: "Who advances the gin middleware chain via Context.Next"
+    anchor_qualified_suffix: "context.go::context.next"
+    traversal_pattern: callers_of
+    expected_neighbor_names: ["handleHTTPRequest", "serveError"]
+    k: 10
+
+# Questions for the agent_baseline benchmark (pure-python grep top-k vs graph
+# query). See docs/REPRODUCING.md for the methodology.
+agent_questions:
+  - "How does Engine.ServeHTTP route an incoming request"
+  - "Where does Context.Next advance the middleware chain"
+  - "How does the node tree match wildcard routes"

code_review_graph/eval/configs/httpx.yaml

@@ -0,0 +1,48 @@
+name: httpx
+url: https://github.com/encode/httpx
+# Pinned to the latest test_commit SHA so the snapshot is deterministic and
+# every test_commit below is reachable as an ancestor.
+commit: b55d4635701d9dc22928ee647880c76b078ba3f2
+language: python
+size_category: small
+
+test_commits:
+  - sha: ae1b9f66238f75ced3ced5e4485408435de10768
+    description: "Expose FunctionAuth in __all__"
+    changed_files: 3
+  - sha: b55d4635701d9dc22928ee647880c76b078ba3f2
+    description: "Upgrade Python type checker mypy"
+    changed_files: 4
+
+entry_points:
+  - "httpx/_client.py::Client"
+  - "httpx/_client.py::AsyncClient"
+
+search_queries:
+  - query: "Client request"
+    expected: "httpx/_client.py::Client"
+  - query: "Response headers"
+    expected: "httpx/_models.py::Response"
+  - query: "BaseClient"
+    expected: "httpx/_client.py::BaseClient"
+
+multi_hop_tasks:
+  - id: httpx-client-request-callers
+    nl_query: "Which HTTP verbs route through the httpx Client.request"
+    anchor_qualified_suffix: "httpx/_client.py::client.request"
+    traversal_pattern: callers_of
+    expected_neighbor_names: ["get", "options", "head", "post", "put", "patch"]
+    k: 10
+  - id: httpx-async-request-tests
+    nl_query: "Tests covering the httpx async client request method"
+    anchor_qualified_suffix: "httpx/_client.py::asyncclient.request"
+    traversal_pattern: callers_of
+    expected_neighbor_names: ["test_raise_for_status"]
+    k: 10
+
+# Questions for the agent_baseline benchmark (pure-python grep top-k vs graph
+# query). See docs/REPRODUCING.md for the methodology.
+agent_questions:
+  - "How does Client.request send an HTTP request"
+  - "Where are Response headers parsed and decoded"
+  - "How does BaseClient build request URLs"

code_review_graph/eval/reporter.py

@@ -0,0 +1,301 @@
+"""Markdown report generator for evaluation benchmark results.
+
+Takes a list of benchmark result dicts and produces a formatted markdown table
+suitable for inclusion in documentation or CI output.
+"""
+
+from __future__ import annotations
+
+import csv
+from pathlib import Path
+from typing import Any
+
+
+def generate_markdown_report(results: list[dict[str, Any]]) -> str:
+    """Generate a markdown report from benchmark results.
+
+    Each result dict should contain at minimum a ``benchmark`` key identifying
+    the benchmark name, plus any metric keys (e.g. ``ratio``,
+    ``reduction_percent``, ``mrr``, ``precision``, ``recall``, ``f1``).
+
+    Args:
+        results: List of result dicts from benchmark runs.
+
+    Returns:
+        A markdown string containing a summary table and per-benchmark details.
+    """
+    if not results:
+        return "# Evaluation Report\n\nNo benchmark results to report.\n"
+
+    lines: list[str] = []
+    lines.append("# Evaluation Report")
+    lines.append("")
+
+    # Collect all metric keys across results (excluding 'benchmark')
+    all_keys: list[str] = []
+    seen: set[str] = set()
+    for r in results:
+        for k in r:
+            if k != "benchmark" and k not in seen:
+                all_keys.append(k)
+                seen.add(k)
+
+    # Summary table
+    lines.append("## Summary")
+    lines.append("")
+
+    header = "| Benchmark | " + " | ".join(all_keys) + " |"
+    separator = "| --- | " + " | ".join("---" for _ in all_keys) + " |"
+    lines.append(header)
+    lines.append(separator)
+
+    for r in results:
+        name = r.get("benchmark", "unknown")
+        values = [str(r.get(k, "-")) for k in all_keys]
+        lines.append(f"| {name} | " + " | ".join(values) + " |")
+
+    lines.append("")
+
+    # Per-benchmark detail sections
+    lines.append("## Details")
+    lines.append("")
+    for r in results:
+        name = r.get("benchmark", "unknown")
+        lines.append(f"### {name}")
+        lines.append("")
+        for k in all_keys:
+            v = r.get(k, "-")
+            lines.append(f"- **{k}**: {v}")
+        lines.append("")
+
+    return "\n".join(lines)
+
+
+def _read_csvs(results_dir: Path, prefix: str) -> list[dict[str, str]]:
+    """Read all CSV files matching a prefix from the results directory."""
+    rows: list[dict[str, str]] = []
+    for p in sorted(results_dir.glob(f"*_{prefix}_*.csv")):
+        with open(p, newline="") as f:
+            reader = csv.DictReader(f)
+            rows.extend(reader)
+    return rows
+
+
+def _md_table(headers: list[str], rows: list[list[str]]) -> str:
+    """Build a markdown table from headers and rows."""
+    lines = []
+    lines.append("| " + " | ".join(headers) + " |")
+    lines.append("| " + " | ".join("---" for _ in headers) + " |")
+    for row in rows:
+        lines.append("| " + " | ".join(row) + " |")
+    return "\n".join(lines)
+
+
+def generate_full_report(results_dir: str | Path) -> str:
+    """Generate a full markdown evaluation report from CSV result files.
+
+    Reads all CSV files in *results_dir*, groups them by benchmark type,
+    and produces a markdown report with methodology notes and per-benchmark
+    result tables.
+
+    Args:
+        results_dir: Directory containing CSV result files.
+
+    Returns:
+        Markdown string with the full report.
+    """
+    results_dir = Path(results_dir)
+    lines: list[str] = []
+    lines.append("# Evaluation Report")
+    lines.append("")
+    lines.append("## Methodology")
+    lines.append("")
+    lines.append("Benchmarks are run against real open-source repositories.")
+    lines.append("Token counts use a consistent `len(text) // 4` approximation.")
+    lines.append(
+        "Impact accuracy reports two ground-truth modes: "
+        "graph-derived (circular — upper bound) and co-change "
+        "(files co-changed in the same commit, seed excluded)."
+    )
+    lines.append(
+        "Rows with `status=error` are kept for forensics but excluded "
+        "from all aggregates."
+    )
+    lines.append("")
+
+    benchmark_types = [
+        "token_efficiency",
+        "impact_accuracy",
+        "agent_baseline",
+        "flow_completeness",
+        "search_quality",
+        "build_performance",
+        "multi_hop_retrieval",
+    ]
+
+    for btype in benchmark_types:
+        rows = _read_csvs(results_dir, btype)
+        if not rows:
+            continue
+
+        title = btype.replace("_", " ").title()
+        lines.append(f"## {title}")
+        lines.append("")
+
+        headers = list(rows[0].keys())
+        table_rows = [[r.get(h, "-") for h in headers] for r in rows]
+        lines.append(_md_table(headers, table_rows))
+        lines.append("")
+
+    if len(lines) <= 6:
+        lines.append("No benchmark results found.")
+        lines.append("")
+
+    return "\n".join(lines)
+
+
+def generate_readme_tables(results_dir: str | Path) -> str:
+    """Generate concise README-ready tables from CSV result files.
+
+    Produces three tables:
+    - Table A: Token Efficiency
+    - Table B: Accuracy & Quality
+    - Table C: Performance
+
+    Args:
+        results_dir: Directory containing CSV result files.
+
+    Returns:
+        Markdown string with the three tables.
+    """
+    results_dir = Path(results_dir)
+    lines: list[str] = []
+
+    # Table A: Token Efficiency
+    te_rows = _read_csvs(results_dir, "token_efficiency")
+    if te_rows:
+        lines.append("### Token Efficiency")
+        lines.append("")
+        headers = [
+            "Repo", "Files", "Naive Tokens", "Standard Tokens",
+            "Graph Tokens", "Naive/Graph", "Std/Graph",
+        ]
+        table_rows = []
+        for r in te_rows:
+            table_rows.append([
+                r.get("repo", "-"),
+                r.get("changed_files", "-"),
+                r.get("naive_tokens", "-"),
+                r.get("standard_tokens", "-"),
+                r.get("graph_tokens", "-"),
+                r.get("naive_to_graph_ratio", "-"),
+                r.get("standard_to_graph_ratio", "-"),
+            ])
+        lines.append(_md_table(headers, table_rows))
+        lines.append("")
+
+    # Table B: Accuracy & Quality
+    ia_rows = _read_csvs(results_dir, "impact_accuracy")
+    fc_rows = _read_csvs(results_dir, "flow_completeness")
+    sq_rows = _read_csvs(results_dir, "search_quality")
+
+    if ia_rows or fc_rows or sq_rows:
+        lines.append("### Accuracy & Quality")
+        lines.append("")
+        headers = ["Repo", "Impact F1 (graph-derived)", "Flow Recall", "Search MRR"]
+        # Build a per-repo summary
+        repo_data: dict[str, dict[str, object]] = {}
+        mrr_accum: dict[str, list[float]] = {}
+        f1_accum: dict[str, list[float]] = {}
+        for r in ia_rows:
+            # Failed rows are kept in the CSV for forensics but must never
+            # contribute to a headline number; co-change rows are a
+            # different metric and get their own reporting.
+            if r.get("status", "ok") not in ("", "ok"):
+                continue
+            mode = r.get("ground_truth_mode", "")
+            if mode and not mode.startswith("graph-derived"):
+                continue
+            repo = r.get("repo", "?")
+            repo_data.setdefault(repo, {})
+            try:
+                f1_accum.setdefault(repo, []).append(float(r.get("f1", "")))
+            except (ValueError, TypeError):
+                pass
+        for r in fc_rows:
+            repo_data.setdefault(r.get("repo", "?"), {})["recall"] = r.get("recall", "-")
+        for r in sq_rows:
+            repo = r.get("repo", "?")
+            repo_data.setdefault(repo, {})
+            try:
+                mrr_accum.setdefault(repo, []).append(float(r.get("reciprocal_rank", 0)))
+            except (ValueError, TypeError):
+                pass
+
+        table_rows = []
+        for repo, d in sorted(repo_data.items()):
+            mrr_vals = mrr_accum.get(repo, [])
+            mrr = (
+                str(round(sum(mrr_vals) / len(mrr_vals), 3))
+                if mrr_vals
+                else "-"
+            )
+            f1_vals = f1_accum.get(repo, [])
+            f1 = (
+                str(round(sum(f1_vals) / len(f1_vals), 3))
+                if f1_vals
+                else "-"
+            )
+            table_rows.append([
+                repo,
+                f1,
+                str(d.get("recall", "-")),
+                mrr,
+            ])
+        lines.append(_md_table(headers, table_rows))
+        lines.append("")
+
+    # Table B2: Agent Baseline (grep top-k vs graph query)
+    ab_rows = _read_csvs(results_dir, "agent_baseline")
+    if ab_rows:
+        lines.append("### Agent Baseline (grep top-k vs graph query)")
+        lines.append("")
+        headers = [
+            "Repo", "Question", "Baseline Tokens", "Graph Tokens",
+            "Baseline/Graph", "Status",
+        ]
+        table_rows = []
+        for r in ab_rows:
+            table_rows.append([
+                r.get("repo", "-"),
+                r.get("question", "-"),
+                r.get("baseline_tokens", "-"),
+                r.get("graph_tokens", "-"),
+                r.get("baseline_to_graph_ratio", "-"),
+                r.get("status", "ok") or "ok",
+            ])
+        lines.append(_md_table(headers, table_rows))
+        lines.append("")
+
+    # Table C: Performance
+    bp_rows = _read_csvs(results_dir, "build_performance")
+    if bp_rows:
+        lines.append("### Performance")
+        lines.append("")
+        headers = ["Repo", "Files", "Nodes", "Flow Det. (s)", "Search (ms)"]
+        table_rows = []
+        for r in bp_rows:
+            table_rows.append([
+                r.get("repo", "-"),
+                r.get("file_count", "-"),
+                r.get("node_count", "-"),
+                r.get("flow_detection_seconds", "-"),
+                r.get("search_avg_ms", "-"),
+            ])
+        lines.append(_md_table(headers, table_rows))
+        lines.append("")
+
+    if not lines:
+        return "No benchmark results found.\n"
+
+    return "\n".join(lines)

code_review_graph/eval/runner.py

@@ -0,0 +1,211 @@
+"""Evaluation runner: orchestrates benchmark execution across repositories."""
+
+from __future__ import annotations
+
+import csv
+import logging
+import subprocess
+from datetime import date
+from pathlib import Path
+
+try:
+    import yaml  # type: ignore[import-untyped]
+except ImportError:
+    yaml = None  # type: ignore[assignment]
+
+from code_review_graph.eval.benchmarks import (
+    agent_baseline,
+    build_performance,
+    flow_completeness,
+    impact_accuracy,
+    multi_hop_retrieval,
+    search_quality,
+    token_efficiency,
+)
+
+logger = logging.getLogger(__name__)
+
+BENCHMARK_REGISTRY = {
+    "token_efficiency": token_efficiency.run,
+    "impact_accuracy": impact_accuracy.run,
+    "flow_completeness": flow_completeness.run,
+    "search_quality": search_quality.run,
+    "build_performance": build_performance.run,
+    "multi_hop_retrieval": multi_hop_retrieval.run,
+    "agent_baseline": agent_baseline.run,
+}
+
+CONFIGS_DIR = Path(__file__).parent / "configs"
+DEFAULT_OUTPUT = Path("evaluate/results")
+DEFAULT_REPOS = Path("evaluate/test_repos")
+
+
+def _require_yaml():
+    if yaml is None:
+        raise ImportError("pyyaml is required: pip install code-review-graph[eval]")
+
+
+def load_config(name: str) -> dict:
+    """Load a single benchmark config by name."""
+    _require_yaml()
+    path = CONFIGS_DIR / f"{name}.yaml"
+    with open(path) as f:
+        return yaml.safe_load(f)
+
+
+def load_all_configs() -> list[dict]:
+    """Load all benchmark configs from the configs directory."""
+    _require_yaml()
+    configs = []
+    for p in sorted(CONFIGS_DIR.glob("*.yaml")):
+        with open(p) as f:
+            configs.append(yaml.safe_load(f))
+    return configs
+
+
+def clone_or_update(config: dict, repos_dir: Path | None = None) -> Path:
+    """Clone or update a repository at the config's pinned ``commit`` SHA.
+
+    Full clones (no ``--depth``) are required: the pinned ``test_commits`` are
+    often older than any reasonable shallow-clone window, and a missed SHA
+    used to silently fall back to ``git diff HEAD~1 HEAD`` — producing
+    benchmark numbers tied to whatever upstream HEAD looked like that day.
+
+    Every subprocess call's exit status is checked; failures raise
+    ``RuntimeError`` so reproducibility issues surface immediately instead of
+    yielding garbage results.
+    """
+    repos_dir = repos_dir or DEFAULT_REPOS
+    repos_dir.mkdir(parents=True, exist_ok=True)
+    repo_path = repos_dir / config["name"]
+
+    if repo_path.exists():
+        proc = subprocess.run(
+            ["git", "fetch", "--all", "--tags"],
+            cwd=str(repo_path),
+            capture_output=True,
+            text=True,
+        )
+        if proc.returncode != 0:
+            raise RuntimeError(
+                f"git fetch failed in {repo_path}: {proc.stderr.strip()}"
+            )
+    else:
+        proc = subprocess.run(
+            ["git", "clone", config["url"], str(repo_path)],
+            capture_output=True,
+            text=True,
+        )
+        if proc.returncode != 0:
+            raise RuntimeError(
+                f"git clone failed for {config['url']}: {proc.stderr.strip()}"
+            )
+
+    commit = config.get("commit", "HEAD")
+    if commit != "HEAD":
+        proc = subprocess.run(
+            ["git", "checkout", commit],
+            cwd=str(repo_path),
+            capture_output=True,
+            text=True,
+        )
+        if proc.returncode != 0:
+            raise RuntimeError(
+                f"git checkout {commit} failed in {repo_path}: "
+                f"{proc.stderr.strip()}"
+            )
+
+    return repo_path
+
+
+def write_csv(results: list[dict], path: Path) -> None:
+    """Write benchmark results to a CSV file."""
+    if not results:
+        return
+    path.parent.mkdir(parents=True, exist_ok=True)
+    fieldnames = list(results[0].keys())
+    with open(path, "w", newline="") as f:
+        writer = csv.DictWriter(f, fieldnames=fieldnames)
+        writer.writeheader()
+        writer.writerows(results)
+
+
+def run_eval(
+    repos: list[str] | None = None,
+    benchmarks: list[str] | None = None,
+    output_dir: str | Path | None = None,
+) -> dict[str, list[dict]]:
+    """Run evaluation benchmarks across repositories.
+
+    Args:
+        repos: List of repo config names to evaluate (None = all).
+        benchmarks: List of benchmark names to run (None = all).
+        output_dir: Directory for CSV output files.
+
+    Returns:
+        Dict mapping ``{repo}_{benchmark}`` to list of result dicts.
+    """
+    output_dir = Path(output_dir) if output_dir else DEFAULT_OUTPUT
+    output_dir.mkdir(parents=True, exist_ok=True)
+
+    if repos:
+        configs = [load_config(r) for r in repos]
+    else:
+        configs = load_all_configs()
+
+    benchmark_names = benchmarks or list(BENCHMARK_REGISTRY.keys())
+    all_results: dict[str, list[dict]] = {}
+    today = date.today().isoformat()
+
+    for config in configs:
+        name = config["name"]
+        logger.info("Evaluating %s...", name)
+
+        # Resolve the repo path to an absolute Path before handing it to
+        # full_build / get_db_path so the stored qualified_names match what
+        # the CLI/MCP layer produces (those paths go through _get_store ->
+        # _validate_repo_root which .resolve()s). Without this, a later
+        # ``code-review-graph update --repo <relative>`` writes the same
+        # function under a new absolute-prefixed qualified_name, leaving the
+        # graph with duplicate nodes for the same source location.
+        repo_path = clone_or_update(config).resolve()
+
+        # Build graph
+        from code_review_graph.graph import GraphStore
+        from code_review_graph.incremental import full_build, get_db_path
+        from code_review_graph.postprocessing import run_post_processing
+
+        db_path = get_db_path(repo_path)
+        store = GraphStore(db_path)
+
+        full_build(repo_path, store)
+        # full_build is the parsing-only primitive; the higher-level CLI/MCP
+        # wrappers run postprocessing on top. The eval framework bypasses
+        # those, so call it directly here. Without this, FTS5 stays empty
+        # and downstream benchmarks (token_efficiency, search_quality)
+        # silently produce useless results. See: search.rebuild_fts_index.
+        pp_result = run_post_processing(store)
+        for warning in pp_result.get("warnings", []):
+            logger.warning("  postprocessing: %s", warning)
+
+        for bench_name in benchmark_names:
+            if bench_name not in BENCHMARK_REGISTRY:
+                logger.warning("Unknown benchmark: %s", bench_name)
+                continue
+
+            logger.info("  Running %s...", bench_name)
+            try:
+                bench_fn = BENCHMARK_REGISTRY[bench_name]
+                results = bench_fn(repo_path, store, config)
+
+                key = f"{name}_{bench_name}"
+                all_results[key] = results
+                write_csv(results, output_dir / f"{key}_{today}.csv")
+                logger.info("  %s: %d result(s)", bench_name, len(results))
+            except Exception as e:
+                logger.error("  %s failed: %s", bench_name, e)
+                all_results[f"{name}_{bench_name}"] = []
+
+        store.close()
+
+    return all_results

code_review_graph/eval/scorer.py

@@ -0,0 +1,85 @@
+"""Scoring metrics for evaluating graph-based code review quality.
+
+Provides:
+- Token efficiency: measures how many tokens the graph saves vs raw context.
+- Mean Reciprocal Rank (MRR): evaluates ranking quality for search results.
+- Precision / Recall / F1: evaluates set-based retrieval accuracy.
+"""
+
+from __future__ import annotations
+
+
+def compute_token_efficiency(raw_tokens: int, graph_tokens: int) -> dict:
+    """Compute token efficiency metrics.
+
+    Args:
+        raw_tokens: Number of tokens when sending raw source code.
+        graph_tokens: Number of tokens when using graph-based context.
+
+    Returns:
+        Dict with keys:
+        - raw_tokens: the raw token count
+        - graph_tokens: the graph token count
+        - ratio: graph_tokens / raw_tokens (lower is better)
+        - reduction_percent: percentage of tokens saved (higher is better)
+    """
+    if raw_tokens <= 0:
+        return {
+            "raw_tokens": raw_tokens,
+            "graph_tokens": graph_tokens,
+            "ratio": 0.0,
+            "reduction_percent": 0.0,
+        }
+    ratio = graph_tokens / raw_tokens
+    reduction = (1.0 - ratio) * 100.0
+    return {
+        "raw_tokens": raw_tokens,
+        "graph_tokens": graph_tokens,
+        "ratio": round(ratio, 4),
+        "reduction_percent": round(reduction, 2),
+    }
+
+
+def compute_mrr(correct: str, results: list[str]) -> float:
+    """Compute Mean Reciprocal Rank for a single query.
+
+    Args:
+        correct: The correct/expected result identifier.
+        results: Ordered list of result identifiers (best first).
+
+    Returns:
+        1/rank if *correct* is found in *results*, else 0.0.
+    """
+    for i, r in enumerate(results, start=1):
+        if r == correct:
+            return 1.0 / i
+    return 0.0
+
+
+def compute_precision_recall(predicted: set, actual: set) -> dict:
+    """Compute precision, recall, and F1 score.
+
+    Args:
+        predicted: Set of predicted/returned items.
+        actual: Set of ground-truth items.
+
+    Returns:
+        Dict with keys: precision, recall, f1.
+    """
+    if not predicted and not actual:
+        return {"precision": 1.0, "recall": 1.0, "f1": 1.0}
+
+    true_positive = len(predicted & actual)
+    precision = true_positive / len(predicted) if predicted else 0.0
+    recall = true_positive / len(actual) if actual else 0.0
+
+    if precision + recall > 0:
+        f1 = 2 * precision * recall / (precision + recall)
+    else:
+        f1 = 0.0
+
+    return {
+        "precision": round(precision, 4),
+        "recall": round(recall, 4),
+        "f1": round(f1, 4),
+    }

code_review_graph/eval/token_benchmark.py

@@ -0,0 +1,182 @@
+"""Measures total tokens consumed by agent workflows against benchmark repos."""
+
+from __future__ import annotations
+
+import json
+import logging
+from typing import Any, Callable
+
+logger = logging.getLogger(__name__)
+
+
+def estimate_tokens(obj: Any) -> int:
+    """Estimate token count from JSON-serializable object.
+
+    Uses character count / 4 as a rough approximation for English + code.
+    """
+    return len(json.dumps(obj, default=str)) // 4
+
+
+def benchmark_review_workflow(repo_root: str, base: str = "HEAD~1") -> dict:
+    """Simulate a review workflow and measure total tokens consumed."""
+    from ..tools.context import get_minimal_context
+    from ..tools.review import detect_changes_func
+
+    total_tokens = 0
+    calls = []
+
+    # Step 1: get_minimal_context
+    result = get_minimal_context(task="review changes", repo_root=repo_root, base=base)
+    tokens = estimate_tokens(result)
+    total_tokens += tokens
+    calls.append({"tool": "get_minimal_context", "tokens": tokens})
+
+    # Step 2: detect_changes (minimal)
+    result = detect_changes_func(base=base, repo_root=repo_root, detail_level="minimal")
+    tokens = estimate_tokens(result)
+    total_tokens += tokens
+    calls.append({"tool": "detect_changes_minimal", "tokens": tokens})
+
+    return {
+        "workflow": "review",
+        "total_tokens": total_tokens,
+        "tool_calls": len(calls),
+        "calls": calls,
+    }
+
+
+def benchmark_architecture_workflow(repo_root: str) -> dict:
+    """Simulate an architecture exploration workflow."""
+    from ..tools.community_tools import list_communities_func
+    from ..tools.context import get_minimal_context
+    from ..tools.flows_tools import list_flows
+
+    total_tokens = 0
+    calls = []
+
+    result = get_minimal_context(task="map architecture", repo_root=repo_root)
+    tokens = estimate_tokens(result)
+    total_tokens += tokens
+    calls.append({"tool": "get_minimal_context", "tokens": tokens})
+
+    result = list_communities_func(repo_root=repo_root, detail_level="minimal")
+    tokens = estimate_tokens(result)
+    total_tokens += tokens
+    calls.append({"tool": "list_communities_minimal", "tokens": tokens})
+
+    result = list_flows(repo_root=repo_root, detail_level="minimal")
+    tokens = estimate_tokens(result)
+    total_tokens += tokens
+    calls.append({"tool": "list_flows_minimal", "tokens": tokens})
+
+    return {
+        "workflow": "architecture",
+        "total_tokens": total_tokens,
+        "tool_calls": len(calls),
+        "calls": calls,
+    }
+
+
+def benchmark_debug_workflow(repo_root: str) -> dict:
+    """Simulate a debug workflow."""
+    from ..tools.context import get_minimal_context
+    from ..tools.query import semantic_search_nodes
+
+    total_tokens = 0
+    calls = []
+
+    result = get_minimal_context(task="debug login bug", repo_root=repo_root)
+    tokens = estimate_tokens(result)
+    total_tokens += tokens
+    calls.append({"tool": "get_minimal_context", "tokens": tokens})
+
+    result = semantic_search_nodes(
+        query="login", repo_root=repo_root, detail_level="minimal",
+    )
+    tokens = estimate_tokens(result)
+    total_tokens += tokens
+    calls.append({"tool": "semantic_search_minimal", "tokens": tokens})
+
+    return {
+        "workflow": "debug",
+        "total_tokens": total_tokens,
+        "tool_calls": len(calls),
+        "calls": calls,
+    }
+
+
+def benchmark_onboard_workflow(repo_root: str) -> dict:
+    """Simulate an onboarding workflow."""
+    from ..tools.context import get_minimal_context
+    from ..tools.query import list_graph_stats
+
+    total_tokens = 0
+    calls = []
+
+    result = get_minimal_context(task="onboard developer", repo_root=repo_root)
+    tokens = estimate_tokens(result)
+    total_tokens += tokens
+    calls.append({"tool": "get_minimal_context", "tokens": tokens})
+
+    result = list_graph_stats(repo_root=repo_root)
+    tokens = estimate_tokens(result)
+    total_tokens += tokens
+    calls.append({"tool": "list_graph_stats", "tokens": tokens})
+
+    return {
+        "workflow": "onboard",
+        "total_tokens": total_tokens,
+        "tool_calls": len(calls),
+        "calls": calls,
+    }
+
+
+def benchmark_pre_merge_workflow(repo_root: str, base: str = "HEAD~1") -> dict:
+    """Simulate a pre-merge check workflow."""
+    from ..tools.context import get_minimal_context
+    from ..tools.review import detect_changes_func
+
+    total_tokens = 0
+    calls = []
+
+    result = get_minimal_context(task="pre-merge check", repo_root=repo_root, base=base)
+    tokens = estimate_tokens(result)
+    total_tokens += tokens
+    calls.append({"tool": "get_minimal_context", "tokens": tokens})
+
+    result = detect_changes_func(base=base, repo_root=repo_root, detail_level="minimal")
+    tokens = estimate_tokens(result)
+    total_tokens += tokens
+    calls.append({"tool": "detect_changes_minimal", "tokens": tokens})
+
+    return {
+        "workflow": "pre_merge",
+        "total_tokens": total_tokens,
+        "tool_calls": len(calls),
+        "calls": calls,
+    }
+
+
+ALL_WORKFLOWS: dict[str, Callable[..., dict]] = {
+    "review": benchmark_review_workflow,
+    "architecture": benchmark_architecture_workflow,
+    "debug": benchmark_debug_workflow,
+    "onboard": benchmark_onboard_workflow,
+    "pre_merge": benchmark_pre_merge_workflow,
+}
+
+
+def run_all_benchmarks(repo_root: str, base: str = "HEAD~1") -> list[dict]:
+    """Run all workflow benchmarks and return results."""
+    results = []
+    for name, fn in ALL_WORKFLOWS.items():
+        try:
+            if "base" in fn.__code__.co_varnames:
+                result = fn(repo_root=repo_root, base=base)
+            else:
+                result = fn(repo_root=repo_root)
+            results.append(result)
+        except Exception as e:
+            logger.warning("Benchmark %s failed: %s", name, e)
+            results.append({"workflow": name, "error": str(e)})
+    return results