chore: import upstream snapshot with attribution

code_review_graph/token_benchmark.py

@@ -0,0 +1,125 @@
+"""Token reduction benchmark -- measures graph query efficiency vs naive file reading."""
+
+from __future__ import annotations
+
+import logging
+import sqlite3
+from pathlib import Path
+from typing import Any
+
+from .graph import GraphStore
+from .search import hybrid_search
+
+logger = logging.getLogger(__name__)
+
+# Sample questions for benchmarking
+_SAMPLE_QUESTIONS = [
+    "how does authentication work",
+    "what is the main entry point",
+    "how are database connections managed",
+    "what error handling patterns are used",
+    "how do tests verify core functionality",
+]
+
+
+def estimate_tokens(text: str) -> int:
+    """Rough token estimate: ~4 chars per token."""
+    return max(1, len(text) // 4)
+
+
+def compute_naive_tokens(repo_root: Path) -> int:
+    """Count tokens in all parseable source files."""
+    total = 0
+    exts = (
+        ".py", ".js", ".ts", ".go", ".rs", ".java",
+        ".c", ".cpp", ".rb", ".php", ".swift", ".kt",
+    )
+    for ext in exts:
+        for f in repo_root.rglob(f"*{ext}"):
+            try:
+                total += estimate_tokens(
+                    f.read_text(errors="replace")
+                )
+            except OSError:
+                continue
+    return total
+
+
+def run_token_benchmark(
+    store: GraphStore,
+    repo_root: Path,
+    questions: list[str] | None = None,
+) -> dict[str, Any]:
+    """Run token reduction benchmark.
+
+    Compares naive full-corpus token cost vs graph query token
+    cost for a set of sample questions.
+
+    The default sample questions are natural language and require semantic
+    search to match. If no embeddings are present in the graph, ``hybrid_search``
+    falls back to FTS5/LIKE matching on node names, which produces no hits for
+    questions like "how does authentication work" — every per-question ratio
+    becomes 0 and the benchmark silently appears to fail. We log a clear
+    warning when that is the case so callers know to run ``embed_graph`` first
+    (or to pass keyword-matching questions).
+    """
+    if questions is None:
+        questions = _SAMPLE_QUESTIONS
+
+    using_default_questions = questions is _SAMPLE_QUESTIONS
+    try:
+        cur = store._conn.execute("SELECT count(*) FROM embeddings")
+        embedding_count = cur.fetchone()[0]
+    except sqlite3.OperationalError:
+        embedding_count = 0
+    if embedding_count == 0 and using_default_questions:
+        logger.warning(
+            "No embeddings found in this graph. The default sample questions "
+            "are natural language and will not match via FTS5/LIKE alone — "
+            "every reduction ratio is likely to be 0. Run "
+            "`code-review-graph embed` first, or pass keyword-matching `questions=`."
+        )
+
+    naive_total = compute_naive_tokens(repo_root)
+
+    results = []
+    for q in questions:
+        search_results = hybrid_search(store, q, limit=5)
+        # Simulate graph context: search results + neighbors
+        graph_tokens = 0
+        for r in search_results:
+            graph_tokens += estimate_tokens(str(r))
+            # Add approximate neighbor context
+            qn = r.get("qualified_name", "")
+            edges = store.get_edges_by_source(qn)[:5]
+            for e in edges:
+                graph_tokens += estimate_tokens(str(e))
+
+        if graph_tokens > 0:
+            ratio = naive_total / graph_tokens
+        else:
+            ratio = 0
+        results.append({
+            "question": q,
+            "naive_tokens": naive_total,
+            "graph_tokens": graph_tokens,
+            "reduction_ratio": round(ratio, 1),
+        })
+
+    if results:
+        total = sum(
+            r["reduction_ratio"] for r in results  # type: ignore[misc]
+        )
+        avg_ratio = float(total) / len(results)  # type: ignore[arg-type]
+    else:
+        avg_ratio = 0.0
+
+    return {
+        "naive_corpus_tokens": naive_total,
+        "per_question": results,
+        "average_reduction_ratio": round(avg_ratio, 1),
+        "summary": (
+            f"Graph queries use ~{avg_ratio:.0f}x fewer tokens "
+            f"than reading all source files"
+        ),
+    }