"""Token efficiency benchmark: compares naive, standard, and graph-based token counts. Failure semantics: if ``get_review_context`` throws, the row is recorded with ``status="error"`` and empty metric fields. It stays in the CSV for forensics but is excluded from every aggregate — a failed tool call is not a measurement. (Previously a failure silently produced ``graph_tokens=0`` and ``ratio = naive / 1``, inflating the results.) """ from __future__ import annotations import json import logging import statistics import subprocess from pathlib import Path logger = logging.getLogger(__name__) def _count_tokens(text: str) -> int: """Approximate token count (1 token ~ 4 chars).""" return len(text) // 4 def _get_changed_files(repo_path: Path, sha: str) -> list[str]: """Get list of changed files for a commit.""" result = subprocess.run( ["git", "diff", "--name-only", f"{sha}~1", sha], cwd=str(repo_path), capture_output=True, text=True, ) if result.returncode != 0: # Fallback: diff against parent result = subprocess.run( ["git", "diff", "--name-only", "HEAD~1", "HEAD"], cwd=str(repo_path), capture_output=True, text=True, ) return [f.strip() for f in result.stdout.strip().splitlines() if f.strip()] def _count_file_tokens(repo_path: Path, files: list[str]) -> int: """Count tokens from full file contents (naive approach).""" total = 0 for f in files: fp = repo_path / f if fp.is_file(): try: total += _count_tokens(fp.read_text(encoding="utf-8", errors="replace")) except OSError: pass return total def _count_diff_tokens(repo_path: Path, sha: str) -> int: """Count tokens from git diff output (standard approach).""" result = subprocess.run( ["git", "diff", f"{sha}~1", sha], cwd=str(repo_path), capture_output=True, text=True, ) if result.returncode != 0: result = subprocess.run( ["git", "diff", "HEAD~1", "HEAD"], cwd=str(repo_path), capture_output=True, text=True, ) return _count_tokens(result.stdout) def run(repo_path: Path, store, config: dict) -> list[dict]: """Run token efficiency benchmark.""" results = [] for tc in config.get("test_commits", []): changed = _get_changed_files(repo_path, tc["sha"]) if not changed: continue naive_tokens = _count_file_tokens(repo_path, changed) standard_tokens = _count_diff_tokens(repo_path, tc["sha"]) row: dict = { "repo": config["name"], "commit": tc["sha"], "description": tc.get("description", ""), "changed_files": len(changed), "naive_tokens": naive_tokens, "standard_tokens": standard_tokens, "graph_tokens": "", "naive_to_graph_ratio": "", "standard_to_graph_ratio": "", "status": "ok", "error": "", } # Graph-based: use get_review_context try: from code_review_graph.tools import get_review_context ctx = get_review_context( changed_files=changed, repo_root=str(repo_path) ) graph_tokens = _count_tokens(json.dumps(ctx)) except Exception as exc: # A failed tool call is not a measurement. Recording # graph_tokens=0 used to turn this into ratio = naive/1 — a # huge fake win. Mark the row failed; aggregate() excludes it. logger.warning("get_review_context failed on %s: %s", tc["sha"], exc) row["status"] = "error" row["error"] = str(exc)[:200] results.append(row) continue row["graph_tokens"] = graph_tokens row["naive_to_graph_ratio"] = round(naive_tokens / max(graph_tokens, 1), 1) row["standard_to_graph_ratio"] = round(standard_tokens / max(graph_tokens, 1), 1) results.append(row) return results def aggregate(results: list[dict]) -> dict: """Aggregate token-efficiency rows, excluding failed measurements. Rows with ``status != "ok"`` stay in the CSV for forensics but must not contribute to any headline number. """ ok = [r for r in results if r.get("status") == "ok"] ratios = [float(r["naive_to_graph_ratio"]) for r in ok] return { "total_rows": len(results), "ok_rows": len(ok), "error_rows": sum(1 for r in results if r.get("status") == "error"), "median_naive_to_graph_ratio": ( round(statistics.median(ratios), 1) if ratios else None ), "mean_naive_to_graph_ratio": ( round(statistics.mean(ratios), 1) if ratios else None ), }