chore: import upstream snapshot with attribution
This commit is contained in:
wehub-resource-sync
@@ -0,0 +1 @@
|
||||
"""Benchmark modules for the evaluation framework."""
|
||||
@@ -0,0 +1,193 @@
|
||||
"""Agent baseline benchmark: grep-and-read-top-k versus a graph query.
|
||||
|
||||
The whole-corpus baseline in the standalone token benchmark is an upper
|
||||
bound no real agent pays: a competent agent greps for identifiers from the
|
||||
question and reads only the best-matching files. This benchmark measures
|
||||
that realistic baseline:
|
||||
|
||||
1. Derive search terms from the question (identifier-shaped tokens via
|
||||
``search.extract_query_identifiers`` plus plain keywords).
|
||||
2. Pure-python grep over the corpus (no external ``rg``/``grep`` binary),
|
||||
ranking files by total case-insensitive match count.
|
||||
3. Read the top-k files (k=3) and token-count them with the chars/4 utility
|
||||
(``token_benchmark.estimate_tokens``) as ``baseline_tokens``.
|
||||
4. Compare against the graph-query cost for the same question — hybrid
|
||||
search hits plus one hop of neighbor edges, the same accounting used by
|
||||
``code_review_graph/token_benchmark.py``.
|
||||
|
||||
Questions come from ``agent_questions:`` in the repo config, falling back to
|
||||
the ``search_queries`` query strings when absent.
|
||||
|
||||
Failure semantics match the other benchmarks: a thrown search is recorded
|
||||
with ``status="error"`` and excluded from aggregates; rows where either side
|
||||
of the ratio is zero get ``status="no_graph_results"`` /
|
||||
``status="no_baseline_match"`` and are likewise excluded.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import statistics
|
||||
from collections.abc import Iterator
|
||||
from pathlib import Path
|
||||
|
||||
from code_review_graph.token_benchmark import estimate_tokens
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
DEFAULT_TOP_K = 3
|
||||
|
||||
_SOURCE_EXTS = (
|
||||
".py", ".js", ".ts", ".tsx", ".go", ".rs", ".java",
|
||||
".c", ".cpp", ".h", ".rb", ".php", ".swift", ".kt",
|
||||
)
|
||||
|
||||
_SKIP_DIRS = {
|
||||
".git", ".hg", ".svn", "node_modules", "__pycache__",
|
||||
".code-review-graph", ".venv", "venv", "dist", "build",
|
||||
}
|
||||
|
||||
_STOPWORDS = {
|
||||
"how", "does", "do", "the", "a", "an", "is", "are", "was", "what",
|
||||
"where", "when", "which", "who", "why", "and", "or", "in", "on", "of",
|
||||
"to", "for", "with", "via", "into", "from", "this", "that", "it", "its",
|
||||
}
|
||||
|
||||
|
||||
def derive_search_terms(question: str) -> list[str]:
|
||||
"""Derive lowercase grep terms: identifiers first, then plain keywords.
|
||||
|
||||
Identifier-shaped tokens (``Client.request``, ``get_users``, ``APIRoute``)
|
||||
are extracted via ``search.extract_query_identifiers``; remaining words of
|
||||
3+ characters that are not stopwords are appended. Order is deterministic.
|
||||
"""
|
||||
from code_review_graph.search import extract_query_identifiers
|
||||
|
||||
terms: list[str] = []
|
||||
seen: set[str] = set()
|
||||
for ident in extract_query_identifiers(question):
|
||||
if ident not in seen:
|
||||
seen.add(ident)
|
||||
terms.append(ident)
|
||||
for word in question.split():
|
||||
w = word.strip(".,;:!?\"'()[]{}`").lower()
|
||||
if len(w) >= 3 and w not in _STOPWORDS and w not in seen:
|
||||
seen.add(w)
|
||||
terms.append(w)
|
||||
return terms
|
||||
|
||||
|
||||
def iter_source_files(repo_path: Path) -> Iterator[Path]:
|
||||
"""Yield source files under *repo_path*, skipping vendored/VCS dirs."""
|
||||
for path in sorted(repo_path.rglob("*")):
|
||||
if path.suffix not in _SOURCE_EXTS or not path.is_file():
|
||||
continue
|
||||
if any(part in _SKIP_DIRS for part in path.parts):
|
||||
continue
|
||||
yield path
|
||||
|
||||
|
||||
def grep_rank(
|
||||
repo_path: Path, terms: list[str], k: int = DEFAULT_TOP_K,
|
||||
) -> list[tuple[str, int]]:
|
||||
"""Rank source files by total case-insensitive term matches; take top-k.
|
||||
|
||||
Pure python — no external grep/rg dependency. Deterministic: ties break
|
||||
on the relative path. Files with zero matches are dropped.
|
||||
"""
|
||||
lowered = [t.lower() for t in terms if t]
|
||||
if not lowered:
|
||||
return []
|
||||
scores: list[tuple[str, int]] = []
|
||||
for path in iter_source_files(repo_path):
|
||||
try:
|
||||
text = path.read_text(encoding="utf-8", errors="replace").lower()
|
||||
except OSError:
|
||||
continue
|
||||
count = sum(text.count(term) for term in lowered)
|
||||
if count > 0:
|
||||
scores.append((str(path.relative_to(repo_path)), count))
|
||||
scores.sort(key=lambda item: (-item[1], item[0]))
|
||||
return scores[:k]
|
||||
|
||||
|
||||
def run(repo_path: Path, store, config: dict) -> list[dict]:
|
||||
"""Run the agent baseline benchmark for one repo."""
|
||||
questions = list(config.get("agent_questions") or [])
|
||||
if not questions:
|
||||
questions = [sq["query"] for sq in config.get("search_queries", [])]
|
||||
|
||||
k = int(config.get("agent_baseline_top_k", DEFAULT_TOP_K))
|
||||
results: list[dict] = []
|
||||
|
||||
for question in questions:
|
||||
terms = derive_search_terms(question)
|
||||
top = grep_rank(repo_path, terms, k=k)
|
||||
baseline_tokens = 0
|
||||
for rel, _count in top:
|
||||
try:
|
||||
baseline_tokens += estimate_tokens(
|
||||
(repo_path / rel).read_text(encoding="utf-8", errors="replace")
|
||||
)
|
||||
except OSError:
|
||||
continue
|
||||
|
||||
row: dict = {
|
||||
"repo": config["name"],
|
||||
"question": question,
|
||||
"terms": " ".join(terms),
|
||||
"files_matched": len(top),
|
||||
"top_files": ";".join(rel for rel, _ in top),
|
||||
"baseline_tokens": baseline_tokens,
|
||||
"graph_tokens": "",
|
||||
"baseline_to_graph_ratio": "",
|
||||
"status": "ok",
|
||||
"error": "",
|
||||
}
|
||||
|
||||
try:
|
||||
from code_review_graph.search import hybrid_search
|
||||
hits = hybrid_search(store, question, limit=5)
|
||||
except Exception as exc:
|
||||
logger.warning("hybrid_search failed on %r: %s", question, exc)
|
||||
row["status"] = "error"
|
||||
row["error"] = str(exc)[:200]
|
||||
results.append(row)
|
||||
continue
|
||||
|
||||
# Same accounting as the standalone token benchmark: search hits
|
||||
# plus up to 5 outgoing edges of neighbor context per hit.
|
||||
graph_tokens = 0
|
||||
for hit in hits:
|
||||
graph_tokens += estimate_tokens(str(hit))
|
||||
qn = hit.get("qualified_name", "")
|
||||
for edge in store.get_edges_by_source(qn)[:5]:
|
||||
graph_tokens += estimate_tokens(str(edge))
|
||||
|
||||
row["graph_tokens"] = graph_tokens
|
||||
if baseline_tokens > 0 and graph_tokens > 0:
|
||||
row["baseline_to_graph_ratio"] = round(baseline_tokens / graph_tokens, 1)
|
||||
elif graph_tokens == 0:
|
||||
row["status"] = "no_graph_results"
|
||||
else:
|
||||
row["status"] = "no_baseline_match"
|
||||
results.append(row)
|
||||
|
||||
return results
|
||||
|
||||
|
||||
def aggregate(results: list[dict]) -> dict:
|
||||
"""Aggregate over rows where both sides of the comparison exist."""
|
||||
ok = [r for r in results if r.get("status") == "ok"]
|
||||
ratios = [float(r["baseline_to_graph_ratio"]) for r in ok]
|
||||
return {
|
||||
"total_rows": len(results),
|
||||
"ok_rows": len(ok),
|
||||
"error_rows": sum(1 for r in results if r.get("status") == "error"),
|
||||
"median_baseline_to_graph_ratio": (
|
||||
round(statistics.median(ratios), 1) if ratios else None
|
||||
),
|
||||
"mean_baseline_to_graph_ratio": (
|
||||
round(statistics.mean(ratios), 1) if ratios else None
|
||||
),
|
||||
}
|
||||
@@ -0,0 +1,60 @@
|
||||
"""Build performance benchmark: measures timing of graph operations."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def run(repo_path: Path, store, config: dict) -> list[dict]:
|
||||
"""Run build performance benchmark."""
|
||||
stats = store.get_stats()
|
||||
|
||||
# Time flow detection
|
||||
try:
|
||||
from code_review_graph.flows import store_flows, trace_flows
|
||||
t0 = time.perf_counter()
|
||||
flows = trace_flows(store)
|
||||
store_flows(store, flows)
|
||||
flow_time = time.perf_counter() - t0
|
||||
except Exception as exc:
|
||||
logger.warning("Flow detection failed: %s", exc)
|
||||
flow_time = 0.0
|
||||
|
||||
# Time community detection
|
||||
try:
|
||||
from code_review_graph.communities import detect_communities, store_communities
|
||||
t0 = time.perf_counter()
|
||||
comms = detect_communities(store)
|
||||
store_communities(store, comms)
|
||||
community_time = time.perf_counter() - t0
|
||||
except Exception as exc:
|
||||
logger.warning("Community detection failed: %s", exc)
|
||||
community_time = 0.0
|
||||
|
||||
# Time search (average of queries)
|
||||
search_times: list[float] = []
|
||||
for sq in config.get("search_queries", [])[:10]:
|
||||
t0 = time.perf_counter()
|
||||
store.search_nodes(sq["query"], limit=20)
|
||||
search_times.append(time.perf_counter() - t0)
|
||||
|
||||
avg_search_ms = round(
|
||||
sum(search_times) / max(len(search_times), 1) * 1000, 1
|
||||
)
|
||||
|
||||
return [{
|
||||
"repo": config["name"],
|
||||
"file_count": stats.files_count,
|
||||
"node_count": stats.total_nodes,
|
||||
"edge_count": stats.total_edges,
|
||||
"flow_detection_seconds": round(flow_time, 3),
|
||||
"community_detection_seconds": round(community_time, 3),
|
||||
"search_avg_ms": avg_search_ms,
|
||||
"nodes_per_second": round(
|
||||
stats.total_nodes / max(flow_time, 0.001)
|
||||
),
|
||||
}]
|
||||
@@ -0,0 +1,36 @@
|
||||
"""Flow completeness benchmark: evaluates entry point detection and flow tracing."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from pathlib import Path
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def run(repo_path: Path, store, config: dict) -> list[dict]:
|
||||
"""Run flow completeness benchmark."""
|
||||
from code_review_graph.flows import store_flows, trace_flows
|
||||
|
||||
flows = trace_flows(store)
|
||||
count = store_flows(store, flows)
|
||||
|
||||
# Get detected entry point names
|
||||
detected_entries = set()
|
||||
for flow in flows:
|
||||
detected_entries.add(flow.get("entry_point") or flow.get("name", ""))
|
||||
|
||||
known = set(config.get("entry_points", []))
|
||||
found = sum(1 for ep in known if any(ep in d for d in detected_entries))
|
||||
|
||||
depths = [f.get("depth", 0) for f in flows]
|
||||
|
||||
return [{
|
||||
"repo": config["name"],
|
||||
"known_entry_points": len(known),
|
||||
"detected_entry_points": found,
|
||||
"recall": round(found / max(len(known), 1), 3),
|
||||
"detected_flows": count,
|
||||
"avg_flow_depth": round(sum(depths) / max(len(depths), 1), 1),
|
||||
"max_flow_depth": max(depths, default=0),
|
||||
}]
|
||||
@@ -0,0 +1,220 @@
|
||||
"""Impact accuracy benchmark: measures precision/recall of change impact analysis.
|
||||
|
||||
Two ground-truth modes are emitted side by side (``ground_truth_mode`` column):
|
||||
|
||||
- **graph-derived (circular — upper bound)** — the historical mode. Ground
|
||||
truth is the changed files plus files with CALLS/IMPORTS_FROM edges into
|
||||
them, i.e. derived from the same graph the predictor traverses. Recall in
|
||||
this mode is an upper bound by construction, not independent evidence.
|
||||
- **co-change (same commit, seed excluded)** — the honest mode. The predictor
|
||||
is seeded with a single changed file and graded against the *other* files
|
||||
the author actually touched in the same commit. The ground truth comes from
|
||||
git history, not from the graph.
|
||||
|
||||
Failure semantics: if ``analyze_changes`` throws, the row is recorded with
|
||||
``status="error"`` and empty metric fields — it stays in the CSV but is
|
||||
excluded from aggregates. (Previously a failure silently set
|
||||
``predicted = set(changed)``, guaranteeing a fake recall of 1.0.)
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import statistics
|
||||
import subprocess
|
||||
from pathlib import Path
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
MODE_GRAPH_DERIVED = "graph-derived (circular — upper bound)"
|
||||
MODE_CO_CHANGE = "co-change (same commit, seed excluded)"
|
||||
|
||||
|
||||
def _get_changed_files(repo_path: Path, sha: str) -> list[str]:
|
||||
"""Get list of changed files for a commit."""
|
||||
result = subprocess.run(
|
||||
["git", "diff", "--name-only", f"{sha}~1", sha],
|
||||
cwd=str(repo_path),
|
||||
capture_output=True,
|
||||
text=True,
|
||||
)
|
||||
if result.returncode != 0:
|
||||
result = subprocess.run(
|
||||
["git", "diff", "--name-only", "HEAD~1", "HEAD"],
|
||||
cwd=str(repo_path),
|
||||
capture_output=True,
|
||||
text=True,
|
||||
)
|
||||
return [f.strip() for f in result.stdout.strip().splitlines() if f.strip()]
|
||||
|
||||
|
||||
def _files_from_analysis(analysis: dict) -> set[str]:
|
||||
"""Extract predicted file paths from an ``analyze_changes`` result."""
|
||||
predicted: set[str] = set()
|
||||
for f in analysis.get("changed_functions", []):
|
||||
if isinstance(f, dict) and "file_path" in f:
|
||||
predicted.add(f["file_path"])
|
||||
elif isinstance(f, dict) and "file" in f:
|
||||
predicted.add(f["file"])
|
||||
for flow in analysis.get("affected_flows", []):
|
||||
if isinstance(flow, dict):
|
||||
for node in flow.get("nodes", []):
|
||||
if isinstance(node, dict) and "file_path" in node:
|
||||
predicted.add(node["file_path"])
|
||||
return predicted
|
||||
|
||||
|
||||
def _graph_neighbor_files(store, files: list[str]) -> set[str]:
|
||||
"""Files with CALLS/IMPORTS_FROM edges into any node of *files* (one hop)."""
|
||||
out: set[str] = set()
|
||||
for f in files:
|
||||
for node in store.get_nodes_by_file(f):
|
||||
for edge in store.get_edges_by_target(node.qualified_name):
|
||||
if edge.kind in ("CALLS", "IMPORTS_FROM"):
|
||||
src_qual = edge.source_qualified
|
||||
src_file = src_qual.split("::")[0] if "::" in src_qual else ""
|
||||
if src_file:
|
||||
out.add(src_file)
|
||||
return out
|
||||
|
||||
|
||||
def _base_row(repo: str, sha: str, mode: str, seed: str) -> dict:
|
||||
return {
|
||||
"repo": repo,
|
||||
"commit": sha,
|
||||
"ground_truth_mode": mode,
|
||||
"seed_file": seed,
|
||||
"predicted_files": "",
|
||||
"actual_files": "",
|
||||
"true_positives": "",
|
||||
"precision": "",
|
||||
"recall": "",
|
||||
"f1": "",
|
||||
"status": "ok",
|
||||
"error": "",
|
||||
}
|
||||
|
||||
|
||||
def _scored_row(
|
||||
repo: str, sha: str, mode: str, seed: str,
|
||||
predicted: set[str], actual: set[str],
|
||||
) -> dict:
|
||||
tp = len(predicted & actual)
|
||||
precision = tp / max(len(predicted), 1)
|
||||
recall = tp / max(len(actual), 1)
|
||||
f1 = 2 * precision * recall / max(precision + recall, 0.001)
|
||||
row = _base_row(repo, sha, mode, seed)
|
||||
row.update({
|
||||
"predicted_files": len(predicted),
|
||||
"actual_files": len(actual),
|
||||
"true_positives": tp,
|
||||
"precision": round(precision, 3),
|
||||
"recall": round(recall, 3),
|
||||
"f1": round(f1, 3),
|
||||
})
|
||||
return row
|
||||
|
||||
|
||||
def _error_row(repo: str, sha: str, mode: str, seed: str, exc: Exception) -> dict:
|
||||
row = _base_row(repo, sha, mode, seed)
|
||||
row["status"] = "error"
|
||||
row["error"] = str(exc)[:200]
|
||||
return row
|
||||
|
||||
|
||||
def run(repo_path: Path, store, config: dict) -> list[dict]:
|
||||
"""Run impact accuracy benchmark (both ground-truth modes)."""
|
||||
from code_review_graph.changes import analyze_changes
|
||||
|
||||
results = []
|
||||
repo = config["name"]
|
||||
for tc in config.get("test_commits", []):
|
||||
sha = tc["sha"]
|
||||
changed = _get_changed_files(repo_path, sha)
|
||||
if not changed:
|
||||
continue
|
||||
|
||||
# --- Mode 1: graph-derived ground truth (circular — upper bound) ---
|
||||
try:
|
||||
analysis = analyze_changes(
|
||||
store, changed, repo_root=str(repo_path), base=sha + "~1",
|
||||
)
|
||||
except Exception as exc:
|
||||
# Old behaviour set predicted = set(changed) here, which
|
||||
# guarantees recall 1.0 on a *failed* run. Mark failed instead.
|
||||
logger.warning("analyze_changes failed on %s: %s", sha, exc)
|
||||
results.append(_error_row(repo, sha, MODE_GRAPH_DERIVED, "", exc))
|
||||
analysis = None
|
||||
|
||||
if analysis is not None:
|
||||
predicted = set(changed) | _files_from_analysis(analysis)
|
||||
actual = set(changed) | _graph_neighbor_files(store, changed)
|
||||
results.append(
|
||||
_scored_row(repo, sha, MODE_GRAPH_DERIVED, "", predicted, actual)
|
||||
)
|
||||
|
||||
# --- Mode 2: co-change ground truth (honest) ---
|
||||
# Seed the predictor with a single changed file and grade against
|
||||
# the other files the author touched in the same commit. Note the
|
||||
# seed analysis deliberately gets no repo_root/diff: it must only
|
||||
# see the seed file, never the full commit diff.
|
||||
seed = sorted(changed)[0]
|
||||
co_actual = set(changed) - {seed}
|
||||
if not co_actual:
|
||||
row = _base_row(repo, sha, MODE_CO_CHANGE, seed)
|
||||
row["status"] = "skipped"
|
||||
row["error"] = "single-file commit: no co-changed files to grade against"
|
||||
results.append(row)
|
||||
continue
|
||||
|
||||
try:
|
||||
seed_analysis = analyze_changes(store, [seed])
|
||||
except Exception as exc:
|
||||
logger.warning("analyze_changes (seed=%s) failed on %s: %s", seed, sha, exc)
|
||||
results.append(_error_row(repo, sha, MODE_CO_CHANGE, seed, exc))
|
||||
continue
|
||||
|
||||
co_predicted = _files_from_analysis(seed_analysis)
|
||||
co_predicted |= _graph_neighbor_files(store, [seed])
|
||||
co_predicted.discard(seed)
|
||||
results.append(
|
||||
_scored_row(repo, sha, MODE_CO_CHANGE, seed, co_predicted, co_actual)
|
||||
)
|
||||
|
||||
return results
|
||||
|
||||
|
||||
def aggregate(results: list[dict]) -> dict:
|
||||
"""Per-mode means over successful rows only.
|
||||
|
||||
Error/skipped rows stay in the CSV but never contribute to a number.
|
||||
"""
|
||||
out: dict = {
|
||||
"total_rows": len(results),
|
||||
"error_rows": sum(1 for r in results if r.get("status") == "error"),
|
||||
"skipped_rows": sum(1 for r in results if r.get("status") == "skipped"),
|
||||
}
|
||||
for key, mode in (
|
||||
("graph_derived", MODE_GRAPH_DERIVED),
|
||||
("co_change", MODE_CO_CHANGE),
|
||||
):
|
||||
rows = [
|
||||
r for r in results
|
||||
if r.get("ground_truth_mode") == mode and r.get("status") == "ok"
|
||||
]
|
||||
out[key] = {
|
||||
"ok_rows": len(rows),
|
||||
"mean_precision": (
|
||||
round(statistics.mean(float(r["precision"]) for r in rows), 3)
|
||||
if rows else None
|
||||
),
|
||||
"mean_recall": (
|
||||
round(statistics.mean(float(r["recall"]) for r in rows), 3)
|
||||
if rows else None
|
||||
),
|
||||
"mean_f1": (
|
||||
round(statistics.mean(float(r["f1"]) for r in rows), 3)
|
||||
if rows else None
|
||||
),
|
||||
}
|
||||
return out
|
||||
@@ -0,0 +1,125 @@
|
||||
"""Multi-hop retrieval benchmark.
|
||||
|
||||
Tests a two-step tool chain that mimics how an LLM agent actually uses the
|
||||
graph for complex tasks:
|
||||
|
||||
1. ``hybrid_search(nl_query)`` to find a starting anchor from a natural-
|
||||
language question.
|
||||
2. ``query_graph(pattern, target=anchor)`` to traverse one hop along the
|
||||
requested edge kind (callers_of / callees_of / tests_for / ...).
|
||||
|
||||
For each task the benchmark records:
|
||||
|
||||
- ``anchor_found`` — did semantic search return a node whose qualified_name
|
||||
ends with the expected suffix in the top-K?
|
||||
- ``anchor_rank`` — index in the search result list (lower is better).
|
||||
- ``neighbor_count`` — number of neighbors returned by the traversal.
|
||||
- ``neighbor_recall`` — fraction of ``expected_neighbor_names`` that appear
|
||||
among the neighbor names.
|
||||
- ``score`` — ``int(anchor_found) * neighbor_recall``. Range 0–1.
|
||||
|
||||
Tasks are defined per-config under ``multi_hop_tasks:`` in
|
||||
``code_review_graph/eval/configs/*.yaml``. See
|
||||
``docs/REPRODUCING.md`` for the schema and the curated canonical task set.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _name_set(rows: list[dict[str, Any]]) -> set[str]:
|
||||
out: set[str] = set()
|
||||
for r in rows:
|
||||
name = (r.get("name") or "").lower()
|
||||
if name:
|
||||
out.add(name)
|
||||
return out
|
||||
|
||||
|
||||
def run(repo_path: Path, store, config: dict) -> list[dict]:
|
||||
"""Run the multi-hop retrieval benchmark for one repo."""
|
||||
# Imports are local so an import-time failure in one optional benchmark
|
||||
# does not poison the whole runner.
|
||||
from code_review_graph.search import hybrid_search
|
||||
from code_review_graph.tools.query import query_graph
|
||||
|
||||
repo_root = str(repo_path)
|
||||
results: list[dict] = []
|
||||
|
||||
for task in config.get("multi_hop_tasks", []):
|
||||
task_id = task["id"]
|
||||
nl_query = task["nl_query"]
|
||||
suffix = task["anchor_qualified_suffix"].lower()
|
||||
traversal = task.get("traversal_pattern", "callers_of")
|
||||
expected = [e.lower() for e in task.get("expected_neighbor_names", [])]
|
||||
k = int(task.get("k", 10))
|
||||
|
||||
# Step 1 — semantic search
|
||||
try:
|
||||
hits = hybrid_search(store, nl_query, limit=k)
|
||||
except Exception as exc: # noqa: BLE001 — benchmark must not abort the runner
|
||||
logger.warning("hybrid_search failed on %s: %s", task_id, exc)
|
||||
hits = []
|
||||
|
||||
anchor = None
|
||||
anchor_rank = -1
|
||||
for i, h in enumerate(hits):
|
||||
qn = (h.get("qualified_name") or "").lower()
|
||||
if qn.endswith(suffix):
|
||||
anchor = h
|
||||
anchor_rank = i
|
||||
break
|
||||
|
||||
if anchor is None:
|
||||
results.append({
|
||||
"repo": config["name"],
|
||||
"task_id": task_id,
|
||||
"nl_query": nl_query,
|
||||
"anchor_found": False,
|
||||
"anchor_rank": -1,
|
||||
"neighbor_count": 0,
|
||||
"expected_count": len(expected),
|
||||
"matched_count": 0,
|
||||
"neighbor_recall": 0.0,
|
||||
"score": 0.0,
|
||||
})
|
||||
continue
|
||||
|
||||
# Step 2 — single-hop graph traversal from the anchor
|
||||
try:
|
||||
trav = query_graph(
|
||||
pattern=traversal,
|
||||
target=anchor["qualified_name"],
|
||||
repo_root=repo_root,
|
||||
detail_level="standard",
|
||||
)
|
||||
except Exception as exc: # noqa: BLE001
|
||||
logger.warning(
|
||||
"query_graph(%s) failed on %s: %s", traversal, task_id, exc,
|
||||
)
|
||||
trav = {}
|
||||
|
||||
rows = trav.get("data") or trav.get("results") or []
|
||||
names = _name_set(rows)
|
||||
matched = sum(1 for e in expected if e in names)
|
||||
recall = matched / len(expected) if expected else 0.0
|
||||
|
||||
results.append({
|
||||
"repo": config["name"],
|
||||
"task_id": task_id,
|
||||
"nl_query": nl_query,
|
||||
"anchor_found": True,
|
||||
"anchor_rank": anchor_rank,
|
||||
"neighbor_count": len(rows),
|
||||
"expected_count": len(expected),
|
||||
"matched_count": matched,
|
||||
"neighbor_recall": round(recall, 3),
|
||||
"score": round(recall, 3),
|
||||
})
|
||||
|
||||
return results
|
||||
@@ -0,0 +1,59 @@
|
||||
"""Search quality benchmark: measures search result ranking via MRR."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import sqlite3
|
||||
from pathlib import Path
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def run(repo_path: Path, store, config: dict) -> list[dict]:
|
||||
"""Run search quality benchmark."""
|
||||
results = []
|
||||
for sq in config.get("search_queries", []):
|
||||
query = sq["query"]
|
||||
expected = sq["expected"]
|
||||
|
||||
try:
|
||||
from code_review_graph.search import hybrid_search
|
||||
search_results = hybrid_search(store, query, limit=20)
|
||||
except (ImportError, sqlite3.OperationalError) as exc:
|
||||
logger.debug("hybrid_search unavailable, using fallback: %s", exc)
|
||||
# Fallback to basic search
|
||||
search_results = [
|
||||
{"qualified_name": n.qualified_name}
|
||||
for n in store.search_nodes(query, limit=20)
|
||||
]
|
||||
|
||||
rank = 0
|
||||
for i, r in enumerate(search_results):
|
||||
if isinstance(r, dict):
|
||||
qn = r.get("qualified_name", "")
|
||||
elif hasattr(r, "qualified_name"):
|
||||
qn = r.qualified_name
|
||||
else:
|
||||
qn = ""
|
||||
qn_lower = qn.lower()
|
||||
exp_lower = expected.lower()
|
||||
# Match if expected is substring of qn, qn is substring of expected,
|
||||
# or the name part after :: matches
|
||||
exp_name = expected.rsplit("::", 1)[-1] if "::" in expected else expected
|
||||
qn_name = qn.rsplit("::", 1)[-1] if "::" in qn else qn
|
||||
if (
|
||||
exp_lower in qn_lower
|
||||
or qn_lower in exp_lower
|
||||
or exp_name.lower() == qn_name.lower()
|
||||
):
|
||||
rank = i + 1
|
||||
break
|
||||
|
||||
results.append({
|
||||
"repo": config["name"],
|
||||
"query": query,
|
||||
"expected": expected,
|
||||
"rank": rank,
|
||||
"reciprocal_rank": round(1.0 / rank if rank > 0 else 0.0, 3),
|
||||
})
|
||||
return results
|
||||
@@ -0,0 +1,143 @@
|
||||
"""Token efficiency benchmark: compares naive, standard, and graph-based token counts.
|
||||
|
||||
Failure semantics: if ``get_review_context`` throws, the row is recorded with
|
||||
``status="error"`` and empty metric fields. It stays in the CSV for forensics
|
||||
but is excluded from every aggregate — a failed tool call is not a
|
||||
measurement. (Previously a failure silently produced ``graph_tokens=0`` and
|
||||
``ratio = naive / 1``, inflating the results.)
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
import statistics
|
||||
import subprocess
|
||||
from pathlib import Path
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _count_tokens(text: str) -> int:
|
||||
"""Approximate token count (1 token ~ 4 chars)."""
|
||||
return len(text) // 4
|
||||
|
||||
|
||||
def _get_changed_files(repo_path: Path, sha: str) -> list[str]:
|
||||
"""Get list of changed files for a commit."""
|
||||
result = subprocess.run(
|
||||
["git", "diff", "--name-only", f"{sha}~1", sha],
|
||||
cwd=str(repo_path),
|
||||
capture_output=True,
|
||||
text=True,
|
||||
)
|
||||
if result.returncode != 0:
|
||||
# Fallback: diff against parent
|
||||
result = subprocess.run(
|
||||
["git", "diff", "--name-only", "HEAD~1", "HEAD"],
|
||||
cwd=str(repo_path),
|
||||
capture_output=True,
|
||||
text=True,
|
||||
)
|
||||
return [f.strip() for f in result.stdout.strip().splitlines() if f.strip()]
|
||||
|
||||
|
||||
def _count_file_tokens(repo_path: Path, files: list[str]) -> int:
|
||||
"""Count tokens from full file contents (naive approach)."""
|
||||
total = 0
|
||||
for f in files:
|
||||
fp = repo_path / f
|
||||
if fp.is_file():
|
||||
try:
|
||||
total += _count_tokens(fp.read_text(encoding="utf-8", errors="replace"))
|
||||
except OSError:
|
||||
pass
|
||||
return total
|
||||
|
||||
|
||||
def _count_diff_tokens(repo_path: Path, sha: str) -> int:
|
||||
"""Count tokens from git diff output (standard approach)."""
|
||||
result = subprocess.run(
|
||||
["git", "diff", f"{sha}~1", sha],
|
||||
cwd=str(repo_path),
|
||||
capture_output=True,
|
||||
text=True,
|
||||
)
|
||||
if result.returncode != 0:
|
||||
result = subprocess.run(
|
||||
["git", "diff", "HEAD~1", "HEAD"],
|
||||
cwd=str(repo_path),
|
||||
capture_output=True,
|
||||
text=True,
|
||||
)
|
||||
return _count_tokens(result.stdout)
|
||||
|
||||
|
||||
def run(repo_path: Path, store, config: dict) -> list[dict]:
|
||||
"""Run token efficiency benchmark."""
|
||||
results = []
|
||||
for tc in config.get("test_commits", []):
|
||||
changed = _get_changed_files(repo_path, tc["sha"])
|
||||
if not changed:
|
||||
continue
|
||||
|
||||
naive_tokens = _count_file_tokens(repo_path, changed)
|
||||
standard_tokens = _count_diff_tokens(repo_path, tc["sha"])
|
||||
|
||||
row: dict = {
|
||||
"repo": config["name"],
|
||||
"commit": tc["sha"],
|
||||
"description": tc.get("description", ""),
|
||||
"changed_files": len(changed),
|
||||
"naive_tokens": naive_tokens,
|
||||
"standard_tokens": standard_tokens,
|
||||
"graph_tokens": "",
|
||||
"naive_to_graph_ratio": "",
|
||||
"standard_to_graph_ratio": "",
|
||||
"status": "ok",
|
||||
"error": "",
|
||||
}
|
||||
|
||||
# Graph-based: use get_review_context
|
||||
try:
|
||||
from code_review_graph.tools import get_review_context
|
||||
ctx = get_review_context(
|
||||
changed_files=changed, repo_root=str(repo_path)
|
||||
)
|
||||
graph_tokens = _count_tokens(json.dumps(ctx))
|
||||
except Exception as exc:
|
||||
# A failed tool call is not a measurement. Recording
|
||||
# graph_tokens=0 used to turn this into ratio = naive/1 — a
|
||||
# huge fake win. Mark the row failed; aggregate() excludes it.
|
||||
logger.warning("get_review_context failed on %s: %s", tc["sha"], exc)
|
||||
row["status"] = "error"
|
||||
row["error"] = str(exc)[:200]
|
||||
results.append(row)
|
||||
continue
|
||||
|
||||
row["graph_tokens"] = graph_tokens
|
||||
row["naive_to_graph_ratio"] = round(naive_tokens / max(graph_tokens, 1), 1)
|
||||
row["standard_to_graph_ratio"] = round(standard_tokens / max(graph_tokens, 1), 1)
|
||||
results.append(row)
|
||||
return results
|
||||
|
||||
|
||||
def aggregate(results: list[dict]) -> dict:
|
||||
"""Aggregate token-efficiency rows, excluding failed measurements.
|
||||
|
||||
Rows with ``status != "ok"`` stay in the CSV for forensics but must not
|
||||
contribute to any headline number.
|
||||
"""
|
||||
ok = [r for r in results if r.get("status") == "ok"]
|
||||
ratios = [float(r["naive_to_graph_ratio"]) for r in ok]
|
||||
return {
|
||||
"total_rows": len(results),
|
||||
"ok_rows": len(ok),
|
||||
"error_rows": sum(1 for r in results if r.get("status") == "error"),
|
||||
"median_naive_to_graph_ratio": (
|
||||
round(statistics.median(ratios), 1) if ratios else None
|
||||
),
|
||||
"mean_naive_to_graph_ratio": (
|
||||
round(statistics.mean(ratios), 1) if ratios else None
|
||||
),
|
||||
}
|
||||
Reference in New Issue
Block a user