chore: import upstream snapshot with attribution
This commit is contained in:
@@ -0,0 +1,20 @@
|
||||
"""Code Review Graph - MCP server for persistent incremental code knowledge graphs."""
|
||||
|
||||
from .context_savings import (
|
||||
attach_context_savings,
|
||||
estimate_context_savings,
|
||||
estimate_file_tokens,
|
||||
estimate_tokens,
|
||||
format_context_savings,
|
||||
)
|
||||
|
||||
__version__ = "2.3.6"
|
||||
|
||||
__all__ = [
|
||||
"__version__",
|
||||
"attach_context_savings",
|
||||
"estimate_context_savings",
|
||||
"estimate_file_tokens",
|
||||
"estimate_tokens",
|
||||
"format_context_savings",
|
||||
]
|
||||
@@ -0,0 +1,4 @@
|
||||
"""Allow running as: python -m code_review_graph"""
|
||||
from .cli import main
|
||||
|
||||
main()
|
||||
@@ -0,0 +1,410 @@
|
||||
"""Graph analysis: hub detection, bridge nodes, knowledge gaps,
|
||||
surprise scoring, suggested questions."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from collections import Counter, defaultdict
|
||||
|
||||
from .graph import GraphStore, _sanitize_name
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def find_hub_nodes(store: GraphStore, top_n: int = 10) -> list[dict]:
|
||||
"""Find the most connected nodes (highest in+out degree), excluding File nodes.
|
||||
|
||||
Returns list of dicts with: name, qualified_name, kind, file,
|
||||
in_degree, out_degree, total_degree, community_id
|
||||
"""
|
||||
# Build degree counts from all edges
|
||||
edges = store.get_all_edges()
|
||||
in_degree: dict[str, int] = Counter()
|
||||
out_degree: dict[str, int] = Counter()
|
||||
for e in edges:
|
||||
out_degree[e.source_qualified] += 1
|
||||
in_degree[e.target_qualified] += 1
|
||||
|
||||
# Get all non-File nodes
|
||||
nodes = store.get_all_nodes(exclude_files=True)
|
||||
community_map = store.get_all_community_ids()
|
||||
|
||||
scored = []
|
||||
for n in nodes:
|
||||
qn = n.qualified_name
|
||||
ind = in_degree.get(qn, 0)
|
||||
outd = out_degree.get(qn, 0)
|
||||
total = ind + outd
|
||||
if total == 0:
|
||||
continue
|
||||
scored.append({
|
||||
"name": _sanitize_name(n.name),
|
||||
"qualified_name": n.qualified_name,
|
||||
"kind": n.kind,
|
||||
"file": n.file_path,
|
||||
"in_degree": ind,
|
||||
"out_degree": outd,
|
||||
"total_degree": total,
|
||||
"community_id": community_map.get(qn),
|
||||
})
|
||||
|
||||
scored.sort(
|
||||
key=lambda x: x.get("total_degree", 0), # type: ignore[arg-type,return-value]
|
||||
reverse=True,
|
||||
)
|
||||
return scored[:top_n]
|
||||
|
||||
|
||||
def find_bridge_nodes(
|
||||
store: GraphStore, top_n: int = 10
|
||||
) -> list[dict]:
|
||||
"""Find nodes with highest betweenness centrality.
|
||||
|
||||
These are architectural chokepoints that sit on shortest paths
|
||||
between many node pairs. If they break, multiple communities
|
||||
lose connectivity.
|
||||
|
||||
Returns list of dicts with: name, qualified_name, kind, file,
|
||||
betweenness, community_id
|
||||
"""
|
||||
import networkx as nx
|
||||
|
||||
# Build the graph — use cached version if available
|
||||
nxg = store._build_networkx_graph()
|
||||
|
||||
# Compute betweenness centrality (approximate for large graphs)
|
||||
n_nodes = nxg.number_of_nodes()
|
||||
if n_nodes > 5000:
|
||||
# Sample-based approximation for large graphs
|
||||
k = min(500, n_nodes)
|
||||
bc = nx.betweenness_centrality(nxg, k=k, normalized=True)
|
||||
elif n_nodes > 0:
|
||||
bc = nx.betweenness_centrality(nxg, normalized=True)
|
||||
else:
|
||||
return []
|
||||
|
||||
community_map = store.get_all_community_ids()
|
||||
node_map = {
|
||||
n.qualified_name: n
|
||||
for n in store.get_all_nodes(exclude_files=True)
|
||||
}
|
||||
|
||||
results = []
|
||||
for qn, score in bc.items():
|
||||
if score <= 0 or qn not in node_map:
|
||||
continue
|
||||
n = node_map[qn]
|
||||
if n.kind == "File":
|
||||
continue
|
||||
results.append({
|
||||
"name": _sanitize_name(n.name),
|
||||
"qualified_name": n.qualified_name,
|
||||
"kind": n.kind,
|
||||
"file": n.file_path,
|
||||
"betweenness": round(score, 6),
|
||||
"community_id": community_map.get(qn),
|
||||
})
|
||||
|
||||
results.sort(
|
||||
key=lambda x: float(x.get("betweenness", 0)), # type: ignore[arg-type,return-value]
|
||||
reverse=True,
|
||||
)
|
||||
return results[:top_n]
|
||||
|
||||
|
||||
def find_knowledge_gaps(store: GraphStore) -> dict[str, list[dict]]:
|
||||
"""Identify structural weaknesses in the codebase graph.
|
||||
|
||||
Returns dict with categories:
|
||||
- isolated_nodes: degree <= 1, disconnected from graph
|
||||
- thin_communities: fewer than 3 members
|
||||
- untested_hotspots: high-degree nodes with no TESTED_BY edges
|
||||
- single_file_communities: entire community in one file
|
||||
"""
|
||||
edges = store.get_all_edges()
|
||||
nodes = store.get_all_nodes(exclude_files=True)
|
||||
community_map = store.get_all_community_ids()
|
||||
|
||||
# Build degree map
|
||||
degree: dict[str, int] = Counter()
|
||||
tested_nodes: set[str] = set()
|
||||
for e in edges:
|
||||
degree[e.source_qualified] += 1
|
||||
degree[e.target_qualified] += 1
|
||||
if e.kind == "TESTED_BY":
|
||||
tested_nodes.add(e.source_qualified)
|
||||
|
||||
# 1. Isolated nodes (degree <= 1, not File)
|
||||
isolated = []
|
||||
for n in nodes:
|
||||
d = degree.get(n.qualified_name, 0)
|
||||
if d <= 1:
|
||||
isolated.append({
|
||||
"name": _sanitize_name(n.name),
|
||||
"qualified_name": n.qualified_name,
|
||||
"kind": n.kind,
|
||||
"file": n.file_path,
|
||||
"degree": d,
|
||||
})
|
||||
|
||||
# 2. Build community sizes and file maps from node data
|
||||
comm_sizes: Counter[int] = Counter()
|
||||
comm_files: dict[int, set[str]] = defaultdict(set)
|
||||
for n in nodes:
|
||||
cid = community_map.get(n.qualified_name)
|
||||
if cid is not None:
|
||||
comm_sizes[cid] += 1
|
||||
comm_files[cid].add(n.file_path)
|
||||
|
||||
# Thin communities (< 3 members)
|
||||
communities = store.get_communities_list()
|
||||
thin = []
|
||||
for c in communities:
|
||||
cid = int(c["id"])
|
||||
size = comm_sizes.get(cid, 0)
|
||||
if size < 3:
|
||||
thin.append({
|
||||
"community_id": cid,
|
||||
"name": str(c["name"]),
|
||||
"size": size,
|
||||
})
|
||||
|
||||
# 3. Untested hotspots (degree >= 5, no TESTED_BY)
|
||||
untested_hotspots = []
|
||||
for n in nodes:
|
||||
d = degree.get(n.qualified_name, 0)
|
||||
if (d >= 5
|
||||
and n.qualified_name not in tested_nodes
|
||||
and not n.is_test):
|
||||
untested_hotspots.append({
|
||||
"name": _sanitize_name(n.name),
|
||||
"qualified_name": n.qualified_name,
|
||||
"kind": n.kind,
|
||||
"file": n.file_path,
|
||||
"degree": d,
|
||||
})
|
||||
untested_hotspots.sort(
|
||||
key=lambda x: x.get("degree", 0), # type: ignore[arg-type,return-value]
|
||||
reverse=True,
|
||||
)
|
||||
|
||||
# 4. Single-file communities
|
||||
single_file = []
|
||||
for c in communities:
|
||||
cid = int(c["id"])
|
||||
files = comm_files.get(cid, set())
|
||||
size = comm_sizes.get(cid, 0)
|
||||
if len(files) == 1 and size >= 3:
|
||||
single_file.append({
|
||||
"community_id": cid,
|
||||
"name": str(c["name"]),
|
||||
"size": size,
|
||||
"file": next(iter(files)),
|
||||
})
|
||||
|
||||
return {
|
||||
"isolated_nodes": isolated[:50],
|
||||
"thin_communities": thin,
|
||||
"untested_hotspots": untested_hotspots[:20],
|
||||
"single_file_communities": single_file,
|
||||
}
|
||||
|
||||
|
||||
def find_surprising_connections(
|
||||
store: GraphStore, top_n: int = 15
|
||||
) -> list[dict]:
|
||||
"""Find edges with high surprise scores.
|
||||
|
||||
Detects unexpected architectural coupling based on:
|
||||
- Cross-community: source and target in different communities
|
||||
- Cross-language: different file languages
|
||||
- Peripheral-to-hub: low-degree node to high-degree node
|
||||
- Cross-file-type: test calling production or vice versa
|
||||
- Non-standard edge kind for the node types
|
||||
"""
|
||||
edges = store.get_all_edges()
|
||||
nodes = store.get_all_nodes(exclude_files=True)
|
||||
community_map = store.get_all_community_ids()
|
||||
|
||||
node_map = {n.qualified_name: n for n in nodes}
|
||||
|
||||
# Build degree map
|
||||
degree: dict[str, int] = Counter()
|
||||
for e in edges:
|
||||
degree[e.source_qualified] += 1
|
||||
degree[e.target_qualified] += 1
|
||||
|
||||
# Median degree for peripheral detection
|
||||
degrees = [d for d in degree.values() if d > 0]
|
||||
if not degrees:
|
||||
return []
|
||||
median_deg = sorted(degrees)[len(degrees) // 2]
|
||||
high_deg_threshold = max(median_deg * 3, 10)
|
||||
|
||||
scored_edges = []
|
||||
for e in edges:
|
||||
src = node_map.get(e.source_qualified)
|
||||
tgt = node_map.get(e.target_qualified)
|
||||
if not src or not tgt:
|
||||
continue
|
||||
if src.kind == "File" or tgt.kind == "File":
|
||||
continue
|
||||
|
||||
score = 0.0
|
||||
reasons = []
|
||||
|
||||
# Cross-community (+0.3)
|
||||
src_cid = community_map.get(e.source_qualified)
|
||||
tgt_cid = community_map.get(e.target_qualified)
|
||||
if (src_cid is not None
|
||||
and tgt_cid is not None
|
||||
and src_cid != tgt_cid):
|
||||
score += 0.3
|
||||
reasons.append("cross-community")
|
||||
|
||||
# Cross-language (+0.2)
|
||||
src_lang = (
|
||||
src.file_path.rsplit(".", 1)[-1]
|
||||
if "." in src.file_path else ""
|
||||
)
|
||||
tgt_lang = (
|
||||
tgt.file_path.rsplit(".", 1)[-1]
|
||||
if "." in tgt.file_path else ""
|
||||
)
|
||||
if src_lang and tgt_lang and src_lang != tgt_lang:
|
||||
score += 0.2
|
||||
reasons.append("cross-language")
|
||||
|
||||
# Peripheral-to-hub (+0.2)
|
||||
src_deg = degree.get(e.source_qualified, 0)
|
||||
tgt_deg = degree.get(e.target_qualified, 0)
|
||||
if ((src_deg <= 2 and tgt_deg >= high_deg_threshold)
|
||||
or (tgt_deg <= 2
|
||||
and src_deg >= high_deg_threshold)):
|
||||
score += 0.2
|
||||
reasons.append("peripheral-to-hub")
|
||||
|
||||
# Cross-file-type: test <-> non-test (+0.15)
|
||||
if src.is_test != tgt.is_test and e.kind == "CALLS":
|
||||
score += 0.15
|
||||
reasons.append("cross-test-boundary")
|
||||
|
||||
# Non-standard edge kind (+0.15)
|
||||
if e.kind == "CALLS" and src.kind == "Type":
|
||||
score += 0.15
|
||||
reasons.append("unusual-edge-kind")
|
||||
|
||||
if score > 0:
|
||||
scored_edges.append({
|
||||
"source": _sanitize_name(src.name),
|
||||
"source_qualified": e.source_qualified,
|
||||
"target": _sanitize_name(tgt.name),
|
||||
"target_qualified": e.target_qualified,
|
||||
"edge_kind": e.kind,
|
||||
"surprise_score": round(score, 2),
|
||||
"reasons": reasons,
|
||||
"source_community": src_cid,
|
||||
"target_community": tgt_cid,
|
||||
})
|
||||
|
||||
scored_edges.sort(
|
||||
key=lambda x: float(x.get("surprise_score", 0)), # type: ignore[arg-type,return-value]
|
||||
reverse=True,
|
||||
)
|
||||
return scored_edges[:top_n]
|
||||
|
||||
|
||||
def generate_suggested_questions(
|
||||
store: GraphStore,
|
||||
) -> list[dict]:
|
||||
"""Auto-generate review questions from graph analysis.
|
||||
|
||||
Categories:
|
||||
- bridge_node: Why does X connect communities A and B?
|
||||
- isolated_node: Is X dead code or dynamically invoked?
|
||||
- low_cohesion: Should community X be split?
|
||||
- hub_risk: Does hub node X have adequate test coverage?
|
||||
- surprising: Why does A call B across community boundary?
|
||||
"""
|
||||
questions = []
|
||||
|
||||
# Bridge node questions
|
||||
bridges = find_bridge_nodes(store, top_n=3)
|
||||
for b in bridges:
|
||||
questions.append({
|
||||
"category": "bridge_node",
|
||||
"question": (
|
||||
f"'{b['name']}' is a critical connector "
|
||||
f"between multiple code regions. Is it "
|
||||
f"adequately tested and documented?"
|
||||
),
|
||||
"target": b["qualified_name"],
|
||||
"priority": "high",
|
||||
})
|
||||
|
||||
# Hub risk questions
|
||||
hubs = find_hub_nodes(store, top_n=3)
|
||||
edges = store.get_all_edges()
|
||||
tested = {
|
||||
e.source_qualified
|
||||
for e in edges if e.kind == "TESTED_BY"
|
||||
}
|
||||
for h in hubs:
|
||||
if h["qualified_name"] not in tested:
|
||||
questions.append({
|
||||
"category": "hub_risk",
|
||||
"question": (
|
||||
f"Hub node '{h['name']}' has "
|
||||
f"{h['total_degree']} connections but no "
|
||||
f"direct test coverage. Should it be "
|
||||
f"tested?"
|
||||
),
|
||||
"target": h["qualified_name"],
|
||||
"priority": "high",
|
||||
})
|
||||
|
||||
# Surprising connection questions
|
||||
surprises = find_surprising_connections(store, top_n=3)
|
||||
for s in surprises:
|
||||
if "cross-community" in s["reasons"]:
|
||||
questions.append({
|
||||
"category": "surprising_connection",
|
||||
"question": (
|
||||
f"'{s['source']}' (community "
|
||||
f"{s['source_community']}) calls "
|
||||
f"'{s['target']}' (community "
|
||||
f"{s['target_community']}). Is this "
|
||||
f"coupling intentional?"
|
||||
),
|
||||
"target": s["source_qualified"],
|
||||
"priority": "medium",
|
||||
})
|
||||
|
||||
# Knowledge gap questions
|
||||
gaps = find_knowledge_gaps(store)
|
||||
|
||||
for c in gaps["thin_communities"][:2]:
|
||||
questions.append({
|
||||
"category": "thin_community",
|
||||
"question": (
|
||||
f"Community '{c['name']}' has only "
|
||||
f"{c['size']} member(s). Should it be "
|
||||
f"merged with a neighbor?"
|
||||
),
|
||||
"target": f"community:{c['community_id']}",
|
||||
"priority": "low",
|
||||
})
|
||||
|
||||
for h in gaps["untested_hotspots"][:2]:
|
||||
questions.append({
|
||||
"category": "untested_hotspot",
|
||||
"question": (
|
||||
f"'{h['name']}' has {h['degree']} "
|
||||
f"connections but no test coverage. "
|
||||
f"Is this a risk?"
|
||||
),
|
||||
"target": h["qualified_name"],
|
||||
"priority": "medium",
|
||||
})
|
||||
|
||||
return questions
|
||||
@@ -0,0 +1,409 @@
|
||||
"""Change impact analysis for code review.
|
||||
|
||||
Maps git/svn diffs to affected functions, flows, communities, and test coverage
|
||||
gaps. Produces risk-scored, priority-ordered review guidance.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
import subprocess
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from .constants import SECURITY_KEYWORDS as _SECURITY_KEYWORDS
|
||||
from .flows import get_affected_flows
|
||||
from .graph import GraphNode, GraphStore, _sanitize_name, node_to_dict
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_GIT_TIMEOUT = int(os.environ.get("CRG_GIT_TIMEOUT", "30")) # seconds, configurable
|
||||
|
||||
_SAFE_GIT_REF = re.compile(r"^[A-Za-z0-9_.~^/@{}\-]+$")
|
||||
_SAFE_SVN_REV = re.compile(r"^r?\d+(:r?\d+|:HEAD|:BASE|:COMMITTED)?$", re.IGNORECASE)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 1. parse_git_diff_ranges / parse_svn_diff_ranges
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def parse_git_diff_ranges(
|
||||
repo_root: str,
|
||||
base: str = "HEAD~1",
|
||||
) -> dict[str, list[tuple[int, int]]]:
|
||||
"""Run ``git diff --unified=0`` and extract changed line ranges per file.
|
||||
|
||||
Args:
|
||||
repo_root: Absolute path to the repository root.
|
||||
base: Git ref to diff against (default: ``HEAD~1``).
|
||||
|
||||
Returns:
|
||||
Mapping of file paths to lists of ``(start_line, end_line)`` tuples.
|
||||
Returns an empty dict on error.
|
||||
"""
|
||||
if not _SAFE_GIT_REF.match(base):
|
||||
logger.warning("Invalid git ref rejected: %s", base)
|
||||
return {}
|
||||
try:
|
||||
result = subprocess.run(
|
||||
["git", "diff", "--unified=0", base, "--"],
|
||||
capture_output=True,
|
||||
stdin=subprocess.DEVNULL,
|
||||
text=True,
|
||||
encoding="utf-8",
|
||||
errors="replace",
|
||||
cwd=repo_root,
|
||||
timeout=_GIT_TIMEOUT,
|
||||
)
|
||||
if result.returncode != 0:
|
||||
logger.warning("git diff failed (rc=%d): %s", result.returncode, result.stderr[:200])
|
||||
return {}
|
||||
except (OSError, subprocess.SubprocessError) as exc:
|
||||
logger.warning("git diff error: %s", exc)
|
||||
return {}
|
||||
|
||||
return _parse_unified_diff(result.stdout)
|
||||
|
||||
|
||||
def parse_svn_diff_ranges(
|
||||
repo_root: str,
|
||||
rev_range: str | None = None,
|
||||
) -> dict[str, list[tuple[int, int]]]:
|
||||
"""Run ``svn diff`` and extract changed line ranges per file.
|
||||
|
||||
Args:
|
||||
repo_root: Absolute path to the SVN working copy root.
|
||||
rev_range: Optional SVN revision range in ``rXXX:HEAD`` format.
|
||||
When *None*, diffs the working copy against BASE (local changes).
|
||||
|
||||
Returns:
|
||||
Mapping of file paths to lists of ``(start_line, end_line)`` tuples.
|
||||
Returns an empty dict on error.
|
||||
"""
|
||||
cmd = ["svn", "diff", "--non-interactive"]
|
||||
if rev_range:
|
||||
if not _SAFE_SVN_REV.match(rev_range):
|
||||
logger.warning("Invalid SVN revision range rejected: %s", rev_range)
|
||||
return {}
|
||||
cmd.extend(["-r", rev_range])
|
||||
try:
|
||||
result = subprocess.run(
|
||||
cmd,
|
||||
capture_output=True,
|
||||
stdin=subprocess.DEVNULL,
|
||||
text=True,
|
||||
encoding="utf-8",
|
||||
errors="replace",
|
||||
cwd=repo_root,
|
||||
timeout=_GIT_TIMEOUT,
|
||||
)
|
||||
if result.returncode != 0:
|
||||
logger.warning("svn diff failed (rc=%d): %s", result.returncode, result.stderr[:200])
|
||||
return {}
|
||||
except (OSError, subprocess.SubprocessError) as exc:
|
||||
logger.warning("svn diff error: %s", exc)
|
||||
return {}
|
||||
|
||||
return _parse_unified_diff(result.stdout)
|
||||
|
||||
|
||||
def parse_diff_ranges(
|
||||
repo_root: str,
|
||||
base: str = "HEAD~1",
|
||||
) -> dict[str, list[tuple[int, int]]]:
|
||||
"""Auto-detect VCS and return changed line ranges per file.
|
||||
|
||||
Dispatches to :func:`parse_git_diff_ranges` for Git repositories and
|
||||
:func:`parse_svn_diff_ranges` for SVN working copies.
|
||||
|
||||
Args:
|
||||
repo_root: Absolute path to the repository/working-copy root.
|
||||
base: For Git: the ref to diff against (default ``HEAD~1``).
|
||||
For SVN: an optional revision range (e.g. ``"r100:HEAD"``);
|
||||
when *base* is not a valid SVN revision, working-copy changes
|
||||
(``svn diff``) are used instead.
|
||||
"""
|
||||
root_path = Path(repo_root)
|
||||
if (root_path / ".svn").exists():
|
||||
rev_range = base if _SAFE_SVN_REV.match(base) else None
|
||||
return parse_svn_diff_ranges(repo_root, rev_range)
|
||||
return parse_git_diff_ranges(repo_root, base)
|
||||
|
||||
|
||||
def _parse_unified_diff(diff_text: str) -> dict[str, list[tuple[int, int]]]:
|
||||
"""Parse unified diff output into file -> line-range mappings.
|
||||
|
||||
Handles the ``@@ -old,count +new,count @@`` hunk header format.
|
||||
"""
|
||||
ranges: dict[str, list[tuple[int, int]]] = {}
|
||||
current_file: str | None = None
|
||||
|
||||
# Match "+++ b/path/to/file"
|
||||
file_pattern = re.compile(r"^\+\+\+ b/(.+)$")
|
||||
# Match "@@ ... +start,count @@" or "@@ ... +start @@"
|
||||
hunk_pattern = re.compile(r"^@@ .+? \+(\d+)(?:,(\d+))? @@")
|
||||
|
||||
for line in diff_text.splitlines():
|
||||
file_match = file_pattern.match(line)
|
||||
if file_match:
|
||||
current_file = file_match.group(1)
|
||||
continue
|
||||
|
||||
hunk_match = hunk_pattern.match(line)
|
||||
if hunk_match and current_file is not None:
|
||||
start = int(hunk_match.group(1))
|
||||
count = int(hunk_match.group(2)) if hunk_match.group(2) else 1
|
||||
if count == 0:
|
||||
# Pure deletion hunk (no lines added); still note the position.
|
||||
end = start
|
||||
else:
|
||||
end = start + count - 1
|
||||
ranges.setdefault(current_file, []).append((start, end))
|
||||
|
||||
return ranges
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 2. map_changes_to_nodes
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def map_changes_to_nodes(
|
||||
store: GraphStore,
|
||||
changed_ranges: dict[str, list[tuple[int, int]]],
|
||||
) -> list[GraphNode]:
|
||||
"""Find graph nodes whose line ranges overlap the changed lines.
|
||||
|
||||
Args:
|
||||
store: The graph store.
|
||||
changed_ranges: Mapping of file paths to ``(start, end)`` tuples.
|
||||
|
||||
Returns:
|
||||
Deduplicated list of overlapping graph nodes.
|
||||
"""
|
||||
seen: set[str] = set()
|
||||
result: list[GraphNode] = []
|
||||
|
||||
for file_path, ranges in changed_ranges.items():
|
||||
# Try the path as-is, then also try all nodes to match relative paths.
|
||||
nodes = store.get_nodes_by_file(file_path)
|
||||
if not nodes:
|
||||
# The graph may store absolute paths; try a suffix match.
|
||||
matched_paths = store.get_files_matching(file_path)
|
||||
for mp in matched_paths:
|
||||
nodes.extend(store.get_nodes_by_file(mp))
|
||||
|
||||
for node in nodes:
|
||||
if node.qualified_name in seen:
|
||||
continue
|
||||
if node.line_start is None or node.line_end is None:
|
||||
continue
|
||||
# Check overlap with any changed range.
|
||||
for start, end in ranges:
|
||||
if node.line_start <= end and node.line_end >= start:
|
||||
result.append(node)
|
||||
seen.add(node.qualified_name)
|
||||
break
|
||||
|
||||
return result
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 3. compute_risk_score
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def compute_risk_score(store: GraphStore, node: GraphNode) -> float:
|
||||
"""Compute a risk score (0.0 - 1.0) for a single node.
|
||||
|
||||
Scoring factors:
|
||||
- Flow participation: 0.05 per flow membership, capped at 0.25
|
||||
- Community crossing: 0.05 per caller from a different community, capped at 0.15
|
||||
- Test coverage: 0.30 (untested) scaling down to 0.05 (5+ TESTED_BY edges)
|
||||
- Security sensitivity: 0.20 if name matches security keywords
|
||||
- Caller count: callers / 20, capped at 0.10
|
||||
"""
|
||||
score = 0.0
|
||||
|
||||
# --- Flow participation (cap 0.25), weighted by criticality ---
|
||||
flow_criticalities = store.get_flow_criticalities_for_node(node.id)
|
||||
if flow_criticalities:
|
||||
score += min(sum(flow_criticalities), 0.25)
|
||||
else:
|
||||
flow_count = store.count_flow_memberships(node.id)
|
||||
score += min(flow_count * 0.05, 0.25)
|
||||
|
||||
# --- Community crossing (cap 0.15) ---
|
||||
callers = store.get_edges_by_target(node.qualified_name)
|
||||
caller_edges = [e for e in callers if e.kind == "CALLS"]
|
||||
|
||||
cross_community = 0
|
||||
node_cid = store.get_node_community_id(node.id)
|
||||
|
||||
if node_cid is not None and caller_edges:
|
||||
caller_qns = [edge.source_qualified for edge in caller_edges]
|
||||
cid_map = store.get_community_ids_by_qualified_names(caller_qns)
|
||||
for cid in cid_map.values():
|
||||
if cid is not None and cid != node_cid:
|
||||
cross_community += 1
|
||||
score += min(cross_community * 0.05, 0.15)
|
||||
|
||||
# --- Test coverage (direct + transitive) ---
|
||||
transitive_tests = store.get_transitive_tests(node.qualified_name)
|
||||
test_count = len(transitive_tests)
|
||||
score += 0.30 - (min(test_count / 5.0, 1.0) * 0.25)
|
||||
|
||||
# --- Security sensitivity ---
|
||||
name_lower = node.name.lower()
|
||||
qn_lower = node.qualified_name.lower()
|
||||
if any(kw in name_lower or kw in qn_lower for kw in _SECURITY_KEYWORDS):
|
||||
score += 0.20
|
||||
|
||||
# --- Caller count (cap 0.10) ---
|
||||
caller_count = len(caller_edges)
|
||||
score += min(caller_count / 20.0, 0.10)
|
||||
|
||||
return round(min(max(score, 0.0), 1.0), 4)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 4. analyze_changes
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def analyze_changes(
|
||||
store: GraphStore,
|
||||
changed_files: list[str],
|
||||
changed_ranges: dict[str, list[tuple[int, int]]] | None = None,
|
||||
repo_root: str | None = None,
|
||||
base: str = "HEAD~1",
|
||||
) -> dict[str, Any]:
|
||||
"""Analyze changes and produce risk-scored review guidance.
|
||||
|
||||
Args:
|
||||
store: The graph store.
|
||||
changed_files: List of changed file paths.
|
||||
changed_ranges: Optional pre-parsed diff ranges. If not provided and
|
||||
``repo_root`` is given, they are computed via the detected VCS
|
||||
(Git or SVN).
|
||||
repo_root: Repository root (for git/svn diff).
|
||||
base: Git ref or SVN revision range to diff against.
|
||||
|
||||
Returns:
|
||||
Dict with ``summary``, ``risk_score``, ``changed_functions``,
|
||||
``affected_flows``, ``test_gaps``, and ``review_priorities``.
|
||||
"""
|
||||
# Compute changed ranges if not provided.
|
||||
if changed_ranges is None and repo_root is not None:
|
||||
# Diff keys are forward-slash paths relative to the repo root, but
|
||||
# the graph stores absolute native paths. Remap so lookups work on
|
||||
# Windows, where the LIKE-suffix fallback cannot bridge
|
||||
# "src/app.py" to "C:\repo\src\app.py" (#528). Keys that are
|
||||
# already absolute pass through pathlib joining unchanged. The
|
||||
# explicit changed_ranges path (MCP) is untouched — tools/review.py
|
||||
# remaps before calling, and remapping twice would corrupt keys.
|
||||
root_path = Path(repo_root)
|
||||
changed_ranges = {
|
||||
str(root_path / key): ranges
|
||||
for key, ranges in parse_diff_ranges(repo_root, base).items()
|
||||
}
|
||||
|
||||
# Map changes to nodes.
|
||||
if changed_ranges:
|
||||
changed_nodes = map_changes_to_nodes(store, changed_ranges)
|
||||
else:
|
||||
# Fallback: all nodes in changed files.
|
||||
changed_nodes = []
|
||||
for fp in changed_files:
|
||||
changed_nodes.extend(store.get_nodes_by_file(fp))
|
||||
|
||||
# Filter to functions/tests for risk scoring (skip File nodes).
|
||||
changed_funcs = [
|
||||
n for n in changed_nodes
|
||||
if n.kind in ("Function", "Test", "Class")
|
||||
]
|
||||
|
||||
# Cap to prevent O(N*M) query explosion on large PRs.
|
||||
_max_funcs = int(os.environ.get("CRG_MAX_CHANGED_FUNCS", "500"))
|
||||
funcs_truncated = len(changed_funcs) > _max_funcs
|
||||
if funcs_truncated:
|
||||
changed_funcs = changed_funcs[:_max_funcs]
|
||||
|
||||
# Compute per-node risk scores.
|
||||
node_risks: list[dict[str, Any]] = []
|
||||
for node in changed_funcs:
|
||||
risk = compute_risk_score(store, node)
|
||||
node_risks.append({
|
||||
**node_to_dict(node),
|
||||
"risk_score": risk,
|
||||
})
|
||||
|
||||
# Overall risk score: max of individual risks, or 0.
|
||||
overall_risk = max((nr["risk_score"] for nr in node_risks), default=0.0)
|
||||
|
||||
# Affected flows.
|
||||
affected = get_affected_flows(store, changed_files)
|
||||
|
||||
# Detect test gaps: changed functions without TESTED_BY edges.
|
||||
test_gaps: list[dict[str, Any]] = []
|
||||
for node in changed_funcs:
|
||||
if node.is_test:
|
||||
continue
|
||||
tested = store.get_edges_by_target(node.qualified_name)
|
||||
if not any(e.kind == "TESTED_BY" for e in tested):
|
||||
test_gaps.append({
|
||||
"name": _sanitize_name(node.name),
|
||||
"qualified_name": _sanitize_name(node.qualified_name),
|
||||
"file": node.file_path,
|
||||
"line_start": node.line_start,
|
||||
"line_end": node.line_end,
|
||||
})
|
||||
|
||||
# Review priorities: top 10 by risk score.
|
||||
review_priorities = sorted(node_risks, key=lambda x: x["risk_score"], reverse=True)[:10]
|
||||
|
||||
# Build summary.
|
||||
summary_parts = [
|
||||
f"Analyzed {len(changed_files)} changed file(s):",
|
||||
f" - {len(changed_funcs)} changed function(s)/class(es)",
|
||||
f" - {affected['total']} affected flow(s)",
|
||||
f" - {len(test_gaps)} test gap(s)",
|
||||
f" - Overall risk score: {overall_risk:.2f}",
|
||||
]
|
||||
if test_gaps:
|
||||
# Dedup by bare name in the human summary. The underlying test_gaps
|
||||
# list keeps every entry (a downstream consumer needs precision via
|
||||
# qualified_name), but a graph that ended up with the same function
|
||||
# stored under two qualified_names (e.g. relative + absolute path
|
||||
# variants) would otherwise print "X, X, Y, Y" — surfacing graph
|
||||
# corruption as a UX bug. The root cause is path normalization;
|
||||
# this is the defensive last line.
|
||||
seen_names: set[str] = set()
|
||||
gap_names: list[str] = []
|
||||
for g in test_gaps:
|
||||
n = g["name"]
|
||||
if n in seen_names:
|
||||
continue
|
||||
seen_names.add(n)
|
||||
gap_names.append(n)
|
||||
if len(gap_names) >= 5:
|
||||
break
|
||||
summary_parts.append(f" - Untested: {', '.join(gap_names)}")
|
||||
if funcs_truncated:
|
||||
summary_parts.append(
|
||||
f" - Warning: analysis capped at {_max_funcs} functions "
|
||||
f"(set CRG_MAX_CHANGED_FUNCS to adjust)"
|
||||
)
|
||||
|
||||
return {
|
||||
"summary": "\n".join(summary_parts),
|
||||
"risk_score": overall_risk,
|
||||
"changed_functions": node_risks,
|
||||
"affected_flows": affected["affected_flows"],
|
||||
"test_gaps": test_gaps,
|
||||
"review_priorities": review_priorities,
|
||||
"functions_truncated": funcs_truncated,
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,874 @@
|
||||
"""Community/cluster detection for the code knowledge graph.
|
||||
|
||||
Detects communities of related code nodes using the Leiden algorithm (via igraph,
|
||||
optional) with a file-based grouping fallback when igraph is not installed.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import random
|
||||
import re
|
||||
from collections import Counter, defaultdict
|
||||
from typing import Any
|
||||
|
||||
from .graph import GraphEdge, GraphNode, GraphStore, _sanitize_name
|
||||
|
||||
# Fixed seed for igraph's RNG so Leiden community detection is reproducible
|
||||
# across runs. Without this, two builds of the same graph produce different
|
||||
# community IDs / sizes, breaking benchmark comparability. Override with
|
||||
# CRG_LEIDEN_SEED env var if you need a different seed.
|
||||
_LEIDEN_SEED = 42
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Stay well under SQLite's default 999-variable limit per statement.
|
||||
_SQL_BATCH = 450
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Optional igraph import
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
try:
|
||||
import igraph as ig # type: ignore[import-untyped]
|
||||
|
||||
IGRAPH_AVAILABLE = True
|
||||
except ImportError:
|
||||
ig = None # type: ignore[assignment]
|
||||
IGRAPH_AVAILABLE = False
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Edge weight mapping
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
EDGE_WEIGHTS: dict[str, float] = {
|
||||
"CALLS": 1.0,
|
||||
"IMPORTS_FROM": 0.5,
|
||||
"INHERITS": 0.8,
|
||||
"IMPLEMENTS": 0.7,
|
||||
"CONTAINS": 0.3,
|
||||
"TESTED_BY": 0.4,
|
||||
"DEPENDS_ON": 0.6,
|
||||
}
|
||||
|
||||
# Common words to filter when generating community names
|
||||
_COMMON_WORDS = frozenset({
|
||||
"get", "set", "self", "init", "new", "create", "update", "delete",
|
||||
"add", "remove", "make", "build", "from", "to", "for", "with",
|
||||
"the", "and", "test", "main", "run", "do", "is", "has", "on",
|
||||
"of", "in", "at", "by", "my", "this", "that", "all", "none",
|
||||
})
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Community naming
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _generate_community_name(members: list[GraphNode]) -> str:
|
||||
"""Generate a meaningful name for a community of nodes.
|
||||
|
||||
Algorithm:
|
||||
1. Find most common module/file prefix among members
|
||||
2. If a dominant class exists (>40% of nodes), use its name
|
||||
3. Fallback: most frequent keyword in function/class names
|
||||
4. Format: "{prefix}-{keyword}"
|
||||
"""
|
||||
if not members:
|
||||
return "empty"
|
||||
|
||||
# 1. Find common file prefix
|
||||
file_paths = [m.file_path for m in members]
|
||||
prefix = _extract_file_prefix(file_paths)
|
||||
|
||||
# 2. Check for dominant class
|
||||
class_names = [m.name for m in members if m.kind == "Class"]
|
||||
if class_names:
|
||||
class_counts = Counter(class_names)
|
||||
top_class, top_count = class_counts.most_common(1)[0]
|
||||
if top_count > len(members) * 0.4:
|
||||
if prefix:
|
||||
return f"{prefix}-{_to_slug(top_class)}"
|
||||
return _to_slug(top_class)
|
||||
|
||||
# 3. Most frequent keyword from function/class names
|
||||
keywords = _extract_keywords(members)
|
||||
keyword = keywords[0] if keywords else ""
|
||||
|
||||
if prefix and keyword:
|
||||
return f"{prefix}-{keyword}"
|
||||
if prefix:
|
||||
return prefix
|
||||
if keyword:
|
||||
return keyword
|
||||
return "cluster"
|
||||
|
||||
|
||||
def _extract_file_prefix(file_paths: list[str]) -> str:
|
||||
"""Find the most common short directory or module name from file paths."""
|
||||
if not file_paths:
|
||||
return ""
|
||||
# Extract the parent directory or file stem
|
||||
parts: list[str] = []
|
||||
for fp in file_paths:
|
||||
# Use the last directory component or file stem
|
||||
segments = fp.replace("\\", "/").split("/")
|
||||
# Take the parent dir if it exists, otherwise the file stem
|
||||
if len(segments) >= 2:
|
||||
parts.append(segments[-2])
|
||||
else:
|
||||
stem = segments[-1].rsplit(".", 1)[0]
|
||||
parts.append(stem)
|
||||
|
||||
counts = Counter(parts)
|
||||
top_part, _ = counts.most_common(1)[0]
|
||||
return _to_slug(top_part)
|
||||
|
||||
|
||||
def _extract_keywords(members: list[GraphNode]) -> list[str]:
|
||||
"""Extract the most frequent meaningful keywords from member names."""
|
||||
word_counts: Counter[str] = Counter()
|
||||
for m in members:
|
||||
if m.kind in ("Function", "Class", "Test", "Type"):
|
||||
words = _split_name(m.name)
|
||||
for w in words:
|
||||
wl = w.lower()
|
||||
if wl not in _COMMON_WORDS and len(wl) > 1:
|
||||
word_counts[wl] += 1
|
||||
|
||||
if not word_counts:
|
||||
return []
|
||||
return [w for w, _ in word_counts.most_common(5)]
|
||||
|
||||
|
||||
def _split_name(name: str) -> list[str]:
|
||||
"""Split a camelCase or snake_case name into words."""
|
||||
# Insert boundary before uppercase letters for camelCase
|
||||
s = re.sub(r"([a-z])([A-Z])", r"\1_\2", name)
|
||||
# Split on underscores, hyphens, dots
|
||||
return [p for p in re.split(r"[_\-.\s]+", s) if p]
|
||||
|
||||
|
||||
def _to_slug(s: str) -> str:
|
||||
"""Convert a string to a short lowercase slug."""
|
||||
return re.sub(r"[^a-z0-9]+", "-", s.lower()).strip("-")[:30]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Cohesion calculation
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _compute_cohesion_batch(
|
||||
community_member_qns: list[set[str]],
|
||||
all_edges: list[GraphEdge],
|
||||
) -> list[float]:
|
||||
"""Compute cohesion for multiple communities in a single O(edges) pass.
|
||||
|
||||
Builds a ``qualified_name -> community_index`` reverse map (each node
|
||||
appears in at most one community since all callers produce partitions),
|
||||
then walks every edge exactly once, bucketing it into internal/external
|
||||
counters per community.
|
||||
|
||||
Total work: O(edges + sum(|members|)) instead of
|
||||
O(edges * communities) for naive per-community cohesion.
|
||||
|
||||
Returns a list of cohesion scores aligned with ``community_member_qns``.
|
||||
"""
|
||||
qn_to_idx: dict[str, int] = {}
|
||||
for idx, members in enumerate(community_member_qns):
|
||||
for qn in members:
|
||||
qn_to_idx[qn] = idx
|
||||
|
||||
n = len(community_member_qns)
|
||||
internal = [0] * n
|
||||
external = [0] * n
|
||||
|
||||
for e in all_edges:
|
||||
sc = qn_to_idx.get(e.source_qualified)
|
||||
tc = qn_to_idx.get(e.target_qualified)
|
||||
if sc is None and tc is None:
|
||||
continue
|
||||
if sc == tc:
|
||||
# Safe: sc is not None here (sc == tc and not both None).
|
||||
assert sc is not None
|
||||
internal[sc] += 1
|
||||
else:
|
||||
if sc is not None:
|
||||
external[sc] += 1
|
||||
if tc is not None:
|
||||
external[tc] += 1
|
||||
|
||||
results: list[float] = []
|
||||
for i in range(n):
|
||||
total = internal[i] + external[i]
|
||||
results.append(internal[i] / total if total > 0 else 0.0)
|
||||
return results
|
||||
|
||||
|
||||
def _build_adjacency(edges: list[GraphEdge]) -> dict[str, list[str]]:
|
||||
"""Build adjacency list from edges (one pass over all edges)."""
|
||||
adj: dict[str, list[str]] = defaultdict(list)
|
||||
for e in edges:
|
||||
adj[e.source_qualified].append(e.target_qualified)
|
||||
adj[e.target_qualified].append(e.source_qualified)
|
||||
return adj
|
||||
|
||||
|
||||
def _compute_cohesion(
|
||||
member_qns: set[str],
|
||||
all_edges: list[GraphEdge],
|
||||
adj: dict[str, list[str]] | None = None,
|
||||
) -> float:
|
||||
"""Compute cohesion: internal_edges / (internal_edges + external_edges).
|
||||
|
||||
For multiple communities, prefer :func:`_compute_cohesion_batch`, which
|
||||
runs in O(edges) total instead of O(edges) per community.
|
||||
"""
|
||||
return _compute_cohesion_batch([member_qns], all_edges)[0]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Leiden-based community detection (igraph)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _detect_leiden(
|
||||
nodes: list[GraphNode],
|
||||
edges: list[GraphEdge],
|
||||
min_size: int,
|
||||
adj: dict[str, list[str]] | None = None,
|
||||
) -> list[dict[str, Any]]:
|
||||
"""Detect communities using Leiden algorithm via igraph.
|
||||
|
||||
Caps Leiden at ``n_iterations=2`` (sufficient for code dependency graphs)
|
||||
and skips the recursive sub-community splitting pass that caused
|
||||
exponential blow-up on large repos (>100k nodes).
|
||||
"""
|
||||
if ig is None:
|
||||
return []
|
||||
|
||||
qn_to_idx: dict[str, int] = {}
|
||||
idx_to_node: dict[int, GraphNode] = {}
|
||||
for i, node in enumerate(nodes):
|
||||
qn_to_idx[node.qualified_name] = i
|
||||
idx_to_node[i] = node
|
||||
|
||||
if not qn_to_idx:
|
||||
return []
|
||||
|
||||
logger.info("Building igraph with %d nodes...", len(qn_to_idx))
|
||||
|
||||
g = ig.Graph(n=len(qn_to_idx), directed=False)
|
||||
edge_list: list[tuple[int, int]] = []
|
||||
weights: list[float] = []
|
||||
seen_edges: set[tuple[int, int]] = set()
|
||||
|
||||
for e in edges:
|
||||
src_idx = qn_to_idx.get(e.source_qualified)
|
||||
tgt_idx = qn_to_idx.get(e.target_qualified)
|
||||
if src_idx is not None and tgt_idx is not None and src_idx != tgt_idx:
|
||||
pair = (min(src_idx, tgt_idx), max(src_idx, tgt_idx))
|
||||
if pair not in seen_edges:
|
||||
seen_edges.add(pair)
|
||||
edge_list.append(pair)
|
||||
weights.append(EDGE_WEIGHTS.get(e.kind, 0.5))
|
||||
|
||||
if not edge_list:
|
||||
return _detect_file_based(nodes, edges, min_size, adj=adj)
|
||||
|
||||
g.add_edges(edge_list)
|
||||
g.es["weight"] = weights
|
||||
|
||||
# Run Leiden -- scale resolution inversely with graph size to get
|
||||
# coarser clusters on large repos. Default resolution=1.0 produces
|
||||
# thousands of tiny communities for 30k+ node graphs.
|
||||
import math
|
||||
n_nodes = g.vcount()
|
||||
resolution = max(0.05, 1.0 / math.log10(max(n_nodes, 10)))
|
||||
|
||||
logger.info(
|
||||
"Running Leiden on %d nodes, %d edges...",
|
||||
g.vcount(), g.ecount(),
|
||||
)
|
||||
|
||||
import os
|
||||
seed = int(os.environ.get("CRG_LEIDEN_SEED", _LEIDEN_SEED))
|
||||
# Deterministic seeding for benchmark reproducibility — community
|
||||
# detection is not a security-sensitive context. nosec B311.
|
||||
ig.set_random_number_generator(random.Random(seed)) # nosec B311
|
||||
partition = g.community_leiden(
|
||||
objective_function="modularity",
|
||||
weights="weight",
|
||||
resolution=resolution,
|
||||
n_iterations=2,
|
||||
)
|
||||
|
||||
logger.info(
|
||||
"Leiden complete, found %d partitions. Computing cohesion...",
|
||||
len(partition),
|
||||
)
|
||||
|
||||
pending: list[tuple[list[GraphNode], set[str]]] = []
|
||||
for cluster_ids in partition:
|
||||
if len(cluster_ids) < min_size:
|
||||
continue
|
||||
members = [idx_to_node[i] for i in cluster_ids if i in idx_to_node]
|
||||
if len(members) < min_size:
|
||||
continue
|
||||
member_qns = {m.qualified_name for m in members}
|
||||
pending.append((members, member_qns))
|
||||
|
||||
cohesions = _compute_cohesion_batch([p[1] for p in pending], edges)
|
||||
|
||||
communities: list[dict[str, Any]] = []
|
||||
for (members, member_qns), cohesion in zip(pending, cohesions):
|
||||
lang_counts = Counter(m.language for m in members if m.language)
|
||||
dominant_lang = lang_counts.most_common(1)[0][0] if lang_counts else ""
|
||||
name = _generate_community_name(members)
|
||||
|
||||
communities.append({
|
||||
"name": name,
|
||||
"level": 0,
|
||||
"size": len(members),
|
||||
"cohesion": round(cohesion, 4),
|
||||
"dominant_language": dominant_lang,
|
||||
"description": f"Community of {len(members)} nodes",
|
||||
"members": [m.qualified_name for m in members],
|
||||
"member_qns": member_qns,
|
||||
})
|
||||
|
||||
logger.info("Community detection complete: %d communities", len(communities))
|
||||
return communities
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# File-based fallback community detection
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _detect_file_based(
|
||||
nodes: list[GraphNode],
|
||||
edges: list[GraphEdge],
|
||||
min_size: int,
|
||||
adj: dict[str, list[str]] | None = None,
|
||||
) -> list[dict[str, Any]]:
|
||||
"""Group nodes by directory when Leiden is unavailable or over-fragments.
|
||||
|
||||
Strips the longest common directory prefix from all file paths, then
|
||||
adaptively picks a grouping depth that yields 10-200 communities.
|
||||
"""
|
||||
# Collect all directory paths (normalized, without filename)
|
||||
all_dir_parts: list[list[str]] = []
|
||||
for n in nodes:
|
||||
parts = n.file_path.replace("\\", "/").split("/")
|
||||
all_dir_parts.append([p for p in parts[:-1] if p])
|
||||
|
||||
# Find the longest common prefix among directory parts
|
||||
prefix_len = 0
|
||||
if all_dir_parts:
|
||||
shortest = min(len(p) for p in all_dir_parts)
|
||||
for i in range(shortest):
|
||||
seg = all_dir_parts[0][i]
|
||||
if all(p[i] == seg for p in all_dir_parts):
|
||||
prefix_len = i + 1
|
||||
else:
|
||||
break
|
||||
|
||||
def _group_at_depth(depth: int) -> dict[str, list[GraphNode]]:
|
||||
groups: dict[str, list[GraphNode]] = defaultdict(list)
|
||||
for n in nodes:
|
||||
parts = n.file_path.replace("\\", "/").split("/")
|
||||
dir_parts = [p for p in parts[:-1] if p]
|
||||
remainder = dir_parts[prefix_len:]
|
||||
if remainder:
|
||||
key = "/".join(remainder[:depth])
|
||||
else:
|
||||
key = parts[-1].rsplit(".", 1)[0] if parts else "root"
|
||||
groups[key].append(n)
|
||||
return groups
|
||||
|
||||
# Try increasing depths until we get 10-200 qualifying groups
|
||||
max_depth = max((len(p) - prefix_len for p in all_dir_parts), default=0)
|
||||
best_groups = _group_at_depth(1) # depth=1 always works (file stem fallback)
|
||||
for depth in range(1, max_depth + 1):
|
||||
groups = _group_at_depth(depth)
|
||||
qualifying = sum(1 for v in groups.values() if len(v) >= min_size)
|
||||
best_groups = groups
|
||||
if qualifying >= 10:
|
||||
break
|
||||
|
||||
by_dir = best_groups
|
||||
|
||||
# Pre-filter to communities meeting min_size and collect their member
|
||||
# sets so we can batch-compute all cohesions in a single O(edges) pass.
|
||||
# Without this, per-community cohesion is O(edges * files), which makes
|
||||
# community detection effectively hang on large repos.
|
||||
pending: list[tuple[str, list[GraphNode], set[str]]] = []
|
||||
for dir_path, members in by_dir.items():
|
||||
if len(members) < min_size:
|
||||
continue
|
||||
member_qns = {m.qualified_name for m in members}
|
||||
pending.append((dir_path, members, member_qns))
|
||||
|
||||
cohesions = _compute_cohesion_batch([p[2] for p in pending], edges)
|
||||
|
||||
communities: list[dict[str, Any]] = []
|
||||
for (dir_path, members, member_qns), cohesion in zip(pending, cohesions):
|
||||
lang_counts = Counter(m.language for m in members if m.language)
|
||||
dominant_lang = lang_counts.most_common(1)[0][0] if lang_counts else ""
|
||||
name = _generate_community_name(members)
|
||||
|
||||
communities.append({
|
||||
"name": name,
|
||||
"level": 0,
|
||||
"size": len(members),
|
||||
"cohesion": round(cohesion, 4),
|
||||
"dominant_language": dominant_lang,
|
||||
"description": f"Directory-based community: {dir_path}",
|
||||
"members": [m.qualified_name for m in members],
|
||||
"member_qns": member_qns,
|
||||
})
|
||||
|
||||
return communities
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Oversized community splitting
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _split_oversized(
|
||||
communities: list[dict],
|
||||
nodes: list[GraphNode],
|
||||
edges: list[GraphEdge],
|
||||
threshold_pct: float = 0.25,
|
||||
min_split_size: int = 10,
|
||||
) -> list[dict]:
|
||||
"""Recursively split communities that exceed threshold_pct of total.
|
||||
|
||||
Uses Leiden on the subgraph of oversized communities. If igraph is
|
||||
not available, returns communities unchanged.
|
||||
"""
|
||||
if not IGRAPH_AVAILABLE:
|
||||
return communities
|
||||
|
||||
total = sum(
|
||||
c.get("size", len(c.get("members", [])))
|
||||
for c in communities
|
||||
)
|
||||
if total == 0:
|
||||
return communities
|
||||
|
||||
threshold = max(int(total * threshold_pct), min_split_size)
|
||||
result: list[dict] = []
|
||||
next_id = max(
|
||||
(c.get("id", 0) for c in communities), default=0
|
||||
) + 1
|
||||
|
||||
for comm in communities:
|
||||
members = set(comm.get("members", []))
|
||||
if len(members) <= threshold:
|
||||
result.append(comm)
|
||||
continue
|
||||
|
||||
# Build subgraph for this community
|
||||
member_nodes = [
|
||||
n for n in nodes
|
||||
if n.qualified_name in members
|
||||
]
|
||||
member_edges = [
|
||||
e for e in edges
|
||||
if (
|
||||
e.source_qualified in members
|
||||
and e.target_qualified in members
|
||||
)
|
||||
]
|
||||
|
||||
if len(member_nodes) < min_split_size:
|
||||
result.append(comm)
|
||||
continue
|
||||
|
||||
# Run Leiden on subgraph
|
||||
qn_to_idx = {
|
||||
n.qualified_name: i
|
||||
for i, n in enumerate(member_nodes)
|
||||
}
|
||||
ig_edges: list[tuple[int, int]] = []
|
||||
ig_weights: list[float] = []
|
||||
for e in member_edges:
|
||||
si = qn_to_idx.get(e.source_qualified)
|
||||
ti = qn_to_idx.get(e.target_qualified)
|
||||
if si is not None and ti is not None and si != ti:
|
||||
ig_edges.append((si, ti))
|
||||
ig_weights.append(
|
||||
EDGE_WEIGHTS.get(e.kind, 0.5)
|
||||
)
|
||||
|
||||
if not ig_edges:
|
||||
result.append(comm)
|
||||
continue
|
||||
|
||||
try:
|
||||
g = ig.Graph(
|
||||
n=len(member_nodes),
|
||||
edges=ig_edges,
|
||||
directed=False,
|
||||
)
|
||||
g.es["weight"] = ig_weights
|
||||
import os
|
||||
seed = int(os.environ.get("CRG_LEIDEN_SEED", _LEIDEN_SEED))
|
||||
# Deterministic seeding for benchmark reproducibility — community
|
||||
# detection is not a security-sensitive context. nosec B311.
|
||||
ig.set_random_number_generator(random.Random(seed)) # nosec B311
|
||||
partition = g.community_leiden(
|
||||
objective_function="modularity",
|
||||
weights="weight",
|
||||
resolution=0.5,
|
||||
)
|
||||
|
||||
sub_communities: dict[int, list[str]] = {}
|
||||
for idx, cid in enumerate(partition.membership):
|
||||
sub_communities.setdefault(cid, []).append(
|
||||
member_nodes[idx].qualified_name
|
||||
)
|
||||
|
||||
if len(sub_communities) <= 1:
|
||||
result.append(comm)
|
||||
continue
|
||||
|
||||
parent_id = comm.get("id", 0)
|
||||
comm_name = comm.get("name", "")
|
||||
for sub_members in sub_communities.values():
|
||||
sub_comm = {
|
||||
"id": next_id,
|
||||
"name": comm_name + f"-sub{next_id}",
|
||||
"level": comm.get("level", 0) + 1,
|
||||
"parent_id": parent_id,
|
||||
"members": sub_members,
|
||||
"size": len(sub_members),
|
||||
"cohesion": 0.0,
|
||||
"dominant_language": comm.get(
|
||||
"dominant_language"
|
||||
),
|
||||
"description": (
|
||||
f"Split from {comm_name}"
|
||||
),
|
||||
}
|
||||
result.append(sub_comm)
|
||||
next_id += 1
|
||||
|
||||
logger.info(
|
||||
"Split oversized community '%s' "
|
||||
"(%d members) into %d",
|
||||
comm_name,
|
||||
len(members),
|
||||
len(sub_communities),
|
||||
)
|
||||
except Exception:
|
||||
logger.warning(
|
||||
"Failed to split community '%s', "
|
||||
"keeping as-is",
|
||||
comm.get("name", ""),
|
||||
exc_info=True,
|
||||
)
|
||||
result.append(comm)
|
||||
|
||||
return result
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Public API
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def detect_communities(
|
||||
store: GraphStore, min_size: int = 2
|
||||
) -> list[dict[str, Any]]:
|
||||
"""Detect communities in the code graph.
|
||||
|
||||
Uses the Leiden algorithm via igraph if available, otherwise falls back to
|
||||
file-based grouping.
|
||||
|
||||
Args:
|
||||
store: The GraphStore instance.
|
||||
min_size: Minimum number of nodes for a community to be included.
|
||||
|
||||
Returns:
|
||||
List of community dicts with keys: name, level, size, cohesion,
|
||||
dominant_language, description, members, member_qns.
|
||||
"""
|
||||
# Gather all nodes (exclude File nodes to focus on code entities)
|
||||
all_edges = store.get_all_edges()
|
||||
unique_nodes = store.get_all_nodes(exclude_files=True)
|
||||
|
||||
# Build adjacency index once for fast cohesion computation
|
||||
adj = _build_adjacency(all_edges)
|
||||
|
||||
logger.info(
|
||||
"Loaded %d unique nodes, %d edges",
|
||||
len(unique_nodes), len(all_edges),
|
||||
)
|
||||
|
||||
if IGRAPH_AVAILABLE:
|
||||
logger.info("Detecting communities with Leiden algorithm (igraph)")
|
||||
results = _detect_leiden(unique_nodes, all_edges, min_size, adj=adj)
|
||||
else:
|
||||
logger.info("igraph not available, using file-based community detection")
|
||||
results = _detect_file_based(unique_nodes, all_edges, min_size, adj=adj)
|
||||
|
||||
# Split oversized communities
|
||||
results = _split_oversized(
|
||||
results, unique_nodes, all_edges,
|
||||
)
|
||||
|
||||
# Convert member_qns (internal set) to a list for serialization safety,
|
||||
# then strip it from the returned dicts to avoid leaking internal state.
|
||||
for comm in results:
|
||||
if "member_qns" in comm:
|
||||
comm["member_qns"] = list(comm["member_qns"])
|
||||
del comm["member_qns"]
|
||||
|
||||
return results
|
||||
|
||||
|
||||
def incremental_detect_communities(
|
||||
store: GraphStore,
|
||||
changed_files: list[str],
|
||||
min_size: int = 2,
|
||||
) -> int:
|
||||
"""Re-detect communities only if changed files affect existing communities.
|
||||
|
||||
If no existing communities contain nodes from changed files, skips
|
||||
re-detection entirely (the common case for small changes). Otherwise
|
||||
re-runs full community detection.
|
||||
|
||||
Args:
|
||||
store: The GraphStore instance.
|
||||
changed_files: List of file paths that have changed.
|
||||
min_size: Minimum number of nodes for a community to be included.
|
||||
|
||||
Returns:
|
||||
Number of communities detected, or 0 if skipped.
|
||||
"""
|
||||
if not changed_files:
|
||||
return 0
|
||||
|
||||
conn = store._conn
|
||||
|
||||
# Check if any communities are affected (batch to stay under SQLite limit)
|
||||
affected_count = 0
|
||||
for i in range(0, len(changed_files), _SQL_BATCH):
|
||||
batch = changed_files[i:i + _SQL_BATCH]
|
||||
placeholders = ",".join("?" * len(batch))
|
||||
row = conn.execute(
|
||||
f"SELECT COUNT(DISTINCT community_id) FROM nodes " # nosec B608
|
||||
f"WHERE community_id IS NOT NULL AND file_path IN ({placeholders})",
|
||||
batch,
|
||||
).fetchone()
|
||||
if row:
|
||||
affected_count += row[0]
|
||||
affected = (affected_count,) if affected_count else None
|
||||
|
||||
if not affected or affected[0] == 0:
|
||||
return 0 # No communities affected, skip
|
||||
|
||||
# Re-run full community detection (correct and fast enough)
|
||||
communities = detect_communities(store, min_size=min_size)
|
||||
return store_communities(store, communities)
|
||||
|
||||
|
||||
def store_communities(
|
||||
store: GraphStore, communities: list[dict[str, Any]]
|
||||
) -> int:
|
||||
"""Store detected communities in the database.
|
||||
|
||||
Clears existing communities and community_id assignments, then inserts
|
||||
the new communities and updates node community_id references.
|
||||
|
||||
Args:
|
||||
store: The GraphStore instance.
|
||||
communities: List of community dicts from detect_communities().
|
||||
|
||||
Returns:
|
||||
Number of communities stored.
|
||||
"""
|
||||
# NOTE: store_communities uses _conn directly because it performs
|
||||
# multi-statement batch writes (DELETE + INSERT loop + UPDATE loop)
|
||||
# that are tightly coupled to the DB transaction lifecycle.
|
||||
conn = store._conn
|
||||
|
||||
if conn.in_transaction:
|
||||
logger.warning("Rolling back uncommitted transaction before BEGIN IMMEDIATE")
|
||||
conn.rollback()
|
||||
# Wrap in explicit transaction so the DELETE + INSERT + UPDATE
|
||||
# sequence is atomic — no partial community data on crash.
|
||||
conn.execute("BEGIN IMMEDIATE")
|
||||
try:
|
||||
conn.execute("DELETE FROM communities")
|
||||
conn.execute("UPDATE nodes SET community_id = NULL")
|
||||
|
||||
count = 0
|
||||
for comm in communities:
|
||||
cursor = conn.execute(
|
||||
"""INSERT INTO communities
|
||||
(name, level, cohesion, size, dominant_language, description)
|
||||
VALUES (?, ?, ?, ?, ?, ?)""",
|
||||
(
|
||||
comm["name"],
|
||||
comm.get("level", 0),
|
||||
comm.get("cohesion", 0.0),
|
||||
comm["size"],
|
||||
comm.get("dominant_language", ""),
|
||||
comm.get("description", ""),
|
||||
),
|
||||
)
|
||||
community_id = cursor.lastrowid
|
||||
|
||||
# Batch update community_id on member nodes
|
||||
member_qns = comm.get("members", [])
|
||||
for j in range(0, len(member_qns), _SQL_BATCH):
|
||||
batch = member_qns[j:j + _SQL_BATCH]
|
||||
placeholders = ",".join("?" * len(batch))
|
||||
conn.execute(
|
||||
f"UPDATE nodes SET community_id = ? WHERE qualified_name IN ({placeholders})", # nosec B608
|
||||
[community_id] + batch,
|
||||
)
|
||||
count += 1
|
||||
|
||||
conn.commit()
|
||||
except BaseException:
|
||||
conn.rollback()
|
||||
raise
|
||||
return count
|
||||
|
||||
|
||||
def get_communities(
|
||||
store: GraphStore, sort_by: str = "size", min_size: int = 0
|
||||
) -> list[dict[str, Any]]:
|
||||
"""Retrieve stored communities from the database.
|
||||
|
||||
Args:
|
||||
store: The GraphStore instance.
|
||||
sort_by: Column to sort by ("size", "cohesion", "name").
|
||||
min_size: Minimum community size to include.
|
||||
|
||||
Returns:
|
||||
List of community dicts.
|
||||
"""
|
||||
valid_sorts = {"size", "cohesion", "name"}
|
||||
if sort_by not in valid_sorts:
|
||||
sort_by = "size"
|
||||
|
||||
order = "DESC" if sort_by in ("size", "cohesion") else "ASC"
|
||||
|
||||
# NOTE: get_communities reads the communities table which has no
|
||||
# dedicated GraphStore method (it's a domain-specific table managed
|
||||
# entirely by the communities module). We use _conn for this query.
|
||||
rows = store._conn.execute(
|
||||
f"SELECT * FROM communities WHERE size >= ? ORDER BY {sort_by} {order}", # nosec B608
|
||||
(min_size,),
|
||||
).fetchall()
|
||||
|
||||
communities: list[dict[str, Any]] = []
|
||||
for row in rows:
|
||||
# Fetch member qualified names for this community
|
||||
member_qns = [
|
||||
_sanitize_name(qn)
|
||||
for qn in store.get_community_member_qns(row["id"])
|
||||
]
|
||||
|
||||
communities.append({
|
||||
"id": row["id"],
|
||||
"name": _sanitize_name(row["name"]),
|
||||
"level": row["level"],
|
||||
"cohesion": row["cohesion"],
|
||||
"size": row["size"],
|
||||
"dominant_language": row["dominant_language"] or "",
|
||||
"description": _sanitize_name(row["description"] or ""),
|
||||
"members": member_qns,
|
||||
})
|
||||
|
||||
return communities
|
||||
|
||||
|
||||
_TEST_COMMUNITY_RE = re.compile(
|
||||
r"(^test[-/]|[-/]test([:/]|$)|it:should|describe:|spec[-/]|[-/]spec$)",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
|
||||
|
||||
def _is_test_community(name: str) -> bool:
|
||||
"""Return True if a community name indicates it is test-dominated."""
|
||||
return bool(_TEST_COMMUNITY_RE.search(name))
|
||||
|
||||
|
||||
def get_architecture_overview(store: GraphStore) -> dict[str, Any]:
|
||||
"""Generate an architecture overview based on community structure.
|
||||
|
||||
Builds a node-to-community mapping, counts cross-community edges,
|
||||
and generates warnings for high coupling.
|
||||
|
||||
Args:
|
||||
store: The GraphStore instance.
|
||||
|
||||
Returns:
|
||||
Dict with keys: communities, cross_community_edges, warnings.
|
||||
"""
|
||||
communities = get_communities(store)
|
||||
|
||||
# Build node -> community_id mapping
|
||||
node_to_community: dict[str, int] = {}
|
||||
for comm in communities:
|
||||
comm_id = comm.get("id", 0)
|
||||
for qn in comm.get("members", []):
|
||||
node_to_community[qn] = comm_id
|
||||
|
||||
# Count cross-community edges
|
||||
all_edges = store.get_all_edges()
|
||||
cross_edges: list[dict[str, Any]] = []
|
||||
cross_counts: Counter[tuple[int, int]] = Counter()
|
||||
|
||||
for e in all_edges:
|
||||
# TESTED_BY edges are expected cross-community coupling (test → code),
|
||||
# not an architectural smell.
|
||||
if e.kind == "TESTED_BY":
|
||||
continue
|
||||
src_comm = node_to_community.get(e.source_qualified)
|
||||
tgt_comm = node_to_community.get(e.target_qualified)
|
||||
if (
|
||||
src_comm is not None
|
||||
and tgt_comm is not None
|
||||
and src_comm != tgt_comm
|
||||
):
|
||||
pair = (min(src_comm, tgt_comm), max(src_comm, tgt_comm))
|
||||
cross_counts[pair] += 1
|
||||
cross_edges.append({
|
||||
"source_community": src_comm,
|
||||
"target_community": tgt_comm,
|
||||
"edge_kind": e.kind,
|
||||
"source": _sanitize_name(e.source_qualified),
|
||||
"target": _sanitize_name(e.target_qualified),
|
||||
})
|
||||
|
||||
# Generate warnings for high coupling, skipping test-dominated pairs.
|
||||
warnings: list[str] = []
|
||||
comm_name_map = {c.get("id", 0): c["name"] for c in communities}
|
||||
for (c1, c2), count in cross_counts.most_common():
|
||||
if count > 10:
|
||||
name1 = comm_name_map.get(c1, f"community-{c1}")
|
||||
name2 = comm_name_map.get(c2, f"community-{c2}")
|
||||
# Skip pairs where either community is test-dominated — coupling
|
||||
# between test and production code is expected, not architectural.
|
||||
if _is_test_community(name1) or _is_test_community(name2):
|
||||
continue
|
||||
warnings.append(
|
||||
f"High coupling ({count} edges) between "
|
||||
f"'{name1}' and '{name2}'"
|
||||
)
|
||||
|
||||
return {
|
||||
"communities": communities,
|
||||
"cross_community_edges": cross_edges,
|
||||
"warnings": warnings,
|
||||
}
|
||||
@@ -0,0 +1,23 @@
|
||||
"""Shared constants for code-review-graph."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
|
||||
SECURITY_KEYWORDS: frozenset[str] = frozenset({
|
||||
"auth", "login", "password", "token", "session", "crypt", "secret",
|
||||
"credential", "permission", "sql", "query", "execute", "connect",
|
||||
"socket", "request", "http", "sanitize", "validate", "encrypt",
|
||||
"decrypt", "hash", "sign", "verify", "admin", "privilege",
|
||||
})
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Configurable limits (override via environment variables)
|
||||
# ---------------------------------------------------------------------------
|
||||
MAX_IMPACT_NODES = int(os.environ.get("CRG_MAX_IMPACT_NODES", "500"))
|
||||
MAX_IMPACT_DEPTH = int(os.environ.get("CRG_MAX_IMPACT_DEPTH", "2"))
|
||||
MAX_BFS_DEPTH = int(os.environ.get("CRG_MAX_BFS_DEPTH", "15"))
|
||||
MAX_SEARCH_RESULTS = int(os.environ.get("CRG_MAX_SEARCH_RESULTS", "20"))
|
||||
|
||||
# BFS engine: "sql" (SQLite recursive CTE) or "networkx" (Python-side BFS)
|
||||
BFS_ENGINE = os.environ.get("CRG_BFS_ENGINE", "sql")
|
||||
@@ -0,0 +1,317 @@
|
||||
"""Compact estimated context savings helpers.
|
||||
|
||||
The project intentionally labels these values as estimates: the helper uses a
|
||||
conservative character-count approximation instead of model-specific tokenizers.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from pathlib import Path
|
||||
from typing import Any, Iterable
|
||||
|
||||
CHARS_PER_TOKEN = 4
|
||||
|
||||
|
||||
def estimate_tokens(value: Any) -> int:
|
||||
"""Estimate token count with a conservative 4 chars/token approximation."""
|
||||
if value is None:
|
||||
return 0
|
||||
if isinstance(value, str):
|
||||
text = value
|
||||
else:
|
||||
text = json.dumps(
|
||||
value,
|
||||
default=str,
|
||||
ensure_ascii=True,
|
||||
separators=(",", ":"),
|
||||
sort_keys=True,
|
||||
)
|
||||
if not text:
|
||||
return 0
|
||||
return max(1, (len(text) + CHARS_PER_TOKEN - 1) // CHARS_PER_TOKEN)
|
||||
|
||||
|
||||
def estimate_file_tokens(repo_root: Path, files: Iterable[str]) -> int:
|
||||
"""Estimate tokens for changed files using file sizes, not file contents."""
|
||||
total = 0
|
||||
root = repo_root.resolve()
|
||||
for file_name in files:
|
||||
path = Path(file_name)
|
||||
full_path = path if path.is_absolute() else root / path
|
||||
try:
|
||||
if full_path.is_file():
|
||||
total += max(
|
||||
1,
|
||||
(full_path.stat().st_size + CHARS_PER_TOKEN - 1)
|
||||
// CHARS_PER_TOKEN,
|
||||
)
|
||||
except OSError:
|
||||
continue
|
||||
return total
|
||||
|
||||
|
||||
def estimate_context_savings(
|
||||
*,
|
||||
original_context: Any | None = None,
|
||||
returned_context: Any | None = None,
|
||||
original_tokens: int | None = None,
|
||||
returned_tokens: int | None = None,
|
||||
) -> dict[str, int | bool] | None:
|
||||
"""Return tiny savings metadata, or None when no baseline is available."""
|
||||
baseline = (
|
||||
original_tokens
|
||||
if original_tokens is not None
|
||||
else estimate_tokens(original_context)
|
||||
)
|
||||
returned = (
|
||||
returned_tokens
|
||||
if returned_tokens is not None
|
||||
else estimate_tokens(returned_context)
|
||||
)
|
||||
|
||||
if baseline <= 0:
|
||||
return None
|
||||
|
||||
saved = max(0, baseline - returned)
|
||||
percent = round((saved / baseline) * 100) if baseline else 0
|
||||
return {
|
||||
"estimated": True,
|
||||
"saved_tokens": int(saved),
|
||||
"saved_percent": int(percent),
|
||||
}
|
||||
|
||||
|
||||
def attach_context_savings(
|
||||
result: dict[str, Any],
|
||||
*,
|
||||
original_context: Any | None = None,
|
||||
original_tokens: int | None = None,
|
||||
returned_context: Any | None = None,
|
||||
returned_tokens: int | None = None,
|
||||
) -> dict[str, Any]:
|
||||
"""Attach compact ``context_savings`` metadata when it can be estimated."""
|
||||
estimate = estimate_context_savings(
|
||||
original_context=original_context,
|
||||
returned_context=result if returned_context is None else returned_context,
|
||||
original_tokens=original_tokens,
|
||||
returned_tokens=returned_tokens,
|
||||
)
|
||||
if estimate is not None:
|
||||
result["context_savings"] = estimate
|
||||
return result
|
||||
|
||||
|
||||
def format_context_savings(estimate: dict[str, Any] | None) -> str | None:
|
||||
"""Format a one-line human summary for CLI output."""
|
||||
if not estimate:
|
||||
return None
|
||||
saved = int(estimate.get("saved_tokens", 0))
|
||||
percent = int(estimate.get("saved_percent", 0))
|
||||
return f"Estimated context saved: ~{saved:,} tokens (~{percent}%)"
|
||||
|
||||
|
||||
def _fmt_compact(n: int) -> str:
|
||||
"""Compact integer formatting: 1234 -> '1.2k', 9876 -> '9.9k', 500 -> '500'."""
|
||||
if n >= 10_000:
|
||||
return f"{n // 1000:,}k"
|
||||
if n >= 1000:
|
||||
return f"{n / 1000:.1f}k"
|
||||
return str(n)
|
||||
|
||||
|
||||
def _breakdown_from_response(response: dict[str, Any]) -> dict[str, int]:
|
||||
"""Pull a per-category token estimate from a detect-changes / review response.
|
||||
|
||||
Only fields that exist and have content are reported, so the breakdown
|
||||
line stays meaningful instead of padding with zeros.
|
||||
"""
|
||||
# Friendly label -> response-dict key
|
||||
fields = [
|
||||
("Functions", "changed_functions"),
|
||||
("Flows", "affected_flows"),
|
||||
("Tests", "test_gaps"),
|
||||
("Risk", "review_priorities"),
|
||||
("Impact", "impacted_nodes"),
|
||||
("Edges", "edges"),
|
||||
("Source", "source_snippets"),
|
||||
("Imports", "imports"),
|
||||
]
|
||||
out: dict[str, int] = {}
|
||||
for label, key in fields:
|
||||
value = response.get(key)
|
||||
if not value:
|
||||
continue
|
||||
tokens = estimate_tokens(value)
|
||||
if tokens > 0:
|
||||
out[label] = tokens
|
||||
return out
|
||||
|
||||
|
||||
def verify_with_tiktoken(
|
||||
repo_root: "Path | str",
|
||||
changed_files: Iterable[str],
|
||||
response: Any,
|
||||
encoding_name: str = "cl100k_base",
|
||||
) -> dict[str, int] | None:
|
||||
"""Calibrate the chars/4 estimate against a real model tokenizer.
|
||||
|
||||
Returns ``{"verified_baseline": int, "verified_returned": int,
|
||||
"verified_saved": int, "verified_percent": int}`` or ``None`` if
|
||||
tiktoken is not installed. Reads every changed file's content (unlike
|
||||
the stat-only ``estimate_file_tokens``) so the numbers reflect what
|
||||
an agent would actually consume.
|
||||
"""
|
||||
try:
|
||||
import tiktoken # type: ignore[import-untyped]
|
||||
except ImportError:
|
||||
return None
|
||||
|
||||
enc = tiktoken.get_encoding(encoding_name)
|
||||
root = Path(repo_root).resolve()
|
||||
|
||||
naive_real = 0
|
||||
for f in changed_files:
|
||||
p = root / f
|
||||
try:
|
||||
if p.is_file():
|
||||
naive_real += len(enc.encode(p.read_text(errors="replace")))
|
||||
except OSError:
|
||||
continue
|
||||
|
||||
if isinstance(response, str):
|
||||
graph_real = len(enc.encode(response))
|
||||
else:
|
||||
text = json.dumps(
|
||||
response, default=str, ensure_ascii=True,
|
||||
separators=(",", ":"), sort_keys=True,
|
||||
)
|
||||
graph_real = len(enc.encode(text))
|
||||
|
||||
saved = max(0, naive_real - graph_real)
|
||||
pct = round(saved * 100 / naive_real) if naive_real > 0 else 0
|
||||
return {
|
||||
"verified_baseline": naive_real,
|
||||
"verified_returned": graph_real,
|
||||
"verified_saved": saved,
|
||||
"verified_percent": pct,
|
||||
}
|
||||
|
||||
|
||||
def format_context_savings_panel(
|
||||
estimate: dict[str, Any] | None,
|
||||
*,
|
||||
original_tokens: int | None = None,
|
||||
returned_tokens: int | None = None,
|
||||
response: dict[str, Any] | None = None,
|
||||
breakdown: dict[str, int] | None = None,
|
||||
verified: dict[str, int] | None = None,
|
||||
title: str = "Token Savings",
|
||||
width: int = 64,
|
||||
) -> str | None:
|
||||
"""Format the savings estimate as a boxed multi-line CLI panel.
|
||||
|
||||
Example output (width=60)::
|
||||
|
||||
┌──────────────── Token Savings ────────────────┐
|
||||
│ Full context would be: 12,932 tokens │
|
||||
│ Graph context used: 773 tokens │
|
||||
│ Saved: 12,159 tokens (~94%) │
|
||||
│ Breakdown: Functions 580 · Tests 120 · ... │
|
||||
└───────────────────────────────────────────────┘
|
||||
|
||||
All numbers are labelled as estimates upstream (``estimated: true`` in the
|
||||
metadata dict) because the project uses a 4-chars-per-token approximation,
|
||||
not model-specific tokenization.
|
||||
|
||||
Args:
|
||||
estimate: The ``context_savings`` dict from a tool response.
|
||||
original_tokens: Optional override for the naive baseline.
|
||||
returned_tokens: Optional override for the graph response size.
|
||||
response: When provided, breakdown is auto-derived from common keys
|
||||
(``changed_functions``, ``affected_flows``, ``test_gaps``,
|
||||
``review_priorities``, ``impacted_nodes``, ``edges``,
|
||||
``source_snippets``, ``imports``).
|
||||
breakdown: Explicit ``{label: tokens}`` map; takes precedence over
|
||||
``response``-derived breakdown when both are provided.
|
||||
title: Title centered in the top border.
|
||||
width: Total panel width, capped at terminal width if larger.
|
||||
|
||||
Returns:
|
||||
The panel as a single ``\\n``-joined string, or ``None`` when there
|
||||
is nothing meaningful to display.
|
||||
"""
|
||||
if not estimate:
|
||||
return None
|
||||
|
||||
saved = int(estimate.get("saved_tokens", 0))
|
||||
percent = int(estimate.get("saved_percent", 0))
|
||||
|
||||
# Derive baseline + returned from saved+percent if not provided
|
||||
if original_tokens is None:
|
||||
if percent > 0:
|
||||
original_tokens = int(round(saved * 100 / percent))
|
||||
else:
|
||||
original_tokens = saved
|
||||
if returned_tokens is None:
|
||||
returned_tokens = max(0, (original_tokens or 0) - saved)
|
||||
|
||||
if breakdown is None and response is not None:
|
||||
breakdown = _breakdown_from_response(response)
|
||||
|
||||
# Top up the breakdown with an "Other" bucket so the parts sum to
|
||||
# ``returned_tokens`` exactly. "Other" covers fields the breakdown
|
||||
# doesn't enumerate (status, summary, risk_score, context_savings
|
||||
# metadata, JSON envelope chars). Skip when there's no positive
|
||||
# remainder — the breakdown already accounts for the whole response.
|
||||
if breakdown and returned_tokens is not None:
|
||||
labelled_sum = sum(breakdown.values())
|
||||
remainder = returned_tokens - labelled_sum
|
||||
if remainder > 0:
|
||||
breakdown = dict(breakdown) # copy before mutating
|
||||
breakdown["Other"] = remainder
|
||||
|
||||
# Lines that go inside the box (without borders)
|
||||
inner_lines: list[str] = [
|
||||
f"Full context would be: {original_tokens:>9,} tokens",
|
||||
f"Graph context used: {returned_tokens:>9,} tokens",
|
||||
f"Saved: {saved:>9,} tokens (~{percent}%)",
|
||||
]
|
||||
if verified:
|
||||
vb = verified["verified_baseline"]
|
||||
vr = verified["verified_returned"]
|
||||
vs = verified["verified_saved"]
|
||||
vp = verified["verified_percent"]
|
||||
inner_lines.append(
|
||||
f"Verified (tiktoken): {vs:>9,} tokens (~{vp}%) "
|
||||
f"[{vb:,} → {vr:,}]"
|
||||
)
|
||||
if breakdown:
|
||||
parts = [f"{label} {_fmt_compact(tok)}" for label, tok in breakdown.items()]
|
||||
bd_line = "Breakdown: " + " · ".join(parts)
|
||||
inner_lines.append(bd_line)
|
||||
|
||||
# Compute final width: at least wide enough for the longest inner line + padding
|
||||
content_width = max(len(s) for s in inner_lines)
|
||||
inner_w = max(width - 2, content_width + 2) # +2 for one space pad each side
|
||||
# Title bar
|
||||
title_str = f" {title} "
|
||||
dash_total = inner_w - len(title_str)
|
||||
if dash_total < 4:
|
||||
dash_total = 4
|
||||
left_dash = dash_total // 2
|
||||
right_dash = dash_total - left_dash
|
||||
top = "┌" + "─" * left_dash + title_str + "─" * right_dash + "┐"
|
||||
bottom = "└" + "─" * inner_w + "┘"
|
||||
|
||||
def _box_line(content: str) -> str:
|
||||
pad = inner_w - 2 - len(content)
|
||||
if pad < 0:
|
||||
pad = 0
|
||||
return f"│ {content}{' ' * pad} │"
|
||||
|
||||
lines = [top]
|
||||
for s in inner_lines:
|
||||
lines.append(_box_line(s))
|
||||
lines.append(bottom)
|
||||
return "\n".join(lines)
|
||||
@@ -0,0 +1,322 @@
|
||||
"""Config-driven custom language support ("bring your own language").
|
||||
|
||||
Repos can teach the parser new tree-sitter languages without forking by
|
||||
dropping a ``languages.toml`` file into ``.code-review-graph/``::
|
||||
|
||||
[languages.erlang]
|
||||
extensions = [".erl", ".hrl"]
|
||||
grammar = "erlang" # tree_sitter_language_pack name
|
||||
function_node_types = ["function_clause"]
|
||||
class_node_types = ["record_decl"]
|
||||
import_node_types = ["import_attribute"]
|
||||
call_node_types = ["call"]
|
||||
comment = "Erlang via the bundled tree-sitter-erlang grammar"
|
||||
|
||||
The loader is deliberately defensive: a broken config must never crash a
|
||||
build. Invalid entries are skipped with a ``logger.warning``, and built-in
|
||||
languages always win — custom entries can neither override built-in file
|
||||
extensions nor reuse built-in language names. At most
|
||||
``MAX_CUSTOM_LANGUAGES`` entries are honoured per repo.
|
||||
|
||||
See docs/CUSTOM_LANGUAGES.md for the full schema reference (answers #320).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import re
|
||||
import sys
|
||||
import threading
|
||||
from collections.abc import Mapping
|
||||
from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
from typing import Optional
|
||||
|
||||
import tree_sitter_language_pack as tslp
|
||||
|
||||
if sys.version_info >= (3, 11):
|
||||
import tomllib
|
||||
else:
|
||||
try:
|
||||
import tomli as tomllib # type: ignore[no-redef]
|
||||
except ImportError:
|
||||
tomllib = None # type: ignore[assignment]
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
#: Location of the config file, relative to the repo root.
|
||||
CONFIG_RELATIVE_PATH = Path(".code-review-graph") / "languages.toml"
|
||||
|
||||
#: Hard cap on the number of custom languages loaded from a single config.
|
||||
MAX_CUSTOM_LANGUAGES = 20
|
||||
|
||||
#: Custom language names: short lowercase identifiers. The name becomes the
|
||||
#: ``language`` field on every node parsed from matching files.
|
||||
_NAME_RE = re.compile(r"^[a-z][a-z0-9_-]{0,31}$")
|
||||
|
||||
#: Extensions: a leading dot followed by 1-15 safe characters (".erl",
|
||||
#: ".cls", ".4gl"). Uppercase input is normalised to lowercase because the
|
||||
#: parser lowercases file suffixes before lookup.
|
||||
_EXTENSION_RE = re.compile(r"^\.[a-z0-9_+-]{1,15}$")
|
||||
|
||||
#: The four node-type lists recognised in each ``[languages.<name>]`` table.
|
||||
_NODE_TYPE_KEYS = (
|
||||
"function_node_types",
|
||||
"class_node_types",
|
||||
"import_node_types",
|
||||
"call_node_types",
|
||||
)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class CustomLanguage:
|
||||
"""One validated ``[languages.<name>]`` entry from languages.toml."""
|
||||
|
||||
name: str
|
||||
grammar: str
|
||||
extensions: tuple[str, ...]
|
||||
function_node_types: tuple[str, ...] = ()
|
||||
class_node_types: tuple[str, ...] = ()
|
||||
import_node_types: tuple[str, ...] = ()
|
||||
call_node_types: tuple[str, ...] = ()
|
||||
comment: str = ""
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class _CacheEntry:
|
||||
mtime_ns: int
|
||||
size: int
|
||||
languages: dict[str, CustomLanguage] = field(default_factory=dict)
|
||||
|
||||
|
||||
# Config files are re-read only when their mtime/size changes. This matters
|
||||
# because full builds construct one CodeParser per worker task, and probing
|
||||
# tree-sitter grammars on every file parse would be wasteful.
|
||||
_cache_lock = threading.Lock()
|
||||
_cache: dict[str, _CacheEntry] = {}
|
||||
|
||||
|
||||
def clear_cache() -> None:
|
||||
"""Drop the loader cache (used by tests)."""
|
||||
with _cache_lock:
|
||||
_cache.clear()
|
||||
|
||||
|
||||
def load_custom_languages(
|
||||
repo_root: Path,
|
||||
*,
|
||||
builtin_extensions: Mapping[str, str],
|
||||
builtin_languages: frozenset[str],
|
||||
) -> dict[str, CustomLanguage]:
|
||||
"""Load and validate ``<repo_root>/.code-review-graph/languages.toml``.
|
||||
|
||||
Returns a mapping of custom language name -> :class:`CustomLanguage`.
|
||||
Always returns (possibly empty) — a broken config never raises.
|
||||
|
||||
Args:
|
||||
repo_root: Repository root containing ``.code-review-graph/``.
|
||||
builtin_extensions: The parser's built-in extension map; custom
|
||||
entries colliding with these are skipped (built-ins win).
|
||||
builtin_languages: All built-in language identifiers; custom names
|
||||
shadowing these are skipped.
|
||||
"""
|
||||
config_path = Path(repo_root) / CONFIG_RELATIVE_PATH
|
||||
try:
|
||||
stat = config_path.stat()
|
||||
except OSError:
|
||||
return {} # No config file — the common case; not worth a log line.
|
||||
|
||||
cache_key = str(config_path)
|
||||
with _cache_lock:
|
||||
cached = _cache.get(cache_key)
|
||||
if (
|
||||
cached is not None
|
||||
and cached.mtime_ns == stat.st_mtime_ns
|
||||
and cached.size == stat.st_size
|
||||
):
|
||||
return dict(cached.languages)
|
||||
|
||||
languages = _load_uncached(config_path, builtin_extensions, builtin_languages)
|
||||
with _cache_lock:
|
||||
_cache[cache_key] = _CacheEntry(stat.st_mtime_ns, stat.st_size, dict(languages))
|
||||
return languages
|
||||
|
||||
|
||||
def _load_uncached(
|
||||
config_path: Path,
|
||||
builtin_extensions: Mapping[str, str],
|
||||
builtin_languages: frozenset[str],
|
||||
) -> dict[str, CustomLanguage]:
|
||||
if tomllib is None:
|
||||
logger.warning(
|
||||
"%s found but TOML parsing requires the 'tomli' package on "
|
||||
"Python < 3.11 — no custom languages loaded",
|
||||
config_path,
|
||||
)
|
||||
return {}
|
||||
try:
|
||||
raw = config_path.read_bytes()
|
||||
except (OSError, PermissionError) as exc:
|
||||
logger.warning("Cannot read %s: %s — no custom languages loaded", config_path, exc)
|
||||
return {}
|
||||
try:
|
||||
data = tomllib.loads(raw.decode("utf-8", errors="replace"))
|
||||
except tomllib.TOMLDecodeError as exc:
|
||||
logger.warning("Malformed TOML in %s: %s — no custom languages loaded", config_path, exc)
|
||||
return {}
|
||||
|
||||
tables = data.get("languages")
|
||||
if tables is None:
|
||||
return {}
|
||||
if not isinstance(tables, dict):
|
||||
logger.warning(
|
||||
"%s: [languages] must be a table of tables — no custom languages loaded",
|
||||
config_path,
|
||||
)
|
||||
return {}
|
||||
|
||||
result: dict[str, CustomLanguage] = {}
|
||||
claimed_extensions: set[str] = set()
|
||||
for name, table in tables.items():
|
||||
if len(result) >= MAX_CUSTOM_LANGUAGES:
|
||||
logger.warning(
|
||||
"%s defines more than %d custom languages — ignoring the rest",
|
||||
config_path, MAX_CUSTOM_LANGUAGES,
|
||||
)
|
||||
break
|
||||
lang = _validate_entry(
|
||||
name, table, builtin_extensions, builtin_languages,
|
||||
claimed_extensions, config_path,
|
||||
)
|
||||
if lang is None:
|
||||
continue
|
||||
result[lang.name] = lang
|
||||
claimed_extensions.update(lang.extensions)
|
||||
return result
|
||||
|
||||
|
||||
def _validate_entry(
|
||||
name: object,
|
||||
table: object,
|
||||
builtin_extensions: Mapping[str, str],
|
||||
builtin_languages: frozenset[str],
|
||||
claimed_extensions: set[str],
|
||||
config_path: Path,
|
||||
) -> Optional[CustomLanguage]:
|
||||
"""Validate one ``[languages.<name>]`` table; None (after a warning) on
|
||||
any problem so a bad entry can never break a build."""
|
||||
label = name if isinstance(name, str) else repr(name)
|
||||
if not isinstance(table, dict):
|
||||
logger.warning("%s: [languages.%s] is not a table — skipping", config_path, label)
|
||||
return None
|
||||
if not isinstance(name, str) or not _NAME_RE.match(name):
|
||||
logger.warning(
|
||||
"%s: invalid custom language name %r (expected lowercase "
|
||||
"letters/digits/_/-, max 32 chars) — skipping",
|
||||
config_path, label,
|
||||
)
|
||||
return None
|
||||
if name in builtin_languages:
|
||||
logger.warning(
|
||||
"%s: custom language %r shadows a built-in language — skipping "
|
||||
"(built-ins cannot be overridden)",
|
||||
config_path, name,
|
||||
)
|
||||
return None
|
||||
|
||||
grammar = table.get("grammar")
|
||||
if not isinstance(grammar, str) or not grammar.strip():
|
||||
logger.warning(
|
||||
"%s: custom language %r needs a non-empty 'grammar' string — skipping",
|
||||
config_path, name,
|
||||
)
|
||||
return None
|
||||
grammar = grammar.strip()
|
||||
|
||||
raw_extensions = table.get("extensions")
|
||||
if not isinstance(raw_extensions, list) or not raw_extensions:
|
||||
logger.warning(
|
||||
"%s: custom language %r needs a non-empty 'extensions' list — skipping",
|
||||
config_path, name,
|
||||
)
|
||||
return None
|
||||
extensions: list[str] = []
|
||||
for ext in raw_extensions:
|
||||
normalized = ext.strip().lower() if isinstance(ext, str) else ""
|
||||
if not normalized.startswith("."):
|
||||
logger.warning(
|
||||
"%s: custom language %r: extension %r must start with a dot — skipping",
|
||||
config_path, name, ext,
|
||||
)
|
||||
return None
|
||||
if not _EXTENSION_RE.match(normalized):
|
||||
logger.warning(
|
||||
"%s: custom language %r: extension %r is not a valid file "
|
||||
"extension — skipping",
|
||||
config_path, name, ext,
|
||||
)
|
||||
return None
|
||||
if normalized in builtin_extensions:
|
||||
logger.warning(
|
||||
"%s: custom language %r: extension %r is already handled by "
|
||||
"the built-in %r parser — skipping (built-ins cannot be overridden)",
|
||||
config_path, name, normalized, builtin_extensions[normalized],
|
||||
)
|
||||
return None
|
||||
if normalized in claimed_extensions:
|
||||
logger.warning(
|
||||
"%s: custom language %r: extension %r is already claimed by "
|
||||
"an earlier custom language — skipping",
|
||||
config_path, name, normalized,
|
||||
)
|
||||
return None
|
||||
if normalized not in extensions:
|
||||
extensions.append(normalized)
|
||||
|
||||
node_types: dict[str, tuple[str, ...]] = {}
|
||||
for key in _NODE_TYPE_KEYS:
|
||||
value = table.get(key, [])
|
||||
if not isinstance(value, list) or any(
|
||||
not isinstance(item, str) or not item.strip() for item in value
|
||||
):
|
||||
logger.warning(
|
||||
"%s: custom language %r: %s must be a list of non-empty "
|
||||
"strings — skipping",
|
||||
config_path, name, key,
|
||||
)
|
||||
return None
|
||||
node_types[key] = tuple(item.strip() for item in value)
|
||||
if not any(node_types.values()):
|
||||
logger.warning(
|
||||
"%s: custom language %r defines no node types — nothing to "
|
||||
"extract, skipping",
|
||||
config_path, name,
|
||||
)
|
||||
return None
|
||||
|
||||
comment = table.get("comment", "")
|
||||
if not isinstance(comment, str):
|
||||
comment = ""
|
||||
|
||||
# Probe the grammar last (it is the expensive check). Parser objects
|
||||
# themselves are created lazily by CodeParser._get_parser.
|
||||
try:
|
||||
tslp.get_language(grammar) # type: ignore[arg-type]
|
||||
except (LookupError, ValueError, ImportError, OSError) as exc:
|
||||
logger.warning(
|
||||
"%s: custom language %r: grammar %r is not available in "
|
||||
"tree_sitter_language_pack (%s) — skipping",
|
||||
config_path, name, grammar, exc,
|
||||
)
|
||||
return None
|
||||
|
||||
return CustomLanguage(
|
||||
name=name,
|
||||
grammar=grammar,
|
||||
extensions=tuple(extensions),
|
||||
function_node_types=node_types["function_node_types"],
|
||||
class_node_types=node_types["class_node_types"],
|
||||
import_node_types=node_types["import_node_types"],
|
||||
call_node_types=node_types["call_node_types"],
|
||||
comment=comment,
|
||||
)
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,320 @@
|
||||
"""CLI entry point for the crg-daemon multi-repo watcher.
|
||||
|
||||
Usage:
|
||||
crg-daemon start [--foreground]
|
||||
crg-daemon stop
|
||||
crg-daemon restart [--foreground]
|
||||
crg-daemon status
|
||||
crg-daemon logs [--repo ALIAS] [--follow] [--lines N]
|
||||
crg-daemon add <path> [--alias ALIAS]
|
||||
crg-daemon remove <path_or_alias>
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import logging
|
||||
import os
|
||||
import signal
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Subcommand handlers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _handle_start(args: argparse.Namespace) -> None:
|
||||
"""Start the daemon process."""
|
||||
from .daemon import WatchDaemon, is_daemon_running, load_config
|
||||
|
||||
if is_daemon_running():
|
||||
print("Error: Daemon is already running.")
|
||||
sys.exit(1)
|
||||
|
||||
config = load_config()
|
||||
daemon = WatchDaemon(config=config)
|
||||
daemon.start()
|
||||
|
||||
if not args.foreground:
|
||||
daemon.daemonize()
|
||||
|
||||
daemon.run_forever()
|
||||
|
||||
|
||||
def _handle_stop(_args: argparse.Namespace) -> None:
|
||||
"""Stop the running daemon process."""
|
||||
from .daemon import clear_pid, is_daemon_running, read_pid
|
||||
|
||||
if not is_daemon_running():
|
||||
print("Daemon is not running.")
|
||||
sys.exit(1)
|
||||
|
||||
pid = read_pid()
|
||||
if pid is None:
|
||||
print("Error: Could not read daemon PID.")
|
||||
sys.exit(1)
|
||||
|
||||
print(f"Stopping daemon (PID {pid})...")
|
||||
try:
|
||||
os.kill(pid, signal.SIGTERM)
|
||||
except ProcessLookupError:
|
||||
clear_pid()
|
||||
print("Daemon stopped (process already gone).")
|
||||
return
|
||||
except PermissionError:
|
||||
print(f"Error: Permission denied sending signal to PID {pid}.")
|
||||
sys.exit(1)
|
||||
|
||||
# Wait up to 5 seconds for process to die
|
||||
for _ in range(50):
|
||||
try:
|
||||
os.kill(pid, 0)
|
||||
except ProcessLookupError:
|
||||
break
|
||||
time.sleep(0.1)
|
||||
else:
|
||||
# Still alive after 5s — send SIGKILL
|
||||
print("Daemon did not stop gracefully, sending SIGKILL...")
|
||||
try:
|
||||
os.kill(pid, signal.SIGKILL)
|
||||
except ProcessLookupError:
|
||||
pass
|
||||
|
||||
clear_pid()
|
||||
print("Daemon stopped.")
|
||||
|
||||
|
||||
def _handle_restart(args: argparse.Namespace) -> None:
|
||||
"""Restart the daemon (stop + start)."""
|
||||
from .daemon import is_daemon_running
|
||||
|
||||
if is_daemon_running():
|
||||
_handle_stop(args)
|
||||
else:
|
||||
print("Daemon is not running, starting fresh.")
|
||||
|
||||
_handle_start(args)
|
||||
|
||||
|
||||
def _handle_status(_args: argparse.Namespace) -> None:
|
||||
"""Show daemon status and configuration."""
|
||||
from .daemon import is_daemon_running, load_config, load_state, pid_alive, read_pid
|
||||
|
||||
config = load_config()
|
||||
running = is_daemon_running()
|
||||
|
||||
if running:
|
||||
pid = read_pid()
|
||||
print(f"Daemon: running (PID {pid})")
|
||||
else:
|
||||
print("Daemon: not running")
|
||||
|
||||
print(f"Name: {config.session_name}")
|
||||
print(f"Log dir: {config.log_dir}")
|
||||
print(f"Poll: {config.poll_interval}s")
|
||||
print()
|
||||
|
||||
if not config.repos:
|
||||
print("No repositories configured.")
|
||||
print("Use: crg-daemon add <path> [--alias NAME]")
|
||||
return
|
||||
|
||||
# Header
|
||||
alias_width = max(len(r.alias) for r in config.repos)
|
||||
alias_width = max(alias_width, 5) # minimum "Alias" header width
|
||||
|
||||
if running:
|
||||
state = load_state()
|
||||
print(f" {'Alias':<{alias_width}} {'Status':<8} {'PID':<8} Path")
|
||||
print(f" {'-' * alias_width} {'-' * 8} {'-' * 8} {'-' * 40}")
|
||||
for repo in config.repos:
|
||||
entry = state.get(repo.alias, {})
|
||||
child_pid: int | None = entry.get("pid")
|
||||
alive = child_pid is not None and pid_alive(child_pid)
|
||||
status_str = "alive" if alive else "dead"
|
||||
pid_str = str(child_pid) if child_pid is not None else "-"
|
||||
print(f" {repo.alias:<{alias_width}} {status_str:<8} {pid_str:<8} {repo.path}")
|
||||
else:
|
||||
print(f" {'Alias':<{alias_width}} Path")
|
||||
print(f" {'-' * alias_width} {'-' * 40}")
|
||||
for repo in config.repos:
|
||||
print(f" {repo.alias:<{alias_width}} {repo.path}")
|
||||
|
||||
|
||||
def _handle_logs(args: argparse.Namespace) -> None:
|
||||
"""Show daemon or per-repo log files."""
|
||||
from .daemon import load_config
|
||||
|
||||
config = load_config()
|
||||
|
||||
if args.repo:
|
||||
log_file = config.log_dir / f"{args.repo}.log"
|
||||
else:
|
||||
log_file = config.log_dir / "daemon.log"
|
||||
|
||||
if not log_file.exists():
|
||||
print(f"Log file not found: {log_file}")
|
||||
sys.exit(1)
|
||||
|
||||
if args.follow:
|
||||
try:
|
||||
subprocess.run(["tail", "-f", str(log_file)], check=False)
|
||||
except KeyboardInterrupt:
|
||||
pass
|
||||
return
|
||||
|
||||
# Read last N lines
|
||||
lines_count = args.lines
|
||||
try:
|
||||
text = log_file.read_text(encoding="utf-8", errors="replace")
|
||||
except OSError as exc:
|
||||
print(f"Error reading log file: {exc}")
|
||||
sys.exit(1)
|
||||
|
||||
lines = text.splitlines()
|
||||
tail = lines[-lines_count:] if len(lines) > lines_count else lines
|
||||
for line in tail:
|
||||
print(line)
|
||||
|
||||
|
||||
def _handle_add(args: argparse.Namespace) -> None:
|
||||
"""Add a repository to the daemon config."""
|
||||
from .daemon import add_repo_to_config, is_daemon_running
|
||||
|
||||
try:
|
||||
add_repo_to_config(args.path, alias=args.alias)
|
||||
except ValueError as exc:
|
||||
print(f"Error: {exc}")
|
||||
sys.exit(1)
|
||||
|
||||
# Find the repo we just added to show confirmation
|
||||
alias = args.alias or os.path.basename(os.path.abspath(args.path))
|
||||
print(f"Added repository: {args.path} (alias: {alias})")
|
||||
|
||||
if is_daemon_running():
|
||||
print("Daemon will pick up the change automatically.")
|
||||
|
||||
|
||||
def _handle_remove(args: argparse.Namespace) -> None:
|
||||
"""Remove a repository from the daemon config."""
|
||||
from .daemon import is_daemon_running, load_config, remove_repo_from_config
|
||||
|
||||
config_before = load_config()
|
||||
count_before = len(config_before.repos)
|
||||
|
||||
config_after = remove_repo_from_config(args.path_or_alias)
|
||||
count_after = len(config_after.repos)
|
||||
|
||||
if count_before == count_after:
|
||||
print(f"No repository matching '{args.path_or_alias}' found in config.")
|
||||
sys.exit(1)
|
||||
|
||||
print(f"Removed repository: {args.path_or_alias}")
|
||||
|
||||
if is_daemon_running():
|
||||
print("Daemon will pick up the change automatically.")
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Main entry point
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def main() -> None:
|
||||
"""Entry point for the crg-daemon CLI."""
|
||||
logging.basicConfig(level=logging.INFO, format="%(levelname)s: %(message)s")
|
||||
|
||||
ap = argparse.ArgumentParser(
|
||||
prog="crg-daemon",
|
||||
description="Multi-repo watch daemon for code-review-graph",
|
||||
)
|
||||
sub = ap.add_subparsers(dest="command")
|
||||
|
||||
# start
|
||||
start_cmd = sub.add_parser("start", help="Start the daemon")
|
||||
start_cmd.add_argument(
|
||||
"--foreground",
|
||||
action="store_true",
|
||||
help="Run in the foreground instead of daemonizing",
|
||||
)
|
||||
|
||||
# stop
|
||||
sub.add_parser("stop", help="Stop the daemon")
|
||||
|
||||
# restart
|
||||
restart_cmd = sub.add_parser("restart", help="Restart the daemon")
|
||||
restart_cmd.add_argument(
|
||||
"--foreground",
|
||||
action="store_true",
|
||||
help="Run in the foreground instead of daemonizing",
|
||||
)
|
||||
|
||||
# status
|
||||
sub.add_parser("status", help="Show daemon status and configuration")
|
||||
|
||||
# logs
|
||||
logs_cmd = sub.add_parser("logs", help="Show daemon or per-repo logs")
|
||||
logs_cmd.add_argument(
|
||||
"--repo",
|
||||
default=None,
|
||||
metavar="ALIAS",
|
||||
help="Show logs for a specific repo (by alias)",
|
||||
)
|
||||
logs_cmd.add_argument(
|
||||
"--follow",
|
||||
"-f",
|
||||
action="store_true",
|
||||
help="Follow log output (tail -f)",
|
||||
)
|
||||
logs_cmd.add_argument(
|
||||
"--lines",
|
||||
"-n",
|
||||
type=int,
|
||||
default=50,
|
||||
help="Number of lines to show (default: 50)",
|
||||
)
|
||||
|
||||
# add
|
||||
add_cmd = sub.add_parser("add", help="Add a repository to the daemon config")
|
||||
add_cmd.add_argument("path", help="Path to the repository")
|
||||
add_cmd.add_argument(
|
||||
"--alias",
|
||||
default=None,
|
||||
help="Short alias for the repository (default: directory name)",
|
||||
)
|
||||
|
||||
# remove
|
||||
remove_cmd = sub.add_parser("remove", help="Remove a repository from the daemon config")
|
||||
remove_cmd.add_argument("path_or_alias", help="Repository path or alias to remove")
|
||||
|
||||
args = ap.parse_args()
|
||||
|
||||
if not args.command:
|
||||
ap.print_help()
|
||||
sys.exit(0)
|
||||
|
||||
handlers: dict[str, object] = {
|
||||
"start": _handle_start,
|
||||
"stop": _handle_stop,
|
||||
"restart": _handle_restart,
|
||||
"status": _handle_status,
|
||||
"logs": _handle_logs,
|
||||
"add": _handle_add,
|
||||
"remove": _handle_remove,
|
||||
}
|
||||
|
||||
handler = handlers.get(args.command)
|
||||
if handler is None:
|
||||
ap.print_help()
|
||||
sys.exit(1)
|
||||
|
||||
handler(args) # type: ignore[operator]
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,303 @@
|
||||
"""PreToolUse search enrichment for Claude Code hooks.
|
||||
|
||||
Intercepts Grep/Glob/Bash/Read tool calls and enriches them with
|
||||
structural context from the code knowledge graph: callers, callees,
|
||||
execution flows, community membership, and test coverage.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Flags that consume the next token in grep/rg commands
|
||||
_RG_FLAGS_WITH_VALUES = frozenset({
|
||||
"-e", "-f", "-m", "-A", "-B", "-C", "-g", "--glob",
|
||||
"-t", "--type", "--include", "--exclude", "--max-count",
|
||||
"--max-depth", "--max-filesize", "--color", "--colors",
|
||||
"--context-separator", "--field-match-separator",
|
||||
"--path-separator", "--replace", "--sort", "--sortr",
|
||||
})
|
||||
|
||||
|
||||
def extract_pattern(tool_name: str, tool_input: dict[str, Any]) -> str | None:
|
||||
"""Extract a search pattern from a tool call's input.
|
||||
|
||||
Returns None if no meaningful pattern can be extracted.
|
||||
"""
|
||||
if tool_name == "Grep":
|
||||
return tool_input.get("pattern")
|
||||
|
||||
if tool_name == "Glob":
|
||||
raw = tool_input.get("pattern", "")
|
||||
# Extract meaningful name from glob: "**/auth*.ts" -> "auth"
|
||||
# Skip pure extension globs like "**/*.ts"
|
||||
match = re.search(r"[*/]([a-zA-Z][a-zA-Z0-9_]{2,})", raw)
|
||||
return match.group(1) if match else None
|
||||
|
||||
if tool_name == "Bash":
|
||||
cmd = tool_input.get("command", "")
|
||||
if not re.search(r"\brg\b|\bgrep\b", cmd):
|
||||
return None
|
||||
tokens = cmd.split()
|
||||
found_cmd = False
|
||||
skip_next = False
|
||||
for token in tokens:
|
||||
if skip_next:
|
||||
skip_next = False
|
||||
continue
|
||||
if not found_cmd:
|
||||
if re.search(r"\brg$|\bgrep$", token):
|
||||
found_cmd = True
|
||||
continue
|
||||
if token.startswith("-"):
|
||||
if token in _RG_FLAGS_WITH_VALUES:
|
||||
skip_next = True
|
||||
continue
|
||||
cleaned = token.strip("'\"")
|
||||
return cleaned if len(cleaned) >= 3 else None
|
||||
return None
|
||||
|
||||
return None
|
||||
|
||||
|
||||
def _make_relative(file_path: str, repo_root: str) -> str:
|
||||
"""Make a file path relative to repo_root for display."""
|
||||
try:
|
||||
return str(Path(file_path).relative_to(repo_root))
|
||||
except ValueError:
|
||||
return file_path
|
||||
|
||||
|
||||
def _get_community_name(conn: Any, community_id: int) -> str:
|
||||
"""Fetch a community name by ID."""
|
||||
row = conn.execute(
|
||||
"SELECT name FROM communities WHERE id = ?", (community_id,)
|
||||
).fetchone()
|
||||
return row["name"] if row else ""
|
||||
|
||||
|
||||
def _get_flow_names_for_node(conn: Any, node_id: int) -> list[str]:
|
||||
"""Fetch execution flow names that a node participates in (max 3)."""
|
||||
rows = conn.execute(
|
||||
"SELECT f.name FROM flow_memberships fm "
|
||||
"JOIN flows f ON fm.flow_id = f.id "
|
||||
"WHERE fm.node_id = ? LIMIT 3",
|
||||
(node_id,),
|
||||
).fetchall()
|
||||
return [r["name"] for r in rows]
|
||||
|
||||
|
||||
def _format_node_context(
|
||||
node: Any,
|
||||
store: Any,
|
||||
conn: Any,
|
||||
repo_root: str,
|
||||
) -> list[str]:
|
||||
"""Format a single node's structural context as plain text lines."""
|
||||
from .graph import GraphNode
|
||||
assert isinstance(node, GraphNode)
|
||||
|
||||
qn = node.qualified_name
|
||||
loc = _make_relative(node.file_path, repo_root)
|
||||
if node.line_start:
|
||||
loc = f"{loc}:{node.line_start}"
|
||||
|
||||
header = f"{node.name} ({loc})"
|
||||
|
||||
# Community
|
||||
if node.extra.get("community_id"):
|
||||
cname = _get_community_name(conn, node.extra["community_id"])
|
||||
if cname:
|
||||
header += f" [{cname}]"
|
||||
else:
|
||||
# Check via direct query
|
||||
row = conn.execute(
|
||||
"SELECT community_id FROM nodes WHERE id = ?", (node.id,)
|
||||
).fetchone()
|
||||
if row and row["community_id"]:
|
||||
cname = _get_community_name(conn, row["community_id"])
|
||||
if cname:
|
||||
header += f" [{cname}]"
|
||||
|
||||
lines = [header]
|
||||
|
||||
# Callers (max 5, deduplicated)
|
||||
callers: list[str] = []
|
||||
seen: set[str] = set()
|
||||
for e in store.get_edges_by_target(qn):
|
||||
if e.kind == "CALLS" and len(callers) < 5:
|
||||
c = store.get_node(e.source_qualified)
|
||||
if c and c.name not in seen:
|
||||
seen.add(c.name)
|
||||
callers.append(c.name)
|
||||
if callers:
|
||||
lines.append(f" Called by: {', '.join(callers)}")
|
||||
|
||||
# Callees (max 5, deduplicated)
|
||||
callees: list[str] = []
|
||||
seen.clear()
|
||||
for e in store.get_edges_by_source(qn):
|
||||
if e.kind == "CALLS" and len(callees) < 5:
|
||||
c = store.get_node(e.target_qualified)
|
||||
if c and c.name not in seen:
|
||||
seen.add(c.name)
|
||||
callees.append(c.name)
|
||||
if callees:
|
||||
lines.append(f" Calls: {', '.join(callees)}")
|
||||
|
||||
# Execution flows
|
||||
flow_names = _get_flow_names_for_node(conn, node.id)
|
||||
if flow_names:
|
||||
lines.append(f" Flows: {', '.join(flow_names)}")
|
||||
|
||||
# Tests
|
||||
tests: list[str] = []
|
||||
for e in store.get_edges_by_target(qn):
|
||||
if e.kind == "TESTED_BY" and len(tests) < 3:
|
||||
t = store.get_node(e.source_qualified)
|
||||
if t:
|
||||
tests.append(t.name)
|
||||
if tests:
|
||||
lines.append(f" Tests: {', '.join(tests)}")
|
||||
|
||||
return lines
|
||||
|
||||
|
||||
def enrich_search(pattern: str, repo_root: str) -> str:
|
||||
"""Search the graph for pattern and return enriched context."""
|
||||
from .graph import GraphStore
|
||||
from .search import _fts_search
|
||||
|
||||
db_path = Path(repo_root) / ".code-review-graph" / "graph.db"
|
||||
if not db_path.exists():
|
||||
return ""
|
||||
|
||||
store = GraphStore(db_path)
|
||||
try:
|
||||
conn = store._conn
|
||||
|
||||
fts_results = _fts_search(conn, pattern, limit=8)
|
||||
if not fts_results:
|
||||
return ""
|
||||
|
||||
all_lines: list[str] = []
|
||||
count = 0
|
||||
for node_id, _score in fts_results:
|
||||
if count >= 5:
|
||||
break
|
||||
node = store.get_node_by_id(node_id)
|
||||
if not node or node.is_test:
|
||||
continue
|
||||
node_lines = _format_node_context(node, store, conn, repo_root)
|
||||
all_lines.extend(node_lines)
|
||||
all_lines.append("")
|
||||
count += 1
|
||||
|
||||
if not all_lines:
|
||||
return ""
|
||||
|
||||
header = f'[code-review-graph] {count} symbol(s) matching "{pattern}":\n'
|
||||
return header + "\n".join(all_lines)
|
||||
finally:
|
||||
store.close()
|
||||
|
||||
|
||||
def enrich_file_read(file_path: str, repo_root: str) -> str:
|
||||
"""Enrich a file read with structural context for functions in that file."""
|
||||
from .graph import GraphStore
|
||||
|
||||
db_path = Path(repo_root) / ".code-review-graph" / "graph.db"
|
||||
if not db_path.exists():
|
||||
return ""
|
||||
|
||||
store = GraphStore(db_path)
|
||||
try:
|
||||
conn = store._conn
|
||||
nodes = store.get_nodes_by_file(file_path)
|
||||
if not nodes:
|
||||
# Try with resolved path
|
||||
try:
|
||||
resolved = str(Path(file_path).resolve())
|
||||
nodes = store.get_nodes_by_file(resolved)
|
||||
except (OSError, ValueError):
|
||||
pass
|
||||
if not nodes:
|
||||
return ""
|
||||
|
||||
# Filter to functions/classes/types (skip File nodes), limit to 10
|
||||
interesting = [
|
||||
n for n in nodes
|
||||
if n.kind in ("Function", "Class", "Type", "Test")
|
||||
][:10]
|
||||
|
||||
if not interesting:
|
||||
return ""
|
||||
|
||||
all_lines: list[str] = []
|
||||
for node in interesting:
|
||||
node_lines = _format_node_context(node, store, conn, repo_root)
|
||||
all_lines.extend(node_lines)
|
||||
all_lines.append("")
|
||||
|
||||
rel_path = _make_relative(file_path, repo_root)
|
||||
header = (
|
||||
f"[code-review-graph] {len(interesting)} symbol(s) in {rel_path}:\n"
|
||||
)
|
||||
return header + "\n".join(all_lines)
|
||||
finally:
|
||||
store.close()
|
||||
|
||||
|
||||
def run_hook() -> None:
|
||||
"""Entry point for the enrich CLI subcommand.
|
||||
|
||||
Reads Claude Code hook JSON from stdin, extracts the search pattern,
|
||||
queries the graph, and outputs hookSpecificOutput JSON to stdout.
|
||||
"""
|
||||
try:
|
||||
hook_input = json.load(sys.stdin)
|
||||
except (json.JSONDecodeError, ValueError):
|
||||
return
|
||||
|
||||
tool_name = hook_input.get("tool_name", "")
|
||||
tool_input = hook_input.get("tool_input", {})
|
||||
cwd = hook_input.get("cwd", os.getcwd())
|
||||
|
||||
# Find repo root by walking up from cwd
|
||||
from .incremental import find_project_root
|
||||
|
||||
repo_root = str(find_project_root(Path(cwd)))
|
||||
db_path = Path(repo_root) / ".code-review-graph" / "graph.db"
|
||||
if not db_path.exists():
|
||||
return
|
||||
|
||||
# Dispatch
|
||||
context = ""
|
||||
if tool_name == "Read":
|
||||
fp = tool_input.get("file_path", "")
|
||||
if fp:
|
||||
context = enrich_file_read(fp, repo_root)
|
||||
else:
|
||||
pattern = extract_pattern(tool_name, tool_input)
|
||||
if not pattern or len(pattern) < 3:
|
||||
return
|
||||
context = enrich_search(pattern, repo_root)
|
||||
|
||||
if not context:
|
||||
return
|
||||
|
||||
response = {
|
||||
"hookSpecificOutput": {
|
||||
"hookEventName": "PreToolUse",
|
||||
"additionalContext": context,
|
||||
}
|
||||
}
|
||||
json.dump(response, sys.stdout)
|
||||
@@ -0,0 +1,33 @@
|
||||
"""Evaluation framework for code-review-graph.
|
||||
|
||||
Provides scoring metrics (token efficiency, MRR, precision/recall),
|
||||
benchmark runners, and report generators for benchmarking graph-based code reviews.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from .reporter import generate_full_report, generate_markdown_report, generate_readme_tables
|
||||
from .scorer import compute_mrr, compute_precision_recall, compute_token_efficiency
|
||||
|
||||
|
||||
def __getattr__(name: str):
|
||||
"""Lazy-import runner functions (require pyyaml)."""
|
||||
_runner_names = {"load_all_configs", "load_config", "run_eval", "write_csv"}
|
||||
if name in _runner_names:
|
||||
from . import runner
|
||||
return getattr(runner, name)
|
||||
raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
|
||||
|
||||
|
||||
__all__ = [
|
||||
"compute_mrr",
|
||||
"compute_precision_recall",
|
||||
"compute_token_efficiency",
|
||||
"generate_full_report",
|
||||
"generate_markdown_report",
|
||||
"generate_readme_tables",
|
||||
"load_all_configs",
|
||||
"load_config",
|
||||
"run_eval",
|
||||
"write_csv",
|
||||
]
|
||||
@@ -0,0 +1 @@
|
||||
"""Benchmark modules for the evaluation framework."""
|
||||
@@ -0,0 +1,193 @@
|
||||
"""Agent baseline benchmark: grep-and-read-top-k versus a graph query.
|
||||
|
||||
The whole-corpus baseline in the standalone token benchmark is an upper
|
||||
bound no real agent pays: a competent agent greps for identifiers from the
|
||||
question and reads only the best-matching files. This benchmark measures
|
||||
that realistic baseline:
|
||||
|
||||
1. Derive search terms from the question (identifier-shaped tokens via
|
||||
``search.extract_query_identifiers`` plus plain keywords).
|
||||
2. Pure-python grep over the corpus (no external ``rg``/``grep`` binary),
|
||||
ranking files by total case-insensitive match count.
|
||||
3. Read the top-k files (k=3) and token-count them with the chars/4 utility
|
||||
(``token_benchmark.estimate_tokens``) as ``baseline_tokens``.
|
||||
4. Compare against the graph-query cost for the same question — hybrid
|
||||
search hits plus one hop of neighbor edges, the same accounting used by
|
||||
``code_review_graph/token_benchmark.py``.
|
||||
|
||||
Questions come from ``agent_questions:`` in the repo config, falling back to
|
||||
the ``search_queries`` query strings when absent.
|
||||
|
||||
Failure semantics match the other benchmarks: a thrown search is recorded
|
||||
with ``status="error"`` and excluded from aggregates; rows where either side
|
||||
of the ratio is zero get ``status="no_graph_results"`` /
|
||||
``status="no_baseline_match"`` and are likewise excluded.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import statistics
|
||||
from collections.abc import Iterator
|
||||
from pathlib import Path
|
||||
|
||||
from code_review_graph.token_benchmark import estimate_tokens
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
DEFAULT_TOP_K = 3
|
||||
|
||||
_SOURCE_EXTS = (
|
||||
".py", ".js", ".ts", ".tsx", ".go", ".rs", ".java",
|
||||
".c", ".cpp", ".h", ".rb", ".php", ".swift", ".kt",
|
||||
)
|
||||
|
||||
_SKIP_DIRS = {
|
||||
".git", ".hg", ".svn", "node_modules", "__pycache__",
|
||||
".code-review-graph", ".venv", "venv", "dist", "build",
|
||||
}
|
||||
|
||||
_STOPWORDS = {
|
||||
"how", "does", "do", "the", "a", "an", "is", "are", "was", "what",
|
||||
"where", "when", "which", "who", "why", "and", "or", "in", "on", "of",
|
||||
"to", "for", "with", "via", "into", "from", "this", "that", "it", "its",
|
||||
}
|
||||
|
||||
|
||||
def derive_search_terms(question: str) -> list[str]:
|
||||
"""Derive lowercase grep terms: identifiers first, then plain keywords.
|
||||
|
||||
Identifier-shaped tokens (``Client.request``, ``get_users``, ``APIRoute``)
|
||||
are extracted via ``search.extract_query_identifiers``; remaining words of
|
||||
3+ characters that are not stopwords are appended. Order is deterministic.
|
||||
"""
|
||||
from code_review_graph.search import extract_query_identifiers
|
||||
|
||||
terms: list[str] = []
|
||||
seen: set[str] = set()
|
||||
for ident in extract_query_identifiers(question):
|
||||
if ident not in seen:
|
||||
seen.add(ident)
|
||||
terms.append(ident)
|
||||
for word in question.split():
|
||||
w = word.strip(".,;:!?\"'()[]{}`").lower()
|
||||
if len(w) >= 3 and w not in _STOPWORDS and w not in seen:
|
||||
seen.add(w)
|
||||
terms.append(w)
|
||||
return terms
|
||||
|
||||
|
||||
def iter_source_files(repo_path: Path) -> Iterator[Path]:
|
||||
"""Yield source files under *repo_path*, skipping vendored/VCS dirs."""
|
||||
for path in sorted(repo_path.rglob("*")):
|
||||
if path.suffix not in _SOURCE_EXTS or not path.is_file():
|
||||
continue
|
||||
if any(part in _SKIP_DIRS for part in path.parts):
|
||||
continue
|
||||
yield path
|
||||
|
||||
|
||||
def grep_rank(
|
||||
repo_path: Path, terms: list[str], k: int = DEFAULT_TOP_K,
|
||||
) -> list[tuple[str, int]]:
|
||||
"""Rank source files by total case-insensitive term matches; take top-k.
|
||||
|
||||
Pure python — no external grep/rg dependency. Deterministic: ties break
|
||||
on the relative path. Files with zero matches are dropped.
|
||||
"""
|
||||
lowered = [t.lower() for t in terms if t]
|
||||
if not lowered:
|
||||
return []
|
||||
scores: list[tuple[str, int]] = []
|
||||
for path in iter_source_files(repo_path):
|
||||
try:
|
||||
text = path.read_text(encoding="utf-8", errors="replace").lower()
|
||||
except OSError:
|
||||
continue
|
||||
count = sum(text.count(term) for term in lowered)
|
||||
if count > 0:
|
||||
scores.append((str(path.relative_to(repo_path)), count))
|
||||
scores.sort(key=lambda item: (-item[1], item[0]))
|
||||
return scores[:k]
|
||||
|
||||
|
||||
def run(repo_path: Path, store, config: dict) -> list[dict]:
|
||||
"""Run the agent baseline benchmark for one repo."""
|
||||
questions = list(config.get("agent_questions") or [])
|
||||
if not questions:
|
||||
questions = [sq["query"] for sq in config.get("search_queries", [])]
|
||||
|
||||
k = int(config.get("agent_baseline_top_k", DEFAULT_TOP_K))
|
||||
results: list[dict] = []
|
||||
|
||||
for question in questions:
|
||||
terms = derive_search_terms(question)
|
||||
top = grep_rank(repo_path, terms, k=k)
|
||||
baseline_tokens = 0
|
||||
for rel, _count in top:
|
||||
try:
|
||||
baseline_tokens += estimate_tokens(
|
||||
(repo_path / rel).read_text(encoding="utf-8", errors="replace")
|
||||
)
|
||||
except OSError:
|
||||
continue
|
||||
|
||||
row: dict = {
|
||||
"repo": config["name"],
|
||||
"question": question,
|
||||
"terms": " ".join(terms),
|
||||
"files_matched": len(top),
|
||||
"top_files": ";".join(rel for rel, _ in top),
|
||||
"baseline_tokens": baseline_tokens,
|
||||
"graph_tokens": "",
|
||||
"baseline_to_graph_ratio": "",
|
||||
"status": "ok",
|
||||
"error": "",
|
||||
}
|
||||
|
||||
try:
|
||||
from code_review_graph.search import hybrid_search
|
||||
hits = hybrid_search(store, question, limit=5)
|
||||
except Exception as exc:
|
||||
logger.warning("hybrid_search failed on %r: %s", question, exc)
|
||||
row["status"] = "error"
|
||||
row["error"] = str(exc)[:200]
|
||||
results.append(row)
|
||||
continue
|
||||
|
||||
# Same accounting as the standalone token benchmark: search hits
|
||||
# plus up to 5 outgoing edges of neighbor context per hit.
|
||||
graph_tokens = 0
|
||||
for hit in hits:
|
||||
graph_tokens += estimate_tokens(str(hit))
|
||||
qn = hit.get("qualified_name", "")
|
||||
for edge in store.get_edges_by_source(qn)[:5]:
|
||||
graph_tokens += estimate_tokens(str(edge))
|
||||
|
||||
row["graph_tokens"] = graph_tokens
|
||||
if baseline_tokens > 0 and graph_tokens > 0:
|
||||
row["baseline_to_graph_ratio"] = round(baseline_tokens / graph_tokens, 1)
|
||||
elif graph_tokens == 0:
|
||||
row["status"] = "no_graph_results"
|
||||
else:
|
||||
row["status"] = "no_baseline_match"
|
||||
results.append(row)
|
||||
|
||||
return results
|
||||
|
||||
|
||||
def aggregate(results: list[dict]) -> dict:
|
||||
"""Aggregate over rows where both sides of the comparison exist."""
|
||||
ok = [r for r in results if r.get("status") == "ok"]
|
||||
ratios = [float(r["baseline_to_graph_ratio"]) for r in ok]
|
||||
return {
|
||||
"total_rows": len(results),
|
||||
"ok_rows": len(ok),
|
||||
"error_rows": sum(1 for r in results if r.get("status") == "error"),
|
||||
"median_baseline_to_graph_ratio": (
|
||||
round(statistics.median(ratios), 1) if ratios else None
|
||||
),
|
||||
"mean_baseline_to_graph_ratio": (
|
||||
round(statistics.mean(ratios), 1) if ratios else None
|
||||
),
|
||||
}
|
||||
@@ -0,0 +1,60 @@
|
||||
"""Build performance benchmark: measures timing of graph operations."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def run(repo_path: Path, store, config: dict) -> list[dict]:
|
||||
"""Run build performance benchmark."""
|
||||
stats = store.get_stats()
|
||||
|
||||
# Time flow detection
|
||||
try:
|
||||
from code_review_graph.flows import store_flows, trace_flows
|
||||
t0 = time.perf_counter()
|
||||
flows = trace_flows(store)
|
||||
store_flows(store, flows)
|
||||
flow_time = time.perf_counter() - t0
|
||||
except Exception as exc:
|
||||
logger.warning("Flow detection failed: %s", exc)
|
||||
flow_time = 0.0
|
||||
|
||||
# Time community detection
|
||||
try:
|
||||
from code_review_graph.communities import detect_communities, store_communities
|
||||
t0 = time.perf_counter()
|
||||
comms = detect_communities(store)
|
||||
store_communities(store, comms)
|
||||
community_time = time.perf_counter() - t0
|
||||
except Exception as exc:
|
||||
logger.warning("Community detection failed: %s", exc)
|
||||
community_time = 0.0
|
||||
|
||||
# Time search (average of queries)
|
||||
search_times: list[float] = []
|
||||
for sq in config.get("search_queries", [])[:10]:
|
||||
t0 = time.perf_counter()
|
||||
store.search_nodes(sq["query"], limit=20)
|
||||
search_times.append(time.perf_counter() - t0)
|
||||
|
||||
avg_search_ms = round(
|
||||
sum(search_times) / max(len(search_times), 1) * 1000, 1
|
||||
)
|
||||
|
||||
return [{
|
||||
"repo": config["name"],
|
||||
"file_count": stats.files_count,
|
||||
"node_count": stats.total_nodes,
|
||||
"edge_count": stats.total_edges,
|
||||
"flow_detection_seconds": round(flow_time, 3),
|
||||
"community_detection_seconds": round(community_time, 3),
|
||||
"search_avg_ms": avg_search_ms,
|
||||
"nodes_per_second": round(
|
||||
stats.total_nodes / max(flow_time, 0.001)
|
||||
),
|
||||
}]
|
||||
@@ -0,0 +1,36 @@
|
||||
"""Flow completeness benchmark: evaluates entry point detection and flow tracing."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from pathlib import Path
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def run(repo_path: Path, store, config: dict) -> list[dict]:
|
||||
"""Run flow completeness benchmark."""
|
||||
from code_review_graph.flows import store_flows, trace_flows
|
||||
|
||||
flows = trace_flows(store)
|
||||
count = store_flows(store, flows)
|
||||
|
||||
# Get detected entry point names
|
||||
detected_entries = set()
|
||||
for flow in flows:
|
||||
detected_entries.add(flow.get("entry_point") or flow.get("name", ""))
|
||||
|
||||
known = set(config.get("entry_points", []))
|
||||
found = sum(1 for ep in known if any(ep in d for d in detected_entries))
|
||||
|
||||
depths = [f.get("depth", 0) for f in flows]
|
||||
|
||||
return [{
|
||||
"repo": config["name"],
|
||||
"known_entry_points": len(known),
|
||||
"detected_entry_points": found,
|
||||
"recall": round(found / max(len(known), 1), 3),
|
||||
"detected_flows": count,
|
||||
"avg_flow_depth": round(sum(depths) / max(len(depths), 1), 1),
|
||||
"max_flow_depth": max(depths, default=0),
|
||||
}]
|
||||
@@ -0,0 +1,220 @@
|
||||
"""Impact accuracy benchmark: measures precision/recall of change impact analysis.
|
||||
|
||||
Two ground-truth modes are emitted side by side (``ground_truth_mode`` column):
|
||||
|
||||
- **graph-derived (circular — upper bound)** — the historical mode. Ground
|
||||
truth is the changed files plus files with CALLS/IMPORTS_FROM edges into
|
||||
them, i.e. derived from the same graph the predictor traverses. Recall in
|
||||
this mode is an upper bound by construction, not independent evidence.
|
||||
- **co-change (same commit, seed excluded)** — the honest mode. The predictor
|
||||
is seeded with a single changed file and graded against the *other* files
|
||||
the author actually touched in the same commit. The ground truth comes from
|
||||
git history, not from the graph.
|
||||
|
||||
Failure semantics: if ``analyze_changes`` throws, the row is recorded with
|
||||
``status="error"`` and empty metric fields — it stays in the CSV but is
|
||||
excluded from aggregates. (Previously a failure silently set
|
||||
``predicted = set(changed)``, guaranteeing a fake recall of 1.0.)
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import statistics
|
||||
import subprocess
|
||||
from pathlib import Path
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
MODE_GRAPH_DERIVED = "graph-derived (circular — upper bound)"
|
||||
MODE_CO_CHANGE = "co-change (same commit, seed excluded)"
|
||||
|
||||
|
||||
def _get_changed_files(repo_path: Path, sha: str) -> list[str]:
|
||||
"""Get list of changed files for a commit."""
|
||||
result = subprocess.run(
|
||||
["git", "diff", "--name-only", f"{sha}~1", sha],
|
||||
cwd=str(repo_path),
|
||||
capture_output=True,
|
||||
text=True,
|
||||
)
|
||||
if result.returncode != 0:
|
||||
result = subprocess.run(
|
||||
["git", "diff", "--name-only", "HEAD~1", "HEAD"],
|
||||
cwd=str(repo_path),
|
||||
capture_output=True,
|
||||
text=True,
|
||||
)
|
||||
return [f.strip() for f in result.stdout.strip().splitlines() if f.strip()]
|
||||
|
||||
|
||||
def _files_from_analysis(analysis: dict) -> set[str]:
|
||||
"""Extract predicted file paths from an ``analyze_changes`` result."""
|
||||
predicted: set[str] = set()
|
||||
for f in analysis.get("changed_functions", []):
|
||||
if isinstance(f, dict) and "file_path" in f:
|
||||
predicted.add(f["file_path"])
|
||||
elif isinstance(f, dict) and "file" in f:
|
||||
predicted.add(f["file"])
|
||||
for flow in analysis.get("affected_flows", []):
|
||||
if isinstance(flow, dict):
|
||||
for node in flow.get("nodes", []):
|
||||
if isinstance(node, dict) and "file_path" in node:
|
||||
predicted.add(node["file_path"])
|
||||
return predicted
|
||||
|
||||
|
||||
def _graph_neighbor_files(store, files: list[str]) -> set[str]:
|
||||
"""Files with CALLS/IMPORTS_FROM edges into any node of *files* (one hop)."""
|
||||
out: set[str] = set()
|
||||
for f in files:
|
||||
for node in store.get_nodes_by_file(f):
|
||||
for edge in store.get_edges_by_target(node.qualified_name):
|
||||
if edge.kind in ("CALLS", "IMPORTS_FROM"):
|
||||
src_qual = edge.source_qualified
|
||||
src_file = src_qual.split("::")[0] if "::" in src_qual else ""
|
||||
if src_file:
|
||||
out.add(src_file)
|
||||
return out
|
||||
|
||||
|
||||
def _base_row(repo: str, sha: str, mode: str, seed: str) -> dict:
|
||||
return {
|
||||
"repo": repo,
|
||||
"commit": sha,
|
||||
"ground_truth_mode": mode,
|
||||
"seed_file": seed,
|
||||
"predicted_files": "",
|
||||
"actual_files": "",
|
||||
"true_positives": "",
|
||||
"precision": "",
|
||||
"recall": "",
|
||||
"f1": "",
|
||||
"status": "ok",
|
||||
"error": "",
|
||||
}
|
||||
|
||||
|
||||
def _scored_row(
|
||||
repo: str, sha: str, mode: str, seed: str,
|
||||
predicted: set[str], actual: set[str],
|
||||
) -> dict:
|
||||
tp = len(predicted & actual)
|
||||
precision = tp / max(len(predicted), 1)
|
||||
recall = tp / max(len(actual), 1)
|
||||
f1 = 2 * precision * recall / max(precision + recall, 0.001)
|
||||
row = _base_row(repo, sha, mode, seed)
|
||||
row.update({
|
||||
"predicted_files": len(predicted),
|
||||
"actual_files": len(actual),
|
||||
"true_positives": tp,
|
||||
"precision": round(precision, 3),
|
||||
"recall": round(recall, 3),
|
||||
"f1": round(f1, 3),
|
||||
})
|
||||
return row
|
||||
|
||||
|
||||
def _error_row(repo: str, sha: str, mode: str, seed: str, exc: Exception) -> dict:
|
||||
row = _base_row(repo, sha, mode, seed)
|
||||
row["status"] = "error"
|
||||
row["error"] = str(exc)[:200]
|
||||
return row
|
||||
|
||||
|
||||
def run(repo_path: Path, store, config: dict) -> list[dict]:
|
||||
"""Run impact accuracy benchmark (both ground-truth modes)."""
|
||||
from code_review_graph.changes import analyze_changes
|
||||
|
||||
results = []
|
||||
repo = config["name"]
|
||||
for tc in config.get("test_commits", []):
|
||||
sha = tc["sha"]
|
||||
changed = _get_changed_files(repo_path, sha)
|
||||
if not changed:
|
||||
continue
|
||||
|
||||
# --- Mode 1: graph-derived ground truth (circular — upper bound) ---
|
||||
try:
|
||||
analysis = analyze_changes(
|
||||
store, changed, repo_root=str(repo_path), base=sha + "~1",
|
||||
)
|
||||
except Exception as exc:
|
||||
# Old behaviour set predicted = set(changed) here, which
|
||||
# guarantees recall 1.0 on a *failed* run. Mark failed instead.
|
||||
logger.warning("analyze_changes failed on %s: %s", sha, exc)
|
||||
results.append(_error_row(repo, sha, MODE_GRAPH_DERIVED, "", exc))
|
||||
analysis = None
|
||||
|
||||
if analysis is not None:
|
||||
predicted = set(changed) | _files_from_analysis(analysis)
|
||||
actual = set(changed) | _graph_neighbor_files(store, changed)
|
||||
results.append(
|
||||
_scored_row(repo, sha, MODE_GRAPH_DERIVED, "", predicted, actual)
|
||||
)
|
||||
|
||||
# --- Mode 2: co-change ground truth (honest) ---
|
||||
# Seed the predictor with a single changed file and grade against
|
||||
# the other files the author touched in the same commit. Note the
|
||||
# seed analysis deliberately gets no repo_root/diff: it must only
|
||||
# see the seed file, never the full commit diff.
|
||||
seed = sorted(changed)[0]
|
||||
co_actual = set(changed) - {seed}
|
||||
if not co_actual:
|
||||
row = _base_row(repo, sha, MODE_CO_CHANGE, seed)
|
||||
row["status"] = "skipped"
|
||||
row["error"] = "single-file commit: no co-changed files to grade against"
|
||||
results.append(row)
|
||||
continue
|
||||
|
||||
try:
|
||||
seed_analysis = analyze_changes(store, [seed])
|
||||
except Exception as exc:
|
||||
logger.warning("analyze_changes (seed=%s) failed on %s: %s", seed, sha, exc)
|
||||
results.append(_error_row(repo, sha, MODE_CO_CHANGE, seed, exc))
|
||||
continue
|
||||
|
||||
co_predicted = _files_from_analysis(seed_analysis)
|
||||
co_predicted |= _graph_neighbor_files(store, [seed])
|
||||
co_predicted.discard(seed)
|
||||
results.append(
|
||||
_scored_row(repo, sha, MODE_CO_CHANGE, seed, co_predicted, co_actual)
|
||||
)
|
||||
|
||||
return results
|
||||
|
||||
|
||||
def aggregate(results: list[dict]) -> dict:
|
||||
"""Per-mode means over successful rows only.
|
||||
|
||||
Error/skipped rows stay in the CSV but never contribute to a number.
|
||||
"""
|
||||
out: dict = {
|
||||
"total_rows": len(results),
|
||||
"error_rows": sum(1 for r in results if r.get("status") == "error"),
|
||||
"skipped_rows": sum(1 for r in results if r.get("status") == "skipped"),
|
||||
}
|
||||
for key, mode in (
|
||||
("graph_derived", MODE_GRAPH_DERIVED),
|
||||
("co_change", MODE_CO_CHANGE),
|
||||
):
|
||||
rows = [
|
||||
r for r in results
|
||||
if r.get("ground_truth_mode") == mode and r.get("status") == "ok"
|
||||
]
|
||||
out[key] = {
|
||||
"ok_rows": len(rows),
|
||||
"mean_precision": (
|
||||
round(statistics.mean(float(r["precision"]) for r in rows), 3)
|
||||
if rows else None
|
||||
),
|
||||
"mean_recall": (
|
||||
round(statistics.mean(float(r["recall"]) for r in rows), 3)
|
||||
if rows else None
|
||||
),
|
||||
"mean_f1": (
|
||||
round(statistics.mean(float(r["f1"]) for r in rows), 3)
|
||||
if rows else None
|
||||
),
|
||||
}
|
||||
return out
|
||||
@@ -0,0 +1,125 @@
|
||||
"""Multi-hop retrieval benchmark.
|
||||
|
||||
Tests a two-step tool chain that mimics how an LLM agent actually uses the
|
||||
graph for complex tasks:
|
||||
|
||||
1. ``hybrid_search(nl_query)`` to find a starting anchor from a natural-
|
||||
language question.
|
||||
2. ``query_graph(pattern, target=anchor)`` to traverse one hop along the
|
||||
requested edge kind (callers_of / callees_of / tests_for / ...).
|
||||
|
||||
For each task the benchmark records:
|
||||
|
||||
- ``anchor_found`` — did semantic search return a node whose qualified_name
|
||||
ends with the expected suffix in the top-K?
|
||||
- ``anchor_rank`` — index in the search result list (lower is better).
|
||||
- ``neighbor_count`` — number of neighbors returned by the traversal.
|
||||
- ``neighbor_recall`` — fraction of ``expected_neighbor_names`` that appear
|
||||
among the neighbor names.
|
||||
- ``score`` — ``int(anchor_found) * neighbor_recall``. Range 0–1.
|
||||
|
||||
Tasks are defined per-config under ``multi_hop_tasks:`` in
|
||||
``code_review_graph/eval/configs/*.yaml``. See
|
||||
``docs/REPRODUCING.md`` for the schema and the curated canonical task set.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _name_set(rows: list[dict[str, Any]]) -> set[str]:
|
||||
out: set[str] = set()
|
||||
for r in rows:
|
||||
name = (r.get("name") or "").lower()
|
||||
if name:
|
||||
out.add(name)
|
||||
return out
|
||||
|
||||
|
||||
def run(repo_path: Path, store, config: dict) -> list[dict]:
|
||||
"""Run the multi-hop retrieval benchmark for one repo."""
|
||||
# Imports are local so an import-time failure in one optional benchmark
|
||||
# does not poison the whole runner.
|
||||
from code_review_graph.search import hybrid_search
|
||||
from code_review_graph.tools.query import query_graph
|
||||
|
||||
repo_root = str(repo_path)
|
||||
results: list[dict] = []
|
||||
|
||||
for task in config.get("multi_hop_tasks", []):
|
||||
task_id = task["id"]
|
||||
nl_query = task["nl_query"]
|
||||
suffix = task["anchor_qualified_suffix"].lower()
|
||||
traversal = task.get("traversal_pattern", "callers_of")
|
||||
expected = [e.lower() for e in task.get("expected_neighbor_names", [])]
|
||||
k = int(task.get("k", 10))
|
||||
|
||||
# Step 1 — semantic search
|
||||
try:
|
||||
hits = hybrid_search(store, nl_query, limit=k)
|
||||
except Exception as exc: # noqa: BLE001 — benchmark must not abort the runner
|
||||
logger.warning("hybrid_search failed on %s: %s", task_id, exc)
|
||||
hits = []
|
||||
|
||||
anchor = None
|
||||
anchor_rank = -1
|
||||
for i, h in enumerate(hits):
|
||||
qn = (h.get("qualified_name") or "").lower()
|
||||
if qn.endswith(suffix):
|
||||
anchor = h
|
||||
anchor_rank = i
|
||||
break
|
||||
|
||||
if anchor is None:
|
||||
results.append({
|
||||
"repo": config["name"],
|
||||
"task_id": task_id,
|
||||
"nl_query": nl_query,
|
||||
"anchor_found": False,
|
||||
"anchor_rank": -1,
|
||||
"neighbor_count": 0,
|
||||
"expected_count": len(expected),
|
||||
"matched_count": 0,
|
||||
"neighbor_recall": 0.0,
|
||||
"score": 0.0,
|
||||
})
|
||||
continue
|
||||
|
||||
# Step 2 — single-hop graph traversal from the anchor
|
||||
try:
|
||||
trav = query_graph(
|
||||
pattern=traversal,
|
||||
target=anchor["qualified_name"],
|
||||
repo_root=repo_root,
|
||||
detail_level="standard",
|
||||
)
|
||||
except Exception as exc: # noqa: BLE001
|
||||
logger.warning(
|
||||
"query_graph(%s) failed on %s: %s", traversal, task_id, exc,
|
||||
)
|
||||
trav = {}
|
||||
|
||||
rows = trav.get("data") or trav.get("results") or []
|
||||
names = _name_set(rows)
|
||||
matched = sum(1 for e in expected if e in names)
|
||||
recall = matched / len(expected) if expected else 0.0
|
||||
|
||||
results.append({
|
||||
"repo": config["name"],
|
||||
"task_id": task_id,
|
||||
"nl_query": nl_query,
|
||||
"anchor_found": True,
|
||||
"anchor_rank": anchor_rank,
|
||||
"neighbor_count": len(rows),
|
||||
"expected_count": len(expected),
|
||||
"matched_count": matched,
|
||||
"neighbor_recall": round(recall, 3),
|
||||
"score": round(recall, 3),
|
||||
})
|
||||
|
||||
return results
|
||||
@@ -0,0 +1,59 @@
|
||||
"""Search quality benchmark: measures search result ranking via MRR."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import sqlite3
|
||||
from pathlib import Path
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def run(repo_path: Path, store, config: dict) -> list[dict]:
|
||||
"""Run search quality benchmark."""
|
||||
results = []
|
||||
for sq in config.get("search_queries", []):
|
||||
query = sq["query"]
|
||||
expected = sq["expected"]
|
||||
|
||||
try:
|
||||
from code_review_graph.search import hybrid_search
|
||||
search_results = hybrid_search(store, query, limit=20)
|
||||
except (ImportError, sqlite3.OperationalError) as exc:
|
||||
logger.debug("hybrid_search unavailable, using fallback: %s", exc)
|
||||
# Fallback to basic search
|
||||
search_results = [
|
||||
{"qualified_name": n.qualified_name}
|
||||
for n in store.search_nodes(query, limit=20)
|
||||
]
|
||||
|
||||
rank = 0
|
||||
for i, r in enumerate(search_results):
|
||||
if isinstance(r, dict):
|
||||
qn = r.get("qualified_name", "")
|
||||
elif hasattr(r, "qualified_name"):
|
||||
qn = r.qualified_name
|
||||
else:
|
||||
qn = ""
|
||||
qn_lower = qn.lower()
|
||||
exp_lower = expected.lower()
|
||||
# Match if expected is substring of qn, qn is substring of expected,
|
||||
# or the name part after :: matches
|
||||
exp_name = expected.rsplit("::", 1)[-1] if "::" in expected else expected
|
||||
qn_name = qn.rsplit("::", 1)[-1] if "::" in qn else qn
|
||||
if (
|
||||
exp_lower in qn_lower
|
||||
or qn_lower in exp_lower
|
||||
or exp_name.lower() == qn_name.lower()
|
||||
):
|
||||
rank = i + 1
|
||||
break
|
||||
|
||||
results.append({
|
||||
"repo": config["name"],
|
||||
"query": query,
|
||||
"expected": expected,
|
||||
"rank": rank,
|
||||
"reciprocal_rank": round(1.0 / rank if rank > 0 else 0.0, 3),
|
||||
})
|
||||
return results
|
||||
@@ -0,0 +1,143 @@
|
||||
"""Token efficiency benchmark: compares naive, standard, and graph-based token counts.
|
||||
|
||||
Failure semantics: if ``get_review_context`` throws, the row is recorded with
|
||||
``status="error"`` and empty metric fields. It stays in the CSV for forensics
|
||||
but is excluded from every aggregate — a failed tool call is not a
|
||||
measurement. (Previously a failure silently produced ``graph_tokens=0`` and
|
||||
``ratio = naive / 1``, inflating the results.)
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
import statistics
|
||||
import subprocess
|
||||
from pathlib import Path
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _count_tokens(text: str) -> int:
|
||||
"""Approximate token count (1 token ~ 4 chars)."""
|
||||
return len(text) // 4
|
||||
|
||||
|
||||
def _get_changed_files(repo_path: Path, sha: str) -> list[str]:
|
||||
"""Get list of changed files for a commit."""
|
||||
result = subprocess.run(
|
||||
["git", "diff", "--name-only", f"{sha}~1", sha],
|
||||
cwd=str(repo_path),
|
||||
capture_output=True,
|
||||
text=True,
|
||||
)
|
||||
if result.returncode != 0:
|
||||
# Fallback: diff against parent
|
||||
result = subprocess.run(
|
||||
["git", "diff", "--name-only", "HEAD~1", "HEAD"],
|
||||
cwd=str(repo_path),
|
||||
capture_output=True,
|
||||
text=True,
|
||||
)
|
||||
return [f.strip() for f in result.stdout.strip().splitlines() if f.strip()]
|
||||
|
||||
|
||||
def _count_file_tokens(repo_path: Path, files: list[str]) -> int:
|
||||
"""Count tokens from full file contents (naive approach)."""
|
||||
total = 0
|
||||
for f in files:
|
||||
fp = repo_path / f
|
||||
if fp.is_file():
|
||||
try:
|
||||
total += _count_tokens(fp.read_text(encoding="utf-8", errors="replace"))
|
||||
except OSError:
|
||||
pass
|
||||
return total
|
||||
|
||||
|
||||
def _count_diff_tokens(repo_path: Path, sha: str) -> int:
|
||||
"""Count tokens from git diff output (standard approach)."""
|
||||
result = subprocess.run(
|
||||
["git", "diff", f"{sha}~1", sha],
|
||||
cwd=str(repo_path),
|
||||
capture_output=True,
|
||||
text=True,
|
||||
)
|
||||
if result.returncode != 0:
|
||||
result = subprocess.run(
|
||||
["git", "diff", "HEAD~1", "HEAD"],
|
||||
cwd=str(repo_path),
|
||||
capture_output=True,
|
||||
text=True,
|
||||
)
|
||||
return _count_tokens(result.stdout)
|
||||
|
||||
|
||||
def run(repo_path: Path, store, config: dict) -> list[dict]:
|
||||
"""Run token efficiency benchmark."""
|
||||
results = []
|
||||
for tc in config.get("test_commits", []):
|
||||
changed = _get_changed_files(repo_path, tc["sha"])
|
||||
if not changed:
|
||||
continue
|
||||
|
||||
naive_tokens = _count_file_tokens(repo_path, changed)
|
||||
standard_tokens = _count_diff_tokens(repo_path, tc["sha"])
|
||||
|
||||
row: dict = {
|
||||
"repo": config["name"],
|
||||
"commit": tc["sha"],
|
||||
"description": tc.get("description", ""),
|
||||
"changed_files": len(changed),
|
||||
"naive_tokens": naive_tokens,
|
||||
"standard_tokens": standard_tokens,
|
||||
"graph_tokens": "",
|
||||
"naive_to_graph_ratio": "",
|
||||
"standard_to_graph_ratio": "",
|
||||
"status": "ok",
|
||||
"error": "",
|
||||
}
|
||||
|
||||
# Graph-based: use get_review_context
|
||||
try:
|
||||
from code_review_graph.tools import get_review_context
|
||||
ctx = get_review_context(
|
||||
changed_files=changed, repo_root=str(repo_path)
|
||||
)
|
||||
graph_tokens = _count_tokens(json.dumps(ctx))
|
||||
except Exception as exc:
|
||||
# A failed tool call is not a measurement. Recording
|
||||
# graph_tokens=0 used to turn this into ratio = naive/1 — a
|
||||
# huge fake win. Mark the row failed; aggregate() excludes it.
|
||||
logger.warning("get_review_context failed on %s: %s", tc["sha"], exc)
|
||||
row["status"] = "error"
|
||||
row["error"] = str(exc)[:200]
|
||||
results.append(row)
|
||||
continue
|
||||
|
||||
row["graph_tokens"] = graph_tokens
|
||||
row["naive_to_graph_ratio"] = round(naive_tokens / max(graph_tokens, 1), 1)
|
||||
row["standard_to_graph_ratio"] = round(standard_tokens / max(graph_tokens, 1), 1)
|
||||
results.append(row)
|
||||
return results
|
||||
|
||||
|
||||
def aggregate(results: list[dict]) -> dict:
|
||||
"""Aggregate token-efficiency rows, excluding failed measurements.
|
||||
|
||||
Rows with ``status != "ok"`` stay in the CSV for forensics but must not
|
||||
contribute to any headline number.
|
||||
"""
|
||||
ok = [r for r in results if r.get("status") == "ok"]
|
||||
ratios = [float(r["naive_to_graph_ratio"]) for r in ok]
|
||||
return {
|
||||
"total_rows": len(results),
|
||||
"ok_rows": len(ok),
|
||||
"error_rows": sum(1 for r in results if r.get("status") == "error"),
|
||||
"median_naive_to_graph_ratio": (
|
||||
round(statistics.median(ratios), 1) if ratios else None
|
||||
),
|
||||
"mean_naive_to_graph_ratio": (
|
||||
round(statistics.mean(ratios), 1) if ratios else None
|
||||
),
|
||||
}
|
||||
@@ -0,0 +1,50 @@
|
||||
name: code-review-graph
|
||||
url: https://github.com/tirth8205/code-review-graph
|
||||
# Pinned to the latest test_commit SHA so the snapshot is deterministic and
|
||||
# every test_commit below is reachable as an ancestor. (This config replaces
|
||||
# the historical "nextjs" entry, which used the same URL but mis-labelled the
|
||||
# target as a Next.js monorepo.)
|
||||
commit: 84bde35459c52e1e0c4b25c6c4799743021e0fc7
|
||||
language: python
|
||||
size_category: medium
|
||||
|
||||
test_commits:
|
||||
- sha: 528801f841e519567ef54d6e52e9b9831d162e1b
|
||||
description: "feat: add multi-platform MCP server installation support"
|
||||
changed_files: 3
|
||||
- sha: 84bde35459c52e1e0c4b25c6c4799743021e0fc7
|
||||
description: "feat: add Google Antigravity platform support for MCP install"
|
||||
changed_files: 2
|
||||
|
||||
entry_points:
|
||||
- "code_review_graph/cli.py::cli"
|
||||
- "code_review_graph/main.py::main"
|
||||
|
||||
search_queries:
|
||||
- query: "GraphStore nodes"
|
||||
expected: "code_review_graph/graph.py::GraphStore"
|
||||
- query: "parse AST"
|
||||
expected: "code_review_graph/parser.py::CodeParser"
|
||||
- query: "full build"
|
||||
expected: "code_review_graph/incremental.py::full_build"
|
||||
|
||||
multi_hop_tasks:
|
||||
- id: crg-parse-file-callers
|
||||
nl_query: "Who invokes the parser entry point on a single source file"
|
||||
anchor_qualified_suffix: "code_review_graph/parser.py::codeparser.parse_file"
|
||||
traversal_pattern: callers_of
|
||||
expected_neighbor_names: ["setup_method"]
|
||||
k: 10
|
||||
- id: crg-upsert-node-callers
|
||||
nl_query: "Where the graph store inserts or updates a node"
|
||||
anchor_qualified_suffix: "code_review_graph/graph.py::graphstore.upsert_node"
|
||||
traversal_pattern: callers_of
|
||||
expected_neighbor_names: ["store_file_nodes_edges"]
|
||||
k: 10
|
||||
|
||||
# Questions for the agent_baseline benchmark (pure-python grep top-k vs graph
|
||||
# query). See docs/REPRODUCING.md for the methodology.
|
||||
agent_questions:
|
||||
- "How does GraphStore upsert_node store a node"
|
||||
- "Where does full_build parse the repository"
|
||||
- "How does hybrid_search rank search results"
|
||||
@@ -0,0 +1,45 @@
|
||||
name: express
|
||||
url: https://github.com/expressjs/express
|
||||
# Pinned to the latest test_commit SHA so the snapshot is deterministic and
|
||||
# every test_commit below is reachable as an ancestor.
|
||||
commit: b4ab7d65d7724d9309b6faaaf82ad492da2a6d35
|
||||
language: javascript
|
||||
size_category: small
|
||||
|
||||
test_commits:
|
||||
- sha: 925a1dff1e42f1b393c977b8b77757fcf633e09f
|
||||
description: "fix: bump qs minimum to ^6.14.2 for CVE-2026-2391"
|
||||
changed_files: 1
|
||||
- sha: b4ab7d65d7724d9309b6faaaf82ad492da2a6d35
|
||||
description: "test: include edge case tests for res.type()"
|
||||
changed_files: 1
|
||||
|
||||
entry_points:
|
||||
- "lib/application.js::app.handle"
|
||||
- "lib/express.js::createApplication"
|
||||
|
||||
search_queries:
|
||||
- query: "app handle"
|
||||
expected: "lib/application.js::app"
|
||||
- query: "response send"
|
||||
expected: "lib/response.js::res"
|
||||
- query: "request"
|
||||
expected: "lib/request.js::req"
|
||||
|
||||
# Express has only one task — JS modules use prototypes + module.exports
|
||||
# heavily, so most "method" callers are not represented as proper Function
|
||||
# edges in the graph. createApplication is the cleanest anchor.
|
||||
multi_hop_tasks:
|
||||
- id: express-create-application-callees
|
||||
nl_query: "What express does when constructing an application"
|
||||
anchor_qualified_suffix: "lib/express.js::createapplication"
|
||||
traversal_pattern: callees_of
|
||||
expected_neighbor_names: ["mixin", "create", "init"]
|
||||
k: 10
|
||||
|
||||
# Questions for the agent_baseline benchmark (pure-python grep top-k vs graph
|
||||
# query). See docs/REPRODUCING.md for the methodology.
|
||||
agent_questions:
|
||||
- "How does app.handle process the middleware stack"
|
||||
- "Where does res.send write the response body"
|
||||
- "How does createApplication initialize an app"
|
||||
@@ -0,0 +1,48 @@
|
||||
name: fastapi
|
||||
url: https://github.com/tiangolo/fastapi
|
||||
# Pinned to the latest test_commit SHA so the snapshot is deterministic and
|
||||
# every test_commit below is reachable as an ancestor.
|
||||
commit: 0227991a01e61bf5cdd93cc00e9e243f52b47a4a
|
||||
language: python
|
||||
size_category: medium
|
||||
|
||||
test_commits:
|
||||
- sha: fa3588c38c7473aca7536b12d686102de4b0f407
|
||||
description: "Fix typo for client_secret in OAuth2 form docstrings"
|
||||
changed_files: 1
|
||||
- sha: 0227991a01e61bf5cdd93cc00e9e243f52b47a4a
|
||||
description: "Exclude spam comments from statistics in scripts/people.py"
|
||||
changed_files: 1
|
||||
|
||||
entry_points:
|
||||
- "fastapi/applications.py::FastAPI"
|
||||
- "fastapi/routing.py::APIRouter"
|
||||
|
||||
search_queries:
|
||||
- query: "FastAPI application"
|
||||
expected: "fastapi/applications.py::FastAPI"
|
||||
- query: "APIRoute routing"
|
||||
expected: "fastapi/routing.py::APIRoute"
|
||||
- query: "Depends injection"
|
||||
expected: "fastapi/params.py::Depends"
|
||||
|
||||
multi_hop_tasks:
|
||||
- id: fastapi-route-handler-callers
|
||||
nl_query: "How fastapi binds a route handler to an APIRoute"
|
||||
anchor_qualified_suffix: "fastapi/routing.py::apiroute.get_route_handler"
|
||||
traversal_pattern: callers_of
|
||||
expected_neighbor_names: ["__init__"]
|
||||
k: 10
|
||||
- id: fastapi-get-dependant-callers
|
||||
nl_query: "Where fastapi resolves dependency declarations into a tree"
|
||||
anchor_qualified_suffix: "fastapi/dependencies/utils.py::get_dependant"
|
||||
traversal_pattern: callers_of
|
||||
expected_neighbor_names: ["get_parameterless_sub_dependant", "solve_dependencies"]
|
||||
k: 10
|
||||
|
||||
# Questions for the agent_baseline benchmark (pure-python grep top-k vs graph
|
||||
# query). See docs/REPRODUCING.md for the methodology.
|
||||
agent_questions:
|
||||
- "How does include_router register routes on the application"
|
||||
- "Where does APIRoute build its route handler"
|
||||
- "How does solve_dependencies resolve Depends parameters"
|
||||
@@ -0,0 +1,50 @@
|
||||
name: flask
|
||||
url: https://github.com/pallets/flask
|
||||
# Pinned to the latest test_commit SHA so the snapshot is deterministic and
|
||||
# every test_commit below is reachable as an ancestor.
|
||||
commit: a29f88ce6f2f9843bd6fcbbfce1390a2071965d6
|
||||
language: python
|
||||
size_category: small
|
||||
|
||||
test_commits:
|
||||
- sha: fbb6f0bc4c60a0bada0e03c3480d0ccf30a3c1df
|
||||
description: "all teardown callbacks are called despite errors"
|
||||
changed_files: 10
|
||||
- sha: a29f88ce6f2f9843bd6fcbbfce1390a2071965d6
|
||||
description: "document that headers must be set before streaming"
|
||||
changed_files: 4
|
||||
|
||||
entry_points:
|
||||
- "src/flask/app.py::Flask.wsgi_app"
|
||||
- "src/flask/sansio/app.py::App.add_url_rule"
|
||||
|
||||
search_queries:
|
||||
- query: "Flask wsgi"
|
||||
expected: "src/flask/app.py::Flask"
|
||||
- query: "AppContext globals"
|
||||
expected: "src/flask/ctx.py::AppContext"
|
||||
- query: "create logger"
|
||||
expected: "src/flask/logging.py::create_logger"
|
||||
|
||||
# Multi-hop retrieval tasks (semantic_search → query_graph one-hop)
|
||||
# See docs/REPRODUCING.md for the schema.
|
||||
multi_hop_tasks:
|
||||
- id: flask-dispatch-callers
|
||||
nl_query: "Where Flask dispatches HTTP requests"
|
||||
anchor_qualified_suffix: "src/flask/app.py::flask.dispatch_request"
|
||||
traversal_pattern: callers_of
|
||||
expected_neighbor_names: ["full_dispatch_request"]
|
||||
k: 10
|
||||
- id: flask-exception-callers
|
||||
nl_query: "Where Flask handles uncaught exceptions"
|
||||
anchor_qualified_suffix: "src/flask/app.py::flask.handle_exception"
|
||||
traversal_pattern: callers_of
|
||||
expected_neighbor_names: ["wsgi_app"]
|
||||
k: 10
|
||||
|
||||
# Questions for the agent_baseline benchmark (pure-python grep top-k vs graph
|
||||
# query). See docs/REPRODUCING.md for the methodology.
|
||||
agent_questions:
|
||||
- "How does dispatch_request route an incoming HTTP request"
|
||||
- "Where is the AppContext pushed and popped"
|
||||
- "How does create_logger configure application logging"
|
||||
@@ -0,0 +1,51 @@
|
||||
name: gin
|
||||
url: https://github.com/gin-gonic/gin
|
||||
# Pinned to the latest test_commit SHA so the snapshot is deterministic and
|
||||
# every test_commit below is reachable as an ancestor.
|
||||
commit: 5c00df8afadd06cc5be530dde00fe6d9fa4a2e4a
|
||||
language: go
|
||||
size_category: small
|
||||
|
||||
test_commits:
|
||||
- sha: 052d1a79aafe3f04078a2716f8e77d4340308383
|
||||
description: "feat(render): add PDF renderer and tests"
|
||||
changed_files: 5
|
||||
- sha: 472d086af2acd924cb4b9d7be0525f7d790f69bc
|
||||
description: "fix(tree): panic in findCaseInsensitivePathRec with RedirectFixedPath"
|
||||
changed_files: 2
|
||||
- sha: 5c00df8afadd06cc5be530dde00fe6d9fa4a2e4a
|
||||
description: "fix(render): write content length in Data.Render"
|
||||
changed_files: 2
|
||||
|
||||
entry_points:
|
||||
- "gin.go::Engine"
|
||||
- "routergroup.go::RouterGroup"
|
||||
|
||||
search_queries:
|
||||
- query: "Engine ServeHTTP"
|
||||
expected: "gin.go::Engine"
|
||||
- query: "Context request"
|
||||
expected: "context.go::Context"
|
||||
- query: "node tree"
|
||||
expected: "tree.go::node"
|
||||
|
||||
multi_hop_tasks:
|
||||
- id: gin-serve-http-callees
|
||||
nl_query: "What does the gin engine do when serving an HTTP request"
|
||||
anchor_qualified_suffix: "gin.go::engine.servehttp"
|
||||
traversal_pattern: callees_of
|
||||
expected_neighbor_names: ["reset"]
|
||||
k: 10
|
||||
- id: gin-context-next-callers
|
||||
nl_query: "Who advances the gin middleware chain via Context.Next"
|
||||
anchor_qualified_suffix: "context.go::context.next"
|
||||
traversal_pattern: callers_of
|
||||
expected_neighbor_names: ["handleHTTPRequest", "serveError"]
|
||||
k: 10
|
||||
|
||||
# Questions for the agent_baseline benchmark (pure-python grep top-k vs graph
|
||||
# query). See docs/REPRODUCING.md for the methodology.
|
||||
agent_questions:
|
||||
- "How does Engine.ServeHTTP route an incoming request"
|
||||
- "Where does Context.Next advance the middleware chain"
|
||||
- "How does the node tree match wildcard routes"
|
||||
@@ -0,0 +1,48 @@
|
||||
name: httpx
|
||||
url: https://github.com/encode/httpx
|
||||
# Pinned to the latest test_commit SHA so the snapshot is deterministic and
|
||||
# every test_commit below is reachable as an ancestor.
|
||||
commit: b55d4635701d9dc22928ee647880c76b078ba3f2
|
||||
language: python
|
||||
size_category: small
|
||||
|
||||
test_commits:
|
||||
- sha: ae1b9f66238f75ced3ced5e4485408435de10768
|
||||
description: "Expose FunctionAuth in __all__"
|
||||
changed_files: 3
|
||||
- sha: b55d4635701d9dc22928ee647880c76b078ba3f2
|
||||
description: "Upgrade Python type checker mypy"
|
||||
changed_files: 4
|
||||
|
||||
entry_points:
|
||||
- "httpx/_client.py::Client"
|
||||
- "httpx/_client.py::AsyncClient"
|
||||
|
||||
search_queries:
|
||||
- query: "Client request"
|
||||
expected: "httpx/_client.py::Client"
|
||||
- query: "Response headers"
|
||||
expected: "httpx/_models.py::Response"
|
||||
- query: "BaseClient"
|
||||
expected: "httpx/_client.py::BaseClient"
|
||||
|
||||
multi_hop_tasks:
|
||||
- id: httpx-client-request-callers
|
||||
nl_query: "Which HTTP verbs route through the httpx Client.request"
|
||||
anchor_qualified_suffix: "httpx/_client.py::client.request"
|
||||
traversal_pattern: callers_of
|
||||
expected_neighbor_names: ["get", "options", "head", "post", "put", "patch"]
|
||||
k: 10
|
||||
- id: httpx-async-request-tests
|
||||
nl_query: "Tests covering the httpx async client request method"
|
||||
anchor_qualified_suffix: "httpx/_client.py::asyncclient.request"
|
||||
traversal_pattern: callers_of
|
||||
expected_neighbor_names: ["test_raise_for_status"]
|
||||
k: 10
|
||||
|
||||
# Questions for the agent_baseline benchmark (pure-python grep top-k vs graph
|
||||
# query). See docs/REPRODUCING.md for the methodology.
|
||||
agent_questions:
|
||||
- "How does Client.request send an HTTP request"
|
||||
- "Where are Response headers parsed and decoded"
|
||||
- "How does BaseClient build request URLs"
|
||||
@@ -0,0 +1,301 @@
|
||||
"""Markdown report generator for evaluation benchmark results.
|
||||
|
||||
Takes a list of benchmark result dicts and produces a formatted markdown table
|
||||
suitable for inclusion in documentation or CI output.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import csv
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
|
||||
def generate_markdown_report(results: list[dict[str, Any]]) -> str:
|
||||
"""Generate a markdown report from benchmark results.
|
||||
|
||||
Each result dict should contain at minimum a ``benchmark`` key identifying
|
||||
the benchmark name, plus any metric keys (e.g. ``ratio``,
|
||||
``reduction_percent``, ``mrr``, ``precision``, ``recall``, ``f1``).
|
||||
|
||||
Args:
|
||||
results: List of result dicts from benchmark runs.
|
||||
|
||||
Returns:
|
||||
A markdown string containing a summary table and per-benchmark details.
|
||||
"""
|
||||
if not results:
|
||||
return "# Evaluation Report\n\nNo benchmark results to report.\n"
|
||||
|
||||
lines: list[str] = []
|
||||
lines.append("# Evaluation Report")
|
||||
lines.append("")
|
||||
|
||||
# Collect all metric keys across results (excluding 'benchmark')
|
||||
all_keys: list[str] = []
|
||||
seen: set[str] = set()
|
||||
for r in results:
|
||||
for k in r:
|
||||
if k != "benchmark" and k not in seen:
|
||||
all_keys.append(k)
|
||||
seen.add(k)
|
||||
|
||||
# Summary table
|
||||
lines.append("## Summary")
|
||||
lines.append("")
|
||||
|
||||
header = "| Benchmark | " + " | ".join(all_keys) + " |"
|
||||
separator = "| --- | " + " | ".join("---" for _ in all_keys) + " |"
|
||||
lines.append(header)
|
||||
lines.append(separator)
|
||||
|
||||
for r in results:
|
||||
name = r.get("benchmark", "unknown")
|
||||
values = [str(r.get(k, "-")) for k in all_keys]
|
||||
lines.append(f"| {name} | " + " | ".join(values) + " |")
|
||||
|
||||
lines.append("")
|
||||
|
||||
# Per-benchmark detail sections
|
||||
lines.append("## Details")
|
||||
lines.append("")
|
||||
for r in results:
|
||||
name = r.get("benchmark", "unknown")
|
||||
lines.append(f"### {name}")
|
||||
lines.append("")
|
||||
for k in all_keys:
|
||||
v = r.get(k, "-")
|
||||
lines.append(f"- **{k}**: {v}")
|
||||
lines.append("")
|
||||
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
def _read_csvs(results_dir: Path, prefix: str) -> list[dict[str, str]]:
|
||||
"""Read all CSV files matching a prefix from the results directory."""
|
||||
rows: list[dict[str, str]] = []
|
||||
for p in sorted(results_dir.glob(f"*_{prefix}_*.csv")):
|
||||
with open(p, newline="") as f:
|
||||
reader = csv.DictReader(f)
|
||||
rows.extend(reader)
|
||||
return rows
|
||||
|
||||
|
||||
def _md_table(headers: list[str], rows: list[list[str]]) -> str:
|
||||
"""Build a markdown table from headers and rows."""
|
||||
lines = []
|
||||
lines.append("| " + " | ".join(headers) + " |")
|
||||
lines.append("| " + " | ".join("---" for _ in headers) + " |")
|
||||
for row in rows:
|
||||
lines.append("| " + " | ".join(row) + " |")
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
def generate_full_report(results_dir: str | Path) -> str:
|
||||
"""Generate a full markdown evaluation report from CSV result files.
|
||||
|
||||
Reads all CSV files in *results_dir*, groups them by benchmark type,
|
||||
and produces a markdown report with methodology notes and per-benchmark
|
||||
result tables.
|
||||
|
||||
Args:
|
||||
results_dir: Directory containing CSV result files.
|
||||
|
||||
Returns:
|
||||
Markdown string with the full report.
|
||||
"""
|
||||
results_dir = Path(results_dir)
|
||||
lines: list[str] = []
|
||||
lines.append("# Evaluation Report")
|
||||
lines.append("")
|
||||
lines.append("## Methodology")
|
||||
lines.append("")
|
||||
lines.append("Benchmarks are run against real open-source repositories.")
|
||||
lines.append("Token counts use a consistent `len(text) // 4` approximation.")
|
||||
lines.append(
|
||||
"Impact accuracy reports two ground-truth modes: "
|
||||
"graph-derived (circular — upper bound) and co-change "
|
||||
"(files co-changed in the same commit, seed excluded)."
|
||||
)
|
||||
lines.append(
|
||||
"Rows with `status=error` are kept for forensics but excluded "
|
||||
"from all aggregates."
|
||||
)
|
||||
lines.append("")
|
||||
|
||||
benchmark_types = [
|
||||
"token_efficiency",
|
||||
"impact_accuracy",
|
||||
"agent_baseline",
|
||||
"flow_completeness",
|
||||
"search_quality",
|
||||
"build_performance",
|
||||
"multi_hop_retrieval",
|
||||
]
|
||||
|
||||
for btype in benchmark_types:
|
||||
rows = _read_csvs(results_dir, btype)
|
||||
if not rows:
|
||||
continue
|
||||
|
||||
title = btype.replace("_", " ").title()
|
||||
lines.append(f"## {title}")
|
||||
lines.append("")
|
||||
|
||||
headers = list(rows[0].keys())
|
||||
table_rows = [[r.get(h, "-") for h in headers] for r in rows]
|
||||
lines.append(_md_table(headers, table_rows))
|
||||
lines.append("")
|
||||
|
||||
if len(lines) <= 6:
|
||||
lines.append("No benchmark results found.")
|
||||
lines.append("")
|
||||
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
def generate_readme_tables(results_dir: str | Path) -> str:
|
||||
"""Generate concise README-ready tables from CSV result files.
|
||||
|
||||
Produces three tables:
|
||||
- Table A: Token Efficiency
|
||||
- Table B: Accuracy & Quality
|
||||
- Table C: Performance
|
||||
|
||||
Args:
|
||||
results_dir: Directory containing CSV result files.
|
||||
|
||||
Returns:
|
||||
Markdown string with the three tables.
|
||||
"""
|
||||
results_dir = Path(results_dir)
|
||||
lines: list[str] = []
|
||||
|
||||
# Table A: Token Efficiency
|
||||
te_rows = _read_csvs(results_dir, "token_efficiency")
|
||||
if te_rows:
|
||||
lines.append("### Token Efficiency")
|
||||
lines.append("")
|
||||
headers = [
|
||||
"Repo", "Files", "Naive Tokens", "Standard Tokens",
|
||||
"Graph Tokens", "Naive/Graph", "Std/Graph",
|
||||
]
|
||||
table_rows = []
|
||||
for r in te_rows:
|
||||
table_rows.append([
|
||||
r.get("repo", "-"),
|
||||
r.get("changed_files", "-"),
|
||||
r.get("naive_tokens", "-"),
|
||||
r.get("standard_tokens", "-"),
|
||||
r.get("graph_tokens", "-"),
|
||||
r.get("naive_to_graph_ratio", "-"),
|
||||
r.get("standard_to_graph_ratio", "-"),
|
||||
])
|
||||
lines.append(_md_table(headers, table_rows))
|
||||
lines.append("")
|
||||
|
||||
# Table B: Accuracy & Quality
|
||||
ia_rows = _read_csvs(results_dir, "impact_accuracy")
|
||||
fc_rows = _read_csvs(results_dir, "flow_completeness")
|
||||
sq_rows = _read_csvs(results_dir, "search_quality")
|
||||
|
||||
if ia_rows or fc_rows or sq_rows:
|
||||
lines.append("### Accuracy & Quality")
|
||||
lines.append("")
|
||||
headers = ["Repo", "Impact F1 (graph-derived)", "Flow Recall", "Search MRR"]
|
||||
# Build a per-repo summary
|
||||
repo_data: dict[str, dict[str, object]] = {}
|
||||
mrr_accum: dict[str, list[float]] = {}
|
||||
f1_accum: dict[str, list[float]] = {}
|
||||
for r in ia_rows:
|
||||
# Failed rows are kept in the CSV for forensics but must never
|
||||
# contribute to a headline number; co-change rows are a
|
||||
# different metric and get their own reporting.
|
||||
if r.get("status", "ok") not in ("", "ok"):
|
||||
continue
|
||||
mode = r.get("ground_truth_mode", "")
|
||||
if mode and not mode.startswith("graph-derived"):
|
||||
continue
|
||||
repo = r.get("repo", "?")
|
||||
repo_data.setdefault(repo, {})
|
||||
try:
|
||||
f1_accum.setdefault(repo, []).append(float(r.get("f1", "")))
|
||||
except (ValueError, TypeError):
|
||||
pass
|
||||
for r in fc_rows:
|
||||
repo_data.setdefault(r.get("repo", "?"), {})["recall"] = r.get("recall", "-")
|
||||
for r in sq_rows:
|
||||
repo = r.get("repo", "?")
|
||||
repo_data.setdefault(repo, {})
|
||||
try:
|
||||
mrr_accum.setdefault(repo, []).append(float(r.get("reciprocal_rank", 0)))
|
||||
except (ValueError, TypeError):
|
||||
pass
|
||||
|
||||
table_rows = []
|
||||
for repo, d in sorted(repo_data.items()):
|
||||
mrr_vals = mrr_accum.get(repo, [])
|
||||
mrr = (
|
||||
str(round(sum(mrr_vals) / len(mrr_vals), 3))
|
||||
if mrr_vals
|
||||
else "-"
|
||||
)
|
||||
f1_vals = f1_accum.get(repo, [])
|
||||
f1 = (
|
||||
str(round(sum(f1_vals) / len(f1_vals), 3))
|
||||
if f1_vals
|
||||
else "-"
|
||||
)
|
||||
table_rows.append([
|
||||
repo,
|
||||
f1,
|
||||
str(d.get("recall", "-")),
|
||||
mrr,
|
||||
])
|
||||
lines.append(_md_table(headers, table_rows))
|
||||
lines.append("")
|
||||
|
||||
# Table B2: Agent Baseline (grep top-k vs graph query)
|
||||
ab_rows = _read_csvs(results_dir, "agent_baseline")
|
||||
if ab_rows:
|
||||
lines.append("### Agent Baseline (grep top-k vs graph query)")
|
||||
lines.append("")
|
||||
headers = [
|
||||
"Repo", "Question", "Baseline Tokens", "Graph Tokens",
|
||||
"Baseline/Graph", "Status",
|
||||
]
|
||||
table_rows = []
|
||||
for r in ab_rows:
|
||||
table_rows.append([
|
||||
r.get("repo", "-"),
|
||||
r.get("question", "-"),
|
||||
r.get("baseline_tokens", "-"),
|
||||
r.get("graph_tokens", "-"),
|
||||
r.get("baseline_to_graph_ratio", "-"),
|
||||
r.get("status", "ok") or "ok",
|
||||
])
|
||||
lines.append(_md_table(headers, table_rows))
|
||||
lines.append("")
|
||||
|
||||
# Table C: Performance
|
||||
bp_rows = _read_csvs(results_dir, "build_performance")
|
||||
if bp_rows:
|
||||
lines.append("### Performance")
|
||||
lines.append("")
|
||||
headers = ["Repo", "Files", "Nodes", "Flow Det. (s)", "Search (ms)"]
|
||||
table_rows = []
|
||||
for r in bp_rows:
|
||||
table_rows.append([
|
||||
r.get("repo", "-"),
|
||||
r.get("file_count", "-"),
|
||||
r.get("node_count", "-"),
|
||||
r.get("flow_detection_seconds", "-"),
|
||||
r.get("search_avg_ms", "-"),
|
||||
])
|
||||
lines.append(_md_table(headers, table_rows))
|
||||
lines.append("")
|
||||
|
||||
if not lines:
|
||||
return "No benchmark results found.\n"
|
||||
|
||||
return "\n".join(lines)
|
||||
@@ -0,0 +1,211 @@
|
||||
"""Evaluation runner: orchestrates benchmark execution across repositories."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import csv
|
||||
import logging
|
||||
import subprocess
|
||||
from datetime import date
|
||||
from pathlib import Path
|
||||
|
||||
try:
|
||||
import yaml # type: ignore[import-untyped]
|
||||
except ImportError:
|
||||
yaml = None # type: ignore[assignment]
|
||||
|
||||
from code_review_graph.eval.benchmarks import (
|
||||
agent_baseline,
|
||||
build_performance,
|
||||
flow_completeness,
|
||||
impact_accuracy,
|
||||
multi_hop_retrieval,
|
||||
search_quality,
|
||||
token_efficiency,
|
||||
)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
BENCHMARK_REGISTRY = {
|
||||
"token_efficiency": token_efficiency.run,
|
||||
"impact_accuracy": impact_accuracy.run,
|
||||
"flow_completeness": flow_completeness.run,
|
||||
"search_quality": search_quality.run,
|
||||
"build_performance": build_performance.run,
|
||||
"multi_hop_retrieval": multi_hop_retrieval.run,
|
||||
"agent_baseline": agent_baseline.run,
|
||||
}
|
||||
|
||||
CONFIGS_DIR = Path(__file__).parent / "configs"
|
||||
DEFAULT_OUTPUT = Path("evaluate/results")
|
||||
DEFAULT_REPOS = Path("evaluate/test_repos")
|
||||
|
||||
|
||||
def _require_yaml():
|
||||
if yaml is None:
|
||||
raise ImportError("pyyaml is required: pip install code-review-graph[eval]")
|
||||
|
||||
|
||||
def load_config(name: str) -> dict:
|
||||
"""Load a single benchmark config by name."""
|
||||
_require_yaml()
|
||||
path = CONFIGS_DIR / f"{name}.yaml"
|
||||
with open(path) as f:
|
||||
return yaml.safe_load(f)
|
||||
|
||||
|
||||
def load_all_configs() -> list[dict]:
|
||||
"""Load all benchmark configs from the configs directory."""
|
||||
_require_yaml()
|
||||
configs = []
|
||||
for p in sorted(CONFIGS_DIR.glob("*.yaml")):
|
||||
with open(p) as f:
|
||||
configs.append(yaml.safe_load(f))
|
||||
return configs
|
||||
|
||||
|
||||
def clone_or_update(config: dict, repos_dir: Path | None = None) -> Path:
|
||||
"""Clone or update a repository at the config's pinned ``commit`` SHA.
|
||||
|
||||
Full clones (no ``--depth``) are required: the pinned ``test_commits`` are
|
||||
often older than any reasonable shallow-clone window, and a missed SHA
|
||||
used to silently fall back to ``git diff HEAD~1 HEAD`` — producing
|
||||
benchmark numbers tied to whatever upstream HEAD looked like that day.
|
||||
|
||||
Every subprocess call's exit status is checked; failures raise
|
||||
``RuntimeError`` so reproducibility issues surface immediately instead of
|
||||
yielding garbage results.
|
||||
"""
|
||||
repos_dir = repos_dir or DEFAULT_REPOS
|
||||
repos_dir.mkdir(parents=True, exist_ok=True)
|
||||
repo_path = repos_dir / config["name"]
|
||||
|
||||
if repo_path.exists():
|
||||
proc = subprocess.run(
|
||||
["git", "fetch", "--all", "--tags"],
|
||||
cwd=str(repo_path),
|
||||
capture_output=True,
|
||||
text=True,
|
||||
)
|
||||
if proc.returncode != 0:
|
||||
raise RuntimeError(
|
||||
f"git fetch failed in {repo_path}: {proc.stderr.strip()}"
|
||||
)
|
||||
else:
|
||||
proc = subprocess.run(
|
||||
["git", "clone", config["url"], str(repo_path)],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
)
|
||||
if proc.returncode != 0:
|
||||
raise RuntimeError(
|
||||
f"git clone failed for {config['url']}: {proc.stderr.strip()}"
|
||||
)
|
||||
|
||||
commit = config.get("commit", "HEAD")
|
||||
if commit != "HEAD":
|
||||
proc = subprocess.run(
|
||||
["git", "checkout", commit],
|
||||
cwd=str(repo_path),
|
||||
capture_output=True,
|
||||
text=True,
|
||||
)
|
||||
if proc.returncode != 0:
|
||||
raise RuntimeError(
|
||||
f"git checkout {commit} failed in {repo_path}: "
|
||||
f"{proc.stderr.strip()}"
|
||||
)
|
||||
|
||||
return repo_path
|
||||
|
||||
|
||||
def write_csv(results: list[dict], path: Path) -> None:
|
||||
"""Write benchmark results to a CSV file."""
|
||||
if not results:
|
||||
return
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
fieldnames = list(results[0].keys())
|
||||
with open(path, "w", newline="") as f:
|
||||
writer = csv.DictWriter(f, fieldnames=fieldnames)
|
||||
writer.writeheader()
|
||||
writer.writerows(results)
|
||||
|
||||
|
||||
def run_eval(
|
||||
repos: list[str] | None = None,
|
||||
benchmarks: list[str] | None = None,
|
||||
output_dir: str | Path | None = None,
|
||||
) -> dict[str, list[dict]]:
|
||||
"""Run evaluation benchmarks across repositories.
|
||||
|
||||
Args:
|
||||
repos: List of repo config names to evaluate (None = all).
|
||||
benchmarks: List of benchmark names to run (None = all).
|
||||
output_dir: Directory for CSV output files.
|
||||
|
||||
Returns:
|
||||
Dict mapping ``{repo}_{benchmark}`` to list of result dicts.
|
||||
"""
|
||||
output_dir = Path(output_dir) if output_dir else DEFAULT_OUTPUT
|
||||
output_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
if repos:
|
||||
configs = [load_config(r) for r in repos]
|
||||
else:
|
||||
configs = load_all_configs()
|
||||
|
||||
benchmark_names = benchmarks or list(BENCHMARK_REGISTRY.keys())
|
||||
all_results: dict[str, list[dict]] = {}
|
||||
today = date.today().isoformat()
|
||||
|
||||
for config in configs:
|
||||
name = config["name"]
|
||||
logger.info("Evaluating %s...", name)
|
||||
|
||||
# Resolve the repo path to an absolute Path before handing it to
|
||||
# full_build / get_db_path so the stored qualified_names match what
|
||||
# the CLI/MCP layer produces (those paths go through _get_store ->
|
||||
# _validate_repo_root which .resolve()s). Without this, a later
|
||||
# ``code-review-graph update --repo <relative>`` writes the same
|
||||
# function under a new absolute-prefixed qualified_name, leaving the
|
||||
# graph with duplicate nodes for the same source location.
|
||||
repo_path = clone_or_update(config).resolve()
|
||||
|
||||
# Build graph
|
||||
from code_review_graph.graph import GraphStore
|
||||
from code_review_graph.incremental import full_build, get_db_path
|
||||
from code_review_graph.postprocessing import run_post_processing
|
||||
|
||||
db_path = get_db_path(repo_path)
|
||||
store = GraphStore(db_path)
|
||||
|
||||
full_build(repo_path, store)
|
||||
# full_build is the parsing-only primitive; the higher-level CLI/MCP
|
||||
# wrappers run postprocessing on top. The eval framework bypasses
|
||||
# those, so call it directly here. Without this, FTS5 stays empty
|
||||
# and downstream benchmarks (token_efficiency, search_quality)
|
||||
# silently produce useless results. See: search.rebuild_fts_index.
|
||||
pp_result = run_post_processing(store)
|
||||
for warning in pp_result.get("warnings", []):
|
||||
logger.warning(" postprocessing: %s", warning)
|
||||
|
||||
for bench_name in benchmark_names:
|
||||
if bench_name not in BENCHMARK_REGISTRY:
|
||||
logger.warning("Unknown benchmark: %s", bench_name)
|
||||
continue
|
||||
|
||||
logger.info(" Running %s...", bench_name)
|
||||
try:
|
||||
bench_fn = BENCHMARK_REGISTRY[bench_name]
|
||||
results = bench_fn(repo_path, store, config)
|
||||
|
||||
key = f"{name}_{bench_name}"
|
||||
all_results[key] = results
|
||||
write_csv(results, output_dir / f"{key}_{today}.csv")
|
||||
logger.info(" %s: %d result(s)", bench_name, len(results))
|
||||
except Exception as e:
|
||||
logger.error(" %s failed: %s", bench_name, e)
|
||||
all_results[f"{name}_{bench_name}"] = []
|
||||
|
||||
store.close()
|
||||
|
||||
return all_results
|
||||
@@ -0,0 +1,85 @@
|
||||
"""Scoring metrics for evaluating graph-based code review quality.
|
||||
|
||||
Provides:
|
||||
- Token efficiency: measures how many tokens the graph saves vs raw context.
|
||||
- Mean Reciprocal Rank (MRR): evaluates ranking quality for search results.
|
||||
- Precision / Recall / F1: evaluates set-based retrieval accuracy.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
|
||||
def compute_token_efficiency(raw_tokens: int, graph_tokens: int) -> dict:
|
||||
"""Compute token efficiency metrics.
|
||||
|
||||
Args:
|
||||
raw_tokens: Number of tokens when sending raw source code.
|
||||
graph_tokens: Number of tokens when using graph-based context.
|
||||
|
||||
Returns:
|
||||
Dict with keys:
|
||||
- raw_tokens: the raw token count
|
||||
- graph_tokens: the graph token count
|
||||
- ratio: graph_tokens / raw_tokens (lower is better)
|
||||
- reduction_percent: percentage of tokens saved (higher is better)
|
||||
"""
|
||||
if raw_tokens <= 0:
|
||||
return {
|
||||
"raw_tokens": raw_tokens,
|
||||
"graph_tokens": graph_tokens,
|
||||
"ratio": 0.0,
|
||||
"reduction_percent": 0.0,
|
||||
}
|
||||
ratio = graph_tokens / raw_tokens
|
||||
reduction = (1.0 - ratio) * 100.0
|
||||
return {
|
||||
"raw_tokens": raw_tokens,
|
||||
"graph_tokens": graph_tokens,
|
||||
"ratio": round(ratio, 4),
|
||||
"reduction_percent": round(reduction, 2),
|
||||
}
|
||||
|
||||
|
||||
def compute_mrr(correct: str, results: list[str]) -> float:
|
||||
"""Compute Mean Reciprocal Rank for a single query.
|
||||
|
||||
Args:
|
||||
correct: The correct/expected result identifier.
|
||||
results: Ordered list of result identifiers (best first).
|
||||
|
||||
Returns:
|
||||
1/rank if *correct* is found in *results*, else 0.0.
|
||||
"""
|
||||
for i, r in enumerate(results, start=1):
|
||||
if r == correct:
|
||||
return 1.0 / i
|
||||
return 0.0
|
||||
|
||||
|
||||
def compute_precision_recall(predicted: set, actual: set) -> dict:
|
||||
"""Compute precision, recall, and F1 score.
|
||||
|
||||
Args:
|
||||
predicted: Set of predicted/returned items.
|
||||
actual: Set of ground-truth items.
|
||||
|
||||
Returns:
|
||||
Dict with keys: precision, recall, f1.
|
||||
"""
|
||||
if not predicted and not actual:
|
||||
return {"precision": 1.0, "recall": 1.0, "f1": 1.0}
|
||||
|
||||
true_positive = len(predicted & actual)
|
||||
precision = true_positive / len(predicted) if predicted else 0.0
|
||||
recall = true_positive / len(actual) if actual else 0.0
|
||||
|
||||
if precision + recall > 0:
|
||||
f1 = 2 * precision * recall / (precision + recall)
|
||||
else:
|
||||
f1 = 0.0
|
||||
|
||||
return {
|
||||
"precision": round(precision, 4),
|
||||
"recall": round(recall, 4),
|
||||
"f1": round(f1, 4),
|
||||
}
|
||||
@@ -0,0 +1,182 @@
|
||||
"""Measures total tokens consumed by agent workflows against benchmark repos."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
from typing import Any, Callable
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def estimate_tokens(obj: Any) -> int:
|
||||
"""Estimate token count from JSON-serializable object.
|
||||
|
||||
Uses character count / 4 as a rough approximation for English + code.
|
||||
"""
|
||||
return len(json.dumps(obj, default=str)) // 4
|
||||
|
||||
|
||||
def benchmark_review_workflow(repo_root: str, base: str = "HEAD~1") -> dict:
|
||||
"""Simulate a review workflow and measure total tokens consumed."""
|
||||
from ..tools.context import get_minimal_context
|
||||
from ..tools.review import detect_changes_func
|
||||
|
||||
total_tokens = 0
|
||||
calls = []
|
||||
|
||||
# Step 1: get_minimal_context
|
||||
result = get_minimal_context(task="review changes", repo_root=repo_root, base=base)
|
||||
tokens = estimate_tokens(result)
|
||||
total_tokens += tokens
|
||||
calls.append({"tool": "get_minimal_context", "tokens": tokens})
|
||||
|
||||
# Step 2: detect_changes (minimal)
|
||||
result = detect_changes_func(base=base, repo_root=repo_root, detail_level="minimal")
|
||||
tokens = estimate_tokens(result)
|
||||
total_tokens += tokens
|
||||
calls.append({"tool": "detect_changes_minimal", "tokens": tokens})
|
||||
|
||||
return {
|
||||
"workflow": "review",
|
||||
"total_tokens": total_tokens,
|
||||
"tool_calls": len(calls),
|
||||
"calls": calls,
|
||||
}
|
||||
|
||||
|
||||
def benchmark_architecture_workflow(repo_root: str) -> dict:
|
||||
"""Simulate an architecture exploration workflow."""
|
||||
from ..tools.community_tools import list_communities_func
|
||||
from ..tools.context import get_minimal_context
|
||||
from ..tools.flows_tools import list_flows
|
||||
|
||||
total_tokens = 0
|
||||
calls = []
|
||||
|
||||
result = get_minimal_context(task="map architecture", repo_root=repo_root)
|
||||
tokens = estimate_tokens(result)
|
||||
total_tokens += tokens
|
||||
calls.append({"tool": "get_minimal_context", "tokens": tokens})
|
||||
|
||||
result = list_communities_func(repo_root=repo_root, detail_level="minimal")
|
||||
tokens = estimate_tokens(result)
|
||||
total_tokens += tokens
|
||||
calls.append({"tool": "list_communities_minimal", "tokens": tokens})
|
||||
|
||||
result = list_flows(repo_root=repo_root, detail_level="minimal")
|
||||
tokens = estimate_tokens(result)
|
||||
total_tokens += tokens
|
||||
calls.append({"tool": "list_flows_minimal", "tokens": tokens})
|
||||
|
||||
return {
|
||||
"workflow": "architecture",
|
||||
"total_tokens": total_tokens,
|
||||
"tool_calls": len(calls),
|
||||
"calls": calls,
|
||||
}
|
||||
|
||||
|
||||
def benchmark_debug_workflow(repo_root: str) -> dict:
|
||||
"""Simulate a debug workflow."""
|
||||
from ..tools.context import get_minimal_context
|
||||
from ..tools.query import semantic_search_nodes
|
||||
|
||||
total_tokens = 0
|
||||
calls = []
|
||||
|
||||
result = get_minimal_context(task="debug login bug", repo_root=repo_root)
|
||||
tokens = estimate_tokens(result)
|
||||
total_tokens += tokens
|
||||
calls.append({"tool": "get_minimal_context", "tokens": tokens})
|
||||
|
||||
result = semantic_search_nodes(
|
||||
query="login", repo_root=repo_root, detail_level="minimal",
|
||||
)
|
||||
tokens = estimate_tokens(result)
|
||||
total_tokens += tokens
|
||||
calls.append({"tool": "semantic_search_minimal", "tokens": tokens})
|
||||
|
||||
return {
|
||||
"workflow": "debug",
|
||||
"total_tokens": total_tokens,
|
||||
"tool_calls": len(calls),
|
||||
"calls": calls,
|
||||
}
|
||||
|
||||
|
||||
def benchmark_onboard_workflow(repo_root: str) -> dict:
|
||||
"""Simulate an onboarding workflow."""
|
||||
from ..tools.context import get_minimal_context
|
||||
from ..tools.query import list_graph_stats
|
||||
|
||||
total_tokens = 0
|
||||
calls = []
|
||||
|
||||
result = get_minimal_context(task="onboard developer", repo_root=repo_root)
|
||||
tokens = estimate_tokens(result)
|
||||
total_tokens += tokens
|
||||
calls.append({"tool": "get_minimal_context", "tokens": tokens})
|
||||
|
||||
result = list_graph_stats(repo_root=repo_root)
|
||||
tokens = estimate_tokens(result)
|
||||
total_tokens += tokens
|
||||
calls.append({"tool": "list_graph_stats", "tokens": tokens})
|
||||
|
||||
return {
|
||||
"workflow": "onboard",
|
||||
"total_tokens": total_tokens,
|
||||
"tool_calls": len(calls),
|
||||
"calls": calls,
|
||||
}
|
||||
|
||||
|
||||
def benchmark_pre_merge_workflow(repo_root: str, base: str = "HEAD~1") -> dict:
|
||||
"""Simulate a pre-merge check workflow."""
|
||||
from ..tools.context import get_minimal_context
|
||||
from ..tools.review import detect_changes_func
|
||||
|
||||
total_tokens = 0
|
||||
calls = []
|
||||
|
||||
result = get_minimal_context(task="pre-merge check", repo_root=repo_root, base=base)
|
||||
tokens = estimate_tokens(result)
|
||||
total_tokens += tokens
|
||||
calls.append({"tool": "get_minimal_context", "tokens": tokens})
|
||||
|
||||
result = detect_changes_func(base=base, repo_root=repo_root, detail_level="minimal")
|
||||
tokens = estimate_tokens(result)
|
||||
total_tokens += tokens
|
||||
calls.append({"tool": "detect_changes_minimal", "tokens": tokens})
|
||||
|
||||
return {
|
||||
"workflow": "pre_merge",
|
||||
"total_tokens": total_tokens,
|
||||
"tool_calls": len(calls),
|
||||
"calls": calls,
|
||||
}
|
||||
|
||||
|
||||
ALL_WORKFLOWS: dict[str, Callable[..., dict]] = {
|
||||
"review": benchmark_review_workflow,
|
||||
"architecture": benchmark_architecture_workflow,
|
||||
"debug": benchmark_debug_workflow,
|
||||
"onboard": benchmark_onboard_workflow,
|
||||
"pre_merge": benchmark_pre_merge_workflow,
|
||||
}
|
||||
|
||||
|
||||
def run_all_benchmarks(repo_root: str, base: str = "HEAD~1") -> list[dict]:
|
||||
"""Run all workflow benchmarks and return results."""
|
||||
results = []
|
||||
for name, fn in ALL_WORKFLOWS.items():
|
||||
try:
|
||||
if "base" in fn.__code__.co_varnames:
|
||||
result = fn(repo_root=repo_root, base=base)
|
||||
else:
|
||||
result = fn(repo_root=repo_root)
|
||||
results.append(result)
|
||||
except Exception as e:
|
||||
logger.warning("Benchmark %s failed: %s", name, e)
|
||||
results.append({"workflow": name, "error": str(e)})
|
||||
return results
|
||||
@@ -0,0 +1,409 @@
|
||||
"""Additional export formats: GraphML, Neo4j Cypher, Obsidian vault, SVG."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import html
|
||||
import logging
|
||||
import re
|
||||
from pathlib import Path
|
||||
|
||||
from .graph import GraphStore, _sanitize_name
|
||||
from .visualization import export_graph_data
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
# -------------------------------------------------------------------
|
||||
# GraphML export (for Gephi, yEd, Cytoscape)
|
||||
# -------------------------------------------------------------------
|
||||
|
||||
def export_graphml(store: GraphStore, output_path: Path) -> Path:
|
||||
"""Export the graph as GraphML XML for Gephi/yEd/Cytoscape.
|
||||
|
||||
Returns the path to the written file.
|
||||
"""
|
||||
data = export_graph_data(store)
|
||||
nodes = data["nodes"]
|
||||
edges = data["edges"]
|
||||
|
||||
lines = [
|
||||
'<?xml version="1.0" encoding="UTF-8"?>',
|
||||
'<graphml xmlns="http://graphml.graphstruct.org/graphml"',
|
||||
' xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"',
|
||||
' xsi:schemaLocation="http://graphml.graphstruct.org/graphml">',
|
||||
' <key id="kind" for="node" attr.name="kind" '
|
||||
'attr.type="string"/>',
|
||||
' <key id="file" for="node" attr.name="file" '
|
||||
'attr.type="string"/>',
|
||||
' <key id="language" for="node" attr.name="language" '
|
||||
'attr.type="string"/>',
|
||||
' <key id="community" for="node" attr.name="community" '
|
||||
'attr.type="int"/>',
|
||||
' <key id="edge_kind" for="edge" attr.name="kind" '
|
||||
'attr.type="string"/>',
|
||||
' <graph id="code-review-graph" edgedefault="directed">',
|
||||
]
|
||||
|
||||
for n in nodes:
|
||||
nid = html.escape(n["qualified_name"], quote=True)
|
||||
lines.append(f' <node id="{nid}">')
|
||||
lines.append(f' <data key="kind">'
|
||||
f'{html.escape(n.get("kind", ""))}</data>')
|
||||
lines.append(f' <data key="file">'
|
||||
f'{html.escape(n.get("file_path", ""))}</data>')
|
||||
lang = n.get("language", "") or ""
|
||||
lines.append(f' <data key="language">'
|
||||
f'{html.escape(lang)}</data>')
|
||||
cid = n.get("community_id")
|
||||
if cid is not None:
|
||||
lines.append(f' <data key="community">'
|
||||
f'{cid}</data>')
|
||||
lines.append(' </node>')
|
||||
|
||||
for i, e in enumerate(edges):
|
||||
src = html.escape(e["source"], quote=True)
|
||||
tgt = html.escape(e["target"], quote=True)
|
||||
kind = html.escape(e.get("kind", ""), quote=True)
|
||||
lines.append(
|
||||
f' <edge id="e{i}" source="{src}" target="{tgt}">'
|
||||
)
|
||||
lines.append(f' <data key="edge_kind">{kind}</data>')
|
||||
lines.append(' </edge>')
|
||||
|
||||
lines.append(' </graph>')
|
||||
lines.append('</graphml>')
|
||||
|
||||
output_path.write_text("\n".join(lines), encoding="utf-8")
|
||||
logger.info("GraphML exported to %s (%d nodes, %d edges)",
|
||||
output_path, len(nodes), len(edges))
|
||||
return output_path
|
||||
|
||||
|
||||
# -------------------------------------------------------------------
|
||||
# Neo4j Cypher export
|
||||
# -------------------------------------------------------------------
|
||||
|
||||
def export_neo4j_cypher(store: GraphStore, output_path: Path) -> Path:
|
||||
"""Export the graph as Neo4j Cypher CREATE statements.
|
||||
|
||||
Returns the path to the written file.
|
||||
"""
|
||||
data = export_graph_data(store)
|
||||
nodes = data["nodes"]
|
||||
edges = data["edges"]
|
||||
|
||||
lines = [
|
||||
"// Generated by code-review-graph",
|
||||
"// Import: paste into Neo4j Browser or run via cypher-shell",
|
||||
"",
|
||||
]
|
||||
|
||||
# Create nodes
|
||||
for n in nodes:
|
||||
kind = n.get("kind", "Node")
|
||||
props = {
|
||||
"qualified_name": n["qualified_name"],
|
||||
"name": n.get("name", ""),
|
||||
"file_path": n.get("file_path", ""),
|
||||
"language": n.get("language", "") or "",
|
||||
}
|
||||
cid = n.get("community_id")
|
||||
if cid is not None:
|
||||
props["community_id"] = cid
|
||||
props_str = _cypher_props(props)
|
||||
lines.append(f"CREATE (:{kind} {props_str});")
|
||||
|
||||
lines.append("")
|
||||
|
||||
# Create edges via MATCH
|
||||
for e in edges:
|
||||
kind = e.get("kind", "RELATES_TO")
|
||||
src_qn = _cypher_escape(e["source"])
|
||||
tgt_qn = _cypher_escape(e["target"])
|
||||
lines.append(
|
||||
f"MATCH (a {{qualified_name: '{src_qn}'}}), "
|
||||
f"(b {{qualified_name: '{tgt_qn}'}}) "
|
||||
f"CREATE (a)-[:{kind}]->(b);"
|
||||
)
|
||||
|
||||
output_path.write_text("\n".join(lines), encoding="utf-8")
|
||||
logger.info("Neo4j Cypher exported to %s (%d nodes, %d edges)",
|
||||
output_path, len(nodes), len(edges))
|
||||
return output_path
|
||||
|
||||
|
||||
def _cypher_escape(s: str) -> str:
|
||||
"""Escape a string for Cypher single-quoted literals."""
|
||||
return s.replace("\\", "\\\\").replace("'", "\\'")
|
||||
|
||||
|
||||
def _cypher_props(d: dict) -> str:
|
||||
"""Format a dict as Cypher property map."""
|
||||
parts = []
|
||||
for k, v in d.items():
|
||||
if isinstance(v, str):
|
||||
parts.append(f"{k}: '{_cypher_escape(v)}'")
|
||||
elif isinstance(v, (int, float)):
|
||||
parts.append(f"{k}: {v}")
|
||||
elif isinstance(v, bool):
|
||||
parts.append(f"{k}: {'true' if v else 'false'}")
|
||||
return "{" + ", ".join(parts) + "}"
|
||||
|
||||
|
||||
# -------------------------------------------------------------------
|
||||
# Obsidian vault export
|
||||
# -------------------------------------------------------------------
|
||||
|
||||
def export_obsidian_vault(
|
||||
store: GraphStore, output_dir: Path
|
||||
) -> Path:
|
||||
"""Export the graph as an Obsidian vault with wikilinks.
|
||||
|
||||
Creates:
|
||||
- One .md per node with YAML frontmatter and [[wikilinks]]
|
||||
- _COMMUNITY_*.md overview notes per community
|
||||
- _INDEX.md with links to all nodes
|
||||
|
||||
Returns the output directory path.
|
||||
"""
|
||||
data = export_graph_data(store)
|
||||
nodes = data["nodes"]
|
||||
edges = data["edges"]
|
||||
communities = data.get("communities", [])
|
||||
|
||||
output_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
# Build adjacency for wikilinks
|
||||
neighbors: dict[str, list[dict]] = {}
|
||||
for e in edges:
|
||||
src = e["source"]
|
||||
tgt = e["target"]
|
||||
kind = e.get("kind", "RELATES_TO")
|
||||
neighbors.setdefault(src, []).append(
|
||||
{"target": tgt, "kind": kind}
|
||||
)
|
||||
neighbors.setdefault(tgt, []).append(
|
||||
{"target": src, "kind": kind}
|
||||
)
|
||||
|
||||
# Node name -> slug mapping
|
||||
slugs: dict[str, str] = {}
|
||||
for n in nodes:
|
||||
slug = _obsidian_slug(n.get("name", n["qualified_name"]))
|
||||
# Handle collisions
|
||||
base_slug = slug
|
||||
counter = 1
|
||||
while slug in slugs.values():
|
||||
slug = f"{base_slug}-{counter}"
|
||||
counter += 1
|
||||
slugs[n["qualified_name"]] = slug
|
||||
|
||||
# Write node pages
|
||||
for n in nodes:
|
||||
qn = n["qualified_name"]
|
||||
slug = slugs[qn]
|
||||
name = n.get("name", qn)
|
||||
|
||||
frontmatter = {
|
||||
"kind": n.get("kind", ""),
|
||||
"file": n.get("file_path", ""),
|
||||
"language": n.get("language", "") or "",
|
||||
"community": n.get("community_id"),
|
||||
"tags": [n.get("kind", "").lower()],
|
||||
}
|
||||
|
||||
lines = ["---"]
|
||||
for k, v in frontmatter.items():
|
||||
if isinstance(v, list):
|
||||
lines.append(f"{k}:")
|
||||
for item in v:
|
||||
lines.append(f" - {item}")
|
||||
elif v is not None:
|
||||
lines.append(f"{k}: {v}")
|
||||
lines.append("---")
|
||||
lines.append(f"# {_sanitize_name(name)}")
|
||||
lines.append("")
|
||||
lines.append(f"**Kind:** {n.get('kind', '')}")
|
||||
lines.append(f"**File:** `{n.get('file_path', '')}`")
|
||||
lines.append("")
|
||||
|
||||
# Wikilinks to neighbors
|
||||
nbrs = neighbors.get(qn, [])
|
||||
if nbrs:
|
||||
lines.append("## Connections")
|
||||
lines.append("")
|
||||
seen = set()
|
||||
for nb in nbrs:
|
||||
tgt_slug = slugs.get(nb["target"])
|
||||
if tgt_slug and tgt_slug not in seen:
|
||||
seen.add(tgt_slug)
|
||||
tgt_name = tgt_slug.replace("-", " ").title()
|
||||
lines.append(
|
||||
f"- {nb['kind']}: "
|
||||
f"[[{tgt_slug}|{tgt_name}]]"
|
||||
)
|
||||
|
||||
page_path = output_dir / f"{slug}.md"
|
||||
page_path.write_text("\n".join(lines), encoding="utf-8")
|
||||
|
||||
# Write community overview pages
|
||||
community_map: dict[int, list[str]] = {}
|
||||
for n in nodes:
|
||||
cid = n.get("community_id")
|
||||
if cid is not None:
|
||||
community_map.setdefault(cid, []).append(
|
||||
n["qualified_name"]
|
||||
)
|
||||
|
||||
for c in communities:
|
||||
cid = c.get("id")
|
||||
cname = c.get("name", f"community-{cid}")
|
||||
members = community_map.get(cid, [])
|
||||
|
||||
lines = [f"# Community: {_sanitize_name(cname)}", ""]
|
||||
lines.append(f"**Size:** {c.get('size', len(members))}")
|
||||
lines.append(f"**Cohesion:** {c.get('cohesion', 0):.2f}")
|
||||
lang = c.get("dominant_language", "")
|
||||
if lang:
|
||||
lines.append(f"**Language:** {lang}")
|
||||
lines.append("")
|
||||
lines.append("## Members")
|
||||
lines.append("")
|
||||
for qn in members[:50]:
|
||||
slug = slugs.get(qn)
|
||||
if slug:
|
||||
lines.append(f"- [[{slug}]]")
|
||||
|
||||
page_path = output_dir / f"_COMMUNITY_{cid}.md"
|
||||
page_path.write_text("\n".join(lines), encoding="utf-8")
|
||||
|
||||
# Write index
|
||||
index_lines = ["# Code Graph Index", ""]
|
||||
index_lines.append(f"**Nodes:** {len(nodes)}")
|
||||
index_lines.append(f"**Edges:** {len(edges)}")
|
||||
index_lines.append(
|
||||
f"**Communities:** {len(communities)}"
|
||||
)
|
||||
index_lines.append("")
|
||||
index_lines.append("## All Nodes")
|
||||
index_lines.append("")
|
||||
for n in sorted(nodes, key=lambda x: x.get("name", "")):
|
||||
slug = slugs.get(n["qualified_name"])
|
||||
if slug:
|
||||
index_lines.append(
|
||||
f"- [[{slug}]] ({n.get('kind', '')})"
|
||||
)
|
||||
|
||||
(output_dir / "_INDEX.md").write_text(
|
||||
"\n".join(index_lines), encoding="utf-8"
|
||||
)
|
||||
|
||||
logger.info(
|
||||
"Obsidian vault exported to %s (%d pages)",
|
||||
output_dir, len(nodes)
|
||||
)
|
||||
return output_dir
|
||||
|
||||
|
||||
def _obsidian_slug(name: str) -> str:
|
||||
"""Convert a name to an Obsidian-friendly filename slug."""
|
||||
slug = re.sub(r"[^\w\s-]", "", name.lower())
|
||||
slug = re.sub(r"[\s_]+", "-", slug).strip("-")
|
||||
return slug[:100] or "unnamed"
|
||||
|
||||
|
||||
# -------------------------------------------------------------------
|
||||
# SVG export (matplotlib-based)
|
||||
# -------------------------------------------------------------------
|
||||
|
||||
def export_svg(store: GraphStore, output_path: Path) -> Path:
|
||||
"""Export a static SVG graph visualization.
|
||||
|
||||
Requires matplotlib (optional dependency).
|
||||
Returns the path to the written file.
|
||||
"""
|
||||
try:
|
||||
import matplotlib
|
||||
matplotlib.use("Agg")
|
||||
import matplotlib.pyplot as plt
|
||||
except ImportError:
|
||||
raise ImportError(
|
||||
"matplotlib is required for SVG export. "
|
||||
"Install with: pip install matplotlib"
|
||||
)
|
||||
|
||||
import networkx as nx
|
||||
|
||||
data = export_graph_data(store)
|
||||
nodes_data = data["nodes"]
|
||||
edges_data = data["edges"]
|
||||
|
||||
nxg: nx.DiGraph = nx.DiGraph() # type: ignore[type-arg]
|
||||
for n in nodes_data:
|
||||
nxg.add_node(
|
||||
n["qualified_name"],
|
||||
label=n.get("name", ""),
|
||||
kind=n.get("kind", ""),
|
||||
)
|
||||
for e in edges_data:
|
||||
if e["source"] in nxg and e["target"] in nxg:
|
||||
nxg.add_edge(e["source"], e["target"])
|
||||
|
||||
if nxg.number_of_nodes() == 0:
|
||||
raise ValueError("Graph is empty, nothing to export")
|
||||
|
||||
# Color by kind
|
||||
kind_colors = {
|
||||
"File": "#6c757d",
|
||||
"Class": "#0d6efd",
|
||||
"Function": "#198754",
|
||||
"Type": "#ffc107",
|
||||
"Test": "#dc3545",
|
||||
}
|
||||
colors = [
|
||||
kind_colors.get(
|
||||
nxg.nodes[n].get("kind", ""), "#adb5bd"
|
||||
)
|
||||
for n in nxg.nodes()
|
||||
]
|
||||
|
||||
fig, ax = plt.subplots(1, 1, figsize=(16, 12))
|
||||
pos = nx.spring_layout(
|
||||
nxg, k=2 / (nxg.number_of_nodes() ** 0.5),
|
||||
iterations=50, seed=42
|
||||
)
|
||||
|
||||
# Limit labels to avoid clutter
|
||||
labels = {}
|
||||
if nxg.number_of_nodes() <= 100:
|
||||
labels = {
|
||||
n: nxg.nodes[n].get("label", n.split("::")[-1])
|
||||
for n in nxg.nodes()
|
||||
}
|
||||
|
||||
nx.draw_networkx_nodes(
|
||||
nxg, pos, ax=ax, node_color=colors,
|
||||
node_size=30, alpha=0.8
|
||||
)
|
||||
nx.draw_networkx_edges(
|
||||
nxg, pos, ax=ax, alpha=0.2,
|
||||
arrows=True, arrowsize=5
|
||||
)
|
||||
if labels:
|
||||
nx.draw_networkx_labels(
|
||||
nxg, pos, labels=labels, ax=ax,
|
||||
font_size=6
|
||||
)
|
||||
|
||||
ax.set_title("Code Review Graph", fontsize=14)
|
||||
ax.axis("off")
|
||||
|
||||
fig.savefig(
|
||||
str(output_path), format="svg",
|
||||
bbox_inches="tight", dpi=150
|
||||
)
|
||||
plt.close(fig)
|
||||
|
||||
logger.info("SVG exported to %s (%d nodes)",
|
||||
output_path, nxg.number_of_nodes())
|
||||
return output_path
|
||||
@@ -0,0 +1,698 @@
|
||||
"""Execution flow detection, tracing, and criticality scoring.
|
||||
|
||||
Detects entry points in the codebase (functions with no incoming CALLS edges,
|
||||
framework-decorated handlers, and conventional name patterns), traces execution
|
||||
paths via forward BFS through CALLS edges, scores each flow for criticality,
|
||||
and persists results to the ``flows`` / ``flow_memberships`` tables.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
import re
|
||||
from collections import deque
|
||||
from typing import Optional
|
||||
|
||||
from .constants import SECURITY_KEYWORDS as _SECURITY_KEYWORDS
|
||||
from .graph import FlowAdjacency, GraphNode, GraphStore, _sanitize_name
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Constants
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
# Decorator patterns that indicate a function is a framework entry point.
|
||||
_FRAMEWORK_DECORATOR_PATTERNS: list[re.Pattern[str]] = [
|
||||
# Python web frameworks
|
||||
re.compile(r"app\.(get|post|put|delete|patch|route|websocket|on_event)", re.IGNORECASE),
|
||||
re.compile(r"router\.(get|post|put|delete|patch|route)", re.IGNORECASE),
|
||||
re.compile(r"blueprint\.(route|before_request|after_request)", re.IGNORECASE),
|
||||
re.compile(r"(before|after)_(request|response)", re.IGNORECASE),
|
||||
# CLI frameworks
|
||||
re.compile(r"click\.(command|group)", re.IGNORECASE),
|
||||
re.compile(r"\w+\.(command|group)\b", re.IGNORECASE), # Click subgroups: @mygroup.command()
|
||||
# Pydantic validators/serializers
|
||||
re.compile(r"(field|model)_(serializer|validator)", re.IGNORECASE),
|
||||
# Task queues
|
||||
re.compile(r"(celery\.)?(task|shared_task|periodic_task)", re.IGNORECASE),
|
||||
# Django
|
||||
re.compile(r"receiver", re.IGNORECASE),
|
||||
re.compile(r"api_view", re.IGNORECASE),
|
||||
re.compile(r"\baction\b", re.IGNORECASE),
|
||||
# Testing
|
||||
re.compile(r"pytest\.(fixture|mark)"),
|
||||
re.compile(r"(override_settings|modify_settings)", re.IGNORECASE),
|
||||
# SQLAlchemy / event systems
|
||||
re.compile(r"(event\.)?listens_for", re.IGNORECASE),
|
||||
# Java Spring
|
||||
re.compile(r"(Get|Post|Put|Delete|Patch|RequestMapping)Mapping", re.IGNORECASE),
|
||||
re.compile(r"(Scheduled|EventListener|Bean|Configuration)", re.IGNORECASE),
|
||||
# JS/TS frameworks
|
||||
re.compile(r"(Component|Injectable|Controller|Module|Guard|Pipe)", re.IGNORECASE),
|
||||
re.compile(r"(Subscribe|Mutation|Query|Resolver)", re.IGNORECASE),
|
||||
# Express / Koa / Hono route handlers
|
||||
re.compile(r"(app|router)\.(get|post|put|delete|patch|use|all)\b"),
|
||||
# Android lifecycle
|
||||
re.compile(r"@(Override|OnLifecycleEvent|Composable)", re.IGNORECASE),
|
||||
# Kotlin coroutines / Android ViewModel
|
||||
re.compile(r"(HiltViewModel|AndroidEntryPoint|Inject)", re.IGNORECASE),
|
||||
# AI/agent frameworks (pydantic-ai, langchain, etc.)
|
||||
re.compile(r"\w+\.(tool|tool_plain|system_prompt|result_validator)\b", re.IGNORECASE),
|
||||
re.compile(r"^tool\b"), # bare @tool (LangChain, etc.)
|
||||
# Middleware and exception handlers (Starlette, FastAPI, Sanic)
|
||||
re.compile(r"\w+\.(middleware|exception_handler|on_exception)\b", re.IGNORECASE),
|
||||
# Generic route decorator (Flask blueprints: @bp.route, @auth_bp.route, etc.)
|
||||
re.compile(r"\w+\.route\b", re.IGNORECASE),
|
||||
]
|
||||
|
||||
# Name patterns that indicate conventional entry points.
|
||||
_ENTRY_NAME_PATTERNS: list[re.Pattern[str]] = [
|
||||
re.compile(r"^main$"),
|
||||
re.compile(r"^__main__$"),
|
||||
re.compile(r"^test_"),
|
||||
re.compile(r"^Test[A-Z]"),
|
||||
re.compile(r"^on_"),
|
||||
re.compile(r"^handle_"),
|
||||
# Lambda / serverless handler functions (wired via config, not code calls)
|
||||
re.compile(r"^handler$"),
|
||||
re.compile(r"^handle$"),
|
||||
re.compile(r"^lambda_handler$"),
|
||||
# Alembic migration entry points
|
||||
re.compile(r"^upgrade$"),
|
||||
re.compile(r"^downgrade$"),
|
||||
# FastAPI lifecycle / dependency injection
|
||||
re.compile(r"^lifespan$"),
|
||||
re.compile(r"^get_db$"),
|
||||
# Android Activity/Fragment lifecycle
|
||||
re.compile(r"^on(Create|Start|Resume|Pause|Stop|Destroy|Bind|Receive)"),
|
||||
# Servlet / JAX-RS
|
||||
re.compile(r"^do(Get|Post|Put|Delete)$"),
|
||||
# Python BaseHTTPRequestHandler
|
||||
re.compile(r"^do_(GET|POST|PUT|DELETE|PATCH|HEAD|OPTIONS)$"),
|
||||
re.compile(r"^log_message$"),
|
||||
# Express middleware signature
|
||||
re.compile(r"^(middleware|errorHandler)$"),
|
||||
# Angular lifecycle hooks
|
||||
re.compile(
|
||||
r"^ng(OnInit|OnChanges|OnDestroy|DoCheck"
|
||||
r"|AfterContentInit|AfterContentChecked|AfterViewInit|AfterViewChecked)$"
|
||||
),
|
||||
# Angular Pipe / ControlValueAccessor / Guards / Resolvers
|
||||
re.compile(r"^(transform|writeValue|registerOnChange|registerOnTouched|setDisabledState)$"),
|
||||
re.compile(r"^(canActivate|canDeactivate|canActivateChild|canLoad|canMatch|resolve)$"),
|
||||
# React class component lifecycle
|
||||
re.compile(
|
||||
r"^(componentDidMount|componentDidUpdate|componentWillUnmount"
|
||||
r"|shouldComponentUpdate|render)$"
|
||||
),
|
||||
]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Entry-point detection
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _has_framework_decorator(node: GraphNode) -> bool:
|
||||
"""Return True if *node* has a decorator matching a framework pattern."""
|
||||
decorators = node.extra.get("decorators")
|
||||
if not decorators:
|
||||
return False
|
||||
if isinstance(decorators, str):
|
||||
decorators = [decorators]
|
||||
for dec in decorators:
|
||||
for pat in _FRAMEWORK_DECORATOR_PATTERNS:
|
||||
if pat.search(dec):
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def _matches_entry_name(node: GraphNode) -> bool:
|
||||
"""Return True if *node*'s name matches a conventional entry-point pattern."""
|
||||
for pat in _ENTRY_NAME_PATTERNS:
|
||||
if pat.search(node.name):
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
_TEST_FILE_RE = re.compile(
|
||||
r"([\\/]__tests__[\\/]|\.spec\.[jt]sx?$|\.test\.[jt]sx?$|[\\/]test_[^/\\]*\.py$)",
|
||||
)
|
||||
|
||||
|
||||
def _is_test_file(file_path: str) -> bool:
|
||||
"""Return True if *file_path* looks like a test file."""
|
||||
return bool(_TEST_FILE_RE.search(file_path))
|
||||
|
||||
|
||||
def detect_entry_points(
|
||||
store: GraphStore,
|
||||
include_tests: bool = False,
|
||||
) -> list[GraphNode]:
|
||||
"""Find functions that are entry points in the graph.
|
||||
|
||||
An entry point is a Function/Test node that either:
|
||||
1. Has no incoming CALLS edges (true root), or
|
||||
2. Has a framework decorator (e.g. ``@app.get``), or
|
||||
3. Matches a conventional name pattern (``main``, ``test_*``, etc.).
|
||||
|
||||
When *include_tests* is False (the default), Test nodes are excluded so
|
||||
that flow analysis focuses on production entry points.
|
||||
"""
|
||||
# Build a set of all qualified names that are CALLS targets. Exclude
|
||||
# edges sourced at File nodes so that script-/notebook-/top-level-only
|
||||
# callees (e.g. ``run_job()`` invoked from module scope, a top-level
|
||||
# ``<App />`` render) remain detectable as entry points.
|
||||
called_qnames = store.get_all_call_targets(include_file_sources=False)
|
||||
|
||||
# Scan all nodes for entry-point candidates.
|
||||
candidate_nodes = store.get_nodes_by_kind(["Function", "Test"])
|
||||
|
||||
entry_points: list[GraphNode] = []
|
||||
seen_qn: set[str] = set()
|
||||
|
||||
for node in candidate_nodes:
|
||||
if not include_tests and (node.is_test or _is_test_file(node.file_path)):
|
||||
continue
|
||||
|
||||
is_entry = False
|
||||
|
||||
# True root: no one calls this function.
|
||||
if node.qualified_name not in called_qnames:
|
||||
is_entry = True
|
||||
|
||||
# Framework decorator match.
|
||||
if _has_framework_decorator(node):
|
||||
is_entry = True
|
||||
|
||||
# Conventional name match.
|
||||
if _matches_entry_name(node):
|
||||
is_entry = True
|
||||
|
||||
if is_entry and node.qualified_name not in seen_qn:
|
||||
entry_points.append(node)
|
||||
seen_qn.add(node.qualified_name)
|
||||
|
||||
return entry_points
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Flow tracing (BFS)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _trace_single_flow(
|
||||
adj: FlowAdjacency,
|
||||
ep: GraphNode,
|
||||
max_depth: int = 15,
|
||||
) -> Optional[dict]:
|
||||
"""Trace a single execution flow from *ep* via forward BFS.
|
||||
|
||||
Returns a flow dict (see :func:`trace_flows` for the schema) or ``None``
|
||||
if the flow is trivial (single-node, no outgoing CALLS that resolve).
|
||||
"""
|
||||
path_ids: list[int] = [ep.id]
|
||||
path_qnames: list[str] = [ep.qualified_name]
|
||||
visited: set[str] = {ep.qualified_name}
|
||||
queue: deque[tuple[str, int]] = deque([(ep.qualified_name, 0)])
|
||||
|
||||
actual_depth = 0
|
||||
nodes_by_qn = adj.nodes_by_qn
|
||||
calls_out = adj.calls_out
|
||||
|
||||
while queue:
|
||||
current_qn, depth = queue.popleft()
|
||||
if depth > actual_depth:
|
||||
actual_depth = depth
|
||||
if depth >= max_depth:
|
||||
continue
|
||||
|
||||
for target_qn in calls_out.get(current_qn, ()):
|
||||
if target_qn in visited:
|
||||
continue
|
||||
target_node = nodes_by_qn.get(target_qn)
|
||||
if target_node is None:
|
||||
continue
|
||||
visited.add(target_qn)
|
||||
path_ids.append(target_node.id)
|
||||
path_qnames.append(target_qn)
|
||||
queue.append((target_qn, depth + 1))
|
||||
|
||||
# Skip trivial single-node flows.
|
||||
if len(path_ids) < 2:
|
||||
return None
|
||||
|
||||
files = list({
|
||||
n.file_path
|
||||
for qn in path_qnames
|
||||
if (n := nodes_by_qn.get(qn)) is not None
|
||||
})
|
||||
|
||||
flow: dict = {
|
||||
"name": _sanitize_name(ep.name),
|
||||
"entry_point": ep.qualified_name,
|
||||
"entry_point_id": ep.id,
|
||||
"path": path_ids,
|
||||
"depth": actual_depth,
|
||||
"node_count": len(path_ids),
|
||||
"file_count": len(files),
|
||||
"files": files,
|
||||
"criticality": 0.0,
|
||||
}
|
||||
flow["criticality"] = compute_criticality(flow, adj)
|
||||
return flow
|
||||
|
||||
|
||||
def trace_flows(
|
||||
store: GraphStore,
|
||||
max_depth: int = 15,
|
||||
include_tests: bool = False,
|
||||
) -> list[dict]:
|
||||
"""Trace execution flows from every entry point via forward BFS.
|
||||
|
||||
Returns a list of flow dicts, each containing:
|
||||
- name: human-readable flow name (entry point name)
|
||||
- entry_point: qualified name of the entry point
|
||||
- entry_point_id: node database id of the entry point
|
||||
- path: ordered list of node IDs in the flow
|
||||
- depth: maximum BFS depth reached
|
||||
- node_count: number of distinct nodes in the path
|
||||
- file_count: number of distinct files touched
|
||||
- files: list of distinct file paths
|
||||
- criticality: computed criticality score (0.0-1.0)
|
||||
"""
|
||||
entry_points = detect_entry_points(store, include_tests=include_tests)
|
||||
if not entry_points:
|
||||
return []
|
||||
|
||||
adj = store.load_flow_adjacency()
|
||||
flows: list[dict] = []
|
||||
|
||||
for ep in entry_points:
|
||||
flow = _trace_single_flow(adj, ep, max_depth)
|
||||
if flow is not None:
|
||||
flows.append(flow)
|
||||
|
||||
# Sort by criticality descending.
|
||||
flows.sort(key=lambda f: f["criticality"], reverse=True)
|
||||
return flows
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Criticality scoring
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def compute_criticality(flow: dict, adj: FlowAdjacency) -> float:
|
||||
"""Score a flow from 0.0 to 1.0 based on multiple weighted factors.
|
||||
|
||||
Weights:
|
||||
- File spread: 0.30
|
||||
- External calls: 0.20
|
||||
- Security sensitivity: 0.25
|
||||
- Test coverage gap: 0.15
|
||||
- Depth: 0.10
|
||||
"""
|
||||
node_ids: list[int] = flow.get("path", [])
|
||||
if not node_ids:
|
||||
return 0.0
|
||||
|
||||
nodes_by_id = adj.nodes_by_id
|
||||
nodes_by_qn = adj.nodes_by_qn
|
||||
calls_out = adj.calls_out
|
||||
has_tested_by = adj.has_tested_by
|
||||
|
||||
nodes: list[GraphNode] = [
|
||||
n for nid in node_ids if (n := nodes_by_id.get(nid)) is not None
|
||||
]
|
||||
if not nodes:
|
||||
return 0.0
|
||||
|
||||
# --- File spread (0.0 - 1.0) ---
|
||||
file_count = len({n.file_path for n in nodes})
|
||||
# Normalize: 1 file => 0.0, 5+ files => 1.0
|
||||
file_spread = min((file_count - 1) / 4.0, 1.0) if file_count > 1 else 0.0
|
||||
|
||||
# --- External calls (0.0 - 1.0) ---
|
||||
# Calls that target nodes NOT in the graph are considered external.
|
||||
external_count = 0
|
||||
for n in nodes:
|
||||
for target_qn in calls_out.get(n.qualified_name, ()):
|
||||
if target_qn not in nodes_by_qn:
|
||||
external_count += 1
|
||||
# Normalize: 0 => 0.0, 5+ => 1.0
|
||||
external_score = min(external_count / 5.0, 1.0)
|
||||
|
||||
# --- Security sensitivity (0.0 - 1.0) ---
|
||||
security_hits = 0
|
||||
for n in nodes:
|
||||
name_lower = n.name.lower()
|
||||
qn_lower = n.qualified_name.lower()
|
||||
for kw in _SECURITY_KEYWORDS:
|
||||
if kw in name_lower or kw in qn_lower:
|
||||
security_hits += 1
|
||||
break # Count each node at most once.
|
||||
security_score = min(security_hits / max(len(nodes), 1), 1.0)
|
||||
|
||||
# --- Test coverage gap (0.0 - 1.0) ---
|
||||
tested_count = sum(1 for n in nodes if n.qualified_name in has_tested_by)
|
||||
coverage = tested_count / max(len(nodes), 1)
|
||||
test_gap = 1.0 - coverage
|
||||
|
||||
# --- Depth (0.0 - 1.0) ---
|
||||
depth = flow.get("depth", 0)
|
||||
# Normalize: 0 => 0.0, 10+ => 1.0
|
||||
depth_score = min(depth / 10.0, 1.0)
|
||||
|
||||
# --- Weighted sum ---
|
||||
criticality = (
|
||||
file_spread * 0.30
|
||||
+ external_score * 0.20
|
||||
+ security_score * 0.25
|
||||
+ test_gap * 0.15
|
||||
+ depth_score * 0.10
|
||||
)
|
||||
return round(min(max(criticality, 0.0), 1.0), 4)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Persistence
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def store_flows(store: GraphStore, flows: list[dict]) -> int:
|
||||
"""Clear existing flows and persist new ones.
|
||||
|
||||
Returns the number of flows stored.
|
||||
"""
|
||||
# NOTE: store_flows uses _conn directly because it performs
|
||||
# multi-statement batch writes (DELETE + INSERT loop) that are
|
||||
# tightly coupled to the DB transaction lifecycle.
|
||||
conn = store._conn
|
||||
|
||||
if conn.in_transaction:
|
||||
logger.warning("Rolling back uncommitted transaction before BEGIN IMMEDIATE")
|
||||
conn.rollback()
|
||||
# Wrap the full DELETE + INSERT sequence in an explicit transaction
|
||||
# so partial writes cannot occur if an exception interrupts the loop.
|
||||
conn.execute("BEGIN IMMEDIATE")
|
||||
try:
|
||||
conn.execute("DELETE FROM flow_memberships")
|
||||
conn.execute("DELETE FROM flows")
|
||||
|
||||
count = 0
|
||||
for flow in flows:
|
||||
path_json = json.dumps(flow.get("path", []))
|
||||
conn.execute(
|
||||
"""INSERT INTO flows
|
||||
(name, entry_point_id, depth, node_count, file_count,
|
||||
criticality, path_json)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?)""",
|
||||
(
|
||||
flow["name"],
|
||||
flow["entry_point_id"],
|
||||
flow["depth"],
|
||||
flow["node_count"],
|
||||
flow["file_count"],
|
||||
flow["criticality"],
|
||||
path_json,
|
||||
),
|
||||
)
|
||||
flow_id = conn.execute("SELECT last_insert_rowid()").fetchone()[0]
|
||||
|
||||
# Insert memberships.
|
||||
node_ids = flow.get("path", [])
|
||||
for position, node_id in enumerate(node_ids):
|
||||
conn.execute(
|
||||
"INSERT OR IGNORE INTO flow_memberships (flow_id, node_id, position) "
|
||||
"VALUES (?, ?, ?)",
|
||||
(flow_id, node_id, position),
|
||||
)
|
||||
count += 1
|
||||
|
||||
conn.commit()
|
||||
except BaseException:
|
||||
conn.rollback()
|
||||
raise
|
||||
return count
|
||||
|
||||
|
||||
def incremental_trace_flows(
|
||||
store: GraphStore,
|
||||
changed_files: list[str],
|
||||
max_depth: int = 15,
|
||||
) -> int:
|
||||
"""Re-trace only flows that touch *changed_files*. Much faster than full trace.
|
||||
|
||||
1. Find flow IDs whose memberships reference nodes in *changed_files*.
|
||||
2. Collect the entry-point node IDs of those flows before deleting them.
|
||||
3. Delete only the affected flows and their memberships.
|
||||
4. Re-detect entry points, keeping those in *changed_files* **or** whose
|
||||
node ID was an entry point of a deleted flow.
|
||||
5. BFS-trace each relevant entry point via :func:`_trace_single_flow`.
|
||||
6. INSERT the new flows (without clearing unrelated flows).
|
||||
|
||||
Returns the number of re-traced flows that were stored.
|
||||
"""
|
||||
if not changed_files:
|
||||
return 0
|
||||
|
||||
conn = store._conn
|
||||
changed_file_set = set(changed_files)
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# 1. Find affected flow IDs
|
||||
# ------------------------------------------------------------------
|
||||
placeholders = ",".join("?" * len(changed_files))
|
||||
affected_rows = conn.execute(
|
||||
f"SELECT DISTINCT fm.flow_id FROM flow_memberships fm " # nosec B608
|
||||
f"JOIN nodes n ON n.id = fm.node_id "
|
||||
f"WHERE n.file_path IN ({placeholders})",
|
||||
changed_files,
|
||||
).fetchall()
|
||||
affected_ids = [r[0] for r in affected_rows]
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# 2. Collect old entry-point node IDs before deletion
|
||||
# ------------------------------------------------------------------
|
||||
entry_point_ids: set[int] = set()
|
||||
if affected_ids:
|
||||
ep_placeholders = ",".join("?" * len(affected_ids))
|
||||
ep_rows = conn.execute(
|
||||
f"SELECT entry_point_id FROM flows " # nosec B608
|
||||
f"WHERE id IN ({ep_placeholders})",
|
||||
affected_ids,
|
||||
).fetchall()
|
||||
entry_point_ids = {r[0] for r in ep_rows}
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# 3. Delete affected flows and their memberships
|
||||
# ------------------------------------------------------------------
|
||||
# Wrap in an explicit transaction so a crash mid-loop cannot leave
|
||||
# orphaned flow_memberships rows pointing at deleted flows. See #258.
|
||||
if affected_ids:
|
||||
if conn.in_transaction:
|
||||
conn.commit()
|
||||
conn.execute("BEGIN IMMEDIATE")
|
||||
try:
|
||||
for fid in affected_ids:
|
||||
conn.execute(
|
||||
"DELETE FROM flow_memberships WHERE flow_id = ?", (fid,),
|
||||
)
|
||||
conn.execute("DELETE FROM flows WHERE id = ?", (fid,))
|
||||
conn.commit()
|
||||
except BaseException:
|
||||
conn.rollback()
|
||||
raise
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# 4. Re-detect entry points and filter to relevant ones
|
||||
# ------------------------------------------------------------------
|
||||
entry_points = detect_entry_points(store)
|
||||
relevant_eps = [
|
||||
ep for ep in entry_points
|
||||
if ep.file_path in changed_file_set or ep.id in entry_point_ids
|
||||
]
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# 5. BFS-trace each relevant entry point
|
||||
# ------------------------------------------------------------------
|
||||
new_flows: list[dict] = []
|
||||
if relevant_eps:
|
||||
adj = store.load_flow_adjacency()
|
||||
for ep in relevant_eps:
|
||||
flow = _trace_single_flow(adj, ep, max_depth)
|
||||
if flow is not None:
|
||||
new_flows.append(flow)
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# 6. INSERT new flows without clearing unrelated ones
|
||||
# ------------------------------------------------------------------
|
||||
count = 0
|
||||
for flow in new_flows:
|
||||
path_json = json.dumps(flow.get("path", []))
|
||||
conn.execute(
|
||||
"""INSERT INTO flows
|
||||
(name, entry_point_id, depth, node_count, file_count,
|
||||
criticality, path_json)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?)""",
|
||||
(
|
||||
flow["name"],
|
||||
flow["entry_point_id"],
|
||||
flow["depth"],
|
||||
flow["node_count"],
|
||||
flow["file_count"],
|
||||
flow["criticality"],
|
||||
path_json,
|
||||
),
|
||||
)
|
||||
flow_id = conn.execute("SELECT last_insert_rowid()").fetchone()[0]
|
||||
|
||||
node_ids = flow.get("path", [])
|
||||
for position, node_id in enumerate(node_ids):
|
||||
conn.execute(
|
||||
"INSERT OR IGNORE INTO flow_memberships (flow_id, node_id, position) "
|
||||
"VALUES (?, ?, ?)",
|
||||
(flow_id, node_id, position),
|
||||
)
|
||||
count += 1
|
||||
|
||||
conn.commit()
|
||||
return count
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Query helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def get_flows(
|
||||
store: GraphStore,
|
||||
sort_by: str = "criticality",
|
||||
limit: int = 50,
|
||||
) -> list[dict]:
|
||||
"""Retrieve stored flows from the database.
|
||||
|
||||
Args:
|
||||
store: The graph store.
|
||||
sort_by: Column to sort by (``criticality``, ``depth``, ``node_count``).
|
||||
limit: Maximum number of flows to return.
|
||||
"""
|
||||
allowed_sort = {"criticality", "depth", "node_count", "file_count", "name"}
|
||||
if sort_by not in allowed_sort:
|
||||
sort_by = "criticality"
|
||||
|
||||
order = "DESC" if sort_by in ("criticality", "depth", "node_count", "file_count") else "ASC"
|
||||
|
||||
# NOTE: get_flows reads from the flows table which is managed by
|
||||
# the flows module; _conn access is documented coupling.
|
||||
rows = store._conn.execute(
|
||||
f"SELECT * FROM flows ORDER BY {sort_by} {order} LIMIT ?", # nosec B608
|
||||
(limit,),
|
||||
).fetchall()
|
||||
|
||||
results: list[dict] = []
|
||||
for row in rows:
|
||||
results.append({
|
||||
"id": row["id"],
|
||||
"name": _sanitize_name(row["name"]),
|
||||
"entry_point_id": row["entry_point_id"],
|
||||
"depth": row["depth"],
|
||||
"node_count": row["node_count"],
|
||||
"file_count": row["file_count"],
|
||||
"criticality": row["criticality"],
|
||||
"path": json.loads(row["path_json"]),
|
||||
"created_at": row["created_at"],
|
||||
"updated_at": row["updated_at"],
|
||||
})
|
||||
return results
|
||||
|
||||
|
||||
def get_flow_by_id(store: GraphStore, flow_id: int) -> Optional[dict]:
|
||||
"""Retrieve a single flow with full path details.
|
||||
|
||||
Returns a dict with the flow metadata plus a ``steps`` list containing
|
||||
each node's name, kind, file, and line info.
|
||||
"""
|
||||
# NOTE: get_flow_by_id reads from the flows table; see store_flows note.
|
||||
row = store._conn.execute(
|
||||
"SELECT * FROM flows WHERE id = ?", (flow_id,)
|
||||
).fetchone()
|
||||
if row is None:
|
||||
return None
|
||||
|
||||
path_ids: list[int] = json.loads(row["path_json"])
|
||||
|
||||
# Build detailed step info.
|
||||
steps: list[dict] = []
|
||||
for nid in path_ids:
|
||||
node = store.get_node_by_id(nid)
|
||||
if node:
|
||||
steps.append({
|
||||
"node_id": node.id,
|
||||
"name": _sanitize_name(node.name),
|
||||
"kind": node.kind,
|
||||
"file": node.file_path,
|
||||
"line_start": node.line_start,
|
||||
"line_end": node.line_end,
|
||||
"qualified_name": _sanitize_name(node.qualified_name),
|
||||
})
|
||||
|
||||
return {
|
||||
"id": row["id"],
|
||||
"name": _sanitize_name(row["name"]),
|
||||
"entry_point_id": row["entry_point_id"],
|
||||
"depth": row["depth"],
|
||||
"node_count": row["node_count"],
|
||||
"file_count": row["file_count"],
|
||||
"criticality": row["criticality"],
|
||||
"path": path_ids,
|
||||
"steps": steps,
|
||||
"created_at": row["created_at"],
|
||||
"updated_at": row["updated_at"],
|
||||
}
|
||||
|
||||
|
||||
def get_affected_flows(
|
||||
store: GraphStore,
|
||||
changed_files: list[str],
|
||||
) -> dict:
|
||||
"""Find flows that include nodes from the given changed files.
|
||||
|
||||
Returns::
|
||||
|
||||
{
|
||||
"affected_flows": [<flow dicts>],
|
||||
"total": <int>,
|
||||
}
|
||||
"""
|
||||
if not changed_files:
|
||||
return {"affected_flows": [], "total": 0}
|
||||
|
||||
# Find node IDs belonging to changed files.
|
||||
node_ids = store.get_node_ids_by_files(changed_files)
|
||||
|
||||
if not node_ids:
|
||||
return {"affected_flows": [], "total": 0}
|
||||
|
||||
# Find flow IDs that contain any of these nodes.
|
||||
flow_ids = store.get_flow_ids_by_node_ids(node_ids)
|
||||
|
||||
if not flow_ids:
|
||||
return {"affected_flows": [], "total": 0}
|
||||
|
||||
affected: list[dict] = []
|
||||
for fid in flow_ids:
|
||||
flow = get_flow_by_id(store, fid)
|
||||
if flow:
|
||||
affected.append(flow)
|
||||
|
||||
# Sort by criticality descending.
|
||||
affected.sort(key=lambda f: f.get("criticality", 0), reverse=True)
|
||||
|
||||
return {
|
||||
"affected_flows": affected,
|
||||
"total": len(affected),
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,122 @@
|
||||
"""Graph snapshot diffing -- compare graph state over time."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from .graph import GraphStore
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def take_snapshot(store: GraphStore) -> dict[str, Any]:
|
||||
"""Take a snapshot of the current graph state.
|
||||
|
||||
Returns a dict with node and edge counts, qualified names,
|
||||
and community assignments for later diffing.
|
||||
"""
|
||||
stats = store.get_stats()
|
||||
nodes = store.get_all_nodes(exclude_files=False)
|
||||
community_map = store.get_all_community_ids()
|
||||
|
||||
return {
|
||||
"node_count": stats.total_nodes,
|
||||
"edge_count": stats.total_edges,
|
||||
"nodes": {
|
||||
n.qualified_name: {
|
||||
"kind": n.kind,
|
||||
"file": n.file_path,
|
||||
"community_id": community_map.get(
|
||||
n.qualified_name
|
||||
),
|
||||
}
|
||||
for n in nodes
|
||||
},
|
||||
"edges": {
|
||||
f"{e.source_qualified}->"
|
||||
f"{e.target_qualified}:{e.kind}"
|
||||
for e in store.get_all_edges()
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def save_snapshot(snapshot: dict, path: Path) -> None:
|
||||
"""Save a snapshot to a JSON file."""
|
||||
data = dict(snapshot)
|
||||
if isinstance(data.get("edges"), set):
|
||||
data["edges"] = sorted(data["edges"])
|
||||
path.write_text(
|
||||
json.dumps(data, indent=2), encoding="utf-8"
|
||||
)
|
||||
|
||||
|
||||
def load_snapshot(path: Path) -> dict:
|
||||
"""Load a snapshot from a JSON file."""
|
||||
data = json.loads(path.read_text(encoding="utf-8"))
|
||||
if isinstance(data.get("edges"), list):
|
||||
data["edges"] = set(data["edges"])
|
||||
return data
|
||||
|
||||
|
||||
def diff_snapshots(
|
||||
before: dict, after: dict,
|
||||
) -> dict[str, Any]:
|
||||
"""Compare two graph snapshots.
|
||||
|
||||
Returns:
|
||||
Dict with new_nodes, removed_nodes, new_edges,
|
||||
removed_edges, community_changes, and summary
|
||||
statistics.
|
||||
"""
|
||||
before_nodes = set(before.get("nodes", {}).keys())
|
||||
after_nodes = set(after.get("nodes", {}).keys())
|
||||
before_edges = before.get("edges", set())
|
||||
after_edges = after.get("edges", set())
|
||||
|
||||
new_nodes = after_nodes - before_nodes
|
||||
removed_nodes = before_nodes - after_nodes
|
||||
new_edges = after_edges - before_edges
|
||||
removed_edges = before_edges - after_edges
|
||||
|
||||
# Community changes for nodes that exist in both
|
||||
community_changes = []
|
||||
for qn in before_nodes & after_nodes:
|
||||
before_cid = before["nodes"][qn].get(
|
||||
"community_id"
|
||||
)
|
||||
after_cid = after["nodes"][qn].get(
|
||||
"community_id"
|
||||
)
|
||||
if before_cid != after_cid:
|
||||
community_changes.append({
|
||||
"node": qn,
|
||||
"before_community": before_cid,
|
||||
"after_community": after_cid,
|
||||
})
|
||||
|
||||
return {
|
||||
"new_nodes": [
|
||||
{"qualified_name": qn, **after["nodes"][qn]}
|
||||
for qn in sorted(new_nodes)
|
||||
][:100],
|
||||
"removed_nodes": sorted(removed_nodes)[:100],
|
||||
"new_edges": sorted(new_edges)[:100],
|
||||
"removed_edges": sorted(removed_edges)[:100],
|
||||
"community_changes": community_changes[:50],
|
||||
"summary": {
|
||||
"nodes_added": len(new_nodes),
|
||||
"nodes_removed": len(removed_nodes),
|
||||
"edges_added": len(new_edges),
|
||||
"edges_removed": len(removed_edges),
|
||||
"community_moves": len(community_changes),
|
||||
"before_total": before.get(
|
||||
"node_count", 0
|
||||
),
|
||||
"after_total": after.get(
|
||||
"node_count", 0
|
||||
),
|
||||
},
|
||||
}
|
||||
@@ -0,0 +1,384 @@
|
||||
"""Context-aware hints system for MCP tool responses.
|
||||
|
||||
Tracks session state (in-memory only) and generates intelligent
|
||||
next-step suggestions after each tool call. Hints are appended as
|
||||
``_hints`` to new tool responses so that Claude Code can propose
|
||||
follow-up actions without the user having to discover them.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import time
|
||||
from collections import deque
|
||||
from typing import Any
|
||||
|
||||
# ---- intent categories and their characteristic tool names ----
|
||||
|
||||
_INTENT_TOOLS: dict[str, set[str]] = {
|
||||
"reviewing": {
|
||||
"detect_changes", "get_review_context", "get_affected_flows", "get_impact_radius",
|
||||
},
|
||||
"debugging": {
|
||||
"query_graph", "get_flow", "semantic_search_nodes",
|
||||
},
|
||||
"refactoring": {
|
||||
"refactor", "find_dead_code", "suggest_refactorings",
|
||||
},
|
||||
"exploring": {
|
||||
"list_communities", "get_architecture_overview", "list_flows", "list_graph_stats",
|
||||
},
|
||||
}
|
||||
|
||||
# ---- workflow adjacency: for each tool, which tools are useful next ----
|
||||
|
||||
_WORKFLOW: dict[str, list[dict[str, str]]] = {
|
||||
"list_flows": [
|
||||
{
|
||||
"tool": "get_flow",
|
||||
"suggestion": "Drill into a specific flow for step-by-step details",
|
||||
},
|
||||
{
|
||||
"tool": "get_affected_flows",
|
||||
"suggestion": "Check which flows are affected by recent changes",
|
||||
},
|
||||
{
|
||||
"tool": "get_architecture_overview",
|
||||
"suggestion": "See the high-level architecture",
|
||||
},
|
||||
],
|
||||
"get_flow": [
|
||||
{
|
||||
"tool": "query_graph",
|
||||
"suggestion": "Inspect callers/callees of a step in this flow",
|
||||
},
|
||||
{
|
||||
"tool": "get_affected_flows",
|
||||
"suggestion": "Check if changes affect this flow",
|
||||
},
|
||||
{
|
||||
"tool": "list_flows",
|
||||
"suggestion": "Browse other execution flows",
|
||||
},
|
||||
],
|
||||
"get_affected_flows": [
|
||||
{
|
||||
"tool": "detect_changes",
|
||||
"suggestion": "Get risk-scored change analysis",
|
||||
},
|
||||
{
|
||||
"tool": "get_flow",
|
||||
"suggestion": "Inspect a specific affected flow",
|
||||
},
|
||||
{
|
||||
"tool": "get_review_context",
|
||||
"suggestion": "Build a full review context for the changes",
|
||||
},
|
||||
],
|
||||
"list_communities": [
|
||||
{
|
||||
"tool": "get_community",
|
||||
"suggestion": "Inspect a specific community's members",
|
||||
},
|
||||
{
|
||||
"tool": "get_architecture_overview",
|
||||
"suggestion": "See cross-community coupling and warnings",
|
||||
},
|
||||
{
|
||||
"tool": "list_flows",
|
||||
"suggestion": "See execution flows across communities",
|
||||
},
|
||||
],
|
||||
"get_community": [
|
||||
{
|
||||
"tool": "query_graph",
|
||||
"suggestion": "Explore callers/callees of community members",
|
||||
},
|
||||
{
|
||||
"tool": "list_communities",
|
||||
"suggestion": "Browse other communities",
|
||||
},
|
||||
{
|
||||
"tool": "get_architecture_overview",
|
||||
"suggestion": "See how this community fits the architecture",
|
||||
},
|
||||
],
|
||||
"get_architecture_overview": [
|
||||
{
|
||||
"tool": "list_communities",
|
||||
"suggestion": "Drill into individual communities",
|
||||
},
|
||||
{
|
||||
"tool": "detect_changes",
|
||||
"suggestion": "See how recent changes affect the architecture",
|
||||
},
|
||||
{
|
||||
"tool": "list_flows",
|
||||
"suggestion": "Explore execution flows",
|
||||
},
|
||||
],
|
||||
"detect_changes": [
|
||||
{
|
||||
"tool": "get_review_context",
|
||||
"suggestion": "Build a full review context with source snippets",
|
||||
},
|
||||
{
|
||||
"tool": "get_affected_flows",
|
||||
"suggestion": "See which execution flows are affected",
|
||||
},
|
||||
{
|
||||
"tool": "get_impact_radius",
|
||||
"suggestion": "Expand the blast radius analysis",
|
||||
},
|
||||
{
|
||||
"tool": "refactor",
|
||||
"suggestion": "Look for refactoring opportunities in changed code",
|
||||
},
|
||||
],
|
||||
"refactor": [
|
||||
{
|
||||
"tool": "query_graph",
|
||||
"suggestion": "Verify call sites before applying a rename",
|
||||
},
|
||||
{
|
||||
"tool": "detect_changes",
|
||||
"suggestion": "Check risk of the refactored code",
|
||||
},
|
||||
{
|
||||
"tool": "semantic_search_nodes",
|
||||
"suggestion": "Find related symbols to also rename",
|
||||
},
|
||||
],
|
||||
"semantic_search_nodes": [
|
||||
{
|
||||
"tool": "query_graph",
|
||||
"suggestion": "Inspect callers/callees of a search result",
|
||||
},
|
||||
{
|
||||
"tool": "get_flow",
|
||||
"suggestion": "See the execution flow through a matched node",
|
||||
},
|
||||
{
|
||||
"tool": "get_impact_radius",
|
||||
"suggestion": "Check the blast radius from matched nodes",
|
||||
},
|
||||
],
|
||||
}
|
||||
|
||||
# Maximum items per hints category returned to the caller.
|
||||
_MAX_PER_CATEGORY = 3
|
||||
|
||||
# Session history caps.
|
||||
_MAX_TOOLS_HISTORY = 100
|
||||
_MAX_NODES_TRACKED = 1000
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# SessionState
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class SessionState:
|
||||
"""In-memory session state for a single MCP connection."""
|
||||
|
||||
def __init__(self) -> None:
|
||||
self.tools_called: deque[str] = deque(maxlen=_MAX_TOOLS_HISTORY)
|
||||
self.nodes_queried: set[str] = set()
|
||||
self.files_touched: set[str] = set()
|
||||
self.inferred_intent: str | None = None
|
||||
self.last_tool_time: float = 0.0
|
||||
|
||||
def record_tool_call(self, tool_name: str) -> None:
|
||||
"""Record a tool invocation (FIFO, capped at 100)."""
|
||||
self.tools_called.append(tool_name)
|
||||
self.last_tool_time = time.time()
|
||||
|
||||
def record_nodes(self, node_ids: list[str]) -> None:
|
||||
"""Record queried node identifiers (capped at 1000)."""
|
||||
for nid in node_ids:
|
||||
if len(self.nodes_queried) >= _MAX_NODES_TRACKED:
|
||||
break
|
||||
self.nodes_queried.add(nid)
|
||||
|
||||
def record_files(self, files: list[str]) -> None:
|
||||
"""Record touched file paths."""
|
||||
self.files_touched.update(files)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Intent inference
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def infer_intent(session: SessionState) -> str:
|
||||
"""Classify the user's likely intent from their tool-call history.
|
||||
|
||||
Returns one of: ``"reviewing"``, ``"debugging"``, ``"refactoring"``,
|
||||
``"exploring"`` (default).
|
||||
"""
|
||||
if not session.tools_called:
|
||||
return "exploring"
|
||||
|
||||
# Score each intent by how many of the last N calls match its tools.
|
||||
recent = list(session.tools_called)[-10:]
|
||||
scores: dict[str, int] = {intent: 0 for intent in _INTENT_TOOLS}
|
||||
for tool in recent:
|
||||
for intent, tools in _INTENT_TOOLS.items():
|
||||
if tool in tools:
|
||||
scores[intent] += 1
|
||||
|
||||
best = max(scores, key=lambda k: scores[k])
|
||||
if scores[best] == 0:
|
||||
return "exploring"
|
||||
return best
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Hints generation
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def generate_hints(
|
||||
tool_name: str,
|
||||
result: dict[str, Any],
|
||||
session: SessionState,
|
||||
) -> dict[str, Any]:
|
||||
"""Build context-aware hints for a tool response.
|
||||
|
||||
Returns::
|
||||
|
||||
{
|
||||
"next_steps": [{"tool": ..., "suggestion": ...}, ...],
|
||||
"related": [...],
|
||||
"warnings": [...],
|
||||
}
|
||||
|
||||
At most ``_MAX_PER_CATEGORY`` items per list. Tools already called
|
||||
in this session are suppressed from ``next_steps``.
|
||||
"""
|
||||
# Update session state.
|
||||
session.record_tool_call(tool_name)
|
||||
session.inferred_intent = infer_intent(session)
|
||||
|
||||
next_steps = _build_next_steps(tool_name, session)
|
||||
warnings = _extract_warnings(result)
|
||||
# Build related BEFORE tracking, so that the current result's files
|
||||
# are not yet in files_touched and can appear as suggestions.
|
||||
related = _build_related(tool_name, result, session)
|
||||
|
||||
# Collect files/nodes from result for session tracking.
|
||||
_track_result(result, session)
|
||||
|
||||
return {
|
||||
"next_steps": next_steps[:_MAX_PER_CATEGORY],
|
||||
"related": related[:_MAX_PER_CATEGORY],
|
||||
"warnings": warnings[:_MAX_PER_CATEGORY],
|
||||
}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Internal helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _track_result(result: dict[str, Any], session: SessionState) -> None:
|
||||
"""Extract node IDs and file paths from a tool result and record them."""
|
||||
# Files
|
||||
for key in ("changed_files", "impacted_files"):
|
||||
files = result.get(key)
|
||||
if isinstance(files, list):
|
||||
session.record_files([f for f in files if isinstance(f, str)])
|
||||
|
||||
# Nodes — look in common result shapes
|
||||
node_ids: list[str] = []
|
||||
for key in ("results", "changed_nodes", "impacted_nodes"):
|
||||
items = result.get(key)
|
||||
if isinstance(items, list):
|
||||
for item in items:
|
||||
if isinstance(item, dict):
|
||||
qn = item.get("qualified_name")
|
||||
if qn:
|
||||
node_ids.append(qn)
|
||||
if node_ids:
|
||||
session.record_nodes(node_ids)
|
||||
|
||||
|
||||
def _build_next_steps(
|
||||
tool_name: str, session: SessionState
|
||||
) -> list[dict[str, str]]:
|
||||
"""Return next-step suggestions, filtering already-called tools."""
|
||||
called = set(session.tools_called)
|
||||
candidates = _WORKFLOW.get(tool_name, [])
|
||||
out: list[dict[str, str]] = []
|
||||
for c in candidates:
|
||||
if c["tool"] not in called:
|
||||
out.append(c)
|
||||
return out
|
||||
|
||||
|
||||
def _extract_warnings(result: dict[str, Any]) -> list[str]:
|
||||
"""Pull warning signals from a tool result."""
|
||||
warnings: list[str] = []
|
||||
|
||||
# Test gaps
|
||||
test_gaps = result.get("test_gaps")
|
||||
if isinstance(test_gaps, list) and test_gaps:
|
||||
names = [g.get("name", g) if isinstance(g, dict) else str(g) for g in test_gaps[:5]]
|
||||
warnings.append(
|
||||
f"Test coverage gaps: {', '.join(names)}"
|
||||
)
|
||||
|
||||
# High risk score
|
||||
risk = result.get("risk_score")
|
||||
if isinstance(risk, (int, float)) and risk > 0.7:
|
||||
warnings.append(f"High risk score ({risk:.2f}) — review carefully")
|
||||
|
||||
# Coupling warnings from architecture overview
|
||||
arch_warnings = result.get("warnings")
|
||||
if isinstance(arch_warnings, list):
|
||||
for w in arch_warnings[:3]:
|
||||
if isinstance(w, str):
|
||||
warnings.append(w)
|
||||
elif isinstance(w, dict) and "message" in w:
|
||||
warnings.append(w["message"])
|
||||
|
||||
return warnings
|
||||
|
||||
|
||||
def _build_related(
|
||||
tool_name: str,
|
||||
result: dict[str, Any],
|
||||
session: SessionState,
|
||||
) -> list[str]:
|
||||
"""Suggest related node/file identifiers from the result."""
|
||||
related: list[str] = []
|
||||
seen: set[str] = set()
|
||||
|
||||
# Suggest impacted files the user hasn't touched yet
|
||||
impacted = result.get("impacted_files")
|
||||
if isinstance(impacted, list):
|
||||
for f in impacted:
|
||||
if isinstance(f, str) and f not in session.files_touched and f not in seen:
|
||||
related.append(f)
|
||||
seen.add(f)
|
||||
if len(related) >= _MAX_PER_CATEGORY:
|
||||
break
|
||||
|
||||
return related
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Module-level session singleton
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
_session = SessionState()
|
||||
|
||||
|
||||
def get_session() -> SessionState:
|
||||
"""Return the global in-memory session state."""
|
||||
return _session
|
||||
|
||||
|
||||
def reset_session() -> None:
|
||||
"""Reset the global session (useful for testing)."""
|
||||
global _session
|
||||
_session = SessionState()
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,303 @@
|
||||
"""Post-build Jedi enrichment for Python call resolution.
|
||||
|
||||
After tree-sitter parsing, many method calls on lowercase-receiver variables
|
||||
are dropped (e.g. ``svc.authenticate()`` where ``svc = factory()``). Jedi
|
||||
can resolve these by tracing return types across files.
|
||||
|
||||
This module runs as a post-build step: it re-walks Python ASTs to find
|
||||
dropped calls, uses ``jedi.Script.goto()`` to resolve them, and adds the
|
||||
resulting CALLS edges to the graph database.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import os
|
||||
from pathlib import Path
|
||||
from typing import Optional
|
||||
|
||||
from .parser import CodeParser, EdgeInfo
|
||||
from .parser import _is_test_file as _parser_is_test_file
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_SELF_NAMES = frozenset({"self", "cls", "super"})
|
||||
|
||||
|
||||
def enrich_jedi_calls(store, repo_root: Path) -> dict:
|
||||
"""Resolve untracked Python method calls via Jedi.
|
||||
|
||||
Walks Python files, finds ``receiver.method()`` calls that tree-sitter
|
||||
dropped (lowercase receiver, not self/cls), resolves them with Jedi,
|
||||
and inserts new CALLS edges.
|
||||
|
||||
Returns stats dict with ``resolved`` count.
|
||||
"""
|
||||
try:
|
||||
import jedi
|
||||
except ImportError:
|
||||
logger.info("Jedi not installed, skipping Python enrichment")
|
||||
return {"skipped": True, "reason": "jedi not installed"}
|
||||
|
||||
repo_root = Path(repo_root).resolve()
|
||||
|
||||
# Get Python files from the graph — skip early if none
|
||||
all_files = store.get_all_files()
|
||||
py_files = [f for f in all_files if f.endswith(".py")]
|
||||
|
||||
if not py_files:
|
||||
return {"resolved": 0, "files": 0}
|
||||
|
||||
# Scope the Jedi project to Python-only directories to avoid scanning
|
||||
# non-Python files (e.g. node_modules, TS sources). This matters for
|
||||
# polyglot monorepos where jedi.Project(path=repo_root) would scan
|
||||
# thousands of irrelevant files during initialization.
|
||||
py_dirs = sorted({str(Path(f).parent) for f in py_files})
|
||||
common_py_root = Path(os.path.commonpath(py_dirs)) if py_dirs else repo_root
|
||||
if not str(common_py_root).startswith(str(repo_root)):
|
||||
common_py_root = repo_root
|
||||
project = jedi.Project(
|
||||
path=str(common_py_root),
|
||||
added_sys_path=[str(repo_root)],
|
||||
smart_sys_path=False,
|
||||
)
|
||||
|
||||
# Pre-parse all Python files to find which ones have pending method calls.
|
||||
# This avoids expensive Jedi Script creation for files with nothing to resolve.
|
||||
parser = CodeParser()
|
||||
ts_parser = parser._get_parser("python")
|
||||
if not ts_parser:
|
||||
return {"resolved": 0, "files": 0}
|
||||
|
||||
# Build set of method names that actually exist in project code.
|
||||
# No point asking Jedi to resolve `logger.getLogger()` if no project
|
||||
# file defines a function called `getLogger`.
|
||||
project_func_names = {
|
||||
r["name"]
|
||||
for r in store._conn.execute(
|
||||
"SELECT DISTINCT name FROM nodes WHERE kind IN ('Function', 'Test')"
|
||||
).fetchall()
|
||||
}
|
||||
|
||||
files_with_pending: list[tuple[str, bytes, list]] = []
|
||||
total_skipped = 0
|
||||
for file_path in py_files:
|
||||
try:
|
||||
source = Path(file_path).read_bytes()
|
||||
except (OSError, PermissionError):
|
||||
continue
|
||||
tree = ts_parser.parse(source)
|
||||
is_test = _parser_is_test_file(file_path)
|
||||
pending = _find_untracked_method_calls(tree.root_node, is_test)
|
||||
if pending:
|
||||
# Only keep calls whose method name exists in project code
|
||||
filtered = [p for p in pending if p[2] in project_func_names]
|
||||
total_skipped += len(pending) - len(filtered)
|
||||
if filtered:
|
||||
files_with_pending.append((file_path, source, filtered))
|
||||
|
||||
if not files_with_pending:
|
||||
return {"resolved": 0, "files": 0}
|
||||
|
||||
logger.debug(
|
||||
"Jedi: %d/%d Python files have pending calls (%d calls skipped — no project target)",
|
||||
len(files_with_pending), len(py_files), total_skipped,
|
||||
)
|
||||
|
||||
resolved_count = 0
|
||||
files_enriched = 0
|
||||
errors = 0
|
||||
|
||||
for file_path, source, pending in files_with_pending:
|
||||
source_text = source.decode("utf-8", errors="replace")
|
||||
|
||||
# Get existing CALLS edges for this file to skip duplicates
|
||||
existing = set()
|
||||
for edge in _get_file_call_edges(store, file_path):
|
||||
existing.add((edge.source_qualified, edge.line))
|
||||
|
||||
# Get function nodes from DB for enclosing-function lookup
|
||||
func_nodes = [
|
||||
n for n in store.get_nodes_by_file(file_path)
|
||||
if n.kind in ("Function", "Test")
|
||||
]
|
||||
|
||||
# Create Jedi script once per file
|
||||
try:
|
||||
script = jedi.Script(source_text, path=file_path, project=project)
|
||||
except Exception as e:
|
||||
logger.debug("Jedi failed to load %s: %s", file_path, e)
|
||||
errors += 1
|
||||
continue
|
||||
|
||||
file_resolved = 0
|
||||
for jedi_line, col, _method_name, _enclosing_name in pending:
|
||||
# Find enclosing function qualified name
|
||||
enclosing = _find_enclosing(func_nodes, jedi_line)
|
||||
if not enclosing:
|
||||
enclosing = file_path # module-level
|
||||
|
||||
# Skip if we already have a CALLS edge from this source at this line
|
||||
if (enclosing, jedi_line) in existing:
|
||||
continue
|
||||
|
||||
# Ask Jedi to resolve
|
||||
try:
|
||||
names = script.goto(jedi_line, col)
|
||||
except Exception: # nosec B112 - Jedi may fail on malformed code
|
||||
continue
|
||||
|
||||
if not names:
|
||||
continue
|
||||
|
||||
name = names[0]
|
||||
if not name.module_path:
|
||||
continue
|
||||
|
||||
module_path = Path(name.module_path).resolve()
|
||||
|
||||
# Only emit edges for project-internal definitions
|
||||
try:
|
||||
module_path.relative_to(repo_root)
|
||||
except ValueError:
|
||||
continue
|
||||
|
||||
# Build qualified target: file_path::Class.method or file_path::func
|
||||
target_file = str(module_path)
|
||||
parent = name.parent()
|
||||
if parent and parent.type == "class":
|
||||
target = f"{target_file}::{parent.name}.{name.name}"
|
||||
else:
|
||||
target = f"{target_file}::{name.name}"
|
||||
|
||||
store.upsert_edge(EdgeInfo(
|
||||
kind="CALLS",
|
||||
source=enclosing,
|
||||
target=target,
|
||||
file_path=file_path,
|
||||
line=jedi_line,
|
||||
))
|
||||
existing.add((enclosing, jedi_line))
|
||||
file_resolved += 1
|
||||
|
||||
if file_resolved:
|
||||
files_enriched += 1
|
||||
resolved_count += file_resolved
|
||||
|
||||
if resolved_count:
|
||||
store.commit()
|
||||
logger.info(
|
||||
"Jedi enrichment: resolved %d calls in %d files",
|
||||
resolved_count, files_enriched,
|
||||
)
|
||||
|
||||
return {
|
||||
"resolved": resolved_count,
|
||||
"files": files_enriched,
|
||||
"errors": errors,
|
||||
}
|
||||
|
||||
|
||||
def _get_file_call_edges(store, file_path: str):
|
||||
"""Get all CALLS edges originating from a file."""
|
||||
conn = store._conn
|
||||
rows = conn.execute(
|
||||
"SELECT * FROM edges WHERE file_path = ? AND kind = 'CALLS'",
|
||||
(file_path,),
|
||||
).fetchall()
|
||||
from .graph import GraphEdge
|
||||
return [
|
||||
GraphEdge(
|
||||
id=r["id"], kind=r["kind"],
|
||||
source_qualified=r["source_qualified"],
|
||||
target_qualified=r["target_qualified"],
|
||||
file_path=r["file_path"], line=r["line"],
|
||||
extra={},
|
||||
)
|
||||
for r in rows
|
||||
]
|
||||
|
||||
|
||||
def _find_enclosing(func_nodes, line: int) -> Optional[str]:
|
||||
"""Find the qualified name of the function enclosing a given line."""
|
||||
best = None
|
||||
best_span = float("inf")
|
||||
for node in func_nodes:
|
||||
if node.line_start <= line <= node.line_end:
|
||||
span = node.line_end - node.line_start
|
||||
if span < best_span:
|
||||
best = node.qualified_name
|
||||
best_span = span
|
||||
return best
|
||||
|
||||
|
||||
def _find_untracked_method_calls(root, is_test_file: bool = False):
|
||||
"""Walk Python AST to find method calls the parser would have dropped.
|
||||
|
||||
Returns list of (jedi_line, col, method_name, enclosing_func_name) tuples.
|
||||
Jedi_line is 1-indexed, col is 0-indexed.
|
||||
"""
|
||||
results: list[tuple[int, int, str, Optional[str]]] = []
|
||||
_walk_calls(root, results, is_test_file, enclosing_func=None)
|
||||
return results
|
||||
|
||||
|
||||
def _walk_calls(node, results, is_test_file, enclosing_func):
|
||||
"""Recursively walk AST collecting dropped method calls."""
|
||||
# Track enclosing function scope
|
||||
if node.type == "function_definition":
|
||||
name = None
|
||||
for child in node.children:
|
||||
if child.type == "identifier":
|
||||
name = child.text.decode("utf-8", errors="replace")
|
||||
break
|
||||
for child in node.children:
|
||||
_walk_calls(child, results, is_test_file, name or enclosing_func)
|
||||
return
|
||||
|
||||
if node.type == "decorated_definition":
|
||||
for child in node.children:
|
||||
_walk_calls(child, results, is_test_file, enclosing_func)
|
||||
return
|
||||
|
||||
# Check for call expressions with attribute access
|
||||
if node.type == "call":
|
||||
first = node.children[0] if node.children else None
|
||||
if first and first.type == "attribute":
|
||||
_check_dropped_call(first, results, is_test_file, enclosing_func)
|
||||
|
||||
for child in node.children:
|
||||
_walk_calls(child, results, is_test_file, enclosing_func)
|
||||
|
||||
|
||||
def _check_dropped_call(attr_node, results, is_test_file, enclosing_func):
|
||||
"""Check if an attribute-based call was dropped by the parser."""
|
||||
children = attr_node.children
|
||||
if len(children) < 2:
|
||||
return
|
||||
|
||||
receiver = children[0]
|
||||
# Only handle simple identifier receivers
|
||||
if receiver.type != "identifier":
|
||||
return
|
||||
|
||||
receiver_text = receiver.text.decode("utf-8", errors="replace")
|
||||
|
||||
# The parser keeps: self/cls/super calls and uppercase-receiver calls
|
||||
# The parser keeps: calls handled by typed-var enrichment (but those are
|
||||
# separate edges -- we check for duplicates via existing-edge set)
|
||||
if receiver_text in _SELF_NAMES:
|
||||
return
|
||||
if receiver_text[:1].isupper():
|
||||
return
|
||||
if is_test_file:
|
||||
return # test files already track all calls
|
||||
|
||||
# Find the method name identifier
|
||||
method_node = children[-1]
|
||||
if method_node.type != "identifier":
|
||||
return
|
||||
|
||||
row, col = method_node.start_point # 0-indexed
|
||||
method_name = method_node.text.decode("utf-8", errors="replace")
|
||||
results.append((row + 1, col, method_name, enclosing_func))
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,142 @@
|
||||
"""Memory/feedback loop -- persist Q&A results for graph enrichment."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import re
|
||||
import time
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def save_result(
|
||||
question: str,
|
||||
answer: str,
|
||||
nodes: list[str] | None = None,
|
||||
result_type: str = "query",
|
||||
memory_dir: Path | None = None,
|
||||
repo_root: Path | None = None,
|
||||
) -> Path:
|
||||
"""Save a Q&A result as markdown for re-ingestion.
|
||||
|
||||
Args:
|
||||
question: The question that was asked.
|
||||
answer: The answer/result.
|
||||
nodes: Related node qualified names.
|
||||
result_type: Type of result (query, review, debug).
|
||||
memory_dir: Directory to save to. Defaults to
|
||||
<repo>/.code-review-graph/memory/
|
||||
repo_root: Repository root for default memory_dir.
|
||||
|
||||
Returns:
|
||||
Path to the saved file.
|
||||
"""
|
||||
if memory_dir is None:
|
||||
if repo_root is None:
|
||||
raise ValueError(
|
||||
"Either memory_dir or repo_root required"
|
||||
)
|
||||
memory_dir = (
|
||||
repo_root / ".code-review-graph" / "memory"
|
||||
)
|
||||
|
||||
memory_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
# Generate filename from question
|
||||
slug = re.sub(r"[^\w\s-]", "", question.lower())
|
||||
slug = re.sub(r"[\s_]+", "-", slug).strip("-")[:60]
|
||||
timestamp = int(time.time())
|
||||
filename = f"{slug}-{timestamp}.md"
|
||||
|
||||
# Build markdown with YAML frontmatter
|
||||
lines = [
|
||||
"---",
|
||||
f"type: {result_type}",
|
||||
f"timestamp: {timestamp}",
|
||||
]
|
||||
if nodes:
|
||||
lines.append("nodes:")
|
||||
for n in nodes[:20]:
|
||||
lines.append(f" - {n}")
|
||||
lines.extend([
|
||||
"---",
|
||||
"",
|
||||
f"# {question}",
|
||||
"",
|
||||
answer,
|
||||
])
|
||||
|
||||
path = memory_dir / filename
|
||||
path.write_text("\n".join(lines), encoding="utf-8")
|
||||
logger.info("Saved result to %s", path)
|
||||
return path
|
||||
|
||||
|
||||
def list_memories(
|
||||
memory_dir: Path | None = None,
|
||||
repo_root: Path | None = None,
|
||||
) -> list[dict[str, Any]]:
|
||||
"""List all saved memory files.
|
||||
|
||||
Returns list of dicts with: path, question, type, timestamp.
|
||||
"""
|
||||
if memory_dir is None:
|
||||
if repo_root is None:
|
||||
return []
|
||||
memory_dir = (
|
||||
repo_root / ".code-review-graph" / "memory"
|
||||
)
|
||||
|
||||
if not memory_dir.exists():
|
||||
return []
|
||||
|
||||
results = []
|
||||
for f in sorted(memory_dir.glob("*.md")):
|
||||
try:
|
||||
text = f.read_text(encoding="utf-8")
|
||||
# Parse frontmatter
|
||||
meta: dict[str, Any] = {"path": str(f)}
|
||||
if text.startswith("---"):
|
||||
parts = text.split("---", 2)
|
||||
if len(parts) >= 3:
|
||||
fm_lines = parts[1].strip().split("\n")
|
||||
for line in fm_lines:
|
||||
if ": " in line and not line.startswith(" "):
|
||||
k, v = line.split(": ", 1)
|
||||
meta[k.strip()] = v.strip()
|
||||
# Extract question from first heading
|
||||
for line in text.split("\n"):
|
||||
if line.startswith("# "):
|
||||
meta["question"] = line[2:].strip()
|
||||
break
|
||||
results.append(meta)
|
||||
except OSError:
|
||||
continue
|
||||
|
||||
return results
|
||||
|
||||
|
||||
def clear_memories(
|
||||
memory_dir: Path | None = None,
|
||||
repo_root: Path | None = None,
|
||||
) -> int:
|
||||
"""Delete all memory files. Returns count deleted."""
|
||||
if memory_dir is None:
|
||||
if repo_root is None:
|
||||
return 0
|
||||
memory_dir = (
|
||||
repo_root / ".code-review-graph" / "memory"
|
||||
)
|
||||
|
||||
if not memory_dir.exists():
|
||||
return 0
|
||||
|
||||
count = 0
|
||||
for f in memory_dir.glob("*.md"):
|
||||
f.unlink()
|
||||
count += 1
|
||||
|
||||
logger.info("Cleared %d memory files", count)
|
||||
return count
|
||||
@@ -0,0 +1,284 @@
|
||||
"""Schema migration framework for the code-review-graph SQLite database.
|
||||
|
||||
Manages incremental schema changes via versioned migration functions.
|
||||
Each migration is idempotent (uses IF NOT EXISTS / column existence checks).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import sqlite3
|
||||
from typing import Callable
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def get_schema_version(conn: sqlite3.Connection) -> int:
|
||||
"""Read the current schema version from the metadata table.
|
||||
|
||||
Returns:
|
||||
int: The schema version (0 if metadata table doesn't exist, 1 if not set).
|
||||
"""
|
||||
try:
|
||||
row = conn.execute(
|
||||
"SELECT value FROM metadata WHERE key = 'schema_version'"
|
||||
).fetchone()
|
||||
if row is None:
|
||||
return 1
|
||||
return int(row[0] if isinstance(row, (tuple, list)) else row["value"])
|
||||
except sqlite3.OperationalError:
|
||||
# metadata table doesn't exist
|
||||
return 0
|
||||
|
||||
|
||||
def _set_schema_version(conn: sqlite3.Connection, version: int) -> None:
|
||||
"""Set the schema version in the metadata table."""
|
||||
conn.execute(
|
||||
"INSERT OR REPLACE INTO metadata (key, value) VALUES ('schema_version', ?)",
|
||||
(str(version),),
|
||||
)
|
||||
|
||||
|
||||
_KNOWN_TABLES = frozenset({
|
||||
"nodes", "edges", "metadata", "communities", "flows", "flow_memberships", "nodes_fts",
|
||||
"community_summaries", "flow_snapshots", "risk_index",
|
||||
})
|
||||
|
||||
|
||||
def _has_column(conn: sqlite3.Connection, table: str, column: str) -> bool:
|
||||
"""Check if a column exists in a table."""
|
||||
if table not in _KNOWN_TABLES:
|
||||
raise ValueError(f"Unknown table: {table}")
|
||||
cursor = conn.execute(f"PRAGMA table_info({table})") # noqa: S608
|
||||
columns = [row[1] if isinstance(row, tuple) else row["name"] for row in cursor]
|
||||
return column in columns
|
||||
|
||||
|
||||
def _table_exists(conn: sqlite3.Connection, table: str) -> bool:
|
||||
"""Check if a table exists."""
|
||||
if table not in _KNOWN_TABLES:
|
||||
raise ValueError(f"Unknown table: {table}")
|
||||
row = conn.execute(
|
||||
"SELECT count(*) FROM sqlite_master WHERE type IN ('table', 'view') "
|
||||
"AND name = ?",
|
||||
(table,),
|
||||
).fetchone()
|
||||
return row[0] > 0
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Migration functions
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _migrate_v2(conn: sqlite3.Connection) -> None:
|
||||
"""v2: Add signature column to nodes table."""
|
||||
if not _has_column(conn, "nodes", "signature"):
|
||||
conn.execute("ALTER TABLE nodes ADD COLUMN signature TEXT")
|
||||
logger.info("Migration v2: added 'signature' column to nodes")
|
||||
|
||||
|
||||
def _migrate_v3(conn: sqlite3.Connection) -> None:
|
||||
"""v3: Create flows and flow_memberships tables."""
|
||||
conn.execute("""
|
||||
CREATE TABLE IF NOT EXISTS flows (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
name TEXT NOT NULL,
|
||||
entry_point_id INTEGER NOT NULL,
|
||||
depth INTEGER NOT NULL,
|
||||
node_count INTEGER NOT NULL,
|
||||
file_count INTEGER NOT NULL,
|
||||
criticality REAL NOT NULL DEFAULT 0.0,
|
||||
path_json TEXT NOT NULL,
|
||||
created_at TEXT NOT NULL DEFAULT (datetime('now')),
|
||||
updated_at TEXT NOT NULL DEFAULT (datetime('now'))
|
||||
)
|
||||
""")
|
||||
conn.execute("""
|
||||
CREATE TABLE IF NOT EXISTS flow_memberships (
|
||||
flow_id INTEGER NOT NULL,
|
||||
node_id INTEGER NOT NULL,
|
||||
position INTEGER NOT NULL,
|
||||
PRIMARY KEY (flow_id, node_id)
|
||||
)
|
||||
""")
|
||||
conn.execute(
|
||||
"CREATE INDEX IF NOT EXISTS idx_flows_criticality ON flows(criticality DESC)"
|
||||
)
|
||||
conn.execute(
|
||||
"CREATE INDEX IF NOT EXISTS idx_flows_entry ON flows(entry_point_id)"
|
||||
)
|
||||
conn.execute(
|
||||
"CREATE INDEX IF NOT EXISTS idx_flow_memberships_node ON flow_memberships(node_id)"
|
||||
)
|
||||
logger.info("Migration v3: created flows and flow_memberships tables")
|
||||
|
||||
|
||||
def _migrate_v4(conn: sqlite3.Connection) -> None:
|
||||
"""v4: Create communities table, add community_id to nodes."""
|
||||
conn.execute("""
|
||||
CREATE TABLE IF NOT EXISTS communities (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
name TEXT NOT NULL,
|
||||
level INTEGER NOT NULL DEFAULT 0,
|
||||
parent_id INTEGER,
|
||||
cohesion REAL NOT NULL DEFAULT 0.0,
|
||||
size INTEGER NOT NULL DEFAULT 0,
|
||||
dominant_language TEXT,
|
||||
description TEXT,
|
||||
created_at TEXT NOT NULL DEFAULT (datetime('now'))
|
||||
)
|
||||
""")
|
||||
if not _has_column(conn, "nodes", "community_id"):
|
||||
conn.execute("ALTER TABLE nodes ADD COLUMN community_id INTEGER")
|
||||
logger.info("Migration v4: added 'community_id' column to nodes")
|
||||
conn.execute(
|
||||
"CREATE INDEX IF NOT EXISTS idx_nodes_community ON nodes(community_id)"
|
||||
)
|
||||
conn.execute(
|
||||
"CREATE INDEX IF NOT EXISTS idx_communities_parent ON communities(parent_id)"
|
||||
)
|
||||
conn.execute(
|
||||
"CREATE INDEX IF NOT EXISTS idx_communities_cohesion ON communities(cohesion DESC)"
|
||||
)
|
||||
logger.info("Migration v4: created communities table")
|
||||
|
||||
|
||||
def _migrate_v5(conn: sqlite3.Connection) -> None:
|
||||
"""v5: Create FTS5 virtual table for nodes."""
|
||||
if not _table_exists(conn, "nodes_fts"):
|
||||
conn.execute("""
|
||||
CREATE VIRTUAL TABLE nodes_fts USING fts5(
|
||||
name, qualified_name, file_path, signature,
|
||||
content='nodes', content_rowid='rowid',
|
||||
tokenize='porter unicode61'
|
||||
)
|
||||
""")
|
||||
logger.info("Migration v5: created nodes_fts FTS5 virtual table")
|
||||
|
||||
|
||||
def _migrate_v6(conn: sqlite3.Connection) -> None:
|
||||
"""v6: Add pre-computed summary tables for token-efficient queries."""
|
||||
conn.execute("""
|
||||
CREATE TABLE IF NOT EXISTS community_summaries (
|
||||
community_id INTEGER PRIMARY KEY,
|
||||
name TEXT NOT NULL,
|
||||
purpose TEXT DEFAULT '',
|
||||
key_symbols TEXT DEFAULT '[]',
|
||||
risk TEXT DEFAULT 'unknown',
|
||||
size INTEGER DEFAULT 0,
|
||||
dominant_language TEXT DEFAULT '',
|
||||
FOREIGN KEY (community_id) REFERENCES communities(id)
|
||||
)
|
||||
""")
|
||||
conn.execute("""
|
||||
CREATE TABLE IF NOT EXISTS flow_snapshots (
|
||||
flow_id INTEGER PRIMARY KEY,
|
||||
name TEXT NOT NULL,
|
||||
entry_point TEXT NOT NULL,
|
||||
critical_path TEXT DEFAULT '[]',
|
||||
criticality REAL DEFAULT 0.0,
|
||||
node_count INTEGER DEFAULT 0,
|
||||
file_count INTEGER DEFAULT 0,
|
||||
FOREIGN KEY (flow_id) REFERENCES flows(id)
|
||||
)
|
||||
""")
|
||||
conn.execute("""
|
||||
CREATE TABLE IF NOT EXISTS risk_index (
|
||||
node_id INTEGER PRIMARY KEY,
|
||||
qualified_name TEXT NOT NULL,
|
||||
risk_score REAL DEFAULT 0.0,
|
||||
caller_count INTEGER DEFAULT 0,
|
||||
test_coverage TEXT DEFAULT 'unknown',
|
||||
security_relevant INTEGER DEFAULT 0,
|
||||
last_computed TEXT DEFAULT '',
|
||||
FOREIGN KEY (node_id) REFERENCES nodes(id)
|
||||
)
|
||||
""")
|
||||
conn.execute(
|
||||
"CREATE INDEX IF NOT EXISTS idx_risk_index_score "
|
||||
"ON risk_index(risk_score DESC)"
|
||||
)
|
||||
logger.info("Migration v6: created summary tables "
|
||||
"(community_summaries, flow_snapshots, risk_index)")
|
||||
|
||||
|
||||
def _migrate_v7(conn: sqlite3.Connection) -> None:
|
||||
"""v7: Add compound edge indexes for summary and risk queries."""
|
||||
conn.execute(
|
||||
"CREATE INDEX IF NOT EXISTS idx_edges_target_kind "
|
||||
"ON edges(target_qualified, kind)"
|
||||
)
|
||||
conn.execute(
|
||||
"CREATE INDEX IF NOT EXISTS idx_edges_source_kind "
|
||||
"ON edges(source_qualified, kind)"
|
||||
)
|
||||
logger.info("Migration v7: added compound edge indexes")
|
||||
|
||||
|
||||
def _migrate_v8(conn: sqlite3.Connection) -> None:
|
||||
"""v8: Add composite index on edges for upsert_edge performance."""
|
||||
conn.execute("""
|
||||
CREATE INDEX IF NOT EXISTS idx_edges_composite
|
||||
ON edges(kind, source_qualified, target_qualified, file_path, line)
|
||||
""")
|
||||
logger.info("Migration v8: created composite edge index")
|
||||
|
||||
|
||||
def _migrate_v9(conn: sqlite3.Connection) -> None:
|
||||
"""v9: Add confidence scoring to edges."""
|
||||
if not _has_column(conn, "edges", "confidence"):
|
||||
conn.execute(
|
||||
"ALTER TABLE edges ADD COLUMN confidence REAL DEFAULT 1.0"
|
||||
)
|
||||
if not _has_column(conn, "edges", "confidence_tier"):
|
||||
conn.execute(
|
||||
"ALTER TABLE edges ADD COLUMN confidence_tier TEXT DEFAULT 'EXTRACTED'"
|
||||
)
|
||||
logger.info("Migration v9: added edge confidence columns")
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Migration registry
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
MIGRATIONS: dict[int, Callable[[sqlite3.Connection], None]] = {
|
||||
2: _migrate_v2,
|
||||
3: _migrate_v3,
|
||||
4: _migrate_v4,
|
||||
5: _migrate_v5,
|
||||
6: _migrate_v6,
|
||||
7: _migrate_v7,
|
||||
8: _migrate_v8,
|
||||
9: _migrate_v9,
|
||||
}
|
||||
|
||||
LATEST_VERSION = max(MIGRATIONS.keys())
|
||||
|
||||
|
||||
def run_migrations(conn: sqlite3.Connection) -> None:
|
||||
"""Run all pending migrations in order.
|
||||
|
||||
Each migration runs in its own transaction. The schema_version metadata
|
||||
entry is updated after each successful migration.
|
||||
"""
|
||||
current = get_schema_version(conn)
|
||||
if current >= LATEST_VERSION:
|
||||
return
|
||||
|
||||
logger.info("Schema version %d -> %d: running migrations", current, LATEST_VERSION)
|
||||
|
||||
for version in sorted(MIGRATIONS.keys()):
|
||||
if version <= current:
|
||||
continue
|
||||
logger.info("Running migration v%d", version)
|
||||
try:
|
||||
MIGRATIONS[version](conn)
|
||||
_set_schema_version(conn, version)
|
||||
conn.commit()
|
||||
except sqlite3.Error:
|
||||
conn.rollback()
|
||||
logger.error("Migration v%d failed, rolling back", version, exc_info=True)
|
||||
raise
|
||||
|
||||
logger.info("Migrations complete, now at schema version %d", LATEST_VERSION)
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,134 @@
|
||||
"""Shared post-build processing pipeline.
|
||||
|
||||
After the core Tree-sitter parse (full_build or incremental_update), four
|
||||
post-processing steps must run to populate derived tables:
|
||||
|
||||
1. Compute node signatures
|
||||
2. Rebuild FTS5 search index
|
||||
3. Trace execution flows
|
||||
4. Detect code communities
|
||||
|
||||
This module extracts that pipeline so every entry point — MCP tool, CLI
|
||||
commands, and watch mode — produces identical results.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import sqlite3
|
||||
from typing import Any
|
||||
|
||||
from .graph import GraphStore
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def run_post_processing(store: GraphStore) -> dict[str, Any]:
|
||||
"""Run all post-build steps on a populated graph.
|
||||
|
||||
Each step is non-fatal: failures are logged and collected as warnings
|
||||
so the primary build result is never lost.
|
||||
|
||||
Args:
|
||||
store: An open GraphStore with nodes and edges already populated.
|
||||
|
||||
Returns:
|
||||
Dict with keys for each step's result count and a ``warnings``
|
||||
list (only present when at least one step failed).
|
||||
"""
|
||||
result: dict[str, Any] = {}
|
||||
warnings: list[str] = []
|
||||
|
||||
_compute_signatures(store, result, warnings)
|
||||
_rebuild_fts_index(store, result, warnings)
|
||||
_trace_flows(store, result, warnings)
|
||||
_detect_communities(store, result, warnings)
|
||||
|
||||
if warnings:
|
||||
result["warnings"] = warnings
|
||||
return result
|
||||
|
||||
|
||||
# -- Individual steps (private) ------------------------------------------
|
||||
|
||||
|
||||
def _compute_signatures(
|
||||
store: GraphStore,
|
||||
result: dict[str, Any],
|
||||
warnings: list[str],
|
||||
) -> None:
|
||||
"""Compute human-readable signatures for nodes that lack one."""
|
||||
try:
|
||||
rows = store.get_nodes_without_signature()
|
||||
for row in rows:
|
||||
node_id, name, kind, params, ret = (
|
||||
row[0],
|
||||
row[1],
|
||||
row[2],
|
||||
row[3],
|
||||
row[4],
|
||||
)
|
||||
if kind in ("Function", "Test"):
|
||||
sig = f"def {name}({params or ''})"
|
||||
if ret:
|
||||
sig += f" -> {ret}"
|
||||
elif kind == "Class":
|
||||
sig = f"class {name}"
|
||||
else:
|
||||
sig = name
|
||||
store.update_node_signature(node_id, sig[:512])
|
||||
store.commit()
|
||||
result["signatures_computed"] = len(rows)
|
||||
except (sqlite3.OperationalError, TypeError, KeyError) as e:
|
||||
logger.warning("Signature computation failed: %s", e)
|
||||
warnings.append(f"Signature computation failed: {type(e).__name__}: {e}")
|
||||
|
||||
|
||||
def _rebuild_fts_index(
|
||||
store: GraphStore,
|
||||
result: dict[str, Any],
|
||||
warnings: list[str],
|
||||
) -> None:
|
||||
"""Rebuild the FTS5 full-text search index."""
|
||||
try:
|
||||
from .search import rebuild_fts_index
|
||||
|
||||
fts_count = rebuild_fts_index(store)
|
||||
result["fts_indexed"] = fts_count
|
||||
except (sqlite3.OperationalError, ImportError) as e:
|
||||
logger.warning("FTS index rebuild failed: %s", e)
|
||||
warnings.append(f"FTS index rebuild failed: {type(e).__name__}: {e}")
|
||||
|
||||
|
||||
def _trace_flows(
|
||||
store: GraphStore,
|
||||
result: dict[str, Any],
|
||||
warnings: list[str],
|
||||
) -> None:
|
||||
"""Trace execution flows from entry points."""
|
||||
try:
|
||||
from .flows import store_flows, trace_flows
|
||||
|
||||
flows = trace_flows(store)
|
||||
count = store_flows(store, flows)
|
||||
result["flows_detected"] = count
|
||||
except (sqlite3.OperationalError, ImportError) as e:
|
||||
logger.warning("Flow detection failed: %s", e)
|
||||
warnings.append(f"Flow detection failed: {type(e).__name__}: {e}")
|
||||
|
||||
|
||||
def _detect_communities(
|
||||
store: GraphStore,
|
||||
result: dict[str, Any],
|
||||
warnings: list[str],
|
||||
) -> None:
|
||||
"""Detect code communities via Leiden algorithm or file grouping."""
|
||||
try:
|
||||
from .communities import detect_communities, store_communities
|
||||
|
||||
comms = detect_communities(store)
|
||||
count = store_communities(store, comms)
|
||||
result["communities_detected"] = count
|
||||
except (sqlite3.OperationalError, ImportError) as e:
|
||||
logger.warning("Community detection failed: %s", e)
|
||||
warnings.append(f"Community detection failed: {type(e).__name__}: {e}")
|
||||
@@ -0,0 +1,159 @@
|
||||
"""MCP prompt templates for Code Review Graph.
|
||||
|
||||
Provides 5 pre-built prompt workflows, all enforcing token-efficient
|
||||
detail_level="minimal" first patterns with get_minimal_context entry point.
|
||||
|
||||
1. review_changes - pre-commit review using detect_changes + affected_flows
|
||||
2. architecture_map - architecture docs using communities, flows, Mermaid
|
||||
3. debug_issue - guided debugging using search, flow tracing
|
||||
4. onboard_developer - new dev orientation using stats, architecture, flows
|
||||
5. pre_merge_check - PR readiness with risk scoring, test gaps, dead code
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from fastmcp.prompts.prompt import Message
|
||||
|
||||
_TOKEN_EFFICIENCY_PREAMBLE = ( # nosec B105 — prompt template, not a password
|
||||
"""\
|
||||
## Rules for Token-Efficient Graph Usage
|
||||
1. ALWAYS call `get_minimal_context` first with a task description.
|
||||
2. Use `detail_level="minimal"` on all tool calls unless the minimal output \
|
||||
is insufficient.
|
||||
3. Only escalate to `detail_level="standard"` or `"verbose"` for the specific \
|
||||
entities that need deeper inspection.
|
||||
4. Never request more than 3 tool calls per turn unless absolutely necessary.
|
||||
5. Prefer targeted queries (query_graph with a specific symbol) over broad \
|
||||
scans (list_communities with full members).
|
||||
6. When reviewing changes: detect_changes(detail_level="minimal") → only \
|
||||
expand on high-risk items.
|
||||
"""
|
||||
)
|
||||
|
||||
|
||||
def _user(content: str) -> list[Message]:
|
||||
"""Wrap content as a single-message user prompt.
|
||||
|
||||
fastmcp >=3.2 rejects raw dicts in prompt return values; each message
|
||||
must be a ``Message`` instance (or a plain ``str``). We standardise on
|
||||
``Message`` so role is explicit and future multi-turn prompts compose
|
||||
naturally.
|
||||
"""
|
||||
return [Message(role="user", content=content)]
|
||||
|
||||
|
||||
def review_changes_prompt(base: str = "HEAD~1") -> list[Message]:
|
||||
"""Pre-commit review workflow.
|
||||
|
||||
Args:
|
||||
base: Git ref to diff against. Default: HEAD~1.
|
||||
"""
|
||||
return _user(
|
||||
f"{_TOKEN_EFFICIENCY_PREAMBLE}\n"
|
||||
f"## Review Workflow\n"
|
||||
f'1. Call `get_minimal_context(task="review changes against '
|
||||
f'{base}")` to get risk overview.\n'
|
||||
f'2. If risk is "low": call '
|
||||
f'`detect_changes(detail_level="minimal")` → report summary '
|
||||
f"+ any test gaps.\n"
|
||||
f'3. If risk is "medium" or "high":\n'
|
||||
f' a. Call `detect_changes(detail_level="standard")` for '
|
||||
f"full change list.\n"
|
||||
f" b. For each high-risk function, call "
|
||||
f'`query_graph(pattern="callers_of", target=<func>, '
|
||||
f'detail_level="minimal")`.\n'
|
||||
f' c. Call `get_affected_flows(detail_level="minimal")` '
|
||||
f"only if >3 changed functions.\n"
|
||||
f"4. Summarize: risk level, what changed, test gaps, "
|
||||
f"specific improvements needed.\n\n"
|
||||
f"Do NOT call get_review_context unless you need source code "
|
||||
f"snippets for a specific function."
|
||||
)
|
||||
|
||||
|
||||
def architecture_map_prompt() -> list[Message]:
|
||||
"""Architecture documentation workflow."""
|
||||
return _user(
|
||||
f"{_TOKEN_EFFICIENCY_PREAMBLE}\n"
|
||||
"## Architecture Mapping Workflow\n"
|
||||
'1. Call `get_minimal_context(task="map architecture")`.\n'
|
||||
'2. Call `get_architecture_overview(detail_level="minimal")` '
|
||||
"for community coupling summary.\n"
|
||||
'3. Call `list_flows(detail_level="minimal")` for critical '
|
||||
"flow names + criticality scores.\n"
|
||||
"4. Only call `get_community(name=<X>, "
|
||||
'detail_level="standard")` for the 1-2 communities the user '
|
||||
"is most interested in.\n"
|
||||
"5. Produce a concise Mermaid diagram showing communities as "
|
||||
"boxes and key flows as arrows."
|
||||
)
|
||||
|
||||
|
||||
def debug_issue_prompt(description: str = "") -> list[Message]:
|
||||
"""Guided debugging workflow.
|
||||
|
||||
Args:
|
||||
description: Description of the issue to debug.
|
||||
"""
|
||||
desc_part = description or "<description>"
|
||||
return _user(
|
||||
f"{_TOKEN_EFFICIENCY_PREAMBLE}\n"
|
||||
"## Debug Workflow\n"
|
||||
f'1. Call `get_minimal_context(task="debug: '
|
||||
f'{desc_part}")`.\n'
|
||||
"2. Call `semantic_search_nodes(query=<keywords from "
|
||||
'description>, detail_level="minimal", limit=5)`.\n'
|
||||
"3. For the top 1-2 results, call "
|
||||
'`query_graph(pattern="callers_of", target=<name>, '
|
||||
'detail_level="minimal")`.\n'
|
||||
"4. If the issue involves execution flow: call "
|
||||
"`get_flow(name=<relevant flow>)` for the single most "
|
||||
"relevant flow.\n"
|
||||
"5. Only call `get_review_context` or `get_impact_radius` "
|
||||
"if you need to trace the blast radius of a specific change."
|
||||
)
|
||||
|
||||
|
||||
def onboard_developer_prompt() -> list[Message]:
|
||||
"""New developer orientation workflow."""
|
||||
return _user(
|
||||
f"{_TOKEN_EFFICIENCY_PREAMBLE}\n"
|
||||
"## Onboarding Workflow\n"
|
||||
'1. Call `get_minimal_context(task="onboard developer")`.\n'
|
||||
"2. Call `list_graph_stats()` for technology overview.\n"
|
||||
'3. Call `get_architecture_overview(detail_level="minimal")` '
|
||||
"for the 30-second mental model.\n"
|
||||
'4. Call `list_communities(detail_level="minimal")` — '
|
||||
"present as a table of module names + sizes.\n"
|
||||
'5. Call `list_flows(detail_level="minimal")` — highlight '
|
||||
"the top 3 critical flows.\n"
|
||||
"6. Only drill into a specific community or flow if the "
|
||||
"developer asks."
|
||||
)
|
||||
|
||||
|
||||
def pre_merge_check_prompt(base: str = "HEAD~1") -> list[Message]:
|
||||
"""PR readiness check workflow.
|
||||
|
||||
Args:
|
||||
base: Git ref to diff against. Default: HEAD~1.
|
||||
"""
|
||||
return _user(
|
||||
f"{_TOKEN_EFFICIENCY_PREAMBLE}\n"
|
||||
"## Pre-Merge Check Workflow\n"
|
||||
'1. Call `get_minimal_context(task="pre-merge check")`.\n'
|
||||
'2. Call `detect_changes(detail_level="minimal")` for risk '
|
||||
"score and test gaps.\n"
|
||||
"3. If risk > 0.4: call "
|
||||
'`get_affected_flows(detail_level="minimal")`.\n'
|
||||
"4. If test_gap_count > 0: call "
|
||||
'`query_graph(pattern="tests_for", '
|
||||
'target=<each untested function>, detail_level="minimal")` '
|
||||
"for up to 3 functions.\n"
|
||||
'5. Call `refactor(mode="dead_code", '
|
||||
'detail_level="minimal")` to check for newly dead code.\n'
|
||||
"6. Only call `find_large_functions` or `get_impact_radius` "
|
||||
"if risk > 0.7.\n"
|
||||
"7. Output: GO/NO-GO recommendation with 1-sentence "
|
||||
"justification + list of required follow-ups."
|
||||
)
|
||||
@@ -0,0 +1,852 @@
|
||||
"""Graph-powered refactoring operations.
|
||||
|
||||
Provides rename previews, dead code detection, refactoring suggestions,
|
||||
and safe application of refactoring edits to source files. All file writes
|
||||
go through a preview-then-apply workflow with expiry enforcement and path
|
||||
traversal prevention.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import functools
|
||||
import logging
|
||||
import re
|
||||
import threading
|
||||
import time
|
||||
import uuid
|
||||
from pathlib import Path
|
||||
from typing import Any, Optional, Union
|
||||
|
||||
from .flows import _has_framework_decorator, _matches_entry_name
|
||||
from .graph import GraphStore, _sanitize_name
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Base class names that indicate a framework-managed class (ORM models,
|
||||
# Pydantic schemas, settings). Classes inheriting from these are invoked
|
||||
# via metaclass/framework magic and should not be flagged as dead code.
|
||||
_FRAMEWORK_BASE_CLASSES = frozenset({
|
||||
"Base", "DeclarativeBase", "Model", "BaseModel", "BaseSettings",
|
||||
"db.Model", "TableBase",
|
||||
# AWS CDK constructs -- instantiated by CDK app wiring, not explicit CALLS.
|
||||
"Stack", "NestedStack", "Construct", "Resource",
|
||||
})
|
||||
|
||||
# Class name suffixes that indicate CDK/IaC constructs.
|
||||
# These are instantiated by framework wiring, not direct CALLS edges.
|
||||
# Used as fallback when INHERITS edges to external base classes are absent.
|
||||
_CDK_CLASS_SUFFIXES = ("Stack", "Construct", "Pipeline", "Resources", "Layer")
|
||||
|
||||
# Patterns for mock/stub variables in test files that should not be flagged dead.
|
||||
_MOCK_NAME_RE = re.compile(
|
||||
r"^(mock[A-Z_]|Mock[A-Z]|createMock[A-Z])|" # mockDynamoClient, MockService, createMockX
|
||||
r"(Mock|Stub|Fake|Spy)$", # s3ClientMock, dbStub
|
||||
re.IGNORECASE,
|
||||
)
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Thread-safe pending refactors storage
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
_refactor_lock = threading.Lock()
|
||||
_pending_refactors: dict[str, dict] = {}
|
||||
REFACTOR_EXPIRY_SECONDS = 600 # 10 minutes
|
||||
|
||||
|
||||
def _cleanup_expired() -> int:
|
||||
"""Remove expired refactors from the pending dict. Returns count removed."""
|
||||
now = time.time()
|
||||
expired = [
|
||||
rid for rid, r in _pending_refactors.items()
|
||||
if now - r["created_at"] > REFACTOR_EXPIRY_SECONDS
|
||||
]
|
||||
for rid in expired:
|
||||
del _pending_refactors[rid]
|
||||
return len(expired)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 1. rename_preview
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def rename_preview(
|
||||
store: GraphStore,
|
||||
old_name: str,
|
||||
new_name: str,
|
||||
) -> Optional[dict[str, Any]]:
|
||||
"""Build a rename edit list for *old_name* -> *new_name*.
|
||||
|
||||
Finds the node via ``store.search_nodes(old_name)``, collects
|
||||
definition and reference sites, generates a unique ``refactor_id``,
|
||||
and stores the preview in the thread-safe ``_pending_refactors`` dict.
|
||||
|
||||
Returns:
|
||||
A refactor preview dict, or ``None`` if the node is not found.
|
||||
"""
|
||||
candidates = store.search_nodes(old_name, limit=10)
|
||||
# Pick the best match: prefer exact name match.
|
||||
node = None
|
||||
for c in candidates:
|
||||
if c.name == old_name:
|
||||
node = c
|
||||
break
|
||||
if node is None and candidates:
|
||||
node = candidates[0]
|
||||
if node is None:
|
||||
logger.warning("rename_preview: node %r not found", old_name)
|
||||
return None
|
||||
|
||||
edits: list[dict[str, Any]] = []
|
||||
|
||||
# --- Definition site ---
|
||||
edits.append({
|
||||
"file": node.file_path,
|
||||
"line": node.line_start,
|
||||
"old": old_name,
|
||||
"new": new_name,
|
||||
"confidence": "high",
|
||||
})
|
||||
|
||||
# --- Call sites (CALLS edges targeting this node) ---
|
||||
call_edges = store.get_edges_by_target(node.qualified_name)
|
||||
for edge in call_edges:
|
||||
if edge.kind == "CALLS":
|
||||
edits.append({
|
||||
"file": edge.file_path,
|
||||
"line": edge.line,
|
||||
"old": old_name,
|
||||
"new": new_name,
|
||||
"confidence": "high",
|
||||
})
|
||||
|
||||
# Also search by bare name for unqualified edges.
|
||||
bare_edges = store.search_edges_by_target_name(old_name, kind="CALLS")
|
||||
seen = {(e["file"], e["line"]) for e in edits}
|
||||
for edge in bare_edges:
|
||||
key = (edge.file_path, edge.line)
|
||||
if key not in seen:
|
||||
edits.append({
|
||||
"file": edge.file_path,
|
||||
"line": edge.line,
|
||||
"old": old_name,
|
||||
"new": new_name,
|
||||
"confidence": "high",
|
||||
})
|
||||
seen.add(key)
|
||||
|
||||
# --- Import sites (IMPORTS_FROM edges targeting this node) ---
|
||||
import_edges = store.get_edges_by_target(node.qualified_name)
|
||||
for edge in import_edges:
|
||||
if edge.kind == "IMPORTS_FROM":
|
||||
key = (edge.file_path, edge.line)
|
||||
if key not in seen:
|
||||
edits.append({
|
||||
"file": edge.file_path,
|
||||
"line": edge.line,
|
||||
"old": old_name,
|
||||
"new": new_name,
|
||||
"confidence": "high",
|
||||
})
|
||||
seen.add(key)
|
||||
|
||||
# --- Stats ---
|
||||
stats = {"high": 0, "medium": 0, "low": 0}
|
||||
for e in edits:
|
||||
stats[e["confidence"]] += 1
|
||||
|
||||
refactor_id = uuid.uuid4().hex[:8]
|
||||
preview: dict[str, Any] = {
|
||||
"refactor_id": refactor_id,
|
||||
"type": "rename",
|
||||
"old_name": _sanitize_name(old_name),
|
||||
"new_name": _sanitize_name(new_name),
|
||||
"edits": edits,
|
||||
"stats": stats,
|
||||
"created_at": time.time(),
|
||||
}
|
||||
|
||||
with _refactor_lock:
|
||||
_cleanup_expired()
|
||||
_pending_refactors[refactor_id] = preview
|
||||
|
||||
logger.info(
|
||||
"rename_preview: created refactor %s (%s -> %s, %d edits)",
|
||||
refactor_id, old_name, new_name, len(edits),
|
||||
)
|
||||
return preview
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 2. find_dead_code
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _is_entry_point(node: Any) -> bool:
|
||||
"""Check if a node looks like an entry point by name or decorator.
|
||||
|
||||
Unlike ``flows.detect_entry_points()`` which treats ALL uncalled functions
|
||||
as entry points, this checks only for conventional name patterns and
|
||||
framework decorators -- the indicators that a function is *intentionally*
|
||||
an entry point rather than simply unreferenced dead code.
|
||||
"""
|
||||
if _has_framework_decorator(node):
|
||||
return True
|
||||
if _matches_entry_name(node):
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
# Matches identifiers inside type annotations (e.g. "GoalCreate" in
|
||||
# "body: GoalCreate", "Optional[UserResponse]", "list[Item]").
|
||||
_TEST_FILE_RE = re.compile(
|
||||
r"([\\/]__tests__[\\/]|\.spec\.[jt]sx?$|\.test\.[jt]sx?$|[\\/]test_[^/\\]*\.py$"
|
||||
r"|[\\/]e2e[_-]?tests?[\\/]|[\\/]test[_-]utils?[\\/])",
|
||||
)
|
||||
|
||||
|
||||
def _is_test_file(file_path: str) -> bool:
|
||||
"""Return True if *file_path* looks like a test file."""
|
||||
return bool(_TEST_FILE_RE.search(file_path))
|
||||
|
||||
|
||||
_MIN_PKG_SEGMENT_LEN = 4 # ignore short dirs like "src", "lib", "app"
|
||||
|
||||
|
||||
@functools.lru_cache(maxsize=4096)
|
||||
def _path_segments(file_path: str) -> tuple[str, ...]:
|
||||
"""Return directory segments long enough to serve as package-name anchors."""
|
||||
parts = file_path.replace("\\", "/").split("/")
|
||||
return tuple(
|
||||
p for p in parts[:-1] # skip the filename itself
|
||||
if len(p) >= _MIN_PKG_SEGMENT_LEN and p not in ("home", "src", "lib", "app")
|
||||
)
|
||||
|
||||
|
||||
_TYPE_IDENT_RE = re.compile(r"[A-Z][A-Za-z0-9_]*")
|
||||
|
||||
|
||||
def _collect_type_referenced_names(store: GraphStore) -> set[str]:
|
||||
"""Collect class names that appear in function params or return types."""
|
||||
funcs = store.get_nodes_by_kind(kinds=["Function", "Test"])
|
||||
names: set[str] = set()
|
||||
for f in funcs:
|
||||
for text in (f.params, f.return_type):
|
||||
if text:
|
||||
names.update(_TYPE_IDENT_RE.findall(text))
|
||||
return names
|
||||
|
||||
|
||||
def find_dead_code(
|
||||
store: GraphStore,
|
||||
kind: Optional[str] = None,
|
||||
file_pattern: Optional[str] = None,
|
||||
root: Optional[Union[str, Path]] = None,
|
||||
) -> list[dict[str, Any]]:
|
||||
"""Find functions/classes with no callers, no test refs, no importers, and no references.
|
||||
|
||||
Entry points (functions matching framework decorators or conventional name
|
||||
patterns like ``main``, ``test_*``, ``handle_*``) are excluded.
|
||||
|
||||
.. note::
|
||||
|
||||
**Caveats — dynamic dispatch patterns.** Static analysis cannot track
|
||||
all runtime-determined call patterns. Functions registered via fully
|
||||
dynamic keys (``map[computedKey()] = fn``), ``Reflect.apply``, or
|
||||
runtime ``require()`` may still appear as dead code. Treat results as
|
||||
hints, especially for TypeScript projects that use map-based dispatch,
|
||||
plugin registries, or dynamic requires.
|
||||
|
||||
Args:
|
||||
store: The GraphStore instance.
|
||||
kind: Optional filter (e.g. ``"Function"`` or ``"Class"``).
|
||||
file_pattern: Optional file-path substring filter.
|
||||
root: Optional repo root path for computing ``relative_path``.
|
||||
|
||||
Returns:
|
||||
List of dead-code dicts with name, qualified_name, kind, file_path,
|
||||
relative_path, line, and language fields.
|
||||
"""
|
||||
# Query candidate nodes.
|
||||
candidates = store.get_nodes_by_kind(
|
||||
kinds=[kind] if kind else ["Function", "Class"],
|
||||
file_pattern=file_pattern,
|
||||
)
|
||||
|
||||
# Build set of class names referenced in function type annotations.
|
||||
type_ref_names = _collect_type_referenced_names(store)
|
||||
|
||||
# Build class hierarchy: class_qualified_name -> [bare_base_names]
|
||||
class_bases: dict[str, list[str]] = {}
|
||||
conn = store._conn
|
||||
for row in conn.execute(
|
||||
"SELECT source_qualified, target_qualified FROM edges WHERE kind = 'INHERITS'"
|
||||
).fetchall():
|
||||
base = row[1].rsplit("::", 1)[-1] if "::" in row[1] else row[1]
|
||||
class_bases.setdefault(row[0], []).append(base)
|
||||
|
||||
# Build import graph: file_path -> set of file_paths it imports from.
|
||||
# Used to filter bare-name caller matches to plausible callers.
|
||||
importer_files: dict[str, set[str]] = {}
|
||||
for row in conn.execute(
|
||||
"SELECT file_path, target_qualified FROM edges WHERE kind = 'IMPORTS_FROM'"
|
||||
).fetchall():
|
||||
importer_files.setdefault(row[0], set()).add(row[1])
|
||||
|
||||
# Build set of globally unique names (only one non-test node with that name).
|
||||
# For unique names, any bare-name CALLS edge is reliable — no ambiguity.
|
||||
name_counts: dict[str, int] = {}
|
||||
for row in conn.execute(
|
||||
"SELECT name, COUNT(*) FROM nodes "
|
||||
"WHERE kind IN ('Function', 'Class') AND is_test = 0 "
|
||||
"GROUP BY name"
|
||||
).fetchall():
|
||||
name_counts[row[0]] = row[1]
|
||||
|
||||
def _is_plausible_caller(
|
||||
edge_file: str, node_file: str, node_name: str = "",
|
||||
) -> bool:
|
||||
"""A bare-name edge is plausible if it comes from the same file,
|
||||
from a file that has an IMPORTS_FROM edge whose target matches
|
||||
the node's file path, or the name is globally unique (no ambiguity)."""
|
||||
if edge_file == node_file:
|
||||
return True
|
||||
# Unique names (only one definition) have no ambiguity -- accept all callers.
|
||||
if node_name and name_counts.get(node_name, 0) == 1:
|
||||
return True
|
||||
for imp_target in importer_files.get(edge_file, ()):
|
||||
# Strip "::name" suffix — workspace-resolved imports may include it
|
||||
imp_path = imp_target.split("::")[0] if "::" in imp_target else imp_target
|
||||
# __init__.py represents its parent package directory
|
||||
if imp_path.endswith("/__init__.py"):
|
||||
imp_dir = imp_path[:-12] # strip "/__init__.py"
|
||||
if node_file.startswith(imp_dir + "/"):
|
||||
return True
|
||||
if imp_path.startswith(node_file) or node_file.startswith(imp_path + "/"):
|
||||
return True
|
||||
# 2-hop: edge_file imports X, X re-exports from node_file (barrel files)
|
||||
for imp2 in importer_files.get(imp_target, ()):
|
||||
imp2_path = imp2.split("::")[0] if "::" in imp2 else imp2
|
||||
if imp2_path.endswith("/__init__.py"):
|
||||
imp2_dir = imp2_path[:-12]
|
||||
if node_file.startswith(imp2_dir + "/"):
|
||||
return True
|
||||
if imp2_path.startswith(node_file) or node_file.startswith(imp2_path + "/"):
|
||||
return True
|
||||
# Package-alias heuristic: monorepo imports like "@scope/pkg-name"
|
||||
# contain the directory name of the target package. Check if the
|
||||
# import target string contains a significant directory segment from
|
||||
# the node's file path (e.g. "lambda-common" in both the import
|
||||
# "@cova-utils/lambda-common" and the path "libraries/lambda-common/...").
|
||||
if not imp_target.startswith("/"):
|
||||
# imp_target is a package specifier, not a file path
|
||||
for seg in _path_segments(node_file):
|
||||
if seg in imp_target:
|
||||
return True
|
||||
return False
|
||||
|
||||
dead: list[dict[str, Any]] = []
|
||||
|
||||
for node in candidates:
|
||||
|
||||
# Skip test nodes and anything defined in test files.
|
||||
if node.is_test or _is_test_file(node.file_path):
|
||||
continue
|
||||
|
||||
# Skip ambient type declarations (.d.ts) — they describe external APIs.
|
||||
if node.file_path.endswith(".d.ts"):
|
||||
continue
|
||||
|
||||
# Skip dunder methods -- invoked by runtime, never have explicit callers.
|
||||
if node.name.startswith("__") and node.name.endswith("__"):
|
||||
continue
|
||||
|
||||
# Skip JS/TS/Java constructors -- invoked via `new ClassName()`, which
|
||||
# creates a CALLS edge to the class, not to `constructor`.
|
||||
if node.name == "constructor" and node.parent_name:
|
||||
continue
|
||||
|
||||
# Skip mock/stub variables in test files -- these are test helpers
|
||||
# referenced via variable assignment, not function calls.
|
||||
if node.is_test or _is_test_file(node.file_path):
|
||||
if _MOCK_NAME_RE.search(node.name):
|
||||
continue
|
||||
|
||||
# Skip entry points (by name pattern or decorator, not just "uncalled").
|
||||
if _is_entry_point(node):
|
||||
continue
|
||||
|
||||
# Check for callers (CALLS), test refs (TESTED_BY), importers (IMPORTS_FROM),
|
||||
# and value references (REFERENCES -- function-as-value in maps, arrays, etc.).
|
||||
|
||||
# Skip classes referenced in type annotations (Pydantic schemas, etc.).
|
||||
if node.kind == "Class" and node.name in type_ref_names:
|
||||
continue
|
||||
|
||||
# Skip Angular/NestJS decorated classes -- they are framework-managed
|
||||
# and instantiated by the DI container, not direct CALLS edges.
|
||||
if node.kind == "Class" and _has_framework_decorator(node):
|
||||
continue
|
||||
|
||||
# Skip classes (and their methods) inheriting from known framework bases.
|
||||
_is_framework_class = False
|
||||
_check_qn = node.qualified_name if node.kind == "Class" else (
|
||||
node.qualified_name.rsplit(".", 1)[0] if node.parent_name else None
|
||||
)
|
||||
if _check_qn:
|
||||
outgoing = store.get_edges_by_source(_check_qn)
|
||||
base_names = {
|
||||
e.target_qualified.rsplit("::", 1)[-1]
|
||||
for e in outgoing if e.kind == "INHERITS"
|
||||
}
|
||||
if base_names & _FRAMEWORK_BASE_CLASSES:
|
||||
_is_framework_class = True
|
||||
if node.kind == "Class":
|
||||
if _is_framework_class:
|
||||
continue
|
||||
# Fallback: CDK class name suffixes (no INHERITS edge for external bases)
|
||||
if any(node.name.endswith(s) for s in _CDK_CLASS_SUFFIXES):
|
||||
continue
|
||||
if node.kind == "Function" and _is_framework_class:
|
||||
continue
|
||||
# Also skip methods whose parent class name matches CDK suffixes
|
||||
# (fallback for external base classes without INHERITS edges).
|
||||
if (
|
||||
node.kind == "Function"
|
||||
and node.parent_name
|
||||
and any(node.parent_name.endswith(s) for s in _CDK_CLASS_SUFFIXES)
|
||||
):
|
||||
continue
|
||||
|
||||
# Skip decorated functions/classes that are invoked implicitly rather
|
||||
# than via explicit CALLS edges.
|
||||
decorators = node.extra.get("decorators", ())
|
||||
if isinstance(decorators, (list, tuple)) and decorators:
|
||||
if node.kind in ("Function", "Test"):
|
||||
# @property -- invoked via attribute access
|
||||
# @abstractmethod -- polymorphic dispatch, never called directly
|
||||
# @classmethod/@staticmethod -- called via Class.method()
|
||||
if any(
|
||||
d in ("property", "abstractmethod", "classmethod", "staticmethod")
|
||||
or d.endswith(".abstractmethod")
|
||||
# Angular @HostListener -- method called by framework event system
|
||||
or d.startswith("HostListener")
|
||||
for d in decorators
|
||||
):
|
||||
continue
|
||||
if node.kind == "Class":
|
||||
# @dataclass classes are instantiated as types, not via CALLS
|
||||
if any("dataclass" in d for d in decorators):
|
||||
continue
|
||||
|
||||
# Skip methods that override an @abstractmethod in a base class --
|
||||
# they are called polymorphically via the base class reference.
|
||||
if node.kind == "Function" and node.parent_name:
|
||||
parent_qn = node.qualified_name.rsplit(".", 1)[0]
|
||||
parent_edges = store.get_edges_by_source(parent_qn)
|
||||
base_class_names = [
|
||||
e.target_qualified for e in parent_edges if e.kind == "INHERITS"
|
||||
]
|
||||
for base_name in base_class_names:
|
||||
# Try fully-qualified base first, then bare name match
|
||||
base_method_qn = f"{base_name}.{node.name}"
|
||||
base_nodes = store.get_node(base_method_qn)
|
||||
if base_nodes is None:
|
||||
# Base class may be bare name -- search in same file
|
||||
base_method_qn2 = (
|
||||
node.file_path + "::" + base_name + "." + node.name
|
||||
)
|
||||
base_nodes = store.get_node(base_method_qn2)
|
||||
if base_nodes is not None:
|
||||
base_decos = base_nodes.extra.get("decorators", ())
|
||||
if isinstance(base_decos, (list, tuple)) and any(
|
||||
"abstractmethod" in d for d in base_decos
|
||||
):
|
||||
break
|
||||
else:
|
||||
base_name = None # no abstract override found
|
||||
if base_name is not None:
|
||||
continue
|
||||
|
||||
incoming = store.get_edges_by_target(node.qualified_name)
|
||||
# Also check class-qualified edges (e.g. "ClassName::method") which
|
||||
# lack the file-path prefix used in node.qualified_name.
|
||||
if not any(e.kind == "CALLS" for e in incoming) and node.parent_name:
|
||||
class_qn = f"{node.parent_name}::{node.name}"
|
||||
incoming = incoming + store.get_edges_by_target(class_qn)
|
||||
# Also check bare-name and partially-qualified edges.
|
||||
# CALLS targets may be bare ("funcName"), class-qualified
|
||||
# ("Class::method"), or workspace-qualified ("pkg/dir::funcName").
|
||||
if not any(e.kind == "CALLS" for e in incoming):
|
||||
bare = store.search_edges_by_target_name(node.name, kind="CALLS")
|
||||
# Also search for partially-qualified targets ending with ::name
|
||||
suffix_rows = conn.execute(
|
||||
"SELECT * FROM edges WHERE kind = 'CALLS'"
|
||||
" AND target_qualified LIKE ?",
|
||||
(f"%::{node.name}",),
|
||||
).fetchall()
|
||||
suffix_edges = [store._row_to_edge(r) for r in suffix_rows]
|
||||
all_bare = bare + suffix_edges
|
||||
all_bare = [
|
||||
e for e in all_bare
|
||||
if _is_plausible_caller(e.file_path, node.file_path, node.name)
|
||||
]
|
||||
incoming = incoming + all_bare
|
||||
if not any(e.kind == "TESTED_BY" for e in incoming):
|
||||
bare_tb = store.search_edges_by_target_name(node.name, kind="TESTED_BY")
|
||||
bare_tb = [
|
||||
e for e in bare_tb
|
||||
if _is_plausible_caller(e.file_path, node.file_path, node.name)
|
||||
]
|
||||
incoming = incoming + bare_tb
|
||||
# Check INHERITS -- classes with subclasses are not dead.
|
||||
if node.kind == "Class" and not any(e.kind == "INHERITS" for e in incoming):
|
||||
bare_inh = store.search_edges_by_target_name(node.name, kind="INHERITS")
|
||||
incoming = incoming + bare_inh
|
||||
has_callers = any(e.kind == "CALLS" for e in incoming)
|
||||
has_test_refs = any(e.kind == "TESTED_BY" for e in incoming)
|
||||
has_importers = any(e.kind == "IMPORTS_FROM" for e in incoming)
|
||||
has_references = any(e.kind == "REFERENCES" for e in incoming)
|
||||
has_subclasses = any(e.kind == "INHERITS" for e in incoming)
|
||||
|
||||
# For classes with no direct references, check if any member has callers.
|
||||
no_refs = not (
|
||||
has_callers or has_test_refs or has_importers
|
||||
or has_references or has_subclasses
|
||||
)
|
||||
if node.kind == "Class" and no_refs:
|
||||
member_prefix = node.qualified_name + "."
|
||||
# Also check bare class-name pattern (unresolved CALLS targets)
|
||||
bare_prefix = node.name + "."
|
||||
member_calls = conn.execute(
|
||||
"SELECT COUNT(*) FROM edges WHERE kind = 'CALLS'"
|
||||
" AND (target_qualified LIKE ? OR target_qualified LIKE ?)",
|
||||
(f"%{member_prefix}%", f"%{bare_prefix}%"),
|
||||
).fetchone()[0]
|
||||
if member_calls > 0:
|
||||
has_callers = True
|
||||
|
||||
if not (
|
||||
has_callers or has_test_refs or has_importers
|
||||
or has_references or has_subclasses
|
||||
):
|
||||
# Check if this is a method override where the base class method
|
||||
# has callers (polymorphic dispatch: callers of Base.method()
|
||||
# implicitly call SubClass.method() at runtime).
|
||||
if node.kind == "Function" and node.parent_name and not has_callers:
|
||||
method_suffix = "." + node.name
|
||||
if node.qualified_name.endswith(method_suffix):
|
||||
class_qn = node.qualified_name[: -len(method_suffix)]
|
||||
for base_name in class_bases.get(class_qn, []):
|
||||
rows = conn.execute(
|
||||
"SELECT n.qualified_name FROM nodes n "
|
||||
"WHERE n.parent_name = ? AND n.name = ? "
|
||||
"AND n.kind IN ('Function', 'Test')",
|
||||
(base_name, node.name),
|
||||
).fetchall()
|
||||
for (base_method_qn,) in rows:
|
||||
if conn.execute(
|
||||
"SELECT 1 FROM edges "
|
||||
"WHERE target_qualified = ? AND kind = 'CALLS' "
|
||||
"LIMIT 1",
|
||||
(base_method_qn,),
|
||||
).fetchone():
|
||||
has_callers = True
|
||||
break
|
||||
if has_callers:
|
||||
break
|
||||
|
||||
if not has_callers:
|
||||
if root:
|
||||
try:
|
||||
rel = str(Path(node.file_path).relative_to(root))
|
||||
except ValueError:
|
||||
rel = node.file_path
|
||||
else:
|
||||
rel = node.file_path
|
||||
dead.append({
|
||||
"name": _sanitize_name(node.name),
|
||||
"qualified_name": _sanitize_name(node.qualified_name),
|
||||
"kind": node.kind,
|
||||
"file": node.file_path,
|
||||
"file_path": node.file_path,
|
||||
"relative_path": rel,
|
||||
"line": node.line_start,
|
||||
"language": node.language,
|
||||
})
|
||||
|
||||
logger.info("find_dead_code: found %d dead symbols", len(dead))
|
||||
return dead
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 3. suggest_refactorings
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def suggest_refactorings(store: GraphStore) -> list[dict[str, Any]]:
|
||||
"""Produce community-driven refactoring suggestions.
|
||||
|
||||
Currently two categories:
|
||||
- **move**: Functions in Community A only called by Community B.
|
||||
- **remove**: Dead code (no callers, tests, or importers and not entry points).
|
||||
|
||||
Returns:
|
||||
List of suggestion dicts with type, description, symbols, rationale.
|
||||
"""
|
||||
suggestions: list[dict[str, Any]] = []
|
||||
|
||||
# --- Dead code suggestions ---
|
||||
dead = find_dead_code(store)
|
||||
for d in dead:
|
||||
suggestions.append({
|
||||
"type": "remove",
|
||||
"description": f"Remove unused {d['kind'].lower()} '{d['name']}'",
|
||||
"symbols": [d["qualified_name"]],
|
||||
"rationale": "No callers, no test references, no importers, not an entry point.",
|
||||
})
|
||||
|
||||
# --- Cross-community move suggestions ---
|
||||
# Only attempt if communities table exists and has data.
|
||||
community_rows = store.get_communities_list()
|
||||
|
||||
if community_rows:
|
||||
# Build node -> community_id mapping.
|
||||
node_community: dict[str, int] = {}
|
||||
for crow in community_rows:
|
||||
cid = crow["id"]
|
||||
member_qns = store.get_community_member_qns(cid)
|
||||
for qn in member_qns:
|
||||
node_community[qn] = cid
|
||||
|
||||
community_names: dict[int, str] = {
|
||||
r["id"]: r["name"] for r in community_rows
|
||||
}
|
||||
|
||||
# Check functions called only by members of a different community.
|
||||
all_funcs = store.get_nodes_by_kind(["Function"])
|
||||
|
||||
for fnode in all_funcs:
|
||||
f_community = node_community.get(fnode.qualified_name)
|
||||
if f_community is None:
|
||||
continue
|
||||
|
||||
incoming_calls = [
|
||||
e for e in store.get_edges_by_target(fnode.qualified_name)
|
||||
if e.kind == "CALLS"
|
||||
]
|
||||
if not incoming_calls:
|
||||
continue
|
||||
|
||||
caller_communities = set()
|
||||
for edge in incoming_calls:
|
||||
c_community = node_community.get(edge.source_qualified)
|
||||
if c_community is not None:
|
||||
caller_communities.add(c_community)
|
||||
|
||||
# If ALL callers are from a single *different* community, suggest move.
|
||||
if len(caller_communities) == 1:
|
||||
target_community = next(iter(caller_communities))
|
||||
if target_community != f_community:
|
||||
src_name = community_names.get(f_community, f"community-{f_community}")
|
||||
tgt_name = community_names.get(
|
||||
target_community, f"community-{target_community}"
|
||||
)
|
||||
suggestions.append({
|
||||
"type": "move",
|
||||
"description": (
|
||||
f"Move '{_sanitize_name(fnode.name)}' from "
|
||||
f"'{src_name}' to '{tgt_name}'"
|
||||
),
|
||||
"symbols": [_sanitize_name(fnode.qualified_name)],
|
||||
"rationale": (
|
||||
f"Function is in community '{src_name}' but only "
|
||||
f"called by members of community '{tgt_name}'."
|
||||
),
|
||||
})
|
||||
|
||||
logger.info("suggest_refactorings: produced %d suggestions", len(suggestions))
|
||||
return suggestions
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 4. apply_refactor
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def apply_refactor(
|
||||
refactor_id: str,
|
||||
repo_root: Path,
|
||||
dry_run: bool = False,
|
||||
) -> dict[str, Any]:
|
||||
"""Apply a previously previewed refactoring to source files.
|
||||
|
||||
Validates the refactor_id, checks expiry, ensures all edit paths are
|
||||
within the repo root, then performs exact string replacements on the
|
||||
target files.
|
||||
|
||||
Args:
|
||||
refactor_id: ID from a prior ``rename_preview`` call.
|
||||
repo_root: Validated repository root path.
|
||||
dry_run: If True, compute the would-be changes and return a
|
||||
unified-diff representation per affected file, but do NOT
|
||||
write anything to disk. The ``refactor_id`` is preserved so
|
||||
the same preview can be committed afterwards via a second
|
||||
call without ``dry_run``. See: #176
|
||||
|
||||
Returns:
|
||||
Status dict with applied count and modified files. When
|
||||
``dry_run=True`` the dict additionally contains:
|
||||
|
||||
- ``dry_run``: ``True``
|
||||
- ``would_modify``: list of file paths that would be changed
|
||||
- ``diffs``: map of file path → unified diff string showing the
|
||||
proposed change
|
||||
"""
|
||||
repo_root = repo_root.resolve()
|
||||
|
||||
with _refactor_lock:
|
||||
_cleanup_expired()
|
||||
preview = _pending_refactors.get(refactor_id)
|
||||
|
||||
if preview is None:
|
||||
logger.warning("apply_refactor: unknown or expired refactor_id %s", refactor_id)
|
||||
return {"status": "error", "error": f"Refactor '{refactor_id}' not found or expired."}
|
||||
|
||||
# Check expiry explicitly.
|
||||
age = time.time() - preview["created_at"]
|
||||
if age > REFACTOR_EXPIRY_SECONDS:
|
||||
with _refactor_lock:
|
||||
_pending_refactors.pop(refactor_id, None)
|
||||
logger.warning("apply_refactor: refactor %s expired (%.0fs old)", refactor_id, age)
|
||||
return {"status": "error", "error": f"Refactor '{refactor_id}' has expired."}
|
||||
|
||||
edits = preview.get("edits", [])
|
||||
if not edits:
|
||||
if dry_run:
|
||||
return {
|
||||
"status": "ok", "dry_run": True, "applied": 0,
|
||||
"files_modified": [], "edits_applied": 0,
|
||||
"would_modify": [], "diffs": {},
|
||||
}
|
||||
return {"status": "ok", "applied": 0, "files_modified": [], "edits_applied": 0}
|
||||
|
||||
# --- Path traversal validation ---
|
||||
for edit in edits:
|
||||
edit_path = Path(edit["file"]).resolve()
|
||||
try:
|
||||
edit_path.relative_to(repo_root)
|
||||
except ValueError:
|
||||
logger.error(
|
||||
"apply_refactor: path traversal blocked for %s (repo_root=%s)",
|
||||
edit_path, repo_root,
|
||||
)
|
||||
return {
|
||||
"status": "error",
|
||||
"error": f"Edit path '{edit['file']}' is outside repo root.",
|
||||
}
|
||||
|
||||
# --- Compute new content for every edit (shared by dry-run and write paths) ---
|
||||
# Group edits by file so multiple edits to the same file apply
|
||||
# sequentially against the updated content rather than stomping each
|
||||
# other. Dry-run and write modes then share this computation.
|
||||
from collections import defaultdict
|
||||
edits_by_file: dict[str, list[dict]] = defaultdict(list)
|
||||
for edit in edits:
|
||||
edits_by_file[edit["file"]].append(edit)
|
||||
|
||||
planned: dict[str, tuple[str, str, int]] = {} # file -> (old_content, new_content, edit_count)
|
||||
for file_str, file_edits in edits_by_file.items():
|
||||
file_path = Path(file_str)
|
||||
if not file_path.is_file():
|
||||
logger.warning("apply_refactor: file not found: %s", file_path)
|
||||
continue
|
||||
try:
|
||||
original = file_path.read_text(encoding="utf-8", errors="replace")
|
||||
except (OSError, UnicodeDecodeError) as exc:
|
||||
logger.warning("apply_refactor: could not read %s: %s", file_path, exc)
|
||||
continue
|
||||
|
||||
content = original
|
||||
file_edits_applied = 0
|
||||
for edit in file_edits:
|
||||
old_text = edit["old"]
|
||||
new_text = edit["new"]
|
||||
if old_text not in content:
|
||||
logger.warning(
|
||||
"apply_refactor: old text %r not found in %s",
|
||||
old_text, file_path,
|
||||
)
|
||||
continue
|
||||
target_line = edit.get("line")
|
||||
if target_line is not None:
|
||||
lines = content.splitlines(keepends=True)
|
||||
idx = target_line - 1
|
||||
if 0 <= idx < len(lines) and old_text in lines[idx]:
|
||||
lines[idx] = lines[idx].replace(old_text, new_text, 1)
|
||||
content = "".join(lines)
|
||||
else:
|
||||
content = content.replace(old_text, new_text, 1)
|
||||
else:
|
||||
content = content.replace(old_text, new_text, 1)
|
||||
file_edits_applied += 1
|
||||
|
||||
if file_edits_applied > 0:
|
||||
planned[file_str] = (original, content, file_edits_applied)
|
||||
|
||||
# --- Dry-run path: return diffs, no writes ---
|
||||
if dry_run:
|
||||
import difflib
|
||||
diffs: dict[str, str] = {}
|
||||
for file_str, (original, new_content, _count) in planned.items():
|
||||
diff_lines = list(difflib.unified_diff(
|
||||
original.splitlines(keepends=True),
|
||||
new_content.splitlines(keepends=True),
|
||||
fromfile=f"a/{file_str}",
|
||||
tofile=f"b/{file_str}",
|
||||
n=3,
|
||||
))
|
||||
diffs[file_str] = "".join(diff_lines)
|
||||
total_edits = sum(count for _o, _n, count in planned.values())
|
||||
result = {
|
||||
"status": "ok",
|
||||
"dry_run": True,
|
||||
"applied": 0,
|
||||
"edits_applied": total_edits,
|
||||
"would_modify": sorted(planned.keys()),
|
||||
"files_modified": [],
|
||||
"diffs": diffs,
|
||||
}
|
||||
logger.info(
|
||||
"apply_refactor: dry-run %s — %d edits would be applied to %d files",
|
||||
refactor_id, total_edits, len(planned),
|
||||
)
|
||||
# Do NOT pop the pending refactor — let the user commit via a
|
||||
# second call with dry_run=False.
|
||||
return result
|
||||
|
||||
# --- Real-write path: write the pre-computed new content ---
|
||||
files_modified: set[str] = set()
|
||||
edits_applied = 0
|
||||
for file_str, (_original, new_content, count) in planned.items():
|
||||
file_path = Path(file_str)
|
||||
try:
|
||||
file_path.write_text(new_content, encoding="utf-8")
|
||||
edits_applied += count
|
||||
files_modified.add(str(file_path))
|
||||
logger.info("apply_refactor: applied %d edit(s) to %s", count, file_path)
|
||||
except OSError as exc:
|
||||
logger.error("apply_refactor: could not write %s: %s", file_path, exc)
|
||||
|
||||
# Remove from pending after successful application.
|
||||
with _refactor_lock:
|
||||
_pending_refactors.pop(refactor_id, None)
|
||||
|
||||
result = {
|
||||
"status": "ok",
|
||||
"applied": edits_applied,
|
||||
"files_modified": sorted(files_modified),
|
||||
"edits_applied": edits_applied,
|
||||
}
|
||||
logger.info("apply_refactor: completed %s — %d edits applied", refactor_id, edits_applied)
|
||||
return result
|
||||
@@ -0,0 +1,319 @@
|
||||
"""Multi-repo registry and connection pool.
|
||||
|
||||
Manages a registry of multiple repositories at ``~/.code-review-graph/registry.json``
|
||||
and provides a connection pool for concurrent access to multiple graph databases.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
import sqlite3
|
||||
import threading
|
||||
from collections import OrderedDict
|
||||
from pathlib import Path
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Default registry path
|
||||
_REGISTRY_DIR = Path.home() / ".code-review-graph"
|
||||
_REGISTRY_PATH = _REGISTRY_DIR / "registry.json"
|
||||
|
||||
|
||||
class Registry:
|
||||
"""Manages a JSON-based registry of code-review-graph repositories.
|
||||
|
||||
Each entry stores the repo path and an optional alias.
|
||||
The registry lives at ``~/.code-review-graph/registry.json``.
|
||||
"""
|
||||
|
||||
def __init__(self, path: Path | None = None) -> None:
|
||||
self._path = path or _REGISTRY_PATH
|
||||
self._path.parent.mkdir(parents=True, exist_ok=True)
|
||||
self._lock = threading.Lock()
|
||||
self._repos: list[dict[str, str]] = []
|
||||
self._load()
|
||||
|
||||
def _load(self) -> None:
|
||||
"""Load registry from disk."""
|
||||
if self._path.exists():
|
||||
try:
|
||||
data = json.loads(self._path.read_text(encoding="utf-8", errors="replace"))
|
||||
self._repos = data.get("repos", [])
|
||||
except (json.JSONDecodeError, KeyError, TypeError):
|
||||
logger.warning("Invalid registry file, starting fresh: %s", self._path)
|
||||
self._repos = []
|
||||
else:
|
||||
self._repos = []
|
||||
|
||||
def _save(self) -> None:
|
||||
"""Write registry to disk."""
|
||||
self._path.parent.mkdir(parents=True, exist_ok=True)
|
||||
data = {"repos": self._repos}
|
||||
self._path.write_text(
|
||||
json.dumps(data, indent=2) + "\n", encoding="utf-8"
|
||||
)
|
||||
|
||||
def register(
|
||||
self, path: str, alias: str | None = None, data_dir: str | None = None,
|
||||
) -> dict[str, str]:
|
||||
"""Register a repository path.
|
||||
|
||||
Validates that the path contains a ``.git`` or ``.code-review-graph``
|
||||
directory.
|
||||
|
||||
Args:
|
||||
path: Absolute or relative path to the repository root.
|
||||
alias: Optional short alias for the repository.
|
||||
data_dir: Optional external directory for graph database.
|
||||
|
||||
Returns:
|
||||
The registered entry dict.
|
||||
|
||||
Raises:
|
||||
ValueError: If the path is not a valid repository.
|
||||
"""
|
||||
resolved = Path(path).resolve()
|
||||
if not resolved.is_dir():
|
||||
raise ValueError(f"Path is not a directory: {resolved}")
|
||||
has_repo_marker = (
|
||||
(resolved / ".git").exists()
|
||||
or (resolved / ".svn").exists()
|
||||
or (resolved / ".code-review-graph").exists()
|
||||
)
|
||||
if not has_repo_marker:
|
||||
raise ValueError(
|
||||
f"Path does not look like a repository "
|
||||
f"(no .git, .svn, or .code-review-graph): {resolved}"
|
||||
)
|
||||
|
||||
with self._lock:
|
||||
# Check for duplicate path
|
||||
str_path = str(resolved)
|
||||
for entry in self._repos:
|
||||
if entry["path"] == str_path:
|
||||
# Update alias and/or data_dir if provided
|
||||
if alias:
|
||||
entry["alias"] = alias
|
||||
if data_dir:
|
||||
entry["data_dir"] = str(Path(data_dir).resolve())
|
||||
self._save()
|
||||
return entry
|
||||
|
||||
new_entry: dict[str, str] = {"path": str_path}
|
||||
if alias:
|
||||
new_entry["alias"] = alias
|
||||
if data_dir:
|
||||
new_entry["data_dir"] = str(Path(data_dir).resolve())
|
||||
self._repos.append(new_entry)
|
||||
self._save()
|
||||
return new_entry
|
||||
|
||||
def unregister(self, path_or_alias: str) -> bool:
|
||||
"""Remove a repository by path or alias.
|
||||
|
||||
Args:
|
||||
path_or_alias: Either the absolute path or the alias.
|
||||
|
||||
Returns:
|
||||
True if an entry was removed, False otherwise.
|
||||
"""
|
||||
with self._lock:
|
||||
resolved = str(Path(path_or_alias).resolve())
|
||||
original_len = len(self._repos)
|
||||
self._repos = [
|
||||
entry for entry in self._repos
|
||||
if entry["path"] != resolved
|
||||
and entry.get("alias") != path_or_alias
|
||||
]
|
||||
if len(self._repos) < original_len:
|
||||
self._save()
|
||||
return True
|
||||
return False
|
||||
|
||||
def list_repos(self) -> list[dict[str, str]]:
|
||||
"""Return list of all registered repositories.
|
||||
|
||||
Returns:
|
||||
List of dicts with 'path' and optional 'alias' keys.
|
||||
"""
|
||||
with self._lock:
|
||||
return list(self._repos)
|
||||
|
||||
def find_by_alias(self, alias: str) -> dict[str, str] | None:
|
||||
"""Look up a repository by its alias.
|
||||
|
||||
Args:
|
||||
alias: The alias to search for.
|
||||
|
||||
Returns:
|
||||
The matching entry, or None.
|
||||
"""
|
||||
with self._lock:
|
||||
for entry in self._repos:
|
||||
if entry.get("alias") == alias:
|
||||
return dict(entry)
|
||||
return None
|
||||
|
||||
def find_by_path(self, path: str) -> dict[str, str] | None:
|
||||
"""Look up a repository by its path.
|
||||
|
||||
Args:
|
||||
path: The path to search for.
|
||||
|
||||
Returns:
|
||||
The matching entry, or None.
|
||||
"""
|
||||
resolved = str(Path(path).resolve())
|
||||
with self._lock:
|
||||
for entry in self._repos:
|
||||
if entry["path"] == resolved:
|
||||
return dict(entry)
|
||||
return None
|
||||
|
||||
def set_data_dir(self, path: str, data_dir: str) -> dict[str, str]:
|
||||
"""Set the external data directory for a repository.
|
||||
|
||||
Args:
|
||||
path: Repository path (absolute or relative).
|
||||
data_dir: External directory path to store graph database.
|
||||
|
||||
Returns:
|
||||
The updated or created registry entry.
|
||||
"""
|
||||
resolved = str(Path(path).resolve())
|
||||
data_resolved = str(Path(data_dir).resolve())
|
||||
|
||||
with self._lock:
|
||||
# Check for existing entry
|
||||
for entry in self._repos:
|
||||
if entry["path"] == resolved:
|
||||
entry["data_dir"] = data_resolved
|
||||
self._save()
|
||||
return dict(entry)
|
||||
|
||||
# Create new entry if not found
|
||||
new_entry = {
|
||||
"path": resolved,
|
||||
"data_dir": data_resolved
|
||||
}
|
||||
self._repos.append(new_entry)
|
||||
self._save()
|
||||
return new_entry
|
||||
|
||||
def get_data_dir_for_repo(self, path: str) -> str | None:
|
||||
"""Get the stored data directory for a repository.
|
||||
|
||||
Args:
|
||||
path: Repository path (absolute or relative).
|
||||
|
||||
Returns:
|
||||
The stored data_dir path, or None if not set.
|
||||
"""
|
||||
resolved = str(Path(path).resolve())
|
||||
with self._lock:
|
||||
for entry in self._repos:
|
||||
if entry["path"] == resolved:
|
||||
return entry.get("data_dir")
|
||||
return None
|
||||
|
||||
|
||||
class ConnectionPool:
|
||||
"""LRU connection pool for SQLite graph databases.
|
||||
|
||||
Caches open connections keyed by database path, evicting the least
|
||||
recently used connection when the pool is full.
|
||||
"""
|
||||
|
||||
def __init__(self, max_size: int = 10) -> None:
|
||||
self._max_size = max_size
|
||||
self._pool: OrderedDict[str, sqlite3.Connection] = OrderedDict()
|
||||
self._lock = threading.Lock()
|
||||
|
||||
def get(self, db_path: str) -> sqlite3.Connection:
|
||||
"""Get or create a connection for the given database path.
|
||||
|
||||
Args:
|
||||
db_path: Path to the SQLite database file.
|
||||
|
||||
Returns:
|
||||
An open SQLite connection.
|
||||
"""
|
||||
key = str(Path(db_path).resolve())
|
||||
with self._lock:
|
||||
if key in self._pool:
|
||||
self._pool.move_to_end(key)
|
||||
return self._pool[key]
|
||||
|
||||
# Evict LRU if full
|
||||
while len(self._pool) >= self._max_size:
|
||||
evict_key, evict_conn = self._pool.popitem(last=False)
|
||||
try:
|
||||
evict_conn.close()
|
||||
except sqlite3.Error:
|
||||
logger.debug("Failed to close evicted connection: %s", evict_key)
|
||||
logger.debug("Evicted connection: %s", evict_key)
|
||||
|
||||
conn = sqlite3.connect(
|
||||
key, timeout=30, check_same_thread=False,
|
||||
isolation_level=None,
|
||||
)
|
||||
conn.row_factory = sqlite3.Row
|
||||
conn.execute("PRAGMA journal_mode=WAL")
|
||||
conn.execute("PRAGMA busy_timeout=5000")
|
||||
self._pool[key] = conn
|
||||
return conn
|
||||
|
||||
def close_all(self) -> None:
|
||||
"""Close all connections in the pool."""
|
||||
with self._lock:
|
||||
for key, conn in self._pool.items():
|
||||
try:
|
||||
conn.close()
|
||||
except sqlite3.Error:
|
||||
logger.debug("Failed to close connection: %s", key)
|
||||
self._pool.clear()
|
||||
|
||||
@property
|
||||
def size(self) -> int:
|
||||
"""Current number of open connections."""
|
||||
with self._lock:
|
||||
return len(self._pool)
|
||||
|
||||
|
||||
def resolve_repo(
|
||||
registry: Registry,
|
||||
repo: str | None,
|
||||
cwd: str | None = None,
|
||||
) -> str | None:
|
||||
"""Resolve a repo parameter to an absolute path.
|
||||
|
||||
Resolution order:
|
||||
1. If repo is given, try as alias first.
|
||||
2. If repo is given and not an alias, try as a direct path.
|
||||
3. If repo is None, use cwd.
|
||||
|
||||
Args:
|
||||
registry: The Registry instance.
|
||||
repo: Alias or path string, or None.
|
||||
cwd: Current working directory fallback.
|
||||
|
||||
Returns:
|
||||
Resolved absolute path string, or None if unresolvable.
|
||||
"""
|
||||
if repo:
|
||||
# Try alias first
|
||||
entry = registry.find_by_alias(repo)
|
||||
if entry:
|
||||
return entry["path"]
|
||||
|
||||
# Try as direct path
|
||||
path = Path(repo).resolve()
|
||||
if path.is_dir():
|
||||
return str(path)
|
||||
|
||||
# Fall back to CWD
|
||||
if cwd:
|
||||
return str(Path(cwd).resolve())
|
||||
|
||||
return None
|
||||
@@ -0,0 +1,206 @@
|
||||
"""Post-build pass that resolves ReScript cross-module references.
|
||||
|
||||
The per-file parser emits CALLS/IMPORTS_FROM edges with bare targets like
|
||||
``LogicUtils.safeParse`` because the parser only sees one file at a time.
|
||||
This module runs after ``full_build`` / incremental updates and rewrites
|
||||
those targets to canonical qualified names like
|
||||
``<abs-path>/LogicUtils.res::safeParse`` so ``callers_of``,
|
||||
``get_impact_radius`` and ``importers_of`` work correctly across files.
|
||||
|
||||
Resolutions performed:
|
||||
1. ``Module.fn`` / ``Module.Sub.fn`` CALLS edges → canonical node
|
||||
when a ``.res`` / ``.resi`` file with matching basename exists.
|
||||
2. Bare ``fn(...)`` CALLS edges in a file that ``open`` / ``include``\\s
|
||||
a module → canonical node in that module's file.
|
||||
3. IMPORTS_FROM edges targeting a module name (open / include / jsx /
|
||||
module_alias / external_module) → the target file path, so
|
||||
``importers_of(<path>)`` finds every consuming file.
|
||||
|
||||
Only the ``target_qualified`` column is updated; source and edge kind are
|
||||
preserved. Edges that cannot be resolved are left unchanged.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
from pathlib import Path
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from .graph import GraphStore
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def resolve_rescript_cross_module(store: GraphStore) -> dict:
|
||||
"""Resolve ReScript cross-module targets in the graph store.
|
||||
|
||||
Safe to call multiple times: already-resolved edges (targets containing
|
||||
``::``) are skipped.
|
||||
|
||||
Returns a dict with resolution counts for telemetry.
|
||||
"""
|
||||
conn = store._conn # intentional: post-build maintenance pass
|
||||
|
||||
# Basename (module name) → absolute file path, preferring .res over .resi.
|
||||
basename_to_path: dict[str, str] = {}
|
||||
rescript_files: set[str] = set()
|
||||
for file_path in store.get_all_files():
|
||||
p = Path(file_path)
|
||||
suffix = p.suffix.lower()
|
||||
if suffix not in (".res", ".resi"):
|
||||
continue
|
||||
rescript_files.add(file_path)
|
||||
stem = p.stem
|
||||
existing = basename_to_path.get(stem)
|
||||
if existing is None or existing.lower().endswith(".resi"):
|
||||
# Prefer implementation (.res) over interface (.resi).
|
||||
basename_to_path[stem] = file_path
|
||||
|
||||
if not basename_to_path:
|
||||
return {"files_indexed": 0, "calls_resolved": 0, "imports_resolved": 0}
|
||||
|
||||
# Per-file opens/includes so we can resolve bare calls.
|
||||
opens_by_file: dict[str, list[str]] = {}
|
||||
imports_rows = conn.execute(
|
||||
"SELECT source_qualified, target_qualified, file_path, extra "
|
||||
"FROM edges WHERE kind = 'IMPORTS_FROM'"
|
||||
).fetchall()
|
||||
for row in imports_rows:
|
||||
fp = row["file_path"]
|
||||
if fp not in rescript_files:
|
||||
continue
|
||||
try:
|
||||
extra = json.loads(row["extra"] or "{}")
|
||||
except (json.JSONDecodeError, TypeError):
|
||||
extra = {}
|
||||
kind = extra.get("rescript_import_kind")
|
||||
if kind in ("open", "include"):
|
||||
# Strip nested submodule — root determines file.
|
||||
root = row["target_qualified"].split(".", 1)[0]
|
||||
opens_by_file.setdefault(fp, []).append(root)
|
||||
|
||||
# --- 1 + 2. Resolve CALLS edges ---
|
||||
call_rows = conn.execute(
|
||||
"SELECT id, source_qualified, target_qualified, file_path "
|
||||
"FROM edges WHERE kind = 'CALLS'"
|
||||
).fetchall()
|
||||
|
||||
call_updates: list[tuple[str, int]] = []
|
||||
for row in call_rows:
|
||||
target = row["target_qualified"]
|
||||
if "::" in target:
|
||||
continue # already resolved
|
||||
|
||||
resolved = _resolve_call_target(
|
||||
target,
|
||||
row["file_path"],
|
||||
basename_to_path,
|
||||
opens_by_file,
|
||||
store,
|
||||
)
|
||||
if resolved and resolved != target:
|
||||
call_updates.append((resolved, row["id"]))
|
||||
|
||||
# --- 3. Resolve IMPORTS_FROM edge targets to file paths ---
|
||||
import_updates: list[tuple[str, int]] = []
|
||||
import_rows_full = conn.execute(
|
||||
"SELECT id, target_qualified, file_path FROM edges "
|
||||
"WHERE kind = 'IMPORTS_FROM'"
|
||||
).fetchall()
|
||||
for row in import_rows_full:
|
||||
target = row["target_qualified"]
|
||||
if target in rescript_files:
|
||||
continue # already a file path
|
||||
if "/" in target or "\\" in target:
|
||||
continue # looks like a path already (e.g. relative JS import)
|
||||
root = target.split(".", 1)[0]
|
||||
file_target = basename_to_path.get(root)
|
||||
if file_target and file_target != target:
|
||||
import_updates.append((file_target, row["id"]))
|
||||
|
||||
cur = conn.cursor()
|
||||
for new_target, edge_id in call_updates:
|
||||
cur.execute(
|
||||
"UPDATE edges SET target_qualified = ? WHERE id = ?",
|
||||
(new_target, edge_id),
|
||||
)
|
||||
for new_target, edge_id in import_updates:
|
||||
cur.execute(
|
||||
"UPDATE edges SET target_qualified = ? WHERE id = ?",
|
||||
(new_target, edge_id),
|
||||
)
|
||||
conn.commit()
|
||||
store._invalidate_cache()
|
||||
|
||||
result = {
|
||||
"files_indexed": len(basename_to_path),
|
||||
"calls_resolved": len(call_updates),
|
||||
"imports_resolved": len(import_updates),
|
||||
}
|
||||
logger.info("ReScript cross-module resolution: %s", result)
|
||||
return result
|
||||
|
||||
|
||||
def _resolve_call_target(
|
||||
target: str,
|
||||
file_path: str,
|
||||
basename_to_path: dict[str, str],
|
||||
opens_by_file: dict[str, list[str]],
|
||||
store: GraphStore,
|
||||
) -> str | None:
|
||||
"""Resolve a CALLS edge's ``target_qualified`` to a canonical qualified
|
||||
node name. Returns None when no resolution is possible.
|
||||
"""
|
||||
# Dotted: `Module.fn` or `Module.Sub.fn`.
|
||||
if "." in target:
|
||||
head, _, rest = target.partition(".")
|
||||
target_file = basename_to_path.get(head)
|
||||
if target_file is None:
|
||||
return None
|
||||
candidate = _pick_existing_qualified(target_file, rest, store)
|
||||
return candidate
|
||||
|
||||
# Bare: `fn` — only resolvable via an open/include in the calling file.
|
||||
for opened in opens_by_file.get(file_path, []):
|
||||
target_file = basename_to_path.get(opened)
|
||||
if target_file is None:
|
||||
continue
|
||||
candidate = f"{target_file}::{target}"
|
||||
if store.get_node(candidate) is not None:
|
||||
return candidate
|
||||
return None
|
||||
|
||||
|
||||
def _pick_existing_qualified(
|
||||
target_file: str, rest: str, store: GraphStore,
|
||||
) -> str | None:
|
||||
"""Given ``LogicUtils.foo.bar``, try ``file::foo.bar`` then
|
||||
``file::Foo.bar`` then ``file::foo``. Return the first one that
|
||||
corresponds to an existing node.
|
||||
"""
|
||||
# Direct: rest as the qualified name tail.
|
||||
direct = f"{target_file}::{rest}"
|
||||
if store.get_node(direct) is not None:
|
||||
return direct
|
||||
|
||||
# Dotted rest like `Sub.fn`: parent_name = Sub, name = fn.
|
||||
# _qualify formats it the same way, so `direct` would already match if
|
||||
# the node was stored with that exact qualified name.
|
||||
|
||||
# Some targets include a trailing member-access that isn't part of
|
||||
# the qualified node (e.g. `LogicUtils.safeParse.resp` — property on
|
||||
# the result). Try peeling from the right.
|
||||
parts = rest.split(".")
|
||||
while len(parts) > 1:
|
||||
parts.pop()
|
||||
candidate = f"{target_file}::{'.'.join(parts)}"
|
||||
if store.get_node(candidate) is not None:
|
||||
return candidate
|
||||
# Last resort: top-level `file::name` (first part only).
|
||||
first = rest.split(".", 1)[0]
|
||||
candidate = f"{target_file}::{first}"
|
||||
if store.get_node(candidate) is not None:
|
||||
return candidate
|
||||
return None
|
||||
@@ -0,0 +1,447 @@
|
||||
"""Hybrid search engine combining FTS5 (BM25) and vector embeddings.
|
||||
|
||||
Uses Reciprocal Rank Fusion (RRF) to merge results from full-text search
|
||||
and semantic similarity, with query-aware kind boosting and context-file
|
||||
boosting for relevance tuning.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import re
|
||||
import sqlite3
|
||||
from typing import Any, Optional
|
||||
|
||||
from .graph import GraphStore, _sanitize_name
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# FTS5 index management
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def rebuild_fts_index(store: GraphStore) -> int:
|
||||
"""Rebuild the FTS5 index from the nodes table.
|
||||
|
||||
Checks whether the ``nodes_fts`` virtual table exists, clears it, then
|
||||
repopulates it from every row in ``nodes``.
|
||||
|
||||
Returns:
|
||||
Number of rows indexed.
|
||||
"""
|
||||
# NOTE: rebuild_fts_index uses store._conn directly because it manages
|
||||
# the FTS5 virtual table DDL, which is tightly coupled to SQLite internals.
|
||||
conn = store._conn
|
||||
|
||||
# Wrap the full DROP + CREATE + INSERT sequence in an explicit transaction
|
||||
# so a crash mid-rebuild cannot leave the DB without an FTS table at all
|
||||
# (DROP succeeded but CREATE/INSERT didn't). See #259.
|
||||
if conn.in_transaction:
|
||||
logger.warning("Rolling back uncommitted transaction before BEGIN IMMEDIATE")
|
||||
conn.rollback()
|
||||
conn.execute("BEGIN IMMEDIATE")
|
||||
try:
|
||||
# Drop and recreate the FTS table with content sync to match migration v5
|
||||
conn.execute("DROP TABLE IF EXISTS nodes_fts")
|
||||
conn.execute("""
|
||||
CREATE VIRTUAL TABLE nodes_fts USING fts5(
|
||||
name, qualified_name, file_path, signature,
|
||||
content='nodes', content_rowid='rowid',
|
||||
tokenize='porter unicode61'
|
||||
)
|
||||
""")
|
||||
|
||||
# Rebuild from the content table (nodes) using the FTS5 rebuild command
|
||||
conn.execute("INSERT INTO nodes_fts(nodes_fts) VALUES('rebuild')")
|
||||
|
||||
|
||||
conn.commit()
|
||||
except BaseException:
|
||||
conn.rollback()
|
||||
raise
|
||||
|
||||
count = conn.execute("SELECT count(*) FROM nodes_fts").fetchone()[0]
|
||||
logger.info("FTS index rebuilt: %d rows indexed", count)
|
||||
return count
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Query kind boosting heuristics
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
_DOTTED_IDENT_RE = re.compile(r'\b[A-Za-z_][\w]*(?:\.[A-Za-z_][\w]*)+\b')
|
||||
_SNAKE_IDENT_RE = re.compile(r'\b[a-z][a-z0-9]*(?:_[a-z0-9]+)+\b')
|
||||
_PASCAL_IDENT_RE = re.compile(r'\b[A-Z][a-z0-9]+(?:[A-Z][a-z0-9]+)+\b')
|
||||
|
||||
|
||||
def extract_query_identifiers(query: str) -> list[str]:
|
||||
"""Pull out identifier-shaped tokens from anywhere in a query.
|
||||
|
||||
Catches dotted forms (``Context.Next``), snake_case (``get_dependant``),
|
||||
and CamelCase (``APIRoute``) even when they're embedded in a natural-
|
||||
language sentence. Used to boost search hits whose qualified_name
|
||||
contains any of these tokens, so an LLM asking "Who advances the gin
|
||||
middleware chain via Context.Next" lands on ``Context.Next`` instead of
|
||||
the bare ``Context`` class.
|
||||
"""
|
||||
found: list[str] = []
|
||||
seen: set[str] = set()
|
||||
for pat in (_DOTTED_IDENT_RE, _SNAKE_IDENT_RE, _PASCAL_IDENT_RE):
|
||||
for match in pat.findall(query):
|
||||
lo = match.lower()
|
||||
if lo not in seen and len(lo) >= 3:
|
||||
seen.add(lo)
|
||||
found.append(lo)
|
||||
return found
|
||||
|
||||
|
||||
def detect_query_kind_boost(query: str) -> dict[str, Any]:
|
||||
"""Detect query patterns and return per-node boost multipliers.
|
||||
|
||||
Heuristics:
|
||||
- PascalCase queries (e.g. ``MyClass``) boost Class/Type by 1.5x
|
||||
- snake_case queries (e.g. ``get_users``) boost Function by 1.5x
|
||||
- Queries containing ``.`` boost qualified name matches by 2.0x
|
||||
- Identifier-shaped tokens *anywhere* in the query (dotted, snake_case,
|
||||
CamelCase) boost results whose qualified_name contains them by 2.0x.
|
||||
See ``extract_query_identifiers``.
|
||||
|
||||
Returns:
|
||||
Dict whose keys are either node kind strings (mapped to float
|
||||
multipliers) or one of the special keys ``_qualified``,
|
||||
``_qualified_identifiers``.
|
||||
"""
|
||||
boosts: dict[str, Any] = {}
|
||||
|
||||
if not query or not query.strip():
|
||||
return boosts
|
||||
|
||||
q = query.strip()
|
||||
|
||||
# PascalCase: starts with uppercase, has at least one lowercase after
|
||||
if re.match(r'^[A-Z][a-z]', q) and not q.isupper():
|
||||
boosts["Class"] = 1.5
|
||||
boosts["Type"] = 1.5
|
||||
|
||||
# snake_case or SCREAMING_SNAKE_CASE: contains underscore with letters
|
||||
if '_' in q and re.search(r'[a-zA-Z]', q):
|
||||
boosts["Function"] = 1.5
|
||||
|
||||
# Dotted path: boost qualified name matches
|
||||
if '.' in q:
|
||||
boosts["_qualified"] = 2.0
|
||||
|
||||
# Identifiers extracted from anywhere in the query
|
||||
idents = extract_query_identifiers(q)
|
||||
if idents:
|
||||
boosts["_qualified_identifiers"] = idents
|
||||
|
||||
return boosts
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Reciprocal Rank Fusion
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def rrf_merge(*result_lists: list[tuple[int, float]], k: int = 60) -> list[tuple[int, float]]:
|
||||
"""Merge multiple ranked result lists using Reciprocal Rank Fusion.
|
||||
|
||||
Each input list contains ``(id, score)`` tuples, ordered by score
|
||||
descending. The RRF score for each item is the sum of
|
||||
``1 / (k + rank + 1)`` across all lists it appears in, where rank is
|
||||
the 0-based position.
|
||||
|
||||
Args:
|
||||
*result_lists: Variable number of ranked result lists.
|
||||
k: RRF constant (default 60). Higher values reduce the impact of
|
||||
rank differences.
|
||||
|
||||
Returns:
|
||||
Merged list of ``(id, rrf_score)`` tuples sorted by score descending.
|
||||
"""
|
||||
scores: dict[int, float] = {}
|
||||
|
||||
for result_list in result_lists:
|
||||
for rank, (item_id, _score) in enumerate(result_list):
|
||||
scores[item_id] = scores.get(item_id, 0.0) + 1.0 / (k + rank + 1)
|
||||
|
||||
merged = sorted(scores.items(), key=lambda x: x[1], reverse=True)
|
||||
return merged
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# FTS5 search
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _fts_search(
|
||||
conn: sqlite3.Connection,
|
||||
query: str,
|
||||
limit: int = 50,
|
||||
) -> list[tuple[int, float]]:
|
||||
"""Run an FTS5 BM25 search against the nodes_fts table.
|
||||
|
||||
Returns list of ``(node_id, bm25_score)`` tuples. The BM25 score is
|
||||
negated so higher = better (FTS5 returns negative BM25).
|
||||
"""
|
||||
# Sanitize: wrap in double quotes to prevent FTS5 operator injection
|
||||
safe_query = '"' + query.replace('"', '""') + '"'
|
||||
|
||||
try:
|
||||
rows = conn.execute(
|
||||
"SELECT rowid, rank FROM nodes_fts WHERE nodes_fts MATCH ? "
|
||||
"ORDER BY rank LIMIT ?",
|
||||
(safe_query, limit),
|
||||
).fetchall()
|
||||
# FTS5 rank is negative BM25 (lower = better), negate for consistency
|
||||
return [(row[0], -row[1]) for row in rows]
|
||||
except sqlite3.OperationalError as e:
|
||||
logger.warning("FTS5 search failed: %s", e)
|
||||
return []
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Embedding search (optional)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _embedding_search(
|
||||
store: GraphStore,
|
||||
query: str,
|
||||
limit: int = 50,
|
||||
model: str | None = None,
|
||||
provider: str | None = None,
|
||||
) -> list[tuple[int, float]]:
|
||||
"""Run a vector similarity search using the embedding store.
|
||||
|
||||
Returns list of ``(node_id, similarity_score)`` tuples.
|
||||
Gracefully returns an empty list if embeddings are not available.
|
||||
"""
|
||||
try:
|
||||
from .embeddings import EmbeddingStore
|
||||
except ImportError:
|
||||
return []
|
||||
|
||||
try:
|
||||
emb_store = EmbeddingStore(store.db_path, provider=provider, model=model)
|
||||
try:
|
||||
if not emb_store.available or emb_store.count() == 0:
|
||||
return []
|
||||
|
||||
results = emb_store.search(query, limit=limit)
|
||||
# Map qualified names back to node IDs
|
||||
id_scores: list[tuple[int, float]] = []
|
||||
for qn, score in results:
|
||||
node = store.get_node(qn)
|
||||
if node:
|
||||
id_scores.append((node.id, score))
|
||||
return id_scores
|
||||
finally:
|
||||
emb_store.close()
|
||||
except Exception as e:
|
||||
logger.warning("Embedding search failed: %s", e)
|
||||
return []
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Keyword LIKE fallback
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _keyword_search(
|
||||
conn: sqlite3.Connection,
|
||||
query: str,
|
||||
limit: int = 50,
|
||||
) -> list[tuple[int, float]]:
|
||||
"""Fall back to simple LIKE keyword matching.
|
||||
|
||||
Each word in the query must match independently (AND logic).
|
||||
Returns ``(node_id, score)`` tuples with a basic relevance score.
|
||||
"""
|
||||
words = query.lower().split()
|
||||
if not words:
|
||||
return []
|
||||
|
||||
conditions: list[str] = []
|
||||
params: list[str | int] = []
|
||||
for word in words:
|
||||
conditions.append(
|
||||
"(LOWER(name) LIKE ? OR LOWER(qualified_name) LIKE ?)"
|
||||
)
|
||||
params.extend([f"%{word}%", f"%{word}%"])
|
||||
|
||||
where = " AND ".join(conditions)
|
||||
params.append(limit)
|
||||
sql = f"SELECT id, name, qualified_name FROM nodes WHERE {where} LIMIT ?" # nosec B608
|
||||
|
||||
try:
|
||||
rows = conn.execute(sql, params).fetchall()
|
||||
except sqlite3.OperationalError:
|
||||
return []
|
||||
|
||||
# Assign a simple relevance score: exact name match > prefix > contains
|
||||
q_lower = query.lower()
|
||||
results: list[tuple[int, float]] = []
|
||||
for row in rows:
|
||||
name_lower = row["name"].lower()
|
||||
if name_lower == q_lower:
|
||||
score = 3.0
|
||||
elif name_lower.startswith(q_lower):
|
||||
score = 2.0
|
||||
else:
|
||||
score = 1.0
|
||||
results.append((row["id"], score))
|
||||
|
||||
results.sort(key=lambda x: x[1], reverse=True)
|
||||
return results
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Main hybrid search
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def hybrid_search(
|
||||
store: GraphStore,
|
||||
query: str,
|
||||
kind: Optional[str] = None,
|
||||
limit: int = 20,
|
||||
context_files: Optional[list[str]] = None,
|
||||
model: Optional[str] = None,
|
||||
provider: Optional[str] = None,
|
||||
) -> list[dict[str, Any]]:
|
||||
"""Hybrid search combining FTS5 BM25 and vector embeddings via RRF.
|
||||
|
||||
Attempts FTS5 + embedding search first, falling back to FTS5-only,
|
||||
then keyword LIKE matching if FTS5 is unavailable.
|
||||
|
||||
Args:
|
||||
store: The graph store to search.
|
||||
query: Search query string.
|
||||
kind: Optional node kind filter (e.g. ``"Function"``, ``"Class"``).
|
||||
limit: Maximum results to return (default 20).
|
||||
context_files: Optional list of file paths. Nodes in these files
|
||||
receive a 1.5x score boost.
|
||||
|
||||
Returns:
|
||||
List of dicts with node metadata and ``score`` field.
|
||||
"""
|
||||
if not query or not query.strip():
|
||||
return []
|
||||
|
||||
# NOTE: hybrid_search uses store._conn for FTS5 and keyword queries
|
||||
# because those operate on the FTS virtual table or need raw Row
|
||||
# access for batch-fetch performance. This is documented coupling.
|
||||
conn = store._conn
|
||||
fetch_limit = limit * 3 # Fetch extra to allow for filtering and boosting
|
||||
|
||||
# ------ Phase 1: Gather ranked lists ------
|
||||
fts_results: list[tuple[int, float]] = []
|
||||
emb_results: list[tuple[int, float]] = []
|
||||
|
||||
# Try FTS5 search
|
||||
try:
|
||||
fts_results = _fts_search(conn, query, limit=fetch_limit)
|
||||
except Exception as e:
|
||||
logger.warning("FTS5 unavailable, will use fallback: %s", e)
|
||||
|
||||
# Try embedding search
|
||||
emb_results = _embedding_search(
|
||||
store, query, limit=fetch_limit, model=model, provider=provider,
|
||||
)
|
||||
|
||||
# ------ Phase 2: Merge via RRF or fallback ------
|
||||
if fts_results or emb_results:
|
||||
lists_to_merge = []
|
||||
if fts_results:
|
||||
lists_to_merge.append(fts_results)
|
||||
if emb_results:
|
||||
lists_to_merge.append(emb_results)
|
||||
merged = rrf_merge(*lists_to_merge)
|
||||
else:
|
||||
# Fallback: keyword LIKE matching
|
||||
keyword_results = _keyword_search(conn, query, limit=fetch_limit)
|
||||
if not keyword_results:
|
||||
return []
|
||||
merged = keyword_results
|
||||
|
||||
# ------ Phase 3+4: Batch-fetch nodes, apply boosting and kind filter ------
|
||||
kind_boosts = detect_query_kind_boost(query)
|
||||
context_set = set(context_files) if context_files else set()
|
||||
|
||||
# Batch-fetch all candidate nodes in one query
|
||||
candidate_ids = [node_id for node_id, _ in merged]
|
||||
node_rows: dict[int, Any] = {}
|
||||
batch_size = 450
|
||||
for i in range(0, len(candidate_ids), batch_size):
|
||||
batch = candidate_ids[i:i + batch_size]
|
||||
placeholders = ",".join("?" for _ in batch)
|
||||
rows = conn.execute(
|
||||
f"SELECT * FROM nodes WHERE id IN ({placeholders})", # nosec B608
|
||||
batch,
|
||||
).fetchall()
|
||||
for row in rows:
|
||||
node_rows[row["id"]] = row
|
||||
|
||||
# Apply boosting
|
||||
boosted: list[tuple[int, float]] = []
|
||||
for node_id, score in merged:
|
||||
row = node_rows.get(node_id)
|
||||
if not row:
|
||||
continue
|
||||
|
||||
node_kind = row["kind"]
|
||||
file_path = row["file_path"]
|
||||
qualified_name = row["qualified_name"]
|
||||
|
||||
boost = 1.0
|
||||
if node_kind in kind_boosts:
|
||||
boost *= kind_boosts[node_kind]
|
||||
if "_qualified" in kind_boosts and '.' in query:
|
||||
if query.lower() in qualified_name.lower():
|
||||
boost *= kind_boosts["_qualified"]
|
||||
idents = kind_boosts.get("_qualified_identifiers")
|
||||
if idents:
|
||||
qn_lo = qualified_name.lower()
|
||||
if any(ident in qn_lo for ident in idents):
|
||||
boost *= 2.0
|
||||
if context_set and file_path in context_set:
|
||||
boost *= 1.5
|
||||
|
||||
boosted.append((node_id, score * boost))
|
||||
|
||||
boosted.sort(key=lambda x: x[1], reverse=True)
|
||||
|
||||
# Build results from the already-fetched rows
|
||||
results: list[dict[str, Any]] = []
|
||||
for node_id, final_score in boosted:
|
||||
if len(results) >= limit:
|
||||
break
|
||||
|
||||
row = node_rows.get(node_id)
|
||||
if not row:
|
||||
continue
|
||||
|
||||
node_kind = row["kind"]
|
||||
if kind and node_kind != kind:
|
||||
continue
|
||||
|
||||
results.append({
|
||||
"name": _sanitize_name(row["name"]),
|
||||
"qualified_name": _sanitize_name(row["qualified_name"]),
|
||||
"kind": node_kind,
|
||||
"file_path": row["file_path"],
|
||||
"line_start": row["line_start"],
|
||||
"line_end": row["line_end"],
|
||||
"language": row["language"] or "",
|
||||
"params": row["params"],
|
||||
"return_type": row["return_type"],
|
||||
"signature": row["signature"] if "signature" in row.keys() else None,
|
||||
"score": round(final_score, 6),
|
||||
})
|
||||
|
||||
return results
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,200 @@
|
||||
"""Post-build Spring DI call resolver.
|
||||
|
||||
After tree-sitter parsing, Java CALLS edges whose target is a bare method
|
||||
name (e.g. ``calculate``) carry ``extra.receiver`` naming the local variable
|
||||
that was called on (e.g. ``invoiceCalculationService``). This module
|
||||
resolves those receivers through the INJECTS map to their declared type, then
|
||||
optionally to the unique concrete implementation via INHERITS edges.
|
||||
|
||||
Resolution chain:
|
||||
receiver variable name
|
||||
→ injected interface/class (from INJECTS.extra.field_name)
|
||||
→ concrete implementation (from INHERITS, when unique)
|
||||
|
||||
Only Java files are processed. Edges that are already qualified (contain
|
||||
``::``) or have no ``receiver`` extra key are skipped.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from .graph import GraphStore
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def resolve_spring_di_calls(store: GraphStore) -> dict:
|
||||
"""Resolve Java CALLS edges whose receiver is a Spring-injected field.
|
||||
|
||||
Safe to call multiple times — already-resolved edges (targets containing
|
||||
``::``) are skipped.
|
||||
|
||||
Returns a dict with resolution counts for telemetry.
|
||||
"""
|
||||
conn = store._conn
|
||||
|
||||
# Only process Java files
|
||||
java_files: set[str] = {
|
||||
row["file_path"]
|
||||
for row in conn.execute(
|
||||
"SELECT DISTINCT file_path FROM nodes WHERE language = 'java'"
|
||||
).fetchall()
|
||||
}
|
||||
if not java_files:
|
||||
return {"files_indexed": 0, "calls_resolved": 0}
|
||||
|
||||
# -----------------------------------------------------------------------
|
||||
# Build field_map: (source_qualified_class, field_name) → injected_type
|
||||
# from INJECTS edges that carry extra.field_name
|
||||
# -----------------------------------------------------------------------
|
||||
field_map: dict[tuple[str, str], str] = {}
|
||||
injects_rows = conn.execute(
|
||||
"SELECT source_qualified, target_qualified, extra FROM edges WHERE kind = 'INJECTS'"
|
||||
).fetchall()
|
||||
for row in injects_rows:
|
||||
try:
|
||||
extra = json.loads(row["extra"] or "{}")
|
||||
except (json.JSONDecodeError, TypeError):
|
||||
extra = {}
|
||||
fname = extra.get("field_name")
|
||||
if not fname:
|
||||
continue
|
||||
# source_qualified is the full class qualified name
|
||||
class_qual = row["source_qualified"]
|
||||
field_map[(class_qual, fname)] = row["target_qualified"]
|
||||
|
||||
if not field_map:
|
||||
logger.info("Spring resolver: no INJECTS edges with field_name found, skipping")
|
||||
return {"files_indexed": len(java_files), "calls_resolved": 0}
|
||||
|
||||
# -----------------------------------------------------------------------
|
||||
# Build class_name → qualified_name lookup from nodes.
|
||||
# Keyed by bare class name; value is the full "file_path::ClassName" form
|
||||
# that callers_of uses for its target_qualified exact-match lookup.
|
||||
# When a name appears in multiple files (e.g. same interface in several
|
||||
# services), we keep the entry with the shortest path as a tiebreaker —
|
||||
# this is overridden by the concrete-implementation lookup below.
|
||||
# -----------------------------------------------------------------------
|
||||
name_to_qual: dict[str, str] = {}
|
||||
for row in conn.execute(
|
||||
"SELECT name, qualified_name FROM nodes WHERE kind = 'Class' AND language = 'java'"
|
||||
).fetchall():
|
||||
bare = row["name"]
|
||||
qual = row["qualified_name"]
|
||||
if bare not in name_to_qual or len(qual) < len(name_to_qual[bare]):
|
||||
name_to_qual[bare] = qual
|
||||
|
||||
# Also index Function nodes so we can build "file::Class.method" targets.
|
||||
# key: (class_name, method_name) → full qualified_name of the method node
|
||||
method_to_qual: dict[tuple[str, str], str] = {}
|
||||
for row in conn.execute(
|
||||
"SELECT name, qualified_name, parent_name FROM nodes "
|
||||
"WHERE kind IN ('Function', 'Test') AND language = 'java' AND parent_name IS NOT NULL"
|
||||
).fetchall():
|
||||
method_to_qual[(row["parent_name"], row["name"])] = row["qualified_name"]
|
||||
|
||||
# -----------------------------------------------------------------------
|
||||
# Build implementors: bare interface name → list of implementing class quals
|
||||
# from INHERITS edges (Java uses INHERITS for both extends and implements)
|
||||
# -----------------------------------------------------------------------
|
||||
implementors: dict[str, list[str]] = {}
|
||||
for row in conn.execute(
|
||||
"SELECT source_qualified, target_qualified FROM edges WHERE kind = 'INHERITS'"
|
||||
).fetchall():
|
||||
iface = row["target_qualified"]
|
||||
impl = row["source_qualified"]
|
||||
if any(impl.startswith(f) for f in java_files) or "::" in impl:
|
||||
implementors.setdefault(iface, []).append(impl)
|
||||
|
||||
# -----------------------------------------------------------------------
|
||||
# Resolve CALLS edges
|
||||
# -----------------------------------------------------------------------
|
||||
calls_rows = conn.execute(
|
||||
"SELECT id, source_qualified, target_qualified, extra, file_path "
|
||||
"FROM edges WHERE kind = 'CALLS'"
|
||||
).fetchall()
|
||||
|
||||
resolved = 0
|
||||
|
||||
for row in calls_rows:
|
||||
if row["file_path"] not in java_files:
|
||||
continue
|
||||
|
||||
try:
|
||||
extra = json.loads(row["extra"] or "{}")
|
||||
except (json.JSONDecodeError, TypeError):
|
||||
extra = {}
|
||||
|
||||
receiver = extra.get("receiver")
|
||||
if not receiver:
|
||||
continue
|
||||
|
||||
# Skip edges already spring-resolved in a previous pass
|
||||
if extra.get("spring_resolved"):
|
||||
continue
|
||||
|
||||
# Strip any prior (possibly wrong) qualification — we have a receiver so
|
||||
# we can do a better resolution. E.g. "file::ClassName.method" → "method"
|
||||
raw_target = row["target_qualified"]
|
||||
if "::" in raw_target:
|
||||
after = raw_target.split("::", 1)[1]
|
||||
method_name = after.split(".")[-1] if "." in after else after
|
||||
else:
|
||||
method_name = raw_target
|
||||
source_qual = row["source_qualified"]
|
||||
|
||||
# Derive the enclosing class qualified name from source
|
||||
# source_qual format: "file_path::ClassName.method_name"
|
||||
enclosing_class_qual: str | None = None
|
||||
if "::" in source_qual:
|
||||
after_sep = source_qual.split("::", 1)[1]
|
||||
if "." in after_sep:
|
||||
class_part = after_sep.split(".")[0]
|
||||
prefix = source_qual.split("::")[0]
|
||||
enclosing_class_qual = f"{prefix}::{class_part}"
|
||||
else:
|
||||
enclosing_class_qual = source_qual
|
||||
|
||||
if not enclosing_class_qual:
|
||||
continue
|
||||
|
||||
# Look up receiver in field_map for this class
|
||||
injected_type = field_map.get((enclosing_class_qual, receiver))
|
||||
if not injected_type:
|
||||
continue
|
||||
|
||||
# Resolve to concrete implementation if unique
|
||||
impls = implementors.get(injected_type, [])
|
||||
if len(impls) == 1:
|
||||
concrete_class = impls[0].split("::")[-1]
|
||||
fallback = f"{impls[0]}.{method_name}"
|
||||
new_target = method_to_qual.get((concrete_class, method_name)) or fallback
|
||||
else:
|
||||
type_bare = injected_type.rsplit(".", 1)[-1]
|
||||
fallback = f"{injected_type}.{method_name}"
|
||||
new_target = method_to_qual.get((type_bare, method_name)) or fallback
|
||||
|
||||
extra["spring_resolved"] = True
|
||||
extra["injected_type"] = injected_type
|
||||
new_extra = json.dumps(extra)
|
||||
|
||||
conn.execute(
|
||||
"UPDATE edges SET target_qualified = ?, extra = ? WHERE id = ?",
|
||||
(new_target, new_extra, row["id"]),
|
||||
)
|
||||
resolved += 1
|
||||
logger.debug(
|
||||
"Spring resolved: %s → %s (was %s, receiver=%s)",
|
||||
source_qual, new_target, method_name, receiver,
|
||||
)
|
||||
|
||||
if resolved:
|
||||
conn.commit()
|
||||
|
||||
logger.info("Spring DI resolver: resolved %d CALLS edges in %d Java files",
|
||||
resolved, len(java_files))
|
||||
return {"files_indexed": len(java_files), "calls_resolved": resolved}
|
||||
@@ -0,0 +1,199 @@
|
||||
"""Post-build Temporal workflow/activity call resolver.
|
||||
|
||||
After tree-sitter parsing, Java CALLS edges whose target is a bare method
|
||||
name carry ``extra.receiver`` naming the local variable called on. This
|
||||
module resolves those receivers through the TEMPORAL_STUB map to their
|
||||
declared Temporal interface type, then optionally to the unique concrete
|
||||
implementation via INHERITS edges.
|
||||
|
||||
Resolution chain:
|
||||
receiver variable name
|
||||
→ temporal stub field type (from TEMPORAL_STUB.extra.field_name)
|
||||
→ concrete implementation (from INHERITS, when unique)
|
||||
|
||||
Only Java files are processed. TEMPORAL_STUB edges whose target is not a
|
||||
node with ``temporal_role`` in extra are silently skipped (they may be
|
||||
non-Temporal types that happen to end in 'Activity'/'Workflow').
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from .graph import GraphStore
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def resolve_temporal_calls(store: GraphStore) -> dict:
|
||||
"""Resolve Java CALLS edges whose receiver is a Temporal activity/workflow stub.
|
||||
|
||||
Safe to call multiple times — already-resolved edges (with
|
||||
``extra.temporal_resolved``) are skipped.
|
||||
|
||||
Returns a dict with resolution counts for telemetry.
|
||||
"""
|
||||
conn = store._conn
|
||||
|
||||
java_files: set[str] = {
|
||||
row["file_path"]
|
||||
for row in conn.execute(
|
||||
"SELECT DISTINCT file_path FROM nodes WHERE language = 'java'"
|
||||
).fetchall()
|
||||
}
|
||||
if not java_files:
|
||||
return {"files_indexed": 0, "calls_resolved": 0}
|
||||
|
||||
# -----------------------------------------------------------------------
|
||||
# Collect Temporal interface nodes: bare name → qualified_name
|
||||
# (nodes whose extra contains temporal_role = workflow_interface|activity_interface)
|
||||
# -----------------------------------------------------------------------
|
||||
temporal_interfaces: dict[str, str] = {} # bare_name → qualified_name
|
||||
for row in conn.execute(
|
||||
"SELECT name, qualified_name, extra FROM nodes "
|
||||
"WHERE language = 'java' AND extra IS NOT NULL AND extra LIKE '%temporal_role%'"
|
||||
).fetchall():
|
||||
try:
|
||||
ex = json.loads(row["extra"] or "{}")
|
||||
except (json.JSONDecodeError, TypeError):
|
||||
ex = {}
|
||||
if ex.get("temporal_role") in ("workflow_interface", "activity_interface"):
|
||||
temporal_interfaces[row["name"]] = row["qualified_name"]
|
||||
|
||||
if not temporal_interfaces:
|
||||
logger.info("Temporal resolver: no Workflow/ActivityInterface nodes, skipping")
|
||||
return {"files_indexed": len(java_files), "calls_resolved": 0}
|
||||
|
||||
# -----------------------------------------------------------------------
|
||||
# Build field_map: (source_qualified_class, field_name) → interface_type
|
||||
# from TEMPORAL_STUB edges whose target is a known Temporal interface
|
||||
# -----------------------------------------------------------------------
|
||||
field_map: dict[tuple[str, str], str] = {}
|
||||
for row in conn.execute(
|
||||
"SELECT source_qualified, target_qualified, extra FROM edges WHERE kind = 'TEMPORAL_STUB'"
|
||||
).fetchall():
|
||||
bare_target = row["target_qualified"]
|
||||
if bare_target not in temporal_interfaces:
|
||||
continue
|
||||
try:
|
||||
extra = json.loads(row["extra"] or "{}")
|
||||
except (json.JSONDecodeError, TypeError):
|
||||
extra = {}
|
||||
fname = extra.get("field_name")
|
||||
if not fname:
|
||||
continue
|
||||
field_map[(row["source_qualified"], fname)] = bare_target
|
||||
|
||||
if not field_map:
|
||||
logger.info("Temporal resolver: no TEMPORAL_STUB edges found, skipping")
|
||||
return {"files_indexed": len(java_files), "calls_resolved": 0}
|
||||
|
||||
# -----------------------------------------------------------------------
|
||||
# method_to_qual: (class_name, method_name) → full qualified_name
|
||||
# -----------------------------------------------------------------------
|
||||
method_to_qual: dict[tuple[str, str], str] = {}
|
||||
for row in conn.execute(
|
||||
"SELECT name, qualified_name, parent_name FROM nodes "
|
||||
"WHERE kind IN ('Function', 'Test') AND language = 'java' AND parent_name IS NOT NULL"
|
||||
).fetchall():
|
||||
method_to_qual[(row["parent_name"], row["name"])] = row["qualified_name"]
|
||||
|
||||
# -----------------------------------------------------------------------
|
||||
# implementors: bare interface name → list of implementing class quals
|
||||
# -----------------------------------------------------------------------
|
||||
implementors: dict[str, list[str]] = {}
|
||||
for row in conn.execute(
|
||||
"SELECT source_qualified, target_qualified FROM edges WHERE kind = 'INHERITS'"
|
||||
).fetchall():
|
||||
iface = row["target_qualified"]
|
||||
impl = row["source_qualified"]
|
||||
if any(impl.startswith(f) for f in java_files) or "::" in impl:
|
||||
implementors.setdefault(iface, []).append(impl)
|
||||
|
||||
# -----------------------------------------------------------------------
|
||||
# Resolve CALLS edges
|
||||
# -----------------------------------------------------------------------
|
||||
calls_rows = conn.execute(
|
||||
"SELECT id, source_qualified, target_qualified, extra, file_path "
|
||||
"FROM edges WHERE kind = 'CALLS'"
|
||||
).fetchall()
|
||||
|
||||
resolved = 0
|
||||
|
||||
for row in calls_rows:
|
||||
if row["file_path"] not in java_files:
|
||||
continue
|
||||
|
||||
try:
|
||||
extra = json.loads(row["extra"] or "{}")
|
||||
except (json.JSONDecodeError, TypeError):
|
||||
extra = {}
|
||||
|
||||
receiver = extra.get("receiver")
|
||||
if not receiver:
|
||||
continue
|
||||
|
||||
if extra.get("temporal_resolved") or extra.get("spring_resolved"):
|
||||
continue
|
||||
|
||||
raw_target = row["target_qualified"]
|
||||
if "::" in raw_target:
|
||||
after = raw_target.split("::", 1)[1]
|
||||
method_name = after.split(".")[-1] if "." in after else after
|
||||
else:
|
||||
method_name = raw_target
|
||||
|
||||
source_qual = row["source_qualified"]
|
||||
|
||||
# Derive enclosing class qualified name
|
||||
enclosing_class_qual: str | None = None
|
||||
if "::" in source_qual:
|
||||
after_sep = source_qual.split("::", 1)[1]
|
||||
if "." in after_sep:
|
||||
class_part = after_sep.split(".")[0]
|
||||
prefix = source_qual.split("::")[0]
|
||||
enclosing_class_qual = f"{prefix}::{class_part}"
|
||||
else:
|
||||
enclosing_class_qual = source_qual
|
||||
|
||||
if not enclosing_class_qual:
|
||||
continue
|
||||
|
||||
interface_bare = field_map.get((enclosing_class_qual, receiver))
|
||||
if not interface_bare:
|
||||
continue
|
||||
|
||||
interface_qual = temporal_interfaces.get(interface_bare, interface_bare)
|
||||
|
||||
impls = implementors.get(interface_qual, [])
|
||||
if len(impls) == 1:
|
||||
concrete_class = impls[0].split("::")[-1]
|
||||
fallback = f"{impls[0]}.{method_name}"
|
||||
new_target = method_to_qual.get((concrete_class, method_name)) or fallback
|
||||
else:
|
||||
fallback = f"{interface_qual}.{method_name}"
|
||||
new_target = method_to_qual.get((interface_bare, method_name)) or fallback
|
||||
|
||||
extra["temporal_resolved"] = True
|
||||
extra["temporal_interface"] = interface_bare
|
||||
new_extra = json.dumps(extra)
|
||||
|
||||
conn.execute(
|
||||
"UPDATE edges SET target_qualified = ?, extra = ? WHERE id = ?",
|
||||
(new_target, new_extra, row["id"]),
|
||||
)
|
||||
resolved += 1
|
||||
logger.debug(
|
||||
"Temporal resolved: %s → %s (receiver=%s, interface=%s)",
|
||||
source_qual, new_target, receiver, interface_bare,
|
||||
)
|
||||
|
||||
if resolved:
|
||||
conn.commit()
|
||||
|
||||
logger.info("Temporal resolver: resolved %d CALLS edges in %d Java files",
|
||||
resolved, len(java_files))
|
||||
return {"files_indexed": len(java_files), "calls_resolved": resolved}
|
||||
@@ -0,0 +1,125 @@
|
||||
"""Token reduction benchmark -- measures graph query efficiency vs naive file reading."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import sqlite3
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from .graph import GraphStore
|
||||
from .search import hybrid_search
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Sample questions for benchmarking
|
||||
_SAMPLE_QUESTIONS = [
|
||||
"how does authentication work",
|
||||
"what is the main entry point",
|
||||
"how are database connections managed",
|
||||
"what error handling patterns are used",
|
||||
"how do tests verify core functionality",
|
||||
]
|
||||
|
||||
|
||||
def estimate_tokens(text: str) -> int:
|
||||
"""Rough token estimate: ~4 chars per token."""
|
||||
return max(1, len(text) // 4)
|
||||
|
||||
|
||||
def compute_naive_tokens(repo_root: Path) -> int:
|
||||
"""Count tokens in all parseable source files."""
|
||||
total = 0
|
||||
exts = (
|
||||
".py", ".js", ".ts", ".go", ".rs", ".java",
|
||||
".c", ".cpp", ".rb", ".php", ".swift", ".kt",
|
||||
)
|
||||
for ext in exts:
|
||||
for f in repo_root.rglob(f"*{ext}"):
|
||||
try:
|
||||
total += estimate_tokens(
|
||||
f.read_text(errors="replace")
|
||||
)
|
||||
except OSError:
|
||||
continue
|
||||
return total
|
||||
|
||||
|
||||
def run_token_benchmark(
|
||||
store: GraphStore,
|
||||
repo_root: Path,
|
||||
questions: list[str] | None = None,
|
||||
) -> dict[str, Any]:
|
||||
"""Run token reduction benchmark.
|
||||
|
||||
Compares naive full-corpus token cost vs graph query token
|
||||
cost for a set of sample questions.
|
||||
|
||||
The default sample questions are natural language and require semantic
|
||||
search to match. If no embeddings are present in the graph, ``hybrid_search``
|
||||
falls back to FTS5/LIKE matching on node names, which produces no hits for
|
||||
questions like "how does authentication work" — every per-question ratio
|
||||
becomes 0 and the benchmark silently appears to fail. We log a clear
|
||||
warning when that is the case so callers know to run ``embed_graph`` first
|
||||
(or to pass keyword-matching questions).
|
||||
"""
|
||||
if questions is None:
|
||||
questions = _SAMPLE_QUESTIONS
|
||||
|
||||
using_default_questions = questions is _SAMPLE_QUESTIONS
|
||||
try:
|
||||
cur = store._conn.execute("SELECT count(*) FROM embeddings")
|
||||
embedding_count = cur.fetchone()[0]
|
||||
except sqlite3.OperationalError:
|
||||
embedding_count = 0
|
||||
if embedding_count == 0 and using_default_questions:
|
||||
logger.warning(
|
||||
"No embeddings found in this graph. The default sample questions "
|
||||
"are natural language and will not match via FTS5/LIKE alone — "
|
||||
"every reduction ratio is likely to be 0. Run "
|
||||
"`code-review-graph embed` first, or pass keyword-matching `questions=`."
|
||||
)
|
||||
|
||||
naive_total = compute_naive_tokens(repo_root)
|
||||
|
||||
results = []
|
||||
for q in questions:
|
||||
search_results = hybrid_search(store, q, limit=5)
|
||||
# Simulate graph context: search results + neighbors
|
||||
graph_tokens = 0
|
||||
for r in search_results:
|
||||
graph_tokens += estimate_tokens(str(r))
|
||||
# Add approximate neighbor context
|
||||
qn = r.get("qualified_name", "")
|
||||
edges = store.get_edges_by_source(qn)[:5]
|
||||
for e in edges:
|
||||
graph_tokens += estimate_tokens(str(e))
|
||||
|
||||
if graph_tokens > 0:
|
||||
ratio = naive_total / graph_tokens
|
||||
else:
|
||||
ratio = 0
|
||||
results.append({
|
||||
"question": q,
|
||||
"naive_tokens": naive_total,
|
||||
"graph_tokens": graph_tokens,
|
||||
"reduction_ratio": round(ratio, 1),
|
||||
})
|
||||
|
||||
if results:
|
||||
total = sum(
|
||||
r["reduction_ratio"] for r in results # type: ignore[misc]
|
||||
)
|
||||
avg_ratio = float(total) / len(results) # type: ignore[arg-type]
|
||||
else:
|
||||
avg_ratio = 0.0
|
||||
|
||||
return {
|
||||
"naive_corpus_tokens": naive_total,
|
||||
"per_question": results,
|
||||
"average_reduction_ratio": round(avg_ratio, 1),
|
||||
"summary": (
|
||||
f"Graph queries use ~{avg_ratio:.0f}x fewer tokens "
|
||||
f"than reading all source files"
|
||||
),
|
||||
}
|
||||
@@ -0,0 +1,156 @@
|
||||
"""MCP tool definitions for the Code Review Graph server.
|
||||
|
||||
Exposes 27 tools:
|
||||
1. build_or_update_graph - full or incremental build
|
||||
2. get_impact_radius - blast radius from changed files
|
||||
3. query_graph - predefined graph queries
|
||||
4. get_review_context - focused subgraph + review prompt
|
||||
5. semantic_search_nodes - keyword + vector search across nodes
|
||||
6. list_graph_stats - aggregate statistics
|
||||
7. embed_graph - compute vector embeddings for semantic search
|
||||
8. get_docs_section - token-optimized documentation retrieval
|
||||
9. find_large_functions - find oversized functions/classes by line count
|
||||
10. list_flows - list execution flows sorted by criticality
|
||||
11. get_flow - get details of a single execution flow
|
||||
12. get_affected_flows - find flows affected by changed files
|
||||
13. list_communities - list detected code communities
|
||||
14. get_community - get details of a single community
|
||||
15. get_architecture_overview - architecture overview from community structure
|
||||
16. detect_changes - risk-scored change impact analysis for code review
|
||||
17. refactor_tool - unified refactoring (rename preview, dead code, suggestions)
|
||||
18. apply_refactor_tool - apply a previously previewed refactoring
|
||||
19. generate_wiki - generate markdown wiki from community structure
|
||||
20. get_wiki_page - retrieve a specific wiki page
|
||||
21. list_repos - list registered repositories
|
||||
22. cross_repo_search - search across all registered repositories
|
||||
23. get_hub_nodes - find most connected nodes (architectural hotspots)
|
||||
24. get_bridge_nodes - find architectural chokepoints (betweenness centrality)
|
||||
25. get_knowledge_gaps - identify structural weaknesses
|
||||
26. get_surprising_connections - find unexpected architectural coupling
|
||||
27. get_suggested_questions - auto-generated review questions from graph analysis
|
||||
28. traverse_graph - BFS/DFS traversal from best-matching node
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
# Re-export names that external code may patch via "code_review_graph.tools.*"
|
||||
from ..changes import parse_diff_ranges as parse_diff_ranges
|
||||
from ..changes import parse_git_diff_ranges as parse_git_diff_ranges
|
||||
from ..changes import parse_svn_diff_ranges as parse_svn_diff_ranges
|
||||
from ..incremental import (
|
||||
get_changed_files as get_changed_files,
|
||||
)
|
||||
from ..incremental import (
|
||||
get_staged_and_unstaged as get_staged_and_unstaged,
|
||||
)
|
||||
|
||||
# -- _common ----------------------------------------------------------------
|
||||
from ._common import (
|
||||
_BUILTIN_CALL_NAMES,
|
||||
_get_store,
|
||||
_validate_repo_root,
|
||||
)
|
||||
|
||||
# -- analysis_tools ---------------------------------------------------------
|
||||
from .analysis_tools import (
|
||||
get_bridge_nodes_func,
|
||||
get_hub_nodes_func,
|
||||
get_knowledge_gaps_func,
|
||||
get_suggested_questions_func,
|
||||
get_surprising_connections_func,
|
||||
)
|
||||
|
||||
# -- build ------------------------------------------------------------------
|
||||
from .build import build_or_update_graph, run_postprocess
|
||||
|
||||
# -- community_tools --------------------------------------------------------
|
||||
from .community_tools import (
|
||||
get_architecture_overview_func,
|
||||
get_community_func,
|
||||
list_communities_func,
|
||||
)
|
||||
|
||||
# -- context ----------------------------------------------------------------
|
||||
from .context import get_minimal_context
|
||||
|
||||
# -- docs -------------------------------------------------------------------
|
||||
from .docs import embed_graph, generate_wiki_func, get_docs_section, get_wiki_page_func
|
||||
|
||||
# -- flows_tools ------------------------------------------------------------
|
||||
from .flows_tools import get_flow, list_flows
|
||||
|
||||
# -- query ------------------------------------------------------------------
|
||||
from .query import (
|
||||
find_large_functions,
|
||||
get_impact_radius,
|
||||
list_graph_stats,
|
||||
query_graph,
|
||||
semantic_search_nodes,
|
||||
traverse_graph_func,
|
||||
)
|
||||
|
||||
# -- refactor_tools ---------------------------------------------------------
|
||||
from .refactor_tools import apply_refactor_func, refactor_func
|
||||
|
||||
# -- registry_tools ---------------------------------------------------------
|
||||
from .registry_tools import cross_repo_search_func, list_repos_func
|
||||
|
||||
# -- review -----------------------------------------------------------------
|
||||
from .review import (
|
||||
detect_changes_func,
|
||||
get_affected_flows_func,
|
||||
get_review_context,
|
||||
)
|
||||
|
||||
__all__ = [
|
||||
# _common
|
||||
"_BUILTIN_CALL_NAMES",
|
||||
"_get_store",
|
||||
"_validate_repo_root",
|
||||
# build
|
||||
"build_or_update_graph",
|
||||
"run_postprocess",
|
||||
# context
|
||||
"get_minimal_context",
|
||||
# community_tools
|
||||
"get_architecture_overview_func",
|
||||
"get_community_func",
|
||||
"list_communities_func",
|
||||
# docs
|
||||
"embed_graph",
|
||||
"generate_wiki_func",
|
||||
"get_docs_section",
|
||||
"get_wiki_page_func",
|
||||
# flows_tools
|
||||
"get_flow",
|
||||
"list_flows",
|
||||
# query
|
||||
"find_large_functions",
|
||||
"get_impact_radius",
|
||||
"list_graph_stats",
|
||||
"query_graph",
|
||||
"semantic_search_nodes",
|
||||
"traverse_graph_func",
|
||||
# refactor_tools
|
||||
"apply_refactor_func",
|
||||
"refactor_func",
|
||||
# registry_tools
|
||||
"cross_repo_search_func",
|
||||
"list_repos_func",
|
||||
# review
|
||||
"detect_changes_func",
|
||||
"get_affected_flows_func",
|
||||
"get_review_context",
|
||||
# analysis_tools
|
||||
"get_bridge_nodes_func",
|
||||
"get_hub_nodes_func",
|
||||
"get_knowledge_gaps_func",
|
||||
"get_suggested_questions_func",
|
||||
"get_surprising_connections_func",
|
||||
# re-exported for backward compat (used in test patches)
|
||||
"get_changed_files",
|
||||
"get_staged_and_unstaged",
|
||||
"parse_git_diff_ranges",
|
||||
"parse_svn_diff_ranges",
|
||||
"parse_diff_ranges",
|
||||
]
|
||||
@@ -0,0 +1,176 @@
|
||||
"""Shared utilities for tool sub-modules."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from ..graph import GraphStore
|
||||
from ..incremental import find_project_root, get_db_path
|
||||
|
||||
|
||||
def _error_response(
|
||||
message: str, status: str = "error", **extra: Any,
|
||||
) -> dict[str, Any]:
|
||||
"""Build a standardised error response dict."""
|
||||
return {"status": status, "error": message, "summary": message, **extra}
|
||||
|
||||
# Common JS/TS builtin method names filtered from callers_of results.
|
||||
# "Who calls .map()?" returns hundreds of hits and is never useful.
|
||||
# These are kept in the graph (callees_of still shows them) but excluded
|
||||
# when doing reverse call tracing to reduce noise.
|
||||
_BUILTIN_CALL_NAMES: set[str] = {
|
||||
"map", "filter", "reduce", "reduceRight", "forEach", "find", "findIndex",
|
||||
"some", "every", "includes", "indexOf", "lastIndexOf",
|
||||
"push", "pop", "shift", "unshift", "splice", "slice",
|
||||
"concat", "join", "flat", "flatMap", "sort", "reverse", "fill",
|
||||
"keys", "values", "entries", "from", "isArray", "of", "at",
|
||||
"trim", "trimStart", "trimEnd", "split", "replace", "replaceAll",
|
||||
"match", "matchAll", "search", "substring", "substr",
|
||||
"toLowerCase", "toUpperCase", "startsWith", "endsWith",
|
||||
"padStart", "padEnd", "repeat", "charAt", "charCodeAt",
|
||||
"assign", "freeze", "defineProperty", "getOwnPropertyNames",
|
||||
"hasOwnProperty", "create", "is", "fromEntries",
|
||||
"log", "warn", "error", "info", "debug", "trace", "dir", "table",
|
||||
"time", "timeEnd", "assert", "clear", "count",
|
||||
"then", "catch", "finally", "resolve", "reject", "all", "allSettled", "race", "any",
|
||||
"parse", "stringify",
|
||||
"floor", "ceil", "round", "random", "max", "min", "abs", "pow", "sqrt",
|
||||
"addEventListener", "removeEventListener", "querySelector", "querySelectorAll",
|
||||
"getElementById", "createElement", "appendChild", "removeChild",
|
||||
"setAttribute", "getAttribute", "preventDefault", "stopPropagation",
|
||||
"setTimeout", "clearTimeout", "setInterval", "clearInterval",
|
||||
"toString", "valueOf", "toJSON", "toISOString",
|
||||
"getTime", "getFullYear", "now",
|
||||
"isNaN", "parseInt", "parseFloat", "toFixed",
|
||||
"encodeURIComponent", "decodeURIComponent",
|
||||
"call", "apply", "bind", "next",
|
||||
"emit", "on", "off", "once",
|
||||
"pipe", "write", "read", "end", "close", "destroy",
|
||||
"send", "status", "json", "redirect",
|
||||
"set", "get", "delete", "has",
|
||||
"findUnique", "findFirst", "findMany", "createMany",
|
||||
"update", "updateMany", "deleteMany", "upsert",
|
||||
"aggregate", "groupBy", "transaction",
|
||||
"describe", "it", "test", "expect", "beforeEach", "afterEach",
|
||||
"beforeAll", "afterAll", "mock", "spyOn",
|
||||
"require", "fetch",
|
||||
}
|
||||
|
||||
|
||||
def _validate_repo_root(path: "Path | str") -> Path:
|
||||
"""Validate that a path is a plausible project root.
|
||||
|
||||
Ensures the path is an existing directory that contains a ``.git``,
|
||||
``.svn``, or ``.code-review-graph`` directory, preventing arbitrary
|
||||
file-system traversal via the ``repo_root`` parameter.
|
||||
"""
|
||||
resolved = Path(path).resolve()
|
||||
if not resolved.is_dir():
|
||||
raise ValueError(
|
||||
f"repo_root is not an existing directory: {resolved}"
|
||||
)
|
||||
has_vcs = (
|
||||
(resolved / ".git").exists()
|
||||
or (resolved / ".svn").exists()
|
||||
or (resolved / ".code-review-graph").exists()
|
||||
)
|
||||
if not has_vcs:
|
||||
raise ValueError(
|
||||
f"repo_root does not look like a project root "
|
||||
f"(no .git, .svn, or .code-review-graph directory found): "
|
||||
f"{resolved}"
|
||||
)
|
||||
return resolved
|
||||
|
||||
|
||||
def _resolve_root(repo_root: str | None = None) -> Path:
|
||||
"""Resolve and validate the repository root without opening a store."""
|
||||
return _validate_repo_root(Path(repo_root)) if repo_root else find_project_root()
|
||||
|
||||
|
||||
def _get_store(repo_root: str | None = None) -> tuple[GraphStore, Path]:
|
||||
"""Resolve repo root and open the graph store.
|
||||
|
||||
Callers own the returned store and must close it (try/finally or
|
||||
context manager) to avoid leaking SQLite file descriptors.
|
||||
"""
|
||||
root = _resolve_root(repo_root)
|
||||
db_path = get_db_path(root)
|
||||
return GraphStore(db_path), root
|
||||
|
||||
|
||||
def _resolve_graph_file_paths(
|
||||
store: GraphStore, root: Path, file_paths: list[str],
|
||||
) -> list[str]:
|
||||
"""Resolve user-facing file paths to the paths stored in the graph.
|
||||
|
||||
Graphs may contain absolute paths, repo-relative paths, or cwd-relative
|
||||
paths depending on how they were built. Tool inputs are usually relative to
|
||||
repo root, so exact matching alone can miss existing graph nodes.
|
||||
"""
|
||||
resolved: list[str] = []
|
||||
seen: set[str] = set()
|
||||
|
||||
def add(path: str) -> None:
|
||||
if path not in seen:
|
||||
resolved.append(path)
|
||||
seen.add(path)
|
||||
|
||||
for file_path in file_paths:
|
||||
raw = file_path.replace("\\", "/")
|
||||
candidates = [raw]
|
||||
path = Path(file_path)
|
||||
if path.is_absolute():
|
||||
try:
|
||||
candidates.append(str(path.resolve().relative_to(root)).replace("\\", "/"))
|
||||
except ValueError:
|
||||
pass
|
||||
else:
|
||||
candidates.append(str(root / path))
|
||||
|
||||
for candidate in candidates:
|
||||
if store.get_nodes_by_file(candidate):
|
||||
add(candidate)
|
||||
|
||||
suffixes = []
|
||||
for candidate in candidates:
|
||||
normalized = candidate.replace("\\", "/")
|
||||
if normalized not in suffixes:
|
||||
suffixes.append(normalized)
|
||||
|
||||
for suffix in suffixes:
|
||||
for matched_path in store.get_files_matching(suffix):
|
||||
add(matched_path)
|
||||
|
||||
return resolved
|
||||
|
||||
|
||||
def compact_response(
|
||||
summary: str,
|
||||
key_entities: list[str] | None = None,
|
||||
risk: str = "unknown",
|
||||
communities: list[str] | None = None,
|
||||
flows_affected: list[str] | None = None,
|
||||
next_tool_suggestions: list[str] | None = None,
|
||||
data: dict[str, Any] | None = None,
|
||||
detail_level: str = "minimal",
|
||||
) -> dict[str, Any]:
|
||||
"""Standard compact response format for token efficiency."""
|
||||
resp: dict[str, Any] = {
|
||||
"status": "ok",
|
||||
"summary": summary,
|
||||
}
|
||||
if key_entities:
|
||||
resp["key_entities"] = key_entities[:10]
|
||||
if risk != "unknown":
|
||||
resp["risk"] = risk
|
||||
if communities:
|
||||
resp["communities"] = communities[:5]
|
||||
if flows_affected:
|
||||
resp["flows_affected"] = flows_affected[:5]
|
||||
if next_tool_suggestions:
|
||||
resp["next_tool_suggestions"] = next_tool_suggestions[:3]
|
||||
if detail_level != "minimal" and data:
|
||||
resp["data"] = data
|
||||
return resp
|
||||
@@ -0,0 +1,184 @@
|
||||
"""MCP tool wrappers for graph analysis features."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
|
||||
from ..analysis import (
|
||||
find_bridge_nodes,
|
||||
find_hub_nodes,
|
||||
find_knowledge_gaps,
|
||||
find_surprising_connections,
|
||||
generate_suggested_questions,
|
||||
)
|
||||
from ._common import _get_store
|
||||
|
||||
|
||||
def get_hub_nodes_func(
|
||||
repo_root: str | None = None,
|
||||
top_n: int = 10,
|
||||
) -> dict[str, Any]:
|
||||
"""Find the most connected nodes in the codebase graph.
|
||||
|
||||
Hub nodes have the highest total degree (in + out edges).
|
||||
These are architectural hotspots -- changes to them have
|
||||
disproportionate blast radius.
|
||||
|
||||
Args:
|
||||
repo_root: Repository root (auto-detected if omitted).
|
||||
top_n: Number of top hubs to return (default 10).
|
||||
"""
|
||||
store, _root = _get_store(repo_root or None)
|
||||
try:
|
||||
hubs = find_hub_nodes(store, top_n=top_n)
|
||||
return {
|
||||
"hub_nodes": hubs,
|
||||
"count": len(hubs),
|
||||
"next_tool_suggestions": [
|
||||
"get_impact_radius -- check blast radius of a hub",
|
||||
"query_graph callers_of -- see what calls a hub",
|
||||
"get_bridge_nodes -- find architectural chokepoints",
|
||||
],
|
||||
}
|
||||
finally:
|
||||
store.close()
|
||||
|
||||
|
||||
def get_bridge_nodes_func(
|
||||
repo_root: str | None = None,
|
||||
top_n: int = 10,
|
||||
) -> dict[str, Any]:
|
||||
"""Find architectural chokepoints via betweenness centrality.
|
||||
|
||||
Bridge nodes sit on the shortest paths between many node
|
||||
pairs. If they break, multiple code regions lose
|
||||
connectivity.
|
||||
|
||||
Args:
|
||||
repo_root: Repository root (auto-detected if omitted).
|
||||
top_n: Number of top bridges to return (default 10).
|
||||
"""
|
||||
store, _root = _get_store(repo_root or None)
|
||||
try:
|
||||
bridges = find_bridge_nodes(store, top_n=top_n)
|
||||
return {
|
||||
"bridge_nodes": bridges,
|
||||
"count": len(bridges),
|
||||
"next_tool_suggestions": [
|
||||
"get_hub_nodes -- find most connected nodes",
|
||||
"get_impact_radius -- check blast radius",
|
||||
"detect_changes -- see if bridges are affected",
|
||||
],
|
||||
}
|
||||
finally:
|
||||
store.close()
|
||||
|
||||
|
||||
def get_knowledge_gaps_func(
|
||||
repo_root: str | None = None,
|
||||
) -> dict[str, Any]:
|
||||
"""Identify structural weaknesses in the codebase.
|
||||
|
||||
Finds: isolated nodes (disconnected), thin communities
|
||||
(< 3 members), untested hotspots (high-degree, no tests),
|
||||
and single-file communities.
|
||||
|
||||
Args:
|
||||
repo_root: Repository root (auto-detected if omitted).
|
||||
"""
|
||||
store, _root = _get_store(repo_root or None)
|
||||
try:
|
||||
gaps = find_knowledge_gaps(store)
|
||||
total = sum(len(v) for v in gaps.values())
|
||||
return {
|
||||
"gaps": gaps,
|
||||
"total_gaps": total,
|
||||
"summary": {
|
||||
"isolated_nodes": len(gaps["isolated_nodes"]),
|
||||
"thin_communities": len(
|
||||
gaps["thin_communities"]
|
||||
),
|
||||
"untested_hotspots": len(
|
||||
gaps["untested_hotspots"]
|
||||
),
|
||||
"single_file_communities": len(
|
||||
gaps["single_file_communities"]
|
||||
),
|
||||
},
|
||||
"next_tool_suggestions": [
|
||||
"refactor dead_code -- find unused symbols",
|
||||
"get_hub_nodes -- find high-impact nodes",
|
||||
"get_suggested_questions -- review prompts",
|
||||
],
|
||||
}
|
||||
finally:
|
||||
store.close()
|
||||
|
||||
|
||||
def get_surprising_connections_func(
|
||||
repo_root: str | None = None,
|
||||
top_n: int = 15,
|
||||
) -> dict[str, Any]:
|
||||
"""Find unexpected architectural coupling in the codebase.
|
||||
|
||||
Scores edges by surprise factors: cross-community,
|
||||
cross-language, peripheral-to-hub, cross-test-boundary.
|
||||
|
||||
Args:
|
||||
repo_root: Repository root (auto-detected if omitted).
|
||||
top_n: Number of top surprises to return (default 15).
|
||||
"""
|
||||
store, _root = _get_store(repo_root or None)
|
||||
try:
|
||||
surprises = find_surprising_connections(
|
||||
store, top_n=top_n
|
||||
)
|
||||
return {
|
||||
"surprising_connections": surprises,
|
||||
"count": len(surprises),
|
||||
"next_tool_suggestions": [
|
||||
"get_architecture_overview -- community structure",
|
||||
"query_graph callers_of -- trace the coupling",
|
||||
"get_bridge_nodes -- find chokepoints",
|
||||
],
|
||||
}
|
||||
finally:
|
||||
store.close()
|
||||
|
||||
|
||||
def get_suggested_questions_func(
|
||||
repo_root: str | None = None,
|
||||
) -> dict[str, Any]:
|
||||
"""Auto-generate review questions from graph analysis.
|
||||
|
||||
Produces questions about: bridge nodes, untested hubs,
|
||||
surprising connections, thin communities, and untested
|
||||
hotspots.
|
||||
|
||||
Args:
|
||||
repo_root: Repository root (auto-detected if omitted).
|
||||
"""
|
||||
store, _root = _get_store(repo_root or None)
|
||||
try:
|
||||
questions = generate_suggested_questions(store)
|
||||
by_priority: dict[str, list[dict[str, Any]]] = {
|
||||
"high": [], "medium": [], "low": [],
|
||||
}
|
||||
for q in questions:
|
||||
prio = q.get("priority", "medium")
|
||||
if prio in by_priority:
|
||||
by_priority[prio].append(q)
|
||||
return {
|
||||
"questions": questions,
|
||||
"count": len(questions),
|
||||
"by_priority": {
|
||||
k: len(v) for k, v in by_priority.items()
|
||||
},
|
||||
"next_tool_suggestions": [
|
||||
"get_knowledge_gaps -- structural weaknesses",
|
||||
"detect_changes -- risk-scored review",
|
||||
"get_architecture_overview -- community map",
|
||||
],
|
||||
}
|
||||
finally:
|
||||
store.close()
|
||||
@@ -0,0 +1,541 @@
|
||||
"""Tool 1: build_or_update_graph + run_postprocess."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import sqlite3
|
||||
import time
|
||||
from typing import Any
|
||||
|
||||
from ..incremental import full_build, incremental_update
|
||||
from ._common import _get_store
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _run_postprocess(
|
||||
store: Any,
|
||||
build_result: dict[str, Any],
|
||||
postprocess: str,
|
||||
full_rebuild: bool = False,
|
||||
changed_files: list[str] | None = None,
|
||||
) -> list[str]:
|
||||
"""Run post-build steps based on *postprocess* level.
|
||||
|
||||
When *full_rebuild* is False and *changed_files* are available,
|
||||
uses incremental flow/community detection for faster updates.
|
||||
|
||||
Returns a list of warning strings (empty on success).
|
||||
"""
|
||||
warnings: list[str] = []
|
||||
build_result["postprocess_level"] = postprocess
|
||||
|
||||
if postprocess == "none":
|
||||
return warnings
|
||||
|
||||
# -- Signatures + FTS (fast, always run unless "none") --
|
||||
try:
|
||||
rows = store.get_nodes_without_signature()
|
||||
for row in rows:
|
||||
node_id, name, kind, params, ret = (
|
||||
row[0],
|
||||
row[1],
|
||||
row[2],
|
||||
row[3],
|
||||
row[4],
|
||||
)
|
||||
if kind in ("Function", "Test"):
|
||||
sig = f"def {name}({params or ''})"
|
||||
if ret:
|
||||
sig += f" -> {ret}"
|
||||
elif kind == "Class":
|
||||
sig = f"class {name}"
|
||||
else:
|
||||
sig = name
|
||||
store.update_node_signature(node_id, sig[:512])
|
||||
store.commit()
|
||||
build_result["signatures_updated"] = True
|
||||
except (sqlite3.OperationalError, TypeError, KeyError) as e:
|
||||
logger.warning("Signature computation failed: %s", e)
|
||||
warnings.append(f"Signature computation failed: {type(e).__name__}: {e}")
|
||||
|
||||
try:
|
||||
from code_review_graph.search import rebuild_fts_index
|
||||
|
||||
fts_count = rebuild_fts_index(store)
|
||||
build_result["fts_indexed"] = fts_count
|
||||
build_result["fts_rebuilt"] = True
|
||||
except (sqlite3.OperationalError, ImportError) as e:
|
||||
logger.warning("FTS index rebuild failed: %s", e)
|
||||
warnings.append(f"FTS index rebuild failed: {type(e).__name__}: {e}")
|
||||
|
||||
if postprocess == "minimal":
|
||||
return warnings
|
||||
|
||||
# -- Expensive: flows + communities (only for "full") --
|
||||
use_incremental = not full_rebuild and bool(changed_files)
|
||||
|
||||
try:
|
||||
if use_incremental:
|
||||
from code_review_graph.flows import incremental_trace_flows
|
||||
|
||||
count = incremental_trace_flows(store, changed_files)
|
||||
else:
|
||||
from code_review_graph.flows import store_flows as _store_flows
|
||||
from code_review_graph.flows import trace_flows as _trace_flows
|
||||
|
||||
flows = _trace_flows(store)
|
||||
count = _store_flows(store, flows)
|
||||
build_result["flows_detected"] = count
|
||||
except (sqlite3.OperationalError, ImportError) as e:
|
||||
logger.warning("Flow detection failed: %s", e)
|
||||
warnings.append(f"Flow detection failed: {type(e).__name__}: {e}")
|
||||
|
||||
try:
|
||||
if use_incremental:
|
||||
from code_review_graph.communities import (
|
||||
incremental_detect_communities,
|
||||
)
|
||||
|
||||
count = incremental_detect_communities(store, changed_files)
|
||||
else:
|
||||
from code_review_graph.communities import (
|
||||
detect_communities as _detect_communities,
|
||||
)
|
||||
from code_review_graph.communities import (
|
||||
store_communities as _store_communities,
|
||||
)
|
||||
|
||||
comms = _detect_communities(store)
|
||||
count = _store_communities(store, comms)
|
||||
build_result["communities_detected"] = count
|
||||
except (sqlite3.OperationalError, ImportError) as e:
|
||||
logger.warning("Community detection failed: %s", e)
|
||||
warnings.append(f"Community detection failed: {type(e).__name__}: {e}")
|
||||
|
||||
# -- Compute pre-computed summary tables --
|
||||
try:
|
||||
_compute_summaries(store)
|
||||
build_result["summaries_computed"] = True
|
||||
except (sqlite3.OperationalError, Exception) as e:
|
||||
logger.warning("Summary computation failed: %s", e)
|
||||
warnings.append(f"Summary computation failed: {type(e).__name__}: {e}")
|
||||
|
||||
store.set_metadata(
|
||||
"last_postprocessed_at",
|
||||
time.strftime("%Y-%m-%dT%H:%M:%S"),
|
||||
)
|
||||
store.set_metadata("postprocess_level", postprocess)
|
||||
|
||||
return warnings
|
||||
|
||||
|
||||
def _compute_summaries(store: Any) -> None:
|
||||
"""Populate community_summaries, flow_snapshots, and risk_index tables.
|
||||
|
||||
Uses batched aggregate queries and in-memory grouping instead of
|
||||
per-community/per-node loops. On graphs with ~100k edges this
|
||||
reduces the work from ``O(nodes + communities)`` SQLite round trips
|
||||
each doing their own B-tree scan to a handful of ``GROUP BY``
|
||||
queries, turning what used to be an effective hang into a few
|
||||
seconds.
|
||||
|
||||
Each summary block (community_summaries, flow_snapshots, risk_index)
|
||||
is wrapped in an explicit transaction so the DELETE + INSERT sequence
|
||||
is atomic. If a table doesn't exist yet the block is silently skipped.
|
||||
"""
|
||||
import json as _json
|
||||
from collections import defaultdict
|
||||
from os.path import commonprefix
|
||||
|
||||
conn = store._conn
|
||||
|
||||
# -- community_summaries --
|
||||
try:
|
||||
conn.execute("BEGIN IMMEDIATE")
|
||||
conn.execute("DELETE FROM community_summaries")
|
||||
|
||||
# Pre-compute per-qualified_name edge counts once. Previously
|
||||
# this section ran a per-community triple-JOIN aggregate query
|
||||
# (nodes LEFT JOIN edges LEFT JOIN edges), which on graphs with
|
||||
# thousands of communities was the second-biggest hang.
|
||||
edge_counts: dict[str, int] = defaultdict(int)
|
||||
for row in conn.execute(
|
||||
"SELECT source_qualified, COUNT(*) FROM edges GROUP BY source_qualified"
|
||||
):
|
||||
edge_counts[row[0]] += row[1]
|
||||
for row in conn.execute(
|
||||
"SELECT target_qualified, COUNT(*) FROM edges GROUP BY target_qualified"
|
||||
):
|
||||
edge_counts[row[0]] += row[1]
|
||||
|
||||
# Group non-File nodes per community for top-symbol selection.
|
||||
nodes_by_comm: dict[int, list[tuple[str, int]]] = defaultdict(list)
|
||||
for row in conn.execute(
|
||||
"SELECT community_id, name, qualified_name FROM nodes "
|
||||
"WHERE community_id IS NOT NULL AND kind != 'File'"
|
||||
):
|
||||
cid, name, qn = row[0], row[1], row[2]
|
||||
nodes_by_comm[cid].append((name, edge_counts.get(qn, 0)))
|
||||
|
||||
# Group distinct file paths per community (preserving first-seen
|
||||
# order for stable output, same as DISTINCT in the old query).
|
||||
files_by_comm: dict[int, list[str]] = defaultdict(list)
|
||||
seen_files: dict[int, set[str]] = defaultdict(set)
|
||||
for row in conn.execute(
|
||||
"SELECT community_id, file_path FROM nodes WHERE community_id IS NOT NULL"
|
||||
):
|
||||
cid, fp = row[0], row[1]
|
||||
if fp not in seen_files[cid]:
|
||||
seen_files[cid].add(fp)
|
||||
files_by_comm[cid].append(fp)
|
||||
|
||||
community_rows = conn.execute(
|
||||
"SELECT id, name, size, dominant_language FROM communities"
|
||||
).fetchall()
|
||||
for r in community_rows:
|
||||
cid, cname, csize, clang = r[0], r[1], r[2], r[3]
|
||||
|
||||
# Top 5 symbols by total edge count (in + out). Python's
|
||||
# sorted() is stable so ties break by original row order.
|
||||
members = sorted(
|
||||
nodes_by_comm.get(cid, []),
|
||||
key=lambda nc: nc[1],
|
||||
reverse=True,
|
||||
)
|
||||
key_syms = _json.dumps([m[0] for m in members[:5]])
|
||||
|
||||
# Auto-generate purpose from common file path prefix.
|
||||
paths = files_by_comm.get(cid, [])[:20]
|
||||
purpose = ""
|
||||
if paths:
|
||||
prefix = commonprefix(paths)
|
||||
if "/" in prefix:
|
||||
purpose = prefix.rsplit("/", 1)[0].split("/")[-1] if "/" in prefix else ""
|
||||
|
||||
conn.execute(
|
||||
"INSERT OR REPLACE INTO community_summaries "
|
||||
"(community_id, name, purpose, key_symbols, size, dominant_language) "
|
||||
"VALUES (?, ?, ?, ?, ?, ?)",
|
||||
(cid, cname, purpose, key_syms, csize, clang or ""),
|
||||
)
|
||||
conn.commit()
|
||||
except sqlite3.OperationalError:
|
||||
conn.rollback() # Table may not exist yet
|
||||
|
||||
# -- flow_snapshots --
|
||||
try:
|
||||
conn.execute("BEGIN IMMEDIATE")
|
||||
conn.execute("DELETE FROM flow_snapshots")
|
||||
flow_rows = conn.execute(
|
||||
"SELECT id, name, entry_point_id, criticality, node_count, "
|
||||
"file_count, path_json FROM flows"
|
||||
).fetchall()
|
||||
|
||||
# Collect every node id referenced by any flow, then fetch
|
||||
# their qualified_names in one batched query instead of per-flow
|
||||
# per-node lookups.
|
||||
needed_ids: set[int] = set()
|
||||
parsed_paths: list[list[int]] = []
|
||||
for r in flow_rows:
|
||||
needed_ids.add(r[2]) # entry_point_id
|
||||
path_ids = _json.loads(r[6]) if r[6] else []
|
||||
parsed_paths.append(path_ids)
|
||||
# Match the old semantics: entry + up to 3 intermediates + last
|
||||
for nid in path_ids[1:4]:
|
||||
needed_ids.add(nid)
|
||||
if path_ids:
|
||||
needed_ids.add(path_ids[-1])
|
||||
|
||||
id_to_name: dict[int, str] = {}
|
||||
if needed_ids:
|
||||
# Batch the IN clause in chunks of 450 to stay under SQLite's
|
||||
# default SQLITE_MAX_VARIABLE_NUMBER (999), same strategy as
|
||||
# GraphStore.get_edges_among.
|
||||
id_list = list(needed_ids)
|
||||
for i in range(0, len(id_list), 450):
|
||||
batch = id_list[i : i + 450]
|
||||
placeholders = ",".join("?" for _ in batch)
|
||||
node_rows = conn.execute(
|
||||
f"SELECT id, qualified_name FROM nodes WHERE id IN ({placeholders})", # nosec B608
|
||||
batch,
|
||||
).fetchall()
|
||||
for nr in node_rows:
|
||||
id_to_name[nr[0]] = nr[1]
|
||||
|
||||
for r, path_ids in zip(flow_rows, parsed_paths):
|
||||
fid, fname, ep_id = r[0], r[1], r[2]
|
||||
crit, ncount, fcount = r[3], r[4], r[5]
|
||||
ep_name = id_to_name.get(ep_id, str(ep_id))
|
||||
critical_path: list[str] = []
|
||||
if path_ids:
|
||||
critical_path.append(ep_name)
|
||||
if len(path_ids) > 2:
|
||||
for nid in path_ids[1:4]:
|
||||
nm = id_to_name.get(nid)
|
||||
if nm:
|
||||
critical_path.append(nm)
|
||||
if len(path_ids) > 1:
|
||||
last = id_to_name.get(path_ids[-1])
|
||||
if last and last not in critical_path:
|
||||
critical_path.append(last)
|
||||
conn.execute(
|
||||
"INSERT OR REPLACE INTO flow_snapshots "
|
||||
"(flow_id, name, entry_point, critical_path, criticality, "
|
||||
"node_count, file_count) VALUES (?, ?, ?, ?, ?, ?, ?)",
|
||||
(fid, fname, ep_name, _json.dumps(critical_path), crit, ncount, fcount),
|
||||
)
|
||||
conn.commit()
|
||||
except sqlite3.OperationalError:
|
||||
conn.rollback()
|
||||
|
||||
# -- risk_index --
|
||||
try:
|
||||
conn.execute("BEGIN IMMEDIATE")
|
||||
conn.execute("DELETE FROM risk_index")
|
||||
|
||||
# Pre-compute caller and test-coverage counts in two aggregate
|
||||
# queries. Previously this section ran two COUNT(*) queries per
|
||||
# candidate node; on a ~100k-edge graph with tens of thousands
|
||||
# of Function/Class/Test nodes that was the primary hang
|
||||
# observed during Godot builds.
|
||||
caller_counts: dict[str, int] = {}
|
||||
for row in conn.execute(
|
||||
"SELECT target_qualified, COUNT(*) FROM edges "
|
||||
"WHERE kind = 'CALLS' GROUP BY target_qualified"
|
||||
):
|
||||
caller_counts[row[0]] = row[1]
|
||||
|
||||
tested_counts: dict[str, int] = {}
|
||||
for row in conn.execute(
|
||||
"SELECT source_qualified, COUNT(*) FROM edges "
|
||||
"WHERE kind = 'TESTED_BY' GROUP BY source_qualified"
|
||||
):
|
||||
tested_counts[row[0]] = row[1]
|
||||
|
||||
risk_nodes = conn.execute(
|
||||
"SELECT id, qualified_name, name FROM nodes WHERE kind IN ('Function', 'Class', 'Test')"
|
||||
).fetchall()
|
||||
security_kw = {
|
||||
"auth",
|
||||
"login",
|
||||
"password",
|
||||
"token",
|
||||
"session",
|
||||
"crypt",
|
||||
"secret",
|
||||
"credential",
|
||||
"permission",
|
||||
"sql",
|
||||
"execute",
|
||||
}
|
||||
for n in risk_nodes:
|
||||
nid, qn, name = n[0], n[1], n[2]
|
||||
caller_count = caller_counts.get(qn, 0)
|
||||
tested = tested_counts.get(qn, 0)
|
||||
coverage = "tested" if tested > 0 else "untested"
|
||||
name_lower = name.lower()
|
||||
sec_relevant = 1 if any(kw in name_lower for kw in security_kw) else 0
|
||||
risk = 0.0
|
||||
if caller_count > 10:
|
||||
risk += 0.3
|
||||
elif caller_count > 3:
|
||||
risk += 0.15
|
||||
if coverage == "untested":
|
||||
risk += 0.3
|
||||
if sec_relevant:
|
||||
risk += 0.4
|
||||
risk = min(risk, 1.0)
|
||||
conn.execute(
|
||||
"INSERT OR REPLACE INTO risk_index "
|
||||
"(node_id, qualified_name, risk_score, caller_count, "
|
||||
"test_coverage, security_relevant, last_computed) "
|
||||
"VALUES (?, ?, ?, ?, ?, ?, datetime('now'))",
|
||||
(nid, qn, risk, caller_count, coverage, sec_relevant),
|
||||
)
|
||||
conn.commit()
|
||||
except sqlite3.OperationalError:
|
||||
conn.rollback()
|
||||
|
||||
|
||||
def build_or_update_graph(
|
||||
full_rebuild: bool = False,
|
||||
repo_root: str | None = None,
|
||||
base: str = "HEAD~1",
|
||||
postprocess: str = "full",
|
||||
recurse_submodules: bool | None = None,
|
||||
) -> dict[str, Any]:
|
||||
"""Build or incrementally update the code knowledge graph.
|
||||
|
||||
Args:
|
||||
full_rebuild: If True, re-parse every file. If False (default),
|
||||
only re-parse files changed since ``base``.
|
||||
repo_root: Path to the repository root. Auto-detected if omitted.
|
||||
base: Git ref for incremental diff (default: HEAD~1).
|
||||
postprocess: Post-processing level after build:
|
||||
``"full"`` (default) — signatures, FTS, flows, communities.
|
||||
``"minimal"`` — signatures + FTS only (fast, keeps search working).
|
||||
``"none"`` — skip all post-processing (raw parse only).
|
||||
recurse_submodules: If True, include files from git submodules
|
||||
via ``git ls-files --recurse-submodules``. When None
|
||||
(default), falls back to the CRG_RECURSE_SUBMODULES
|
||||
environment variable. Default: disabled.
|
||||
|
||||
Returns:
|
||||
Summary with files_parsed/updated, node/edge counts, and errors.
|
||||
"""
|
||||
store, root = _get_store(repo_root)
|
||||
try:
|
||||
if full_rebuild:
|
||||
result = full_build(root, store, recurse_submodules)
|
||||
build_result = {
|
||||
"status": "ok",
|
||||
"build_type": "full",
|
||||
"summary": (
|
||||
f"Full build complete: parsed {result['files_parsed']} files, "
|
||||
f"created {result['total_nodes']} nodes and "
|
||||
f"{result['total_edges']} edges."
|
||||
),
|
||||
**result,
|
||||
}
|
||||
else:
|
||||
result = incremental_update(root, store, base=base)
|
||||
if result["files_updated"] == 0:
|
||||
return {
|
||||
"status": "ok",
|
||||
"build_type": "incremental",
|
||||
"summary": "No changes detected. Graph is up to date.",
|
||||
"postprocess_level": postprocess,
|
||||
**result,
|
||||
}
|
||||
build_result = {
|
||||
"status": "ok",
|
||||
"build_type": "incremental",
|
||||
"summary": (
|
||||
f"Incremental update: {result['files_updated']} files re-parsed, "
|
||||
f"{result['total_nodes']} nodes and "
|
||||
f"{result['total_edges']} edges updated. "
|
||||
f"Changed: {result['changed_files']}. "
|
||||
f"Dependents also updated: {result['dependent_files']}."
|
||||
),
|
||||
**result,
|
||||
}
|
||||
|
||||
# Pass changed_files for incremental flow/community detection
|
||||
changed = result.get("changed_files") if not full_rebuild else None
|
||||
warnings = _run_postprocess(
|
||||
store,
|
||||
build_result,
|
||||
postprocess,
|
||||
full_rebuild=full_rebuild,
|
||||
changed_files=changed,
|
||||
)
|
||||
if warnings:
|
||||
build_result["warnings"] = warnings
|
||||
return build_result
|
||||
finally:
|
||||
store.close()
|
||||
|
||||
|
||||
def run_postprocess(
|
||||
flows: bool = True,
|
||||
communities: bool = True,
|
||||
fts: bool = True,
|
||||
repo_root: str | None = None,
|
||||
) -> dict[str, Any]:
|
||||
"""Run post-processing steps on an existing graph.
|
||||
|
||||
Useful for running expensive steps (flows, communities) separately
|
||||
from the build, or for re-running after the graph has been updated
|
||||
with ``postprocess="none"``.
|
||||
|
||||
Args:
|
||||
flows: Run flow detection. Default: True.
|
||||
communities: Run community detection. Default: True.
|
||||
fts: Rebuild FTS index. Default: True.
|
||||
repo_root: Repository root path. Auto-detected if omitted.
|
||||
|
||||
Returns:
|
||||
Summary of what was computed.
|
||||
"""
|
||||
store, _root = _get_store(repo_root)
|
||||
result: dict[str, Any] = {"status": "ok"}
|
||||
warnings: list[str] = []
|
||||
|
||||
try:
|
||||
try:
|
||||
rows = store.get_nodes_without_signature()
|
||||
for row in rows:
|
||||
node_id, name, kind, params, ret = (
|
||||
row[0],
|
||||
row[1],
|
||||
row[2],
|
||||
row[3],
|
||||
row[4],
|
||||
)
|
||||
if kind in ("Function", "Test"):
|
||||
sig = f"def {name}({params or ''})"
|
||||
if ret:
|
||||
sig += f" -> {ret}"
|
||||
elif kind == "Class":
|
||||
sig = f"class {name}"
|
||||
else:
|
||||
sig = name
|
||||
store.update_node_signature(node_id, sig[:512])
|
||||
store.commit()
|
||||
result["signatures_updated"] = True
|
||||
except (sqlite3.OperationalError, TypeError, KeyError) as e:
|
||||
logger.warning("Signature computation failed: %s", e)
|
||||
warnings.append(f"Signature computation failed: {type(e).__name__}: {e}")
|
||||
|
||||
if fts:
|
||||
try:
|
||||
from code_review_graph.search import rebuild_fts_index
|
||||
|
||||
fts_count = rebuild_fts_index(store)
|
||||
result["fts_indexed"] = fts_count
|
||||
except (sqlite3.OperationalError, ImportError) as e:
|
||||
store.rollback()
|
||||
logger.warning("FTS index rebuild failed: %s", e)
|
||||
warnings.append(f"FTS index rebuild failed: {type(e).__name__}: {e}")
|
||||
|
||||
if flows:
|
||||
try:
|
||||
from code_review_graph.flows import store_flows as _store_flows
|
||||
from code_review_graph.flows import trace_flows as _trace_flows
|
||||
|
||||
traced = _trace_flows(store)
|
||||
count = _store_flows(store, traced)
|
||||
result["flows_detected"] = count
|
||||
except (sqlite3.OperationalError, ImportError) as e:
|
||||
store.rollback()
|
||||
logger.warning("Flow detection failed: %s", e)
|
||||
warnings.append(f"Flow detection failed: {type(e).__name__}: {e}")
|
||||
|
||||
if communities:
|
||||
try:
|
||||
from code_review_graph.communities import (
|
||||
detect_communities as _detect_communities,
|
||||
)
|
||||
from code_review_graph.communities import (
|
||||
store_communities as _store_communities,
|
||||
)
|
||||
|
||||
comms = _detect_communities(store)
|
||||
count = _store_communities(store, comms)
|
||||
result["communities_detected"] = count
|
||||
except (sqlite3.OperationalError, ImportError) as e:
|
||||
store.rollback()
|
||||
logger.warning("Community detection failed: %s", e)
|
||||
warnings.append(f"Community detection failed: {type(e).__name__}: {e}")
|
||||
|
||||
store.set_metadata(
|
||||
"last_postprocessed_at",
|
||||
time.strftime("%Y-%m-%dT%H:%M:%S"),
|
||||
)
|
||||
result["summary"] = "Post-processing complete."
|
||||
if warnings:
|
||||
result["warnings"] = warnings
|
||||
return result
|
||||
finally:
|
||||
store.close()
|
||||
@@ -0,0 +1,246 @@
|
||||
"""Tools 13, 14, 15: community listing, detail, architecture overview."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from collections import Counter
|
||||
from typing import Any
|
||||
|
||||
from ..communities import get_architecture_overview, get_communities
|
||||
from ..context_savings import attach_context_savings
|
||||
from ..graph import node_to_dict
|
||||
from ..hints import generate_hints, get_session
|
||||
from ._common import _get_store
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Tool 13: list_communities [EXPLORE]
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def list_communities_func(
|
||||
repo_root: str | None = None,
|
||||
sort_by: str = "size",
|
||||
min_size: int = 0,
|
||||
detail_level: str = "standard",
|
||||
) -> dict[str, Any]:
|
||||
"""List detected code communities in the codebase.
|
||||
|
||||
[EXPLORE] Retrieves stored communities from the knowledge graph.
|
||||
Each community represents a cluster of related code entities
|
||||
(functions, classes) detected via the Leiden algorithm or
|
||||
file-based grouping.
|
||||
|
||||
Args:
|
||||
repo_root: Repository root path. Auto-detected if omitted.
|
||||
sort_by: Sort column: size, cohesion, or name.
|
||||
min_size: Minimum community size to include (default: 0).
|
||||
detail_level: "standard" (default) returns full community data;
|
||||
"minimal" returns only name, size, and cohesion
|
||||
per community.
|
||||
|
||||
Returns:
|
||||
List of communities with size and cohesion scores.
|
||||
"""
|
||||
store, root = _get_store(repo_root)
|
||||
try:
|
||||
communities = get_communities(
|
||||
store, sort_by=sort_by, min_size=min_size
|
||||
)
|
||||
if detail_level == "minimal":
|
||||
communities = [
|
||||
{"name": c["name"], "size": c["size"], "cohesion": c["cohesion"]}
|
||||
for c in communities
|
||||
]
|
||||
result: dict[str, object] = {
|
||||
"status": "ok",
|
||||
"summary": f"Found {len(communities)} communities",
|
||||
"communities": communities,
|
||||
}
|
||||
result["_hints"] = generate_hints(
|
||||
"list_communities", result, get_session()
|
||||
)
|
||||
return result
|
||||
except Exception as exc:
|
||||
return {"status": "error", "error": str(exc)}
|
||||
finally:
|
||||
store.close()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Tool 14: get_community [EXPLORE]
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def get_community_func(
|
||||
community_name: str | None = None,
|
||||
community_id: int | None = None,
|
||||
include_members: bool = False,
|
||||
repo_root: str | None = None,
|
||||
) -> dict[str, Any]:
|
||||
"""Get details of a single code community.
|
||||
|
||||
[EXPLORE] Retrieves a community by its database ID or by name match.
|
||||
Optionally includes the full list of member nodes.
|
||||
|
||||
Args:
|
||||
community_name: Name to search for (partial match). Ignored if
|
||||
community_id given.
|
||||
community_id: Database ID of the community.
|
||||
include_members: If True, include full member node details.
|
||||
repo_root: Repository root path. Auto-detected if omitted.
|
||||
|
||||
Returns:
|
||||
Community details, or not_found status.
|
||||
"""
|
||||
store, root = _get_store(repo_root)
|
||||
try:
|
||||
community: dict | None = None
|
||||
all_communities = get_communities(store)
|
||||
|
||||
if community_id is not None:
|
||||
for c in all_communities:
|
||||
if c.get("id") == community_id:
|
||||
community = c
|
||||
break
|
||||
elif community_name is not None:
|
||||
for c in all_communities:
|
||||
if community_name.lower() in c["name"].lower():
|
||||
community = c
|
||||
break
|
||||
|
||||
if community is None:
|
||||
return {
|
||||
"status": "not_found",
|
||||
"summary": (
|
||||
"No community found matching the given criteria."
|
||||
),
|
||||
}
|
||||
|
||||
if include_members:
|
||||
cid = community.get("id")
|
||||
if cid is not None:
|
||||
member_nodes = store.get_nodes_by_community_id(cid)
|
||||
members = [node_to_dict(n) for n in member_nodes]
|
||||
community["member_details"] = members
|
||||
|
||||
result = {
|
||||
"status": "ok",
|
||||
"summary": (
|
||||
f"Community '{community['name']}': "
|
||||
f"{community['size']} nodes, "
|
||||
f"cohesion {community['cohesion']:.4f}"
|
||||
),
|
||||
"community": community,
|
||||
}
|
||||
result["_hints"] = generate_hints(
|
||||
"get_community", result, get_session()
|
||||
)
|
||||
return result
|
||||
except Exception as exc:
|
||||
return {"status": "error", "error": str(exc)}
|
||||
finally:
|
||||
store.close()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Tool 15: get_architecture_overview [EXPLORE]
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
_MINIMAL_COMMUNITY_FIELDS = ("id", "name", "size", "cohesion", "dominant_language")
|
||||
|
||||
|
||||
def _minimal_overview(overview: dict[str, Any]) -> dict[str, Any]:
|
||||
"""Compress overview for ``detail_level="minimal"``.
|
||||
|
||||
The full overview can exceed 600KB on medium repos because it embeds
|
||||
every community's member list and every individual cross-community
|
||||
edge. Minimal mode drops member lists and aggregates the edge list
|
||||
to one row per community pair with a count and the top edge kinds —
|
||||
enough to spot coupling smells without exploding token budgets.
|
||||
"""
|
||||
communities = [
|
||||
{k: c[k] for k in _MINIMAL_COMMUNITY_FIELDS if k in c}
|
||||
for c in overview.get("communities", [])
|
||||
]
|
||||
id_to_name = {c["id"]: c["name"] for c in communities if "id" in c}
|
||||
|
||||
edge_pair_counts: Counter[tuple[int, int]] = Counter()
|
||||
edge_pair_kinds: dict[tuple[int, int], Counter[str]] = {}
|
||||
for e in overview.get("cross_community_edges", []):
|
||||
# Use canonical (low, high) ordering so A↔B and B↔A aggregate together.
|
||||
a, b = e["source_community"], e["target_community"]
|
||||
pair = (a, b) if a <= b else (b, a)
|
||||
edge_pair_counts[pair] += 1
|
||||
edge_pair_kinds.setdefault(pair, Counter())[e["edge_kind"]] += 1
|
||||
|
||||
cross_pairs = [
|
||||
{
|
||||
"source_community": id_to_name.get(a, f"community-{a}"),
|
||||
"target_community": id_to_name.get(b, f"community-{b}"),
|
||||
"edge_count": count,
|
||||
"top_kinds": [k for k, _ in edge_pair_kinds[(a, b)].most_common(3)],
|
||||
}
|
||||
for (a, b), count in edge_pair_counts.most_common()
|
||||
]
|
||||
return {
|
||||
"communities": communities,
|
||||
"cross_community_edges": cross_pairs,
|
||||
"warnings": overview.get("warnings", []),
|
||||
}
|
||||
|
||||
|
||||
def get_architecture_overview_func(
|
||||
repo_root: str | None = None,
|
||||
detail_level: str = "minimal",
|
||||
) -> dict[str, Any]:
|
||||
"""Generate an architecture overview based on community structure.
|
||||
|
||||
[EXPLORE] Builds a high-level view of the codebase architecture by
|
||||
analyzing community boundaries and cross-community coupling.
|
||||
Includes warnings for high coupling between communities.
|
||||
|
||||
Args:
|
||||
repo_root: Repository root path. Auto-detected if omitted.
|
||||
detail_level: "minimal" (default) drops community member lists
|
||||
and aggregates edges to one row per community pair
|
||||
(typical reduction: 600KB -> <5KB);
|
||||
"standard" returns the full overview including
|
||||
per-edge cross-community detail.
|
||||
|
||||
Returns:
|
||||
Architecture overview with communities, cross-community edges,
|
||||
and warnings.
|
||||
"""
|
||||
store, root = _get_store(repo_root)
|
||||
try:
|
||||
full_overview = get_architecture_overview(store)
|
||||
overview = full_overview
|
||||
if detail_level == "minimal":
|
||||
overview = _minimal_overview(full_overview)
|
||||
n_communities = len(overview["communities"])
|
||||
n_cross = len(overview["cross_community_edges"])
|
||||
n_warnings = len(overview["warnings"])
|
||||
cross_label = (
|
||||
"community pairs"
|
||||
if detail_level == "minimal"
|
||||
else "cross-community edges"
|
||||
)
|
||||
result = {
|
||||
"status": "ok",
|
||||
"summary": (
|
||||
f"Architecture: {n_communities} communities, "
|
||||
f"{n_cross} {cross_label}, "
|
||||
f"{n_warnings} warning(s)"
|
||||
),
|
||||
**overview,
|
||||
}
|
||||
result["_hints"] = generate_hints(
|
||||
"get_architecture_overview", result, get_session()
|
||||
)
|
||||
if detail_level == "minimal":
|
||||
attach_context_savings(result, original_context=full_overview)
|
||||
return result
|
||||
except Exception as exc:
|
||||
return {"status": "error", "error": str(exc)}
|
||||
finally:
|
||||
store.close()
|
||||
@@ -0,0 +1,152 @@
|
||||
"""Tool: get_minimal_context — ultra-compact context for token-efficient workflows."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import sqlite3
|
||||
import subprocess
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from ._common import _get_store, compact_response
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _has_git_changes(root: Path, base: str) -> bool:
|
||||
"""Quick check for uncommitted or diffed changes."""
|
||||
try:
|
||||
result = subprocess.run(
|
||||
["git", "diff", "--name-only", base, "--"],
|
||||
capture_output=True, stdin=subprocess.DEVNULL, text=True,
|
||||
cwd=str(root), timeout=10,
|
||||
)
|
||||
if result.returncode == 0 and result.stdout.strip():
|
||||
return True
|
||||
# Also check staged/unstaged
|
||||
result2 = subprocess.run(
|
||||
["git", "status", "--porcelain"],
|
||||
capture_output=True, stdin=subprocess.DEVNULL, text=True,
|
||||
cwd=str(root), timeout=10,
|
||||
)
|
||||
return bool(result2.stdout.strip())
|
||||
except (FileNotFoundError, subprocess.TimeoutExpired):
|
||||
return False
|
||||
|
||||
|
||||
def get_minimal_context(
|
||||
task: str = "",
|
||||
changed_files: list[str] | None = None,
|
||||
repo_root: str | None = None,
|
||||
base: str = "HEAD~1",
|
||||
) -> dict[str, Any]:
|
||||
"""Return minimum context an agent needs to start any task (~100 tokens).
|
||||
|
||||
Combines graph stats, top communities, top flows, risk score,
|
||||
and suggested next tools into an ultra-compact response.
|
||||
|
||||
Args:
|
||||
task: Natural language description of what the agent is doing
|
||||
(e.g. "review PR #42", "debug login timeout").
|
||||
changed_files: Explicit changed files. Auto-detected from git if None.
|
||||
repo_root: Repository root path. Auto-detected if None.
|
||||
base: Git ref for diff comparison.
|
||||
"""
|
||||
store, root = _get_store(repo_root)
|
||||
try:
|
||||
# 1. Quick stats
|
||||
stats = store.get_stats()
|
||||
|
||||
# 2. Risk from changed files
|
||||
risk = "unknown"
|
||||
risk_score = 0.0
|
||||
top_affected: list[str] = []
|
||||
test_gap_count = 0
|
||||
if changed_files or _has_git_changes(root, base):
|
||||
try:
|
||||
from ..changes import analyze_changes
|
||||
from ..incremental import get_changed_files as _get_changed
|
||||
|
||||
files = changed_files
|
||||
if not files:
|
||||
files = _get_changed(root, base)
|
||||
if files:
|
||||
abs_files = [str(root / f) for f in files]
|
||||
analysis = analyze_changes(
|
||||
store, abs_files, repo_root=str(root), base=base,
|
||||
)
|
||||
risk_score = analysis.get("risk_score", 0.0)
|
||||
risk = (
|
||||
"high" if risk_score > 0.7
|
||||
else "medium" if risk_score > 0.4
|
||||
else "low"
|
||||
)
|
||||
top_affected = [
|
||||
f.get("name", "")
|
||||
for f in analysis.get("changed_functions", [])[:5]
|
||||
]
|
||||
test_gap_count = len(analysis.get("test_gaps", []))
|
||||
except (
|
||||
ImportError, OSError, ValueError,
|
||||
sqlite3.Error, subprocess.SubprocessError,
|
||||
):
|
||||
logger.debug("Risk analysis failed in get_minimal_context", exc_info=True)
|
||||
|
||||
# 3. Top 3 communities
|
||||
communities: list[str] = []
|
||||
try:
|
||||
rows = store._conn.execute(
|
||||
"SELECT name FROM communities ORDER BY size DESC LIMIT 3"
|
||||
).fetchall()
|
||||
communities = [r[0] for r in rows]
|
||||
except sqlite3.OperationalError: # nosec B110 — table may not exist yet
|
||||
logger.debug("communities table not yet populated")
|
||||
|
||||
# 4. Top 3 critical flows
|
||||
flows: list[str] = []
|
||||
try:
|
||||
rows = store._conn.execute(
|
||||
"SELECT name FROM flows ORDER BY criticality DESC LIMIT 3"
|
||||
).fetchall()
|
||||
flows = [r[0] for r in rows]
|
||||
except sqlite3.OperationalError: # nosec B110 — table may not exist yet
|
||||
logger.debug("flows table not yet populated")
|
||||
|
||||
# 5. Suggest next tools based on task keywords
|
||||
task_lower = task.lower()
|
||||
if any(w in task_lower for w in ("review", "pr", "merge", "diff")):
|
||||
suggestions = ["detect_changes", "get_affected_flows", "get_review_context"]
|
||||
elif any(w in task_lower for w in ("debug", "bug", "error", "fix")):
|
||||
suggestions = ["semantic_search_nodes", "query_graph", "get_flow"]
|
||||
elif any(w in task_lower for w in ("refactor", "rename", "dead", "clean")):
|
||||
suggestions = ["refactor", "find_large_functions", "get_architecture_overview"]
|
||||
elif any(w in task_lower for w in ("onboard", "understand", "explore", "arch")):
|
||||
suggestions = [
|
||||
"get_architecture_overview", "list_communities", "list_flows",
|
||||
]
|
||||
else:
|
||||
suggestions = [
|
||||
"detect_changes", "semantic_search_nodes",
|
||||
"get_architecture_overview",
|
||||
]
|
||||
|
||||
# Build summary
|
||||
summary_parts = [
|
||||
f"{stats.total_nodes} nodes, {stats.total_edges} edges"
|
||||
f" across {stats.files_count} files.",
|
||||
]
|
||||
if risk != "unknown":
|
||||
summary_parts.append(f"Risk: {risk} ({risk_score:.2f}).")
|
||||
if test_gap_count:
|
||||
summary_parts.append(f"{test_gap_count} test gaps.")
|
||||
|
||||
return compact_response(
|
||||
summary=" ".join(summary_parts),
|
||||
key_entities=top_affected or None,
|
||||
risk=risk,
|
||||
communities=communities or None,
|
||||
flows_affected=flows or None,
|
||||
next_tool_suggestions=suggestions,
|
||||
)
|
||||
finally:
|
||||
store.close()
|
||||
@@ -0,0 +1,274 @@
|
||||
"""Tools 7, 8, 19, 20: embed_graph, get_docs_section, wiki tools."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from ..embeddings import EmbeddingStore, embed_all_nodes
|
||||
from ..incremental import find_project_root, get_db_path
|
||||
from ._common import _get_store, _resolve_root, _validate_repo_root
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Tool 7: embed_graph
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def embed_graph(
|
||||
repo_root: str | None = None,
|
||||
model: str | None = None,
|
||||
provider: str | None = None,
|
||||
) -> dict[str, Any]:
|
||||
"""Compute vector embeddings for all graph nodes to enable semantic search.
|
||||
|
||||
Requires: ``pip install code-review-graph[embeddings]`` (local provider only;
|
||||
cloud providers like ``openai`` / ``google`` / ``minimax`` use stdlib ``urllib``).
|
||||
Default model: all-MiniLM-L6-v2. Override via ``model`` param or
|
||||
CRG_EMBEDDING_MODEL env var.
|
||||
Changing the model or provider re-embeds all nodes automatically.
|
||||
|
||||
Only embeds nodes that don't already have up-to-date embeddings.
|
||||
|
||||
Args:
|
||||
repo_root: Repository root path. Auto-detected if omitted.
|
||||
model: Embedding model name. For local: HuggingFace ID or path;
|
||||
for openai: model ID (e.g. ``text-embedding-3-small``);
|
||||
for google: Gemini model ID. Falls back to
|
||||
CRG_EMBEDDING_MODEL / CRG_OPENAI_MODEL env vars as appropriate.
|
||||
provider: Provider name: ``local`` (default), ``openai``, ``google``,
|
||||
or ``minimax``. ``openai`` requires CRG_OPENAI_BASE_URL +
|
||||
CRG_OPENAI_API_KEY + CRG_OPENAI_MODEL env vars and accepts
|
||||
any OpenAI-compatible endpoint (real OpenAI, Azure, new-api,
|
||||
LiteLLM, vLLM, LocalAI, Ollama openai-mode, etc.).
|
||||
|
||||
Returns:
|
||||
Number of nodes embedded and total embedding count.
|
||||
"""
|
||||
store, root = _get_store(repo_root)
|
||||
try:
|
||||
db_path = get_db_path(root)
|
||||
try:
|
||||
emb_store = EmbeddingStore(db_path, provider=provider, model=model)
|
||||
except ValueError as exc:
|
||||
# Unknown provider name or missing provider env vars — surface
|
||||
# as a structured error rather than a traceback.
|
||||
logger.error("embed_graph: %s", exc)
|
||||
return {"status": "error", "error": str(exc)}
|
||||
try:
|
||||
if not emb_store.available:
|
||||
if provider in ("openai", "google", "minimax"):
|
||||
err = (
|
||||
f"The '{provider}' embedding provider is not available. "
|
||||
"Check the required environment variables "
|
||||
"(see README and `get_provider()` docstring) and that "
|
||||
"the endpoint is reachable."
|
||||
)
|
||||
else:
|
||||
err = (
|
||||
"The local embedding provider needs sentence-transformers. "
|
||||
"Install with: pip install code-review-graph[embeddings] — "
|
||||
"or switch provider to 'openai' / 'google' / 'minimax'."
|
||||
)
|
||||
return {"status": "error", "error": err}
|
||||
|
||||
newly_embedded = embed_all_nodes(store, emb_store)
|
||||
total = emb_store.count()
|
||||
|
||||
return {
|
||||
"status": "ok",
|
||||
"summary": (
|
||||
f"Embedded {newly_embedded} new node(s). "
|
||||
f"Total embeddings: {total}. "
|
||||
"Semantic search is now active."
|
||||
),
|
||||
"newly_embedded": newly_embedded,
|
||||
"total_embeddings": total,
|
||||
}
|
||||
finally:
|
||||
emb_store.close()
|
||||
finally:
|
||||
store.close()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Tool 8: get_docs_section
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def get_docs_section(
|
||||
section_name: str, repo_root: str | None = None
|
||||
) -> dict[str, Any]:
|
||||
"""Return a specific section from the LLM-optimized reference.
|
||||
|
||||
Used by skills and Claude Code to load only the exact documentation
|
||||
section needed, keeping token usage minimal (90%+ savings).
|
||||
|
||||
Args:
|
||||
section_name: Exact section name. One of: usage, review-delta,
|
||||
review-pr, commands, legal, watch, embeddings,
|
||||
languages, troubleshooting.
|
||||
repo_root: Repository root path. Auto-detected from current
|
||||
directory if omitted.
|
||||
|
||||
Returns:
|
||||
The section content, or an error if not found.
|
||||
"""
|
||||
import re as _re
|
||||
|
||||
search_roots: list[Path] = []
|
||||
|
||||
# Wheel install: docs are packaged inside code_review_graph/docs.
|
||||
in_pkg_docs = (
|
||||
Path(__file__).parent.parent
|
||||
/ "docs"
|
||||
/ "LLM-OPTIMIZED-REFERENCE.md"
|
||||
)
|
||||
if repo_root:
|
||||
try:
|
||||
search_roots.append(_validate_repo_root(Path(repo_root)))
|
||||
except ValueError:
|
||||
pass
|
||||
elif in_pkg_docs.exists():
|
||||
in_pkg_root = in_pkg_docs.parent.parent
|
||||
search_roots.append(in_pkg_root)
|
||||
|
||||
if not repo_root:
|
||||
project_root = find_project_root()
|
||||
if project_root not in search_roots:
|
||||
search_roots.append(project_root)
|
||||
|
||||
# Editable/source-tree fallback: docs live next to code_review_graph/.
|
||||
pkg_docs = (
|
||||
Path(__file__).parent.parent.parent
|
||||
/ "docs"
|
||||
/ "LLM-OPTIMIZED-REFERENCE.md"
|
||||
)
|
||||
if pkg_docs.exists():
|
||||
pkg_root = pkg_docs.parent.parent
|
||||
if pkg_root not in search_roots:
|
||||
search_roots.append(pkg_root)
|
||||
|
||||
for search_root in search_roots:
|
||||
candidate = search_root / "docs" / "LLM-OPTIMIZED-REFERENCE.md"
|
||||
if candidate.exists():
|
||||
content = candidate.read_text(encoding="utf-8", errors="replace")
|
||||
match = _re.search(
|
||||
rf'<section name="{_re.escape(section_name)}">'
|
||||
r"(.*?)</section>",
|
||||
content,
|
||||
_re.DOTALL | _re.IGNORECASE,
|
||||
)
|
||||
if match:
|
||||
return {
|
||||
"status": "ok",
|
||||
"section": section_name,
|
||||
"content": match.group(1).strip(),
|
||||
}
|
||||
|
||||
available = [
|
||||
"usage", "review-delta", "review-pr", "commands",
|
||||
"legal", "watch", "embeddings", "languages", "troubleshooting",
|
||||
]
|
||||
return {
|
||||
"status": "not_found",
|
||||
"error": (
|
||||
f"Section '{section_name}' not found. "
|
||||
f"Available: {', '.join(available)}"
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Tool 19: generate_wiki [DOCS]
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def generate_wiki_func(
|
||||
repo_root: str | None = None,
|
||||
force: bool = False,
|
||||
) -> dict[str, Any]:
|
||||
"""Generate a markdown wiki from the community structure.
|
||||
|
||||
[DOCS] Creates a wiki page for each detected community and an index
|
||||
page. Pages are written to ``.code-review-graph/wiki/`` inside the
|
||||
repository. Only regenerates pages whose content has changed unless
|
||||
force=True.
|
||||
|
||||
Args:
|
||||
repo_root: Repository root path. Auto-detected if omitted.
|
||||
force: If True, regenerate all pages even if content is unchanged.
|
||||
|
||||
Returns:
|
||||
Status with pages_generated, pages_updated, pages_unchanged counts.
|
||||
"""
|
||||
from ..incremental import get_data_dir
|
||||
from ..wiki import generate_wiki
|
||||
|
||||
store, root = _get_store(repo_root)
|
||||
try:
|
||||
wiki_dir = get_data_dir(root) / "wiki"
|
||||
result = generate_wiki(store, wiki_dir, force=force)
|
||||
total = (
|
||||
result["pages_generated"]
|
||||
+ result["pages_updated"]
|
||||
+ result["pages_unchanged"]
|
||||
)
|
||||
return {
|
||||
"status": "ok",
|
||||
"summary": (
|
||||
f"Wiki generated: {result['pages_generated']} new, "
|
||||
f"{result['pages_updated']} updated, "
|
||||
f"{result['pages_unchanged']} unchanged "
|
||||
f"({total} total pages)"
|
||||
),
|
||||
"wiki_dir": str(wiki_dir),
|
||||
**result,
|
||||
}
|
||||
except Exception as exc:
|
||||
return {"status": "error", "error": str(exc)}
|
||||
finally:
|
||||
store.close()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Tool 20: get_wiki_page [DOCS]
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def get_wiki_page_func(
|
||||
community_name: str,
|
||||
repo_root: str | None = None,
|
||||
) -> dict[str, Any]:
|
||||
"""Retrieve a specific wiki page by community name.
|
||||
|
||||
[DOCS] Returns the markdown content of the wiki page for the given
|
||||
community. The wiki must have been generated first via generate_wiki.
|
||||
|
||||
Args:
|
||||
community_name: Community name to look up (slugified for filename).
|
||||
repo_root: Repository root path. Auto-detected if omitted.
|
||||
|
||||
Returns:
|
||||
Page content or not_found status.
|
||||
"""
|
||||
from ..incremental import get_data_dir
|
||||
from ..wiki import get_wiki_page
|
||||
|
||||
root = _resolve_root(repo_root)
|
||||
wiki_dir = get_data_dir(root) / "wiki"
|
||||
content = get_wiki_page(wiki_dir, community_name)
|
||||
if content is None:
|
||||
return {
|
||||
"status": "not_found",
|
||||
"summary": f"No wiki page found for '{community_name}'.",
|
||||
}
|
||||
return {
|
||||
"status": "ok",
|
||||
"summary": (
|
||||
f"Wiki page for '{community_name}' ({len(content)} chars)"
|
||||
),
|
||||
"content": content,
|
||||
}
|
||||
@@ -0,0 +1,176 @@
|
||||
"""Tools 10, 11: list_flows, get_flow."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from ..flows import get_flow_by_id, get_flows
|
||||
from ..hints import generate_hints, get_session
|
||||
from ._common import _get_store
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Tool 10: list_flows [EXPLORE]
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def list_flows(
|
||||
repo_root: str | None = None,
|
||||
sort_by: str = "criticality",
|
||||
limit: int = 50,
|
||||
kind: str | None = None,
|
||||
detail_level: str = "standard",
|
||||
) -> dict[str, Any]:
|
||||
"""List execution flows in the codebase, sorted by criticality.
|
||||
|
||||
[EXPLORE] Retrieves stored execution flows from the knowledge graph.
|
||||
Each flow represents a call chain starting from an entry point
|
||||
(e.g. HTTP handler, CLI command, test function).
|
||||
|
||||
Args:
|
||||
repo_root: Repository root path. Auto-detected if omitted.
|
||||
sort_by: Sort column: criticality, depth, node_count, file_count,
|
||||
or name.
|
||||
limit: Maximum flows to return (default: 50).
|
||||
kind: Optional filter by entry point kind (e.g. "Test", "Function").
|
||||
detail_level: "standard" (default) returns full flow data;
|
||||
"minimal" returns only name, criticality, and
|
||||
node_count per flow.
|
||||
|
||||
Returns:
|
||||
List of flows with criticality scores.
|
||||
"""
|
||||
store, root = _get_store(repo_root)
|
||||
try:
|
||||
fetch_limit = (
|
||||
limit if not kind else limit * 10
|
||||
) # fetch more when filtering
|
||||
flows = get_flows(store, sort_by=sort_by, limit=fetch_limit)
|
||||
|
||||
if kind:
|
||||
filtered = []
|
||||
for f in flows:
|
||||
ep_id = f.get("entry_point_id")
|
||||
if ep_id is not None:
|
||||
node_kind = store.get_node_kind_by_id(ep_id)
|
||||
if node_kind == kind:
|
||||
filtered.append(f)
|
||||
flows = filtered[:limit]
|
||||
|
||||
if detail_level == "minimal":
|
||||
flows = [
|
||||
{
|
||||
"name": f["name"],
|
||||
"criticality": f["criticality"],
|
||||
"node_count": f["node_count"],
|
||||
}
|
||||
for f in flows
|
||||
]
|
||||
|
||||
result: dict[str, object] = {
|
||||
"status": "ok",
|
||||
"summary": f"Found {len(flows)} execution flow(s)",
|
||||
"flows": flows,
|
||||
}
|
||||
result["_hints"] = generate_hints(
|
||||
"list_flows", result, get_session()
|
||||
)
|
||||
return result
|
||||
except Exception as exc:
|
||||
return {"status": "error", "error": str(exc)}
|
||||
finally:
|
||||
store.close()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Tool 11: get_flow [EXPLORE]
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def get_flow(
|
||||
flow_id: int | None = None,
|
||||
flow_name: str | None = None,
|
||||
include_source: bool = False,
|
||||
repo_root: str | None = None,
|
||||
) -> dict[str, Any]:
|
||||
"""Get details of a single execution flow.
|
||||
|
||||
[EXPLORE] Retrieves full path details for a flow, including each step's
|
||||
function name, file, and line numbers. Optionally includes source
|
||||
snippets for every step in the path.
|
||||
|
||||
Args:
|
||||
flow_id: Database ID of the flow (from list_flows).
|
||||
flow_name: Name to search for (partial match). Ignored if flow_id
|
||||
given.
|
||||
include_source: If True, include source code snippets for each step.
|
||||
repo_root: Repository root path. Auto-detected if omitted.
|
||||
|
||||
Returns:
|
||||
Flow details with steps, or not_found status.
|
||||
"""
|
||||
store, root = _get_store(repo_root)
|
||||
try:
|
||||
flow: dict | None = None
|
||||
|
||||
if flow_id is not None:
|
||||
flow = get_flow_by_id(store, flow_id)
|
||||
elif flow_name is not None:
|
||||
# Search flows by name match
|
||||
all_flows = get_flows(
|
||||
store, sort_by="criticality", limit=500
|
||||
)
|
||||
for f in all_flows:
|
||||
if flow_name.lower() in f["name"].lower():
|
||||
flow = get_flow_by_id(store, f["id"])
|
||||
break
|
||||
|
||||
if flow is None:
|
||||
return {
|
||||
"status": "not_found",
|
||||
"summary": "No flow found matching the given criteria.",
|
||||
}
|
||||
|
||||
# Optionally include source snippets for each step
|
||||
if include_source and "steps" in flow:
|
||||
for step in flow["steps"]:
|
||||
fp = Path(step["file"]) if step.get("file") else None
|
||||
if fp is not None and not fp.is_absolute():
|
||||
fp = root / fp
|
||||
file_path = fp
|
||||
if file_path and file_path.is_file():
|
||||
try:
|
||||
lines = file_path.read_text(
|
||||
errors="replace"
|
||||
).splitlines()
|
||||
start = max(
|
||||
0, (step.get("line_start") or 1) - 1
|
||||
)
|
||||
end = min(
|
||||
len(lines),
|
||||
step.get("line_end") or len(lines),
|
||||
)
|
||||
step["source"] = "\n".join(
|
||||
f"{i + 1}: {lines[i]}"
|
||||
for i in range(start, end)
|
||||
)
|
||||
except (OSError, UnicodeDecodeError):
|
||||
step["source"] = "(could not read file)"
|
||||
|
||||
result = {
|
||||
"status": "ok",
|
||||
"summary": (
|
||||
f"Flow '{flow['name']}': {flow['node_count']} nodes, "
|
||||
f"depth {flow['depth']}, "
|
||||
f"criticality {flow['criticality']:.4f}"
|
||||
),
|
||||
"flow": flow,
|
||||
}
|
||||
result["_hints"] = generate_hints(
|
||||
"get_flow", result, get_session()
|
||||
)
|
||||
return result
|
||||
except Exception as exc:
|
||||
return {"status": "error", "error": str(exc)}
|
||||
finally:
|
||||
store.close()
|
||||
@@ -0,0 +1,692 @@
|
||||
"""Tools 2, 3, 5, 6, 9: query / search / stats helpers."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from ..context_savings import attach_context_savings, estimate_file_tokens
|
||||
from ..embeddings import EmbeddingStore
|
||||
from ..graph import _sanitize_name, edge_to_dict, node_to_dict
|
||||
from ..hints import generate_hints, get_session
|
||||
from ..incremental import get_changed_files, get_db_path, get_staged_and_unstaged
|
||||
from ..search import hybrid_search
|
||||
from ._common import _BUILTIN_CALL_NAMES, _get_store, _resolve_graph_file_paths
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Tool 2: get_impact_radius
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
_QUERY_PATTERNS = {
|
||||
"callers_of": "Find all functions that call a given function",
|
||||
"callees_of": "Find all functions called by a given function",
|
||||
"imports_of": "Find all imports of a given file or module",
|
||||
"importers_of": "Find all files that import a given file or module",
|
||||
"children_of": "Find all nodes contained in a file or class",
|
||||
"tests_for": "Find all tests for a given function or class",
|
||||
"inheritors_of": "Find all classes that inherit from a given class",
|
||||
"file_summary": "Get a summary of all nodes in a file",
|
||||
}
|
||||
|
||||
|
||||
def get_impact_radius(
|
||||
changed_files: list[str] | None = None,
|
||||
max_depth: int = 2,
|
||||
max_results: int = 500,
|
||||
repo_root: str | None = None,
|
||||
base: str = "HEAD~1",
|
||||
detail_level: str = "standard",
|
||||
) -> dict[str, Any]:
|
||||
"""Analyze the blast radius of changed files.
|
||||
|
||||
Args:
|
||||
changed_files: Explicit list of changed file paths (relative to repo root).
|
||||
If omitted, auto-detects from git diff.
|
||||
max_depth: How many hops to traverse in the graph (default: 2).
|
||||
max_results: Maximum impacted nodes to return (default: 500).
|
||||
repo_root: Repository root path. Auto-detected if omitted.
|
||||
base: Git ref for auto-detecting changes (default: HEAD~1).
|
||||
detail_level: "standard" (full output) or "minimal" (summary only).
|
||||
|
||||
Returns:
|
||||
Changed nodes, impacted nodes, impacted files, connecting edges,
|
||||
plus ``truncated`` flag and ``total_impacted`` count.
|
||||
"""
|
||||
store, root = _get_store(repo_root)
|
||||
try:
|
||||
if changed_files is None:
|
||||
changed_files = get_changed_files(root, base)
|
||||
if not changed_files:
|
||||
changed_files = get_staged_and_unstaged(root)
|
||||
|
||||
if not changed_files:
|
||||
return {
|
||||
"status": "ok",
|
||||
"summary": "No changed files detected.",
|
||||
"changed_nodes": [],
|
||||
"impacted_nodes": [],
|
||||
"impacted_files": [],
|
||||
"truncated": False,
|
||||
"total_impacted": 0,
|
||||
}
|
||||
|
||||
# Resolve user-facing paths to the file paths stored in the graph.
|
||||
original_tokens = estimate_file_tokens(root, changed_files)
|
||||
abs_files = _resolve_graph_file_paths(store, root, changed_files)
|
||||
result = store.get_impact_radius(
|
||||
abs_files, max_depth=max_depth, max_nodes=max_results
|
||||
)
|
||||
|
||||
changed_dicts = [node_to_dict(n) for n in result["changed_nodes"]]
|
||||
impacted_dicts = [node_to_dict(n) for n in result["impacted_nodes"]]
|
||||
edge_dicts = [edge_to_dict(e) for e in result["edges"]]
|
||||
truncated = result["truncated"]
|
||||
total_impacted = result["total_impacted"]
|
||||
|
||||
summary_parts = [
|
||||
f"Blast radius for {len(changed_files)} changed file(s):",
|
||||
f" - {len(changed_dicts)} nodes directly changed",
|
||||
f" - {len(impacted_dicts)} nodes impacted (within {max_depth} hops)",
|
||||
f" - {len(result['impacted_files'])} additional files affected",
|
||||
]
|
||||
if truncated:
|
||||
summary_parts.append(
|
||||
f" - Results truncated: showing {len(impacted_dicts)}"
|
||||
f" of {total_impacted} impacted nodes"
|
||||
)
|
||||
|
||||
if detail_level == "minimal":
|
||||
impacted_count = len(impacted_dicts)
|
||||
if impacted_count > 20:
|
||||
risk = "high"
|
||||
elif impacted_count > 5:
|
||||
risk = "medium"
|
||||
else:
|
||||
risk = "low"
|
||||
key_entities = [
|
||||
n["name"] for n in impacted_dicts[:5]
|
||||
]
|
||||
minimal_response = {
|
||||
"status": "ok",
|
||||
"summary": "\n".join(summary_parts),
|
||||
"risk": risk,
|
||||
"impacted_file_count": len(result["impacted_files"]),
|
||||
"key_entities": key_entities,
|
||||
"truncated": truncated,
|
||||
}
|
||||
attach_context_savings(minimal_response, original_tokens=original_tokens)
|
||||
return minimal_response
|
||||
|
||||
response = {
|
||||
"status": "ok",
|
||||
"summary": "\n".join(summary_parts),
|
||||
"changed_files": changed_files,
|
||||
"changed_nodes": changed_dicts,
|
||||
"impacted_nodes": impacted_dicts,
|
||||
"impacted_files": result["impacted_files"],
|
||||
"edges": edge_dicts,
|
||||
"truncated": truncated,
|
||||
"total_impacted": total_impacted,
|
||||
}
|
||||
attach_context_savings(response, original_tokens=original_tokens)
|
||||
return response
|
||||
finally:
|
||||
store.close()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Tool 3: query_graph
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def query_graph(
|
||||
pattern: str,
|
||||
target: str,
|
||||
repo_root: str | None = None,
|
||||
detail_level: str = "standard",
|
||||
) -> dict[str, Any]:
|
||||
"""Run a predefined graph query.
|
||||
|
||||
Args:
|
||||
pattern: Query pattern. One of: callers_of, callees_of, imports_of,
|
||||
importers_of, children_of, tests_for, inheritors_of, file_summary.
|
||||
target: The node name, qualified name, or file path to query about.
|
||||
repo_root: Repository root path. Auto-detected if omitted.
|
||||
detail_level: "standard" (full output) or "minimal" (summary only).
|
||||
|
||||
Returns:
|
||||
Matching nodes and edges for the query.
|
||||
"""
|
||||
store, root = _get_store(repo_root)
|
||||
try:
|
||||
if pattern not in _QUERY_PATTERNS:
|
||||
return {
|
||||
"status": "error",
|
||||
"error": (
|
||||
f"Unknown pattern '{pattern}'. "
|
||||
f"Available: {list(_QUERY_PATTERNS.keys())}"
|
||||
),
|
||||
}
|
||||
|
||||
results: list[dict] = []
|
||||
edges_out: list[dict] = []
|
||||
|
||||
# For callers_of, skip common builtins early (bare names only)
|
||||
# "Who calls .map()?" returns hundreds of useless hits.
|
||||
# Qualified names (e.g. "utils.py::map") bypass this filter.
|
||||
if (
|
||||
pattern == "callers_of"
|
||||
and target in _BUILTIN_CALL_NAMES
|
||||
and "::" not in target
|
||||
):
|
||||
return {
|
||||
"status": "ok", "pattern": pattern, "target": target,
|
||||
"description": _QUERY_PATTERNS[pattern],
|
||||
"summary": (
|
||||
f"'{target}' is a common builtin "
|
||||
"— callers_of skipped to avoid noise."
|
||||
),
|
||||
"results": [], "edges": [],
|
||||
}
|
||||
|
||||
# Resolve target - try as-is, then as absolute path, then search.
|
||||
# file_summary targets are paths, so skip broad node search.
|
||||
node = None
|
||||
if pattern != "file_summary":
|
||||
node = store.get_node(target)
|
||||
if not node:
|
||||
abs_target = str(root / target)
|
||||
node = store.get_node(abs_target)
|
||||
if not node:
|
||||
# Search by name
|
||||
candidates = store.search_nodes(target, limit=5)
|
||||
if len(candidates) == 1:
|
||||
node = candidates[0]
|
||||
target = node.qualified_name
|
||||
elif len(candidates) > 1:
|
||||
return {
|
||||
"status": "ambiguous",
|
||||
"summary": (
|
||||
f"Multiple matches for '{target}'. "
|
||||
"Please use a qualified name."
|
||||
),
|
||||
"candidates": [node_to_dict(c) for c in candidates],
|
||||
}
|
||||
|
||||
if not node and pattern != "file_summary":
|
||||
return {
|
||||
"status": "not_found",
|
||||
"summary": f"No node found matching '{target}'.",
|
||||
}
|
||||
|
||||
qn = node.qualified_name if node else target
|
||||
|
||||
if pattern == "callers_of":
|
||||
seen_sources: set[str] = set()
|
||||
for e in store.get_edges_by_target(qn):
|
||||
if e.kind == "CALLS":
|
||||
if e.source_qualified not in seen_sources:
|
||||
seen_sources.add(e.source_qualified)
|
||||
caller = store.get_node(e.source_qualified)
|
||||
if caller:
|
||||
results.append(node_to_dict(caller))
|
||||
edges_out.append(edge_to_dict(e))
|
||||
# Fallback: CALLS edges store unqualified target names
|
||||
# (e.g. "generateTestCode") while qn is fully qualified
|
||||
# (e.g. "file.ts::generateTestCode"). Search by plain name too.
|
||||
if node:
|
||||
for e in store.search_edges_by_target_name(node.name):
|
||||
if e.source_qualified not in seen_sources:
|
||||
seen_sources.add(e.source_qualified)
|
||||
caller = store.get_node(e.source_qualified)
|
||||
if caller:
|
||||
results.append(node_to_dict(caller))
|
||||
edges_out.append(edge_to_dict(e))
|
||||
|
||||
elif pattern == "callees_of":
|
||||
seen_targets: set[str] = set()
|
||||
for e in store.get_edges_by_source(qn):
|
||||
if e.kind == "CALLS":
|
||||
if e.target_qualified not in seen_targets:
|
||||
seen_targets.add(e.target_qualified)
|
||||
callee = store.get_node(e.target_qualified)
|
||||
if callee:
|
||||
results.append(node_to_dict(callee))
|
||||
elif "::" not in e.target_qualified:
|
||||
results.append({
|
||||
"kind": "Function",
|
||||
"name": e.target_qualified,
|
||||
"qualified_name": e.target_qualified,
|
||||
})
|
||||
edges_out.append(edge_to_dict(e))
|
||||
|
||||
elif pattern == "imports_of":
|
||||
for e in store.get_edges_by_source(qn):
|
||||
if e.kind == "IMPORTS_FROM":
|
||||
results.append({"import_target": e.target_qualified})
|
||||
edges_out.append(edge_to_dict(e))
|
||||
|
||||
elif pattern == "importers_of":
|
||||
# Find edges where target matches this file.
|
||||
# Use resolve() to canonicalize the path, matching how
|
||||
# _resolve_module_to_file stores edge targets.
|
||||
abs_target = (
|
||||
str((root / target).resolve()) if node is None
|
||||
else node.file_path
|
||||
)
|
||||
for e in store.get_edges_by_target(abs_target):
|
||||
if e.kind == "IMPORTS_FROM":
|
||||
results.append({
|
||||
"importer": e.source_qualified,
|
||||
"file": e.file_path,
|
||||
})
|
||||
edges_out.append(edge_to_dict(e))
|
||||
|
||||
elif pattern == "children_of":
|
||||
for e in store.get_edges_by_source(qn):
|
||||
if e.kind == "CONTAINS":
|
||||
child = store.get_node(e.target_qualified)
|
||||
if child:
|
||||
results.append(node_to_dict(child))
|
||||
|
||||
elif pattern == "tests_for":
|
||||
for e in store.get_edges_by_target(qn):
|
||||
if e.kind == "TESTED_BY":
|
||||
test = store.get_node(e.source_qualified)
|
||||
if test:
|
||||
results.append(node_to_dict(test))
|
||||
# Also search by naming convention
|
||||
name = node.name if node else target
|
||||
test_nodes = store.search_nodes(f"test_{name}", limit=10)
|
||||
test_nodes += store.search_nodes(f"Test{name}", limit=10)
|
||||
seen = {r.get("qualified_name") for r in results}
|
||||
for t in test_nodes:
|
||||
if t.qualified_name not in seen and t.is_test:
|
||||
results.append(node_to_dict(t))
|
||||
|
||||
elif pattern == "inheritors_of":
|
||||
for e in store.get_edges_by_target(qn):
|
||||
if e.kind in ("INHERITS", "IMPLEMENTS"):
|
||||
child = store.get_node(e.source_qualified)
|
||||
if child:
|
||||
results.append(node_to_dict(child))
|
||||
edges_out.append(edge_to_dict(e))
|
||||
# Fallback: INHERITS/IMPLEMENTS edges store unqualified base names
|
||||
# (e.g. "Animal") while qn is fully qualified
|
||||
# (e.g. "sample.dart::Animal"). Search by plain name too. See: #87
|
||||
if not results and node:
|
||||
for kind in ("INHERITS", "IMPLEMENTS"):
|
||||
for e in store.search_edges_by_target_name(node.name, kind=kind):
|
||||
child = store.get_node(e.source_qualified)
|
||||
if child:
|
||||
results.append(node_to_dict(child))
|
||||
edges_out.append(edge_to_dict(e))
|
||||
|
||||
elif pattern == "file_summary":
|
||||
graph_paths = _resolve_graph_file_paths(store, root, [target])
|
||||
for graph_path in graph_paths:
|
||||
for n in store.get_nodes_by_file(graph_path):
|
||||
results.append(node_to_dict(n))
|
||||
|
||||
summary = (
|
||||
f"Found {len(results)} result(s) "
|
||||
f"for {pattern}('{target}')"
|
||||
)
|
||||
|
||||
if detail_level == "minimal":
|
||||
minimal_results = [
|
||||
{
|
||||
k: r[k]
|
||||
for k in ("name", "kind", "file_path")
|
||||
if k in r
|
||||
}
|
||||
for r in results[:5]
|
||||
]
|
||||
return {
|
||||
"status": "ok",
|
||||
"pattern": pattern,
|
||||
"target": target,
|
||||
"description": _QUERY_PATTERNS[pattern],
|
||||
"summary": summary,
|
||||
"result_count": len(results),
|
||||
"results": minimal_results,
|
||||
}
|
||||
|
||||
return {
|
||||
"status": "ok",
|
||||
"pattern": pattern,
|
||||
"target": target,
|
||||
"description": _QUERY_PATTERNS[pattern],
|
||||
"summary": summary,
|
||||
"results": results,
|
||||
"edges": edges_out,
|
||||
}
|
||||
finally:
|
||||
store.close()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Tool 5: semantic_search_nodes
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def semantic_search_nodes(
|
||||
query: str,
|
||||
kind: str | None = None,
|
||||
limit: int = 20,
|
||||
repo_root: str | None = None,
|
||||
context_files: list[str] | None = None,
|
||||
model: str | None = None,
|
||||
provider: str | None = None,
|
||||
detail_level: str = "standard",
|
||||
) -> dict[str, Any]:
|
||||
"""Search for nodes by name, keyword, or semantic similarity.
|
||||
|
||||
Uses hybrid search (FTS5 BM25 + vector embeddings merged via Reciprocal
|
||||
Rank Fusion) as the primary search path, with graceful fallback to
|
||||
keyword matching.
|
||||
|
||||
Args:
|
||||
query: Search string to match against node names and qualified names.
|
||||
kind: Optional filter by node kind (File, Class, Function, Type, Test).
|
||||
limit: Maximum results to return (default: 20).
|
||||
repo_root: Repository root path. Auto-detected if omitted.
|
||||
context_files: Optional list of file paths. Nodes in these files
|
||||
receive a relevance boost.
|
||||
detail_level: "standard" (full output) or "minimal" (summary only).
|
||||
|
||||
Returns:
|
||||
Ranked list of matching nodes.
|
||||
"""
|
||||
store, root = _get_store(repo_root)
|
||||
try:
|
||||
results = hybrid_search(
|
||||
store, query, kind=kind, limit=limit, context_files=context_files,
|
||||
model=model, provider=provider,
|
||||
)
|
||||
|
||||
search_mode = "hybrid"
|
||||
if not results:
|
||||
search_mode = "keyword"
|
||||
|
||||
summary = f"Found {len(results)} node(s) matching '{query}'" + (
|
||||
f" (kind={kind})" if kind else ""
|
||||
)
|
||||
|
||||
if detail_level == "minimal":
|
||||
minimal_results = [
|
||||
{
|
||||
k: r[k]
|
||||
for k in ("name", "kind", "file_path", "score")
|
||||
if k in r
|
||||
}
|
||||
for r in results[:5]
|
||||
]
|
||||
return {
|
||||
"status": "ok",
|
||||
"query": query,
|
||||
"search_mode": search_mode,
|
||||
"summary": summary,
|
||||
"results": minimal_results,
|
||||
}
|
||||
|
||||
result: dict[str, object] = {
|
||||
"status": "ok",
|
||||
"query": query,
|
||||
"search_mode": search_mode,
|
||||
"summary": summary,
|
||||
"results": results,
|
||||
}
|
||||
result["_hints"] = generate_hints(
|
||||
"semantic_search_nodes", result, get_session()
|
||||
)
|
||||
return result
|
||||
finally:
|
||||
store.close()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Tool 6: list_graph_stats
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def list_graph_stats(repo_root: str | None = None) -> dict[str, Any]:
|
||||
"""Get aggregate statistics about the knowledge graph.
|
||||
|
||||
Args:
|
||||
repo_root: Repository root path. Auto-detected if omitted.
|
||||
|
||||
Returns:
|
||||
Total nodes, edges, breakdown by kind, languages, and last update time.
|
||||
"""
|
||||
store, root = _get_store(repo_root)
|
||||
try:
|
||||
stats = store.get_stats()
|
||||
|
||||
summary_parts = [
|
||||
f"Graph statistics for {root.name}:",
|
||||
f" Files: {stats.files_count}",
|
||||
f" Total nodes: {stats.total_nodes}",
|
||||
f" Total edges: {stats.total_edges}",
|
||||
f" Languages: {', '.join(stats.languages) if stats.languages else 'none'}",
|
||||
f" Last updated: {stats.last_updated or 'never'}",
|
||||
"",
|
||||
"Nodes by kind:",
|
||||
]
|
||||
for kind, count in sorted(stats.nodes_by_kind.items()):
|
||||
summary_parts.append(f" {kind}: {count}")
|
||||
summary_parts.append("")
|
||||
summary_parts.append("Edges by kind:")
|
||||
for kind, count in sorted(stats.edges_by_kind.items()):
|
||||
summary_parts.append(f" {kind}: {count}")
|
||||
|
||||
# Add embedding info if available
|
||||
emb_store = EmbeddingStore(get_db_path(root))
|
||||
try:
|
||||
emb_count = emb_store.count()
|
||||
summary_parts.append("")
|
||||
summary_parts.append(f"Embeddings: {emb_count} nodes embedded")
|
||||
if not emb_store.available:
|
||||
summary_parts.append(
|
||||
" (install sentence-transformers for semantic search)"
|
||||
)
|
||||
finally:
|
||||
emb_store.close()
|
||||
|
||||
return {
|
||||
"status": "ok",
|
||||
"summary": "\n".join(summary_parts),
|
||||
"total_nodes": stats.total_nodes,
|
||||
"total_edges": stats.total_edges,
|
||||
"nodes_by_kind": stats.nodes_by_kind,
|
||||
"edges_by_kind": stats.edges_by_kind,
|
||||
"languages": stats.languages,
|
||||
"files_count": stats.files_count,
|
||||
"last_updated": stats.last_updated,
|
||||
"embeddings_count": emb_count,
|
||||
}
|
||||
finally:
|
||||
store.close()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Tool 9: find_large_functions
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def find_large_functions(
|
||||
min_lines: int = 50,
|
||||
kind: str | None = None,
|
||||
file_path_pattern: str | None = None,
|
||||
limit: int = 50,
|
||||
repo_root: str | None = None,
|
||||
) -> dict[str, Any]:
|
||||
"""Find functions, classes, or files exceeding a line-count threshold.
|
||||
|
||||
Useful for identifying decomposition targets, code-quality audits,
|
||||
and enforcing size limits during code review.
|
||||
|
||||
Args:
|
||||
min_lines: Minimum line count to flag (default: 50).
|
||||
kind: Filter by node kind: Function, Class, File, or Test.
|
||||
file_path_pattern: Filter by file path substring (e.g. "components/").
|
||||
limit: Maximum results (default: 50).
|
||||
repo_root: Repository root path. Auto-detected if omitted.
|
||||
|
||||
Returns:
|
||||
Oversized nodes with line counts, ordered largest first.
|
||||
"""
|
||||
store, root = _get_store(repo_root)
|
||||
try:
|
||||
nodes = store.get_nodes_by_size(
|
||||
min_lines=min_lines,
|
||||
kind=kind,
|
||||
file_path_pattern=file_path_pattern,
|
||||
limit=limit,
|
||||
)
|
||||
|
||||
results = []
|
||||
for n in nodes:
|
||||
d = node_to_dict(n)
|
||||
d["line_count"] = (
|
||||
(n.line_end - n.line_start + 1)
|
||||
if n.line_start and n.line_end
|
||||
else 0
|
||||
)
|
||||
# Make file_path relative for readability
|
||||
try:
|
||||
d["relative_path"] = str(Path(n.file_path).relative_to(root))
|
||||
except ValueError:
|
||||
d["relative_path"] = n.file_path
|
||||
results.append(d)
|
||||
|
||||
summary_parts = [
|
||||
f"Found {len(results)} node(s) with >= {min_lines} lines"
|
||||
+ (f" (kind={kind})" if kind else "")
|
||||
+ (f" matching '{file_path_pattern}'" if file_path_pattern else "")
|
||||
+ ":",
|
||||
]
|
||||
for r in results[:10]:
|
||||
summary_parts.append(
|
||||
f" {r['line_count']:>4} lines | {r['kind']:>8} | "
|
||||
f"{r['name']} ({r['relative_path']}:{r['line_start']})"
|
||||
)
|
||||
if len(results) > 10:
|
||||
summary_parts.append(f" ... and {len(results) - 10} more")
|
||||
|
||||
return {
|
||||
"status": "ok",
|
||||
"summary": "\n".join(summary_parts),
|
||||
"total_found": len(results),
|
||||
"min_lines": min_lines,
|
||||
"results": results,
|
||||
}
|
||||
finally:
|
||||
store.close()
|
||||
|
||||
|
||||
# -------------------------------------------------------------------
|
||||
# traverse_graph: free-form BFS / DFS traversal
|
||||
# -------------------------------------------------------------------
|
||||
|
||||
|
||||
def traverse_graph_func(
|
||||
query: str,
|
||||
mode: str = "bfs",
|
||||
depth: int = 3,
|
||||
token_budget: int = 2000,
|
||||
repo_root: str | None = None,
|
||||
) -> dict[str, Any]:
|
||||
"""BFS/DFS traversal from best-matching node.
|
||||
|
||||
Args:
|
||||
query: Search string to find the starting node.
|
||||
mode: "bfs" (breadth-first) or "dfs" (depth-first).
|
||||
depth: Max traversal depth (1-6). Default: 3.
|
||||
token_budget: Approximate token limit for results.
|
||||
repo_root: Repository root path.
|
||||
"""
|
||||
store, root = _get_store(repo_root)
|
||||
try:
|
||||
results = hybrid_search(store, query, limit=1)
|
||||
if not results:
|
||||
return {
|
||||
"error": f"No node matching '{query}'",
|
||||
"nodes": [],
|
||||
}
|
||||
|
||||
start_qn = results[0]["qualified_name"]
|
||||
depth = max(1, min(depth, 6))
|
||||
|
||||
# BFS / DFS traversal
|
||||
visited: dict[str, int] = {} # qn -> depth
|
||||
queue: list[tuple[str, int]] = [
|
||||
(start_qn, 0),
|
||||
]
|
||||
traversal: list[dict] = []
|
||||
approx_tokens = 0
|
||||
|
||||
while queue:
|
||||
if mode == "bfs":
|
||||
current_qn, cur_depth = queue.pop(0)
|
||||
else:
|
||||
current_qn, cur_depth = queue.pop()
|
||||
|
||||
if current_qn in visited:
|
||||
continue
|
||||
if cur_depth > depth:
|
||||
continue
|
||||
|
||||
visited[current_qn] = cur_depth
|
||||
node = store.get_node(current_qn)
|
||||
if not node:
|
||||
continue
|
||||
|
||||
entry = {
|
||||
"name": _sanitize_name(node.name),
|
||||
"qualified_name": node.qualified_name,
|
||||
"kind": node.kind,
|
||||
"file": node.file_path,
|
||||
"depth": cur_depth,
|
||||
}
|
||||
approx_tokens += len(str(entry)) // 4
|
||||
if approx_tokens > token_budget:
|
||||
break
|
||||
|
||||
traversal.append(entry)
|
||||
|
||||
# Get neighbours
|
||||
out_edges = store.get_edges_by_source(
|
||||
current_qn
|
||||
)
|
||||
in_edges = store.get_edges_by_target(
|
||||
current_qn
|
||||
)
|
||||
for e in out_edges:
|
||||
tgt = e.target_qualified
|
||||
if tgt not in visited:
|
||||
queue.append((tgt, cur_depth + 1))
|
||||
for e in in_edges:
|
||||
src = e.source_qualified
|
||||
if src not in visited:
|
||||
queue.append((src, cur_depth + 1))
|
||||
|
||||
return {
|
||||
"start_node": start_qn,
|
||||
"mode": mode,
|
||||
"max_depth": depth,
|
||||
"nodes_visited": len(traversal),
|
||||
"traversal": traversal,
|
||||
"truncated": approx_tokens > token_budget,
|
||||
"next_tool_suggestions": [
|
||||
"query_graph callers_of"
|
||||
" -- focused relationship query",
|
||||
"get_impact_radius"
|
||||
" -- blast radius analysis",
|
||||
],
|
||||
}
|
||||
finally:
|
||||
store.close()
|
||||
@@ -0,0 +1,168 @@
|
||||
"""Tools 17, 18: refactor_func, apply_refactor_func."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from ..hints import generate_hints, get_session
|
||||
from ..incremental import find_project_root
|
||||
from ..refactor import (
|
||||
apply_refactor,
|
||||
find_dead_code,
|
||||
rename_preview,
|
||||
suggest_refactorings,
|
||||
)
|
||||
from ._common import _get_store, _validate_repo_root
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Tool 17: refactor_tool [REFACTOR]
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def refactor_func(
|
||||
mode: str = "rename",
|
||||
old_name: str | None = None,
|
||||
new_name: str | None = None,
|
||||
kind: str | None = None,
|
||||
file_pattern: str | None = None,
|
||||
repo_root: str | None = None,
|
||||
) -> dict[str, Any]:
|
||||
"""Unified refactoring entry point.
|
||||
|
||||
[REFACTOR] Supports three modes:
|
||||
- ``rename``: Preview renaming a symbol (requires *old_name* and
|
||||
*new_name*).
|
||||
- ``dead_code``: Find unreferenced functions/classes.
|
||||
- ``suggest``: Get community-driven refactoring suggestions.
|
||||
|
||||
Args:
|
||||
mode: One of ``"rename"``, ``"dead_code"``, or ``"suggest"``.
|
||||
old_name: (rename mode) Current symbol name.
|
||||
new_name: (rename mode) Desired new name.
|
||||
kind: (dead_code mode) Optional node kind filter.
|
||||
file_pattern: (dead_code mode) Optional file path substring filter.
|
||||
repo_root: Repository root path. Auto-detected if omitted.
|
||||
|
||||
Returns:
|
||||
Mode-specific results dict.
|
||||
"""
|
||||
valid_modes = {"rename", "dead_code", "suggest"}
|
||||
if mode not in valid_modes:
|
||||
return {
|
||||
"status": "error",
|
||||
"error": (
|
||||
f"Invalid mode '{mode}'. "
|
||||
f"Must be one of: {', '.join(sorted(valid_modes))}"
|
||||
),
|
||||
}
|
||||
|
||||
store, root = _get_store(repo_root)
|
||||
try:
|
||||
if mode == "rename":
|
||||
if not old_name or not new_name:
|
||||
return {
|
||||
"status": "error",
|
||||
"error": (
|
||||
"rename mode requires both old_name and new_name."
|
||||
),
|
||||
}
|
||||
preview = rename_preview(store, old_name, new_name)
|
||||
if preview is None:
|
||||
return {
|
||||
"status": "not_found",
|
||||
"summary": f"No node found matching '{old_name}'.",
|
||||
}
|
||||
result = {
|
||||
"status": "ok",
|
||||
"summary": (
|
||||
f"Rename preview: {old_name} -> {new_name}, "
|
||||
f"{len(preview['edits'])} edit(s). "
|
||||
f"Use apply_refactor_tool(refactor_id="
|
||||
f"'{preview['refactor_id']}') to apply."
|
||||
),
|
||||
**preview,
|
||||
}
|
||||
result["_hints"] = generate_hints(
|
||||
"refactor", result, get_session()
|
||||
)
|
||||
return result
|
||||
|
||||
elif mode == "dead_code":
|
||||
dead = find_dead_code(
|
||||
store, kind=kind, file_pattern=file_pattern, root=root
|
||||
)
|
||||
result = {
|
||||
"status": "ok",
|
||||
"summary": f"Found {len(dead)} dead code symbol(s).",
|
||||
"dead_code": dead,
|
||||
"total": len(dead),
|
||||
}
|
||||
result["_hints"] = generate_hints(
|
||||
"refactor", result, get_session()
|
||||
)
|
||||
return result
|
||||
|
||||
else: # suggest
|
||||
suggestions = suggest_refactorings(store)
|
||||
result = {
|
||||
"status": "ok",
|
||||
"summary": (
|
||||
f"Generated {len(suggestions)} "
|
||||
"refactoring suggestion(s)."
|
||||
),
|
||||
"suggestions": suggestions,
|
||||
"total": len(suggestions),
|
||||
}
|
||||
result["_hints"] = generate_hints(
|
||||
"refactor", result, get_session()
|
||||
)
|
||||
return result
|
||||
|
||||
except Exception as exc:
|
||||
return {"status": "error", "error": str(exc)}
|
||||
finally:
|
||||
store.close()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Tool 18: apply_refactor_tool [REFACTOR]
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def apply_refactor_func(
|
||||
refactor_id: str,
|
||||
repo_root: str | None = None,
|
||||
dry_run: bool = False,
|
||||
) -> dict[str, Any]:
|
||||
"""Apply a previously previewed refactoring to source files.
|
||||
|
||||
[REFACTOR] Validates the refactor_id, checks expiry, ensures all edit
|
||||
paths are within the repo root, then performs exact string replacements.
|
||||
|
||||
Args:
|
||||
refactor_id: ID returned by a prior ``refactor_tool(mode="rename")``
|
||||
call.
|
||||
repo_root: Repository root path. Auto-detected if omitted.
|
||||
dry_run: If True, return a unified diff of what would change
|
||||
without touching disk. The refactor_id remains valid so the
|
||||
user can review the diff, then call again with ``dry_run=False``
|
||||
to actually write the changes. See: #176
|
||||
|
||||
Returns:
|
||||
Status with count of applied edits and modified files. When
|
||||
``dry_run=True`` the response additionally contains ``would_modify``
|
||||
(list of file paths) and ``diffs`` (map of file -> unified-diff
|
||||
string).
|
||||
"""
|
||||
try:
|
||||
root = (
|
||||
_validate_repo_root(Path(repo_root))
|
||||
if repo_root
|
||||
else find_project_root()
|
||||
)
|
||||
except (RuntimeError, ValueError) as exc:
|
||||
return {"status": "error", "error": str(exc)}
|
||||
|
||||
result = apply_refactor(refactor_id, root, dry_run=dry_run)
|
||||
return result
|
||||
@@ -0,0 +1,125 @@
|
||||
"""Tools 21, 22: list_repos_func, cross_repo_search_func."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from ..graph import GraphStore
|
||||
from ..incremental import get_db_path
|
||||
from ..search import hybrid_search
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Tool 21: list_repos [REGISTRY]
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def list_repos_func() -> dict[str, Any]:
|
||||
"""List all registered repositories.
|
||||
|
||||
[REGISTRY] Returns the list of repositories registered in the global
|
||||
multi-repo registry at ``~/.code-review-graph/registry.json``.
|
||||
|
||||
Returns:
|
||||
List of registered repos with paths and aliases.
|
||||
"""
|
||||
from ..registry import Registry
|
||||
|
||||
try:
|
||||
registry = Registry()
|
||||
repos = registry.list_repos()
|
||||
return {
|
||||
"status": "ok",
|
||||
"summary": f"{len(repos)} registered repository(ies)",
|
||||
"repos": repos,
|
||||
}
|
||||
except Exception as exc:
|
||||
return {"status": "error", "error": str(exc)}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Tool 22: cross_repo_search [REGISTRY]
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def cross_repo_search_func(
|
||||
query: str,
|
||||
kind: str | None = None,
|
||||
limit: int = 20,
|
||||
) -> dict[str, Any]:
|
||||
"""Search across all registered repositories.
|
||||
|
||||
[REGISTRY] Runs hybrid_search on each registered repo's graph database
|
||||
and merges the results.
|
||||
|
||||
Args:
|
||||
query: Search query string.
|
||||
kind: Optional node kind filter (e.g. "Function", "Class").
|
||||
limit: Maximum results per repo (default: 20).
|
||||
|
||||
Returns:
|
||||
Combined search results from all registered repos.
|
||||
"""
|
||||
from ..registry import Registry
|
||||
|
||||
try:
|
||||
registry = Registry()
|
||||
repos = registry.list_repos()
|
||||
if not repos:
|
||||
return {
|
||||
"status": "ok",
|
||||
"summary": (
|
||||
"No repositories registered. "
|
||||
"Use 'register' to add repos."
|
||||
),
|
||||
"results": [],
|
||||
}
|
||||
|
||||
all_results: list[dict[str, Any]] = []
|
||||
searched_repos: list[str] = []
|
||||
|
||||
for repo_entry in repos:
|
||||
repo_path = Path(repo_entry["path"])
|
||||
db_path = get_db_path(repo_path)
|
||||
if not db_path.exists():
|
||||
continue
|
||||
|
||||
try:
|
||||
store = GraphStore(str(db_path))
|
||||
try:
|
||||
results = hybrid_search(
|
||||
store, query, kind=kind, limit=limit
|
||||
)
|
||||
alias = repo_entry.get("alias", repo_path.name)
|
||||
for r in results:
|
||||
r["repo"] = alias
|
||||
r["repo_path"] = str(repo_path)
|
||||
all_results.extend(results)
|
||||
searched_repos.append(alias)
|
||||
finally:
|
||||
store.close()
|
||||
except Exception as exc:
|
||||
logger.warning(
|
||||
"Search failed for %s: %s", repo_path, exc
|
||||
)
|
||||
|
||||
# Sort all results by score descending
|
||||
all_results.sort(
|
||||
key=lambda r: r.get("score", 0), reverse=True
|
||||
)
|
||||
|
||||
return {
|
||||
"status": "ok",
|
||||
"summary": (
|
||||
f"Found {len(all_results)} result(s) across "
|
||||
f"{len(searched_repos)} repo(s) for '{query}'"
|
||||
),
|
||||
"results": all_results[:limit],
|
||||
"repos_searched": searched_repos,
|
||||
}
|
||||
except Exception as exc:
|
||||
return {"status": "error", "error": str(exc)}
|
||||
@@ -0,0 +1,477 @@
|
||||
"""Tools 4, 12, 16: review context, affected flows, detect changes."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from ..changes import analyze_changes, parse_diff_ranges, parse_git_diff_ranges # noqa: F401
|
||||
from ..context_savings import attach_context_savings, estimate_file_tokens
|
||||
from ..flows import get_affected_flows as _get_affected_flows
|
||||
from ..graph import edge_to_dict, node_to_dict
|
||||
from ..hints import generate_hints, get_session
|
||||
from ..incremental import get_changed_files, get_staged_and_unstaged
|
||||
from ._common import _get_store, _resolve_graph_file_paths
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Tool 4: get_review_context
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def get_review_context(
|
||||
changed_files: list[str] | None = None,
|
||||
max_depth: int = 2,
|
||||
include_source: bool = True,
|
||||
max_lines_per_file: int = 200,
|
||||
repo_root: str | None = None,
|
||||
base: str = "HEAD~1",
|
||||
detail_level: str = "standard",
|
||||
) -> dict[str, Any]:
|
||||
"""Generate a focused review context from changed files.
|
||||
|
||||
Builds a token-optimized subgraph + source snippets for code review.
|
||||
|
||||
Args:
|
||||
changed_files: Files to review (auto-detected from git diff if omitted).
|
||||
max_depth: Impact radius depth (default: 2).
|
||||
include_source: Whether to include source code snippets (default: True).
|
||||
max_lines_per_file: Max source lines per file in output (default: 200).
|
||||
repo_root: Repository root path. Auto-detected if omitted.
|
||||
base: Git ref for change detection (default: HEAD~1).
|
||||
detail_level: Output detail level. "standard" returns full context;
|
||||
"minimal" returns summary, risk level, changed/impacted file counts,
|
||||
top 5 key entity names, test gap count, and next tool suggestions.
|
||||
Default: "standard".
|
||||
|
||||
Returns:
|
||||
Structured review context with subgraph, source snippets, and
|
||||
review guidance.
|
||||
"""
|
||||
store, root = _get_store(repo_root)
|
||||
try:
|
||||
# Get impact radius first
|
||||
if changed_files is None:
|
||||
changed_files = get_changed_files(root, base)
|
||||
if not changed_files:
|
||||
changed_files = get_staged_and_unstaged(root)
|
||||
|
||||
if not changed_files:
|
||||
return {
|
||||
"status": "ok",
|
||||
"summary": "No changes detected. Nothing to review.",
|
||||
"context": {},
|
||||
}
|
||||
|
||||
graph_files = _resolve_graph_file_paths(store, root, changed_files)
|
||||
original_tokens = estimate_file_tokens(root, changed_files)
|
||||
impact = store.get_impact_radius(graph_files, max_depth=max_depth)
|
||||
|
||||
if detail_level == "minimal":
|
||||
impacted_count = len(impact["impacted_nodes"])
|
||||
if impacted_count > 20:
|
||||
risk = "high"
|
||||
elif impacted_count > 5:
|
||||
risk = "medium"
|
||||
else:
|
||||
risk = "low"
|
||||
|
||||
key_entities = [
|
||||
n.name for n in impact["changed_nodes"][:5]
|
||||
]
|
||||
|
||||
# Count test gaps among changed functions.
|
||||
changed_funcs = [
|
||||
n for n in impact["changed_nodes"]
|
||||
if n.kind == "Function" and not n.is_test
|
||||
]
|
||||
test_edges = [
|
||||
e for e in impact["edges"] if e.kind == "TESTED_BY"
|
||||
]
|
||||
tested_qualified = {e.source_qualified for e in test_edges}
|
||||
test_gap_count = sum(
|
||||
1 for f in changed_funcs
|
||||
if f.qualified_name not in tested_qualified
|
||||
)
|
||||
|
||||
summary_parts = [
|
||||
f"Review context for {len(changed_files)} changed file(s):",
|
||||
f" - Risk: {risk}",
|
||||
f" - {len(impact['impacted_nodes'])} impacted nodes"
|
||||
f" in {len(impact['impacted_files'])} files",
|
||||
]
|
||||
|
||||
result = {
|
||||
"status": "ok",
|
||||
"summary": "\n".join(summary_parts),
|
||||
"risk": risk,
|
||||
"changed_file_count": len(changed_files),
|
||||
"impacted_file_count": len(impact["impacted_files"]),
|
||||
"key_entities": key_entities,
|
||||
"test_gaps": test_gap_count,
|
||||
"next_tool_suggestions": [
|
||||
"detect_changes",
|
||||
"get_affected_flows",
|
||||
"get_impact_radius",
|
||||
],
|
||||
}
|
||||
attach_context_savings(result, original_tokens=original_tokens)
|
||||
return result
|
||||
|
||||
# Build review context
|
||||
context: dict[str, Any] = {
|
||||
"changed_files": changed_files,
|
||||
"impacted_files": impact["impacted_files"],
|
||||
"graph": {
|
||||
"changed_nodes": [
|
||||
node_to_dict(n) for n in impact["changed_nodes"]
|
||||
],
|
||||
"impacted_nodes": [
|
||||
node_to_dict(n) for n in impact["impacted_nodes"]
|
||||
],
|
||||
"edges": [edge_to_dict(e) for e in impact["edges"]],
|
||||
},
|
||||
}
|
||||
|
||||
# Add source snippets for changed files
|
||||
if include_source:
|
||||
snippets = {}
|
||||
for rel_path in changed_files:
|
||||
full_path = root / rel_path
|
||||
if full_path.is_file():
|
||||
try:
|
||||
lines = full_path.read_text(
|
||||
errors="replace"
|
||||
).splitlines()
|
||||
if len(lines) > max_lines_per_file:
|
||||
# Include only the relevant functions/classes
|
||||
relevant_lines = _extract_relevant_lines(
|
||||
lines,
|
||||
impact["changed_nodes"],
|
||||
str(full_path),
|
||||
)
|
||||
snippets[rel_path] = relevant_lines
|
||||
else:
|
||||
snippets[rel_path] = "\n".join(
|
||||
f"{i+1}: {line}"
|
||||
for i, line in enumerate(lines)
|
||||
)
|
||||
except (OSError, UnicodeDecodeError):
|
||||
snippets[rel_path] = "(could not read file)"
|
||||
context["source_snippets"] = snippets
|
||||
|
||||
# Generate review guidance
|
||||
guidance = _generate_review_guidance(impact, changed_files)
|
||||
context["review_guidance"] = guidance
|
||||
|
||||
summary_parts = [
|
||||
f"Review context for {len(changed_files)} changed file(s):",
|
||||
f" - {len(impact['changed_nodes'])} directly changed nodes",
|
||||
f" - {len(impact['impacted_nodes'])} impacted nodes"
|
||||
f" in {len(impact['impacted_files'])} files",
|
||||
"",
|
||||
"Review guidance:",
|
||||
guidance,
|
||||
]
|
||||
|
||||
result = {
|
||||
"status": "ok",
|
||||
"summary": "\n".join(summary_parts),
|
||||
"context": context,
|
||||
}
|
||||
attach_context_savings(result, original_tokens=original_tokens)
|
||||
return result
|
||||
finally:
|
||||
store.close()
|
||||
|
||||
|
||||
def _extract_relevant_lines(
|
||||
lines: list[str], nodes: list, file_path: str
|
||||
) -> str:
|
||||
"""Extract only the lines relevant to changed nodes."""
|
||||
ranges = []
|
||||
for n in nodes:
|
||||
if n.file_path == file_path:
|
||||
start = max(0, n.line_start - 3) # 2 lines context before
|
||||
end = min(len(lines), n.line_end + 2) # 1 line context after
|
||||
ranges.append((start, end))
|
||||
|
||||
if not ranges:
|
||||
# Show first N lines as fallback
|
||||
return "\n".join(
|
||||
f"{i+1}: {line}" for i, line in enumerate(lines[:50])
|
||||
)
|
||||
|
||||
# Merge overlapping ranges
|
||||
ranges.sort()
|
||||
merged = [ranges[0]]
|
||||
for start, end in ranges[1:]:
|
||||
if start <= merged[-1][1] + 1:
|
||||
merged[-1] = (merged[-1][0], max(merged[-1][1], end))
|
||||
else:
|
||||
merged.append((start, end))
|
||||
|
||||
parts: list[str] = []
|
||||
for start, end in merged:
|
||||
if parts:
|
||||
parts.append("...")
|
||||
for i in range(start, end):
|
||||
parts.append(f"{i+1}: {lines[i]}")
|
||||
|
||||
return "\n".join(parts)
|
||||
|
||||
|
||||
def _generate_review_guidance(
|
||||
impact: dict, changed_files: list[str]
|
||||
) -> str:
|
||||
"""Generate review guidance based on the impact analysis."""
|
||||
guidance_parts = []
|
||||
|
||||
# Check for test coverage
|
||||
changed_funcs = [
|
||||
n for n in impact["changed_nodes"] if n.kind == "Function"
|
||||
]
|
||||
test_edges = [e for e in impact["edges"] if e.kind == "TESTED_BY"]
|
||||
tested_funcs = {e.source_qualified for e in test_edges}
|
||||
|
||||
untested = [
|
||||
f for f in changed_funcs
|
||||
if f.qualified_name not in tested_funcs and not f.is_test
|
||||
]
|
||||
if untested:
|
||||
guidance_parts.append(
|
||||
f"- {len(untested)} changed function(s) lack test coverage: "
|
||||
+ ", ".join(n.name for n in untested[:5])
|
||||
)
|
||||
|
||||
# Check for wide blast radius
|
||||
if len(impact["impacted_nodes"]) > 20:
|
||||
guidance_parts.append(
|
||||
f"- Wide blast radius: {len(impact['impacted_nodes'])} "
|
||||
"nodes impacted. "
|
||||
"Review callers and dependents carefully."
|
||||
)
|
||||
|
||||
# Check for inheritance changes
|
||||
inheritance_edges = [
|
||||
e for e in impact["edges"]
|
||||
if e.kind in ("INHERITS", "IMPLEMENTS")
|
||||
]
|
||||
if inheritance_edges:
|
||||
guidance_parts.append(
|
||||
f"- {len(inheritance_edges)} inheritance/implementation "
|
||||
"relationship(s) affected. "
|
||||
"Check for Liskov substitution violations."
|
||||
)
|
||||
|
||||
# Check for cross-file impact
|
||||
impacted_file_count = len(impact["impacted_files"])
|
||||
if impacted_file_count > 3:
|
||||
guidance_parts.append(
|
||||
f"- Changes impact {impacted_file_count} other files."
|
||||
" Consider splitting into smaller PRs."
|
||||
)
|
||||
|
||||
if not guidance_parts:
|
||||
guidance_parts.append(
|
||||
"- Changes appear well-contained with minimal blast radius."
|
||||
)
|
||||
|
||||
return "\n".join(guidance_parts)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Tool 12: get_affected_flows [REVIEW]
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def get_affected_flows_func(
|
||||
changed_files: list[str] | None = None,
|
||||
base: str = "HEAD~1",
|
||||
repo_root: str | None = None,
|
||||
) -> dict[str, Any]:
|
||||
"""Find execution flows affected by changed files.
|
||||
|
||||
[REVIEW] Identifies which execution flows pass through nodes in the
|
||||
changed files. Useful during code review to understand which user-facing
|
||||
or critical paths are affected by a change.
|
||||
|
||||
Args:
|
||||
changed_files: List of changed file paths (relative to repo root).
|
||||
Auto-detected from git diff if omitted.
|
||||
base: Git ref for auto-detecting changes (default: HEAD~1).
|
||||
repo_root: Repository root path. Auto-detected if omitted.
|
||||
|
||||
Returns:
|
||||
Affected flows sorted by criticality, with step details.
|
||||
"""
|
||||
store, root = _get_store(repo_root)
|
||||
try:
|
||||
if changed_files is None:
|
||||
changed_files = get_changed_files(root, base)
|
||||
if not changed_files:
|
||||
changed_files = get_staged_and_unstaged(root)
|
||||
|
||||
if not changed_files:
|
||||
return {
|
||||
"status": "ok",
|
||||
"summary": "No changed files detected.",
|
||||
"affected_flows": [],
|
||||
"total": 0,
|
||||
}
|
||||
|
||||
# Convert to absolute paths for graph lookup
|
||||
abs_files = [str(root / f) for f in changed_files]
|
||||
result = _get_affected_flows(store, abs_files)
|
||||
|
||||
total = result["total"]
|
||||
out = {
|
||||
"status": "ok",
|
||||
"summary": (
|
||||
f"{total} flow(s) affected by changes "
|
||||
f"in {len(changed_files)} file(s)"
|
||||
),
|
||||
"changed_files": changed_files,
|
||||
"affected_flows": result["affected_flows"],
|
||||
"total": total,
|
||||
}
|
||||
out["_hints"] = generate_hints(
|
||||
"get_affected_flows", out, get_session()
|
||||
)
|
||||
return out
|
||||
except Exception as exc:
|
||||
return {"status": "error", "error": str(exc)}
|
||||
finally:
|
||||
store.close()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Tool 16: detect_changes [REVIEW]
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def detect_changes_func(
|
||||
base: str = "HEAD~1",
|
||||
changed_files: list[str] | None = None,
|
||||
include_source: bool = False,
|
||||
max_depth: int = 2,
|
||||
repo_root: str | None = None,
|
||||
detail_level: str = "standard",
|
||||
) -> dict[str, Any]:
|
||||
"""Detect changes and produce risk-scored review guidance.
|
||||
|
||||
[REVIEW] Primary tool for code review. Maps git diffs to affected
|
||||
functions, flows, communities, and test coverage gaps. Returns
|
||||
priority-ordered review guidance with risk scores.
|
||||
|
||||
Args:
|
||||
base: Git ref to diff against (default: HEAD~1).
|
||||
changed_files: Explicit list of changed file paths (relative to repo
|
||||
root). Auto-detected from git diff if omitted.
|
||||
include_source: If True, include source code snippets for changed
|
||||
functions. Default: False.
|
||||
max_depth: Impact radius depth for BFS traversal. Default: 2.
|
||||
repo_root: Repository root path. Auto-detected if omitted.
|
||||
detail_level: Output detail level. "standard" returns full analysis;
|
||||
"minimal" returns only summary, risk_score, changed_file_count,
|
||||
test_gap_count, and top 3 review priorities (text only).
|
||||
Default: "standard".
|
||||
|
||||
Returns:
|
||||
Risk-scored analysis with changed functions, affected flows,
|
||||
test gaps, and review priorities.
|
||||
"""
|
||||
store, root = _get_store(repo_root)
|
||||
try:
|
||||
# Detect changed files if not provided.
|
||||
if changed_files is None:
|
||||
changed_files = get_changed_files(root, base)
|
||||
if not changed_files:
|
||||
changed_files = get_staged_and_unstaged(root)
|
||||
|
||||
if not changed_files:
|
||||
return {
|
||||
"status": "ok",
|
||||
"summary": "No changed files detected.",
|
||||
"risk_score": 0.0,
|
||||
"changed_functions": [],
|
||||
"affected_flows": [],
|
||||
"test_gaps": [],
|
||||
"review_priorities": [],
|
||||
}
|
||||
|
||||
original_tokens = estimate_file_tokens(root, changed_files)
|
||||
|
||||
# Convert to absolute paths for graph lookup.
|
||||
abs_files = [str(root / f) for f in changed_files]
|
||||
|
||||
# Parse diff ranges for line-level mapping.
|
||||
diff_ranges = parse_diff_ranges(str(root), base)
|
||||
# Remap to absolute paths so they match graph file_paths.
|
||||
abs_ranges: dict[str, list[tuple[int, int]]] = {}
|
||||
for rel_path, ranges in diff_ranges.items():
|
||||
abs_path = str(root / rel_path)
|
||||
abs_ranges[abs_path] = ranges
|
||||
|
||||
analysis = analyze_changes(
|
||||
store,
|
||||
changed_files=abs_files,
|
||||
changed_ranges=abs_ranges if abs_ranges else None,
|
||||
repo_root=str(root),
|
||||
base=base,
|
||||
)
|
||||
|
||||
# Optionally include source snippets for changed functions.
|
||||
if include_source:
|
||||
for func in analysis.get("changed_functions", []):
|
||||
fp = func.get("file_path")
|
||||
ls = func.get("line_start")
|
||||
le = func.get("line_end")
|
||||
if fp and ls and le:
|
||||
file_path = Path(fp)
|
||||
if file_path.is_file():
|
||||
try:
|
||||
lines = file_path.read_text(
|
||||
errors="replace"
|
||||
).splitlines()
|
||||
start = max(0, ls - 1)
|
||||
end = min(len(lines), le)
|
||||
func["source"] = "\n".join(
|
||||
f"{i + 1}: {lines[i]}"
|
||||
for i in range(start, end)
|
||||
)
|
||||
except (OSError, UnicodeDecodeError):
|
||||
func["source"] = "(could not read file)"
|
||||
|
||||
if detail_level == "minimal":
|
||||
priorities = analysis.get("review_priorities", [])
|
||||
top_priorities = [
|
||||
p.get("name", p.get("qualified_name", ""))
|
||||
for p in priorities[:3]
|
||||
]
|
||||
result: dict[str, Any] = {
|
||||
"status": "ok",
|
||||
"summary": analysis.get("summary", ""),
|
||||
"risk_score": analysis.get("risk_score", 0.0),
|
||||
"changed_file_count": len(changed_files),
|
||||
"test_gap_count": len(analysis.get("test_gaps", [])),
|
||||
"review_priorities": top_priorities,
|
||||
}
|
||||
else:
|
||||
result = {
|
||||
"status": "ok",
|
||||
"changed_files": changed_files,
|
||||
**analysis,
|
||||
}
|
||||
result["_hints"] = generate_hints(
|
||||
"detect_changes", result, get_session()
|
||||
)
|
||||
attach_context_savings(result, original_tokens=original_tokens)
|
||||
return result
|
||||
except Exception as exc:
|
||||
return {"status": "error", "error": str(exc)}
|
||||
finally:
|
||||
store.close()
|
||||
@@ -0,0 +1,257 @@
|
||||
"""TypeScript tsconfig.json path alias resolver.
|
||||
|
||||
Resolves TypeScript path aliases (e.g., ``@/ -> src/``) declared in
|
||||
``compilerOptions.paths`` so that ``IMPORTS_FROM`` edges can point to
|
||||
real file paths instead of raw alias strings.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
import re
|
||||
from pathlib import Path
|
||||
from typing import Optional
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Extensions probed when resolving an alias target
|
||||
_PROBE_EXTENSIONS = [".ts", ".tsx", ".js", ".jsx", ".vue"]
|
||||
|
||||
# Tsconfig filenames to look for when walking up the directory tree
|
||||
_TSCONFIG_NAMES = ["tsconfig.json", "tsconfig.app.json"]
|
||||
|
||||
|
||||
class TsconfigResolver:
|
||||
"""Resolves TypeScript path aliases (e.g., @/ -> src/) using tsconfig.json."""
|
||||
|
||||
def __init__(self) -> None:
|
||||
self._cache: dict[str, Optional[dict]] = {}
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Public API
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def resolve_alias(self, import_str: str, file_path: str) -> Optional[str]:
|
||||
"""Resolve a TS path alias to an absolute file path, or None."""
|
||||
try:
|
||||
config = self._load_tsconfig_for_file(file_path)
|
||||
if config is None:
|
||||
return None
|
||||
|
||||
base_url: Optional[str] = config.get("baseUrl")
|
||||
paths: dict[str, list[str]] = config.get("paths", {})
|
||||
tsconfig_dir: str = config.get("_tsconfig_dir", "")
|
||||
|
||||
if not paths:
|
||||
return None
|
||||
|
||||
if base_url:
|
||||
base_dir = (Path(tsconfig_dir) / base_url).resolve()
|
||||
else:
|
||||
base_dir = Path(tsconfig_dir).resolve()
|
||||
|
||||
return self._match_and_probe(import_str, paths, base_dir)
|
||||
except (OSError, ValueError, TypeError):
|
||||
logger.debug(
|
||||
"TsconfigResolver: unexpected error for %s", file_path, exc_info=True,
|
||||
)
|
||||
return None
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Internal helpers
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def _load_tsconfig_for_file(self, file_path: str) -> Optional[dict]:
|
||||
"""Find and load tsconfig.json for the given file."""
|
||||
start_dir = Path(file_path).parent.resolve()
|
||||
current = start_dir
|
||||
visited: list[str] = []
|
||||
|
||||
while True:
|
||||
dir_str = str(current)
|
||||
if dir_str in self._cache:
|
||||
result = self._cache[dir_str]
|
||||
for visited_dir in visited:
|
||||
self._cache[visited_dir] = result
|
||||
return result
|
||||
|
||||
visited.append(dir_str)
|
||||
|
||||
for name in _TSCONFIG_NAMES:
|
||||
candidate = current / name
|
||||
if candidate.is_file():
|
||||
config = self._parse_tsconfig(candidate)
|
||||
config["_tsconfig_dir"] = dir_str
|
||||
for visited_dir in visited:
|
||||
self._cache[visited_dir] = config
|
||||
return config
|
||||
|
||||
parent = current.parent
|
||||
if parent == current:
|
||||
for visited_dir in visited:
|
||||
self._cache[visited_dir] = None
|
||||
return None
|
||||
current = parent
|
||||
|
||||
def _parse_tsconfig(self, tsconfig_path: Path) -> dict:
|
||||
"""Parse a tsconfig.json file (supports JSONC comments)."""
|
||||
seen: set[str] = set()
|
||||
return self._resolve_extends(tsconfig_path, seen)
|
||||
|
||||
def _resolve_extends(self, tsconfig_path: Path, seen: set[str]) -> dict:
|
||||
"""Recursively resolve the tsconfig extends chain."""
|
||||
canonical = str(tsconfig_path.resolve())
|
||||
if canonical in seen:
|
||||
logger.debug("TsconfigResolver: cycle detected at %s", canonical)
|
||||
return {}
|
||||
seen = seen | {canonical}
|
||||
|
||||
try:
|
||||
raw = tsconfig_path.read_text(encoding="utf-8", errors="replace")
|
||||
except OSError:
|
||||
logger.debug("TsconfigResolver: cannot read %s", tsconfig_path)
|
||||
return {}
|
||||
|
||||
stripped = self._strip_jsonc_comments(raw)
|
||||
try:
|
||||
data: dict = json.loads(stripped)
|
||||
except json.JSONDecodeError:
|
||||
logger.debug("TsconfigResolver: invalid JSON in %s", tsconfig_path)
|
||||
return {}
|
||||
|
||||
result: dict = {}
|
||||
|
||||
extends: Optional[str] = data.get("extends")
|
||||
if extends and isinstance(extends, str) and extends.startswith("."):
|
||||
parent_path = (tsconfig_path.parent / extends).resolve()
|
||||
if not parent_path.suffix:
|
||||
parent_path = parent_path.with_suffix(".json")
|
||||
if parent_path.is_file():
|
||||
parent_config = self._resolve_extends(parent_path, seen)
|
||||
parent_opts = parent_config.get("compilerOptions", {})
|
||||
result.setdefault("compilerOptions", {}).update(parent_opts)
|
||||
|
||||
child_opts: dict = data.get("compilerOptions", {})
|
||||
result.setdefault("compilerOptions", {}).update(child_opts)
|
||||
|
||||
compiler_options = result.get("compilerOptions", {})
|
||||
if "baseUrl" in compiler_options:
|
||||
result["baseUrl"] = compiler_options["baseUrl"]
|
||||
if "paths" in compiler_options:
|
||||
result["paths"] = compiler_options["paths"]
|
||||
|
||||
return result
|
||||
|
||||
def _strip_jsonc_comments(self, text: str) -> str:
|
||||
"""Remove // and /* */ comments and trailing commas from JSONC."""
|
||||
result: list[str] = []
|
||||
i = 0
|
||||
n = len(text)
|
||||
|
||||
while i < n:
|
||||
ch = text[i]
|
||||
|
||||
if ch == '"':
|
||||
result.append(ch)
|
||||
i += 1
|
||||
while i < n:
|
||||
c = text[i]
|
||||
result.append(c)
|
||||
if c == "\\" and i + 1 < n:
|
||||
i += 1
|
||||
result.append(text[i])
|
||||
elif c == '"':
|
||||
break
|
||||
i += 1
|
||||
i += 1
|
||||
continue
|
||||
|
||||
if ch == "/" and i + 1 < n and text[i + 1] == "*":
|
||||
i += 2
|
||||
while i < n - 1:
|
||||
if text[i] == "*" and text[i + 1] == "/":
|
||||
i += 2
|
||||
break
|
||||
i += 1
|
||||
else:
|
||||
i = n
|
||||
continue
|
||||
|
||||
if ch == "/" and i + 1 < n and text[i + 1] == "/":
|
||||
i += 2
|
||||
while i < n and text[i] != "\n":
|
||||
i += 1
|
||||
continue
|
||||
|
||||
result.append(ch)
|
||||
i += 1
|
||||
|
||||
stripped = "".join(result)
|
||||
stripped = re.sub(r",\s*([\]}])", r"\1", stripped)
|
||||
return stripped
|
||||
|
||||
def _match_and_probe(
|
||||
self,
|
||||
import_str: str,
|
||||
paths: dict[str, list[str]],
|
||||
base_dir: Path,
|
||||
) -> Optional[str]:
|
||||
"""Match import_str against alias patterns and probe the filesystem."""
|
||||
def _pattern_specificity(item: tuple[str, list[str]]) -> int:
|
||||
pat = item[0]
|
||||
return len(pat.partition("*")[0])
|
||||
|
||||
sorted_paths = sorted(paths.items(), key=_pattern_specificity, reverse=True)
|
||||
|
||||
for pattern, replacements in sorted_paths:
|
||||
suffix = _match_pattern(pattern, import_str)
|
||||
if suffix is None:
|
||||
continue
|
||||
|
||||
for replacement in replacements:
|
||||
if "*" in replacement:
|
||||
mapped = replacement.replace("*", suffix, 1)
|
||||
else:
|
||||
mapped = replacement
|
||||
|
||||
candidate_base = (base_dir / mapped).resolve()
|
||||
found = _probe_path(candidate_base)
|
||||
if found:
|
||||
return str(found)
|
||||
|
||||
return None
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Module-level helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _match_pattern(pattern: str, import_str: str) -> Optional[str]:
|
||||
"""Return the wildcard-matched suffix if pattern matches import_str."""
|
||||
if "*" not in pattern:
|
||||
return "" if import_str == pattern else None
|
||||
|
||||
prefix, _, suffix_pat = pattern.partition("*")
|
||||
if not (import_str.startswith(prefix) and import_str.endswith(suffix_pat)):
|
||||
return None
|
||||
|
||||
end = len(import_str) - len(suffix_pat) if suffix_pat else len(import_str)
|
||||
return import_str[len(prefix):end]
|
||||
|
||||
|
||||
def _probe_path(base: Path) -> Optional[Path]:
|
||||
"""Probe base and base + extensions for an existing file."""
|
||||
if base.is_file():
|
||||
return base
|
||||
for ext in _PROBE_EXTENSIONS:
|
||||
candidate = base.with_suffix(ext) if not base.suffix else Path(str(base) + ext)
|
||||
if candidate.is_file():
|
||||
return candidate
|
||||
if base.is_dir():
|
||||
for ext in _PROBE_EXTENSIONS:
|
||||
candidate = base / f"index{ext}"
|
||||
if candidate.is_file():
|
||||
return candidate
|
||||
return None
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,305 @@
|
||||
"""Wiki generation from community structure.
|
||||
|
||||
Generates markdown pages for each detected community and an index page,
|
||||
providing a navigable documentation wiki for the codebase architecture.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import re
|
||||
import sqlite3
|
||||
from collections import Counter
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from .communities import get_communities
|
||||
from .flows import get_flows
|
||||
from .graph import GraphStore, _sanitize_name
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _slugify(name: str) -> str:
|
||||
"""Convert a community name to a safe filename slug."""
|
||||
slug = re.sub(r"[^a-z0-9]+", "-", name.lower()).strip("-")
|
||||
return slug[:80] or "unnamed"
|
||||
|
||||
|
||||
def _generate_community_page(store: GraphStore, community: dict[str, Any]) -> str:
|
||||
"""Build markdown content for a single community.
|
||||
|
||||
Includes: heading, overview (size, cohesion, language), members table
|
||||
(top 50), execution flows through the community, and dependencies.
|
||||
|
||||
Args:
|
||||
store: The graph store.
|
||||
community: Community dict from get_communities().
|
||||
|
||||
Returns:
|
||||
Markdown string for the community page.
|
||||
"""
|
||||
name = community["name"]
|
||||
size = community["size"]
|
||||
cohesion = community.get("cohesion", 0.0)
|
||||
lang = community.get("dominant_language", "")
|
||||
description = community.get("description", "")
|
||||
|
||||
lines: list[str] = []
|
||||
lines.append(f"# {name}")
|
||||
lines.append("")
|
||||
|
||||
# Overview section
|
||||
lines.append("## Overview")
|
||||
lines.append("")
|
||||
if description:
|
||||
lines.append(f"{description}")
|
||||
lines.append("")
|
||||
lines.append(f"- **Size**: {size} nodes")
|
||||
lines.append(f"- **Cohesion**: {cohesion:.4f}")
|
||||
if lang:
|
||||
lines.append(f"- **Dominant Language**: {lang}")
|
||||
lines.append("")
|
||||
|
||||
# Members table (top 50)
|
||||
member_qns = community.get("members", [])
|
||||
lines.append("## Members")
|
||||
lines.append("")
|
||||
if member_qns:
|
||||
lines.append("| Name | Kind | File | Lines |")
|
||||
lines.append("|------|------|------|-------|")
|
||||
|
||||
# Fetch node details for members (limit to 50)
|
||||
member_count = 0
|
||||
for qn in member_qns[:50]:
|
||||
node = store.get_node(qn)
|
||||
if node and node.kind != "File":
|
||||
node_name = _sanitize_name(node.name)
|
||||
lines.append(
|
||||
f"| {node_name} | {node.kind} | {node.file_path} "
|
||||
f"| {node.line_start}-{node.line_end} |"
|
||||
)
|
||||
member_count += 1
|
||||
|
||||
if not member_count:
|
||||
# Remove the table headers if no members were added
|
||||
lines.pop() # header separator
|
||||
lines.pop() # header
|
||||
lines.append("No non-file members found.")
|
||||
|
||||
if len(member_qns) > 50:
|
||||
lines.append("")
|
||||
lines.append(f"*... and {len(member_qns) - 50} more members.*")
|
||||
else:
|
||||
lines.append("No members found.")
|
||||
lines.append("")
|
||||
|
||||
# Execution flows through community
|
||||
lines.append("## Execution Flows")
|
||||
lines.append("")
|
||||
member_set = set(member_qns)
|
||||
try:
|
||||
all_flows = get_flows(store, sort_by="criticality", limit=200)
|
||||
community_flows: list[dict] = []
|
||||
for flow in all_flows:
|
||||
# Check if this flow passes through any community member
|
||||
flow_qns = store.get_flow_qualified_names(flow["id"])
|
||||
if flow_qns & member_set:
|
||||
community_flows.append(flow)
|
||||
|
||||
if community_flows:
|
||||
for flow in community_flows[:10]:
|
||||
flow_name = _sanitize_name(flow.get("name", "unnamed"))
|
||||
criticality = flow.get("criticality", 0.0)
|
||||
depth = flow.get("depth", 0)
|
||||
lines.append(
|
||||
f"- **{flow_name}** (criticality: {criticality:.2f}, depth: {depth})"
|
||||
)
|
||||
if len(community_flows) > 10:
|
||||
lines.append(f"- *... and {len(community_flows) - 10} more flows.*")
|
||||
else:
|
||||
lines.append("No execution flows pass through this community.")
|
||||
except sqlite3.OperationalError as exc:
|
||||
logger.debug("wiki: flows table unavailable: %s", exc)
|
||||
lines.append("Execution flow data not available.")
|
||||
lines.append("")
|
||||
|
||||
# Dependencies (cross-community edges)
|
||||
lines.append("## Dependencies")
|
||||
lines.append("")
|
||||
try:
|
||||
outgoing_targets: Counter[str] = Counter()
|
||||
incoming_sources: Counter[str] = Counter()
|
||||
if member_qns:
|
||||
qns = list(member_qns)
|
||||
|
||||
# Outgoing: source is a member
|
||||
for t in store.get_outgoing_targets(qns):
|
||||
if t not in member_set:
|
||||
outgoing_targets[t] += 1
|
||||
|
||||
# Incoming: target is a member
|
||||
for s in store.get_incoming_sources(qns):
|
||||
if s not in member_set:
|
||||
incoming_sources[s] += 1
|
||||
|
||||
if outgoing_targets:
|
||||
lines.append("### Outgoing")
|
||||
lines.append("")
|
||||
for target, count in outgoing_targets.most_common(15):
|
||||
lines.append(f"- `{_sanitize_name(target)}` ({count} edge(s))")
|
||||
lines.append("")
|
||||
|
||||
if incoming_sources:
|
||||
lines.append("### Incoming")
|
||||
lines.append("")
|
||||
for source, count in incoming_sources.most_common(15):
|
||||
lines.append(f"- `{_sanitize_name(source)}` ({count} edge(s))")
|
||||
lines.append("")
|
||||
|
||||
if not outgoing_targets and not incoming_sources:
|
||||
lines.append("No cross-community dependencies detected.")
|
||||
lines.append("")
|
||||
except sqlite3.OperationalError as exc:
|
||||
logger.debug("wiki: dependency edges unavailable: %s", exc)
|
||||
lines.append("Dependency data not available.")
|
||||
lines.append("")
|
||||
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
def generate_wiki(
|
||||
store: GraphStore,
|
||||
wiki_dir: str | Path,
|
||||
force: bool = False,
|
||||
) -> dict[str, Any]:
|
||||
"""Generate a markdown wiki from the community structure.
|
||||
|
||||
For each community, generates a markdown page. Also generates an
|
||||
index.md with links to all community pages.
|
||||
|
||||
Args:
|
||||
store: The graph store.
|
||||
wiki_dir: Directory to write wiki pages into.
|
||||
force: If True, regenerate all pages even if content unchanged.
|
||||
|
||||
Returns:
|
||||
Dict with pages_generated, pages_updated, pages_unchanged counts.
|
||||
"""
|
||||
wiki_path = Path(wiki_dir)
|
||||
wiki_path.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
communities = get_communities(store)
|
||||
|
||||
pages_generated = 0
|
||||
pages_updated = 0
|
||||
pages_unchanged = 0
|
||||
|
||||
page_entries: list[tuple[str, str, int]] = [] # (slug, name, size)
|
||||
|
||||
# Track slugs we've already used in THIS run so two communities that
|
||||
# slugify to the same filename don't overwrite each other (#222 follow-up).
|
||||
# Previously "Data Processing" and "data processing" both became
|
||||
# "data-processing.md", causing silent data loss and inflated "updated"
|
||||
# counters (each collision was counted as an update while only one file
|
||||
# made it to disk).
|
||||
used_slugs: set[str] = set()
|
||||
|
||||
for comm in communities:
|
||||
name = comm["name"]
|
||||
base_slug = _slugify(name)
|
||||
slug = base_slug
|
||||
suffix = 2
|
||||
while slug in used_slugs:
|
||||
slug = f"{base_slug}-{suffix}"
|
||||
suffix += 1
|
||||
used_slugs.add(slug)
|
||||
|
||||
filename = f"{slug}.md"
|
||||
filepath = wiki_path / filename
|
||||
|
||||
content = _generate_community_page(store, comm)
|
||||
|
||||
if filepath.exists() and not force:
|
||||
existing = filepath.read_text(encoding="utf-8", errors="replace")
|
||||
if existing == content:
|
||||
pages_unchanged += 1
|
||||
page_entries.append((slug, name, comm["size"]))
|
||||
continue
|
||||
|
||||
already_existed = filepath.exists()
|
||||
filepath.write_text(content, encoding="utf-8")
|
||||
if already_existed:
|
||||
pages_updated += 1
|
||||
else:
|
||||
pages_generated += 1
|
||||
page_entries.append((slug, name, comm["size"]))
|
||||
|
||||
# Generate index.md
|
||||
index_lines: list[str] = []
|
||||
index_lines.append("# Code Wiki")
|
||||
index_lines.append("")
|
||||
index_lines.append(
|
||||
"Auto-generated documentation from the code knowledge graph community structure."
|
||||
)
|
||||
index_lines.append("")
|
||||
index_lines.append(f"**Total communities**: {len(communities)}")
|
||||
index_lines.append("")
|
||||
index_lines.append("## Communities")
|
||||
index_lines.append("")
|
||||
index_lines.append("| Community | Size | Link |")
|
||||
index_lines.append("|-----------|------|------|")
|
||||
for slug, name, size in sorted(page_entries, key=lambda x: x[1]):
|
||||
index_lines.append(f"| {name} | {size} | [{slug}.md]({slug}.md) |")
|
||||
index_lines.append("")
|
||||
|
||||
index_content = "\n".join(index_lines)
|
||||
index_path = wiki_path / "index.md"
|
||||
|
||||
if index_path.exists() and not force:
|
||||
existing_index = index_path.read_text(encoding="utf-8", errors="replace")
|
||||
if existing_index == index_content:
|
||||
pages_unchanged += 1
|
||||
else:
|
||||
index_path.write_text(index_content, encoding="utf-8")
|
||||
pages_updated += 1
|
||||
else:
|
||||
index_path.write_text(index_content, encoding="utf-8")
|
||||
pages_generated += 1
|
||||
|
||||
return {
|
||||
"pages_generated": pages_generated,
|
||||
"pages_updated": pages_updated,
|
||||
"pages_unchanged": pages_unchanged,
|
||||
}
|
||||
|
||||
|
||||
def get_wiki_page(wiki_dir: str | Path, page_name: str) -> str | None:
|
||||
"""Retrieve a specific wiki page by community name.
|
||||
|
||||
Args:
|
||||
wiki_dir: Directory containing wiki pages.
|
||||
page_name: Community name (will be slugified for filename lookup).
|
||||
|
||||
Returns:
|
||||
Page content as a string, or None if the page does not exist.
|
||||
"""
|
||||
wiki_path = Path(wiki_dir)
|
||||
slug = _slugify(page_name)
|
||||
filepath = wiki_path / f"{slug}.md"
|
||||
|
||||
if filepath.is_file():
|
||||
return filepath.read_text(encoding="utf-8", errors="replace")
|
||||
|
||||
# Fallback: try exact filename match — with path traversal protection
|
||||
exact_path = (wiki_path / page_name).resolve()
|
||||
if exact_path.is_file() and exact_path.is_relative_to(wiki_path.resolve()):
|
||||
return exact_path.read_text(encoding="utf-8", errors="replace")
|
||||
|
||||
# Fallback: search for partial match
|
||||
if wiki_path.is_dir():
|
||||
for p in wiki_path.iterdir():
|
||||
if p.suffix == ".md" and slug in p.stem:
|
||||
return p.read_text(encoding="utf-8", errors="replace")
|
||||
|
||||
return None
|
||||
Reference in New Issue
Block a user