chore: import upstream snapshot with attribution

This commit is contained in:
wehub-resource-sync
2026-07-13 12:42:18 +08:00
commit 05f60106aa
288 changed files with 76871 additions and 0 deletions
+20
View File
@@ -0,0 +1,20 @@
"""Code Review Graph - MCP server for persistent incremental code knowledge graphs."""
from .context_savings import (
attach_context_savings,
estimate_context_savings,
estimate_file_tokens,
estimate_tokens,
format_context_savings,
)
__version__ = "2.3.6"
__all__ = [
"__version__",
"attach_context_savings",
"estimate_context_savings",
"estimate_file_tokens",
"estimate_tokens",
"format_context_savings",
]
+4
View File
@@ -0,0 +1,4 @@
"""Allow running as: python -m code_review_graph"""
from .cli import main
main()
+410
View File
@@ -0,0 +1,410 @@
"""Graph analysis: hub detection, bridge nodes, knowledge gaps,
surprise scoring, suggested questions."""
from __future__ import annotations
import logging
from collections import Counter, defaultdict
from .graph import GraphStore, _sanitize_name
logger = logging.getLogger(__name__)
def find_hub_nodes(store: GraphStore, top_n: int = 10) -> list[dict]:
"""Find the most connected nodes (highest in+out degree), excluding File nodes.
Returns list of dicts with: name, qualified_name, kind, file,
in_degree, out_degree, total_degree, community_id
"""
# Build degree counts from all edges
edges = store.get_all_edges()
in_degree: dict[str, int] = Counter()
out_degree: dict[str, int] = Counter()
for e in edges:
out_degree[e.source_qualified] += 1
in_degree[e.target_qualified] += 1
# Get all non-File nodes
nodes = store.get_all_nodes(exclude_files=True)
community_map = store.get_all_community_ids()
scored = []
for n in nodes:
qn = n.qualified_name
ind = in_degree.get(qn, 0)
outd = out_degree.get(qn, 0)
total = ind + outd
if total == 0:
continue
scored.append({
"name": _sanitize_name(n.name),
"qualified_name": n.qualified_name,
"kind": n.kind,
"file": n.file_path,
"in_degree": ind,
"out_degree": outd,
"total_degree": total,
"community_id": community_map.get(qn),
})
scored.sort(
key=lambda x: x.get("total_degree", 0), # type: ignore[arg-type,return-value]
reverse=True,
)
return scored[:top_n]
def find_bridge_nodes(
store: GraphStore, top_n: int = 10
) -> list[dict]:
"""Find nodes with highest betweenness centrality.
These are architectural chokepoints that sit on shortest paths
between many node pairs. If they break, multiple communities
lose connectivity.
Returns list of dicts with: name, qualified_name, kind, file,
betweenness, community_id
"""
import networkx as nx
# Build the graph — use cached version if available
nxg = store._build_networkx_graph()
# Compute betweenness centrality (approximate for large graphs)
n_nodes = nxg.number_of_nodes()
if n_nodes > 5000:
# Sample-based approximation for large graphs
k = min(500, n_nodes)
bc = nx.betweenness_centrality(nxg, k=k, normalized=True)
elif n_nodes > 0:
bc = nx.betweenness_centrality(nxg, normalized=True)
else:
return []
community_map = store.get_all_community_ids()
node_map = {
n.qualified_name: n
for n in store.get_all_nodes(exclude_files=True)
}
results = []
for qn, score in bc.items():
if score <= 0 or qn not in node_map:
continue
n = node_map[qn]
if n.kind == "File":
continue
results.append({
"name": _sanitize_name(n.name),
"qualified_name": n.qualified_name,
"kind": n.kind,
"file": n.file_path,
"betweenness": round(score, 6),
"community_id": community_map.get(qn),
})
results.sort(
key=lambda x: float(x.get("betweenness", 0)), # type: ignore[arg-type,return-value]
reverse=True,
)
return results[:top_n]
def find_knowledge_gaps(store: GraphStore) -> dict[str, list[dict]]:
"""Identify structural weaknesses in the codebase graph.
Returns dict with categories:
- isolated_nodes: degree <= 1, disconnected from graph
- thin_communities: fewer than 3 members
- untested_hotspots: high-degree nodes with no TESTED_BY edges
- single_file_communities: entire community in one file
"""
edges = store.get_all_edges()
nodes = store.get_all_nodes(exclude_files=True)
community_map = store.get_all_community_ids()
# Build degree map
degree: dict[str, int] = Counter()
tested_nodes: set[str] = set()
for e in edges:
degree[e.source_qualified] += 1
degree[e.target_qualified] += 1
if e.kind == "TESTED_BY":
tested_nodes.add(e.source_qualified)
# 1. Isolated nodes (degree <= 1, not File)
isolated = []
for n in nodes:
d = degree.get(n.qualified_name, 0)
if d <= 1:
isolated.append({
"name": _sanitize_name(n.name),
"qualified_name": n.qualified_name,
"kind": n.kind,
"file": n.file_path,
"degree": d,
})
# 2. Build community sizes and file maps from node data
comm_sizes: Counter[int] = Counter()
comm_files: dict[int, set[str]] = defaultdict(set)
for n in nodes:
cid = community_map.get(n.qualified_name)
if cid is not None:
comm_sizes[cid] += 1
comm_files[cid].add(n.file_path)
# Thin communities (< 3 members)
communities = store.get_communities_list()
thin = []
for c in communities:
cid = int(c["id"])
size = comm_sizes.get(cid, 0)
if size < 3:
thin.append({
"community_id": cid,
"name": str(c["name"]),
"size": size,
})
# 3. Untested hotspots (degree >= 5, no TESTED_BY)
untested_hotspots = []
for n in nodes:
d = degree.get(n.qualified_name, 0)
if (d >= 5
and n.qualified_name not in tested_nodes
and not n.is_test):
untested_hotspots.append({
"name": _sanitize_name(n.name),
"qualified_name": n.qualified_name,
"kind": n.kind,
"file": n.file_path,
"degree": d,
})
untested_hotspots.sort(
key=lambda x: x.get("degree", 0), # type: ignore[arg-type,return-value]
reverse=True,
)
# 4. Single-file communities
single_file = []
for c in communities:
cid = int(c["id"])
files = comm_files.get(cid, set())
size = comm_sizes.get(cid, 0)
if len(files) == 1 and size >= 3:
single_file.append({
"community_id": cid,
"name": str(c["name"]),
"size": size,
"file": next(iter(files)),
})
return {
"isolated_nodes": isolated[:50],
"thin_communities": thin,
"untested_hotspots": untested_hotspots[:20],
"single_file_communities": single_file,
}
def find_surprising_connections(
store: GraphStore, top_n: int = 15
) -> list[dict]:
"""Find edges with high surprise scores.
Detects unexpected architectural coupling based on:
- Cross-community: source and target in different communities
- Cross-language: different file languages
- Peripheral-to-hub: low-degree node to high-degree node
- Cross-file-type: test calling production or vice versa
- Non-standard edge kind for the node types
"""
edges = store.get_all_edges()
nodes = store.get_all_nodes(exclude_files=True)
community_map = store.get_all_community_ids()
node_map = {n.qualified_name: n for n in nodes}
# Build degree map
degree: dict[str, int] = Counter()
for e in edges:
degree[e.source_qualified] += 1
degree[e.target_qualified] += 1
# Median degree for peripheral detection
degrees = [d for d in degree.values() if d > 0]
if not degrees:
return []
median_deg = sorted(degrees)[len(degrees) // 2]
high_deg_threshold = max(median_deg * 3, 10)
scored_edges = []
for e in edges:
src = node_map.get(e.source_qualified)
tgt = node_map.get(e.target_qualified)
if not src or not tgt:
continue
if src.kind == "File" or tgt.kind == "File":
continue
score = 0.0
reasons = []
# Cross-community (+0.3)
src_cid = community_map.get(e.source_qualified)
tgt_cid = community_map.get(e.target_qualified)
if (src_cid is not None
and tgt_cid is not None
and src_cid != tgt_cid):
score += 0.3
reasons.append("cross-community")
# Cross-language (+0.2)
src_lang = (
src.file_path.rsplit(".", 1)[-1]
if "." in src.file_path else ""
)
tgt_lang = (
tgt.file_path.rsplit(".", 1)[-1]
if "." in tgt.file_path else ""
)
if src_lang and tgt_lang and src_lang != tgt_lang:
score += 0.2
reasons.append("cross-language")
# Peripheral-to-hub (+0.2)
src_deg = degree.get(e.source_qualified, 0)
tgt_deg = degree.get(e.target_qualified, 0)
if ((src_deg <= 2 and tgt_deg >= high_deg_threshold)
or (tgt_deg <= 2
and src_deg >= high_deg_threshold)):
score += 0.2
reasons.append("peripheral-to-hub")
# Cross-file-type: test <-> non-test (+0.15)
if src.is_test != tgt.is_test and e.kind == "CALLS":
score += 0.15
reasons.append("cross-test-boundary")
# Non-standard edge kind (+0.15)
if e.kind == "CALLS" and src.kind == "Type":
score += 0.15
reasons.append("unusual-edge-kind")
if score > 0:
scored_edges.append({
"source": _sanitize_name(src.name),
"source_qualified": e.source_qualified,
"target": _sanitize_name(tgt.name),
"target_qualified": e.target_qualified,
"edge_kind": e.kind,
"surprise_score": round(score, 2),
"reasons": reasons,
"source_community": src_cid,
"target_community": tgt_cid,
})
scored_edges.sort(
key=lambda x: float(x.get("surprise_score", 0)), # type: ignore[arg-type,return-value]
reverse=True,
)
return scored_edges[:top_n]
def generate_suggested_questions(
store: GraphStore,
) -> list[dict]:
"""Auto-generate review questions from graph analysis.
Categories:
- bridge_node: Why does X connect communities A and B?
- isolated_node: Is X dead code or dynamically invoked?
- low_cohesion: Should community X be split?
- hub_risk: Does hub node X have adequate test coverage?
- surprising: Why does A call B across community boundary?
"""
questions = []
# Bridge node questions
bridges = find_bridge_nodes(store, top_n=3)
for b in bridges:
questions.append({
"category": "bridge_node",
"question": (
f"'{b['name']}' is a critical connector "
f"between multiple code regions. Is it "
f"adequately tested and documented?"
),
"target": b["qualified_name"],
"priority": "high",
})
# Hub risk questions
hubs = find_hub_nodes(store, top_n=3)
edges = store.get_all_edges()
tested = {
e.source_qualified
for e in edges if e.kind == "TESTED_BY"
}
for h in hubs:
if h["qualified_name"] not in tested:
questions.append({
"category": "hub_risk",
"question": (
f"Hub node '{h['name']}' has "
f"{h['total_degree']} connections but no "
f"direct test coverage. Should it be "
f"tested?"
),
"target": h["qualified_name"],
"priority": "high",
})
# Surprising connection questions
surprises = find_surprising_connections(store, top_n=3)
for s in surprises:
if "cross-community" in s["reasons"]:
questions.append({
"category": "surprising_connection",
"question": (
f"'{s['source']}' (community "
f"{s['source_community']}) calls "
f"'{s['target']}' (community "
f"{s['target_community']}). Is this "
f"coupling intentional?"
),
"target": s["source_qualified"],
"priority": "medium",
})
# Knowledge gap questions
gaps = find_knowledge_gaps(store)
for c in gaps["thin_communities"][:2]:
questions.append({
"category": "thin_community",
"question": (
f"Community '{c['name']}' has only "
f"{c['size']} member(s). Should it be "
f"merged with a neighbor?"
),
"target": f"community:{c['community_id']}",
"priority": "low",
})
for h in gaps["untested_hotspots"][:2]:
questions.append({
"category": "untested_hotspot",
"question": (
f"'{h['name']}' has {h['degree']} "
f"connections but no test coverage. "
f"Is this a risk?"
),
"target": h["qualified_name"],
"priority": "medium",
})
return questions
+409
View File
@@ -0,0 +1,409 @@
"""Change impact analysis for code review.
Maps git/svn diffs to affected functions, flows, communities, and test coverage
gaps. Produces risk-scored, priority-ordered review guidance.
"""
from __future__ import annotations
import logging
import os
import re
import subprocess
from pathlib import Path
from typing import Any
from .constants import SECURITY_KEYWORDS as _SECURITY_KEYWORDS
from .flows import get_affected_flows
from .graph import GraphNode, GraphStore, _sanitize_name, node_to_dict
logger = logging.getLogger(__name__)
_GIT_TIMEOUT = int(os.environ.get("CRG_GIT_TIMEOUT", "30")) # seconds, configurable
_SAFE_GIT_REF = re.compile(r"^[A-Za-z0-9_.~^/@{}\-]+$")
_SAFE_SVN_REV = re.compile(r"^r?\d+(:r?\d+|:HEAD|:BASE|:COMMITTED)?$", re.IGNORECASE)
# ---------------------------------------------------------------------------
# 1. parse_git_diff_ranges / parse_svn_diff_ranges
# ---------------------------------------------------------------------------
def parse_git_diff_ranges(
repo_root: str,
base: str = "HEAD~1",
) -> dict[str, list[tuple[int, int]]]:
"""Run ``git diff --unified=0`` and extract changed line ranges per file.
Args:
repo_root: Absolute path to the repository root.
base: Git ref to diff against (default: ``HEAD~1``).
Returns:
Mapping of file paths to lists of ``(start_line, end_line)`` tuples.
Returns an empty dict on error.
"""
if not _SAFE_GIT_REF.match(base):
logger.warning("Invalid git ref rejected: %s", base)
return {}
try:
result = subprocess.run(
["git", "diff", "--unified=0", base, "--"],
capture_output=True,
stdin=subprocess.DEVNULL,
text=True,
encoding="utf-8",
errors="replace",
cwd=repo_root,
timeout=_GIT_TIMEOUT,
)
if result.returncode != 0:
logger.warning("git diff failed (rc=%d): %s", result.returncode, result.stderr[:200])
return {}
except (OSError, subprocess.SubprocessError) as exc:
logger.warning("git diff error: %s", exc)
return {}
return _parse_unified_diff(result.stdout)
def parse_svn_diff_ranges(
repo_root: str,
rev_range: str | None = None,
) -> dict[str, list[tuple[int, int]]]:
"""Run ``svn diff`` and extract changed line ranges per file.
Args:
repo_root: Absolute path to the SVN working copy root.
rev_range: Optional SVN revision range in ``rXXX:HEAD`` format.
When *None*, diffs the working copy against BASE (local changes).
Returns:
Mapping of file paths to lists of ``(start_line, end_line)`` tuples.
Returns an empty dict on error.
"""
cmd = ["svn", "diff", "--non-interactive"]
if rev_range:
if not _SAFE_SVN_REV.match(rev_range):
logger.warning("Invalid SVN revision range rejected: %s", rev_range)
return {}
cmd.extend(["-r", rev_range])
try:
result = subprocess.run(
cmd,
capture_output=True,
stdin=subprocess.DEVNULL,
text=True,
encoding="utf-8",
errors="replace",
cwd=repo_root,
timeout=_GIT_TIMEOUT,
)
if result.returncode != 0:
logger.warning("svn diff failed (rc=%d): %s", result.returncode, result.stderr[:200])
return {}
except (OSError, subprocess.SubprocessError) as exc:
logger.warning("svn diff error: %s", exc)
return {}
return _parse_unified_diff(result.stdout)
def parse_diff_ranges(
repo_root: str,
base: str = "HEAD~1",
) -> dict[str, list[tuple[int, int]]]:
"""Auto-detect VCS and return changed line ranges per file.
Dispatches to :func:`parse_git_diff_ranges` for Git repositories and
:func:`parse_svn_diff_ranges` for SVN working copies.
Args:
repo_root: Absolute path to the repository/working-copy root.
base: For Git: the ref to diff against (default ``HEAD~1``).
For SVN: an optional revision range (e.g. ``"r100:HEAD"``);
when *base* is not a valid SVN revision, working-copy changes
(``svn diff``) are used instead.
"""
root_path = Path(repo_root)
if (root_path / ".svn").exists():
rev_range = base if _SAFE_SVN_REV.match(base) else None
return parse_svn_diff_ranges(repo_root, rev_range)
return parse_git_diff_ranges(repo_root, base)
def _parse_unified_diff(diff_text: str) -> dict[str, list[tuple[int, int]]]:
"""Parse unified diff output into file -> line-range mappings.
Handles the ``@@ -old,count +new,count @@`` hunk header format.
"""
ranges: dict[str, list[tuple[int, int]]] = {}
current_file: str | None = None
# Match "+++ b/path/to/file"
file_pattern = re.compile(r"^\+\+\+ b/(.+)$")
# Match "@@ ... +start,count @@" or "@@ ... +start @@"
hunk_pattern = re.compile(r"^@@ .+? \+(\d+)(?:,(\d+))? @@")
for line in diff_text.splitlines():
file_match = file_pattern.match(line)
if file_match:
current_file = file_match.group(1)
continue
hunk_match = hunk_pattern.match(line)
if hunk_match and current_file is not None:
start = int(hunk_match.group(1))
count = int(hunk_match.group(2)) if hunk_match.group(2) else 1
if count == 0:
# Pure deletion hunk (no lines added); still note the position.
end = start
else:
end = start + count - 1
ranges.setdefault(current_file, []).append((start, end))
return ranges
# ---------------------------------------------------------------------------
# 2. map_changes_to_nodes
# ---------------------------------------------------------------------------
def map_changes_to_nodes(
store: GraphStore,
changed_ranges: dict[str, list[tuple[int, int]]],
) -> list[GraphNode]:
"""Find graph nodes whose line ranges overlap the changed lines.
Args:
store: The graph store.
changed_ranges: Mapping of file paths to ``(start, end)`` tuples.
Returns:
Deduplicated list of overlapping graph nodes.
"""
seen: set[str] = set()
result: list[GraphNode] = []
for file_path, ranges in changed_ranges.items():
# Try the path as-is, then also try all nodes to match relative paths.
nodes = store.get_nodes_by_file(file_path)
if not nodes:
# The graph may store absolute paths; try a suffix match.
matched_paths = store.get_files_matching(file_path)
for mp in matched_paths:
nodes.extend(store.get_nodes_by_file(mp))
for node in nodes:
if node.qualified_name in seen:
continue
if node.line_start is None or node.line_end is None:
continue
# Check overlap with any changed range.
for start, end in ranges:
if node.line_start <= end and node.line_end >= start:
result.append(node)
seen.add(node.qualified_name)
break
return result
# ---------------------------------------------------------------------------
# 3. compute_risk_score
# ---------------------------------------------------------------------------
def compute_risk_score(store: GraphStore, node: GraphNode) -> float:
"""Compute a risk score (0.0 - 1.0) for a single node.
Scoring factors:
- Flow participation: 0.05 per flow membership, capped at 0.25
- Community crossing: 0.05 per caller from a different community, capped at 0.15
- Test coverage: 0.30 (untested) scaling down to 0.05 (5+ TESTED_BY edges)
- Security sensitivity: 0.20 if name matches security keywords
- Caller count: callers / 20, capped at 0.10
"""
score = 0.0
# --- Flow participation (cap 0.25), weighted by criticality ---
flow_criticalities = store.get_flow_criticalities_for_node(node.id)
if flow_criticalities:
score += min(sum(flow_criticalities), 0.25)
else:
flow_count = store.count_flow_memberships(node.id)
score += min(flow_count * 0.05, 0.25)
# --- Community crossing (cap 0.15) ---
callers = store.get_edges_by_target(node.qualified_name)
caller_edges = [e for e in callers if e.kind == "CALLS"]
cross_community = 0
node_cid = store.get_node_community_id(node.id)
if node_cid is not None and caller_edges:
caller_qns = [edge.source_qualified for edge in caller_edges]
cid_map = store.get_community_ids_by_qualified_names(caller_qns)
for cid in cid_map.values():
if cid is not None and cid != node_cid:
cross_community += 1
score += min(cross_community * 0.05, 0.15)
# --- Test coverage (direct + transitive) ---
transitive_tests = store.get_transitive_tests(node.qualified_name)
test_count = len(transitive_tests)
score += 0.30 - (min(test_count / 5.0, 1.0) * 0.25)
# --- Security sensitivity ---
name_lower = node.name.lower()
qn_lower = node.qualified_name.lower()
if any(kw in name_lower or kw in qn_lower for kw in _SECURITY_KEYWORDS):
score += 0.20
# --- Caller count (cap 0.10) ---
caller_count = len(caller_edges)
score += min(caller_count / 20.0, 0.10)
return round(min(max(score, 0.0), 1.0), 4)
# ---------------------------------------------------------------------------
# 4. analyze_changes
# ---------------------------------------------------------------------------
def analyze_changes(
store: GraphStore,
changed_files: list[str],
changed_ranges: dict[str, list[tuple[int, int]]] | None = None,
repo_root: str | None = None,
base: str = "HEAD~1",
) -> dict[str, Any]:
"""Analyze changes and produce risk-scored review guidance.
Args:
store: The graph store.
changed_files: List of changed file paths.
changed_ranges: Optional pre-parsed diff ranges. If not provided and
``repo_root`` is given, they are computed via the detected VCS
(Git or SVN).
repo_root: Repository root (for git/svn diff).
base: Git ref or SVN revision range to diff against.
Returns:
Dict with ``summary``, ``risk_score``, ``changed_functions``,
``affected_flows``, ``test_gaps``, and ``review_priorities``.
"""
# Compute changed ranges if not provided.
if changed_ranges is None and repo_root is not None:
# Diff keys are forward-slash paths relative to the repo root, but
# the graph stores absolute native paths. Remap so lookups work on
# Windows, where the LIKE-suffix fallback cannot bridge
# "src/app.py" to "C:\repo\src\app.py" (#528). Keys that are
# already absolute pass through pathlib joining unchanged. The
# explicit changed_ranges path (MCP) is untouched — tools/review.py
# remaps before calling, and remapping twice would corrupt keys.
root_path = Path(repo_root)
changed_ranges = {
str(root_path / key): ranges
for key, ranges in parse_diff_ranges(repo_root, base).items()
}
# Map changes to nodes.
if changed_ranges:
changed_nodes = map_changes_to_nodes(store, changed_ranges)
else:
# Fallback: all nodes in changed files.
changed_nodes = []
for fp in changed_files:
changed_nodes.extend(store.get_nodes_by_file(fp))
# Filter to functions/tests for risk scoring (skip File nodes).
changed_funcs = [
n for n in changed_nodes
if n.kind in ("Function", "Test", "Class")
]
# Cap to prevent O(N*M) query explosion on large PRs.
_max_funcs = int(os.environ.get("CRG_MAX_CHANGED_FUNCS", "500"))
funcs_truncated = len(changed_funcs) > _max_funcs
if funcs_truncated:
changed_funcs = changed_funcs[:_max_funcs]
# Compute per-node risk scores.
node_risks: list[dict[str, Any]] = []
for node in changed_funcs:
risk = compute_risk_score(store, node)
node_risks.append({
**node_to_dict(node),
"risk_score": risk,
})
# Overall risk score: max of individual risks, or 0.
overall_risk = max((nr["risk_score"] for nr in node_risks), default=0.0)
# Affected flows.
affected = get_affected_flows(store, changed_files)
# Detect test gaps: changed functions without TESTED_BY edges.
test_gaps: list[dict[str, Any]] = []
for node in changed_funcs:
if node.is_test:
continue
tested = store.get_edges_by_target(node.qualified_name)
if not any(e.kind == "TESTED_BY" for e in tested):
test_gaps.append({
"name": _sanitize_name(node.name),
"qualified_name": _sanitize_name(node.qualified_name),
"file": node.file_path,
"line_start": node.line_start,
"line_end": node.line_end,
})
# Review priorities: top 10 by risk score.
review_priorities = sorted(node_risks, key=lambda x: x["risk_score"], reverse=True)[:10]
# Build summary.
summary_parts = [
f"Analyzed {len(changed_files)} changed file(s):",
f" - {len(changed_funcs)} changed function(s)/class(es)",
f" - {affected['total']} affected flow(s)",
f" - {len(test_gaps)} test gap(s)",
f" - Overall risk score: {overall_risk:.2f}",
]
if test_gaps:
# Dedup by bare name in the human summary. The underlying test_gaps
# list keeps every entry (a downstream consumer needs precision via
# qualified_name), but a graph that ended up with the same function
# stored under two qualified_names (e.g. relative + absolute path
# variants) would otherwise print "X, X, Y, Y" — surfacing graph
# corruption as a UX bug. The root cause is path normalization;
# this is the defensive last line.
seen_names: set[str] = set()
gap_names: list[str] = []
for g in test_gaps:
n = g["name"]
if n in seen_names:
continue
seen_names.add(n)
gap_names.append(n)
if len(gap_names) >= 5:
break
summary_parts.append(f" - Untested: {', '.join(gap_names)}")
if funcs_truncated:
summary_parts.append(
f" - Warning: analysis capped at {_max_funcs} functions "
f"(set CRG_MAX_CHANGED_FUNCS to adjust)"
)
return {
"summary": "\n".join(summary_parts),
"risk_score": overall_risk,
"changed_functions": node_risks,
"affected_flows": affected["affected_flows"],
"test_gaps": test_gaps,
"review_priorities": review_priorities,
"functions_truncated": funcs_truncated,
}
File diff suppressed because it is too large Load Diff
+874
View File
@@ -0,0 +1,874 @@
"""Community/cluster detection for the code knowledge graph.
Detects communities of related code nodes using the Leiden algorithm (via igraph,
optional) with a file-based grouping fallback when igraph is not installed.
"""
from __future__ import annotations
import logging
import random
import re
from collections import Counter, defaultdict
from typing import Any
from .graph import GraphEdge, GraphNode, GraphStore, _sanitize_name
# Fixed seed for igraph's RNG so Leiden community detection is reproducible
# across runs. Without this, two builds of the same graph produce different
# community IDs / sizes, breaking benchmark comparability. Override with
# CRG_LEIDEN_SEED env var if you need a different seed.
_LEIDEN_SEED = 42
logger = logging.getLogger(__name__)
# Stay well under SQLite's default 999-variable limit per statement.
_SQL_BATCH = 450
# ---------------------------------------------------------------------------
# Optional igraph import
# ---------------------------------------------------------------------------
try:
import igraph as ig # type: ignore[import-untyped]
IGRAPH_AVAILABLE = True
except ImportError:
ig = None # type: ignore[assignment]
IGRAPH_AVAILABLE = False
# ---------------------------------------------------------------------------
# Edge weight mapping
# ---------------------------------------------------------------------------
EDGE_WEIGHTS: dict[str, float] = {
"CALLS": 1.0,
"IMPORTS_FROM": 0.5,
"INHERITS": 0.8,
"IMPLEMENTS": 0.7,
"CONTAINS": 0.3,
"TESTED_BY": 0.4,
"DEPENDS_ON": 0.6,
}
# Common words to filter when generating community names
_COMMON_WORDS = frozenset({
"get", "set", "self", "init", "new", "create", "update", "delete",
"add", "remove", "make", "build", "from", "to", "for", "with",
"the", "and", "test", "main", "run", "do", "is", "has", "on",
"of", "in", "at", "by", "my", "this", "that", "all", "none",
})
# ---------------------------------------------------------------------------
# Community naming
# ---------------------------------------------------------------------------
def _generate_community_name(members: list[GraphNode]) -> str:
"""Generate a meaningful name for a community of nodes.
Algorithm:
1. Find most common module/file prefix among members
2. If a dominant class exists (>40% of nodes), use its name
3. Fallback: most frequent keyword in function/class names
4. Format: "{prefix}-{keyword}"
"""
if not members:
return "empty"
# 1. Find common file prefix
file_paths = [m.file_path for m in members]
prefix = _extract_file_prefix(file_paths)
# 2. Check for dominant class
class_names = [m.name for m in members if m.kind == "Class"]
if class_names:
class_counts = Counter(class_names)
top_class, top_count = class_counts.most_common(1)[0]
if top_count > len(members) * 0.4:
if prefix:
return f"{prefix}-{_to_slug(top_class)}"
return _to_slug(top_class)
# 3. Most frequent keyword from function/class names
keywords = _extract_keywords(members)
keyword = keywords[0] if keywords else ""
if prefix and keyword:
return f"{prefix}-{keyword}"
if prefix:
return prefix
if keyword:
return keyword
return "cluster"
def _extract_file_prefix(file_paths: list[str]) -> str:
"""Find the most common short directory or module name from file paths."""
if not file_paths:
return ""
# Extract the parent directory or file stem
parts: list[str] = []
for fp in file_paths:
# Use the last directory component or file stem
segments = fp.replace("\\", "/").split("/")
# Take the parent dir if it exists, otherwise the file stem
if len(segments) >= 2:
parts.append(segments[-2])
else:
stem = segments[-1].rsplit(".", 1)[0]
parts.append(stem)
counts = Counter(parts)
top_part, _ = counts.most_common(1)[0]
return _to_slug(top_part)
def _extract_keywords(members: list[GraphNode]) -> list[str]:
"""Extract the most frequent meaningful keywords from member names."""
word_counts: Counter[str] = Counter()
for m in members:
if m.kind in ("Function", "Class", "Test", "Type"):
words = _split_name(m.name)
for w in words:
wl = w.lower()
if wl not in _COMMON_WORDS and len(wl) > 1:
word_counts[wl] += 1
if not word_counts:
return []
return [w for w, _ in word_counts.most_common(5)]
def _split_name(name: str) -> list[str]:
"""Split a camelCase or snake_case name into words."""
# Insert boundary before uppercase letters for camelCase
s = re.sub(r"([a-z])([A-Z])", r"\1_\2", name)
# Split on underscores, hyphens, dots
return [p for p in re.split(r"[_\-.\s]+", s) if p]
def _to_slug(s: str) -> str:
"""Convert a string to a short lowercase slug."""
return re.sub(r"[^a-z0-9]+", "-", s.lower()).strip("-")[:30]
# ---------------------------------------------------------------------------
# Cohesion calculation
# ---------------------------------------------------------------------------
def _compute_cohesion_batch(
community_member_qns: list[set[str]],
all_edges: list[GraphEdge],
) -> list[float]:
"""Compute cohesion for multiple communities in a single O(edges) pass.
Builds a ``qualified_name -> community_index`` reverse map (each node
appears in at most one community since all callers produce partitions),
then walks every edge exactly once, bucketing it into internal/external
counters per community.
Total work: O(edges + sum(|members|)) instead of
O(edges * communities) for naive per-community cohesion.
Returns a list of cohesion scores aligned with ``community_member_qns``.
"""
qn_to_idx: dict[str, int] = {}
for idx, members in enumerate(community_member_qns):
for qn in members:
qn_to_idx[qn] = idx
n = len(community_member_qns)
internal = [0] * n
external = [0] * n
for e in all_edges:
sc = qn_to_idx.get(e.source_qualified)
tc = qn_to_idx.get(e.target_qualified)
if sc is None and tc is None:
continue
if sc == tc:
# Safe: sc is not None here (sc == tc and not both None).
assert sc is not None
internal[sc] += 1
else:
if sc is not None:
external[sc] += 1
if tc is not None:
external[tc] += 1
results: list[float] = []
for i in range(n):
total = internal[i] + external[i]
results.append(internal[i] / total if total > 0 else 0.0)
return results
def _build_adjacency(edges: list[GraphEdge]) -> dict[str, list[str]]:
"""Build adjacency list from edges (one pass over all edges)."""
adj: dict[str, list[str]] = defaultdict(list)
for e in edges:
adj[e.source_qualified].append(e.target_qualified)
adj[e.target_qualified].append(e.source_qualified)
return adj
def _compute_cohesion(
member_qns: set[str],
all_edges: list[GraphEdge],
adj: dict[str, list[str]] | None = None,
) -> float:
"""Compute cohesion: internal_edges / (internal_edges + external_edges).
For multiple communities, prefer :func:`_compute_cohesion_batch`, which
runs in O(edges) total instead of O(edges) per community.
"""
return _compute_cohesion_batch([member_qns], all_edges)[0]
# ---------------------------------------------------------------------------
# Leiden-based community detection (igraph)
# ---------------------------------------------------------------------------
def _detect_leiden(
nodes: list[GraphNode],
edges: list[GraphEdge],
min_size: int,
adj: dict[str, list[str]] | None = None,
) -> list[dict[str, Any]]:
"""Detect communities using Leiden algorithm via igraph.
Caps Leiden at ``n_iterations=2`` (sufficient for code dependency graphs)
and skips the recursive sub-community splitting pass that caused
exponential blow-up on large repos (>100k nodes).
"""
if ig is None:
return []
qn_to_idx: dict[str, int] = {}
idx_to_node: dict[int, GraphNode] = {}
for i, node in enumerate(nodes):
qn_to_idx[node.qualified_name] = i
idx_to_node[i] = node
if not qn_to_idx:
return []
logger.info("Building igraph with %d nodes...", len(qn_to_idx))
g = ig.Graph(n=len(qn_to_idx), directed=False)
edge_list: list[tuple[int, int]] = []
weights: list[float] = []
seen_edges: set[tuple[int, int]] = set()
for e in edges:
src_idx = qn_to_idx.get(e.source_qualified)
tgt_idx = qn_to_idx.get(e.target_qualified)
if src_idx is not None and tgt_idx is not None and src_idx != tgt_idx:
pair = (min(src_idx, tgt_idx), max(src_idx, tgt_idx))
if pair not in seen_edges:
seen_edges.add(pair)
edge_list.append(pair)
weights.append(EDGE_WEIGHTS.get(e.kind, 0.5))
if not edge_list:
return _detect_file_based(nodes, edges, min_size, adj=adj)
g.add_edges(edge_list)
g.es["weight"] = weights
# Run Leiden -- scale resolution inversely with graph size to get
# coarser clusters on large repos. Default resolution=1.0 produces
# thousands of tiny communities for 30k+ node graphs.
import math
n_nodes = g.vcount()
resolution = max(0.05, 1.0 / math.log10(max(n_nodes, 10)))
logger.info(
"Running Leiden on %d nodes, %d edges...",
g.vcount(), g.ecount(),
)
import os
seed = int(os.environ.get("CRG_LEIDEN_SEED", _LEIDEN_SEED))
# Deterministic seeding for benchmark reproducibility — community
# detection is not a security-sensitive context. nosec B311.
ig.set_random_number_generator(random.Random(seed)) # nosec B311
partition = g.community_leiden(
objective_function="modularity",
weights="weight",
resolution=resolution,
n_iterations=2,
)
logger.info(
"Leiden complete, found %d partitions. Computing cohesion...",
len(partition),
)
pending: list[tuple[list[GraphNode], set[str]]] = []
for cluster_ids in partition:
if len(cluster_ids) < min_size:
continue
members = [idx_to_node[i] for i in cluster_ids if i in idx_to_node]
if len(members) < min_size:
continue
member_qns = {m.qualified_name for m in members}
pending.append((members, member_qns))
cohesions = _compute_cohesion_batch([p[1] for p in pending], edges)
communities: list[dict[str, Any]] = []
for (members, member_qns), cohesion in zip(pending, cohesions):
lang_counts = Counter(m.language for m in members if m.language)
dominant_lang = lang_counts.most_common(1)[0][0] if lang_counts else ""
name = _generate_community_name(members)
communities.append({
"name": name,
"level": 0,
"size": len(members),
"cohesion": round(cohesion, 4),
"dominant_language": dominant_lang,
"description": f"Community of {len(members)} nodes",
"members": [m.qualified_name for m in members],
"member_qns": member_qns,
})
logger.info("Community detection complete: %d communities", len(communities))
return communities
# ---------------------------------------------------------------------------
# File-based fallback community detection
# ---------------------------------------------------------------------------
def _detect_file_based(
nodes: list[GraphNode],
edges: list[GraphEdge],
min_size: int,
adj: dict[str, list[str]] | None = None,
) -> list[dict[str, Any]]:
"""Group nodes by directory when Leiden is unavailable or over-fragments.
Strips the longest common directory prefix from all file paths, then
adaptively picks a grouping depth that yields 10-200 communities.
"""
# Collect all directory paths (normalized, without filename)
all_dir_parts: list[list[str]] = []
for n in nodes:
parts = n.file_path.replace("\\", "/").split("/")
all_dir_parts.append([p for p in parts[:-1] if p])
# Find the longest common prefix among directory parts
prefix_len = 0
if all_dir_parts:
shortest = min(len(p) for p in all_dir_parts)
for i in range(shortest):
seg = all_dir_parts[0][i]
if all(p[i] == seg for p in all_dir_parts):
prefix_len = i + 1
else:
break
def _group_at_depth(depth: int) -> dict[str, list[GraphNode]]:
groups: dict[str, list[GraphNode]] = defaultdict(list)
for n in nodes:
parts = n.file_path.replace("\\", "/").split("/")
dir_parts = [p for p in parts[:-1] if p]
remainder = dir_parts[prefix_len:]
if remainder:
key = "/".join(remainder[:depth])
else:
key = parts[-1].rsplit(".", 1)[0] if parts else "root"
groups[key].append(n)
return groups
# Try increasing depths until we get 10-200 qualifying groups
max_depth = max((len(p) - prefix_len for p in all_dir_parts), default=0)
best_groups = _group_at_depth(1) # depth=1 always works (file stem fallback)
for depth in range(1, max_depth + 1):
groups = _group_at_depth(depth)
qualifying = sum(1 for v in groups.values() if len(v) >= min_size)
best_groups = groups
if qualifying >= 10:
break
by_dir = best_groups
# Pre-filter to communities meeting min_size and collect their member
# sets so we can batch-compute all cohesions in a single O(edges) pass.
# Without this, per-community cohesion is O(edges * files), which makes
# community detection effectively hang on large repos.
pending: list[tuple[str, list[GraphNode], set[str]]] = []
for dir_path, members in by_dir.items():
if len(members) < min_size:
continue
member_qns = {m.qualified_name for m in members}
pending.append((dir_path, members, member_qns))
cohesions = _compute_cohesion_batch([p[2] for p in pending], edges)
communities: list[dict[str, Any]] = []
for (dir_path, members, member_qns), cohesion in zip(pending, cohesions):
lang_counts = Counter(m.language for m in members if m.language)
dominant_lang = lang_counts.most_common(1)[0][0] if lang_counts else ""
name = _generate_community_name(members)
communities.append({
"name": name,
"level": 0,
"size": len(members),
"cohesion": round(cohesion, 4),
"dominant_language": dominant_lang,
"description": f"Directory-based community: {dir_path}",
"members": [m.qualified_name for m in members],
"member_qns": member_qns,
})
return communities
# ---------------------------------------------------------------------------
# Oversized community splitting
# ---------------------------------------------------------------------------
def _split_oversized(
communities: list[dict],
nodes: list[GraphNode],
edges: list[GraphEdge],
threshold_pct: float = 0.25,
min_split_size: int = 10,
) -> list[dict]:
"""Recursively split communities that exceed threshold_pct of total.
Uses Leiden on the subgraph of oversized communities. If igraph is
not available, returns communities unchanged.
"""
if not IGRAPH_AVAILABLE:
return communities
total = sum(
c.get("size", len(c.get("members", [])))
for c in communities
)
if total == 0:
return communities
threshold = max(int(total * threshold_pct), min_split_size)
result: list[dict] = []
next_id = max(
(c.get("id", 0) for c in communities), default=0
) + 1
for comm in communities:
members = set(comm.get("members", []))
if len(members) <= threshold:
result.append(comm)
continue
# Build subgraph for this community
member_nodes = [
n for n in nodes
if n.qualified_name in members
]
member_edges = [
e for e in edges
if (
e.source_qualified in members
and e.target_qualified in members
)
]
if len(member_nodes) < min_split_size:
result.append(comm)
continue
# Run Leiden on subgraph
qn_to_idx = {
n.qualified_name: i
for i, n in enumerate(member_nodes)
}
ig_edges: list[tuple[int, int]] = []
ig_weights: list[float] = []
for e in member_edges:
si = qn_to_idx.get(e.source_qualified)
ti = qn_to_idx.get(e.target_qualified)
if si is not None and ti is not None and si != ti:
ig_edges.append((si, ti))
ig_weights.append(
EDGE_WEIGHTS.get(e.kind, 0.5)
)
if not ig_edges:
result.append(comm)
continue
try:
g = ig.Graph(
n=len(member_nodes),
edges=ig_edges,
directed=False,
)
g.es["weight"] = ig_weights
import os
seed = int(os.environ.get("CRG_LEIDEN_SEED", _LEIDEN_SEED))
# Deterministic seeding for benchmark reproducibility — community
# detection is not a security-sensitive context. nosec B311.
ig.set_random_number_generator(random.Random(seed)) # nosec B311
partition = g.community_leiden(
objective_function="modularity",
weights="weight",
resolution=0.5,
)
sub_communities: dict[int, list[str]] = {}
for idx, cid in enumerate(partition.membership):
sub_communities.setdefault(cid, []).append(
member_nodes[idx].qualified_name
)
if len(sub_communities) <= 1:
result.append(comm)
continue
parent_id = comm.get("id", 0)
comm_name = comm.get("name", "")
for sub_members in sub_communities.values():
sub_comm = {
"id": next_id,
"name": comm_name + f"-sub{next_id}",
"level": comm.get("level", 0) + 1,
"parent_id": parent_id,
"members": sub_members,
"size": len(sub_members),
"cohesion": 0.0,
"dominant_language": comm.get(
"dominant_language"
),
"description": (
f"Split from {comm_name}"
),
}
result.append(sub_comm)
next_id += 1
logger.info(
"Split oversized community '%s' "
"(%d members) into %d",
comm_name,
len(members),
len(sub_communities),
)
except Exception:
logger.warning(
"Failed to split community '%s', "
"keeping as-is",
comm.get("name", ""),
exc_info=True,
)
result.append(comm)
return result
# ---------------------------------------------------------------------------
# Public API
# ---------------------------------------------------------------------------
def detect_communities(
store: GraphStore, min_size: int = 2
) -> list[dict[str, Any]]:
"""Detect communities in the code graph.
Uses the Leiden algorithm via igraph if available, otherwise falls back to
file-based grouping.
Args:
store: The GraphStore instance.
min_size: Minimum number of nodes for a community to be included.
Returns:
List of community dicts with keys: name, level, size, cohesion,
dominant_language, description, members, member_qns.
"""
# Gather all nodes (exclude File nodes to focus on code entities)
all_edges = store.get_all_edges()
unique_nodes = store.get_all_nodes(exclude_files=True)
# Build adjacency index once for fast cohesion computation
adj = _build_adjacency(all_edges)
logger.info(
"Loaded %d unique nodes, %d edges",
len(unique_nodes), len(all_edges),
)
if IGRAPH_AVAILABLE:
logger.info("Detecting communities with Leiden algorithm (igraph)")
results = _detect_leiden(unique_nodes, all_edges, min_size, adj=adj)
else:
logger.info("igraph not available, using file-based community detection")
results = _detect_file_based(unique_nodes, all_edges, min_size, adj=adj)
# Split oversized communities
results = _split_oversized(
results, unique_nodes, all_edges,
)
# Convert member_qns (internal set) to a list for serialization safety,
# then strip it from the returned dicts to avoid leaking internal state.
for comm in results:
if "member_qns" in comm:
comm["member_qns"] = list(comm["member_qns"])
del comm["member_qns"]
return results
def incremental_detect_communities(
store: GraphStore,
changed_files: list[str],
min_size: int = 2,
) -> int:
"""Re-detect communities only if changed files affect existing communities.
If no existing communities contain nodes from changed files, skips
re-detection entirely (the common case for small changes). Otherwise
re-runs full community detection.
Args:
store: The GraphStore instance.
changed_files: List of file paths that have changed.
min_size: Minimum number of nodes for a community to be included.
Returns:
Number of communities detected, or 0 if skipped.
"""
if not changed_files:
return 0
conn = store._conn
# Check if any communities are affected (batch to stay under SQLite limit)
affected_count = 0
for i in range(0, len(changed_files), _SQL_BATCH):
batch = changed_files[i:i + _SQL_BATCH]
placeholders = ",".join("?" * len(batch))
row = conn.execute(
f"SELECT COUNT(DISTINCT community_id) FROM nodes " # nosec B608
f"WHERE community_id IS NOT NULL AND file_path IN ({placeholders})",
batch,
).fetchone()
if row:
affected_count += row[0]
affected = (affected_count,) if affected_count else None
if not affected or affected[0] == 0:
return 0 # No communities affected, skip
# Re-run full community detection (correct and fast enough)
communities = detect_communities(store, min_size=min_size)
return store_communities(store, communities)
def store_communities(
store: GraphStore, communities: list[dict[str, Any]]
) -> int:
"""Store detected communities in the database.
Clears existing communities and community_id assignments, then inserts
the new communities and updates node community_id references.
Args:
store: The GraphStore instance.
communities: List of community dicts from detect_communities().
Returns:
Number of communities stored.
"""
# NOTE: store_communities uses _conn directly because it performs
# multi-statement batch writes (DELETE + INSERT loop + UPDATE loop)
# that are tightly coupled to the DB transaction lifecycle.
conn = store._conn
if conn.in_transaction:
logger.warning("Rolling back uncommitted transaction before BEGIN IMMEDIATE")
conn.rollback()
# Wrap in explicit transaction so the DELETE + INSERT + UPDATE
# sequence is atomic — no partial community data on crash.
conn.execute("BEGIN IMMEDIATE")
try:
conn.execute("DELETE FROM communities")
conn.execute("UPDATE nodes SET community_id = NULL")
count = 0
for comm in communities:
cursor = conn.execute(
"""INSERT INTO communities
(name, level, cohesion, size, dominant_language, description)
VALUES (?, ?, ?, ?, ?, ?)""",
(
comm["name"],
comm.get("level", 0),
comm.get("cohesion", 0.0),
comm["size"],
comm.get("dominant_language", ""),
comm.get("description", ""),
),
)
community_id = cursor.lastrowid
# Batch update community_id on member nodes
member_qns = comm.get("members", [])
for j in range(0, len(member_qns), _SQL_BATCH):
batch = member_qns[j:j + _SQL_BATCH]
placeholders = ",".join("?" * len(batch))
conn.execute(
f"UPDATE nodes SET community_id = ? WHERE qualified_name IN ({placeholders})", # nosec B608
[community_id] + batch,
)
count += 1
conn.commit()
except BaseException:
conn.rollback()
raise
return count
def get_communities(
store: GraphStore, sort_by: str = "size", min_size: int = 0
) -> list[dict[str, Any]]:
"""Retrieve stored communities from the database.
Args:
store: The GraphStore instance.
sort_by: Column to sort by ("size", "cohesion", "name").
min_size: Minimum community size to include.
Returns:
List of community dicts.
"""
valid_sorts = {"size", "cohesion", "name"}
if sort_by not in valid_sorts:
sort_by = "size"
order = "DESC" if sort_by in ("size", "cohesion") else "ASC"
# NOTE: get_communities reads the communities table which has no
# dedicated GraphStore method (it's a domain-specific table managed
# entirely by the communities module). We use _conn for this query.
rows = store._conn.execute(
f"SELECT * FROM communities WHERE size >= ? ORDER BY {sort_by} {order}", # nosec B608
(min_size,),
).fetchall()
communities: list[dict[str, Any]] = []
for row in rows:
# Fetch member qualified names for this community
member_qns = [
_sanitize_name(qn)
for qn in store.get_community_member_qns(row["id"])
]
communities.append({
"id": row["id"],
"name": _sanitize_name(row["name"]),
"level": row["level"],
"cohesion": row["cohesion"],
"size": row["size"],
"dominant_language": row["dominant_language"] or "",
"description": _sanitize_name(row["description"] or ""),
"members": member_qns,
})
return communities
_TEST_COMMUNITY_RE = re.compile(
r"(^test[-/]|[-/]test([:/]|$)|it:should|describe:|spec[-/]|[-/]spec$)",
re.IGNORECASE,
)
def _is_test_community(name: str) -> bool:
"""Return True if a community name indicates it is test-dominated."""
return bool(_TEST_COMMUNITY_RE.search(name))
def get_architecture_overview(store: GraphStore) -> dict[str, Any]:
"""Generate an architecture overview based on community structure.
Builds a node-to-community mapping, counts cross-community edges,
and generates warnings for high coupling.
Args:
store: The GraphStore instance.
Returns:
Dict with keys: communities, cross_community_edges, warnings.
"""
communities = get_communities(store)
# Build node -> community_id mapping
node_to_community: dict[str, int] = {}
for comm in communities:
comm_id = comm.get("id", 0)
for qn in comm.get("members", []):
node_to_community[qn] = comm_id
# Count cross-community edges
all_edges = store.get_all_edges()
cross_edges: list[dict[str, Any]] = []
cross_counts: Counter[tuple[int, int]] = Counter()
for e in all_edges:
# TESTED_BY edges are expected cross-community coupling (test → code),
# not an architectural smell.
if e.kind == "TESTED_BY":
continue
src_comm = node_to_community.get(e.source_qualified)
tgt_comm = node_to_community.get(e.target_qualified)
if (
src_comm is not None
and tgt_comm is not None
and src_comm != tgt_comm
):
pair = (min(src_comm, tgt_comm), max(src_comm, tgt_comm))
cross_counts[pair] += 1
cross_edges.append({
"source_community": src_comm,
"target_community": tgt_comm,
"edge_kind": e.kind,
"source": _sanitize_name(e.source_qualified),
"target": _sanitize_name(e.target_qualified),
})
# Generate warnings for high coupling, skipping test-dominated pairs.
warnings: list[str] = []
comm_name_map = {c.get("id", 0): c["name"] for c in communities}
for (c1, c2), count in cross_counts.most_common():
if count > 10:
name1 = comm_name_map.get(c1, f"community-{c1}")
name2 = comm_name_map.get(c2, f"community-{c2}")
# Skip pairs where either community is test-dominated — coupling
# between test and production code is expected, not architectural.
if _is_test_community(name1) or _is_test_community(name2):
continue
warnings.append(
f"High coupling ({count} edges) between "
f"'{name1}' and '{name2}'"
)
return {
"communities": communities,
"cross_community_edges": cross_edges,
"warnings": warnings,
}
+23
View File
@@ -0,0 +1,23 @@
"""Shared constants for code-review-graph."""
from __future__ import annotations
import os
SECURITY_KEYWORDS: frozenset[str] = frozenset({
"auth", "login", "password", "token", "session", "crypt", "secret",
"credential", "permission", "sql", "query", "execute", "connect",
"socket", "request", "http", "sanitize", "validate", "encrypt",
"decrypt", "hash", "sign", "verify", "admin", "privilege",
})
# ---------------------------------------------------------------------------
# Configurable limits (override via environment variables)
# ---------------------------------------------------------------------------
MAX_IMPACT_NODES = int(os.environ.get("CRG_MAX_IMPACT_NODES", "500"))
MAX_IMPACT_DEPTH = int(os.environ.get("CRG_MAX_IMPACT_DEPTH", "2"))
MAX_BFS_DEPTH = int(os.environ.get("CRG_MAX_BFS_DEPTH", "15"))
MAX_SEARCH_RESULTS = int(os.environ.get("CRG_MAX_SEARCH_RESULTS", "20"))
# BFS engine: "sql" (SQLite recursive CTE) or "networkx" (Python-side BFS)
BFS_ENGINE = os.environ.get("CRG_BFS_ENGINE", "sql")
+317
View File
@@ -0,0 +1,317 @@
"""Compact estimated context savings helpers.
The project intentionally labels these values as estimates: the helper uses a
conservative character-count approximation instead of model-specific tokenizers.
"""
from __future__ import annotations
import json
from pathlib import Path
from typing import Any, Iterable
CHARS_PER_TOKEN = 4
def estimate_tokens(value: Any) -> int:
"""Estimate token count with a conservative 4 chars/token approximation."""
if value is None:
return 0
if isinstance(value, str):
text = value
else:
text = json.dumps(
value,
default=str,
ensure_ascii=True,
separators=(",", ":"),
sort_keys=True,
)
if not text:
return 0
return max(1, (len(text) + CHARS_PER_TOKEN - 1) // CHARS_PER_TOKEN)
def estimate_file_tokens(repo_root: Path, files: Iterable[str]) -> int:
"""Estimate tokens for changed files using file sizes, not file contents."""
total = 0
root = repo_root.resolve()
for file_name in files:
path = Path(file_name)
full_path = path if path.is_absolute() else root / path
try:
if full_path.is_file():
total += max(
1,
(full_path.stat().st_size + CHARS_PER_TOKEN - 1)
// CHARS_PER_TOKEN,
)
except OSError:
continue
return total
def estimate_context_savings(
*,
original_context: Any | None = None,
returned_context: Any | None = None,
original_tokens: int | None = None,
returned_tokens: int | None = None,
) -> dict[str, int | bool] | None:
"""Return tiny savings metadata, or None when no baseline is available."""
baseline = (
original_tokens
if original_tokens is not None
else estimate_tokens(original_context)
)
returned = (
returned_tokens
if returned_tokens is not None
else estimate_tokens(returned_context)
)
if baseline <= 0:
return None
saved = max(0, baseline - returned)
percent = round((saved / baseline) * 100) if baseline else 0
return {
"estimated": True,
"saved_tokens": int(saved),
"saved_percent": int(percent),
}
def attach_context_savings(
result: dict[str, Any],
*,
original_context: Any | None = None,
original_tokens: int | None = None,
returned_context: Any | None = None,
returned_tokens: int | None = None,
) -> dict[str, Any]:
"""Attach compact ``context_savings`` metadata when it can be estimated."""
estimate = estimate_context_savings(
original_context=original_context,
returned_context=result if returned_context is None else returned_context,
original_tokens=original_tokens,
returned_tokens=returned_tokens,
)
if estimate is not None:
result["context_savings"] = estimate
return result
def format_context_savings(estimate: dict[str, Any] | None) -> str | None:
"""Format a one-line human summary for CLI output."""
if not estimate:
return None
saved = int(estimate.get("saved_tokens", 0))
percent = int(estimate.get("saved_percent", 0))
return f"Estimated context saved: ~{saved:,} tokens (~{percent}%)"
def _fmt_compact(n: int) -> str:
"""Compact integer formatting: 1234 -> '1.2k', 9876 -> '9.9k', 500 -> '500'."""
if n >= 10_000:
return f"{n // 1000:,}k"
if n >= 1000:
return f"{n / 1000:.1f}k"
return str(n)
def _breakdown_from_response(response: dict[str, Any]) -> dict[str, int]:
"""Pull a per-category token estimate from a detect-changes / review response.
Only fields that exist and have content are reported, so the breakdown
line stays meaningful instead of padding with zeros.
"""
# Friendly label -> response-dict key
fields = [
("Functions", "changed_functions"),
("Flows", "affected_flows"),
("Tests", "test_gaps"),
("Risk", "review_priorities"),
("Impact", "impacted_nodes"),
("Edges", "edges"),
("Source", "source_snippets"),
("Imports", "imports"),
]
out: dict[str, int] = {}
for label, key in fields:
value = response.get(key)
if not value:
continue
tokens = estimate_tokens(value)
if tokens > 0:
out[label] = tokens
return out
def verify_with_tiktoken(
repo_root: "Path | str",
changed_files: Iterable[str],
response: Any,
encoding_name: str = "cl100k_base",
) -> dict[str, int] | None:
"""Calibrate the chars/4 estimate against a real model tokenizer.
Returns ``{"verified_baseline": int, "verified_returned": int,
"verified_saved": int, "verified_percent": int}`` or ``None`` if
tiktoken is not installed. Reads every changed file's content (unlike
the stat-only ``estimate_file_tokens``) so the numbers reflect what
an agent would actually consume.
"""
try:
import tiktoken # type: ignore[import-untyped]
except ImportError:
return None
enc = tiktoken.get_encoding(encoding_name)
root = Path(repo_root).resolve()
naive_real = 0
for f in changed_files:
p = root / f
try:
if p.is_file():
naive_real += len(enc.encode(p.read_text(errors="replace")))
except OSError:
continue
if isinstance(response, str):
graph_real = len(enc.encode(response))
else:
text = json.dumps(
response, default=str, ensure_ascii=True,
separators=(",", ":"), sort_keys=True,
)
graph_real = len(enc.encode(text))
saved = max(0, naive_real - graph_real)
pct = round(saved * 100 / naive_real) if naive_real > 0 else 0
return {
"verified_baseline": naive_real,
"verified_returned": graph_real,
"verified_saved": saved,
"verified_percent": pct,
}
def format_context_savings_panel(
estimate: dict[str, Any] | None,
*,
original_tokens: int | None = None,
returned_tokens: int | None = None,
response: dict[str, Any] | None = None,
breakdown: dict[str, int] | None = None,
verified: dict[str, int] | None = None,
title: str = "Token Savings",
width: int = 64,
) -> str | None:
"""Format the savings estimate as a boxed multi-line CLI panel.
Example output (width=60)::
┌──────────────── Token Savings ────────────────┐
│ Full context would be: 12,932 tokens │
│ Graph context used: 773 tokens │
│ Saved: 12,159 tokens (~94%) │
│ Breakdown: Functions 580 · Tests 120 · ... │
└───────────────────────────────────────────────┘
All numbers are labelled as estimates upstream (``estimated: true`` in the
metadata dict) because the project uses a 4-chars-per-token approximation,
not model-specific tokenization.
Args:
estimate: The ``context_savings`` dict from a tool response.
original_tokens: Optional override for the naive baseline.
returned_tokens: Optional override for the graph response size.
response: When provided, breakdown is auto-derived from common keys
(``changed_functions``, ``affected_flows``, ``test_gaps``,
``review_priorities``, ``impacted_nodes``, ``edges``,
``source_snippets``, ``imports``).
breakdown: Explicit ``{label: tokens}`` map; takes precedence over
``response``-derived breakdown when both are provided.
title: Title centered in the top border.
width: Total panel width, capped at terminal width if larger.
Returns:
The panel as a single ``\\n``-joined string, or ``None`` when there
is nothing meaningful to display.
"""
if not estimate:
return None
saved = int(estimate.get("saved_tokens", 0))
percent = int(estimate.get("saved_percent", 0))
# Derive baseline + returned from saved+percent if not provided
if original_tokens is None:
if percent > 0:
original_tokens = int(round(saved * 100 / percent))
else:
original_tokens = saved
if returned_tokens is None:
returned_tokens = max(0, (original_tokens or 0) - saved)
if breakdown is None and response is not None:
breakdown = _breakdown_from_response(response)
# Top up the breakdown with an "Other" bucket so the parts sum to
# ``returned_tokens`` exactly. "Other" covers fields the breakdown
# doesn't enumerate (status, summary, risk_score, context_savings
# metadata, JSON envelope chars). Skip when there's no positive
# remainder — the breakdown already accounts for the whole response.
if breakdown and returned_tokens is not None:
labelled_sum = sum(breakdown.values())
remainder = returned_tokens - labelled_sum
if remainder > 0:
breakdown = dict(breakdown) # copy before mutating
breakdown["Other"] = remainder
# Lines that go inside the box (without borders)
inner_lines: list[str] = [
f"Full context would be: {original_tokens:>9,} tokens",
f"Graph context used: {returned_tokens:>9,} tokens",
f"Saved: {saved:>9,} tokens (~{percent}%)",
]
if verified:
vb = verified["verified_baseline"]
vr = verified["verified_returned"]
vs = verified["verified_saved"]
vp = verified["verified_percent"]
inner_lines.append(
f"Verified (tiktoken): {vs:>9,} tokens (~{vp}%) "
f"[{vb:,}{vr:,}]"
)
if breakdown:
parts = [f"{label} {_fmt_compact(tok)}" for label, tok in breakdown.items()]
bd_line = "Breakdown: " + " · ".join(parts)
inner_lines.append(bd_line)
# Compute final width: at least wide enough for the longest inner line + padding
content_width = max(len(s) for s in inner_lines)
inner_w = max(width - 2, content_width + 2) # +2 for one space pad each side
# Title bar
title_str = f" {title} "
dash_total = inner_w - len(title_str)
if dash_total < 4:
dash_total = 4
left_dash = dash_total // 2
right_dash = dash_total - left_dash
top = "" + "" * left_dash + title_str + "" * right_dash + ""
bottom = "" + "" * inner_w + ""
def _box_line(content: str) -> str:
pad = inner_w - 2 - len(content)
if pad < 0:
pad = 0
return f"{content}{' ' * pad}"
lines = [top]
for s in inner_lines:
lines.append(_box_line(s))
lines.append(bottom)
return "\n".join(lines)
+322
View File
@@ -0,0 +1,322 @@
"""Config-driven custom language support ("bring your own language").
Repos can teach the parser new tree-sitter languages without forking by
dropping a ``languages.toml`` file into ``.code-review-graph/``::
[languages.erlang]
extensions = [".erl", ".hrl"]
grammar = "erlang" # tree_sitter_language_pack name
function_node_types = ["function_clause"]
class_node_types = ["record_decl"]
import_node_types = ["import_attribute"]
call_node_types = ["call"]
comment = "Erlang via the bundled tree-sitter-erlang grammar"
The loader is deliberately defensive: a broken config must never crash a
build. Invalid entries are skipped with a ``logger.warning``, and built-in
languages always win — custom entries can neither override built-in file
extensions nor reuse built-in language names. At most
``MAX_CUSTOM_LANGUAGES`` entries are honoured per repo.
See docs/CUSTOM_LANGUAGES.md for the full schema reference (answers #320).
"""
from __future__ import annotations
import logging
import re
import sys
import threading
from collections.abc import Mapping
from dataclasses import dataclass, field
from pathlib import Path
from typing import Optional
import tree_sitter_language_pack as tslp
if sys.version_info >= (3, 11):
import tomllib
else:
try:
import tomli as tomllib # type: ignore[no-redef]
except ImportError:
tomllib = None # type: ignore[assignment]
logger = logging.getLogger(__name__)
#: Location of the config file, relative to the repo root.
CONFIG_RELATIVE_PATH = Path(".code-review-graph") / "languages.toml"
#: Hard cap on the number of custom languages loaded from a single config.
MAX_CUSTOM_LANGUAGES = 20
#: Custom language names: short lowercase identifiers. The name becomes the
#: ``language`` field on every node parsed from matching files.
_NAME_RE = re.compile(r"^[a-z][a-z0-9_-]{0,31}$")
#: Extensions: a leading dot followed by 1-15 safe characters (".erl",
#: ".cls", ".4gl"). Uppercase input is normalised to lowercase because the
#: parser lowercases file suffixes before lookup.
_EXTENSION_RE = re.compile(r"^\.[a-z0-9_+-]{1,15}$")
#: The four node-type lists recognised in each ``[languages.<name>]`` table.
_NODE_TYPE_KEYS = (
"function_node_types",
"class_node_types",
"import_node_types",
"call_node_types",
)
@dataclass(frozen=True)
class CustomLanguage:
"""One validated ``[languages.<name>]`` entry from languages.toml."""
name: str
grammar: str
extensions: tuple[str, ...]
function_node_types: tuple[str, ...] = ()
class_node_types: tuple[str, ...] = ()
import_node_types: tuple[str, ...] = ()
call_node_types: tuple[str, ...] = ()
comment: str = ""
@dataclass(frozen=True)
class _CacheEntry:
mtime_ns: int
size: int
languages: dict[str, CustomLanguage] = field(default_factory=dict)
# Config files are re-read only when their mtime/size changes. This matters
# because full builds construct one CodeParser per worker task, and probing
# tree-sitter grammars on every file parse would be wasteful.
_cache_lock = threading.Lock()
_cache: dict[str, _CacheEntry] = {}
def clear_cache() -> None:
"""Drop the loader cache (used by tests)."""
with _cache_lock:
_cache.clear()
def load_custom_languages(
repo_root: Path,
*,
builtin_extensions: Mapping[str, str],
builtin_languages: frozenset[str],
) -> dict[str, CustomLanguage]:
"""Load and validate ``<repo_root>/.code-review-graph/languages.toml``.
Returns a mapping of custom language name -> :class:`CustomLanguage`.
Always returns (possibly empty) — a broken config never raises.
Args:
repo_root: Repository root containing ``.code-review-graph/``.
builtin_extensions: The parser's built-in extension map; custom
entries colliding with these are skipped (built-ins win).
builtin_languages: All built-in language identifiers; custom names
shadowing these are skipped.
"""
config_path = Path(repo_root) / CONFIG_RELATIVE_PATH
try:
stat = config_path.stat()
except OSError:
return {} # No config file — the common case; not worth a log line.
cache_key = str(config_path)
with _cache_lock:
cached = _cache.get(cache_key)
if (
cached is not None
and cached.mtime_ns == stat.st_mtime_ns
and cached.size == stat.st_size
):
return dict(cached.languages)
languages = _load_uncached(config_path, builtin_extensions, builtin_languages)
with _cache_lock:
_cache[cache_key] = _CacheEntry(stat.st_mtime_ns, stat.st_size, dict(languages))
return languages
def _load_uncached(
config_path: Path,
builtin_extensions: Mapping[str, str],
builtin_languages: frozenset[str],
) -> dict[str, CustomLanguage]:
if tomllib is None:
logger.warning(
"%s found but TOML parsing requires the 'tomli' package on "
"Python < 3.11 — no custom languages loaded",
config_path,
)
return {}
try:
raw = config_path.read_bytes()
except (OSError, PermissionError) as exc:
logger.warning("Cannot read %s: %s — no custom languages loaded", config_path, exc)
return {}
try:
data = tomllib.loads(raw.decode("utf-8", errors="replace"))
except tomllib.TOMLDecodeError as exc:
logger.warning("Malformed TOML in %s: %s — no custom languages loaded", config_path, exc)
return {}
tables = data.get("languages")
if tables is None:
return {}
if not isinstance(tables, dict):
logger.warning(
"%s: [languages] must be a table of tables — no custom languages loaded",
config_path,
)
return {}
result: dict[str, CustomLanguage] = {}
claimed_extensions: set[str] = set()
for name, table in tables.items():
if len(result) >= MAX_CUSTOM_LANGUAGES:
logger.warning(
"%s defines more than %d custom languages — ignoring the rest",
config_path, MAX_CUSTOM_LANGUAGES,
)
break
lang = _validate_entry(
name, table, builtin_extensions, builtin_languages,
claimed_extensions, config_path,
)
if lang is None:
continue
result[lang.name] = lang
claimed_extensions.update(lang.extensions)
return result
def _validate_entry(
name: object,
table: object,
builtin_extensions: Mapping[str, str],
builtin_languages: frozenset[str],
claimed_extensions: set[str],
config_path: Path,
) -> Optional[CustomLanguage]:
"""Validate one ``[languages.<name>]`` table; None (after a warning) on
any problem so a bad entry can never break a build."""
label = name if isinstance(name, str) else repr(name)
if not isinstance(table, dict):
logger.warning("%s: [languages.%s] is not a table — skipping", config_path, label)
return None
if not isinstance(name, str) or not _NAME_RE.match(name):
logger.warning(
"%s: invalid custom language name %r (expected lowercase "
"letters/digits/_/-, max 32 chars) — skipping",
config_path, label,
)
return None
if name in builtin_languages:
logger.warning(
"%s: custom language %r shadows a built-in language — skipping "
"(built-ins cannot be overridden)",
config_path, name,
)
return None
grammar = table.get("grammar")
if not isinstance(grammar, str) or not grammar.strip():
logger.warning(
"%s: custom language %r needs a non-empty 'grammar' string — skipping",
config_path, name,
)
return None
grammar = grammar.strip()
raw_extensions = table.get("extensions")
if not isinstance(raw_extensions, list) or not raw_extensions:
logger.warning(
"%s: custom language %r needs a non-empty 'extensions' list — skipping",
config_path, name,
)
return None
extensions: list[str] = []
for ext in raw_extensions:
normalized = ext.strip().lower() if isinstance(ext, str) else ""
if not normalized.startswith("."):
logger.warning(
"%s: custom language %r: extension %r must start with a dot — skipping",
config_path, name, ext,
)
return None
if not _EXTENSION_RE.match(normalized):
logger.warning(
"%s: custom language %r: extension %r is not a valid file "
"extension — skipping",
config_path, name, ext,
)
return None
if normalized in builtin_extensions:
logger.warning(
"%s: custom language %r: extension %r is already handled by "
"the built-in %r parser — skipping (built-ins cannot be overridden)",
config_path, name, normalized, builtin_extensions[normalized],
)
return None
if normalized in claimed_extensions:
logger.warning(
"%s: custom language %r: extension %r is already claimed by "
"an earlier custom language — skipping",
config_path, name, normalized,
)
return None
if normalized not in extensions:
extensions.append(normalized)
node_types: dict[str, tuple[str, ...]] = {}
for key in _NODE_TYPE_KEYS:
value = table.get(key, [])
if not isinstance(value, list) or any(
not isinstance(item, str) or not item.strip() for item in value
):
logger.warning(
"%s: custom language %r: %s must be a list of non-empty "
"strings — skipping",
config_path, name, key,
)
return None
node_types[key] = tuple(item.strip() for item in value)
if not any(node_types.values()):
logger.warning(
"%s: custom language %r defines no node types — nothing to "
"extract, skipping",
config_path, name,
)
return None
comment = table.get("comment", "")
if not isinstance(comment, str):
comment = ""
# Probe the grammar last (it is the expensive check). Parser objects
# themselves are created lazily by CodeParser._get_parser.
try:
tslp.get_language(grammar) # type: ignore[arg-type]
except (LookupError, ValueError, ImportError, OSError) as exc:
logger.warning(
"%s: custom language %r: grammar %r is not available in "
"tree_sitter_language_pack (%s) — skipping",
config_path, name, grammar, exc,
)
return None
return CustomLanguage(
name=name,
grammar=grammar,
extensions=tuple(extensions),
function_node_types=node_types["function_node_types"],
class_node_types=node_types["class_node_types"],
import_node_types=node_types["import_node_types"],
call_node_types=node_types["call_node_types"],
comment=comment,
)
File diff suppressed because it is too large Load Diff
+320
View File
@@ -0,0 +1,320 @@
"""CLI entry point for the crg-daemon multi-repo watcher.
Usage:
crg-daemon start [--foreground]
crg-daemon stop
crg-daemon restart [--foreground]
crg-daemon status
crg-daemon logs [--repo ALIAS] [--follow] [--lines N]
crg-daemon add <path> [--alias ALIAS]
crg-daemon remove <path_or_alias>
"""
from __future__ import annotations
import argparse
import logging
import os
import signal
import subprocess
import sys
import time
logger = logging.getLogger(__name__)
# ---------------------------------------------------------------------------
# Subcommand handlers
# ---------------------------------------------------------------------------
def _handle_start(args: argparse.Namespace) -> None:
"""Start the daemon process."""
from .daemon import WatchDaemon, is_daemon_running, load_config
if is_daemon_running():
print("Error: Daemon is already running.")
sys.exit(1)
config = load_config()
daemon = WatchDaemon(config=config)
daemon.start()
if not args.foreground:
daemon.daemonize()
daemon.run_forever()
def _handle_stop(_args: argparse.Namespace) -> None:
"""Stop the running daemon process."""
from .daemon import clear_pid, is_daemon_running, read_pid
if not is_daemon_running():
print("Daemon is not running.")
sys.exit(1)
pid = read_pid()
if pid is None:
print("Error: Could not read daemon PID.")
sys.exit(1)
print(f"Stopping daemon (PID {pid})...")
try:
os.kill(pid, signal.SIGTERM)
except ProcessLookupError:
clear_pid()
print("Daemon stopped (process already gone).")
return
except PermissionError:
print(f"Error: Permission denied sending signal to PID {pid}.")
sys.exit(1)
# Wait up to 5 seconds for process to die
for _ in range(50):
try:
os.kill(pid, 0)
except ProcessLookupError:
break
time.sleep(0.1)
else:
# Still alive after 5s — send SIGKILL
print("Daemon did not stop gracefully, sending SIGKILL...")
try:
os.kill(pid, signal.SIGKILL)
except ProcessLookupError:
pass
clear_pid()
print("Daemon stopped.")
def _handle_restart(args: argparse.Namespace) -> None:
"""Restart the daemon (stop + start)."""
from .daemon import is_daemon_running
if is_daemon_running():
_handle_stop(args)
else:
print("Daemon is not running, starting fresh.")
_handle_start(args)
def _handle_status(_args: argparse.Namespace) -> None:
"""Show daemon status and configuration."""
from .daemon import is_daemon_running, load_config, load_state, pid_alive, read_pid
config = load_config()
running = is_daemon_running()
if running:
pid = read_pid()
print(f"Daemon: running (PID {pid})")
else:
print("Daemon: not running")
print(f"Name: {config.session_name}")
print(f"Log dir: {config.log_dir}")
print(f"Poll: {config.poll_interval}s")
print()
if not config.repos:
print("No repositories configured.")
print("Use: crg-daemon add <path> [--alias NAME]")
return
# Header
alias_width = max(len(r.alias) for r in config.repos)
alias_width = max(alias_width, 5) # minimum "Alias" header width
if running:
state = load_state()
print(f" {'Alias':<{alias_width}} {'Status':<8} {'PID':<8} Path")
print(f" {'-' * alias_width} {'-' * 8} {'-' * 8} {'-' * 40}")
for repo in config.repos:
entry = state.get(repo.alias, {})
child_pid: int | None = entry.get("pid")
alive = child_pid is not None and pid_alive(child_pid)
status_str = "alive" if alive else "dead"
pid_str = str(child_pid) if child_pid is not None else "-"
print(f" {repo.alias:<{alias_width}} {status_str:<8} {pid_str:<8} {repo.path}")
else:
print(f" {'Alias':<{alias_width}} Path")
print(f" {'-' * alias_width} {'-' * 40}")
for repo in config.repos:
print(f" {repo.alias:<{alias_width}} {repo.path}")
def _handle_logs(args: argparse.Namespace) -> None:
"""Show daemon or per-repo log files."""
from .daemon import load_config
config = load_config()
if args.repo:
log_file = config.log_dir / f"{args.repo}.log"
else:
log_file = config.log_dir / "daemon.log"
if not log_file.exists():
print(f"Log file not found: {log_file}")
sys.exit(1)
if args.follow:
try:
subprocess.run(["tail", "-f", str(log_file)], check=False)
except KeyboardInterrupt:
pass
return
# Read last N lines
lines_count = args.lines
try:
text = log_file.read_text(encoding="utf-8", errors="replace")
except OSError as exc:
print(f"Error reading log file: {exc}")
sys.exit(1)
lines = text.splitlines()
tail = lines[-lines_count:] if len(lines) > lines_count else lines
for line in tail:
print(line)
def _handle_add(args: argparse.Namespace) -> None:
"""Add a repository to the daemon config."""
from .daemon import add_repo_to_config, is_daemon_running
try:
add_repo_to_config(args.path, alias=args.alias)
except ValueError as exc:
print(f"Error: {exc}")
sys.exit(1)
# Find the repo we just added to show confirmation
alias = args.alias or os.path.basename(os.path.abspath(args.path))
print(f"Added repository: {args.path} (alias: {alias})")
if is_daemon_running():
print("Daemon will pick up the change automatically.")
def _handle_remove(args: argparse.Namespace) -> None:
"""Remove a repository from the daemon config."""
from .daemon import is_daemon_running, load_config, remove_repo_from_config
config_before = load_config()
count_before = len(config_before.repos)
config_after = remove_repo_from_config(args.path_or_alias)
count_after = len(config_after.repos)
if count_before == count_after:
print(f"No repository matching '{args.path_or_alias}' found in config.")
sys.exit(1)
print(f"Removed repository: {args.path_or_alias}")
if is_daemon_running():
print("Daemon will pick up the change automatically.")
# ---------------------------------------------------------------------------
# Main entry point
# ---------------------------------------------------------------------------
def main() -> None:
"""Entry point for the crg-daemon CLI."""
logging.basicConfig(level=logging.INFO, format="%(levelname)s: %(message)s")
ap = argparse.ArgumentParser(
prog="crg-daemon",
description="Multi-repo watch daemon for code-review-graph",
)
sub = ap.add_subparsers(dest="command")
# start
start_cmd = sub.add_parser("start", help="Start the daemon")
start_cmd.add_argument(
"--foreground",
action="store_true",
help="Run in the foreground instead of daemonizing",
)
# stop
sub.add_parser("stop", help="Stop the daemon")
# restart
restart_cmd = sub.add_parser("restart", help="Restart the daemon")
restart_cmd.add_argument(
"--foreground",
action="store_true",
help="Run in the foreground instead of daemonizing",
)
# status
sub.add_parser("status", help="Show daemon status and configuration")
# logs
logs_cmd = sub.add_parser("logs", help="Show daemon or per-repo logs")
logs_cmd.add_argument(
"--repo",
default=None,
metavar="ALIAS",
help="Show logs for a specific repo (by alias)",
)
logs_cmd.add_argument(
"--follow",
"-f",
action="store_true",
help="Follow log output (tail -f)",
)
logs_cmd.add_argument(
"--lines",
"-n",
type=int,
default=50,
help="Number of lines to show (default: 50)",
)
# add
add_cmd = sub.add_parser("add", help="Add a repository to the daemon config")
add_cmd.add_argument("path", help="Path to the repository")
add_cmd.add_argument(
"--alias",
default=None,
help="Short alias for the repository (default: directory name)",
)
# remove
remove_cmd = sub.add_parser("remove", help="Remove a repository from the daemon config")
remove_cmd.add_argument("path_or_alias", help="Repository path or alias to remove")
args = ap.parse_args()
if not args.command:
ap.print_help()
sys.exit(0)
handlers: dict[str, object] = {
"start": _handle_start,
"stop": _handle_stop,
"restart": _handle_restart,
"status": _handle_status,
"logs": _handle_logs,
"add": _handle_add,
"remove": _handle_remove,
}
handler = handlers.get(args.command)
if handler is None:
ap.print_help()
sys.exit(1)
handler(args) # type: ignore[operator]
if __name__ == "__main__":
main()
File diff suppressed because it is too large Load Diff
+303
View File
@@ -0,0 +1,303 @@
"""PreToolUse search enrichment for Claude Code hooks.
Intercepts Grep/Glob/Bash/Read tool calls and enriches them with
structural context from the code knowledge graph: callers, callees,
execution flows, community membership, and test coverage.
"""
from __future__ import annotations
import json
import logging
import os
import re
import sys
from pathlib import Path
from typing import Any
logger = logging.getLogger(__name__)
# Flags that consume the next token in grep/rg commands
_RG_FLAGS_WITH_VALUES = frozenset({
"-e", "-f", "-m", "-A", "-B", "-C", "-g", "--glob",
"-t", "--type", "--include", "--exclude", "--max-count",
"--max-depth", "--max-filesize", "--color", "--colors",
"--context-separator", "--field-match-separator",
"--path-separator", "--replace", "--sort", "--sortr",
})
def extract_pattern(tool_name: str, tool_input: dict[str, Any]) -> str | None:
"""Extract a search pattern from a tool call's input.
Returns None if no meaningful pattern can be extracted.
"""
if tool_name == "Grep":
return tool_input.get("pattern")
if tool_name == "Glob":
raw = tool_input.get("pattern", "")
# Extract meaningful name from glob: "**/auth*.ts" -> "auth"
# Skip pure extension globs like "**/*.ts"
match = re.search(r"[*/]([a-zA-Z][a-zA-Z0-9_]{2,})", raw)
return match.group(1) if match else None
if tool_name == "Bash":
cmd = tool_input.get("command", "")
if not re.search(r"\brg\b|\bgrep\b", cmd):
return None
tokens = cmd.split()
found_cmd = False
skip_next = False
for token in tokens:
if skip_next:
skip_next = False
continue
if not found_cmd:
if re.search(r"\brg$|\bgrep$", token):
found_cmd = True
continue
if token.startswith("-"):
if token in _RG_FLAGS_WITH_VALUES:
skip_next = True
continue
cleaned = token.strip("'\"")
return cleaned if len(cleaned) >= 3 else None
return None
return None
def _make_relative(file_path: str, repo_root: str) -> str:
"""Make a file path relative to repo_root for display."""
try:
return str(Path(file_path).relative_to(repo_root))
except ValueError:
return file_path
def _get_community_name(conn: Any, community_id: int) -> str:
"""Fetch a community name by ID."""
row = conn.execute(
"SELECT name FROM communities WHERE id = ?", (community_id,)
).fetchone()
return row["name"] if row else ""
def _get_flow_names_for_node(conn: Any, node_id: int) -> list[str]:
"""Fetch execution flow names that a node participates in (max 3)."""
rows = conn.execute(
"SELECT f.name FROM flow_memberships fm "
"JOIN flows f ON fm.flow_id = f.id "
"WHERE fm.node_id = ? LIMIT 3",
(node_id,),
).fetchall()
return [r["name"] for r in rows]
def _format_node_context(
node: Any,
store: Any,
conn: Any,
repo_root: str,
) -> list[str]:
"""Format a single node's structural context as plain text lines."""
from .graph import GraphNode
assert isinstance(node, GraphNode)
qn = node.qualified_name
loc = _make_relative(node.file_path, repo_root)
if node.line_start:
loc = f"{loc}:{node.line_start}"
header = f"{node.name} ({loc})"
# Community
if node.extra.get("community_id"):
cname = _get_community_name(conn, node.extra["community_id"])
if cname:
header += f" [{cname}]"
else:
# Check via direct query
row = conn.execute(
"SELECT community_id FROM nodes WHERE id = ?", (node.id,)
).fetchone()
if row and row["community_id"]:
cname = _get_community_name(conn, row["community_id"])
if cname:
header += f" [{cname}]"
lines = [header]
# Callers (max 5, deduplicated)
callers: list[str] = []
seen: set[str] = set()
for e in store.get_edges_by_target(qn):
if e.kind == "CALLS" and len(callers) < 5:
c = store.get_node(e.source_qualified)
if c and c.name not in seen:
seen.add(c.name)
callers.append(c.name)
if callers:
lines.append(f" Called by: {', '.join(callers)}")
# Callees (max 5, deduplicated)
callees: list[str] = []
seen.clear()
for e in store.get_edges_by_source(qn):
if e.kind == "CALLS" and len(callees) < 5:
c = store.get_node(e.target_qualified)
if c and c.name not in seen:
seen.add(c.name)
callees.append(c.name)
if callees:
lines.append(f" Calls: {', '.join(callees)}")
# Execution flows
flow_names = _get_flow_names_for_node(conn, node.id)
if flow_names:
lines.append(f" Flows: {', '.join(flow_names)}")
# Tests
tests: list[str] = []
for e in store.get_edges_by_target(qn):
if e.kind == "TESTED_BY" and len(tests) < 3:
t = store.get_node(e.source_qualified)
if t:
tests.append(t.name)
if tests:
lines.append(f" Tests: {', '.join(tests)}")
return lines
def enrich_search(pattern: str, repo_root: str) -> str:
"""Search the graph for pattern and return enriched context."""
from .graph import GraphStore
from .search import _fts_search
db_path = Path(repo_root) / ".code-review-graph" / "graph.db"
if not db_path.exists():
return ""
store = GraphStore(db_path)
try:
conn = store._conn
fts_results = _fts_search(conn, pattern, limit=8)
if not fts_results:
return ""
all_lines: list[str] = []
count = 0
for node_id, _score in fts_results:
if count >= 5:
break
node = store.get_node_by_id(node_id)
if not node or node.is_test:
continue
node_lines = _format_node_context(node, store, conn, repo_root)
all_lines.extend(node_lines)
all_lines.append("")
count += 1
if not all_lines:
return ""
header = f'[code-review-graph] {count} symbol(s) matching "{pattern}":\n'
return header + "\n".join(all_lines)
finally:
store.close()
def enrich_file_read(file_path: str, repo_root: str) -> str:
"""Enrich a file read with structural context for functions in that file."""
from .graph import GraphStore
db_path = Path(repo_root) / ".code-review-graph" / "graph.db"
if not db_path.exists():
return ""
store = GraphStore(db_path)
try:
conn = store._conn
nodes = store.get_nodes_by_file(file_path)
if not nodes:
# Try with resolved path
try:
resolved = str(Path(file_path).resolve())
nodes = store.get_nodes_by_file(resolved)
except (OSError, ValueError):
pass
if not nodes:
return ""
# Filter to functions/classes/types (skip File nodes), limit to 10
interesting = [
n for n in nodes
if n.kind in ("Function", "Class", "Type", "Test")
][:10]
if not interesting:
return ""
all_lines: list[str] = []
for node in interesting:
node_lines = _format_node_context(node, store, conn, repo_root)
all_lines.extend(node_lines)
all_lines.append("")
rel_path = _make_relative(file_path, repo_root)
header = (
f"[code-review-graph] {len(interesting)} symbol(s) in {rel_path}:\n"
)
return header + "\n".join(all_lines)
finally:
store.close()
def run_hook() -> None:
"""Entry point for the enrich CLI subcommand.
Reads Claude Code hook JSON from stdin, extracts the search pattern,
queries the graph, and outputs hookSpecificOutput JSON to stdout.
"""
try:
hook_input = json.load(sys.stdin)
except (json.JSONDecodeError, ValueError):
return
tool_name = hook_input.get("tool_name", "")
tool_input = hook_input.get("tool_input", {})
cwd = hook_input.get("cwd", os.getcwd())
# Find repo root by walking up from cwd
from .incremental import find_project_root
repo_root = str(find_project_root(Path(cwd)))
db_path = Path(repo_root) / ".code-review-graph" / "graph.db"
if not db_path.exists():
return
# Dispatch
context = ""
if tool_name == "Read":
fp = tool_input.get("file_path", "")
if fp:
context = enrich_file_read(fp, repo_root)
else:
pattern = extract_pattern(tool_name, tool_input)
if not pattern or len(pattern) < 3:
return
context = enrich_search(pattern, repo_root)
if not context:
return
response = {
"hookSpecificOutput": {
"hookEventName": "PreToolUse",
"additionalContext": context,
}
}
json.dump(response, sys.stdout)
+33
View File
@@ -0,0 +1,33 @@
"""Evaluation framework for code-review-graph.
Provides scoring metrics (token efficiency, MRR, precision/recall),
benchmark runners, and report generators for benchmarking graph-based code reviews.
"""
from __future__ import annotations
from .reporter import generate_full_report, generate_markdown_report, generate_readme_tables
from .scorer import compute_mrr, compute_precision_recall, compute_token_efficiency
def __getattr__(name: str):
"""Lazy-import runner functions (require pyyaml)."""
_runner_names = {"load_all_configs", "load_config", "run_eval", "write_csv"}
if name in _runner_names:
from . import runner
return getattr(runner, name)
raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
__all__ = [
"compute_mrr",
"compute_precision_recall",
"compute_token_efficiency",
"generate_full_report",
"generate_markdown_report",
"generate_readme_tables",
"load_all_configs",
"load_config",
"run_eval",
"write_csv",
]
@@ -0,0 +1 @@
"""Benchmark modules for the evaluation framework."""
@@ -0,0 +1,193 @@
"""Agent baseline benchmark: grep-and-read-top-k versus a graph query.
The whole-corpus baseline in the standalone token benchmark is an upper
bound no real agent pays: a competent agent greps for identifiers from the
question and reads only the best-matching files. This benchmark measures
that realistic baseline:
1. Derive search terms from the question (identifier-shaped tokens via
``search.extract_query_identifiers`` plus plain keywords).
2. Pure-python grep over the corpus (no external ``rg``/``grep`` binary),
ranking files by total case-insensitive match count.
3. Read the top-k files (k=3) and token-count them with the chars/4 utility
(``token_benchmark.estimate_tokens``) as ``baseline_tokens``.
4. Compare against the graph-query cost for the same question — hybrid
search hits plus one hop of neighbor edges, the same accounting used by
``code_review_graph/token_benchmark.py``.
Questions come from ``agent_questions:`` in the repo config, falling back to
the ``search_queries`` query strings when absent.
Failure semantics match the other benchmarks: a thrown search is recorded
with ``status="error"`` and excluded from aggregates; rows where either side
of the ratio is zero get ``status="no_graph_results"`` /
``status="no_baseline_match"`` and are likewise excluded.
"""
from __future__ import annotations
import logging
import statistics
from collections.abc import Iterator
from pathlib import Path
from code_review_graph.token_benchmark import estimate_tokens
logger = logging.getLogger(__name__)
DEFAULT_TOP_K = 3
_SOURCE_EXTS = (
".py", ".js", ".ts", ".tsx", ".go", ".rs", ".java",
".c", ".cpp", ".h", ".rb", ".php", ".swift", ".kt",
)
_SKIP_DIRS = {
".git", ".hg", ".svn", "node_modules", "__pycache__",
".code-review-graph", ".venv", "venv", "dist", "build",
}
_STOPWORDS = {
"how", "does", "do", "the", "a", "an", "is", "are", "was", "what",
"where", "when", "which", "who", "why", "and", "or", "in", "on", "of",
"to", "for", "with", "via", "into", "from", "this", "that", "it", "its",
}
def derive_search_terms(question: str) -> list[str]:
"""Derive lowercase grep terms: identifiers first, then plain keywords.
Identifier-shaped tokens (``Client.request``, ``get_users``, ``APIRoute``)
are extracted via ``search.extract_query_identifiers``; remaining words of
3+ characters that are not stopwords are appended. Order is deterministic.
"""
from code_review_graph.search import extract_query_identifiers
terms: list[str] = []
seen: set[str] = set()
for ident in extract_query_identifiers(question):
if ident not in seen:
seen.add(ident)
terms.append(ident)
for word in question.split():
w = word.strip(".,;:!?\"'()[]{}`").lower()
if len(w) >= 3 and w not in _STOPWORDS and w not in seen:
seen.add(w)
terms.append(w)
return terms
def iter_source_files(repo_path: Path) -> Iterator[Path]:
"""Yield source files under *repo_path*, skipping vendored/VCS dirs."""
for path in sorted(repo_path.rglob("*")):
if path.suffix not in _SOURCE_EXTS or not path.is_file():
continue
if any(part in _SKIP_DIRS for part in path.parts):
continue
yield path
def grep_rank(
repo_path: Path, terms: list[str], k: int = DEFAULT_TOP_K,
) -> list[tuple[str, int]]:
"""Rank source files by total case-insensitive term matches; take top-k.
Pure python — no external grep/rg dependency. Deterministic: ties break
on the relative path. Files with zero matches are dropped.
"""
lowered = [t.lower() for t in terms if t]
if not lowered:
return []
scores: list[tuple[str, int]] = []
for path in iter_source_files(repo_path):
try:
text = path.read_text(encoding="utf-8", errors="replace").lower()
except OSError:
continue
count = sum(text.count(term) for term in lowered)
if count > 0:
scores.append((str(path.relative_to(repo_path)), count))
scores.sort(key=lambda item: (-item[1], item[0]))
return scores[:k]
def run(repo_path: Path, store, config: dict) -> list[dict]:
"""Run the agent baseline benchmark for one repo."""
questions = list(config.get("agent_questions") or [])
if not questions:
questions = [sq["query"] for sq in config.get("search_queries", [])]
k = int(config.get("agent_baseline_top_k", DEFAULT_TOP_K))
results: list[dict] = []
for question in questions:
terms = derive_search_terms(question)
top = grep_rank(repo_path, terms, k=k)
baseline_tokens = 0
for rel, _count in top:
try:
baseline_tokens += estimate_tokens(
(repo_path / rel).read_text(encoding="utf-8", errors="replace")
)
except OSError:
continue
row: dict = {
"repo": config["name"],
"question": question,
"terms": " ".join(terms),
"files_matched": len(top),
"top_files": ";".join(rel for rel, _ in top),
"baseline_tokens": baseline_tokens,
"graph_tokens": "",
"baseline_to_graph_ratio": "",
"status": "ok",
"error": "",
}
try:
from code_review_graph.search import hybrid_search
hits = hybrid_search(store, question, limit=5)
except Exception as exc:
logger.warning("hybrid_search failed on %r: %s", question, exc)
row["status"] = "error"
row["error"] = str(exc)[:200]
results.append(row)
continue
# Same accounting as the standalone token benchmark: search hits
# plus up to 5 outgoing edges of neighbor context per hit.
graph_tokens = 0
for hit in hits:
graph_tokens += estimate_tokens(str(hit))
qn = hit.get("qualified_name", "")
for edge in store.get_edges_by_source(qn)[:5]:
graph_tokens += estimate_tokens(str(edge))
row["graph_tokens"] = graph_tokens
if baseline_tokens > 0 and graph_tokens > 0:
row["baseline_to_graph_ratio"] = round(baseline_tokens / graph_tokens, 1)
elif graph_tokens == 0:
row["status"] = "no_graph_results"
else:
row["status"] = "no_baseline_match"
results.append(row)
return results
def aggregate(results: list[dict]) -> dict:
"""Aggregate over rows where both sides of the comparison exist."""
ok = [r for r in results if r.get("status") == "ok"]
ratios = [float(r["baseline_to_graph_ratio"]) for r in ok]
return {
"total_rows": len(results),
"ok_rows": len(ok),
"error_rows": sum(1 for r in results if r.get("status") == "error"),
"median_baseline_to_graph_ratio": (
round(statistics.median(ratios), 1) if ratios else None
),
"mean_baseline_to_graph_ratio": (
round(statistics.mean(ratios), 1) if ratios else None
),
}
@@ -0,0 +1,60 @@
"""Build performance benchmark: measures timing of graph operations."""
from __future__ import annotations
import logging
import time
from pathlib import Path
logger = logging.getLogger(__name__)
def run(repo_path: Path, store, config: dict) -> list[dict]:
"""Run build performance benchmark."""
stats = store.get_stats()
# Time flow detection
try:
from code_review_graph.flows import store_flows, trace_flows
t0 = time.perf_counter()
flows = trace_flows(store)
store_flows(store, flows)
flow_time = time.perf_counter() - t0
except Exception as exc:
logger.warning("Flow detection failed: %s", exc)
flow_time = 0.0
# Time community detection
try:
from code_review_graph.communities import detect_communities, store_communities
t0 = time.perf_counter()
comms = detect_communities(store)
store_communities(store, comms)
community_time = time.perf_counter() - t0
except Exception as exc:
logger.warning("Community detection failed: %s", exc)
community_time = 0.0
# Time search (average of queries)
search_times: list[float] = []
for sq in config.get("search_queries", [])[:10]:
t0 = time.perf_counter()
store.search_nodes(sq["query"], limit=20)
search_times.append(time.perf_counter() - t0)
avg_search_ms = round(
sum(search_times) / max(len(search_times), 1) * 1000, 1
)
return [{
"repo": config["name"],
"file_count": stats.files_count,
"node_count": stats.total_nodes,
"edge_count": stats.total_edges,
"flow_detection_seconds": round(flow_time, 3),
"community_detection_seconds": round(community_time, 3),
"search_avg_ms": avg_search_ms,
"nodes_per_second": round(
stats.total_nodes / max(flow_time, 0.001)
),
}]
@@ -0,0 +1,36 @@
"""Flow completeness benchmark: evaluates entry point detection and flow tracing."""
from __future__ import annotations
import logging
from pathlib import Path
logger = logging.getLogger(__name__)
def run(repo_path: Path, store, config: dict) -> list[dict]:
"""Run flow completeness benchmark."""
from code_review_graph.flows import store_flows, trace_flows
flows = trace_flows(store)
count = store_flows(store, flows)
# Get detected entry point names
detected_entries = set()
for flow in flows:
detected_entries.add(flow.get("entry_point") or flow.get("name", ""))
known = set(config.get("entry_points", []))
found = sum(1 for ep in known if any(ep in d for d in detected_entries))
depths = [f.get("depth", 0) for f in flows]
return [{
"repo": config["name"],
"known_entry_points": len(known),
"detected_entry_points": found,
"recall": round(found / max(len(known), 1), 3),
"detected_flows": count,
"avg_flow_depth": round(sum(depths) / max(len(depths), 1), 1),
"max_flow_depth": max(depths, default=0),
}]
@@ -0,0 +1,220 @@
"""Impact accuracy benchmark: measures precision/recall of change impact analysis.
Two ground-truth modes are emitted side by side (``ground_truth_mode`` column):
- **graph-derived (circular — upper bound)** — the historical mode. Ground
truth is the changed files plus files with CALLS/IMPORTS_FROM edges into
them, i.e. derived from the same graph the predictor traverses. Recall in
this mode is an upper bound by construction, not independent evidence.
- **co-change (same commit, seed excluded)** — the honest mode. The predictor
is seeded with a single changed file and graded against the *other* files
the author actually touched in the same commit. The ground truth comes from
git history, not from the graph.
Failure semantics: if ``analyze_changes`` throws, the row is recorded with
``status="error"`` and empty metric fields — it stays in the CSV but is
excluded from aggregates. (Previously a failure silently set
``predicted = set(changed)``, guaranteeing a fake recall of 1.0.)
"""
from __future__ import annotations
import logging
import statistics
import subprocess
from pathlib import Path
logger = logging.getLogger(__name__)
MODE_GRAPH_DERIVED = "graph-derived (circular — upper bound)"
MODE_CO_CHANGE = "co-change (same commit, seed excluded)"
def _get_changed_files(repo_path: Path, sha: str) -> list[str]:
"""Get list of changed files for a commit."""
result = subprocess.run(
["git", "diff", "--name-only", f"{sha}~1", sha],
cwd=str(repo_path),
capture_output=True,
text=True,
)
if result.returncode != 0:
result = subprocess.run(
["git", "diff", "--name-only", "HEAD~1", "HEAD"],
cwd=str(repo_path),
capture_output=True,
text=True,
)
return [f.strip() for f in result.stdout.strip().splitlines() if f.strip()]
def _files_from_analysis(analysis: dict) -> set[str]:
"""Extract predicted file paths from an ``analyze_changes`` result."""
predicted: set[str] = set()
for f in analysis.get("changed_functions", []):
if isinstance(f, dict) and "file_path" in f:
predicted.add(f["file_path"])
elif isinstance(f, dict) and "file" in f:
predicted.add(f["file"])
for flow in analysis.get("affected_flows", []):
if isinstance(flow, dict):
for node in flow.get("nodes", []):
if isinstance(node, dict) and "file_path" in node:
predicted.add(node["file_path"])
return predicted
def _graph_neighbor_files(store, files: list[str]) -> set[str]:
"""Files with CALLS/IMPORTS_FROM edges into any node of *files* (one hop)."""
out: set[str] = set()
for f in files:
for node in store.get_nodes_by_file(f):
for edge in store.get_edges_by_target(node.qualified_name):
if edge.kind in ("CALLS", "IMPORTS_FROM"):
src_qual = edge.source_qualified
src_file = src_qual.split("::")[0] if "::" in src_qual else ""
if src_file:
out.add(src_file)
return out
def _base_row(repo: str, sha: str, mode: str, seed: str) -> dict:
return {
"repo": repo,
"commit": sha,
"ground_truth_mode": mode,
"seed_file": seed,
"predicted_files": "",
"actual_files": "",
"true_positives": "",
"precision": "",
"recall": "",
"f1": "",
"status": "ok",
"error": "",
}
def _scored_row(
repo: str, sha: str, mode: str, seed: str,
predicted: set[str], actual: set[str],
) -> dict:
tp = len(predicted & actual)
precision = tp / max(len(predicted), 1)
recall = tp / max(len(actual), 1)
f1 = 2 * precision * recall / max(precision + recall, 0.001)
row = _base_row(repo, sha, mode, seed)
row.update({
"predicted_files": len(predicted),
"actual_files": len(actual),
"true_positives": tp,
"precision": round(precision, 3),
"recall": round(recall, 3),
"f1": round(f1, 3),
})
return row
def _error_row(repo: str, sha: str, mode: str, seed: str, exc: Exception) -> dict:
row = _base_row(repo, sha, mode, seed)
row["status"] = "error"
row["error"] = str(exc)[:200]
return row
def run(repo_path: Path, store, config: dict) -> list[dict]:
"""Run impact accuracy benchmark (both ground-truth modes)."""
from code_review_graph.changes import analyze_changes
results = []
repo = config["name"]
for tc in config.get("test_commits", []):
sha = tc["sha"]
changed = _get_changed_files(repo_path, sha)
if not changed:
continue
# --- Mode 1: graph-derived ground truth (circular — upper bound) ---
try:
analysis = analyze_changes(
store, changed, repo_root=str(repo_path), base=sha + "~1",
)
except Exception as exc:
# Old behaviour set predicted = set(changed) here, which
# guarantees recall 1.0 on a *failed* run. Mark failed instead.
logger.warning("analyze_changes failed on %s: %s", sha, exc)
results.append(_error_row(repo, sha, MODE_GRAPH_DERIVED, "", exc))
analysis = None
if analysis is not None:
predicted = set(changed) | _files_from_analysis(analysis)
actual = set(changed) | _graph_neighbor_files(store, changed)
results.append(
_scored_row(repo, sha, MODE_GRAPH_DERIVED, "", predicted, actual)
)
# --- Mode 2: co-change ground truth (honest) ---
# Seed the predictor with a single changed file and grade against
# the other files the author touched in the same commit. Note the
# seed analysis deliberately gets no repo_root/diff: it must only
# see the seed file, never the full commit diff.
seed = sorted(changed)[0]
co_actual = set(changed) - {seed}
if not co_actual:
row = _base_row(repo, sha, MODE_CO_CHANGE, seed)
row["status"] = "skipped"
row["error"] = "single-file commit: no co-changed files to grade against"
results.append(row)
continue
try:
seed_analysis = analyze_changes(store, [seed])
except Exception as exc:
logger.warning("analyze_changes (seed=%s) failed on %s: %s", seed, sha, exc)
results.append(_error_row(repo, sha, MODE_CO_CHANGE, seed, exc))
continue
co_predicted = _files_from_analysis(seed_analysis)
co_predicted |= _graph_neighbor_files(store, [seed])
co_predicted.discard(seed)
results.append(
_scored_row(repo, sha, MODE_CO_CHANGE, seed, co_predicted, co_actual)
)
return results
def aggregate(results: list[dict]) -> dict:
"""Per-mode means over successful rows only.
Error/skipped rows stay in the CSV but never contribute to a number.
"""
out: dict = {
"total_rows": len(results),
"error_rows": sum(1 for r in results if r.get("status") == "error"),
"skipped_rows": sum(1 for r in results if r.get("status") == "skipped"),
}
for key, mode in (
("graph_derived", MODE_GRAPH_DERIVED),
("co_change", MODE_CO_CHANGE),
):
rows = [
r for r in results
if r.get("ground_truth_mode") == mode and r.get("status") == "ok"
]
out[key] = {
"ok_rows": len(rows),
"mean_precision": (
round(statistics.mean(float(r["precision"]) for r in rows), 3)
if rows else None
),
"mean_recall": (
round(statistics.mean(float(r["recall"]) for r in rows), 3)
if rows else None
),
"mean_f1": (
round(statistics.mean(float(r["f1"]) for r in rows), 3)
if rows else None
),
}
return out
@@ -0,0 +1,125 @@
"""Multi-hop retrieval benchmark.
Tests a two-step tool chain that mimics how an LLM agent actually uses the
graph for complex tasks:
1. ``hybrid_search(nl_query)`` to find a starting anchor from a natural-
language question.
2. ``query_graph(pattern, target=anchor)`` to traverse one hop along the
requested edge kind (callers_of / callees_of / tests_for / ...).
For each task the benchmark records:
- ``anchor_found`` — did semantic search return a node whose qualified_name
ends with the expected suffix in the top-K?
- ``anchor_rank`` — index in the search result list (lower is better).
- ``neighbor_count`` — number of neighbors returned by the traversal.
- ``neighbor_recall`` — fraction of ``expected_neighbor_names`` that appear
among the neighbor names.
- ``score`` — ``int(anchor_found) * neighbor_recall``. Range 01.
Tasks are defined per-config under ``multi_hop_tasks:`` in
``code_review_graph/eval/configs/*.yaml``. See
``docs/REPRODUCING.md`` for the schema and the curated canonical task set.
"""
from __future__ import annotations
import logging
from pathlib import Path
from typing import Any
logger = logging.getLogger(__name__)
def _name_set(rows: list[dict[str, Any]]) -> set[str]:
out: set[str] = set()
for r in rows:
name = (r.get("name") or "").lower()
if name:
out.add(name)
return out
def run(repo_path: Path, store, config: dict) -> list[dict]:
"""Run the multi-hop retrieval benchmark for one repo."""
# Imports are local so an import-time failure in one optional benchmark
# does not poison the whole runner.
from code_review_graph.search import hybrid_search
from code_review_graph.tools.query import query_graph
repo_root = str(repo_path)
results: list[dict] = []
for task in config.get("multi_hop_tasks", []):
task_id = task["id"]
nl_query = task["nl_query"]
suffix = task["anchor_qualified_suffix"].lower()
traversal = task.get("traversal_pattern", "callers_of")
expected = [e.lower() for e in task.get("expected_neighbor_names", [])]
k = int(task.get("k", 10))
# Step 1 — semantic search
try:
hits = hybrid_search(store, nl_query, limit=k)
except Exception as exc: # noqa: BLE001 — benchmark must not abort the runner
logger.warning("hybrid_search failed on %s: %s", task_id, exc)
hits = []
anchor = None
anchor_rank = -1
for i, h in enumerate(hits):
qn = (h.get("qualified_name") or "").lower()
if qn.endswith(suffix):
anchor = h
anchor_rank = i
break
if anchor is None:
results.append({
"repo": config["name"],
"task_id": task_id,
"nl_query": nl_query,
"anchor_found": False,
"anchor_rank": -1,
"neighbor_count": 0,
"expected_count": len(expected),
"matched_count": 0,
"neighbor_recall": 0.0,
"score": 0.0,
})
continue
# Step 2 — single-hop graph traversal from the anchor
try:
trav = query_graph(
pattern=traversal,
target=anchor["qualified_name"],
repo_root=repo_root,
detail_level="standard",
)
except Exception as exc: # noqa: BLE001
logger.warning(
"query_graph(%s) failed on %s: %s", traversal, task_id, exc,
)
trav = {}
rows = trav.get("data") or trav.get("results") or []
names = _name_set(rows)
matched = sum(1 for e in expected if e in names)
recall = matched / len(expected) if expected else 0.0
results.append({
"repo": config["name"],
"task_id": task_id,
"nl_query": nl_query,
"anchor_found": True,
"anchor_rank": anchor_rank,
"neighbor_count": len(rows),
"expected_count": len(expected),
"matched_count": matched,
"neighbor_recall": round(recall, 3),
"score": round(recall, 3),
})
return results
@@ -0,0 +1,59 @@
"""Search quality benchmark: measures search result ranking via MRR."""
from __future__ import annotations
import logging
import sqlite3
from pathlib import Path
logger = logging.getLogger(__name__)
def run(repo_path: Path, store, config: dict) -> list[dict]:
"""Run search quality benchmark."""
results = []
for sq in config.get("search_queries", []):
query = sq["query"]
expected = sq["expected"]
try:
from code_review_graph.search import hybrid_search
search_results = hybrid_search(store, query, limit=20)
except (ImportError, sqlite3.OperationalError) as exc:
logger.debug("hybrid_search unavailable, using fallback: %s", exc)
# Fallback to basic search
search_results = [
{"qualified_name": n.qualified_name}
for n in store.search_nodes(query, limit=20)
]
rank = 0
for i, r in enumerate(search_results):
if isinstance(r, dict):
qn = r.get("qualified_name", "")
elif hasattr(r, "qualified_name"):
qn = r.qualified_name
else:
qn = ""
qn_lower = qn.lower()
exp_lower = expected.lower()
# Match if expected is substring of qn, qn is substring of expected,
# or the name part after :: matches
exp_name = expected.rsplit("::", 1)[-1] if "::" in expected else expected
qn_name = qn.rsplit("::", 1)[-1] if "::" in qn else qn
if (
exp_lower in qn_lower
or qn_lower in exp_lower
or exp_name.lower() == qn_name.lower()
):
rank = i + 1
break
results.append({
"repo": config["name"],
"query": query,
"expected": expected,
"rank": rank,
"reciprocal_rank": round(1.0 / rank if rank > 0 else 0.0, 3),
})
return results
@@ -0,0 +1,143 @@
"""Token efficiency benchmark: compares naive, standard, and graph-based token counts.
Failure semantics: if ``get_review_context`` throws, the row is recorded with
``status="error"`` and empty metric fields. It stays in the CSV for forensics
but is excluded from every aggregate — a failed tool call is not a
measurement. (Previously a failure silently produced ``graph_tokens=0`` and
``ratio = naive / 1``, inflating the results.)
"""
from __future__ import annotations
import json
import logging
import statistics
import subprocess
from pathlib import Path
logger = logging.getLogger(__name__)
def _count_tokens(text: str) -> int:
"""Approximate token count (1 token ~ 4 chars)."""
return len(text) // 4
def _get_changed_files(repo_path: Path, sha: str) -> list[str]:
"""Get list of changed files for a commit."""
result = subprocess.run(
["git", "diff", "--name-only", f"{sha}~1", sha],
cwd=str(repo_path),
capture_output=True,
text=True,
)
if result.returncode != 0:
# Fallback: diff against parent
result = subprocess.run(
["git", "diff", "--name-only", "HEAD~1", "HEAD"],
cwd=str(repo_path),
capture_output=True,
text=True,
)
return [f.strip() for f in result.stdout.strip().splitlines() if f.strip()]
def _count_file_tokens(repo_path: Path, files: list[str]) -> int:
"""Count tokens from full file contents (naive approach)."""
total = 0
for f in files:
fp = repo_path / f
if fp.is_file():
try:
total += _count_tokens(fp.read_text(encoding="utf-8", errors="replace"))
except OSError:
pass
return total
def _count_diff_tokens(repo_path: Path, sha: str) -> int:
"""Count tokens from git diff output (standard approach)."""
result = subprocess.run(
["git", "diff", f"{sha}~1", sha],
cwd=str(repo_path),
capture_output=True,
text=True,
)
if result.returncode != 0:
result = subprocess.run(
["git", "diff", "HEAD~1", "HEAD"],
cwd=str(repo_path),
capture_output=True,
text=True,
)
return _count_tokens(result.stdout)
def run(repo_path: Path, store, config: dict) -> list[dict]:
"""Run token efficiency benchmark."""
results = []
for tc in config.get("test_commits", []):
changed = _get_changed_files(repo_path, tc["sha"])
if not changed:
continue
naive_tokens = _count_file_tokens(repo_path, changed)
standard_tokens = _count_diff_tokens(repo_path, tc["sha"])
row: dict = {
"repo": config["name"],
"commit": tc["sha"],
"description": tc.get("description", ""),
"changed_files": len(changed),
"naive_tokens": naive_tokens,
"standard_tokens": standard_tokens,
"graph_tokens": "",
"naive_to_graph_ratio": "",
"standard_to_graph_ratio": "",
"status": "ok",
"error": "",
}
# Graph-based: use get_review_context
try:
from code_review_graph.tools import get_review_context
ctx = get_review_context(
changed_files=changed, repo_root=str(repo_path)
)
graph_tokens = _count_tokens(json.dumps(ctx))
except Exception as exc:
# A failed tool call is not a measurement. Recording
# graph_tokens=0 used to turn this into ratio = naive/1 — a
# huge fake win. Mark the row failed; aggregate() excludes it.
logger.warning("get_review_context failed on %s: %s", tc["sha"], exc)
row["status"] = "error"
row["error"] = str(exc)[:200]
results.append(row)
continue
row["graph_tokens"] = graph_tokens
row["naive_to_graph_ratio"] = round(naive_tokens / max(graph_tokens, 1), 1)
row["standard_to_graph_ratio"] = round(standard_tokens / max(graph_tokens, 1), 1)
results.append(row)
return results
def aggregate(results: list[dict]) -> dict:
"""Aggregate token-efficiency rows, excluding failed measurements.
Rows with ``status != "ok"`` stay in the CSV for forensics but must not
contribute to any headline number.
"""
ok = [r for r in results if r.get("status") == "ok"]
ratios = [float(r["naive_to_graph_ratio"]) for r in ok]
return {
"total_rows": len(results),
"ok_rows": len(ok),
"error_rows": sum(1 for r in results if r.get("status") == "error"),
"median_naive_to_graph_ratio": (
round(statistics.median(ratios), 1) if ratios else None
),
"mean_naive_to_graph_ratio": (
round(statistics.mean(ratios), 1) if ratios else None
),
}
@@ -0,0 +1,50 @@
name: code-review-graph
url: https://github.com/tirth8205/code-review-graph
# Pinned to the latest test_commit SHA so the snapshot is deterministic and
# every test_commit below is reachable as an ancestor. (This config replaces
# the historical "nextjs" entry, which used the same URL but mis-labelled the
# target as a Next.js monorepo.)
commit: 84bde35459c52e1e0c4b25c6c4799743021e0fc7
language: python
size_category: medium
test_commits:
- sha: 528801f841e519567ef54d6e52e9b9831d162e1b
description: "feat: add multi-platform MCP server installation support"
changed_files: 3
- sha: 84bde35459c52e1e0c4b25c6c4799743021e0fc7
description: "feat: add Google Antigravity platform support for MCP install"
changed_files: 2
entry_points:
- "code_review_graph/cli.py::cli"
- "code_review_graph/main.py::main"
search_queries:
- query: "GraphStore nodes"
expected: "code_review_graph/graph.py::GraphStore"
- query: "parse AST"
expected: "code_review_graph/parser.py::CodeParser"
- query: "full build"
expected: "code_review_graph/incremental.py::full_build"
multi_hop_tasks:
- id: crg-parse-file-callers
nl_query: "Who invokes the parser entry point on a single source file"
anchor_qualified_suffix: "code_review_graph/parser.py::codeparser.parse_file"
traversal_pattern: callers_of
expected_neighbor_names: ["setup_method"]
k: 10
- id: crg-upsert-node-callers
nl_query: "Where the graph store inserts or updates a node"
anchor_qualified_suffix: "code_review_graph/graph.py::graphstore.upsert_node"
traversal_pattern: callers_of
expected_neighbor_names: ["store_file_nodes_edges"]
k: 10
# Questions for the agent_baseline benchmark (pure-python grep top-k vs graph
# query). See docs/REPRODUCING.md for the methodology.
agent_questions:
- "How does GraphStore upsert_node store a node"
- "Where does full_build parse the repository"
- "How does hybrid_search rank search results"
@@ -0,0 +1,45 @@
name: express
url: https://github.com/expressjs/express
# Pinned to the latest test_commit SHA so the snapshot is deterministic and
# every test_commit below is reachable as an ancestor.
commit: b4ab7d65d7724d9309b6faaaf82ad492da2a6d35
language: javascript
size_category: small
test_commits:
- sha: 925a1dff1e42f1b393c977b8b77757fcf633e09f
description: "fix: bump qs minimum to ^6.14.2 for CVE-2026-2391"
changed_files: 1
- sha: b4ab7d65d7724d9309b6faaaf82ad492da2a6d35
description: "test: include edge case tests for res.type()"
changed_files: 1
entry_points:
- "lib/application.js::app.handle"
- "lib/express.js::createApplication"
search_queries:
- query: "app handle"
expected: "lib/application.js::app"
- query: "response send"
expected: "lib/response.js::res"
- query: "request"
expected: "lib/request.js::req"
# Express has only one task — JS modules use prototypes + module.exports
# heavily, so most "method" callers are not represented as proper Function
# edges in the graph. createApplication is the cleanest anchor.
multi_hop_tasks:
- id: express-create-application-callees
nl_query: "What express does when constructing an application"
anchor_qualified_suffix: "lib/express.js::createapplication"
traversal_pattern: callees_of
expected_neighbor_names: ["mixin", "create", "init"]
k: 10
# Questions for the agent_baseline benchmark (pure-python grep top-k vs graph
# query). See docs/REPRODUCING.md for the methodology.
agent_questions:
- "How does app.handle process the middleware stack"
- "Where does res.send write the response body"
- "How does createApplication initialize an app"
@@ -0,0 +1,48 @@
name: fastapi
url: https://github.com/tiangolo/fastapi
# Pinned to the latest test_commit SHA so the snapshot is deterministic and
# every test_commit below is reachable as an ancestor.
commit: 0227991a01e61bf5cdd93cc00e9e243f52b47a4a
language: python
size_category: medium
test_commits:
- sha: fa3588c38c7473aca7536b12d686102de4b0f407
description: "Fix typo for client_secret in OAuth2 form docstrings"
changed_files: 1
- sha: 0227991a01e61bf5cdd93cc00e9e243f52b47a4a
description: "Exclude spam comments from statistics in scripts/people.py"
changed_files: 1
entry_points:
- "fastapi/applications.py::FastAPI"
- "fastapi/routing.py::APIRouter"
search_queries:
- query: "FastAPI application"
expected: "fastapi/applications.py::FastAPI"
- query: "APIRoute routing"
expected: "fastapi/routing.py::APIRoute"
- query: "Depends injection"
expected: "fastapi/params.py::Depends"
multi_hop_tasks:
- id: fastapi-route-handler-callers
nl_query: "How fastapi binds a route handler to an APIRoute"
anchor_qualified_suffix: "fastapi/routing.py::apiroute.get_route_handler"
traversal_pattern: callers_of
expected_neighbor_names: ["__init__"]
k: 10
- id: fastapi-get-dependant-callers
nl_query: "Where fastapi resolves dependency declarations into a tree"
anchor_qualified_suffix: "fastapi/dependencies/utils.py::get_dependant"
traversal_pattern: callers_of
expected_neighbor_names: ["get_parameterless_sub_dependant", "solve_dependencies"]
k: 10
# Questions for the agent_baseline benchmark (pure-python grep top-k vs graph
# query). See docs/REPRODUCING.md for the methodology.
agent_questions:
- "How does include_router register routes on the application"
- "Where does APIRoute build its route handler"
- "How does solve_dependencies resolve Depends parameters"
+50
View File
@@ -0,0 +1,50 @@
name: flask
url: https://github.com/pallets/flask
# Pinned to the latest test_commit SHA so the snapshot is deterministic and
# every test_commit below is reachable as an ancestor.
commit: a29f88ce6f2f9843bd6fcbbfce1390a2071965d6
language: python
size_category: small
test_commits:
- sha: fbb6f0bc4c60a0bada0e03c3480d0ccf30a3c1df
description: "all teardown callbacks are called despite errors"
changed_files: 10
- sha: a29f88ce6f2f9843bd6fcbbfce1390a2071965d6
description: "document that headers must be set before streaming"
changed_files: 4
entry_points:
- "src/flask/app.py::Flask.wsgi_app"
- "src/flask/sansio/app.py::App.add_url_rule"
search_queries:
- query: "Flask wsgi"
expected: "src/flask/app.py::Flask"
- query: "AppContext globals"
expected: "src/flask/ctx.py::AppContext"
- query: "create logger"
expected: "src/flask/logging.py::create_logger"
# Multi-hop retrieval tasks (semantic_search → query_graph one-hop)
# See docs/REPRODUCING.md for the schema.
multi_hop_tasks:
- id: flask-dispatch-callers
nl_query: "Where Flask dispatches HTTP requests"
anchor_qualified_suffix: "src/flask/app.py::flask.dispatch_request"
traversal_pattern: callers_of
expected_neighbor_names: ["full_dispatch_request"]
k: 10
- id: flask-exception-callers
nl_query: "Where Flask handles uncaught exceptions"
anchor_qualified_suffix: "src/flask/app.py::flask.handle_exception"
traversal_pattern: callers_of
expected_neighbor_names: ["wsgi_app"]
k: 10
# Questions for the agent_baseline benchmark (pure-python grep top-k vs graph
# query). See docs/REPRODUCING.md for the methodology.
agent_questions:
- "How does dispatch_request route an incoming HTTP request"
- "Where is the AppContext pushed and popped"
- "How does create_logger configure application logging"
+51
View File
@@ -0,0 +1,51 @@
name: gin
url: https://github.com/gin-gonic/gin
# Pinned to the latest test_commit SHA so the snapshot is deterministic and
# every test_commit below is reachable as an ancestor.
commit: 5c00df8afadd06cc5be530dde00fe6d9fa4a2e4a
language: go
size_category: small
test_commits:
- sha: 052d1a79aafe3f04078a2716f8e77d4340308383
description: "feat(render): add PDF renderer and tests"
changed_files: 5
- sha: 472d086af2acd924cb4b9d7be0525f7d790f69bc
description: "fix(tree): panic in findCaseInsensitivePathRec with RedirectFixedPath"
changed_files: 2
- sha: 5c00df8afadd06cc5be530dde00fe6d9fa4a2e4a
description: "fix(render): write content length in Data.Render"
changed_files: 2
entry_points:
- "gin.go::Engine"
- "routergroup.go::RouterGroup"
search_queries:
- query: "Engine ServeHTTP"
expected: "gin.go::Engine"
- query: "Context request"
expected: "context.go::Context"
- query: "node tree"
expected: "tree.go::node"
multi_hop_tasks:
- id: gin-serve-http-callees
nl_query: "What does the gin engine do when serving an HTTP request"
anchor_qualified_suffix: "gin.go::engine.servehttp"
traversal_pattern: callees_of
expected_neighbor_names: ["reset"]
k: 10
- id: gin-context-next-callers
nl_query: "Who advances the gin middleware chain via Context.Next"
anchor_qualified_suffix: "context.go::context.next"
traversal_pattern: callers_of
expected_neighbor_names: ["handleHTTPRequest", "serveError"]
k: 10
# Questions for the agent_baseline benchmark (pure-python grep top-k vs graph
# query). See docs/REPRODUCING.md for the methodology.
agent_questions:
- "How does Engine.ServeHTTP route an incoming request"
- "Where does Context.Next advance the middleware chain"
- "How does the node tree match wildcard routes"
+48
View File
@@ -0,0 +1,48 @@
name: httpx
url: https://github.com/encode/httpx
# Pinned to the latest test_commit SHA so the snapshot is deterministic and
# every test_commit below is reachable as an ancestor.
commit: b55d4635701d9dc22928ee647880c76b078ba3f2
language: python
size_category: small
test_commits:
- sha: ae1b9f66238f75ced3ced5e4485408435de10768
description: "Expose FunctionAuth in __all__"
changed_files: 3
- sha: b55d4635701d9dc22928ee647880c76b078ba3f2
description: "Upgrade Python type checker mypy"
changed_files: 4
entry_points:
- "httpx/_client.py::Client"
- "httpx/_client.py::AsyncClient"
search_queries:
- query: "Client request"
expected: "httpx/_client.py::Client"
- query: "Response headers"
expected: "httpx/_models.py::Response"
- query: "BaseClient"
expected: "httpx/_client.py::BaseClient"
multi_hop_tasks:
- id: httpx-client-request-callers
nl_query: "Which HTTP verbs route through the httpx Client.request"
anchor_qualified_suffix: "httpx/_client.py::client.request"
traversal_pattern: callers_of
expected_neighbor_names: ["get", "options", "head", "post", "put", "patch"]
k: 10
- id: httpx-async-request-tests
nl_query: "Tests covering the httpx async client request method"
anchor_qualified_suffix: "httpx/_client.py::asyncclient.request"
traversal_pattern: callers_of
expected_neighbor_names: ["test_raise_for_status"]
k: 10
# Questions for the agent_baseline benchmark (pure-python grep top-k vs graph
# query). See docs/REPRODUCING.md for the methodology.
agent_questions:
- "How does Client.request send an HTTP request"
- "Where are Response headers parsed and decoded"
- "How does BaseClient build request URLs"
+301
View File
@@ -0,0 +1,301 @@
"""Markdown report generator for evaluation benchmark results.
Takes a list of benchmark result dicts and produces a formatted markdown table
suitable for inclusion in documentation or CI output.
"""
from __future__ import annotations
import csv
from pathlib import Path
from typing import Any
def generate_markdown_report(results: list[dict[str, Any]]) -> str:
"""Generate a markdown report from benchmark results.
Each result dict should contain at minimum a ``benchmark`` key identifying
the benchmark name, plus any metric keys (e.g. ``ratio``,
``reduction_percent``, ``mrr``, ``precision``, ``recall``, ``f1``).
Args:
results: List of result dicts from benchmark runs.
Returns:
A markdown string containing a summary table and per-benchmark details.
"""
if not results:
return "# Evaluation Report\n\nNo benchmark results to report.\n"
lines: list[str] = []
lines.append("# Evaluation Report")
lines.append("")
# Collect all metric keys across results (excluding 'benchmark')
all_keys: list[str] = []
seen: set[str] = set()
for r in results:
for k in r:
if k != "benchmark" and k not in seen:
all_keys.append(k)
seen.add(k)
# Summary table
lines.append("## Summary")
lines.append("")
header = "| Benchmark | " + " | ".join(all_keys) + " |"
separator = "| --- | " + " | ".join("---" for _ in all_keys) + " |"
lines.append(header)
lines.append(separator)
for r in results:
name = r.get("benchmark", "unknown")
values = [str(r.get(k, "-")) for k in all_keys]
lines.append(f"| {name} | " + " | ".join(values) + " |")
lines.append("")
# Per-benchmark detail sections
lines.append("## Details")
lines.append("")
for r in results:
name = r.get("benchmark", "unknown")
lines.append(f"### {name}")
lines.append("")
for k in all_keys:
v = r.get(k, "-")
lines.append(f"- **{k}**: {v}")
lines.append("")
return "\n".join(lines)
def _read_csvs(results_dir: Path, prefix: str) -> list[dict[str, str]]:
"""Read all CSV files matching a prefix from the results directory."""
rows: list[dict[str, str]] = []
for p in sorted(results_dir.glob(f"*_{prefix}_*.csv")):
with open(p, newline="") as f:
reader = csv.DictReader(f)
rows.extend(reader)
return rows
def _md_table(headers: list[str], rows: list[list[str]]) -> str:
"""Build a markdown table from headers and rows."""
lines = []
lines.append("| " + " | ".join(headers) + " |")
lines.append("| " + " | ".join("---" for _ in headers) + " |")
for row in rows:
lines.append("| " + " | ".join(row) + " |")
return "\n".join(lines)
def generate_full_report(results_dir: str | Path) -> str:
"""Generate a full markdown evaluation report from CSV result files.
Reads all CSV files in *results_dir*, groups them by benchmark type,
and produces a markdown report with methodology notes and per-benchmark
result tables.
Args:
results_dir: Directory containing CSV result files.
Returns:
Markdown string with the full report.
"""
results_dir = Path(results_dir)
lines: list[str] = []
lines.append("# Evaluation Report")
lines.append("")
lines.append("## Methodology")
lines.append("")
lines.append("Benchmarks are run against real open-source repositories.")
lines.append("Token counts use a consistent `len(text) // 4` approximation.")
lines.append(
"Impact accuracy reports two ground-truth modes: "
"graph-derived (circular — upper bound) and co-change "
"(files co-changed in the same commit, seed excluded)."
)
lines.append(
"Rows with `status=error` are kept for forensics but excluded "
"from all aggregates."
)
lines.append("")
benchmark_types = [
"token_efficiency",
"impact_accuracy",
"agent_baseline",
"flow_completeness",
"search_quality",
"build_performance",
"multi_hop_retrieval",
]
for btype in benchmark_types:
rows = _read_csvs(results_dir, btype)
if not rows:
continue
title = btype.replace("_", " ").title()
lines.append(f"## {title}")
lines.append("")
headers = list(rows[0].keys())
table_rows = [[r.get(h, "-") for h in headers] for r in rows]
lines.append(_md_table(headers, table_rows))
lines.append("")
if len(lines) <= 6:
lines.append("No benchmark results found.")
lines.append("")
return "\n".join(lines)
def generate_readme_tables(results_dir: str | Path) -> str:
"""Generate concise README-ready tables from CSV result files.
Produces three tables:
- Table A: Token Efficiency
- Table B: Accuracy & Quality
- Table C: Performance
Args:
results_dir: Directory containing CSV result files.
Returns:
Markdown string with the three tables.
"""
results_dir = Path(results_dir)
lines: list[str] = []
# Table A: Token Efficiency
te_rows = _read_csvs(results_dir, "token_efficiency")
if te_rows:
lines.append("### Token Efficiency")
lines.append("")
headers = [
"Repo", "Files", "Naive Tokens", "Standard Tokens",
"Graph Tokens", "Naive/Graph", "Std/Graph",
]
table_rows = []
for r in te_rows:
table_rows.append([
r.get("repo", "-"),
r.get("changed_files", "-"),
r.get("naive_tokens", "-"),
r.get("standard_tokens", "-"),
r.get("graph_tokens", "-"),
r.get("naive_to_graph_ratio", "-"),
r.get("standard_to_graph_ratio", "-"),
])
lines.append(_md_table(headers, table_rows))
lines.append("")
# Table B: Accuracy & Quality
ia_rows = _read_csvs(results_dir, "impact_accuracy")
fc_rows = _read_csvs(results_dir, "flow_completeness")
sq_rows = _read_csvs(results_dir, "search_quality")
if ia_rows or fc_rows or sq_rows:
lines.append("### Accuracy & Quality")
lines.append("")
headers = ["Repo", "Impact F1 (graph-derived)", "Flow Recall", "Search MRR"]
# Build a per-repo summary
repo_data: dict[str, dict[str, object]] = {}
mrr_accum: dict[str, list[float]] = {}
f1_accum: dict[str, list[float]] = {}
for r in ia_rows:
# Failed rows are kept in the CSV for forensics but must never
# contribute to a headline number; co-change rows are a
# different metric and get their own reporting.
if r.get("status", "ok") not in ("", "ok"):
continue
mode = r.get("ground_truth_mode", "")
if mode and not mode.startswith("graph-derived"):
continue
repo = r.get("repo", "?")
repo_data.setdefault(repo, {})
try:
f1_accum.setdefault(repo, []).append(float(r.get("f1", "")))
except (ValueError, TypeError):
pass
for r in fc_rows:
repo_data.setdefault(r.get("repo", "?"), {})["recall"] = r.get("recall", "-")
for r in sq_rows:
repo = r.get("repo", "?")
repo_data.setdefault(repo, {})
try:
mrr_accum.setdefault(repo, []).append(float(r.get("reciprocal_rank", 0)))
except (ValueError, TypeError):
pass
table_rows = []
for repo, d in sorted(repo_data.items()):
mrr_vals = mrr_accum.get(repo, [])
mrr = (
str(round(sum(mrr_vals) / len(mrr_vals), 3))
if mrr_vals
else "-"
)
f1_vals = f1_accum.get(repo, [])
f1 = (
str(round(sum(f1_vals) / len(f1_vals), 3))
if f1_vals
else "-"
)
table_rows.append([
repo,
f1,
str(d.get("recall", "-")),
mrr,
])
lines.append(_md_table(headers, table_rows))
lines.append("")
# Table B2: Agent Baseline (grep top-k vs graph query)
ab_rows = _read_csvs(results_dir, "agent_baseline")
if ab_rows:
lines.append("### Agent Baseline (grep top-k vs graph query)")
lines.append("")
headers = [
"Repo", "Question", "Baseline Tokens", "Graph Tokens",
"Baseline/Graph", "Status",
]
table_rows = []
for r in ab_rows:
table_rows.append([
r.get("repo", "-"),
r.get("question", "-"),
r.get("baseline_tokens", "-"),
r.get("graph_tokens", "-"),
r.get("baseline_to_graph_ratio", "-"),
r.get("status", "ok") or "ok",
])
lines.append(_md_table(headers, table_rows))
lines.append("")
# Table C: Performance
bp_rows = _read_csvs(results_dir, "build_performance")
if bp_rows:
lines.append("### Performance")
lines.append("")
headers = ["Repo", "Files", "Nodes", "Flow Det. (s)", "Search (ms)"]
table_rows = []
for r in bp_rows:
table_rows.append([
r.get("repo", "-"),
r.get("file_count", "-"),
r.get("node_count", "-"),
r.get("flow_detection_seconds", "-"),
r.get("search_avg_ms", "-"),
])
lines.append(_md_table(headers, table_rows))
lines.append("")
if not lines:
return "No benchmark results found.\n"
return "\n".join(lines)
+211
View File
@@ -0,0 +1,211 @@
"""Evaluation runner: orchestrates benchmark execution across repositories."""
from __future__ import annotations
import csv
import logging
import subprocess
from datetime import date
from pathlib import Path
try:
import yaml # type: ignore[import-untyped]
except ImportError:
yaml = None # type: ignore[assignment]
from code_review_graph.eval.benchmarks import (
agent_baseline,
build_performance,
flow_completeness,
impact_accuracy,
multi_hop_retrieval,
search_quality,
token_efficiency,
)
logger = logging.getLogger(__name__)
BENCHMARK_REGISTRY = {
"token_efficiency": token_efficiency.run,
"impact_accuracy": impact_accuracy.run,
"flow_completeness": flow_completeness.run,
"search_quality": search_quality.run,
"build_performance": build_performance.run,
"multi_hop_retrieval": multi_hop_retrieval.run,
"agent_baseline": agent_baseline.run,
}
CONFIGS_DIR = Path(__file__).parent / "configs"
DEFAULT_OUTPUT = Path("evaluate/results")
DEFAULT_REPOS = Path("evaluate/test_repos")
def _require_yaml():
if yaml is None:
raise ImportError("pyyaml is required: pip install code-review-graph[eval]")
def load_config(name: str) -> dict:
"""Load a single benchmark config by name."""
_require_yaml()
path = CONFIGS_DIR / f"{name}.yaml"
with open(path) as f:
return yaml.safe_load(f)
def load_all_configs() -> list[dict]:
"""Load all benchmark configs from the configs directory."""
_require_yaml()
configs = []
for p in sorted(CONFIGS_DIR.glob("*.yaml")):
with open(p) as f:
configs.append(yaml.safe_load(f))
return configs
def clone_or_update(config: dict, repos_dir: Path | None = None) -> Path:
"""Clone or update a repository at the config's pinned ``commit`` SHA.
Full clones (no ``--depth``) are required: the pinned ``test_commits`` are
often older than any reasonable shallow-clone window, and a missed SHA
used to silently fall back to ``git diff HEAD~1 HEAD`` — producing
benchmark numbers tied to whatever upstream HEAD looked like that day.
Every subprocess call's exit status is checked; failures raise
``RuntimeError`` so reproducibility issues surface immediately instead of
yielding garbage results.
"""
repos_dir = repos_dir or DEFAULT_REPOS
repos_dir.mkdir(parents=True, exist_ok=True)
repo_path = repos_dir / config["name"]
if repo_path.exists():
proc = subprocess.run(
["git", "fetch", "--all", "--tags"],
cwd=str(repo_path),
capture_output=True,
text=True,
)
if proc.returncode != 0:
raise RuntimeError(
f"git fetch failed in {repo_path}: {proc.stderr.strip()}"
)
else:
proc = subprocess.run(
["git", "clone", config["url"], str(repo_path)],
capture_output=True,
text=True,
)
if proc.returncode != 0:
raise RuntimeError(
f"git clone failed for {config['url']}: {proc.stderr.strip()}"
)
commit = config.get("commit", "HEAD")
if commit != "HEAD":
proc = subprocess.run(
["git", "checkout", commit],
cwd=str(repo_path),
capture_output=True,
text=True,
)
if proc.returncode != 0:
raise RuntimeError(
f"git checkout {commit} failed in {repo_path}: "
f"{proc.stderr.strip()}"
)
return repo_path
def write_csv(results: list[dict], path: Path) -> None:
"""Write benchmark results to a CSV file."""
if not results:
return
path.parent.mkdir(parents=True, exist_ok=True)
fieldnames = list(results[0].keys())
with open(path, "w", newline="") as f:
writer = csv.DictWriter(f, fieldnames=fieldnames)
writer.writeheader()
writer.writerows(results)
def run_eval(
repos: list[str] | None = None,
benchmarks: list[str] | None = None,
output_dir: str | Path | None = None,
) -> dict[str, list[dict]]:
"""Run evaluation benchmarks across repositories.
Args:
repos: List of repo config names to evaluate (None = all).
benchmarks: List of benchmark names to run (None = all).
output_dir: Directory for CSV output files.
Returns:
Dict mapping ``{repo}_{benchmark}`` to list of result dicts.
"""
output_dir = Path(output_dir) if output_dir else DEFAULT_OUTPUT
output_dir.mkdir(parents=True, exist_ok=True)
if repos:
configs = [load_config(r) for r in repos]
else:
configs = load_all_configs()
benchmark_names = benchmarks or list(BENCHMARK_REGISTRY.keys())
all_results: dict[str, list[dict]] = {}
today = date.today().isoformat()
for config in configs:
name = config["name"]
logger.info("Evaluating %s...", name)
# Resolve the repo path to an absolute Path before handing it to
# full_build / get_db_path so the stored qualified_names match what
# the CLI/MCP layer produces (those paths go through _get_store ->
# _validate_repo_root which .resolve()s). Without this, a later
# ``code-review-graph update --repo <relative>`` writes the same
# function under a new absolute-prefixed qualified_name, leaving the
# graph with duplicate nodes for the same source location.
repo_path = clone_or_update(config).resolve()
# Build graph
from code_review_graph.graph import GraphStore
from code_review_graph.incremental import full_build, get_db_path
from code_review_graph.postprocessing import run_post_processing
db_path = get_db_path(repo_path)
store = GraphStore(db_path)
full_build(repo_path, store)
# full_build is the parsing-only primitive; the higher-level CLI/MCP
# wrappers run postprocessing on top. The eval framework bypasses
# those, so call it directly here. Without this, FTS5 stays empty
# and downstream benchmarks (token_efficiency, search_quality)
# silently produce useless results. See: search.rebuild_fts_index.
pp_result = run_post_processing(store)
for warning in pp_result.get("warnings", []):
logger.warning(" postprocessing: %s", warning)
for bench_name in benchmark_names:
if bench_name not in BENCHMARK_REGISTRY:
logger.warning("Unknown benchmark: %s", bench_name)
continue
logger.info(" Running %s...", bench_name)
try:
bench_fn = BENCHMARK_REGISTRY[bench_name]
results = bench_fn(repo_path, store, config)
key = f"{name}_{bench_name}"
all_results[key] = results
write_csv(results, output_dir / f"{key}_{today}.csv")
logger.info(" %s: %d result(s)", bench_name, len(results))
except Exception as e:
logger.error(" %s failed: %s", bench_name, e)
all_results[f"{name}_{bench_name}"] = []
store.close()
return all_results
+85
View File
@@ -0,0 +1,85 @@
"""Scoring metrics for evaluating graph-based code review quality.
Provides:
- Token efficiency: measures how many tokens the graph saves vs raw context.
- Mean Reciprocal Rank (MRR): evaluates ranking quality for search results.
- Precision / Recall / F1: evaluates set-based retrieval accuracy.
"""
from __future__ import annotations
def compute_token_efficiency(raw_tokens: int, graph_tokens: int) -> dict:
"""Compute token efficiency metrics.
Args:
raw_tokens: Number of tokens when sending raw source code.
graph_tokens: Number of tokens when using graph-based context.
Returns:
Dict with keys:
- raw_tokens: the raw token count
- graph_tokens: the graph token count
- ratio: graph_tokens / raw_tokens (lower is better)
- reduction_percent: percentage of tokens saved (higher is better)
"""
if raw_tokens <= 0:
return {
"raw_tokens": raw_tokens,
"graph_tokens": graph_tokens,
"ratio": 0.0,
"reduction_percent": 0.0,
}
ratio = graph_tokens / raw_tokens
reduction = (1.0 - ratio) * 100.0
return {
"raw_tokens": raw_tokens,
"graph_tokens": graph_tokens,
"ratio": round(ratio, 4),
"reduction_percent": round(reduction, 2),
}
def compute_mrr(correct: str, results: list[str]) -> float:
"""Compute Mean Reciprocal Rank for a single query.
Args:
correct: The correct/expected result identifier.
results: Ordered list of result identifiers (best first).
Returns:
1/rank if *correct* is found in *results*, else 0.0.
"""
for i, r in enumerate(results, start=1):
if r == correct:
return 1.0 / i
return 0.0
def compute_precision_recall(predicted: set, actual: set) -> dict:
"""Compute precision, recall, and F1 score.
Args:
predicted: Set of predicted/returned items.
actual: Set of ground-truth items.
Returns:
Dict with keys: precision, recall, f1.
"""
if not predicted and not actual:
return {"precision": 1.0, "recall": 1.0, "f1": 1.0}
true_positive = len(predicted & actual)
precision = true_positive / len(predicted) if predicted else 0.0
recall = true_positive / len(actual) if actual else 0.0
if precision + recall > 0:
f1 = 2 * precision * recall / (precision + recall)
else:
f1 = 0.0
return {
"precision": round(precision, 4),
"recall": round(recall, 4),
"f1": round(f1, 4),
}
+182
View File
@@ -0,0 +1,182 @@
"""Measures total tokens consumed by agent workflows against benchmark repos."""
from __future__ import annotations
import json
import logging
from typing import Any, Callable
logger = logging.getLogger(__name__)
def estimate_tokens(obj: Any) -> int:
"""Estimate token count from JSON-serializable object.
Uses character count / 4 as a rough approximation for English + code.
"""
return len(json.dumps(obj, default=str)) // 4
def benchmark_review_workflow(repo_root: str, base: str = "HEAD~1") -> dict:
"""Simulate a review workflow and measure total tokens consumed."""
from ..tools.context import get_minimal_context
from ..tools.review import detect_changes_func
total_tokens = 0
calls = []
# Step 1: get_minimal_context
result = get_minimal_context(task="review changes", repo_root=repo_root, base=base)
tokens = estimate_tokens(result)
total_tokens += tokens
calls.append({"tool": "get_minimal_context", "tokens": tokens})
# Step 2: detect_changes (minimal)
result = detect_changes_func(base=base, repo_root=repo_root, detail_level="minimal")
tokens = estimate_tokens(result)
total_tokens += tokens
calls.append({"tool": "detect_changes_minimal", "tokens": tokens})
return {
"workflow": "review",
"total_tokens": total_tokens,
"tool_calls": len(calls),
"calls": calls,
}
def benchmark_architecture_workflow(repo_root: str) -> dict:
"""Simulate an architecture exploration workflow."""
from ..tools.community_tools import list_communities_func
from ..tools.context import get_minimal_context
from ..tools.flows_tools import list_flows
total_tokens = 0
calls = []
result = get_minimal_context(task="map architecture", repo_root=repo_root)
tokens = estimate_tokens(result)
total_tokens += tokens
calls.append({"tool": "get_minimal_context", "tokens": tokens})
result = list_communities_func(repo_root=repo_root, detail_level="minimal")
tokens = estimate_tokens(result)
total_tokens += tokens
calls.append({"tool": "list_communities_minimal", "tokens": tokens})
result = list_flows(repo_root=repo_root, detail_level="minimal")
tokens = estimate_tokens(result)
total_tokens += tokens
calls.append({"tool": "list_flows_minimal", "tokens": tokens})
return {
"workflow": "architecture",
"total_tokens": total_tokens,
"tool_calls": len(calls),
"calls": calls,
}
def benchmark_debug_workflow(repo_root: str) -> dict:
"""Simulate a debug workflow."""
from ..tools.context import get_minimal_context
from ..tools.query import semantic_search_nodes
total_tokens = 0
calls = []
result = get_minimal_context(task="debug login bug", repo_root=repo_root)
tokens = estimate_tokens(result)
total_tokens += tokens
calls.append({"tool": "get_minimal_context", "tokens": tokens})
result = semantic_search_nodes(
query="login", repo_root=repo_root, detail_level="minimal",
)
tokens = estimate_tokens(result)
total_tokens += tokens
calls.append({"tool": "semantic_search_minimal", "tokens": tokens})
return {
"workflow": "debug",
"total_tokens": total_tokens,
"tool_calls": len(calls),
"calls": calls,
}
def benchmark_onboard_workflow(repo_root: str) -> dict:
"""Simulate an onboarding workflow."""
from ..tools.context import get_minimal_context
from ..tools.query import list_graph_stats
total_tokens = 0
calls = []
result = get_minimal_context(task="onboard developer", repo_root=repo_root)
tokens = estimate_tokens(result)
total_tokens += tokens
calls.append({"tool": "get_minimal_context", "tokens": tokens})
result = list_graph_stats(repo_root=repo_root)
tokens = estimate_tokens(result)
total_tokens += tokens
calls.append({"tool": "list_graph_stats", "tokens": tokens})
return {
"workflow": "onboard",
"total_tokens": total_tokens,
"tool_calls": len(calls),
"calls": calls,
}
def benchmark_pre_merge_workflow(repo_root: str, base: str = "HEAD~1") -> dict:
"""Simulate a pre-merge check workflow."""
from ..tools.context import get_minimal_context
from ..tools.review import detect_changes_func
total_tokens = 0
calls = []
result = get_minimal_context(task="pre-merge check", repo_root=repo_root, base=base)
tokens = estimate_tokens(result)
total_tokens += tokens
calls.append({"tool": "get_minimal_context", "tokens": tokens})
result = detect_changes_func(base=base, repo_root=repo_root, detail_level="minimal")
tokens = estimate_tokens(result)
total_tokens += tokens
calls.append({"tool": "detect_changes_minimal", "tokens": tokens})
return {
"workflow": "pre_merge",
"total_tokens": total_tokens,
"tool_calls": len(calls),
"calls": calls,
}
ALL_WORKFLOWS: dict[str, Callable[..., dict]] = {
"review": benchmark_review_workflow,
"architecture": benchmark_architecture_workflow,
"debug": benchmark_debug_workflow,
"onboard": benchmark_onboard_workflow,
"pre_merge": benchmark_pre_merge_workflow,
}
def run_all_benchmarks(repo_root: str, base: str = "HEAD~1") -> list[dict]:
"""Run all workflow benchmarks and return results."""
results = []
for name, fn in ALL_WORKFLOWS.items():
try:
if "base" in fn.__code__.co_varnames:
result = fn(repo_root=repo_root, base=base)
else:
result = fn(repo_root=repo_root)
results.append(result)
except Exception as e:
logger.warning("Benchmark %s failed: %s", name, e)
results.append({"workflow": name, "error": str(e)})
return results
+409
View File
@@ -0,0 +1,409 @@
"""Additional export formats: GraphML, Neo4j Cypher, Obsidian vault, SVG."""
from __future__ import annotations
import html
import logging
import re
from pathlib import Path
from .graph import GraphStore, _sanitize_name
from .visualization import export_graph_data
logger = logging.getLogger(__name__)
# -------------------------------------------------------------------
# GraphML export (for Gephi, yEd, Cytoscape)
# -------------------------------------------------------------------
def export_graphml(store: GraphStore, output_path: Path) -> Path:
"""Export the graph as GraphML XML for Gephi/yEd/Cytoscape.
Returns the path to the written file.
"""
data = export_graph_data(store)
nodes = data["nodes"]
edges = data["edges"]
lines = [
'<?xml version="1.0" encoding="UTF-8"?>',
'<graphml xmlns="http://graphml.graphstruct.org/graphml"',
' xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"',
' xsi:schemaLocation="http://graphml.graphstruct.org/graphml">',
' <key id="kind" for="node" attr.name="kind" '
'attr.type="string"/>',
' <key id="file" for="node" attr.name="file" '
'attr.type="string"/>',
' <key id="language" for="node" attr.name="language" '
'attr.type="string"/>',
' <key id="community" for="node" attr.name="community" '
'attr.type="int"/>',
' <key id="edge_kind" for="edge" attr.name="kind" '
'attr.type="string"/>',
' <graph id="code-review-graph" edgedefault="directed">',
]
for n in nodes:
nid = html.escape(n["qualified_name"], quote=True)
lines.append(f' <node id="{nid}">')
lines.append(f' <data key="kind">'
f'{html.escape(n.get("kind", ""))}</data>')
lines.append(f' <data key="file">'
f'{html.escape(n.get("file_path", ""))}</data>')
lang = n.get("language", "") or ""
lines.append(f' <data key="language">'
f'{html.escape(lang)}</data>')
cid = n.get("community_id")
if cid is not None:
lines.append(f' <data key="community">'
f'{cid}</data>')
lines.append(' </node>')
for i, e in enumerate(edges):
src = html.escape(e["source"], quote=True)
tgt = html.escape(e["target"], quote=True)
kind = html.escape(e.get("kind", ""), quote=True)
lines.append(
f' <edge id="e{i}" source="{src}" target="{tgt}">'
)
lines.append(f' <data key="edge_kind">{kind}</data>')
lines.append(' </edge>')
lines.append(' </graph>')
lines.append('</graphml>')
output_path.write_text("\n".join(lines), encoding="utf-8")
logger.info("GraphML exported to %s (%d nodes, %d edges)",
output_path, len(nodes), len(edges))
return output_path
# -------------------------------------------------------------------
# Neo4j Cypher export
# -------------------------------------------------------------------
def export_neo4j_cypher(store: GraphStore, output_path: Path) -> Path:
"""Export the graph as Neo4j Cypher CREATE statements.
Returns the path to the written file.
"""
data = export_graph_data(store)
nodes = data["nodes"]
edges = data["edges"]
lines = [
"// Generated by code-review-graph",
"// Import: paste into Neo4j Browser or run via cypher-shell",
"",
]
# Create nodes
for n in nodes:
kind = n.get("kind", "Node")
props = {
"qualified_name": n["qualified_name"],
"name": n.get("name", ""),
"file_path": n.get("file_path", ""),
"language": n.get("language", "") or "",
}
cid = n.get("community_id")
if cid is not None:
props["community_id"] = cid
props_str = _cypher_props(props)
lines.append(f"CREATE (:{kind} {props_str});")
lines.append("")
# Create edges via MATCH
for e in edges:
kind = e.get("kind", "RELATES_TO")
src_qn = _cypher_escape(e["source"])
tgt_qn = _cypher_escape(e["target"])
lines.append(
f"MATCH (a {{qualified_name: '{src_qn}'}}), "
f"(b {{qualified_name: '{tgt_qn}'}}) "
f"CREATE (a)-[:{kind}]->(b);"
)
output_path.write_text("\n".join(lines), encoding="utf-8")
logger.info("Neo4j Cypher exported to %s (%d nodes, %d edges)",
output_path, len(nodes), len(edges))
return output_path
def _cypher_escape(s: str) -> str:
"""Escape a string for Cypher single-quoted literals."""
return s.replace("\\", "\\\\").replace("'", "\\'")
def _cypher_props(d: dict) -> str:
"""Format a dict as Cypher property map."""
parts = []
for k, v in d.items():
if isinstance(v, str):
parts.append(f"{k}: '{_cypher_escape(v)}'")
elif isinstance(v, (int, float)):
parts.append(f"{k}: {v}")
elif isinstance(v, bool):
parts.append(f"{k}: {'true' if v else 'false'}")
return "{" + ", ".join(parts) + "}"
# -------------------------------------------------------------------
# Obsidian vault export
# -------------------------------------------------------------------
def export_obsidian_vault(
store: GraphStore, output_dir: Path
) -> Path:
"""Export the graph as an Obsidian vault with wikilinks.
Creates:
- One .md per node with YAML frontmatter and [[wikilinks]]
- _COMMUNITY_*.md overview notes per community
- _INDEX.md with links to all nodes
Returns the output directory path.
"""
data = export_graph_data(store)
nodes = data["nodes"]
edges = data["edges"]
communities = data.get("communities", [])
output_dir.mkdir(parents=True, exist_ok=True)
# Build adjacency for wikilinks
neighbors: dict[str, list[dict]] = {}
for e in edges:
src = e["source"]
tgt = e["target"]
kind = e.get("kind", "RELATES_TO")
neighbors.setdefault(src, []).append(
{"target": tgt, "kind": kind}
)
neighbors.setdefault(tgt, []).append(
{"target": src, "kind": kind}
)
# Node name -> slug mapping
slugs: dict[str, str] = {}
for n in nodes:
slug = _obsidian_slug(n.get("name", n["qualified_name"]))
# Handle collisions
base_slug = slug
counter = 1
while slug in slugs.values():
slug = f"{base_slug}-{counter}"
counter += 1
slugs[n["qualified_name"]] = slug
# Write node pages
for n in nodes:
qn = n["qualified_name"]
slug = slugs[qn]
name = n.get("name", qn)
frontmatter = {
"kind": n.get("kind", ""),
"file": n.get("file_path", ""),
"language": n.get("language", "") or "",
"community": n.get("community_id"),
"tags": [n.get("kind", "").lower()],
}
lines = ["---"]
for k, v in frontmatter.items():
if isinstance(v, list):
lines.append(f"{k}:")
for item in v:
lines.append(f" - {item}")
elif v is not None:
lines.append(f"{k}: {v}")
lines.append("---")
lines.append(f"# {_sanitize_name(name)}")
lines.append("")
lines.append(f"**Kind:** {n.get('kind', '')}")
lines.append(f"**File:** `{n.get('file_path', '')}`")
lines.append("")
# Wikilinks to neighbors
nbrs = neighbors.get(qn, [])
if nbrs:
lines.append("## Connections")
lines.append("")
seen = set()
for nb in nbrs:
tgt_slug = slugs.get(nb["target"])
if tgt_slug and tgt_slug not in seen:
seen.add(tgt_slug)
tgt_name = tgt_slug.replace("-", " ").title()
lines.append(
f"- {nb['kind']}: "
f"[[{tgt_slug}|{tgt_name}]]"
)
page_path = output_dir / f"{slug}.md"
page_path.write_text("\n".join(lines), encoding="utf-8")
# Write community overview pages
community_map: dict[int, list[str]] = {}
for n in nodes:
cid = n.get("community_id")
if cid is not None:
community_map.setdefault(cid, []).append(
n["qualified_name"]
)
for c in communities:
cid = c.get("id")
cname = c.get("name", f"community-{cid}")
members = community_map.get(cid, [])
lines = [f"# Community: {_sanitize_name(cname)}", ""]
lines.append(f"**Size:** {c.get('size', len(members))}")
lines.append(f"**Cohesion:** {c.get('cohesion', 0):.2f}")
lang = c.get("dominant_language", "")
if lang:
lines.append(f"**Language:** {lang}")
lines.append("")
lines.append("## Members")
lines.append("")
for qn in members[:50]:
slug = slugs.get(qn)
if slug:
lines.append(f"- [[{slug}]]")
page_path = output_dir / f"_COMMUNITY_{cid}.md"
page_path.write_text("\n".join(lines), encoding="utf-8")
# Write index
index_lines = ["# Code Graph Index", ""]
index_lines.append(f"**Nodes:** {len(nodes)}")
index_lines.append(f"**Edges:** {len(edges)}")
index_lines.append(
f"**Communities:** {len(communities)}"
)
index_lines.append("")
index_lines.append("## All Nodes")
index_lines.append("")
for n in sorted(nodes, key=lambda x: x.get("name", "")):
slug = slugs.get(n["qualified_name"])
if slug:
index_lines.append(
f"- [[{slug}]] ({n.get('kind', '')})"
)
(output_dir / "_INDEX.md").write_text(
"\n".join(index_lines), encoding="utf-8"
)
logger.info(
"Obsidian vault exported to %s (%d pages)",
output_dir, len(nodes)
)
return output_dir
def _obsidian_slug(name: str) -> str:
"""Convert a name to an Obsidian-friendly filename slug."""
slug = re.sub(r"[^\w\s-]", "", name.lower())
slug = re.sub(r"[\s_]+", "-", slug).strip("-")
return slug[:100] or "unnamed"
# -------------------------------------------------------------------
# SVG export (matplotlib-based)
# -------------------------------------------------------------------
def export_svg(store: GraphStore, output_path: Path) -> Path:
"""Export a static SVG graph visualization.
Requires matplotlib (optional dependency).
Returns the path to the written file.
"""
try:
import matplotlib
matplotlib.use("Agg")
import matplotlib.pyplot as plt
except ImportError:
raise ImportError(
"matplotlib is required for SVG export. "
"Install with: pip install matplotlib"
)
import networkx as nx
data = export_graph_data(store)
nodes_data = data["nodes"]
edges_data = data["edges"]
nxg: nx.DiGraph = nx.DiGraph() # type: ignore[type-arg]
for n in nodes_data:
nxg.add_node(
n["qualified_name"],
label=n.get("name", ""),
kind=n.get("kind", ""),
)
for e in edges_data:
if e["source"] in nxg and e["target"] in nxg:
nxg.add_edge(e["source"], e["target"])
if nxg.number_of_nodes() == 0:
raise ValueError("Graph is empty, nothing to export")
# Color by kind
kind_colors = {
"File": "#6c757d",
"Class": "#0d6efd",
"Function": "#198754",
"Type": "#ffc107",
"Test": "#dc3545",
}
colors = [
kind_colors.get(
nxg.nodes[n].get("kind", ""), "#adb5bd"
)
for n in nxg.nodes()
]
fig, ax = plt.subplots(1, 1, figsize=(16, 12))
pos = nx.spring_layout(
nxg, k=2 / (nxg.number_of_nodes() ** 0.5),
iterations=50, seed=42
)
# Limit labels to avoid clutter
labels = {}
if nxg.number_of_nodes() <= 100:
labels = {
n: nxg.nodes[n].get("label", n.split("::")[-1])
for n in nxg.nodes()
}
nx.draw_networkx_nodes(
nxg, pos, ax=ax, node_color=colors,
node_size=30, alpha=0.8
)
nx.draw_networkx_edges(
nxg, pos, ax=ax, alpha=0.2,
arrows=True, arrowsize=5
)
if labels:
nx.draw_networkx_labels(
nxg, pos, labels=labels, ax=ax,
font_size=6
)
ax.set_title("Code Review Graph", fontsize=14)
ax.axis("off")
fig.savefig(
str(output_path), format="svg",
bbox_inches="tight", dpi=150
)
plt.close(fig)
logger.info("SVG exported to %s (%d nodes)",
output_path, nxg.number_of_nodes())
return output_path
+698
View File
@@ -0,0 +1,698 @@
"""Execution flow detection, tracing, and criticality scoring.
Detects entry points in the codebase (functions with no incoming CALLS edges,
framework-decorated handlers, and conventional name patterns), traces execution
paths via forward BFS through CALLS edges, scores each flow for criticality,
and persists results to the ``flows`` / ``flow_memberships`` tables.
"""
from __future__ import annotations
import json
import logging
import re
from collections import deque
from typing import Optional
from .constants import SECURITY_KEYWORDS as _SECURITY_KEYWORDS
from .graph import FlowAdjacency, GraphNode, GraphStore, _sanitize_name
logger = logging.getLogger(__name__)
# ---------------------------------------------------------------------------
# Constants
# ---------------------------------------------------------------------------
# Decorator patterns that indicate a function is a framework entry point.
_FRAMEWORK_DECORATOR_PATTERNS: list[re.Pattern[str]] = [
# Python web frameworks
re.compile(r"app\.(get|post|put|delete|patch|route|websocket|on_event)", re.IGNORECASE),
re.compile(r"router\.(get|post|put|delete|patch|route)", re.IGNORECASE),
re.compile(r"blueprint\.(route|before_request|after_request)", re.IGNORECASE),
re.compile(r"(before|after)_(request|response)", re.IGNORECASE),
# CLI frameworks
re.compile(r"click\.(command|group)", re.IGNORECASE),
re.compile(r"\w+\.(command|group)\b", re.IGNORECASE), # Click subgroups: @mygroup.command()
# Pydantic validators/serializers
re.compile(r"(field|model)_(serializer|validator)", re.IGNORECASE),
# Task queues
re.compile(r"(celery\.)?(task|shared_task|periodic_task)", re.IGNORECASE),
# Django
re.compile(r"receiver", re.IGNORECASE),
re.compile(r"api_view", re.IGNORECASE),
re.compile(r"\baction\b", re.IGNORECASE),
# Testing
re.compile(r"pytest\.(fixture|mark)"),
re.compile(r"(override_settings|modify_settings)", re.IGNORECASE),
# SQLAlchemy / event systems
re.compile(r"(event\.)?listens_for", re.IGNORECASE),
# Java Spring
re.compile(r"(Get|Post|Put|Delete|Patch|RequestMapping)Mapping", re.IGNORECASE),
re.compile(r"(Scheduled|EventListener|Bean|Configuration)", re.IGNORECASE),
# JS/TS frameworks
re.compile(r"(Component|Injectable|Controller|Module|Guard|Pipe)", re.IGNORECASE),
re.compile(r"(Subscribe|Mutation|Query|Resolver)", re.IGNORECASE),
# Express / Koa / Hono route handlers
re.compile(r"(app|router)\.(get|post|put|delete|patch|use|all)\b"),
# Android lifecycle
re.compile(r"@(Override|OnLifecycleEvent|Composable)", re.IGNORECASE),
# Kotlin coroutines / Android ViewModel
re.compile(r"(HiltViewModel|AndroidEntryPoint|Inject)", re.IGNORECASE),
# AI/agent frameworks (pydantic-ai, langchain, etc.)
re.compile(r"\w+\.(tool|tool_plain|system_prompt|result_validator)\b", re.IGNORECASE),
re.compile(r"^tool\b"), # bare @tool (LangChain, etc.)
# Middleware and exception handlers (Starlette, FastAPI, Sanic)
re.compile(r"\w+\.(middleware|exception_handler|on_exception)\b", re.IGNORECASE),
# Generic route decorator (Flask blueprints: @bp.route, @auth_bp.route, etc.)
re.compile(r"\w+\.route\b", re.IGNORECASE),
]
# Name patterns that indicate conventional entry points.
_ENTRY_NAME_PATTERNS: list[re.Pattern[str]] = [
re.compile(r"^main$"),
re.compile(r"^__main__$"),
re.compile(r"^test_"),
re.compile(r"^Test[A-Z]"),
re.compile(r"^on_"),
re.compile(r"^handle_"),
# Lambda / serverless handler functions (wired via config, not code calls)
re.compile(r"^handler$"),
re.compile(r"^handle$"),
re.compile(r"^lambda_handler$"),
# Alembic migration entry points
re.compile(r"^upgrade$"),
re.compile(r"^downgrade$"),
# FastAPI lifecycle / dependency injection
re.compile(r"^lifespan$"),
re.compile(r"^get_db$"),
# Android Activity/Fragment lifecycle
re.compile(r"^on(Create|Start|Resume|Pause|Stop|Destroy|Bind|Receive)"),
# Servlet / JAX-RS
re.compile(r"^do(Get|Post|Put|Delete)$"),
# Python BaseHTTPRequestHandler
re.compile(r"^do_(GET|POST|PUT|DELETE|PATCH|HEAD|OPTIONS)$"),
re.compile(r"^log_message$"),
# Express middleware signature
re.compile(r"^(middleware|errorHandler)$"),
# Angular lifecycle hooks
re.compile(
r"^ng(OnInit|OnChanges|OnDestroy|DoCheck"
r"|AfterContentInit|AfterContentChecked|AfterViewInit|AfterViewChecked)$"
),
# Angular Pipe / ControlValueAccessor / Guards / Resolvers
re.compile(r"^(transform|writeValue|registerOnChange|registerOnTouched|setDisabledState)$"),
re.compile(r"^(canActivate|canDeactivate|canActivateChild|canLoad|canMatch|resolve)$"),
# React class component lifecycle
re.compile(
r"^(componentDidMount|componentDidUpdate|componentWillUnmount"
r"|shouldComponentUpdate|render)$"
),
]
# ---------------------------------------------------------------------------
# Entry-point detection
# ---------------------------------------------------------------------------
def _has_framework_decorator(node: GraphNode) -> bool:
"""Return True if *node* has a decorator matching a framework pattern."""
decorators = node.extra.get("decorators")
if not decorators:
return False
if isinstance(decorators, str):
decorators = [decorators]
for dec in decorators:
for pat in _FRAMEWORK_DECORATOR_PATTERNS:
if pat.search(dec):
return True
return False
def _matches_entry_name(node: GraphNode) -> bool:
"""Return True if *node*'s name matches a conventional entry-point pattern."""
for pat in _ENTRY_NAME_PATTERNS:
if pat.search(node.name):
return True
return False
_TEST_FILE_RE = re.compile(
r"([\\/]__tests__[\\/]|\.spec\.[jt]sx?$|\.test\.[jt]sx?$|[\\/]test_[^/\\]*\.py$)",
)
def _is_test_file(file_path: str) -> bool:
"""Return True if *file_path* looks like a test file."""
return bool(_TEST_FILE_RE.search(file_path))
def detect_entry_points(
store: GraphStore,
include_tests: bool = False,
) -> list[GraphNode]:
"""Find functions that are entry points in the graph.
An entry point is a Function/Test node that either:
1. Has no incoming CALLS edges (true root), or
2. Has a framework decorator (e.g. ``@app.get``), or
3. Matches a conventional name pattern (``main``, ``test_*``, etc.).
When *include_tests* is False (the default), Test nodes are excluded so
that flow analysis focuses on production entry points.
"""
# Build a set of all qualified names that are CALLS targets. Exclude
# edges sourced at File nodes so that script-/notebook-/top-level-only
# callees (e.g. ``run_job()`` invoked from module scope, a top-level
# ``<App />`` render) remain detectable as entry points.
called_qnames = store.get_all_call_targets(include_file_sources=False)
# Scan all nodes for entry-point candidates.
candidate_nodes = store.get_nodes_by_kind(["Function", "Test"])
entry_points: list[GraphNode] = []
seen_qn: set[str] = set()
for node in candidate_nodes:
if not include_tests and (node.is_test or _is_test_file(node.file_path)):
continue
is_entry = False
# True root: no one calls this function.
if node.qualified_name not in called_qnames:
is_entry = True
# Framework decorator match.
if _has_framework_decorator(node):
is_entry = True
# Conventional name match.
if _matches_entry_name(node):
is_entry = True
if is_entry and node.qualified_name not in seen_qn:
entry_points.append(node)
seen_qn.add(node.qualified_name)
return entry_points
# ---------------------------------------------------------------------------
# Flow tracing (BFS)
# ---------------------------------------------------------------------------
def _trace_single_flow(
adj: FlowAdjacency,
ep: GraphNode,
max_depth: int = 15,
) -> Optional[dict]:
"""Trace a single execution flow from *ep* via forward BFS.
Returns a flow dict (see :func:`trace_flows` for the schema) or ``None``
if the flow is trivial (single-node, no outgoing CALLS that resolve).
"""
path_ids: list[int] = [ep.id]
path_qnames: list[str] = [ep.qualified_name]
visited: set[str] = {ep.qualified_name}
queue: deque[tuple[str, int]] = deque([(ep.qualified_name, 0)])
actual_depth = 0
nodes_by_qn = adj.nodes_by_qn
calls_out = adj.calls_out
while queue:
current_qn, depth = queue.popleft()
if depth > actual_depth:
actual_depth = depth
if depth >= max_depth:
continue
for target_qn in calls_out.get(current_qn, ()):
if target_qn in visited:
continue
target_node = nodes_by_qn.get(target_qn)
if target_node is None:
continue
visited.add(target_qn)
path_ids.append(target_node.id)
path_qnames.append(target_qn)
queue.append((target_qn, depth + 1))
# Skip trivial single-node flows.
if len(path_ids) < 2:
return None
files = list({
n.file_path
for qn in path_qnames
if (n := nodes_by_qn.get(qn)) is not None
})
flow: dict = {
"name": _sanitize_name(ep.name),
"entry_point": ep.qualified_name,
"entry_point_id": ep.id,
"path": path_ids,
"depth": actual_depth,
"node_count": len(path_ids),
"file_count": len(files),
"files": files,
"criticality": 0.0,
}
flow["criticality"] = compute_criticality(flow, adj)
return flow
def trace_flows(
store: GraphStore,
max_depth: int = 15,
include_tests: bool = False,
) -> list[dict]:
"""Trace execution flows from every entry point via forward BFS.
Returns a list of flow dicts, each containing:
- name: human-readable flow name (entry point name)
- entry_point: qualified name of the entry point
- entry_point_id: node database id of the entry point
- path: ordered list of node IDs in the flow
- depth: maximum BFS depth reached
- node_count: number of distinct nodes in the path
- file_count: number of distinct files touched
- files: list of distinct file paths
- criticality: computed criticality score (0.0-1.0)
"""
entry_points = detect_entry_points(store, include_tests=include_tests)
if not entry_points:
return []
adj = store.load_flow_adjacency()
flows: list[dict] = []
for ep in entry_points:
flow = _trace_single_flow(adj, ep, max_depth)
if flow is not None:
flows.append(flow)
# Sort by criticality descending.
flows.sort(key=lambda f: f["criticality"], reverse=True)
return flows
# ---------------------------------------------------------------------------
# Criticality scoring
# ---------------------------------------------------------------------------
def compute_criticality(flow: dict, adj: FlowAdjacency) -> float:
"""Score a flow from 0.0 to 1.0 based on multiple weighted factors.
Weights:
- File spread: 0.30
- External calls: 0.20
- Security sensitivity: 0.25
- Test coverage gap: 0.15
- Depth: 0.10
"""
node_ids: list[int] = flow.get("path", [])
if not node_ids:
return 0.0
nodes_by_id = adj.nodes_by_id
nodes_by_qn = adj.nodes_by_qn
calls_out = adj.calls_out
has_tested_by = adj.has_tested_by
nodes: list[GraphNode] = [
n for nid in node_ids if (n := nodes_by_id.get(nid)) is not None
]
if not nodes:
return 0.0
# --- File spread (0.0 - 1.0) ---
file_count = len({n.file_path for n in nodes})
# Normalize: 1 file => 0.0, 5+ files => 1.0
file_spread = min((file_count - 1) / 4.0, 1.0) if file_count > 1 else 0.0
# --- External calls (0.0 - 1.0) ---
# Calls that target nodes NOT in the graph are considered external.
external_count = 0
for n in nodes:
for target_qn in calls_out.get(n.qualified_name, ()):
if target_qn not in nodes_by_qn:
external_count += 1
# Normalize: 0 => 0.0, 5+ => 1.0
external_score = min(external_count / 5.0, 1.0)
# --- Security sensitivity (0.0 - 1.0) ---
security_hits = 0
for n in nodes:
name_lower = n.name.lower()
qn_lower = n.qualified_name.lower()
for kw in _SECURITY_KEYWORDS:
if kw in name_lower or kw in qn_lower:
security_hits += 1
break # Count each node at most once.
security_score = min(security_hits / max(len(nodes), 1), 1.0)
# --- Test coverage gap (0.0 - 1.0) ---
tested_count = sum(1 for n in nodes if n.qualified_name in has_tested_by)
coverage = tested_count / max(len(nodes), 1)
test_gap = 1.0 - coverage
# --- Depth (0.0 - 1.0) ---
depth = flow.get("depth", 0)
# Normalize: 0 => 0.0, 10+ => 1.0
depth_score = min(depth / 10.0, 1.0)
# --- Weighted sum ---
criticality = (
file_spread * 0.30
+ external_score * 0.20
+ security_score * 0.25
+ test_gap * 0.15
+ depth_score * 0.10
)
return round(min(max(criticality, 0.0), 1.0), 4)
# ---------------------------------------------------------------------------
# Persistence
# ---------------------------------------------------------------------------
def store_flows(store: GraphStore, flows: list[dict]) -> int:
"""Clear existing flows and persist new ones.
Returns the number of flows stored.
"""
# NOTE: store_flows uses _conn directly because it performs
# multi-statement batch writes (DELETE + INSERT loop) that are
# tightly coupled to the DB transaction lifecycle.
conn = store._conn
if conn.in_transaction:
logger.warning("Rolling back uncommitted transaction before BEGIN IMMEDIATE")
conn.rollback()
# Wrap the full DELETE + INSERT sequence in an explicit transaction
# so partial writes cannot occur if an exception interrupts the loop.
conn.execute("BEGIN IMMEDIATE")
try:
conn.execute("DELETE FROM flow_memberships")
conn.execute("DELETE FROM flows")
count = 0
for flow in flows:
path_json = json.dumps(flow.get("path", []))
conn.execute(
"""INSERT INTO flows
(name, entry_point_id, depth, node_count, file_count,
criticality, path_json)
VALUES (?, ?, ?, ?, ?, ?, ?)""",
(
flow["name"],
flow["entry_point_id"],
flow["depth"],
flow["node_count"],
flow["file_count"],
flow["criticality"],
path_json,
),
)
flow_id = conn.execute("SELECT last_insert_rowid()").fetchone()[0]
# Insert memberships.
node_ids = flow.get("path", [])
for position, node_id in enumerate(node_ids):
conn.execute(
"INSERT OR IGNORE INTO flow_memberships (flow_id, node_id, position) "
"VALUES (?, ?, ?)",
(flow_id, node_id, position),
)
count += 1
conn.commit()
except BaseException:
conn.rollback()
raise
return count
def incremental_trace_flows(
store: GraphStore,
changed_files: list[str],
max_depth: int = 15,
) -> int:
"""Re-trace only flows that touch *changed_files*. Much faster than full trace.
1. Find flow IDs whose memberships reference nodes in *changed_files*.
2. Collect the entry-point node IDs of those flows before deleting them.
3. Delete only the affected flows and their memberships.
4. Re-detect entry points, keeping those in *changed_files* **or** whose
node ID was an entry point of a deleted flow.
5. BFS-trace each relevant entry point via :func:`_trace_single_flow`.
6. INSERT the new flows (without clearing unrelated flows).
Returns the number of re-traced flows that were stored.
"""
if not changed_files:
return 0
conn = store._conn
changed_file_set = set(changed_files)
# ------------------------------------------------------------------
# 1. Find affected flow IDs
# ------------------------------------------------------------------
placeholders = ",".join("?" * len(changed_files))
affected_rows = conn.execute(
f"SELECT DISTINCT fm.flow_id FROM flow_memberships fm " # nosec B608
f"JOIN nodes n ON n.id = fm.node_id "
f"WHERE n.file_path IN ({placeholders})",
changed_files,
).fetchall()
affected_ids = [r[0] for r in affected_rows]
# ------------------------------------------------------------------
# 2. Collect old entry-point node IDs before deletion
# ------------------------------------------------------------------
entry_point_ids: set[int] = set()
if affected_ids:
ep_placeholders = ",".join("?" * len(affected_ids))
ep_rows = conn.execute(
f"SELECT entry_point_id FROM flows " # nosec B608
f"WHERE id IN ({ep_placeholders})",
affected_ids,
).fetchall()
entry_point_ids = {r[0] for r in ep_rows}
# ------------------------------------------------------------------
# 3. Delete affected flows and their memberships
# ------------------------------------------------------------------
# Wrap in an explicit transaction so a crash mid-loop cannot leave
# orphaned flow_memberships rows pointing at deleted flows. See #258.
if affected_ids:
if conn.in_transaction:
conn.commit()
conn.execute("BEGIN IMMEDIATE")
try:
for fid in affected_ids:
conn.execute(
"DELETE FROM flow_memberships WHERE flow_id = ?", (fid,),
)
conn.execute("DELETE FROM flows WHERE id = ?", (fid,))
conn.commit()
except BaseException:
conn.rollback()
raise
# ------------------------------------------------------------------
# 4. Re-detect entry points and filter to relevant ones
# ------------------------------------------------------------------
entry_points = detect_entry_points(store)
relevant_eps = [
ep for ep in entry_points
if ep.file_path in changed_file_set or ep.id in entry_point_ids
]
# ------------------------------------------------------------------
# 5. BFS-trace each relevant entry point
# ------------------------------------------------------------------
new_flows: list[dict] = []
if relevant_eps:
adj = store.load_flow_adjacency()
for ep in relevant_eps:
flow = _trace_single_flow(adj, ep, max_depth)
if flow is not None:
new_flows.append(flow)
# ------------------------------------------------------------------
# 6. INSERT new flows without clearing unrelated ones
# ------------------------------------------------------------------
count = 0
for flow in new_flows:
path_json = json.dumps(flow.get("path", []))
conn.execute(
"""INSERT INTO flows
(name, entry_point_id, depth, node_count, file_count,
criticality, path_json)
VALUES (?, ?, ?, ?, ?, ?, ?)""",
(
flow["name"],
flow["entry_point_id"],
flow["depth"],
flow["node_count"],
flow["file_count"],
flow["criticality"],
path_json,
),
)
flow_id = conn.execute("SELECT last_insert_rowid()").fetchone()[0]
node_ids = flow.get("path", [])
for position, node_id in enumerate(node_ids):
conn.execute(
"INSERT OR IGNORE INTO flow_memberships (flow_id, node_id, position) "
"VALUES (?, ?, ?)",
(flow_id, node_id, position),
)
count += 1
conn.commit()
return count
# ---------------------------------------------------------------------------
# Query helpers
# ---------------------------------------------------------------------------
def get_flows(
store: GraphStore,
sort_by: str = "criticality",
limit: int = 50,
) -> list[dict]:
"""Retrieve stored flows from the database.
Args:
store: The graph store.
sort_by: Column to sort by (``criticality``, ``depth``, ``node_count``).
limit: Maximum number of flows to return.
"""
allowed_sort = {"criticality", "depth", "node_count", "file_count", "name"}
if sort_by not in allowed_sort:
sort_by = "criticality"
order = "DESC" if sort_by in ("criticality", "depth", "node_count", "file_count") else "ASC"
# NOTE: get_flows reads from the flows table which is managed by
# the flows module; _conn access is documented coupling.
rows = store._conn.execute(
f"SELECT * FROM flows ORDER BY {sort_by} {order} LIMIT ?", # nosec B608
(limit,),
).fetchall()
results: list[dict] = []
for row in rows:
results.append({
"id": row["id"],
"name": _sanitize_name(row["name"]),
"entry_point_id": row["entry_point_id"],
"depth": row["depth"],
"node_count": row["node_count"],
"file_count": row["file_count"],
"criticality": row["criticality"],
"path": json.loads(row["path_json"]),
"created_at": row["created_at"],
"updated_at": row["updated_at"],
})
return results
def get_flow_by_id(store: GraphStore, flow_id: int) -> Optional[dict]:
"""Retrieve a single flow with full path details.
Returns a dict with the flow metadata plus a ``steps`` list containing
each node's name, kind, file, and line info.
"""
# NOTE: get_flow_by_id reads from the flows table; see store_flows note.
row = store._conn.execute(
"SELECT * FROM flows WHERE id = ?", (flow_id,)
).fetchone()
if row is None:
return None
path_ids: list[int] = json.loads(row["path_json"])
# Build detailed step info.
steps: list[dict] = []
for nid in path_ids:
node = store.get_node_by_id(nid)
if node:
steps.append({
"node_id": node.id,
"name": _sanitize_name(node.name),
"kind": node.kind,
"file": node.file_path,
"line_start": node.line_start,
"line_end": node.line_end,
"qualified_name": _sanitize_name(node.qualified_name),
})
return {
"id": row["id"],
"name": _sanitize_name(row["name"]),
"entry_point_id": row["entry_point_id"],
"depth": row["depth"],
"node_count": row["node_count"],
"file_count": row["file_count"],
"criticality": row["criticality"],
"path": path_ids,
"steps": steps,
"created_at": row["created_at"],
"updated_at": row["updated_at"],
}
def get_affected_flows(
store: GraphStore,
changed_files: list[str],
) -> dict:
"""Find flows that include nodes from the given changed files.
Returns::
{
"affected_flows": [<flow dicts>],
"total": <int>,
}
"""
if not changed_files:
return {"affected_flows": [], "total": 0}
# Find node IDs belonging to changed files.
node_ids = store.get_node_ids_by_files(changed_files)
if not node_ids:
return {"affected_flows": [], "total": 0}
# Find flow IDs that contain any of these nodes.
flow_ids = store.get_flow_ids_by_node_ids(node_ids)
if not flow_ids:
return {"affected_flows": [], "total": 0}
affected: list[dict] = []
for fid in flow_ids:
flow = get_flow_by_id(store, fid)
if flow:
affected.append(flow)
# Sort by criticality descending.
affected.sort(key=lambda f: f.get("criticality", 0), reverse=True)
return {
"affected_flows": affected,
"total": len(affected),
}
File diff suppressed because it is too large Load Diff
+122
View File
@@ -0,0 +1,122 @@
"""Graph snapshot diffing -- compare graph state over time."""
from __future__ import annotations
import json
import logging
from pathlib import Path
from typing import Any
from .graph import GraphStore
logger = logging.getLogger(__name__)
def take_snapshot(store: GraphStore) -> dict[str, Any]:
"""Take a snapshot of the current graph state.
Returns a dict with node and edge counts, qualified names,
and community assignments for later diffing.
"""
stats = store.get_stats()
nodes = store.get_all_nodes(exclude_files=False)
community_map = store.get_all_community_ids()
return {
"node_count": stats.total_nodes,
"edge_count": stats.total_edges,
"nodes": {
n.qualified_name: {
"kind": n.kind,
"file": n.file_path,
"community_id": community_map.get(
n.qualified_name
),
}
for n in nodes
},
"edges": {
f"{e.source_qualified}->"
f"{e.target_qualified}:{e.kind}"
for e in store.get_all_edges()
},
}
def save_snapshot(snapshot: dict, path: Path) -> None:
"""Save a snapshot to a JSON file."""
data = dict(snapshot)
if isinstance(data.get("edges"), set):
data["edges"] = sorted(data["edges"])
path.write_text(
json.dumps(data, indent=2), encoding="utf-8"
)
def load_snapshot(path: Path) -> dict:
"""Load a snapshot from a JSON file."""
data = json.loads(path.read_text(encoding="utf-8"))
if isinstance(data.get("edges"), list):
data["edges"] = set(data["edges"])
return data
def diff_snapshots(
before: dict, after: dict,
) -> dict[str, Any]:
"""Compare two graph snapshots.
Returns:
Dict with new_nodes, removed_nodes, new_edges,
removed_edges, community_changes, and summary
statistics.
"""
before_nodes = set(before.get("nodes", {}).keys())
after_nodes = set(after.get("nodes", {}).keys())
before_edges = before.get("edges", set())
after_edges = after.get("edges", set())
new_nodes = after_nodes - before_nodes
removed_nodes = before_nodes - after_nodes
new_edges = after_edges - before_edges
removed_edges = before_edges - after_edges
# Community changes for nodes that exist in both
community_changes = []
for qn in before_nodes & after_nodes:
before_cid = before["nodes"][qn].get(
"community_id"
)
after_cid = after["nodes"][qn].get(
"community_id"
)
if before_cid != after_cid:
community_changes.append({
"node": qn,
"before_community": before_cid,
"after_community": after_cid,
})
return {
"new_nodes": [
{"qualified_name": qn, **after["nodes"][qn]}
for qn in sorted(new_nodes)
][:100],
"removed_nodes": sorted(removed_nodes)[:100],
"new_edges": sorted(new_edges)[:100],
"removed_edges": sorted(removed_edges)[:100],
"community_changes": community_changes[:50],
"summary": {
"nodes_added": len(new_nodes),
"nodes_removed": len(removed_nodes),
"edges_added": len(new_edges),
"edges_removed": len(removed_edges),
"community_moves": len(community_changes),
"before_total": before.get(
"node_count", 0
),
"after_total": after.get(
"node_count", 0
),
},
}
+384
View File
@@ -0,0 +1,384 @@
"""Context-aware hints system for MCP tool responses.
Tracks session state (in-memory only) and generates intelligent
next-step suggestions after each tool call. Hints are appended as
``_hints`` to new tool responses so that Claude Code can propose
follow-up actions without the user having to discover them.
"""
from __future__ import annotations
import time
from collections import deque
from typing import Any
# ---- intent categories and their characteristic tool names ----
_INTENT_TOOLS: dict[str, set[str]] = {
"reviewing": {
"detect_changes", "get_review_context", "get_affected_flows", "get_impact_radius",
},
"debugging": {
"query_graph", "get_flow", "semantic_search_nodes",
},
"refactoring": {
"refactor", "find_dead_code", "suggest_refactorings",
},
"exploring": {
"list_communities", "get_architecture_overview", "list_flows", "list_graph_stats",
},
}
# ---- workflow adjacency: for each tool, which tools are useful next ----
_WORKFLOW: dict[str, list[dict[str, str]]] = {
"list_flows": [
{
"tool": "get_flow",
"suggestion": "Drill into a specific flow for step-by-step details",
},
{
"tool": "get_affected_flows",
"suggestion": "Check which flows are affected by recent changes",
},
{
"tool": "get_architecture_overview",
"suggestion": "See the high-level architecture",
},
],
"get_flow": [
{
"tool": "query_graph",
"suggestion": "Inspect callers/callees of a step in this flow",
},
{
"tool": "get_affected_flows",
"suggestion": "Check if changes affect this flow",
},
{
"tool": "list_flows",
"suggestion": "Browse other execution flows",
},
],
"get_affected_flows": [
{
"tool": "detect_changes",
"suggestion": "Get risk-scored change analysis",
},
{
"tool": "get_flow",
"suggestion": "Inspect a specific affected flow",
},
{
"tool": "get_review_context",
"suggestion": "Build a full review context for the changes",
},
],
"list_communities": [
{
"tool": "get_community",
"suggestion": "Inspect a specific community's members",
},
{
"tool": "get_architecture_overview",
"suggestion": "See cross-community coupling and warnings",
},
{
"tool": "list_flows",
"suggestion": "See execution flows across communities",
},
],
"get_community": [
{
"tool": "query_graph",
"suggestion": "Explore callers/callees of community members",
},
{
"tool": "list_communities",
"suggestion": "Browse other communities",
},
{
"tool": "get_architecture_overview",
"suggestion": "See how this community fits the architecture",
},
],
"get_architecture_overview": [
{
"tool": "list_communities",
"suggestion": "Drill into individual communities",
},
{
"tool": "detect_changes",
"suggestion": "See how recent changes affect the architecture",
},
{
"tool": "list_flows",
"suggestion": "Explore execution flows",
},
],
"detect_changes": [
{
"tool": "get_review_context",
"suggestion": "Build a full review context with source snippets",
},
{
"tool": "get_affected_flows",
"suggestion": "See which execution flows are affected",
},
{
"tool": "get_impact_radius",
"suggestion": "Expand the blast radius analysis",
},
{
"tool": "refactor",
"suggestion": "Look for refactoring opportunities in changed code",
},
],
"refactor": [
{
"tool": "query_graph",
"suggestion": "Verify call sites before applying a rename",
},
{
"tool": "detect_changes",
"suggestion": "Check risk of the refactored code",
},
{
"tool": "semantic_search_nodes",
"suggestion": "Find related symbols to also rename",
},
],
"semantic_search_nodes": [
{
"tool": "query_graph",
"suggestion": "Inspect callers/callees of a search result",
},
{
"tool": "get_flow",
"suggestion": "See the execution flow through a matched node",
},
{
"tool": "get_impact_radius",
"suggestion": "Check the blast radius from matched nodes",
},
],
}
# Maximum items per hints category returned to the caller.
_MAX_PER_CATEGORY = 3
# Session history caps.
_MAX_TOOLS_HISTORY = 100
_MAX_NODES_TRACKED = 1000
# ---------------------------------------------------------------------------
# SessionState
# ---------------------------------------------------------------------------
class SessionState:
"""In-memory session state for a single MCP connection."""
def __init__(self) -> None:
self.tools_called: deque[str] = deque(maxlen=_MAX_TOOLS_HISTORY)
self.nodes_queried: set[str] = set()
self.files_touched: set[str] = set()
self.inferred_intent: str | None = None
self.last_tool_time: float = 0.0
def record_tool_call(self, tool_name: str) -> None:
"""Record a tool invocation (FIFO, capped at 100)."""
self.tools_called.append(tool_name)
self.last_tool_time = time.time()
def record_nodes(self, node_ids: list[str]) -> None:
"""Record queried node identifiers (capped at 1000)."""
for nid in node_ids:
if len(self.nodes_queried) >= _MAX_NODES_TRACKED:
break
self.nodes_queried.add(nid)
def record_files(self, files: list[str]) -> None:
"""Record touched file paths."""
self.files_touched.update(files)
# ---------------------------------------------------------------------------
# Intent inference
# ---------------------------------------------------------------------------
def infer_intent(session: SessionState) -> str:
"""Classify the user's likely intent from their tool-call history.
Returns one of: ``"reviewing"``, ``"debugging"``, ``"refactoring"``,
``"exploring"`` (default).
"""
if not session.tools_called:
return "exploring"
# Score each intent by how many of the last N calls match its tools.
recent = list(session.tools_called)[-10:]
scores: dict[str, int] = {intent: 0 for intent in _INTENT_TOOLS}
for tool in recent:
for intent, tools in _INTENT_TOOLS.items():
if tool in tools:
scores[intent] += 1
best = max(scores, key=lambda k: scores[k])
if scores[best] == 0:
return "exploring"
return best
# ---------------------------------------------------------------------------
# Hints generation
# ---------------------------------------------------------------------------
def generate_hints(
tool_name: str,
result: dict[str, Any],
session: SessionState,
) -> dict[str, Any]:
"""Build context-aware hints for a tool response.
Returns::
{
"next_steps": [{"tool": ..., "suggestion": ...}, ...],
"related": [...],
"warnings": [...],
}
At most ``_MAX_PER_CATEGORY`` items per list. Tools already called
in this session are suppressed from ``next_steps``.
"""
# Update session state.
session.record_tool_call(tool_name)
session.inferred_intent = infer_intent(session)
next_steps = _build_next_steps(tool_name, session)
warnings = _extract_warnings(result)
# Build related BEFORE tracking, so that the current result's files
# are not yet in files_touched and can appear as suggestions.
related = _build_related(tool_name, result, session)
# Collect files/nodes from result for session tracking.
_track_result(result, session)
return {
"next_steps": next_steps[:_MAX_PER_CATEGORY],
"related": related[:_MAX_PER_CATEGORY],
"warnings": warnings[:_MAX_PER_CATEGORY],
}
# ---------------------------------------------------------------------------
# Internal helpers
# ---------------------------------------------------------------------------
def _track_result(result: dict[str, Any], session: SessionState) -> None:
"""Extract node IDs and file paths from a tool result and record them."""
# Files
for key in ("changed_files", "impacted_files"):
files = result.get(key)
if isinstance(files, list):
session.record_files([f for f in files if isinstance(f, str)])
# Nodes — look in common result shapes
node_ids: list[str] = []
for key in ("results", "changed_nodes", "impacted_nodes"):
items = result.get(key)
if isinstance(items, list):
for item in items:
if isinstance(item, dict):
qn = item.get("qualified_name")
if qn:
node_ids.append(qn)
if node_ids:
session.record_nodes(node_ids)
def _build_next_steps(
tool_name: str, session: SessionState
) -> list[dict[str, str]]:
"""Return next-step suggestions, filtering already-called tools."""
called = set(session.tools_called)
candidates = _WORKFLOW.get(tool_name, [])
out: list[dict[str, str]] = []
for c in candidates:
if c["tool"] not in called:
out.append(c)
return out
def _extract_warnings(result: dict[str, Any]) -> list[str]:
"""Pull warning signals from a tool result."""
warnings: list[str] = []
# Test gaps
test_gaps = result.get("test_gaps")
if isinstance(test_gaps, list) and test_gaps:
names = [g.get("name", g) if isinstance(g, dict) else str(g) for g in test_gaps[:5]]
warnings.append(
f"Test coverage gaps: {', '.join(names)}"
)
# High risk score
risk = result.get("risk_score")
if isinstance(risk, (int, float)) and risk > 0.7:
warnings.append(f"High risk score ({risk:.2f}) — review carefully")
# Coupling warnings from architecture overview
arch_warnings = result.get("warnings")
if isinstance(arch_warnings, list):
for w in arch_warnings[:3]:
if isinstance(w, str):
warnings.append(w)
elif isinstance(w, dict) and "message" in w:
warnings.append(w["message"])
return warnings
def _build_related(
tool_name: str,
result: dict[str, Any],
session: SessionState,
) -> list[str]:
"""Suggest related node/file identifiers from the result."""
related: list[str] = []
seen: set[str] = set()
# Suggest impacted files the user hasn't touched yet
impacted = result.get("impacted_files")
if isinstance(impacted, list):
for f in impacted:
if isinstance(f, str) and f not in session.files_touched and f not in seen:
related.append(f)
seen.add(f)
if len(related) >= _MAX_PER_CATEGORY:
break
return related
# ---------------------------------------------------------------------------
# Module-level session singleton
# ---------------------------------------------------------------------------
_session = SessionState()
def get_session() -> SessionState:
"""Return the global in-memory session state."""
return _session
def reset_session() -> None:
"""Reset the global session (useful for testing)."""
global _session
_session = SessionState()
File diff suppressed because it is too large Load Diff
+303
View File
@@ -0,0 +1,303 @@
"""Post-build Jedi enrichment for Python call resolution.
After tree-sitter parsing, many method calls on lowercase-receiver variables
are dropped (e.g. ``svc.authenticate()`` where ``svc = factory()``). Jedi
can resolve these by tracing return types across files.
This module runs as a post-build step: it re-walks Python ASTs to find
dropped calls, uses ``jedi.Script.goto()`` to resolve them, and adds the
resulting CALLS edges to the graph database.
"""
from __future__ import annotations
import logging
import os
from pathlib import Path
from typing import Optional
from .parser import CodeParser, EdgeInfo
from .parser import _is_test_file as _parser_is_test_file
logger = logging.getLogger(__name__)
_SELF_NAMES = frozenset({"self", "cls", "super"})
def enrich_jedi_calls(store, repo_root: Path) -> dict:
"""Resolve untracked Python method calls via Jedi.
Walks Python files, finds ``receiver.method()`` calls that tree-sitter
dropped (lowercase receiver, not self/cls), resolves them with Jedi,
and inserts new CALLS edges.
Returns stats dict with ``resolved`` count.
"""
try:
import jedi
except ImportError:
logger.info("Jedi not installed, skipping Python enrichment")
return {"skipped": True, "reason": "jedi not installed"}
repo_root = Path(repo_root).resolve()
# Get Python files from the graph — skip early if none
all_files = store.get_all_files()
py_files = [f for f in all_files if f.endswith(".py")]
if not py_files:
return {"resolved": 0, "files": 0}
# Scope the Jedi project to Python-only directories to avoid scanning
# non-Python files (e.g. node_modules, TS sources). This matters for
# polyglot monorepos where jedi.Project(path=repo_root) would scan
# thousands of irrelevant files during initialization.
py_dirs = sorted({str(Path(f).parent) for f in py_files})
common_py_root = Path(os.path.commonpath(py_dirs)) if py_dirs else repo_root
if not str(common_py_root).startswith(str(repo_root)):
common_py_root = repo_root
project = jedi.Project(
path=str(common_py_root),
added_sys_path=[str(repo_root)],
smart_sys_path=False,
)
# Pre-parse all Python files to find which ones have pending method calls.
# This avoids expensive Jedi Script creation for files with nothing to resolve.
parser = CodeParser()
ts_parser = parser._get_parser("python")
if not ts_parser:
return {"resolved": 0, "files": 0}
# Build set of method names that actually exist in project code.
# No point asking Jedi to resolve `logger.getLogger()` if no project
# file defines a function called `getLogger`.
project_func_names = {
r["name"]
for r in store._conn.execute(
"SELECT DISTINCT name FROM nodes WHERE kind IN ('Function', 'Test')"
).fetchall()
}
files_with_pending: list[tuple[str, bytes, list]] = []
total_skipped = 0
for file_path in py_files:
try:
source = Path(file_path).read_bytes()
except (OSError, PermissionError):
continue
tree = ts_parser.parse(source)
is_test = _parser_is_test_file(file_path)
pending = _find_untracked_method_calls(tree.root_node, is_test)
if pending:
# Only keep calls whose method name exists in project code
filtered = [p for p in pending if p[2] in project_func_names]
total_skipped += len(pending) - len(filtered)
if filtered:
files_with_pending.append((file_path, source, filtered))
if not files_with_pending:
return {"resolved": 0, "files": 0}
logger.debug(
"Jedi: %d/%d Python files have pending calls (%d calls skipped — no project target)",
len(files_with_pending), len(py_files), total_skipped,
)
resolved_count = 0
files_enriched = 0
errors = 0
for file_path, source, pending in files_with_pending:
source_text = source.decode("utf-8", errors="replace")
# Get existing CALLS edges for this file to skip duplicates
existing = set()
for edge in _get_file_call_edges(store, file_path):
existing.add((edge.source_qualified, edge.line))
# Get function nodes from DB for enclosing-function lookup
func_nodes = [
n for n in store.get_nodes_by_file(file_path)
if n.kind in ("Function", "Test")
]
# Create Jedi script once per file
try:
script = jedi.Script(source_text, path=file_path, project=project)
except Exception as e:
logger.debug("Jedi failed to load %s: %s", file_path, e)
errors += 1
continue
file_resolved = 0
for jedi_line, col, _method_name, _enclosing_name in pending:
# Find enclosing function qualified name
enclosing = _find_enclosing(func_nodes, jedi_line)
if not enclosing:
enclosing = file_path # module-level
# Skip if we already have a CALLS edge from this source at this line
if (enclosing, jedi_line) in existing:
continue
# Ask Jedi to resolve
try:
names = script.goto(jedi_line, col)
except Exception: # nosec B112 - Jedi may fail on malformed code
continue
if not names:
continue
name = names[0]
if not name.module_path:
continue
module_path = Path(name.module_path).resolve()
# Only emit edges for project-internal definitions
try:
module_path.relative_to(repo_root)
except ValueError:
continue
# Build qualified target: file_path::Class.method or file_path::func
target_file = str(module_path)
parent = name.parent()
if parent and parent.type == "class":
target = f"{target_file}::{parent.name}.{name.name}"
else:
target = f"{target_file}::{name.name}"
store.upsert_edge(EdgeInfo(
kind="CALLS",
source=enclosing,
target=target,
file_path=file_path,
line=jedi_line,
))
existing.add((enclosing, jedi_line))
file_resolved += 1
if file_resolved:
files_enriched += 1
resolved_count += file_resolved
if resolved_count:
store.commit()
logger.info(
"Jedi enrichment: resolved %d calls in %d files",
resolved_count, files_enriched,
)
return {
"resolved": resolved_count,
"files": files_enriched,
"errors": errors,
}
def _get_file_call_edges(store, file_path: str):
"""Get all CALLS edges originating from a file."""
conn = store._conn
rows = conn.execute(
"SELECT * FROM edges WHERE file_path = ? AND kind = 'CALLS'",
(file_path,),
).fetchall()
from .graph import GraphEdge
return [
GraphEdge(
id=r["id"], kind=r["kind"],
source_qualified=r["source_qualified"],
target_qualified=r["target_qualified"],
file_path=r["file_path"], line=r["line"],
extra={},
)
for r in rows
]
def _find_enclosing(func_nodes, line: int) -> Optional[str]:
"""Find the qualified name of the function enclosing a given line."""
best = None
best_span = float("inf")
for node in func_nodes:
if node.line_start <= line <= node.line_end:
span = node.line_end - node.line_start
if span < best_span:
best = node.qualified_name
best_span = span
return best
def _find_untracked_method_calls(root, is_test_file: bool = False):
"""Walk Python AST to find method calls the parser would have dropped.
Returns list of (jedi_line, col, method_name, enclosing_func_name) tuples.
Jedi_line is 1-indexed, col is 0-indexed.
"""
results: list[tuple[int, int, str, Optional[str]]] = []
_walk_calls(root, results, is_test_file, enclosing_func=None)
return results
def _walk_calls(node, results, is_test_file, enclosing_func):
"""Recursively walk AST collecting dropped method calls."""
# Track enclosing function scope
if node.type == "function_definition":
name = None
for child in node.children:
if child.type == "identifier":
name = child.text.decode("utf-8", errors="replace")
break
for child in node.children:
_walk_calls(child, results, is_test_file, name or enclosing_func)
return
if node.type == "decorated_definition":
for child in node.children:
_walk_calls(child, results, is_test_file, enclosing_func)
return
# Check for call expressions with attribute access
if node.type == "call":
first = node.children[0] if node.children else None
if first and first.type == "attribute":
_check_dropped_call(first, results, is_test_file, enclosing_func)
for child in node.children:
_walk_calls(child, results, is_test_file, enclosing_func)
def _check_dropped_call(attr_node, results, is_test_file, enclosing_func):
"""Check if an attribute-based call was dropped by the parser."""
children = attr_node.children
if len(children) < 2:
return
receiver = children[0]
# Only handle simple identifier receivers
if receiver.type != "identifier":
return
receiver_text = receiver.text.decode("utf-8", errors="replace")
# The parser keeps: self/cls/super calls and uppercase-receiver calls
# The parser keeps: calls handled by typed-var enrichment (but those are
# separate edges -- we check for duplicates via existing-edge set)
if receiver_text in _SELF_NAMES:
return
if receiver_text[:1].isupper():
return
if is_test_file:
return # test files already track all calls
# Find the method name identifier
method_node = children[-1]
if method_node.type != "identifier":
return
row, col = method_node.start_point # 0-indexed
method_name = method_node.text.decode("utf-8", errors="replace")
results.append((row + 1, col, method_name, enclosing_func))
File diff suppressed because it is too large Load Diff
+142
View File
@@ -0,0 +1,142 @@
"""Memory/feedback loop -- persist Q&A results for graph enrichment."""
from __future__ import annotations
import logging
import re
import time
from pathlib import Path
from typing import Any
logger = logging.getLogger(__name__)
def save_result(
question: str,
answer: str,
nodes: list[str] | None = None,
result_type: str = "query",
memory_dir: Path | None = None,
repo_root: Path | None = None,
) -> Path:
"""Save a Q&A result as markdown for re-ingestion.
Args:
question: The question that was asked.
answer: The answer/result.
nodes: Related node qualified names.
result_type: Type of result (query, review, debug).
memory_dir: Directory to save to. Defaults to
<repo>/.code-review-graph/memory/
repo_root: Repository root for default memory_dir.
Returns:
Path to the saved file.
"""
if memory_dir is None:
if repo_root is None:
raise ValueError(
"Either memory_dir or repo_root required"
)
memory_dir = (
repo_root / ".code-review-graph" / "memory"
)
memory_dir.mkdir(parents=True, exist_ok=True)
# Generate filename from question
slug = re.sub(r"[^\w\s-]", "", question.lower())
slug = re.sub(r"[\s_]+", "-", slug).strip("-")[:60]
timestamp = int(time.time())
filename = f"{slug}-{timestamp}.md"
# Build markdown with YAML frontmatter
lines = [
"---",
f"type: {result_type}",
f"timestamp: {timestamp}",
]
if nodes:
lines.append("nodes:")
for n in nodes[:20]:
lines.append(f" - {n}")
lines.extend([
"---",
"",
f"# {question}",
"",
answer,
])
path = memory_dir / filename
path.write_text("\n".join(lines), encoding="utf-8")
logger.info("Saved result to %s", path)
return path
def list_memories(
memory_dir: Path | None = None,
repo_root: Path | None = None,
) -> list[dict[str, Any]]:
"""List all saved memory files.
Returns list of dicts with: path, question, type, timestamp.
"""
if memory_dir is None:
if repo_root is None:
return []
memory_dir = (
repo_root / ".code-review-graph" / "memory"
)
if not memory_dir.exists():
return []
results = []
for f in sorted(memory_dir.glob("*.md")):
try:
text = f.read_text(encoding="utf-8")
# Parse frontmatter
meta: dict[str, Any] = {"path": str(f)}
if text.startswith("---"):
parts = text.split("---", 2)
if len(parts) >= 3:
fm_lines = parts[1].strip().split("\n")
for line in fm_lines:
if ": " in line and not line.startswith(" "):
k, v = line.split(": ", 1)
meta[k.strip()] = v.strip()
# Extract question from first heading
for line in text.split("\n"):
if line.startswith("# "):
meta["question"] = line[2:].strip()
break
results.append(meta)
except OSError:
continue
return results
def clear_memories(
memory_dir: Path | None = None,
repo_root: Path | None = None,
) -> int:
"""Delete all memory files. Returns count deleted."""
if memory_dir is None:
if repo_root is None:
return 0
memory_dir = (
repo_root / ".code-review-graph" / "memory"
)
if not memory_dir.exists():
return 0
count = 0
for f in memory_dir.glob("*.md"):
f.unlink()
count += 1
logger.info("Cleared %d memory files", count)
return count
+284
View File
@@ -0,0 +1,284 @@
"""Schema migration framework for the code-review-graph SQLite database.
Manages incremental schema changes via versioned migration functions.
Each migration is idempotent (uses IF NOT EXISTS / column existence checks).
"""
from __future__ import annotations
import logging
import sqlite3
from typing import Callable
logger = logging.getLogger(__name__)
def get_schema_version(conn: sqlite3.Connection) -> int:
"""Read the current schema version from the metadata table.
Returns:
int: The schema version (0 if metadata table doesn't exist, 1 if not set).
"""
try:
row = conn.execute(
"SELECT value FROM metadata WHERE key = 'schema_version'"
).fetchone()
if row is None:
return 1
return int(row[0] if isinstance(row, (tuple, list)) else row["value"])
except sqlite3.OperationalError:
# metadata table doesn't exist
return 0
def _set_schema_version(conn: sqlite3.Connection, version: int) -> None:
"""Set the schema version in the metadata table."""
conn.execute(
"INSERT OR REPLACE INTO metadata (key, value) VALUES ('schema_version', ?)",
(str(version),),
)
_KNOWN_TABLES = frozenset({
"nodes", "edges", "metadata", "communities", "flows", "flow_memberships", "nodes_fts",
"community_summaries", "flow_snapshots", "risk_index",
})
def _has_column(conn: sqlite3.Connection, table: str, column: str) -> bool:
"""Check if a column exists in a table."""
if table not in _KNOWN_TABLES:
raise ValueError(f"Unknown table: {table}")
cursor = conn.execute(f"PRAGMA table_info({table})") # noqa: S608
columns = [row[1] if isinstance(row, tuple) else row["name"] for row in cursor]
return column in columns
def _table_exists(conn: sqlite3.Connection, table: str) -> bool:
"""Check if a table exists."""
if table not in _KNOWN_TABLES:
raise ValueError(f"Unknown table: {table}")
row = conn.execute(
"SELECT count(*) FROM sqlite_master WHERE type IN ('table', 'view') "
"AND name = ?",
(table,),
).fetchone()
return row[0] > 0
# ---------------------------------------------------------------------------
# Migration functions
# ---------------------------------------------------------------------------
def _migrate_v2(conn: sqlite3.Connection) -> None:
"""v2: Add signature column to nodes table."""
if not _has_column(conn, "nodes", "signature"):
conn.execute("ALTER TABLE nodes ADD COLUMN signature TEXT")
logger.info("Migration v2: added 'signature' column to nodes")
def _migrate_v3(conn: sqlite3.Connection) -> None:
"""v3: Create flows and flow_memberships tables."""
conn.execute("""
CREATE TABLE IF NOT EXISTS flows (
id INTEGER PRIMARY KEY AUTOINCREMENT,
name TEXT NOT NULL,
entry_point_id INTEGER NOT NULL,
depth INTEGER NOT NULL,
node_count INTEGER NOT NULL,
file_count INTEGER NOT NULL,
criticality REAL NOT NULL DEFAULT 0.0,
path_json TEXT NOT NULL,
created_at TEXT NOT NULL DEFAULT (datetime('now')),
updated_at TEXT NOT NULL DEFAULT (datetime('now'))
)
""")
conn.execute("""
CREATE TABLE IF NOT EXISTS flow_memberships (
flow_id INTEGER NOT NULL,
node_id INTEGER NOT NULL,
position INTEGER NOT NULL,
PRIMARY KEY (flow_id, node_id)
)
""")
conn.execute(
"CREATE INDEX IF NOT EXISTS idx_flows_criticality ON flows(criticality DESC)"
)
conn.execute(
"CREATE INDEX IF NOT EXISTS idx_flows_entry ON flows(entry_point_id)"
)
conn.execute(
"CREATE INDEX IF NOT EXISTS idx_flow_memberships_node ON flow_memberships(node_id)"
)
logger.info("Migration v3: created flows and flow_memberships tables")
def _migrate_v4(conn: sqlite3.Connection) -> None:
"""v4: Create communities table, add community_id to nodes."""
conn.execute("""
CREATE TABLE IF NOT EXISTS communities (
id INTEGER PRIMARY KEY AUTOINCREMENT,
name TEXT NOT NULL,
level INTEGER NOT NULL DEFAULT 0,
parent_id INTEGER,
cohesion REAL NOT NULL DEFAULT 0.0,
size INTEGER NOT NULL DEFAULT 0,
dominant_language TEXT,
description TEXT,
created_at TEXT NOT NULL DEFAULT (datetime('now'))
)
""")
if not _has_column(conn, "nodes", "community_id"):
conn.execute("ALTER TABLE nodes ADD COLUMN community_id INTEGER")
logger.info("Migration v4: added 'community_id' column to nodes")
conn.execute(
"CREATE INDEX IF NOT EXISTS idx_nodes_community ON nodes(community_id)"
)
conn.execute(
"CREATE INDEX IF NOT EXISTS idx_communities_parent ON communities(parent_id)"
)
conn.execute(
"CREATE INDEX IF NOT EXISTS idx_communities_cohesion ON communities(cohesion DESC)"
)
logger.info("Migration v4: created communities table")
def _migrate_v5(conn: sqlite3.Connection) -> None:
"""v5: Create FTS5 virtual table for nodes."""
if not _table_exists(conn, "nodes_fts"):
conn.execute("""
CREATE VIRTUAL TABLE nodes_fts USING fts5(
name, qualified_name, file_path, signature,
content='nodes', content_rowid='rowid',
tokenize='porter unicode61'
)
""")
logger.info("Migration v5: created nodes_fts FTS5 virtual table")
def _migrate_v6(conn: sqlite3.Connection) -> None:
"""v6: Add pre-computed summary tables for token-efficient queries."""
conn.execute("""
CREATE TABLE IF NOT EXISTS community_summaries (
community_id INTEGER PRIMARY KEY,
name TEXT NOT NULL,
purpose TEXT DEFAULT '',
key_symbols TEXT DEFAULT '[]',
risk TEXT DEFAULT 'unknown',
size INTEGER DEFAULT 0,
dominant_language TEXT DEFAULT '',
FOREIGN KEY (community_id) REFERENCES communities(id)
)
""")
conn.execute("""
CREATE TABLE IF NOT EXISTS flow_snapshots (
flow_id INTEGER PRIMARY KEY,
name TEXT NOT NULL,
entry_point TEXT NOT NULL,
critical_path TEXT DEFAULT '[]',
criticality REAL DEFAULT 0.0,
node_count INTEGER DEFAULT 0,
file_count INTEGER DEFAULT 0,
FOREIGN KEY (flow_id) REFERENCES flows(id)
)
""")
conn.execute("""
CREATE TABLE IF NOT EXISTS risk_index (
node_id INTEGER PRIMARY KEY,
qualified_name TEXT NOT NULL,
risk_score REAL DEFAULT 0.0,
caller_count INTEGER DEFAULT 0,
test_coverage TEXT DEFAULT 'unknown',
security_relevant INTEGER DEFAULT 0,
last_computed TEXT DEFAULT '',
FOREIGN KEY (node_id) REFERENCES nodes(id)
)
""")
conn.execute(
"CREATE INDEX IF NOT EXISTS idx_risk_index_score "
"ON risk_index(risk_score DESC)"
)
logger.info("Migration v6: created summary tables "
"(community_summaries, flow_snapshots, risk_index)")
def _migrate_v7(conn: sqlite3.Connection) -> None:
"""v7: Add compound edge indexes for summary and risk queries."""
conn.execute(
"CREATE INDEX IF NOT EXISTS idx_edges_target_kind "
"ON edges(target_qualified, kind)"
)
conn.execute(
"CREATE INDEX IF NOT EXISTS idx_edges_source_kind "
"ON edges(source_qualified, kind)"
)
logger.info("Migration v7: added compound edge indexes")
def _migrate_v8(conn: sqlite3.Connection) -> None:
"""v8: Add composite index on edges for upsert_edge performance."""
conn.execute("""
CREATE INDEX IF NOT EXISTS idx_edges_composite
ON edges(kind, source_qualified, target_qualified, file_path, line)
""")
logger.info("Migration v8: created composite edge index")
def _migrate_v9(conn: sqlite3.Connection) -> None:
"""v9: Add confidence scoring to edges."""
if not _has_column(conn, "edges", "confidence"):
conn.execute(
"ALTER TABLE edges ADD COLUMN confidence REAL DEFAULT 1.0"
)
if not _has_column(conn, "edges", "confidence_tier"):
conn.execute(
"ALTER TABLE edges ADD COLUMN confidence_tier TEXT DEFAULT 'EXTRACTED'"
)
logger.info("Migration v9: added edge confidence columns")
# ---------------------------------------------------------------------------
# Migration registry
# ---------------------------------------------------------------------------
MIGRATIONS: dict[int, Callable[[sqlite3.Connection], None]] = {
2: _migrate_v2,
3: _migrate_v3,
4: _migrate_v4,
5: _migrate_v5,
6: _migrate_v6,
7: _migrate_v7,
8: _migrate_v8,
9: _migrate_v9,
}
LATEST_VERSION = max(MIGRATIONS.keys())
def run_migrations(conn: sqlite3.Connection) -> None:
"""Run all pending migrations in order.
Each migration runs in its own transaction. The schema_version metadata
entry is updated after each successful migration.
"""
current = get_schema_version(conn)
if current >= LATEST_VERSION:
return
logger.info("Schema version %d -> %d: running migrations", current, LATEST_VERSION)
for version in sorted(MIGRATIONS.keys()):
if version <= current:
continue
logger.info("Running migration v%d", version)
try:
MIGRATIONS[version](conn)
_set_schema_version(conn, version)
conn.commit()
except sqlite3.Error:
conn.rollback()
logger.error("Migration v%d failed, rolling back", version, exc_info=True)
raise
logger.info("Migrations complete, now at schema version %d", LATEST_VERSION)
File diff suppressed because it is too large Load Diff
+134
View File
@@ -0,0 +1,134 @@
"""Shared post-build processing pipeline.
After the core Tree-sitter parse (full_build or incremental_update), four
post-processing steps must run to populate derived tables:
1. Compute node signatures
2. Rebuild FTS5 search index
3. Trace execution flows
4. Detect code communities
This module extracts that pipeline so every entry point — MCP tool, CLI
commands, and watch mode — produces identical results.
"""
from __future__ import annotations
import logging
import sqlite3
from typing import Any
from .graph import GraphStore
logger = logging.getLogger(__name__)
def run_post_processing(store: GraphStore) -> dict[str, Any]:
"""Run all post-build steps on a populated graph.
Each step is non-fatal: failures are logged and collected as warnings
so the primary build result is never lost.
Args:
store: An open GraphStore with nodes and edges already populated.
Returns:
Dict with keys for each step's result count and a ``warnings``
list (only present when at least one step failed).
"""
result: dict[str, Any] = {}
warnings: list[str] = []
_compute_signatures(store, result, warnings)
_rebuild_fts_index(store, result, warnings)
_trace_flows(store, result, warnings)
_detect_communities(store, result, warnings)
if warnings:
result["warnings"] = warnings
return result
# -- Individual steps (private) ------------------------------------------
def _compute_signatures(
store: GraphStore,
result: dict[str, Any],
warnings: list[str],
) -> None:
"""Compute human-readable signatures for nodes that lack one."""
try:
rows = store.get_nodes_without_signature()
for row in rows:
node_id, name, kind, params, ret = (
row[0],
row[1],
row[2],
row[3],
row[4],
)
if kind in ("Function", "Test"):
sig = f"def {name}({params or ''})"
if ret:
sig += f" -> {ret}"
elif kind == "Class":
sig = f"class {name}"
else:
sig = name
store.update_node_signature(node_id, sig[:512])
store.commit()
result["signatures_computed"] = len(rows)
except (sqlite3.OperationalError, TypeError, KeyError) as e:
logger.warning("Signature computation failed: %s", e)
warnings.append(f"Signature computation failed: {type(e).__name__}: {e}")
def _rebuild_fts_index(
store: GraphStore,
result: dict[str, Any],
warnings: list[str],
) -> None:
"""Rebuild the FTS5 full-text search index."""
try:
from .search import rebuild_fts_index
fts_count = rebuild_fts_index(store)
result["fts_indexed"] = fts_count
except (sqlite3.OperationalError, ImportError) as e:
logger.warning("FTS index rebuild failed: %s", e)
warnings.append(f"FTS index rebuild failed: {type(e).__name__}: {e}")
def _trace_flows(
store: GraphStore,
result: dict[str, Any],
warnings: list[str],
) -> None:
"""Trace execution flows from entry points."""
try:
from .flows import store_flows, trace_flows
flows = trace_flows(store)
count = store_flows(store, flows)
result["flows_detected"] = count
except (sqlite3.OperationalError, ImportError) as e:
logger.warning("Flow detection failed: %s", e)
warnings.append(f"Flow detection failed: {type(e).__name__}: {e}")
def _detect_communities(
store: GraphStore,
result: dict[str, Any],
warnings: list[str],
) -> None:
"""Detect code communities via Leiden algorithm or file grouping."""
try:
from .communities import detect_communities, store_communities
comms = detect_communities(store)
count = store_communities(store, comms)
result["communities_detected"] = count
except (sqlite3.OperationalError, ImportError) as e:
logger.warning("Community detection failed: %s", e)
warnings.append(f"Community detection failed: {type(e).__name__}: {e}")
+159
View File
@@ -0,0 +1,159 @@
"""MCP prompt templates for Code Review Graph.
Provides 5 pre-built prompt workflows, all enforcing token-efficient
detail_level="minimal" first patterns with get_minimal_context entry point.
1. review_changes - pre-commit review using detect_changes + affected_flows
2. architecture_map - architecture docs using communities, flows, Mermaid
3. debug_issue - guided debugging using search, flow tracing
4. onboard_developer - new dev orientation using stats, architecture, flows
5. pre_merge_check - PR readiness with risk scoring, test gaps, dead code
"""
from __future__ import annotations
from fastmcp.prompts.prompt import Message
_TOKEN_EFFICIENCY_PREAMBLE = ( # nosec B105 — prompt template, not a password
"""\
## Rules for Token-Efficient Graph Usage
1. ALWAYS call `get_minimal_context` first with a task description.
2. Use `detail_level="minimal"` on all tool calls unless the minimal output \
is insufficient.
3. Only escalate to `detail_level="standard"` or `"verbose"` for the specific \
entities that need deeper inspection.
4. Never request more than 3 tool calls per turn unless absolutely necessary.
5. Prefer targeted queries (query_graph with a specific symbol) over broad \
scans (list_communities with full members).
6. When reviewing changes: detect_changes(detail_level="minimal") → only \
expand on high-risk items.
"""
)
def _user(content: str) -> list[Message]:
"""Wrap content as a single-message user prompt.
fastmcp >=3.2 rejects raw dicts in prompt return values; each message
must be a ``Message`` instance (or a plain ``str``). We standardise on
``Message`` so role is explicit and future multi-turn prompts compose
naturally.
"""
return [Message(role="user", content=content)]
def review_changes_prompt(base: str = "HEAD~1") -> list[Message]:
"""Pre-commit review workflow.
Args:
base: Git ref to diff against. Default: HEAD~1.
"""
return _user(
f"{_TOKEN_EFFICIENCY_PREAMBLE}\n"
f"## Review Workflow\n"
f'1. Call `get_minimal_context(task="review changes against '
f'{base}")` to get risk overview.\n'
f'2. If risk is "low": call '
f'`detect_changes(detail_level="minimal")` → report summary '
f"+ any test gaps.\n"
f'3. If risk is "medium" or "high":\n'
f' a. Call `detect_changes(detail_level="standard")` for '
f"full change list.\n"
f" b. For each high-risk function, call "
f'`query_graph(pattern="callers_of", target=<func>, '
f'detail_level="minimal")`.\n'
f' c. Call `get_affected_flows(detail_level="minimal")` '
f"only if >3 changed functions.\n"
f"4. Summarize: risk level, what changed, test gaps, "
f"specific improvements needed.\n\n"
f"Do NOT call get_review_context unless you need source code "
f"snippets for a specific function."
)
def architecture_map_prompt() -> list[Message]:
"""Architecture documentation workflow."""
return _user(
f"{_TOKEN_EFFICIENCY_PREAMBLE}\n"
"## Architecture Mapping Workflow\n"
'1. Call `get_minimal_context(task="map architecture")`.\n'
'2. Call `get_architecture_overview(detail_level="minimal")` '
"for community coupling summary.\n"
'3. Call `list_flows(detail_level="minimal")` for critical '
"flow names + criticality scores.\n"
"4. Only call `get_community(name=<X>, "
'detail_level="standard")` for the 1-2 communities the user '
"is most interested in.\n"
"5. Produce a concise Mermaid diagram showing communities as "
"boxes and key flows as arrows."
)
def debug_issue_prompt(description: str = "") -> list[Message]:
"""Guided debugging workflow.
Args:
description: Description of the issue to debug.
"""
desc_part = description or "<description>"
return _user(
f"{_TOKEN_EFFICIENCY_PREAMBLE}\n"
"## Debug Workflow\n"
f'1. Call `get_minimal_context(task="debug: '
f'{desc_part}")`.\n'
"2. Call `semantic_search_nodes(query=<keywords from "
'description>, detail_level="minimal", limit=5)`.\n'
"3. For the top 1-2 results, call "
'`query_graph(pattern="callers_of", target=<name>, '
'detail_level="minimal")`.\n'
"4. If the issue involves execution flow: call "
"`get_flow(name=<relevant flow>)` for the single most "
"relevant flow.\n"
"5. Only call `get_review_context` or `get_impact_radius` "
"if you need to trace the blast radius of a specific change."
)
def onboard_developer_prompt() -> list[Message]:
"""New developer orientation workflow."""
return _user(
f"{_TOKEN_EFFICIENCY_PREAMBLE}\n"
"## Onboarding Workflow\n"
'1. Call `get_minimal_context(task="onboard developer")`.\n'
"2. Call `list_graph_stats()` for technology overview.\n"
'3. Call `get_architecture_overview(detail_level="minimal")` '
"for the 30-second mental model.\n"
'4. Call `list_communities(detail_level="minimal")` — '
"present as a table of module names + sizes.\n"
'5. Call `list_flows(detail_level="minimal")` — highlight '
"the top 3 critical flows.\n"
"6. Only drill into a specific community or flow if the "
"developer asks."
)
def pre_merge_check_prompt(base: str = "HEAD~1") -> list[Message]:
"""PR readiness check workflow.
Args:
base: Git ref to diff against. Default: HEAD~1.
"""
return _user(
f"{_TOKEN_EFFICIENCY_PREAMBLE}\n"
"## Pre-Merge Check Workflow\n"
'1. Call `get_minimal_context(task="pre-merge check")`.\n'
'2. Call `detect_changes(detail_level="minimal")` for risk '
"score and test gaps.\n"
"3. If risk > 0.4: call "
'`get_affected_flows(detail_level="minimal")`.\n'
"4. If test_gap_count > 0: call "
'`query_graph(pattern="tests_for", '
'target=<each untested function>, detail_level="minimal")` '
"for up to 3 functions.\n"
'5. Call `refactor(mode="dead_code", '
'detail_level="minimal")` to check for newly dead code.\n'
"6. Only call `find_large_functions` or `get_impact_radius` "
"if risk > 0.7.\n"
"7. Output: GO/NO-GO recommendation with 1-sentence "
"justification + list of required follow-ups."
)
+852
View File
@@ -0,0 +1,852 @@
"""Graph-powered refactoring operations.
Provides rename previews, dead code detection, refactoring suggestions,
and safe application of refactoring edits to source files. All file writes
go through a preview-then-apply workflow with expiry enforcement and path
traversal prevention.
"""
from __future__ import annotations
import functools
import logging
import re
import threading
import time
import uuid
from pathlib import Path
from typing import Any, Optional, Union
from .flows import _has_framework_decorator, _matches_entry_name
from .graph import GraphStore, _sanitize_name
logger = logging.getLogger(__name__)
# Base class names that indicate a framework-managed class (ORM models,
# Pydantic schemas, settings). Classes inheriting from these are invoked
# via metaclass/framework magic and should not be flagged as dead code.
_FRAMEWORK_BASE_CLASSES = frozenset({
"Base", "DeclarativeBase", "Model", "BaseModel", "BaseSettings",
"db.Model", "TableBase",
# AWS CDK constructs -- instantiated by CDK app wiring, not explicit CALLS.
"Stack", "NestedStack", "Construct", "Resource",
})
# Class name suffixes that indicate CDK/IaC constructs.
# These are instantiated by framework wiring, not direct CALLS edges.
# Used as fallback when INHERITS edges to external base classes are absent.
_CDK_CLASS_SUFFIXES = ("Stack", "Construct", "Pipeline", "Resources", "Layer")
# Patterns for mock/stub variables in test files that should not be flagged dead.
_MOCK_NAME_RE = re.compile(
r"^(mock[A-Z_]|Mock[A-Z]|createMock[A-Z])|" # mockDynamoClient, MockService, createMockX
r"(Mock|Stub|Fake|Spy)$", # s3ClientMock, dbStub
re.IGNORECASE,
)
# ---------------------------------------------------------------------------
# Thread-safe pending refactors storage
# ---------------------------------------------------------------------------
_refactor_lock = threading.Lock()
_pending_refactors: dict[str, dict] = {}
REFACTOR_EXPIRY_SECONDS = 600 # 10 minutes
def _cleanup_expired() -> int:
"""Remove expired refactors from the pending dict. Returns count removed."""
now = time.time()
expired = [
rid for rid, r in _pending_refactors.items()
if now - r["created_at"] > REFACTOR_EXPIRY_SECONDS
]
for rid in expired:
del _pending_refactors[rid]
return len(expired)
# ---------------------------------------------------------------------------
# 1. rename_preview
# ---------------------------------------------------------------------------
def rename_preview(
store: GraphStore,
old_name: str,
new_name: str,
) -> Optional[dict[str, Any]]:
"""Build a rename edit list for *old_name* -> *new_name*.
Finds the node via ``store.search_nodes(old_name)``, collects
definition and reference sites, generates a unique ``refactor_id``,
and stores the preview in the thread-safe ``_pending_refactors`` dict.
Returns:
A refactor preview dict, or ``None`` if the node is not found.
"""
candidates = store.search_nodes(old_name, limit=10)
# Pick the best match: prefer exact name match.
node = None
for c in candidates:
if c.name == old_name:
node = c
break
if node is None and candidates:
node = candidates[0]
if node is None:
logger.warning("rename_preview: node %r not found", old_name)
return None
edits: list[dict[str, Any]] = []
# --- Definition site ---
edits.append({
"file": node.file_path,
"line": node.line_start,
"old": old_name,
"new": new_name,
"confidence": "high",
})
# --- Call sites (CALLS edges targeting this node) ---
call_edges = store.get_edges_by_target(node.qualified_name)
for edge in call_edges:
if edge.kind == "CALLS":
edits.append({
"file": edge.file_path,
"line": edge.line,
"old": old_name,
"new": new_name,
"confidence": "high",
})
# Also search by bare name for unqualified edges.
bare_edges = store.search_edges_by_target_name(old_name, kind="CALLS")
seen = {(e["file"], e["line"]) for e in edits}
for edge in bare_edges:
key = (edge.file_path, edge.line)
if key not in seen:
edits.append({
"file": edge.file_path,
"line": edge.line,
"old": old_name,
"new": new_name,
"confidence": "high",
})
seen.add(key)
# --- Import sites (IMPORTS_FROM edges targeting this node) ---
import_edges = store.get_edges_by_target(node.qualified_name)
for edge in import_edges:
if edge.kind == "IMPORTS_FROM":
key = (edge.file_path, edge.line)
if key not in seen:
edits.append({
"file": edge.file_path,
"line": edge.line,
"old": old_name,
"new": new_name,
"confidence": "high",
})
seen.add(key)
# --- Stats ---
stats = {"high": 0, "medium": 0, "low": 0}
for e in edits:
stats[e["confidence"]] += 1
refactor_id = uuid.uuid4().hex[:8]
preview: dict[str, Any] = {
"refactor_id": refactor_id,
"type": "rename",
"old_name": _sanitize_name(old_name),
"new_name": _sanitize_name(new_name),
"edits": edits,
"stats": stats,
"created_at": time.time(),
}
with _refactor_lock:
_cleanup_expired()
_pending_refactors[refactor_id] = preview
logger.info(
"rename_preview: created refactor %s (%s -> %s, %d edits)",
refactor_id, old_name, new_name, len(edits),
)
return preview
# ---------------------------------------------------------------------------
# 2. find_dead_code
# ---------------------------------------------------------------------------
def _is_entry_point(node: Any) -> bool:
"""Check if a node looks like an entry point by name or decorator.
Unlike ``flows.detect_entry_points()`` which treats ALL uncalled functions
as entry points, this checks only for conventional name patterns and
framework decorators -- the indicators that a function is *intentionally*
an entry point rather than simply unreferenced dead code.
"""
if _has_framework_decorator(node):
return True
if _matches_entry_name(node):
return True
return False
# Matches identifiers inside type annotations (e.g. "GoalCreate" in
# "body: GoalCreate", "Optional[UserResponse]", "list[Item]").
_TEST_FILE_RE = re.compile(
r"([\\/]__tests__[\\/]|\.spec\.[jt]sx?$|\.test\.[jt]sx?$|[\\/]test_[^/\\]*\.py$"
r"|[\\/]e2e[_-]?tests?[\\/]|[\\/]test[_-]utils?[\\/])",
)
def _is_test_file(file_path: str) -> bool:
"""Return True if *file_path* looks like a test file."""
return bool(_TEST_FILE_RE.search(file_path))
_MIN_PKG_SEGMENT_LEN = 4 # ignore short dirs like "src", "lib", "app"
@functools.lru_cache(maxsize=4096)
def _path_segments(file_path: str) -> tuple[str, ...]:
"""Return directory segments long enough to serve as package-name anchors."""
parts = file_path.replace("\\", "/").split("/")
return tuple(
p for p in parts[:-1] # skip the filename itself
if len(p) >= _MIN_PKG_SEGMENT_LEN and p not in ("home", "src", "lib", "app")
)
_TYPE_IDENT_RE = re.compile(r"[A-Z][A-Za-z0-9_]*")
def _collect_type_referenced_names(store: GraphStore) -> set[str]:
"""Collect class names that appear in function params or return types."""
funcs = store.get_nodes_by_kind(kinds=["Function", "Test"])
names: set[str] = set()
for f in funcs:
for text in (f.params, f.return_type):
if text:
names.update(_TYPE_IDENT_RE.findall(text))
return names
def find_dead_code(
store: GraphStore,
kind: Optional[str] = None,
file_pattern: Optional[str] = None,
root: Optional[Union[str, Path]] = None,
) -> list[dict[str, Any]]:
"""Find functions/classes with no callers, no test refs, no importers, and no references.
Entry points (functions matching framework decorators or conventional name
patterns like ``main``, ``test_*``, ``handle_*``) are excluded.
.. note::
**Caveats — dynamic dispatch patterns.** Static analysis cannot track
all runtime-determined call patterns. Functions registered via fully
dynamic keys (``map[computedKey()] = fn``), ``Reflect.apply``, or
runtime ``require()`` may still appear as dead code. Treat results as
hints, especially for TypeScript projects that use map-based dispatch,
plugin registries, or dynamic requires.
Args:
store: The GraphStore instance.
kind: Optional filter (e.g. ``"Function"`` or ``"Class"``).
file_pattern: Optional file-path substring filter.
root: Optional repo root path for computing ``relative_path``.
Returns:
List of dead-code dicts with name, qualified_name, kind, file_path,
relative_path, line, and language fields.
"""
# Query candidate nodes.
candidates = store.get_nodes_by_kind(
kinds=[kind] if kind else ["Function", "Class"],
file_pattern=file_pattern,
)
# Build set of class names referenced in function type annotations.
type_ref_names = _collect_type_referenced_names(store)
# Build class hierarchy: class_qualified_name -> [bare_base_names]
class_bases: dict[str, list[str]] = {}
conn = store._conn
for row in conn.execute(
"SELECT source_qualified, target_qualified FROM edges WHERE kind = 'INHERITS'"
).fetchall():
base = row[1].rsplit("::", 1)[-1] if "::" in row[1] else row[1]
class_bases.setdefault(row[0], []).append(base)
# Build import graph: file_path -> set of file_paths it imports from.
# Used to filter bare-name caller matches to plausible callers.
importer_files: dict[str, set[str]] = {}
for row in conn.execute(
"SELECT file_path, target_qualified FROM edges WHERE kind = 'IMPORTS_FROM'"
).fetchall():
importer_files.setdefault(row[0], set()).add(row[1])
# Build set of globally unique names (only one non-test node with that name).
# For unique names, any bare-name CALLS edge is reliable — no ambiguity.
name_counts: dict[str, int] = {}
for row in conn.execute(
"SELECT name, COUNT(*) FROM nodes "
"WHERE kind IN ('Function', 'Class') AND is_test = 0 "
"GROUP BY name"
).fetchall():
name_counts[row[0]] = row[1]
def _is_plausible_caller(
edge_file: str, node_file: str, node_name: str = "",
) -> bool:
"""A bare-name edge is plausible if it comes from the same file,
from a file that has an IMPORTS_FROM edge whose target matches
the node's file path, or the name is globally unique (no ambiguity)."""
if edge_file == node_file:
return True
# Unique names (only one definition) have no ambiguity -- accept all callers.
if node_name and name_counts.get(node_name, 0) == 1:
return True
for imp_target in importer_files.get(edge_file, ()):
# Strip "::name" suffix — workspace-resolved imports may include it
imp_path = imp_target.split("::")[0] if "::" in imp_target else imp_target
# __init__.py represents its parent package directory
if imp_path.endswith("/__init__.py"):
imp_dir = imp_path[:-12] # strip "/__init__.py"
if node_file.startswith(imp_dir + "/"):
return True
if imp_path.startswith(node_file) or node_file.startswith(imp_path + "/"):
return True
# 2-hop: edge_file imports X, X re-exports from node_file (barrel files)
for imp2 in importer_files.get(imp_target, ()):
imp2_path = imp2.split("::")[0] if "::" in imp2 else imp2
if imp2_path.endswith("/__init__.py"):
imp2_dir = imp2_path[:-12]
if node_file.startswith(imp2_dir + "/"):
return True
if imp2_path.startswith(node_file) or node_file.startswith(imp2_path + "/"):
return True
# Package-alias heuristic: monorepo imports like "@scope/pkg-name"
# contain the directory name of the target package. Check if the
# import target string contains a significant directory segment from
# the node's file path (e.g. "lambda-common" in both the import
# "@cova-utils/lambda-common" and the path "libraries/lambda-common/...").
if not imp_target.startswith("/"):
# imp_target is a package specifier, not a file path
for seg in _path_segments(node_file):
if seg in imp_target:
return True
return False
dead: list[dict[str, Any]] = []
for node in candidates:
# Skip test nodes and anything defined in test files.
if node.is_test or _is_test_file(node.file_path):
continue
# Skip ambient type declarations (.d.ts) — they describe external APIs.
if node.file_path.endswith(".d.ts"):
continue
# Skip dunder methods -- invoked by runtime, never have explicit callers.
if node.name.startswith("__") and node.name.endswith("__"):
continue
# Skip JS/TS/Java constructors -- invoked via `new ClassName()`, which
# creates a CALLS edge to the class, not to `constructor`.
if node.name == "constructor" and node.parent_name:
continue
# Skip mock/stub variables in test files -- these are test helpers
# referenced via variable assignment, not function calls.
if node.is_test or _is_test_file(node.file_path):
if _MOCK_NAME_RE.search(node.name):
continue
# Skip entry points (by name pattern or decorator, not just "uncalled").
if _is_entry_point(node):
continue
# Check for callers (CALLS), test refs (TESTED_BY), importers (IMPORTS_FROM),
# and value references (REFERENCES -- function-as-value in maps, arrays, etc.).
# Skip classes referenced in type annotations (Pydantic schemas, etc.).
if node.kind == "Class" and node.name in type_ref_names:
continue
# Skip Angular/NestJS decorated classes -- they are framework-managed
# and instantiated by the DI container, not direct CALLS edges.
if node.kind == "Class" and _has_framework_decorator(node):
continue
# Skip classes (and their methods) inheriting from known framework bases.
_is_framework_class = False
_check_qn = node.qualified_name if node.kind == "Class" else (
node.qualified_name.rsplit(".", 1)[0] if node.parent_name else None
)
if _check_qn:
outgoing = store.get_edges_by_source(_check_qn)
base_names = {
e.target_qualified.rsplit("::", 1)[-1]
for e in outgoing if e.kind == "INHERITS"
}
if base_names & _FRAMEWORK_BASE_CLASSES:
_is_framework_class = True
if node.kind == "Class":
if _is_framework_class:
continue
# Fallback: CDK class name suffixes (no INHERITS edge for external bases)
if any(node.name.endswith(s) for s in _CDK_CLASS_SUFFIXES):
continue
if node.kind == "Function" and _is_framework_class:
continue
# Also skip methods whose parent class name matches CDK suffixes
# (fallback for external base classes without INHERITS edges).
if (
node.kind == "Function"
and node.parent_name
and any(node.parent_name.endswith(s) for s in _CDK_CLASS_SUFFIXES)
):
continue
# Skip decorated functions/classes that are invoked implicitly rather
# than via explicit CALLS edges.
decorators = node.extra.get("decorators", ())
if isinstance(decorators, (list, tuple)) and decorators:
if node.kind in ("Function", "Test"):
# @property -- invoked via attribute access
# @abstractmethod -- polymorphic dispatch, never called directly
# @classmethod/@staticmethod -- called via Class.method()
if any(
d in ("property", "abstractmethod", "classmethod", "staticmethod")
or d.endswith(".abstractmethod")
# Angular @HostListener -- method called by framework event system
or d.startswith("HostListener")
for d in decorators
):
continue
if node.kind == "Class":
# @dataclass classes are instantiated as types, not via CALLS
if any("dataclass" in d for d in decorators):
continue
# Skip methods that override an @abstractmethod in a base class --
# they are called polymorphically via the base class reference.
if node.kind == "Function" and node.parent_name:
parent_qn = node.qualified_name.rsplit(".", 1)[0]
parent_edges = store.get_edges_by_source(parent_qn)
base_class_names = [
e.target_qualified for e in parent_edges if e.kind == "INHERITS"
]
for base_name in base_class_names:
# Try fully-qualified base first, then bare name match
base_method_qn = f"{base_name}.{node.name}"
base_nodes = store.get_node(base_method_qn)
if base_nodes is None:
# Base class may be bare name -- search in same file
base_method_qn2 = (
node.file_path + "::" + base_name + "." + node.name
)
base_nodes = store.get_node(base_method_qn2)
if base_nodes is not None:
base_decos = base_nodes.extra.get("decorators", ())
if isinstance(base_decos, (list, tuple)) and any(
"abstractmethod" in d for d in base_decos
):
break
else:
base_name = None # no abstract override found
if base_name is not None:
continue
incoming = store.get_edges_by_target(node.qualified_name)
# Also check class-qualified edges (e.g. "ClassName::method") which
# lack the file-path prefix used in node.qualified_name.
if not any(e.kind == "CALLS" for e in incoming) and node.parent_name:
class_qn = f"{node.parent_name}::{node.name}"
incoming = incoming + store.get_edges_by_target(class_qn)
# Also check bare-name and partially-qualified edges.
# CALLS targets may be bare ("funcName"), class-qualified
# ("Class::method"), or workspace-qualified ("pkg/dir::funcName").
if not any(e.kind == "CALLS" for e in incoming):
bare = store.search_edges_by_target_name(node.name, kind="CALLS")
# Also search for partially-qualified targets ending with ::name
suffix_rows = conn.execute(
"SELECT * FROM edges WHERE kind = 'CALLS'"
" AND target_qualified LIKE ?",
(f"%::{node.name}",),
).fetchall()
suffix_edges = [store._row_to_edge(r) for r in suffix_rows]
all_bare = bare + suffix_edges
all_bare = [
e for e in all_bare
if _is_plausible_caller(e.file_path, node.file_path, node.name)
]
incoming = incoming + all_bare
if not any(e.kind == "TESTED_BY" for e in incoming):
bare_tb = store.search_edges_by_target_name(node.name, kind="TESTED_BY")
bare_tb = [
e for e in bare_tb
if _is_plausible_caller(e.file_path, node.file_path, node.name)
]
incoming = incoming + bare_tb
# Check INHERITS -- classes with subclasses are not dead.
if node.kind == "Class" and not any(e.kind == "INHERITS" for e in incoming):
bare_inh = store.search_edges_by_target_name(node.name, kind="INHERITS")
incoming = incoming + bare_inh
has_callers = any(e.kind == "CALLS" for e in incoming)
has_test_refs = any(e.kind == "TESTED_BY" for e in incoming)
has_importers = any(e.kind == "IMPORTS_FROM" for e in incoming)
has_references = any(e.kind == "REFERENCES" for e in incoming)
has_subclasses = any(e.kind == "INHERITS" for e in incoming)
# For classes with no direct references, check if any member has callers.
no_refs = not (
has_callers or has_test_refs or has_importers
or has_references or has_subclasses
)
if node.kind == "Class" and no_refs:
member_prefix = node.qualified_name + "."
# Also check bare class-name pattern (unresolved CALLS targets)
bare_prefix = node.name + "."
member_calls = conn.execute(
"SELECT COUNT(*) FROM edges WHERE kind = 'CALLS'"
" AND (target_qualified LIKE ? OR target_qualified LIKE ?)",
(f"%{member_prefix}%", f"%{bare_prefix}%"),
).fetchone()[0]
if member_calls > 0:
has_callers = True
if not (
has_callers or has_test_refs or has_importers
or has_references or has_subclasses
):
# Check if this is a method override where the base class method
# has callers (polymorphic dispatch: callers of Base.method()
# implicitly call SubClass.method() at runtime).
if node.kind == "Function" and node.parent_name and not has_callers:
method_suffix = "." + node.name
if node.qualified_name.endswith(method_suffix):
class_qn = node.qualified_name[: -len(method_suffix)]
for base_name in class_bases.get(class_qn, []):
rows = conn.execute(
"SELECT n.qualified_name FROM nodes n "
"WHERE n.parent_name = ? AND n.name = ? "
"AND n.kind IN ('Function', 'Test')",
(base_name, node.name),
).fetchall()
for (base_method_qn,) in rows:
if conn.execute(
"SELECT 1 FROM edges "
"WHERE target_qualified = ? AND kind = 'CALLS' "
"LIMIT 1",
(base_method_qn,),
).fetchone():
has_callers = True
break
if has_callers:
break
if not has_callers:
if root:
try:
rel = str(Path(node.file_path).relative_to(root))
except ValueError:
rel = node.file_path
else:
rel = node.file_path
dead.append({
"name": _sanitize_name(node.name),
"qualified_name": _sanitize_name(node.qualified_name),
"kind": node.kind,
"file": node.file_path,
"file_path": node.file_path,
"relative_path": rel,
"line": node.line_start,
"language": node.language,
})
logger.info("find_dead_code: found %d dead symbols", len(dead))
return dead
# ---------------------------------------------------------------------------
# 3. suggest_refactorings
# ---------------------------------------------------------------------------
def suggest_refactorings(store: GraphStore) -> list[dict[str, Any]]:
"""Produce community-driven refactoring suggestions.
Currently two categories:
- **move**: Functions in Community A only called by Community B.
- **remove**: Dead code (no callers, tests, or importers and not entry points).
Returns:
List of suggestion dicts with type, description, symbols, rationale.
"""
suggestions: list[dict[str, Any]] = []
# --- Dead code suggestions ---
dead = find_dead_code(store)
for d in dead:
suggestions.append({
"type": "remove",
"description": f"Remove unused {d['kind'].lower()} '{d['name']}'",
"symbols": [d["qualified_name"]],
"rationale": "No callers, no test references, no importers, not an entry point.",
})
# --- Cross-community move suggestions ---
# Only attempt if communities table exists and has data.
community_rows = store.get_communities_list()
if community_rows:
# Build node -> community_id mapping.
node_community: dict[str, int] = {}
for crow in community_rows:
cid = crow["id"]
member_qns = store.get_community_member_qns(cid)
for qn in member_qns:
node_community[qn] = cid
community_names: dict[int, str] = {
r["id"]: r["name"] for r in community_rows
}
# Check functions called only by members of a different community.
all_funcs = store.get_nodes_by_kind(["Function"])
for fnode in all_funcs:
f_community = node_community.get(fnode.qualified_name)
if f_community is None:
continue
incoming_calls = [
e for e in store.get_edges_by_target(fnode.qualified_name)
if e.kind == "CALLS"
]
if not incoming_calls:
continue
caller_communities = set()
for edge in incoming_calls:
c_community = node_community.get(edge.source_qualified)
if c_community is not None:
caller_communities.add(c_community)
# If ALL callers are from a single *different* community, suggest move.
if len(caller_communities) == 1:
target_community = next(iter(caller_communities))
if target_community != f_community:
src_name = community_names.get(f_community, f"community-{f_community}")
tgt_name = community_names.get(
target_community, f"community-{target_community}"
)
suggestions.append({
"type": "move",
"description": (
f"Move '{_sanitize_name(fnode.name)}' from "
f"'{src_name}' to '{tgt_name}'"
),
"symbols": [_sanitize_name(fnode.qualified_name)],
"rationale": (
f"Function is in community '{src_name}' but only "
f"called by members of community '{tgt_name}'."
),
})
logger.info("suggest_refactorings: produced %d suggestions", len(suggestions))
return suggestions
# ---------------------------------------------------------------------------
# 4. apply_refactor
# ---------------------------------------------------------------------------
def apply_refactor(
refactor_id: str,
repo_root: Path,
dry_run: bool = False,
) -> dict[str, Any]:
"""Apply a previously previewed refactoring to source files.
Validates the refactor_id, checks expiry, ensures all edit paths are
within the repo root, then performs exact string replacements on the
target files.
Args:
refactor_id: ID from a prior ``rename_preview`` call.
repo_root: Validated repository root path.
dry_run: If True, compute the would-be changes and return a
unified-diff representation per affected file, but do NOT
write anything to disk. The ``refactor_id`` is preserved so
the same preview can be committed afterwards via a second
call without ``dry_run``. See: #176
Returns:
Status dict with applied count and modified files. When
``dry_run=True`` the dict additionally contains:
- ``dry_run``: ``True``
- ``would_modify``: list of file paths that would be changed
- ``diffs``: map of file path → unified diff string showing the
proposed change
"""
repo_root = repo_root.resolve()
with _refactor_lock:
_cleanup_expired()
preview = _pending_refactors.get(refactor_id)
if preview is None:
logger.warning("apply_refactor: unknown or expired refactor_id %s", refactor_id)
return {"status": "error", "error": f"Refactor '{refactor_id}' not found or expired."}
# Check expiry explicitly.
age = time.time() - preview["created_at"]
if age > REFACTOR_EXPIRY_SECONDS:
with _refactor_lock:
_pending_refactors.pop(refactor_id, None)
logger.warning("apply_refactor: refactor %s expired (%.0fs old)", refactor_id, age)
return {"status": "error", "error": f"Refactor '{refactor_id}' has expired."}
edits = preview.get("edits", [])
if not edits:
if dry_run:
return {
"status": "ok", "dry_run": True, "applied": 0,
"files_modified": [], "edits_applied": 0,
"would_modify": [], "diffs": {},
}
return {"status": "ok", "applied": 0, "files_modified": [], "edits_applied": 0}
# --- Path traversal validation ---
for edit in edits:
edit_path = Path(edit["file"]).resolve()
try:
edit_path.relative_to(repo_root)
except ValueError:
logger.error(
"apply_refactor: path traversal blocked for %s (repo_root=%s)",
edit_path, repo_root,
)
return {
"status": "error",
"error": f"Edit path '{edit['file']}' is outside repo root.",
}
# --- Compute new content for every edit (shared by dry-run and write paths) ---
# Group edits by file so multiple edits to the same file apply
# sequentially against the updated content rather than stomping each
# other. Dry-run and write modes then share this computation.
from collections import defaultdict
edits_by_file: dict[str, list[dict]] = defaultdict(list)
for edit in edits:
edits_by_file[edit["file"]].append(edit)
planned: dict[str, tuple[str, str, int]] = {} # file -> (old_content, new_content, edit_count)
for file_str, file_edits in edits_by_file.items():
file_path = Path(file_str)
if not file_path.is_file():
logger.warning("apply_refactor: file not found: %s", file_path)
continue
try:
original = file_path.read_text(encoding="utf-8", errors="replace")
except (OSError, UnicodeDecodeError) as exc:
logger.warning("apply_refactor: could not read %s: %s", file_path, exc)
continue
content = original
file_edits_applied = 0
for edit in file_edits:
old_text = edit["old"]
new_text = edit["new"]
if old_text not in content:
logger.warning(
"apply_refactor: old text %r not found in %s",
old_text, file_path,
)
continue
target_line = edit.get("line")
if target_line is not None:
lines = content.splitlines(keepends=True)
idx = target_line - 1
if 0 <= idx < len(lines) and old_text in lines[idx]:
lines[idx] = lines[idx].replace(old_text, new_text, 1)
content = "".join(lines)
else:
content = content.replace(old_text, new_text, 1)
else:
content = content.replace(old_text, new_text, 1)
file_edits_applied += 1
if file_edits_applied > 0:
planned[file_str] = (original, content, file_edits_applied)
# --- Dry-run path: return diffs, no writes ---
if dry_run:
import difflib
diffs: dict[str, str] = {}
for file_str, (original, new_content, _count) in planned.items():
diff_lines = list(difflib.unified_diff(
original.splitlines(keepends=True),
new_content.splitlines(keepends=True),
fromfile=f"a/{file_str}",
tofile=f"b/{file_str}",
n=3,
))
diffs[file_str] = "".join(diff_lines)
total_edits = sum(count for _o, _n, count in planned.values())
result = {
"status": "ok",
"dry_run": True,
"applied": 0,
"edits_applied": total_edits,
"would_modify": sorted(planned.keys()),
"files_modified": [],
"diffs": diffs,
}
logger.info(
"apply_refactor: dry-run %s%d edits would be applied to %d files",
refactor_id, total_edits, len(planned),
)
# Do NOT pop the pending refactor — let the user commit via a
# second call with dry_run=False.
return result
# --- Real-write path: write the pre-computed new content ---
files_modified: set[str] = set()
edits_applied = 0
for file_str, (_original, new_content, count) in planned.items():
file_path = Path(file_str)
try:
file_path.write_text(new_content, encoding="utf-8")
edits_applied += count
files_modified.add(str(file_path))
logger.info("apply_refactor: applied %d edit(s) to %s", count, file_path)
except OSError as exc:
logger.error("apply_refactor: could not write %s: %s", file_path, exc)
# Remove from pending after successful application.
with _refactor_lock:
_pending_refactors.pop(refactor_id, None)
result = {
"status": "ok",
"applied": edits_applied,
"files_modified": sorted(files_modified),
"edits_applied": edits_applied,
}
logger.info("apply_refactor: completed %s%d edits applied", refactor_id, edits_applied)
return result
+319
View File
@@ -0,0 +1,319 @@
"""Multi-repo registry and connection pool.
Manages a registry of multiple repositories at ``~/.code-review-graph/registry.json``
and provides a connection pool for concurrent access to multiple graph databases.
"""
from __future__ import annotations
import json
import logging
import sqlite3
import threading
from collections import OrderedDict
from pathlib import Path
logger = logging.getLogger(__name__)
# Default registry path
_REGISTRY_DIR = Path.home() / ".code-review-graph"
_REGISTRY_PATH = _REGISTRY_DIR / "registry.json"
class Registry:
"""Manages a JSON-based registry of code-review-graph repositories.
Each entry stores the repo path and an optional alias.
The registry lives at ``~/.code-review-graph/registry.json``.
"""
def __init__(self, path: Path | None = None) -> None:
self._path = path or _REGISTRY_PATH
self._path.parent.mkdir(parents=True, exist_ok=True)
self._lock = threading.Lock()
self._repos: list[dict[str, str]] = []
self._load()
def _load(self) -> None:
"""Load registry from disk."""
if self._path.exists():
try:
data = json.loads(self._path.read_text(encoding="utf-8", errors="replace"))
self._repos = data.get("repos", [])
except (json.JSONDecodeError, KeyError, TypeError):
logger.warning("Invalid registry file, starting fresh: %s", self._path)
self._repos = []
else:
self._repos = []
def _save(self) -> None:
"""Write registry to disk."""
self._path.parent.mkdir(parents=True, exist_ok=True)
data = {"repos": self._repos}
self._path.write_text(
json.dumps(data, indent=2) + "\n", encoding="utf-8"
)
def register(
self, path: str, alias: str | None = None, data_dir: str | None = None,
) -> dict[str, str]:
"""Register a repository path.
Validates that the path contains a ``.git`` or ``.code-review-graph``
directory.
Args:
path: Absolute or relative path to the repository root.
alias: Optional short alias for the repository.
data_dir: Optional external directory for graph database.
Returns:
The registered entry dict.
Raises:
ValueError: If the path is not a valid repository.
"""
resolved = Path(path).resolve()
if not resolved.is_dir():
raise ValueError(f"Path is not a directory: {resolved}")
has_repo_marker = (
(resolved / ".git").exists()
or (resolved / ".svn").exists()
or (resolved / ".code-review-graph").exists()
)
if not has_repo_marker:
raise ValueError(
f"Path does not look like a repository "
f"(no .git, .svn, or .code-review-graph): {resolved}"
)
with self._lock:
# Check for duplicate path
str_path = str(resolved)
for entry in self._repos:
if entry["path"] == str_path:
# Update alias and/or data_dir if provided
if alias:
entry["alias"] = alias
if data_dir:
entry["data_dir"] = str(Path(data_dir).resolve())
self._save()
return entry
new_entry: dict[str, str] = {"path": str_path}
if alias:
new_entry["alias"] = alias
if data_dir:
new_entry["data_dir"] = str(Path(data_dir).resolve())
self._repos.append(new_entry)
self._save()
return new_entry
def unregister(self, path_or_alias: str) -> bool:
"""Remove a repository by path or alias.
Args:
path_or_alias: Either the absolute path or the alias.
Returns:
True if an entry was removed, False otherwise.
"""
with self._lock:
resolved = str(Path(path_or_alias).resolve())
original_len = len(self._repos)
self._repos = [
entry for entry in self._repos
if entry["path"] != resolved
and entry.get("alias") != path_or_alias
]
if len(self._repos) < original_len:
self._save()
return True
return False
def list_repos(self) -> list[dict[str, str]]:
"""Return list of all registered repositories.
Returns:
List of dicts with 'path' and optional 'alias' keys.
"""
with self._lock:
return list(self._repos)
def find_by_alias(self, alias: str) -> dict[str, str] | None:
"""Look up a repository by its alias.
Args:
alias: The alias to search for.
Returns:
The matching entry, or None.
"""
with self._lock:
for entry in self._repos:
if entry.get("alias") == alias:
return dict(entry)
return None
def find_by_path(self, path: str) -> dict[str, str] | None:
"""Look up a repository by its path.
Args:
path: The path to search for.
Returns:
The matching entry, or None.
"""
resolved = str(Path(path).resolve())
with self._lock:
for entry in self._repos:
if entry["path"] == resolved:
return dict(entry)
return None
def set_data_dir(self, path: str, data_dir: str) -> dict[str, str]:
"""Set the external data directory for a repository.
Args:
path: Repository path (absolute or relative).
data_dir: External directory path to store graph database.
Returns:
The updated or created registry entry.
"""
resolved = str(Path(path).resolve())
data_resolved = str(Path(data_dir).resolve())
with self._lock:
# Check for existing entry
for entry in self._repos:
if entry["path"] == resolved:
entry["data_dir"] = data_resolved
self._save()
return dict(entry)
# Create new entry if not found
new_entry = {
"path": resolved,
"data_dir": data_resolved
}
self._repos.append(new_entry)
self._save()
return new_entry
def get_data_dir_for_repo(self, path: str) -> str | None:
"""Get the stored data directory for a repository.
Args:
path: Repository path (absolute or relative).
Returns:
The stored data_dir path, or None if not set.
"""
resolved = str(Path(path).resolve())
with self._lock:
for entry in self._repos:
if entry["path"] == resolved:
return entry.get("data_dir")
return None
class ConnectionPool:
"""LRU connection pool for SQLite graph databases.
Caches open connections keyed by database path, evicting the least
recently used connection when the pool is full.
"""
def __init__(self, max_size: int = 10) -> None:
self._max_size = max_size
self._pool: OrderedDict[str, sqlite3.Connection] = OrderedDict()
self._lock = threading.Lock()
def get(self, db_path: str) -> sqlite3.Connection:
"""Get or create a connection for the given database path.
Args:
db_path: Path to the SQLite database file.
Returns:
An open SQLite connection.
"""
key = str(Path(db_path).resolve())
with self._lock:
if key in self._pool:
self._pool.move_to_end(key)
return self._pool[key]
# Evict LRU if full
while len(self._pool) >= self._max_size:
evict_key, evict_conn = self._pool.popitem(last=False)
try:
evict_conn.close()
except sqlite3.Error:
logger.debug("Failed to close evicted connection: %s", evict_key)
logger.debug("Evicted connection: %s", evict_key)
conn = sqlite3.connect(
key, timeout=30, check_same_thread=False,
isolation_level=None,
)
conn.row_factory = sqlite3.Row
conn.execute("PRAGMA journal_mode=WAL")
conn.execute("PRAGMA busy_timeout=5000")
self._pool[key] = conn
return conn
def close_all(self) -> None:
"""Close all connections in the pool."""
with self._lock:
for key, conn in self._pool.items():
try:
conn.close()
except sqlite3.Error:
logger.debug("Failed to close connection: %s", key)
self._pool.clear()
@property
def size(self) -> int:
"""Current number of open connections."""
with self._lock:
return len(self._pool)
def resolve_repo(
registry: Registry,
repo: str | None,
cwd: str | None = None,
) -> str | None:
"""Resolve a repo parameter to an absolute path.
Resolution order:
1. If repo is given, try as alias first.
2. If repo is given and not an alias, try as a direct path.
3. If repo is None, use cwd.
Args:
registry: The Registry instance.
repo: Alias or path string, or None.
cwd: Current working directory fallback.
Returns:
Resolved absolute path string, or None if unresolvable.
"""
if repo:
# Try alias first
entry = registry.find_by_alias(repo)
if entry:
return entry["path"]
# Try as direct path
path = Path(repo).resolve()
if path.is_dir():
return str(path)
# Fall back to CWD
if cwd:
return str(Path(cwd).resolve())
return None
+206
View File
@@ -0,0 +1,206 @@
"""Post-build pass that resolves ReScript cross-module references.
The per-file parser emits CALLS/IMPORTS_FROM edges with bare targets like
``LogicUtils.safeParse`` because the parser only sees one file at a time.
This module runs after ``full_build`` / incremental updates and rewrites
those targets to canonical qualified names like
``<abs-path>/LogicUtils.res::safeParse`` so ``callers_of``,
``get_impact_radius`` and ``importers_of`` work correctly across files.
Resolutions performed:
1. ``Module.fn`` / ``Module.Sub.fn`` CALLS edges → canonical node
when a ``.res`` / ``.resi`` file with matching basename exists.
2. Bare ``fn(...)`` CALLS edges in a file that ``open`` / ``include``\\s
a module → canonical node in that module's file.
3. IMPORTS_FROM edges targeting a module name (open / include / jsx /
module_alias / external_module) → the target file path, so
``importers_of(<path>)`` finds every consuming file.
Only the ``target_qualified`` column is updated; source and edge kind are
preserved. Edges that cannot be resolved are left unchanged.
"""
from __future__ import annotations
import json
import logging
from pathlib import Path
from typing import TYPE_CHECKING
if TYPE_CHECKING:
from .graph import GraphStore
logger = logging.getLogger(__name__)
def resolve_rescript_cross_module(store: GraphStore) -> dict:
"""Resolve ReScript cross-module targets in the graph store.
Safe to call multiple times: already-resolved edges (targets containing
``::``) are skipped.
Returns a dict with resolution counts for telemetry.
"""
conn = store._conn # intentional: post-build maintenance pass
# Basename (module name) → absolute file path, preferring .res over .resi.
basename_to_path: dict[str, str] = {}
rescript_files: set[str] = set()
for file_path in store.get_all_files():
p = Path(file_path)
suffix = p.suffix.lower()
if suffix not in (".res", ".resi"):
continue
rescript_files.add(file_path)
stem = p.stem
existing = basename_to_path.get(stem)
if existing is None or existing.lower().endswith(".resi"):
# Prefer implementation (.res) over interface (.resi).
basename_to_path[stem] = file_path
if not basename_to_path:
return {"files_indexed": 0, "calls_resolved": 0, "imports_resolved": 0}
# Per-file opens/includes so we can resolve bare calls.
opens_by_file: dict[str, list[str]] = {}
imports_rows = conn.execute(
"SELECT source_qualified, target_qualified, file_path, extra "
"FROM edges WHERE kind = 'IMPORTS_FROM'"
).fetchall()
for row in imports_rows:
fp = row["file_path"]
if fp not in rescript_files:
continue
try:
extra = json.loads(row["extra"] or "{}")
except (json.JSONDecodeError, TypeError):
extra = {}
kind = extra.get("rescript_import_kind")
if kind in ("open", "include"):
# Strip nested submodule — root determines file.
root = row["target_qualified"].split(".", 1)[0]
opens_by_file.setdefault(fp, []).append(root)
# --- 1 + 2. Resolve CALLS edges ---
call_rows = conn.execute(
"SELECT id, source_qualified, target_qualified, file_path "
"FROM edges WHERE kind = 'CALLS'"
).fetchall()
call_updates: list[tuple[str, int]] = []
for row in call_rows:
target = row["target_qualified"]
if "::" in target:
continue # already resolved
resolved = _resolve_call_target(
target,
row["file_path"],
basename_to_path,
opens_by_file,
store,
)
if resolved and resolved != target:
call_updates.append((resolved, row["id"]))
# --- 3. Resolve IMPORTS_FROM edge targets to file paths ---
import_updates: list[tuple[str, int]] = []
import_rows_full = conn.execute(
"SELECT id, target_qualified, file_path FROM edges "
"WHERE kind = 'IMPORTS_FROM'"
).fetchall()
for row in import_rows_full:
target = row["target_qualified"]
if target in rescript_files:
continue # already a file path
if "/" in target or "\\" in target:
continue # looks like a path already (e.g. relative JS import)
root = target.split(".", 1)[0]
file_target = basename_to_path.get(root)
if file_target and file_target != target:
import_updates.append((file_target, row["id"]))
cur = conn.cursor()
for new_target, edge_id in call_updates:
cur.execute(
"UPDATE edges SET target_qualified = ? WHERE id = ?",
(new_target, edge_id),
)
for new_target, edge_id in import_updates:
cur.execute(
"UPDATE edges SET target_qualified = ? WHERE id = ?",
(new_target, edge_id),
)
conn.commit()
store._invalidate_cache()
result = {
"files_indexed": len(basename_to_path),
"calls_resolved": len(call_updates),
"imports_resolved": len(import_updates),
}
logger.info("ReScript cross-module resolution: %s", result)
return result
def _resolve_call_target(
target: str,
file_path: str,
basename_to_path: dict[str, str],
opens_by_file: dict[str, list[str]],
store: GraphStore,
) -> str | None:
"""Resolve a CALLS edge's ``target_qualified`` to a canonical qualified
node name. Returns None when no resolution is possible.
"""
# Dotted: `Module.fn` or `Module.Sub.fn`.
if "." in target:
head, _, rest = target.partition(".")
target_file = basename_to_path.get(head)
if target_file is None:
return None
candidate = _pick_existing_qualified(target_file, rest, store)
return candidate
# Bare: `fn` — only resolvable via an open/include in the calling file.
for opened in opens_by_file.get(file_path, []):
target_file = basename_to_path.get(opened)
if target_file is None:
continue
candidate = f"{target_file}::{target}"
if store.get_node(candidate) is not None:
return candidate
return None
def _pick_existing_qualified(
target_file: str, rest: str, store: GraphStore,
) -> str | None:
"""Given ``LogicUtils.foo.bar``, try ``file::foo.bar`` then
``file::Foo.bar`` then ``file::foo``. Return the first one that
corresponds to an existing node.
"""
# Direct: rest as the qualified name tail.
direct = f"{target_file}::{rest}"
if store.get_node(direct) is not None:
return direct
# Dotted rest like `Sub.fn`: parent_name = Sub, name = fn.
# _qualify formats it the same way, so `direct` would already match if
# the node was stored with that exact qualified name.
# Some targets include a trailing member-access that isn't part of
# the qualified node (e.g. `LogicUtils.safeParse.resp` — property on
# the result). Try peeling from the right.
parts = rest.split(".")
while len(parts) > 1:
parts.pop()
candidate = f"{target_file}::{'.'.join(parts)}"
if store.get_node(candidate) is not None:
return candidate
# Last resort: top-level `file::name` (first part only).
first = rest.split(".", 1)[0]
candidate = f"{target_file}::{first}"
if store.get_node(candidate) is not None:
return candidate
return None
+447
View File
@@ -0,0 +1,447 @@
"""Hybrid search engine combining FTS5 (BM25) and vector embeddings.
Uses Reciprocal Rank Fusion (RRF) to merge results from full-text search
and semantic similarity, with query-aware kind boosting and context-file
boosting for relevance tuning.
"""
from __future__ import annotations
import logging
import re
import sqlite3
from typing import Any, Optional
from .graph import GraphStore, _sanitize_name
logger = logging.getLogger(__name__)
# ---------------------------------------------------------------------------
# FTS5 index management
# ---------------------------------------------------------------------------
def rebuild_fts_index(store: GraphStore) -> int:
"""Rebuild the FTS5 index from the nodes table.
Checks whether the ``nodes_fts`` virtual table exists, clears it, then
repopulates it from every row in ``nodes``.
Returns:
Number of rows indexed.
"""
# NOTE: rebuild_fts_index uses store._conn directly because it manages
# the FTS5 virtual table DDL, which is tightly coupled to SQLite internals.
conn = store._conn
# Wrap the full DROP + CREATE + INSERT sequence in an explicit transaction
# so a crash mid-rebuild cannot leave the DB without an FTS table at all
# (DROP succeeded but CREATE/INSERT didn't). See #259.
if conn.in_transaction:
logger.warning("Rolling back uncommitted transaction before BEGIN IMMEDIATE")
conn.rollback()
conn.execute("BEGIN IMMEDIATE")
try:
# Drop and recreate the FTS table with content sync to match migration v5
conn.execute("DROP TABLE IF EXISTS nodes_fts")
conn.execute("""
CREATE VIRTUAL TABLE nodes_fts USING fts5(
name, qualified_name, file_path, signature,
content='nodes', content_rowid='rowid',
tokenize='porter unicode61'
)
""")
# Rebuild from the content table (nodes) using the FTS5 rebuild command
conn.execute("INSERT INTO nodes_fts(nodes_fts) VALUES('rebuild')")
conn.commit()
except BaseException:
conn.rollback()
raise
count = conn.execute("SELECT count(*) FROM nodes_fts").fetchone()[0]
logger.info("FTS index rebuilt: %d rows indexed", count)
return count
# ---------------------------------------------------------------------------
# Query kind boosting heuristics
# ---------------------------------------------------------------------------
_DOTTED_IDENT_RE = re.compile(r'\b[A-Za-z_][\w]*(?:\.[A-Za-z_][\w]*)+\b')
_SNAKE_IDENT_RE = re.compile(r'\b[a-z][a-z0-9]*(?:_[a-z0-9]+)+\b')
_PASCAL_IDENT_RE = re.compile(r'\b[A-Z][a-z0-9]+(?:[A-Z][a-z0-9]+)+\b')
def extract_query_identifiers(query: str) -> list[str]:
"""Pull out identifier-shaped tokens from anywhere in a query.
Catches dotted forms (``Context.Next``), snake_case (``get_dependant``),
and CamelCase (``APIRoute``) even when they're embedded in a natural-
language sentence. Used to boost search hits whose qualified_name
contains any of these tokens, so an LLM asking "Who advances the gin
middleware chain via Context.Next" lands on ``Context.Next`` instead of
the bare ``Context`` class.
"""
found: list[str] = []
seen: set[str] = set()
for pat in (_DOTTED_IDENT_RE, _SNAKE_IDENT_RE, _PASCAL_IDENT_RE):
for match in pat.findall(query):
lo = match.lower()
if lo not in seen and len(lo) >= 3:
seen.add(lo)
found.append(lo)
return found
def detect_query_kind_boost(query: str) -> dict[str, Any]:
"""Detect query patterns and return per-node boost multipliers.
Heuristics:
- PascalCase queries (e.g. ``MyClass``) boost Class/Type by 1.5x
- snake_case queries (e.g. ``get_users``) boost Function by 1.5x
- Queries containing ``.`` boost qualified name matches by 2.0x
- Identifier-shaped tokens *anywhere* in the query (dotted, snake_case,
CamelCase) boost results whose qualified_name contains them by 2.0x.
See ``extract_query_identifiers``.
Returns:
Dict whose keys are either node kind strings (mapped to float
multipliers) or one of the special keys ``_qualified``,
``_qualified_identifiers``.
"""
boosts: dict[str, Any] = {}
if not query or not query.strip():
return boosts
q = query.strip()
# PascalCase: starts with uppercase, has at least one lowercase after
if re.match(r'^[A-Z][a-z]', q) and not q.isupper():
boosts["Class"] = 1.5
boosts["Type"] = 1.5
# snake_case or SCREAMING_SNAKE_CASE: contains underscore with letters
if '_' in q and re.search(r'[a-zA-Z]', q):
boosts["Function"] = 1.5
# Dotted path: boost qualified name matches
if '.' in q:
boosts["_qualified"] = 2.0
# Identifiers extracted from anywhere in the query
idents = extract_query_identifiers(q)
if idents:
boosts["_qualified_identifiers"] = idents
return boosts
# ---------------------------------------------------------------------------
# Reciprocal Rank Fusion
# ---------------------------------------------------------------------------
def rrf_merge(*result_lists: list[tuple[int, float]], k: int = 60) -> list[tuple[int, float]]:
"""Merge multiple ranked result lists using Reciprocal Rank Fusion.
Each input list contains ``(id, score)`` tuples, ordered by score
descending. The RRF score for each item is the sum of
``1 / (k + rank + 1)`` across all lists it appears in, where rank is
the 0-based position.
Args:
*result_lists: Variable number of ranked result lists.
k: RRF constant (default 60). Higher values reduce the impact of
rank differences.
Returns:
Merged list of ``(id, rrf_score)`` tuples sorted by score descending.
"""
scores: dict[int, float] = {}
for result_list in result_lists:
for rank, (item_id, _score) in enumerate(result_list):
scores[item_id] = scores.get(item_id, 0.0) + 1.0 / (k + rank + 1)
merged = sorted(scores.items(), key=lambda x: x[1], reverse=True)
return merged
# ---------------------------------------------------------------------------
# FTS5 search
# ---------------------------------------------------------------------------
def _fts_search(
conn: sqlite3.Connection,
query: str,
limit: int = 50,
) -> list[tuple[int, float]]:
"""Run an FTS5 BM25 search against the nodes_fts table.
Returns list of ``(node_id, bm25_score)`` tuples. The BM25 score is
negated so higher = better (FTS5 returns negative BM25).
"""
# Sanitize: wrap in double quotes to prevent FTS5 operator injection
safe_query = '"' + query.replace('"', '""') + '"'
try:
rows = conn.execute(
"SELECT rowid, rank FROM nodes_fts WHERE nodes_fts MATCH ? "
"ORDER BY rank LIMIT ?",
(safe_query, limit),
).fetchall()
# FTS5 rank is negative BM25 (lower = better), negate for consistency
return [(row[0], -row[1]) for row in rows]
except sqlite3.OperationalError as e:
logger.warning("FTS5 search failed: %s", e)
return []
# ---------------------------------------------------------------------------
# Embedding search (optional)
# ---------------------------------------------------------------------------
def _embedding_search(
store: GraphStore,
query: str,
limit: int = 50,
model: str | None = None,
provider: str | None = None,
) -> list[tuple[int, float]]:
"""Run a vector similarity search using the embedding store.
Returns list of ``(node_id, similarity_score)`` tuples.
Gracefully returns an empty list if embeddings are not available.
"""
try:
from .embeddings import EmbeddingStore
except ImportError:
return []
try:
emb_store = EmbeddingStore(store.db_path, provider=provider, model=model)
try:
if not emb_store.available or emb_store.count() == 0:
return []
results = emb_store.search(query, limit=limit)
# Map qualified names back to node IDs
id_scores: list[tuple[int, float]] = []
for qn, score in results:
node = store.get_node(qn)
if node:
id_scores.append((node.id, score))
return id_scores
finally:
emb_store.close()
except Exception as e:
logger.warning("Embedding search failed: %s", e)
return []
# ---------------------------------------------------------------------------
# Keyword LIKE fallback
# ---------------------------------------------------------------------------
def _keyword_search(
conn: sqlite3.Connection,
query: str,
limit: int = 50,
) -> list[tuple[int, float]]:
"""Fall back to simple LIKE keyword matching.
Each word in the query must match independently (AND logic).
Returns ``(node_id, score)`` tuples with a basic relevance score.
"""
words = query.lower().split()
if not words:
return []
conditions: list[str] = []
params: list[str | int] = []
for word in words:
conditions.append(
"(LOWER(name) LIKE ? OR LOWER(qualified_name) LIKE ?)"
)
params.extend([f"%{word}%", f"%{word}%"])
where = " AND ".join(conditions)
params.append(limit)
sql = f"SELECT id, name, qualified_name FROM nodes WHERE {where} LIMIT ?" # nosec B608
try:
rows = conn.execute(sql, params).fetchall()
except sqlite3.OperationalError:
return []
# Assign a simple relevance score: exact name match > prefix > contains
q_lower = query.lower()
results: list[tuple[int, float]] = []
for row in rows:
name_lower = row["name"].lower()
if name_lower == q_lower:
score = 3.0
elif name_lower.startswith(q_lower):
score = 2.0
else:
score = 1.0
results.append((row["id"], score))
results.sort(key=lambda x: x[1], reverse=True)
return results
# ---------------------------------------------------------------------------
# Main hybrid search
# ---------------------------------------------------------------------------
def hybrid_search(
store: GraphStore,
query: str,
kind: Optional[str] = None,
limit: int = 20,
context_files: Optional[list[str]] = None,
model: Optional[str] = None,
provider: Optional[str] = None,
) -> list[dict[str, Any]]:
"""Hybrid search combining FTS5 BM25 and vector embeddings via RRF.
Attempts FTS5 + embedding search first, falling back to FTS5-only,
then keyword LIKE matching if FTS5 is unavailable.
Args:
store: The graph store to search.
query: Search query string.
kind: Optional node kind filter (e.g. ``"Function"``, ``"Class"``).
limit: Maximum results to return (default 20).
context_files: Optional list of file paths. Nodes in these files
receive a 1.5x score boost.
Returns:
List of dicts with node metadata and ``score`` field.
"""
if not query or not query.strip():
return []
# NOTE: hybrid_search uses store._conn for FTS5 and keyword queries
# because those operate on the FTS virtual table or need raw Row
# access for batch-fetch performance. This is documented coupling.
conn = store._conn
fetch_limit = limit * 3 # Fetch extra to allow for filtering and boosting
# ------ Phase 1: Gather ranked lists ------
fts_results: list[tuple[int, float]] = []
emb_results: list[tuple[int, float]] = []
# Try FTS5 search
try:
fts_results = _fts_search(conn, query, limit=fetch_limit)
except Exception as e:
logger.warning("FTS5 unavailable, will use fallback: %s", e)
# Try embedding search
emb_results = _embedding_search(
store, query, limit=fetch_limit, model=model, provider=provider,
)
# ------ Phase 2: Merge via RRF or fallback ------
if fts_results or emb_results:
lists_to_merge = []
if fts_results:
lists_to_merge.append(fts_results)
if emb_results:
lists_to_merge.append(emb_results)
merged = rrf_merge(*lists_to_merge)
else:
# Fallback: keyword LIKE matching
keyword_results = _keyword_search(conn, query, limit=fetch_limit)
if not keyword_results:
return []
merged = keyword_results
# ------ Phase 3+4: Batch-fetch nodes, apply boosting and kind filter ------
kind_boosts = detect_query_kind_boost(query)
context_set = set(context_files) if context_files else set()
# Batch-fetch all candidate nodes in one query
candidate_ids = [node_id for node_id, _ in merged]
node_rows: dict[int, Any] = {}
batch_size = 450
for i in range(0, len(candidate_ids), batch_size):
batch = candidate_ids[i:i + batch_size]
placeholders = ",".join("?" for _ in batch)
rows = conn.execute(
f"SELECT * FROM nodes WHERE id IN ({placeholders})", # nosec B608
batch,
).fetchall()
for row in rows:
node_rows[row["id"]] = row
# Apply boosting
boosted: list[tuple[int, float]] = []
for node_id, score in merged:
row = node_rows.get(node_id)
if not row:
continue
node_kind = row["kind"]
file_path = row["file_path"]
qualified_name = row["qualified_name"]
boost = 1.0
if node_kind in kind_boosts:
boost *= kind_boosts[node_kind]
if "_qualified" in kind_boosts and '.' in query:
if query.lower() in qualified_name.lower():
boost *= kind_boosts["_qualified"]
idents = kind_boosts.get("_qualified_identifiers")
if idents:
qn_lo = qualified_name.lower()
if any(ident in qn_lo for ident in idents):
boost *= 2.0
if context_set and file_path in context_set:
boost *= 1.5
boosted.append((node_id, score * boost))
boosted.sort(key=lambda x: x[1], reverse=True)
# Build results from the already-fetched rows
results: list[dict[str, Any]] = []
for node_id, final_score in boosted:
if len(results) >= limit:
break
row = node_rows.get(node_id)
if not row:
continue
node_kind = row["kind"]
if kind and node_kind != kind:
continue
results.append({
"name": _sanitize_name(row["name"]),
"qualified_name": _sanitize_name(row["qualified_name"]),
"kind": node_kind,
"file_path": row["file_path"],
"line_start": row["line_start"],
"line_end": row["line_end"],
"language": row["language"] or "",
"params": row["params"],
"return_type": row["return_type"],
"signature": row["signature"] if "signature" in row.keys() else None,
"score": round(final_score, 6),
})
return results
File diff suppressed because it is too large Load Diff
+200
View File
@@ -0,0 +1,200 @@
"""Post-build Spring DI call resolver.
After tree-sitter parsing, Java CALLS edges whose target is a bare method
name (e.g. ``calculate``) carry ``extra.receiver`` naming the local variable
that was called on (e.g. ``invoiceCalculationService``). This module
resolves those receivers through the INJECTS map to their declared type, then
optionally to the unique concrete implementation via INHERITS edges.
Resolution chain:
receiver variable name
→ injected interface/class (from INJECTS.extra.field_name)
→ concrete implementation (from INHERITS, when unique)
Only Java files are processed. Edges that are already qualified (contain
``::``) or have no ``receiver`` extra key are skipped.
"""
from __future__ import annotations
import json
import logging
from typing import TYPE_CHECKING
if TYPE_CHECKING:
from .graph import GraphStore
logger = logging.getLogger(__name__)
def resolve_spring_di_calls(store: GraphStore) -> dict:
"""Resolve Java CALLS edges whose receiver is a Spring-injected field.
Safe to call multiple times — already-resolved edges (targets containing
``::``) are skipped.
Returns a dict with resolution counts for telemetry.
"""
conn = store._conn
# Only process Java files
java_files: set[str] = {
row["file_path"]
for row in conn.execute(
"SELECT DISTINCT file_path FROM nodes WHERE language = 'java'"
).fetchall()
}
if not java_files:
return {"files_indexed": 0, "calls_resolved": 0}
# -----------------------------------------------------------------------
# Build field_map: (source_qualified_class, field_name) → injected_type
# from INJECTS edges that carry extra.field_name
# -----------------------------------------------------------------------
field_map: dict[tuple[str, str], str] = {}
injects_rows = conn.execute(
"SELECT source_qualified, target_qualified, extra FROM edges WHERE kind = 'INJECTS'"
).fetchall()
for row in injects_rows:
try:
extra = json.loads(row["extra"] or "{}")
except (json.JSONDecodeError, TypeError):
extra = {}
fname = extra.get("field_name")
if not fname:
continue
# source_qualified is the full class qualified name
class_qual = row["source_qualified"]
field_map[(class_qual, fname)] = row["target_qualified"]
if not field_map:
logger.info("Spring resolver: no INJECTS edges with field_name found, skipping")
return {"files_indexed": len(java_files), "calls_resolved": 0}
# -----------------------------------------------------------------------
# Build class_name → qualified_name lookup from nodes.
# Keyed by bare class name; value is the full "file_path::ClassName" form
# that callers_of uses for its target_qualified exact-match lookup.
# When a name appears in multiple files (e.g. same interface in several
# services), we keep the entry with the shortest path as a tiebreaker —
# this is overridden by the concrete-implementation lookup below.
# -----------------------------------------------------------------------
name_to_qual: dict[str, str] = {}
for row in conn.execute(
"SELECT name, qualified_name FROM nodes WHERE kind = 'Class' AND language = 'java'"
).fetchall():
bare = row["name"]
qual = row["qualified_name"]
if bare not in name_to_qual or len(qual) < len(name_to_qual[bare]):
name_to_qual[bare] = qual
# Also index Function nodes so we can build "file::Class.method" targets.
# key: (class_name, method_name) → full qualified_name of the method node
method_to_qual: dict[tuple[str, str], str] = {}
for row in conn.execute(
"SELECT name, qualified_name, parent_name FROM nodes "
"WHERE kind IN ('Function', 'Test') AND language = 'java' AND parent_name IS NOT NULL"
).fetchall():
method_to_qual[(row["parent_name"], row["name"])] = row["qualified_name"]
# -----------------------------------------------------------------------
# Build implementors: bare interface name → list of implementing class quals
# from INHERITS edges (Java uses INHERITS for both extends and implements)
# -----------------------------------------------------------------------
implementors: dict[str, list[str]] = {}
for row in conn.execute(
"SELECT source_qualified, target_qualified FROM edges WHERE kind = 'INHERITS'"
).fetchall():
iface = row["target_qualified"]
impl = row["source_qualified"]
if any(impl.startswith(f) for f in java_files) or "::" in impl:
implementors.setdefault(iface, []).append(impl)
# -----------------------------------------------------------------------
# Resolve CALLS edges
# -----------------------------------------------------------------------
calls_rows = conn.execute(
"SELECT id, source_qualified, target_qualified, extra, file_path "
"FROM edges WHERE kind = 'CALLS'"
).fetchall()
resolved = 0
for row in calls_rows:
if row["file_path"] not in java_files:
continue
try:
extra = json.loads(row["extra"] or "{}")
except (json.JSONDecodeError, TypeError):
extra = {}
receiver = extra.get("receiver")
if not receiver:
continue
# Skip edges already spring-resolved in a previous pass
if extra.get("spring_resolved"):
continue
# Strip any prior (possibly wrong) qualification — we have a receiver so
# we can do a better resolution. E.g. "file::ClassName.method" → "method"
raw_target = row["target_qualified"]
if "::" in raw_target:
after = raw_target.split("::", 1)[1]
method_name = after.split(".")[-1] if "." in after else after
else:
method_name = raw_target
source_qual = row["source_qualified"]
# Derive the enclosing class qualified name from source
# source_qual format: "file_path::ClassName.method_name"
enclosing_class_qual: str | None = None
if "::" in source_qual:
after_sep = source_qual.split("::", 1)[1]
if "." in after_sep:
class_part = after_sep.split(".")[0]
prefix = source_qual.split("::")[0]
enclosing_class_qual = f"{prefix}::{class_part}"
else:
enclosing_class_qual = source_qual
if not enclosing_class_qual:
continue
# Look up receiver in field_map for this class
injected_type = field_map.get((enclosing_class_qual, receiver))
if not injected_type:
continue
# Resolve to concrete implementation if unique
impls = implementors.get(injected_type, [])
if len(impls) == 1:
concrete_class = impls[0].split("::")[-1]
fallback = f"{impls[0]}.{method_name}"
new_target = method_to_qual.get((concrete_class, method_name)) or fallback
else:
type_bare = injected_type.rsplit(".", 1)[-1]
fallback = f"{injected_type}.{method_name}"
new_target = method_to_qual.get((type_bare, method_name)) or fallback
extra["spring_resolved"] = True
extra["injected_type"] = injected_type
new_extra = json.dumps(extra)
conn.execute(
"UPDATE edges SET target_qualified = ?, extra = ? WHERE id = ?",
(new_target, new_extra, row["id"]),
)
resolved += 1
logger.debug(
"Spring resolved: %s%s (was %s, receiver=%s)",
source_qual, new_target, method_name, receiver,
)
if resolved:
conn.commit()
logger.info("Spring DI resolver: resolved %d CALLS edges in %d Java files",
resolved, len(java_files))
return {"files_indexed": len(java_files), "calls_resolved": resolved}
+199
View File
@@ -0,0 +1,199 @@
"""Post-build Temporal workflow/activity call resolver.
After tree-sitter parsing, Java CALLS edges whose target is a bare method
name carry ``extra.receiver`` naming the local variable called on. This
module resolves those receivers through the TEMPORAL_STUB map to their
declared Temporal interface type, then optionally to the unique concrete
implementation via INHERITS edges.
Resolution chain:
receiver variable name
→ temporal stub field type (from TEMPORAL_STUB.extra.field_name)
→ concrete implementation (from INHERITS, when unique)
Only Java files are processed. TEMPORAL_STUB edges whose target is not a
node with ``temporal_role`` in extra are silently skipped (they may be
non-Temporal types that happen to end in 'Activity'/'Workflow').
"""
from __future__ import annotations
import json
import logging
from typing import TYPE_CHECKING
if TYPE_CHECKING:
from .graph import GraphStore
logger = logging.getLogger(__name__)
def resolve_temporal_calls(store: GraphStore) -> dict:
"""Resolve Java CALLS edges whose receiver is a Temporal activity/workflow stub.
Safe to call multiple times — already-resolved edges (with
``extra.temporal_resolved``) are skipped.
Returns a dict with resolution counts for telemetry.
"""
conn = store._conn
java_files: set[str] = {
row["file_path"]
for row in conn.execute(
"SELECT DISTINCT file_path FROM nodes WHERE language = 'java'"
).fetchall()
}
if not java_files:
return {"files_indexed": 0, "calls_resolved": 0}
# -----------------------------------------------------------------------
# Collect Temporal interface nodes: bare name → qualified_name
# (nodes whose extra contains temporal_role = workflow_interface|activity_interface)
# -----------------------------------------------------------------------
temporal_interfaces: dict[str, str] = {} # bare_name → qualified_name
for row in conn.execute(
"SELECT name, qualified_name, extra FROM nodes "
"WHERE language = 'java' AND extra IS NOT NULL AND extra LIKE '%temporal_role%'"
).fetchall():
try:
ex = json.loads(row["extra"] or "{}")
except (json.JSONDecodeError, TypeError):
ex = {}
if ex.get("temporal_role") in ("workflow_interface", "activity_interface"):
temporal_interfaces[row["name"]] = row["qualified_name"]
if not temporal_interfaces:
logger.info("Temporal resolver: no Workflow/ActivityInterface nodes, skipping")
return {"files_indexed": len(java_files), "calls_resolved": 0}
# -----------------------------------------------------------------------
# Build field_map: (source_qualified_class, field_name) → interface_type
# from TEMPORAL_STUB edges whose target is a known Temporal interface
# -----------------------------------------------------------------------
field_map: dict[tuple[str, str], str] = {}
for row in conn.execute(
"SELECT source_qualified, target_qualified, extra FROM edges WHERE kind = 'TEMPORAL_STUB'"
).fetchall():
bare_target = row["target_qualified"]
if bare_target not in temporal_interfaces:
continue
try:
extra = json.loads(row["extra"] or "{}")
except (json.JSONDecodeError, TypeError):
extra = {}
fname = extra.get("field_name")
if not fname:
continue
field_map[(row["source_qualified"], fname)] = bare_target
if not field_map:
logger.info("Temporal resolver: no TEMPORAL_STUB edges found, skipping")
return {"files_indexed": len(java_files), "calls_resolved": 0}
# -----------------------------------------------------------------------
# method_to_qual: (class_name, method_name) → full qualified_name
# -----------------------------------------------------------------------
method_to_qual: dict[tuple[str, str], str] = {}
for row in conn.execute(
"SELECT name, qualified_name, parent_name FROM nodes "
"WHERE kind IN ('Function', 'Test') AND language = 'java' AND parent_name IS NOT NULL"
).fetchall():
method_to_qual[(row["parent_name"], row["name"])] = row["qualified_name"]
# -----------------------------------------------------------------------
# implementors: bare interface name → list of implementing class quals
# -----------------------------------------------------------------------
implementors: dict[str, list[str]] = {}
for row in conn.execute(
"SELECT source_qualified, target_qualified FROM edges WHERE kind = 'INHERITS'"
).fetchall():
iface = row["target_qualified"]
impl = row["source_qualified"]
if any(impl.startswith(f) for f in java_files) or "::" in impl:
implementors.setdefault(iface, []).append(impl)
# -----------------------------------------------------------------------
# Resolve CALLS edges
# -----------------------------------------------------------------------
calls_rows = conn.execute(
"SELECT id, source_qualified, target_qualified, extra, file_path "
"FROM edges WHERE kind = 'CALLS'"
).fetchall()
resolved = 0
for row in calls_rows:
if row["file_path"] not in java_files:
continue
try:
extra = json.loads(row["extra"] or "{}")
except (json.JSONDecodeError, TypeError):
extra = {}
receiver = extra.get("receiver")
if not receiver:
continue
if extra.get("temporal_resolved") or extra.get("spring_resolved"):
continue
raw_target = row["target_qualified"]
if "::" in raw_target:
after = raw_target.split("::", 1)[1]
method_name = after.split(".")[-1] if "." in after else after
else:
method_name = raw_target
source_qual = row["source_qualified"]
# Derive enclosing class qualified name
enclosing_class_qual: str | None = None
if "::" in source_qual:
after_sep = source_qual.split("::", 1)[1]
if "." in after_sep:
class_part = after_sep.split(".")[0]
prefix = source_qual.split("::")[0]
enclosing_class_qual = f"{prefix}::{class_part}"
else:
enclosing_class_qual = source_qual
if not enclosing_class_qual:
continue
interface_bare = field_map.get((enclosing_class_qual, receiver))
if not interface_bare:
continue
interface_qual = temporal_interfaces.get(interface_bare, interface_bare)
impls = implementors.get(interface_qual, [])
if len(impls) == 1:
concrete_class = impls[0].split("::")[-1]
fallback = f"{impls[0]}.{method_name}"
new_target = method_to_qual.get((concrete_class, method_name)) or fallback
else:
fallback = f"{interface_qual}.{method_name}"
new_target = method_to_qual.get((interface_bare, method_name)) or fallback
extra["temporal_resolved"] = True
extra["temporal_interface"] = interface_bare
new_extra = json.dumps(extra)
conn.execute(
"UPDATE edges SET target_qualified = ?, extra = ? WHERE id = ?",
(new_target, new_extra, row["id"]),
)
resolved += 1
logger.debug(
"Temporal resolved: %s%s (receiver=%s, interface=%s)",
source_qual, new_target, receiver, interface_bare,
)
if resolved:
conn.commit()
logger.info("Temporal resolver: resolved %d CALLS edges in %d Java files",
resolved, len(java_files))
return {"files_indexed": len(java_files), "calls_resolved": resolved}
+125
View File
@@ -0,0 +1,125 @@
"""Token reduction benchmark -- measures graph query efficiency vs naive file reading."""
from __future__ import annotations
import logging
import sqlite3
from pathlib import Path
from typing import Any
from .graph import GraphStore
from .search import hybrid_search
logger = logging.getLogger(__name__)
# Sample questions for benchmarking
_SAMPLE_QUESTIONS = [
"how does authentication work",
"what is the main entry point",
"how are database connections managed",
"what error handling patterns are used",
"how do tests verify core functionality",
]
def estimate_tokens(text: str) -> int:
"""Rough token estimate: ~4 chars per token."""
return max(1, len(text) // 4)
def compute_naive_tokens(repo_root: Path) -> int:
"""Count tokens in all parseable source files."""
total = 0
exts = (
".py", ".js", ".ts", ".go", ".rs", ".java",
".c", ".cpp", ".rb", ".php", ".swift", ".kt",
)
for ext in exts:
for f in repo_root.rglob(f"*{ext}"):
try:
total += estimate_tokens(
f.read_text(errors="replace")
)
except OSError:
continue
return total
def run_token_benchmark(
store: GraphStore,
repo_root: Path,
questions: list[str] | None = None,
) -> dict[str, Any]:
"""Run token reduction benchmark.
Compares naive full-corpus token cost vs graph query token
cost for a set of sample questions.
The default sample questions are natural language and require semantic
search to match. If no embeddings are present in the graph, ``hybrid_search``
falls back to FTS5/LIKE matching on node names, which produces no hits for
questions like "how does authentication work" — every per-question ratio
becomes 0 and the benchmark silently appears to fail. We log a clear
warning when that is the case so callers know to run ``embed_graph`` first
(or to pass keyword-matching questions).
"""
if questions is None:
questions = _SAMPLE_QUESTIONS
using_default_questions = questions is _SAMPLE_QUESTIONS
try:
cur = store._conn.execute("SELECT count(*) FROM embeddings")
embedding_count = cur.fetchone()[0]
except sqlite3.OperationalError:
embedding_count = 0
if embedding_count == 0 and using_default_questions:
logger.warning(
"No embeddings found in this graph. The default sample questions "
"are natural language and will not match via FTS5/LIKE alone — "
"every reduction ratio is likely to be 0. Run "
"`code-review-graph embed` first, or pass keyword-matching `questions=`."
)
naive_total = compute_naive_tokens(repo_root)
results = []
for q in questions:
search_results = hybrid_search(store, q, limit=5)
# Simulate graph context: search results + neighbors
graph_tokens = 0
for r in search_results:
graph_tokens += estimate_tokens(str(r))
# Add approximate neighbor context
qn = r.get("qualified_name", "")
edges = store.get_edges_by_source(qn)[:5]
for e in edges:
graph_tokens += estimate_tokens(str(e))
if graph_tokens > 0:
ratio = naive_total / graph_tokens
else:
ratio = 0
results.append({
"question": q,
"naive_tokens": naive_total,
"graph_tokens": graph_tokens,
"reduction_ratio": round(ratio, 1),
})
if results:
total = sum(
r["reduction_ratio"] for r in results # type: ignore[misc]
)
avg_ratio = float(total) / len(results) # type: ignore[arg-type]
else:
avg_ratio = 0.0
return {
"naive_corpus_tokens": naive_total,
"per_question": results,
"average_reduction_ratio": round(avg_ratio, 1),
"summary": (
f"Graph queries use ~{avg_ratio:.0f}x fewer tokens "
f"than reading all source files"
),
}
+156
View File
@@ -0,0 +1,156 @@
"""MCP tool definitions for the Code Review Graph server.
Exposes 27 tools:
1. build_or_update_graph - full or incremental build
2. get_impact_radius - blast radius from changed files
3. query_graph - predefined graph queries
4. get_review_context - focused subgraph + review prompt
5. semantic_search_nodes - keyword + vector search across nodes
6. list_graph_stats - aggregate statistics
7. embed_graph - compute vector embeddings for semantic search
8. get_docs_section - token-optimized documentation retrieval
9. find_large_functions - find oversized functions/classes by line count
10. list_flows - list execution flows sorted by criticality
11. get_flow - get details of a single execution flow
12. get_affected_flows - find flows affected by changed files
13. list_communities - list detected code communities
14. get_community - get details of a single community
15. get_architecture_overview - architecture overview from community structure
16. detect_changes - risk-scored change impact analysis for code review
17. refactor_tool - unified refactoring (rename preview, dead code, suggestions)
18. apply_refactor_tool - apply a previously previewed refactoring
19. generate_wiki - generate markdown wiki from community structure
20. get_wiki_page - retrieve a specific wiki page
21. list_repos - list registered repositories
22. cross_repo_search - search across all registered repositories
23. get_hub_nodes - find most connected nodes (architectural hotspots)
24. get_bridge_nodes - find architectural chokepoints (betweenness centrality)
25. get_knowledge_gaps - identify structural weaknesses
26. get_surprising_connections - find unexpected architectural coupling
27. get_suggested_questions - auto-generated review questions from graph analysis
28. traverse_graph - BFS/DFS traversal from best-matching node
"""
from __future__ import annotations
# Re-export names that external code may patch via "code_review_graph.tools.*"
from ..changes import parse_diff_ranges as parse_diff_ranges
from ..changes import parse_git_diff_ranges as parse_git_diff_ranges
from ..changes import parse_svn_diff_ranges as parse_svn_diff_ranges
from ..incremental import (
get_changed_files as get_changed_files,
)
from ..incremental import (
get_staged_and_unstaged as get_staged_and_unstaged,
)
# -- _common ----------------------------------------------------------------
from ._common import (
_BUILTIN_CALL_NAMES,
_get_store,
_validate_repo_root,
)
# -- analysis_tools ---------------------------------------------------------
from .analysis_tools import (
get_bridge_nodes_func,
get_hub_nodes_func,
get_knowledge_gaps_func,
get_suggested_questions_func,
get_surprising_connections_func,
)
# -- build ------------------------------------------------------------------
from .build import build_or_update_graph, run_postprocess
# -- community_tools --------------------------------------------------------
from .community_tools import (
get_architecture_overview_func,
get_community_func,
list_communities_func,
)
# -- context ----------------------------------------------------------------
from .context import get_minimal_context
# -- docs -------------------------------------------------------------------
from .docs import embed_graph, generate_wiki_func, get_docs_section, get_wiki_page_func
# -- flows_tools ------------------------------------------------------------
from .flows_tools import get_flow, list_flows
# -- query ------------------------------------------------------------------
from .query import (
find_large_functions,
get_impact_radius,
list_graph_stats,
query_graph,
semantic_search_nodes,
traverse_graph_func,
)
# -- refactor_tools ---------------------------------------------------------
from .refactor_tools import apply_refactor_func, refactor_func
# -- registry_tools ---------------------------------------------------------
from .registry_tools import cross_repo_search_func, list_repos_func
# -- review -----------------------------------------------------------------
from .review import (
detect_changes_func,
get_affected_flows_func,
get_review_context,
)
__all__ = [
# _common
"_BUILTIN_CALL_NAMES",
"_get_store",
"_validate_repo_root",
# build
"build_or_update_graph",
"run_postprocess",
# context
"get_minimal_context",
# community_tools
"get_architecture_overview_func",
"get_community_func",
"list_communities_func",
# docs
"embed_graph",
"generate_wiki_func",
"get_docs_section",
"get_wiki_page_func",
# flows_tools
"get_flow",
"list_flows",
# query
"find_large_functions",
"get_impact_radius",
"list_graph_stats",
"query_graph",
"semantic_search_nodes",
"traverse_graph_func",
# refactor_tools
"apply_refactor_func",
"refactor_func",
# registry_tools
"cross_repo_search_func",
"list_repos_func",
# review
"detect_changes_func",
"get_affected_flows_func",
"get_review_context",
# analysis_tools
"get_bridge_nodes_func",
"get_hub_nodes_func",
"get_knowledge_gaps_func",
"get_suggested_questions_func",
"get_surprising_connections_func",
# re-exported for backward compat (used in test patches)
"get_changed_files",
"get_staged_and_unstaged",
"parse_git_diff_ranges",
"parse_svn_diff_ranges",
"parse_diff_ranges",
]
+176
View File
@@ -0,0 +1,176 @@
"""Shared utilities for tool sub-modules."""
from __future__ import annotations
from pathlib import Path
from typing import Any
from ..graph import GraphStore
from ..incremental import find_project_root, get_db_path
def _error_response(
message: str, status: str = "error", **extra: Any,
) -> dict[str, Any]:
"""Build a standardised error response dict."""
return {"status": status, "error": message, "summary": message, **extra}
# Common JS/TS builtin method names filtered from callers_of results.
# "Who calls .map()?" returns hundreds of hits and is never useful.
# These are kept in the graph (callees_of still shows them) but excluded
# when doing reverse call tracing to reduce noise.
_BUILTIN_CALL_NAMES: set[str] = {
"map", "filter", "reduce", "reduceRight", "forEach", "find", "findIndex",
"some", "every", "includes", "indexOf", "lastIndexOf",
"push", "pop", "shift", "unshift", "splice", "slice",
"concat", "join", "flat", "flatMap", "sort", "reverse", "fill",
"keys", "values", "entries", "from", "isArray", "of", "at",
"trim", "trimStart", "trimEnd", "split", "replace", "replaceAll",
"match", "matchAll", "search", "substring", "substr",
"toLowerCase", "toUpperCase", "startsWith", "endsWith",
"padStart", "padEnd", "repeat", "charAt", "charCodeAt",
"assign", "freeze", "defineProperty", "getOwnPropertyNames",
"hasOwnProperty", "create", "is", "fromEntries",
"log", "warn", "error", "info", "debug", "trace", "dir", "table",
"time", "timeEnd", "assert", "clear", "count",
"then", "catch", "finally", "resolve", "reject", "all", "allSettled", "race", "any",
"parse", "stringify",
"floor", "ceil", "round", "random", "max", "min", "abs", "pow", "sqrt",
"addEventListener", "removeEventListener", "querySelector", "querySelectorAll",
"getElementById", "createElement", "appendChild", "removeChild",
"setAttribute", "getAttribute", "preventDefault", "stopPropagation",
"setTimeout", "clearTimeout", "setInterval", "clearInterval",
"toString", "valueOf", "toJSON", "toISOString",
"getTime", "getFullYear", "now",
"isNaN", "parseInt", "parseFloat", "toFixed",
"encodeURIComponent", "decodeURIComponent",
"call", "apply", "bind", "next",
"emit", "on", "off", "once",
"pipe", "write", "read", "end", "close", "destroy",
"send", "status", "json", "redirect",
"set", "get", "delete", "has",
"findUnique", "findFirst", "findMany", "createMany",
"update", "updateMany", "deleteMany", "upsert",
"aggregate", "groupBy", "transaction",
"describe", "it", "test", "expect", "beforeEach", "afterEach",
"beforeAll", "afterAll", "mock", "spyOn",
"require", "fetch",
}
def _validate_repo_root(path: "Path | str") -> Path:
"""Validate that a path is a plausible project root.
Ensures the path is an existing directory that contains a ``.git``,
``.svn``, or ``.code-review-graph`` directory, preventing arbitrary
file-system traversal via the ``repo_root`` parameter.
"""
resolved = Path(path).resolve()
if not resolved.is_dir():
raise ValueError(
f"repo_root is not an existing directory: {resolved}"
)
has_vcs = (
(resolved / ".git").exists()
or (resolved / ".svn").exists()
or (resolved / ".code-review-graph").exists()
)
if not has_vcs:
raise ValueError(
f"repo_root does not look like a project root "
f"(no .git, .svn, or .code-review-graph directory found): "
f"{resolved}"
)
return resolved
def _resolve_root(repo_root: str | None = None) -> Path:
"""Resolve and validate the repository root without opening a store."""
return _validate_repo_root(Path(repo_root)) if repo_root else find_project_root()
def _get_store(repo_root: str | None = None) -> tuple[GraphStore, Path]:
"""Resolve repo root and open the graph store.
Callers own the returned store and must close it (try/finally or
context manager) to avoid leaking SQLite file descriptors.
"""
root = _resolve_root(repo_root)
db_path = get_db_path(root)
return GraphStore(db_path), root
def _resolve_graph_file_paths(
store: GraphStore, root: Path, file_paths: list[str],
) -> list[str]:
"""Resolve user-facing file paths to the paths stored in the graph.
Graphs may contain absolute paths, repo-relative paths, or cwd-relative
paths depending on how they were built. Tool inputs are usually relative to
repo root, so exact matching alone can miss existing graph nodes.
"""
resolved: list[str] = []
seen: set[str] = set()
def add(path: str) -> None:
if path not in seen:
resolved.append(path)
seen.add(path)
for file_path in file_paths:
raw = file_path.replace("\\", "/")
candidates = [raw]
path = Path(file_path)
if path.is_absolute():
try:
candidates.append(str(path.resolve().relative_to(root)).replace("\\", "/"))
except ValueError:
pass
else:
candidates.append(str(root / path))
for candidate in candidates:
if store.get_nodes_by_file(candidate):
add(candidate)
suffixes = []
for candidate in candidates:
normalized = candidate.replace("\\", "/")
if normalized not in suffixes:
suffixes.append(normalized)
for suffix in suffixes:
for matched_path in store.get_files_matching(suffix):
add(matched_path)
return resolved
def compact_response(
summary: str,
key_entities: list[str] | None = None,
risk: str = "unknown",
communities: list[str] | None = None,
flows_affected: list[str] | None = None,
next_tool_suggestions: list[str] | None = None,
data: dict[str, Any] | None = None,
detail_level: str = "minimal",
) -> dict[str, Any]:
"""Standard compact response format for token efficiency."""
resp: dict[str, Any] = {
"status": "ok",
"summary": summary,
}
if key_entities:
resp["key_entities"] = key_entities[:10]
if risk != "unknown":
resp["risk"] = risk
if communities:
resp["communities"] = communities[:5]
if flows_affected:
resp["flows_affected"] = flows_affected[:5]
if next_tool_suggestions:
resp["next_tool_suggestions"] = next_tool_suggestions[:3]
if detail_level != "minimal" and data:
resp["data"] = data
return resp
+184
View File
@@ -0,0 +1,184 @@
"""MCP tool wrappers for graph analysis features."""
from __future__ import annotations
from typing import Any
from ..analysis import (
find_bridge_nodes,
find_hub_nodes,
find_knowledge_gaps,
find_surprising_connections,
generate_suggested_questions,
)
from ._common import _get_store
def get_hub_nodes_func(
repo_root: str | None = None,
top_n: int = 10,
) -> dict[str, Any]:
"""Find the most connected nodes in the codebase graph.
Hub nodes have the highest total degree (in + out edges).
These are architectural hotspots -- changes to them have
disproportionate blast radius.
Args:
repo_root: Repository root (auto-detected if omitted).
top_n: Number of top hubs to return (default 10).
"""
store, _root = _get_store(repo_root or None)
try:
hubs = find_hub_nodes(store, top_n=top_n)
return {
"hub_nodes": hubs,
"count": len(hubs),
"next_tool_suggestions": [
"get_impact_radius -- check blast radius of a hub",
"query_graph callers_of -- see what calls a hub",
"get_bridge_nodes -- find architectural chokepoints",
],
}
finally:
store.close()
def get_bridge_nodes_func(
repo_root: str | None = None,
top_n: int = 10,
) -> dict[str, Any]:
"""Find architectural chokepoints via betweenness centrality.
Bridge nodes sit on the shortest paths between many node
pairs. If they break, multiple code regions lose
connectivity.
Args:
repo_root: Repository root (auto-detected if omitted).
top_n: Number of top bridges to return (default 10).
"""
store, _root = _get_store(repo_root or None)
try:
bridges = find_bridge_nodes(store, top_n=top_n)
return {
"bridge_nodes": bridges,
"count": len(bridges),
"next_tool_suggestions": [
"get_hub_nodes -- find most connected nodes",
"get_impact_radius -- check blast radius",
"detect_changes -- see if bridges are affected",
],
}
finally:
store.close()
def get_knowledge_gaps_func(
repo_root: str | None = None,
) -> dict[str, Any]:
"""Identify structural weaknesses in the codebase.
Finds: isolated nodes (disconnected), thin communities
(< 3 members), untested hotspots (high-degree, no tests),
and single-file communities.
Args:
repo_root: Repository root (auto-detected if omitted).
"""
store, _root = _get_store(repo_root or None)
try:
gaps = find_knowledge_gaps(store)
total = sum(len(v) for v in gaps.values())
return {
"gaps": gaps,
"total_gaps": total,
"summary": {
"isolated_nodes": len(gaps["isolated_nodes"]),
"thin_communities": len(
gaps["thin_communities"]
),
"untested_hotspots": len(
gaps["untested_hotspots"]
),
"single_file_communities": len(
gaps["single_file_communities"]
),
},
"next_tool_suggestions": [
"refactor dead_code -- find unused symbols",
"get_hub_nodes -- find high-impact nodes",
"get_suggested_questions -- review prompts",
],
}
finally:
store.close()
def get_surprising_connections_func(
repo_root: str | None = None,
top_n: int = 15,
) -> dict[str, Any]:
"""Find unexpected architectural coupling in the codebase.
Scores edges by surprise factors: cross-community,
cross-language, peripheral-to-hub, cross-test-boundary.
Args:
repo_root: Repository root (auto-detected if omitted).
top_n: Number of top surprises to return (default 15).
"""
store, _root = _get_store(repo_root or None)
try:
surprises = find_surprising_connections(
store, top_n=top_n
)
return {
"surprising_connections": surprises,
"count": len(surprises),
"next_tool_suggestions": [
"get_architecture_overview -- community structure",
"query_graph callers_of -- trace the coupling",
"get_bridge_nodes -- find chokepoints",
],
}
finally:
store.close()
def get_suggested_questions_func(
repo_root: str | None = None,
) -> dict[str, Any]:
"""Auto-generate review questions from graph analysis.
Produces questions about: bridge nodes, untested hubs,
surprising connections, thin communities, and untested
hotspots.
Args:
repo_root: Repository root (auto-detected if omitted).
"""
store, _root = _get_store(repo_root or None)
try:
questions = generate_suggested_questions(store)
by_priority: dict[str, list[dict[str, Any]]] = {
"high": [], "medium": [], "low": [],
}
for q in questions:
prio = q.get("priority", "medium")
if prio in by_priority:
by_priority[prio].append(q)
return {
"questions": questions,
"count": len(questions),
"by_priority": {
k: len(v) for k, v in by_priority.items()
},
"next_tool_suggestions": [
"get_knowledge_gaps -- structural weaknesses",
"detect_changes -- risk-scored review",
"get_architecture_overview -- community map",
],
}
finally:
store.close()
+541
View File
@@ -0,0 +1,541 @@
"""Tool 1: build_or_update_graph + run_postprocess."""
from __future__ import annotations
import logging
import sqlite3
import time
from typing import Any
from ..incremental import full_build, incremental_update
from ._common import _get_store
logger = logging.getLogger(__name__)
def _run_postprocess(
store: Any,
build_result: dict[str, Any],
postprocess: str,
full_rebuild: bool = False,
changed_files: list[str] | None = None,
) -> list[str]:
"""Run post-build steps based on *postprocess* level.
When *full_rebuild* is False and *changed_files* are available,
uses incremental flow/community detection for faster updates.
Returns a list of warning strings (empty on success).
"""
warnings: list[str] = []
build_result["postprocess_level"] = postprocess
if postprocess == "none":
return warnings
# -- Signatures + FTS (fast, always run unless "none") --
try:
rows = store.get_nodes_without_signature()
for row in rows:
node_id, name, kind, params, ret = (
row[0],
row[1],
row[2],
row[3],
row[4],
)
if kind in ("Function", "Test"):
sig = f"def {name}({params or ''})"
if ret:
sig += f" -> {ret}"
elif kind == "Class":
sig = f"class {name}"
else:
sig = name
store.update_node_signature(node_id, sig[:512])
store.commit()
build_result["signatures_updated"] = True
except (sqlite3.OperationalError, TypeError, KeyError) as e:
logger.warning("Signature computation failed: %s", e)
warnings.append(f"Signature computation failed: {type(e).__name__}: {e}")
try:
from code_review_graph.search import rebuild_fts_index
fts_count = rebuild_fts_index(store)
build_result["fts_indexed"] = fts_count
build_result["fts_rebuilt"] = True
except (sqlite3.OperationalError, ImportError) as e:
logger.warning("FTS index rebuild failed: %s", e)
warnings.append(f"FTS index rebuild failed: {type(e).__name__}: {e}")
if postprocess == "minimal":
return warnings
# -- Expensive: flows + communities (only for "full") --
use_incremental = not full_rebuild and bool(changed_files)
try:
if use_incremental:
from code_review_graph.flows import incremental_trace_flows
count = incremental_trace_flows(store, changed_files)
else:
from code_review_graph.flows import store_flows as _store_flows
from code_review_graph.flows import trace_flows as _trace_flows
flows = _trace_flows(store)
count = _store_flows(store, flows)
build_result["flows_detected"] = count
except (sqlite3.OperationalError, ImportError) as e:
logger.warning("Flow detection failed: %s", e)
warnings.append(f"Flow detection failed: {type(e).__name__}: {e}")
try:
if use_incremental:
from code_review_graph.communities import (
incremental_detect_communities,
)
count = incremental_detect_communities(store, changed_files)
else:
from code_review_graph.communities import (
detect_communities as _detect_communities,
)
from code_review_graph.communities import (
store_communities as _store_communities,
)
comms = _detect_communities(store)
count = _store_communities(store, comms)
build_result["communities_detected"] = count
except (sqlite3.OperationalError, ImportError) as e:
logger.warning("Community detection failed: %s", e)
warnings.append(f"Community detection failed: {type(e).__name__}: {e}")
# -- Compute pre-computed summary tables --
try:
_compute_summaries(store)
build_result["summaries_computed"] = True
except (sqlite3.OperationalError, Exception) as e:
logger.warning("Summary computation failed: %s", e)
warnings.append(f"Summary computation failed: {type(e).__name__}: {e}")
store.set_metadata(
"last_postprocessed_at",
time.strftime("%Y-%m-%dT%H:%M:%S"),
)
store.set_metadata("postprocess_level", postprocess)
return warnings
def _compute_summaries(store: Any) -> None:
"""Populate community_summaries, flow_snapshots, and risk_index tables.
Uses batched aggregate queries and in-memory grouping instead of
per-community/per-node loops. On graphs with ~100k edges this
reduces the work from ``O(nodes + communities)`` SQLite round trips
each doing their own B-tree scan to a handful of ``GROUP BY``
queries, turning what used to be an effective hang into a few
seconds.
Each summary block (community_summaries, flow_snapshots, risk_index)
is wrapped in an explicit transaction so the DELETE + INSERT sequence
is atomic. If a table doesn't exist yet the block is silently skipped.
"""
import json as _json
from collections import defaultdict
from os.path import commonprefix
conn = store._conn
# -- community_summaries --
try:
conn.execute("BEGIN IMMEDIATE")
conn.execute("DELETE FROM community_summaries")
# Pre-compute per-qualified_name edge counts once. Previously
# this section ran a per-community triple-JOIN aggregate query
# (nodes LEFT JOIN edges LEFT JOIN edges), which on graphs with
# thousands of communities was the second-biggest hang.
edge_counts: dict[str, int] = defaultdict(int)
for row in conn.execute(
"SELECT source_qualified, COUNT(*) FROM edges GROUP BY source_qualified"
):
edge_counts[row[0]] += row[1]
for row in conn.execute(
"SELECT target_qualified, COUNT(*) FROM edges GROUP BY target_qualified"
):
edge_counts[row[0]] += row[1]
# Group non-File nodes per community for top-symbol selection.
nodes_by_comm: dict[int, list[tuple[str, int]]] = defaultdict(list)
for row in conn.execute(
"SELECT community_id, name, qualified_name FROM nodes "
"WHERE community_id IS NOT NULL AND kind != 'File'"
):
cid, name, qn = row[0], row[1], row[2]
nodes_by_comm[cid].append((name, edge_counts.get(qn, 0)))
# Group distinct file paths per community (preserving first-seen
# order for stable output, same as DISTINCT in the old query).
files_by_comm: dict[int, list[str]] = defaultdict(list)
seen_files: dict[int, set[str]] = defaultdict(set)
for row in conn.execute(
"SELECT community_id, file_path FROM nodes WHERE community_id IS NOT NULL"
):
cid, fp = row[0], row[1]
if fp not in seen_files[cid]:
seen_files[cid].add(fp)
files_by_comm[cid].append(fp)
community_rows = conn.execute(
"SELECT id, name, size, dominant_language FROM communities"
).fetchall()
for r in community_rows:
cid, cname, csize, clang = r[0], r[1], r[2], r[3]
# Top 5 symbols by total edge count (in + out). Python's
# sorted() is stable so ties break by original row order.
members = sorted(
nodes_by_comm.get(cid, []),
key=lambda nc: nc[1],
reverse=True,
)
key_syms = _json.dumps([m[0] for m in members[:5]])
# Auto-generate purpose from common file path prefix.
paths = files_by_comm.get(cid, [])[:20]
purpose = ""
if paths:
prefix = commonprefix(paths)
if "/" in prefix:
purpose = prefix.rsplit("/", 1)[0].split("/")[-1] if "/" in prefix else ""
conn.execute(
"INSERT OR REPLACE INTO community_summaries "
"(community_id, name, purpose, key_symbols, size, dominant_language) "
"VALUES (?, ?, ?, ?, ?, ?)",
(cid, cname, purpose, key_syms, csize, clang or ""),
)
conn.commit()
except sqlite3.OperationalError:
conn.rollback() # Table may not exist yet
# -- flow_snapshots --
try:
conn.execute("BEGIN IMMEDIATE")
conn.execute("DELETE FROM flow_snapshots")
flow_rows = conn.execute(
"SELECT id, name, entry_point_id, criticality, node_count, "
"file_count, path_json FROM flows"
).fetchall()
# Collect every node id referenced by any flow, then fetch
# their qualified_names in one batched query instead of per-flow
# per-node lookups.
needed_ids: set[int] = set()
parsed_paths: list[list[int]] = []
for r in flow_rows:
needed_ids.add(r[2]) # entry_point_id
path_ids = _json.loads(r[6]) if r[6] else []
parsed_paths.append(path_ids)
# Match the old semantics: entry + up to 3 intermediates + last
for nid in path_ids[1:4]:
needed_ids.add(nid)
if path_ids:
needed_ids.add(path_ids[-1])
id_to_name: dict[int, str] = {}
if needed_ids:
# Batch the IN clause in chunks of 450 to stay under SQLite's
# default SQLITE_MAX_VARIABLE_NUMBER (999), same strategy as
# GraphStore.get_edges_among.
id_list = list(needed_ids)
for i in range(0, len(id_list), 450):
batch = id_list[i : i + 450]
placeholders = ",".join("?" for _ in batch)
node_rows = conn.execute(
f"SELECT id, qualified_name FROM nodes WHERE id IN ({placeholders})", # nosec B608
batch,
).fetchall()
for nr in node_rows:
id_to_name[nr[0]] = nr[1]
for r, path_ids in zip(flow_rows, parsed_paths):
fid, fname, ep_id = r[0], r[1], r[2]
crit, ncount, fcount = r[3], r[4], r[5]
ep_name = id_to_name.get(ep_id, str(ep_id))
critical_path: list[str] = []
if path_ids:
critical_path.append(ep_name)
if len(path_ids) > 2:
for nid in path_ids[1:4]:
nm = id_to_name.get(nid)
if nm:
critical_path.append(nm)
if len(path_ids) > 1:
last = id_to_name.get(path_ids[-1])
if last and last not in critical_path:
critical_path.append(last)
conn.execute(
"INSERT OR REPLACE INTO flow_snapshots "
"(flow_id, name, entry_point, critical_path, criticality, "
"node_count, file_count) VALUES (?, ?, ?, ?, ?, ?, ?)",
(fid, fname, ep_name, _json.dumps(critical_path), crit, ncount, fcount),
)
conn.commit()
except sqlite3.OperationalError:
conn.rollback()
# -- risk_index --
try:
conn.execute("BEGIN IMMEDIATE")
conn.execute("DELETE FROM risk_index")
# Pre-compute caller and test-coverage counts in two aggregate
# queries. Previously this section ran two COUNT(*) queries per
# candidate node; on a ~100k-edge graph with tens of thousands
# of Function/Class/Test nodes that was the primary hang
# observed during Godot builds.
caller_counts: dict[str, int] = {}
for row in conn.execute(
"SELECT target_qualified, COUNT(*) FROM edges "
"WHERE kind = 'CALLS' GROUP BY target_qualified"
):
caller_counts[row[0]] = row[1]
tested_counts: dict[str, int] = {}
for row in conn.execute(
"SELECT source_qualified, COUNT(*) FROM edges "
"WHERE kind = 'TESTED_BY' GROUP BY source_qualified"
):
tested_counts[row[0]] = row[1]
risk_nodes = conn.execute(
"SELECT id, qualified_name, name FROM nodes WHERE kind IN ('Function', 'Class', 'Test')"
).fetchall()
security_kw = {
"auth",
"login",
"password",
"token",
"session",
"crypt",
"secret",
"credential",
"permission",
"sql",
"execute",
}
for n in risk_nodes:
nid, qn, name = n[0], n[1], n[2]
caller_count = caller_counts.get(qn, 0)
tested = tested_counts.get(qn, 0)
coverage = "tested" if tested > 0 else "untested"
name_lower = name.lower()
sec_relevant = 1 if any(kw in name_lower for kw in security_kw) else 0
risk = 0.0
if caller_count > 10:
risk += 0.3
elif caller_count > 3:
risk += 0.15
if coverage == "untested":
risk += 0.3
if sec_relevant:
risk += 0.4
risk = min(risk, 1.0)
conn.execute(
"INSERT OR REPLACE INTO risk_index "
"(node_id, qualified_name, risk_score, caller_count, "
"test_coverage, security_relevant, last_computed) "
"VALUES (?, ?, ?, ?, ?, ?, datetime('now'))",
(nid, qn, risk, caller_count, coverage, sec_relevant),
)
conn.commit()
except sqlite3.OperationalError:
conn.rollback()
def build_or_update_graph(
full_rebuild: bool = False,
repo_root: str | None = None,
base: str = "HEAD~1",
postprocess: str = "full",
recurse_submodules: bool | None = None,
) -> dict[str, Any]:
"""Build or incrementally update the code knowledge graph.
Args:
full_rebuild: If True, re-parse every file. If False (default),
only re-parse files changed since ``base``.
repo_root: Path to the repository root. Auto-detected if omitted.
base: Git ref for incremental diff (default: HEAD~1).
postprocess: Post-processing level after build:
``"full"`` (default) — signatures, FTS, flows, communities.
``"minimal"`` — signatures + FTS only (fast, keeps search working).
``"none"`` — skip all post-processing (raw parse only).
recurse_submodules: If True, include files from git submodules
via ``git ls-files --recurse-submodules``. When None
(default), falls back to the CRG_RECURSE_SUBMODULES
environment variable. Default: disabled.
Returns:
Summary with files_parsed/updated, node/edge counts, and errors.
"""
store, root = _get_store(repo_root)
try:
if full_rebuild:
result = full_build(root, store, recurse_submodules)
build_result = {
"status": "ok",
"build_type": "full",
"summary": (
f"Full build complete: parsed {result['files_parsed']} files, "
f"created {result['total_nodes']} nodes and "
f"{result['total_edges']} edges."
),
**result,
}
else:
result = incremental_update(root, store, base=base)
if result["files_updated"] == 0:
return {
"status": "ok",
"build_type": "incremental",
"summary": "No changes detected. Graph is up to date.",
"postprocess_level": postprocess,
**result,
}
build_result = {
"status": "ok",
"build_type": "incremental",
"summary": (
f"Incremental update: {result['files_updated']} files re-parsed, "
f"{result['total_nodes']} nodes and "
f"{result['total_edges']} edges updated. "
f"Changed: {result['changed_files']}. "
f"Dependents also updated: {result['dependent_files']}."
),
**result,
}
# Pass changed_files for incremental flow/community detection
changed = result.get("changed_files") if not full_rebuild else None
warnings = _run_postprocess(
store,
build_result,
postprocess,
full_rebuild=full_rebuild,
changed_files=changed,
)
if warnings:
build_result["warnings"] = warnings
return build_result
finally:
store.close()
def run_postprocess(
flows: bool = True,
communities: bool = True,
fts: bool = True,
repo_root: str | None = None,
) -> dict[str, Any]:
"""Run post-processing steps on an existing graph.
Useful for running expensive steps (flows, communities) separately
from the build, or for re-running after the graph has been updated
with ``postprocess="none"``.
Args:
flows: Run flow detection. Default: True.
communities: Run community detection. Default: True.
fts: Rebuild FTS index. Default: True.
repo_root: Repository root path. Auto-detected if omitted.
Returns:
Summary of what was computed.
"""
store, _root = _get_store(repo_root)
result: dict[str, Any] = {"status": "ok"}
warnings: list[str] = []
try:
try:
rows = store.get_nodes_without_signature()
for row in rows:
node_id, name, kind, params, ret = (
row[0],
row[1],
row[2],
row[3],
row[4],
)
if kind in ("Function", "Test"):
sig = f"def {name}({params or ''})"
if ret:
sig += f" -> {ret}"
elif kind == "Class":
sig = f"class {name}"
else:
sig = name
store.update_node_signature(node_id, sig[:512])
store.commit()
result["signatures_updated"] = True
except (sqlite3.OperationalError, TypeError, KeyError) as e:
logger.warning("Signature computation failed: %s", e)
warnings.append(f"Signature computation failed: {type(e).__name__}: {e}")
if fts:
try:
from code_review_graph.search import rebuild_fts_index
fts_count = rebuild_fts_index(store)
result["fts_indexed"] = fts_count
except (sqlite3.OperationalError, ImportError) as e:
store.rollback()
logger.warning("FTS index rebuild failed: %s", e)
warnings.append(f"FTS index rebuild failed: {type(e).__name__}: {e}")
if flows:
try:
from code_review_graph.flows import store_flows as _store_flows
from code_review_graph.flows import trace_flows as _trace_flows
traced = _trace_flows(store)
count = _store_flows(store, traced)
result["flows_detected"] = count
except (sqlite3.OperationalError, ImportError) as e:
store.rollback()
logger.warning("Flow detection failed: %s", e)
warnings.append(f"Flow detection failed: {type(e).__name__}: {e}")
if communities:
try:
from code_review_graph.communities import (
detect_communities as _detect_communities,
)
from code_review_graph.communities import (
store_communities as _store_communities,
)
comms = _detect_communities(store)
count = _store_communities(store, comms)
result["communities_detected"] = count
except (sqlite3.OperationalError, ImportError) as e:
store.rollback()
logger.warning("Community detection failed: %s", e)
warnings.append(f"Community detection failed: {type(e).__name__}: {e}")
store.set_metadata(
"last_postprocessed_at",
time.strftime("%Y-%m-%dT%H:%M:%S"),
)
result["summary"] = "Post-processing complete."
if warnings:
result["warnings"] = warnings
return result
finally:
store.close()
+246
View File
@@ -0,0 +1,246 @@
"""Tools 13, 14, 15: community listing, detail, architecture overview."""
from __future__ import annotations
from collections import Counter
from typing import Any
from ..communities import get_architecture_overview, get_communities
from ..context_savings import attach_context_savings
from ..graph import node_to_dict
from ..hints import generate_hints, get_session
from ._common import _get_store
# ---------------------------------------------------------------------------
# Tool 13: list_communities [EXPLORE]
# ---------------------------------------------------------------------------
def list_communities_func(
repo_root: str | None = None,
sort_by: str = "size",
min_size: int = 0,
detail_level: str = "standard",
) -> dict[str, Any]:
"""List detected code communities in the codebase.
[EXPLORE] Retrieves stored communities from the knowledge graph.
Each community represents a cluster of related code entities
(functions, classes) detected via the Leiden algorithm or
file-based grouping.
Args:
repo_root: Repository root path. Auto-detected if omitted.
sort_by: Sort column: size, cohesion, or name.
min_size: Minimum community size to include (default: 0).
detail_level: "standard" (default) returns full community data;
"minimal" returns only name, size, and cohesion
per community.
Returns:
List of communities with size and cohesion scores.
"""
store, root = _get_store(repo_root)
try:
communities = get_communities(
store, sort_by=sort_by, min_size=min_size
)
if detail_level == "minimal":
communities = [
{"name": c["name"], "size": c["size"], "cohesion": c["cohesion"]}
for c in communities
]
result: dict[str, object] = {
"status": "ok",
"summary": f"Found {len(communities)} communities",
"communities": communities,
}
result["_hints"] = generate_hints(
"list_communities", result, get_session()
)
return result
except Exception as exc:
return {"status": "error", "error": str(exc)}
finally:
store.close()
# ---------------------------------------------------------------------------
# Tool 14: get_community [EXPLORE]
# ---------------------------------------------------------------------------
def get_community_func(
community_name: str | None = None,
community_id: int | None = None,
include_members: bool = False,
repo_root: str | None = None,
) -> dict[str, Any]:
"""Get details of a single code community.
[EXPLORE] Retrieves a community by its database ID or by name match.
Optionally includes the full list of member nodes.
Args:
community_name: Name to search for (partial match). Ignored if
community_id given.
community_id: Database ID of the community.
include_members: If True, include full member node details.
repo_root: Repository root path. Auto-detected if omitted.
Returns:
Community details, or not_found status.
"""
store, root = _get_store(repo_root)
try:
community: dict | None = None
all_communities = get_communities(store)
if community_id is not None:
for c in all_communities:
if c.get("id") == community_id:
community = c
break
elif community_name is not None:
for c in all_communities:
if community_name.lower() in c["name"].lower():
community = c
break
if community is None:
return {
"status": "not_found",
"summary": (
"No community found matching the given criteria."
),
}
if include_members:
cid = community.get("id")
if cid is not None:
member_nodes = store.get_nodes_by_community_id(cid)
members = [node_to_dict(n) for n in member_nodes]
community["member_details"] = members
result = {
"status": "ok",
"summary": (
f"Community '{community['name']}': "
f"{community['size']} nodes, "
f"cohesion {community['cohesion']:.4f}"
),
"community": community,
}
result["_hints"] = generate_hints(
"get_community", result, get_session()
)
return result
except Exception as exc:
return {"status": "error", "error": str(exc)}
finally:
store.close()
# ---------------------------------------------------------------------------
# Tool 15: get_architecture_overview [EXPLORE]
# ---------------------------------------------------------------------------
_MINIMAL_COMMUNITY_FIELDS = ("id", "name", "size", "cohesion", "dominant_language")
def _minimal_overview(overview: dict[str, Any]) -> dict[str, Any]:
"""Compress overview for ``detail_level="minimal"``.
The full overview can exceed 600KB on medium repos because it embeds
every community's member list and every individual cross-community
edge. Minimal mode drops member lists and aggregates the edge list
to one row per community pair with a count and the top edge kinds —
enough to spot coupling smells without exploding token budgets.
"""
communities = [
{k: c[k] for k in _MINIMAL_COMMUNITY_FIELDS if k in c}
for c in overview.get("communities", [])
]
id_to_name = {c["id"]: c["name"] for c in communities if "id" in c}
edge_pair_counts: Counter[tuple[int, int]] = Counter()
edge_pair_kinds: dict[tuple[int, int], Counter[str]] = {}
for e in overview.get("cross_community_edges", []):
# Use canonical (low, high) ordering so A↔B and B↔A aggregate together.
a, b = e["source_community"], e["target_community"]
pair = (a, b) if a <= b else (b, a)
edge_pair_counts[pair] += 1
edge_pair_kinds.setdefault(pair, Counter())[e["edge_kind"]] += 1
cross_pairs = [
{
"source_community": id_to_name.get(a, f"community-{a}"),
"target_community": id_to_name.get(b, f"community-{b}"),
"edge_count": count,
"top_kinds": [k for k, _ in edge_pair_kinds[(a, b)].most_common(3)],
}
for (a, b), count in edge_pair_counts.most_common()
]
return {
"communities": communities,
"cross_community_edges": cross_pairs,
"warnings": overview.get("warnings", []),
}
def get_architecture_overview_func(
repo_root: str | None = None,
detail_level: str = "minimal",
) -> dict[str, Any]:
"""Generate an architecture overview based on community structure.
[EXPLORE] Builds a high-level view of the codebase architecture by
analyzing community boundaries and cross-community coupling.
Includes warnings for high coupling between communities.
Args:
repo_root: Repository root path. Auto-detected if omitted.
detail_level: "minimal" (default) drops community member lists
and aggregates edges to one row per community pair
(typical reduction: 600KB -> <5KB);
"standard" returns the full overview including
per-edge cross-community detail.
Returns:
Architecture overview with communities, cross-community edges,
and warnings.
"""
store, root = _get_store(repo_root)
try:
full_overview = get_architecture_overview(store)
overview = full_overview
if detail_level == "minimal":
overview = _minimal_overview(full_overview)
n_communities = len(overview["communities"])
n_cross = len(overview["cross_community_edges"])
n_warnings = len(overview["warnings"])
cross_label = (
"community pairs"
if detail_level == "minimal"
else "cross-community edges"
)
result = {
"status": "ok",
"summary": (
f"Architecture: {n_communities} communities, "
f"{n_cross} {cross_label}, "
f"{n_warnings} warning(s)"
),
**overview,
}
result["_hints"] = generate_hints(
"get_architecture_overview", result, get_session()
)
if detail_level == "minimal":
attach_context_savings(result, original_context=full_overview)
return result
except Exception as exc:
return {"status": "error", "error": str(exc)}
finally:
store.close()
+152
View File
@@ -0,0 +1,152 @@
"""Tool: get_minimal_context — ultra-compact context for token-efficient workflows."""
from __future__ import annotations
import logging
import sqlite3
import subprocess
from pathlib import Path
from typing import Any
from ._common import _get_store, compact_response
logger = logging.getLogger(__name__)
def _has_git_changes(root: Path, base: str) -> bool:
"""Quick check for uncommitted or diffed changes."""
try:
result = subprocess.run(
["git", "diff", "--name-only", base, "--"],
capture_output=True, stdin=subprocess.DEVNULL, text=True,
cwd=str(root), timeout=10,
)
if result.returncode == 0 and result.stdout.strip():
return True
# Also check staged/unstaged
result2 = subprocess.run(
["git", "status", "--porcelain"],
capture_output=True, stdin=subprocess.DEVNULL, text=True,
cwd=str(root), timeout=10,
)
return bool(result2.stdout.strip())
except (FileNotFoundError, subprocess.TimeoutExpired):
return False
def get_minimal_context(
task: str = "",
changed_files: list[str] | None = None,
repo_root: str | None = None,
base: str = "HEAD~1",
) -> dict[str, Any]:
"""Return minimum context an agent needs to start any task (~100 tokens).
Combines graph stats, top communities, top flows, risk score,
and suggested next tools into an ultra-compact response.
Args:
task: Natural language description of what the agent is doing
(e.g. "review PR #42", "debug login timeout").
changed_files: Explicit changed files. Auto-detected from git if None.
repo_root: Repository root path. Auto-detected if None.
base: Git ref for diff comparison.
"""
store, root = _get_store(repo_root)
try:
# 1. Quick stats
stats = store.get_stats()
# 2. Risk from changed files
risk = "unknown"
risk_score = 0.0
top_affected: list[str] = []
test_gap_count = 0
if changed_files or _has_git_changes(root, base):
try:
from ..changes import analyze_changes
from ..incremental import get_changed_files as _get_changed
files = changed_files
if not files:
files = _get_changed(root, base)
if files:
abs_files = [str(root / f) for f in files]
analysis = analyze_changes(
store, abs_files, repo_root=str(root), base=base,
)
risk_score = analysis.get("risk_score", 0.0)
risk = (
"high" if risk_score > 0.7
else "medium" if risk_score > 0.4
else "low"
)
top_affected = [
f.get("name", "")
for f in analysis.get("changed_functions", [])[:5]
]
test_gap_count = len(analysis.get("test_gaps", []))
except (
ImportError, OSError, ValueError,
sqlite3.Error, subprocess.SubprocessError,
):
logger.debug("Risk analysis failed in get_minimal_context", exc_info=True)
# 3. Top 3 communities
communities: list[str] = []
try:
rows = store._conn.execute(
"SELECT name FROM communities ORDER BY size DESC LIMIT 3"
).fetchall()
communities = [r[0] for r in rows]
except sqlite3.OperationalError: # nosec B110 — table may not exist yet
logger.debug("communities table not yet populated")
# 4. Top 3 critical flows
flows: list[str] = []
try:
rows = store._conn.execute(
"SELECT name FROM flows ORDER BY criticality DESC LIMIT 3"
).fetchall()
flows = [r[0] for r in rows]
except sqlite3.OperationalError: # nosec B110 — table may not exist yet
logger.debug("flows table not yet populated")
# 5. Suggest next tools based on task keywords
task_lower = task.lower()
if any(w in task_lower for w in ("review", "pr", "merge", "diff")):
suggestions = ["detect_changes", "get_affected_flows", "get_review_context"]
elif any(w in task_lower for w in ("debug", "bug", "error", "fix")):
suggestions = ["semantic_search_nodes", "query_graph", "get_flow"]
elif any(w in task_lower for w in ("refactor", "rename", "dead", "clean")):
suggestions = ["refactor", "find_large_functions", "get_architecture_overview"]
elif any(w in task_lower for w in ("onboard", "understand", "explore", "arch")):
suggestions = [
"get_architecture_overview", "list_communities", "list_flows",
]
else:
suggestions = [
"detect_changes", "semantic_search_nodes",
"get_architecture_overview",
]
# Build summary
summary_parts = [
f"{stats.total_nodes} nodes, {stats.total_edges} edges"
f" across {stats.files_count} files.",
]
if risk != "unknown":
summary_parts.append(f"Risk: {risk} ({risk_score:.2f}).")
if test_gap_count:
summary_parts.append(f"{test_gap_count} test gaps.")
return compact_response(
summary=" ".join(summary_parts),
key_entities=top_affected or None,
risk=risk,
communities=communities or None,
flows_affected=flows or None,
next_tool_suggestions=suggestions,
)
finally:
store.close()
+274
View File
@@ -0,0 +1,274 @@
"""Tools 7, 8, 19, 20: embed_graph, get_docs_section, wiki tools."""
from __future__ import annotations
import logging
from pathlib import Path
from typing import Any
from ..embeddings import EmbeddingStore, embed_all_nodes
from ..incremental import find_project_root, get_db_path
from ._common import _get_store, _resolve_root, _validate_repo_root
logger = logging.getLogger(__name__)
# ---------------------------------------------------------------------------
# Tool 7: embed_graph
# ---------------------------------------------------------------------------
def embed_graph(
repo_root: str | None = None,
model: str | None = None,
provider: str | None = None,
) -> dict[str, Any]:
"""Compute vector embeddings for all graph nodes to enable semantic search.
Requires: ``pip install code-review-graph[embeddings]`` (local provider only;
cloud providers like ``openai`` / ``google`` / ``minimax`` use stdlib ``urllib``).
Default model: all-MiniLM-L6-v2. Override via ``model`` param or
CRG_EMBEDDING_MODEL env var.
Changing the model or provider re-embeds all nodes automatically.
Only embeds nodes that don't already have up-to-date embeddings.
Args:
repo_root: Repository root path. Auto-detected if omitted.
model: Embedding model name. For local: HuggingFace ID or path;
for openai: model ID (e.g. ``text-embedding-3-small``);
for google: Gemini model ID. Falls back to
CRG_EMBEDDING_MODEL / CRG_OPENAI_MODEL env vars as appropriate.
provider: Provider name: ``local`` (default), ``openai``, ``google``,
or ``minimax``. ``openai`` requires CRG_OPENAI_BASE_URL +
CRG_OPENAI_API_KEY + CRG_OPENAI_MODEL env vars and accepts
any OpenAI-compatible endpoint (real OpenAI, Azure, new-api,
LiteLLM, vLLM, LocalAI, Ollama openai-mode, etc.).
Returns:
Number of nodes embedded and total embedding count.
"""
store, root = _get_store(repo_root)
try:
db_path = get_db_path(root)
try:
emb_store = EmbeddingStore(db_path, provider=provider, model=model)
except ValueError as exc:
# Unknown provider name or missing provider env vars — surface
# as a structured error rather than a traceback.
logger.error("embed_graph: %s", exc)
return {"status": "error", "error": str(exc)}
try:
if not emb_store.available:
if provider in ("openai", "google", "minimax"):
err = (
f"The '{provider}' embedding provider is not available. "
"Check the required environment variables "
"(see README and `get_provider()` docstring) and that "
"the endpoint is reachable."
)
else:
err = (
"The local embedding provider needs sentence-transformers. "
"Install with: pip install code-review-graph[embeddings] — "
"or switch provider to 'openai' / 'google' / 'minimax'."
)
return {"status": "error", "error": err}
newly_embedded = embed_all_nodes(store, emb_store)
total = emb_store.count()
return {
"status": "ok",
"summary": (
f"Embedded {newly_embedded} new node(s). "
f"Total embeddings: {total}. "
"Semantic search is now active."
),
"newly_embedded": newly_embedded,
"total_embeddings": total,
}
finally:
emb_store.close()
finally:
store.close()
# ---------------------------------------------------------------------------
# Tool 8: get_docs_section
# ---------------------------------------------------------------------------
def get_docs_section(
section_name: str, repo_root: str | None = None
) -> dict[str, Any]:
"""Return a specific section from the LLM-optimized reference.
Used by skills and Claude Code to load only the exact documentation
section needed, keeping token usage minimal (90%+ savings).
Args:
section_name: Exact section name. One of: usage, review-delta,
review-pr, commands, legal, watch, embeddings,
languages, troubleshooting.
repo_root: Repository root path. Auto-detected from current
directory if omitted.
Returns:
The section content, or an error if not found.
"""
import re as _re
search_roots: list[Path] = []
# Wheel install: docs are packaged inside code_review_graph/docs.
in_pkg_docs = (
Path(__file__).parent.parent
/ "docs"
/ "LLM-OPTIMIZED-REFERENCE.md"
)
if repo_root:
try:
search_roots.append(_validate_repo_root(Path(repo_root)))
except ValueError:
pass
elif in_pkg_docs.exists():
in_pkg_root = in_pkg_docs.parent.parent
search_roots.append(in_pkg_root)
if not repo_root:
project_root = find_project_root()
if project_root not in search_roots:
search_roots.append(project_root)
# Editable/source-tree fallback: docs live next to code_review_graph/.
pkg_docs = (
Path(__file__).parent.parent.parent
/ "docs"
/ "LLM-OPTIMIZED-REFERENCE.md"
)
if pkg_docs.exists():
pkg_root = pkg_docs.parent.parent
if pkg_root not in search_roots:
search_roots.append(pkg_root)
for search_root in search_roots:
candidate = search_root / "docs" / "LLM-OPTIMIZED-REFERENCE.md"
if candidate.exists():
content = candidate.read_text(encoding="utf-8", errors="replace")
match = _re.search(
rf'<section name="{_re.escape(section_name)}">'
r"(.*?)</section>",
content,
_re.DOTALL | _re.IGNORECASE,
)
if match:
return {
"status": "ok",
"section": section_name,
"content": match.group(1).strip(),
}
available = [
"usage", "review-delta", "review-pr", "commands",
"legal", "watch", "embeddings", "languages", "troubleshooting",
]
return {
"status": "not_found",
"error": (
f"Section '{section_name}' not found. "
f"Available: {', '.join(available)}"
),
}
# ---------------------------------------------------------------------------
# Tool 19: generate_wiki [DOCS]
# ---------------------------------------------------------------------------
def generate_wiki_func(
repo_root: str | None = None,
force: bool = False,
) -> dict[str, Any]:
"""Generate a markdown wiki from the community structure.
[DOCS] Creates a wiki page for each detected community and an index
page. Pages are written to ``.code-review-graph/wiki/`` inside the
repository. Only regenerates pages whose content has changed unless
force=True.
Args:
repo_root: Repository root path. Auto-detected if omitted.
force: If True, regenerate all pages even if content is unchanged.
Returns:
Status with pages_generated, pages_updated, pages_unchanged counts.
"""
from ..incremental import get_data_dir
from ..wiki import generate_wiki
store, root = _get_store(repo_root)
try:
wiki_dir = get_data_dir(root) / "wiki"
result = generate_wiki(store, wiki_dir, force=force)
total = (
result["pages_generated"]
+ result["pages_updated"]
+ result["pages_unchanged"]
)
return {
"status": "ok",
"summary": (
f"Wiki generated: {result['pages_generated']} new, "
f"{result['pages_updated']} updated, "
f"{result['pages_unchanged']} unchanged "
f"({total} total pages)"
),
"wiki_dir": str(wiki_dir),
**result,
}
except Exception as exc:
return {"status": "error", "error": str(exc)}
finally:
store.close()
# ---------------------------------------------------------------------------
# Tool 20: get_wiki_page [DOCS]
# ---------------------------------------------------------------------------
def get_wiki_page_func(
community_name: str,
repo_root: str | None = None,
) -> dict[str, Any]:
"""Retrieve a specific wiki page by community name.
[DOCS] Returns the markdown content of the wiki page for the given
community. The wiki must have been generated first via generate_wiki.
Args:
community_name: Community name to look up (slugified for filename).
repo_root: Repository root path. Auto-detected if omitted.
Returns:
Page content or not_found status.
"""
from ..incremental import get_data_dir
from ..wiki import get_wiki_page
root = _resolve_root(repo_root)
wiki_dir = get_data_dir(root) / "wiki"
content = get_wiki_page(wiki_dir, community_name)
if content is None:
return {
"status": "not_found",
"summary": f"No wiki page found for '{community_name}'.",
}
return {
"status": "ok",
"summary": (
f"Wiki page for '{community_name}' ({len(content)} chars)"
),
"content": content,
}
+176
View File
@@ -0,0 +1,176 @@
"""Tools 10, 11: list_flows, get_flow."""
from __future__ import annotations
from pathlib import Path
from typing import Any
from ..flows import get_flow_by_id, get_flows
from ..hints import generate_hints, get_session
from ._common import _get_store
# ---------------------------------------------------------------------------
# Tool 10: list_flows [EXPLORE]
# ---------------------------------------------------------------------------
def list_flows(
repo_root: str | None = None,
sort_by: str = "criticality",
limit: int = 50,
kind: str | None = None,
detail_level: str = "standard",
) -> dict[str, Any]:
"""List execution flows in the codebase, sorted by criticality.
[EXPLORE] Retrieves stored execution flows from the knowledge graph.
Each flow represents a call chain starting from an entry point
(e.g. HTTP handler, CLI command, test function).
Args:
repo_root: Repository root path. Auto-detected if omitted.
sort_by: Sort column: criticality, depth, node_count, file_count,
or name.
limit: Maximum flows to return (default: 50).
kind: Optional filter by entry point kind (e.g. "Test", "Function").
detail_level: "standard" (default) returns full flow data;
"minimal" returns only name, criticality, and
node_count per flow.
Returns:
List of flows with criticality scores.
"""
store, root = _get_store(repo_root)
try:
fetch_limit = (
limit if not kind else limit * 10
) # fetch more when filtering
flows = get_flows(store, sort_by=sort_by, limit=fetch_limit)
if kind:
filtered = []
for f in flows:
ep_id = f.get("entry_point_id")
if ep_id is not None:
node_kind = store.get_node_kind_by_id(ep_id)
if node_kind == kind:
filtered.append(f)
flows = filtered[:limit]
if detail_level == "minimal":
flows = [
{
"name": f["name"],
"criticality": f["criticality"],
"node_count": f["node_count"],
}
for f in flows
]
result: dict[str, object] = {
"status": "ok",
"summary": f"Found {len(flows)} execution flow(s)",
"flows": flows,
}
result["_hints"] = generate_hints(
"list_flows", result, get_session()
)
return result
except Exception as exc:
return {"status": "error", "error": str(exc)}
finally:
store.close()
# ---------------------------------------------------------------------------
# Tool 11: get_flow [EXPLORE]
# ---------------------------------------------------------------------------
def get_flow(
flow_id: int | None = None,
flow_name: str | None = None,
include_source: bool = False,
repo_root: str | None = None,
) -> dict[str, Any]:
"""Get details of a single execution flow.
[EXPLORE] Retrieves full path details for a flow, including each step's
function name, file, and line numbers. Optionally includes source
snippets for every step in the path.
Args:
flow_id: Database ID of the flow (from list_flows).
flow_name: Name to search for (partial match). Ignored if flow_id
given.
include_source: If True, include source code snippets for each step.
repo_root: Repository root path. Auto-detected if omitted.
Returns:
Flow details with steps, or not_found status.
"""
store, root = _get_store(repo_root)
try:
flow: dict | None = None
if flow_id is not None:
flow = get_flow_by_id(store, flow_id)
elif flow_name is not None:
# Search flows by name match
all_flows = get_flows(
store, sort_by="criticality", limit=500
)
for f in all_flows:
if flow_name.lower() in f["name"].lower():
flow = get_flow_by_id(store, f["id"])
break
if flow is None:
return {
"status": "not_found",
"summary": "No flow found matching the given criteria.",
}
# Optionally include source snippets for each step
if include_source and "steps" in flow:
for step in flow["steps"]:
fp = Path(step["file"]) if step.get("file") else None
if fp is not None and not fp.is_absolute():
fp = root / fp
file_path = fp
if file_path and file_path.is_file():
try:
lines = file_path.read_text(
errors="replace"
).splitlines()
start = max(
0, (step.get("line_start") or 1) - 1
)
end = min(
len(lines),
step.get("line_end") or len(lines),
)
step["source"] = "\n".join(
f"{i + 1}: {lines[i]}"
for i in range(start, end)
)
except (OSError, UnicodeDecodeError):
step["source"] = "(could not read file)"
result = {
"status": "ok",
"summary": (
f"Flow '{flow['name']}': {flow['node_count']} nodes, "
f"depth {flow['depth']}, "
f"criticality {flow['criticality']:.4f}"
),
"flow": flow,
}
result["_hints"] = generate_hints(
"get_flow", result, get_session()
)
return result
except Exception as exc:
return {"status": "error", "error": str(exc)}
finally:
store.close()
+692
View File
@@ -0,0 +1,692 @@
"""Tools 2, 3, 5, 6, 9: query / search / stats helpers."""
from __future__ import annotations
import logging
from pathlib import Path
from typing import Any
from ..context_savings import attach_context_savings, estimate_file_tokens
from ..embeddings import EmbeddingStore
from ..graph import _sanitize_name, edge_to_dict, node_to_dict
from ..hints import generate_hints, get_session
from ..incremental import get_changed_files, get_db_path, get_staged_and_unstaged
from ..search import hybrid_search
from ._common import _BUILTIN_CALL_NAMES, _get_store, _resolve_graph_file_paths
logger = logging.getLogger(__name__)
# ---------------------------------------------------------------------------
# Tool 2: get_impact_radius
# ---------------------------------------------------------------------------
_QUERY_PATTERNS = {
"callers_of": "Find all functions that call a given function",
"callees_of": "Find all functions called by a given function",
"imports_of": "Find all imports of a given file or module",
"importers_of": "Find all files that import a given file or module",
"children_of": "Find all nodes contained in a file or class",
"tests_for": "Find all tests for a given function or class",
"inheritors_of": "Find all classes that inherit from a given class",
"file_summary": "Get a summary of all nodes in a file",
}
def get_impact_radius(
changed_files: list[str] | None = None,
max_depth: int = 2,
max_results: int = 500,
repo_root: str | None = None,
base: str = "HEAD~1",
detail_level: str = "standard",
) -> dict[str, Any]:
"""Analyze the blast radius of changed files.
Args:
changed_files: Explicit list of changed file paths (relative to repo root).
If omitted, auto-detects from git diff.
max_depth: How many hops to traverse in the graph (default: 2).
max_results: Maximum impacted nodes to return (default: 500).
repo_root: Repository root path. Auto-detected if omitted.
base: Git ref for auto-detecting changes (default: HEAD~1).
detail_level: "standard" (full output) or "minimal" (summary only).
Returns:
Changed nodes, impacted nodes, impacted files, connecting edges,
plus ``truncated`` flag and ``total_impacted`` count.
"""
store, root = _get_store(repo_root)
try:
if changed_files is None:
changed_files = get_changed_files(root, base)
if not changed_files:
changed_files = get_staged_and_unstaged(root)
if not changed_files:
return {
"status": "ok",
"summary": "No changed files detected.",
"changed_nodes": [],
"impacted_nodes": [],
"impacted_files": [],
"truncated": False,
"total_impacted": 0,
}
# Resolve user-facing paths to the file paths stored in the graph.
original_tokens = estimate_file_tokens(root, changed_files)
abs_files = _resolve_graph_file_paths(store, root, changed_files)
result = store.get_impact_radius(
abs_files, max_depth=max_depth, max_nodes=max_results
)
changed_dicts = [node_to_dict(n) for n in result["changed_nodes"]]
impacted_dicts = [node_to_dict(n) for n in result["impacted_nodes"]]
edge_dicts = [edge_to_dict(e) for e in result["edges"]]
truncated = result["truncated"]
total_impacted = result["total_impacted"]
summary_parts = [
f"Blast radius for {len(changed_files)} changed file(s):",
f" - {len(changed_dicts)} nodes directly changed",
f" - {len(impacted_dicts)} nodes impacted (within {max_depth} hops)",
f" - {len(result['impacted_files'])} additional files affected",
]
if truncated:
summary_parts.append(
f" - Results truncated: showing {len(impacted_dicts)}"
f" of {total_impacted} impacted nodes"
)
if detail_level == "minimal":
impacted_count = len(impacted_dicts)
if impacted_count > 20:
risk = "high"
elif impacted_count > 5:
risk = "medium"
else:
risk = "low"
key_entities = [
n["name"] for n in impacted_dicts[:5]
]
minimal_response = {
"status": "ok",
"summary": "\n".join(summary_parts),
"risk": risk,
"impacted_file_count": len(result["impacted_files"]),
"key_entities": key_entities,
"truncated": truncated,
}
attach_context_savings(minimal_response, original_tokens=original_tokens)
return minimal_response
response = {
"status": "ok",
"summary": "\n".join(summary_parts),
"changed_files": changed_files,
"changed_nodes": changed_dicts,
"impacted_nodes": impacted_dicts,
"impacted_files": result["impacted_files"],
"edges": edge_dicts,
"truncated": truncated,
"total_impacted": total_impacted,
}
attach_context_savings(response, original_tokens=original_tokens)
return response
finally:
store.close()
# ---------------------------------------------------------------------------
# Tool 3: query_graph
# ---------------------------------------------------------------------------
def query_graph(
pattern: str,
target: str,
repo_root: str | None = None,
detail_level: str = "standard",
) -> dict[str, Any]:
"""Run a predefined graph query.
Args:
pattern: Query pattern. One of: callers_of, callees_of, imports_of,
importers_of, children_of, tests_for, inheritors_of, file_summary.
target: The node name, qualified name, or file path to query about.
repo_root: Repository root path. Auto-detected if omitted.
detail_level: "standard" (full output) or "minimal" (summary only).
Returns:
Matching nodes and edges for the query.
"""
store, root = _get_store(repo_root)
try:
if pattern not in _QUERY_PATTERNS:
return {
"status": "error",
"error": (
f"Unknown pattern '{pattern}'. "
f"Available: {list(_QUERY_PATTERNS.keys())}"
),
}
results: list[dict] = []
edges_out: list[dict] = []
# For callers_of, skip common builtins early (bare names only)
# "Who calls .map()?" returns hundreds of useless hits.
# Qualified names (e.g. "utils.py::map") bypass this filter.
if (
pattern == "callers_of"
and target in _BUILTIN_CALL_NAMES
and "::" not in target
):
return {
"status": "ok", "pattern": pattern, "target": target,
"description": _QUERY_PATTERNS[pattern],
"summary": (
f"'{target}' is a common builtin "
"— callers_of skipped to avoid noise."
),
"results": [], "edges": [],
}
# Resolve target - try as-is, then as absolute path, then search.
# file_summary targets are paths, so skip broad node search.
node = None
if pattern != "file_summary":
node = store.get_node(target)
if not node:
abs_target = str(root / target)
node = store.get_node(abs_target)
if not node:
# Search by name
candidates = store.search_nodes(target, limit=5)
if len(candidates) == 1:
node = candidates[0]
target = node.qualified_name
elif len(candidates) > 1:
return {
"status": "ambiguous",
"summary": (
f"Multiple matches for '{target}'. "
"Please use a qualified name."
),
"candidates": [node_to_dict(c) for c in candidates],
}
if not node and pattern != "file_summary":
return {
"status": "not_found",
"summary": f"No node found matching '{target}'.",
}
qn = node.qualified_name if node else target
if pattern == "callers_of":
seen_sources: set[str] = set()
for e in store.get_edges_by_target(qn):
if e.kind == "CALLS":
if e.source_qualified not in seen_sources:
seen_sources.add(e.source_qualified)
caller = store.get_node(e.source_qualified)
if caller:
results.append(node_to_dict(caller))
edges_out.append(edge_to_dict(e))
# Fallback: CALLS edges store unqualified target names
# (e.g. "generateTestCode") while qn is fully qualified
# (e.g. "file.ts::generateTestCode"). Search by plain name too.
if node:
for e in store.search_edges_by_target_name(node.name):
if e.source_qualified not in seen_sources:
seen_sources.add(e.source_qualified)
caller = store.get_node(e.source_qualified)
if caller:
results.append(node_to_dict(caller))
edges_out.append(edge_to_dict(e))
elif pattern == "callees_of":
seen_targets: set[str] = set()
for e in store.get_edges_by_source(qn):
if e.kind == "CALLS":
if e.target_qualified not in seen_targets:
seen_targets.add(e.target_qualified)
callee = store.get_node(e.target_qualified)
if callee:
results.append(node_to_dict(callee))
elif "::" not in e.target_qualified:
results.append({
"kind": "Function",
"name": e.target_qualified,
"qualified_name": e.target_qualified,
})
edges_out.append(edge_to_dict(e))
elif pattern == "imports_of":
for e in store.get_edges_by_source(qn):
if e.kind == "IMPORTS_FROM":
results.append({"import_target": e.target_qualified})
edges_out.append(edge_to_dict(e))
elif pattern == "importers_of":
# Find edges where target matches this file.
# Use resolve() to canonicalize the path, matching how
# _resolve_module_to_file stores edge targets.
abs_target = (
str((root / target).resolve()) if node is None
else node.file_path
)
for e in store.get_edges_by_target(abs_target):
if e.kind == "IMPORTS_FROM":
results.append({
"importer": e.source_qualified,
"file": e.file_path,
})
edges_out.append(edge_to_dict(e))
elif pattern == "children_of":
for e in store.get_edges_by_source(qn):
if e.kind == "CONTAINS":
child = store.get_node(e.target_qualified)
if child:
results.append(node_to_dict(child))
elif pattern == "tests_for":
for e in store.get_edges_by_target(qn):
if e.kind == "TESTED_BY":
test = store.get_node(e.source_qualified)
if test:
results.append(node_to_dict(test))
# Also search by naming convention
name = node.name if node else target
test_nodes = store.search_nodes(f"test_{name}", limit=10)
test_nodes += store.search_nodes(f"Test{name}", limit=10)
seen = {r.get("qualified_name") for r in results}
for t in test_nodes:
if t.qualified_name not in seen and t.is_test:
results.append(node_to_dict(t))
elif pattern == "inheritors_of":
for e in store.get_edges_by_target(qn):
if e.kind in ("INHERITS", "IMPLEMENTS"):
child = store.get_node(e.source_qualified)
if child:
results.append(node_to_dict(child))
edges_out.append(edge_to_dict(e))
# Fallback: INHERITS/IMPLEMENTS edges store unqualified base names
# (e.g. "Animal") while qn is fully qualified
# (e.g. "sample.dart::Animal"). Search by plain name too. See: #87
if not results and node:
for kind in ("INHERITS", "IMPLEMENTS"):
for e in store.search_edges_by_target_name(node.name, kind=kind):
child = store.get_node(e.source_qualified)
if child:
results.append(node_to_dict(child))
edges_out.append(edge_to_dict(e))
elif pattern == "file_summary":
graph_paths = _resolve_graph_file_paths(store, root, [target])
for graph_path in graph_paths:
for n in store.get_nodes_by_file(graph_path):
results.append(node_to_dict(n))
summary = (
f"Found {len(results)} result(s) "
f"for {pattern}('{target}')"
)
if detail_level == "minimal":
minimal_results = [
{
k: r[k]
for k in ("name", "kind", "file_path")
if k in r
}
for r in results[:5]
]
return {
"status": "ok",
"pattern": pattern,
"target": target,
"description": _QUERY_PATTERNS[pattern],
"summary": summary,
"result_count": len(results),
"results": minimal_results,
}
return {
"status": "ok",
"pattern": pattern,
"target": target,
"description": _QUERY_PATTERNS[pattern],
"summary": summary,
"results": results,
"edges": edges_out,
}
finally:
store.close()
# ---------------------------------------------------------------------------
# Tool 5: semantic_search_nodes
# ---------------------------------------------------------------------------
def semantic_search_nodes(
query: str,
kind: str | None = None,
limit: int = 20,
repo_root: str | None = None,
context_files: list[str] | None = None,
model: str | None = None,
provider: str | None = None,
detail_level: str = "standard",
) -> dict[str, Any]:
"""Search for nodes by name, keyword, or semantic similarity.
Uses hybrid search (FTS5 BM25 + vector embeddings merged via Reciprocal
Rank Fusion) as the primary search path, with graceful fallback to
keyword matching.
Args:
query: Search string to match against node names and qualified names.
kind: Optional filter by node kind (File, Class, Function, Type, Test).
limit: Maximum results to return (default: 20).
repo_root: Repository root path. Auto-detected if omitted.
context_files: Optional list of file paths. Nodes in these files
receive a relevance boost.
detail_level: "standard" (full output) or "minimal" (summary only).
Returns:
Ranked list of matching nodes.
"""
store, root = _get_store(repo_root)
try:
results = hybrid_search(
store, query, kind=kind, limit=limit, context_files=context_files,
model=model, provider=provider,
)
search_mode = "hybrid"
if not results:
search_mode = "keyword"
summary = f"Found {len(results)} node(s) matching '{query}'" + (
f" (kind={kind})" if kind else ""
)
if detail_level == "minimal":
minimal_results = [
{
k: r[k]
for k in ("name", "kind", "file_path", "score")
if k in r
}
for r in results[:5]
]
return {
"status": "ok",
"query": query,
"search_mode": search_mode,
"summary": summary,
"results": minimal_results,
}
result: dict[str, object] = {
"status": "ok",
"query": query,
"search_mode": search_mode,
"summary": summary,
"results": results,
}
result["_hints"] = generate_hints(
"semantic_search_nodes", result, get_session()
)
return result
finally:
store.close()
# ---------------------------------------------------------------------------
# Tool 6: list_graph_stats
# ---------------------------------------------------------------------------
def list_graph_stats(repo_root: str | None = None) -> dict[str, Any]:
"""Get aggregate statistics about the knowledge graph.
Args:
repo_root: Repository root path. Auto-detected if omitted.
Returns:
Total nodes, edges, breakdown by kind, languages, and last update time.
"""
store, root = _get_store(repo_root)
try:
stats = store.get_stats()
summary_parts = [
f"Graph statistics for {root.name}:",
f" Files: {stats.files_count}",
f" Total nodes: {stats.total_nodes}",
f" Total edges: {stats.total_edges}",
f" Languages: {', '.join(stats.languages) if stats.languages else 'none'}",
f" Last updated: {stats.last_updated or 'never'}",
"",
"Nodes by kind:",
]
for kind, count in sorted(stats.nodes_by_kind.items()):
summary_parts.append(f" {kind}: {count}")
summary_parts.append("")
summary_parts.append("Edges by kind:")
for kind, count in sorted(stats.edges_by_kind.items()):
summary_parts.append(f" {kind}: {count}")
# Add embedding info if available
emb_store = EmbeddingStore(get_db_path(root))
try:
emb_count = emb_store.count()
summary_parts.append("")
summary_parts.append(f"Embeddings: {emb_count} nodes embedded")
if not emb_store.available:
summary_parts.append(
" (install sentence-transformers for semantic search)"
)
finally:
emb_store.close()
return {
"status": "ok",
"summary": "\n".join(summary_parts),
"total_nodes": stats.total_nodes,
"total_edges": stats.total_edges,
"nodes_by_kind": stats.nodes_by_kind,
"edges_by_kind": stats.edges_by_kind,
"languages": stats.languages,
"files_count": stats.files_count,
"last_updated": stats.last_updated,
"embeddings_count": emb_count,
}
finally:
store.close()
# ---------------------------------------------------------------------------
# Tool 9: find_large_functions
# ---------------------------------------------------------------------------
def find_large_functions(
min_lines: int = 50,
kind: str | None = None,
file_path_pattern: str | None = None,
limit: int = 50,
repo_root: str | None = None,
) -> dict[str, Any]:
"""Find functions, classes, or files exceeding a line-count threshold.
Useful for identifying decomposition targets, code-quality audits,
and enforcing size limits during code review.
Args:
min_lines: Minimum line count to flag (default: 50).
kind: Filter by node kind: Function, Class, File, or Test.
file_path_pattern: Filter by file path substring (e.g. "components/").
limit: Maximum results (default: 50).
repo_root: Repository root path. Auto-detected if omitted.
Returns:
Oversized nodes with line counts, ordered largest first.
"""
store, root = _get_store(repo_root)
try:
nodes = store.get_nodes_by_size(
min_lines=min_lines,
kind=kind,
file_path_pattern=file_path_pattern,
limit=limit,
)
results = []
for n in nodes:
d = node_to_dict(n)
d["line_count"] = (
(n.line_end - n.line_start + 1)
if n.line_start and n.line_end
else 0
)
# Make file_path relative for readability
try:
d["relative_path"] = str(Path(n.file_path).relative_to(root))
except ValueError:
d["relative_path"] = n.file_path
results.append(d)
summary_parts = [
f"Found {len(results)} node(s) with >= {min_lines} lines"
+ (f" (kind={kind})" if kind else "")
+ (f" matching '{file_path_pattern}'" if file_path_pattern else "")
+ ":",
]
for r in results[:10]:
summary_parts.append(
f" {r['line_count']:>4} lines | {r['kind']:>8} | "
f"{r['name']} ({r['relative_path']}:{r['line_start']})"
)
if len(results) > 10:
summary_parts.append(f" ... and {len(results) - 10} more")
return {
"status": "ok",
"summary": "\n".join(summary_parts),
"total_found": len(results),
"min_lines": min_lines,
"results": results,
}
finally:
store.close()
# -------------------------------------------------------------------
# traverse_graph: free-form BFS / DFS traversal
# -------------------------------------------------------------------
def traverse_graph_func(
query: str,
mode: str = "bfs",
depth: int = 3,
token_budget: int = 2000,
repo_root: str | None = None,
) -> dict[str, Any]:
"""BFS/DFS traversal from best-matching node.
Args:
query: Search string to find the starting node.
mode: "bfs" (breadth-first) or "dfs" (depth-first).
depth: Max traversal depth (1-6). Default: 3.
token_budget: Approximate token limit for results.
repo_root: Repository root path.
"""
store, root = _get_store(repo_root)
try:
results = hybrid_search(store, query, limit=1)
if not results:
return {
"error": f"No node matching '{query}'",
"nodes": [],
}
start_qn = results[0]["qualified_name"]
depth = max(1, min(depth, 6))
# BFS / DFS traversal
visited: dict[str, int] = {} # qn -> depth
queue: list[tuple[str, int]] = [
(start_qn, 0),
]
traversal: list[dict] = []
approx_tokens = 0
while queue:
if mode == "bfs":
current_qn, cur_depth = queue.pop(0)
else:
current_qn, cur_depth = queue.pop()
if current_qn in visited:
continue
if cur_depth > depth:
continue
visited[current_qn] = cur_depth
node = store.get_node(current_qn)
if not node:
continue
entry = {
"name": _sanitize_name(node.name),
"qualified_name": node.qualified_name,
"kind": node.kind,
"file": node.file_path,
"depth": cur_depth,
}
approx_tokens += len(str(entry)) // 4
if approx_tokens > token_budget:
break
traversal.append(entry)
# Get neighbours
out_edges = store.get_edges_by_source(
current_qn
)
in_edges = store.get_edges_by_target(
current_qn
)
for e in out_edges:
tgt = e.target_qualified
if tgt not in visited:
queue.append((tgt, cur_depth + 1))
for e in in_edges:
src = e.source_qualified
if src not in visited:
queue.append((src, cur_depth + 1))
return {
"start_node": start_qn,
"mode": mode,
"max_depth": depth,
"nodes_visited": len(traversal),
"traversal": traversal,
"truncated": approx_tokens > token_budget,
"next_tool_suggestions": [
"query_graph callers_of"
" -- focused relationship query",
"get_impact_radius"
" -- blast radius analysis",
],
}
finally:
store.close()
+168
View File
@@ -0,0 +1,168 @@
"""Tools 17, 18: refactor_func, apply_refactor_func."""
from __future__ import annotations
from pathlib import Path
from typing import Any
from ..hints import generate_hints, get_session
from ..incremental import find_project_root
from ..refactor import (
apply_refactor,
find_dead_code,
rename_preview,
suggest_refactorings,
)
from ._common import _get_store, _validate_repo_root
# ---------------------------------------------------------------------------
# Tool 17: refactor_tool [REFACTOR]
# ---------------------------------------------------------------------------
def refactor_func(
mode: str = "rename",
old_name: str | None = None,
new_name: str | None = None,
kind: str | None = None,
file_pattern: str | None = None,
repo_root: str | None = None,
) -> dict[str, Any]:
"""Unified refactoring entry point.
[REFACTOR] Supports three modes:
- ``rename``: Preview renaming a symbol (requires *old_name* and
*new_name*).
- ``dead_code``: Find unreferenced functions/classes.
- ``suggest``: Get community-driven refactoring suggestions.
Args:
mode: One of ``"rename"``, ``"dead_code"``, or ``"suggest"``.
old_name: (rename mode) Current symbol name.
new_name: (rename mode) Desired new name.
kind: (dead_code mode) Optional node kind filter.
file_pattern: (dead_code mode) Optional file path substring filter.
repo_root: Repository root path. Auto-detected if omitted.
Returns:
Mode-specific results dict.
"""
valid_modes = {"rename", "dead_code", "suggest"}
if mode not in valid_modes:
return {
"status": "error",
"error": (
f"Invalid mode '{mode}'. "
f"Must be one of: {', '.join(sorted(valid_modes))}"
),
}
store, root = _get_store(repo_root)
try:
if mode == "rename":
if not old_name or not new_name:
return {
"status": "error",
"error": (
"rename mode requires both old_name and new_name."
),
}
preview = rename_preview(store, old_name, new_name)
if preview is None:
return {
"status": "not_found",
"summary": f"No node found matching '{old_name}'.",
}
result = {
"status": "ok",
"summary": (
f"Rename preview: {old_name} -> {new_name}, "
f"{len(preview['edits'])} edit(s). "
f"Use apply_refactor_tool(refactor_id="
f"'{preview['refactor_id']}') to apply."
),
**preview,
}
result["_hints"] = generate_hints(
"refactor", result, get_session()
)
return result
elif mode == "dead_code":
dead = find_dead_code(
store, kind=kind, file_pattern=file_pattern, root=root
)
result = {
"status": "ok",
"summary": f"Found {len(dead)} dead code symbol(s).",
"dead_code": dead,
"total": len(dead),
}
result["_hints"] = generate_hints(
"refactor", result, get_session()
)
return result
else: # suggest
suggestions = suggest_refactorings(store)
result = {
"status": "ok",
"summary": (
f"Generated {len(suggestions)} "
"refactoring suggestion(s)."
),
"suggestions": suggestions,
"total": len(suggestions),
}
result["_hints"] = generate_hints(
"refactor", result, get_session()
)
return result
except Exception as exc:
return {"status": "error", "error": str(exc)}
finally:
store.close()
# ---------------------------------------------------------------------------
# Tool 18: apply_refactor_tool [REFACTOR]
# ---------------------------------------------------------------------------
def apply_refactor_func(
refactor_id: str,
repo_root: str | None = None,
dry_run: bool = False,
) -> dict[str, Any]:
"""Apply a previously previewed refactoring to source files.
[REFACTOR] Validates the refactor_id, checks expiry, ensures all edit
paths are within the repo root, then performs exact string replacements.
Args:
refactor_id: ID returned by a prior ``refactor_tool(mode="rename")``
call.
repo_root: Repository root path. Auto-detected if omitted.
dry_run: If True, return a unified diff of what would change
without touching disk. The refactor_id remains valid so the
user can review the diff, then call again with ``dry_run=False``
to actually write the changes. See: #176
Returns:
Status with count of applied edits and modified files. When
``dry_run=True`` the response additionally contains ``would_modify``
(list of file paths) and ``diffs`` (map of file -> unified-diff
string).
"""
try:
root = (
_validate_repo_root(Path(repo_root))
if repo_root
else find_project_root()
)
except (RuntimeError, ValueError) as exc:
return {"status": "error", "error": str(exc)}
result = apply_refactor(refactor_id, root, dry_run=dry_run)
return result
+125
View File
@@ -0,0 +1,125 @@
"""Tools 21, 22: list_repos_func, cross_repo_search_func."""
from __future__ import annotations
import logging
from pathlib import Path
from typing import Any
from ..graph import GraphStore
from ..incremental import get_db_path
from ..search import hybrid_search
logger = logging.getLogger(__name__)
# ---------------------------------------------------------------------------
# Tool 21: list_repos [REGISTRY]
# ---------------------------------------------------------------------------
def list_repos_func() -> dict[str, Any]:
"""List all registered repositories.
[REGISTRY] Returns the list of repositories registered in the global
multi-repo registry at ``~/.code-review-graph/registry.json``.
Returns:
List of registered repos with paths and aliases.
"""
from ..registry import Registry
try:
registry = Registry()
repos = registry.list_repos()
return {
"status": "ok",
"summary": f"{len(repos)} registered repository(ies)",
"repos": repos,
}
except Exception as exc:
return {"status": "error", "error": str(exc)}
# ---------------------------------------------------------------------------
# Tool 22: cross_repo_search [REGISTRY]
# ---------------------------------------------------------------------------
def cross_repo_search_func(
query: str,
kind: str | None = None,
limit: int = 20,
) -> dict[str, Any]:
"""Search across all registered repositories.
[REGISTRY] Runs hybrid_search on each registered repo's graph database
and merges the results.
Args:
query: Search query string.
kind: Optional node kind filter (e.g. "Function", "Class").
limit: Maximum results per repo (default: 20).
Returns:
Combined search results from all registered repos.
"""
from ..registry import Registry
try:
registry = Registry()
repos = registry.list_repos()
if not repos:
return {
"status": "ok",
"summary": (
"No repositories registered. "
"Use 'register' to add repos."
),
"results": [],
}
all_results: list[dict[str, Any]] = []
searched_repos: list[str] = []
for repo_entry in repos:
repo_path = Path(repo_entry["path"])
db_path = get_db_path(repo_path)
if not db_path.exists():
continue
try:
store = GraphStore(str(db_path))
try:
results = hybrid_search(
store, query, kind=kind, limit=limit
)
alias = repo_entry.get("alias", repo_path.name)
for r in results:
r["repo"] = alias
r["repo_path"] = str(repo_path)
all_results.extend(results)
searched_repos.append(alias)
finally:
store.close()
except Exception as exc:
logger.warning(
"Search failed for %s: %s", repo_path, exc
)
# Sort all results by score descending
all_results.sort(
key=lambda r: r.get("score", 0), reverse=True
)
return {
"status": "ok",
"summary": (
f"Found {len(all_results)} result(s) across "
f"{len(searched_repos)} repo(s) for '{query}'"
),
"results": all_results[:limit],
"repos_searched": searched_repos,
}
except Exception as exc:
return {"status": "error", "error": str(exc)}
+477
View File
@@ -0,0 +1,477 @@
"""Tools 4, 12, 16: review context, affected flows, detect changes."""
from __future__ import annotations
import logging
from pathlib import Path
from typing import Any
from ..changes import analyze_changes, parse_diff_ranges, parse_git_diff_ranges # noqa: F401
from ..context_savings import attach_context_savings, estimate_file_tokens
from ..flows import get_affected_flows as _get_affected_flows
from ..graph import edge_to_dict, node_to_dict
from ..hints import generate_hints, get_session
from ..incremental import get_changed_files, get_staged_and_unstaged
from ._common import _get_store, _resolve_graph_file_paths
logger = logging.getLogger(__name__)
# ---------------------------------------------------------------------------
# Tool 4: get_review_context
# ---------------------------------------------------------------------------
def get_review_context(
changed_files: list[str] | None = None,
max_depth: int = 2,
include_source: bool = True,
max_lines_per_file: int = 200,
repo_root: str | None = None,
base: str = "HEAD~1",
detail_level: str = "standard",
) -> dict[str, Any]:
"""Generate a focused review context from changed files.
Builds a token-optimized subgraph + source snippets for code review.
Args:
changed_files: Files to review (auto-detected from git diff if omitted).
max_depth: Impact radius depth (default: 2).
include_source: Whether to include source code snippets (default: True).
max_lines_per_file: Max source lines per file in output (default: 200).
repo_root: Repository root path. Auto-detected if omitted.
base: Git ref for change detection (default: HEAD~1).
detail_level: Output detail level. "standard" returns full context;
"minimal" returns summary, risk level, changed/impacted file counts,
top 5 key entity names, test gap count, and next tool suggestions.
Default: "standard".
Returns:
Structured review context with subgraph, source snippets, and
review guidance.
"""
store, root = _get_store(repo_root)
try:
# Get impact radius first
if changed_files is None:
changed_files = get_changed_files(root, base)
if not changed_files:
changed_files = get_staged_and_unstaged(root)
if not changed_files:
return {
"status": "ok",
"summary": "No changes detected. Nothing to review.",
"context": {},
}
graph_files = _resolve_graph_file_paths(store, root, changed_files)
original_tokens = estimate_file_tokens(root, changed_files)
impact = store.get_impact_radius(graph_files, max_depth=max_depth)
if detail_level == "minimal":
impacted_count = len(impact["impacted_nodes"])
if impacted_count > 20:
risk = "high"
elif impacted_count > 5:
risk = "medium"
else:
risk = "low"
key_entities = [
n.name for n in impact["changed_nodes"][:5]
]
# Count test gaps among changed functions.
changed_funcs = [
n for n in impact["changed_nodes"]
if n.kind == "Function" and not n.is_test
]
test_edges = [
e for e in impact["edges"] if e.kind == "TESTED_BY"
]
tested_qualified = {e.source_qualified for e in test_edges}
test_gap_count = sum(
1 for f in changed_funcs
if f.qualified_name not in tested_qualified
)
summary_parts = [
f"Review context for {len(changed_files)} changed file(s):",
f" - Risk: {risk}",
f" - {len(impact['impacted_nodes'])} impacted nodes"
f" in {len(impact['impacted_files'])} files",
]
result = {
"status": "ok",
"summary": "\n".join(summary_parts),
"risk": risk,
"changed_file_count": len(changed_files),
"impacted_file_count": len(impact["impacted_files"]),
"key_entities": key_entities,
"test_gaps": test_gap_count,
"next_tool_suggestions": [
"detect_changes",
"get_affected_flows",
"get_impact_radius",
],
}
attach_context_savings(result, original_tokens=original_tokens)
return result
# Build review context
context: dict[str, Any] = {
"changed_files": changed_files,
"impacted_files": impact["impacted_files"],
"graph": {
"changed_nodes": [
node_to_dict(n) for n in impact["changed_nodes"]
],
"impacted_nodes": [
node_to_dict(n) for n in impact["impacted_nodes"]
],
"edges": [edge_to_dict(e) for e in impact["edges"]],
},
}
# Add source snippets for changed files
if include_source:
snippets = {}
for rel_path in changed_files:
full_path = root / rel_path
if full_path.is_file():
try:
lines = full_path.read_text(
errors="replace"
).splitlines()
if len(lines) > max_lines_per_file:
# Include only the relevant functions/classes
relevant_lines = _extract_relevant_lines(
lines,
impact["changed_nodes"],
str(full_path),
)
snippets[rel_path] = relevant_lines
else:
snippets[rel_path] = "\n".join(
f"{i+1}: {line}"
for i, line in enumerate(lines)
)
except (OSError, UnicodeDecodeError):
snippets[rel_path] = "(could not read file)"
context["source_snippets"] = snippets
# Generate review guidance
guidance = _generate_review_guidance(impact, changed_files)
context["review_guidance"] = guidance
summary_parts = [
f"Review context for {len(changed_files)} changed file(s):",
f" - {len(impact['changed_nodes'])} directly changed nodes",
f" - {len(impact['impacted_nodes'])} impacted nodes"
f" in {len(impact['impacted_files'])} files",
"",
"Review guidance:",
guidance,
]
result = {
"status": "ok",
"summary": "\n".join(summary_parts),
"context": context,
}
attach_context_savings(result, original_tokens=original_tokens)
return result
finally:
store.close()
def _extract_relevant_lines(
lines: list[str], nodes: list, file_path: str
) -> str:
"""Extract only the lines relevant to changed nodes."""
ranges = []
for n in nodes:
if n.file_path == file_path:
start = max(0, n.line_start - 3) # 2 lines context before
end = min(len(lines), n.line_end + 2) # 1 line context after
ranges.append((start, end))
if not ranges:
# Show first N lines as fallback
return "\n".join(
f"{i+1}: {line}" for i, line in enumerate(lines[:50])
)
# Merge overlapping ranges
ranges.sort()
merged = [ranges[0]]
for start, end in ranges[1:]:
if start <= merged[-1][1] + 1:
merged[-1] = (merged[-1][0], max(merged[-1][1], end))
else:
merged.append((start, end))
parts: list[str] = []
for start, end in merged:
if parts:
parts.append("...")
for i in range(start, end):
parts.append(f"{i+1}: {lines[i]}")
return "\n".join(parts)
def _generate_review_guidance(
impact: dict, changed_files: list[str]
) -> str:
"""Generate review guidance based on the impact analysis."""
guidance_parts = []
# Check for test coverage
changed_funcs = [
n for n in impact["changed_nodes"] if n.kind == "Function"
]
test_edges = [e for e in impact["edges"] if e.kind == "TESTED_BY"]
tested_funcs = {e.source_qualified for e in test_edges}
untested = [
f for f in changed_funcs
if f.qualified_name not in tested_funcs and not f.is_test
]
if untested:
guidance_parts.append(
f"- {len(untested)} changed function(s) lack test coverage: "
+ ", ".join(n.name for n in untested[:5])
)
# Check for wide blast radius
if len(impact["impacted_nodes"]) > 20:
guidance_parts.append(
f"- Wide blast radius: {len(impact['impacted_nodes'])} "
"nodes impacted. "
"Review callers and dependents carefully."
)
# Check for inheritance changes
inheritance_edges = [
e for e in impact["edges"]
if e.kind in ("INHERITS", "IMPLEMENTS")
]
if inheritance_edges:
guidance_parts.append(
f"- {len(inheritance_edges)} inheritance/implementation "
"relationship(s) affected. "
"Check for Liskov substitution violations."
)
# Check for cross-file impact
impacted_file_count = len(impact["impacted_files"])
if impacted_file_count > 3:
guidance_parts.append(
f"- Changes impact {impacted_file_count} other files."
" Consider splitting into smaller PRs."
)
if not guidance_parts:
guidance_parts.append(
"- Changes appear well-contained with minimal blast radius."
)
return "\n".join(guidance_parts)
# ---------------------------------------------------------------------------
# Tool 12: get_affected_flows [REVIEW]
# ---------------------------------------------------------------------------
def get_affected_flows_func(
changed_files: list[str] | None = None,
base: str = "HEAD~1",
repo_root: str | None = None,
) -> dict[str, Any]:
"""Find execution flows affected by changed files.
[REVIEW] Identifies which execution flows pass through nodes in the
changed files. Useful during code review to understand which user-facing
or critical paths are affected by a change.
Args:
changed_files: List of changed file paths (relative to repo root).
Auto-detected from git diff if omitted.
base: Git ref for auto-detecting changes (default: HEAD~1).
repo_root: Repository root path. Auto-detected if omitted.
Returns:
Affected flows sorted by criticality, with step details.
"""
store, root = _get_store(repo_root)
try:
if changed_files is None:
changed_files = get_changed_files(root, base)
if not changed_files:
changed_files = get_staged_and_unstaged(root)
if not changed_files:
return {
"status": "ok",
"summary": "No changed files detected.",
"affected_flows": [],
"total": 0,
}
# Convert to absolute paths for graph lookup
abs_files = [str(root / f) for f in changed_files]
result = _get_affected_flows(store, abs_files)
total = result["total"]
out = {
"status": "ok",
"summary": (
f"{total} flow(s) affected by changes "
f"in {len(changed_files)} file(s)"
),
"changed_files": changed_files,
"affected_flows": result["affected_flows"],
"total": total,
}
out["_hints"] = generate_hints(
"get_affected_flows", out, get_session()
)
return out
except Exception as exc:
return {"status": "error", "error": str(exc)}
finally:
store.close()
# ---------------------------------------------------------------------------
# Tool 16: detect_changes [REVIEW]
# ---------------------------------------------------------------------------
def detect_changes_func(
base: str = "HEAD~1",
changed_files: list[str] | None = None,
include_source: bool = False,
max_depth: int = 2,
repo_root: str | None = None,
detail_level: str = "standard",
) -> dict[str, Any]:
"""Detect changes and produce risk-scored review guidance.
[REVIEW] Primary tool for code review. Maps git diffs to affected
functions, flows, communities, and test coverage gaps. Returns
priority-ordered review guidance with risk scores.
Args:
base: Git ref to diff against (default: HEAD~1).
changed_files: Explicit list of changed file paths (relative to repo
root). Auto-detected from git diff if omitted.
include_source: If True, include source code snippets for changed
functions. Default: False.
max_depth: Impact radius depth for BFS traversal. Default: 2.
repo_root: Repository root path. Auto-detected if omitted.
detail_level: Output detail level. "standard" returns full analysis;
"minimal" returns only summary, risk_score, changed_file_count,
test_gap_count, and top 3 review priorities (text only).
Default: "standard".
Returns:
Risk-scored analysis with changed functions, affected flows,
test gaps, and review priorities.
"""
store, root = _get_store(repo_root)
try:
# Detect changed files if not provided.
if changed_files is None:
changed_files = get_changed_files(root, base)
if not changed_files:
changed_files = get_staged_and_unstaged(root)
if not changed_files:
return {
"status": "ok",
"summary": "No changed files detected.",
"risk_score": 0.0,
"changed_functions": [],
"affected_flows": [],
"test_gaps": [],
"review_priorities": [],
}
original_tokens = estimate_file_tokens(root, changed_files)
# Convert to absolute paths for graph lookup.
abs_files = [str(root / f) for f in changed_files]
# Parse diff ranges for line-level mapping.
diff_ranges = parse_diff_ranges(str(root), base)
# Remap to absolute paths so they match graph file_paths.
abs_ranges: dict[str, list[tuple[int, int]]] = {}
for rel_path, ranges in diff_ranges.items():
abs_path = str(root / rel_path)
abs_ranges[abs_path] = ranges
analysis = analyze_changes(
store,
changed_files=abs_files,
changed_ranges=abs_ranges if abs_ranges else None,
repo_root=str(root),
base=base,
)
# Optionally include source snippets for changed functions.
if include_source:
for func in analysis.get("changed_functions", []):
fp = func.get("file_path")
ls = func.get("line_start")
le = func.get("line_end")
if fp and ls and le:
file_path = Path(fp)
if file_path.is_file():
try:
lines = file_path.read_text(
errors="replace"
).splitlines()
start = max(0, ls - 1)
end = min(len(lines), le)
func["source"] = "\n".join(
f"{i + 1}: {lines[i]}"
for i in range(start, end)
)
except (OSError, UnicodeDecodeError):
func["source"] = "(could not read file)"
if detail_level == "minimal":
priorities = analysis.get("review_priorities", [])
top_priorities = [
p.get("name", p.get("qualified_name", ""))
for p in priorities[:3]
]
result: dict[str, Any] = {
"status": "ok",
"summary": analysis.get("summary", ""),
"risk_score": analysis.get("risk_score", 0.0),
"changed_file_count": len(changed_files),
"test_gap_count": len(analysis.get("test_gaps", [])),
"review_priorities": top_priorities,
}
else:
result = {
"status": "ok",
"changed_files": changed_files,
**analysis,
}
result["_hints"] = generate_hints(
"detect_changes", result, get_session()
)
attach_context_savings(result, original_tokens=original_tokens)
return result
except Exception as exc:
return {"status": "error", "error": str(exc)}
finally:
store.close()
+257
View File
@@ -0,0 +1,257 @@
"""TypeScript tsconfig.json path alias resolver.
Resolves TypeScript path aliases (e.g., ``@/ -> src/``) declared in
``compilerOptions.paths`` so that ``IMPORTS_FROM`` edges can point to
real file paths instead of raw alias strings.
"""
from __future__ import annotations
import json
import logging
import re
from pathlib import Path
from typing import Optional
logger = logging.getLogger(__name__)
# Extensions probed when resolving an alias target
_PROBE_EXTENSIONS = [".ts", ".tsx", ".js", ".jsx", ".vue"]
# Tsconfig filenames to look for when walking up the directory tree
_TSCONFIG_NAMES = ["tsconfig.json", "tsconfig.app.json"]
class TsconfigResolver:
"""Resolves TypeScript path aliases (e.g., @/ -> src/) using tsconfig.json."""
def __init__(self) -> None:
self._cache: dict[str, Optional[dict]] = {}
# ------------------------------------------------------------------
# Public API
# ------------------------------------------------------------------
def resolve_alias(self, import_str: str, file_path: str) -> Optional[str]:
"""Resolve a TS path alias to an absolute file path, or None."""
try:
config = self._load_tsconfig_for_file(file_path)
if config is None:
return None
base_url: Optional[str] = config.get("baseUrl")
paths: dict[str, list[str]] = config.get("paths", {})
tsconfig_dir: str = config.get("_tsconfig_dir", "")
if not paths:
return None
if base_url:
base_dir = (Path(tsconfig_dir) / base_url).resolve()
else:
base_dir = Path(tsconfig_dir).resolve()
return self._match_and_probe(import_str, paths, base_dir)
except (OSError, ValueError, TypeError):
logger.debug(
"TsconfigResolver: unexpected error for %s", file_path, exc_info=True,
)
return None
# ------------------------------------------------------------------
# Internal helpers
# ------------------------------------------------------------------
def _load_tsconfig_for_file(self, file_path: str) -> Optional[dict]:
"""Find and load tsconfig.json for the given file."""
start_dir = Path(file_path).parent.resolve()
current = start_dir
visited: list[str] = []
while True:
dir_str = str(current)
if dir_str in self._cache:
result = self._cache[dir_str]
for visited_dir in visited:
self._cache[visited_dir] = result
return result
visited.append(dir_str)
for name in _TSCONFIG_NAMES:
candidate = current / name
if candidate.is_file():
config = self._parse_tsconfig(candidate)
config["_tsconfig_dir"] = dir_str
for visited_dir in visited:
self._cache[visited_dir] = config
return config
parent = current.parent
if parent == current:
for visited_dir in visited:
self._cache[visited_dir] = None
return None
current = parent
def _parse_tsconfig(self, tsconfig_path: Path) -> dict:
"""Parse a tsconfig.json file (supports JSONC comments)."""
seen: set[str] = set()
return self._resolve_extends(tsconfig_path, seen)
def _resolve_extends(self, tsconfig_path: Path, seen: set[str]) -> dict:
"""Recursively resolve the tsconfig extends chain."""
canonical = str(tsconfig_path.resolve())
if canonical in seen:
logger.debug("TsconfigResolver: cycle detected at %s", canonical)
return {}
seen = seen | {canonical}
try:
raw = tsconfig_path.read_text(encoding="utf-8", errors="replace")
except OSError:
logger.debug("TsconfigResolver: cannot read %s", tsconfig_path)
return {}
stripped = self._strip_jsonc_comments(raw)
try:
data: dict = json.loads(stripped)
except json.JSONDecodeError:
logger.debug("TsconfigResolver: invalid JSON in %s", tsconfig_path)
return {}
result: dict = {}
extends: Optional[str] = data.get("extends")
if extends and isinstance(extends, str) and extends.startswith("."):
parent_path = (tsconfig_path.parent / extends).resolve()
if not parent_path.suffix:
parent_path = parent_path.with_suffix(".json")
if parent_path.is_file():
parent_config = self._resolve_extends(parent_path, seen)
parent_opts = parent_config.get("compilerOptions", {})
result.setdefault("compilerOptions", {}).update(parent_opts)
child_opts: dict = data.get("compilerOptions", {})
result.setdefault("compilerOptions", {}).update(child_opts)
compiler_options = result.get("compilerOptions", {})
if "baseUrl" in compiler_options:
result["baseUrl"] = compiler_options["baseUrl"]
if "paths" in compiler_options:
result["paths"] = compiler_options["paths"]
return result
def _strip_jsonc_comments(self, text: str) -> str:
"""Remove // and /* */ comments and trailing commas from JSONC."""
result: list[str] = []
i = 0
n = len(text)
while i < n:
ch = text[i]
if ch == '"':
result.append(ch)
i += 1
while i < n:
c = text[i]
result.append(c)
if c == "\\" and i + 1 < n:
i += 1
result.append(text[i])
elif c == '"':
break
i += 1
i += 1
continue
if ch == "/" and i + 1 < n and text[i + 1] == "*":
i += 2
while i < n - 1:
if text[i] == "*" and text[i + 1] == "/":
i += 2
break
i += 1
else:
i = n
continue
if ch == "/" and i + 1 < n and text[i + 1] == "/":
i += 2
while i < n and text[i] != "\n":
i += 1
continue
result.append(ch)
i += 1
stripped = "".join(result)
stripped = re.sub(r",\s*([\]}])", r"\1", stripped)
return stripped
def _match_and_probe(
self,
import_str: str,
paths: dict[str, list[str]],
base_dir: Path,
) -> Optional[str]:
"""Match import_str against alias patterns and probe the filesystem."""
def _pattern_specificity(item: tuple[str, list[str]]) -> int:
pat = item[0]
return len(pat.partition("*")[0])
sorted_paths = sorted(paths.items(), key=_pattern_specificity, reverse=True)
for pattern, replacements in sorted_paths:
suffix = _match_pattern(pattern, import_str)
if suffix is None:
continue
for replacement in replacements:
if "*" in replacement:
mapped = replacement.replace("*", suffix, 1)
else:
mapped = replacement
candidate_base = (base_dir / mapped).resolve()
found = _probe_path(candidate_base)
if found:
return str(found)
return None
# ---------------------------------------------------------------------------
# Module-level helpers
# ---------------------------------------------------------------------------
def _match_pattern(pattern: str, import_str: str) -> Optional[str]:
"""Return the wildcard-matched suffix if pattern matches import_str."""
if "*" not in pattern:
return "" if import_str == pattern else None
prefix, _, suffix_pat = pattern.partition("*")
if not (import_str.startswith(prefix) and import_str.endswith(suffix_pat)):
return None
end = len(import_str) - len(suffix_pat) if suffix_pat else len(import_str)
return import_str[len(prefix):end]
def _probe_path(base: Path) -> Optional[Path]:
"""Probe base and base + extensions for an existing file."""
if base.is_file():
return base
for ext in _PROBE_EXTENSIONS:
candidate = base.with_suffix(ext) if not base.suffix else Path(str(base) + ext)
if candidate.is_file():
return candidate
if base.is_dir():
for ext in _PROBE_EXTENSIONS:
candidate = base / f"index{ext}"
if candidate.is_file():
return candidate
return None
File diff suppressed because it is too large Load Diff
+305
View File
@@ -0,0 +1,305 @@
"""Wiki generation from community structure.
Generates markdown pages for each detected community and an index page,
providing a navigable documentation wiki for the codebase architecture.
"""
from __future__ import annotations
import logging
import re
import sqlite3
from collections import Counter
from pathlib import Path
from typing import Any
from .communities import get_communities
from .flows import get_flows
from .graph import GraphStore, _sanitize_name
logger = logging.getLogger(__name__)
def _slugify(name: str) -> str:
"""Convert a community name to a safe filename slug."""
slug = re.sub(r"[^a-z0-9]+", "-", name.lower()).strip("-")
return slug[:80] or "unnamed"
def _generate_community_page(store: GraphStore, community: dict[str, Any]) -> str:
"""Build markdown content for a single community.
Includes: heading, overview (size, cohesion, language), members table
(top 50), execution flows through the community, and dependencies.
Args:
store: The graph store.
community: Community dict from get_communities().
Returns:
Markdown string for the community page.
"""
name = community["name"]
size = community["size"]
cohesion = community.get("cohesion", 0.0)
lang = community.get("dominant_language", "")
description = community.get("description", "")
lines: list[str] = []
lines.append(f"# {name}")
lines.append("")
# Overview section
lines.append("## Overview")
lines.append("")
if description:
lines.append(f"{description}")
lines.append("")
lines.append(f"- **Size**: {size} nodes")
lines.append(f"- **Cohesion**: {cohesion:.4f}")
if lang:
lines.append(f"- **Dominant Language**: {lang}")
lines.append("")
# Members table (top 50)
member_qns = community.get("members", [])
lines.append("## Members")
lines.append("")
if member_qns:
lines.append("| Name | Kind | File | Lines |")
lines.append("|------|------|------|-------|")
# Fetch node details for members (limit to 50)
member_count = 0
for qn in member_qns[:50]:
node = store.get_node(qn)
if node and node.kind != "File":
node_name = _sanitize_name(node.name)
lines.append(
f"| {node_name} | {node.kind} | {node.file_path} "
f"| {node.line_start}-{node.line_end} |"
)
member_count += 1
if not member_count:
# Remove the table headers if no members were added
lines.pop() # header separator
lines.pop() # header
lines.append("No non-file members found.")
if len(member_qns) > 50:
lines.append("")
lines.append(f"*... and {len(member_qns) - 50} more members.*")
else:
lines.append("No members found.")
lines.append("")
# Execution flows through community
lines.append("## Execution Flows")
lines.append("")
member_set = set(member_qns)
try:
all_flows = get_flows(store, sort_by="criticality", limit=200)
community_flows: list[dict] = []
for flow in all_flows:
# Check if this flow passes through any community member
flow_qns = store.get_flow_qualified_names(flow["id"])
if flow_qns & member_set:
community_flows.append(flow)
if community_flows:
for flow in community_flows[:10]:
flow_name = _sanitize_name(flow.get("name", "unnamed"))
criticality = flow.get("criticality", 0.0)
depth = flow.get("depth", 0)
lines.append(
f"- **{flow_name}** (criticality: {criticality:.2f}, depth: {depth})"
)
if len(community_flows) > 10:
lines.append(f"- *... and {len(community_flows) - 10} more flows.*")
else:
lines.append("No execution flows pass through this community.")
except sqlite3.OperationalError as exc:
logger.debug("wiki: flows table unavailable: %s", exc)
lines.append("Execution flow data not available.")
lines.append("")
# Dependencies (cross-community edges)
lines.append("## Dependencies")
lines.append("")
try:
outgoing_targets: Counter[str] = Counter()
incoming_sources: Counter[str] = Counter()
if member_qns:
qns = list(member_qns)
# Outgoing: source is a member
for t in store.get_outgoing_targets(qns):
if t not in member_set:
outgoing_targets[t] += 1
# Incoming: target is a member
for s in store.get_incoming_sources(qns):
if s not in member_set:
incoming_sources[s] += 1
if outgoing_targets:
lines.append("### Outgoing")
lines.append("")
for target, count in outgoing_targets.most_common(15):
lines.append(f"- `{_sanitize_name(target)}` ({count} edge(s))")
lines.append("")
if incoming_sources:
lines.append("### Incoming")
lines.append("")
for source, count in incoming_sources.most_common(15):
lines.append(f"- `{_sanitize_name(source)}` ({count} edge(s))")
lines.append("")
if not outgoing_targets and not incoming_sources:
lines.append("No cross-community dependencies detected.")
lines.append("")
except sqlite3.OperationalError as exc:
logger.debug("wiki: dependency edges unavailable: %s", exc)
lines.append("Dependency data not available.")
lines.append("")
return "\n".join(lines)
def generate_wiki(
store: GraphStore,
wiki_dir: str | Path,
force: bool = False,
) -> dict[str, Any]:
"""Generate a markdown wiki from the community structure.
For each community, generates a markdown page. Also generates an
index.md with links to all community pages.
Args:
store: The graph store.
wiki_dir: Directory to write wiki pages into.
force: If True, regenerate all pages even if content unchanged.
Returns:
Dict with pages_generated, pages_updated, pages_unchanged counts.
"""
wiki_path = Path(wiki_dir)
wiki_path.mkdir(parents=True, exist_ok=True)
communities = get_communities(store)
pages_generated = 0
pages_updated = 0
pages_unchanged = 0
page_entries: list[tuple[str, str, int]] = [] # (slug, name, size)
# Track slugs we've already used in THIS run so two communities that
# slugify to the same filename don't overwrite each other (#222 follow-up).
# Previously "Data Processing" and "data processing" both became
# "data-processing.md", causing silent data loss and inflated "updated"
# counters (each collision was counted as an update while only one file
# made it to disk).
used_slugs: set[str] = set()
for comm in communities:
name = comm["name"]
base_slug = _slugify(name)
slug = base_slug
suffix = 2
while slug in used_slugs:
slug = f"{base_slug}-{suffix}"
suffix += 1
used_slugs.add(slug)
filename = f"{slug}.md"
filepath = wiki_path / filename
content = _generate_community_page(store, comm)
if filepath.exists() and not force:
existing = filepath.read_text(encoding="utf-8", errors="replace")
if existing == content:
pages_unchanged += 1
page_entries.append((slug, name, comm["size"]))
continue
already_existed = filepath.exists()
filepath.write_text(content, encoding="utf-8")
if already_existed:
pages_updated += 1
else:
pages_generated += 1
page_entries.append((slug, name, comm["size"]))
# Generate index.md
index_lines: list[str] = []
index_lines.append("# Code Wiki")
index_lines.append("")
index_lines.append(
"Auto-generated documentation from the code knowledge graph community structure."
)
index_lines.append("")
index_lines.append(f"**Total communities**: {len(communities)}")
index_lines.append("")
index_lines.append("## Communities")
index_lines.append("")
index_lines.append("| Community | Size | Link |")
index_lines.append("|-----------|------|------|")
for slug, name, size in sorted(page_entries, key=lambda x: x[1]):
index_lines.append(f"| {name} | {size} | [{slug}.md]({slug}.md) |")
index_lines.append("")
index_content = "\n".join(index_lines)
index_path = wiki_path / "index.md"
if index_path.exists() and not force:
existing_index = index_path.read_text(encoding="utf-8", errors="replace")
if existing_index == index_content:
pages_unchanged += 1
else:
index_path.write_text(index_content, encoding="utf-8")
pages_updated += 1
else:
index_path.write_text(index_content, encoding="utf-8")
pages_generated += 1
return {
"pages_generated": pages_generated,
"pages_updated": pages_updated,
"pages_unchanged": pages_unchanged,
}
def get_wiki_page(wiki_dir: str | Path, page_name: str) -> str | None:
"""Retrieve a specific wiki page by community name.
Args:
wiki_dir: Directory containing wiki pages.
page_name: Community name (will be slugified for filename lookup).
Returns:
Page content as a string, or None if the page does not exist.
"""
wiki_path = Path(wiki_dir)
slug = _slugify(page_name)
filepath = wiki_path / f"{slug}.md"
if filepath.is_file():
return filepath.read_text(encoding="utf-8", errors="replace")
# Fallback: try exact filename match — with path traversal protection
exact_path = (wiki_path / page_name).resolve()
if exact_path.is_file() and exact_path.is_relative_to(wiki_path.resolve()):
return exact_path.read_text(encoding="utf-8", errors="replace")
# Fallback: search for partial match
if wiki_path.is_dir():
for p in wiki_path.iterdir():
if p.suffix == ".md" and slug in p.stem:
return p.read_text(encoding="utf-8", errors="replace")
return None