chore: import upstream snapshot with attribution
This commit is contained in:
wehub-resource-sync
@@ -0,0 +1,107 @@
|
||||
# Migrating a language extractor out of extract.py
|
||||
|
||||
`graphify/extract.py` is being split into this package, one language per PR
|
||||
(upstream issue #1212). This is the playbook for porting ONE language. It is
|
||||
written so an AI agent can execute it in a single session.
|
||||
|
||||
## Status
|
||||
|
||||
| module | migrated |
|
||||
|---|---|
|
||||
| blade | yes |
|
||||
| zig | yes |
|
||||
| elixir | yes |
|
||||
| razor | yes |
|
||||
| dart | yes |
|
||||
| rust | yes |
|
||||
| go | yes |
|
||||
| powershell (ps1 + psd1 manifest) | yes |
|
||||
| fortran | yes |
|
||||
| sql | yes |
|
||||
| dm (dm/dmm/dmi/dmf) | yes |
|
||||
| bash | yes |
|
||||
| apex | yes |
|
||||
| terraform | yes |
|
||||
| sln | yes |
|
||||
| pascal_forms (dfm + lfm) | yes |
|
||||
| json_config | yes |
|
||||
| (config-driven core: python, js, java, c, cpp, csharp, kotlin, scala, php, lua, swift, groovy, vue, svelte, astro, xaml, groovy) | no — shared _extract_generic core, move as one batch |
|
||||
| (other bespoke: julia, verilog, markdown, objc, csproj, slnx, lazarus_package, pascal) | no |
|
||||
|
||||
Note: config-driven extractors (python, js, java, c, cpp, ruby, csharp,
|
||||
kotlin, scala, php, lua, swift, groovy) depend on the shared
|
||||
`_extract_generic` core (~1,300 lines). Do NOT port them one-by-one; the core
|
||||
must move first as its own coordinated batch. Pick a bespoke extractor.
|
||||
|
||||
## Invariants (non-negotiable)
|
||||
|
||||
1. **Verbatim moves only.** No renames, no docstring edits, no reformatting,
|
||||
no added annotations, no "improvements". Verify: save the block before
|
||||
cutting and confirm the pasted block is byte-identical.
|
||||
2. **One language per PR.** Small diffs keep review trivial and avoid
|
||||
conflicts with other in-flight ports.
|
||||
3. **Facade re-export is mandatory.** `extract.py` must keep exporting every
|
||||
moved name (`from graphify.extractors.<mod> import extract_<lang> # noqa: F401`
|
||||
in the marked migration block, kept alphabetical). Existing importers
|
||||
(`__main__.py`, `watch.py`, `pg_introspect.py`, tests) must not change.
|
||||
4. **Never import from `graphify.extract` inside this package.** Import
|
||||
direction is strictly extract.py -> extractors/. If you need a helper that
|
||||
lives in extract.py, classify it (below) and move it.
|
||||
5. **Zero test edits** outside `tests/test_extractors_registry.py`. The
|
||||
untouched language tests passing IS the proof of behavior preservation.
|
||||
|
||||
## Helper classification
|
||||
|
||||
For every `_name` your function references that is defined OUTSIDE it:
|
||||
|
||||
- run `grep -c '_name' graphify/extract.py` AFTER your candidate move;
|
||||
- remaining uses > 0 -> **shared**: move it to `base.py` and add it to the
|
||||
facade re-import in extract.py;
|
||||
- remaining uses = 0 -> **private**: move it into your language module.
|
||||
|
||||
Closures, constants, and `import` statements defined INSIDE your function
|
||||
move with it for free — leave them exactly where they are. Only add a
|
||||
module-header import for names the pasted code references at module scope
|
||||
that are not satisfied internally, and verify each header import is used.
|
||||
|
||||
## Pre-flight
|
||||
|
||||
1. Check upstream for conflicts: open PRs/issues mentioning your language,
|
||||
and churn: `git log --oneline --since="3 months ago" upstream/<default> | grep -i <lang>`.
|
||||
High churn -> pick another language.
|
||||
2. Confirm your extractor is bespoke (its `extract_<lang>` is a full function,
|
||||
not a 5-line `_extract_generic(path, LanguageConfig(...))` wrapper).
|
||||
3. Check whether tests/ exercises your language's behavior (grep for
|
||||
`test_<lang>`). If it has no behavioral tests, the byte-identity check in
|
||||
step 3 below is the ENTIRE proof of preservation — include the
|
||||
`git diff --color-moved` evidence in your PR description.
|
||||
|
||||
## Steps
|
||||
|
||||
1. Append a failing test to `tests/test_extractors_registry.py`:
|
||||
module import + facade identity + registry identity (copy an existing
|
||||
`test_<lang>_migrated` as the template).
|
||||
2. `grep -n 'def extract_<lang>' graphify/extract.py`; the span ends at the
|
||||
line before the next top-level statement (`^def ` or `^_CONST`). Beware
|
||||
neighbors: top-level constants AFTER your function may belong to the NEXT
|
||||
function (e.g. `_CONFIG_JSON_*` sit after where extract_razor used to be
|
||||
but were never razor's).
|
||||
3. Save the span to a temp file. Create `graphify/extractors/<lang>.py` with
|
||||
module docstring (`"""<Lang> extractor. Moved verbatim from graphify/extract.py."""`),
|
||||
`from __future__ import annotations`, minimal stdlib imports, base imports,
|
||||
then paste the function. Verify byte-identity against the temp file.
|
||||
4. Delete the span from extract.py, leaving exactly two blank lines between
|
||||
the now-adjacent top-level definitions; add the facade re-import; add the
|
||||
registry entry in `__init__.py` (alphabetical); update the Status table
|
||||
above.
|
||||
5. `uv run pytest -q` -> 0 failures, no test file changed except the registry
|
||||
test. If ImportError/NameError: a helper was misclassified — go to
|
||||
Helper classification.
|
||||
6. One commit: `refactor(extract): move extract_<lang> to extractors/<lang>.py (verbatim)`.
|
||||
|
||||
## What NOT to do
|
||||
|
||||
- Do not rewire dispatch, add classes, or add lazy imports — mechanism layers
|
||||
come later, by separate agreement (see #1212 discussion).
|
||||
- Do not port two languages in one PR "while you're at it".
|
||||
- Do not touch `__main__.py`.
|
||||
@@ -0,0 +1,64 @@
|
||||
"""Per-language extractors, incrementally migrated out of graphify/extract.py.
|
||||
|
||||
Dispatch still flows through graphify.extract (the facade re-exports every
|
||||
moved name), so importing from graphify.extract keeps working unchanged.
|
||||
LANGUAGE_EXTRACTORS is the registry seed; wiring dispatch through it is a
|
||||
later, separate step. See MIGRATION.md for how to port another language.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
from typing import Callable
|
||||
|
||||
from graphify.extractors.apex import extract_apex
|
||||
from graphify.extractors.bash import extract_bash
|
||||
from graphify.extractors.blade import extract_blade
|
||||
from graphify.extractors.dart import extract_dart
|
||||
from graphify.extractors.dm import extract_dm, extract_dmf, extract_dmi, extract_dmm
|
||||
from graphify.extractors.elixir import extract_elixir
|
||||
from graphify.extractors.fortran import extract_fortran
|
||||
from graphify.extractors.go import extract_go
|
||||
from graphify.extractors.json_config import extract_json
|
||||
from graphify.extractors.julia import extract_julia
|
||||
from graphify.extractors.markdown import extract_markdown
|
||||
from graphify.extractors.objc import extract_objc
|
||||
from graphify.extractors.pascal import extract_pascal
|
||||
from graphify.extractors.pascal_forms import extract_delphi_form, extract_lazarus_form
|
||||
from graphify.extractors.powershell import extract_powershell, extract_powershell_manifest
|
||||
from graphify.extractors.razor import extract_razor
|
||||
from graphify.extractors.rust import extract_rust
|
||||
from graphify.extractors.sln import extract_sln
|
||||
from graphify.extractors.sql import extract_sql
|
||||
from graphify.extractors.terraform import extract_terraform
|
||||
from graphify.extractors.verilog import extract_verilog
|
||||
from graphify.extractors.zig import extract_zig
|
||||
|
||||
LANGUAGE_EXTRACTORS: dict[str, Callable[[Path], dict]] = {
|
||||
"apex": extract_apex,
|
||||
"bash": extract_bash,
|
||||
"blade": extract_blade,
|
||||
"dart": extract_dart,
|
||||
"delphi_form": extract_delphi_form,
|
||||
"dm": extract_dm,
|
||||
"dmf": extract_dmf,
|
||||
"dmi": extract_dmi,
|
||||
"dmm": extract_dmm,
|
||||
"elixir": extract_elixir,
|
||||
"fortran": extract_fortran,
|
||||
"go": extract_go,
|
||||
"json": extract_json,
|
||||
"julia": extract_julia,
|
||||
"lazarus_form": extract_lazarus_form,
|
||||
"markdown": extract_markdown,
|
||||
"objc": extract_objc,
|
||||
"pascal": extract_pascal,
|
||||
"powershell": extract_powershell,
|
||||
"powershell_manifest": extract_powershell_manifest,
|
||||
"razor": extract_razor,
|
||||
"rust": extract_rust,
|
||||
"sln": extract_sln,
|
||||
"sql": extract_sql,
|
||||
"terraform": extract_terraform,
|
||||
"verilog": extract_verilog,
|
||||
"zig": extract_zig,
|
||||
}
|
||||
@@ -0,0 +1,215 @@
|
||||
"""Apex extractor. Moved verbatim from graphify/extract.py."""
|
||||
from __future__ import annotations
|
||||
|
||||
|
||||
from pathlib import Path
|
||||
from graphify.extractors.base import _file_stem, _make_id
|
||||
|
||||
|
||||
def extract_apex(path: Path) -> dict:
|
||||
"""Extract classes, interfaces, enums, methods, and Salesforce constructs from
|
||||
Apex .cls and .trigger files using regex (no tree-sitter grammar on PyPI)."""
|
||||
import re as _re
|
||||
try:
|
||||
source = path.read_text(encoding="utf-8", errors="replace")
|
||||
except OSError:
|
||||
return {"nodes": [], "edges": []}
|
||||
|
||||
str_path = str(path)
|
||||
stem = _file_stem(path)
|
||||
file_nid = _make_id(str_path)
|
||||
|
||||
nodes: list[dict] = []
|
||||
edges: list[dict] = []
|
||||
seen_ids: set[str] = set()
|
||||
|
||||
def add_node(nid: str, label: str, line: int) -> None:
|
||||
if nid not in seen_ids:
|
||||
seen_ids.add(nid)
|
||||
nodes.append({
|
||||
"id": nid,
|
||||
"label": label,
|
||||
"file_type": "code",
|
||||
"source_file": str_path,
|
||||
"source_location": f"L{line}",
|
||||
})
|
||||
|
||||
def add_edge(src: str, tgt: str, relation: str, line: int,
|
||||
confidence: str = "EXTRACTED") -> None:
|
||||
edges.append({
|
||||
"source": src,
|
||||
"target": tgt,
|
||||
"relation": relation,
|
||||
"confidence": confidence,
|
||||
"source_file": str_path,
|
||||
"source_location": f"L{line}",
|
||||
"weight": 1.0,
|
||||
})
|
||||
|
||||
add_node(file_nid, path.name, 1)
|
||||
|
||||
lines = source.splitlines()
|
||||
|
||||
_ACCESS = r"(?:public|private|protected|global|webService)?"
|
||||
_SHARING = r"(?:\s+(?:with|without|inherited)\s+sharing)?"
|
||||
_MOD = r"(?:\s+(?:abstract|virtual|override|static|final|transient|testMethod))?"
|
||||
_ANNOTATION = r"(?:\s*@\w+(?:\s*\([^)]*\))?\s*)*"
|
||||
|
||||
cls_re = _re.compile(
|
||||
rf"^{_ANNOTATION}\s*{_ACCESS}{_SHARING}{_MOD}\s*class\s+(\w+)"
|
||||
rf"(?:\s+extends\s+(\w+))?(?:\s+implements\s+([\w,\s]+))?\s*\{{?",
|
||||
_re.IGNORECASE,
|
||||
)
|
||||
iface_re = _re.compile(
|
||||
rf"^{_ANNOTATION}\s*{_ACCESS}{_SHARING}{_MOD}\s*interface\s+(\w+)"
|
||||
rf"(?:\s+extends\s+([\w,\s]+))?\s*\{{?",
|
||||
_re.IGNORECASE,
|
||||
)
|
||||
enum_re = _re.compile(
|
||||
rf"^{_ANNOTATION}\s*{_ACCESS}{_SHARING}{_MOD}\s*enum\s+(\w+)\s*\{{?",
|
||||
_re.IGNORECASE,
|
||||
)
|
||||
trigger_re = _re.compile(
|
||||
r"^\s*trigger\s+(\w+)\s+on\s+(\w+)\s*\(",
|
||||
_re.IGNORECASE,
|
||||
)
|
||||
method_re = _re.compile(
|
||||
rf"^{_ANNOTATION}\s*{_ACCESS}{_MOD}\s*(?:static\s+)?[\w<>\[\]]+\s+(\w+)\s*\([^)]*\)\s*(?:throws\s+\w+\s*)?\{{?",
|
||||
_re.IGNORECASE,
|
||||
)
|
||||
annotation_re = _re.compile(r"@(\w+)", _re.IGNORECASE)
|
||||
soql_re = _re.compile(r"\[\s*SELECT\b[^\]]+FROM\s+(\w+)", _re.IGNORECASE)
|
||||
dml_re = _re.compile(r"\b(insert|update|delete|upsert|merge|undelete)\s+\w", _re.IGNORECASE)
|
||||
|
||||
_CONTROL_FLOW = frozenset({
|
||||
"if", "else", "for", "while", "do", "switch", "try", "catch",
|
||||
"finally", "return", "throw", "new", "void", "null",
|
||||
"true", "false", "this", "super", "class", "interface", "enum",
|
||||
"trigger", "on",
|
||||
})
|
||||
|
||||
current_class_nid: str | None = None
|
||||
pending_annotations: list[str] = []
|
||||
|
||||
for lineno, line_text in enumerate(lines, start=1):
|
||||
stripped = line_text.strip()
|
||||
|
||||
if stripped.startswith("@"):
|
||||
for m in annotation_re.finditer(stripped):
|
||||
pending_annotations.append(m.group(1).lower())
|
||||
continue
|
||||
|
||||
tm = trigger_re.match(stripped)
|
||||
if tm:
|
||||
trig_name, sobject = tm.group(1), tm.group(2)
|
||||
trig_nid = _make_id(stem, trig_name)
|
||||
add_node(trig_nid, trig_name, lineno)
|
||||
add_edge(file_nid, trig_nid, "contains", lineno)
|
||||
sob_nid = _make_id(sobject)
|
||||
if sob_nid not in seen_ids:
|
||||
add_node(sob_nid, sobject, lineno)
|
||||
add_edge(trig_nid, sob_nid, "uses", lineno, confidence="INFERRED")
|
||||
current_class_nid = trig_nid
|
||||
pending_annotations = []
|
||||
continue
|
||||
|
||||
cm = cls_re.match(stripped)
|
||||
if cm:
|
||||
class_name = cm.group(1)
|
||||
if class_name.lower() in _CONTROL_FLOW:
|
||||
pending_annotations = []
|
||||
continue
|
||||
class_nid = _make_id(stem, class_name)
|
||||
add_node(class_nid, class_name, lineno)
|
||||
add_edge(file_nid, class_nid, "contains", lineno)
|
||||
if cm.group(2):
|
||||
base = cm.group(2).strip()
|
||||
base_nid = _make_id(stem, base)
|
||||
if base_nid not in seen_ids:
|
||||
base_nid = _make_id(base)
|
||||
if base_nid not in seen_ids:
|
||||
add_node(base_nid, base, lineno)
|
||||
add_edge(class_nid, base_nid, "extends", lineno, confidence="INFERRED")
|
||||
if cm.group(3):
|
||||
for iface in cm.group(3).split(","):
|
||||
iface = iface.strip()
|
||||
if iface:
|
||||
iface_nid = _make_id(stem, iface)
|
||||
if iface_nid not in seen_ids:
|
||||
iface_nid = _make_id(iface)
|
||||
if iface_nid not in seen_ids:
|
||||
add_node(iface_nid, iface, lineno)
|
||||
add_edge(class_nid, iface_nid, "implements", lineno, confidence="INFERRED")
|
||||
current_class_nid = class_nid
|
||||
pending_annotations = []
|
||||
continue
|
||||
|
||||
im = iface_re.match(stripped)
|
||||
if im:
|
||||
iface_name = im.group(1)
|
||||
if iface_name.lower() in _CONTROL_FLOW:
|
||||
pending_annotations = []
|
||||
continue
|
||||
iface_nid = _make_id(stem, iface_name)
|
||||
add_node(iface_nid, iface_name, lineno)
|
||||
add_edge(file_nid if current_class_nid is None else current_class_nid,
|
||||
iface_nid, "contains", lineno)
|
||||
if im.group(2):
|
||||
for parent in im.group(2).split(","):
|
||||
parent = parent.strip()
|
||||
if parent:
|
||||
parent_nid = _make_id(stem, parent)
|
||||
if parent_nid not in seen_ids:
|
||||
parent_nid = _make_id(parent)
|
||||
if parent_nid not in seen_ids:
|
||||
add_node(parent_nid, parent, lineno)
|
||||
add_edge(iface_nid, parent_nid, "extends", lineno, confidence="INFERRED")
|
||||
pending_annotations = []
|
||||
continue
|
||||
|
||||
em = enum_re.match(stripped)
|
||||
if em:
|
||||
enum_name = em.group(1)
|
||||
if enum_name.lower() in _CONTROL_FLOW:
|
||||
pending_annotations = []
|
||||
continue
|
||||
enum_nid = _make_id(stem, enum_name)
|
||||
add_node(enum_nid, enum_name, lineno)
|
||||
add_edge(file_nid if current_class_nid is None else current_class_nid,
|
||||
enum_nid, "contains", lineno)
|
||||
pending_annotations = []
|
||||
continue
|
||||
|
||||
if current_class_nid is not None:
|
||||
mm = method_re.match(stripped)
|
||||
if mm:
|
||||
method_name = mm.group(1)
|
||||
if method_name.lower() not in _CONTROL_FLOW:
|
||||
method_nid = _make_id(current_class_nid, method_name)
|
||||
method_label = f".{method_name}()"
|
||||
add_node(method_nid, method_label, lineno)
|
||||
add_edge(current_class_nid, method_nid, "method", lineno)
|
||||
if "auraenabled" in pending_annotations or "invocablemethod" in pending_annotations:
|
||||
add_edge(file_nid, method_nid, "contains", lineno, confidence="INFERRED")
|
||||
pending_annotations = []
|
||||
continue
|
||||
|
||||
pending_annotations = []
|
||||
|
||||
for sm in soql_re.finditer(line_text):
|
||||
sobject = sm.group(1)
|
||||
sob_nid = _make_id(sobject)
|
||||
if sob_nid not in seen_ids:
|
||||
add_node(sob_nid, sobject, lineno)
|
||||
src = current_class_nid or file_nid
|
||||
add_edge(src, sob_nid, "uses", lineno, confidence="INFERRED")
|
||||
|
||||
for dm in dml_re.finditer(line_text):
|
||||
dml_op = dm.group(1).lower()
|
||||
dml_nid = _make_id(f"dml_{dml_op}")
|
||||
if dml_nid not in seen_ids:
|
||||
add_node(dml_nid, dml_op, lineno)
|
||||
src = current_class_nid or file_nid
|
||||
add_edge(src, dml_nid, "uses", lineno, confidence="INFERRED")
|
||||
|
||||
return {"nodes": nodes, "edges": edges}
|
||||
@@ -0,0 +1,66 @@
|
||||
# DO NOT import from graphify.extract here — direction is extract.py → extractors/ only.
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
from graphify.ids import make_id
|
||||
|
||||
# Language built-in globals that AST may classify as call targets when used as
|
||||
# constructors or coercion functions (e.g. String(x), Number(x), Boolean(x)).
|
||||
# Without this filter they become god-nodes accumulating spurious edges from
|
||||
# every call site. Filter applied at same-file and cross-file resolution.
|
||||
# See issue #726.
|
||||
_LANGUAGE_BUILTIN_GLOBALS: frozenset[str] = frozenset({
|
||||
# JavaScript / TypeScript ECMAScript built-ins
|
||||
"String", "Number", "Boolean", "Object", "Array", "Symbol", "BigInt",
|
||||
"Date", "RegExp", "Error", "TypeError", "RangeError", "SyntaxError",
|
||||
"ReferenceError", "EvalError", "URIError",
|
||||
"Promise", "Map", "Set", "WeakMap", "WeakSet", "JSON", "Math",
|
||||
"Reflect", "Proxy", "Intl",
|
||||
"parseInt", "parseFloat", "isNaN", "isFinite",
|
||||
"encodeURIComponent", "decodeURIComponent", "encodeURI", "decodeURI",
|
||||
# Browser / Node common globals
|
||||
"URL", "URLSearchParams", "FormData", "Blob", "File",
|
||||
"Headers", "Request", "Response", "AbortController", "AbortSignal",
|
||||
"TextEncoder", "TextDecoder", "console",
|
||||
# Python built-in callables
|
||||
"str", "int", "float", "bool", "list", "dict", "set", "tuple", "bytes",
|
||||
"len", "range", "enumerate", "zip", "map", "filter", "sum", "min", "max",
|
||||
"print", "open", "isinstance", "type", "super", "sorted", "reversed",
|
||||
"any", "all", "abs", "round", "next", "iter", "hash", "id", "repr",
|
||||
"callable", "getattr", "setattr", "hasattr", "delattr", "vars", "dir",
|
||||
})
|
||||
|
||||
|
||||
def _make_id(*parts: str) -> str:
|
||||
return make_id(*parts)
|
||||
|
||||
|
||||
def _file_stem(path: Path) -> str:
|
||||
"""Stem used as the node-ID prefix for a file and its symbols.
|
||||
|
||||
The full path (extension dropped) is preserved as path segments; ``make_id``
|
||||
later collapses the separators to underscores. Using every segment — not just
|
||||
the immediate parent dir (#1504) — means same-named files in different
|
||||
directories get distinct IDs instead of colliding into one
|
||||
last-writer-wins node:
|
||||
|
||||
docs/v1/api/README.md -> docs/v1/api/README -> docs_v1_api_readme
|
||||
docs/v2/api/README.md -> docs/v2/api/README -> docs_v2_api_readme
|
||||
|
||||
Top-level files keep a bare stem (``setup.py`` -> ``setup``). When passed an
|
||||
absolute path the whole path is encoded; the extract() id-remap post-pass
|
||||
re-derives the canonical repo-relative form from ``source_file`` so the on-disk
|
||||
location can't leak into the persisted IDs (#502).
|
||||
|
||||
Returns "" for a path with no name (``Path('.')`` — a source_file that equals
|
||||
the scan root, so it has no per-file stem). Guarding here keeps
|
||||
``path.with_suffix("")`` from raising ``ValueError: '.' has an empty name`` and
|
||||
protects every caller, not just ``_semantic_id_remap`` (#1618)."""
|
||||
if not path.name:
|
||||
return ""
|
||||
return path.with_suffix("").as_posix()
|
||||
|
||||
|
||||
def _read_text(node, source: bytes) -> str:
|
||||
return source[node.start_byte:node.end_byte].decode("utf-8", errors="replace")
|
||||
@@ -0,0 +1,248 @@
|
||||
"""Bash extractor. Moved verbatim from graphify/extract.py."""
|
||||
from __future__ import annotations
|
||||
|
||||
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
from graphify.extractors.base import _file_stem, _make_id, _read_text
|
||||
|
||||
|
||||
def extract_bash(path: Path) -> dict:
|
||||
"""Extract functions, source imports, and cross-function calls from a .sh file."""
|
||||
try:
|
||||
import tree_sitter_bash as tsbash
|
||||
from tree_sitter import Language, Parser
|
||||
except ImportError:
|
||||
return {"nodes": [], "edges": [], "error": "tree-sitter-bash not installed"}
|
||||
|
||||
try:
|
||||
language = Language(tsbash.language())
|
||||
parser = Parser(language)
|
||||
source = path.read_bytes()
|
||||
tree = parser.parse(source)
|
||||
root = tree.root_node
|
||||
except Exception as e:
|
||||
return {"nodes": [], "edges": [], "error": str(e)}
|
||||
|
||||
stem = _file_stem(path)
|
||||
str_path = str(path)
|
||||
nodes: list[dict] = []
|
||||
edges: list[dict] = []
|
||||
seen_ids: set[str] = set()
|
||||
function_bodies: list[tuple[str, Any]] = []
|
||||
defined_functions: set[str] = set()
|
||||
|
||||
from graphify.security import sanitize_metadata # module-level cached import
|
||||
|
||||
def add_node(nid: str, label: str, line: int, kind: str = "code") -> None:
|
||||
if nid and nid not in seen_ids:
|
||||
seen_ids.add(nid)
|
||||
nodes.append({"id": nid, "label": label, "file_type": "code",
|
||||
"source_file": str_path, "source_location": f"L{line}",
|
||||
"metadata": sanitize_metadata({"language": "bash", "kind": kind})}) # noqa: E501
|
||||
|
||||
def add_edge(src: str, tgt: str, relation: str, line: int,
|
||||
confidence: str = "EXTRACTED", weight: float = 1.0,
|
||||
context: str | None = None) -> None:
|
||||
if not src or not tgt or src == tgt:
|
||||
return
|
||||
edge = {"source": src, "target": tgt, "relation": relation,
|
||||
"confidence": confidence, "source_file": str_path,
|
||||
"source_location": f"L{line}", "weight": weight}
|
||||
if context:
|
||||
edge["context"] = context
|
||||
edges.append(edge)
|
||||
|
||||
file_nid = _make_id(str(path))
|
||||
# file_nid is fully path-derived and never produced by _make_id(stem, func_name),
|
||||
# so appending "__entry" guarantees a distinct ID from any function node.
|
||||
entry_nid = file_nid + "__entry"
|
||||
add_node(file_nid, path.name, 1, kind="file")
|
||||
add_node(entry_nid, f"{path.name} script", 1, kind="bash_entrypoint")
|
||||
add_edge(file_nid, entry_nid, "contains", 1)
|
||||
|
||||
_BASH_SOURCE_COMMANDS = frozenset({"source", "."})
|
||||
_BASH_SCRIPT_RUNNERS = frozenset({"bash", "sh", "zsh", "ksh", "dash"})
|
||||
# Parent node types that mean a contained command is part of a substitution
|
||||
# or expansion, not a real function call. Token-level filtering misses
|
||||
# these because `$(build)` exposes `build` as a child command whose name
|
||||
# token has no metacharacters — only the parent does.
|
||||
_BASH_EXPANSION_PARENTS = frozenset({
|
||||
"command_substitution",
|
||||
"process_substitution",
|
||||
})
|
||||
|
||||
def text(node) -> str:
|
||||
return source[node.start_byte:node.end_byte].decode("utf-8", errors="replace")
|
||||
|
||||
def is_inside_expansion(node) -> bool:
|
||||
parent = node.parent
|
||||
while parent is not None:
|
||||
if parent.type in _BASH_EXPANSION_PARENTS:
|
||||
return True
|
||||
parent = parent.parent
|
||||
return False
|
||||
|
||||
def literal(node) -> str | None:
|
||||
# Token-level filter: rejects names containing shell metacharacters.
|
||||
# Combined with `is_inside_expansion` for parent-context rejection.
|
||||
raw = text(node).strip()
|
||||
if not raw:
|
||||
return None
|
||||
if raw[0:1] in {"'", '"'} and raw[-1:] == raw[0]:
|
||||
raw = raw[1:-1]
|
||||
if any(token in raw for token in ("$", "`", "$(", "<(", ">", "|", ";", "&")):
|
||||
return None
|
||||
return raw
|
||||
|
||||
def _bash_func_name(node) -> str | None:
|
||||
"""Get the name from a function_definition node."""
|
||||
# bash grammar: function_definition has a word child (the name)
|
||||
for child in node.children:
|
||||
if child.type == "word":
|
||||
return literal(child)
|
||||
return None
|
||||
|
||||
def walk_calls(body_node, func_nid: str, seen_calls: set) -> None:
|
||||
if body_node is None:
|
||||
return
|
||||
for child in body_node.children:
|
||||
if child.type == "function_definition":
|
||||
# Skip nested function definitions — their bodies are walked
|
||||
# separately, so we don't attribute their calls to the
|
||||
# enclosing scope.
|
||||
continue
|
||||
if child.type == "command" and not is_inside_expansion(child):
|
||||
cmd_name_node = child.child_by_field_name("name")
|
||||
if cmd_name_node is None and child.children:
|
||||
cmd_name_node = child.children[0]
|
||||
if cmd_name_node:
|
||||
name = literal(cmd_name_node)
|
||||
# Defined-functions wins. Skip-lists for external commands
|
||||
# would create false negatives when a user defines a
|
||||
# function shadowing an external (`install`, `find`, etc.).
|
||||
if name and name in defined_functions:
|
||||
tgt = _make_id(stem, name)
|
||||
key = (func_nid, tgt)
|
||||
if tgt and key not in seen_calls:
|
||||
seen_calls.add(key)
|
||||
add_edge(func_nid, tgt, "calls",
|
||||
child.start_point[0] + 1,
|
||||
confidence="EXTRACTED", context="call")
|
||||
walk_calls(child, func_nid, seen_calls)
|
||||
|
||||
def walk(node, parent_nid: str) -> None:
|
||||
t = node.type
|
||||
if t == "function_definition":
|
||||
name = _bash_func_name(node)
|
||||
if name:
|
||||
fn_nid = _make_id(stem, name)
|
||||
line = node.start_point[0] + 1
|
||||
add_node(fn_nid, f"{name}()", line, kind="bash_function")
|
||||
add_edge(parent_nid, fn_nid, "defines", line)
|
||||
defined_functions.add(name)
|
||||
# find the compound_statement body
|
||||
body = None
|
||||
for child in node.children:
|
||||
if child.type == "compound_statement":
|
||||
body = child
|
||||
break
|
||||
function_bodies.append((fn_nid, body))
|
||||
# Recurse into the body so nested function definitions are discovered
|
||||
# and added to function_bodies for the second-pass walk_calls.
|
||||
if body is not None:
|
||||
walk(body, fn_nid)
|
||||
return
|
||||
|
||||
if t == "command":
|
||||
if is_inside_expansion(node):
|
||||
return
|
||||
cmd_name_node = node.child_by_field_name("name")
|
||||
if cmd_name_node is None and node.children:
|
||||
cmd_name_node = node.children[0]
|
||||
if cmd_name_node:
|
||||
cmd = literal(cmd_name_node)
|
||||
args = [c for c in node.children
|
||||
if c.type in ("word", "string", "concatenation")
|
||||
and c != cmd_name_node]
|
||||
if cmd in _BASH_SOURCE_COMMANDS and cmd not in defined_functions:
|
||||
# find the path argument (first word after command name)
|
||||
if args:
|
||||
raw = _read_text(args[0], source).strip().strip("'\"")
|
||||
line = node.start_point[0] + 1
|
||||
if raw.startswith((".", "/")):
|
||||
resolved = (path.parent / raw).resolve()
|
||||
# Only emit the edge if the target actually exists on
|
||||
# disk — prevents graph pollution from crafted paths
|
||||
# like `source ../../etc/passwd` that traverse outside
|
||||
# the project tree (B-1).
|
||||
if resolved.exists():
|
||||
tgt_nid = _make_id(str(resolved))
|
||||
add_edge(file_nid, tgt_nid, "imports_from", line,
|
||||
context="import")
|
||||
else:
|
||||
tgt_nid = _make_id(raw)
|
||||
if tgt_nid:
|
||||
add_edge(file_nid, tgt_nid, "imports", line,
|
||||
context="import")
|
||||
elif cmd and cmd not in defined_functions:
|
||||
raw = cmd if cmd.endswith(".sh") else None
|
||||
if cmd in _BASH_SCRIPT_RUNNERS and args:
|
||||
raw = literal(args[0])
|
||||
if raw and raw.endswith(".sh"):
|
||||
resolved = (path.parent / raw).resolve()
|
||||
if resolved.is_file():
|
||||
target_path = resolved
|
||||
if not path.is_absolute():
|
||||
try:
|
||||
target_path = resolved.relative_to(Path.cwd().resolve())
|
||||
except ValueError:
|
||||
pass
|
||||
caller_nid = entry_nid if parent_nid == file_nid else parent_nid
|
||||
add_edge(caller_nid, _make_id(str(target_path)) + "__entry",
|
||||
"calls", node.start_point[0] + 1,
|
||||
context="script_invocation")
|
||||
return
|
||||
|
||||
if t == "declaration_command":
|
||||
# export/declare/readonly VAR=value at program level
|
||||
if node.parent and node.parent.type == "program":
|
||||
for child in node.children:
|
||||
if child.type == "variable_assignment":
|
||||
var_node = child.child_by_field_name("name")
|
||||
if var_node:
|
||||
var = _read_text(var_node, source).strip()
|
||||
if var:
|
||||
var_nid = _make_id(stem, var)
|
||||
line = child.start_point[0] + 1
|
||||
add_node(var_nid, var, line)
|
||||
add_edge(file_nid, var_nid, "defines", line)
|
||||
return
|
||||
|
||||
for child in node.children:
|
||||
walk(child, parent_nid)
|
||||
|
||||
# Pre-pass: collect all defined function names so the source-command handler
|
||||
# in walk() can detect user-defined functions that shadow 'source' / '.'
|
||||
# regardless of definition order in the file.
|
||||
def _prescan_functions(node) -> None:
|
||||
if node.type == "function_definition":
|
||||
name = _bash_func_name(node)
|
||||
if name:
|
||||
defined_functions.add(name)
|
||||
for child in node.children:
|
||||
_prescan_functions(child)
|
||||
else:
|
||||
for child in node.children:
|
||||
_prescan_functions(child)
|
||||
|
||||
_prescan_functions(root)
|
||||
walk(root, file_nid)
|
||||
|
||||
# Second pass: cross-function calls
|
||||
top_seen: set = set()
|
||||
walk_calls(root, entry_nid, top_seen) # top-level calls attributed to the entrypoint
|
||||
for fn_nid, body in function_bodies:
|
||||
walk_calls(body, fn_nid, set())
|
||||
|
||||
return {"nodes": nodes, "edges": edges}
|
||||
@@ -0,0 +1,53 @@
|
||||
"""Laravel Blade template extractor. Moved verbatim from graphify/extract.py."""
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
from graphify.extractors.base import _make_id
|
||||
|
||||
|
||||
def extract_blade(path: Path) -> dict:
|
||||
"""Extract @include, <livewire:> components, and wire:click bindings from Blade templates."""
|
||||
import re
|
||||
try:
|
||||
src = path.read_text(encoding="utf-8", errors="replace")
|
||||
except OSError:
|
||||
return {"error": f"cannot read {path}"}
|
||||
|
||||
file_nid = _make_id(str(path))
|
||||
nodes = [{"id": file_nid, "label": path.name, "file_type": "code",
|
||||
"source_file": str(path), "source_location": None}]
|
||||
edges = []
|
||||
|
||||
# @include('path.to.partial') or @include("path.to.partial")
|
||||
for m in re.finditer(r"@include\(['\"]([^'\"]+)['\"]", src):
|
||||
tgt = m.group(1).replace(".", "/")
|
||||
tgt_nid = _make_id(tgt)
|
||||
if tgt_nid not in {n["id"] for n in nodes}:
|
||||
nodes.append({"id": tgt_nid, "label": m.group(1), "file_type": "code",
|
||||
"source_file": str(path), "source_location": None})
|
||||
edges.append({"source": file_nid, "target": tgt_nid, "relation": "includes",
|
||||
"confidence": "EXTRACTED", "confidence_score": 1.0,
|
||||
"source_file": str(path), "source_location": None, "weight": 1.0})
|
||||
|
||||
# <livewire:component.name /> or <livewire:component.name>
|
||||
for m in re.finditer(r"<livewire:([\w.\-]+)", src):
|
||||
tgt_nid = _make_id(m.group(1))
|
||||
if tgt_nid not in {n["id"] for n in nodes}:
|
||||
nodes.append({"id": tgt_nid, "label": m.group(1), "file_type": "code",
|
||||
"source_file": str(path), "source_location": None})
|
||||
edges.append({"source": file_nid, "target": tgt_nid, "relation": "uses_component",
|
||||
"confidence": "EXTRACTED", "confidence_score": 1.0,
|
||||
"source_file": str(path), "source_location": None, "weight": 1.0})
|
||||
|
||||
# wire:click="methodName"
|
||||
for m in re.finditer(r'wire:click=["\']([^"\']+)["\']', src):
|
||||
tgt_nid = _make_id(m.group(1))
|
||||
if tgt_nid not in {n["id"] for n in nodes}:
|
||||
nodes.append({"id": tgt_nid, "label": m.group(1), "file_type": "code",
|
||||
"source_file": str(path), "source_location": None})
|
||||
edges.append({"source": file_nid, "target": tgt_nid, "relation": "binds_method",
|
||||
"confidence": "EXTRACTED", "confidence_score": 1.0,
|
||||
"source_file": str(path), "source_location": None, "weight": 1.0})
|
||||
|
||||
return {"nodes": nodes, "edges": edges}
|
||||
@@ -0,0 +1,393 @@
|
||||
"""C# cross-file resolution.
|
||||
|
||||
The config-driven C# *extractor* (``extract_csharp`` → ``_extract_generic``)
|
||||
still lives in ``graphify/extract.py``; per ``extractors/MIGRATION.md`` the
|
||||
config-driven languages cannot be ported one-by-one until the shared
|
||||
``_extract_generic`` core moves as its own coordinated batch. This module is
|
||||
the C# home for the parts that *are* cleanly separable — today, the cross-file
|
||||
type-reference resolver below — and is where ``extract_csharp`` will land when
|
||||
the core migration happens.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import html
|
||||
from pathlib import Path
|
||||
|
||||
from graphify.extractors.base import _make_id
|
||||
|
||||
|
||||
def _build_csharp_type_def_index(all_nodes: list[dict]) -> dict[tuple[str, str], str]:
|
||||
"""Return deterministic ``(namespace, name) -> node_id`` C# type definitions."""
|
||||
candidates: dict[tuple[str, str], list[dict]] = {}
|
||||
for node in all_nodes:
|
||||
if node.get("type") == "namespace":
|
||||
continue
|
||||
metadata = node.get("metadata") or {}
|
||||
if not isinstance(metadata, dict):
|
||||
metadata = {}
|
||||
if metadata.get("is_nested_type"):
|
||||
continue
|
||||
nid = node.get("id")
|
||||
label = node.get("label")
|
||||
if not (isinstance(nid, str) and nid and isinstance(label, str) and label):
|
||||
continue
|
||||
source_file = node.get("source_file")
|
||||
if (
|
||||
not isinstance(source_file, str)
|
||||
or not source_file.endswith(".cs")
|
||||
or node.get("file_type") != "code"
|
||||
):
|
||||
continue
|
||||
if label.endswith(")") or label.startswith(".") or "." in label:
|
||||
continue
|
||||
namespace = metadata.get("namespace", "")
|
||||
if not isinstance(namespace, str):
|
||||
namespace = ""
|
||||
candidates.setdefault((namespace, label), []).append(node)
|
||||
|
||||
return {
|
||||
key: sorted(
|
||||
nodes,
|
||||
key=lambda node: (
|
||||
str(node.get("source_file") or ""),
|
||||
str(node.get("source_location") or ""),
|
||||
str(node.get("id") or ""),
|
||||
),
|
||||
)[0]["id"]
|
||||
for key, nodes in candidates.items()
|
||||
}
|
||||
|
||||
|
||||
def _strip_trailing_csharp_generic_args(target_fqn: str) -> str:
|
||||
target_fqn = target_fqn.strip()
|
||||
if not target_fqn.endswith(">"):
|
||||
return target_fqn
|
||||
depth = 0
|
||||
for index in range(len(target_fqn) - 1, -1, -1):
|
||||
char = target_fqn[index]
|
||||
if char == ">":
|
||||
depth += 1
|
||||
elif char == "<":
|
||||
depth -= 1
|
||||
if depth == 0:
|
||||
return target_fqn[:index].strip()
|
||||
return target_fqn
|
||||
|
||||
|
||||
def _resolve_cross_file_csharp_imports(
|
||||
per_file: list[dict],
|
||||
paths: list[Path],
|
||||
all_nodes: list[dict],
|
||||
all_edges: list[dict],
|
||||
) -> None:
|
||||
"""Re-point resolvable C# ``using`` import edges to canonical internal nodes.
|
||||
|
||||
Namespace imports resolve only to canonical C# namespace nodes. Alias imports
|
||||
resolve only when the alias target's prefix is a known canonical namespace and
|
||||
the simple type name exists in the shared C# type-definition index. ``using
|
||||
static`` and nested type aliases remain deliberate gaps because they need
|
||||
member/nested-type modeling beyond this import pass.
|
||||
"""
|
||||
_ = (per_file, paths)
|
||||
namespace_id_by_label: dict[str, str] = {}
|
||||
for node in sorted(
|
||||
all_nodes,
|
||||
key=lambda node: (
|
||||
str(node.get("source_file") or ""),
|
||||
str(node.get("source_location") or ""),
|
||||
str(node.get("id") or ""),
|
||||
),
|
||||
):
|
||||
if node.get("type") != "namespace":
|
||||
continue
|
||||
label = node.get("label")
|
||||
nid = node.get("id")
|
||||
if isinstance(label, str) and label and isinstance(nid, str) and nid:
|
||||
namespace_id_by_label.setdefault(label, nid)
|
||||
|
||||
type_def_index = _build_csharp_type_def_index(all_nodes)
|
||||
if not namespace_id_by_label and not type_def_index:
|
||||
return
|
||||
|
||||
repointed_from: set[str] = set()
|
||||
for edge in all_edges:
|
||||
if edge.get("relation") != "imports":
|
||||
continue
|
||||
metadata = edge.get("metadata") or {}
|
||||
if not isinstance(metadata, dict):
|
||||
continue
|
||||
using_kind = metadata.get("using_kind")
|
||||
target_fqn = metadata.get("target_fqn")
|
||||
if not using_kind or not isinstance(target_fqn, str) or not target_fqn:
|
||||
continue
|
||||
|
||||
resolved = None
|
||||
if using_kind == "namespace":
|
||||
resolved = namespace_id_by_label.get(target_fqn)
|
||||
elif using_kind == "alias":
|
||||
base_fqn = _strip_trailing_csharp_generic_args(html.unescape(target_fqn))
|
||||
prefix, sep, name = base_fqn.rpartition(".")
|
||||
if sep and prefix in namespace_id_by_label:
|
||||
resolved = type_def_index.get((prefix, name))
|
||||
|
||||
old_target = edge.get("target")
|
||||
if resolved and resolved != old_target:
|
||||
edge["target"] = resolved
|
||||
if isinstance(old_target, str) and old_target:
|
||||
repointed_from.add(old_target)
|
||||
|
||||
if not repointed_from:
|
||||
return
|
||||
|
||||
still_referenced: set[str] = set()
|
||||
for edge in all_edges:
|
||||
still_referenced.add(edge.get("source"))
|
||||
still_referenced.add(edge.get("target"))
|
||||
all_nodes[:] = [
|
||||
node for node in all_nodes
|
||||
if node.get("id") not in repointed_from or node.get("id") in still_referenced
|
||||
]
|
||||
|
||||
|
||||
def _resolve_csharp_type_references(
|
||||
per_file: list[dict],
|
||||
paths: list[Path],
|
||||
all_nodes: list[dict],
|
||||
all_edges: list[dict],
|
||||
) -> None:
|
||||
"""Arbitrate all C# ``inherits``/``implements``/``references`` targets.
|
||||
|
||||
The extractor emits provisional same-file bindings and sourceless stubs. This
|
||||
pass is the single soundness gate: it uses only graph-stamped namespace/import
|
||||
facts, keeps a binding only when the referenced simple name resolves to one
|
||||
in-scope real type definition, and otherwise leaves the edge on a dangling stub.
|
||||
"""
|
||||
_ = (per_file, paths)
|
||||
|
||||
def _is_cs_file(value: object) -> bool:
|
||||
return isinstance(value, str) and value.endswith(".cs")
|
||||
|
||||
def _metadata(value: object) -> dict:
|
||||
return value if isinstance(value, dict) else {}
|
||||
|
||||
def _namespace(node: dict | None) -> str:
|
||||
metadata = _metadata((node or {}).get("metadata"))
|
||||
namespace = metadata.get("namespace", "")
|
||||
return namespace if isinstance(namespace, str) else ""
|
||||
|
||||
def _append_unique(items: list[str], value: str) -> None:
|
||||
if value not in items:
|
||||
items.append(value)
|
||||
|
||||
node_by_id = {
|
||||
node["id"]: node
|
||||
for node in all_nodes
|
||||
if isinstance(node.get("id"), str) and node.get("id")
|
||||
}
|
||||
type_def_index = _build_csharp_type_def_index(all_nodes)
|
||||
known_namespaces = {
|
||||
node.get("label")
|
||||
for node in all_nodes
|
||||
if node.get("type") == "namespace" and isinstance(node.get("label"), str)
|
||||
}
|
||||
|
||||
# Each using carries its lexical scope: ("file", None) applies file-wide;
|
||||
# ("namespace", scope_id) applies only where scope_id is in the ref's scope_chain.
|
||||
namespace_usings_by_file: dict[str, list[tuple[str, str, str | None]]] = {}
|
||||
aliases_by_file: dict[str, dict[str, list[tuple[str, str, str | None]]]] = {}
|
||||
|
||||
for edge in all_edges:
|
||||
if edge.get("relation") != "imports":
|
||||
continue
|
||||
source_node = node_by_id.get(edge.get("source"))
|
||||
if not (
|
||||
source_node
|
||||
and isinstance(source_node.get("label"), str)
|
||||
and source_node.get("label", "").endswith(".cs")
|
||||
):
|
||||
continue
|
||||
source_file = source_node.get("source_file")
|
||||
if not _is_cs_file(source_file):
|
||||
continue
|
||||
metadata = _metadata(edge.get("metadata"))
|
||||
target_fqn = metadata.get("target_fqn")
|
||||
if not isinstance(target_fqn, str) or not target_fqn:
|
||||
continue
|
||||
scope_kind = metadata.get("scope_kind") or "file"
|
||||
scope_id = metadata.get("scope_id")
|
||||
using_kind = metadata.get("using_kind")
|
||||
if using_kind == "namespace":
|
||||
entry = (target_fqn, scope_kind, scope_id)
|
||||
bucket = namespace_usings_by_file.setdefault(source_file, [])
|
||||
if entry not in bucket:
|
||||
bucket.append(entry)
|
||||
elif using_kind == "alias":
|
||||
alias = metadata.get("alias")
|
||||
if isinstance(alias, str) and alias:
|
||||
entry = (target_fqn, scope_kind, scope_id)
|
||||
bucket = aliases_by_file.setdefault(source_file, {}).setdefault(alias, [])
|
||||
if entry not in bucket:
|
||||
bucket.append(entry)
|
||||
|
||||
def _scope_chain(node: dict) -> list[str]:
|
||||
chain = _metadata(node.get("metadata")).get("scope_chain")
|
||||
return chain if isinstance(chain, list) else []
|
||||
|
||||
def _using_in_scope(scope_kind: str, scope_id: str | None, source_node: dict) -> bool:
|
||||
if scope_kind == "file":
|
||||
return True
|
||||
return scope_id is not None and scope_id in _scope_chain(source_node)
|
||||
|
||||
def _scopes_for(source_node: dict, source_file: str) -> list[str]:
|
||||
scopes: list[str] = []
|
||||
_append_unique(scopes, _namespace(source_node))
|
||||
_append_unique(scopes, "")
|
||||
for namespace, scope_kind, scope_id in namespace_usings_by_file.get(source_file, []):
|
||||
if _using_in_scope(scope_kind, scope_id, source_node):
|
||||
_append_unique(scopes, namespace)
|
||||
return scopes
|
||||
|
||||
def _resolve_alias(label: str, source_node: dict, source_file: str) -> str | None:
|
||||
hits = set()
|
||||
for target_fqn, scope_kind, scope_id in aliases_by_file.get(source_file, {}).get(label, []):
|
||||
if not _using_in_scope(scope_kind, scope_id, source_node):
|
||||
continue
|
||||
base_fqn = _strip_trailing_csharp_generic_args(html.unescape(target_fqn))
|
||||
namespace, sep, simple_name = base_fqn.rpartition(".")
|
||||
if not sep:
|
||||
simple_name = namespace
|
||||
namespace = ""
|
||||
if not simple_name:
|
||||
continue
|
||||
hit = type_def_index.get((namespace, simple_name))
|
||||
if hit:
|
||||
hits.add(hit)
|
||||
return next(iter(hits)) if len(hits) == 1 else None
|
||||
|
||||
def _resolve_label(label: str, source_node: dict, source_file: str) -> str | None:
|
||||
if label in aliases_by_file.get(source_file, {}):
|
||||
return _resolve_alias(label, source_node, source_file)
|
||||
candidates: list[str] = []
|
||||
for namespace in _scopes_for(source_node, source_file):
|
||||
hit = type_def_index.get((namespace, label))
|
||||
if hit and hit not in candidates:
|
||||
candidates.append(hit)
|
||||
return candidates[0] if len(candidates) == 1 else None
|
||||
|
||||
def _resolve_qualified(label: str, qualifier: object, source_node: dict, source_file: str) -> str | None:
|
||||
# Sound qualified resolution: an in-scope alias for Q shadows the namespace Q. For a qualified
|
||||
# ref Q.label, look up (alias_target_namespace, label). If no in-scope alias, fall through to an
|
||||
# exact known namespace. Dangle on ambiguity / no hit / unknown qualifier.
|
||||
if not isinstance(qualifier, str) or not qualifier:
|
||||
return None
|
||||
in_scope = [
|
||||
entry for entry in aliases_by_file.get(source_file, {}).get(qualifier, [])
|
||||
if _using_in_scope(entry[1], entry[2], source_node)
|
||||
]
|
||||
if in_scope:
|
||||
hits = set()
|
||||
for target_fqn, _scope_kind, _scope_id in in_scope:
|
||||
alias_ns = _strip_trailing_csharp_generic_args(html.unescape(target_fqn))
|
||||
hit = type_def_index.get((alias_ns, label))
|
||||
if hit:
|
||||
hits.add(hit)
|
||||
return next(iter(hits)) if len(hits) == 1 else None
|
||||
if qualifier in known_namespaces:
|
||||
return type_def_index.get((qualifier, label))
|
||||
return None
|
||||
|
||||
def _is_placeholder(node: dict | None) -> bool:
|
||||
return bool(node) and not node.get("source_file")
|
||||
|
||||
def _is_csharp_relevant_target(node: dict) -> bool:
|
||||
if node.get("type") == "namespace":
|
||||
return True
|
||||
source_file = node.get("source_file")
|
||||
return not source_file or _is_cs_file(source_file)
|
||||
|
||||
def _label_for_type_ref_target(target_node: dict, source_file: str) -> str | None:
|
||||
label = target_node.get("label")
|
||||
if not isinstance(label, str) or not label:
|
||||
return None
|
||||
if not label.endswith(".cs"):
|
||||
return label
|
||||
|
||||
stem = label[:-3]
|
||||
for alias in aliases_by_file.get(source_file, {}):
|
||||
if alias.lower() == stem.lower() or _make_id(alias) == _make_id(stem):
|
||||
return alias
|
||||
return stem or None
|
||||
|
||||
def _dangling_stub_id(label: str, current_target: object) -> str:
|
||||
current = node_by_id.get(current_target)
|
||||
if _is_placeholder(current) and current.get("label") == label:
|
||||
return str(current_target)
|
||||
|
||||
for node in all_nodes:
|
||||
nid = node.get("id")
|
||||
if (
|
||||
isinstance(nid, str)
|
||||
and node.get("label") == label
|
||||
and _is_placeholder(node)
|
||||
):
|
||||
return nid
|
||||
|
||||
stem = _make_id(label)
|
||||
stub_id = stem
|
||||
if stub_id in node_by_id:
|
||||
stub_id = _make_id("csharp_type_ref", label)
|
||||
suffix = 2
|
||||
while stub_id in node_by_id:
|
||||
stub_id = _make_id("csharp_type_ref", label, str(suffix))
|
||||
suffix += 1
|
||||
node = {
|
||||
"id": stub_id,
|
||||
"label": label,
|
||||
"file_type": "code",
|
||||
"source_file": "",
|
||||
"source_location": "",
|
||||
}
|
||||
all_nodes.append(node)
|
||||
node_by_id[stub_id] = node
|
||||
return stub_id
|
||||
|
||||
REPOINT_RELATIONS = {"implements", "inherits", "references"}
|
||||
repointed_from: set[str] = set()
|
||||
for edge in all_edges:
|
||||
if edge.get("relation") not in REPOINT_RELATIONS:
|
||||
continue
|
||||
source_file = edge.get("source_file")
|
||||
if not _is_cs_file(source_file):
|
||||
continue
|
||||
source_node = node_by_id.get(edge.get("source"))
|
||||
target_node = node_by_id.get(edge.get("target"))
|
||||
if not source_node or not target_node:
|
||||
continue
|
||||
if not _is_csharp_relevant_target(target_node):
|
||||
continue
|
||||
metadata = _metadata(edge.get("metadata"))
|
||||
label = metadata.get("ref_token") or _label_for_type_ref_target(target_node, source_file)
|
||||
if not label:
|
||||
continue
|
||||
if metadata.get("qualified"):
|
||||
resolved = _resolve_qualified(label, metadata.get("ref_qualifier"), source_node, source_file)
|
||||
else:
|
||||
resolved = _resolve_label(label, source_node, source_file)
|
||||
target = edge.get("target")
|
||||
desired = resolved or _dangling_stub_id(label, target)
|
||||
if desired != target:
|
||||
edge["target"] = desired
|
||||
if isinstance(target, str) and _is_placeholder(target_node):
|
||||
repointed_from.add(target)
|
||||
|
||||
if not repointed_from:
|
||||
return
|
||||
|
||||
still_referenced: set[str] = set()
|
||||
for edge in all_edges:
|
||||
still_referenced.add(edge.get("source"))
|
||||
still_referenced.add(edge.get("target"))
|
||||
all_nodes[:] = [
|
||||
node for node in all_nodes
|
||||
if node.get("id") not in repointed_from or node.get("id") in still_referenced
|
||||
]
|
||||
@@ -0,0 +1,528 @@
|
||||
"""Dart extractor. Moved verbatim from graphify/extract.py."""
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
|
||||
from pathlib import Path
|
||||
from graphify.extractors.base import _file_stem, _make_id
|
||||
|
||||
|
||||
def extract_dart(path: Path) -> dict:
|
||||
"""Extract classes, mixins, functions, imports, generic calls, and annotations from a .dart file using regex."""
|
||||
try:
|
||||
src = path.read_text(encoding="utf-8", errors="replace")
|
||||
except OSError:
|
||||
return {"error": f"cannot read {path}"}
|
||||
|
||||
# Remove inline and multi-line comments while leaving string literals untouched to prevent stripping URLs/paths inside strings
|
||||
comment_string_pattern = re.compile(
|
||||
r'"""(?:\\.|[\s\S])*?"""'
|
||||
r"|'''(?:\\.|[\s\S])*?'''"
|
||||
r'|"(?:\\.|[^"\\])*"'
|
||||
r"|'(?:\\.|[^'\\])*'"
|
||||
r"|/\*[\s\S]*?\*/"
|
||||
r"|//[^\n]*"
|
||||
)
|
||||
def _comment_replace(match: re.Match) -> str:
|
||||
token = match.group(0)
|
||||
if token.startswith("/"):
|
||||
return ""
|
||||
return token
|
||||
src_clean = comment_string_pattern.sub(_comment_replace, src)
|
||||
|
||||
stem = _file_stem(path)
|
||||
file_nid = _make_id(str(path))
|
||||
|
||||
# Check if this is a part-of file and redirect to parent
|
||||
part_of_match = re.search(r"^\s*part\s+of\s+['\"]([^'\"]+)['\"]", src_clean, re.MULTILINE)
|
||||
is_part = False
|
||||
if part_of_match:
|
||||
parent_ref = part_of_match.group(1)
|
||||
if parent_ref.endswith(".dart"):
|
||||
try:
|
||||
parent_path = (path.parent / parent_ref).resolve()
|
||||
if parent_path.exists():
|
||||
stem = _file_stem(parent_path)
|
||||
file_nid = _make_id(str(parent_path))
|
||||
is_part = True
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
nodes = []
|
||||
if not is_part:
|
||||
nodes.append({"id": file_nid, "label": path.name, "file_type": "code",
|
||||
"source_file": str(path), "source_location": None})
|
||||
edges = []
|
||||
defined: set[str] = set()
|
||||
|
||||
def add_node(nid: str, label: str, ftype: str = "code", source_file: str | None = str(path)) -> None:
|
||||
if nid not in defined:
|
||||
nodes.append({"id": nid, "label": label, "file_type": ftype,
|
||||
"source_file": source_file, "source_location": None})
|
||||
defined.add(nid)
|
||||
|
||||
def add_edge(src_id: str, tgt_id: str, relation: str, weight: float = 1.0, context: str | None = None) -> None:
|
||||
edge = {"source": src_id, "target": tgt_id, "relation": relation,
|
||||
"confidence": "EXTRACTED", "confidence_score": 1.0,
|
||||
"source_file": str(path), "source_location": None, "weight": weight}
|
||||
if context:
|
||||
edge["context"] = context
|
||||
edges.append(edge)
|
||||
|
||||
def _split_types(text: str) -> list[str]:
|
||||
parts = []
|
||||
current = []
|
||||
depth = 0
|
||||
for char in text:
|
||||
if char == "<":
|
||||
depth += 1
|
||||
current.append(char)
|
||||
elif char == ">":
|
||||
depth -= 1
|
||||
current.append(char)
|
||||
elif char == "," and depth == 0:
|
||||
parts.append("".join(current).strip())
|
||||
current = []
|
||||
else:
|
||||
current.append(char)
|
||||
if current:
|
||||
parts.append("".join(current).strip())
|
||||
return [p for p in parts if p]
|
||||
|
||||
def _find_matching_brace(text: str, start_pos: int) -> int:
|
||||
brace_count = 0
|
||||
in_double_quote = False
|
||||
in_single_quote = False
|
||||
escape = False
|
||||
|
||||
first_brace = text.find("{", start_pos)
|
||||
if first_brace == -1:
|
||||
return len(text)
|
||||
|
||||
brace_count = 1
|
||||
i = first_brace + 1
|
||||
n = len(text)
|
||||
while i < n:
|
||||
char = text[i]
|
||||
if escape:
|
||||
escape = False
|
||||
i += 1
|
||||
continue
|
||||
if char == "\\":
|
||||
escape = True
|
||||
i += 1
|
||||
continue
|
||||
if text[i:i+3] == '"""' and not in_single_quote:
|
||||
i += 3
|
||||
end = text.find('"""', i)
|
||||
i = end + 3 if end != -1 else n
|
||||
continue
|
||||
if text[i:i+3] == "'''" and not in_double_quote:
|
||||
i += 3
|
||||
end = text.find("'''", i)
|
||||
i = end + 3 if end != -1 else n
|
||||
continue
|
||||
if char == '"' and not in_single_quote:
|
||||
in_double_quote = not in_double_quote
|
||||
elif char == "'" and not in_double_quote:
|
||||
in_single_quote = not in_single_quote
|
||||
elif not in_double_quote and not in_single_quote:
|
||||
if char == "{":
|
||||
brace_count += 1
|
||||
elif char == "}":
|
||||
brace_count -= 1
|
||||
if brace_count == 0:
|
||||
return i + 1
|
||||
i += 1
|
||||
return len(text)
|
||||
|
||||
# 1. Classes, mixins, and enums declarations (with inheritance, mixins, interfaces, and generics)
|
||||
# Supports multiple combined modifiers (e.g., abstract base class, mixin class) without capturing "class" as a name
|
||||
class_pattern = r"^\s*(?:(?:abstract|sealed|base|interface|final|mixin)\s+)*(?:class|mixin|enum|extension\s+type)\s+(\w+)"
|
||||
for m in re.finditer(class_pattern, src_clean, re.MULTILINE):
|
||||
class_name = m.group(1)
|
||||
class_nid = _make_id(stem, class_name)
|
||||
add_node(class_nid, class_name)
|
||||
add_edge(file_nid, class_nid, "defines")
|
||||
|
||||
# Manually parse extends/on, with, and implements in header to handle nested generics brackets balanced
|
||||
start_idx = m.end()
|
||||
rest = src_clean[start_idx : start_idx + 500]
|
||||
|
||||
# Skip class generic parameters
|
||||
if rest.lstrip().startswith("<"):
|
||||
offset = rest.find("<")
|
||||
depth = 1
|
||||
i = offset + 1
|
||||
while i < len(rest) and depth > 0:
|
||||
if rest[i] == "<": depth += 1
|
||||
elif rest[i] == ">": depth -= 1
|
||||
i += 1
|
||||
rest = rest[i:]
|
||||
|
||||
# Skip primary constructor (e.g. extension type MyExt(int id))
|
||||
if rest.lstrip().startswith("("):
|
||||
offset = rest.find("(")
|
||||
depth = 1
|
||||
i = offset + 1
|
||||
while i < len(rest) and depth > 0:
|
||||
if rest[i] == "(": depth += 1
|
||||
elif rest[i] == ")": depth -= 1
|
||||
i += 1
|
||||
rest = rest[i:]
|
||||
|
||||
header_end = rest.find("{")
|
||||
if header_end == -1:
|
||||
header_end = rest.find(";")
|
||||
if header_end == -1:
|
||||
header_end = len(rest)
|
||||
header = rest[:header_end]
|
||||
|
||||
base_class = None
|
||||
generics = None
|
||||
mixins_list = []
|
||||
interfaces_list = []
|
||||
|
||||
# Parse extends or on
|
||||
extends_m = re.search(r"^\s*(?:extends|on)\s+([a-zA-Z0-9_.]+)", header)
|
||||
if extends_m:
|
||||
base_class = extends_m.group(1)
|
||||
rest_header = header[extends_m.end():]
|
||||
if rest_header.strip().startswith("<"):
|
||||
start_idx = rest_header.find("<")
|
||||
depth = 1
|
||||
i = start_idx + 1
|
||||
while i < len(rest_header) and depth > 0:
|
||||
if rest_header[i] == "<":
|
||||
depth += 1
|
||||
elif rest_header[i] == ">":
|
||||
depth -= 1
|
||||
if depth == 0:
|
||||
generics = rest_header[start_idx + 1 : i]
|
||||
break
|
||||
i += 1
|
||||
if generics is not None:
|
||||
header = rest_header[i + 1:]
|
||||
else:
|
||||
header = rest_header
|
||||
else:
|
||||
header = rest_header
|
||||
|
||||
# Parse with
|
||||
with_m = re.search(r"^\s*with\s+", header)
|
||||
if with_m:
|
||||
rest_header = header[with_m.end():]
|
||||
impl_idx = rest_header.find("implements")
|
||||
if impl_idx != -1:
|
||||
mixins_str = rest_header[:impl_idx]
|
||||
header = rest_header[impl_idx:]
|
||||
else:
|
||||
mixins_str = rest_header
|
||||
header = ""
|
||||
mixins_list = _split_types(mixins_str)
|
||||
|
||||
# Parse implements
|
||||
impl_m = re.search(r"^\s*implements\s+", header)
|
||||
if impl_m:
|
||||
interfaces_list = _split_types(header[impl_m.end():])
|
||||
|
||||
# Map extends inheritance relation
|
||||
if base_class:
|
||||
base_nid = _make_id(base_class)
|
||||
add_node(base_nid, base_class, source_file=None)
|
||||
add_edge(class_nid, base_nid, "inherits")
|
||||
|
||||
# Map generic type arguments (e.g. MyBloc extends Bloc<MyEvent, MyState>)
|
||||
if generics:
|
||||
for gen in _split_types(generics):
|
||||
gen_clean = gen.split("<")[0].strip()
|
||||
if gen_clean not in {"String", "int", "double", "bool", "num", "dynamic", "Object", "void"}:
|
||||
gen_nid = _make_id(gen_clean)
|
||||
add_node(gen_nid, gen_clean, source_file=None)
|
||||
add_edge(class_nid, gen_nid, "references")
|
||||
|
||||
# Map mixins
|
||||
for mixin in mixins_list:
|
||||
mixin_clean = mixin.split("<")[0].strip()
|
||||
mixin_nid = _make_id(mixin_clean)
|
||||
add_node(mixin_nid, mixin_clean, source_file=None)
|
||||
add_edge(class_nid, mixin_nid, "mixes_in")
|
||||
|
||||
# Map interfaces
|
||||
for interface in interfaces_list:
|
||||
interface_clean = interface.split("<")[0].strip()
|
||||
interface_nid = _make_id(interface_clean)
|
||||
add_node(interface_nid, interface_clean, source_file=None)
|
||||
add_edge(class_nid, interface_nid, "implements")
|
||||
|
||||
# Extract class body for precise framework dependencies and event handling
|
||||
start_idx = m.start()
|
||||
brace_pos = src_clean.find("{", start_idx)
|
||||
semi_pos = src_clean.find(";", start_idx)
|
||||
|
||||
has_body = brace_pos != -1
|
||||
if has_body and semi_pos != -1 and semi_pos < brace_pos:
|
||||
has_body = False
|
||||
|
||||
if has_body:
|
||||
end_pos = _find_matching_brace(src_clean, start_idx)
|
||||
class_body = src_clean[brace_pos:end_pos]
|
||||
|
||||
# Bloc event registration: on<MyEvent>()
|
||||
for em in re.finditer(r"\bon<(\w+)>\s*\(", class_body):
|
||||
event_name = em.group(1)
|
||||
event_nid = _make_id(event_name)
|
||||
add_node(event_nid, event_name, source_file=None)
|
||||
add_edge(class_nid, event_nid, "calls", context="bloc_event")
|
||||
|
||||
# Bloc state emissions: emit(MyState) or yield MyState
|
||||
for sm in re.finditer(r"\b(?:emit|yield)\s*\(?\s*(?:const\s+)?([A-Z]\w*)\b", class_body):
|
||||
state_name = sm.group(1)
|
||||
if state_name not in {"String", "List", "Map", "Set", "Future", "Stream", "Object"}:
|
||||
state_nid = _make_id(state_name)
|
||||
add_node(state_nid, state_name, source_file=None)
|
||||
add_edge(class_nid, state_nid, "calls", context="emit_state")
|
||||
|
||||
# Bloc event additions: widget.add(MyEvent()) or bloc.add(MyEvent())
|
||||
for am in re.finditer(r"\b(?:\w*[Bb]loc\w*|context\.read<\w+>\(\))\.add\(\s*(?:const\s+)?([A-Z]\w*)\b", class_body):
|
||||
event_name = am.group(1)
|
||||
if event_name not in {"String", "List", "Map", "Set", "Future", "Stream", "Object"}:
|
||||
event_nid = _make_id(event_name)
|
||||
add_node(event_nid, event_name, source_file=None)
|
||||
add_edge(class_nid, event_nid, "calls", context="bloc_add_event")
|
||||
|
||||
# Riverpod provider references: ref.watch(provider)
|
||||
for rm in re.finditer(r"\bref\.(?:watch|read|listen)\s*\(\s*(\w+)\b", class_body):
|
||||
provider_name = rm.group(1)
|
||||
provider_nid = _make_id(provider_name)
|
||||
add_node(provider_nid, provider_name, source_file=None)
|
||||
add_edge(class_nid, provider_nid, "references", context="riverpod_reference")
|
||||
|
||||
# Widget to Bloc references: BlocBuilder<MyBloc, ...>
|
||||
for bm in re.finditer(r"\bBloc(?:Builder|Listener|Consumer|Provider|Selector)\s*<\s*([a-zA-Z0-9_]+)\b", class_body):
|
||||
bloc_name = bm.group(1)
|
||||
if bloc_name not in {"String", "int", "double", "bool", "num", "dynamic", "Object", "void"}:
|
||||
bloc_nid = _make_id(bloc_name)
|
||||
add_node(bloc_nid, bloc_name, source_file=None)
|
||||
add_edge(class_nid, bloc_nid, "references", context="bloc_widget_binding")
|
||||
|
||||
# context.read<MyBloc>() or BlocProvider.of<MyBloc>(context)
|
||||
for lm in re.finditer(r"\b(?:read|watch|select|of)\s*<([a-zA-Z0-9_]+)>", class_body):
|
||||
bloc_name = lm.group(1)
|
||||
if bloc_name not in {"String", "int", "double", "bool", "num", "dynamic", "Object", "void"}:
|
||||
bloc_nid = _make_id(bloc_name)
|
||||
add_node(bloc_nid, bloc_name, source_file=None)
|
||||
add_edge(class_nid, bloc_nid, "references", context="bloc_lookup")
|
||||
|
||||
# 2. Annotations mapping (class, mixin, enum, or function level annotations)
|
||||
# Support: @riverpod, @Riverpod(...), @injectable, @singleton, @RoutePage(), @HiveType(typeId: 0), @RestApi()
|
||||
# Matches `@annotation` and links it to the next class/mixin/enum/function declaration in the file
|
||||
annotation_pattern = r"@(\w+)(?:\([^)]*\))?"
|
||||
for am in re.finditer(annotation_pattern, src_clean):
|
||||
annotation_name = am.group(1)
|
||||
if annotation_name in {"override", "deprecated", "required", "protected", "mustCallSuper"}:
|
||||
continue
|
||||
annotation_pos = am.end()
|
||||
intervening_text = src_clean[annotation_pos : annotation_pos + 300]
|
||||
|
||||
class_m = re.search(r"^\s*(?:(?:abstract|sealed|base|interface|final|mixin)\s+)*(?:class|mixin|enum|extension\s+type)\s+(\w+)", intervening_text, re.MULTILINE)
|
||||
func_m = re.search(r"^\s*(?:factory\s+|static\s+|async\s+|external\s+|abstract\s+)?(?:\([^)]+\)|[a-zA-Z0-9_<>,.?]+)(?:\s+[a-zA-Z0-9_<>,.?]+){0,3}\s+(\w+)\s*\(", intervening_text, re.MULTILINE)
|
||||
|
||||
target_nid = None
|
||||
target_name = None
|
||||
target_type = None
|
||||
|
||||
if class_m and func_m:
|
||||
if class_m.start() < func_m.start():
|
||||
target_name = class_m.group(1)
|
||||
target_type = "class"
|
||||
target_nid = _make_id(stem, target_name)
|
||||
else:
|
||||
target_name = func_m.group(1)
|
||||
target_type = "function"
|
||||
target_nid = _make_id(stem, target_name)
|
||||
elif class_m:
|
||||
target_name = class_m.group(1)
|
||||
target_type = "class"
|
||||
target_nid = _make_id(stem, target_name)
|
||||
elif func_m:
|
||||
target_name = func_m.group(1)
|
||||
target_type = "function"
|
||||
target_nid = _make_id(stem, target_name)
|
||||
|
||||
if target_nid and target_name:
|
||||
actual_intervening = intervening_text[:min(class_m.start() if class_m else 300, func_m.start() if func_m else 300)]
|
||||
if ";" not in actual_intervening and "}" not in actual_intervening and "{" not in actual_intervening:
|
||||
annotation_nid = _make_id("annotation", annotation_name.lower())
|
||||
add_node(annotation_nid, f"@{annotation_name}", ftype="concept", source_file=None)
|
||||
add_edge(target_nid, annotation_nid, "configures")
|
||||
|
||||
# Riverpod specific provider generation mapping (supports camelCase class and functional providers)
|
||||
if annotation_name.lower() == "riverpod":
|
||||
if target_type == "class":
|
||||
provider_name = target_name[0].lower() + target_name[1:] + "Provider" if len(target_name) > 1 else target_name.lower() + "Provider"
|
||||
else:
|
||||
provider_name = target_name + "Provider"
|
||||
provider_nid = _make_id(provider_name)
|
||||
add_node(provider_nid, provider_name, ftype="concept", source_file=str(path))
|
||||
add_edge(target_nid, provider_nid, "defines", context="riverpod_provider")
|
||||
|
||||
# 2.5 Typedefs (Type Aliases)
|
||||
typedef_pattern = r"^\s*typedef\s+(\w+)\s*(?:<[^>]+>)?\s*=\s*([a-zA-Z0-9_<>,.?\s]+);"
|
||||
for m in re.finditer(typedef_pattern, src_clean, re.MULTILINE):
|
||||
typedef_name = m.group(1)
|
||||
target_type = m.group(2).split("<")[0].split(".")[-1].strip()
|
||||
if target_type not in {"String", "int", "double", "bool", "num", "dynamic", "Object", "List", "Map", "Set", "void", "Function"}:
|
||||
typedef_nid = _make_id(stem, typedef_name)
|
||||
add_node(typedef_nid, typedef_name)
|
||||
add_edge(file_nid, typedef_nid, "defines")
|
||||
target_nid = _make_id(target_type)
|
||||
add_node(target_nid, target_type, source_file=None)
|
||||
add_edge(typedef_nid, target_nid, "references", context="typedef")
|
||||
|
||||
# 3. Extensions (extension MyExt on MyClass)
|
||||
ext_pattern = r"^\s{0,4}extension\s+(\w+)?(?:<[^>]+>)?\s+on\s+(\w+)"
|
||||
for m in re.finditer(ext_pattern, src_clean, re.MULTILINE):
|
||||
ext_name = m.group(1) or f"{stem}_anonymous_extension"
|
||||
target_class = m.group(2)
|
||||
|
||||
ext_nid = _make_id(stem, ext_name)
|
||||
label = m.group(1) or f"Extension on {target_class}"
|
||||
add_node(ext_nid, label)
|
||||
add_edge(file_nid, ext_nid, "defines")
|
||||
|
||||
target_nid = _make_id(target_class)
|
||||
add_node(target_nid, target_class, source_file=None)
|
||||
add_edge(ext_nid, target_nid, "extends")
|
||||
|
||||
# 4. Top-level and class-level variable declarations (generic variables, records, late, and destructuring)
|
||||
# Restrict indentation to 0-2 spaces to avoid matching local variables inside functions or switch expressions
|
||||
var_pattern = r"^\s{0,2}(?:late\s+)?(?:(?:final|const|var)\s+)?(?:\([^)]+\)\s+|([a-zA-Z0-9_<>,.?]+(?:\s+[a-zA-Z0-9_<>,.?]+){0,3})\s+)?(?:(\w+)|(?:\w+\s*)?\(([^)]+)\))\s*(?:=|$|;)"
|
||||
for m in re.finditer(var_pattern, src_clean, re.MULTILINE):
|
||||
var_type = m.group(1)
|
||||
single_name = m.group(2)
|
||||
destructured_names = m.group(3)
|
||||
|
||||
if not re.match(r"^\s*(?:late|final|const|var)\b", m.group(0)) and not var_type:
|
||||
continue
|
||||
|
||||
if single_name:
|
||||
if single_name not in {"if", "for", "while", "switch", "catch", "return"}:
|
||||
var_nid = _make_id(stem, single_name)
|
||||
add_node(var_nid, single_name)
|
||||
add_edge(file_nid, var_nid, "defines")
|
||||
|
||||
if var_type and var_type not in {"String", "int", "double", "bool", "num", "dynamic", "Object", "List", "Map", "Set", "void"}:
|
||||
clean_type = var_type.split("<")[0].split(".")[-1].strip()
|
||||
type_nid = _make_id(clean_type)
|
||||
add_node(type_nid, clean_type, source_file=None)
|
||||
add_edge(file_nid, type_nid, "references", context="variable_type")
|
||||
elif destructured_names:
|
||||
for name in [n.strip() for n in destructured_names.split(",") if n.strip()]:
|
||||
if ":" in name:
|
||||
name = name.split(":")[-1].strip()
|
||||
if re.match(r"^[a-zA-Z_]\w*$", name) and not re.match(r"^[A-Z]", name):
|
||||
if name not in {"if", "for", "while", "switch", "catch", "return"}:
|
||||
var_nid = _make_id(stem, name)
|
||||
add_node(var_nid, name)
|
||||
add_edge(file_nid, var_nid, "defines")
|
||||
|
||||
# 5. Top-level and member functions/methods (supports typed/generic/record return types and Riverpod/Bloc references)
|
||||
# Restrict indentation to 0-2 spaces to avoid matching nested local functions or methods inside multiline switch statements
|
||||
method_pattern = r"^\s{0,2}(?:factory\s+|static\s+|async\s+|external\s+|abstract\s+)?(?:\([^)]+\)|[a-zA-Z0-9_<>,.?]+)(?:\s+[a-zA-Z0-9_<>,.?]+){0,3}\s+(\w+(?:\.\w+)?)\s*\("
|
||||
for m in re.finditer(method_pattern, src_clean, re.MULTILINE):
|
||||
raw_name = m.group(1)
|
||||
name = raw_name.split(".")[-1]
|
||||
if name in {"if", "for", "while", "switch", "catch", "return", "void", "dynamic", "final", "const", "get", "set"}:
|
||||
continue
|
||||
if re.match(r"^[A-Z]", name):
|
||||
continue
|
||||
nid = _make_id(stem, name)
|
||||
add_node(nid, name)
|
||||
add_edge(file_nid, nid, "defines")
|
||||
|
||||
# Get function body using matching brace to extract Riverpod reference patterns
|
||||
start_idx = m.start()
|
||||
brace_pos = src_clean.find("{", start_idx)
|
||||
semi_pos = src_clean.find(";", start_idx)
|
||||
arrow_pos = src_clean.find("=>", start_idx)
|
||||
|
||||
has_body = brace_pos != -1
|
||||
if has_body and semi_pos != -1 and semi_pos < brace_pos:
|
||||
has_body = False
|
||||
if has_body and arrow_pos != -1 and arrow_pos < brace_pos:
|
||||
has_body = False
|
||||
|
||||
if has_body:
|
||||
end_pos = _find_matching_brace(src_clean, start_idx)
|
||||
func_body = src_clean[brace_pos:end_pos]
|
||||
|
||||
# Extract Riverpod provider references: ref.watch(provider)
|
||||
for rm in re.finditer(r"\bref\.(?:watch|read|listen)\s*\(\s*(\w+)\b", func_body):
|
||||
provider_name = rm.group(1)
|
||||
provider_nid = _make_id(provider_name)
|
||||
add_node(provider_nid, provider_name, source_file=None)
|
||||
add_edge(nid, provider_nid, "references", context="riverpod_reference")
|
||||
|
||||
# Extract Bloc event additions: widget.add(MyEvent()) or bloc.add(MyEvent())
|
||||
for am in re.finditer(r"\b(?:\w*[Bb]loc\w*|context\.read<\w+>\(\))\.add\(\s*(?:const\s+)?([A-Z]\w*)\b", func_body):
|
||||
event_name = am.group(1)
|
||||
if event_name not in {"String", "List", "Map", "Set", "Future", "Stream", "Object"}:
|
||||
event_nid = _make_id(event_name)
|
||||
add_node(event_nid, event_name, source_file=None)
|
||||
add_edge(nid, event_nid, "calls", context="bloc_add_event")
|
||||
|
||||
# context.read<MyBloc>() or BlocProvider.of<MyBloc>(context)
|
||||
for lm in re.finditer(r"\b(?:read|watch|select|of)\s*<([a-zA-Z0-9_]+)>", func_body):
|
||||
bloc_name = lm.group(1)
|
||||
if bloc_name not in {"String", "int", "double", "bool", "num", "dynamic", "Object", "void"}:
|
||||
bloc_nid = _make_id(bloc_name)
|
||||
add_node(bloc_nid, bloc_name, source_file=None)
|
||||
add_edge(nid, bloc_nid, "references", context="bloc_lookup")
|
||||
|
||||
# Universal Navigation Patters (GoRouter, AutoRoute, Navigator)
|
||||
for nm in re.finditer(r"\b(?:go|push|goNamed|pushNamed|replace|replaceNamed)\s*\(\s*(?:context\s*,\s*)?['\"]([a-zA-Z0-9_/?=&%-]+)['\"]", func_body):
|
||||
route_path = nm.group(1)
|
||||
route_nid = _make_id("route", route_path.replace("/", "_").replace("?", "_").replace("=", "_").replace("&", "_"))
|
||||
add_node(route_nid, f"Route {route_path}", ftype="concept", source_file=None)
|
||||
add_edge(nid, route_nid, "navigates", context="route_path")
|
||||
|
||||
for cm in re.finditer(r"\b(?:go|push|goNamed|pushNamed|replace|replaceNamed)\s*\(\s*(?:context\s*,\s*)?([A-Z][a-zA-Z0-9_]*\.[a-zA-Z0-9_]+)", func_body):
|
||||
route_const = cm.group(1)
|
||||
route_nid = _make_id("route", route_const.replace(".", "_"))
|
||||
add_node(route_nid, route_const, ftype="concept", source_file=None)
|
||||
add_edge(nid, route_nid, "navigates", context="route_const")
|
||||
|
||||
for om in re.finditer(r"\b(?:push|replace)\s*\(\s*(?:context\s*,\s*)?.*?\b([A-Z]\w*(?:Route|Screen|Page))\b", func_body):
|
||||
route_class = om.group(1)
|
||||
route_nid = _make_id(route_class)
|
||||
add_node(route_nid, route_class, source_file=None)
|
||||
add_edge(nid, route_nid, "navigates", context="route_object")
|
||||
|
||||
# 6. Imports and Exports
|
||||
for m in re.finditer(r"""^\s*import\s+['"]([^'"]+)['"]""", src_clean, re.MULTILINE):
|
||||
pkg = m.group(1)
|
||||
tgt_nid = _make_id(pkg)
|
||||
add_node(tgt_nid, pkg, source_file=None)
|
||||
add_edge(file_nid, tgt_nid, "imports")
|
||||
|
||||
for m in re.finditer(r"""^\s*export\s+['"]([^'"]+)['"]""", src_clean, re.MULTILINE):
|
||||
pkg = m.group(1)
|
||||
tgt_nid = _make_id(pkg)
|
||||
add_node(tgt_nid, pkg, source_file=None)
|
||||
add_edge(file_nid, tgt_nid, "exports")
|
||||
|
||||
# 7. Generic Invocations / Type Lookups (Universal Dependency Lookup)
|
||||
# Matches any method call with type parameters: methodName<Type>() or object.methodName<Type>()
|
||||
# Automatically extracts GetIt, Injectable, Riverpod, Provider, BlocProvider, and InheritedWidget type lookups!
|
||||
generic_call_pattern = r"\b\w+<([a-zA-Z0-9_.]+(?:<[a-zA-Z0-9_.,\s<>]+>)?)\s*>\s*\("
|
||||
type_blacklist = {"String", "int", "double", "bool", "num", "dynamic", "Object", "List", "Map", "Set", "Future", "Stream", "void"}
|
||||
for m in re.finditer(generic_call_pattern, src_clean):
|
||||
type_name = m.group(1).split(".")[-1].strip()
|
||||
clean_name = type_name.split("<")[0].strip()
|
||||
if clean_name not in type_blacklist:
|
||||
target_nid = _make_id(clean_name)
|
||||
add_node(target_nid, clean_name, source_file=None)
|
||||
add_edge(file_nid, target_nid, "references", context="type_lookup")
|
||||
|
||||
return {"nodes": nodes, "edges": edges}
|
||||
@@ -0,0 +1,494 @@
|
||||
"""Dm extractor. Moved verbatim from graphify/extract.py."""
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
from graphify.extractors.base import _file_stem, _make_id, _read_text
|
||||
|
||||
|
||||
def extract_dm(path: Path) -> dict:
|
||||
"""Extract types, procs, includes, and calls from a .dm/.dme file."""
|
||||
try:
|
||||
import tree_sitter_dm as tsdm
|
||||
from tree_sitter import Language, Parser
|
||||
except ImportError:
|
||||
return {"nodes": [], "edges": [], "error": "tree-sitter-dm not installed"}
|
||||
try:
|
||||
language = Language(tsdm.language())
|
||||
parser = Parser(language)
|
||||
source = path.read_bytes()
|
||||
tree = parser.parse(source)
|
||||
root = tree.root_node
|
||||
except Exception as e:
|
||||
return {"nodes": [], "edges": [], "error": str(e)}
|
||||
|
||||
stem = _file_stem(path)
|
||||
str_path = str(path)
|
||||
nodes: list[dict] = []
|
||||
edges: list[dict] = []
|
||||
seen_ids: set[str] = set()
|
||||
function_bodies: list[tuple[str, Any, "str | None"]] = []
|
||||
|
||||
def add_node(nid: str, label: str, line: int) -> None:
|
||||
if nid and nid not in seen_ids:
|
||||
seen_ids.add(nid)
|
||||
nodes.append({"id": nid, "label": label, "file_type": "code",
|
||||
"source_file": str_path, "source_location": f"L{line}"})
|
||||
|
||||
def add_edge(src: str, tgt: str, relation: str, line: int,
|
||||
confidence: str = "EXTRACTED", weight: float = 1.0,
|
||||
context: str | None = None) -> None:
|
||||
if not src or not tgt or src == tgt:
|
||||
return
|
||||
edge: dict = {"source": src, "target": tgt, "relation": relation,
|
||||
"confidence": confidence, "source_file": str_path,
|
||||
"source_location": f"L{line}", "weight": weight}
|
||||
if context:
|
||||
edge["context"] = context
|
||||
edges.append(edge)
|
||||
|
||||
file_nid = _make_id(str(path))
|
||||
add_node(file_nid, path.name, 1)
|
||||
|
||||
def _type_path_text(node) -> str:
|
||||
return _read_text(node, source).strip()
|
||||
|
||||
def _ensure_type(path_text: str, line: int) -> str:
|
||||
nid = _make_id(stem, path_text)
|
||||
add_node(nid, path_text, line)
|
||||
return nid
|
||||
|
||||
def _find_child(node, type_name: str):
|
||||
for c in node.children:
|
||||
if c.type == type_name:
|
||||
return c
|
||||
return None
|
||||
|
||||
def _read_include_path(file_node) -> str:
|
||||
if file_node is None:
|
||||
return ""
|
||||
if file_node.type == "string_literal":
|
||||
parts = []
|
||||
for c in file_node.children:
|
||||
if c.type == "string_content":
|
||||
parts.append(_read_text(c, source))
|
||||
return "".join(parts)
|
||||
return _read_text(file_node, source).strip("'\"")
|
||||
|
||||
def walk(node, parent_type_path: "str | None" = None,
|
||||
parent_type_nid: "str | None" = None) -> None:
|
||||
t = node.type
|
||||
line = node.start_point[0] + 1
|
||||
|
||||
if t == "preproc_include":
|
||||
file_node = node.child_by_field_name("file")
|
||||
raw = _read_include_path(file_node)
|
||||
if raw:
|
||||
norm = raw.replace("\\", "/").lstrip("./")
|
||||
resolved = (path.parent / norm).resolve()
|
||||
edge: dict = {
|
||||
"source": file_nid,
|
||||
"target": _make_id(str(resolved)) if resolved.exists() else _make_id(norm),
|
||||
"relation": "imports_from" if resolved.exists() else "imports",
|
||||
"context": "import",
|
||||
"confidence": "EXTRACTED",
|
||||
"source_file": str_path,
|
||||
"source_location": f"L{line}",
|
||||
"weight": 1.0,
|
||||
}
|
||||
if not resolved.exists():
|
||||
edge["external"] = True
|
||||
edges.append(edge)
|
||||
return
|
||||
|
||||
if t == "type_definition":
|
||||
tp_node = _find_child(node, "type_path")
|
||||
if tp_node is None:
|
||||
return
|
||||
type_path_str = _type_path_text(tp_node)
|
||||
type_nid = _ensure_type(type_path_str, line)
|
||||
add_edge(file_nid, type_nid, "contains", line)
|
||||
body = _find_child(node, "type_body")
|
||||
if body is not None:
|
||||
for c in body.children:
|
||||
walk(c, parent_type_path=type_path_str, parent_type_nid=type_nid)
|
||||
return
|
||||
|
||||
if t in ("type_body_intended", "type_body_braced"):
|
||||
for c in node.children:
|
||||
walk(c, parent_type_path, parent_type_nid)
|
||||
return
|
||||
|
||||
if t in ("type_proc_definition", "type_proc_override"):
|
||||
if parent_type_nid is None or parent_type_path is None:
|
||||
return
|
||||
name_node = node.child_by_field_name("name")
|
||||
if name_node is None:
|
||||
return
|
||||
proc_name = _read_text(name_node, source)
|
||||
proc_nid = _make_id(stem, parent_type_path, proc_name)
|
||||
add_node(proc_nid, f"{parent_type_path}/{proc_name}()", line)
|
||||
add_edge(parent_type_nid, proc_nid, "method", line)
|
||||
block = _find_child(node, "block")
|
||||
if block is not None:
|
||||
function_bodies.append((proc_nid, block, parent_type_path))
|
||||
return
|
||||
|
||||
if t in ("proc_definition", "proc_override"):
|
||||
tp_node = _find_child(node, "type_path")
|
||||
owner_path: "str | None" = None
|
||||
owner_nid: "str | None" = None
|
||||
if tp_node is not None:
|
||||
owner_path = _type_path_text(tp_node)
|
||||
owner_nid = _ensure_type(owner_path, line)
|
||||
add_edge(file_nid, owner_nid, "contains", line)
|
||||
name_node = node.child_by_field_name("name")
|
||||
if name_node is None:
|
||||
return
|
||||
proc_name = _read_text(name_node, source)
|
||||
if owner_path and owner_nid:
|
||||
proc_nid = _make_id(stem, owner_path, proc_name)
|
||||
add_node(proc_nid, f"{owner_path}/{proc_name}()", line)
|
||||
add_edge(owner_nid, proc_nid, "method", line)
|
||||
else:
|
||||
proc_nid = _make_id(stem, proc_name)
|
||||
add_node(proc_nid, f"{proc_name}()", line)
|
||||
add_edge(file_nid, proc_nid, "contains", line)
|
||||
block = _find_child(node, "block")
|
||||
if block is not None:
|
||||
function_bodies.append((proc_nid, block, owner_path))
|
||||
return
|
||||
|
||||
if t in ("operator_override", "type_operator_override"):
|
||||
return
|
||||
|
||||
for child in node.children:
|
||||
walk(child, parent_type_path, parent_type_nid)
|
||||
|
||||
walk(root)
|
||||
|
||||
label_to_nids: dict[str, list[str]] = {}
|
||||
path_to_nids: dict[str, list[str]] = {}
|
||||
for n in nodes:
|
||||
label = n["label"].strip("()")
|
||||
last = label.rsplit("/", 1)[-1] if "/" in label else label
|
||||
if last:
|
||||
label_to_nids.setdefault(last.lower(), []).append(n["id"])
|
||||
if label.startswith("/"):
|
||||
path_to_nids.setdefault(label.lower(), []).append(n["id"])
|
||||
|
||||
seen_call_pairs: set[tuple[str, str]] = set()
|
||||
raw_calls: list[dict] = []
|
||||
|
||||
def _emit_call(caller_nid: str, callee: str, line: int, is_member: bool) -> None:
|
||||
candidates = label_to_nids.get(callee.lower(), [])
|
||||
tgt_nid = candidates[0] if len(candidates) == 1 else None
|
||||
if tgt_nid and tgt_nid != caller_nid:
|
||||
pair = (caller_nid, tgt_nid)
|
||||
if pair in seen_call_pairs:
|
||||
return
|
||||
seen_call_pairs.add(pair)
|
||||
edges.append({
|
||||
"source": caller_nid, "target": tgt_nid, "relation": "calls",
|
||||
"context": "call", "confidence": "EXTRACTED",
|
||||
"source_file": str_path, "source_location": f"L{line}", "weight": 1.0,
|
||||
})
|
||||
else:
|
||||
raw_calls.append({
|
||||
"caller_nid": caller_nid, "callee": callee,
|
||||
"is_member_call": is_member, "source_file": str_path,
|
||||
"source_location": f"L{line}",
|
||||
})
|
||||
|
||||
def walk_calls(body_node, caller_nid: str) -> None:
|
||||
if body_node is None:
|
||||
return
|
||||
t = body_node.type
|
||||
if t in ("proc_definition", "proc_override", "type_proc_definition",
|
||||
"type_proc_override", "type_definition"):
|
||||
return
|
||||
if t == "call_expression":
|
||||
name_node = body_node.child_by_field_name("name")
|
||||
if name_node is not None:
|
||||
callee = _read_text(name_node, source)
|
||||
if callee and callee != "..":
|
||||
_emit_call(caller_nid, callee, body_node.start_point[0] + 1,
|
||||
is_member=False)
|
||||
elif t == "field_proc_expression":
|
||||
proc_field = body_node.child_by_field_name("proc")
|
||||
if proc_field is not None:
|
||||
callee = _read_text(proc_field, source)
|
||||
if callee:
|
||||
_emit_call(caller_nid, callee, body_node.start_point[0] + 1,
|
||||
is_member=True)
|
||||
elif t == "new_expression":
|
||||
tp_node = _find_child(body_node, "type_path")
|
||||
if tp_node is not None:
|
||||
target_text = _type_path_text(tp_node)
|
||||
candidates = path_to_nids.get(target_text.lower(), [])
|
||||
tgt_nid = candidates[0] if len(candidates) == 1 else None
|
||||
if tgt_nid and tgt_nid != caller_nid:
|
||||
pair = (caller_nid, tgt_nid)
|
||||
if pair not in seen_call_pairs:
|
||||
seen_call_pairs.add(pair)
|
||||
edges.append({
|
||||
"source": caller_nid, "target": tgt_nid,
|
||||
"relation": "instantiates", "context": "call",
|
||||
"confidence": "EXTRACTED", "source_file": str_path,
|
||||
"source_location": f"L{body_node.start_point[0] + 1}",
|
||||
"weight": 1.0,
|
||||
})
|
||||
for child in body_node.children:
|
||||
walk_calls(child, caller_nid)
|
||||
|
||||
for proc_nid, block, _owner_path in function_bodies:
|
||||
walk_calls(block, proc_nid)
|
||||
|
||||
return {"nodes": nodes, "edges": edges, "raw_calls": raw_calls}
|
||||
|
||||
def _read_dmi_description(data: bytes) -> str:
|
||||
"""Pull the BYOND metadata text out of a .dmi PNG, or empty string on failure."""
|
||||
import struct
|
||||
import zlib as _zlib
|
||||
if not data.startswith(b"\x89PNG\r\n\x1a\n"):
|
||||
return ""
|
||||
i = 8
|
||||
while i + 8 <= len(data):
|
||||
length = struct.unpack(">I", data[i:i + 4])[0]
|
||||
chunk_type = data[i + 4:i + 8]
|
||||
payload = data[i + 8:i + 8 + length]
|
||||
if chunk_type in (b"tEXt", b"zTXt"):
|
||||
try:
|
||||
null = payload.index(b"\x00")
|
||||
except ValueError:
|
||||
return ""
|
||||
keyword = payload[:null]
|
||||
if keyword == b"Description":
|
||||
if chunk_type == b"zTXt":
|
||||
return _zlib.decompressobj().decompress(payload[null + 2:], max_length=1024 * 1024).decode("utf-8", errors="replace")
|
||||
return payload[null + 1:].decode("utf-8", errors="replace")
|
||||
i += 8 + length + 4
|
||||
return ""
|
||||
|
||||
def extract_dmi(path: Path) -> dict:
|
||||
"""Extract icon state names from a .dmi (BYOND PNG icon sheet)."""
|
||||
try:
|
||||
data = path.read_bytes()
|
||||
except Exception as e:
|
||||
return {"nodes": [], "edges": [], "error": str(e)}
|
||||
|
||||
str_path = str(path)
|
||||
stem = _file_stem(path)
|
||||
file_nid = _make_id(str(path))
|
||||
nodes: list[dict] = [{"id": file_nid, "label": path.name, "file_type": "code",
|
||||
"source_file": str_path, "source_location": "L1"}]
|
||||
edges: list[dict] = []
|
||||
seen: set[str] = {file_nid}
|
||||
|
||||
description = _read_dmi_description(data)
|
||||
if not description:
|
||||
return {"nodes": nodes, "edges": edges}
|
||||
|
||||
line_no = 0
|
||||
for raw_line in description.splitlines():
|
||||
line_no += 1
|
||||
stripped = raw_line.strip()
|
||||
if not stripped.startswith("state ="):
|
||||
continue
|
||||
value = stripped.split("=", 1)[1].strip()
|
||||
if value.startswith('"') and value.endswith('"') and len(value) >= 2:
|
||||
state_name = value[1:-1]
|
||||
else:
|
||||
state_name = value
|
||||
if not state_name:
|
||||
continue
|
||||
nid = _make_id(stem, "state", state_name)
|
||||
if nid in seen:
|
||||
continue
|
||||
seen.add(nid)
|
||||
nodes.append({"id": nid, "label": f'"{state_name}"', "file_type": "code",
|
||||
"source_file": str_path, "source_location": f"L{line_no}"})
|
||||
edges.append({"source": file_nid, "target": nid, "relation": "contains",
|
||||
"confidence": "EXTRACTED", "source_file": str_path,
|
||||
"source_location": f"L{line_no}", "weight": 1.0})
|
||||
|
||||
return {"nodes": nodes, "edges": edges}
|
||||
|
||||
_DMM_GRID_RE = re.compile(r"^\(\s*\d+\s*,\s*\d+\s*,\s*\d+\s*\)\s*=", re.MULTILINE)
|
||||
|
||||
def _split_dmm_tile(body: str) -> list[str]:
|
||||
out: list[str] = []
|
||||
buf: list[str] = []
|
||||
depth = 0
|
||||
in_string = False
|
||||
escape = False
|
||||
for ch in body:
|
||||
if escape:
|
||||
buf.append(ch)
|
||||
escape = False
|
||||
continue
|
||||
if in_string:
|
||||
buf.append(ch)
|
||||
if ch == "\\":
|
||||
escape = True
|
||||
elif ch == '"':
|
||||
in_string = False
|
||||
continue
|
||||
if ch == '"':
|
||||
in_string = True
|
||||
buf.append(ch)
|
||||
elif ch in "({[":
|
||||
depth += 1
|
||||
buf.append(ch)
|
||||
elif ch in ")}]":
|
||||
depth -= 1
|
||||
buf.append(ch)
|
||||
elif ch == "," and depth == 0:
|
||||
out.append("".join(buf).strip())
|
||||
buf = []
|
||||
else:
|
||||
buf.append(ch)
|
||||
tail = "".join(buf).strip()
|
||||
if tail:
|
||||
out.append(tail)
|
||||
return out
|
||||
|
||||
def _dmm_type_path(entry: str) -> str:
|
||||
brace = entry.find("{")
|
||||
if brace != -1:
|
||||
entry = entry[:brace]
|
||||
return entry.strip()
|
||||
|
||||
def extract_dmm(path: Path) -> dict:
|
||||
"""Extract type-path references from a .dmm map file's tile dictionary."""
|
||||
try:
|
||||
if path.stat().st_size > 50 * 1024 * 1024:
|
||||
return {"nodes": [], "edges": [], "error": "file too large (>50 MB)"}
|
||||
text = path.read_text(encoding="utf-8", errors="replace")
|
||||
except Exception as e:
|
||||
return {"nodes": [], "edges": [], "error": str(e)}
|
||||
|
||||
str_path = str(path)
|
||||
file_nid = _make_id(str(path))
|
||||
nodes: list[dict] = [{"id": file_nid, "label": path.name, "file_type": "code",
|
||||
"source_file": str_path, "source_location": "L1"}]
|
||||
edges: list[dict] = []
|
||||
|
||||
grid_match = _DMM_GRID_RE.search(text)
|
||||
dict_text = text[:grid_match.start()] if grid_match else text
|
||||
|
||||
seen_targets: set[str] = set()
|
||||
buf: list[str] = []
|
||||
open_line = 0
|
||||
depth = 0
|
||||
in_string = False
|
||||
escape = False
|
||||
for line_idx, line in enumerate(dict_text.splitlines(), start=1):
|
||||
for ch in line:
|
||||
if escape:
|
||||
escape = False
|
||||
elif in_string:
|
||||
if ch == "\\":
|
||||
escape = True
|
||||
elif ch == '"':
|
||||
in_string = False
|
||||
elif ch == '"':
|
||||
in_string = True
|
||||
elif ch == "(":
|
||||
if depth == 0:
|
||||
open_line = line_idx
|
||||
depth += 1
|
||||
elif ch == ")":
|
||||
depth -= 1
|
||||
buf.append(ch)
|
||||
buf.append("\n")
|
||||
if depth == 0 and buf:
|
||||
chunk = "".join(buf)
|
||||
buf = []
|
||||
lp = chunk.find("(")
|
||||
rp = chunk.rfind(")")
|
||||
if lp == -1 or rp == -1 or rp <= lp:
|
||||
continue
|
||||
inner = chunk[lp + 1:rp]
|
||||
for entry in _split_dmm_tile(inner):
|
||||
tpath = _dmm_type_path(entry)
|
||||
if not tpath.startswith("/"):
|
||||
continue
|
||||
tgt = _make_id(tpath)
|
||||
if tgt in seen_targets:
|
||||
continue
|
||||
seen_targets.add(tgt)
|
||||
edges.append({"source": file_nid, "target": tgt, "relation": "uses",
|
||||
"context": "map", "confidence": "EXTRACTED",
|
||||
"source_file": str_path,
|
||||
"source_location": f"L{open_line}", "weight": 1.0})
|
||||
|
||||
return {"nodes": nodes, "edges": edges}
|
||||
|
||||
_DMF_WINDOW_RE = re.compile(r'^\s*window\s+"([^"]+)"\s*$')
|
||||
|
||||
_DMF_ELEM_RE = re.compile(r'^\s*elem\s+"([^"]+)"\s*$')
|
||||
|
||||
_DMF_TYPE_RE = re.compile(r'^\s*type\s*=\s*(\S+)\s*$')
|
||||
|
||||
def extract_dmf(path: Path) -> dict:
|
||||
"""Extract windows and controls from a .dmf interface file."""
|
||||
try:
|
||||
text = path.read_text(encoding="utf-8", errors="replace")
|
||||
except Exception as e:
|
||||
return {"nodes": [], "edges": [], "error": str(e)}
|
||||
|
||||
str_path = str(path)
|
||||
stem = _file_stem(path)
|
||||
file_nid = _make_id(str(path))
|
||||
nodes: list[dict] = [{"id": file_nid, "label": path.name, "file_type": "code",
|
||||
"source_file": str_path, "source_location": "L1"}]
|
||||
edges: list[dict] = []
|
||||
seen: set[str] = {file_nid}
|
||||
|
||||
current_window_nid: str | None = None
|
||||
current_elem_nid: str | None = None
|
||||
current_elem_name: str | None = None
|
||||
|
||||
for line_idx, line in enumerate(text.splitlines(), start=1):
|
||||
m = _DMF_WINDOW_RE.match(line)
|
||||
if m:
|
||||
name = m.group(1)
|
||||
nid = _make_id(stem, "window", name)
|
||||
if nid not in seen:
|
||||
seen.add(nid)
|
||||
nodes.append({"id": nid, "label": f'window "{name}"', "file_type": "code",
|
||||
"source_file": str_path, "source_location": f"L{line_idx}"})
|
||||
edges.append({"source": file_nid, "target": nid, "relation": "contains",
|
||||
"confidence": "EXTRACTED", "source_file": str_path,
|
||||
"source_location": f"L{line_idx}", "weight": 1.0})
|
||||
current_window_nid = nid
|
||||
current_elem_nid = None
|
||||
current_elem_name = None
|
||||
continue
|
||||
m = _DMF_ELEM_RE.match(line)
|
||||
if m and current_window_nid is not None:
|
||||
name = m.group(1)
|
||||
nid = _make_id(stem, "elem", current_window_nid, name)
|
||||
if nid not in seen:
|
||||
seen.add(nid)
|
||||
nodes.append({"id": nid, "label": f'elem "{name}"', "file_type": "code",
|
||||
"source_file": str_path, "source_location": f"L{line_idx}"})
|
||||
edges.append({"source": current_window_nid, "target": nid,
|
||||
"relation": "contains", "confidence": "EXTRACTED",
|
||||
"source_file": str_path, "source_location": f"L{line_idx}",
|
||||
"weight": 1.0})
|
||||
current_elem_nid = nid
|
||||
current_elem_name = name
|
||||
continue
|
||||
m = _DMF_TYPE_RE.match(line)
|
||||
if m and current_elem_nid is not None and current_elem_name is not None:
|
||||
ctype = m.group(1)
|
||||
for n in nodes:
|
||||
if n["id"] == current_elem_nid and " [" not in n["label"]:
|
||||
n["label"] = f'elem "{current_elem_name}" [{ctype}]'
|
||||
break
|
||||
|
||||
return {"nodes": nodes, "edges": edges}
|
||||
@@ -0,0 +1,228 @@
|
||||
"""Elixir extractor. Moved verbatim from graphify/extract.py."""
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from graphify.extractors.base import _LANGUAGE_BUILTIN_GLOBALS, _file_stem, _make_id
|
||||
|
||||
|
||||
def extract_elixir(path: Path) -> dict:
|
||||
"""Extract modules, functions, imports, and calls from a .ex/.exs file."""
|
||||
try:
|
||||
import tree_sitter_elixir as tselixir
|
||||
from tree_sitter import Language, Parser
|
||||
except ImportError:
|
||||
return {"nodes": [], "edges": [], "error": "tree_sitter_elixir not installed"}
|
||||
|
||||
try:
|
||||
language = Language(tselixir.language())
|
||||
parser = Parser(language)
|
||||
source = path.read_bytes()
|
||||
tree = parser.parse(source)
|
||||
root = tree.root_node
|
||||
except Exception as e:
|
||||
return {"nodes": [], "edges": [], "error": str(e)}
|
||||
|
||||
stem = _file_stem(path)
|
||||
str_path = str(path)
|
||||
nodes: list[dict] = []
|
||||
edges: list[dict] = []
|
||||
seen_ids: set[str] = set()
|
||||
function_bodies: list[tuple[str, Any]] = []
|
||||
|
||||
def add_node(nid: str, label: str, line: int) -> None:
|
||||
if nid not in seen_ids:
|
||||
seen_ids.add(nid)
|
||||
nodes.append({"id": nid, "label": label, "file_type": "code",
|
||||
"source_file": str_path, "source_location": f"L{line}"})
|
||||
|
||||
def add_edge(src: str, tgt: str, relation: str, line: int,
|
||||
confidence: str = "EXTRACTED", weight: float = 1.0,
|
||||
context: str | None = None) -> None:
|
||||
edge = {"source": src, "target": tgt, "relation": relation,
|
||||
"confidence": confidence, "source_file": str_path,
|
||||
"source_location": f"L{line}", "weight": weight}
|
||||
if context:
|
||||
edge["context"] = context
|
||||
edges.append(edge)
|
||||
|
||||
file_nid = _make_id(str(path))
|
||||
add_node(file_nid, path.name, 1)
|
||||
|
||||
_IMPORT_KEYWORDS = frozenset({"alias", "import", "require", "use"})
|
||||
|
||||
def _get_alias_text(node) -> str | None:
|
||||
for child in node.children:
|
||||
if child.type == "alias":
|
||||
return source[child.start_byte:child.end_byte].decode("utf-8", errors="replace")
|
||||
return None
|
||||
|
||||
def _get_alias_modules(node) -> list[str]:
|
||||
"""Every module named by an alias/import/require/use argument.
|
||||
|
||||
Handles the single form (``alias Foo.Bar`` -> ``["Foo.Bar"]``) and the
|
||||
multi-alias brace form (``alias Foo.{Bar, Baz}`` ->
|
||||
``["Foo.Bar", "Foo.Baz"]``), which the grammar represents as a ``dot``
|
||||
node holding the base alias and a trailing ``tuple`` of member aliases.
|
||||
"""
|
||||
def _text(n) -> str:
|
||||
return source[n.start_byte:n.end_byte].decode("utf-8", errors="replace")
|
||||
|
||||
for child in node.children:
|
||||
if child.type == "alias":
|
||||
return [_text(child)]
|
||||
if child.type == "dot":
|
||||
base = None
|
||||
tuple_node = None
|
||||
for sub in child.children:
|
||||
if sub.type == "alias" and base is None:
|
||||
base = _text(sub)
|
||||
elif sub.type == "tuple":
|
||||
tuple_node = sub
|
||||
if base and tuple_node is not None:
|
||||
members = [_text(m) for m in tuple_node.children if m.type == "alias"]
|
||||
if members:
|
||||
return [f"{base}.{m}" for m in members]
|
||||
return [_text(child)]
|
||||
return []
|
||||
|
||||
def walk(node, parent_module_nid: str | None = None) -> None:
|
||||
if node.type != "call":
|
||||
for child in node.children:
|
||||
walk(child, parent_module_nid)
|
||||
return
|
||||
|
||||
identifier_node = None
|
||||
arguments_node = None
|
||||
do_block_node = None
|
||||
for child in node.children:
|
||||
if child.type == "identifier":
|
||||
identifier_node = child
|
||||
elif child.type == "arguments":
|
||||
arguments_node = child
|
||||
elif child.type == "do_block":
|
||||
do_block_node = child
|
||||
|
||||
if identifier_node is None:
|
||||
for child in node.children:
|
||||
walk(child, parent_module_nid)
|
||||
return
|
||||
|
||||
keyword = source[identifier_node.start_byte:identifier_node.end_byte].decode("utf-8", errors="replace")
|
||||
line = node.start_point[0] + 1
|
||||
|
||||
if keyword == "defmodule":
|
||||
module_name = _get_alias_text(arguments_node) if arguments_node else None
|
||||
if not module_name:
|
||||
return
|
||||
module_nid = _make_id(stem, module_name)
|
||||
add_node(module_nid, module_name, line)
|
||||
add_edge(file_nid, module_nid, "contains", line)
|
||||
if do_block_node:
|
||||
for child in do_block_node.children:
|
||||
walk(child, parent_module_nid=module_nid)
|
||||
return
|
||||
|
||||
if keyword in ("def", "defp"):
|
||||
func_name = None
|
||||
if arguments_node:
|
||||
for child in arguments_node.children:
|
||||
if child.type == "call":
|
||||
for sub in child.children:
|
||||
if sub.type == "identifier":
|
||||
func_name = source[sub.start_byte:sub.end_byte].decode("utf-8", errors="replace")
|
||||
break
|
||||
elif child.type == "identifier":
|
||||
func_name = source[child.start_byte:child.end_byte].decode("utf-8", errors="replace")
|
||||
break
|
||||
if not func_name:
|
||||
return
|
||||
container = parent_module_nid or file_nid
|
||||
func_nid = _make_id(container, func_name)
|
||||
add_node(func_nid, f"{func_name}()", line)
|
||||
if parent_module_nid:
|
||||
add_edge(parent_module_nid, func_nid, "method", line)
|
||||
else:
|
||||
add_edge(file_nid, func_nid, "contains", line)
|
||||
if do_block_node:
|
||||
function_bodies.append((func_nid, do_block_node))
|
||||
return
|
||||
|
||||
if keyword in _IMPORT_KEYWORDS and arguments_node:
|
||||
for module_name in _get_alias_modules(arguments_node):
|
||||
tgt_nid = _make_id(module_name)
|
||||
add_edge(file_nid, tgt_nid, "imports", line, context="import")
|
||||
return
|
||||
|
||||
for child in node.children:
|
||||
walk(child, parent_module_nid)
|
||||
|
||||
walk(root)
|
||||
|
||||
label_to_nid: dict[str, str] = {}
|
||||
for n in nodes:
|
||||
normalised = n["label"].strip("()").lstrip(".")
|
||||
label_to_nid[normalised] = n["id"]
|
||||
|
||||
seen_call_pairs: set[tuple[str, str]] = set()
|
||||
raw_calls: list[dict] = []
|
||||
_SKIP_KEYWORDS = frozenset({
|
||||
"def", "defp", "defmodule", "defmacro", "defmacrop",
|
||||
"defstruct", "defprotocol", "defimpl", "defguard",
|
||||
"alias", "import", "require", "use",
|
||||
"if", "unless", "case", "cond", "with", "for",
|
||||
})
|
||||
|
||||
def walk_calls(node, caller_nid: str) -> None:
|
||||
if node.type != "call":
|
||||
for child in node.children:
|
||||
walk_calls(child, caller_nid)
|
||||
return
|
||||
for child in node.children:
|
||||
if child.type == "identifier":
|
||||
kw = source[child.start_byte:child.end_byte].decode("utf-8", errors="replace")
|
||||
if kw in _SKIP_KEYWORDS:
|
||||
for c in node.children:
|
||||
walk_calls(c, caller_nid)
|
||||
return
|
||||
break
|
||||
callee_name: str | None = None
|
||||
is_member_call: bool = False
|
||||
for child in node.children:
|
||||
if child.type == "dot":
|
||||
is_member_call = True
|
||||
dot_text = source[child.start_byte:child.end_byte].decode("utf-8", errors="replace")
|
||||
parts = dot_text.rstrip(".").split(".")
|
||||
if parts:
|
||||
callee_name = parts[-1]
|
||||
break
|
||||
if child.type == "identifier":
|
||||
callee_name = source[child.start_byte:child.end_byte].decode("utf-8", errors="replace")
|
||||
break
|
||||
if callee_name and callee_name not in _LANGUAGE_BUILTIN_GLOBALS:
|
||||
tgt_nid = label_to_nid.get(callee_name)
|
||||
if tgt_nid and tgt_nid != caller_nid:
|
||||
pair = (caller_nid, tgt_nid)
|
||||
if pair not in seen_call_pairs:
|
||||
seen_call_pairs.add(pair)
|
||||
add_edge(caller_nid, tgt_nid, "calls",
|
||||
node.start_point[0] + 1, confidence="EXTRACTED", weight=1.0,
|
||||
context="call")
|
||||
else:
|
||||
raw_calls.append({
|
||||
"caller_nid": caller_nid,
|
||||
"callee": callee_name,
|
||||
"is_member_call": is_member_call,
|
||||
"source_file": str_path,
|
||||
"source_location": f"L{node.start_point[0] + 1}",
|
||||
})
|
||||
for child in node.children:
|
||||
walk_calls(child, caller_nid)
|
||||
|
||||
for caller_nid, body in function_bodies:
|
||||
walk_calls(body, caller_nid)
|
||||
|
||||
clean_edges = [e for e in edges if e["source"] in seen_ids and
|
||||
(e["target"] in seen_ids or e["relation"] == "imports")]
|
||||
return {"nodes": nodes, "edges": clean_edges, "raw_calls": raw_calls, "input_tokens": 0, "output_tokens": 0}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,309 @@
|
||||
"""Fortran extractor. Moved verbatim from graphify/extract.py."""
|
||||
from __future__ import annotations
|
||||
|
||||
|
||||
from pathlib import Path
|
||||
from graphify.extractors.base import _file_stem, _make_id, _read_text
|
||||
|
||||
|
||||
_FORTRAN_CPP_EXTS = {".F", ".F90", ".F95", ".F03", ".F08"}
|
||||
|
||||
def _cpp_preprocess(path: Path) -> bytes:
|
||||
"""Run cpp -w -P on a capital-F Fortran file and return preprocessed bytes.
|
||||
|
||||
Falls back to raw file bytes if cpp is not available. Capital-F extensions
|
||||
conventionally require C preprocessor expansion (#ifdef MPI, #define REAL8, etc.)
|
||||
before parsing.
|
||||
|
||||
Security (F-007): we pass `-nostdinc` and `-I /dev/null` so a malicious
|
||||
source file containing `#include "/home/victim/.ssh/id_rsa"` (or any other
|
||||
include directive) cannot inline arbitrary host files into the output that
|
||||
we then ship to an LLM. Without these flags `cpp` happily resolves any
|
||||
relative or absolute include path it can read, which is a corpus-side
|
||||
file-exfiltration vector.
|
||||
"""
|
||||
import shutil
|
||||
import subprocess
|
||||
if not shutil.which("cpp"):
|
||||
return path.read_bytes()
|
||||
try:
|
||||
# Pass an absolute path so a corpus file named like "-I/etc/x.F90" cannot
|
||||
# be parsed by cpp as an option (cpp does not accept a "--" end-of-options
|
||||
# terminator). An absolute path always begins with "/".
|
||||
result = subprocess.run(
|
||||
["cpp", "-w", "-P", "-nostdinc", "-I", "/dev/null", str(path.resolve())],
|
||||
capture_output=True,
|
||||
timeout=30,
|
||||
)
|
||||
if result.returncode == 0 and result.stdout:
|
||||
return result.stdout
|
||||
except Exception:
|
||||
pass
|
||||
return path.read_bytes()
|
||||
|
||||
def extract_fortran(path: Path) -> dict:
|
||||
"""Extract programs, modules, subroutines, functions, use statements, and calls from Fortran files.
|
||||
|
||||
Capital-F extensions (.F, .F90, etc.) are run through the C preprocessor before
|
||||
parsing so #ifdef/#define macros are resolved.
|
||||
"""
|
||||
try:
|
||||
import tree_sitter_fortran as tsfortran
|
||||
from tree_sitter import Language, Parser
|
||||
except ImportError:
|
||||
return {"nodes": [], "edges": [], "error": "tree-sitter-fortran not installed"}
|
||||
|
||||
try:
|
||||
language = Language(tsfortran.language())
|
||||
parser = Parser(language)
|
||||
source = _cpp_preprocess(path) if path.suffix in _FORTRAN_CPP_EXTS else path.read_bytes()
|
||||
tree = parser.parse(source)
|
||||
root = tree.root_node
|
||||
except Exception as e:
|
||||
return {"nodes": [], "edges": [], "error": str(e)}
|
||||
|
||||
stem = _file_stem(path)
|
||||
str_path = str(path)
|
||||
nodes: list[dict] = []
|
||||
edges: list[dict] = []
|
||||
seen_ids: set[str] = set()
|
||||
scope_bodies: list[tuple[str, object]] = []
|
||||
|
||||
def add_node(nid: str, label: str, line: int) -> None:
|
||||
if nid not in seen_ids:
|
||||
seen_ids.add(nid)
|
||||
nodes.append({
|
||||
"id": nid,
|
||||
"label": label,
|
||||
"file_type": "code",
|
||||
"source_file": str_path,
|
||||
"source_location": f"L{line}",
|
||||
})
|
||||
|
||||
def add_edge(src: str, tgt: str, relation: str, line: int,
|
||||
confidence: str = "EXTRACTED", weight: float = 1.0,
|
||||
context: str | None = None) -> None:
|
||||
edge = {
|
||||
"source": src,
|
||||
"target": tgt,
|
||||
"relation": relation,
|
||||
"confidence": confidence,
|
||||
"source_file": str_path,
|
||||
"source_location": f"L{line}",
|
||||
"weight": weight,
|
||||
}
|
||||
if context:
|
||||
edge["context"] = context
|
||||
edges.append(edge)
|
||||
|
||||
file_nid = _make_id(str(path))
|
||||
add_node(file_nid, path.name, 1)
|
||||
|
||||
def _fortran_name(stmt_node) -> str | None:
|
||||
"""Extract name from a *_statement node. Fortran is case-insensitive; lowercase."""
|
||||
for child in stmt_node.children:
|
||||
if child.type in ("name", "identifier"):
|
||||
return _read_text(child, source).lower()
|
||||
return None
|
||||
|
||||
def ensure_named_node(name: str, line: int) -> str:
|
||||
nid = _make_id(stem, name)
|
||||
if nid in seen_ids:
|
||||
return nid
|
||||
nid = _make_id(name)
|
||||
if nid not in seen_ids:
|
||||
# The name isn't defined in this file, so this is a cross-file reference
|
||||
# (e.g. a `Thing` type annotation imported from another module). Emit a
|
||||
# SOURCELESS stub — like the inheritance-base path below — so the
|
||||
# corpus-level rewire can collapse it onto the real definition. A sourced
|
||||
# stub here makes _disambiguate_colliding_node_ids bake the referencing
|
||||
# file's path (with extension) into the id and blocks the rewire, which is
|
||||
# the phantom-duplicate-node bug (#1402).
|
||||
seen_ids.add(nid)
|
||||
nodes.append({
|
||||
"id": nid,
|
||||
"label": name,
|
||||
"file_type": "code",
|
||||
"source_file": "",
|
||||
"source_location": "",
|
||||
"origin_file": str_path,
|
||||
})
|
||||
return nid
|
||||
|
||||
def emit_signature_refs(scope_node, fn_nid: str, is_function: bool) -> None:
|
||||
"""Emit references[parameter_type] / references[return_type] edges for
|
||||
a subroutine/function based on its variable_declaration siblings."""
|
||||
stmt_type = "function_statement" if is_function else "subroutine_statement"
|
||||
stmt = next((c for c in scope_node.children if c.type == stmt_type), None)
|
||||
if stmt is None:
|
||||
return
|
||||
param_names: set[str] = set()
|
||||
params_node = next((c for c in stmt.children if c.type == "parameters"), None)
|
||||
if params_node is not None:
|
||||
for c in params_node.children:
|
||||
if c.type == "identifier":
|
||||
param_names.add(_read_text(c, source).lower())
|
||||
result_name: str | None = None
|
||||
if is_function:
|
||||
result_node = next((c for c in stmt.children if c.type == "function_result"), None)
|
||||
if result_node is not None:
|
||||
res_id = next((c for c in result_node.children if c.type == "identifier"), None)
|
||||
if res_id is not None:
|
||||
result_name = _read_text(res_id, source).lower()
|
||||
else:
|
||||
# implicit result variable: same name as the function
|
||||
result_name = _fortran_name(stmt)
|
||||
for child in scope_node.children:
|
||||
if child.type != "variable_declaration":
|
||||
continue
|
||||
derived = next((c for c in child.children if c.type == "derived_type"), None)
|
||||
if derived is None:
|
||||
continue
|
||||
type_name_node = next((c for c in derived.children if c.type == "type_name"), None)
|
||||
if type_name_node is None:
|
||||
continue
|
||||
type_name = _read_text(type_name_node, source).lower()
|
||||
for var in child.children:
|
||||
if var.type != "identifier":
|
||||
continue
|
||||
var_name = _read_text(var, source).lower()
|
||||
var_line = var.start_point[0] + 1
|
||||
if var_name in param_names:
|
||||
tgt = ensure_named_node(type_name, var_line)
|
||||
if tgt != fn_nid:
|
||||
add_edge(fn_nid, tgt, "references", var_line, context="parameter_type")
|
||||
elif is_function and var_name == result_name:
|
||||
tgt = ensure_named_node(type_name, var_line)
|
||||
if tgt != fn_nid:
|
||||
add_edge(fn_nid, tgt, "references", var_line, context="return_type")
|
||||
|
||||
def walk_calls(node, scope_nid: str) -> None:
|
||||
if node is None:
|
||||
return
|
||||
t = node.type
|
||||
if t in ("subroutine", "function", "module", "program", "internal_procedures"):
|
||||
return
|
||||
# call FOO(args) — tree-sitter-fortran uses subroutine_call
|
||||
if t == "subroutine_call":
|
||||
name_node = next((c for c in node.children if c.type == "identifier"), None)
|
||||
if name_node:
|
||||
callee = _read_text(name_node, source).lower()
|
||||
target_nid = _make_id(stem, callee)
|
||||
add_edge(scope_nid, target_nid, "calls", node.start_point[0] + 1,
|
||||
confidence="EXTRACTED", context="call")
|
||||
# x = compute(args) — function invocations are `call_expression`, which
|
||||
# shares Fortran's `name(...)` syntax with array indexing. Only emit a
|
||||
# call edge when the callee resolves to a procedure defined in this file
|
||||
# (an array variable produces no matching node), so array accesses can't
|
||||
# fabricate spurious `calls` edges.
|
||||
elif t == "call_expression":
|
||||
name_node = next((c for c in node.children if c.type == "identifier"), None)
|
||||
if name_node:
|
||||
callee = _read_text(name_node, source).lower()
|
||||
target_nid = _make_id(stem, callee)
|
||||
if target_nid in seen_ids and target_nid != scope_nid:
|
||||
add_edge(scope_nid, target_nid, "calls", node.start_point[0] + 1,
|
||||
confidence="EXTRACTED", context="call")
|
||||
for child in node.children:
|
||||
walk_calls(child, scope_nid)
|
||||
|
||||
def walk(node, scope_nid: str) -> None:
|
||||
t = node.type
|
||||
|
||||
if t == "program":
|
||||
stmt = next((c for c in node.children if c.type == "program_statement"), None)
|
||||
name = _fortran_name(stmt) if stmt else None
|
||||
if name:
|
||||
nid = _make_id(stem, name)
|
||||
line = node.start_point[0] + 1
|
||||
add_node(nid, name, line)
|
||||
add_edge(file_nid, nid, "defines", line)
|
||||
scope_bodies.append((nid, node))
|
||||
for child in node.children:
|
||||
walk(child, nid)
|
||||
return
|
||||
|
||||
if t == "module":
|
||||
stmt = next((c for c in node.children if c.type == "module_statement"), None)
|
||||
name = _fortran_name(stmt) if stmt else None
|
||||
if name:
|
||||
nid = _make_id(stem, name)
|
||||
line = node.start_point[0] + 1
|
||||
add_node(nid, name, line)
|
||||
add_edge(file_nid, nid, "defines", line)
|
||||
for child in node.children:
|
||||
walk(child, nid)
|
||||
return
|
||||
|
||||
# subroutines/functions inside a module live under internal_procedures
|
||||
if t == "internal_procedures":
|
||||
for child in node.children:
|
||||
walk(child, scope_nid)
|
||||
return
|
||||
|
||||
if t == "derived_type_definition":
|
||||
stmt = next((c for c in node.children if c.type == "derived_type_statement"), None)
|
||||
if stmt is not None:
|
||||
name_node = next((c for c in stmt.children if c.type == "type_name"), None)
|
||||
if name_node is not None:
|
||||
type_name = _read_text(name_node, source).lower()
|
||||
type_nid = _make_id(stem, type_name)
|
||||
line = node.start_point[0] + 1
|
||||
add_node(type_nid, type_name, line)
|
||||
add_edge(scope_nid, type_nid, "defines", line)
|
||||
return
|
||||
|
||||
if t == "subroutine":
|
||||
stmt = next((c for c in node.children if c.type == "subroutine_statement"), None)
|
||||
name = _fortran_name(stmt) if stmt else None
|
||||
if name:
|
||||
nid = _make_id(stem, name)
|
||||
line = node.start_point[0] + 1
|
||||
add_node(nid, f"{name}()", line)
|
||||
add_edge(scope_nid, nid, "defines", line)
|
||||
scope_bodies.append((nid, node))
|
||||
emit_signature_refs(node, nid, is_function=False)
|
||||
for child in node.children:
|
||||
walk(child, nid)
|
||||
return
|
||||
|
||||
if t == "function":
|
||||
stmt = next((c for c in node.children if c.type == "function_statement"), None)
|
||||
name = _fortran_name(stmt) if stmt else None
|
||||
if name:
|
||||
nid = _make_id(stem, name)
|
||||
line = node.start_point[0] + 1
|
||||
add_node(nid, f"{name}()", line)
|
||||
add_edge(scope_nid, nid, "defines", line)
|
||||
scope_bodies.append((nid, node))
|
||||
emit_signature_refs(node, nid, is_function=True)
|
||||
for child in node.children:
|
||||
walk(child, nid)
|
||||
return
|
||||
|
||||
if t == "use_statement":
|
||||
line = node.start_point[0] + 1
|
||||
# tree-sitter-fortran uses module_name node for the used module
|
||||
name_node = next((c for c in node.children if c.type in ("module_name", "name", "identifier")), None)
|
||||
if name_node:
|
||||
mod_name = _read_text(name_node, source).lower()
|
||||
imp_nid = _make_id(mod_name)
|
||||
add_node(imp_nid, mod_name, line)
|
||||
add_edge(scope_nid, imp_nid, "imports", line, context="use")
|
||||
return
|
||||
|
||||
for child in node.children:
|
||||
walk(child, scope_nid)
|
||||
|
||||
walk(root, file_nid)
|
||||
|
||||
_stmt_headers = {
|
||||
"subroutine_statement", "function_statement",
|
||||
"program_statement", "module_statement",
|
||||
}
|
||||
for scope_nid, body_node in scope_bodies:
|
||||
for child in body_node.children:
|
||||
if child.type not in _stmt_headers:
|
||||
walk_calls(child, scope_nid)
|
||||
|
||||
return {"nodes": nodes, "edges": edges}
|
||||
@@ -0,0 +1,396 @@
|
||||
"""Go extractor. Moved verbatim from graphify/extract.py."""
|
||||
from __future__ import annotations
|
||||
|
||||
|
||||
from pathlib import Path
|
||||
from graphify.extractors.base import _LANGUAGE_BUILTIN_GLOBALS, _file_stem, _make_id, _read_text
|
||||
|
||||
|
||||
_GO_PREDECLARED_TYPES = frozenset({
|
||||
"bool", "byte", "complex64", "complex128", "error", "float32", "float64",
|
||||
"int", "int8", "int16", "int32", "int64", "rune", "string",
|
||||
"uint", "uint8", "uint16", "uint32", "uint64", "uintptr", "any", "comparable",
|
||||
})
|
||||
|
||||
def _go_collect_type_refs(node, source: bytes, generic: bool, out: list[tuple[str, str]]) -> None:
|
||||
"""Walk a Go type expression; append (name, role) tuples."""
|
||||
if node is None:
|
||||
return
|
||||
t = node.type
|
||||
if t == "type_identifier":
|
||||
text = _read_text(node, source)
|
||||
if text and text not in _GO_PREDECLARED_TYPES:
|
||||
out.append((text, "generic_arg" if generic else "type"))
|
||||
return
|
||||
if t == "qualified_type":
|
||||
text = _read_text(node, source).rsplit(".", 1)[-1]
|
||||
if text and text not in _GO_PREDECLARED_TYPES:
|
||||
out.append((text, "generic_arg" if generic else "type"))
|
||||
return
|
||||
if t == "generic_type":
|
||||
type_field = node.child_by_field_name("type")
|
||||
if type_field is not None:
|
||||
sub: list[tuple[str, str]] = []
|
||||
_go_collect_type_refs(type_field, source, generic, sub)
|
||||
out.extend(sub)
|
||||
for c in node.children:
|
||||
if c.type == "type_arguments":
|
||||
for arg in c.children:
|
||||
if arg.is_named:
|
||||
_go_collect_type_refs(arg, source, True, out)
|
||||
return
|
||||
if t in ("pointer_type", "slice_type", "array_type", "map_type",
|
||||
"channel_type", "parenthesized_type"):
|
||||
for c in node.children:
|
||||
if c.is_named:
|
||||
_go_collect_type_refs(c, source, generic, out)
|
||||
return
|
||||
if node.is_named:
|
||||
for c in node.children:
|
||||
if c.is_named:
|
||||
_go_collect_type_refs(c, source, generic, out)
|
||||
|
||||
def extract_go(path: Path) -> dict:
|
||||
"""Extract functions, methods, type declarations, and imports from a .go file."""
|
||||
try:
|
||||
import tree_sitter_go as tsgo
|
||||
from tree_sitter import Language, Parser
|
||||
except ImportError:
|
||||
return {"nodes": [], "edges": [], "error": "tree-sitter-go not installed"}
|
||||
|
||||
try:
|
||||
language = Language(tsgo.language())
|
||||
parser = Parser(language)
|
||||
source = path.read_bytes()
|
||||
tree = parser.parse(source)
|
||||
root = tree.root_node
|
||||
except Exception as e:
|
||||
return {"nodes": [], "edges": [], "error": str(e)}
|
||||
|
||||
stem = _file_stem(path)
|
||||
# Use directory name as package scope so methods on the same type across
|
||||
# multiple files in a package share one canonical type node.
|
||||
pkg_scope = path.parent.name or stem
|
||||
str_path = str(path)
|
||||
nodes: list[dict] = []
|
||||
edges: list[dict] = []
|
||||
seen_ids: set[str] = set()
|
||||
function_bodies: list[tuple[str, object]] = []
|
||||
go_imported_pkgs: set[str] = set() # local names of imported packages
|
||||
|
||||
def add_node(nid: str, label: str, line: int) -> None:
|
||||
if nid not in seen_ids:
|
||||
seen_ids.add(nid)
|
||||
nodes.append({
|
||||
"id": nid,
|
||||
"label": label,
|
||||
"file_type": "code",
|
||||
"source_file": str_path,
|
||||
"source_location": f"L{line}",
|
||||
})
|
||||
|
||||
def add_edge(src: str, tgt: str, relation: str, line: int,
|
||||
confidence: str = "EXTRACTED", weight: float = 1.0,
|
||||
context: str | None = None) -> None:
|
||||
edge = {
|
||||
"source": src,
|
||||
"target": tgt,
|
||||
"relation": relation,
|
||||
"confidence": confidence,
|
||||
"source_file": str_path,
|
||||
"source_location": f"L{line}",
|
||||
"weight": weight,
|
||||
}
|
||||
if context:
|
||||
edge["context"] = context
|
||||
edges.append(edge)
|
||||
|
||||
file_nid = _make_id(str(path))
|
||||
add_node(file_nid, path.name, 1)
|
||||
|
||||
def ensure_named_node(name: str, line: int) -> str:
|
||||
nid = _make_id(pkg_scope, name)
|
||||
if nid in seen_ids:
|
||||
return nid
|
||||
nid = _make_id(name)
|
||||
if nid not in seen_ids:
|
||||
# The name isn't declared in this file, so this is a cross-file reference
|
||||
# (e.g. a type defined in another file of the package). Emit a SOURCELESS
|
||||
# stub — like the inheritance-base path in the other extractors — so the
|
||||
# corpus-level rewire can collapse it onto the real definition. A sourced
|
||||
# stub here makes _disambiguate_colliding_node_ids bake the referencing
|
||||
# file's path (with extension) into the id and blocks the rewire, which is
|
||||
# the phantom-duplicate-node bug (#1402).
|
||||
seen_ids.add(nid)
|
||||
nodes.append({
|
||||
"id": nid,
|
||||
"label": name,
|
||||
"file_type": "code",
|
||||
"source_file": "",
|
||||
"source_location": "",
|
||||
"origin_file": str_path,
|
||||
})
|
||||
return nid
|
||||
|
||||
def emit_go_method_refs(func_node, func_nid: str, line: int) -> None:
|
||||
params = func_node.child_by_field_name("parameters")
|
||||
if params is not None:
|
||||
for p in params.children:
|
||||
if p.type != "parameter_declaration":
|
||||
continue
|
||||
type_node = p.child_by_field_name("type")
|
||||
refs: list[tuple[str, str]] = []
|
||||
_go_collect_type_refs(type_node, source, False, refs)
|
||||
for ref_name, role in refs:
|
||||
ctx = "generic_arg" if role == "generic_arg" else "parameter_type"
|
||||
tgt = ensure_named_node(ref_name, line)
|
||||
if tgt != func_nid:
|
||||
add_edge(func_nid, tgt, "references", line, context=ctx)
|
||||
result = func_node.child_by_field_name("result")
|
||||
if result is not None:
|
||||
if result.type == "parameter_list":
|
||||
for p in result.children:
|
||||
if p.type != "parameter_declaration":
|
||||
continue
|
||||
type_node = p.child_by_field_name("type")
|
||||
if type_node is None:
|
||||
for c in p.children:
|
||||
if c.is_named:
|
||||
type_node = c
|
||||
break
|
||||
refs = []
|
||||
_go_collect_type_refs(type_node, source, False, refs)
|
||||
for ref_name, role in refs:
|
||||
ctx = "generic_arg" if role == "generic_arg" else "return_type"
|
||||
tgt = ensure_named_node(ref_name, line)
|
||||
if tgt != func_nid:
|
||||
add_edge(func_nid, tgt, "references", line, context=ctx)
|
||||
else:
|
||||
refs = []
|
||||
_go_collect_type_refs(result, source, False, refs)
|
||||
for ref_name, role in refs:
|
||||
ctx = "generic_arg" if role == "generic_arg" else "return_type"
|
||||
tgt = ensure_named_node(ref_name, line)
|
||||
if tgt != func_nid:
|
||||
add_edge(func_nid, tgt, "references", line, context=ctx)
|
||||
|
||||
def walk(node) -> None:
|
||||
t = node.type
|
||||
|
||||
if t == "function_declaration":
|
||||
name_node = node.child_by_field_name("name")
|
||||
if name_node:
|
||||
func_name = _read_text(name_node, source)
|
||||
line = node.start_point[0] + 1
|
||||
func_nid = _make_id(stem, func_name)
|
||||
add_node(func_nid, f"{func_name}()", line)
|
||||
add_edge(file_nid, func_nid, "contains", line)
|
||||
emit_go_method_refs(node, func_nid, line)
|
||||
body = node.child_by_field_name("body")
|
||||
if body:
|
||||
function_bodies.append((func_nid, body))
|
||||
return
|
||||
|
||||
if t == "method_declaration":
|
||||
receiver = node.child_by_field_name("receiver")
|
||||
receiver_type: str | None = None
|
||||
if receiver:
|
||||
for param in receiver.children:
|
||||
if param.type == "parameter_declaration":
|
||||
type_node = param.child_by_field_name("type")
|
||||
if type_node:
|
||||
receiver_type = _read_text(type_node, source).lstrip("*").strip()
|
||||
break
|
||||
name_node = node.child_by_field_name("name")
|
||||
if not name_node:
|
||||
return
|
||||
method_name = _read_text(name_node, source)
|
||||
line = node.start_point[0] + 1
|
||||
|
||||
if receiver_type:
|
||||
parent_nid = _make_id(pkg_scope, receiver_type)
|
||||
add_node(parent_nid, receiver_type, line)
|
||||
method_nid = _make_id(parent_nid, method_name)
|
||||
add_node(method_nid, f".{method_name}()", line)
|
||||
add_edge(parent_nid, method_nid, "method", line)
|
||||
else:
|
||||
method_nid = _make_id(stem, method_name)
|
||||
add_node(method_nid, f"{method_name}()", line)
|
||||
add_edge(file_nid, method_nid, "contains", line)
|
||||
|
||||
emit_go_method_refs(node, method_nid, line)
|
||||
body = node.child_by_field_name("body")
|
||||
if body:
|
||||
function_bodies.append((method_nid, body))
|
||||
return
|
||||
|
||||
if t == "type_declaration":
|
||||
for child in node.children:
|
||||
if child.type != "type_spec":
|
||||
continue
|
||||
name_node = child.child_by_field_name("name")
|
||||
if not name_node:
|
||||
continue
|
||||
type_name = _read_text(name_node, source)
|
||||
line = child.start_point[0] + 1
|
||||
type_nid = _make_id(pkg_scope, type_name)
|
||||
add_node(type_nid, type_name, line)
|
||||
add_edge(file_nid, type_nid, "contains", line)
|
||||
# Type body: struct fields (with embeds) or interface embedding.
|
||||
type_body = None
|
||||
for tc in child.children:
|
||||
if tc.type in ("struct_type", "interface_type"):
|
||||
type_body = tc
|
||||
break
|
||||
if type_body is None:
|
||||
continue
|
||||
if type_body.type == "struct_type":
|
||||
for fdl in type_body.children:
|
||||
if fdl.type != "field_declaration_list":
|
||||
continue
|
||||
for field in fdl.children:
|
||||
if field.type != "field_declaration":
|
||||
continue
|
||||
has_name = any(
|
||||
fc.type == "field_identifier" for fc in field.children
|
||||
)
|
||||
type_node = field.child_by_field_name("type")
|
||||
if type_node is None:
|
||||
for fc in field.children:
|
||||
if fc.is_named and fc.type != "field_identifier":
|
||||
type_node = fc
|
||||
break
|
||||
refs: list[tuple[str, str]] = []
|
||||
_go_collect_type_refs(type_node, source, False, refs)
|
||||
for ref_name, role in refs:
|
||||
tgt = ensure_named_node(ref_name, field.start_point[0] + 1)
|
||||
if tgt == type_nid:
|
||||
continue
|
||||
if not has_name and role == "type":
|
||||
add_edge(type_nid, tgt, "embeds",
|
||||
field.start_point[0] + 1)
|
||||
else:
|
||||
ctx = "generic_arg" if role == "generic_arg" else "field"
|
||||
add_edge(type_nid, tgt, "references",
|
||||
field.start_point[0] + 1, context=ctx)
|
||||
elif type_body.type == "interface_type":
|
||||
for elem in type_body.children:
|
||||
if elem.type != "type_elem":
|
||||
continue
|
||||
refs = []
|
||||
for sub in elem.children:
|
||||
if sub.is_named:
|
||||
_go_collect_type_refs(sub, source, False, refs)
|
||||
for ref_name, role in refs:
|
||||
tgt = ensure_named_node(ref_name, elem.start_point[0] + 1)
|
||||
if tgt == type_nid:
|
||||
continue
|
||||
if role == "type":
|
||||
add_edge(type_nid, tgt, "embeds",
|
||||
elem.start_point[0] + 1)
|
||||
else:
|
||||
add_edge(type_nid, tgt, "references",
|
||||
elem.start_point[0] + 1, context="generic_arg")
|
||||
return
|
||||
|
||||
if t == "import_declaration":
|
||||
for child in node.children:
|
||||
if child.type == "import_spec_list":
|
||||
for spec in child.children:
|
||||
if spec.type == "import_spec":
|
||||
path_node = spec.child_by_field_name("path")
|
||||
if path_node:
|
||||
raw = _read_text(path_node, source).strip('"')
|
||||
# Prefix with go_pkg_ so stdlib names (e.g. "context")
|
||||
# don't collide with local files of the same basename.
|
||||
tgt_nid = _make_id("go", "pkg", raw)
|
||||
add_edge(file_nid, tgt_nid, "imports_from", spec.start_point[0] + 1, context="import")
|
||||
# Track local name (alias or last path segment)
|
||||
alias = spec.child_by_field_name("name")
|
||||
local_name = _read_text(alias, source) if alias else raw.split("/")[-1]
|
||||
if local_name and local_name != "_" and local_name != ".":
|
||||
go_imported_pkgs.add(local_name)
|
||||
elif child.type == "import_spec":
|
||||
path_node = child.child_by_field_name("path")
|
||||
if path_node:
|
||||
raw = _read_text(path_node, source).strip('"')
|
||||
tgt_nid = _make_id("go", "pkg", raw)
|
||||
add_edge(file_nid, tgt_nid, "imports_from", child.start_point[0] + 1, context="import")
|
||||
alias = child.child_by_field_name("name")
|
||||
local_name = _read_text(alias, source) if alias else raw.split("/")[-1]
|
||||
if local_name and local_name != "_" and local_name != ".":
|
||||
go_imported_pkgs.add(local_name)
|
||||
return
|
||||
|
||||
for child in node.children:
|
||||
walk(child)
|
||||
|
||||
walk(root)
|
||||
|
||||
label_to_nid: dict[str, str] = {}
|
||||
for n in nodes:
|
||||
raw = n["label"]
|
||||
normalised = raw.strip("()").lstrip(".")
|
||||
label_to_nid[normalised] = n["id"]
|
||||
|
||||
seen_call_pairs: set[tuple[str, str]] = set()
|
||||
raw_calls: list[dict] = []
|
||||
|
||||
def walk_calls(node, caller_nid: str) -> None:
|
||||
if node.type in ("function_declaration", "method_declaration"):
|
||||
return
|
||||
if node.type == "call_expression":
|
||||
func_node = node.child_by_field_name("function")
|
||||
callee_name: str | None = None
|
||||
is_member_call: bool = False
|
||||
if func_node:
|
||||
if func_node.type == "identifier":
|
||||
callee_name = _read_text(func_node, source)
|
||||
elif func_node.type == "selector_expression":
|
||||
field = func_node.child_by_field_name("field")
|
||||
operand = func_node.child_by_field_name("operand")
|
||||
receiver_name = _read_text(operand, source) if operand else ""
|
||||
# Package-qualified call (e.g. fmt.Println) → allow cross-file resolution.
|
||||
# Receiver method call (e.g. s.logger.Log) → skip, no import evidence.
|
||||
is_member_call = receiver_name not in go_imported_pkgs
|
||||
if field:
|
||||
callee_name = _read_text(field, source)
|
||||
if callee_name and callee_name not in _LANGUAGE_BUILTIN_GLOBALS:
|
||||
tgt_nid = label_to_nid.get(callee_name)
|
||||
if tgt_nid and tgt_nid != caller_nid:
|
||||
pair = (caller_nid, tgt_nid)
|
||||
if pair not in seen_call_pairs:
|
||||
seen_call_pairs.add(pair)
|
||||
line = node.start_point[0] + 1
|
||||
edges.append({
|
||||
"source": caller_nid,
|
||||
"target": tgt_nid,
|
||||
"relation": "calls",
|
||||
"context": "call",
|
||||
"confidence": "EXTRACTED",
|
||||
"source_file": str_path,
|
||||
"source_location": f"L{line}",
|
||||
"weight": 1.0,
|
||||
})
|
||||
elif callee_name:
|
||||
raw_calls.append({
|
||||
"caller_nid": caller_nid,
|
||||
"callee": callee_name,
|
||||
"is_member_call": is_member_call,
|
||||
"source_file": str_path,
|
||||
"source_location": f"L{node.start_point[0] + 1}",
|
||||
})
|
||||
for child in node.children:
|
||||
walk_calls(child, caller_nid)
|
||||
|
||||
for caller_nid, body_node in function_bodies:
|
||||
walk_calls(body_node, caller_nid)
|
||||
|
||||
valid_ids = seen_ids
|
||||
clean_edges = []
|
||||
for edge in edges:
|
||||
src, tgt = edge["source"], edge["target"]
|
||||
if src in valid_ids and (tgt in valid_ids or edge["relation"] in ("imports", "imports_from")):
|
||||
clean_edges.append(edge)
|
||||
|
||||
return {"nodes": nodes, "edges": clean_edges, "raw_calls": raw_calls}
|
||||
@@ -0,0 +1,209 @@
|
||||
"""Json_config extractor. Moved verbatim from graphify/extract.py."""
|
||||
from __future__ import annotations
|
||||
|
||||
|
||||
from pathlib import Path
|
||||
from graphify.extractors.base import _file_stem, _make_id, _read_text
|
||||
|
||||
|
||||
_CONFIG_JSON_NAMES = frozenset({
|
||||
"package.json", "tsconfig.json", "jsconfig.json", "composer.json",
|
||||
"deno.json", "deno.jsonc", "bower.json", "manifest.json",
|
||||
"app.json", "now.json", "vercel.json", "angular.json", "nest-cli.json",
|
||||
"biome.json", "biome.jsonc", "renovate.json", ".babelrc", ".babelrc.json",
|
||||
".eslintrc.json", ".prettierrc.json", ".prettierrc", "babel.config.json",
|
||||
})
|
||||
|
||||
_CONFIG_JSON_KEYS = frozenset({
|
||||
"dependencies", "devDependencies", "peerDependencies",
|
||||
"optionalDependencies", "bundleDependencies", "bundledDependencies",
|
||||
"extends", "$ref", "$schema", "compilerOptions",
|
||||
})
|
||||
|
||||
def _is_config_json(path: Path, obj_node, source: bytes) -> bool:
|
||||
"""True if a .json file is a recognized config/manifest worth AST-extracting.
|
||||
|
||||
Matches by filename first (cheap), then falls back to a top-level key probe
|
||||
so arbitrarily-named config files (e.g. ``api.tsconfig.json``,
|
||||
``foo.eslintrc.json``) are still picked up. Returns False for data JSON so it
|
||||
is skipped by the structural pass (#1224)."""
|
||||
name = path.name.casefold()
|
||||
if name in _CONFIG_JSON_NAMES:
|
||||
return True
|
||||
# Common compound config names: *.eslintrc.json, *.prettierrc.json, etc.
|
||||
if name.endswith((".eslintrc.json", ".prettierrc.json", ".babelrc.json",
|
||||
"tsconfig.json", "jsconfig.json")):
|
||||
return True
|
||||
# Top-level key probe: scan the root object's immediate keys (no deep walk).
|
||||
for top_key in obj_node.children:
|
||||
if top_key.type != "pair":
|
||||
continue
|
||||
key_node = top_key.child_by_field_name("key")
|
||||
if key_node is None:
|
||||
continue
|
||||
kc = key_node.child_by_field_name("string_content")
|
||||
text = _read_text(kc, source) if kc else _read_text(key_node, source).strip('"\'')
|
||||
if text in _CONFIG_JSON_KEYS:
|
||||
return True
|
||||
return False
|
||||
|
||||
def extract_json(path: Path) -> dict:
|
||||
"""Extract structure and dependency edges from a *config/manifest* .json file.
|
||||
|
||||
Data-shaped JSON (eval fixtures, datasets, GeoJSON, API response dumps) is
|
||||
deliberately skipped — AST-walking it produced hundreds of orphan key-nodes
|
||||
and duplicate communities that swamped real structure (#1224). Recognition
|
||||
is by filename (package.json, tsconfig.json, …) or a top-level key probe
|
||||
(dependencies / extends / $ref / $schema / compilerOptions)."""
|
||||
_JSON_MAX_BYTES = 1_048_576 # 1 MiB — skip large fixture dumps / GeoJSON blobs
|
||||
|
||||
try:
|
||||
import tree_sitter_json as tsjson
|
||||
from tree_sitter import Language, Parser
|
||||
except ImportError:
|
||||
return {"nodes": [], "edges": [], "error": "tree-sitter-json not installed"}
|
||||
|
||||
try:
|
||||
# Bounded read instead of stat()+read() to eliminate TOCTOU (J-1):
|
||||
# read one byte beyond the limit so we can detect oversized files even
|
||||
# if the file grows between stat and read.
|
||||
with path.open("rb") as _f:
|
||||
source = _f.read(_JSON_MAX_BYTES + 1)
|
||||
if len(source) > _JSON_MAX_BYTES:
|
||||
return {"nodes": [], "edges": [], "error": "json file too large to index"}
|
||||
language = Language(tsjson.language())
|
||||
parser = Parser(language)
|
||||
tree = parser.parse(source)
|
||||
root = tree.root_node
|
||||
except Exception as e:
|
||||
return {"nodes": [], "edges": [], "error": str(e)}
|
||||
|
||||
stem = _file_stem(path)
|
||||
str_path = str(path)
|
||||
nodes: list[dict] = []
|
||||
edges: list[dict] = []
|
||||
seen_ids: set[str] = set()
|
||||
|
||||
# Keys whose string values become imports (package.json dep blocks)
|
||||
_DEP_KEYS = frozenset({
|
||||
"dependencies", "devDependencies", "peerDependencies",
|
||||
"optionalDependencies", "bundleDependencies", "bundledDependencies",
|
||||
})
|
||||
|
||||
def add_node(nid: str, label: str, line: int, file_type: str = "code") -> None:
|
||||
if nid and nid not in seen_ids:
|
||||
seen_ids.add(nid)
|
||||
nodes.append({"id": nid, "label": label, "file_type": file_type,
|
||||
"source_file": str_path, "source_location": f"L{line}"})
|
||||
|
||||
def add_edge(src: str, tgt: str, relation: str, line: int,
|
||||
context: str | None = None) -> None:
|
||||
if not src or not tgt or src == tgt:
|
||||
return
|
||||
edge = {"source": src, "target": tgt, "relation": relation,
|
||||
"confidence": "EXTRACTED", "source_file": str_path,
|
||||
"source_location": f"L{line}", "weight": 1.0}
|
||||
if context:
|
||||
edge["context"] = context
|
||||
edges.append(edge)
|
||||
|
||||
file_nid = _make_id(str(path))
|
||||
add_node(file_nid, path.name, 1)
|
||||
|
||||
def _key_text(pair_node) -> str | None:
|
||||
"""Extract the string content of a pair's key."""
|
||||
key_node = pair_node.child_by_field_name("key")
|
||||
if key_node is None:
|
||||
return None
|
||||
if key_node.type == "string":
|
||||
content = key_node.child_by_field_name("string_content")
|
||||
if content:
|
||||
return _read_text(content, source)
|
||||
# fallback: strip surrounding quotes
|
||||
raw = _read_text(key_node, source)
|
||||
return raw.strip('"\'')
|
||||
return _read_text(key_node, source)
|
||||
|
||||
def _val_node(pair_node):
|
||||
return pair_node.child_by_field_name("value")
|
||||
|
||||
def walk_object(obj_node, parent_nid: str, parent_key: str | None,
|
||||
depth: int, pair_count: list) -> None:
|
||||
if depth > 6:
|
||||
return
|
||||
for child in obj_node.children:
|
||||
if child.type != "pair":
|
||||
continue
|
||||
if pair_count[0] >= 500: # check per-pair so the cap is honoured exactly (J-3)
|
||||
return
|
||||
pair_count[0] += 1
|
||||
key = _key_text(child)
|
||||
if not key:
|
||||
continue
|
||||
key_nid = _make_id(stem, *(([parent_key] if parent_key else []) + [key]))
|
||||
if not key_nid:
|
||||
continue
|
||||
line = child.start_point[0] + 1
|
||||
add_node(key_nid, key, line)
|
||||
add_edge(parent_nid, key_nid, "contains", line)
|
||||
|
||||
val = _val_node(child)
|
||||
if val is None:
|
||||
continue
|
||||
|
||||
if val.type == "object":
|
||||
walk_object(val, key_nid, key, depth + 1, pair_count)
|
||||
|
||||
elif val.type == "array":
|
||||
# For "extends" arrays (tsconfig, eslint): each string element.
|
||||
# Prefix with "ref_" so external refs don't collide with real
|
||||
# code/file node IDs that share the same collapsed _make_id (J-4).
|
||||
for item in val.children:
|
||||
if item.type == "string":
|
||||
content = item.child_by_field_name("string_content")
|
||||
ref = _read_text(content, source) if content else _read_text(item, source).strip('"\'')
|
||||
if ref:
|
||||
ref_nid = _make_id("ref", ref)
|
||||
if ref_nid:
|
||||
add_node(ref_nid, ref, line, file_type="concept")
|
||||
add_edge(key_nid, ref_nid, "extends", line, context="import")
|
||||
|
||||
elif val.type == "string":
|
||||
content = val.child_by_field_name("string_content")
|
||||
val_text = _read_text(content, source) if content else _read_text(val, source).strip('"\'')
|
||||
|
||||
if key == "extends" and val_text:
|
||||
# Namespace external refs to avoid ID collision with file nodes (J-4)
|
||||
ref_nid = _make_id("ref", val_text)
|
||||
if ref_nid:
|
||||
add_node(ref_nid, val_text, line, file_type="concept")
|
||||
add_edge(file_nid, ref_nid, "extends", line, context="import")
|
||||
|
||||
elif key == "$ref" and val_text:
|
||||
# Namespace $ref values to prevent edge hijacking into code nodes (J-4)
|
||||
ref_nid = _make_id("ref", val_text)
|
||||
if ref_nid:
|
||||
add_edge(parent_nid, ref_nid, "references", line)
|
||||
|
||||
elif parent_key in _DEP_KEYS and val_text:
|
||||
dep_nid = _make_id(key)
|
||||
if dep_nid:
|
||||
add_node(dep_nid, key, line, file_type="concept")
|
||||
add_edge(key_nid, dep_nid, "imports", line, context="import")
|
||||
|
||||
# Entry: find root document → object
|
||||
doc = root
|
||||
if doc.type == "document" and doc.child_count > 0:
|
||||
doc = doc.children[0]
|
||||
if doc.type == "object":
|
||||
# Only AST-extract recognized config/manifest JSON. Data JSON (fixtures,
|
||||
# datasets, GeoJSON, API dumps) is skipped so it doesn't explode into
|
||||
# orphan key-nodes (#1224); it's left to the LLM semantic pass.
|
||||
if not _is_config_json(path, doc, source):
|
||||
return {"nodes": [], "edges": [], "skipped": "data json (not a config/manifest)"}
|
||||
walk_object(doc, file_nid, None, 0, [0])
|
||||
else:
|
||||
# Top-level array or scalar => data JSON, never a config/manifest.
|
||||
return {"nodes": [], "edges": [], "skipped": "data json (non-object root)"}
|
||||
|
||||
return {"nodes": nodes, "edges": edges}
|
||||
@@ -0,0 +1,275 @@
|
||||
"""julia — moved verbatim from graphify/extract.py."""
|
||||
from __future__ import annotations
|
||||
|
||||
from graphify.extractors.base import _file_stem, _make_id, _read_text
|
||||
from graphify.extractors.engine import _semantic_reference_edge
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
def extract_julia(path: Path) -> dict:
|
||||
"""Extract modules, structs, functions, imports, and calls from a .jl file."""
|
||||
try:
|
||||
import tree_sitter_julia as tsjulia
|
||||
from tree_sitter import Language, Parser
|
||||
except ImportError:
|
||||
return {"nodes": [], "edges": [], "error": "tree-sitter-julia not installed"}
|
||||
|
||||
try:
|
||||
language = Language(tsjulia.language())
|
||||
parser = Parser(language)
|
||||
source = path.read_bytes()
|
||||
tree = parser.parse(source)
|
||||
root = tree.root_node
|
||||
except Exception as e:
|
||||
return {"nodes": [], "edges": [], "error": str(e)}
|
||||
|
||||
stem = _file_stem(path)
|
||||
str_path = str(path)
|
||||
nodes: list[dict] = []
|
||||
edges: list[dict] = []
|
||||
seen_ids: set[str] = set()
|
||||
function_bodies: list[tuple[str, object]] = []
|
||||
|
||||
def add_node(nid: str, label: str, line: int) -> None:
|
||||
if nid not in seen_ids:
|
||||
seen_ids.add(nid)
|
||||
nodes.append({
|
||||
"id": nid,
|
||||
"label": label,
|
||||
"file_type": "code",
|
||||
"source_file": str_path,
|
||||
"source_location": f"L{line}",
|
||||
})
|
||||
|
||||
def add_edge(src: str, tgt: str, relation: str, line: int,
|
||||
confidence: str = "EXTRACTED", weight: float = 1.0,
|
||||
context: str | None = None) -> None:
|
||||
edge = {
|
||||
"source": src,
|
||||
"target": tgt,
|
||||
"relation": relation,
|
||||
"confidence": confidence,
|
||||
"source_file": str_path,
|
||||
"source_location": f"L{line}",
|
||||
"weight": weight,
|
||||
}
|
||||
if context:
|
||||
edge["context"] = context
|
||||
edges.append(edge)
|
||||
|
||||
file_nid = _make_id(str(path))
|
||||
add_node(file_nid, path.name, 1)
|
||||
|
||||
def ensure_named_node(name: str, line: int) -> str:
|
||||
nid = _make_id(stem, name)
|
||||
if nid in seen_ids:
|
||||
return nid
|
||||
nid = _make_id(name)
|
||||
if nid not in seen_ids:
|
||||
# The name isn't defined in this file, so this is a cross-file reference
|
||||
# (e.g. a `Thing` type annotation imported from another module). Emit a
|
||||
# SOURCELESS stub — like the inheritance-base path below — so the
|
||||
# corpus-level rewire can collapse it onto the real definition. A sourced
|
||||
# stub here makes _disambiguate_colliding_node_ids bake the referencing
|
||||
# file's path (with extension) into the id and blocks the rewire, which is
|
||||
# the phantom-duplicate-node bug (#1402).
|
||||
seen_ids.add(nid)
|
||||
nodes.append({
|
||||
"id": nid,
|
||||
"label": name,
|
||||
"file_type": "code",
|
||||
"source_file": "",
|
||||
"source_location": "",
|
||||
"origin_file": str_path,
|
||||
})
|
||||
return nid
|
||||
|
||||
def _func_name_from_signature(sig_node) -> str | None:
|
||||
"""Extract function name from a Julia signature node (call_expression > identifier)."""
|
||||
for child in sig_node.children:
|
||||
if child.type == "call_expression":
|
||||
callee = child.children[0] if child.children else None
|
||||
if callee and callee.type == "identifier":
|
||||
return _read_text(callee, source)
|
||||
return None
|
||||
|
||||
def walk_calls(body_node, func_nid: str) -> None:
|
||||
if body_node is None:
|
||||
return
|
||||
t = body_node.type
|
||||
if t in ("function_definition", "short_function_definition"):
|
||||
return
|
||||
if t == "call_expression" and body_node.children:
|
||||
callee = body_node.children[0]
|
||||
# Direct call: foo(...)
|
||||
if callee.type == "identifier":
|
||||
callee_name = _read_text(callee, source)
|
||||
target_nid = _make_id(stem, callee_name)
|
||||
add_edge(func_nid, target_nid, "calls", body_node.start_point[0] + 1,
|
||||
confidence="EXTRACTED", context="call")
|
||||
# Method call: obj.method(...)
|
||||
elif callee.type == "field_expression" and len(callee.children) >= 3:
|
||||
method_node = callee.children[-1]
|
||||
method_name = _read_text(method_node, source)
|
||||
target_nid = _make_id(stem, method_name)
|
||||
add_edge(func_nid, target_nid, "calls", body_node.start_point[0] + 1,
|
||||
confidence="EXTRACTED", context="call")
|
||||
for child in body_node.children:
|
||||
walk_calls(child, func_nid)
|
||||
|
||||
def walk(node, scope_nid: str) -> None:
|
||||
t = node.type
|
||||
|
||||
# Module
|
||||
if t == "module_definition":
|
||||
name_node = next((c for c in node.children if c.type == "identifier"), None)
|
||||
if name_node:
|
||||
mod_name = _read_text(name_node, source)
|
||||
mod_nid = _make_id(stem, mod_name)
|
||||
line = node.start_point[0] + 1
|
||||
add_node(mod_nid, mod_name, line)
|
||||
add_edge(file_nid, mod_nid, "defines", line)
|
||||
for child in node.children:
|
||||
walk(child, mod_nid)
|
||||
return
|
||||
|
||||
# Struct (struct / mutable struct — both map to struct_definition in tree-sitter-julia)
|
||||
if t == "struct_definition":
|
||||
# type_head may contain: identifier (simple) or binary_expression (Foo <: Bar)
|
||||
type_head = next((c for c in node.children if c.type == "type_head"), None)
|
||||
if not type_head:
|
||||
return
|
||||
struct_name: str | None = None
|
||||
super_name: str | None = None
|
||||
bin_expr = next((c for c in type_head.children if c.type == "binary_expression"), None)
|
||||
if bin_expr:
|
||||
identifiers = [c for c in bin_expr.children if c.type == "identifier"]
|
||||
if identifiers:
|
||||
struct_name = _read_text(identifiers[0], source)
|
||||
if len(identifiers) >= 2:
|
||||
super_name = _read_text(identifiers[-1], source)
|
||||
else:
|
||||
name_node = next((c for c in type_head.children if c.type == "identifier"), None)
|
||||
if name_node:
|
||||
struct_name = _read_text(name_node, source)
|
||||
if not struct_name:
|
||||
return
|
||||
struct_nid = _make_id(stem, struct_name)
|
||||
line = node.start_point[0] + 1
|
||||
add_node(struct_nid, struct_name, line)
|
||||
add_edge(scope_nid, struct_nid, "defines", line)
|
||||
if super_name:
|
||||
add_edge(struct_nid, ensure_named_node(super_name, line),
|
||||
"inherits", line, confidence="EXTRACTED")
|
||||
# Field types: each `name::Type` lowers to a typed_expression child of struct_definition
|
||||
for child in node.children:
|
||||
if child.type == "typed_expression":
|
||||
type_ids = [c for c in child.children if c.type == "identifier"]
|
||||
if len(type_ids) >= 2:
|
||||
field_line = child.start_point[0] + 1
|
||||
type_name = _read_text(type_ids[-1], source)
|
||||
type_nid = ensure_named_node(type_name, field_line)
|
||||
edges.append(_semantic_reference_edge(
|
||||
struct_nid, type_nid, "field", str_path, field_line))
|
||||
return
|
||||
|
||||
# Abstract type
|
||||
if t == "abstract_definition":
|
||||
# type_head > identifier
|
||||
type_head = next((c for c in node.children if c.type == "type_head"), None)
|
||||
if type_head:
|
||||
name_node = next((c for c in type_head.children if c.type == "identifier"), None)
|
||||
if name_node:
|
||||
abs_name = _read_text(name_node, source)
|
||||
abs_nid = _make_id(stem, abs_name)
|
||||
line = node.start_point[0] + 1
|
||||
add_node(abs_nid, abs_name, line)
|
||||
add_edge(scope_nid, abs_nid, "defines", line)
|
||||
return
|
||||
|
||||
# Function: function foo(...) ... end
|
||||
if t == "function_definition":
|
||||
sig_node = next((c for c in node.children if c.type == "signature"), None)
|
||||
if sig_node:
|
||||
func_name = _func_name_from_signature(sig_node)
|
||||
if func_name:
|
||||
func_nid = _make_id(stem, func_name)
|
||||
line = node.start_point[0] + 1
|
||||
add_node(func_nid, f"{func_name}()", line)
|
||||
add_edge(scope_nid, func_nid, "defines", line)
|
||||
function_bodies.append((func_nid, node))
|
||||
return
|
||||
|
||||
# Short function: foo(x) = expr
|
||||
if t == "assignment":
|
||||
lhs = node.children[0] if node.children else None
|
||||
if lhs and lhs.type == "call_expression" and lhs.children:
|
||||
callee = lhs.children[0]
|
||||
if callee.type == "identifier":
|
||||
func_name = _read_text(callee, source)
|
||||
func_nid = _make_id(stem, func_name)
|
||||
line = node.start_point[0] + 1
|
||||
add_node(func_nid, f"{func_name}()", line)
|
||||
add_edge(scope_nid, func_nid, "defines", line)
|
||||
# Only walk the RHS (index 2 after lhs and operator) to avoid self-loops
|
||||
rhs = node.children[-1] if len(node.children) >= 3 else None
|
||||
if rhs:
|
||||
function_bodies.append((func_nid, rhs))
|
||||
return
|
||||
|
||||
# Using / Import
|
||||
if t in ("using_statement", "import_statement"):
|
||||
line = node.start_point[0] + 1
|
||||
|
||||
def _julia_mod_name(n):
|
||||
# identifier (`Foo`), scoped_identifier (`Base.Threads`), or
|
||||
# import_path (relative `..Sibling`) -> the module name. Only bare
|
||||
# identifiers were handled, so qualified/relative imports — and the
|
||||
# scoped package of a `selected_import` — were silently dropped.
|
||||
if n.type == "import_path":
|
||||
ids = [c for c in n.children if c.type == "identifier"]
|
||||
return _read_text(ids[-1], source) if ids else None
|
||||
if n.type in ("identifier", "scoped_identifier"):
|
||||
return _read_text(n, source)
|
||||
return None
|
||||
|
||||
def _emit_import(name):
|
||||
if not name:
|
||||
return
|
||||
imp_nid = _make_id(name)
|
||||
add_node(imp_nid, name, line)
|
||||
add_edge(scope_nid, imp_nid, "imports", line, context="import")
|
||||
|
||||
for child in node.children:
|
||||
if child.type in ("identifier", "scoped_identifier", "import_path"):
|
||||
_emit_import(_julia_mod_name(child))
|
||||
elif child.type == "selected_import":
|
||||
# `import Base.Threads: nthreads` — the package (first named
|
||||
# child) may itself be a scoped_identifier/import_path.
|
||||
pkg = next(
|
||||
(c for c in child.children
|
||||
if c.type in ("identifier", "scoped_identifier", "import_path")),
|
||||
None,
|
||||
)
|
||||
if pkg is not None:
|
||||
_emit_import(_julia_mod_name(pkg))
|
||||
return
|
||||
|
||||
for child in node.children:
|
||||
walk(child, scope_nid)
|
||||
|
||||
walk(root, file_nid)
|
||||
|
||||
for func_nid, body_node in function_bodies:
|
||||
# For function_definition nodes, walk children directly to avoid
|
||||
# the boundary check returning early on the top-level node itself.
|
||||
# Skip the "signature" child — it contains the function's own call_expression
|
||||
# which would create a self-loop.
|
||||
if body_node.type == "function_definition":
|
||||
for child in body_node.children:
|
||||
if child.type != "signature":
|
||||
walk_calls(child, func_nid)
|
||||
else:
|
||||
walk_calls(body_node, func_nid)
|
||||
|
||||
return {"nodes": nodes, "edges": edges}
|
||||
@@ -0,0 +1,176 @@
|
||||
"""Markdown extractor. Moved verbatim from graphify/extract.py."""
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
import os
|
||||
|
||||
from pathlib import Path
|
||||
from graphify.extractors.base import _file_stem, _make_id
|
||||
|
||||
|
||||
_MD_INLINE_LINK_RE = re.compile(r'(?<!\!)\[[^\]]*\]\(\s*<?([^)\s>]+)>?(?:\s+[^)]*)?\)')
|
||||
|
||||
_MD_REF_DEF_RE = re.compile(r'^\s{0,3}\[[^\]]+\]:\s*<?([^\s>]+)>?')
|
||||
|
||||
_MD_WIKILINK_RE = re.compile(r'(?<!\!)\[\[([^\]|#]+)(?:[#|][^\]]*)?\]\]')
|
||||
|
||||
_MD_LINKABLE_EXTS = {".md", ".mdx", ".qmd", ".markdown", ".rst", ".txt"}
|
||||
|
||||
def _resolve_markdown_link(raw: str, source_dir: Path) -> "Path | None":
|
||||
"""Resolve a markdown link target to the absolute path of a sibling document.
|
||||
|
||||
Returns the resolved (normalized, not necessarily existing) path when the
|
||||
target is a *local* relative/absolute file-path link to a document, or None
|
||||
when it should be skipped: external URLs (http/https/mailto/protocol-
|
||||
relative/data), pure in-page anchors (``#section``), and links to non-doc
|
||||
file types (code/assets are handled by their own extractors).
|
||||
|
||||
The anchor fragment (``#section``) and query (``?x=1``) are stripped before
|
||||
resolution so ``./repo.md#setup`` resolves to the same node as ``./repo.md``.
|
||||
Extension-less targets (typical of wikilinks) are treated as sibling ``.md``.
|
||||
"""
|
||||
target = raw.strip()
|
||||
if not target:
|
||||
return None
|
||||
# Drop anchor / query so #section links still resolve to the target doc.
|
||||
target = target.split("#", 1)[0].split("?", 1)[0].strip()
|
||||
if not target:
|
||||
return None
|
||||
low = target.lower()
|
||||
if "://" in target or low.startswith(("mailto:", "tel:", "//", "data:")):
|
||||
return None
|
||||
suffix = Path(target).suffix.lower()
|
||||
if suffix == "":
|
||||
target = target + ".md"
|
||||
suffix = ".md"
|
||||
if suffix not in _MD_LINKABLE_EXTS:
|
||||
return None
|
||||
candidate = Path(target)
|
||||
if not candidate.is_absolute():
|
||||
candidate = source_dir / candidate
|
||||
return Path(os.path.normpath(str(candidate)))
|
||||
|
||||
def extract_markdown(path: Path) -> dict:
|
||||
"""Extract structural nodes and edges from a Markdown file.
|
||||
|
||||
Produces nodes for:
|
||||
- The file itself
|
||||
- Each heading (# / ## / ### etc.)
|
||||
|
||||
Produces edges for:
|
||||
- file --contains--> heading
|
||||
- parent heading --contains--> child heading (nesting by level)
|
||||
- heading --references--> other node (when backtick `Name` matches a known pattern)
|
||||
- file --references--> linked document, for inline ``[text](./other.md)``,
|
||||
reference-style ``[label]: ./other.md`` and ``[[wikilink]]`` links, so a
|
||||
hub doc (``index.md`` / ``table-of-contents.md``) becomes a real hub node
|
||||
instead of an under-connected orphan (#1376). The target node ID is built
|
||||
from the resolved target path with the same recipe as the target file's
|
||||
own node, so the edge merges into that node (no ghost node). External
|
||||
URLs, in-page anchors, images and non-document targets are skipped.
|
||||
|
||||
Fenced code blocks (``` ... ```) are skipped during parsing so their
|
||||
contents don't get treated as headings, but no node is emitted for
|
||||
them — they were always orphans (only a single contains edge to the
|
||||
parent doc) and inflated the disconnected-component count (#1077).
|
||||
|
||||
No tree-sitter dependency — pure line-by-line parsing.
|
||||
"""
|
||||
try:
|
||||
source = path.read_text(encoding="utf-8", errors="replace")
|
||||
except Exception as e:
|
||||
return {"nodes": [], "edges": [], "error": str(e)}
|
||||
|
||||
stem = _file_stem(path)
|
||||
str_path = str(path)
|
||||
nodes: list[dict] = []
|
||||
edges: list[dict] = []
|
||||
seen_ids: set[str] = set()
|
||||
|
||||
def add_node(nid: str, label: str, line: int, file_type: str = "document") -> None:
|
||||
if nid not in seen_ids:
|
||||
seen_ids.add(nid)
|
||||
nodes.append({"id": nid, "label": label, "file_type": file_type,
|
||||
"source_file": str_path, "source_location": f"L{line}"})
|
||||
|
||||
def add_edge(src: str, tgt: str, relation: str, line: int,
|
||||
confidence: str = "EXTRACTED", weight: float = 1.0) -> None:
|
||||
edges.append({"source": src, "target": tgt, "relation": relation,
|
||||
"confidence": confidence, "source_file": str_path,
|
||||
"source_location": f"L{line}", "weight": weight})
|
||||
|
||||
file_nid = _make_id(str(path))
|
||||
add_node(file_nid, path.name, 1)
|
||||
|
||||
source_dir = path.parent
|
||||
# Dedup link edges by resolved target node so a hub doc that links to the
|
||||
# same sibling many times yields one edge, not N (keeps weights meaningful).
|
||||
linked_targets: set[str] = set()
|
||||
|
||||
def add_link(raw: str, line: int) -> None:
|
||||
resolved = _resolve_markdown_link(raw, source_dir)
|
||||
if resolved is None:
|
||||
return
|
||||
# Build the target ID with the SAME recipe as the target file's own
|
||||
# node (_make_id(str(path)) at extract time, canonicalized to
|
||||
# _file_node_id(rel) by the extract() post-pass). Using the absolute
|
||||
# resolved path means both endpoints get remapped identically, so the
|
||||
# edge merges into the existing doc node instead of spawning a ghost.
|
||||
tgt_nid = _make_id(str(resolved))
|
||||
if tgt_nid == file_nid or tgt_nid in linked_targets:
|
||||
return
|
||||
linked_targets.add(tgt_nid)
|
||||
add_edge(file_nid, tgt_nid, "references", line)
|
||||
|
||||
# Track heading stack for nesting: [(level, nid), ...]
|
||||
heading_stack: list[tuple[int, str]] = []
|
||||
in_code_block = False
|
||||
|
||||
lines = source.splitlines()
|
||||
for line_num_0, line_text in enumerate(lines):
|
||||
line_num = line_num_0 + 1
|
||||
|
||||
# Skip over fenced code blocks so their contents are not parsed as
|
||||
# headings, but do not emit nodes/edges for them (#1077).
|
||||
stripped = line_text.strip()
|
||||
if stripped.startswith("```"):
|
||||
in_code_block = not in_code_block
|
||||
continue
|
||||
|
||||
if in_code_block:
|
||||
continue
|
||||
|
||||
# Markdown links -> document references (#1376). Scanned on every
|
||||
# non-fenced line (including heading lines, which the heading branch
|
||||
# below `continue`s past) so links anywhere in the doc are captured.
|
||||
for m in _MD_INLINE_LINK_RE.finditer(line_text):
|
||||
add_link(m.group(1), line_num)
|
||||
for m in _MD_WIKILINK_RE.finditer(line_text):
|
||||
add_link(m.group(1), line_num)
|
||||
ref_def = _MD_REF_DEF_RE.match(line_text)
|
||||
if ref_def:
|
||||
add_link(ref_def.group(1), line_num)
|
||||
|
||||
# Detect headings: # Heading, ## Heading, etc.
|
||||
heading_match = re.match(r'^(#{1,6})\s+(.+)', line_text)
|
||||
if heading_match:
|
||||
level = len(heading_match.group(1))
|
||||
title = heading_match.group(2).strip()
|
||||
h_nid = _make_id(stem, title)
|
||||
# Avoid duplicate heading IDs by appending line number
|
||||
if h_nid in seen_ids:
|
||||
h_nid = _make_id(stem, title, str(line_num))
|
||||
add_node(h_nid, title, line_num)
|
||||
|
||||
# Pop headings at same or deeper level
|
||||
while heading_stack and heading_stack[-1][0] >= level:
|
||||
heading_stack.pop()
|
||||
|
||||
# Connect to parent heading or file
|
||||
parent = heading_stack[-1][1] if heading_stack else file_nid
|
||||
add_edge(parent, h_nid, "contains", line_num)
|
||||
|
||||
heading_stack.append((level, h_nid))
|
||||
continue
|
||||
|
||||
return {"nodes": nodes, "edges": edges, "input_tokens": 0, "output_tokens": 0}
|
||||
@@ -0,0 +1,119 @@
|
||||
"""models — moved verbatim from graphify/extract.py."""
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any, Callable
|
||||
from pathlib import Path
|
||||
from dataclasses import dataclass, field
|
||||
|
||||
|
||||
_WORKSPACE_PACKAGE_CACHE: dict[str, dict[str, Path]] = {}
|
||||
|
||||
_JS_CACHE_BYPASS_SUFFIXES = {".js", ".jsx", ".mjs", ".ts", ".tsx", ".mts", ".cts", ".vue", ".svelte"}
|
||||
|
||||
@dataclass
|
||||
class LanguageConfig:
|
||||
ts_module: str # e.g. "tree_sitter_python"
|
||||
ts_language_fn: str = "language" # attr to call: e.g. tslang.language()
|
||||
|
||||
class_types: frozenset = frozenset()
|
||||
function_types: frozenset = frozenset()
|
||||
import_types: frozenset = frozenset()
|
||||
call_types: frozenset = frozenset()
|
||||
static_prop_types: frozenset = frozenset()
|
||||
helper_fn_names: frozenset = frozenset()
|
||||
container_bind_methods: frozenset = frozenset()
|
||||
event_listener_properties: frozenset = frozenset()
|
||||
|
||||
# Name extraction
|
||||
name_field: str = "name"
|
||||
name_fallback_child_types: tuple = ()
|
||||
|
||||
# Body detection
|
||||
body_field: str = "body"
|
||||
body_fallback_child_types: tuple = () # e.g. ("declaration_list", "compound_statement")
|
||||
|
||||
# Call name extraction
|
||||
call_function_field: str = "function" # field on call node for callee
|
||||
call_accessor_node_types: frozenset = frozenset() # member/attribute nodes
|
||||
call_accessor_field: str = "attribute" # field on accessor for method name
|
||||
call_accessor_object_field: str = "" # field on accessor for the receiver/object
|
||||
|
||||
# Stop recursion at these types in walk_calls
|
||||
function_boundary_types: frozenset = frozenset()
|
||||
|
||||
# Import handler: called for import nodes instead of generic handling
|
||||
import_handler: Callable | None = None
|
||||
|
||||
# Optional custom name resolver for functions (C, C++ declarator unwrapping)
|
||||
resolve_function_name_fn: Callable | None = None
|
||||
|
||||
# Extra label formatting for functions: if True, functions get "name()" label
|
||||
function_label_parens: bool = True
|
||||
|
||||
# Extra walk hook called after generic dispatch (for JS arrow functions, C# namespaces, etc.)
|
||||
extra_walk_fn: Callable | None = None
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class _SymbolDeclarationFact:
|
||||
file_path: Path
|
||||
name: str
|
||||
line: int
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class _SymbolImportFact:
|
||||
file_path: Path
|
||||
local_name: str
|
||||
target_path: Path
|
||||
imported_name: str
|
||||
line: int
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class _SymbolAliasFact:
|
||||
file_path: Path
|
||||
alias: str
|
||||
target_name: str
|
||||
line: int
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class _SymbolExportFact:
|
||||
file_path: Path
|
||||
exported_name: str
|
||||
line: int
|
||||
local_name: str | None = None
|
||||
target_path: Path | None = None
|
||||
target_name: str | None = None
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class _StarExportFact:
|
||||
file_path: Path
|
||||
target_path: Path
|
||||
line: int
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class _NamespaceExportFact:
|
||||
file_path: Path
|
||||
exported_name: str
|
||||
target_path: Path
|
||||
line: int
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class _SymbolUseFact:
|
||||
file_path: Path
|
||||
source_id: str
|
||||
local_name: str
|
||||
relation: str
|
||||
context: str
|
||||
line: int
|
||||
|
||||
@dataclass
|
||||
class _SymbolResolutionFacts:
|
||||
declarations: list[_SymbolDeclarationFact] = field(default_factory=list)
|
||||
imports: list[_SymbolImportFact] = field(default_factory=list)
|
||||
aliases: list[_SymbolAliasFact] = field(default_factory=list)
|
||||
exports: list[_SymbolExportFact] = field(default_factory=list)
|
||||
star_exports: list[_StarExportFact] = field(default_factory=list)
|
||||
namespace_exports: list[_NamespaceExportFact] = field(default_factory=list)
|
||||
uses: list[_SymbolUseFact] = field(default_factory=list)
|
||||
# File-to-file submodule imports from `from pkg import submod` (#1146).
|
||||
# Each entry is (importing_file, submodule_file, line).
|
||||
module_imports: list[tuple[Path, Path, int]] = field(default_factory=list)
|
||||
@@ -0,0 +1,430 @@
|
||||
"""objc — moved verbatim from graphify/extract.py."""
|
||||
from __future__ import annotations
|
||||
|
||||
from graphify.extractors.base import _file_stem, _make_id, _read_text
|
||||
from graphify.extractors.engine import _cpp_declarator_name, _semantic_reference_edge
|
||||
from graphify.extractors.resolution import _resolve_c_include_path
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
|
||||
def _objc_local_var_types(body_node, source: bytes, table: dict[str, str]) -> None:
|
||||
"""Collect ``var -> ClassName`` from ObjC local declarations (``Foo *f = ...;``)
|
||||
in a method body, for receiver typing in the cross-file message-send pass
|
||||
(#1556). Only a capitalized ``type_identifier`` with a single named declarator
|
||||
is recorded; a built-in/lower-cased type or an un-nameable declarator is skipped
|
||||
(precision over recall). Reuses the C++ declarator unwrapper (identical grammar).
|
||||
"""
|
||||
stack = [body_node]
|
||||
while stack:
|
||||
n = stack.pop()
|
||||
if n.type == "method_definition" and n is not body_node:
|
||||
continue
|
||||
if n.type == "declaration":
|
||||
type_node = n.child_by_field_name("type")
|
||||
if type_node is None:
|
||||
for c in n.children:
|
||||
if c.type == "type_identifier":
|
||||
type_node = c
|
||||
break
|
||||
if type_node is not None and type_node.type == "type_identifier":
|
||||
type_name = _read_text(type_node, source).strip()
|
||||
declarators = [
|
||||
c for c in n.children
|
||||
if c.type in ("identifier", "pointer_declarator", "init_declarator")
|
||||
]
|
||||
if type_name and type_name[:1].isupper() and len(declarators) == 1:
|
||||
var = _cpp_declarator_name(declarators[0], source)
|
||||
if var and var not in table:
|
||||
table[var] = type_name
|
||||
for c in n.children:
|
||||
stack.append(c)
|
||||
|
||||
def extract_objc(path: Path) -> dict:
|
||||
"""Extract interfaces, implementations, protocols, methods, and imports from .m/.mm/.h files."""
|
||||
try:
|
||||
import tree_sitter_objc as tsobjc
|
||||
from tree_sitter import Language, Parser
|
||||
except ImportError:
|
||||
return {"nodes": [], "edges": [], "error": "tree_sitter_objc not installed"}
|
||||
|
||||
try:
|
||||
language = Language(tsobjc.language())
|
||||
parser = Parser(language)
|
||||
source = path.read_bytes()
|
||||
# tree-sitter-objc cannot expand these argument-less annotation macros (no
|
||||
# trailing ';'), and their presence before @interface makes the parser fail to
|
||||
# emit a class_interface node (#1475). Blank them to equal-length spaces so byte
|
||||
# offsets / line numbers are preserved and the interface parses.
|
||||
_OBJC_BLANK_MACROS = (b"NS_ASSUME_NONNULL_BEGIN", b"NS_ASSUME_NONNULL_END")
|
||||
for _m in _OBJC_BLANK_MACROS:
|
||||
source = source.replace(_m, b" " * len(_m))
|
||||
tree = parser.parse(source)
|
||||
root = tree.root_node
|
||||
except Exception as e:
|
||||
return {"nodes": [], "edges": [], "error": str(e)}
|
||||
|
||||
stem = _file_stem(path)
|
||||
str_path = str(path)
|
||||
nodes: list[dict] = []
|
||||
edges: list[dict] = []
|
||||
seen_ids: set[str] = set()
|
||||
method_bodies: list[tuple[str, Any, str]] = []
|
||||
# #1556: unresolved message sends saved for the cross-file ObjC resolver, plus a
|
||||
# per-file `var -> ClassName` table from `Foo *f = ...;` local declarations.
|
||||
raw_calls: list[dict] = []
|
||||
objc_type_table: dict[str, str] = {}
|
||||
|
||||
def add_node(nid: str, label: str, line: int) -> None:
|
||||
if nid not in seen_ids:
|
||||
seen_ids.add(nid)
|
||||
nodes.append({"id": nid, "label": label, "file_type": "code",
|
||||
"source_file": str_path, "source_location": f"L{line}"})
|
||||
|
||||
def add_edge(src: str, tgt: str, relation: str, line: int,
|
||||
confidence: str = "EXTRACTED", weight: float = 1.0,
|
||||
context: str | None = None) -> None:
|
||||
edge = {"source": src, "target": tgt, "relation": relation,
|
||||
"confidence": confidence, "source_file": str_path,
|
||||
"source_location": f"L{line}", "weight": weight}
|
||||
if context:
|
||||
edge["context"] = context
|
||||
edges.append(edge)
|
||||
|
||||
file_nid = _make_id(str(path))
|
||||
add_node(file_nid, path.name, 1)
|
||||
|
||||
def _read(node) -> str:
|
||||
return source[node.start_byte:node.end_byte].decode("utf-8", errors="replace")
|
||||
|
||||
def _get_name(node, field: str) -> str | None:
|
||||
n = node.child_by_field_name(field)
|
||||
return _read(n) if n else None
|
||||
|
||||
def _type_identifiers(node):
|
||||
"""Yield every type_identifier under a property's type node, descending
|
||||
through generic_specifier/type_name so NSArray<Product *> yields both
|
||||
NSArray and the element type Product (the generic case was invisible
|
||||
because the type was wrapped in a generic_specifier, not a bare
|
||||
type_identifier child) (#1475)."""
|
||||
if node.type == "type_identifier":
|
||||
yield node
|
||||
return
|
||||
for c in node.children:
|
||||
yield from _type_identifiers(c)
|
||||
|
||||
def ensure_named_node(name: str, line: int) -> str:
|
||||
nid = _make_id(stem, name)
|
||||
if nid in seen_ids:
|
||||
return nid
|
||||
nid = _make_id(name)
|
||||
if nid not in seen_ids:
|
||||
# The name isn't defined in this file, so this is a cross-file reference
|
||||
# (e.g. a `Thing` type annotation imported from another module). Emit a
|
||||
# SOURCELESS stub — like the inheritance-base path below — so the
|
||||
# corpus-level rewire can collapse it onto the real definition. A sourced
|
||||
# stub here makes _disambiguate_colliding_node_ids bake the referencing
|
||||
# file's path (with extension) into the id and blocks the rewire, which is
|
||||
# the phantom-duplicate-node bug (#1402).
|
||||
seen_ids.add(nid)
|
||||
nodes.append({
|
||||
"id": nid,
|
||||
"label": name,
|
||||
"file_type": "code",
|
||||
"source_file": "",
|
||||
"source_location": "",
|
||||
"origin_file": str_path,
|
||||
})
|
||||
return nid
|
||||
|
||||
def walk(node, parent_nid: str | None = None) -> None:
|
||||
t = node.type
|
||||
line = node.start_point[0] + 1
|
||||
|
||||
if t == "preproc_include":
|
||||
# #import <Foundation/Foundation.h> or #import "MyClass.h"
|
||||
for child in node.children:
|
||||
if child.type == "system_lib_string":
|
||||
raw = _read(child).strip("<>")
|
||||
module = raw.split("/")[-1].replace(".h", "")
|
||||
if module:
|
||||
tgt_nid = _make_id(module)
|
||||
add_edge(file_nid, tgt_nid, "imports", line, context="import")
|
||||
elif child.type == "string_literal":
|
||||
# recurse into string_literal to find string_content
|
||||
for sub in child.children:
|
||||
if sub.type == "string_content":
|
||||
raw = _read(sub)
|
||||
# Resolve the quoted include to a real file so the target id
|
||||
# matches the (possibly disambiguated) node id _make_id gives
|
||||
# that file; the bare-stem id never survives
|
||||
# _disambiguate_colliding_node_ids when a .h/.m pair exists,
|
||||
# so the edge dangled and was dropped (#1475).
|
||||
resolved = _resolve_c_include_path(raw, str_path)
|
||||
if resolved is not None:
|
||||
add_edge(file_nid, _make_id(str(resolved)), "imports", line, context="import")
|
||||
else:
|
||||
module = raw.split("/")[-1].replace(".h", "")
|
||||
if module:
|
||||
add_edge(file_nid, _make_id(module), "imports", line, context="import")
|
||||
return
|
||||
|
||||
if t == "module_import":
|
||||
# @import Foundation; / @import Foundation.NSString;
|
||||
path_node = node.child_by_field_name("path")
|
||||
if path_node is not None:
|
||||
module = _read(path_node).split(".")[0].strip()
|
||||
if module:
|
||||
add_edge(file_nid, _make_id(module), "imports", line, context="import")
|
||||
return
|
||||
|
||||
if t == "class_interface":
|
||||
# @interface ClassName : SuperClass <Protocols>
|
||||
# children: @interface, identifier(name), ':', identifier(super), parameterized_arguments, ...
|
||||
identifiers = [c for c in node.children if c.type == "identifier"]
|
||||
if not identifiers:
|
||||
for child in node.children:
|
||||
walk(child, parent_nid)
|
||||
return
|
||||
name = _read(identifiers[0])
|
||||
cls_nid = _make_id(stem, name)
|
||||
add_node(cls_nid, name, line)
|
||||
add_edge(file_nid, cls_nid, "contains", line)
|
||||
# superclass is second identifier after ':'
|
||||
colon_seen = False
|
||||
for child in node.children:
|
||||
if child.type == ":":
|
||||
colon_seen = True
|
||||
elif colon_seen and child.type == "identifier":
|
||||
super_nid = ensure_named_node(_read(child), line)
|
||||
add_edge(cls_nid, super_nid, "inherits", line)
|
||||
colon_seen = False
|
||||
elif child.type == "parameterized_arguments":
|
||||
# protocols adopted: @interface Foo : Bar <Proto1, Proto2>
|
||||
for sub in child.children:
|
||||
if sub.type == "type_name":
|
||||
for s in sub.children:
|
||||
if s.type == "type_identifier":
|
||||
proto_nid = ensure_named_node(_read(s), line)
|
||||
add_edge(cls_nid, proto_nid, "implements", line)
|
||||
elif child.type == "property_declaration":
|
||||
prop_line = child.start_point[0] + 1
|
||||
for sub in child.children:
|
||||
if sub.type == "struct_declaration":
|
||||
# The type is either a direct type_identifier
|
||||
# (NSString *x) or wrapped in a generic_specifier
|
||||
# (NSArray<Product *> *xs). Walk every type name in the
|
||||
# type portion, skipping the declarator (the *field
|
||||
# name), so generic collections are no longer invisible.
|
||||
seen_types: set[str] = set()
|
||||
for s in sub.children:
|
||||
if s.type in ("struct_declarator", ";"):
|
||||
continue
|
||||
for ti in _type_identifiers(s):
|
||||
tname = _read(ti)
|
||||
if tname in seen_types:
|
||||
continue
|
||||
seen_types.add(tname)
|
||||
type_nid = ensure_named_node(tname, prop_line)
|
||||
edges.append(_semantic_reference_edge(
|
||||
cls_nid, type_nid, "field", str_path, prop_line))
|
||||
elif child.type == "method_declaration":
|
||||
walk(child, cls_nid)
|
||||
return
|
||||
|
||||
if t == "class_implementation":
|
||||
# @implementation ClassName
|
||||
name = None
|
||||
for child in node.children:
|
||||
if child.type == "identifier":
|
||||
name = _read(child)
|
||||
break
|
||||
if not name:
|
||||
for child in node.children:
|
||||
walk(child, parent_nid)
|
||||
return
|
||||
impl_nid = _make_id(stem, name)
|
||||
if impl_nid not in seen_ids:
|
||||
add_node(impl_nid, name, line)
|
||||
add_edge(file_nid, impl_nid, "contains", line)
|
||||
for child in node.children:
|
||||
if child.type == "implementation_definition":
|
||||
for sub in child.children:
|
||||
walk(sub, impl_nid)
|
||||
return
|
||||
|
||||
if t == "protocol_declaration":
|
||||
name = None
|
||||
for child in node.children:
|
||||
if child.type == "identifier":
|
||||
name = _read(child)
|
||||
break
|
||||
if name:
|
||||
proto_nid = _make_id(stem, name)
|
||||
add_node(proto_nid, f"<{name}>", line)
|
||||
add_edge(file_nid, proto_nid, "contains", line)
|
||||
# Adopted protocols: `@protocol Derived <Base, Other>`. These
|
||||
# nest under a protocol_reference_list node (distinct from the
|
||||
# parameterized_arguments node used by @interface adoption), so
|
||||
# they were never emitted. Emit an `implements` edge for each,
|
||||
# matching how @interface protocol adoption is handled.
|
||||
for child in node.children:
|
||||
if child.type == "protocol_reference_list":
|
||||
for sub in child.children:
|
||||
if sub.type == "identifier":
|
||||
base_nid = ensure_named_node(_read(sub), line)
|
||||
if base_nid != proto_nid:
|
||||
add_edge(proto_nid, base_nid, "implements", line)
|
||||
for child in node.children:
|
||||
walk(child, proto_nid)
|
||||
return
|
||||
|
||||
if t in ("method_declaration", "method_definition"):
|
||||
container = parent_nid or file_nid
|
||||
# Class methods start with '+', instance methods with '-' (the grammar
|
||||
# emits the sigil as the first child). The selector is the concatenation
|
||||
# of the direct identifier children: one for a simple selector (-go),
|
||||
# several for a compound one (-tableView:numberOfRowsInSection: ->
|
||||
# "tableViewnumberOfRowsInSection"); method_parameter holds the arg
|
||||
# types/names, not selector keywords, so it is correctly skipped.
|
||||
prefix = "-"
|
||||
for child in node.children:
|
||||
if child.type in ("+", "-"):
|
||||
prefix = child.type
|
||||
break
|
||||
parts = [_read(c) for c in node.children if c.type == "identifier"]
|
||||
method_name = "".join(parts) if parts else None
|
||||
if method_name:
|
||||
method_nid = _make_id(container, method_name)
|
||||
add_node(method_nid, f"{prefix}{method_name}", line)
|
||||
add_edge(container, method_nid, "method", line)
|
||||
if t == "method_definition":
|
||||
method_bodies.append((method_nid, node, container))
|
||||
return
|
||||
|
||||
for child in node.children:
|
||||
walk(child, parent_nid)
|
||||
|
||||
walk(root)
|
||||
|
||||
# Second pass: resolve calls inside method bodies
|
||||
all_method_nids = {n["id"] for n in nodes if n["id"] != file_nid}
|
||||
class_method_nids: dict[str, set[str]] = {}
|
||||
for m_nid, _, container_nid in method_bodies:
|
||||
class_method_nids.setdefault(container_nid, set()).add(m_nid)
|
||||
seen_calls: set[tuple[str, str]] = set()
|
||||
# #1556: per-file `var -> ClassName` table from local declarations in every
|
||||
# method body, so the cross-file resolver can type a `[f doThing]` receiver.
|
||||
for _m_nid, body_node, _container in method_bodies:
|
||||
_objc_local_var_types(body_node, source, objc_type_table)
|
||||
|
||||
for caller_nid, body_node, container_nid in method_bodies:
|
||||
sibling_nids = class_method_nids.get(container_nid, set())
|
||||
|
||||
def walk_calls(n) -> None:
|
||||
if n.type == "message_expression":
|
||||
# `[[Foo alloc] init]` is a message_expression whose method is the
|
||||
# identifier `alloc` and whose receiver is the bare class identifier
|
||||
# `Foo`; resolve that class name and emit a `references` edge so the
|
||||
# allocating method links to the allocated type. ensure_named_node
|
||||
# emits a sourceless stub for unknown names, which the corpus rewire
|
||||
# collapses ONLY when exactly one real class of that name exists, so an
|
||||
# unknown/ambiguous class produces no false resolved edge (#1475).
|
||||
meth = n.child_by_field_name("method")
|
||||
recv = n.child_by_field_name("receiver")
|
||||
if (meth is not None and meth.type == "identifier" and _read(meth) == "alloc"
|
||||
and recv is not None and recv.type == "identifier"):
|
||||
tname = _read(recv)
|
||||
ref_line = n.start_point[0] + 1
|
||||
type_nid = ensure_named_node(tname, ref_line)
|
||||
if type_nid != caller_nid:
|
||||
edges.append(_semantic_reference_edge(
|
||||
caller_nid, type_nid, "type", str_path, ref_line))
|
||||
# [receiver sel] and [receiver kw1:a kw2:b] both parse to a
|
||||
# message_expression whose selector parts carry the field name
|
||||
# "method" (one for a simple selector, several for a compound one);
|
||||
# the receiver carries field name "receiver". Reconstruct the
|
||||
# selector from every "method" child so self/super/ClassName
|
||||
# receivers are never mistaken for a selector, and compound sends
|
||||
# resolve too (the whole second pass was previously dead code for
|
||||
# ObjC because the grammar emits these as `identifier`, not
|
||||
# `selector`/`keyword_argument_list`) (#1475).
|
||||
sel_parts = [
|
||||
_read(child)
|
||||
for i, child in enumerate(n.children)
|
||||
if n.field_name_for_child(i) == "method" and child.type == "identifier"
|
||||
]
|
||||
method_name = "".join(sel_parts)
|
||||
if method_name:
|
||||
needle = _make_id("", method_name).lstrip("_")
|
||||
for candidate in all_method_nids:
|
||||
if candidate.endswith(needle):
|
||||
pair = (caller_nid, candidate)
|
||||
if pair not in seen_calls and caller_nid != candidate:
|
||||
seen_calls.add(pair)
|
||||
add_edge(caller_nid, candidate, "calls", n.start_point[0] + 1,
|
||||
confidence="EXTRACTED", weight=1.0, context="call")
|
||||
# #1556: also emit a raw_call so the cross-file resolver can type
|
||||
# the receiver and link to a method in ANOTHER file. A bare
|
||||
# identifier receiver (`f`, `self`, `Foo`) is captured; a nested
|
||||
# message send (`[[Foo alloc] init]`) has no simple receiver name
|
||||
# to type, so it is left to the alloc/init `references` edge above.
|
||||
if recv is not None and recv.type == "identifier":
|
||||
raw_calls.append({
|
||||
"caller_nid": caller_nid,
|
||||
"callee": method_name,
|
||||
"is_member_call": True,
|
||||
"source_file": str_path,
|
||||
"source_location": f"L{n.start_point[0] + 1}",
|
||||
"receiver": _read(recv),
|
||||
"lang": "objc",
|
||||
})
|
||||
elif n.type == "field_expression":
|
||||
# self.name / self.product.name — dot-syntax sugar for [self name].
|
||||
# Resolve to a sibling method of the SAME class, matched by EXACT
|
||||
# node id (a method id is _make_id(container, name)). A suffix
|
||||
# substring match would mis-resolve self.name -> -surname and would
|
||||
# let a substring-colliding sibling (-surname) suppress the real
|
||||
# -name edge, so it must be an exact match (#1475).
|
||||
for child in n.children:
|
||||
if child.type == "field_identifier":
|
||||
field_name = _read(child)
|
||||
target = _make_id(container_nid, field_name)
|
||||
if target in sibling_nids and target != caller_nid:
|
||||
pair = (caller_nid, target)
|
||||
if pair not in seen_calls:
|
||||
seen_calls.add(pair)
|
||||
add_edge(caller_nid, target, "accesses",
|
||||
n.start_point[0] + 1,
|
||||
confidence="EXTRACTED", weight=1.0)
|
||||
elif n.type == "selector_expression":
|
||||
# @selector(doSomething:withParam:) — compile-time method ref.
|
||||
# Match the selector name EXACTLY (a method id is
|
||||
# _make_id(container, name)) against every class's methods, and emit
|
||||
# only when exactly one method matches, to avoid ambiguous fan-out.
|
||||
# Exact match (not a suffix) keeps -doThing distinct from
|
||||
# -reallyDoThing (#1475).
|
||||
sel_parts = [_read(c) for c in n.children if c.type == "identifier"]
|
||||
sel_name = "".join(sel_parts)
|
||||
if sel_name:
|
||||
matches = sorted({
|
||||
m for m, _, cont in method_bodies
|
||||
if m == _make_id(cont, sel_name) and m != caller_nid
|
||||
})
|
||||
if len(matches) == 1:
|
||||
pair = (caller_nid, matches[0])
|
||||
if pair not in seen_calls:
|
||||
seen_calls.add(pair)
|
||||
add_edge(caller_nid, matches[0], "calls",
|
||||
n.start_point[0] + 1,
|
||||
confidence="EXTRACTED", weight=1.0,
|
||||
context="call")
|
||||
for child in n.children:
|
||||
walk_calls(child)
|
||||
walk_calls(body_node)
|
||||
|
||||
result = {"nodes": nodes, "edges": edges, "raw_calls": raw_calls,
|
||||
"input_tokens": 0, "output_tokens": 0}
|
||||
if objc_type_table:
|
||||
result["objc_type_table"] = {"path": str_path, "table": objc_type_table}
|
||||
return result
|
||||
@@ -0,0 +1,688 @@
|
||||
"""pascal — moved verbatim from graphify/extract.py."""
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from graphify.extractors.base import _file_stem, _make_id
|
||||
from graphify.extractors.resolution import _pascal_resolve_class, _pascal_resolve_unit
|
||||
from pathlib import Path
|
||||
from typing import Any, Callable
|
||||
|
||||
|
||||
_PAS_TOKEN_RE = re.compile(
|
||||
r"'(?:''|[^'])*'"
|
||||
r"|\{[^}]*\}"
|
||||
r"|\(\*.*?\*\)"
|
||||
r"|//[^\n]*",
|
||||
re.DOTALL,
|
||||
)
|
||||
|
||||
_PAS_MODULE_RE = re.compile(
|
||||
r"\b(unit|program|library)\s+([A-Za-z_][\w.]*)\s*;",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
|
||||
_PAS_USES_RE = re.compile(
|
||||
r"\buses\b\s*([^;]+);",
|
||||
re.IGNORECASE | re.DOTALL,
|
||||
)
|
||||
|
||||
_PAS_TYPE_HEADER_RE = re.compile(
|
||||
r"\b(?P<name>[A-Za-z_]\w*)(?:\s*<[^>]+>)?\s*=\s*(?:packed\s+)?"
|
||||
r"(?P<kind>class|interface)\b"
|
||||
r"(?:\s*\(\s*(?P<bases>[^)]*)\s*\))?",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
|
||||
_PAS_END_SEMI_RE = re.compile(r"\bend\s*;", re.IGNORECASE)
|
||||
|
||||
_PAS_METHOD_DECL_RE = re.compile(
|
||||
r"\b(?:procedure|function|constructor|destructor)\s+"
|
||||
r"(?P<name>[A-Za-z_]\w*)"
|
||||
r"(?:\s*\([^)]*\))?"
|
||||
r"(?:\s*:\s*[\w<>,\s.]+)?"
|
||||
r"\s*;",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
|
||||
_PAS_IMPL_HEADER_RE = re.compile(
|
||||
r"\b(?:procedure|function|constructor|destructor)\s+"
|
||||
r"(?P<qual>[A-Za-z_]\w*(?:\.[A-Za-z_]\w*)?)"
|
||||
r"(?:\s*<[^>]+>)?"
|
||||
r"(?:\s*\([^)]*\))?"
|
||||
r"(?:\s*:\s*[\w<>,\s.]+)?"
|
||||
r"\s*;",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
|
||||
_PAS_BEGIN_END_TOKEN_RE = re.compile(
|
||||
r"\b(begin|end|case|try|asm|record)\b", re.IGNORECASE
|
||||
)
|
||||
|
||||
_PAS_CALL_RE = re.compile(r"\b([A-Za-z_]\w*(?:\.[A-Za-z_]\w*)*)\s*[(;]")
|
||||
|
||||
_PAS_KEYWORDS = frozenset({
|
||||
"begin", "end", "if", "then", "else", "while", "do", "for", "to",
|
||||
"downto", "repeat", "until", "case", "of", "try", "finally", "except",
|
||||
"with", "inherited", "result", "var", "const", "type", "nil", "true",
|
||||
"false", "exit", "break", "continue", "uses", "unit", "program",
|
||||
"library", "interface", "implementation", "initialization", "finalization",
|
||||
"procedure", "function", "constructor", "destructor", "class", "record",
|
||||
"object", "array", "string", "integer", "boolean", "real", "char",
|
||||
"writeln", "write", "readln", "read", "assigned", "length", "high",
|
||||
"low", "inc", "dec", "new", "dispose", "setlength", "copy", "pos",
|
||||
"trim", "format", "inttostr", "strtoint", "ord", "chr", "sizeof",
|
||||
"create", "free", "destroy",
|
||||
})
|
||||
|
||||
def _pascal_strip_comments(text: str) -> str:
|
||||
"""Strip Pascal comments ({}, (* *), //) while preserving newlines."""
|
||||
def _sub(m: re.Match) -> str:
|
||||
tok = m.group(0)
|
||||
if tok.startswith("'"):
|
||||
return tok
|
||||
return "".join(c if c == "\n" else " " for c in tok)
|
||||
return _PAS_TOKEN_RE.sub(_sub, text)
|
||||
|
||||
def _pascal_split_sections(text: str) -> tuple[str, int, str, int]:
|
||||
"""Split into (iface_text, iface_offset, impl_text, impl_offset).
|
||||
Files without interface/implementation sections (dpr/lpr/inc) return
|
||||
the whole text as impl with offset 0.
|
||||
"""
|
||||
iface_m = re.search(r"\binterface\b", text, re.IGNORECASE)
|
||||
impl_m = re.search(r"\bimplementation\b", text, re.IGNORECASE)
|
||||
if iface_m and impl_m:
|
||||
iface_off = iface_m.end()
|
||||
impl_off = impl_m.end()
|
||||
end_m = re.search(
|
||||
r"\b(initialization|finalization)\b", text[impl_off:], re.IGNORECASE
|
||||
)
|
||||
impl_end = impl_off + end_m.start() if end_m else len(text)
|
||||
return text[iface_off:impl_m.start()], iface_off, text[impl_off:impl_end], impl_off
|
||||
return "", 0, text, 0
|
||||
|
||||
def _pascal_split_uses(s: str) -> list[str]:
|
||||
"""Split a uses list string, handling 'Foo in ''bar.pas''' syntax."""
|
||||
out = []
|
||||
for chunk in s.split(","):
|
||||
name = re.split(r"\s+in\s+", chunk.strip(), maxsplit=1, flags=re.IGNORECASE)[0]
|
||||
name = name.strip().strip(";")
|
||||
if name and re.match(r"[A-Za-z_][\w.]*$", name):
|
||||
out.append(name)
|
||||
return out
|
||||
|
||||
def _pascal_split_bases(s: str) -> list[str]:
|
||||
"""Split inheritance list, handling generics like TList<T, U>."""
|
||||
out, depth, buf = [], 0, []
|
||||
for ch in s:
|
||||
if ch == "<":
|
||||
depth += 1
|
||||
buf.append(ch)
|
||||
elif ch == ">":
|
||||
depth -= 1
|
||||
buf.append(ch)
|
||||
elif ch == "," and depth == 0:
|
||||
name = re.sub(r"<.*$", "", "".join(buf).strip())
|
||||
if name:
|
||||
out.append(name)
|
||||
buf = []
|
||||
else:
|
||||
buf.append(ch)
|
||||
name = re.sub(r"<.*$", "", "".join(buf).strip())
|
||||
if name:
|
||||
out.append(name)
|
||||
return [n for n in out if re.match(r"[A-Za-z_]\w*$", n)]
|
||||
|
||||
def _pascal_find_body(text: str, start: int) -> tuple[int, int]:
|
||||
"""Find balanced begin..end after start. Returns (body_start, body_end).
|
||||
Returns (0, 0) if no begin found.
|
||||
"""
|
||||
m = re.search(r"\bbegin\b", text[start:], re.IGNORECASE)
|
||||
if not m:
|
||||
return (0, 0)
|
||||
body_start = start + m.end()
|
||||
depth = 1
|
||||
for tok in _PAS_BEGIN_END_TOKEN_RE.finditer(text, body_start):
|
||||
kw = tok.group(1).lower()
|
||||
if kw in ("begin", "case", "try", "asm", "record"):
|
||||
depth += 1
|
||||
elif kw == "end":
|
||||
depth -= 1
|
||||
if depth == 0:
|
||||
return (body_start, tok.start())
|
||||
return (body_start, len(text))
|
||||
|
||||
def _resolve_pascal_callee_factory(
|
||||
records: list[tuple],
|
||||
edges: list[dict],
|
||||
module_nid: str,
|
||||
) -> Callable[[str, str], str | None]:
|
||||
"""Build a scoped call resolver for a single Pascal/Delphi file.
|
||||
|
||||
``records`` is the list of raw per-procedure tuples produced by either
|
||||
Pascal extractor; only the trailing ``(..., container, name_lower)``
|
||||
fields and the leading ``proc_nid`` are used, so both extractors' tuple
|
||||
shapes work unmodified (regex: proc_nid, line, body_text, container,
|
||||
name_lower; tree-sitter: proc_nid, body_node, container, name_lower).
|
||||
|
||||
Resolution order for a call to ``name_lower`` from ``caller_nid``:
|
||||
1. A method declared on the caller's own class.
|
||||
2. A method declared on an ancestor class (BFS up ``inherits`` edges,
|
||||
which are already resolved by this point).
|
||||
3. A file-level free function (declared directly under the module).
|
||||
4. A global by-name match, but only when unambiguous (exactly one
|
||||
procedure with that name anywhere in the file).
|
||||
|
||||
Returns None (no edge emitted) when the name is ambiguous at every
|
||||
level -- guessing at a same-named method on an unrelated class is worse
|
||||
than omitting the edge. Same-named methods on unrelated classes are a
|
||||
common Pascal/Delphi pattern (property accessors, generated wrapper
|
||||
classes such as TLB import units): without this scoping, a flat
|
||||
file-wide by-name lookup silently collapses onto whichever declaration
|
||||
happens to be inserted last, producing wrong cross-class edges. Mirrors
|
||||
the "god-node guard" already used by ``resolve_ruby_member_calls`` for
|
||||
the analogous Ruby ambiguous-method-name problem.
|
||||
"""
|
||||
class_bases: dict[str, list[str]] = {}
|
||||
for e in edges:
|
||||
if e.get("relation") == "inherits":
|
||||
class_bases.setdefault(e["source"], []).append(e["target"])
|
||||
|
||||
class_procs: dict[str, dict[str, list[str]]] = {}
|
||||
module_procs: dict[str, list[str]] = {}
|
||||
global_procs: dict[str, list[str]] = {}
|
||||
proc_owner: dict[str, str] = {}
|
||||
for rec in records:
|
||||
proc_nid, container, name_lower = rec[0], rec[-2], rec[-1]
|
||||
proc_owner[proc_nid] = container
|
||||
global_procs.setdefault(name_lower, []).append(proc_nid)
|
||||
if container == module_nid:
|
||||
module_procs.setdefault(name_lower, []).append(proc_nid)
|
||||
else:
|
||||
class_procs.setdefault(container, {}).setdefault(name_lower, []).append(proc_nid)
|
||||
|
||||
def _resolve(caller_nid: str, name_lower: str) -> str | None:
|
||||
owner = proc_owner.get(caller_nid)
|
||||
if owner is not None:
|
||||
candidates = class_procs.get(owner, {}).get(name_lower)
|
||||
if candidates:
|
||||
return candidates[0] if len(candidates) == 1 else None
|
||||
seen_bases: set[str] = set()
|
||||
queue = list(class_bases.get(owner, []))
|
||||
while queue:
|
||||
base = queue.pop(0)
|
||||
if base in seen_bases:
|
||||
continue
|
||||
seen_bases.add(base)
|
||||
candidates = class_procs.get(base, {}).get(name_lower)
|
||||
if candidates:
|
||||
return candidates[0] if len(candidates) == 1 else None
|
||||
queue.extend(class_bases.get(base, []))
|
||||
candidates = module_procs.get(name_lower)
|
||||
if candidates:
|
||||
return candidates[0] if len(candidates) == 1 else None
|
||||
candidates = global_procs.get(name_lower)
|
||||
if candidates and len(candidates) == 1:
|
||||
return candidates[0]
|
||||
return None
|
||||
|
||||
return _resolve
|
||||
|
||||
|
||||
def _extract_pascal_regex(path: Path) -> dict:
|
||||
"""Regex fallback for Pascal/Delphi extraction when tree-sitter-pascal
|
||||
is unavailable. Produces the same node/edge schema as the tree-sitter pass.
|
||||
"""
|
||||
try:
|
||||
raw = path.read_text(encoding="utf-8", errors="replace")
|
||||
except Exception as exc:
|
||||
return {"nodes": [], "edges": [], "error": str(exc)}
|
||||
|
||||
str_path = str(path)
|
||||
stem = _file_stem(path)
|
||||
nodes: list[dict] = []
|
||||
edges: list[dict] = []
|
||||
seen_ids: set[str] = set()
|
||||
seen_call_pairs: set[tuple[str, str]] = set()
|
||||
seen_edges: set[tuple[str, str, str]] = set()
|
||||
|
||||
def _add_node(nid: str, label: str, line: int) -> None:
|
||||
if nid not in seen_ids:
|
||||
seen_ids.add(nid)
|
||||
nodes.append({
|
||||
"id": nid,
|
||||
"label": label,
|
||||
"file_type": "code",
|
||||
"source_file": str_path,
|
||||
"source_location": f"L{line}",
|
||||
})
|
||||
|
||||
def _add_edge(src: str, tgt: str, relation: str, line: int, context: str | None = None) -> None:
|
||||
# A class method declared in the interface section and defined in the
|
||||
# implementation section both emit a `method` edge to the same node, so
|
||||
# dedup on (src, tgt, relation) to keep the graph from carrying doubled
|
||||
# method/contains/inherits edges (mirrors _add_node's seen_ids guard).
|
||||
key = (src, tgt, relation)
|
||||
if key in seen_edges:
|
||||
return
|
||||
seen_edges.add(key)
|
||||
edge: dict = {
|
||||
"source": src,
|
||||
"target": tgt,
|
||||
"relation": relation,
|
||||
"confidence": "EXTRACTED",
|
||||
"source_file": str_path,
|
||||
"source_location": f"L{line}",
|
||||
"weight": 1.0,
|
||||
}
|
||||
if context:
|
||||
edge["context"] = context
|
||||
edges.append(edge)
|
||||
|
||||
def _lineno(text: str, offset: int) -> int:
|
||||
return text.count("\n", 0, offset) + 1
|
||||
|
||||
file_nid = _make_id(str_path)
|
||||
_add_node(file_nid, path.name, 1)
|
||||
|
||||
stripped = _pascal_strip_comments(raw)
|
||||
|
||||
# Module header
|
||||
module_nid = file_nid
|
||||
mod_m = _PAS_MODULE_RE.search(stripped)
|
||||
if mod_m:
|
||||
mod_name = mod_m.group(2)
|
||||
module_nid = _make_id(stem, mod_name)
|
||||
_add_node(module_nid, mod_name, _lineno(stripped, mod_m.start()))
|
||||
_add_edge(file_nid, module_nid, "contains", _lineno(stripped, mod_m.start()))
|
||||
|
||||
iface_text, iface_off, impl_text, impl_off = _pascal_split_sections(stripped)
|
||||
|
||||
# Uses clauses
|
||||
for section_text, section_off in ((iface_text, iface_off), (impl_text, impl_off)):
|
||||
for um in _PAS_USES_RE.finditer(section_text):
|
||||
line = _lineno(stripped, section_off + um.start())
|
||||
for unit_name in _pascal_split_uses(um.group(1)):
|
||||
tgt_nid = _pascal_resolve_unit(path, unit_name)
|
||||
_add_edge(module_nid, tgt_nid, "imports", line, context="import")
|
||||
|
||||
# Type declarations (classes / interfaces) in interface section
|
||||
search_text = iface_text if iface_text else stripped
|
||||
search_off = iface_off if iface_text else 0
|
||||
pos = 0
|
||||
while pos < len(search_text):
|
||||
hm = _PAS_TYPE_HEADER_RE.search(search_text, pos)
|
||||
if not hm:
|
||||
break
|
||||
type_name = hm.group("name")
|
||||
bases_raw = hm.group("bases") or ""
|
||||
line = _lineno(stripped, search_off + hm.start())
|
||||
cls_nid = _make_id(stem, type_name)
|
||||
_add_node(cls_nid, type_name, line)
|
||||
_add_edge(module_nid, cls_nid, "contains", line)
|
||||
|
||||
for base_name in _pascal_split_bases(bases_raw):
|
||||
same_file_nid = _make_id(stem, base_name)
|
||||
if same_file_nid in seen_ids:
|
||||
# Base class already declared earlier in this same file --
|
||||
# reuse its real node instead of the cross-file/stub lookup
|
||||
# below (which assumes one-class-per-file and would create a
|
||||
# duplicate node for a base class that shares this file).
|
||||
base_nid = same_file_nid
|
||||
else:
|
||||
resolved = _pascal_resolve_class(path, base_name)
|
||||
if resolved:
|
||||
# Cross-file base class found on disk -- its real node
|
||||
# arrives via THAT file's own extraction. Do not add a
|
||||
# duplicate stub here: it would carry this file's
|
||||
# source_file (wrong -- it belongs to the base class's
|
||||
# own file) and collide with the real node under
|
||||
# cross-file id disambiguation, producing two different
|
||||
# salted ids for what should be one class (breaks
|
||||
# cross-file `inherits`-chain resolution downstream).
|
||||
base_nid = resolved
|
||||
else:
|
||||
base_nid = _make_id(base_name)
|
||||
if base_nid not in seen_ids:
|
||||
_add_node(base_nid, base_name, line)
|
||||
_add_edge(cls_nid, base_nid, "inherits", line)
|
||||
|
||||
# Find class body (up to next end;)
|
||||
end_m = _PAS_END_SEMI_RE.search(search_text, hm.end())
|
||||
body_text = search_text[hm.end():end_m.start()] if end_m else ""
|
||||
body_off = search_off + hm.end()
|
||||
|
||||
# Forward method declarations inside the class body
|
||||
for mm in _PAS_METHOD_DECL_RE.finditer(body_text):
|
||||
mname = mm.group("name")
|
||||
mline = _lineno(stripped, body_off + mm.start())
|
||||
method_nid = _make_id(cls_nid, mname)
|
||||
_add_node(method_nid, f"{mname}()", mline)
|
||||
_add_edge(cls_nid, method_nid, "method", mline)
|
||||
|
||||
pos = end_m.end() if end_m else len(search_text)
|
||||
|
||||
# Implementation headers (procedure/function/constructor/destructor)
|
||||
impl_records: list[tuple[str, int, str, str, str]] = []
|
||||
# (proc_nid, line, body_text, container, name_lower)
|
||||
for fm in _PAS_IMPL_HEADER_RE.finditer(impl_text):
|
||||
qualified = fm.group("qual")
|
||||
line = _lineno(stripped, impl_off + fm.start())
|
||||
if "." in qualified:
|
||||
cls_part, method_part = qualified.split(".", 1)
|
||||
cls_nid = _make_id(stem, cls_part)
|
||||
container = cls_nid if cls_nid in seen_ids else module_nid
|
||||
relation = "method" if cls_nid in seen_ids else "contains"
|
||||
label = f"{method_part}()"
|
||||
name_lower = method_part.lower()
|
||||
else:
|
||||
container, relation = module_nid, "contains"
|
||||
label = f"{qualified}()"
|
||||
name_lower = qualified.lower()
|
||||
proc_nid = _make_id(stem, qualified)
|
||||
_add_node(proc_nid, label, line)
|
||||
_add_edge(container, proc_nid, relation, line)
|
||||
|
||||
body_start, body_end = _pascal_find_body(impl_text, fm.end())
|
||||
body_text = impl_text[body_start:body_end] if body_start else ""
|
||||
impl_records.append((proc_nid, line, body_text, container, name_lower))
|
||||
|
||||
# Intra-file call edges, scoped by the caller's own class, then its
|
||||
# ancestor chain (via `inherits` edges already emitted above), then
|
||||
# file-level free functions; fall back to a global by-name match only
|
||||
# when it is unambiguous (single owner across the file). Prevents
|
||||
# same-named methods on unrelated classes (property accessors, generated
|
||||
# wrapper classes such as TLB import units, etc. -- a common Pascal/Delphi
|
||||
# pattern) from collapsing into an arbitrary cross-class edge.
|
||||
callee_nid = _resolve_pascal_callee_factory(impl_records, edges, module_nid)
|
||||
raw_calls: list[dict] = []
|
||||
for caller_nid, caller_line, body_text, _container, _name_lower in impl_records:
|
||||
for cm in _PAS_CALL_RE.finditer(body_text):
|
||||
callee_name = cm.group(1).split(".")[-1].lower()
|
||||
if callee_name in _PAS_KEYWORDS:
|
||||
continue
|
||||
call_line = caller_line + body_text.count("\n", 0, cm.start())
|
||||
target_nid = callee_nid(caller_nid, callee_name)
|
||||
if target_nid == caller_nid:
|
||||
continue
|
||||
if not target_nid:
|
||||
# Not resolvable within this file (e.g. inherited from a base
|
||||
# class declared in another file) -- report for the
|
||||
# cross-file resolver (graphify.pascal_resolution) instead of
|
||||
# guessing or dropping it silently.
|
||||
raw_calls.append({
|
||||
"source_file": str_path,
|
||||
"source_location": f"L{call_line}",
|
||||
"caller_nid": caller_nid,
|
||||
"callee": callee_name,
|
||||
})
|
||||
continue
|
||||
pair = (caller_nid, target_nid)
|
||||
if pair in seen_call_pairs:
|
||||
continue
|
||||
seen_call_pairs.add(pair)
|
||||
_add_edge(caller_nid, target_nid, "calls", call_line, context="call")
|
||||
|
||||
return {
|
||||
"nodes": nodes, "edges": edges, "input_tokens": 0, "output_tokens": 0,
|
||||
"raw_calls": raw_calls,
|
||||
}
|
||||
|
||||
def extract_pascal(path: Path) -> dict:
|
||||
"""Extract units, classes, procedures, uses-imports, and calls from Pascal/Delphi files.
|
||||
|
||||
Produces nodes for:
|
||||
- The file itself
|
||||
- unit / program / library declarations
|
||||
- class and interface type declarations
|
||||
- procedure / function implementations (including qualified TClass.Method names)
|
||||
|
||||
Produces edges for:
|
||||
- file --contains--> module
|
||||
- module --imports--> other file node (via uses clause, resolved to path-based IDs)
|
||||
- class --inherits--> base class
|
||||
- class/module --contains--> method forward declaration
|
||||
- class/module --contains--> procedure/function implementation
|
||||
- procedure --calls--> other procedure (within the same file)
|
||||
|
||||
Uses tree-sitter-pascal when available; falls back to a regex-based extractor
|
||||
(_extract_pascal_regex) when it isn't installed or fails to parse, so Pascal
|
||||
extraction works out of the box without an extra pip install.
|
||||
"""
|
||||
try:
|
||||
import tree_sitter_pascal as tspascal
|
||||
from tree_sitter import Language, Parser
|
||||
except ImportError:
|
||||
return _extract_pascal_regex(path)
|
||||
|
||||
try:
|
||||
language = Language(tspascal.language())
|
||||
parser = Parser(language)
|
||||
source = path.read_bytes()
|
||||
tree = parser.parse(source)
|
||||
root = tree.root_node
|
||||
except Exception:
|
||||
return _extract_pascal_regex(path)
|
||||
|
||||
stem = _file_stem(path)
|
||||
str_path = str(path)
|
||||
nodes: list[dict] = []
|
||||
edges: list[dict] = []
|
||||
seen_ids: set[str] = set()
|
||||
seen_edges: set[tuple[str, str, str]] = set()
|
||||
proc_bodies: list[tuple[str, Any, str, str]] = []
|
||||
# (proc_nid, body_node, container, name_lower)
|
||||
|
||||
def _read(node) -> str: # type: ignore[no-untyped-def]
|
||||
return source[node.start_byte:node.end_byte].decode("utf-8", errors="replace")
|
||||
|
||||
def add_node(nid: str, label: str, line: int) -> None:
|
||||
if nid not in seen_ids:
|
||||
seen_ids.add(nid)
|
||||
nodes.append({
|
||||
"id": nid, "label": label, "file_type": "code",
|
||||
"source_file": str_path, "source_location": f"L{line}",
|
||||
})
|
||||
|
||||
def add_edge(
|
||||
src: str, tgt: str, relation: str, line: int,
|
||||
confidence: str = "EXTRACTED", weight: float = 1.0,
|
||||
context: str | None = None,
|
||||
) -> None:
|
||||
# A class method declared in the interface section and defined in the
|
||||
# implementation section both emit a `method` edge to the same node, so
|
||||
# dedup on (src, tgt, relation) to keep the graph from carrying doubled
|
||||
# method/contains/inherits edges (mirrors add_node's seen_ids guard).
|
||||
key = (src, tgt, relation)
|
||||
if key in seen_edges:
|
||||
return
|
||||
seen_edges.add(key)
|
||||
edge: dict[str, Any] = {
|
||||
"source": src, "target": tgt, "relation": relation,
|
||||
"confidence": confidence, "source_file": str_path,
|
||||
"source_location": f"L{line}", "weight": weight,
|
||||
}
|
||||
if context:
|
||||
edge["context"] = context
|
||||
edges.append(edge)
|
||||
|
||||
file_nid = _make_id(str(path))
|
||||
add_node(file_nid, path.name, 1)
|
||||
module_nid = file_nid
|
||||
|
||||
def _proc_name(header_node) -> str | None: # type: ignore[no-untyped-def]
|
||||
name_node = header_node.child_by_field_name("name")
|
||||
if name_node:
|
||||
return _read(name_node)
|
||||
for child in header_node.children:
|
||||
if child.type in ("identifier", "genericDot", "genericTpl"):
|
||||
return _read(child)
|
||||
return None
|
||||
|
||||
def walk(node, parent_nid: str) -> None: # type: ignore[no-untyped-def]
|
||||
nonlocal module_nid
|
||||
t = node.type
|
||||
line = node.start_point[0] + 1
|
||||
|
||||
if t in ("unit", "program", "library"):
|
||||
name_node = next((c for c in node.children if c.type == "moduleName"), None)
|
||||
mod_name = _read(name_node) if name_node else path.stem
|
||||
mod_nid = _make_id(stem, mod_name)
|
||||
add_node(mod_nid, mod_name, line)
|
||||
add_edge(file_nid, mod_nid, "contains", line)
|
||||
module_nid = mod_nid
|
||||
for child in node.children:
|
||||
walk(child, mod_nid)
|
||||
return
|
||||
|
||||
if t == "declUses":
|
||||
for child in node.children:
|
||||
if child.type == "moduleName":
|
||||
mod_name = _read(child)
|
||||
tgt_nid = _pascal_resolve_unit(path, mod_name)
|
||||
add_edge(parent_nid, tgt_nid, "imports", line, context="import")
|
||||
return
|
||||
|
||||
if t == "declType":
|
||||
type_name = None
|
||||
kind_node = None
|
||||
for child in node.children:
|
||||
if child.type == "identifier" and type_name is None:
|
||||
type_name = _read(child)
|
||||
elif child.type in ("declClass", "declIntf", "declHelper") and kind_node is None:
|
||||
kind_node = child
|
||||
if type_name and kind_node:
|
||||
cls_nid = _make_id(stem, type_name)
|
||||
add_node(cls_nid, type_name, line)
|
||||
add_edge(parent_nid, cls_nid, "contains", line)
|
||||
for child in kind_node.children:
|
||||
if child.type == "typeref":
|
||||
base_name = _read(child)
|
||||
base_nid = _make_id(stem, base_name)
|
||||
if base_nid not in seen_ids:
|
||||
# Try cross-file resolution (TFooBar → FooBar.pas)
|
||||
resolved = _pascal_resolve_class(path, base_name)
|
||||
if resolved:
|
||||
# Cross-file base class found on disk -- its
|
||||
# real node arrives via THAT file's own
|
||||
# extraction. Do not add a duplicate stub
|
||||
# here: it would carry this file's
|
||||
# source_file (wrong) and collide with the
|
||||
# real node under cross-file id
|
||||
# disambiguation, producing two different
|
||||
# salted ids for what should be one class.
|
||||
base_nid = resolved
|
||||
else:
|
||||
base_nid = _make_id(base_name)
|
||||
if base_nid not in seen_ids:
|
||||
# Stub for RTL/external base classes.
|
||||
add_node(base_nid, base_name, line)
|
||||
add_edge(cls_nid, base_nid, "inherits", line)
|
||||
for child in kind_node.children:
|
||||
walk(child, cls_nid)
|
||||
return
|
||||
for child in node.children:
|
||||
walk(child, parent_nid)
|
||||
return
|
||||
|
||||
if t == "declProcFwd":
|
||||
header = next((c for c in node.children if c.type == "declProc"), None)
|
||||
if header:
|
||||
name = _proc_name(header)
|
||||
if name and "." not in name:
|
||||
method_nid = _make_id(parent_nid, name)
|
||||
add_node(method_nid, f"{name}()", line)
|
||||
add_edge(parent_nid, method_nid, "method", line)
|
||||
return
|
||||
|
||||
if t == "defProc":
|
||||
header = next((c for c in node.children if c.type == "declProc"), None)
|
||||
body_node = next((c for c in node.children if c.type == "block"), None)
|
||||
if not header:
|
||||
for child in node.children:
|
||||
walk(child, parent_nid)
|
||||
return
|
||||
name = _proc_name(header)
|
||||
if not name:
|
||||
for child in node.children:
|
||||
walk(child, parent_nid)
|
||||
return
|
||||
container = parent_nid
|
||||
if "." in name:
|
||||
parts = name.split(".", 1)
|
||||
cls_nid = _make_id(stem, parts[0])
|
||||
if cls_nid in seen_ids:
|
||||
container = cls_nid
|
||||
label = f"{parts[-1]}()"
|
||||
else:
|
||||
label = f"{name}()"
|
||||
proc_nid = _make_id(stem, name)
|
||||
add_node(proc_nid, label, line)
|
||||
add_edge(
|
||||
container, proc_nid,
|
||||
"method" if container != parent_nid else "contains",
|
||||
line,
|
||||
)
|
||||
if body_node:
|
||||
proc_bodies.append((proc_nid, body_node, container, label.removesuffix("()").lower()))
|
||||
return
|
||||
|
||||
for child in node.children:
|
||||
walk(child, parent_nid)
|
||||
|
||||
walk(root, file_nid)
|
||||
|
||||
# Second pass: resolve calls inside procedure/function bodies, scoped by
|
||||
# the caller's own class, then its ancestor chain, then file-level free
|
||||
# functions, falling back to an unambiguous global match (see
|
||||
# _resolve_pascal_callee_factory).
|
||||
resolve_callee = _resolve_pascal_callee_factory(proc_bodies, edges, module_nid)
|
||||
seen_call_pairs: set[tuple[str, str]] = set()
|
||||
raw_calls: list[dict] = []
|
||||
|
||||
def _emit_or_report(caller_nid: str, name_lower: str, line: int) -> None:
|
||||
target = resolve_callee(caller_nid, name_lower)
|
||||
if target == caller_nid:
|
||||
return
|
||||
if not target:
|
||||
# Not resolvable within this file (e.g. inherited from a base
|
||||
# class declared in another file) -- report for the cross-file
|
||||
# resolver (graphify.pascal_resolution) instead of guessing or
|
||||
# dropping it silently.
|
||||
raw_calls.append({
|
||||
"source_file": str_path,
|
||||
"source_location": f"L{line}",
|
||||
"caller_nid": caller_nid,
|
||||
"callee": name_lower,
|
||||
})
|
||||
return
|
||||
pair = (caller_nid, target)
|
||||
if pair not in seen_call_pairs:
|
||||
seen_call_pairs.add(pair)
|
||||
add_edge(caller_nid, target, "calls", line, context="call")
|
||||
|
||||
def walk_calls(node, caller_nid: str) -> None: # type: ignore[no-untyped-def]
|
||||
if node.type == "exprCall":
|
||||
callee_text = None
|
||||
for child in node.children:
|
||||
if child.is_named and child.type not in ("exprArgs",):
|
||||
callee_text = _read(child).split(".")[-1]
|
||||
break
|
||||
if callee_text:
|
||||
_emit_or_report(caller_nid, callee_text.lower(), node.start_point[0] + 1)
|
||||
elif node.type == "statement":
|
||||
# Pascal bare procedure calls with no args: `Reset;`
|
||||
# tree-sitter represents these as statement → identifier (no exprCall wrapper)
|
||||
named = [c for c in node.children if c.is_named]
|
||||
if len(named) == 1 and named[0].type == "identifier":
|
||||
callee_text = _read(named[0])
|
||||
_emit_or_report(caller_nid, callee_text.lower(), node.start_point[0] + 1)
|
||||
for child in node.children:
|
||||
walk_calls(child, caller_nid)
|
||||
|
||||
for proc_nid, body_node, _container, _name_lower in proc_bodies:
|
||||
walk_calls(body_node, proc_nid)
|
||||
|
||||
return {
|
||||
"nodes": nodes, "edges": edges, "input_tokens": 0, "output_tokens": 0,
|
||||
"raw_calls": raw_calls,
|
||||
}
|
||||
@@ -0,0 +1,196 @@
|
||||
"""Pascal_forms extractor. Moved verbatim from graphify/extract.py."""
|
||||
from __future__ import annotations
|
||||
|
||||
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
from graphify.extractors.base import _file_stem, _make_id
|
||||
|
||||
|
||||
def extract_lazarus_form(path: Path) -> dict:
|
||||
"""Extract component hierarchy from Lazarus .lfm form files.
|
||||
|
||||
.lfm is a text-based declarative format for UI component trees, structured as:
|
||||
object ComponentName: TClassName
|
||||
PropertyName = Value
|
||||
OnEvent = HandlerName
|
||||
object ChildName: TChildClass
|
||||
...
|
||||
end
|
||||
end
|
||||
|
||||
Produces nodes for:
|
||||
- The form file itself
|
||||
- Each component class encountered (TForm1, TButton, TPanel, ...)
|
||||
- Event handler names referenced by OnXxx properties
|
||||
|
||||
Produces edges for:
|
||||
- file --contains--> root form class
|
||||
- parent component --contains--> child component class
|
||||
- component --references--> event handler (context: "event")
|
||||
"""
|
||||
try:
|
||||
text = path.read_text(encoding="utf-8", errors="replace")
|
||||
except Exception as e:
|
||||
return {"nodes": [], "edges": [], "error": str(e)}
|
||||
|
||||
import re
|
||||
str_path = str(path)
|
||||
stem = _file_stem(path)
|
||||
nodes: list[dict] = []
|
||||
edges: list[dict] = []
|
||||
seen_ids: set[str] = set()
|
||||
seen_edge_pairs: set[tuple[str, str, str]] = set()
|
||||
|
||||
def add_node(nid: str, label: str, line: int) -> None:
|
||||
if nid not in seen_ids:
|
||||
seen_ids.add(nid)
|
||||
nodes.append({
|
||||
"id": nid, "label": label, "file_type": "code",
|
||||
"source_file": str_path, "source_location": f"L{line}",
|
||||
})
|
||||
|
||||
def add_edge(
|
||||
src: str, tgt: str, relation: str, line: int,
|
||||
context: str | None = None,
|
||||
) -> None:
|
||||
key = (src, tgt, relation)
|
||||
if key in seen_edge_pairs:
|
||||
return
|
||||
seen_edge_pairs.add(key)
|
||||
edge: dict[str, Any] = {
|
||||
"source": src, "target": tgt, "relation": relation,
|
||||
"confidence": "EXTRACTED", "source_file": str_path,
|
||||
"source_location": f"L{line}", "weight": 1.0,
|
||||
}
|
||||
if context:
|
||||
edge["context"] = context
|
||||
edges.append(edge)
|
||||
|
||||
file_nid = _make_id(str(path))
|
||||
add_node(file_nid, path.name, 1)
|
||||
|
||||
obj_re = re.compile(r"^\s*object\s+\w+\s*:\s*(\w+)", re.IGNORECASE)
|
||||
event_re = re.compile(r"^\s*On\w+\s*=\s*(\w+)", re.IGNORECASE)
|
||||
end_re = re.compile(r"^\s*end\s*$", re.IGNORECASE)
|
||||
|
||||
# Stack of node IDs representing the nesting of object...end blocks
|
||||
stack: list[str] = [file_nid]
|
||||
|
||||
for lineno, line in enumerate(text.splitlines(), 1):
|
||||
m = obj_re.match(line)
|
||||
if m:
|
||||
class_name = m.group(1)
|
||||
nid = _make_id(stem, class_name)
|
||||
add_node(nid, class_name, lineno)
|
||||
add_edge(stack[-1], nid, "contains", lineno)
|
||||
stack.append(nid)
|
||||
continue
|
||||
|
||||
m = event_re.match(line)
|
||||
if m and len(stack) > 1:
|
||||
handler = m.group(1)
|
||||
handler_nid = _make_id(stem, handler)
|
||||
add_node(handler_nid, f"{handler}()", lineno)
|
||||
add_edge(stack[-1], handler_nid, "references", lineno, context="event")
|
||||
continue
|
||||
|
||||
if end_re.match(line) and len(stack) > 1:
|
||||
stack.pop()
|
||||
|
||||
return {"nodes": nodes, "edges": edges, "input_tokens": 0, "output_tokens": 0}
|
||||
|
||||
def extract_delphi_form(path: Path) -> dict:
|
||||
"""Extract component hierarchy from Delphi .dfm form files.
|
||||
|
||||
.dfm files come in two formats:
|
||||
- Text (same `object Name: TClassName ... end` syntax as .lfm)
|
||||
- Binary (starts with a TPF0/FF0A magic header — unreadable as text)
|
||||
|
||||
Binary .dfm files are skipped gracefully: an empty result is returned
|
||||
so the rest of the pipeline is unaffected. Convert binary forms to
|
||||
text in the Delphi IDE via File → Save As (Text DFM) if you want them
|
||||
indexed.
|
||||
|
||||
Text .dfm files are parsed identically to .lfm: component containment
|
||||
(`contains`) and event handler references (`references`, context "event").
|
||||
"""
|
||||
try:
|
||||
raw = path.read_bytes()
|
||||
except Exception as e:
|
||||
return {"nodes": [], "edges": [], "error": str(e)}
|
||||
|
||||
# Detect binary DFM: Delphi binary resource streams start with FF 0A
|
||||
if raw[:2] == b"\xff\x0a":
|
||||
return {
|
||||
"nodes": [], "edges": [],
|
||||
"error": f"binary DFM (convert to text in Delphi IDE to index): {path.name}",
|
||||
}
|
||||
|
||||
# Text DFM — delegate to the shared form parser (same syntax as .lfm)
|
||||
try:
|
||||
text = raw.decode("utf-8", errors="replace")
|
||||
except Exception as e:
|
||||
return {"nodes": [], "edges": [], "error": str(e)}
|
||||
|
||||
import re
|
||||
str_path = str(path)
|
||||
stem = _file_stem(path)
|
||||
nodes: list[dict] = []
|
||||
edges: list[dict] = []
|
||||
seen_ids: set[str] = set()
|
||||
seen_edge_pairs: set[tuple[str, str, str]] = set()
|
||||
|
||||
def add_node(nid: str, label: str, line: int) -> None:
|
||||
if nid not in seen_ids:
|
||||
seen_ids.add(nid)
|
||||
nodes.append({
|
||||
"id": nid, "label": label, "file_type": "code",
|
||||
"source_file": str_path, "source_location": f"L{line}",
|
||||
})
|
||||
|
||||
def add_edge(
|
||||
src: str, tgt: str, relation: str, line: int,
|
||||
context: str | None = None,
|
||||
) -> None:
|
||||
key = (src, tgt, relation)
|
||||
if key in seen_edge_pairs:
|
||||
return
|
||||
seen_edge_pairs.add(key)
|
||||
edge: dict[str, Any] = {
|
||||
"source": src, "target": tgt, "relation": relation,
|
||||
"confidence": "EXTRACTED", "source_file": str_path,
|
||||
"source_location": f"L{line}", "weight": 1.0,
|
||||
}
|
||||
if context:
|
||||
edge["context"] = context
|
||||
edges.append(edge)
|
||||
|
||||
file_nid = _make_id(str(path))
|
||||
add_node(file_nid, path.name, 1)
|
||||
|
||||
obj_re = re.compile(r"^\s*object\s+\w+\s*:\s*(\w+)", re.IGNORECASE)
|
||||
event_re = re.compile(r"^\s*On\w+\s*=\s*(\w+)", re.IGNORECASE)
|
||||
end_re = re.compile(r"^\s*end\s*$", re.IGNORECASE)
|
||||
stack: list[str] = [file_nid]
|
||||
|
||||
for lineno, line in enumerate(text.splitlines(), 1):
|
||||
m = obj_re.match(line)
|
||||
if m:
|
||||
class_name = m.group(1)
|
||||
nid = _make_id(stem, class_name)
|
||||
add_node(nid, class_name, lineno)
|
||||
add_edge(stack[-1], nid, "contains", lineno)
|
||||
stack.append(nid)
|
||||
continue
|
||||
m = event_re.match(line)
|
||||
if m and len(stack) > 1:
|
||||
handler = m.group(1)
|
||||
handler_nid = _make_id(stem, handler)
|
||||
add_node(handler_nid, f"{handler}()", lineno)
|
||||
add_edge(stack[-1], handler_nid, "references", lineno, context="event")
|
||||
continue
|
||||
if end_re.match(line) and len(stack) > 1:
|
||||
stack.pop()
|
||||
|
||||
return {"nodes": nodes, "edges": edges, "input_tokens": 0, "output_tokens": 0}
|
||||
@@ -0,0 +1,496 @@
|
||||
"""Powershell extractor. Moved verbatim from graphify/extract.py."""
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
from graphify.extractors.base import _file_stem, _make_id, _read_text
|
||||
|
||||
|
||||
def extract_powershell(path: Path) -> dict:
|
||||
"""Extract functions, classes, methods, and using statements from a .ps1 file."""
|
||||
try:
|
||||
import tree_sitter_powershell as tsps
|
||||
from tree_sitter import Language, Parser
|
||||
except ImportError:
|
||||
return {"nodes": [], "edges": [], "error": "tree_sitter_powershell not installed"}
|
||||
|
||||
try:
|
||||
language = Language(tsps.language())
|
||||
parser = Parser(language)
|
||||
source = path.read_bytes()
|
||||
tree = parser.parse(source)
|
||||
root = tree.root_node
|
||||
except Exception as e:
|
||||
return {"nodes": [], "edges": [], "error": str(e)}
|
||||
|
||||
stem = _file_stem(path)
|
||||
str_path = str(path)
|
||||
nodes: list[dict] = []
|
||||
edges: list[dict] = []
|
||||
seen_ids: set[str] = set()
|
||||
function_bodies: list[tuple[str, Any]] = []
|
||||
|
||||
def add_node(nid: str, label: str, line: int) -> None:
|
||||
if nid not in seen_ids:
|
||||
seen_ids.add(nid)
|
||||
nodes.append({"id": nid, "label": label, "file_type": "code",
|
||||
"source_file": str_path, "source_location": f"L{line}"})
|
||||
|
||||
def add_edge(src: str, tgt: str, relation: str, line: int,
|
||||
confidence: str = "EXTRACTED", weight: float = 1.0,
|
||||
context: str | None = None) -> None:
|
||||
edge = {"source": src, "target": tgt, "relation": relation,
|
||||
"confidence": confidence, "source_file": str_path,
|
||||
"source_location": f"L{line}", "weight": weight}
|
||||
if context:
|
||||
edge["context"] = context
|
||||
edges.append(edge)
|
||||
|
||||
file_nid = _make_id(str(path))
|
||||
add_node(file_nid, path.name, 1)
|
||||
|
||||
_PS_SKIP = frozenset({
|
||||
"using", "return", "if", "else", "elseif", "foreach", "for",
|
||||
"while", "do", "switch", "try", "catch", "finally", "throw",
|
||||
"break", "continue", "exit", "param", "begin", "process", "end",
|
||||
# Import commands — handled as import edges, not function calls
|
||||
"import-module",
|
||||
})
|
||||
|
||||
def _find_script_block_body(node):
|
||||
for child in node.children:
|
||||
if child.type == "script_block":
|
||||
for sc in child.children:
|
||||
if sc.type == "script_block_body":
|
||||
return sc
|
||||
return child
|
||||
return None
|
||||
|
||||
def ensure_named_node(name: str, line: int) -> str:
|
||||
nid = _make_id(stem, name)
|
||||
if nid in seen_ids:
|
||||
return nid
|
||||
nid = _make_id(name)
|
||||
if nid not in seen_ids:
|
||||
# The name isn't defined in this file, so this is a cross-file reference
|
||||
# (e.g. a `Thing` type annotation imported from another module). Emit a
|
||||
# SOURCELESS stub — like the inheritance-base path below — so the
|
||||
# corpus-level rewire can collapse it onto the real definition. A sourced
|
||||
# stub here makes _disambiguate_colliding_node_ids bake the referencing
|
||||
# file's path (with extension) into the id and blocks the rewire, which is
|
||||
# the phantom-duplicate-node bug (#1402).
|
||||
seen_ids.add(nid)
|
||||
nodes.append({
|
||||
"id": nid,
|
||||
"label": name,
|
||||
"file_type": "code",
|
||||
"source_file": "",
|
||||
"source_location": "",
|
||||
"origin_file": str_path,
|
||||
})
|
||||
return nid
|
||||
|
||||
def _ps_type_name(type_literal_node) -> str | None:
|
||||
"""Drill into a type_literal node and return the inner type_identifier text."""
|
||||
if type_literal_node is None:
|
||||
return None
|
||||
for spec in type_literal_node.children:
|
||||
if spec.type != "type_spec":
|
||||
continue
|
||||
for tname in spec.children:
|
||||
if tname.type != "type_name":
|
||||
continue
|
||||
for tid in tname.children:
|
||||
if tid.type == "type_identifier":
|
||||
return _read_text(tid, source)
|
||||
return None
|
||||
|
||||
def walk(node, parent_class_nid: str | None = None) -> None:
|
||||
t = node.type
|
||||
|
||||
if t == "function_statement":
|
||||
name_node = next((c for c in node.children if c.type == "function_name"), None)
|
||||
if name_node:
|
||||
func_name = _read_text(name_node, source)
|
||||
line = node.start_point[0] + 1
|
||||
func_nid = _make_id(stem, func_name)
|
||||
add_node(func_nid, f"{func_name}()", line)
|
||||
add_edge(file_nid, func_nid, "contains", line)
|
||||
body = _find_script_block_body(node)
|
||||
if body:
|
||||
function_bodies.append((func_nid, body))
|
||||
# Also walk the body during the main pass so that
|
||||
# Import-Module / dot-source inside functions emit
|
||||
# file-level imports_from edges (#1331).
|
||||
walk(body, parent_class_nid)
|
||||
return
|
||||
|
||||
if t == "class_statement":
|
||||
name_node = next((c for c in node.children if c.type == "simple_name"), None)
|
||||
if name_node:
|
||||
class_name = _read_text(name_node, source)
|
||||
line = node.start_point[0] + 1
|
||||
class_nid = _make_id(stem, class_name)
|
||||
add_node(class_nid, class_name, line)
|
||||
add_edge(file_nid, class_nid, "contains", line)
|
||||
# Base type(s) after ':'. PowerShell has no syntactic base vs
|
||||
# interface split, so (matching the C# convention) treat the
|
||||
# first base as the superclass (inherits) and the rest as
|
||||
# interfaces (implements). Bases are the simple_name children
|
||||
# after the ':' token.
|
||||
colon_seen = False
|
||||
base_index = 0
|
||||
for child in node.children:
|
||||
if child.type == ":":
|
||||
colon_seen = True
|
||||
elif colon_seen and child.type == "simple_name":
|
||||
base_nid = ensure_named_node(_read_text(child, source), line)
|
||||
if base_nid != class_nid:
|
||||
rel = "inherits" if base_index == 0 else "implements"
|
||||
add_edge(class_nid, base_nid, rel, line)
|
||||
base_index += 1
|
||||
for child in node.children:
|
||||
walk(child, parent_class_nid=class_nid)
|
||||
return
|
||||
|
||||
if t == "class_property_definition" and parent_class_nid:
|
||||
type_literal = next((c for c in node.children if c.type == "type_literal"), None)
|
||||
type_name = _ps_type_name(type_literal)
|
||||
if type_name:
|
||||
line = node.start_point[0] + 1
|
||||
target_nid = ensure_named_node(type_name, line)
|
||||
if target_nid != parent_class_nid:
|
||||
add_edge(parent_class_nid, target_nid, "references",
|
||||
line, context="field")
|
||||
return
|
||||
|
||||
if t == "class_method_definition":
|
||||
name_node = next((c for c in node.children if c.type == "simple_name"), None)
|
||||
if name_node:
|
||||
method_name = _read_text(name_node, source)
|
||||
line = node.start_point[0] + 1
|
||||
if parent_class_nid:
|
||||
method_nid = _make_id(parent_class_nid, method_name)
|
||||
add_node(method_nid, f".{method_name}()", line)
|
||||
add_edge(parent_class_nid, method_nid, "method", line)
|
||||
else:
|
||||
method_nid = _make_id(stem, method_name)
|
||||
add_node(method_nid, f"{method_name}()", line)
|
||||
add_edge(file_nid, method_nid, "contains", line)
|
||||
# Return type: type_literal sibling of simple_name
|
||||
return_type_literal = next(
|
||||
(c for c in node.children if c.type == "type_literal"), None)
|
||||
return_type_name = _ps_type_name(return_type_literal)
|
||||
if return_type_name:
|
||||
target_nid = ensure_named_node(return_type_name, line)
|
||||
if target_nid != method_nid:
|
||||
add_edge(method_nid, target_nid, "references",
|
||||
line, context="return_type")
|
||||
# Parameter types: class_method_parameter_list
|
||||
param_list = next(
|
||||
(c for c in node.children if c.type == "class_method_parameter_list"), None)
|
||||
if param_list is not None:
|
||||
for p in param_list.children:
|
||||
if p.type != "class_method_parameter":
|
||||
continue
|
||||
ptype_literal = next(
|
||||
(c for c in p.children if c.type == "type_literal"), None)
|
||||
ptype_name = _ps_type_name(ptype_literal)
|
||||
if not ptype_name:
|
||||
continue
|
||||
p_line = p.start_point[0] + 1
|
||||
target_nid = ensure_named_node(ptype_name, p_line)
|
||||
if target_nid != method_nid:
|
||||
add_edge(method_nid, target_nid, "references",
|
||||
p_line, context="parameter_type")
|
||||
body = _find_script_block_body(node)
|
||||
if body:
|
||||
function_bodies.append((method_nid, body))
|
||||
return
|
||||
|
||||
if t == "command":
|
||||
# Dot-sourcing: `. ./Shared.psm1`
|
||||
# Uses command_invokation_operator '.' + command_name_expr (not command_name)
|
||||
invoke_op = next(
|
||||
(c for c in node.children if c.type == "command_invokation_operator"), None
|
||||
)
|
||||
if invoke_op is not None and _read_text(invoke_op, source).strip() == ".":
|
||||
name_expr = next(
|
||||
(c for c in node.children if c.type == "command_name_expr"), None
|
||||
)
|
||||
if name_expr is not None:
|
||||
name_node = next(
|
||||
(c for c in name_expr.children if c.type == "command_name"), None
|
||||
)
|
||||
if name_node:
|
||||
raw_path = _read_text(name_node, source)
|
||||
# Strip relative path prefix (./ or .\ or just the dot)
|
||||
module_stem = re.sub(r'^[./\\]+', '', raw_path)
|
||||
# Drop extension to get bare module name
|
||||
module_stem = re.sub(r'\.[^.]+$', '', module_stem).replace('\\', '/')
|
||||
module_name = module_stem.split('/')[-1]
|
||||
if module_name:
|
||||
add_edge(file_nid, _make_id(module_name), "imports_from",
|
||||
node.start_point[0] + 1)
|
||||
return
|
||||
|
||||
cmd_name_node = next((c for c in node.children if c.type == "command_name"), None)
|
||||
if cmd_name_node:
|
||||
cmd_text = _read_text(cmd_name_node, source).lower()
|
||||
if cmd_text == "using":
|
||||
tokens = []
|
||||
for child in node.children:
|
||||
if child.type == "command_elements":
|
||||
for el in child.children:
|
||||
if el.type == "generic_token":
|
||||
tokens.append(_read_text(el, source))
|
||||
module_tokens = [t for t in tokens
|
||||
if t.lower() not in ("namespace", "module", "assembly")]
|
||||
if module_tokens:
|
||||
module_name = module_tokens[-1].split(".")[-1]
|
||||
add_edge(file_nid, _make_id(module_name), "imports_from",
|
||||
node.start_point[0] + 1)
|
||||
elif cmd_text == "import-module":
|
||||
# Collect generic_token args; skip command_parameter flags like -Name
|
||||
# The module name is the first generic_token (or the one after -Name)
|
||||
module_name: str | None = None
|
||||
expect_name = False
|
||||
for child in node.children:
|
||||
if child.type != "command_elements":
|
||||
continue
|
||||
for el in child.children:
|
||||
if el.type == "command_parameter":
|
||||
param_text = _read_text(el, source).lstrip("-").lower()
|
||||
expect_name = param_text in ("name", "n")
|
||||
elif el.type == "generic_token":
|
||||
token = _read_text(el, source)
|
||||
if module_name is None or expect_name:
|
||||
module_name = token
|
||||
expect_name = False
|
||||
if module_name:
|
||||
# Strip extension; keep only the stem for the node ID
|
||||
bare = re.sub(r'\.[^.]+$', '', module_name).split('/')[-1].split('\\')[-1]
|
||||
if bare:
|
||||
add_edge(file_nid, _make_id(bare), "imports_from",
|
||||
node.start_point[0] + 1)
|
||||
return
|
||||
|
||||
for child in node.children:
|
||||
walk(child, parent_class_nid)
|
||||
|
||||
walk(root)
|
||||
|
||||
label_to_nid = {n["label"].strip("()").lstrip(".").lower(): n["id"] for n in nodes}
|
||||
seen_call_pairs: set[tuple[str, str]] = set()
|
||||
raw_calls: list[dict] = []
|
||||
|
||||
def walk_calls(node, caller_nid: str) -> None:
|
||||
if node.type in ("function_statement", "class_statement"):
|
||||
return
|
||||
if node.type == "command":
|
||||
cmd_name_node = next((c for c in node.children if c.type == "command_name"), None)
|
||||
if cmd_name_node:
|
||||
cmd_text = _read_text(cmd_name_node, source)
|
||||
if cmd_text.lower() not in _PS_SKIP:
|
||||
tgt_nid = label_to_nid.get(cmd_text.lower())
|
||||
if tgt_nid and tgt_nid != caller_nid:
|
||||
pair = (caller_nid, tgt_nid)
|
||||
if pair not in seen_call_pairs:
|
||||
seen_call_pairs.add(pair)
|
||||
add_edge(caller_nid, tgt_nid, "calls",
|
||||
node.start_point[0] + 1,
|
||||
confidence="EXTRACTED", weight=1.0)
|
||||
elif cmd_text:
|
||||
raw_calls.append({
|
||||
"caller_nid": caller_nid,
|
||||
"callee": cmd_text,
|
||||
"is_member_call": False,
|
||||
"source_file": str_path,
|
||||
"source_location": f"L{node.start_point[0] + 1}",
|
||||
})
|
||||
for child in node.children:
|
||||
walk_calls(child, caller_nid)
|
||||
|
||||
for caller_nid, body_node in function_bodies:
|
||||
walk_calls(body_node, caller_nid)
|
||||
|
||||
clean_edges = [e for e in edges if e["source"] in seen_ids and
|
||||
(e["target"] in seen_ids or e["relation"] in ("imports_from", "imports"))]
|
||||
return {"nodes": nodes, "edges": clean_edges, "raw_calls": raw_calls}
|
||||
|
||||
_PSD1_IMPORT_KEYS = frozenset({"RootModule", "NestedModules", "RequiredModules"})
|
||||
|
||||
def _psd1_collect_string_literals(node, source: bytes) -> list[str]:
|
||||
"""Recursively collect all string_literal text values under *node*."""
|
||||
results: list[str] = []
|
||||
|
||||
def _walk(n) -> None:
|
||||
if n.type == "string_literal":
|
||||
raw = source[n.start_byte:n.end_byte].decode(errors="replace")
|
||||
# Strip surrounding quote chars (' or ")
|
||||
results.append(raw.strip("'\""))
|
||||
return
|
||||
for child in n.children:
|
||||
_walk(child)
|
||||
|
||||
_walk(node)
|
||||
return results
|
||||
|
||||
def _psd1_module_name(raw: str) -> str:
|
||||
"""Derive a bare module name from a raw string value.
|
||||
|
||||
e.g. 'MyModule.psm1' → 'MyModule', './sub/Util.psm1' → 'Util', 'PSReadLine' → 'PSReadLine'
|
||||
"""
|
||||
# Strip path prefix and extension
|
||||
name = raw.replace("\\", "/").split("/")[-1]
|
||||
name = re.sub(r"\.[^.]+$", "", name) # remove last extension
|
||||
return name.strip()
|
||||
|
||||
def extract_powershell_manifest(path: Path) -> dict:
|
||||
"""Extract module dependency edges from a PowerShell .psd1 manifest file.
|
||||
|
||||
.psd1 files are PowerShell data hashtables, not scripts. tree-sitter-powershell
|
||||
parses them correctly (they are syntactically valid PS). We walk the AST looking
|
||||
for RootModule, NestedModules, and RequiredModules keys and emit imports_from
|
||||
edges for every referenced module.
|
||||
|
||||
RequiredModules supports two forms:
|
||||
- Simple string: 'PSReadLine'
|
||||
- Module specification: @{ ModuleName = 'Pester'; ModuleVersion = '5.0' }
|
||||
For the hashtable form we only follow the ModuleName key.
|
||||
"""
|
||||
try:
|
||||
import tree_sitter_powershell as tsps
|
||||
from tree_sitter import Language, Parser
|
||||
except ImportError:
|
||||
return {"nodes": [], "edges": [], "error": "tree_sitter_powershell not installed"}
|
||||
|
||||
try:
|
||||
language = Language(tsps.language())
|
||||
parser = Parser(language)
|
||||
source = path.read_bytes()
|
||||
tree = parser.parse(source)
|
||||
root = tree.root_node
|
||||
except Exception as e:
|
||||
return {"nodes": [], "edges": [], "error": str(e)}
|
||||
|
||||
str_path = str(path)
|
||||
nodes: list[dict] = []
|
||||
edges: list[dict] = []
|
||||
seen_ids: set[str] = set()
|
||||
|
||||
def add_node(nid: str, label: str, line: int) -> None:
|
||||
if nid not in seen_ids:
|
||||
seen_ids.add(nid)
|
||||
nodes.append({"id": nid, "label": label, "file_type": "code",
|
||||
"source_file": str_path, "source_location": f"L{line}"})
|
||||
|
||||
def add_import_edge(src: str, module_raw: str, line: int) -> None:
|
||||
name = _psd1_module_name(module_raw)
|
||||
if not name:
|
||||
return
|
||||
tgt_nid = _make_id(name)
|
||||
edges.append({
|
||||
"source": src,
|
||||
"target": tgt_nid,
|
||||
"relation": "imports_from",
|
||||
"confidence": "EXTRACTED",
|
||||
"source_file": str_path,
|
||||
"source_location": f"L{line}",
|
||||
"weight": 1.0,
|
||||
"context": "import",
|
||||
})
|
||||
|
||||
file_nid = _make_id(str(path))
|
||||
add_node(file_nid, path.name, 1)
|
||||
|
||||
def walk_manifest(node) -> None:
|
||||
"""Walk the AST and emit edges for import-relevant hash_entry nodes."""
|
||||
if node.type != "hash_entry":
|
||||
for child in node.children:
|
||||
walk_manifest(child)
|
||||
return
|
||||
|
||||
# Identify the key
|
||||
key_node = next((c for c in node.children if c.type == "key_expression"), None)
|
||||
if key_node is None:
|
||||
return
|
||||
key_text = source[key_node.start_byte:key_node.end_byte].decode(errors="replace").strip()
|
||||
|
||||
if key_text not in _PSD1_IMPORT_KEYS:
|
||||
# Still recurse in case there are nested hashes (e.g. ModuleVersion entries
|
||||
# contain sub-hashes, but we only care about top-level keys for imports)
|
||||
return
|
||||
|
||||
line = node.start_point[0] + 1
|
||||
value_node = next((c for c in node.children if c.type == "pipeline"), None)
|
||||
if value_node is None:
|
||||
return
|
||||
|
||||
if key_text == "RootModule":
|
||||
# Value is a single string
|
||||
strings = _psd1_collect_string_literals(value_node, source)
|
||||
for s in strings:
|
||||
add_import_edge(file_nid, s, line)
|
||||
|
||||
elif key_text == "NestedModules":
|
||||
# Value is a string or @('a', 'b', ...) array — collect all string literals
|
||||
strings = _psd1_collect_string_literals(value_node, source)
|
||||
for s in strings:
|
||||
add_import_edge(file_nid, s, line)
|
||||
|
||||
elif key_text == "RequiredModules":
|
||||
# Two forms:
|
||||
# 1) 'SimpleModule' — direct string literals in the array
|
||||
# 2) @{ ModuleName = 'Foo'; ModuleVersion = '2.0' } — use ModuleName only
|
||||
#
|
||||
# Strategy: walk the value for hash_entry nodes whose key is 'ModuleName';
|
||||
# collect their string values. For the remaining string_literal nodes that
|
||||
# are NOT inside a hash_entry subtree, treat them as simple module names.
|
||||
module_name_strings: list[str] = []
|
||||
inside_hash_entries: set[int] = set() # byte offsets of handled strings
|
||||
|
||||
def find_modulename_entries(n) -> None:
|
||||
if n.type == "hash_entry":
|
||||
sub_key = next((c for c in n.children if c.type == "key_expression"), None)
|
||||
if sub_key is not None:
|
||||
sk_text = source[sub_key.start_byte:sub_key.end_byte].decode(errors="replace").strip()
|
||||
# Collect strings inside *all* sub-keys so we can exclude them
|
||||
for c in n.children:
|
||||
if c.type == "pipeline":
|
||||
for s_node in _collect_string_nodes(c):
|
||||
inside_hash_entries.add(s_node.start_byte)
|
||||
if sk_text == "ModuleName":
|
||||
for c in n.children:
|
||||
if c.type == "pipeline":
|
||||
for s in _psd1_collect_string_literals(c, source):
|
||||
module_name_strings.append(s)
|
||||
return # don't recurse further into this hash_entry
|
||||
for child in n.children:
|
||||
find_modulename_entries(child)
|
||||
|
||||
def _collect_string_nodes(n):
|
||||
"""Return all string_literal nodes in subtree."""
|
||||
if n.type == "string_literal":
|
||||
yield n
|
||||
return
|
||||
for child in n.children:
|
||||
yield from _collect_string_nodes(child)
|
||||
|
||||
find_modulename_entries(value_node)
|
||||
|
||||
# Now gather direct string literals not inside hash entries
|
||||
direct_strings: list[str] = []
|
||||
for s_node in _collect_string_nodes(value_node):
|
||||
if s_node.start_byte not in inside_hash_entries:
|
||||
raw = source[s_node.start_byte:s_node.end_byte].decode(errors="replace")
|
||||
direct_strings.append(raw.strip("'\""))
|
||||
|
||||
for s in direct_strings + module_name_strings:
|
||||
add_import_edge(file_nid, s, line)
|
||||
|
||||
walk_manifest(root)
|
||||
|
||||
return {"nodes": nodes, "edges": edges, "raw_calls": []}
|
||||
@@ -0,0 +1,119 @@
|
||||
"""ASP.NET Razor component extractor. Moved verbatim from graphify/extract.py."""
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from pathlib import Path
|
||||
|
||||
from graphify.extractors.base import _file_stem, _make_id
|
||||
|
||||
|
||||
def extract_razor(path: Path) -> dict:
|
||||
"""Extract directives, component refs, and @code methods from .razor/.cshtml."""
|
||||
try:
|
||||
src = path.read_text(encoding="utf-8", errors="replace")
|
||||
except OSError:
|
||||
return {"nodes": [], "edges": [], "error": f"cannot read {path}"}
|
||||
|
||||
file_nid = _make_id(str(path))
|
||||
str_path = str(path)
|
||||
nodes: list[dict] = [{"id": file_nid, "label": path.name, "file_type": "code",
|
||||
"source_file": str_path, "source_location": None}]
|
||||
edges: list[dict] = []
|
||||
seen_ids: set[str] = set()
|
||||
seen_ids.add(file_nid)
|
||||
|
||||
def _add_ref(target_name: str, relation: str, line: int) -> None:
|
||||
tgt_nid = _make_id(target_name)
|
||||
if not tgt_nid:
|
||||
return
|
||||
if tgt_nid not in seen_ids:
|
||||
seen_ids.add(tgt_nid)
|
||||
nodes.append({"id": tgt_nid, "label": target_name,
|
||||
"file_type": "code", "source_file": str_path,
|
||||
"source_location": f"L{line}"})
|
||||
edges.append({"source": file_nid, "target": tgt_nid,
|
||||
"relation": relation, "confidence": "EXTRACTED",
|
||||
"source_file": str_path, "source_location": f"L{line}",
|
||||
"weight": 1.0})
|
||||
|
||||
for i, line in enumerate(src.splitlines(), 1):
|
||||
m = re.match(r'@using\s+([\w.]+)', line)
|
||||
if m:
|
||||
_add_ref(m.group(1), "imports", i)
|
||||
continue
|
||||
|
||||
m = re.match(r'@inject\s+([\w.<>\[\]]+)\s+(\w+)', line)
|
||||
if m:
|
||||
_add_ref(m.group(1), "imports", i)
|
||||
continue
|
||||
|
||||
m = re.match(r'@inherits\s+([\w.<>\[\]]+)', line)
|
||||
if m:
|
||||
_add_ref(m.group(1), "inherits", i)
|
||||
continue
|
||||
|
||||
m = re.match(r'@model\s+([\w.<>\[\]]+)', line)
|
||||
if m:
|
||||
_add_ref(m.group(1), "references", i)
|
||||
continue
|
||||
|
||||
m = re.match(r'@page\s+"([^"]+)"', line)
|
||||
if m:
|
||||
route = m.group(1)
|
||||
route_nid = _make_id("route", route)
|
||||
if route_nid and route_nid not in seen_ids:
|
||||
seen_ids.add(route_nid)
|
||||
nodes.append({"id": route_nid, "label": f"route:{route}",
|
||||
"file_type": "concept", "source_file": str_path,
|
||||
"source_location": f"L{i}"})
|
||||
edges.append({"source": file_nid, "target": route_nid,
|
||||
"relation": "references", "confidence": "EXTRACTED",
|
||||
"source_file": str_path, "weight": 1.0})
|
||||
continue
|
||||
|
||||
_COMPONENT_RE = re.compile(r'<([A-Z][A-Za-z0-9]+)[\s/>]')
|
||||
_HTML_TAGS = frozenset({
|
||||
"DOCTYPE", "Html", "Head", "Body", "Div", "Span", "Table", "Form",
|
||||
"Input", "Button", "Select", "Option", "Label", "Textarea",
|
||||
"Script", "Style", "Link", "Meta", "Title", "Header", "Footer",
|
||||
"Nav", "Main", "Section", "Article", "Aside",
|
||||
})
|
||||
for m in _COMPONENT_RE.finditer(src):
|
||||
comp_name = m.group(1)
|
||||
if comp_name in _HTML_TAGS:
|
||||
continue
|
||||
line_num = src[:m.start()].count("\n") + 1
|
||||
_add_ref(comp_name, "calls", line_num)
|
||||
|
||||
_CODE_BLOCK_RE = re.compile(r'@code\s*\{', re.MULTILINE)
|
||||
for m in _CODE_BLOCK_RE.finditer(src):
|
||||
block_start = m.end()
|
||||
depth = 1
|
||||
pos = block_start
|
||||
while pos < len(src) and depth > 0:
|
||||
if src[pos] == '{':
|
||||
depth += 1
|
||||
elif src[pos] == '}':
|
||||
depth -= 1
|
||||
pos += 1
|
||||
code_block = src[block_start:pos - 1] if depth == 0 else ""
|
||||
|
||||
_METHOD_RE = re.compile(
|
||||
r'(?:public|private|protected|internal|static|async|override|virtual|abstract)\s+'
|
||||
r'[\w<>\[\],\s]+\s+(\w+)\s*\('
|
||||
)
|
||||
for mm in _METHOD_RE.finditer(code_block):
|
||||
method_name = mm.group(1)
|
||||
abs_pos = block_start + mm.start()
|
||||
method_line = src[:abs_pos].count("\n") + 1
|
||||
method_nid = _make_id(_file_stem(path), method_name)
|
||||
if method_nid and method_nid not in seen_ids:
|
||||
seen_ids.add(method_nid)
|
||||
nodes.append({"id": method_nid, "label": method_name,
|
||||
"file_type": "code", "source_file": str_path,
|
||||
"source_location": f"L{method_line}"})
|
||||
edges.append({"source": file_nid, "target": method_nid,
|
||||
"relation": "contains", "confidence": "EXTRACTED",
|
||||
"source_file": str_path, "weight": 1.0})
|
||||
|
||||
return {"nodes": nodes, "edges": edges}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,410 @@
|
||||
"""Rust extractor. Moved verbatim from graphify/extract.py."""
|
||||
from __future__ import annotations
|
||||
|
||||
|
||||
from pathlib import Path
|
||||
from graphify.extractors.base import _LANGUAGE_BUILTIN_GLOBALS, _file_stem, _make_id, _read_text
|
||||
|
||||
|
||||
def _rust_collect_type_refs(node, source: bytes, generic: bool, out: list[tuple[str, str]]) -> None:
|
||||
"""Walk a Rust type expression; append (name, role) tuples."""
|
||||
if node is None:
|
||||
return
|
||||
t = node.type
|
||||
if t == "primitive_type":
|
||||
return
|
||||
if t == "type_identifier":
|
||||
text = _read_text(node, source)
|
||||
if text:
|
||||
out.append((text, "generic_arg" if generic else "type"))
|
||||
return
|
||||
if t == "scoped_type_identifier":
|
||||
text = _read_text(node, source).rsplit("::", 1)[-1]
|
||||
if text:
|
||||
out.append((text, "generic_arg" if generic else "type"))
|
||||
return
|
||||
if t == "generic_type":
|
||||
name_node = node.child_by_field_name("type")
|
||||
if name_node is None:
|
||||
for c in node.children:
|
||||
if c.type in ("type_identifier", "scoped_type_identifier"):
|
||||
name_node = c
|
||||
break
|
||||
if name_node is not None:
|
||||
text = _read_text(name_node, source).rsplit("::", 1)[-1]
|
||||
if text:
|
||||
out.append((text, "generic_arg" if generic else "type"))
|
||||
for c in node.children:
|
||||
if c.type == "type_arguments":
|
||||
for arg in c.children:
|
||||
if arg.is_named:
|
||||
_rust_collect_type_refs(arg, source, True, out)
|
||||
return
|
||||
if t in ("reference_type", "pointer_type", "array_type", "tuple_type", "slice_type"):
|
||||
for c in node.children:
|
||||
if c.is_named:
|
||||
_rust_collect_type_refs(c, source, generic, out)
|
||||
return
|
||||
if node.is_named:
|
||||
for c in node.children:
|
||||
if c.is_named:
|
||||
_rust_collect_type_refs(c, source, generic, out)
|
||||
|
||||
_RUST_TRAIT_METHOD_BLOCKLIST: frozenset[str] = frozenset({
|
||||
"new", "default", "parse", "from_str", "now", "clone", "into", "from",
|
||||
"to_string", "to_owned", "len", "is_empty", "iter", "next", "build",
|
||||
"start", "run", "init", "app", "get", "set", "push", "pop", "insert",
|
||||
"remove", "contains", "collect", "map", "filter", "unwrap", "expect",
|
||||
"ok", "err", "some", "none", "send", "recv", "lock", "read", "write",
|
||||
})
|
||||
|
||||
def extract_rust(path: Path) -> dict:
|
||||
"""Extract functions, structs, enums, traits, impl methods, and use declarations from a .rs file."""
|
||||
try:
|
||||
import tree_sitter_rust as tsrust
|
||||
from tree_sitter import Language, Parser
|
||||
except ImportError:
|
||||
return {"nodes": [], "edges": [], "error": "tree-sitter-rust not installed"}
|
||||
|
||||
try:
|
||||
language = Language(tsrust.language())
|
||||
parser = Parser(language)
|
||||
source = path.read_bytes()
|
||||
tree = parser.parse(source)
|
||||
root = tree.root_node
|
||||
except Exception as e:
|
||||
return {"nodes": [], "edges": [], "error": str(e)}
|
||||
|
||||
stem = _file_stem(path)
|
||||
str_path = str(path)
|
||||
nodes: list[dict] = []
|
||||
edges: list[dict] = []
|
||||
seen_ids: set[str] = set()
|
||||
function_bodies: list[tuple[str, object]] = []
|
||||
|
||||
def add_node(nid: str, label: str, line: int) -> None:
|
||||
if nid not in seen_ids:
|
||||
seen_ids.add(nid)
|
||||
nodes.append({
|
||||
"id": nid,
|
||||
"label": label,
|
||||
"file_type": "code",
|
||||
"source_file": str_path,
|
||||
"source_location": f"L{line}",
|
||||
})
|
||||
|
||||
def add_edge(src: str, tgt: str, relation: str, line: int,
|
||||
confidence: str = "EXTRACTED", weight: float = 1.0,
|
||||
context: str | None = None) -> None:
|
||||
edge = {
|
||||
"source": src,
|
||||
"target": tgt,
|
||||
"relation": relation,
|
||||
"confidence": confidence,
|
||||
"source_file": str_path,
|
||||
"source_location": f"L{line}",
|
||||
"weight": weight,
|
||||
}
|
||||
if context:
|
||||
edge["context"] = context
|
||||
edges.append(edge)
|
||||
|
||||
file_nid = _make_id(str(path))
|
||||
add_node(file_nid, path.name, 1)
|
||||
|
||||
def ensure_named_node(name: str, line: int) -> str:
|
||||
nid = _make_id(stem, name)
|
||||
if nid in seen_ids:
|
||||
return nid
|
||||
nid = _make_id(name)
|
||||
if nid not in seen_ids:
|
||||
# The name isn't defined in this file, so this is a cross-file reference
|
||||
# (e.g. a `Thing` type annotation imported from another module). Emit a
|
||||
# SOURCELESS stub — like the inheritance-base path below — so the
|
||||
# corpus-level rewire can collapse it onto the real definition. A sourced
|
||||
# stub here makes _disambiguate_colliding_node_ids bake the referencing
|
||||
# file's path (with extension) into the id and blocks the rewire, which is
|
||||
# the phantom-duplicate-node bug (#1402).
|
||||
seen_ids.add(nid)
|
||||
nodes.append({
|
||||
"id": nid,
|
||||
"label": name,
|
||||
"file_type": "code",
|
||||
"source_file": "",
|
||||
"source_location": "",
|
||||
"origin_file": str_path,
|
||||
})
|
||||
return nid
|
||||
|
||||
def emit_param_return_refs(func_node, func_nid: str, line: int) -> None:
|
||||
params = func_node.child_by_field_name("parameters")
|
||||
if params is not None:
|
||||
for p in params.children:
|
||||
if p.type != "parameter":
|
||||
continue
|
||||
type_node = p.child_by_field_name("type")
|
||||
refs: list[tuple[str, str]] = []
|
||||
_rust_collect_type_refs(type_node, source, False, refs)
|
||||
for ref_name, role in refs:
|
||||
ctx = "generic_arg" if role == "generic_arg" else "parameter_type"
|
||||
tgt = ensure_named_node(ref_name, line)
|
||||
if tgt != func_nid:
|
||||
add_edge(func_nid, tgt, "references", line, context=ctx)
|
||||
return_type = func_node.child_by_field_name("return_type")
|
||||
if return_type is not None:
|
||||
refs = []
|
||||
_rust_collect_type_refs(return_type, source, False, refs)
|
||||
for ref_name, role in refs:
|
||||
ctx = "generic_arg" if role == "generic_arg" else "return_type"
|
||||
tgt = ensure_named_node(ref_name, line)
|
||||
if tgt != func_nid:
|
||||
add_edge(func_nid, tgt, "references", line, context=ctx)
|
||||
|
||||
def walk(node, parent_impl_nid: str | None = None) -> None:
|
||||
t = node.type
|
||||
|
||||
if t == "function_item":
|
||||
name_node = node.child_by_field_name("name")
|
||||
if name_node:
|
||||
func_name = _read_text(name_node, source)
|
||||
line = node.start_point[0] + 1
|
||||
if parent_impl_nid:
|
||||
func_nid = _make_id(parent_impl_nid, func_name)
|
||||
add_node(func_nid, f".{func_name}()", line)
|
||||
add_edge(parent_impl_nid, func_nid, "method", line)
|
||||
else:
|
||||
func_nid = _make_id(stem, func_name)
|
||||
add_node(func_nid, f"{func_name}()", line)
|
||||
add_edge(file_nid, func_nid, "contains", line)
|
||||
emit_param_return_refs(node, func_nid, line)
|
||||
body = node.child_by_field_name("body")
|
||||
if body:
|
||||
function_bodies.append((func_nid, body))
|
||||
return
|
||||
|
||||
if t in ("struct_item", "enum_item", "trait_item"):
|
||||
name_node = node.child_by_field_name("name")
|
||||
if name_node:
|
||||
item_name = _read_text(name_node, source)
|
||||
line = node.start_point[0] + 1
|
||||
item_nid = _make_id(stem, item_name)
|
||||
add_node(item_nid, item_name, line)
|
||||
add_edge(file_nid, item_nid, "contains", line)
|
||||
if t == "trait_item":
|
||||
for c in node.children:
|
||||
if c.type != "trait_bounds":
|
||||
continue
|
||||
for sub in c.children:
|
||||
if not sub.is_named:
|
||||
continue
|
||||
refs: list[tuple[str, str]] = []
|
||||
_rust_collect_type_refs(sub, source, False, refs)
|
||||
for idx, (ref_name, _role) in enumerate(refs):
|
||||
tgt = ensure_named_node(ref_name, line)
|
||||
if tgt == item_nid:
|
||||
continue
|
||||
rel = "inherits" if idx == 0 else "references"
|
||||
if rel == "inherits":
|
||||
add_edge(item_nid, tgt, "inherits", line)
|
||||
else:
|
||||
add_edge(item_nid, tgt, "references", line,
|
||||
context="generic_arg")
|
||||
if t == "struct_item":
|
||||
for c in node.children:
|
||||
if c.type != "field_declaration_list":
|
||||
continue
|
||||
for field in c.children:
|
||||
if field.type != "field_declaration":
|
||||
continue
|
||||
type_node = field.child_by_field_name("type")
|
||||
if type_node is None:
|
||||
for fc in field.children:
|
||||
if fc.type in ("type_identifier", "generic_type",
|
||||
"scoped_type_identifier",
|
||||
"reference_type", "primitive_type"):
|
||||
type_node = fc
|
||||
break
|
||||
refs = []
|
||||
_rust_collect_type_refs(type_node, source, False, refs)
|
||||
for ref_name, role in refs:
|
||||
ctx = "generic_arg" if role == "generic_arg" else "field"
|
||||
tgt = ensure_named_node(ref_name, field.start_point[0] + 1)
|
||||
if tgt != item_nid:
|
||||
add_edge(item_nid, tgt, "references",
|
||||
field.start_point[0] + 1, context=ctx)
|
||||
# Tuple structs (`struct Wrapper(pub Logger, Config);`) nest their
|
||||
# positional field types directly under ordered_field_declaration_list
|
||||
# with no field_declaration wrapper -- the same shape handled for tuple
|
||||
# enum variants below. Without this branch these field type references
|
||||
# are silently dropped.
|
||||
for c in node.children:
|
||||
if c.type != "ordered_field_declaration_list":
|
||||
continue
|
||||
fline = c.start_point[0] + 1
|
||||
for tc in c.children:
|
||||
if tc.type not in ("type_identifier", "generic_type",
|
||||
"scoped_type_identifier", "reference_type",
|
||||
"primitive_type", "tuple_type", "array_type"):
|
||||
continue
|
||||
refs = []
|
||||
_rust_collect_type_refs(tc, source, False, refs)
|
||||
for ref_name, role in refs:
|
||||
ctx = "generic_arg" if role == "generic_arg" else "field"
|
||||
tgt = ensure_named_node(ref_name, fline)
|
||||
if tgt != item_nid:
|
||||
add_edge(item_nid, tgt, "references", fline, context=ctx)
|
||||
if t == "enum_item":
|
||||
# Variant payload types nest under enum_variant_list ->
|
||||
# enum_variant -> ordered_field_declaration_list (tuple variant,
|
||||
# `Click(Logger)`) | field_declaration_list (struct variant,
|
||||
# `Resize { size: Dim }`). Neither was traversed, so every
|
||||
# enum-variant type reference was silently dropped.
|
||||
_TYPE_NODES = ("type_identifier", "generic_type",
|
||||
"scoped_type_identifier", "reference_type",
|
||||
"primitive_type", "tuple_type", "array_type")
|
||||
|
||||
def _emit_enum_type(type_node, at_line):
|
||||
if type_node is None:
|
||||
return
|
||||
refs2: list[tuple[str, str]] = []
|
||||
_rust_collect_type_refs(type_node, source, False, refs2)
|
||||
for ref_name, role in refs2:
|
||||
ctx = "generic_arg" if role == "generic_arg" else "field"
|
||||
tgt = ensure_named_node(ref_name, at_line)
|
||||
if tgt != item_nid:
|
||||
add_edge(item_nid, tgt, "references", at_line, context=ctx)
|
||||
|
||||
for c in node.children:
|
||||
if c.type != "enum_variant_list":
|
||||
continue
|
||||
for variant in c.children:
|
||||
if variant.type != "enum_variant":
|
||||
continue
|
||||
vline = variant.start_point[0] + 1
|
||||
for vc in variant.children:
|
||||
if vc.type == "ordered_field_declaration_list":
|
||||
for tc in vc.children:
|
||||
if tc.type in _TYPE_NODES:
|
||||
_emit_enum_type(tc, vline)
|
||||
elif vc.type == "field_declaration_list":
|
||||
for field in vc.children:
|
||||
if field.type != "field_declaration":
|
||||
continue
|
||||
type_node = field.child_by_field_name("type")
|
||||
_emit_enum_type(type_node, field.start_point[0] + 1)
|
||||
return
|
||||
|
||||
if t == "impl_item":
|
||||
type_node = node.child_by_field_name("type")
|
||||
trait_node = node.child_by_field_name("trait")
|
||||
impl_nid: str | None = None
|
||||
if type_node:
|
||||
type_name = _read_text(type_node, source).strip()
|
||||
impl_nid = _make_id(stem, type_name)
|
||||
add_node(impl_nid, type_name, node.start_point[0] + 1)
|
||||
if trait_node is not None and impl_nid is not None:
|
||||
refs: list[tuple[str, str]] = []
|
||||
_rust_collect_type_refs(trait_node, source, False, refs)
|
||||
for idx, (ref_name, _role) in enumerate(refs):
|
||||
tgt = ensure_named_node(ref_name, node.start_point[0] + 1)
|
||||
if tgt == impl_nid:
|
||||
continue
|
||||
if idx == 0:
|
||||
add_edge(impl_nid, tgt, "implements", node.start_point[0] + 1)
|
||||
else:
|
||||
add_edge(impl_nid, tgt, "references", node.start_point[0] + 1,
|
||||
context="generic_arg")
|
||||
body = node.child_by_field_name("body")
|
||||
if body:
|
||||
for child in body.children:
|
||||
walk(child, parent_impl_nid=impl_nid)
|
||||
return
|
||||
|
||||
if t == "use_declaration":
|
||||
arg = node.child_by_field_name("argument")
|
||||
if arg:
|
||||
raw = _read_text(arg, source)
|
||||
clean = raw.split("{")[0].rstrip(":").rstrip("*").rstrip(":")
|
||||
module_name = clean.split("::")[-1].strip()
|
||||
if module_name:
|
||||
tgt_nid = _make_id(module_name)
|
||||
add_edge(file_nid, tgt_nid, "imports_from", node.start_point[0] + 1, context="import")
|
||||
return
|
||||
|
||||
for child in node.children:
|
||||
walk(child, parent_impl_nid=None)
|
||||
|
||||
walk(root)
|
||||
|
||||
label_to_nid: dict[str, str] = {}
|
||||
for n in nodes:
|
||||
raw = n["label"]
|
||||
normalised = raw.strip("()").lstrip(".")
|
||||
label_to_nid[normalised] = n["id"]
|
||||
|
||||
seen_call_pairs: set[tuple[str, str]] = set()
|
||||
raw_calls: list[dict] = []
|
||||
|
||||
def walk_calls(node, caller_nid: str) -> None:
|
||||
if node.type == "function_item":
|
||||
return
|
||||
if node.type == "call_expression":
|
||||
func_node = node.child_by_field_name("function")
|
||||
callee_name: str | None = None
|
||||
is_member_call: bool = False
|
||||
is_scoped_call: bool = False
|
||||
if func_node:
|
||||
if func_node.type == "identifier":
|
||||
callee_name = _read_text(func_node, source)
|
||||
elif func_node.type == "field_expression":
|
||||
is_member_call = True
|
||||
field = func_node.child_by_field_name("field")
|
||||
if field:
|
||||
callee_name = _read_text(field, source)
|
||||
elif func_node.type == "scoped_identifier":
|
||||
# Type::method() — still allow in-file EXTRACTED match, but
|
||||
# skip cross-file resolution: bare last-segment lookup ignores
|
||||
# crate boundaries and produces spurious INFERRED edges (#908).
|
||||
is_scoped_call = True
|
||||
name = func_node.child_by_field_name("name")
|
||||
if name:
|
||||
callee_name = _read_text(name, source)
|
||||
if callee_name and callee_name not in _LANGUAGE_BUILTIN_GLOBALS:
|
||||
tgt_nid = label_to_nid.get(callee_name)
|
||||
if tgt_nid and tgt_nid != caller_nid:
|
||||
pair = (caller_nid, tgt_nid)
|
||||
if pair not in seen_call_pairs:
|
||||
seen_call_pairs.add(pair)
|
||||
line = node.start_point[0] + 1
|
||||
edges.append({
|
||||
"source": caller_nid,
|
||||
"target": tgt_nid,
|
||||
"relation": "calls",
|
||||
"context": "call",
|
||||
"confidence": "EXTRACTED",
|
||||
"source_file": str_path,
|
||||
"source_location": f"L{line}",
|
||||
"weight": 1.0,
|
||||
})
|
||||
elif not is_scoped_call and callee_name.lower() not in _RUST_TRAIT_METHOD_BLOCKLIST:
|
||||
raw_calls.append({
|
||||
"caller_nid": caller_nid,
|
||||
"callee": callee_name,
|
||||
"is_member_call": is_member_call,
|
||||
"source_file": str_path,
|
||||
"source_location": f"L{node.start_point[0] + 1}",
|
||||
})
|
||||
for child in node.children:
|
||||
walk_calls(child, caller_nid)
|
||||
|
||||
for caller_nid, body_node in function_bodies:
|
||||
walk_calls(body_node, caller_nid)
|
||||
|
||||
valid_ids = seen_ids
|
||||
clean_edges = []
|
||||
for edge in edges:
|
||||
src, tgt = edge["source"], edge["target"]
|
||||
if src in valid_ids and (tgt in valid_ids or edge["relation"] in ("imports", "imports_from")):
|
||||
clean_edges.append(edge)
|
||||
|
||||
return {"nodes": nodes, "edges": clean_edges, "raw_calls": raw_calls}
|
||||
@@ -0,0 +1,81 @@
|
||||
"""Sln extractor. Moved verbatim from graphify/extract.py."""
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
|
||||
from pathlib import Path
|
||||
from graphify.extractors.base import _make_id
|
||||
|
||||
|
||||
def extract_sln(path: Path) -> dict:
|
||||
"""Extract projects and inter-project dependencies from a .sln file."""
|
||||
try:
|
||||
src = path.read_text(encoding="utf-8", errors="replace")
|
||||
except OSError:
|
||||
return {"nodes": [], "edges": [], "error": f"cannot read {path}"}
|
||||
|
||||
file_nid = _make_id(str(path))
|
||||
str_path = str(path)
|
||||
nodes: list[dict] = [{"id": file_nid, "label": path.name, "file_type": "code",
|
||||
"source_file": str_path, "source_location": None}]
|
||||
edges: list[dict] = []
|
||||
seen_ids: set[str] = set()
|
||||
seen_ids.add(file_nid)
|
||||
|
||||
_PROJECT_RE = re.compile(
|
||||
r'Project\("[^"]*"\)\s*=\s*"([^"]+)"\s*,\s*"([^"]+)"\s*,\s*"([^"]*)"'
|
||||
)
|
||||
_DEP_RE = re.compile(r'\{([0-9a-fA-F-]+)\}\s*=\s*\{([0-9a-fA-F-]+)\}')
|
||||
|
||||
guid_to_nid: dict[str, str] = {}
|
||||
|
||||
for m in _PROJECT_RE.finditer(src):
|
||||
proj_name = m.group(1)
|
||||
proj_path = m.group(2).replace("\\", "/")
|
||||
proj_guid = m.group(3).strip("{}")
|
||||
|
||||
try:
|
||||
abs_proj = str((path.parent / proj_path).resolve())
|
||||
except Exception:
|
||||
abs_proj = proj_path
|
||||
proj_nid = _make_id(abs_proj)
|
||||
if proj_nid and proj_nid not in seen_ids:
|
||||
seen_ids.add(proj_nid)
|
||||
nodes.append({"id": proj_nid, "label": proj_name,
|
||||
"file_type": "code", "source_file": abs_proj,
|
||||
"source_location": None})
|
||||
edges.append({"source": file_nid, "target": proj_nid,
|
||||
"relation": "contains", "confidence": "EXTRACTED",
|
||||
"source_file": str_path, "weight": 1.0})
|
||||
if proj_guid:
|
||||
guid_to_nid[proj_guid.lower()] = proj_nid
|
||||
|
||||
in_dep_section = False
|
||||
current_proj_guid: str | None = None
|
||||
_PROJECT_LINE_RE = re.compile(r'Project\("[^"]*"\)\s*=\s*"[^"]+"\s*,\s*"[^"]+"\s*,\s*"\{([^}]+)\}"')
|
||||
for line in src.splitlines():
|
||||
proj_line_m = _PROJECT_LINE_RE.search(line)
|
||||
if proj_line_m:
|
||||
current_proj_guid = proj_line_m.group(1).lower()
|
||||
continue
|
||||
if line.strip() == "EndProject":
|
||||
current_proj_guid = None
|
||||
continue
|
||||
if "ProjectSection(ProjectDependencies)" in line:
|
||||
in_dep_section = True
|
||||
continue
|
||||
if in_dep_section and "EndProjectSection" in line:
|
||||
in_dep_section = False
|
||||
continue
|
||||
if in_dep_section and current_proj_guid:
|
||||
dep_m = _DEP_RE.search(line)
|
||||
if dep_m:
|
||||
to_guid = dep_m.group(1).lower()
|
||||
from_nid = guid_to_nid.get(current_proj_guid)
|
||||
to_nid = guid_to_nid.get(to_guid)
|
||||
if from_nid and to_nid and from_nid != to_nid:
|
||||
edges.append({"source": from_nid, "target": to_nid,
|
||||
"relation": "imports", "confidence": "EXTRACTED",
|
||||
"source_file": str_path, "weight": 1.0})
|
||||
|
||||
return {"nodes": nodes, "edges": edges}
|
||||
@@ -0,0 +1,276 @@
|
||||
"""Sql extractor. Moved verbatim from graphify/extract.py."""
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
|
||||
from pathlib import Path
|
||||
from graphify.extractors.base import _file_stem, _make_id
|
||||
|
||||
|
||||
def extract_sql(path: Path, content: str | bytes | None = None) -> dict:
|
||||
"""Extract tables, views, functions, and relationships from .sql files via tree-sitter."""
|
||||
try:
|
||||
import tree_sitter_sql as tssql
|
||||
from tree_sitter import Language, Parser
|
||||
except ImportError:
|
||||
return {"nodes": [], "edges": [], "error": "tree_sitter_sql not installed. Run: pip install tree-sitter-sql"}
|
||||
|
||||
try:
|
||||
language = Language(tssql.language())
|
||||
parser = Parser(language)
|
||||
source = (
|
||||
content.encode("utf-8") if isinstance(content, str)
|
||||
else content if content is not None
|
||||
else path.read_bytes()
|
||||
)
|
||||
tree = parser.parse(source)
|
||||
root = tree.root_node
|
||||
except Exception as e:
|
||||
return {"nodes": [], "edges": [], "error": str(e)}
|
||||
|
||||
|
||||
stem = _file_stem(path)
|
||||
str_path = str(path)
|
||||
file_nid = _make_id(str_path)
|
||||
nodes: list[dict] = [{"id": file_nid, "label": path.name, "file_type": "code",
|
||||
"source_file": str_path, "source_location": None}]
|
||||
edges: list[dict] = []
|
||||
seen_ids: set[str] = {file_nid}
|
||||
table_nids: dict[str, str] = {} # name → nid for reference resolution
|
||||
|
||||
def _read(n) -> str:
|
||||
return source[n.start_byte:n.end_byte].decode("utf-8", errors="replace")
|
||||
|
||||
def _obj_name(n) -> str | None:
|
||||
for c in n.children:
|
||||
if c.type == "object_reference":
|
||||
return _read(c)
|
||||
return None
|
||||
|
||||
def _add_node(nid: str, label: str, line: int) -> None:
|
||||
if nid not in seen_ids:
|
||||
seen_ids.add(nid)
|
||||
nodes.append({"id": nid, "label": label, "file_type": "code",
|
||||
"source_file": str_path, "source_location": f"L{line}"})
|
||||
edges.append({"source": file_nid, "target": nid, "relation": "contains",
|
||||
"confidence": "EXTRACTED", "source_file": str_path,
|
||||
"source_location": f"L{line}", "weight": 1.0})
|
||||
|
||||
def _add_edge(src: str, tgt: str, relation: str, line: int) -> None:
|
||||
edges.append({"source": src, "target": tgt, "relation": relation,
|
||||
"confidence": "EXTRACTED", "source_file": str_path,
|
||||
"source_location": f"L{line}", "weight": 1.0})
|
||||
|
||||
def walk(node) -> None:
|
||||
t = node.type
|
||||
line = node.start_point[0] + 1
|
||||
|
||||
if t == "create_table":
|
||||
name = _obj_name(node)
|
||||
if name:
|
||||
nid = _make_id(stem, name)
|
||||
_add_node(nid, name, line)
|
||||
table_nids[name.lower()] = nid
|
||||
# Foreign key REFERENCES
|
||||
for col in node.children:
|
||||
if col.type == "column_definitions":
|
||||
has_error = any(cd.type == "ERROR" for cd in col.children)
|
||||
seen_refs: set[str] = set()
|
||||
for cd in col.children:
|
||||
if cd.type == "column_definition":
|
||||
# Inline column-level REFERENCES
|
||||
ref_name: str | None = None
|
||||
found_ref = False
|
||||
for cc in cd.children:
|
||||
if cc.type == "keyword_references":
|
||||
found_ref = True
|
||||
elif found_ref and cc.type == "object_reference":
|
||||
ref_name = _read(cc)
|
||||
break
|
||||
if ref_name:
|
||||
ref_nid = table_nids.get(ref_name.lower()) or _make_id(stem, ref_name)
|
||||
_add_edge(nid, ref_nid, "references", line)
|
||||
seen_refs.add(ref_name.lower())
|
||||
elif cd.type == "constraints":
|
||||
# Table-level FOREIGN KEY ... REFERENCES ... constraints
|
||||
for constraint in cd.children:
|
||||
if constraint.type != "constraint":
|
||||
continue
|
||||
ref_name = None
|
||||
found_ref = False
|
||||
for cc in constraint.children:
|
||||
if cc.type == "keyword_references":
|
||||
found_ref = True
|
||||
elif found_ref and cc.type == "object_reference":
|
||||
ref_name = _read(cc)
|
||||
break
|
||||
if ref_name:
|
||||
ref_nid = table_nids.get(ref_name.lower()) or _make_id(stem, ref_name)
|
||||
_add_edge(nid, ref_nid, "references", line)
|
||||
seen_refs.add(ref_name.lower())
|
||||
if has_error:
|
||||
# Dialect-specific syntax (e.g. Firebird COMPUTED BY) causes ERROR
|
||||
# nodes that make the parser drop the trailing constraints block.
|
||||
# Regex-scan the raw column_definitions text as fallback.
|
||||
col_text = _read(col)
|
||||
for rm in re.finditer(r"\bREFERENCES\s+([\w$]+)", col_text, re.IGNORECASE):
|
||||
ref_name = rm.group(1)
|
||||
if ref_name.lower() not in seen_refs:
|
||||
ref_nid = table_nids.get(ref_name.lower()) or _make_id(stem, ref_name)
|
||||
_add_edge(nid, ref_nid, "references", line)
|
||||
seen_refs.add(ref_name.lower())
|
||||
|
||||
elif t == "create_view":
|
||||
name = _obj_name(node)
|
||||
if name:
|
||||
nid = _make_id(stem, name)
|
||||
_add_node(nid, name, line)
|
||||
table_nids[name.lower()] = nid
|
||||
# FROM/JOIN table references inside view body
|
||||
_walk_from_refs(node, nid, line)
|
||||
|
||||
elif t == "create_function":
|
||||
name = _obj_name(node)
|
||||
if name:
|
||||
nid = _make_id(stem, name)
|
||||
_add_node(nid, f"{name}()", line)
|
||||
_walk_from_refs(node, nid, line)
|
||||
|
||||
elif t == "create_procedure":
|
||||
name = _obj_name(node)
|
||||
if name:
|
||||
nid = _make_id(stem, name)
|
||||
_add_node(nid, f"{name}()", line)
|
||||
_walk_from_refs(node, nid, line)
|
||||
|
||||
elif t == "alter_table":
|
||||
name = _obj_name(node)
|
||||
if name:
|
||||
src_nid = table_nids.get(name.lower())
|
||||
if not src_nid:
|
||||
src_nid = _make_id(stem, name)
|
||||
_add_node(src_nid, name, line)
|
||||
table_nids[name.lower()] = src_nid
|
||||
for child in node.children:
|
||||
if child.type == "add_constraint":
|
||||
for cc in child.children:
|
||||
if cc.type != "constraint":
|
||||
continue
|
||||
found_ref = False
|
||||
ref_name: str | None = None
|
||||
for ccc in cc.children:
|
||||
if ccc.type == "keyword_references":
|
||||
found_ref = True
|
||||
elif found_ref and ccc.type == "object_reference":
|
||||
ref_name = _read(ccc)
|
||||
break
|
||||
if ref_name:
|
||||
ref_nid = table_nids.get(ref_name.lower())
|
||||
if not ref_nid:
|
||||
ref_nid = _make_id(stem, ref_name)
|
||||
_add_edge(src_nid, ref_nid, "references", line)
|
||||
|
||||
elif t == "create_trigger":
|
||||
trig_name: str | None = None
|
||||
tbl_name: str | None = None
|
||||
after_trigger = False
|
||||
after_for = False
|
||||
for c in node.children:
|
||||
if c.type == "keyword_trigger":
|
||||
after_trigger = True
|
||||
elif after_trigger and not trig_name and c.type == "object_reference":
|
||||
trig_name = _read(c)
|
||||
elif c.type == "keyword_for":
|
||||
after_for = True
|
||||
elif after_for and not tbl_name and c.type == "object_reference":
|
||||
tbl_name = _read(c)
|
||||
if trig_name:
|
||||
trig_nid = _make_id(stem, trig_name)
|
||||
_add_node(trig_nid, trig_name, line)
|
||||
if tbl_name:
|
||||
tbl_nid = table_nids.get(tbl_name.lower()) or _make_id(stem, tbl_name)
|
||||
_add_edge(trig_nid, tbl_nid, "triggers", line)
|
||||
|
||||
elif t == "fb_proc_or_trigger":
|
||||
text = _read(node)
|
||||
m = re.match(
|
||||
r"CREATE\s+(?:OR\s+(?:REPLACE|ALTER)\s+)?"
|
||||
r"(PROCEDURE|TRIGGER|FUNCTION)\s+([\w$]+)",
|
||||
text, re.IGNORECASE,
|
||||
)
|
||||
if m:
|
||||
obj_type = m.group(1).upper()
|
||||
obj_name = m.group(2)
|
||||
obj_nid = _make_id(stem, obj_name)
|
||||
label = obj_name if obj_type == "TRIGGER" else f"{obj_name}()"
|
||||
_add_node(obj_nid, label, line)
|
||||
if obj_type == "TRIGGER":
|
||||
fm = re.search(r"\bFOR\s+([\w$]+)", text, re.IGNORECASE)
|
||||
if fm:
|
||||
tbl = fm.group(1)
|
||||
tbl_nid = table_nids.get(tbl.lower()) or _make_id(stem, tbl)
|
||||
_add_edge(obj_nid, tbl_nid, "triggers", line)
|
||||
_NON_TABLES = {
|
||||
"select", "where", "set", "dual", "null", "true", "false",
|
||||
"first", "skip", "rows", "next", "only", "lateral",
|
||||
}
|
||||
seen_tbls: set[str] = set()
|
||||
for rm in re.finditer(r"\b(?:FROM|JOIN|INTO)\s+([\w$]+)", text, re.IGNORECASE):
|
||||
tbl = rm.group(1)
|
||||
if tbl.lower() not in _NON_TABLES and tbl.lower() not in seen_tbls:
|
||||
seen_tbls.add(tbl.lower())
|
||||
tbl_nid = table_nids.get(tbl.lower()) or _make_id(stem, tbl)
|
||||
_add_edge(obj_nid, tbl_nid, "reads_from", line)
|
||||
for rm in re.finditer(r"\bUPDATE\s+([\w$]+)", text, re.IGNORECASE):
|
||||
tbl = rm.group(1)
|
||||
if tbl.lower() not in _NON_TABLES and tbl.lower() not in seen_tbls:
|
||||
seen_tbls.add(tbl.lower())
|
||||
tbl_nid = table_nids.get(tbl.lower()) or _make_id(stem, tbl)
|
||||
_add_edge(obj_nid, tbl_nid, "reads_from", line)
|
||||
|
||||
for child in node.children:
|
||||
walk(child)
|
||||
|
||||
def _walk_from_refs(node, caller_nid: str, line: int) -> None:
|
||||
"""Recursively find FROM/JOIN table references inside a node."""
|
||||
if node.type in ("from", "join"):
|
||||
for c in node.children:
|
||||
if c.type == "relation":
|
||||
for cc in c.children:
|
||||
if cc.type == "object_reference":
|
||||
tbl = _read(cc)
|
||||
tbl_nid = _make_id(stem, tbl)
|
||||
_add_edge(caller_nid, tbl_nid, "reads_from",
|
||||
c.start_point[0] + 1)
|
||||
for child in node.children:
|
||||
_walk_from_refs(child, caller_nid, line)
|
||||
|
||||
for stmt in root.children:
|
||||
if stmt.type == "statement":
|
||||
for child in stmt.children:
|
||||
walk(child)
|
||||
elif stmt.type in ("fb_proc_or_trigger", "set_term", "declare_external_function"):
|
||||
walk(stmt)
|
||||
|
||||
# Global regex fallback: catch any REFERENCES missed due to ERROR nodes in the parse tree
|
||||
# (e.g. Firebird COMPUTED BY columns push constraints out of the tree entirely).
|
||||
# Snapshot after tree walk so we don't re-emit edges already captured above.
|
||||
emitted = {(e["source"], e["target"]) for e in edges if e["relation"] == "references"}
|
||||
src_text = source.decode("utf-8", errors="replace")
|
||||
for m in re.finditer(r"CREATE\s+TABLE\s+([\w$]+)\s*\(", src_text, re.IGNORECASE):
|
||||
tbl_name = m.group(1)
|
||||
tbl_nid = table_nids.get(tbl_name.lower())
|
||||
if tbl_nid is None:
|
||||
continue
|
||||
tbl_line = src_text[: m.start()].count("\n") + 1
|
||||
tail = src_text[m.start():]
|
||||
end = re.search(r"(?:^|\n)(?:CREATE|SET\s+TERM|ALTER)\s", tail[1:], re.IGNORECASE)
|
||||
block = tail[: end.start() + 1] if end else tail
|
||||
for rm in re.finditer(r"\bREFERENCES\s+([\w$]+)", block, re.IGNORECASE):
|
||||
ref_name = rm.group(1)
|
||||
ref_nid = table_nids.get(ref_name.lower()) or _make_id(stem, ref_name)
|
||||
if (tbl_nid, ref_nid) not in emitted:
|
||||
_add_edge(tbl_nid, ref_nid, "references", tbl_line)
|
||||
emitted.add((tbl_nid, ref_nid))
|
||||
|
||||
return {"nodes": nodes, "edges": edges}
|
||||
@@ -0,0 +1,181 @@
|
||||
"""Terraform extractor. Moved verbatim from graphify/extract.py."""
|
||||
from __future__ import annotations
|
||||
|
||||
|
||||
from pathlib import Path
|
||||
from graphify.extractors.base import _make_id
|
||||
|
||||
|
||||
_TF_META_HEADS = frozenset({"count", "each", "self", "path", "terraform"})
|
||||
|
||||
def extract_terraform(path: Path) -> dict:
|
||||
"""Extract Terraform/HCL blocks and the references between them via tree-sitter.
|
||||
|
||||
Nodes: resources, data sources, modules, variables, outputs, providers, and
|
||||
locals. Edges: `contains` (file -> block), `references` (block -> the blocks
|
||||
it interpolates, e.g. `aws_instance.web` -> `var.region`), and `depends_on`
|
||||
(explicit dependency edges).
|
||||
|
||||
Node IDs are scoped by the parent directory, not the file stem, because
|
||||
Terraform resources are module(directory)-scoped: a resource defined in
|
||||
main.tf is referenced from other .tf files in the same directory. Directory
|
||||
scoping lets those cross-file references resolve when per-file extractions
|
||||
are merged (stem scoping would split a definition from its references).
|
||||
"""
|
||||
try:
|
||||
import tree_sitter_hcl as tshcl
|
||||
from tree_sitter import Language, Parser
|
||||
except ImportError:
|
||||
return {"nodes": [], "edges": [], "error": "tree_sitter_hcl not installed. Run: pip install tree-sitter-hcl"}
|
||||
|
||||
try:
|
||||
language = Language(tshcl.language())
|
||||
parser = Parser(language)
|
||||
source = path.read_bytes()
|
||||
tree = parser.parse(source)
|
||||
root = tree.root_node
|
||||
except Exception as e:
|
||||
return {"nodes": [], "edges": [], "error": str(e)}
|
||||
|
||||
str_path = str(path)
|
||||
file_nid = _make_id(str_path)
|
||||
scope = path.parent.name or "tf"
|
||||
|
||||
nodes: list[dict] = [{"id": file_nid, "label": path.name, "file_type": "code",
|
||||
"source_file": str_path, "source_location": None}]
|
||||
edges: list[dict] = []
|
||||
seen_ids: set[str] = {file_nid}
|
||||
seen_edges: set[tuple[str, str, str]] = set()
|
||||
|
||||
def _read(n) -> str:
|
||||
return source[n.start_byte:n.end_byte].decode("utf-8", errors="replace")
|
||||
|
||||
def _label_text(n) -> str:
|
||||
return _read(n).strip().strip('"')
|
||||
|
||||
def _add_node(address: str, label: str, line: int) -> str:
|
||||
nid = _make_id(scope, address)
|
||||
if nid not in seen_ids:
|
||||
seen_ids.add(nid)
|
||||
nodes.append({"id": nid, "label": label, "file_type": "code",
|
||||
"source_file": str_path, "source_location": f"L{line}"})
|
||||
edges.append({"source": file_nid, "target": nid, "relation": "contains",
|
||||
"confidence": "EXTRACTED", "source_file": str_path,
|
||||
"source_location": f"L{line}", "weight": 1.0})
|
||||
return nid
|
||||
|
||||
def _add_edge(src: str, address: str, relation: str, line: int) -> None:
|
||||
tgt = _make_id(scope, address)
|
||||
if src == tgt:
|
||||
return
|
||||
key = (src, tgt, relation)
|
||||
if key in seen_edges:
|
||||
return
|
||||
seen_edges.add(key)
|
||||
edges.append({"source": src, "target": tgt, "relation": relation,
|
||||
"confidence": "EXTRACTED", "source_file": str_path,
|
||||
"source_location": f"L{line}", "weight": 1.0})
|
||||
|
||||
def _block_parts(block) -> tuple:
|
||||
btype = None
|
||||
labels: list[str] = []
|
||||
for c in block.children:
|
||||
if c.type in ("block_start", "body", "block_end"):
|
||||
break
|
||||
if c.type == "identifier" and btype is None:
|
||||
btype = _read(c)
|
||||
elif c.type in ("string_lit", "identifier"):
|
||||
labels.append(_label_text(c))
|
||||
return btype, labels
|
||||
|
||||
def _ref_address(expr):
|
||||
head = _read(expr)
|
||||
parent = expr.parent
|
||||
attrs: list[str] = []
|
||||
if parent is not None:
|
||||
seen_self = False
|
||||
for c in parent.children:
|
||||
if c.id == expr.id:
|
||||
seen_self = True
|
||||
continue
|
||||
if seen_self and c.type == "get_attr":
|
||||
name = None
|
||||
for gc in c.children:
|
||||
if gc.type == "identifier":
|
||||
name = _read(gc)
|
||||
break
|
||||
if name is None:
|
||||
break
|
||||
attrs.append(name)
|
||||
elif seen_self and c.type not in ("get_attr",):
|
||||
break
|
||||
if head in _TF_META_HEADS or not head:
|
||||
return None
|
||||
if head == "var":
|
||||
return f"var.{attrs[0]}" if attrs else None
|
||||
if head == "local":
|
||||
return f"local.{attrs[0]}" if attrs else None
|
||||
if head == "module":
|
||||
return f"module.{attrs[0]}" if attrs else None
|
||||
if head == "data":
|
||||
return f"data.{attrs[0]}.{attrs[1]}" if len(attrs) >= 2 else None
|
||||
return f"{head}.{attrs[0]}" if attrs else None
|
||||
|
||||
def _collect_refs(node, owner_nid: str, relation: str) -> None:
|
||||
rel = relation
|
||||
if node.type == "attribute":
|
||||
key_node = node.child_by_field_name("key") or (
|
||||
node.children[0] if node.children else None
|
||||
)
|
||||
if key_node is not None and _read(key_node) == "depends_on":
|
||||
rel = "depends_on"
|
||||
if node.type == "variable_expr":
|
||||
addr = _ref_address(node)
|
||||
if addr:
|
||||
_add_edge(owner_nid, addr, rel, node.start_point[0] + 1)
|
||||
for c in node.children:
|
||||
if c.is_named:
|
||||
_collect_refs(c, owner_nid, rel)
|
||||
|
||||
def _body_of(block):
|
||||
for c in block.children:
|
||||
if c.type == "body":
|
||||
return c
|
||||
return None
|
||||
|
||||
body = next((c for c in root.children if c.type == "body"), root)
|
||||
for block in body.children:
|
||||
if block.type != "block":
|
||||
continue
|
||||
btype, labels = _block_parts(block)
|
||||
line = block.start_point[0] + 1
|
||||
blk_body = _body_of(block)
|
||||
if btype == "resource" and len(labels) >= 2:
|
||||
owner = _add_node(f"{labels[0]}.{labels[1]}", f"{labels[0]}.{labels[1]}", line)
|
||||
elif btype == "data" and len(labels) >= 2:
|
||||
owner = _add_node(f"data.{labels[0]}.{labels[1]}", f"data.{labels[0]}.{labels[1]}", line)
|
||||
elif btype == "module" and labels:
|
||||
owner = _add_node(f"module.{labels[0]}", f"module.{labels[0]}", line)
|
||||
elif btype == "variable" and labels:
|
||||
owner = _add_node(f"var.{labels[0]}", f"var.{labels[0]}", line)
|
||||
elif btype == "output" and labels:
|
||||
owner = _add_node(f"output.{labels[0]}", f"output.{labels[0]}", line)
|
||||
elif btype == "provider" and labels:
|
||||
owner = _add_node(f"provider.{labels[0]}", f"provider.{labels[0]}", line)
|
||||
elif btype == "locals" and blk_body is not None:
|
||||
for attr in blk_body.children:
|
||||
if attr.type != "attribute":
|
||||
continue
|
||||
key_node = attr.children[0] if attr.children else None
|
||||
if key_node is None:
|
||||
continue
|
||||
key = _read(key_node)
|
||||
lnid = _add_node(f"local.{key}", f"local.{key}", attr.start_point[0] + 1)
|
||||
_collect_refs(attr, lnid, "references")
|
||||
continue
|
||||
else:
|
||||
continue
|
||||
if blk_body is not None:
|
||||
_collect_refs(blk_body, owner, "references")
|
||||
|
||||
return {"nodes": nodes, "edges": edges}
|
||||
@@ -0,0 +1,329 @@
|
||||
"""Verilog extractor. Moved verbatim from graphify/extract.py."""
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
|
||||
from pathlib import Path
|
||||
from graphify.extractors.base import _file_stem, _make_id, _read_text
|
||||
|
||||
|
||||
def _sv_first_identifier(node, source: bytes) -> str | None:
|
||||
"""First `simple_identifier` under node in pre-order, or None.
|
||||
|
||||
tree-sitter-verilog 1.0.3 nests declaration names a few levels deep instead
|
||||
of exposing a `name` field. Scope the search to the right child node (e.g.
|
||||
`function_identifier`) or this returns the return-type instead of the name.
|
||||
"""
|
||||
if node is None:
|
||||
return None
|
||||
for child in node.children:
|
||||
if child.type == "simple_identifier":
|
||||
return _read_text(child, source)
|
||||
found = _sv_first_identifier(child, source)
|
||||
if found:
|
||||
return found
|
||||
return None
|
||||
|
||||
def _sv_child(node, type_name: str) -> object | None:
|
||||
if node is None:
|
||||
return None
|
||||
for child in node.children:
|
||||
if child.type == type_name:
|
||||
return child
|
||||
return None
|
||||
|
||||
_SV_BUILTIN_TYPES = frozenset({
|
||||
"bit", "logic", "reg", "wire", "int", "integer", "shortint", "longint",
|
||||
"byte", "time", "real", "shortreal", "void", "string", "type", "event",
|
||||
"mailbox", "semaphore", "process", "chandle",
|
||||
})
|
||||
|
||||
_SV_NON_TYPE_WORDS = frozenset({
|
||||
"return", "if", "else", "for", "foreach", "while", "case", "begin", "end",
|
||||
"function", "task", "class", "endclass", "endfunction", "endtask",
|
||||
})
|
||||
|
||||
_SV_PARENS_INNER = r"(?:[^()]|\([^()]*\))*"
|
||||
|
||||
_SV_PARENS = r"\(" + _SV_PARENS_INNER + r"\)"
|
||||
|
||||
_SV_FUNC_RE = re.compile(
|
||||
r"\bfunction\s+([A-Za-z_]\w*(?:\s*#\s*" + _SV_PARENS + r")?)\s+(\w+)\s*"
|
||||
r"\((" + _SV_PARENS_INNER + r")\)\s*;",
|
||||
re.MULTILINE,
|
||||
)
|
||||
|
||||
_SV_PARAM_RE = re.compile(
|
||||
r"\s*(?:input|output|inout|ref|const\s+ref)?\s*"
|
||||
r"([A-Za-z_]\w*(?:\s*#\s*" + _SV_PARENS + r")?)\s+\w+"
|
||||
)
|
||||
|
||||
def _sv_strip_comments(text: str) -> str:
|
||||
text = re.sub(r"/\*.*?\*/", "", text, flags=re.DOTALL)
|
||||
return re.sub(r"//.*", "", text)
|
||||
|
||||
def _sv_split_type_list(text: str) -> list[str]:
|
||||
parts: list[str] = []
|
||||
depth = 0
|
||||
start = 0
|
||||
for idx, ch in enumerate(text):
|
||||
if ch == "(":
|
||||
depth += 1
|
||||
elif ch == ")":
|
||||
depth = max(0, depth - 1)
|
||||
elif ch == "," and depth == 0:
|
||||
item = text[start:idx].strip()
|
||||
if item:
|
||||
parts.append(item)
|
||||
start = idx + 1
|
||||
item = text[start:].strip()
|
||||
if item:
|
||||
parts.append(item)
|
||||
return parts
|
||||
|
||||
def _sv_collect_type_refs(type_text: str, generic: bool = False,
|
||||
skip: frozenset[str] = frozenset()) -> list[tuple[str, str]]:
|
||||
refs: list[tuple[str, str]] = []
|
||||
text = type_text.strip()
|
||||
if not text:
|
||||
return refs
|
||||
head = re.match(r"([A-Za-z_]\w*)", text)
|
||||
if head:
|
||||
name = head.group(1)
|
||||
# `skip` carries the enclosing class's `#(type T = ...)` parameters so
|
||||
# they are not mistaken for referenced types.
|
||||
if name not in _SV_BUILTIN_TYPES and name not in _SV_NON_TYPE_WORDS and name not in skip:
|
||||
refs.append((name, "generic_arg" if generic else "type"))
|
||||
params = re.search(r"#\s*\((" + _SV_PARENS_INNER + r")\)", text)
|
||||
if params:
|
||||
for arg in _sv_split_type_list(params.group(1)):
|
||||
refs.extend(_sv_collect_type_refs(arg, generic=True, skip=skip))
|
||||
return refs
|
||||
|
||||
def _augment_systemverilog_semantics(
|
||||
raw: str,
|
||||
stem: str,
|
||||
str_path: str,
|
||||
file_nid: str,
|
||||
nodes: list[dict],
|
||||
edges: list[dict],
|
||||
seen_ids: set[str],
|
||||
) -> None:
|
||||
label_to_nid = {node["label"]: node["id"] for node in nodes}
|
||||
|
||||
def line_for(offset: int) -> int:
|
||||
return raw.count("\n", 0, offset) + 1
|
||||
|
||||
def add_node(nid: str, label: str, line: int) -> None:
|
||||
if nid not in seen_ids:
|
||||
seen_ids.add(nid)
|
||||
nodes.append({"id": nid, "label": label, "file_type": "code",
|
||||
"source_file": str_path, "source_location": f"L{line}",
|
||||
"confidence_score": 1.0})
|
||||
label_to_nid[label] = nid
|
||||
|
||||
def ensure_type(label: str, line: int) -> str:
|
||||
if label in label_to_nid:
|
||||
return label_to_nid[label]
|
||||
nid = _make_id(stem, label)
|
||||
add_node(nid, label, line)
|
||||
return nid
|
||||
|
||||
def add_edge(src: str, target_label: str, relation: str, line: int, context: str | None = None) -> None:
|
||||
tgt = ensure_type(target_label, line)
|
||||
edge = {"source": src, "target": tgt, "relation": relation,
|
||||
"confidence": "EXTRACTED", "confidence_score": 1.0,
|
||||
"source_file": str_path, "source_location": f"L{line}", "weight": 1.0}
|
||||
if context:
|
||||
edge["context"] = context
|
||||
edges.append(edge)
|
||||
|
||||
text = _sv_strip_comments(raw)
|
||||
# Consuming `endclass` (rather than a lookahead) makes each match own its
|
||||
# terminator, so back-to-back or malformed classes cannot bleed bodies.
|
||||
class_re = re.compile(
|
||||
r"\b(?:(interface)\s+)?class\s+(\w+)([^;{]*)\s*;(.*?)\bendclass\b",
|
||||
re.DOTALL,
|
||||
)
|
||||
for match in class_re.finditer(text):
|
||||
class_name = match.group(2)
|
||||
header = match.group(3) or ""
|
||||
body = match.group(4) or ""
|
||||
line = line_for(match.start())
|
||||
# `#(type T = Payload)` declares `T` as a class type parameter, not a
|
||||
# referenced type — collect these to skip below.
|
||||
type_params = frozenset(re.findall(r"\btype\s+(\w+)", header))
|
||||
class_nid = _make_id(stem, class_name)
|
||||
add_node(class_nid, class_name, line)
|
||||
edges.append({"source": file_nid, "target": class_nid, "relation": "defines",
|
||||
"confidence": "EXTRACTED", "confidence_score": 1.0,
|
||||
"source_file": str_path, "source_location": f"L{line}", "weight": 1.0})
|
||||
|
||||
ext = re.search(r"\bextends\s+(\w+)", header)
|
||||
if ext:
|
||||
add_edge(class_nid, ext.group(1), "inherits", line)
|
||||
impl = re.search(r"\bimplements\s+([^;{]+)", header)
|
||||
if impl:
|
||||
for iface_name in _sv_split_type_list(impl.group(1)):
|
||||
add_edge(class_nid, iface_name.split("#", 1)[0].strip(), "implements", line)
|
||||
|
||||
body_without_functions = re.sub(
|
||||
r"\bfunction\b.*?\bendfunction\b",
|
||||
lambda m: "\n" * m.group(0).count("\n"),
|
||||
body,
|
||||
flags=re.DOTALL,
|
||||
)
|
||||
# Optional leading class-property qualifiers (rand/local/protected/etc.)
|
||||
# must be consumed: otherwise a qualified field like `rand Config x;`
|
||||
# (three tokens) fails the `<type> <name>;` shape and its type reference
|
||||
# is silently dropped.
|
||||
for field in re.finditer(r"^\s*(?:(?:rand|randc|local|protected|static|const|automatic|var)\s+)*([A-Za-z_]\w*(?:\s*#\s*\([^;]+?\))?)\s+\w+\s*;", body_without_functions, re.MULTILINE):
|
||||
# Count to the start of the type token (group 1), not the match
|
||||
# start: `^\s*` consumes the leading newline(s), so field.start()
|
||||
# would resolve to the class's line instead of the field's.
|
||||
field_line = line + body_without_functions.count("\n", 0, field.start(1))
|
||||
for ref_name, role in _sv_collect_type_refs(field.group(1), skip=type_params):
|
||||
add_edge(class_nid, ref_name, "references", field_line, "generic_arg" if role == "generic_arg" else "field")
|
||||
|
||||
for fm in _SV_FUNC_RE.finditer(body):
|
||||
return_type, func_name, params = fm.group(1), fm.group(2), fm.group(3)
|
||||
func_line = line + body.count("\n", 0, fm.start())
|
||||
func_nid = _make_id(class_nid, func_name)
|
||||
add_node(func_nid, func_name, func_line)
|
||||
edges.append({"source": class_nid, "target": func_nid, "relation": "method",
|
||||
"confidence": "EXTRACTED", "confidence_score": 1.0,
|
||||
"source_file": str_path, "source_location": f"L{func_line}", "weight": 1.0})
|
||||
for ref_name, role in _sv_collect_type_refs(return_type, skip=type_params):
|
||||
add_edge(func_nid, ref_name, "references", func_line, "generic_arg" if role == "generic_arg" else "return_type")
|
||||
for param in _sv_split_type_list(params):
|
||||
pm = _SV_PARAM_RE.match(param)
|
||||
if not pm:
|
||||
continue
|
||||
for ref_name, role in _sv_collect_type_refs(pm.group(1), skip=type_params):
|
||||
add_edge(func_nid, ref_name, "references", func_line, "generic_arg" if role == "generic_arg" else "parameter_type")
|
||||
|
||||
def extract_verilog(path: Path) -> dict:
|
||||
"""Extract modules, functions, tasks, package imports, instantiations, and
|
||||
SystemVerilog class semantics (inherits/implements edges, field/parameter/
|
||||
return-type references) from .v/.sv files."""
|
||||
try:
|
||||
import tree_sitter_verilog as tsverilog
|
||||
from tree_sitter import Language, Parser
|
||||
except ImportError:
|
||||
return {"nodes": [], "edges": [], "error": "tree_sitter_verilog not installed"}
|
||||
|
||||
try:
|
||||
language = Language(tsverilog.language())
|
||||
parser = Parser(language)
|
||||
source = path.read_bytes()
|
||||
tree = parser.parse(source)
|
||||
root = tree.root_node
|
||||
except Exception as e:
|
||||
return {"nodes": [], "edges": [], "error": str(e)}
|
||||
|
||||
stem = _file_stem(path)
|
||||
str_path = str(path)
|
||||
nodes: list[dict] = []
|
||||
edges: list[dict] = []
|
||||
seen_ids: set[str] = set()
|
||||
|
||||
def add_node(nid: str, label: str, line: int) -> None:
|
||||
if nid not in seen_ids:
|
||||
seen_ids.add(nid)
|
||||
nodes.append({"id": nid, "label": label, "file_type": "code",
|
||||
"source_file": str_path, "source_location": f"L{line}",
|
||||
"confidence_score": 1.0})
|
||||
|
||||
def add_edge(src: str, tgt: str, relation: str, line: int,
|
||||
confidence: str = "EXTRACTED", score: float = 1.0) -> None:
|
||||
edges.append({"source": src, "target": tgt, "relation": relation,
|
||||
"confidence": confidence, "confidence_score": score,
|
||||
"source_file": str_path, "source_location": f"L{line}", "weight": 1.0})
|
||||
|
||||
file_nid = _make_id(str(path))
|
||||
add_node(file_nid, path.name, 1)
|
||||
|
||||
def walk(node, module_nid: str | None = None) -> None:
|
||||
t = node.type
|
||||
|
||||
# SystemVerilog class bodies are handled by _augment_systemverilog_semantics
|
||||
# (regex over source text). Skip their subtrees so in-class methods are not
|
||||
# double-emitted here — and with the wrong, return-type-derived name.
|
||||
if t in ("class_declaration", "interface_class_declaration"):
|
||||
return
|
||||
|
||||
if t == "module_declaration":
|
||||
mod_name = _sv_first_identifier(_sv_child(node, "module_header"), source)
|
||||
if mod_name:
|
||||
line = node.start_point[0] + 1
|
||||
nid = _make_id(stem, mod_name)
|
||||
add_node(nid, mod_name, line)
|
||||
add_edge(file_nid, nid, "defines", line)
|
||||
for child in node.children:
|
||||
walk(child, nid)
|
||||
return
|
||||
|
||||
# `function_prototype` only appears inside class/interface-class bodies
|
||||
# (skipped above) and nests its name differently; it is intentionally not
|
||||
# handled here.
|
||||
elif t == "function_declaration":
|
||||
fn_body = _sv_child(node, "function_body_declaration")
|
||||
func_name = _sv_first_identifier(_sv_child(fn_body, "function_identifier"), source)
|
||||
if func_name:
|
||||
line = node.start_point[0] + 1
|
||||
parent = module_nid or file_nid
|
||||
nid = _make_id(parent, func_name)
|
||||
add_node(nid, f"{func_name}()", line)
|
||||
add_edge(parent, nid, "contains", line)
|
||||
|
||||
elif t == "task_declaration":
|
||||
tk_body = _sv_child(node, "task_body_declaration")
|
||||
task_name = _sv_first_identifier(_sv_child(tk_body, "task_identifier"), source)
|
||||
if task_name:
|
||||
line = node.start_point[0] + 1
|
||||
parent = module_nid or file_nid
|
||||
nid = _make_id(parent, task_name)
|
||||
add_node(nid, task_name, line)
|
||||
add_edge(parent, nid, "contains", line)
|
||||
|
||||
elif t == "package_import_declaration":
|
||||
for child in node.children:
|
||||
if child.type == "package_import_item":
|
||||
pkg_text = _read_text(child, source)
|
||||
pkg_name = pkg_text.split("::")[0].strip()
|
||||
if pkg_name:
|
||||
line = node.start_point[0] + 1
|
||||
tgt_nid = _make_id(pkg_name)
|
||||
add_node(tgt_nid, pkg_name, line)
|
||||
src_nid = module_nid or file_nid
|
||||
add_edge(src_nid, tgt_nid, "imports_from", line)
|
||||
|
||||
elif t in ("module_instantiation", "checker_instantiation"):
|
||||
# `leaf u_leaf();` parses as checker_instantiation in 1.0.3;
|
||||
# module_instantiation (when it occurs) exposes a `module_type` field.
|
||||
# Both reduce to the first identifier under the node — the instantiated
|
||||
# type, not the instance name (which appears later).
|
||||
if module_nid:
|
||||
type_node = node.child_by_field_name("module_type")
|
||||
inst_type = (_read_text(type_node, source).strip() if type_node
|
||||
else _sv_first_identifier(node, source))
|
||||
if inst_type:
|
||||
line = node.start_point[0] + 1
|
||||
tgt_nid = _make_id(inst_type)
|
||||
add_node(tgt_nid, inst_type, line)
|
||||
add_edge(module_nid, tgt_nid, "instantiates", line)
|
||||
|
||||
for child in node.children:
|
||||
walk(child, module_nid)
|
||||
|
||||
walk(root)
|
||||
_augment_systemverilog_semantics(
|
||||
source.decode("utf-8", errors="replace"),
|
||||
stem,
|
||||
str_path,
|
||||
file_nid,
|
||||
nodes,
|
||||
edges,
|
||||
seen_ids,
|
||||
)
|
||||
return {"nodes": nodes, "edges": edges}
|
||||
@@ -0,0 +1,175 @@
|
||||
"""Zig extractor (tree-sitter). Moved verbatim from graphify/extract.py."""
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from graphify.extractors.base import _file_stem, _make_id, _read_text
|
||||
|
||||
|
||||
def extract_zig(path: Path) -> dict:
|
||||
"""Extract functions, structs, enums, unions, and imports from a .zig file."""
|
||||
try:
|
||||
import tree_sitter_zig as tszig
|
||||
from tree_sitter import Language, Parser
|
||||
except ImportError:
|
||||
return {"nodes": [], "edges": [], "error": "tree_sitter_zig not installed"}
|
||||
|
||||
try:
|
||||
language = Language(tszig.language())
|
||||
parser = Parser(language)
|
||||
source = path.read_bytes()
|
||||
tree = parser.parse(source)
|
||||
root = tree.root_node
|
||||
except Exception as e:
|
||||
return {"nodes": [], "edges": [], "error": str(e)}
|
||||
|
||||
stem = _file_stem(path)
|
||||
str_path = str(path)
|
||||
nodes: list[dict] = []
|
||||
edges: list[dict] = []
|
||||
seen_ids: set[str] = set()
|
||||
function_bodies: list[tuple[str, Any]] = []
|
||||
|
||||
def add_node(nid: str, label: str, line: int) -> None:
|
||||
if nid not in seen_ids:
|
||||
seen_ids.add(nid)
|
||||
nodes.append({"id": nid, "label": label, "file_type": "code",
|
||||
"source_file": str_path, "source_location": f"L{line}"})
|
||||
|
||||
def add_edge(src: str, tgt: str, relation: str, line: int,
|
||||
confidence: str = "EXTRACTED", weight: float = 1.0,
|
||||
context: str | None = None) -> None:
|
||||
edge = {"source": src, "target": tgt, "relation": relation,
|
||||
"confidence": confidence, "source_file": str_path,
|
||||
"source_location": f"L{line}", "weight": weight}
|
||||
if context:
|
||||
edge["context"] = context
|
||||
edges.append(edge)
|
||||
|
||||
file_nid = _make_id(str(path))
|
||||
add_node(file_nid, path.name, 1)
|
||||
|
||||
def _extract_import(node) -> None:
|
||||
for child in node.children:
|
||||
if child.type == "builtin_function":
|
||||
bi = None
|
||||
args = None
|
||||
for c in child.children:
|
||||
if c.type == "builtin_identifier":
|
||||
bi = _read_text(c, source)
|
||||
elif c.type == "arguments":
|
||||
args = c
|
||||
if bi in ("@import", "@cImport") and args:
|
||||
for arg in args.children:
|
||||
if arg.type in ("string_literal", "string"):
|
||||
raw = _read_text(arg, source).strip('"')
|
||||
module_name = raw.split("/")[-1].split(".")[0]
|
||||
if module_name:
|
||||
tgt_nid = _make_id(module_name)
|
||||
add_edge(file_nid, tgt_nid, "imports_from",
|
||||
node.start_point[0] + 1)
|
||||
return
|
||||
elif child.type == "field_expression":
|
||||
_extract_import(child)
|
||||
return
|
||||
|
||||
def walk(node, parent_struct_nid: str | None = None) -> None:
|
||||
t = node.type
|
||||
|
||||
if t == "function_declaration":
|
||||
name_node = node.child_by_field_name("name")
|
||||
if name_node:
|
||||
func_name = _read_text(name_node, source)
|
||||
line = node.start_point[0] + 1
|
||||
if parent_struct_nid:
|
||||
func_nid = _make_id(parent_struct_nid, func_name)
|
||||
add_node(func_nid, f".{func_name}()", line)
|
||||
add_edge(parent_struct_nid, func_nid, "method", line)
|
||||
else:
|
||||
func_nid = _make_id(stem, func_name)
|
||||
add_node(func_nid, f"{func_name}()", line)
|
||||
add_edge(file_nid, func_nid, "contains", line)
|
||||
body = node.child_by_field_name("body")
|
||||
if body:
|
||||
function_bodies.append((func_nid, body))
|
||||
return
|
||||
|
||||
if t == "variable_declaration":
|
||||
name_node = None
|
||||
value_node = None
|
||||
for child in node.children:
|
||||
if child.type == "identifier":
|
||||
name_node = child
|
||||
elif child.type in ("struct_declaration", "enum_declaration",
|
||||
"union_declaration", "builtin_function",
|
||||
"field_expression"):
|
||||
value_node = child
|
||||
|
||||
if value_node and value_node.type == "struct_declaration":
|
||||
if name_node:
|
||||
struct_name = _read_text(name_node, source)
|
||||
line = node.start_point[0] + 1
|
||||
struct_nid = _make_id(stem, struct_name)
|
||||
add_node(struct_nid, struct_name, line)
|
||||
add_edge(file_nid, struct_nid, "contains", line)
|
||||
for child in value_node.children:
|
||||
walk(child, parent_struct_nid=struct_nid)
|
||||
return
|
||||
|
||||
if value_node and value_node.type in ("enum_declaration", "union_declaration"):
|
||||
if name_node:
|
||||
type_name = _read_text(name_node, source)
|
||||
line = node.start_point[0] + 1
|
||||
type_nid = _make_id(stem, type_name)
|
||||
add_node(type_nid, type_name, line)
|
||||
add_edge(file_nid, type_nid, "contains", line)
|
||||
return
|
||||
|
||||
if value_node and value_node.type in ("builtin_function", "field_expression"):
|
||||
_extract_import(node)
|
||||
return
|
||||
|
||||
for child in node.children:
|
||||
walk(child, parent_struct_nid)
|
||||
|
||||
walk(root)
|
||||
|
||||
seen_call_pairs: set[tuple[str, str]] = set()
|
||||
raw_calls: list[dict] = []
|
||||
|
||||
def walk_calls(node, caller_nid: str) -> None:
|
||||
if node.type == "function_declaration":
|
||||
return
|
||||
if node.type == "call_expression":
|
||||
fn = node.child_by_field_name("function")
|
||||
if fn:
|
||||
fn_text = _read_text(fn, source)
|
||||
callee = fn_text.split(".")[-1]
|
||||
is_member_call = "." in fn_text
|
||||
tgt_nid = next((n["id"] for n in nodes if n["label"] in
|
||||
(f"{callee}()", f".{callee}()")), None)
|
||||
if tgt_nid and tgt_nid != caller_nid:
|
||||
pair = (caller_nid, tgt_nid)
|
||||
if pair not in seen_call_pairs:
|
||||
seen_call_pairs.add(pair)
|
||||
add_edge(caller_nid, tgt_nid, "calls",
|
||||
node.start_point[0] + 1,
|
||||
confidence="EXTRACTED", weight=1.0)
|
||||
elif callee:
|
||||
raw_calls.append({
|
||||
"caller_nid": caller_nid,
|
||||
"callee": callee,
|
||||
"is_member_call": is_member_call,
|
||||
"source_file": str_path,
|
||||
"source_location": f"L{node.start_point[0] + 1}",
|
||||
})
|
||||
for child in node.children:
|
||||
walk_calls(child, caller_nid)
|
||||
|
||||
for caller_nid, body_node in function_bodies:
|
||||
walk_calls(body_node, caller_nid)
|
||||
|
||||
clean_edges = [e for e in edges if e["source"] in seen_ids and
|
||||
(e["target"] in seen_ids or e["relation"] == "imports_from")]
|
||||
return {"nodes": nodes, "edges": clean_edges, "raw_calls": raw_calls}
|
||||
Reference in New Issue
Block a user