"""Deterministic structural extraction from source code using tree-sitter. Outputs nodes+edges dicts.""" from __future__ import annotations import hashlib import importlib import json import os import re import sys from dataclasses import dataclass, field from pathlib import Path from typing import Any, Callable from .cache import load_cached, save_cached from .mcp_ingest import extract_mcp_config, is_mcp_config_path from .manifest_ingest import extract_package_manifest, is_package_manifest_path from .resolver_registry import ( LanguageResolver, register as register_language_resolver, run_language_resolvers, ) from .ruby_resolution import resolve_ruby_member_calls from .pascal_resolution import resolve_pascal_inherited_calls # --- migrated to graphify/extractors/ (see graphify/extractors/MIGRATION.md) --- from graphify.extractors.base import ( # noqa: F401 _LANGUAGE_BUILTIN_GLOBALS, _file_stem, _make_id, _read_text, ) from graphify.extractors.apex import extract_apex # noqa: F401 from graphify.extractors.bash import extract_bash # noqa: F401 from graphify.extractors.blade import extract_blade # noqa: F401 from graphify.extractors.csharp import ( _resolve_cross_file_csharp_imports, _resolve_csharp_type_references, ) from graphify.extractors.dart import extract_dart # noqa: F401 from graphify.extractors.dm import extract_dm, extract_dmf, extract_dmi, extract_dmm # noqa: F401 from graphify.extractors.elixir import extract_elixir # noqa: F401 from graphify.extractors.fortran import _cpp_preprocess, extract_fortran # noqa: F401 from graphify.extractors.go import extract_go # noqa: F401 from graphify.extractors.json_config import extract_json # noqa: F401 from graphify.extractors.markdown import extract_markdown # noqa: F401 from graphify.extractors.pascal_forms import extract_delphi_form, extract_lazarus_form # noqa: F401 from graphify.extractors.powershell import extract_powershell, extract_powershell_manifest # noqa: F401 from graphify.extractors.razor import extract_razor # noqa: F401 from graphify.extractors.rust import extract_rust # noqa: F401 from graphify.extractors.sln import extract_sln # noqa: F401 from graphify.extractors.sql import extract_sql # noqa: F401 from graphify.extractors.terraform import extract_terraform # noqa: F401 from graphify.extractors.verilog import extract_verilog # noqa: F401 from graphify.extractors.zig import extract_zig # noqa: F401 from graphify.security import sanitize_metadata from graphify.paths import disambiguate_ambiguous_candidates from graphify.extractors.models import LanguageConfig, _JS_CACHE_BYPASS_SUFFIXES, _NamespaceExportFact, _StarExportFact, _SymbolAliasFact, _SymbolDeclarationFact, _SymbolExportFact, _SymbolImportFact, _SymbolResolutionFacts, _SymbolUseFact, _WORKSPACE_PACKAGE_CACHE # noqa: E402,F401 from graphify.extractors.resolution import ( # noqa: E402,F401 _DECLDEF_HEADER_SUFFIXES, _DECLDEF_IMPL_SUFFIXES, _EXPORT_CONDITION_PRIORITY, _JS_INDEX_FILES, _JS_PRIMITIVE_TYPES, _JS_RESOLVE_EXTS, _TSCONFIG_ALIAS_CACHE, _VUE_SCRIPT_LANG_RE, _VUE_SCRIPT_RE, _WORKSPACE_MANIFEST_NAMES, _apply_symbol_resolution_facts, _augment_symbol_resolution_edges, _collect_js_symbol_resolution_facts, _collect_python_symbol_resolution_facts, _contained_in_package, _decldef_class_stem, _disambiguate_colliding_node_ids, _find_workspace_root, _is_type_like_definition, _js_call_identifier, _js_default_export_name, _js_default_import_name, _js_export_clause, _js_export_statement_is_star, _js_exported_declaration_names, _js_lexical_aliases, _js_module_specifier, _js_named_specifiers, _js_namespace_export_name, _js_source_path, _js_top_level_function_bodies, _load_tsconfig_aliases, _load_workspace_packages, _match_tsconfig_alias, _merge_decl_def_classes, _node_disambiguation_source_key, _package_entry_candidates, _parse_js_tree, _parse_python_tree, _pascal_class_stem_cache, _pascal_project_root, _pascal_resolve_class, _pascal_resolve_unit, _pascal_unit_cache, _pnpm_workspace_globs, _python_call_identifier, _python_import_from_module, _python_imported_names, _python_top_level_function_bodies, _read_tsconfig_aliases, _resolve_c_include_path, _resolve_cross_file_imports, _resolve_cross_file_java_imports, _resolve_export_target, _resolve_java_type_references, _resolve_js_import_path, _resolve_js_import_target, _resolve_js_module_path, _resolve_lua_import_target, _resolve_python_module_path, _resolve_tsconfig_alias, _resolve_workspace_import, _source_key, _strip_jsonc, _ts_collect_type_refs, _ts_heritage_clause_entries, _ts_walk_class_members, _vue_mask_non_script, _walk_js_tree, _walk_python_tree, _workspace_globs, ) from graphify.extractors.engine import REFERENCE_CONTEXTS, _CSHARP_TYPE_PARAMETER_SCOPE_DECLARATIONS, _C_PRIMITIVE_TYPE_NODES, _JAVA_BUILTIN_TYPES, _JAVA_TYPE_PARAMETER_SCOPE_DECLARATIONS, _JS_FUNCTION_VALUE_TYPES, _JS_SCOPE_BOUNDARY, _PYTHON_ANNOTATION_NOISE, _PYTHON_TYPE_CONTAINERS, _RUBY_CLASS_FACTORIES, _c_collect_type_refs, _cpp_collect_type_refs, _cpp_declarator_name, _cpp_local_var_types, _csharp_attribute_names, _csharp_classify_base, _csharp_collect_type_refs, _csharp_extra_walk, _csharp_member_type_table, _csharp_namespace_id, _csharp_namespace_name, _csharp_pre_scan_interfaces, _csharp_type_parameters_in_scope, _dynamic_import_js, _extract_generic, _find_body, _find_require_call, _get_cpp_func_name, _java_annotation_names, _java_collect_type_refs, _java_extra_walk, _java_type_parameters_in_scope, _js_collect_pattern_idents, _js_dispatch_value_idents, _js_extra_walk, _js_local_bound_names, _js_member_assignment_target, _js_module_bound_names, _kotlin_collect_type_refs, _kotlin_function_return_type_node, _kotlin_property_type_node, _kotlin_user_type_name, _php_collect_type_refs, _php_method_return_type_node, _php_name_text, _python_collect_assignment_targets, _python_collect_param_refs, _python_collect_type_refs, _python_local_bound_names, _python_module_bound_names, _python_param_names, _read_csharp_type_name, _require_imports_js, _ruby_const_last_name, _ruby_extra_walk, _ruby_local_class_bindings, _ruby_new_class_name, _scala_collect_type_refs, _semantic_reference_edge, _source_location, _swift_classify_base, _swift_collect_type_refs, _swift_constructor_type, _swift_declaration_keyword, _swift_extra_walk, _swift_local_var_types, _swift_pre_scan, _swift_property_name, _swift_property_type_node, _swift_receiver_name, _swift_user_type_name, _ts_decorator_name, _ts_descendant_decorators, _ts_emit_decorator_edges, _ts_extra_walk, _ts_method_name, _ts_receiver_type_table # noqa: E402,F401 from graphify.extractors.pascal import _PAS_BEGIN_END_TOKEN_RE, _PAS_CALL_RE, _PAS_END_SEMI_RE, _PAS_IMPL_HEADER_RE, _PAS_KEYWORDS, _PAS_METHOD_DECL_RE, _PAS_MODULE_RE, _PAS_TOKEN_RE, _PAS_TYPE_HEADER_RE, _PAS_USES_RE, _extract_pascal_regex, _pascal_find_body, _pascal_split_bases, _pascal_split_sections, _pascal_split_uses, _pascal_strip_comments, extract_pascal # noqa: E402,F401 from graphify.extractors.objc import _objc_local_var_types, extract_objc # noqa: E402,F401 from graphify.extractors.julia import extract_julia # noqa: E402,F401 _RECURSION_LIMIT = 10_000 # Language built-in globals that AST may classify as call targets when used as # constructors or coercion functions (e.g. String(x), Number(x), Boolean(x)). # Without this filter they become god-nodes accumulating spurious edges from # every call site. Filter applied at same-file and cross-file resolution. # See issue #726. def _raise_recursion_limit() -> None: if sys.getrecursionlimit() < _RECURSION_LIMIT: sys.setrecursionlimit(_RECURSION_LIMIT) def _safe_extract(extractor: Callable, path: Path) -> dict: try: return extractor(path) except RecursionError: print(f" warning: skipped {path} (recursion limit exceeded)", file=sys.stderr, flush=True) return {"nodes": [], "edges": [], "error": "recursion_limit_exceeded"} except Exception as e: if os.environ.get("GRAPHIFY_DEBUG"): import traceback traceback.print_exc(file=sys.stderr) print(f" warning: skipped {path} ({type(e).__name__}: {e})", file=sys.stderr, flush=True) return {"nodes": [], "edges": [], "error": f"{type(e).__name__}: {e}"} def _file_node_id(rel_path: Path) -> str: """File-level node ID matching the skill.md spec: ``{parent_dir}_{stem}`` — one parent directory level, no extension. ``rel_path`` MUST be relative to the project root so top-level files collapse to a bare stem (``setup.py`` -> ``setup``) instead of picking up the root directory name. This must equal the ID semantic subagents generate, or AST and semantic extraction split a file into two disconnected ghost nodes (#1033).""" return _make_id(_file_stem(rel_path)) SEMANTIC_RELATIONS = frozenset({ "inherits", "implements", "mixes_in", "embeds", "references", "calls", "imports", "imports_from", "re_exports", "contains", "method", }) # Condition keys consulted when resolving an `exports` target, in priority # order. `default` is Node's catch-all and must be consulted LAST so a more # specific condition (source/import/module/etc.) wins when several match. # ── LanguageConfig dataclass ───────────────────────────────────────────────── # ── Generic helpers ─────────────────────────────────────────────────────────── # Scalar builtins and test-mock names that appear as type annotations but carry # no useful semantic meaning as graph nodes (#1147). Suppressed at the annotation # walker level so they are never created as nodes or emitted as edges. # java.lang (auto-imported) plus the ubiquitous java.util / java.io / java.time / # java.util.{stream,function,concurrent} / java.math / java.nio.file types that # appear as field, parameter, return, and generic-argument annotations. They never # resolve to a project node, so emitting `references` edges to them is pure noise # (mirrors _GO_PREDECLARED_TYPES / _PYTHON_ANNOTATION_NOISE). Suppressed at the # type-ref walker so they are never created as nodes or emitted as edges. The # boxed-scalar/`void` primitives are already dropped by grammar node type above; # these are the class/interface names the grammar reports as identifiers. # ── C / C++ type-ref helpers ───────────────────────────────────────────────── # ── Scala type-ref helpers ─────────────────────────────────────────────────── def _resolve_name(node, source: bytes, config: LanguageConfig) -> str | None: """Get the name from a node using config.name_field, falling back to child types.""" if config.resolve_function_name_fn is not None: # For C/C++ where the name is inside a declarator return None # caller handles this separately n = node.child_by_field_name(config.name_field) if n: return _read_text(n, source) for child in node.children: if child.type in config.name_fallback_child_types: return _read_text(child, source) return None # ── Import handlers ─────────────────────────────────────────────────────────── def _import_python(node, source: bytes, file_nid: str, stem: str, edges: list, str_path: str, scope_stack: list[str] | None = None) -> None: t = node.type if t == "import_statement": for child in node.children: if child.type in ("dotted_name", "aliased_import"): raw = _read_text(child, source) module_name = raw.split(" as ")[0].strip().lstrip(".") tgt_nid = _make_id(module_name) edges.append({ "source": file_nid, "target": tgt_nid, "relation": "imports", "context": "import", "confidence": "EXTRACTED", "source_file": str_path, "source_location": f"L{node.start_point[0] + 1}", "weight": 1.0, }) elif t == "import_from_statement": module_node = node.child_by_field_name("module_name") if module_node: raw = _read_text(module_node, source) if raw.startswith("."): # Relative import - resolve to full path so IDs match file node IDs dots = len(raw) - len(raw.lstrip(".")) module_name = raw.lstrip(".") base = Path(str_path).parent for _ in range(dots - 1): base = base.parent rel = (module_name.replace(".", "/") + ".py") if module_name else "__init__.py" tgt_nid = _make_id(str(base / rel)) else: tgt_nid = _make_id(raw) edges.append({ "source": file_nid, "target": tgt_nid, "relation": "imports_from", "context": "import", "confidence": "EXTRACTED", "source_file": str_path, "source_location": f"L{node.start_point[0] + 1}", "weight": 1.0, }) def _import_js(node, source: bytes, file_nid: str, stem: str, edges: list, str_path: str, scope_stack: list[str] | None = None) -> None: is_reexport = node.type == "export_statement" # Only handle export_statement if it has a `from` clause (re-export). # Pure exports like `export const x = 1` or `export { localVar }` have no source module. if is_reexport: has_from = any(child.type == "from" or (_read_text(child, source) == "from") for child in node.children if child.type in ("from", "identifier")) if not has_from: # Check for string child (source path) as a more reliable indicator has_from = any(child.type == "string" for child in node.children) if not has_from: return resolved_path: "Path | None" = None module_string = None for child in node.children: if child.type == "string": module_string = child break if child.type == "import_require_clause": # TS import-equals form: `import x = require("./m")`. The module # string sits inside the clause, not on the import_statement # itself, so the direct-child scan above never sees it. module_string = next( (sub for sub in child.children if sub.type == "string"), None ) break if module_string is not None: raw = _read_text(module_string, source).strip("'\"` ") resolved = _resolve_js_import_target(raw, str_path) if resolved is not None: tgt_nid, resolved_path = resolved edges.append({ "source": file_nid, "target": tgt_nid, "relation": "imports_from", "context": "re-export" if is_reexport else "import", "confidence": "EXTRACTED", "source_file": str_path, "source_location": f"L{node.start_point[0] + 1}", "weight": 1.0, }) # Emit symbol-level edges for named imports/re-exports from local/aliased files. # e.g. `import { Foo, type Bar } from './bar'` → file → Foo, file → Bar (EXTRACTED) # e.g. `export { Foo } from './bar'` → file → Foo (re_exports edge) # Uses the same _make_id(target_stem, name) key that _extract_generic emits when # defining the symbol, so these edges wire importers directly to existing symbol nodes. if resolved_path is not None: target_stem = _file_stem(resolved_path) line = node.start_point[0] + 1 if is_reexport: # Handle: export { foo, bar } from './module' # export { default as baz } from './module' for child in node.children: if child.type == "export_clause": for spec in child.children: if spec.type == "export_specifier": # The exported name is the local name from the source module name_node = spec.child_by_field_name("name") if name_node: sym = _read_text(name_node, source) if sym == "default": continue # skip default re-exports for ID matching edges.append({ "source": file_nid, "target": _make_id(target_stem, sym), "relation": "re_exports", "context": "re-export", "confidence": "EXTRACTED", "source_file": str_path, "source_location": f"L{line}", "weight": 1.0, }) else: # Handle: import { Foo, type Bar } from './bar' for child in node.children: if child.type == "import_clause": for sub in child.children: if sub.type == "named_imports": for spec in sub.children: if spec.type == "import_specifier": name_node = spec.child_by_field_name("name") if name_node: sym = _read_text(name_node, source) edges.append({ "source": file_nid, "target": _make_id(target_stem, sym), "relation": "imports", "context": "import", "confidence": "EXTRACTED", "source_file": str_path, "source_location": f"L{line}", "weight": 1.0, }) def _import_java(node, source: bytes, file_nid: str, stem: str, edges: list, str_path: str, scope_stack: list[str] | None = None) -> None: def _walk_scoped(n) -> str: parts: list[str] = [] cur = n while cur: if cur.type == "scoped_identifier": name_node = cur.child_by_field_name("name") if name_node: parts.append(_read_text(name_node, source)) cur = cur.child_by_field_name("scope") elif cur.type == "identifier": parts.append(_read_text(cur, source)) break else: break parts.reverse() return ".".join(parts) for child in node.children: if child.type in ("scoped_identifier", "identifier"): path_str = _walk_scoped(child) module_name = path_str.split(".")[-1].strip("*").strip(".") or ( path_str.split(".")[-2] if len(path_str.split(".")) > 1 else path_str ) if module_name: tgt_nid = _make_id(module_name) edges.append({ "source": file_nid, "target": tgt_nid, "relation": "imports", "context": "import", "confidence": "EXTRACTED", "source_file": str_path, "source_location": f"L{node.start_point[0] + 1}", "weight": 1.0, }) break def _import_c(node, source: bytes, file_nid: str, stem: str, edges: list, str_path: str, scope_stack: list[str] | None = None) -> None: for child in node.children: if child.type in ("string_literal", "system_lib_string", "string"): raw = _read_text(child, source).strip('"<> ') # Quoted includes: try to resolve to a real file so the target ID # matches the node ID _extract_generic creates for that file. if child.type != "system_lib_string": resolved = _resolve_c_include_path(raw, str_path) if resolved is not None: tgt_nid = _make_id(str(resolved)) edges.append({ "source": file_nid, "target": tgt_nid, "relation": "imports", "context": "import", "confidence": "EXTRACTED", "source_file": str_path, "source_location": f"L{node.start_point[0] + 1}", "weight": 1.0, }) break module_name = raw.split("/")[-1].split(".")[0] if module_name: tgt_nid = _make_id(module_name) edges.append({ "source": file_nid, "target": tgt_nid, "relation": "imports", "context": "import", "confidence": "EXTRACTED", "source_file": str_path, "source_location": f"L{node.start_point[0] + 1}", "weight": 1.0, }) break def _import_csharp(node, source: bytes, file_nid: str, stem: str, edges: list, str_path: str, scope_stack: list[str] | None = None) -> None: text = _read_text(node, source).strip().rstrip(";") if text.startswith("global "): text = text[len("global "):].strip() if not text.startswith("using"): return body = text[len("using"):].strip() using_kind, alias, target_fqn = "namespace", None, body if body.startswith("static "): using_kind, target_fqn = "static", body[len("static "):].strip() elif "=" in body: lhs, rhs = body.split("=", 1) using_kind, alias, target_fqn = "alias", lhs.strip(), rhs.strip() if not target_fqn: return edges.append({ "source": file_nid, "target": _make_id(target_fqn), "relation": "imports", "context": "import", "confidence": "EXTRACTED", "source_file": str_path, "source_location": f"L{node.start_point[0] + 1}", "weight": 1.0, "metadata": sanitize_metadata({k: v for k, v in {"using_kind": using_kind, "alias": alias, "target_fqn": target_fqn, "scope_kind": "namespace" if scope_stack else "file", "scope_id": scope_stack[-1] if scope_stack else None}.items() if v is not None}), }) def _import_kotlin(node, source: bytes, file_nid: str, stem: str, edges: list, str_path: str, scope_stack: list[str] | None = None) -> None: path_node = node.child_by_field_name("path") if path_node: raw = _read_text(path_node, source) module_name = raw.split(".")[-1].strip() if module_name: tgt_nid = _make_id(module_name) edges.append({ "source": file_nid, "target": tgt_nid, "relation": "imports", "context": "import", "confidence": "EXTRACTED", "source_file": str_path, "source_location": f"L{node.start_point[0] + 1}", "weight": 1.0, }) return # Fallback: find identifier child for child in node.children: if child.type == "identifier": raw = _read_text(child, source) tgt_nid = _make_id(raw) edges.append({ "source": file_nid, "target": tgt_nid, "relation": "imports", "context": "import", "confidence": "EXTRACTED", "source_file": str_path, "source_location": f"L{node.start_point[0] + 1}", "weight": 1.0, }) break def _import_scala(node, source: bytes, file_nid: str, stem: str, edges: list, str_path: str, scope_stack: list[str] | None = None) -> None: for child in node.children: if child.type in ("stable_id", "identifier"): raw = _read_text(child, source) module_name = raw.split(".")[-1].strip("{} ") if module_name and module_name != "_": tgt_nid = _make_id(module_name) edges.append({ "source": file_nid, "target": tgt_nid, "relation": "imports", "context": "import", "confidence": "EXTRACTED", "source_file": str_path, "source_location": f"L{node.start_point[0] + 1}", "weight": 1.0, }) break def _import_php(node, source: bytes, file_nid: str, stem: str, edges: list, str_path: str, scope_stack: list[str] | None = None) -> None: for child in node.children: if child.type in ("qualified_name", "name", "identifier"): raw = _read_text(child, source) module_name = raw.split("\\")[-1].strip() if module_name: tgt_nid = _make_id(module_name) edges.append({ "source": file_nid, "target": tgt_nid, "relation": "imports", "context": "import", "confidence": "EXTRACTED", "source_file": str_path, "source_location": f"L{node.start_point[0] + 1}", "weight": 1.0, }) break # ── C/C++ function name helpers ─────────────────────────────────────────────── def _get_c_func_name(node, source: bytes) -> str | None: """Recursively unwrap declarator to find the innermost identifier (C).""" if node.type == "identifier": return _read_text(node, source) decl = node.child_by_field_name("declarator") if decl: return _get_c_func_name(decl, source) for child in node.children: if child.type == "identifier": return _read_text(child, source) return None # ── JS/TS extra walk for arrow functions ────────────────────────────────────── # Node types whose value is a callable, for the JS/TS assignment / class-field # / function-expression forms below. Older tree-sitter-javascript grammars # label a function expression `function`; current ones use `function_expression`. # ── TS extra walk for namespace / module declarations ───────────────────────── # ── C# extra walk for namespace declarations ────────────────────────────────── # ── Swift extra walk for enum cases ────────────────────────────────────────── # ── Java extra walk for enum constants ─────────────────────────────────────── # ── Language configs ────────────────────────────────────────────────────────── _PYTHON_CONFIG = LanguageConfig( ts_module="tree_sitter_python", class_types=frozenset({"class_definition"}), function_types=frozenset({"function_definition"}), import_types=frozenset({"import_statement", "import_from_statement"}), call_types=frozenset({"call"}), call_function_field="function", call_accessor_node_types=frozenset({"attribute"}), call_accessor_field="attribute", call_accessor_object_field="object", function_boundary_types=frozenset({"function_definition"}), import_handler=_import_python, ) _JS_CONFIG = LanguageConfig( ts_module="tree_sitter_javascript", class_types=frozenset({"class_declaration"}), function_types=frozenset({"function_declaration", "generator_function_declaration", "method_definition"}), import_types=frozenset({"import_statement", "export_statement"}), call_types=frozenset({"call_expression", "new_expression"}), call_function_field="function", call_accessor_node_types=frozenset({"member_expression"}), call_accessor_field="property", call_accessor_object_field="object", function_boundary_types=frozenset({"function_declaration", "generator_function_declaration", "arrow_function", "method_definition"}), import_handler=_import_js, ) _TS_CONFIG = LanguageConfig( ts_module="tree_sitter_typescript", ts_language_fn="language_typescript", class_types=frozenset({ "class_declaration", "abstract_class_declaration", # TS abstract class "interface_declaration", # parity with Java/C# "enum_declaration", # named enums "type_alias_declaration", # named type aliases }), function_types=frozenset({"function_declaration", "generator_function_declaration", "method_definition", "method_signature"}), import_types=frozenset({"import_statement", "export_statement"}), call_types=frozenset({"call_expression", "new_expression"}), call_function_field="function", call_accessor_node_types=frozenset({"member_expression"}), call_accessor_field="property", call_accessor_object_field="object", function_boundary_types=frozenset({"function_declaration", "generator_function_declaration", "arrow_function", "method_definition"}), import_handler=_import_js, ) # .tsx files must use the TSX grammar (JSX-aware), not the plain TypeScript grammar. # tree-sitter-typescript ships two languages: language_typescript (for .ts) and # language_tsx (for .tsx). Parsing .tsx with language_typescript silently fails on # JSX expressions, dropping any call_expression nested inside JSX (e.g. {fmtDate(x)}). _TSX_CONFIG = LanguageConfig( ts_module="tree_sitter_typescript", ts_language_fn="language_tsx", class_types=_TS_CONFIG.class_types, function_types=_TS_CONFIG.function_types, import_types=_TS_CONFIG.import_types, call_types=_TS_CONFIG.call_types, call_function_field=_TS_CONFIG.call_function_field, call_accessor_node_types=_TS_CONFIG.call_accessor_node_types, call_accessor_field=_TS_CONFIG.call_accessor_field, call_accessor_object_field=_TS_CONFIG.call_accessor_object_field, function_boundary_types=_TS_CONFIG.function_boundary_types, import_handler=_TS_CONFIG.import_handler, ) _JAVA_CONFIG = LanguageConfig( ts_module="tree_sitter_java", # record_declaration shares class_declaration's name/body/interfaces fields, # so it becomes a first-class type node instead of an isolated file (#1373). # Enums and annotation declarations use the same name/body contract. class_types=frozenset({ "class_declaration", "interface_declaration", "record_declaration", "enum_declaration", "annotation_type_declaration", }), function_types=frozenset({"method_declaration", "constructor_declaration"}), import_types=frozenset({"import_declaration"}), # object_creation_expression (`new Foo(...)`) is handled by a dedicated Java # branch in walk_calls below — its callee is in the `type` field, not `name`. call_types=frozenset({"method_invocation", "object_creation_expression"}), call_function_field="name", call_accessor_node_types=frozenset(), function_boundary_types=frozenset({"method_declaration", "constructor_declaration"}), import_handler=_import_java, ) _GROOVY_CONFIG = LanguageConfig( ts_module="tree_sitter_groovy", class_types=frozenset({"class_declaration", "interface_declaration"}), function_types=frozenset({"method_declaration", "constructor_declaration"}), import_types=frozenset({"import_declaration"}), call_types=frozenset({"method_invocation"}), call_function_field="name", call_accessor_node_types=frozenset(), function_boundary_types=frozenset({"method_declaration", "constructor_declaration"}), import_handler=_import_java, ) _C_CONFIG = LanguageConfig( ts_module="tree_sitter_c", class_types=frozenset(), function_types=frozenset({"function_definition"}), import_types=frozenset({"preproc_include"}), call_types=frozenset({"call_expression"}), call_function_field="function", call_accessor_node_types=frozenset({"field_expression"}), call_accessor_field="field", function_boundary_types=frozenset({"function_definition"}), import_handler=_import_c, resolve_function_name_fn=_get_c_func_name, ) _CPP_CONFIG = LanguageConfig( ts_module="tree_sitter_cpp", class_types=frozenset({"class_specifier", "struct_specifier"}), function_types=frozenset({"function_definition"}), import_types=frozenset({"preproc_include"}), call_types=frozenset({"call_expression"}), call_function_field="function", call_accessor_node_types=frozenset({"field_expression", "qualified_identifier"}), call_accessor_field="field", function_boundary_types=frozenset({"function_definition"}), import_handler=_import_c, resolve_function_name_fn=_get_cpp_func_name, ) _RUBY_CONFIG = LanguageConfig( ts_module="tree_sitter_ruby", # `module Foo` is a container node just like `class Foo` in tree-sitter's # Ruby grammar (name in a `constant` child, body in `body_statement`), so it # gets a node and its methods attach via `method` (#1640). Without it, plain # utility/`module_function` modules produced no node and their methods hung # off the file via `contains` with dot-less labels. class_types=frozenset({"class", "module"}), function_types=frozenset({"method", "singleton_method"}), import_types=frozenset(), call_types=frozenset({"call"}), call_function_field="method", call_accessor_node_types=frozenset(), name_fallback_child_types=("constant", "scope_resolution", "identifier"), body_fallback_child_types=("body_statement",), function_boundary_types=frozenset({"method", "singleton_method"}), ) _CSHARP_CONFIG = LanguageConfig( ts_module="tree_sitter_c_sharp", class_types=frozenset({ "class_declaration", "interface_declaration", "enum_declaration", "struct_declaration", "record_declaration", }), function_types=frozenset({"method_declaration"}), import_types=frozenset({"using_directive"}), call_types=frozenset({"invocation_expression"}), call_function_field="function", call_accessor_node_types=frozenset({"member_access_expression"}), call_accessor_field="name", body_fallback_child_types=("declaration_list",), function_boundary_types=frozenset({"method_declaration"}), import_handler=_import_csharp, ) _KOTLIN_CONFIG = LanguageConfig( ts_module="tree_sitter_kotlin", class_types=frozenset({"class_declaration", "object_declaration"}), function_types=frozenset({"function_declaration"}), import_types=frozenset({"import_header"}), call_types=frozenset({"call_expression"}), call_function_field="", call_accessor_node_types=frozenset({"navigation_expression"}), call_accessor_field="", # Different tree-sitter-kotlin grammar versions name plain identifier # nodes differently: PyPI's `tree_sitter_kotlin` uses `identifier`, # older forks use `simple_identifier`. Accept both so the extractor # works across grammar generations. name_fallback_child_types=("simple_identifier", "identifier"), body_fallback_child_types=("function_body", "class_body", "enum_class_body"), function_boundary_types=frozenset({"function_declaration"}), import_handler=_import_kotlin, ) _SCALA_CONFIG = LanguageConfig( ts_module="tree_sitter_scala", class_types=frozenset({"class_definition", "object_definition"}), function_types=frozenset({"function_definition"}), import_types=frozenset({"import_declaration"}), call_types=frozenset({"call_expression"}), call_function_field="", call_accessor_node_types=frozenset({"field_expression"}), call_accessor_field="field", name_fallback_child_types=("identifier",), body_fallback_child_types=("template_body",), function_boundary_types=frozenset({"function_definition"}), import_handler=_import_scala, ) _PHP_CONFIG = LanguageConfig( ts_module="tree_sitter_php", ts_language_fn="language_php", class_types=frozenset({"class_declaration"}), function_types=frozenset({"function_definition", "method_declaration"}), import_types=frozenset({"namespace_use_clause"}), call_types=frozenset({"function_call_expression", "member_call_expression", "scoped_call_expression", "class_constant_access_expression"}), static_prop_types=frozenset({"scoped_property_access_expression"}), helper_fn_names=frozenset({"config"}), container_bind_methods=frozenset({"bind", "singleton", "scoped", "instance"}), event_listener_properties=frozenset({"listen", "subscribe"}), call_function_field="function", call_accessor_node_types=frozenset({"member_call_expression"}), call_accessor_field="name", name_fallback_child_types=("name",), body_fallback_child_types=("declaration_list", "compound_statement"), function_boundary_types=frozenset({"function_definition", "method_declaration"}), import_handler=_import_php, ) def _import_lua(node, source: bytes, file_nid: str, stem: str, edges: list, str_path: str, scope_stack: list[str] | None = None) -> None: """Extract require('module') from Lua variable_declaration nodes.""" text = _read_text(node, source) import re m = re.search(r"""require\s*[\('"]\s*['"]?([^'")\s]+)""", text) if m: raw_module = m.group(1) if raw_module: tgt_nid = _resolve_lua_import_target(raw_module, str_path) if tgt_nid: edges.append({ "source": file_nid, "target": tgt_nid, "relation": "imports", "context": "import", "confidence": "EXTRACTED", "confidence_score": 1.0, "source_file": str_path, "source_location": str(node.start_point[0] + 1), "weight": 1.0, }) _LUA_CONFIG = LanguageConfig( ts_module="tree_sitter_lua", ts_language_fn="language", class_types=frozenset(), function_types=frozenset({"function_declaration"}), import_types=frozenset({"variable_declaration"}), call_types=frozenset({"function_call"}), call_function_field="name", call_accessor_node_types=frozenset({"method_index_expression"}), call_accessor_field="name", name_fallback_child_types=("identifier", "method_index_expression"), body_fallback_child_types=("block",), function_boundary_types=frozenset({"function_declaration"}), import_handler=_import_lua, ) def _import_swift(node, source: bytes, file_nid: str, stem: str, edges: list, str_path: str, scope_stack: list[str] | None = None) -> list[tuple[str, str]]: """Emit module-level ``imports`` edges and report the imported modules. A Swift ``import CoreKit`` names a module, not a file path, so — unlike the file-resolving JS/TS handlers — there is no existing node for the edge to point at. The returned ``(id, label)`` pairs let the extractor materialize a ``type=module`` anchor node so the edge survives; without it ``build_from_json`` prunes every Swift import edge as a dangling/external reference (#1327). """ modules: list[tuple[str, str]] = [] for child in node.children: if child.type == "identifier": raw = _read_text(child, source) tgt_nid = _make_id(raw) edges.append({ "source": file_nid, "target": tgt_nid, "relation": "imports", "context": "import", "confidence": "EXTRACTED", "source_file": str_path, "source_location": f"L{node.start_point[0] + 1}", "weight": 1.0, }) modules.append((tgt_nid, raw)) break return modules _SWIFT_CONFIG = LanguageConfig( ts_module="tree_sitter_swift", class_types=frozenset({"class_declaration", "protocol_declaration"}), function_types=frozenset({"function_declaration", "init_declaration", "deinit_declaration", "subscript_declaration"}), import_types=frozenset({"import_declaration"}), call_types=frozenset({"call_expression"}), call_function_field="", call_accessor_node_types=frozenset({"navigation_expression"}), call_accessor_field="", name_fallback_child_types=("simple_identifier", "type_identifier", "user_type"), body_fallback_child_types=("class_body", "protocol_body", "function_body", "enum_class_body"), function_boundary_types=frozenset({"function_declaration", "init_declaration", "deinit_declaration", "subscript_declaration"}), import_handler=_import_swift, ) # ── Ruby local type inference (for member-call resolution) ───────────────────── # `Const = (...)` shapes that define a lightweight class named after the # constant. tree-sitter parses each as an `assignment`, not a `class`, so the # generic class branch never saw them (#1640). # ── Generic extractor ───────────────────────────────────────────────────────── # ── Python rationale extraction ─────────────────────────────────────────────── _RATIONALE_PREFIXES = ("# NOTE:", "# IMPORTANT:", "# HACK:", "# WHY:", "# RATIONALE:", "# TODO:", "# FIXME:") def _is_autogenerated_python(source: bytes) -> bool: """Return True if this Python file is auto-generated and its module docstring is noise. Covers: Alembic/Flask-Migrate revisions, Django migrations, protobuf/gRPC/OpenAPI stubs. Module docstrings in these files are change annotations or boilerplate, not rationale. """ head = source[:2048].decode("utf-8", errors="replace") # Generic generated-file markers (protobuf, gRPC, OpenAPI codegen, etc.) if any(m in head for m in ("DO NOT EDIT", "@generated", "Generated by the protocol buffer")): return True # Alembic / Flask-Migrate revision files if (re.search(r"^revision\s*[:=]", head, re.MULTILINE) and "def upgrade(" in head and "down_revision" in head): return True # Django migrations if "class Migration(migrations.Migration)" in head and "operations" in head: return True return False def _extract_python_rationale(path: Path, result: dict) -> None: """Post-pass: extract docstrings and rationale comments from Python source. Mutates result in-place by appending to result['nodes'] and result['edges']. """ try: import tree_sitter_python as tspython from tree_sitter import Language, Parser language = Language(tspython.language()) parser = Parser(language) source = path.read_bytes() tree = parser.parse(source) root = tree.root_node except Exception: return stem = _file_stem(path) str_path = str(path) nodes = result["nodes"] edges = result["edges"] seen_ids = {n["id"] for n in nodes} file_nid = _make_id(str(path)) def _get_docstring(body_node) -> tuple[str, int] | None: if not body_node: return None for child in body_node.children: if child.type == "expression_statement": for sub in child.children: if sub.type in ("string", "concatenated_string"): text = source[sub.start_byte:sub.end_byte].decode("utf-8", errors="replace") text = text.strip("\"'").strip('"""').strip("'''").strip() if len(text) > 20: return text, child.start_point[0] + 1 break return None def _add_rationale(text: str, line: int, parent_nid: str) -> None: label = text[:80].replace("\r\n", " ").replace("\r", " ").replace("\n", " ").strip() rid = _make_id(stem, "rationale", str(line)) if rid not in seen_ids: seen_ids.add(rid) nodes.append({ "id": rid, "label": label, "file_type": "rationale", "source_file": str_path, "source_location": f"L{line}", }) edges.append({ "source": rid, "target": parent_nid, "relation": "rationale_for", "confidence": "EXTRACTED", "source_file": str_path, "source_location": f"L{line}", "weight": 1.0, }) # Module-level docstring — skip for auto-generated files (Alembic, Django # migrations, protobuf stubs, etc.) whose module docstrings are revision # annotations, not architectural rationale. if not _is_autogenerated_python(source): ds = _get_docstring(root) if ds: _add_rationale(ds[0], ds[1], file_nid) # Class and function docstrings def walk_docstrings(node, parent_nid: str) -> None: t = node.type if t == "class_definition": name_node = node.child_by_field_name("name") body = node.child_by_field_name("body") if name_node and body: class_name = source[name_node.start_byte:name_node.end_byte].decode("utf-8", errors="replace") nid = _make_id(stem, class_name) ds = _get_docstring(body) if ds: _add_rationale(ds[0], ds[1], nid) for child in body.children: walk_docstrings(child, nid) return if t == "function_definition": name_node = node.child_by_field_name("name") body = node.child_by_field_name("body") if name_node and body: func_name = source[name_node.start_byte:name_node.end_byte].decode("utf-8", errors="replace") nid = _make_id(parent_nid, func_name) if parent_nid != file_nid else _make_id(stem, func_name) ds = _get_docstring(body) if ds: _add_rationale(ds[0], ds[1], nid) return for child in node.children: walk_docstrings(child, parent_nid) walk_docstrings(root, file_nid) # Rationale comments (# NOTE:, # IMPORTANT:, etc.) source_text = source.decode("utf-8", errors="replace") for lineno, line_text in enumerate(source_text.splitlines(), start=1): stripped = line_text.strip() if any(stripped.startswith(p) for p in _RATIONALE_PREFIXES): _add_rationale(stripped, lineno, file_nid) # ── Public API ──────────────────────────────────────────────────────────────── def extract_python(path: Path) -> dict: """Extract classes, functions, and imports from a .py file via tree-sitter AST.""" result = _extract_generic(path, _PYTHON_CONFIG) if "error" not in result: _extract_python_rationale(path, result) return result def extract_js(path: Path) -> dict: """Extract classes, functions, arrow functions, and imports from a .js/.ts/.tsx/.mts/.cts file.""" if path.suffix == ".tsx": config = _TSX_CONFIG elif path.suffix in (".ts", ".mts", ".cts"): config = _TS_CONFIG else: config = _JS_CONFIG result = _extract_generic(path, config) if "error" not in result: _extract_js_rationale(path, result) return result # ── JS/TS rationale + doc-reference extraction ──────────────────────────────── # # Parity with _extract_python_rationale: Python files get rationale nodes from # docstrings and `# NOTE:`-style comments, but JS/TS comments were discarded # entirely. That silently drops two high-value signals in mixed corpora: # 1. rationale comments (`// NOTE:`, `// WHY:`, ...) — same as Python; # 2. architecture-decision references (`ADR-0011`, `RFC 793`) that teams # conventionally cite in file/function headers. These are the natural # join points between code and design docs in the same graph — without # them, code<->ADR edges never form even when the code cites the ADR. _JS_RATIONALE_PREFIXES = ( "// NOTE:", "// IMPORTANT:", "// HACK:", "// WHY:", "// RATIONALE:", "// TODO:", "// FIXME:", "* NOTE:", "* IMPORTANT:", "* HACK:", "* WHY:", "* RATIONALE:", "* TODO:", "* FIXME:", ) # Doc-reference tokens worth first-classing as graph nodes. Deliberately # conservative: ADR-NNNN (Architecture Decision Records, any zero padding) # and RFC NNNN / RFC-NNNN. _JS_DOC_REF_RE = re.compile(r"\b(ADR[- ]?\d{1,5}|RFC[- ]?\d{1,5})\b", re.IGNORECASE) # Only look for doc references inside comments, not string literals or code. _JS_COMMENT_LINE_RE = re.compile(r"^\s*(//|/\*|\*)") def _extract_js_rationale(path: Path, result: dict) -> None: """Post-pass: extract rationale comments and doc references from JS/TS source. Mutates result in-place by appending to result['nodes'] and result['edges']. """ try: source_text = path.read_text(encoding="utf-8", errors="replace") except Exception: return stem = _file_stem(path) str_path = str(path) nodes = result["nodes"] edges = result["edges"] seen_ids = {n["id"] for n in nodes} file_nid = _make_id(str(path)) seen_doc_refs: set[str] = set() def _add_rationale(text: str, line: int) -> None: label = text[:80].replace("\r\n", " ").replace("\r", " ").replace("\n", " ").strip() rid = _make_id(stem, "rationale", str(line)) if rid not in seen_ids: seen_ids.add(rid) nodes.append({ "id": rid, "label": label, "file_type": "rationale", "source_file": str_path, "source_location": f"L{line}", }) edges.append({ "source": rid, "target": file_nid, "relation": "rationale_for", "confidence": "EXTRACTED", "source_file": str_path, "source_location": f"L{line}", "weight": 1.0, }) def _add_doc_ref(token: str, line: int) -> None: # Normalize "adr 11" / "ADR-0011" spellings to a canonical "ADR-0011" # style label so references to the same document collapse to one node. kind, num = re.match(r"([A-Za-z]+)[- ]?(\d+)", token).groups() kind = kind.upper() label = f"{kind}-{num.zfill(4)}" if kind == "ADR" else f"{kind}-{num}" if label in seen_doc_refs: return seen_doc_refs.add(label) rid = _make_id("docref", label) if rid not in seen_ids: seen_ids.add(rid) nodes.append({ "id": rid, "label": label, "file_type": "doc_ref", "source_file": str_path, "source_location": f"L{line}", }) edges.append({ "source": file_nid, "target": rid, "relation": "cites", "confidence": "EXTRACTED", "source_file": str_path, "source_location": f"L{line}", "weight": 1.0, }) for lineno, line_text in enumerate(source_text.splitlines(), start=1): stripped = line_text.strip() if any(stripped.startswith(p) for p in _JS_RATIONALE_PREFIXES): _add_rationale(stripped.lstrip("/* "), lineno) if _JS_COMMENT_LINE_RE.match(line_text): for m in _JS_DOC_REF_RE.finditer(stripped): _add_doc_ref(m.group(1), lineno) def extract_svelte(path: Path) -> dict: """Extract imports from .svelte files: script-block via JS AST + template regex fallback. Tree-sitter only sees the