wehub-resource-sync
177 lines
7.3 KiB
Python
177 lines
7.3 KiB
Python
"""Markdown extractor. Moved verbatim from graphify/extract.py."""
|
|
from __future__ import annotations
|
|
|
|
import re
|
|
import os
|
|
|
|
from pathlib import Path
|
|
from graphify.extractors.base import _file_stem, _make_id
|
|
|
|
|
|
_MD_INLINE_LINK_RE = re.compile(r'(?<!\!)\[[^\]]*\]\(\s*<?([^)\s>]+)>?(?:\s+[^)]*)?\)')
|
|
|
|
_MD_REF_DEF_RE = re.compile(r'^\s{0,3}\[[^\]]+\]:\s*<?([^\s>]+)>?')
|
|
|
|
_MD_WIKILINK_RE = re.compile(r'(?<!\!)\[\[([^\]|#]+)(?:[#|][^\]]*)?\]\]')
|
|
|
|
_MD_LINKABLE_EXTS = {".md", ".mdx", ".qmd", ".markdown", ".rst", ".txt"}
|
|
|
|
def _resolve_markdown_link(raw: str, source_dir: Path) -> "Path | None":
|
|
"""Resolve a markdown link target to the absolute path of a sibling document.
|
|
|
|
Returns the resolved (normalized, not necessarily existing) path when the
|
|
target is a *local* relative/absolute file-path link to a document, or None
|
|
when it should be skipped: external URLs (http/https/mailto/protocol-
|
|
relative/data), pure in-page anchors (``#section``), and links to non-doc
|
|
file types (code/assets are handled by their own extractors).
|
|
|
|
The anchor fragment (``#section``) and query (``?x=1``) are stripped before
|
|
resolution so ``./repo.md#setup`` resolves to the same node as ``./repo.md``.
|
|
Extension-less targets (typical of wikilinks) are treated as sibling ``.md``.
|
|
"""
|
|
target = raw.strip()
|
|
if not target:
|
|
return None
|
|
# Drop anchor / query so #section links still resolve to the target doc.
|
|
target = target.split("#", 1)[0].split("?", 1)[0].strip()
|
|
if not target:
|
|
return None
|
|
low = target.lower()
|
|
if "://" in target or low.startswith(("mailto:", "tel:", "//", "data:")):
|
|
return None
|
|
suffix = Path(target).suffix.lower()
|
|
if suffix == "":
|
|
target = target + ".md"
|
|
suffix = ".md"
|
|
if suffix not in _MD_LINKABLE_EXTS:
|
|
return None
|
|
candidate = Path(target)
|
|
if not candidate.is_absolute():
|
|
candidate = source_dir / candidate
|
|
return Path(os.path.normpath(str(candidate)))
|
|
|
|
def extract_markdown(path: Path) -> dict:
|
|
"""Extract structural nodes and edges from a Markdown file.
|
|
|
|
Produces nodes for:
|
|
- The file itself
|
|
- Each heading (# / ## / ### etc.)
|
|
|
|
Produces edges for:
|
|
- file --contains--> heading
|
|
- parent heading --contains--> child heading (nesting by level)
|
|
- heading --references--> other node (when backtick `Name` matches a known pattern)
|
|
- file --references--> linked document, for inline ``[text](./other.md)``,
|
|
reference-style ``[label]: ./other.md`` and ``[[wikilink]]`` links, so a
|
|
hub doc (``index.md`` / ``table-of-contents.md``) becomes a real hub node
|
|
instead of an under-connected orphan (#1376). The target node ID is built
|
|
from the resolved target path with the same recipe as the target file's
|
|
own node, so the edge merges into that node (no ghost node). External
|
|
URLs, in-page anchors, images and non-document targets are skipped.
|
|
|
|
Fenced code blocks (``` ... ```) are skipped during parsing so their
|
|
contents don't get treated as headings, but no node is emitted for
|
|
them — they were always orphans (only a single contains edge to the
|
|
parent doc) and inflated the disconnected-component count (#1077).
|
|
|
|
No tree-sitter dependency — pure line-by-line parsing.
|
|
"""
|
|
try:
|
|
source = path.read_text(encoding="utf-8", errors="replace")
|
|
except Exception as e:
|
|
return {"nodes": [], "edges": [], "error": str(e)}
|
|
|
|
stem = _file_stem(path)
|
|
str_path = str(path)
|
|
nodes: list[dict] = []
|
|
edges: list[dict] = []
|
|
seen_ids: set[str] = set()
|
|
|
|
def add_node(nid: str, label: str, line: int, file_type: str = "document") -> None:
|
|
if nid not in seen_ids:
|
|
seen_ids.add(nid)
|
|
nodes.append({"id": nid, "label": label, "file_type": file_type,
|
|
"source_file": str_path, "source_location": f"L{line}"})
|
|
|
|
def add_edge(src: str, tgt: str, relation: str, line: int,
|
|
confidence: str = "EXTRACTED", weight: float = 1.0) -> None:
|
|
edges.append({"source": src, "target": tgt, "relation": relation,
|
|
"confidence": confidence, "source_file": str_path,
|
|
"source_location": f"L{line}", "weight": weight})
|
|
|
|
file_nid = _make_id(str(path))
|
|
add_node(file_nid, path.name, 1)
|
|
|
|
source_dir = path.parent
|
|
# Dedup link edges by resolved target node so a hub doc that links to the
|
|
# same sibling many times yields one edge, not N (keeps weights meaningful).
|
|
linked_targets: set[str] = set()
|
|
|
|
def add_link(raw: str, line: int) -> None:
|
|
resolved = _resolve_markdown_link(raw, source_dir)
|
|
if resolved is None:
|
|
return
|
|
# Build the target ID with the SAME recipe as the target file's own
|
|
# node (_make_id(str(path)) at extract time, canonicalized to
|
|
# _file_node_id(rel) by the extract() post-pass). Using the absolute
|
|
# resolved path means both endpoints get remapped identically, so the
|
|
# edge merges into the existing doc node instead of spawning a ghost.
|
|
tgt_nid = _make_id(str(resolved))
|
|
if tgt_nid == file_nid or tgt_nid in linked_targets:
|
|
return
|
|
linked_targets.add(tgt_nid)
|
|
add_edge(file_nid, tgt_nid, "references", line)
|
|
|
|
# Track heading stack for nesting: [(level, nid), ...]
|
|
heading_stack: list[tuple[int, str]] = []
|
|
in_code_block = False
|
|
|
|
lines = source.splitlines()
|
|
for line_num_0, line_text in enumerate(lines):
|
|
line_num = line_num_0 + 1
|
|
|
|
# Skip over fenced code blocks so their contents are not parsed as
|
|
# headings, but do not emit nodes/edges for them (#1077).
|
|
stripped = line_text.strip()
|
|
if stripped.startswith("```"):
|
|
in_code_block = not in_code_block
|
|
continue
|
|
|
|
if in_code_block:
|
|
continue
|
|
|
|
# Markdown links -> document references (#1376). Scanned on every
|
|
# non-fenced line (including heading lines, which the heading branch
|
|
# below `continue`s past) so links anywhere in the doc are captured.
|
|
for m in _MD_INLINE_LINK_RE.finditer(line_text):
|
|
add_link(m.group(1), line_num)
|
|
for m in _MD_WIKILINK_RE.finditer(line_text):
|
|
add_link(m.group(1), line_num)
|
|
ref_def = _MD_REF_DEF_RE.match(line_text)
|
|
if ref_def:
|
|
add_link(ref_def.group(1), line_num)
|
|
|
|
# Detect headings: # Heading, ## Heading, etc.
|
|
heading_match = re.match(r'^(#{1,6})\s+(.+)', line_text)
|
|
if heading_match:
|
|
level = len(heading_match.group(1))
|
|
title = heading_match.group(2).strip()
|
|
h_nid = _make_id(stem, title)
|
|
# Avoid duplicate heading IDs by appending line number
|
|
if h_nid in seen_ids:
|
|
h_nid = _make_id(stem, title, str(line_num))
|
|
add_node(h_nid, title, line_num)
|
|
|
|
# Pop headings at same or deeper level
|
|
while heading_stack and heading_stack[-1][0] >= level:
|
|
heading_stack.pop()
|
|
|
|
# Connect to parent heading or file
|
|
parent = heading_stack[-1][1] if heading_stack else file_nid
|
|
add_edge(parent, h_nid, "contains", line_num)
|
|
|
|
heading_stack.append((level, h_nid))
|
|
continue
|
|
|
|
return {"nodes": nodes, "edges": edges, "input_tokens": 0, "output_tokens": 0}
|