wehub-resource-sync
1104 lines
40 KiB
Python
1104 lines
40 KiB
Python
#!/usr/bin/env python3
|
|
"""Convert pdoc3 HTML API reference docs to markdown.
|
|
|
|
This script converts the already-built pdoc3 HTML docs (from CloudFront/S3) into
|
|
clean markdown files suitable for consumption by LLMs via the MCP server.
|
|
|
|
The generated files match the ``python-md/`` convention expected by
|
|
``python-reference.ts`` in the docs MCP server.
|
|
|
|
Path mapping (canonical, no ambiguity):
|
|
livekit/index.html -> livekit.md
|
|
livekit/agents/index.html -> livekit/agents.md
|
|
livekit/agents/vad.html -> livekit/agents/vad.md
|
|
|
|
Usage:
|
|
uv run python .github/convert_html_docs.py INPUT_DIR OUTPUT_DIR [options]
|
|
--validate-only Only validate existing markdown against HTML
|
|
--no-validate Skip validation after conversion
|
|
--verbose / -v Show per-file validation details
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import html
|
|
import os
|
|
import re
|
|
import sys
|
|
from dataclasses import dataclass, field
|
|
from pathlib import Path
|
|
|
|
from bs4 import BeautifulSoup, Tag
|
|
from markdownify import MarkdownConverter
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Path mapping
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def map_output_path(html_path: Path, input_dir: Path) -> Path:
|
|
"""Map an HTML input path to its markdown output path.
|
|
|
|
Canonical rule (zero collisions across all 264 Python HTML files):
|
|
dir/index.html -> dir.md
|
|
dir/file.html -> dir/file.md
|
|
|
|
Raises ValueError for paths containing ``..`` or other unsafe components.
|
|
"""
|
|
rel = html_path.relative_to(input_dir)
|
|
|
|
# Path safety: reject traversal
|
|
for part in rel.parts:
|
|
if part in ("..", "~") or part.startswith(".."):
|
|
raise ValueError(f"Unsafe path component in {rel}: {part!r}")
|
|
|
|
parts = list(rel.parts)
|
|
|
|
# Strip leading "livekit" prefix directory if present — the HTML bucket
|
|
# has livekit/agents/index.html but we want livekit/agents.md
|
|
# (we do NOT strip "livekit" — keep it in output path)
|
|
|
|
if parts[-1] == "index.html":
|
|
# dir/index.html -> dir.md
|
|
parts = parts[:-1]
|
|
if not parts:
|
|
return Path("index.md")
|
|
return (
|
|
Path(*parts[:-1], parts[-1] + ".md")
|
|
if len(parts) == 1
|
|
else Path(*parts[:-1], parts[-1] + ".md")
|
|
)
|
|
else:
|
|
# dir/file.html -> dir/file.md
|
|
stem = parts[-1].removesuffix(".html")
|
|
parts[-1] = stem + ".md"
|
|
return Path(*parts)
|
|
|
|
|
|
def map_link(href: str) -> str:
|
|
"""Convert a relative .html link to .md for cross-references."""
|
|
if not href or href.startswith(("http://", "https://", "#", "mailto:")):
|
|
return href
|
|
# Strip fragment
|
|
base, _, fragment = href.partition("#")
|
|
if not base.endswith(".html"):
|
|
return href
|
|
# index.html -> parent .md
|
|
if base.endswith("/index.html"):
|
|
base = base[: -len("/index.html")] + ".md"
|
|
elif base == "index.html":
|
|
base = "../index.md"
|
|
else:
|
|
base = base.removesuffix(".html") + ".md"
|
|
return base + (f"#{fragment}" if fragment else "")
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# PdocMarkdownConverter — markdownify subclass for <div class="desc"> content
|
|
# ---------------------------------------------------------------------------
|
|
|
|
_ORPHAN_REF_LINK_RE = re.compile(r"\[([^\]]*)\]\[([^\]]*)\]")
|
|
|
|
|
|
class PdocMarkdownConverter(MarkdownConverter):
|
|
"""Custom markdownify converter for pdoc3 description blocks."""
|
|
|
|
def __init__(self, *, page_converter: PageConverter | None = None, **kwargs: object) -> None:
|
|
super().__init__(**kwargs)
|
|
self._page_converter = page_converter
|
|
|
|
def convert_details(self, el: Tag, text: str, parent_tags: set[str] | None = None) -> str: # type: ignore[override]
|
|
"""Skip <details class="source"> (handled separately by PageConverter)."""
|
|
if "source" in (el.get("class") or []):
|
|
return ""
|
|
return text
|
|
|
|
def convert_div(self, el: Tag, text: str, parent_tags: set[str] | None = None) -> str: # type: ignore[override]
|
|
classes = el.get("class") or []
|
|
if "admonition" in classes:
|
|
title_el = el.find(class_="admonition-title")
|
|
title = title_el.get_text(strip=True) if title_el else "Note"
|
|
# Remove the title element text from the body
|
|
body = text
|
|
if title_el:
|
|
body = body.replace(title_el.get_text(), "", 1).strip()
|
|
lines = [
|
|
f"> **{title}:** {line}" if i == 0 else f"> {line}"
|
|
for i, line in enumerate(body.split("\n"))
|
|
if line.strip()
|
|
]
|
|
return "\n".join(lines) + "\n\n"
|
|
return text
|
|
|
|
def convert_pre(self, el: Tag, text: str, parent_tags: set[str] | None = None) -> str: # type: ignore[override]
|
|
code_el = el.find("code")
|
|
if code_el and isinstance(code_el, Tag):
|
|
classes = code_el.get("class") or []
|
|
lang = ""
|
|
for cls in classes:
|
|
if isinstance(cls, str) and cls != "hljs":
|
|
lang = cls.removeprefix("language-")
|
|
break
|
|
code_text = code_el.get_text()
|
|
return f"\n```{lang}\n{code_text}\n```\n"
|
|
return f"\n```\n{text}\n```\n"
|
|
|
|
def convert_a(self, el: Tag, text: str, parent_tags: set[str] | None = None) -> str: # type: ignore[override]
|
|
href = el.get("href", "")
|
|
if isinstance(href, list):
|
|
href = href[0] if href else ""
|
|
if self._page_converter is not None:
|
|
href = self._page_converter._rewrite_link(href)
|
|
else:
|
|
href = map_link(href)
|
|
title = el.get("title", "")
|
|
if not href:
|
|
return text
|
|
if title:
|
|
return f'[{text}]({href} "{title}")'
|
|
return f"[{text}]({href})"
|
|
|
|
def convert_dl(self, el: Tag, text: str, parent_tags: set[str] | None = None) -> str: # type: ignore[override]
|
|
"""Convert definition lists (e.g. parameter docs) to markdown lists."""
|
|
lines: list[str] = []
|
|
for child in el.children:
|
|
if isinstance(child, Tag):
|
|
if child.name == "dt":
|
|
code = child.find("code")
|
|
if code:
|
|
term = code.get_text(strip=True)
|
|
lines.append(f"- **`{term}`**")
|
|
else:
|
|
term = child.get_text(strip=True)
|
|
lines.append(f"- **{term}**")
|
|
elif child.name == "dd":
|
|
# Use markdownify to preserve spaces around inline <code>
|
|
# elements and properly convert them to backticks.
|
|
inner_html = child.decode_contents()
|
|
desc = MarkdownConverter().convert(inner_html).strip()
|
|
# Collapse to single line for list-item format
|
|
desc = " ".join(desc.split())
|
|
# Escape orphan reference-link patterns ([text][ref])
|
|
# from unresolved docstring cross-references (e.g. Pydantic)
|
|
desc = _ORPHAN_REF_LINK_RE.sub(r"\\[\1\\]\\[\2\\]", desc)
|
|
if desc and lines:
|
|
lines[-1] += f": {desc}"
|
|
return "\n".join(lines) + "\n\n" if lines else text
|
|
|
|
|
|
def _desc_to_markdown(el: Tag | None, *, page_converter: PageConverter | None = None) -> str:
|
|
"""Convert a <div class="desc"> element to markdown."""
|
|
if el is None:
|
|
return ""
|
|
text = PdocMarkdownConverter(
|
|
page_converter=page_converter,
|
|
heading_style="ATX",
|
|
bullets="-",
|
|
strip=["img"],
|
|
).convert(str(el))
|
|
return text.strip()
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Signature / source extraction helpers
|
|
# ---------------------------------------------------------------------------
|
|
|
|
_NBSP_RE = re.compile(r"\u00a0|\u2011|\u2010|\u2012|\u2013")
|
|
|
|
|
|
def _normalize_sig(text: str) -> str:
|
|
"""Normalize a signature string: unescape HTML, fix non-breaking hyphens."""
|
|
text = html.unescape(text)
|
|
text = _NBSP_RE.sub(lambda m: "-" if m.group() in "\u2011\u2010\u2012\u2013" else " ", text)
|
|
text = text.replace("\u2002", " ").replace("\u2003", " ") # en/em space
|
|
text = re.sub(r"\s+", " ", text).strip()
|
|
return text
|
|
|
|
|
|
def _extract_signature(dt: Tag) -> str:
|
|
"""Extract a function/class signature from a <dt> element.
|
|
|
|
Handles <br> tags (multi-line signatures) and non-breaking hyphens.
|
|
"""
|
|
code = dt.find("code", class_="name")
|
|
if not code:
|
|
return ""
|
|
# Replace <br> with newline before extracting text
|
|
for br in code.find_all("br"):
|
|
br.replace_with("\n")
|
|
raw = code.get_text()
|
|
return _normalize_sig(raw)
|
|
|
|
|
|
def _extract_source(dd: Tag) -> str | None:
|
|
"""Extract source code from <details class="source"> inside a <dd>."""
|
|
details = dd.find("details", class_="source")
|
|
if not details or not isinstance(details, Tag):
|
|
return None
|
|
pre = details.find("pre")
|
|
if not pre or not isinstance(pre, Tag):
|
|
return None
|
|
code_el = pre.find("code")
|
|
if not code_el:
|
|
return None
|
|
raw = code_el.get_text() if isinstance(code_el, Tag) else str(code_el)
|
|
return html.unescape(raw).strip() or None
|
|
|
|
|
|
def _extract_ident(dt: Tag) -> str:
|
|
"""Extract the short identifier name from a <dt> element."""
|
|
span = dt.find("span", class_="ident")
|
|
if span:
|
|
return span.get_text(strip=True)
|
|
return ""
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# PageConverter — structured DOM walker
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
class PageConverter:
|
|
"""Convert a single pdoc3 HTML page to markdown."""
|
|
|
|
def __init__(
|
|
self,
|
|
html_content: str,
|
|
*,
|
|
html_rel_path: Path | None = None,
|
|
input_dir: Path | None = None,
|
|
) -> None:
|
|
self.soup = BeautifulSoup(html_content, "html.parser")
|
|
self.lines: list[str] = []
|
|
# Context for link rewriting — when set, links are resolved relative to
|
|
# the source HTML file's location and re-computed relative to the output
|
|
# markdown file's location.
|
|
self._html_rel_path = html_rel_path
|
|
self._input_dir = input_dir
|
|
|
|
def _rewrite_link(self, href: str) -> str:
|
|
"""Rewrite an HTML href to a correct relative .md path.
|
|
|
|
When source context is available, resolves the link against the source
|
|
HTML file's directory, maps through ``map_output_path``, and computes
|
|
the correct relative path from the output markdown file.
|
|
|
|
Falls back to the context-free ``map_link`` when source context is
|
|
unavailable.
|
|
"""
|
|
if not href or href.startswith(("http://", "https://", "#", "mailto:")):
|
|
return href
|
|
base, _, fragment = href.partition("#")
|
|
if not base.endswith(".html"):
|
|
return href
|
|
|
|
if self._html_rel_path is None or self._input_dir is None:
|
|
return map_link(href)
|
|
|
|
# Resolve the target HTML path relative to the source HTML's directory
|
|
source_html_dir = self._html_rel_path.parent
|
|
target_html_abs = (self._input_dir / source_html_dir / base).resolve()
|
|
# Make it relative to input_dir again
|
|
try:
|
|
target_html_rel = target_html_abs.relative_to(self._input_dir.resolve())
|
|
except ValueError:
|
|
return map_link(href)
|
|
|
|
# Map both source and target through map_output_path
|
|
try:
|
|
source_md = map_output_path(self._input_dir / self._html_rel_path, self._input_dir)
|
|
target_md = map_output_path(self._input_dir / target_html_rel, self._input_dir)
|
|
except ValueError:
|
|
return map_link(href)
|
|
|
|
# Compute the relative path from source .md's directory to target .md
|
|
rel = os.path.relpath(str(target_md), str(source_md.parent))
|
|
result = rel
|
|
if fragment:
|
|
result += f"#{fragment}"
|
|
return result
|
|
|
|
def convert(self) -> str:
|
|
article = self.soup.find("article", id="content")
|
|
if not article or not isinstance(article, Tag):
|
|
return ""
|
|
|
|
self._convert_title(article)
|
|
self._convert_intro(article)
|
|
self._convert_submodules(article)
|
|
self._convert_variables(article)
|
|
self._convert_functions(article)
|
|
self._convert_classes(article)
|
|
|
|
return "\n".join(self.lines).rstrip() + "\n"
|
|
|
|
# --- Title ---
|
|
|
|
def _convert_title(self, article: Tag) -> None:
|
|
h1 = article.find("h1", class_="title")
|
|
if not h1 or not isinstance(h1, Tag):
|
|
return
|
|
code = h1.find("code")
|
|
if code:
|
|
module_name = code.get_text(strip=True)
|
|
else:
|
|
module_name = h1.get_text(strip=True)
|
|
# Determine type prefix (Module, Package, Namespace)
|
|
full_text = h1.get_text(strip=True)
|
|
prefix = "Module"
|
|
for p in ("Namespace", "Package", "Module"):
|
|
if full_text.startswith(p):
|
|
prefix = p
|
|
break
|
|
self.lines.append(f"# {prefix} `{module_name}`")
|
|
self.lines.append("")
|
|
|
|
# --- Section intro ---
|
|
|
|
def _convert_intro(self, article: Tag) -> None:
|
|
intro = article.find("section", id="section-intro")
|
|
if not intro or not isinstance(intro, Tag):
|
|
return
|
|
# Convert the intro section content (skip the section tag itself)
|
|
text = _desc_to_markdown(intro, page_converter=self)
|
|
if text:
|
|
self.lines.append(text)
|
|
self.lines.append("")
|
|
|
|
# --- Sub-modules ---
|
|
|
|
def _convert_submodules(self, article: Tag) -> None:
|
|
h2 = article.find("h2", id="header-submodules")
|
|
if not h2 or not isinstance(h2, Tag):
|
|
return
|
|
self.lines.append("## Sub-modules")
|
|
self.lines.append("")
|
|
|
|
# The <dl> follows the <h2> in the next <section> or same section
|
|
section = h2.find_parent("section")
|
|
if not section:
|
|
return
|
|
dl = section.find("dl")
|
|
if not dl or not isinstance(dl, Tag):
|
|
return
|
|
|
|
for dt in dl.find_all("dt", recursive=False):
|
|
a = dt.find("a")
|
|
if not a:
|
|
continue
|
|
name = a.get_text(strip=True)
|
|
href = self._rewrite_link(a.get("href", ""))
|
|
dd = dt.find_next_sibling("dd")
|
|
desc_el = dd.find("div", class_="desc") if dd else None
|
|
desc_text = ""
|
|
if desc_el and isinstance(desc_el, Tag):
|
|
desc_text = " ".join(desc_el.get_text().strip().split())
|
|
|
|
if desc_text:
|
|
self.lines.append(f"- [`{name}`]({href}) - {desc_text}")
|
|
else:
|
|
self.lines.append(f"- [`{name}`]({href})")
|
|
|
|
self.lines.append("")
|
|
|
|
# --- Variables ---
|
|
|
|
def _convert_variables(self, article: Tag) -> None:
|
|
h2 = article.find("h2", id="header-variables")
|
|
if not h2 or not isinstance(h2, Tag):
|
|
return
|
|
self.lines.append("## Global variables")
|
|
self.lines.append("")
|
|
|
|
section = h2.find_parent("section")
|
|
if not section:
|
|
return
|
|
dl = section.find("dl")
|
|
if not dl or not isinstance(dl, Tag):
|
|
return
|
|
|
|
for dt in dl.find_all("dt", recursive=False):
|
|
self._convert_variable_or_property(dt, heading_level=3)
|
|
|
|
# --- Functions ---
|
|
|
|
def _convert_functions(self, article: Tag) -> None:
|
|
h2 = article.find("h2", id="header-functions")
|
|
if not h2 or not isinstance(h2, Tag):
|
|
return
|
|
self.lines.append("## Functions")
|
|
self.lines.append("")
|
|
|
|
section = h2.find_parent("section")
|
|
if not section:
|
|
return
|
|
dl = section.find("dl")
|
|
if not dl or not isinstance(dl, Tag):
|
|
return
|
|
|
|
for dt in dl.find_all("dt", recursive=False):
|
|
self._convert_function(dt, heading_level=3)
|
|
|
|
def _convert_function(self, dt: Tag, heading_level: int) -> None:
|
|
name = _extract_ident(dt)
|
|
if not name:
|
|
return
|
|
sig = _extract_signature(dt)
|
|
dd = dt.find_next_sibling("dd")
|
|
full_id = dt.get("id", "")
|
|
|
|
self.lines.append(f"{'#' * heading_level} `{name}`")
|
|
self.lines.append("")
|
|
if full_id:
|
|
self.lines.append(f"Full name: `{full_id}`")
|
|
self.lines.append("")
|
|
if sig:
|
|
self.lines.append(f"```python\n{sig}\n```")
|
|
self.lines.append("")
|
|
|
|
if dd and isinstance(dd, Tag):
|
|
# Description
|
|
desc_el = dd.find("div", class_="desc", recursive=False)
|
|
if desc_el and isinstance(desc_el, Tag):
|
|
desc_text = _desc_to_markdown(desc_el, page_converter=self)
|
|
if desc_text:
|
|
self.lines.append(desc_text)
|
|
self.lines.append("")
|
|
|
|
# Source
|
|
source = _extract_source(dd)
|
|
if source:
|
|
self.lines.append("<details><summary>Source</summary>")
|
|
self.lines.append("")
|
|
self.lines.append(f"```python\n{source}\n```")
|
|
self.lines.append("")
|
|
self.lines.append("</details>")
|
|
self.lines.append("")
|
|
|
|
# --- Classes ---
|
|
|
|
def _convert_classes(self, article: Tag) -> None:
|
|
h2 = article.find("h2", id="header-classes")
|
|
if not h2 or not isinstance(h2, Tag):
|
|
return
|
|
self.lines.append("## Classes")
|
|
self.lines.append("")
|
|
|
|
section = h2.find_parent("section")
|
|
if not section:
|
|
return
|
|
dl = section.find("dl")
|
|
if not dl or not isinstance(dl, Tag):
|
|
return
|
|
|
|
for dt in dl.find_all("dt", recursive=False):
|
|
self._convert_class(dt)
|
|
|
|
def _convert_class(self, dt: Tag) -> None:
|
|
name = _extract_ident(dt)
|
|
if not name:
|
|
return
|
|
sig = _extract_signature(dt)
|
|
dd = dt.find_next_sibling("dd")
|
|
full_id = dt.get("id", "")
|
|
|
|
self.lines.append(f"### `{name}`")
|
|
self.lines.append("")
|
|
if full_id:
|
|
self.lines.append(f"Full name: `{full_id}`")
|
|
self.lines.append("")
|
|
if sig:
|
|
self.lines.append(f"```python\n{sig}\n```")
|
|
self.lines.append("")
|
|
|
|
if not dd or not isinstance(dd, Tag):
|
|
return
|
|
|
|
# Class description
|
|
desc_el = dd.find("div", class_="desc", recursive=False)
|
|
if desc_el and isinstance(desc_el, Tag):
|
|
desc_text = _desc_to_markdown(desc_el, page_converter=self)
|
|
if desc_text:
|
|
self.lines.append(desc_text)
|
|
self.lines.append("")
|
|
|
|
# Source
|
|
source = _extract_source(dd)
|
|
if source:
|
|
self.lines.append("<details><summary>Source</summary>")
|
|
self.lines.append("")
|
|
self.lines.append(f"```python\n{source}\n```")
|
|
self.lines.append("")
|
|
self.lines.append("</details>")
|
|
self.lines.append("")
|
|
|
|
# Walk h3 subsections within this <dd>
|
|
for h3 in dd.find_all("h3", recursive=False):
|
|
section_title = h3.get_text(strip=True)
|
|
self._convert_class_section(dd, h3, section_title)
|
|
|
|
def _convert_class_section(self, dd: Tag, h3: Tag, title: str) -> None:
|
|
title_lower = title.lower()
|
|
|
|
if title_lower in ("ancestors", "subclasses"):
|
|
self.lines.append(f"#### {title}")
|
|
self.lines.append("")
|
|
ul = h3.find_next_sibling("ul")
|
|
if ul and isinstance(ul, Tag):
|
|
for li in ul.find_all("li"):
|
|
text = li.get_text(strip=True)
|
|
self.lines.append(f"- {text}")
|
|
self.lines.append("")
|
|
|
|
elif title_lower in (
|
|
"class variables",
|
|
"instance variables",
|
|
"static methods",
|
|
):
|
|
self.lines.append(f"#### {title}")
|
|
self.lines.append("")
|
|
dl = h3.find_next_sibling("dl")
|
|
if dl and isinstance(dl, Tag):
|
|
for sub_dt in dl.find_all("dt", recursive=False):
|
|
self._convert_variable_or_property(sub_dt, heading_level=5)
|
|
|
|
elif title_lower == "methods":
|
|
self.lines.append("#### Methods")
|
|
self.lines.append("")
|
|
dl = h3.find_next_sibling("dl")
|
|
if dl and isinstance(dl, Tag):
|
|
for sub_dt in dl.find_all("dt", recursive=False):
|
|
self._convert_function(sub_dt, heading_level=5)
|
|
|
|
elif title_lower == "inherited members":
|
|
self.lines.append("#### Inherited members")
|
|
self.lines.append("")
|
|
ul = h3.find_next_sibling("ul")
|
|
if ul and isinstance(ul, Tag):
|
|
for li in ul.find_all("li", recursive=False):
|
|
# Each top-level li has a parent class link + nested ul of members
|
|
parent_code = li.find("code", recursive=False)
|
|
if not parent_code:
|
|
parent_code = li.find("b")
|
|
parent_link = parent_code.find("a") if parent_code else None
|
|
parent_name = (
|
|
parent_link.get_text(strip=True)
|
|
if parent_link
|
|
else (parent_code.get_text(strip=True) if parent_code else "")
|
|
)
|
|
nested_ul = li.find("ul")
|
|
members: list[str] = []
|
|
if nested_ul and isinstance(nested_ul, Tag):
|
|
for member_li in nested_ul.find_all("li"):
|
|
members.append(member_li.get_text(strip=True))
|
|
if members:
|
|
self.lines.append(f"- **{parent_name}**: {', '.join(members)}")
|
|
else:
|
|
self.lines.append(f"- **{parent_name}**")
|
|
self.lines.append("")
|
|
|
|
else:
|
|
# Fallback: just emit the heading
|
|
self.lines.append(f"#### {title}")
|
|
self.lines.append("")
|
|
|
|
def _convert_variable_or_property(self, dt: Tag, heading_level: int) -> None:
|
|
"""Convert a class/instance variable or property <dt>/<dd> pair."""
|
|
name = _extract_ident(dt)
|
|
if not name:
|
|
return
|
|
full_id = dt.get("id", "")
|
|
# Get the full signature line (var/prop name : type)
|
|
code = dt.find("code", class_="name")
|
|
sig_text = ""
|
|
if code:
|
|
sig_text = _normalize_sig(code.get_text())
|
|
|
|
dd = dt.find_next_sibling("dd")
|
|
|
|
self.lines.append(f"{'#' * heading_level} `{name}`")
|
|
self.lines.append("")
|
|
if full_id:
|
|
self.lines.append(f"Full name: `{full_id}`")
|
|
self.lines.append("")
|
|
if sig_text:
|
|
self.lines.append(f"`{sig_text}`")
|
|
self.lines.append("")
|
|
|
|
if dd and isinstance(dd, Tag):
|
|
desc_el = dd.find("div", class_="desc", recursive=False)
|
|
if desc_el and isinstance(desc_el, Tag):
|
|
desc_text = _desc_to_markdown(desc_el, page_converter=self)
|
|
if desc_text:
|
|
self.lines.append(desc_text)
|
|
self.lines.append("")
|
|
|
|
source = _extract_source(dd)
|
|
if source:
|
|
self.lines.append("<details><summary>Source</summary>")
|
|
self.lines.append("")
|
|
self.lines.append(f"```python\n{source}\n```")
|
|
self.lines.append("")
|
|
self.lines.append("</details>")
|
|
self.lines.append("")
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Validation
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
@dataclass
|
|
class ContentFingerprint:
|
|
"""Structural fingerprint extracted from HTML for validation."""
|
|
|
|
identifiers: list[str] = field(default_factory=list)
|
|
source_blocks: list[str] = field(default_factory=list)
|
|
signatures: list[str] = field(default_factory=list)
|
|
description_texts: list[str] = field(default_factory=list)
|
|
|
|
|
|
@dataclass
|
|
class ValidationResult:
|
|
"""Result of validating markdown against an HTML fingerprint."""
|
|
|
|
is_valid: bool
|
|
score: float
|
|
identifier_found: int = 0
|
|
identifier_total: int = 0
|
|
source_found: int = 0
|
|
source_total: int = 0
|
|
signature_found: int = 0
|
|
signature_total: int = 0
|
|
description_score: float = 1.0
|
|
missing_identifiers: list[str] = field(default_factory=list)
|
|
missing_sources: int = 0
|
|
missing_signatures: list[str] = field(default_factory=list)
|
|
description_warnings: list[str] = field(default_factory=list)
|
|
|
|
|
|
def extract_fingerprint(html_content: str) -> ContentFingerprint:
|
|
"""Extract a structural fingerprint from pdoc3 HTML for validation."""
|
|
soup = BeautifulSoup(html_content, "html.parser")
|
|
fp = ContentFingerprint()
|
|
|
|
# Identifiers: all dt[id] values
|
|
for dt in soup.find_all("dt", id=True):
|
|
fp.identifiers.append(dt["id"])
|
|
|
|
# Source blocks
|
|
for details in soup.find_all("details", class_="source"):
|
|
pre = details.find("pre")
|
|
if pre:
|
|
code_el = pre.find("code")
|
|
if code_el:
|
|
raw = code_el.get_text() if isinstance(code_el, Tag) else str(code_el)
|
|
fp.source_blocks.append(html.unescape(raw).strip())
|
|
|
|
# Signatures: function/class names from <dt><code class="name"><span class="ident">
|
|
for dt in soup.find_all("dt"):
|
|
span = dt.find("span", class_="ident")
|
|
if span:
|
|
fp.signatures.append(span.get_text(strip=True))
|
|
|
|
# Description texts
|
|
for desc in soup.find_all("div", class_="desc"):
|
|
text = desc.get_text(strip=True)
|
|
if text:
|
|
fp.description_texts.append(text)
|
|
|
|
return fp
|
|
|
|
|
|
def _normalize_ws(text: str) -> str:
|
|
"""Normalize whitespace for comparison."""
|
|
return re.sub(r"\s+", " ", text).strip()
|
|
|
|
|
|
def validate_markdown(fingerprint: ContentFingerprint, markdown: str) -> ValidationResult:
|
|
"""Validate that markdown contains the expected content from the HTML.
|
|
|
|
Exact checks (identifiers, sources, signatures) are gates.
|
|
Fuzzy description overlap is warning-only.
|
|
"""
|
|
result = ValidationResult(is_valid=True, score=1.0)
|
|
|
|
# 1. Identifier check — exact substring match (gates)
|
|
result.identifier_total = len(fingerprint.identifiers)
|
|
for ident in fingerprint.identifiers:
|
|
if ident in markdown:
|
|
result.identifier_found += 1
|
|
else:
|
|
result.missing_identifiers.append(ident)
|
|
|
|
# 2. Source block check — whitespace-normalized line-by-line (gates)
|
|
result.source_total = len(fingerprint.source_blocks)
|
|
for source in fingerprint.source_blocks:
|
|
norm_source_lines = [_normalize_ws(line) for line in source.split("\n") if line.strip()]
|
|
# Check that all significant lines appear in the markdown
|
|
found = True
|
|
for line in norm_source_lines[:5]: # check first 5 lines for efficiency
|
|
if len(line) > 10 and _normalize_ws(line) not in _normalize_ws(markdown):
|
|
found = False
|
|
break
|
|
if found:
|
|
result.source_found += 1
|
|
else:
|
|
result.missing_sources += 1
|
|
|
|
# 3. Signature check — function/class name must appear (gates)
|
|
result.signature_total = len(fingerprint.signatures)
|
|
for sig_name in fingerprint.signatures:
|
|
if sig_name in markdown:
|
|
result.signature_found += 1
|
|
else:
|
|
result.missing_signatures.append(sig_name)
|
|
|
|
# 4. Description check — fuzzy token overlap (warning-only, does NOT gate)
|
|
if fingerprint.description_texts:
|
|
desc_scores: list[float] = []
|
|
for desc_text in fingerprint.description_texts:
|
|
tokens = {w for w in re.findall(r"\w+", desc_text.lower()) if len(w) >= 4}
|
|
if not tokens:
|
|
desc_scores.append(1.0)
|
|
continue
|
|
found_tokens = sum(1 for t in tokens if t in markdown.lower())
|
|
overlap = found_tokens / len(tokens)
|
|
if overlap < 0.9:
|
|
result.description_warnings.append(
|
|
f"Low overlap ({overlap:.0%}): {desc_text[:80]}..."
|
|
)
|
|
desc_scores.append(overlap)
|
|
result.description_score = sum(desc_scores) / len(desc_scores)
|
|
|
|
# Compute gate result: exact checks only
|
|
id_ok = result.identifier_found == result.identifier_total
|
|
source_ok = result.source_found == result.source_total
|
|
sig_ok = result.signature_found == result.signature_total
|
|
result.is_valid = id_ok and source_ok and sig_ok
|
|
|
|
# Compute overall score (for reporting)
|
|
counts = [
|
|
(result.identifier_found, result.identifier_total),
|
|
(result.source_found, result.source_total),
|
|
(result.signature_found, result.signature_total),
|
|
]
|
|
total = sum(t for _, t in counts)
|
|
found = sum(f for f, _ in counts)
|
|
result.score = found / total if total > 0 else 1.0
|
|
|
|
return result
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Batch conversion
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def convert_file(html_path: Path) -> str:
|
|
"""Convert a single HTML file to markdown."""
|
|
html_content = html_path.read_text(encoding="utf-8")
|
|
converter = PageConverter(html_content)
|
|
return converter.convert()
|
|
|
|
|
|
def convert_directory(
|
|
input_dir: Path,
|
|
output_dir: Path,
|
|
*,
|
|
validate: bool = True,
|
|
verbose: bool = False,
|
|
) -> tuple[int, int, int]:
|
|
"""Convert all HTML files in input_dir to markdown in output_dir.
|
|
|
|
Returns (total_files, valid_files, failed_files).
|
|
"""
|
|
html_files = sorted(input_dir.rglob("*.html"))
|
|
if not html_files:
|
|
print(f"No HTML files found in {input_dir}")
|
|
return 0, 0, 0
|
|
|
|
total = 0
|
|
valid = 0
|
|
failed = 0
|
|
|
|
for html_path in html_files:
|
|
try:
|
|
out_path = map_output_path(html_path, input_dir)
|
|
except ValueError as e:
|
|
print(f" SKIP {html_path}: {e}")
|
|
continue
|
|
|
|
total += 1
|
|
full_out = output_dir / out_path
|
|
full_out.parent.mkdir(parents=True, exist_ok=True)
|
|
|
|
html_content = html_path.read_text(encoding="utf-8")
|
|
html_rel = html_path.relative_to(input_dir)
|
|
markdown = PageConverter(
|
|
html_content,
|
|
html_rel_path=html_rel,
|
|
input_dir=input_dir,
|
|
).convert()
|
|
full_out.write_text(markdown, encoding="utf-8")
|
|
|
|
if validate:
|
|
fp = extract_fingerprint(html_content)
|
|
vr = validate_markdown(fp, markdown)
|
|
if vr.is_valid:
|
|
valid += 1
|
|
if verbose:
|
|
status = "VALID"
|
|
if vr.description_warnings:
|
|
status += f" (desc warnings: {len(vr.description_warnings)})"
|
|
print(f" {status} {out_path} (score={vr.score:.2f})")
|
|
else:
|
|
failed += 1
|
|
print(f" FAIL {out_path} (score={vr.score:.2f})")
|
|
if verbose:
|
|
if vr.missing_identifiers:
|
|
print(f" missing identifiers: {vr.missing_identifiers[:5]}")
|
|
if vr.missing_sources:
|
|
print(f" missing sources: {vr.missing_sources}")
|
|
if vr.missing_signatures:
|
|
print(f" missing signatures: {vr.missing_signatures[:5]}")
|
|
else:
|
|
valid += 1
|
|
if verbose:
|
|
print(f" OK {out_path}")
|
|
|
|
return total, valid, failed
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Index generation
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def generate_index(input_dir: Path, output_dir: Path) -> None:
|
|
"""Generate an index.md listing all converted modules."""
|
|
md_files = sorted(output_dir.rglob("*.md"))
|
|
lines = [
|
|
"# LiveKit Python SDK API Reference",
|
|
"",
|
|
"Auto-generated API reference for the LiveKit Python SDK and plugins.",
|
|
"",
|
|
"## Modules",
|
|
"",
|
|
]
|
|
for md_file in md_files:
|
|
rel = md_file.relative_to(output_dir)
|
|
if str(rel) == "index.md":
|
|
continue
|
|
# Convert path back to module name for display
|
|
module_name = str(rel).removesuffix(".md").replace("/", ".")
|
|
lines.append(f"- [`{module_name}`]({rel})")
|
|
|
|
lines.append("")
|
|
index_path = output_dir / "index.md"
|
|
index_path.write_text("\n".join(lines), encoding="utf-8")
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Link integrity validation
|
|
# ---------------------------------------------------------------------------
|
|
|
|
_MD_LINK_RE = re.compile(r"\[.*?\]\(([^)]+)\)")
|
|
|
|
# Known external hrefs from third-party docstrings. Matched against the raw
|
|
# href (before resolution) stripped of any fragment. Be specific — each entry
|
|
# is a literal string, not a pattern, so we never accidentally suppress a real
|
|
# broken link.
|
|
#
|
|
# To find new entries: run with --check-links -v after adding a new dependency
|
|
# that injects cross-site links in its docstrings.
|
|
_KNOWN_EXTERNAL_HREFS: set[str] = {
|
|
# Pydantic BaseModel docstring links to docs.pydantic.dev/concepts/models
|
|
"../concepts/models.md",
|
|
}
|
|
|
|
|
|
@dataclass
|
|
class LinkCheckResult:
|
|
"""Result of validating internal links across generated markdown files."""
|
|
|
|
broken: list[tuple[Path, str]] = field(default_factory=list)
|
|
"""Links whose target file should exist but doesn't."""
|
|
|
|
external: list[tuple[Path, str]] = field(default_factory=list)
|
|
"""Links whose raw href matches a known third-party docstring cross-reference
|
|
(e.g. Pydantic's ``../concepts/models.md``). Not failures."""
|
|
|
|
|
|
def validate_links(output_dir: Path) -> LinkCheckResult:
|
|
"""Walk all .md files and check that internal .md links resolve to existing files.
|
|
|
|
Returns a ``LinkCheckResult`` with broken and external links separated.
|
|
"""
|
|
result = LinkCheckResult()
|
|
resolved_root = output_dir.resolve()
|
|
|
|
for md_file in sorted(output_dir.rglob("*.md")):
|
|
content = md_file.read_text(encoding="utf-8")
|
|
for match in _MD_LINK_RE.finditer(content):
|
|
href = match.group(1)
|
|
# Strip title from link: [text](url "title")
|
|
href = href.split('"')[0].strip()
|
|
# Skip external URLs, anchors, non-.md links
|
|
if href.startswith(("http://", "https://", "#", "mailto:")):
|
|
continue
|
|
# Split off fragment for file-existence check
|
|
base = href.split("#")[0]
|
|
if not base:
|
|
continue
|
|
if not base.endswith(".md"):
|
|
continue
|
|
|
|
# Resolve relative to the source file's directory
|
|
target = (md_file.parent / base).resolve()
|
|
rel_source = md_file.relative_to(output_dir)
|
|
|
|
# Links that escape the output tree are always broken.
|
|
try:
|
|
target.relative_to(resolved_root)
|
|
except ValueError:
|
|
result.broken.append((rel_source, href))
|
|
continue
|
|
|
|
# If the target file exists, the link is valid regardless of
|
|
# whether the href matches a known-external pattern.
|
|
if target.exists():
|
|
continue
|
|
|
|
# Target is missing. If the href matches a known third-party
|
|
# docstring cross-reference, classify as external; otherwise
|
|
# it's a broken internal link.
|
|
if base in _KNOWN_EXTERNAL_HREFS:
|
|
result.external.append((rel_source, href))
|
|
else:
|
|
result.broken.append((rel_source, href))
|
|
|
|
return result
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# CLI
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def main() -> int:
|
|
parser = argparse.ArgumentParser(
|
|
description="Convert pdoc3 HTML docs to markdown for LLM consumption."
|
|
)
|
|
parser.add_argument("input_dir", type=Path, help="Directory containing HTML docs")
|
|
parser.add_argument("output_dir", type=Path, help="Directory to write markdown files")
|
|
parser.add_argument(
|
|
"--validate-only",
|
|
action="store_true",
|
|
help="Only validate existing markdown against HTML (no conversion)",
|
|
)
|
|
parser.add_argument(
|
|
"--no-validate",
|
|
action="store_true",
|
|
help="Skip validation after conversion",
|
|
)
|
|
parser.add_argument(
|
|
"--check-links",
|
|
action="store_true",
|
|
help="Validate that all internal .md links resolve to existing files",
|
|
)
|
|
parser.add_argument(
|
|
"-v",
|
|
"--verbose",
|
|
action="store_true",
|
|
help="Show per-file validation details",
|
|
)
|
|
args = parser.parse_args()
|
|
|
|
input_dir: Path = args.input_dir
|
|
output_dir: Path = args.output_dir
|
|
|
|
if not input_dir.is_dir():
|
|
print(f"Error: {input_dir} is not a directory", file=sys.stderr)
|
|
return 1
|
|
|
|
if args.validate_only:
|
|
if not output_dir.is_dir():
|
|
print(f"Error: {output_dir} is not a directory", file=sys.stderr)
|
|
return 1
|
|
# Validate existing markdown against HTML
|
|
html_files = sorted(input_dir.rglob("*.html"))
|
|
total = 0
|
|
valid = 0
|
|
failed = 0
|
|
for html_path in html_files:
|
|
try:
|
|
out_path = map_output_path(html_path, input_dir)
|
|
except ValueError:
|
|
continue
|
|
md_path = output_dir / out_path
|
|
if not md_path.exists():
|
|
print(f" MISSING {out_path}")
|
|
failed += 1
|
|
total += 1
|
|
continue
|
|
total += 1
|
|
html_content = html_path.read_text(encoding="utf-8")
|
|
markdown = md_path.read_text(encoding="utf-8")
|
|
fp = extract_fingerprint(html_content)
|
|
vr = validate_markdown(fp, markdown)
|
|
if vr.is_valid:
|
|
valid += 1
|
|
if args.verbose:
|
|
print(f" VALID {out_path} (score={vr.score:.2f})")
|
|
else:
|
|
failed += 1
|
|
print(f" FAIL {out_path} (score={vr.score:.2f})")
|
|
if args.verbose and vr.missing_identifiers:
|
|
print(f" missing: {vr.missing_identifiers[:5]}")
|
|
print(f"\nValidation: {valid}/{total} valid, {failed} failed")
|
|
return 0 if failed == 0 else 1
|
|
|
|
# Convert
|
|
print(f"Converting {input_dir} -> {output_dir}")
|
|
output_dir.mkdir(parents=True, exist_ok=True)
|
|
|
|
total, valid, failed = convert_directory(
|
|
input_dir,
|
|
output_dir,
|
|
validate=not args.no_validate,
|
|
verbose=args.verbose,
|
|
)
|
|
|
|
# Generate index
|
|
generate_index(input_dir, output_dir)
|
|
total += 1 # count index
|
|
valid += 1
|
|
|
|
print(f"\nDone! Converted {total} files to {output_dir}/")
|
|
if not args.no_validate:
|
|
print(f"Validation: {valid}/{total} valid, {failed} failed")
|
|
pct = valid / total * 100 if total > 0 else 0
|
|
print(f"Pass rate: {pct:.1f}%")
|
|
|
|
# Link integrity check
|
|
if args.check_links:
|
|
link_result = validate_links(output_dir)
|
|
n_broken = len(link_result.broken)
|
|
n_external = len(link_result.external)
|
|
print(f"\nLinks: {n_broken} broken, {n_external} external (skipped)")
|
|
if link_result.broken:
|
|
for src, href in link_result.broken:
|
|
print(f" BROKEN {src} -> {href}")
|
|
return 1
|
|
if args.verbose and link_result.external:
|
|
for src, href in link_result.external:
|
|
print(f" external {src} -> {href}")
|
|
|
|
return 0 if failed == 0 else 1
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main())
|