"""Shared HTML-table parsing helpers for parser engines.
Pure functions that recover structural facts from an HTML ```` string
without a heavy dependency: row/column counts (colspan-aware), the verbatim
```` substring, table-payload detection, and ``/`` wrapper
stripping. Originally private to the mineru IR builder; lifted into a leaf
module so the native markdown IR builder can reuse the exact same logic
(merged-cell semantics must survive identically across engines).
Leaf module with no parser-layer imports — safe for any engine to import.
"""
from __future__ import annotations
import re
from dataclasses import dataclass
from html.parser import HTMLParser
from lightrag.utils import logger
@dataclass
class HTMLTableInfo:
num_rows: int = 0
num_cols: int = 0
class _HTMLTableInfoParser(HTMLParser):
"""Count ```` rows and their (colspan-aware) column widths.
Used only to recover ``num_rows`` / ``num_cols`` when the engine did not
supply them; the ```` header itself is preserved verbatim by
:func:`extract_thead_html`, not reconstructed here.
"""
def __init__(self) -> None:
super().__init__(convert_charrefs=True)
# ``col_count`` (sum of colspans) for each completed top-level ````.
self.row_col_counts: list[int] = []
self._tr_depth = 0
self._cell_depth = 0
self._row_col_count = 0
self._cell_colspan = 1
def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None:
tag = tag.lower()
if tag == "tr":
if self._tr_depth == 0:
self._row_col_count = 0
self._tr_depth += 1
return
if tag in {"td", "th"} and self._tr_depth > 0:
if self._cell_depth == 0:
self._cell_colspan = _cell_span(attrs, "colspan")
self._cell_depth += 1
def handle_endtag(self, tag: str) -> None:
tag = tag.lower()
if tag in {"td", "th"} and self._cell_depth > 0:
self._cell_depth -= 1
if self._cell_depth == 0:
self._row_col_count += self._cell_colspan
self._cell_colspan = 1
return
if tag == "tr" and self._tr_depth > 0:
self._tr_depth -= 1
# ``col_count > 0`` ⇔ the row held at least one cell (colspan ≥ 1),
# so an empty ``
|
`` is skipped exactly as before.
if self._tr_depth == 0 and self._row_col_count > 0:
self.row_col_counts.append(self._row_col_count)
self._row_col_count = 0
def _cell_span(attrs: list[tuple[str, str | None]], name: str) -> int:
"""Read a ``colspan``/``rowspan`` attribute as an int ``>= 1`` (default 1)."""
for key, value in attrs:
if key.lower() != name:
continue
try:
return max(int(value or "1"), 1)
except ValueError:
return 1
return 1
def extract_html_table_info(html: str) -> HTMLTableInfo:
parser = _HTMLTableInfoParser()
try:
parser.feed(html or "")
parser.close()
except Exception as exc: # pragma: no cover - HTMLParser is forgiving.
logger.debug("[html_table] failed to parse table HTML: %s", exc)
return HTMLTableInfo()
return HTMLTableInfo(
num_rows=len(parser.row_col_counts),
num_cols=max(parser.row_col_counts, default=0),
)
def extract_thead_html(html: str) -> str | None:
"""Return the first top-level ``…`` substring verbatim.
The raw markup is kept so merged-cell semantics (``rowspan`` / ``colspan``)
survive into ``tables.json`` and, later, into every repeated header chunk
of a split table. Returns ``None`` when the table has no ```` or the
```` carries no visible text (a blank spacer row, which would
otherwise emit empty ```` headers).
"""
stripped = (html or "").strip()
lower = stripped.lower()
start = find_html_tag(lower, "thead")
if start < 0:
return None
close = lower.find(" | ", start)
if close < 0:
return None
thead = stripped[start : close + len("")]
# Blank check: drop a header whose cells hold no non-whitespace text.
if not re.sub(r"<[^>]+>", "", thead).strip():
return None
return thead
def looks_like_html_table_payload(body: str) -> bool:
lower = (body or "").lstrip().lower()
return any(
starts_with_html_tag(lower, tag)
for tag in ("table", "thead", "tbody", "tfoot", "tr", "html", "body")
)
def unwrap_html_table(payload: str) -> str:
"""Strip a ``/`` document wrapper that a table model
sometimes emits, returning the outermost ``…
`` span. Keeps
a single clean ```` so the writer does not nest tables and the
non-greedy ``TABLE_TAG_RE`` is not truncated at an inner ``
``.
Falls back to the stripped payload when no ```` element exists."""
stripped = (payload or "").strip()
lower = stripped.lower()
start = _find_table_open(lower)
if start < 0:
return stripped
close = lower.rfind("
")
if close < start:
return stripped
return stripped[start : close + len("")]
def _find_table_open(lower: str) -> int:
"""First index of a real `` int:
"""First index of a real ``= len(lower) or lower[nxt] in {" ", "\t", "\r", "\n", ">", "/"}:
return idx
idx = nxt
def starts_with_html_tag(lower: str, tag: str) -> bool:
prefix = f"<{tag}"
if not lower.startswith(prefix):
return False
if len(lower) == len(prefix):
return True
return lower[len(prefix)] in {" ", "\t", "\r", "\n", ">", "/"}
def html_table_inner_body(html: str) -> str:
stripped = (html or "").strip()
lower = stripped.lower()
if not starts_with_html_tag(lower, "table"):
return stripped
open_end = _open_tag_end(stripped)
close_start = lower.rfind("")
if open_end < 0 or close_start <= open_end:
return stripped
return stripped[open_end + 1 : close_start].strip()
def _open_tag_end(html: str) -> int:
"""Index of the ``>`` closing the leading tag, skipping quoted attribute
values so a ``>`` inside an attribute (e.g. ````) does
not terminate the tag early. Returns -1 when no closing ``>`` is found."""
quote: str | None = None
for idx, ch in enumerate(html):
if quote is not None:
if ch == quote:
quote = None
elif ch in {'"', "'"}:
quote = ch
elif ch == ">":
return idx
return -1
__all__ = [
"HTMLTableInfo",
"extract_html_table_info",
"extract_thead_html",
"looks_like_html_table_payload",
"unwrap_html_table",
"find_html_tag",
"starts_with_html_tag",
"html_table_inner_body",
]