chore: import upstream snapshot with attribution

lightrag/parser/docx/__init__.py

@@ -0,0 +1,14 @@
+"""LightRAG native DOCX parser package.
+
+The :mod:`parse_document` / :mod:`numbering_resolver` / :mod:`table_extractor` /
+:mod:`drawing_image_extractor` / :mod:`utils` / :mod:`omml` modules ship the
+upstream DOCX extraction logic verbatim (with imports localized for the new
+package path).
+
+The pipeline-side orchestration (extract → IR → sidecar) now lives in
+:meth:`lightrag.pipeline._PipelineMixin.parse_native` so the native and
+MinerU engines share one shape; see :mod:`lightrag.parser.docx.ir_builder`
+for the engine IR builder.
+"""
+
+__all__: list[str] = []

lightrag/parser/docx/drawing_image_extractor.py

@@ -0,0 +1,445 @@
+#!/usr/bin/env python3
+"""
+ABOUTME: Shared drawing/image extraction utilities for DOCX parsing and editing
+ABOUTME: Resolves w:drawing -> a:blip relationships, exports embedded images, builds placeholders
+"""
+
+from __future__ import annotations
+
+import posixpath
+import re
+import shutil
+import zipfile
+from dataclasses import dataclass, field
+from html import escape, unescape
+from pathlib import Path, PurePosixPath
+from typing import Dict, Optional, Tuple
+from urllib.parse import urlparse
+
+try:
+    from defusedxml import ElementTree as ET
+except ImportError:  # pragma: no cover
+    from xml.etree import ElementTree as ET
+
+
+NS = {
+    "w": "http://schemas.openxmlformats.org/wordprocessingml/2006/main",
+    "wp": "http://schemas.openxmlformats.org/drawingml/2006/wordprocessingDrawing",
+    "a": "http://schemas.openxmlformats.org/drawingml/2006/main",
+    "r": "http://schemas.openxmlformats.org/officeDocument/2006/relationships",
+    "v": "urn:schemas-microsoft-com:vml",
+}
+
+REL_NS = "http://schemas.openxmlformats.org/package/2006/relationships"
+CONTENT_TYPE_NS = "http://schemas.openxmlformats.org/package/2006/content-types"
+IMAGE_REL_TYPE = (
+    "http://schemas.openxmlformats.org/officeDocument/2006/relationships/image"
+)
+SOURCE_DOCUMENT_PART = "/word/document.xml"
+
+# Match old and new drawing placeholders (requires id/name, allows extra attributes)
+DRAWING_PATTERN = re.compile(
+    r'<drawing\b(?=[^>]*\bid="[^"]*")(?=[^>]*\bname="[^"]*")[^>]*/>'
+)
+DRAWING_TAG_PATTERN = re.compile(r"<drawing\b[^>]*/>")
+DRAWING_ATTR_PATTERN = re.compile(r'([a-zA-Z_][\w:.-]*)="([^"]*)"')
+
+
+@dataclass
+class DrawingRelationship:
+    """Relationship metadata for a single relationship ID."""
+
+    rel_id: str
+    target: str
+    target_mode: str
+    rel_type: str
+    part_name: Optional[str] = None
+    content_type: Optional[str] = None
+    image_format: Optional[str] = None
+
+
+@dataclass
+class DrawingExtractionContext:
+    """Context used to resolve and export drawing images for one DOCX file."""
+
+    docx_path: Path
+    blocks_output_path: Optional[Path] = None
+    export_dir_name: Optional[str] = None
+    export_dir_path: Optional[Path] = None
+    relationships: Dict[str, DrawingRelationship] = field(default_factory=dict)
+    _exported_part_to_relpath: Dict[str, str] = field(default_factory=dict)
+    _used_filenames: Dict[str, str] = field(default_factory=dict)
+
+    def resolve_relationship(self, rel_id: str) -> Optional[DrawingRelationship]:
+        return self.relationships.get(rel_id)
+
+    def export_embedded_image(self, rel: DrawingRelationship) -> Optional[str]:
+        """
+        Export an embedded image relationship target to export_dir.
+
+        Returns:
+            Relative path like "<blocks_stem>.image/image1.png" if exported,
+            or None when export is not applicable.
+        """
+        if not self.export_dir_path or not self.export_dir_name:
+            return None
+        if rel.target_mode.lower() == "external":
+            return None
+        if not rel.part_name:
+            return None
+        if rel.part_name in self._exported_part_to_relpath:
+            return self._exported_part_to_relpath[rel.part_name]
+
+        zip_member = rel.part_name.lstrip("/")
+        try:
+            with zipfile.ZipFile(self.docx_path, "r") as zf:
+                blob = zf.read(zip_member)
+        except Exception:
+            return None
+
+        filename = self._dedupe_filename(PurePosixPath(rel.part_name).name or "image")
+        output_file = self.export_dir_path / filename
+        output_file.write_bytes(blob)
+
+        rel_path = str(PurePosixPath(self.export_dir_name) / filename)
+        self._exported_part_to_relpath[rel.part_name] = rel_path
+        return rel_path
+
+    def _dedupe_filename(self, base_name: str) -> str:
+        if base_name not in self._used_filenames:
+            self._used_filenames[base_name] = base_name
+            return base_name
+
+        stem = Path(base_name).stem
+        suffix = Path(base_name).suffix
+        index = 2
+        while True:
+            candidate = f"{stem}_{index}{suffix}"
+            if candidate not in self._used_filenames:
+                self._used_filenames[candidate] = candidate
+                return candidate
+            index += 1
+
+
+def _normalize_image_format(ext_or_type: str) -> Optional[str]:
+    if not ext_or_type:
+        return None
+    value = ext_or_type.strip().lower()
+
+    # Content-Type
+    if value.startswith("image/"):
+        value = value.split("/", 1)[1]
+        if "+" in value:
+            value = value.split("+", 1)[0]
+        if value.startswith("x-"):
+            value = value[2:]
+
+    # Extension (with or without leading dot)
+    value = value.lstrip(".")
+    if value == "jpg":
+        return "jpeg"
+    if value in {"jpeg", "png", "gif", "bmp", "tiff", "webp", "svg", "emf", "wmf"}:
+        return value
+    return value or None
+
+
+def _infer_format_from_target(target: str) -> Optional[str]:
+    if not target:
+        return None
+    parsed = urlparse(target)
+    path = parsed.path if parsed.scheme else target
+    suffix = PurePosixPath(path).suffix
+    return _normalize_image_format(suffix)
+
+
+def _resolve_part_name(source_part_name: str, target: str) -> str:
+    if target.startswith("/"):
+        return posixpath.normpath(target)
+    source_dir = posixpath.dirname(source_part_name)
+    joined = posixpath.join(source_dir, target)
+    normalized = posixpath.normpath(joined)
+    if not normalized.startswith("/"):
+        normalized = "/" + normalized
+    return normalized
+
+
+def create_drawing_context(
+    docx_path: str,
+    blocks_output_path: Optional[str] = None,
+) -> DrawingExtractionContext:
+    """
+    Create extraction context for a DOCX file.
+
+    If blocks_output_path is provided, this also prepares `<blocks_stem>.image/`
+    beside the blocks file and clears any previous content.
+    """
+    docx_file = Path(docx_path)
+    ctx = DrawingExtractionContext(docx_path=docx_file)
+
+    if blocks_output_path:
+        output_path = Path(blocks_output_path)
+        export_dir_name = f"{output_path.stem}.image"
+        export_dir_path = output_path.parent / export_dir_name
+        if export_dir_path.exists():
+            shutil.rmtree(export_dir_path)
+        export_dir_path.mkdir(parents=True, exist_ok=True)
+        ctx.blocks_output_path = output_path
+        ctx.export_dir_name = export_dir_name
+        ctx.export_dir_path = export_dir_path
+
+    load_relationships(ctx)
+    return ctx
+
+
+def load_relationships(ctx: DrawingExtractionContext) -> None:
+    rels_xml = "word/_rels/document.xml.rels"
+    content_types_xml = "[Content_Types].xml"
+
+    overrides: Dict[str, str] = {}
+    defaults: Dict[str, str] = {}
+
+    try:
+        with zipfile.ZipFile(ctx.docx_path, "r") as zf:
+            if content_types_xml in zf.namelist():
+                ct_root = ET.parse(zf.open(content_types_xml)).getroot()
+                for node in ct_root.findall(f".//{{{CONTENT_TYPE_NS}}}Override"):
+                    part_name = node.get("PartName")
+                    content_type = node.get("ContentType")
+                    if part_name and content_type:
+                        overrides[part_name] = content_type
+                for node in ct_root.findall(f".//{{{CONTENT_TYPE_NS}}}Default"):
+                    ext = node.get("Extension")
+                    content_type = node.get("ContentType")
+                    if ext and content_type:
+                        defaults[ext.lower()] = content_type
+
+            if rels_xml not in zf.namelist():
+                return
+            rels_root = ET.parse(zf.open(rels_xml)).getroot()
+    except Exception:
+        return
+
+    for rel in rels_root.findall(f".//{{{REL_NS}}}Relationship"):
+        rel_id = rel.get("Id")
+        target = rel.get("Target", "")
+        target_mode = rel.get("TargetMode", "")
+        rel_type = rel.get("Type", "")
+        if not rel_id:
+            continue
+
+        part_name = None
+        content_type = None
+        image_format = None
+
+        if target_mode.lower() != "external":
+            part_name = _resolve_part_name(SOURCE_DOCUMENT_PART, target)
+            if part_name:
+                content_type = overrides.get(part_name)
+                if not content_type:
+                    ext = PurePosixPath(part_name).suffix.lower().lstrip(".")
+                    content_type = defaults.get(ext)
+                image_format = _normalize_image_format(
+                    content_type or _infer_format_from_target(part_name)
+                )
+        else:
+            image_format = _normalize_image_format(_infer_format_from_target(target))
+
+        ctx.relationships[rel_id] = DrawingRelationship(
+            rel_id=rel_id,
+            target=target,
+            target_mode=target_mode,
+            rel_type=rel_type,
+            part_name=part_name,
+            content_type=content_type,
+            image_format=image_format,
+        )
+
+
+def _extract_blip_relationship(drawing_elem) -> Optional[Tuple[str, str]]:
+    for blip in drawing_elem.findall(".//a:blip", NS):
+        # Prefer explicit external links when both link/embed are present on one blip.
+        # Word may keep an embedded cache for linked pictures.
+        rel_link = blip.get(f"{{{NS['r']}}}link")
+        if rel_link:
+            return "link", rel_link
+        rel_embed = blip.get(f"{{{NS['r']}}}embed")
+        if rel_embed:
+            return "embed", rel_embed
+    return None
+
+
+def _extract_imagedata_relationship(container_elem) -> Optional[str]:
+    """Find an image relationship id from a w:pict / w:object via v:imagedata.
+
+    These legacy VML containers are how Word references EMF/WMF metafiles
+    (and the rendered preview of any embedded OLE object). v:imagedata uses
+    ``r:id`` to point at the image part for both embedded and externally
+    linked images — the relationship's ``TargetMode`` is what disambiguates
+    the two cases, so the caller must inspect the resolved relationship.
+    """
+    r_id_attr = f"{{{NS['r']}}}id"
+    for imgdata in container_elem.findall(".//v:imagedata", NS):
+        rel_id = imgdata.get(r_id_attr)
+        if rel_id:
+            return rel_id
+    return None
+
+
+def _build_placeholder(attrs: Dict[str, str]) -> str:
+    ordered_keys = ["id", "name", "path", "format"]
+    pieces = []
+    for key in ordered_keys:
+        if key in attrs and attrs[key] is not None:
+            pieces.append(f'{key}="{escape(str(attrs[key]), quote=True)}"')
+
+    # Preserve extra attributes deterministically (sorted by name)
+    for key in sorted(k for k in attrs.keys() if k not in ordered_keys):
+        value = attrs[key]
+        if value is not None:
+            pieces.append(f'{key}="{escape(str(value), quote=True)}"')
+
+    return f"<drawing {' '.join(pieces)} />"
+
+
+def extract_drawing_placeholder_from_element(
+    drawing_elem,
+    context: Optional[DrawingExtractionContext] = None,
+    include_extended_attrs: bool = True,
+) -> str:
+    """
+    Build a <drawing ... /> placeholder from a w:drawing element.
+
+    Behavior:
+    - Always emits id/name from wp:docPr when present.
+    - For embedded images (a:blip@r:embed): exports image and sets path/format.
+    - For linked images (a:blip@r:link): does not download; path is original link target.
+    - When no image reference exists (e.g. chart drawing): keeps id/name only.
+    """
+    doc_pr = drawing_elem.find(".//wp:docPr", NS)
+    attrs = {
+        "id": doc_pr.get("id", "") if doc_pr is not None else "",
+        "name": doc_pr.get("name", "") if doc_pr is not None else "",
+    }
+
+    if include_extended_attrs:
+        rel_ref = _extract_blip_relationship(drawing_elem)
+        if rel_ref is not None and context is not None:
+            rel_kind, rel_id = rel_ref
+            rel = context.resolve_relationship(rel_id)
+            if rel is not None:
+                if rel_kind == "embed" and rel.rel_type == IMAGE_REL_TYPE:
+                    rel_path = context.export_embedded_image(rel)
+                    if rel_path:
+                        attrs["path"] = rel_path
+                    if rel.image_format:
+                        attrs["format"] = rel.image_format
+                elif rel_kind == "link":
+                    if rel.target:
+                        attrs["path"] = rel.target
+                    if rel.image_format:
+                        attrs["format"] = rel.image_format
+
+    return _build_placeholder(attrs)
+
+
+def extract_vml_image_placeholder_from_element(
+    container_elem,
+    context: Optional[DrawingExtractionContext] = None,
+    include_extended_attrs: bool = True,
+) -> str:
+    """
+    Build a <drawing ... /> placeholder from a w:pict or w:object element.
+
+    Legacy Word documents and OLE-embedded objects (Visio diagrams, equation
+    editor previews, etc.) expose their rendered image via VML rather than
+    DrawingML. The image is referenced through ``<v:imagedata r:id="..."/>``
+    inside ``<v:shape>``, and the underlying bytes are commonly EMF/WMF
+    metafiles. This function exports those bytes through the same context as
+    DrawingML images so EMF/WMF assets land in the blocks.assets directory
+    alongside PNG/JPEG ones.
+
+    The output placeholder format matches
+    ``extract_drawing_placeholder_from_element`` so downstream consumers
+    treat both paths uniformly.
+    """
+    shape = container_elem.find(".//v:shape", NS)
+    attrs = {
+        "id": shape.get("id", "") if shape is not None else "",
+        "name": shape.get("alt", "") if shape is not None else "",
+    }
+
+    if include_extended_attrs:
+        rel_id = _extract_imagedata_relationship(container_elem)
+        if rel_id and context is not None:
+            rel = context.resolve_relationship(rel_id)
+            if rel is not None and rel.rel_type == IMAGE_REL_TYPE:
+                # VML reuses r:id for both embedded image parts and externally
+                # linked images; only the resolved TargetMode tells us which.
+                # Treating an external relationship as embedded would call
+                # export_embedded_image() (which short-circuits on external)
+                # and silently drop the linked path.
+                if rel.target_mode.lower() == "external":
+                    if rel.target:
+                        attrs["path"] = rel.target
+                    if rel.image_format:
+                        attrs["format"] = rel.image_format
+                else:
+                    rel_path = context.export_embedded_image(rel)
+                    if rel_path:
+                        attrs["path"] = rel_path
+                    if rel.image_format:
+                        attrs["format"] = rel.image_format
+
+    return _build_placeholder(attrs)
+
+
+def parse_drawing_attributes(placeholder: str) -> Dict[str, str]:
+    """Parse attributes from a <drawing ... /> placeholder."""
+    return {
+        name: unescape(value)
+        for name, value in DRAWING_ATTR_PATTERN.findall(placeholder)
+    }
+
+
+def normalize_drawing_placeholder(
+    placeholder: str,
+    include_extended_attrs: bool = False,
+) -> str:
+    """
+    Normalize one drawing placeholder into canonical attribute order.
+
+    Args:
+        placeholder: Input placeholder string
+        include_extended_attrs: If False, keeps only id/name.
+    """
+    attrs = parse_drawing_attributes(placeholder)
+    normalized = {
+        "id": attrs.get("id", ""),
+        "name": attrs.get("name", ""),
+    }
+    if include_extended_attrs:
+        if "path" in attrs:
+            normalized["path"] = attrs["path"]
+        if "format" in attrs:
+            normalized["format"] = attrs["format"]
+        for key, value in attrs.items():
+            if key not in {"id", "name", "path", "format"}:
+                normalized[key] = value
+    return _build_placeholder(normalized)
+
+
+def normalize_drawing_placeholders_in_text(
+    text: str,
+    include_extended_attrs: bool = False,
+) -> str:
+    """Normalize all drawing placeholders inside a text blob."""
+    if not text:
+        return text
+
+    def _replace(match: re.Match) -> str:
+        return normalize_drawing_placeholder(
+            match.group(0),
+            include_extended_attrs=include_extended_attrs,
+        )
+
+    return DRAWING_TAG_PATTERN.sub(_replace, text)

lightrag/parser/docx/ir_builder.py

@@ -0,0 +1,339 @@
+"""Native DOCX IR builder: ``extract_docx_blocks`` output → :class:`IRDoc`.
+
+Input contract: a list of block dicts as produced by
+``lightrag.parser.docx.parse_document.extract_docx_blocks``. Each
+block carries ``content`` text in which ``<table>``, ``<equation>`` and
+``<drawing …/>`` placeholders are already embedded by the upstream parser.
+The builder rewrites those placeholders into IR placeholder tokens
+(``{{TBL:k}} / {{EQ:k}} / {{EQI:k}} / {{IMG:k}}``) and builds the matching
+``IRTable`` / ``IREquation`` / ``IRDrawing`` items.
+
+Asset bytes are extracted to disk by the upstream parser *before* this
+builder runs (via ``DrawingExtractionContext`` passed to
+``extract_docx_blocks``). The builder therefore declares assets with
+``AssetSpec.source=None`` — the writer records each entry's size without
+copying.
+
+Block-vs-inline equation distinction follows the legacy native rule: an
+``<equation>…</equation>`` tag is *block* iff each side is either the
+content boundary or a ``\\n`` character. Anything else stays inline,
+keeps its tag in block text without an id, and never enters
+``equations.json``.
+
+Positions are always emitted as ``IRPosition(type="paraid", range=[start,
+end])`` where each side may be ``None`` (legacy / non-Word docx authors
+sometimes omit ``w14:paraId``). The writer's ``to_jsonable`` faithfully
+preserves the per-side null so consumers can distinguish "start missing"
+vs "both missing".
+"""
+
+from __future__ import annotations
+
+import itertools
+import json
+import re
+from collections.abc import Callable
+from dataclasses import dataclass, field
+from pathlib import Path, PurePosixPath
+from typing import Any
+
+from lightrag.parser.docx.drawing_image_extractor import (
+    DRAWING_TAG_PATTERN,
+    parse_drawing_attributes,
+)
+from lightrag.sidecar.ir import (
+    AssetSpec,
+    IRBlock,
+    IRDoc,
+    IRDrawing,
+    IREquation,
+    IRPosition,
+    IRTable,
+)
+
+
+_TABLE_TAG_RE = re.compile(r"<table>(.*?)</table>", re.DOTALL)
+_EQUATION_TAG_RE = re.compile(r"<equation>(.*?)</equation>", re.DOTALL)
+
+
+def _normalize_dimension(rows_value: Any) -> tuple[int, int]:
+    if not isinstance(rows_value, list):
+        return 0, 0
+    num_rows = len(rows_value)
+    num_cols = max((len(r) for r in rows_value if isinstance(r, list)), default=0)
+    return num_rows, num_cols
+
+
+def _placeholder_keyspace() -> Callable[[str], str]:
+    """Return a fresh counter producing ``{prefix}{N}`` keys (1-indexed)."""
+    counter = itertools.count(1)
+    return lambda prefix: f"{prefix}{next(counter)}"
+
+
+def _safe_asset_ref_from_path(path_val: str, asset_prefix: str) -> str | None:
+    """Return the path inside ``asset_prefix`` only when it is safe.
+
+    Native DOCX images are pre-extracted into ``<base>.blocks.assets/``.
+    Treat a drawing path as local only when the suffix is a clean POSIX
+    relative path. Unsafe local-looking paths are dropped instead of being
+    registered as assets or preserved as linked references.
+    """
+    if not asset_prefix or not path_val.startswith(asset_prefix):
+        return None
+
+    rel_raw = path_val[len(asset_prefix) :]
+    if not rel_raw or "\\" in rel_raw:
+        return None
+
+    rel_path = PurePosixPath(rel_raw)
+    if rel_path.is_absolute():
+        return None
+    if any(part == ".." for part in rel_path.parts):
+        return None
+
+    rel = rel_path.as_posix()
+    if rel in {"", "."}:
+        return None
+    return rel
+
+
+@dataclass
+class _BlockBuilder:
+    """Per-block scratch state for the three ``re.sub`` rewrite passes.
+
+    Keeping the replacer routines as bound methods (rather than closures
+    redefined inside the per-block loop) means they're compiled once at
+    class-load and the state they mutate — ``tables`` / ``drawings`` /
+    ``equations`` / ``table_position`` — is held explicitly rather than
+    captured implicitly from the enclosing frame.
+    """
+
+    next_key: Callable[[str], str]
+    assets: list[AssetSpec]
+    seen_asset_refs: set[str]
+    asset_prefix: str
+    block_table_headers: list[Any]
+    tables: list[IRTable] = field(default_factory=list)
+    drawings: list[IRDrawing] = field(default_factory=list)
+    equations: list[IREquation] = field(default_factory=list)
+    # Position of the *next* ``<table>`` placeholder within this block,
+    # used to look up the matching entry in ``block_table_headers``.
+    table_position: int = 0
+
+    def replace_table(self, match: "re.Match[str]") -> str:
+        table_body_raw = match.group(1)
+        try:
+            rows = json.loads(table_body_raw)
+            if not isinstance(rows, list):
+                rows = None
+        except json.JSONDecodeError:
+            rows = None
+
+        if rows is not None:
+            parsed_rows: list[list[str]] | None = [
+                [str(c) for c in r] if isinstance(r, list) else [str(r)] for r in rows
+            ]
+            html: str | None = None
+        else:
+            parsed_rows = None
+            html = table_body_raw
+
+        num_rows, num_cols = _normalize_dimension(parsed_rows)
+
+        header_pos = self.table_position
+        self.table_position += 1
+        header_rows = (
+            self.block_table_headers[header_pos]
+            if header_pos < len(self.block_table_headers)
+            else None
+        )
+        # Treat empty list / explicit None identically: no header
+        # entry on the sidecar item.
+        table_header = header_rows if header_rows else None
+
+        placeholder = self.next_key("tb")
+        self.tables.append(
+            IRTable(
+                placeholder_key=placeholder,
+                rows=parsed_rows,
+                html=html,
+                num_rows=num_rows,
+                num_cols=num_cols,
+                caption="",
+                footnotes=[],
+                table_header=table_header,
+                body_override=table_body_raw,
+            )
+        )
+        return f"{{{{TBL:{placeholder}}}}}"
+
+    def replace_equation(self, match: "re.Match[str]") -> str:
+        latex = match.group(1)
+        source = match.string
+        start, end = match.start(), match.end()
+        is_block = (start == 0 or source[start - 1] == "\n") and (
+            end == len(source) or source[end] == "\n"
+        )
+        placeholder = self.next_key("eq")
+        self.equations.append(
+            IREquation(
+                placeholder_key=placeholder,
+                latex=latex,
+                is_block=is_block,
+                caption="",
+                footnotes=[],
+            )
+        )
+        token = "EQ" if is_block else "EQI"
+        return f"{{{{{token}:{placeholder}}}}}"
+
+    def replace_drawing(self, match: "re.Match[str]") -> str:
+        attrs = parse_drawing_attributes(match.group(0))
+        path_val = attrs.get("path", "") or ""
+        src_val = attrs.get("src", "") or ""
+        fmt = attrs.get("format", "") or ""
+        if not fmt and path_val:
+            fmt = Path(path_val).suffix.lower().lstrip(".")
+
+        # Two flavours of <drawing path="…">:
+        #   1. Local asset under <base>.blocks.assets/ — already
+        #      extracted to disk by DrawingExtractionContext;
+        #      register as AssetSpec(source=None) and let the
+        #      writer resolve the path via asset_paths.
+        #   2. External/linked path (URL, or any path that does
+        #      not live under asset_prefix) — pass through
+        #      verbatim via IRDrawing.path_override; do NOT emit
+        #      an AssetSpec (no on-disk bytes to materialize).
+        rel_inside_assets = _safe_asset_ref_from_path(path_val, self.asset_prefix)
+        if rel_inside_assets is not None:
+            asset_ref = rel_inside_assets
+            suggested_name = Path(rel_inside_assets).name or rel_inside_assets
+            if asset_ref and asset_ref not in self.seen_asset_refs:
+                self.assets.append(
+                    AssetSpec(
+                        ref=asset_ref,
+                        suggested_name=suggested_name,
+                        source=None,  # already extracted to disk
+                    )
+                )
+                self.seen_asset_refs.add(asset_ref)
+            path_override: str | None = None
+        else:
+            asset_ref = ""
+            # Only mark as an external/linked reference when the
+            # upstream parser actually emitted a path. An empty
+            # ``path=""`` should fall back to the regular asset-
+            # resolution path (which will also produce ``path=""``
+            # downstream) rather than masquerading as an explicit
+            # builder override.
+            path_override = (
+                None
+                if self.asset_prefix and path_val.startswith(self.asset_prefix)
+                else path_val or None
+            )
+
+        placeholder = self.next_key("im")
+        self.drawings.append(
+            IRDrawing(
+                placeholder_key=placeholder,
+                asset_ref=asset_ref,
+                fmt=fmt,
+                caption="",
+                footnotes=[],
+                src=src_val,
+                path_override=path_override,
+            )
+        )
+        return f"{{{{IMG:{placeholder}}}}}"
+
+
+class NativeDocxIRBuilder:
+    """Translate ``extract_docx_blocks`` output into an :class:`IRDoc`.
+
+    The builder is stateless — instantiate per call. ``asset_dir_name`` is
+    the relative name (without trailing slash) of ``<base>.blocks.assets/``
+    that the upstream parser used when emitting ``<drawing path>``
+    attributes; the builder strips that prefix when building
+    :attr:`AssetSpec.ref` so the writer's ref↔filename mapping has
+    predictable keys.
+    """
+
+    def normalize(
+        self,
+        blocks: list[dict[str, Any]],
+        *,
+        document_name: str,
+        asset_dir_name: str,
+        parse_metadata: dict[str, Any] | None = None,
+    ) -> IRDoc:
+        next_key = _placeholder_keyspace()
+        ir_blocks: list[IRBlock] = []
+        assets: list[AssetSpec] = []
+        seen_asset_refs: set[str] = set()
+
+        asset_prefix = f"{asset_dir_name}/" if asset_dir_name else ""
+
+        for block in blocks:
+            raw_content = block.get("content") or ""
+            heading = block.get("heading") or ""
+            level = int(block.get("level", 0) or 0)
+            parent_headings = list(block.get("parent_headings") or [])
+            # Preserve per-side nulls in [start, end].
+            uuid_start = block.get("uuid") or None
+            uuid_end = block.get("uuid_end") or None
+
+            builder = _BlockBuilder(
+                next_key=next_key,
+                assets=assets,
+                seen_asset_refs=seen_asset_refs,
+                asset_prefix=asset_prefix,
+                block_table_headers=list(block.get("table_headers") or []),
+            )
+
+            # Rewrite order matches the legacy native flow: tables, then
+            # equations, then drawings — each ``re.sub`` operates on the
+            # output of the previous pass.
+            content_template = _TABLE_TAG_RE.sub(builder.replace_table, raw_content)
+            content_template = _EQUATION_TAG_RE.sub(
+                builder.replace_equation, content_template
+            )
+            content_template = DRAWING_TAG_PATTERN.sub(
+                builder.replace_drawing, content_template
+            )
+
+            positions = [
+                IRPosition(type="paraid", range=[uuid_start, uuid_end]),
+            ]
+
+            ir_blocks.append(
+                IRBlock(
+                    content_template=content_template,
+                    heading=heading,
+                    level=level,
+                    parent_headings=parent_headings,
+                    positions=positions,
+                    tables=builder.tables,
+                    drawings=builder.drawings,
+                    equations=builder.equations,
+                )
+            )
+
+        # doc_title: parse_metadata["first_heading"] when present, else file
+        # stem fallback (resolved here so the writer doesn't have to know).
+        first_heading = ""
+        if isinstance(parse_metadata, dict):
+            first_heading = str(parse_metadata.get("first_heading") or "")
+        doc_title = first_heading or (Path(document_name).stem or document_name)
+
+        return IRDoc(
+            document_name=document_name,
+            document_format=Path(document_name).suffix.lower().lstrip("."),
+            doc_title=doc_title,
+            split_option={},
+            blocks=ir_blocks,
+            assets=assets,
+            bbox_attributes=None,
+        )
+
+
+__all__ = ["NativeDocxIRBuilder"]

lightrag/parser/docx/numbering_resolver.py

@@ -0,0 +1,423 @@
+#!/usr/bin/env python3
+"""
+ABOUTME: Resolves automatic numbering labels from DOCX documents
+ABOUTME: Parses numbering.xml and computes rendered number strings
+"""
+
+import zipfile
+from defusedxml import ElementTree as ET
+from typing import Dict
+
+NSMAP = {"w": "http://schemas.openxmlformats.org/wordprocessingml/2006/main"}
+
+
+class NumberingResolver:
+    """
+    Resolves paragraph numbering to rendered label strings.
+
+    DOCX stores numbering definitions in numbering.xml:
+    - abstractNum: Defines format templates (lvlText like "%1.%2.")
+    - num: Links numId to abstractNumId
+
+    Each paragraph references: numId (which definition) + ilvl (which level)
+    """
+
+    # Number format converters
+    FORMAT_CONVERTERS = {
+        "decimal": lambda n: str(n),
+        "lowerLetter": lambda n: chr(ord("a") + (n - 1) % 26),
+        "upperLetter": lambda n: chr(ord("A") + (n - 1) % 26),
+        "lowerRoman": lambda n: NumberingResolver._to_roman(n).lower(),
+        "upperRoman": lambda n: NumberingResolver._to_roman(n),
+        "chineseCountingThousand": lambda n: NumberingResolver._to_chinese(n),
+        "ideographTraditional": lambda n: "甲乙丙丁戊己庚辛壬癸"[(n - 1) % 10],
+        "bullet": lambda n: "•",
+        "none": lambda n: "",
+    }
+
+    def __init__(self, docx_path: str):
+        self.abstract_nums: Dict[str, dict] = {}  # abstractNumId -> level definitions
+        self.num_to_abstract: Dict[str, str] = {}  # numId -> abstractNumId
+        self.counters: Dict[
+            str, Dict[int, int]
+        ] = {}  # numId -> {ilvl -> current_count}
+        self.start_overrides: Dict[
+            str, Dict[int, int]
+        ] = {}  # numId -> {ilvl -> start_value}
+        self.style_numpr: Dict[
+            str, dict
+        ] = {}  # styleId -> {numId, ilvl} from styles.xml
+        self.style_based_on: Dict[str, str] = {}  # styleId -> basedOn styleId
+        # Smart numbering merge state (Word's rendering behavior)
+        self.last_numId: str = None  # Previous paragraph's numId
+        self.last_abstract_id: str = None  # Previous paragraph's abstractNumId
+        self.last_style_id: str = None  # Previous paragraph's style ID
+        self._parse_numbering_xml(docx_path)
+        self._parse_styles_xml(docx_path)
+
+    def _parse_numbering_xml(self, docx_path: str):
+        """Parse numbering.xml from DOCX archive"""
+        try:
+            with zipfile.ZipFile(docx_path, "r") as zf:
+                if "word/numbering.xml" not in zf.namelist():
+                    return
+
+                tree = ET.parse(zf.open("word/numbering.xml"))
+                root = tree.getroot()
+
+                # Parse abstractNum definitions
+                for abstract in root.findall(".//w:abstractNum", NSMAP):
+                    abstract_id = abstract.get(f"{{{NSMAP['w']}}}abstractNumId")
+                    levels = {}
+
+                    for lvl in abstract.findall("w:lvl", NSMAP):
+                        ilvl = int(lvl.get(f"{{{NSMAP['w']}}}ilvl"))
+
+                        start_elem = lvl.find("w:start", NSMAP)
+                        start = (
+                            int(start_elem.get(f"{{{NSMAP['w']}}}val"))
+                            if start_elem is not None
+                            else 1
+                        )
+
+                        num_fmt_elem = lvl.find("w:numFmt", NSMAP)
+                        num_fmt = (
+                            num_fmt_elem.get(f"{{{NSMAP['w']}}}val")
+                            if num_fmt_elem is not None
+                            else "decimal"
+                        )
+
+                        lvl_text_elem = lvl.find("w:lvlText", NSMAP)
+                        lvl_text = (
+                            lvl_text_elem.get(f"{{{NSMAP['w']}}}val")
+                            if lvl_text_elem is not None
+                            else "%1."
+                        )
+
+                        is_lgl_elem = lvl.find("w:isLgl", NSMAP)
+                        is_lgl = False
+                        if is_lgl_elem is not None:
+                            val = is_lgl_elem.get(f"{{{NSMAP['w']}}}val")
+                            is_lgl = val is None or val not in ("0", "false")
+
+                        levels[ilvl] = {
+                            "start": start,
+                            "numFmt": num_fmt,
+                            "lvlText": lvl_text,
+                            "isLgl": is_lgl,
+                        }
+
+                    self.abstract_nums[abstract_id] = levels
+
+                # Parse num -> abstractNum mapping and startOverride
+                for num in root.findall(".//w:num", NSMAP):
+                    num_id = num.get(f"{{{NSMAP['w']}}}numId")
+                    abstract_ref = num.find("w:abstractNumId", NSMAP)
+                    if abstract_ref is not None:
+                        self.num_to_abstract[num_id] = abstract_ref.get(
+                            f"{{{NSMAP['w']}}}val"
+                        )
+
+                    # Parse lvlOverride/startOverride for this num
+                    for lvl_override in num.findall("w:lvlOverride", NSMAP):
+                        ilvl = int(lvl_override.get(f"{{{NSMAP['w']}}}ilvl"))
+                        start_override = lvl_override.find("w:startOverride", NSMAP)
+                        if start_override is not None:
+                            start_val = int(start_override.get(f"{{{NSMAP['w']}}}val"))
+                            if num_id not in self.start_overrides:
+                                self.start_overrides[num_id] = {}
+                            self.start_overrides[num_id][ilvl] = start_val
+        except Exception:
+            # Silently ignore parsing errors - document may not have numbering
+            pass
+
+    def _parse_styles_xml(self, docx_path: str):
+        """Parse styles.xml to get style-inherited numbering definitions"""
+        try:
+            with zipfile.ZipFile(docx_path, "r") as zf:
+                if "word/styles.xml" not in zf.namelist():
+                    return
+
+                tree = ET.parse(zf.open("word/styles.xml"))
+                root = tree.getroot()
+
+                # Parse style definitions
+                for style in root.findall(".//w:style", NSMAP):
+                    style_id = style.get(f"{{{NSMAP['w']}}}styleId")
+                    if not style_id:
+                        continue
+
+                    # Check for basedOn (style inheritance)
+                    based_on = style.find("w:basedOn", NSMAP)
+                    if based_on is not None:
+                        parent_id = based_on.get(f"{{{NSMAP['w']}}}val")
+                        if parent_id:
+                            self.style_based_on[style_id] = parent_id
+
+                    # Check for numPr in style's pPr
+                    pPr = style.find("w:pPr", NSMAP)
+                    if pPr is not None:
+                        numPr = pPr.find("w:numPr", NSMAP)
+                        if numPr is not None:
+                            num_id_elem = numPr.find("w:numId", NSMAP)
+                            ilvl_elem = numPr.find("w:ilvl", NSMAP)
+
+                            if num_id_elem is not None:
+                                num_id = num_id_elem.get(f"{{{NSMAP['w']}}}val")
+                                ilvl = (
+                                    int(ilvl_elem.get(f"{{{NSMAP['w']}}}val"))
+                                    if ilvl_elem is not None
+                                    else 0
+                                )
+                                self.style_numpr[style_id] = {
+                                    "numId": num_id,
+                                    "ilvl": ilvl,
+                                }
+        except Exception:
+            # Silently ignore parsing errors
+            pass
+
+    def _get_numbering_from_style(self, style_id: str, visited=None) -> dict:
+        """
+        Get numbering definition from style, following inheritance chain.
+
+        Args:
+            style_id: Style ID to look up
+            visited: Set of visited style IDs (to prevent circular references)
+
+        Returns:
+            dict with 'numId' and 'ilvl', or None
+        """
+        if visited is None:
+            visited = set()
+
+        # Prevent circular references
+        if style_id in visited:
+            return None
+        visited.add(style_id)
+
+        # Check if this style has numPr
+        if style_id in self.style_numpr:
+            return self.style_numpr[style_id]
+
+        # Check parent style
+        if style_id in self.style_based_on:
+            parent_id = self.style_based_on[style_id]
+            return self._get_numbering_from_style(parent_id, visited)
+
+        return None
+
+    def reset_tracking_state(self):
+        """
+        Reset numbering tracking state.
+
+        Call this when encountering structural breaks that should
+        interrupt numbering continuity:
+        - Section breaks (sectPr)
+        - Table boundaries (before and after tables)
+
+        This prevents incorrect numbering continuation across
+        document structure boundaries.
+        """
+        self.last_numId = None
+        self.last_abstract_id = None
+        self.last_style_id = None
+
+    def get_label(self, para_element) -> str:
+        """
+        Get rendered numbering label for a paragraph.
+
+        Checks both direct numPr and style-inherited numbering. Direct numPr
+        is a paragraph-local override and applies only to the current
+        paragraph; subsequent paragraphs that carry only pStyle fall back to
+        the style's numPr declared in styles.xml.
+
+        Args:
+            para_element: lxml Element for <w:p>
+
+        Returns:
+            Rendered label string (e.g., "1.1", "a)", "第一章") or empty string
+        """
+        try:
+            pPr = para_element.find(f"{{{NSMAP['w']}}}pPr")
+            if pPr is None:
+                return ""
+
+            num_id = None
+            ilvl = 0
+            style_id = None
+
+            # Get pStyle (if present)
+            pStyle = pPr.find(f"{{{NSMAP['w']}}}pStyle")
+            if pStyle is not None:
+                style_id = pStyle.get(f"{{{NSMAP['w']}}}val")
+
+            # Check for direct numPr in paragraph
+            numPr = pPr.find(f"{{{NSMAP['w']}}}numPr")
+            if numPr is not None:
+                num_id_elem = numPr.find(f"{{{NSMAP['w']}}}numId")
+                ilvl_elem = numPr.find(f"{{{NSMAP['w']}}}ilvl")
+
+                if num_id_elem is not None:
+                    num_id = num_id_elem.get(f"{{{NSMAP['w']}}}val")
+                    ilvl = (
+                        int(ilvl_elem.get(f"{{{NSMAP['w']}}}val"))
+                        if ilvl_elem is not None
+                        else 0
+                    )
+
+            # If no direct numPr, fall back to style-inherited numbering.
+            # Direct numPr is a paragraph-local override in Word; it must not
+            # persist as a runtime default for the style, otherwise subsequent
+            # paragraphs that only carry pStyle will keep following the local
+            # override instead of the style's declared numPr.
+            if num_id is None and style_id:
+                style_num = self._get_numbering_from_style(style_id)
+                if style_num:
+                    num_id = style_num["numId"]
+                    ilvl = style_num["ilvl"]
+
+            # If still no numbering found, clear state and return empty
+            if num_id is None:
+                # We should use list structure breaking logic to reset last_numId, last_abstract_id and last_style_id
+                return ""
+
+            # Get abstract definition
+            abstract_id = self.num_to_abstract.get(num_id)
+            if abstract_id is None or abstract_id not in self.abstract_nums:
+                # Clear state for invalid numbering
+                self.last_numId = None
+                self.last_abstract_id = None
+                return ""
+
+            levels = self.abstract_nums[abstract_id]
+            if ilvl not in levels:
+                # Clear state for invalid level
+                self.last_numId = None
+                self.last_abstract_id = None
+                return ""
+
+            # Smart numbering merge: (Word's rendering behavior)
+            # When consecutive paragraphs have different numId but same abstractNumId,
+            # Word continues the numbering sequence rather than restarting.
+            # This happens regardless of whether the numId is new or style matches.
+
+            if (
+                self.last_numId is not None
+                and self.last_numId != num_id
+                and self.last_abstract_id == abstract_id
+                and self.last_numId in self.counters
+            ):
+                # Merge: copy previous numId's counter to current numId
+                self.counters[num_id] = self.counters[self.last_numId].copy()
+
+            # Initialize/update counter
+            if num_id not in self.counters:
+                self.counters[num_id] = {}
+
+            # Initialize all parent levels if not present (for deep nested numbering)
+            for i in range(ilvl):
+                if i not in self.counters[num_id] and i in levels:
+                    # Use startOverride if exists, otherwise use abstractNum's start value
+                    if (
+                        num_id in self.start_overrides
+                        and i in self.start_overrides[num_id]
+                    ):
+                        self.counters[num_id][i] = self.start_overrides[num_id][i]
+                    else:
+                        self.counters[num_id][i] = levels[i]["start"]
+
+            # Reset lower levels when higher level increments
+            for i in range(ilvl + 1, 10):
+                if i in self.counters[num_id]:
+                    del self.counters[num_id][i]
+
+            # Initialize current level if needed
+            if ilvl not in self.counters[num_id]:
+                # Use startOverride if exists, otherwise use abstractNum's start value
+                if (
+                    num_id in self.start_overrides
+                    and ilvl in self.start_overrides[num_id]
+                ):
+                    self.counters[num_id][ilvl] = self.start_overrides[num_id][ilvl]
+                else:
+                    self.counters[num_id][ilvl] = levels[ilvl]["start"]
+            else:
+                self.counters[num_id][ilvl] += 1
+
+            # Format the label using lvlText template
+            label = self._format_label(num_id, ilvl, levels)
+
+            # Update tracking state for next paragraph
+            self.last_numId = num_id
+            self.last_abstract_id = abstract_id
+            self.last_style_id = style_id
+
+            return label
+        except Exception:
+            # Return empty on any error to avoid breaking document parsing
+            return ""
+
+    def _format_label(self, num_id: str, ilvl: int, levels: dict) -> str:
+        """Format label string by replacing %1, %2, etc."""
+        try:
+            lvl_text = levels[ilvl]["lvlText"]
+            result = lvl_text
+            current_is_lgl = levels[ilvl].get("isLgl", False)
+
+            for i in range(ilvl + 1):
+                if i in levels and i in self.counters.get(num_id, {}):
+                    num_fmt = levels[i]["numFmt"]
+                    if current_is_lgl and i < ilvl:
+                        num_fmt = "decimal"
+                    count = self.counters[num_id][i]
+                    converter = self.FORMAT_CONVERTERS.get(num_fmt, lambda n: str(n))
+                    formatted = converter(count)
+                    result = result.replace(f"%{i + 1}", formatted)
+
+            return result
+        except Exception:
+            return ""
+
+    @staticmethod
+    def _to_roman(n: int) -> str:
+        """Convert integer to Roman numeral"""
+        if n <= 0 or n >= 4000:
+            return str(n)
+        values = [
+            (1000, "M"),
+            (900, "CM"),
+            (500, "D"),
+            (400, "CD"),
+            (100, "C"),
+            (90, "XC"),
+            (50, "L"),
+            (40, "XL"),
+            (10, "X"),
+            (9, "IX"),
+            (5, "V"),
+            (4, "IV"),
+            (1, "I"),
+        ]
+        result = ""
+        for value, numeral in values:
+            while n >= value:
+                result += numeral
+                n -= value
+        return result
+
+    @staticmethod
+    def _to_chinese(n: int) -> str:
+        """Convert integer to Chinese numeral"""
+        digits = "零一二三四五六七八九"
+        if n <= 0 or n > 99:
+            return str(n)
+        if n < 10:
+            return digits[n]
+        if n < 20:
+            return "十" + (digits[n % 10] if n % 10 else "")
+        if n < 100:
+            tens = n // 10
+            ones = n % 10
+            return digits[tens] + "十" + (digits[ones] if ones else "")
+        return str(n)

lightrag/parser/docx/omml/__init__.py

@@ -0,0 +1,10 @@
+"""
+ABOUTME: OMML (Office Math Markup Language) to LaTeX conversion
+"""
+
+from .ommlparser import OMMLParser
+
+
+def convert_omml_to_latex(omml_element) -> str:
+    """Convert an m:oMath XML element to a LaTeX string."""
+    return OMMLParser().parse(omml_element)

lightrag/parser/docx/omml/cleaners.py

@@ -0,0 +1,38 @@
+"""
+Postprocessing functions for cleaning up latex equations in linear format which don't give valid LaTeX.
+"""
+
+import re
+
+clean_exps = {
+    r"\\degf": "°F",
+    r"\\degc": "°C",
+    r"(\\cbrt)(\w+)": r"\\sqrt[3]{\2}",
+    r"(\\qdrt)(\w+)": r"\\sqrt[4]{\2}",
+    r"\\sfrac": r"\\frac",
+    r"(\\o[i]+nt)(\w+)": r"\1{\2}",
+    r"\\bullet(\w+)": r"\\bullet \1",
+    r"\\sum([a-zA-Z0-9]+)": r"\\sum{\1}",
+    r"\\prod([a-zA-Z0-9]+)": r"\\prod{\1}",
+    r"\\amalg([a-zA-Z0-9]+)": r"\\amalg{\1}",
+    r"\\bigcup([a-zA-Z0-9]+)": r"\\bigcup{\1}",
+    r"\\bigcap([a-zA-Z0-9]+)": r"\\bigcap{\1}",
+    r"\\bigvee([a-zA-Z0-9]+)": r"\\bigvee{\1}",
+    r"\\bigwedge([a-zA-Z0-9]+)": r"\\bigwedge{\1}",
+    r"\\lfloor([a-zA-Z0-9]+)": r"\\lfloor{\1}",
+    r"\\lceil([a-zA-Z0-9]+)": r"\\lceil{\1}",
+    r"\\lim\\below\{(.+)\}\{(.+)\}": r"\\lim_{\1}{\2}",
+    r"\\min\\below\{(.+)\}\{(.+)\}": r"\\min_{\1}{\2}",
+    r"\\max\\below\{(.+)\}\{(.+)\}": r"\\max_{\1}{\2}",
+}
+
+
+def clean_exp(exp):
+    """
+    Takes in a linear expression and converts known invalid LaTeX equations to valid LaTeX
+    :param exp:str - An equation in invalid syntax
+    :return :str - A valid equation
+    """
+    for e in clean_exps:
+        exp = re.sub(e, clean_exps[e], exp)
+    return exp

lightrag/parser/docx/omml/ommlparser.py

@@ -0,0 +1,511 @@
+from xml.etree.cElementTree import Element
+
+from .utils import qn
+
+
+class OMMLParser:
+    """
+    Parser class for reading OMML and converting it into LaTeX.
+    """
+
+    FUNCTION_MAP = {
+        "sin": "\\sin",
+        "cos": "\\cos",
+        "tan": "\\tan",
+        "cot": "\\cot",
+        "sec": "\\sec",
+        "csc": "\\csc",
+        "sinh": "\\sinh",
+        "cosh": "\\cosh",
+        "tanh": "\\tanh",
+        "coth": "\\coth",
+        "sech": "\\operatorname{sech}",
+        "csch": "\\operatorname{csch}",
+        "log": "\\log",
+        "ln": "\\ln",
+        "min": "\\min",
+        "max": "\\max",
+        "lim": "\\lim",
+    }
+
+    def _normalize_func_name(self, content: str) -> str:
+        if not content:
+            return content
+        if content.startswith("\\"):
+            return content
+        key = content.strip()
+        mapped = self.FUNCTION_MAP.get(key)
+        return mapped if mapped else content
+
+    def parse(self, root: Element) -> str:
+        """
+        Parses an m:oMath OMML tag into LaTeX.
+        :param root: An m:oMath OMML tag
+        :return: The LaTeX representation of the OMML input
+        """
+        text = ""
+        try:
+            if root.tag == qn("m:t"):
+                return self.parse_t(root)
+            for child in root:
+                if child.tag in self.parsers:
+                    text += self.parsers[child.tag](self, child)
+        except AttributeError:
+            # In case of missing attributes on OMML tags,
+            # we return an empty string (ref:issue_14)
+            return ""
+        return text
+
+    def parse_e(self, root: Element) -> str:
+        text = ""
+        for child in root:
+            text += self.parse(child)
+        return text
+
+    def parse_r(self, root: Element) -> str:
+        # TODO: Add support for m:rPr and m:scr to support different character styles
+        #    For now, we just parse the text content of m:r
+        text = ""
+        for child in root:
+            text += self.parse(child)
+        return text
+
+    def parse_t(self, root: Element):
+        symbol_map = {
+            "≜": "\\triangleq",
+            "≝": "\\stackrel{\\tiny def}{=}",
+            "≞": "\\stackrel{\\tiny m}{=}",
+        }
+        replacements = {
+            "<": "\\lt ",
+            ">": "\\gt ",
+            "≤": "\\leq ",
+            "≥": "\\geq ",
+            "∞": "\\infty ",
+            "<": "\\lt ",
+            ">": "\\gt ",
+            "≤": "\\leq ",
+            "≥": "\\geq ",
+        }
+        text = root.text.split()
+        if not text:
+            return " "
+        for i, t in enumerate(text):
+            if t in symbol_map:
+                text[i] = symbol_map[t]
+        for key, value in replacements.items():
+            for i, t in enumerate(text):
+                text[i] = t.replace(key, value)
+        return " ".join(text)
+
+    def parse_acc(self, root: Element) -> str:
+        character_map = {
+            768: "\\grave",
+            769: "\\acute",
+            770: "\\hat",
+            771: "\\tilde",
+            773: "\\bar",
+            774: "\\breve",
+            775: "\\dot",
+            776: "\\ddot",
+            780: "\\check",
+            831: "\\overline{\\overline",
+            8400: "\\overset\\leftharpoonup",
+            8401: "\\overset\\rightharpoonup",
+            8406: "\\overleftarrow",
+            8407: "\\overrightarrow",
+            8411: "\\dddot",
+            8417: "\\overset\\leftrightarrow",
+        }
+        text = ""
+        accent = 770
+        for child in root:
+            if child.tag == qn("m:accPr"):
+                for child2 in child:
+                    if child2.tag == qn("m:chr"):
+                        val = child2.attrib.get(qn("m:val"))
+                        if val:
+                            try:
+                                accent = ord(val)
+                            except TypeError:
+                                pass
+
+        accent_cmd = character_map.get(accent)
+        if accent_cmd is None:
+            accent_cmd = character_map.get(770, "\\hat")
+        text += accent_cmd + "{"
+        for child in root:
+            if child.tag == qn("m:e"):
+                text += self.parse(child)
+        text += "}"
+        if accent == 831:
+            text += "}"
+        return text
+
+    def parse_bar(self, root: Element) -> str:
+        text = "\\overline{"
+        for child in root:
+            if child.tag == qn("m:barPr"):
+                for child2 in child:
+                    if child2.tag == qn("m:pos"):
+                        if child2.attrib.get(qn("m:val")) == "bot":
+                            text = "\\underline{"
+
+        for child in root:
+            if child.tag == qn("m:e"):
+                text += self.parse(child)
+        text += "}"
+        return text
+
+    def parse_border_box(self, root: Element) -> str:
+        text = "\\boxed{"
+        for child in root:
+            if child.tag == qn("m:e"):
+                text += self.parse(child)
+        text += "}"
+        return text
+
+    def parse_box(self, root: Element) -> str:
+        text = ""
+        for child in root:
+            text += self.parse(child)
+        return text
+
+    def parse_group_chr(self, root: Element) -> str:
+        character_map = {
+            "←": "\\leftarrow",
+            "→": "\\rightarrow",
+            "↔": "\\leftrightarrow",
+            "⇐": "\\Leftarrow",
+            "⇒": "\\Rightarrow",
+            "⇔": "\\Leftrightarrow",
+        }
+        text = "\\underbrace{"
+        bottom = False
+        for child in root:
+            if child.tag == qn("m:groupChrPr"):
+                for child2 in child:
+                    if child2.tag == qn("m:chr"):
+                        char = child2.attrib.get(qn("m:val"))
+                        if char in character_map:
+                            text = character_map[char]
+                for child2 in child:
+                    if (
+                        child2.tag == qn("m:pos")
+                        and child2.attrib.get(qn("m:val")) == "top"
+                    ):
+                        # If m:pos is set to "top", the symbol is supposed to
+                        # be on top and the text is actually supposed to be under
+                        bottom = True
+
+        content = ""
+        for child in root:
+            if child.tag == qn("m:e"):
+                content = self.parse(child)
+        if text == "\\underbrace{":
+            if bottom:
+                text = "\\overbrace{" + content + "}"
+            else:
+                text += content + "}"
+        else:
+            if not bottom:
+                text = "\\overset{" + content + "}" + "{" + text + "}"
+            else:
+                text = "\\underset{" + content + "}" + "{" + text + "}"
+        return text
+
+    def parse_d(self, root: Element) -> str:
+        bracket_map = {
+            "(": "\\left(",
+            ")": "\\right)",
+            "[": "\\left[",
+            "]": "\\right]",
+            "{": "\\left{",
+            "}": "\\right}",
+            "〈": "\\left\\langle",
+            "〉": "\\right\\rangle",
+            "⟨": "\\left\\langle",
+            "⟩": "\\right\\rangle",
+            "⌊": "\\left\\lfloor",
+            "⌋": "\\right\\rfloor",
+            "⌈": "\\left\\lceil",
+            "⌉": "\\right\\rceil",
+            "|": "\\left|",
+            "‖": "\\left\\|",
+            "⟦": "[\\![",
+            "⟧": "]\\!]",
+        }
+        text = ""
+        start_bracket = "("
+        end_bracket = ")"
+        seperator = "|"
+        is_matrix = False
+        for child in root:
+            for child2 in child:
+                if child.tag == qn("m:dPr"):
+                    if child2.tag == qn("m:begChr"):
+                        start_bracket = child2.attrib.get(qn("m:val"))
+                    if child2.tag == qn("m:endChr"):
+                        end_bracket = child2.attrib.get(qn("m:val"))
+                    if child2.tag == qn("m:sepChr"):
+                        seperator = child2.attrib.get(qn("m:val"))
+                if child2.tag == qn("m:m"):
+                    is_matrix = True
+
+        for child in root:
+            if child.tag == qn("m:e"):
+                if text:
+                    text += seperator
+                text += self.parse(child)
+        end_bracket_replacements = {
+            "|": "\\right|",
+            "‖": "\\right\\|",
+            "[": "\\right[",
+        }
+        start_bracket_replacements = {
+            "]": "\\left]",
+        }
+        start = ""
+        end = ""
+        if start_bracket:
+            if start_bracket in start_bracket_replacements:
+                start = start_bracket_replacements[start_bracket] + " "
+            elif start_bracket in bracket_map:
+                start = bracket_map[start_bracket] + " "
+            else:
+                start = "\\left(" + " "
+        if end_bracket:
+            if end_bracket in end_bracket_replacements:
+                end = " " + end_bracket_replacements[end_bracket]
+            elif end_bracket in bracket_map:
+                end = " " + bracket_map[end_bracket]
+            else:
+                end = " " + "\\right)"
+        # If there is no end bracket and this tag contains an m:eqArr tag as a
+        # child, we assume that the eqArr should be translated to a cases environment
+        # instead of an eqnarray* environment.
+        else:
+            for child in root:
+                if child.tag == qn("m:e"):
+                    for child2 in child:
+                        if child2.tag == qn("m:eqArr"):
+                            text = text.replace("\\begin{eqnarray*}", "")
+                            text = text.replace("\\end{eqnarray*}", "")
+                            return "\\begin{cases} " + text + " \\end{cases}"
+        if is_matrix:
+            if start_bracket == "(" and end_bracket == ")":
+                return text.replace("{matrix}", "{pmatrix}")
+            elif start_bracket == "|" and end_bracket == "|":
+                return text.replace("{matrix}", "{vmatrix}")
+            elif start_bracket == "‖" and end_bracket == "‖":
+                return text.replace("{matrix}", "{Vmatrix}")
+            else:
+                return text.replace("{matrix}", "{bmatrix}")
+        return start + text + end
+
+    def parse_eq_arr(self, root: Element) -> str:
+        text = "\\begin{eqnarray*}"
+        for child in root:
+            if child.tag == qn("m:e"):
+                text += self.parse(child) + " \\\\"
+        text += "\\end{eqnarray*}"
+        return text
+
+    def parse_f(self, root: Element) -> str:
+        text = "\\frac{"
+        num = ""
+        den = ""
+        is_binom = False
+        for child in root:
+            if child.tag == qn("m:fPr"):
+                for child2 in child:
+                    if (
+                        child2.tag == qn("m:type")
+                        and child2.attrib.get(qn("m:val")) == "noBar"
+                    ):
+                        is_binom = True
+            if child.tag == qn("m:num"):
+                num = self.parse(child)
+            if child.tag == qn("m:den"):
+                den = self.parse(child)
+        if is_binom:
+            text = "\\genfrac{}{}{0pt}{}{"
+        text += num + "}{" + den + "}"
+        return text
+
+    def parse_m(self, root: Element) -> str:
+        text = "\\begin{matrix} "
+        text += self.parse(root)[:-3]  # Remove the last ' \\'
+        text += "\\end{matrix}"
+        return text
+
+    def parse_mr(self, root: Element) -> str:
+        text = ""
+        for child in root:
+            if child.tag == qn("m:e"):
+                text += self.parse(child) + " & "
+        return text[:-2] + "\\\\ "  # Remove the last ' & '
+
+    def parse_func(self, root: Element) -> str:
+        subscript = ""
+        superscript = ""
+        text = ""
+        func_name = "sin"
+        for child in root:
+            if child.tag == qn("m:fName"):
+                for child2 in child:
+                    if child2.tag in [qn("m:sSup"), qn("m:sSub"), qn("m:r")]:
+                        for child3 in child2:
+                            if child3.tag == qn("m:sub"):
+                                subscript = self.parse(child3)
+                            if child3.tag == qn("m:sup"):
+                                superscript = self.parse(child3)
+                            if child3.tag == qn("m:t") or child3.tag == qn("m:e"):
+                                func_name = self.parse(child3)
+                    elif child2.tag == qn("m:limLow"):
+                        for child3 in child2:
+                            if child3.tag == qn("m:lim"):
+                                for child4 in child3:
+                                    subscript += self.parse(child4)
+                            if child3.tag == qn("m:e"):
+                                func_name = self.parse(child3)
+
+            if child.tag == qn("m:e"):
+                text += self.parse(child)
+        if func_name in ["lim", "max", "min"]:
+            return f"\\{func_name}\\limits_{{{subscript}}}^{{{superscript}}}{{{text}}}"
+        if func_name not in self.FUNCTION_MAP:
+            return f"{{{func_name}}}^{{{superscript}}}_{{{subscript}}}{{{text}}}"
+        return (
+            self.FUNCTION_MAP[func_name]
+            + f"_{{{subscript}}}^{{{superscript}}}{{{text}}}"
+        )
+
+    def parse_s_sup(self, root: Element) -> str:
+        content = ""
+        exp_content = ""
+        for child in root:
+            if child.tag == qn("m:e"):
+                content = self.parse(child)
+            if child.tag == qn("m:sup"):
+                exp_content = self.parse(child)
+        content = self._normalize_func_name(content)
+        return f"{{{content}}}^{{{exp_content}}}"
+
+    def parse_s_sub(self, root: Element) -> str:
+        content = ""
+        sub_content = ""
+        for child in root:
+            if child.tag == qn("m:e"):
+                content = self.parse(child)
+            if child.tag == qn("m:sub"):
+                sub_content = self.parse(child)
+        content = self._normalize_func_name(content)
+        return f"{{{content}}}_{{{sub_content}}}"
+
+    def parse_s_sub_sup(self, root: Element) -> str:
+        content = ""
+        sub_content = ""
+        exp_content = ""
+        for child in root:
+            if child.tag == qn("m:e"):
+                content = self.parse(child)
+            if child.tag == qn("m:sub"):
+                sub_content = self.parse(child)
+            if child.tag == qn("m:sup"):
+                exp_content = self.parse(child)
+        content = self._normalize_func_name(content)
+        return f"{{{content}}}_{{{sub_content}}}^{{{exp_content}}}"
+
+    def parse_s_pre(self, root: Element) -> str:
+        content = ""
+        sub_content = ""
+        exp_content = ""
+        for child in root:
+            if child.tag == qn("m:e"):
+                content = self.parse(child)
+            if child.tag == qn("m:sub"):
+                sub_content = self.parse(child)
+            if child.tag == qn("m:sup"):
+                exp_content = self.parse(child)
+        return "{}^{" + exp_content + "}_{" + sub_content + "}{" + content + "}"
+
+    def parse_rad(self, root: Element) -> str:
+        content = ""
+        order = ""
+        for child in root:
+            if child.tag == qn("m:deg"):
+                order = self.parse(child)
+            if child.tag == qn("m:e"):
+                content += self.parse(child)
+        if order:
+            return f"\\sqrt[{order}]{{{content}}}"
+        return f"\\sqrt{{{content}}}"
+
+    def parse_nary(self, root: Element) -> str:
+        character_map = {
+            8719: "\\prod",
+            8720: "\\coprod",
+            8721: "\\sum",
+            8747: "\\int",
+            8748: "\\iint",
+            8749: "\\iiint",
+            8750: "\\oint",
+            8751: "\\oiint",
+            8752: "\\oiiint",
+            8896: "\\bigwedge",
+            8897: "\\bigvee",
+            8898: "\\bigcap",
+            8899: "\\bigcup",
+        }
+        char = 8747
+        for child in root:
+            if child.tag == qn("m:naryPr"):
+                for child2 in child:
+                    if child2.tag == qn("m:chr"):
+                        val = child2.attrib.get(qn("m:val"))
+                        if val:
+                            try:
+                                char = ord(val)
+                            except TypeError:
+                                pass
+        text = character_map.get(char, character_map[8721])
+        sub = ""
+        sup = ""
+        content = ""
+        for child in root:
+            if child.tag == qn("m:sub"):
+                sub = self.parse(child)
+            if child.tag == qn("m:sup"):
+                sup = self.parse(child)
+            if child.tag == qn("m:e"):
+                content = self.parse(child)
+        if sub:
+            text += f"_{{{sub}}}"
+        if sup:
+            text += f"^{{{sup}}}"
+        text += "{" + content + "}"
+        return text
+
+    parsers = {
+        qn("m:r"): parse_r,
+        qn("m:acc"): parse_acc,
+        qn("m:borderBox"): parse_border_box,
+        qn("m:bar"): parse_bar,
+        qn("m:box"): parse_box,
+        qn("m:d"): parse_d,
+        qn("m:e"): parse_e,
+        qn("m:groupChr"): parse_group_chr,
+        qn("m:f"): parse_f,
+        qn("m:sSup"): parse_s_sup,
+        qn("m:sSub"): parse_s_sub,
+        qn("m:sSubSup"): parse_s_sub_sup,
+        qn("m:sPre"): parse_s_pre,
+        qn("m:t"): parse_t,
+        qn("m:rad"): parse_rad,
+        qn("m:nary"): parse_nary,
+        qn("m:eqArr"): parse_eq_arr,
+        qn("m:func"): parse_func,
+        qn("m:m"): parse_m,
+        qn("m:mr"): parse_mr,
+    }

lightrag/parser/docx/omml/utils.py

@@ -0,0 +1,40 @@
+"""
+Utility functions to extract text from the supported mathematical equations from xml tags and
+convert them into LaTeX
+"""
+
+from .cleaners import clean_exp
+
+ns_map = {
+    "w": "http://schemas.openxmlformats.org/wordprocessingml/2006/main",
+    "m": "http://schemas.openxmlformats.org/officeDocument/2006/math",
+}
+
+
+def linear_expression(tag):
+    """
+    Just returns the text contained in the given tag while setting docxlatex_skip_iteration flags
+    for all its children.
+    :param tag:defusedxml.Element - An xml element which contains a math equation in linear form
+    :return text:str - The equation in valid LaTeX syntax
+    """
+    text = ""
+    for child in tag.iter():
+        child.set("docxlatex_skip_iteration", True)
+        text += child.text if child.text is not None else ""
+    text = clean_exp(text)
+    return text
+
+
+def qn(tag):
+    """
+    A utility function to turn a namespace
+    prefixed tag name into a Clark-notation qualified tag name for lxml. For
+    example, qn('m:oMath') returns '{http://schemas.openxmlformats.org/officeDocument/2006/math}oMath'
+
+    :param tag:str - A namespace-prefixed tag name
+    :return qn:str - A Clark-notation qualified name tag for lxml.
+    """
+    prefix, tag_root = tag.split(":")
+    uri = ns_map[prefix]
+    return "{{{}}}{}".format(uri, tag_root)

lightrag/parser/docx/parse_document.py

@@ -0,0 +1,932 @@
+#!/usr/bin/env python3
+"""
+ABOUTME: Parses DOCX documents into text blocks using python-docx
+ABOUTME: Extracts automatic numbering, splits by headings, converts tables to JSON
+"""
+
+import json
+import sys
+
+try:
+    from docx import Document
+    from docx.opc.exceptions import PackageNotFoundError
+except ImportError as exc:
+    # Raise instead of sys.exit: this module is imported in-process by the
+    # gunicorn/uvicorn worker, where a SystemExit would tear down the whole
+    # worker rather than surfacing a normal, catchable error.
+    raise ImportError(
+        "python-docx not installed. Run: pip install python-docx"
+    ) from exc
+
+from lightrag.parser._markdown import (
+    render_heading_line,
+    strip_heading_markdown_prefix,
+)
+from .numbering_resolver import NumberingResolver
+from .table_extractor import TableExtractor
+from .drawing_image_extractor import (
+    DrawingExtractionContext,
+    extract_drawing_placeholder_from_element,
+    extract_vml_image_placeholder_from_element,
+)
+
+
+# Constants for content validation (character-based for UI/display)
+MAX_HEADING_LENGTH = 200  # Maximum heading length in characters (UI constraint)
+
+# OOXML tracked-change/comment tags whose subtree must be dropped so we only
+# surface the *final* revised text. w:ins / w:moveTo are kept via default
+# recursion so inserted/moved-in content survives.
+_SKIP_REVISION_TAGS = frozenset({"del", "moveFrom"})
+_SKIP_COMMENT_TAGS = frozenset(
+    {"commentRangeStart", "commentRangeEnd", "commentReference", "annotationRef"}
+)
+_SKIP_PARAGRAPH_TAGS = _SKIP_REVISION_TAGS | _SKIP_COMMENT_TAGS
+
+
+class DocxContentError(ValueError):
+    """DOCX content violates a parsing constraint (heading/table/anchor limits).
+
+    Raised instead of calling ``sys.exit`` so the pipeline's per-document
+    ``except Exception`` handler marks just that document FAILED while the
+    gunicorn/uvicorn worker process keeps running. Subclasses ``ValueError``
+    (i.e. an ``Exception``, not ``BaseException``) so the existing pipeline
+    handlers catch it.
+    """
+
+
+def format_error(title: str, details: str, solution: str) -> str:
+    """
+    Build a friendly, formatted error message (title / details / SOLUTION).
+
+    Args:
+        title: Error title
+        details: Detailed error information
+        solution: Suggested solution steps
+
+    Returns:
+        str: The formatted multi-line message.
+    """
+    return (
+        "\n"
+        + "=" * 68
+        + f"\nERROR: {title}\n"
+        + "=" * 68
+        + f"\n\n{details}"
+        + "\n\nSOLUTION:\n"
+        + solution
+        + "\n\n"
+        + "=" * 68
+        + "\n"
+    )
+
+
+def print_error(title: str, details: str, solution: str):
+    """Print a friendly, formatted error message to stderr."""
+    print(format_error(title, details, solution), file=sys.stderr)
+
+
+def _diagnose_invalid_docx(file_path: str) -> tuple[str, str]:
+    """Diagnose why a ``.docx`` file is not a valid OOXML/ZIP package.
+
+    python-docx raises ``PackageNotFoundError("Package not found at '...'")``
+    both when the path is missing AND when the file exists but is not a valid
+    zip. By the time this runs the file has already been confirmed to exist
+    (the native worker validates ``p.exists()`` first), so the real cause is a
+    corrupt file or a non-DOCX payload wearing a ``.docx`` extension. Sniff the
+    magic bytes to name the actual format so the error message reflects the
+    real problem instead of an empty "not found".
+
+    Returns a ``(details, solution)`` tuple for :func:`format_error`. Reads only
+    the file header and never raises — any IO failure degrades to a generic
+    "cannot read" diagnosis.
+    """
+    import zipfile
+
+    convert_solution = (
+        "  1. Open the file in Microsoft Word or WPS\n"
+        '  2. Use "Save As" and choose "Word Document (*.docx)"\n'
+        "  3. Re-upload the converted .docx to LightRAG"
+    )
+
+    try:
+        with open(file_path, "rb") as f:
+            head = f.read(8)
+    except OSError as exc:
+        return (
+            f"The file at '{file_path}' could not be read: {exc}",
+            "  1. Verify the file exists and is readable\n"
+            "  2. Re-upload it to LightRAG",
+        )
+
+    if not head:
+        return (
+            f"The file at '{file_path}' is empty (0 bytes). The upload was "
+            "likely truncated or the source file is corrupt.",
+            "  1. Check the original document opens correctly\n"
+            "  2. Re-upload a complete copy to LightRAG",
+        )
+
+    if head.startswith(b"\xd0\xcf\x11\xe0\xa1\xb1\x1a\xe1"):
+        # OLE2 Compound File — the legacy binary Word 97-2003 .doc format.
+        return (
+            f"The file at '{file_path}' is a legacy Word 97-2003 (.doc) "
+            "document saved with a .docx extension. The .doc binary format is "
+            "not a ZIP/OOXML package and cannot be parsed by the native engine.",
+            convert_solution,
+        )
+
+    if head.startswith(b"{\\rtf"):
+        return (
+            f"The file at '{file_path}' is an RTF document saved with a .docx "
+            "extension. RTF is not a ZIP/OOXML package.",
+            convert_solution,
+        )
+
+    if head.startswith(b"%PDF"):
+        return (
+            f"The file at '{file_path}' is a PDF saved with a .docx extension. "
+            "It is not a ZIP/OOXML package.",
+            "  1. Convert the PDF to .docx, or upload it through a PDF-capable "
+            "parser engine (e.g. mineru/docling)\n"
+            "  2. Re-upload to LightRAG",
+        )
+
+    stripped = head.lstrip()
+    if stripped.startswith(b"<"):
+        # <?xml ...>, <html ...>, or Word 2003 "<w:wordDocument>" flat XML.
+        return (
+            f"The file at '{file_path}' is an HTML or XML document saved with a "
+            ".docx extension, not a ZIP/OOXML package.",
+            convert_solution,
+        )
+
+    if head.startswith(b"PK\x03\x04") and not zipfile.is_zipfile(file_path):
+        # Has the ZIP local-file-header magic but the archive is unreadable.
+        return (
+            f"The file at '{file_path}' starts like a ZIP archive but is "
+            "truncated or corrupt, so it cannot be opened as a DOCX package.",
+            "  1. Check the original document opens correctly\n"
+            "  2. Re-upload a complete, uncorrupted copy to LightRAG",
+        )
+
+    return (
+        f"The file at '{file_path}' is not a valid DOCX (ZIP/OOXML) package. "
+        "It is either corrupt or a different file format saved with a .docx "
+        "extension.",
+        convert_solution,
+    )
+
+
+def truncate_heading(heading_text: str, para_id: str = None) -> str:
+    """
+    Truncate heading if it exceeds MAX_HEADING_LENGTH.
+
+    Args:
+        heading_text: The heading text to check
+        para_id: Optional paragraph ID for warning message
+
+    Returns:
+        str: Original heading if within limit, truncated heading with "..." if too long
+    """
+    if len(heading_text) > MAX_HEADING_LENGTH:
+        truncated = heading_text[: MAX_HEADING_LENGTH - 3] + "..."
+        location = f" (para_id: {para_id})" if para_id else ""
+        print(
+            f"Warning: Heading truncated (length {len(heading_text)} > max {MAX_HEADING_LENGTH}){location}: "
+            f'"{truncated}"',
+            file=sys.stderr,
+        )
+        return truncated
+    return heading_text
+
+
+def validate_heading_length(heading_text: str, para_id: str):
+    """
+    Validate that heading length does not exceed MAX_HEADING_LENGTH.
+
+    Args:
+        heading_text: The heading text to validate
+        para_id: The paragraph ID for error reporting
+
+    Raises:
+        DocxContentError: if heading exceeds maximum length
+    """
+    if len(heading_text) > MAX_HEADING_LENGTH:
+        preview = (
+            heading_text[:100] + "..." if len(heading_text) > 100 else heading_text
+        )
+        raise DocxContentError(
+            format_error(
+                f"Heading too long ({len(heading_text)} characters, max {MAX_HEADING_LENGTH})",
+                f"The following heading exceeds the maximum allowed length:\n\n{preview}\n\n"
+                f"Location(para_id): {para_id}\n"
+                f"Actual length: {len(heading_text)} characters",
+                "  1. Open the document in Microsoft Word\n"
+                f"  2. Shorten this heading to {MAX_HEADING_LENGTH} characters or less\n"
+                "  3. Re-upload it to LightRAG",
+            )
+        )
+
+
+def find_first_valid_para_id(para_ids: list) -> str | None:
+    """
+    Find the first valid paraId in a 2D array of paraIds.
+
+    Args:
+        para_ids: 2D list of paraIds from table cells
+
+    Returns:
+        First non-None paraId found, or None when every cell lacks a paraId.
+        Callers must tolerate ``None`` and treat it as a tracking gap rather
+        than a fatal error (legacy / non-Word docx authors omit ``w14:paraId``
+        attributes and we want to keep parsing).
+    """
+    for row in para_ids:
+        for para_id in row:
+            if para_id:
+                return para_id
+    return None
+
+
+def find_last_valid_para_id(para_ids: list) -> str | None:
+    """
+    Find the last valid paraId in a 2D array of paraIds.
+
+    Returns the last non-None paraId, falling back to the first valid one
+    when reverse-iteration does not yield anything (single-paraId tables),
+    and finally ``None`` when every cell lacks a paraId.
+    """
+    for row in reversed(para_ids):
+        for para_id in reversed(row):
+            if para_id:
+                return para_id
+
+    return find_first_valid_para_id(para_ids)
+
+
+def _table_has_any_paraid(para_ids: list) -> bool:
+    """True when at least one cell in the 2D paraId grid carries an id."""
+    return find_first_valid_para_id(para_ids) is not None
+
+
+def extract_para_id(para_element) -> str:
+    """
+    Extract w14:paraId attribute from paragraph element.
+
+    Args:
+        para_element: lxml paragraph element
+
+    Returns:
+        8-character hex paraId, or ``None`` when the paragraph carries no
+        ``w14:paraId`` attribute (legacy / non-Word docx authors). Callers
+        propagate the ``None`` upward — the LightRAG adapter counts these
+        and surfaces a single warning per document.
+    """
+    return para_element.get(
+        "{http://schemas.microsoft.com/office/word/2010/wordml}paraId"
+    )
+
+
+def parse_styles_outline_levels(docx_path: str) -> dict:
+    """
+    Parse styles.xml to extract outlineLvl definitions for each style,
+    following style inheritance chain (basedOn).
+
+    Args:
+        docx_path: Path to DOCX file
+
+    Returns:
+        dict: styleId -> outlineLvl (0-8 for headings, 9 for body text)
+    """
+    import zipfile
+
+    try:
+        from defusedxml import ElementTree as ET
+    except ImportError:
+        from xml.etree import ElementTree as ET
+
+    styles_outline = {}  # styleId -> outlineLvl (directly defined)
+    style_based_on = {}  # styleId -> parent styleId
+
+    try:
+        with zipfile.ZipFile(docx_path, "r") as zf:
+            if "word/styles.xml" not in zf.namelist():
+                return styles_outline
+
+            tree = ET.parse(zf.open("word/styles.xml"))
+            root = tree.getroot()
+
+            ns = "http://schemas.openxmlformats.org/wordprocessingml/2006/main"
+
+            # First pass: collect outlineLvl and basedOn for all styles
+            for style in root.findall(f".//{{{ns}}}style"):
+                style_id = style.get(f"{{{ns}}}styleId")
+                if not style_id:
+                    continue
+
+                # Check for basedOn (style inheritance)
+                based_on = style.find(f"{{{ns}}}basedOn")
+                if based_on is not None:
+                    parent_id = based_on.get(f"{{{ns}}}val")
+                    if parent_id:
+                        style_based_on[style_id] = parent_id
+
+                # Check for outlineLvl in style's pPr
+                pPr = style.find(f"{{{ns}}}pPr")
+                if pPr is not None:
+                    outline_lvl_elem = pPr.find(f"{{{ns}}}outlineLvl")
+                    if outline_lvl_elem is not None:
+                        level = int(outline_lvl_elem.get(f"{{{ns}}}val"))
+                        styles_outline[style_id] = level
+
+            # Second pass: resolve inheritance chain for styles without direct outlineLvl
+            def get_outline_level(style_id: str, visited: set = None) -> int:
+                if visited is None:
+                    visited = set()
+                if style_id in visited:
+                    return None  # Prevent circular references
+                visited.add(style_id)
+
+                # If this style directly defines outlineLvl, return it
+                if style_id in styles_outline:
+                    return styles_outline[style_id]
+
+                # Otherwise check parent style
+                if style_id in style_based_on:
+                    parent_id = style_based_on[style_id]
+                    return get_outline_level(parent_id, visited)
+
+                return None
+
+            # Fill in missing outlineLvl from inheritance chain
+            all_style_ids = set(styles_outline.keys()) | set(style_based_on.keys())
+            for style_id in all_style_ids:
+                if style_id not in styles_outline:
+                    level = get_outline_level(style_id)
+                    if level is not None:
+                        styles_outline[style_id] = level
+    except Exception:
+        # Silently ignore parsing errors
+        pass
+
+    return styles_outline
+
+
+def get_heading_level(para_element, styles_outline_map: dict) -> int:
+    """
+    Get heading level from paragraph, checking both direct format and style.
+
+    Priority: paragraph outlineLvl > style outlineLvl
+
+    Args:
+        para_element: lxml paragraph element
+        styles_outline_map: dict of styleId -> outlineLvl from styles.xml
+
+    Returns:
+        int: 0-8 for heading levels (0=level 1, 1=level 2, etc.), None for non-heading
+    """
+    # 1. Check paragraph direct format
+    pPr = para_element.find(
+        "{http://schemas.openxmlformats.org/wordprocessingml/2006/main}pPr"
+    )
+    if pPr is not None:
+        outline_elem = pPr.find(
+            "{http://schemas.openxmlformats.org/wordprocessingml/2006/main}outlineLvl"
+        )
+        if outline_elem is not None:
+            level = int(
+                outline_elem.get(
+                    "{http://schemas.openxmlformats.org/wordprocessingml/2006/main}val"
+                )
+            )
+            # Only 0-8 are true heading levels (9 is body text)
+            if level < 9:
+                return level
+            else:
+                return None  # Level 9 is body text
+
+    # 2. Check style definition's outlineLvl
+    if pPr is not None:
+        pStyle_elem = pPr.find(
+            "{http://schemas.openxmlformats.org/wordprocessingml/2006/main}pStyle"
+        )
+        if pStyle_elem is not None:
+            style_id = pStyle_elem.get(
+                "{http://schemas.openxmlformats.org/wordprocessingml/2006/main}val"
+            )
+            if style_id and style_id in styles_outline_map:
+                level = styles_outline_map[style_id]
+                if level < 9:
+                    return level
+                else:
+                    return None
+
+    return None
+
+
+def extract_text_from_run(
+    run,
+    ns: dict,
+    drawing_context: DrawingExtractionContext = None,
+) -> str:
+    """
+    Extract text from a run element, preserving superscript/subscript with markup.
+
+    Converts Word formatting to HTML-like tags:
+    - Superscript: <sup>text</sup>
+    - Subscript: <sub>text</sub>
+    - Normal text: unchanged
+
+    Args:
+        run: lxml run element (w:r)
+        ns: XML namespace dictionary
+
+    Returns:
+        Text string with <sup>/<sub> markup for formatted portions
+    """
+    text = ""
+
+    # Check for vertAlign in rPr (superscript/subscript)
+    vert_align = None
+    rPr = run.find("w:rPr", ns)
+    if rPr is not None:
+        vert_elem = rPr.find("w:vertAlign", ns)
+        if vert_elem is not None:
+            vert_align = vert_elem.get(
+                "{http://schemas.openxmlformats.org/wordprocessingml/2006/main}val"
+            )
+
+    # Extract text content from run children
+    for child in run:
+        tag = child.tag.split("}")[-1]  # Remove namespace
+        if tag == "t" and child.text:
+            text += child.text
+        elif tag == "tab":
+            text += "\t"
+        elif tag == "br":
+            # Handle line breaks - textWrapping or no type = soft line break
+            br_type = child.get(
+                "{http://schemas.openxmlformats.org/wordprocessingml/2006/main}type"
+            )
+            if br_type in (None, "textWrapping"):
+                text += "\n"
+            # Skip page and column breaks (layout elements)
+        elif tag == "drawing":
+            text += extract_drawing_placeholder_from_element(
+                child,
+                context=drawing_context,
+                include_extended_attrs=True,
+            )
+        elif tag in ("pict", "object"):
+            text += extract_vml_image_placeholder_from_element(
+                child,
+                context=drawing_context,
+                include_extended_attrs=True,
+            )
+
+    # Apply superscript/subscript markup if needed
+    if text and vert_align == "superscript":
+        return f"<sup>{text}</sup>"
+    elif text and vert_align == "subscript":
+        return f"<sub>{text}</sub>"
+
+    return text
+
+
+def extract_paragraph_content(
+    element,
+    ns,
+    drawing_context: DrawingExtractionContext = None,
+) -> str:
+    """
+    Extract text and equations from a paragraph element in document order.
+
+    Handles w:r (text runs), m:oMath (inline equations), and m:oMathPara
+    (block equations). Recurses into container elements (e.g., w:hyperlink,
+    w:ins, w:sdt, w:fldSimple, w:smartTag) to avoid dropping content.
+
+    Args:
+        element: lxml paragraph element (w:p)
+        ns: XML namespace dictionary
+
+    Returns:
+        Text string with equations wrapped in <equation> tags
+    """
+    parts = []
+
+    def append_from(node) -> None:
+        tag = node.tag.split("}")[-1]
+        # Drop tracked-change deletions (w:del/w:moveFrom) and comment markers
+        # (w:commentRangeStart/End, w:commentReference, w:annotationRef) so the
+        # output only contains the final revised text without annotation glyphs.
+        if tag in _SKIP_PARAGRAPH_TAGS:
+            return
+        if tag == "r":
+            parts.append(
+                extract_text_from_run(node, ns, drawing_context=drawing_context)
+            )
+            return
+        if tag == "oMath":
+            from .omml import convert_omml_to_latex
+
+            latex = convert_omml_to_latex(node)
+            if latex:
+                parts.append(f"<equation>{latex}</equation>")
+            return
+        if tag == "oMathPara":
+            from .omml import convert_omml_to_latex
+
+            for omath in node:
+                if omath.tag.split("}")[-1] == "oMath":
+                    latex = convert_omml_to_latex(omath)
+                    if latex:
+                        parts.append(f"<equation>{latex}</equation>")
+            return
+        for child in node:
+            append_from(child)
+
+    for child in element:
+        append_from(child)
+
+    return "".join(parts)
+
+
+def _is_table_empty(rows: list) -> bool:
+    """Return True iff every cell in ``rows`` is whitespace-only."""
+    return all(not (cell or "").strip() for row in rows for cell in row)
+
+
+def _collect_table_headers(paragraphs: list) -> list:
+    """Collect per-table cross-page header rows from ``is_table`` paragraphs.
+
+    The returned list is aligned 1:1 with the order of ``<table>`` placeholder
+    tags emitted into the block's content; entries are either the list of
+    header rows captured from ``w:tblHeader`` or ``None`` when the table has
+    no cross-page repeating header.
+    """
+    return [p.get("_table_header") for p in paragraphs if p.get("is_table")]
+
+
+def _build_unsplit_block(
+    heading: str, paragraphs: list, parent_headings: list, level: int
+) -> dict:
+    """Build a single block from paragraphs without size-based splitting."""
+    last_para = paragraphs[-1]
+    block = {
+        "uuid": paragraphs[0]["para_id"],
+        "uuid_end": last_para.get("para_id_end") or last_para.get("para_id"),
+        "heading": heading,
+        "content": "\n".join(p["text"] for p in paragraphs),
+        "type": "text",
+        "parent_headings": parent_headings,
+        "level": level,
+    }
+    table_headers = _collect_table_headers(paragraphs)
+    if table_headers:
+        block["table_headers"] = table_headers
+    return block
+
+
+def _flush_current_block(
+    blocks: list,
+    heading: str,
+    paragraphs: list,
+    parent_headings: list,
+    level: int,
+) -> None:
+    """Flush accumulated paragraphs into a single heading-scoped block.
+
+    The native parser performs only heading-driven structural splitting; block
+    sizing (long-block anchor splitting, table row splitting, small-block
+    merging) is the downstream paragraph-semantic chunker's responsibility.
+    """
+    if not paragraphs:
+        return
+
+    blocks.append(_build_unsplit_block(heading, paragraphs, parent_headings, level))
+
+
+def extract_docx_blocks(
+    file_path: str,
+    drawing_context: DrawingExtractionContext = None,
+    parse_warnings: dict | None = None,
+    parse_metadata: dict | None = None,
+) -> list:
+    """
+    Extract heading-scoped text blocks from a DOCX file.
+
+    Uses python-docx with a custom numbering resolver to:
+    1. Capture automatic numbering (list labels)
+    2. Split the document into one block per heading (structural splitting)
+    3. Convert tables to JSON (2D array) and emit them as <table> placeholders
+    4. Preserve superscript/subscript formatting with <sup>/<sub> markup
+
+    Block sizing — long-block anchor splitting, table row splitting, and
+    small-block merging — is intentionally NOT done here; it is the downstream
+    paragraph-semantic chunker's responsibility. Blocks emitted here may
+    therefore be arbitrarily large.
+
+    Args:
+        file_path: Path to the DOCX file
+        parse_warnings: Optional out-dict that this function mutates with
+            non-fatal warnings observed during parsing. Currently used for
+            ``missing_paraid_count`` — incremented once per body-level
+            paragraph (heading or text) that lacks a ``w14:paraId`` and once
+            per table whose every cell lacks one. Callers (the LightRAG
+            adapter / debug CLI) read this to surface a one-line warning per
+            document instead of crashing.
+        parse_metadata: Optional out-dict that this function mutates with
+            document-level metadata derived during parsing. Currently used
+            for ``first_heading`` — the text of the first heading encountered
+            in document order (regardless of level). Used by the LightRAG
+            adapter to populate ``meta.doc_title`` in ``.blocks.jsonl``.
+
+    Returns:
+        List of block dictionaries with heading, content, type, and metadata
+    """
+    try:
+        doc = Document(file_path)
+    except PackageNotFoundError as exc:
+        # python-docx surfaces a misleading "Package not found at '...'" for any
+        # file it cannot open as a ZIP/OOXML package — including files that
+        # exist but are corrupt or a different format wearing a .docx extension.
+        # Diagnose the real cause from the magic bytes and raise a DocxContentError
+        # (a ValueError) so the pipeline's per-document handler marks just this
+        # document FAILED with an accurate, actionable message.
+        details, solution = _diagnose_invalid_docx(file_path)
+        raise DocxContentError(
+            format_error("File is not a valid DOCX document", details, solution)
+        ) from exc
+    resolver = NumberingResolver(file_path)
+    styles_outline = parse_styles_outline_levels(file_path)
+
+    blocks = []
+    current_heading = "Preface/Uncategorized"
+    current_heading_level = 1  # Default level for "Preface/Uncategorized"
+    current_heading_stack = {}  # {level: heading_text} - Use dict to correctly track heading hierarchy
+    current_parent_headings = []  # Parent headings for current block
+    current_paragraphs = []  # Track paragraphs with metadata for splitting
+    first_heading_recorded = (
+        False  # Track whether the document's first heading has been captured
+    )
+
+    # Iterate through document body elements (paragraphs and tables)
+    body = doc._element.body
+
+    for element in body:
+        tag = element.tag.split("}")[-1]  # Remove namespace
+
+        if tag == "sectPr":  # Document-level section break
+            resolver.reset_tracking_state()
+            continue
+
+        if tag == "p":  # Paragraph
+            # Get paragraph text with superscript/subscript markup and equations
+            para_text = ""
+            ns = {
+                "w": "http://schemas.openxmlformats.org/wordprocessingml/2006/main",
+                "wp": "http://schemas.openxmlformats.org/drawingml/2006/wordprocessingDrawing",
+                "m": "http://schemas.openxmlformats.org/officeDocument/2006/math",
+            }
+            para_text = extract_paragraph_content(
+                element,
+                ns,
+                drawing_context=drawing_context,
+            )
+
+            para_text = para_text.strip()
+            if not para_text:
+                continue
+
+            # Get numbering label using our resolver
+            label = resolver.get_label(element)
+            full_text = f"{label} {para_text}".strip() if label else para_text
+
+            # Check if this is a heading using the new function
+            outline_level = get_heading_level(element, styles_outline)
+
+            # A "heading" longer than MAX_HEADING_LENGTH is not a real heading.
+            # The common cause (WPS/Word): the author set an outline level on a
+            # paragraph but typed the body with soft line breaks (Shift+Enter →
+            # <w:br/> → '\n') instead of starting a new paragraph, so heading
+            # text + body live in one <w:p>. Split at the first soft break: the
+            # first line stays the heading, the remainder becomes body text. If
+            # there is no usable soft break (a genuine single-line over-long
+            # heading), demote the whole paragraph to body text. Either way we
+            # avoid crashing via validate_heading_length() and never drop content.
+            demoted_body_text = None
+            if outline_level is not None and len(full_text) > MAX_HEADING_LENGTH:
+                head, sep, rest = full_text.partition("\n")
+                if sep and len(head) <= MAX_HEADING_LENGTH:
+                    full_text = head
+                    demoted_body_text = rest.strip() or None
+                    if parse_warnings is not None:
+                        parse_warnings["heading_softbreak_split_count"] = (
+                            parse_warnings.get("heading_softbreak_split_count", 0) + 1
+                        )
+                    print(
+                        f"Warning: heading paragraph exceeded {MAX_HEADING_LENGTH} "
+                        "chars; split at soft line break — kept first line as "
+                        "heading, rest as body.",
+                        file=sys.stderr,
+                    )
+                else:
+                    outline_level = None
+                    if parse_warnings is not None:
+                        parse_warnings["demoted_oversize_heading_count"] = (
+                            parse_warnings.get("demoted_oversize_heading_count", 0) + 1
+                        )
+                    print(
+                        f"Warning: paragraph has outline level but is "
+                        f"{len(full_text)} chars (> {MAX_HEADING_LENGTH}); treating "
+                        "as body text, not a heading.",
+                        file=sys.stderr,
+                    )
+
+            if outline_level is not None:
+                # This is a heading (outline level 0-8)
+                # Convert 0-based to 1-based level
+                level = outline_level + 1
+
+                # Extract paraId for this heading
+                heading_para_id = extract_para_id(element)
+                if parse_warnings is not None and not heading_para_id:
+                    parse_warnings["missing_paraid_count"] = (
+                        parse_warnings.get("missing_paraid_count", 0) + 1
+                    )
+
+                # Validate heading length
+                validate_heading_length(full_text, heading_para_id)
+
+                # Truncate heading if needed before storing
+                truncated_text = truncate_heading(full_text, heading_para_id)
+                clean_heading_text = strip_heading_markdown_prefix(truncated_text)
+
+                # Record the document's first heading (any level) for meta.doc_title.
+                if not first_heading_recorded:
+                    if parse_metadata is not None:
+                        parse_metadata["first_heading"] = clean_heading_text
+                    first_heading_recorded = True
+
+                # Every recognized heading starts its own block. Always flush the
+                # accumulated paragraphs so a heading with no body becomes a
+                # standalone block whose content is just the heading text,
+                # instead of being folded into the next heading's block.
+                if current_paragraphs:
+                    _flush_current_block(
+                        blocks,
+                        current_heading,
+                        current_paragraphs,
+                        current_parent_headings,
+                        current_heading_level,
+                    )
+
+                    # Reset for new block
+                    current_paragraphs = []
+
+                # Add heading to current_paragraphs. The content line gets
+                # a markdown ``#`` prefix (capped at 6) via
+                # render_heading_line; ``clean_heading_text`` is kept
+                # for the heading field / stack / parent_headings below.
+                current_paragraphs.append(
+                    {
+                        "text": render_heading_line(level, truncated_text),
+                        "para_id": heading_para_id,
+                        "is_table": False,
+                    }
+                )
+
+                # Update current_heading and parent_headings for the FIRST heading in a block
+                # (when current_paragraphs just had this heading added as its first element)
+                if len(current_paragraphs) == 1:
+                    current_heading = clean_heading_text
+                    current_heading_level = level  # Only set level when setting heading
+                    # Parent headings = all headings from levels strictly less than current level
+                    # Sort by level to maintain hierarchy order
+                    current_parent_headings = [
+                        current_heading_stack[lvl]
+                        for lvl in sorted(current_heading_stack.keys())
+                        if lvl < level
+                    ]
+
+                # Update heading stack: remove current level and all lower levels, then add current
+                current_heading_stack = {
+                    k: v for k, v in current_heading_stack.items() if k < level
+                }
+                current_heading_stack[level] = clean_heading_text
+
+                # Carry the body text that followed a soft break in an over-long
+                # heading paragraph as a regular body paragraph in the same block.
+                if demoted_body_text:
+                    current_paragraphs.append(
+                        {
+                            "text": demoted_body_text,
+                            "para_id": heading_para_id,
+                            "is_table": False,
+                        }
+                    )
+            else:
+                # Regular paragraph content
+                para_id = extract_para_id(element)
+                if parse_warnings is not None and not para_id:
+                    parse_warnings["missing_paraid_count"] = (
+                        parse_warnings.get("missing_paraid_count", 0) + 1
+                    )
+
+                # Store paragraph with metadata for potential splitting
+                current_paragraphs.append(
+                    {"text": full_text, "para_id": para_id, "is_table": False}
+                )
+
+            # Check for paragraph-level section break (after processing paragraph)
+            # sectPr in pPr means this paragraph ends a section
+            pPr = element.find(
+                "{http://schemas.openxmlformats.org/wordprocessingml/2006/main}pPr"
+            )
+            if pPr is not None:
+                sectPr = pPr.find(
+                    "{http://schemas.openxmlformats.org/wordprocessingml/2006/main}sectPr"
+                )
+                if sectPr is not None:
+                    # Section break after this paragraph - reset tracking
+                    resolver.reset_tracking_state()
+
+        elif tag == "tbl":  # Table
+            # Reset numbering tracking before table (table start boundary)
+            resolver.reset_tracking_state()
+
+            # Directly create Table object from XML element to avoid index mismatch
+            # (doc.tables may have different order due to nested tables)
+            from docx.table import Table
+
+            table = Table(element, doc)
+            table_metadata = TableExtractor.extract_with_metadata(
+                table,
+                numbering_resolver=resolver,
+                drawing_context=drawing_context,
+            )
+
+            table_rows = table_metadata["rows"]
+            para_ids = table_metadata["para_ids"]
+            para_ids_end = table_metadata["para_ids_end"]  # Last paraId in each cell
+            header_indices = table_metadata["header_indices"]
+
+            # Skip tables whose every cell is whitespace-only — otherwise an
+            # empty `<table>[[""]]</table>` placeholder would leak into block
+            # content and a useless IRTable would appear in tables.json.
+            if _is_table_empty(table_rows):
+                resolver.reset_tracking_state()
+                continue
+
+            # Count tables whose cells carry no w14:paraId. Legacy / non-Word
+            # docx authors omit these attributes; we no longer fail-fast, but
+            # the adapter surfaces a single warning so the user knows the edit
+            # range hints will be missing for these tables.
+            if parse_warnings is not None and not _table_has_any_paraid(para_ids):
+                parse_warnings["missing_paraid_count"] = (
+                    parse_warnings.get("missing_paraid_count", 0) + 1
+                )
+
+            # Convert table to JSON
+            table_json = json.dumps(table_rows, ensure_ascii=False)
+
+            # Extract cross-page repeating header rows (w:tblHeader) once per
+            # table so we can surface them to the sidecar via the block-level
+            # ``table_headers`` list.
+            header_rows = []
+            if header_indices:
+                header_rows = [
+                    table_rows[idx] for idx in header_indices if idx < len(table_rows)
+                ]
+            header_rows_or_none = header_rows if header_rows else None
+
+            # Emit the whole table as a single <table> placeholder. Token-based
+            # table row splitting is the downstream chunker's responsibility.
+            # Use first valid paraId from table, and last valid paraId (from
+            # para_ids_end) for uuid_end.
+            table_para_id = find_first_valid_para_id(para_ids)
+            table_para_id_end = find_last_valid_para_id(para_ids_end)
+            current_paragraphs.append(
+                {
+                    "text": f"<table>{table_json}</table>",
+                    "para_id": table_para_id,
+                    "para_id_end": table_para_id_end,  # Store end paraId for uuid_end calculation
+                    "is_table": True,
+                    "_table_header": header_rows_or_none,
+                }
+            )
+
+            # Reset numbering tracking after table (table end boundary)
+            resolver.reset_tracking_state()
+
+    # Save final block
+    _flush_current_block(
+        blocks,
+        current_heading,
+        current_paragraphs,
+        current_parent_headings,
+        current_heading_level,
+    )
+
+    return blocks

lightrag/parser/docx/parser.py

@@ -0,0 +1,96 @@
+"""Native DOCX engine adapter (implements NativeParserBase hooks)."""
+
+from __future__ import annotations
+
+from pathlib import Path
+from typing import TYPE_CHECKING, Any
+
+from lightrag.constants import PARSER_ENGINE_NATIVE
+from lightrag.parser.native_base import NativeParserBase
+from lightrag.utils import logger
+
+if TYPE_CHECKING:
+    from lightrag.sidecar.ir import IRDoc
+
+
+class NativeDocxParser(NativeParserBase):
+    """Native DOCX parser for LightRAG's production parsing path.
+
+    ``extract_docx_blocks`` performs only heading-driven structural splitting
+    (one block per DOCX heading). Block sizing is intentionally left to the
+    downstream paragraph-semantic chunker, so this parser emits the
+    one-heading-one-block sidecar contract that chunking consumes.
+    """
+
+    engine_name = PARSER_ENGINE_NATIVE
+    sidecar_path_style = "basename_only"  # legacy native docx convention
+    empty_content_label = "DOCX"
+
+    def validate_source(self, source: Path, file_path: str) -> None:
+        if not (
+            source.exists() and source.is_file() and source.suffix.lower() == ".docx"
+        ):
+            raise ValueError(
+                f"Native parser does not support pending file: {file_path}"
+            )
+
+    def extract(
+        self, source: Path, *, parsed_dir: Path, asset_dir: Path, base_name: str
+    ) -> tuple[list[dict[str, Any]], dict[str, Any], dict[str, Any]]:
+        """Extract heading-scoped DOCX blocks (sizing left to the chunker)."""
+
+        from lightrag.parser.docx.drawing_image_extractor import (
+            DrawingExtractionContext,
+            load_relationships,
+        )
+        from lightrag.parser.docx.parse_document import extract_docx_blocks
+
+        ctx = DrawingExtractionContext(
+            docx_path=source,
+            blocks_output_path=parsed_dir / f"{base_name}.blocks.jsonl",
+            export_dir_name=asset_dir.name,
+            export_dir_path=asset_dir,
+        )
+        load_relationships(ctx)
+        warnings: dict[str, Any] = {}
+        metadata: dict[str, Any] = {}
+        blocks = extract_docx_blocks(
+            str(source),
+            drawing_context=ctx,
+            parse_warnings=warnings,
+            parse_metadata=metadata,
+        )
+        return blocks, warnings, metadata
+
+    def build_ir(
+        self,
+        blocks: list[dict[str, Any]],
+        *,
+        document_name: str,
+        asset_dir_name: str,
+        metadata: dict[str, Any],
+    ) -> "IRDoc":
+        from lightrag.parser.docx.ir_builder import NativeDocxIRBuilder
+
+        return NativeDocxIRBuilder().normalize(
+            blocks,
+            document_name=document_name,
+            asset_dir_name=asset_dir_name,
+            parse_metadata=metadata,
+        )
+
+    def surface_warnings(
+        self, warnings: dict[str, Any], source: Path
+    ) -> dict[str, Any] | None:
+        missing = int(warnings.get("missing_paraid_count", 0) or 0)
+        if missing > 0:
+            # Surface once per document; affected blocks emit
+            # ``positions: [{"type": "paraid", "range": null}]``.
+            logger.warning(
+                "[parse_native] %s: %d paragraphs lack paraId; "
+                "Re-saving file in Word 2013+ to regenerate ids.",
+                source.name,
+                missing,
+            )
+            return {"missing_paraid_count": missing}
+        return None

lightrag/parser/docx/table_extractor.py

@@ -0,0 +1,419 @@
+#!/usr/bin/env python3
+"""
+ABOUTME: Extracts tables from DOCX with proper merged cell handling
+ABOUTME: Vertically merged cells: content repeated in all rows with shared paraId
+ABOUTME: Horizontally merged cells: content in first cell only
+ABOUTME: Preserves superscript/subscript formatting with <sup>/<sub> markup
+"""
+
+from docx.table import Table
+from docx.oxml.ns import qn
+from typing import List
+
+from .drawing_image_extractor import (
+    DrawingExtractionContext,
+    extract_drawing_placeholder_from_element,
+    extract_vml_image_placeholder_from_element,
+)
+
+# Keep in sync with parse_document._SKIP_PARAGRAPH_TAGS — duplicated here to
+# avoid a circular import between parse_document and table_extractor.
+_SKIP_PARAGRAPH_TAGS = frozenset(
+    {
+        "del",
+        "moveFrom",
+        "commentRangeStart",
+        "commentRangeEnd",
+        "commentReference",
+        "annotationRef",
+    }
+)
+
+
+def extract_text_from_run_table(
+    run_elem,
+    qn_func,
+    drawing_context: DrawingExtractionContext = None,
+) -> str:
+    """
+    Extract text from a run element in table cell, preserving superscript/subscript with markup.
+
+    Converts Word formatting to HTML-like tags:
+    - Superscript: <sup>text</sup>
+    - Subscript: <sub>text</sub>
+    - Normal text: unchanged
+
+    Args:
+        run_elem: lxml run element (w:r)
+        qn_func: qn function for namespace handling
+
+    Returns:
+        Text string with <sup>/<sub> markup for formatted portions
+    """
+    text = ""
+
+    # Check for vertAlign in rPr (superscript/subscript)
+    vert_align = None
+    rPr = run_elem.find(qn_func("w:rPr"))
+    if rPr is not None:
+        vert_elem = rPr.find(qn_func("w:vertAlign"))
+        if vert_elem is not None:
+            vert_align = vert_elem.get(qn_func("w:val"))
+
+    # Extract text content from run children
+    for child in run_elem:
+        tag = child.tag.split("}")[-1]  # Remove namespace
+        if tag == "t" and child.text:
+            text += child.text
+        elif tag == "tab":
+            text += "\t"
+        elif tag == "br":
+            # Handle line breaks - textWrapping or no type = soft line break
+            br_type = child.get(qn_func("w:type"))
+            if br_type in (None, "textWrapping"):
+                text += "\n"
+            # Skip page and column breaks (layout elements)
+        elif tag == "drawing":
+            text += extract_drawing_placeholder_from_element(
+                child,
+                context=drawing_context,
+                include_extended_attrs=True,
+            )
+        elif tag in ("pict", "object"):
+            text += extract_vml_image_placeholder_from_element(
+                child,
+                context=drawing_context,
+                include_extended_attrs=True,
+            )
+
+    # Apply superscript/subscript markup if needed
+    if text and vert_align == "superscript":
+        return f"<sup>{text}</sup>"
+    elif text and vert_align == "subscript":
+        return f"<sub>{text}</sub>"
+
+    return text
+
+
+def extract_paragraph_content_table(
+    para_elem,
+    qn_func,
+    drawing_context: DrawingExtractionContext = None,
+) -> str:
+    """
+    Extract text and equations from a table cell paragraph in document order.
+
+    Handles w:r (text runs), m:oMath (inline equations), and m:oMathPara
+    (block equations). Recurses into container elements (e.g., w:hyperlink,
+    w:ins, w:sdt, w:fldSimple, w:smartTag) to avoid dropping content.
+
+    Args:
+        para_elem: lxml paragraph element (w:p)
+        qn_func: qn function for namespace handling
+
+    Returns:
+        Text string with equations wrapped in <equation> tags
+    """
+    parts = []
+
+    def append_from(node) -> None:
+        tag = node.tag.split("}")[-1]
+        # Drop tracked-change deletions (w:del/w:moveFrom) and comment markers
+        # (w:commentRangeStart/End, w:commentReference, w:annotationRef) so the
+        # output only contains the final revised text without annotation glyphs.
+        if tag in _SKIP_PARAGRAPH_TAGS:
+            return
+        if tag == "r":
+            parts.append(
+                extract_text_from_run_table(
+                    node,
+                    qn_func,
+                    drawing_context=drawing_context,
+                )
+            )
+            return
+        if tag == "oMath":
+            from omml import convert_omml_to_latex
+
+            latex = convert_omml_to_latex(node)
+            if latex:
+                parts.append(f"<equation>{latex}</equation>")
+            return
+        if tag == "oMathPara":
+            from omml import convert_omml_to_latex
+
+            for omath in node:
+                if omath.tag.split("}")[-1] == "oMath":
+                    latex = convert_omml_to_latex(omath)
+                    if latex:
+                        parts.append(f"<equation>{latex}</equation>")
+            return
+        for child in node:
+            append_from(child)
+
+    for child in para_elem:
+        append_from(child)
+
+    return "".join(parts)
+
+
+class TableExtractor:
+    """
+    Extract table content handling merged cells correctly.
+
+    Merged cells in DOCX:
+    - Horizontal: w:gridSpan specifies how many columns cell spans
+    - Vertical: w:vMerge with val="restart" starts merge, subsequent cells continue
+
+    Output format:
+    - 2D list of strings
+    - Vertically merged cells: content repeated in all rows, all rows use the same paraId (from start cell)
+    - Horizontally merged cells: content in left-most position only, other positions empty
+    """
+
+    @staticmethod
+    def extract(
+        table: Table,
+        numbering_resolver=None,
+        drawing_context: DrawingExtractionContext = None,
+    ) -> List[List[str]]:
+        """
+        Extract table to 2D string array.
+
+        Args:
+            table: python-docx Table object
+            numbering_resolver: Optional NumberingResolver for extracting numbering
+
+        Returns:
+            List of rows, each row is list of cell text strings
+        """
+        result = TableExtractor.extract_with_metadata(
+            table,
+            numbering_resolver=numbering_resolver,
+            drawing_context=drawing_context,
+        )
+        return result["rows"]
+
+    @staticmethod
+    def extract_with_metadata(
+        table: Table,
+        numbering_resolver=None,
+        drawing_context: DrawingExtractionContext = None,
+    ) -> dict:
+        """
+        Extract table to 2D string array with metadata (paraIds, header info).
+
+        Vertical merge behavior:
+        - All rows in a vertically merged region share the same content
+        - All rows use the paraId from the merge start cell (for precise edit targeting)
+
+        Args:
+            table: python-docx Table object
+            numbering_resolver: Optional NumberingResolver for extracting numbering
+
+        Returns:
+            Dict with:
+            - rows: 2D list of cell text strings
+            - para_ids: 2D list of paraIds (first paraId in each cell, or None)
+                        For vertically merged cells, all rows share the start cell's paraId
+            - para_ids_end: 2D list of paraIds (last paraId in each cell, or None)
+                            For vertically merged cells, all rows share the start cell's paraId
+            - header_indices: List of row indices marked as table headers
+        """
+        tbl = table._tbl
+
+        # Get number of columns from tblGrid
+        tbl_grid = tbl.find(qn("w:tblGrid"))
+        num_cols = 0
+        if tbl_grid is not None:
+            num_cols = len(tbl_grid.findall(qn("w:gridCol")))
+
+        if num_cols == 0:
+            return {
+                "rows": [],
+                "para_ids": [],
+                "para_ids_end": [],
+                "header_indices": [],
+            }
+
+        # Detect header rows using w:tblHeader attribute
+        header_indices = []
+        for idx, tr in enumerate(tbl.findall(qn("w:tr"))):
+            trPr = tr.find(qn("w:trPr"))
+            if trPr is not None:
+                tbl_header = trPr.find(qn("w:tblHeader"))
+                if tbl_header is not None:
+                    header_indices.append(idx)
+
+        # Process each row by directly iterating <w:tr> elements
+        grid = []
+        para_ids_grid = []
+        para_ids_end_grid = []  # Track last paraId in each cell
+        vmerge_content = {}  # Track vertical merge by column: {col: {'text': str, 'para_id': str, 'para_id_end': str}}
+
+        for tr in tbl.findall(qn("w:tr")):
+            row_data = [""] * num_cols  # Pre-fill with empty strings
+            row_para_ids = [None] * num_cols  # Pre-fill with None
+            row_para_ids_end = [None] * num_cols  # Pre-fill with None for last paraId
+            grid_col = 0
+
+            # Iterate actual <w:tc> elements (each physical cell appears once)
+            for tc in tr.findall(qn("w:tc")):
+                # Reset numbering state when cell changes to prevent incorrect continuation
+                if numbering_resolver is not None:
+                    numbering_resolver.reset_tracking_state()
+
+                tcPr = tc.find(qn("w:tcPr"))
+
+                # Check gridSpan (horizontal merge)
+                grid_span = 1
+                if tcPr is not None:
+                    gs = tcPr.find(qn("w:gridSpan"))
+                    if gs is not None:
+                        grid_span = int(gs.get(qn("w:val")))
+
+                # Check vMerge (vertical merge)
+                vmerge_elem = None
+                vmerge_val = None
+                if tcPr is not None:
+                    vmerge_elem = tcPr.find(qn("w:vMerge"))
+                    if vmerge_elem is not None:
+                        vmerge_val = vmerge_elem.get(
+                            qn("w:val")
+                        )  # 'restart' or None (means 'continue')
+
+                # Determine vMerge status
+                is_vmerge_restart = vmerge_elem is not None and vmerge_val == "restart"
+                is_vmerge_continue = vmerge_elem is not None and vmerge_val in (
+                    None,
+                    "continue",
+                )
+                is_normal_cell = vmerge_elem is None
+
+                cell_text = ""
+                cell_para_id = None
+                cell_para_id_end = None  # Track last paraId in cell
+
+                # Handle different vMerge cases
+                if is_vmerge_restart or is_normal_cell:
+                    # Extract content for restart or normal cells
+                    # Get cell text with numbering support and format preservation
+                    if numbering_resolver is not None:
+                        # Extract text with numbering labels and superscript/subscript markup
+                        cell_paragraphs = []
+                        for para_elem in tc.findall(qn("w:p")):
+                            # Capture paraId from each paragraph
+                            para_id_attr = para_elem.get(
+                                "{http://schemas.microsoft.com/office/word/2010/wordml}paraId"
+                            )
+                            if para_id_attr:
+                                if cell_para_id is None:
+                                    cell_para_id = para_id_attr  # First paraId
+                                cell_para_id_end = (
+                                    para_id_attr  # Always update to get last
+                                )
+
+                            # Get text content with format preservation (superscript/subscript/equations)
+                            para_text = extract_paragraph_content_table(
+                                para_elem,
+                                qn,
+                                drawing_context=drawing_context,
+                            )
+
+                            # Get numbering label
+                            label = numbering_resolver.get_label(para_elem)
+
+                            # Combine label and text
+                            if label:
+                                full_text = f"{label} {para_text}".strip()
+                            else:
+                                full_text = para_text.strip()
+
+                            if full_text:
+                                cell_paragraphs.append(full_text)
+
+                        cell_text = "\n".join(cell_paragraphs).replace("\x07", "")
+                    else:
+                        # Fallback to simple text extraction with format preservation
+                        # Cannot use cell.text here, must extract from XML
+                        para_texts = []
+                        for para_elem in tc.findall(qn("w:p")):
+                            # Capture paraId from each paragraph
+                            para_id_attr = para_elem.get(
+                                "{http://schemas.microsoft.com/office/word/2010/wordml}paraId"
+                            )
+                            if para_id_attr:
+                                if cell_para_id is None:
+                                    cell_para_id = para_id_attr  # First paraId
+                                cell_para_id_end = (
+                                    para_id_attr  # Always update to get last
+                                )
+
+                            # Extract text with format preservation (superscript/subscript/equations)
+                            para_text = extract_paragraph_content_table(
+                                para_elem,
+                                qn,
+                                drawing_context=drawing_context,
+                            )
+
+                            if para_text:
+                                para_texts.append(para_text.strip())
+                        cell_text = "\n".join(para_texts).replace("\x07", "")
+
+                    # Store content and paraIds for vMerge restart
+                    if is_vmerge_restart:
+                        vmerge_content[grid_col] = {
+                            "text": cell_text,
+                            "para_id": cell_para_id,
+                            "para_id_end": cell_para_id_end,
+                        }
+                    elif is_normal_cell:
+                        # For normal cells: if empty and we have active vMerge, copy all from start
+                        # If non-empty, this ends the vMerge region
+                        if not cell_text and grid_col in vmerge_content:
+                            # Empty cell in vMerge region - copy content and paraIds from start
+                            cell_text = vmerge_content[grid_col]["text"]
+                            cell_para_id = vmerge_content[grid_col]["para_id"]
+                            cell_para_id_end = vmerge_content[grid_col]["para_id_end"]
+                        elif cell_text:
+                            # Non-empty cell - this ends the vMerge for this column
+                            vmerge_content.pop(grid_col, None)
+
+                elif is_vmerge_continue:
+                    # Copy content and para_id from previous merge start
+                    # But extract actual para_id_end from this continue cell for range boundary
+                    if grid_col in vmerge_content:
+                        cell_text = vmerge_content[grid_col]["text"]
+                        cell_para_id = vmerge_content[grid_col][
+                            "para_id"
+                        ]  # Use restart's paraId for edit targeting
+
+                        # Extract actual paraId from this continue cell for uuid_end (range boundary)
+                        for para_elem in tc.findall(qn("w:p")):
+                            para_id_attr = para_elem.get(
+                                "{http://schemas.microsoft.com/office/word/2010/wordml}paraId"
+                            )
+                            if para_id_attr:
+                                cell_para_id_end = (
+                                    para_id_attr  # Use actual paraId for range boundary
+                                )
+
+                # Place content at starting grid position only
+                if grid_col < num_cols:
+                    row_data[grid_col] = cell_text
+                    row_para_ids[grid_col] = cell_para_id
+                    row_para_ids_end[grid_col] = cell_para_id_end
+
+                # Move grid position by gridSpan
+                grid_col += grid_span
+
+            grid.append(row_data)
+            para_ids_grid.append(row_para_ids)
+            para_ids_end_grid.append(row_para_ids_end)
+
+        return {
+            "rows": grid,
+            "para_ids": para_ids_grid,
+            "para_ids_end": para_ids_end_grid,
+            "header_indices": header_indices,
+        }

lightrag/parser/docx/utils.py

@@ -0,0 +1,791 @@
+#!/usr/bin/env python3
+"""
+ABOUTME: Shared token estimation utilities for audit scripts
+ABOUTME: XML sanitization helpers for document processing
+"""
+
+import json
+import os
+import re
+
+try:
+    from google import genai
+    from google.genai import types
+
+    HAS_GEMINI = True
+except ImportError:  # pragma: no cover - optional dependency
+    genai = None
+    types = None
+    HAS_GEMINI = False
+
+try:
+    import openai
+
+    HAS_OPENAI = True
+except ImportError:  # pragma: no cover - optional dependency
+    openai = None
+    HAS_OPENAI = False
+
+
+def estimate_tokens(text: str) -> int:
+    """
+    Estimate token count for LLM context management.
+
+    Uses a weighted formula based on character types:
+    - Chinese characters: ~0.75 tokens per character (subword tokenization)
+    - JSON structural characters (brackets, quotes, commas): ~1 tokens per character
+    - Other characters (English, numbers, symbols): ~0.4 tokens per character (~3 chars/token)
+
+    Includes 5% buffer and safety offset for special formatting and system prompt overhead.
+
+    Args:
+        text: Input text to estimate tokens for
+
+    Returns:
+        int: Estimated token count
+    """
+    if not text:
+        return 0
+
+    chinese_count = len(re.findall(r"[\u4e00-\u9fa5]", text))
+    json_chars_count = len(re.findall(r'[\[\]",{}]', text))
+    other_count = len(text) - chinese_count - json_chars_count
+
+    base_estimate = (
+        (chinese_count * 0.75) + (json_chars_count * 1) + (other_count * 0.4)
+    )
+    final_tokens = int(base_estimate * 1.05) + 2
+    return final_tokens
+
+
+def sanitize_xml_string(text: str) -> str:
+    """
+    Remove control characters that are illegal in XML 1.0.
+
+    XML 1.0 allows: #x9 (tab), #xA (LF), #xD (CR), and #x20-#xD7FF, #xE000-#xFFFD, #x10000-#x10FFFF
+    This function removes all other control characters (0x00-0x08, 0x0B, 0x0C, 0x0E-0x1F).
+
+    Args:
+        text: Text that may contain control characters
+
+    Returns:
+        Sanitized text safe for XML. Returns input unchanged if not a non-empty string.
+    """
+    if not text or not isinstance(text, str):
+        return text
+    # Build a translation table to remove illegal control characters
+    # Keep: \t (0x09), \n (0x0A), \r (0x0D)
+    # Remove: 0x00-0x08, 0x0B, 0x0C, 0x0E-0x1F
+    illegal_chars = "".join(chr(c) for c in range(0x20) if c not in (0x09, 0x0A, 0x0D))
+    return text.translate(str.maketrans("", "", illegal_chars))
+
+
+def is_vertex_ai_mode() -> bool:
+    """
+    Check if Vertex AI mode is enabled via environment variable.
+
+    Returns:
+        True if GOOGLE_GENAI_USE_VERTEXAI is set to 'true', False otherwise
+    """
+    return os.getenv("GOOGLE_GENAI_USE_VERTEXAI", "").lower() == "true"
+
+
+def create_gemini_client(use_async: bool = False):
+    """
+    Create Gemini client for AI Studio or Vertex AI.
+
+    Supports two modes:
+    - AI Studio (default): Uses GOOGLE_API_KEY for authentication
+    - Vertex AI: Uses ADC (GOOGLE_APPLICATION_CREDENTIALS or gcloud auth)
+
+    Environment variables for Vertex AI mode:
+    - GOOGLE_GENAI_USE_VERTEXAI: Set to 'true' to enable Vertex AI mode
+    - GOOGLE_CLOUD_PROJECT: Required GCP project ID
+    - GOOGLE_CLOUD_LOCATION: Optional region (default: us-central1)
+    - GOOGLE_VERTEX_BASE_URL: Optional custom API endpoint (for API gateway proxies)
+    - GOOGLE_APPLICATION_CREDENTIALS: Path to service account JSON (or use gcloud auth)
+
+    Args:
+        use_async: If True, return the async client (.aio), otherwise return sync client
+
+    Returns:
+        Gemini client instance (sync or async based on use_async parameter)
+
+    Raises:
+        ValueError: If required environment variables are not set
+    """
+    use_vertex = is_vertex_ai_mode()
+
+    if use_vertex:
+        # Vertex AI mode - uses ADC (GOOGLE_APPLICATION_CREDENTIALS or gcloud auth)
+        project = os.getenv("GOOGLE_CLOUD_PROJECT")
+        location = os.getenv("GOOGLE_CLOUD_LOCATION", "us-central1")
+        base_url = os.getenv("GOOGLE_VERTEX_BASE_URL")
+
+        if not project:
+            raise ValueError(
+                "GOOGLE_CLOUD_PROJECT is required for Vertex AI mode. "
+                "Set GOOGLE_GENAI_USE_VERTEXAI=false to use AI Studio mode instead."
+            )
+
+        # Build http_options only if custom base_url is specified
+        http_options = None
+        if base_url:
+            http_options = {"base_url": base_url}
+
+        # Note: ADC handles authentication automatically
+        # via GOOGLE_APPLICATION_CREDENTIALS env var or gcloud auth
+        client = genai.Client(
+            vertexai=True, project=project, location=location, http_options=http_options
+        )
+    else:
+        # AI Studio mode - requires API key
+        api_key = os.getenv("GOOGLE_API_KEY")
+        if not api_key:
+            raise ValueError(
+                "GOOGLE_API_KEY is required for AI Studio mode. "
+                "Set GOOGLE_GENAI_USE_VERTEXAI=true and configure GCP credentials for Vertex AI mode."
+            )
+
+        client = genai.Client(api_key=api_key)
+
+    # Return async or sync client based on parameter
+    return client.aio if use_async else client
+
+
+def get_gemini_provider_name() -> str:
+    """
+    Get the Gemini provider name based on current mode.
+
+    Returns:
+        Provider name string for display purposes
+    """
+    if is_vertex_ai_mode():
+        project = os.getenv("GOOGLE_CLOUD_PROJECT", "unknown")
+        location = os.getenv("GOOGLE_CLOUD_LOCATION", "us-central1")
+        return f"Google Gemini (Vertex AI: {project}/{location})"
+    return "Google Gemini (AI Studio)"
+
+
+def create_openai_client(use_async: bool = True):
+    """
+    Create OpenAI client with optional custom base URL.
+
+    Environment variables:
+    - OPENAI_API_KEY: Required API key
+    - OPENAI_BASE_URL: Optional custom API endpoint (for proxies, Azure, etc.)
+
+    Args:
+        use_async: If True, return AsyncOpenAI, otherwise return OpenAI
+
+    Returns:
+        OpenAI client instance (async or sync based on use_async parameter)
+
+    Raises:
+        ValueError: If OPENAI_API_KEY is not set
+    """
+    if not HAS_OPENAI:
+        raise ValueError("openai library is not installed.")
+    api_key = os.getenv("OPENAI_API_KEY")
+    if not api_key:
+        raise ValueError("OPENAI_API_KEY is required for OpenAI mode.")
+
+    base_url = os.getenv("OPENAI_BASE_URL")
+
+    if use_async:
+        return openai.AsyncOpenAI(base_url=base_url)
+    return openai.OpenAI(base_url=base_url)
+
+
+def get_openai_provider_name() -> str:
+    """
+    Get the OpenAI provider name, including custom endpoint if configured.
+
+    Returns:
+        Provider name string for display purposes
+    """
+    base_url = os.getenv("OPENAI_BASE_URL")
+    if base_url:
+        return f"OpenAI (Custom: {base_url})"
+    return "OpenAI"
+
+
+def is_openai_reasoning_model(model_name: str) -> bool:
+    """
+    Check if the OpenAI model supports reasoning_effort parameter.
+
+    Models that support reasoning_effort:
+    - o-series: o1, o3, o4 and their variants (o1-mini, o1-2024-12-17, etc.)
+    - gpt-5 series: gpt-5, gpt-5.2, gpt-5-turbo, etc.
+
+    Non-reasoning models like gpt-4.1, gpt-4o, etc. will reject this parameter.
+
+    Handles proxy/router prefixes like "openai/o1-mini" or "openrouter/gpt-5.2".
+
+    Args:
+        model_name: The OpenAI model name (may include path prefix)
+
+    Returns:
+        True if the model supports reasoning_effort, False otherwise
+    """
+    model_lower = model_name.lower()
+
+    # Handle proxy/router prefixes like "openai/o1-mini", "openrouter/gpt-5.2"
+    # Extract the base model name after the last "/"
+    if "/" in model_lower:
+        model_lower = model_lower.rsplit("/", 1)[-1]
+
+    # Match o-series and gpt-5 series
+    return model_lower.startswith(("o1", "o3", "o4", "gpt-5"))
+
+
+def is_openai_retryable(error: Exception) -> bool:
+    """
+    Determine if an OpenAI error should be retried.
+
+    Non-retryable errors:
+    - AuthenticationError (401): Invalid API key
+    - PermissionDeniedError (403): No access to resource
+    - BadRequestError (400): Invalid request format
+    - NotFoundError (404): Model or resource not found
+
+    Retryable errors:
+    - RateLimitError (429): Rate limit exceeded
+    - APIConnectionError: Network issues
+    - InternalServerError (500): Server errors
+    - APIStatusError with 502, 503, 504: Gateway/service errors
+
+    Args:
+        error: The exception from OpenAI API call
+
+    Returns:
+        True if the error should be retried, False otherwise
+    """
+    if not HAS_OPENAI:
+        return True
+
+    # Authentication error - invalid API key (401)
+    if isinstance(error, openai.AuthenticationError):
+        return False
+
+    # Permission denied - no access to resource (403)
+    if isinstance(error, openai.PermissionDeniedError):
+        return False
+
+    # Bad request - invalid request format (400)
+    if isinstance(error, openai.BadRequestError):
+        return False
+
+    # Not found - model or resource doesn't exist (404)
+    if isinstance(error, openai.NotFoundError):
+        return False
+
+    # Rate limit exceeded - should retry with backoff (429)
+    if isinstance(error, openai.RateLimitError):
+        return True
+
+    # API connection error - network issues, should retry
+    if isinstance(error, openai.APIConnectionError):
+        return True
+
+    # Internal server error - should retry (500)
+    if isinstance(error, openai.InternalServerError):
+        return True
+
+    # For other APIStatusError, check HTTP status code
+    if isinstance(error, openai.APIStatusError):
+        # Retryable server-side errors
+        return error.status_code in (429, 500, 502, 503, 504)
+
+    # For unknown errors, default to retry (network issues, timeouts, etc.)
+    return True
+
+
+def is_gemini_retryable(error: Exception) -> bool:
+    """
+    Determine if a Gemini error should be retried.
+
+    Uses string matching on error messages since google-genai may not have
+    well-defined exception types for all error cases.
+
+    Non-retryable errors:
+    - API key errors
+    - Authentication/permission errors
+    - Invalid request errors
+    - Model not found errors
+    - Billing/quota permanently exceeded
+
+    Retryable errors:
+    - Rate limit (429)
+    - Server errors (500, 502, 503, 504)
+    - Timeout/connection errors
+
+    Args:
+        error: The exception from Gemini API call
+
+    Returns:
+        True if the error should be retried, False otherwise
+    """
+    error_str = str(error).lower()
+
+    # API key / authentication errors - do not retry
+    if "api_key" in error_str or "api key" in error_str:
+        return False
+    if "authentication" in error_str or "authenticate" in error_str:
+        return False
+    if "invalid_api_key" in error_str or "invalid api key" in error_str:
+        return False
+
+    # Permission / forbidden errors - do not retry
+    if "permission" in error_str and "denied" in error_str:
+        return False
+    if "forbidden" in error_str or "403" in error_str:
+        return False
+
+    # Invalid request errors - do not retry
+    if "invalid" in error_str and ("request" in error_str or "argument" in error_str):
+        return False
+    if "400" in error_str and "bad request" in error_str:
+        return False
+
+    # Model not found - do not retry
+    if "model" in error_str and ("not found" in error_str or "not exist" in error_str):
+        return False
+    if "404" in error_str:
+        return False
+
+    # Billing / permanent quota errors - do not retry
+    if "billing" in error_str:
+        return False
+    if "quota" in error_str and ("exceeded" in error_str or "exhausted" in error_str):
+        # Check if it mentions billing which indicates permanent quota issue
+        if "billing" in error_str or "payment" in error_str:
+            return False
+        # Temporary quota (rate limit) - should retry
+        return True
+
+    # Rate limit errors - should retry (429)
+    if "rate" in error_str and "limit" in error_str:
+        return True
+    if "429" in error_str or "resource_exhausted" in error_str:
+        return True
+
+    # Server errors - should retry (500, 502, 503, 504)
+    if any(code in error_str for code in ["500", "502", "503", "504"]):
+        return True
+    if "internal" in error_str and ("error" in error_str or "server" in error_str):
+        return True
+    if "service" in error_str and "unavailable" in error_str:
+        return True
+    if "gateway" in error_str:
+        return True
+
+    # Timeout / connection errors - should retry
+    if "timeout" in error_str or "timed out" in error_str:
+        return True
+    if "connection" in error_str:
+        return True
+    if "network" in error_str:
+        return True
+
+    # Unknown errors - default to retry with limited attempts
+    return True
+
+
+# JSON Schema for LLM structured output
+AUDIT_RESULT_SCHEMA = {
+    "type": "object",
+    "additionalProperties": False,
+    "properties": {
+        "is_violation": {
+            "type": "boolean",
+            "description": "Whether any violations were found",
+        },
+        "violations": {
+            "type": "array",
+            "description": "List of violations found",
+            "items": {
+                "type": "object",
+                "additionalProperties": False,
+                "properties": {
+                    "rule_id": {
+                        "type": "string",
+                        "description": "ID of the violated rule (e.g., R001)",
+                    },
+                    "violation_text": {
+                        "type": "string",
+                        "description": "The problematic text directly verbatim quote from the source content, and not span multiple cells",
+                    },
+                    "violation_reason": {
+                        "type": "string",
+                        "description": "Explanation of why this violates the rule",
+                    },
+                    "fix_action": {
+                        "type": "string",
+                        "enum": ["replace", "manual"],
+                        "description": "Action type: replace substitutes text (including deletion-via-replace), manual requires human review",
+                    },
+                    "revised_text": {
+                        "type": "string",
+                        "description": "For replace: complete replacement text (including deletion-via-replace). For manual: additional guidance for human reviewer",
+                    },
+                },
+                "required": [
+                    "rule_id",
+                    "violation_text",
+                    "violation_reason",
+                    "fix_action",
+                    "revised_text",
+                ],
+            },
+        },
+    },
+    "required": ["is_violation", "violations"],
+}
+
+# JSON Schema for global extraction output
+GLOBAL_EXTRACT_SCHEMA = {
+    "type": "object",
+    "additionalProperties": False,
+    "properties": {
+        "results": {
+            "type": "array",
+            "items": {
+                "type": "object",
+                "additionalProperties": False,
+                "properties": {
+                    "rule_id": {"type": "string"},
+                    "extracted_results": {
+                        "type": "array",
+                        "items": {
+                            "type": "object",
+                            "additionalProperties": False,
+                            "properties": {
+                                "entity": {"type": "string"},
+                                "fields": {
+                                    "type": "array",
+                                    "items": {
+                                        "type": "object",
+                                        "additionalProperties": False,
+                                        "properties": {
+                                            "name": {"type": "string"},
+                                            "value": {"type": "string"},
+                                            "evidence": {"type": "string"},
+                                        },
+                                        "required": ["name", "value", "evidence"],
+                                    },
+                                },
+                            },
+                            "required": ["entity", "fields"],
+                        },
+                    },
+                },
+                "required": ["rule_id", "extracted_results"],
+            },
+        }
+    },
+    "required": ["results"],
+}
+
+# JSON Schema for global verification output
+GLOBAL_VERIFY_SCHEMA = {
+    "type": "object",
+    "additionalProperties": False,
+    "properties": {
+        "violations": {
+            "type": "array",
+            "items": {
+                "type": "object",
+                "additionalProperties": False,
+                "properties": {
+                    "rule_id": {"type": "string"},
+                    "uuid": {"type": "string"},
+                    "uuid_end": {"type": "string"},
+                    "violation_text": {"type": "string"},
+                    "violation_reason": {"type": "string"},
+                    "fix_action": {"type": "string", "enum": ["replace", "manual"]},
+                    "revised_text": {"type": "string"},
+                },
+                "required": [
+                    "rule_id",
+                    "uuid",
+                    "uuid_end",
+                    "violation_text",
+                    "violation_reason",
+                    "fix_action",
+                    "revised_text",
+                ],
+            },
+        }
+    },
+    "required": ["violations"],
+}
+
+
+async def global_extract_gemini_async(
+    user_prompt: str,
+    system_prompt: str,
+    model_name: str,
+    client,
+    thinking_level: str = None,
+    thinking_budget: int = None,
+) -> dict:
+    thinking_config = None
+    if thinking_level and thinking_level.upper() in (
+        "MINIMAL",
+        "LOW",
+        "MEDIUM",
+        "HIGH",
+    ):
+        level_map = {
+            "MINIMAL": types.ThinkingLevel.MINIMAL,
+            "LOW": types.ThinkingLevel.LOW,
+            "MEDIUM": types.ThinkingLevel.MEDIUM,
+            "HIGH": types.ThinkingLevel.HIGH,
+        }
+        thinking_config = types.ThinkingConfig(
+            thinking_level=level_map[thinking_level.upper()]
+        )
+    elif thinking_budget is not None:
+        thinking_config = types.ThinkingConfig(thinking_budget=int(thinking_budget))
+
+    config_params = {
+        "system_instruction": system_prompt,
+        "response_mime_type": "application/json",
+        "response_schema": GLOBAL_EXTRACT_SCHEMA,
+    }
+    if thinking_config:
+        config_params["thinking_config"] = thinking_config
+
+    response = await client.models.generate_content(
+        model=model_name,
+        contents=user_prompt,
+        config=types.GenerateContentConfig(**config_params),
+    )
+    return json.loads(response.text)
+
+
+async def global_extract_openai_async(
+    user_prompt: str,
+    system_prompt: str,
+    model_name: str,
+    client,
+    reasoning_effort: str = None,
+) -> dict:
+    request_params = {
+        "model": model_name,
+        "messages": [
+            {"role": "system", "content": system_prompt},
+            {"role": "user", "content": user_prompt},
+        ],
+        "response_format": {
+            "type": "json_schema",
+            "json_schema": {
+                "name": "global_extract",
+                "strict": True,
+                "schema": GLOBAL_EXTRACT_SCHEMA,
+            },
+        },
+    }
+    if (
+        reasoning_effort
+        and reasoning_effort.lower() in ("low", "medium", "high")
+        and is_openai_reasoning_model(model_name)
+    ):
+        request_params["reasoning_effort"] = reasoning_effort.lower()
+
+    response = await client.chat.completions.create(**request_params)
+    return json.loads(response.choices[0].message.content)
+
+
+async def global_verify_gemini_async(
+    user_prompt: str,
+    system_prompt: str,
+    model_name: str,
+    client,
+    thinking_level: str = None,
+    thinking_budget: int = None,
+) -> dict:
+    thinking_config = None
+    if thinking_level and thinking_level.upper() in (
+        "MINIMAL",
+        "LOW",
+        "MEDIUM",
+        "HIGH",
+    ):
+        level_map = {
+            "MINIMAL": types.ThinkingLevel.MINIMAL,
+            "LOW": types.ThinkingLevel.LOW,
+            "MEDIUM": types.ThinkingLevel.MEDIUM,
+            "HIGH": types.ThinkingLevel.HIGH,
+        }
+        thinking_config = types.ThinkingConfig(
+            thinking_level=level_map[thinking_level.upper()]
+        )
+    elif thinking_budget is not None:
+        thinking_config = types.ThinkingConfig(thinking_budget=int(thinking_budget))
+
+    config_params = {
+        "system_instruction": system_prompt,
+        "response_mime_type": "application/json",
+        "response_schema": GLOBAL_VERIFY_SCHEMA,
+    }
+    if thinking_config:
+        config_params["thinking_config"] = thinking_config
+
+    response = await client.models.generate_content(
+        model=model_name,
+        contents=user_prompt,
+        config=types.GenerateContentConfig(**config_params),
+    )
+    return json.loads(response.text)
+
+
+async def global_verify_openai_async(
+    user_prompt: str,
+    system_prompt: str,
+    model_name: str,
+    client,
+    reasoning_effort: str = None,
+) -> dict:
+    request_params = {
+        "model": model_name,
+        "messages": [
+            {"role": "system", "content": system_prompt},
+            {"role": "user", "content": user_prompt},
+        ],
+        "response_format": {
+            "type": "json_schema",
+            "json_schema": {
+                "name": "global_verify",
+                "strict": True,
+                "schema": GLOBAL_VERIFY_SCHEMA,
+            },
+        },
+    }
+    if (
+        reasoning_effort
+        and reasoning_effort.lower() in ("low", "medium", "high")
+        and is_openai_reasoning_model(model_name)
+    ):
+        request_params["reasoning_effort"] = reasoning_effort.lower()
+
+    response = await client.chat.completions.create(**request_params)
+    return json.loads(response.choices[0].message.content)
+
+
+async def audit_block_gemini_async(
+    user_prompt: str,
+    system_prompt: str,
+    model_name: str,
+    client,
+    thinking_level: str = None,
+    thinking_budget: int = None,
+) -> dict:
+    """
+    Audit a text block using Google Gemini with strict JSON mode (async version).
+
+    Args:
+        user_prompt: User prompt to audit
+        system_prompt: Cached system prompt with rules and instructions
+        model_name: Gemini model to use
+        client: Gemini async client instance (client.aio)
+        thinking_level: Thinking level for Gemini 3 models (MINIMAL, LOW, MEDIUM, HIGH)
+        thinking_budget: Thinking token budget for Gemini 2.5 models (integer)
+
+    Returns:
+        Audit result dictionary
+    """
+    # Build thinking config based on model and parameters
+    thinking_config = None
+
+    if thinking_level and thinking_level.upper() in (
+        "MINIMAL",
+        "LOW",
+        "MEDIUM",
+        "HIGH",
+    ):
+        # For Gemini 3 models
+        level_map = {
+            "MINIMAL": types.ThinkingLevel.MINIMAL,
+            "LOW": types.ThinkingLevel.LOW,
+            "MEDIUM": types.ThinkingLevel.MEDIUM,
+            "HIGH": types.ThinkingLevel.HIGH,
+        }
+        thinking_config = types.ThinkingConfig(
+            thinking_level=level_map[thinking_level.upper()]
+        )
+    elif thinking_budget is not None:
+        # For Gemini 2.5 models
+        thinking_config = types.ThinkingConfig(thinking_budget=int(thinking_budget))
+
+    config_params = {
+        "system_instruction": system_prompt,
+        "response_mime_type": "application/json",
+        "response_schema": AUDIT_RESULT_SCHEMA,
+    }
+
+    # Only add thinking_config if it's configured
+    if thinking_config:
+        config_params["thinking_config"] = thinking_config
+
+    response = await client.models.generate_content(
+        model=model_name,
+        contents=user_prompt,
+        config=types.GenerateContentConfig(**config_params),
+    )
+
+    # With structured output, response is guaranteed to be valid JSON
+    result = json.loads(response.text)
+    return result
+
+
+async def audit_block_openai_async(
+    user_prompt: str,
+    system_prompt: str,
+    model_name: str,
+    client,
+    reasoning_effort: str = None,
+) -> dict:
+    """
+    Audit a text block using OpenAI with strict JSON mode (async version).
+
+    Args:
+        user_prompt: User prompt to audit
+        system_prompt: Cached system prompt with rules and instructions
+        model_name: OpenAI model to use
+        client: AsyncOpenAI client instance
+        reasoning_effort: Reasoning effort for o-series models (low, medium, high)
+
+    Returns:
+        Audit result dictionary
+    """
+    request_params = {
+        "model": model_name,
+        "messages": [
+            {"role": "system", "content": system_prompt},
+            {"role": "user", "content": user_prompt},
+        ],
+        "response_format": {
+            "type": "json_schema",
+            "json_schema": {
+                "name": "audit_result",
+                "strict": True,
+                "schema": AUDIT_RESULT_SCHEMA,
+            },
+        },
+    }
+
+    # Add reasoning_effort only for o-series models that support it
+    if (
+        reasoning_effort
+        and reasoning_effort.lower() in ("low", "medium", "high")
+        and is_openai_reasoning_model(model_name)
+    ):
+        request_params["reasoning_effort"] = reasoning_effort.lower()
+
+    response = await client.chat.completions.create(**request_params)
+
+    # With structured output, response is guaranteed to be valid JSON
+    result = json.loads(response.choices[0].message.content)
+    return result