wehub/hkuds--lightrag
1
0
Fork 0
Code Issues Pull Requests Actions Deployments Models Agent Notes Packages Projects Releases Wiki Activity

chore: import upstream snapshot with attribution

Browse Source
This commit is contained in:
wehub-resource-sync
2026-07-13 12:08:54 +08:00
commit 4a4a1fed67
721 changed files with 262090 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/parser/docx/__init__.py
View File
tests/parser/docx/_native_docx_fixtures.py
+329
View File
@@ -0,0 +1,329 @@
"""Scenario fixtures for the native docx → SidecarWriter migration tests.
Each scenario describes:
- ``blocks`` — what ``extract_docx_blocks`` would return (the synthetic
block dicts that the adapter consumes).
- ``parse_metadata`` — the dict the upstream parser fills in (only
``first_heading`` is currently consumed by the adapter).
- ``assets`` — files the upstream extractor would have written into
``<base>.blocks.assets/`` before the IR builder runs. Maps relative names
inside the asset dir → byte content.
- ``doc_id`` — fixed so blockid + sidecar ids are deterministic.
- ``file_path`` — used for canonical basename / doc_title fallback.
The captured outputs (``blocks.jsonl`` + per-modality JSONs + assets) live
under ``tests/parser/docx/golden/native_docx/<scenario>/``. The
production path (``LightRAG.parse_native``) must produce byte-identical
bytes vs those fixtures; the regen script under ``scripts/`` rewrites
them when the format intentionally changes.
"""
from __future__ import annotations
from dataclasses import dataclass, field
from typing import Any
def _block(
content: str,
*,
heading: str = "",
level: int = 0,
parent: list[str] | None = None,
uuid: str = "p1",
uuid_end: str | None = None,
table_headers: list[Any] | None = None,
table_chunk_role: str = "none",
) -> dict[str, Any]:
"""Build a synthetic block matching ``extract_docx_blocks`` output."""
out: dict[str, Any] = {
"uuid": uuid,
"uuid_end": uuid_end if uuid_end is not None else uuid,
"heading": heading,
"content": content,
"type": "text",
"parent_headings": list(parent or []),
"level": level,
"table_chunk_role": table_chunk_role,
}
if table_headers is not None:
out["table_headers"] = table_headers
return out
@dataclass
class Scenario:
name: str
doc_id: str
file_path: str # canonical-ish; what the pipeline would pass
blocks: list[dict[str, Any]]
parse_metadata: dict[str, Any] = field(default_factory=dict)
assets: dict[str, bytes] = field(default_factory=dict)
SCENARIOS: list[Scenario] = [
# --- 1: text-only, multi-heading -----------------------------------
Scenario(
name="text_only_hierarchy",
doc_id="doc-aaaa111122223333aaaa111122223333",
file_path="paper.docx",
parse_metadata={"first_heading": "Introduction"},
blocks=[
_block(
"# Introduction",
heading="Introduction",
level=1,
uuid="h1",
),
_block(
"Body paragraph one.",
heading="Introduction",
level=1,
uuid="p1",
uuid_end="p2",
),
_block(
"## Background",
heading="Background",
level=2,
parent=["Introduction"],
uuid="h2",
),
_block(
"Sub body.",
heading="Background",
level=2,
parent=["Introduction"],
uuid="p3",
),
],
),
# --- 2: block + inline equations -----------------------------------
Scenario(
name="equations_block_and_inline",
doc_id="doc-bbbb222233334444bbbb222233334444",
file_path="formulas.docx",
parse_metadata={"first_heading": "Equations"},
blocks=[
_block(
"# Equations",
heading="Equations",
level=1,
uuid="h1",
),
_block(
# Inline equation (no surrounding \n on either side)
"Energy is <equation>E=mc^2</equation> per Einstein.",
heading="Equations",
level=1,
uuid="p1",
),
_block(
# Block equation (wedged between newlines)
"Consider:\n<equation>x^2 + y^2 = r^2</equation>\nThe circle equation.",
heading="Equations",
level=1,
uuid="p2",
),
_block(
# Block at content edge (start == 0)
"<equation>a + b = c</equation>\ntext after",
heading="Equations",
level=1,
uuid="p3",
),
],
),
# --- 3: tables with and without table_headers ----------------------
Scenario(
name="tables_mixed",
doc_id="doc-cccc333344445555cccc333344445555",
file_path="report.docx",
parse_metadata={"first_heading": "Report"},
blocks=[
_block(
"# Report",
heading="Report",
level=1,
uuid="h1",
),
_block(
# Table with table_headers (cross-page repeating)
'See table:\n<table>[["X","Y"],["1","2"],["3","4"]]</table>',
heading="Report",
level=1,
uuid="t1",
table_headers=[[["X", "Y"]]], # one table, one header row
),
_block(
# Table without table_headers
'Plain table:\n<table>[["a","b"]]</table>',
heading="Report",
level=1,
uuid="t2",
),
_block(
# Two tables in one block
'<table>[["p"]]</table>\nthen\n<table>[["q","r"],["s","t"]]</table>',
heading="Report",
level=1,
uuid="t3",
table_headers=[None, [["q", "r"]]],
),
],
),
# --- 4: drawings + assets ------------------------------------------
Scenario(
name="drawings_with_assets",
doc_id="doc-dddd444455556666dddd444455556666",
file_path="diagrams.docx",
parse_metadata={"first_heading": "Diagrams"},
assets={
"fig1.png": b"\x89PNG\r\n\x1a\n-fig1-fake",
"fig2.jpg": b"\xff\xd8\xff\xe0-fig2-fake",
},
blocks=[
_block(
"# Diagrams",
heading="Diagrams",
level=1,
uuid="h1",
),
_block(
"Figure one:\n"
'<drawing id="x" format="png" '
'path="diagrams.blocks.assets/fig1.png" '
'src="docx://image1" />\n'
"Figure two:\n"
'<drawing id="y" format="jpg" '
'path="diagrams.blocks.assets/fig2.jpg" '
'src="docx://image2" />',
heading="Diagrams",
level=1,
uuid="p1",
),
],
),
# --- 5: all modalities mixed ---------------------------------------
Scenario(
name="all_modalities",
doc_id="doc-eeee555566667777eeee555566667777",
file_path="combo.docx",
parse_metadata={"first_heading": "Combined"},
assets={"pic.png": b"PNG-combo"},
blocks=[
_block(
"# Combined",
heading="Combined",
level=1,
uuid="h1",
),
_block(
"Look at this figure:\n"
'<drawing id="z" format="png" '
'path="combo.blocks.assets/pic.png" '
'src="docx://img" />\n'
"Plus a table:\n"
'<table>[["α","β"],["γ","δ"]]</table>\n'
"And a block equation:\n"
"<equation>F = ma</equation>\n"
"And an inline <equation>v=d/t</equation> here.",
heading="Combined",
level=1,
uuid="p1",
),
],
),
# --- 6: empty block dropped ----------------------------------------
Scenario(
name="empty_block_dropped",
doc_id="doc-ffff666677778888ffff666677778888",
file_path="sparse.docx",
parse_metadata={"first_heading": "Sparse"},
blocks=[
_block(
"# Sparse",
heading="Sparse",
level=1,
uuid="h1",
),
_block(
" \n ", # strips to empty — must be dropped
heading="Sparse",
level=1,
uuid="p_empty",
),
_block(
"Real content after empty.",
heading="Sparse",
level=1,
uuid="p_real",
),
],
),
# --- 7: external / linked image references ------------------------
# DOCX can carry ``<a:blip r:link="rId…"/>`` references to image
# targets that live outside the package — the upstream extractor
# then emits ``<drawing path="<external URL or unresolved path>" />``
# WITHOUT writing bytes into ``<base>.blocks.assets/``. The adapter
# must pass those paths through verbatim (both in ``blocks.jsonl``
# and ``drawings.json``); turning them into AssetSpecs with
# ``source=None`` would make the writer warn-and-skip → ``path=""``,
# losing the only reference downstream consumers have.
Scenario(
name="external_image_link",
doc_id="doc-1111aaaa2222bbbb1111aaaa2222bbbb",
file_path="linked.docx",
parse_metadata={"first_heading": "Linked"},
# No on-disk assets — the path points elsewhere.
assets={},
blocks=[
_block(
"# Linked",
heading="Linked",
level=1,
uuid="h1",
),
_block(
"See the diagram online:\n"
'<drawing id="z" format="png" '
'path="https://example.com/diagrams/architecture.png" '
'src="docx://external" />\n'
"And a relative-but-not-asset path:\n"
'<drawing id="z2" format="gif" '
'path="../images/legacy.gif" '
'src="docx://legacy" />',
heading="Linked",
level=1,
uuid="p1",
),
],
),
# --- 8: missing paraid ---------------------------------------------
Scenario(
name="missing_paraid",
doc_id="doc-99990000111122229999000011112222",
file_path="legacy.docx",
parse_metadata={"first_heading": ""}, # no headings at all
blocks=[
_block(
"Just plain text without a heading.",
heading="",
level=0,
uuid="", # missing
uuid_end="",
),
_block(
"Another paragraph with no paraId.",
heading="",
level=0,
uuid="",
uuid_end="",
),
],
),
]
__all__ = ["Scenario", "SCENARIOS", "_block"]
tests/parser/docx/golden/native_docx/all_modalities/combo.blocks.assets/pic.png
+1
View File
@@ -0,0 +1 @@
PNG-combo
tests/parser/docx/golden/native_docx/all_modalities/combo.blocks.jsonl
+3
View File
@@ -0,0 +1,3 @@
{"type": "meta", "format": "lightrag", "version": "1.0", "document_name": "combo.docx", "document_format": "docx", "document_hash": "sha256:2aa5d36cf1126af704cc0b7b924e1b8d9eb8fa576b19807f9fb1fbc80d0fefb7", "table_file": true, "equation_file": true, "drawing_file": true, "asset_dir": true, "blocks": 2, "doc_id": "doc-eeee555566667777eeee555566667777", "parse_engine": "native", "parse_time": "2026-01-01T00:00:00+00:00", "doc_title": "Combined"}
{"type": "content", "blockid": "bea7fc7932485628877517b6b5a0e80c", "format": "plain_text", "content": "# Combined", "heading": "Combined", "parent_headings": [], "level": 1, "session_type": "body", "table_slice": "none", "positions": [{"type": "paraid", "range": ["h1", "h1"]}]}
{"type": "content", "blockid": "8caba9870df85b9357e9521a70d82225", "format": "plain_text", "content": "Look at this figure:\n<drawing id=\"im-eeee555566667777eeee555566667777-0001\" format=\"png\" path=\"pic.png\" src=\"docx://img\" />\nPlus a table:\n<table id=\"tb-eeee555566667777eeee555566667777-0001\" format=\"json\">[[\"α\",\"β\"],[\"γ\",\"δ\"]]</table>\nAnd a block equation:\n<equation id=\"eq-eeee555566667777eeee555566667777-0001\" format=\"latex\">F = ma</equation>\nAnd an inline <equation format=\"latex\">v=d/t</equation> here.", "heading": "Combined", "parent_headings": [], "level": 1, "session_type": "body", "table_slice": "none", "positions": [{"type": "paraid", "range": ["p1", "p1"]}]}
tests/parser/docx/golden/native_docx/all_modalities/combo.drawings.json
+16
View File
@@ -0,0 +1,16 @@
{
"version": "1.0",
"drawings": {
"im-eeee555566667777eeee555566667777-0001": {
"id": "im-eeee555566667777eeee555566667777-0001",
"blockid": "8caba9870df85b9357e9521a70d82225",
"heading": "Combined",
"parent_headings": [],
"format": "png",
"path": "combo.blocks.assets/pic.png",
"src": "docx://img",
"caption": "",
"footnotes": []
}
}
}
tests/parser/docx/golden/native_docx/all_modalities/combo.equations.json
+15
View File
@@ -0,0 +1,15 @@
{
"version": "1.0",
"equations": {
"eq-eeee555566667777eeee555566667777-0001": {
"id": "eq-eeee555566667777eeee555566667777-0001",
"blockid": "8caba9870df85b9357e9521a70d82225",
"heading": "Combined",
"parent_headings": [],
"format": "latex",
"content": "F = ma",
"caption": "",
"footnotes": []
}
}
}
tests/parser/docx/golden/native_docx/all_modalities/combo.tables.json
+19
View File
@@ -0,0 +1,19 @@
{
"version": "1.0",
"tables": {
"tb-eeee555566667777eeee555566667777-0001": {
"id": "tb-eeee555566667777eeee555566667777-0001",
"blockid": "8caba9870df85b9357e9521a70d82225",
"heading": "Combined",
"parent_headings": [],
"dimension": [
2,
2
],
"format": "json",
"content": "[[\"α\", \"β\"], [\"γ\", \"δ\"]]",
"caption": "",
"footnotes": []
}
}
}
tests/parser/docx/golden/native_docx/drawings_with_assets/diagrams.blocks.assets/fig1.png
+3
View File
@@ -0,0 +1,3 @@
‰PNG

-fig1-fake
Side by Side

After

Width:  |  Height:  |  Size: 18 B

tests/parser/docx/golden/native_docx/drawings_with_assets/diagrams.blocks.assets/fig2.jpg
+1
View File
@@ -0,0 +1 @@
�ﾘ��-fig2-fake
Side by Side

After

Width:  |  Height:  |  Size: 14 B

tests/parser/docx/golden/native_docx/drawings_with_assets/diagrams.blocks.jsonl
+3
View File
@@ -0,0 +1,3 @@
{"type": "meta", "format": "lightrag", "version": "1.0", "document_name": "diagrams.docx", "document_format": "docx", "document_hash": "sha256:9ad9b5e89e06783368565eda8ff1cd44e6b2f9a4d678f30e204b7c4984da524e", "table_file": false, "equation_file": false, "drawing_file": true, "asset_dir": true, "blocks": 2, "doc_id": "doc-dddd444455556666dddd444455556666", "parse_engine": "native", "parse_time": "2026-01-01T00:00:00+00:00", "doc_title": "Diagrams"}
{"type": "content", "blockid": "14a79f4f059cb16effe2d6cfe5a18e9b", "format": "plain_text", "content": "# Diagrams", "heading": "Diagrams", "parent_headings": [], "level": 1, "session_type": "body", "table_slice": "none", "positions": [{"type": "paraid", "range": ["h1", "h1"]}]}
{"type": "content", "blockid": "d10a9ed61f86247aeedfefb785c919b3", "format": "plain_text", "content": "Figure one:\n<drawing id=\"im-dddd444455556666dddd444455556666-0001\" format=\"png\" path=\"fig1.png\" src=\"docx://image1\" />\nFigure two:\n<drawing id=\"im-dddd444455556666dddd444455556666-0002\" format=\"jpg\" path=\"fig2.jpg\" src=\"docx://image2\" />", "heading": "Diagrams", "parent_headings": [], "level": 1, "session_type": "body", "table_slice": "none", "positions": [{"type": "paraid", "range": ["p1", "p1"]}]}
tests/parser/docx/golden/native_docx/drawings_with_assets/diagrams.drawings.json
+27
View File
@@ -0,0 +1,27 @@
{
"version": "1.0",
"drawings": {
"im-dddd444455556666dddd444455556666-0001": {
"id": "im-dddd444455556666dddd444455556666-0001",
"blockid": "d10a9ed61f86247aeedfefb785c919b3",
"heading": "Diagrams",
"parent_headings": [],
"format": "png",
"path": "diagrams.blocks.assets/fig1.png",
"src": "docx://image1",
"caption": "",
"footnotes": []
},
"im-dddd444455556666dddd444455556666-0002": {
"id": "im-dddd444455556666dddd444455556666-0002",
"blockid": "d10a9ed61f86247aeedfefb785c919b3",
"heading": "Diagrams",
"parent_headings": [],
"format": "jpg",
"path": "diagrams.blocks.assets/fig2.jpg",
"src": "docx://image2",
"caption": "",
"footnotes": []
}
}
}
tests/parser/docx/golden/native_docx/empty_block_dropped/sparse.blocks.jsonl
+3
View File
@@ -0,0 +1,3 @@
{"type": "meta", "format": "lightrag", "version": "1.0", "document_name": "sparse.docx", "document_format": "docx", "document_hash": "sha256:9f42b2b11e8a06b78fbd7fd815f2d800027d33c2bd6f1aa0f92feab32a37a0cb", "table_file": false, "equation_file": false, "drawing_file": false, "asset_dir": false, "blocks": 2, "doc_id": "doc-ffff666677778888ffff666677778888", "parse_engine": "native", "parse_time": "2026-01-01T00:00:00+00:00", "doc_title": "Sparse"}
{"type": "content", "blockid": "1fb5dc88cfc19f236d324df359829044", "format": "plain_text", "content": "# Sparse", "heading": "Sparse", "parent_headings": [], "level": 1, "session_type": "body", "table_slice": "none", "positions": [{"type": "paraid", "range": ["h1", "h1"]}]}
{"type": "content", "blockid": "39f44666ee7d52f6fc680bad4f93ffa8", "format": "plain_text", "content": "Real content after empty.", "heading": "Sparse", "parent_headings": [], "level": 1, "session_type": "body", "table_slice": "none", "positions": [{"type": "paraid", "range": ["p_real", "p_real"]}]}
tests/parser/docx/golden/native_docx/equations_block_and_inline/formulas.blocks.jsonl
+5
View File
@@ -0,0 +1,5 @@
{"type": "meta", "format": "lightrag", "version": "1.0", "document_name": "formulas.docx", "document_format": "docx", "document_hash": "sha256:403a469531b150e2c1c14a3728ea6d2e3b48b3c86cf4d4896b94d5c0d6d07fcb", "table_file": false, "equation_file": true, "drawing_file": false, "asset_dir": false, "blocks": 4, "doc_id": "doc-bbbb222233334444bbbb222233334444", "parse_engine": "native", "parse_time": "2026-01-01T00:00:00+00:00", "doc_title": "Equations"}
{"type": "content", "blockid": "49b6d85e4de7d0a8a8024d68b297885d", "format": "plain_text", "content": "# Equations", "heading": "Equations", "parent_headings": [], "level": 1, "session_type": "body", "table_slice": "none", "positions": [{"type": "paraid", "range": ["h1", "h1"]}]}
{"type": "content", "blockid": "3caba68eda5e6f4f3288d3c8a338b76e", "format": "plain_text", "content": "Energy is <equation format=\"latex\">E=mc^2</equation> per Einstein.", "heading": "Equations", "parent_headings": [], "level": 1, "session_type": "body", "table_slice": "none", "positions": [{"type": "paraid", "range": ["p1", "p1"]}]}
{"type": "content", "blockid": "603a388c6d9c45900dcbde3a86a0c3b5", "format": "plain_text", "content": "Consider:\n<equation id=\"eq-bbbb222233334444bbbb222233334444-0001\" format=\"latex\">x^2 + y^2 = r^2</equation>\nThe circle equation.", "heading": "Equations", "parent_headings": [], "level": 1, "session_type": "body", "table_slice": "none", "positions": [{"type": "paraid", "range": ["p2", "p2"]}]}
{"type": "content", "blockid": "e4c6683e6107d75437b7d0712cf75c82", "format": "plain_text", "content": "<equation id=\"eq-bbbb222233334444bbbb222233334444-0002\" format=\"latex\">a + b = c</equation>\ntext after", "heading": "Equations", "parent_headings": [], "level": 1, "session_type": "body", "table_slice": "none", "positions": [{"type": "paraid", "range": ["p3", "p3"]}]}
tests/parser/docx/golden/native_docx/equations_block_and_inline/formulas.equations.json
+25
View File
@@ -0,0 +1,25 @@
{
"version": "1.0",
"equations": {
"eq-bbbb222233334444bbbb222233334444-0001": {
"id": "eq-bbbb222233334444bbbb222233334444-0001",
"blockid": "603a388c6d9c45900dcbde3a86a0c3b5",
"heading": "Equations",
"parent_headings": [],
"format": "latex",
"content": "x^2 + y^2 = r^2",
"caption": "",
"footnotes": []
},
"eq-bbbb222233334444bbbb222233334444-0002": {
"id": "eq-bbbb222233334444bbbb222233334444-0002",
"blockid": "e4c6683e6107d75437b7d0712cf75c82",
"heading": "Equations",
"parent_headings": [],
"format": "latex",
"content": "a + b = c",
"caption": "",
"footnotes": []
}
}
}
tests/parser/docx/golden/native_docx/external_image_link/linked.blocks.jsonl
+3
View File
@@ -0,0 +1,3 @@
{"type": "meta", "format": "lightrag", "version": "1.0", "document_name": "linked.docx", "document_format": "docx", "document_hash": "sha256:4d984f8d2a4cb95808c403800f17e4f5043f48cc9d72e6062c93b5150e389320", "table_file": false, "equation_file": false, "drawing_file": true, "asset_dir": false, "blocks": 2, "doc_id": "doc-1111aaaa2222bbbb1111aaaa2222bbbb", "parse_engine": "native", "parse_time": "2026-01-01T00:00:00+00:00", "doc_title": "Linked"}
{"type": "content", "blockid": "573c0f2fc108950c0ce0060429a9de39", "format": "plain_text", "content": "# Linked", "heading": "Linked", "parent_headings": [], "level": 1, "session_type": "body", "table_slice": "none", "positions": [{"type": "paraid", "range": ["h1", "h1"]}]}
{"type": "content", "blockid": "42b4dd9bfb50ba8f1c4b60229ea4605a", "format": "plain_text", "content": "See the diagram online:\n<drawing id=\"im-1111aaaa2222bbbb1111aaaa2222bbbb-0001\" format=\"png\" path=\"https://example.com/diagrams/architecture.png\" src=\"docx://external\" />\nAnd a relative-but-not-asset path:\n<drawing id=\"im-1111aaaa2222bbbb1111aaaa2222bbbb-0002\" format=\"gif\" path=\"../images/legacy.gif\" src=\"docx://legacy\" />", "heading": "Linked", "parent_headings": [], "level": 1, "session_type": "body", "table_slice": "none", "positions": [{"type": "paraid", "range": ["p1", "p1"]}]}
tests/parser/docx/golden/native_docx/external_image_link/linked.drawings.json
+27
View File
@@ -0,0 +1,27 @@
{
"version": "1.0",
"drawings": {
"im-1111aaaa2222bbbb1111aaaa2222bbbb-0001": {
"id": "im-1111aaaa2222bbbb1111aaaa2222bbbb-0001",
"blockid": "42b4dd9bfb50ba8f1c4b60229ea4605a",
"heading": "Linked",
"parent_headings": [],
"format": "png",
"path": "https://example.com/diagrams/architecture.png",
"src": "docx://external",
"caption": "",
"footnotes": []
},
"im-1111aaaa2222bbbb1111aaaa2222bbbb-0002": {
"id": "im-1111aaaa2222bbbb1111aaaa2222bbbb-0002",
"blockid": "42b4dd9bfb50ba8f1c4b60229ea4605a",
"heading": "Linked",
"parent_headings": [],
"format": "gif",
"path": "../images/legacy.gif",
"src": "docx://legacy",
"caption": "",
"footnotes": []
}
}
}
tests/parser/docx/golden/native_docx/missing_paraid/legacy.blocks.jsonl
+3
View File
@@ -0,0 +1,3 @@
{"type": "meta", "format": "lightrag", "version": "1.0", "document_name": "legacy.docx", "document_format": "docx", "document_hash": "sha256:1b888273b2772355165b6e6bc69092a35f421fb382e4e1cf5bdb2a92db70ee86", "table_file": false, "equation_file": false, "drawing_file": false, "asset_dir": false, "blocks": 2, "doc_id": "doc-99990000111122229999000011112222", "parse_engine": "native", "parse_time": "2026-01-01T00:00:00+00:00", "doc_title": "legacy"}
{"type": "content", "blockid": "11ffd40d2d43ce0c105a7d0ffe3c803f", "format": "plain_text", "content": "Just plain text without a heading.", "heading": "", "parent_headings": [], "level": 0, "session_type": "body", "table_slice": "none", "positions": [{"type": "paraid", "range": [null, null]}]}
{"type": "content", "blockid": "75c3c3be70534741d0ce0eeecf308502", "format": "plain_text", "content": "Another paragraph with no paraId.", "heading": "", "parent_headings": [], "level": 0, "session_type": "body", "table_slice": "none", "positions": [{"type": "paraid", "range": [null, null]}]}
tests/parser/docx/golden/native_docx/tables_mixed/report.blocks.jsonl
+5
View File
@@ -0,0 +1,5 @@
{"type": "meta", "format": "lightrag", "version": "1.0", "document_name": "report.docx", "document_format": "docx", "document_hash": "sha256:ac2fac1eece1383be12ebb2ca8ace735bfcb3078af0740e22d35508cf8b7421c", "table_file": true, "equation_file": false, "drawing_file": false, "asset_dir": false, "blocks": 4, "doc_id": "doc-cccc333344445555cccc333344445555", "parse_engine": "native", "parse_time": "2026-01-01T00:00:00+00:00", "doc_title": "Report"}
{"type": "content", "blockid": "4ede43f1d15b69ff71c86035d0f7cfac", "format": "plain_text", "content": "# Report", "heading": "Report", "parent_headings": [], "level": 1, "session_type": "body", "table_slice": "none", "positions": [{"type": "paraid", "range": ["h1", "h1"]}]}
{"type": "content", "blockid": "c2f594c8d5882977f79ad8ec6da3338f", "format": "plain_text", "content": "See table:\n<table id=\"tb-cccc333344445555cccc333344445555-0001\" format=\"json\">[[\"X\",\"Y\"],[\"1\",\"2\"],[\"3\",\"4\"]]</table>", "heading": "Report", "parent_headings": [], "level": 1, "session_type": "body", "table_slice": "none", "positions": [{"type": "paraid", "range": ["t1", "t1"]}]}
{"type": "content", "blockid": "bc35d892e4c8f05c9eaf758a4f78878e", "format": "plain_text", "content": "Plain table:\n<table id=\"tb-cccc333344445555cccc333344445555-0002\" format=\"json\">[[\"a\",\"b\"]]</table>", "heading": "Report", "parent_headings": [], "level": 1, "session_type": "body", "table_slice": "none", "positions": [{"type": "paraid", "range": ["t2", "t2"]}]}
{"type": "content", "blockid": "fd0febb81cacf41fe81aea2812bd1bd8", "format": "plain_text", "content": "<table id=\"tb-cccc333344445555cccc333344445555-0003\" format=\"json\">[[\"p\"]]</table>\nthen\n<table id=\"tb-cccc333344445555cccc333344445555-0004\" format=\"json\">[[\"q\",\"r\"],[\"s\",\"t\"]]</table>", "heading": "Report", "parent_headings": [], "level": 1, "session_type": "body", "table_slice": "none", "positions": [{"type": "paraid", "range": ["t3", "t3"]}]}
tests/parser/docx/golden/native_docx/tables_mixed/report.tables.json
+63
View File
@@ -0,0 +1,63 @@
{
"version": "1.0",
"tables": {
"tb-cccc333344445555cccc333344445555-0001": {
"id": "tb-cccc333344445555cccc333344445555-0001",
"blockid": "c2f594c8d5882977f79ad8ec6da3338f",
"heading": "Report",
"parent_headings": [],
"dimension": [
3,
2
],
"format": "json",
"content": "[[\"X\", \"Y\"], [\"1\", \"2\"], [\"3\", \"4\"]]",
"caption": "",
"footnotes": [],
"table_header": "[[\"X\", \"Y\"]]"
},
"tb-cccc333344445555cccc333344445555-0002": {
"id": "tb-cccc333344445555cccc333344445555-0002",
"blockid": "bc35d892e4c8f05c9eaf758a4f78878e",
"heading": "Report",
"parent_headings": [],
"dimension": [
1,
2
],
"format": "json",
"content": "[[\"a\", \"b\"]]",
"caption": "",
"footnotes": []
},
"tb-cccc333344445555cccc333344445555-0003": {
"id": "tb-cccc333344445555cccc333344445555-0003",
"blockid": "fd0febb81cacf41fe81aea2812bd1bd8",
"heading": "Report",
"parent_headings": [],
"dimension": [
1,
1
],
"format": "json",
"content": "[[\"p\"]]",
"caption": "",
"footnotes": []
},
"tb-cccc333344445555cccc333344445555-0004": {
"id": "tb-cccc333344445555cccc333344445555-0004",
"blockid": "fd0febb81cacf41fe81aea2812bd1bd8",
"heading": "Report",
"parent_headings": [],
"dimension": [
2,
2
],
"format": "json",
"content": "[[\"q\", \"r\"], [\"s\", \"t\"]]",
"caption": "",
"footnotes": [],
"table_header": "[[\"q\", \"r\"]]"
}
}
}
tests/parser/docx/golden/native_docx/text_only_hierarchy/paper.blocks.jsonl
+5
View File
@@ -0,0 +1,5 @@
{"type": "meta", "format": "lightrag", "version": "1.0", "document_name": "paper.docx", "document_format": "docx", "document_hash": "sha256:e3fd11b45f12fe0fe0ca22cd6526b7d65336e58cfd2c6585fca4511fa967a55a", "table_file": false, "equation_file": false, "drawing_file": false, "asset_dir": false, "blocks": 4, "doc_id": "doc-aaaa111122223333aaaa111122223333", "parse_engine": "native", "parse_time": "2026-01-01T00:00:00+00:00", "doc_title": "Introduction"}
{"type": "content", "blockid": "5c647a54224deea018b593b7bbef2348", "format": "plain_text", "content": "# Introduction", "heading": "Introduction", "parent_headings": [], "level": 1, "session_type": "body", "table_slice": "none", "positions": [{"type": "paraid", "range": ["h1", "h1"]}]}
{"type": "content", "blockid": "11d7395dfeaf573464e88d3d7f736f5c", "format": "plain_text", "content": "Body paragraph one.", "heading": "Introduction", "parent_headings": [], "level": 1, "session_type": "body", "table_slice": "none", "positions": [{"type": "paraid", "range": ["p1", "p2"]}]}
{"type": "content", "blockid": "7bf205c142f2012fbe586007949c21d1", "format": "plain_text", "content": "## Background", "heading": "Background", "parent_headings": ["Introduction"], "level": 2, "session_type": "body", "table_slice": "none", "positions": [{"type": "paraid", "range": ["h2", "h2"]}]}
{"type": "content", "blockid": "337190582281b148ff9732cb7a04661e", "format": "plain_text", "content": "Sub body.", "heading": "Background", "parent_headings": ["Introduction"], "level": 2, "session_type": "body", "table_slice": "none", "positions": [{"type": "paraid", "range": ["p3", "p3"]}]}
tests/parser/docx/test_ir_builder_security.py
+69
View File
@@ -0,0 +1,69 @@
"""Security-focused tests for native DOCX IR drawing asset handling."""
from __future__ import annotations
import pytest
from lightrag.parser.docx.ir_builder import NativeDocxIRBuilder
def _build_ir(content: str):
return NativeDocxIRBuilder().normalize(
[
{
"uuid": "p1",
"uuid_end": "p1",
"heading": "Section",
"content": content,
"parent_headings": [],
"level": 1,
}
],
document_name="doc.docx",
asset_dir_name="doc.blocks.assets",
)
@pytest.mark.offline
def test_native_docx_ir_accepts_safe_local_drawing_asset() -> None:
ir = _build_ir(
'<drawing id="1" name="fig" format="png" path="doc.blocks.assets/fig.png" />'
)
drawing = ir.blocks[0].drawings[0]
assert drawing.asset_ref == "fig.png"
assert drawing.path_override is None
assert len(ir.assets) == 1
assert ir.assets[0].ref == "fig.png"
assert ir.assets[0].suggested_name == "fig.png"
assert ir.assets[0].source is None
@pytest.mark.offline
@pytest.mark.parametrize(
"path",
[
"doc.blocks.assets/../secret.png",
"doc.blocks.assets//tmp/secret.png",
r"doc.blocks.assets/..\secret.png",
],
)
def test_native_docx_ir_rejects_unsafe_local_drawing_asset(path: str) -> None:
ir = _build_ir(f'<drawing id="1" name="fig" format="png" path="{path}" />')
drawing = ir.blocks[0].drawings[0]
assert drawing.asset_ref == ""
assert drawing.path_override is None
assert ir.assets == []
@pytest.mark.offline
def test_native_docx_ir_preserves_non_asset_external_drawing_path() -> None:
ir = _build_ir(
'<drawing id="1" name="fig" format="gif" path="../images/legacy.gif" />'
)
drawing = ir.blocks[0].drawings[0]
assert drawing.asset_ref == ""
assert drawing.path_override == "../images/legacy.gif"
assert ir.assets == []
tests/parser/docx/test_native_docx_extract.py
+405
View File
@@ -0,0 +1,405 @@
"""Unit tests for native docx tracked-change / comment / empty-table handling.
Locks in the contract that:
- ``w:ins`` content survives (final revised text),
- ``w:del`` / ``w:moveFrom`` content is dropped,
- ``w:commentRangeStart`` / ``w:commentRangeEnd`` / ``w:commentReference`` /
``w:annotationRef`` markers are dropped,
- tables whose every cell is whitespace-only are omitted from the parser
output (no ``<table>`` placeholder, so no IRTable downstream).
These were previously emergent properties (skip lists in two places + a
run-level white-list); regressions now fail loudly.
"""
from __future__ import annotations
from io import BytesIO
import pytest
from docx import Document
from docx.oxml import OxmlElement
from docx.oxml.ns import qn
from lxml import etree
from lightrag.parser.docx.parse_document import (
DocxContentError,
extract_docx_blocks,
extract_paragraph_content,
)
from lightrag.parser.docx.table_extractor import extract_paragraph_content_table
W_NS = "http://schemas.openxmlformats.org/wordprocessingml/2006/main"
PARAGRAPH_NS = {
"w": W_NS,
"wp": "http://schemas.openxmlformats.org/drawingml/2006/wordprocessingDrawing",
"m": "http://schemas.openxmlformats.org/officeDocument/2006/math",
}
def _p(inner_xml: str):
"""Build a ``<w:p>`` lxml element from inner OOXML."""
return etree.fromstring(f'<w:p xmlns:w="{W_NS}">{inner_xml}</w:p>'.encode("utf-8"))
# --- paragraph walker (parse_document.extract_paragraph_content) -----------
@pytest.mark.offline
def test_paragraph_keeps_w_ins_content() -> None:
para = _p("<w:ins><w:r><w:t>kept</w:t></w:r></w:ins>")
assert extract_paragraph_content(para, PARAGRAPH_NS) == "kept"
@pytest.mark.offline
def test_paragraph_drops_w_del_content() -> None:
para = _p("<w:del><w:r><w:t>removed</w:t></w:r></w:del>")
assert extract_paragraph_content(para, PARAGRAPH_NS) == ""
@pytest.mark.offline
def test_paragraph_drops_w_movefrom_content() -> None:
para = _p("<w:moveFrom><w:r><w:t>moved away</w:t></w:r></w:moveFrom>")
assert extract_paragraph_content(para, PARAGRAPH_NS) == ""
@pytest.mark.offline
def test_paragraph_drops_comment_markers_but_keeps_surrounding_text() -> None:
para = _p(
'<w:commentRangeStart w:id="0"/>'
"<w:r><w:t>visible</w:t></w:r>"
'<w:commentRangeEnd w:id="0"/>'
"<w:r>"
' <w:rPr><w:rStyle w:val="CommentReference"/></w:rPr>'
' <w:commentReference w:id="0"/>'
"</w:r>"
)
assert extract_paragraph_content(para, PARAGRAPH_NS) == "visible"
# --- table-cell walker (table_extractor.extract_paragraph_content_table) ---
@pytest.mark.offline
def test_table_cell_keeps_w_ins_content() -> None:
para = _p("<w:ins><w:r><w:t>kept</w:t></w:r></w:ins>")
assert extract_paragraph_content_table(para, qn) == "kept"
@pytest.mark.offline
def test_table_cell_drops_w_del_content() -> None:
para = _p("<w:del><w:r><w:t>removed</w:t></w:r></w:del>")
assert extract_paragraph_content_table(para, qn) == ""
@pytest.mark.offline
def test_table_cell_drops_comment_markers() -> None:
para = _p(
'<w:commentRangeStart w:id="0"/>'
"<w:r><w:t>cell-text</w:t></w:r>"
'<w:commentRangeEnd w:id="0"/>'
)
assert extract_paragraph_content_table(para, qn) == "cell-text"
# --- end-to-end: empty tables vanish from blocks output --------------------
def _populate_cell(cell, text: str) -> None:
cell.paragraphs[0].text = text
def _build_docx_with_three_tables() -> BytesIO:
"""One real table, one all-empty table, one whitespace-only table.
Returns a BytesIO containing the rendered .docx so it can be fed to
``extract_docx_blocks`` via a tempfile-backed path or directly.
"""
doc = Document()
doc.add_paragraph("intro")
real = doc.add_table(rows=1, cols=2)
_populate_cell(real.rows[0].cells[0], "A")
_populate_cell(real.rows[0].cells[1], "B")
empty = doc.add_table(rows=2, cols=2)
# leave every cell at python-docx default (empty paragraph)
assert all(c.text == "" for row in empty.rows for c in row.cells)
whitespace = doc.add_table(rows=1, cols=2)
_populate_cell(whitespace.rows[0].cells[0], " ")
_populate_cell(whitespace.rows[0].cells[1], "\t")
buf = BytesIO()
doc.save(buf)
buf.seek(0)
return buf
# --- end-to-end: every heading becomes its own block ----------------------
def _add_heading(doc, text: str, level: int) -> None:
"""Append a heading paragraph with an explicit ``w:outlineLvl``.
Setting outlineLvl directly on the paragraph (rather than relying on the
template's built-in Heading styles) keeps ``get_heading_level`` detection
deterministic. ``level`` is 1-based (1 = H1); outlineLvl val is 0-based.
"""
para = doc.add_paragraph(text)
pPr = para._p.get_or_add_pPr()
outline = OxmlElement("w:outlineLvl")
outline.set(qn("w:val"), str(level - 1))
pPr.append(outline)
@pytest.mark.offline
def test_each_heading_becomes_its_own_block(tmp_path) -> None:
"""Headings with no body must each form a standalone block.
Layout: H1 (no body) → H2 (no body) → H3 (with body) → H1 (no body, last).
Previously the empty H1/H2 were folded into the next heading's block; now
every recognized heading starts its own block. A heading with no following
body yields a block whose ``content`` is just the heading text, while a
heading with body merges that body into the same block. ``parent_headings``
must track the outline hierarchy correctly.
"""
doc = Document()
_add_heading(doc, "Chapter One", level=1)
_add_heading(doc, "Section 1.1", level=2)
_add_heading(doc, "Subsection 1.1.1", level=3)
doc.add_paragraph("Body text under subsection.")
_add_heading(doc, "Appendix", level=1)
buf = BytesIO()
doc.save(buf)
docx_path = tmp_path / "headings.docx"
docx_path.write_bytes(buf.getvalue())
# The native parser splits at every heading level and never does
# token-based splitting or small-block merging (that is the chunker's job).
blocks = extract_docx_blocks(str(docx_path))
# The content line carries a markdown ``#`` prefix matching the level,
# while the ``heading`` field stays clean (no prefix).
summary = [
(b["heading"], b["content"], b["level"], b["parent_headings"]) for b in blocks
]
assert summary == [
("Chapter One", "# Chapter One", 1, []),
("Section 1.1", "## Section 1.1", 2, ["Chapter One"]),
(
"Subsection 1.1.1",
"### Subsection 1.1.1\nBody text under subsection.",
3,
["Chapter One", "Section 1.1"],
),
("Appendix", "# Appendix", 1, []),
]
@pytest.mark.offline
def test_heading_markdown_prefix_capped_at_six(tmp_path) -> None:
"""Heading content lines get a markdown ``#`` prefix matching the level,
capped at 6 ``#`` (a level-7 heading still renders ``######``)."""
doc = Document()
_add_heading(doc, "Top", level=1)
doc.add_paragraph("body under top.")
_add_heading(doc, "Deep Seven", level=7)
doc.add_paragraph("body under deep.")
buf = BytesIO()
doc.save(buf)
docx_path = tmp_path / "deep.docx"
docx_path.write_bytes(buf.getvalue())
blocks = extract_docx_blocks(str(docx_path))
summary = [(b["heading"], b["content"].split("\n")[0], b["level"]) for b in blocks]
assert summary == [
("Top", "# Top", 1),
# level 7 outline → clamped to six "#".
("Deep Seven", "###### Deep Seven", 7),
]
def _add_outline_para_with_softbreak(doc, head_text: str, body_text: str, level: int):
"""Append one ``<w:p>`` carrying an outline level whose runs are
``head_text`` + a soft line break (``<w:br/>``) + ``body_text``.
Mirrors the real-world WPS/Word pattern where an author sets an outline
level on a paragraph but types the body with Shift+Enter (a soft break)
instead of a new paragraph, so heading + body live in a single paragraph.
"""
para = doc.add_paragraph(head_text)
pPr = para._p.get_or_add_pPr()
outline = OxmlElement("w:outlineLvl")
outline.set(qn("w:val"), str(level - 1))
pPr.append(outline)
para.add_run().add_break() # default WD_BREAK.LINE → soft <w:br/> → '\n'
para.add_run(body_text)
return para
@pytest.mark.offline
def test_oversize_outline_heading_with_softbreak_splits(tmp_path) -> None:
"""An over-long heading paragraph that contains a soft break must split:
the first line stays the heading, the remainder becomes body text. No
DocxContentError, and the body content is preserved in full."""
head = "A、处置程序"
body = "施工现场一旦发生事故时应立即采取相应的应急处置措施并以最快速度报警。" * 8
assert len(head) + 1 + len(body) > 200 # would trip validate_heading_length
doc = Document()
_add_outline_para_with_softbreak(doc, head, body, level=4)
buf = BytesIO()
doc.save(buf)
docx_path = tmp_path / "softbreak.docx"
docx_path.write_bytes(buf.getvalue())
blocks = extract_docx_blocks(str(docx_path))
# The short first line becomes the heading; the long remainder is body.
heading_block = next(b for b in blocks if b["heading"] == head)
assert body in heading_block["content"]
# Heading field carries only the first line, not the body.
assert body not in heading_block["heading"]
@pytest.mark.offline
def test_oversize_outline_heading_no_softbreak_demoted_to_body(tmp_path) -> None:
"""A single-line over-long outline paragraph (no soft break) is demoted to
body text rather than raising — content preserved, not treated as a heading."""
long_text = "施工现场应急处置措施说明，" * 30 # well over 200 chars, no '\n'
assert len(long_text) > 200
doc = Document()
_add_heading(doc, long_text, level=2)
buf = BytesIO()
doc.save(buf)
docx_path = tmp_path / "demote.docx"
docx_path.write_bytes(buf.getvalue())
blocks = extract_docx_blocks(str(docx_path))
# The text survives as body content, and no block adopts it as a heading.
assert any(long_text in b["content"] for b in blocks)
assert all(b["heading"] != long_text for b in blocks)
@pytest.mark.offline
def test_short_outline_paragraph_still_heading(tmp_path) -> None:
"""A within-limit outline paragraph must still be recognized as a heading —
the over-long handling never demotes short headings."""
doc = Document()
_add_heading(doc, "处置程序", level=3)
doc.add_paragraph("body under heading.")
buf = BytesIO()
doc.save(buf)
docx_path = tmp_path / "short.docx"
docx_path.write_bytes(buf.getvalue())
blocks = extract_docx_blocks(str(docx_path))
heading_block = next(b for b in blocks if b["heading"] == "处置程序")
assert heading_block["level"] == 3
assert heading_block["content"].startswith("### 处置程序")
@pytest.mark.offline
def test_existing_markdown_heading_keeps_content_but_metadata_is_clean(
tmp_path,
) -> None:
doc = Document()
_add_heading(doc, "# Already MD", level=1)
doc.add_paragraph("Body.")
buf = BytesIO()
doc.save(buf)
docx_path = tmp_path / "markdown-heading.docx"
docx_path.write_bytes(buf.getvalue())
parse_metadata: dict[str, str] = {}
blocks = extract_docx_blocks(str(docx_path), parse_metadata=parse_metadata)
assert parse_metadata["first_heading"] == "Already MD"
assert blocks[0]["heading"] == "Already MD"
assert blocks[0]["parent_headings"] == []
assert blocks[0]["content"].splitlines()[0] == "# Already MD"
@pytest.mark.offline
def test_empty_tables_are_skipped(tmp_path) -> None:
docx_bytes = _build_docx_with_three_tables()
docx_path = tmp_path / "three_tables.docx"
docx_path.write_bytes(docx_bytes.getvalue())
blocks = extract_docx_blocks(str(docx_path))
table_placeholders = sum(b["content"].count("<table>") for b in blocks)
assert table_placeholders == 1, (
"exactly one real table should survive; empty + whitespace tables "
"must be dropped before the placeholder is emitted"
)
assert any('"A"' in b["content"] and '"B"' in b["content"] for b in blocks)
# --- invalid (non-ZIP) .docx files surface an accurate error ---------------
@pytest.mark.offline
@pytest.mark.parametrize(
("header", "format_hint"),
[
(b"\xd0\xcf\x11\xe0\xa1\xb1\x1a\xe1" + b"junk", "Word 97-2003"),
(b"{\\rtf1\\ansi}", "RTF"),
(b"%PDF-1.7\n%binary", "PDF"),
(b"<?xml version='1.0'?><html></html>", "HTML or XML"),
(b"plain text masquerading as docx", "not a valid DOCX"),
],
)
def test_invalid_docx_raises_accurate_error(tmp_path, header, format_hint) -> None:
"""A file that exists but is not a ZIP/OOXML package must fail with an
accurate message — never the misleading python-docx "Package not found".
The native worker confirms the file exists before parsing, so the only way
``PackageNotFoundError`` can fire here is a corrupt file or a non-DOCX
payload wearing a .docx extension. The error must name the real format and
keep the file path.
"""
docx_path = tmp_path / "masquerade.docx"
docx_path.write_bytes(header)
with pytest.raises(DocxContentError) as excinfo:
extract_docx_blocks(str(docx_path))
message = str(excinfo.value)
assert "Package not found" not in message
assert "valid DOCX" in message
assert str(docx_path) in message
assert format_hint in message
@pytest.mark.offline
def test_empty_docx_file_reports_truncation(tmp_path) -> None:
"""A 0-byte .docx must be diagnosed as empty/truncated, not 'not found'."""
docx_path = tmp_path / "empty.docx"
docx_path.write_bytes(b"")
with pytest.raises(DocxContentError) as excinfo:
extract_docx_blocks(str(docx_path))
message = str(excinfo.value)
assert "Package not found" not in message
assert "empty" in message
assert str(docx_path) in message
tests/parser/docx/test_native_docx_golden.py
+165
View File
@@ -0,0 +1,165 @@
"""Byte-equivalence golden tests for the native docx → SidecarWriter migration.
These tests run the production code path (``LightRAG.parse_native``) on
each scenario in ``_native_docx_fixtures.SCENARIOS`` and assert that
every produced file matches the captured baseline bytes under
``tests/parser/docx/golden/native_docx/<scenario>/``.
The baseline was generated by ``scripts/regen_native_docx_golden.py``;
the script lives in-tree so the snapshots can be regenerated when
intentional changes to the format are made (e.g. spec evolution).
"""
from __future__ import annotations
import asyncio
import sys
from pathlib import Path
from unittest import mock
import pytest
# Make the sibling _native_docx_fixtures module importable as if it were a
# top-level helper. Avoids needing a package __init__ rename.
HERE = Path(__file__).resolve().parent
if str(HERE) not in sys.path:
sys.path.insert(0, str(HERE))
from _native_docx_fixtures import SCENARIOS, Scenario # noqa: E402
from lightrag.parser.base import ParseContext # noqa: E402
from lightrag.parser.debug import ( # noqa: E402
FrozenDateTime,
build_debug_rag,
)
from lightrag.parser.registry import get_parser # noqa: E402
GOLDEN_ROOT = HERE / "golden" / "native_docx"
def _run_new_path(
scenario: Scenario, input_dir: Path, monkeypatch: pytest.MonkeyPatch
) -> Path:
"""Invoke ``LightRAG.parse_native`` on a single scenario.
The upstream ``extract_docx_blocks`` is stubbed to feed the synthetic
blocks and produce the matching asset files inside
``<base>.blocks.assets/``. ``datetime.now`` is frozen so ``parse_time``
matches the captured baseline exactly.
Returns the parsed directory containing the produced artifacts.
"""
from lightrag.constants import (
FULL_DOCS_FORMAT_PENDING_PARSE,
PARSED_DIR_NAME,
)
monkeypatch.setenv("INPUT_DIR", str(input_dir))
# parse_native archives the source after parsing; for the golden test
# we don't want the docx moving around between scenarios.
async def _noop_archive(_p: str) -> None:
return None
import lightrag.pipeline as pipeline_module
monkeypatch.setattr(
pipeline_module, "archive_docx_source_after_full_docs_sync", _noop_archive
)
source_path = input_dir / scenario.file_path
source_path.parent.mkdir(parents=True, exist_ok=True)
source_path.write_bytes(b"fake-docx")
def _stub_extract(
file_path,
*,
drawing_context=None,
parse_warnings=None,
parse_metadata=None,
**_kwargs,
):
if drawing_context is not None and scenario.assets:
drawing_context.export_dir_path.mkdir(parents=True, exist_ok=True)
for name, data in scenario.assets.items():
(drawing_context.export_dir_path / name).write_bytes(data)
if parse_metadata is not None:
parse_metadata.update(scenario.parse_metadata)
return [dict(b) for b in scenario.blocks]
rag = build_debug_rag()
with (
mock.patch(
"lightrag.parser.docx.parse_document.extract_docx_blocks",
_stub_extract,
),
mock.patch("lightrag.sidecar.writer.datetime", FrozenDateTime),
):
async def _go() -> None:
await get_parser("native").parse(
ParseContext(
rag,
scenario.doc_id,
str(source_path),
{
"parse_format": FULL_DOCS_FORMAT_PENDING_PARSE,
"content": "",
},
)
)
asyncio.run(_go())
return input_dir / PARSED_DIR_NAME / f"{scenario.file_path}.parsed"
def _read_bytes(path: Path) -> bytes:
with path.open("rb") as fh:
return fh.read()
@pytest.mark.offline
@pytest.mark.parametrize(
"scenario",
SCENARIOS,
ids=[s.name for s in SCENARIOS],
)
def test_native_docx_migration_is_byte_equivalent(
scenario: Scenario,
tmp_path: Path,
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""Every file produced by parse_native must be byte-identical to the
captured legacy baseline."""
input_dir = tmp_path / "inputs"
input_dir.mkdir()
parsed_dir = _run_new_path(scenario, input_dir, monkeypatch)
expected_root = GOLDEN_ROOT / scenario.name
assert expected_root.is_dir(), (
f"missing golden fixture for scenario {scenario.name!r}; "
f"run scripts/regen_native_docx_golden.py to regenerate"
)
# Collect both file sets relative to their roots, then compare.
expected_files = {
p.relative_to(expected_root): p for p in expected_root.rglob("*") if p.is_file()
}
produced_files = {
p.relative_to(parsed_dir): p for p in parsed_dir.rglob("*") if p.is_file()
}
extra = sorted(produced_files.keys() - expected_files.keys())
missing = sorted(expected_files.keys() - produced_files.keys())
assert not extra, f"unexpected produced files: {extra}"
assert not missing, f"missing produced files (legacy had them): {missing}"
mismatches: list[str] = []
for rel, expected_path in expected_files.items():
produced_path = produced_files[rel]
if _read_bytes(produced_path) != _read_bytes(expected_path):
mismatches.append(str(rel))
assert not mismatches, (
f"byte mismatch in scenario {scenario.name!r} for files: {mismatches}"
)
tests/parser/docx/test_validation_raises.py
+45
View File
@@ -0,0 +1,45 @@
"""Native DOCX content-limit validation must *raise*, never ``sys.exit``.
The DOCX parser runs in-process inside the gunicorn/uvicorn worker (via
``LightRAG.parse_native`` → ``extract_docx_blocks``). A ``sys.exit`` there
raises ``SystemExit`` (a ``BaseException``), which slips past the pipeline's
per-document ``except Exception`` handler and tears down the whole worker —
gunicorn then reports ``WORKER TIMEOUT`` and kills it with SIGABRT (code 134),
interrupting the entire processing pipeline.
These tests lock in that the content-limit checks raise ``DocxContentError``
(an ``Exception`` subclass) so just the offending document fails while the
worker keeps running.
"""
from __future__ import annotations
import pytest
from lightrag.parser.docx.parse_document import (
DocxContentError,
MAX_HEADING_LENGTH,
validate_heading_length,
)
def test_docx_content_error_is_an_exception_not_just_baseexception():
# The pipeline parse worker catches ``except Exception``; SystemExit
# (BaseException) would bypass it. Guard against a future regression that
# reintroduces a BaseException-only error type.
assert issubclass(DocxContentError, Exception)
def test_validate_heading_length_raises_instead_of_exiting():
long_heading = "x" * (MAX_HEADING_LENGTH + 213) # mirrors the 413-char report
with pytest.raises(DocxContentError) as exc_info:
validate_heading_length(long_heading, "p1")
message = str(exc_info.value)
assert f"Heading too long ({len(long_heading)} characters" in message
assert "SOLUTION:" in message
def test_validate_heading_length_does_not_raise_within_limit():
# Exactly at the limit and below must pass through silently.
validate_heading_length("x" * MAX_HEADING_LENGTH, "p2")
validate_heading_length("short heading", "p3")