chore: import upstream snapshot with attribution

This commit is contained in:
wehub-resource-sync
2026-07-13 12:08:54 +08:00
commit 4a4a1fed67
721 changed files with 262090 additions and 0 deletions
View File
View File
+329
View File
@@ -0,0 +1,329 @@
"""Scenario fixtures for the native docx → SidecarWriter migration tests.
Each scenario describes:
- ``blocks`` — what ``extract_docx_blocks`` would return (the synthetic
block dicts that the adapter consumes).
- ``parse_metadata`` — the dict the upstream parser fills in (only
``first_heading`` is currently consumed by the adapter).
- ``assets`` — files the upstream extractor would have written into
``<base>.blocks.assets/`` before the IR builder runs. Maps relative names
inside the asset dir → byte content.
- ``doc_id`` — fixed so blockid + sidecar ids are deterministic.
- ``file_path`` — used for canonical basename / doc_title fallback.
The captured outputs (``blocks.jsonl`` + per-modality JSONs + assets) live
under ``tests/parser/docx/golden/native_docx/<scenario>/``. The
production path (``LightRAG.parse_native``) must produce byte-identical
bytes vs those fixtures; the regen script under ``scripts/`` rewrites
them when the format intentionally changes.
"""
from __future__ import annotations
from dataclasses import dataclass, field
from typing import Any
def _block(
content: str,
*,
heading: str = "",
level: int = 0,
parent: list[str] | None = None,
uuid: str = "p1",
uuid_end: str | None = None,
table_headers: list[Any] | None = None,
table_chunk_role: str = "none",
) -> dict[str, Any]:
"""Build a synthetic block matching ``extract_docx_blocks`` output."""
out: dict[str, Any] = {
"uuid": uuid,
"uuid_end": uuid_end if uuid_end is not None else uuid,
"heading": heading,
"content": content,
"type": "text",
"parent_headings": list(parent or []),
"level": level,
"table_chunk_role": table_chunk_role,
}
if table_headers is not None:
out["table_headers"] = table_headers
return out
@dataclass
class Scenario:
name: str
doc_id: str
file_path: str # canonical-ish; what the pipeline would pass
blocks: list[dict[str, Any]]
parse_metadata: dict[str, Any] = field(default_factory=dict)
assets: dict[str, bytes] = field(default_factory=dict)
SCENARIOS: list[Scenario] = [
# --- 1: text-only, multi-heading -----------------------------------
Scenario(
name="text_only_hierarchy",
doc_id="doc-aaaa111122223333aaaa111122223333",
file_path="paper.docx",
parse_metadata={"first_heading": "Introduction"},
blocks=[
_block(
"# Introduction",
heading="Introduction",
level=1,
uuid="h1",
),
_block(
"Body paragraph one.",
heading="Introduction",
level=1,
uuid="p1",
uuid_end="p2",
),
_block(
"## Background",
heading="Background",
level=2,
parent=["Introduction"],
uuid="h2",
),
_block(
"Sub body.",
heading="Background",
level=2,
parent=["Introduction"],
uuid="p3",
),
],
),
# --- 2: block + inline equations -----------------------------------
Scenario(
name="equations_block_and_inline",
doc_id="doc-bbbb222233334444bbbb222233334444",
file_path="formulas.docx",
parse_metadata={"first_heading": "Equations"},
blocks=[
_block(
"# Equations",
heading="Equations",
level=1,
uuid="h1",
),
_block(
# Inline equation (no surrounding \n on either side)
"Energy is <equation>E=mc^2</equation> per Einstein.",
heading="Equations",
level=1,
uuid="p1",
),
_block(
# Block equation (wedged between newlines)
"Consider:\n<equation>x^2 + y^2 = r^2</equation>\nThe circle equation.",
heading="Equations",
level=1,
uuid="p2",
),
_block(
# Block at content edge (start == 0)
"<equation>a + b = c</equation>\ntext after",
heading="Equations",
level=1,
uuid="p3",
),
],
),
# --- 3: tables with and without table_headers ----------------------
Scenario(
name="tables_mixed",
doc_id="doc-cccc333344445555cccc333344445555",
file_path="report.docx",
parse_metadata={"first_heading": "Report"},
blocks=[
_block(
"# Report",
heading="Report",
level=1,
uuid="h1",
),
_block(
# Table with table_headers (cross-page repeating)
'See table:\n<table>[["X","Y"],["1","2"],["3","4"]]</table>',
heading="Report",
level=1,
uuid="t1",
table_headers=[[["X", "Y"]]], # one table, one header row
),
_block(
# Table without table_headers
'Plain table:\n<table>[["a","b"]]</table>',
heading="Report",
level=1,
uuid="t2",
),
_block(
# Two tables in one block
'<table>[["p"]]</table>\nthen\n<table>[["q","r"],["s","t"]]</table>',
heading="Report",
level=1,
uuid="t3",
table_headers=[None, [["q", "r"]]],
),
],
),
# --- 4: drawings + assets ------------------------------------------
Scenario(
name="drawings_with_assets",
doc_id="doc-dddd444455556666dddd444455556666",
file_path="diagrams.docx",
parse_metadata={"first_heading": "Diagrams"},
assets={
"fig1.png": b"\x89PNG\r\n\x1a\n-fig1-fake",
"fig2.jpg": b"\xff\xd8\xff\xe0-fig2-fake",
},
blocks=[
_block(
"# Diagrams",
heading="Diagrams",
level=1,
uuid="h1",
),
_block(
"Figure one:\n"
'<drawing id="x" format="png" '
'path="diagrams.blocks.assets/fig1.png" '
'src="docx://image1" />\n'
"Figure two:\n"
'<drawing id="y" format="jpg" '
'path="diagrams.blocks.assets/fig2.jpg" '
'src="docx://image2" />',
heading="Diagrams",
level=1,
uuid="p1",
),
],
),
# --- 5: all modalities mixed ---------------------------------------
Scenario(
name="all_modalities",
doc_id="doc-eeee555566667777eeee555566667777",
file_path="combo.docx",
parse_metadata={"first_heading": "Combined"},
assets={"pic.png": b"PNG-combo"},
blocks=[
_block(
"# Combined",
heading="Combined",
level=1,
uuid="h1",
),
_block(
"Look at this figure:\n"
'<drawing id="z" format="png" '
'path="combo.blocks.assets/pic.png" '
'src="docx://img" />\n'
"Plus a table:\n"
'<table>[["α","β"],["γ","δ"]]</table>\n'
"And a block equation:\n"
"<equation>F = ma</equation>\n"
"And an inline <equation>v=d/t</equation> here.",
heading="Combined",
level=1,
uuid="p1",
),
],
),
# --- 6: empty block dropped ----------------------------------------
Scenario(
name="empty_block_dropped",
doc_id="doc-ffff666677778888ffff666677778888",
file_path="sparse.docx",
parse_metadata={"first_heading": "Sparse"},
blocks=[
_block(
"# Sparse",
heading="Sparse",
level=1,
uuid="h1",
),
_block(
" \n ", # strips to empty — must be dropped
heading="Sparse",
level=1,
uuid="p_empty",
),
_block(
"Real content after empty.",
heading="Sparse",
level=1,
uuid="p_real",
),
],
),
# --- 7: external / linked image references ------------------------
# DOCX can carry ``<a:blip r:link="rId…"/>`` references to image
# targets that live outside the package — the upstream extractor
# then emits ``<drawing path="<external URL or unresolved path>" />``
# WITHOUT writing bytes into ``<base>.blocks.assets/``. The adapter
# must pass those paths through verbatim (both in ``blocks.jsonl``
# and ``drawings.json``); turning them into AssetSpecs with
# ``source=None`` would make the writer warn-and-skip → ``path=""``,
# losing the only reference downstream consumers have.
Scenario(
name="external_image_link",
doc_id="doc-1111aaaa2222bbbb1111aaaa2222bbbb",
file_path="linked.docx",
parse_metadata={"first_heading": "Linked"},
# No on-disk assets — the path points elsewhere.
assets={},
blocks=[
_block(
"# Linked",
heading="Linked",
level=1,
uuid="h1",
),
_block(
"See the diagram online:\n"
'<drawing id="z" format="png" '
'path="https://example.com/diagrams/architecture.png" '
'src="docx://external" />\n'
"And a relative-but-not-asset path:\n"
'<drawing id="z2" format="gif" '
'path="../images/legacy.gif" '
'src="docx://legacy" />',
heading="Linked",
level=1,
uuid="p1",
),
],
),
# --- 8: missing paraid ---------------------------------------------
Scenario(
name="missing_paraid",
doc_id="doc-99990000111122229999000011112222",
file_path="legacy.docx",
parse_metadata={"first_heading": ""}, # no headings at all
blocks=[
_block(
"Just plain text without a heading.",
heading="",
level=0,
uuid="", # missing
uuid_end="",
),
_block(
"Another paragraph with no paraId.",
heading="",
level=0,
uuid="",
uuid_end="",
),
],
),
]
__all__ = ["Scenario", "SCENARIOS", "_block"]
@@ -0,0 +1,3 @@
{"type": "meta", "format": "lightrag", "version": "1.0", "document_name": "combo.docx", "document_format": "docx", "document_hash": "sha256:2aa5d36cf1126af704cc0b7b924e1b8d9eb8fa576b19807f9fb1fbc80d0fefb7", "table_file": true, "equation_file": true, "drawing_file": true, "asset_dir": true, "blocks": 2, "doc_id": "doc-eeee555566667777eeee555566667777", "parse_engine": "native", "parse_time": "2026-01-01T00:00:00+00:00", "doc_title": "Combined"}
{"type": "content", "blockid": "bea7fc7932485628877517b6b5a0e80c", "format": "plain_text", "content": "# Combined", "heading": "Combined", "parent_headings": [], "level": 1, "session_type": "body", "table_slice": "none", "positions": [{"type": "paraid", "range": ["h1", "h1"]}]}
{"type": "content", "blockid": "8caba9870df85b9357e9521a70d82225", "format": "plain_text", "content": "Look at this figure:\n<drawing id=\"im-eeee555566667777eeee555566667777-0001\" format=\"png\" path=\"pic.png\" src=\"docx://img\" />\nPlus a table:\n<table id=\"tb-eeee555566667777eeee555566667777-0001\" format=\"json\">[[\"α\",\"β\"],[\"γ\",\"δ\"]]</table>\nAnd a block equation:\n<equation id=\"eq-eeee555566667777eeee555566667777-0001\" format=\"latex\">F = ma</equation>\nAnd an inline <equation format=\"latex\">v=d/t</equation> here.", "heading": "Combined", "parent_headings": [], "level": 1, "session_type": "body", "table_slice": "none", "positions": [{"type": "paraid", "range": ["p1", "p1"]}]}
@@ -0,0 +1,16 @@
{
"version": "1.0",
"drawings": {
"im-eeee555566667777eeee555566667777-0001": {
"id": "im-eeee555566667777eeee555566667777-0001",
"blockid": "8caba9870df85b9357e9521a70d82225",
"heading": "Combined",
"parent_headings": [],
"format": "png",
"path": "combo.blocks.assets/pic.png",
"src": "docx://img",
"caption": "",
"footnotes": []
}
}
}
@@ -0,0 +1,15 @@
{
"version": "1.0",
"equations": {
"eq-eeee555566667777eeee555566667777-0001": {
"id": "eq-eeee555566667777eeee555566667777-0001",
"blockid": "8caba9870df85b9357e9521a70d82225",
"heading": "Combined",
"parent_headings": [],
"format": "latex",
"content": "F = ma",
"caption": "",
"footnotes": []
}
}
}
@@ -0,0 +1,19 @@
{
"version": "1.0",
"tables": {
"tb-eeee555566667777eeee555566667777-0001": {
"id": "tb-eeee555566667777eeee555566667777-0001",
"blockid": "8caba9870df85b9357e9521a70d82225",
"heading": "Combined",
"parent_headings": [],
"dimension": [
2,
2
],
"format": "json",
"content": "[[\"α\", \"β\"], [\"γ\", \"δ\"]]",
"caption": "",
"footnotes": []
}
}
}
@@ -0,0 +1,3 @@
‰PNG

-fig1-fake

After

Width:  |  Height:  |  Size: 18 B

@@ -0,0 +1 @@
-fig2-fake

After

Width:  |  Height:  |  Size: 14 B

@@ -0,0 +1,3 @@
{"type": "meta", "format": "lightrag", "version": "1.0", "document_name": "diagrams.docx", "document_format": "docx", "document_hash": "sha256:9ad9b5e89e06783368565eda8ff1cd44e6b2f9a4d678f30e204b7c4984da524e", "table_file": false, "equation_file": false, "drawing_file": true, "asset_dir": true, "blocks": 2, "doc_id": "doc-dddd444455556666dddd444455556666", "parse_engine": "native", "parse_time": "2026-01-01T00:00:00+00:00", "doc_title": "Diagrams"}
{"type": "content", "blockid": "14a79f4f059cb16effe2d6cfe5a18e9b", "format": "plain_text", "content": "# Diagrams", "heading": "Diagrams", "parent_headings": [], "level": 1, "session_type": "body", "table_slice": "none", "positions": [{"type": "paraid", "range": ["h1", "h1"]}]}
{"type": "content", "blockid": "d10a9ed61f86247aeedfefb785c919b3", "format": "plain_text", "content": "Figure one:\n<drawing id=\"im-dddd444455556666dddd444455556666-0001\" format=\"png\" path=\"fig1.png\" src=\"docx://image1\" />\nFigure two:\n<drawing id=\"im-dddd444455556666dddd444455556666-0002\" format=\"jpg\" path=\"fig2.jpg\" src=\"docx://image2\" />", "heading": "Diagrams", "parent_headings": [], "level": 1, "session_type": "body", "table_slice": "none", "positions": [{"type": "paraid", "range": ["p1", "p1"]}]}
@@ -0,0 +1,27 @@
{
"version": "1.0",
"drawings": {
"im-dddd444455556666dddd444455556666-0001": {
"id": "im-dddd444455556666dddd444455556666-0001",
"blockid": "d10a9ed61f86247aeedfefb785c919b3",
"heading": "Diagrams",
"parent_headings": [],
"format": "png",
"path": "diagrams.blocks.assets/fig1.png",
"src": "docx://image1",
"caption": "",
"footnotes": []
},
"im-dddd444455556666dddd444455556666-0002": {
"id": "im-dddd444455556666dddd444455556666-0002",
"blockid": "d10a9ed61f86247aeedfefb785c919b3",
"heading": "Diagrams",
"parent_headings": [],
"format": "jpg",
"path": "diagrams.blocks.assets/fig2.jpg",
"src": "docx://image2",
"caption": "",
"footnotes": []
}
}
}
@@ -0,0 +1,3 @@
{"type": "meta", "format": "lightrag", "version": "1.0", "document_name": "sparse.docx", "document_format": "docx", "document_hash": "sha256:9f42b2b11e8a06b78fbd7fd815f2d800027d33c2bd6f1aa0f92feab32a37a0cb", "table_file": false, "equation_file": false, "drawing_file": false, "asset_dir": false, "blocks": 2, "doc_id": "doc-ffff666677778888ffff666677778888", "parse_engine": "native", "parse_time": "2026-01-01T00:00:00+00:00", "doc_title": "Sparse"}
{"type": "content", "blockid": "1fb5dc88cfc19f236d324df359829044", "format": "plain_text", "content": "# Sparse", "heading": "Sparse", "parent_headings": [], "level": 1, "session_type": "body", "table_slice": "none", "positions": [{"type": "paraid", "range": ["h1", "h1"]}]}
{"type": "content", "blockid": "39f44666ee7d52f6fc680bad4f93ffa8", "format": "plain_text", "content": "Real content after empty.", "heading": "Sparse", "parent_headings": [], "level": 1, "session_type": "body", "table_slice": "none", "positions": [{"type": "paraid", "range": ["p_real", "p_real"]}]}
@@ -0,0 +1,5 @@
{"type": "meta", "format": "lightrag", "version": "1.0", "document_name": "formulas.docx", "document_format": "docx", "document_hash": "sha256:403a469531b150e2c1c14a3728ea6d2e3b48b3c86cf4d4896b94d5c0d6d07fcb", "table_file": false, "equation_file": true, "drawing_file": false, "asset_dir": false, "blocks": 4, "doc_id": "doc-bbbb222233334444bbbb222233334444", "parse_engine": "native", "parse_time": "2026-01-01T00:00:00+00:00", "doc_title": "Equations"}
{"type": "content", "blockid": "49b6d85e4de7d0a8a8024d68b297885d", "format": "plain_text", "content": "# Equations", "heading": "Equations", "parent_headings": [], "level": 1, "session_type": "body", "table_slice": "none", "positions": [{"type": "paraid", "range": ["h1", "h1"]}]}
{"type": "content", "blockid": "3caba68eda5e6f4f3288d3c8a338b76e", "format": "plain_text", "content": "Energy is <equation format=\"latex\">E=mc^2</equation> per Einstein.", "heading": "Equations", "parent_headings": [], "level": 1, "session_type": "body", "table_slice": "none", "positions": [{"type": "paraid", "range": ["p1", "p1"]}]}
{"type": "content", "blockid": "603a388c6d9c45900dcbde3a86a0c3b5", "format": "plain_text", "content": "Consider:\n<equation id=\"eq-bbbb222233334444bbbb222233334444-0001\" format=\"latex\">x^2 + y^2 = r^2</equation>\nThe circle equation.", "heading": "Equations", "parent_headings": [], "level": 1, "session_type": "body", "table_slice": "none", "positions": [{"type": "paraid", "range": ["p2", "p2"]}]}
{"type": "content", "blockid": "e4c6683e6107d75437b7d0712cf75c82", "format": "plain_text", "content": "<equation id=\"eq-bbbb222233334444bbbb222233334444-0002\" format=\"latex\">a + b = c</equation>\ntext after", "heading": "Equations", "parent_headings": [], "level": 1, "session_type": "body", "table_slice": "none", "positions": [{"type": "paraid", "range": ["p3", "p3"]}]}
@@ -0,0 +1,25 @@
{
"version": "1.0",
"equations": {
"eq-bbbb222233334444bbbb222233334444-0001": {
"id": "eq-bbbb222233334444bbbb222233334444-0001",
"blockid": "603a388c6d9c45900dcbde3a86a0c3b5",
"heading": "Equations",
"parent_headings": [],
"format": "latex",
"content": "x^2 + y^2 = r^2",
"caption": "",
"footnotes": []
},
"eq-bbbb222233334444bbbb222233334444-0002": {
"id": "eq-bbbb222233334444bbbb222233334444-0002",
"blockid": "e4c6683e6107d75437b7d0712cf75c82",
"heading": "Equations",
"parent_headings": [],
"format": "latex",
"content": "a + b = c",
"caption": "",
"footnotes": []
}
}
}
@@ -0,0 +1,3 @@
{"type": "meta", "format": "lightrag", "version": "1.0", "document_name": "linked.docx", "document_format": "docx", "document_hash": "sha256:4d984f8d2a4cb95808c403800f17e4f5043f48cc9d72e6062c93b5150e389320", "table_file": false, "equation_file": false, "drawing_file": true, "asset_dir": false, "blocks": 2, "doc_id": "doc-1111aaaa2222bbbb1111aaaa2222bbbb", "parse_engine": "native", "parse_time": "2026-01-01T00:00:00+00:00", "doc_title": "Linked"}
{"type": "content", "blockid": "573c0f2fc108950c0ce0060429a9de39", "format": "plain_text", "content": "# Linked", "heading": "Linked", "parent_headings": [], "level": 1, "session_type": "body", "table_slice": "none", "positions": [{"type": "paraid", "range": ["h1", "h1"]}]}
{"type": "content", "blockid": "42b4dd9bfb50ba8f1c4b60229ea4605a", "format": "plain_text", "content": "See the diagram online:\n<drawing id=\"im-1111aaaa2222bbbb1111aaaa2222bbbb-0001\" format=\"png\" path=\"https://example.com/diagrams/architecture.png\" src=\"docx://external\" />\nAnd a relative-but-not-asset path:\n<drawing id=\"im-1111aaaa2222bbbb1111aaaa2222bbbb-0002\" format=\"gif\" path=\"../images/legacy.gif\" src=\"docx://legacy\" />", "heading": "Linked", "parent_headings": [], "level": 1, "session_type": "body", "table_slice": "none", "positions": [{"type": "paraid", "range": ["p1", "p1"]}]}
@@ -0,0 +1,27 @@
{
"version": "1.0",
"drawings": {
"im-1111aaaa2222bbbb1111aaaa2222bbbb-0001": {
"id": "im-1111aaaa2222bbbb1111aaaa2222bbbb-0001",
"blockid": "42b4dd9bfb50ba8f1c4b60229ea4605a",
"heading": "Linked",
"parent_headings": [],
"format": "png",
"path": "https://example.com/diagrams/architecture.png",
"src": "docx://external",
"caption": "",
"footnotes": []
},
"im-1111aaaa2222bbbb1111aaaa2222bbbb-0002": {
"id": "im-1111aaaa2222bbbb1111aaaa2222bbbb-0002",
"blockid": "42b4dd9bfb50ba8f1c4b60229ea4605a",
"heading": "Linked",
"parent_headings": [],
"format": "gif",
"path": "../images/legacy.gif",
"src": "docx://legacy",
"caption": "",
"footnotes": []
}
}
}
@@ -0,0 +1,3 @@
{"type": "meta", "format": "lightrag", "version": "1.0", "document_name": "legacy.docx", "document_format": "docx", "document_hash": "sha256:1b888273b2772355165b6e6bc69092a35f421fb382e4e1cf5bdb2a92db70ee86", "table_file": false, "equation_file": false, "drawing_file": false, "asset_dir": false, "blocks": 2, "doc_id": "doc-99990000111122229999000011112222", "parse_engine": "native", "parse_time": "2026-01-01T00:00:00+00:00", "doc_title": "legacy"}
{"type": "content", "blockid": "11ffd40d2d43ce0c105a7d0ffe3c803f", "format": "plain_text", "content": "Just plain text without a heading.", "heading": "", "parent_headings": [], "level": 0, "session_type": "body", "table_slice": "none", "positions": [{"type": "paraid", "range": [null, null]}]}
{"type": "content", "blockid": "75c3c3be70534741d0ce0eeecf308502", "format": "plain_text", "content": "Another paragraph with no paraId.", "heading": "", "parent_headings": [], "level": 0, "session_type": "body", "table_slice": "none", "positions": [{"type": "paraid", "range": [null, null]}]}
@@ -0,0 +1,5 @@
{"type": "meta", "format": "lightrag", "version": "1.0", "document_name": "report.docx", "document_format": "docx", "document_hash": "sha256:ac2fac1eece1383be12ebb2ca8ace735bfcb3078af0740e22d35508cf8b7421c", "table_file": true, "equation_file": false, "drawing_file": false, "asset_dir": false, "blocks": 4, "doc_id": "doc-cccc333344445555cccc333344445555", "parse_engine": "native", "parse_time": "2026-01-01T00:00:00+00:00", "doc_title": "Report"}
{"type": "content", "blockid": "4ede43f1d15b69ff71c86035d0f7cfac", "format": "plain_text", "content": "# Report", "heading": "Report", "parent_headings": [], "level": 1, "session_type": "body", "table_slice": "none", "positions": [{"type": "paraid", "range": ["h1", "h1"]}]}
{"type": "content", "blockid": "c2f594c8d5882977f79ad8ec6da3338f", "format": "plain_text", "content": "See table:\n<table id=\"tb-cccc333344445555cccc333344445555-0001\" format=\"json\">[[\"X\",\"Y\"],[\"1\",\"2\"],[\"3\",\"4\"]]</table>", "heading": "Report", "parent_headings": [], "level": 1, "session_type": "body", "table_slice": "none", "positions": [{"type": "paraid", "range": ["t1", "t1"]}]}
{"type": "content", "blockid": "bc35d892e4c8f05c9eaf758a4f78878e", "format": "plain_text", "content": "Plain table:\n<table id=\"tb-cccc333344445555cccc333344445555-0002\" format=\"json\">[[\"a\",\"b\"]]</table>", "heading": "Report", "parent_headings": [], "level": 1, "session_type": "body", "table_slice": "none", "positions": [{"type": "paraid", "range": ["t2", "t2"]}]}
{"type": "content", "blockid": "fd0febb81cacf41fe81aea2812bd1bd8", "format": "plain_text", "content": "<table id=\"tb-cccc333344445555cccc333344445555-0003\" format=\"json\">[[\"p\"]]</table>\nthen\n<table id=\"tb-cccc333344445555cccc333344445555-0004\" format=\"json\">[[\"q\",\"r\"],[\"s\",\"t\"]]</table>", "heading": "Report", "parent_headings": [], "level": 1, "session_type": "body", "table_slice": "none", "positions": [{"type": "paraid", "range": ["t3", "t3"]}]}
@@ -0,0 +1,63 @@
{
"version": "1.0",
"tables": {
"tb-cccc333344445555cccc333344445555-0001": {
"id": "tb-cccc333344445555cccc333344445555-0001",
"blockid": "c2f594c8d5882977f79ad8ec6da3338f",
"heading": "Report",
"parent_headings": [],
"dimension": [
3,
2
],
"format": "json",
"content": "[[\"X\", \"Y\"], [\"1\", \"2\"], [\"3\", \"4\"]]",
"caption": "",
"footnotes": [],
"table_header": "[[\"X\", \"Y\"]]"
},
"tb-cccc333344445555cccc333344445555-0002": {
"id": "tb-cccc333344445555cccc333344445555-0002",
"blockid": "bc35d892e4c8f05c9eaf758a4f78878e",
"heading": "Report",
"parent_headings": [],
"dimension": [
1,
2
],
"format": "json",
"content": "[[\"a\", \"b\"]]",
"caption": "",
"footnotes": []
},
"tb-cccc333344445555cccc333344445555-0003": {
"id": "tb-cccc333344445555cccc333344445555-0003",
"blockid": "fd0febb81cacf41fe81aea2812bd1bd8",
"heading": "Report",
"parent_headings": [],
"dimension": [
1,
1
],
"format": "json",
"content": "[[\"p\"]]",
"caption": "",
"footnotes": []
},
"tb-cccc333344445555cccc333344445555-0004": {
"id": "tb-cccc333344445555cccc333344445555-0004",
"blockid": "fd0febb81cacf41fe81aea2812bd1bd8",
"heading": "Report",
"parent_headings": [],
"dimension": [
2,
2
],
"format": "json",
"content": "[[\"q\", \"r\"], [\"s\", \"t\"]]",
"caption": "",
"footnotes": [],
"table_header": "[[\"q\", \"r\"]]"
}
}
}
@@ -0,0 +1,5 @@
{"type": "meta", "format": "lightrag", "version": "1.0", "document_name": "paper.docx", "document_format": "docx", "document_hash": "sha256:e3fd11b45f12fe0fe0ca22cd6526b7d65336e58cfd2c6585fca4511fa967a55a", "table_file": false, "equation_file": false, "drawing_file": false, "asset_dir": false, "blocks": 4, "doc_id": "doc-aaaa111122223333aaaa111122223333", "parse_engine": "native", "parse_time": "2026-01-01T00:00:00+00:00", "doc_title": "Introduction"}
{"type": "content", "blockid": "5c647a54224deea018b593b7bbef2348", "format": "plain_text", "content": "# Introduction", "heading": "Introduction", "parent_headings": [], "level": 1, "session_type": "body", "table_slice": "none", "positions": [{"type": "paraid", "range": ["h1", "h1"]}]}
{"type": "content", "blockid": "11d7395dfeaf573464e88d3d7f736f5c", "format": "plain_text", "content": "Body paragraph one.", "heading": "Introduction", "parent_headings": [], "level": 1, "session_type": "body", "table_slice": "none", "positions": [{"type": "paraid", "range": ["p1", "p2"]}]}
{"type": "content", "blockid": "7bf205c142f2012fbe586007949c21d1", "format": "plain_text", "content": "## Background", "heading": "Background", "parent_headings": ["Introduction"], "level": 2, "session_type": "body", "table_slice": "none", "positions": [{"type": "paraid", "range": ["h2", "h2"]}]}
{"type": "content", "blockid": "337190582281b148ff9732cb7a04661e", "format": "plain_text", "content": "Sub body.", "heading": "Background", "parent_headings": ["Introduction"], "level": 2, "session_type": "body", "table_slice": "none", "positions": [{"type": "paraid", "range": ["p3", "p3"]}]}
@@ -0,0 +1,69 @@
"""Security-focused tests for native DOCX IR drawing asset handling."""
from __future__ import annotations
import pytest
from lightrag.parser.docx.ir_builder import NativeDocxIRBuilder
def _build_ir(content: str):
return NativeDocxIRBuilder().normalize(
[
{
"uuid": "p1",
"uuid_end": "p1",
"heading": "Section",
"content": content,
"parent_headings": [],
"level": 1,
}
],
document_name="doc.docx",
asset_dir_name="doc.blocks.assets",
)
@pytest.mark.offline
def test_native_docx_ir_accepts_safe_local_drawing_asset() -> None:
ir = _build_ir(
'<drawing id="1" name="fig" format="png" path="doc.blocks.assets/fig.png" />'
)
drawing = ir.blocks[0].drawings[0]
assert drawing.asset_ref == "fig.png"
assert drawing.path_override is None
assert len(ir.assets) == 1
assert ir.assets[0].ref == "fig.png"
assert ir.assets[0].suggested_name == "fig.png"
assert ir.assets[0].source is None
@pytest.mark.offline
@pytest.mark.parametrize(
"path",
[
"doc.blocks.assets/../secret.png",
"doc.blocks.assets//tmp/secret.png",
r"doc.blocks.assets/..\secret.png",
],
)
def test_native_docx_ir_rejects_unsafe_local_drawing_asset(path: str) -> None:
ir = _build_ir(f'<drawing id="1" name="fig" format="png" path="{path}" />')
drawing = ir.blocks[0].drawings[0]
assert drawing.asset_ref == ""
assert drawing.path_override is None
assert ir.assets == []
@pytest.mark.offline
def test_native_docx_ir_preserves_non_asset_external_drawing_path() -> None:
ir = _build_ir(
'<drawing id="1" name="fig" format="gif" path="../images/legacy.gif" />'
)
drawing = ir.blocks[0].drawings[0]
assert drawing.asset_ref == ""
assert drawing.path_override == "../images/legacy.gif"
assert ir.assets == []
@@ -0,0 +1,405 @@
"""Unit tests for native docx tracked-change / comment / empty-table handling.
Locks in the contract that:
- ``w:ins`` content survives (final revised text),
- ``w:del`` / ``w:moveFrom`` content is dropped,
- ``w:commentRangeStart`` / ``w:commentRangeEnd`` / ``w:commentReference`` /
``w:annotationRef`` markers are dropped,
- tables whose every cell is whitespace-only are omitted from the parser
output (no ``<table>`` placeholder, so no IRTable downstream).
These were previously emergent properties (skip lists in two places + a
run-level white-list); regressions now fail loudly.
"""
from __future__ import annotations
from io import BytesIO
import pytest
from docx import Document
from docx.oxml import OxmlElement
from docx.oxml.ns import qn
from lxml import etree
from lightrag.parser.docx.parse_document import (
DocxContentError,
extract_docx_blocks,
extract_paragraph_content,
)
from lightrag.parser.docx.table_extractor import extract_paragraph_content_table
W_NS = "http://schemas.openxmlformats.org/wordprocessingml/2006/main"
PARAGRAPH_NS = {
"w": W_NS,
"wp": "http://schemas.openxmlformats.org/drawingml/2006/wordprocessingDrawing",
"m": "http://schemas.openxmlformats.org/officeDocument/2006/math",
}
def _p(inner_xml: str):
"""Build a ``<w:p>`` lxml element from inner OOXML."""
return etree.fromstring(f'<w:p xmlns:w="{W_NS}">{inner_xml}</w:p>'.encode("utf-8"))
# --- paragraph walker (parse_document.extract_paragraph_content) -----------
@pytest.mark.offline
def test_paragraph_keeps_w_ins_content() -> None:
para = _p("<w:ins><w:r><w:t>kept</w:t></w:r></w:ins>")
assert extract_paragraph_content(para, PARAGRAPH_NS) == "kept"
@pytest.mark.offline
def test_paragraph_drops_w_del_content() -> None:
para = _p("<w:del><w:r><w:t>removed</w:t></w:r></w:del>")
assert extract_paragraph_content(para, PARAGRAPH_NS) == ""
@pytest.mark.offline
def test_paragraph_drops_w_movefrom_content() -> None:
para = _p("<w:moveFrom><w:r><w:t>moved away</w:t></w:r></w:moveFrom>")
assert extract_paragraph_content(para, PARAGRAPH_NS) == ""
@pytest.mark.offline
def test_paragraph_drops_comment_markers_but_keeps_surrounding_text() -> None:
para = _p(
'<w:commentRangeStart w:id="0"/>'
"<w:r><w:t>visible</w:t></w:r>"
'<w:commentRangeEnd w:id="0"/>'
"<w:r>"
' <w:rPr><w:rStyle w:val="CommentReference"/></w:rPr>'
' <w:commentReference w:id="0"/>'
"</w:r>"
)
assert extract_paragraph_content(para, PARAGRAPH_NS) == "visible"
# --- table-cell walker (table_extractor.extract_paragraph_content_table) ---
@pytest.mark.offline
def test_table_cell_keeps_w_ins_content() -> None:
para = _p("<w:ins><w:r><w:t>kept</w:t></w:r></w:ins>")
assert extract_paragraph_content_table(para, qn) == "kept"
@pytest.mark.offline
def test_table_cell_drops_w_del_content() -> None:
para = _p("<w:del><w:r><w:t>removed</w:t></w:r></w:del>")
assert extract_paragraph_content_table(para, qn) == ""
@pytest.mark.offline
def test_table_cell_drops_comment_markers() -> None:
para = _p(
'<w:commentRangeStart w:id="0"/>'
"<w:r><w:t>cell-text</w:t></w:r>"
'<w:commentRangeEnd w:id="0"/>'
)
assert extract_paragraph_content_table(para, qn) == "cell-text"
# --- end-to-end: empty tables vanish from blocks output --------------------
def _populate_cell(cell, text: str) -> None:
cell.paragraphs[0].text = text
def _build_docx_with_three_tables() -> BytesIO:
"""One real table, one all-empty table, one whitespace-only table.
Returns a BytesIO containing the rendered .docx so it can be fed to
``extract_docx_blocks`` via a tempfile-backed path or directly.
"""
doc = Document()
doc.add_paragraph("intro")
real = doc.add_table(rows=1, cols=2)
_populate_cell(real.rows[0].cells[0], "A")
_populate_cell(real.rows[0].cells[1], "B")
empty = doc.add_table(rows=2, cols=2)
# leave every cell at python-docx default (empty paragraph)
assert all(c.text == "" for row in empty.rows for c in row.cells)
whitespace = doc.add_table(rows=1, cols=2)
_populate_cell(whitespace.rows[0].cells[0], " ")
_populate_cell(whitespace.rows[0].cells[1], "\t")
buf = BytesIO()
doc.save(buf)
buf.seek(0)
return buf
# --- end-to-end: every heading becomes its own block ----------------------
def _add_heading(doc, text: str, level: int) -> None:
"""Append a heading paragraph with an explicit ``w:outlineLvl``.
Setting outlineLvl directly on the paragraph (rather than relying on the
template's built-in Heading styles) keeps ``get_heading_level`` detection
deterministic. ``level`` is 1-based (1 = H1); outlineLvl val is 0-based.
"""
para = doc.add_paragraph(text)
pPr = para._p.get_or_add_pPr()
outline = OxmlElement("w:outlineLvl")
outline.set(qn("w:val"), str(level - 1))
pPr.append(outline)
@pytest.mark.offline
def test_each_heading_becomes_its_own_block(tmp_path) -> None:
"""Headings with no body must each form a standalone block.
Layout: H1 (no body) → H2 (no body) → H3 (with body) → H1 (no body, last).
Previously the empty H1/H2 were folded into the next heading's block; now
every recognized heading starts its own block. A heading with no following
body yields a block whose ``content`` is just the heading text, while a
heading with body merges that body into the same block. ``parent_headings``
must track the outline hierarchy correctly.
"""
doc = Document()
_add_heading(doc, "Chapter One", level=1)
_add_heading(doc, "Section 1.1", level=2)
_add_heading(doc, "Subsection 1.1.1", level=3)
doc.add_paragraph("Body text under subsection.")
_add_heading(doc, "Appendix", level=1)
buf = BytesIO()
doc.save(buf)
docx_path = tmp_path / "headings.docx"
docx_path.write_bytes(buf.getvalue())
# The native parser splits at every heading level and never does
# token-based splitting or small-block merging (that is the chunker's job).
blocks = extract_docx_blocks(str(docx_path))
# The content line carries a markdown ``#`` prefix matching the level,
# while the ``heading`` field stays clean (no prefix).
summary = [
(b["heading"], b["content"], b["level"], b["parent_headings"]) for b in blocks
]
assert summary == [
("Chapter One", "# Chapter One", 1, []),
("Section 1.1", "## Section 1.1", 2, ["Chapter One"]),
(
"Subsection 1.1.1",
"### Subsection 1.1.1\nBody text under subsection.",
3,
["Chapter One", "Section 1.1"],
),
("Appendix", "# Appendix", 1, []),
]
@pytest.mark.offline
def test_heading_markdown_prefix_capped_at_six(tmp_path) -> None:
"""Heading content lines get a markdown ``#`` prefix matching the level,
capped at 6 ``#`` (a level-7 heading still renders ``######``)."""
doc = Document()
_add_heading(doc, "Top", level=1)
doc.add_paragraph("body under top.")
_add_heading(doc, "Deep Seven", level=7)
doc.add_paragraph("body under deep.")
buf = BytesIO()
doc.save(buf)
docx_path = tmp_path / "deep.docx"
docx_path.write_bytes(buf.getvalue())
blocks = extract_docx_blocks(str(docx_path))
summary = [(b["heading"], b["content"].split("\n")[0], b["level"]) for b in blocks]
assert summary == [
("Top", "# Top", 1),
# level 7 outline → clamped to six "#".
("Deep Seven", "###### Deep Seven", 7),
]
def _add_outline_para_with_softbreak(doc, head_text: str, body_text: str, level: int):
"""Append one ``<w:p>`` carrying an outline level whose runs are
``head_text`` + a soft line break (``<w:br/>``) + ``body_text``.
Mirrors the real-world WPS/Word pattern where an author sets an outline
level on a paragraph but types the body with Shift+Enter (a soft break)
instead of a new paragraph, so heading + body live in a single paragraph.
"""
para = doc.add_paragraph(head_text)
pPr = para._p.get_or_add_pPr()
outline = OxmlElement("w:outlineLvl")
outline.set(qn("w:val"), str(level - 1))
pPr.append(outline)
para.add_run().add_break() # default WD_BREAK.LINE → soft <w:br/> → '\n'
para.add_run(body_text)
return para
@pytest.mark.offline
def test_oversize_outline_heading_with_softbreak_splits(tmp_path) -> None:
"""An over-long heading paragraph that contains a soft break must split:
the first line stays the heading, the remainder becomes body text. No
DocxContentError, and the body content is preserved in full."""
head = "A、处置程序"
body = "施工现场一旦发生事故时应立即采取相应的应急处置措施并以最快速度报警。" * 8
assert len(head) + 1 + len(body) > 200 # would trip validate_heading_length
doc = Document()
_add_outline_para_with_softbreak(doc, head, body, level=4)
buf = BytesIO()
doc.save(buf)
docx_path = tmp_path / "softbreak.docx"
docx_path.write_bytes(buf.getvalue())
blocks = extract_docx_blocks(str(docx_path))
# The short first line becomes the heading; the long remainder is body.
heading_block = next(b for b in blocks if b["heading"] == head)
assert body in heading_block["content"]
# Heading field carries only the first line, not the body.
assert body not in heading_block["heading"]
@pytest.mark.offline
def test_oversize_outline_heading_no_softbreak_demoted_to_body(tmp_path) -> None:
"""A single-line over-long outline paragraph (no soft break) is demoted to
body text rather than raising — content preserved, not treated as a heading."""
long_text = "施工现场应急处置措施说明," * 30 # well over 200 chars, no '\n'
assert len(long_text) > 200
doc = Document()
_add_heading(doc, long_text, level=2)
buf = BytesIO()
doc.save(buf)
docx_path = tmp_path / "demote.docx"
docx_path.write_bytes(buf.getvalue())
blocks = extract_docx_blocks(str(docx_path))
# The text survives as body content, and no block adopts it as a heading.
assert any(long_text in b["content"] for b in blocks)
assert all(b["heading"] != long_text for b in blocks)
@pytest.mark.offline
def test_short_outline_paragraph_still_heading(tmp_path) -> None:
"""A within-limit outline paragraph must still be recognized as a heading —
the over-long handling never demotes short headings."""
doc = Document()
_add_heading(doc, "处置程序", level=3)
doc.add_paragraph("body under heading.")
buf = BytesIO()
doc.save(buf)
docx_path = tmp_path / "short.docx"
docx_path.write_bytes(buf.getvalue())
blocks = extract_docx_blocks(str(docx_path))
heading_block = next(b for b in blocks if b["heading"] == "处置程序")
assert heading_block["level"] == 3
assert heading_block["content"].startswith("### 处置程序")
@pytest.mark.offline
def test_existing_markdown_heading_keeps_content_but_metadata_is_clean(
tmp_path,
) -> None:
doc = Document()
_add_heading(doc, "# Already MD", level=1)
doc.add_paragraph("Body.")
buf = BytesIO()
doc.save(buf)
docx_path = tmp_path / "markdown-heading.docx"
docx_path.write_bytes(buf.getvalue())
parse_metadata: dict[str, str] = {}
blocks = extract_docx_blocks(str(docx_path), parse_metadata=parse_metadata)
assert parse_metadata["first_heading"] == "Already MD"
assert blocks[0]["heading"] == "Already MD"
assert blocks[0]["parent_headings"] == []
assert blocks[0]["content"].splitlines()[0] == "# Already MD"
@pytest.mark.offline
def test_empty_tables_are_skipped(tmp_path) -> None:
docx_bytes = _build_docx_with_three_tables()
docx_path = tmp_path / "three_tables.docx"
docx_path.write_bytes(docx_bytes.getvalue())
blocks = extract_docx_blocks(str(docx_path))
table_placeholders = sum(b["content"].count("<table>") for b in blocks)
assert table_placeholders == 1, (
"exactly one real table should survive; empty + whitespace tables "
"must be dropped before the placeholder is emitted"
)
assert any('"A"' in b["content"] and '"B"' in b["content"] for b in blocks)
# --- invalid (non-ZIP) .docx files surface an accurate error ---------------
@pytest.mark.offline
@pytest.mark.parametrize(
("header", "format_hint"),
[
(b"\xd0\xcf\x11\xe0\xa1\xb1\x1a\xe1" + b"junk", "Word 97-2003"),
(b"{\\rtf1\\ansi}", "RTF"),
(b"%PDF-1.7\n%binary", "PDF"),
(b"<?xml version='1.0'?><html></html>", "HTML or XML"),
(b"plain text masquerading as docx", "not a valid DOCX"),
],
)
def test_invalid_docx_raises_accurate_error(tmp_path, header, format_hint) -> None:
"""A file that exists but is not a ZIP/OOXML package must fail with an
accurate message — never the misleading python-docx "Package not found".
The native worker confirms the file exists before parsing, so the only way
``PackageNotFoundError`` can fire here is a corrupt file or a non-DOCX
payload wearing a .docx extension. The error must name the real format and
keep the file path.
"""
docx_path = tmp_path / "masquerade.docx"
docx_path.write_bytes(header)
with pytest.raises(DocxContentError) as excinfo:
extract_docx_blocks(str(docx_path))
message = str(excinfo.value)
assert "Package not found" not in message
assert "valid DOCX" in message
assert str(docx_path) in message
assert format_hint in message
@pytest.mark.offline
def test_empty_docx_file_reports_truncation(tmp_path) -> None:
"""A 0-byte .docx must be diagnosed as empty/truncated, not 'not found'."""
docx_path = tmp_path / "empty.docx"
docx_path.write_bytes(b"")
with pytest.raises(DocxContentError) as excinfo:
extract_docx_blocks(str(docx_path))
message = str(excinfo.value)
assert "Package not found" not in message
assert "empty" in message
assert str(docx_path) in message
@@ -0,0 +1,165 @@
"""Byte-equivalence golden tests for the native docx → SidecarWriter migration.
These tests run the production code path (``LightRAG.parse_native``) on
each scenario in ``_native_docx_fixtures.SCENARIOS`` and assert that
every produced file matches the captured baseline bytes under
``tests/parser/docx/golden/native_docx/<scenario>/``.
The baseline was generated by ``scripts/regen_native_docx_golden.py``;
the script lives in-tree so the snapshots can be regenerated when
intentional changes to the format are made (e.g. spec evolution).
"""
from __future__ import annotations
import asyncio
import sys
from pathlib import Path
from unittest import mock
import pytest
# Make the sibling _native_docx_fixtures module importable as if it were a
# top-level helper. Avoids needing a package __init__ rename.
HERE = Path(__file__).resolve().parent
if str(HERE) not in sys.path:
sys.path.insert(0, str(HERE))
from _native_docx_fixtures import SCENARIOS, Scenario # noqa: E402
from lightrag.parser.base import ParseContext # noqa: E402
from lightrag.parser.debug import ( # noqa: E402
FrozenDateTime,
build_debug_rag,
)
from lightrag.parser.registry import get_parser # noqa: E402
GOLDEN_ROOT = HERE / "golden" / "native_docx"
def _run_new_path(
scenario: Scenario, input_dir: Path, monkeypatch: pytest.MonkeyPatch
) -> Path:
"""Invoke ``LightRAG.parse_native`` on a single scenario.
The upstream ``extract_docx_blocks`` is stubbed to feed the synthetic
blocks and produce the matching asset files inside
``<base>.blocks.assets/``. ``datetime.now`` is frozen so ``parse_time``
matches the captured baseline exactly.
Returns the parsed directory containing the produced artifacts.
"""
from lightrag.constants import (
FULL_DOCS_FORMAT_PENDING_PARSE,
PARSED_DIR_NAME,
)
monkeypatch.setenv("INPUT_DIR", str(input_dir))
# parse_native archives the source after parsing; for the golden test
# we don't want the docx moving around between scenarios.
async def _noop_archive(_p: str) -> None:
return None
import lightrag.pipeline as pipeline_module
monkeypatch.setattr(
pipeline_module, "archive_docx_source_after_full_docs_sync", _noop_archive
)
source_path = input_dir / scenario.file_path
source_path.parent.mkdir(parents=True, exist_ok=True)
source_path.write_bytes(b"fake-docx")
def _stub_extract(
file_path,
*,
drawing_context=None,
parse_warnings=None,
parse_metadata=None,
**_kwargs,
):
if drawing_context is not None and scenario.assets:
drawing_context.export_dir_path.mkdir(parents=True, exist_ok=True)
for name, data in scenario.assets.items():
(drawing_context.export_dir_path / name).write_bytes(data)
if parse_metadata is not None:
parse_metadata.update(scenario.parse_metadata)
return [dict(b) for b in scenario.blocks]
rag = build_debug_rag()
with (
mock.patch(
"lightrag.parser.docx.parse_document.extract_docx_blocks",
_stub_extract,
),
mock.patch("lightrag.sidecar.writer.datetime", FrozenDateTime),
):
async def _go() -> None:
await get_parser("native").parse(
ParseContext(
rag,
scenario.doc_id,
str(source_path),
{
"parse_format": FULL_DOCS_FORMAT_PENDING_PARSE,
"content": "",
},
)
)
asyncio.run(_go())
return input_dir / PARSED_DIR_NAME / f"{scenario.file_path}.parsed"
def _read_bytes(path: Path) -> bytes:
with path.open("rb") as fh:
return fh.read()
@pytest.mark.offline
@pytest.mark.parametrize(
"scenario",
SCENARIOS,
ids=[s.name for s in SCENARIOS],
)
def test_native_docx_migration_is_byte_equivalent(
scenario: Scenario,
tmp_path: Path,
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""Every file produced by parse_native must be byte-identical to the
captured legacy baseline."""
input_dir = tmp_path / "inputs"
input_dir.mkdir()
parsed_dir = _run_new_path(scenario, input_dir, monkeypatch)
expected_root = GOLDEN_ROOT / scenario.name
assert expected_root.is_dir(), (
f"missing golden fixture for scenario {scenario.name!r}; "
f"run scripts/regen_native_docx_golden.py to regenerate"
)
# Collect both file sets relative to their roots, then compare.
expected_files = {
p.relative_to(expected_root): p for p in expected_root.rglob("*") if p.is_file()
}
produced_files = {
p.relative_to(parsed_dir): p for p in parsed_dir.rglob("*") if p.is_file()
}
extra = sorted(produced_files.keys() - expected_files.keys())
missing = sorted(expected_files.keys() - produced_files.keys())
assert not extra, f"unexpected produced files: {extra}"
assert not missing, f"missing produced files (legacy had them): {missing}"
mismatches: list[str] = []
for rel, expected_path in expected_files.items():
produced_path = produced_files[rel]
if _read_bytes(produced_path) != _read_bytes(expected_path):
mismatches.append(str(rel))
assert not mismatches, (
f"byte mismatch in scenario {scenario.name!r} for files: {mismatches}"
)
@@ -0,0 +1,45 @@
"""Native DOCX content-limit validation must *raise*, never ``sys.exit``.
The DOCX parser runs in-process inside the gunicorn/uvicorn worker (via
``LightRAG.parse_native`` → ``extract_docx_blocks``). A ``sys.exit`` there
raises ``SystemExit`` (a ``BaseException``), which slips past the pipeline's
per-document ``except Exception`` handler and tears down the whole worker —
gunicorn then reports ``WORKER TIMEOUT`` and kills it with SIGABRT (code 134),
interrupting the entire processing pipeline.
These tests lock in that the content-limit checks raise ``DocxContentError``
(an ``Exception`` subclass) so just the offending document fails while the
worker keeps running.
"""
from __future__ import annotations
import pytest
from lightrag.parser.docx.parse_document import (
DocxContentError,
MAX_HEADING_LENGTH,
validate_heading_length,
)
def test_docx_content_error_is_an_exception_not_just_baseexception():
# The pipeline parse worker catches ``except Exception``; SystemExit
# (BaseException) would bypass it. Guard against a future regression that
# reintroduces a BaseException-only error type.
assert issubclass(DocxContentError, Exception)
def test_validate_heading_length_raises_instead_of_exiting():
long_heading = "x" * (MAX_HEADING_LENGTH + 213) # mirrors the 413-char report
with pytest.raises(DocxContentError) as exc_info:
validate_heading_length(long_heading, "p1")
message = str(exc_info.value)
assert f"Heading too long ({len(long_heading)} characters" in message
assert "SOLUTION:" in message
def test_validate_heading_length_does_not_raise_within_limit():
# Exactly at the limit and below must pass through silently.
validate_heading_length("x" * MAX_HEADING_LENGTH, "p2")
validate_heading_length("short heading", "p3")
View File
View File
+222
View File
@@ -0,0 +1,222 @@
"""Tests for ``lightrag/parser/external/docling/cache.py``.
Covers the cache-miss conditions enumerated in the module docstring:
- missing / malformed / wrong-engine manifest
- source size or hash mismatch
- engine_version / endpoint_signature env mismatch
- options_signature env mismatch
- critical-file size / sha256 mismatch
- non-critical file size mismatch
"""
from __future__ import annotations
from pathlib import Path
import pytest
from lightrag.parser.external import Manifest, ManifestFile, write_manifest
from lightrag.parser.external._common import compute_size_and_hash
from lightrag.parser.external.docling.cache import (
compute_options_signature,
is_bundle_valid,
snapshot_tunable_env,
)
from lightrag.parser.external.docling.client import FIXED_CONSTANTS
@pytest.fixture(autouse=True)
def _clear_envs(monkeypatch: pytest.MonkeyPatch) -> None:
for name in (
"DOCLING_DO_OCR",
"DOCLING_FORCE_OCR",
"DOCLING_OCR_ENGINE",
"DOCLING_OCR_PRESET",
"DOCLING_OCR_LANG",
"DOCLING_DO_FORMULA_ENRICHMENT",
"DOCLING_ENGINE_VERSION",
"DOCLING_ENDPOINT",
):
monkeypatch.delenv(name, raising=False)
monkeypatch.setenv("DOCLING_ENDPOINT", "http://docling.test")
@pytest.fixture
def source_file(tmp_path: Path) -> Path:
p = tmp_path / "src.pdf"
p.write_bytes(b"hello pdf payload" * 64)
return p
def test_snapshot_tunable_env_uses_effective_defaults(
monkeypatch: pytest.MonkeyPatch,
) -> None:
unset = snapshot_tunable_env()
monkeypatch.setenv("DOCLING_DO_OCR", "true")
monkeypatch.setenv("DOCLING_FORCE_OCR", "true")
monkeypatch.setenv("DOCLING_OCR_ENGINE", "auto")
monkeypatch.setenv("DOCLING_OCR_PRESET", "auto")
monkeypatch.setenv("DOCLING_OCR_LANG", "")
monkeypatch.setenv("DOCLING_DO_FORMULA_ENRICHMENT", "false")
assert snapshot_tunable_env() == unset
def _build_valid_bundle(
tmp_path: Path,
source_file: Path,
*,
options_signature: str | None = None,
) -> Path:
raw_dir = tmp_path / "src.docling_raw"
raw_dir.mkdir()
main_json = raw_dir / "src.json"
main_json.write_text('{"schema_name": "DoclingDocument"}', encoding="utf-8")
md = raw_dir / "src.md"
md.write_text("# title", encoding="utf-8")
src_size, src_hash = compute_size_and_hash(source_file)
crit_size, crit_hash = compute_size_and_hash(main_json)
sig = options_signature
if sig is None:
sig = compute_options_signature(
tunable_env=snapshot_tunable_env(),
fixed_constants=FIXED_CONSTANTS,
)
manifest = Manifest(
engine="docling",
source_content_hash=src_hash,
source_size_bytes=src_size,
source_filename_at_parse=source_file.name,
critical_file=ManifestFile(path="src.json", size=crit_size, sha256=crit_hash),
files=[ManifestFile(path="src.md", size=md.stat().st_size)],
total_size_bytes=crit_size + md.stat().st_size,
task_id="task-1",
endpoint_signature="http://docling.test",
engine_version="",
options_signature=sig,
extras={"fixed_constants": dict(FIXED_CONSTANTS)},
)
write_manifest(raw_dir, manifest)
return raw_dir
def test_is_bundle_valid_happy_path(tmp_path: Path, source_file: Path) -> None:
raw = _build_valid_bundle(tmp_path, source_file)
assert is_bundle_valid(raw, source_file) is True
def test_is_bundle_valid_missing_dir(tmp_path: Path, source_file: Path) -> None:
assert is_bundle_valid(tmp_path / "ghost", source_file) is False
def test_is_bundle_valid_missing_manifest(tmp_path: Path, source_file: Path) -> None:
raw = tmp_path / "src.docling_raw"
raw.mkdir()
(raw / "src.json").write_text("{}")
assert is_bundle_valid(raw, source_file) is False
def test_is_bundle_valid_wrong_engine(tmp_path: Path, source_file: Path) -> None:
raw = _build_valid_bundle(tmp_path, source_file)
manifest_path = raw / "_manifest.json"
data = manifest_path.read_text(encoding="utf-8")
manifest_path.write_text(data.replace('"docling"', '"mineru"'), encoding="utf-8")
assert is_bundle_valid(raw, source_file) is False
def test_is_bundle_valid_source_size_mismatch(
tmp_path: Path, source_file: Path
) -> None:
raw = _build_valid_bundle(tmp_path, source_file)
source_file.write_bytes(source_file.read_bytes() + b"!")
assert is_bundle_valid(raw, source_file) is False
def test_is_bundle_valid_source_hash_mismatch(
tmp_path: Path, source_file: Path
) -> None:
raw = _build_valid_bundle(tmp_path, source_file)
# Replace contents with same length but different bytes
new = b"Y" * source_file.stat().st_size
source_file.write_bytes(new)
assert is_bundle_valid(raw, source_file) is False
def test_is_bundle_valid_endpoint_change(
tmp_path: Path, source_file: Path, monkeypatch: pytest.MonkeyPatch
) -> None:
raw = _build_valid_bundle(tmp_path, source_file)
monkeypatch.setenv("DOCLING_ENDPOINT", "http://other:5001")
assert is_bundle_valid(raw, source_file) is False
def test_is_bundle_valid_options_signature_change(
tmp_path: Path, source_file: Path, monkeypatch: pytest.MonkeyPatch
) -> None:
raw = _build_valid_bundle(tmp_path, source_file)
monkeypatch.setenv("DOCLING_FORCE_OCR", "false")
assert is_bundle_valid(raw, source_file) is False
def test_is_bundle_valid_legacy_bundle_misses_with_overrides(
tmp_path: Path, source_file: Path
) -> None:
# A legacy bundle predating signature recording (empty options_signature)
# is leniently accepted with no overrides, but a per-file override
# (docling(force_ocr=...)) must force a miss — we cannot prove the bundle
# was produced with that override, and silently reusing it would drop the
# user's explicit param. Mirrors MinerU, which misses on any absent sig.
raw = _build_valid_bundle(tmp_path, source_file, options_signature="")
assert is_bundle_valid(raw, source_file) is True
assert is_bundle_valid(raw, source_file, overrides={"force_ocr": False}) is False
assert is_bundle_valid(raw, source_file, overrides={"force_ocr": True}) is False
def test_is_bundle_valid_fixed_constants_code_change(
tmp_path: Path, source_file: Path
) -> None:
# Simulate a code-only change to one of the fixed pipeline constants
# (e.g. image_export_mode flipped from "referenced" to "embedded"
# between parse time and validation time). The manifest stores both
# the stale constants and a signature computed from them; validation
# must compare against current FIXED_CONSTANTS and miss, not against
# the manifest's own copy (which would always match).
stale_constants = {**FIXED_CONSTANTS, "image_export_mode": "embedded"}
stale_signature = compute_options_signature(
tunable_env=snapshot_tunable_env(),
fixed_constants=stale_constants,
)
raw = _build_valid_bundle(tmp_path, source_file, options_signature=stale_signature)
# Overwrite the manifest's extras to record the stale constants too —
# this is the bug surface: if validation rehydrated from extras, it
# would reproduce stale_signature and falsely accept the bundle.
import json as _json
mp = raw / "_manifest.json"
data = _json.loads(mp.read_text(encoding="utf-8"))
data["extras"] = {"fixed_constants": stale_constants}
mp.write_text(_json.dumps(data), encoding="utf-8")
assert is_bundle_valid(raw, source_file) is False
def test_is_bundle_valid_critical_file_corrupt(
tmp_path: Path, source_file: Path
) -> None:
raw = _build_valid_bundle(tmp_path, source_file)
# Corrupt the JSON: same length, different bytes — defeats size check,
# so the sha256 path must catch it.
current = (raw / "src.json").read_bytes()
(raw / "src.json").write_bytes(b"X" * len(current))
assert is_bundle_valid(raw, source_file) is False
def test_is_bundle_valid_other_file_size_mismatch(
tmp_path: Path, source_file: Path
) -> None:
raw = _build_valid_bundle(tmp_path, source_file)
(raw / "src.md").write_text("totally different content here that is longer")
assert is_bundle_valid(raw, source_file) is False
+596
View File
@@ -0,0 +1,596 @@
"""Tests for :class:`DoclingRawClient`.
Cover the contract guarantees that protect the sidecar pipeline:
- the fixed pipeline constants (``pipeline=standard`` / ``target_type=zip``
/ ``to_formats=[json,md]`` / ``image_export_mode=referenced``) are sent
on every upload, regardless of env;
- terminal non-success states (``failure`` / ``partial_success`` /
``skipped``) abort the run **before** any result download;
- ``DOCLING_OCR_LANG`` is omitted when empty so docling-serve falls back
to its own default.
Uses an in-process fake httpx client mirroring ``tests/parser/external/mineru/test_client.py``
so we don't trip httpx's sync/async stream guard on multipart uploads.
"""
from __future__ import annotations
import io
import json
import zipfile
from pathlib import Path
from typing import Any
from urllib.parse import quote
import pytest
from lightrag.parser.external.docling.client import (
CONVERT_PATH,
POLL_PATH,
RESULT_PATH,
DoclingRawClient,
)
# ---------------------------------------------------------------------------
# Minimal httpx fake (no MockTransport — avoids the multipart encode path)
# ---------------------------------------------------------------------------
class _FakeResponse:
def __init__(
self,
*,
status_code: int = 200,
text: str = "",
content: bytes = b"",
headers: dict[str, str] | None = None,
) -> None:
self.status_code = status_code
self.text = text
self.content = content or text.encode("utf-8")
self.headers = headers or {}
def json(self) -> Any:
return json.loads(self.text) if self.text else {}
def raise_for_status(self) -> None:
if self.status_code >= 400:
raise RuntimeError(f"HTTP {self.status_code}")
class _Recorder:
def __init__(
self,
*,
terminal_status: str,
zip_bytes: bytes,
task_id: str = "task-abc",
submit_status_code: int = 200,
submit_text: str | None = None,
poll_status_code: int = 200,
poll_text: str | None = None,
result_status_code: int = 200,
result_text: str | None = None,
) -> None:
self.terminal_status = terminal_status
self.zip_bytes = zip_bytes
self.task_id = task_id
self.submit_status_code = submit_status_code
self.submit_text = submit_text
self.poll_status_code = poll_status_code
self.poll_text = poll_text
self.result_status_code = result_status_code
self.result_text = result_text
self.post_calls: list[dict] = []
self.get_calls: list[dict] = []
self.result_calls = 0
_CURRENT: dict[str, _Recorder] = {}
class _FakeAsyncClient:
def __init__(self, *_: Any, **__: Any) -> None:
pass
async def __aenter__(self) -> "_FakeAsyncClient":
return self
async def __aexit__(self, *_: Any) -> None:
pass
async def post(
self,
url: str,
files: Any = None,
data: Any = None,
json: Any = None,
headers: Any = None,
) -> _FakeResponse:
recorder = _CURRENT["recorder"]
# Production passes a file handle inside a `with` block — by the time
# tests inspect `post_calls` it's already closed. Drain the stream
# here so assertions can keep reading the payload as bytes.
snapshot_files = files
if files and "files" in files:
name, payload, ctype = files["files"]
if hasattr(payload, "read"):
payload = payload.read()
snapshot_files = {"files": (name, payload, ctype)}
recorder.post_calls.append(
{"url": url, "files": snapshot_files, "data": data, "json": json}
)
if CONVERT_PATH in url:
if recorder.submit_status_code != 200:
return _FakeResponse(
status_code=recorder.submit_status_code,
text=recorder.submit_text or "",
)
return _FakeResponse(
status_code=200,
text=json_dump({"task_id": recorder.task_id}),
)
raise AssertionError(f"unexpected POST {url}")
async def get(
self, url: str, params: Any = None, headers: Any = None
) -> _FakeResponse:
recorder = _CURRENT["recorder"]
recorder.get_calls.append({"url": url, "params": params})
# Mirror production: the client encodes the task id into a single path
# segment, so route on the encoded form (a no-op for URL-safe ids).
encoded = quote(recorder.task_id, safe="")
if POLL_PATH.format(task_id=encoded) in url:
if recorder.poll_status_code != 200:
return _FakeResponse(
status_code=recorder.poll_status_code,
text=recorder.poll_text or "",
)
payload: dict[str, Any] = {
"task_id": recorder.task_id,
"task_status": recorder.terminal_status,
}
if recorder.terminal_status != "success":
payload["error_message"] = "synthetic-failure"
return _FakeResponse(status_code=200, text=json_dump(payload))
if RESULT_PATH.format(task_id=encoded) in url:
recorder.result_calls += 1
if recorder.result_status_code != 200:
return _FakeResponse(
status_code=recorder.result_status_code,
text=recorder.result_text or "",
)
return _FakeResponse(
status_code=200,
content=recorder.zip_bytes,
headers={"content-type": "application/zip"},
)
raise AssertionError(f"unexpected GET {url}")
def json_dump(payload: Any) -> str:
return json.dumps(payload)
def _form_pairs(data: Any) -> list[tuple[str, str]]:
"""Normalize httpx form data into repeated ``(name, value)`` pairs.
Production passes a mapping so httpx 0.28 keeps multipart ``files=`` on
the async path. List values in that mapping represent repeated form keys.
Older tests used tuple lists directly; accepting both keeps assertions
focused on the wire contract instead of the container type.
"""
if isinstance(data, dict):
pairs: list[tuple[str, str]] = []
for name, value in data.items():
values = value if isinstance(value, list) else [value]
pairs.extend((str(name), str(v)) for v in values)
return pairs
return [(str(name), str(value)) for name, value in data]
def _fake_zip_with_main_json(stem: str) -> bytes:
buf = io.BytesIO()
with zipfile.ZipFile(buf, "w") as zf:
zf.writestr(f"{stem}.json", b'{"schema_name": "DoclingDocument"}')
zf.writestr(f"{stem}.md", b"# hello")
return buf.getvalue()
def _install_fake_httpx(monkeypatch: pytest.MonkeyPatch) -> None:
"""Replace ``httpx.AsyncClient`` and ``httpx.Timeout`` references in
the docling client module with no-arg fakes."""
monkeypatch.setattr(
"lightrag.parser.external.docling.client.httpx.AsyncClient",
_FakeAsyncClient,
)
monkeypatch.setattr(
"lightrag.parser.external.docling.client.httpx.Timeout",
lambda *a, **kw: None,
)
# ---------------------------------------------------------------------------
# Fixtures
# ---------------------------------------------------------------------------
@pytest.fixture
def source_pdf(tmp_path: Path) -> Path:
p = tmp_path / "demo.pdf"
p.write_bytes(b"%PDF-1.4 fake")
return p
@pytest.fixture(autouse=True)
def docling_endpoint(monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.setenv("DOCLING_ENDPOINT", "http://docling.test")
for name in (
"DOCLING_DO_OCR",
"DOCLING_FORCE_OCR",
"DOCLING_OCR_ENGINE",
"DOCLING_OCR_PRESET",
"DOCLING_OCR_LANG",
"DOCLING_DO_FORMULA_ENRICHMENT",
"DOCLING_ENGINE_VERSION",
):
monkeypatch.delenv(name, raising=False)
# ---------------------------------------------------------------------------
# Tests
# ---------------------------------------------------------------------------
async def test_docling_client_sends_fixed_constants(
monkeypatch: pytest.MonkeyPatch,
tmp_path: Path,
source_pdf: Path,
) -> None:
recorder = _Recorder(
terminal_status="success",
zip_bytes=_fake_zip_with_main_json("demo"),
)
_CURRENT["recorder"] = recorder
_install_fake_httpx(monkeypatch)
raw_dir = tmp_path / "demo.docling_raw"
manifest = await DoclingRawClient().download_into(raw_dir, source_pdf)
assert len(recorder.post_calls) == 1
data = recorder.post_calls[0]["data"]
field_map: dict[str, list[str]] = {}
for name, value in _form_pairs(data):
field_map.setdefault(name, []).append(value)
assert field_map["pipeline"] == ["standard"]
assert field_map["target_type"] == ["zip"]
assert field_map["image_export_mode"] == ["referenced"]
assert sorted(field_map["to_formats"]) == ["json", "md"]
files = recorder.post_calls[0]["files"]
assert "files" in files
name, blob, ctype = files["files"]
assert name == "demo.pdf"
assert blob.startswith(b"%PDF-1.4")
assert ctype == "application/octet-stream"
assert manifest.task_id == recorder.task_id
assert manifest.engine == "docling"
assert manifest.extras["fixed_constants"]["pipeline"] == "standard"
assert manifest.endpoint_signature == "http://docling.test"
async def test_docling_client_partial_success_aborts(
monkeypatch: pytest.MonkeyPatch,
tmp_path: Path,
source_pdf: Path,
) -> None:
recorder = _Recorder(
terminal_status="partial_success",
zip_bytes=_fake_zip_with_main_json("demo"),
)
_CURRENT["recorder"] = recorder
_install_fake_httpx(monkeypatch)
with pytest.raises(RuntimeError) as excinfo:
await DoclingRawClient().download_into(
tmp_path / "demo.docling_raw", source_pdf
)
msg = str(excinfo.value)
assert recorder.task_id in msg
assert "partial_success" in msg
assert "synthetic-failure" in msg
assert recorder.result_calls == 0
async def test_docling_client_failure_aborts(
monkeypatch: pytest.MonkeyPatch,
tmp_path: Path,
source_pdf: Path,
) -> None:
recorder = _Recorder(
terminal_status="failure",
zip_bytes=_fake_zip_with_main_json("demo"),
)
_CURRENT["recorder"] = recorder
_install_fake_httpx(monkeypatch)
with pytest.raises(RuntimeError):
await DoclingRawClient().download_into(
tmp_path / "demo.docling_raw", source_pdf
)
assert recorder.result_calls == 0
async def test_docling_client_skipped_aborts(
monkeypatch: pytest.MonkeyPatch,
tmp_path: Path,
source_pdf: Path,
) -> None:
recorder = _Recorder(
terminal_status="skipped",
zip_bytes=_fake_zip_with_main_json("demo"),
)
_CURRENT["recorder"] = recorder
_install_fake_httpx(monkeypatch)
with pytest.raises(RuntimeError):
await DoclingRawClient().download_into(
tmp_path / "demo.docling_raw", source_pdf
)
assert recorder.result_calls == 0
async def test_docling_client_upload_http_error_preserves_response_body(
monkeypatch: pytest.MonkeyPatch,
tmp_path: Path,
source_pdf: Path,
) -> None:
recorder = _Recorder(
terminal_status="success",
zip_bytes=_fake_zip_with_main_json("demo"),
submit_status_code=400,
submit_text=json_dump({"detail": "unsupported file type"}),
)
_CURRENT["recorder"] = recorder
_install_fake_httpx(monkeypatch)
with pytest.raises(RuntimeError) as excinfo:
await DoclingRawClient().download_into(
tmp_path / "demo.docling_raw", source_pdf
)
message = str(excinfo.value)
assert "Docling upload for 'demo.pdf'" in message
assert "HTTP 400" in message
assert "unsupported file type" in message
async def test_docling_client_poll_http_error_preserves_response_body(
monkeypatch: pytest.MonkeyPatch,
tmp_path: Path,
source_pdf: Path,
) -> None:
recorder = _Recorder(
terminal_status="success",
zip_bytes=_fake_zip_with_main_json("demo"),
poll_status_code=503,
poll_text=json_dump({"message": "queue unavailable"}),
)
_CURRENT["recorder"] = recorder
_install_fake_httpx(monkeypatch)
with pytest.raises(RuntimeError) as excinfo:
await DoclingRawClient().download_into(
tmp_path / "demo.docling_raw", source_pdf
)
message = str(excinfo.value)
assert "Docling task task-abc poll" in message
assert "HTTP 503" in message
assert "queue unavailable" in message
async def test_docling_client_result_redirect_treated_as_error(
monkeypatch: pytest.MonkeyPatch,
tmp_path: Path,
source_pdf: Path,
) -> None:
# docling-serve fronted by a misconfigured proxy could emit a 302 to a
# CDN that httpx (default ``follow_redirects=False``) won't follow.
# Without the explicit non-2xx guard the redirect body would fall into
# the zip-decoder and surface as a cryptic "bad zip" error.
recorder = _Recorder(
terminal_status="success",
zip_bytes=_fake_zip_with_main_json("demo"),
result_status_code=302,
result_text="",
)
_CURRENT["recorder"] = recorder
_install_fake_httpx(monkeypatch)
with pytest.raises(RuntimeError) as excinfo:
await DoclingRawClient().download_into(
tmp_path / "demo.docling_raw", source_pdf
)
message = str(excinfo.value)
assert "Docling result task-abc download" in message
assert "HTTP 302" in message
async def test_docling_client_result_http_error_preserves_response_body(
monkeypatch: pytest.MonkeyPatch,
tmp_path: Path,
source_pdf: Path,
) -> None:
recorder = _Recorder(
terminal_status="success",
zip_bytes=_fake_zip_with_main_json("demo"),
result_status_code=500,
result_text="zip artifact missing",
)
_CURRENT["recorder"] = recorder
_install_fake_httpx(monkeypatch)
with pytest.raises(RuntimeError) as excinfo:
await DoclingRawClient().download_into(
tmp_path / "demo.docling_raw", source_pdf
)
message = str(excinfo.value)
assert "Docling result task-abc download" in message
assert "HTTP 500" in message
assert "zip artifact missing" in message
async def test_docling_client_encodes_task_id_into_url_path_segment(
monkeypatch: pytest.MonkeyPatch,
tmp_path: Path,
source_pdf: Path,
) -> None:
# Security regression (CWE-116): the poll/result task id is service-returned.
# A crafted value with ``/`` / ``?`` / ``..`` must be percent-encoded into a
# single path segment so it cannot escape ``/v1/status/poll/{id}`` or append
# a query string to the request the client issues.
malicious = "../admin?x=1"
recorder = _Recorder(
terminal_status="success",
zip_bytes=_fake_zip_with_main_json("demo"),
task_id=malicious,
)
_CURRENT["recorder"] = recorder
_install_fake_httpx(monkeypatch)
await DoclingRawClient().download_into(tmp_path / "demo.docling_raw", source_pdf)
poll_url = recorder.get_calls[0]["url"]
result_url = recorder.get_calls[-1]["url"]
for url in (poll_url, result_url):
# ``..`` survives (dot is unreserved) but the separators that grant
# request-structure control are neutralized.
assert "..%2Fadmin%3Fx%3D1" in url
assert "/admin" not in url
assert "?x=1" not in url
async def test_docling_client_ocr_lang_omitted_when_empty(
monkeypatch: pytest.MonkeyPatch,
tmp_path: Path,
source_pdf: Path,
) -> None:
recorder = _Recorder(
terminal_status="success",
zip_bytes=_fake_zip_with_main_json("demo"),
)
_CURRENT["recorder"] = recorder
_install_fake_httpx(monkeypatch)
await DoclingRawClient().download_into(tmp_path / "demo.docling_raw", source_pdf)
data = recorder.post_calls[0]["data"]
names = [name for name, _ in _form_pairs(data)]
assert "ocr_lang" not in names
async def test_docling_client_ocr_lang_sent_when_set(
monkeypatch: pytest.MonkeyPatch,
tmp_path: Path,
source_pdf: Path,
) -> None:
monkeypatch.setenv("DOCLING_OCR_LANG", '["en","zh"]')
recorder = _Recorder(
terminal_status="success",
zip_bytes=_fake_zip_with_main_json("demo"),
)
_CURRENT["recorder"] = recorder
_install_fake_httpx(monkeypatch)
await DoclingRawClient().download_into(tmp_path / "demo.docling_raw", source_pdf)
data = recorder.post_calls[0]["data"]
langs = [v for name, v in _form_pairs(data) if name == "ocr_lang"]
assert langs == ["en", "zh"]
async def test_docling_client_ocr_lang_csv_form(
monkeypatch: pytest.MonkeyPatch,
tmp_path: Path,
source_pdf: Path,
) -> None:
"""CSV fallback when value isn't valid JSON."""
monkeypatch.setenv("DOCLING_OCR_LANG", "en, fr")
recorder = _Recorder(
terminal_status="success",
zip_bytes=_fake_zip_with_main_json("demo"),
)
_CURRENT["recorder"] = recorder
_install_fake_httpx(monkeypatch)
await DoclingRawClient().download_into(tmp_path / "demo.docling_raw", source_pdf)
data = recorder.post_calls[0]["data"]
langs = [v for name, v in _form_pairs(data) if name == "ocr_lang"]
assert langs == ["en", "fr"]
async def test_docling_client_rejects_missing_endpoint(
monkeypatch: pytest.MonkeyPatch,
) -> None:
monkeypatch.setenv("DOCLING_ENDPOINT", "")
with pytest.raises(ValueError, match="DOCLING_ENDPOINT"):
DoclingRawClient()
async def test_docling_client_strips_parser_hint_from_upload_filename(
monkeypatch: pytest.MonkeyPatch, tmp_path: Path
) -> None:
# Regression: a hinted source (``report.[docling].pdf``) used to cause
# docling-serve to name its bundle JSON ``report.[docling].json``, which
# the adapter (looking for ``report.json``) could not locate. The
# pipeline now passes the canonical name as ``upload_filename`` so the
# bundle is canonical-stem from the start.
hinted = tmp_path / "report.[docling].pdf"
hinted.write_bytes(b"%PDF-1.4 fake")
# The fake zip mimics docling-serve responding with the *canonical* stem,
# which is what would happen once we send the canonical filename.
recorder = _Recorder(
terminal_status="success",
zip_bytes=_fake_zip_with_main_json("report"),
)
_CURRENT["recorder"] = recorder
_install_fake_httpx(monkeypatch)
raw_dir = tmp_path / "report.docling_raw"
manifest = await DoclingRawClient().download_into(
raw_dir, hinted, upload_filename="report.pdf"
)
name, _blob, _ctype = recorder.post_calls[0]["files"]["files"]
assert name == "report.pdf"
assert manifest.source_filename_at_parse == "report.pdf"
assert manifest.critical_file.path == "report.json"
assert (raw_dir / "report.json").is_file()
async def test_docling_client_default_upload_filename_falls_back_to_source_name(
monkeypatch: pytest.MonkeyPatch, tmp_path: Path, source_pdf: Path
) -> None:
# Back-compat guard: callers that don't pass ``upload_filename`` (any
# path other than the production pipeline) keep the legacy behavior of
# using the on-disk source filename.
recorder = _Recorder(
terminal_status="success",
zip_bytes=_fake_zip_with_main_json("demo"),
)
_CURRENT["recorder"] = recorder
_install_fake_httpx(monkeypatch)
await DoclingRawClient().download_into(tmp_path / "demo.docling_raw", source_pdf)
name, _blob, _ctype = recorder.post_calls[0]["files"]["files"]
assert name == "demo.pdf"
File diff suppressed because it is too large Load Diff
+57
View File
@@ -0,0 +1,57 @@
"""Tests for ``lightrag/parser/external/docling/manifest.py`` helpers.
Targets the contract guarantees that the rest of the docling flow relies on:
``select_main_json`` must find the bundle's main JSON even when ``_manifest.json``
sits alongside it, and the preferred-path lookup must take priority over the
fallback glob.
"""
from __future__ import annotations
from pathlib import Path
import pytest
from lightrag.parser.external.docling.manifest import select_main_json
def _touch(path: Path, content: str = "{}") -> None:
path.parent.mkdir(parents=True, exist_ok=True)
path.write_text(content, encoding="utf-8")
def test_select_main_json_preferred_path_hits(tmp_path: Path) -> None:
# Manifest is present (the typical post-download state), but the preferred
# ``<stem>.json`` exists, so the fallback glob is not consulted at all.
_touch(tmp_path / "report.json")
_touch(tmp_path / "_manifest.json", '{"engine":"docling"}')
assert select_main_json(tmp_path, Path("report.pdf")) == tmp_path / "report.json"
def test_select_main_json_fallback_ignores_manifest(tmp_path: Path) -> None:
# Defensive: when the preferred path misses (e.g. docling-serve renamed
# the stem for whatever reason), the fallback glob must NOT confuse
# ``_manifest.json`` for a bundle JSON. Pre-fix this case raised
# "multiple .json candidates".
_touch(tmp_path / "report.json")
_touch(tmp_path / "_manifest.json", '{"engine":"docling"}')
assert select_main_json(tmp_path, Path("other.pdf")) == tmp_path / "report.json"
def test_select_main_json_raises_when_only_manifest_present(tmp_path: Path) -> None:
# If the bundle JSON is genuinely missing, the manifest alone is not a
# valid substitute — the helper must still raise rather than silently
# returning the manifest.
_touch(tmp_path / "_manifest.json", '{"engine":"docling"}')
with pytest.raises(RuntimeError, match="contains no .json file"):
select_main_json(tmp_path, Path("report.pdf"))
def test_select_main_json_raises_on_real_ambiguity(tmp_path: Path) -> None:
# Two genuine bundle JSONs is still an error; the manifest filter must
# not mask multi-candidate detection.
_touch(tmp_path / "report.json")
_touch(tmp_path / "extra.json")
_touch(tmp_path / "_manifest.json", '{"engine":"docling"}')
with pytest.raises(RuntimeError, match="multiple .json candidates"):
select_main_json(tmp_path, Path("other.pdf"))
@@ -0,0 +1,668 @@
"""Integration tests for ``parse_docling`` with the unified sidecar pipeline.
Stubs :class:`DoclingRawClient.download_into` so no real docling-serve is
contacted; the focus is on:
- happy path: cache miss → fake bundle written → sidecar emitted with all
expected files at the spec-compliant locations
- cache hit: a pre-existing valid ``*.docling_raw/`` + manifest causes
``DoclingRawClient.download_into`` NOT to be called
- ``LIGHTRAG_FORCE_REPARSE_DOCLING=true`` forces a re-download even when
the manifest is valid
- source content swap → cache miss
- options_signature change (``DOCLING_OCR_LANG`` toggle) → cache miss
- adapter sees zero blocks → parse fails loudly (no half-baked sidecar)
"""
from __future__ import annotations
import asyncio
import json
from datetime import datetime, timezone
from pathlib import Path
from typing import Any
import numpy as np
import pytest
from lightrag import LightRAG
from lightrag.constants import FULL_DOCS_FORMAT_LIGHTRAG
from lightrag.parser.external import (
Manifest,
ManifestFile,
compute_size_and_hash,
write_manifest,
)
from lightrag.parser.external.docling.cache import (
compute_options_signature,
snapshot_tunable_env,
)
from lightrag.parser.external.docling.client import FIXED_CONSTANTS
from lightrag.parser.base import ParseContext
from lightrag.parser.registry import get_parser
from lightrag.utils import EmbeddingFunc, Tokenizer
async def _parse_via_registry(rag, engine, doc_id, file_path, content_data):
"""Drive a parser the way the pipeline worker does (registry dispatch)."""
result = await get_parser(engine).parse(
ParseContext(rag, doc_id, file_path, content_data)
)
return result.to_dict()
class _SimpleTokenizerImpl:
def encode(self, content: str) -> list[int]:
return [ord(ch) for ch in content]
def decode(self, tokens: list[int]) -> str:
return "".join(chr(t) for t in tokens)
async def _mock_embedding(texts: list[str]) -> np.ndarray:
return np.random.rand(len(texts), 32)
async def _mock_llm(prompt: Any, **kwargs: Any) -> str:
return '{"name":"x","summary":"s","detail_description":"d"}'
def _new_rag(tmp_path: Path) -> LightRAG:
return LightRAG(
working_dir=str(tmp_path),
workspace=f"test-docling-sidecar-{tmp_path.name}",
llm_model_func=_mock_llm,
embedding_func=EmbeddingFunc(
embedding_dim=32,
max_token_size=4096,
func=_mock_embedding,
),
tokenizer=Tokenizer("mock-tokenizer", _SimpleTokenizerImpl()),
vlm_process_enable=False,
)
_FAKE_DOCLING_JSON = {
"schema_name": "DoclingDocument",
"version": "1.10.0",
"origin": {"filename": "demo.pdf", "mimetype": "application/pdf"},
"body": {
"self_ref": "#/body",
"children": [
{"$ref": "#/texts/0"},
],
"content_layer": "body",
"label": "unspecified",
},
"groups": [],
"texts": [
{
"self_ref": "#/texts/0",
"label": "section_header",
"text": "Intro",
"orig": "Intro",
"level": 1,
"content_layer": "body",
"children": [
{"$ref": "#/texts/1"},
{"$ref": "#/tables/0"},
{"$ref": "#/pictures/0"},
{"$ref": "#/texts/2"},
],
"prov": [
{
"page_no": 1,
"bbox": {
"l": 10.0,
"t": 100.0,
"r": 200.0,
"b": 80.0,
"coord_origin": "BOTTOMLEFT",
},
"charspan": [0, 5],
}
],
},
{
"self_ref": "#/texts/1",
"label": "text",
"text": "Body paragraph.",
"orig": "Body paragraph.",
"content_layer": "body",
"prov": [
{
"page_no": 1,
"bbox": {
"l": 10.0,
"t": 60.0,
"r": 200.0,
"b": 40.0,
"coord_origin": "BOTTOMLEFT",
},
"charspan": [0, 15],
}
],
},
{
"self_ref": "#/texts/2",
"label": "formula",
"text": "E = mc^2",
"orig": "E = mc^2",
"content_layer": "body",
"prov": [],
},
],
"tables": [
{
"self_ref": "#/tables/0",
"label": "table",
"content_layer": "body",
"data": {
"num_rows": 2,
"num_cols": 2,
"grid": [
[{"text": "h1"}, {"text": "h2"}],
[{"text": "a"}, {"text": "b"}],
],
},
"prov": [],
}
],
"pictures": [
{
"self_ref": "#/pictures/0",
"label": "picture",
"content_layer": "body",
"image": {"uri": "artifacts/img_000000.png", "mimetype": "image/png"},
"prov": [],
}
],
"key_value_items": [],
"form_items": [],
"pages": {"1": {"size": {"width": 612.0, "height": 792.0}, "page_no": 1}},
}
def _install_fake_download(monkeypatch: pytest.MonkeyPatch) -> dict[str, int]:
"""Replace ``DoclingRawClient.download_into`` with a recorder that
writes a synthetic raw bundle and a valid manifest."""
import lightrag.parser.external.docling.client as client_mod
counters = {"calls": 0}
async def _fake_download(self, raw_dir: Path, source_file_path: Path, **_kwargs):
counters["calls"] += 1
raw_dir.mkdir(parents=True, exist_ok=True)
main_json = raw_dir / "demo.json"
main_json.write_text(json.dumps(_FAKE_DOCLING_JSON), encoding="utf-8")
(raw_dir / "demo.md").write_text("# fake md", encoding="utf-8")
art = raw_dir / "artifacts"
art.mkdir(exist_ok=True)
(art / "img_000000.png").write_bytes(b"\x89PNG fake")
src_size, src_hash = compute_size_and_hash(source_file_path)
crit_size, crit_hash = compute_size_and_hash(main_json)
others = [
ManifestFile(path="demo.md", size=(raw_dir / "demo.md").stat().st_size),
ManifestFile(
path="artifacts/img_000000.png",
size=(art / "img_000000.png").stat().st_size,
),
]
options_signature = compute_options_signature(
tunable_env=snapshot_tunable_env(),
fixed_constants=FIXED_CONSTANTS,
)
manifest = Manifest(
engine="docling",
source_content_hash=src_hash,
source_size_bytes=src_size,
source_filename_at_parse=source_file_path.name,
critical_file=ManifestFile(
path="demo.json", size=crit_size, sha256=crit_hash
),
files=others,
total_size_bytes=crit_size + sum(f.size for f in others),
task_id=f"fake-{counters['calls']}",
endpoint_signature="http://docling.test",
options_signature=options_signature,
extras={"fixed_constants": dict(FIXED_CONSTANTS)},
downloaded_at=datetime.now(timezone.utc).isoformat(timespec="seconds"),
)
write_manifest(raw_dir, manifest)
return manifest
monkeypatch.setattr(client_mod.DoclingRawClient, "download_into", _fake_download)
return counters
def _stub_pipeline(monkeypatch: pytest.MonkeyPatch, rag: LightRAG, src: Path) -> None:
"""Common pipeline-level stubs: avoid moving the source file and pin
the file resolver to the synthetic path."""
async def _noop_archive(_p: str) -> None:
return None
import lightrag.pipeline as pipeline_module
monkeypatch.setattr(
pipeline_module,
"archive_docx_source_after_full_docs_sync",
_noop_archive,
)
monkeypatch.setattr(rag, "_resolve_source_file_for_parser", lambda _p: str(src))
def _seed_doc_status(rag: LightRAG, doc_id: str) -> Any:
return rag.doc_status.upsert(
{
doc_id: {
"status": "PARSING",
"content_summary": "",
"content_length": 0,
"chunks_count": 0,
"chunks_list": [],
"created_at": "2026-05-18T00:00:00+00:00",
"updated_at": "2026-05-18T00:00:00+00:00",
"file_path": "demo.pdf",
"track_id": "trk",
"content_hash": "",
"metadata": {},
}
}
)
# ---------------------------------------------------------------------------
# Tests
# ---------------------------------------------------------------------------
@pytest.mark.offline
def test_parse_docling_emits_compliant_sidecar(
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
) -> None:
async def _run() -> None:
monkeypatch.setenv("DOCLING_ENDPOINT", "http://docling.test")
counters = _install_fake_download(monkeypatch)
input_dir = tmp_path / "inputs" / "ws"
input_dir.mkdir(parents=True)
src = input_dir / "demo.pdf"
src.write_bytes(b"PDFPDF" * 256)
rag = _new_rag(tmp_path)
await rag.initialize_storages()
try:
_stub_pipeline(monkeypatch, rag, src)
doc_id = "doc-abcdef0123456789abcdef0123456789"
await _seed_doc_status(rag, doc_id)
parsed = await _parse_via_registry(
rag,
"docling",
doc_id=doc_id,
file_path="demo.pdf",
content_data={},
)
assert counters["calls"] == 1
parsed_dir = Path(parsed["blocks_path"]).parent
assert parsed["parse_format"] == FULL_DOCS_FORMAT_LIGHTRAG
assert parsed_dir.name == "demo.pdf.parsed"
files = {p.name for p in parsed_dir.iterdir() if p.is_file()}
assert "demo.blocks.jsonl" in files
assert "demo.tables.json" in files
assert "demo.drawings.json" in files
assert "demo.equations.json" in files
assert (parsed_dir / "demo.blocks.assets").is_dir()
assert (parsed_dir / "demo.blocks.assets" / "img_000000.png").is_file()
blocks_raw = (parsed_dir / "demo.blocks.jsonl").read_text()
lines = blocks_raw.splitlines()
meta = json.loads(lines[0])
rows = [json.loads(line) for line in lines[1:]]
assert meta["parse_engine"] == "docling"
assert meta["bbox_attributes"] == {"origin": "LEFTBOTTOM"}
assert "max" not in meta["bbox_attributes"]
assert "page_sizes" not in meta["bbox_attributes"]
assert meta["table_file"] is True
assert meta["drawing_file"] is True
assert meta["equation_file"] is True
# No label="title" in the fixture (matches the typical PDF case
# where docling produces only section_headers) → doc_title falls
# back to the document stem.
assert meta["doc_title"] == "demo"
contents = " ".join(row.get("content", "") for row in rows)
assert '<table id="tb-' in contents
assert "<drawing" in contents
assert "<equation" in contents
# Raw bundle preserved next to sidecar
raw_dir = parsed_dir.parent / "demo.pdf.docling_raw"
assert (raw_dir / "_manifest.json").is_file()
assert (raw_dir / "demo.json").is_file()
assert (raw_dir / "demo.md").is_file()
assert (raw_dir / "artifacts" / "img_000000.png").is_file()
# Drawing path correctly resolved
drawings = json.loads((parsed_dir / "demo.drawings.json").read_text())[
"drawings"
]
(drawing_id, drawing_item) = next(iter(drawings.items()))
assert drawing_id.startswith("im-")
assert drawing_item["path"] == "demo.blocks.assets/img_000000.png"
# Table self_ref propagated
tables = json.loads((parsed_dir / "demo.tables.json").read_text())["tables"]
(_, table_item) = next(iter(tables.items()))
assert table_item.get("self_ref") == "#/tables/0"
finally:
await rag.finalize_storages()
asyncio.new_event_loop().run_until_complete(_run())
@pytest.mark.offline
def test_parse_docling_cache_hit_skips_download(
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
) -> None:
async def _run() -> None:
monkeypatch.setenv("DOCLING_ENDPOINT", "http://docling.test")
counters = _install_fake_download(monkeypatch)
input_dir = tmp_path / "inputs" / "ws"
input_dir.mkdir(parents=True)
src = input_dir / "demo.pdf"
src.write_bytes(b"PDFPDF" * 256)
rag = _new_rag(tmp_path)
await rag.initialize_storages()
try:
_stub_pipeline(monkeypatch, rag, src)
doc_id = "doc-abcdef0123456789abcdef0123456789"
await _seed_doc_status(rag, doc_id)
await _parse_via_registry(
rag,
"docling",
doc_id=doc_id,
file_path="demo.pdf",
content_data={},
)
assert counters["calls"] == 1
await _parse_via_registry(
rag,
"docling",
doc_id=doc_id,
file_path="demo.pdf",
content_data={},
)
assert counters["calls"] == 1, "cache hit must not re-download"
monkeypatch.setenv("LIGHTRAG_FORCE_REPARSE_DOCLING", "true")
await _parse_via_registry(
rag,
"docling",
doc_id=doc_id,
file_path="demo.pdf",
content_data={},
)
assert counters["calls"] == 2
finally:
await rag.finalize_storages()
asyncio.new_event_loop().run_until_complete(_run())
@pytest.mark.offline
def test_parse_docling_cache_invalidates_on_source_change(
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
) -> None:
async def _run() -> None:
monkeypatch.setenv("DOCLING_ENDPOINT", "http://docling.test")
counters = _install_fake_download(monkeypatch)
input_dir = tmp_path / "inputs" / "ws"
input_dir.mkdir(parents=True)
src = input_dir / "demo.pdf"
src.write_bytes(b"PDFPDF" * 256)
rag = _new_rag(tmp_path)
await rag.initialize_storages()
try:
_stub_pipeline(monkeypatch, rag, src)
doc_id = "doc-abcdef0123456789abcdef0123456789"
await _seed_doc_status(rag, doc_id)
await _parse_via_registry(
rag,
"docling",
doc_id=doc_id,
file_path="demo.pdf",
content_data={},
)
assert counters["calls"] == 1
data = src.read_bytes()
src.write_bytes(b"\x00" + data[1:])
await _parse_via_registry(
rag,
"docling",
doc_id=doc_id,
file_path="demo.pdf",
content_data={},
)
assert counters["calls"] == 2
finally:
await rag.finalize_storages()
asyncio.new_event_loop().run_until_complete(_run())
@pytest.mark.offline
def test_parse_docling_options_signature_invalidates_cache(
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
) -> None:
async def _run() -> None:
monkeypatch.setenv("DOCLING_ENDPOINT", "http://docling.test")
counters = _install_fake_download(monkeypatch)
input_dir = tmp_path / "inputs" / "ws"
input_dir.mkdir(parents=True)
src = input_dir / "demo.pdf"
src.write_bytes(b"PDFPDF" * 256)
rag = _new_rag(tmp_path)
await rag.initialize_storages()
try:
_stub_pipeline(monkeypatch, rag, src)
doc_id = "doc-abcdef0123456789abcdef0123456789"
await _seed_doc_status(rag, doc_id)
await _parse_via_registry(
rag,
"docling",
doc_id=doc_id,
file_path="demo.pdf",
content_data={},
)
assert counters["calls"] == 1
# Flip an env var that participates in the options signature
monkeypatch.setenv("DOCLING_OCR_LANG", "en,zh")
await _parse_via_registry(
rag,
"docling",
doc_id=doc_id,
file_path="demo.pdf",
content_data={},
)
assert counters["calls"] == 2, (
"DOCLING_OCR_LANG change must invalidate the bundle cache"
)
finally:
await rag.finalize_storages()
asyncio.new_event_loop().run_until_complete(_run())
@pytest.mark.offline
def test_parse_docling_endpoint_signature_invalidates_cache(
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
) -> None:
async def _run() -> None:
monkeypatch.setenv("DOCLING_ENDPOINT", "http://docling.test")
counters = _install_fake_download(monkeypatch)
input_dir = tmp_path / "inputs" / "ws"
input_dir.mkdir(parents=True)
src = input_dir / "demo.pdf"
src.write_bytes(b"PDFPDF" * 256)
rag = _new_rag(tmp_path)
await rag.initialize_storages()
try:
_stub_pipeline(monkeypatch, rag, src)
doc_id = "doc-abcdef0123456789abcdef0123456789"
await _seed_doc_status(rag, doc_id)
await _parse_via_registry(
rag,
"docling",
doc_id=doc_id,
file_path="demo.pdf",
content_data={},
)
assert counters["calls"] == 1
# Pointing at a different docling-serve instance must not silently
# reuse a bundle that was produced by the previous one.
monkeypatch.setenv("DOCLING_ENDPOINT", "http://docling-other.test")
await _parse_via_registry(
rag,
"docling",
doc_id=doc_id,
file_path="demo.pdf",
content_data={},
)
assert counters["calls"] == 2, (
"DOCLING_ENDPOINT change must invalidate the bundle cache"
)
finally:
await rag.finalize_storages()
asyncio.new_event_loop().run_until_complete(_run())
@pytest.mark.offline
def test_parse_docling_zero_blocks_raises(
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
) -> None:
"""When the docling bundle yields no body blocks (e.g. everything was
classified as furniture/background) ``parse_docling`` must fail loudly
so the document is marked failed — never persist a half-baked sidecar.
"""
async def _run() -> None:
monkeypatch.setenv("DOCLING_ENDPOINT", "http://docling.test")
# Install a fake download that writes a valid bundle whose body has
# no children — the adapter then produces zero IR blocks.
import lightrag.parser.external.docling.client as client_mod
empty_json: dict[str, Any] = {
"schema_name": "DoclingDocument",
"version": "1.10.0",
"origin": {"filename": "demo.pdf", "mimetype": "application/pdf"},
"body": {
"self_ref": "#/body",
"children": [],
"content_layer": "body",
"label": "unspecified",
},
"groups": [],
"texts": [],
"tables": [],
"pictures": [],
"key_value_items": [],
"form_items": [],
"pages": {},
}
async def _fake_download(
self, raw_dir: Path, source_file_path: Path, **_kwargs
):
raw_dir.mkdir(parents=True, exist_ok=True)
main_json = raw_dir / "demo.json"
main_json.write_text(json.dumps(empty_json), encoding="utf-8")
(raw_dir / "demo.md").write_text("# empty", encoding="utf-8")
src_size, src_hash = compute_size_and_hash(source_file_path)
crit_size, crit_hash = compute_size_and_hash(main_json)
others = [
ManifestFile(path="demo.md", size=(raw_dir / "demo.md").stat().st_size),
]
options_signature = compute_options_signature(
tunable_env=snapshot_tunable_env(),
fixed_constants=FIXED_CONSTANTS,
)
manifest = Manifest(
engine="docling",
source_content_hash=src_hash,
source_size_bytes=src_size,
source_filename_at_parse=source_file_path.name,
critical_file=ManifestFile(
path="demo.json", size=crit_size, sha256=crit_hash
),
files=others,
total_size_bytes=crit_size + sum(f.size for f in others),
task_id="fake-empty",
endpoint_signature="http://docling.test",
options_signature=options_signature,
extras={"fixed_constants": dict(FIXED_CONSTANTS)},
downloaded_at=datetime.now(timezone.utc).isoformat(timespec="seconds"),
)
write_manifest(raw_dir, manifest)
return manifest
monkeypatch.setattr(
client_mod.DoclingRawClient, "download_into", _fake_download
)
input_dir = tmp_path / "inputs" / "ws"
input_dir.mkdir(parents=True)
src = input_dir / "demo.pdf"
src.write_bytes(b"PDFPDF" * 256)
rag = _new_rag(tmp_path)
await rag.initialize_storages()
try:
_stub_pipeline(monkeypatch, rag, src)
doc_id = "doc-abcdef0123456789abcdef0123456789"
await _seed_doc_status(rag, doc_id)
with pytest.raises(ValueError, match="zero blocks"):
await _parse_via_registry(
rag,
"docling",
doc_id=doc_id,
file_path="demo.pdf",
content_data={},
)
# Sidecar must NOT have been emitted: ``write_sidecar`` is reached
# only after the zero-blocks check, so no ``*.blocks.jsonl`` may
# exist anywhere under the workspace.
blocks_files = list(tmp_path.rglob("*.blocks.jsonl"))
assert not blocks_files, (
f"sidecar emitted despite zero-blocks failure: {blocks_files}"
)
finally:
await rag.finalize_storages()
asyncio.new_event_loop().run_until_complete(_run())
View File
+461
View File
@@ -0,0 +1,461 @@
"""``*.mineru_raw/`` cache validation tests.
Covers every failure mode that triggers a re-download:
- missing / malformed manifest
- source file size mismatch (fast-path)
- source file content_hash mismatch
- parser options signature missing / mismatch
- engine version / endpoint env mismatch
- critical_file (content_list.json) size or sha256 mismatch
- any non-critical file size mismatch
"""
from __future__ import annotations
import json
from pathlib import Path
import pytest
from lightrag.parser.external.mineru import (
Manifest,
ManifestFile,
clear_dir_contents,
compute_size_and_hash,
is_bundle_valid,
raw_dir_for_parsed_dir,
)
from lightrag.parser.external.mineru.cache import current_mineru_options_signature
from lightrag.parser.external.mineru.manifest import write_manifest
# ---------------------------------------------------------------------------
# Fixtures
# ---------------------------------------------------------------------------
@pytest.fixture
def source_file(tmp_path: Path) -> Path:
p = tmp_path / "src.pdf"
p.write_bytes(b"Hello PDF" * 100)
return p
@pytest.fixture
def fresh_bundle(tmp_path: Path, source_file: Path) -> tuple[Path, Manifest]:
"""Build a fully-valid bundle alongside ``source_file`` and return
``(raw_dir, manifest)``."""
raw = tmp_path / "src.mineru_raw"
raw.mkdir()
content_list = raw / "content_list.json"
content_list.write_text('[{"type":"text","text":"hi"}]', encoding="utf-8")
images = raw / "images"
images.mkdir()
(images / "img1.png").write_bytes(b"PNG" * 50)
(images / "img2.png").write_bytes(b"PNG" * 60)
src_size, src_hash = compute_size_and_hash(source_file)
crit_size, crit_hash = compute_size_and_hash(content_list)
files = [
ManifestFile(path="images/img1.png", size=(images / "img1.png").stat().st_size),
ManifestFile(path="images/img2.png", size=(images / "img2.png").stat().st_size),
]
manifest = Manifest(
source_content_hash=src_hash,
source_size_bytes=src_size,
source_filename_at_parse=source_file.name,
critical_file=ManifestFile(
path="content_list.json", size=crit_size, sha256=crit_hash
),
files=files,
total_size_bytes=crit_size + sum(f.size for f in files),
task_id="task-1",
api_mode="local",
options_signature=current_mineru_options_signature(),
)
write_manifest(raw, manifest)
return raw, manifest
# ---------------------------------------------------------------------------
# Layout helpers
# ---------------------------------------------------------------------------
@pytest.mark.offline
def test_raw_dir_naming(tmp_path: Path) -> None:
parsed = tmp_path / "report.pdf.parsed"
raw = raw_dir_for_parsed_dir(parsed)
assert raw.name == "report.pdf.mineru_raw"
assert raw.parent == parsed.parent
# ---------------------------------------------------------------------------
# Validation: happy path + every individual failure mode
# ---------------------------------------------------------------------------
@pytest.mark.offline
def test_is_bundle_valid_happy_path(
fresh_bundle: tuple[Path, Manifest], source_file: Path
) -> None:
raw, _ = fresh_bundle
assert is_bundle_valid(raw, source_file) is True
@pytest.mark.offline
def test_invalid_when_manifest_missing(
fresh_bundle: tuple[Path, Manifest], source_file: Path
) -> None:
raw, _ = fresh_bundle
(raw / "_manifest.json").unlink()
assert is_bundle_valid(raw, source_file) is False
@pytest.mark.offline
def test_invalid_when_manifest_malformed(
fresh_bundle: tuple[Path, Manifest], source_file: Path
) -> None:
raw, _ = fresh_bundle
(raw / "_manifest.json").write_text("not json")
assert is_bundle_valid(raw, source_file) is False
@pytest.mark.offline
def test_invalid_when_manifest_wrong_engine(
fresh_bundle: tuple[Path, Manifest], source_file: Path
) -> None:
raw, _ = fresh_bundle
payload = json.loads((raw / "_manifest.json").read_text())
payload["engine"] = "docling"
(raw / "_manifest.json").write_text(json.dumps(payload))
assert is_bundle_valid(raw, source_file) is False
@pytest.mark.offline
def test_invalid_when_source_size_changes(
fresh_bundle: tuple[Path, Manifest], source_file: Path
) -> None:
# Append bytes to the source file — size diverges from manifest fast-path.
raw, _ = fresh_bundle
with source_file.open("ab") as fh:
fh.write(b"x")
assert is_bundle_valid(raw, source_file) is False
@pytest.mark.offline
def test_invalid_when_source_hash_changes_but_size_same(
fresh_bundle: tuple[Path, Manifest], source_file: Path
) -> None:
"""In-place rewrite that preserves byte size but mutates content. The
fast-path passes but the full hash check catches it."""
raw, _ = fresh_bundle
data = source_file.read_bytes()
# Flip first byte; keep length identical.
mutated = bytes([data[0] ^ 0xFF]) + data[1:]
assert len(mutated) == len(data)
source_file.write_bytes(mutated)
assert is_bundle_valid(raw, source_file) is False
@pytest.mark.offline
def test_invalid_when_engine_version_mismatch(
fresh_bundle: tuple[Path, Manifest],
source_file: Path,
monkeypatch: pytest.MonkeyPatch,
) -> None:
raw, _ = fresh_bundle
payload = json.loads((raw / "_manifest.json").read_text())
payload["engine_version"] = "magic-pdf 1.5.4"
(raw / "_manifest.json").write_text(json.dumps(payload))
monkeypatch.setenv("MINERU_ENGINE_VERSION", "magic-pdf 1.6.0")
assert is_bundle_valid(raw, source_file) is False
@pytest.mark.offline
def test_engine_version_match_passes(
fresh_bundle: tuple[Path, Manifest],
source_file: Path,
monkeypatch: pytest.MonkeyPatch,
) -> None:
raw, _ = fresh_bundle
payload = json.loads((raw / "_manifest.json").read_text())
payload["engine_version"] = "magic-pdf 1.5.4"
(raw / "_manifest.json").write_text(json.dumps(payload))
monkeypatch.setenv("MINERU_ENGINE_VERSION", "magic-pdf 1.5.4")
assert is_bundle_valid(raw, source_file) is True
@pytest.mark.offline
def test_engine_version_skip_when_either_side_blank(
fresh_bundle: tuple[Path, Manifest],
source_file: Path,
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""Blank manifest engine_version + non-blank env should NOT invalidate
(no signal from manifest); same for the reverse."""
raw, _ = fresh_bundle
# Manifest engine_version is empty by default.
monkeypatch.setenv("MINERU_ENGINE_VERSION", "anything")
assert is_bundle_valid(raw, source_file) is True
@pytest.mark.offline
def test_invalid_when_api_mode_mismatch(
fresh_bundle: tuple[Path, Manifest],
source_file: Path,
monkeypatch: pytest.MonkeyPatch,
) -> None:
raw, _ = fresh_bundle
payload = json.loads((raw / "_manifest.json").read_text())
payload["api_mode"] = "local"
(raw / "_manifest.json").write_text(json.dumps(payload))
monkeypatch.setenv("MINERU_API_MODE", "official")
monkeypatch.setenv("MINERU_API_TOKEN", "token")
assert is_bundle_valid(raw, source_file) is False
@pytest.mark.offline
def test_invalid_when_options_signature_missing(
fresh_bundle: tuple[Path, Manifest], source_file: Path
) -> None:
raw, _ = fresh_bundle
payload = json.loads((raw / "_manifest.json").read_text())
payload.pop("options_signature", None)
(raw / "_manifest.json").write_text(json.dumps(payload))
assert is_bundle_valid(raw, source_file) is False
@pytest.mark.offline
@pytest.mark.parametrize(
("key", "value"),
[
("MINERU_LOCAL_BACKEND", "pipeline"),
("MINERU_LOCAL_PARSE_METHOD", "ocr"),
("MINERU_LOCAL_IMAGE_ANALYSIS", "true"),
("MINERU_LOCAL_START_PAGE_ID", "1"),
],
)
def test_invalid_when_local_parser_options_change(
fresh_bundle: tuple[Path, Manifest],
source_file: Path,
monkeypatch: pytest.MonkeyPatch,
key: str,
value: str,
) -> None:
raw, _ = fresh_bundle
monkeypatch.setenv("MINERU_API_MODE", "local")
monkeypatch.setenv(key, value)
assert is_bundle_valid(raw, source_file) is False
@pytest.mark.offline
@pytest.mark.parametrize(
("key", "value"),
[
("MINERU_MODEL_VERSION", "pipeline"),
("MINERU_IS_OCR", "true"),
("MINERU_PAGE_RANGES", "1-5"),
("MINERU_LANGUAGE", "en"),
("MINERU_ENABLE_TABLE", "false"),
("MINERU_ENABLE_FORMULA", "false"),
],
)
def test_invalid_when_official_parser_options_change(
tmp_path: Path,
source_file: Path,
monkeypatch: pytest.MonkeyPatch,
key: str,
value: str,
) -> None:
"""Symmetric coverage for the official-mode partition of the signature.
Build a bundle whose ``options_signature`` reflects the official defaults,
sanity-check that it validates, then flip ``key`` and assert a cache miss.
"""
monkeypatch.setenv("MINERU_API_MODE", "official")
raw = tmp_path / "src.mineru_raw"
raw.mkdir()
content_list = raw / "content_list.json"
content_list.write_text('[{"type":"text","text":"hi"}]', encoding="utf-8")
crit_size, crit_hash = compute_size_and_hash(content_list)
src_size, src_hash = compute_size_and_hash(source_file)
manifest = Manifest(
source_content_hash=src_hash,
source_size_bytes=src_size,
source_filename_at_parse=source_file.name,
critical_file=ManifestFile(
path="content_list.json", size=crit_size, sha256=crit_hash
),
files=[],
total_size_bytes=crit_size,
task_id="task-official",
api_mode="official",
options_signature=current_mineru_options_signature(),
)
write_manifest(raw, manifest)
assert is_bundle_valid(raw, source_file) is True
monkeypatch.setenv(key, value)
assert is_bundle_valid(raw, source_file) is False
@pytest.mark.offline
def test_invalid_when_endpoint_signature_mismatch(
fresh_bundle: tuple[Path, Manifest],
source_file: Path,
monkeypatch: pytest.MonkeyPatch,
) -> None:
raw, _ = fresh_bundle
payload = json.loads((raw / "_manifest.json").read_text())
payload["api_mode"] = "local"
payload["endpoint_signature"] = "http://old.example"
(raw / "_manifest.json").write_text(json.dumps(payload))
monkeypatch.setenv("MINERU_API_MODE", "local")
monkeypatch.setenv("MINERU_LOCAL_ENDPOINT", "http://new.example")
assert is_bundle_valid(raw, source_file) is False
@pytest.mark.offline
def test_endpoint_signature_uses_mode_specific_endpoint(
fresh_bundle: tuple[Path, Manifest],
source_file: Path,
monkeypatch: pytest.MonkeyPatch,
) -> None:
raw, _ = fresh_bundle
payload = json.loads((raw / "_manifest.json").read_text())
payload["api_mode"] = "local"
payload["endpoint_signature"] = "http://old.example"
(raw / "_manifest.json").write_text(json.dumps(payload))
monkeypatch.setenv("MINERU_API_MODE", "local")
monkeypatch.setenv("MINERU_LOCAL_ENDPOINT", "http://new.example")
assert is_bundle_valid(raw, source_file) is False
@pytest.mark.offline
def test_endpoint_signature_ignores_trailing_slash(
fresh_bundle: tuple[Path, Manifest],
source_file: Path,
monkeypatch: pytest.MonkeyPatch,
) -> None:
raw, _ = fresh_bundle
payload = json.loads((raw / "_manifest.json").read_text())
payload["api_mode"] = "local"
payload["endpoint_signature"] = "http://old.example"
(raw / "_manifest.json").write_text(json.dumps(payload))
monkeypatch.setenv("MINERU_API_MODE", "local")
monkeypatch.setenv("MINERU_LOCAL_ENDPOINT", "http://old.example/")
assert is_bundle_valid(raw, source_file) is True
@pytest.mark.offline
def test_invalid_when_critical_file_missing(
fresh_bundle: tuple[Path, Manifest], source_file: Path
) -> None:
raw, _ = fresh_bundle
(raw / "content_list.json").unlink()
assert is_bundle_valid(raw, source_file) is False
@pytest.mark.offline
def test_invalid_when_critical_file_size_changes(
fresh_bundle: tuple[Path, Manifest], source_file: Path
) -> None:
raw, _ = fresh_bundle
cl = raw / "content_list.json"
cl.write_text(cl.read_text() + "/* extra */")
assert is_bundle_valid(raw, source_file) is False
@pytest.mark.offline
def test_invalid_when_critical_file_hash_changes(
fresh_bundle: tuple[Path, Manifest], source_file: Path
) -> None:
"""Same size, different bytes. sha256 is the terminal check."""
raw, _ = fresh_bundle
cl = raw / "content_list.json"
data = cl.read_text()
mutated = data[:-1] + "X" # swap last char; size preserved
assert len(mutated) == len(data)
cl.write_text(mutated)
assert is_bundle_valid(raw, source_file) is False
@pytest.mark.offline
def test_invalid_when_aux_file_size_changes(
fresh_bundle: tuple[Path, Manifest], source_file: Path
) -> None:
raw, _ = fresh_bundle
p = raw / "images" / "img1.png"
p.write_bytes(p.read_bytes() + b"corruption")
assert is_bundle_valid(raw, source_file) is False
@pytest.mark.offline
def test_invalid_when_aux_file_missing(
fresh_bundle: tuple[Path, Manifest], source_file: Path
) -> None:
raw, _ = fresh_bundle
(raw / "images" / "img2.png").unlink()
assert is_bundle_valid(raw, source_file) is False
# ---------------------------------------------------------------------------
# Helpers
# ---------------------------------------------------------------------------
@pytest.mark.offline
def test_clear_dir_contents_preserves_directory(tmp_path: Path) -> None:
d = tmp_path / "raw"
d.mkdir()
(d / "a.txt").write_text("a")
(d / "sub").mkdir()
(d / "sub" / "b.txt").write_text("b")
clear_dir_contents(d)
assert d.exists()
assert list(d.iterdir()) == []
@pytest.mark.offline
def test_compute_size_and_hash_consistency(tmp_path: Path) -> None:
"""Both values describe the same byte stream."""
p = tmp_path / "f.bin"
payload = b"abc" * 1000
p.write_bytes(payload)
size, h = compute_size_and_hash(p)
assert size == len(payload)
assert h.startswith("sha256:") and len(h) == len("sha256:") + 64
@pytest.mark.offline
def test_manifest_round_trip_via_disk(tmp_path: Path) -> None:
"""Write → read recovers all fields."""
raw = tmp_path / "rt.mineru_raw"
raw.mkdir()
m = Manifest(
source_content_hash="sha256:abc",
source_size_bytes=10,
source_filename_at_parse="x.pdf",
critical_file=ManifestFile(
path="content_list.json", size=5, sha256="sha256:cl"
),
files=[ManifestFile(path="images/i.png", size=3)],
total_size_bytes=8,
task_id="t1",
engine_version="v",
endpoint_signature="ep",
options_signature="sha256:opts",
)
write_manifest(raw, m)
from lightrag.parser.external.mineru.manifest import load_manifest
loaded = load_manifest(raw)
assert loaded is not None
assert loaded.source_content_hash == "sha256:abc"
assert loaded.critical_file.sha256 == "sha256:cl"
assert [f.path for f in loaded.files] == ["images/i.png"]
assert loaded.task_id == "t1"
assert loaded.options_signature == "sha256:opts"
File diff suppressed because it is too large Load Diff
@@ -0,0 +1,114 @@
"""``delete_file_variants_by_file_path`` now also clears the
MinerU raw bundle (``*.mineru_raw/``) alongside the sidecar
(``*.parsed/``) and source file when ``delete_file=True`` is selected.
"""
from __future__ import annotations
import sys
from pathlib import Path
import pytest
# document_routes pulls in api.auth → api.config.parse_args(); blank argv
# to keep argparse from rejecting pytest's flags at import time.
sys.argv = sys.argv[:1]
from lightrag.api.routers.document_routes import ( # noqa: E402
_file_path_for_parsed_artifact_dir,
delete_file_variants_by_file_path,
)
from lightrag.constants import PARSED_DIR_NAME # noqa: E402
@pytest.mark.offline
def test_canonical_basename_recognizes_both_suffixes() -> None:
assert _file_path_for_parsed_artifact_dir("foo.pdf.parsed") == "foo.pdf"
assert _file_path_for_parsed_artifact_dir("foo.pdf.mineru_raw") == "foo.pdf"
# archive variants (parsed_001, mineru_raw_002, ...) handled
assert _file_path_for_parsed_artifact_dir("foo.pdf.parsed_001") == "foo.pdf"
assert _file_path_for_parsed_artifact_dir("foo.pdf.mineru_raw_002") == "foo.pdf"
# unrelated names don't match
assert _file_path_for_parsed_artifact_dir("foo.parsed.bak") is None
assert _file_path_for_parsed_artifact_dir("notes.txt") is None
@pytest.mark.offline
def test_delete_file_variants_removes_parsed_and_mineru_raw(
tmp_path: Path,
) -> None:
"""Both sidecar dir and raw bundle dir should disappear in one call."""
input_dir = tmp_path / "inputs"
input_dir.mkdir()
parsed_root = input_dir / PARSED_DIR_NAME
parsed_root.mkdir()
# Original source file (already moved into __parsed__/ post-archive).
archived_source = parsed_root / "demo.pdf"
archived_source.write_bytes(b"PDFCONTENT")
parsed_dir = parsed_root / "demo.pdf.parsed"
parsed_dir.mkdir()
(parsed_dir / "demo.blocks.jsonl").write_text("{}\n")
raw_dir = parsed_root / "demo.pdf.mineru_raw"
raw_dir.mkdir()
(raw_dir / "_manifest.json").write_text("{}")
(raw_dir / "content_list.json").write_text("[]")
(raw_dir / "images").mkdir()
(raw_dir / "images" / "img.png").write_bytes(b"png")
deleted, errors = delete_file_variants_by_file_path(input_dir, file_path="demo.pdf")
# Both directories and the archived source file were deleted.
assert errors == []
deleted_names = {Path(p).name for p in deleted}
assert "demo.pdf.parsed" in deleted_names
assert "demo.pdf.mineru_raw" in deleted_names
assert "demo.pdf" in deleted_names
assert not parsed_dir.exists()
assert not raw_dir.exists()
assert not archived_source.exists()
@pytest.mark.offline
def test_delete_file_variants_handles_only_raw_dir(tmp_path: Path) -> None:
"""The raw bundle may exist without a corresponding sidecar (parse
aborted between download and adapter). Delete should still pick it up.
"""
input_dir = tmp_path / "inputs"
input_dir.mkdir()
parsed_root = input_dir / PARSED_DIR_NAME
parsed_root.mkdir()
raw_dir = parsed_root / "demo.pdf.mineru_raw"
raw_dir.mkdir()
(raw_dir / "_manifest.json").write_text("{}")
deleted, errors = delete_file_variants_by_file_path(input_dir, file_path="demo.pdf")
assert errors == []
assert any("demo.pdf.mineru_raw" in p for p in deleted)
assert not raw_dir.exists()
@pytest.mark.offline
def test_delete_file_variants_leaves_unrelated_dirs(tmp_path: Path) -> None:
"""Directories that don't match the canonical filename are untouched."""
input_dir = tmp_path / "inputs"
input_dir.mkdir()
parsed_root = input_dir / PARSED_DIR_NAME
parsed_root.mkdir()
target_raw = parsed_root / "demo.pdf.mineru_raw"
target_raw.mkdir()
other_raw = parsed_root / "other.pdf.mineru_raw"
other_raw.mkdir()
other_parsed = parsed_root / "other.pdf.parsed"
other_parsed.mkdir()
deleted, errors = delete_file_variants_by_file_path(input_dir, file_path="demo.pdf")
assert errors == []
assert not target_raw.exists()
assert other_raw.exists(), "siblings for a different basename must survive"
assert other_parsed.exists()
+909
View File
@@ -0,0 +1,909 @@
"""MinerU IR builder tests: content_list.json → IR translation."""
from __future__ import annotations
import json
from pathlib import Path
import pytest
from lightrag.parser.external.mineru import MinerUIRBuilder
def _write_bundle(tmp_path: Path, content_list: list[dict]) -> Path:
"""Build a minimal *.mineru_raw/ directory."""
raw = tmp_path / "doc.mineru_raw"
raw.mkdir()
(raw / "content_list.json").write_text(json.dumps(content_list, ensure_ascii=False))
return raw
@pytest.mark.offline
def test_adapter_simple_text_and_heading(tmp_path: Path) -> None:
raw = _write_bundle(
tmp_path,
[
{"type": "text", "text": "1 Introduction", "text_level": 1},
{"type": "text", "text": "Body paragraph."},
{"type": "text", "text": "1.1 Sub", "text_level": 2},
{"type": "text", "text": "Sub body."},
],
)
ir = MinerUIRBuilder().normalize_from_workdir(raw, document_name="x.pdf")
assert ir.doc_title == "1 Introduction"
assert ir.document_format == "pdf"
# Heading + body merge into a single block per heading.
assert len(ir.blocks) == 2
assert ir.blocks[0].heading == "1 Introduction"
assert ir.blocks[0].level == 1
# Heading line is rendered with markdown ``#`` prefix matching the level.
assert ir.blocks[0].content_template == "# 1 Introduction\nBody paragraph."
# Sub-heading updates stack and records parent.
assert ir.blocks[1].heading == "1.1 Sub"
assert ir.blocks[1].level == 2
assert ir.blocks[1].parent_headings == ["1 Introduction"]
assert ir.blocks[1].content_template == "## 1.1 Sub\nSub body."
@pytest.mark.offline
def test_adapter_preface_block_for_pre_heading_content(tmp_path: Path) -> None:
"""Items emitted before the first heading land in a synthetic
``Preface/Uncategorized`` block at level 0."""
raw = _write_bundle(
tmp_path,
[
{"type": "text", "text": "Floating intro line."},
{"type": "list", "list_items": ["a", "b"]},
{"type": "text", "text": "Section A", "text_level": 1},
{"type": "text", "text": "A body."},
],
)
ir = MinerUIRBuilder().normalize_from_workdir(raw, document_name="p.pdf")
assert len(ir.blocks) == 2
preface = ir.blocks[0]
assert preface.heading == "Preface/Uncategorized"
assert preface.level == 0
assert preface.parent_headings == []
assert preface.content_template == "Floating intro line.\na\nb"
section = ir.blocks[1]
assert section.heading == "Section A"
assert section.level == 1
assert section.content_template == "# Section A\nA body."
@pytest.mark.offline
def test_adapter_merges_mixed_payloads_under_heading(tmp_path: Path) -> None:
"""Tables / images / equations / code under the same heading merge into
one block; their placeholders appear in document order."""
raw = _write_bundle(
tmp_path,
[
{"type": "text", "text": "Methods", "text_level": 1},
{"type": "text", "text": "We did stuff."},
{
"type": "table",
"table_body": [["a", "b"], ["1", "2"]],
"num_rows": 2,
"num_cols": 2,
},
{"type": "image", "img_path": "images/fig1.png"},
{"type": "equation", "text": "$$E = mc^2$$"},
{"type": "code", "code_body": "print('ok')"},
],
)
ir = MinerUIRBuilder().normalize_from_workdir(raw, document_name="m.pdf")
assert len(ir.blocks) == 1
block = ir.blocks[0]
assert block.heading == "Methods"
assert block.level == 1
assert len(block.tables) == 1
assert len(block.drawings) == 1
assert len(block.equations) == 1
# Lines are joined in source order; the heading carries its ``#`` prefix.
expected_lines = [
"# Methods",
"We did stuff.",
f"{{{{TBL:{block.tables[0].placeholder_key}}}}}",
f"{{{{IMG:{block.drawings[0].placeholder_key}}}}}",
f"{{{{EQ:{block.equations[0].placeholder_key}}}}}",
"print('ok')",
]
assert block.content_template == "\n".join(expected_lines)
@pytest.mark.offline
def test_adapter_table_and_drawing_and_equation(tmp_path: Path) -> None:
raw = _write_bundle(
tmp_path,
[
{
"type": "table",
"table_body": [["a", "b"], ["1", "2"]],
"num_rows": 2,
"num_cols": 2,
"table_caption": ["Tbl"],
"header": [["a", "b"]],
},
{
"type": "image",
"img_path": "images/img_001.jpg",
"image_caption": ["Fig 1"],
"page_idx": 1,
"bbox": [10, 20, 30, 40],
},
{"type": "equation", "text": "$E = mc^2$", "caption": "Eq 1"},
],
)
# The drawing references images/img_001.jpg — adapter accepts missing
# files and produces an AssetSpec with source=None.
ir = MinerUIRBuilder().normalize_from_workdir(raw, document_name="d.pdf")
table_block = next(b for b in ir.blocks if b.tables)
table = table_block.tables[0]
assert table.rows == [["a", "b"], ["1", "2"]]
assert table.num_rows == 2 and table.num_cols == 2
assert table.caption == "Tbl"
assert table.table_header == [["a", "b"]]
assert table.self_ref == "content_list.json#/0"
drawing_block = next(b for b in ir.blocks if b.drawings)
drawing = drawing_block.drawings[0]
assert drawing.fmt == "jpg"
assert drawing.caption == "Fig 1"
assert drawing.self_ref == "content_list.json#/1"
# Position carried through. The bbox-bearing item produces exactly one
# fine-grained position (anchor + range) and is NOT also rolled into the
# page-only summary channel — so the block has a single position entry,
# not a duplicate summary + bbox pair.
assert len(drawing_block.positions) == 1
assert drawing_block.positions[0].type == "bbox"
# Anchor is always serialized as a string (uniform on-disk format,
# accommodates book pagination labels like Roman "ii").
assert drawing_block.positions[0].anchor == "2" # page_idx+1
assert drawing_block.positions[0].range == [10.0, 20.0, 30.0, 40.0]
# Asset is declared with the relative path as ref.
assert any(a.ref == "images/img_001.jpg" for a in ir.assets)
equation_block = next(b for b in ir.blocks if b.equations)
eq = equation_block.equations[0]
# IREquation.latex preserves MinerU's raw form so blocks.jsonl shows it
# verbatim; equations.json strips the ``$`` wrappers downstream (writer).
assert eq.latex == "$E = mc^2$"
assert eq.is_block is True
assert eq.caption == "Eq 1"
assert eq.self_ref == "content_list.json#/2"
@pytest.mark.offline
def test_adapter_page_idx_aggregated_and_deduped_when_no_bbox(
tmp_path: Path,
) -> None:
"""Real MinerU output carries ``page_idx`` on every item but rarely a
``bbox``. Each unique page contributing to a merged block must surface as
one anchor-only ``{type:"bbox", anchor:<page+1>}`` entry, sorted, no
duplicates, no ``range``.
"""
raw = _write_bundle(
tmp_path,
[
{"type": "text", "text": "Section", "text_level": 1, "page_idx": 0},
{"type": "text", "text": "line A", "page_idx": 0},
{"type": "text", "text": "line B", "page_idx": 1},
{"type": "text", "text": "line C", "page_idx": 1},
{"type": "text", "text": "line D", "page_idx": 2},
],
)
ir = MinerUIRBuilder().normalize_from_workdir(raw, document_name="p.pdf")
assert len(ir.blocks) == 1
block = ir.blocks[0]
# Pages 0, 1, 2 → anchors "1", "2", "3" — one entry per unique page.
# Anchors are persisted as strings for on-disk uniformity.
assert len(block.positions) == 3
anchors = [p.anchor for p in block.positions]
assert anchors == ["1", "2", "3"]
for pos in block.positions:
assert pos.type == "bbox"
# Page-only summary entries have no range; ``to_jsonable`` must omit
# the key entirely.
assert pos.range is None
assert "range" not in pos.to_jsonable()
@pytest.mark.offline
def test_adapter_bbox_items_and_page_only_items_coexist(tmp_path: Path) -> None:
"""When a block merges both bbox-bearing and bbox-less items, the bbox
items are emitted per-item (no dedupe, with ``range``) and only the
bbox-less items contribute to the page-only summary. Ordering: summary
first (sorted by anchor), bbox entries after (source order).
"""
raw = _write_bundle(
tmp_path,
[
{"type": "text", "text": "Mixed", "text_level": 1, "page_idx": 1},
{
"type": "image",
"img_path": "images/fig.png",
"page_idx": 1,
"bbox": [10, 20, 30, 40],
},
{"type": "text", "text": "tail line", "page_idx": 2},
],
)
ir = MinerUIRBuilder().normalize_from_workdir(raw, document_name="m.pdf")
assert len(ir.blocks) == 1
positions = ir.blocks[0].positions
# One page-only summary for page 3 (the bbox-less tail line) and one
# bbox entry for page 2 (the image). The heading item has page_idx=1
# but no bbox, so it adds anchor 2 to the page set — combined with the
# tail item's anchor 3 the summary section has TWO anchors (1+1, 2+1).
assert [(p.anchor, p.range) for p in positions] == [
("2", None),
("3", None),
("2", [10.0, 20.0, 30.0, 40.0]),
]
@pytest.mark.offline
def test_adapter_page_sort_books_convention_with_mixed_anchors(
tmp_path: Path,
) -> None:
"""Block merges items with Roman preface labels and Arabic numerals.
Two guarantees:
1. The adapter must not crash when sorting heterogeneous anchors — a
previous bug surfaced ``TypeError: '<' not supported between
instances of 'str' and 'int'`` whenever ``page_idx`` mixed types.
2. Output order follows book pagination convention: Roman / letter
labels first (lexical), then numeric pages by integer value, so
``"2"`` precedes ``"10"`` (not ``"10"`` before ``"2"`` as a naive
lexical sort would do).
"""
raw = _write_bundle(
tmp_path,
[
{
"type": "text",
"text": "Mixed Pagination",
"text_level": 1,
"page_idx": "i",
},
{"type": "text", "text": "preface intro", "page_idx": "i"},
{"type": "text", "text": "preface tail", "page_idx": "ii"},
{"type": "text", "text": "chapter line A", "page_idx": 1}, # → "2"
{"type": "text", "text": "chapter line B", "page_idx": 9}, # → "10"
],
)
ir = MinerUIRBuilder().normalize_from_workdir(raw, document_name="mix.pdf")
assert len(ir.blocks) == 1
anchors = [p.anchor for p in ir.blocks[0].positions]
# Roman labels first (lex order), then numerics by int value.
assert anchors == ["i", "ii", "2", "10"]
@pytest.mark.offline
def test_adapter_empty_text_item_does_not_leak_page_to_block(
tmp_path: Path,
) -> None:
"""An item whose body is empty must NOT contribute its ``page_idx`` to
the current block's positions — otherwise spurious pages from
content-less items poison provenance.
Regression: empty text on page 99 sits between two real headings; its
page must not appear under either block.
"""
raw = _write_bundle(
tmp_path,
[
{"type": "text", "text": "Section A", "text_level": 1, "page_idx": 0},
{"type": "text", "text": "real body", "page_idx": 0},
# Empty body — should be silently dropped, page_idx not recorded.
{"type": "text", "text": "", "page_idx": 98},
{"type": "text", "text": "Section B", "text_level": 1, "page_idx": 1},
{"type": "text", "text": "next body", "page_idx": 1},
],
)
ir = MinerUIRBuilder().normalize_from_workdir(raw, document_name="leak.pdf")
assert len(ir.blocks) == 2
a_anchors = [p.anchor for p in ir.blocks[0].positions]
b_anchors = [p.anchor for p in ir.blocks[1].positions]
# Section A only mentions page 1 (page_idx 0 + 1) — NOT 99 from the
# dropped empty item.
assert a_anchors == ["1"]
assert "99" not in a_anchors and "99" not in b_anchors
# Section B only mentions page 2 (page_idx 1 + 1).
assert b_anchors == ["2"]
@pytest.mark.offline
def test_adapter_each_heading_starts_its_own_block(tmp_path: Path) -> None:
"""Every recognized heading starts its own block. Back-to-back headings
with no body between them are NOT folded — each becomes a standalone
block whose content is just the heading line; the heading that does have
a following body merges that body in. Mirrors the native docx parser.
"""
raw = _write_bundle(
tmp_path,
[
{"type": "text", "text": "1 Top", "text_level": 1},
{"type": "text", "text": "1.1 Mid", "text_level": 2},
{"type": "text", "text": "1.1.1 Deep", "text_level": 3},
{"type": "text", "text": "Body for deep."},
{"type": "text", "text": "2 Top Again", "text_level": 1},
{"type": "text", "text": "More body."},
],
)
ir = MinerUIRBuilder().normalize_from_workdir(raw, document_name="m.pdf")
summary = [
(b.heading, b.content_template, b.level, b.parent_headings) for b in ir.blocks
]
assert summary == [
("1 Top", "# 1 Top", 1, []),
("1.1 Mid", "## 1.1 Mid", 2, ["1 Top"]),
(
"1.1.1 Deep",
"### 1.1.1 Deep\nBody for deep.",
3,
["1 Top", "1.1 Mid"],
),
("2 Top Again", "# 2 Top Again\nMore body.", 1, []),
]
@pytest.mark.offline
def test_adapter_heading_markdown_prefix_cap_and_existing_marker(
tmp_path: Path,
) -> None:
"""Heading prefix is capped at six ``#`` and a heading whose text already
carries a markdown prefix is not double-prefixed in content but is stored
cleanly in metadata."""
raw = _write_bundle(
tmp_path,
[
# level 7 → clamped to six "#".
{"type": "text", "text": "Deep", "text_level": 7},
{"type": "text", "text": "deep body."},
# Source text already a markdown heading → kept verbatim.
{"type": "text", "text": "# Already MD", "text_level": 1},
{"type": "text", "text": "md body."},
{"type": "text", "text": "## Child MD", "text_level": 2},
{"type": "text", "text": "child body."},
],
)
ir = MinerUIRBuilder().normalize_from_workdir(raw, document_name="m.pdf")
first_lines = [b.content_template.split("\n")[0] for b in ir.blocks]
assert first_lines == ["###### Deep", "# Already MD", "## Child MD"]
assert [(b.heading, b.parent_headings) for b in ir.blocks] == [
("Deep", []),
("Already MD", []),
("Child MD", ["Already MD"]),
]
@pytest.mark.offline
def test_adapter_adjacent_shallower_heading_starts_new_block(
tmp_path: Path,
) -> None:
"""Inverse case: when the second adjacent heading is shallower (level
number smaller or equal), it must NOT merge — it starts a new block.
"""
raw = _write_bundle(
tmp_path,
[
{"type": "text", "text": "1.1 Mid first", "text_level": 2},
{"type": "text", "text": "2 Top after", "text_level": 1},
{"type": "text", "text": "body"},
],
)
ir = MinerUIRBuilder().normalize_from_workdir(raw, document_name="m.pdf")
# The first block is heading-only; the writer downstream will keep it
# (the merged-heading rule only forwards DEEPER headings).
assert len(ir.blocks) == 2
assert ir.blocks[0].heading == "1.1 Mid first"
assert ir.blocks[0].level == 2
assert ir.blocks[0].content_template == "## 1.1 Mid first"
assert ir.blocks[1].heading == "2 Top after"
assert ir.blocks[1].level == 1
assert ir.blocks[1].content_template == "# 2 Top after\nbody"
@pytest.mark.offline
def test_adapter_body_breaks_adjacent_heading_merge(tmp_path: Path) -> None:
"""Once any body content lands in the current block, the next heading —
even a deeper one — must flush and open a fresh block (no merge)."""
raw = _write_bundle(
tmp_path,
[
{"type": "text", "text": "1 Top", "text_level": 1},
{"type": "text", "text": "Intro line under 1."},
{"type": "text", "text": "1.1 Mid", "text_level": 2},
{"type": "text", "text": "Mid body."},
],
)
ir = MinerUIRBuilder().normalize_from_workdir(raw, document_name="m.pdf")
assert len(ir.blocks) == 2
assert ir.blocks[0].content_template == "# 1 Top\nIntro line under 1."
assert ir.blocks[1].heading == "1.1 Mid"
assert ir.blocks[1].parent_headings == ["1 Top"]
assert ir.blocks[1].content_template == "## 1.1 Mid\nMid body."
@pytest.mark.offline
def test_adapter_block_equation_preserves_dollar_wrappers(tmp_path: Path) -> None:
"""Block equations keep the ``$$`` markers verbatim on IREquation.latex
so the writer renders blocks.jsonl's ``<equation>`` body byte-identical
to MinerU's source. The downstream writer is responsible for stripping
them when generating equations.json."""
raw = _write_bundle(
tmp_path,
[
{
"type": "equation",
"text": "$$\n\\int_0^1 x dx = \\tfrac{1}{2}\n$$",
"text_format": "block",
"caption": "Eq A",
},
],
)
ir = MinerUIRBuilder().normalize_from_workdir(raw, document_name="b.pdf")
eq = ir.blocks[0].equations[0]
assert eq.is_block is True
# No stripping in the adapter; whitespace.strip() only.
assert eq.latex == "$$\n\\int_0^1 x dx = \\tfrac{1}{2}\n$$"
@pytest.mark.offline
def test_adapter_empty_equation_dropped(tmp_path: Path) -> None:
"""Fix 2: equation items with empty text MUST NOT enter the IR (and
consequently not the sidecar). They previously left dangling sidecar
entries."""
raw = _write_bundle(
tmp_path,
[
{"type": "equation", "text": "", "caption": "ghost"},
{"type": "equation", "text": " ", "caption": "ghost"},
{"type": "text", "text": "kept"},
],
)
ir = MinerUIRBuilder().normalize_from_workdir(raw, document_name="g.pdf")
eq_count = sum(len(b.equations) for b in ir.blocks)
assert eq_count == 0
assert any(b.content_template == "kept" for b in ir.blocks)
@pytest.mark.offline
def test_adapter_empty_table_dropped(tmp_path: Path) -> None:
"""Table items with no usable body MUST NOT enter the IR.
MinerU sometimes misidentifies a page-number / blank region as a table
and emits a body-less ``table`` item (missing ``table_body``/``rows``,
or with an empty string / empty grid). Leaving such items in the IR
would later trip the analyze worker's hard-failure path on empty
``content``. The IR builder filters them upstream.
"""
raw = _write_bundle(
tmp_path,
[
# 1) Body field completely absent.
{"type": "table", "num_rows": 0, "num_cols": 0},
# 2) Empty string body (matches the real m012-manual.pdf bug).
{"type": "table", "table_body": ""},
# 3) Empty list body.
{"type": "table", "rows": []},
# 4) Grid with only blank cells.
{"type": "table", "rows": [["", " "], ["\t", ""]]},
# 5) A real text item so the IR is not entirely empty.
{"type": "text", "text": "kept"},
],
)
ir = MinerUIRBuilder().normalize_from_workdir(raw, document_name="t.pdf")
table_count = sum(len(b.tables) for b in ir.blocks)
assert table_count == 0
# No table placeholder should leak into the rendered content either.
joined = "\n".join(b.content_template for b in ir.blocks)
assert "TBL:" not in joined
assert "kept" in joined
@pytest.mark.offline
def test_adapter_page_number_dropped(tmp_path: Path) -> None:
"""``page_number`` items are layout noise and MUST NOT enter the IR.
MinerU emits a ``page_number`` item per page. Empty-text page numbers were
already dropped by the fallback's _append_text guard, but page numbers that
carry real text (``"12"``, ``"iii"``) previously leaked into block content
and contaminated the block's positions with their page_idx. The IR builder
now skips them outright regardless of text.
"""
raw = _write_bundle(
tmp_path,
[
{"type": "text", "text": "kept", "page_idx": 0},
# Page number carrying real text — the regression case.
{"type": "page_number", "text": "12", "page_idx": 7},
# Roman-numeral page number.
{"type": "page_number", "text": "iii", "page_idx": 8},
# Blank/whitespace page number.
{"type": "page_number", "text": "- ", "page_idx": 9},
],
)
ir = MinerUIRBuilder().normalize_from_workdir(raw, document_name="p.pdf")
joined = "\n".join(b.content_template for b in ir.blocks)
assert "12" not in joined
assert "iii" not in joined
assert "kept" in joined
# The page_idx of skipped page numbers must not leak into block positions.
# Anchors are 1-based strings, so page_idx 7/8/9 would surface as 8/9/10.
anchors = {pos.anchor for b in ir.blocks for pos in b.positions}
assert anchors.isdisjoint({"8", "9", "10"})
@pytest.mark.offline
def test_adapter_bbox_attributes_default_and_override(tmp_path: Path) -> None:
raw = _write_bundle(tmp_path, [{"type": "text", "text": "x"}])
adapter = MinerUIRBuilder()
ir = adapter.normalize_from_workdir(raw, document_name="x.pdf")
assert ir.bbox_attributes == {"origin": "LEFTTOP", "max": 1000}
@pytest.mark.offline
def test_adapter_bbox_attributes_env_override(
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
) -> None:
monkeypatch.setenv(
"MINERU_BBOX_ATTRIBUTES",
'{"origin": "LEFTBOTTOM", "max": 612}',
)
raw = _write_bundle(tmp_path, [{"type": "text", "text": "x"}])
adapter = MinerUIRBuilder()
ir = adapter.normalize_from_workdir(raw, document_name="x.pdf")
assert ir.bbox_attributes == {"origin": "LEFTBOTTOM", "max": 612}
@pytest.mark.offline
def test_adapter_engine_version_recorded_in_split_option(
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
) -> None:
monkeypatch.setenv("MINERU_ENGINE_VERSION", "magic-pdf 1.5.4")
raw = _write_bundle(tmp_path, [{"type": "text", "text": "x"}])
ir = MinerUIRBuilder().normalize_from_workdir(raw, document_name="x.pdf")
assert ir.split_option == {"engine_version": "magic-pdf 1.5.4"}
@pytest.mark.offline
def test_adapter_missing_content_list_raises(tmp_path: Path) -> None:
raw_dir = tmp_path / "bad.mineru_raw"
raw_dir.mkdir()
with pytest.raises(FileNotFoundError):
MinerUIRBuilder().normalize_from_workdir(raw_dir, document_name="x.pdf")
@pytest.mark.offline
def test_adapter_html_table_preserves_merged_cells_and_extracts_thead(
tmp_path: Path,
) -> None:
"""MinerU HTML table_body must stay HTML so rowspan/colspan survive."""
table_html = (
'<table><thead><tr><th rowspan="2">Metric</th>'
'<th colspan="2">Group</th></tr><tr><th>A</th><th>B</th></tr></thead>'
"<tbody><tr><td>Accuracy</td><td>91%</td><td>93%</td></tr></tbody></table>"
)
raw = _write_bundle(
tmp_path,
[
{
"type": "table",
"table_body": table_html,
"table_caption": ["Tbl"],
}
],
)
ir = MinerUIRBuilder().normalize_from_workdir(raw, document_name="h.pdf")
table = ir.blocks[0].tables[0]
assert table.rows is None
assert table.html == table_html
assert 'rowspan="2"' in table.html
assert 'colspan="2"' in table.html
assert table.body_override == table_html.removeprefix("<table>").removesuffix(
"</table>"
)
assert table.num_rows == 3
assert table.num_cols == 3
assert table.caption == "Tbl"
# The <thead> is preserved verbatim (raw HTML) so rowspan/colspan survive —
# not flattened into a grid.
assert table.table_header == (
'<thead><tr><th rowspan="2">Metric</th>'
'<th colspan="2">Group</th></tr><tr><th>A</th><th>B</th></tr></thead>'
)
assert 'rowspan="2"' in table.table_header
assert 'colspan="2"' in table.table_header
@pytest.mark.offline
def test_adapter_html_table_header_preserves_spans(tmp_path: Path) -> None:
"""A spanned <thead> is stored verbatim — colspan/rowspan markup is kept
intact rather than flattened into a rectangular grid (so merged-cell
semantics survive into every repeated header chunk downstream)."""
thead = (
"<thead>"
'<tr><th>ID</th><th colspan="3">Scores</th></tr>'
'<tr><th>n</th><th>A</th><th colspan="2">BC</th></tr>'
"</thead>"
)
table_html = (
f"<table>{thead}<tbody>"
"<tr><td>1</td><td>x</td><td>y</td><td>z</td></tr>"
"</tbody></table>"
)
raw = _write_bundle(tmp_path, [{"type": "table", "table_body": table_html}])
ir = MinerUIRBuilder().normalize_from_workdir(raw, document_name="h.pdf")
table = ir.blocks[0].tables[0]
assert table.num_cols == 4
assert table.table_header == thead
assert table.table_header.count('colspan="3"') == 1
assert table.table_header.count('colspan="2"') == 1
@pytest.mark.offline
def test_adapter_html_table_without_thead_does_not_guess_header(
tmp_path: Path,
) -> None:
"""Only a real HTML <thead> should populate table_header."""
table_html = (
"<table><tbody><tr><td>H1</td><td>H2</td></tr>"
"<tr><td>a</td><td>b</td></tr></tbody></table>"
)
raw = _write_bundle(
tmp_path,
[
{
"type": "table",
"table_body": table_html,
"num_rows": 1,
"num_cols": 2,
}
],
)
ir = MinerUIRBuilder().normalize_from_workdir(raw, document_name="h.pdf")
table = ir.blocks[0].tables[0]
assert table.rows is None
assert table.html == table_html
assert table.table_header is None
@pytest.mark.offline
def test_adapter_html_table_without_thead_falls_back_to_header_grid(
tmp_path: Path,
) -> None:
"""An HTML table whose markup carries no <thead> but for which MinerU
supplied a separate ``header`` grid keeps that grid (the writer renders it to
a span-less <thead>) rather than silently dropping the recovered header."""
table_html = (
"<table><tbody><tr><td>a</td><td>b</td></tr></tbody></table>" # no <thead>
)
raw = _write_bundle(
tmp_path,
[
{
"type": "table",
"table_body": table_html,
"header": [["H1", "H2"]],
"num_rows": 1,
"num_cols": 2,
}
],
)
ir = MinerUIRBuilder().normalize_from_workdir(raw, document_name="h.pdf")
table = ir.blocks[0].tables[0]
assert table.html == table_html
assert table.table_header == [["H1", "H2"]] # grid kept, not dropped
@pytest.mark.offline
def test_adapter_html_table_attr_with_gt_keeps_inner_body(tmp_path: Path) -> None:
"""A ``>`` inside a ``<table>`` attribute must not truncate the open tag,
so ``body_override`` still strips the full ``<table …>`` wrapper."""
table_html = '<table data-x="a>b"><tbody><tr><td>v</td></tr></tbody></table>'
raw = _write_bundle(
tmp_path,
[{"type": "table", "table_body": table_html}],
)
ir = MinerUIRBuilder().normalize_from_workdir(raw, document_name="h.pdf")
table = ir.blocks[0].tables[0]
assert table.html == table_html
assert table.body_override == "<tbody><tr><td>v</td></tr></tbody>"
@pytest.mark.offline
def test_adapter_empty_html_table_falls_back_to_full_html(tmp_path: Path) -> None:
"""A degenerate ``<table></table>`` has no inner body, so ``body_override``
stays None and the writer renders ``table.html`` verbatim."""
raw = _write_bundle(
tmp_path,
[{"type": "table", "table_body": "<table></table>"}],
)
ir = MinerUIRBuilder().normalize_from_workdir(raw, document_name="h.pdf")
table = ir.blocks[0].tables[0]
assert table.html == "<table></table>"
assert table.body_override is None
@pytest.mark.offline
def test_adapter_html_table_strips_html_body_wrapper(tmp_path: Path) -> None:
"""MinerU's table model may wrap output in ``<html><body>``; the adapter
must unwrap to a single ``<table>`` so the writer does not nest tables and
``TABLE_TAG_RE`` consumers are not truncated at the inner ``</table>``."""
inner = (
'<thead><tr><th colspan="2">G</th></tr></thead>'
"<tbody><tr><td>a</td><td>b</td></tr></tbody>"
)
payload = f"<html><body><table>{inner}</table></body></html>"
raw = _write_bundle(tmp_path, [{"type": "table", "table_body": payload}])
ir = MinerUIRBuilder().normalize_from_workdir(raw, document_name="h.pdf")
table = ir.blocks[0].tables[0]
assert table.html == f"<table>{inner}</table>" # <html>/<body> wrapper gone
assert table.body_override == inner # block text renders only the inner body
assert table.num_cols == 2
# The <thead> is kept verbatim (colspan preserved), not flattened to a grid.
assert table.table_header == '<thead><tr><th colspan="2">G</th></tr></thead>'
@pytest.mark.offline
def test_adapter_list_items_joined_with_newline(tmp_path: Path) -> None:
raw = _write_bundle(
tmp_path,
[
{"type": "list", "list_items": ["one", "two", "three"]},
],
)
ir = MinerUIRBuilder().normalize_from_workdir(raw, document_name="l.pdf")
assert ir.blocks[0].content_template == "one\ntwo\nthree"
@pytest.mark.offline
def test_adapter_drawing_asset_source_only_when_file_exists(
tmp_path: Path,
) -> None:
"""The adapter should declare an AssetSpec for the drawing in both
cases, but ``source`` is set only when the bytes are on disk; the
writer then warns and skips a missing-source asset."""
raw = _write_bundle(
tmp_path,
[
{"type": "image", "img_path": "images/exists.png"},
{"type": "image", "img_path": "images/missing.png"},
],
)
(raw / "images").mkdir()
(raw / "images" / "exists.png").write_bytes(b"\x89PNG")
ir = MinerUIRBuilder().normalize_from_workdir(raw, document_name="a.pdf")
by_ref = {a.ref: a for a in ir.assets}
assert by_ref["images/exists.png"].source is not None
assert by_ref["images/missing.png"].source is None
@pytest.mark.offline
def test_adapter_refuses_path_traversal_img_path(tmp_path: Path) -> None:
"""Untrusted img_path with ``..`` or absolute filesystem segments must
not be allowed to point ``AssetSpec.source`` outside ``raw_dir``.
Otherwise the writer would copy attacker-named files from the host into
the sidecar's ``*.blocks.assets/`` directory (file-disclosure path).
"""
# Place a "secret" file outside the raw bundle that should never be
# selectable as an asset source.
secret = tmp_path / "secret.txt"
secret.write_bytes(b"private")
raw = _write_bundle(
tmp_path,
[
{"type": "image", "img_path": "../secret.txt"},
{"type": "image", "img_path": str(secret)}, # absolute path
],
)
ir = MinerUIRBuilder().normalize_from_workdir(raw, document_name="x.pdf")
by_ref = {a.ref: a for a in ir.assets}
# Relative ``..`` escape is rejected outright.
assert by_ref["../secret.txt"].source is None
# Absolute filesystem path is reinterpreted as ``images/<basename>``
# inside raw_dir. Since no such file exists, source must remain None
# (and crucially must not point at the original secret file).
abs_asset = by_ref[str(secret)]
assert abs_asset.source is None
@pytest.mark.offline
def test_adapter_absolute_url_img_path_resolves_to_images_basename(
tmp_path: Path,
) -> None:
"""When MinerU emits an absolute URL in img_path, the downloader saves
it as ``images/<basename>``; the adapter must look there too."""
raw = _write_bundle(
tmp_path,
[
{
"type": "image",
"img_path": "https://cdn.example.com/imgs/figure_42.png",
},
],
)
(raw / "images").mkdir()
(raw / "images" / "figure_42.png").write_bytes(b"\x89PNGfake")
ir = MinerUIRBuilder().normalize_from_workdir(raw, document_name="u.pdf")
asset = ir.assets[0]
assert asset.ref == "https://cdn.example.com/imgs/figure_42.png"
assert asset.suggested_name == "figure_42.png"
assert asset.source is not None
assert asset.source.read_bytes() == b"\x89PNGfake"
@pytest.mark.offline
def test_adapter_image_url_template_mode_maps_relative_to_images_basename(
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
) -> None:
"""When MINERU_IMAGE_URL_TEMPLATE is set, MinerURawClient stores every
image reference — including relative ones — at ``images/<basename>``.
The adapter must mirror that lookup so the asset is wired up, otherwise
the downloaded bytes are silently dropped from the sidecar."""
monkeypatch.setenv(
"MINERU_IMAGE_URL_TEMPLATE",
"http://mineru.internal/assets/{name}",
)
raw = _write_bundle(
tmp_path,
[
{"type": "image", "img_path": "page/img.png"},
],
)
# Downloader's actual landing spot in template mode.
(raw / "images").mkdir()
(raw / "images" / "img.png").write_bytes(b"\x89PNGtemplate")
# The "naive" location (raw_dir/page/img.png) does NOT exist; in
# template mode the downloader does not write there.
assert not (raw / "page" / "img.png").exists()
ir = MinerUIRBuilder().normalize_from_workdir(raw, document_name="t.pdf")
asset = ir.assets[0]
assert asset.source is not None
assert asset.source.read_bytes() == b"\x89PNGtemplate"
@pytest.mark.offline
def test_adapter_no_template_keeps_relative_path_lookup(
tmp_path: Path,
) -> None:
"""Sanity: without MINERU_IMAGE_URL_TEMPLATE, a relative img_path still
resolves under raw_dir at its original location (regression guard for
the template-mode change above)."""
raw = _write_bundle(
tmp_path,
[
{"type": "image", "img_path": "page/img.png"},
],
)
(raw / "page").mkdir()
(raw / "page" / "img.png").write_bytes(b"\x89PNGrel")
ir = MinerUIRBuilder().normalize_from_workdir(raw, document_name="r.pdf")
asset = ir.assets[0]
assert asset.source is not None
assert asset.source.read_bytes() == b"\x89PNGrel"
@@ -0,0 +1,553 @@
"""Integration tests for ``parse_mineru`` with the unified sidecar pipeline.
These tests stub :class:`MinerURawClient.download_into` so no real MinerU
service is contacted; the focus is on:
- happy path: cache miss → download → sidecar emitted with all expected
files in the spec-compliant locations
- cache hit: a pre-existing valid ``*.mineru_raw/`` + manifest causes
``MinerURawClient.download_into`` NOT to be called
- ``LIGHTRAG_FORCE_REPARSE_MINERU=true`` forces a re-download even when
the manifest is valid
"""
from __future__ import annotations
import asyncio
import json
from pathlib import Path
from typing import Any
import numpy as np
import pytest
from lightrag import LightRAG
from lightrag.constants import (
FULL_DOCS_FORMAT_LIGHTRAG,
)
from lightrag.parser.external.mineru import compute_size_and_hash
from lightrag.parser.external.mineru.cache import current_mineru_options_signature
from lightrag.parser.external.mineru.manifest import (
Manifest,
ManifestFile,
write_manifest,
)
from lightrag.parser.base import ParseContext
from lightrag.parser.registry import get_parser
from lightrag.utils import EmbeddingFunc, Tokenizer
async def _parse_via_registry(rag, engine, doc_id, file_path, content_data):
"""Drive a parser the way the pipeline worker does (registry dispatch)."""
result = await get_parser(engine).parse(
ParseContext(rag, doc_id, file_path, content_data)
)
return result.to_dict()
class _SimpleTokenizerImpl:
def encode(self, content: str) -> list[int]:
return [ord(ch) for ch in content]
def decode(self, tokens: list[int]) -> str:
return "".join(chr(t) for t in tokens)
async def _mock_embedding(texts: list[str]) -> np.ndarray:
return np.random.rand(len(texts), 32)
async def _mock_llm(prompt: Any, **kwargs: Any) -> str:
return '{"name":"x","summary":"s","detail_description":"d"}'
def _new_rag(tmp_path: Path) -> LightRAG:
return LightRAG(
working_dir=str(tmp_path),
workspace=f"test-mineru-sidecar-{tmp_path.name}",
llm_model_func=_mock_llm,
embedding_func=EmbeddingFunc(
embedding_dim=32,
max_token_size=4096,
func=_mock_embedding,
),
tokenizer=Tokenizer("mock-tokenizer", _SimpleTokenizerImpl()),
vlm_process_enable=False,
)
_FAKE_TABLE_HTML = (
'<table><thead><tr><th rowspan="2">Metric</th>'
'<th colspan="2">Score</th></tr><tr><th>A</th><th>B</th></tr></thead>'
"<tbody><tr><td>Accuracy</td><td>91%</td><td>93%</td></tr></tbody></table>"
)
_FAKE_CONTENT_LIST = [
{"type": "text", "text": "1 Introduction", "text_level": 1},
{"type": "text", "text": "Body paragraph."},
{
"type": "table",
"table_body": _FAKE_TABLE_HTML,
"table_caption": ["Tbl"],
"page_idx": 0,
"bbox": [10, 10, 100, 50],
},
{
"type": "image",
"img_path": "images/img_001.jpg",
"image_caption": ["Fig 1"],
"page_idx": 1,
"bbox": [20, 20, 200, 100],
},
{"type": "equation", "text": "$E = mc^2$", "caption": "Eq 1", "page_idx": 1},
]
def _install_fake_download(monkeypatch: pytest.MonkeyPatch) -> dict[str, int]:
"""Replace :meth:`MinerURawClient.download_into` with a recorder that
writes a synthetic bundle (content_list.json + one image + manifest).
"""
import lightrag.parser.external.mineru.client as client_mod
counters = {"calls": 0, "upload_names": []}
async def _fake_download(
self,
raw_dir: Path,
source_file_path: Path,
*,
upload_name: str | None = None,
):
counters["calls"] += 1
counters["upload_names"].append(upload_name)
raw_dir.mkdir(parents=True, exist_ok=True)
(raw_dir / "content_list.json").write_text(
json.dumps(_FAKE_CONTENT_LIST, ensure_ascii=False),
encoding="utf-8",
)
(raw_dir / "images").mkdir(exist_ok=True)
(raw_dir / "images" / "img_001.jpg").write_bytes(b"\xff\xd8\xff\xe0fakeJPEG")
src_size, src_hash = compute_size_and_hash(source_file_path)
crit_size, crit_hash = compute_size_and_hash(raw_dir / "content_list.json")
files = [
ManifestFile(
path="images/img_001.jpg",
size=(raw_dir / "images" / "img_001.jpg").stat().st_size,
)
]
manifest = Manifest(
source_content_hash=src_hash,
source_size_bytes=src_size,
source_filename_at_parse=upload_name or source_file_path.name,
critical_file=ManifestFile(
path="content_list.json", size=crit_size, sha256=crit_hash
),
files=files,
total_size_bytes=crit_size + sum(f.size for f in files),
task_id=f"fake-{counters['calls']}",
api_mode="local",
options_signature=current_mineru_options_signature(),
)
write_manifest(raw_dir, manifest)
return manifest
monkeypatch.setattr(client_mod.MinerURawClient, "download_into", _fake_download)
return counters
@pytest.mark.offline
def test_parse_mineru_emits_compliant_sidecar(
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
) -> None:
"""End-to-end: parse_mineru produces *.parsed/ with spec-compliant
blocks.jsonl + per-modality JSONs + assets dir; *.mineru_raw/ kept."""
async def _run() -> None:
monkeypatch.setenv("MINERU_LOCAL_ENDPOINT", "http://mineru.example")
counters = _install_fake_download(monkeypatch)
# Don't move the source out from under the cache validator between
# repeated parse_mineru calls.
async def _noop_archive(_p: str) -> None:
return None
import lightrag.pipeline as pipeline_module
monkeypatch.setattr(
pipeline_module,
"archive_docx_source_after_full_docs_sync",
_noop_archive,
)
input_dir = tmp_path / "inputs" / "ws"
input_dir.mkdir(parents=True)
src = input_dir / "demo.pdf"
src.write_bytes(b"PDFPDF" * 256)
rag = _new_rag(tmp_path)
await rag.initialize_storages()
try:
doc_id = "doc-abcdef0123456789abcdef0123456789"
await rag.doc_status.upsert(
{
doc_id: {
"status": "PARSING",
"content_summary": "",
"content_length": 0,
"chunks_count": 0,
"chunks_list": [],
"created_at": "2026-05-15T00:00:00+00:00",
"updated_at": "2026-05-15T00:00:00+00:00",
"file_path": "demo.pdf",
"track_id": "trk",
"content_hash": "",
"metadata": {},
}
}
)
monkeypatch.setattr(
rag,
"_resolve_source_file_for_parser",
lambda _p: str(src),
)
parsed = await _parse_via_registry(
rag,
"mineru",
doc_id=doc_id,
file_path="demo.pdf",
content_data={},
)
assert counters["calls"] == 1, "download_into should run once on miss"
parsed_dir = Path(parsed["blocks_path"]).parent
assert parsed["parse_format"] == FULL_DOCS_FORMAT_LIGHTRAG
assert parsed_dir.name == "demo.pdf.parsed"
# Sidecar files present
files = {p.name for p in parsed_dir.iterdir() if p.is_file()}
assert "demo.blocks.jsonl" in files
assert "demo.tables.json" in files
assert "demo.drawings.json" in files
assert "demo.equations.json" in files
assert (parsed_dir / "demo.blocks.assets").is_dir()
assert (parsed_dir / "demo.blocks.assets" / "img_001.jpg").is_file()
# Content of blocks.jsonl
blocks_raw = (parsed_dir / "demo.blocks.jsonl").read_text()
lines = blocks_raw.splitlines()
meta = json.loads(lines[0])
rows = [json.loads(line) for line in lines[1:]]
assert meta["parse_engine"] == "mineru"
assert meta["table_file"] is True
assert meta["drawing_file"] is True
assert meta["equation_file"] is True
assert meta["asset_dir"] is True
assert meta["doc_title"] == "1 Introduction"
# bbox_attributes present for mineru (PDF coordinate context)
assert meta["bbox_attributes"] == {"origin": "LEFTTOP", "max": 1000}
# Spec fix: <table> placeholder inline, not <cite>
contents = " ".join(row.get("content", "") for row in rows)
assert '<table id="tb-' in contents
assert 'format="html"' in contents
assert 'format="json"' not in contents
assert 'rowspan="2"' in contents
assert 'colspan="2"' in contents
assert "<cite" not in contents
# bbox positions present on at least one block
assert any(
p.get("type") == "bbox"
for row in rows
for p in row.get("positions") or []
)
# Drawing path points inside *.blocks.assets/
drawings = json.loads((parsed_dir / "demo.drawings.json").read_text())[
"drawings"
]
(drawing_id, drawing_item) = next(iter(drawings.items()))
assert drawing_id.startswith("im-")
assert drawing_item["path"] == "demo.blocks.assets/img_001.jpg"
assert drawing_item["self_ref"] == "content_list.json#/3"
# Raw bundle preserved next to sidecar
raw_dir = parsed_dir.parent / "demo.pdf.mineru_raw"
assert (raw_dir / "_manifest.json").is_file()
assert (raw_dir / "content_list.json").is_file()
assert (raw_dir / "images" / "img_001.jpg").is_file()
# No legacy non-spec image field on tables
tables = json.loads((parsed_dir / "demo.tables.json").read_text())["tables"]
(_, table_item) = next(iter(tables.items()))
assert "image" not in table_item
assert table_item["format"] == "html"
assert table_item["content"] == _FAKE_TABLE_HTML
assert table_item["dimension"] == [3, 3]
assert 'rowspan="2"' in table_item["content"]
assert 'colspan="2"' in table_item["content"]
# HTML tables store table_header as the raw <thead> (rowspan/colspan
# preserved), not a flattened JSON grid.
assert table_item["table_header"] == (
'<thead><tr><th rowspan="2">Metric</th>'
'<th colspan="2">Score</th></tr>'
"<tr><th>A</th><th>B</th></tr></thead>"
)
assert 'rowspan="2"' in table_item["table_header"]
assert 'colspan="2"' in table_item["table_header"]
assert table_item["self_ref"] == "content_list.json#/2"
equations = json.loads((parsed_dir / "demo.equations.json").read_text())[
"equations"
]
(_, equation_item) = next(iter(equations.items()))
assert equation_item["self_ref"] == "content_list.json#/4"
finally:
await rag.finalize_storages()
asyncio.new_event_loop().run_until_complete(_run())
@pytest.mark.offline
def test_parse_mineru_cache_hit_skips_download(
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
) -> None:
"""A pre-existing valid bundle short-circuits the network call entirely."""
async def _run() -> None:
monkeypatch.setenv("MINERU_LOCAL_ENDPOINT", "http://mineru.example")
counters = _install_fake_download(monkeypatch)
# Don't move the source out from under the cache validator between
# repeated parse_mineru calls.
async def _noop_archive(_p: str) -> None:
return None
import lightrag.pipeline as pipeline_module
monkeypatch.setattr(
pipeline_module,
"archive_docx_source_after_full_docs_sync",
_noop_archive,
)
input_dir = tmp_path / "inputs" / "ws"
input_dir.mkdir(parents=True)
src = input_dir / "demo.pdf"
src.write_bytes(b"PDFPDF" * 256)
rag = _new_rag(tmp_path)
await rag.initialize_storages()
try:
doc_id = "doc-abcdef0123456789abcdef0123456789"
await rag.doc_status.upsert(
{
doc_id: {
"status": "PARSING",
"content_summary": "",
"content_length": 0,
"chunks_count": 0,
"chunks_list": [],
"created_at": "2026-05-15T00:00:00+00:00",
"updated_at": "2026-05-15T00:00:00+00:00",
"file_path": "demo.pdf",
"track_id": "trk",
"content_hash": "",
"metadata": {},
}
}
)
monkeypatch.setattr(
rag,
"_resolve_source_file_for_parser",
lambda _p: str(src),
)
# First call: cache miss → download once.
await _parse_via_registry(
rag,
"mineru",
doc_id=doc_id,
file_path="demo.pdf",
content_data={},
)
assert counters["calls"] == 1
# Second call: should hit cache.
await _parse_via_registry(
rag,
"mineru",
doc_id=doc_id,
file_path="demo.pdf",
content_data={},
)
assert counters["calls"] == 1, "cache hit must not re-download"
# Third call with force-reparse: cache invalidated.
monkeypatch.setenv("LIGHTRAG_FORCE_REPARSE_MINERU", "true")
await _parse_via_registry(
rag,
"mineru",
doc_id=doc_id,
file_path="demo.pdf",
content_data={},
)
assert counters["calls"] == 2
finally:
await rag.finalize_storages()
asyncio.new_event_loop().run_until_complete(_run())
@pytest.mark.offline
def test_parse_mineru_upload_name_strips_parser_hint(
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
) -> None:
"""MinerU upload name should use the canonical filename, not parser
hints embedded in the source basename."""
async def _run() -> None:
monkeypatch.setenv("MINERU_LOCAL_ENDPOINT", "http://mineru.example")
counters = _install_fake_download(monkeypatch)
input_dir = tmp_path / "inputs" / "ws"
input_dir.mkdir(parents=True)
src = input_dir / "demo.[mineru-iet].pdf"
src.write_bytes(b"PDFPDF" * 256)
rag = _new_rag(tmp_path)
await rag.initialize_storages()
try:
doc_id = "doc-abcdef0123456789abcdef0123456789"
await rag.doc_status.upsert(
{
doc_id: {
"status": "PARSING",
"content_summary": "",
"content_length": 0,
"chunks_count": 0,
"chunks_list": [],
"created_at": "2026-05-15T00:00:00+00:00",
"updated_at": "2026-05-15T00:00:00+00:00",
"file_path": src.name,
"track_id": "trk",
"content_hash": "",
"metadata": {},
}
}
)
monkeypatch.setattr(
rag,
"_resolve_source_file_for_parser",
lambda _p: str(src),
)
parsed = await _parse_via_registry(
rag,
"mineru",
doc_id=doc_id,
file_path=src.name,
content_data={},
)
assert counters["upload_names"] == ["demo.pdf"]
parsed_dir = Path(parsed["blocks_path"]).parent
assert parsed_dir.name == "demo.pdf.parsed"
manifest = json.loads(
(
parsed_dir.parent / "demo.pdf.mineru_raw" / "_manifest.json"
).read_text(encoding="utf-8")
)
assert manifest["source_filename_at_parse"] == "demo.pdf"
finally:
await rag.finalize_storages()
asyncio.new_event_loop().run_until_complete(_run())
@pytest.mark.offline
def test_parse_mineru_cache_invalidates_on_source_change(
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
) -> None:
"""Source file content swapped (same/different size) → cache miss."""
async def _run() -> None:
monkeypatch.setenv("MINERU_LOCAL_ENDPOINT", "http://mineru.example")
counters = _install_fake_download(monkeypatch)
# Don't move the source out from under the cache validator between
# repeated parse_mineru calls.
async def _noop_archive(_p: str) -> None:
return None
import lightrag.pipeline as pipeline_module
monkeypatch.setattr(
pipeline_module,
"archive_docx_source_after_full_docs_sync",
_noop_archive,
)
input_dir = tmp_path / "inputs" / "ws"
input_dir.mkdir(parents=True)
src = input_dir / "demo.pdf"
src.write_bytes(b"PDFPDF" * 256)
rag = _new_rag(tmp_path)
await rag.initialize_storages()
try:
doc_id = "doc-abcdef0123456789abcdef0123456789"
await rag.doc_status.upsert(
{
doc_id: {
"status": "PARSING",
"content_summary": "",
"content_length": 0,
"chunks_count": 0,
"chunks_list": [],
"created_at": "2026-05-15T00:00:00+00:00",
"updated_at": "2026-05-15T00:00:00+00:00",
"file_path": "demo.pdf",
"track_id": "trk",
"content_hash": "",
"metadata": {},
}
}
)
monkeypatch.setattr(
rag,
"_resolve_source_file_for_parser",
lambda _p: str(src),
)
await _parse_via_registry(
rag,
"mineru",
doc_id=doc_id,
file_path="demo.pdf",
content_data={},
)
assert counters["calls"] == 1
# Same length, different bytes → fast-path passes, hash fails.
data = src.read_bytes()
src.write_bytes(b"\x00" + data[1:])
await _parse_via_registry(
rag,
"mineru",
doc_id=doc_id,
file_path="demo.pdf",
content_data={},
)
assert counters["calls"] == 2
finally:
await rag.finalize_storages()
asyncio.new_event_loop().run_until_complete(_run())
+235
View File
@@ -0,0 +1,235 @@
"""Tests for shared helpers in ``lightrag/parser/external/``.
These cover the pure functions reused across engine integrations:
- ``compute_size_and_hash`` — single-read (size, hash) pair
- ``clear_dir_contents`` — empty a directory while keeping it
- ``raw_dir_for_parsed_dir`` — suffix-bound raw dir naming
- ``safe_extract_zip`` — refuses path traversal and absolute paths
- ``env_bool`` / ``env_int`` / ``env_json`` — env parsing
- ``Manifest`` round-trip via ``write_manifest`` / ``load_manifest``
"""
from __future__ import annotations
import io
import json
import zipfile
from pathlib import Path
import pytest
from lightrag.parser.external import (
Manifest,
ManifestFile,
clear_dir_contents,
compute_size_and_hash,
env_bool,
env_int,
env_json,
load_manifest,
raw_dir_for_parsed_dir,
safe_extract_zip,
write_manifest,
)
# ---------------------------------------------------------------------------
# compute_size_and_hash
# ---------------------------------------------------------------------------
def test_compute_size_and_hash_stable(tmp_path: Path) -> None:
p = tmp_path / "f.bin"
payload = b"hello-external-parser" * 1024
p.write_bytes(payload)
size_a, hash_a = compute_size_and_hash(p)
size_b, hash_b = compute_size_and_hash(p)
assert size_a == len(payload) == size_b
assert hash_a == hash_b
assert hash_a.startswith("sha256:") and len(hash_a) == len("sha256:") + 64
# ---------------------------------------------------------------------------
# clear_dir_contents
# ---------------------------------------------------------------------------
def test_clear_dir_contents_keeps_dir_and_removes_children(tmp_path: Path) -> None:
d = tmp_path / "raw"
d.mkdir()
(d / "a.txt").write_text("hi")
sub = d / "nested"
sub.mkdir()
(sub / "b.bin").write_bytes(b"x" * 10)
clear_dir_contents(d)
assert d.is_dir()
assert list(d.iterdir()) == []
def test_clear_dir_contents_noop_when_missing(tmp_path: Path) -> None:
clear_dir_contents(tmp_path / "does-not-exist")
# ---------------------------------------------------------------------------
# raw_dir_for_parsed_dir
# ---------------------------------------------------------------------------
def test_raw_dir_for_parsed_dir_with_suffix(tmp_path: Path) -> None:
parsed = tmp_path / "demo.pdf.parsed"
raw = raw_dir_for_parsed_dir(parsed, suffix=".docling_raw")
assert raw == tmp_path / "demo.pdf.docling_raw"
def test_raw_dir_for_parsed_dir_without_parsed_suffix(tmp_path: Path) -> None:
parsed = tmp_path / "other_dir"
raw = raw_dir_for_parsed_dir(parsed, suffix=".docling_raw")
assert raw == tmp_path / "other_dir.docling_raw"
def test_raw_dir_for_parsed_dir_rejects_bad_suffix(tmp_path: Path) -> None:
with pytest.raises(ValueError):
raw_dir_for_parsed_dir(tmp_path / "x.parsed", suffix="docling_raw")
# ---------------------------------------------------------------------------
# safe_extract_zip
# ---------------------------------------------------------------------------
def _make_zip(entries: dict[str, bytes]) -> bytes:
buf = io.BytesIO()
with zipfile.ZipFile(buf, "w") as zf:
for name, payload in entries.items():
zf.writestr(name, payload)
return buf.getvalue()
def test_safe_extract_zip_extracts_flat_bundle(tmp_path: Path) -> None:
payload = _make_zip(
{
"demo.json": b'{"schema_name": "DoclingDocument"}',
"demo.md": b"# demo",
"artifacts/image_000000.png": b"\x89PNG fake",
}
)
dest = tmp_path / "raw"
names = safe_extract_zip(payload, dest)
assert (dest / "demo.json").read_bytes().startswith(b'{"schema_name"')
assert (dest / "demo.md").read_text(encoding="utf-8") == "# demo"
assert (dest / "artifacts" / "image_000000.png").is_file()
assert sorted(names) == sorted(
["demo.json", "demo.md", "artifacts/image_000000.png"]
)
def test_safe_extract_zip_rejects_path_traversal(tmp_path: Path) -> None:
payload = _make_zip({"../evil.txt": b"oops"})
with pytest.raises(RuntimeError, match="unsafe path"):
safe_extract_zip(payload, tmp_path / "raw")
def test_safe_extract_zip_rejects_absolute_path(tmp_path: Path) -> None:
payload = _make_zip({"/etc/passwd": b"oops"})
with pytest.raises(RuntimeError, match="unsafe path"):
safe_extract_zip(payload, tmp_path / "raw")
# ---------------------------------------------------------------------------
# env coercion
# ---------------------------------------------------------------------------
def test_env_bool_truthy_falsy(monkeypatch: pytest.MonkeyPatch) -> None:
for raw in ("1", "true", "yes", "ON"):
monkeypatch.setenv("X", raw)
assert env_bool("X", False) is True
for raw in ("0", "false", "no", "off"):
monkeypatch.setenv("X", raw)
assert env_bool("X", True) is False
def test_env_bool_falls_back_on_unrecognized(monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.setenv("X", "maybe")
assert env_bool("X", True) is True
assert env_bool("X", False) is False
def test_env_int_falls_back_on_garbage(monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.setenv("X", "not-an-int")
assert env_int("X", 7) == 7
def test_env_json_returns_default_on_garbage(monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.setenv("X", "{bad json")
assert env_json("X", {"origin": "LEFTBOTTOM"}) == {"origin": "LEFTBOTTOM"}
def test_env_json_parses_object(monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.setenv("X", '{"a": 1, "b": [2, 3]}')
assert env_json("X", None) == {"a": 1, "b": [2, 3]}
# ---------------------------------------------------------------------------
# Manifest round-trip
# ---------------------------------------------------------------------------
def test_manifest_round_trip(tmp_path: Path) -> None:
raw = tmp_path / "demo.docling_raw"
raw.mkdir()
crit = ManifestFile(path="demo.json", size=42, sha256="sha256:" + "a" * 64)
other = ManifestFile(path="demo.md", size=10)
manifest = Manifest(
engine="docling",
source_content_hash="sha256:" + "b" * 64,
source_size_bytes=100,
source_filename_at_parse="demo.pdf",
critical_file=crit,
files=[other],
total_size_bytes=52,
task_id="task-xyz",
endpoint_signature="http://l4ai:5001",
engine_version="1.18.0",
options_signature="sha256:" + "c" * 64,
downloaded_at="2026-05-18T00:00:00Z",
extras={"to_formats": ["json", "md"]},
)
write_manifest(raw, manifest)
payload = json.loads((raw / "_manifest.json").read_text(encoding="utf-8"))
assert payload["engine"] == "docling"
assert payload["options_signature"] == "sha256:" + "c" * 64
assert payload["extras"] == {"to_formats": ["json", "md"]}
loaded = load_manifest(raw, expected_engine="docling")
assert loaded is not None
assert loaded.task_id == "task-xyz"
assert loaded.critical_file.size == 42
assert loaded.files[0].path == "demo.md"
def test_manifest_load_rejects_wrong_engine(tmp_path: Path) -> None:
raw = tmp_path / "demo.docling_raw"
raw.mkdir()
manifest = Manifest(
engine="mineru",
source_content_hash="sha256:" + "0" * 64,
source_size_bytes=1,
source_filename_at_parse="x",
critical_file=ManifestFile(path="c", size=1, sha256="sha256:" + "1" * 64),
files=[],
total_size_bytes=1,
)
write_manifest(raw, manifest)
assert load_manifest(raw, expected_engine="docling") is None
def test_manifest_load_handles_missing_file(tmp_path: Path) -> None:
assert load_manifest(tmp_path / "no-such-dir", expected_engine="docling") is None
+104
View File
@@ -0,0 +1,104 @@
"""Proof that engine params thread identically into both external-parser hooks.
The ONLY test exercising ``ExternalParserBase.parse``'s decode + thread: the
per-file engine params decoded from ``content_data['parse_engine']`` must reach
BOTH ``is_bundle_valid`` (cache-hit check) and ``download_into`` (request), or a
cache signature could be computed with different params than the bundle was
parsed with. Also asserts a malformed stored directive fails the doc loudly.
"""
from __future__ import annotations
import asyncio
from pathlib import Path
from types import SimpleNamespace
import pytest
from lightrag.parser.base import ParseContext
from lightrag.parser.external._base import ExternalParserBase
class _Stop(Exception):
"""Sentinel to halt parse() right after the hook under test runs."""
def _make_ctx(tmp_path: Path, parse_engine: str) -> ParseContext:
source = tmp_path / "doc.pdf"
source.write_bytes(b"%PDF-1.4 test")
ctx = ParseContext(
rag=SimpleNamespace(),
doc_id="doc-1",
file_path=str(source),
content_data={"parse_engine": parse_engine},
)
# Avoid the pipeline-layer source resolution; hand the template a simple
# resolved-source stand-in pointing at our temp file.
rs = SimpleNamespace(
source_path=source, parsed_dir=tmp_path / "parsed", document_name="doc.pdf"
)
ctx.resolve = lambda engine_name: rs # type: ignore[assignment]
return ctx
class _RecordingParser(ExternalParserBase):
engine_name = "mineru" # registered engine that accepts page_range
raw_dir_suffix = ".raw"
force_reparse_env = "LIGHTRAG_TEST_FORCE_REPARSE"
def __init__(self, *, hit: bool) -> None:
self._hit = hit
self.seen: dict[str, object] = {}
self.download_called = False
def is_bundle_valid(self, raw_dir, source_path, *, engine_params=None):
self.seen["is_bundle_valid"] = engine_params
return self._hit
async def download_into(
self, raw_dir, source_path, *, upload_name, engine_params=None
):
self.seen["download_into"] = engine_params
self.download_called = True
raise _Stop()
def build_ir(self, raw_dir, document_name): # hit path halts here
raise _Stop()
def test_miss_path_threads_identical_engine_params(tmp_path):
ctx = _make_ctx(tmp_path, "mineru(page_range=1-3)")
parser = _RecordingParser(hit=False)
with pytest.raises(_Stop):
asyncio.run(parser.parse(ctx))
assert parser.download_called is True
# Both hooks received the SAME decoded params dict.
assert parser.seen["is_bundle_valid"] == {"page_range": "1-3"}
assert parser.seen["download_into"] == {"page_range": "1-3"}
def test_hit_path_skips_client(tmp_path):
ctx = _make_ctx(tmp_path, "mineru(page_range=1-3)")
parser = _RecordingParser(hit=True)
with pytest.raises(_Stop): # halts in build_ir
asyncio.run(parser.parse(ctx))
assert parser.download_called is False
assert parser.seen["is_bundle_valid"] == {"page_range": "1-3"}
def test_bare_engine_threads_none(tmp_path):
ctx = _make_ctx(tmp_path, "mineru")
parser = _RecordingParser(hit=False)
with pytest.raises(_Stop):
asyncio.run(parser.parse(ctx))
assert parser.seen["is_bundle_valid"] is None
assert parser.seen["download_into"] is None
def test_malformed_stored_parse_engine_fails_loudly(tmp_path):
ctx = _make_ctx(tmp_path, "mineru(page_range=") # unbalanced
parser = _RecordingParser(hit=False)
with pytest.raises(ValueError, match="invalid parse_engine"):
asyncio.run(parser.parse(ctx))
# Failed before reaching either hook.
assert "is_bundle_valid" not in parser.seen
+1
View File
@@ -0,0 +1 @@
"""Tests for the native markdown parser engine (.md / .textpack)."""
+101
View File
@@ -0,0 +1,101 @@
"""End-to-end: markdown → IRDoc → sidecar files, and re-parse stability.
Drives the parser's ``extract`` + ``build_ir`` hooks through the real
``write_sidecar`` (the same path ``NativeParserBase.parse`` uses) and asserts
the produced sidecar shape, then re-runs to confirm byte-stable content blocks
(no runtime stamping leaks into the per-block content).
"""
from __future__ import annotations
import json
from pathlib import Path
from lightrag.parser.markdown.parser import NativeMarkdownParser
from lightrag.sidecar import write_sidecar
_MD = """# Doc Title
Intro ![x](data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mP8z8BQDwAEhQGAhKmMIQAAAABJRU5ErkJggg==).
## Section A
| Name | Age |
|------|-----|
| Bob | 30 |
$$
E = mc^2
$$
<table><thead><tr><th>K</th></tr></thead><tbody><tr><td>a</td></tr></tbody></table>
"""
def _build_sidecar(tmp_path: Path) -> Path:
parser = NativeMarkdownParser()
parsed_dir = tmp_path / "doc.md.parsed"
asset_dir = parsed_dir / "doc.blocks.assets"
parsed_dir.mkdir(parents=True)
asset_dir.mkdir()
blocks, _, meta = parser._extract_text(_MD, bundle_root=None)
ir = parser.build_ir(
blocks,
document_name="doc.md",
asset_dir_name=asset_dir.name,
metadata=meta,
)
write_sidecar(
ir,
parsed_dir=parsed_dir,
doc_id="doc-test",
engine="native",
clean_parsed_dir=False,
block_drawing_path_style=parser.sidecar_path_style,
)
return parsed_dir
def _content_lines(parsed_dir: Path) -> list[str]:
lines = (parsed_dir / "doc.blocks.jsonl").read_text().splitlines()
return [line for line in lines if json.loads(line).get("type") == "content"]
def test_sidecar_files_and_payload(tmp_path: Path):
parsed_dir = _build_sidecar(tmp_path)
assert (parsed_dir / "doc.blocks.jsonl").is_file()
assert (parsed_dir / "doc.tables.json").is_file()
assert (parsed_dir / "doc.equations.json").is_file()
assert (parsed_dir / "doc.drawings.json").is_file()
assets = list((parsed_dir / "doc.blocks.assets").iterdir())
assert len(assets) == 1 and assets[0].read_bytes().startswith(b"\x89PNG")
meta_line = json.loads(
(parsed_dir / "doc.blocks.jsonl").read_text().splitlines()[0]
)
assert meta_line["document_format"] == "md"
assert meta_line["doc_title"] == "Doc Title"
assert meta_line["parse_engine"] == "native"
tables = json.loads((parsed_dir / "doc.tables.json").read_text())["tables"]
fmts = sorted(t["format"] for t in tables.values())
assert fmts == ["html", "json"]
# Pipe-table header survives as a JSON grid; HTML header as a <thead>.
json_table = next(t for t in tables.values() if t["format"] == "json")
assert json.loads(json_table["table_header"]) == [["Name", "Age"]]
html_table = next(t for t in tables.values() if t["format"] == "html")
assert "<thead>" in html_table["table_header"]
equations = json.loads((parsed_dir / "doc.equations.json").read_text())["equations"]
assert any(e["content"] == "E = mc^2" for e in equations.values())
drawings = json.loads((parsed_dir / "doc.drawings.json").read_text())["drawings"]
(drawing,) = drawings.values()
assert drawing["path"].startswith("doc.blocks.assets/")
assert drawing["format"] == "png"
def test_reparse_content_is_byte_stable(tmp_path: Path):
first = _content_lines(_build_sidecar(tmp_path / "a"))
second = _content_lines(_build_sidecar(tmp_path / "b"))
assert first == second
+182
View File
@@ -0,0 +1,182 @@
"""Unit tests for ``lightrag.parser.markdown.extract`` (pure extraction).
Locks in the supported-subset contract: ATX heading splitting, fenced-code
suppression, pipe / HTML table recognition with headers, block-level ``$$``
math (and inline ``$`` left alone), and inline image resolution via a stubbed
resolver.
"""
from __future__ import annotations
from lightrag.parser.markdown.extract import (
PREFACE_HEADING,
ResolvedImage,
extract_markdown,
)
class _StubResolver:
"""Resolves ``http(s)`` → external link, everything else → local bytes."""
def __init__(self) -> None:
self.calls: list[str] = []
def resolve(self, src: str) -> ResolvedImage:
self.calls.append(src)
if src.startswith(("http://", "https://")):
return ResolvedImage(kind="external", url=src, fmt="png")
return ResolvedImage(
kind="local",
asset_ref="sha:" + src,
data=b"BYTES:" + src.encode(),
suggested_name="img.png",
fmt="png",
)
def _extract(md: str):
return extract_markdown(md, image_resolver=_StubResolver())
def test_headings_split_blocks_and_track_parents():
md = "# A\nintro\n## B\nbody\n### C\ndeep\n# D\nlast"
ex = _extract(md)
summary = [(b["heading"], b["level"], b["parent_headings"]) for b in ex.blocks]
assert summary == [
("A", 1, []),
("B", 2, ["A"]),
("C", 3, ["A", "B"]),
("D", 1, []),
]
# Heading line is rendered into the block body (markdown-style).
assert ex.blocks[0]["content"].startswith("# A")
def test_content_before_first_heading_is_preface():
ex = _extract("loose intro line\n\n# Real")
assert ex.blocks[0]["heading"] == PREFACE_HEADING
assert ex.blocks[0]["level"] == 0
assert "loose intro line" in ex.blocks[0]["content"]
def test_fenced_code_suppresses_all_detection():
md = "# H\n```python\n# not a heading\n$$ not eq $$\n| not | table |\n![x](no.png)\n```\n"
ex = _extract(md)
# Only the single real heading produced a block; fence content stays verbatim.
assert len(ex.blocks) == 1
assert not ex.tables and not ex.equations and not ex.drawings
assert "# not a heading" in ex.blocks[0]["content"]
assert "$$ not eq $$" in ex.blocks[0]["content"]
def test_tilde_fence_supported():
md = "# H\n~~~\n## still code\n~~~\nafter"
ex = _extract(md)
assert len(ex.blocks) == 1
assert "## still code" in ex.blocks[0]["content"]
def test_pipe_table_with_header():
md = "| Name | Age |\n|------|-----|\n| Bob | 30 |\n| Sue | 25 |\n"
ex = _extract(md)
assert len(ex.tables) == 1
(table,) = ex.tables.values()
assert table["kind"] == "pipe"
assert table["rows"] == [["Bob", "30"], ["Sue", "25"]]
assert table["header"] == [["Name", "Age"]]
def test_pipe_table_requires_delimiter_row():
# A pipe-containing line with no delimiter row underneath is plain text.
md = "| just | text |\nnot a delimiter\n"
ex = _extract(md)
assert not ex.tables
def test_pipe_line_over_thematic_break_is_not_a_table():
# A pipe-containing paragraph followed by a bare ``---`` (thematic break /
# setext underline) has mismatched column counts (2 vs 1) and so is NOT a
# GFM table — it must stay plain text rather than be misrecognised.
md = "foo | bar\n---\nnext paragraph\n"
ex = _extract(md)
assert not ex.tables
assert "foo | bar" in ex.blocks[0]["content"]
def test_pipe_table_column_count_must_match_header():
# Delimiter row column count differs from the header → not a table.
md = "| a | b | c |\n| --- | --- |\n| 1 | 2 | 3 |\n"
ex = _extract(md)
assert not ex.tables
def test_html_table_captured_verbatim_spanning_lines():
md = (
"<table>\n"
"<thead><tr><th>K</th></tr></thead>\n"
"<tbody><tr><td>a</td></tr></tbody>\n"
"</table>\n"
)
ex = _extract(md)
assert len(ex.tables) == 1
(table,) = ex.tables.values()
assert table["kind"] == "html"
assert "<thead>" in table["html"] and "</table>" in table["html"]
def test_block_equation_single_and_multiline():
md = "$$ E = mc^2 $$\n\ntext\n\n$$\n\\sum x\n$$\n"
ex = _extract(md)
latexes = list(ex.equations.values())
assert "E = mc^2" in latexes
assert "\\sum x" in latexes
def test_inline_dollar_not_recognized_as_equation():
ex = _extract("cost is $5 and $10 today")
assert not ex.equations
assert "$5" in ex.blocks[0]["content"]
def test_unclosed_block_equation_is_plain_text():
ex = _extract("$$\nE = mc^2\nno closing")
assert not ex.equations
assert "$$" in ex.blocks[0]["content"]
def test_inline_image_local_and_external_and_dedup():
md = "![a](one.png) and ![b](one.png) and ![c](http://x/y.png)"
resolver = _StubResolver()
ex = extract_markdown(md, image_resolver=resolver)
# Three occurrences → three drawings; same local src deduped to one asset.
assert len(ex.drawings) == 3
assert len(ex.assets) == 1
kinds = sorted(d["kind"] for d in ex.drawings.values())
assert kinds == ["external", "local", "local"]
# Asset dedup is by ``asset_ref`` in extract (the resolver itself owns any
# per-src call caching), so the stub sees one call per occurrence.
assert resolver.calls.count("one.png") == 2
def test_base64_src_not_echoed_into_drawing_src():
md = "![x](data:image/png;base64,QUJD)"
class _B64:
def resolve(self, src):
return ResolvedImage(
kind="local",
asset_ref="sha:b64",
data=b"ABC",
suggested_name="image.png",
fmt="png",
)
ex = extract_markdown(md, image_resolver=_B64())
(drawing,) = ex.drawings.values()
assert drawing["src"] == ""
def test_reference_style_image_not_recognized():
# ``![alt][id]`` reference-style is out of the supported subset.
ex = _extract("![alt][id]\n\n[id]: real.png")
assert not ex.drawings
+155
View File
@@ -0,0 +1,155 @@
"""Unit tests for ``NativeMarkdownIRBuilder`` (markers + side tables → IRDoc)."""
from __future__ import annotations
from lightrag.parser.markdown.extract import (
drawing_marker,
equation_marker,
table_marker,
)
from lightrag.parser.markdown.ir_builder import NativeMarkdownIRBuilder
def _normalize(blocks, meta, *, document_name="doc.md"):
return NativeMarkdownIRBuilder().normalize(
blocks,
document_name=document_name,
asset_dir_name="doc.blocks.assets",
parse_metadata=meta,
)
def test_pipe_table_becomes_json_irtable_with_header_grid():
blocks = [
{
"heading": "H",
"level": 1,
"parent_headings": [],
"content": table_marker("t1"),
}
]
meta = {
"md_tables": {
"t1": {"kind": "pipe", "rows": [["a", "b"]], "header": [["H1", "H2"]]}
}
}
ir = _normalize(blocks, meta)
(table,) = ir.blocks[0].tables
assert "{{TBL:" in ir.blocks[0].content_template
assert table.rows == [["a", "b"]]
assert table.html is None
assert table.table_header == [["H1", "H2"]]
assert (table.num_rows, table.num_cols) == (1, 2)
def test_html_table_becomes_html_irtable_with_thead_and_dims():
html = "<table><thead><tr><th>K</th><th>V</th></tr></thead><tbody><tr><td>a</td><td>1</td></tr></tbody></table>"
blocks = [
{
"heading": "H",
"level": 1,
"parent_headings": [],
"content": table_marker("t1"),
}
]
ir = _normalize(blocks, {"md_tables": {"t1": {"kind": "html", "html": html}}})
(table,) = ir.blocks[0].tables
assert table.rows is None
assert table.html and table.html.startswith("<table>")
assert isinstance(table.table_header, str) and "<thead>" in table.table_header
assert table.body_override and "<thead>" in table.body_override
assert (table.num_rows, table.num_cols) == (2, 2)
def test_equation_is_block_level():
blocks = [
{
"heading": "H",
"level": 1,
"parent_headings": [],
"content": equation_marker("e1"),
}
]
ir = _normalize(blocks, {"md_equations": {"e1": "E = mc^2"}})
(eq,) = ir.blocks[0].equations
assert eq.is_block is True
assert eq.latex == "E = mc^2"
assert "{{EQ:" in ir.blocks[0].content_template
def test_local_drawing_dedups_assetspec_across_occurrences():
content = f"{drawing_marker('d1')} {drawing_marker('d2')}"
blocks = [{"heading": "H", "level": 1, "parent_headings": [], "content": content}]
meta = {
"md_drawings": {
"d1": {"kind": "local", "asset_ref": "sha:1", "fmt": "png", "src": "a.png"},
"d2": {"kind": "local", "asset_ref": "sha:1", "fmt": "png", "src": "a.png"},
},
"md_assets": {"sha:1": {"suggested_name": "a.png", "data": b"BYTES"}},
}
ir = _normalize(blocks, meta)
assert len(ir.blocks[0].drawings) == 2
assert all(d.asset_ref == "sha:1" for d in ir.blocks[0].drawings)
# One shared on-disk asset, carried as bytes for the writer to materialize.
assert len(ir.assets) == 1
assert ir.assets[0].ref == "sha:1"
assert ir.assets[0].source == b"BYTES"
def test_external_drawing_uses_path_override_and_no_asset():
blocks = [
{
"heading": "H",
"level": 1,
"parent_headings": [],
"content": drawing_marker("d1"),
}
]
meta = {
"md_drawings": {
"d1": {
"kind": "external",
"url": "http://x/y.png",
"fmt": "png",
"src": "http://x/y.png",
}
}
}
ir = _normalize(blocks, meta)
(drawing,) = ir.blocks[0].drawings
assert drawing.path_override == "http://x/y.png"
assert drawing.asset_ref == ""
assert ir.assets == []
def test_doc_title_from_first_h1_else_filename():
ir = _normalize(
[{"heading": "First", "level": 1, "parent_headings": [], "content": "# First"}],
{},
)
assert ir.doc_title == "First"
ir2 = _normalize(
[{"heading": "Sub", "level": 2, "parent_headings": [], "content": "## Sub"}],
{},
document_name="myfile.md",
)
assert ir2.doc_title == "myfile"
def test_positions_use_heading_anchor():
ir = _normalize(
[{"heading": "Sec", "level": 2, "parent_headings": [], "content": "## Sec"}],
{},
)
(pos,) = ir.blocks[0].positions
assert pos.type == "heading"
assert pos.anchor == "Sec"
def test_document_format_from_suffix():
ir = _normalize(
[{"heading": "X", "level": 1, "parent_headings": [], "content": "# X"}],
{},
document_name="a.textpack",
)
assert ir.document_format == "textpack"
+446
View File
@@ -0,0 +1,446 @@
"""Tests for ``NativeMarkdownParser`` extract/resolver behaviour.
Covers base64 decoding, ``.textpack`` extraction + file-reference resolution
with a traversal guard, zip-bomb rejection, and the SSRF-guarded remote image
download (monkeypatched ``urllib``), including the env-controlled failure
policy.
"""
from __future__ import annotations
import io
import zipfile
from pathlib import Path
import pytest
from lightrag.parser.markdown import parser as md_parser
from lightrag.parser.markdown.parser import (
NativeMarkdownParser,
_host_is_public,
_image_bytes_and_ext,
_image_ext_from_magic,
_looks_like_svg,
_pin_socket,
)
# A 1x1 transparent PNG.
_PNG_BYTES = (
b"\x89PNG\r\n\x1a\n\x00\x00\x00\rIHDR\x00\x00\x00\x01\x00\x00\x00\x01\x08\x06"
b"\x00\x00\x00\x1f\x15\xc4\x89\x00\x00\x00\nIDATx\x9cc\x00\x01\x00\x00\x05"
b"\x00\x01\r\n-\xb4\x00\x00\x00\x00IEND\xaeB`\x82"
)
_PNG_B64 = "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mP8z8BQDwAEhQGAhKmMIQAAAABJRU5ErkJggg=="
def _make_parser() -> NativeMarkdownParser:
return NativeMarkdownParser()
_SVG_BYTES = (
b'<?xml version="1.0"?>\n'
b'<svg xmlns="http://www.w3.org/2000/svg" width="4" height="4">'
b'<rect width="4" height="4" fill="red"/></svg>'
)
def test_magic_detection():
assert _image_ext_from_magic(_PNG_BYTES) == "png"
assert _image_ext_from_magic(b"\xff\xd8\xff\xe0junk") == "jpg"
assert _image_ext_from_magic(b"not an image") is None
# SVG is text, not a raster magic — recognised only by the SVG sniffer.
assert _image_ext_from_magic(_SVG_BYTES) is None
def test_looks_like_svg():
assert _looks_like_svg(_SVG_BYTES) is True
assert _looks_like_svg(b"\xef\xbb\xbf \n<svg></svg>") is True # BOM + space
assert _looks_like_svg(_PNG_BYTES) is False
assert _looks_like_svg(b"<html><body>no svg</body></html>") is False
def test_svg_rasterized_to_png_via_coerce():
coerced = _image_bytes_and_ext(
_SVG_BYTES, max_bytes=25 * 1024 * 1024, max_svg_pixels=16_000_000
)
assert coerced is not None
data, ext = coerced
assert ext == "png"
assert data.startswith(b"\x89PNG")
def test_svg_coerce_skips_when_rasterization_fails(monkeypatch):
# cairosvg unavailable / render failure -> coerce returns None (caller skips).
monkeypatch.setattr(md_parser, "_rasterize_svg", lambda raw, **kw: None)
assert (
_image_bytes_and_ext(_SVG_BYTES, max_bytes=25 * 1024 * 1024, max_svg_pixels=1)
is None
)
def test_svg_coerce_skips_when_rasterized_png_too_large(monkeypatch):
monkeypatch.setattr(md_parser, "_rasterize_svg", lambda raw, **kw: _PNG_BYTES)
assert (
_image_bytes_and_ext(_SVG_BYTES, max_bytes=4, max_svg_pixels=16_000_000) is None
)
def test_svg_dimensions_parsed_from_width_height_and_viewbox():
assert md_parser._svg_pixel_dimensions(_SVG_BYTES) == (4, 4)
# Unit conversion: 1in = 96px, 2in = 192px.
svg_in = b'<svg xmlns="http://www.w3.org/2000/svg" width="1in" height="2in"/>'
assert md_parser._svg_pixel_dimensions(svg_in) == (96, 192)
# width/height missing or relative (%) -> fall back to viewBox w/h.
svg_vb = (
b'<svg xmlns="http://www.w3.org/2000/svg" width="100%" viewBox="0 0 30 20"/>'
)
assert md_parser._svg_pixel_dimensions(svg_vb) == (30, 20)
# Neither resolvable -> None (skipped by the rasterizer).
svg_none = b'<svg xmlns="http://www.w3.org/2000/svg"/>'
assert md_parser._svg_pixel_dimensions(svg_none) is None
def test_svg_oversized_canvas_rejected_before_render(monkeypatch):
# A tiny SVG declaring a huge canvas is rejected on the pre-render pixel
# budget — cairosvg.svg2png must never be reached.
import cairosvg
def _boom(*a, **k): # pragma: no cover - must not be called
raise AssertionError("cairosvg.svg2png should not run for oversized canvas")
monkeypatch.setattr(cairosvg, "svg2png", _boom)
big = b'<svg xmlns="http://www.w3.org/2000/svg" width="100000" height="100000"/>'
assert md_parser._rasterize_svg(big, max_pixels=16_000_000) is None
def test_base64_svg_decoded_and_rasterized():
import base64 as _b64
p = _make_parser()
b64 = _b64.b64encode(_SVG_BYTES).decode()
md = f"# H\n\n![x](data:image/svg+xml;base64,{b64})\n"
_, warnings, meta = p._extract_text(md, bundle_root=None)
(asset,) = meta["md_assets"].values()
assert asset["fmt"] == "png"
assert asset["data"].startswith(b"\x89PNG")
(drawing,) = meta["md_drawings"].values()
assert drawing["kind"] == "local"
assert not warnings
def test_textpack_svg_file_rasterized_to_png(tmp_path: Path):
pack = tmp_path / "note.textpack"
with zipfile.ZipFile(pack, "w") as zf:
zf.writestr("text.markdown", "# N\n\n![logo](assets/logo.svg)\n")
zf.writestr("assets/logo.svg", _SVG_BYTES)
p = _make_parser()
_, warnings, meta = p.extract(
pack, parsed_dir=tmp_path, asset_dir=tmp_path, base_name="note"
)
(asset,) = meta["md_assets"].values()
assert asset["fmt"] == "png"
assert asset["data"].startswith(b"\x89PNG")
# The on-disk name takes the rasterized extension.
assert asset["suggested_name"] == "logo.png"
assert not warnings
def test_host_is_public_rejects_internal(monkeypatch):
monkeypatch.delenv("NATIVE_MD_IMAGE_ALLOWED_NON_PUBLIC_CIDRS", raising=False)
assert _host_is_public("8.8.8.8") is True
assert _host_is_public("127.0.0.1") is False
assert _host_is_public("10.0.0.1") is False
assert _host_is_public("169.254.0.1") is False
assert _host_is_public("::1") is False
# CGNAT 100.64.0.0/10 is not private/reserved in Python's flags but is not
# globally routable — default-deny via is_global must still reject it.
assert _host_is_public("100.64.0.1") is False
# TEST-NET (192.0.2.0/24) is likewise non-global.
assert _host_is_public("192.0.2.1") is False
def test_allowlist_permits_configured_non_public(monkeypatch):
monkeypatch.setenv("NATIVE_MD_IMAGE_ALLOWED_NON_PUBLIC_CIDRS", "100.64.0.0/10")
assert _host_is_public("100.64.0.1") is True
# An address outside the allowlist is still rejected.
assert _host_is_public("10.0.0.1") is False
def test_pin_socket_refuses_internal_host(monkeypatch):
# The connection pins to the same resolution that authorised the host, so
# an internal target cannot be reached even if it would resolve.
monkeypatch.delenv("NATIVE_MD_IMAGE_ALLOWED_NON_PUBLIC_CIDRS", raising=False)
with pytest.raises(OSError, match="non-public host"):
_pin_socket("127.0.0.1", 80, None, None)
def test_redirect_to_non_public_host_blocked(monkeypatch):
import email.message
monkeypatch.delenv("NATIVE_MD_IMAGE_ALLOWED_NON_PUBLIC_CIDRS", raising=False)
handler = md_parser._GuardedRedirectHandler()
req = md_parser.urllib.request.Request("http://example.com/a")
with pytest.raises(md_parser.urllib.error.HTTPError):
handler.redirect_request(
req, None, 302, "Found", email.message.Message(), "http://10.0.0.1/evil"
)
def test_guarded_https_handler_open_does_not_raise_attribute_error(monkeypatch):
# Regression: ``https_open`` must forward only the kwargs the stdlib handler
# actually has on this Python version. On 3.12 ``HTTPSHandler`` has no
# ``_check_hostname`` attribute, so a hard reference raised AttributeError
# and every real download silently fell back to an external link.
handler = md_parser._GuardedHTTPSHandler()
captured: dict = {}
def _fake_do_open(conn_cls, req, **kwargs):
captured["conn_cls"] = conn_cls
captured["kwargs"] = kwargs
return "ok"
monkeypatch.setattr(handler, "do_open", _fake_do_open)
req = md_parser.urllib.request.Request("https://example.com/x.png")
assert handler.https_open(req) == "ok"
assert captured["conn_cls"] is md_parser._GuardedHTTPSConnection
assert "context" in captured["kwargs"]
def test_build_guarded_opener_constructs_with_guarded_handlers():
opener = md_parser._build_guarded_opener()
handler_types = {type(h) for h in opener.handlers}
assert md_parser._GuardedHTTPSHandler in handler_types
assert md_parser._GuardedHTTPHandler in handler_types
def test_base64_image_decoded_to_asset():
p = _make_parser()
md = f"# H\n\n![x]({_PNG_B64})\n"
blocks, warnings, meta = p._extract_text(md, bundle_root=None)
assert len(meta["md_assets"]) == 1
(asset,) = meta["md_assets"].values()
assert asset["data"].startswith(b"\x89PNG")
assert asset["fmt"] == "png"
(drawing,) = meta["md_drawings"].values()
assert drawing["kind"] == "local"
assert not warnings
def test_base64_non_image_bytes_rejected():
# ``data:image/png;base64,QUJD`` decodes to b"ABC" — declared PNG but not a
# real image. Magic bytes are authoritative, so it must be skipped.
p = _make_parser()
md = "# H\n\n![x](data:image/png;base64,QUJD)\n"
_, warnings, meta = p._extract_text(md, bundle_root=None)
assert meta["md_drawings"] == {}
assert meta["md_assets"] == {}
assert warnings.get("images_skipped") == 1
def test_standalone_md_relative_image_is_skipped():
p = _make_parser()
blocks, warnings, meta = p._extract_text(
"# H\n\n![x](assets/pic.png)\n", bundle_root=None
)
assert meta["md_drawings"] == {}
assert warnings.get("images_skipped") == 1
def _write_textpack(path: Path, *, text: str, assets: dict[str, bytes]) -> None:
with zipfile.ZipFile(path, "w") as zf:
zf.writestr("text.markdown", text)
for name, data in assets.items():
zf.writestr(f"assets/{name}", data)
def test_textpack_file_reference_resolved(tmp_path: Path):
pack = tmp_path / "note.textpack"
_write_textpack(
pack,
text="# Note\n\n![pic](assets/pic.png)\n",
assets={"pic.png": _PNG_BYTES},
)
p = _make_parser()
blocks, warnings, meta = p.extract(
pack, parsed_dir=tmp_path, asset_dir=tmp_path, base_name="note"
)
assert len(meta["md_assets"]) == 1
(asset,) = meta["md_assets"].values()
assert asset["data"] == _PNG_BYTES
assert asset["suggested_name"] == "pic.png"
assert not warnings
def test_textpack_image_outside_assets_dir_is_resolved(tmp_path: Path):
# Compatibility: a reference may point anywhere under the bundle root, not
# only ``assets/`` — here ``media/pic.png`` and a root-level ``cover.png``.
pack = tmp_path / "note.textpack"
with zipfile.ZipFile(pack, "w") as zf:
zf.writestr("text.markdown", "# N\n\n![a](media/pic.png)\n\n![b](cover.png)\n")
zf.writestr("media/pic.png", _PNG_BYTES)
zf.writestr("cover.png", _PNG_BYTES)
p = _make_parser()
_, warnings, meta = p.extract(
pack, parsed_dir=tmp_path, asset_dir=tmp_path, base_name="note"
)
assert len(meta["md_drawings"]) == 2
assert all(d["kind"] == "local" for d in meta["md_drawings"].values())
# Same bytes → deduped to a single asset.
assert len(meta["md_assets"]) == 1
assert not warnings
def test_textpack_non_image_file_reference_is_skipped(tmp_path: Path):
pack = tmp_path / "note.textpack"
with zipfile.ZipFile(pack, "w") as zf:
zf.writestr("text.markdown", "# N\n\n![x](assets/fake.png)\n")
zf.writestr("assets/fake.png", b"this is not an image")
p = _make_parser()
_, warnings, meta = p.extract(
pack, parsed_dir=tmp_path, asset_dir=tmp_path, base_name="note"
)
assert meta["md_drawings"] == {}
assert warnings.get("images_skipped") == 1
def test_textpack_oversized_image_is_skipped(tmp_path: Path, monkeypatch):
# A bundled asset larger than NATIVE_MD_IMAGE_MAX_BYTES is skipped before it
# is read into memory (the per-file cap, separate from the zip-bomb total).
monkeypatch.setenv("NATIVE_MD_IMAGE_MAX_BYTES", "16")
pack = tmp_path / "big.textpack"
with zipfile.ZipFile(pack, "w") as zf:
zf.writestr("text.markdown", "# N\n\n![x](assets/big.png)\n")
zf.writestr("assets/big.png", _PNG_BYTES) # well over 16 bytes
p = _make_parser()
_, warnings, meta = p.extract(
pack, parsed_dir=tmp_path, asset_dir=tmp_path, base_name="big"
)
assert meta["md_drawings"] == {}
assert meta["md_assets"] == {}
assert warnings.get("images_skipped") == 1
def test_textpack_traversal_reference_is_skipped(tmp_path: Path):
pack = tmp_path / "evil.textpack"
_write_textpack(
pack,
text="# Evil\n\n![x](../../../etc/passwd)\n",
assets={"pic.png": _PNG_BYTES},
)
p = _make_parser()
_, warnings, meta = p.extract(
pack, parsed_dir=tmp_path, asset_dir=tmp_path, base_name="evil"
)
assert meta["md_drawings"] == {}
assert warnings.get("images_skipped") == 1
def test_textpack_zip_bomb_entry_count_rejected(tmp_path: Path, monkeypatch):
monkeypatch.setattr(md_parser, "_TEXTPACK_MAX_ENTRIES", 3)
pack = tmp_path / "bomb.textpack"
with zipfile.ZipFile(pack, "w") as zf:
zf.writestr("text.markdown", "# H\n")
for i in range(10):
zf.writestr(f"assets/f{i}.bin", b"x")
p = _make_parser()
with pytest.raises(RuntimeError, match="entries"):
p.extract(pack, parsed_dir=tmp_path, asset_dir=tmp_path, base_name="bomb")
# --- remote download (monkeypatched urllib) --------------------------------
class _FakeHeaders:
def __init__(self, content_type: str) -> None:
self._ct = content_type
def get_content_type(self) -> str:
return self._ct
class _FakeResponse(io.BytesIO):
def __init__(self, data: bytes, content_type: str) -> None:
super().__init__(data)
self.headers = _FakeHeaders(content_type)
def __enter__(self):
return self
def __exit__(self, *exc):
self.close()
def _patch_download(
monkeypatch, *, data=_PNG_BYTES, content_type="image/png", host_public=True
):
monkeypatch.setattr(md_parser, "_host_is_public", lambda host: host_public)
class _Opener:
def open(self, req, timeout=None):
return _FakeResponse(data, content_type)
monkeypatch.setattr(
md_parser.urllib.request, "build_opener", lambda *a, **k: _Opener()
)
def test_remote_image_dropped_when_download_disabled(monkeypatch):
# With downloading explicitly disabled an external image is dropped entirely
# — no drawing, no asset — so a doc whose only image is an external link
# produces no drawings.json. (Downloading is ON by default.)
monkeypatch.setenv("NATIVE_MD_IMAGE_DOWNLOAD_ENABLED", "false")
p = _make_parser()
_, warnings, meta = p._extract_text(
"# H\n\n![x](http://host/y.png)\n", bundle_root=None
)
assert meta["md_drawings"] == {}
assert meta["md_assets"] == {}
assert warnings.get("images_external_dropped") == 1
def test_remote_image_downloaded_when_enabled(monkeypatch):
monkeypatch.setenv("NATIVE_MD_IMAGE_DOWNLOAD_ENABLED", "true")
_patch_download(monkeypatch)
p = _make_parser()
_, warnings, meta = p._extract_text(
"# H\n\n![x](http://host/y.png)\n", bundle_root=None
)
(drawing,) = meta["md_drawings"].values()
assert drawing["kind"] == "local"
(asset,) = meta["md_assets"].values()
assert asset["data"] == _PNG_BYTES
def test_remote_image_failure_kept_external_by_default(monkeypatch):
monkeypatch.setenv("NATIVE_MD_IMAGE_DOWNLOAD_ENABLED", "true")
monkeypatch.delenv("NATIVE_MD_IMAGE_DOWNLOAD_REQUIRED", raising=False)
_patch_download(monkeypatch, content_type="text/html") # explicit non-image
p = _make_parser()
_, warnings, meta = p._extract_text(
"# H\n\n![x](http://host/y.png)\n", bundle_root=None
)
(drawing,) = meta["md_drawings"].values()
assert drawing["kind"] == "external"
assert warnings.get("images_download_failed") == 1
def test_remote_image_failure_raises_when_required(monkeypatch):
monkeypatch.setenv("NATIVE_MD_IMAGE_DOWNLOAD_ENABLED", "true")
monkeypatch.setenv("NATIVE_MD_IMAGE_DOWNLOAD_REQUIRED", "true")
_patch_download(monkeypatch, content_type="text/html")
p = _make_parser()
with pytest.raises(Exception):
p._extract_text("# H\n\n![x](http://host/y.png)\n", bundle_root=None)
def test_remote_image_private_host_blocked(monkeypatch):
monkeypatch.setenv("NATIVE_MD_IMAGE_DOWNLOAD_ENABLED", "true")
monkeypatch.delenv("NATIVE_MD_IMAGE_DOWNLOAD_REQUIRED", raising=False)
_patch_download(monkeypatch, host_public=False)
p = _make_parser()
_, warnings, meta = p._extract_text(
"# H\n\n![x](http://169.254.0.1/y.png)\n", bundle_root=None
)
(drawing,) = meta["md_drawings"].values()
assert drawing["kind"] == "external"
assert warnings.get("images_download_failed") == 1
+214
View File
@@ -0,0 +1,214 @@
"""Tests for the native-markdown downloaded-image cache (`.native_raw/`).
The cache lets a re-parse of an unchanged file reuse already-downloaded images
instead of hitting the network. ``_download`` is monkeypatched so a cache hit is
proven by it NOT being called (it raises) while assets still materialize.
"""
from __future__ import annotations
import shutil
from pathlib import Path
import pytest
from lightrag.parser.markdown import parser as md_parser
from lightrag.parser.markdown.parser import NativeMarkdownParser
# A 1x1 transparent PNG.
_PNG_BYTES = (
b"\x89PNG\r\n\x1a\n\x00\x00\x00\rIHDR\x00\x00\x00\x01\x00\x00\x00\x01\x08\x06"
b"\x00\x00\x00\x1f\x15\xc4\x89\x00\x00\x00\nIDATx\x9cc\x00\x01\x00\x00\x05"
b"\x00\x01\r\n-\xb4\x00\x00\x00\x00IEND\xaeB`\x82"
)
_URL = "http://host/y.png"
_MD = f"# H\n\n![x]({_URL})\n"
@pytest.fixture(autouse=True)
def _enable_download(monkeypatch):
monkeypatch.setenv("NATIVE_MD_IMAGE_DOWNLOAD_ENABLED", "true")
monkeypatch.delenv("LIGHTRAG_FORCE_REPARSE_NATIVE", raising=False)
monkeypatch.delenv("NATIVE_MD_IMAGE_MAX_BYTES", raising=False)
monkeypatch.delenv("NATIVE_MD_IMAGE_MAX_SVG_PIXELS", raising=False)
monkeypatch.delenv("NATIVE_MD_IMAGE_ALLOWED_NON_PUBLIC_CIDRS", raising=False)
def _patch_download(monkeypatch, payload=(_PNG_BYTES, "png")):
"""Patch ``_download`` to count calls and return fixed bytes."""
counter = {"n": 0}
def _fake(self, src):
counter["n"] += 1
return payload
monkeypatch.setattr(md_parser._MarkdownImageResolver, "_download", _fake)
return counter
def _forbid_download(monkeypatch):
"""Patch ``_download`` to fail the test if the network is touched."""
def _boom(self, src): # pragma: no cover - must not be called on a cache hit
raise AssertionError("network download must not run on a cache hit")
monkeypatch.setattr(md_parser._MarkdownImageResolver, "_download", _boom)
def _make_doc(tmp_path: Path, text: str = _MD) -> tuple[Path, Path]:
src = tmp_path / "doc.md"
src.write_text(text)
parsed = tmp_path / "__parsed__" / "doc.md.parsed"
parsed.mkdir(parents=True)
(parsed / "doc.blocks.assets").mkdir()
return src, parsed
def _extract(p: NativeMarkdownParser, src: Path, parsed: Path):
return p.extract(
src, parsed_dir=parsed, asset_dir=parsed / "doc.blocks.assets", base_name="doc"
)
def _wipe_parsed(parsed: Path) -> None:
"""Mirror NativeParserBase.parse's rmtree(parsed_dir) before a re-extract."""
shutil.rmtree(parsed)
parsed.mkdir(parents=True)
(parsed / "doc.blocks.assets").mkdir()
def _raw_dir(tmp_path: Path) -> Path:
return tmp_path / "__parsed__" / "doc.md.native_raw"
def test_first_parse_downloads_and_writes_bundle(tmp_path, monkeypatch):
counter = _patch_download(monkeypatch)
src, parsed = _make_doc(tmp_path)
_, _, meta = _extract(NativeMarkdownParser(), src, parsed)
assert counter["n"] == 1
assert len(meta["md_assets"]) == 1
raw = _raw_dir(tmp_path)
assert (raw / "_manifest.json").is_file()
# Exactly one cached image file alongside the manifest.
files = sorted(c.name for c in raw.iterdir())
assert len(files) == 2 and "_manifest.json" in files
def test_reparse_unchanged_source_is_cache_hit(tmp_path, monkeypatch):
_patch_download(monkeypatch)
p = NativeMarkdownParser()
src, parsed = _make_doc(tmp_path)
_extract(p, src, parsed) # populate cache
_wipe_parsed(parsed) # base parser would rmtree the .parsed dir
_forbid_download(monkeypatch) # any download now fails the test
_, warnings, meta = _extract(p, src, parsed)
assert warnings.get("images_cache_hit") == 1
assert len(meta["md_assets"]) == 1
(asset,) = meta["md_assets"].values()
assert asset["data"] == _PNG_BYTES
def test_pure_cache_hit_does_not_touch_bundle(tmp_path, monkeypatch):
# A pure hit must write nothing: the manifest + image mtimes stay frozen, so
# on-disk timestamps alone reveal whether the cache was hit.
_patch_download(monkeypatch)
p = NativeMarkdownParser()
src, parsed = _make_doc(tmp_path)
_extract(p, src, parsed)
raw = _raw_dir(tmp_path)
before = {c.name: c.stat().st_mtime_ns for c in raw.iterdir()}
_wipe_parsed(parsed)
_forbid_download(monkeypatch) # pure hit: no download allowed
_, warnings, _ = _extract(p, src, parsed)
assert warnings.get("images_cache_hit") == 1
after = {c.name: c.stat().st_mtime_ns for c in raw.iterdir()}
assert after == before # nothing rewritten
def test_source_change_invalidates_cache(tmp_path, monkeypatch):
counter = _patch_download(monkeypatch)
p = NativeMarkdownParser()
src, parsed = _make_doc(tmp_path)
_extract(p, src, parsed)
assert counter["n"] == 1
src.write_text("# Changed\n\n![x](http://host/y.png)\n")
_wipe_parsed(parsed)
_extract(p, src, parsed)
assert counter["n"] == 2 # re-downloaded
def test_options_signature_change_invalidates_cache(tmp_path, monkeypatch):
counter = _patch_download(monkeypatch)
p = NativeMarkdownParser()
src, parsed = _make_doc(tmp_path)
_extract(p, src, parsed)
assert counter["n"] == 1
# Changing a byte-affecting knob busts the bundle.
monkeypatch.setenv("NATIVE_MD_IMAGE_MAX_BYTES", "12345")
_wipe_parsed(parsed)
_extract(p, src, parsed)
assert counter["n"] == 2
def test_force_reparse_discards_cache(tmp_path, monkeypatch):
counter = _patch_download(monkeypatch)
p = NativeMarkdownParser()
src, parsed = _make_doc(tmp_path)
_extract(p, src, parsed)
assert counter["n"] == 1
monkeypatch.setenv("LIGHTRAG_FORCE_REPARSE_NATIVE", "true")
_wipe_parsed(parsed)
_extract(p, src, parsed)
assert counter["n"] == 2
def test_tampered_cache_file_falls_back_to_download(tmp_path, monkeypatch):
counter = _patch_download(monkeypatch)
p = NativeMarkdownParser()
src, parsed = _make_doc(tmp_path)
_extract(p, src, parsed)
assert counter["n"] == 1
# Corrupt the cached image bytes so the sha256 no longer matches the manifest.
raw = _raw_dir(tmp_path)
img = next(c for c in raw.iterdir() if c.name != "_manifest.json")
img.write_bytes(b"corrupted")
_wipe_parsed(parsed)
_extract(p, src, parsed)
assert counter["n"] == 2 # integrity mismatch -> miss -> re-download
def test_svg_cached_as_png_and_reused_without_rasterizing(tmp_path, monkeypatch):
# _download returns the post-rasterization PNG, so a cache hit reuses PNG
# bytes and never calls cairosvg.
_patch_download(monkeypatch, payload=(_PNG_BYTES, "png"))
p = NativeMarkdownParser()
src, parsed = _make_doc(tmp_path, text=f"# H\n\n![s]({_URL})\n")
_, _, meta = _extract(p, src, parsed)
(asset,) = meta["md_assets"].values()
assert asset["fmt"] == "png" and asset["data"].startswith(b"\x89PNG")
_wipe_parsed(parsed)
_forbid_download(monkeypatch)
import cairosvg
def _boom(*a, **k): # pragma: no cover - must not run on a cache hit
raise AssertionError("cairosvg must not run on a cache hit")
monkeypatch.setattr(cairosvg, "svg2png", _boom)
_, warnings, meta = _extract(p, src, parsed)
assert warnings.get("images_cache_hit") == 1
def test_native_raw_is_sibling_and_survives_parsed_rmtree(tmp_path, monkeypatch):
_patch_download(monkeypatch)
src, parsed = _make_doc(tmp_path)
_extract(NativeMarkdownParser(), src, parsed)
raw = _raw_dir(tmp_path)
assert raw.is_dir()
assert raw.parent == parsed.parent # sibling of .parsed, not nested
shutil.rmtree(parsed) # base parser wipes parsed_dir
assert raw.is_dir() # cache survives
+81
View File
@@ -0,0 +1,81 @@
"""Routing + registry wiring for the native markdown engine.
``parser_rules=""`` is passed explicitly so these assertions are independent of
any ambient ``LIGHTRAG_PARSER`` in the dev environment.
"""
from __future__ import annotations
import asyncio
from lightrag.parser import registry
from lightrag.parser.native_dispatch import NativeParser
from lightrag.parser.routing import resolve_file_parser_directives
def _engine(path: str, rules: str = "") -> str:
return resolve_file_parser_directives(path, parser_rules=rules)[0]
def test_native_declares_markdown_suffixes():
assert registry.suffix_capabilities("native") == frozenset(
{"docx", "md", "textpack"}
)
def test_textpack_and_md_in_upload_allowlist():
suffixes = registry.available_engine_suffixes()
assert "textpack" in suffixes
assert "md" in suffixes
def test_get_parser_native_is_dispatcher():
parser = registry.get_parser("native")
assert type(parser).__name__ == "NativeParser"
def test_textpack_defaults_to_native_without_hint():
assert _engine("note.textpack") == "native"
def test_md_defaults_to_legacy_opt_in_for_native():
assert _engine("doc.md") == "legacy"
assert _engine("doc.[native].md") == "native"
assert _engine("doc.md", rules="md:native") == "native"
def test_docx_default_unchanged():
assert _engine("report.docx") == "legacy"
def test_textpack_falls_back_to_native_when_rule_engine_cannot_handle_it():
# legacy cannot parse .textpack; the per-suffix default still wins.
assert _engine("note.textpack", rules="textpack:legacy") == "native"
class _SentinelParser:
def __init__(self, name: str) -> None:
self.name = name
async def parse(self, ctx):
return self.name
class _Ctx:
def __init__(self, file_path: str) -> None:
self.file_path = file_path
def _dispatch(file_path: str) -> str:
dispatcher = NativeParser()
dispatcher._docx = _SentinelParser("docx")
dispatcher._markdown = _SentinelParser("markdown")
return asyncio.run(dispatcher.parse(_Ctx(file_path)))
def test_dispatcher_routes_by_suffix():
assert _dispatch("a.md") == "markdown"
assert _dispatch("a.textpack") == "markdown"
assert _dispatch("a.[native-iet].md") == "markdown"
assert _dispatch("a.docx") == "docx"
assert _dispatch("report.[native].docx") == "docx"
@@ -0,0 +1,106 @@
"""Cache-correctness proof for per-file engine params (Phase 2).
A per-file override MUST participate in the raw-bundle cache signature (so an
overridden document does not hit a bundle parsed with different params) AND must
reach the live request. These tests assert both for MinerU and Docling.
"""
from __future__ import annotations
# --------------------------------------------------------------------------- #
# MinerU
# --------------------------------------------------------------------------- #
def test_mineru_signature_changes_with_page_range(monkeypatch):
monkeypatch.setenv("MINERU_API_MODE", "official")
from lightrag.parser.external.mineru.cache import current_mineru_options_signature
base = current_mineru_options_signature()
ov1 = current_mineru_options_signature({"page_range": "1-3"})
ov2 = current_mineru_options_signature({"page_range": "1-3,5"})
assert base != ov1 != ov2 and base != ov2
# No spurious invalidation: no override == bare env signature.
assert current_mineru_options_signature(None) == base
def test_mineru_signature_changes_with_language_and_parse_method(monkeypatch):
monkeypatch.setenv("MINERU_API_MODE", "local")
from lightrag.parser.external.mineru.cache import current_mineru_options_signature
base = current_mineru_options_signature()
assert current_mineru_options_signature({"language": "en"}) != base
assert current_mineru_options_signature({"local_parse_method": "ocr"}) != base
def test_mineru_options_reflect_override(monkeypatch):
monkeypatch.setenv("MINERU_API_MODE", "official")
from lightrag.parser.external.mineru.cache import MinerUParserOptions
opts = MinerUParserOptions.from_env(
overrides={"language": "en", "page_range": "1-3,5"}
)
assert opts.language == "en" and opts.page_ranges == "1-3,5"
def test_mineru_local_bounds_from_page_range_override():
from lightrag.parser.external.mineru.cache import MinerUParserOptions
opts = MinerUParserOptions.from_env(
api_mode="local", overrides={"page_range": "2-4"}
)
# local_page_bounds is 0-based.
assert opts.local_start_page_id == 1 and opts.local_end_page_id == 3
def test_mineru_request_payload_reflects_override(monkeypatch):
monkeypatch.setenv("MINERU_API_MODE", "official")
monkeypatch.setenv("MINERU_OFFICIAL_ENDPOINT", "https://mineru.net")
monkeypatch.setenv("MINERU_API_TOKEN", "test-token")
from lightrag.parser.external.mineru.client import MinerURawClient
client = MinerURawClient(overrides={"language": "en", "page_range": "1-3,5"})
payload = client._official_payload("doc.pdf")
assert payload["language"] == "en"
assert payload["files"][0]["page_ranges"] == "1-3,5"
def test_mineru_local_form_reflects_parse_method(monkeypatch):
monkeypatch.setenv("MINERU_API_MODE", "local")
monkeypatch.setenv("MINERU_LOCAL_ENDPOINT", "http://127.0.0.1:8000")
from lightrag.parser.external.mineru.client import MinerURawClient
client = MinerURawClient(overrides={"local_parse_method": "ocr"})
assert client._local_form_data()["parse_method"] == "ocr"
# --------------------------------------------------------------------------- #
# Docling
# --------------------------------------------------------------------------- #
def test_docling_snapshot_and_signature_reflect_force_ocr():
from lightrag.parser.external.docling.cache import (
compute_options_signature,
snapshot_tunable_env,
)
from lightrag.parser.external.docling.client import FIXED_CONSTANTS
assert snapshot_tunable_env({"force_ocr": False})["DOCLING_FORCE_OCR"] == "false"
base = compute_options_signature(
tunable_env=snapshot_tunable_env(), fixed_constants=FIXED_CONSTANTS
)
ov = compute_options_signature(
tunable_env=snapshot_tunable_env({"force_ocr": False}),
fixed_constants=FIXED_CONSTANTS,
)
assert base != ov
def test_docling_client_reflects_force_ocr_override(monkeypatch):
monkeypatch.setenv("DOCLING_ENDPOINT", "http://localhost:5001")
from lightrag.parser.external.docling.client import DoclingRawClient
client = DoclingRawClient(overrides={"force_ocr": False})
assert client.force_ocr is False
+295
View File
@@ -0,0 +1,295 @@
"""Tests for per-file ENGINE parameters (Phase 2).
Engine params attach to the engine token of a hint / LIGHTRAG_PARSER rule
(``mineru(page_range=1-3,language=en)`` / ``docling(force_ocr=true)``) and ride
the existing ``parse_engine`` field encoded in hint syntax.
The suite's conftest strips ``MINERU_API_MODE`` (tests default to local, which
allows only a single page_range segment), so multi-segment cases explicitly set
official mode via monkeypatch. ``parser_rules=""`` keeps assertions independent
of any ambient ``LIGHTRAG_PARSER``.
"""
from __future__ import annotations
import importlib
import sys
import pytest
from lightrag.parser.param_schema import (
normalize_engine_params,
parse_engine_params,
render_engine_params,
)
from lightrag.parser.routing import (
FilenameParserHintError,
ParserRoutingConfigError,
decode_parse_engine,
encode_parse_engine,
normalize_parser_engine,
resolve_file_parser_directives,
resolve_parser_directives,
validate_parser_routing_config,
)
# Importing document_routes runs argparse over sys.argv at import time;
# neutralise pytest's argv during that first import.
_original_argv = sys.argv[:]
sys.argv = [sys.argv[0]]
importlib.import_module("lightrag.api.routers.document_routes")
sys.argv = _original_argv
# --------------------------------------------------------------------------- #
# parse_engine_params
# --------------------------------------------------------------------------- #
def test_parse_engine_params_basic_and_alias():
parsed, errors = parse_engine_params(
"page_range=1-3,language=en", engine="mineru", label="x"
)
assert errors == []
assert parsed == {"page_range": "1-3", "language": "en"}
# alias pr -> page_range
parsed, errors = parse_engine_params("pr=2-4", engine="mineru", label="x")
assert errors == [] and parsed == {"page_range": "2-4"}
# alias local_pm -> local_parse_method
parsed, errors = parse_engine_params("local_pm=ocr", engine="mineru", label="x")
assert errors == [] and parsed == {"local_parse_method": "ocr"}
def test_parse_engine_params_bool_coercion():
parsed, errors = parse_engine_params("force_ocr=false", engine="docling", label="x")
assert errors == [] and parsed == {"force_ocr": False}
parsed, errors = parse_engine_params("ocr=yes", engine="docling", label="x")
assert errors == [] and parsed == {"force_ocr": True}
def test_parse_engine_params_multi_segment_official(monkeypatch):
monkeypatch.setenv("MINERU_API_MODE", "official")
parsed, errors = parse_engine_params(
"page_range=1-3,page_range=5,page_range=7-9", engine="mineru", label="x"
)
assert errors == [] and parsed == {"page_range": "1-3,5,7-9"}
def test_parse_engine_params_multi_segment_rejected_local(monkeypatch):
monkeypatch.delenv("MINERU_API_MODE", raising=False) # default local
parsed, errors = parse_engine_params(
"page_range=1-3,page_range=5", engine="mineru", label="x"
)
assert parsed == {} or "page_range" not in parsed
assert errors and "local supports only a single page" in errors[0]
def test_parse_engine_params_local_parse_method_accepted_local(monkeypatch):
monkeypatch.delenv("MINERU_API_MODE", raising=False) # default local
parsed, errors = parse_engine_params(
"local_parse_method=ocr", engine="mineru", label="x"
)
assert errors == [] and parsed == {"local_parse_method": "ocr"}
def test_parse_engine_params_local_parse_method_rejected_official(monkeypatch):
# local_parse_method is local-only: official neither sends it nor folds it
# into the cache key, so accepting it would persist a silent no-op directive.
monkeypatch.setenv("MINERU_API_MODE", "official")
parsed, errors = parse_engine_params(
"local_parse_method=ocr", engine="mineru", label="x"
)
assert "local_parse_method" not in parsed
assert errors and "MINERU_API_MODE=local" in errors[0]
@pytest.mark.parametrize(
"block,engine,needle",
[
("page_range=1", "legacy", "does not accept parameters"),
("page_range=1", "native", "does not accept parameters"),
("foo=1", "mineru", "unknown parameter 'foo'"),
("local_parse_method=bad", "mineru", "must be one of"),
("force_ocr=maybe", "docling", "must be a boolean"),
("language=", "mineru", "must be non-empty"),
("page_range=1-3,5", "mineru", "page lists must repeat the key"),
],
)
def test_parse_engine_params_errors(block, engine, needle):
_parsed, errors = parse_engine_params(block, engine=engine, label="x")
assert errors and any(needle in e for e in errors)
# --------------------------------------------------------------------------- #
# normalize_engine_params (resolved-dict path) + encode/decode round-trip
# --------------------------------------------------------------------------- #
def test_normalize_engine_params_coerces_bool_string():
# A direct caller may pass force_ocr as the string "false" — must coerce.
norm, errors = normalize_engine_params("docling", {"force_ocr": "false"})
assert errors == [] and norm == {"force_ocr": False}
def test_normalize_engine_params_accepts_page_range_list_or_string(monkeypatch):
monkeypatch.setenv("MINERU_API_MODE", "official")
by_list, e1 = normalize_engine_params("mineru", {"page_range": ["1-3", "5"]})
by_str, e2 = normalize_engine_params("mineru", {"page_range": "1-3,5"})
assert e1 == [] and e2 == [] and by_list == by_str == {"page_range": "1-3,5"}
def test_normalize_engine_params_rejects_unregistered_engine():
_norm, errors = normalize_engine_params("legacy", {"page_range": "1"})
assert errors and "does not accept parameters" in errors[0]
def test_encode_decode_round_trip(monkeypatch):
monkeypatch.setenv("MINERU_API_MODE", "official")
enc = encode_parse_engine("mineru", {"page_range": "1-3,5", "language": "en"})
# list param emitted as repeated keys; deterministic (sorted)
assert enc == "mineru(language=en,page_range=1-3,page_range=5)"
engine, params, errors = decode_parse_engine(enc)
assert engine == "mineru" and errors == []
assert params == {"page_range": "1-3,5", "language": "en"}
def test_encode_bare_when_no_params():
assert encode_parse_engine("mineru", {}) == "mineru"
assert encode_parse_engine("legacy", None) == "legacy"
def test_render_engine_params_bool_and_list(monkeypatch):
monkeypatch.setenv("MINERU_API_MODE", "official")
inner, errs = render_engine_params("mineru", {"page_range": "1-3,5"})
assert errs == [] and inner == "page_range=1-3,page_range=5"
inner, errs = render_engine_params("docling", {"force_ocr": False})
assert errs == [] and inner == "force_ocr=false"
def test_normalize_parser_engine_strips_params():
assert normalize_parser_engine("mineru(page_range=1-3)") == "mineru"
assert normalize_parser_engine("MINERU(page_range=1-3,language=en)") == "mineru"
assert normalize_parser_engine("mineru") == "mineru"
def test_decode_malformed_reports_errors():
_e, _p, errors = decode_parse_engine("mineru(page_range=")
assert errors and "unbalanced" in errors[0]
_e, _p, errors = decode_parse_engine("mineru(badkey=1)")
assert errors and "unknown parameter" in errors[0]
# --------------------------------------------------------------------------- #
# resolve_parser_directives — filename hint + rule + overlay + drop
# --------------------------------------------------------------------------- #
def test_resolve_filename_engine_params(monkeypatch):
monkeypatch.delenv("MINERU_API_MODE", raising=False) # local single segment
d = resolve_parser_directives(
"paper.[mineru(page_range=1-3,language=en)].pdf",
parser_rules="",
require_external_endpoint=False,
)
assert d.engine == "mineru"
assert d.engine_params == {"page_range": "1-3", "language": "en"}
# selector stays a pure (here empty) string
assert "(" not in d.process_options
def test_resolve_rule_engine_params():
d = resolve_parser_directives(
"a.pdf",
parser_rules="pdf:mineru(language=en)",
require_external_endpoint=False,
)
assert d.engine == "mineru" and d.engine_params == {"language": "en"}
def test_resolve_drop_rule_params_when_hint_engine_wins():
# Rule names mineru(language); filename hint names a different usable engine
# (docling) which wins -> mineru's engine params are dropped.
d = resolve_parser_directives(
"paper.[docling(force_ocr=true)].pdf",
parser_rules="pdf:mineru(language=en)",
require_external_endpoint=False,
)
assert d.engine == "docling"
assert d.engine_params == {"force_ocr": True}
@pytest.mark.parametrize(
"name",
[
"x.[mineru(language=)].pdf", # empty value
"x.[mineru(local_parse_method=bad)].pdf", # bad enum
"x.[docling(force_ocr=maybe)].pdf", # bad bool
"x.[legacy(page_range=1)].txt", # engine declares no params
"x.[mineru(page_range=1-3,5)].pdf", # env-style comma list
],
)
def test_upload_validation_raises_on_bad_engine_params(name):
with pytest.raises(FilenameParserHintError):
resolve_file_parser_directives(
name, parser_rules="", require_external_endpoint=False
)
# --------------------------------------------------------------------------- #
# validate_parser_routing_config (startup)
# --------------------------------------------------------------------------- #
def test_validate_rule_engine_params_ok(monkeypatch):
# mineru requires its endpoint configured before its rule validates.
monkeypatch.setenv("MINERU_API_MODE", "local")
monkeypatch.setenv("MINERU_LOCAL_ENDPOINT", "http://127.0.0.1:8000")
validate_parser_routing_config("pdf:mineru(language=en);*:legacy-R")
@pytest.mark.parametrize(
"rules,needle",
[
("pdf:legacy(page_range=1)", "does not accept parameters"),
("pdf:mineru(foo=1)", "unknown parameter 'foo'"),
("pdf:mineru(local_parse_method=bad)", "must be one of"),
("pdf:docling(force_ocr=maybe)", "must be a boolean"),
],
)
def test_validate_rule_engine_params_rejected(rules, needle):
with pytest.raises(ParserRoutingConfigError, match=needle):
validate_parser_routing_config(rules)
# --------------------------------------------------------------------------- #
# Direct-API guard via apipeline_enqueue_documents -> _parse_engine_at
# --------------------------------------------------------------------------- #
def _decode_via_parse_engine_at(raw: str):
"""Exercise the same decode+validate+re-encode that _parse_engine_at runs."""
engine, params, errs = decode_parse_engine(raw)
if errs:
raise ValueError("; ".join(errs))
return encode_parse_engine(engine, params) if params else engine
def test_direct_parse_engine_field_canonicalises(monkeypatch):
monkeypatch.setenv("MINERU_API_MODE", "official")
assert (
_decode_via_parse_engine_at("mineru(page_range=1-3,page_range=5)")
== "mineru(page_range=1-3,page_range=5)"
)
@pytest.mark.parametrize(
"raw",
[
"legacy(page_range=1)", # legacy declares no params
"mineru(badkey=1)", # unknown
"mineru(page_range=", # malformed
],
)
def test_direct_parse_engine_field_rejected(raw):
with pytest.raises(ValueError):
_decode_via_parse_engine_at(raw)
+408
View File
@@ -0,0 +1,408 @@
"""Tests for parameterised parser hints (``[-R(chunk_ts=800,chunk_ol=80)]``).
Phase 1 wires only ``chunk_token_size``/``chunk_ts`` and
``chunk_overlap_token_size``/``chunk_ol`` through the existing per-document
``chunk_options`` channel. ``parser_rules=""`` is passed explicitly so the
assertions are independent of any ambient ``LIGHTRAG_PARSER``.
"""
from __future__ import annotations
import importlib
import sys
import pytest
from lightrag.parser.param_schema import (
parse_chunk_params,
split_top_level,
take_paren_block,
)
from lightrag.parser.routing import (
FilenameParserHintError,
ParserRoutingConfigError,
canonicalize_parser_hinted_basename,
filename_parser_directives,
resolve_file_parser_directives,
resolve_parser_directives,
validate_parser_routing_config,
)
# Importing document_routes runs an argparse over sys.argv at import time;
# neutralise pytest's argv during that first import (subsequent imports hit the
# module cache and skip argparse). Mirrors tests/api/routes/test_document_routes_chunking.py.
_original_argv = sys.argv[:]
sys.argv = [sys.argv[0]]
importlib.import_module("lightrag.api.routers.document_routes")
sys.argv = _original_argv
# --------------------------------------------------------------------------- #
# param_schema: scanners
# --------------------------------------------------------------------------- #
def test_split_top_level_protects_commas_inside_parens():
# A comma inside a parameter block must not split the surrounding rules.
assert split_top_level(
"pdf:legacy-R(chunk_ts=800,chunk_ol=80);*:legacy-R", ";,"
) == ["pdf:legacy-R(chunk_ts=800,chunk_ol=80)", "*:legacy-R"]
# Legacy comma rule separator still works at depth 0.
assert split_top_level("a:native-F,b:legacy-R", ";,") == [
"a:native-F",
"b:legacy-R",
]
def test_take_paren_block():
assert take_paren_block("R(chunk_ts=800)", 1) == ("chunk_ts=800", 15)
assert take_paren_block("R", 1) == (None, 1) # no block
assert take_paren_block("R(unterminated", 1) == (None, 1) # unbalanced
# --------------------------------------------------------------------------- #
# param_schema: parse_chunk_params (alias / type / target / repeat)
# --------------------------------------------------------------------------- #
def test_parse_chunk_params_alias_normalised_to_canonical():
parsed, errors = parse_chunk_params(
"chunk_ts=800,chunk_ol=80", selector="R", label="x"
)
assert errors == []
assert parsed == {"chunk_token_size": 800, "chunk_overlap_token_size": 80}
def test_parse_chunk_params_canonical_names_accepted():
parsed, errors = parse_chunk_params(
"chunk_token_size=1500", selector="V", label="x"
)
assert errors == [] and parsed == {"chunk_token_size": 1500}
def test_parse_chunk_params_overlap_rejected_for_vector():
parsed, errors = parse_chunk_params("chunk_ol=80", selector="V", label="x")
assert parsed == {}
assert errors and "not supported for chunk strategy 'V'" in errors[0]
@pytest.mark.parametrize(
"block,expected",
[
("drop_rf=true", True),
("drop_references=true", True),
("drop_rf=false", False),
("drop_rf=yes", True),
("drop_rf=0", False),
],
)
def test_parse_chunk_params_drop_references_bool(block, expected):
parsed, errors = parse_chunk_params(block, selector="P", label="x")
assert errors == []
assert parsed == {"drop_references": expected}
def test_parse_chunk_params_drop_references_invalid_bool():
parsed, errors = parse_chunk_params("drop_rf=maybe", selector="P", label="x")
assert parsed == {}
assert errors and "must be a boolean" in errors[0]
@pytest.mark.parametrize("block", ["drop_rf", "drop_references"])
def test_parse_chunk_params_drop_references_bare_flag_is_true(block):
# A bare boolean flag is shorthand for ``=true``.
parsed, errors = parse_chunk_params(block, selector="P", label="x")
assert errors == []
assert parsed == {"drop_references": True}
@pytest.mark.parametrize("selector", ["F", "R", "V"])
def test_parse_chunk_params_drop_references_rejected_off_paragraph(selector):
parsed, errors = parse_chunk_params("drop_rf=true", selector=selector, label="x")
assert parsed == {}
assert errors and f"not supported for chunk strategy {selector!r}" in errors[0]
def test_filename_hint_drop_references_extracted():
d = resolve_parser_directives("notes.[-P(drop_rf=true)].md", parser_rules="")
assert d.process_options == "P" # pure selector, no params
assert d.chunk_params == {"P": {"drop_references": True}}
def test_filename_hint_drop_references_bare_flag():
# ``P(drop_rf)`` with no value is shorthand for ``drop_rf=true``.
d = resolve_parser_directives("notes.[-P(drop_rf)].md", parser_rules="")
assert d.process_options == "P"
assert d.chunk_params == {"P": {"drop_references": True}}
@pytest.mark.parametrize(
"block,needle",
[
("chunk_ts=abc", "must be an integer"),
("chunk_ts=0", "must be >= 1"),
("foo=1", "unknown parameter 'foo'"),
("chunk_ts=1,chunk_ts=2", "may not be repeated"),
("correct_tl", "unknown parameter 'correct_tl'"),
# A non-boolean parameter cannot be written bare (only flags can).
("chunk_ts", "must be written as 'key=value'"),
("", "empty parameter"),
# Cross-field invariant: explicit overlap >= size is rejected here, so
# both rule and filename-hint validation reject it (not just enqueue).
("chunk_ts=50,chunk_ol=100", "must be < chunk_token_size"),
("chunk_ts=100,chunk_ol=100", "must be < chunk_token_size"),
],
)
def test_parse_chunk_params_errors(block, needle):
_parsed, errors = parse_chunk_params(block, selector="F", label="x")
assert errors and any(needle in e for e in errors)
# --------------------------------------------------------------------------- #
# routing: filename hint params
# --------------------------------------------------------------------------- #
def test_filename_hint_selector_stays_pure_and_params_extracted():
d = resolve_parser_directives(
"notes.[-R(chunk_ts=800,chunk_ol=80)].md", parser_rules=""
)
assert d.engine == "legacy"
assert d.process_options == "R" # pure selector, no params
assert d.chunk_params == {
"R": {"chunk_token_size": 800, "chunk_overlap_token_size": 80}
}
assert d.engine_params == {}
def test_back_compat_two_tuple_and_directives_unaffected_by_params():
# The legacy 2-tuple accessor must keep returning (engine, pure-selector).
assert resolve_file_parser_directives(
"notes.[-R(chunk_ts=800)].md", parser_rules=""
) == ("legacy", "R")
assert filename_parser_directives("notes.[-R(chunk_ts=800)].md") == (None, "R")
# A param-free hint is completely unchanged.
assert filename_parser_directives("paper.[native-iet].docx") == ("native", "iet")
def test_canonicalize_strips_parameterised_hint():
assert (
canonicalize_parser_hinted_basename("notes.[-R(chunk_ts=800,chunk_ol=80)].md")
== "notes.md"
)
def test_engine_params_rejected_for_engine_that_declares_none():
# Chunk params on a chunk char still work...
d = resolve_parser_directives("doc.[native-F(chunk_ts=900)].docx", parser_rules="")
assert d.chunk_params == {"F": {"chunk_token_size": 900}}
# ...but native declares no engine params, so an engine-level block on it is
# a hard error on ingestion (Phase 2: only mineru/docling accept params).
with pytest.raises(FilenameParserHintError, match="does not accept parameters"):
resolve_file_parser_directives(
"doc.[native(do_ocr=true)-F].docx", parser_rules=""
)
@pytest.mark.parametrize(
"name",
[
"x.[-V(chunk_ol=80)].md", # overlap invalid for V
"x.[-R(chunk_ts=abc)].md", # non-integer
"x.[native-R(foo=1)].md", # unknown parameter
"x.[-R(chunk_ts=50,chunk_ol=100)].md", # explicit overlap >= size
],
)
def test_upload_validation_raises_on_bad_filename_params(name):
with pytest.raises(FilenameParserHintError):
resolve_file_parser_directives(name, parser_rules="")
# --------------------------------------------------------------------------- #
# routing: LIGHTRAG_PARSER rule params + overlay merge
# --------------------------------------------------------------------------- #
def test_rule_params_with_semicolon_separator_and_comma_in_parens():
d = resolve_parser_directives(
"a.md", parser_rules="md:legacy-R(chunk_ts=800,chunk_ol=80);*:legacy-R"
)
assert d.engine == "legacy"
assert d.chunk_params == {
"R": {"chunk_token_size": 800, "chunk_overlap_token_size": 80}
}
def test_rule_params_with_comma_separator_and_comma_in_parens():
# Rule splitting is parenthesis-aware: a comma inside a parameter block is
# NOT a rule separator, so ',' still separates rules even when a rule carries
# parameters (the docs recommend ';' but ',' must keep working).
assert split_top_level(
"md:legacy-R(chunk_ts=800,chunk_ol=80),*:legacy-R", ";,"
) == ["md:legacy-R(chunk_ts=800,chunk_ol=80)", "*:legacy-R"]
d = resolve_parser_directives(
"a.md", parser_rules="md:legacy-R(chunk_ts=800,chunk_ol=80),*:legacy-R"
)
assert d.engine == "legacy"
assert d.chunk_params == {
"R": {"chunk_token_size": 800, "chunk_overlap_token_size": 80}
}
def test_overlay_rule_then_filename_hint_wins_per_key():
# Design worked example: rule supplies chunk_ol on P, the filename hint
# supplies chunk_ts on P; the surviving selector is the filename's "P".
d = resolve_parser_directives(
"paper.[-P(chunk_ts=3000)].pdf", parser_rules="pdf:legacy-iteP(chunk_ol=100)"
)
assert d.process_options == "P" # filename options wholesale-override rule
assert d.chunk_params["P"] == {
"chunk_overlap_token_size": 100, # inherited from the rule
"chunk_token_size": 3000, # from the filename hint
}
def test_filename_hint_key_overrides_rule_key():
d = resolve_parser_directives(
"paper.[-R(chunk_ts=500)].md",
parser_rules="md:legacy-R(chunk_ts=900,chunk_ol=50)",
)
assert d.chunk_params["R"] == {
"chunk_token_size": 500, # filename wins over the rule's 900
"chunk_overlap_token_size": 50, # inherited from the rule
}
# --------------------------------------------------------------------------- #
# routing: startup validation
# --------------------------------------------------------------------------- #
def test_validate_parser_routing_config_accepts_good_params():
validate_parser_routing_config("pdf:legacy-R(chunk_ts=800,chunk_ol=80);*:legacy-R")
@pytest.mark.parametrize(
"rules,needle",
[
("pdf:legacy-V(chunk_ol=80)", "not supported for chunk strategy 'V'"),
("pdf:legacy-R(chunk_ts=abc)", "must be an integer"),
("pdf:legacy-R(foo=1)", "unknown parameter 'foo'"),
# Cross-field invariant: explicit overlap >= size is rejected at startup
# so a global rule cannot make every matching upload fail at enqueue.
("pdf:legacy-R(chunk_ts=50,chunk_ol=100)", "must be < chunk_token_size"),
("pdf:legacy-R(chunk_ts=100,chunk_ol=100)", "must be < chunk_token_size"),
],
)
def test_validate_parser_routing_config_rejects_bad_params(rules, needle):
with pytest.raises(ParserRoutingConfigError, match=needle):
validate_parser_routing_config(rules)
def test_validate_parser_routing_config_allows_one_sided_overlap():
# Only overlap is explicit: the effective size depends on env / addon_params
# and is unknown at config time, so startup accepts it and defers to the
# upload-time effective-overlap check on the resolved snapshot.
validate_parser_routing_config("pdf:legacy-R(chunk_ol=100);*:legacy-R")
def test_legacy_comma_rules_without_params_still_valid():
# Pure back-compat: legacy comma-separated, param-free rules unchanged.
validate_parser_routing_config("docx:native-iteP,pdf:legacy-R,*:legacy-R")
# --------------------------------------------------------------------------- #
# upload path: filename-hint params reach the enqueued chunk_options
# --------------------------------------------------------------------------- #
def _recording_rag():
import asyncio # noqa: F401 - imported for clarity at call sites
from types import SimpleNamespace
captured: dict = {}
async def fake_enqueue(content, **kwargs):
captured.update(kwargs)
return "track-xyz" # non-None -> success path
async def fake_error(files, track_id):
captured["error_files"] = files
rag = SimpleNamespace(
addon_params={},
apipeline_enqueue_documents=fake_enqueue,
apipeline_enqueue_error_documents=fake_error,
)
return rag, captured
def test_upload_path_applies_hint_chunk_params(tmp_path):
import asyncio
from lightrag.api.routers.document_routes import pipeline_enqueue_file
rag, captured = _recording_rag()
f = tmp_path / "notes.[-R(chunk_ts=800,chunk_ol=80)].md"
f.write_text("hello world", encoding="utf-8")
ok, _track = asyncio.run(pipeline_enqueue_file(rag, f))
assert ok is True
assert "error_files" not in captured
assert captured["parse_engine"] == "legacy"
assert captured["process_options"] == "R" # pure selector forwarded
sub = captured["chunk_options"]["recursive_character"]
assert sub["chunk_token_size"] == 800
assert sub["chunk_overlap_token_size"] == 80
def test_upload_path_without_params_omits_chunk_options(tmp_path):
import asyncio
from lightrag.api.routers.document_routes import pipeline_enqueue_file
rag, captured = _recording_rag()
f = tmp_path / "plain.[native-R].md"
f.write_text("hello world", encoding="utf-8")
ok, _track = asyncio.run(pipeline_enqueue_file(rag, f))
assert ok is True
# Legacy behaviour: no per-file chunk_options when the hint has no params.
assert "chunk_options" not in captured
def test_upload_path_rejects_bad_hint_params(tmp_path):
import asyncio
from lightrag.api.routers.document_routes import pipeline_enqueue_file
rag, captured = _recording_rag()
f = tmp_path / "bad.[-V(chunk_ol=80)].md"
f.write_text("hello world", encoding="utf-8")
ok, _track = asyncio.run(pipeline_enqueue_file(rag, f))
assert ok is False
assert captured["error_files"] # an error document was enqueued
assert "chunk_options" not in captured
def test_upload_path_rejects_effective_overlap_violation(tmp_path):
import asyncio
from lightrag.api.routers.document_routes import pipeline_enqueue_file
rag, captured = _recording_rag()
# An explicit overlap (100) >= size (50) pair is rejected up front by
# filename-hint validation (uniform with rule validation), so the file
# never enqueues and the overlap message is surfaced in the error document.
f = tmp_path / "tiny.[-R(chunk_ts=50,chunk_ol=100)].md"
f.write_text("hello world", encoding="utf-8")
ok, _track = asyncio.run(pipeline_enqueue_file(rag, f))
assert ok is False
assert captured["error_files"]
assert "must be < chunk_token_size" in captured["error_files"][0]["original_error"]
+121
View File
@@ -0,0 +1,121 @@
"""Unit tests for the legacy engine adapter (lightrag.parser.legacy.parser).
Covers the worker-stage extraction contract directly: success path
(persist + archive + RAW ParseResult), the unsupported-suffix gate, the
whitespace-only extraction guard (scanned-PDF case) and the
``PDF_DECRYPT_PASSWORD`` plumbing.
"""
import pytest
import lightrag.pipeline as _pipeline
from lightrag.constants import FULL_DOCS_FORMAT_RAW
from lightrag.parser.base import ParseContext
from lightrag.parser.legacy.extractors import LegacyExtractionError
from lightrag.parser.legacy.parser import LegacyParser
pytestmark = pytest.mark.offline
class _FakeRag:
def __init__(self):
self.persisted = []
async def _persist_parsed_full_docs(self, doc_id, payload):
self.persisted.append((doc_id, payload))
def _resolve_source_file_for_parser(
self, file_path, *, source_file=None, parser_engine=None
):
return file_path
@pytest.fixture
def archived(monkeypatch):
"""Record archive calls instead of moving files into __parsed__."""
calls = []
async def _record(source_path):
calls.append(source_path)
return source_path
monkeypatch.setattr(_pipeline, "archive_docx_source_after_full_docs_sync", _record)
return calls
def _ctx(rag, source_path):
return ParseContext(rag, "doc-legacy", str(source_path), {})
async def test_legacy_parse_txt_persists_and_archives(tmp_path, archived):
source = tmp_path / "notes.txt"
source.write_text("plain text body", encoding="utf-8")
rag = _FakeRag()
result = await LegacyParser().parse(_ctx(rag, source))
assert result.parse_format == FULL_DOCS_FORMAT_RAW
assert result.content == "plain text body"
assert result.parse_engine == "legacy"
assert result.blocks_path == ""
doc_id, payload = rag.persisted[0]
assert doc_id == "doc-legacy"
assert payload["content"] == "plain text body"
assert payload["parse_format"] == FULL_DOCS_FORMAT_RAW
assert archived == [str(source)]
# to_dict stays byte-compatible: no spurious skip/warning keys.
assert "parse_stage_skipped" not in result.to_dict()
async def test_legacy_parse_unsupported_suffix_raises(tmp_path, archived):
source = tmp_path / "image.xyz"
source.write_bytes(b"not parseable")
rag = _FakeRag()
with pytest.raises(ValueError, match=r"does not support \.xyz"):
await LegacyParser().parse(_ctx(rag, source))
assert rag.persisted == []
assert archived == []
async def test_legacy_parse_whitespace_only_extraction_raises(
tmp_path, archived, monkeypatch
):
# A scanned PDF (no text layer) extracts to pure whitespace; the parser
# must fail the doc instead of persisting an empty document.
monkeypatch.setattr(
"lightrag.parser.legacy.extractors.extract_text",
lambda file_bytes, suffix, *, pdf_password=None: "\n \n\t\n",
)
source = tmp_path / "scanned.pdf"
source.write_bytes(b"%PDF-fake")
rag = _FakeRag()
with pytest.raises(LegacyExtractionError, match="no usable text"):
await LegacyParser().parse(_ctx(rag, source))
assert rag.persisted == []
assert archived == []
async def test_legacy_parse_passes_pdf_password_from_env(
tmp_path, archived, monkeypatch
):
seen = {}
def _capture(file_bytes, suffix, *, pdf_password=None):
seen["suffix"] = suffix
seen["pdf_password"] = pdf_password
return "decrypted text"
monkeypatch.setattr("lightrag.parser.legacy.extractors.extract_text", _capture)
monkeypatch.setenv("PDF_DECRYPT_PASSWORD", "s3cret")
source = tmp_path / "locked.pdf"
source.write_bytes(b"%PDF-fake")
rag = _FakeRag()
result = await LegacyParser().parse(_ctx(rag, source))
assert seen == {"suffix": "pdf", "pdf_password": "s3cret"}
assert result.content == "decrypted text"
+72
View File
@@ -0,0 +1,72 @@
"""Unit tests for the shared markdown heading renderer."""
from __future__ import annotations
import pytest
from lightrag.parser._markdown import (
render_heading_line,
strip_heading_markdown_prefix,
)
@pytest.mark.offline
@pytest.mark.parametrize(
("level", "text", "expected"),
[
(1, "Intro", "# Intro"),
(2, "Sub", "## Sub"),
(6, "Deep", "###### Deep"),
# level >= 7 is clamped to six "#".
(7, "Deeper", "###### Deeper"),
(99, "Deepest", "###### Deepest"),
# level < 1 falls back to a single "#".
(0, "Zero", "# Zero"),
(-3, "Neg", "# Neg"),
],
)
def test_render_heading_line_prefix_and_cap(level, text, expected) -> None:
assert render_heading_line(level, text) == expected
@pytest.mark.offline
@pytest.mark.parametrize(
"already",
["# Foo", "## Bar", "###### Six"],
)
def test_render_heading_line_keeps_existing_markdown(already) -> None:
# Already a markdown heading (1-6 "#" + space) → returned unchanged,
# regardless of the requested level (no double-prefixing).
assert render_heading_line(3, already) == already
@pytest.mark.offline
@pytest.mark.parametrize(
("text", "expected"),
[
# 7 "#" is NOT a valid markdown heading → still gets prefixed.
("####### Seven", "# ####### Seven"),
# "#" with no following space → not a heading marker → prefixed.
("#NoSpace", "# #NoSpace"),
],
)
def test_render_heading_line_non_heading_hash_is_prefixed(text, expected) -> None:
assert render_heading_line(1, text) == expected
@pytest.mark.offline
@pytest.mark.parametrize(
("text", "expected"),
[
("# Foo", "Foo"),
("## Bar", "Bar"),
("###### Six", "Six"),
# The whole "#" run plus all following spaces are stripped — no
# leading space leaks into the cleaned metadata.
("# Extra space", "Extra space"),
("#NoSpace", "#NoSpace"),
("####### Seven", "####### Seven"),
],
)
def test_strip_heading_markdown_prefix(text, expected) -> None:
assert strip_heading_markdown_prefix(text) == expected
@@ -0,0 +1,430 @@
"""End-to-end test: native docx → LightRAG Document → stable cache key.
The original bug this guards against: ``parse_native`` used to write a
runtime-stamped structured parser payload into ``full_docs.content``, so
re-parsing the same docx produced different
chunk-0 content and therefore different LLM cache keys.
After the fix, ``parse_native`` writes ``.blocks.jsonl`` + sidecars and
``full_docs`` is in LIGHTRAG format. ``_load_lightrag_document_content``
skips the ``meta`` line (which contains ``parse_time``) and concatenates
only ``"type": "content"`` rows, so re-parsing must yield byte-identical
``merged_text`` and stable downstream chunk-0 content.
"""
import asyncio
import json
from pathlib import Path
import pytest
from lightrag import LightRAG
from lightrag.constants import (
FULL_DOCS_FORMAT_PENDING_PARSE,
PARSED_DIR_NAME,
)
from lightrag.parser.base import ParseContext
from lightrag.parser.registry import get_parser
from lightrag.utils import Tokenizer, TokenizerInterface, compute_args_hash
async def _parse_via_registry(rag, engine, doc_id, file_path, content_data):
"""Drive a parser the way the pipeline worker does (registry dispatch)."""
result = await get_parser(engine).parse(
ParseContext(rag, doc_id, file_path, content_data)
)
return result.to_dict()
def _block(content, *, heading="", level=0, parent=None, uuid="p1"):
"""Build a synthetic block dict matching extract_docx_blocks output."""
return {
"uuid": uuid,
"uuid_end": uuid,
"heading": heading,
"content": content,
"type": "text",
"parent_headings": list(parent or []),
"level": level,
"table_chunk_role": "none",
}
class _MiniFullDocs:
def __init__(self):
self.data = {}
async def upsert(self, payload):
self.data.update(payload)
async def get_by_id(self, doc_id):
return self.data.get(doc_id)
async def index_done_callback(self):
return None
class _MiniDocStatus:
async def get_by_id(self, doc_id):
return None
async def upsert(self, data):
return None
class _CharTokenizer(TokenizerInterface):
def encode(self, content: str):
return [ord(ch) for ch in content]
def decode(self, tokens):
return "".join(chr(t) for t in tokens)
class _MiniRag:
"""Just enough surface for parse_native + parser/docx adapter."""
_persist_parsed_full_docs = LightRAG._persist_parsed_full_docs
def __init__(self, working_dir):
self.working_dir = str(working_dir)
self.full_docs = _MiniFullDocs()
self.doc_status = _MiniDocStatus()
self.tokenizer = Tokenizer(model_name="char", tokenizer=_CharTokenizer())
def _resolve_source_file_for_parser(self, file_path):
return file_path
@pytest.mark.offline
def test_native_lightrag_path_produces_stable_merged_text(tmp_path, monkeypatch):
"""Re-parsing the same docx must yield byte-identical merged_text and
therefore identical chunk_args_hash on chunk-0."""
async def _run():
input_dir = tmp_path / "input"
input_dir.mkdir()
monkeypatch.setenv("INPUT_DIR", str(input_dir))
source_path = input_dir / "stable.docx"
source_path.write_bytes(b"fake docx bytes")
# Stub extract_docx_blocks at the adapter so the upstream DOCX
# parser is never invoked. The adapter still does all the
# LightRAG-specific writing — that is what we want under test.
stable_blocks = [
_block(
"Title\nFirst paragraph body.\nSecond paragraph body.",
heading="Title",
level=1,
),
]
def _stub_extract(file_path, drawing_context=None, **kwargs):
return [dict(b) for b in stable_blocks]
monkeypatch.setattr(
"lightrag.parser.docx.parse_document.extract_docx_blocks",
_stub_extract,
)
rag = _MiniRag(tmp_path / "work")
# ---- First parse ----
# parse_native archives the source after writing, so re-create it
# before the second parse for a fair comparison.
result1 = await _parse_via_registry(
rag,
"native",
"doc-stable",
str(source_path),
{"parse_format": FULL_DOCS_FORMAT_PENDING_PARSE, "content": ""},
)
merged1 = result1["content"]
assert merged1, "first parse produced empty merged_text"
# ---- Second parse ----
# Restore the source file (archive moved it), reset the in-memory
# full_docs row, and remove the parsed_dir so the writer rewrites
# both meta (with a fresh parse_time) and content lines.
source_path.write_bytes(b"fake docx bytes")
rag.full_docs.data.clear()
parsed_artifact_dir = input_dir / PARSED_DIR_NAME / f"{source_path.name}.parsed"
if parsed_artifact_dir.exists():
import shutil
shutil.rmtree(parsed_artifact_dir)
result2 = await _parse_via_registry(
rag,
"native",
"doc-stable",
str(source_path),
{"parse_format": FULL_DOCS_FORMAT_PENDING_PARSE, "content": ""},
)
merged2 = result2["content"]
# Core invariant: merged_text byte-identical across runs even
# though parse_time in the .blocks.jsonl meta line differs.
assert merged1 == merged2
# And: a hash computed over a chunk-0 derived from merged_text
# must also be identical — that is what powers LLM cache hits.
prompt_template = "EXTRACT_PROMPT::{text}"
chunk0_a = prompt_template.format(text=merged1[:200])
chunk0_b = prompt_template.format(text=merged2[:200])
assert chunk0_a == chunk0_b
assert compute_args_hash(chunk0_a) == compute_args_hash(chunk0_b)
# And: full_docs.content uses the {{LRdoc}} marker plus a leading
# summary derived from merged_text (not the legacy placeholder).
record = rag.full_docs.data["doc-stable"]
assert record["parse_format"] == "lightrag"
assert record["content"].startswith("{{LRdoc}}")
assert merged1[:40] in record["content"]
asyncio.run(_run())
@pytest.mark.offline
def test_native_lightrag_path_writes_blocks_jsonl_and_skips_meta_on_load(
tmp_path, monkeypatch
):
"""Sanity check: ``_load_lightrag_document_content`` must skip the
meta line (where the runtime ``parse_time`` lives) and only return
body content. This is what lets re-parsing produce stable text."""
async def _run():
input_dir = tmp_path / "input"
input_dir.mkdir()
monkeypatch.setenv("INPUT_DIR", str(input_dir))
source_path = input_dir / "skipmeta.docx"
source_path.write_bytes(b"fake")
def _stub_extract(file_path, drawing_context=None, **kwargs):
return [_block("the body")]
monkeypatch.setattr(
"lightrag.parser.docx.parse_document.extract_docx_blocks",
_stub_extract,
)
rag = _MiniRag(tmp_path / "work")
result = await _parse_via_registry(
rag,
"native",
"doc-skip",
str(source_path),
{"parse_format": FULL_DOCS_FORMAT_PENDING_PARSE, "content": ""},
)
# The .blocks.jsonl on disk DOES contain "parse_time" inside the
# meta line; the merged_text returned by parse_native MUST NOT.
blocks_path = result["blocks_path"]
on_disk = open(blocks_path, "r", encoding="utf-8").read()
assert "parse_time" in on_disk
assert "parse_time" not in result["content"]
assert result["content"].strip() == "the body"
asyncio.run(_run())
@pytest.mark.offline
def test_native_lightrag_path_leaves_unknown_table_caption_empty(tmp_path, monkeypatch):
"""The native DOCX parser does not infer table titles, so its table
sidecar must not synthesize captions like ``表1``.
"""
async def _run():
input_dir = tmp_path / "input"
input_dir.mkdir()
monkeypatch.setenv("INPUT_DIR", str(input_dir))
source_path = input_dir / "table.docx"
source_path.write_bytes(b"fake")
def _stub_extract(file_path, drawing_context=None, **kwargs):
return [_block('before\n<table>[["A"]]</table>\nafter')]
monkeypatch.setattr(
"lightrag.parser.docx.parse_document.extract_docx_blocks",
_stub_extract,
)
rag = _MiniRag(tmp_path / "work")
result = await _parse_via_registry(
rag,
"native",
"doc-table",
str(source_path),
{"parse_format": FULL_DOCS_FORMAT_PENDING_PARSE, "content": ""},
)
blocks_path = Path(result["blocks_path"])
lines = blocks_path.read_text(encoding="utf-8").splitlines()
block = json.loads(lines[1])
assert "caption=" not in block["content"]
assert "表1" not in block["content"]
tables_path = blocks_path.with_suffix("").with_suffix(".tables.json")
tables = json.loads(tables_path.read_text(encoding="utf-8"))
table_entry = tables["tables"]["tb-table-0001"]
assert table_entry["caption"] == ""
# Surrounding is now backfilled at analyze_multimodal entry, not in
# parse_native — invoke the same routine directly to mirror that.
from lightrag.multimodal_context import enrich_sidecars_with_surrounding
enrich_sidecars_with_surrounding(
blocks_path=str(blocks_path),
enabled_modalities={"tables"},
tokenizer=rag.tokenizer,
)
tables = json.loads(tables_path.read_text(encoding="utf-8"))
table_entry = tables["tables"]["tb-table-0001"]
assert table_entry["surrounding"] == {
"leading": "before\n",
"trailing": "\nafter",
}
asyncio.run(_run())
@pytest.mark.offline
def test_analyze_entrypoint_backfills_surrounding_for_all_sidecars(
tmp_path, monkeypatch
):
"""Surrounding is backfilled at analyze_multimodal entry, covering native
parse output as well as any other sidecar-producing engine."""
async def _run():
input_dir = tmp_path / "input"
input_dir.mkdir()
monkeypatch.setenv("INPUT_DIR", str(input_dir))
source_path = input_dir / "all_modalities.docx"
source_path.write_bytes(b"fake")
def _stub_extract(file_path, drawing_context=None, **kwargs):
assert drawing_context is not None
assert drawing_context.export_dir_path is not None
(drawing_context.export_dir_path / "pic.png").write_bytes(b"PNG")
return [
_block(
'alpha <drawing id="1" format="png" '
'path="all_modalities.blocks.assets/pic.png" /> beta\n'
'<table>[["A"]]</table> gamma\n'
"<equation>E=mc^2</equation>\n"
"delta"
)
]
monkeypatch.setattr(
"lightrag.parser.docx.parse_document.extract_docx_blocks",
_stub_extract,
)
rag = _MiniRag(tmp_path / "work")
result = await _parse_via_registry(
rag,
"native",
"doc-mm",
str(source_path),
{"parse_format": FULL_DOCS_FORMAT_PENDING_PARSE, "content": ""},
)
blocks_path = Path(result["blocks_path"])
base = str(blocks_path)[: -len(".blocks.jsonl")]
# Parse-time sidecars must NOT contain surrounding — that field is
# now produced at analyze_multimodal entry.
for root in ("drawings", "tables", "equations"):
payload = json.loads(Path(base + f".{root}.json").read_text("utf-8"))
for item in payload[root].values():
assert "surrounding" not in item
# Now invoke the same routine analyze_multimodal calls and verify
# all modalities get populated.
from lightrag.multimodal_context import enrich_sidecars_with_surrounding
enrich_sidecars_with_surrounding(
blocks_path=str(blocks_path),
enabled_modalities={"drawings", "tables", "equations"},
tokenizer=rag.tokenizer,
)
for root in ("drawings", "tables", "equations"):
payload = json.loads(Path(base + f".{root}.json").read_text("utf-8"))
items = payload[root]
assert items
for item in items.values():
assert "surrounding" in item
assert set(item["surrounding"]) == {"leading", "trailing"}
asyncio.run(_run())
@pytest.mark.offline
def test_native_lightrag_path_writes_image_assets_to_blocks_assets_dir(
tmp_path, monkeypatch
):
"""Native parsing must drop image bytes into ``<base>.blocks.assets/``
after the adapter creates the parsed dir (which it wipes at the start),
and the drawings sidecar must reference the rewritten ids.
"""
from pathlib import Path
async def _run():
input_dir = tmp_path / "input"
input_dir.mkdir()
monkeypatch.setenv("INPUT_DIR", str(input_dir))
source_path = input_dir / "with_pics.docx"
source_path.write_bytes(b"fake")
def _stub_extract(file_path, drawing_context=None, **kwargs):
# The adapter already created the asset dir before calling us;
# write the fake image bytes there as a side-effect, then return
# a block whose content references that asset via <drawing .../>.
assert drawing_context is not None
assert drawing_context.export_dir_path is not None
(drawing_context.export_dir_path / "pic.png").write_bytes(b"PNG-BYTES")
return [
_block(
"intro\n"
'<drawing id="1" name="pic" format="png" '
'path="with_pics.blocks.assets/pic.png" />\n'
"outro",
heading="intro",
level=1,
),
]
monkeypatch.setattr(
"lightrag.parser.docx.parse_document.extract_docx_blocks",
_stub_extract,
)
rag = _MiniRag(tmp_path / "work")
result = await _parse_via_registry(
rag,
"native",
"doc-pic",
str(source_path),
{"parse_format": FULL_DOCS_FORMAT_PENDING_PARSE, "content": ""},
)
blocks_path = Path(result["blocks_path"])
parsed_dir = blocks_path.parent
asset_dir = parsed_dir / "with_pics.blocks.assets"
# Asset dir must exist alongside .blocks.jsonl and survive the
# adapter's parsed_dir cleanup step.
assert asset_dir.is_dir(), (
f"asset dir not created at {asset_dir}; parsed_dir contents: "
f"{list(parsed_dir.iterdir())}"
)
assert (asset_dir / "pic.png").read_bytes() == b"PNG-BYTES"
# And drawings.json sidecar should also be there since the block
# contained a <drawing .../> markup the adapter had to record.
assert (parsed_dir / "with_pics.drawings.json").is_file()
asyncio.run(_run())
+269
View File
@@ -0,0 +1,269 @@
"""Tests for the unified parser debug CLI (``lightrag/parser/cli.py``).
The CLI behaviour under test is engine-agnostic: argument parsing, the
flat sidecar layout (no ``__parsed__/`` middle layer), the lenient raw
cache strategy (non-empty raw_dir reused without manifest checks), and
the no-archive guarantee on the source file.
We drive these checks via the **docling** engine path because docling's
raw bundle is the easiest to construct as static fixture (a single JSON
file) with zero external service or fixture-file dependency. The other
two engines exercise the same CLI code path:
- ``native`` would need a real ``.docx`` byte stream end-to-end (golden
fixtures live under ``tests/parser/docx/golden/`` and have
their own coverage via ``test_native_docx_golden.py``).
- ``mineru`` would need to mock ``MinerURawClient.download_into`` on
the cache-miss path, or seed a mineru raw bundle layout (more files
than docling's). Cache-hit reuses the same CLI orchestration as
docling, so coverage here implicitly validates mineru's CLI wiring
too.
"""
from __future__ import annotations
import json
import subprocess
import sys
from pathlib import Path
from typing import Any
import pytest
from lightrag.parser.cli import main
def _make_main_json(
*,
origin_filename: str = "demo.pdf",
with_table: bool = False,
) -> dict[str, Any]:
payload: dict[str, Any] = {
"schema_name": "DoclingDocument",
"version": "1.10.0",
"origin": {"filename": origin_filename, "mimetype": "application/pdf"},
"body": {
"self_ref": "#/body",
"children": [{"$ref": "#/texts/0"}, {"$ref": "#/texts/1"}],
"content_layer": "body",
"label": "unspecified",
},
"groups": [],
"texts": [
{
"self_ref": "#/texts/0",
"label": "title",
"text": "Hello Title",
"orig": "Hello Title",
"content_layer": "body",
"prov": [],
},
{
"self_ref": "#/texts/1",
"label": "text",
"text": "Body line.",
"orig": "Body line.",
"content_layer": "body",
"prov": [],
},
],
"pictures": [],
"tables": [],
"key_value_items": [],
"form_items": [],
}
if with_table:
payload["body"]["children"].append({"$ref": "#/tables/0"})
payload["tables"].append(
{
"self_ref": "#/tables/0",
"label": "table",
"content_layer": "body",
"data": {
"num_rows": 1,
"num_cols": 2,
"grid": [[{"text": "A"}, {"text": "B"}]],
},
"prov": [],
}
)
return payload
def _seed_raw_dir(raw_dir: Path, *, with_table: bool = False) -> None:
raw_dir.mkdir(parents=True, exist_ok=True)
(raw_dir / "demo.json").write_text(
json.dumps(_make_main_json(with_table=with_table)),
encoding="utf-8",
)
def _read_meta(blocks_path: Path) -> dict[str, Any]:
return json.loads(blocks_path.read_text(encoding="utf-8").splitlines()[0])
@pytest.fixture(autouse=True)
def _clean_env(monkeypatch: pytest.MonkeyPatch) -> None:
for name in (
"DOCLING_BBOX_ATTRIBUTES",
"DOCLING_ENGINE_VERSION",
"LIGHTRAG_FORCE_REPARSE_DOCLING",
):
monkeypatch.delenv(name, raising=False)
def test_cli_writes_sidecar_from_existing_raw_dir(tmp_path: Path) -> None:
source = tmp_path / "demo.pdf"
source.write_bytes(b"%PDF-1.4\n") # never read; presence is the only check
_seed_raw_dir(tmp_path / "demo.pdf.docling_raw", with_table=True)
rc = main([str(source), "--engine", "docling"])
assert rc == 0
parsed_dir = tmp_path / "demo.pdf.parsed"
blocks_path = parsed_dir / "demo.blocks.jsonl"
assert blocks_path.is_file()
assert (parsed_dir / "demo.tables.json").is_file()
meta = _read_meta(blocks_path)
assert meta["parse_engine"] == "docling"
assert meta["document_name"] == "demo.pdf"
assert meta["table_file"] is True
# Source file stays where it was — the CLI mocks the archive step.
assert source.is_file()
def test_cli_doc_id_default_is_stable_across_runs(tmp_path: Path) -> None:
source = tmp_path / "demo.pdf"
source.write_bytes(b"%PDF-1.4\n")
_seed_raw_dir(tmp_path / "demo.pdf.docling_raw")
blocks_path = tmp_path / "demo.pdf.parsed" / "demo.blocks.jsonl"
assert main([str(source), "--engine", "docling"]) == 0
first_lines = blocks_path.read_text(encoding="utf-8").splitlines()
first_meta = json.loads(first_lines[0])
first_block_ids = [json.loads(line)["blockid"] for line in first_lines[1:]]
assert main([str(source), "--engine", "docling"]) == 0
second_lines = blocks_path.read_text(encoding="utf-8").splitlines()
second_meta = json.loads(second_lines[0])
second_block_ids = [json.loads(line)["blockid"] for line in second_lines[1:]]
assert first_meta["doc_id"].startswith("doc-")
assert first_meta["doc_id"] == second_meta["doc_id"]
assert first_block_ids and first_block_ids == second_block_ids
def test_cli_doc_id_override(tmp_path: Path) -> None:
source = tmp_path / "demo.pdf"
source.write_bytes(b"%PDF-1.4\n")
_seed_raw_dir(tmp_path / "demo.pdf.docling_raw", with_table=True)
override = "doc-" + "a" * 32
rc = main([str(source), "--engine", "docling", "--doc-id", override])
assert rc == 0
parsed_dir = tmp_path / "demo.pdf.parsed"
meta = _read_meta(parsed_dir / "demo.blocks.jsonl")
assert meta["doc_id"] == override
tables = json.loads((parsed_dir / "demo.tables.json").read_text(encoding="utf-8"))[
"tables"
]
assert tables
assert all(tid.startswith("tb-" + "a" * 32 + "-") for tid in tables)
def test_cli_custom_sidecar_parent_dir(tmp_path: Path) -> None:
source = tmp_path / "demo.pdf"
source.write_bytes(b"%PDF-1.4\n")
custom_parent = tmp_path / "elsewhere"
custom_parent.mkdir()
_seed_raw_dir(custom_parent / "demo.pdf.docling_raw")
rc = main([str(source), "--engine", "docling", "-o", str(custom_parent)])
assert rc == 0
assert (custom_parent / "demo.pdf.parsed" / "demo.blocks.jsonl").is_file()
# Nothing should land in the source's parent directory.
assert not (tmp_path / "demo.pdf.parsed").exists()
# Source file is preserved in place.
assert source.is_file()
def test_cli_missing_input_file_returns_error(
tmp_path: Path, capsys: pytest.CaptureFixture[str]
) -> None:
missing = tmp_path / "nope.pdf"
rc = main([str(missing), "--engine", "docling"])
assert rc == 1
err = capsys.readouterr().err
assert str(missing.resolve()) in err
def test_cli_rejects_suffix_engine_mismatch(
tmp_path: Path, capsys: pytest.CaptureFixture[str]
) -> None:
# native only accepts .docx; feeding it a .pdf should fail up-front with
# a clear error rather than crashing deep inside the IR builder.
source = tmp_path / "demo.pdf"
source.write_bytes(b"%PDF-1.4\n")
rc = main([str(source), "--engine", "native"])
assert rc == 1
err = capsys.readouterr().err
assert "native" in err
assert "pdf" in err
assert "docx" in err # supported suffix list mentions docx
def test_cli_legacy_engine_prints_raw_summary(
tmp_path: Path, capsys: pytest.CaptureFixture[str]
) -> None:
# legacy is registry-selectable and emits plain content with no sidecar
# (blocks_path=""). The CLI must summarize the content instead of trying
# to open a non-existent blocks file.
source = tmp_path / "notes.txt"
source.write_text("hello world\nsecond line\n", encoding="utf-8")
rc = main([str(source), "--engine", "legacy"])
assert rc == 0
out = capsys.readouterr().out
assert "engine : legacy" in out
assert "format : raw" in out
assert "hello world" in out
def test_cli_direct_script_native_does_not_shadow_python_docx(
tmp_path: Path,
) -> None:
from docx import Document
source = tmp_path / "demo.docx"
doc = Document()
doc.add_heading("Hello Title", level=1)
doc.add_paragraph("Body line.")
doc.save(source)
repo_root = Path(__file__).resolve().parents[2]
cli_path = repo_root / "lightrag" / "parser" / "cli.py"
proc = subprocess.run(
[
sys.executable,
str(cli_path),
str(source),
"--engine",
"native",
"--preview",
"0",
],
cwd=repo_root,
text=True,
capture_output=True,
check=False,
)
assert proc.returncode == 0, proc.stderr + proc.stdout
assert "python-docx not installed" not in proc.stderr
assert (tmp_path / "demo.docx.parsed" / "demo.blocks.jsonl").is_file()
assert source.is_file()
+132
View File
@@ -0,0 +1,132 @@
"""Unit tests for third-party parser discovery (lightrag.parser.plugins)."""
import pytest
from lightrag.parser import plugins, registry
class _FakeEntryPoint:
"""Stand-in for importlib.metadata.EntryPoint (name/value/load)."""
def __init__(self, name, value, fn):
self.name = name
self.value = value
self._fn = fn
def load(self):
if isinstance(self._fn, Exception):
raise self._fn
return self._fn
@pytest.fixture(autouse=True)
def _reset_loaded_flag(monkeypatch):
"""Each test starts from the not-yet-loaded state."""
monkeypatch.setattr(plugins, "_loaded", False)
@pytest.fixture
def third_party_cleanup():
"""Remove engines a test registered so the module-level registry stays
clean for other tests."""
names = []
yield names
for name in names:
registry._REGISTRY.pop(name, None)
def _install_eps(monkeypatch, eps):
def _fake_entry_points(*, group):
assert group == plugins.ENTRY_POINT_GROUP
return list(eps)
monkeypatch.setattr(plugins, "entry_points", _fake_entry_points)
def test_load_discovers_and_registers(monkeypatch, third_party_cleanup):
def _register():
registry.register_parser(
registry.ParserSpec(
engine_name="plug-engine",
impl="x:Y",
suffixes=frozenset({"foo"}),
queue_group="plug-engine",
concurrency=1,
)
)
_install_eps(monkeypatch, [_FakeEntryPoint("plug", "pkg.mod:register", _register)])
third_party_cleanup.append("plug-engine")
assert plugins.load_third_party_parsers() == ["plug"]
assert "plug-engine" in registry.supported_parser_engines()
def test_load_is_idempotent_per_process(monkeypatch):
calls = {"n": 0}
def _register():
calls["n"] += 1
_install_eps(monkeypatch, [_FakeEntryPoint("plug", "pkg.mod:register", _register)])
assert plugins.load_third_party_parsers() == ["plug"]
# Second call is a no-op (server lifespan + CLI may both call it).
assert plugins.load_third_party_parsers() == []
assert calls["n"] == 1
# force=True re-runs (test escape hatch).
assert plugins.load_third_party_parsers(force=True) == ["plug"]
assert calls["n"] == 2
def test_broken_plugin_is_skipped_not_fatal(monkeypatch, third_party_cleanup):
def _good_register():
registry.register_parser(
registry.ParserSpec(
engine_name="good-engine",
impl="x:Y",
suffixes=frozenset({"bar"}),
queue_group="good-engine",
concurrency=1,
)
)
_install_eps(
monkeypatch,
[
_FakeEntryPoint("broken", "bad.mod:register", ImportError("boom")),
_FakeEntryPoint("good", "ok.mod:register", _good_register),
],
)
third_party_cleanup.append("good-engine")
# The broken plugin is logged + skipped; the good one still loads.
assert plugins.load_third_party_parsers() == ["good"]
assert "good-engine" in registry.supported_parser_engines()
def test_cli_sees_entry_point_engine(monkeypatch, third_party_cleanup):
"""End-to-end: a plugin-registered engine becomes a valid --engine choice
in the debug CLI (which calls load_third_party_parsers in main)."""
from lightrag.parser import cli
def _register():
registry.register_parser(
registry.ParserSpec(
engine_name="cli-plug-engine",
impl="x:Y",
suffixes=frozenset({"baz"}),
queue_group="cli-plug-engine",
concurrency=1,
)
)
_install_eps(
monkeypatch, [_FakeEntryPoint("cliplug", "pkg.mod:register", _register)]
)
third_party_cleanup.append("cli-plug-engine")
# Missing input file -> exit 1 AFTER argparse accepted the engine choice
# (an unknown engine would exit 2 from argparse instead).
rc = cli.main(["/nonexistent/x.baz", "--engine", "cli-plug-engine"])
assert rc == 1
+77
View File
@@ -0,0 +1,77 @@
"""Unit tests for the parser registry (lightrag.parser.registry)."""
import subprocess
import sys
from lightrag.parser import registry
def test_supported_engines_are_user_selectable_only():
engines = registry.supported_parser_engines()
assert engines == frozenset({"native", "legacy", "mineru", "docling"})
# Internal format handlers are registered but not user-selectable.
assert "reuse" not in engines
assert "passthrough" not in engines
def test_suffix_capabilities_lookup():
assert "pdf" in registry.suffix_capabilities("mineru")
assert registry.suffix_capabilities("native") == frozenset(
{"docx", "md", "textpack"}
)
assert registry.suffix_capabilities("unknown-engine") == frozenset()
def test_get_parser_unknown_returns_none():
assert registry.get_parser("does-not-exist") is None
def test_get_parser_instances_are_cached():
a = registry.get_parser("native")
b = registry.get_parser("native")
assert a is b and a is not None
def test_register_parser_roundtrips_concurrency():
# The registrant bakes any env override into a concrete ``concurrency``
# value at registration; the spec carries it verbatim.
spec = registry.ParserSpec(
engine_name="third-party-engine",
impl="x:Y",
suffixes=frozenset({"foo"}),
queue_group="third-party-engine",
concurrency=7,
)
try:
registry.register_parser(spec)
snapshot = registry.parser_specs_snapshot()
assert snapshot["third-party-engine"].concurrency == 7
assert snapshot["third-party-engine"].queue_group == "third-party-engine"
finally:
# Keep the module-level registry clean for other tests.
registry._REGISTRY.pop("third-party-engine", None)
def test_mineru_endpoint_requirement_tracks_api_mode(monkeypatch):
monkeypatch.setenv("MINERU_API_MODE", "official")
assert registry.engine_endpoint_requirement("mineru") == "MINERU_API_TOKEN"
monkeypatch.setenv("MINERU_API_MODE", "local")
assert registry.engine_endpoint_requirement("mineru") == "MINERU_LOCAL_ENDPOINT"
def test_capability_queries_do_not_import_parser_impls():
"""Importing the registry + querying capabilities must not pull a parser
implementation (and therefore httpx). Run in a clean subprocess so other
tests' imports don't pollute sys.modules."""
code = (
"import sys; import lightrag.parser.registry as r; "
"r.supported_parser_engines(); r.suffix_capabilities('mineru'); "
"r.engine_endpoint_configured('docling'); "
"assert 'httpx' not in sys.modules, 'httpx leaked'; "
"assert 'lightrag.parser.external.mineru.parser' not in sys.modules; "
"print('clean')"
)
proc = subprocess.run([sys.executable, "-c", code], capture_output=True, text=True)
assert proc.returncode == 0, proc.stderr
assert "clean" in proc.stdout