chore: import upstream snapshot with attribution
This commit is contained in:
@@ -0,0 +1,329 @@
|
||||
"""Scenario fixtures for the native docx → SidecarWriter migration tests.
|
||||
|
||||
Each scenario describes:
|
||||
|
||||
- ``blocks`` — what ``extract_docx_blocks`` would return (the synthetic
|
||||
block dicts that the adapter consumes).
|
||||
- ``parse_metadata`` — the dict the upstream parser fills in (only
|
||||
``first_heading`` is currently consumed by the adapter).
|
||||
- ``assets`` — files the upstream extractor would have written into
|
||||
``<base>.blocks.assets/`` before the IR builder runs. Maps relative names
|
||||
inside the asset dir → byte content.
|
||||
- ``doc_id`` — fixed so blockid + sidecar ids are deterministic.
|
||||
- ``file_path`` — used for canonical basename / doc_title fallback.
|
||||
|
||||
The captured outputs (``blocks.jsonl`` + per-modality JSONs + assets) live
|
||||
under ``tests/parser/docx/golden/native_docx/<scenario>/``. The
|
||||
production path (``LightRAG.parse_native``) must produce byte-identical
|
||||
bytes vs those fixtures; the regen script under ``scripts/`` rewrites
|
||||
them when the format intentionally changes.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any
|
||||
|
||||
|
||||
def _block(
|
||||
content: str,
|
||||
*,
|
||||
heading: str = "",
|
||||
level: int = 0,
|
||||
parent: list[str] | None = None,
|
||||
uuid: str = "p1",
|
||||
uuid_end: str | None = None,
|
||||
table_headers: list[Any] | None = None,
|
||||
table_chunk_role: str = "none",
|
||||
) -> dict[str, Any]:
|
||||
"""Build a synthetic block matching ``extract_docx_blocks`` output."""
|
||||
out: dict[str, Any] = {
|
||||
"uuid": uuid,
|
||||
"uuid_end": uuid_end if uuid_end is not None else uuid,
|
||||
"heading": heading,
|
||||
"content": content,
|
||||
"type": "text",
|
||||
"parent_headings": list(parent or []),
|
||||
"level": level,
|
||||
"table_chunk_role": table_chunk_role,
|
||||
}
|
||||
if table_headers is not None:
|
||||
out["table_headers"] = table_headers
|
||||
return out
|
||||
|
||||
|
||||
@dataclass
|
||||
class Scenario:
|
||||
name: str
|
||||
doc_id: str
|
||||
file_path: str # canonical-ish; what the pipeline would pass
|
||||
blocks: list[dict[str, Any]]
|
||||
parse_metadata: dict[str, Any] = field(default_factory=dict)
|
||||
assets: dict[str, bytes] = field(default_factory=dict)
|
||||
|
||||
|
||||
SCENARIOS: list[Scenario] = [
|
||||
# --- 1: text-only, multi-heading -----------------------------------
|
||||
Scenario(
|
||||
name="text_only_hierarchy",
|
||||
doc_id="doc-aaaa111122223333aaaa111122223333",
|
||||
file_path="paper.docx",
|
||||
parse_metadata={"first_heading": "Introduction"},
|
||||
blocks=[
|
||||
_block(
|
||||
"# Introduction",
|
||||
heading="Introduction",
|
||||
level=1,
|
||||
uuid="h1",
|
||||
),
|
||||
_block(
|
||||
"Body paragraph one.",
|
||||
heading="Introduction",
|
||||
level=1,
|
||||
uuid="p1",
|
||||
uuid_end="p2",
|
||||
),
|
||||
_block(
|
||||
"## Background",
|
||||
heading="Background",
|
||||
level=2,
|
||||
parent=["Introduction"],
|
||||
uuid="h2",
|
||||
),
|
||||
_block(
|
||||
"Sub body.",
|
||||
heading="Background",
|
||||
level=2,
|
||||
parent=["Introduction"],
|
||||
uuid="p3",
|
||||
),
|
||||
],
|
||||
),
|
||||
# --- 2: block + inline equations -----------------------------------
|
||||
Scenario(
|
||||
name="equations_block_and_inline",
|
||||
doc_id="doc-bbbb222233334444bbbb222233334444",
|
||||
file_path="formulas.docx",
|
||||
parse_metadata={"first_heading": "Equations"},
|
||||
blocks=[
|
||||
_block(
|
||||
"# Equations",
|
||||
heading="Equations",
|
||||
level=1,
|
||||
uuid="h1",
|
||||
),
|
||||
_block(
|
||||
# Inline equation (no surrounding \n on either side)
|
||||
"Energy is <equation>E=mc^2</equation> per Einstein.",
|
||||
heading="Equations",
|
||||
level=1,
|
||||
uuid="p1",
|
||||
),
|
||||
_block(
|
||||
# Block equation (wedged between newlines)
|
||||
"Consider:\n<equation>x^2 + y^2 = r^2</equation>\nThe circle equation.",
|
||||
heading="Equations",
|
||||
level=1,
|
||||
uuid="p2",
|
||||
),
|
||||
_block(
|
||||
# Block at content edge (start == 0)
|
||||
"<equation>a + b = c</equation>\ntext after",
|
||||
heading="Equations",
|
||||
level=1,
|
||||
uuid="p3",
|
||||
),
|
||||
],
|
||||
),
|
||||
# --- 3: tables with and without table_headers ----------------------
|
||||
Scenario(
|
||||
name="tables_mixed",
|
||||
doc_id="doc-cccc333344445555cccc333344445555",
|
||||
file_path="report.docx",
|
||||
parse_metadata={"first_heading": "Report"},
|
||||
blocks=[
|
||||
_block(
|
||||
"# Report",
|
||||
heading="Report",
|
||||
level=1,
|
||||
uuid="h1",
|
||||
),
|
||||
_block(
|
||||
# Table with table_headers (cross-page repeating)
|
||||
'See table:\n<table>[["X","Y"],["1","2"],["3","4"]]</table>',
|
||||
heading="Report",
|
||||
level=1,
|
||||
uuid="t1",
|
||||
table_headers=[[["X", "Y"]]], # one table, one header row
|
||||
),
|
||||
_block(
|
||||
# Table without table_headers
|
||||
'Plain table:\n<table>[["a","b"]]</table>',
|
||||
heading="Report",
|
||||
level=1,
|
||||
uuid="t2",
|
||||
),
|
||||
_block(
|
||||
# Two tables in one block
|
||||
'<table>[["p"]]</table>\nthen\n<table>[["q","r"],["s","t"]]</table>',
|
||||
heading="Report",
|
||||
level=1,
|
||||
uuid="t3",
|
||||
table_headers=[None, [["q", "r"]]],
|
||||
),
|
||||
],
|
||||
),
|
||||
# --- 4: drawings + assets ------------------------------------------
|
||||
Scenario(
|
||||
name="drawings_with_assets",
|
||||
doc_id="doc-dddd444455556666dddd444455556666",
|
||||
file_path="diagrams.docx",
|
||||
parse_metadata={"first_heading": "Diagrams"},
|
||||
assets={
|
||||
"fig1.png": b"\x89PNG\r\n\x1a\n-fig1-fake",
|
||||
"fig2.jpg": b"\xff\xd8\xff\xe0-fig2-fake",
|
||||
},
|
||||
blocks=[
|
||||
_block(
|
||||
"# Diagrams",
|
||||
heading="Diagrams",
|
||||
level=1,
|
||||
uuid="h1",
|
||||
),
|
||||
_block(
|
||||
"Figure one:\n"
|
||||
'<drawing id="x" format="png" '
|
||||
'path="diagrams.blocks.assets/fig1.png" '
|
||||
'src="docx://image1" />\n'
|
||||
"Figure two:\n"
|
||||
'<drawing id="y" format="jpg" '
|
||||
'path="diagrams.blocks.assets/fig2.jpg" '
|
||||
'src="docx://image2" />',
|
||||
heading="Diagrams",
|
||||
level=1,
|
||||
uuid="p1",
|
||||
),
|
||||
],
|
||||
),
|
||||
# --- 5: all modalities mixed ---------------------------------------
|
||||
Scenario(
|
||||
name="all_modalities",
|
||||
doc_id="doc-eeee555566667777eeee555566667777",
|
||||
file_path="combo.docx",
|
||||
parse_metadata={"first_heading": "Combined"},
|
||||
assets={"pic.png": b"PNG-combo"},
|
||||
blocks=[
|
||||
_block(
|
||||
"# Combined",
|
||||
heading="Combined",
|
||||
level=1,
|
||||
uuid="h1",
|
||||
),
|
||||
_block(
|
||||
"Look at this figure:\n"
|
||||
'<drawing id="z" format="png" '
|
||||
'path="combo.blocks.assets/pic.png" '
|
||||
'src="docx://img" />\n'
|
||||
"Plus a table:\n"
|
||||
'<table>[["α","β"],["γ","δ"]]</table>\n'
|
||||
"And a block equation:\n"
|
||||
"<equation>F = ma</equation>\n"
|
||||
"And an inline <equation>v=d/t</equation> here.",
|
||||
heading="Combined",
|
||||
level=1,
|
||||
uuid="p1",
|
||||
),
|
||||
],
|
||||
),
|
||||
# --- 6: empty block dropped ----------------------------------------
|
||||
Scenario(
|
||||
name="empty_block_dropped",
|
||||
doc_id="doc-ffff666677778888ffff666677778888",
|
||||
file_path="sparse.docx",
|
||||
parse_metadata={"first_heading": "Sparse"},
|
||||
blocks=[
|
||||
_block(
|
||||
"# Sparse",
|
||||
heading="Sparse",
|
||||
level=1,
|
||||
uuid="h1",
|
||||
),
|
||||
_block(
|
||||
" \n ", # strips to empty — must be dropped
|
||||
heading="Sparse",
|
||||
level=1,
|
||||
uuid="p_empty",
|
||||
),
|
||||
_block(
|
||||
"Real content after empty.",
|
||||
heading="Sparse",
|
||||
level=1,
|
||||
uuid="p_real",
|
||||
),
|
||||
],
|
||||
),
|
||||
# --- 7: external / linked image references ------------------------
|
||||
# DOCX can carry ``<a:blip r:link="rId…"/>`` references to image
|
||||
# targets that live outside the package — the upstream extractor
|
||||
# then emits ``<drawing path="<external URL or unresolved path>" />``
|
||||
# WITHOUT writing bytes into ``<base>.blocks.assets/``. The adapter
|
||||
# must pass those paths through verbatim (both in ``blocks.jsonl``
|
||||
# and ``drawings.json``); turning them into AssetSpecs with
|
||||
# ``source=None`` would make the writer warn-and-skip → ``path=""``,
|
||||
# losing the only reference downstream consumers have.
|
||||
Scenario(
|
||||
name="external_image_link",
|
||||
doc_id="doc-1111aaaa2222bbbb1111aaaa2222bbbb",
|
||||
file_path="linked.docx",
|
||||
parse_metadata={"first_heading": "Linked"},
|
||||
# No on-disk assets — the path points elsewhere.
|
||||
assets={},
|
||||
blocks=[
|
||||
_block(
|
||||
"# Linked",
|
||||
heading="Linked",
|
||||
level=1,
|
||||
uuid="h1",
|
||||
),
|
||||
_block(
|
||||
"See the diagram online:\n"
|
||||
'<drawing id="z" format="png" '
|
||||
'path="https://example.com/diagrams/architecture.png" '
|
||||
'src="docx://external" />\n'
|
||||
"And a relative-but-not-asset path:\n"
|
||||
'<drawing id="z2" format="gif" '
|
||||
'path="../images/legacy.gif" '
|
||||
'src="docx://legacy" />',
|
||||
heading="Linked",
|
||||
level=1,
|
||||
uuid="p1",
|
||||
),
|
||||
],
|
||||
),
|
||||
# --- 8: missing paraid ---------------------------------------------
|
||||
Scenario(
|
||||
name="missing_paraid",
|
||||
doc_id="doc-99990000111122229999000011112222",
|
||||
file_path="legacy.docx",
|
||||
parse_metadata={"first_heading": ""}, # no headings at all
|
||||
blocks=[
|
||||
_block(
|
||||
"Just plain text without a heading.",
|
||||
heading="",
|
||||
level=0,
|
||||
uuid="", # missing
|
||||
uuid_end="",
|
||||
),
|
||||
_block(
|
||||
"Another paragraph with no paraId.",
|
||||
heading="",
|
||||
level=0,
|
||||
uuid="",
|
||||
uuid_end="",
|
||||
),
|
||||
],
|
||||
),
|
||||
]
|
||||
|
||||
|
||||
__all__ = ["Scenario", "SCENARIOS", "_block"]
|
||||
@@ -0,0 +1 @@
|
||||
PNG-combo
|
||||
@@ -0,0 +1,3 @@
|
||||
{"type": "meta", "format": "lightrag", "version": "1.0", "document_name": "combo.docx", "document_format": "docx", "document_hash": "sha256:2aa5d36cf1126af704cc0b7b924e1b8d9eb8fa576b19807f9fb1fbc80d0fefb7", "table_file": true, "equation_file": true, "drawing_file": true, "asset_dir": true, "blocks": 2, "doc_id": "doc-eeee555566667777eeee555566667777", "parse_engine": "native", "parse_time": "2026-01-01T00:00:00+00:00", "doc_title": "Combined"}
|
||||
{"type": "content", "blockid": "bea7fc7932485628877517b6b5a0e80c", "format": "plain_text", "content": "# Combined", "heading": "Combined", "parent_headings": [], "level": 1, "session_type": "body", "table_slice": "none", "positions": [{"type": "paraid", "range": ["h1", "h1"]}]}
|
||||
{"type": "content", "blockid": "8caba9870df85b9357e9521a70d82225", "format": "plain_text", "content": "Look at this figure:\n<drawing id=\"im-eeee555566667777eeee555566667777-0001\" format=\"png\" path=\"pic.png\" src=\"docx://img\" />\nPlus a table:\n<table id=\"tb-eeee555566667777eeee555566667777-0001\" format=\"json\">[[\"α\",\"β\"],[\"γ\",\"δ\"]]</table>\nAnd a block equation:\n<equation id=\"eq-eeee555566667777eeee555566667777-0001\" format=\"latex\">F = ma</equation>\nAnd an inline <equation format=\"latex\">v=d/t</equation> here.", "heading": "Combined", "parent_headings": [], "level": 1, "session_type": "body", "table_slice": "none", "positions": [{"type": "paraid", "range": ["p1", "p1"]}]}
|
||||
@@ -0,0 +1,16 @@
|
||||
{
|
||||
"version": "1.0",
|
||||
"drawings": {
|
||||
"im-eeee555566667777eeee555566667777-0001": {
|
||||
"id": "im-eeee555566667777eeee555566667777-0001",
|
||||
"blockid": "8caba9870df85b9357e9521a70d82225",
|
||||
"heading": "Combined",
|
||||
"parent_headings": [],
|
||||
"format": "png",
|
||||
"path": "combo.blocks.assets/pic.png",
|
||||
"src": "docx://img",
|
||||
"caption": "",
|
||||
"footnotes": []
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,15 @@
|
||||
{
|
||||
"version": "1.0",
|
||||
"equations": {
|
||||
"eq-eeee555566667777eeee555566667777-0001": {
|
||||
"id": "eq-eeee555566667777eeee555566667777-0001",
|
||||
"blockid": "8caba9870df85b9357e9521a70d82225",
|
||||
"heading": "Combined",
|
||||
"parent_headings": [],
|
||||
"format": "latex",
|
||||
"content": "F = ma",
|
||||
"caption": "",
|
||||
"footnotes": []
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,19 @@
|
||||
{
|
||||
"version": "1.0",
|
||||
"tables": {
|
||||
"tb-eeee555566667777eeee555566667777-0001": {
|
||||
"id": "tb-eeee555566667777eeee555566667777-0001",
|
||||
"blockid": "8caba9870df85b9357e9521a70d82225",
|
||||
"heading": "Combined",
|
||||
"parent_headings": [],
|
||||
"dimension": [
|
||||
2,
|
||||
2
|
||||
],
|
||||
"format": "json",
|
||||
"content": "[[\"α\", \"β\"], [\"γ\", \"δ\"]]",
|
||||
"caption": "",
|
||||
"footnotes": []
|
||||
}
|
||||
}
|
||||
}
|
||||
+3
@@ -0,0 +1,3 @@
|
||||
‰PNG
|
||||
|
||||
-fig1-fake
|
||||
|
After Width: | Height: | Size: 18 B |
+1
@@ -0,0 +1 @@
|
||||
�リ��-fig2-fake
|
||||
|
After Width: | Height: | Size: 14 B |
@@ -0,0 +1,3 @@
|
||||
{"type": "meta", "format": "lightrag", "version": "1.0", "document_name": "diagrams.docx", "document_format": "docx", "document_hash": "sha256:9ad9b5e89e06783368565eda8ff1cd44e6b2f9a4d678f30e204b7c4984da524e", "table_file": false, "equation_file": false, "drawing_file": true, "asset_dir": true, "blocks": 2, "doc_id": "doc-dddd444455556666dddd444455556666", "parse_engine": "native", "parse_time": "2026-01-01T00:00:00+00:00", "doc_title": "Diagrams"}
|
||||
{"type": "content", "blockid": "14a79f4f059cb16effe2d6cfe5a18e9b", "format": "plain_text", "content": "# Diagrams", "heading": "Diagrams", "parent_headings": [], "level": 1, "session_type": "body", "table_slice": "none", "positions": [{"type": "paraid", "range": ["h1", "h1"]}]}
|
||||
{"type": "content", "blockid": "d10a9ed61f86247aeedfefb785c919b3", "format": "plain_text", "content": "Figure one:\n<drawing id=\"im-dddd444455556666dddd444455556666-0001\" format=\"png\" path=\"fig1.png\" src=\"docx://image1\" />\nFigure two:\n<drawing id=\"im-dddd444455556666dddd444455556666-0002\" format=\"jpg\" path=\"fig2.jpg\" src=\"docx://image2\" />", "heading": "Diagrams", "parent_headings": [], "level": 1, "session_type": "body", "table_slice": "none", "positions": [{"type": "paraid", "range": ["p1", "p1"]}]}
|
||||
@@ -0,0 +1,27 @@
|
||||
{
|
||||
"version": "1.0",
|
||||
"drawings": {
|
||||
"im-dddd444455556666dddd444455556666-0001": {
|
||||
"id": "im-dddd444455556666dddd444455556666-0001",
|
||||
"blockid": "d10a9ed61f86247aeedfefb785c919b3",
|
||||
"heading": "Diagrams",
|
||||
"parent_headings": [],
|
||||
"format": "png",
|
||||
"path": "diagrams.blocks.assets/fig1.png",
|
||||
"src": "docx://image1",
|
||||
"caption": "",
|
||||
"footnotes": []
|
||||
},
|
||||
"im-dddd444455556666dddd444455556666-0002": {
|
||||
"id": "im-dddd444455556666dddd444455556666-0002",
|
||||
"blockid": "d10a9ed61f86247aeedfefb785c919b3",
|
||||
"heading": "Diagrams",
|
||||
"parent_headings": [],
|
||||
"format": "jpg",
|
||||
"path": "diagrams.blocks.assets/fig2.jpg",
|
||||
"src": "docx://image2",
|
||||
"caption": "",
|
||||
"footnotes": []
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,3 @@
|
||||
{"type": "meta", "format": "lightrag", "version": "1.0", "document_name": "sparse.docx", "document_format": "docx", "document_hash": "sha256:9f42b2b11e8a06b78fbd7fd815f2d800027d33c2bd6f1aa0f92feab32a37a0cb", "table_file": false, "equation_file": false, "drawing_file": false, "asset_dir": false, "blocks": 2, "doc_id": "doc-ffff666677778888ffff666677778888", "parse_engine": "native", "parse_time": "2026-01-01T00:00:00+00:00", "doc_title": "Sparse"}
|
||||
{"type": "content", "blockid": "1fb5dc88cfc19f236d324df359829044", "format": "plain_text", "content": "# Sparse", "heading": "Sparse", "parent_headings": [], "level": 1, "session_type": "body", "table_slice": "none", "positions": [{"type": "paraid", "range": ["h1", "h1"]}]}
|
||||
{"type": "content", "blockid": "39f44666ee7d52f6fc680bad4f93ffa8", "format": "plain_text", "content": "Real content after empty.", "heading": "Sparse", "parent_headings": [], "level": 1, "session_type": "body", "table_slice": "none", "positions": [{"type": "paraid", "range": ["p_real", "p_real"]}]}
|
||||
@@ -0,0 +1,5 @@
|
||||
{"type": "meta", "format": "lightrag", "version": "1.0", "document_name": "formulas.docx", "document_format": "docx", "document_hash": "sha256:403a469531b150e2c1c14a3728ea6d2e3b48b3c86cf4d4896b94d5c0d6d07fcb", "table_file": false, "equation_file": true, "drawing_file": false, "asset_dir": false, "blocks": 4, "doc_id": "doc-bbbb222233334444bbbb222233334444", "parse_engine": "native", "parse_time": "2026-01-01T00:00:00+00:00", "doc_title": "Equations"}
|
||||
{"type": "content", "blockid": "49b6d85e4de7d0a8a8024d68b297885d", "format": "plain_text", "content": "# Equations", "heading": "Equations", "parent_headings": [], "level": 1, "session_type": "body", "table_slice": "none", "positions": [{"type": "paraid", "range": ["h1", "h1"]}]}
|
||||
{"type": "content", "blockid": "3caba68eda5e6f4f3288d3c8a338b76e", "format": "plain_text", "content": "Energy is <equation format=\"latex\">E=mc^2</equation> per Einstein.", "heading": "Equations", "parent_headings": [], "level": 1, "session_type": "body", "table_slice": "none", "positions": [{"type": "paraid", "range": ["p1", "p1"]}]}
|
||||
{"type": "content", "blockid": "603a388c6d9c45900dcbde3a86a0c3b5", "format": "plain_text", "content": "Consider:\n<equation id=\"eq-bbbb222233334444bbbb222233334444-0001\" format=\"latex\">x^2 + y^2 = r^2</equation>\nThe circle equation.", "heading": "Equations", "parent_headings": [], "level": 1, "session_type": "body", "table_slice": "none", "positions": [{"type": "paraid", "range": ["p2", "p2"]}]}
|
||||
{"type": "content", "blockid": "e4c6683e6107d75437b7d0712cf75c82", "format": "plain_text", "content": "<equation id=\"eq-bbbb222233334444bbbb222233334444-0002\" format=\"latex\">a + b = c</equation>\ntext after", "heading": "Equations", "parent_headings": [], "level": 1, "session_type": "body", "table_slice": "none", "positions": [{"type": "paraid", "range": ["p3", "p3"]}]}
|
||||
+25
@@ -0,0 +1,25 @@
|
||||
{
|
||||
"version": "1.0",
|
||||
"equations": {
|
||||
"eq-bbbb222233334444bbbb222233334444-0001": {
|
||||
"id": "eq-bbbb222233334444bbbb222233334444-0001",
|
||||
"blockid": "603a388c6d9c45900dcbde3a86a0c3b5",
|
||||
"heading": "Equations",
|
||||
"parent_headings": [],
|
||||
"format": "latex",
|
||||
"content": "x^2 + y^2 = r^2",
|
||||
"caption": "",
|
||||
"footnotes": []
|
||||
},
|
||||
"eq-bbbb222233334444bbbb222233334444-0002": {
|
||||
"id": "eq-bbbb222233334444bbbb222233334444-0002",
|
||||
"blockid": "e4c6683e6107d75437b7d0712cf75c82",
|
||||
"heading": "Equations",
|
||||
"parent_headings": [],
|
||||
"format": "latex",
|
||||
"content": "a + b = c",
|
||||
"caption": "",
|
||||
"footnotes": []
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,3 @@
|
||||
{"type": "meta", "format": "lightrag", "version": "1.0", "document_name": "linked.docx", "document_format": "docx", "document_hash": "sha256:4d984f8d2a4cb95808c403800f17e4f5043f48cc9d72e6062c93b5150e389320", "table_file": false, "equation_file": false, "drawing_file": true, "asset_dir": false, "blocks": 2, "doc_id": "doc-1111aaaa2222bbbb1111aaaa2222bbbb", "parse_engine": "native", "parse_time": "2026-01-01T00:00:00+00:00", "doc_title": "Linked"}
|
||||
{"type": "content", "blockid": "573c0f2fc108950c0ce0060429a9de39", "format": "plain_text", "content": "# Linked", "heading": "Linked", "parent_headings": [], "level": 1, "session_type": "body", "table_slice": "none", "positions": [{"type": "paraid", "range": ["h1", "h1"]}]}
|
||||
{"type": "content", "blockid": "42b4dd9bfb50ba8f1c4b60229ea4605a", "format": "plain_text", "content": "See the diagram online:\n<drawing id=\"im-1111aaaa2222bbbb1111aaaa2222bbbb-0001\" format=\"png\" path=\"https://example.com/diagrams/architecture.png\" src=\"docx://external\" />\nAnd a relative-but-not-asset path:\n<drawing id=\"im-1111aaaa2222bbbb1111aaaa2222bbbb-0002\" format=\"gif\" path=\"../images/legacy.gif\" src=\"docx://legacy\" />", "heading": "Linked", "parent_headings": [], "level": 1, "session_type": "body", "table_slice": "none", "positions": [{"type": "paraid", "range": ["p1", "p1"]}]}
|
||||
@@ -0,0 +1,27 @@
|
||||
{
|
||||
"version": "1.0",
|
||||
"drawings": {
|
||||
"im-1111aaaa2222bbbb1111aaaa2222bbbb-0001": {
|
||||
"id": "im-1111aaaa2222bbbb1111aaaa2222bbbb-0001",
|
||||
"blockid": "42b4dd9bfb50ba8f1c4b60229ea4605a",
|
||||
"heading": "Linked",
|
||||
"parent_headings": [],
|
||||
"format": "png",
|
||||
"path": "https://example.com/diagrams/architecture.png",
|
||||
"src": "docx://external",
|
||||
"caption": "",
|
||||
"footnotes": []
|
||||
},
|
||||
"im-1111aaaa2222bbbb1111aaaa2222bbbb-0002": {
|
||||
"id": "im-1111aaaa2222bbbb1111aaaa2222bbbb-0002",
|
||||
"blockid": "42b4dd9bfb50ba8f1c4b60229ea4605a",
|
||||
"heading": "Linked",
|
||||
"parent_headings": [],
|
||||
"format": "gif",
|
||||
"path": "../images/legacy.gif",
|
||||
"src": "docx://legacy",
|
||||
"caption": "",
|
||||
"footnotes": []
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,3 @@
|
||||
{"type": "meta", "format": "lightrag", "version": "1.0", "document_name": "legacy.docx", "document_format": "docx", "document_hash": "sha256:1b888273b2772355165b6e6bc69092a35f421fb382e4e1cf5bdb2a92db70ee86", "table_file": false, "equation_file": false, "drawing_file": false, "asset_dir": false, "blocks": 2, "doc_id": "doc-99990000111122229999000011112222", "parse_engine": "native", "parse_time": "2026-01-01T00:00:00+00:00", "doc_title": "legacy"}
|
||||
{"type": "content", "blockid": "11ffd40d2d43ce0c105a7d0ffe3c803f", "format": "plain_text", "content": "Just plain text without a heading.", "heading": "", "parent_headings": [], "level": 0, "session_type": "body", "table_slice": "none", "positions": [{"type": "paraid", "range": [null, null]}]}
|
||||
{"type": "content", "blockid": "75c3c3be70534741d0ce0eeecf308502", "format": "plain_text", "content": "Another paragraph with no paraId.", "heading": "", "parent_headings": [], "level": 0, "session_type": "body", "table_slice": "none", "positions": [{"type": "paraid", "range": [null, null]}]}
|
||||
@@ -0,0 +1,5 @@
|
||||
{"type": "meta", "format": "lightrag", "version": "1.0", "document_name": "report.docx", "document_format": "docx", "document_hash": "sha256:ac2fac1eece1383be12ebb2ca8ace735bfcb3078af0740e22d35508cf8b7421c", "table_file": true, "equation_file": false, "drawing_file": false, "asset_dir": false, "blocks": 4, "doc_id": "doc-cccc333344445555cccc333344445555", "parse_engine": "native", "parse_time": "2026-01-01T00:00:00+00:00", "doc_title": "Report"}
|
||||
{"type": "content", "blockid": "4ede43f1d15b69ff71c86035d0f7cfac", "format": "plain_text", "content": "# Report", "heading": "Report", "parent_headings": [], "level": 1, "session_type": "body", "table_slice": "none", "positions": [{"type": "paraid", "range": ["h1", "h1"]}]}
|
||||
{"type": "content", "blockid": "c2f594c8d5882977f79ad8ec6da3338f", "format": "plain_text", "content": "See table:\n<table id=\"tb-cccc333344445555cccc333344445555-0001\" format=\"json\">[[\"X\",\"Y\"],[\"1\",\"2\"],[\"3\",\"4\"]]</table>", "heading": "Report", "parent_headings": [], "level": 1, "session_type": "body", "table_slice": "none", "positions": [{"type": "paraid", "range": ["t1", "t1"]}]}
|
||||
{"type": "content", "blockid": "bc35d892e4c8f05c9eaf758a4f78878e", "format": "plain_text", "content": "Plain table:\n<table id=\"tb-cccc333344445555cccc333344445555-0002\" format=\"json\">[[\"a\",\"b\"]]</table>", "heading": "Report", "parent_headings": [], "level": 1, "session_type": "body", "table_slice": "none", "positions": [{"type": "paraid", "range": ["t2", "t2"]}]}
|
||||
{"type": "content", "blockid": "fd0febb81cacf41fe81aea2812bd1bd8", "format": "plain_text", "content": "<table id=\"tb-cccc333344445555cccc333344445555-0003\" format=\"json\">[[\"p\"]]</table>\nthen\n<table id=\"tb-cccc333344445555cccc333344445555-0004\" format=\"json\">[[\"q\",\"r\"],[\"s\",\"t\"]]</table>", "heading": "Report", "parent_headings": [], "level": 1, "session_type": "body", "table_slice": "none", "positions": [{"type": "paraid", "range": ["t3", "t3"]}]}
|
||||
@@ -0,0 +1,63 @@
|
||||
{
|
||||
"version": "1.0",
|
||||
"tables": {
|
||||
"tb-cccc333344445555cccc333344445555-0001": {
|
||||
"id": "tb-cccc333344445555cccc333344445555-0001",
|
||||
"blockid": "c2f594c8d5882977f79ad8ec6da3338f",
|
||||
"heading": "Report",
|
||||
"parent_headings": [],
|
||||
"dimension": [
|
||||
3,
|
||||
2
|
||||
],
|
||||
"format": "json",
|
||||
"content": "[[\"X\", \"Y\"], [\"1\", \"2\"], [\"3\", \"4\"]]",
|
||||
"caption": "",
|
||||
"footnotes": [],
|
||||
"table_header": "[[\"X\", \"Y\"]]"
|
||||
},
|
||||
"tb-cccc333344445555cccc333344445555-0002": {
|
||||
"id": "tb-cccc333344445555cccc333344445555-0002",
|
||||
"blockid": "bc35d892e4c8f05c9eaf758a4f78878e",
|
||||
"heading": "Report",
|
||||
"parent_headings": [],
|
||||
"dimension": [
|
||||
1,
|
||||
2
|
||||
],
|
||||
"format": "json",
|
||||
"content": "[[\"a\", \"b\"]]",
|
||||
"caption": "",
|
||||
"footnotes": []
|
||||
},
|
||||
"tb-cccc333344445555cccc333344445555-0003": {
|
||||
"id": "tb-cccc333344445555cccc333344445555-0003",
|
||||
"blockid": "fd0febb81cacf41fe81aea2812bd1bd8",
|
||||
"heading": "Report",
|
||||
"parent_headings": [],
|
||||
"dimension": [
|
||||
1,
|
||||
1
|
||||
],
|
||||
"format": "json",
|
||||
"content": "[[\"p\"]]",
|
||||
"caption": "",
|
||||
"footnotes": []
|
||||
},
|
||||
"tb-cccc333344445555cccc333344445555-0004": {
|
||||
"id": "tb-cccc333344445555cccc333344445555-0004",
|
||||
"blockid": "fd0febb81cacf41fe81aea2812bd1bd8",
|
||||
"heading": "Report",
|
||||
"parent_headings": [],
|
||||
"dimension": [
|
||||
2,
|
||||
2
|
||||
],
|
||||
"format": "json",
|
||||
"content": "[[\"q\", \"r\"], [\"s\", \"t\"]]",
|
||||
"caption": "",
|
||||
"footnotes": [],
|
||||
"table_header": "[[\"q\", \"r\"]]"
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,5 @@
|
||||
{"type": "meta", "format": "lightrag", "version": "1.0", "document_name": "paper.docx", "document_format": "docx", "document_hash": "sha256:e3fd11b45f12fe0fe0ca22cd6526b7d65336e58cfd2c6585fca4511fa967a55a", "table_file": false, "equation_file": false, "drawing_file": false, "asset_dir": false, "blocks": 4, "doc_id": "doc-aaaa111122223333aaaa111122223333", "parse_engine": "native", "parse_time": "2026-01-01T00:00:00+00:00", "doc_title": "Introduction"}
|
||||
{"type": "content", "blockid": "5c647a54224deea018b593b7bbef2348", "format": "plain_text", "content": "# Introduction", "heading": "Introduction", "parent_headings": [], "level": 1, "session_type": "body", "table_slice": "none", "positions": [{"type": "paraid", "range": ["h1", "h1"]}]}
|
||||
{"type": "content", "blockid": "11d7395dfeaf573464e88d3d7f736f5c", "format": "plain_text", "content": "Body paragraph one.", "heading": "Introduction", "parent_headings": [], "level": 1, "session_type": "body", "table_slice": "none", "positions": [{"type": "paraid", "range": ["p1", "p2"]}]}
|
||||
{"type": "content", "blockid": "7bf205c142f2012fbe586007949c21d1", "format": "plain_text", "content": "## Background", "heading": "Background", "parent_headings": ["Introduction"], "level": 2, "session_type": "body", "table_slice": "none", "positions": [{"type": "paraid", "range": ["h2", "h2"]}]}
|
||||
{"type": "content", "blockid": "337190582281b148ff9732cb7a04661e", "format": "plain_text", "content": "Sub body.", "heading": "Background", "parent_headings": ["Introduction"], "level": 2, "session_type": "body", "table_slice": "none", "positions": [{"type": "paraid", "range": ["p3", "p3"]}]}
|
||||
@@ -0,0 +1,69 @@
|
||||
"""Security-focused tests for native DOCX IR drawing asset handling."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
|
||||
from lightrag.parser.docx.ir_builder import NativeDocxIRBuilder
|
||||
|
||||
|
||||
def _build_ir(content: str):
|
||||
return NativeDocxIRBuilder().normalize(
|
||||
[
|
||||
{
|
||||
"uuid": "p1",
|
||||
"uuid_end": "p1",
|
||||
"heading": "Section",
|
||||
"content": content,
|
||||
"parent_headings": [],
|
||||
"level": 1,
|
||||
}
|
||||
],
|
||||
document_name="doc.docx",
|
||||
asset_dir_name="doc.blocks.assets",
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.offline
|
||||
def test_native_docx_ir_accepts_safe_local_drawing_asset() -> None:
|
||||
ir = _build_ir(
|
||||
'<drawing id="1" name="fig" format="png" path="doc.blocks.assets/fig.png" />'
|
||||
)
|
||||
|
||||
drawing = ir.blocks[0].drawings[0]
|
||||
assert drawing.asset_ref == "fig.png"
|
||||
assert drawing.path_override is None
|
||||
assert len(ir.assets) == 1
|
||||
assert ir.assets[0].ref == "fig.png"
|
||||
assert ir.assets[0].suggested_name == "fig.png"
|
||||
assert ir.assets[0].source is None
|
||||
|
||||
|
||||
@pytest.mark.offline
|
||||
@pytest.mark.parametrize(
|
||||
"path",
|
||||
[
|
||||
"doc.blocks.assets/../secret.png",
|
||||
"doc.blocks.assets//tmp/secret.png",
|
||||
r"doc.blocks.assets/..\secret.png",
|
||||
],
|
||||
)
|
||||
def test_native_docx_ir_rejects_unsafe_local_drawing_asset(path: str) -> None:
|
||||
ir = _build_ir(f'<drawing id="1" name="fig" format="png" path="{path}" />')
|
||||
|
||||
drawing = ir.blocks[0].drawings[0]
|
||||
assert drawing.asset_ref == ""
|
||||
assert drawing.path_override is None
|
||||
assert ir.assets == []
|
||||
|
||||
|
||||
@pytest.mark.offline
|
||||
def test_native_docx_ir_preserves_non_asset_external_drawing_path() -> None:
|
||||
ir = _build_ir(
|
||||
'<drawing id="1" name="fig" format="gif" path="../images/legacy.gif" />'
|
||||
)
|
||||
|
||||
drawing = ir.blocks[0].drawings[0]
|
||||
assert drawing.asset_ref == ""
|
||||
assert drawing.path_override == "../images/legacy.gif"
|
||||
assert ir.assets == []
|
||||
@@ -0,0 +1,405 @@
|
||||
"""Unit tests for native docx tracked-change / comment / empty-table handling.
|
||||
|
||||
Locks in the contract that:
|
||||
- ``w:ins`` content survives (final revised text),
|
||||
- ``w:del`` / ``w:moveFrom`` content is dropped,
|
||||
- ``w:commentRangeStart`` / ``w:commentRangeEnd`` / ``w:commentReference`` /
|
||||
``w:annotationRef`` markers are dropped,
|
||||
- tables whose every cell is whitespace-only are omitted from the parser
|
||||
output (no ``<table>`` placeholder, so no IRTable downstream).
|
||||
|
||||
These were previously emergent properties (skip lists in two places + a
|
||||
run-level white-list); regressions now fail loudly.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from io import BytesIO
|
||||
|
||||
import pytest
|
||||
from docx import Document
|
||||
from docx.oxml import OxmlElement
|
||||
from docx.oxml.ns import qn
|
||||
from lxml import etree
|
||||
|
||||
from lightrag.parser.docx.parse_document import (
|
||||
DocxContentError,
|
||||
extract_docx_blocks,
|
||||
extract_paragraph_content,
|
||||
)
|
||||
from lightrag.parser.docx.table_extractor import extract_paragraph_content_table
|
||||
|
||||
W_NS = "http://schemas.openxmlformats.org/wordprocessingml/2006/main"
|
||||
PARAGRAPH_NS = {
|
||||
"w": W_NS,
|
||||
"wp": "http://schemas.openxmlformats.org/drawingml/2006/wordprocessingDrawing",
|
||||
"m": "http://schemas.openxmlformats.org/officeDocument/2006/math",
|
||||
}
|
||||
|
||||
|
||||
def _p(inner_xml: str):
|
||||
"""Build a ``<w:p>`` lxml element from inner OOXML."""
|
||||
return etree.fromstring(f'<w:p xmlns:w="{W_NS}">{inner_xml}</w:p>'.encode("utf-8"))
|
||||
|
||||
|
||||
# --- paragraph walker (parse_document.extract_paragraph_content) -----------
|
||||
|
||||
|
||||
@pytest.mark.offline
|
||||
def test_paragraph_keeps_w_ins_content() -> None:
|
||||
para = _p("<w:ins><w:r><w:t>kept</w:t></w:r></w:ins>")
|
||||
|
||||
assert extract_paragraph_content(para, PARAGRAPH_NS) == "kept"
|
||||
|
||||
|
||||
@pytest.mark.offline
|
||||
def test_paragraph_drops_w_del_content() -> None:
|
||||
para = _p("<w:del><w:r><w:t>removed</w:t></w:r></w:del>")
|
||||
|
||||
assert extract_paragraph_content(para, PARAGRAPH_NS) == ""
|
||||
|
||||
|
||||
@pytest.mark.offline
|
||||
def test_paragraph_drops_w_movefrom_content() -> None:
|
||||
para = _p("<w:moveFrom><w:r><w:t>moved away</w:t></w:r></w:moveFrom>")
|
||||
|
||||
assert extract_paragraph_content(para, PARAGRAPH_NS) == ""
|
||||
|
||||
|
||||
@pytest.mark.offline
|
||||
def test_paragraph_drops_comment_markers_but_keeps_surrounding_text() -> None:
|
||||
para = _p(
|
||||
'<w:commentRangeStart w:id="0"/>'
|
||||
"<w:r><w:t>visible</w:t></w:r>"
|
||||
'<w:commentRangeEnd w:id="0"/>'
|
||||
"<w:r>"
|
||||
' <w:rPr><w:rStyle w:val="CommentReference"/></w:rPr>'
|
||||
' <w:commentReference w:id="0"/>'
|
||||
"</w:r>"
|
||||
)
|
||||
|
||||
assert extract_paragraph_content(para, PARAGRAPH_NS) == "visible"
|
||||
|
||||
|
||||
# --- table-cell walker (table_extractor.extract_paragraph_content_table) ---
|
||||
|
||||
|
||||
@pytest.mark.offline
|
||||
def test_table_cell_keeps_w_ins_content() -> None:
|
||||
para = _p("<w:ins><w:r><w:t>kept</w:t></w:r></w:ins>")
|
||||
|
||||
assert extract_paragraph_content_table(para, qn) == "kept"
|
||||
|
||||
|
||||
@pytest.mark.offline
|
||||
def test_table_cell_drops_w_del_content() -> None:
|
||||
para = _p("<w:del><w:r><w:t>removed</w:t></w:r></w:del>")
|
||||
|
||||
assert extract_paragraph_content_table(para, qn) == ""
|
||||
|
||||
|
||||
@pytest.mark.offline
|
||||
def test_table_cell_drops_comment_markers() -> None:
|
||||
para = _p(
|
||||
'<w:commentRangeStart w:id="0"/>'
|
||||
"<w:r><w:t>cell-text</w:t></w:r>"
|
||||
'<w:commentRangeEnd w:id="0"/>'
|
||||
)
|
||||
|
||||
assert extract_paragraph_content_table(para, qn) == "cell-text"
|
||||
|
||||
|
||||
# --- end-to-end: empty tables vanish from blocks output --------------------
|
||||
|
||||
|
||||
def _populate_cell(cell, text: str) -> None:
|
||||
cell.paragraphs[0].text = text
|
||||
|
||||
|
||||
def _build_docx_with_three_tables() -> BytesIO:
|
||||
"""One real table, one all-empty table, one whitespace-only table.
|
||||
|
||||
Returns a BytesIO containing the rendered .docx so it can be fed to
|
||||
``extract_docx_blocks`` via a tempfile-backed path or directly.
|
||||
"""
|
||||
doc = Document()
|
||||
doc.add_paragraph("intro")
|
||||
|
||||
real = doc.add_table(rows=1, cols=2)
|
||||
_populate_cell(real.rows[0].cells[0], "A")
|
||||
_populate_cell(real.rows[0].cells[1], "B")
|
||||
|
||||
empty = doc.add_table(rows=2, cols=2)
|
||||
# leave every cell at python-docx default (empty paragraph)
|
||||
assert all(c.text == "" for row in empty.rows for c in row.cells)
|
||||
|
||||
whitespace = doc.add_table(rows=1, cols=2)
|
||||
_populate_cell(whitespace.rows[0].cells[0], " ")
|
||||
_populate_cell(whitespace.rows[0].cells[1], "\t")
|
||||
|
||||
buf = BytesIO()
|
||||
doc.save(buf)
|
||||
buf.seek(0)
|
||||
return buf
|
||||
|
||||
|
||||
# --- end-to-end: every heading becomes its own block ----------------------
|
||||
|
||||
|
||||
def _add_heading(doc, text: str, level: int) -> None:
|
||||
"""Append a heading paragraph with an explicit ``w:outlineLvl``.
|
||||
|
||||
Setting outlineLvl directly on the paragraph (rather than relying on the
|
||||
template's built-in Heading styles) keeps ``get_heading_level`` detection
|
||||
deterministic. ``level`` is 1-based (1 = H1); outlineLvl val is 0-based.
|
||||
"""
|
||||
para = doc.add_paragraph(text)
|
||||
pPr = para._p.get_or_add_pPr()
|
||||
outline = OxmlElement("w:outlineLvl")
|
||||
outline.set(qn("w:val"), str(level - 1))
|
||||
pPr.append(outline)
|
||||
|
||||
|
||||
@pytest.mark.offline
|
||||
def test_each_heading_becomes_its_own_block(tmp_path) -> None:
|
||||
"""Headings with no body must each form a standalone block.
|
||||
|
||||
Layout: H1 (no body) → H2 (no body) → H3 (with body) → H1 (no body, last).
|
||||
Previously the empty H1/H2 were folded into the next heading's block; now
|
||||
every recognized heading starts its own block. A heading with no following
|
||||
body yields a block whose ``content`` is just the heading text, while a
|
||||
heading with body merges that body into the same block. ``parent_headings``
|
||||
must track the outline hierarchy correctly.
|
||||
"""
|
||||
doc = Document()
|
||||
_add_heading(doc, "Chapter One", level=1)
|
||||
_add_heading(doc, "Section 1.1", level=2)
|
||||
_add_heading(doc, "Subsection 1.1.1", level=3)
|
||||
doc.add_paragraph("Body text under subsection.")
|
||||
_add_heading(doc, "Appendix", level=1)
|
||||
|
||||
buf = BytesIO()
|
||||
doc.save(buf)
|
||||
docx_path = tmp_path / "headings.docx"
|
||||
docx_path.write_bytes(buf.getvalue())
|
||||
|
||||
# The native parser splits at every heading level and never does
|
||||
# token-based splitting or small-block merging (that is the chunker's job).
|
||||
blocks = extract_docx_blocks(str(docx_path))
|
||||
|
||||
# The content line carries a markdown ``#`` prefix matching the level,
|
||||
# while the ``heading`` field stays clean (no prefix).
|
||||
summary = [
|
||||
(b["heading"], b["content"], b["level"], b["parent_headings"]) for b in blocks
|
||||
]
|
||||
assert summary == [
|
||||
("Chapter One", "# Chapter One", 1, []),
|
||||
("Section 1.1", "## Section 1.1", 2, ["Chapter One"]),
|
||||
(
|
||||
"Subsection 1.1.1",
|
||||
"### Subsection 1.1.1\nBody text under subsection.",
|
||||
3,
|
||||
["Chapter One", "Section 1.1"],
|
||||
),
|
||||
("Appendix", "# Appendix", 1, []),
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.offline
|
||||
def test_heading_markdown_prefix_capped_at_six(tmp_path) -> None:
|
||||
"""Heading content lines get a markdown ``#`` prefix matching the level,
|
||||
capped at 6 ``#`` (a level-7 heading still renders ``######``)."""
|
||||
doc = Document()
|
||||
_add_heading(doc, "Top", level=1)
|
||||
doc.add_paragraph("body under top.")
|
||||
_add_heading(doc, "Deep Seven", level=7)
|
||||
doc.add_paragraph("body under deep.")
|
||||
|
||||
buf = BytesIO()
|
||||
doc.save(buf)
|
||||
docx_path = tmp_path / "deep.docx"
|
||||
docx_path.write_bytes(buf.getvalue())
|
||||
|
||||
blocks = extract_docx_blocks(str(docx_path))
|
||||
|
||||
summary = [(b["heading"], b["content"].split("\n")[0], b["level"]) for b in blocks]
|
||||
assert summary == [
|
||||
("Top", "# Top", 1),
|
||||
# level 7 outline → clamped to six "#".
|
||||
("Deep Seven", "###### Deep Seven", 7),
|
||||
]
|
||||
|
||||
|
||||
def _add_outline_para_with_softbreak(doc, head_text: str, body_text: str, level: int):
|
||||
"""Append one ``<w:p>`` carrying an outline level whose runs are
|
||||
``head_text`` + a soft line break (``<w:br/>``) + ``body_text``.
|
||||
|
||||
Mirrors the real-world WPS/Word pattern where an author sets an outline
|
||||
level on a paragraph but types the body with Shift+Enter (a soft break)
|
||||
instead of a new paragraph, so heading + body live in a single paragraph.
|
||||
"""
|
||||
para = doc.add_paragraph(head_text)
|
||||
pPr = para._p.get_or_add_pPr()
|
||||
outline = OxmlElement("w:outlineLvl")
|
||||
outline.set(qn("w:val"), str(level - 1))
|
||||
pPr.append(outline)
|
||||
para.add_run().add_break() # default WD_BREAK.LINE → soft <w:br/> → '\n'
|
||||
para.add_run(body_text)
|
||||
return para
|
||||
|
||||
|
||||
@pytest.mark.offline
|
||||
def test_oversize_outline_heading_with_softbreak_splits(tmp_path) -> None:
|
||||
"""An over-long heading paragraph that contains a soft break must split:
|
||||
the first line stays the heading, the remainder becomes body text. No
|
||||
DocxContentError, and the body content is preserved in full."""
|
||||
head = "A、处置程序"
|
||||
body = "施工现场一旦发生事故时应立即采取相应的应急处置措施并以最快速度报警。" * 8
|
||||
assert len(head) + 1 + len(body) > 200 # would trip validate_heading_length
|
||||
|
||||
doc = Document()
|
||||
_add_outline_para_with_softbreak(doc, head, body, level=4)
|
||||
|
||||
buf = BytesIO()
|
||||
doc.save(buf)
|
||||
docx_path = tmp_path / "softbreak.docx"
|
||||
docx_path.write_bytes(buf.getvalue())
|
||||
|
||||
blocks = extract_docx_blocks(str(docx_path))
|
||||
|
||||
# The short first line becomes the heading; the long remainder is body.
|
||||
heading_block = next(b for b in blocks if b["heading"] == head)
|
||||
assert body in heading_block["content"]
|
||||
# Heading field carries only the first line, not the body.
|
||||
assert body not in heading_block["heading"]
|
||||
|
||||
|
||||
@pytest.mark.offline
|
||||
def test_oversize_outline_heading_no_softbreak_demoted_to_body(tmp_path) -> None:
|
||||
"""A single-line over-long outline paragraph (no soft break) is demoted to
|
||||
body text rather than raising — content preserved, not treated as a heading."""
|
||||
long_text = "施工现场应急处置措施说明," * 30 # well over 200 chars, no '\n'
|
||||
assert len(long_text) > 200
|
||||
|
||||
doc = Document()
|
||||
_add_heading(doc, long_text, level=2)
|
||||
|
||||
buf = BytesIO()
|
||||
doc.save(buf)
|
||||
docx_path = tmp_path / "demote.docx"
|
||||
docx_path.write_bytes(buf.getvalue())
|
||||
|
||||
blocks = extract_docx_blocks(str(docx_path))
|
||||
|
||||
# The text survives as body content, and no block adopts it as a heading.
|
||||
assert any(long_text in b["content"] for b in blocks)
|
||||
assert all(b["heading"] != long_text for b in blocks)
|
||||
|
||||
|
||||
@pytest.mark.offline
|
||||
def test_short_outline_paragraph_still_heading(tmp_path) -> None:
|
||||
"""A within-limit outline paragraph must still be recognized as a heading —
|
||||
the over-long handling never demotes short headings."""
|
||||
doc = Document()
|
||||
_add_heading(doc, "处置程序", level=3)
|
||||
doc.add_paragraph("body under heading.")
|
||||
|
||||
buf = BytesIO()
|
||||
doc.save(buf)
|
||||
docx_path = tmp_path / "short.docx"
|
||||
docx_path.write_bytes(buf.getvalue())
|
||||
|
||||
blocks = extract_docx_blocks(str(docx_path))
|
||||
|
||||
heading_block = next(b for b in blocks if b["heading"] == "处置程序")
|
||||
assert heading_block["level"] == 3
|
||||
assert heading_block["content"].startswith("### 处置程序")
|
||||
|
||||
|
||||
@pytest.mark.offline
|
||||
def test_existing_markdown_heading_keeps_content_but_metadata_is_clean(
|
||||
tmp_path,
|
||||
) -> None:
|
||||
doc = Document()
|
||||
_add_heading(doc, "# Already MD", level=1)
|
||||
doc.add_paragraph("Body.")
|
||||
|
||||
buf = BytesIO()
|
||||
doc.save(buf)
|
||||
docx_path = tmp_path / "markdown-heading.docx"
|
||||
docx_path.write_bytes(buf.getvalue())
|
||||
|
||||
parse_metadata: dict[str, str] = {}
|
||||
blocks = extract_docx_blocks(str(docx_path), parse_metadata=parse_metadata)
|
||||
|
||||
assert parse_metadata["first_heading"] == "Already MD"
|
||||
assert blocks[0]["heading"] == "Already MD"
|
||||
assert blocks[0]["parent_headings"] == []
|
||||
assert blocks[0]["content"].splitlines()[0] == "# Already MD"
|
||||
|
||||
|
||||
@pytest.mark.offline
|
||||
def test_empty_tables_are_skipped(tmp_path) -> None:
|
||||
docx_bytes = _build_docx_with_three_tables()
|
||||
docx_path = tmp_path / "three_tables.docx"
|
||||
docx_path.write_bytes(docx_bytes.getvalue())
|
||||
|
||||
blocks = extract_docx_blocks(str(docx_path))
|
||||
|
||||
table_placeholders = sum(b["content"].count("<table>") for b in blocks)
|
||||
assert table_placeholders == 1, (
|
||||
"exactly one real table should survive; empty + whitespace tables "
|
||||
"must be dropped before the placeholder is emitted"
|
||||
)
|
||||
assert any('"A"' in b["content"] and '"B"' in b["content"] for b in blocks)
|
||||
|
||||
|
||||
# --- invalid (non-ZIP) .docx files surface an accurate error ---------------
|
||||
|
||||
|
||||
@pytest.mark.offline
|
||||
@pytest.mark.parametrize(
|
||||
("header", "format_hint"),
|
||||
[
|
||||
(b"\xd0\xcf\x11\xe0\xa1\xb1\x1a\xe1" + b"junk", "Word 97-2003"),
|
||||
(b"{\\rtf1\\ansi}", "RTF"),
|
||||
(b"%PDF-1.7\n%binary", "PDF"),
|
||||
(b"<?xml version='1.0'?><html></html>", "HTML or XML"),
|
||||
(b"plain text masquerading as docx", "not a valid DOCX"),
|
||||
],
|
||||
)
|
||||
def test_invalid_docx_raises_accurate_error(tmp_path, header, format_hint) -> None:
|
||||
"""A file that exists but is not a ZIP/OOXML package must fail with an
|
||||
accurate message — never the misleading python-docx "Package not found".
|
||||
|
||||
The native worker confirms the file exists before parsing, so the only way
|
||||
``PackageNotFoundError`` can fire here is a corrupt file or a non-DOCX
|
||||
payload wearing a .docx extension. The error must name the real format and
|
||||
keep the file path.
|
||||
"""
|
||||
docx_path = tmp_path / "masquerade.docx"
|
||||
docx_path.write_bytes(header)
|
||||
|
||||
with pytest.raises(DocxContentError) as excinfo:
|
||||
extract_docx_blocks(str(docx_path))
|
||||
|
||||
message = str(excinfo.value)
|
||||
assert "Package not found" not in message
|
||||
assert "valid DOCX" in message
|
||||
assert str(docx_path) in message
|
||||
assert format_hint in message
|
||||
|
||||
|
||||
@pytest.mark.offline
|
||||
def test_empty_docx_file_reports_truncation(tmp_path) -> None:
|
||||
"""A 0-byte .docx must be diagnosed as empty/truncated, not 'not found'."""
|
||||
docx_path = tmp_path / "empty.docx"
|
||||
docx_path.write_bytes(b"")
|
||||
|
||||
with pytest.raises(DocxContentError) as excinfo:
|
||||
extract_docx_blocks(str(docx_path))
|
||||
|
||||
message = str(excinfo.value)
|
||||
assert "Package not found" not in message
|
||||
assert "empty" in message
|
||||
assert str(docx_path) in message
|
||||
@@ -0,0 +1,165 @@
|
||||
"""Byte-equivalence golden tests for the native docx → SidecarWriter migration.
|
||||
|
||||
These tests run the production code path (``LightRAG.parse_native``) on
|
||||
each scenario in ``_native_docx_fixtures.SCENARIOS`` and assert that
|
||||
every produced file matches the captured baseline bytes under
|
||||
``tests/parser/docx/golden/native_docx/<scenario>/``.
|
||||
|
||||
The baseline was generated by ``scripts/regen_native_docx_golden.py``;
|
||||
the script lives in-tree so the snapshots can be regenerated when
|
||||
intentional changes to the format are made (e.g. spec evolution).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from unittest import mock
|
||||
|
||||
import pytest
|
||||
|
||||
# Make the sibling _native_docx_fixtures module importable as if it were a
|
||||
# top-level helper. Avoids needing a package __init__ rename.
|
||||
HERE = Path(__file__).resolve().parent
|
||||
if str(HERE) not in sys.path:
|
||||
sys.path.insert(0, str(HERE))
|
||||
|
||||
from _native_docx_fixtures import SCENARIOS, Scenario # noqa: E402
|
||||
from lightrag.parser.base import ParseContext # noqa: E402
|
||||
from lightrag.parser.debug import ( # noqa: E402
|
||||
FrozenDateTime,
|
||||
build_debug_rag,
|
||||
)
|
||||
from lightrag.parser.registry import get_parser # noqa: E402
|
||||
|
||||
GOLDEN_ROOT = HERE / "golden" / "native_docx"
|
||||
|
||||
|
||||
def _run_new_path(
|
||||
scenario: Scenario, input_dir: Path, monkeypatch: pytest.MonkeyPatch
|
||||
) -> Path:
|
||||
"""Invoke ``LightRAG.parse_native`` on a single scenario.
|
||||
|
||||
The upstream ``extract_docx_blocks`` is stubbed to feed the synthetic
|
||||
blocks and produce the matching asset files inside
|
||||
``<base>.blocks.assets/``. ``datetime.now`` is frozen so ``parse_time``
|
||||
matches the captured baseline exactly.
|
||||
|
||||
Returns the parsed directory containing the produced artifacts.
|
||||
"""
|
||||
from lightrag.constants import (
|
||||
FULL_DOCS_FORMAT_PENDING_PARSE,
|
||||
PARSED_DIR_NAME,
|
||||
)
|
||||
|
||||
monkeypatch.setenv("INPUT_DIR", str(input_dir))
|
||||
|
||||
# parse_native archives the source after parsing; for the golden test
|
||||
# we don't want the docx moving around between scenarios.
|
||||
async def _noop_archive(_p: str) -> None:
|
||||
return None
|
||||
|
||||
import lightrag.pipeline as pipeline_module
|
||||
|
||||
monkeypatch.setattr(
|
||||
pipeline_module, "archive_docx_source_after_full_docs_sync", _noop_archive
|
||||
)
|
||||
|
||||
source_path = input_dir / scenario.file_path
|
||||
source_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
source_path.write_bytes(b"fake-docx")
|
||||
|
||||
def _stub_extract(
|
||||
file_path,
|
||||
*,
|
||||
drawing_context=None,
|
||||
parse_warnings=None,
|
||||
parse_metadata=None,
|
||||
**_kwargs,
|
||||
):
|
||||
if drawing_context is not None and scenario.assets:
|
||||
drawing_context.export_dir_path.mkdir(parents=True, exist_ok=True)
|
||||
for name, data in scenario.assets.items():
|
||||
(drawing_context.export_dir_path / name).write_bytes(data)
|
||||
if parse_metadata is not None:
|
||||
parse_metadata.update(scenario.parse_metadata)
|
||||
return [dict(b) for b in scenario.blocks]
|
||||
|
||||
rag = build_debug_rag()
|
||||
|
||||
with (
|
||||
mock.patch(
|
||||
"lightrag.parser.docx.parse_document.extract_docx_blocks",
|
||||
_stub_extract,
|
||||
),
|
||||
mock.patch("lightrag.sidecar.writer.datetime", FrozenDateTime),
|
||||
):
|
||||
|
||||
async def _go() -> None:
|
||||
await get_parser("native").parse(
|
||||
ParseContext(
|
||||
rag,
|
||||
scenario.doc_id,
|
||||
str(source_path),
|
||||
{
|
||||
"parse_format": FULL_DOCS_FORMAT_PENDING_PARSE,
|
||||
"content": "",
|
||||
},
|
||||
)
|
||||
)
|
||||
|
||||
asyncio.run(_go())
|
||||
|
||||
return input_dir / PARSED_DIR_NAME / f"{scenario.file_path}.parsed"
|
||||
|
||||
|
||||
def _read_bytes(path: Path) -> bytes:
|
||||
with path.open("rb") as fh:
|
||||
return fh.read()
|
||||
|
||||
|
||||
@pytest.mark.offline
|
||||
@pytest.mark.parametrize(
|
||||
"scenario",
|
||||
SCENARIOS,
|
||||
ids=[s.name for s in SCENARIOS],
|
||||
)
|
||||
def test_native_docx_migration_is_byte_equivalent(
|
||||
scenario: Scenario,
|
||||
tmp_path: Path,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
"""Every file produced by parse_native must be byte-identical to the
|
||||
captured legacy baseline."""
|
||||
input_dir = tmp_path / "inputs"
|
||||
input_dir.mkdir()
|
||||
parsed_dir = _run_new_path(scenario, input_dir, monkeypatch)
|
||||
|
||||
expected_root = GOLDEN_ROOT / scenario.name
|
||||
assert expected_root.is_dir(), (
|
||||
f"missing golden fixture for scenario {scenario.name!r}; "
|
||||
f"run scripts/regen_native_docx_golden.py to regenerate"
|
||||
)
|
||||
|
||||
# Collect both file sets relative to their roots, then compare.
|
||||
expected_files = {
|
||||
p.relative_to(expected_root): p for p in expected_root.rglob("*") if p.is_file()
|
||||
}
|
||||
produced_files = {
|
||||
p.relative_to(parsed_dir): p for p in parsed_dir.rglob("*") if p.is_file()
|
||||
}
|
||||
|
||||
extra = sorted(produced_files.keys() - expected_files.keys())
|
||||
missing = sorted(expected_files.keys() - produced_files.keys())
|
||||
assert not extra, f"unexpected produced files: {extra}"
|
||||
assert not missing, f"missing produced files (legacy had them): {missing}"
|
||||
|
||||
mismatches: list[str] = []
|
||||
for rel, expected_path in expected_files.items():
|
||||
produced_path = produced_files[rel]
|
||||
if _read_bytes(produced_path) != _read_bytes(expected_path):
|
||||
mismatches.append(str(rel))
|
||||
assert not mismatches, (
|
||||
f"byte mismatch in scenario {scenario.name!r} for files: {mismatches}"
|
||||
)
|
||||
@@ -0,0 +1,45 @@
|
||||
"""Native DOCX content-limit validation must *raise*, never ``sys.exit``.
|
||||
|
||||
The DOCX parser runs in-process inside the gunicorn/uvicorn worker (via
|
||||
``LightRAG.parse_native`` → ``extract_docx_blocks``). A ``sys.exit`` there
|
||||
raises ``SystemExit`` (a ``BaseException``), which slips past the pipeline's
|
||||
per-document ``except Exception`` handler and tears down the whole worker —
|
||||
gunicorn then reports ``WORKER TIMEOUT`` and kills it with SIGABRT (code 134),
|
||||
interrupting the entire processing pipeline.
|
||||
|
||||
These tests lock in that the content-limit checks raise ``DocxContentError``
|
||||
(an ``Exception`` subclass) so just the offending document fails while the
|
||||
worker keeps running.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
|
||||
from lightrag.parser.docx.parse_document import (
|
||||
DocxContentError,
|
||||
MAX_HEADING_LENGTH,
|
||||
validate_heading_length,
|
||||
)
|
||||
|
||||
|
||||
def test_docx_content_error_is_an_exception_not_just_baseexception():
|
||||
# The pipeline parse worker catches ``except Exception``; SystemExit
|
||||
# (BaseException) would bypass it. Guard against a future regression that
|
||||
# reintroduces a BaseException-only error type.
|
||||
assert issubclass(DocxContentError, Exception)
|
||||
|
||||
|
||||
def test_validate_heading_length_raises_instead_of_exiting():
|
||||
long_heading = "x" * (MAX_HEADING_LENGTH + 213) # mirrors the 413-char report
|
||||
with pytest.raises(DocxContentError) as exc_info:
|
||||
validate_heading_length(long_heading, "p1")
|
||||
message = str(exc_info.value)
|
||||
assert f"Heading too long ({len(long_heading)} characters" in message
|
||||
assert "SOLUTION:" in message
|
||||
|
||||
|
||||
def test_validate_heading_length_does_not_raise_within_limit():
|
||||
# Exactly at the limit and below must pass through silently.
|
||||
validate_heading_length("x" * MAX_HEADING_LENGTH, "p2")
|
||||
validate_heading_length("short heading", "p3")
|
||||
+222
@@ -0,0 +1,222 @@
|
||||
"""Tests for ``lightrag/parser/external/docling/cache.py``.
|
||||
|
||||
Covers the cache-miss conditions enumerated in the module docstring:
|
||||
|
||||
- missing / malformed / wrong-engine manifest
|
||||
- source size or hash mismatch
|
||||
- engine_version / endpoint_signature env mismatch
|
||||
- options_signature env mismatch
|
||||
- critical-file size / sha256 mismatch
|
||||
- non-critical file size mismatch
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from lightrag.parser.external import Manifest, ManifestFile, write_manifest
|
||||
from lightrag.parser.external._common import compute_size_and_hash
|
||||
from lightrag.parser.external.docling.cache import (
|
||||
compute_options_signature,
|
||||
is_bundle_valid,
|
||||
snapshot_tunable_env,
|
||||
)
|
||||
from lightrag.parser.external.docling.client import FIXED_CONSTANTS
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _clear_envs(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
for name in (
|
||||
"DOCLING_DO_OCR",
|
||||
"DOCLING_FORCE_OCR",
|
||||
"DOCLING_OCR_ENGINE",
|
||||
"DOCLING_OCR_PRESET",
|
||||
"DOCLING_OCR_LANG",
|
||||
"DOCLING_DO_FORMULA_ENRICHMENT",
|
||||
"DOCLING_ENGINE_VERSION",
|
||||
"DOCLING_ENDPOINT",
|
||||
):
|
||||
monkeypatch.delenv(name, raising=False)
|
||||
monkeypatch.setenv("DOCLING_ENDPOINT", "http://docling.test")
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def source_file(tmp_path: Path) -> Path:
|
||||
p = tmp_path / "src.pdf"
|
||||
p.write_bytes(b"hello pdf payload" * 64)
|
||||
return p
|
||||
|
||||
|
||||
def test_snapshot_tunable_env_uses_effective_defaults(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
unset = snapshot_tunable_env()
|
||||
|
||||
monkeypatch.setenv("DOCLING_DO_OCR", "true")
|
||||
monkeypatch.setenv("DOCLING_FORCE_OCR", "true")
|
||||
monkeypatch.setenv("DOCLING_OCR_ENGINE", "auto")
|
||||
monkeypatch.setenv("DOCLING_OCR_PRESET", "auto")
|
||||
monkeypatch.setenv("DOCLING_OCR_LANG", "")
|
||||
monkeypatch.setenv("DOCLING_DO_FORMULA_ENRICHMENT", "false")
|
||||
|
||||
assert snapshot_tunable_env() == unset
|
||||
|
||||
|
||||
def _build_valid_bundle(
|
||||
tmp_path: Path,
|
||||
source_file: Path,
|
||||
*,
|
||||
options_signature: str | None = None,
|
||||
) -> Path:
|
||||
raw_dir = tmp_path / "src.docling_raw"
|
||||
raw_dir.mkdir()
|
||||
main_json = raw_dir / "src.json"
|
||||
main_json.write_text('{"schema_name": "DoclingDocument"}', encoding="utf-8")
|
||||
md = raw_dir / "src.md"
|
||||
md.write_text("# title", encoding="utf-8")
|
||||
|
||||
src_size, src_hash = compute_size_and_hash(source_file)
|
||||
crit_size, crit_hash = compute_size_and_hash(main_json)
|
||||
sig = options_signature
|
||||
if sig is None:
|
||||
sig = compute_options_signature(
|
||||
tunable_env=snapshot_tunable_env(),
|
||||
fixed_constants=FIXED_CONSTANTS,
|
||||
)
|
||||
manifest = Manifest(
|
||||
engine="docling",
|
||||
source_content_hash=src_hash,
|
||||
source_size_bytes=src_size,
|
||||
source_filename_at_parse=source_file.name,
|
||||
critical_file=ManifestFile(path="src.json", size=crit_size, sha256=crit_hash),
|
||||
files=[ManifestFile(path="src.md", size=md.stat().st_size)],
|
||||
total_size_bytes=crit_size + md.stat().st_size,
|
||||
task_id="task-1",
|
||||
endpoint_signature="http://docling.test",
|
||||
engine_version="",
|
||||
options_signature=sig,
|
||||
extras={"fixed_constants": dict(FIXED_CONSTANTS)},
|
||||
)
|
||||
write_manifest(raw_dir, manifest)
|
||||
return raw_dir
|
||||
|
||||
|
||||
def test_is_bundle_valid_happy_path(tmp_path: Path, source_file: Path) -> None:
|
||||
raw = _build_valid_bundle(tmp_path, source_file)
|
||||
assert is_bundle_valid(raw, source_file) is True
|
||||
|
||||
|
||||
def test_is_bundle_valid_missing_dir(tmp_path: Path, source_file: Path) -> None:
|
||||
assert is_bundle_valid(tmp_path / "ghost", source_file) is False
|
||||
|
||||
|
||||
def test_is_bundle_valid_missing_manifest(tmp_path: Path, source_file: Path) -> None:
|
||||
raw = tmp_path / "src.docling_raw"
|
||||
raw.mkdir()
|
||||
(raw / "src.json").write_text("{}")
|
||||
assert is_bundle_valid(raw, source_file) is False
|
||||
|
||||
|
||||
def test_is_bundle_valid_wrong_engine(tmp_path: Path, source_file: Path) -> None:
|
||||
raw = _build_valid_bundle(tmp_path, source_file)
|
||||
manifest_path = raw / "_manifest.json"
|
||||
data = manifest_path.read_text(encoding="utf-8")
|
||||
manifest_path.write_text(data.replace('"docling"', '"mineru"'), encoding="utf-8")
|
||||
assert is_bundle_valid(raw, source_file) is False
|
||||
|
||||
|
||||
def test_is_bundle_valid_source_size_mismatch(
|
||||
tmp_path: Path, source_file: Path
|
||||
) -> None:
|
||||
raw = _build_valid_bundle(tmp_path, source_file)
|
||||
source_file.write_bytes(source_file.read_bytes() + b"!")
|
||||
assert is_bundle_valid(raw, source_file) is False
|
||||
|
||||
|
||||
def test_is_bundle_valid_source_hash_mismatch(
|
||||
tmp_path: Path, source_file: Path
|
||||
) -> None:
|
||||
raw = _build_valid_bundle(tmp_path, source_file)
|
||||
# Replace contents with same length but different bytes
|
||||
new = b"Y" * source_file.stat().st_size
|
||||
source_file.write_bytes(new)
|
||||
assert is_bundle_valid(raw, source_file) is False
|
||||
|
||||
|
||||
def test_is_bundle_valid_endpoint_change(
|
||||
tmp_path: Path, source_file: Path, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
raw = _build_valid_bundle(tmp_path, source_file)
|
||||
monkeypatch.setenv("DOCLING_ENDPOINT", "http://other:5001")
|
||||
assert is_bundle_valid(raw, source_file) is False
|
||||
|
||||
|
||||
def test_is_bundle_valid_options_signature_change(
|
||||
tmp_path: Path, source_file: Path, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
raw = _build_valid_bundle(tmp_path, source_file)
|
||||
monkeypatch.setenv("DOCLING_FORCE_OCR", "false")
|
||||
assert is_bundle_valid(raw, source_file) is False
|
||||
|
||||
|
||||
def test_is_bundle_valid_legacy_bundle_misses_with_overrides(
|
||||
tmp_path: Path, source_file: Path
|
||||
) -> None:
|
||||
# A legacy bundle predating signature recording (empty options_signature)
|
||||
# is leniently accepted with no overrides, but a per-file override
|
||||
# (docling(force_ocr=...)) must force a miss — we cannot prove the bundle
|
||||
# was produced with that override, and silently reusing it would drop the
|
||||
# user's explicit param. Mirrors MinerU, which misses on any absent sig.
|
||||
raw = _build_valid_bundle(tmp_path, source_file, options_signature="")
|
||||
assert is_bundle_valid(raw, source_file) is True
|
||||
assert is_bundle_valid(raw, source_file, overrides={"force_ocr": False}) is False
|
||||
assert is_bundle_valid(raw, source_file, overrides={"force_ocr": True}) is False
|
||||
|
||||
|
||||
def test_is_bundle_valid_fixed_constants_code_change(
|
||||
tmp_path: Path, source_file: Path
|
||||
) -> None:
|
||||
# Simulate a code-only change to one of the fixed pipeline constants
|
||||
# (e.g. image_export_mode flipped from "referenced" to "embedded"
|
||||
# between parse time and validation time). The manifest stores both
|
||||
# the stale constants and a signature computed from them; validation
|
||||
# must compare against current FIXED_CONSTANTS and miss, not against
|
||||
# the manifest's own copy (which would always match).
|
||||
stale_constants = {**FIXED_CONSTANTS, "image_export_mode": "embedded"}
|
||||
stale_signature = compute_options_signature(
|
||||
tunable_env=snapshot_tunable_env(),
|
||||
fixed_constants=stale_constants,
|
||||
)
|
||||
raw = _build_valid_bundle(tmp_path, source_file, options_signature=stale_signature)
|
||||
# Overwrite the manifest's extras to record the stale constants too —
|
||||
# this is the bug surface: if validation rehydrated from extras, it
|
||||
# would reproduce stale_signature and falsely accept the bundle.
|
||||
import json as _json
|
||||
|
||||
mp = raw / "_manifest.json"
|
||||
data = _json.loads(mp.read_text(encoding="utf-8"))
|
||||
data["extras"] = {"fixed_constants": stale_constants}
|
||||
mp.write_text(_json.dumps(data), encoding="utf-8")
|
||||
|
||||
assert is_bundle_valid(raw, source_file) is False
|
||||
|
||||
|
||||
def test_is_bundle_valid_critical_file_corrupt(
|
||||
tmp_path: Path, source_file: Path
|
||||
) -> None:
|
||||
raw = _build_valid_bundle(tmp_path, source_file)
|
||||
# Corrupt the JSON: same length, different bytes — defeats size check,
|
||||
# so the sha256 path must catch it.
|
||||
current = (raw / "src.json").read_bytes()
|
||||
(raw / "src.json").write_bytes(b"X" * len(current))
|
||||
assert is_bundle_valid(raw, source_file) is False
|
||||
|
||||
|
||||
def test_is_bundle_valid_other_file_size_mismatch(
|
||||
tmp_path: Path, source_file: Path
|
||||
) -> None:
|
||||
raw = _build_valid_bundle(tmp_path, source_file)
|
||||
(raw / "src.md").write_text("totally different content here that is longer")
|
||||
assert is_bundle_valid(raw, source_file) is False
|
||||
+596
@@ -0,0 +1,596 @@
|
||||
"""Tests for :class:`DoclingRawClient`.
|
||||
|
||||
Cover the contract guarantees that protect the sidecar pipeline:
|
||||
|
||||
- the fixed pipeline constants (``pipeline=standard`` / ``target_type=zip``
|
||||
/ ``to_formats=[json,md]`` / ``image_export_mode=referenced``) are sent
|
||||
on every upload, regardless of env;
|
||||
- terminal non-success states (``failure`` / ``partial_success`` /
|
||||
``skipped``) abort the run **before** any result download;
|
||||
- ``DOCLING_OCR_LANG`` is omitted when empty so docling-serve falls back
|
||||
to its own default.
|
||||
|
||||
Uses an in-process fake httpx client mirroring ``tests/parser/external/mineru/test_client.py``
|
||||
so we don't trip httpx's sync/async stream guard on multipart uploads.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import io
|
||||
import json
|
||||
import zipfile
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
from urllib.parse import quote
|
||||
|
||||
import pytest
|
||||
|
||||
from lightrag.parser.external.docling.client import (
|
||||
CONVERT_PATH,
|
||||
POLL_PATH,
|
||||
RESULT_PATH,
|
||||
DoclingRawClient,
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Minimal httpx fake (no MockTransport — avoids the multipart encode path)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class _FakeResponse:
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
status_code: int = 200,
|
||||
text: str = "",
|
||||
content: bytes = b"",
|
||||
headers: dict[str, str] | None = None,
|
||||
) -> None:
|
||||
self.status_code = status_code
|
||||
self.text = text
|
||||
self.content = content or text.encode("utf-8")
|
||||
self.headers = headers or {}
|
||||
|
||||
def json(self) -> Any:
|
||||
return json.loads(self.text) if self.text else {}
|
||||
|
||||
def raise_for_status(self) -> None:
|
||||
if self.status_code >= 400:
|
||||
raise RuntimeError(f"HTTP {self.status_code}")
|
||||
|
||||
|
||||
class _Recorder:
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
terminal_status: str,
|
||||
zip_bytes: bytes,
|
||||
task_id: str = "task-abc",
|
||||
submit_status_code: int = 200,
|
||||
submit_text: str | None = None,
|
||||
poll_status_code: int = 200,
|
||||
poll_text: str | None = None,
|
||||
result_status_code: int = 200,
|
||||
result_text: str | None = None,
|
||||
) -> None:
|
||||
self.terminal_status = terminal_status
|
||||
self.zip_bytes = zip_bytes
|
||||
self.task_id = task_id
|
||||
self.submit_status_code = submit_status_code
|
||||
self.submit_text = submit_text
|
||||
self.poll_status_code = poll_status_code
|
||||
self.poll_text = poll_text
|
||||
self.result_status_code = result_status_code
|
||||
self.result_text = result_text
|
||||
|
||||
self.post_calls: list[dict] = []
|
||||
self.get_calls: list[dict] = []
|
||||
self.result_calls = 0
|
||||
|
||||
|
||||
_CURRENT: dict[str, _Recorder] = {}
|
||||
|
||||
|
||||
class _FakeAsyncClient:
|
||||
def __init__(self, *_: Any, **__: Any) -> None:
|
||||
pass
|
||||
|
||||
async def __aenter__(self) -> "_FakeAsyncClient":
|
||||
return self
|
||||
|
||||
async def __aexit__(self, *_: Any) -> None:
|
||||
pass
|
||||
|
||||
async def post(
|
||||
self,
|
||||
url: str,
|
||||
files: Any = None,
|
||||
data: Any = None,
|
||||
json: Any = None,
|
||||
headers: Any = None,
|
||||
) -> _FakeResponse:
|
||||
recorder = _CURRENT["recorder"]
|
||||
# Production passes a file handle inside a `with` block — by the time
|
||||
# tests inspect `post_calls` it's already closed. Drain the stream
|
||||
# here so assertions can keep reading the payload as bytes.
|
||||
snapshot_files = files
|
||||
if files and "files" in files:
|
||||
name, payload, ctype = files["files"]
|
||||
if hasattr(payload, "read"):
|
||||
payload = payload.read()
|
||||
snapshot_files = {"files": (name, payload, ctype)}
|
||||
recorder.post_calls.append(
|
||||
{"url": url, "files": snapshot_files, "data": data, "json": json}
|
||||
)
|
||||
if CONVERT_PATH in url:
|
||||
if recorder.submit_status_code != 200:
|
||||
return _FakeResponse(
|
||||
status_code=recorder.submit_status_code,
|
||||
text=recorder.submit_text or "",
|
||||
)
|
||||
return _FakeResponse(
|
||||
status_code=200,
|
||||
text=json_dump({"task_id": recorder.task_id}),
|
||||
)
|
||||
raise AssertionError(f"unexpected POST {url}")
|
||||
|
||||
async def get(
|
||||
self, url: str, params: Any = None, headers: Any = None
|
||||
) -> _FakeResponse:
|
||||
recorder = _CURRENT["recorder"]
|
||||
recorder.get_calls.append({"url": url, "params": params})
|
||||
# Mirror production: the client encodes the task id into a single path
|
||||
# segment, so route on the encoded form (a no-op for URL-safe ids).
|
||||
encoded = quote(recorder.task_id, safe="")
|
||||
if POLL_PATH.format(task_id=encoded) in url:
|
||||
if recorder.poll_status_code != 200:
|
||||
return _FakeResponse(
|
||||
status_code=recorder.poll_status_code,
|
||||
text=recorder.poll_text or "",
|
||||
)
|
||||
payload: dict[str, Any] = {
|
||||
"task_id": recorder.task_id,
|
||||
"task_status": recorder.terminal_status,
|
||||
}
|
||||
if recorder.terminal_status != "success":
|
||||
payload["error_message"] = "synthetic-failure"
|
||||
return _FakeResponse(status_code=200, text=json_dump(payload))
|
||||
if RESULT_PATH.format(task_id=encoded) in url:
|
||||
recorder.result_calls += 1
|
||||
if recorder.result_status_code != 200:
|
||||
return _FakeResponse(
|
||||
status_code=recorder.result_status_code,
|
||||
text=recorder.result_text or "",
|
||||
)
|
||||
return _FakeResponse(
|
||||
status_code=200,
|
||||
content=recorder.zip_bytes,
|
||||
headers={"content-type": "application/zip"},
|
||||
)
|
||||
raise AssertionError(f"unexpected GET {url}")
|
||||
|
||||
|
||||
def json_dump(payload: Any) -> str:
|
||||
return json.dumps(payload)
|
||||
|
||||
|
||||
def _form_pairs(data: Any) -> list[tuple[str, str]]:
|
||||
"""Normalize httpx form data into repeated ``(name, value)`` pairs.
|
||||
|
||||
Production passes a mapping so httpx 0.28 keeps multipart ``files=`` on
|
||||
the async path. List values in that mapping represent repeated form keys.
|
||||
Older tests used tuple lists directly; accepting both keeps assertions
|
||||
focused on the wire contract instead of the container type.
|
||||
"""
|
||||
if isinstance(data, dict):
|
||||
pairs: list[tuple[str, str]] = []
|
||||
for name, value in data.items():
|
||||
values = value if isinstance(value, list) else [value]
|
||||
pairs.extend((str(name), str(v)) for v in values)
|
||||
return pairs
|
||||
return [(str(name), str(value)) for name, value in data]
|
||||
|
||||
|
||||
def _fake_zip_with_main_json(stem: str) -> bytes:
|
||||
buf = io.BytesIO()
|
||||
with zipfile.ZipFile(buf, "w") as zf:
|
||||
zf.writestr(f"{stem}.json", b'{"schema_name": "DoclingDocument"}')
|
||||
zf.writestr(f"{stem}.md", b"# hello")
|
||||
return buf.getvalue()
|
||||
|
||||
|
||||
def _install_fake_httpx(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
"""Replace ``httpx.AsyncClient`` and ``httpx.Timeout`` references in
|
||||
the docling client module with no-arg fakes."""
|
||||
monkeypatch.setattr(
|
||||
"lightrag.parser.external.docling.client.httpx.AsyncClient",
|
||||
_FakeAsyncClient,
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
"lightrag.parser.external.docling.client.httpx.Timeout",
|
||||
lambda *a, **kw: None,
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Fixtures
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def source_pdf(tmp_path: Path) -> Path:
|
||||
p = tmp_path / "demo.pdf"
|
||||
p.write_bytes(b"%PDF-1.4 fake")
|
||||
return p
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def docling_endpoint(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
monkeypatch.setenv("DOCLING_ENDPOINT", "http://docling.test")
|
||||
for name in (
|
||||
"DOCLING_DO_OCR",
|
||||
"DOCLING_FORCE_OCR",
|
||||
"DOCLING_OCR_ENGINE",
|
||||
"DOCLING_OCR_PRESET",
|
||||
"DOCLING_OCR_LANG",
|
||||
"DOCLING_DO_FORMULA_ENRICHMENT",
|
||||
"DOCLING_ENGINE_VERSION",
|
||||
):
|
||||
monkeypatch.delenv(name, raising=False)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Tests
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
async def test_docling_client_sends_fixed_constants(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
tmp_path: Path,
|
||||
source_pdf: Path,
|
||||
) -> None:
|
||||
recorder = _Recorder(
|
||||
terminal_status="success",
|
||||
zip_bytes=_fake_zip_with_main_json("demo"),
|
||||
)
|
||||
_CURRENT["recorder"] = recorder
|
||||
_install_fake_httpx(monkeypatch)
|
||||
|
||||
raw_dir = tmp_path / "demo.docling_raw"
|
||||
manifest = await DoclingRawClient().download_into(raw_dir, source_pdf)
|
||||
|
||||
assert len(recorder.post_calls) == 1
|
||||
data = recorder.post_calls[0]["data"]
|
||||
field_map: dict[str, list[str]] = {}
|
||||
for name, value in _form_pairs(data):
|
||||
field_map.setdefault(name, []).append(value)
|
||||
|
||||
assert field_map["pipeline"] == ["standard"]
|
||||
assert field_map["target_type"] == ["zip"]
|
||||
assert field_map["image_export_mode"] == ["referenced"]
|
||||
assert sorted(field_map["to_formats"]) == ["json", "md"]
|
||||
|
||||
files = recorder.post_calls[0]["files"]
|
||||
assert "files" in files
|
||||
name, blob, ctype = files["files"]
|
||||
assert name == "demo.pdf"
|
||||
assert blob.startswith(b"%PDF-1.4")
|
||||
assert ctype == "application/octet-stream"
|
||||
|
||||
assert manifest.task_id == recorder.task_id
|
||||
assert manifest.engine == "docling"
|
||||
assert manifest.extras["fixed_constants"]["pipeline"] == "standard"
|
||||
assert manifest.endpoint_signature == "http://docling.test"
|
||||
|
||||
|
||||
async def test_docling_client_partial_success_aborts(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
tmp_path: Path,
|
||||
source_pdf: Path,
|
||||
) -> None:
|
||||
recorder = _Recorder(
|
||||
terminal_status="partial_success",
|
||||
zip_bytes=_fake_zip_with_main_json("demo"),
|
||||
)
|
||||
_CURRENT["recorder"] = recorder
|
||||
_install_fake_httpx(monkeypatch)
|
||||
|
||||
with pytest.raises(RuntimeError) as excinfo:
|
||||
await DoclingRawClient().download_into(
|
||||
tmp_path / "demo.docling_raw", source_pdf
|
||||
)
|
||||
msg = str(excinfo.value)
|
||||
assert recorder.task_id in msg
|
||||
assert "partial_success" in msg
|
||||
assert "synthetic-failure" in msg
|
||||
assert recorder.result_calls == 0
|
||||
|
||||
|
||||
async def test_docling_client_failure_aborts(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
tmp_path: Path,
|
||||
source_pdf: Path,
|
||||
) -> None:
|
||||
recorder = _Recorder(
|
||||
terminal_status="failure",
|
||||
zip_bytes=_fake_zip_with_main_json("demo"),
|
||||
)
|
||||
_CURRENT["recorder"] = recorder
|
||||
_install_fake_httpx(monkeypatch)
|
||||
|
||||
with pytest.raises(RuntimeError):
|
||||
await DoclingRawClient().download_into(
|
||||
tmp_path / "demo.docling_raw", source_pdf
|
||||
)
|
||||
assert recorder.result_calls == 0
|
||||
|
||||
|
||||
async def test_docling_client_skipped_aborts(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
tmp_path: Path,
|
||||
source_pdf: Path,
|
||||
) -> None:
|
||||
recorder = _Recorder(
|
||||
terminal_status="skipped",
|
||||
zip_bytes=_fake_zip_with_main_json("demo"),
|
||||
)
|
||||
_CURRENT["recorder"] = recorder
|
||||
_install_fake_httpx(monkeypatch)
|
||||
|
||||
with pytest.raises(RuntimeError):
|
||||
await DoclingRawClient().download_into(
|
||||
tmp_path / "demo.docling_raw", source_pdf
|
||||
)
|
||||
assert recorder.result_calls == 0
|
||||
|
||||
|
||||
async def test_docling_client_upload_http_error_preserves_response_body(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
tmp_path: Path,
|
||||
source_pdf: Path,
|
||||
) -> None:
|
||||
recorder = _Recorder(
|
||||
terminal_status="success",
|
||||
zip_bytes=_fake_zip_with_main_json("demo"),
|
||||
submit_status_code=400,
|
||||
submit_text=json_dump({"detail": "unsupported file type"}),
|
||||
)
|
||||
_CURRENT["recorder"] = recorder
|
||||
_install_fake_httpx(monkeypatch)
|
||||
|
||||
with pytest.raises(RuntimeError) as excinfo:
|
||||
await DoclingRawClient().download_into(
|
||||
tmp_path / "demo.docling_raw", source_pdf
|
||||
)
|
||||
|
||||
message = str(excinfo.value)
|
||||
assert "Docling upload for 'demo.pdf'" in message
|
||||
assert "HTTP 400" in message
|
||||
assert "unsupported file type" in message
|
||||
|
||||
|
||||
async def test_docling_client_poll_http_error_preserves_response_body(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
tmp_path: Path,
|
||||
source_pdf: Path,
|
||||
) -> None:
|
||||
recorder = _Recorder(
|
||||
terminal_status="success",
|
||||
zip_bytes=_fake_zip_with_main_json("demo"),
|
||||
poll_status_code=503,
|
||||
poll_text=json_dump({"message": "queue unavailable"}),
|
||||
)
|
||||
_CURRENT["recorder"] = recorder
|
||||
_install_fake_httpx(monkeypatch)
|
||||
|
||||
with pytest.raises(RuntimeError) as excinfo:
|
||||
await DoclingRawClient().download_into(
|
||||
tmp_path / "demo.docling_raw", source_pdf
|
||||
)
|
||||
|
||||
message = str(excinfo.value)
|
||||
assert "Docling task task-abc poll" in message
|
||||
assert "HTTP 503" in message
|
||||
assert "queue unavailable" in message
|
||||
|
||||
|
||||
async def test_docling_client_result_redirect_treated_as_error(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
tmp_path: Path,
|
||||
source_pdf: Path,
|
||||
) -> None:
|
||||
# docling-serve fronted by a misconfigured proxy could emit a 302 to a
|
||||
# CDN that httpx (default ``follow_redirects=False``) won't follow.
|
||||
# Without the explicit non-2xx guard the redirect body would fall into
|
||||
# the zip-decoder and surface as a cryptic "bad zip" error.
|
||||
recorder = _Recorder(
|
||||
terminal_status="success",
|
||||
zip_bytes=_fake_zip_with_main_json("demo"),
|
||||
result_status_code=302,
|
||||
result_text="",
|
||||
)
|
||||
_CURRENT["recorder"] = recorder
|
||||
_install_fake_httpx(monkeypatch)
|
||||
|
||||
with pytest.raises(RuntimeError) as excinfo:
|
||||
await DoclingRawClient().download_into(
|
||||
tmp_path / "demo.docling_raw", source_pdf
|
||||
)
|
||||
|
||||
message = str(excinfo.value)
|
||||
assert "Docling result task-abc download" in message
|
||||
assert "HTTP 302" in message
|
||||
|
||||
|
||||
async def test_docling_client_result_http_error_preserves_response_body(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
tmp_path: Path,
|
||||
source_pdf: Path,
|
||||
) -> None:
|
||||
recorder = _Recorder(
|
||||
terminal_status="success",
|
||||
zip_bytes=_fake_zip_with_main_json("demo"),
|
||||
result_status_code=500,
|
||||
result_text="zip artifact missing",
|
||||
)
|
||||
_CURRENT["recorder"] = recorder
|
||||
_install_fake_httpx(monkeypatch)
|
||||
|
||||
with pytest.raises(RuntimeError) as excinfo:
|
||||
await DoclingRawClient().download_into(
|
||||
tmp_path / "demo.docling_raw", source_pdf
|
||||
)
|
||||
|
||||
message = str(excinfo.value)
|
||||
assert "Docling result task-abc download" in message
|
||||
assert "HTTP 500" in message
|
||||
assert "zip artifact missing" in message
|
||||
|
||||
|
||||
async def test_docling_client_encodes_task_id_into_url_path_segment(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
tmp_path: Path,
|
||||
source_pdf: Path,
|
||||
) -> None:
|
||||
# Security regression (CWE-116): the poll/result task id is service-returned.
|
||||
# A crafted value with ``/`` / ``?`` / ``..`` must be percent-encoded into a
|
||||
# single path segment so it cannot escape ``/v1/status/poll/{id}`` or append
|
||||
# a query string to the request the client issues.
|
||||
malicious = "../admin?x=1"
|
||||
recorder = _Recorder(
|
||||
terminal_status="success",
|
||||
zip_bytes=_fake_zip_with_main_json("demo"),
|
||||
task_id=malicious,
|
||||
)
|
||||
_CURRENT["recorder"] = recorder
|
||||
_install_fake_httpx(monkeypatch)
|
||||
|
||||
await DoclingRawClient().download_into(tmp_path / "demo.docling_raw", source_pdf)
|
||||
|
||||
poll_url = recorder.get_calls[0]["url"]
|
||||
result_url = recorder.get_calls[-1]["url"]
|
||||
for url in (poll_url, result_url):
|
||||
# ``..`` survives (dot is unreserved) but the separators that grant
|
||||
# request-structure control are neutralized.
|
||||
assert "..%2Fadmin%3Fx%3D1" in url
|
||||
assert "/admin" not in url
|
||||
assert "?x=1" not in url
|
||||
|
||||
|
||||
async def test_docling_client_ocr_lang_omitted_when_empty(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
tmp_path: Path,
|
||||
source_pdf: Path,
|
||||
) -> None:
|
||||
recorder = _Recorder(
|
||||
terminal_status="success",
|
||||
zip_bytes=_fake_zip_with_main_json("demo"),
|
||||
)
|
||||
_CURRENT["recorder"] = recorder
|
||||
_install_fake_httpx(monkeypatch)
|
||||
|
||||
await DoclingRawClient().download_into(tmp_path / "demo.docling_raw", source_pdf)
|
||||
|
||||
data = recorder.post_calls[0]["data"]
|
||||
names = [name for name, _ in _form_pairs(data)]
|
||||
assert "ocr_lang" not in names
|
||||
|
||||
|
||||
async def test_docling_client_ocr_lang_sent_when_set(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
tmp_path: Path,
|
||||
source_pdf: Path,
|
||||
) -> None:
|
||||
monkeypatch.setenv("DOCLING_OCR_LANG", '["en","zh"]')
|
||||
recorder = _Recorder(
|
||||
terminal_status="success",
|
||||
zip_bytes=_fake_zip_with_main_json("demo"),
|
||||
)
|
||||
_CURRENT["recorder"] = recorder
|
||||
_install_fake_httpx(monkeypatch)
|
||||
|
||||
await DoclingRawClient().download_into(tmp_path / "demo.docling_raw", source_pdf)
|
||||
|
||||
data = recorder.post_calls[0]["data"]
|
||||
langs = [v for name, v in _form_pairs(data) if name == "ocr_lang"]
|
||||
assert langs == ["en", "zh"]
|
||||
|
||||
|
||||
async def test_docling_client_ocr_lang_csv_form(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
tmp_path: Path,
|
||||
source_pdf: Path,
|
||||
) -> None:
|
||||
"""CSV fallback when value isn't valid JSON."""
|
||||
monkeypatch.setenv("DOCLING_OCR_LANG", "en, fr")
|
||||
recorder = _Recorder(
|
||||
terminal_status="success",
|
||||
zip_bytes=_fake_zip_with_main_json("demo"),
|
||||
)
|
||||
_CURRENT["recorder"] = recorder
|
||||
_install_fake_httpx(monkeypatch)
|
||||
|
||||
await DoclingRawClient().download_into(tmp_path / "demo.docling_raw", source_pdf)
|
||||
|
||||
data = recorder.post_calls[0]["data"]
|
||||
langs = [v for name, v in _form_pairs(data) if name == "ocr_lang"]
|
||||
assert langs == ["en", "fr"]
|
||||
|
||||
|
||||
async def test_docling_client_rejects_missing_endpoint(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
monkeypatch.setenv("DOCLING_ENDPOINT", "")
|
||||
with pytest.raises(ValueError, match="DOCLING_ENDPOINT"):
|
||||
DoclingRawClient()
|
||||
|
||||
|
||||
async def test_docling_client_strips_parser_hint_from_upload_filename(
|
||||
monkeypatch: pytest.MonkeyPatch, tmp_path: Path
|
||||
) -> None:
|
||||
# Regression: a hinted source (``report.[docling].pdf``) used to cause
|
||||
# docling-serve to name its bundle JSON ``report.[docling].json``, which
|
||||
# the adapter (looking for ``report.json``) could not locate. The
|
||||
# pipeline now passes the canonical name as ``upload_filename`` so the
|
||||
# bundle is canonical-stem from the start.
|
||||
hinted = tmp_path / "report.[docling].pdf"
|
||||
hinted.write_bytes(b"%PDF-1.4 fake")
|
||||
# The fake zip mimics docling-serve responding with the *canonical* stem,
|
||||
# which is what would happen once we send the canonical filename.
|
||||
recorder = _Recorder(
|
||||
terminal_status="success",
|
||||
zip_bytes=_fake_zip_with_main_json("report"),
|
||||
)
|
||||
_CURRENT["recorder"] = recorder
|
||||
_install_fake_httpx(monkeypatch)
|
||||
|
||||
raw_dir = tmp_path / "report.docling_raw"
|
||||
manifest = await DoclingRawClient().download_into(
|
||||
raw_dir, hinted, upload_filename="report.pdf"
|
||||
)
|
||||
|
||||
name, _blob, _ctype = recorder.post_calls[0]["files"]["files"]
|
||||
assert name == "report.pdf"
|
||||
assert manifest.source_filename_at_parse == "report.pdf"
|
||||
assert manifest.critical_file.path == "report.json"
|
||||
assert (raw_dir / "report.json").is_file()
|
||||
|
||||
|
||||
async def test_docling_client_default_upload_filename_falls_back_to_source_name(
|
||||
monkeypatch: pytest.MonkeyPatch, tmp_path: Path, source_pdf: Path
|
||||
) -> None:
|
||||
# Back-compat guard: callers that don't pass ``upload_filename`` (any
|
||||
# path other than the production pipeline) keep the legacy behavior of
|
||||
# using the on-disk source filename.
|
||||
recorder = _Recorder(
|
||||
terminal_status="success",
|
||||
zip_bytes=_fake_zip_with_main_json("demo"),
|
||||
)
|
||||
_CURRENT["recorder"] = recorder
|
||||
_install_fake_httpx(monkeypatch)
|
||||
|
||||
await DoclingRawClient().download_into(tmp_path / "demo.docling_raw", source_pdf)
|
||||
|
||||
name, _blob, _ctype = recorder.post_calls[0]["files"]["files"]
|
||||
assert name == "demo.pdf"
|
||||
+1138
File diff suppressed because it is too large
Load Diff
+57
@@ -0,0 +1,57 @@
|
||||
"""Tests for ``lightrag/parser/external/docling/manifest.py`` helpers.
|
||||
|
||||
Targets the contract guarantees that the rest of the docling flow relies on:
|
||||
``select_main_json`` must find the bundle's main JSON even when ``_manifest.json``
|
||||
sits alongside it, and the preferred-path lookup must take priority over the
|
||||
fallback glob.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from lightrag.parser.external.docling.manifest import select_main_json
|
||||
|
||||
|
||||
def _touch(path: Path, content: str = "{}") -> None:
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
path.write_text(content, encoding="utf-8")
|
||||
|
||||
|
||||
def test_select_main_json_preferred_path_hits(tmp_path: Path) -> None:
|
||||
# Manifest is present (the typical post-download state), but the preferred
|
||||
# ``<stem>.json`` exists, so the fallback glob is not consulted at all.
|
||||
_touch(tmp_path / "report.json")
|
||||
_touch(tmp_path / "_manifest.json", '{"engine":"docling"}')
|
||||
assert select_main_json(tmp_path, Path("report.pdf")) == tmp_path / "report.json"
|
||||
|
||||
|
||||
def test_select_main_json_fallback_ignores_manifest(tmp_path: Path) -> None:
|
||||
# Defensive: when the preferred path misses (e.g. docling-serve renamed
|
||||
# the stem for whatever reason), the fallback glob must NOT confuse
|
||||
# ``_manifest.json`` for a bundle JSON. Pre-fix this case raised
|
||||
# "multiple .json candidates".
|
||||
_touch(tmp_path / "report.json")
|
||||
_touch(tmp_path / "_manifest.json", '{"engine":"docling"}')
|
||||
assert select_main_json(tmp_path, Path("other.pdf")) == tmp_path / "report.json"
|
||||
|
||||
|
||||
def test_select_main_json_raises_when_only_manifest_present(tmp_path: Path) -> None:
|
||||
# If the bundle JSON is genuinely missing, the manifest alone is not a
|
||||
# valid substitute — the helper must still raise rather than silently
|
||||
# returning the manifest.
|
||||
_touch(tmp_path / "_manifest.json", '{"engine":"docling"}')
|
||||
with pytest.raises(RuntimeError, match="contains no .json file"):
|
||||
select_main_json(tmp_path, Path("report.pdf"))
|
||||
|
||||
|
||||
def test_select_main_json_raises_on_real_ambiguity(tmp_path: Path) -> None:
|
||||
# Two genuine bundle JSONs is still an error; the manifest filter must
|
||||
# not mask multi-candidate detection.
|
||||
_touch(tmp_path / "report.json")
|
||||
_touch(tmp_path / "extra.json")
|
||||
_touch(tmp_path / "_manifest.json", '{"engine":"docling"}')
|
||||
with pytest.raises(RuntimeError, match="multiple .json candidates"):
|
||||
select_main_json(tmp_path, Path("other.pdf"))
|
||||
@@ -0,0 +1,668 @@
|
||||
"""Integration tests for ``parse_docling`` with the unified sidecar pipeline.
|
||||
|
||||
Stubs :class:`DoclingRawClient.download_into` so no real docling-serve is
|
||||
contacted; the focus is on:
|
||||
|
||||
- happy path: cache miss → fake bundle written → sidecar emitted with all
|
||||
expected files at the spec-compliant locations
|
||||
- cache hit: a pre-existing valid ``*.docling_raw/`` + manifest causes
|
||||
``DoclingRawClient.download_into`` NOT to be called
|
||||
- ``LIGHTRAG_FORCE_REPARSE_DOCLING=true`` forces a re-download even when
|
||||
the manifest is valid
|
||||
- source content swap → cache miss
|
||||
- options_signature change (``DOCLING_OCR_LANG`` toggle) → cache miss
|
||||
- adapter sees zero blocks → parse fails loudly (no half-baked sidecar)
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
|
||||
from lightrag import LightRAG
|
||||
from lightrag.constants import FULL_DOCS_FORMAT_LIGHTRAG
|
||||
from lightrag.parser.external import (
|
||||
Manifest,
|
||||
ManifestFile,
|
||||
compute_size_and_hash,
|
||||
write_manifest,
|
||||
)
|
||||
from lightrag.parser.external.docling.cache import (
|
||||
compute_options_signature,
|
||||
snapshot_tunable_env,
|
||||
)
|
||||
from lightrag.parser.external.docling.client import FIXED_CONSTANTS
|
||||
from lightrag.parser.base import ParseContext
|
||||
from lightrag.parser.registry import get_parser
|
||||
from lightrag.utils import EmbeddingFunc, Tokenizer
|
||||
|
||||
|
||||
async def _parse_via_registry(rag, engine, doc_id, file_path, content_data):
|
||||
"""Drive a parser the way the pipeline worker does (registry dispatch)."""
|
||||
result = await get_parser(engine).parse(
|
||||
ParseContext(rag, doc_id, file_path, content_data)
|
||||
)
|
||||
return result.to_dict()
|
||||
|
||||
|
||||
class _SimpleTokenizerImpl:
|
||||
def encode(self, content: str) -> list[int]:
|
||||
return [ord(ch) for ch in content]
|
||||
|
||||
def decode(self, tokens: list[int]) -> str:
|
||||
return "".join(chr(t) for t in tokens)
|
||||
|
||||
|
||||
async def _mock_embedding(texts: list[str]) -> np.ndarray:
|
||||
return np.random.rand(len(texts), 32)
|
||||
|
||||
|
||||
async def _mock_llm(prompt: Any, **kwargs: Any) -> str:
|
||||
return '{"name":"x","summary":"s","detail_description":"d"}'
|
||||
|
||||
|
||||
def _new_rag(tmp_path: Path) -> LightRAG:
|
||||
return LightRAG(
|
||||
working_dir=str(tmp_path),
|
||||
workspace=f"test-docling-sidecar-{tmp_path.name}",
|
||||
llm_model_func=_mock_llm,
|
||||
embedding_func=EmbeddingFunc(
|
||||
embedding_dim=32,
|
||||
max_token_size=4096,
|
||||
func=_mock_embedding,
|
||||
),
|
||||
tokenizer=Tokenizer("mock-tokenizer", _SimpleTokenizerImpl()),
|
||||
vlm_process_enable=False,
|
||||
)
|
||||
|
||||
|
||||
_FAKE_DOCLING_JSON = {
|
||||
"schema_name": "DoclingDocument",
|
||||
"version": "1.10.0",
|
||||
"origin": {"filename": "demo.pdf", "mimetype": "application/pdf"},
|
||||
"body": {
|
||||
"self_ref": "#/body",
|
||||
"children": [
|
||||
{"$ref": "#/texts/0"},
|
||||
],
|
||||
"content_layer": "body",
|
||||
"label": "unspecified",
|
||||
},
|
||||
"groups": [],
|
||||
"texts": [
|
||||
{
|
||||
"self_ref": "#/texts/0",
|
||||
"label": "section_header",
|
||||
"text": "Intro",
|
||||
"orig": "Intro",
|
||||
"level": 1,
|
||||
"content_layer": "body",
|
||||
"children": [
|
||||
{"$ref": "#/texts/1"},
|
||||
{"$ref": "#/tables/0"},
|
||||
{"$ref": "#/pictures/0"},
|
||||
{"$ref": "#/texts/2"},
|
||||
],
|
||||
"prov": [
|
||||
{
|
||||
"page_no": 1,
|
||||
"bbox": {
|
||||
"l": 10.0,
|
||||
"t": 100.0,
|
||||
"r": 200.0,
|
||||
"b": 80.0,
|
||||
"coord_origin": "BOTTOMLEFT",
|
||||
},
|
||||
"charspan": [0, 5],
|
||||
}
|
||||
],
|
||||
},
|
||||
{
|
||||
"self_ref": "#/texts/1",
|
||||
"label": "text",
|
||||
"text": "Body paragraph.",
|
||||
"orig": "Body paragraph.",
|
||||
"content_layer": "body",
|
||||
"prov": [
|
||||
{
|
||||
"page_no": 1,
|
||||
"bbox": {
|
||||
"l": 10.0,
|
||||
"t": 60.0,
|
||||
"r": 200.0,
|
||||
"b": 40.0,
|
||||
"coord_origin": "BOTTOMLEFT",
|
||||
},
|
||||
"charspan": [0, 15],
|
||||
}
|
||||
],
|
||||
},
|
||||
{
|
||||
"self_ref": "#/texts/2",
|
||||
"label": "formula",
|
||||
"text": "E = mc^2",
|
||||
"orig": "E = mc^2",
|
||||
"content_layer": "body",
|
||||
"prov": [],
|
||||
},
|
||||
],
|
||||
"tables": [
|
||||
{
|
||||
"self_ref": "#/tables/0",
|
||||
"label": "table",
|
||||
"content_layer": "body",
|
||||
"data": {
|
||||
"num_rows": 2,
|
||||
"num_cols": 2,
|
||||
"grid": [
|
||||
[{"text": "h1"}, {"text": "h2"}],
|
||||
[{"text": "a"}, {"text": "b"}],
|
||||
],
|
||||
},
|
||||
"prov": [],
|
||||
}
|
||||
],
|
||||
"pictures": [
|
||||
{
|
||||
"self_ref": "#/pictures/0",
|
||||
"label": "picture",
|
||||
"content_layer": "body",
|
||||
"image": {"uri": "artifacts/img_000000.png", "mimetype": "image/png"},
|
||||
"prov": [],
|
||||
}
|
||||
],
|
||||
"key_value_items": [],
|
||||
"form_items": [],
|
||||
"pages": {"1": {"size": {"width": 612.0, "height": 792.0}, "page_no": 1}},
|
||||
}
|
||||
|
||||
|
||||
def _install_fake_download(monkeypatch: pytest.MonkeyPatch) -> dict[str, int]:
|
||||
"""Replace ``DoclingRawClient.download_into`` with a recorder that
|
||||
writes a synthetic raw bundle and a valid manifest."""
|
||||
import lightrag.parser.external.docling.client as client_mod
|
||||
|
||||
counters = {"calls": 0}
|
||||
|
||||
async def _fake_download(self, raw_dir: Path, source_file_path: Path, **_kwargs):
|
||||
counters["calls"] += 1
|
||||
raw_dir.mkdir(parents=True, exist_ok=True)
|
||||
main_json = raw_dir / "demo.json"
|
||||
main_json.write_text(json.dumps(_FAKE_DOCLING_JSON), encoding="utf-8")
|
||||
(raw_dir / "demo.md").write_text("# fake md", encoding="utf-8")
|
||||
art = raw_dir / "artifacts"
|
||||
art.mkdir(exist_ok=True)
|
||||
(art / "img_000000.png").write_bytes(b"\x89PNG fake")
|
||||
|
||||
src_size, src_hash = compute_size_and_hash(source_file_path)
|
||||
crit_size, crit_hash = compute_size_and_hash(main_json)
|
||||
others = [
|
||||
ManifestFile(path="demo.md", size=(raw_dir / "demo.md").stat().st_size),
|
||||
ManifestFile(
|
||||
path="artifacts/img_000000.png",
|
||||
size=(art / "img_000000.png").stat().st_size,
|
||||
),
|
||||
]
|
||||
options_signature = compute_options_signature(
|
||||
tunable_env=snapshot_tunable_env(),
|
||||
fixed_constants=FIXED_CONSTANTS,
|
||||
)
|
||||
manifest = Manifest(
|
||||
engine="docling",
|
||||
source_content_hash=src_hash,
|
||||
source_size_bytes=src_size,
|
||||
source_filename_at_parse=source_file_path.name,
|
||||
critical_file=ManifestFile(
|
||||
path="demo.json", size=crit_size, sha256=crit_hash
|
||||
),
|
||||
files=others,
|
||||
total_size_bytes=crit_size + sum(f.size for f in others),
|
||||
task_id=f"fake-{counters['calls']}",
|
||||
endpoint_signature="http://docling.test",
|
||||
options_signature=options_signature,
|
||||
extras={"fixed_constants": dict(FIXED_CONSTANTS)},
|
||||
downloaded_at=datetime.now(timezone.utc).isoformat(timespec="seconds"),
|
||||
)
|
||||
write_manifest(raw_dir, manifest)
|
||||
return manifest
|
||||
|
||||
monkeypatch.setattr(client_mod.DoclingRawClient, "download_into", _fake_download)
|
||||
return counters
|
||||
|
||||
|
||||
def _stub_pipeline(monkeypatch: pytest.MonkeyPatch, rag: LightRAG, src: Path) -> None:
|
||||
"""Common pipeline-level stubs: avoid moving the source file and pin
|
||||
the file resolver to the synthetic path."""
|
||||
|
||||
async def _noop_archive(_p: str) -> None:
|
||||
return None
|
||||
|
||||
import lightrag.pipeline as pipeline_module
|
||||
|
||||
monkeypatch.setattr(
|
||||
pipeline_module,
|
||||
"archive_docx_source_after_full_docs_sync",
|
||||
_noop_archive,
|
||||
)
|
||||
monkeypatch.setattr(rag, "_resolve_source_file_for_parser", lambda _p: str(src))
|
||||
|
||||
|
||||
def _seed_doc_status(rag: LightRAG, doc_id: str) -> Any:
|
||||
return rag.doc_status.upsert(
|
||||
{
|
||||
doc_id: {
|
||||
"status": "PARSING",
|
||||
"content_summary": "",
|
||||
"content_length": 0,
|
||||
"chunks_count": 0,
|
||||
"chunks_list": [],
|
||||
"created_at": "2026-05-18T00:00:00+00:00",
|
||||
"updated_at": "2026-05-18T00:00:00+00:00",
|
||||
"file_path": "demo.pdf",
|
||||
"track_id": "trk",
|
||||
"content_hash": "",
|
||||
"metadata": {},
|
||||
}
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Tests
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@pytest.mark.offline
|
||||
def test_parse_docling_emits_compliant_sidecar(
|
||||
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
async def _run() -> None:
|
||||
monkeypatch.setenv("DOCLING_ENDPOINT", "http://docling.test")
|
||||
counters = _install_fake_download(monkeypatch)
|
||||
|
||||
input_dir = tmp_path / "inputs" / "ws"
|
||||
input_dir.mkdir(parents=True)
|
||||
src = input_dir / "demo.pdf"
|
||||
src.write_bytes(b"PDFPDF" * 256)
|
||||
|
||||
rag = _new_rag(tmp_path)
|
||||
await rag.initialize_storages()
|
||||
try:
|
||||
_stub_pipeline(monkeypatch, rag, src)
|
||||
doc_id = "doc-abcdef0123456789abcdef0123456789"
|
||||
await _seed_doc_status(rag, doc_id)
|
||||
|
||||
parsed = await _parse_via_registry(
|
||||
rag,
|
||||
"docling",
|
||||
doc_id=doc_id,
|
||||
file_path="demo.pdf",
|
||||
content_data={},
|
||||
)
|
||||
assert counters["calls"] == 1
|
||||
|
||||
parsed_dir = Path(parsed["blocks_path"]).parent
|
||||
assert parsed["parse_format"] == FULL_DOCS_FORMAT_LIGHTRAG
|
||||
assert parsed_dir.name == "demo.pdf.parsed"
|
||||
|
||||
files = {p.name for p in parsed_dir.iterdir() if p.is_file()}
|
||||
assert "demo.blocks.jsonl" in files
|
||||
assert "demo.tables.json" in files
|
||||
assert "demo.drawings.json" in files
|
||||
assert "demo.equations.json" in files
|
||||
assert (parsed_dir / "demo.blocks.assets").is_dir()
|
||||
assert (parsed_dir / "demo.blocks.assets" / "img_000000.png").is_file()
|
||||
|
||||
blocks_raw = (parsed_dir / "demo.blocks.jsonl").read_text()
|
||||
lines = blocks_raw.splitlines()
|
||||
meta = json.loads(lines[0])
|
||||
rows = [json.loads(line) for line in lines[1:]]
|
||||
assert meta["parse_engine"] == "docling"
|
||||
assert meta["bbox_attributes"] == {"origin": "LEFTBOTTOM"}
|
||||
assert "max" not in meta["bbox_attributes"]
|
||||
assert "page_sizes" not in meta["bbox_attributes"]
|
||||
assert meta["table_file"] is True
|
||||
assert meta["drawing_file"] is True
|
||||
assert meta["equation_file"] is True
|
||||
# No label="title" in the fixture (matches the typical PDF case
|
||||
# where docling produces only section_headers) → doc_title falls
|
||||
# back to the document stem.
|
||||
assert meta["doc_title"] == "demo"
|
||||
|
||||
contents = " ".join(row.get("content", "") for row in rows)
|
||||
assert '<table id="tb-' in contents
|
||||
assert "<drawing" in contents
|
||||
assert "<equation" in contents
|
||||
|
||||
# Raw bundle preserved next to sidecar
|
||||
raw_dir = parsed_dir.parent / "demo.pdf.docling_raw"
|
||||
assert (raw_dir / "_manifest.json").is_file()
|
||||
assert (raw_dir / "demo.json").is_file()
|
||||
assert (raw_dir / "demo.md").is_file()
|
||||
assert (raw_dir / "artifacts" / "img_000000.png").is_file()
|
||||
|
||||
# Drawing path correctly resolved
|
||||
drawings = json.loads((parsed_dir / "demo.drawings.json").read_text())[
|
||||
"drawings"
|
||||
]
|
||||
(drawing_id, drawing_item) = next(iter(drawings.items()))
|
||||
assert drawing_id.startswith("im-")
|
||||
assert drawing_item["path"] == "demo.blocks.assets/img_000000.png"
|
||||
|
||||
# Table self_ref propagated
|
||||
tables = json.loads((parsed_dir / "demo.tables.json").read_text())["tables"]
|
||||
(_, table_item) = next(iter(tables.items()))
|
||||
assert table_item.get("self_ref") == "#/tables/0"
|
||||
finally:
|
||||
await rag.finalize_storages()
|
||||
|
||||
asyncio.new_event_loop().run_until_complete(_run())
|
||||
|
||||
|
||||
@pytest.mark.offline
|
||||
def test_parse_docling_cache_hit_skips_download(
|
||||
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
async def _run() -> None:
|
||||
monkeypatch.setenv("DOCLING_ENDPOINT", "http://docling.test")
|
||||
counters = _install_fake_download(monkeypatch)
|
||||
|
||||
input_dir = tmp_path / "inputs" / "ws"
|
||||
input_dir.mkdir(parents=True)
|
||||
src = input_dir / "demo.pdf"
|
||||
src.write_bytes(b"PDFPDF" * 256)
|
||||
|
||||
rag = _new_rag(tmp_path)
|
||||
await rag.initialize_storages()
|
||||
try:
|
||||
_stub_pipeline(monkeypatch, rag, src)
|
||||
doc_id = "doc-abcdef0123456789abcdef0123456789"
|
||||
await _seed_doc_status(rag, doc_id)
|
||||
|
||||
await _parse_via_registry(
|
||||
rag,
|
||||
"docling",
|
||||
doc_id=doc_id,
|
||||
file_path="demo.pdf",
|
||||
content_data={},
|
||||
)
|
||||
assert counters["calls"] == 1
|
||||
|
||||
await _parse_via_registry(
|
||||
rag,
|
||||
"docling",
|
||||
doc_id=doc_id,
|
||||
file_path="demo.pdf",
|
||||
content_data={},
|
||||
)
|
||||
assert counters["calls"] == 1, "cache hit must not re-download"
|
||||
|
||||
monkeypatch.setenv("LIGHTRAG_FORCE_REPARSE_DOCLING", "true")
|
||||
await _parse_via_registry(
|
||||
rag,
|
||||
"docling",
|
||||
doc_id=doc_id,
|
||||
file_path="demo.pdf",
|
||||
content_data={},
|
||||
)
|
||||
assert counters["calls"] == 2
|
||||
finally:
|
||||
await rag.finalize_storages()
|
||||
|
||||
asyncio.new_event_loop().run_until_complete(_run())
|
||||
|
||||
|
||||
@pytest.mark.offline
|
||||
def test_parse_docling_cache_invalidates_on_source_change(
|
||||
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
async def _run() -> None:
|
||||
monkeypatch.setenv("DOCLING_ENDPOINT", "http://docling.test")
|
||||
counters = _install_fake_download(monkeypatch)
|
||||
|
||||
input_dir = tmp_path / "inputs" / "ws"
|
||||
input_dir.mkdir(parents=True)
|
||||
src = input_dir / "demo.pdf"
|
||||
src.write_bytes(b"PDFPDF" * 256)
|
||||
|
||||
rag = _new_rag(tmp_path)
|
||||
await rag.initialize_storages()
|
||||
try:
|
||||
_stub_pipeline(monkeypatch, rag, src)
|
||||
doc_id = "doc-abcdef0123456789abcdef0123456789"
|
||||
await _seed_doc_status(rag, doc_id)
|
||||
|
||||
await _parse_via_registry(
|
||||
rag,
|
||||
"docling",
|
||||
doc_id=doc_id,
|
||||
file_path="demo.pdf",
|
||||
content_data={},
|
||||
)
|
||||
assert counters["calls"] == 1
|
||||
|
||||
data = src.read_bytes()
|
||||
src.write_bytes(b"\x00" + data[1:])
|
||||
|
||||
await _parse_via_registry(
|
||||
rag,
|
||||
"docling",
|
||||
doc_id=doc_id,
|
||||
file_path="demo.pdf",
|
||||
content_data={},
|
||||
)
|
||||
assert counters["calls"] == 2
|
||||
finally:
|
||||
await rag.finalize_storages()
|
||||
|
||||
asyncio.new_event_loop().run_until_complete(_run())
|
||||
|
||||
|
||||
@pytest.mark.offline
|
||||
def test_parse_docling_options_signature_invalidates_cache(
|
||||
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
async def _run() -> None:
|
||||
monkeypatch.setenv("DOCLING_ENDPOINT", "http://docling.test")
|
||||
counters = _install_fake_download(monkeypatch)
|
||||
|
||||
input_dir = tmp_path / "inputs" / "ws"
|
||||
input_dir.mkdir(parents=True)
|
||||
src = input_dir / "demo.pdf"
|
||||
src.write_bytes(b"PDFPDF" * 256)
|
||||
|
||||
rag = _new_rag(tmp_path)
|
||||
await rag.initialize_storages()
|
||||
try:
|
||||
_stub_pipeline(monkeypatch, rag, src)
|
||||
doc_id = "doc-abcdef0123456789abcdef0123456789"
|
||||
await _seed_doc_status(rag, doc_id)
|
||||
|
||||
await _parse_via_registry(
|
||||
rag,
|
||||
"docling",
|
||||
doc_id=doc_id,
|
||||
file_path="demo.pdf",
|
||||
content_data={},
|
||||
)
|
||||
assert counters["calls"] == 1
|
||||
|
||||
# Flip an env var that participates in the options signature
|
||||
monkeypatch.setenv("DOCLING_OCR_LANG", "en,zh")
|
||||
await _parse_via_registry(
|
||||
rag,
|
||||
"docling",
|
||||
doc_id=doc_id,
|
||||
file_path="demo.pdf",
|
||||
content_data={},
|
||||
)
|
||||
assert counters["calls"] == 2, (
|
||||
"DOCLING_OCR_LANG change must invalidate the bundle cache"
|
||||
)
|
||||
finally:
|
||||
await rag.finalize_storages()
|
||||
|
||||
asyncio.new_event_loop().run_until_complete(_run())
|
||||
|
||||
|
||||
@pytest.mark.offline
|
||||
def test_parse_docling_endpoint_signature_invalidates_cache(
|
||||
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
async def _run() -> None:
|
||||
monkeypatch.setenv("DOCLING_ENDPOINT", "http://docling.test")
|
||||
counters = _install_fake_download(monkeypatch)
|
||||
|
||||
input_dir = tmp_path / "inputs" / "ws"
|
||||
input_dir.mkdir(parents=True)
|
||||
src = input_dir / "demo.pdf"
|
||||
src.write_bytes(b"PDFPDF" * 256)
|
||||
|
||||
rag = _new_rag(tmp_path)
|
||||
await rag.initialize_storages()
|
||||
try:
|
||||
_stub_pipeline(monkeypatch, rag, src)
|
||||
doc_id = "doc-abcdef0123456789abcdef0123456789"
|
||||
await _seed_doc_status(rag, doc_id)
|
||||
|
||||
await _parse_via_registry(
|
||||
rag,
|
||||
"docling",
|
||||
doc_id=doc_id,
|
||||
file_path="demo.pdf",
|
||||
content_data={},
|
||||
)
|
||||
assert counters["calls"] == 1
|
||||
|
||||
# Pointing at a different docling-serve instance must not silently
|
||||
# reuse a bundle that was produced by the previous one.
|
||||
monkeypatch.setenv("DOCLING_ENDPOINT", "http://docling-other.test")
|
||||
await _parse_via_registry(
|
||||
rag,
|
||||
"docling",
|
||||
doc_id=doc_id,
|
||||
file_path="demo.pdf",
|
||||
content_data={},
|
||||
)
|
||||
assert counters["calls"] == 2, (
|
||||
"DOCLING_ENDPOINT change must invalidate the bundle cache"
|
||||
)
|
||||
finally:
|
||||
await rag.finalize_storages()
|
||||
|
||||
asyncio.new_event_loop().run_until_complete(_run())
|
||||
|
||||
|
||||
@pytest.mark.offline
|
||||
def test_parse_docling_zero_blocks_raises(
|
||||
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
"""When the docling bundle yields no body blocks (e.g. everything was
|
||||
classified as furniture/background) ``parse_docling`` must fail loudly
|
||||
so the document is marked failed — never persist a half-baked sidecar.
|
||||
"""
|
||||
|
||||
async def _run() -> None:
|
||||
monkeypatch.setenv("DOCLING_ENDPOINT", "http://docling.test")
|
||||
|
||||
# Install a fake download that writes a valid bundle whose body has
|
||||
# no children — the adapter then produces zero IR blocks.
|
||||
import lightrag.parser.external.docling.client as client_mod
|
||||
|
||||
empty_json: dict[str, Any] = {
|
||||
"schema_name": "DoclingDocument",
|
||||
"version": "1.10.0",
|
||||
"origin": {"filename": "demo.pdf", "mimetype": "application/pdf"},
|
||||
"body": {
|
||||
"self_ref": "#/body",
|
||||
"children": [],
|
||||
"content_layer": "body",
|
||||
"label": "unspecified",
|
||||
},
|
||||
"groups": [],
|
||||
"texts": [],
|
||||
"tables": [],
|
||||
"pictures": [],
|
||||
"key_value_items": [],
|
||||
"form_items": [],
|
||||
"pages": {},
|
||||
}
|
||||
|
||||
async def _fake_download(
|
||||
self, raw_dir: Path, source_file_path: Path, **_kwargs
|
||||
):
|
||||
raw_dir.mkdir(parents=True, exist_ok=True)
|
||||
main_json = raw_dir / "demo.json"
|
||||
main_json.write_text(json.dumps(empty_json), encoding="utf-8")
|
||||
(raw_dir / "demo.md").write_text("# empty", encoding="utf-8")
|
||||
|
||||
src_size, src_hash = compute_size_and_hash(source_file_path)
|
||||
crit_size, crit_hash = compute_size_and_hash(main_json)
|
||||
others = [
|
||||
ManifestFile(path="demo.md", size=(raw_dir / "demo.md").stat().st_size),
|
||||
]
|
||||
options_signature = compute_options_signature(
|
||||
tunable_env=snapshot_tunable_env(),
|
||||
fixed_constants=FIXED_CONSTANTS,
|
||||
)
|
||||
manifest = Manifest(
|
||||
engine="docling",
|
||||
source_content_hash=src_hash,
|
||||
source_size_bytes=src_size,
|
||||
source_filename_at_parse=source_file_path.name,
|
||||
critical_file=ManifestFile(
|
||||
path="demo.json", size=crit_size, sha256=crit_hash
|
||||
),
|
||||
files=others,
|
||||
total_size_bytes=crit_size + sum(f.size for f in others),
|
||||
task_id="fake-empty",
|
||||
endpoint_signature="http://docling.test",
|
||||
options_signature=options_signature,
|
||||
extras={"fixed_constants": dict(FIXED_CONSTANTS)},
|
||||
downloaded_at=datetime.now(timezone.utc).isoformat(timespec="seconds"),
|
||||
)
|
||||
write_manifest(raw_dir, manifest)
|
||||
return manifest
|
||||
|
||||
monkeypatch.setattr(
|
||||
client_mod.DoclingRawClient, "download_into", _fake_download
|
||||
)
|
||||
|
||||
input_dir = tmp_path / "inputs" / "ws"
|
||||
input_dir.mkdir(parents=True)
|
||||
src = input_dir / "demo.pdf"
|
||||
src.write_bytes(b"PDFPDF" * 256)
|
||||
|
||||
rag = _new_rag(tmp_path)
|
||||
await rag.initialize_storages()
|
||||
try:
|
||||
_stub_pipeline(monkeypatch, rag, src)
|
||||
doc_id = "doc-abcdef0123456789abcdef0123456789"
|
||||
await _seed_doc_status(rag, doc_id)
|
||||
|
||||
with pytest.raises(ValueError, match="zero blocks"):
|
||||
await _parse_via_registry(
|
||||
rag,
|
||||
"docling",
|
||||
doc_id=doc_id,
|
||||
file_path="demo.pdf",
|
||||
content_data={},
|
||||
)
|
||||
|
||||
# Sidecar must NOT have been emitted: ``write_sidecar`` is reached
|
||||
# only after the zero-blocks check, so no ``*.blocks.jsonl`` may
|
||||
# exist anywhere under the workspace.
|
||||
blocks_files = list(tmp_path.rglob("*.blocks.jsonl"))
|
||||
assert not blocks_files, (
|
||||
f"sidecar emitted despite zero-blocks failure: {blocks_files}"
|
||||
)
|
||||
finally:
|
||||
await rag.finalize_storages()
|
||||
|
||||
asyncio.new_event_loop().run_until_complete(_run())
|
||||
+461
@@ -0,0 +1,461 @@
|
||||
"""``*.mineru_raw/`` cache validation tests.
|
||||
|
||||
Covers every failure mode that triggers a re-download:
|
||||
|
||||
- missing / malformed manifest
|
||||
- source file size mismatch (fast-path)
|
||||
- source file content_hash mismatch
|
||||
- parser options signature missing / mismatch
|
||||
- engine version / endpoint env mismatch
|
||||
- critical_file (content_list.json) size or sha256 mismatch
|
||||
- any non-critical file size mismatch
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from lightrag.parser.external.mineru import (
|
||||
Manifest,
|
||||
ManifestFile,
|
||||
clear_dir_contents,
|
||||
compute_size_and_hash,
|
||||
is_bundle_valid,
|
||||
raw_dir_for_parsed_dir,
|
||||
)
|
||||
from lightrag.parser.external.mineru.cache import current_mineru_options_signature
|
||||
from lightrag.parser.external.mineru.manifest import write_manifest
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Fixtures
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def source_file(tmp_path: Path) -> Path:
|
||||
p = tmp_path / "src.pdf"
|
||||
p.write_bytes(b"Hello PDF" * 100)
|
||||
return p
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def fresh_bundle(tmp_path: Path, source_file: Path) -> tuple[Path, Manifest]:
|
||||
"""Build a fully-valid bundle alongside ``source_file`` and return
|
||||
``(raw_dir, manifest)``."""
|
||||
raw = tmp_path / "src.mineru_raw"
|
||||
raw.mkdir()
|
||||
content_list = raw / "content_list.json"
|
||||
content_list.write_text('[{"type":"text","text":"hi"}]', encoding="utf-8")
|
||||
images = raw / "images"
|
||||
images.mkdir()
|
||||
(images / "img1.png").write_bytes(b"PNG" * 50)
|
||||
(images / "img2.png").write_bytes(b"PNG" * 60)
|
||||
|
||||
src_size, src_hash = compute_size_and_hash(source_file)
|
||||
crit_size, crit_hash = compute_size_and_hash(content_list)
|
||||
files = [
|
||||
ManifestFile(path="images/img1.png", size=(images / "img1.png").stat().st_size),
|
||||
ManifestFile(path="images/img2.png", size=(images / "img2.png").stat().st_size),
|
||||
]
|
||||
manifest = Manifest(
|
||||
source_content_hash=src_hash,
|
||||
source_size_bytes=src_size,
|
||||
source_filename_at_parse=source_file.name,
|
||||
critical_file=ManifestFile(
|
||||
path="content_list.json", size=crit_size, sha256=crit_hash
|
||||
),
|
||||
files=files,
|
||||
total_size_bytes=crit_size + sum(f.size for f in files),
|
||||
task_id="task-1",
|
||||
api_mode="local",
|
||||
options_signature=current_mineru_options_signature(),
|
||||
)
|
||||
write_manifest(raw, manifest)
|
||||
return raw, manifest
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Layout helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@pytest.mark.offline
|
||||
def test_raw_dir_naming(tmp_path: Path) -> None:
|
||||
parsed = tmp_path / "report.pdf.parsed"
|
||||
raw = raw_dir_for_parsed_dir(parsed)
|
||||
assert raw.name == "report.pdf.mineru_raw"
|
||||
assert raw.parent == parsed.parent
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Validation: happy path + every individual failure mode
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@pytest.mark.offline
|
||||
def test_is_bundle_valid_happy_path(
|
||||
fresh_bundle: tuple[Path, Manifest], source_file: Path
|
||||
) -> None:
|
||||
raw, _ = fresh_bundle
|
||||
assert is_bundle_valid(raw, source_file) is True
|
||||
|
||||
|
||||
@pytest.mark.offline
|
||||
def test_invalid_when_manifest_missing(
|
||||
fresh_bundle: tuple[Path, Manifest], source_file: Path
|
||||
) -> None:
|
||||
raw, _ = fresh_bundle
|
||||
(raw / "_manifest.json").unlink()
|
||||
assert is_bundle_valid(raw, source_file) is False
|
||||
|
||||
|
||||
@pytest.mark.offline
|
||||
def test_invalid_when_manifest_malformed(
|
||||
fresh_bundle: tuple[Path, Manifest], source_file: Path
|
||||
) -> None:
|
||||
raw, _ = fresh_bundle
|
||||
(raw / "_manifest.json").write_text("not json")
|
||||
assert is_bundle_valid(raw, source_file) is False
|
||||
|
||||
|
||||
@pytest.mark.offline
|
||||
def test_invalid_when_manifest_wrong_engine(
|
||||
fresh_bundle: tuple[Path, Manifest], source_file: Path
|
||||
) -> None:
|
||||
raw, _ = fresh_bundle
|
||||
payload = json.loads((raw / "_manifest.json").read_text())
|
||||
payload["engine"] = "docling"
|
||||
(raw / "_manifest.json").write_text(json.dumps(payload))
|
||||
assert is_bundle_valid(raw, source_file) is False
|
||||
|
||||
|
||||
@pytest.mark.offline
|
||||
def test_invalid_when_source_size_changes(
|
||||
fresh_bundle: tuple[Path, Manifest], source_file: Path
|
||||
) -> None:
|
||||
# Append bytes to the source file — size diverges from manifest fast-path.
|
||||
raw, _ = fresh_bundle
|
||||
with source_file.open("ab") as fh:
|
||||
fh.write(b"x")
|
||||
assert is_bundle_valid(raw, source_file) is False
|
||||
|
||||
|
||||
@pytest.mark.offline
|
||||
def test_invalid_when_source_hash_changes_but_size_same(
|
||||
fresh_bundle: tuple[Path, Manifest], source_file: Path
|
||||
) -> None:
|
||||
"""In-place rewrite that preserves byte size but mutates content. The
|
||||
fast-path passes but the full hash check catches it."""
|
||||
raw, _ = fresh_bundle
|
||||
data = source_file.read_bytes()
|
||||
# Flip first byte; keep length identical.
|
||||
mutated = bytes([data[0] ^ 0xFF]) + data[1:]
|
||||
assert len(mutated) == len(data)
|
||||
source_file.write_bytes(mutated)
|
||||
assert is_bundle_valid(raw, source_file) is False
|
||||
|
||||
|
||||
@pytest.mark.offline
|
||||
def test_invalid_when_engine_version_mismatch(
|
||||
fresh_bundle: tuple[Path, Manifest],
|
||||
source_file: Path,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
raw, _ = fresh_bundle
|
||||
payload = json.loads((raw / "_manifest.json").read_text())
|
||||
payload["engine_version"] = "magic-pdf 1.5.4"
|
||||
(raw / "_manifest.json").write_text(json.dumps(payload))
|
||||
monkeypatch.setenv("MINERU_ENGINE_VERSION", "magic-pdf 1.6.0")
|
||||
assert is_bundle_valid(raw, source_file) is False
|
||||
|
||||
|
||||
@pytest.mark.offline
|
||||
def test_engine_version_match_passes(
|
||||
fresh_bundle: tuple[Path, Manifest],
|
||||
source_file: Path,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
raw, _ = fresh_bundle
|
||||
payload = json.loads((raw / "_manifest.json").read_text())
|
||||
payload["engine_version"] = "magic-pdf 1.5.4"
|
||||
(raw / "_manifest.json").write_text(json.dumps(payload))
|
||||
monkeypatch.setenv("MINERU_ENGINE_VERSION", "magic-pdf 1.5.4")
|
||||
assert is_bundle_valid(raw, source_file) is True
|
||||
|
||||
|
||||
@pytest.mark.offline
|
||||
def test_engine_version_skip_when_either_side_blank(
|
||||
fresh_bundle: tuple[Path, Manifest],
|
||||
source_file: Path,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
"""Blank manifest engine_version + non-blank env should NOT invalidate
|
||||
(no signal from manifest); same for the reverse."""
|
||||
raw, _ = fresh_bundle
|
||||
# Manifest engine_version is empty by default.
|
||||
monkeypatch.setenv("MINERU_ENGINE_VERSION", "anything")
|
||||
assert is_bundle_valid(raw, source_file) is True
|
||||
|
||||
|
||||
@pytest.mark.offline
|
||||
def test_invalid_when_api_mode_mismatch(
|
||||
fresh_bundle: tuple[Path, Manifest],
|
||||
source_file: Path,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
raw, _ = fresh_bundle
|
||||
payload = json.loads((raw / "_manifest.json").read_text())
|
||||
payload["api_mode"] = "local"
|
||||
(raw / "_manifest.json").write_text(json.dumps(payload))
|
||||
monkeypatch.setenv("MINERU_API_MODE", "official")
|
||||
monkeypatch.setenv("MINERU_API_TOKEN", "token")
|
||||
assert is_bundle_valid(raw, source_file) is False
|
||||
|
||||
|
||||
@pytest.mark.offline
|
||||
def test_invalid_when_options_signature_missing(
|
||||
fresh_bundle: tuple[Path, Manifest], source_file: Path
|
||||
) -> None:
|
||||
raw, _ = fresh_bundle
|
||||
payload = json.loads((raw / "_manifest.json").read_text())
|
||||
payload.pop("options_signature", None)
|
||||
(raw / "_manifest.json").write_text(json.dumps(payload))
|
||||
assert is_bundle_valid(raw, source_file) is False
|
||||
|
||||
|
||||
@pytest.mark.offline
|
||||
@pytest.mark.parametrize(
|
||||
("key", "value"),
|
||||
[
|
||||
("MINERU_LOCAL_BACKEND", "pipeline"),
|
||||
("MINERU_LOCAL_PARSE_METHOD", "ocr"),
|
||||
("MINERU_LOCAL_IMAGE_ANALYSIS", "true"),
|
||||
("MINERU_LOCAL_START_PAGE_ID", "1"),
|
||||
],
|
||||
)
|
||||
def test_invalid_when_local_parser_options_change(
|
||||
fresh_bundle: tuple[Path, Manifest],
|
||||
source_file: Path,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
key: str,
|
||||
value: str,
|
||||
) -> None:
|
||||
raw, _ = fresh_bundle
|
||||
monkeypatch.setenv("MINERU_API_MODE", "local")
|
||||
monkeypatch.setenv(key, value)
|
||||
assert is_bundle_valid(raw, source_file) is False
|
||||
|
||||
|
||||
@pytest.mark.offline
|
||||
@pytest.mark.parametrize(
|
||||
("key", "value"),
|
||||
[
|
||||
("MINERU_MODEL_VERSION", "pipeline"),
|
||||
("MINERU_IS_OCR", "true"),
|
||||
("MINERU_PAGE_RANGES", "1-5"),
|
||||
("MINERU_LANGUAGE", "en"),
|
||||
("MINERU_ENABLE_TABLE", "false"),
|
||||
("MINERU_ENABLE_FORMULA", "false"),
|
||||
],
|
||||
)
|
||||
def test_invalid_when_official_parser_options_change(
|
||||
tmp_path: Path,
|
||||
source_file: Path,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
key: str,
|
||||
value: str,
|
||||
) -> None:
|
||||
"""Symmetric coverage for the official-mode partition of the signature.
|
||||
|
||||
Build a bundle whose ``options_signature`` reflects the official defaults,
|
||||
sanity-check that it validates, then flip ``key`` and assert a cache miss.
|
||||
"""
|
||||
monkeypatch.setenv("MINERU_API_MODE", "official")
|
||||
|
||||
raw = tmp_path / "src.mineru_raw"
|
||||
raw.mkdir()
|
||||
content_list = raw / "content_list.json"
|
||||
content_list.write_text('[{"type":"text","text":"hi"}]', encoding="utf-8")
|
||||
crit_size, crit_hash = compute_size_and_hash(content_list)
|
||||
src_size, src_hash = compute_size_and_hash(source_file)
|
||||
manifest = Manifest(
|
||||
source_content_hash=src_hash,
|
||||
source_size_bytes=src_size,
|
||||
source_filename_at_parse=source_file.name,
|
||||
critical_file=ManifestFile(
|
||||
path="content_list.json", size=crit_size, sha256=crit_hash
|
||||
),
|
||||
files=[],
|
||||
total_size_bytes=crit_size,
|
||||
task_id="task-official",
|
||||
api_mode="official",
|
||||
options_signature=current_mineru_options_signature(),
|
||||
)
|
||||
write_manifest(raw, manifest)
|
||||
|
||||
assert is_bundle_valid(raw, source_file) is True
|
||||
monkeypatch.setenv(key, value)
|
||||
assert is_bundle_valid(raw, source_file) is False
|
||||
|
||||
|
||||
@pytest.mark.offline
|
||||
def test_invalid_when_endpoint_signature_mismatch(
|
||||
fresh_bundle: tuple[Path, Manifest],
|
||||
source_file: Path,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
raw, _ = fresh_bundle
|
||||
payload = json.loads((raw / "_manifest.json").read_text())
|
||||
payload["api_mode"] = "local"
|
||||
payload["endpoint_signature"] = "http://old.example"
|
||||
(raw / "_manifest.json").write_text(json.dumps(payload))
|
||||
monkeypatch.setenv("MINERU_API_MODE", "local")
|
||||
monkeypatch.setenv("MINERU_LOCAL_ENDPOINT", "http://new.example")
|
||||
assert is_bundle_valid(raw, source_file) is False
|
||||
|
||||
|
||||
@pytest.mark.offline
|
||||
def test_endpoint_signature_uses_mode_specific_endpoint(
|
||||
fresh_bundle: tuple[Path, Manifest],
|
||||
source_file: Path,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
raw, _ = fresh_bundle
|
||||
payload = json.loads((raw / "_manifest.json").read_text())
|
||||
payload["api_mode"] = "local"
|
||||
payload["endpoint_signature"] = "http://old.example"
|
||||
(raw / "_manifest.json").write_text(json.dumps(payload))
|
||||
monkeypatch.setenv("MINERU_API_MODE", "local")
|
||||
monkeypatch.setenv("MINERU_LOCAL_ENDPOINT", "http://new.example")
|
||||
assert is_bundle_valid(raw, source_file) is False
|
||||
|
||||
|
||||
@pytest.mark.offline
|
||||
def test_endpoint_signature_ignores_trailing_slash(
|
||||
fresh_bundle: tuple[Path, Manifest],
|
||||
source_file: Path,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
raw, _ = fresh_bundle
|
||||
payload = json.loads((raw / "_manifest.json").read_text())
|
||||
payload["api_mode"] = "local"
|
||||
payload["endpoint_signature"] = "http://old.example"
|
||||
(raw / "_manifest.json").write_text(json.dumps(payload))
|
||||
monkeypatch.setenv("MINERU_API_MODE", "local")
|
||||
monkeypatch.setenv("MINERU_LOCAL_ENDPOINT", "http://old.example/")
|
||||
assert is_bundle_valid(raw, source_file) is True
|
||||
|
||||
|
||||
@pytest.mark.offline
|
||||
def test_invalid_when_critical_file_missing(
|
||||
fresh_bundle: tuple[Path, Manifest], source_file: Path
|
||||
) -> None:
|
||||
raw, _ = fresh_bundle
|
||||
(raw / "content_list.json").unlink()
|
||||
assert is_bundle_valid(raw, source_file) is False
|
||||
|
||||
|
||||
@pytest.mark.offline
|
||||
def test_invalid_when_critical_file_size_changes(
|
||||
fresh_bundle: tuple[Path, Manifest], source_file: Path
|
||||
) -> None:
|
||||
raw, _ = fresh_bundle
|
||||
cl = raw / "content_list.json"
|
||||
cl.write_text(cl.read_text() + "/* extra */")
|
||||
assert is_bundle_valid(raw, source_file) is False
|
||||
|
||||
|
||||
@pytest.mark.offline
|
||||
def test_invalid_when_critical_file_hash_changes(
|
||||
fresh_bundle: tuple[Path, Manifest], source_file: Path
|
||||
) -> None:
|
||||
"""Same size, different bytes. sha256 is the terminal check."""
|
||||
raw, _ = fresh_bundle
|
||||
cl = raw / "content_list.json"
|
||||
data = cl.read_text()
|
||||
mutated = data[:-1] + "X" # swap last char; size preserved
|
||||
assert len(mutated) == len(data)
|
||||
cl.write_text(mutated)
|
||||
assert is_bundle_valid(raw, source_file) is False
|
||||
|
||||
|
||||
@pytest.mark.offline
|
||||
def test_invalid_when_aux_file_size_changes(
|
||||
fresh_bundle: tuple[Path, Manifest], source_file: Path
|
||||
) -> None:
|
||||
raw, _ = fresh_bundle
|
||||
p = raw / "images" / "img1.png"
|
||||
p.write_bytes(p.read_bytes() + b"corruption")
|
||||
assert is_bundle_valid(raw, source_file) is False
|
||||
|
||||
|
||||
@pytest.mark.offline
|
||||
def test_invalid_when_aux_file_missing(
|
||||
fresh_bundle: tuple[Path, Manifest], source_file: Path
|
||||
) -> None:
|
||||
raw, _ = fresh_bundle
|
||||
(raw / "images" / "img2.png").unlink()
|
||||
assert is_bundle_valid(raw, source_file) is False
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@pytest.mark.offline
|
||||
def test_clear_dir_contents_preserves_directory(tmp_path: Path) -> None:
|
||||
d = tmp_path / "raw"
|
||||
d.mkdir()
|
||||
(d / "a.txt").write_text("a")
|
||||
(d / "sub").mkdir()
|
||||
(d / "sub" / "b.txt").write_text("b")
|
||||
clear_dir_contents(d)
|
||||
assert d.exists()
|
||||
assert list(d.iterdir()) == []
|
||||
|
||||
|
||||
@pytest.mark.offline
|
||||
def test_compute_size_and_hash_consistency(tmp_path: Path) -> None:
|
||||
"""Both values describe the same byte stream."""
|
||||
p = tmp_path / "f.bin"
|
||||
payload = b"abc" * 1000
|
||||
p.write_bytes(payload)
|
||||
size, h = compute_size_and_hash(p)
|
||||
assert size == len(payload)
|
||||
assert h.startswith("sha256:") and len(h) == len("sha256:") + 64
|
||||
|
||||
|
||||
@pytest.mark.offline
|
||||
def test_manifest_round_trip_via_disk(tmp_path: Path) -> None:
|
||||
"""Write → read recovers all fields."""
|
||||
raw = tmp_path / "rt.mineru_raw"
|
||||
raw.mkdir()
|
||||
m = Manifest(
|
||||
source_content_hash="sha256:abc",
|
||||
source_size_bytes=10,
|
||||
source_filename_at_parse="x.pdf",
|
||||
critical_file=ManifestFile(
|
||||
path="content_list.json", size=5, sha256="sha256:cl"
|
||||
),
|
||||
files=[ManifestFile(path="images/i.png", size=3)],
|
||||
total_size_bytes=8,
|
||||
task_id="t1",
|
||||
engine_version="v",
|
||||
endpoint_signature="ep",
|
||||
options_signature="sha256:opts",
|
||||
)
|
||||
write_manifest(raw, m)
|
||||
from lightrag.parser.external.mineru.manifest import load_manifest
|
||||
|
||||
loaded = load_manifest(raw)
|
||||
assert loaded is not None
|
||||
assert loaded.source_content_hash == "sha256:abc"
|
||||
assert loaded.critical_file.sha256 == "sha256:cl"
|
||||
assert [f.path for f in loaded.files] == ["images/i.png"]
|
||||
assert loaded.task_id == "t1"
|
||||
assert loaded.options_signature == "sha256:opts"
|
||||
+1071
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,114 @@
|
||||
"""``delete_file_variants_by_file_path`` now also clears the
|
||||
MinerU raw bundle (``*.mineru_raw/``) alongside the sidecar
|
||||
(``*.parsed/``) and source file when ``delete_file=True`` is selected.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
# document_routes pulls in api.auth → api.config.parse_args(); blank argv
|
||||
# to keep argparse from rejecting pytest's flags at import time.
|
||||
sys.argv = sys.argv[:1]
|
||||
|
||||
from lightrag.api.routers.document_routes import ( # noqa: E402
|
||||
_file_path_for_parsed_artifact_dir,
|
||||
delete_file_variants_by_file_path,
|
||||
)
|
||||
from lightrag.constants import PARSED_DIR_NAME # noqa: E402
|
||||
|
||||
|
||||
@pytest.mark.offline
|
||||
def test_canonical_basename_recognizes_both_suffixes() -> None:
|
||||
assert _file_path_for_parsed_artifact_dir("foo.pdf.parsed") == "foo.pdf"
|
||||
assert _file_path_for_parsed_artifact_dir("foo.pdf.mineru_raw") == "foo.pdf"
|
||||
# archive variants (parsed_001, mineru_raw_002, ...) handled
|
||||
assert _file_path_for_parsed_artifact_dir("foo.pdf.parsed_001") == "foo.pdf"
|
||||
assert _file_path_for_parsed_artifact_dir("foo.pdf.mineru_raw_002") == "foo.pdf"
|
||||
# unrelated names don't match
|
||||
assert _file_path_for_parsed_artifact_dir("foo.parsed.bak") is None
|
||||
assert _file_path_for_parsed_artifact_dir("notes.txt") is None
|
||||
|
||||
|
||||
@pytest.mark.offline
|
||||
def test_delete_file_variants_removes_parsed_and_mineru_raw(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
"""Both sidecar dir and raw bundle dir should disappear in one call."""
|
||||
input_dir = tmp_path / "inputs"
|
||||
input_dir.mkdir()
|
||||
parsed_root = input_dir / PARSED_DIR_NAME
|
||||
parsed_root.mkdir()
|
||||
|
||||
# Original source file (already moved into __parsed__/ post-archive).
|
||||
archived_source = parsed_root / "demo.pdf"
|
||||
archived_source.write_bytes(b"PDFCONTENT")
|
||||
|
||||
parsed_dir = parsed_root / "demo.pdf.parsed"
|
||||
parsed_dir.mkdir()
|
||||
(parsed_dir / "demo.blocks.jsonl").write_text("{}\n")
|
||||
|
||||
raw_dir = parsed_root / "demo.pdf.mineru_raw"
|
||||
raw_dir.mkdir()
|
||||
(raw_dir / "_manifest.json").write_text("{}")
|
||||
(raw_dir / "content_list.json").write_text("[]")
|
||||
(raw_dir / "images").mkdir()
|
||||
(raw_dir / "images" / "img.png").write_bytes(b"png")
|
||||
|
||||
deleted, errors = delete_file_variants_by_file_path(input_dir, file_path="demo.pdf")
|
||||
|
||||
# Both directories and the archived source file were deleted.
|
||||
assert errors == []
|
||||
deleted_names = {Path(p).name for p in deleted}
|
||||
assert "demo.pdf.parsed" in deleted_names
|
||||
assert "demo.pdf.mineru_raw" in deleted_names
|
||||
assert "demo.pdf" in deleted_names
|
||||
|
||||
assert not parsed_dir.exists()
|
||||
assert not raw_dir.exists()
|
||||
assert not archived_source.exists()
|
||||
|
||||
|
||||
@pytest.mark.offline
|
||||
def test_delete_file_variants_handles_only_raw_dir(tmp_path: Path) -> None:
|
||||
"""The raw bundle may exist without a corresponding sidecar (parse
|
||||
aborted between download and adapter). Delete should still pick it up.
|
||||
"""
|
||||
input_dir = tmp_path / "inputs"
|
||||
input_dir.mkdir()
|
||||
parsed_root = input_dir / PARSED_DIR_NAME
|
||||
parsed_root.mkdir()
|
||||
|
||||
raw_dir = parsed_root / "demo.pdf.mineru_raw"
|
||||
raw_dir.mkdir()
|
||||
(raw_dir / "_manifest.json").write_text("{}")
|
||||
|
||||
deleted, errors = delete_file_variants_by_file_path(input_dir, file_path="demo.pdf")
|
||||
assert errors == []
|
||||
assert any("demo.pdf.mineru_raw" in p for p in deleted)
|
||||
assert not raw_dir.exists()
|
||||
|
||||
|
||||
@pytest.mark.offline
|
||||
def test_delete_file_variants_leaves_unrelated_dirs(tmp_path: Path) -> None:
|
||||
"""Directories that don't match the canonical filename are untouched."""
|
||||
input_dir = tmp_path / "inputs"
|
||||
input_dir.mkdir()
|
||||
parsed_root = input_dir / PARSED_DIR_NAME
|
||||
parsed_root.mkdir()
|
||||
|
||||
target_raw = parsed_root / "demo.pdf.mineru_raw"
|
||||
target_raw.mkdir()
|
||||
other_raw = parsed_root / "other.pdf.mineru_raw"
|
||||
other_raw.mkdir()
|
||||
other_parsed = parsed_root / "other.pdf.parsed"
|
||||
other_parsed.mkdir()
|
||||
|
||||
deleted, errors = delete_file_variants_by_file_path(input_dir, file_path="demo.pdf")
|
||||
assert errors == []
|
||||
assert not target_raw.exists()
|
||||
assert other_raw.exists(), "siblings for a different basename must survive"
|
||||
assert other_parsed.exists()
|
||||
+909
@@ -0,0 +1,909 @@
|
||||
"""MinerU IR builder tests: content_list.json → IR translation."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from lightrag.parser.external.mineru import MinerUIRBuilder
|
||||
|
||||
|
||||
def _write_bundle(tmp_path: Path, content_list: list[dict]) -> Path:
|
||||
"""Build a minimal *.mineru_raw/ directory."""
|
||||
raw = tmp_path / "doc.mineru_raw"
|
||||
raw.mkdir()
|
||||
(raw / "content_list.json").write_text(json.dumps(content_list, ensure_ascii=False))
|
||||
return raw
|
||||
|
||||
|
||||
@pytest.mark.offline
|
||||
def test_adapter_simple_text_and_heading(tmp_path: Path) -> None:
|
||||
raw = _write_bundle(
|
||||
tmp_path,
|
||||
[
|
||||
{"type": "text", "text": "1 Introduction", "text_level": 1},
|
||||
{"type": "text", "text": "Body paragraph."},
|
||||
{"type": "text", "text": "1.1 Sub", "text_level": 2},
|
||||
{"type": "text", "text": "Sub body."},
|
||||
],
|
||||
)
|
||||
ir = MinerUIRBuilder().normalize_from_workdir(raw, document_name="x.pdf")
|
||||
|
||||
assert ir.doc_title == "1 Introduction"
|
||||
assert ir.document_format == "pdf"
|
||||
# Heading + body merge into a single block per heading.
|
||||
assert len(ir.blocks) == 2
|
||||
assert ir.blocks[0].heading == "1 Introduction"
|
||||
assert ir.blocks[0].level == 1
|
||||
# Heading line is rendered with markdown ``#`` prefix matching the level.
|
||||
assert ir.blocks[0].content_template == "# 1 Introduction\nBody paragraph."
|
||||
# Sub-heading updates stack and records parent.
|
||||
assert ir.blocks[1].heading == "1.1 Sub"
|
||||
assert ir.blocks[1].level == 2
|
||||
assert ir.blocks[1].parent_headings == ["1 Introduction"]
|
||||
assert ir.blocks[1].content_template == "## 1.1 Sub\nSub body."
|
||||
|
||||
|
||||
@pytest.mark.offline
|
||||
def test_adapter_preface_block_for_pre_heading_content(tmp_path: Path) -> None:
|
||||
"""Items emitted before the first heading land in a synthetic
|
||||
``Preface/Uncategorized`` block at level 0."""
|
||||
raw = _write_bundle(
|
||||
tmp_path,
|
||||
[
|
||||
{"type": "text", "text": "Floating intro line."},
|
||||
{"type": "list", "list_items": ["a", "b"]},
|
||||
{"type": "text", "text": "Section A", "text_level": 1},
|
||||
{"type": "text", "text": "A body."},
|
||||
],
|
||||
)
|
||||
ir = MinerUIRBuilder().normalize_from_workdir(raw, document_name="p.pdf")
|
||||
|
||||
assert len(ir.blocks) == 2
|
||||
preface = ir.blocks[0]
|
||||
assert preface.heading == "Preface/Uncategorized"
|
||||
assert preface.level == 0
|
||||
assert preface.parent_headings == []
|
||||
assert preface.content_template == "Floating intro line.\na\nb"
|
||||
|
||||
section = ir.blocks[1]
|
||||
assert section.heading == "Section A"
|
||||
assert section.level == 1
|
||||
assert section.content_template == "# Section A\nA body."
|
||||
|
||||
|
||||
@pytest.mark.offline
|
||||
def test_adapter_merges_mixed_payloads_under_heading(tmp_path: Path) -> None:
|
||||
"""Tables / images / equations / code under the same heading merge into
|
||||
one block; their placeholders appear in document order."""
|
||||
raw = _write_bundle(
|
||||
tmp_path,
|
||||
[
|
||||
{"type": "text", "text": "Methods", "text_level": 1},
|
||||
{"type": "text", "text": "We did stuff."},
|
||||
{
|
||||
"type": "table",
|
||||
"table_body": [["a", "b"], ["1", "2"]],
|
||||
"num_rows": 2,
|
||||
"num_cols": 2,
|
||||
},
|
||||
{"type": "image", "img_path": "images/fig1.png"},
|
||||
{"type": "equation", "text": "$$E = mc^2$$"},
|
||||
{"type": "code", "code_body": "print('ok')"},
|
||||
],
|
||||
)
|
||||
ir = MinerUIRBuilder().normalize_from_workdir(raw, document_name="m.pdf")
|
||||
assert len(ir.blocks) == 1
|
||||
block = ir.blocks[0]
|
||||
assert block.heading == "Methods"
|
||||
assert block.level == 1
|
||||
assert len(block.tables) == 1
|
||||
assert len(block.drawings) == 1
|
||||
assert len(block.equations) == 1
|
||||
# Lines are joined in source order; the heading carries its ``#`` prefix.
|
||||
expected_lines = [
|
||||
"# Methods",
|
||||
"We did stuff.",
|
||||
f"{{{{TBL:{block.tables[0].placeholder_key}}}}}",
|
||||
f"{{{{IMG:{block.drawings[0].placeholder_key}}}}}",
|
||||
f"{{{{EQ:{block.equations[0].placeholder_key}}}}}",
|
||||
"print('ok')",
|
||||
]
|
||||
assert block.content_template == "\n".join(expected_lines)
|
||||
|
||||
|
||||
@pytest.mark.offline
|
||||
def test_adapter_table_and_drawing_and_equation(tmp_path: Path) -> None:
|
||||
raw = _write_bundle(
|
||||
tmp_path,
|
||||
[
|
||||
{
|
||||
"type": "table",
|
||||
"table_body": [["a", "b"], ["1", "2"]],
|
||||
"num_rows": 2,
|
||||
"num_cols": 2,
|
||||
"table_caption": ["Tbl"],
|
||||
"header": [["a", "b"]],
|
||||
},
|
||||
{
|
||||
"type": "image",
|
||||
"img_path": "images/img_001.jpg",
|
||||
"image_caption": ["Fig 1"],
|
||||
"page_idx": 1,
|
||||
"bbox": [10, 20, 30, 40],
|
||||
},
|
||||
{"type": "equation", "text": "$E = mc^2$", "caption": "Eq 1"},
|
||||
],
|
||||
)
|
||||
# The drawing references images/img_001.jpg — adapter accepts missing
|
||||
# files and produces an AssetSpec with source=None.
|
||||
ir = MinerUIRBuilder().normalize_from_workdir(raw, document_name="d.pdf")
|
||||
|
||||
table_block = next(b for b in ir.blocks if b.tables)
|
||||
table = table_block.tables[0]
|
||||
assert table.rows == [["a", "b"], ["1", "2"]]
|
||||
assert table.num_rows == 2 and table.num_cols == 2
|
||||
assert table.caption == "Tbl"
|
||||
assert table.table_header == [["a", "b"]]
|
||||
assert table.self_ref == "content_list.json#/0"
|
||||
|
||||
drawing_block = next(b for b in ir.blocks if b.drawings)
|
||||
drawing = drawing_block.drawings[0]
|
||||
assert drawing.fmt == "jpg"
|
||||
assert drawing.caption == "Fig 1"
|
||||
assert drawing.self_ref == "content_list.json#/1"
|
||||
# Position carried through. The bbox-bearing item produces exactly one
|
||||
# fine-grained position (anchor + range) and is NOT also rolled into the
|
||||
# page-only summary channel — so the block has a single position entry,
|
||||
# not a duplicate summary + bbox pair.
|
||||
assert len(drawing_block.positions) == 1
|
||||
assert drawing_block.positions[0].type == "bbox"
|
||||
# Anchor is always serialized as a string (uniform on-disk format,
|
||||
# accommodates book pagination labels like Roman "ii").
|
||||
assert drawing_block.positions[0].anchor == "2" # page_idx+1
|
||||
assert drawing_block.positions[0].range == [10.0, 20.0, 30.0, 40.0]
|
||||
|
||||
# Asset is declared with the relative path as ref.
|
||||
assert any(a.ref == "images/img_001.jpg" for a in ir.assets)
|
||||
|
||||
equation_block = next(b for b in ir.blocks if b.equations)
|
||||
eq = equation_block.equations[0]
|
||||
# IREquation.latex preserves MinerU's raw form so blocks.jsonl shows it
|
||||
# verbatim; equations.json strips the ``$`` wrappers downstream (writer).
|
||||
assert eq.latex == "$E = mc^2$"
|
||||
assert eq.is_block is True
|
||||
assert eq.caption == "Eq 1"
|
||||
assert eq.self_ref == "content_list.json#/2"
|
||||
|
||||
|
||||
@pytest.mark.offline
|
||||
def test_adapter_page_idx_aggregated_and_deduped_when_no_bbox(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
"""Real MinerU output carries ``page_idx`` on every item but rarely a
|
||||
``bbox``. Each unique page contributing to a merged block must surface as
|
||||
one anchor-only ``{type:"bbox", anchor:<page+1>}`` entry, sorted, no
|
||||
duplicates, no ``range``.
|
||||
"""
|
||||
raw = _write_bundle(
|
||||
tmp_path,
|
||||
[
|
||||
{"type": "text", "text": "Section", "text_level": 1, "page_idx": 0},
|
||||
{"type": "text", "text": "line A", "page_idx": 0},
|
||||
{"type": "text", "text": "line B", "page_idx": 1},
|
||||
{"type": "text", "text": "line C", "page_idx": 1},
|
||||
{"type": "text", "text": "line D", "page_idx": 2},
|
||||
],
|
||||
)
|
||||
ir = MinerUIRBuilder().normalize_from_workdir(raw, document_name="p.pdf")
|
||||
|
||||
assert len(ir.blocks) == 1
|
||||
block = ir.blocks[0]
|
||||
# Pages 0, 1, 2 → anchors "1", "2", "3" — one entry per unique page.
|
||||
# Anchors are persisted as strings for on-disk uniformity.
|
||||
assert len(block.positions) == 3
|
||||
anchors = [p.anchor for p in block.positions]
|
||||
assert anchors == ["1", "2", "3"]
|
||||
for pos in block.positions:
|
||||
assert pos.type == "bbox"
|
||||
# Page-only summary entries have no range; ``to_jsonable`` must omit
|
||||
# the key entirely.
|
||||
assert pos.range is None
|
||||
assert "range" not in pos.to_jsonable()
|
||||
|
||||
|
||||
@pytest.mark.offline
|
||||
def test_adapter_bbox_items_and_page_only_items_coexist(tmp_path: Path) -> None:
|
||||
"""When a block merges both bbox-bearing and bbox-less items, the bbox
|
||||
items are emitted per-item (no dedupe, with ``range``) and only the
|
||||
bbox-less items contribute to the page-only summary. Ordering: summary
|
||||
first (sorted by anchor), bbox entries after (source order).
|
||||
"""
|
||||
raw = _write_bundle(
|
||||
tmp_path,
|
||||
[
|
||||
{"type": "text", "text": "Mixed", "text_level": 1, "page_idx": 1},
|
||||
{
|
||||
"type": "image",
|
||||
"img_path": "images/fig.png",
|
||||
"page_idx": 1,
|
||||
"bbox": [10, 20, 30, 40],
|
||||
},
|
||||
{"type": "text", "text": "tail line", "page_idx": 2},
|
||||
],
|
||||
)
|
||||
ir = MinerUIRBuilder().normalize_from_workdir(raw, document_name="m.pdf")
|
||||
|
||||
assert len(ir.blocks) == 1
|
||||
positions = ir.blocks[0].positions
|
||||
# One page-only summary for page 3 (the bbox-less tail line) and one
|
||||
# bbox entry for page 2 (the image). The heading item has page_idx=1
|
||||
# but no bbox, so it adds anchor 2 to the page set — combined with the
|
||||
# tail item's anchor 3 the summary section has TWO anchors (1+1, 2+1).
|
||||
assert [(p.anchor, p.range) for p in positions] == [
|
||||
("2", None),
|
||||
("3", None),
|
||||
("2", [10.0, 20.0, 30.0, 40.0]),
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.offline
|
||||
def test_adapter_page_sort_books_convention_with_mixed_anchors(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
"""Block merges items with Roman preface labels and Arabic numerals.
|
||||
|
||||
Two guarantees:
|
||||
|
||||
1. The adapter must not crash when sorting heterogeneous anchors — a
|
||||
previous bug surfaced ``TypeError: '<' not supported between
|
||||
instances of 'str' and 'int'`` whenever ``page_idx`` mixed types.
|
||||
2. Output order follows book pagination convention: Roman / letter
|
||||
labels first (lexical), then numeric pages by integer value, so
|
||||
``"2"`` precedes ``"10"`` (not ``"10"`` before ``"2"`` as a naive
|
||||
lexical sort would do).
|
||||
"""
|
||||
raw = _write_bundle(
|
||||
tmp_path,
|
||||
[
|
||||
{
|
||||
"type": "text",
|
||||
"text": "Mixed Pagination",
|
||||
"text_level": 1,
|
||||
"page_idx": "i",
|
||||
},
|
||||
{"type": "text", "text": "preface intro", "page_idx": "i"},
|
||||
{"type": "text", "text": "preface tail", "page_idx": "ii"},
|
||||
{"type": "text", "text": "chapter line A", "page_idx": 1}, # → "2"
|
||||
{"type": "text", "text": "chapter line B", "page_idx": 9}, # → "10"
|
||||
],
|
||||
)
|
||||
ir = MinerUIRBuilder().normalize_from_workdir(raw, document_name="mix.pdf")
|
||||
|
||||
assert len(ir.blocks) == 1
|
||||
anchors = [p.anchor for p in ir.blocks[0].positions]
|
||||
# Roman labels first (lex order), then numerics by int value.
|
||||
assert anchors == ["i", "ii", "2", "10"]
|
||||
|
||||
|
||||
@pytest.mark.offline
|
||||
def test_adapter_empty_text_item_does_not_leak_page_to_block(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
"""An item whose body is empty must NOT contribute its ``page_idx`` to
|
||||
the current block's positions — otherwise spurious pages from
|
||||
content-less items poison provenance.
|
||||
|
||||
Regression: empty text on page 99 sits between two real headings; its
|
||||
page must not appear under either block.
|
||||
"""
|
||||
raw = _write_bundle(
|
||||
tmp_path,
|
||||
[
|
||||
{"type": "text", "text": "Section A", "text_level": 1, "page_idx": 0},
|
||||
{"type": "text", "text": "real body", "page_idx": 0},
|
||||
# Empty body — should be silently dropped, page_idx not recorded.
|
||||
{"type": "text", "text": "", "page_idx": 98},
|
||||
{"type": "text", "text": "Section B", "text_level": 1, "page_idx": 1},
|
||||
{"type": "text", "text": "next body", "page_idx": 1},
|
||||
],
|
||||
)
|
||||
ir = MinerUIRBuilder().normalize_from_workdir(raw, document_name="leak.pdf")
|
||||
|
||||
assert len(ir.blocks) == 2
|
||||
a_anchors = [p.anchor for p in ir.blocks[0].positions]
|
||||
b_anchors = [p.anchor for p in ir.blocks[1].positions]
|
||||
# Section A only mentions page 1 (page_idx 0 + 1) — NOT 99 from the
|
||||
# dropped empty item.
|
||||
assert a_anchors == ["1"]
|
||||
assert "99" not in a_anchors and "99" not in b_anchors
|
||||
# Section B only mentions page 2 (page_idx 1 + 1).
|
||||
assert b_anchors == ["2"]
|
||||
|
||||
|
||||
@pytest.mark.offline
|
||||
def test_adapter_each_heading_starts_its_own_block(tmp_path: Path) -> None:
|
||||
"""Every recognized heading starts its own block. Back-to-back headings
|
||||
with no body between them are NOT folded — each becomes a standalone
|
||||
block whose content is just the heading line; the heading that does have
|
||||
a following body merges that body in. Mirrors the native docx parser.
|
||||
"""
|
||||
raw = _write_bundle(
|
||||
tmp_path,
|
||||
[
|
||||
{"type": "text", "text": "1 Top", "text_level": 1},
|
||||
{"type": "text", "text": "1.1 Mid", "text_level": 2},
|
||||
{"type": "text", "text": "1.1.1 Deep", "text_level": 3},
|
||||
{"type": "text", "text": "Body for deep."},
|
||||
{"type": "text", "text": "2 Top Again", "text_level": 1},
|
||||
{"type": "text", "text": "More body."},
|
||||
],
|
||||
)
|
||||
ir = MinerUIRBuilder().normalize_from_workdir(raw, document_name="m.pdf")
|
||||
|
||||
summary = [
|
||||
(b.heading, b.content_template, b.level, b.parent_headings) for b in ir.blocks
|
||||
]
|
||||
assert summary == [
|
||||
("1 Top", "# 1 Top", 1, []),
|
||||
("1.1 Mid", "## 1.1 Mid", 2, ["1 Top"]),
|
||||
(
|
||||
"1.1.1 Deep",
|
||||
"### 1.1.1 Deep\nBody for deep.",
|
||||
3,
|
||||
["1 Top", "1.1 Mid"],
|
||||
),
|
||||
("2 Top Again", "# 2 Top Again\nMore body.", 1, []),
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.offline
|
||||
def test_adapter_heading_markdown_prefix_cap_and_existing_marker(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
"""Heading prefix is capped at six ``#`` and a heading whose text already
|
||||
carries a markdown prefix is not double-prefixed in content but is stored
|
||||
cleanly in metadata."""
|
||||
raw = _write_bundle(
|
||||
tmp_path,
|
||||
[
|
||||
# level 7 → clamped to six "#".
|
||||
{"type": "text", "text": "Deep", "text_level": 7},
|
||||
{"type": "text", "text": "deep body."},
|
||||
# Source text already a markdown heading → kept verbatim.
|
||||
{"type": "text", "text": "# Already MD", "text_level": 1},
|
||||
{"type": "text", "text": "md body."},
|
||||
{"type": "text", "text": "## Child MD", "text_level": 2},
|
||||
{"type": "text", "text": "child body."},
|
||||
],
|
||||
)
|
||||
ir = MinerUIRBuilder().normalize_from_workdir(raw, document_name="m.pdf")
|
||||
|
||||
first_lines = [b.content_template.split("\n")[0] for b in ir.blocks]
|
||||
assert first_lines == ["###### Deep", "# Already MD", "## Child MD"]
|
||||
assert [(b.heading, b.parent_headings) for b in ir.blocks] == [
|
||||
("Deep", []),
|
||||
("Already MD", []),
|
||||
("Child MD", ["Already MD"]),
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.offline
|
||||
def test_adapter_adjacent_shallower_heading_starts_new_block(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
"""Inverse case: when the second adjacent heading is shallower (level
|
||||
number smaller or equal), it must NOT merge — it starts a new block.
|
||||
"""
|
||||
raw = _write_bundle(
|
||||
tmp_path,
|
||||
[
|
||||
{"type": "text", "text": "1.1 Mid first", "text_level": 2},
|
||||
{"type": "text", "text": "2 Top after", "text_level": 1},
|
||||
{"type": "text", "text": "body"},
|
||||
],
|
||||
)
|
||||
ir = MinerUIRBuilder().normalize_from_workdir(raw, document_name="m.pdf")
|
||||
|
||||
# The first block is heading-only; the writer downstream will keep it
|
||||
# (the merged-heading rule only forwards DEEPER headings).
|
||||
assert len(ir.blocks) == 2
|
||||
assert ir.blocks[0].heading == "1.1 Mid first"
|
||||
assert ir.blocks[0].level == 2
|
||||
assert ir.blocks[0].content_template == "## 1.1 Mid first"
|
||||
|
||||
assert ir.blocks[1].heading == "2 Top after"
|
||||
assert ir.blocks[1].level == 1
|
||||
assert ir.blocks[1].content_template == "# 2 Top after\nbody"
|
||||
|
||||
|
||||
@pytest.mark.offline
|
||||
def test_adapter_body_breaks_adjacent_heading_merge(tmp_path: Path) -> None:
|
||||
"""Once any body content lands in the current block, the next heading —
|
||||
even a deeper one — must flush and open a fresh block (no merge)."""
|
||||
raw = _write_bundle(
|
||||
tmp_path,
|
||||
[
|
||||
{"type": "text", "text": "1 Top", "text_level": 1},
|
||||
{"type": "text", "text": "Intro line under 1."},
|
||||
{"type": "text", "text": "1.1 Mid", "text_level": 2},
|
||||
{"type": "text", "text": "Mid body."},
|
||||
],
|
||||
)
|
||||
ir = MinerUIRBuilder().normalize_from_workdir(raw, document_name="m.pdf")
|
||||
|
||||
assert len(ir.blocks) == 2
|
||||
assert ir.blocks[0].content_template == "# 1 Top\nIntro line under 1."
|
||||
assert ir.blocks[1].heading == "1.1 Mid"
|
||||
assert ir.blocks[1].parent_headings == ["1 Top"]
|
||||
assert ir.blocks[1].content_template == "## 1.1 Mid\nMid body."
|
||||
|
||||
|
||||
@pytest.mark.offline
|
||||
def test_adapter_block_equation_preserves_dollar_wrappers(tmp_path: Path) -> None:
|
||||
"""Block equations keep the ``$$`` markers verbatim on IREquation.latex
|
||||
so the writer renders blocks.jsonl's ``<equation>`` body byte-identical
|
||||
to MinerU's source. The downstream writer is responsible for stripping
|
||||
them when generating equations.json."""
|
||||
raw = _write_bundle(
|
||||
tmp_path,
|
||||
[
|
||||
{
|
||||
"type": "equation",
|
||||
"text": "$$\n\\int_0^1 x dx = \\tfrac{1}{2}\n$$",
|
||||
"text_format": "block",
|
||||
"caption": "Eq A",
|
||||
},
|
||||
],
|
||||
)
|
||||
ir = MinerUIRBuilder().normalize_from_workdir(raw, document_name="b.pdf")
|
||||
eq = ir.blocks[0].equations[0]
|
||||
assert eq.is_block is True
|
||||
# No stripping in the adapter; whitespace.strip() only.
|
||||
assert eq.latex == "$$\n\\int_0^1 x dx = \\tfrac{1}{2}\n$$"
|
||||
|
||||
|
||||
@pytest.mark.offline
|
||||
def test_adapter_empty_equation_dropped(tmp_path: Path) -> None:
|
||||
"""Fix 2: equation items with empty text MUST NOT enter the IR (and
|
||||
consequently not the sidecar). They previously left dangling sidecar
|
||||
entries."""
|
||||
raw = _write_bundle(
|
||||
tmp_path,
|
||||
[
|
||||
{"type": "equation", "text": "", "caption": "ghost"},
|
||||
{"type": "equation", "text": " ", "caption": "ghost"},
|
||||
{"type": "text", "text": "kept"},
|
||||
],
|
||||
)
|
||||
ir = MinerUIRBuilder().normalize_from_workdir(raw, document_name="g.pdf")
|
||||
eq_count = sum(len(b.equations) for b in ir.blocks)
|
||||
assert eq_count == 0
|
||||
assert any(b.content_template == "kept" for b in ir.blocks)
|
||||
|
||||
|
||||
@pytest.mark.offline
|
||||
def test_adapter_empty_table_dropped(tmp_path: Path) -> None:
|
||||
"""Table items with no usable body MUST NOT enter the IR.
|
||||
|
||||
MinerU sometimes misidentifies a page-number / blank region as a table
|
||||
and emits a body-less ``table`` item (missing ``table_body``/``rows``,
|
||||
or with an empty string / empty grid). Leaving such items in the IR
|
||||
would later trip the analyze worker's hard-failure path on empty
|
||||
``content``. The IR builder filters them upstream.
|
||||
"""
|
||||
raw = _write_bundle(
|
||||
tmp_path,
|
||||
[
|
||||
# 1) Body field completely absent.
|
||||
{"type": "table", "num_rows": 0, "num_cols": 0},
|
||||
# 2) Empty string body (matches the real m012-manual.pdf bug).
|
||||
{"type": "table", "table_body": ""},
|
||||
# 3) Empty list body.
|
||||
{"type": "table", "rows": []},
|
||||
# 4) Grid with only blank cells.
|
||||
{"type": "table", "rows": [["", " "], ["\t", ""]]},
|
||||
# 5) A real text item so the IR is not entirely empty.
|
||||
{"type": "text", "text": "kept"},
|
||||
],
|
||||
)
|
||||
ir = MinerUIRBuilder().normalize_from_workdir(raw, document_name="t.pdf")
|
||||
table_count = sum(len(b.tables) for b in ir.blocks)
|
||||
assert table_count == 0
|
||||
# No table placeholder should leak into the rendered content either.
|
||||
joined = "\n".join(b.content_template for b in ir.blocks)
|
||||
assert "TBL:" not in joined
|
||||
assert "kept" in joined
|
||||
|
||||
|
||||
@pytest.mark.offline
|
||||
def test_adapter_page_number_dropped(tmp_path: Path) -> None:
|
||||
"""``page_number`` items are layout noise and MUST NOT enter the IR.
|
||||
|
||||
MinerU emits a ``page_number`` item per page. Empty-text page numbers were
|
||||
already dropped by the fallback's _append_text guard, but page numbers that
|
||||
carry real text (``"12"``, ``"iii"``) previously leaked into block content
|
||||
and contaminated the block's positions with their page_idx. The IR builder
|
||||
now skips them outright regardless of text.
|
||||
"""
|
||||
raw = _write_bundle(
|
||||
tmp_path,
|
||||
[
|
||||
{"type": "text", "text": "kept", "page_idx": 0},
|
||||
# Page number carrying real text — the regression case.
|
||||
{"type": "page_number", "text": "12", "page_idx": 7},
|
||||
# Roman-numeral page number.
|
||||
{"type": "page_number", "text": "iii", "page_idx": 8},
|
||||
# Blank/whitespace page number.
|
||||
{"type": "page_number", "text": "- ", "page_idx": 9},
|
||||
],
|
||||
)
|
||||
ir = MinerUIRBuilder().normalize_from_workdir(raw, document_name="p.pdf")
|
||||
joined = "\n".join(b.content_template for b in ir.blocks)
|
||||
assert "12" not in joined
|
||||
assert "iii" not in joined
|
||||
assert "kept" in joined
|
||||
# The page_idx of skipped page numbers must not leak into block positions.
|
||||
# Anchors are 1-based strings, so page_idx 7/8/9 would surface as 8/9/10.
|
||||
anchors = {pos.anchor for b in ir.blocks for pos in b.positions}
|
||||
assert anchors.isdisjoint({"8", "9", "10"})
|
||||
|
||||
|
||||
@pytest.mark.offline
|
||||
def test_adapter_bbox_attributes_default_and_override(tmp_path: Path) -> None:
|
||||
raw = _write_bundle(tmp_path, [{"type": "text", "text": "x"}])
|
||||
adapter = MinerUIRBuilder()
|
||||
ir = adapter.normalize_from_workdir(raw, document_name="x.pdf")
|
||||
assert ir.bbox_attributes == {"origin": "LEFTTOP", "max": 1000}
|
||||
|
||||
|
||||
@pytest.mark.offline
|
||||
def test_adapter_bbox_attributes_env_override(
|
||||
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
monkeypatch.setenv(
|
||||
"MINERU_BBOX_ATTRIBUTES",
|
||||
'{"origin": "LEFTBOTTOM", "max": 612}',
|
||||
)
|
||||
raw = _write_bundle(tmp_path, [{"type": "text", "text": "x"}])
|
||||
adapter = MinerUIRBuilder()
|
||||
ir = adapter.normalize_from_workdir(raw, document_name="x.pdf")
|
||||
assert ir.bbox_attributes == {"origin": "LEFTBOTTOM", "max": 612}
|
||||
|
||||
|
||||
@pytest.mark.offline
|
||||
def test_adapter_engine_version_recorded_in_split_option(
|
||||
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
monkeypatch.setenv("MINERU_ENGINE_VERSION", "magic-pdf 1.5.4")
|
||||
raw = _write_bundle(tmp_path, [{"type": "text", "text": "x"}])
|
||||
ir = MinerUIRBuilder().normalize_from_workdir(raw, document_name="x.pdf")
|
||||
assert ir.split_option == {"engine_version": "magic-pdf 1.5.4"}
|
||||
|
||||
|
||||
@pytest.mark.offline
|
||||
def test_adapter_missing_content_list_raises(tmp_path: Path) -> None:
|
||||
raw_dir = tmp_path / "bad.mineru_raw"
|
||||
raw_dir.mkdir()
|
||||
with pytest.raises(FileNotFoundError):
|
||||
MinerUIRBuilder().normalize_from_workdir(raw_dir, document_name="x.pdf")
|
||||
|
||||
|
||||
@pytest.mark.offline
|
||||
def test_adapter_html_table_preserves_merged_cells_and_extracts_thead(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
"""MinerU HTML table_body must stay HTML so rowspan/colspan survive."""
|
||||
table_html = (
|
||||
'<table><thead><tr><th rowspan="2">Metric</th>'
|
||||
'<th colspan="2">Group</th></tr><tr><th>A</th><th>B</th></tr></thead>'
|
||||
"<tbody><tr><td>Accuracy</td><td>91%</td><td>93%</td></tr></tbody></table>"
|
||||
)
|
||||
raw = _write_bundle(
|
||||
tmp_path,
|
||||
[
|
||||
{
|
||||
"type": "table",
|
||||
"table_body": table_html,
|
||||
"table_caption": ["Tbl"],
|
||||
}
|
||||
],
|
||||
)
|
||||
ir = MinerUIRBuilder().normalize_from_workdir(raw, document_name="h.pdf")
|
||||
table = ir.blocks[0].tables[0]
|
||||
|
||||
assert table.rows is None
|
||||
assert table.html == table_html
|
||||
assert 'rowspan="2"' in table.html
|
||||
assert 'colspan="2"' in table.html
|
||||
assert table.body_override == table_html.removeprefix("<table>").removesuffix(
|
||||
"</table>"
|
||||
)
|
||||
assert table.num_rows == 3
|
||||
assert table.num_cols == 3
|
||||
assert table.caption == "Tbl"
|
||||
# The <thead> is preserved verbatim (raw HTML) so rowspan/colspan survive —
|
||||
# not flattened into a grid.
|
||||
assert table.table_header == (
|
||||
'<thead><tr><th rowspan="2">Metric</th>'
|
||||
'<th colspan="2">Group</th></tr><tr><th>A</th><th>B</th></tr></thead>'
|
||||
)
|
||||
assert 'rowspan="2"' in table.table_header
|
||||
assert 'colspan="2"' in table.table_header
|
||||
|
||||
|
||||
@pytest.mark.offline
|
||||
def test_adapter_html_table_header_preserves_spans(tmp_path: Path) -> None:
|
||||
"""A spanned <thead> is stored verbatim — colspan/rowspan markup is kept
|
||||
intact rather than flattened into a rectangular grid (so merged-cell
|
||||
semantics survive into every repeated header chunk downstream)."""
|
||||
thead = (
|
||||
"<thead>"
|
||||
'<tr><th>ID</th><th colspan="3">Scores</th></tr>'
|
||||
'<tr><th>n</th><th>A</th><th colspan="2">BC</th></tr>'
|
||||
"</thead>"
|
||||
)
|
||||
table_html = (
|
||||
f"<table>{thead}<tbody>"
|
||||
"<tr><td>1</td><td>x</td><td>y</td><td>z</td></tr>"
|
||||
"</tbody></table>"
|
||||
)
|
||||
raw = _write_bundle(tmp_path, [{"type": "table", "table_body": table_html}])
|
||||
ir = MinerUIRBuilder().normalize_from_workdir(raw, document_name="h.pdf")
|
||||
table = ir.blocks[0].tables[0]
|
||||
assert table.num_cols == 4
|
||||
assert table.table_header == thead
|
||||
assert table.table_header.count('colspan="3"') == 1
|
||||
assert table.table_header.count('colspan="2"') == 1
|
||||
|
||||
|
||||
@pytest.mark.offline
|
||||
def test_adapter_html_table_without_thead_does_not_guess_header(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
"""Only a real HTML <thead> should populate table_header."""
|
||||
table_html = (
|
||||
"<table><tbody><tr><td>H1</td><td>H2</td></tr>"
|
||||
"<tr><td>a</td><td>b</td></tr></tbody></table>"
|
||||
)
|
||||
raw = _write_bundle(
|
||||
tmp_path,
|
||||
[
|
||||
{
|
||||
"type": "table",
|
||||
"table_body": table_html,
|
||||
"num_rows": 1,
|
||||
"num_cols": 2,
|
||||
}
|
||||
],
|
||||
)
|
||||
ir = MinerUIRBuilder().normalize_from_workdir(raw, document_name="h.pdf")
|
||||
table = ir.blocks[0].tables[0]
|
||||
assert table.rows is None
|
||||
assert table.html == table_html
|
||||
assert table.table_header is None
|
||||
|
||||
|
||||
@pytest.mark.offline
|
||||
def test_adapter_html_table_without_thead_falls_back_to_header_grid(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
"""An HTML table whose markup carries no <thead> but for which MinerU
|
||||
supplied a separate ``header`` grid keeps that grid (the writer renders it to
|
||||
a span-less <thead>) rather than silently dropping the recovered header."""
|
||||
table_html = (
|
||||
"<table><tbody><tr><td>a</td><td>b</td></tr></tbody></table>" # no <thead>
|
||||
)
|
||||
raw = _write_bundle(
|
||||
tmp_path,
|
||||
[
|
||||
{
|
||||
"type": "table",
|
||||
"table_body": table_html,
|
||||
"header": [["H1", "H2"]],
|
||||
"num_rows": 1,
|
||||
"num_cols": 2,
|
||||
}
|
||||
],
|
||||
)
|
||||
ir = MinerUIRBuilder().normalize_from_workdir(raw, document_name="h.pdf")
|
||||
table = ir.blocks[0].tables[0]
|
||||
assert table.html == table_html
|
||||
assert table.table_header == [["H1", "H2"]] # grid kept, not dropped
|
||||
|
||||
|
||||
@pytest.mark.offline
|
||||
def test_adapter_html_table_attr_with_gt_keeps_inner_body(tmp_path: Path) -> None:
|
||||
"""A ``>`` inside a ``<table>`` attribute must not truncate the open tag,
|
||||
so ``body_override`` still strips the full ``<table …>`` wrapper."""
|
||||
table_html = '<table data-x="a>b"><tbody><tr><td>v</td></tr></tbody></table>'
|
||||
raw = _write_bundle(
|
||||
tmp_path,
|
||||
[{"type": "table", "table_body": table_html}],
|
||||
)
|
||||
ir = MinerUIRBuilder().normalize_from_workdir(raw, document_name="h.pdf")
|
||||
table = ir.blocks[0].tables[0]
|
||||
assert table.html == table_html
|
||||
assert table.body_override == "<tbody><tr><td>v</td></tr></tbody>"
|
||||
|
||||
|
||||
@pytest.mark.offline
|
||||
def test_adapter_empty_html_table_falls_back_to_full_html(tmp_path: Path) -> None:
|
||||
"""A degenerate ``<table></table>`` has no inner body, so ``body_override``
|
||||
stays None and the writer renders ``table.html`` verbatim."""
|
||||
raw = _write_bundle(
|
||||
tmp_path,
|
||||
[{"type": "table", "table_body": "<table></table>"}],
|
||||
)
|
||||
ir = MinerUIRBuilder().normalize_from_workdir(raw, document_name="h.pdf")
|
||||
table = ir.blocks[0].tables[0]
|
||||
assert table.html == "<table></table>"
|
||||
assert table.body_override is None
|
||||
|
||||
|
||||
@pytest.mark.offline
|
||||
def test_adapter_html_table_strips_html_body_wrapper(tmp_path: Path) -> None:
|
||||
"""MinerU's table model may wrap output in ``<html><body>``; the adapter
|
||||
must unwrap to a single ``<table>`` so the writer does not nest tables and
|
||||
``TABLE_TAG_RE`` consumers are not truncated at the inner ``</table>``."""
|
||||
inner = (
|
||||
'<thead><tr><th colspan="2">G</th></tr></thead>'
|
||||
"<tbody><tr><td>a</td><td>b</td></tr></tbody>"
|
||||
)
|
||||
payload = f"<html><body><table>{inner}</table></body></html>"
|
||||
raw = _write_bundle(tmp_path, [{"type": "table", "table_body": payload}])
|
||||
ir = MinerUIRBuilder().normalize_from_workdir(raw, document_name="h.pdf")
|
||||
table = ir.blocks[0].tables[0]
|
||||
assert table.html == f"<table>{inner}</table>" # <html>/<body> wrapper gone
|
||||
assert table.body_override == inner # block text renders only the inner body
|
||||
assert table.num_cols == 2
|
||||
# The <thead> is kept verbatim (colspan preserved), not flattened to a grid.
|
||||
assert table.table_header == '<thead><tr><th colspan="2">G</th></tr></thead>'
|
||||
|
||||
|
||||
@pytest.mark.offline
|
||||
def test_adapter_list_items_joined_with_newline(tmp_path: Path) -> None:
|
||||
raw = _write_bundle(
|
||||
tmp_path,
|
||||
[
|
||||
{"type": "list", "list_items": ["one", "two", "three"]},
|
||||
],
|
||||
)
|
||||
ir = MinerUIRBuilder().normalize_from_workdir(raw, document_name="l.pdf")
|
||||
assert ir.blocks[0].content_template == "one\ntwo\nthree"
|
||||
|
||||
|
||||
@pytest.mark.offline
|
||||
def test_adapter_drawing_asset_source_only_when_file_exists(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
"""The adapter should declare an AssetSpec for the drawing in both
|
||||
cases, but ``source`` is set only when the bytes are on disk; the
|
||||
writer then warns and skips a missing-source asset."""
|
||||
raw = _write_bundle(
|
||||
tmp_path,
|
||||
[
|
||||
{"type": "image", "img_path": "images/exists.png"},
|
||||
{"type": "image", "img_path": "images/missing.png"},
|
||||
],
|
||||
)
|
||||
(raw / "images").mkdir()
|
||||
(raw / "images" / "exists.png").write_bytes(b"\x89PNG")
|
||||
|
||||
ir = MinerUIRBuilder().normalize_from_workdir(raw, document_name="a.pdf")
|
||||
by_ref = {a.ref: a for a in ir.assets}
|
||||
assert by_ref["images/exists.png"].source is not None
|
||||
assert by_ref["images/missing.png"].source is None
|
||||
|
||||
|
||||
@pytest.mark.offline
|
||||
def test_adapter_refuses_path_traversal_img_path(tmp_path: Path) -> None:
|
||||
"""Untrusted img_path with ``..`` or absolute filesystem segments must
|
||||
not be allowed to point ``AssetSpec.source`` outside ``raw_dir``.
|
||||
|
||||
Otherwise the writer would copy attacker-named files from the host into
|
||||
the sidecar's ``*.blocks.assets/`` directory (file-disclosure path).
|
||||
"""
|
||||
# Place a "secret" file outside the raw bundle that should never be
|
||||
# selectable as an asset source.
|
||||
secret = tmp_path / "secret.txt"
|
||||
secret.write_bytes(b"private")
|
||||
raw = _write_bundle(
|
||||
tmp_path,
|
||||
[
|
||||
{"type": "image", "img_path": "../secret.txt"},
|
||||
{"type": "image", "img_path": str(secret)}, # absolute path
|
||||
],
|
||||
)
|
||||
ir = MinerUIRBuilder().normalize_from_workdir(raw, document_name="x.pdf")
|
||||
by_ref = {a.ref: a for a in ir.assets}
|
||||
|
||||
# Relative ``..`` escape is rejected outright.
|
||||
assert by_ref["../secret.txt"].source is None
|
||||
|
||||
# Absolute filesystem path is reinterpreted as ``images/<basename>``
|
||||
# inside raw_dir. Since no such file exists, source must remain None
|
||||
# (and crucially must not point at the original secret file).
|
||||
abs_asset = by_ref[str(secret)]
|
||||
assert abs_asset.source is None
|
||||
|
||||
|
||||
@pytest.mark.offline
|
||||
def test_adapter_absolute_url_img_path_resolves_to_images_basename(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
"""When MinerU emits an absolute URL in img_path, the downloader saves
|
||||
it as ``images/<basename>``; the adapter must look there too."""
|
||||
raw = _write_bundle(
|
||||
tmp_path,
|
||||
[
|
||||
{
|
||||
"type": "image",
|
||||
"img_path": "https://cdn.example.com/imgs/figure_42.png",
|
||||
},
|
||||
],
|
||||
)
|
||||
(raw / "images").mkdir()
|
||||
(raw / "images" / "figure_42.png").write_bytes(b"\x89PNGfake")
|
||||
|
||||
ir = MinerUIRBuilder().normalize_from_workdir(raw, document_name="u.pdf")
|
||||
asset = ir.assets[0]
|
||||
assert asset.ref == "https://cdn.example.com/imgs/figure_42.png"
|
||||
assert asset.suggested_name == "figure_42.png"
|
||||
assert asset.source is not None
|
||||
assert asset.source.read_bytes() == b"\x89PNGfake"
|
||||
|
||||
|
||||
@pytest.mark.offline
|
||||
def test_adapter_image_url_template_mode_maps_relative_to_images_basename(
|
||||
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
"""When MINERU_IMAGE_URL_TEMPLATE is set, MinerURawClient stores every
|
||||
image reference — including relative ones — at ``images/<basename>``.
|
||||
The adapter must mirror that lookup so the asset is wired up, otherwise
|
||||
the downloaded bytes are silently dropped from the sidecar."""
|
||||
monkeypatch.setenv(
|
||||
"MINERU_IMAGE_URL_TEMPLATE",
|
||||
"http://mineru.internal/assets/{name}",
|
||||
)
|
||||
raw = _write_bundle(
|
||||
tmp_path,
|
||||
[
|
||||
{"type": "image", "img_path": "page/img.png"},
|
||||
],
|
||||
)
|
||||
# Downloader's actual landing spot in template mode.
|
||||
(raw / "images").mkdir()
|
||||
(raw / "images" / "img.png").write_bytes(b"\x89PNGtemplate")
|
||||
# The "naive" location (raw_dir/page/img.png) does NOT exist; in
|
||||
# template mode the downloader does not write there.
|
||||
assert not (raw / "page" / "img.png").exists()
|
||||
|
||||
ir = MinerUIRBuilder().normalize_from_workdir(raw, document_name="t.pdf")
|
||||
asset = ir.assets[0]
|
||||
assert asset.source is not None
|
||||
assert asset.source.read_bytes() == b"\x89PNGtemplate"
|
||||
|
||||
|
||||
@pytest.mark.offline
|
||||
def test_adapter_no_template_keeps_relative_path_lookup(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
"""Sanity: without MINERU_IMAGE_URL_TEMPLATE, a relative img_path still
|
||||
resolves under raw_dir at its original location (regression guard for
|
||||
the template-mode change above)."""
|
||||
raw = _write_bundle(
|
||||
tmp_path,
|
||||
[
|
||||
{"type": "image", "img_path": "page/img.png"},
|
||||
],
|
||||
)
|
||||
(raw / "page").mkdir()
|
||||
(raw / "page" / "img.png").write_bytes(b"\x89PNGrel")
|
||||
|
||||
ir = MinerUIRBuilder().normalize_from_workdir(raw, document_name="r.pdf")
|
||||
asset = ir.assets[0]
|
||||
assert asset.source is not None
|
||||
assert asset.source.read_bytes() == b"\x89PNGrel"
|
||||
@@ -0,0 +1,553 @@
|
||||
"""Integration tests for ``parse_mineru`` with the unified sidecar pipeline.
|
||||
|
||||
These tests stub :class:`MinerURawClient.download_into` so no real MinerU
|
||||
service is contacted; the focus is on:
|
||||
|
||||
- happy path: cache miss → download → sidecar emitted with all expected
|
||||
files in the spec-compliant locations
|
||||
- cache hit: a pre-existing valid ``*.mineru_raw/`` + manifest causes
|
||||
``MinerURawClient.download_into`` NOT to be called
|
||||
- ``LIGHTRAG_FORCE_REPARSE_MINERU=true`` forces a re-download even when
|
||||
the manifest is valid
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
|
||||
from lightrag import LightRAG
|
||||
from lightrag.constants import (
|
||||
FULL_DOCS_FORMAT_LIGHTRAG,
|
||||
)
|
||||
from lightrag.parser.external.mineru import compute_size_and_hash
|
||||
from lightrag.parser.external.mineru.cache import current_mineru_options_signature
|
||||
from lightrag.parser.external.mineru.manifest import (
|
||||
Manifest,
|
||||
ManifestFile,
|
||||
write_manifest,
|
||||
)
|
||||
from lightrag.parser.base import ParseContext
|
||||
from lightrag.parser.registry import get_parser
|
||||
from lightrag.utils import EmbeddingFunc, Tokenizer
|
||||
|
||||
|
||||
async def _parse_via_registry(rag, engine, doc_id, file_path, content_data):
|
||||
"""Drive a parser the way the pipeline worker does (registry dispatch)."""
|
||||
result = await get_parser(engine).parse(
|
||||
ParseContext(rag, doc_id, file_path, content_data)
|
||||
)
|
||||
return result.to_dict()
|
||||
|
||||
|
||||
class _SimpleTokenizerImpl:
|
||||
def encode(self, content: str) -> list[int]:
|
||||
return [ord(ch) for ch in content]
|
||||
|
||||
def decode(self, tokens: list[int]) -> str:
|
||||
return "".join(chr(t) for t in tokens)
|
||||
|
||||
|
||||
async def _mock_embedding(texts: list[str]) -> np.ndarray:
|
||||
return np.random.rand(len(texts), 32)
|
||||
|
||||
|
||||
async def _mock_llm(prompt: Any, **kwargs: Any) -> str:
|
||||
return '{"name":"x","summary":"s","detail_description":"d"}'
|
||||
|
||||
|
||||
def _new_rag(tmp_path: Path) -> LightRAG:
|
||||
return LightRAG(
|
||||
working_dir=str(tmp_path),
|
||||
workspace=f"test-mineru-sidecar-{tmp_path.name}",
|
||||
llm_model_func=_mock_llm,
|
||||
embedding_func=EmbeddingFunc(
|
||||
embedding_dim=32,
|
||||
max_token_size=4096,
|
||||
func=_mock_embedding,
|
||||
),
|
||||
tokenizer=Tokenizer("mock-tokenizer", _SimpleTokenizerImpl()),
|
||||
vlm_process_enable=False,
|
||||
)
|
||||
|
||||
|
||||
_FAKE_TABLE_HTML = (
|
||||
'<table><thead><tr><th rowspan="2">Metric</th>'
|
||||
'<th colspan="2">Score</th></tr><tr><th>A</th><th>B</th></tr></thead>'
|
||||
"<tbody><tr><td>Accuracy</td><td>91%</td><td>93%</td></tr></tbody></table>"
|
||||
)
|
||||
|
||||
|
||||
_FAKE_CONTENT_LIST = [
|
||||
{"type": "text", "text": "1 Introduction", "text_level": 1},
|
||||
{"type": "text", "text": "Body paragraph."},
|
||||
{
|
||||
"type": "table",
|
||||
"table_body": _FAKE_TABLE_HTML,
|
||||
"table_caption": ["Tbl"],
|
||||
"page_idx": 0,
|
||||
"bbox": [10, 10, 100, 50],
|
||||
},
|
||||
{
|
||||
"type": "image",
|
||||
"img_path": "images/img_001.jpg",
|
||||
"image_caption": ["Fig 1"],
|
||||
"page_idx": 1,
|
||||
"bbox": [20, 20, 200, 100],
|
||||
},
|
||||
{"type": "equation", "text": "$E = mc^2$", "caption": "Eq 1", "page_idx": 1},
|
||||
]
|
||||
|
||||
|
||||
def _install_fake_download(monkeypatch: pytest.MonkeyPatch) -> dict[str, int]:
|
||||
"""Replace :meth:`MinerURawClient.download_into` with a recorder that
|
||||
writes a synthetic bundle (content_list.json + one image + manifest).
|
||||
"""
|
||||
import lightrag.parser.external.mineru.client as client_mod
|
||||
|
||||
counters = {"calls": 0, "upload_names": []}
|
||||
|
||||
async def _fake_download(
|
||||
self,
|
||||
raw_dir: Path,
|
||||
source_file_path: Path,
|
||||
*,
|
||||
upload_name: str | None = None,
|
||||
):
|
||||
counters["calls"] += 1
|
||||
counters["upload_names"].append(upload_name)
|
||||
raw_dir.mkdir(parents=True, exist_ok=True)
|
||||
(raw_dir / "content_list.json").write_text(
|
||||
json.dumps(_FAKE_CONTENT_LIST, ensure_ascii=False),
|
||||
encoding="utf-8",
|
||||
)
|
||||
(raw_dir / "images").mkdir(exist_ok=True)
|
||||
(raw_dir / "images" / "img_001.jpg").write_bytes(b"\xff\xd8\xff\xe0fakeJPEG")
|
||||
|
||||
src_size, src_hash = compute_size_and_hash(source_file_path)
|
||||
crit_size, crit_hash = compute_size_and_hash(raw_dir / "content_list.json")
|
||||
files = [
|
||||
ManifestFile(
|
||||
path="images/img_001.jpg",
|
||||
size=(raw_dir / "images" / "img_001.jpg").stat().st_size,
|
||||
)
|
||||
]
|
||||
manifest = Manifest(
|
||||
source_content_hash=src_hash,
|
||||
source_size_bytes=src_size,
|
||||
source_filename_at_parse=upload_name or source_file_path.name,
|
||||
critical_file=ManifestFile(
|
||||
path="content_list.json", size=crit_size, sha256=crit_hash
|
||||
),
|
||||
files=files,
|
||||
total_size_bytes=crit_size + sum(f.size for f in files),
|
||||
task_id=f"fake-{counters['calls']}",
|
||||
api_mode="local",
|
||||
options_signature=current_mineru_options_signature(),
|
||||
)
|
||||
write_manifest(raw_dir, manifest)
|
||||
return manifest
|
||||
|
||||
monkeypatch.setattr(client_mod.MinerURawClient, "download_into", _fake_download)
|
||||
return counters
|
||||
|
||||
|
||||
@pytest.mark.offline
|
||||
def test_parse_mineru_emits_compliant_sidecar(
|
||||
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
"""End-to-end: parse_mineru produces *.parsed/ with spec-compliant
|
||||
blocks.jsonl + per-modality JSONs + assets dir; *.mineru_raw/ kept."""
|
||||
|
||||
async def _run() -> None:
|
||||
monkeypatch.setenv("MINERU_LOCAL_ENDPOINT", "http://mineru.example")
|
||||
counters = _install_fake_download(monkeypatch)
|
||||
|
||||
# Don't move the source out from under the cache validator between
|
||||
# repeated parse_mineru calls.
|
||||
async def _noop_archive(_p: str) -> None:
|
||||
return None
|
||||
|
||||
import lightrag.pipeline as pipeline_module
|
||||
|
||||
monkeypatch.setattr(
|
||||
pipeline_module,
|
||||
"archive_docx_source_after_full_docs_sync",
|
||||
_noop_archive,
|
||||
)
|
||||
|
||||
input_dir = tmp_path / "inputs" / "ws"
|
||||
input_dir.mkdir(parents=True)
|
||||
src = input_dir / "demo.pdf"
|
||||
src.write_bytes(b"PDFPDF" * 256)
|
||||
|
||||
rag = _new_rag(tmp_path)
|
||||
await rag.initialize_storages()
|
||||
try:
|
||||
doc_id = "doc-abcdef0123456789abcdef0123456789"
|
||||
await rag.doc_status.upsert(
|
||||
{
|
||||
doc_id: {
|
||||
"status": "PARSING",
|
||||
"content_summary": "",
|
||||
"content_length": 0,
|
||||
"chunks_count": 0,
|
||||
"chunks_list": [],
|
||||
"created_at": "2026-05-15T00:00:00+00:00",
|
||||
"updated_at": "2026-05-15T00:00:00+00:00",
|
||||
"file_path": "demo.pdf",
|
||||
"track_id": "trk",
|
||||
"content_hash": "",
|
||||
"metadata": {},
|
||||
}
|
||||
}
|
||||
)
|
||||
|
||||
monkeypatch.setattr(
|
||||
rag,
|
||||
"_resolve_source_file_for_parser",
|
||||
lambda _p: str(src),
|
||||
)
|
||||
|
||||
parsed = await _parse_via_registry(
|
||||
rag,
|
||||
"mineru",
|
||||
doc_id=doc_id,
|
||||
file_path="demo.pdf",
|
||||
content_data={},
|
||||
)
|
||||
assert counters["calls"] == 1, "download_into should run once on miss"
|
||||
|
||||
parsed_dir = Path(parsed["blocks_path"]).parent
|
||||
assert parsed["parse_format"] == FULL_DOCS_FORMAT_LIGHTRAG
|
||||
assert parsed_dir.name == "demo.pdf.parsed"
|
||||
|
||||
# Sidecar files present
|
||||
files = {p.name for p in parsed_dir.iterdir() if p.is_file()}
|
||||
assert "demo.blocks.jsonl" in files
|
||||
assert "demo.tables.json" in files
|
||||
assert "demo.drawings.json" in files
|
||||
assert "demo.equations.json" in files
|
||||
assert (parsed_dir / "demo.blocks.assets").is_dir()
|
||||
assert (parsed_dir / "demo.blocks.assets" / "img_001.jpg").is_file()
|
||||
|
||||
# Content of blocks.jsonl
|
||||
blocks_raw = (parsed_dir / "demo.blocks.jsonl").read_text()
|
||||
lines = blocks_raw.splitlines()
|
||||
meta = json.loads(lines[0])
|
||||
rows = [json.loads(line) for line in lines[1:]]
|
||||
assert meta["parse_engine"] == "mineru"
|
||||
assert meta["table_file"] is True
|
||||
assert meta["drawing_file"] is True
|
||||
assert meta["equation_file"] is True
|
||||
assert meta["asset_dir"] is True
|
||||
assert meta["doc_title"] == "1 Introduction"
|
||||
# bbox_attributes present for mineru (PDF coordinate context)
|
||||
assert meta["bbox_attributes"] == {"origin": "LEFTTOP", "max": 1000}
|
||||
|
||||
# Spec fix: <table> placeholder inline, not <cite>
|
||||
contents = " ".join(row.get("content", "") for row in rows)
|
||||
assert '<table id="tb-' in contents
|
||||
assert 'format="html"' in contents
|
||||
assert 'format="json"' not in contents
|
||||
assert 'rowspan="2"' in contents
|
||||
assert 'colspan="2"' in contents
|
||||
assert "<cite" not in contents
|
||||
|
||||
# bbox positions present on at least one block
|
||||
assert any(
|
||||
p.get("type") == "bbox"
|
||||
for row in rows
|
||||
for p in row.get("positions") or []
|
||||
)
|
||||
|
||||
# Drawing path points inside *.blocks.assets/
|
||||
drawings = json.loads((parsed_dir / "demo.drawings.json").read_text())[
|
||||
"drawings"
|
||||
]
|
||||
(drawing_id, drawing_item) = next(iter(drawings.items()))
|
||||
assert drawing_id.startswith("im-")
|
||||
assert drawing_item["path"] == "demo.blocks.assets/img_001.jpg"
|
||||
assert drawing_item["self_ref"] == "content_list.json#/3"
|
||||
|
||||
# Raw bundle preserved next to sidecar
|
||||
raw_dir = parsed_dir.parent / "demo.pdf.mineru_raw"
|
||||
assert (raw_dir / "_manifest.json").is_file()
|
||||
assert (raw_dir / "content_list.json").is_file()
|
||||
assert (raw_dir / "images" / "img_001.jpg").is_file()
|
||||
|
||||
# No legacy non-spec image field on tables
|
||||
tables = json.loads((parsed_dir / "demo.tables.json").read_text())["tables"]
|
||||
(_, table_item) = next(iter(tables.items()))
|
||||
assert "image" not in table_item
|
||||
assert table_item["format"] == "html"
|
||||
assert table_item["content"] == _FAKE_TABLE_HTML
|
||||
assert table_item["dimension"] == [3, 3]
|
||||
assert 'rowspan="2"' in table_item["content"]
|
||||
assert 'colspan="2"' in table_item["content"]
|
||||
# HTML tables store table_header as the raw <thead> (rowspan/colspan
|
||||
# preserved), not a flattened JSON grid.
|
||||
assert table_item["table_header"] == (
|
||||
'<thead><tr><th rowspan="2">Metric</th>'
|
||||
'<th colspan="2">Score</th></tr>'
|
||||
"<tr><th>A</th><th>B</th></tr></thead>"
|
||||
)
|
||||
assert 'rowspan="2"' in table_item["table_header"]
|
||||
assert 'colspan="2"' in table_item["table_header"]
|
||||
assert table_item["self_ref"] == "content_list.json#/2"
|
||||
|
||||
equations = json.loads((parsed_dir / "demo.equations.json").read_text())[
|
||||
"equations"
|
||||
]
|
||||
(_, equation_item) = next(iter(equations.items()))
|
||||
assert equation_item["self_ref"] == "content_list.json#/4"
|
||||
finally:
|
||||
await rag.finalize_storages()
|
||||
|
||||
asyncio.new_event_loop().run_until_complete(_run())
|
||||
|
||||
|
||||
@pytest.mark.offline
|
||||
def test_parse_mineru_cache_hit_skips_download(
|
||||
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
"""A pre-existing valid bundle short-circuits the network call entirely."""
|
||||
|
||||
async def _run() -> None:
|
||||
monkeypatch.setenv("MINERU_LOCAL_ENDPOINT", "http://mineru.example")
|
||||
counters = _install_fake_download(monkeypatch)
|
||||
|
||||
# Don't move the source out from under the cache validator between
|
||||
# repeated parse_mineru calls.
|
||||
async def _noop_archive(_p: str) -> None:
|
||||
return None
|
||||
|
||||
import lightrag.pipeline as pipeline_module
|
||||
|
||||
monkeypatch.setattr(
|
||||
pipeline_module,
|
||||
"archive_docx_source_after_full_docs_sync",
|
||||
_noop_archive,
|
||||
)
|
||||
|
||||
input_dir = tmp_path / "inputs" / "ws"
|
||||
input_dir.mkdir(parents=True)
|
||||
src = input_dir / "demo.pdf"
|
||||
src.write_bytes(b"PDFPDF" * 256)
|
||||
|
||||
rag = _new_rag(tmp_path)
|
||||
await rag.initialize_storages()
|
||||
try:
|
||||
doc_id = "doc-abcdef0123456789abcdef0123456789"
|
||||
await rag.doc_status.upsert(
|
||||
{
|
||||
doc_id: {
|
||||
"status": "PARSING",
|
||||
"content_summary": "",
|
||||
"content_length": 0,
|
||||
"chunks_count": 0,
|
||||
"chunks_list": [],
|
||||
"created_at": "2026-05-15T00:00:00+00:00",
|
||||
"updated_at": "2026-05-15T00:00:00+00:00",
|
||||
"file_path": "demo.pdf",
|
||||
"track_id": "trk",
|
||||
"content_hash": "",
|
||||
"metadata": {},
|
||||
}
|
||||
}
|
||||
)
|
||||
|
||||
monkeypatch.setattr(
|
||||
rag,
|
||||
"_resolve_source_file_for_parser",
|
||||
lambda _p: str(src),
|
||||
)
|
||||
|
||||
# First call: cache miss → download once.
|
||||
await _parse_via_registry(
|
||||
rag,
|
||||
"mineru",
|
||||
doc_id=doc_id,
|
||||
file_path="demo.pdf",
|
||||
content_data={},
|
||||
)
|
||||
assert counters["calls"] == 1
|
||||
|
||||
# Second call: should hit cache.
|
||||
await _parse_via_registry(
|
||||
rag,
|
||||
"mineru",
|
||||
doc_id=doc_id,
|
||||
file_path="demo.pdf",
|
||||
content_data={},
|
||||
)
|
||||
assert counters["calls"] == 1, "cache hit must not re-download"
|
||||
|
||||
# Third call with force-reparse: cache invalidated.
|
||||
monkeypatch.setenv("LIGHTRAG_FORCE_REPARSE_MINERU", "true")
|
||||
await _parse_via_registry(
|
||||
rag,
|
||||
"mineru",
|
||||
doc_id=doc_id,
|
||||
file_path="demo.pdf",
|
||||
content_data={},
|
||||
)
|
||||
assert counters["calls"] == 2
|
||||
finally:
|
||||
await rag.finalize_storages()
|
||||
|
||||
asyncio.new_event_loop().run_until_complete(_run())
|
||||
|
||||
|
||||
@pytest.mark.offline
|
||||
def test_parse_mineru_upload_name_strips_parser_hint(
|
||||
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
"""MinerU upload name should use the canonical filename, not parser
|
||||
hints embedded in the source basename."""
|
||||
|
||||
async def _run() -> None:
|
||||
monkeypatch.setenv("MINERU_LOCAL_ENDPOINT", "http://mineru.example")
|
||||
counters = _install_fake_download(monkeypatch)
|
||||
|
||||
input_dir = tmp_path / "inputs" / "ws"
|
||||
input_dir.mkdir(parents=True)
|
||||
src = input_dir / "demo.[mineru-iet].pdf"
|
||||
src.write_bytes(b"PDFPDF" * 256)
|
||||
|
||||
rag = _new_rag(tmp_path)
|
||||
await rag.initialize_storages()
|
||||
try:
|
||||
doc_id = "doc-abcdef0123456789abcdef0123456789"
|
||||
await rag.doc_status.upsert(
|
||||
{
|
||||
doc_id: {
|
||||
"status": "PARSING",
|
||||
"content_summary": "",
|
||||
"content_length": 0,
|
||||
"chunks_count": 0,
|
||||
"chunks_list": [],
|
||||
"created_at": "2026-05-15T00:00:00+00:00",
|
||||
"updated_at": "2026-05-15T00:00:00+00:00",
|
||||
"file_path": src.name,
|
||||
"track_id": "trk",
|
||||
"content_hash": "",
|
||||
"metadata": {},
|
||||
}
|
||||
}
|
||||
)
|
||||
|
||||
monkeypatch.setattr(
|
||||
rag,
|
||||
"_resolve_source_file_for_parser",
|
||||
lambda _p: str(src),
|
||||
)
|
||||
|
||||
parsed = await _parse_via_registry(
|
||||
rag,
|
||||
"mineru",
|
||||
doc_id=doc_id,
|
||||
file_path=src.name,
|
||||
content_data={},
|
||||
)
|
||||
|
||||
assert counters["upload_names"] == ["demo.pdf"]
|
||||
parsed_dir = Path(parsed["blocks_path"]).parent
|
||||
assert parsed_dir.name == "demo.pdf.parsed"
|
||||
manifest = json.loads(
|
||||
(
|
||||
parsed_dir.parent / "demo.pdf.mineru_raw" / "_manifest.json"
|
||||
).read_text(encoding="utf-8")
|
||||
)
|
||||
assert manifest["source_filename_at_parse"] == "demo.pdf"
|
||||
finally:
|
||||
await rag.finalize_storages()
|
||||
|
||||
asyncio.new_event_loop().run_until_complete(_run())
|
||||
|
||||
|
||||
@pytest.mark.offline
|
||||
def test_parse_mineru_cache_invalidates_on_source_change(
|
||||
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
"""Source file content swapped (same/different size) → cache miss."""
|
||||
|
||||
async def _run() -> None:
|
||||
monkeypatch.setenv("MINERU_LOCAL_ENDPOINT", "http://mineru.example")
|
||||
counters = _install_fake_download(monkeypatch)
|
||||
|
||||
# Don't move the source out from under the cache validator between
|
||||
# repeated parse_mineru calls.
|
||||
async def _noop_archive(_p: str) -> None:
|
||||
return None
|
||||
|
||||
import lightrag.pipeline as pipeline_module
|
||||
|
||||
monkeypatch.setattr(
|
||||
pipeline_module,
|
||||
"archive_docx_source_after_full_docs_sync",
|
||||
_noop_archive,
|
||||
)
|
||||
|
||||
input_dir = tmp_path / "inputs" / "ws"
|
||||
input_dir.mkdir(parents=True)
|
||||
src = input_dir / "demo.pdf"
|
||||
src.write_bytes(b"PDFPDF" * 256)
|
||||
|
||||
rag = _new_rag(tmp_path)
|
||||
await rag.initialize_storages()
|
||||
try:
|
||||
doc_id = "doc-abcdef0123456789abcdef0123456789"
|
||||
await rag.doc_status.upsert(
|
||||
{
|
||||
doc_id: {
|
||||
"status": "PARSING",
|
||||
"content_summary": "",
|
||||
"content_length": 0,
|
||||
"chunks_count": 0,
|
||||
"chunks_list": [],
|
||||
"created_at": "2026-05-15T00:00:00+00:00",
|
||||
"updated_at": "2026-05-15T00:00:00+00:00",
|
||||
"file_path": "demo.pdf",
|
||||
"track_id": "trk",
|
||||
"content_hash": "",
|
||||
"metadata": {},
|
||||
}
|
||||
}
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
rag,
|
||||
"_resolve_source_file_for_parser",
|
||||
lambda _p: str(src),
|
||||
)
|
||||
|
||||
await _parse_via_registry(
|
||||
rag,
|
||||
"mineru",
|
||||
doc_id=doc_id,
|
||||
file_path="demo.pdf",
|
||||
content_data={},
|
||||
)
|
||||
assert counters["calls"] == 1
|
||||
|
||||
# Same length, different bytes → fast-path passes, hash fails.
|
||||
data = src.read_bytes()
|
||||
src.write_bytes(b"\x00" + data[1:])
|
||||
|
||||
await _parse_via_registry(
|
||||
rag,
|
||||
"mineru",
|
||||
doc_id=doc_id,
|
||||
file_path="demo.pdf",
|
||||
content_data={},
|
||||
)
|
||||
assert counters["calls"] == 2
|
||||
finally:
|
||||
await rag.finalize_storages()
|
||||
|
||||
asyncio.new_event_loop().run_until_complete(_run())
|
||||
Vendored
+235
@@ -0,0 +1,235 @@
|
||||
"""Tests for shared helpers in ``lightrag/parser/external/``.
|
||||
|
||||
These cover the pure functions reused across engine integrations:
|
||||
|
||||
- ``compute_size_and_hash`` — single-read (size, hash) pair
|
||||
- ``clear_dir_contents`` — empty a directory while keeping it
|
||||
- ``raw_dir_for_parsed_dir`` — suffix-bound raw dir naming
|
||||
- ``safe_extract_zip`` — refuses path traversal and absolute paths
|
||||
- ``env_bool`` / ``env_int`` / ``env_json`` — env parsing
|
||||
- ``Manifest`` round-trip via ``write_manifest`` / ``load_manifest``
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import io
|
||||
import json
|
||||
import zipfile
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from lightrag.parser.external import (
|
||||
Manifest,
|
||||
ManifestFile,
|
||||
clear_dir_contents,
|
||||
compute_size_and_hash,
|
||||
env_bool,
|
||||
env_int,
|
||||
env_json,
|
||||
load_manifest,
|
||||
raw_dir_for_parsed_dir,
|
||||
safe_extract_zip,
|
||||
write_manifest,
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# compute_size_and_hash
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_compute_size_and_hash_stable(tmp_path: Path) -> None:
|
||||
p = tmp_path / "f.bin"
|
||||
payload = b"hello-external-parser" * 1024
|
||||
p.write_bytes(payload)
|
||||
|
||||
size_a, hash_a = compute_size_and_hash(p)
|
||||
size_b, hash_b = compute_size_and_hash(p)
|
||||
|
||||
assert size_a == len(payload) == size_b
|
||||
assert hash_a == hash_b
|
||||
assert hash_a.startswith("sha256:") and len(hash_a) == len("sha256:") + 64
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# clear_dir_contents
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_clear_dir_contents_keeps_dir_and_removes_children(tmp_path: Path) -> None:
|
||||
d = tmp_path / "raw"
|
||||
d.mkdir()
|
||||
(d / "a.txt").write_text("hi")
|
||||
sub = d / "nested"
|
||||
sub.mkdir()
|
||||
(sub / "b.bin").write_bytes(b"x" * 10)
|
||||
|
||||
clear_dir_contents(d)
|
||||
|
||||
assert d.is_dir()
|
||||
assert list(d.iterdir()) == []
|
||||
|
||||
|
||||
def test_clear_dir_contents_noop_when_missing(tmp_path: Path) -> None:
|
||||
clear_dir_contents(tmp_path / "does-not-exist")
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# raw_dir_for_parsed_dir
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_raw_dir_for_parsed_dir_with_suffix(tmp_path: Path) -> None:
|
||||
parsed = tmp_path / "demo.pdf.parsed"
|
||||
raw = raw_dir_for_parsed_dir(parsed, suffix=".docling_raw")
|
||||
assert raw == tmp_path / "demo.pdf.docling_raw"
|
||||
|
||||
|
||||
def test_raw_dir_for_parsed_dir_without_parsed_suffix(tmp_path: Path) -> None:
|
||||
parsed = tmp_path / "other_dir"
|
||||
raw = raw_dir_for_parsed_dir(parsed, suffix=".docling_raw")
|
||||
assert raw == tmp_path / "other_dir.docling_raw"
|
||||
|
||||
|
||||
def test_raw_dir_for_parsed_dir_rejects_bad_suffix(tmp_path: Path) -> None:
|
||||
with pytest.raises(ValueError):
|
||||
raw_dir_for_parsed_dir(tmp_path / "x.parsed", suffix="docling_raw")
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# safe_extract_zip
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _make_zip(entries: dict[str, bytes]) -> bytes:
|
||||
buf = io.BytesIO()
|
||||
with zipfile.ZipFile(buf, "w") as zf:
|
||||
for name, payload in entries.items():
|
||||
zf.writestr(name, payload)
|
||||
return buf.getvalue()
|
||||
|
||||
|
||||
def test_safe_extract_zip_extracts_flat_bundle(tmp_path: Path) -> None:
|
||||
payload = _make_zip(
|
||||
{
|
||||
"demo.json": b'{"schema_name": "DoclingDocument"}',
|
||||
"demo.md": b"# demo",
|
||||
"artifacts/image_000000.png": b"\x89PNG fake",
|
||||
}
|
||||
)
|
||||
dest = tmp_path / "raw"
|
||||
names = safe_extract_zip(payload, dest)
|
||||
|
||||
assert (dest / "demo.json").read_bytes().startswith(b'{"schema_name"')
|
||||
assert (dest / "demo.md").read_text(encoding="utf-8") == "# demo"
|
||||
assert (dest / "artifacts" / "image_000000.png").is_file()
|
||||
assert sorted(names) == sorted(
|
||||
["demo.json", "demo.md", "artifacts/image_000000.png"]
|
||||
)
|
||||
|
||||
|
||||
def test_safe_extract_zip_rejects_path_traversal(tmp_path: Path) -> None:
|
||||
payload = _make_zip({"../evil.txt": b"oops"})
|
||||
with pytest.raises(RuntimeError, match="unsafe path"):
|
||||
safe_extract_zip(payload, tmp_path / "raw")
|
||||
|
||||
|
||||
def test_safe_extract_zip_rejects_absolute_path(tmp_path: Path) -> None:
|
||||
payload = _make_zip({"/etc/passwd": b"oops"})
|
||||
with pytest.raises(RuntimeError, match="unsafe path"):
|
||||
safe_extract_zip(payload, tmp_path / "raw")
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# env coercion
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_env_bool_truthy_falsy(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
for raw in ("1", "true", "yes", "ON"):
|
||||
monkeypatch.setenv("X", raw)
|
||||
assert env_bool("X", False) is True
|
||||
for raw in ("0", "false", "no", "off"):
|
||||
monkeypatch.setenv("X", raw)
|
||||
assert env_bool("X", True) is False
|
||||
|
||||
|
||||
def test_env_bool_falls_back_on_unrecognized(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
monkeypatch.setenv("X", "maybe")
|
||||
assert env_bool("X", True) is True
|
||||
assert env_bool("X", False) is False
|
||||
|
||||
|
||||
def test_env_int_falls_back_on_garbage(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
monkeypatch.setenv("X", "not-an-int")
|
||||
assert env_int("X", 7) == 7
|
||||
|
||||
|
||||
def test_env_json_returns_default_on_garbage(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
monkeypatch.setenv("X", "{bad json")
|
||||
assert env_json("X", {"origin": "LEFTBOTTOM"}) == {"origin": "LEFTBOTTOM"}
|
||||
|
||||
|
||||
def test_env_json_parses_object(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
monkeypatch.setenv("X", '{"a": 1, "b": [2, 3]}')
|
||||
assert env_json("X", None) == {"a": 1, "b": [2, 3]}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Manifest round-trip
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_manifest_round_trip(tmp_path: Path) -> None:
|
||||
raw = tmp_path / "demo.docling_raw"
|
||||
raw.mkdir()
|
||||
crit = ManifestFile(path="demo.json", size=42, sha256="sha256:" + "a" * 64)
|
||||
other = ManifestFile(path="demo.md", size=10)
|
||||
manifest = Manifest(
|
||||
engine="docling",
|
||||
source_content_hash="sha256:" + "b" * 64,
|
||||
source_size_bytes=100,
|
||||
source_filename_at_parse="demo.pdf",
|
||||
critical_file=crit,
|
||||
files=[other],
|
||||
total_size_bytes=52,
|
||||
task_id="task-xyz",
|
||||
endpoint_signature="http://l4ai:5001",
|
||||
engine_version="1.18.0",
|
||||
options_signature="sha256:" + "c" * 64,
|
||||
downloaded_at="2026-05-18T00:00:00Z",
|
||||
extras={"to_formats": ["json", "md"]},
|
||||
)
|
||||
write_manifest(raw, manifest)
|
||||
|
||||
payload = json.loads((raw / "_manifest.json").read_text(encoding="utf-8"))
|
||||
assert payload["engine"] == "docling"
|
||||
assert payload["options_signature"] == "sha256:" + "c" * 64
|
||||
assert payload["extras"] == {"to_formats": ["json", "md"]}
|
||||
|
||||
loaded = load_manifest(raw, expected_engine="docling")
|
||||
assert loaded is not None
|
||||
assert loaded.task_id == "task-xyz"
|
||||
assert loaded.critical_file.size == 42
|
||||
assert loaded.files[0].path == "demo.md"
|
||||
|
||||
|
||||
def test_manifest_load_rejects_wrong_engine(tmp_path: Path) -> None:
|
||||
raw = tmp_path / "demo.docling_raw"
|
||||
raw.mkdir()
|
||||
manifest = Manifest(
|
||||
engine="mineru",
|
||||
source_content_hash="sha256:" + "0" * 64,
|
||||
source_size_bytes=1,
|
||||
source_filename_at_parse="x",
|
||||
critical_file=ManifestFile(path="c", size=1, sha256="sha256:" + "1" * 64),
|
||||
files=[],
|
||||
total_size_bytes=1,
|
||||
)
|
||||
write_manifest(raw, manifest)
|
||||
assert load_manifest(raw, expected_engine="docling") is None
|
||||
|
||||
|
||||
def test_manifest_load_handles_missing_file(tmp_path: Path) -> None:
|
||||
assert load_manifest(tmp_path / "no-such-dir", expected_engine="docling") is None
|
||||
+104
@@ -0,0 +1,104 @@
|
||||
"""Proof that engine params thread identically into both external-parser hooks.
|
||||
|
||||
The ONLY test exercising ``ExternalParserBase.parse``'s decode + thread: the
|
||||
per-file engine params decoded from ``content_data['parse_engine']`` must reach
|
||||
BOTH ``is_bundle_valid`` (cache-hit check) and ``download_into`` (request), or a
|
||||
cache signature could be computed with different params than the bundle was
|
||||
parsed with. Also asserts a malformed stored directive fails the doc loudly.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
|
||||
from lightrag.parser.base import ParseContext
|
||||
from lightrag.parser.external._base import ExternalParserBase
|
||||
|
||||
|
||||
class _Stop(Exception):
|
||||
"""Sentinel to halt parse() right after the hook under test runs."""
|
||||
|
||||
|
||||
def _make_ctx(tmp_path: Path, parse_engine: str) -> ParseContext:
|
||||
source = tmp_path / "doc.pdf"
|
||||
source.write_bytes(b"%PDF-1.4 test")
|
||||
ctx = ParseContext(
|
||||
rag=SimpleNamespace(),
|
||||
doc_id="doc-1",
|
||||
file_path=str(source),
|
||||
content_data={"parse_engine": parse_engine},
|
||||
)
|
||||
# Avoid the pipeline-layer source resolution; hand the template a simple
|
||||
# resolved-source stand-in pointing at our temp file.
|
||||
rs = SimpleNamespace(
|
||||
source_path=source, parsed_dir=tmp_path / "parsed", document_name="doc.pdf"
|
||||
)
|
||||
ctx.resolve = lambda engine_name: rs # type: ignore[assignment]
|
||||
return ctx
|
||||
|
||||
|
||||
class _RecordingParser(ExternalParserBase):
|
||||
engine_name = "mineru" # registered engine that accepts page_range
|
||||
raw_dir_suffix = ".raw"
|
||||
force_reparse_env = "LIGHTRAG_TEST_FORCE_REPARSE"
|
||||
|
||||
def __init__(self, *, hit: bool) -> None:
|
||||
self._hit = hit
|
||||
self.seen: dict[str, object] = {}
|
||||
self.download_called = False
|
||||
|
||||
def is_bundle_valid(self, raw_dir, source_path, *, engine_params=None):
|
||||
self.seen["is_bundle_valid"] = engine_params
|
||||
return self._hit
|
||||
|
||||
async def download_into(
|
||||
self, raw_dir, source_path, *, upload_name, engine_params=None
|
||||
):
|
||||
self.seen["download_into"] = engine_params
|
||||
self.download_called = True
|
||||
raise _Stop()
|
||||
|
||||
def build_ir(self, raw_dir, document_name): # hit path halts here
|
||||
raise _Stop()
|
||||
|
||||
|
||||
def test_miss_path_threads_identical_engine_params(tmp_path):
|
||||
ctx = _make_ctx(tmp_path, "mineru(page_range=1-3)")
|
||||
parser = _RecordingParser(hit=False)
|
||||
with pytest.raises(_Stop):
|
||||
asyncio.run(parser.parse(ctx))
|
||||
assert parser.download_called is True
|
||||
# Both hooks received the SAME decoded params dict.
|
||||
assert parser.seen["is_bundle_valid"] == {"page_range": "1-3"}
|
||||
assert parser.seen["download_into"] == {"page_range": "1-3"}
|
||||
|
||||
|
||||
def test_hit_path_skips_client(tmp_path):
|
||||
ctx = _make_ctx(tmp_path, "mineru(page_range=1-3)")
|
||||
parser = _RecordingParser(hit=True)
|
||||
with pytest.raises(_Stop): # halts in build_ir
|
||||
asyncio.run(parser.parse(ctx))
|
||||
assert parser.download_called is False
|
||||
assert parser.seen["is_bundle_valid"] == {"page_range": "1-3"}
|
||||
|
||||
|
||||
def test_bare_engine_threads_none(tmp_path):
|
||||
ctx = _make_ctx(tmp_path, "mineru")
|
||||
parser = _RecordingParser(hit=False)
|
||||
with pytest.raises(_Stop):
|
||||
asyncio.run(parser.parse(ctx))
|
||||
assert parser.seen["is_bundle_valid"] is None
|
||||
assert parser.seen["download_into"] is None
|
||||
|
||||
|
||||
def test_malformed_stored_parse_engine_fails_loudly(tmp_path):
|
||||
ctx = _make_ctx(tmp_path, "mineru(page_range=") # unbalanced
|
||||
parser = _RecordingParser(hit=False)
|
||||
with pytest.raises(ValueError, match="invalid parse_engine"):
|
||||
asyncio.run(parser.parse(ctx))
|
||||
# Failed before reaching either hook.
|
||||
assert "is_bundle_valid" not in parser.seen
|
||||
@@ -0,0 +1 @@
|
||||
"""Tests for the native markdown parser engine (.md / .textpack)."""
|
||||
@@ -0,0 +1,101 @@
|
||||
"""End-to-end: markdown → IRDoc → sidecar files, and re-parse stability.
|
||||
|
||||
Drives the parser's ``extract`` + ``build_ir`` hooks through the real
|
||||
``write_sidecar`` (the same path ``NativeParserBase.parse`` uses) and asserts
|
||||
the produced sidecar shape, then re-runs to confirm byte-stable content blocks
|
||||
(no runtime stamping leaks into the per-block content).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
from lightrag.parser.markdown.parser import NativeMarkdownParser
|
||||
from lightrag.sidecar import write_sidecar
|
||||
|
||||
_MD = """# Doc Title
|
||||
|
||||
Intro .
|
||||
|
||||
## Section A
|
||||
|
||||
| Name | Age |
|
||||
|------|-----|
|
||||
| Bob | 30 |
|
||||
|
||||
$$
|
||||
E = mc^2
|
||||
$$
|
||||
|
||||
<table><thead><tr><th>K</th></tr></thead><tbody><tr><td>a</td></tr></tbody></table>
|
||||
"""
|
||||
|
||||
|
||||
def _build_sidecar(tmp_path: Path) -> Path:
|
||||
parser = NativeMarkdownParser()
|
||||
parsed_dir = tmp_path / "doc.md.parsed"
|
||||
asset_dir = parsed_dir / "doc.blocks.assets"
|
||||
parsed_dir.mkdir(parents=True)
|
||||
asset_dir.mkdir()
|
||||
blocks, _, meta = parser._extract_text(_MD, bundle_root=None)
|
||||
ir = parser.build_ir(
|
||||
blocks,
|
||||
document_name="doc.md",
|
||||
asset_dir_name=asset_dir.name,
|
||||
metadata=meta,
|
||||
)
|
||||
write_sidecar(
|
||||
ir,
|
||||
parsed_dir=parsed_dir,
|
||||
doc_id="doc-test",
|
||||
engine="native",
|
||||
clean_parsed_dir=False,
|
||||
block_drawing_path_style=parser.sidecar_path_style,
|
||||
)
|
||||
return parsed_dir
|
||||
|
||||
|
||||
def _content_lines(parsed_dir: Path) -> list[str]:
|
||||
lines = (parsed_dir / "doc.blocks.jsonl").read_text().splitlines()
|
||||
return [line for line in lines if json.loads(line).get("type") == "content"]
|
||||
|
||||
|
||||
def test_sidecar_files_and_payload(tmp_path: Path):
|
||||
parsed_dir = _build_sidecar(tmp_path)
|
||||
assert (parsed_dir / "doc.blocks.jsonl").is_file()
|
||||
assert (parsed_dir / "doc.tables.json").is_file()
|
||||
assert (parsed_dir / "doc.equations.json").is_file()
|
||||
assert (parsed_dir / "doc.drawings.json").is_file()
|
||||
assets = list((parsed_dir / "doc.blocks.assets").iterdir())
|
||||
assert len(assets) == 1 and assets[0].read_bytes().startswith(b"\x89PNG")
|
||||
|
||||
meta_line = json.loads(
|
||||
(parsed_dir / "doc.blocks.jsonl").read_text().splitlines()[0]
|
||||
)
|
||||
assert meta_line["document_format"] == "md"
|
||||
assert meta_line["doc_title"] == "Doc Title"
|
||||
assert meta_line["parse_engine"] == "native"
|
||||
|
||||
tables = json.loads((parsed_dir / "doc.tables.json").read_text())["tables"]
|
||||
fmts = sorted(t["format"] for t in tables.values())
|
||||
assert fmts == ["html", "json"]
|
||||
# Pipe-table header survives as a JSON grid; HTML header as a <thead>.
|
||||
json_table = next(t for t in tables.values() if t["format"] == "json")
|
||||
assert json.loads(json_table["table_header"]) == [["Name", "Age"]]
|
||||
html_table = next(t for t in tables.values() if t["format"] == "html")
|
||||
assert "<thead>" in html_table["table_header"]
|
||||
|
||||
equations = json.loads((parsed_dir / "doc.equations.json").read_text())["equations"]
|
||||
assert any(e["content"] == "E = mc^2" for e in equations.values())
|
||||
|
||||
drawings = json.loads((parsed_dir / "doc.drawings.json").read_text())["drawings"]
|
||||
(drawing,) = drawings.values()
|
||||
assert drawing["path"].startswith("doc.blocks.assets/")
|
||||
assert drawing["format"] == "png"
|
||||
|
||||
|
||||
def test_reparse_content_is_byte_stable(tmp_path: Path):
|
||||
first = _content_lines(_build_sidecar(tmp_path / "a"))
|
||||
second = _content_lines(_build_sidecar(tmp_path / "b"))
|
||||
assert first == second
|
||||
@@ -0,0 +1,182 @@
|
||||
"""Unit tests for ``lightrag.parser.markdown.extract`` (pure extraction).
|
||||
|
||||
Locks in the supported-subset contract: ATX heading splitting, fenced-code
|
||||
suppression, pipe / HTML table recognition with headers, block-level ``$$``
|
||||
math (and inline ``$`` left alone), and inline image resolution via a stubbed
|
||||
resolver.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from lightrag.parser.markdown.extract import (
|
||||
PREFACE_HEADING,
|
||||
ResolvedImage,
|
||||
extract_markdown,
|
||||
)
|
||||
|
||||
|
||||
class _StubResolver:
|
||||
"""Resolves ``http(s)`` → external link, everything else → local bytes."""
|
||||
|
||||
def __init__(self) -> None:
|
||||
self.calls: list[str] = []
|
||||
|
||||
def resolve(self, src: str) -> ResolvedImage:
|
||||
self.calls.append(src)
|
||||
if src.startswith(("http://", "https://")):
|
||||
return ResolvedImage(kind="external", url=src, fmt="png")
|
||||
return ResolvedImage(
|
||||
kind="local",
|
||||
asset_ref="sha:" + src,
|
||||
data=b"BYTES:" + src.encode(),
|
||||
suggested_name="img.png",
|
||||
fmt="png",
|
||||
)
|
||||
|
||||
|
||||
def _extract(md: str):
|
||||
return extract_markdown(md, image_resolver=_StubResolver())
|
||||
|
||||
|
||||
def test_headings_split_blocks_and_track_parents():
|
||||
md = "# A\nintro\n## B\nbody\n### C\ndeep\n# D\nlast"
|
||||
ex = _extract(md)
|
||||
summary = [(b["heading"], b["level"], b["parent_headings"]) for b in ex.blocks]
|
||||
assert summary == [
|
||||
("A", 1, []),
|
||||
("B", 2, ["A"]),
|
||||
("C", 3, ["A", "B"]),
|
||||
("D", 1, []),
|
||||
]
|
||||
# Heading line is rendered into the block body (markdown-style).
|
||||
assert ex.blocks[0]["content"].startswith("# A")
|
||||
|
||||
|
||||
def test_content_before_first_heading_is_preface():
|
||||
ex = _extract("loose intro line\n\n# Real")
|
||||
assert ex.blocks[0]["heading"] == PREFACE_HEADING
|
||||
assert ex.blocks[0]["level"] == 0
|
||||
assert "loose intro line" in ex.blocks[0]["content"]
|
||||
|
||||
|
||||
def test_fenced_code_suppresses_all_detection():
|
||||
md = "# H\n```python\n# not a heading\n$$ not eq $$\n| not | table |\n\n```\n"
|
||||
ex = _extract(md)
|
||||
# Only the single real heading produced a block; fence content stays verbatim.
|
||||
assert len(ex.blocks) == 1
|
||||
assert not ex.tables and not ex.equations and not ex.drawings
|
||||
assert "# not a heading" in ex.blocks[0]["content"]
|
||||
assert "$$ not eq $$" in ex.blocks[0]["content"]
|
||||
|
||||
|
||||
def test_tilde_fence_supported():
|
||||
md = "# H\n~~~\n## still code\n~~~\nafter"
|
||||
ex = _extract(md)
|
||||
assert len(ex.blocks) == 1
|
||||
assert "## still code" in ex.blocks[0]["content"]
|
||||
|
||||
|
||||
def test_pipe_table_with_header():
|
||||
md = "| Name | Age |\n|------|-----|\n| Bob | 30 |\n| Sue | 25 |\n"
|
||||
ex = _extract(md)
|
||||
assert len(ex.tables) == 1
|
||||
(table,) = ex.tables.values()
|
||||
assert table["kind"] == "pipe"
|
||||
assert table["rows"] == [["Bob", "30"], ["Sue", "25"]]
|
||||
assert table["header"] == [["Name", "Age"]]
|
||||
|
||||
|
||||
def test_pipe_table_requires_delimiter_row():
|
||||
# A pipe-containing line with no delimiter row underneath is plain text.
|
||||
md = "| just | text |\nnot a delimiter\n"
|
||||
ex = _extract(md)
|
||||
assert not ex.tables
|
||||
|
||||
|
||||
def test_pipe_line_over_thematic_break_is_not_a_table():
|
||||
# A pipe-containing paragraph followed by a bare ``---`` (thematic break /
|
||||
# setext underline) has mismatched column counts (2 vs 1) and so is NOT a
|
||||
# GFM table — it must stay plain text rather than be misrecognised.
|
||||
md = "foo | bar\n---\nnext paragraph\n"
|
||||
ex = _extract(md)
|
||||
assert not ex.tables
|
||||
assert "foo | bar" in ex.blocks[0]["content"]
|
||||
|
||||
|
||||
def test_pipe_table_column_count_must_match_header():
|
||||
# Delimiter row column count differs from the header → not a table.
|
||||
md = "| a | b | c |\n| --- | --- |\n| 1 | 2 | 3 |\n"
|
||||
ex = _extract(md)
|
||||
assert not ex.tables
|
||||
|
||||
|
||||
def test_html_table_captured_verbatim_spanning_lines():
|
||||
md = (
|
||||
"<table>\n"
|
||||
"<thead><tr><th>K</th></tr></thead>\n"
|
||||
"<tbody><tr><td>a</td></tr></tbody>\n"
|
||||
"</table>\n"
|
||||
)
|
||||
ex = _extract(md)
|
||||
assert len(ex.tables) == 1
|
||||
(table,) = ex.tables.values()
|
||||
assert table["kind"] == "html"
|
||||
assert "<thead>" in table["html"] and "</table>" in table["html"]
|
||||
|
||||
|
||||
def test_block_equation_single_and_multiline():
|
||||
md = "$$ E = mc^2 $$\n\ntext\n\n$$\n\\sum x\n$$\n"
|
||||
ex = _extract(md)
|
||||
latexes = list(ex.equations.values())
|
||||
assert "E = mc^2" in latexes
|
||||
assert "\\sum x" in latexes
|
||||
|
||||
|
||||
def test_inline_dollar_not_recognized_as_equation():
|
||||
ex = _extract("cost is $5 and $10 today")
|
||||
assert not ex.equations
|
||||
assert "$5" in ex.blocks[0]["content"]
|
||||
|
||||
|
||||
def test_unclosed_block_equation_is_plain_text():
|
||||
ex = _extract("$$\nE = mc^2\nno closing")
|
||||
assert not ex.equations
|
||||
assert "$$" in ex.blocks[0]["content"]
|
||||
|
||||
|
||||
def test_inline_image_local_and_external_and_dedup():
|
||||
md = " and  and "
|
||||
resolver = _StubResolver()
|
||||
ex = extract_markdown(md, image_resolver=resolver)
|
||||
# Three occurrences → three drawings; same local src deduped to one asset.
|
||||
assert len(ex.drawings) == 3
|
||||
assert len(ex.assets) == 1
|
||||
kinds = sorted(d["kind"] for d in ex.drawings.values())
|
||||
assert kinds == ["external", "local", "local"]
|
||||
# Asset dedup is by ``asset_ref`` in extract (the resolver itself owns any
|
||||
# per-src call caching), so the stub sees one call per occurrence.
|
||||
assert resolver.calls.count("one.png") == 2
|
||||
|
||||
|
||||
def test_base64_src_not_echoed_into_drawing_src():
|
||||
md = ""
|
||||
|
||||
class _B64:
|
||||
def resolve(self, src):
|
||||
return ResolvedImage(
|
||||
kind="local",
|
||||
asset_ref="sha:b64",
|
||||
data=b"ABC",
|
||||
suggested_name="image.png",
|
||||
fmt="png",
|
||||
)
|
||||
|
||||
ex = extract_markdown(md, image_resolver=_B64())
|
||||
(drawing,) = ex.drawings.values()
|
||||
assert drawing["src"] == ""
|
||||
|
||||
|
||||
def test_reference_style_image_not_recognized():
|
||||
# ``![alt][id]`` reference-style is out of the supported subset.
|
||||
ex = _extract("![alt][id]\n\n[id]: real.png")
|
||||
assert not ex.drawings
|
||||
@@ -0,0 +1,155 @@
|
||||
"""Unit tests for ``NativeMarkdownIRBuilder`` (markers + side tables → IRDoc)."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from lightrag.parser.markdown.extract import (
|
||||
drawing_marker,
|
||||
equation_marker,
|
||||
table_marker,
|
||||
)
|
||||
from lightrag.parser.markdown.ir_builder import NativeMarkdownIRBuilder
|
||||
|
||||
|
||||
def _normalize(blocks, meta, *, document_name="doc.md"):
|
||||
return NativeMarkdownIRBuilder().normalize(
|
||||
blocks,
|
||||
document_name=document_name,
|
||||
asset_dir_name="doc.blocks.assets",
|
||||
parse_metadata=meta,
|
||||
)
|
||||
|
||||
|
||||
def test_pipe_table_becomes_json_irtable_with_header_grid():
|
||||
blocks = [
|
||||
{
|
||||
"heading": "H",
|
||||
"level": 1,
|
||||
"parent_headings": [],
|
||||
"content": table_marker("t1"),
|
||||
}
|
||||
]
|
||||
meta = {
|
||||
"md_tables": {
|
||||
"t1": {"kind": "pipe", "rows": [["a", "b"]], "header": [["H1", "H2"]]}
|
||||
}
|
||||
}
|
||||
ir = _normalize(blocks, meta)
|
||||
(table,) = ir.blocks[0].tables
|
||||
assert "{{TBL:" in ir.blocks[0].content_template
|
||||
assert table.rows == [["a", "b"]]
|
||||
assert table.html is None
|
||||
assert table.table_header == [["H1", "H2"]]
|
||||
assert (table.num_rows, table.num_cols) == (1, 2)
|
||||
|
||||
|
||||
def test_html_table_becomes_html_irtable_with_thead_and_dims():
|
||||
html = "<table><thead><tr><th>K</th><th>V</th></tr></thead><tbody><tr><td>a</td><td>1</td></tr></tbody></table>"
|
||||
blocks = [
|
||||
{
|
||||
"heading": "H",
|
||||
"level": 1,
|
||||
"parent_headings": [],
|
||||
"content": table_marker("t1"),
|
||||
}
|
||||
]
|
||||
ir = _normalize(blocks, {"md_tables": {"t1": {"kind": "html", "html": html}}})
|
||||
(table,) = ir.blocks[0].tables
|
||||
assert table.rows is None
|
||||
assert table.html and table.html.startswith("<table>")
|
||||
assert isinstance(table.table_header, str) and "<thead>" in table.table_header
|
||||
assert table.body_override and "<thead>" in table.body_override
|
||||
assert (table.num_rows, table.num_cols) == (2, 2)
|
||||
|
||||
|
||||
def test_equation_is_block_level():
|
||||
blocks = [
|
||||
{
|
||||
"heading": "H",
|
||||
"level": 1,
|
||||
"parent_headings": [],
|
||||
"content": equation_marker("e1"),
|
||||
}
|
||||
]
|
||||
ir = _normalize(blocks, {"md_equations": {"e1": "E = mc^2"}})
|
||||
(eq,) = ir.blocks[0].equations
|
||||
assert eq.is_block is True
|
||||
assert eq.latex == "E = mc^2"
|
||||
assert "{{EQ:" in ir.blocks[0].content_template
|
||||
|
||||
|
||||
def test_local_drawing_dedups_assetspec_across_occurrences():
|
||||
content = f"{drawing_marker('d1')} {drawing_marker('d2')}"
|
||||
blocks = [{"heading": "H", "level": 1, "parent_headings": [], "content": content}]
|
||||
meta = {
|
||||
"md_drawings": {
|
||||
"d1": {"kind": "local", "asset_ref": "sha:1", "fmt": "png", "src": "a.png"},
|
||||
"d2": {"kind": "local", "asset_ref": "sha:1", "fmt": "png", "src": "a.png"},
|
||||
},
|
||||
"md_assets": {"sha:1": {"suggested_name": "a.png", "data": b"BYTES"}},
|
||||
}
|
||||
ir = _normalize(blocks, meta)
|
||||
assert len(ir.blocks[0].drawings) == 2
|
||||
assert all(d.asset_ref == "sha:1" for d in ir.blocks[0].drawings)
|
||||
# One shared on-disk asset, carried as bytes for the writer to materialize.
|
||||
assert len(ir.assets) == 1
|
||||
assert ir.assets[0].ref == "sha:1"
|
||||
assert ir.assets[0].source == b"BYTES"
|
||||
|
||||
|
||||
def test_external_drawing_uses_path_override_and_no_asset():
|
||||
blocks = [
|
||||
{
|
||||
"heading": "H",
|
||||
"level": 1,
|
||||
"parent_headings": [],
|
||||
"content": drawing_marker("d1"),
|
||||
}
|
||||
]
|
||||
meta = {
|
||||
"md_drawings": {
|
||||
"d1": {
|
||||
"kind": "external",
|
||||
"url": "http://x/y.png",
|
||||
"fmt": "png",
|
||||
"src": "http://x/y.png",
|
||||
}
|
||||
}
|
||||
}
|
||||
ir = _normalize(blocks, meta)
|
||||
(drawing,) = ir.blocks[0].drawings
|
||||
assert drawing.path_override == "http://x/y.png"
|
||||
assert drawing.asset_ref == ""
|
||||
assert ir.assets == []
|
||||
|
||||
|
||||
def test_doc_title_from_first_h1_else_filename():
|
||||
ir = _normalize(
|
||||
[{"heading": "First", "level": 1, "parent_headings": [], "content": "# First"}],
|
||||
{},
|
||||
)
|
||||
assert ir.doc_title == "First"
|
||||
ir2 = _normalize(
|
||||
[{"heading": "Sub", "level": 2, "parent_headings": [], "content": "## Sub"}],
|
||||
{},
|
||||
document_name="myfile.md",
|
||||
)
|
||||
assert ir2.doc_title == "myfile"
|
||||
|
||||
|
||||
def test_positions_use_heading_anchor():
|
||||
ir = _normalize(
|
||||
[{"heading": "Sec", "level": 2, "parent_headings": [], "content": "## Sec"}],
|
||||
{},
|
||||
)
|
||||
(pos,) = ir.blocks[0].positions
|
||||
assert pos.type == "heading"
|
||||
assert pos.anchor == "Sec"
|
||||
|
||||
|
||||
def test_document_format_from_suffix():
|
||||
ir = _normalize(
|
||||
[{"heading": "X", "level": 1, "parent_headings": [], "content": "# X"}],
|
||||
{},
|
||||
document_name="a.textpack",
|
||||
)
|
||||
assert ir.document_format == "textpack"
|
||||
@@ -0,0 +1,446 @@
|
||||
"""Tests for ``NativeMarkdownParser`` extract/resolver behaviour.
|
||||
|
||||
Covers base64 decoding, ``.textpack`` extraction + file-reference resolution
|
||||
with a traversal guard, zip-bomb rejection, and the SSRF-guarded remote image
|
||||
download (monkeypatched ``urllib``), including the env-controlled failure
|
||||
policy.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import io
|
||||
import zipfile
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from lightrag.parser.markdown import parser as md_parser
|
||||
from lightrag.parser.markdown.parser import (
|
||||
NativeMarkdownParser,
|
||||
_host_is_public,
|
||||
_image_bytes_and_ext,
|
||||
_image_ext_from_magic,
|
||||
_looks_like_svg,
|
||||
_pin_socket,
|
||||
)
|
||||
|
||||
# A 1x1 transparent PNG.
|
||||
_PNG_BYTES = (
|
||||
b"\x89PNG\r\n\x1a\n\x00\x00\x00\rIHDR\x00\x00\x00\x01\x00\x00\x00\x01\x08\x06"
|
||||
b"\x00\x00\x00\x1f\x15\xc4\x89\x00\x00\x00\nIDATx\x9cc\x00\x01\x00\x00\x05"
|
||||
b"\x00\x01\r\n-\xb4\x00\x00\x00\x00IEND\xaeB`\x82"
|
||||
)
|
||||
_PNG_B64 = "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mP8z8BQDwAEhQGAhKmMIQAAAABJRU5ErkJggg=="
|
||||
|
||||
|
||||
def _make_parser() -> NativeMarkdownParser:
|
||||
return NativeMarkdownParser()
|
||||
|
||||
|
||||
_SVG_BYTES = (
|
||||
b'<?xml version="1.0"?>\n'
|
||||
b'<svg xmlns="http://www.w3.org/2000/svg" width="4" height="4">'
|
||||
b'<rect width="4" height="4" fill="red"/></svg>'
|
||||
)
|
||||
|
||||
|
||||
def test_magic_detection():
|
||||
assert _image_ext_from_magic(_PNG_BYTES) == "png"
|
||||
assert _image_ext_from_magic(b"\xff\xd8\xff\xe0junk") == "jpg"
|
||||
assert _image_ext_from_magic(b"not an image") is None
|
||||
# SVG is text, not a raster magic — recognised only by the SVG sniffer.
|
||||
assert _image_ext_from_magic(_SVG_BYTES) is None
|
||||
|
||||
|
||||
def test_looks_like_svg():
|
||||
assert _looks_like_svg(_SVG_BYTES) is True
|
||||
assert _looks_like_svg(b"\xef\xbb\xbf \n<svg></svg>") is True # BOM + space
|
||||
assert _looks_like_svg(_PNG_BYTES) is False
|
||||
assert _looks_like_svg(b"<html><body>no svg</body></html>") is False
|
||||
|
||||
|
||||
def test_svg_rasterized_to_png_via_coerce():
|
||||
coerced = _image_bytes_and_ext(
|
||||
_SVG_BYTES, max_bytes=25 * 1024 * 1024, max_svg_pixels=16_000_000
|
||||
)
|
||||
assert coerced is not None
|
||||
data, ext = coerced
|
||||
assert ext == "png"
|
||||
assert data.startswith(b"\x89PNG")
|
||||
|
||||
|
||||
def test_svg_coerce_skips_when_rasterization_fails(monkeypatch):
|
||||
# cairosvg unavailable / render failure -> coerce returns None (caller skips).
|
||||
monkeypatch.setattr(md_parser, "_rasterize_svg", lambda raw, **kw: None)
|
||||
assert (
|
||||
_image_bytes_and_ext(_SVG_BYTES, max_bytes=25 * 1024 * 1024, max_svg_pixels=1)
|
||||
is None
|
||||
)
|
||||
|
||||
|
||||
def test_svg_coerce_skips_when_rasterized_png_too_large(monkeypatch):
|
||||
monkeypatch.setattr(md_parser, "_rasterize_svg", lambda raw, **kw: _PNG_BYTES)
|
||||
assert (
|
||||
_image_bytes_and_ext(_SVG_BYTES, max_bytes=4, max_svg_pixels=16_000_000) is None
|
||||
)
|
||||
|
||||
|
||||
def test_svg_dimensions_parsed_from_width_height_and_viewbox():
|
||||
assert md_parser._svg_pixel_dimensions(_SVG_BYTES) == (4, 4)
|
||||
# Unit conversion: 1in = 96px, 2in = 192px.
|
||||
svg_in = b'<svg xmlns="http://www.w3.org/2000/svg" width="1in" height="2in"/>'
|
||||
assert md_parser._svg_pixel_dimensions(svg_in) == (96, 192)
|
||||
# width/height missing or relative (%) -> fall back to viewBox w/h.
|
||||
svg_vb = (
|
||||
b'<svg xmlns="http://www.w3.org/2000/svg" width="100%" viewBox="0 0 30 20"/>'
|
||||
)
|
||||
assert md_parser._svg_pixel_dimensions(svg_vb) == (30, 20)
|
||||
# Neither resolvable -> None (skipped by the rasterizer).
|
||||
svg_none = b'<svg xmlns="http://www.w3.org/2000/svg"/>'
|
||||
assert md_parser._svg_pixel_dimensions(svg_none) is None
|
||||
|
||||
|
||||
def test_svg_oversized_canvas_rejected_before_render(monkeypatch):
|
||||
# A tiny SVG declaring a huge canvas is rejected on the pre-render pixel
|
||||
# budget — cairosvg.svg2png must never be reached.
|
||||
import cairosvg
|
||||
|
||||
def _boom(*a, **k): # pragma: no cover - must not be called
|
||||
raise AssertionError("cairosvg.svg2png should not run for oversized canvas")
|
||||
|
||||
monkeypatch.setattr(cairosvg, "svg2png", _boom)
|
||||
big = b'<svg xmlns="http://www.w3.org/2000/svg" width="100000" height="100000"/>'
|
||||
assert md_parser._rasterize_svg(big, max_pixels=16_000_000) is None
|
||||
|
||||
|
||||
def test_base64_svg_decoded_and_rasterized():
|
||||
import base64 as _b64
|
||||
|
||||
p = _make_parser()
|
||||
b64 = _b64.b64encode(_SVG_BYTES).decode()
|
||||
md = f"# H\n\n\n"
|
||||
_, warnings, meta = p._extract_text(md, bundle_root=None)
|
||||
(asset,) = meta["md_assets"].values()
|
||||
assert asset["fmt"] == "png"
|
||||
assert asset["data"].startswith(b"\x89PNG")
|
||||
(drawing,) = meta["md_drawings"].values()
|
||||
assert drawing["kind"] == "local"
|
||||
assert not warnings
|
||||
|
||||
|
||||
def test_textpack_svg_file_rasterized_to_png(tmp_path: Path):
|
||||
pack = tmp_path / "note.textpack"
|
||||
with zipfile.ZipFile(pack, "w") as zf:
|
||||
zf.writestr("text.markdown", "# N\n\n\n")
|
||||
zf.writestr("assets/logo.svg", _SVG_BYTES)
|
||||
p = _make_parser()
|
||||
_, warnings, meta = p.extract(
|
||||
pack, parsed_dir=tmp_path, asset_dir=tmp_path, base_name="note"
|
||||
)
|
||||
(asset,) = meta["md_assets"].values()
|
||||
assert asset["fmt"] == "png"
|
||||
assert asset["data"].startswith(b"\x89PNG")
|
||||
# The on-disk name takes the rasterized extension.
|
||||
assert asset["suggested_name"] == "logo.png"
|
||||
assert not warnings
|
||||
|
||||
|
||||
def test_host_is_public_rejects_internal(monkeypatch):
|
||||
monkeypatch.delenv("NATIVE_MD_IMAGE_ALLOWED_NON_PUBLIC_CIDRS", raising=False)
|
||||
assert _host_is_public("8.8.8.8") is True
|
||||
assert _host_is_public("127.0.0.1") is False
|
||||
assert _host_is_public("10.0.0.1") is False
|
||||
assert _host_is_public("169.254.0.1") is False
|
||||
assert _host_is_public("::1") is False
|
||||
# CGNAT 100.64.0.0/10 is not private/reserved in Python's flags but is not
|
||||
# globally routable — default-deny via is_global must still reject it.
|
||||
assert _host_is_public("100.64.0.1") is False
|
||||
# TEST-NET (192.0.2.0/24) is likewise non-global.
|
||||
assert _host_is_public("192.0.2.1") is False
|
||||
|
||||
|
||||
def test_allowlist_permits_configured_non_public(monkeypatch):
|
||||
monkeypatch.setenv("NATIVE_MD_IMAGE_ALLOWED_NON_PUBLIC_CIDRS", "100.64.0.0/10")
|
||||
assert _host_is_public("100.64.0.1") is True
|
||||
# An address outside the allowlist is still rejected.
|
||||
assert _host_is_public("10.0.0.1") is False
|
||||
|
||||
|
||||
def test_pin_socket_refuses_internal_host(monkeypatch):
|
||||
# The connection pins to the same resolution that authorised the host, so
|
||||
# an internal target cannot be reached even if it would resolve.
|
||||
monkeypatch.delenv("NATIVE_MD_IMAGE_ALLOWED_NON_PUBLIC_CIDRS", raising=False)
|
||||
with pytest.raises(OSError, match="non-public host"):
|
||||
_pin_socket("127.0.0.1", 80, None, None)
|
||||
|
||||
|
||||
def test_redirect_to_non_public_host_blocked(monkeypatch):
|
||||
import email.message
|
||||
|
||||
monkeypatch.delenv("NATIVE_MD_IMAGE_ALLOWED_NON_PUBLIC_CIDRS", raising=False)
|
||||
handler = md_parser._GuardedRedirectHandler()
|
||||
req = md_parser.urllib.request.Request("http://example.com/a")
|
||||
with pytest.raises(md_parser.urllib.error.HTTPError):
|
||||
handler.redirect_request(
|
||||
req, None, 302, "Found", email.message.Message(), "http://10.0.0.1/evil"
|
||||
)
|
||||
|
||||
|
||||
def test_guarded_https_handler_open_does_not_raise_attribute_error(monkeypatch):
|
||||
# Regression: ``https_open`` must forward only the kwargs the stdlib handler
|
||||
# actually has on this Python version. On 3.12 ``HTTPSHandler`` has no
|
||||
# ``_check_hostname`` attribute, so a hard reference raised AttributeError
|
||||
# and every real download silently fell back to an external link.
|
||||
handler = md_parser._GuardedHTTPSHandler()
|
||||
captured: dict = {}
|
||||
|
||||
def _fake_do_open(conn_cls, req, **kwargs):
|
||||
captured["conn_cls"] = conn_cls
|
||||
captured["kwargs"] = kwargs
|
||||
return "ok"
|
||||
|
||||
monkeypatch.setattr(handler, "do_open", _fake_do_open)
|
||||
req = md_parser.urllib.request.Request("https://example.com/x.png")
|
||||
assert handler.https_open(req) == "ok"
|
||||
assert captured["conn_cls"] is md_parser._GuardedHTTPSConnection
|
||||
assert "context" in captured["kwargs"]
|
||||
|
||||
|
||||
def test_build_guarded_opener_constructs_with_guarded_handlers():
|
||||
opener = md_parser._build_guarded_opener()
|
||||
handler_types = {type(h) for h in opener.handlers}
|
||||
assert md_parser._GuardedHTTPSHandler in handler_types
|
||||
assert md_parser._GuardedHTTPHandler in handler_types
|
||||
|
||||
|
||||
def test_base64_image_decoded_to_asset():
|
||||
p = _make_parser()
|
||||
md = f"# H\n\n\n"
|
||||
blocks, warnings, meta = p._extract_text(md, bundle_root=None)
|
||||
assert len(meta["md_assets"]) == 1
|
||||
(asset,) = meta["md_assets"].values()
|
||||
assert asset["data"].startswith(b"\x89PNG")
|
||||
assert asset["fmt"] == "png"
|
||||
(drawing,) = meta["md_drawings"].values()
|
||||
assert drawing["kind"] == "local"
|
||||
assert not warnings
|
||||
|
||||
|
||||
def test_base64_non_image_bytes_rejected():
|
||||
# ``data:image/png;base64,QUJD`` decodes to b"ABC" — declared PNG but not a
|
||||
# real image. Magic bytes are authoritative, so it must be skipped.
|
||||
p = _make_parser()
|
||||
md = "# H\n\n\n"
|
||||
_, warnings, meta = p._extract_text(md, bundle_root=None)
|
||||
assert meta["md_drawings"] == {}
|
||||
assert meta["md_assets"] == {}
|
||||
assert warnings.get("images_skipped") == 1
|
||||
|
||||
|
||||
def test_standalone_md_relative_image_is_skipped():
|
||||
p = _make_parser()
|
||||
blocks, warnings, meta = p._extract_text(
|
||||
"# H\n\n\n", bundle_root=None
|
||||
)
|
||||
assert meta["md_drawings"] == {}
|
||||
assert warnings.get("images_skipped") == 1
|
||||
|
||||
|
||||
def _write_textpack(path: Path, *, text: str, assets: dict[str, bytes]) -> None:
|
||||
with zipfile.ZipFile(path, "w") as zf:
|
||||
zf.writestr("text.markdown", text)
|
||||
for name, data in assets.items():
|
||||
zf.writestr(f"assets/{name}", data)
|
||||
|
||||
|
||||
def test_textpack_file_reference_resolved(tmp_path: Path):
|
||||
pack = tmp_path / "note.textpack"
|
||||
_write_textpack(
|
||||
pack,
|
||||
text="# Note\n\n\n",
|
||||
assets={"pic.png": _PNG_BYTES},
|
||||
)
|
||||
p = _make_parser()
|
||||
blocks, warnings, meta = p.extract(
|
||||
pack, parsed_dir=tmp_path, asset_dir=tmp_path, base_name="note"
|
||||
)
|
||||
assert len(meta["md_assets"]) == 1
|
||||
(asset,) = meta["md_assets"].values()
|
||||
assert asset["data"] == _PNG_BYTES
|
||||
assert asset["suggested_name"] == "pic.png"
|
||||
assert not warnings
|
||||
|
||||
|
||||
def test_textpack_image_outside_assets_dir_is_resolved(tmp_path: Path):
|
||||
# Compatibility: a reference may point anywhere under the bundle root, not
|
||||
# only ``assets/`` — here ``media/pic.png`` and a root-level ``cover.png``.
|
||||
pack = tmp_path / "note.textpack"
|
||||
with zipfile.ZipFile(pack, "w") as zf:
|
||||
zf.writestr("text.markdown", "# N\n\n\n\n\n")
|
||||
zf.writestr("media/pic.png", _PNG_BYTES)
|
||||
zf.writestr("cover.png", _PNG_BYTES)
|
||||
p = _make_parser()
|
||||
_, warnings, meta = p.extract(
|
||||
pack, parsed_dir=tmp_path, asset_dir=tmp_path, base_name="note"
|
||||
)
|
||||
assert len(meta["md_drawings"]) == 2
|
||||
assert all(d["kind"] == "local" for d in meta["md_drawings"].values())
|
||||
# Same bytes → deduped to a single asset.
|
||||
assert len(meta["md_assets"]) == 1
|
||||
assert not warnings
|
||||
|
||||
|
||||
def test_textpack_non_image_file_reference_is_skipped(tmp_path: Path):
|
||||
pack = tmp_path / "note.textpack"
|
||||
with zipfile.ZipFile(pack, "w") as zf:
|
||||
zf.writestr("text.markdown", "# N\n\n\n")
|
||||
zf.writestr("assets/fake.png", b"this is not an image")
|
||||
p = _make_parser()
|
||||
_, warnings, meta = p.extract(
|
||||
pack, parsed_dir=tmp_path, asset_dir=tmp_path, base_name="note"
|
||||
)
|
||||
assert meta["md_drawings"] == {}
|
||||
assert warnings.get("images_skipped") == 1
|
||||
|
||||
|
||||
def test_textpack_oversized_image_is_skipped(tmp_path: Path, monkeypatch):
|
||||
# A bundled asset larger than NATIVE_MD_IMAGE_MAX_BYTES is skipped before it
|
||||
# is read into memory (the per-file cap, separate from the zip-bomb total).
|
||||
monkeypatch.setenv("NATIVE_MD_IMAGE_MAX_BYTES", "16")
|
||||
pack = tmp_path / "big.textpack"
|
||||
with zipfile.ZipFile(pack, "w") as zf:
|
||||
zf.writestr("text.markdown", "# N\n\n\n")
|
||||
zf.writestr("assets/big.png", _PNG_BYTES) # well over 16 bytes
|
||||
p = _make_parser()
|
||||
_, warnings, meta = p.extract(
|
||||
pack, parsed_dir=tmp_path, asset_dir=tmp_path, base_name="big"
|
||||
)
|
||||
assert meta["md_drawings"] == {}
|
||||
assert meta["md_assets"] == {}
|
||||
assert warnings.get("images_skipped") == 1
|
||||
|
||||
|
||||
def test_textpack_traversal_reference_is_skipped(tmp_path: Path):
|
||||
pack = tmp_path / "evil.textpack"
|
||||
_write_textpack(
|
||||
pack,
|
||||
text="# Evil\n\n\n",
|
||||
assets={"pic.png": _PNG_BYTES},
|
||||
)
|
||||
p = _make_parser()
|
||||
_, warnings, meta = p.extract(
|
||||
pack, parsed_dir=tmp_path, asset_dir=tmp_path, base_name="evil"
|
||||
)
|
||||
assert meta["md_drawings"] == {}
|
||||
assert warnings.get("images_skipped") == 1
|
||||
|
||||
|
||||
def test_textpack_zip_bomb_entry_count_rejected(tmp_path: Path, monkeypatch):
|
||||
monkeypatch.setattr(md_parser, "_TEXTPACK_MAX_ENTRIES", 3)
|
||||
pack = tmp_path / "bomb.textpack"
|
||||
with zipfile.ZipFile(pack, "w") as zf:
|
||||
zf.writestr("text.markdown", "# H\n")
|
||||
for i in range(10):
|
||||
zf.writestr(f"assets/f{i}.bin", b"x")
|
||||
p = _make_parser()
|
||||
with pytest.raises(RuntimeError, match="entries"):
|
||||
p.extract(pack, parsed_dir=tmp_path, asset_dir=tmp_path, base_name="bomb")
|
||||
|
||||
|
||||
# --- remote download (monkeypatched urllib) --------------------------------
|
||||
|
||||
|
||||
class _FakeHeaders:
|
||||
def __init__(self, content_type: str) -> None:
|
||||
self._ct = content_type
|
||||
|
||||
def get_content_type(self) -> str:
|
||||
return self._ct
|
||||
|
||||
|
||||
class _FakeResponse(io.BytesIO):
|
||||
def __init__(self, data: bytes, content_type: str) -> None:
|
||||
super().__init__(data)
|
||||
self.headers = _FakeHeaders(content_type)
|
||||
|
||||
def __enter__(self):
|
||||
return self
|
||||
|
||||
def __exit__(self, *exc):
|
||||
self.close()
|
||||
|
||||
|
||||
def _patch_download(
|
||||
monkeypatch, *, data=_PNG_BYTES, content_type="image/png", host_public=True
|
||||
):
|
||||
monkeypatch.setattr(md_parser, "_host_is_public", lambda host: host_public)
|
||||
|
||||
class _Opener:
|
||||
def open(self, req, timeout=None):
|
||||
return _FakeResponse(data, content_type)
|
||||
|
||||
monkeypatch.setattr(
|
||||
md_parser.urllib.request, "build_opener", lambda *a, **k: _Opener()
|
||||
)
|
||||
|
||||
|
||||
def test_remote_image_dropped_when_download_disabled(monkeypatch):
|
||||
# With downloading explicitly disabled an external image is dropped entirely
|
||||
# — no drawing, no asset — so a doc whose only image is an external link
|
||||
# produces no drawings.json. (Downloading is ON by default.)
|
||||
monkeypatch.setenv("NATIVE_MD_IMAGE_DOWNLOAD_ENABLED", "false")
|
||||
p = _make_parser()
|
||||
_, warnings, meta = p._extract_text(
|
||||
"# H\n\n\n", bundle_root=None
|
||||
)
|
||||
assert meta["md_drawings"] == {}
|
||||
assert meta["md_assets"] == {}
|
||||
assert warnings.get("images_external_dropped") == 1
|
||||
|
||||
|
||||
def test_remote_image_downloaded_when_enabled(monkeypatch):
|
||||
monkeypatch.setenv("NATIVE_MD_IMAGE_DOWNLOAD_ENABLED", "true")
|
||||
_patch_download(monkeypatch)
|
||||
p = _make_parser()
|
||||
_, warnings, meta = p._extract_text(
|
||||
"# H\n\n\n", bundle_root=None
|
||||
)
|
||||
(drawing,) = meta["md_drawings"].values()
|
||||
assert drawing["kind"] == "local"
|
||||
(asset,) = meta["md_assets"].values()
|
||||
assert asset["data"] == _PNG_BYTES
|
||||
|
||||
|
||||
def test_remote_image_failure_kept_external_by_default(monkeypatch):
|
||||
monkeypatch.setenv("NATIVE_MD_IMAGE_DOWNLOAD_ENABLED", "true")
|
||||
monkeypatch.delenv("NATIVE_MD_IMAGE_DOWNLOAD_REQUIRED", raising=False)
|
||||
_patch_download(monkeypatch, content_type="text/html") # explicit non-image
|
||||
p = _make_parser()
|
||||
_, warnings, meta = p._extract_text(
|
||||
"# H\n\n\n", bundle_root=None
|
||||
)
|
||||
(drawing,) = meta["md_drawings"].values()
|
||||
assert drawing["kind"] == "external"
|
||||
assert warnings.get("images_download_failed") == 1
|
||||
|
||||
|
||||
def test_remote_image_failure_raises_when_required(monkeypatch):
|
||||
monkeypatch.setenv("NATIVE_MD_IMAGE_DOWNLOAD_ENABLED", "true")
|
||||
monkeypatch.setenv("NATIVE_MD_IMAGE_DOWNLOAD_REQUIRED", "true")
|
||||
_patch_download(monkeypatch, content_type="text/html")
|
||||
p = _make_parser()
|
||||
with pytest.raises(Exception):
|
||||
p._extract_text("# H\n\n\n", bundle_root=None)
|
||||
|
||||
|
||||
def test_remote_image_private_host_blocked(monkeypatch):
|
||||
monkeypatch.setenv("NATIVE_MD_IMAGE_DOWNLOAD_ENABLED", "true")
|
||||
monkeypatch.delenv("NATIVE_MD_IMAGE_DOWNLOAD_REQUIRED", raising=False)
|
||||
_patch_download(monkeypatch, host_public=False)
|
||||
p = _make_parser()
|
||||
_, warnings, meta = p._extract_text(
|
||||
"# H\n\n\n", bundle_root=None
|
||||
)
|
||||
(drawing,) = meta["md_drawings"].values()
|
||||
assert drawing["kind"] == "external"
|
||||
assert warnings.get("images_download_failed") == 1
|
||||
@@ -0,0 +1,214 @@
|
||||
"""Tests for the native-markdown downloaded-image cache (`.native_raw/`).
|
||||
|
||||
The cache lets a re-parse of an unchanged file reuse already-downloaded images
|
||||
instead of hitting the network. ``_download`` is monkeypatched so a cache hit is
|
||||
proven by it NOT being called (it raises) while assets still materialize.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import shutil
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from lightrag.parser.markdown import parser as md_parser
|
||||
from lightrag.parser.markdown.parser import NativeMarkdownParser
|
||||
|
||||
# A 1x1 transparent PNG.
|
||||
_PNG_BYTES = (
|
||||
b"\x89PNG\r\n\x1a\n\x00\x00\x00\rIHDR\x00\x00\x00\x01\x00\x00\x00\x01\x08\x06"
|
||||
b"\x00\x00\x00\x1f\x15\xc4\x89\x00\x00\x00\nIDATx\x9cc\x00\x01\x00\x00\x05"
|
||||
b"\x00\x01\r\n-\xb4\x00\x00\x00\x00IEND\xaeB`\x82"
|
||||
)
|
||||
_URL = "http://host/y.png"
|
||||
_MD = f"# H\n\n\n"
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _enable_download(monkeypatch):
|
||||
monkeypatch.setenv("NATIVE_MD_IMAGE_DOWNLOAD_ENABLED", "true")
|
||||
monkeypatch.delenv("LIGHTRAG_FORCE_REPARSE_NATIVE", raising=False)
|
||||
monkeypatch.delenv("NATIVE_MD_IMAGE_MAX_BYTES", raising=False)
|
||||
monkeypatch.delenv("NATIVE_MD_IMAGE_MAX_SVG_PIXELS", raising=False)
|
||||
monkeypatch.delenv("NATIVE_MD_IMAGE_ALLOWED_NON_PUBLIC_CIDRS", raising=False)
|
||||
|
||||
|
||||
def _patch_download(monkeypatch, payload=(_PNG_BYTES, "png")):
|
||||
"""Patch ``_download`` to count calls and return fixed bytes."""
|
||||
counter = {"n": 0}
|
||||
|
||||
def _fake(self, src):
|
||||
counter["n"] += 1
|
||||
return payload
|
||||
|
||||
monkeypatch.setattr(md_parser._MarkdownImageResolver, "_download", _fake)
|
||||
return counter
|
||||
|
||||
|
||||
def _forbid_download(monkeypatch):
|
||||
"""Patch ``_download`` to fail the test if the network is touched."""
|
||||
|
||||
def _boom(self, src): # pragma: no cover - must not be called on a cache hit
|
||||
raise AssertionError("network download must not run on a cache hit")
|
||||
|
||||
monkeypatch.setattr(md_parser._MarkdownImageResolver, "_download", _boom)
|
||||
|
||||
|
||||
def _make_doc(tmp_path: Path, text: str = _MD) -> tuple[Path, Path]:
|
||||
src = tmp_path / "doc.md"
|
||||
src.write_text(text)
|
||||
parsed = tmp_path / "__parsed__" / "doc.md.parsed"
|
||||
parsed.mkdir(parents=True)
|
||||
(parsed / "doc.blocks.assets").mkdir()
|
||||
return src, parsed
|
||||
|
||||
|
||||
def _extract(p: NativeMarkdownParser, src: Path, parsed: Path):
|
||||
return p.extract(
|
||||
src, parsed_dir=parsed, asset_dir=parsed / "doc.blocks.assets", base_name="doc"
|
||||
)
|
||||
|
||||
|
||||
def _wipe_parsed(parsed: Path) -> None:
|
||||
"""Mirror NativeParserBase.parse's rmtree(parsed_dir) before a re-extract."""
|
||||
shutil.rmtree(parsed)
|
||||
parsed.mkdir(parents=True)
|
||||
(parsed / "doc.blocks.assets").mkdir()
|
||||
|
||||
|
||||
def _raw_dir(tmp_path: Path) -> Path:
|
||||
return tmp_path / "__parsed__" / "doc.md.native_raw"
|
||||
|
||||
|
||||
def test_first_parse_downloads_and_writes_bundle(tmp_path, monkeypatch):
|
||||
counter = _patch_download(monkeypatch)
|
||||
src, parsed = _make_doc(tmp_path)
|
||||
_, _, meta = _extract(NativeMarkdownParser(), src, parsed)
|
||||
assert counter["n"] == 1
|
||||
assert len(meta["md_assets"]) == 1
|
||||
raw = _raw_dir(tmp_path)
|
||||
assert (raw / "_manifest.json").is_file()
|
||||
# Exactly one cached image file alongside the manifest.
|
||||
files = sorted(c.name for c in raw.iterdir())
|
||||
assert len(files) == 2 and "_manifest.json" in files
|
||||
|
||||
|
||||
def test_reparse_unchanged_source_is_cache_hit(tmp_path, monkeypatch):
|
||||
_patch_download(monkeypatch)
|
||||
p = NativeMarkdownParser()
|
||||
src, parsed = _make_doc(tmp_path)
|
||||
_extract(p, src, parsed) # populate cache
|
||||
_wipe_parsed(parsed) # base parser would rmtree the .parsed dir
|
||||
|
||||
_forbid_download(monkeypatch) # any download now fails the test
|
||||
_, warnings, meta = _extract(p, src, parsed)
|
||||
assert warnings.get("images_cache_hit") == 1
|
||||
assert len(meta["md_assets"]) == 1
|
||||
(asset,) = meta["md_assets"].values()
|
||||
assert asset["data"] == _PNG_BYTES
|
||||
|
||||
|
||||
def test_pure_cache_hit_does_not_touch_bundle(tmp_path, monkeypatch):
|
||||
# A pure hit must write nothing: the manifest + image mtimes stay frozen, so
|
||||
# on-disk timestamps alone reveal whether the cache was hit.
|
||||
_patch_download(monkeypatch)
|
||||
p = NativeMarkdownParser()
|
||||
src, parsed = _make_doc(tmp_path)
|
||||
_extract(p, src, parsed)
|
||||
raw = _raw_dir(tmp_path)
|
||||
before = {c.name: c.stat().st_mtime_ns for c in raw.iterdir()}
|
||||
|
||||
_wipe_parsed(parsed)
|
||||
_forbid_download(monkeypatch) # pure hit: no download allowed
|
||||
_, warnings, _ = _extract(p, src, parsed)
|
||||
assert warnings.get("images_cache_hit") == 1
|
||||
|
||||
after = {c.name: c.stat().st_mtime_ns for c in raw.iterdir()}
|
||||
assert after == before # nothing rewritten
|
||||
|
||||
|
||||
def test_source_change_invalidates_cache(tmp_path, monkeypatch):
|
||||
counter = _patch_download(monkeypatch)
|
||||
p = NativeMarkdownParser()
|
||||
src, parsed = _make_doc(tmp_path)
|
||||
_extract(p, src, parsed)
|
||||
assert counter["n"] == 1
|
||||
src.write_text("# Changed\n\n\n")
|
||||
_wipe_parsed(parsed)
|
||||
_extract(p, src, parsed)
|
||||
assert counter["n"] == 2 # re-downloaded
|
||||
|
||||
|
||||
def test_options_signature_change_invalidates_cache(tmp_path, monkeypatch):
|
||||
counter = _patch_download(monkeypatch)
|
||||
p = NativeMarkdownParser()
|
||||
src, parsed = _make_doc(tmp_path)
|
||||
_extract(p, src, parsed)
|
||||
assert counter["n"] == 1
|
||||
# Changing a byte-affecting knob busts the bundle.
|
||||
monkeypatch.setenv("NATIVE_MD_IMAGE_MAX_BYTES", "12345")
|
||||
_wipe_parsed(parsed)
|
||||
_extract(p, src, parsed)
|
||||
assert counter["n"] == 2
|
||||
|
||||
|
||||
def test_force_reparse_discards_cache(tmp_path, monkeypatch):
|
||||
counter = _patch_download(monkeypatch)
|
||||
p = NativeMarkdownParser()
|
||||
src, parsed = _make_doc(tmp_path)
|
||||
_extract(p, src, parsed)
|
||||
assert counter["n"] == 1
|
||||
monkeypatch.setenv("LIGHTRAG_FORCE_REPARSE_NATIVE", "true")
|
||||
_wipe_parsed(parsed)
|
||||
_extract(p, src, parsed)
|
||||
assert counter["n"] == 2
|
||||
|
||||
|
||||
def test_tampered_cache_file_falls_back_to_download(tmp_path, monkeypatch):
|
||||
counter = _patch_download(monkeypatch)
|
||||
p = NativeMarkdownParser()
|
||||
src, parsed = _make_doc(tmp_path)
|
||||
_extract(p, src, parsed)
|
||||
assert counter["n"] == 1
|
||||
# Corrupt the cached image bytes so the sha256 no longer matches the manifest.
|
||||
raw = _raw_dir(tmp_path)
|
||||
img = next(c for c in raw.iterdir() if c.name != "_manifest.json")
|
||||
img.write_bytes(b"corrupted")
|
||||
_wipe_parsed(parsed)
|
||||
_extract(p, src, parsed)
|
||||
assert counter["n"] == 2 # integrity mismatch -> miss -> re-download
|
||||
|
||||
|
||||
def test_svg_cached_as_png_and_reused_without_rasterizing(tmp_path, monkeypatch):
|
||||
# _download returns the post-rasterization PNG, so a cache hit reuses PNG
|
||||
# bytes and never calls cairosvg.
|
||||
_patch_download(monkeypatch, payload=(_PNG_BYTES, "png"))
|
||||
p = NativeMarkdownParser()
|
||||
src, parsed = _make_doc(tmp_path, text=f"# H\n\n\n")
|
||||
_, _, meta = _extract(p, src, parsed)
|
||||
(asset,) = meta["md_assets"].values()
|
||||
assert asset["fmt"] == "png" and asset["data"].startswith(b"\x89PNG")
|
||||
|
||||
_wipe_parsed(parsed)
|
||||
_forbid_download(monkeypatch)
|
||||
|
||||
import cairosvg
|
||||
|
||||
def _boom(*a, **k): # pragma: no cover - must not run on a cache hit
|
||||
raise AssertionError("cairosvg must not run on a cache hit")
|
||||
|
||||
monkeypatch.setattr(cairosvg, "svg2png", _boom)
|
||||
_, warnings, meta = _extract(p, src, parsed)
|
||||
assert warnings.get("images_cache_hit") == 1
|
||||
|
||||
|
||||
def test_native_raw_is_sibling_and_survives_parsed_rmtree(tmp_path, monkeypatch):
|
||||
_patch_download(monkeypatch)
|
||||
src, parsed = _make_doc(tmp_path)
|
||||
_extract(NativeMarkdownParser(), src, parsed)
|
||||
raw = _raw_dir(tmp_path)
|
||||
assert raw.is_dir()
|
||||
assert raw.parent == parsed.parent # sibling of .parsed, not nested
|
||||
shutil.rmtree(parsed) # base parser wipes parsed_dir
|
||||
assert raw.is_dir() # cache survives
|
||||
@@ -0,0 +1,81 @@
|
||||
"""Routing + registry wiring for the native markdown engine.
|
||||
|
||||
``parser_rules=""`` is passed explicitly so these assertions are independent of
|
||||
any ambient ``LIGHTRAG_PARSER`` in the dev environment.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
|
||||
from lightrag.parser import registry
|
||||
from lightrag.parser.native_dispatch import NativeParser
|
||||
from lightrag.parser.routing import resolve_file_parser_directives
|
||||
|
||||
|
||||
def _engine(path: str, rules: str = "") -> str:
|
||||
return resolve_file_parser_directives(path, parser_rules=rules)[0]
|
||||
|
||||
|
||||
def test_native_declares_markdown_suffixes():
|
||||
assert registry.suffix_capabilities("native") == frozenset(
|
||||
{"docx", "md", "textpack"}
|
||||
)
|
||||
|
||||
|
||||
def test_textpack_and_md_in_upload_allowlist():
|
||||
suffixes = registry.available_engine_suffixes()
|
||||
assert "textpack" in suffixes
|
||||
assert "md" in suffixes
|
||||
|
||||
|
||||
def test_get_parser_native_is_dispatcher():
|
||||
parser = registry.get_parser("native")
|
||||
assert type(parser).__name__ == "NativeParser"
|
||||
|
||||
|
||||
def test_textpack_defaults_to_native_without_hint():
|
||||
assert _engine("note.textpack") == "native"
|
||||
|
||||
|
||||
def test_md_defaults_to_legacy_opt_in_for_native():
|
||||
assert _engine("doc.md") == "legacy"
|
||||
assert _engine("doc.[native].md") == "native"
|
||||
assert _engine("doc.md", rules="md:native") == "native"
|
||||
|
||||
|
||||
def test_docx_default_unchanged():
|
||||
assert _engine("report.docx") == "legacy"
|
||||
|
||||
|
||||
def test_textpack_falls_back_to_native_when_rule_engine_cannot_handle_it():
|
||||
# legacy cannot parse .textpack; the per-suffix default still wins.
|
||||
assert _engine("note.textpack", rules="textpack:legacy") == "native"
|
||||
|
||||
|
||||
class _SentinelParser:
|
||||
def __init__(self, name: str) -> None:
|
||||
self.name = name
|
||||
|
||||
async def parse(self, ctx):
|
||||
return self.name
|
||||
|
||||
|
||||
class _Ctx:
|
||||
def __init__(self, file_path: str) -> None:
|
||||
self.file_path = file_path
|
||||
|
||||
|
||||
def _dispatch(file_path: str) -> str:
|
||||
dispatcher = NativeParser()
|
||||
dispatcher._docx = _SentinelParser("docx")
|
||||
dispatcher._markdown = _SentinelParser("markdown")
|
||||
return asyncio.run(dispatcher.parse(_Ctx(file_path)))
|
||||
|
||||
|
||||
def test_dispatcher_routes_by_suffix():
|
||||
assert _dispatch("a.md") == "markdown"
|
||||
assert _dispatch("a.textpack") == "markdown"
|
||||
assert _dispatch("a.[native-iet].md") == "markdown"
|
||||
assert _dispatch("a.docx") == "docx"
|
||||
assert _dispatch("report.[native].docx") == "docx"
|
||||
@@ -0,0 +1,106 @@
|
||||
"""Cache-correctness proof for per-file engine params (Phase 2).
|
||||
|
||||
A per-file override MUST participate in the raw-bundle cache signature (so an
|
||||
overridden document does not hit a bundle parsed with different params) AND must
|
||||
reach the live request. These tests assert both for MinerU and Docling.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# MinerU
|
||||
# --------------------------------------------------------------------------- #
|
||||
|
||||
|
||||
def test_mineru_signature_changes_with_page_range(monkeypatch):
|
||||
monkeypatch.setenv("MINERU_API_MODE", "official")
|
||||
from lightrag.parser.external.mineru.cache import current_mineru_options_signature
|
||||
|
||||
base = current_mineru_options_signature()
|
||||
ov1 = current_mineru_options_signature({"page_range": "1-3"})
|
||||
ov2 = current_mineru_options_signature({"page_range": "1-3,5"})
|
||||
assert base != ov1 != ov2 and base != ov2
|
||||
# No spurious invalidation: no override == bare env signature.
|
||||
assert current_mineru_options_signature(None) == base
|
||||
|
||||
|
||||
def test_mineru_signature_changes_with_language_and_parse_method(monkeypatch):
|
||||
monkeypatch.setenv("MINERU_API_MODE", "local")
|
||||
from lightrag.parser.external.mineru.cache import current_mineru_options_signature
|
||||
|
||||
base = current_mineru_options_signature()
|
||||
assert current_mineru_options_signature({"language": "en"}) != base
|
||||
assert current_mineru_options_signature({"local_parse_method": "ocr"}) != base
|
||||
|
||||
|
||||
def test_mineru_options_reflect_override(monkeypatch):
|
||||
monkeypatch.setenv("MINERU_API_MODE", "official")
|
||||
from lightrag.parser.external.mineru.cache import MinerUParserOptions
|
||||
|
||||
opts = MinerUParserOptions.from_env(
|
||||
overrides={"language": "en", "page_range": "1-3,5"}
|
||||
)
|
||||
assert opts.language == "en" and opts.page_ranges == "1-3,5"
|
||||
|
||||
|
||||
def test_mineru_local_bounds_from_page_range_override():
|
||||
from lightrag.parser.external.mineru.cache import MinerUParserOptions
|
||||
|
||||
opts = MinerUParserOptions.from_env(
|
||||
api_mode="local", overrides={"page_range": "2-4"}
|
||||
)
|
||||
# local_page_bounds is 0-based.
|
||||
assert opts.local_start_page_id == 1 and opts.local_end_page_id == 3
|
||||
|
||||
|
||||
def test_mineru_request_payload_reflects_override(monkeypatch):
|
||||
monkeypatch.setenv("MINERU_API_MODE", "official")
|
||||
monkeypatch.setenv("MINERU_OFFICIAL_ENDPOINT", "https://mineru.net")
|
||||
monkeypatch.setenv("MINERU_API_TOKEN", "test-token")
|
||||
from lightrag.parser.external.mineru.client import MinerURawClient
|
||||
|
||||
client = MinerURawClient(overrides={"language": "en", "page_range": "1-3,5"})
|
||||
payload = client._official_payload("doc.pdf")
|
||||
assert payload["language"] == "en"
|
||||
assert payload["files"][0]["page_ranges"] == "1-3,5"
|
||||
|
||||
|
||||
def test_mineru_local_form_reflects_parse_method(monkeypatch):
|
||||
monkeypatch.setenv("MINERU_API_MODE", "local")
|
||||
monkeypatch.setenv("MINERU_LOCAL_ENDPOINT", "http://127.0.0.1:8000")
|
||||
from lightrag.parser.external.mineru.client import MinerURawClient
|
||||
|
||||
client = MinerURawClient(overrides={"local_parse_method": "ocr"})
|
||||
assert client._local_form_data()["parse_method"] == "ocr"
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# Docling
|
||||
# --------------------------------------------------------------------------- #
|
||||
|
||||
|
||||
def test_docling_snapshot_and_signature_reflect_force_ocr():
|
||||
from lightrag.parser.external.docling.cache import (
|
||||
compute_options_signature,
|
||||
snapshot_tunable_env,
|
||||
)
|
||||
from lightrag.parser.external.docling.client import FIXED_CONSTANTS
|
||||
|
||||
assert snapshot_tunable_env({"force_ocr": False})["DOCLING_FORCE_OCR"] == "false"
|
||||
base = compute_options_signature(
|
||||
tunable_env=snapshot_tunable_env(), fixed_constants=FIXED_CONSTANTS
|
||||
)
|
||||
ov = compute_options_signature(
|
||||
tunable_env=snapshot_tunable_env({"force_ocr": False}),
|
||||
fixed_constants=FIXED_CONSTANTS,
|
||||
)
|
||||
assert base != ov
|
||||
|
||||
|
||||
def test_docling_client_reflects_force_ocr_override(monkeypatch):
|
||||
monkeypatch.setenv("DOCLING_ENDPOINT", "http://localhost:5001")
|
||||
from lightrag.parser.external.docling.client import DoclingRawClient
|
||||
|
||||
client = DoclingRawClient(overrides={"force_ocr": False})
|
||||
assert client.force_ocr is False
|
||||
@@ -0,0 +1,295 @@
|
||||
"""Tests for per-file ENGINE parameters (Phase 2).
|
||||
|
||||
Engine params attach to the engine token of a hint / LIGHTRAG_PARSER rule
|
||||
(``mineru(page_range=1-3,language=en)`` / ``docling(force_ocr=true)``) and ride
|
||||
the existing ``parse_engine`` field encoded in hint syntax.
|
||||
|
||||
The suite's conftest strips ``MINERU_API_MODE`` (tests default to local, which
|
||||
allows only a single page_range segment), so multi-segment cases explicitly set
|
||||
official mode via monkeypatch. ``parser_rules=""`` keeps assertions independent
|
||||
of any ambient ``LIGHTRAG_PARSER``.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import importlib
|
||||
import sys
|
||||
|
||||
import pytest
|
||||
|
||||
from lightrag.parser.param_schema import (
|
||||
normalize_engine_params,
|
||||
parse_engine_params,
|
||||
render_engine_params,
|
||||
)
|
||||
from lightrag.parser.routing import (
|
||||
FilenameParserHintError,
|
||||
ParserRoutingConfigError,
|
||||
decode_parse_engine,
|
||||
encode_parse_engine,
|
||||
normalize_parser_engine,
|
||||
resolve_file_parser_directives,
|
||||
resolve_parser_directives,
|
||||
validate_parser_routing_config,
|
||||
)
|
||||
|
||||
# Importing document_routes runs argparse over sys.argv at import time;
|
||||
# neutralise pytest's argv during that first import.
|
||||
_original_argv = sys.argv[:]
|
||||
sys.argv = [sys.argv[0]]
|
||||
importlib.import_module("lightrag.api.routers.document_routes")
|
||||
sys.argv = _original_argv
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# parse_engine_params
|
||||
# --------------------------------------------------------------------------- #
|
||||
|
||||
|
||||
def test_parse_engine_params_basic_and_alias():
|
||||
parsed, errors = parse_engine_params(
|
||||
"page_range=1-3,language=en", engine="mineru", label="x"
|
||||
)
|
||||
assert errors == []
|
||||
assert parsed == {"page_range": "1-3", "language": "en"}
|
||||
# alias pr -> page_range
|
||||
parsed, errors = parse_engine_params("pr=2-4", engine="mineru", label="x")
|
||||
assert errors == [] and parsed == {"page_range": "2-4"}
|
||||
# alias local_pm -> local_parse_method
|
||||
parsed, errors = parse_engine_params("local_pm=ocr", engine="mineru", label="x")
|
||||
assert errors == [] and parsed == {"local_parse_method": "ocr"}
|
||||
|
||||
|
||||
def test_parse_engine_params_bool_coercion():
|
||||
parsed, errors = parse_engine_params("force_ocr=false", engine="docling", label="x")
|
||||
assert errors == [] and parsed == {"force_ocr": False}
|
||||
parsed, errors = parse_engine_params("ocr=yes", engine="docling", label="x")
|
||||
assert errors == [] and parsed == {"force_ocr": True}
|
||||
|
||||
|
||||
def test_parse_engine_params_multi_segment_official(monkeypatch):
|
||||
monkeypatch.setenv("MINERU_API_MODE", "official")
|
||||
parsed, errors = parse_engine_params(
|
||||
"page_range=1-3,page_range=5,page_range=7-9", engine="mineru", label="x"
|
||||
)
|
||||
assert errors == [] and parsed == {"page_range": "1-3,5,7-9"}
|
||||
|
||||
|
||||
def test_parse_engine_params_multi_segment_rejected_local(monkeypatch):
|
||||
monkeypatch.delenv("MINERU_API_MODE", raising=False) # default local
|
||||
parsed, errors = parse_engine_params(
|
||||
"page_range=1-3,page_range=5", engine="mineru", label="x"
|
||||
)
|
||||
assert parsed == {} or "page_range" not in parsed
|
||||
assert errors and "local supports only a single page" in errors[0]
|
||||
|
||||
|
||||
def test_parse_engine_params_local_parse_method_accepted_local(monkeypatch):
|
||||
monkeypatch.delenv("MINERU_API_MODE", raising=False) # default local
|
||||
parsed, errors = parse_engine_params(
|
||||
"local_parse_method=ocr", engine="mineru", label="x"
|
||||
)
|
||||
assert errors == [] and parsed == {"local_parse_method": "ocr"}
|
||||
|
||||
|
||||
def test_parse_engine_params_local_parse_method_rejected_official(monkeypatch):
|
||||
# local_parse_method is local-only: official neither sends it nor folds it
|
||||
# into the cache key, so accepting it would persist a silent no-op directive.
|
||||
monkeypatch.setenv("MINERU_API_MODE", "official")
|
||||
parsed, errors = parse_engine_params(
|
||||
"local_parse_method=ocr", engine="mineru", label="x"
|
||||
)
|
||||
assert "local_parse_method" not in parsed
|
||||
assert errors and "MINERU_API_MODE=local" in errors[0]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"block,engine,needle",
|
||||
[
|
||||
("page_range=1", "legacy", "does not accept parameters"),
|
||||
("page_range=1", "native", "does not accept parameters"),
|
||||
("foo=1", "mineru", "unknown parameter 'foo'"),
|
||||
("local_parse_method=bad", "mineru", "must be one of"),
|
||||
("force_ocr=maybe", "docling", "must be a boolean"),
|
||||
("language=", "mineru", "must be non-empty"),
|
||||
("page_range=1-3,5", "mineru", "page lists must repeat the key"),
|
||||
],
|
||||
)
|
||||
def test_parse_engine_params_errors(block, engine, needle):
|
||||
_parsed, errors = parse_engine_params(block, engine=engine, label="x")
|
||||
assert errors and any(needle in e for e in errors)
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# normalize_engine_params (resolved-dict path) + encode/decode round-trip
|
||||
# --------------------------------------------------------------------------- #
|
||||
|
||||
|
||||
def test_normalize_engine_params_coerces_bool_string():
|
||||
# A direct caller may pass force_ocr as the string "false" — must coerce.
|
||||
norm, errors = normalize_engine_params("docling", {"force_ocr": "false"})
|
||||
assert errors == [] and norm == {"force_ocr": False}
|
||||
|
||||
|
||||
def test_normalize_engine_params_accepts_page_range_list_or_string(monkeypatch):
|
||||
monkeypatch.setenv("MINERU_API_MODE", "official")
|
||||
by_list, e1 = normalize_engine_params("mineru", {"page_range": ["1-3", "5"]})
|
||||
by_str, e2 = normalize_engine_params("mineru", {"page_range": "1-3,5"})
|
||||
assert e1 == [] and e2 == [] and by_list == by_str == {"page_range": "1-3,5"}
|
||||
|
||||
|
||||
def test_normalize_engine_params_rejects_unregistered_engine():
|
||||
_norm, errors = normalize_engine_params("legacy", {"page_range": "1"})
|
||||
assert errors and "does not accept parameters" in errors[0]
|
||||
|
||||
|
||||
def test_encode_decode_round_trip(monkeypatch):
|
||||
monkeypatch.setenv("MINERU_API_MODE", "official")
|
||||
enc = encode_parse_engine("mineru", {"page_range": "1-3,5", "language": "en"})
|
||||
# list param emitted as repeated keys; deterministic (sorted)
|
||||
assert enc == "mineru(language=en,page_range=1-3,page_range=5)"
|
||||
engine, params, errors = decode_parse_engine(enc)
|
||||
assert engine == "mineru" and errors == []
|
||||
assert params == {"page_range": "1-3,5", "language": "en"}
|
||||
|
||||
|
||||
def test_encode_bare_when_no_params():
|
||||
assert encode_parse_engine("mineru", {}) == "mineru"
|
||||
assert encode_parse_engine("legacy", None) == "legacy"
|
||||
|
||||
|
||||
def test_render_engine_params_bool_and_list(monkeypatch):
|
||||
monkeypatch.setenv("MINERU_API_MODE", "official")
|
||||
inner, errs = render_engine_params("mineru", {"page_range": "1-3,5"})
|
||||
assert errs == [] and inner == "page_range=1-3,page_range=5"
|
||||
inner, errs = render_engine_params("docling", {"force_ocr": False})
|
||||
assert errs == [] and inner == "force_ocr=false"
|
||||
|
||||
|
||||
def test_normalize_parser_engine_strips_params():
|
||||
assert normalize_parser_engine("mineru(page_range=1-3)") == "mineru"
|
||||
assert normalize_parser_engine("MINERU(page_range=1-3,language=en)") == "mineru"
|
||||
assert normalize_parser_engine("mineru") == "mineru"
|
||||
|
||||
|
||||
def test_decode_malformed_reports_errors():
|
||||
_e, _p, errors = decode_parse_engine("mineru(page_range=")
|
||||
assert errors and "unbalanced" in errors[0]
|
||||
_e, _p, errors = decode_parse_engine("mineru(badkey=1)")
|
||||
assert errors and "unknown parameter" in errors[0]
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# resolve_parser_directives — filename hint + rule + overlay + drop
|
||||
# --------------------------------------------------------------------------- #
|
||||
|
||||
|
||||
def test_resolve_filename_engine_params(monkeypatch):
|
||||
monkeypatch.delenv("MINERU_API_MODE", raising=False) # local single segment
|
||||
d = resolve_parser_directives(
|
||||
"paper.[mineru(page_range=1-3,language=en)].pdf",
|
||||
parser_rules="",
|
||||
require_external_endpoint=False,
|
||||
)
|
||||
assert d.engine == "mineru"
|
||||
assert d.engine_params == {"page_range": "1-3", "language": "en"}
|
||||
# selector stays a pure (here empty) string
|
||||
assert "(" not in d.process_options
|
||||
|
||||
|
||||
def test_resolve_rule_engine_params():
|
||||
d = resolve_parser_directives(
|
||||
"a.pdf",
|
||||
parser_rules="pdf:mineru(language=en)",
|
||||
require_external_endpoint=False,
|
||||
)
|
||||
assert d.engine == "mineru" and d.engine_params == {"language": "en"}
|
||||
|
||||
|
||||
def test_resolve_drop_rule_params_when_hint_engine_wins():
|
||||
# Rule names mineru(language); filename hint names a different usable engine
|
||||
# (docling) which wins -> mineru's engine params are dropped.
|
||||
d = resolve_parser_directives(
|
||||
"paper.[docling(force_ocr=true)].pdf",
|
||||
parser_rules="pdf:mineru(language=en)",
|
||||
require_external_endpoint=False,
|
||||
)
|
||||
assert d.engine == "docling"
|
||||
assert d.engine_params == {"force_ocr": True}
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"name",
|
||||
[
|
||||
"x.[mineru(language=)].pdf", # empty value
|
||||
"x.[mineru(local_parse_method=bad)].pdf", # bad enum
|
||||
"x.[docling(force_ocr=maybe)].pdf", # bad bool
|
||||
"x.[legacy(page_range=1)].txt", # engine declares no params
|
||||
"x.[mineru(page_range=1-3,5)].pdf", # env-style comma list
|
||||
],
|
||||
)
|
||||
def test_upload_validation_raises_on_bad_engine_params(name):
|
||||
with pytest.raises(FilenameParserHintError):
|
||||
resolve_file_parser_directives(
|
||||
name, parser_rules="", require_external_endpoint=False
|
||||
)
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# validate_parser_routing_config (startup)
|
||||
# --------------------------------------------------------------------------- #
|
||||
|
||||
|
||||
def test_validate_rule_engine_params_ok(monkeypatch):
|
||||
# mineru requires its endpoint configured before its rule validates.
|
||||
monkeypatch.setenv("MINERU_API_MODE", "local")
|
||||
monkeypatch.setenv("MINERU_LOCAL_ENDPOINT", "http://127.0.0.1:8000")
|
||||
validate_parser_routing_config("pdf:mineru(language=en);*:legacy-R")
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"rules,needle",
|
||||
[
|
||||
("pdf:legacy(page_range=1)", "does not accept parameters"),
|
||||
("pdf:mineru(foo=1)", "unknown parameter 'foo'"),
|
||||
("pdf:mineru(local_parse_method=bad)", "must be one of"),
|
||||
("pdf:docling(force_ocr=maybe)", "must be a boolean"),
|
||||
],
|
||||
)
|
||||
def test_validate_rule_engine_params_rejected(rules, needle):
|
||||
with pytest.raises(ParserRoutingConfigError, match=needle):
|
||||
validate_parser_routing_config(rules)
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# Direct-API guard via apipeline_enqueue_documents -> _parse_engine_at
|
||||
# --------------------------------------------------------------------------- #
|
||||
|
||||
|
||||
def _decode_via_parse_engine_at(raw: str):
|
||||
"""Exercise the same decode+validate+re-encode that _parse_engine_at runs."""
|
||||
engine, params, errs = decode_parse_engine(raw)
|
||||
if errs:
|
||||
raise ValueError("; ".join(errs))
|
||||
return encode_parse_engine(engine, params) if params else engine
|
||||
|
||||
|
||||
def test_direct_parse_engine_field_canonicalises(monkeypatch):
|
||||
monkeypatch.setenv("MINERU_API_MODE", "official")
|
||||
assert (
|
||||
_decode_via_parse_engine_at("mineru(page_range=1-3,page_range=5)")
|
||||
== "mineru(page_range=1-3,page_range=5)"
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"raw",
|
||||
[
|
||||
"legacy(page_range=1)", # legacy declares no params
|
||||
"mineru(badkey=1)", # unknown
|
||||
"mineru(page_range=", # malformed
|
||||
],
|
||||
)
|
||||
def test_direct_parse_engine_field_rejected(raw):
|
||||
with pytest.raises(ValueError):
|
||||
_decode_via_parse_engine_at(raw)
|
||||
@@ -0,0 +1,408 @@
|
||||
"""Tests for parameterised parser hints (``[-R(chunk_ts=800,chunk_ol=80)]``).
|
||||
|
||||
Phase 1 wires only ``chunk_token_size``/``chunk_ts`` and
|
||||
``chunk_overlap_token_size``/``chunk_ol`` through the existing per-document
|
||||
``chunk_options`` channel. ``parser_rules=""`` is passed explicitly so the
|
||||
assertions are independent of any ambient ``LIGHTRAG_PARSER``.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import importlib
|
||||
import sys
|
||||
|
||||
import pytest
|
||||
|
||||
from lightrag.parser.param_schema import (
|
||||
parse_chunk_params,
|
||||
split_top_level,
|
||||
take_paren_block,
|
||||
)
|
||||
from lightrag.parser.routing import (
|
||||
FilenameParserHintError,
|
||||
ParserRoutingConfigError,
|
||||
canonicalize_parser_hinted_basename,
|
||||
filename_parser_directives,
|
||||
resolve_file_parser_directives,
|
||||
resolve_parser_directives,
|
||||
validate_parser_routing_config,
|
||||
)
|
||||
|
||||
# Importing document_routes runs an argparse over sys.argv at import time;
|
||||
# neutralise pytest's argv during that first import (subsequent imports hit the
|
||||
# module cache and skip argparse). Mirrors tests/api/routes/test_document_routes_chunking.py.
|
||||
_original_argv = sys.argv[:]
|
||||
sys.argv = [sys.argv[0]]
|
||||
importlib.import_module("lightrag.api.routers.document_routes")
|
||||
sys.argv = _original_argv
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# param_schema: scanners
|
||||
# --------------------------------------------------------------------------- #
|
||||
|
||||
|
||||
def test_split_top_level_protects_commas_inside_parens():
|
||||
# A comma inside a parameter block must not split the surrounding rules.
|
||||
assert split_top_level(
|
||||
"pdf:legacy-R(chunk_ts=800,chunk_ol=80);*:legacy-R", ";,"
|
||||
) == ["pdf:legacy-R(chunk_ts=800,chunk_ol=80)", "*:legacy-R"]
|
||||
# Legacy comma rule separator still works at depth 0.
|
||||
assert split_top_level("a:native-F,b:legacy-R", ";,") == [
|
||||
"a:native-F",
|
||||
"b:legacy-R",
|
||||
]
|
||||
|
||||
|
||||
def test_take_paren_block():
|
||||
assert take_paren_block("R(chunk_ts=800)", 1) == ("chunk_ts=800", 15)
|
||||
assert take_paren_block("R", 1) == (None, 1) # no block
|
||||
assert take_paren_block("R(unterminated", 1) == (None, 1) # unbalanced
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# param_schema: parse_chunk_params (alias / type / target / repeat)
|
||||
# --------------------------------------------------------------------------- #
|
||||
|
||||
|
||||
def test_parse_chunk_params_alias_normalised_to_canonical():
|
||||
parsed, errors = parse_chunk_params(
|
||||
"chunk_ts=800,chunk_ol=80", selector="R", label="x"
|
||||
)
|
||||
assert errors == []
|
||||
assert parsed == {"chunk_token_size": 800, "chunk_overlap_token_size": 80}
|
||||
|
||||
|
||||
def test_parse_chunk_params_canonical_names_accepted():
|
||||
parsed, errors = parse_chunk_params(
|
||||
"chunk_token_size=1500", selector="V", label="x"
|
||||
)
|
||||
assert errors == [] and parsed == {"chunk_token_size": 1500}
|
||||
|
||||
|
||||
def test_parse_chunk_params_overlap_rejected_for_vector():
|
||||
parsed, errors = parse_chunk_params("chunk_ol=80", selector="V", label="x")
|
||||
assert parsed == {}
|
||||
assert errors and "not supported for chunk strategy 'V'" in errors[0]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"block,expected",
|
||||
[
|
||||
("drop_rf=true", True),
|
||||
("drop_references=true", True),
|
||||
("drop_rf=false", False),
|
||||
("drop_rf=yes", True),
|
||||
("drop_rf=0", False),
|
||||
],
|
||||
)
|
||||
def test_parse_chunk_params_drop_references_bool(block, expected):
|
||||
parsed, errors = parse_chunk_params(block, selector="P", label="x")
|
||||
assert errors == []
|
||||
assert parsed == {"drop_references": expected}
|
||||
|
||||
|
||||
def test_parse_chunk_params_drop_references_invalid_bool():
|
||||
parsed, errors = parse_chunk_params("drop_rf=maybe", selector="P", label="x")
|
||||
assert parsed == {}
|
||||
assert errors and "must be a boolean" in errors[0]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("block", ["drop_rf", "drop_references"])
|
||||
def test_parse_chunk_params_drop_references_bare_flag_is_true(block):
|
||||
# A bare boolean flag is shorthand for ``=true``.
|
||||
parsed, errors = parse_chunk_params(block, selector="P", label="x")
|
||||
assert errors == []
|
||||
assert parsed == {"drop_references": True}
|
||||
|
||||
|
||||
@pytest.mark.parametrize("selector", ["F", "R", "V"])
|
||||
def test_parse_chunk_params_drop_references_rejected_off_paragraph(selector):
|
||||
parsed, errors = parse_chunk_params("drop_rf=true", selector=selector, label="x")
|
||||
assert parsed == {}
|
||||
assert errors and f"not supported for chunk strategy {selector!r}" in errors[0]
|
||||
|
||||
|
||||
def test_filename_hint_drop_references_extracted():
|
||||
d = resolve_parser_directives("notes.[-P(drop_rf=true)].md", parser_rules="")
|
||||
assert d.process_options == "P" # pure selector, no params
|
||||
assert d.chunk_params == {"P": {"drop_references": True}}
|
||||
|
||||
|
||||
def test_filename_hint_drop_references_bare_flag():
|
||||
# ``P(drop_rf)`` with no value is shorthand for ``drop_rf=true``.
|
||||
d = resolve_parser_directives("notes.[-P(drop_rf)].md", parser_rules="")
|
||||
assert d.process_options == "P"
|
||||
assert d.chunk_params == {"P": {"drop_references": True}}
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"block,needle",
|
||||
[
|
||||
("chunk_ts=abc", "must be an integer"),
|
||||
("chunk_ts=0", "must be >= 1"),
|
||||
("foo=1", "unknown parameter 'foo'"),
|
||||
("chunk_ts=1,chunk_ts=2", "may not be repeated"),
|
||||
("correct_tl", "unknown parameter 'correct_tl'"),
|
||||
# A non-boolean parameter cannot be written bare (only flags can).
|
||||
("chunk_ts", "must be written as 'key=value'"),
|
||||
("", "empty parameter"),
|
||||
# Cross-field invariant: explicit overlap >= size is rejected here, so
|
||||
# both rule and filename-hint validation reject it (not just enqueue).
|
||||
("chunk_ts=50,chunk_ol=100", "must be < chunk_token_size"),
|
||||
("chunk_ts=100,chunk_ol=100", "must be < chunk_token_size"),
|
||||
],
|
||||
)
|
||||
def test_parse_chunk_params_errors(block, needle):
|
||||
_parsed, errors = parse_chunk_params(block, selector="F", label="x")
|
||||
assert errors and any(needle in e for e in errors)
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# routing: filename hint params
|
||||
# --------------------------------------------------------------------------- #
|
||||
|
||||
|
||||
def test_filename_hint_selector_stays_pure_and_params_extracted():
|
||||
d = resolve_parser_directives(
|
||||
"notes.[-R(chunk_ts=800,chunk_ol=80)].md", parser_rules=""
|
||||
)
|
||||
assert d.engine == "legacy"
|
||||
assert d.process_options == "R" # pure selector, no params
|
||||
assert d.chunk_params == {
|
||||
"R": {"chunk_token_size": 800, "chunk_overlap_token_size": 80}
|
||||
}
|
||||
assert d.engine_params == {}
|
||||
|
||||
|
||||
def test_back_compat_two_tuple_and_directives_unaffected_by_params():
|
||||
# The legacy 2-tuple accessor must keep returning (engine, pure-selector).
|
||||
assert resolve_file_parser_directives(
|
||||
"notes.[-R(chunk_ts=800)].md", parser_rules=""
|
||||
) == ("legacy", "R")
|
||||
assert filename_parser_directives("notes.[-R(chunk_ts=800)].md") == (None, "R")
|
||||
# A param-free hint is completely unchanged.
|
||||
assert filename_parser_directives("paper.[native-iet].docx") == ("native", "iet")
|
||||
|
||||
|
||||
def test_canonicalize_strips_parameterised_hint():
|
||||
assert (
|
||||
canonicalize_parser_hinted_basename("notes.[-R(chunk_ts=800,chunk_ol=80)].md")
|
||||
== "notes.md"
|
||||
)
|
||||
|
||||
|
||||
def test_engine_params_rejected_for_engine_that_declares_none():
|
||||
# Chunk params on a chunk char still work...
|
||||
d = resolve_parser_directives("doc.[native-F(chunk_ts=900)].docx", parser_rules="")
|
||||
assert d.chunk_params == {"F": {"chunk_token_size": 900}}
|
||||
# ...but native declares no engine params, so an engine-level block on it is
|
||||
# a hard error on ingestion (Phase 2: only mineru/docling accept params).
|
||||
with pytest.raises(FilenameParserHintError, match="does not accept parameters"):
|
||||
resolve_file_parser_directives(
|
||||
"doc.[native(do_ocr=true)-F].docx", parser_rules=""
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"name",
|
||||
[
|
||||
"x.[-V(chunk_ol=80)].md", # overlap invalid for V
|
||||
"x.[-R(chunk_ts=abc)].md", # non-integer
|
||||
"x.[native-R(foo=1)].md", # unknown parameter
|
||||
"x.[-R(chunk_ts=50,chunk_ol=100)].md", # explicit overlap >= size
|
||||
],
|
||||
)
|
||||
def test_upload_validation_raises_on_bad_filename_params(name):
|
||||
with pytest.raises(FilenameParserHintError):
|
||||
resolve_file_parser_directives(name, parser_rules="")
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# routing: LIGHTRAG_PARSER rule params + overlay merge
|
||||
# --------------------------------------------------------------------------- #
|
||||
|
||||
|
||||
def test_rule_params_with_semicolon_separator_and_comma_in_parens():
|
||||
d = resolve_parser_directives(
|
||||
"a.md", parser_rules="md:legacy-R(chunk_ts=800,chunk_ol=80);*:legacy-R"
|
||||
)
|
||||
assert d.engine == "legacy"
|
||||
assert d.chunk_params == {
|
||||
"R": {"chunk_token_size": 800, "chunk_overlap_token_size": 80}
|
||||
}
|
||||
|
||||
|
||||
def test_rule_params_with_comma_separator_and_comma_in_parens():
|
||||
# Rule splitting is parenthesis-aware: a comma inside a parameter block is
|
||||
# NOT a rule separator, so ',' still separates rules even when a rule carries
|
||||
# parameters (the docs recommend ';' but ',' must keep working).
|
||||
assert split_top_level(
|
||||
"md:legacy-R(chunk_ts=800,chunk_ol=80),*:legacy-R", ";,"
|
||||
) == ["md:legacy-R(chunk_ts=800,chunk_ol=80)", "*:legacy-R"]
|
||||
d = resolve_parser_directives(
|
||||
"a.md", parser_rules="md:legacy-R(chunk_ts=800,chunk_ol=80),*:legacy-R"
|
||||
)
|
||||
assert d.engine == "legacy"
|
||||
assert d.chunk_params == {
|
||||
"R": {"chunk_token_size": 800, "chunk_overlap_token_size": 80}
|
||||
}
|
||||
|
||||
|
||||
def test_overlay_rule_then_filename_hint_wins_per_key():
|
||||
# Design worked example: rule supplies chunk_ol on P, the filename hint
|
||||
# supplies chunk_ts on P; the surviving selector is the filename's "P".
|
||||
d = resolve_parser_directives(
|
||||
"paper.[-P(chunk_ts=3000)].pdf", parser_rules="pdf:legacy-iteP(chunk_ol=100)"
|
||||
)
|
||||
assert d.process_options == "P" # filename options wholesale-override rule
|
||||
assert d.chunk_params["P"] == {
|
||||
"chunk_overlap_token_size": 100, # inherited from the rule
|
||||
"chunk_token_size": 3000, # from the filename hint
|
||||
}
|
||||
|
||||
|
||||
def test_filename_hint_key_overrides_rule_key():
|
||||
d = resolve_parser_directives(
|
||||
"paper.[-R(chunk_ts=500)].md",
|
||||
parser_rules="md:legacy-R(chunk_ts=900,chunk_ol=50)",
|
||||
)
|
||||
assert d.chunk_params["R"] == {
|
||||
"chunk_token_size": 500, # filename wins over the rule's 900
|
||||
"chunk_overlap_token_size": 50, # inherited from the rule
|
||||
}
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# routing: startup validation
|
||||
# --------------------------------------------------------------------------- #
|
||||
|
||||
|
||||
def test_validate_parser_routing_config_accepts_good_params():
|
||||
validate_parser_routing_config("pdf:legacy-R(chunk_ts=800,chunk_ol=80);*:legacy-R")
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"rules,needle",
|
||||
[
|
||||
("pdf:legacy-V(chunk_ol=80)", "not supported for chunk strategy 'V'"),
|
||||
("pdf:legacy-R(chunk_ts=abc)", "must be an integer"),
|
||||
("pdf:legacy-R(foo=1)", "unknown parameter 'foo'"),
|
||||
# Cross-field invariant: explicit overlap >= size is rejected at startup
|
||||
# so a global rule cannot make every matching upload fail at enqueue.
|
||||
("pdf:legacy-R(chunk_ts=50,chunk_ol=100)", "must be < chunk_token_size"),
|
||||
("pdf:legacy-R(chunk_ts=100,chunk_ol=100)", "must be < chunk_token_size"),
|
||||
],
|
||||
)
|
||||
def test_validate_parser_routing_config_rejects_bad_params(rules, needle):
|
||||
with pytest.raises(ParserRoutingConfigError, match=needle):
|
||||
validate_parser_routing_config(rules)
|
||||
|
||||
|
||||
def test_validate_parser_routing_config_allows_one_sided_overlap():
|
||||
# Only overlap is explicit: the effective size depends on env / addon_params
|
||||
# and is unknown at config time, so startup accepts it and defers to the
|
||||
# upload-time effective-overlap check on the resolved snapshot.
|
||||
validate_parser_routing_config("pdf:legacy-R(chunk_ol=100);*:legacy-R")
|
||||
|
||||
|
||||
def test_legacy_comma_rules_without_params_still_valid():
|
||||
# Pure back-compat: legacy comma-separated, param-free rules unchanged.
|
||||
validate_parser_routing_config("docx:native-iteP,pdf:legacy-R,*:legacy-R")
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# upload path: filename-hint params reach the enqueued chunk_options
|
||||
# --------------------------------------------------------------------------- #
|
||||
|
||||
|
||||
def _recording_rag():
|
||||
import asyncio # noqa: F401 - imported for clarity at call sites
|
||||
from types import SimpleNamespace
|
||||
|
||||
captured: dict = {}
|
||||
|
||||
async def fake_enqueue(content, **kwargs):
|
||||
captured.update(kwargs)
|
||||
return "track-xyz" # non-None -> success path
|
||||
|
||||
async def fake_error(files, track_id):
|
||||
captured["error_files"] = files
|
||||
|
||||
rag = SimpleNamespace(
|
||||
addon_params={},
|
||||
apipeline_enqueue_documents=fake_enqueue,
|
||||
apipeline_enqueue_error_documents=fake_error,
|
||||
)
|
||||
return rag, captured
|
||||
|
||||
|
||||
def test_upload_path_applies_hint_chunk_params(tmp_path):
|
||||
import asyncio
|
||||
|
||||
from lightrag.api.routers.document_routes import pipeline_enqueue_file
|
||||
|
||||
rag, captured = _recording_rag()
|
||||
f = tmp_path / "notes.[-R(chunk_ts=800,chunk_ol=80)].md"
|
||||
f.write_text("hello world", encoding="utf-8")
|
||||
|
||||
ok, _track = asyncio.run(pipeline_enqueue_file(rag, f))
|
||||
|
||||
assert ok is True
|
||||
assert "error_files" not in captured
|
||||
assert captured["parse_engine"] == "legacy"
|
||||
assert captured["process_options"] == "R" # pure selector forwarded
|
||||
sub = captured["chunk_options"]["recursive_character"]
|
||||
assert sub["chunk_token_size"] == 800
|
||||
assert sub["chunk_overlap_token_size"] == 80
|
||||
|
||||
|
||||
def test_upload_path_without_params_omits_chunk_options(tmp_path):
|
||||
import asyncio
|
||||
|
||||
from lightrag.api.routers.document_routes import pipeline_enqueue_file
|
||||
|
||||
rag, captured = _recording_rag()
|
||||
f = tmp_path / "plain.[native-R].md"
|
||||
f.write_text("hello world", encoding="utf-8")
|
||||
|
||||
ok, _track = asyncio.run(pipeline_enqueue_file(rag, f))
|
||||
|
||||
assert ok is True
|
||||
# Legacy behaviour: no per-file chunk_options when the hint has no params.
|
||||
assert "chunk_options" not in captured
|
||||
|
||||
|
||||
def test_upload_path_rejects_bad_hint_params(tmp_path):
|
||||
import asyncio
|
||||
|
||||
from lightrag.api.routers.document_routes import pipeline_enqueue_file
|
||||
|
||||
rag, captured = _recording_rag()
|
||||
f = tmp_path / "bad.[-V(chunk_ol=80)].md"
|
||||
f.write_text("hello world", encoding="utf-8")
|
||||
|
||||
ok, _track = asyncio.run(pipeline_enqueue_file(rag, f))
|
||||
|
||||
assert ok is False
|
||||
assert captured["error_files"] # an error document was enqueued
|
||||
assert "chunk_options" not in captured
|
||||
|
||||
|
||||
def test_upload_path_rejects_effective_overlap_violation(tmp_path):
|
||||
import asyncio
|
||||
|
||||
from lightrag.api.routers.document_routes import pipeline_enqueue_file
|
||||
|
||||
rag, captured = _recording_rag()
|
||||
# An explicit overlap (100) >= size (50) pair is rejected up front by
|
||||
# filename-hint validation (uniform with rule validation), so the file
|
||||
# never enqueues and the overlap message is surfaced in the error document.
|
||||
f = tmp_path / "tiny.[-R(chunk_ts=50,chunk_ol=100)].md"
|
||||
f.write_text("hello world", encoding="utf-8")
|
||||
|
||||
ok, _track = asyncio.run(pipeline_enqueue_file(rag, f))
|
||||
|
||||
assert ok is False
|
||||
assert captured["error_files"]
|
||||
assert "must be < chunk_token_size" in captured["error_files"][0]["original_error"]
|
||||
@@ -0,0 +1,121 @@
|
||||
"""Unit tests for the legacy engine adapter (lightrag.parser.legacy.parser).
|
||||
|
||||
Covers the worker-stage extraction contract directly: success path
|
||||
(persist + archive + RAW ParseResult), the unsupported-suffix gate, the
|
||||
whitespace-only extraction guard (scanned-PDF case) and the
|
||||
``PDF_DECRYPT_PASSWORD`` plumbing.
|
||||
"""
|
||||
|
||||
import pytest
|
||||
|
||||
import lightrag.pipeline as _pipeline
|
||||
from lightrag.constants import FULL_DOCS_FORMAT_RAW
|
||||
from lightrag.parser.base import ParseContext
|
||||
from lightrag.parser.legacy.extractors import LegacyExtractionError
|
||||
from lightrag.parser.legacy.parser import LegacyParser
|
||||
|
||||
pytestmark = pytest.mark.offline
|
||||
|
||||
|
||||
class _FakeRag:
|
||||
def __init__(self):
|
||||
self.persisted = []
|
||||
|
||||
async def _persist_parsed_full_docs(self, doc_id, payload):
|
||||
self.persisted.append((doc_id, payload))
|
||||
|
||||
def _resolve_source_file_for_parser(
|
||||
self, file_path, *, source_file=None, parser_engine=None
|
||||
):
|
||||
return file_path
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def archived(monkeypatch):
|
||||
"""Record archive calls instead of moving files into __parsed__."""
|
||||
calls = []
|
||||
|
||||
async def _record(source_path):
|
||||
calls.append(source_path)
|
||||
return source_path
|
||||
|
||||
monkeypatch.setattr(_pipeline, "archive_docx_source_after_full_docs_sync", _record)
|
||||
return calls
|
||||
|
||||
|
||||
def _ctx(rag, source_path):
|
||||
return ParseContext(rag, "doc-legacy", str(source_path), {})
|
||||
|
||||
|
||||
async def test_legacy_parse_txt_persists_and_archives(tmp_path, archived):
|
||||
source = tmp_path / "notes.txt"
|
||||
source.write_text("plain text body", encoding="utf-8")
|
||||
rag = _FakeRag()
|
||||
|
||||
result = await LegacyParser().parse(_ctx(rag, source))
|
||||
|
||||
assert result.parse_format == FULL_DOCS_FORMAT_RAW
|
||||
assert result.content == "plain text body"
|
||||
assert result.parse_engine == "legacy"
|
||||
assert result.blocks_path == ""
|
||||
doc_id, payload = rag.persisted[0]
|
||||
assert doc_id == "doc-legacy"
|
||||
assert payload["content"] == "plain text body"
|
||||
assert payload["parse_format"] == FULL_DOCS_FORMAT_RAW
|
||||
assert archived == [str(source)]
|
||||
# to_dict stays byte-compatible: no spurious skip/warning keys.
|
||||
assert "parse_stage_skipped" not in result.to_dict()
|
||||
|
||||
|
||||
async def test_legacy_parse_unsupported_suffix_raises(tmp_path, archived):
|
||||
source = tmp_path / "image.xyz"
|
||||
source.write_bytes(b"not parseable")
|
||||
rag = _FakeRag()
|
||||
|
||||
with pytest.raises(ValueError, match=r"does not support \.xyz"):
|
||||
await LegacyParser().parse(_ctx(rag, source))
|
||||
|
||||
assert rag.persisted == []
|
||||
assert archived == []
|
||||
|
||||
|
||||
async def test_legacy_parse_whitespace_only_extraction_raises(
|
||||
tmp_path, archived, monkeypatch
|
||||
):
|
||||
# A scanned PDF (no text layer) extracts to pure whitespace; the parser
|
||||
# must fail the doc instead of persisting an empty document.
|
||||
monkeypatch.setattr(
|
||||
"lightrag.parser.legacy.extractors.extract_text",
|
||||
lambda file_bytes, suffix, *, pdf_password=None: "\n \n\t\n",
|
||||
)
|
||||
source = tmp_path / "scanned.pdf"
|
||||
source.write_bytes(b"%PDF-fake")
|
||||
rag = _FakeRag()
|
||||
|
||||
with pytest.raises(LegacyExtractionError, match="no usable text"):
|
||||
await LegacyParser().parse(_ctx(rag, source))
|
||||
|
||||
assert rag.persisted == []
|
||||
assert archived == []
|
||||
|
||||
|
||||
async def test_legacy_parse_passes_pdf_password_from_env(
|
||||
tmp_path, archived, monkeypatch
|
||||
):
|
||||
seen = {}
|
||||
|
||||
def _capture(file_bytes, suffix, *, pdf_password=None):
|
||||
seen["suffix"] = suffix
|
||||
seen["pdf_password"] = pdf_password
|
||||
return "decrypted text"
|
||||
|
||||
monkeypatch.setattr("lightrag.parser.legacy.extractors.extract_text", _capture)
|
||||
monkeypatch.setenv("PDF_DECRYPT_PASSWORD", "s3cret")
|
||||
source = tmp_path / "locked.pdf"
|
||||
source.write_bytes(b"%PDF-fake")
|
||||
rag = _FakeRag()
|
||||
|
||||
result = await LegacyParser().parse(_ctx(rag, source))
|
||||
|
||||
assert seen == {"suffix": "pdf", "pdf_password": "s3cret"}
|
||||
assert result.content == "decrypted text"
|
||||
@@ -0,0 +1,72 @@
|
||||
"""Unit tests for the shared markdown heading renderer."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
|
||||
from lightrag.parser._markdown import (
|
||||
render_heading_line,
|
||||
strip_heading_markdown_prefix,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.offline
|
||||
@pytest.mark.parametrize(
|
||||
("level", "text", "expected"),
|
||||
[
|
||||
(1, "Intro", "# Intro"),
|
||||
(2, "Sub", "## Sub"),
|
||||
(6, "Deep", "###### Deep"),
|
||||
# level >= 7 is clamped to six "#".
|
||||
(7, "Deeper", "###### Deeper"),
|
||||
(99, "Deepest", "###### Deepest"),
|
||||
# level < 1 falls back to a single "#".
|
||||
(0, "Zero", "# Zero"),
|
||||
(-3, "Neg", "# Neg"),
|
||||
],
|
||||
)
|
||||
def test_render_heading_line_prefix_and_cap(level, text, expected) -> None:
|
||||
assert render_heading_line(level, text) == expected
|
||||
|
||||
|
||||
@pytest.mark.offline
|
||||
@pytest.mark.parametrize(
|
||||
"already",
|
||||
["# Foo", "## Bar", "###### Six"],
|
||||
)
|
||||
def test_render_heading_line_keeps_existing_markdown(already) -> None:
|
||||
# Already a markdown heading (1-6 "#" + space) → returned unchanged,
|
||||
# regardless of the requested level (no double-prefixing).
|
||||
assert render_heading_line(3, already) == already
|
||||
|
||||
|
||||
@pytest.mark.offline
|
||||
@pytest.mark.parametrize(
|
||||
("text", "expected"),
|
||||
[
|
||||
# 7 "#" is NOT a valid markdown heading → still gets prefixed.
|
||||
("####### Seven", "# ####### Seven"),
|
||||
# "#" with no following space → not a heading marker → prefixed.
|
||||
("#NoSpace", "# #NoSpace"),
|
||||
],
|
||||
)
|
||||
def test_render_heading_line_non_heading_hash_is_prefixed(text, expected) -> None:
|
||||
assert render_heading_line(1, text) == expected
|
||||
|
||||
|
||||
@pytest.mark.offline
|
||||
@pytest.mark.parametrize(
|
||||
("text", "expected"),
|
||||
[
|
||||
("# Foo", "Foo"),
|
||||
("## Bar", "Bar"),
|
||||
("###### Six", "Six"),
|
||||
# The whole "#" run plus all following spaces are stripped — no
|
||||
# leading space leaks into the cleaned metadata.
|
||||
("# Extra space", "Extra space"),
|
||||
("#NoSpace", "#NoSpace"),
|
||||
("####### Seven", "####### Seven"),
|
||||
],
|
||||
)
|
||||
def test_strip_heading_markdown_prefix(text, expected) -> None:
|
||||
assert strip_heading_markdown_prefix(text) == expected
|
||||
@@ -0,0 +1,430 @@
|
||||
"""End-to-end test: native docx → LightRAG Document → stable cache key.
|
||||
|
||||
The original bug this guards against: ``parse_native`` used to write a
|
||||
runtime-stamped structured parser payload into ``full_docs.content``, so
|
||||
re-parsing the same docx produced different
|
||||
chunk-0 content and therefore different LLM cache keys.
|
||||
|
||||
After the fix, ``parse_native`` writes ``.blocks.jsonl`` + sidecars and
|
||||
``full_docs`` is in LIGHTRAG format. ``_load_lightrag_document_content``
|
||||
skips the ``meta`` line (which contains ``parse_time``) and concatenates
|
||||
only ``"type": "content"`` rows, so re-parsing must yield byte-identical
|
||||
``merged_text`` and stable downstream chunk-0 content.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from lightrag import LightRAG
|
||||
from lightrag.constants import (
|
||||
FULL_DOCS_FORMAT_PENDING_PARSE,
|
||||
PARSED_DIR_NAME,
|
||||
)
|
||||
from lightrag.parser.base import ParseContext
|
||||
from lightrag.parser.registry import get_parser
|
||||
from lightrag.utils import Tokenizer, TokenizerInterface, compute_args_hash
|
||||
|
||||
|
||||
async def _parse_via_registry(rag, engine, doc_id, file_path, content_data):
|
||||
"""Drive a parser the way the pipeline worker does (registry dispatch)."""
|
||||
result = await get_parser(engine).parse(
|
||||
ParseContext(rag, doc_id, file_path, content_data)
|
||||
)
|
||||
return result.to_dict()
|
||||
|
||||
|
||||
def _block(content, *, heading="", level=0, parent=None, uuid="p1"):
|
||||
"""Build a synthetic block dict matching extract_docx_blocks output."""
|
||||
return {
|
||||
"uuid": uuid,
|
||||
"uuid_end": uuid,
|
||||
"heading": heading,
|
||||
"content": content,
|
||||
"type": "text",
|
||||
"parent_headings": list(parent or []),
|
||||
"level": level,
|
||||
"table_chunk_role": "none",
|
||||
}
|
||||
|
||||
|
||||
class _MiniFullDocs:
|
||||
def __init__(self):
|
||||
self.data = {}
|
||||
|
||||
async def upsert(self, payload):
|
||||
self.data.update(payload)
|
||||
|
||||
async def get_by_id(self, doc_id):
|
||||
return self.data.get(doc_id)
|
||||
|
||||
async def index_done_callback(self):
|
||||
return None
|
||||
|
||||
|
||||
class _MiniDocStatus:
|
||||
async def get_by_id(self, doc_id):
|
||||
return None
|
||||
|
||||
async def upsert(self, data):
|
||||
return None
|
||||
|
||||
|
||||
class _CharTokenizer(TokenizerInterface):
|
||||
def encode(self, content: str):
|
||||
return [ord(ch) for ch in content]
|
||||
|
||||
def decode(self, tokens):
|
||||
return "".join(chr(t) for t in tokens)
|
||||
|
||||
|
||||
class _MiniRag:
|
||||
"""Just enough surface for parse_native + parser/docx adapter."""
|
||||
|
||||
_persist_parsed_full_docs = LightRAG._persist_parsed_full_docs
|
||||
|
||||
def __init__(self, working_dir):
|
||||
self.working_dir = str(working_dir)
|
||||
self.full_docs = _MiniFullDocs()
|
||||
self.doc_status = _MiniDocStatus()
|
||||
self.tokenizer = Tokenizer(model_name="char", tokenizer=_CharTokenizer())
|
||||
|
||||
def _resolve_source_file_for_parser(self, file_path):
|
||||
return file_path
|
||||
|
||||
|
||||
@pytest.mark.offline
|
||||
def test_native_lightrag_path_produces_stable_merged_text(tmp_path, monkeypatch):
|
||||
"""Re-parsing the same docx must yield byte-identical merged_text and
|
||||
therefore identical chunk_args_hash on chunk-0."""
|
||||
|
||||
async def _run():
|
||||
input_dir = tmp_path / "input"
|
||||
input_dir.mkdir()
|
||||
monkeypatch.setenv("INPUT_DIR", str(input_dir))
|
||||
|
||||
source_path = input_dir / "stable.docx"
|
||||
source_path.write_bytes(b"fake docx bytes")
|
||||
|
||||
# Stub extract_docx_blocks at the adapter so the upstream DOCX
|
||||
# parser is never invoked. The adapter still does all the
|
||||
# LightRAG-specific writing — that is what we want under test.
|
||||
stable_blocks = [
|
||||
_block(
|
||||
"Title\nFirst paragraph body.\nSecond paragraph body.",
|
||||
heading="Title",
|
||||
level=1,
|
||||
),
|
||||
]
|
||||
|
||||
def _stub_extract(file_path, drawing_context=None, **kwargs):
|
||||
return [dict(b) for b in stable_blocks]
|
||||
|
||||
monkeypatch.setattr(
|
||||
"lightrag.parser.docx.parse_document.extract_docx_blocks",
|
||||
_stub_extract,
|
||||
)
|
||||
|
||||
rag = _MiniRag(tmp_path / "work")
|
||||
|
||||
# ---- First parse ----
|
||||
# parse_native archives the source after writing, so re-create it
|
||||
# before the second parse for a fair comparison.
|
||||
result1 = await _parse_via_registry(
|
||||
rag,
|
||||
"native",
|
||||
"doc-stable",
|
||||
str(source_path),
|
||||
{"parse_format": FULL_DOCS_FORMAT_PENDING_PARSE, "content": ""},
|
||||
)
|
||||
merged1 = result1["content"]
|
||||
assert merged1, "first parse produced empty merged_text"
|
||||
|
||||
# ---- Second parse ----
|
||||
# Restore the source file (archive moved it), reset the in-memory
|
||||
# full_docs row, and remove the parsed_dir so the writer rewrites
|
||||
# both meta (with a fresh parse_time) and content lines.
|
||||
source_path.write_bytes(b"fake docx bytes")
|
||||
rag.full_docs.data.clear()
|
||||
parsed_artifact_dir = input_dir / PARSED_DIR_NAME / f"{source_path.name}.parsed"
|
||||
if parsed_artifact_dir.exists():
|
||||
import shutil
|
||||
|
||||
shutil.rmtree(parsed_artifact_dir)
|
||||
|
||||
result2 = await _parse_via_registry(
|
||||
rag,
|
||||
"native",
|
||||
"doc-stable",
|
||||
str(source_path),
|
||||
{"parse_format": FULL_DOCS_FORMAT_PENDING_PARSE, "content": ""},
|
||||
)
|
||||
merged2 = result2["content"]
|
||||
|
||||
# Core invariant: merged_text byte-identical across runs even
|
||||
# though parse_time in the .blocks.jsonl meta line differs.
|
||||
assert merged1 == merged2
|
||||
|
||||
# And: a hash computed over a chunk-0 derived from merged_text
|
||||
# must also be identical — that is what powers LLM cache hits.
|
||||
prompt_template = "EXTRACT_PROMPT::{text}"
|
||||
chunk0_a = prompt_template.format(text=merged1[:200])
|
||||
chunk0_b = prompt_template.format(text=merged2[:200])
|
||||
assert chunk0_a == chunk0_b
|
||||
assert compute_args_hash(chunk0_a) == compute_args_hash(chunk0_b)
|
||||
|
||||
# And: full_docs.content uses the {{LRdoc}} marker plus a leading
|
||||
# summary derived from merged_text (not the legacy placeholder).
|
||||
record = rag.full_docs.data["doc-stable"]
|
||||
assert record["parse_format"] == "lightrag"
|
||||
assert record["content"].startswith("{{LRdoc}}")
|
||||
assert merged1[:40] in record["content"]
|
||||
|
||||
asyncio.run(_run())
|
||||
|
||||
|
||||
@pytest.mark.offline
|
||||
def test_native_lightrag_path_writes_blocks_jsonl_and_skips_meta_on_load(
|
||||
tmp_path, monkeypatch
|
||||
):
|
||||
"""Sanity check: ``_load_lightrag_document_content`` must skip the
|
||||
meta line (where the runtime ``parse_time`` lives) and only return
|
||||
body content. This is what lets re-parsing produce stable text."""
|
||||
|
||||
async def _run():
|
||||
input_dir = tmp_path / "input"
|
||||
input_dir.mkdir()
|
||||
monkeypatch.setenv("INPUT_DIR", str(input_dir))
|
||||
|
||||
source_path = input_dir / "skipmeta.docx"
|
||||
source_path.write_bytes(b"fake")
|
||||
|
||||
def _stub_extract(file_path, drawing_context=None, **kwargs):
|
||||
return [_block("the body")]
|
||||
|
||||
monkeypatch.setattr(
|
||||
"lightrag.parser.docx.parse_document.extract_docx_blocks",
|
||||
_stub_extract,
|
||||
)
|
||||
|
||||
rag = _MiniRag(tmp_path / "work")
|
||||
result = await _parse_via_registry(
|
||||
rag,
|
||||
"native",
|
||||
"doc-skip",
|
||||
str(source_path),
|
||||
{"parse_format": FULL_DOCS_FORMAT_PENDING_PARSE, "content": ""},
|
||||
)
|
||||
|
||||
# The .blocks.jsonl on disk DOES contain "parse_time" inside the
|
||||
# meta line; the merged_text returned by parse_native MUST NOT.
|
||||
blocks_path = result["blocks_path"]
|
||||
on_disk = open(blocks_path, "r", encoding="utf-8").read()
|
||||
assert "parse_time" in on_disk
|
||||
assert "parse_time" not in result["content"]
|
||||
assert result["content"].strip() == "the body"
|
||||
|
||||
asyncio.run(_run())
|
||||
|
||||
|
||||
@pytest.mark.offline
|
||||
def test_native_lightrag_path_leaves_unknown_table_caption_empty(tmp_path, monkeypatch):
|
||||
"""The native DOCX parser does not infer table titles, so its table
|
||||
sidecar must not synthesize captions like ``表1``.
|
||||
"""
|
||||
|
||||
async def _run():
|
||||
input_dir = tmp_path / "input"
|
||||
input_dir.mkdir()
|
||||
monkeypatch.setenv("INPUT_DIR", str(input_dir))
|
||||
|
||||
source_path = input_dir / "table.docx"
|
||||
source_path.write_bytes(b"fake")
|
||||
|
||||
def _stub_extract(file_path, drawing_context=None, **kwargs):
|
||||
return [_block('before\n<table>[["A"]]</table>\nafter')]
|
||||
|
||||
monkeypatch.setattr(
|
||||
"lightrag.parser.docx.parse_document.extract_docx_blocks",
|
||||
_stub_extract,
|
||||
)
|
||||
|
||||
rag = _MiniRag(tmp_path / "work")
|
||||
result = await _parse_via_registry(
|
||||
rag,
|
||||
"native",
|
||||
"doc-table",
|
||||
str(source_path),
|
||||
{"parse_format": FULL_DOCS_FORMAT_PENDING_PARSE, "content": ""},
|
||||
)
|
||||
|
||||
blocks_path = Path(result["blocks_path"])
|
||||
lines = blocks_path.read_text(encoding="utf-8").splitlines()
|
||||
block = json.loads(lines[1])
|
||||
assert "caption=" not in block["content"]
|
||||
assert "表1" not in block["content"]
|
||||
|
||||
tables_path = blocks_path.with_suffix("").with_suffix(".tables.json")
|
||||
tables = json.loads(tables_path.read_text(encoding="utf-8"))
|
||||
table_entry = tables["tables"]["tb-table-0001"]
|
||||
assert table_entry["caption"] == ""
|
||||
|
||||
# Surrounding is now backfilled at analyze_multimodal entry, not in
|
||||
# parse_native — invoke the same routine directly to mirror that.
|
||||
from lightrag.multimodal_context import enrich_sidecars_with_surrounding
|
||||
|
||||
enrich_sidecars_with_surrounding(
|
||||
blocks_path=str(blocks_path),
|
||||
enabled_modalities={"tables"},
|
||||
tokenizer=rag.tokenizer,
|
||||
)
|
||||
tables = json.loads(tables_path.read_text(encoding="utf-8"))
|
||||
table_entry = tables["tables"]["tb-table-0001"]
|
||||
assert table_entry["surrounding"] == {
|
||||
"leading": "before\n",
|
||||
"trailing": "\nafter",
|
||||
}
|
||||
|
||||
asyncio.run(_run())
|
||||
|
||||
|
||||
@pytest.mark.offline
|
||||
def test_analyze_entrypoint_backfills_surrounding_for_all_sidecars(
|
||||
tmp_path, monkeypatch
|
||||
):
|
||||
"""Surrounding is backfilled at analyze_multimodal entry, covering native
|
||||
parse output as well as any other sidecar-producing engine."""
|
||||
|
||||
async def _run():
|
||||
input_dir = tmp_path / "input"
|
||||
input_dir.mkdir()
|
||||
monkeypatch.setenv("INPUT_DIR", str(input_dir))
|
||||
|
||||
source_path = input_dir / "all_modalities.docx"
|
||||
source_path.write_bytes(b"fake")
|
||||
|
||||
def _stub_extract(file_path, drawing_context=None, **kwargs):
|
||||
assert drawing_context is not None
|
||||
assert drawing_context.export_dir_path is not None
|
||||
(drawing_context.export_dir_path / "pic.png").write_bytes(b"PNG")
|
||||
return [
|
||||
_block(
|
||||
'alpha <drawing id="1" format="png" '
|
||||
'path="all_modalities.blocks.assets/pic.png" /> beta\n'
|
||||
'<table>[["A"]]</table> gamma\n'
|
||||
"<equation>E=mc^2</equation>\n"
|
||||
"delta"
|
||||
)
|
||||
]
|
||||
|
||||
monkeypatch.setattr(
|
||||
"lightrag.parser.docx.parse_document.extract_docx_blocks",
|
||||
_stub_extract,
|
||||
)
|
||||
|
||||
rag = _MiniRag(tmp_path / "work")
|
||||
result = await _parse_via_registry(
|
||||
rag,
|
||||
"native",
|
||||
"doc-mm",
|
||||
str(source_path),
|
||||
{"parse_format": FULL_DOCS_FORMAT_PENDING_PARSE, "content": ""},
|
||||
)
|
||||
|
||||
blocks_path = Path(result["blocks_path"])
|
||||
base = str(blocks_path)[: -len(".blocks.jsonl")]
|
||||
|
||||
# Parse-time sidecars must NOT contain surrounding — that field is
|
||||
# now produced at analyze_multimodal entry.
|
||||
for root in ("drawings", "tables", "equations"):
|
||||
payload = json.loads(Path(base + f".{root}.json").read_text("utf-8"))
|
||||
for item in payload[root].values():
|
||||
assert "surrounding" not in item
|
||||
|
||||
# Now invoke the same routine analyze_multimodal calls and verify
|
||||
# all modalities get populated.
|
||||
from lightrag.multimodal_context import enrich_sidecars_with_surrounding
|
||||
|
||||
enrich_sidecars_with_surrounding(
|
||||
blocks_path=str(blocks_path),
|
||||
enabled_modalities={"drawings", "tables", "equations"},
|
||||
tokenizer=rag.tokenizer,
|
||||
)
|
||||
for root in ("drawings", "tables", "equations"):
|
||||
payload = json.loads(Path(base + f".{root}.json").read_text("utf-8"))
|
||||
items = payload[root]
|
||||
assert items
|
||||
for item in items.values():
|
||||
assert "surrounding" in item
|
||||
assert set(item["surrounding"]) == {"leading", "trailing"}
|
||||
|
||||
asyncio.run(_run())
|
||||
|
||||
|
||||
@pytest.mark.offline
|
||||
def test_native_lightrag_path_writes_image_assets_to_blocks_assets_dir(
|
||||
tmp_path, monkeypatch
|
||||
):
|
||||
"""Native parsing must drop image bytes into ``<base>.blocks.assets/``
|
||||
after the adapter creates the parsed dir (which it wipes at the start),
|
||||
and the drawings sidecar must reference the rewritten ids.
|
||||
"""
|
||||
from pathlib import Path
|
||||
|
||||
async def _run():
|
||||
input_dir = tmp_path / "input"
|
||||
input_dir.mkdir()
|
||||
monkeypatch.setenv("INPUT_DIR", str(input_dir))
|
||||
|
||||
source_path = input_dir / "with_pics.docx"
|
||||
source_path.write_bytes(b"fake")
|
||||
|
||||
def _stub_extract(file_path, drawing_context=None, **kwargs):
|
||||
# The adapter already created the asset dir before calling us;
|
||||
# write the fake image bytes there as a side-effect, then return
|
||||
# a block whose content references that asset via <drawing .../>.
|
||||
assert drawing_context is not None
|
||||
assert drawing_context.export_dir_path is not None
|
||||
(drawing_context.export_dir_path / "pic.png").write_bytes(b"PNG-BYTES")
|
||||
return [
|
||||
_block(
|
||||
"intro\n"
|
||||
'<drawing id="1" name="pic" format="png" '
|
||||
'path="with_pics.blocks.assets/pic.png" />\n'
|
||||
"outro",
|
||||
heading="intro",
|
||||
level=1,
|
||||
),
|
||||
]
|
||||
|
||||
monkeypatch.setattr(
|
||||
"lightrag.parser.docx.parse_document.extract_docx_blocks",
|
||||
_stub_extract,
|
||||
)
|
||||
|
||||
rag = _MiniRag(tmp_path / "work")
|
||||
result = await _parse_via_registry(
|
||||
rag,
|
||||
"native",
|
||||
"doc-pic",
|
||||
str(source_path),
|
||||
{"parse_format": FULL_DOCS_FORMAT_PENDING_PARSE, "content": ""},
|
||||
)
|
||||
|
||||
blocks_path = Path(result["blocks_path"])
|
||||
parsed_dir = blocks_path.parent
|
||||
asset_dir = parsed_dir / "with_pics.blocks.assets"
|
||||
# Asset dir must exist alongside .blocks.jsonl and survive the
|
||||
# adapter's parsed_dir cleanup step.
|
||||
assert asset_dir.is_dir(), (
|
||||
f"asset dir not created at {asset_dir}; parsed_dir contents: "
|
||||
f"{list(parsed_dir.iterdir())}"
|
||||
)
|
||||
assert (asset_dir / "pic.png").read_bytes() == b"PNG-BYTES"
|
||||
# And drawings.json sidecar should also be there since the block
|
||||
# contained a <drawing .../> markup the adapter had to record.
|
||||
assert (parsed_dir / "with_pics.drawings.json").is_file()
|
||||
|
||||
asyncio.run(_run())
|
||||
@@ -0,0 +1,269 @@
|
||||
"""Tests for the unified parser debug CLI (``lightrag/parser/cli.py``).
|
||||
|
||||
The CLI behaviour under test is engine-agnostic: argument parsing, the
|
||||
flat sidecar layout (no ``__parsed__/`` middle layer), the lenient raw
|
||||
cache strategy (non-empty raw_dir reused without manifest checks), and
|
||||
the no-archive guarantee on the source file.
|
||||
|
||||
We drive these checks via the **docling** engine path because docling's
|
||||
raw bundle is the easiest to construct as static fixture (a single JSON
|
||||
file) with zero external service or fixture-file dependency. The other
|
||||
two engines exercise the same CLI code path:
|
||||
|
||||
- ``native`` would need a real ``.docx`` byte stream end-to-end (golden
|
||||
fixtures live under ``tests/parser/docx/golden/`` and have
|
||||
their own coverage via ``test_native_docx_golden.py``).
|
||||
- ``mineru`` would need to mock ``MinerURawClient.download_into`` on
|
||||
the cache-miss path, or seed a mineru raw bundle layout (more files
|
||||
than docling's). Cache-hit reuses the same CLI orchestration as
|
||||
docling, so coverage here implicitly validates mineru's CLI wiring
|
||||
too.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import pytest
|
||||
|
||||
from lightrag.parser.cli import main
|
||||
|
||||
|
||||
def _make_main_json(
|
||||
*,
|
||||
origin_filename: str = "demo.pdf",
|
||||
with_table: bool = False,
|
||||
) -> dict[str, Any]:
|
||||
payload: dict[str, Any] = {
|
||||
"schema_name": "DoclingDocument",
|
||||
"version": "1.10.0",
|
||||
"origin": {"filename": origin_filename, "mimetype": "application/pdf"},
|
||||
"body": {
|
||||
"self_ref": "#/body",
|
||||
"children": [{"$ref": "#/texts/0"}, {"$ref": "#/texts/1"}],
|
||||
"content_layer": "body",
|
||||
"label": "unspecified",
|
||||
},
|
||||
"groups": [],
|
||||
"texts": [
|
||||
{
|
||||
"self_ref": "#/texts/0",
|
||||
"label": "title",
|
||||
"text": "Hello Title",
|
||||
"orig": "Hello Title",
|
||||
"content_layer": "body",
|
||||
"prov": [],
|
||||
},
|
||||
{
|
||||
"self_ref": "#/texts/1",
|
||||
"label": "text",
|
||||
"text": "Body line.",
|
||||
"orig": "Body line.",
|
||||
"content_layer": "body",
|
||||
"prov": [],
|
||||
},
|
||||
],
|
||||
"pictures": [],
|
||||
"tables": [],
|
||||
"key_value_items": [],
|
||||
"form_items": [],
|
||||
}
|
||||
if with_table:
|
||||
payload["body"]["children"].append({"$ref": "#/tables/0"})
|
||||
payload["tables"].append(
|
||||
{
|
||||
"self_ref": "#/tables/0",
|
||||
"label": "table",
|
||||
"content_layer": "body",
|
||||
"data": {
|
||||
"num_rows": 1,
|
||||
"num_cols": 2,
|
||||
"grid": [[{"text": "A"}, {"text": "B"}]],
|
||||
},
|
||||
"prov": [],
|
||||
}
|
||||
)
|
||||
return payload
|
||||
|
||||
|
||||
def _seed_raw_dir(raw_dir: Path, *, with_table: bool = False) -> None:
|
||||
raw_dir.mkdir(parents=True, exist_ok=True)
|
||||
(raw_dir / "demo.json").write_text(
|
||||
json.dumps(_make_main_json(with_table=with_table)),
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
|
||||
def _read_meta(blocks_path: Path) -> dict[str, Any]:
|
||||
return json.loads(blocks_path.read_text(encoding="utf-8").splitlines()[0])
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _clean_env(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
for name in (
|
||||
"DOCLING_BBOX_ATTRIBUTES",
|
||||
"DOCLING_ENGINE_VERSION",
|
||||
"LIGHTRAG_FORCE_REPARSE_DOCLING",
|
||||
):
|
||||
monkeypatch.delenv(name, raising=False)
|
||||
|
||||
|
||||
def test_cli_writes_sidecar_from_existing_raw_dir(tmp_path: Path) -> None:
|
||||
source = tmp_path / "demo.pdf"
|
||||
source.write_bytes(b"%PDF-1.4\n") # never read; presence is the only check
|
||||
_seed_raw_dir(tmp_path / "demo.pdf.docling_raw", with_table=True)
|
||||
|
||||
rc = main([str(source), "--engine", "docling"])
|
||||
assert rc == 0
|
||||
|
||||
parsed_dir = tmp_path / "demo.pdf.parsed"
|
||||
blocks_path = parsed_dir / "demo.blocks.jsonl"
|
||||
assert blocks_path.is_file()
|
||||
assert (parsed_dir / "demo.tables.json").is_file()
|
||||
|
||||
meta = _read_meta(blocks_path)
|
||||
assert meta["parse_engine"] == "docling"
|
||||
assert meta["document_name"] == "demo.pdf"
|
||||
assert meta["table_file"] is True
|
||||
# Source file stays where it was — the CLI mocks the archive step.
|
||||
assert source.is_file()
|
||||
|
||||
|
||||
def test_cli_doc_id_default_is_stable_across_runs(tmp_path: Path) -> None:
|
||||
source = tmp_path / "demo.pdf"
|
||||
source.write_bytes(b"%PDF-1.4\n")
|
||||
_seed_raw_dir(tmp_path / "demo.pdf.docling_raw")
|
||||
|
||||
blocks_path = tmp_path / "demo.pdf.parsed" / "demo.blocks.jsonl"
|
||||
|
||||
assert main([str(source), "--engine", "docling"]) == 0
|
||||
first_lines = blocks_path.read_text(encoding="utf-8").splitlines()
|
||||
first_meta = json.loads(first_lines[0])
|
||||
first_block_ids = [json.loads(line)["blockid"] for line in first_lines[1:]]
|
||||
|
||||
assert main([str(source), "--engine", "docling"]) == 0
|
||||
second_lines = blocks_path.read_text(encoding="utf-8").splitlines()
|
||||
second_meta = json.loads(second_lines[0])
|
||||
second_block_ids = [json.loads(line)["blockid"] for line in second_lines[1:]]
|
||||
|
||||
assert first_meta["doc_id"].startswith("doc-")
|
||||
assert first_meta["doc_id"] == second_meta["doc_id"]
|
||||
assert first_block_ids and first_block_ids == second_block_ids
|
||||
|
||||
|
||||
def test_cli_doc_id_override(tmp_path: Path) -> None:
|
||||
source = tmp_path / "demo.pdf"
|
||||
source.write_bytes(b"%PDF-1.4\n")
|
||||
_seed_raw_dir(tmp_path / "demo.pdf.docling_raw", with_table=True)
|
||||
|
||||
override = "doc-" + "a" * 32
|
||||
rc = main([str(source), "--engine", "docling", "--doc-id", override])
|
||||
assert rc == 0
|
||||
|
||||
parsed_dir = tmp_path / "demo.pdf.parsed"
|
||||
meta = _read_meta(parsed_dir / "demo.blocks.jsonl")
|
||||
assert meta["doc_id"] == override
|
||||
|
||||
tables = json.loads((parsed_dir / "demo.tables.json").read_text(encoding="utf-8"))[
|
||||
"tables"
|
||||
]
|
||||
assert tables
|
||||
assert all(tid.startswith("tb-" + "a" * 32 + "-") for tid in tables)
|
||||
|
||||
|
||||
def test_cli_custom_sidecar_parent_dir(tmp_path: Path) -> None:
|
||||
source = tmp_path / "demo.pdf"
|
||||
source.write_bytes(b"%PDF-1.4\n")
|
||||
custom_parent = tmp_path / "elsewhere"
|
||||
custom_parent.mkdir()
|
||||
_seed_raw_dir(custom_parent / "demo.pdf.docling_raw")
|
||||
|
||||
rc = main([str(source), "--engine", "docling", "-o", str(custom_parent)])
|
||||
assert rc == 0
|
||||
assert (custom_parent / "demo.pdf.parsed" / "demo.blocks.jsonl").is_file()
|
||||
# Nothing should land in the source's parent directory.
|
||||
assert not (tmp_path / "demo.pdf.parsed").exists()
|
||||
# Source file is preserved in place.
|
||||
assert source.is_file()
|
||||
|
||||
|
||||
def test_cli_missing_input_file_returns_error(
|
||||
tmp_path: Path, capsys: pytest.CaptureFixture[str]
|
||||
) -> None:
|
||||
missing = tmp_path / "nope.pdf"
|
||||
rc = main([str(missing), "--engine", "docling"])
|
||||
assert rc == 1
|
||||
err = capsys.readouterr().err
|
||||
assert str(missing.resolve()) in err
|
||||
|
||||
|
||||
def test_cli_rejects_suffix_engine_mismatch(
|
||||
tmp_path: Path, capsys: pytest.CaptureFixture[str]
|
||||
) -> None:
|
||||
# native only accepts .docx; feeding it a .pdf should fail up-front with
|
||||
# a clear error rather than crashing deep inside the IR builder.
|
||||
source = tmp_path / "demo.pdf"
|
||||
source.write_bytes(b"%PDF-1.4\n")
|
||||
|
||||
rc = main([str(source), "--engine", "native"])
|
||||
assert rc == 1
|
||||
err = capsys.readouterr().err
|
||||
assert "native" in err
|
||||
assert "pdf" in err
|
||||
assert "docx" in err # supported suffix list mentions docx
|
||||
|
||||
|
||||
def test_cli_legacy_engine_prints_raw_summary(
|
||||
tmp_path: Path, capsys: pytest.CaptureFixture[str]
|
||||
) -> None:
|
||||
# legacy is registry-selectable and emits plain content with no sidecar
|
||||
# (blocks_path=""). The CLI must summarize the content instead of trying
|
||||
# to open a non-existent blocks file.
|
||||
source = tmp_path / "notes.txt"
|
||||
source.write_text("hello world\nsecond line\n", encoding="utf-8")
|
||||
|
||||
rc = main([str(source), "--engine", "legacy"])
|
||||
assert rc == 0
|
||||
out = capsys.readouterr().out
|
||||
assert "engine : legacy" in out
|
||||
assert "format : raw" in out
|
||||
assert "hello world" in out
|
||||
|
||||
|
||||
def test_cli_direct_script_native_does_not_shadow_python_docx(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
from docx import Document
|
||||
|
||||
source = tmp_path / "demo.docx"
|
||||
doc = Document()
|
||||
doc.add_heading("Hello Title", level=1)
|
||||
doc.add_paragraph("Body line.")
|
||||
doc.save(source)
|
||||
|
||||
repo_root = Path(__file__).resolve().parents[2]
|
||||
cli_path = repo_root / "lightrag" / "parser" / "cli.py"
|
||||
proc = subprocess.run(
|
||||
[
|
||||
sys.executable,
|
||||
str(cli_path),
|
||||
str(source),
|
||||
"--engine",
|
||||
"native",
|
||||
"--preview",
|
||||
"0",
|
||||
],
|
||||
cwd=repo_root,
|
||||
text=True,
|
||||
capture_output=True,
|
||||
check=False,
|
||||
)
|
||||
|
||||
assert proc.returncode == 0, proc.stderr + proc.stdout
|
||||
assert "python-docx not installed" not in proc.stderr
|
||||
assert (tmp_path / "demo.docx.parsed" / "demo.blocks.jsonl").is_file()
|
||||
assert source.is_file()
|
||||
@@ -0,0 +1,132 @@
|
||||
"""Unit tests for third-party parser discovery (lightrag.parser.plugins)."""
|
||||
|
||||
import pytest
|
||||
|
||||
from lightrag.parser import plugins, registry
|
||||
|
||||
|
||||
class _FakeEntryPoint:
|
||||
"""Stand-in for importlib.metadata.EntryPoint (name/value/load)."""
|
||||
|
||||
def __init__(self, name, value, fn):
|
||||
self.name = name
|
||||
self.value = value
|
||||
self._fn = fn
|
||||
|
||||
def load(self):
|
||||
if isinstance(self._fn, Exception):
|
||||
raise self._fn
|
||||
return self._fn
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _reset_loaded_flag(monkeypatch):
|
||||
"""Each test starts from the not-yet-loaded state."""
|
||||
monkeypatch.setattr(plugins, "_loaded", False)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def third_party_cleanup():
|
||||
"""Remove engines a test registered so the module-level registry stays
|
||||
clean for other tests."""
|
||||
names = []
|
||||
yield names
|
||||
for name in names:
|
||||
registry._REGISTRY.pop(name, None)
|
||||
|
||||
|
||||
def _install_eps(monkeypatch, eps):
|
||||
def _fake_entry_points(*, group):
|
||||
assert group == plugins.ENTRY_POINT_GROUP
|
||||
return list(eps)
|
||||
|
||||
monkeypatch.setattr(plugins, "entry_points", _fake_entry_points)
|
||||
|
||||
|
||||
def test_load_discovers_and_registers(monkeypatch, third_party_cleanup):
|
||||
def _register():
|
||||
registry.register_parser(
|
||||
registry.ParserSpec(
|
||||
engine_name="plug-engine",
|
||||
impl="x:Y",
|
||||
suffixes=frozenset({"foo"}),
|
||||
queue_group="plug-engine",
|
||||
concurrency=1,
|
||||
)
|
||||
)
|
||||
|
||||
_install_eps(monkeypatch, [_FakeEntryPoint("plug", "pkg.mod:register", _register)])
|
||||
third_party_cleanup.append("plug-engine")
|
||||
|
||||
assert plugins.load_third_party_parsers() == ["plug"]
|
||||
assert "plug-engine" in registry.supported_parser_engines()
|
||||
|
||||
|
||||
def test_load_is_idempotent_per_process(monkeypatch):
|
||||
calls = {"n": 0}
|
||||
|
||||
def _register():
|
||||
calls["n"] += 1
|
||||
|
||||
_install_eps(monkeypatch, [_FakeEntryPoint("plug", "pkg.mod:register", _register)])
|
||||
|
||||
assert plugins.load_third_party_parsers() == ["plug"]
|
||||
# Second call is a no-op (server lifespan + CLI may both call it).
|
||||
assert plugins.load_third_party_parsers() == []
|
||||
assert calls["n"] == 1
|
||||
# force=True re-runs (test escape hatch).
|
||||
assert plugins.load_third_party_parsers(force=True) == ["plug"]
|
||||
assert calls["n"] == 2
|
||||
|
||||
|
||||
def test_broken_plugin_is_skipped_not_fatal(monkeypatch, third_party_cleanup):
|
||||
def _good_register():
|
||||
registry.register_parser(
|
||||
registry.ParserSpec(
|
||||
engine_name="good-engine",
|
||||
impl="x:Y",
|
||||
suffixes=frozenset({"bar"}),
|
||||
queue_group="good-engine",
|
||||
concurrency=1,
|
||||
)
|
||||
)
|
||||
|
||||
_install_eps(
|
||||
monkeypatch,
|
||||
[
|
||||
_FakeEntryPoint("broken", "bad.mod:register", ImportError("boom")),
|
||||
_FakeEntryPoint("good", "ok.mod:register", _good_register),
|
||||
],
|
||||
)
|
||||
third_party_cleanup.append("good-engine")
|
||||
|
||||
# The broken plugin is logged + skipped; the good one still loads.
|
||||
assert plugins.load_third_party_parsers() == ["good"]
|
||||
assert "good-engine" in registry.supported_parser_engines()
|
||||
|
||||
|
||||
def test_cli_sees_entry_point_engine(monkeypatch, third_party_cleanup):
|
||||
"""End-to-end: a plugin-registered engine becomes a valid --engine choice
|
||||
in the debug CLI (which calls load_third_party_parsers in main)."""
|
||||
from lightrag.parser import cli
|
||||
|
||||
def _register():
|
||||
registry.register_parser(
|
||||
registry.ParserSpec(
|
||||
engine_name="cli-plug-engine",
|
||||
impl="x:Y",
|
||||
suffixes=frozenset({"baz"}),
|
||||
queue_group="cli-plug-engine",
|
||||
concurrency=1,
|
||||
)
|
||||
)
|
||||
|
||||
_install_eps(
|
||||
monkeypatch, [_FakeEntryPoint("cliplug", "pkg.mod:register", _register)]
|
||||
)
|
||||
third_party_cleanup.append("cli-plug-engine")
|
||||
|
||||
# Missing input file -> exit 1 AFTER argparse accepted the engine choice
|
||||
# (an unknown engine would exit 2 from argparse instead).
|
||||
rc = cli.main(["/nonexistent/x.baz", "--engine", "cli-plug-engine"])
|
||||
assert rc == 1
|
||||
@@ -0,0 +1,77 @@
|
||||
"""Unit tests for the parser registry (lightrag.parser.registry)."""
|
||||
|
||||
import subprocess
|
||||
import sys
|
||||
|
||||
|
||||
from lightrag.parser import registry
|
||||
|
||||
|
||||
def test_supported_engines_are_user_selectable_only():
|
||||
engines = registry.supported_parser_engines()
|
||||
assert engines == frozenset({"native", "legacy", "mineru", "docling"})
|
||||
# Internal format handlers are registered but not user-selectable.
|
||||
assert "reuse" not in engines
|
||||
assert "passthrough" not in engines
|
||||
|
||||
|
||||
def test_suffix_capabilities_lookup():
|
||||
assert "pdf" in registry.suffix_capabilities("mineru")
|
||||
assert registry.suffix_capabilities("native") == frozenset(
|
||||
{"docx", "md", "textpack"}
|
||||
)
|
||||
assert registry.suffix_capabilities("unknown-engine") == frozenset()
|
||||
|
||||
|
||||
def test_get_parser_unknown_returns_none():
|
||||
assert registry.get_parser("does-not-exist") is None
|
||||
|
||||
|
||||
def test_get_parser_instances_are_cached():
|
||||
a = registry.get_parser("native")
|
||||
b = registry.get_parser("native")
|
||||
assert a is b and a is not None
|
||||
|
||||
|
||||
def test_register_parser_roundtrips_concurrency():
|
||||
# The registrant bakes any env override into a concrete ``concurrency``
|
||||
# value at registration; the spec carries it verbatim.
|
||||
spec = registry.ParserSpec(
|
||||
engine_name="third-party-engine",
|
||||
impl="x:Y",
|
||||
suffixes=frozenset({"foo"}),
|
||||
queue_group="third-party-engine",
|
||||
concurrency=7,
|
||||
)
|
||||
try:
|
||||
registry.register_parser(spec)
|
||||
snapshot = registry.parser_specs_snapshot()
|
||||
assert snapshot["third-party-engine"].concurrency == 7
|
||||
assert snapshot["third-party-engine"].queue_group == "third-party-engine"
|
||||
finally:
|
||||
# Keep the module-level registry clean for other tests.
|
||||
registry._REGISTRY.pop("third-party-engine", None)
|
||||
|
||||
|
||||
def test_mineru_endpoint_requirement_tracks_api_mode(monkeypatch):
|
||||
monkeypatch.setenv("MINERU_API_MODE", "official")
|
||||
assert registry.engine_endpoint_requirement("mineru") == "MINERU_API_TOKEN"
|
||||
monkeypatch.setenv("MINERU_API_MODE", "local")
|
||||
assert registry.engine_endpoint_requirement("mineru") == "MINERU_LOCAL_ENDPOINT"
|
||||
|
||||
|
||||
def test_capability_queries_do_not_import_parser_impls():
|
||||
"""Importing the registry + querying capabilities must not pull a parser
|
||||
implementation (and therefore httpx). Run in a clean subprocess so other
|
||||
tests' imports don't pollute sys.modules."""
|
||||
code = (
|
||||
"import sys; import lightrag.parser.registry as r; "
|
||||
"r.supported_parser_engines(); r.suffix_capabilities('mineru'); "
|
||||
"r.engine_endpoint_configured('docling'); "
|
||||
"assert 'httpx' not in sys.modules, 'httpx leaked'; "
|
||||
"assert 'lightrag.parser.external.mineru.parser' not in sys.modules; "
|
||||
"print('clean')"
|
||||
)
|
||||
proc = subprocess.run([sys.executable, "-c", code], capture_output=True, text=True)
|
||||
assert proc.returncode == 0, proc.stderr
|
||||
assert "clean" in proc.stdout
|
||||
Reference in New Issue
Block a user