"""Unified sidecar debug CLI for any registered parser engine. Dispatches one source file through the parser registry (``get_parser(engine).parse(...)``) and writes the resulting sidecar (and raw bundle, for external engines) into a flat layout — no ``__parsed__/`` middle layer, source file never archived — so the artifacts can be inspected next to the input file. Because dispatch goes through the registry, a third-party engine registered via ``register_parser`` is a valid ``--engine`` choice with no CLI changes. Invocation:: python -m lightrag.parser.cli path/to/sample.docx --engine native python -m lightrag.parser.cli path/to/sample.pdf --engine mineru python -m lightrag.parser.cli path/to/sample.pdf --engine docling --force-reparse See ``docs/ParserDebugCLI-zh.md`` for the full reference. """ from __future__ import annotations import argparse import asyncio import json import sys from contextlib import ExitStack from pathlib import Path from typing import Any from unittest import mock def _normalize_direct_script_sys_path() -> None: if __package__: return parser_dir = Path(__file__).resolve().parent repo_root = parser_dir.parent.parent # Direct execution adds lightrag/parser to sys.path, which makes the # native parser's third-party ``docx`` import resolve to parser/docx. sys.path[:] = [ entry for entry in sys.path if Path(entry or ".").resolve() != parser_dir ] repo_root_str = str(repo_root) if repo_root_str in sys.path: sys.path.remove(repo_root_str) sys.path.insert(0, repo_root_str) _normalize_direct_script_sys_path() def _build_parser() -> argparse.ArgumentParser: # Registry import is cheap by design (no parser impl is pulled in), so # deriving --engine choices here keeps third-party engines selectable # without making --help pay for a heavy import. from lightrag.parser.registry import supported_parser_engines engines = sorted(supported_parser_engines()) parser = argparse.ArgumentParser( prog="parse_sidecar", description=( "Run a registered parser engine on a single file and emit sidecar " "artifacts (plus a raw bundle for external engines) into a flat " "layout alongside the source. No __parsed__/ middle layer; the " "source file is never moved." ), ) parser.add_argument("input_file", type=Path, help="Source file to parse.") parser.add_argument( "--engine", required=True, choices=engines, help="Parser engine to drive.", ) parser.add_argument( "-o", "--sidecar-parent-dir", type=Path, default=None, help=( "Parent directory for .parsed/ and ._raw/. " "Default: the source file's parent directory." ), ) parser.add_argument( "--doc-id", default=None, help="Override the doc id. Default: doc-.", ) parser.add_argument( "--force-reparse", action="store_true", help=( "Only affects mineru/docling. By default a non-empty raw_dir is " "treated as a valid cache and reused without manifest checks; " "this flag clears raw_dir and forces a fresh download/parse." ), ) parser.add_argument( "--preview", type=int, default=5, metavar="N", help="Number of block rows to preview after parsing (0 disables).", ) return parser def _print_summary(blocks_path: Path, raw_dir: Path | None, preview: int) -> None: with blocks_path.open("r", encoding="utf-8") as fh: meta_line = fh.readline().strip() if not meta_line: raise SystemExit(f"empty blocks file at {blocks_path}") meta = json.loads(meta_line) rows = [json.loads(line) for line in fh if line.strip()] parsed_dir = blocks_path.parent print(f"parsed dir : {parsed_dir} (exists={parsed_dir.exists()})") if raw_dir is not None: print(f"raw dir : {raw_dir} (exists={raw_dir.exists()})") print(f"document : {meta.get('document_name')}") print(f"doc_id : {meta.get('doc_id')}") print(f"engine : {meta.get('parse_engine')}") print(f"blocks : {meta.get('blocks')}") print( f"sidecars : tables={meta.get('table_file')} " f"drawings={meta.get('drawing_file')} " f"equations={meta.get('equation_file')} " f"asset_dir={meta.get('asset_dir')}" ) if preview > 0 and rows: shown = min(preview, len(rows)) print(f"--- preview (first {shown} of {len(rows)} blocks) ---") for row in rows[:preview]: heading = row.get("heading") or "" content = (row.get("content") or "").replace("\n", " ") snippet = content if len(content) <= 80 else content[:77] + "..." print(f" [{row.get('blockid', '')[:8]}] heading={heading!r} :: {snippet}") def _print_raw_summary(result: dict, preview: int) -> None: """Summary for engines that emit plain content with no sidecar (legacy / any non-sidecar third-party engine: ``blocks_path`` is empty).""" content = result.get("content") or "" print(f"engine : {result.get('parse_engine')}") print(f"format : {result.get('parse_format')}") print(f"content : {len(content)} chars") if preview > 0 and content: snippet = content[:400] print(f"--- preview (first {len(snippet)} of {len(content)} chars) ---") print(snippet + ("..." if len(content) > len(snippet) else "")) async def _run(args: argparse.Namespace) -> int: # Pipeline + heavy parser imports are deferred so ``--help`` and the # input-file existence check don't pay for them. from lightrag.constants import FULL_DOCS_FORMAT_PENDING_PARSE from lightrag.parser.base import ParseContext from lightrag.parser.external._base import ExternalParserBase from lightrag.parser.registry import get_parser, suffix_capabilities from lightrag.parser.debug import build_debug_rag from lightrag.parser.docx.parse_document import DocxContentError from lightrag.utils import compute_mdhash_id import lightrag.pipeline as pipeline_mod import lightrag.utils_pipeline as utils_pipeline_mod source = args.input_file.resolve() if not source.is_file(): print(f"error: input file does not exist: {source}", file=sys.stderr) return 1 # Reject suffix/engine mismatches up-front: the pipeline would otherwise # fail deep inside the IR builder with a less helpful message. suffix = source.suffix.lstrip(".").lower() supported = suffix_capabilities(args.engine) if suffix not in supported: print( f"error: engine '{args.engine}' does not support .{suffix or ''} " f"files (supported: {', '.join(sorted(supported))})", file=sys.stderr, ) return 1 parser = get_parser(args.engine) if parser is None: print(f"error: engine '{args.engine}' is not registered", file=sys.stderr) return 1 is_external = isinstance(parser, ExternalParserBase) sidecar_parent = (args.sidecar_parent_dir or source.parent).resolve() sidecar_parent.mkdir(parents=True, exist_ok=True) parsed_dir = sidecar_parent / f"{source.name}.parsed" # External engines preserve a raw bundle next to the sidecar; its name is # derived from the engine's own raw_dir_suffix (mirrors the flattened # raw_dir_for_parsed_dir layout), so any external engine works here. raw_dir = ( sidecar_parent / f"{source.name}{parser.raw_dir_suffix}" if is_external else None ) doc_id = args.doc_id or compute_mdhash_id(str(source), prefix="doc-") def _patched_artifact_dir( file_path: str | None = None, *, parent_hint: Any | None = None, ) -> Path: # Flatten the production "/__parsed__/.parsed/" # layout to "/.parsed/" so the sidecar # and the source file sit side by side. return parsed_dir def _lenient_bundle( raw_dir_arg: Path, _source_file: Path | None = None, *_args: Any, **_kwargs: Any ) -> bool: # Tolerate the ExternalParserBase hook's keyword-only ``engine_params`` # (and any future args) — this stub replaces ``is_bundle_valid``. return raw_dir_arg.exists() and any(raw_dir_arg.iterdir()) def _force_miss(*_args: Any, **_kwargs: Any) -> bool: return False bundle_check = _force_miss if args.force_reparse else _lenient_bundle async def _noop_archive(*_args: Any, **_kwargs: Any) -> None: return None rag = build_debug_rag() with ExitStack() as stack: # Patch 1: redirect sidecar output to the flat layout. # parsed_artifact_dir_for is from-imported into pipeline at # module load, so patch both namespaces. stack.enter_context( mock.patch.object( utils_pipeline_mod, "parsed_artifact_dir_for", _patched_artifact_dir, ) ) stack.enter_context( mock.patch.object( pipeline_mod, "parsed_artifact_dir_for", _patched_artifact_dir, ) ) # Patch 2: raw cache strategy. ExternalParserBase.parse calls # ``self.is_bundle_valid(...)``, so patch the resolved instance method # directly — works for any external engine without knowing its module. if is_external: stack.enter_context( mock.patch.object(parser, "is_bundle_valid", bundle_check) ) # Patch 3: keep the source file in place. Every engine archives via # ctx.archive_source -> lightrag.pipeline.archive_docx_source_after_... stack.enter_context( mock.patch.object( pipeline_mod, "archive_docx_source_after_full_docs_sync", _noop_archive, ) ) try: result = ( await parser.parse( ParseContext( rag, doc_id, str(source), { "parse_format": FULL_DOCS_FORMAT_PENDING_PARSE, "content": "", }, ) ) ).to_dict() except DocxContentError as exc: # The native DOCX parser now raises (instead of sys.exit) on a # content-limit violation so the server pipeline can fail just the # offending document. Preserve the friendly CLI UX: print the # formatted message and return a non-zero exit code, no traceback. print(str(exc), file=sys.stderr) return 1 blocks_path = result.get("blocks_path") or "" if blocks_path: _print_summary(Path(blocks_path), raw_dir, args.preview) else: # No sidecar (legacy / non-sidecar third-party engine): the result # carries plain content, so summarize that instead of a blocks file. _print_raw_summary(result, args.preview) return 0 def main(argv: list[str] | None = None) -> int: # Discover third-party parser engines before --engine choices are built, # so a `lightrag.parsers` entry-point engine is selectable here exactly # like in the server (which loads plugins in create_app). from lightrag.parser.plugins import load_third_party_parsers load_third_party_parsers() args = _build_parser().parse_args(argv) return asyncio.run(_run(args)) if __name__ == "__main__": sys.exit(main())