chore: import upstream snapshot with attribution

lightrag/parser/external/docling/__init__.py

@@ -0,0 +1,41 @@
+"""Docling parser integration (raw client, cache, manifest, IR adapter).
+
+Public surface for the rest of the codebase. ``parse_docling`` imports
+only from this facade so the inner module layout stays free to evolve.
+"""
+
+from lightrag.constants import DOCLING_RAW_DIR_SUFFIX
+from lightrag.parser.external._common import (
+    clear_dir_contents,
+    raw_dir_for_parsed_dir as _raw_dir_for_parsed_dir,
+)
+
+MANIFEST_ENGINE = "docling"
+
+
+def raw_dir_for_parsed_dir(parsed_dir):
+    """``foo.parsed/`` → ``foo.docling_raw/`` (docling-specific binding)."""
+    return _raw_dir_for_parsed_dir(parsed_dir, suffix=DOCLING_RAW_DIR_SUFFIX)
+
+
+# Imported after ``MANIFEST_ENGINE`` / ``DOCLING_RAW_DIR_SUFFIX`` because
+# the submodules read those constants at import time.
+from lightrag.parser.external.docling.ir_builder import (  # noqa: E402
+    DoclingIRBuilder,
+)
+from lightrag.parser.external.docling.cache import (  # noqa: E402
+    is_bundle_valid,
+)
+from lightrag.parser.external.docling.client import (  # noqa: E402
+    DoclingRawClient,
+)
+
+__all__ = [
+    "DOCLING_RAW_DIR_SUFFIX",
+    "MANIFEST_ENGINE",
+    "DoclingIRBuilder",
+    "DoclingRawClient",
+    "clear_dir_contents",
+    "is_bundle_valid",
+    "raw_dir_for_parsed_dir",
+]

lightrag/parser/external/docling/cache.py

@@ -0,0 +1,257 @@
+"""Cache validation for ``*.docling_raw/`` bundles.
+
+Validation policy (settled in
+``docs/DoclingSidecarRefactorPlan-zh.md`` §4.1):
+
+1. ``_manifest.json`` exists, parses, ``engine="docling"`` ∧ schema version
+   matches.
+2. **Source size fast-path**: ``source_file.stat().st_size`` matches the
+   manifest; mismatch → miss without hashing.
+3. **Source content_hash**: full sha256 of the current source file matches
+   the manifest.
+4. **Engine version**: if ``DOCLING_ENGINE_VERSION`` is set in env and the
+   manifest recorded a non-empty value, they must match.
+5. **Endpoint signature**: if the active ``DOCLING_ENDPOINT`` differs from
+   what was recorded at parse time, miss (avoids re-using a bundle produced
+   by a different docling-serve instance).
+6. **Options signature**: covers every env or fixed constant that changes
+   the produced bundle (OCR flags, language list, formula enrichment,
+   target format and pipeline). Any change → miss.
+7. **Critical file**: the main JSON must exist with matching size **and**
+   sha256 — final tie-breaker against silent corruption affecting the file
+   the adapter depends on.
+8. **Other files**: size-only verification (cheap; covers most corruption
+   modes for markdown / artifacts).
+
+Any failed step ⇒ cache miss; the caller wipes the directory contents and
+re-runs the download.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import json
+import os
+from collections.abc import Mapping
+from pathlib import Path
+from typing import Any
+
+from lightrag.parser.external._common import compute_size_and_hash, env_bool
+from lightrag.parser.external._manifest import load_manifest
+from lightrag.parser.external.docling import MANIFEST_ENGINE
+from lightrag.utils import logger
+
+# Legacy upload-path suffix. ``env.example`` historically documented
+# ``DOCLING_ENDPOINT=http://host:5001/v1/convert/file/async`` (the full
+# upload URL); the current client expects a base URL and appends the path
+# itself. Strip the suffix so an unmodified pre-refactor ``.env`` keeps
+# working instead of producing
+# ``/v1/convert/file/async/v1/convert/file/async`` requests.
+_LEGACY_UPLOAD_PATH_SUFFIX = "/v1/convert/file/async"
+_legacy_endpoint_warned = False
+
+# Envs that change the bytes docling-serve produces. Any change here must
+# invalidate the bundle cache. ``DOCLING_BBOX_ATTRIBUTES`` is intentionally
+# NOT in this list: it only affects how the adapter writes IR meta, not the
+# docling bundle, so flipping it should re-emit the sidecar (which we always
+# do) without forcing a re-download.
+DOCLING_TUNABLE_ENVS: tuple[str, ...] = (
+    "DOCLING_DO_OCR",
+    "DOCLING_FORCE_OCR",
+    "DOCLING_OCR_ENGINE",
+    "DOCLING_OCR_PRESET",
+    "DOCLING_OCR_LANG",
+    "DOCLING_DO_FORMULA_ENRICHMENT",
+)
+
+
+def current_endpoint_signature() -> str:
+    """The active docling endpoint, normalized to a base URL.
+
+    Normalization:
+
+    - Trims surrounding whitespace and strips trailing slashes.
+    - Strips the legacy ``/v1/convert/file/async`` upload suffix if present,
+      preserving backwards compatibility with the pre-refactor ``env.example``
+      that documented the full upload URL.
+
+    Returns ``""`` if ``DOCLING_ENDPOINT`` is unset — callers that need a
+    real endpoint (``DoclingRawClient``) raise on empty; callers that only
+    compare against a recorded manifest field (``is_bundle_valid``) silently
+    skip the check when either side is empty.
+    """
+    global _legacy_endpoint_warned
+    endpoint = os.getenv("DOCLING_ENDPOINT", "").strip().rstrip("/")
+    if endpoint.endswith(_LEGACY_UPLOAD_PATH_SUFFIX):
+        endpoint = endpoint[: -len(_LEGACY_UPLOAD_PATH_SUFFIX)]
+        if not _legacy_endpoint_warned:
+            _legacy_endpoint_warned = True
+            logger.warning(
+                "DOCLING_ENDPOINT still includes the legacy %r upload suffix; "
+                "stripping it. Update your .env to a base URL "
+                "(e.g. http://host:5001).",
+                _LEGACY_UPLOAD_PATH_SUFFIX,
+            )
+    return endpoint
+
+
+def compute_options_signature(
+    *,
+    tunable_env: dict[str, str],
+    fixed_constants: dict[str, object],
+) -> str:
+    """Stable signature over user-tunable env values and fixed pipeline
+    constants.
+
+    Storing the constants in the signature means a future code change that
+    flips e.g. ``image_export_mode`` from ``referenced`` to ``embedded``
+    invalidates every existing cache without anyone having to remember to
+    bump a version.
+    """
+    payload = json.dumps(
+        {"env": tunable_env, "fixed": fixed_constants},
+        ensure_ascii=False,
+        sort_keys=True,
+        separators=(",", ":"),
+    )
+    return "sha256:" + hashlib.sha256(payload.encode("utf-8")).hexdigest()
+
+
+def snapshot_tunable_env(
+    overrides: "Mapping[str, Any] | None" = None,
+) -> dict[str, str]:
+    """Read effective docling tunables so equivalent requests share a signature.
+
+    ``overrides`` carries decoded per-file engine params (Phase 2: ``force_ocr``)
+    and replaces the corresponding env value so the override feeds BOTH the live
+    request and the cache signature.
+    """
+    overrides = overrides or {}
+    force_ocr = (
+        bool(overrides["force_ocr"])
+        if "force_ocr" in overrides
+        else env_bool("DOCLING_FORCE_OCR", True)
+    )
+    return {
+        "DOCLING_DO_OCR": str(env_bool("DOCLING_DO_OCR", True)).lower(),
+        "DOCLING_FORCE_OCR": str(force_ocr).lower(),
+        "DOCLING_OCR_ENGINE": os.getenv("DOCLING_OCR_ENGINE", "auto").strip() or "auto",
+        "DOCLING_OCR_PRESET": os.getenv("DOCLING_OCR_PRESET", "auto").strip() or "auto",
+        "DOCLING_OCR_LANG": os.getenv("DOCLING_OCR_LANG", "").strip(),
+        "DOCLING_DO_FORMULA_ENRICHMENT": str(
+            env_bool("DOCLING_DO_FORMULA_ENRICHMENT", False)
+        ).lower(),
+    }
+
+
+def is_bundle_valid(
+    raw_dir: Path,
+    source_file: Path,
+    *,
+    overrides: "Mapping[str, Any] | None" = None,
+) -> bool:
+    """Return True iff the bundle matches the current source + env state."""
+    if not raw_dir.is_dir():
+        return False
+
+    manifest = load_manifest(raw_dir, expected_engine=MANIFEST_ENGINE)
+    if manifest is None:
+        return False
+
+    # 1. Source size fast-path
+    try:
+        cur_size = source_file.stat().st_size
+    except OSError:
+        return False
+    if cur_size != int(manifest.source_size_bytes):
+        return False
+
+    # 2. Source content_hash
+    _, cur_hash = compute_size_and_hash(source_file)
+    if cur_hash != manifest.source_content_hash:
+        return False
+
+    # 3. Engine version. Skip the comparison when either side is empty so
+    #    operators can opt out by unsetting the env, and so bundles from
+    #    earlier code that never recorded the field aren't force-invalidated.
+    cur_engine_version = os.getenv("DOCLING_ENGINE_VERSION", "").strip()
+    if (
+        cur_engine_version
+        and manifest.engine_version
+        and cur_engine_version != manifest.engine_version
+    ):
+        return False
+
+    # 4. Endpoint signature. Same "both non-empty to compare" rule: a bundle
+    #    parsed against a different docling-serve URL must not be reused, but
+    #    we don't reject the cache just because the env happens to be unset
+    #    at validation time (e.g. CLI tooling that only reads the cache).
+    cur_endpoint = current_endpoint_signature()
+    if (
+        cur_endpoint
+        and manifest.endpoint_signature
+        and cur_endpoint != manifest.endpoint_signature
+    ):
+        return False
+
+    # 5. Options signature: only enforced if the manifest recorded one
+    #    (manifests written before this commit have it empty — they are
+    #    treated as stale and re-downloaded the next time the env changes).
+    #
+    #    Compare against the *current* fixed constants from client.py, not
+    #    the copy stashed in the manifest — using the manifest's copy would
+    #    always reproduce the recorded signature and silently swallow
+    #    code-only changes (e.g. flipping image_export_mode or to_formats),
+    #    defeating the invalidation this step is supposed to provide.
+    #    Lazy import: client.py imports from cache.py.
+    #
+    #    When per-file overrides are requested (e.g. docling(force_ocr=true)) but
+    #    the manifest predates signature recording, we cannot prove the bundle was
+    #    produced with those overrides — accepting it would silently drop the
+    #    user's explicit param. Treat that as a miss so the override is honored
+    #    (mirrors MinerU, which misses on any absent signature). The no-override
+    #    case keeps the deliberate leniency above for legacy bundles.
+    if overrides and not manifest.options_signature:
+        return False
+    if manifest.options_signature:
+        from lightrag.parser.external.docling.client import FIXED_CONSTANTS
+
+        cur_options = compute_options_signature(
+            tunable_env=snapshot_tunable_env(overrides),
+            fixed_constants=FIXED_CONSTANTS,
+        )
+        if cur_options != manifest.options_signature:
+            return False
+
+    # 6. Critical file: size + sha256
+    crit = manifest.critical_file
+    crit_path = raw_dir / crit.path
+    try:
+        if crit_path.stat().st_size != int(crit.size):
+            return False
+    except OSError:
+        return False
+    if crit.sha256:
+        _, crit_actual = compute_size_and_hash(crit_path)
+        if crit_actual != crit.sha256:
+            return False
+
+    # 7. Other files: size only
+    for entry in manifest.files:
+        ep = raw_dir / entry.path
+        try:
+            if ep.stat().st_size != int(entry.size):
+                return False
+        except OSError:
+            return False
+
+    return True
+
+
+__all__ = [
+    "DOCLING_TUNABLE_ENVS",
+    "compute_options_signature",
+    "current_endpoint_signature",
+    "is_bundle_valid",
+    "snapshot_tunable_env",
+]

lightrag/parser/external/docling/client.py

@@ -0,0 +1,353 @@
+"""Docling raw bundle downloader.
+
+Talks to Docling Serve v1 over HTTP:
+
+- ``POST /v1/convert/file/async`` — multipart upload, returns ``task_id``,
+- ``GET  /v1/status/poll/{task_id}?wait=5`` — long-poll for terminal state,
+- ``GET  /v1/result/{task_id}`` — zip download (only on ``success``).
+
+The zip is extracted safely under ``raw_dir/`` (refusing path traversal /
+absolute entries). A success manifest is written atomically at the very
+end; mid-run crashes therefore leave the directory in a state the cache
+layer marks as invalid (no manifest → miss → re-download).
+
+Pipeline constants (``pipeline``, ``target_type``, ``to_formats``,
+``image_export_mode``) are intentionally **not** env-driven — the sidecar
+flow depends on them — and are recorded inside the manifest so a future
+code change automatically invalidates pre-existing caches.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import json
+import os
+import time
+from collections.abc import Mapping
+from pathlib import Path
+from typing import TYPE_CHECKING, Any
+from urllib.parse import quote
+
+from lightrag.parser.external._common import (
+    env_bool,
+    env_int,
+    raise_for_status_with_detail,
+)
+from lightrag.parser.external._zip import safe_extract_zip
+from lightrag.parser.external.docling.cache import (
+    compute_options_signature,
+    current_endpoint_signature,
+    snapshot_tunable_env,
+)
+from lightrag.parser.external.docling.manifest import (
+    build_and_write_docling_manifest,
+    select_main_json,
+)
+from lightrag.utils import logger
+
+if TYPE_CHECKING:
+    import httpx
+else:
+    try:
+        import httpx
+    except ImportError:  # pragma: no cover
+        httpx = None
+
+# ---------------------------------------------------------------------------
+# Fixed pipeline constants (NOT env-driven)
+# ---------------------------------------------------------------------------
+
+PIPELINE = "standard"
+TARGET_TYPE = "zip"
+TO_FORMATS: tuple[str, ...] = ("json", "md")
+IMAGE_EXPORT_MODE = "referenced"
+
+FIXED_CONSTANTS: dict[str, object] = {
+    "pipeline": PIPELINE,
+    "target_type": TARGET_TYPE,
+    "to_formats": list(TO_FORMATS),
+    "image_export_mode": IMAGE_EXPORT_MODE,
+}
+
+CONVERT_PATH = "/v1/convert/file/async"
+POLL_PATH = "/v1/status/poll/{task_id}"
+RESULT_PATH = "/v1/result/{task_id}"
+
+DEFAULT_POLL_WAIT_SECONDS = 5
+DEFAULT_MAX_POLLS = 240  # 240 * 5s long-poll ≈ 20 min worst case
+
+# ConversionStatus enum from the docling-serve OpenAPI
+SUCCESS_STATES = {"success"}
+FAILURE_STATES = {"failure", "partial_success", "skipped"}
+IN_PROGRESS_STATES = {"pending", "started"}
+
+
+class DoclingRawClient:
+    """Downloads docling-serve bundles into ``raw_dir``.
+
+    Construct once per parse call (cheap). Reads ``DOCLING_*`` envs at
+    ``__init__`` time, so callers can flip env between calls and pick up
+    the new values without holding a stale instance.
+    """
+
+    def __init__(self, *, overrides: "Mapping[str, Any] | None" = None) -> None:
+        self._overrides = overrides or {}
+        self.endpoint = current_endpoint_signature()
+        if not self.endpoint:
+            raise ValueError("DOCLING_ENDPOINT is required")
+        self.engine_version = os.getenv("DOCLING_ENGINE_VERSION", "").strip()
+
+        self.do_ocr = env_bool("DOCLING_DO_OCR", True)
+        self.force_ocr = (
+            bool(self._overrides["force_ocr"])
+            if "force_ocr" in self._overrides
+            else env_bool("DOCLING_FORCE_OCR", True)
+        )
+        self.ocr_engine = os.getenv("DOCLING_OCR_ENGINE", "auto").strip() or "auto"
+        self.ocr_preset = os.getenv("DOCLING_OCR_PRESET", "auto").strip() or "auto"
+        self.ocr_lang_raw = os.getenv("DOCLING_OCR_LANG", "").strip()
+        self.do_formula_enrichment = env_bool("DOCLING_DO_FORMULA_ENRICHMENT", False)
+
+        # Poll cadence: docling-serve's ``?wait=N`` is a server-side long-poll
+        # window. ``DOCLING_POLL_INTERVAL_SECONDS`` sets that window; the
+        # client does NOT add its own sleep between polls. ``DOCLING_MAX_POLLS``
+        # bounds the total polling budget — exceeding it raises ``TimeoutError``.
+        wait = env_int("DOCLING_POLL_INTERVAL_SECONDS", DEFAULT_POLL_WAIT_SECONDS)
+        self.poll_wait_seconds = wait if wait > 0 else DEFAULT_POLL_WAIT_SECONDS
+        max_polls = env_int("DOCLING_MAX_POLLS", DEFAULT_MAX_POLLS)
+        self.max_poll_attempts = max_polls if max_polls > 0 else DEFAULT_MAX_POLLS
+
+    # ------------------------------------------------------------------
+    # Public API
+    # ------------------------------------------------------------------
+
+    async def download_into(
+        self,
+        raw_dir: Path,
+        source_file_path: Path,
+        *,
+        upload_filename: str | None = None,
+    ):
+        """Upload, poll, download, extract, and write the manifest.
+
+        ``upload_filename`` overrides the multipart filename sent to
+        docling-serve (defaults to ``source_file_path.name``). The pipeline
+        passes the canonical, hint-stripped document name here so the
+        bundle's ``<stem>.json`` ends up canonical too — otherwise a file
+        named ``report.[docling].pdf`` would produce ``report.[docling].json``
+        inside the bundle, and the adapter (which only knows the canonical
+        ``report.pdf``) would not be able to locate it via the preferred
+        ``<stem>.json`` lookup.
+
+        Pre-condition: caller cleared ``raw_dir`` (e.g. via
+        :func:`lightrag.parser.external.clear_dir_contents`). This method
+        does not clean the directory itself — keeping that explicit at the
+        ``parse_docling`` entry point.
+        """
+        if httpx is None:
+            raise RuntimeError(
+                "httpx is required for Docling parsing but is not installed"
+            )
+        raw_dir.mkdir(parents=True, exist_ok=True)
+
+        effective_filename = upload_filename or source_file_path.name
+
+        timeout = httpx.Timeout(120.0, connect=30.0)
+        async with httpx.AsyncClient(timeout=timeout) as client:
+            task_id = await self._submit(
+                client, source_file_path, filename=effective_filename
+            )
+            await self._poll_until_done(client, task_id)
+            payload = await self._download_zip_bytes(client, task_id)
+
+        safe_extract_zip(payload, raw_dir)
+        # Defensive: confirm the main JSON exists before anyone reads the
+        # bundle. Look it up by the *uploaded* filename's stem — that's
+        # what docling-serve uses to name the JSON inside the zip.
+        select_main_json(raw_dir, Path(effective_filename))
+
+        options_signature = compute_options_signature(
+            tunable_env=snapshot_tunable_env(self._overrides),
+            fixed_constants=FIXED_CONSTANTS,
+        )
+        return build_and_write_docling_manifest(
+            raw_dir,
+            source_file_path=source_file_path,
+            task_id=task_id,
+            endpoint_signature=self.endpoint,
+            engine_version=self.engine_version,
+            options_signature=options_signature,
+            fixed_constants=FIXED_CONSTANTS,
+            recorded_filename=effective_filename,
+        )
+
+    # ------------------------------------------------------------------
+    # Upload + poll + download
+    # ------------------------------------------------------------------
+
+    def _build_multipart_data(self) -> dict[str, str | list[str]]:
+        """Form fields (everything except the file payload).
+
+        Returns a ``dict`` (not a list of tuples): httpx ≥ 0.28 short-circuits
+        non-``Mapping`` ``data`` into raw-content encoding and ignores
+        ``files=`` entirely, producing a sync-only stream that an
+        ``AsyncClient`` then rejects. List-valued entries are emitted as
+        repeated form keys by ``MultipartStream``, matching docling-serve's
+        pydantic ``List[Enum]`` form parsing. ``ocr_lang`` is omitted entirely
+        when empty so the engine uses its own default.
+        """
+        data: dict[str, str | list[str]] = {
+            "pipeline": PIPELINE,
+            "target_type": TARGET_TYPE,
+            "image_export_mode": IMAGE_EXPORT_MODE,
+            "do_ocr": _bool_form(self.do_ocr),
+            "force_ocr": _bool_form(self.force_ocr),
+            "ocr_engine": self.ocr_engine,
+            "ocr_preset": self.ocr_preset,
+            "do_formula_enrichment": _bool_form(self.do_formula_enrichment),
+            "to_formats": list(TO_FORMATS),
+        }
+        if self.ocr_lang_raw:
+            langs = _parse_ocr_lang(self.ocr_lang_raw)
+            if langs:
+                data["ocr_lang"] = langs
+        return data
+
+    async def _submit(
+        self,
+        client: "httpx.AsyncClient",
+        source_file_path: Path,
+        *,
+        filename: str,
+    ) -> str:
+        url = f"{self.endpoint}{CONVERT_PATH}"
+        # Hand httpx a file object so its MultipartStream reads the body in
+        # chunks instead of materializing the whole PDF/PPTX in worker memory.
+        # With ``max_parallel_parse_docling > 1`` a per-doc bytes copy can
+        # OOM the worker before docling-serve ever sees the request.
+        with source_file_path.open("rb") as fh:
+            files = {"files": (filename, fh, "application/octet-stream")}
+            resp = await client.post(
+                url, data=self._build_multipart_data(), files=files
+            )
+        raise_for_status_with_detail(resp, f"Docling upload for {filename!r}")
+        payload = resp.json() if resp.text else {}
+        task_id = str(payload.get("task_id") or payload.get("id") or "").strip()
+        if not task_id:
+            raise RuntimeError(f"Docling upload response missing task_id: {payload!r}")
+        return task_id
+
+    async def _poll_until_done(
+        self,
+        client: "httpx.AsyncClient",
+        task_id: str,
+    ) -> None:
+        encoded_task_id = quote(task_id, safe="")
+        url = f"{self.endpoint}{POLL_PATH.format(task_id=encoded_task_id)}"
+        params = {"wait": self.poll_wait_seconds}
+        for _ in range(self.max_poll_attempts):
+            iteration_started = time.monotonic()
+            resp = await client.get(url, params=params)
+            raise_for_status_with_detail(resp, f"Docling task {task_id} poll")
+            payload = resp.json() if resp.text else {}
+            status = str(
+                payload.get("task_status") or payload.get("status") or ""
+            ).lower()
+
+            if status in SUCCESS_STATES:
+                return
+            if status in FAILURE_STATES:
+                raise RuntimeError(_format_failure(task_id, status, payload))
+            if status not in IN_PROGRESS_STATES:
+                # Unknown status: keep polling, but surface it so operators notice.
+                logger.warning(
+                    "[docling] unknown task status %r for task %s; continuing to poll",
+                    status,
+                    task_id,
+                )
+
+            # The intended cadence is one poll per ``poll_wait_seconds`` — the
+            # design relies on docling-serve's ``?wait=N`` long-polling for
+            # that. Some deployments return immediately instead, which would
+            # burn through ``max_poll_attempts`` in milliseconds and fail
+            # with a spurious timeout. Cap each iteration at the configured
+            # interval ourselves so the total budget holds either way.
+            elapsed = time.monotonic() - iteration_started
+            remaining = self.poll_wait_seconds - elapsed
+            if remaining > 0:
+                await asyncio.sleep(remaining)
+
+        raise TimeoutError(f"Docling task {task_id} polling timeout")
+
+    async def _download_zip_bytes(
+        self,
+        client: "httpx.AsyncClient",
+        task_id: str,
+    ) -> bytes:
+        encoded_task_id = quote(task_id, safe="")
+        url = f"{self.endpoint}{RESULT_PATH.format(task_id=encoded_task_id)}"
+        resp = await client.get(url)
+        raise_for_status_with_detail(resp, f"Docling result {task_id} download")
+        ctype = resp.headers.get("content-type", "")
+        if "zip" not in ctype.lower():
+            raise RuntimeError(
+                f"Docling result {task_id} returned non-zip content-type "
+                f"{ctype!r}; body prefix={resp.text[:400]!r}"
+            )
+        return resp.content
+
+
+# ---------------------------------------------------------------------------
+# Helpers
+# ---------------------------------------------------------------------------
+
+
+def _bool_form(v: bool) -> str:
+    return "true" if v else "false"
+
+
+def _parse_ocr_lang(raw: str) -> list[str]:
+    """Best-effort parser for ``DOCLING_OCR_LANG``.
+
+    Accepts a JSON array (``["en","zh"]``) or a comma-separated list
+    (``en,zh``). Returns a list of stripped non-empty strings; empty in →
+    empty out.
+    """
+    try:
+        parsed = json.loads(raw)
+    except json.JSONDecodeError:
+        parsed = None
+    if isinstance(parsed, list):
+        return [str(x).strip() for x in parsed if str(x).strip()]
+    return [item.strip() for item in raw.split(",") if item.strip()]
+
+
+def _format_failure(task_id: str, status: str, payload: Any) -> str:
+    if isinstance(payload, dict):
+        err = (
+            payload.get("error_message")
+            or payload.get("error")
+            or payload.get("message")
+            or "<no error_message>"
+        )
+    else:
+        err = "<no error_message>"
+    truncated = json.dumps(payload, ensure_ascii=False)[:400]
+    return f"Docling task {task_id} ended in {status}: {err}; payload={truncated}"
+
+
+__all__ = [
+    "DoclingRawClient",
+    "CONVERT_PATH",
+    "DEFAULT_MAX_POLLS",
+    "DEFAULT_POLL_WAIT_SECONDS",
+    "FIXED_CONSTANTS",
+    "IMAGE_EXPORT_MODE",
+    "PIPELINE",
+    "POLL_PATH",
+    "RESULT_PATH",
+    "SUCCESS_STATES",
+    "FAILURE_STATES",
+    "TARGET_TYPE",
+    "TO_FORMATS",
+]

lightrag/parser/external/docling/manifest.py

@@ -0,0 +1,130 @@
+"""Helpers for building ``_manifest.json`` for docling raw bundles.
+
+Wraps the generic :class:`Manifest` schema with docling-specific knowledge:
+
+- the critical file is the main ``<stem>.json`` produced by docling-serve,
+- non-critical files are the markdown + every entry under ``artifacts/``,
+- ``extras`` carries the fixed pipeline constants so the options signature
+  remains reproducible across runs.
+"""
+
+from __future__ import annotations
+
+from datetime import datetime, timezone
+from pathlib import Path
+
+from lightrag.parser.external._common import compute_size_and_hash
+from lightrag.parser.external._manifest import (
+    MANIFEST_FILENAME,
+    Manifest,
+    ManifestFile,
+    write_manifest,
+)
+from lightrag.parser.external.docling import MANIFEST_ENGINE
+
+
+def select_main_json(raw_dir: Path, source_file_path: Path) -> Path:
+    """Locate the primary docling JSON inside ``raw_dir``.
+
+    Priority: ``<source_stem>.json`` if present, else the single ``*.json``
+    sitting at ``raw_dir`` root (excluding ``_manifest.json``, which always
+    sits in the bundle once a download has completed and would otherwise
+    collide with the bundle JSON in the fallback). Raises ``RuntimeError``
+    if zero or multiple candidates exist.
+    """
+    preferred = raw_dir / f"{source_file_path.stem}.json"
+    if preferred.is_file():
+        return preferred
+
+    candidates = sorted(
+        p for p in raw_dir.glob("*.json") if p.is_file() and p.name != MANIFEST_FILENAME
+    )
+    if len(candidates) == 1:
+        return candidates[0]
+    if not candidates:
+        raise RuntimeError(f"Docling raw bundle at {raw_dir} contains no .json file")
+    names = ", ".join(p.name for p in candidates)
+    raise RuntimeError(
+        f"Docling raw bundle at {raw_dir} has multiple .json candidates ({names}); "
+        f"expected exactly one to derive the critical file from"
+    )
+
+
+def select_main_md(raw_dir: Path, source_file_path: Path) -> Path | None:
+    """Locate the markdown twin of the main JSON. Returns ``None`` if no
+    markdown was produced (defensive — docling-serve always emits one for
+    ``to_formats=["json","md"]`` but we don't want to crash if it is
+    missing)."""
+    preferred = raw_dir / f"{source_file_path.stem}.md"
+    if preferred.is_file():
+        return preferred
+    candidates = sorted(p for p in raw_dir.glob("*.md") if p.is_file())
+    return candidates[0] if candidates else None
+
+
+def build_and_write_docling_manifest(
+    raw_dir: Path,
+    *,
+    source_file_path: Path,
+    task_id: str,
+    endpoint_signature: str,
+    engine_version: str,
+    options_signature: str,
+    fixed_constants: dict[str, object],
+    recorded_filename: str | None = None,
+) -> Manifest:
+    """Construct the manifest for a freshly downloaded docling bundle and
+    persist it atomically. Returns the in-memory manifest for callers that
+    need the task_id / signatures for logging.
+
+    ``recorded_filename`` is the name passed to docling-serve at upload
+    time (canonical, hint-stripped form when called from the pipeline).
+    It governs both the preferred-path lookup for the bundle JSON and the
+    value persisted as ``source_filename_at_parse``. When ``None``, falls
+    back to ``source_file_path.name`` for backward compatibility.
+    """
+    lookup_path = Path(recorded_filename) if recorded_filename else source_file_path
+    main_json = select_main_json(raw_dir, lookup_path)
+    crit_size, crit_hash = compute_size_and_hash(main_json)
+    critical = ManifestFile(
+        path=main_json.relative_to(raw_dir).as_posix(),
+        size=crit_size,
+        sha256=crit_hash,
+    )
+
+    others: list[ManifestFile] = []
+    for path in sorted(raw_dir.rglob("*")):
+        if not path.is_file():
+            continue
+        rel = path.relative_to(raw_dir).as_posix()
+        if rel == critical.path or rel.startswith("_manifest"):
+            continue
+        others.append(ManifestFile(path=rel, size=path.stat().st_size))
+
+    source_size, source_hash = compute_size_and_hash(source_file_path)
+    total = crit_size + sum(f.size for f in others)
+
+    manifest = Manifest(
+        engine=MANIFEST_ENGINE,
+        source_content_hash=source_hash,
+        source_size_bytes=source_size,
+        source_filename_at_parse=recorded_filename or source_file_path.name,
+        critical_file=critical,
+        files=others,
+        total_size_bytes=total,
+        task_id=task_id,
+        endpoint_signature=endpoint_signature,
+        engine_version=engine_version,
+        options_signature=options_signature,
+        downloaded_at=datetime.now(timezone.utc).isoformat(timespec="seconds"),
+        extras={"fixed_constants": dict(fixed_constants)},
+    )
+    write_manifest(raw_dir, manifest)
+    return manifest
+
+
+__all__ = [
+    "build_and_write_docling_manifest",
+    "select_main_json",
+    "select_main_md",
+]

lightrag/parser/external/docling/parser.py

@@ -0,0 +1,61 @@
+"""Docling engine adapter (implements ExternalParserBase hooks)."""
+
+from __future__ import annotations
+
+from collections.abc import Mapping
+from pathlib import Path
+from typing import TYPE_CHECKING, Any
+
+from lightrag.constants import DOCLING_RAW_DIR_SUFFIX, PARSER_ENGINE_DOCLING
+from lightrag.parser.external._base import ExternalParserBase
+
+if TYPE_CHECKING:
+    from lightrag.sidecar.ir import IRDoc
+
+
+class DoclingParser(ExternalParserBase):
+    engine_name = PARSER_ENGINE_DOCLING
+    raw_dir_suffix = DOCLING_RAW_DIR_SUFFIX
+    force_reparse_env = "LIGHTRAG_FORCE_REPARSE_DOCLING"
+
+    def is_bundle_valid(
+        self,
+        raw_dir: Path,
+        source_path: Path,
+        *,
+        engine_params: "Mapping[str, Any] | None" = None,
+    ) -> bool:
+        from lightrag.parser.external.docling import is_bundle_valid
+
+        return is_bundle_valid(raw_dir, source_path, overrides=engine_params)
+
+    async def download_into(
+        self,
+        raw_dir: Path,
+        source_path: Path,
+        *,
+        upload_name: str,
+        engine_params: "Mapping[str, Any] | None" = None,
+    ) -> None:
+        from lightrag.parser.external.docling import DoclingRawClient
+
+        # Map the canonical ``upload_name`` onto docling-serve's multipart
+        # filename so the bundle's main JSON is named ``<canonical_stem>.json``
+        # (the IR builder locates it via that canonical stem).
+        await DoclingRawClient(overrides=engine_params).download_into(
+            raw_dir, source_path, upload_filename=upload_name
+        )
+
+    def build_ir(self, raw_dir: Path, document_name: str) -> "IRDoc":
+        from lightrag.parser.external.docling import DoclingIRBuilder
+
+        return DoclingIRBuilder().normalize_from_workdir(
+            raw_dir, document_name=document_name
+        )
+
+    def validate_ir(self, ir: "IRDoc", *, file_path: str, raw_dir: Path) -> None:
+        if not ir.blocks:
+            raise ValueError(
+                f"Docling IR builder produced zero blocks for {file_path} "
+                f"(raw_dir={raw_dir})"
+            )