chore: import upstream snapshot with attribution
This commit is contained in:
wehub-resource-sync
+257
@@ -0,0 +1,257 @@
|
||||
"""Cache validation for ``*.docling_raw/`` bundles.
|
||||
|
||||
Validation policy (settled in
|
||||
``docs/DoclingSidecarRefactorPlan-zh.md`` §4.1):
|
||||
|
||||
1. ``_manifest.json`` exists, parses, ``engine="docling"`` ∧ schema version
|
||||
matches.
|
||||
2. **Source size fast-path**: ``source_file.stat().st_size`` matches the
|
||||
manifest; mismatch → miss without hashing.
|
||||
3. **Source content_hash**: full sha256 of the current source file matches
|
||||
the manifest.
|
||||
4. **Engine version**: if ``DOCLING_ENGINE_VERSION`` is set in env and the
|
||||
manifest recorded a non-empty value, they must match.
|
||||
5. **Endpoint signature**: if the active ``DOCLING_ENDPOINT`` differs from
|
||||
what was recorded at parse time, miss (avoids re-using a bundle produced
|
||||
by a different docling-serve instance).
|
||||
6. **Options signature**: covers every env or fixed constant that changes
|
||||
the produced bundle (OCR flags, language list, formula enrichment,
|
||||
target format and pipeline). Any change → miss.
|
||||
7. **Critical file**: the main JSON must exist with matching size **and**
|
||||
sha256 — final tie-breaker against silent corruption affecting the file
|
||||
the adapter depends on.
|
||||
8. **Other files**: size-only verification (cheap; covers most corruption
|
||||
modes for markdown / artifacts).
|
||||
|
||||
Any failed step ⇒ cache miss; the caller wipes the directory contents and
|
||||
re-runs the download.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
from collections.abc import Mapping
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from lightrag.parser.external._common import compute_size_and_hash, env_bool
|
||||
from lightrag.parser.external._manifest import load_manifest
|
||||
from lightrag.parser.external.docling import MANIFEST_ENGINE
|
||||
from lightrag.utils import logger
|
||||
|
||||
# Legacy upload-path suffix. ``env.example`` historically documented
|
||||
# ``DOCLING_ENDPOINT=http://host:5001/v1/convert/file/async`` (the full
|
||||
# upload URL); the current client expects a base URL and appends the path
|
||||
# itself. Strip the suffix so an unmodified pre-refactor ``.env`` keeps
|
||||
# working instead of producing
|
||||
# ``/v1/convert/file/async/v1/convert/file/async`` requests.
|
||||
_LEGACY_UPLOAD_PATH_SUFFIX = "/v1/convert/file/async"
|
||||
_legacy_endpoint_warned = False
|
||||
|
||||
# Envs that change the bytes docling-serve produces. Any change here must
|
||||
# invalidate the bundle cache. ``DOCLING_BBOX_ATTRIBUTES`` is intentionally
|
||||
# NOT in this list: it only affects how the adapter writes IR meta, not the
|
||||
# docling bundle, so flipping it should re-emit the sidecar (which we always
|
||||
# do) without forcing a re-download.
|
||||
DOCLING_TUNABLE_ENVS: tuple[str, ...] = (
|
||||
"DOCLING_DO_OCR",
|
||||
"DOCLING_FORCE_OCR",
|
||||
"DOCLING_OCR_ENGINE",
|
||||
"DOCLING_OCR_PRESET",
|
||||
"DOCLING_OCR_LANG",
|
||||
"DOCLING_DO_FORMULA_ENRICHMENT",
|
||||
)
|
||||
|
||||
|
||||
def current_endpoint_signature() -> str:
|
||||
"""The active docling endpoint, normalized to a base URL.
|
||||
|
||||
Normalization:
|
||||
|
||||
- Trims surrounding whitespace and strips trailing slashes.
|
||||
- Strips the legacy ``/v1/convert/file/async`` upload suffix if present,
|
||||
preserving backwards compatibility with the pre-refactor ``env.example``
|
||||
that documented the full upload URL.
|
||||
|
||||
Returns ``""`` if ``DOCLING_ENDPOINT`` is unset — callers that need a
|
||||
real endpoint (``DoclingRawClient``) raise on empty; callers that only
|
||||
compare against a recorded manifest field (``is_bundle_valid``) silently
|
||||
skip the check when either side is empty.
|
||||
"""
|
||||
global _legacy_endpoint_warned
|
||||
endpoint = os.getenv("DOCLING_ENDPOINT", "").strip().rstrip("/")
|
||||
if endpoint.endswith(_LEGACY_UPLOAD_PATH_SUFFIX):
|
||||
endpoint = endpoint[: -len(_LEGACY_UPLOAD_PATH_SUFFIX)]
|
||||
if not _legacy_endpoint_warned:
|
||||
_legacy_endpoint_warned = True
|
||||
logger.warning(
|
||||
"DOCLING_ENDPOINT still includes the legacy %r upload suffix; "
|
||||
"stripping it. Update your .env to a base URL "
|
||||
"(e.g. http://host:5001).",
|
||||
_LEGACY_UPLOAD_PATH_SUFFIX,
|
||||
)
|
||||
return endpoint
|
||||
|
||||
|
||||
def compute_options_signature(
|
||||
*,
|
||||
tunable_env: dict[str, str],
|
||||
fixed_constants: dict[str, object],
|
||||
) -> str:
|
||||
"""Stable signature over user-tunable env values and fixed pipeline
|
||||
constants.
|
||||
|
||||
Storing the constants in the signature means a future code change that
|
||||
flips e.g. ``image_export_mode`` from ``referenced`` to ``embedded``
|
||||
invalidates every existing cache without anyone having to remember to
|
||||
bump a version.
|
||||
"""
|
||||
payload = json.dumps(
|
||||
{"env": tunable_env, "fixed": fixed_constants},
|
||||
ensure_ascii=False,
|
||||
sort_keys=True,
|
||||
separators=(",", ":"),
|
||||
)
|
||||
return "sha256:" + hashlib.sha256(payload.encode("utf-8")).hexdigest()
|
||||
|
||||
|
||||
def snapshot_tunable_env(
|
||||
overrides: "Mapping[str, Any] | None" = None,
|
||||
) -> dict[str, str]:
|
||||
"""Read effective docling tunables so equivalent requests share a signature.
|
||||
|
||||
``overrides`` carries decoded per-file engine params (Phase 2: ``force_ocr``)
|
||||
and replaces the corresponding env value so the override feeds BOTH the live
|
||||
request and the cache signature.
|
||||
"""
|
||||
overrides = overrides or {}
|
||||
force_ocr = (
|
||||
bool(overrides["force_ocr"])
|
||||
if "force_ocr" in overrides
|
||||
else env_bool("DOCLING_FORCE_OCR", True)
|
||||
)
|
||||
return {
|
||||
"DOCLING_DO_OCR": str(env_bool("DOCLING_DO_OCR", True)).lower(),
|
||||
"DOCLING_FORCE_OCR": str(force_ocr).lower(),
|
||||
"DOCLING_OCR_ENGINE": os.getenv("DOCLING_OCR_ENGINE", "auto").strip() or "auto",
|
||||
"DOCLING_OCR_PRESET": os.getenv("DOCLING_OCR_PRESET", "auto").strip() or "auto",
|
||||
"DOCLING_OCR_LANG": os.getenv("DOCLING_OCR_LANG", "").strip(),
|
||||
"DOCLING_DO_FORMULA_ENRICHMENT": str(
|
||||
env_bool("DOCLING_DO_FORMULA_ENRICHMENT", False)
|
||||
).lower(),
|
||||
}
|
||||
|
||||
|
||||
def is_bundle_valid(
|
||||
raw_dir: Path,
|
||||
source_file: Path,
|
||||
*,
|
||||
overrides: "Mapping[str, Any] | None" = None,
|
||||
) -> bool:
|
||||
"""Return True iff the bundle matches the current source + env state."""
|
||||
if not raw_dir.is_dir():
|
||||
return False
|
||||
|
||||
manifest = load_manifest(raw_dir, expected_engine=MANIFEST_ENGINE)
|
||||
if manifest is None:
|
||||
return False
|
||||
|
||||
# 1. Source size fast-path
|
||||
try:
|
||||
cur_size = source_file.stat().st_size
|
||||
except OSError:
|
||||
return False
|
||||
if cur_size != int(manifest.source_size_bytes):
|
||||
return False
|
||||
|
||||
# 2. Source content_hash
|
||||
_, cur_hash = compute_size_and_hash(source_file)
|
||||
if cur_hash != manifest.source_content_hash:
|
||||
return False
|
||||
|
||||
# 3. Engine version. Skip the comparison when either side is empty so
|
||||
# operators can opt out by unsetting the env, and so bundles from
|
||||
# earlier code that never recorded the field aren't force-invalidated.
|
||||
cur_engine_version = os.getenv("DOCLING_ENGINE_VERSION", "").strip()
|
||||
if (
|
||||
cur_engine_version
|
||||
and manifest.engine_version
|
||||
and cur_engine_version != manifest.engine_version
|
||||
):
|
||||
return False
|
||||
|
||||
# 4. Endpoint signature. Same "both non-empty to compare" rule: a bundle
|
||||
# parsed against a different docling-serve URL must not be reused, but
|
||||
# we don't reject the cache just because the env happens to be unset
|
||||
# at validation time (e.g. CLI tooling that only reads the cache).
|
||||
cur_endpoint = current_endpoint_signature()
|
||||
if (
|
||||
cur_endpoint
|
||||
and manifest.endpoint_signature
|
||||
and cur_endpoint != manifest.endpoint_signature
|
||||
):
|
||||
return False
|
||||
|
||||
# 5. Options signature: only enforced if the manifest recorded one
|
||||
# (manifests written before this commit have it empty — they are
|
||||
# treated as stale and re-downloaded the next time the env changes).
|
||||
#
|
||||
# Compare against the *current* fixed constants from client.py, not
|
||||
# the copy stashed in the manifest — using the manifest's copy would
|
||||
# always reproduce the recorded signature and silently swallow
|
||||
# code-only changes (e.g. flipping image_export_mode or to_formats),
|
||||
# defeating the invalidation this step is supposed to provide.
|
||||
# Lazy import: client.py imports from cache.py.
|
||||
#
|
||||
# When per-file overrides are requested (e.g. docling(force_ocr=true)) but
|
||||
# the manifest predates signature recording, we cannot prove the bundle was
|
||||
# produced with those overrides — accepting it would silently drop the
|
||||
# user's explicit param. Treat that as a miss so the override is honored
|
||||
# (mirrors MinerU, which misses on any absent signature). The no-override
|
||||
# case keeps the deliberate leniency above for legacy bundles.
|
||||
if overrides and not manifest.options_signature:
|
||||
return False
|
||||
if manifest.options_signature:
|
||||
from lightrag.parser.external.docling.client import FIXED_CONSTANTS
|
||||
|
||||
cur_options = compute_options_signature(
|
||||
tunable_env=snapshot_tunable_env(overrides),
|
||||
fixed_constants=FIXED_CONSTANTS,
|
||||
)
|
||||
if cur_options != manifest.options_signature:
|
||||
return False
|
||||
|
||||
# 6. Critical file: size + sha256
|
||||
crit = manifest.critical_file
|
||||
crit_path = raw_dir / crit.path
|
||||
try:
|
||||
if crit_path.stat().st_size != int(crit.size):
|
||||
return False
|
||||
except OSError:
|
||||
return False
|
||||
if crit.sha256:
|
||||
_, crit_actual = compute_size_and_hash(crit_path)
|
||||
if crit_actual != crit.sha256:
|
||||
return False
|
||||
|
||||
# 7. Other files: size only
|
||||
for entry in manifest.files:
|
||||
ep = raw_dir / entry.path
|
||||
try:
|
||||
if ep.stat().st_size != int(entry.size):
|
||||
return False
|
||||
except OSError:
|
||||
return False
|
||||
|
||||
return True
|
||||
|
||||
|
||||
__all__ = [
|
||||
"DOCLING_TUNABLE_ENVS",
|
||||
"compute_options_signature",
|
||||
"current_endpoint_signature",
|
||||
"is_bundle_valid",
|
||||
"snapshot_tunable_env",
|
||||
]
|
||||
Reference in New Issue
Block a user