chore: import upstream snapshot with attribution

python/opendataloader-pdf/.gitignore

@@ -0,0 +1,13 @@
+# Build artifacts
+dist/
+build/
+*.egg-info/
+
+# Ignore JAR file
+*.jar
+
+# Docs
+README.md
+LICENSE
+NOTICE
+THIRD_PARTY/

python/opendataloader-pdf/hatch_build.py

@@ -0,0 +1,69 @@
+"""Custom build hook for hatch to copy JAR and license files."""
+
+import glob
+import shutil
+from pathlib import Path
+
+from hatchling.builders.hooks.plugin.interface import BuildHookInterface
+
+
+class CustomBuildHook(BuildHookInterface):
+    def initialize(self, version, build_data):
+        root_dir = Path(self.root)
+        pkg_dir = root_dir / "src/opendataloader_pdf"
+        dest_jar_dir = pkg_dir / "jar"
+        dest_jar_path = dest_jar_dir / "opendataloader-pdf-cli.jar"
+        license_path = pkg_dir / "LICENSE"
+        notice_path = pkg_dir / "NOTICE"
+        third_party_dest = pkg_dir / "THIRD_PARTY"
+
+        readme_path = root_dir / "README.md"
+
+        # sdist-install code path: when users `pip install <sdist>.tar.gz`,
+        # the extracted sdist already contains JAR/LICENSE/NOTICE/THIRD_PARTY
+        # (force-included via [tool.hatch.build] artifacts in pyproject.toml),
+        # and there is no java/ tree to rebuild from. Do not remove — sdist
+        # installs would break with a spurious "mvn package" error.
+        if (
+            dest_jar_path.exists()
+            and license_path.exists()
+            and notice_path.exists()
+            and third_party_dest.exists()
+            and readme_path.exists()
+        ):
+            print("All required files already exist (building from sdist), skipping copy")
+            return
+
+        # --- Copy JAR ---
+        print(f"Root DIR: {root_dir}")
+        source_jar_glob = str(
+            root_dir / "../../java/opendataloader-pdf-cli/target/opendataloader-pdf-cli-*.jar"
+        )
+        resolved_glob_path = Path(source_jar_glob).resolve()
+        print(f"Searching for JAR file in: {resolved_glob_path}")
+
+        source_jar_paths = glob.glob(source_jar_glob)
+        if not source_jar_paths:
+            raise RuntimeError(
+                f"Could not find the JAR file. Please run 'mvn package' in the 'java/' directory first. Searched in: {resolved_glob_path}"
+            )
+        if len(source_jar_paths) > 1:
+            raise RuntimeError(f"Found multiple JAR files, expected one: {source_jar_paths}")
+        source_jar_path = source_jar_paths[0]
+        print(f"Found source JAR: {source_jar_path}")
+
+        dest_jar_dir.mkdir(parents=True, exist_ok=True)
+        print(f"Copying JAR to {dest_jar_path}")
+        shutil.copy(source_jar_path, dest_jar_path)
+
+        # --- Copy LICENSE, NOTICE ---
+        # README is copied by build-python.sh before this hook runs, because
+        # hatchling validates [project.readme] during metadata parsing, which
+        # happens before build hooks. Do not copy README here.
+        shutil.copy(root_dir / "../../LICENSE", license_path)
+        shutil.copy(root_dir / "../../NOTICE", notice_path)
+        third_party_src = root_dir / "../../THIRD_PARTY"
+        print(f"Copying THIRD_PARTY directory to {third_party_dest}")
+        if third_party_dest.exists():
+            shutil.rmtree(third_party_dest)
+        shutil.copytree(third_party_src, third_party_dest)

python/opendataloader-pdf/pyproject.toml

@@ -0,0 +1,79 @@
+[project]
+name = "opendataloader-pdf"
+version = "0.0.0"
+description = "A Python wrapper for the opendataloader-pdf Java CLI."
+readme = "README.md"
+license = "Apache-2.0"
+requires-python = ">=3.10"
+authors = [
+    { name = "opendataloader-project", email = "open.dataloader@hancom.com" }
+]
+classifiers = [
+    "Programming Language :: Python :: 3",
+    "Operating System :: OS Independent",
+]
+dependencies = []
+
+[project.optional-dependencies]
+hybrid = [
+    "docling[easyocr]>=2.91.0",
+    "fastapi>=0.136.1",
+    "uvicorn>=0.46.0",
+    "python-multipart>=0.0.28",
+]
+
+[project.scripts]
+opendataloader-pdf = "opendataloader_pdf.wrapper:main"
+opendataloader-pdf-hybrid = "opendataloader_pdf.hybrid_server:main"
+
+[project.urls]
+Homepage = "https://github.com/opendataloader-project/opendataloader-pdf"
+
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+# Shared by wheel and sdist: force-include gitignored build outputs
+# (JAR, LICENSE, NOTICE, THIRD_PARTY) that hatch_build.py copies at build time.
+[tool.hatch.build]
+artifacts = [
+    "src/opendataloader_pdf/jar/*.jar",
+    "src/opendataloader_pdf/LICENSE",
+    "src/opendataloader_pdf/NOTICE",
+    "src/opendataloader_pdf/THIRD_PARTY/**",
+]
+
+[tool.hatch.build.targets.wheel]
+packages = ["src/opendataloader_pdf"]
+
+[tool.hatch.build.targets.sdist]
+include = [
+    "src/opendataloader_pdf/**",
+    "README.md",
+    "hatch_build.py",
+]
+
+[tool.hatch.build.hooks.custom]
+path = "hatch_build.py"
+
+[dependency-groups]
+dev = [
+    "pytest>=9.0.3",
+    "pytest-asyncio>=1.3.0",
+    "httpx>=0.28.1",
+]
+
+[tool.uv]
+package = true
+
+[tool.black]
+line-length = 100
+
+[tool.ruff]
+line-length = 100
+target-version = "py310"
+exclude = ["dist", "build"]
+
+[tool.ruff.lint]
+select = ["E", "F", "I"]
+ignore = []

python/opendataloader-pdf/src/opendataloader_pdf/__init__.py

@@ -0,0 +1,3 @@
+from .wrapper import run, convert, run_jar
+
+__all__ = ["run", "convert", "run_jar"]

python/opendataloader-pdf/src/opendataloader_pdf/__main__.py

@@ -0,0 +1,5 @@
+from .wrapper import main
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

python/opendataloader-pdf/src/opendataloader_pdf/cli_options_generated.py

@@ -0,0 +1,308 @@
+# AUTO-GENERATED FROM options.json - DO NOT EDIT DIRECTLY
+# Run `npm run generate-options` to regenerate
+
+"""
+CLI option definitions for opendataloader-pdf.
+"""
+from typing import Any, Dict, List
+
+
+# Option metadata list
+CLI_OPTIONS: List[Dict[str, Any]] = [
+    {
+        "name": "output-dir",
+        "python_name": "output_dir",
+        "short_name": "o",
+        "type": "string",
+        "required": False,
+        "default": None,
+        "description": "Directory where output files are written. Default: input file directory",
+    },
+    {
+        "name": "password",
+        "python_name": "password",
+        "short_name": "p",
+        "type": "string",
+        "required": False,
+        "default": None,
+        "description": "Password for encrypted PDF files",
+    },
+    {
+        "name": "format",
+        "python_name": "format",
+        "short_name": "f",
+        "type": "string",
+        "required": False,
+        "default": None,
+        "description": "Output formats (comma-separated). Values: json, text, html, pdf, markdown, tagged-pdf. Default: json. For HTML inside Markdown use --markdown-with-html. For image extraction control use --image-output.",
+    },
+    {
+        "name": "quiet",
+        "python_name": "quiet",
+        "short_name": "q",
+        "type": "boolean",
+        "required": False,
+        "default": False,
+        "description": "Suppress console logging output",
+    },
+    {
+        "name": "content-safety-off",
+        "python_name": "content_safety_off",
+        "short_name": None,
+        "type": "string",
+        "required": False,
+        "default": None,
+        "description": "Disable content safety filters. Values: all, hidden-text, off-page, tiny, hidden-ocg",
+    },
+    {
+        "name": "sanitize",
+        "python_name": "sanitize",
+        "short_name": None,
+        "type": "boolean",
+        "required": False,
+        "default": False,
+        "description": "Enable sensitive data sanitization. Replaces emails, phone numbers, IPs, credit cards, and URLs with placeholders",
+    },
+    {
+        "name": "keep-line-breaks",
+        "python_name": "keep_line_breaks",
+        "short_name": None,
+        "type": "boolean",
+        "required": False,
+        "default": False,
+        "description": "Preserve original line breaks in extracted text",
+    },
+    {
+        "name": "replace-invalid-chars",
+        "python_name": "replace_invalid_chars",
+        "short_name": None,
+        "type": "string",
+        "required": False,
+        "default": " ",
+        "description": "Replacement character for invalid/unrecognized characters. Default: space",
+    },
+    {
+        "name": "use-struct-tree",
+        "python_name": "use_struct_tree",
+        "short_name": None,
+        "type": "boolean",
+        "required": False,
+        "default": False,
+        "description": "Use PDF structure tree (tagged PDF) for reading order and semantic structure. Output quality depends on tag quality",
+    },
+    {
+        "name": "table-method",
+        "python_name": "table_method",
+        "short_name": None,
+        "type": "string",
+        "required": False,
+        "default": "default",
+        "description": "Table detection method. Values: default (border-based), cluster (border + cluster). Default: default",
+    },
+    {
+        "name": "reading-order",
+        "python_name": "reading_order",
+        "short_name": None,
+        "type": "string",
+        "required": False,
+        "default": "xycut",
+        "description": "Reading order algorithm. Values: off, xycut. Default: xycut",
+    },
+    {
+        "name": "markdown-page-separator",
+        "python_name": "markdown_page_separator",
+        "short_name": None,
+        "type": "string",
+        "required": False,
+        "default": None,
+        "description": "Separator between pages in Markdown output. Use %%page-number%% for page numbers. Default: none",
+    },
+    {
+        "name": "markdown-with-html",
+        "python_name": "markdown_with_html",
+        "short_name": None,
+        "type": "boolean",
+        "required": False,
+        "default": False,
+        "description": "Allow HTML tags inside Markdown output for complex structures such as multi-row-span tables. Implies --format markdown.",
+    },
+    {
+        "name": "text-page-separator",
+        "python_name": "text_page_separator",
+        "short_name": None,
+        "type": "string",
+        "required": False,
+        "default": None,
+        "description": "Separator between pages in text output. Use %%page-number%% for page numbers. Default: none",
+    },
+    {
+        "name": "html-page-separator",
+        "python_name": "html_page_separator",
+        "short_name": None,
+        "type": "string",
+        "required": False,
+        "default": None,
+        "description": "Separator between pages in HTML output. Use %%page-number%% for page numbers. Default: none",
+    },
+    {
+        "name": "image-output",
+        "python_name": "image_output",
+        "short_name": None,
+        "type": "string",
+        "required": False,
+        "default": "external",
+        "description": "Image output mode. Values: off (no images), embedded (Base64 data URIs), external (file references). Default: external",
+    },
+    {
+        "name": "image-format",
+        "python_name": "image_format",
+        "short_name": None,
+        "type": "string",
+        "required": False,
+        "default": "png",
+        "description": "Output format for extracted images. Values: png, jpeg. Default: png",
+    },
+    {
+        "name": "image-dir",
+        "python_name": "image_dir",
+        "short_name": None,
+        "type": "string",
+        "required": False,
+        "default": None,
+        "description": "Directory for extracted images (applies only with --image-output external)",
+    },
+    {
+        "name": "pages",
+        "python_name": "pages",
+        "short_name": None,
+        "type": "string",
+        "required": False,
+        "default": None,
+        "description": "Pages to extract (e.g., \"1,3,5-7\"). Default: all pages",
+    },
+    {
+        "name": "include-header-footer",
+        "python_name": "include_header_footer",
+        "short_name": None,
+        "type": "boolean",
+        "required": False,
+        "default": False,
+        "description": "Include page headers and footers in output",
+    },
+    {
+        "name": "detect-strikethrough",
+        "python_name": "detect_strikethrough",
+        "short_name": None,
+        "type": "boolean",
+        "required": False,
+        "default": False,
+        "description": "Detect strikethrough text and wrap with ~~ in Markdown output or <del></del> tag in HTML output (experimental)",
+    },
+    {
+        "name": "hybrid",
+        "python_name": "hybrid",
+        "short_name": None,
+        "type": "string",
+        "required": False,
+        "default": "off",
+        "description": "Hybrid backend (requires a running server). Quick start: pip install \"opendataloader-pdf[hybrid]\" && opendataloader-pdf-hybrid --port 5002. For remote servers use --hybrid-url. Values: off (default), docling-fast, hancom-ai",
+    },
+    {
+        "name": "hybrid-mode",
+        "python_name": "hybrid_mode",
+        "short_name": None,
+        "type": "string",
+        "required": False,
+        "default": "auto",
+        "description": "Hybrid triage mode. Values: auto (default, dynamic triage), full (skip triage, all pages to backend)",
+    },
+    {
+        "name": "hybrid-url",
+        "python_name": "hybrid_url",
+        "short_name": None,
+        "type": "string",
+        "required": False,
+        "default": None,
+        "description": "Hybrid backend server URL (overrides default)",
+    },
+    {
+        "name": "hybrid-timeout",
+        "python_name": "hybrid_timeout",
+        "short_name": None,
+        "type": "string",
+        "required": False,
+        "default": "0",
+        "description": "Hybrid backend request timeout in milliseconds (0 = no timeout). Default: 0",
+    },
+    {
+        "name": "hybrid-fallback",
+        "python_name": "hybrid_fallback",
+        "short_name": None,
+        "type": "boolean",
+        "required": False,
+        "default": False,
+        "description": "Opt in to Java fallback on hybrid backend error (default: disabled)",
+    },
+    {
+        "name": "hybrid-hancom-ai-regionlist-strategy",
+        "python_name": "hybrid_hancom_ai_regionlist_strategy",
+        "short_name": None,
+        "type": "string",
+        "required": False,
+        "default": "table-first",
+        "description": "DLA label 7 (regionlist) handling. Requires --hybrid=hancom-ai. Values: table-first (default; check TSR overlap), list-only (skip TSR, always treat as list)",
+    },
+    {
+        "name": "hybrid-hancom-ai-ocr-strategy",
+        "python_name": "hybrid_hancom_ai_ocr_strategy",
+        "short_name": None,
+        "type": "string",
+        "required": False,
+        "default": "auto",
+        "description": "OCR strategy. Requires --hybrid=hancom-ai. Values: off (stream-only), auto (default; stream first, OCR fallback), force (OCR-only)",
+    },
+    {
+        "name": "hybrid-hancom-ai-image-cache",
+        "python_name": "hybrid_hancom_ai_image_cache",
+        "short_name": None,
+        "type": "string",
+        "required": False,
+        "default": "memory",
+        "description": "Page image cache backing. Requires --hybrid=hancom-ai. Values: memory (default), disk",
+    },
+    {
+        "name": "to-stdout",
+        "python_name": "to_stdout",
+        "short_name": None,
+        "type": "boolean",
+        "required": False,
+        "default": False,
+        "description": "Write output to stdout instead of file (single format only)",
+    },
+    {
+        "name": "threads",
+        "python_name": "threads",
+        "short_name": None,
+        "type": "string",
+        "required": False,
+        "default": "1",
+        "description": "Number of worker threads for per-page processing. Default: 1 (sequential, stable). Values >1 (experimental) run pages in parallel for faster throughput; output may vary slightly on some PDFs. Capped at the number of available CPU cores. Applies to the native Java pipeline only; ignored in --hybrid mode",
+    },
+]
+
+
+def add_options_to_parser(parser) -> None:
+    """Add all CLI options to an argparse.ArgumentParser."""
+    for opt in CLI_OPTIONS:
+        flags = []
+        if opt["short_name"]:
+            flags.append(f'-{opt["short_name"]}')
+        flags.append(f'--{opt["name"]}')
+
+        kwargs = {"help": opt["description"]}
+        if opt["type"] == "boolean":
+            kwargs["action"] = "store_true"
+        else:
+            kwargs["default"] = None
+
+        parser.add_argument(*flags, **kwargs)

python/opendataloader-pdf/src/opendataloader_pdf/convert_generated.py

@@ -0,0 +1,162 @@
+# AUTO-GENERATED FROM options.json - DO NOT EDIT DIRECTLY
+# Run `npm run generate-options` to regenerate
+
+"""
+Auto-generated convert function for opendataloader-pdf.
+"""
+from typing import List, Optional, Union
+
+from .runner import run_jar
+
+
+def convert(
+    input_path: Union[str, List[str]],
+    output_dir: Optional[str] = None,
+    password: Optional[str] = None,
+    format: Optional[Union[str, List[str]]] = None,
+    quiet: bool = False,
+    content_safety_off: Optional[Union[str, List[str]]] = None,
+    sanitize: bool = False,
+    keep_line_breaks: bool = False,
+    replace_invalid_chars: Optional[str] = None,
+    use_struct_tree: bool = False,
+    table_method: Optional[str] = None,
+    reading_order: Optional[str] = None,
+    markdown_page_separator: Optional[str] = None,
+    markdown_with_html: bool = False,
+    text_page_separator: Optional[str] = None,
+    html_page_separator: Optional[str] = None,
+    image_output: Optional[str] = None,
+    image_format: Optional[str] = None,
+    image_dir: Optional[str] = None,
+    pages: Optional[str] = None,
+    include_header_footer: bool = False,
+    detect_strikethrough: bool = False,
+    hybrid: Optional[str] = None,
+    hybrid_mode: Optional[str] = None,
+    hybrid_url: Optional[str] = None,
+    hybrid_timeout: Optional[str] = None,
+    hybrid_fallback: bool = False,
+    hybrid_hancom_ai_regionlist_strategy: Optional[str] = None,
+    hybrid_hancom_ai_ocr_strategy: Optional[str] = None,
+    hybrid_hancom_ai_image_cache: Optional[str] = None,
+    to_stdout: bool = False,
+    threads: Optional[str] = None,
+) -> None:
+    """
+    Convert PDF(s) into the requested output format(s).
+
+    Args:
+        input_path: One or more input PDF file paths or directories
+        output_dir: Directory where output files are written. Default: input file directory
+        password: Password for encrypted PDF files
+        format: Output formats (comma-separated). Values: json, text, html, pdf, markdown, tagged-pdf. Default: json. For HTML inside Markdown use --markdown-with-html. For image extraction control use --image-output.
+        quiet: Suppress console logging output
+        content_safety_off: Disable content safety filters. Values: all, hidden-text, off-page, tiny, hidden-ocg
+        sanitize: Enable sensitive data sanitization. Replaces emails, phone numbers, IPs, credit cards, and URLs with placeholders
+        keep_line_breaks: Preserve original line breaks in extracted text
+        replace_invalid_chars: Replacement character for invalid/unrecognized characters. Default: space
+        use_struct_tree: Use PDF structure tree (tagged PDF) for reading order and semantic structure. Output quality depends on tag quality
+        table_method: Table detection method. Values: default (border-based), cluster (border + cluster). Default: default
+        reading_order: Reading order algorithm. Values: off, xycut. Default: xycut
+        markdown_page_separator: Separator between pages in Markdown output. Use %page-number% for page numbers. Default: none
+        markdown_with_html: Allow HTML tags inside Markdown output for complex structures such as multi-row-span tables. Implies --format markdown.
+        text_page_separator: Separator between pages in text output. Use %page-number% for page numbers. Default: none
+        html_page_separator: Separator between pages in HTML output. Use %page-number% for page numbers. Default: none
+        image_output: Image output mode. Values: off (no images), embedded (Base64 data URIs), external (file references). Default: external
+        image_format: Output format for extracted images. Values: png, jpeg. Default: png
+        image_dir: Directory for extracted images (applies only with --image-output external)
+        pages: Pages to extract (e.g., "1,3,5-7"). Default: all pages
+        include_header_footer: Include page headers and footers in output
+        detect_strikethrough: Detect strikethrough text and wrap with ~~ in Markdown output or <del></del> tag in HTML output (experimental)
+        hybrid: Hybrid backend (requires a running server). Quick start: pip install "opendataloader-pdf[hybrid]" && opendataloader-pdf-hybrid --port 5002. For remote servers use --hybrid-url. Values: off (default), docling-fast, hancom-ai
+        hybrid_mode: Hybrid triage mode. Values: auto (default, dynamic triage), full (skip triage, all pages to backend)
+        hybrid_url: Hybrid backend server URL (overrides default)
+        hybrid_timeout: Hybrid backend request timeout in milliseconds (0 = no timeout). Default: 0
+        hybrid_fallback: Opt in to Java fallback on hybrid backend error (default: disabled)
+        hybrid_hancom_ai_regionlist_strategy: DLA label 7 (regionlist) handling. Requires --hybrid=hancom-ai. Values: table-first (default; check TSR overlap), list-only (skip TSR, always treat as list)
+        hybrid_hancom_ai_ocr_strategy: OCR strategy. Requires --hybrid=hancom-ai. Values: off (stream-only), auto (default; stream first, OCR fallback), force (OCR-only)
+        hybrid_hancom_ai_image_cache: Page image cache backing. Requires --hybrid=hancom-ai. Values: memory (default), disk
+        to_stdout: Write output to stdout instead of file (single format only)
+        threads: Number of worker threads for per-page processing. Default: 1 (sequential, stable). Values >1 (experimental) run pages in parallel for faster throughput; output may vary slightly on some PDFs. Capped at the number of available CPU cores. Applies to the native Java pipeline only; ignored in --hybrid mode
+    """
+    args: List[str] = []
+
+    # Build input paths
+    if isinstance(input_path, list):
+        args.extend(input_path)
+    else:
+        args.append(input_path)
+
+    if output_dir:
+        args.extend(["--output-dir", output_dir])
+    if password:
+        args.extend(["--password", password])
+    if format:
+        if isinstance(format, list):
+            if format:
+                args.extend(["--format", ",".join(format)])
+        else:
+            args.extend(["--format", format])
+    if quiet:
+        args.append("--quiet")
+    if content_safety_off:
+        if isinstance(content_safety_off, list):
+            if content_safety_off:
+                args.extend(["--content-safety-off", ",".join(content_safety_off)])
+        else:
+            args.extend(["--content-safety-off", content_safety_off])
+    if sanitize:
+        args.append("--sanitize")
+    if keep_line_breaks:
+        args.append("--keep-line-breaks")
+    if replace_invalid_chars:
+        args.extend(["--replace-invalid-chars", replace_invalid_chars])
+    if use_struct_tree:
+        args.append("--use-struct-tree")
+    if table_method:
+        args.extend(["--table-method", table_method])
+    if reading_order:
+        args.extend(["--reading-order", reading_order])
+    if markdown_page_separator:
+        args.extend(["--markdown-page-separator", markdown_page_separator])
+    if markdown_with_html:
+        args.append("--markdown-with-html")
+    if text_page_separator:
+        args.extend(["--text-page-separator", text_page_separator])
+    if html_page_separator:
+        args.extend(["--html-page-separator", html_page_separator])
+    if image_output:
+        args.extend(["--image-output", image_output])
+    if image_format:
+        args.extend(["--image-format", image_format])
+    if image_dir:
+        args.extend(["--image-dir", image_dir])
+    if pages:
+        args.extend(["--pages", pages])
+    if include_header_footer:
+        args.append("--include-header-footer")
+    if detect_strikethrough:
+        args.append("--detect-strikethrough")
+    if hybrid:
+        args.extend(["--hybrid", hybrid])
+    if hybrid_mode:
+        args.extend(["--hybrid-mode", hybrid_mode])
+    if hybrid_url:
+        args.extend(["--hybrid-url", hybrid_url])
+    if hybrid_timeout:
+        args.extend(["--hybrid-timeout", hybrid_timeout])
+    if hybrid_fallback:
+        args.append("--hybrid-fallback")
+    if hybrid_hancom_ai_regionlist_strategy:
+        args.extend(["--hybrid-hancom-ai-regionlist-strategy", hybrid_hancom_ai_regionlist_strategy])
+    if hybrid_hancom_ai_ocr_strategy:
+        args.extend(["--hybrid-hancom-ai-ocr-strategy", hybrid_hancom_ai_ocr_strategy])
+    if hybrid_hancom_ai_image_cache:
+        args.extend(["--hybrid-hancom-ai-image-cache", hybrid_hancom_ai_image_cache])
+    if to_stdout:
+        args.append("--to-stdout")
+    if threads:
+        args.extend(["--threads", threads])
+
+    run_jar(args, quiet)

python/opendataloader-pdf/src/opendataloader_pdf/runner.py

@@ -0,0 +1,103 @@
+"""
+Low-level JAR runner for opendataloader-pdf.
+"""
+import subprocess
+import sys
+import importlib.resources as resources
+from typing import List
+
+# The consistent name of the JAR file bundled with the package
+_JAR_NAME = "opendataloader-pdf-cli.jar"
+
+
+def run_jar(args: List[str], quiet: bool = False) -> str:
+    """Run the opendataloader-pdf JAR with the given arguments."""
+    try:
+        # Access the embedded JAR inside the package
+        jar_ref = resources.files("opendataloader_pdf").joinpath("jar", _JAR_NAME)
+        with resources.as_file(jar_ref) as jar_path:
+            # Force headless AWT so macOS doesn't surface a Dock icon (and
+            # steal focus) every time the JVM touches ImageIO/PDFBox
+            # rendering. Safe on all OSes — the CLI never opens a UI window,
+            # only manipulates BufferedImages.
+            command = [
+                "java",
+                "-Djava.awt.headless=true",
+                "-Dapple.awt.UIElement=true",
+                "-jar",
+                str(jar_path),
+                *args,
+            ]
+
+            if quiet:
+                # Quiet mode → suppress the JAR's log stream (stderr) but
+                # relay its stdout to the caller: --to-stdout content and the
+                # folder summary line arrive on stdout, and swallowing them
+                # breaks pipe consumers (`... --quiet --to-stdout | jq`).
+                result = subprocess.run(
+                    command,
+                    stdout=subprocess.PIPE,
+                    stderr=subprocess.PIPE,
+                    text=True,
+                    check=True,
+                    encoding="utf-8",
+                    errors="replace",
+                )
+                if result.stdout:
+                    if hasattr(sys.stdout, "buffer"):
+                        sys.stdout.buffer.write(
+                            result.stdout.encode("utf-8", errors="replace")
+                        )
+                        sys.stdout.buffer.flush()
+                    else:
+                        sys.stdout.write(result.stdout)
+                return result.stdout
+
+            # Streaming mode → live output
+            with subprocess.Popen(
+                command,
+                stdout=subprocess.PIPE,
+                stderr=subprocess.STDOUT,
+                text=True,
+                encoding="utf-8",
+                errors="replace",
+            ) as process:
+                output_lines: List[str] = []
+                for line in process.stdout:
+                    if hasattr(sys.stdout, "buffer"):
+                        sys.stdout.buffer.write(line.encode("utf-8", errors="replace"))
+                        sys.stdout.buffer.flush()
+                    else:
+                        sys.stdout.write(line)
+                    output_lines.append(line)
+
+                return_code = process.wait()
+                captured_output = "".join(output_lines)
+
+                if return_code:
+                    raise subprocess.CalledProcessError(
+                        return_code, command, output=captured_output
+                    )
+                return captured_output
+
+    except FileNotFoundError:
+        print(
+            "Error: 'java' command not found. Please ensure Java is installed and in your system's PATH.",
+            file=sys.stderr,
+        )
+        raise
+
+    except subprocess.CalledProcessError as error:
+        print("Error running opendataloader-pdf CLI.", file=sys.stderr)
+        print(f"Return code: {error.returncode}", file=sys.stderr)
+        # Streaming mode already wrote the JAR's output live to stdout, so
+        # re-printing the captured copy would duplicate it. Only surface the
+        # captured streams in quiet mode, where the caller has not seen them.
+        # Note: CalledProcessError.output and .stdout are aliases for the same
+        # attribute — printing both produces the same content twice.
+        if quiet:
+            if error.stdout:
+                print(f"Stdout: {error.stdout}", file=sys.stderr)
+            if error.stderr:
+                print(f"Stderr: {error.stderr}", file=sys.stderr)
+        raise

python/opendataloader-pdf/src/opendataloader_pdf/wrapper.py

@@ -0,0 +1,117 @@
+import argparse
+import subprocess
+import sys
+import warnings
+from typing import List, Optional
+
+from .cli_options_generated import add_options_to_parser
+from .convert_generated import convert
+from .runner import run_jar
+
+# Re-export for backward compatibility
+__all__ = ["convert", "run", "run_jar", "main"]
+
+
+# Deprecated : Use `convert()` instead. This function will be removed in a future version.
+def run(
+    input_path: str,
+    output_folder: Optional[str] = None,
+    password: Optional[str] = None,
+    replace_invalid_chars: Optional[str] = None,
+    generate_markdown: bool = False,
+    generate_html: bool = False,
+    generate_annotated_pdf: bool = False,
+    keep_line_breaks: bool = False,
+    content_safety_off: Optional[str] = None,
+    html_in_markdown: bool = False,
+    add_image_to_markdown: bool = False,
+    no_json: bool = False,
+    debug: bool = False,
+    use_struct_tree: bool = False,
+):
+    """
+    Runs the opendataloader-pdf with the given arguments.
+
+    .. deprecated::
+        Use :func:`convert` instead. This function will be removed in a future version.
+
+    Args:
+        input_path: Path to the input PDF file or folder.
+        output_folder: Path to the output folder. Defaults to the input folder.
+        password: Password for the PDF file.
+        replace_invalid_chars: Character to replace invalid or unrecognized characters (e.g., , \\u0000) with.
+        generate_markdown: If True, generates a Markdown output file.
+        generate_html: If True, generates an HTML output file.
+        generate_annotated_pdf: If True, generates an annotated PDF output file.
+        keep_line_breaks: If True, keeps line breaks in the output.
+        html_in_markdown: If True, uses HTML in the Markdown output.
+        add_image_to_markdown: If True, adds images to the Markdown output.
+        no_json: If True, disable the JSON output.
+        debug: If True, prints all messages from the CLI to the console during execution.
+        use_struct_tree: If True, enable processing structure tree (disabled by default)
+
+    Raises:
+        FileNotFoundError: If the 'java' command is not found or input_path is invalid.
+        subprocess.CalledProcessError: If the CLI tool returns a non-zero exit code.
+    """
+    warnings.warn(
+        "run() is deprecated and will be removed in a future version. Use convert() instead.",
+        DeprecationWarning,
+        stacklevel=2,
+    )
+
+    # Build format list based on legacy boolean options
+    formats: List[str] = []
+    if not no_json:
+        formats.append("json")
+    if generate_markdown:
+        if add_image_to_markdown:
+            formats.append("markdown-with-images")
+        elif html_in_markdown:
+            formats.append("markdown-with-html")
+        else:
+            formats.append("markdown")
+    if generate_html:
+        formats.append("html")
+    if generate_annotated_pdf:
+        formats.append("pdf")
+
+    convert(
+        input_path=input_path,
+        output_dir=output_folder,
+        password=password,
+        replace_invalid_chars=replace_invalid_chars,
+        keep_line_breaks=keep_line_breaks,
+        content_safety_off=content_safety_off,
+        use_struct_tree=use_struct_tree,
+        format=formats if formats else None,
+        quiet=not debug,
+    )
+
+
+def main(argv=None) -> int:
+    """CLI entry point for running the wrapper from the command line."""
+    parser = argparse.ArgumentParser(
+        description="Run the opendataloader-pdf CLI using the bundled JAR."
+    )
+    parser.add_argument(
+        "input_path", nargs="+", help="Path to the input PDF file or directory."
+    )
+
+    # Register CLI options from auto-generated module
+    add_options_to_parser(parser)
+
+    args = parser.parse_args(argv)
+
+    try:
+        convert(**vars(args))
+        return 0
+    except FileNotFoundError as err:
+        print(err, file=sys.stderr)
+        return 1
+    except subprocess.CalledProcessError as err:
+        return err.returncode or 1
+
+
+if __name__ == "__main__":
+    sys.exit(main())

python/opendataloader-pdf/tests/conftest.py

@@ -0,0 +1,23 @@
+import shutil
+from pathlib import Path
+
+import pytest
+
+
+@pytest.fixture
+def input_pdf():
+    return Path(__file__).resolve().parents[3] / "samples" / "pdf" / "1901.03003.pdf"
+
+
+@pytest.fixture
+def output_dir():
+    path = (
+        Path(__file__).resolve().parents[3]
+        / "python"
+        / "opendataloader-pdf"
+        / "tests"
+        / "temp"
+    )
+    path.mkdir(exist_ok=True)
+    yield path
+    shutil.rmtree(path, ignore_errors=True)

python/opendataloader-pdf/tests/test_cli_options.py

@@ -0,0 +1,142 @@
+"""Unit tests for auto-generated cli_options module"""
+
+import pytest
+from opendataloader_pdf.cli_options_generated import CLI_OPTIONS, add_options_to_parser
+
+
+class TestCLIOptions:
+    """Tests for CLI_OPTIONS metadata list"""
+
+    def test_cli_options_is_list(self):
+        """CLI_OPTIONS should be a list"""
+        assert isinstance(CLI_OPTIONS, list)
+
+    def test_cli_options_not_empty(self):
+        """CLI_OPTIONS should not be empty"""
+        assert len(CLI_OPTIONS) > 0
+
+    def test_each_option_has_required_fields(self):
+        """Each option should have all required fields"""
+        required_fields = [
+            "name",
+            "python_name",
+            "short_name",
+            "type",
+            "required",
+            "default",
+            "description",
+        ]
+        for opt in CLI_OPTIONS:
+            for field in required_fields:
+                assert field in opt, f"Option {opt.get('name', 'unknown')} missing field: {field}"
+
+    def test_option_types_are_valid(self):
+        """Option types should be 'string' or 'boolean'"""
+        valid_types = {"string", "boolean"}
+        for opt in CLI_OPTIONS:
+            assert opt["type"] in valid_types, f"Invalid type for {opt['name']}: {opt['type']}"
+
+    def test_python_name_is_snake_case(self):
+        """Python names should be snake_case (no hyphens)"""
+        for opt in CLI_OPTIONS:
+            assert "-" not in opt["python_name"], f"Python name should not contain hyphen: {opt['python_name']}"
+
+    def test_known_options_exist(self):
+        """Known options should exist in the list"""
+        option_names = {opt["name"] for opt in CLI_OPTIONS}
+        expected_options = {
+            "output-dir",
+            "password",
+            "format",
+            "quiet",
+            "content-safety-off",
+            "keep-line-breaks",
+            "image-output",
+            "image-format",
+        }
+        for expected in expected_options:
+            assert expected in option_names, f"Expected option not found: {expected}"
+
+    def test_sanitize_option_exists(self):
+        option_names = [opt["name"] for opt in CLI_OPTIONS]
+        assert "sanitize" in option_names
+        sanitize_opt = next(opt for opt in CLI_OPTIONS if opt["name"] == "sanitize")
+        assert sanitize_opt["type"] == "boolean"
+        assert sanitize_opt["default"] == False
+
+
+
+class TestAddOptionsToParser:
+    """Tests for add_options_to_parser function"""
+
+    def test_adds_all_options(self):
+        """Should add all options to argparse parser"""
+        import argparse
+
+        parser = argparse.ArgumentParser()
+        add_options_to_parser(parser)
+
+        # Parse empty args to get defaults
+        args = parser.parse_args([])
+
+        # Check that all options are added
+        for opt in CLI_OPTIONS:
+            python_name = opt["python_name"]
+            assert hasattr(args, python_name.replace("-", "_")), f"Option {python_name} not added to parser"
+
+    def test_boolean_options_default_to_false(self):
+        """Boolean options should default to False"""
+        import argparse
+
+        parser = argparse.ArgumentParser()
+        add_options_to_parser(parser)
+        args = parser.parse_args([])
+
+        for opt in CLI_OPTIONS:
+            if opt["type"] == "boolean":
+                python_name = opt["python_name"].replace("-", "_")
+                assert getattr(args, python_name) is False, f"Boolean option {python_name} should default to False"
+
+    def test_string_options_default_to_none(self):
+        """String options should default to None"""
+        import argparse
+
+        parser = argparse.ArgumentParser()
+        add_options_to_parser(parser)
+        args = parser.parse_args([])
+
+        for opt in CLI_OPTIONS:
+            if opt["type"] == "string":
+                python_name = opt["python_name"].replace("-", "_")
+                assert getattr(args, python_name) is None, f"String option {python_name} should default to None"
+
+    def test_short_options_work(self):
+        """Short option flags should work"""
+        import argparse
+
+        parser = argparse.ArgumentParser()
+        add_options_to_parser(parser)
+
+        # Test with -o (short for --output-dir)
+        args = parser.parse_args(["-o", "/output"])
+        assert args.output_dir == "/output"
+
+        # Test with -f (short for --format)
+        args = parser.parse_args(["-f", "json"])
+        assert args.format == "json"
+
+        # Test with -q (short for --quiet)
+        args = parser.parse_args(["-q"])
+        assert args.quiet is True
+
+    def test_long_options_work(self):
+        """Long option flags should work"""
+        import argparse
+
+        parser = argparse.ArgumentParser()
+        add_options_to_parser(parser)
+
+        args = parser.parse_args(["--output-dir", "/output", "--format", "json,markdown", "--quiet"])
+        assert args.output_dir == "/output"
+        assert args.format == "json,markdown"
+        assert args.quiet is True

python/opendataloader-pdf/tests/test_convert_integration.py

@@ -0,0 +1,16 @@
+"""Integration tests that actually run the JAR (slow)"""
+
+import opendataloader_pdf
+
+
+def test_convert_generates_output(input_pdf, output_dir):
+    """Verify that convert() actually generates output files"""
+    opendataloader_pdf.convert(
+        input_path=str(input_pdf),
+        output_dir=str(output_dir),
+        format="json",
+        quiet=True,
+    )
+    output = output_dir / "1901.03003.json"
+    assert output.exists(), f"Output file not found at {output}"
+    assert output.stat().st_size > 0, "Output file is empty"

python/opendataloader-pdf/tests/test_hybrid_server.py

@@ -0,0 +1,91 @@
+"""Tests for hybrid_server."""
+
+import logging
+import sys
+from unittest.mock import MagicMock, patch
+
+
+def test_gpu_detected_logging(caplog):
+    """GPU detection should log GPU name and CUDA version when available."""
+    mock_torch = MagicMock()
+    mock_torch.cuda.is_available.return_value = True
+    mock_torch.cuda.get_device_name.return_value = "NVIDIA A100"
+    mock_torch.version.cuda = "12.1"
+
+    with patch.dict("sys.modules", {"torch": mock_torch}):
+        # Re-import to pick up the mock
+        import importlib
+        from opendataloader_pdf import hybrid_server
+
+        importlib.reload(hybrid_server)
+
+        with caplog.at_level(logging.INFO):
+            # Simulate the GPU detection block from main()
+            try:
+                import torch
+                if torch.cuda.is_available():
+                    gpu_name = torch.cuda.get_device_name(0)
+                    cuda_version = torch.version.cuda
+                    logging.getLogger(__name__).info(
+                        f"GPU detected: {gpu_name} (CUDA {cuda_version})"
+                    )
+            except ImportError:
+                pass
+
+    assert "GPU detected: NVIDIA A100 (CUDA 12.1)" in caplog.text
+
+
+def test_no_gpu_logging(caplog):
+    """Should log CPU fallback when no GPU is available."""
+    mock_torch = MagicMock()
+    mock_torch.cuda.is_available.return_value = False
+
+    with patch.dict("sys.modules", {"torch": mock_torch}):
+        with caplog.at_level(logging.INFO):
+            try:
+                import torch
+                if torch.cuda.is_available():
+                    pass
+                else:
+                    logging.getLogger(__name__).info("No GPU detected, using CPU.")
+            except ImportError:
+                pass
+
+    assert "No GPU detected, using CPU." in caplog.text
+
+
+def test_no_pytorch_logging(caplog):
+    """Should log CPU fallback when PyTorch is not installed."""
+    with patch.dict("sys.modules", {"torch": None}):
+        with caplog.at_level(logging.INFO):
+            try:
+                import torch  # noqa: F811
+                if torch.cuda.is_available():
+                    pass
+                else:
+                    logging.getLogger(__name__).info("No GPU detected, using CPU.")
+            except (ImportError, TypeError):
+                logging.getLogger(__name__).info(
+                    "No GPU detected, using CPU. (PyTorch not installed)"
+                )
+
+    assert "No GPU detected, using CPU. (PyTorch not installed)" in caplog.text
+
+
+def test_get_loop_setting_returns_asyncio_on_windows():
+    """On Windows, should return 'asyncio' to avoid uvloop errors (#323)."""
+    from opendataloader_pdf.hybrid_server import _get_loop_setting
+
+    with patch("sys.platform", "win32"):
+        assert _get_loop_setting() == "asyncio"
+
+
+def test_get_loop_setting_returns_auto_on_non_windows():
+    """On non-Windows platforms, should return 'auto' (uvloop if available)."""
+    from opendataloader_pdf.hybrid_server import _get_loop_setting
+
+    with patch("sys.platform", "darwin"):
+        assert _get_loop_setting() == "auto"
+
+    with patch("sys.platform", "linux"):
+        assert _get_loop_setting() == "auto"

python/opendataloader-pdf/tests/test_hybrid_server_nonblocking.py

@@ -0,0 +1,139 @@
+"""Tests for hybrid_server non-blocking conversion.
+
+Verifies that converter.convert() runs in a thread pool rather than blocking
+the event loop. This is the root cause of issue #301: the Java client's
+3-second health check times out when the server is busy with a synchronous
+conversion call inside an async endpoint.
+"""
+
+import asyncio
+import threading
+import time
+from unittest.mock import MagicMock, patch
+
+import pytest
+
+
+@pytest.fixture
+def mock_docling():
+    """Mock docling modules so tests don't need the actual dependency."""
+    mock_converter = MagicMock()
+    mock_result = MagicMock()
+    mock_result.status.value = "success"
+    mock_result.errors = []
+    mock_result.input.page_count = 1
+    mock_result.document.export_to_dict.return_value = {
+        "pages": {"1": {}},
+        "body": {},
+    }
+
+    # Track which thread the conversion runs on
+    convert_thread_name = {}
+
+    def tracking_convert(path, page_range=None):
+        convert_thread_name["thread"] = threading.current_thread().name
+        time.sleep(2)
+        return mock_result
+
+    mock_converter.convert = tracking_convert
+    mock_converter._convert_thread = convert_thread_name
+
+    mock_conversion_status = MagicMock()
+    mock_conversion_status.PARTIAL_SUCCESS = "partial_success"
+
+    with patch.dict("sys.modules", {
+        "docling": MagicMock(),
+        "docling.datamodel.base_models": MagicMock(
+            InputFormat=MagicMock(PDF="pdf"),
+            ConversionStatus=mock_conversion_status,
+        ),
+        "docling.datamodel.pipeline_options": MagicMock(),
+        "docling.document_converter": MagicMock(),
+        "uvicorn": MagicMock(),
+    }):
+        yield mock_converter
+
+
+@pytest.fixture
+def app_with_converter(mock_docling):
+    """Create a FastAPI app with the mock converter."""
+    import importlib
+    from opendataloader_pdf import hybrid_server
+
+    importlib.reload(hybrid_server)
+
+    app = hybrid_server.create_app()
+    hybrid_server.converter = mock_docling
+    return app
+
+
+@pytest.mark.asyncio
+async def test_convert_runs_in_thread_pool(app_with_converter, mock_docling):
+    """converter.convert() must run in a worker thread, not on the event loop.
+
+    When converter.convert() runs directly on the async event loop thread,
+    it blocks all concurrent request handling — including /health checks.
+    The fix wraps converter.convert() with asyncio.to_thread() so it runs
+    in a separate thread.
+
+    Reproduces issue #301.
+    """
+    from httpx import ASGITransport, AsyncClient
+
+    transport = ASGITransport(app=app_with_converter)
+    async with AsyncClient(transport=transport, base_url="http://test") as client:
+        event_loop_thread = threading.current_thread().name
+
+        response = await client.post(
+            "/v1/convert/file",
+            files={"files": ("test.pdf", b"%PDF-1.4 minimal", "application/pdf")},
+        )
+
+        assert response.status_code == 200
+
+        convert_thread = mock_docling._convert_thread.get("thread")
+        assert convert_thread is not None, "converter.convert() was not called"
+        assert convert_thread != event_loop_thread, (
+            f"converter.convert() ran on the event loop thread '{event_loop_thread}'. "
+            f"It must run in a worker thread via asyncio.to_thread() to avoid "
+            f"blocking /health and other endpoints during long conversions."
+        )
+
+
+@pytest.mark.asyncio
+async def test_health_responds_during_conversion(app_with_converter):
+    """Health endpoint must respond quickly even during active conversion.
+
+    This is the user-facing symptom of issue #301: the Java CLI gets
+    SocketTimeoutException when the hybrid server is busy processing
+    another document. Mock sleep is 2s; health must respond under 0.2s.
+    """
+    from httpx import ASGITransport, AsyncClient
+
+    transport = ASGITransport(app=app_with_converter)
+    async with AsyncClient(transport=transport, base_url="http://test") as client:
+        # Start conversion (takes 2s in mock)
+        convert_task = asyncio.create_task(
+            client.post(
+                "/v1/convert/file",
+                files={"files": ("test.pdf", b"%PDF-1.4 minimal", "application/pdf")},
+            )
+        )
+
+        # Wait for conversion to start in the worker thread
+        await asyncio.sleep(0.3)
+
+        # Health check should respond quickly — well under the 2s conversion
+        start = time.monotonic()
+        health_response = await client.get("/health")
+        health_time = time.monotonic() - start
+
+        assert health_response.status_code == 200
+        assert health_response.json() == {"status": "ok"}
+        assert health_time < 0.2, (
+            f"Health endpoint took {health_time:.2f}s during conversion. "
+            f"Expected < 0.2s. The event loop is likely blocked."
+        )
+
+        convert_response = await convert_task
+        assert convert_response.status_code == 200

python/opendataloader-pdf/tests/test_hybrid_server_ocr_options.py

@@ -0,0 +1,447 @@
+"""Tests for the OCR option surface added to the hybrid server.
+
+Covers:
+    * `create_converter` delegation to docling's `get_ocr_factory`
+    * Defaults preserve prior behavior (do_ocr=True, engine=easyocr)
+    * `disable_ocr=True` -> do_ocr=False (#387)
+    * Engine selection produces the matching OcrOptions subclass (#436, #439)
+    * `psm` is applied only for Tesseract engines (silently ignored otherwise)
+    * Unknown / denylisted engines exit with a clear message
+    * argparse `--no-ocr` and `--force-ocr` are mutually exclusive
+    * argparse `--ocr-engine` choices exclude `kserve_v2_ocr`
+"""
+
+import argparse
+import io
+from contextlib import redirect_stderr
+from unittest.mock import patch
+
+import pytest
+
+from opendataloader_pdf import hybrid_server
+
+
+# ---------- create_converter: factory delegation ----------
+
+def _capture_pipeline_options(**kwargs):
+    """Call create_converter while mocking the docling document_converter so we
+    can introspect the PdfPipelineOptions instance that would be passed in.
+    """
+    captured = {}
+
+    def fake_pdf_format_option(*, pipeline_options):
+        captured["pipeline_options"] = pipeline_options
+        return object()
+
+    with patch(
+        "docling.document_converter.DocumentConverter"
+    ) as mock_dc, patch(
+        "docling.document_converter.PdfFormatOption", side_effect=fake_pdf_format_option
+    ):
+        mock_dc.return_value = object()
+        hybrid_server.create_converter(**kwargs)
+
+    return captured["pipeline_options"]
+
+
+def test_defaults_preserve_prior_behavior():
+    """No flags -> do_ocr=True, EasyOcrOptions (matches pre-change behavior)."""
+    from docling.datamodel.pipeline_options import EasyOcrOptions
+
+    opts = _capture_pipeline_options()
+    assert opts.do_ocr is True
+    assert isinstance(opts.ocr_options, EasyOcrOptions)
+
+
+def test_disable_ocr_sets_do_ocr_false():
+    """`disable_ocr=True` -> do_ocr=False (#387)."""
+    opts = _capture_pipeline_options(disable_ocr=True)
+    assert opts.do_ocr is False
+
+
+def test_ocr_engine_tesseract_yields_tesseract_options():
+    """`ocr_engine='tesseract'` -> TesseractCliOcrOptions (#439 path)."""
+    from docling.datamodel.pipeline_options import TesseractCliOcrOptions
+
+    opts = _capture_pipeline_options(ocr_engine="tesseract")
+    assert isinstance(opts.ocr_options, TesseractCliOcrOptions)
+
+
+def test_ocr_engine_rapidocr_yields_rapidocr_options():
+    """`ocr_engine='rapidocr'` -> RapidOcrOptions (#436 path)."""
+    from docling.datamodel.pipeline_options import RapidOcrOptions
+
+    opts = _capture_pipeline_options(ocr_engine="rapidocr")
+    assert isinstance(opts.ocr_options, RapidOcrOptions)
+
+
+def test_force_full_page_ocr_propagates_to_engine_options():
+    """`force_full_page_ocr=True` flows to the engine's options instance."""
+    opts = _capture_pipeline_options(
+        ocr_engine="rapidocr", force_full_page_ocr=True
+    )
+    assert opts.ocr_options.force_full_page_ocr is True
+
+
+def test_ocr_lang_overrides_engine_default():
+    """A non-empty ocr_lang list replaces the engine's default lang."""
+    opts = _capture_pipeline_options(
+        ocr_engine="tesseract", ocr_lang=["mal", "eng"]
+    )
+    assert opts.ocr_options.lang == ["mal", "eng"]
+
+
+def test_psm_is_applied_to_tesseract():
+    """`psm` is applied when the engine is Tesseract."""
+    opts = _capture_pipeline_options(ocr_engine="tesseract", psm=6)
+    assert opts.ocr_options.psm == 6
+
+
+def test_psm_is_ignored_for_non_tesseract_engines():
+    """`psm` is silently ignored for engines that do not expose it."""
+    # EasyOcrOptions has no `psm` field; passing psm should not raise.
+    opts = _capture_pipeline_options(ocr_engine="easyocr", psm=6)
+    assert not hasattr(opts.ocr_options, "psm")
+
+
+def test_ocr_engine_auto_yields_auto_options():
+    """`ocr_engine='auto'` -> OcrAutoOptions; engine choice is deferred to docling."""
+    from docling.datamodel.pipeline_options import OcrAutoOptions
+
+    opts = _capture_pipeline_options(ocr_engine="auto")
+    assert isinstance(opts.ocr_options, OcrAutoOptions)
+
+
+def test_unknown_engine_raises_value_error_with_clear_message():
+    """An unknown engine kind raises ValueError listing valid engines.
+
+    ValueError (not SystemExit) keeps the function library-friendly: programmatic
+    callers can catch and retry with a different engine. main() relies on
+    argparse `choices` to gate invalid CLI input separately.
+    """
+    with pytest.raises(ValueError) as excinfo:
+        _capture_pipeline_options(ocr_engine="bogus-engine")
+    msg = str(excinfo.value)
+    assert "bogus-engine" in msg
+    assert "Available engines" in msg
+
+
+def test_create_converter_rejects_denylisted_engine():
+    """Programmatic callers cannot bypass `_OCR_ENGINE_DENYLIST` via create_converter().
+
+    The CLI layer rejects denylisted engines via argparse `choices`. This test
+    locks in the same enforcement at the function-call layer, so importing the
+    module and passing `ocr_engine="kserve_v2_ocr"` directly fails with a clear
+    ValueError instead of building a converter that fails opaquely on the first
+    request.
+    """
+    with pytest.raises(ValueError) as excinfo:
+        _capture_pipeline_options(ocr_engine="kserve_v2_ocr")
+    msg = str(excinfo.value)
+    assert "kserve_v2_ocr" in msg
+    assert "hybrid local mode" in msg
+    assert "Available engines" in msg
+    # Must also mention the denylist explicitly so a future maintainer searching
+    # for the constant lands here.
+    assert "_OCR_ENGINE_DENYLIST" in msg
+
+
+# ---------- argparse: --no-ocr / --force-ocr / --ocr-engine / --psm ----------
+
+def _build_parser_subset():
+    """Reconstruct the OCR-relevant argparse subset from main().
+
+    Mirrors the layout in hybrid_server.main() so changes to the CLI surface
+    are caught by these tests. Uses the same `_OCR_ENGINE_DENYLIST` module
+    constant as production code so tests stay in sync with main() automatically.
+    """
+    from docling.models.factories import get_ocr_factory
+
+    parser = argparse.ArgumentParser()
+    ocr_mode = parser.add_mutually_exclusive_group()
+    ocr_mode.add_argument("--force-ocr", action="store_true")
+    ocr_mode.add_argument("--no-ocr", action="store_true")
+
+    choices = sorted(
+        set(get_ocr_factory(allow_external_plugins=False).registered_kind)
+        - hybrid_server._OCR_ENGINE_DENYLIST
+    )
+    parser.add_argument(
+        "--ocr-engine", default="easyocr", choices=choices
+    )
+    parser.add_argument("--psm", type=int, default=None)
+    parser.add_argument("--ocr-lang", default=None)
+    return parser
+
+
+def test_argparse_defaults():
+    """No flags -> no_ocr=False, force_ocr=False, engine=easyocr, psm=None."""
+    args = _build_parser_subset().parse_args([])
+    assert args.no_ocr is False
+    assert args.force_ocr is False
+    assert args.ocr_engine == "easyocr"
+    assert args.psm is None
+
+
+def test_argparse_no_ocr_and_force_ocr_are_mutually_exclusive():
+    """Passing both --no-ocr and --force-ocr exits with an argparse error."""
+    err = io.StringIO()
+    with pytest.raises(SystemExit), redirect_stderr(err):
+        _build_parser_subset().parse_args(["--no-ocr", "--force-ocr"])
+    assert "not allowed with argument" in err.getvalue()
+
+
+def test_argparse_kserve_engine_is_rejected():
+    """`--ocr-engine kserve_v2_ocr` is rejected (denylist filter)."""
+    err = io.StringIO()
+    with pytest.raises(SystemExit), redirect_stderr(err):
+        _build_parser_subset().parse_args(["--ocr-engine", "kserve_v2_ocr"])
+    assert "invalid choice" in err.getvalue()
+
+
+def test_argparse_tesseract_path_for_issue_439():
+    """The CLI invocation that resolves #439 parses cleanly."""
+    args = _build_parser_subset().parse_args(
+        ["--ocr-engine", "tesseract", "--ocr-lang", "mal", "--force-ocr"]
+    )
+    assert args.ocr_engine == "tesseract"
+    assert args.ocr_lang == "mal"
+    assert args.force_ocr is True
+    assert args.no_ocr is False
+
+
+def test_argparse_psm_accepted_as_integer():
+    """`--psm 6` is parsed as an integer.
+
+    Range and semantics belong to Tesseract / docling; this server passes the
+    integer through unchanged. Out-of-range values surface from docling /
+    Tesseract at conversion time, not here.
+    """
+    args = _build_parser_subset().parse_args(
+        ["--ocr-engine", "tesseract", "--psm", "6"]
+    )
+    assert args.psm == 6
+
+
+# ---------- _check_ocr_engine_available ----------
+
+def test_engine_check_easyocr_always_ok():
+    """easyocr ships with the `[hybrid]` extra; check is a no-op."""
+    ok, msg = hybrid_server._check_ocr_engine_available("easyocr")
+    assert ok is True
+    assert msg == ""
+
+
+def test_engine_check_auto_always_ok():
+    """`auto` defers engine choice to docling; check is a no-op."""
+    ok, msg = hybrid_server._check_ocr_engine_available("auto")
+    assert ok is True
+    assert msg == ""
+
+
+def test_engine_check_unknown_kind_returns_false():
+    """Unknown engine kinds fail closed instead of silently passing.
+
+    Before this contract was tightened, the helper returned `(True, "")` for any
+    engine name it did not recognize, so a docling release that registered a
+    new engine kind would slip past the probe and fail at first conversion.
+    Locks in the fail-closed behavior.
+    """
+    ok, msg = hybrid_server._check_ocr_engine_available("hypothetical_new_engine")
+    assert ok is False
+    assert "hypothetical_new_engine" in msg
+    # Maintainer-targeted hint: where to add the probe branch.
+    assert "_check_ocr_engine_available" in msg
+
+
+def test_engine_check_tesseract_missing_binary():
+    """`tesseract` engine without the binary on PATH fails with an actionable message."""
+    with patch("shutil.which", return_value=None):
+        ok, msg = hybrid_server._check_ocr_engine_available("tesseract")
+    assert ok is False
+    assert "tesseract" in msg.lower()
+    assert "PATH" in msg
+
+
+def test_engine_check_tesseract_present():
+    """`tesseract` engine passes when the binary resolves on PATH."""
+    with patch("shutil.which", return_value="/usr/bin/tesseract"):
+        ok, msg = hybrid_server._check_ocr_engine_available("tesseract")
+    assert ok is True
+    assert msg == ""
+
+
+def test_engine_check_tesserocr_missing_package():
+    """`tesserocr` engine without the Python package fails with install hint."""
+    with patch("importlib.util.find_spec", return_value=None):
+        ok, msg = hybrid_server._check_ocr_engine_available("tesserocr")
+    assert ok is False
+    assert "tesserocr" in msg
+    assert "pip install" in msg
+
+
+def test_engine_check_rapidocr_missing_package():
+    """`rapidocr` engine without the Python package fails with install hint."""
+    with patch("importlib.util.find_spec", return_value=None):
+        ok, msg = hybrid_server._check_ocr_engine_available("rapidocr")
+    assert ok is False
+    assert "rapidocr" in msg
+    assert "onnxruntime" in msg
+
+
+def test_engine_check_rapidocr_missing_onnxruntime():
+    """`rapidocr` installed without `onnxruntime` fails with a backend-specific hint."""
+    # Return a truthy spec for rapidocr, None for onnxruntime, None for anything else.
+    def fake_find_spec(name):
+        return object() if name == "rapidocr" else None
+
+    with patch("importlib.util.find_spec", side_effect=fake_find_spec):
+        ok, msg = hybrid_server._check_ocr_engine_available("rapidocr")
+    assert ok is False
+    assert "onnxruntime" in msg
+    # The message should not duplicate the "rapidocr is not installed" wording.
+    assert "rapidocr` Python package" not in msg
+
+
+def test_engine_check_ocrmac_off_macos():
+    """`ocrmac` selected on a non-Darwin platform fails with a clear platform message."""
+    with patch("sys.platform", "linux"):
+        ok, msg = hybrid_server._check_ocr_engine_available("ocrmac")
+    assert ok is False
+    assert "macOS" in msg
+
+
+def test_engine_check_ocrmac_on_macos_missing_package():
+    """`ocrmac` on macOS without the `ocrmac` Python package fails with install hint."""
+    with patch("sys.platform", "darwin"), patch(
+        "importlib.util.find_spec", return_value=None
+    ):
+        ok, msg = hybrid_server._check_ocr_engine_available("ocrmac")
+    assert ok is False
+    assert "ocrmac" in msg
+    assert "pip install" in msg
+    # Should NOT be the platform-mismatch message.
+    assert "macOS only" not in msg and "not macOS" not in msg
+
+
+# ---------- --no-ocr ignored-flag warning ----------
+
+def _run_main_to_warning(argv, monkeypatch, caplog):
+    """Drive `main()` far enough to capture the --no-ocr warning, then short-circuit.
+
+    `main()` ends in `uvicorn.run(...)`; we patch `create_app` and `uvicorn.run`
+    so the function returns cleanly after argparse + the warning logic without
+    starting a server.
+    """
+    monkeypatch.setattr("sys.argv", ["opendataloader-pdf-hybrid", *argv])
+
+    # Skip dep import checks — pytest already runs inside the [hybrid] env.
+    monkeypatch.setattr(hybrid_server, "_check_dependencies", lambda: None)
+
+    # Avoid building a real DocumentConverter / FastAPI app.
+    monkeypatch.setattr(hybrid_server, "create_app", lambda **kwargs: object())
+
+    # Stub uvicorn so main() returns instead of starting a server.
+    fake_uvicorn = type("FakeUvicorn", (), {"run": staticmethod(lambda *a, **k: None)})
+    monkeypatch.setitem(__import__("sys").modules, "uvicorn", fake_uvicorn)
+
+    caplog.set_level("WARNING", logger=hybrid_server.logger.name)
+    hybrid_server.main()
+
+
+def test_no_ocr_warns_when_engine_explicitly_set(monkeypatch, caplog):
+    """`--no-ocr --ocr-engine tesseract` warns that --ocr-engine has no effect."""
+    _run_main_to_warning(
+        ["--no-ocr", "--ocr-engine", "tesseract"], monkeypatch, caplog
+    )
+    warnings = [r.message for r in caplog.records if r.levelname == "WARNING"]
+    assert any("--ocr-engine tesseract" in w for w in warnings), warnings
+
+
+def test_no_ocr_warns_when_engine_explicitly_set_to_easyocr(monkeypatch, caplog):
+    """Explicit `--ocr-engine easyocr` under --no-ocr is still inert and must warn.
+
+    argparse cannot distinguish "user typed easyocr" from "default was used", so
+    main() inspects argv directly. This regression test locks the path in.
+    """
+    _run_main_to_warning(
+        ["--no-ocr", "--ocr-engine", "easyocr"], monkeypatch, caplog
+    )
+    warnings = [r.message for r in caplog.records if r.levelname == "WARNING"]
+    assert any("--ocr-engine easyocr" in w for w in warnings), warnings
+
+
+def test_no_ocr_no_warning_when_engine_left_default(monkeypatch, caplog):
+    """`--no-ocr` without --ocr-engine does not falsely report easyocr as inert."""
+    _run_main_to_warning(["--no-ocr"], monkeypatch, caplog)
+    warnings = [r.message for r in caplog.records if r.levelname == "WARNING"]
+    # No "--ocr-engine" mention because the user did not type it.
+    assert not any("--ocr-engine" in w for w in warnings), warnings
+
+
+def test_no_ocr_warns_when_ocr_lang_set(monkeypatch, caplog):
+    """`--no-ocr --ocr-lang ko` warns that --ocr-lang has no effect."""
+    _run_main_to_warning(["--no-ocr", "--ocr-lang", "ko"], monkeypatch, caplog)
+    warnings = [r.message for r in caplog.records if r.levelname == "WARNING"]
+    assert any("--ocr-lang" in w for w in warnings), warnings
+
+
+def test_no_ocr_warns_when_psm_set(monkeypatch, caplog):
+    """`--no-ocr --psm 6` warns that --psm has no effect."""
+    _run_main_to_warning(["--no-ocr", "--psm", "6"], monkeypatch, caplog)
+    warnings = [r.message for r in caplog.records if r.levelname == "WARNING"]
+    assert any("--psm 6" in w for w in warnings), warnings
+
+
+def test_no_ocr_alone_emits_no_warning(monkeypatch, caplog):
+    """`--no-ocr` on its own does not warn — there are no inert flags to call out."""
+    _run_main_to_warning(["--no-ocr"], monkeypatch, caplog)
+    warnings = [
+        r.message
+        for r in caplog.records
+        if r.levelname == "WARNING" and "no effect" in r.message
+    ]
+    assert warnings == []
+
+
+# ---------- main() exits when engine prerequisites are missing ----------
+
+def test_main_exits_when_tesseract_binary_missing(monkeypatch, caplog):
+    """Selecting `--ocr-engine tesseract` without the binary on PATH exits at startup."""
+    monkeypatch.setattr(
+        "sys.argv", ["opendataloader-pdf-hybrid", "--ocr-engine", "tesseract"]
+    )
+    monkeypatch.setattr(hybrid_server, "_check_dependencies", lambda: None)
+    monkeypatch.setattr(hybrid_server, "create_app", lambda **kwargs: object())
+    monkeypatch.setattr("shutil.which", lambda _name: None)
+    fake_uvicorn = type("FakeUvicorn", (), {"run": staticmethod(lambda *a, **k: None)})
+    monkeypatch.setitem(__import__("sys").modules, "uvicorn", fake_uvicorn)
+
+    caplog.set_level("ERROR", logger=hybrid_server.logger.name)
+    with pytest.raises(SystemExit) as excinfo:
+        hybrid_server.main()
+    assert excinfo.value.code == 2
+    errors = [r.message for r in caplog.records if r.levelname == "ERROR"]
+    assert any("tesseract" in e.lower() for e in errors), errors
+
+
+def test_main_skips_engine_check_when_no_ocr(monkeypatch, caplog):
+    """`--no-ocr --ocr-engine tesseract` skips the binary check entirely."""
+    called = {"which": False}
+
+    def spy_which(_name):
+        called["which"] = True
+        return None  # Would fail the check if it were called.
+
+    monkeypatch.setattr(
+        "sys.argv",
+        ["opendataloader-pdf-hybrid", "--no-ocr", "--ocr-engine", "tesseract"],
+    )
+    monkeypatch.setattr(hybrid_server, "_check_dependencies", lambda: None)
+    monkeypatch.setattr(hybrid_server, "create_app", lambda **kwargs: object())
+    monkeypatch.setattr("shutil.which", spy_which)
+    fake_uvicorn = type("FakeUvicorn", (), {"run": staticmethod(lambda *a, **k: None)})
+    monkeypatch.setitem(__import__("sys").modules, "uvicorn", fake_uvicorn)
+
+    hybrid_server.main()  # must not raise SystemExit
+    assert called["which"] is False

python/opendataloader-pdf/tests/test_hybrid_server_partial_success.py

@@ -0,0 +1,299 @@
+"""Tests for PARTIAL_SUCCESS handling in hybrid server responses.
+
+Validates that when Docling encounters errors during PDF preprocessing
+(e.g., Invalid code point), the hybrid server correctly reports:
+- partial_success status instead of success
+- list of failed page numbers
+- error messages from Docling
+"""
+
+from opendataloader_pdf.hybrid_server import (
+    _extract_failed_pages_from_errors,
+    build_conversion_response,
+)
+
+
+class TestBuildConversionResponse:
+    """Tests for the build_conversion_response function."""
+
+    def test_success_status(self):
+        """Fully successful conversion should return status=success."""
+        response = build_conversion_response(
+            status_value="success",
+            json_content={"pages": {"1": {}, "2": {}, "3": {}}},
+            processing_time=1.5,
+            errors=[],
+            requested_pages=None,
+        )
+        assert response["status"] == "success"
+        assert response["failed_pages"] == []
+        assert response["processing_time"] == 1.5
+
+    def test_partial_success_status(self):
+        """PARTIAL_SUCCESS should return status=partial_success with failed pages."""
+        response = build_conversion_response(
+            status_value="partial_success",
+            json_content={"pages": {"1": {}, "2": {}, "4": {}, "5": {}}},
+            processing_time=2.0,
+            errors=["Unknown page: pipeline terminated early"],
+            requested_pages=(1, 5),
+        )
+        assert response["status"] == "partial_success"
+        assert response["failed_pages"] == [3]
+        assert response["errors"] == ["Unknown page: pipeline terminated early"]
+
+    def test_partial_success_multiple_failed_pages(self):
+        """Multiple failed pages should all be reported."""
+        response = build_conversion_response(
+            status_value="partial_success",
+            json_content={"pages": {"1": {}, "3": {}, "5": {}}},
+            processing_time=3.0,
+            errors=[
+                "Unknown page: pipeline terminated early",
+                "Unknown page: pipeline terminated early",
+            ],
+            requested_pages=(1, 5),
+        )
+        assert response["status"] == "partial_success"
+        assert sorted(response["failed_pages"]) == [2, 4]
+
+    def test_partial_success_no_page_range_with_total_pages(self):
+        """When total_pages is provided, boundary page failures are detected."""
+        # 5-page document, page 1 (first) and page 5 (last) failed
+        response = build_conversion_response(
+            status_value="partial_success",
+            json_content={"pages": {"2": {}, "3": {}, "4": {}}},
+            processing_time=2.0,
+            errors=["error1", "error2"],
+            requested_pages=None,
+            total_pages=5,
+        )
+        assert response["status"] == "partial_success"
+        assert response["failed_pages"] == [1, 5]
+
+    def test_partial_success_no_page_range_fallback(self):
+        """When no page range or total_pages, interior gaps are still detected."""
+        response = build_conversion_response(
+            status_value="partial_success",
+            json_content={"pages": {"1": {}, "2": {}, "4": {}, "5": {}}},
+            processing_time=2.0,
+            errors=["Unknown page: pipeline terminated early"],
+            requested_pages=None,
+        )
+        assert response["status"] == "partial_success"
+        assert response["failed_pages"] == [3]
+
+    def test_success_no_errors_field(self):
+        """Successful conversion should have empty errors list."""
+        response = build_conversion_response(
+            status_value="success",
+            json_content={"pages": {"1": {}, "2": {}}},
+            processing_time=1.0,
+            errors=[],
+            requested_pages=None,
+        )
+        assert response["errors"] == []
+
+    def test_document_field_present(self):
+        """Response should contain document.json_content."""
+        json_content = {"pages": {"1": {}}, "body": {"text": "hello"}}
+        response = build_conversion_response(
+            status_value="success",
+            json_content=json_content,
+            processing_time=1.0,
+            errors=[],
+            requested_pages=None,
+        )
+        assert response["document"]["json_content"] == json_content
+
+    def test_partial_success_first_page_failed_with_page_range(self):
+        """First page failure should be detected when page range is specified."""
+        response = build_conversion_response(
+            status_value="partial_success",
+            json_content={"pages": {"2": {}, "3": {}}},
+            processing_time=1.0,
+            errors=["error"],
+            requested_pages=(1, 3),
+        )
+        assert response["failed_pages"] == [1]
+
+    def test_partial_success_last_page_failed_with_page_range(self):
+        """Last page failure should be detected when page range is specified."""
+        response = build_conversion_response(
+            status_value="partial_success",
+            json_content={"pages": {"1": {}, "2": {}}},
+            processing_time=1.0,
+            errors=["error"],
+            requested_pages=(1, 3),
+        )
+        assert response["failed_pages"] == [3]
+
+    def test_partial_success_all_pages_failed(self):
+        """All pages failing should report every page in failed_pages."""
+        response = build_conversion_response(
+            status_value="partial_success",
+            json_content={"pages": {}},
+            processing_time=2.0,
+            errors=["error1", "error2", "error3"],
+            requested_pages=(1, 3),
+        )
+        assert response["status"] == "partial_success"
+        assert response["failed_pages"] == [1, 2, 3]
+
+    def test_partial_success_all_pages_failed_with_total_pages(self):
+        """All pages failing with total_pages should report every page."""
+        response = build_conversion_response(
+            status_value="partial_success",
+            json_content={"pages": {}},
+            processing_time=2.0,
+            errors=["error1", "error2"],
+            requested_pages=None,
+            total_pages=3,
+        )
+        assert response["status"] == "partial_success"
+        assert response["failed_pages"] == [1, 2, 3]
+
+    def test_failure_status_no_failed_pages_detection(self):
+        """Failure status should not trigger failed page detection."""
+        response = build_conversion_response(
+            status_value="failure",
+            json_content={"pages": {"1": {}}},
+            processing_time=1.0,
+            errors=["PDF conversion failed"],
+            requested_pages=(1, 3),
+        )
+        assert response["status"] == "failure"
+        assert response["failed_pages"] == []
+
+    def test_partial_success_missing_pages_key(self):
+        """json_content without 'pages' key should mark all requested pages as failed."""
+        response = build_conversion_response(
+            status_value="partial_success",
+            json_content={"body": {"text": "hello"}},
+            processing_time=1.0,
+            errors=["error"],
+            requested_pages=(1, 3),
+        )
+        assert response["status"] == "partial_success"
+        assert response["failed_pages"] == [1, 2, 3]
+
+
+class TestExtractFailedPagesFromErrors:
+    """Tests for error message-based failed page detection."""
+
+    def test_std_bad_alloc_errors(self):
+        """Page numbers should be extracted from 'Page N: std::bad_alloc' messages."""
+        errors = [
+            "Page 26: std::bad_alloc",
+            "Page 27: std::bad_alloc",
+            "Page 28: std::bad_alloc",
+        ]
+        assert _extract_failed_pages_from_errors(errors) == [26, 27, 28]
+
+    def test_mixed_error_formats(self):
+        """Only 'Page N:' prefixed messages should be matched."""
+        errors = [
+            "Page 5: Invalid code point",
+            "Unknown page: pipeline terminated early",
+            "Page 10: std::bad_alloc",
+        ]
+        assert _extract_failed_pages_from_errors(errors) == [5, 10]
+
+    def test_no_page_errors(self):
+        """Non-page errors should return empty list."""
+        errors = [
+            "Unknown page: pipeline terminated early",
+            "General error occurred",
+        ]
+        assert _extract_failed_pages_from_errors(errors) == []
+
+    def test_bare_page_no_colon(self):
+        """Bare 'Page N' (no colon) should be matched when error_msg is empty."""
+        assert _extract_failed_pages_from_errors(["Page 26"]) == [26]
+
+
+class TestBuildConversionResponseErrorParsing:
+    """Tests for failed page detection via error message parsing.
+
+    When docling includes failed pages as empty entries in the pages dict,
+    gap detection fails. Error message parsing handles this case.
+    """
+
+    def test_failed_pages_with_empty_entries_in_pages_dict(self):
+        """Failed pages present as empty entries should still be detected via errors."""
+        response = build_conversion_response(
+            status_value="partial_success",
+            json_content={"pages": {"1": {}, "2": {}, "3": {}, "4": {}, "5": {}}},
+            processing_time=2.0,
+            errors=["Page 4: std::bad_alloc", "Page 5: std::bad_alloc"],
+            requested_pages=(1, 5),
+        )
+        assert response["failed_pages"] == [4, 5]
+
+    def test_boundary_pages_detected_via_errors(self):
+        """Boundary page failures should be detected even without page range."""
+        response = build_conversion_response(
+            status_value="partial_success",
+            json_content={"pages": {"1": {}, "2": {}, "3": {}, "4": {}, "5": {}}},
+            processing_time=2.0,
+            errors=["Page 4: std::bad_alloc", "Page 5: std::bad_alloc"],
+            requested_pages=None,
+            total_pages=None,
+        )
+        assert response["failed_pages"] == [4, 5]
+
+    def test_both_strategies_combined(self):
+        """Union of error-parsed and gap-detected pages should be reported.
+
+        Page 2 is missing from dict (gap), pages 4-5 have error messages
+        but are present as empty entries. All three must appear.
+        """
+        response = build_conversion_response(
+            status_value="partial_success",
+            json_content={"pages": {"1": {}, "3": {}, "4": {}, "5": {}}},
+            processing_time=2.0,
+            errors=["Page 4: std::bad_alloc", "Page 5: std::bad_alloc"],
+            requested_pages=(1, 5),
+        )
+        assert response["failed_pages"] == [2, 4, 5]
+
+    def test_overlap_between_gap_and_error_is_deduplicated(self):
+        """Same page from gap and error parsing should appear once."""
+        response = build_conversion_response(
+            status_value="partial_success",
+            json_content={"pages": {"1": {}, "3": {}}},
+            processing_time=1.0,
+            errors=["Page 2: std::bad_alloc"],
+            requested_pages=(1, 3),
+        )
+        assert response["failed_pages"] == [2]
+
+    def test_duplicate_page_in_errors(self):
+        """Duplicate page numbers in error messages should be deduplicated."""
+        errors = [
+            "Page 3: std::bad_alloc",
+            "Page 3: Invalid code point",
+        ]
+        assert _extract_failed_pages_from_errors(errors) == [3]
+
+    def test_no_page_pattern_errors_falls_back_to_gap(self):
+        """When errors lack 'Page N:' pattern, gap detection should still work."""
+        response = build_conversion_response(
+            status_value="partial_success",
+            json_content={"pages": {"1": {}, "3": {}}},
+            processing_time=1.0,
+            errors=["Unknown page: pipeline terminated early"],
+            requested_pages=(1, 3),
+        )
+        assert response["failed_pages"] == [2]
+
+    def test_empty_errors_with_partial_success(self):
+        """Partial success with no error messages should still detect gaps."""
+        response = build_conversion_response(
+            status_value="partial_success",
+            json_content={"pages": {"1": {}, "3": {}}},
+            processing_time=1.0,
+            errors=[],
+            requested_pages=(1, 3),
+        )
+        assert response["failed_pages"] == [2]

python/opendataloader-pdf/tests/test_hybrid_server_picture_description.py

@@ -0,0 +1,101 @@
+"""Tests for `--picture-description-prompt` propagation in the hybrid server.
+
+Covers PDFDLOSP-20: the CLI accepted `--picture-description-prompt` without
+error but the prompt was never written into `PictureDescriptionVlmOptions`,
+so output was byte-identical regardless of the user's prompt. These tests
+lock in both halves of the contract:
+
+    * A custom prompt reaches `PictureDescriptionVlmOptions.prompt`.
+    * Omitting the prompt preserves docling's built-in default.
+    * Blank / whitespace-only prompts are treated as "not provided" so they
+      never silently inject an empty prompt into the VLM.
+    * The whole feature is gated by `enrich_picture_description=True`.
+"""
+
+from unittest.mock import patch
+
+import pytest
+
+from opendataloader_pdf import hybrid_server
+
+
+def _capture_pipeline_options(**kwargs):
+    """Build a converter while mocking out docling's heavy bits so we can
+    introspect the PdfPipelineOptions instance.
+    """
+    captured = {}
+
+    def fake_pdf_format_option(*, pipeline_options):
+        captured["pipeline_options"] = pipeline_options
+        return object()
+
+    with patch(
+        "docling.document_converter.DocumentConverter"
+    ) as mock_dc, patch(
+        "docling.document_converter.PdfFormatOption", side_effect=fake_pdf_format_option
+    ):
+        mock_dc.return_value = object()
+        hybrid_server.create_converter(**kwargs)
+
+    return captured["pipeline_options"]
+
+
+def test_custom_prompt_is_forwarded_to_vlm_options():
+    """The user-supplied prompt must end up on PictureDescriptionVlmOptions.
+
+    Regression for PDFDLOSP-20.
+    """
+    opts = _capture_pipeline_options(
+        enrich_picture_description=True,
+        picture_description_prompt="HELLO WORLD",
+    )
+    assert opts.picture_description_options is not None
+    assert opts.picture_description_options.prompt == "HELLO WORLD"
+
+
+def test_default_prompt_is_preserved_when_user_omits_flag():
+    """When no prompt is given, docling's built-in default must survive.
+
+    Locks the current docling default. If a docling upgrade changes the
+    phrase, this canary fires — at which point we decide whether to track
+    the new default or pin our own.
+    """
+    opts = _capture_pipeline_options(enrich_picture_description=True)
+    assert opts.picture_description_options is not None
+    assert (
+        opts.picture_description_options.prompt
+        == "Describe this image in a few sentences."
+    )
+
+
+@pytest.mark.parametrize("blank", ["", "   ", "\t\n"])
+def test_blank_prompt_falls_back_to_default(blank):
+    """Empty / whitespace-only prompts must not inject an empty prompt.
+
+    Otherwise we recreate the same class of silent-flag misbehavior that
+    PDFDLOSP-20 reported.
+    """
+    opts = _capture_pipeline_options(
+        enrich_picture_description=True,
+        picture_description_prompt=blank,
+    )
+    assert opts.picture_description_options is not None
+    assert (
+        opts.picture_description_options.prompt
+        == "Describe this image in a few sentences."
+    )
+
+
+def test_prompt_ignored_when_enrichment_disabled():
+    """Without --enrich-picture-description, the prompt has no destination.
+
+    Docling populates a default `picture_description_options` object on
+    `PdfPipelineOptions` even when do_picture_description=False, so we
+    assert on the gate (`do_picture_description`) rather than identity of
+    the options object.
+    """
+    opts = _capture_pipeline_options(
+        enrich_picture_description=False,
+        picture_description_prompt="HELLO WORLD",
+    )
+    assert opts.do_picture_description is False

python/opendataloader-pdf/tests/test_hybrid_server_unicode.py

@@ -0,0 +1,86 @@
+"""Tests for Unicode sanitization in hybrid server responses.
+
+Validates that lone surrogates and null characters from Docling OCR output
+are sanitized before JSON serialization to prevent UnicodeEncodeError in
+Starlette's JSONResponse.render().
+"""
+
+import json
+
+import pytest
+
+from opendataloader_pdf.hybrid_server import sanitize_unicode
+
+
+class TestSanitizeUnicode:
+    """Tests for the sanitize_unicode function."""
+
+    def test_lone_surrogate_replaced(self):
+        """Lone surrogates should be replaced with U+FFFD."""
+        data = {"text": "Hello \ud800 World"}
+        result = sanitize_unicode(data)
+        assert "\ud800" not in result["text"]
+        assert "\ufffd" in result["text"]
+
+    def test_all_surrogate_range_replaced(self):
+        """All surrogate code points (U+D800 to U+DFFF) should be replaced."""
+        data = {"text": "\ud800\udbff\udc00\udfff"}
+        result = sanitize_unicode(data)
+        assert result["text"] == "\ufffd" * 4
+
+    def test_null_character_replaced(self):
+        """Null characters should be replaced with U+FFFD."""
+        data = {"text": "Hello\x00World"}
+        result = sanitize_unicode(data)
+        assert "\x00" not in result["text"]
+        assert result["text"] == "Hello\ufffdWorld"
+
+    def test_nested_dict_sanitized(self):
+        """Nested dictionaries should be sanitized recursively."""
+        data = {"level1": {"level2": {"text": "bad\ud800char"}}}
+        result = sanitize_unicode(data)
+        assert "\ud800" not in result["level1"]["level2"]["text"]
+        assert "\ufffd" in result["level1"]["level2"]["text"]
+
+    def test_list_sanitized(self):
+        """Lists within the data should be sanitized."""
+        data = {"items": ["good", "bad\ud800text", "also\x00bad"]}
+        result = sanitize_unicode(data)
+        assert result["items"][0] == "good"
+        assert "\ud800" not in result["items"][1]
+        assert "\x00" not in result["items"][2]
+
+    def test_clean_data_unchanged(self):
+        """Clean data without problematic characters should pass through unchanged."""
+        data = {"text": "Hello World", "number": 42, "flag": True, "nothing": None}
+        result = sanitize_unicode(data)
+        assert result == data
+
+    def test_non_string_values_preserved(self):
+        """Non-string values (int, float, bool, None) should be preserved as-is."""
+        data = {"int": 42, "float": 3.14, "bool": True, "none": None}
+        result = sanitize_unicode(data)
+        assert result == data
+
+    def test_sanitized_output_json_serializable(self):
+        """Sanitized output must survive json.dumps + encode('utf-8') without error."""
+        data = {
+            "status": "success",
+            "document": {
+                "json_content": {
+                    "body": {"text": "OCR text with \ud800 lone surrogate and \x00 null"}
+                }
+            },
+        }
+        result = sanitize_unicode(data)
+        # This is the exact operation that Starlette's JSONResponse.render() performs
+        json_bytes = json.dumps(result, ensure_ascii=False).encode("utf-8")
+        assert isinstance(json_bytes, bytes)
+
+    def test_mixed_valid_and_invalid_unicode(self):
+        """Valid Unicode (including CJK, emoji) should be preserved alongside sanitization."""
+        data = {"text": "Valid \u4e16\u754c \ud800 text"}
+        result = sanitize_unicode(data)
+        assert "\u4e16\u754c" in result["text"]  # CJK preserved
+        assert "\ud800" not in result["text"]  # surrogate removed
+        assert "\ufffd" in result["text"]  # replacement added

python/opendataloader-pdf/tests/test_runner.py

@@ -0,0 +1,153 @@
+"""Unit tests for runner.py error-handling behaviour.
+
+Regression: when the JAR fails, the streaming branch already wrote the
+JAR's stdout to the console live, so the except handler must not re-emit
+the captured copy. The quiet branch, conversely, has not surfaced anything
+yet and is allowed to print the captured streams — but only once
+(``CalledProcessError.output`` and ``.stdout`` are the same attribute).
+"""
+
+import io
+import subprocess
+from unittest.mock import MagicMock
+
+import pytest
+
+from opendataloader_pdf import runner
+
+
+class _FakeAsFile:
+    def __init__(self, path):
+        self._path = path
+
+    def __enter__(self):
+        return self._path
+
+    def __exit__(self, *_args):
+        return False
+
+
+@pytest.fixture
+def patched_jar(monkeypatch, tmp_path):
+    """Bypass the real resources lookup so run_jar reaches subprocess."""
+    fake_jar = tmp_path / "opendataloader-pdf-cli.jar"
+    fake_jar.write_bytes(b"")
+    fake_traversable = MagicMock()
+    fake_traversable.joinpath = lambda *_a, **_kw: fake_jar
+    monkeypatch.setattr(runner.resources, "files", lambda _pkg: fake_traversable)
+    monkeypatch.setattr(runner.resources, "as_file", lambda p: _FakeAsFile(p))
+
+
+def test_streaming_failure_does_not_duplicate_output(monkeypatch, capsys, patched_jar):
+    """Streaming mode prints JAR output live; the except handler must not
+    re-emit the captured copy on stderr."""
+    jar_output = "Invalid page range format: '-10'\nusage: [options] ...\n"
+
+    fake_process = MagicMock()
+    fake_process.stdout = iter([jar_output])
+    fake_process.wait.return_value = 2
+    fake_process.__enter__ = lambda self: self
+    fake_process.__exit__ = lambda self, *_a: False
+
+    monkeypatch.setattr(runner.subprocess, "Popen", lambda *_a, **_kw: fake_process)
+
+    with pytest.raises(subprocess.CalledProcessError):
+        runner.run_jar(["--bogus"], quiet=False)
+
+    captured = capsys.readouterr()
+    # JAR text appears exactly once: the live streaming write.
+    assert "Invalid page range format" in captured.out
+    assert captured.out.count("usage: [options]") == 1
+    # The except handler did NOT re-emit the captured copy on stderr.
+    assert "Invalid page range format" not in captured.err
+    assert "usage: [options]" not in captured.err
+    # Meta info is still surfaced.
+    assert "Error running opendataloader-pdf CLI." in captured.err
+    assert "Return code: 2" in captured.err
+
+
+def test_quiet_success_relays_stdout_but_not_stderr(monkeypatch, capsys, patched_jar):
+    """Quiet mode must relay the JAR's stdout (--to-stdout payload, folder
+    summary line) to the caller while still suppressing the JAR's log
+    stream (stderr). Regression: --quiet + --to-stdout produced no output,
+    breaking pipe consumers."""
+    result = subprocess.CompletedProcess(
+        args=["java", "-jar", "fake.jar"],
+        returncode=0,
+        stdout="extracted text payload\n",
+        stderr="[INFO] java log noise\n",
+    )
+    monkeypatch.setattr(runner.subprocess, "run", MagicMock(return_value=result))
+
+    returned = runner.run_jar(["doc.pdf", "--quiet", "--to-stdout"], quiet=True)
+
+    captured = capsys.readouterr()
+    # Payload reaches the caller's stdout exactly once.
+    assert captured.out.count("extracted text payload") == 1
+    # The JAR's log stream stays suppressed (that is what quiet means).
+    assert "java log noise" not in captured.out
+    assert "java log noise" not in captured.err
+    # Return value is unchanged for library callers.
+    assert returned == "extracted text payload\n"
+
+
+def test_quiet_relays_through_stdout_buffer_byte_path(monkeypatch, patched_jar):
+    """The production CLI writes the relayed payload through
+    ``sys.stdout.buffer`` (bytes), not the ``sys.stdout.write`` (str)
+    fallback. ``capsys`` replaces ``sys.stdout`` with an object whose
+    ``.buffer`` semantics differ, so the sibling test only exercises the
+    str branch. This drives the real byte-relay path and asserts the
+    payload is encoded to UTF-8 and written exactly once, intact — covering
+    the ``encode("utf-8", "replace")`` step that the str branch skips.
+    """
+    payload = "héllo 한글 payload\n"  # non-ASCII: exercises the utf-8 encode
+    result = subprocess.CompletedProcess(
+        args=["java", "-jar", "fake.jar"],
+        returncode=0,
+        stdout=payload,
+        stderr="[INFO] java log noise\n",
+    )
+    monkeypatch.setattr(runner.subprocess, "run", MagicMock(return_value=result))
+
+    # Fake stdout with a real binary buffer, mirroring a genuine TextIOWrapper
+    # (hasattr(sys.stdout, "buffer") is True), so run_jar takes the byte path.
+    fake_stdout = MagicMock()
+    fake_stdout.buffer = io.BytesIO()
+    monkeypatch.setattr(runner.sys, "stdout", fake_stdout)
+
+    returned = runner.run_jar(["doc.pdf", "--quiet", "--to-stdout"], quiet=True)
+
+    written = fake_stdout.buffer.getvalue()
+    # Byte path was taken: payload written as UTF-8 exactly once, intact.
+    assert written == payload.encode("utf-8")
+    assert written.decode("utf-8") == payload
+    # The str fallback branch was NOT used.
+    fake_stdout.write.assert_not_called()
+    fake_stdout.buffer.flush()  # buffer is flushed by run_jar; no error
+    # Return value is unchanged for library callers.
+    assert returned == payload
+
+
+def test_quiet_failure_prints_captured_streams_once(monkeypatch, capsys, patched_jar):
+    """Quiet mode captures output, so the except handler surfaces it — but
+    must avoid the old bug where Output and Stdout (aliases) both printed."""
+    error = subprocess.CalledProcessError(
+        returncode=2,
+        cmd=["java", "-jar", "fake.jar"],
+        output="captured stdout text",
+        stderr="captured stderr text",
+    )
+    monkeypatch.setattr(runner.subprocess, "run", MagicMock(side_effect=error))
+
+    with pytest.raises(subprocess.CalledProcessError):
+        runner.run_jar(["--bogus"], quiet=True)
+
+    err = capsys.readouterr().err
+    assert err.count("captured stdout text") == 1
+    assert err.count("captured stderr text") == 1
+    # The pre-fix code printed both "Output:" and "Stdout:" with the same text.
+    assert "Output:" not in err
+    assert "Stdout: captured stdout text" in err
+    assert "Stderr: captured stderr text" in err
+    assert "Error running opendataloader-pdf CLI." in err
+    assert "Return code: 2" in err