chore: import upstream snapshot with attribution

This commit is contained in:
wehub-resource-sync
2026-07-13 12:59:42 +08:00
commit 59f8f60dad
348 changed files with 139133 additions and 0 deletions
+45
View File
@@ -0,0 +1,45 @@
# Batch Processing Example
Demonstrates processing multiple PDFs in a single invocation to avoid repeated Java JVM startup overhead.
## Prerequisites
- Python 3.10+
- Java 11+ (on PATH)
## Example
[`batch_processing.py`](batch_processing.py) shows two methods for batch conversion:
1. **File list** — Pass multiple PDF paths as a list
2. **Directory** — Pass a directory path (recursively finds all PDFs)
Both methods use a single JVM invocation, which is significantly faster than calling the CLI once per file.
**Run:**
```bash
pip install -r requirements.txt
python batch_processing.py
```
## Sample Output
```
Found 4 PDFs in pdf/
==========================================================
Method 1: Batch convert with file list
==========================================================
Document Pages Top-level
----------------------------------------------------------
1901.03003 15 241
2408.02509v1 14 365
chinese_scan 1 1
lorem 1 2
----------------------------------------------------------
Total 31 609
Processed 4 documents
Time: 7.95s (single JVM invocation)
```
+119
View File
@@ -0,0 +1,119 @@
#!/usr/bin/env python3
"""
Batch Processing Example
Demonstrates processing multiple PDFs in a single invocation to avoid
repeated Java JVM startup overhead. This is the recommended approach
for large-scale document pipelines.
Requires Python 3.10+.
Usage:
pip install opendataloader-pdf
python batch_processing.py
"""
from __future__ import annotations
import json
import tempfile
import time
from pathlib import Path
import opendataloader_pdf
def batch_convert(pdf_paths: list[str], output_dir: str) -> list[Path]:
"""Convert multiple PDFs in a single JVM invocation."""
opendataloader_pdf.convert(
input_path=pdf_paths,
output_dir=output_dir,
format="json,markdown",
quiet=True,
)
# Collect output JSON files
return sorted(Path(output_dir).glob("*.json"))
def convert_directory(directory: str, output_dir: str) -> list[Path]:
"""Convert all PDFs in a directory (recursive)."""
opendataloader_pdf.convert(
input_path=directory,
output_dir=output_dir,
format="json,markdown",
quiet=True,
)
return sorted(Path(output_dir).glob("*.json"))
def summarize_results(json_files: list[Path]) -> None:
"""Print a summary of all converted documents."""
total_pages = 0
total_elements = 0
print(f"\n{'Document':<40} {'Pages':>6} {'Top-level':>9}")
print("-" * 58)
for json_path in json_files:
with open(json_path, encoding="utf-8") as f:
doc = json.load(f)
pages = doc.get("number of pages", 0)
elements = len(doc.get("kids", []))
total_pages += pages
total_elements += elements
print(f"{json_path.stem:<40} {pages:>6} {elements:>9}")
print("-" * 58)
print(f"{'Total':<40} {total_pages:>6} {total_elements:>9}")
print(f"\nProcessed {len(json_files)} documents")
def main():
# Find sample PDFs relative to this script
script_dir = Path(__file__).resolve().parent
repo_root = script_dir.parent.parent.parent
samples_dir = repo_root / "samples" / "pdf"
pdf_files = sorted(samples_dir.glob("*.pdf"))
if not pdf_files:
print(f"No sample PDFs found at: {samples_dir}")
return
print(f"Found {len(pdf_files)} PDFs in {samples_dir.name}/")
for p in pdf_files:
print(f" - {p.name}")
# --- Method 1: Pass a list of files ---
print("\n" + "=" * 58)
print("Method 1: Batch convert with file list")
print("=" * 58)
with tempfile.TemporaryDirectory() as temp_dir:
start = time.perf_counter()
json_files = batch_convert(
[str(p) for p in pdf_files],
temp_dir,
)
elapsed = time.perf_counter() - start
summarize_results(json_files)
print(f"Time: {elapsed:.2f}s (single JVM invocation)")
# --- Method 2: Pass a directory ---
# Note: directory input recursively finds PDFs in subdirectories,
# so the file count may differ from Method 1 (which uses top-level glob).
print("\n" + "=" * 58)
print("Method 2: Convert entire directory")
print("=" * 58)
with tempfile.TemporaryDirectory() as temp_dir:
start = time.perf_counter()
json_files = convert_directory(str(samples_dir), temp_dir)
elapsed = time.perf_counter() - start
summarize_results(json_files)
print(f"Time: {elapsed:.2f}s (single JVM invocation)")
if __name__ == "__main__":
main()
+2
View File
@@ -0,0 +1,2 @@
# Requires Python 3.10+
opendataloader-pdf>=2.2.1