wehub/opendataloader-project--opendataloader-pdf
1
0
Fork 0
Code Issues Pull Requests Actions Deployments Models Agent Notes Packages Projects Releases Wiki Activity

chore: import upstream snapshot with attribution

Browse Source
This commit is contained in:
wehub-resource-sync
2026-07-13 12:59:42 +08:00
commit 59f8f60dad
348 changed files with 139133 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
examples/python/batch/README.md
+45
View File
@@ -0,0 +1,45 @@
# Batch Processing Example
Demonstrates processing multiple PDFs in a single invocation to avoid repeated Java JVM startup overhead.
## Prerequisites
- Python 3.10+
- Java 11+ (on PATH)
## Example
[`batch_processing.py`](batch_processing.py) shows two methods for batch conversion:
1. **File list** — Pass multiple PDF paths as a list
2. **Directory** — Pass a directory path (recursively finds all PDFs)
Both methods use a single JVM invocation, which is significantly faster than calling the CLI once per file.
**Run:**
```bash
pip install -r requirements.txt
python batch_processing.py
```
## Sample Output
```
Found 4 PDFs in pdf/
==========================================================
Method 1: Batch convert with file list
==========================================================
Document Pages Top-level
----------------------------------------------------------
1901.03003 15 241
2408.02509v1 14 365
chinese_scan 1 1
lorem 1 2
----------------------------------------------------------
Total 31 609
Processed 4 documents
Time: 7.95s (single JVM invocation)
```
examples/python/batch/batch_processing.py
+119
View File
@@ -0,0 +1,119 @@
#!/usr/bin/env python3
"""
Batch Processing Example
Demonstrates processing multiple PDFs in a single invocation to avoid
repeated Java JVM startup overhead. This is the recommended approach
for large-scale document pipelines.
Requires Python 3.10+.
Usage:
pip install opendataloader-pdf
python batch_processing.py
"""
from __future__ import annotations
import json
import tempfile
import time
from pathlib import Path
import opendataloader_pdf
def batch_convert(pdf_paths: list[str], output_dir: str) -> list[Path]:
"""Convert multiple PDFs in a single JVM invocation."""
opendataloader_pdf.convert(
input_path=pdf_paths,
output_dir=output_dir,
format="json,markdown",
quiet=True,
)
# Collect output JSON files
return sorted(Path(output_dir).glob("*.json"))
def convert_directory(directory: str, output_dir: str) -> list[Path]:
"""Convert all PDFs in a directory (recursive)."""
opendataloader_pdf.convert(
input_path=directory,
output_dir=output_dir,
format="json,markdown",
quiet=True,
)
return sorted(Path(output_dir).glob("*.json"))
def summarize_results(json_files: list[Path]) -> None:
"""Print a summary of all converted documents."""
total_pages = 0
total_elements = 0
print(f"\n{'Document':<40} {'Pages':>6} {'Top-level':>9}")
print("-" * 58)
for json_path in json_files:
with open(json_path, encoding="utf-8") as f:
doc = json.load(f)
pages = doc.get("number of pages", 0)
elements = len(doc.get("kids", []))
total_pages += pages
total_elements += elements
print(f"{json_path.stem:<40} {pages:>6} {elements:>9}")
print("-" * 58)
print(f"{'Total':<40} {total_pages:>6} {total_elements:>9}")
print(f"\nProcessed {len(json_files)} documents")
def main():
# Find sample PDFs relative to this script
script_dir = Path(__file__).resolve().parent
repo_root = script_dir.parent.parent.parent
samples_dir = repo_root / "samples" / "pdf"
pdf_files = sorted(samples_dir.glob("*.pdf"))
if not pdf_files:
print(f"No sample PDFs found at: {samples_dir}")
return
print(f"Found {len(pdf_files)} PDFs in {samples_dir.name}/")
for p in pdf_files:
print(f" - {p.name}")
# --- Method 1: Pass a list of files ---
print("\n" + "=" * 58)
print("Method 1: Batch convert with file list")
print("=" * 58)
with tempfile.TemporaryDirectory() as temp_dir:
start = time.perf_counter()
json_files = batch_convert(
[str(p) for p in pdf_files],
temp_dir,
)
elapsed = time.perf_counter() - start
summarize_results(json_files)
print(f"Time: {elapsed:.2f}s (single JVM invocation)")
# --- Method 2: Pass a directory ---
# Note: directory input recursively finds PDFs in subdirectories,
# so the file count may differ from Method 1 (which uses top-level glob).
print("\n" + "=" * 58)
print("Method 2: Convert entire directory")
print("=" * 58)
with tempfile.TemporaryDirectory() as temp_dir:
start = time.perf_counter()
json_files = convert_directory(str(samples_dir), temp_dir)
elapsed = time.perf_counter() - start
summarize_results(json_files)
print(f"Time: {elapsed:.2f}s (single JVM invocation)")
if __name__ == "__main__":
main()
examples/python/batch/requirements.txt
+2
View File
@@ -0,0 +1,2 @@
# Requires Python 3.10+
opendataloader-pdf>=2.2.1
examples/python/rag/README.md
+95
View File
@@ -0,0 +1,95 @@
# RAG Examples for OpenDataLoader PDF
Working examples demonstrating how to use OpenDataLoader PDF in RAG (Retrieval-Augmented Generation) pipelines.
## Prerequisites
- Python 3.10+
- Java 11+ (on PATH)
## Sample PDF
Examples use `samples/pdf/1901.03003.pdf` - a multi-page academic paper (arXiv:1901.03003) with:
- Two-column layout
- Multiple sections and headings
- Tables and figures
- Complex reading order
## Examples
### 1. Basic Chunking (No External Dependencies)
[`basic_chunking.py`](basic_chunking.py) demonstrates PDF-to-chunks conversion using only `opendataloader-pdf` and Python standard library. No external embedding or vector store dependencies.
**Features:**
- PDF to JSON conversion with reading order
- Three chunking strategies:
1. By element (paragraph, heading, list)
2. By section (grouped under headings)
3. Merged chunks (minimum size threshold)
- Bounding box metadata for citations
**Run:**
```bash
pip install opendataloader-pdf
python basic_chunking.py
```
### 2. LangChain Integration
[`langchain_example.py`](langchain_example.py) shows integration with the official LangChain loader.
**Features:**
- OpenDataLoaderPDFLoader usage
- Returns LangChain Document objects
- Ready for any LangChain pipeline
**Run:**
```bash
pip install -r requirements.txt
python langchain_example.py
```
## Sample Output
```
Processing: 1901.03003.pdf
==================================================
Document: 1901.03003.pdf
Pages: 9
Elements: 187
--- Strategy 1: Chunk by Element ---
Created 156 chunks
[1] RoBERTa: A Robustly Optimized BERT Pretraining Approach
Source: 1901.03003.pdf, Page 1, Position (108, 655)
[2] Yinhan Liu† Myle Ott† Naman Goyal† Jingfei Du† ...
Source: 1901.03003.pdf, Page 1, Position (142, 603)
--- Strategy 2: Chunk by Section ---
Created 12 chunks
Section: RoBERTa: A Robustly Optimized BERT Pretraining Approach
Section: 1 Introduction
Section: 2 Background
...
```
## Next Steps
After chunking, integrate with your preferred:
- **Embedding model**: OpenAI, Cohere, HuggingFace, etc.
- **Vector store**: Chroma, FAISS, Pinecone, Weaviate, etc.
Each chunk includes `text` and `metadata` ready for embedding:
```python
{
"text": "Language model pretraining has led to significant...",
"metadata": {
"type": "paragraph",
"page": 1,
"bbox": [108.0, 526.2, 286.5, 592.8],
"source": "1901.03003.pdf"
}
}
```
examples/python/rag/basic_chunking.py
+227
View File
@@ -0,0 +1,227 @@
#!/usr/bin/env python3
"""
Basic RAG Chunking Example - No External Dependencies
Demonstrates PDF-to-chunks conversion using only opendataloader-pdf
and Python standard library. Ready for integration with any embedding
model or vector store.
Usage:
pip install opendataloader-pdf
python basic_chunking.py
"""
import json
import tempfile
from pathlib import Path
import opendataloader_pdf
def convert_pdf_to_json(pdf_path: str, output_dir: str) -> Path:
"""Convert PDF to JSON and Markdown with reading order enabled."""
opendataloader_pdf.convert(
input_path=pdf_path,
output_dir=output_dir,
format="json,markdown",
reading_order="xycut",
quiet=True,
)
pdf_name = Path(pdf_path).stem
return Path(output_dir) / f"{pdf_name}.json"
def load_document(json_path: Path) -> dict:
"""Load the JSON output from OpenDataLoader."""
with open(json_path, encoding="utf-8") as f:
return json.load(f)
def chunk_by_element(doc: dict) -> list[dict]:
"""
Strategy 1: Chunk by semantic element.
Creates one chunk per paragraph, heading, or list element.
Best for: Fine-grained retrieval, precise citations.
"""
chunks = []
for element in doc.get("kids", []):
if element.get("type") in ("paragraph", "heading", "list"):
chunks.append({
"text": element.get("content", ""),
"metadata": {
"type": element["type"],
"page": element.get("page number"),
"bbox": element.get("bounding box"),
"source": doc.get("file name"),
}
})
return chunks
def chunk_by_section(doc: dict) -> list[dict]:
"""
Strategy 2: Chunk by heading/section.
Groups content under headings into coherent sections.
Best for: Context-rich retrieval, topic-based search.
"""
chunks = []
current_heading = None
current_content: list[str] = []
current_start_page = None
for element in doc.get("kids", []):
element_type = element.get("type")
if element_type == "heading":
# Save previous section
if current_content:
chunks.append({
"text": "\n".join(current_content),
"metadata": {
"heading": current_heading,
"page": current_start_page,
"source": doc.get("file name"),
}
})
current_heading = element.get("content", "")
current_content = [current_heading]
current_start_page = element.get("page number")
elif element_type in ("paragraph", "list"):
content = element.get("content", "")
if content:
current_content.append(content)
# Save the last section
if current_content:
chunks.append({
"text": "\n".join(current_content),
"metadata": {
"heading": current_heading,
"page": current_start_page,
"source": doc.get("file name"),
}
})
return chunks
def chunk_with_min_size(doc: dict, min_chars: int = 200) -> list[dict]:
"""
Strategy 3: Merge adjacent elements until minimum size.
Combines small paragraphs to avoid overly fragmented chunks.
Best for: Balanced chunk sizes, reducing noise.
"""
chunks = []
buffer_text = ""
buffer_pages: list[int] = []
for element in doc.get("kids", []):
if element.get("type") in ("paragraph", "heading", "list"):
content = element.get("content", "")
page = element.get("page number")
buffer_text += content + "\n"
if page and page not in buffer_pages:
buffer_pages.append(page)
if len(buffer_text) >= min_chars:
chunks.append({
"text": buffer_text.strip(),
"metadata": {
"pages": buffer_pages.copy(),
"source": doc.get("file name"),
}
})
buffer_text = ""
buffer_pages = []
# Save remaining buffer
if buffer_text.strip():
chunks.append({
"text": buffer_text.strip(),
"metadata": {
"pages": buffer_pages,
"source": doc.get("file name"),
}
})
return chunks
def format_citation(metadata: dict) -> str:
"""Generate a citation string from chunk metadata."""
source = metadata.get("source", "unknown")
page = metadata.get("page") or (metadata.get("pages", [None]) or [None])[0]
bbox = metadata.get("bbox")
citation = f"Source: {source}"
if page:
citation += f", Page {page}"
if bbox:
citation += f", Position ({bbox[0]:.0f}, {bbox[1]:.0f})"
return citation
def main():
# Find sample PDF relative to this script
# Using 1901.03003.pdf - a multi-page academic paper with complex layout
script_dir = Path(__file__).resolve().parent
repo_root = script_dir.parent.parent.parent
sample_pdf = repo_root / "samples" / "pdf" / "1901.03003.pdf"
if not sample_pdf.exists():
print(f"Sample PDF not found at: {sample_pdf}")
print("Make sure you're running from the repository.")
return
print(f"Processing: {sample_pdf.name}")
print("=" * 50)
# Convert PDF to JSON in a temp directory
with tempfile.TemporaryDirectory() as temp_dir:
json_path = convert_pdf_to_json(str(sample_pdf), temp_dir)
doc = load_document(json_path)
print(f"Document: {doc.get('file name')}")
print(f"Pages: {doc.get('number of pages')}")
print(f"Elements: {len(doc.get('kids', []))}")
# Strategy 1: By element
print("\n--- Strategy 1: Chunk by Element ---")
element_chunks = chunk_by_element(doc)
print(f"Created {len(element_chunks)} chunks")
for i, chunk in enumerate(element_chunks[:3]):
text_preview = chunk["text"][:60] + "..." if len(chunk["text"]) > 60 else chunk["text"]
print(f" [{i+1}] {text_preview}")
print(f" {format_citation(chunk['metadata'])}")
# Strategy 2: By section
print("\n--- Strategy 2: Chunk by Section ---")
section_chunks = chunk_by_section(doc)
print(f"Created {len(section_chunks)} chunks")
for i, chunk in enumerate(section_chunks[:2]):
heading = chunk["metadata"].get("heading", "No heading")
print(f" Section: {heading}")
print(f" Text: {chunk['text'][:60]}...")
# Strategy 3: Merged
print("\n--- Strategy 3: Merged Chunks (min 200 chars) ---")
merged_chunks = chunk_with_min_size(doc, min_chars=200)
print(f"Created {len(merged_chunks)} chunks")
for i, chunk in enumerate(merged_chunks[:2]):
print(f" [{i+1}] {len(chunk['text'])} chars: {chunk['text'][:50]}...")
# Show example chunk structure
print("\n--- Example Chunk Structure ---")
print("Each chunk has 'text' and 'metadata' ready for embedding:")
if element_chunks:
print(json.dumps(element_chunks[0], indent=2, ensure_ascii=False))
if __name__ == "__main__":
main()
examples/python/rag/langchain_example.py
+79
View File
@@ -0,0 +1,79 @@
#!/usr/bin/env python3
"""
LangChain Integration Example
Demonstrates using the official langchain-opendataloader-pdf package
for seamless RAG pipeline integration.
Usage:
pip install langchain-opendataloader-pdf
python langchain_example.py
"""
from pathlib import Path
from langchain_opendataloader_pdf import OpenDataLoaderPDFLoader
def main():
# Find sample PDF relative to this script
# Using 1901.03003.pdf - a multi-page academic paper with complex layout
script_dir = Path(__file__).resolve().parent
repo_root = script_dir.parent.parent.parent
sample_pdf = repo_root / "samples" / "pdf" / "1901.03003.pdf"
if not sample_pdf.exists():
print(f"Sample PDF not found at: {sample_pdf}")
print("Make sure you're running from the repository.")
return
print(f"Loading: {sample_pdf.name}")
print("=" * 50)
# Create loader with LangChain integration
loader = OpenDataLoaderPDFLoader(
file_path=[str(sample_pdf)],
format="text",
quiet=True,
)
# Load documents (returns LangChain Document objects)
documents = loader.load()
print(f"Loaded {len(documents)} document(s)\n")
for i, doc in enumerate(documents):
print(f"--- Document {i+1} ---")
print(f"Metadata: {doc.metadata}")
content_preview = doc.page_content[:200] + "..." if len(doc.page_content) > 200 else doc.page_content
print(f"Content:\n{content_preview}\n")
# Show integration points
print("--- LangChain Integration ---")
print("These Document objects work directly with:")
print(" - Text splitters: RecursiveCharacterTextSplitter, etc.")
print(" - Vector stores: Chroma, FAISS, Pinecone, etc.")
print(" - Retrievers: vectorstore.as_retriever()")
print(" - Chains: RetrievalQA, ConversationalRetrievalChain, etc.")
# Example: Using with a text splitter
print("\n--- Example: Text Splitting ---")
try:
from langchain_text_splitters import RecursiveCharacterTextSplitter
splitter = RecursiveCharacterTextSplitter(
chunk_size=500,
chunk_overlap=50,
)
chunks = splitter.split_documents(documents)
print(f"Split into {len(chunks)} chunks")
if chunks:
print(f"First chunk ({len(chunks[0].page_content)} chars):")
print(f" {chunks[0].page_content[:100]}...")
except ImportError:
print("Install langchain-text-splitters to see this example:")
print(" pip install langchain-text-splitters")
if __name__ == "__main__":
main()
examples/python/rag/requirements.txt
+3
View File
@@ -0,0 +1,3 @@
opendataloader-pdf>=2.2.1
langchain-opendataloader-pdf>=2.0.0
langchain-text-splitters>=1.1.2