chore: import upstream snapshot with attribution
This commit is contained in:
wehub-resource-sync
@@ -0,0 +1,158 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Baseline benchmark using docling-serve HTTP API.
|
||||
|
||||
Measures current docling-serve performance for comparison with
|
||||
FastAPI and subprocess approaches.
|
||||
|
||||
Usage:
|
||||
python scripts/experiments/docling_baseline_bench.py
|
||||
|
||||
Requirements:
|
||||
- docling-serve running on localhost:5001
|
||||
- requests package installed
|
||||
"""
|
||||
|
||||
import json
|
||||
import sys
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
import requests
|
||||
|
||||
# Configuration
|
||||
DOCLING_URL = "http://localhost:5001/v1/convert/file"
|
||||
PDF_DIR = Path(__file__).parent.parent.parent / "tests" / "benchmark" / "pdfs"
|
||||
RESULTS_DIR = Path(__file__).parent.parent.parent / "docs" / "hybrid" / "experiments"
|
||||
RESULTS_FILE = RESULTS_DIR / "baseline_results.json"
|
||||
|
||||
|
||||
def convert_pdf(pdf_path: Path) -> dict:
|
||||
"""Convert a single PDF using docling-serve API."""
|
||||
with open(pdf_path, "rb") as f:
|
||||
files = {"files": (pdf_path.name, f, "application/pdf")}
|
||||
data = {
|
||||
"to_formats": "md",
|
||||
"do_ocr": "true",
|
||||
"do_table_structure": "true",
|
||||
}
|
||||
|
||||
start_time = time.perf_counter()
|
||||
response = requests.post(DOCLING_URL, files=files, data=data, timeout=300)
|
||||
elapsed = time.perf_counter() - start_time
|
||||
|
||||
return {
|
||||
"filename": pdf_path.name,
|
||||
"status": "success" if response.status_code == 200 else "error",
|
||||
"elapsed": elapsed,
|
||||
"status_code": response.status_code,
|
||||
}
|
||||
|
||||
|
||||
def main():
|
||||
"""Run baseline benchmark."""
|
||||
# Check server health
|
||||
try:
|
||||
health = requests.get("http://localhost:5001/health", timeout=5)
|
||||
if health.status_code != 200:
|
||||
print("ERROR: docling-serve is not healthy", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
except requests.RequestException as e:
|
||||
print(f"ERROR: Cannot connect to docling-serve: {e}", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
print("=" * 60)
|
||||
print("Docling-serve Baseline Benchmark")
|
||||
print("=" * 60)
|
||||
print(f"PDF directory: {PDF_DIR}")
|
||||
print(f"Server URL: {DOCLING_URL}")
|
||||
print()
|
||||
|
||||
# Get PDF files
|
||||
pdf_files = sorted(PDF_DIR.glob("*.pdf"))
|
||||
total_files = len(pdf_files)
|
||||
print(f"Found {total_files} PDF files")
|
||||
print()
|
||||
|
||||
# Process each PDF
|
||||
results = []
|
||||
total_start = time.perf_counter()
|
||||
|
||||
for i, pdf_path in enumerate(pdf_files, 1):
|
||||
print(f"[{i:3d}/{total_files}] Processing {pdf_path.name}...", end=" ", flush=True)
|
||||
|
||||
try:
|
||||
result = convert_pdf(pdf_path)
|
||||
results.append(result)
|
||||
print(f"{result['elapsed']:.2f}s ({result['status']})")
|
||||
except Exception as e:
|
||||
results.append({
|
||||
"filename": pdf_path.name,
|
||||
"status": "error",
|
||||
"elapsed": 0,
|
||||
"error": str(e),
|
||||
})
|
||||
print(f"ERROR: {e}")
|
||||
|
||||
total_elapsed = time.perf_counter() - total_start
|
||||
|
||||
# Calculate statistics
|
||||
successful = [r for r in results if r["status"] == "success"]
|
||||
failed = [r for r in results if r["status"] != "success"]
|
||||
|
||||
if successful:
|
||||
elapsed_times = [r["elapsed"] for r in successful]
|
||||
avg_time = sum(elapsed_times) / len(elapsed_times)
|
||||
min_time = min(elapsed_times)
|
||||
max_time = max(elapsed_times)
|
||||
else:
|
||||
avg_time = min_time = max_time = 0
|
||||
|
||||
# Print summary
|
||||
print()
|
||||
print("=" * 60)
|
||||
print("RESULTS SUMMARY")
|
||||
print("=" * 60)
|
||||
print(f"Total documents: {total_files}")
|
||||
print(f"Successful: {len(successful)}")
|
||||
print(f"Failed: {len(failed)}")
|
||||
print()
|
||||
print(f"Total elapsed: {total_elapsed:.1f}s")
|
||||
print(f"Average per doc: {avg_time:.3f}s")
|
||||
print(f"Min: {min_time:.3f}s")
|
||||
print(f"Max: {max_time:.3f}s")
|
||||
print("=" * 60)
|
||||
|
||||
# Save results
|
||||
RESULTS_DIR.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
summary = {
|
||||
"approach": "baseline",
|
||||
"description": "docling-serve HTTP API",
|
||||
"timestamp": time.strftime("%Y-%m-%d %H:%M:%S"),
|
||||
"config": {
|
||||
"do_ocr": True,
|
||||
"do_table_structure": True,
|
||||
"server_url": DOCLING_URL,
|
||||
},
|
||||
"statistics": {
|
||||
"total_documents": total_files,
|
||||
"successful": len(successful),
|
||||
"failed": len(failed),
|
||||
"total_elapsed": round(total_elapsed, 2),
|
||||
"elapsed_per_doc": round(avg_time, 4),
|
||||
"min_elapsed": round(min_time, 4),
|
||||
"max_elapsed": round(max_time, 4),
|
||||
},
|
||||
"details": results,
|
||||
}
|
||||
|
||||
with open(RESULTS_FILE, "w", encoding="utf-8") as f:
|
||||
json.dump(summary, f, indent=2, ensure_ascii=False)
|
||||
|
||||
print(f"\nResults saved to: {RESULTS_FILE}")
|
||||
|
||||
return avg_time
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,289 @@
|
||||
#!/usr/bin/env python3
|
||||
"""FastAPI experiment benchmark using docling SDK directly.
|
||||
|
||||
Tests the hypothesis that a lightweight FastAPI server with
|
||||
DocumentConverter singleton is faster than docling-serve.
|
||||
|
||||
This script:
|
||||
1. Starts an embedded FastAPI server (port 5002)
|
||||
2. Converts all 200 benchmark PDFs
|
||||
3. Measures and reports performance
|
||||
|
||||
Usage:
|
||||
python scripts/experiments/docling_fastapi_bench.py
|
||||
|
||||
Requirements:
|
||||
- docling package installed
|
||||
- fastapi, uvicorn packages installed
|
||||
"""
|
||||
|
||||
import json
|
||||
import multiprocessing
|
||||
import os
|
||||
import sys
|
||||
import tempfile
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
import requests
|
||||
|
||||
# Configuration
|
||||
FASTAPI_PORT = 5002
|
||||
FASTAPI_URL = f"http://localhost:{FASTAPI_PORT}/convert"
|
||||
PDF_DIR = Path(__file__).parent.parent.parent / "tests" / "benchmark" / "pdfs"
|
||||
RESULTS_DIR = Path(__file__).parent.parent.parent / "docs" / "hybrid" / "experiments"
|
||||
RESULTS_FILE = RESULTS_DIR / "fastapi_results.json"
|
||||
|
||||
|
||||
def run_server():
|
||||
"""Run FastAPI server in a subprocess."""
|
||||
import uvicorn
|
||||
from fastapi import FastAPI, File, UploadFile
|
||||
from fastapi.responses import JSONResponse
|
||||
|
||||
# Import docling after fork to avoid issues
|
||||
from docling.datamodel.base_models import InputFormat
|
||||
from docling.datamodel.pipeline_options import (
|
||||
EasyOcrOptions,
|
||||
OcrOptions,
|
||||
PdfPipelineOptions,
|
||||
TableFormerMode,
|
||||
TableStructureOptions,
|
||||
)
|
||||
from docling.document_converter import DocumentConverter, PdfFormatOption
|
||||
|
||||
app = FastAPI()
|
||||
|
||||
# Create singleton DocumentConverter with warm-up
|
||||
print("Initializing DocumentConverter...", flush=True)
|
||||
|
||||
pipeline_options = PdfPipelineOptions(
|
||||
do_ocr=True,
|
||||
do_table_structure=True,
|
||||
ocr_options=EasyOcrOptions(force_full_page_ocr=False),
|
||||
table_structure_options=TableStructureOptions(
|
||||
mode=TableFormerMode.ACCURATE
|
||||
),
|
||||
)
|
||||
|
||||
converter = DocumentConverter(
|
||||
format_options={
|
||||
InputFormat.PDF: PdfFormatOption(pipeline_options=pipeline_options)
|
||||
}
|
||||
)
|
||||
print("DocumentConverter initialized.", flush=True)
|
||||
|
||||
@app.get("/health")
|
||||
def health():
|
||||
return {"status": "ok"}
|
||||
|
||||
@app.post("/convert")
|
||||
async def convert(file: UploadFile = File(...)):
|
||||
# Save uploaded file to temp location
|
||||
with tempfile.NamedTemporaryFile(suffix=".pdf", delete=False) as tmp:
|
||||
content = await file.read()
|
||||
tmp.write(content)
|
||||
tmp_path = tmp.name
|
||||
|
||||
try:
|
||||
start = time.perf_counter()
|
||||
result = converter.convert(tmp_path)
|
||||
elapsed = time.perf_counter() - start
|
||||
|
||||
md_content = result.document.export_to_markdown()
|
||||
|
||||
return JSONResponse({
|
||||
"status": "success",
|
||||
"markdown": md_content,
|
||||
"processing_time": elapsed,
|
||||
})
|
||||
except Exception:
|
||||
return JSONResponse({
|
||||
"status": "error",
|
||||
"error": "PDF conversion failed",
|
||||
}, status_code=500)
|
||||
finally:
|
||||
os.unlink(tmp_path)
|
||||
|
||||
uvicorn.run(app, host="0.0.0.0", port=FASTAPI_PORT, log_level="warning")
|
||||
|
||||
|
||||
def convert_pdf(pdf_path: Path) -> dict:
|
||||
"""Convert a single PDF using FastAPI server."""
|
||||
with open(pdf_path, "rb") as f:
|
||||
files = {"file": (pdf_path.name, f, "application/pdf")}
|
||||
|
||||
start_time = time.perf_counter()
|
||||
response = requests.post(FASTAPI_URL, files=files, timeout=300)
|
||||
elapsed = time.perf_counter() - start_time
|
||||
|
||||
if response.status_code == 200:
|
||||
data = response.json()
|
||||
return {
|
||||
"filename": pdf_path.name,
|
||||
"status": "success",
|
||||
"elapsed": elapsed,
|
||||
"server_time": data.get("processing_time", 0),
|
||||
}
|
||||
else:
|
||||
return {
|
||||
"filename": pdf_path.name,
|
||||
"status": "error",
|
||||
"elapsed": elapsed,
|
||||
"error": response.text,
|
||||
}
|
||||
|
||||
|
||||
def wait_for_server(max_retries=60, delay=1.0):
|
||||
"""Wait for server to be ready."""
|
||||
for i in range(max_retries):
|
||||
try:
|
||||
resp = requests.get(f"http://localhost:{FASTAPI_PORT}/health", timeout=5)
|
||||
if resp.status_code == 200:
|
||||
return True
|
||||
except requests.RequestException:
|
||||
pass
|
||||
time.sleep(delay)
|
||||
return False
|
||||
|
||||
|
||||
def main():
|
||||
"""Run FastAPI benchmark."""
|
||||
print("=" * 60)
|
||||
print("FastAPI Experiment Benchmark")
|
||||
print("=" * 60)
|
||||
print(f"PDF directory: {PDF_DIR}")
|
||||
print(f"Server URL: {FASTAPI_URL}")
|
||||
print()
|
||||
|
||||
# Start server in subprocess
|
||||
print("Starting FastAPI server...", flush=True)
|
||||
server_process = multiprocessing.Process(target=run_server, daemon=True)
|
||||
server_process.start()
|
||||
|
||||
# Wait for server to be ready
|
||||
print("Waiting for server to initialize (including model loading)...", flush=True)
|
||||
if not wait_for_server(max_retries=120, delay=1.0):
|
||||
print("ERROR: Server failed to start", file=sys.stderr)
|
||||
server_process.terminate()
|
||||
sys.exit(1)
|
||||
|
||||
print("Server is ready.", flush=True)
|
||||
print()
|
||||
|
||||
# Get PDF files
|
||||
pdf_files = sorted(PDF_DIR.glob("*.pdf"))
|
||||
total_files = len(pdf_files)
|
||||
print(f"Found {total_files} PDF files")
|
||||
print()
|
||||
|
||||
# Process each PDF
|
||||
results = []
|
||||
total_start = time.perf_counter()
|
||||
|
||||
try:
|
||||
for i, pdf_path in enumerate(pdf_files, 1):
|
||||
print(f"[{i:3d}/{total_files}] Processing {pdf_path.name}...", end=" ", flush=True)
|
||||
|
||||
try:
|
||||
result = convert_pdf(pdf_path)
|
||||
results.append(result)
|
||||
server_time = result.get("server_time", 0)
|
||||
print(f"{result['elapsed']:.2f}s (server: {server_time:.2f}s) ({result['status']})")
|
||||
except Exception as e:
|
||||
results.append({
|
||||
"filename": pdf_path.name,
|
||||
"status": "error",
|
||||
"elapsed": 0,
|
||||
"error": str(e),
|
||||
})
|
||||
print(f"ERROR: {e}")
|
||||
|
||||
total_elapsed = time.perf_counter() - total_start
|
||||
|
||||
finally:
|
||||
# Shutdown server
|
||||
print("\nShutting down server...", flush=True)
|
||||
server_process.terminate()
|
||||
server_process.join(timeout=5)
|
||||
|
||||
# Calculate statistics
|
||||
successful = [r for r in results if r["status"] == "success"]
|
||||
failed = [r for r in results if r["status"] != "success"]
|
||||
|
||||
if successful:
|
||||
elapsed_times = [r["elapsed"] for r in successful]
|
||||
server_times = [r.get("server_time", 0) for r in successful]
|
||||
avg_time = sum(elapsed_times) / len(elapsed_times)
|
||||
avg_server_time = sum(server_times) / len(server_times)
|
||||
min_time = min(elapsed_times)
|
||||
max_time = max(elapsed_times)
|
||||
else:
|
||||
avg_time = avg_server_time = min_time = max_time = 0
|
||||
|
||||
# Print summary
|
||||
print()
|
||||
print("=" * 60)
|
||||
print("RESULTS SUMMARY")
|
||||
print("=" * 60)
|
||||
print(f"Total documents: {total_files}")
|
||||
print(f"Successful: {len(successful)}")
|
||||
print(f"Failed: {len(failed)}")
|
||||
print()
|
||||
print(f"Total elapsed: {total_elapsed:.1f}s")
|
||||
print(f"Average per doc: {avg_time:.3f}s (target: < 0.8s)")
|
||||
print(f"Avg server time: {avg_server_time:.3f}s")
|
||||
print(f"Min: {min_time:.3f}s")
|
||||
print(f"Max: {max_time:.3f}s")
|
||||
print()
|
||||
|
||||
# Success/Failure check
|
||||
if avg_time < 0.8:
|
||||
print("✅ SUCCESS: Average time is below 0.8s threshold!")
|
||||
else:
|
||||
print("❌ FAILURE: Average time exceeds 0.8s threshold")
|
||||
print(" Plan may need to be discarded.")
|
||||
|
||||
print("=" * 60)
|
||||
|
||||
# Save results
|
||||
RESULTS_DIR.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
summary = {
|
||||
"approach": "fastapi",
|
||||
"description": "FastAPI server with docling SDK singleton",
|
||||
"timestamp": time.strftime("%Y-%m-%d %H:%M:%S"),
|
||||
"config": {
|
||||
"do_ocr": True,
|
||||
"do_table_structure": True,
|
||||
"server_port": FASTAPI_PORT,
|
||||
},
|
||||
"statistics": {
|
||||
"total_documents": total_files,
|
||||
"successful": len(successful),
|
||||
"failed": len(failed),
|
||||
"total_elapsed": round(total_elapsed, 2),
|
||||
"elapsed_per_doc": round(avg_time, 4),
|
||||
"server_time_per_doc": round(avg_server_time, 4),
|
||||
"min_elapsed": round(min_time, 4),
|
||||
"max_elapsed": round(max_time, 4),
|
||||
},
|
||||
"threshold": {
|
||||
"target": 0.8,
|
||||
"passed": avg_time < 0.8,
|
||||
},
|
||||
"details": results,
|
||||
}
|
||||
|
||||
with open(RESULTS_FILE, "w", encoding="utf-8") as f:
|
||||
json.dump(summary, f, indent=2, ensure_ascii=False)
|
||||
|
||||
print(f"\nResults saved to: {RESULTS_FILE}")
|
||||
|
||||
return avg_time
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
# Required for multiprocessing on macOS
|
||||
multiprocessing.set_start_method("spawn", force=True)
|
||||
main()
|
||||
@@ -0,0 +1,211 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Generate speed comparison report for docling experiments.
|
||||
|
||||
Reads results from all experiment runs and generates a summary report.
|
||||
|
||||
Usage:
|
||||
python scripts/experiments/docling_speed_report.py
|
||||
"""
|
||||
|
||||
import json
|
||||
import sys
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
|
||||
RESULTS_DIR = Path(__file__).parent.parent.parent / "docs" / "hybrid" / "experiments"
|
||||
REPORT_FILE = RESULTS_DIR / f"speed-experiment-{datetime.now().strftime('%Y-%m-%d')}.md"
|
||||
|
||||
|
||||
def load_results(filename: str) -> dict | None:
|
||||
"""Load results from JSON file."""
|
||||
path = RESULTS_DIR / filename
|
||||
if not path.exists():
|
||||
return None
|
||||
with open(path, encoding="utf-8") as f:
|
||||
return json.load(f)
|
||||
|
||||
|
||||
def main():
|
||||
"""Generate comparison report."""
|
||||
print("Loading experiment results...")
|
||||
|
||||
baseline = load_results("baseline_results.json")
|
||||
fastapi = load_results("fastapi_results.json")
|
||||
subprocess = load_results("subprocess_results.json")
|
||||
|
||||
if not any([baseline, fastapi, subprocess]):
|
||||
print("ERROR: No experiment results found", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
# Print console summary
|
||||
print()
|
||||
print("=" * 70)
|
||||
print("DOCLING SPEED EXPERIMENT RESULTS")
|
||||
print("=" * 70)
|
||||
print()
|
||||
|
||||
approaches = []
|
||||
if baseline:
|
||||
approaches.append(("baseline", "docling-serve HTTP", baseline))
|
||||
if fastapi:
|
||||
approaches.append(("fastapi", "FastAPI + SDK singleton", fastapi))
|
||||
if subprocess:
|
||||
approaches.append(("subprocess", "Persistent subprocess", subprocess))
|
||||
|
||||
# Table header
|
||||
print(f"{'Approach':<15} {'Description':<25} {'Avg (s/doc)':<12} {'Target':<10} {'Status':<10} {'Speedup':<10}")
|
||||
print("-" * 70)
|
||||
|
||||
baseline_time = baseline["statistics"]["elapsed_per_doc"] if baseline else None
|
||||
|
||||
for name, desc, data in approaches:
|
||||
stats = data["statistics"]
|
||||
avg_time = stats["elapsed_per_doc"]
|
||||
|
||||
threshold = data.get("threshold", {})
|
||||
target = threshold.get("target", "-")
|
||||
passed = threshold.get("passed", None)
|
||||
|
||||
if passed is True:
|
||||
status = "PASS"
|
||||
elif passed is False:
|
||||
status = "FAIL"
|
||||
else:
|
||||
status = "-"
|
||||
|
||||
# Calculate speedup vs baseline
|
||||
if baseline_time and name != "baseline":
|
||||
speedup = f"{baseline_time / avg_time:.1f}x"
|
||||
else:
|
||||
speedup = "-"
|
||||
|
||||
print(f"{name:<15} {desc:<25} {avg_time:<12.3f} {str(target):<10} {status:<10} {speedup:<10}")
|
||||
|
||||
print("-" * 70)
|
||||
print()
|
||||
|
||||
# Decision summary
|
||||
print("DECISION SUMMARY:")
|
||||
print("-" * 40)
|
||||
|
||||
fastapi_passed = fastapi and fastapi.get("threshold", {}).get("passed", False)
|
||||
subprocess_passed = subprocess and subprocess.get("threshold", {}).get("passed", False)
|
||||
|
||||
if fastapi_passed:
|
||||
print("FastAPI approach: APPROVED (proceed to Phase 1)")
|
||||
else:
|
||||
print("FastAPI approach: REJECTED (plan discarded)")
|
||||
|
||||
if subprocess_passed:
|
||||
print("Subprocess approach: APPROVED (proceed to Phase 1)")
|
||||
else:
|
||||
print("Subprocess approach: REJECTED (excluded from plan)")
|
||||
|
||||
print()
|
||||
|
||||
if fastapi_passed:
|
||||
print("OVERALL: Phase 0 PASSED - Proceed to implementation")
|
||||
print()
|
||||
|
||||
# Recommendation
|
||||
if subprocess_passed:
|
||||
fastapi_time = fastapi["statistics"]["elapsed_per_doc"]
|
||||
subprocess_time = subprocess["statistics"]["elapsed_per_doc"]
|
||||
if subprocess_time < fastapi_time:
|
||||
print(f"RECOMMENDATION: subprocess approach is slightly faster ({subprocess_time:.3f}s vs {fastapi_time:.3f}s)")
|
||||
print(" However, FastAPI is more production-ready (health checks, easier deployment)")
|
||||
else:
|
||||
print(f"RECOMMENDATION: FastAPI approach is faster and more production-ready")
|
||||
else:
|
||||
print("OVERALL: Phase 0 FAILED - Plan should be discarded")
|
||||
|
||||
print("=" * 70)
|
||||
|
||||
# Generate markdown report
|
||||
RESULTS_DIR.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
report = []
|
||||
report.append("# Docling Speed Experiment Results")
|
||||
report.append("")
|
||||
report.append(f"**Date**: {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}")
|
||||
report.append("")
|
||||
|
||||
report.append("## Summary")
|
||||
report.append("")
|
||||
report.append("| Approach | Description | Avg (s/doc) | Target | Status | Speedup |")
|
||||
report.append("|----------|-------------|-------------|--------|--------|---------|")
|
||||
|
||||
for name, desc, data in approaches:
|
||||
stats = data["statistics"]
|
||||
avg_time = stats["elapsed_per_doc"]
|
||||
threshold = data.get("threshold", {})
|
||||
target = threshold.get("target", "-")
|
||||
passed = threshold.get("passed", None)
|
||||
|
||||
if passed is True:
|
||||
status = "PASS"
|
||||
elif passed is False:
|
||||
status = "FAIL"
|
||||
else:
|
||||
status = "-"
|
||||
|
||||
if baseline_time and name != "baseline":
|
||||
speedup = f"{baseline_time / avg_time:.1f}x"
|
||||
else:
|
||||
speedup = "-"
|
||||
|
||||
report.append(f"| {name} | {desc} | {avg_time:.3f} | {target} | {status} | {speedup} |")
|
||||
|
||||
report.append("")
|
||||
report.append("## Decision")
|
||||
report.append("")
|
||||
|
||||
if fastapi_passed:
|
||||
report.append("**Phase 0 PASSED** - FastAPI approach meets the < 0.8s threshold.")
|
||||
report.append("")
|
||||
report.append("Proceed to Phase 1 implementation:")
|
||||
report.append("")
|
||||
report.append("- [ ] Task 1.1: docling_subprocess_worker.py")
|
||||
report.append("- [ ] Task 1.2: docling_fast_server.py")
|
||||
report.append("- [ ] Task 2.1: DoclingSubprocessClient.java")
|
||||
report.append("- [ ] Task 2.2: DoclingFastServerClient.java")
|
||||
report.append("- [ ] Task 2.3: HybridClientFactory modification")
|
||||
report.append("- [ ] Task 3: Benchmark integration")
|
||||
report.append("- [ ] Task 4: Final validation")
|
||||
|
||||
if subprocess_passed:
|
||||
report.append("")
|
||||
report.append("Subprocess approach also passed - both approaches available for implementation.")
|
||||
else:
|
||||
report.append("**Phase 0 FAILED** - FastAPI approach exceeds 0.8s threshold.")
|
||||
report.append("")
|
||||
report.append("Plan should be discarded. Consider alternative approaches.")
|
||||
|
||||
report.append("")
|
||||
report.append("## Detailed Statistics")
|
||||
report.append("")
|
||||
|
||||
for name, desc, data in approaches:
|
||||
stats = data["statistics"]
|
||||
report.append(f"### {name.title()}")
|
||||
report.append("")
|
||||
report.append(f"- **Description**: {data['description']}")
|
||||
report.append(f"- **Timestamp**: {data['timestamp']}")
|
||||
report.append(f"- **Total documents**: {stats['total_documents']}")
|
||||
report.append(f"- **Successful**: {stats['successful']}")
|
||||
report.append(f"- **Failed**: {stats['failed']}")
|
||||
report.append(f"- **Total elapsed**: {stats['total_elapsed']:.1f}s")
|
||||
report.append(f"- **Average per doc**: {stats['elapsed_per_doc']:.4f}s")
|
||||
report.append(f"- **Min**: {stats['min_elapsed']:.4f}s")
|
||||
report.append(f"- **Max**: {stats['max_elapsed']:.4f}s")
|
||||
report.append("")
|
||||
|
||||
# Write report
|
||||
with open(REPORT_FILE, "w", encoding="utf-8") as f:
|
||||
f.write("\n".join(report))
|
||||
|
||||
print(f"\nReport saved to: {REPORT_FILE}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,331 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Subprocess experiment benchmark using docling SDK directly.
|
||||
|
||||
Tests the subprocess approach where each PDF is processed by
|
||||
invoking a Python worker script via subprocess.
|
||||
|
||||
This approach has overhead from:
|
||||
1. Python interpreter startup
|
||||
2. Model loading per-process (unless using persistent worker)
|
||||
|
||||
For this experiment, we test a persistent worker approach:
|
||||
- Single Python process stays alive
|
||||
- Receives PDF paths via stdin, outputs JSON via stdout
|
||||
|
||||
Usage:
|
||||
python scripts/experiments/docling_subprocess_bench.py
|
||||
|
||||
Requirements:
|
||||
- docling package installed
|
||||
"""
|
||||
|
||||
import base64
|
||||
import json
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
# Configuration
|
||||
PDF_DIR = Path(__file__).parent.parent.parent / "tests" / "benchmark" / "pdfs"
|
||||
RESULTS_DIR = Path(__file__).parent.parent.parent / "docs" / "hybrid" / "experiments"
|
||||
RESULTS_FILE = RESULTS_DIR / "subprocess_results.json"
|
||||
|
||||
# Worker script inline - will be written to temp file
|
||||
WORKER_SCRIPT = '''
|
||||
import base64
|
||||
import json
|
||||
import sys
|
||||
import time
|
||||
import tempfile
|
||||
import os
|
||||
|
||||
# Initialize docling once
|
||||
from docling.datamodel.base_models import InputFormat
|
||||
from docling.datamodel.pipeline_options import (
|
||||
EasyOcrOptions,
|
||||
PdfPipelineOptions,
|
||||
TableFormerMode,
|
||||
TableStructureOptions,
|
||||
)
|
||||
from docling.document_converter import DocumentConverter, PdfFormatOption
|
||||
|
||||
print("WORKER_READY", file=sys.stderr, flush=True)
|
||||
|
||||
pipeline_options = PdfPipelineOptions(
|
||||
do_ocr=True,
|
||||
do_table_structure=True,
|
||||
ocr_options=EasyOcrOptions(force_full_page_ocr=False),
|
||||
table_structure_options=TableStructureOptions(
|
||||
mode=TableFormerMode.ACCURATE
|
||||
),
|
||||
)
|
||||
|
||||
converter = DocumentConverter(
|
||||
format_options={
|
||||
InputFormat.PDF: PdfFormatOption(pipeline_options=pipeline_options)
|
||||
}
|
||||
)
|
||||
|
||||
print("CONVERTER_READY", file=sys.stderr, flush=True)
|
||||
|
||||
# Process requests from stdin
|
||||
for line in sys.stdin:
|
||||
line = line.strip()
|
||||
if not line:
|
||||
continue
|
||||
|
||||
try:
|
||||
request = json.loads(line)
|
||||
pdf_base64 = request.get("pdf_base64")
|
||||
filename = request.get("filename", "document.pdf")
|
||||
|
||||
# Decode and write to temp file
|
||||
pdf_bytes = base64.b64decode(pdf_base64)
|
||||
with tempfile.NamedTemporaryFile(suffix=".pdf", delete=False) as tmp:
|
||||
tmp.write(pdf_bytes)
|
||||
tmp_path = tmp.name
|
||||
|
||||
try:
|
||||
start = time.perf_counter()
|
||||
result = converter.convert(tmp_path)
|
||||
elapsed = time.perf_counter() - start
|
||||
|
||||
md_content = result.document.export_to_markdown()
|
||||
|
||||
response = {
|
||||
"status": "success",
|
||||
"filename": filename,
|
||||
"markdown": md_content,
|
||||
"processing_time": elapsed,
|
||||
}
|
||||
except Exception as e:
|
||||
response = {
|
||||
"status": "error",
|
||||
"filename": filename,
|
||||
"error": str(e),
|
||||
}
|
||||
finally:
|
||||
os.unlink(tmp_path)
|
||||
|
||||
print(json.dumps(response), flush=True)
|
||||
|
||||
except Exception as e:
|
||||
response = {
|
||||
"status": "error",
|
||||
"error": str(e),
|
||||
}
|
||||
print(json.dumps(response), flush=True)
|
||||
'''
|
||||
|
||||
|
||||
def convert_pdf(process: subprocess.Popen, pdf_path: Path) -> dict:
|
||||
"""Convert a single PDF using subprocess worker."""
|
||||
# Read PDF and encode as base64
|
||||
with open(pdf_path, "rb") as f:
|
||||
pdf_bytes = f.read()
|
||||
pdf_base64 = base64.b64encode(pdf_bytes).decode("ascii")
|
||||
|
||||
# Send request
|
||||
request = {
|
||||
"pdf_base64": pdf_base64,
|
||||
"filename": pdf_path.name,
|
||||
}
|
||||
|
||||
start_time = time.perf_counter()
|
||||
process.stdin.write(json.dumps(request) + "\n")
|
||||
process.stdin.flush()
|
||||
|
||||
# Read response
|
||||
response_line = process.stdout.readline()
|
||||
elapsed = time.perf_counter() - start_time
|
||||
|
||||
if response_line:
|
||||
try:
|
||||
response = json.loads(response_line)
|
||||
response["client_elapsed"] = elapsed
|
||||
return response
|
||||
except json.JSONDecodeError as e:
|
||||
return {
|
||||
"filename": pdf_path.name,
|
||||
"status": "error",
|
||||
"error": f"JSON decode error: {e}",
|
||||
"client_elapsed": elapsed,
|
||||
}
|
||||
else:
|
||||
return {
|
||||
"filename": pdf_path.name,
|
||||
"status": "error",
|
||||
"error": "No response from worker",
|
||||
"client_elapsed": elapsed,
|
||||
}
|
||||
|
||||
|
||||
def main():
|
||||
"""Run subprocess benchmark."""
|
||||
print("=" * 60)
|
||||
print("Subprocess Experiment Benchmark")
|
||||
print("=" * 60)
|
||||
print(f"PDF directory: {PDF_DIR}")
|
||||
print()
|
||||
|
||||
# Write worker script to temp file
|
||||
with tempfile.NamedTemporaryFile(mode="w", suffix=".py", delete=False) as f:
|
||||
f.write(WORKER_SCRIPT)
|
||||
worker_path = f.name
|
||||
|
||||
print("Starting worker process...", flush=True)
|
||||
|
||||
try:
|
||||
# Start worker process
|
||||
process = subprocess.Popen(
|
||||
[sys.executable, worker_path],
|
||||
stdin=subprocess.PIPE,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE,
|
||||
text=True,
|
||||
bufsize=1, # Line buffered
|
||||
)
|
||||
|
||||
# Wait for worker to be ready (read stderr for status messages)
|
||||
print("Waiting for worker to initialize (including model loading)...", flush=True)
|
||||
|
||||
ready_count = 0
|
||||
while ready_count < 2:
|
||||
line = process.stderr.readline()
|
||||
if "WORKER_READY" in line:
|
||||
ready_count += 1
|
||||
print(" - Worker process started", flush=True)
|
||||
elif "CONVERTER_READY" in line:
|
||||
ready_count += 1
|
||||
print(" - DocumentConverter initialized", flush=True)
|
||||
elif process.poll() is not None:
|
||||
print("ERROR: Worker process died unexpectedly", file=sys.stderr)
|
||||
remaining_stderr = process.stderr.read()
|
||||
print(remaining_stderr, file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
print("Worker is ready.", flush=True)
|
||||
print()
|
||||
|
||||
# Get PDF files
|
||||
pdf_files = sorted(PDF_DIR.glob("*.pdf"))
|
||||
total_files = len(pdf_files)
|
||||
print(f"Found {total_files} PDF files")
|
||||
print()
|
||||
|
||||
# Process each PDF
|
||||
results = []
|
||||
total_start = time.perf_counter()
|
||||
|
||||
for i, pdf_path in enumerate(pdf_files, 1):
|
||||
print(f"[{i:3d}/{total_files}] Processing {pdf_path.name}...", end=" ", flush=True)
|
||||
|
||||
try:
|
||||
result = convert_pdf(process, pdf_path)
|
||||
results.append(result)
|
||||
server_time = result.get("processing_time", 0)
|
||||
client_time = result.get("client_elapsed", 0)
|
||||
print(f"{client_time:.2f}s (server: {server_time:.2f}s) ({result['status']})")
|
||||
except Exception as e:
|
||||
results.append({
|
||||
"filename": pdf_path.name,
|
||||
"status": "error",
|
||||
"client_elapsed": 0,
|
||||
"error": str(e),
|
||||
})
|
||||
print(f"ERROR: {e}")
|
||||
|
||||
total_elapsed = time.perf_counter() - total_start
|
||||
|
||||
finally:
|
||||
# Shutdown worker
|
||||
print("\nShutting down worker...", flush=True)
|
||||
if process.poll() is None:
|
||||
process.stdin.close()
|
||||
process.terminate()
|
||||
process.wait(timeout=5)
|
||||
|
||||
# Clean up worker script
|
||||
import os
|
||||
os.unlink(worker_path)
|
||||
|
||||
# Calculate statistics
|
||||
successful = [r for r in results if r["status"] == "success"]
|
||||
failed = [r for r in results if r["status"] != "success"]
|
||||
|
||||
if successful:
|
||||
client_times = [r.get("client_elapsed", 0) for r in successful]
|
||||
server_times = [r.get("processing_time", 0) for r in successful]
|
||||
avg_client_time = sum(client_times) / len(client_times)
|
||||
avg_server_time = sum(server_times) / len(server_times)
|
||||
min_time = min(client_times)
|
||||
max_time = max(client_times)
|
||||
else:
|
||||
avg_client_time = avg_server_time = min_time = max_time = 0
|
||||
|
||||
# Print summary
|
||||
print()
|
||||
print("=" * 60)
|
||||
print("RESULTS SUMMARY")
|
||||
print("=" * 60)
|
||||
print(f"Total documents: {total_files}")
|
||||
print(f"Successful: {len(successful)}")
|
||||
print(f"Failed: {len(failed)}")
|
||||
print()
|
||||
print(f"Total elapsed: {total_elapsed:.1f}s")
|
||||
print(f"Average per doc: {avg_client_time:.3f}s (target: < 1.0s)")
|
||||
print(f"Avg server time: {avg_server_time:.3f}s")
|
||||
print(f"Min: {min_time:.3f}s")
|
||||
print(f"Max: {max_time:.3f}s")
|
||||
print()
|
||||
|
||||
# Success/Failure check
|
||||
if avg_client_time < 1.0:
|
||||
print("✅ SUCCESS: Average time is below 1.0s threshold!")
|
||||
else:
|
||||
print("❌ FAILURE: Average time exceeds 1.0s threshold")
|
||||
print(" Subprocess approach will be excluded.")
|
||||
|
||||
print("=" * 60)
|
||||
|
||||
# Save results
|
||||
RESULTS_DIR.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
summary = {
|
||||
"approach": "subprocess",
|
||||
"description": "Persistent Python subprocess with docling SDK",
|
||||
"timestamp": time.strftime("%Y-%m-%d %H:%M:%S"),
|
||||
"config": {
|
||||
"do_ocr": True,
|
||||
"do_table_structure": True,
|
||||
"worker_type": "persistent",
|
||||
},
|
||||
"statistics": {
|
||||
"total_documents": total_files,
|
||||
"successful": len(successful),
|
||||
"failed": len(failed),
|
||||
"total_elapsed": round(total_elapsed, 2),
|
||||
"elapsed_per_doc": round(avg_client_time, 4),
|
||||
"server_time_per_doc": round(avg_server_time, 4),
|
||||
"min_elapsed": round(min_time, 4),
|
||||
"max_elapsed": round(max_time, 4),
|
||||
},
|
||||
"threshold": {
|
||||
"target": 1.0,
|
||||
"passed": avg_client_time < 1.0,
|
||||
},
|
||||
"details": results,
|
||||
}
|
||||
|
||||
with open(RESULTS_FILE, "w", encoding="utf-8") as f:
|
||||
json.dump(summary, f, indent=2, ensure_ascii=False)
|
||||
|
||||
print(f"\nResults saved to: {RESULTS_FILE}")
|
||||
|
||||
return avg_client_time
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user