chore: import upstream snapshot with attribution
This commit is contained in:
wehub-resource-sync
Executable
+85
@@ -0,0 +1,85 @@
|
||||
#!/usr/bin/env bash
|
||||
# Benchmark script for opendataloader-pdf
|
||||
#
|
||||
# Clones opendataloader-bench and runs benchmark against locally built JAR.
|
||||
#
|
||||
# Usage:
|
||||
# ./scripts/bench.sh # Run full benchmark
|
||||
# ./scripts/bench.sh --doc-id 01030... # Run for specific document
|
||||
# ./scripts/bench.sh --check-regression # Run with regression check (CI)
|
||||
# ./scripts/bench.sh --skip-build # Skip Java build step
|
||||
#
|
||||
# Environment:
|
||||
# BENCH_DIR Override bench repo clone location (default: /tmp/opendataloader-bench)
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
PROJECT_ROOT="$(dirname "$SCRIPT_DIR")"
|
||||
BENCH_REPO="https://github.com/opendataloader-project/opendataloader-bench.git"
|
||||
BENCH_DIR="${BENCH_DIR:-/tmp/opendataloader-bench}"
|
||||
|
||||
# Parse --skip-build flag (pass everything else through)
|
||||
SKIP_BUILD=false
|
||||
ARGS=()
|
||||
for arg in "$@"; do
|
||||
if [[ "$arg" == "--skip-build" ]]; then
|
||||
SKIP_BUILD=true
|
||||
else
|
||||
ARGS+=("$arg")
|
||||
fi
|
||||
done
|
||||
|
||||
# Find CLI JAR (shaded first, then regular, excluding sources/javadoc/original)
|
||||
find_jar() {
|
||||
local target_dir="$PROJECT_ROOT/java/opendataloader-pdf-cli/target"
|
||||
local jar
|
||||
jar=$(find "$target_dir" -name "opendataloader-pdf-cli-*-shaded.jar" 2>/dev/null | head -1)
|
||||
if [[ -z "$jar" ]]; then
|
||||
jar=$(find "$target_dir" -name "opendataloader-pdf-cli-*.jar" \
|
||||
! -name "*-sources.jar" ! -name "*-javadoc.jar" ! -name "original-*" \
|
||||
2>/dev/null | head -1)
|
||||
fi
|
||||
echo "$jar"
|
||||
}
|
||||
|
||||
# Step 1: Build Java if needed
|
||||
if [[ "$SKIP_BUILD" == "false" ]]; then
|
||||
JAR_PATH=$(find_jar)
|
||||
if [[ -z "$JAR_PATH" ]]; then
|
||||
echo "Building Java..."
|
||||
"$SCRIPT_DIR/build-java.sh"
|
||||
else
|
||||
echo "Using existing JAR: $JAR_PATH"
|
||||
fi
|
||||
fi
|
||||
|
||||
# Step 2: Clone or update bench repo
|
||||
if [[ -d "$BENCH_DIR/.git" ]]; then
|
||||
echo "Updating bench repo..."
|
||||
git -C "$BENCH_DIR" pull --ff-only --quiet 2>/dev/null || true
|
||||
else
|
||||
echo "Cloning bench repo..."
|
||||
git clone --depth 1 "$BENCH_REPO" "$BENCH_DIR"
|
||||
fi
|
||||
|
||||
# Step 3: Find JAR path
|
||||
JAR_PATH=$(find_jar)
|
||||
if [[ -z "$JAR_PATH" ]]; then
|
||||
echo "Error: No JAR found. Run ./scripts/build-java.sh first."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Step 4: Run benchmark with JAR
|
||||
echo "Running benchmark with JAR: $JAR_PATH"
|
||||
cd "$BENCH_DIR"
|
||||
|
||||
if ! command -v uv &> /dev/null; then
|
||||
echo "Error: uv is not installed."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
uv sync --quiet
|
||||
OPENDATALOADER_JAR="$JAR_PATH" uv run python src/run.py \
|
||||
--engine opendataloader \
|
||||
"${ARGS[@]}"
|
||||
Executable
+81
@@ -0,0 +1,81 @@
|
||||
#!/bin/bash
|
||||
|
||||
# Build and test all packages: Java, Python, Node.js
|
||||
# Usage: ./scripts/build-all.sh [VERSION]
|
||||
# Example: ./scripts/build-all.sh 1.0.0
|
||||
# If VERSION is not provided, defaults to "0.0.0"
|
||||
|
||||
set -e
|
||||
|
||||
# =================================================================
|
||||
# Configuration
|
||||
# =================================================================
|
||||
VERSION="${1:-0.0.0}"
|
||||
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
|
||||
ROOT_DIR="$(cd "$SCRIPT_DIR/.." && pwd)"
|
||||
|
||||
# =================================================================
|
||||
# Prerequisites Check
|
||||
# =================================================================
|
||||
echo "Checking prerequisites..."
|
||||
|
||||
command -v java >/dev/null || { echo "Error: java not found"; exit 1; }
|
||||
command -v mvn >/dev/null || { echo "Error: mvn not found"; exit 1; }
|
||||
command -v uv >/dev/null || { echo "Error: uv not found. Install with: curl -LsSf https://astral.sh/uv/install.sh | sh"; exit 1; }
|
||||
command -v node >/dev/null || { echo "Error: node not found"; exit 1; }
|
||||
command -v pnpm >/dev/null || { echo "Error: pnpm not found"; exit 1; }
|
||||
|
||||
echo "All prerequisites found."
|
||||
|
||||
echo ""
|
||||
echo "========================================"
|
||||
echo "Building all packages (version: $VERSION)"
|
||||
echo "========================================"
|
||||
|
||||
# =================================================================
|
||||
# Java Build & Test
|
||||
# =================================================================
|
||||
echo ""
|
||||
echo "[1/3] Java: Building and testing..."
|
||||
echo "----------------------------------------"
|
||||
|
||||
cd "$ROOT_DIR/java"
|
||||
mvn versions:set -DnewVersion="$VERSION" -DgenerateBackupPoms=false
|
||||
"$SCRIPT_DIR/build-java.sh"
|
||||
|
||||
echo "[1/3] Java: Done"
|
||||
|
||||
# =================================================================
|
||||
# Python Build & Test
|
||||
# =================================================================
|
||||
echo ""
|
||||
echo "[2/3] Python: Building and testing..."
|
||||
echo "----------------------------------------"
|
||||
|
||||
cd "$ROOT_DIR/python/opendataloader-pdf"
|
||||
sed -i.bak "s/^version = \"[^\"]*\"/version = \"$VERSION\"/" pyproject.toml && rm -f pyproject.toml.bak
|
||||
"$SCRIPT_DIR/build-python.sh"
|
||||
|
||||
echo "[2/3] Python: Done"
|
||||
|
||||
# =================================================================
|
||||
# Node.js Build & Test
|
||||
# =================================================================
|
||||
echo ""
|
||||
echo "[3/3] Node.js: Building and testing..."
|
||||
echo "----------------------------------------"
|
||||
|
||||
cd "$ROOT_DIR/node/opendataloader-pdf"
|
||||
pnpm version "$VERSION" --no-git-tag-version --allow-same-version
|
||||
"$SCRIPT_DIR/build-node.sh"
|
||||
|
||||
echo "[3/3] Node.js: Done"
|
||||
|
||||
# =================================================================
|
||||
# Summary
|
||||
# =================================================================
|
||||
echo ""
|
||||
echo "========================================"
|
||||
echo "All builds completed successfully!"
|
||||
echo "Version: $VERSION"
|
||||
echo "========================================"
|
||||
Executable
+18
@@ -0,0 +1,18 @@
|
||||
#!/bin/bash
|
||||
|
||||
# CI/CD build script for Java package
|
||||
# For local development, use test-java.sh instead
|
||||
|
||||
set -e
|
||||
|
||||
# Prerequisites
|
||||
command -v java >/dev/null || { echo "Error: java not found"; exit 1; }
|
||||
command -v mvn >/dev/null || { echo "Error: mvn not found"; exit 1; }
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
|
||||
ROOT_DIR="$SCRIPT_DIR/.."
|
||||
PACKAGE_DIR="$ROOT_DIR/java"
|
||||
cd "$PACKAGE_DIR"
|
||||
|
||||
# Build and test
|
||||
mvn -B clean package -P release
|
||||
Executable
+24
@@ -0,0 +1,24 @@
|
||||
#!/bin/bash
|
||||
|
||||
# CI/CD build script for Node.js package
|
||||
# For local development, use test-node.sh instead
|
||||
|
||||
set -e
|
||||
|
||||
# Prerequisites
|
||||
command -v node >/dev/null || { echo "Error: node not found"; exit 1; }
|
||||
command -v pnpm >/dev/null || { echo "Error: pnpm not found"; exit 1; }
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
|
||||
ROOT_DIR="$SCRIPT_DIR/.."
|
||||
PACKAGE_DIR="$ROOT_DIR/node/opendataloader-pdf"
|
||||
cd "$PACKAGE_DIR"
|
||||
|
||||
# Install dependencies
|
||||
pnpm install --frozen-lockfile
|
||||
|
||||
# Build
|
||||
pnpm run build
|
||||
|
||||
# Run tests
|
||||
pnpm test
|
||||
Executable
+36
@@ -0,0 +1,36 @@
|
||||
#!/bin/bash
|
||||
|
||||
# CI/CD build script for Python package using uv
|
||||
# For local development, use test-python.sh instead
|
||||
|
||||
set -e
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
|
||||
ROOT_DIR="$SCRIPT_DIR/.."
|
||||
PACKAGE_DIR="$ROOT_DIR/python/opendataloader-pdf"
|
||||
cd "$PACKAGE_DIR"
|
||||
|
||||
# Check uv is available
|
||||
command -v uv >/dev/null || { echo "Error: uv not found. Install with: curl -LsSf https://astral.sh/uv/install.sh | sh"; exit 1; }
|
||||
|
||||
# Clean previous build
|
||||
rm -rf dist/
|
||||
|
||||
# Copy README.md from repo root *before* build. hatchling validates [project.readme]
|
||||
# during metadata parsing, which runs BEFORE hatch_build.py's build hook — so we
|
||||
# cannot rely on the hook to provide it. Clean up on exit so branch switches
|
||||
# don't leave a stale copy in the package dir.
|
||||
cp "$ROOT_DIR/README.md" "$PACKAGE_DIR/README.md"
|
||||
trap 'rm -f "$PACKAGE_DIR/README.md"' EXIT
|
||||
|
||||
# Build sdist and wheel packages (hatch_build.py copies JAR/LICENSE/NOTICE/THIRD_PARTY)
|
||||
uv build
|
||||
|
||||
# Verify sdist contains required artifacts (JAR, LICENSE, NOTICE, THIRD_PARTY)
|
||||
"$SCRIPT_DIR/verify-python-sdist.sh"
|
||||
|
||||
# Install and run tests (include hybrid extras for full test coverage)
|
||||
uv sync --extra hybrid
|
||||
uv run pytest tests -v -s
|
||||
|
||||
echo "Build completed successfully."
|
||||
@@ -0,0 +1,158 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Baseline benchmark using docling-serve HTTP API.
|
||||
|
||||
Measures current docling-serve performance for comparison with
|
||||
FastAPI and subprocess approaches.
|
||||
|
||||
Usage:
|
||||
python scripts/experiments/docling_baseline_bench.py
|
||||
|
||||
Requirements:
|
||||
- docling-serve running on localhost:5001
|
||||
- requests package installed
|
||||
"""
|
||||
|
||||
import json
|
||||
import sys
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
import requests
|
||||
|
||||
# Configuration
|
||||
DOCLING_URL = "http://localhost:5001/v1/convert/file"
|
||||
PDF_DIR = Path(__file__).parent.parent.parent / "tests" / "benchmark" / "pdfs"
|
||||
RESULTS_DIR = Path(__file__).parent.parent.parent / "docs" / "hybrid" / "experiments"
|
||||
RESULTS_FILE = RESULTS_DIR / "baseline_results.json"
|
||||
|
||||
|
||||
def convert_pdf(pdf_path: Path) -> dict:
|
||||
"""Convert a single PDF using docling-serve API."""
|
||||
with open(pdf_path, "rb") as f:
|
||||
files = {"files": (pdf_path.name, f, "application/pdf")}
|
||||
data = {
|
||||
"to_formats": "md",
|
||||
"do_ocr": "true",
|
||||
"do_table_structure": "true",
|
||||
}
|
||||
|
||||
start_time = time.perf_counter()
|
||||
response = requests.post(DOCLING_URL, files=files, data=data, timeout=300)
|
||||
elapsed = time.perf_counter() - start_time
|
||||
|
||||
return {
|
||||
"filename": pdf_path.name,
|
||||
"status": "success" if response.status_code == 200 else "error",
|
||||
"elapsed": elapsed,
|
||||
"status_code": response.status_code,
|
||||
}
|
||||
|
||||
|
||||
def main():
|
||||
"""Run baseline benchmark."""
|
||||
# Check server health
|
||||
try:
|
||||
health = requests.get("http://localhost:5001/health", timeout=5)
|
||||
if health.status_code != 200:
|
||||
print("ERROR: docling-serve is not healthy", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
except requests.RequestException as e:
|
||||
print(f"ERROR: Cannot connect to docling-serve: {e}", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
print("=" * 60)
|
||||
print("Docling-serve Baseline Benchmark")
|
||||
print("=" * 60)
|
||||
print(f"PDF directory: {PDF_DIR}")
|
||||
print(f"Server URL: {DOCLING_URL}")
|
||||
print()
|
||||
|
||||
# Get PDF files
|
||||
pdf_files = sorted(PDF_DIR.glob("*.pdf"))
|
||||
total_files = len(pdf_files)
|
||||
print(f"Found {total_files} PDF files")
|
||||
print()
|
||||
|
||||
# Process each PDF
|
||||
results = []
|
||||
total_start = time.perf_counter()
|
||||
|
||||
for i, pdf_path in enumerate(pdf_files, 1):
|
||||
print(f"[{i:3d}/{total_files}] Processing {pdf_path.name}...", end=" ", flush=True)
|
||||
|
||||
try:
|
||||
result = convert_pdf(pdf_path)
|
||||
results.append(result)
|
||||
print(f"{result['elapsed']:.2f}s ({result['status']})")
|
||||
except Exception as e:
|
||||
results.append({
|
||||
"filename": pdf_path.name,
|
||||
"status": "error",
|
||||
"elapsed": 0,
|
||||
"error": str(e),
|
||||
})
|
||||
print(f"ERROR: {e}")
|
||||
|
||||
total_elapsed = time.perf_counter() - total_start
|
||||
|
||||
# Calculate statistics
|
||||
successful = [r for r in results if r["status"] == "success"]
|
||||
failed = [r for r in results if r["status"] != "success"]
|
||||
|
||||
if successful:
|
||||
elapsed_times = [r["elapsed"] for r in successful]
|
||||
avg_time = sum(elapsed_times) / len(elapsed_times)
|
||||
min_time = min(elapsed_times)
|
||||
max_time = max(elapsed_times)
|
||||
else:
|
||||
avg_time = min_time = max_time = 0
|
||||
|
||||
# Print summary
|
||||
print()
|
||||
print("=" * 60)
|
||||
print("RESULTS SUMMARY")
|
||||
print("=" * 60)
|
||||
print(f"Total documents: {total_files}")
|
||||
print(f"Successful: {len(successful)}")
|
||||
print(f"Failed: {len(failed)}")
|
||||
print()
|
||||
print(f"Total elapsed: {total_elapsed:.1f}s")
|
||||
print(f"Average per doc: {avg_time:.3f}s")
|
||||
print(f"Min: {min_time:.3f}s")
|
||||
print(f"Max: {max_time:.3f}s")
|
||||
print("=" * 60)
|
||||
|
||||
# Save results
|
||||
RESULTS_DIR.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
summary = {
|
||||
"approach": "baseline",
|
||||
"description": "docling-serve HTTP API",
|
||||
"timestamp": time.strftime("%Y-%m-%d %H:%M:%S"),
|
||||
"config": {
|
||||
"do_ocr": True,
|
||||
"do_table_structure": True,
|
||||
"server_url": DOCLING_URL,
|
||||
},
|
||||
"statistics": {
|
||||
"total_documents": total_files,
|
||||
"successful": len(successful),
|
||||
"failed": len(failed),
|
||||
"total_elapsed": round(total_elapsed, 2),
|
||||
"elapsed_per_doc": round(avg_time, 4),
|
||||
"min_elapsed": round(min_time, 4),
|
||||
"max_elapsed": round(max_time, 4),
|
||||
},
|
||||
"details": results,
|
||||
}
|
||||
|
||||
with open(RESULTS_FILE, "w", encoding="utf-8") as f:
|
||||
json.dump(summary, f, indent=2, ensure_ascii=False)
|
||||
|
||||
print(f"\nResults saved to: {RESULTS_FILE}")
|
||||
|
||||
return avg_time
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,289 @@
|
||||
#!/usr/bin/env python3
|
||||
"""FastAPI experiment benchmark using docling SDK directly.
|
||||
|
||||
Tests the hypothesis that a lightweight FastAPI server with
|
||||
DocumentConverter singleton is faster than docling-serve.
|
||||
|
||||
This script:
|
||||
1. Starts an embedded FastAPI server (port 5002)
|
||||
2. Converts all 200 benchmark PDFs
|
||||
3. Measures and reports performance
|
||||
|
||||
Usage:
|
||||
python scripts/experiments/docling_fastapi_bench.py
|
||||
|
||||
Requirements:
|
||||
- docling package installed
|
||||
- fastapi, uvicorn packages installed
|
||||
"""
|
||||
|
||||
import json
|
||||
import multiprocessing
|
||||
import os
|
||||
import sys
|
||||
import tempfile
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
import requests
|
||||
|
||||
# Configuration
|
||||
FASTAPI_PORT = 5002
|
||||
FASTAPI_URL = f"http://localhost:{FASTAPI_PORT}/convert"
|
||||
PDF_DIR = Path(__file__).parent.parent.parent / "tests" / "benchmark" / "pdfs"
|
||||
RESULTS_DIR = Path(__file__).parent.parent.parent / "docs" / "hybrid" / "experiments"
|
||||
RESULTS_FILE = RESULTS_DIR / "fastapi_results.json"
|
||||
|
||||
|
||||
def run_server():
|
||||
"""Run FastAPI server in a subprocess."""
|
||||
import uvicorn
|
||||
from fastapi import FastAPI, File, UploadFile
|
||||
from fastapi.responses import JSONResponse
|
||||
|
||||
# Import docling after fork to avoid issues
|
||||
from docling.datamodel.base_models import InputFormat
|
||||
from docling.datamodel.pipeline_options import (
|
||||
EasyOcrOptions,
|
||||
OcrOptions,
|
||||
PdfPipelineOptions,
|
||||
TableFormerMode,
|
||||
TableStructureOptions,
|
||||
)
|
||||
from docling.document_converter import DocumentConverter, PdfFormatOption
|
||||
|
||||
app = FastAPI()
|
||||
|
||||
# Create singleton DocumentConverter with warm-up
|
||||
print("Initializing DocumentConverter...", flush=True)
|
||||
|
||||
pipeline_options = PdfPipelineOptions(
|
||||
do_ocr=True,
|
||||
do_table_structure=True,
|
||||
ocr_options=EasyOcrOptions(force_full_page_ocr=False),
|
||||
table_structure_options=TableStructureOptions(
|
||||
mode=TableFormerMode.ACCURATE
|
||||
),
|
||||
)
|
||||
|
||||
converter = DocumentConverter(
|
||||
format_options={
|
||||
InputFormat.PDF: PdfFormatOption(pipeline_options=pipeline_options)
|
||||
}
|
||||
)
|
||||
print("DocumentConverter initialized.", flush=True)
|
||||
|
||||
@app.get("/health")
|
||||
def health():
|
||||
return {"status": "ok"}
|
||||
|
||||
@app.post("/convert")
|
||||
async def convert(file: UploadFile = File(...)):
|
||||
# Save uploaded file to temp location
|
||||
with tempfile.NamedTemporaryFile(suffix=".pdf", delete=False) as tmp:
|
||||
content = await file.read()
|
||||
tmp.write(content)
|
||||
tmp_path = tmp.name
|
||||
|
||||
try:
|
||||
start = time.perf_counter()
|
||||
result = converter.convert(tmp_path)
|
||||
elapsed = time.perf_counter() - start
|
||||
|
||||
md_content = result.document.export_to_markdown()
|
||||
|
||||
return JSONResponse({
|
||||
"status": "success",
|
||||
"markdown": md_content,
|
||||
"processing_time": elapsed,
|
||||
})
|
||||
except Exception:
|
||||
return JSONResponse({
|
||||
"status": "error",
|
||||
"error": "PDF conversion failed",
|
||||
}, status_code=500)
|
||||
finally:
|
||||
os.unlink(tmp_path)
|
||||
|
||||
uvicorn.run(app, host="0.0.0.0", port=FASTAPI_PORT, log_level="warning")
|
||||
|
||||
|
||||
def convert_pdf(pdf_path: Path) -> dict:
|
||||
"""Convert a single PDF using FastAPI server."""
|
||||
with open(pdf_path, "rb") as f:
|
||||
files = {"file": (pdf_path.name, f, "application/pdf")}
|
||||
|
||||
start_time = time.perf_counter()
|
||||
response = requests.post(FASTAPI_URL, files=files, timeout=300)
|
||||
elapsed = time.perf_counter() - start_time
|
||||
|
||||
if response.status_code == 200:
|
||||
data = response.json()
|
||||
return {
|
||||
"filename": pdf_path.name,
|
||||
"status": "success",
|
||||
"elapsed": elapsed,
|
||||
"server_time": data.get("processing_time", 0),
|
||||
}
|
||||
else:
|
||||
return {
|
||||
"filename": pdf_path.name,
|
||||
"status": "error",
|
||||
"elapsed": elapsed,
|
||||
"error": response.text,
|
||||
}
|
||||
|
||||
|
||||
def wait_for_server(max_retries=60, delay=1.0):
|
||||
"""Wait for server to be ready."""
|
||||
for i in range(max_retries):
|
||||
try:
|
||||
resp = requests.get(f"http://localhost:{FASTAPI_PORT}/health", timeout=5)
|
||||
if resp.status_code == 200:
|
||||
return True
|
||||
except requests.RequestException:
|
||||
pass
|
||||
time.sleep(delay)
|
||||
return False
|
||||
|
||||
|
||||
def main():
|
||||
"""Run FastAPI benchmark."""
|
||||
print("=" * 60)
|
||||
print("FastAPI Experiment Benchmark")
|
||||
print("=" * 60)
|
||||
print(f"PDF directory: {PDF_DIR}")
|
||||
print(f"Server URL: {FASTAPI_URL}")
|
||||
print()
|
||||
|
||||
# Start server in subprocess
|
||||
print("Starting FastAPI server...", flush=True)
|
||||
server_process = multiprocessing.Process(target=run_server, daemon=True)
|
||||
server_process.start()
|
||||
|
||||
# Wait for server to be ready
|
||||
print("Waiting for server to initialize (including model loading)...", flush=True)
|
||||
if not wait_for_server(max_retries=120, delay=1.0):
|
||||
print("ERROR: Server failed to start", file=sys.stderr)
|
||||
server_process.terminate()
|
||||
sys.exit(1)
|
||||
|
||||
print("Server is ready.", flush=True)
|
||||
print()
|
||||
|
||||
# Get PDF files
|
||||
pdf_files = sorted(PDF_DIR.glob("*.pdf"))
|
||||
total_files = len(pdf_files)
|
||||
print(f"Found {total_files} PDF files")
|
||||
print()
|
||||
|
||||
# Process each PDF
|
||||
results = []
|
||||
total_start = time.perf_counter()
|
||||
|
||||
try:
|
||||
for i, pdf_path in enumerate(pdf_files, 1):
|
||||
print(f"[{i:3d}/{total_files}] Processing {pdf_path.name}...", end=" ", flush=True)
|
||||
|
||||
try:
|
||||
result = convert_pdf(pdf_path)
|
||||
results.append(result)
|
||||
server_time = result.get("server_time", 0)
|
||||
print(f"{result['elapsed']:.2f}s (server: {server_time:.2f}s) ({result['status']})")
|
||||
except Exception as e:
|
||||
results.append({
|
||||
"filename": pdf_path.name,
|
||||
"status": "error",
|
||||
"elapsed": 0,
|
||||
"error": str(e),
|
||||
})
|
||||
print(f"ERROR: {e}")
|
||||
|
||||
total_elapsed = time.perf_counter() - total_start
|
||||
|
||||
finally:
|
||||
# Shutdown server
|
||||
print("\nShutting down server...", flush=True)
|
||||
server_process.terminate()
|
||||
server_process.join(timeout=5)
|
||||
|
||||
# Calculate statistics
|
||||
successful = [r for r in results if r["status"] == "success"]
|
||||
failed = [r for r in results if r["status"] != "success"]
|
||||
|
||||
if successful:
|
||||
elapsed_times = [r["elapsed"] for r in successful]
|
||||
server_times = [r.get("server_time", 0) for r in successful]
|
||||
avg_time = sum(elapsed_times) / len(elapsed_times)
|
||||
avg_server_time = sum(server_times) / len(server_times)
|
||||
min_time = min(elapsed_times)
|
||||
max_time = max(elapsed_times)
|
||||
else:
|
||||
avg_time = avg_server_time = min_time = max_time = 0
|
||||
|
||||
# Print summary
|
||||
print()
|
||||
print("=" * 60)
|
||||
print("RESULTS SUMMARY")
|
||||
print("=" * 60)
|
||||
print(f"Total documents: {total_files}")
|
||||
print(f"Successful: {len(successful)}")
|
||||
print(f"Failed: {len(failed)}")
|
||||
print()
|
||||
print(f"Total elapsed: {total_elapsed:.1f}s")
|
||||
print(f"Average per doc: {avg_time:.3f}s (target: < 0.8s)")
|
||||
print(f"Avg server time: {avg_server_time:.3f}s")
|
||||
print(f"Min: {min_time:.3f}s")
|
||||
print(f"Max: {max_time:.3f}s")
|
||||
print()
|
||||
|
||||
# Success/Failure check
|
||||
if avg_time < 0.8:
|
||||
print("✅ SUCCESS: Average time is below 0.8s threshold!")
|
||||
else:
|
||||
print("❌ FAILURE: Average time exceeds 0.8s threshold")
|
||||
print(" Plan may need to be discarded.")
|
||||
|
||||
print("=" * 60)
|
||||
|
||||
# Save results
|
||||
RESULTS_DIR.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
summary = {
|
||||
"approach": "fastapi",
|
||||
"description": "FastAPI server with docling SDK singleton",
|
||||
"timestamp": time.strftime("%Y-%m-%d %H:%M:%S"),
|
||||
"config": {
|
||||
"do_ocr": True,
|
||||
"do_table_structure": True,
|
||||
"server_port": FASTAPI_PORT,
|
||||
},
|
||||
"statistics": {
|
||||
"total_documents": total_files,
|
||||
"successful": len(successful),
|
||||
"failed": len(failed),
|
||||
"total_elapsed": round(total_elapsed, 2),
|
||||
"elapsed_per_doc": round(avg_time, 4),
|
||||
"server_time_per_doc": round(avg_server_time, 4),
|
||||
"min_elapsed": round(min_time, 4),
|
||||
"max_elapsed": round(max_time, 4),
|
||||
},
|
||||
"threshold": {
|
||||
"target": 0.8,
|
||||
"passed": avg_time < 0.8,
|
||||
},
|
||||
"details": results,
|
||||
}
|
||||
|
||||
with open(RESULTS_FILE, "w", encoding="utf-8") as f:
|
||||
json.dump(summary, f, indent=2, ensure_ascii=False)
|
||||
|
||||
print(f"\nResults saved to: {RESULTS_FILE}")
|
||||
|
||||
return avg_time
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
# Required for multiprocessing on macOS
|
||||
multiprocessing.set_start_method("spawn", force=True)
|
||||
main()
|
||||
@@ -0,0 +1,211 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Generate speed comparison report for docling experiments.
|
||||
|
||||
Reads results from all experiment runs and generates a summary report.
|
||||
|
||||
Usage:
|
||||
python scripts/experiments/docling_speed_report.py
|
||||
"""
|
||||
|
||||
import json
|
||||
import sys
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
|
||||
RESULTS_DIR = Path(__file__).parent.parent.parent / "docs" / "hybrid" / "experiments"
|
||||
REPORT_FILE = RESULTS_DIR / f"speed-experiment-{datetime.now().strftime('%Y-%m-%d')}.md"
|
||||
|
||||
|
||||
def load_results(filename: str) -> dict | None:
|
||||
"""Load results from JSON file."""
|
||||
path = RESULTS_DIR / filename
|
||||
if not path.exists():
|
||||
return None
|
||||
with open(path, encoding="utf-8") as f:
|
||||
return json.load(f)
|
||||
|
||||
|
||||
def main():
|
||||
"""Generate comparison report."""
|
||||
print("Loading experiment results...")
|
||||
|
||||
baseline = load_results("baseline_results.json")
|
||||
fastapi = load_results("fastapi_results.json")
|
||||
subprocess = load_results("subprocess_results.json")
|
||||
|
||||
if not any([baseline, fastapi, subprocess]):
|
||||
print("ERROR: No experiment results found", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
# Print console summary
|
||||
print()
|
||||
print("=" * 70)
|
||||
print("DOCLING SPEED EXPERIMENT RESULTS")
|
||||
print("=" * 70)
|
||||
print()
|
||||
|
||||
approaches = []
|
||||
if baseline:
|
||||
approaches.append(("baseline", "docling-serve HTTP", baseline))
|
||||
if fastapi:
|
||||
approaches.append(("fastapi", "FastAPI + SDK singleton", fastapi))
|
||||
if subprocess:
|
||||
approaches.append(("subprocess", "Persistent subprocess", subprocess))
|
||||
|
||||
# Table header
|
||||
print(f"{'Approach':<15} {'Description':<25} {'Avg (s/doc)':<12} {'Target':<10} {'Status':<10} {'Speedup':<10}")
|
||||
print("-" * 70)
|
||||
|
||||
baseline_time = baseline["statistics"]["elapsed_per_doc"] if baseline else None
|
||||
|
||||
for name, desc, data in approaches:
|
||||
stats = data["statistics"]
|
||||
avg_time = stats["elapsed_per_doc"]
|
||||
|
||||
threshold = data.get("threshold", {})
|
||||
target = threshold.get("target", "-")
|
||||
passed = threshold.get("passed", None)
|
||||
|
||||
if passed is True:
|
||||
status = "PASS"
|
||||
elif passed is False:
|
||||
status = "FAIL"
|
||||
else:
|
||||
status = "-"
|
||||
|
||||
# Calculate speedup vs baseline
|
||||
if baseline_time and name != "baseline":
|
||||
speedup = f"{baseline_time / avg_time:.1f}x"
|
||||
else:
|
||||
speedup = "-"
|
||||
|
||||
print(f"{name:<15} {desc:<25} {avg_time:<12.3f} {str(target):<10} {status:<10} {speedup:<10}")
|
||||
|
||||
print("-" * 70)
|
||||
print()
|
||||
|
||||
# Decision summary
|
||||
print("DECISION SUMMARY:")
|
||||
print("-" * 40)
|
||||
|
||||
fastapi_passed = fastapi and fastapi.get("threshold", {}).get("passed", False)
|
||||
subprocess_passed = subprocess and subprocess.get("threshold", {}).get("passed", False)
|
||||
|
||||
if fastapi_passed:
|
||||
print("FastAPI approach: APPROVED (proceed to Phase 1)")
|
||||
else:
|
||||
print("FastAPI approach: REJECTED (plan discarded)")
|
||||
|
||||
if subprocess_passed:
|
||||
print("Subprocess approach: APPROVED (proceed to Phase 1)")
|
||||
else:
|
||||
print("Subprocess approach: REJECTED (excluded from plan)")
|
||||
|
||||
print()
|
||||
|
||||
if fastapi_passed:
|
||||
print("OVERALL: Phase 0 PASSED - Proceed to implementation")
|
||||
print()
|
||||
|
||||
# Recommendation
|
||||
if subprocess_passed:
|
||||
fastapi_time = fastapi["statistics"]["elapsed_per_doc"]
|
||||
subprocess_time = subprocess["statistics"]["elapsed_per_doc"]
|
||||
if subprocess_time < fastapi_time:
|
||||
print(f"RECOMMENDATION: subprocess approach is slightly faster ({subprocess_time:.3f}s vs {fastapi_time:.3f}s)")
|
||||
print(" However, FastAPI is more production-ready (health checks, easier deployment)")
|
||||
else:
|
||||
print(f"RECOMMENDATION: FastAPI approach is faster and more production-ready")
|
||||
else:
|
||||
print("OVERALL: Phase 0 FAILED - Plan should be discarded")
|
||||
|
||||
print("=" * 70)
|
||||
|
||||
# Generate markdown report
|
||||
RESULTS_DIR.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
report = []
|
||||
report.append("# Docling Speed Experiment Results")
|
||||
report.append("")
|
||||
report.append(f"**Date**: {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}")
|
||||
report.append("")
|
||||
|
||||
report.append("## Summary")
|
||||
report.append("")
|
||||
report.append("| Approach | Description | Avg (s/doc) | Target | Status | Speedup |")
|
||||
report.append("|----------|-------------|-------------|--------|--------|---------|")
|
||||
|
||||
for name, desc, data in approaches:
|
||||
stats = data["statistics"]
|
||||
avg_time = stats["elapsed_per_doc"]
|
||||
threshold = data.get("threshold", {})
|
||||
target = threshold.get("target", "-")
|
||||
passed = threshold.get("passed", None)
|
||||
|
||||
if passed is True:
|
||||
status = "PASS"
|
||||
elif passed is False:
|
||||
status = "FAIL"
|
||||
else:
|
||||
status = "-"
|
||||
|
||||
if baseline_time and name != "baseline":
|
||||
speedup = f"{baseline_time / avg_time:.1f}x"
|
||||
else:
|
||||
speedup = "-"
|
||||
|
||||
report.append(f"| {name} | {desc} | {avg_time:.3f} | {target} | {status} | {speedup} |")
|
||||
|
||||
report.append("")
|
||||
report.append("## Decision")
|
||||
report.append("")
|
||||
|
||||
if fastapi_passed:
|
||||
report.append("**Phase 0 PASSED** - FastAPI approach meets the < 0.8s threshold.")
|
||||
report.append("")
|
||||
report.append("Proceed to Phase 1 implementation:")
|
||||
report.append("")
|
||||
report.append("- [ ] Task 1.1: docling_subprocess_worker.py")
|
||||
report.append("- [ ] Task 1.2: docling_fast_server.py")
|
||||
report.append("- [ ] Task 2.1: DoclingSubprocessClient.java")
|
||||
report.append("- [ ] Task 2.2: DoclingFastServerClient.java")
|
||||
report.append("- [ ] Task 2.3: HybridClientFactory modification")
|
||||
report.append("- [ ] Task 3: Benchmark integration")
|
||||
report.append("- [ ] Task 4: Final validation")
|
||||
|
||||
if subprocess_passed:
|
||||
report.append("")
|
||||
report.append("Subprocess approach also passed - both approaches available for implementation.")
|
||||
else:
|
||||
report.append("**Phase 0 FAILED** - FastAPI approach exceeds 0.8s threshold.")
|
||||
report.append("")
|
||||
report.append("Plan should be discarded. Consider alternative approaches.")
|
||||
|
||||
report.append("")
|
||||
report.append("## Detailed Statistics")
|
||||
report.append("")
|
||||
|
||||
for name, desc, data in approaches:
|
||||
stats = data["statistics"]
|
||||
report.append(f"### {name.title()}")
|
||||
report.append("")
|
||||
report.append(f"- **Description**: {data['description']}")
|
||||
report.append(f"- **Timestamp**: {data['timestamp']}")
|
||||
report.append(f"- **Total documents**: {stats['total_documents']}")
|
||||
report.append(f"- **Successful**: {stats['successful']}")
|
||||
report.append(f"- **Failed**: {stats['failed']}")
|
||||
report.append(f"- **Total elapsed**: {stats['total_elapsed']:.1f}s")
|
||||
report.append(f"- **Average per doc**: {stats['elapsed_per_doc']:.4f}s")
|
||||
report.append(f"- **Min**: {stats['min_elapsed']:.4f}s")
|
||||
report.append(f"- **Max**: {stats['max_elapsed']:.4f}s")
|
||||
report.append("")
|
||||
|
||||
# Write report
|
||||
with open(REPORT_FILE, "w", encoding="utf-8") as f:
|
||||
f.write("\n".join(report))
|
||||
|
||||
print(f"\nReport saved to: {REPORT_FILE}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,331 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Subprocess experiment benchmark using docling SDK directly.
|
||||
|
||||
Tests the subprocess approach where each PDF is processed by
|
||||
invoking a Python worker script via subprocess.
|
||||
|
||||
This approach has overhead from:
|
||||
1. Python interpreter startup
|
||||
2. Model loading per-process (unless using persistent worker)
|
||||
|
||||
For this experiment, we test a persistent worker approach:
|
||||
- Single Python process stays alive
|
||||
- Receives PDF paths via stdin, outputs JSON via stdout
|
||||
|
||||
Usage:
|
||||
python scripts/experiments/docling_subprocess_bench.py
|
||||
|
||||
Requirements:
|
||||
- docling package installed
|
||||
"""
|
||||
|
||||
import base64
|
||||
import json
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
# Configuration
|
||||
PDF_DIR = Path(__file__).parent.parent.parent / "tests" / "benchmark" / "pdfs"
|
||||
RESULTS_DIR = Path(__file__).parent.parent.parent / "docs" / "hybrid" / "experiments"
|
||||
RESULTS_FILE = RESULTS_DIR / "subprocess_results.json"
|
||||
|
||||
# Worker script inline - will be written to temp file
|
||||
WORKER_SCRIPT = '''
|
||||
import base64
|
||||
import json
|
||||
import sys
|
||||
import time
|
||||
import tempfile
|
||||
import os
|
||||
|
||||
# Initialize docling once
|
||||
from docling.datamodel.base_models import InputFormat
|
||||
from docling.datamodel.pipeline_options import (
|
||||
EasyOcrOptions,
|
||||
PdfPipelineOptions,
|
||||
TableFormerMode,
|
||||
TableStructureOptions,
|
||||
)
|
||||
from docling.document_converter import DocumentConverter, PdfFormatOption
|
||||
|
||||
print("WORKER_READY", file=sys.stderr, flush=True)
|
||||
|
||||
pipeline_options = PdfPipelineOptions(
|
||||
do_ocr=True,
|
||||
do_table_structure=True,
|
||||
ocr_options=EasyOcrOptions(force_full_page_ocr=False),
|
||||
table_structure_options=TableStructureOptions(
|
||||
mode=TableFormerMode.ACCURATE
|
||||
),
|
||||
)
|
||||
|
||||
converter = DocumentConverter(
|
||||
format_options={
|
||||
InputFormat.PDF: PdfFormatOption(pipeline_options=pipeline_options)
|
||||
}
|
||||
)
|
||||
|
||||
print("CONVERTER_READY", file=sys.stderr, flush=True)
|
||||
|
||||
# Process requests from stdin
|
||||
for line in sys.stdin:
|
||||
line = line.strip()
|
||||
if not line:
|
||||
continue
|
||||
|
||||
try:
|
||||
request = json.loads(line)
|
||||
pdf_base64 = request.get("pdf_base64")
|
||||
filename = request.get("filename", "document.pdf")
|
||||
|
||||
# Decode and write to temp file
|
||||
pdf_bytes = base64.b64decode(pdf_base64)
|
||||
with tempfile.NamedTemporaryFile(suffix=".pdf", delete=False) as tmp:
|
||||
tmp.write(pdf_bytes)
|
||||
tmp_path = tmp.name
|
||||
|
||||
try:
|
||||
start = time.perf_counter()
|
||||
result = converter.convert(tmp_path)
|
||||
elapsed = time.perf_counter() - start
|
||||
|
||||
md_content = result.document.export_to_markdown()
|
||||
|
||||
response = {
|
||||
"status": "success",
|
||||
"filename": filename,
|
||||
"markdown": md_content,
|
||||
"processing_time": elapsed,
|
||||
}
|
||||
except Exception as e:
|
||||
response = {
|
||||
"status": "error",
|
||||
"filename": filename,
|
||||
"error": str(e),
|
||||
}
|
||||
finally:
|
||||
os.unlink(tmp_path)
|
||||
|
||||
print(json.dumps(response), flush=True)
|
||||
|
||||
except Exception as e:
|
||||
response = {
|
||||
"status": "error",
|
||||
"error": str(e),
|
||||
}
|
||||
print(json.dumps(response), flush=True)
|
||||
'''
|
||||
|
||||
|
||||
def convert_pdf(process: subprocess.Popen, pdf_path: Path) -> dict:
|
||||
"""Convert a single PDF using subprocess worker."""
|
||||
# Read PDF and encode as base64
|
||||
with open(pdf_path, "rb") as f:
|
||||
pdf_bytes = f.read()
|
||||
pdf_base64 = base64.b64encode(pdf_bytes).decode("ascii")
|
||||
|
||||
# Send request
|
||||
request = {
|
||||
"pdf_base64": pdf_base64,
|
||||
"filename": pdf_path.name,
|
||||
}
|
||||
|
||||
start_time = time.perf_counter()
|
||||
process.stdin.write(json.dumps(request) + "\n")
|
||||
process.stdin.flush()
|
||||
|
||||
# Read response
|
||||
response_line = process.stdout.readline()
|
||||
elapsed = time.perf_counter() - start_time
|
||||
|
||||
if response_line:
|
||||
try:
|
||||
response = json.loads(response_line)
|
||||
response["client_elapsed"] = elapsed
|
||||
return response
|
||||
except json.JSONDecodeError as e:
|
||||
return {
|
||||
"filename": pdf_path.name,
|
||||
"status": "error",
|
||||
"error": f"JSON decode error: {e}",
|
||||
"client_elapsed": elapsed,
|
||||
}
|
||||
else:
|
||||
return {
|
||||
"filename": pdf_path.name,
|
||||
"status": "error",
|
||||
"error": "No response from worker",
|
||||
"client_elapsed": elapsed,
|
||||
}
|
||||
|
||||
|
||||
def main():
|
||||
"""Run subprocess benchmark."""
|
||||
print("=" * 60)
|
||||
print("Subprocess Experiment Benchmark")
|
||||
print("=" * 60)
|
||||
print(f"PDF directory: {PDF_DIR}")
|
||||
print()
|
||||
|
||||
# Write worker script to temp file
|
||||
with tempfile.NamedTemporaryFile(mode="w", suffix=".py", delete=False) as f:
|
||||
f.write(WORKER_SCRIPT)
|
||||
worker_path = f.name
|
||||
|
||||
print("Starting worker process...", flush=True)
|
||||
|
||||
try:
|
||||
# Start worker process
|
||||
process = subprocess.Popen(
|
||||
[sys.executable, worker_path],
|
||||
stdin=subprocess.PIPE,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE,
|
||||
text=True,
|
||||
bufsize=1, # Line buffered
|
||||
)
|
||||
|
||||
# Wait for worker to be ready (read stderr for status messages)
|
||||
print("Waiting for worker to initialize (including model loading)...", flush=True)
|
||||
|
||||
ready_count = 0
|
||||
while ready_count < 2:
|
||||
line = process.stderr.readline()
|
||||
if "WORKER_READY" in line:
|
||||
ready_count += 1
|
||||
print(" - Worker process started", flush=True)
|
||||
elif "CONVERTER_READY" in line:
|
||||
ready_count += 1
|
||||
print(" - DocumentConverter initialized", flush=True)
|
||||
elif process.poll() is not None:
|
||||
print("ERROR: Worker process died unexpectedly", file=sys.stderr)
|
||||
remaining_stderr = process.stderr.read()
|
||||
print(remaining_stderr, file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
print("Worker is ready.", flush=True)
|
||||
print()
|
||||
|
||||
# Get PDF files
|
||||
pdf_files = sorted(PDF_DIR.glob("*.pdf"))
|
||||
total_files = len(pdf_files)
|
||||
print(f"Found {total_files} PDF files")
|
||||
print()
|
||||
|
||||
# Process each PDF
|
||||
results = []
|
||||
total_start = time.perf_counter()
|
||||
|
||||
for i, pdf_path in enumerate(pdf_files, 1):
|
||||
print(f"[{i:3d}/{total_files}] Processing {pdf_path.name}...", end=" ", flush=True)
|
||||
|
||||
try:
|
||||
result = convert_pdf(process, pdf_path)
|
||||
results.append(result)
|
||||
server_time = result.get("processing_time", 0)
|
||||
client_time = result.get("client_elapsed", 0)
|
||||
print(f"{client_time:.2f}s (server: {server_time:.2f}s) ({result['status']})")
|
||||
except Exception as e:
|
||||
results.append({
|
||||
"filename": pdf_path.name,
|
||||
"status": "error",
|
||||
"client_elapsed": 0,
|
||||
"error": str(e),
|
||||
})
|
||||
print(f"ERROR: {e}")
|
||||
|
||||
total_elapsed = time.perf_counter() - total_start
|
||||
|
||||
finally:
|
||||
# Shutdown worker
|
||||
print("\nShutting down worker...", flush=True)
|
||||
if process.poll() is None:
|
||||
process.stdin.close()
|
||||
process.terminate()
|
||||
process.wait(timeout=5)
|
||||
|
||||
# Clean up worker script
|
||||
import os
|
||||
os.unlink(worker_path)
|
||||
|
||||
# Calculate statistics
|
||||
successful = [r for r in results if r["status"] == "success"]
|
||||
failed = [r for r in results if r["status"] != "success"]
|
||||
|
||||
if successful:
|
||||
client_times = [r.get("client_elapsed", 0) for r in successful]
|
||||
server_times = [r.get("processing_time", 0) for r in successful]
|
||||
avg_client_time = sum(client_times) / len(client_times)
|
||||
avg_server_time = sum(server_times) / len(server_times)
|
||||
min_time = min(client_times)
|
||||
max_time = max(client_times)
|
||||
else:
|
||||
avg_client_time = avg_server_time = min_time = max_time = 0
|
||||
|
||||
# Print summary
|
||||
print()
|
||||
print("=" * 60)
|
||||
print("RESULTS SUMMARY")
|
||||
print("=" * 60)
|
||||
print(f"Total documents: {total_files}")
|
||||
print(f"Successful: {len(successful)}")
|
||||
print(f"Failed: {len(failed)}")
|
||||
print()
|
||||
print(f"Total elapsed: {total_elapsed:.1f}s")
|
||||
print(f"Average per doc: {avg_client_time:.3f}s (target: < 1.0s)")
|
||||
print(f"Avg server time: {avg_server_time:.3f}s")
|
||||
print(f"Min: {min_time:.3f}s")
|
||||
print(f"Max: {max_time:.3f}s")
|
||||
print()
|
||||
|
||||
# Success/Failure check
|
||||
if avg_client_time < 1.0:
|
||||
print("✅ SUCCESS: Average time is below 1.0s threshold!")
|
||||
else:
|
||||
print("❌ FAILURE: Average time exceeds 1.0s threshold")
|
||||
print(" Subprocess approach will be excluded.")
|
||||
|
||||
print("=" * 60)
|
||||
|
||||
# Save results
|
||||
RESULTS_DIR.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
summary = {
|
||||
"approach": "subprocess",
|
||||
"description": "Persistent Python subprocess with docling SDK",
|
||||
"timestamp": time.strftime("%Y-%m-%d %H:%M:%S"),
|
||||
"config": {
|
||||
"do_ocr": True,
|
||||
"do_table_structure": True,
|
||||
"worker_type": "persistent",
|
||||
},
|
||||
"statistics": {
|
||||
"total_documents": total_files,
|
||||
"successful": len(successful),
|
||||
"failed": len(failed),
|
||||
"total_elapsed": round(total_elapsed, 2),
|
||||
"elapsed_per_doc": round(avg_client_time, 4),
|
||||
"server_time_per_doc": round(avg_server_time, 4),
|
||||
"min_elapsed": round(min_time, 4),
|
||||
"max_elapsed": round(max_time, 4),
|
||||
},
|
||||
"threshold": {
|
||||
"target": 1.0,
|
||||
"passed": avg_client_time < 1.0,
|
||||
},
|
||||
"details": results,
|
||||
}
|
||||
|
||||
with open(RESULTS_FILE, "w", encoding="utf-8") as f:
|
||||
json.dump(summary, f, indent=2, ensure_ascii=False)
|
||||
|
||||
print(f"\nResults saved to: {RESULTS_FILE}")
|
||||
|
||||
return avg_client_time
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,547 @@
|
||||
#!/usr/bin/env node
|
||||
/**
|
||||
* Generates CLI option definitions for Node.js, Python, and documentation
|
||||
* from the single source of truth (options.json).
|
||||
*
|
||||
* Usage: node scripts/generate-options.mjs
|
||||
*/
|
||||
|
||||
import { readFileSync, writeFileSync, mkdirSync } from 'node:fs';
|
||||
import { dirname, join } from 'node:path';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
import { escapeMarkdown, formatTable } from './utils.mjs';
|
||||
|
||||
const __dirname = dirname(fileURLToPath(import.meta.url));
|
||||
const ROOT_DIR = join(__dirname, '..');
|
||||
|
||||
// Read options.json
|
||||
const optionsPath = join(ROOT_DIR, 'options.json');
|
||||
const options = JSON.parse(readFileSync(optionsPath, 'utf-8'));
|
||||
|
||||
const AUTO_GENERATED_HEADER = `// AUTO-GENERATED FROM options.json - DO NOT EDIT DIRECTLY
|
||||
// Run \`npm run generate-options\` to regenerate
|
||||
`;
|
||||
|
||||
const AUTO_GENERATED_HEADER_PYTHON = `# AUTO-GENERATED FROM options.json - DO NOT EDIT DIRECTLY
|
||||
# Run \`npm run generate-options\` to regenerate
|
||||
`;
|
||||
|
||||
const AUTO_GENERATED_HEADER_MDX = `{/* AUTO-GENERATED FROM options.json - DO NOT EDIT DIRECTLY */}
|
||||
{/* Run \`npm run generate-options\` to regenerate */}
|
||||
|
||||
`;
|
||||
|
||||
/**
|
||||
* Convert kebab-case to camelCase
|
||||
*/
|
||||
function toCamelCase(str) {
|
||||
return str.replace(/-([a-z])/g, (_, letter) => letter.toUpperCase());
|
||||
}
|
||||
|
||||
/**
|
||||
* Convert kebab-case to snake_case
|
||||
*/
|
||||
function toSnakeCase(str) {
|
||||
return str.replace(/-/g, '_');
|
||||
}
|
||||
|
||||
/**
|
||||
* Options that accept comma-separated list values.
|
||||
*/
|
||||
const LIST_OPTIONS = new Set(['format', 'content-safety-off']);
|
||||
|
||||
/**
|
||||
* Check if option supports list values.
|
||||
*/
|
||||
function isListOption(opt) {
|
||||
return LIST_OPTIONS.has(opt.name);
|
||||
}
|
||||
|
||||
/**
|
||||
* Escape string for use in generated code.
|
||||
* @param {string} str - The string to escape
|
||||
* @param {string} quote - The quote character (' or ")
|
||||
* @param {object} options - Additional escape options
|
||||
* @param {boolean} options.escapePercent - Escape % as %% for Python argparse
|
||||
*/
|
||||
function escapeString(str, quote = "'", { escapePercent = false } = {}) {
|
||||
let result = str.replace(/\\/g, '\\\\'); // escape backslashes first
|
||||
if (quote === "'") {
|
||||
result = result.replace(/'/g, "\\'");
|
||||
} else {
|
||||
result = result.replace(/"/g, '\\"');
|
||||
}
|
||||
if (escapePercent) {
|
||||
result = result.replace(/%/g, '%%');
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
/**
|
||||
* Generate Node.js CLI options file
|
||||
*/
|
||||
function generateNodeCliOptions() {
|
||||
const lines = [AUTO_GENERATED_HEADER];
|
||||
lines.push(`import { Command } from 'commander';`);
|
||||
lines.push('');
|
||||
lines.push('/**');
|
||||
lines.push(' * Register all CLI options on the given Commander program.');
|
||||
lines.push(' */');
|
||||
lines.push('export function registerCliOptions(program: Command): void {');
|
||||
|
||||
for (const opt of options.options) {
|
||||
const flags = opt.shortName
|
||||
? `-${opt.shortName}, --${opt.name}${opt.type === 'string' ? ' <value>' : ''}`
|
||||
: `--${opt.name}${opt.type === 'string' ? ' <value>' : ''}`;
|
||||
|
||||
const description = escapeString(opt.description, "'");
|
||||
lines.push(` program.option('${flags}', '${description}');`);
|
||||
}
|
||||
|
||||
lines.push('}');
|
||||
lines.push('');
|
||||
|
||||
const outputPath = join(ROOT_DIR, 'node/opendataloader-pdf/src/cli-options.generated.ts');
|
||||
writeFileSync(outputPath, lines.join('\n'));
|
||||
console.log(`Generated: ${outputPath}`);
|
||||
}
|
||||
|
||||
/**
|
||||
* Generate Node.js ConvertOptions interface and helper functions
|
||||
*/
|
||||
function generateNodeConvertOptions() {
|
||||
const lines = [AUTO_GENERATED_HEADER];
|
||||
|
||||
// Generate ConvertOptions interface
|
||||
lines.push('/**');
|
||||
lines.push(' * Options for the convert function.');
|
||||
lines.push(' */');
|
||||
lines.push('export interface ConvertOptions {');
|
||||
|
||||
for (const opt of options.options) {
|
||||
const camelName = toCamelCase(opt.name);
|
||||
let tsType = 'string';
|
||||
|
||||
if (opt.type === 'boolean') {
|
||||
tsType = 'boolean';
|
||||
} else if (isListOption(opt)) {
|
||||
tsType = 'string | string[]';
|
||||
}
|
||||
|
||||
lines.push(` /** ${opt.description} */`);
|
||||
lines.push(` ${camelName}?: ${tsType};`);
|
||||
}
|
||||
|
||||
lines.push('}');
|
||||
lines.push('');
|
||||
|
||||
// Generate CliOptions interface (for CLI parsing - all values are strings from commander)
|
||||
lines.push('/**');
|
||||
lines.push(' * Options as parsed from CLI (all values are strings from commander).');
|
||||
lines.push(' */');
|
||||
lines.push('export interface CliOptions {');
|
||||
|
||||
for (const opt of options.options) {
|
||||
const camelName = toCamelCase(opt.name);
|
||||
const tsType = opt.type === 'boolean' ? 'boolean' : 'string';
|
||||
lines.push(` ${camelName}?: ${tsType};`);
|
||||
}
|
||||
|
||||
lines.push('}');
|
||||
lines.push('');
|
||||
|
||||
// Generate buildConvertOptions function
|
||||
lines.push('/**');
|
||||
lines.push(' * Convert CLI options to ConvertOptions.');
|
||||
lines.push(' */');
|
||||
lines.push('export function buildConvertOptions(cliOptions: CliOptions): ConvertOptions {');
|
||||
lines.push(' const convertOptions: ConvertOptions = {};');
|
||||
lines.push('');
|
||||
|
||||
for (const opt of options.options) {
|
||||
const camelName = toCamelCase(opt.name);
|
||||
if (opt.type === 'boolean') {
|
||||
lines.push(` if (cliOptions.${camelName}) {`);
|
||||
lines.push(` convertOptions.${camelName} = true;`);
|
||||
lines.push(' }');
|
||||
} else {
|
||||
lines.push(` if (cliOptions.${camelName}) {`);
|
||||
lines.push(` convertOptions.${camelName} = cliOptions.${camelName};`);
|
||||
lines.push(' }');
|
||||
}
|
||||
}
|
||||
|
||||
lines.push('');
|
||||
lines.push(' return convertOptions;');
|
||||
lines.push('}');
|
||||
lines.push('');
|
||||
|
||||
// Generate buildArgs function
|
||||
lines.push('/**');
|
||||
lines.push(' * Build CLI arguments array from ConvertOptions.');
|
||||
lines.push(' */');
|
||||
lines.push('export function buildArgs(options: ConvertOptions): string[] {');
|
||||
lines.push(' const args: string[] = [];');
|
||||
lines.push('');
|
||||
|
||||
for (const opt of options.options) {
|
||||
const camelName = toCamelCase(opt.name);
|
||||
const cliFlag = `--${opt.name}`;
|
||||
|
||||
if (opt.type === 'boolean') {
|
||||
lines.push(` if (options.${camelName}) {`);
|
||||
lines.push(` args.push('${cliFlag}');`);
|
||||
lines.push(' }');
|
||||
} else if (isListOption(opt)) {
|
||||
lines.push(` if (options.${camelName}) {`);
|
||||
lines.push(` if (Array.isArray(options.${camelName})) {`);
|
||||
lines.push(` if (options.${camelName}.length > 0) {`);
|
||||
lines.push(` args.push('${cliFlag}', options.${camelName}.join(','));`);
|
||||
lines.push(' }');
|
||||
lines.push(' } else {');
|
||||
lines.push(` args.push('${cliFlag}', options.${camelName});`);
|
||||
lines.push(' }');
|
||||
lines.push(' }');
|
||||
} else {
|
||||
lines.push(` if (options.${camelName}) {`);
|
||||
lines.push(` args.push('${cliFlag}', options.${camelName});`);
|
||||
lines.push(' }');
|
||||
}
|
||||
}
|
||||
|
||||
lines.push('');
|
||||
lines.push(' return args;');
|
||||
lines.push('}');
|
||||
lines.push('');
|
||||
|
||||
const outputPath = join(ROOT_DIR, 'node/opendataloader-pdf/src/convert-options.generated.ts');
|
||||
writeFileSync(outputPath, lines.join('\n'));
|
||||
console.log(`Generated: ${outputPath}`);
|
||||
}
|
||||
|
||||
/**
|
||||
* Generate Python CLI options file (cli_options.py)
|
||||
*/
|
||||
function generatePythonCliOptions() {
|
||||
const lines = [AUTO_GENERATED_HEADER_PYTHON];
|
||||
lines.push('"""');
|
||||
lines.push('CLI option definitions for opendataloader-pdf.');
|
||||
lines.push('"""');
|
||||
lines.push('from typing import Any, Dict, List');
|
||||
lines.push('');
|
||||
lines.push('');
|
||||
lines.push('# Option metadata list');
|
||||
lines.push('CLI_OPTIONS: List[Dict[str, Any]] = [');
|
||||
|
||||
for (const opt of options.options) {
|
||||
const snakeName = toSnakeCase(opt.name);
|
||||
const defaultValue = opt.default === null ? 'None'
|
||||
: typeof opt.default === 'boolean' ? (opt.default ? 'True' : 'False')
|
||||
: `"${opt.default}"`;
|
||||
|
||||
lines.push(' {');
|
||||
lines.push(` "name": "${opt.name}",`);
|
||||
lines.push(` "python_name": "${snakeName}",`);
|
||||
lines.push(` "short_name": ${opt.shortName ? `"${opt.shortName}"` : 'None'},`);
|
||||
lines.push(` "type": "${opt.type}",`);
|
||||
lines.push(` "required": ${opt.required ? 'True' : 'False'},`);
|
||||
lines.push(` "default": ${defaultValue},`);
|
||||
lines.push(` "description": "${escapeString(opt.description, '"', { escapePercent: true })}",`);
|
||||
lines.push(' },');
|
||||
}
|
||||
|
||||
lines.push(']');
|
||||
lines.push('');
|
||||
lines.push('');
|
||||
lines.push('def add_options_to_parser(parser) -> None:');
|
||||
lines.push(' """Add all CLI options to an argparse.ArgumentParser."""');
|
||||
lines.push(' for opt in CLI_OPTIONS:');
|
||||
lines.push(' flags = []');
|
||||
lines.push(' if opt["short_name"]:');
|
||||
lines.push(" flags.append(f'-{opt[\"short_name\"]}')");
|
||||
lines.push(" flags.append(f'--{opt[\"name\"]}')");
|
||||
lines.push('');
|
||||
lines.push(' kwargs = {"help": opt["description"]}');
|
||||
lines.push(' if opt["type"] == "boolean":');
|
||||
lines.push(' kwargs["action"] = "store_true"');
|
||||
lines.push(' else:');
|
||||
lines.push(' kwargs["default"] = None');
|
||||
lines.push('');
|
||||
lines.push(' parser.add_argument(*flags, **kwargs)');
|
||||
lines.push('');
|
||||
|
||||
const outputPath = join(ROOT_DIR, 'python/opendataloader-pdf/src/opendataloader_pdf/cli_options_generated.py');
|
||||
mkdirSync(dirname(outputPath), { recursive: true });
|
||||
writeFileSync(outputPath, lines.join('\n'));
|
||||
console.log(`Generated: ${outputPath}`);
|
||||
}
|
||||
|
||||
/**
|
||||
* Generate Python convert function (convert.py)
|
||||
*/
|
||||
function generatePythonConvert() {
|
||||
const lines = [AUTO_GENERATED_HEADER_PYTHON];
|
||||
lines.push('"""');
|
||||
lines.push('Auto-generated convert function for opendataloader-pdf.');
|
||||
lines.push('"""');
|
||||
lines.push('from typing import List, Optional, Union');
|
||||
lines.push('');
|
||||
lines.push('from .runner import run_jar');
|
||||
lines.push('');
|
||||
lines.push('');
|
||||
|
||||
// Generate function signature
|
||||
lines.push('def convert(');
|
||||
lines.push(' input_path: Union[str, List[str]],');
|
||||
|
||||
for (const opt of options.options) {
|
||||
const snakeName = toSnakeCase(opt.name);
|
||||
let typeHint;
|
||||
let defaultVal;
|
||||
|
||||
if (opt.type === 'boolean') {
|
||||
typeHint = 'bool';
|
||||
defaultVal = opt.default ? 'True' : 'False';
|
||||
} else if (isListOption(opt)) {
|
||||
typeHint = 'Optional[Union[str, List[str]]]';
|
||||
defaultVal = 'None';
|
||||
} else {
|
||||
typeHint = 'Optional[str]';
|
||||
defaultVal = 'None';
|
||||
}
|
||||
|
||||
lines.push(` ${snakeName}: ${typeHint} = ${defaultVal},`);
|
||||
}
|
||||
|
||||
lines.push(') -> None:');
|
||||
lines.push(' """');
|
||||
lines.push(' Convert PDF(s) into the requested output format(s).');
|
||||
lines.push('');
|
||||
lines.push(' Args:');
|
||||
lines.push(' input_path: One or more input PDF file paths or directories');
|
||||
|
||||
for (const opt of options.options) {
|
||||
const snakeName = toSnakeCase(opt.name);
|
||||
lines.push(` ${snakeName}: ${opt.description}`);
|
||||
}
|
||||
|
||||
lines.push(' """');
|
||||
|
||||
// Generate function body
|
||||
lines.push(' args: List[str] = []');
|
||||
lines.push('');
|
||||
lines.push(' # Build input paths');
|
||||
lines.push(' if isinstance(input_path, list):');
|
||||
lines.push(' args.extend(input_path)');
|
||||
lines.push(' else:');
|
||||
lines.push(' args.append(input_path)');
|
||||
lines.push('');
|
||||
|
||||
// Generate args building for each option
|
||||
for (const opt of options.options) {
|
||||
const snakeName = toSnakeCase(opt.name);
|
||||
const cliFlag = `--${opt.name}`;
|
||||
|
||||
if (opt.type === 'boolean') {
|
||||
lines.push(` if ${snakeName}:`);
|
||||
lines.push(` args.append("${cliFlag}")`);
|
||||
} else if (isListOption(opt)) {
|
||||
lines.push(` if ${snakeName}:`);
|
||||
lines.push(` if isinstance(${snakeName}, list):`);
|
||||
lines.push(` if ${snakeName}:`);
|
||||
lines.push(` args.extend(["${cliFlag}", ",".join(${snakeName})])`);
|
||||
lines.push(` else:`);
|
||||
lines.push(` args.extend(["${cliFlag}", ${snakeName}])`);
|
||||
} else {
|
||||
lines.push(` if ${snakeName}:`);
|
||||
lines.push(` args.extend(["${cliFlag}", ${snakeName}])`);
|
||||
}
|
||||
}
|
||||
|
||||
lines.push('');
|
||||
lines.push(' run_jar(args, quiet)');
|
||||
lines.push('');
|
||||
|
||||
const outputPath = join(ROOT_DIR, 'python/opendataloader-pdf/src/opendataloader_pdf/convert_generated.py');
|
||||
mkdirSync(dirname(outputPath), { recursive: true });
|
||||
writeFileSync(outputPath, lines.join('\n'));
|
||||
console.log(`Generated: ${outputPath}`);
|
||||
}
|
||||
|
||||
/**
|
||||
* Generate Python convert() options table (MDX snippet)
|
||||
*/
|
||||
function generatePythonConvertOptionsMdx() {
|
||||
const lines = [];
|
||||
lines.push('---');
|
||||
lines.push('title: Python Convert Options');
|
||||
lines.push('description: Options for the Python convert function');
|
||||
lines.push('---');
|
||||
lines.push('');
|
||||
lines.push(AUTO_GENERATED_HEADER_MDX);
|
||||
|
||||
// Build rows array
|
||||
const rows = [];
|
||||
|
||||
// Add input_path first (not in options.json)
|
||||
rows.push(['`input_path`', String.raw`\`str \| list[str]\``, 'required', 'One or more input PDF file paths or directories']);
|
||||
|
||||
for (const opt of options.options) {
|
||||
const snakeName = toSnakeCase(opt.name);
|
||||
let pyType = 'str';
|
||||
if (opt.type === 'boolean') {
|
||||
pyType = 'bool';
|
||||
} else if (isListOption(opt)) {
|
||||
pyType = String.raw`str \| list[str]`;
|
||||
}
|
||||
|
||||
const defaultVal = opt.default === null ? '-'
|
||||
: typeof opt.default === 'boolean' ? (opt.default ? '`True`' : '`False`')
|
||||
: `\`"${opt.default}"\``;
|
||||
|
||||
const description = escapeMarkdown(opt.description);
|
||||
rows.push([`\`${snakeName}\``, `\`${pyType}\``, defaultVal, description]);
|
||||
}
|
||||
|
||||
lines.push(...formatTable(['Parameter', 'Type', 'Default', 'Description'], rows));
|
||||
lines.push('');
|
||||
|
||||
const outputPath = join(ROOT_DIR, 'content/docs/reference/python-convert-options.mdx');
|
||||
mkdirSync(dirname(outputPath), { recursive: true });
|
||||
writeFileSync(outputPath, lines.join('\n'));
|
||||
console.log(`Generated: ${outputPath}`);
|
||||
}
|
||||
|
||||
/**
|
||||
* Generate Node.js convert() options table (MDX snippet)
|
||||
*/
|
||||
function generateNodeConvertOptionsMdx() {
|
||||
const lines = [];
|
||||
lines.push('---');
|
||||
lines.push('title: Node.js Convert Options');
|
||||
lines.push('description: Options for the Node.js convert function');
|
||||
lines.push('---');
|
||||
lines.push('');
|
||||
lines.push(AUTO_GENERATED_HEADER_MDX);
|
||||
|
||||
// Build rows array
|
||||
const rows = [];
|
||||
|
||||
for (const opt of options.options) {
|
||||
const camelName = toCamelCase(opt.name);
|
||||
let tsType = 'string';
|
||||
if (opt.type === 'boolean') {
|
||||
tsType = 'boolean';
|
||||
} else if (isListOption(opt)) {
|
||||
tsType = String.raw`string \| string[]`;
|
||||
}
|
||||
|
||||
const defaultVal = opt.default === null ? '-'
|
||||
: typeof opt.default === 'boolean' ? `\`${opt.default}\``
|
||||
: `\`"${opt.default}"\``;
|
||||
|
||||
const description = escapeMarkdown(opt.description);
|
||||
rows.push([`\`${camelName}\``, `\`${tsType}\``, defaultVal, description]);
|
||||
}
|
||||
|
||||
lines.push(...formatTable(['Option', 'Type', 'Default', 'Description'], rows));
|
||||
lines.push('');
|
||||
|
||||
const outputPath = join(ROOT_DIR, 'content/docs/reference/node-convert-options.mdx');
|
||||
mkdirSync(dirname(outputPath), { recursive: true });
|
||||
writeFileSync(outputPath, lines.join('\n'));
|
||||
console.log(`Generated: ${outputPath}`);
|
||||
}
|
||||
|
||||
/**
|
||||
* Generate options reference documentation (MDX)
|
||||
*/
|
||||
function generateOptionsReferenceMdx() {
|
||||
// Build rows array
|
||||
const rows = [];
|
||||
|
||||
for (const opt of options.options) {
|
||||
const longOpt = `\`--${opt.name}\``;
|
||||
const shortOpt = opt.shortName ? `\`-${opt.shortName}\`` : '-';
|
||||
const type = `\`${opt.type}\``;
|
||||
const defaultVal = opt.default === null ? '-'
|
||||
: typeof opt.default === 'boolean' ? `\`${opt.default}\``
|
||||
: `\`"${opt.default}"\``;
|
||||
const description = escapeMarkdown(opt.description);
|
||||
|
||||
rows.push([longOpt, shortOpt, type, defaultVal, description]);
|
||||
}
|
||||
|
||||
const lines = [
|
||||
'---',
|
||||
'title: CLI Options Reference',
|
||||
'description: Complete reference for all CLI options',
|
||||
'---',
|
||||
'',
|
||||
AUTO_GENERATED_HEADER_MDX.trimEnd(),
|
||||
'# CLI Options Reference',
|
||||
'',
|
||||
'This page documents all available CLI options for opendataloader-pdf.',
|
||||
'',
|
||||
'## Options',
|
||||
'',
|
||||
...formatTable(['Option', 'Short', 'Type', 'Default', 'Description'], rows),
|
||||
'',
|
||||
'## Examples',
|
||||
'',
|
||||
'### Basic conversion',
|
||||
'',
|
||||
'```bash',
|
||||
'opendataloader-pdf document.pdf -o ./output -f json,markdown',
|
||||
'```',
|
||||
'',
|
||||
'### Convert entire folder',
|
||||
'',
|
||||
'```bash',
|
||||
'opendataloader-pdf ./pdf-folder -o ./output -f json',
|
||||
'```',
|
||||
'',
|
||||
'### Save images as external files',
|
||||
'',
|
||||
'```bash',
|
||||
'opendataloader-pdf document.pdf -f markdown --image-output external',
|
||||
'```',
|
||||
'',
|
||||
'### Disable reading order sorting',
|
||||
'',
|
||||
'```bash',
|
||||
'opendataloader-pdf document.pdf -f json --reading-order off',
|
||||
'```',
|
||||
'',
|
||||
'### Add page separators in output',
|
||||
'',
|
||||
'```bash',
|
||||
'opendataloader-pdf document.pdf -f markdown --markdown-page-separator "--- Page %page-number% ---"',
|
||||
'```',
|
||||
'',
|
||||
'### Encrypted PDF',
|
||||
'',
|
||||
'```bash',
|
||||
'opendataloader-pdf encrypted.pdf -p mypassword -o ./output',
|
||||
'```',
|
||||
'',
|
||||
];
|
||||
|
||||
const outputPath = join(ROOT_DIR, 'content/docs/reference/cli-options.mdx');
|
||||
mkdirSync(dirname(outputPath), { recursive: true });
|
||||
writeFileSync(outputPath, lines.join('\n'));
|
||||
console.log(`Generated: ${outputPath}`);
|
||||
}
|
||||
|
||||
// Run all generators
|
||||
console.log('Generating files from options.json...\n');
|
||||
|
||||
generateNodeCliOptions();
|
||||
generateNodeConvertOptions();
|
||||
generatePythonCliOptions();
|
||||
generatePythonConvert();
|
||||
generateOptionsReferenceMdx();
|
||||
generatePythonConvertOptionsMdx();
|
||||
generateNodeConvertOptionsMdx();
|
||||
|
||||
console.log('\nDone!');
|
||||
@@ -0,0 +1,263 @@
|
||||
#!/usr/bin/env node
|
||||
/**
|
||||
* Generates JSON Schema documentation from the single source of truth (schema.json).
|
||||
*
|
||||
* Usage: node scripts/generate-schema.mjs
|
||||
*/
|
||||
|
||||
import { readFileSync, writeFileSync, mkdirSync } from 'node:fs';
|
||||
import { dirname, join } from 'node:path';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
import { escapeMarkdown, formatTable } from './utils.mjs';
|
||||
|
||||
const __dirname = dirname(fileURLToPath(import.meta.url));
|
||||
const ROOT_DIR = join(__dirname, '..');
|
||||
|
||||
// Read schema.json
|
||||
const schemaPath = join(ROOT_DIR, 'schema.json');
|
||||
const schema = JSON.parse(readFileSync(schemaPath, 'utf-8'));
|
||||
|
||||
const AUTO_GENERATED_HEADER_MDX = `{/* AUTO-GENERATED FROM schema.json - DO NOT EDIT DIRECTLY */}
|
||||
{/* Run \`npm run generate-schema\` to regenerate */}
|
||||
|
||||
`;
|
||||
|
||||
/**
|
||||
* Get JSON Schema type as a readable string.
|
||||
*/
|
||||
function formatType(prop) {
|
||||
if (!prop) return 'any';
|
||||
|
||||
if (prop.$ref) {
|
||||
const refName = prop.$ref.split('/').pop();
|
||||
return `\`${refName}\``;
|
||||
}
|
||||
|
||||
if (prop.oneOf) {
|
||||
return prop.oneOf.map(formatType).join(' \\| ');
|
||||
}
|
||||
|
||||
if (prop.const) {
|
||||
return `\`"${prop.const}"\``;
|
||||
}
|
||||
|
||||
if (prop.enum) {
|
||||
return prop.enum.map(v => `\`${v}\``).join(', ');
|
||||
}
|
||||
|
||||
if (Array.isArray(prop.type)) {
|
||||
return prop.type.map(t => `\`${t}\``).join(' \\| ');
|
||||
}
|
||||
|
||||
if (prop.type === 'array') {
|
||||
if (prop.items) {
|
||||
return `\`array\``;
|
||||
}
|
||||
return `\`array\``;
|
||||
}
|
||||
|
||||
return `\`${prop.type || 'any'}\``;
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if a property is required.
|
||||
*/
|
||||
function isRequired(propName, requiredList) {
|
||||
return requiredList && requiredList.includes(propName);
|
||||
}
|
||||
|
||||
/**
|
||||
* Generate JSON Schema documentation (MDX).
|
||||
*/
|
||||
function generateJsonSchemaMdx() {
|
||||
const lines = [];
|
||||
lines.push('---');
|
||||
lines.push('title: JSON Schema');
|
||||
lines.push('description: Understand the layout structure emitted by OpenDataLoader PDF');
|
||||
lines.push('---');
|
||||
lines.push('');
|
||||
lines.push(AUTO_GENERATED_HEADER_MDX);
|
||||
|
||||
lines.push('Every conversion that includes the `json` format produces a hierarchical document describing detected elements (pages, tables, lists, captions, etc.). Use the following reference to map fields into your downstream processors.');
|
||||
lines.push('');
|
||||
|
||||
// Helper to build rows from schema properties
|
||||
const buildRows = (properties, requiredList) =>
|
||||
Object.entries(properties).map(([name, prop]) => [
|
||||
`\`${name}\``,
|
||||
formatType(prop),
|
||||
isRequired(name, requiredList) ? 'Yes' : 'No',
|
||||
escapeMarkdown(prop.description || '')
|
||||
]);
|
||||
|
||||
// Root node
|
||||
const rootRows = buildRows(schema.properties, schema.required);
|
||||
lines.push(
|
||||
'## Root node',
|
||||
'',
|
||||
...formatTable(['Field', 'Type', 'Required', 'Description'], rootRows),
|
||||
''
|
||||
);
|
||||
|
||||
// Common content fields (baseElement)
|
||||
const baseElement = schema.$defs.baseElement;
|
||||
const baseRows = buildRows(baseElement.properties, baseElement.required);
|
||||
lines.push(
|
||||
'## Common content fields',
|
||||
'',
|
||||
'All content elements share these base properties:',
|
||||
'',
|
||||
...formatTable(['Field', 'Type', 'Required', 'Description'], baseRows),
|
||||
''
|
||||
);
|
||||
|
||||
// Text properties
|
||||
const textProps = schema.$defs.textProperties;
|
||||
const textRows = buildRows(textProps.properties, textProps.required);
|
||||
lines.push(
|
||||
'## Text properties',
|
||||
'',
|
||||
'Text nodes (`paragraph`, `heading`, `caption`, `list item`) include these additional fields:',
|
||||
'',
|
||||
...formatTable(['Field', 'Type', 'Required', 'Description'], textRows),
|
||||
''
|
||||
);
|
||||
|
||||
// Headings
|
||||
lines.push(
|
||||
'## Headings',
|
||||
'',
|
||||
...formatTable(['Field', 'Type', 'Required', 'Description'], [
|
||||
['`heading level`', '`integer`', 'Yes', 'Heading level (e.g., 1 for h1)']
|
||||
]),
|
||||
''
|
||||
);
|
||||
|
||||
// Captions
|
||||
lines.push(
|
||||
'## Captions',
|
||||
'',
|
||||
...formatTable(['Field', 'Type', 'Required', 'Description'], [
|
||||
['`linked content id`', '`integer`', 'No', 'ID of the linked content element (table, image, etc.)']
|
||||
]),
|
||||
''
|
||||
);
|
||||
|
||||
// Tables
|
||||
lines.push(
|
||||
'## Tables',
|
||||
'',
|
||||
...formatTable(['Field', 'Type', 'Required', 'Description'], [
|
||||
['`number of rows`', '`integer`', 'Yes', 'Row count'],
|
||||
['`number of columns`', '`integer`', 'Yes', 'Column count'],
|
||||
['`previous table id`', '`integer`', 'No', 'Linked table identifier (if broken across pages)'],
|
||||
['`next table id`', '`integer`', 'No', 'Linked table identifier'],
|
||||
['`rows`', '`array`', 'Yes', 'Row objects']
|
||||
]),
|
||||
''
|
||||
);
|
||||
|
||||
// Table rows
|
||||
lines.push(
|
||||
'### Table rows',
|
||||
'',
|
||||
...formatTable(['Field', 'Type', 'Required', 'Description'], [
|
||||
['`type`', '`"table row"`', 'Yes', 'Element type'],
|
||||
['`row number`', '`integer`', 'Yes', 'Row index (1-indexed)'],
|
||||
['`cells`', '`array`', 'Yes', 'Cell objects']
|
||||
]),
|
||||
''
|
||||
);
|
||||
|
||||
// Table cells
|
||||
lines.push(
|
||||
'### Table cells',
|
||||
'',
|
||||
...formatTable(['Field', 'Type', 'Required', 'Description'], [
|
||||
['`row number`', '`integer`', 'Yes', 'Row index of the cell (1-indexed)'],
|
||||
['`column number`', '`integer`', 'Yes', 'Column index of the cell (1-indexed)'],
|
||||
['`row span`', '`integer`', 'Yes', 'Number of rows spanned'],
|
||||
['`column span`', '`integer`', 'Yes', 'Number of columns spanned'],
|
||||
['`kids`', '`array`', 'Yes', 'Nested content elements']
|
||||
]),
|
||||
''
|
||||
);
|
||||
|
||||
// Lists
|
||||
lines.push(
|
||||
'## Lists',
|
||||
'',
|
||||
...formatTable(['Field', 'Type', 'Required', 'Description'], [
|
||||
['`numbering style`', '`string`', 'Yes', 'Marker style (ordered, bullet, etc.)'],
|
||||
['`number of list items`', '`integer`', 'Yes', 'Item count'],
|
||||
['`previous list id`', '`integer`', 'No', 'Linked list identifier'],
|
||||
['`next list id`', '`integer`', 'No', 'Linked list identifier'],
|
||||
['`list items`', '`array`', 'Yes', 'Item nodes']
|
||||
]),
|
||||
''
|
||||
);
|
||||
|
||||
// List items
|
||||
lines.push(
|
||||
'### List items',
|
||||
'',
|
||||
'List items include text properties plus:',
|
||||
'',
|
||||
...formatTable(['Field', 'Type', 'Required', 'Description'], [
|
||||
['`kids`', '`array`', 'Yes', 'Nested content elements']
|
||||
]),
|
||||
''
|
||||
);
|
||||
|
||||
// Images
|
||||
lines.push(
|
||||
'## Images',
|
||||
'',
|
||||
...formatTable(['Field', 'Type', 'Required', 'Description'], [
|
||||
['`source`', '`string`', 'No', 'Relative path to the image file'],
|
||||
['`data`', '`string`', 'No', 'Base64 data URI (when image-output is "embedded")'],
|
||||
['`format`', '`string`', 'No', 'Image format (`png`, `jpeg`)']
|
||||
]),
|
||||
''
|
||||
);
|
||||
|
||||
// Headers and footers
|
||||
lines.push(
|
||||
'## Headers and footers',
|
||||
'',
|
||||
...formatTable(['Field', 'Type', 'Required', 'Description'], [
|
||||
['`type`', '`string`', 'Yes', 'Either `header` or `footer`'],
|
||||
['`kids`', '`array`', 'Yes', 'Content elements within the header or footer']
|
||||
]),
|
||||
''
|
||||
);
|
||||
|
||||
// Text blocks
|
||||
lines.push(
|
||||
'## Text blocks',
|
||||
'',
|
||||
...formatTable(['Field', 'Type', 'Required', 'Description'], [
|
||||
['`kids`', '`array`', 'Yes', 'Text block children']
|
||||
]),
|
||||
''
|
||||
);
|
||||
|
||||
lines.push(
|
||||
'## JSON Schema',
|
||||
'',
|
||||
'The complete JSON Schema is available at [`schema.json`](https://github.com/opendataloader-project/opendataloader-pdf/blob/main/schema.json) in the repository root.'
|
||||
);
|
||||
lines.push('');
|
||||
|
||||
const outputPath = join(ROOT_DIR, 'content/docs/reference/json-schema.mdx');
|
||||
mkdirSync(dirname(outputPath), { recursive: true });
|
||||
writeFileSync(outputPath, lines.join('\n'));
|
||||
console.log(`Generated: ${outputPath}`);
|
||||
}
|
||||
|
||||
// Run all generators
|
||||
console.log('Generating files from schema.json...\n');
|
||||
|
||||
generateJsonSchemaMdx();
|
||||
|
||||
console.log('\nDone!');
|
||||
Executable
+59
@@ -0,0 +1,59 @@
|
||||
#!/bin/bash
|
||||
|
||||
# Run the Java CLI directly
|
||||
# Usage: ./scripts/run-cli.sh [options] [input...]
|
||||
#
|
||||
# If no arguments: uses DEFAULT_ARGS
|
||||
# If arguments given: uses only the provided arguments
|
||||
#
|
||||
# Examples:
|
||||
# ./scripts/run-cli.sh # Use defaults
|
||||
# ./scripts/run-cli.sh -f markdown my.pdf # Custom args only
|
||||
|
||||
set -e
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
|
||||
ROOT_DIR="$SCRIPT_DIR/.."
|
||||
JAR_DIR="$ROOT_DIR/java/opendataloader-pdf-cli/target"
|
||||
|
||||
# Defaults (used only when no arguments provided)
|
||||
DEFAULT_ARGS=("-f" "json,markdown,html,pdf" "-o" "$ROOT_DIR/samples/temp" "$ROOT_DIR/samples/pdf")
|
||||
|
||||
# Check if Java is installed
|
||||
command -v java >/dev/null || { echo "Error: java not found"; exit 1; }
|
||||
|
||||
# Find the shaded JAR (excludes original-* and *-sources.jar)
|
||||
find_jar() {
|
||||
find "$JAR_DIR" -maxdepth 1 -name "opendataloader-pdf-cli-*.jar" \
|
||||
! -name "original-*" \
|
||||
! -name "*-sources.jar" \
|
||||
! -name "*-javadoc.jar" \
|
||||
2>/dev/null | head -1
|
||||
}
|
||||
|
||||
JAR_PATH=$(find_jar)
|
||||
|
||||
# Build JAR if it doesn't exist
|
||||
if [ -z "$JAR_PATH" ] || [ ! -f "$JAR_PATH" ]; then
|
||||
echo "JAR not found. Building..."
|
||||
cd "$ROOT_DIR/java"
|
||||
mvn -B package -DskipTests -q
|
||||
cd "$ROOT_DIR"
|
||||
JAR_PATH=$(find_jar)
|
||||
fi
|
||||
|
||||
if [ -z "$JAR_PATH" ]; then
|
||||
echo "Error: Could not find CLI JAR file"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Use defaults if no arguments, otherwise use provided arguments only
|
||||
if [ $# -eq 0 ]; then
|
||||
ARGS=("${DEFAULT_ARGS[@]}")
|
||||
else
|
||||
ARGS=("$@")
|
||||
fi
|
||||
|
||||
# Run the CLI. Headless flags suppress the macOS Dock icon / focus theft when
|
||||
# AWT/ImageIO/PDFBox initialize. Safe on all OSes — the CLI never opens a UI.
|
||||
java -Djava.awt.headless=true -Dapple.awt.UIElement=true -jar "$JAR_PATH" "${ARGS[@]}"
|
||||
Executable
+18
@@ -0,0 +1,18 @@
|
||||
#!/bin/bash
|
||||
|
||||
# Local development test script for Java package
|
||||
# For CI/CD builds, use build-java.sh instead
|
||||
|
||||
set -e
|
||||
|
||||
# Prerequisites
|
||||
command -v java >/dev/null || { echo "Error: java not found"; exit 1; }
|
||||
command -v mvn >/dev/null || { echo "Error: mvn not found"; exit 1; }
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
|
||||
ROOT_DIR="$SCRIPT_DIR/.."
|
||||
PACKAGE_DIR="$ROOT_DIR/java"
|
||||
cd "$PACKAGE_DIR"
|
||||
|
||||
# Run tests
|
||||
mvn test "$@"
|
||||
Executable
+21
@@ -0,0 +1,21 @@
|
||||
#!/bin/bash
|
||||
|
||||
# Local development test script for Node.js package
|
||||
# For CI/CD builds, use build-node.sh instead
|
||||
|
||||
set -e
|
||||
|
||||
# Prerequisites
|
||||
command -v node >/dev/null || { echo "Error: node not found"; exit 1; }
|
||||
command -v pnpm >/dev/null || { echo "Error: pnpm not found"; exit 1; }
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
|
||||
ROOT_DIR="$SCRIPT_DIR/.."
|
||||
PACKAGE_DIR="$ROOT_DIR/node/opendataloader-pdf"
|
||||
cd "$PACKAGE_DIR"
|
||||
|
||||
# Install dependencies (if needed)
|
||||
pnpm install
|
||||
|
||||
# Run tests
|
||||
pnpm test "$@"
|
||||
Executable
+18
@@ -0,0 +1,18 @@
|
||||
#!/bin/bash
|
||||
|
||||
# Local development test script for Python package using uv
|
||||
# For CI/CD builds, use build-python.sh instead
|
||||
|
||||
set -e
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
|
||||
ROOT_DIR="$SCRIPT_DIR/.."
|
||||
PACKAGE_DIR="$ROOT_DIR/python/opendataloader-pdf"
|
||||
cd "$PACKAGE_DIR"
|
||||
|
||||
# Check uv is available
|
||||
command -v uv >/dev/null || { echo "Error: uv not found. Install with: curl -LsSf https://astral.sh/uv/install.sh | sh"; exit 1; }
|
||||
|
||||
# Sync dependencies and run tests
|
||||
uv sync
|
||||
uv run pytest tests -v -s "$@"
|
||||
@@ -0,0 +1,48 @@
|
||||
/**
|
||||
* Shared utilities for code generation scripts.
|
||||
*/
|
||||
|
||||
/**
|
||||
* Escape string for use in Markdown table cells.
|
||||
* @param {string} str - The string to escape
|
||||
* @returns {string} - Escaped string safe for Markdown tables
|
||||
*/
|
||||
export function escapeMarkdown(str) {
|
||||
if (!str) return '';
|
||||
return str
|
||||
.replace(/\\/g, String.raw`\\`) // escape backslashes first
|
||||
.replace(/\|/g, String.raw`\|`) // escape pipe characters
|
||||
.replace(/`/g, String.raw`\``) // escape backticks
|
||||
.replace(/\*/g, String.raw`\*`) // escape asterisks
|
||||
.replace(/_/g, String.raw`\_`) // escape underscores
|
||||
.replace(/</g, '<') // escape HTML angle brackets
|
||||
.replace(/>/g, '>');
|
||||
}
|
||||
|
||||
/**
|
||||
* Format a markdown table with aligned columns.
|
||||
* @param {string[]} headers - Table headers
|
||||
* @param {string[][]} rows - Table rows (each row is an array of cell values)
|
||||
* @returns {string[]} - Formatted table lines
|
||||
*/
|
||||
export function formatTable(headers, rows) {
|
||||
// Calculate max width for each column
|
||||
const colWidths = headers.map((h, i) => {
|
||||
const headerLen = h.length;
|
||||
const maxRowLen = rows.reduce((max, row) => Math.max(max, (row[i] || '').length), 0);
|
||||
return Math.max(headerLen, maxRowLen);
|
||||
});
|
||||
|
||||
// Build header row
|
||||
const headerRow = '| ' + headers.map((h, i) => h.padEnd(colWidths[i])).join(' | ') + ' |';
|
||||
|
||||
// Build separator row
|
||||
const separatorRow = '|' + colWidths.map(w => '-'.repeat(w + 2)).join('|') + '|';
|
||||
|
||||
// Build data rows
|
||||
const dataRows = rows.map(row =>
|
||||
'| ' + row.map((cell, i) => (cell || '').padEnd(colWidths[i])).join(' | ') + ' |'
|
||||
);
|
||||
|
||||
return [headerRow, separatorRow, ...dataRows];
|
||||
}
|
||||
Executable
+67
@@ -0,0 +1,67 @@
|
||||
#!/bin/bash
|
||||
|
||||
# Verify the Python sdist contains all required files (JAR, LICENSE, etc).
|
||||
# The files listed below are gitignored in the package dir and only exist in
|
||||
# the dist because [tool.hatch.build] artifacts force-includes them. This
|
||||
# script guards against silent regressions if that config ever drifts.
|
||||
#
|
||||
# Usage: ./scripts/verify-python-sdist.sh
|
||||
# Prerequisite: run 'uv build' (or scripts/build-python.sh) first.
|
||||
|
||||
set -e
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
|
||||
ROOT_DIR="$SCRIPT_DIR/.."
|
||||
DIST_DIR="$ROOT_DIR/python/opendataloader-pdf/dist"
|
||||
|
||||
shopt -s nullglob
|
||||
SDIST_CANDIDATES=("$DIST_DIR"/*.tar.gz)
|
||||
shopt -u nullglob
|
||||
|
||||
if [ ${#SDIST_CANDIDATES[@]} -eq 0 ]; then
|
||||
echo "Error: no sdist found in $DIST_DIR. Run 'uv build' first." >&2
|
||||
exit 1
|
||||
fi
|
||||
if [ ${#SDIST_CANDIDATES[@]} -gt 1 ]; then
|
||||
echo "Error: multiple sdists found in $DIST_DIR. Remove stale ones first:" >&2
|
||||
printf ' - %s\n' "${SDIST_CANDIDATES[@]}" >&2
|
||||
exit 1
|
||||
fi
|
||||
SDIST="${SDIST_CANDIDATES[0]}"
|
||||
|
||||
echo "Verifying sdist: $(basename "$SDIST")"
|
||||
|
||||
REQUIRED=(
|
||||
"jar/opendataloader-pdf-cli.jar"
|
||||
"LICENSE"
|
||||
"NOTICE"
|
||||
"THIRD_PARTY/"
|
||||
)
|
||||
|
||||
CONTENTS=$(tar -tzf "$SDIST")
|
||||
MISSING=()
|
||||
for path in "${REQUIRED[@]}"; do
|
||||
# Directory prefixes (trailing '/') match any entry under that prefix.
|
||||
# File paths are anchored to end-of-line so "LICENSE" does not match "LICENSE.bak".
|
||||
# (^|/) prefix keeps the check layout-tolerant if hatchling ever emits
|
||||
# a differently-rooted sdist (e.g., without the top-level pkgname-version/ dir).
|
||||
if [[ "$path" == */ ]]; then
|
||||
pattern="(^|/)src/opendataloader_pdf/${path}"
|
||||
else
|
||||
pattern="(^|/)src/opendataloader_pdf/${path}\$"
|
||||
fi
|
||||
if ! echo "$CONTENTS" | grep -qE "$pattern"; then
|
||||
MISSING+=("$path")
|
||||
fi
|
||||
done
|
||||
|
||||
if [ ${#MISSING[@]} -gt 0 ]; then
|
||||
echo "Error: sdist is missing required files:" >&2
|
||||
printf ' - src/opendataloader_pdf/%s\n' "${MISSING[@]}" >&2
|
||||
echo "" >&2
|
||||
echo "Fix: ensure [tool.hatch.build] in pyproject.toml lists these under 'artifacts'." >&2
|
||||
echo "(They are gitignored, so hatch drops them unless force-included.)" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "OK: sdist contains all required files."
|
||||
Reference in New Issue
Block a user