wehub/opendataloader-project--opendataloader-pdf
1
0
Fork 0
Code Issues Pull Requests Actions Deployments Models Agent Notes Packages Projects Releases Wiki Activity

chore: import upstream snapshot with attribution

Browse Source
This commit is contained in:
wehub-resource-sync
2026-07-13 12:59:42 +08:00
commit 59f8f60dad
348 changed files with 139133 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.editorconfig
+12
View File
@@ -0,0 +1,12 @@
root = true
[*]
charset = utf-8
indent_style = space
indent_size = 4
end_of_line = lf
insert_final_newline = true
trim_trailing_whitespace = true
[*.md]
trim_trailing_whitespace = false
.gitattributes
+16
View File
@@ -0,0 +1,16 @@
# Unify all text files to LF line endings
* text eol=lf
# Binary files should not have line ending conversions
*.exe binary
*.dll binary
*.so binary
*.dylib binary
*.class binary
*.jar binary
*.zip binary
*.png binary
*.jpg binary
*.jpeg binary
*.gif binary
*.pdf binary
.github/CODEOWNERS
+5
View File
@@ -0,0 +1,5 @@
# Owner for all documents
*.md @bdoubrov @hnc-sujicho
# Default owners for everything else in the repository
* @MaximPlusov @LonelyMidoriya @hyunhee-jo @bundolee @hnc-jglee
.github/ISSUE_TEMPLATE/bug_report.md
+33
View File
@@ -0,0 +1,33 @@
---
name: Bug report
about: Report an issue
title: ""
labels: bug
assignees: ""
---
### Bug
<!-- Describe the buggy behavior you have observed. -->
...
### Steps to reproduce
<!-- Describe the sequence of steps for reproducing the bug. -->
...
### Version
<!-- Copy version. -->
...
### Java version
<!-- Copy the output of `java --version`. -->
...
<!-- ⚠️ ATTENTION: When sharing screenshots, attachments, or other data make sure not to include any sensitive information. -->
.github/ISSUE_TEMPLATE/config.yml
+1
View File
@@ -0,0 +1 @@
blank_issues_enabled: false
.github/ISSUE_TEMPLATE/feature_request.md
+21
View File
@@ -0,0 +1,21 @@
---
name: Feature request
about: Suggest an idea
title: ""
labels: enhancement
assignees: ""
---
### Requested feature
<!-- Describe the feature you have in mind and the user need it addresses. -->
...
### Alternatives
<!-- Describe any alternatives you have considered. -->
...
<!-- ⚠️ ATTENTION: When sharing screenshots, attachments, or other data make sure not to include any sensitive information. -->
.github/ISSUE_TEMPLATE/question.md
+15
View File
@@ -0,0 +1,15 @@
---
name: Question
about: Ask a question
title: ""
labels: question
assignees: ""
---
### Question
<!-- Describe what you would like to achieve and which part you need help with. -->
...
<!-- ⚠️ ATTENTION: When sharing screenshots, attachments, or other data make sure not to include any sensitive information. -->
.github/PULL_REQUEST_TEMPLATE.md
+19
View File
@@ -0,0 +1,19 @@
<!-- Thank you for your contribution! -->
<!-- STEPS TO FOLLOW:
1. Add a description of the changes (frequently the same as the commit description)
2. Enter the issue number next to "Resolves #" below (if there is no tracking issue resolved, **remove that section**)
3. Make sure the PR title follows the **Commit Message Formatting**: https://www.conventionalcommits.org/en/v1.0.0/#summary.
4. Follow the steps in the checklist below, starting with the **Commit Message Formatting**.
-->
<!-- Uncomment this section with the issue number if an issue is being resolved
**Issue resolved by this Pull Request:**
Resolves #
--->
**Checklist:**
- [ ] Documentation has been updated, if necessary.
- [ ] Examples have been added, if necessary.
- [ ] Tests have been added, if necessary.
.github/SECURITY.md
+24
View File
@@ -0,0 +1,24 @@
# Security Policy
## Reporting a Vulnerability
If you think you've identified a security issue in the project repository, please DO NOT report the issue publicly via
the GitHub issue tracker, etc.
Instead, send an email with as many details as possible. This is a private mailing list for the maintainers team.
Please do not create a public issue.
### Security Vulnerability Response
Each report is acknowledged and analyzed by the core maintainers within 3 working days.
Any vulnerability information shared with core maintainers stays within the project and will not be disseminated to
other projects unless it is necessary to get the issue fixed.
After the initial reply to your report, the security team will keep you informed of the progress towards a fix and full
announcement, and may ask for additional information or guidance.
## Security Alerts
We will send announcements of security vulnerabilities and steps to remediate on the project announcements.
.github/dependabot.yml
+71
View File
@@ -0,0 +1,71 @@
version: 2
updates:
# Java — multi-module Maven (parent /java declares core/cli child modules; Dependabot auto-detects them)
- package-ecosystem: "maven"
directory: "/java"
schedule:
interval: "weekly"
day: "monday"
time: "09:00"
timezone: "Asia/Seoul"
open-pull-requests-limit: 10
labels:
- "dependencies"
groups:
maven-minor-patch:
update-types:
- "minor"
- "patch"
# Node.js SDK (pnpm; Dependabot handles pnpm under the "npm" ecosystem).
# Root package.json is a scripts-only workspace meta-file with no deps, so it is omitted.
- package-ecosystem: "npm"
directory: "/node/opendataloader-pdf"
schedule:
interval: "weekly"
day: "monday"
time: "09:00"
timezone: "Asia/Seoul"
open-pull-requests-limit: 10
labels:
- "dependencies"
groups:
npm-minor-patch:
update-types:
- "minor"
- "patch"
# Python SDK + MCP server (example requirements.txt excluded to avoid noise)
- package-ecosystem: "pip"
directories:
- "/python/opendataloader-pdf"
- "/python/opendataloader-pdf-mcp"
schedule:
interval: "weekly"
day: "monday"
time: "09:00"
timezone: "Asia/Seoul"
open-pull-requests-limit: 10
labels:
- "dependencies"
groups:
pip-minor-patch:
update-types:
- "minor"
- "patch"
# GitHub Actions workflows
- package-ecosystem: "github-actions"
directory: "/"
schedule:
interval: "weekly"
day: "monday"
time: "09:00"
timezone: "Asia/Seoul"
labels:
- "dependencies"
groups:
github-actions:
update-types:
- "minor"
- "patch"
.github/workflows/release.yml
+167
View File
@@ -0,0 +1,167 @@
name: Release
on:
push:
tags:
- 'v*'
workflow_dispatch:
jobs:
release:
runs-on: ubuntu-latest
env:
VERSION: '0.0.0'
permissions:
contents: write
id-token: write
steps:
# =================================================================
# 1. SETUP
# =================================================================
- name: Checkout code
uses: actions/checkout@v7
- name: Initialize VERSION
run: |
if [[ "${GITHUB_REF}" == refs/tags/v* ]]; then
echo "VERSION=${GITHUB_REF_NAME#v}" >> $GITHUB_ENV
else
echo "VERSION=0.0.0" >> $GITHUB_ENV
fi
- name: Set up Java
uses: actions/setup-java@v5
with:
java-version: '21'
distribution: 'temurin'
cache: 'maven'
server-id: central
server-username: MAVEN_CENTRAL_USERNAME
server-password: MAVEN_CENTRAL_PASSWORD
gpg-private-key: ${{ secrets.MAVEN_GPG_KEY }}
gpg-passphrase: ${{ secrets.MAVEN_GPG_PASSPHRASE }}
- name: Set up Python
uses: actions/setup-python@v6
with:
python-version: '3.12'
- name: Install uv
uses: astral-sh/setup-uv@v7
- name: Set up Node.js and pnpm
uses: actions/setup-node@v6
with:
node-version: '20'
registry-url: 'https://registry.npmjs.org'
- name: Install pnpm
uses: pnpm/action-setup@v6
with:
version: 9
# =================================================================
# 2. BUILD & TEST
# =================================================================
- name: Build and test all packages
run: ./scripts/build-all.sh ${{ env.VERSION }}
# =================================================================
# 3. VERIFY CLI OPTIONS (before deploy)
# =================================================================
- name: Install opendataloader-pdf CLI
# --system installs into the setup-python interpreter; without it (or an
# active venv) `uv pip install` aborts with "No virtual environment found".
run: uv pip install --system ./python/opendataloader-pdf/dist/*.whl
- name: Run CLI verification
# ci-verify.py writes its own markdown table to $GITHUB_STEP_SUMMARY,
# so no separate summary step is needed here.
run: python verification/ci-verify.py
- name: Upload verification report
if: always()
uses: actions/upload-artifact@v7
with:
name: verification-report-release
path: verification/verification-report-ci.txt
retention-days: 30
# =================================================================
# 4. DEPLOY (only on tag push)
# =================================================================
- name: '[Java] Deploy to Maven Central'
if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v')
run: mvn -B -pl opendataloader-pdf-core deploy -P release
working-directory: ./java
env:
MAVEN_CENTRAL_USERNAME: ${{ secrets.MAVEN_CENTRAL_USERNAME }}
MAVEN_CENTRAL_PASSWORD: ${{ secrets.MAVEN_CENTRAL_PASSWORD }}
MAVEN_GPG_KEY: ${{ secrets.MAVEN_GPG_KEY }}
MAVEN_GPG_PASSPHRASE: ${{ secrets.MAVEN_GPG_PASSPHRASE }}
- name: '[Python] Publish to PyPI'
if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v')
uses: pypa/gh-action-pypi-publish@release/v1
with:
packages-dir: ./python/opendataloader-pdf/dist
- name: '[Node.js] Publish to npm'
if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v')
run: pnpm publish --no-git-checks
working-directory: ./node/opendataloader-pdf
env:
NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }}
# =================================================================
# 5. GITHUB RELEASE (only on tag push)
# =================================================================
- name: Package CLI as ZIP
if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v')
run: |
cd java/opendataloader-pdf-cli/target
mkdir -p release
cp "opendataloader-pdf-cli-${{ env.VERSION }}.jar" release/
cp ../../../README.md release/
cp ../../../LICENSE release/
cp ../../../NOTICE release/
cp -r ../../../THIRD_PARTY release/
cd release
zip -r "../opendataloader-pdf-cli-${{ env.VERSION }}.zip" .
cd ../..
- name: Create GitHub Release
if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v')
uses: softprops/action-gh-release@v3
with:
tag_name: ${{ github.ref_name }}
name: Release ${{ github.ref_name }}
generate_release_notes: true
files: |
java/opendataloader-pdf-cli/target/opendataloader-pdf-cli-${{ env.VERSION }}.zip
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
# =================================================================
# 6. SYNC DOCS TO HOMEPAGE (only on tag push)
# =================================================================
- name: Generate reference docs
if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v')
run: |
mkdir -p content/docs/reference
node scripts/generate-options.mjs
node scripts/generate-schema.mjs
- name: Push reference docs to homepage
if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v')
# Pinned to v1.7.3 for security - verify before updating
uses: cpina/github-action-push-to-another-repository@55306faa4ed53b815ae49e564af8cfb359d32ae2 # v1.7.3
with:
source-directory: 'content/docs/reference'
destination-github-username: 'opendataloader-project'
destination-repository-name: 'opendataloader.org'
target-directory: 'apps/v1/content/docs/reference'
target-branch: main
env:
API_TOKEN_GITHUB: ${{ secrets.HOMEPAGE_SYNC_TOKEN }}
.github/workflows/test-benchmark.yml
+200
View File
@@ -0,0 +1,200 @@
name: Test & Benchmark
on:
pull_request:
branches: [main]
paths:
- 'java/**'
- 'python/**'
- 'node/**'
- 'scripts/**'
- 'verification/**'
- 'samples/**'
- '.github/workflows/**'
workflow_dispatch:
concurrency:
group: ci-${{ github.event.pull_request.number || github.ref }}
cancel-in-progress: true
permissions:
contents: read
jobs:
test:
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v7
- name: Setup Java
uses: actions/setup-java@v5
with:
distribution: 'temurin'
java-version: '21'
- name: Setup Python
uses: actions/setup-python@v6
with:
python-version: '3.12'
- name: Setup uv
uses: astral-sh/setup-uv@v7
- name: Setup Node.js
uses: actions/setup-node@v6
with:
node-version: '20'
- name: Setup pnpm
run: npm install -g pnpm
- name: Build & Test All
run: ./scripts/build-all.sh
- name: Install opendataloader-pdf CLI
# --system installs into the setup-python interpreter; without it (or an
# active venv) `uv pip install` aborts with "No virtual environment found".
run: uv pip install --system ./python/opendataloader-pdf/dist/*.whl
- name: Run CLI verification
# ci-verify.py writes its own markdown table to $GITHUB_STEP_SUMMARY,
# so no separate summary step is needed here.
run: python verification/ci-verify.py
- name: Upload verification report
if: always()
uses: actions/upload-artifact@v7
with:
name: verification-report-ci
path: verification/verification-report-ci.txt
retention-days: 7
- name: Upload coverage to Codecov
uses: codecov/codecov-action@v7
with:
files: java/opendataloader-pdf-core/target/site/jacoco/jacoco.xml
fail_ci_if_error: false
token: ${{ secrets.CODECOV_TOKEN }}
- name: Upload build artifacts
uses: actions/upload-artifact@v7
with:
name: java-build
path: java/opendataloader-pdf-cli/target/*.jar
retention-days: 1
benchmark:
needs: test
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v7
- name: Setup Java
uses: actions/setup-java@v5
with:
distribution: 'temurin'
java-version: '21'
- name: Download build artifacts
uses: actions/download-artifact@v8
with:
name: java-build
path: java/opendataloader-pdf-cli/target/
- name: Setup Python
uses: actions/setup-python@v6
with:
python-version: '3.13'
- name: Setup uv
uses: astral-sh/setup-uv@v7
- name: Run benchmark
run: ./scripts/bench.sh --skip-build --check-regression
- name: Benchmark summary
if: always()
run: |
python3 << 'PYEOF'
import json, os, sys
from pathlib import Path
eval_path = Path("/tmp/opendataloader-bench/prediction/opendataloader/evaluation.json")
thresh_path = Path("/tmp/opendataloader-bench/thresholds.json")
summary_file = os.environ.get("GITHUB_STEP_SUMMARY", "/dev/null")
if not eval_path.exists() or not thresh_path.exists():
with open(summary_file, "a") as f:
f.write("## Benchmark Results\n\nBenchmark did not produce evaluation results.\n")
sys.exit(0)
try:
with open(eval_path) as f:
eval_data = json.load(f)
with open(thresh_path) as f:
thresholds = json.load(f)
except json.JSONDecodeError as e:
with open(summary_file, "a") as f:
f.write(f"## Benchmark Results\n\nFailed to parse results: {e}\n")
sys.exit(0)
scores = eval_data.get("metrics", {}).get("score", {})
table_detection = eval_data.get("table_detection", {})
speed = eval_data.get("speed", {})
triage = eval_data.get("triage", {})
tol = thresholds.get("regression_tolerance", 0)
rows = []
for key, label, src in [
("nid", "NID", scores.get("nid_mean")),
("teds", "TEDS", scores.get("teds_mean")),
("mhs", "MHS", scores.get("mhs_mean")),
("table_detection_f1", "Table Detection F1", table_detection.get("f1")),
]:
t = thresholds.get(key)
if src is not None and t is not None:
effective = t - tol
status = "✅" if src >= effective else "❌"
rows.append(f"| {label} | {src:.4f} | ≥ {effective:.2f} | {status} |")
elapsed = speed.get("elapsed_per_doc")
elapsed_thresh = thresholds.get("elapsed_per_doc")
if elapsed is not None and elapsed_thresh is not None:
status = "✅" if elapsed <= elapsed_thresh else "❌"
rows.append(f"| Speed | {elapsed:.2f}s/doc | ≤ {elapsed_thresh}s/doc | {status} |")
if triage:
tr_recall = triage.get("recall")
tr_thresh = thresholds.get("triage_recall")
if tr_recall is not None and tr_thresh is not None:
effective = tr_thresh - tol
status = "✅" if tr_recall >= effective else "❌"
rows.append(f"| Triage Recall | {tr_recall:.4f} | ≥ {effective:.2f} | {status} |")
tr_fn = triage.get("fn_count")
tr_fn_max = thresholds.get("triage_fn_max")
if tr_fn is not None and tr_fn_max is not None:
status = "✅" if tr_fn <= tr_fn_max else "❌"
rows.append(f"| Triage FN | {tr_fn} | ≤ {tr_fn_max} | {status} |")
with open(summary_file, "a") as f:
f.write("## Benchmark Results\n\n")
f.write("| Metric | Score | Threshold | Status |\n")
f.write("|--------|-------|-----------|--------|\n")
for row in rows:
f.write(row + "\n")
if not rows:
f.write("| (no metrics found) | | | |\n")
PYEOF
- name: Upload evaluation results
uses: actions/upload-artifact@v7
if: always()
with:
name: benchmark-results
path: /tmp/opendataloader-bench/prediction/opendataloader/evaluation.json
.gitignore
+89
View File
@@ -0,0 +1,89 @@
*.class
# OS specific files
.DS_Store
Thumbs.db
# IDE / Editor files
**/.idea/
**/.vscode/
*.iml
*.ipr
*.iws
# Emacs
\#*\#
*~
.#*
# Vim
*.swp
*.swo
*.swn
# Java (Maven & Gradle)
**/target/
**/build/
**/bin/
**/.gradle/
**/*.jar
**/*.war
**/*.ear
**/dependency-reduced-pom.xml
**/.flattened-pom.xml
# Logs
*.log
logs/
**/logs/
**/npm-debug.log*
**/yarn-debug.log*
**/yarn-error.log*
**/pnpm-debug.log*
# Node.js
**/node_modules/
**/.next/
**/.turbo/
**/dist/
**/.cache/
*.tsbuildinfo
# Python
**/__pycache__/
**/*.py[cod]
*.pyo
*.pyd
*.so
**/.venv/
**/.env
**/.env.*
*.egg-info/
**/.eggs/
**/.mypy_cache/
**/.pytest_cache/
**/.coverage
**/htmlcov/
# Temporary / Generated
**/tmp/
**/temp/
# Git worktrees
.worktrees/
# Auto-generated reference docs (CI-only, not tracked)
content/docs/
# Configuration files
.claude/settings.local.json
.claude/plans/
# Verification reports (local runs only; CI reports are uploaded as artifacts)
verification/verification-report-local.txt
verification/verification-report-ci.txt
# Local fixture generator (kept out of the repo: fixtures are committed
# binaries; regeneration is rare and the generator needs pikepdf/reportlab)
scripts/generate-test-fixtures.py
CHANGELOG.md
+5
View File
@@ -0,0 +1,5 @@
# Changelog
## 0.1.0
- Initial release.
CLAUDE.md
+25
View File
@@ -0,0 +1,25 @@
# CLAUDE.md
## Gotchas
After changing CLI options in Java, **must** run `npm run sync` — this regenerates `options.json` and all Python/Node.js bindings. Forgetting this silently breaks the wrappers.
When using `--enrich-formula` or `--enrich-picture-description` on the hybrid server, the client **must** use `--hybrid-mode full`. Otherwise enrichments are silently skipped (they only run on the backend, not in Java).
Processing uses `ForkJoinPool(availableProcessors)` for per-page parallelism. All `StaticContainers` and `StaticLayoutContainers` ThreadLocal state must be propagated to worker threads via `propagateState.run()` — missing a ThreadLocal causes silent data loss or NPE in parallel mode.
Hidden text detection (`--filter-hidden-text`) is **off by default** — it requires per-page PDF rendering via `ContrastRatioConsumer` which cannot be parallelized safely.
`--format` values name **output file kinds only** (json, text, html, pdf, markdown, tagged-pdf). Markdown rendering modifiers and image extraction are separate flags: `--markdown-with-html` for HTML-in-Markdown, `--image-output off|embedded|external` for image control. `markdown-with-html` and `markdown-with-images` are still accepted as `--format` values for one major release but emit a deprecation warning and will be removed.
## Conventions
Manual docs live in opendataloader.org repo. Reference docs (CLI options, JSON schema) are auto-generated by CI at release time and pushed to opendataloader.org. No MDX files are tracked in this repo.
## Benchmark
- `./scripts/bench.sh` — Run benchmark (auto-clones opendataloader-bench for PDFs and evaluation logic)
- `./scripts/bench.sh --doc-id <id>` — Debug specific document
- `./scripts/bench.sh --check-regression` — CI mode with threshold check
- Benchmark code lives in [opendataloader-bench](https://github.com/opendataloader-project/opendataloader-bench)
- Metrics: **NID** (reading order), **TEDS** (table structure), **MHS** (heading structure), **Table Detection F1**, **Speed**
CODE_OF_CONDUCT.md
+129
View File
@@ -0,0 +1,129 @@
# Contributor Covenant Code of Conduct
## Our Pledge
We as members, contributors, and leaders pledge to make participation in our
community a harassment-free experience for everyone, regardless of age, body
size, visible or invisible disability, ethnicity, sex characteristics, gender
identity and expression, level of experience, education, socio-economic status,
nationality, personal appearance, race, religion, or sexual identity
and orientation.
We pledge to act and interact in ways that contribute to an open, welcoming,
diverse, inclusive, and healthy community.
## Our Standards
Examples of behavior that contributes to a positive environment for our
community include:
- Demonstrating empathy and kindness toward other people
- Being respectful of differing opinions, viewpoints, and experiences
- Giving and gracefully accepting constructive feedback
- Accepting responsibility and apologizing to those affected by our mistakes,
and learning from the experience
- Focusing on what is best not just for us as individuals, but for the
overall community
Examples of unacceptable behavior include:
- The use of sexualized language or imagery, and sexual attention or
advances of any kind
- Trolling, insulting or derogatory comments, and personal or political attacks
- Public or private harassment
- Publishing others' private information, such as a physical or email
address, without their explicit permission
- Other conduct which could reasonably be considered inappropriate in a
professional setting
## Enforcement Responsibilities
Community leaders are responsible for clarifying and enforcing our standards of
acceptable behavior and will take appropriate and fair corrective action in
response to any behavior that they deem inappropriate, threatening, offensive,
or harmful.
Community leaders have the right and responsibility to remove, edit, or reject
comments, commits, code, wiki edits, issues, and other contributions that are
not aligned to this Code of Conduct, and will communicate reasons for moderation
decisions when appropriate.
## Scope
This Code of Conduct applies within all community spaces, and also applies when
an individual is officially representing the community in public spaces.
Examples of representing our community include using an official e-mail address,
posting via an official social media account, or acting as an appointed
representative at an online or offline event.
## Enforcement
Instances of abusive, harassing, or otherwise unacceptable behavior may be
reported to the community leaders responsible for enforcement using
[open.dataloader@hancom.com](mailto:open.dataloader@hancom.com).
All complaints will be reviewed and investigated promptly and fairly.
All community leaders are obligated to respect the privacy and security of the
reporter of any incident.
## Enforcement Guidelines
Community leaders will follow these Community Impact Guidelines in determining
the consequences for any action they deem in violation of this Code of Conduct:
### 1. Correction
**Community Impact**: Use of inappropriate language or other behavior deemed
unprofessional or unwelcome in the community.
**Consequence**: A private, written warning from community leaders, providing
clarity around the nature of the violation and an explanation of why the
behavior was inappropriate. A public apology may be requested.
### 2. Warning
**Community Impact**: A violation through a single incident or series
of actions.
**Consequence**: A warning with consequences for continued behavior. No
interaction with the people involved, including unsolicited interaction with
those enforcing the Code of Conduct, for a specified period of time. This
includes avoiding interactions in community spaces as well as external channels
like social media. Violating these terms may lead to a temporary or
permanent ban.
### 3. Temporary Ban
**Community Impact**: A serious violation of community standards, including
sustained inappropriate behavior.
**Consequence**: A temporary ban from any sort of interaction or public
communication with the community for a specified period of time. No public or
private interaction with the people involved, including unsolicited interaction
with those enforcing the Code of Conduct, is allowed during this period.
Violating these terms may lead to a permanent ban.
### 4. Permanent Ban
**Community Impact**: Demonstrating a pattern of violation of community
standards, including sustained inappropriate behavior, harassment of an
individual, or aggression toward or disparagement of classes of individuals.
**Consequence**: A permanent ban from any sort of public interaction within
the community.
## Attribution
This Code of Conduct is adapted from the [Contributor Covenant](https://www.contributor-covenant.org)],
version 2.0, available at
[https://www.contributor-covenant.org/version/2/0/code_of_conduct.html](https://www.contributor-covenant.org/version/2/0/code_of_conduct.html).
Community Impact Guidelines were inspired by [Mozilla's code of conduct
enforcement ladder](https://github.com/mozilla/diversity).
Homepage: [https://www.contributor-covenant.org](https://www.contributor-covenant.org)
For answers to common questions about this code of conduct, see the FAQ at
[https://www.contributor-covenant.org/faq](https://www.contributor-covenant.org/faq). Translations are available at
[https://www.contributor-covenant.org/translations](https://www.contributor-covenant.org/translations).
CONTRIBUTING.md
+138
View File
@@ -0,0 +1,138 @@
# Contributing to This Project
Thank you for your interest in contributing!
We welcome contributions from everyone. This document outlines the guidelines for how to contribute effectively and
respectfully.
---
## 📌 Types of Contributions We Welcome
We appreciate various kinds of contributions, including but not limited to:
- 🛠️ **Code contributions** (bug fixes, performance improvements, new features)
- 🐞 **Bug reports**
- 💡 **Feature suggestions**
- ❓ **Questions and discussions**
- 📚 **Improving documentation**
---
## ❓ How to Ask Questions
If you have questions:
1. Check the [README](./README.md) and
existing [issues](https://github.com/opendataloader-project/opendataloader-pdf/issues) first.
2. If your question hasn't been addressed, open a new issue using the `Question` label.
---
## 🐛 How to Report Bugs
When reporting a bug, please include the following:
- A clear and descriptive title
- Steps to reproduce the issue
- Expected vs actual behavior
- Environment info (OS, version, etc.)
- Logs or screenshots if available
Use the **Bug Report** issue template when creating the issue.
---
## 💡 How to Suggest a Feature
To suggest a new feature:
1. Search existing issues to avoid duplicates.
2. If it's new, open a new issue using the **Feature Request** template.
3. Describe your idea, use cases, and possible alternatives.
---
## 🔧 How to Contribute Code
### Step-by-Step Process
1. **Fork** the repository.
2. **Clone** your fork:
```bash
git clone https://github.com/your-username/opendataloader-pdf.git
cd opendataloader-pdf
```
3. **Create a feature branch:**
```bash
git checkout -b my-feature
```
4. **Build** the project:
**Prerequisites:** Java 11+, Maven, Python 3.10+, uv, Node.js 20+, pnpm
See the [Development Workflow guide](https://opendataloader.org/docs/development-workflow) for OS-specific install instructions.
```bash
# Build Java packages
npm run build-java
# If you changed CLI options in Java, sync bindings (regenerates options.json, Python/Node.js wrappers)
npm run sync
```
> **Important**: If you modified any CLI options in Java, you **must** run `npm run sync` before committing. This regenerates `options.json` and all Python/Node.js bindings. Forgetting this silently breaks the wrappers.
5. Make your changes and commit them.
6. **Push** your branch:
```bash
git push origin my-feature
```
7. **Open a Pull Request** (PR) against the `main` branch.
8. Respond to review comments and update your PR as needed.
---
## 🧹 Coding Style & Guidelines
- Follow existing code conventions.
- Run linters/formatters before committing.
- Write unit tests for any new or changed logic.
- Run `./scripts/bench.sh` before submitting a PR — CI will fail if benchmark scores drop below thresholds.
- Keep your changes minimal and focused.
## ✅ Commit Message Guidelines
Use the following format:
```
<type> <short summary>
```
### Common types:
- Add: New feature
- Fix: Bug fix
- Update: Code update
## 📝 CLA / DCO Requirements
Depending on your contribution, we may ask you to sign:
- CLA – Contributor License Agreement
- DCO – Developer Certificate of Origin
To sign the DCO, add `Signed-off-by` to your commit message:
```
git commit -s -m "your message"
```
Make sure your Git config contains your real name and email.
Thank you again for helping us improve this project! 🙌
If you have any questions, open an issue or join the discussion.
LICENSE
+202
View File
@@ -0,0 +1,202 @@
Apache License
Version 2.0, January 2004
http://www.apache.org/licenses/
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
1. Definitions.
"License" shall mean the terms and conditions for use, reproduction,
and distribution as defined by Sections 1 through 9 of this document.
"Licensor" shall mean the copyright owner or entity authorized by
the copyright owner that is granting the License.
"Legal Entity" shall mean the union of the acting entity and all
other entities that control, are controlled by, or are under common
control with that entity. For the purposes of this definition,
"control" means (i) the power, direct or indirect, to cause the
direction or management of such entity, whether by contract or
otherwise, or (ii) ownership of fifty percent (50%) or more of the
outstanding shares, or (iii) beneficial ownership of such entity.
"You" (or "Your") shall mean an individual or Legal Entity
exercising permissions granted by this License.
"Source" form shall mean the preferred form for making modifications,
including but not limited to software source code, documentation
source, and configuration files.
"Object" form shall mean any form resulting from mechanical
transformation or translation of a Source form, including but
not limited to compiled object code, generated documentation,
and conversions to other media types.
"Work" shall mean the work of authorship, whether in Source or
Object form, made available under the License, as indicated by a
copyright notice that is included in or attached to the work
(an example is provided in the Appendix below).
"Derivative Works" shall mean any work, whether in Source or Object
form, that is based on (or derived from) the Work and for which the
editorial revisions, annotations, elaborations, or other modifications
represent, as a whole, an original work of authorship. For the purposes
of this License, Derivative Works shall not include works that remain
separable from, or merely link (or bind by name) to the interfaces of,
the Work and Derivative Works thereof.
"Contribution" shall mean any work of authorship, including
the original version of the Work and any modifications or additions
to that Work or Derivative Works thereof, that is intentionally
submitted to Licensor for inclusion in the Work by the copyright owner
or by an individual or Legal Entity authorized to submit on behalf of
the copyright owner. For the purposes of this definition, "submitted"
means any form of electronic, verbal, or written communication sent
to the Licensor or its representatives, including but not limited to
communication on electronic mailing lists, source code control systems,
and issue tracking systems that are managed by, or on behalf of, the
Licensor for the purpose of discussing and improving the Work, but
excluding communication that is conspicuously marked or otherwise
designated in writing by the copyright owner as "Not a Contribution."
"Contributor" shall mean Licensor and any individual or Legal Entity
on behalf of whom a Contribution has been received by Licensor and
subsequently incorporated within the Work.
2. Grant of Copyright License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
copyright license to reproduce, prepare Derivative Works of,
publicly display, publicly perform, sublicense, and distribute the
Work and such Derivative Works in Source or Object form.
3. Grant of Patent License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
(except as stated in this section) patent license to make, have made,
use, offer to sell, sell, import, and otherwise transfer the Work,
where such license applies only to those patent claims licensable
by such Contributor that are necessarily infringed by their
Contribution(s) alone or by combination of their Contribution(s)
with the Work to which such Contribution(s) was submitted. If You
institute patent litigation against any entity (including a
cross-claim or counterclaim in a lawsuit) alleging that the Work
or a Contribution incorporated within the Work constitutes direct
or contributory patent infringement, then any patent licenses
granted to You under this License for that Work shall terminate
as of the date such litigation is filed.
4. Redistribution. You may reproduce and distribute copies of the
Work or Derivative Works thereof in any medium, with or without
modifications, and in Source or Object form, provided that You
meet the following conditions:
(a) You must give any other recipients of the Work or
Derivative Works a copy of this License; and
(b) You must cause any modified files to carry prominent notices
stating that You changed the files; and
(c) You must retain, in the Source form of any Derivative Works
that You distribute, all copyright, patent, trademark, and
attribution notices from the Source form of the Work,
excluding those notices that do not pertain to any part of
the Derivative Works; and
(d) If the Work includes a "NOTICE" text file as part of its
distribution, then any Derivative Works that You distribute must
include a readable copy of the attribution notices contained
within such NOTICE file, excluding those notices that do not
pertain to any part of the Derivative Works, in at least one
of the following places: within a NOTICE text file distributed
as part of the Derivative Works; within the Source form or
documentation, if provided along with the Derivative Works; or,
within a display generated by the Derivative Works, if and
wherever such third-party notices normally appear. The contents
of the NOTICE file are for informational purposes only and
do not modify the License. You may add Your own attribution
notices within Derivative Works that You distribute, alongside
or as an addendum to the NOTICE text from the Work, provided
that such additional attribution notices cannot be construed
as modifying the License.
You may add Your own copyright statement to Your modifications and
may provide additional or different license terms and conditions
for use, reproduction, or distribution of Your modifications, or
for any such Derivative Works as a whole, provided Your use,
reproduction, and distribution of the Work otherwise complies with
the conditions stated in this License.
5. Submission of Contributions. Unless You explicitly state otherwise,
any Contribution intentionally submitted for inclusion in the Work
by You to the Licensor shall be under the terms and conditions of
this License, without any additional terms or conditions.
Notwithstanding the above, nothing herein shall supersede or modify
the terms of any separate license agreement you may have executed
with Licensor regarding such Contributions.
6. Trademarks. This License does not grant permission to use the trade
names, trademarks, service marks, or product names of the Licensor,
except as required for reasonable and customary use in describing the
origin of the Work and reproducing the content of the NOTICE file.
7. Disclaimer of Warranty. Unless required by applicable law or
agreed to in writing, Licensor provides the Work (and each
Contributor provides its Contributions) on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
implied, including, without limitation, any warranties or conditions
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
PARTICULAR PURPOSE. You are solely responsible for determining the
appropriateness of using or redistributing the Work and assume any
risks associated with Your exercise of permissions under this License.
8. Limitation of Liability. In no event and under no legal theory,
whether in tort (including negligence), contract, or otherwise,
unless required by applicable law (such as deliberate and grossly
negligent acts) or agreed to in writing, shall any Contributor be
liable to You for damages, including any direct, indirect, special,
incidental, or consequential damages of any character arising as a
result of this License or out of the use or inability to use the
Work (including but not limited to damages for loss of goodwill,
work stoppage, computer failure or malfunction, or any and all
other commercial damages or losses), even if such Contributor
has been advised of the possibility of such damages.
9. Accepting Warranty or Additional Liability. While redistributing
the Work or Derivative Works thereof, You may choose to offer,
and charge a fee for, acceptance of support, warranty, indemnity,
or other liability obligations and/or rights consistent with this
License. However, in accepting such obligations, You may act only
on Your own behalf and on Your sole responsibility, not on behalf
of any other Contributor, and only if You agree to indemnify,
defend, and hold each Contributor harmless for any liability
incurred by, or claims asserted against, such Contributor by reason
of your accepting any such warranty or additional liability.
END OF TERMS AND CONDITIONS
APPENDIX: How to apply the Apache License to your work.
To apply the Apache License to your work, attach the following
boilerplate notice, with the fields enclosed by brackets "[]"
replaced with your own identifying information. (Don't include
the brackets!) The text should be enclosed in the appropriate
comment syntax for the file format. We also recommend that a
file or class name and description of purpose be included on the
same "printed page" as the copyright notice for easier
identification within third-party archives.
Copyright [yyyy] [name of copyright owner]
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
LICENSE_TEMPLATE/license.txt
+13
View File
@@ -0,0 +1,13 @@
Copyright 2025-2026 Hancom Inc.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
NOTICE
+38
View File
@@ -0,0 +1,38 @@
OpenDataLoader PDF
Copyright 2025-2026 Hancom, Inc.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
This product includes 'OpenDataLoader PDF' distributed under the Apache
License 2.0, along with various third-party software components. For the
complete source code and detailed copyright notices and license information
for each third-party component, please visit:
https://github.com/opendataloader-project/opendataloader-pdf
THIRD-PARTY LICENSES
This project includes third-party libraries and components, licensed under
their respective open source licenses. For details, see:
THIRD_PARTY/THIRD_PARTY_LICENSES.md
THIRD_PARTY/THIRD_PARTY_NOTICES.md
THIRD_PARTY/licenses/
HISTORICAL NOTE
Versions of OpenDataLoader PDF prior to 2.0 were licensed under the
Mozilla Public License 2.0 (MPL-2.0). From version 2.0 onwards, the
project is licensed under the Apache License 2.0.
README.md
+603
View File
@@ -0,0 +1,603 @@
<!-- AI-AGENT-SUMMARY
name: opendataloader-pdf
category: PDF data extraction, PDF accessibility automation
license: Apache-2.0
solves: [PDF to structured data for RAG/LLM pipelines, accelerate PDF accessibility remediation — layout analysis + auto-tagging to Tagged PDF as foundation for PDF/UA (first open-source end-to-end)]
input: PDF files (digital, scanned, tagged)
output: Markdown, JSON (with bounding boxes), HTML, Tagged PDF, PDF/UA (enterprise)
sdk: Python, Node.js, Java
requirements: Java 11+
pricing: open-source core (data extraction, layout analysis, auto-tagging to Tagged PDF), enterprise add-on (PDF/UA export, accessibility studio)
extraction-benchmark: #1 overall extraction accuracy (0.907) in hybrid mode, 0.928 table extraction accuracy, 0.015s/page local mode
accessibility-validation: PDF Association collaboration, Well-Tagged PDF specification, veraPDF automated validation
key-differentiators: [benchmark #1 PDF parser, deterministic output, bounding boxes for every element, XY-Cut++ reading order, AI safety filters, hybrid AI mode, first open-source PDF auto-tagging to Tagged PDF, PDF Association + Dual Lab (veraPDF) collaboration, Well-Tagged PDF spec compliance]
-->
# OpenDataLoader PDF
**PDF Parser for AI-ready data. Automate PDF accessibility. Open-source.**
[![License](https://img.shields.io/badge/License-Apache_2.0-blue.svg)](https://github.com/opendataloader-project/opendataloader-pdf/blob/main/LICENSE)
[![PyPI version](https://img.shields.io/pypi/v/opendataloader-pdf.svg)](https://pypi.org/project/opendataloader-pdf/)
[![npm version](https://img.shields.io/npm/v/@opendataloader/pdf.svg)](https://www.npmjs.com/package/@opendataloader/pdf)
[![Maven Central](https://img.shields.io/maven-central/v/org.opendataloader/opendataloader-pdf-core.svg)](https://search.maven.org/artifact/org.opendataloader/opendataloader-pdf-core)
[![Java](https://img.shields.io/badge/Java-11%2B-blue.svg)](https://github.com/opendataloader-project/opendataloader-pdf#java)
<a href="https://trendshift.io/repositories/21917" target="_blank"><img src="https://trendshift.io/api/badge/repositories/21917" alt="opendataloader-project%2Fopendataloader-pdf | Trendshift" style="width: 250px; height: 55px;" width="250" height="55"/></a>
🔍 **PDF parser for AI data extraction** — Extract Markdown, JSON (with bounding boxes), and HTML from any PDF. #1 in benchmarks (0.907 overall). Deterministic local mode + AI hybrid mode for complex pages.
- **How accurate is it?** — #1 in benchmarks: 0.907 overall, 0.928 table accuracy across 200 real-world PDFs including multi-column and scientific papers. Deterministic local mode + AI hybrid mode for complex pages ([benchmarks](#extraction-benchmarks))
- **Scanned PDFs and OCR?** — Yes. Built-in OCR (80+ languages) in hybrid mode. Works with poor-quality scans at 300 DPI+ ([hybrid mode](#hybrid-mode-1-accuracy-for-complex-pdfs))
- **Tables, formulas, images, charts?** — Yes. Complex/borderless tables, LaTeX formulas, and AI-generated picture/chart descriptions all via hybrid mode ([hybrid mode](#hybrid-mode-1-accuracy-for-complex-pdfs))
- **How do I use this for RAG?** — `pip install opendataloader-pdf`, convert in 3 lines. Outputs structured Markdown for chunking, JSON with bounding boxes for source citations, and HTML. LangChain integration available. Python, Node.js, Java SDKs ([quick start](#get-started-in-30-seconds) | [LangChain](#langchain-integration))
♿ **PDF accessibility automation** — Auto-tag untagged PDFs into screen-reader-ready Tagged PDFs at scale. First open-source tool to generate Tagged PDFs end-to-end.
- **What's the problem?** — Accessibility regulations are now enforced worldwide. Manual PDF remediation costs $50–200 per document and doesn't scale ([regulations](#pdf-accessibility--pdfua-conversion))
- **What's free?** — Layout analysis + auto-tagging (Apache 2.0). Untagged PDF in → Tagged PDF out. No proprietary SDK dependency ([auto-tagging](#auto-tagging))
- **What about PDF/UA compliance?** — Converting Tagged PDF to PDF/UA-1 or PDF/UA-2 is an enterprise add-on. Auto-tagging generates the Tagged PDF; PDF/UA export is the final step ([pipeline](#accessibility-pipeline))
- **Why trust this?** — Built in collaboration with [Dual Lab](https://duallab.com) ([veraPDF](https://verapdf.org) developers) based on [PDF Association](https://pdfa.org) specifications, best practice guides and expertise of the [PDF Community](https://pdfa.org/community/). Auto-tagging follows the [Well-Tagged PDF specification](https://pdfa.org/wtpdf/), validated with veraPDF ([collaboration](https://opendataloader.org/docs/tagged-pdf-collaboration))
## Get Started in 30 Seconds
**Requires**: Java 11+ and Python 3.10+ ([Node.js](https://opendataloader.org/docs/quick-start-nodejs) | [Java](https://opendataloader.org/docs/quick-start-java) also available)
> Before you start: run `java -version`. If not found, install JDK 11+ from [Adoptium](https://adoptium.net/).
```bash
pip install -U opendataloader-pdf
```
```python
import opendataloader_pdf
# Batch all files in one call — each convert() spawns a JVM process, so repeated calls are slow
opendataloader_pdf.convert(
input_path=["file1.pdf", "file2.pdf", "folder/"],
output_dir="output/",
format="markdown,json"
)
```
![OpenDataLoader PDF layout analysis — headings, tables, images detected with bounding boxes](https://raw.githubusercontent.com/opendataloader-project/opendataloader-pdf/main/samples/image/example_annotated_pdf.png)
*Annotated PDF output — each element (heading, paragraph, table, image) detected with bounding boxes and semantic type.*
## What Problems Does This Solve?
| Problem | Solution | Status |
|---------|----------|--------|
| **PDF structure lost during parsing** — wrong reading order, broken tables, no element coordinates | Deterministic local PDF to Markdown/JSON with bounding boxes, XY-Cut++ reading order | Shipped |
| **Complex tables, scanned PDFs, formulas, charts** need AI-level understanding | Hybrid mode routes complex pages to AI backend (#1 in benchmarks) | Shipped |
| **Manual PDF remediation cost** — Accessibility regulations (EAA, ADA, Section 508) demand Tagged PDFs. Manual remediation costs $50–200/doc | Auto-tag untagged PDFs into Tagged PDFs (free, Apache 2.0). Foundation for PDF/UA workflows; full PDF/UA-1/2 export is an enterprise add-on | Auto-tag: Shipped. PDF/UA export: Enterprise |
## Capability Matrix
| Capability | Supported | Tier |
|------------|-----------|------|
| **Data extraction** | | |
| Extract text with correct reading order | Yes | Free |
| Bounding boxes for every element | Yes | Free |
| Table extraction (simple borders) | Yes | Free |
| Table extraction (complex/borderless) | Yes | Free (Hybrid) |
| Heading hierarchy detection | Yes | Free |
| List detection (numbered, bulleted, nested) | Yes | Free |
| Image extraction with coordinates | Yes | Free |
| AI chart/image description | Yes | Free (Hybrid) |
| OCR for scanned PDFs | Yes | Free (Hybrid) |
| Formula extraction (LaTeX) | Yes | Free (Hybrid) |
| Tagged PDF structure extraction | Yes | Free |
| AI safety (prompt injection filtering) | Yes | Free |
| Header/footer/watermark filtering | Yes | Free |
| **Accessibility** | | |
| Auto-tagging → Tagged PDF for untagged PDFs | Yes | Free (Apache 2.0) |
| PDF/UA-1, PDF/UA-2 export | 💼 Available | Enterprise |
| Accessibility studio (visual editor) | 💼 Available | Enterprise |
| **Limitations** | | |
| Process Word/Excel/PPT | No | — |
| GPU required | No | — |
## Extraction Benchmarks
**opendataloader-pdf [hybrid] ranks #1 overall (0.907)** across reading order, table, and heading extraction accuracy.
| Engine | Overall | Reading Order | Table | Heading | Speed (s/page) | License |
|--------|---------|---------------|-------|---------|----------------|---------|
| **opendataloader [hybrid]** | **0.907** | **0.934** | **0.928** | 0.821 | 0.463 | Apache-2.0 |
| nutrient | 0.885 | 0.925 | 0.708 | 0.819 | **0.008** | Commercial |
| docling | 0.882 | 0.898 | 0.887 | **0.824** | 0.762 | MIT |
| marker | 0.861 | 0.890 | 0.808 | 0.796 | 53.932 | GPL-3.0 |
| unstructured [hi_res] | 0.841 | 0.904 | 0.588 | 0.749 | 3.008 | Apache-2.0 |
| edgeparse | 0.837 | 0.894 | 0.717 | 0.706 | 0.036 | Apache-2.0 |
| opendataloader | 0.831 | 0.902 | 0.489 | 0.739 | 0.015 | Apache-2.0 |
| mineru | 0.831 | 0.857 | 0.873 | 0.743 | 5.962 | AGPL-3.0 |
| pymupdf4llm | 0.732 | 0.885 | 0.401 | 0.412 | 0.091 | AGPL-3.0 |
| unstructured | 0.686 | 0.882 | 0.000 | 0.388 | 0.077 | Apache-2.0 |
| markitdown | 0.589 | 0.844 | 0.273 | 0.000 | 0.114 | MIT |
| liteparse | 0.576 | 0.866 | 0.000 | 0.000 | 1.061 | Apache-2.0 |
> Scores normalized to [0, 1]. Higher is better for accuracy; lower is better for speed. **Bold** = best. [Full benchmark details](https://github.com/opendataloader-project/opendataloader-bench)
[![Benchmark](https://github.com/opendataloader-project/opendataloader-bench/raw/refs/heads/main/charts/benchmark.png)](https://github.com/opendataloader-project/opendataloader-bench)
[![Quality Breakdown](https://github.com/opendataloader-project/opendataloader-bench/raw/refs/heads/main/charts/benchmark_quality.png)](https://github.com/opendataloader-project/opendataloader-bench)
## Which Mode Should I Use?
| Your Document | Mode | Install | Server Command | Client Command |
|---------------|------|---------|----------------|----------------|
| Standard digital PDF | Fast (default) | `pip install opendataloader-pdf` | None needed | `opendataloader-pdf file1.pdf file2.pdf folder/` |
| Complex or nested tables | **Hybrid** | `pip install "opendataloader-pdf[hybrid]"` | `opendataloader-pdf-hybrid --port 5002` | `opendataloader-pdf --hybrid docling-fast file1.pdf file2.pdf folder/` |
| Scanned / image-based PDF | Hybrid + OCR | `pip install "opendataloader-pdf[hybrid]"` | `opendataloader-pdf-hybrid --port 5002 --force-ocr` | `opendataloader-pdf --hybrid docling-fast file1.pdf file2.pdf folder/` |
| Non-English scanned PDF | Hybrid + OCR | `pip install "opendataloader-pdf[hybrid]"` | `opendataloader-pdf-hybrid --port 5002 --force-ocr --ocr-lang "ko,en"` | `opendataloader-pdf --hybrid docling-fast file1.pdf file2.pdf folder/` |
| Mathematical formulas | Hybrid + formula | `pip install "opendataloader-pdf[hybrid]"` | `opendataloader-pdf-hybrid --enrich-formula` | `opendataloader-pdf --hybrid docling-fast --hybrid-mode full file1.pdf file2.pdf folder/` |
| Charts needing description | Hybrid + picture | `pip install "opendataloader-pdf[hybrid]"` | `opendataloader-pdf-hybrid --enrich-picture-description` | `opendataloader-pdf --hybrid docling-fast --hybrid-mode full file1.pdf file2.pdf folder/` |
| Untagged PDFs needing accessibility | Auto-tagging → Tagged PDF | `pip install opendataloader-pdf` | None needed | `opendataloader-pdf --format tagged-pdf file1.pdf file2.pdf folder/` |
## Quick Start
### Python
```bash
pip install -U opendataloader-pdf
```
```python
import opendataloader_pdf
# Batch all files in one call — each convert() spawns a JVM process, so repeated calls are slow
opendataloader_pdf.convert(
input_path=["file1.pdf", "file2.pdf", "folder/"],
output_dir="output/",
format="markdown,json"
)
```
### Node.js
```bash
npm install @opendataloader/pdf
```
```typescript
import { convert } from '@opendataloader/pdf';
await convert(['file1.pdf', 'file2.pdf', 'folder/'], {
outputDir: 'output/',
format: 'markdown,json'
});
```
### Java
```xml
<dependency>
<groupId>org.opendataloader</groupId>
<artifactId>opendataloader-pdf-core</artifactId>
</dependency>
```
[Python Quick Start](https://opendataloader.org/docs/quick-start-python) | [Node.js Quick Start](https://opendataloader.org/docs/quick-start-nodejs) | [Java Quick Start](https://opendataloader.org/docs/quick-start-java)
## Hybrid Mode: #1 Accuracy for Complex PDFs
Hybrid mode combines fast local Java processing with AI backends. Simple pages stay local (0.02s); complex pages route to AI for +90% table accuracy.
```bash
pip install -U "opendataloader-pdf[hybrid]"
```
**Terminal 1** — Start the backend server:
```bash
opendataloader-pdf-hybrid --port 5002
```
**Terminal 2** — Process PDFs:
```bash
# Batch all files in one call — each invocation spawns a JVM process, so repeated calls are slow
opendataloader-pdf --hybrid docling-fast file1.pdf file2.pdf folder/
```
**Python:**
```python
# Batch all files in one call — each convert() spawns a JVM process, so repeated calls are slow
opendataloader_pdf.convert(
input_path=["file1.pdf", "file2.pdf", "folder/"],
output_dir="output/",
hybrid="docling-fast"
)
```
### OCR for Scanned PDFs
Start the backend with `--force-ocr` for image-based PDFs with no selectable text:
```bash
opendataloader-pdf-hybrid --port 5002 --force-ocr
```
For non-English documents, specify the language:
```bash
opendataloader-pdf-hybrid --port 5002 --force-ocr --ocr-lang "ko,en"
```
Supported languages: `en`, `ko`, `ja`, `ch_sim`, `ch_tra`, `de`, `fr`, `ar`, and more.
### Formula Extraction (LaTeX)
Extract mathematical formulas as LaTeX from scientific PDFs:
```bash
# Server: enable formula enrichment
opendataloader-pdf-hybrid --enrich-formula
# Batch all files in one call — each invocation spawns a JVM process, so repeated calls are slow
opendataloader-pdf --hybrid docling-fast --hybrid-mode full file1.pdf file2.pdf folder/
```
Output in JSON:
```json
{
"type": "formula",
"page number": 1,
"bounding box": [226.2, 144.7, 377.1, 168.7],
"content": "\\frac{f(x+h) - f(x)}{h}"
}
```
> **Note**: Formula and picture description enrichments require `--hybrid-mode full` on the client side.
### Chart & Image Description
Generate AI descriptions for charts and images — useful for RAG search and accessibility alt text:
```bash
# Server
opendataloader-pdf-hybrid --enrich-picture-description
# Batch all files in one call — each invocation spawns a JVM process, so repeated calls are slow
opendataloader-pdf --hybrid docling-fast --hybrid-mode full file1.pdf file2.pdf folder/
```
Output in JSON:
```json
{
"type": "picture",
"page number": 1,
"bounding box": [72.0, 400.0, 540.0, 650.0],
"description": "A bar chart showing waste generation by region from 2016 to 2030..."
}
```
> Uses SmolVLM (256M), a lightweight vision model. Custom prompts supported via `--picture-description-prompt`.
### Hancom Data Loader Integration — Coming Soon
Enterprise-grade AI document analysis via [Hancom Data Loader](https://sdk.hancom.com/en/services/1?utm_source=github&utm_medium=readme&utm_campaign=opendataloader-pdf) — customer-customized models trained on your domain-specific documents. 30+ element types (tables, charts, formulas, captions, footnotes, etc.), VLM-based image/chart understanding, complex table extraction (merged cells, nested tables), SLA-backed OCR for scanned documents, and native HWP/HWPX support. Supports PDF, DOCX, XLSX, PPTX, HWP, PNG, JPG. [Live demo](https://livedemo.sdk.hancom.com/en/dataloader?utm_source=github&utm_medium=readme&utm_campaign=opendataloader-pdf)
[Hybrid Mode Guide](https://opendataloader.org/docs/hybrid-mode)
## Output Formats
| Format | Use Case |
|--------|----------|
| **JSON** | Structured data with bounding boxes, semantic types |
| **Markdown** | Clean text for LLM context, RAG chunks |
| **HTML** | Web display with styling |
| **Annotated PDF** | Visual debugging — see detected structures ([sample](https://opendataloader.org/demo/samples/01030000000000)) |
| **Text** | Plain text extraction |
Combine formats: `format="json,markdown"`
### JSON Output Example
```json
{
"type": "heading",
"id": 42,
"level": "Title",
"page number": 1,
"bounding box": [72.0, 700.0, 540.0, 730.0],
"heading level": 1,
"font": "Helvetica-Bold",
"font size": 24.0,
"text color": "[0.0]",
"content": "Introduction"
}
```
| Field | Description |
|-------|-------------|
| `type` | Element type: heading, paragraph, table, list, image, caption, formula |
| `id` | Unique identifier for cross-referencing |
| `page number` | 1-indexed page reference |
| `bounding box` | `[left, bottom, right, top]` in PDF points (72pt = 1 inch) |
| `heading level` | Heading depth (1+) |
| `content` | Extracted text |
[Full JSON Schema](https://opendataloader.org/docs/reference/json-schema)
## Advanced Features
### Tagged PDF Support
When a PDF has structure tags, OpenDataLoader extracts the **exact layout** the author intended — no guessing, no heuristics. Headings, lists, tables, and reading order are preserved from the source.
> **Output quality depends on tag quality.** Not all tagged PDFs are well-tagged. For PDFs with sparse or incorrect tags, the default heuristic mode or `--hybrid docling-fast` often produces better results.
```python
# Batch all files in one call — each convert() spawns a JVM process, so repeated calls are slow
opendataloader_pdf.convert(
input_path=["file1.pdf", "file2.pdf", "folder/"],
output_dir="output/",
use_struct_tree=True # Use native PDF structure tags
)
```
Most PDF parsers ignore structure tags entirely. [Learn more](https://opendataloader.org/docs/tagged-pdf)
### AI Safety: Prompt Injection Protection
PDFs can contain hidden prompt injection attacks. OpenDataLoader automatically filters:
- Hidden text (transparent, zero-size fonts)
- Off-page content
- Suspicious invisible layers
To sanitize sensitive data (emails, URLs, phone numbers → placeholders), enable it explicitly:
```bash
# Batch all files in one call — each invocation spawns a JVM process, so repeated calls are slow
opendataloader-pdf file1.pdf file2.pdf folder/ --sanitize
```
[AI Safety Guide](https://opendataloader.org/docs/ai-safety)
### LangChain Integration
```bash
pip install -U langchain-opendataloader-pdf
```
```python
from langchain_opendataloader_pdf import OpenDataLoaderPDFLoader
loader = OpenDataLoaderPDFLoader(
file_path=["file1.pdf", "file2.pdf", "folder/"],
format="text"
)
documents = loader.load()
```
[LangChain Docs](https://docs.langchain.com/oss/python/integrations/document_loaders/opendataloader_pdf) | [GitHub](https://github.com/opendataloader-project/langchain-opendataloader-pdf) | [PyPI](https://pypi.org/project/langchain-opendataloader-pdf/)
### Advanced Options
```python
# Batch all files in one call — each convert() spawns a JVM process, so repeated calls are slow
opendataloader_pdf.convert(
input_path=["file1.pdf", "file2.pdf", "folder/"],
output_dir="output/",
format="json,markdown,pdf",
image_output="embedded", # "off", "embedded" (Base64), or "external" (default)
image_format="jpeg", # "png" or "jpeg"
use_struct_tree=True, # Use native PDF structure
)
```
[Full CLI Options Reference](https://opendataloader.org/docs/reference/cli-options)
## PDF Accessibility & PDF/UA Conversion
**Problem**: Millions of existing PDFs lack structure tags, failing accessibility regulations (EAA, ADA/Section 508, Korea Digital Inclusion Act). Manual remediation costs $50–200 per document and doesn't scale.
**OpenDataLoader's approach**: Built in collaboration with [PDF Association](https://pdfa.org) and [Dual Lab](https://duallab.com) (developers of [veraPDF](https://verapdf.org), the industry-reference open-source PDF/A and PDF/UA validator). Auto-tagging follows the [Well-Tagged PDF specification](https://pdfa.org/resource/well-tagged-pdf/) and is validated programmatically using veraPDF — automated conformance checks against PDF accessibility standards, not manual review. No existing open-source tool generates Tagged PDFs end-to-end — most rely on proprietary SDKs for the tag-writing step. OpenDataLoader does it all under Apache 2.0. ([collaboration details](https://opendataloader.org/docs/tagged-pdf-collaboration))
| Regulation | Deadline | Requirement |
|------------|----------|-------------|
| **European Accessibility Act (EAA)** | June 28, 2025 | Accessible digital products across the EU |
| **ADA & Section 508** | In effect | U.S. federal agencies and public accommodations |
| **Digital Inclusion Act** | In effect | South Korea digital service accessibility |
### Standards & Validation
| Aspect | Detail |
|--------|--------|
| **Specification** | [Well-Tagged PDF](https://pdfa.org/resource/well-tagged-pdf/) by PDF Association |
| **Validation** | [veraPDF](https://verapdf.org) — industry-reference open-source PDF/A & PDF/UA validator |
| **Collaboration** | PDF Association + [Dual Lab](https://duallab.com) (veraPDF developers) co-develop tagging and validation |
| **License** | Auto-tagging → Tagged PDF: Apache 2.0 (free). PDF/UA export: Enterprise |
### Accessibility Pipeline
| Step | Feature | Status | Tier |
|------|---------|--------|------|
| 1. **Audit** | Read existing PDF tags, detect untagged PDFs | Shipped | Free |
| 2. **Auto-tag → Tagged PDF** | Generate structure tags for untagged PDFs | Shipped | Free (Apache 2.0) |
| 3. **Export PDF/UA** | Convert to PDF/UA-1 or PDF/UA-2 compliant files | 💼 Available | Enterprise |
| 4. **Visual editing** | Accessibility studio — review and fix tags | 💼 Available | Enterprise |
> **💼 Enterprise features** are available on request. [Contact us](https://opendataloader.org/contact) to get started.
### Auto-Tagging
Generate Tagged PDFs from untagged PDFs — output is a screen-reader-ready PDF with structure tags (headings, paragraphs, lists, tables, reading order).
```python
import opendataloader_pdf
# Untagged PDF in → Tagged PDF out
opendataloader_pdf.convert(
input_path=["file1.pdf", "file2.pdf", "folder/"],
output_dir="output/",
format="tagged-pdf"
)
```
```bash
# CLI
opendataloader-pdf --format tagged-pdf file1.pdf file2.pdf folder/
```
Combine with other formats: `format="json,tagged-pdf"`.
### End-to-End Compliance Workflow
```
Existing PDFs (untagged)
│
▼
┌─────────────────┐ ┌──────────────────┐ ┌─────────────────┐ ┌──────────────────┐
│ 1. Audit │───>│ 2. Auto-Tag │───>│ 3. Export │───>│ 4. Studio │
│ (check tags) │ │ (→ Tagged PDF) │ │ (PDF/UA) │ │ (visual editor) │
└─────────────────┘ └──────────────────┘ └─────────────────┘ └──────────────────┘
│ │ │ │
▼ ▼ ▼ ▼
use_struct_tree format="tagged-pdf" PDF/UA export Accessibility Studio
(Available now) (Available, Apache 2.0) (Enterprise) (Enterprise)
```
[PDF Accessibility Guide](https://opendataloader.org/docs/accessibility-compliance)
## Roadmap
| Feature | Timeline | Tier |
|---------|----------|------|
| **[Hancom Data Loader](https://sdk.hancom.com/en/services/1?utm_source=github&utm_medium=readme&utm_campaign=opendataloader-pdf)** — Enterprise AI document analysis, customer-customized models, VLM-based chart/image understanding, production-grade OCR | Q2-Q3 2026 | Planned |
| **Structure validation** — Verify PDF tag trees | Q3 2026 | Planned |
[Full Roadmap](https://opendataloader.org/docs/upcoming-roadmap)
## Frequently Asked Questions
### What is the best PDF parser for RAG?
For RAG pipelines, you need a parser that preserves document structure, maintains correct reading order, and provides element coordinates for citations. OpenDataLoader is designed specifically for this — it outputs structured JSON with bounding boxes, handles multi-column layouts with XY-Cut++, and runs locally without GPU. In hybrid mode, it ranks #1 overall (0.907) in benchmarks.
### What is the best open-source PDF parser?
OpenDataLoader PDF is the only open-source parser that combines: rule-based deterministic extraction (no GPU), bounding boxes for every element, XY-Cut++ reading order, built-in AI safety filters, native Tagged PDF support, and hybrid AI mode for complex documents. It ranks #1 in overall accuracy (0.907) while running locally on CPU.
### How do I extract tables from PDF for LLM?
OpenDataLoader detects tables using border analysis and text clustering, preserving row/column structure. For complex tables, enable hybrid mode for +90% accuracy improvement (0.489 to 0.928 TEDS score):
```python
# Batch all files in one call — each convert() spawns a JVM process, so repeated calls are slow
opendataloader_pdf.convert(
input_path=["file1.pdf", "file2.pdf", "folder/"],
output_dir="output/",
format="json",
hybrid="docling-fast" # For complex tables
)
```
### How does it compare to docling, marker, or pymupdf4llm?
OpenDataLoader [hybrid] ranks #1 overall (0.907) across reading order, table, and heading accuracy. Key differences: docling (0.882) is strong but lacks bounding boxes and AI safety filters. marker (0.861) requires GPU and is 1000x slower (53.932s/page). pymupdf4llm (0.732) is fast but has poor table (0.401) and heading (0.412) accuracy. OpenDataLoader is the only parser that combines deterministic local extraction, bounding boxes for every element, and built-in prompt injection protection. See [full benchmark](https://github.com/opendataloader-project/opendataloader-bench).
### Can I use this without sending data to the cloud?
Yes. OpenDataLoader runs 100% locally. No API calls, no data transmission — your documents never leave your environment. The hybrid mode backend also runs locally on your machine. Ideal for legal, healthcare, and financial documents.
### Does it support OCR for scanned PDFs?
Yes, via hybrid mode. Install with `pip install "opendataloader-pdf[hybrid]"`, start the backend with `--force-ocr`, then process as usual. Supports multiple languages including Korean, Japanese, Chinese, Arabic, and more via `--ocr-lang`.
### Does it work with Korean, Japanese, or Chinese documents?
Yes. For digital PDFs, text extraction works out of the box. For scanned PDFs, use hybrid mode with `--force-ocr --ocr-lang "ko,en"` (or `ja`, `ch_sim`, `ch_tra`). Coming soon: [Hancom Data Loader](https://sdk.hancom.com/en/services/1?utm_source=github&utm_medium=readme&utm_campaign=opendataloader-pdf) integration — enterprise-grade AI document analysis with built-in production-grade OCR and customer-customized models optimized for your specific document types and workflows.
### How fast is it?
Local mode processes 60+ pages per second on CPU (0.02s/page). Hybrid mode processes 2+ pages per second (0.46s/page) with significantly higher accuracy for complex documents. No GPU required. Benchmarked on Apple M4. [Full benchmark details](https://github.com/opendataloader-project/opendataloader-bench). With multi-process batch processing, throughput exceeds 100 pages per second on 8+ core machines.
### Does it handle multi-column layouts?
Yes. OpenDataLoader uses XY-Cut++ reading order analysis to correctly sequence text across multi-column pages, sidebars, and mixed layouts. This works in both local and hybrid modes without any configuration.
### What is hybrid mode?
Hybrid mode combines fast local Java processing with an AI backend. Simple pages are processed locally (0.02s/page); complex pages (tables, scanned content, formulas, charts) are automatically routed to the AI backend for higher accuracy. The backend runs locally on your machine — no cloud required. See [Which Mode Should I Use?](#which-mode-should-i-use) and [Hybrid Mode Guide](https://opendataloader.org/docs/hybrid-mode).
### Does it work with LangChain?
Yes. Install `langchain-opendataloader-pdf` for an official LangChain document loader integration. See [LangChain docs](https://docs.langchain.com/oss/python/integrations/document_loaders/opendataloader_pdf).
### How do I chunk PDFs for RAG?
OpenDataLoader outputs structured Markdown with headings, tables, and lists preserved — ideal input for semantic chunking. Each element in JSON output includes `type`, `heading level`, and `page number`, so you can split by section or page boundary. For most RAG pipelines: parse with `format="markdown"` for text chunks, or `format="json"` when you need element-level control. Pair with LangChain's `RecursiveCharacterTextSplitter` or your own heading-based splitter for best results.
### How do I cite PDF sources in RAG answers?
Every element in JSON output includes a `bounding box` (`[left, bottom, right, top]` in PDF points) and `page number`. When your RAG pipeline returns an answer, map the source chunk back to its bounding box to highlight the exact location in the original PDF. This enables "click to source" UX — users see which paragraph, table, or figure the answer came from. No other open-source parser provides bounding boxes for every element by default.
### How do I convert PDF to Markdown for LLM?
```python
import opendataloader_pdf
# Batch all files in one call — each convert() spawns a JVM process, so repeated calls are slow
opendataloader_pdf.convert(
input_path=["file1.pdf", "file2.pdf", "folder/"],
output_dir="output/",
format="markdown"
)
```
OpenDataLoader preserves heading hierarchy, table structure, and reading order in the Markdown output. For complex documents with borderless tables or scanned pages, use hybrid mode (`hybrid="docling-fast"`) for higher accuracy. The output is clean enough to feed directly into LLM context windows or RAG chunking pipelines.
### Is there an automated PDF accessibility remediation tool?
Yes. OpenDataLoader is the first open-source tool that automates PDF accessibility end-to-end. Built in collaboration with [PDF Association](https://pdfa.org) and [Dual Lab](https://duallab.com) (veraPDF developers), auto-tagging follows the Well-Tagged PDF specification and is validated programmatically using veraPDF. The layout analysis engine detects document structure (headings, tables, lists, reading order) and generates accessibility tags automatically. Auto-tagging converts untagged PDFs into Tagged PDFs under Apache 2.0 — no proprietary SDK dependency. Use `format="tagged-pdf"` (Python/Node.js) or `--format tagged-pdf` (CLI). For organizations needing full PDF/UA compliance, enterprise add-ons provide PDF/UA export and a visual tag editor. This replaces manual remediation workflows that typically cost $50–200+ per document.
### Is this really the first open-source PDF auto-tagging tool?
Yes. Existing tools either depend on proprietary SDKs for writing structure tags, only output non-PDF formats (e.g., Docling outputs Markdown/JSON but cannot produce Tagged PDFs), or require manual intervention. OpenDataLoader is the first to do layout analysis → tag generation → Tagged PDF output entirely under an open-source license (Apache 2.0), with no proprietary dependency. Auto-tagging follows the PDF Association's Well-Tagged PDF specification and is validated using veraPDF, the industry-reference open-source PDF/A and PDF/UA validator.
### How do I convert existing PDFs to PDF/UA?
OpenDataLoader provides an end-to-end pipeline: audit existing PDFs for tags (`use_struct_tree=True`), auto-tag untagged PDFs into Tagged PDFs (`format="tagged-pdf"`, free under Apache 2.0), and export as PDF/UA-1 or PDF/UA-2 (enterprise add-on). Auto-tagging follows the PDF Association's Well-Tagged PDF specification and is validated using veraPDF. Auto-tagging generates the Tagged PDF; PDF/UA export is the final step. [Contact us](https://opendataloader.org/contact) for enterprise integration.
### How do I make my PDFs accessible for EAA compliance?
The European Accessibility Act requires accessible digital products by June 28, 2025. OpenDataLoader supports the full remediation workflow: audit → auto-tag → Tagged PDF → PDF/UA export. Auto-tagging follows the PDF Association's Well-Tagged PDF specification and is validated using veraPDF, ensuring standards-compliant output. Auto-tagging to Tagged PDF is open-source under Apache 2.0. PDF/UA export and accessibility studio are enterprise add-ons. See our [Accessibility Guide](https://opendataloader.org/docs/accessibility-compliance).
### Is OpenDataLoader PDF free?
The core library is **open-source under Apache 2.0** — free for commercial use. This includes all extraction features (text, tables, images, OCR, formulas, charts via hybrid mode), AI safety filters, Tagged PDF support, and auto-tagging to Tagged PDF. We are committed to keeping the core accessibility pipeline (layout analysis → auto-tagging → Tagged PDF) free and open-source. Enterprise add-ons (PDF/UA export, accessibility studio) are available for organizations needing end-to-end regulatory compliance.
### Why did the license change from MPL 2.0 to Apache 2.0?
MPL 2.0 requires file-level copyleft, which often triggers legal review before enterprise adoption. Apache 2.0 is fully permissive — no copyleft obligations, easier to integrate into commercial projects. If you are using a pre-2.0 version, it remains under MPL 2.0 and you can continue using it. Upgrading to 2.0+ means your project follows Apache 2.0 terms, which are strictly more permissive — no additional obligations, no action needed on your side.
## Documentation
- [Quick Start (Python)](https://opendataloader.org/docs/quick-start-python)
- [Quick Start (Node.js)](https://opendataloader.org/docs/quick-start-nodejs)
- [Quick Start (Java)](https://opendataloader.org/docs/quick-start-java)
- [JSON Schema Reference](https://opendataloader.org/docs/reference/json-schema)
- [CLI Options](https://opendataloader.org/docs/reference/cli-options)
- [Hybrid Mode Guide](https://opendataloader.org/docs/hybrid-mode)
- [Tagged PDF Support](https://opendataloader.org/docs/tagged-pdf)
- [AI Safety Features](https://opendataloader.org/docs/ai-safety)
- [PDF Accessibility](https://opendataloader.org/docs/accessibility-compliance)
## Contributing
We welcome contributions! See [CONTRIBUTING.md](CONTRIBUTING.md) for guidelines.
## License
[Apache License 2.0](LICENSE)
> **Note:** Versions prior to 2.0 are licensed under the [Mozilla Public License 2.0](https://www.mozilla.org/MPL/2.0/).
---
**Found this useful?** Give us a star to help others discover OpenDataLoader.
README.wehub.md
+7
View File
@@ -0,0 +1,7 @@
# WeHub 来源说明
- 原始项目：`opendataloader-project/opendataloader-pdf`
- 原始仓库：https://github.com/opendataloader-project/opendataloader-pdf
- 导入方式：上游默认分支的最新快照
- 原作者、版权和许可证信息以原始仓库及本仓库 LICENSE 为准
- 本文件仅用于记录来源，不代表 WeHub 是原项目作者
SUPPORT.md
+32
View File
@@ -0,0 +1,32 @@
# Support
This project uses GitHub Issues to track bugs and feature requests. Please search the existing
issues before filing new issues to avoid duplicates. For new issues, file your bug or
feature request as a new Issue.
For help and questions about using this project, please contact our team via Teams or tag us in the issues.
## AI-Powered Issue Processing
This project uses AI to automatically process GitHub issues through a three-stage workflow:
### How It Works
1. **Triage**: Validates your issue (checks for duplicates, spam, and project scope)
2. **Analyze**: Analyzes the codebase to understand the issue and determine the best approach
3. **Fix**: Automatically creates a PR for eligible issues
### What to Expect
After submitting an issue, you may see these labels:
| Label | Meaning |
|-------|---------|
| `fix/auto-eligible` | AI can automatically fix this issue |
| `fix/manual-required` | Requires human expert review |
| `fix/comment-only` | No code change needed; resolved via comment |
### Commands (CODEOWNERS only)
- `@ai-issue analyze` - Request re-analysis of an issue
- `@ai-issue fix` - Trigger automatic fix attempt
THIRD_PARTY/THIRD_PARTY_LICENSES.md
+323
View File
@@ -0,0 +1,323 @@
# THIRD-PARTY LICENSES
This project includes third-party libraries and components, licensed under their respective open source licenses.
Hancom, Inc. distributes the veraPDF components under the Mozilla Public License 2.0 (MPL-2.0), chosen from dual-licensed options.
| Component | Version | License | Download URL |
| :--- | :--- | :--- | :--- |
| @esbuild/aix-ppc64 | 0.25.12, 0.27.0 | MIT | https://github.com/evanw/esbuild |
| @esbuild/android-arm | 0.25.12, 0.27.0 | MIT | https://github.com/evanw/esbuild |
| @esbuild/android-arm64 | 0.25.12, 0.27.0 | MIT | https://github.com/evanw/esbuild |
| @esbuild/android-x64 | 0.25.12, 0.27.0 | MIT | https://github.com/evanw/esbuild |
| @esbuild/darwin-arm64 | 0.25.12, 0.27.0 | MIT | https://github.com/evanw/esbuild |
| @esbuild/darwin-x64 | 0.25.12, 0.27.0 | MIT | https://github.com/evanw/esbuild |
| @esbuild/freebsd-arm64 | 0.25.12, 0.27.0 | MIT | https://github.com/evanw/esbuild |
| @esbuild/freebsd-x64 | 0.25.12, 0.27.0 | MIT | https://github.com/evanw/esbuild |
| @esbuild/linux-arm | 0.25.12, 0.27.0 | MIT | https://github.com/evanw/esbuild |
| @esbuild/linux-arm64 | 0.25.12, 0.27.0 | MIT | https://github.com/evanw/esbuild |
| @esbuild/linux-ia32 | 0.25.12, 0.27.0 | MIT | https://github.com/evanw/esbuild |
| @esbuild/linux-loong64 | 0.25.12, 0.27.0 | MIT | https://github.com/evanw/esbuild |
| @esbuild/linux-mips64el | 0.25.12, 0.27.0 | MIT | https://github.com/evanw/esbuild |
| @esbuild/linux-ppc64 | 0.25.12, 0.27.0 | MIT | https://github.com/evanw/esbuild |
| @esbuild/linux-riscv64 | 0.25.12, 0.27.0 | MIT | https://github.com/evanw/esbuild |
| @esbuild/linux-s390x | 0.25.12, 0.27.0 | MIT | https://github.com/evanw/esbuild |
| @esbuild/linux-x64 | 0.25.12, 0.27.0 | MIT | https://github.com/evanw/esbuild |
| @esbuild/netbsd-arm64 | 0.25.12, 0.27.0 | MIT | https://github.com/evanw/esbuild |
| @esbuild/netbsd-x64 | 0.25.12, 0.27.0 | MIT | https://github.com/evanw/esbuild |
| @esbuild/openbsd-arm64 | 0.25.12, 0.27.0 | MIT | https://github.com/evanw/esbuild |
| @esbuild/openbsd-x64 | 0.25.12, 0.27.0 | MIT | https://github.com/evanw/esbuild |
| @esbuild/openharmony-arm64 | 0.25.12, 0.27.0 | MIT | https://github.com/evanw/esbuild |
| @esbuild/sunos-x64 | 0.25.12, 0.27.0 | MIT | https://github.com/evanw/esbuild |
| @esbuild/win32-arm64 | 0.25.12, 0.27.0 | MIT | https://github.com/evanw/esbuild |
| @esbuild/win32-ia32 | 0.25.12, 0.27.0 | MIT | https://github.com/evanw/esbuild |
| @esbuild/win32-x64 | 0.25.12, 0.27.0 | MIT | https://github.com/evanw/esbuild |
| @eslint-community/eslint-utils | 4.9.0, 4.9.1 | MIT | https://github.com/eslint-community/eslint-utils |
| @eslint-community/regexpp | 4.12.2 | MIT | https://github.com/eslint-community/regexpp |
| @eslint/config-array | 0.21.1 | Apache-2.0 | https://github.com/eslint/rewrite |
| @eslint/config-helpers | 0.4.2 | Apache-2.0 | https://github.com/eslint/rewrite/tree/main/packages/config-helpers |
| @eslint/core | 0.17.0 | Apache-2.0 | https://github.com/eslint/rewrite |
| @eslint/eslintrc | 3.3.3 | MIT | https://github.com/eslint/eslintrc |
| @eslint/js | 9.39.2 | MIT | https://eslint.org |
| @eslint/object-schema | 2.1.7 | Apache-2.0 | https://github.com/eslint/rewrite |
| @eslint/plugin-kit | 0.4.1 | Apache-2.0 | https://github.com/eslint/rewrite |
| @humanfs/core | 0.19.1 | Apache-2.0 | https://github.com/humanwhocodes/humanfs |
| @humanfs/node | 0.16.7 | Apache-2.0 | https://github.com/humanwhocodes/humanfs |
| @humanwhocodes/module-importer | 1.0.1 | Apache-2.0 | https://github.com/humanwhocodes/module-importer |
| @humanwhocodes/retry | 0.4.3 | Apache-2.0 | https://github.com/humanwhocodes/retrier |
| @isaacs/balanced-match | 4.0.1 | MIT | https://github.com/isaacs/balanced-match |
| @isaacs/brace-expansion | 5.0.1 | MIT | https://github.com/isaacs/brace-expansion |
| @isaacs/cliui | 8.0.2 | ISC | https://github.com/yargs/cliui |
| @jridgewell/gen-mapping | 0.3.13 | MIT | https://github.com/jridgewell/gen-mapping |
| @jridgewell/resolve-uri | 3.1.2 | MIT | https://github.com/jridgewell/resolve-uri |
| @jridgewell/sourcemap-codec | 1.5.5 | MIT | https://github.com/jridgewell/sourcemap-codec |
| @jridgewell/trace-mapping | 0.3.31 | MIT | https://github.com/jridgewell/trace-mapping |
| @pkgjs/parseargs | 0.11.0 | MIT | https://github.com/pkgjs/parseargs |
| @rollup/rollup-android-arm-eabi | 4.53.2 | MIT | https://rollupjs.org/ |
| @rollup/rollup-android-arm64 | 4.53.2 | MIT | https://rollupjs.org/ |
| @rollup/rollup-darwin-arm64 | 4.53.2 | MIT | https://rollupjs.org/ |
| @rollup/rollup-darwin-x64 | 4.53.2 | MIT | https://rollupjs.org/ |
| @rollup/rollup-freebsd-arm64 | 4.53.2 | MIT | https://rollupjs.org/ |
| @rollup/rollup-freebsd-x64 | 4.53.2 | MIT | https://rollupjs.org/ |
| @rollup/rollup-linux-arm-gnueabihf | 4.53.2 | MIT | https://rollupjs.org/ |
| @rollup/rollup-linux-arm-musleabihf | 4.53.2 | MIT | https://rollupjs.org/ |
| @rollup/rollup-linux-arm64-gnu | 4.53.2 | MIT | https://rollupjs.org/ |
| @rollup/rollup-linux-arm64-musl | 4.53.2 | MIT | https://rollupjs.org/ |
| @rollup/rollup-linux-loong64-gnu | 4.53.2 | MIT | https://rollupjs.org/ |
| @rollup/rollup-linux-ppc64-gnu | 4.53.2 | MIT | https://rollupjs.org/ |
| @rollup/rollup-linux-riscv64-gnu | 4.53.2 | MIT | https://rollupjs.org/ |
| @rollup/rollup-linux-riscv64-musl | 4.53.2 | MIT | https://rollupjs.org/ |
| @rollup/rollup-linux-s390x-gnu | 4.53.2 | MIT | https://rollupjs.org/ |
| @rollup/rollup-linux-x64-gnu | 4.53.2 | MIT | https://rollupjs.org/ |
| @rollup/rollup-linux-x64-musl | 4.53.2 | MIT | https://rollupjs.org/ |
| @rollup/rollup-openharmony-arm64 | 4.53.2 | MIT | https://rollupjs.org/ |
| @rollup/rollup-win32-arm64-msvc | 4.53.2 | MIT | https://rollupjs.org/ |
| @rollup/rollup-win32-ia32-msvc | 4.53.2 | MIT | https://rollupjs.org/ |
| @rollup/rollup-win32-x64-gnu | 4.53.2 | MIT | https://rollupjs.org/ |
| @rollup/rollup-win32-x64-msvc | 4.53.2 | MIT | https://rollupjs.org/ |
| @standard-schema/spec | 1.0.0 | MIT | https://github.com/standard-schema/standard-schema |
| @types/chai | 5.2.3 | MIT | https://github.com/DefinitelyTyped/DefinitelyTyped/tree/master/types/chai |
| @types/deep-eql | 4.0.2 | MIT | https://github.com/DefinitelyTyped/DefinitelyTyped/tree/master/types/deep-eql |
| @types/estree | 1.0.8 | MIT | https://github.com/DefinitelyTyped/DefinitelyTyped/tree/master/types/estree |
| @types/json-schema | 7.0.15 | MIT | https://github.com/DefinitelyTyped/DefinitelyTyped/tree/master/types/json-schema |
| @types/node | 25.2.0 | MIT | https://github.com/DefinitelyTyped/DefinitelyTyped/tree/master/types/node |
| @typescript-eslint/eslint-plugin | 8.54.0 | MIT | https://github.com/typescript-eslint/typescript-eslint |
| @typescript-eslint/parser | 8.54.0 | MIT | https://github.com/typescript-eslint/typescript-eslint |
| @typescript-eslint/project-service | 8.54.0 | MIT | https://typescript-eslint.io |
| @typescript-eslint/scope-manager | 8.54.0 | MIT | https://github.com/typescript-eslint/typescript-eslint |
| @typescript-eslint/tsconfig-utils | 8.54.0 | MIT | https://typescript-eslint.io |
| @typescript-eslint/type-utils | 8.54.0 | MIT | https://github.com/typescript-eslint/typescript-eslint |
| @typescript-eslint/types | 8.54.0 | MIT | https://github.com/typescript-eslint/typescript-eslint |
| @typescript-eslint/typescript-estree | 8.54.0 | MIT | https://github.com/typescript-eslint/typescript-eslint |
| @typescript-eslint/utils | 8.54.0 | MIT | https://github.com/typescript-eslint/typescript-eslint |
| @typescript-eslint/visitor-keys | 8.54.0 | MIT | https://github.com/typescript-eslint/typescript-eslint |
| @vitest/expect | 4.0.18 | MIT | https://github.com/vitest-dev/vitest |
| @vitest/mocker | 4.0.18 | MIT | https://github.com/vitest-dev/vitest/tree/main/packages/mocker |
| @vitest/pretty-format | 4.0.18 | MIT | https://github.com/vitest-dev/vitest/tree/main/packages/utils |
| @vitest/runner | 4.0.18 | MIT | https://github.com/vitest-dev/vitest |
| @vitest/snapshot | 4.0.18 | MIT | https://github.com/vitest-dev/vitest |
| @vitest/spy | 4.0.18 | MIT | https://github.com/vitest-dev/vitest |
| @vitest/utils | 4.0.18 | MIT | https://github.com/vitest-dev/vitest |
| Acorn | 8.15.0 | MIT | https://github.com/ternjs/Acorn |
| Acorn-JSX | 5.3.2 | MIT | https://github.com/RReverser/Acorn-JSX |
| ajv | 6.12.6 | MIT | https://github.com/ajv-validator/ajv.git |
| annotated-types | 0.7.0 | MIT | https://github.com/annotated-types/annotated-types |
| ansi-regex | 6.2.2 | MIT | https://github.com/sindresorhus/ansi-regex |
| ansi-styles | v4.3.0, 6.2.3 | MIT | https://github.com/sindresorhus/ansi-styles |
| any-promise | 1.3.0 | MIT | http://github.com/kevinbeaty/any-promise |
| anyio | 4.9.0 | MIT | https://pypi.org/project/anyio/ |
| Apache Commons Logging | 1.3.4 | Apache-2.0 | http://commons.apache.org/proper/commons-logging/ |
| Apache PDFBox | 3.0.4 | Apache-2.0 | https://pdfbox.apache.org/ |
| Apache PDFBox io | 3.0.4 | Apache-2.0 | https://repo1.maven.org/maven2/org/apache/pdfbox/pdfbox-io/ |
| API Guardian | 1.1.2 | Apache-2.0 | https://repo1.maven.org/maven2/org/apiguardian/apiguardian-api/1.1.2/ |
| assertion-error | 2.0.1 | MIT | https://github.com/chaijs/assertion-error |
| AssertJ - Fluent Assertions for Java | 3.27.7 | Apache-2.0 | https://assertj.github.io/doc/ |
| balanced-match | 1.0.2 | MIT | https://github.com/juliangruber/balanced-match |
| brace-expansion | 1.1.12, 2.0.2 | MIT | https://github.com/juliangruber/brace-expansion |
| bundle-require | 5.1.0 | MIT | https://www.npmjs.com/package/bundle-require |
| Byte Buddy | 1.18.3 | Apache-2.0 | http://bytebuddy.net |
| cac | 6.7.14 | MIT | https://github.com/egoist/cac |
| callsites | 3.1.0 | MIT | https://github.com/sindresorhus/callsites |
| Chai | 6.2.1 | MIT | http://chaijs.com/ |
| Chalk | 4.1.2 | MIT | https://github.com/sindresorhus/chalk |
| chokidar | 4.0.3 | MIT | https://github.com/paulmillr/chokidar |
| color-name | 1.1.4 | MIT | https://github.com/colorjs/color-name |
| com.sun.xml.bind:jaxb-impl | 2.3.2 | EDL-1.0 | https://mvnrepository.com/artifact/com.sun.xml.bind/jaxb-impl |
| commander | 14.0.3 | MIT | https://github.com/tj/commander.js |
| Commander.js | 4.1.1 | MIT | https://github.com/tj/commander.js |
| commons-cli | 1.10.0 | Apache-2.0 | http://commons.apache.org/cli/ |
| confbox | 0.1.8 | MIT | https://github.com/unjs/confbox |
| consola | 3.4.2 | MIT | https://github.com/nuxt/consola |
| debug-js/debug | 4.4.3 | MIT | https://github.com/debug-js/debug |
| deep-is | 0.1.4 | MIT | https://github.com/thlorenz/deep-is |
| eastasianwidth | 0.2.0 | MIT | https://github.com/komagata/eastasianwidth |
| emoji-regex | 9.2.2 | MIT | https://github.com/mathiasbynens/emoji-regex |
| es-module-lexer | 1.7.0 | MIT | https://github.com/guybedford/es-module-lexer |
| esbuild | 0.25.12, 0.27.0 | MIT | https://esbuild.github.io/ |
| escape-string-regexp | v4.0.0 | MIT | https://github.com/sindresorhus/escape-string-regexp |
| ESLint | 9.39.2 | MIT | http://eslint.org/ |
| eslint-scope | 8.4.0 | BSD-2-Clause | https://github.com/eslint/eslint-scope |
| eslint-visitor-keys | 3.4.3, 4.2.1 | Apache-2.0 | https://github.com/eslint/eslint-visitor-keys |
| espree | 10.4.0 | BSD-2-Clause | https://github.com/eslint/espree |
| esquery | 1.6.0 | BSD-3-Clause | https://github.com/jrfeenst/esquery |
| esrecurse | v4.3.0 | BSD-2-Clause | https://github.com/estools/esrecurse |
| estraverse | 5.3.0 | BSD-2-Clause | https://github.com/Constellation/estraverse |
| estree-walker | 3.0.3 | MIT | https://github.com/Rich-Harris/estree-walker |
| esutils | 2.0.3 | BSD-2-Clause | https://github.com/Constellation/esutils |
| expect-type | 1.2.2 | Apache-2.0 | https://github.com/mmkal/ts/tree/master/packages/expect-type |
| fast-deep-equal | v3.1.3 | MIT | https://github.com/epoberezkin/fast-deep-equal |
| fast-json-stable-stringify | 2.1.0 | MIT | https://github.com/epoberezkin/fast-json-stable-stringify |
| fast-levenshtein | 2.0.6 | MIT | https://github.com/hiddentao/fast-levenshtein |
| fdir | 6.5.0 | MIT | https://github.com/thecodrr/fdir |
| file-entry-cache | 8.0.0 | MIT | https://github.com/royriojas/file-entry-cache |
| find-up | v5.0.0 | MIT | https://github.com/sindresorhus/find-up |
| fix-dts-default-cjs-exports | 1.0.1 | MIT | https://github.com/userquin/fix-dts-default-cjs-exports |
| flat-cache | 4.0.1 | MIT | https://github.com/royriojas/flat-cache |
| flatted | 3.3.3 | ISC | https://github.com/WebReflection/flatted |
| foreground-child | 3.3.1 | ISC | https://github.com/isaacs/foreground-child |
| fsevents | 2.3.3 | MIT | https://github.com/fsevents/fsevents |
| get-tsconfig | 4.13.0 | MIT | https://github.com/typeslick/get-tsconfig |
| glob | 13.0.1 | Blue Oak 1.0.0 | https://github.com/isaacs/node-glob |
| glob-parent | 6.0.2 | ISC | https://github.com/es128/glob-parent |
| h11 | 0.16.0 | MIT | https://github.com/njsmith/h11 |
| Hamcrest | 1.3 | BSD-3-Clause | http://hamcrest.org/ |
| has-flag | 4.0.0 | MIT | https://github.com/sindresorhus/has-flag |
| httpcore | 1.0.9 | BSD-3-Clause | https://github.com/encode/httpcore |
| httpx | 0.27.2 | BSD-3-Clause | https://www.python-httpx.org/ |
| idna | 3.10 | BSD-3-Clause | https://github.com/kjd/idna |
| import-fresh | 3.3.1 | MIT | https://github.com/sindresorhus/import-fresh |
| imurmurhash | 0.1.4 | MIT | https://github.com/jensyt/imurmurhash-js |
| is-extglob | 2.1.1 | MIT | https://github.com/jonschlinkert/is-extglob |
| is-glob | 4.0.3 | MIT | https://www.npmjs.com/package/is-glob |
| isaacs/jackspeak | 3.4.3 | Blue Oak 1.0.0 | https://github.com/isaacs/jackspeak |
| isexe | 2.0.0 | ISC | https://github.com/isaacs/isexe |
| jackson-annotations | 2.15.0 | Apache-2.0 | https://github.com/FasterXML/jackson-annotations |
| jackson-core | 2.15.0 | Apache-2.0 | https://github.com/FasterXML/jackson-core |
| jackson-databind | 2.15.0 | Apache-2.0 | https://github.com/FasterXML/jackson-databind |
| Jakarta Activation API | 1.2.0 | CDDL-1.1 | https://eclipse-ee4j.github.io/jaf/ |
| Java Advanced Imaging Image I/O Tools API core (standalone) | 1.4.0 | BSD-3-Clause | https://github.com/jai-imageio/jai-imageio-core |
| JAXB CORE | 2.3.0.1 | CDDL-1.1 | http://jaxb.java.net/ |
| jaxb-api | 2.4.0-b180830.0359 | CDDL-1.1 | https://jakarta.ee/specifications/xml-binding |
| Jetbrains annotations | 13.0 | Apache-2.0 | http://www.jetbrains.org |
| joycon | 3.1.1 | MIT | https://github.com/egoist/joycon |
| JPEG2000 support for Java Advanced Imaging Image I/O Tools API | 1.3.0 | Sun BSD | https://repo1.maven.org/maven2/com/github/jai-imageio/jai-imageio-jpeg2000/ |
| js-yaml | 4.1.1 | MIT | https://github.com/nodeca/js-yaml |
| json-buffer | 3.0.1 | MIT | https://github.com/dominictarr/json-buffer |
| json-schema-traverse | 0.4.1 | MIT | https://github.com/epoberezkin/json-schema-traverse |
| JUnit | 4.13.2 | EPL-1.0 | https://junit.org/junit5/ |
| JUnit Jupiter (Aggregator) | 5.14.2 | EPL-2.0 | https://junit.org/ |
| keyv | 4.5.4 | MIT | https://github.com/lukechilds/keyv |
| Kotlin | 1.8.21 | Apache-2.0 | http://kotlin.jetbrains.org |
| kotlin-stdlib-common | 1.9.10 | Apache-2.0 | https://kotlinlang.org/ |
| langchain | 0.3.80 | MIT | https://github.com/langchain-ai/langchain |
| langchain-text-splitters | 0.3.9 | MIT | https://github.com/langchain-ai/langchain |
| langsmith | 0.3.45 | MIT | https://smith.langchain.com/ |
| levn | 0.4.1 | MIT | https://github.com/gkz/levn |
| libcspice-sys | 0.1.1 | MIT | https://crates.io/crates/libcspice-sys |
| lilconfig | 3.1.3 | MIT | https://github.com/antonk52/lilconfig |
| lines-and-columns | 1.2.4 | MIT | https://github.com/eventualbuddha/lines-and-columns |
| load-tsconfig | 0.2.5 | MIT | https://www.npmjs.com/package/load-tsconfig |
| locate-path | v6.0.0 | MIT | https://github.com/sindresorhus/locate-path |
| lodash.merge | 4.6.2 | MIT | https://lodash.com/ |
| mafintosh/why-is-node-running | 2.3.0 | MIT | https://github.com/mafintosh/why-is-node-running |
| magic-string | 0.30.21 | MIT | https://github.com/rich-harris/magic-string |
| mdBook | 0.4.36 | MPL-2.0 | https://github.com/rust-lang/mdBook |
| minimatch | 3.1.2, 9.0.5 | ISC | https://github.com/isaacs/minimatch |
| minimatch | 10.1.2 | Blue Oak 1.0.0 | https://github.com/isaacs/minimatch |
| minipass | 7.1.2 | ISC | https://github.com/isaacs/minipass |
| mlly | 1.8.0 | MIT | https://github.com/unjs/mlly |
| MockWebServer | 4.12.0 | Apache-2.0 | https://github.com/square/okhttp/ |
| moxystudio/node-cross-spawn | 7.0.6 | MIT | https://github.com/moxystudio/node-cross-spawn |
| Mozilla Rhino | 1.7.14.1 | MPL-2.0 | http://www.mozilla.org/rhino/ |
| ms.js | 2.1.3 | MIT | https://github.com/guille/ms.js |
| mz | 2.7.0 | MIT | https://github.com/normalize/mz |
| nanoid | 3.3.11 | MIT | https://github.com/ai/nanoid |
| natural-compare | 1.4.0 | MIT | https://github.com/litejs/natural-compare |
| nobody | 2.1.1 | MIT | https://github.com/debug-js/debug |
| node-concat-map | 0.0.1 | MIT | https://github.com/substack/node-concat-map |
| node-glob | 10.5.0 | ISC | http://github.com/isaacs/node-glob |
| node-ignore | 5.3.2, 7.0.5 | MIT | https://github.com/kaelzhang/node-ignore |
| node-lru-cache | 10.4.3, 11.2.2 | ISC | http://github.com/isaacs/node-lru-cache |
| node-semver | 7.7.3 | ISC | https://github.com/npm/node-semver |
| nodeca-argparse | 2.0.1 | PSF-2.0 | https://github.com/nodeca/argparse |
| object-assign | 4.1.1 | MIT | https://github.com/sindresorhus/object-assign |
| OkHttp | 4.12.0 | Apache-2.0 | https://github.com/square/okhttp |
| OkIO | 3.6.0 | Apache-2.0 | https://square.github.io/okio/ |
| optionator | 0.9.4 | MIT | https://github.com/gkz/optionator |
| org.apiguardian:apiguardian-api | 1.1.2 | Apache-2.0 | https://github.com/apiguardian-team/apiguardian |
| org.jetbrains.kotlin:kotlin-stdlib-jdk7 | 1.8.21 | Apache-2.0 | https://kotlinlang.org/ |
| org.jetbrains.kotlin:kotlin-stdlib-jdk8 | 1.8.21 | Apache-2.0 | https://kotlinlang.org/ |
| org.junit.jupiter:junit-jupiter-api | 5.14.2 | EPL-2.0 | https://junit.org/ |
| org.junit.jupiter:junit-jupiter-engine | 5.14.2 | EPL-2.0 | https://junit.org/ |
| org.junit.jupiter:junit-jupiter-params | 5.14.2 | EPL-2.0 | https://junit.org/ |
| org.junit.platform:junit-platform-commons | 1.14.2 | EPL-2.0 | https://junit.org/ |
| org.junit.platform:junit-platform-engine | 1.14.2 | EPL-2.0 | https://junit.org/ |
| org.opentest4j:opentest4j | 1.3.0 | Apache-2.0 | https://github.com/ota4j-team/opentest4j |
| org.verapdf:core | 1.29.56 | MPL-2.0 | https://repo1.maven.org/maven2/org/verapdf/ |
| org.verapdf:metadata-fixer | 1.29.194 | MPL-2.0 | https://repo1.maven.org/maven2/org/verapdf/ |
| org.verapdf:parser | 1.29.64 | MPL-2.0 | https://repo1.maven.org/maven2/org/verapdf/ |
| org.verapdf:pdf-model | 1.29.12 | MPL-2.0 | https://repo1.maven.org/maven2/org/verapdf/ |
| org.verapdf:validation-model | 1.29.194 | MPL-2.0 | https://repo1.maven.org/maven2/org/verapdf/ |
| org.verapdf:verapdf-xmp-core | 1.29.56 | MPL-2.0 | https://repo1.maven.org/maven2/org/verapdf/ |
| org.verapdf:wcag-algorithms | 1.29.43 | MPL-2.0 | https://repo1.maven.org/maven2/org/verapdf/ |
| org.verapdf:wcag-validation | 1.29.194 | MPL-2.0 | https://repo1.maven.org/maven2/org/verapdf/ |
| orjson | 3.10.15 | Apache-2.0 | https://github.com/ijl/orjson |
| p-limit | 3.1.0 | MIT | https://github.com/sindresorhus/p-limit |
| p-locate | v5.0.0 | MIT | https://github.com/sindresorhus/p-locate |
| package-json-from-dist | 1.0.1 | Blue Oak 1.0.0 | https://github.com/isaacs/package-json-from-dist |
| Packaging | 24.2 | BSD-2-Clause | https://github.com/pypa/packaging |
| parent-module | 1.0.1 | MIT | https://github.com/sindresorhus/parent-module |
| path-exists | 4.0.0 | MIT | https://github.com/sindresorhus/path-exists |
| path-key | 3.1.1 | MIT | https://github.com/sindresorhus/path-key |
| path-scurry | 1.11.1, 2.0.1 | Blue Oak 1.0.0 | https://github.com/isaacs/path-walker |
| pathe | 2.0.3 | MIT | https://github.com/unjs/pathe |
| PDFBox JBIG2 ImageIO plugin | 3.0.3 | Apache-2.0 | https://www.apache.org/jbig2-imageio/ |
| picocolors | 1.1.1 | ISC | https://github.com/alexeyraspopov/picocolors |
| picomatch | 4.0.3 | MIT | https://github.com/micromatch/picomatch |
| pirates | 4.0.7 | MIT | https://github.com/ariporad/pirates |
| pkg-types | 1.3.1 | MIT | https://github.com/unjs/pkg-types |
| PostCSS | 8.5.6 | MIT | http://postcss.org/ |
| postcss-load-config | 6.0.1 | MIT | https://github.com/michael-ciniawsky/postcss-load-config |
| prelude-ls | 1.2.1 | MIT | https://github.com/gkz/prelude-ls |
| Prettier IO | 3.8.1 | MIT | https://prettier.io |
| psf-requests | 2.32.5 | Apache-2.0 | http://docs.python-requests.org |
| Punycode.js | 2.3.1 | MIT | http://mths.be/punycode |
| pydantic | 2.10.6 | MIT | https://pydantic-docs.helpmanual.io/ |
| pydantic-core | 2.27.2 | MIT | https://github.com/pydantic/pydantic-core |
| python-certifi | 2024.12.14 | MPL-2.0 | https://certifiio.readthedocs.io/en/latest/ |
| python-json-patch | 1.33 | BSD-3-Clause | https://github.com/stefankoegl/python-json-patch/ |
| python-json-pointer | 3.0.0 | BSD-3-Clause | https://github.com/stefankoegl/python-json-pointer |
| python-typing-extensions | 4.15.0 | PSF-2.0 | https://tracker.debian.org/pkg/python-typing-extensions |
| python3-charset-normalizer | 3.4.1 | MIT | https://github.com/ousret/charset_normalizer |
| PyYAML | 6.0.2 | MIT | https://pyyaml.org/ |
| Qix-/color-convert | 2.0.1 | MIT | https://github.com/Qix-/color-convert |
| readdirp | 4.1.2 | MIT | https://github.com/thlorenz/readdirp |
| requests-toolbelt | 1.0.0 | Apache-2.0 | https://toolbelt.readthedocs.io |
| resolve-from | 4.0.0, 5.0.0 | MIT | https://github.com/sindresorhus/resolve-from |
| resolve-pkg-maps | 1.0.0 | MIT | https://github.com/privatenumber/resolve-pkg-maps |
| rollup/rollup | 4.53.2 | MIT | https://github.com/rollup/rollup |
| Saxon XSLT and XQuery Processor | 12.8 | MPL-2.0 | http://saxon.sourceforge.net |
| shebang-command | 2.0.0 | MIT | https://github.com/kevva/shebang-command |
| shebang-regex | 3.0.0 | MIT | https://github.com/sindresorhus/shebang-regex |
| siginfo | 2.0.0 | ISC | https://github.com/emilbayes/siginfo |
| sindresorhus/globals | 14.0.0 | MIT | https://github.com/sindresorhus/globals |
| sindresorhus/supports-color | v7.2.0 | MIT | https://github.com/sindresorhus/supports-color |
| sniffio | 1.3.1 | Apache-2.0 | https://github.com/python-trio/sniffio |
| source-map | 0.7.6 | BSD-3-Clause | https://github.com/mozilla/source-map |
| source-map-js | 1.2.1 | BSD-3-Clause | https://github.com/7rulnik/source-map |
| stable-stringify | 1.0.1 | MIT | https://github.com/samn/json-stable-stringify |
| stackback | 0.0.2 | MIT | https://github.com/defunctzombie/node-stackback |
| StAX Utilities Project | 20070216 | BSD-3-Clause | http://java.net/projects/stax-utils/ |
| std-env | 3.10.0 | MIT | https://github.com/pi0/std-env |
| string-width | 4.2.3, 5.1.2 | MIT | https://github.com/sindresorhus/string-width |
| Strip ANSI | 6.0.1, 7.1.2 | MIT | https://github.com/chalk/strip-ansi |
| strip-json-comments | 3.1.1 | MIT | https://github.com/sindresorhus/strip-json-comments |
| sucrase | 3.35.0 | MIT | https://github.com/decaffeinate/bulk-decaffeinate |
| tapjs/signal-exit | 4.1.0 | ISC | https://github.com/tapjs/signal-exit |
| tenacity | 8.5.0 | Apache-2.0 | https://github.com/jd/tenacity |
| thenify | 3.3.1 | MIT | https://github.com/thenables/thenify |
| thenify-all | 1.6.0 | MIT | https://github.com/thenables/thenify-all |
| tinybench | 2.9.0 | MIT | https://github.com/tinylibs/tinybench |
| tinyexec | 0.3.2, 1.0.2 | MIT | https://github.com/tinylibs/tinyexec |
| tinyglobby | 0.2.15 | MIT | https://github.com/SuperchupuDev/tinyglobby |
| tinyrainbow | 3.0.3 | MIT | https://github.com/tinylibs/tinyrainbow |
| tree-kill | v1.2.2 | MIT | https://github.com/pkrumins/node-tree-kill |
| ts-api-utils | 2.4.0 | MIT | https://github.com/JoshuaKGoldberg/ts-api-utils |
| ts-interface-checker | 0.1.13 | Apache-2.0 | https://github.com/gristlabs/ts-interface-checker |
| tsup | 8.5.1 | MIT | https://github.com/egoist/tsup |
| tsx | 4.20.5 | MIT | https://github.com/basarat/tsx |
| type-check | 0.4.0 | MIT | https://github.com/gkz/type-check |
| TypeScript | 5.9.3 | Apache-2.0 | http://www.typescriptlang.org/ |
| ufo | 1.6.1 | MIT | https://github.com/nuxt-contrib/ufo |
| undici-types | 7.16.0 | MIT | https://undici.nodejs.org |
| upstage/dp-bench | - | MIT | https://huggingface.co/datasets/upstage/dp-bench |
| uri-js | 4.4.1 | BSD-2-Clause | https://github.com/garycourt/uri-js |
| urllib3 | 2.6.1 | MIT | https://urllib3.readthedocs.io/en/stable |
| vitejs | 7.3.1 | MIT | http://vitejs.dev/ |
| vitest | 4.0.18 | MIT | https://github.com/vitest-dev/vitest |
| which | 2.0.2 | ISC | https://github.com/isaacs/node-which |
| word-wrap | 1.2.5 | MIT | https://github.com/jonschlinkert |
| wrap-ansi | v7.0.0, 8.1.0 | MIT | https://github.com/chalk/wrap-ansi |
| XML Resolver | 5.3.3 | Apache-2.0 | https://github.com/ndw/xmlresolver |
| yocto-queue | 0.1.0 | MIT | https://github.com/sindresorhus/yocto-queue |
| zstandard | 0.23.0 | BSD-3-Clause | https://github.com/indygreg/python-zstandard |
THIRD_PARTY/THIRD_PARTY_NOTICES.md
+198
View File
@@ -0,0 +1,198 @@
# THIRD-PARTY NOTICES (Copyright & Attributions)
Copyright © 2025-2026 Hancom, Inc. All rights reserved.
Below are copyright and notice texts for third-party libraries and components used in this project.
Full license texts are provided in the `licenses/` directory.
See also: [THIRD_PARTY_LICENSES](./THIRD_PARTY_LICENSES.md) for details.
---
##### Apache Software Foundation notices
The following components include software developed at The Apache Software Foundation (https://www.apache.org/).
- Apache Commons Logging
- Apache PDFBox (FontBox, PDFBox, PDFBox-IO, JBIG2 ImageIO plugin)
- Apache Commons CLI
- Apache Maven (Artifact, Plugin API, Reporting API, Shared, Wagon etc.)
- Apache Maven Doxia
- XML Resolver
"This product includes software developed at The Apache Software Foundation (https://www.apache.org/)."
---
##### Component Attributions
- `@esbuild/android-arm` (0.25.12, 0.27.0): Copyright (c) 2018, 2021 The Go Authors. All rights reserved.
- `@esbuild/android-x64` (0.25.12, 0.27.0): Copyright (c) 2018, 2021 The Go Authors. All rights reserved.
- `@esbuild/openharmony-arm64` (0.25.12, 0.27.0): Copyright (c) 2018, 2021 The Go Authors. All rights reserved.
- `@eslint-community/eslint-utils` (4.9.0, 4.9.1): Copyright (c) 2018 Toru Nagashima
- `@eslint-community/regexpp` (4.12.2): Copyright (c) 2018 Toru Nagashima
- `@eslint/config-array` (0.21.1): Copyright (c) 2018-2025 the Deno authors.
- `@eslint/eslintrc` (3.3.3): Copyright (c) 2015-2017 Evgeny Poberezkin
- `@humanwhocodes/retry` (0.4.3): Copyright (c) 2011-2023 Isaac Z. Schlueter, Ben Noordhuis, and Contributors
- `@isaacs/cliui` (8.0.2): Copyright (c) 2015 Contributors
- `@jridgewell/gen-mapping` (0.3.13): Copyright (c) 2024 Justin Ridgewell
- `@jridgewell/resolve-uri` (3.1.2): Copyright (c) 2019 Justin Ridgewell
- `@jridgewell/sourcemap-codec` (1.5.5): Copyright (c) 2024 Justin Ridgewell
- `@jridgewell/trace-mapping` (0.3.31): Copyright (c) 2024 Justin Ridgewell
- `@standard-schema/spec` (1.0.0): Copyright (c) 2024 Colin McDonnell
- `@typescript-eslint/eslint-plugin` (8.54.0): Copyright (c) 2019 typescript-eslint and other contributors
- `@typescript-eslint/parser` (8.54.0): Copyright (c) 2019 typescript-eslint and other contributors
- `@typescript-eslint/project-service` (8.54.0): Copyright (c) 2025 typescript-eslint and other contributors
- `@typescript-eslint/scope-manager` (8.54.0): Copyright (c) 2019 typescript-eslint and other contributors
- `@typescript-eslint/tsconfig-utils` (8.54.0): Copyright (c) 2025 typescript-eslint and other contributors
- `@typescript-eslint/type-utils` (8.54.0): Copyright (c) 2021 typescript-eslint and other contributors
- `@typescript-eslint/types` (8.54.0): Copyright (c) 2019 typescript-eslint and other contributors
- `@typescript-eslint/typescript-estree` (8.54.0): Copyright (c) 2019 typescript-eslint and other contributors
- `@typescript-eslint/utils` (8.54.0): Copyright (c) 2019 typescript-eslint and other contributors
- `@typescript-eslint/visitor-keys` (8.54.0): Copyright (c) 2019 typescript-eslint and other contributors
- `@vitest/expect` (4.0.18): Copyright (c) 2021-Present VoidZero Inc. and Vitest contributors
- `@vitest/mocker` (4.0.18): Copyright (c) 2021-Present VoidZero Inc. and Vitest contributors
- `@vitest/pretty-format` (4.0.18): Copyright (c) 2021-Present VoidZero Inc. and Vitest contributors
- `@vitest/runner` (4.0.18): Copyright (c) 2021-Present VoidZero Inc. and Vitest contributors
- `@vitest/snapshot` (4.0.18): Copyright (c) 2021-Present VoidZero Inc. and Vitest contributors
- `@vitest/spy` (4.0.18): Copyright (c) 2021-Present VoidZero Inc. and Vitest contributors
- `@vitest/utils` (4.0.18): Copyright (c) 2021-Present VoidZero Inc. and Vitest contributors, Copyright (c) 2014-2023 Simon Lydell, Copyright (c) 2018 The diff-match-patch Authors., Copyright (c) 2013 Jake Luer
- `Acorn` (8.15.0): Copyright (c) 2012-2022 by various contributors (see AUTHORS)
- `Acorn-JSX` (5.3.2): Copyright (c) 2012-2017 by Ingvar Stepanyan
- `ajv` (6.12.6): Copyright (c) 2011 Gary Court, Copyright (c) 2015-2017 Evgeny Poberezkin
- `annotated-types` (0.7.0): Copyright (c) 2022 the contributors
- `any-promise` (1.3.0): Copyright (c) 2014-2016 Kevin Beaty
- `anyio` (4.9.0): Copyright (c) 2018 Alex Grönholm
- `Apache Commons Logging` (1.3.4): Copyright (c) 1989-2024 Free Software Foundation and The Apache Software Foundation, Copyright (c) 2013-2022 Oracle and/or its affiliates.
- `Apache PDFBox` (3.0.4): Copyright (c) 1990-2024 Adobe Systems Incorporated, www.pdfbox.org, Harald Kuhr, Google Corporation, Red Hat Inc., Unicode Inc., The Apache Software Foundation, GitHub Inc.
- `Apache PDFBox FontBox` (3.0.4): Copyright (c) 2006-2024 www.fontbox.org, The Apache Software Foundation, Grzegorz Luk, Lohit Fonts Project, Unicode Inc.
- `Apache PDFBox io` (3.0.4): Copyright (c) 2002-2024 The Apache Software Foundation
- `assertion-error` (2.0.1): Copyright (c) 2013 Jake Luer
- `AssertJ - Fluent Assertions for Java` (3.27.7): Copyright (c) 1989-2026 Free Software Foundation, Bitstream Inc., Tavmjong Bah, Oracle and/or its affiliates, the original author or authors.
- `balanced-match` (1.0.2): Copyright (c) 2013 Julian Gruber
- `brace-expansion` (1.1.12, 2.0.2): Copyright (c) 2013 Julian Gruber
- `bundle-require` (5.1.0): Copyright (c) 2021 EGOIST
- `Byte Buddy` (1.18.3): Copyright (c) 2000-2011 INRIA, France Telecom, Copyright (c) 2014-Present Rafael Winterhalter
- `Chai` (6.2.1): Copyright (c) 2011-2017 Chai.js Assertion Library, Jake Luer, Sakthipriyan Vairamani
- `chokidar` (4.0.3): Copyright (c) 2012-2019 Paul Miller, Elan Shanker
- `color-name` (1.1.4): Copyright (c) 2015 Dmitry Ivanov
- `com.sun.xml.bind:jaxb-impl` (2.3.2): Copyright (c) 1995-2018 Jean-loup Gailly, Mark Adler, Stuart Knightley, David Duponchel, Vitaly Puzrin, Andrey Tupitsin, Oracle, jQuery Foundation
- `Commander.js` (4.1.1): Copyright (c) 2011 TJ Holowaychuk
- `commons-cli` (1.10.0): Copyright (c) 1989-2025 Free Software Foundation, Oracle, The Apache Software Foundation
- `confbox` (0.1.8): Copyright (c) 2011-2018 Vitaly Puzrin, Aseem Kishore
- `debug-js/debug` (4.4.3): Copyright (c) 2014-2021 TJ Holowaychuk, Josh Junon
- `deep-is` (0.1.4): Copyright (c) 2009-2013 Thomas Robinson, James Halliday, Thorsten Lorenz
- `es-module-lexer` (1.7.0): Copyright (c) 2012-2022 by various contributors, Guy Bedford
- `esbuild` (0.25.12, 0.27.0): Copyright (c) 2020 Evan Wallace
- `ESLint` (9.39.2): Copyright (c) 2013 Joel Feenstra
- `eslint-scope` (8.4.0): Copyright (c) 2012-2015 Yusuke Suzuki, Alex Seville, Thiago de Arruda
- `espree` (10.4.0): Copyright (c) 2012-2015 Acorn Contributors, Sebastian McKenzie
- `esquery` (1.6.0): Copyright (c) 2012-2013 Ariya Hidayat, Yusuke Suzuki, Joel Feenstra
- `esrecurse` (v4.3.0): Copyright (c) 2014 Yusuke Suzuki
- `estraverse` (5.3.0): Copyright (c) 2012-2016 Ariya Hidayat, Yusuke Suzuki
- `esutils` (2.0.3): Copyright (c) 2013-2014 Yusuke Suzuki, Ivan Nikulin
- `expect-type` (1.2.2): Copyright (c) 2024 Misha Kaletsky
- `fast-deep-equal` (v3.1.3): Copyright (c) 2017 Evgeny Poberezkin
- `fdir` (6.5.0): Copyright (c) 2023-2024 Abdullah Atta
- `fix-dts-default-cjs-exports` (1.0.1): Copyright (c) 2025-Present Joaquin
- `flatted` (3.3.3): Copyright (c) 2018-2025 Andrea Giammarchi
- `foreground-child` (3.3.1): Copyright (c) 2015-2023 Isaac Z. Schlueter and Contributors
- `fsevents` (2.3.3): Copyright (c) 2010-2020 Philipp Dunkel, Ben Noordhuis, Elan Shankar, Paul Miller
- `glob-parent` (6.0.2): Copyright (c) 2015-2021 Elan Shanker, Blaine Bublitz, Eric Schoffstall
- `h11` (0.16.0): Copyright (c) 2006-2016 Jonathan E. Taylor, Scipy Developers, Statsmodels Developers, Nathaniel J. Smith, Chris Wanstrath
- `Hamcrest` (1.3): Copyright (c) 2000-2010 hamcrest.org
- `httpcore` (1.0.9): Copyright (c) 2020 Encode OSS Ltd.
- `httpx` (0.27.2): Copyright (c) 2019 Encode OSS Ltd.
- `idna` (3.10): Copyright (c) 2013-2024 Kim Davies and contributors.
- `imurmurhash` (0.1.4): Copyright (c) 2013 Gary Court, Jens Taylor
- `is-extglob` (2.1.1): Copyright (c) 2014-2016 Jon Schlinkert
- `is-glob` (4.0.3): Copyright (c) 2014-2017 Jon Schlinkert
- `jackson-annotations` (2.15.0): Copyright (c) 2007 Tatu Saloranta
- `jackson-core` (2.15.0): Copyright (c) 2007-2020 Tatu Saloranta, Raffaello Giulietti
- `jackson-databind` (2.15.0): Copyright (c) 2007-2011 Tatu Saloranta, Google Inc.
- `Jakarta Activation API` (1.2.0): Copyright (c) 1989-2017 Free Software Foundation, Oracle
- `Java Advanced Imaging Image I/O Tools API core` (1.4.0): Copyright (c) 1990-2018 Wang Labs Inc., Sun Microsystems, Stian Soiland-Reyes, University of Manchester, Butch Howard, Mark Carroll, Peter Hull, Robin Stevens, Yannick De Turck, Luca Bellonda, Curtis Rueden, Ghislain Bonamy, Mykola Pavluchynskyi, Roger Leigh, Sebastien Besson, Peter Jodeleit
- `JAXB CORE` (2.3.0.1): Copyright (c) 1997-2018 Stuart Knightley, David Duponchel, Oracle, jQuery Foundation
- `jaxb-api` (2.4.0-b180830.0359): Copyright (c) 1989-2018 Free Software Foundation, Stuart Knightley, David Duponchel, Oracle, jQuery Foundation
- `Jetbrains annotations` (13.0): Copyright (c) 2000-2013 JetBrains s.r.o., Sascha Weinreuter
- `JPEG2000 support for Java Advanced Imaging Image I/O Tools API` (1.3.0): Copyright (c) 1999-2006 JJ2000 Partners, Sun Microsystems
- `js-yaml` (4.1.1): Copyright (c) 2011-2015 Vitaly Puzrin
- `json-buffer` (3.0.1): Copyright (c) 2013 Dominic Tarr
- `json-schema-traverse` (0.4.1): Copyright (c) 2017 Evgeny Poberezkin
- `JUnit Jupiter (Aggregator)` (5.14.2): Copyright (c) 2015-2026 the original author or authors.
- `Kotlin` (1.8.21): Copyright (c) 2010-2023 JetBrains s.r.o. and Kotlin Programming Language contributors.
- `kotlin-stdlib-common` (1.9.10): Copyright (c) 2007-2023 Google Inc., JetBrains s.r.o., The Guava Authors
- `langchain-opendataloader-pdf` (1.0.1): Copyright (c) 2024 LangChain, Inc.
- `libcspice-sys` (0.1.1): Copyright (c) 2025 libcspice-sys contributors
- `lilconfig` (3.1.3): Copyright (c) 2022 Anton Kastritskiy
- `lines-and-columns` (1.2.4): Copyright (c) 2015 Brian Donovan
- `load-tsconfig` (0.2.5): Copyright (c) 2021 EGOIST
- `mafintosh/why-is-node-running` (2.3.0): Copyright (c) 2016 Mathias Buus
- `magic-string` (0.30.21): Copyright (c) 2018 Rich Harris
- `mdBook` (0.4.36): Copyright (c) 2006-2020 Ivan Sagalaev, Ajax.org B.V., Oliver Nightingale, Wei Song
- `minimatch` (9.0.5): Copyright (c) 2011-2023 Isaac Z. Schlueter and Contributors
- `minipass` (7.1.2): Copyright (c) 2017-2023 npm, Inc., Isaac Z. Schlueter, and Contributors
- `MockWebServer` (4.12.0): Copyright (c) 2011-2019 Google Inc., Square, Inc.
- `moxystudio/node-cross-spawn` (7.0.6): Copyright (c) 2018 Made With MOXY Lda
- `Mozilla Rhino` (1.7.14.1): Copyright (c) 1991-2022 Lucent Technologies, Free Software Foundation, Stuart Knightley, Oracle, Vitaly Puzrin, Sun Microsystems, V8 project authors, Raffaello Giulietti
- `ms.js` (2.1.3): Copyright (c) 2020 Vercel, Inc.
- `mz` (2.7.0): Copyright (c) 2014-2016 Jonathan Ong
- `nanoid` (3.3.11): Copyright (c) 2017 Andrey Sitnik
- `natural-compare` (1.4.0): Copyright (c) 2012-2015 Lauri Rooden
- `nobody` (2.1.1): Copyright (c) 2014-2025 TJ Holowaychuk, Josh Junon, Kevin Deng
- `node-glob` (10.5.0): Copyright (c) 2009-2023 Isaac Z. Schlueter and Contributors
- `node-ignore` (5.3.2, 7.0.5): Copyright (c) 2013 Kael Zhang
- `node-lru-cache` (10.4.3, 11.2.2): Copyright (c) 2010-2023 Isaac Z. Schlueter and Contributors
- `nodeca-argparse` (2.0.1): Copyright (c) 1991-2020 Stichting Mathematisch Centrum Amsterdam, Gregory P. Ward, Python Software Foundation, argparse.js authors
- `OkHttp` (4.12.0): Copyright (c) 2010-2020 The Android Open Source Project, Square Inc., Twitter Inc.
- `OkIO` (3.6.0): Copyright (c) 2014-2023 Square, Inc.
- `org.apiguardian:apiguardian-api` (1.1.2): Copyright (c) 1995-2018 Jean-loup Gailly, Mark Adler, Stuart Knightley, Vitaly Puzrin, Oracle, jQuery Foundation
- `org.junit.jupiter:junit-jupiter-*` (5.14.2): Copyright (c) 1989-2026 Free Software Foundation, Oracle, the original author or authors.
- `org.junit.platform:junit-platform-*` (1.14.2): Copyright (c) 1989-2026 Free Software Foundation, Oracle, the original author or authors.
- `org.opentest4j:opentest4j` (1.3.0): Copyright (c) 1989-2023 Free Software Foundation, Oracle, the original author or authors.
- `orjson` (3.10.15): Copyright (c) 1991-2023 Alex Crichton, Milo Yip, Ryohei Machida, The Rust Project Developers, Andrew Gallant, Nicholas Allegra, Nikolai Vazquez, The bytecount Developers, PyO3 Project, Sergio Benitez, Ashley Mannix, The Servo Project Developers, YaoYuan, Parker Timmerman, Stephen M. Coakley, The Uuid Project Developers, Ulf Adams, Unicode Inc., winapi-rs developers
- `Packaging` (24.2): Copyright (c) 2017-Present Ofek Lev
- `pathe` (2.0.3): Copyright (c) 2023-Present Fabio Spampinato
- `PDFBox JBIG2 ImageIO plugin` (3.0.3): Copyright (c) 1995-2019 levigo holding GmbH, The Apache Software Foundation
- `picocolors` (1.1.1): Copyright (c) 2021-2024 Oleksii Raspopov, Kostiantyn Denysov, Anton Verinov
- `picomatch` (4.0.3): Copyright (c) 2017-Present Jon Schlinkert
- `pirates` (4.0.7): Copyright (c) 2015-2018 Ari Porad
- `PostCSS` (8.5.6): Copyright (c) 2013 Andrey Sitnik
- `Prettier IO` (3.8.1): Copyright (c) 2009-2026 Google LLC, Kevin Decker, Vitaly Puzrin, Woong Jun, Raynos, Ingvar Stepanyan, Aseem Kishore, Andrey Sitnik, Dominic Tarr, James Halliday, Kael Zhang, Liucw, Mikola Lysenko, Alex Bell, Stefan Thomas, Yehuda Katz, Jon Schlinkert, Teambition, Simon Lydell, Sebastian McKenzie, Tilde Inc., Titus Wormer, Elan Shanker, Matteo Collina, Denys Kniazevych, Joshua Holbrook, Mark Wubben, Pat Sissons, Thomas Watson Steen, Andrew Powell, Evgeny Poberezkin, Luke Childs, Andrea Giammarchi, KFlash, typescript-eslint, Fabio Spampinato, Jared Wray, Oleksii Raspopov, Eemeli Aro, EditorConfig Team.
- `psf-requests` (2.32.5): Copyright (c) 2012-2019 Kenneth Reitz
- `pydantic` (2.10.6): Copyright (c) 2017-Present Pydantic Services Inc.
- `pydantic-core` (2.27.2): Copyright (c) 2022 Samuel Colvin
- `python-json-patch` (1.33): Copyright (c) 2011 Stefan Kögl
- `python-json-pointer` (3.0.0): Copyright (c) 2011 Stefan Kögl
- `python-typing-extensions` (4.15.0): Copyright (c) 1991-1995 Stichting Mathematisch Centrum Amsterdam
- `python3-charset-normalizer` (3.4.1): Copyright (c) 2021-2025 Ahmed TAHRI
- `PyYAML` (6.0.2): Copyright (c) 2006-2021 Kirill Simonov, Ingy döt Net
- `Qix-/color-convert` (2.0.1): Copyright (c) 2011-2016 Heather Arthur, Josh Junon
- `readdirp` (4.1.2): Copyright (c) 2012-2019 Thorsten Lorenz, Paul Miller
- `requests-toolbelt` (1.0.0): Copyright (c) 2014 Ian Cordasco, Cory Benfield
- `rollup/rollup` (4.53.2): Copyright (c) 2012-2024 Paul Miller, Elan Shanker, Thorsten Lorenz, Jon Schlinkert, Benjamin Coe, Isaac Z. Schlueter, RollupJS Plugin Contributors, Sindre Sorhus, Oleksii Raspopov, Rich Harris, Justin Ridgewell
- `Saxon XSLT and XQuery Processor` (12.8): Copyright (c) 1998-2025 James Clark, Saxonica Limited, Michael Froh, Oracle
- `siginfo` (2.0.0): Copyright (c) 2017 Emil Bay
- `source-map` (0.7.6): Copyright (c) 2009-2014 Mozilla Foundation, The Closure Compiler Authors
- `source-map-js` (1.2.1): Copyright (c) 2009-2014 Mozilla Foundation, The Closure Compiler Authors
- `stackback` (0.0.2): Copyright (c) 2012 the V8 project authors.
- `StAX Utilities Project` (20070216): Copyright (c) 2004-2006 Christian Niles, Sun Microsystems, John Kristian
- `sucrase` (3.35.0): Copyright (c) 2012-2018 various contributors
- `tapjs/signal-exit` (4.1.0): Copyright (c) 2015-2023 Benjamin Coe, Isaac Z. Schlueter
- `tenacity` (8.5.0): Copyright (c) 2013-2018 Ray Holder, Joshua Harlow, Julien Danjou, Elisey Zanko
- `thenify` (3.3.1): Copyright (c) 2014-2016 Jonathan Ong
- `thenify-all` (1.6.0): Copyright (c) 2014 Jonathan Ong
- `tinybench` (2.9.0): Copyright (c) 2022 Tinylibs
- `tinyexec` (0.3.2, 1.0.2): Copyright (c) 2024 Tinylibs
- `tinyglobby` (0.2.15): Copyright (c) 2024 Madeline Gurriar
- `tinyrainbow` (3.0.3): Copyright (c) 2022 Tinylibs
- `tree-kill` (v1.2.2): Copyright (c) 2018 Peter Krumins
- `tsup` (8.5.1): Copyright (c) 2021 EGOIST
- `TypeScript` (5.9.3): Copyright (c) 1991-2018 Unicode Inc., The Khronos Group Inc., WHATWG
- `uri-js` (4.4.1): Copyright (c) 2011 Gary Court
- `urllib3` (2.6.1): Copyright (c) 2008-2020 Andrey Petrov and contributors
- `vitejs` (7.3.1): Copyright (c) 2010-2025 Sencha Inc., William Stein, Einar Otto Stangvik, LearnBoost, TJ Holowaychuk, Paul Miller, Elan Shanker, Thorsten Lorenz, Arnout Kazemier, James Halliday, Troy Goode, Jonathan Ong, Jared Hanson, Ivan Nikulin, Maxime Thirouin, Nathan Rajlich, Jon Schlinkert, Douglas Christopher Wilson, Simon Lydell, Alexey Litvinov, Andreas Lubbe, Glen Maddern, Tiancheng Gu, Scott Motte, Facebook Inc., Luigi Pinca, Yuxi You, MOXY Lda, Josh Junon, Guy Bedford, Rich Harris, Sindre Sorhus, VoidZero Inc., The Preact Authors, dominikg, Anthony Fu, Anton Kastritskiy, sapphi-red, Mark Dalgleish, Alexander Madyankin, Justin Ridgewell, Kevin Deng.
- `word-wrap` (1.2.5): Copyright (c) 2014-2023 Jon Schlinkert
- `XML Resolver` (5.3.3): Copyright (c) 1989-2023 Free Software Foundation, The Open Healthcare Group, Jonathan Borden, Oracle, W3C, The Internet Society
- `zstandard` (0.23.0): Copyright (c) 1989-2021 Free Software Foundation, Yuta Mori, Gregory Szorc, Tino Reichardt
THIRD_PARTY/licenses/BSD-2-Clause.txt
+22
View File
@@ -0,0 +1,22 @@
BSD Two Clause License
======================
Redistribution and use in source and binary forms, with or without modification,
are permitted provided that the following conditions are met:
1. Redistributions of source code must retain the above copyright notice, this
list of conditions and the following disclaimer.
2. Redistributions in binary form must reproduce the above copyright notice,
this list of conditions and the following disclaimer in the documentation
and/or other materials provided with the distribution.
THIS SOFTWARE IS PROVIDED BY THE AUTHOR "AS IS" AND ANY EXPRESS OR IMPLIED
WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT
SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT
OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
DAMAGE.
THIRD_PARTY/licenses/BSD-3-Clause.txt
+30
View File
@@ -0,0 +1,30 @@
BSD 3-clause "New" or "Revised" License
Copyright (c) <YEAR>, <OWNER>
All rights reserved.
Redistribution and use in source and binary forms, with or without modification,
are permitted provided that the following conditions are met:
* Redistributions of source code must retain the above copyright notice, this
list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above copyright notice,
this list of conditions and the following disclaimer in the documentation
and/or other materials provided with the distribution.
* Neither the name of the <ORGANIZATION> nor the names of its contributors may
be used to endorse or promote products derived from this software without
specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS
OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN
IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
THIRD_PARTY/licenses/Blue-Oak-1.0.0.txt
+56
View File
@@ -0,0 +1,56 @@
Blue Oak Model License
======================
Version 1.0.0
Purpose
-------
This license gives everyone as much permission to work with this software as
possible, while protecting contributors from liability.
Acceptance
----------
In order to receive this license, you must agree to its rules. The rules of this
license are both obligations under that agreement and conditions to your license.
You must not do anything with this software that triggers a rule that you cannot
or will not follow.
Copyright
---------
Each contributor licenses you to do everything with this software that would
otherwise infringe that contributor's copyright in it.
Notices
-------
You must ensure that everyone who gets a copy of any part of this software from
you, with or without changes, also gets the text of this license or a link to
https://blueoakcouncil.org/license/1.0.0.
Excuse
------
If anyone notifies you in writing that you have not complied with Notices, you
can keep your license by taking all practical steps to comply within 30 days
after the notice. If you do not do so, your license ends immediately.
Patent
------
Each contributor licenses you to do everything with this software that would
otherwise infringe any patent claims they can license or become able to license.
Reliability
-----------
No contributor can revoke this license.
No Liability
------------
As far as the law allows, this software comes as is, without any warranty or
condition, and no contributor will be liable to anyone for any damages related to
this software or this license, under any kind of legal claim.
THIRD_PARTY/licenses/CDDL-1.1.txt
+352
View File
@@ -0,0 +1,352 @@
COMMON DEVELOPMENT AND DISTRIBUTION LICENSE (CDDL) Version 1.1
==============================================================
1. Definitions.
1.1. “Contributor” means each individual or entity that creates or contributes
to the creation of Modifications.
1.2. “Contributor Version” means the combination of the Original Software,
prior Modifications used by a Contributor (if any), and the Modifications made
by that particular Contributor.
1.3. “Covered Software” means (a) the Original Software, or (b) Modifications,
or (c) the combination of files containing Original Software with files
containing Modifications, in each case including portions thereof.
1.4. “Executable” means the Covered Software in any form other than Source
Code.
1.5. “Initial Developer” means the individual or entity that first makes
Original Software available under this License.
1.6. “Larger Work” means a work which combines Covered Software or portions
thereof with code not governed by the terms of this License.
1.7. “License” means this document.
1.8. “Licensable” means having the right to grant, to the maximum extent
possible, whether at the time of the initial grant or subsequently acquired,
any and all of the rights conveyed herein.
1.9. “Modifications” means the Source Code and Executable form of any of the
following:
A. Any file that results from an addition to, deletion from or modification
of the contents of a file containing Original Software or previous
Modifications;
B. Any new file that contains any part of the Original Software or previous
Modification; or
C. Any new file that is contributed or otherwise made available under the
terms of this License.
1.10. “Original Software” means the Source Code and Executable form of computer
software code that is originally released under this License.
1.11. “Patent Claims” means any patent claim(s), now owned or hereafter
acquired, including without limitation, method, process, and apparatus claims,
in any patent Licensable by grantor.
1.12. “Source Code” means (a) the common form of computer software code in
which modifications are made and (b) associated documentation included in or
with such code.
1.13. “You” (or “Your”) means an individual or a legal entity exercising rights
under, and complying with all of the terms of, this License. For legal
entities, “You” includes any entity which controls, is controlled by, or is
under common control with You. For purposes of this definition, “control” means
(a) the power, direct or indirect, to cause the direction or management of such
entity, whether by contract or otherwise, or (b) ownership of more than fifty
percent (50%) of the outstanding shares or beneficial ownership of such entity.
2. License Grants.
2.1. The Initial Developer Grant.
Conditioned upon Your compliance with Section 3.1 below and subject to third
party intellectual property claims, the Initial Developer hereby grants You a
world-wide, royalty-free, non-exclusive license:
(a) under intellectual property rights (other than patent or trademark)
Licensable by Initial Developer, to use, reproduce, modify, display, perform,
sublicense and distribute the Original Software (or portions thereof), with
or without Modifications, and/or as part of a Larger Work; and
(b) under Patent Claims infringed by the making, using or selling of Original
Software, to make, have made, use, practice, sell, and offer for sale, and/or
otherwise dispose of the Original Software (or portions thereof).
(c) The licenses granted in Sections 2.1(a) and (b) are effective on the date
Initial Developer first distributes or otherwise makes the Original Software
available to a third party under the terms of this License.
(d) Notwithstanding Section 2.1(b) above, no patent license is granted: (1)
for code that You delete from the Original Software, or (2) for infringements
caused by: (i) the modification of the Original Software, or (ii) the
combination of the Original Software with other software or devices.
2.2. Contributor Grant.
Conditioned upon Your compliance with Section 3.1 below and subject to third
party intellectual property claims, each Contributor hereby grants You a
world-wide, royalty-free, non-exclusive license:
(a) under intellectual property rights (other than patent or trademark)
Licensable by Contributor to use, reproduce, modify, display, perform,
sublicense and distribute the Modifications created by such Contributor (or
portions thereof), either on an unmodified basis, with other Modifications,
as Covered Software and/or as part of a Larger Work; and
(b) under Patent Claims infringed by the making, using, or selling of
Modifications made by that Contributor either alone and/or in combination
with its Contributor Version (or portions of such combination), to make, use,
sell, offer for sale, have made, and/or otherwise dispose of: (1)
Modifications made by that Contributor (or portions thereof); and (2) the
combination of Modifications made by that Contributor with its Contributor
Version (or portions of such combination).
(c) The licenses granted in Sections 2.2(a) and 2.2(b) are effective on the
date Contributor first distributes or otherwise makes the Modifications
available to a third party.
(d) Notwithstanding Section 2.2(b) above, no patent license is granted:
(1) for any code that Contributor has deleted from the Contributor Version;
(2) for infringements caused by: (i) third party modifications of
Contributor Version, or (ii) the combination of Modifications made by that
Contributor with other software (except as part of the Contributor Version)
or other devices; or
(3) under Patent Claims infringed by Covered Software in the absence of
Modifications made by that Contributor.
3. Distribution Obligations.
3.1. Availability of Source Code.
Any Covered Software that You distribute or otherwise make available in
Executable form must also be made available in Source Code form and that
Source Code form must be distributed only under the terms of this License.
You must include a copy of this License with every copy of the Source Code
form of the Covered Software You distribute or otherwise make available. You
must inform recipients of any such Covered Software in Executable form as to
how they can obtain such Covered Software in Source Code form in a reasonable
manner on or through a medium customarily used for software exchange.
3.2. Modifications.
The Modifications that You create or to which You contribute are governed by
the terms of this License. You represent that You believe Your Modifications
are Your original creation(s) and/or You have sufficient rights to grant the
rights conveyed by this License.
3.3. Required Notices.
You must include a notice in each of Your Modifications that identifies You
as the Contributor of the Modification. You may not remove or alter any
copyright, patent or trademark notices contained within the Covered Software,
or any notices of licensing or any descriptive text giving attribution to any
Contributor or the Initial Developer.
3.4. Application of Additional Terms.
You may not offer or impose any terms on any Covered Software in Source Code
form that alters or restricts the applicable version of this License or the
recipients' rights hereunder. You may choose to offer, and to charge a fee
for, warranty, support, indemnity or liability obligations to one or more
recipients of Covered Software. However, you may do so only on Your own
behalf, and not on behalf of the Initial Developer or any Contributor. You
must make it absolutely clear that any such warranty, support, indemnity or
liability obligation is offered by You alone, and You hereby agree to
indemnify the Initial Developer and every Contributor for any liability
incurred by the Initial Developer or such Contributor as a result of
warranty, support, indemnity or liability terms You offer.
3.5. Distribution of Executable Versions.
You may distribute the Executable form of the Covered Software under the
terms of this License or under the terms of a license of Your choice, which
may contain terms different from this License, provided that You are in
compliance with the terms of this License and that the license for the
Executable form does not attempt to limit or alter the recipient's rights in
the Source Code form from the rights set forth in this License. If You
distribute the Covered Software in Executable form under a different license,
You must make it absolutely clear that any terms which differ from this
License are offered by You alone, not by the Initial Developer or
Contributor. You hereby agree to indemnify the Initial Developer and every
Contributor for any liability incurred by the Initial Developer or such
Contributor as a result of any such terms You offer.
3.6. Larger Works.
You may create a Larger Work by combining Covered Software with other code
not governed by the terms of this License and distribute the Larger Work as a
single product. In such a case, You must make sure the requirements of this
License are fulfilled for the Covered Software.
4. Versions of the License.
4.1. New Versions.
Oracle is the initial license steward and may publish revised and/or new
versions of this License from time to time. Each version will be given a
distinguishing version number. Except as provided in Section 4.3, no one
other than the license steward has the right to modify this License.
4.2. Effect of New Versions.
You may always continue to use, distribute or otherwise make the Covered
Software available under the terms of the version of the License under which
You originally received the Covered Software. If the Initial Developer
includes a notice in the Original Software prohibiting it from being
distributed or otherwise made available under any subsequent version of the
License, You must distribute and make the Covered Software available under
the terms of the version of the License under which You originally received
the Covered Software. Otherwise, You may also choose to use, distribute or
otherwise make the Covered Software available under the terms of any
subsequent version of the License published by the license steward.
4.3. Modified Versions.
When You are an Initial Developer and You want to create a new license for
Your Original Software, You may create and use a modified version of this
License if You: (a) rename the license and remove any references to the name
of the license steward (except to note that the license differs from this
License); and (b) otherwise make it clear that the license contains terms
which differ from this License.
5. DISCLAIMER OF WARRANTY.
COVERED SOFTWARE IS PROVIDED UNDER THIS LICENSE ON AN “AS IS” BASIS, WITHOUT
WARRANTY OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, WITHOUT
LIMITATION, WARRANTIES THAT THE COVERED SOFTWARE IS FREE OF DEFECTS,
MERCHANTABLE, FIT FOR A PARTICULAR PURPOSE OR NON-INFRINGING. THE ENTIRE RISK
AS TO THE QUALITY AND PERFORMANCE OF THE COVERED SOFTWARE IS WITH YOU. SHOULD
ANY COVERED SOFTWARE PROVE DEFECTIVE IN ANY RESPECT, YOU (NOT THE INITIAL
DEVELOPER OR ANY OTHER CONTRIBUTOR) ASSUME THE COST OF ANY NECESSARY SERVICING,
REPAIR OR CORRECTION. THIS DISCLAIMER OF WARRANTY CONSTITUTES AN ESSENTIAL PART
OF THIS LICENSE. NO USE OF ANY COVERED SOFTWARE IS AUTHORIZED HEREUNDER EXCEPT
UNDER THIS DISCLAIMER.
6. TERMINATION.
6.1. This License and the rights granted hereunder will terminate
automatically if You fail to comply with terms herein and fail to cure such
breach within 30 days of becoming aware of the breach. Provisions which, by
their nature, must remain in effect beyond the termination of this License
shall survive.
6.2. If You assert a patent infringement claim (excluding declaratory
judgment actions) against Initial Developer or a Contributor (the Initial
Developer or Contributor against whom You assert such claim is referred to as
“Participant”) alleging that the Participant Software (meaning the
Contributor Version where the Participant is a Contributor or the Original
Software where the Participant is the Initial Developer) directly or
indirectly infringes any patent, then any and all rights granted directly or
indirectly to You by such Participant, the Initial Developer (if the Initial
Developer is not the Participant) and all Contributors under Sections 2.1
and/or 2.2 of this License shall, upon 60 days notice from Participant
terminate prospectively and automatically at the expiration of such 60 day
notice period, unless if within such 60 day period You withdraw Your claim
with respect to the Participant Software against such Participant either
unilaterally or pursuant to a written agreement with Participant.
6.3. If You assert a patent infringement claim against Participant alleging
that the Participant Software directly or indirectly infringes any patent
where such claim is resolved (such as by license or settlement) prior to the
initiation of patent infringement litigation, then the reasonable value of
the licenses granted by such Participant under Sections 2.1 or 2.2 shall be
taken into account in determining the amount or value of any payment or
license.
6.4. In the event of termination under Sections 6.1 or 6.2 above, all end
user licenses that have been validly granted by You or any distributor
hereunder prior to termination (excluding licenses granted to You by any
distributor) shall survive termination.
7. LIMITATION OF LIABILITY.
UNDER NO CIRCUMSTANCES AND UNDER NO LEGAL THEORY, WHETHER TORT (INCLUDING
NEGLIGENCE), CONTRACT, OR OTHERWISE, SHALL YOU, THE INITIAL DEVELOPER, ANY
OTHER CONTRIBUTOR, OR ANY DISTRIBUTOR OF COVERED SOFTWARE, OR ANY SUPPLIER OF
ANY OF SUCH PARTIES, BE LIABLE TO ANY PERSON FOR ANY INDIRECT, SPECIAL,
INCIDENTAL, OR CONSEQUENTIAL DAMAGES OF ANY CHARACTER INCLUDING, WITHOUT
LIMITATION, DAMAGES FOR LOSS OF GOODWILL, WORK STOPPAGE, COMPUTER FAILURE OR
MALFUNCTION, OR ANY AND ALL OTHER COMMERCIAL DAMAGES OR LOSSES, EVEN IF SUCH
PARTY SHALL HAVE BEEN INFORMED OF THE POSSIBILITY OF SUCH DAMAGES. THIS
LIMITATION OF LIABILITY SHALL NOT APPLY TO LIABILITY FOR DEATH OR PERSONAL
INJURY RESULTING FROM SUCH PARTY'S NEGLIGENCE TO THE EXTENT APPLICABLE LAW
PROHIBITS SUCH LIMITATION. SOME JURISDICTIONS DO NOT ALLOW THE EXCLUSION OR
LIMITATION OF INCIDENTAL OR CONSEQUENTIAL DAMAGES, SO THIS EXCLUSION AND
LIMITATION MAY NOT APPLY TO YOU.
8. U.S. GOVERNMENT END USERS.
The Covered Software is a “commercial item,” as that term is defined in 48
C.F.R. 2.101 (Oct. 1995), consisting of “commercial computer software” (as that
term is defined at 48 C.F.R. § 252.227-7014(a)(1)) and “commercial computer
software documentation” as such terms are used in 48 C.F.R. 12.212 (Sept.
1995). Consistent with 48 C.F.R. 12.212 and 48 C.F.R. 227.7202-1 through
227.7202-4 (June 1995), all U.S. Government End Users acquire Covered Software
with only those rights set forth herein. This U.S. Government Rights clause is
in lieu of, and supersedes, any other FAR, DFAR, or other clause or provision
that addresses Government rights in computer software under this License.
9. MISCELLANEOUS.
This License represents the complete agreement concerning subject matter
hereof. If any provision of this License is held to be unenforceable, such
provision shall be reformed only to the extent necessary to make it
enforceable. This License shall be governed by the law of the jurisdiction
specified in a notice contained within the Original Software (except to the
extent applicable law, if any, provides otherwise), excluding such
jurisdiction's conflict-of-law provisions. Any litigation relating to this
License shall be subject to the jurisdiction of the courts located in the
jurisdiction and venue specified in a notice contained within the Original
Software, with the losing party responsible for costs, including, without
limitation, court costs and reasonable attorneys' fees and expenses. The
application of the United Nations Convention on Contracts for the International
Sale of Goods is expressly excluded. Any law or regulation which provides that
the language of a contract shall be construed against the drafter shall not
apply to this License. You agree that You alone are responsible for compliance
with the United States export administration regulations (and the export
control laws and regulation of any other countries) when You use, distribute or
otherwise make available any Covered Software.
10. RESPONSIBILITY FOR CLAIMS.
As between Initial Developer and the Contributors, each party is responsible
for claims and damages arising, directly or indirectly, out of its utilization
of rights under this License and You agree to work with Initial Developer and
Contributors to distribute such responsibility on an equitable basis. Nothing
herein is intended or shall be deemed to constitute any admission of liability.
------------------------------------------------------------------------------
NOTICE PURSUANT TO SECTION 9 OF THE COMMON DEVELOPMENT AND DISTRIBUTION LICENSE
(CDDL)
The code released under the CDDL shall be governed by the laws of the State of
California (excluding conflict-of-law provisions). Any litigation relating to
this License shall be subject to the jurisdiction of the Federal Courts of the
Northern District of California and the state courts of the State of
California, with venue lying in Santa Clara County, California.
THIRD_PARTY/licenses/EDL-1.0.txt
+31
View File
@@ -0,0 +1,31 @@
Eclipse Distribution License - v 1.0
====================================
Copyright (c) 2007, Eclipse Foundation, Inc. and its licensors.
All rights reserved.
Redistribution and use in source and binary forms, with or without modification,
are permitted provided that the following conditions are met:
* Redistributions of source code must retain the above copyright notice, this
list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above copyright notice,
this list of conditions and the following disclaimer in the documentation
and/or other materials provided with the distribution.
* Neither the name of the Eclipse Foundation, Inc. nor the names of its
contributors may be used to endorse or promote products derived from this
software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS
OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN
IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
THIRD_PARTY/licenses/EPL-1.0.txt
+211
View File
@@ -0,0 +1,211 @@
Eclipse Public License - v 1.0
==============================
THE ACCOMPANYING PROGRAM IS PROVIDED UNDER THE TERMS OF THIS ECLIPSE PUBLIC
LICENSE ("AGREEMENT"). ANY USE, REPRODUCTION OR DISTRIBUTION OF THE PROGRAM
CONSTITUTES RECIPIENT'S ACCEPTANCE OF THIS AGREEMENT.
1. DEFINITIONS
"Contribution" means:
a) in the case of the initial Contributor, the initial code and documentation
distributed under this Agreement, and
b) in the case of each subsequent Contributor:
i) changes to the Program, and
ii) additions to the Program;
where such changes and/or additions to the Program originate from and are
distributed by that particular Contributor. A Contribution 'originates' from a
Contributor if it was added to the Program by such Contributor itself or anyone
acting on such Contributor's behalf. Contributions do not include additions to
the Program which: (i) are separate modules of software distributed in
conjunction with the Program under their own license agreement, and (ii) are not
derivative works of the Program.
"Contributor" means any person or entity that distributes the Program.
"Licensed Patents " mean patent claims licensable by a Contributor which are
necessarily infringed by the use or sale of its Contribution alone or when
combined with the Program.
"Program" means the Contributions distributed in accordance with this Agreement.
"Recipient" means anyone who receives the Program under this Agreement, including
all Contributors.
2. GRANT OF RIGHTS
a) Subject to the terms of this Agreement, each Contributor hereby grants
Recipient a non-exclusive, worldwide, royalty-free copyright license to reproduce,
prepare derivative works of, publicly display, publicly perform, distribute and
sublicense the Contribution of such Contributor, if any, and such derivative
works, in source code and object code form.
b) Subject to the terms of this Agreement, each Contributor hereby grants
Recipient a non-exclusive, worldwide, royalty-free patent license under Licensed
Patents to make, use, sell, offer to sell, import and otherwise transfer the
Contribution of such Contributor, if any, in source code and object code form.
This patent license shall apply to the combination of the Contribution and the
Program if, at the time the Contribution is added by the Contributor, such
addition of the Contribution causes such combination to be covered by the
Licensed Patents. The patent license shall not apply to any other combinations
which include the Contribution. No hardware per se is licensed hereunder.
c) Recipient understands that although each Contributor grants the licenses to
its Contributions set forth herein, no assurances are provided by any Contributor
that the Program does not infringe the patent or other intellectual property
rights of any other entity. Each Contributor disclaims any liability to Recipient
for claims brought by any other entity based on infringement of intellectual
property rights or otherwise. As a condition to exercising the rights and
licenses granted hereunder, each Recipient hereby assumes sole responsibility to
secure any other intellectual property rights needed, if any. For example, if a
third party patent license is required to allow Recipient to distribute the
Program, it is Recipient's responsibility to acquire that license before
distributing the Program.
d) Each Contributor represents that to its knowledge it has sufficient copyright
rights in its Contribution, if any, to grant the copyright license set forth in
this Agreement.
3. REQUIREMENTS
A Contributor may choose to distribute the Program in object code form under its
own license agreement, provided that:
a) it complies with the terms and conditions of this Agreement;
and
b) its license agreement:
i) effectively disclaims on behalf of all Contributors all warranties and
conditions, express and implied, including warranties or conditions of title and
non-infringement, and implied warranties or conditions of merchantability and
fitness for a particular purpose;
ii) effectively excludes on behalf of all Contributors all liability for damages,
including direct, indirect, special, incidental and consequential damages, such
as lost profits;
iii) states that any provisions which differ from this Agreement are offered by
that Contributor alone and not by any other party;
and
iv) states that source code for the Program is available from such Contributor,
and informs licensees how to obtain it in a reasonable manner on or through a
medium customarily used for software exchange.
When the Program is made available in source code form:
a) it must be made available under this Agreement;
and
b) a copy of this Agreement must be included with each copy of the Program.
Contributors may not remove or alter any copyright notices contained within the
Program.
Each Contributor must identify itself as the originator of its Contribution, if
any, in a manner that reasonably allows subsequent Recipients to identify the
originator of the Contribution.
4. COMMERCIAL DISTRIBUTION
Commercial distributors of software may accept certain responsibilities with
respect to end users, business partners and the like. While this license is
intended to facilitate the commercial use of the Program, the Contributor who
includes the Program in a commercial product offering should do so in a manner
which does not create potential liability for other Contributors. Therefore, if a
Contributor includes the Program in a commercial product offering, such
Contributor ("Commercial Contributor") hereby agrees to defend and indemnify
every other Contributor ("Indemnified Contributor") against any losses, damages
and costs (collectively "Losses") arising from claims, lawsuits and other legal
actions brought by a third party against the Indemnified Contributor to the
extent caused by the acts or omissions of such Commercial Contributor in
connection with its distribution of the Program in a commercial product offering.
The obligations in this section do not apply to any claims or Losses relating to
any actual or alleged intellectual property infringement. In order to qualify, an
Indemnified Contributor must: a) promptly notify the Commercial Contributor in
writing of such claim, and b) allow the Commercial Contributor to control, and
cooperate with the Commercial Contributor in, the defense and any related
settlement negotiations. The Indemnified Contributor may participate in any such
claim at its own expense.
For example, a Contributor might include the Program in a commercial product
offering, Product X. That Contributor is then a Commercial Contributor. If that
Commercial Contributor then makes performance claims, or offers warranties
related to Product X, those performance claims and warranties are such Commercial
Contributor's responsibility alone. Under this section, the Commercial
Contributor would have to defend claims against the other Contributors related to
those performance claims and warranties, and if a court requires any other
Contributor to pay any damages as a result, the Commercial Contributor must pay
those damages.
5. NO WARRANTY
EXCEPT AS EXPRESSLY SET FORTH IN THIS AGREEMENT, THE PROGRAM IS PROVIDED ON AN
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, EITHER EXPRESS OR
IMPLIED INCLUDING, WITHOUT LIMITATION, ANY WARRANTIES OR CONDITIONS OF TITLE,
NON-INFRINGEMENT, MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE. Each
Recipient is solely responsible for determining the appropriateness of using and
distributing the Program and assumes all risks associated with its exercise of
rights under this Agreement , including but not limited to the risks and costs of
program errors, compliance with applicable laws, damage to or loss of data,
programs or equipment, and unavailability or interruption of operations.
6. DISCLAIMER OF LIABILITY
EXCEPT AS EXPRESSLY SET FORTH IN THIS AGREEMENT, NEITHER RECIPIENT NOR ANY
CONTRIBUTORS SHALL HAVE ANY LIABILITY FOR ANY DIRECT, INDIRECT, INCIDENTAL,
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING WITHOUT LIMITATION LOST
PROFITS), HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
OUT OF THE USE OR DISTRIBUTION OF THE PROGRAM OR THE EXERCISE OF ANY RIGHTS
GRANTED HEREUNDER, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGES.
7. GENERAL
If any provision of this Agreement is invalid or unenforceable under applicable
law, it shall not affect the validity or enforceability of the remainder of the
terms of this Agreement, and without further action by the parties hereto, such
provision shall be reformed to the minimum extent necessary to make such
provision valid and enforceable.
If Recipient institutes patent litigation against any entity (including a
cross-claim or counterclaim in a lawsuit) alleging that the Program itself
(excluding combinations of the Program with other software or hardware) infringes
such Recipient's patent(s), then such Recipient's rights granted under Section
2(b) shall terminate as of the date such litigation is filed.
All Recipient's rights under this Agreement shall terminate if it fails to comply
with any of the material terms or conditions of this Agreement and does not cure
such failure in a reasonable period of time after becoming aware of such
noncompliance. If all Recipient's rights under this Agreement terminate,
Recipient agrees to cease use and distribution of the Program as soon as
reasonably practicable. However, Recipient's obligations under this Agreement and
any licenses granted by Recipient relating to the Program shall continue and
survive.
Everyone is permitted to copy and distribute copies of this Agreement, but in
order to avoid inconsistency the Agreement is copyrighted and may only be
modified in the following manner. The Agreement Steward reserves the right to
publish new versions (including revisions) of this Agreement from time to time.
No one other than the Agreement Steward has the right to modify this Agreement.
The Eclipse Foundation is the initial Agreement Steward. The Eclipse Foundation
may assign the responsibility to serve as the Agreement Steward to a suitable
separate entity. Each new version of the Agreement will be given a distinguishing
version number. The Program (including Contributions) may always be distributed
subject to the version of the Agreement under which it was received. In addition,
after a new version of the Agreement is published, Contributor may elect to
distribute the Program (including its Contributions) under the new version.
Except as expressly stated in Sections 2(a) and 2(b) above, Recipient receives no
rights or licenses to the intellectual property of any Contributor under this
Agreement, whether expressly, by implication, estoppel or otherwise. All rights
in the Program not expressly granted under this Agreement are reserved.
This Agreement is governed by the laws of the State of New York and the
intellectual property laws of the United States of America. No party to this
Agreement will bring a legal action under this Agreement more than one year after
the cause of action arose. Each party waives its rights to a jury trial in any
resulting litigation.
THIRD_PARTY/licenses/EPL-2.0.txt
+267
View File
@@ -0,0 +1,267 @@
Eclipse Public License - v 2.0
==============================
THE ACCOMPANYING PROGRAM IS PROVIDED UNDER THE TERMS OF THIS ECLIPSE PUBLIC
LICENSE (“AGREEMENT”). ANY USE, REPRODUCTION OR DISTRIBUTION OF THE PROGRAM
CONSTITUTES RECIPIENT'S ACCEPTANCE OF THIS AGREEMENT.
1. DEFINITIONS
--------------
“Contribution” means:
a) in the case of the initial Contributor, the initial content Distributed
under this Agreement, and
b) in the case of each subsequent Contributor:
i) changes to the Program, and
ii) additions to the Program;
where such changes and/or additions to the Program originate from and are
Distributed by that particular Contributor. A Contribution “originates” from a
Contributor if it was added to the Program by such Contributor itself or anyone
acting on such Contributor's behalf. Contributions do not include changes or
additions to the Program that are not Modified Works.
“Contributor” means any person or entity that Distributes the Program.
“Licensed Patents” mean patent claims licensable by a Contributor which are
necessarily infringed by the use or sale of its Contribution alone or when
combined with the Program.
“Program” means the Contributions Distributed in accordance with this Agreement.
“Recipient” means anyone who receives the Program under this Agreement or any
Secondary License (as applicable), including Contributors.
“Derivative Works” shall mean any work, whether in Source Code or other form,
that is based on (or derived from) the Program and for which the editorial
revisions, annotations, elaborations, or other modifications represent, as a
whole, an original work of authorship.
“Modified Works” shall mean any work in Source Code or other form that results
from an addition to, deletion from, or modification of the contents of the
Program, including, for purposes of clarity any new file in Source Code form that
contains any contents of the Program. Modified Works shall not include works that
contain only declarations, interfaces, types, classes, structures, or files of
the Program solely in each case in order to link to, bind by name, or subclass
the Program or Modified Works thereof.
“Distribute” means the acts of a) distributing or b) making available in any
manner that enables the transfer of a copy.
“Source Code” means the form of a Program preferred for making modifications,
including but not limited to software source code, documentation source, and
configuration files.
“Secondary License” means either the GNU General Public License, Version 2.0, or
any later versions of that license, including any exceptions or additional
permissions as identified by the initial Contributor.
2. GRANT OF RIGHTS
------------------
a) Subject to the terms of this Agreement, each Contributor hereby grants
Recipient a non-exclusive, worldwide, royalty-free copyright license to
reproduce, prepare Derivative Works of, publicly display, publicly perform,
Distribute and sublicense the Contribution of such Contributor, if any, and
such Derivative Works.
b) Subject to the terms of this Agreement, each Contributor hereby grants
Recipient a non-exclusive, worldwide, royalty-free patent license under
Licensed Patents to make, use, sell, offer to sell, import and otherwise
transfer the Contribution of such Contributor, if any, in Source Code or other
form. This patent license shall apply to the combination of the Contribution
and the Program if, at the time the Contribution is added by the Contributor,
such addition of the Contribution causes such combination to be covered by the
Licensed Patents. The patent license shall not apply to any other combinations
which include the Contribution. No hardware per se is licensed hereunder.
c) Recipient understands that although each Contributor grants the licenses to
its Contributions set forth herein, no assurances are provided by any
Contributor that the Program does not infringe the patent or other intellectual
property rights of any other entity. Each Contributor disclaims any liability
to Recipient for claims brought by any other entity based on infringement of
intellectual property rights or otherwise. As a condition to exercising the
rights and licenses granted hereunder, each Recipient hereby assumes sole
responsibility to secure any other intellectual property rights needed, if any.
For example, if a third party patent license is required to allow Recipient to
Distribute the Program, it is Recipient's responsibility to acquire that
license before distributing the Program.
d) Each Contributor represents that to its knowledge it has sufficient
copyright rights in its Contribution, if any, to grant the copyright license
set forth in this Agreement.
e) Notwithstanding the terms of any Secondary License, no Contributor makes
additional grants to any Recipient (other than those set forth in this
Agreement) as a result of such Recipient's receipt of the Program under the
terms of a Secondary License (if permitted under the terms of Section 3).
3. REQUIREMENTS
---------------
3.1 If a Contributor Distributes the Program in any form, then:
a) the Program must also be made available as Source Code, in accordance with
section 3.2, and the Contributor must accompany the Program with a statement
that the Source Code for the Program is available under this Agreement, and
informs Recipients how to obtain it in a reasonable manner on or through a
medium customarily used for software exchange; and
b) the Contributor may Distribute the Program under a license different than
this Agreement, provided that such license:
i) effectively disclaims on behalf of all other Contributors all warranties
and conditions, express and implied, including warranties or conditions of
title and non-infringement, and implied warranties or conditions of
merchantability and fitness for a particular purpose;
ii) effectively excludes on behalf of all other Contributors all liability
for damages, including direct, indirect, special, incidental and
consequential damages, such as lost profits;
iii) does not attempt to limit or alter the recipients' rights in the Source
Code under section 3.2; and
iv) requires any subsequent distribution of the Program by any party to be
under a license that satisfies the requirements of this section 3.
3.2 When the Program is Distributed as Source Code:
a) it must be made available under this Agreement, or if the Program
(i) is combined with other material in a separate file or files made
available under a Secondary License, and
(ii) the initial Contributor attached to the Source Code the notice described
in Exhibit A of this Agreement, then the Program may be made available under
the terms of such Secondary Licenses, and
b) a copy of this Agreement must be included with each copy of the Program.
3.3 Contributors may not remove or alter any copyright, patent, trademark,
attribution notices, disclaimers of warranty, or limitations of liability
(‘notices’) contained within the Program from any copy of the Program which they
Distribute, provided that Contributors may add their own appropriate notices.
4. COMMERCIAL DISTRIBUTION
--------------------------
Commercial distributors of software may accept certain responsibilities with
respect to end users, business partners and the like. While this license is
intended to facilitate the commercial use of the Program, the Contributor who
includes the Program in a commercial product offering should do so in a manner
which does not create potential liability for other Contributors. Therefore, if a
Contributor includes the Program in a commercial product offering, such
Contributor (“Commercial Contributor”) hereby agrees to defend and indemnify
every other Contributor (“Indemnified Contributor”) against any losses, damages
and costs (collectively “Losses”) arising from claims, lawsuits and other legal
actions brought by a third party against the Indemnified Contributor to the
extent caused by the acts or omissions of such Commercial Contributor in
connection with its distribution of the Program in a commercial product offering.
The obligations in this section do not apply to any claims or Losses relating to
any actual or alleged intellectual property infringement. In order to qualify, an
Indemnified Contributor must: a) promptly notify the Commercial Contributor in
writing of such claim, and b) allow the Commercial Contributor to control, and
cooperate with the Commercial Contributor in, the defense and any related
settlement negotiations. The Indemnified Contributor may participate in any such
claim at its own expense.
For example, a Contributor might include the Program in a commercial product
offering, Product X. That Contributor is then a Commercial Contributor. If that
Commercial Contributor then makes performance claims, or offers warranties
related to Product X, those performance claims and warranties are such Commercial
Contributor's responsibility alone. Under this section, the Commercial
Contributor would have to defend claims against the other Contributors related to
those performance claims and warranties, and if a court requires any other
Contributor to pay any damages as a result, the Commercial Contributor must pay
those damages.
5. NO WARRANTY
--------------
EXCEPT AS EXPRESSLY SET FORTH IN THIS AGREEMENT, AND TO THE EXTENT PERMITTED BY
APPLICABLE LAW, THE PROGRAM IS PROVIDED ON AN “AS IS” BASIS, WITHOUT WARRANTIES
OR CONDITIONS OF ANY KIND, EITHER EXPRESS OR IMPLIED INCLUDING, WITHOUT
LIMITATION, ANY WARRANTIES OR CONDITIONS OF TITLE, NON-INFRINGEMENT,
MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE. Each Recipient is solely
responsible for determining the appropriateness of using and distributing the
Program and assumes all risks associated with its exercise of rights under this
Agreement, including but not limited to the risks and costs of program errors,
compliance with applicable laws, damage to or loss of data, programs or
equipment, and unavailability or interruption of operations.
6. DISCLAIMER OF LIABILITY
--------------------------
EXCEPT AS EXPRESSLY SET FORTH IN THIS AGREEMENT, AND TO THE EXTENT PERMITTED BY
APPLICABLE LAW, NEITHER RECIPIENT NOR ANY CONTRIBUTORS SHALL HAVE ANY LIABILITY
FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
DAMAGES (INCLUDING WITHOUT LIMITATION LOST PROFITS), HOWEVER CAUSED AND ON ANY
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OR DISTRIBUTION OF THE
PROGRAM OR THE EXERCISE OF ANY RIGHTS GRANTED HEREUNDER, EVEN IF ADVISED OF THE
POSSIBILITY OF SUCH DAMAGES.
7. GENERAL
----------
If any provision of this Agreement is invalid or unenforceable under applicable
law, it shall not affect the validity or enforceability of the remainder of the
terms of this Agreement, and without further action by the parties hereto, such
provision shall be reformed to the minimum extent necessary to make such
provision valid and enforceable.
If Recipient institutes patent litigation against any entity (including a
cross-claim or counterclaim in a lawsuit) alleging that the Program itself
(excluding combinations of the Program with other software or hardware) infringes
such Recipient's patent(s), then such Recipient's rights granted under Section
2(b) shall terminate as of the date such litigation is filed.
All Recipient's rights under this Agreement shall terminate if it fails to comply
with any of the material terms or conditions of this Agreement and does not cure
such failure in a reasonable period of time after becoming aware of such
noncompliance. If all Recipient's rights under this Agreement terminate,
Recipient agrees to cease use and distribution of the Program as soon as
reasonably practicable. However, Recipient's obligations under this Agreement and
any licenses granted by Recipient relating to the Program shall continue and
survive.
Everyone is permitted to copy and distribute copies of this Agreement, but in
order to avoid inconsistency the Agreement is copyrighted and may only be
modified in the following manner. The Agreement Steward reserves the right to
publish new versions (including revisions) of this Agreement from time to time.
No one other than the Agreement Steward has the right to modify this Agreement.
The Eclipse Foundation is the initial Agreement Steward. The Eclipse Foundation
may assign the responsibility to serve as the Agreement Steward to a suitable
separate entity. Each new version of the Agreement will be given a distinguishing
version number. The Program (including Contributions) may always be Distributed
subject to the version of the Agreement under which it was received. In addition,
after a new version of the Agreement is published, Contributor may elect to
Distribute the Program (including its Contributions) under the new version.
Except as expressly stated in Sections 2(a) and 2(b) above, Recipient receives no
rights or licenses to the intellectual property of any Contributor under this
Agreement, whether expressly, by implication, estoppel or otherwise. All rights
in the Program not expressly granted under this Agreement are reserved. Nothing
in this Agreement is intended to be enforceable by any entity that is not a
Contributor or Recipient. No third-party beneficiary rights are created under
this Agreement.
Exhibit A – Form of Secondary Licenses Notice
---------------------------------------------
“This Source Code may also be made available under the following Secondary
Licenses when the conditions for such availability set forth in the Eclipse
Public License, v. 2.0 are satisfied: {name license(s), version(s), and
exceptions or additional permissions here}.”
Simply including a copy of this Agreement, including this Exhibit A is not
sufficient to license the Source Code under Secondary Licenses.
If it is not possible or desirable to put the notice in a particular file,
then You may include the notice in a location (such as a LICENSE file in a
relevant directory) where a recipient would be likely to look for such a
notice.
You may add additional accurate notices of copyright ownership.
THIRD_PARTY/licenses/ISC.txt
+16
View File
@@ -0,0 +1,16 @@
ISC License (ISCL)
==================
Copyright (c) 4-digit year, Company or Person's Name
Permission to use, copy, modify, and/or distribute this software for any purpose
with or without fee is hereby granted, provided that the above copyright notice
and this permission notice appear in all copies.
THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH
REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND
FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT,
INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS
OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER
TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF
THIS SOFTWARE.
THIRD_PARTY/licenses/LICENSE-JJ2000.txt
+28
View File
@@ -0,0 +1,28 @@
This software module was originally developed by Raphaël Grosbois and
Diego Santa Cruz (Swiss Federal Institute of Technology-EPFL); Joel
Askelöf (Ericsson Radio Systems AB); and Bertrand Berthelot, David
Bouchard, Félix Henry, Gerard Mozelle and Patrice Onno (Canon Research
Centre France S.A) in the course of development of the JPEG2000
standard as specified by ISO/IEC 15444 (JPEG 2000 Standard). This
software module is an implementation of a part of the JPEG 2000
Standard. Swiss Federal Institute of Technology-EPFL, Ericsson Radio
Systems AB and Canon Research Centre France S.A (collectively JJ2000
Partners) agree not to assert against ISO/IEC and users of the JPEG
2000 Standard (Users) any of their rights under the copyright, not
including other intellectual property rights, for this software module
with respect to the usage by ISO/IEC and Users of this software module
or modifications thereof for use in hardware or software products
claiming conformance to the JPEG 2000 Standard. Those intending to use
this software module in hardware or software products are advised that
their use may infringe existing patents. The original developers of
this software module, JJ2000 Partners and ISO/IEC assume no liability
for use of this software module or modifications thereof. No license
or right to this software module is granted for non JPEG 2000 Standard
conforming products. JJ2000 Partners have full right to use this
software module for his/her own purpose, assign or donate this
software module to any third party and to inhibit third parties from
using this software module for non JPEG 2000 Standard conforming
products. This copyright notice must be included in all copies or
derivative works of this software module.
Copyright (c) 1999/2000 JJ2000 Partners.
THIRD_PARTY/licenses/MIT.txt
+21
View File
@@ -0,0 +1,21 @@
MIT License
Copyright (c) <year> <copyright holders>
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
THIRD_PARTY/licenses/MPL-2.0.txt
+408
View File
@@ -0,0 +1,408 @@
Mozilla Public License
Version 2.0
======================
1. Definitions
--------------
1.1. "Contributor"
means each individual or legal entity that creates, contributes to the creation
of, or owns Covered Software.
1.2. "Contributor Version"
means the combination of the Contributions of others (if any) used by a
Contributor and that particular Contributor's Contribution.
1.3. "Contribution"
means Covered Software of a particular Contributor.
1.4. "Covered Software"
means Source Code Form to which the initial Contributor has attached the notice
in Exhibit A, the Executable Form of such Source Code Form, and Modifications
of such Source Code Form, in each case including portions thereof.
1.5. "Incompatible With Secondary Licenses"
means
a.
that the initial Contributor has attached the notice described in Exhibit B
to the Covered Software; or
b.
that the Covered Software was made available under the terms of version 1.1
or earlier of the License, but not also under the terms of a Secondary
License.
1.6. "Executable Form"
means any form of the work other than Source Code Form.
1.7. "Larger Work"
means a work that combines Covered Software with other material, in a separate
file or files, that is not Covered Software.
1.8. "License"
means this document.
1.9. "Licensable"
means having the right to grant, to the maximum extent possible, whether at the
time of the initial grant or subsequently, any and all of the rights conveyed
by this License.
1.10. "Modifications"
means any of the following:
a.
any file in Source Code Form that results from an addition to, deletion
from, or modification of the contents of Covered Software; or
b.
any new file in Source Code Form that contains any Covered Software.
1.11. "Patent Claims" of a Contributor
means any patent claim(s), including without limitation, method, process, and
apparatus claims, in any patent Licensable by such Contributor that would be
infringed, but for the grant of the License, by the making, using, selling,
offering for sale, having made, import, or transfer of either its Contributions
or its Contributor Version.
1.12. "Secondary License"
means either the GNU General Public License, Version 2.0, the GNU Lesser
General Public License, Version 2.1, the GNU Affero General Public License,
Version 3.0, or any later versions of those licenses.
1.13. "Source Code Form"
means the form of the work preferred for making modifications.
1.14. "You" (or "Your")
means an individual or a legal entity exercising rights under this License. For
legal entities, "You" includes any entity that controls, is controlled by, or
is under common control with You. For purposes of this definition, "control"
means (a) the power, direct or indirect, to cause the direction or management
of such entity, whether by contract or otherwise, or (b) ownership of more than
fifty percent (50%) of the outstanding shares or beneficial ownership of such
entity.
2. License Grants and Conditions
--------------------------------
2.1. Grants
Each Contributor hereby grants You a world-wide, royalty-free, non-exclusive
license:
a.
under intellectual property rights (other than patent or trademark)
Licensable by such Contributor to use, reproduce, make available, modify,
display, perform, distribute, and otherwise exploit its Contributions,
either on an unmodified basis, with Modifications, or as part of a Larger
Work; and
b.
under Patent Claims of such Contributor to make, use, sell, offer for sale,
have made, import, and otherwise transfer either its Contributions or its
Contributor Version.
2.2. Effective Date
The licenses granted in Section 2.1 with respect to any Contribution become
effective for each Contribution on the date the Contributor first distributes
such Contribution.
2.3. Limitations on Grant Scope
The licenses granted in this Section 2 are the only rights granted under this
License. No additional rights or licenses will be implied from the distribution
or licensing of Covered Software under this License. Notwithstanding
Section 2.1(b) above, no patent license is granted by a Contributor:
a.
for any code that a Contributor has removed from Covered Software; or
b.
for infringements caused by: (i) Your and any other third party's
modifications of Covered Software, or (ii) the combination of its
Contributions with other software (except as part of its Contributor
Version); or
c.
under Patent Claims infringed by Covered Software in the absence of its
Contributions.
This License does not grant any rights in the trademarks, service marks, or
logos of any Contributor (except as may be necessary to comply with the notice
requirements in Section 3.4).
2.4. Subsequent Licenses
No Contributor makes additional grants as a result of Your choice to distribute
the Covered Software under a subsequent version of this License (see
Section 10.2) or under the terms of a Secondary License (if permitted under the
terms of Section 3.3).
2.5. Representation
Each Contributor represents that the Contributor believes its Contributions are
its original creation(s) or it has sufficient rights to grant the rights to its
Contributions conveyed by this License.
2.6. Fair Use
This License is not intended to limit any rights You have under applicable
copyright doctrines of fair use, fair dealing, or other equivalents.
2.7. Conditions
Sections 3.1, 3.2, 3.3, and 3.4 are conditions of the licenses granted in
Section 2.1.
3. Responsibilities
-------------------
3.1. Distribution of Source Form
All distribution of Covered Software in Source Code Form, including any
Modifications that You create or to which You contribute, must be under the
terms of this License. You must inform recipients that the Source Code Form of
the Covered Software is governed by the terms of this License, and how they can
obtain a copy of this License. You may not attempt to alter or restrict the
recipients' rights in the Source Code Form.
3.2. Distribution of Executable Form
If You distribute Covered Software in Executable Form then:
a.
such Covered Software must also be made available in Source Code Form, as
described in Section 3.1, and You must inform recipients of the Executable
Form how they can obtain a copy of such Source Code Form by reasonable
means in a timely manner, at a charge no more than the cost of distribution
to the recipient; and
b.
You may distribute such Executable Form under the terms of this License, or
sublicense it under different terms, provided that the license for the
Executable Form does not attempt to limit or alter the recipients' rights
in the Source Code Form under this License.
3.3. Distribution of a Larger Work
You may create and distribute a Larger Work under terms of Your choice,
provided that You also comply with the requirements of this License for the
Covered Software. If the Larger Work is a combination of Covered Software with
a work governed by one or more Secondary Licenses, and the Covered Software is
not Incompatible With Secondary Licenses, this License permits You to
additionally distribute such Covered Software under the terms of such Secondary
License(s), so that the recipient of the Larger Work may, at their option,
further distribute the Covered Software under the terms of either this License
or such Secondary License(s).
3.4. Notices
You may not remove or alter the substance of any license notices (including
copyright notices, patent notices, disclaimers of warranty, or limitations of
liability) contained within the Source Code Form of the Covered Software,
except that You may alter any license notices to the extent required to remedy
known factual inaccuracies.
3.5. Application of Additional Terms
You may choose to offer, and to charge a fee for, warranty, support, indemnity
or liability obligations to one or more recipients of Covered Software.
However, You may do so only on Your own behalf, and not on behalf of any
Contributor. You must make it absolutely clear that any such warranty, support,
indemnity, or liability obligation is offered by You alone, and You hereby
agree to indemnify every Contributor for any liability incurred by such
Contributor as a result of warranty, support, indemnity or liability terms You
offer. You may include additional disclaimers of warranty and limitations of
liability specific to any jurisdiction.
4. Inability to Comply Due to Statute or Regulation
---------------------------------------------------
If it is impossible for You to comply with any of the terms of this License with
respect to some or all of the Covered Software due to statute, judicial order, or
regulation then You must: (a) comply with the terms of this License to the
maximum extent possible; and (b) describe the limitations and the code they
affect. Such description must be placed in a text file included with all
distributions of the Covered Software under this License. Except to the extent
prohibited by statute or regulation, such description must be sufficiently
detailed for a recipient of ordinary skill to be able to understand it.
5. Termination
--------------
5.1. The rights granted under this License will terminate automatically if You
fail to comply with any of its terms. However, if You become compliant, then
the rights granted under this License from a particular Contributor are
reinstated (a) provisionally, unless and until such Contributor explicitly and
finally terminates Your grants, and (b) on an ongoing basis, if such
Contributor fails to notify You of the non-compliance by some reasonable means
prior to 60 days after You have come back into compliance. Moreover, Your
grants from a particular Contributor are reinstated on an ongoing basis if such
Contributor notifies You of the non-compliance by some reasonable means, this
is the first time You have received notice of non-compliance with this License
from such Contributor, and You become compliant prior to 30 days after Your
receipt of the notice.
5.2. If You initiate litigation against any entity by asserting a patent
infringement claim (excluding declaratory judgment actions, counter-claims, and
cross-claims) alleging that a Contributor Version directly or indirectly
infringes any patent, then the rights granted to You by any and all
Contributors for the Covered Software under Section 2.1 of this License shall
terminate.
5.3. In the event of termination under Sections 5.1 or 5.2 above, all end user
license agreements (excluding distributors and resellers) which have been
validly granted by You or Your distributors under this License prior to
termination shall survive termination.
6. Disclaimer of Warranty
-------------------------
Covered Software is provided under this License on an "as is" basis, without
warranty of any kind, either expressed, implied, or statutory, including, without
limitation, warranties that the Covered Software is free of defects,
merchantable, fit for a particular purpose or non-infringing. The entire risk as
to the quality and performance of the Covered Software is with You. Should any
Covered Software prove defective in any respect, You (not any Contributor) assume
the cost of any necessary servicing, repair, or correction. This disclaimer of
warranty constitutes an essential part of this License. No use of any Covered
Software is authorized under this License except under this disclaimer.
7. Limitation of Liability
--------------------------
Under no circumstances and under no legal theory, whether tort (including
negligence), contract, or otherwise, shall any Contributor, or anyone who
distributes Covered Software as permitted above, be liable to You for any direct,
indirect, special, incidental, or consequential damages of any character
including, without limitation, damages for lost profits, loss of goodwill, work
stoppage, computer failure or malfunction, or any and all other commercial
damages or losses, even if such party shall have been informed of the possibility
of such damages. This limitation of liability shall not apply to liability for
death or personal injury resulting from such party's negligence to the extent
applicable law prohibits such limitation. Some jurisdictions do not allow the
exclusion or limitation of incidental or consequential damages, so this exclusion
and limitation may not apply to You.
8. Litigation
-------------
Any litigation relating to this License may be brought only in the courts of a
jurisdiction where the defendant maintains its principal place of business and
such litigation shall be governed by laws of that jurisdiction, without reference
to its conflict-of-law provisions. Nothing in this Section shall prevent a
party's ability to bring cross-claims or counter-claims.
9. Miscellaneous
----------------
This License represents the complete agreement concerning the subject matter
hereof. If any provision of this License is held to be unenforceable, such
provision shall be reformed only to the extent necessary to make it enforceable.
Any law or regulation which provides that the language of a contract shall be
construed against the drafter shall not be used to construe this License against
a Contributor.
10. Versions of the License
---------------------------
10.1. New Versions
Mozilla Foundation is the license steward. Except as provided in Section 10.3,
no one other than the license steward has the right to modify or publish new
versions of this License. Each version will be given a distinguishing version
number.
10.2. Effect of New Versions
You may distribute the Covered Software under the terms of the version of the
License under which You originally received the Covered Software, or under the
terms of any subsequent version published by the license steward.
10.3. Modified Versions
If you create software not governed by this License, and you want to create a
new license for such software, you may create and use a modified version of
this License if you rename the license and remove any references to the name of
the license steward (except to note that such modified license differs from
this License).
10.4. Distributing Source Code Form that is Incompatible With Secondary
Licenses
If You choose to distribute Source Code Form that is Incompatible With
Secondary Licenses under the terms of this version of the License, the notice
described in Exhibit B of this License must be attached.
Exhibit A - Source Code Form License Notice
-------------------------------------------
This Source Code Form is subject to the terms of the Mozilla Public License,
v. 2.0. If a copy of the MPL was not distributed with this file, You can
obtain one at http://mozilla.org/MPL/2.0/.
If it is not possible or desirable to put the notice in a particular file, then
You may include the notice in a location (such as a LICENSE file in a relevant
directory) where a recipient would be likely to look for such a notice.
You may add additional accurate notices of copyright ownership.
Exhibit B - "Incompatible With Secondary Licenses" Notice
---------------------------------------------------------
This Source Code Form is "Incompatible With Secondary Licenses", as defined
by the Mozilla Public License, v. 2.0.
THIRD_PARTY/licenses/PSF-2.0.txt
+46
View File
@@ -0,0 +1,46 @@
PYTHON SOFTWARE FOUNDATION LICENSE VERSION 2
============================================
--------------------------------------------
1. This LICENSE AGREEMENT is between the Python Software Foundation ("PSF"),
and the Individual or Organization ("Licensee") accessing and otherwise using
this software ("Python") in source or binary form and its associated
documentation.
2. Subject to the terms and conditions of this License Agreement, PSF hereby
grants Licensee a nonexclusive, royalty-free, world-wide license to
reproduce, analyze, test, perform and/or display publicly, prepare derivative
works, distribute, and otherwise use Python alone or in any derivative
version, provided, however, that PSF's License Agreement and PSF's notice of
copyright, i.e., "Copyright (c) 2001, 2002, 2003, 2004 Python Software
Foundation; All Rights Reserved" are retained in Python alone or in any
derivative version prepared by Licensee.
3. In the event Licensee prepares a derivative work that is based on or
incorporates Python or any part thereof, and wants to make the derivative
work available to others as provided herein, then Licensee hereby agrees to
include in any such work a brief summary of the changes made to Python.
4. PSF is making Python available to Licensee on an "AS IS" basis. PSF MAKES
NO REPRESENTATIONS OR WARRANTIES, EXPRESS OR IMPLIED. BY WAY OF EXAMPLE, BUT
NOT LIMITATION, PSF MAKES NO AND DISCLAIMS ANY REPRESENTATION OR WARRANTY OF
MERCHANTABILITY OR FITNESS FOR ANY PARTICULAR PURPOSE OR THAT THE USE OF
PYTHON WILL NOT INFRINGE ANY THIRD PARTY RIGHTS.
5. PSF SHALL NOT BE LIABLE TO LICENSEE OR ANY OTHER USERS OF PYTHON FOR ANY
INCIDENTAL, SPECIAL, OR CONSEQUENTIAL DAMAGES OR LOSS AS A RESULT OF
MODIFYING, DISTRIBUTING, OR OTHERWISE USING PYTHON, OR ANY DERIVATIVE
THEREOF, EVEN IF ADVISED OF THE POSSIBILITY THEREOF.
6. This License Agreement will automatically terminate upon a material breach
of its terms and conditions.
7. Nothing in this License Agreement shall be deemed to create any
relationship of agency, partnership, or joint venture between PSF and
Licensee. This License Agreement does not grant permission to use PSF
trademarks or trade name in a trademark sense to endorse or promote products
or services of Licensee, or any third party.
8. By copying, installing or otherwise using Python, Licensee agrees to be
bound by the terms and conditions of this License Agreement.
THIRD_PARTY/licenses/Plexus Classworlds License.txt
+37
View File
@@ -0,0 +1,37 @@
Plexus Classworlds License
==========================
Copyright 2002 (C) The Codehaus. All Rights Reserved.
Redistribution and use of this software and associated documentation
("Software"), with or without modification, are permitted provided that the
following conditions are met:
1. Redistributions of source code must retain copyright statements and notices.
Redistributions must also contain a copy of this document.
2. Redistributions in binary form must reproduce the above copyright notice,
this list of conditions and the following disclaimer in the documentation
and/or other materials provided with the distribution.
3. The name "classworlds" must not be used to endorse or promote products
derived from this Software without prior written permission of The Codehaus.
For written permission, please contact bob@codehaus.org.
4. Products derived from this Software may not be called "classworlds" nor may
"classworlds" appear in their names without prior written permission of The
Codehaus. "classworlds" is a registered trademark of The Codehaus.
5. Due credit should be given to The Codehaus.
(http://classworlds.codehaus.org/).
THIS SOFTWARE IS PROVIDED BY THE CODEHAUS AND CONTRIBUTORS "AS IS" AND ANY
EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
DISCLAIMED. IN NO EVENT SHALL THE CODEHAUS OR ITS CONTRIBUTORS BE LIABLE FOR ANY
DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS
OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN
IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
build-scripts/fetch_shaded_jar.py
+80
View File
@@ -0,0 +1,80 @@
"""
Finds and copies the latest shaded JAR from the Java build to the Python package source.
This script is intended to be run from the monorepo root, typically as part of a
CI/CD pipeline, before the Python package is built.
"""
import argparse
import logging
import re
import shutil
import sys
from pathlib import Path
from typing import Optional
# Requires 'packaging' library (pip install packaging)
from packaging.version import parse as parse_version
def find_latest_jar_by_semver(target_dir: Path) -> Optional[Path]:
"""Finds the shaded JAR with the highest semantic version in its filename."""
# Example filename: opendataloader-pdf-runtime-0.1.0.jar
jar_pattern = "opendataloader-pdf-runtime-*.jar"
version_regex = re.compile(r"opendataloader-pdf-runtime-(.+?)\.jar")
latest_version = parse_version("0.0.0")
latest_jar_path = None
# Exclude Maven's 'original' JARs to ensure we get the shaded (fat) JAR.
potential_jars = [p for p in target_dir.glob(jar_pattern) if 'original' not in p.name]
if not potential_jars:
return None
# Iterate through potential JARs to find the one with the highest version number.
for jar_path in potential_jars:
match = version_regex.search(jar_path.name)
if match:
try:
current_version = parse_version(match.group(1))
if current_version > latest_version:
latest_version = current_version
latest_jar_path = jar_path
except Exception:
# Ignore files with non-parseable version strings.
continue
return latest_jar_path
def main():
"""Parse command-line arguments and orchestrate the copy process."""
logging.basicConfig(level=logging.INFO, format='%(levelname)s: %(message)s', stream=sys.stdout)
parser = argparse.ArgumentParser(description="Copies the latest shaded JAR to the Python source tree.")
parser.add_argument("java_target_dir", type=Path, help="Path to the Java module's 'target' directory.")
parser.add_argument("python_jars_dir", type=Path, help="Path to the Python package's destination directory for JARs.")
args = parser.parse_args()
java_target_path: Path = args.java_target_dir.resolve()
python_jars_path: Path = args.python_jars_dir.resolve()
if not java_target_path.is_dir():
parser.error(f"Java target directory not found: {java_target_path}")
# Ensure the destination directory exists.
python_jars_path.mkdir(parents=True, exist_ok=True)
source_jar_path = find_latest_jar_by_semver(java_target_path)
if not source_jar_path:
parser.error(f"No versioned shaded JAR found in: {java_target_path}")
# Standardize the destination name for consistent access within the Python package.
destination_jar_path = python_jars_path / 'runtime.jar'
shutil.copy2(source_jar_path, destination_jar_path)
logging.info(f"Copied '{source_jar_path.name}' to '{destination_jar_path}'")
if __name__ == "__main__":
main()
build-scripts/set_version.py
+45
View File
@@ -0,0 +1,45 @@
# build-scripts/set_version.py
import os
import re
import sys
def set_version(version_file, pom_file, pyproject_toml_file):
with open(version_file, 'r') as f:
version = f.read().strip()
# Update Maven POM
with open(pom_file, 'r') as f:
pom_content = f.read()
pom_content = re.sub(r'<version>.*</version>', f'<version>{version}</version>', pom_content, count=1)
with open(pom_file, 'w') as f:
f.write(pom_content)
print(f"Updated Maven POM version to {version}")
# Update Python pyproject.toml
with open(pyproject_toml_file, 'r') as f:
pyproject_content = f.read()
pyproject_content = re.sub(r'version = ".*"', f'version = "{version}"', pyproject_content, count=1)
with open(pyproject_toml_file, 'w') as f:
f.write(pyproject_content)
print(f"Updated Python pyproject.toml version to {version}")
if __name__ == "__main__":
# Paths are relative to the monorepo root
root_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))
version_path = os.path.join(root_dir, 'VERSION')
java_pom_path = os.path.join(root_dir, 'java', 'pom.xml')
python_pyproject_path = os.path.join(root_dir, 'python', 'packages', 'opendataloader_pdf', 'pyproject.toml')
if not os.path.exists(version_path):
print(f"Error: VERSION file not found at {version_path}")
sys.exit(1)
if not os.path.exists(java_pom_path):
print(f"Error: Java pom.xml not found at {java_pom_path}")
sys.exit(1)
if not os.path.exists(python_pyproject_path):
print(f"Error: Python pyproject.toml not found at {python_pyproject_path}")
sys.exit(1)
set_version(version_path, java_pom_path, python_pyproject_path)
docs/hybrid/docling-speed-optimization-plan.md
+301
View File
@@ -0,0 +1,301 @@
# Docling Speed Optimization Plan
## Progress Tracker
| Task | Status | Completed | Result |
|------|--------|-----------|--------|
| Phase 0: Baseline measurement | ✅ completed | 2026-01-03 | 2.283s/doc |
| Phase 0: FastAPI experiment | ✅ completed | 2026-01-03 | 0.685s/doc (PASS < 0.8s) |
| Phase 0: subprocess experiment | ✅ completed | 2026-01-03 | 0.661s/doc (PASS < 1.0s) |
| Phase 0: Results comparison | ✅ completed | 2026-01-03 | 3.3x-3.5x speedup |
| Task 1.1: docling_subprocess_worker.py | ⏭️ skipped | - | FastAPI only |
| Task 1.2: hybrid_server.py | ✅ completed | 2026-01-03 | opendataloader-pdf-hybrid |
| Task 2.1: DoclingSubprocessClient.java | ⏭️ skipped | - | FastAPI only |
| Task 2.2: DoclingFastServerClient.java | ✅ completed | 2026-01-03 | - |
| Task 2.3: HybridClientFactory modification | ✅ completed | 2026-01-03 | docling-fast only |
| Task 3.1: pdf_parser modules | ✅ completed | 2026-01-03 | docling-fast only |
| Task 3.2: engine_registry.py | ✅ completed | 2026-01-03 | - |
| Task 3.3: run.py CLI options | ✅ completed | 2026-01-03 | - |
| Task 4.1: Full benchmark | ✅ completed | 2026-01-03 | See experiments/speed/ |
| Task 4.2: Results documentation | ✅ completed | 2026-01-03 | speed-experiment-2026-01-03.md |
**Status Legend:**
- ⬜ `not_started` - Not yet begun
- 🔄 `in_progress` - Currently working
- ✅ `completed` - Done and verified
- ⏭️ `skipped` - Excluded from plan
- ⏸️ `blocked` - Waiting on dependency
- ❌ `failed` - Did not meet criteria
- 🚫 `discarded` - Plan abandoned
---
## 1. Background
### Current Problem
- **DoclingClient** (docling-serve HTTP API): ~2 seconds per page
- **docling SDK direct call**: ~0.5 seconds per document (user-reported)
- HTTP overhead negates the speed benefits of hybrid mode
### Goal
Implement alternative approaches to efficiently call the docling SDK, then compare benchmark speeds
---
## 2. Experiment Phase (Phase 0)
### Purpose
Validate the speed improvement hypothesis before full implementation
### Experiment Targets
| Approach | Description |
|----------|-------------|
| baseline | Current docling-serve (reference) |
| fastapi | Optimized FastAPI server |
| subprocess | Direct Python subprocess call |
### Success Criteria
| Approach | Threshold | Condition |
|----------|-----------|-----------|
| fastapi | **< 0.8 sec/doc** (average) | Based on 200 documents |
| subprocess | **< 1.0 sec/doc** (average) | Based on 200 documents |
### Failure Conditions
- If fastapi approach exceeds 0.8 sec/doc: **Discard entire plan**
- If only subprocess fails: Exclude that approach only
### Experiment Environment
- Benchmark PDFs: `tests/benchmark/pdfs/` (200 files)
- Settings: `do_ocr=true`, `do_table_structure=true`
- Measurement: `total_time / document_count`
### Experiment Scripts
```
scripts/experiments/
├── docling_baseline_bench.py # docling-serve speed measurement
├── docling_fastapi_bench.py # FastAPI server + client test
├── docling_subprocess_bench.py # subprocess approach test
└── docling_speed_report.py # Results comparison report
```
### Experiment Execution
```bash
# 1. baseline (requires docling-serve running)
python scripts/experiments/docling_baseline_bench.py
# 2. fastapi (server auto-starts)
python scripts/experiments/docling_fastapi_bench.py
# 3. subprocess
python scripts/experiments/docling_subprocess_bench.py
# 4. compare results
python scripts/experiments/docling_speed_report.py
```
### Results Recording
```
docs/hybrid/experiments/
└── speed-experiment-YYYY-MM-DD.md
```
---
## 3. Implementation Tasks (After Phase 0 Success)
### Task 1: Python Scripts
#### Task 1.1: docling_subprocess_worker.py
| Item | Details |
|------|---------|
| File | `scripts/docling_subprocess_worker.py` |
| Prerequisites | docling package installed |
| Description | stdin JSON → stdout JSON conversion |
| Success Criteria | Single PDF conversion succeeds, JSON output parseable |
| Test | `echo '{"pdf_path":"/path/to.pdf"}' \| python scripts/docling_subprocess_worker.py` |
#### Task 1.2: hybrid_server.py
| Item | Details |
|------|---------|
| File | `python/opendataloader-pdf/src/opendataloader_pdf/hybrid_server.py` |
| Prerequisites | `pip install opendataloader-pdf[hybrid]` |
| Description | POST /convert endpoint, DocumentConverter singleton |
| Success Criteria | curl PDF upload returns JSON response |
| Test | `opendataloader-pdf-hybrid &` then `curl -F "file=@test.pdf" http://localhost:5002/v1/convert/file` |
### Task 2: Java Client Implementation
#### Task 2.1: DoclingSubprocessClient.java
| Item | Details |
|------|---------|
| File | `java/.../hybrid/DoclingSubprocessClient.java` |
| Prerequisites | Task 1.1 complete |
| Description | ProcessBuilder executes Python, stdin/stdout JSON |
| Success Criteria | Implements HybridClient interface, single PDF conversion succeeds |
| Test | `DoclingSubprocessClientTest.java` unit tests pass |
#### Task 2.2: DoclingFastServerClient.java
| Item | Details |
|------|---------|
| File | `java/.../hybrid/DoclingFastServerClient.java` |
| Prerequisites | Task 1.2 complete |
| Description | OkHttp calls FastAPI server |
| Success Criteria | Implements HybridClient interface, single PDF conversion succeeds |
| Test | `DoclingFastServerClientTest.java` unit tests pass |
#### Task 2.3: HybridClientFactory Modification
| Item | Details |
|------|---------|
| File | `java/.../hybrid/HybridClientFactory.java` |
| Prerequisites | Task 2.1, 2.2 complete |
| Description | Register `docling-subprocess`, `docling-fast` backends |
| Success Criteria | `HybridClientFactory.getOrCreate("docling-fast", config)` works |
| Test | Extend `HybridClientFactoryTest.java` |
### Task 3: Benchmark Integration
#### Task 3.1: Add pdf_parser Modules
| Item | Details |
|------|---------|
| Files | `tests/benchmark/src/pdf_parser_opendataloader_hybrid_subprocess.py` |
| | `tests/benchmark/src/pdf_parser_opendataloader_hybrid_fast.py` |
| Prerequisites | Task 2.3 complete, JAR built |
| Success Criteria | Benchmark runs with `--hybrid docling-subprocess` option |
#### Task 3.2: Modify engine_registry.py
| Item | Details |
|------|---------|
| File | `tests/benchmark/src/engine_registry.py` |
| Description | Register new engines |
| Success Criteria | New engines queryable from ENGINE_DISPATCH |
#### Task 3.3: Add run.py CLI Options
| Item | Details |
|------|---------|
| File | `tests/benchmark/run.py` |
| Description | Extend `--hybrid` choices |
| Success Criteria | `./scripts/bench.sh --hybrid docling-fast` runs |
### Task 4: Final Validation
#### Task 4.1: Full Benchmark Execution
| Item | Details |
|------|---------|
| Prerequisites | Task 3 complete |
| Execution | Benchmark 200 documents with all 3 approaches |
| Success Criteria | elapsed_per_doc comparison shows meaningful improvement |
#### Task 4.2: Results Documentation
| Item | Details |
|------|---------|
| File | `docs/hybrid/docling-speed-optimization-results.md` |
| Content | Speed comparison table, recommended approach, usage guide |
---
## 4. Task Workflow
```
Phase 0: Experiment
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
┌─────────────────┐
│ baseline measure │
└────────┬────────┘
│
┌──────────────┴──────────────┐
▼ ▼
┌─────────────────┐ ┌─────────────────┐
│ fastapi test │ │ subprocess test │
└────────┬────────┘ └────────┬────────┘
│ │
└──────────────┬──────────────┘
▼
┌─────────────────┐
│ compare results │
│ < 0.8 sec/doc? │
└────────┬────────┘
│
┌──────────────┴──────────────┐
▼ ▼
[SUCCESS] [FAILURE]
Proceed to Discard plan
Phase 1
Phase 1~4: Implementation (parallelizable)
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
Task 1.1 ─────────────────► Task 2.1 ─┐
(subprocess worker) (Java client) │
│
Task 1.2 ─────────────────► Task 2.2 ─┼─► Task 2.3 ─► Task 3 ─► Task 4
(fastapi server) (Java client) │ (Factory) (Bench) (Validate)
│
◄──── parallelizable ────► │
```
### Parallelizable Tasks
| Group | Tasks | Notes |
|-------|-------|-------|
| Phase 0 | fastapi test, subprocess test | After baseline measurement |
| Phase 1 | Task 1.1, Task 1.2 | Independent |
| Phase 2 | Task 2.1, Task 2.2 | Depend on Task 1.1, 1.2 respectively |
| Phase 3 | Task 3.1, 3.2, 3.3 | After Task 2.3 complete |
### Dependencies
```
Task 1.1 → Task 2.1 ─┐
├─► Task 2.3 → Task 3.* → Task 4.*
Task 1.2 → Task 2.2 ─┘
```
---
## 5. File List
### New Files
| File | Phase | Description |
|------|-------|-------------|
| `scripts/experiments/docling_baseline_bench.py` | 0 | Baseline measurement |
| `scripts/experiments/docling_fastapi_bench.py` | 0 | FastAPI experiment |
| `scripts/experiments/docling_subprocess_bench.py` | 0 | Subprocess experiment |
| `scripts/experiments/docling_speed_report.py` | 0 | Results report |
| `scripts/docling_subprocess_worker.py` | 1 | Subprocess worker (skipped) |
| `python/.../hybrid_server.py` | 1 | FastAPI server (opendataloader-pdf-hybrid) |
| `java/.../hybrid/DoclingSubprocessClient.java` | 2 | Java client |
| `java/.../hybrid/DoclingFastServerClient.java` | 2 | Java client |
| `tests/.../pdf_parser_opendataloader_hybrid_subprocess.py` | 3 | Benchmark parser |
| `tests/.../pdf_parser_opendataloader_hybrid_fast.py` | 3 | Benchmark parser |
### Modified Files
| File | Phase | Changes |
|------|-------|---------|
| `java/.../hybrid/HybridClientFactory.java` | 2 | Register new backends |
| `tests/benchmark/src/engine_registry.py` | 3 | Register engines |
| `tests/benchmark/run.py` | 3 | CLI options |
---
## 6. Risks and Mitigations
| Risk | Probability | Mitigation |
|------|-------------|------------|
| FastAPI speed below threshold | Medium | Discard plan, explore other approaches |
| subprocess overhead | Medium | Consider process pooling |
| docling SDK version compatibility | Low | Pin version, test |
| Memory exhaustion | Low | Adjust batch size |
---
## 7. Checklist
### Phase 0 Completion Criteria
- [ ] Baseline speed measurement complete
- [ ] FastAPI experiment: < 0.8 sec/doc
- [ ] subprocess experiment: < 1.0 sec/doc
- [ ] Experiment results documented
### Overall Completion Criteria
- [ ] All Tasks complete
- [ ] Benchmark runs successfully with all 3 approaches
- [ ] Speed improvement confirmed (vs baseline)
- [ ] Results documented
docs/hybrid/experiments/chunking_strategy/conclusion.json
+32
View File
@@ -0,0 +1,32 @@
{
"conclusion": "Optimized ranges (consecutive page merging) is always the best strategy",
"recommendation": "Merge consecutive target pages into ranges before calling convert()",
"results": {
"25% pages": {
"best_method": "optimized_ranges",
"best_time": 4.223,
"chunks": 2
},
"50% pages": {
"best_method": "optimized_ranges",
"best_time": 6.052,
"chunks": 3
},
"75% pages": {
"best_method": "optimized_ranges",
"best_time": 9.347,
"chunks": 5
},
"100% pages": {
"best_method": "optimized_ranges",
"best_time": 10.277,
"chunks": 1
}
},
"key_findings": [
"Fixed chunk sizes always have overhead due to processing unnecessary pages",
"Single page chunks (chunk_1) have 7-33% overhead from extra API calls",
"Larger fixed chunks (chunk_2, 3, 5) have 17-88% overhead from processing unneeded pages",
"Optimized ranges minimize both API calls and unnecessary page processing"
]
}
docs/hybrid/experiments/chunking_strategy/docling_benchmark_report.json
+816
View File
@@ -0,0 +1,816 @@
{
"metadata": {
"pdf_file": "1901.03003.pdf",
"total_pages": 15,
"warmup_runs": 1,
"measure_runs": 3,
"chunk_sizes": [
1,
2,
3,
5
],
"random_seed": 42,
"timestamp": "2026-01-02T16:04:13.655948"
},
"scenarios": [
{
"scenario": "25% pages",
"total_pages": 15,
"target_pages": [
1,
2,
11
],
"target_page_count": 3,
"percentage": 20.0,
"results": [
{
"method": "optimized_ranges",
"ranges": [
[
1,
2
],
[
11,
11
]
],
"num_chunks": 2,
"avg_time": 4.223,
"std_time": 0.042,
"times": [
4.203,
4.282,
4.184
],
"overhead_pct": 0.0
},
{
"method": "chunk_1",
"chunk_size": 1,
"chunks": [
[
1,
1
],
[
2,
2
],
[
11,
11
]
],
"num_chunks": 3,
"avg_time": 4.517,
"std_time": 0.082,
"times": [
4.444,
4.475,
4.631
],
"overhead_pct": 7.0
},
{
"method": "chunk_2",
"chunk_size": 2,
"chunks": [
[
1,
2
],
[
11,
12
]
],
"num_chunks": 2,
"avg_time": 5.257,
"std_time": 0.177,
"times": [
5.502,
5.088,
5.182
],
"overhead_pct": 24.5
},
{
"method": "chunk_3",
"chunk_size": 3,
"chunks": [
[
1,
3
],
[
10,
12
]
],
"num_chunks": 2,
"avg_time": 6.502,
"std_time": 0.137,
"times": [
6.309,
6.612,
6.584
],
"overhead_pct": 54.0
},
{
"method": "chunk_5",
"chunk_size": 5,
"chunks": [
[
1,
5
],
[
11,
15
]
],
"num_chunks": 2,
"avg_time": 7.122,
"std_time": 0.025,
"times": [
7.13,
7.148,
7.089
],
"overhead_pct": 68.7
}
],
"best_method": "Optimized ranges",
"best_time": 4.223
},
{
"scenario": "50% pages",
"total_pages": 15,
"target_pages": [
2,
3,
4,
5,
9,
12,
13
],
"target_page_count": 7,
"percentage": 46.7,
"results": [
{
"method": "optimized_ranges",
"ranges": [
[
2,
5
],
[
9,
9
],
[
12,
13
]
],
"num_chunks": 3,
"avg_time": 6.052,
"std_time": 0.128,
"times": [
5.899,
6.212,
6.045
],
"overhead_pct": 0.0
},
{
"method": "chunk_1",
"chunk_size": 1,
"chunks": [
[
2,
2
],
[
3,
3
],
[
4,
4
],
[
5,
5
],
[
9,
9
],
[
12,
12
],
[
13,
13
]
],
"num_chunks": 7,
"avg_time": 6.548,
"std_time": 0.128,
"times": [
6.714,
6.528,
6.403
],
"overhead_pct": 8.2
},
{
"method": "chunk_2",
"chunk_size": 2,
"chunks": [
[
1,
2
],
[
3,
4
],
[
5,
6
],
[
9,
10
],
[
11,
12
],
[
13,
14
]
],
"num_chunks": 6,
"avg_time": 10.657,
"std_time": 0.062,
"times": [
10.731,
10.659,
10.58
],
"overhead_pct": 76.1
},
{
"method": "chunk_3",
"chunk_size": 3,
"chunks": [
[
1,
3
],
[
4,
6
],
[
7,
9
],
[
10,
12
],
[
13,
15
]
],
"num_chunks": 5,
"avg_time": 11.412,
"std_time": 0.071,
"times": [
11.442,
11.479,
11.313
],
"overhead_pct": 88.6
},
{
"method": "chunk_5",
"chunk_size": 5,
"chunks": [
[
1,
5
],
[
6,
10
],
[
11,
15
]
],
"num_chunks": 3,
"avg_time": 10.899,
"std_time": 0.016,
"times": [
10.922,
10.889,
10.886
],
"overhead_pct": 80.1
}
],
"best_method": "Optimized ranges",
"best_time": 6.052
},
{
"scenario": "75% pages",
"total_pages": 15,
"target_pages": [
1,
2,
4,
5,
7,
9,
10,
11,
12,
13,
15
],
"target_page_count": 11,
"percentage": 73.3,
"results": [
{
"method": "optimized_ranges",
"ranges": [
[
1,
2
],
[
4,
5
],
[
7,
7
],
[
9,
13
],
[
15,
15
]
],
"num_chunks": 5,
"avg_time": 9.347,
"std_time": 0.112,
"times": [
9.19,
9.412,
9.44
],
"overhead_pct": 0.0
},
{
"method": "chunk_1",
"chunk_size": 1,
"chunks": [
[
1,
1
],
[
2,
2
],
[
4,
4
],
[
5,
5
],
[
7,
7
],
[
9,
9
],
[
10,
10
],
[
11,
11
],
[
12,
12
],
[
13,
13
],
[
15,
15
]
],
"num_chunks": 11,
"avg_time": 11.15,
"std_time": 0.082,
"times": [
11.043,
11.242,
11.165
],
"overhead_pct": 19.3
},
{
"method": "chunk_2",
"chunk_size": 2,
"chunks": [
[
1,
2
],
[
3,
4
],
[
5,
6
],
[
7,
8
],
[
9,
10
],
[
11,
12
],
[
13,
14
],
[
15,
15
]
],
"num_chunks": 8,
"avg_time": 12.039,
"std_time": 0.112,
"times": [
12.141,
12.092,
11.883
],
"overhead_pct": 28.8
},
{
"method": "chunk_3",
"chunk_size": 3,
"chunks": [
[
1,
3
],
[
4,
6
],
[
7,
9
],
[
10,
12
],
[
13,
15
]
],
"num_chunks": 5,
"avg_time": 12.503,
"std_time": 0.374,
"times": [
12.279,
13.029,
12.2
],
"overhead_pct": 33.8
},
{
"method": "chunk_5",
"chunk_size": 5,
"chunks": [
[
1,
5
],
[
6,
10
],
[
11,
15
]
],
"num_chunks": 3,
"avg_time": 11.288,
"std_time": 0.39,
"times": [
11.06,
11.838,
10.967
],
"overhead_pct": 20.8
}
],
"best_method": "Optimized ranges",
"best_time": 9.347
},
{
"scenario": "100% pages",
"total_pages": 15,
"target_pages": [
1,
2,
3,
4,
5,
6,
7,
8,
9,
10,
11,
12,
13,
14,
15
],
"target_page_count": 15,
"percentage": 100.0,
"results": [
{
"method": "optimized_ranges",
"ranges": [
[
1,
15
]
],
"num_chunks": 1,
"avg_time": 10.277,
"std_time": 0.263,
"times": [
10.074,
10.648,
10.11
],
"overhead_pct": 0.0
},
{
"method": "chunk_1",
"chunk_size": 1,
"chunks": [
[
1,
1
],
[
2,
2
],
[
3,
3
],
[
4,
4
],
[
5,
5
],
[
6,
6
],
[
7,
7
],
[
8,
8
],
[
9,
9
],
[
10,
10
],
[
11,
11
],
[
12,
12
],
[
13,
13
],
[
14,
14
],
[
15,
15
]
],
"num_chunks": 15,
"avg_time": 13.625,
"std_time": 0.154,
"times": [
13.509,
13.524,
13.843
],
"overhead_pct": 32.6
},
{
"method": "chunk_2",
"chunk_size": 2,
"chunks": [
[
1,
2
],
[
3,
4
],
[
5,
6
],
[
7,
8
],
[
9,
10
],
[
11,
12
],
[
13,
14
],
[
15,
15
]
],
"num_chunks": 8,
"avg_time": 12.48,
"std_time": 0.039,
"times": [
12.531,
12.437,
12.472
],
"overhead_pct": 21.4
},
{
"method": "chunk_3",
"chunk_size": 3,
"chunks": [
[
1,
3
],
[
4,
6
],
[
7,
9
],
[
10,
12
],
[
13,
15
]
],
"num_chunks": 5,
"avg_time": 12.104,
"std_time": 0.214,
"times": [
12.301,
12.204,
11.806
],
"overhead_pct": 17.8
},
{
"method": "chunk_5",
"chunk_size": 5,
"chunks": [
[
1,
5
],
[
6,
10
],
[
11,
15
]
],
"num_chunks": 3,
"avg_time": 12.277,
"std_time": 0.334,
"times": [
12.108,
12.742,
11.98
],
"overhead_pct": 19.5
}
],
"best_method": "Optimized ranges",
"best_time": 10.277
}
],
"summary": {
"25% pages": {
"best_method": "optimized_ranges",
"best_time": 4.223,
"best_chunks": 2
},
"50% pages": {
"best_method": "optimized_ranges",
"best_time": 6.052,
"best_chunks": 3
},
"75% pages": {
"best_method": "optimized_ranges",
"best_time": 9.347,
"best_chunks": 5
},
"100% pages": {
"best_method": "optimized_ranges",
"best_time": 10.277,
"best_chunks": 1
}
}
}
docs/hybrid/experiments/chunking_strategy/docling_page_range_benchmark.py
+291
View File
@@ -0,0 +1,291 @@
#!/usr/bin/env python3
"""
Docling Page Range Benchmark
페이지 범위별 변환 성능 비교:
- 25%, 50%, 75%, 100% 페이지 시나리오
- 각 시나리오별 최적 청크 크기 탐색
워밍업 후 여러 번 실행하여 평균 측정
결과는 JSON으로 저장
"""
import json
import time
import random
from pathlib import Path
from dataclasses import dataclass, asdict
from datetime import datetime
from docling.document_converter import DocumentConverter, PdfFormatOption
from docling.datamodel.base_models import InputFormat
from docling.datamodel.pipeline_options import PdfPipelineOptions
WARMUP_RUNS = 1
MEASURE_RUNS = 3
RANDOM_SEED = 42
@dataclass
class BenchmarkResult:
name: str
avg_time: float
std_time: float
times: list[float]
chunk_size: int
num_chunks: int
def get_project_root() -> Path:
"""프로젝트 루트 디렉토리 반환"""
return Path(__file__).parent.parent.parent
def create_converter() -> DocumentConverter:
"""DocumentConverter 인스턴스 생성"""
pipeline_options = PdfPipelineOptions()
return DocumentConverter(
format_options={
InputFormat.PDF: PdfFormatOption(pipeline_options=pipeline_options)
}
)
def convert_with_page_range(
converter: DocumentConverter, pdf_path: Path, start: int, end: int
) -> float:
"""지정된 페이지 범위로 변환하고 소요 시간 반환"""
start_time = time.perf_counter()
converter.convert(pdf_path, page_range=(start, end))
return time.perf_counter() - start_time
def pages_to_ranges(pages: list[int]) -> list[tuple[int, int]]:
"""페이지 리스트를 연속 범위로 변환"""
if not pages:
return []
pages = sorted(pages)
ranges = []
start = pages[0]
end = pages[0]
for p in pages[1:]:
if p == end + 1:
end = p
else:
ranges.append((start, end))
start = p
end = p
ranges.append((start, end))
return ranges
def run_benchmark_for_ranges(
pdf_path: Path,
ranges: list[tuple[int, int]],
name: str,
) -> BenchmarkResult:
"""주어진 범위들에 대해 벤치마크 실행"""
def run_once():
converter = create_converter()
total_time = 0.0
for start, end in ranges:
total_time += convert_with_page_range(converter, pdf_path, start, end)
return total_time
# 워밍업
for _ in range(WARMUP_RUNS):
run_once()
# 측정
times = []
for _ in range(MEASURE_RUNS):
times.append(run_once())
avg_time = sum(times) / len(times)
std_time = (sum((t - avg_time) ** 2 for t in times) / len(times)) ** 0.5
return BenchmarkResult(
name=name,
avg_time=avg_time,
std_time=std_time,
times=times,
chunk_size=0,
num_chunks=len(ranges),
)
def get_chunks_for_pages(
target_pages: list[int], chunk_size: int, total_pages: int
) -> list[tuple[int, int]]:
"""타겟 페이지들을 청크 크기로 그룹화"""
chunks = []
for page in target_pages:
chunk_start = ((page - 1) // chunk_size) * chunk_size + 1
chunk_end = min(chunk_start + chunk_size - 1, total_pages)
if (chunk_start, chunk_end) not in chunks:
chunks.append((chunk_start, chunk_end))
return chunks
def run_scenario_benchmark(
pdf_path: Path,
total_pages: int,
target_pages: list[int],
chunk_sizes: list[int],
scenario_name: str,
) -> dict:
"""단일 시나리오 벤치마크 실행"""
print(f"\n{'='*60}")
print(f"Scenario: {scenario_name}")
print(f"{'='*60}")
print(f"Target pages ({len(target_pages)}): {target_pages}")
print()
results = []
scenario_data = {
"scenario": scenario_name,
"total_pages": total_pages,
"target_pages": target_pages,
"target_page_count": len(target_pages),
"percentage": round(len(target_pages) / total_pages * 100, 1),
"results": [],
}
# 1. 연속 범위 최적화
optimized_ranges = pages_to_ranges(target_pages)
print(f"[1] Optimized ranges: {optimized_ranges} ({len(optimized_ranges)} ranges)")
opt_result = run_benchmark_for_ranges(pdf_path, optimized_ranges, "Optimized ranges")
results.append(opt_result)
print(f" Avg: {opt_result.avg_time:.2f}s (±{opt_result.std_time:.2f}s)")
scenario_data["results"].append({
"method": "optimized_ranges",
"ranges": optimized_ranges,
"num_chunks": len(optimized_ranges),
"avg_time": round(opt_result.avg_time, 3),
"std_time": round(opt_result.std_time, 3),
"times": [round(t, 3) for t in opt_result.times],
"overhead_pct": 0.0,
})
# 2. 각 청크 크기별 테스트
for chunk_size in chunk_sizes:
chunks = get_chunks_for_pages(target_pages, chunk_size, total_pages)
print(f"[{len(results) + 1}] {chunk_size} page(s)/chunk ({len(chunks)} chunks)")
result = run_benchmark_for_ranges(pdf_path, chunks, f"{chunk_size} page(s)/chunk")
result.chunk_size = chunk_size
results.append(result)
overhead_pct = ((result.avg_time - opt_result.avg_time) / opt_result.avg_time) * 100
print(f" Avg: {result.avg_time:.2f}s (±{result.std_time:.2f}s) [{overhead_pct:+.1f}%]")
scenario_data["results"].append({
"method": f"chunk_{chunk_size}",
"chunk_size": chunk_size,
"chunks": chunks,
"num_chunks": len(chunks),
"avg_time": round(result.avg_time, 3),
"std_time": round(result.std_time, 3),
"times": [round(t, 3) for t in result.times],
"overhead_pct": round(overhead_pct, 1),
})
# Best 찾기
best_result = min(results, key=lambda r: r.avg_time)
scenario_data["best_method"] = best_result.name
scenario_data["best_time"] = round(best_result.avg_time, 3)
print()
print(f" >> Best: {best_result.name} ({best_result.avg_time:.2f}s)")
return scenario_data
def main():
project_root = get_project_root()
pdf_path = project_root / "samples" / "pdf" / "1901.03003.pdf"
if not pdf_path.exists():
print(f"Error: PDF not found at {pdf_path}")
return 1
total_pages = 15
chunk_sizes = [1, 2, 3, 5]
percentages = [25, 50, 75, 100]
print("=" * 60)
print("Docling Page Range Benchmark - Multi Scenario")
print("=" * 60)
print(f"PDF: {pdf_path.name} ({total_pages} pages)")
print(f"Warmup: {WARMUP_RUNS} run(s), Measure: {MEASURE_RUNS} run(s)")
print(f"Chunk sizes: {chunk_sizes}")
print(f"Scenarios: {percentages}%")
random.seed(RANDOM_SEED)
report = {
"metadata": {
"pdf_file": pdf_path.name,
"total_pages": total_pages,
"warmup_runs": WARMUP_RUNS,
"measure_runs": MEASURE_RUNS,
"chunk_sizes": chunk_sizes,
"random_seed": RANDOM_SEED,
"timestamp": datetime.now().isoformat(),
},
"scenarios": [],
"summary": {},
}
for pct in percentages:
num_pages = max(1, total_pages * pct // 100)
if pct == 100:
target_pages = list(range(1, total_pages + 1))
else:
target_pages = sorted(random.sample(range(1, total_pages + 1), num_pages))
scenario_data = run_scenario_benchmark(
pdf_path,
total_pages,
target_pages,
chunk_sizes,
f"{pct}% pages",
)
report["scenarios"].append(scenario_data)
# Summary 생성
print("\n" + "=" * 60)
print("SUMMARY")
print("=" * 60)
print(f"{'Scenario':<15} {'Best Method':<20} {'Time':>8} {'Chunks':>8}")
print("-" * 60)
for scenario in report["scenarios"]:
best = min(scenario["results"], key=lambda r: r["avg_time"])
print(f"{scenario['scenario']:<15} {best['method']:<20} {best['avg_time']:>7.2f}s {best['num_chunks']:>7}")
report["summary"][scenario["scenario"]] = {
"best_method": best["method"],
"best_time": best["avg_time"],
"best_chunks": best["num_chunks"],
}
# JSON 저장
output_path = project_root / "tests" / "docling_chunking_strategy" / "docling_benchmark_report.json"
with open(output_path, "w", encoding="utf-8") as f:
json.dump(report, f, indent=2, ensure_ascii=False)
print()
print(f"Report saved to: {output_path}")
return 0
if __name__ == "__main__":
exit(main())
docs/hybrid/experiments/speed/baseline_results.json
+1221
View File
File diff suppressed because it is too large Load Diff
docs/hybrid/experiments/speed/fastapi_results.json
+1226
View File
File diff suppressed because it is too large Load Diff
docs/hybrid/experiments/speed/speed-experiment-2026-01-03.md
+65
View File
@@ -0,0 +1,65 @@
# Docling Speed Experiment Results
**Date**: 2026-01-03 14:31:43
## Summary
| Approach | Description | Avg (s/doc) | Target | Status | Speedup |
|----------|-------------|-------------|--------|--------|---------|
| baseline | docling-serve HTTP | 2.283 | - | - | - |
| fastapi | FastAPI + SDK singleton | 0.685 | 0.8 | PASS | 3.3x |
| subprocess | Persistent subprocess | 0.661 | 1.0 | PASS | 3.5x |
## Decision
**Phase 0 PASSED** - FastAPI approach meets the < 0.8s threshold.
Proceed to Phase 1 implementation:
- [x] Task 1.1: docling_subprocess_worker.py (skipped - FastAPI only)
- [x] Task 1.2: hybrid_server.py (opendataloader-pdf-hybrid CLI)
- [x] Task 2.1: DoclingSubprocessClient.java (skipped - FastAPI only)
- [x] Task 2.2: DoclingFastServerClient.java
- [x] Task 2.3: HybridClientFactory modification
- [x] Task 3: Benchmark integration
- [x] Task 4: Final validation
Subprocess approach also passed - both approaches available for implementation.
## Detailed Statistics
### Baseline
- **Description**: docling-serve HTTP API
- **Timestamp**: 2026-01-03 14:23:41
- **Total documents**: 200
- **Successful**: 200
- **Failed**: 0
- **Total elapsed**: 456.6s
- **Average per doc**: 2.2825s
- **Min**: 2.0045s
- **Max**: 8.0182s
### Fastapi
- **Description**: FastAPI server with docling SDK singleton
- **Timestamp**: 2026-01-03 14:27:18
- **Total documents**: 200
- **Successful**: 200
- **Failed**: 0
- **Total elapsed**: 137.1s
- **Average per doc**: 0.6855s
- **Min**: 0.1912s
- **Max**: 4.2420s
### Subprocess
- **Description**: Persistent Python subprocess with docling SDK
- **Timestamp**: 2026-01-03 14:30:50
- **Total documents**: 200
- **Successful**: 200
- **Failed**: 0
- **Total elapsed**: 132.4s
- **Average per doc**: 0.6612s
- **Min**: 0.1908s
- **Max**: 4.2498s
docs/hybrid/experiments/speed/subprocess_results.json
+1426
View File
File diff suppressed because one or more lines are too long
docs/hybrid/experiments/triage/triage-experiments.md
+349
View File
@@ -0,0 +1,349 @@
---
name: triage-lab
description: Triage logic experiment records and optimization history
---
# Triage Lab - Experiment Records
This skill manages experiment records and optimization history for triage logic.
## Current Implementation
**File**: `java/opendataloader-pdf-core/src/main/java/org/opendataloader/pdf/hybrid/TriageProcessor.java`
### Signal Priority (classifyPage method)
1. `hasTableBorder` - TableBorder presence (confidence: 1.0)
2. `hasVectorTableSignal` - Grid lines, border lines, line art (confidence: 0.95)
3. `hasTextTablePattern` - Text patterns with consecutive validation (confidence: 0.9)
4. `hasSuspiciousPattern` - Y-overlap or large gap detection (confidence: 0.85)
5. `lineToTextRatio > 0.3` - High line chunk ratio (confidence: 0.8)
6. `alignedLineGroups >= 5` - Aligned baseline groups (confidence: 0.7)
### Key Thresholds
| Parameter | Value | Location |
|-----------|-------|----------|
| LINE_RATIO_THRESHOLD | 0.3 | TriageProcessor:41 |
| ALIGNED_LINE_GROUPS_THRESHOLD | 5 | TriageProcessor:46 |
| GRID_GAP_MULTIPLIER | 3.0 | TriageProcessor:49 |
| MIN_LINE_COUNT_FOR_TABLE | 8 | TriageProcessor:57 |
| MIN_GRID_LINES | 3 | TriageProcessor:60 |
| MIN_CONSECUTIVE_PATTERNS | 2 | TriageProcessor:79 |
---
## Experiment History
### Experiment 001 (2026-01-03): FP Cause Analysis
**Goal**: Identify root causes of high False Positive rate
**Baseline**:
- Documents: 200 (42 with tables)
- TP: 41, TN: 48, FP: 110, FN: 1
- Precision: 27.15%, Recall: 97.62%, F1: 42.49%
**FP by Signal**:
| Signal | Count | % |
|--------|-------|---|
| hasSuspiciousPattern | 65 | 59.1% |
| hasVectorTableSignal | 23 | 20.9% |
| hasTableBorder | 14 | 12.7% |
| hasTextTablePattern | 5 | 4.5% |
| alignedLineGroups | 2 | 1.8% |
| highLineRatio | 1 | 0.9% |
**Root Cause**: Y-overlap check in `hasSuspiciousPattern` is too sensitive
- Condition `previous.getTopY() < current.getBottomY()` triggers on normal multi-column layouts
**Experiments**:
| Config | Precision | Recall | F1 | FP | FN |
|--------|-----------|--------|-----|-----|-----|
| Baseline | 27.15% | 97.62% | 42.49% | 110 | 1 |
| Disable Y-overlap | 36.28% | 97.62% | 52.90% | ~69 | 1 |
| Only Reliable Signals | 50.67% | 90.48% | 64.96% | ~38 | 4 |
| Disable SuspiciousPattern | 39.22% | 95.24% | 55.56% | ~64 | 2 |
| Require 3+ patterns | 37.38% | 95.24% | 53.69% | ~67 | 2 |
**Recommendation**:
- To maintain recall: Remove Y-overlap check (Precision +9%, Recall unchanged)
- To optimize F1: Use only reliable signals (F1 +22%, Recall -7%)
**FN Documents**:
- `01030000000110`: Missed by all experiments (needs investigation)
- `01030000000122`, `01030000000116`, `01030000000117`: Only detected by SuspiciousPattern
**Applied**: Y-overlap check removed (2026-01-03)
---
### Experiment 002 (2026-01-03): Further FP Reduction
**Goal**: Reduce remaining 72 FPs after Y-overlap removal
**Current FP by Signal** (after Experiment 001):
| Signal | Count | % |
|--------|-------|---|
| hasSuspiciousPattern | 21 | 29.2% |
| hasTableBorder | 14 | 19.4% |
| hasVectorTableSignal | 13 | 18.1% |
| alignedLineGroups | 10 | 13.9% |
| unknown | 8 | 11.1% |
| hasTextTablePattern | 5 | 6.9% |
| highLineRatio | 1 | 1.4% |
**Experiment 2A: Gap Multiplier** (hasSuspiciousPattern)
| Gap | Precision | Recall | F1 | FP | FN |
|-----|-----------|--------|-----|-----|-----|
| 3.0 (current) | 37.86% | 92.86% | 53.79% | 64 | 3 |
| 4.0 | 37.86% | 92.86% | 53.79% | 64 | 3 |
| 5.0 | 37.86% | 92.86% | 53.79% | 64 | 3 |
| 6.0 | 37.86% | 92.86% | 53.79% | 64 | 3 |
→ No effect (Y-overlap removal already optimized this signal)
**Experiment 2B: AlignedLineGroups Threshold**
| Threshold | Precision | Recall | F1 | FP | FN |
|-----------|-----------|--------|-----|-----|-----|
| 3 (current) | 37.86% | 92.86% | 53.79% | 64 | 3 |
| 4 | 39.39% | 92.86% | 55.32% | 60 | 3 |
| **5** | **39.80%** | **92.86%** | **55.71%** | **59** | **3** |
| 6 | 39.80% | 92.86% | 55.71% | 59 | 3 |
→ **Recommended**: Threshold 5 (FP -5, Recall maintained)
**Experiment 2C: Vector Signal Criteria**
| LineCount | GridLines | Precision | Recall | F1 | FP | FN |
|-----------|-----------|-----------|--------|-----|-----|-----|
| 8, 3 (current) | | 37.86% | 92.86% | 53.79% | 64 | 3 |
| 10, 4 | | 38.24% | 92.86% | 54.17% | 63 | 3 |
| 12, 4 | | 37.62% | 90.48% | 53.15% | 63 | 4 |
→ Minimal effect (FP -1, higher values reduce Recall)
**Recommendation**:
- Apply `alignedLineGroups` threshold 3 → 5
- Expected: FP 64 → 59 (-5), Recall 92.86% (maintained), F1 +1.92%
**Applied**: alignedLineGroups threshold 3 → 5 (2026-01-03)
**Actual Results**:
| Metric | Before (Exp 001) | After (Exp 002) | Change |
|--------|------------------|-----------------|--------|
| FP | 72 | 67 | -5 |
| FN | 1 | 1 | 0 |
| Precision | 36.28% | 37.96% | +1.68% |
| Recall | 97.62% | 97.62% | 0 |
| F1 | 52.90% | 54.67% | +1.77% |
**Next Steps**:
- Investigate `hasTableBorder` FPs (14 cases, external library)
- Investigate `unknown` FPs (8 cases)
---
### Experiment 003 (2026-01-03): VectorTableSignal & SuspiciousPattern Analysis
**Goal**: Further reduce FP 67 while maintaining high Recall
**Current FP by Signal** (after Experiment 002):
| Signal | Count | % |
|--------|-------|---|
| hasVectorTableSignal | 23 | 34.3% |
| hasSuspiciousPattern | 19 | 28.4% |
| hasTableBorder | 14 | 20.9% |
| hasTextTablePattern | 5 | 7.5% |
| alignedLineGroups | 5 | 7.5% |
| highLineRatio | 1 | 1.5% |
**VectorTableSignal Sub-signal Analysis** (23 FPs):
| Sub-signal | Count |
|------------|-------|
| hasAlignedShortLines | 30 |
| hasTableBorderLines | 22 |
| hasGridLines | 16 |
| lineArt>=8 | 13 |
| hasRowSeparatorPattern | 12 |
→ `hasAlignedShortLines` is the primary cause of VectorSignal FPs
**Experiments**:
| Config | Precision | Recall | F1 | FP | FN |
|--------|-----------|--------|-----|-----|-----|
| Current (Exp 002) | 37.96% | 97.62% | 54.67% | 67 | 1 |
| 003B: Disable VectorSignal | 40.00% | 90.48% | 55.47% | 57 | 4 |
| 003C: Grid OR BorderLines only | 40.21% | 92.86% | 56.12% | 58 | 3 |
| **003D: Disable SuspiciousPattern** | **42.11%** | **95.24%** | **58.39%** | **55** | **2** |
| 003F: Only Reliable Signals | 56.25% | 85.71% | 67.92% | 28 | 6 |
| 003G: Disable AlignedShortLines | 40.00% | 95.24% | 56.34% | 60 | 2 |
| 003I: Combined (NoAlign+NoSusp) | 44.71% | 90.48% | 59.84% | 47 | 4 |
**Analysis**:
- `hasTableBorder` (14 FPs): External library, cannot modify
- `hasVectorTableSignal` (23 FPs): `hasAlignedShortLines` too aggressive
- `hasSuspiciousPattern` (19 FPs): Gap detection catches non-table layouts
**Recommendation**:
- **Best for Recall**: 003D (Disable SuspiciousPattern)
- FP: 67 → 55 (-12), FN: 1 → 2 (+1), Recall: 95.24%
- **Best for F1**: 003I (Combined)
- FP: 67 → 47 (-20), FN: 1 → 4 (+3), F1: 59.84%
**FN Documents** (003D):
- `01030000000122`: Only detected by SuspiciousPattern (gap-based)
- `01030000000110`: Never detected (needs separate investigation)
**Applied**: 003D - Disabled hasSuspiciousPattern (2026-01-03)
**Actual Results**:
| Metric | Before (Exp 002) | After (Exp 003) | Change |
|--------|------------------|-----------------|--------|
| FP | 67 | 55 | -12 |
| FN | 1 | 2 | +1 |
| Precision | 37.96% | 42.11% | +4.15% |
| Recall | 97.62% | 95.24% | -2.38% |
| F1 | 54.67% | 58.39% | +3.72% |
---
### Experiment 004 (2026-01-03): AlignedLineGroups Signal Analysis
**Goal**: Further reduce FP 55 while maintaining Recall 95.24%
**Current FP by Signal** (after Experiment 003):
| Signal | Count | % |
|--------|-------|---|
| hasVectorTableSignal | 23 | 41.8% |
| hasTableBorder | 14 | 25.5% |
| alignedLineGroups | 12 | 21.8% |
| hasTextTablePattern | 5 | 9.1% |
| highLineRatio | 1 | 1.8% |
**VectorTableSignal Sub-signal Analysis** (23 FPs):
| Sub-signal | Count |
|------------|-------|
| hasAlignedShortLines | 16 |
| hasTableBorderLines | 10 |
| lineArt>=8 | 8 |
| hasRowSeparatorPattern | 7 |
| hasGridLines | 5 |
**Experiments**:
| Config | Precision | Recall | F1 | FP | FN |
|--------|-----------|--------|-----|-----|-----|
| Current (Exp 003) | 42.11% | 95.24% | 58.39% | 55 | 2 |
| 004A: NoAlignedShortLines | 44.71% | 90.48% | 59.84% | 47 | 4 |
| 004B: Grid+BorderLines only | 45.12% | 88.10% | 59.68% | 45 | 5 |
| **004D: No alignedLineGroups** | **48.19%** | **95.24%** | **64.00%** | **43** | **2** |
| 004E: alignedLineGroups>=7 | 44.94% | 95.24% | 61.07% | 49 | 2 |
| 004G: NoAlignShort+Groups>=7 | 48.10% | 90.48% | 62.81% | 41 | 4 |
| 004I: Reliable Only | 54.41% | 88.10% | 67.27% | 31 | 5 |
**Analysis**:
- `alignedLineGroups` signal caused 12 FPs but detected no additional true tables
- Disabling it removes all 12 FPs without any FN increase
- Best option for maintaining Recall while improving Precision
**Applied**: 004D - Disabled alignedLineGroups signal (2026-01-03)
**Actual Results**:
| Metric | Before (Exp 003) | After (Exp 004) | Change |
|--------|------------------|-----------------|--------|
| FP | 55 | 43 | -12 |
| FN | 2 | 2 | 0 |
| Precision | 42.11% | 48.19% | +6.08% |
| Recall | 95.24% | 95.24% | 0 |
| F1 | 58.39% | 64.00% | +5.61% |
**Next Steps**:
- Investigate `hasVectorTableSignal` FPs (23 remaining) - hasAlignedShortLines main cause
- Investigate `hasTableBorder` FPs (14 cases, external library limitation)
---
### Experiment 005 (2026-01-03): Large Image Signal for FN Reduction
**Goal**: Reduce FN by detecting pages with large images (potential table/chart images)
**Background**:
- FN documents `01030000000110` and `01030000000122` contain images with tables
- 110: 28.64% page area (graph image)
- 122: 11.73% page area (table image)
**Implementation**:
- Added `hasLargeImage` signal to TriageProcessor
- Detects ImageChunk objects and calculates max image area / page area ratio
**Experiments**:
| Threshold | Precision | Recall | F1 | FP | FN |
|-----------|-----------|--------|-----|-----|-----|
| Baseline (no image) | 48.19% | 95.24% | 64.00% | 43 | 2 |
| 10% | 33.07% | **100%** | 49.70% | 85 | 0 |
| **11%** | **33.60%** | **100%** | **50.30%** | **83** | **0** |
| 15% | 35.96% | 97.62% | 52.56% | 73 | 1 |
**Analysis**:
- 11% threshold achieves **100% Recall** (all 42 table documents detected)
- Trade-off: FP increases from 43 → 83 (+40), Precision drops from 48% → 34%
- F1 decreases from 64% → 50% due to high FP increase
- Many FPs are documents with decorative images, diagrams, photos
**Experiment 005B: Adding Aspect Ratio Condition**
Observation: FN documents have wide images (ratio 1.79, 3.68), while FP documents often have square/tall images (ratio 0.6~1.5).
| Config | Precision | Recall | F1 | FP | FN |
|--------|-----------|--------|-----|-----|-----|
| Baseline (no image) | 48.19% | 95.24% | 64.00% | 43 | 2 |
| 11% only | 33.60% | 100% | 50.30% | 83 | 0 |
| 11% + ratio 1.7 | 42.86% | 100% | 60.00% | 56 | 0 |
| **11% + ratio 1.75** | **43.30%** | **100%** | **60.43%** | **55** | **0** |
| 11% + ratio 2.0 | 46.59% | 97.62% | 63.08% | 47 | 1 |
**Final Configuration**:
- Image area >= 11% of page area
- Image aspect ratio (width/height) >= 1.75
**Trade-off**:
- Achieves **100% Recall** (all 42 table documents detected)
- FP increases from 43 → 55 (+12)
- F1 decreases from 64% → 60.43% (-3.57%)
**Applied**: 11% + aspect ratio 1.75 (2026-01-03)
---
## Template for New Experiments
```markdown
### Experiment XXX (YYYY-MM-DD): [Title]
**Goal**: [What are you trying to improve?]
**Changes**: [What did you modify?]
**Results**:
| Config | Precision | Recall | F1 | FP | FN |
|--------|-----------|--------|-----|-----|-----|
| Before | | | | | |
| After | | | | | |
**Conclusion**: [What did you learn? Should this be applied?]
**Next Steps**: [What to try next?]
```
---
## How to Run Experiments
```bash
# Run triage accuracy test
./scripts/test-java.sh -Dtest=TriageProcessorIntegrationTest#testTriageAccuracyOnBenchmarkPDFs
# Debug specific document
./scripts/bench.sh --doc-id 01030000000110
```
## Related Files
- [TriageProcessor.java](../../java/opendataloader-pdf-core/src/main/java/org/opendataloader/pdf/hybrid/TriageProcessor.java)
- [TriageProcessorIntegrationTest.java](../../java/opendataloader-pdf-core/src/test/java/org/opendataloader/pdf/hybrid/TriageProcessorIntegrationTest.java)
docs/hybrid/hybrid-mode-design.md
+121
View File
@@ -0,0 +1,121 @@
# Hybrid PDF Processing System - Design Document
## Overview
Hybrid PDF processing system combining Java heuristics + external AI backends.
Routes pages via per-page Triage: simple pages to fast Java path, complex tables/OCR to AI backend.
## Key Decisions
| Item | Decision |
|------|----------|
| CLI Option | `--hybrid <off\|docling\|hancom\|...>` |
| Default | `off` (Java-only, no external dependency) |
| First Backend | `docling` (docling-serve REST API) |
| Automation | Semi-automatic (benchmark/analysis auto, code changes require approval) |
| Triage Strategy | Conservative (minimize FN, accept FP, route uncertain pages to backend) |
---
## CLI Usage
```bash
# Default: Java-only processing
opendataloader-pdf input.pdf
opendataloader-pdf --hybrid off input.pdf
# Use docling backend
opendataloader-pdf --hybrid docling input.pdf
# With custom backend URL
opendataloader-pdf --hybrid docling --hybrid-url http://localhost:5001 input.pdf
# Future backends
opendataloader-pdf --hybrid hancom input.pdf
```
## Hybrid Options
| Option | Description |
|--------|-------------|
| `--hybrid <name>` | Hybrid backend: `off` (default), `docling`, `hancom`, etc. |
| `--hybrid-url <url>` | Backend server URL (overrides default) |
| `--hybrid-timeout <ms>` | Request timeout in milliseconds (default: 0, no timeout) |
| `--hybrid-fallback` | Opt in to Java fallback on backend error (default: disabled — backend failures fail fast with non-zero exit) |
## Supported Backends
| Backend | Status | Description |
|---------|--------|-------------|
| `off` | ✅ Default | Java-only, no external calls |
| `docling-fast` | ✅ Available | docling-serve (local) |
| `hancom` | 📋 Future (Priority) | Hancom Document AI |
| `azure` | 📋 Future | Azure Document Intelligence |
| `google` | 📋 Future | Google Document AI |
---
## Architecture
```
┌─────────────────────────────────────────────────────────────────┐
│ PDF Input │
└─────────────────────────────────────────────────────────────────┘
│
▼
┌─────────────────────────────────────────────────────────────────┐
│ ContentFilterProcessor │
│ (existing: text filtering) │
└─────────────────────────────────────────────────────────────────┘
│
▼
┌─────────────────────────────────────────────────────────────────┐
│ TriageProcessor.triageAllPages() │
│ - Batch triage all pages │
│ - Output: Map<PageNumber, TriageResult> │
└─────────────────────────────────────────────────────────────────┘
│
┌───────────────┴───────────────┐
│ │
▼ ▼
┌─────────────────────────┐ ┌─────────────────────────┐
│ JAVA Path │ │ BACKEND Path │
│ (parallel processing) │ │ (single batch API call)│
│ │ │ │
│ ExecutorService │ │ BackendClient │
│ - TableBorderProcessor │ │ - Send all pages once │
│ - TextLineProcessor │ │ - Receive all results │
│ - ParagraphProcessor │ │ SchemaTransformer │
└─────────────────────────┘ └─────────────────────────┘
│ │
│ CONCURRENT │
└───────────────┬───────────────┘
│
▼
┌─────────────────────────────────────────────────────────────────┐
│ Result Merger │
│ (preserve page order) │
└─────────────────────────────────────────────────────────────────┘
│
▼
┌─────────────────────────────────────────────────────────────────┐
│ Post-processing & Output Generation │
└─────────────────────────────────────────────────────────────────┘
```
---
## Risks and Mitigations
| Risk | Mitigation |
|------|------------|
| Backend unavailable or returns failed pages | Fail fast with non-zero exit (default). Opt in to `--hybrid-fallback` to fall back to Java for backend-routed pages. |
| Triage FN (missed tables) | Conservative threshold, benchmark monitoring |
| Schema mismatch | Step-by-step validation, type checking |
| Slow processing | Parallel execution, batch API calls |
---
## Related Documents
- **Implementation Tasks**: [hybrid-mode-tasks.md](hybrid-mode-tasks.md)
docs/hybrid/hybrid-mode-tasks.md
+1265
View File
File diff suppressed because it is too large Load Diff
docs/hybrid/research/comparison-summary.md
+73
View File
@@ -0,0 +1,73 @@
# Docling vs OpenDataLoader Output Comparison
## Test Document
- File: `01030000000045.pdf` (1 page with table)
## Element Count Comparison
| Category | Docling | OpenDataLoader |
|----------|---------|----------------|
| Tables | 1 | 1 |
| Text elements | 5 | 4 paragraphs |
| Images | 0 | 1 |
| Headings | (N/A - uses labels) | 1 |
## Text Element Labels (Docling)
| Label | Count |
|-------|-------|
| caption | 1 |
| footnote | 1 |
| page_footer | 1 |
| page_header | 1 |
| text | 1 |
## Table Structure Comparison
| Property | Docling | OpenDataLoader |
|----------|---------|----------------|
| Rows | 9 | 3 |
| Columns | 3 | 3 |
| Total cells | 26 | 9 |
**Note**: Docling detects more rows in the table structure. This may be due to:
- Different table detection algorithms
- OpenDataLoader may have merged some rows
- Different handling of header rows
## Bounding Box Comparison (Table)
| System | l/left | t/top | r/right | b/bottom | Origin |
|--------|--------|-------|---------|----------|--------|
| Docling | 53.22 | 439.98 | 373.94 | 234.74 | BOTTOMLEFT |
| OpenDataLoader | 54.0 | 234.44 | 372.73 | 440.21 | BOTTOMLEFT |
**Coordinate mapping**: Both use BOTTOMLEFT origin.
- Docling: `{l, t, r, b}` where t=top, b=bottom
- OpenDataLoader: `[left, bottom, right, top]`
So the actual coordinates match closely:
- Left: 53.22 ≈ 54.0
- Bottom: 234.74 ≈ 234.44
- Right: 373.94 ≈ 372.73
- Top: 439.98 ≈ 440.21
## Schema Mapping Summary
| Docling Type | OpenDataLoader Type |
|--------------|---------------------|
| texts (label: text) | paragraph |
| texts (label: section_header) | heading |
| tables | table |
| pictures | image |
| texts (label: page_header) | paragraph (filtered as header) |
| texts (label: page_footer) | paragraph (filtered as footer) |
| texts (label: caption) | paragraph |
| texts (label: footnote) | paragraph |
## Key Differences
1. **Type naming**: Docling uses `label` field for text types, OpenDataLoader uses `type`
2. **Table structure**: Docling detects more detailed row structure
3. **Coordinate format**: Same origin but different field order
4. **Heading detection**: Docling uses `SectionHeaderItem` with `level`, OpenDataLoader uses `heading` type with `level`
docs/hybrid/research/docling-openapi.json
+1
View File
File diff suppressed because one or more lines are too long
docs/hybrid/research/docling-sample-response-lorem.json
+1
View File
File diff suppressed because one or more lines are too long
docs/hybrid/research/docling-sample-response.json
+1
View File
File diff suppressed because one or more lines are too long
docs/hybrid/research/documents-with-tables.txt
+42
View File
@@ -0,0 +1,42 @@
01030000000045.pdf
01030000000046.pdf
01030000000047.pdf
01030000000051.pdf
01030000000052.pdf
01030000000053.pdf
01030000000064.pdf
01030000000078.pdf
01030000000081.pdf
01030000000082.pdf
01030000000083.pdf
01030000000084.pdf
01030000000088.pdf
01030000000089.pdf
01030000000090.pdf
01030000000110.pdf
01030000000116.pdf
01030000000117.pdf
01030000000119.pdf
01030000000120.pdf
01030000000121.pdf
01030000000122.pdf
01030000000127.pdf
01030000000128.pdf
01030000000130.pdf
01030000000132.pdf
01030000000146.pdf
01030000000147.pdf
01030000000149.pdf
01030000000150.pdf
01030000000165.pdf
01030000000166.pdf
01030000000170.pdf
01030000000178.pdf
01030000000180.pdf
01030000000182.pdf
01030000000187.pdf
01030000000188.pdf
01030000000189.pdf
01030000000190.pdf
01030000000197.pdf
01030000000200.pdf
docs/hybrid/research/iobject-structure.md
+105
View File
@@ -0,0 +1,105 @@
# IObject Class Structure
## Overview
IObject is imported from `org.verapdf.wcag.algorithms.entities.IObject` (external verapdf-wcag-algs library).
## JSON Output Types
Based on sample response analysis, OpenDataLoader produces the following element types:
### Element Types
| Type | JSON `type` field | Description |
|------|-------------------|-------------|
| Paragraph | `paragraph` | Text paragraph with font info |
| Heading | `heading` | Section heading with level |
| Table | `table` | Table with rows and cells |
| Image | `image` | Image/figure element |
| List | `list` | Bulleted or numbered list |
### Common Fields (all types)
```json
{
"type": "paragraph",
"id": 17,
"page number": 1,
"bounding box": [left, bottom, right, top] // PDF points, origin at bottom-left
}
```
### Paragraph Fields
```json
{
"type": "paragraph",
"font": "ArialMT",
"font size": 8.0,
"text color": "[0.0, 0.0, 0.0, 0.7]",
"content": "Text content here"
}
```
### Heading Fields
```json
{
"type": "heading",
"level": "1",
"content": "Heading text"
}
```
### Table Structure
```json
{
"type": "table",
"level": "1",
"number of rows": 3,
"number of columns": 3,
"rows": [
{
"type": "table row",
"row number": 1,
"cells": [
{
"type": "table cell",
"page number": 1,
"bounding box": [left, bottom, right, top],
"row number": 1,
"column number": 1,
"row span": 1,
"column span": 1,
"kids": [
{
"type": "paragraph",
"content": "Cell text"
}
]
}
]
}
]
}
```
## Bounding Box Coordinate System
- **OpenDataLoader**: `[left, bottom, right, top]` in PDF points, origin at BOTTOMLEFT
- **Docling**: `{l, t, r, b}` with `coord_origin: "BOTTOMLEFT"` or `"TOPLEFT"`
### Conversion Notes
- If docling uses TOPLEFT origin: `bottom = page_height - docling_t`, `top = page_height - docling_b`
- If docling uses BOTTOMLEFT origin: direct mapping `[l, b, r, t]` → `[left, bottom, right, top]`
## Key Java Classes
From the codebase:
- `TableBorder` - Table with border-based detection
- `TableBorderRow` - Table row
- `TableBorderCell` - Table cell with contents, rowSpan, colSpan
- `BoundingBox` - PDF coordinates (page, left, bottom, right, top)
- Processors: `TextLineProcessor`, `TableBorderProcessor`, `HeadingProcessor`, `ListProcessor`
docs/hybrid/research/opendataloader-sample-response.json
+370
View File
@@ -0,0 +1,370 @@
{
"file name" : "01030000000045.pdf",
"number of pages" : 1,
"author" : null,
"title" : null,
"creation date" : null,
"modification date" : null,
"kids" : [ {
"type" : "paragraph",
"id" : 17,
"page number" : 1,
"bounding box" : [ 281.571, 551.756, 372.715, 560.694 ],
"font" : "ArialMT",
"font size" : 8.0,
"text color" : "[0.0, 0.0, 0.0, 0.7]",
"content" : "Civil Society Engagement"
}, {
"type" : "paragraph",
"id" : 19,
"page number" : 1,
"bounding box" : [ 54.0, 499.92, 372.727, 539.282 ],
"font" : "Georgia",
"font size" : 10.0,
"text color" : "[0.0, 0.0, 0.0, 1.0]",
"content" : "election integrity. The registration of local election observers runs until 25 May, and the NEC is still reviewing the application of nearly 5,000 observers."
}, {
"type" : "heading",
"id" : 20,
"level" : "Doctitle",
"page number" : 1,
"bounding box" : [ 54.0, 455.381, 350.475, 480.87 ],
"heading level" : 1,
"font" : "Arial-BoldMT",
"font size" : 11.0,
"text color" : "[0.0, 0.0, 0.0, 1.0]",
"content" : "Table: The number of accredited observers as of 28 April 202215"
}, {
"type" : "table",
"id" : 21,
"level" : "1",
"page number" : 1,
"bounding box" : [ 54.0, 234.441, 372.727, 440.212 ],
"number of rows" : 3,
"number of columns" : 3,
"rows" : [ {
"type" : "table row",
"row number" : 1,
"cells" : [ {
"type" : "table cell",
"page number" : 1,
"bounding box" : [ 54.375, 413.86, 83.52, 440.087 ],
"row number" : 1,
"column number" : 1,
"row span" : 1,
"column span" : 1,
"kids" : [ {
"type" : "paragraph",
"id" : 1,
"page number" : 1,
"bounding box" : [ 61.757, 427.253, 75.761, 437.307 ],
"font" : "ArialMT",
"font size" : 9.0,
"text color" : "[0.0, 0.0, 0.0, 1.0]",
"content" : "No."
} ]
}, {
"type" : "table cell",
"page number" : 1,
"bounding box" : [ 83.52, 413.86, 273.472, 440.087 ],
"row number" : 1,
"column number" : 2,
"row span" : 1,
"column span" : 1,
"kids" : [ {
"type" : "paragraph",
"id" : 2,
"page number" : 1,
"bounding box" : [ 87.52, 427.253, 173.056, 437.307 ],
"font" : "ArialMT",
"font size" : 9.0,
"text color" : "[0.0, 0.0, 0.0, 1.0]",
"content" : "Name of organization"
} ]
}, {
"type" : "table cell",
"page number" : 1,
"bounding box" : [ 273.472, 413.86, 372.352, 440.087 ],
"row number" : 1,
"column number" : 3,
"row span" : 1,
"column span" : 1,
"kids" : [ {
"type" : "paragraph",
"id" : 3,
"page number" : 1,
"bounding box" : [ 280.08, 416.453, 366.111, 437.307 ],
"font" : "ArialMT",
"font size" : 9.0,
"text color" : "[0.0, 0.0, 0.0, 1.0]",
"content" : "Number of accredited observers"
} ]
} ]
}, {
"type" : "table row",
"row number" : 2,
"cells" : [ {
"type" : "table cell",
"page number" : 1,
"bounding box" : [ 54.375, 249.993, 83.52, 413.86 ],
"row number" : 2,
"column number" : 1,
"row span" : 1,
"column span" : 1,
"kids" : [ {
"type" : "list",
"id" : 4,
"level" : "1",
"page number" : 1,
"bounding box" : [ 66.257, 263.386, 71.261, 410.955 ],
"numbering style" : "arabic numbers",
"number of list items" : 7,
"list items" : [ {
"type" : "list item",
"page number" : 1,
"bounding box" : [ 66.257, 400.9, 71.261, 410.955 ],
"font" : "ArialMT",
"font size" : 9.0,
"text color" : "[0.0, 0.0, 0.0, 1.0]",
"content" : "1",
"kids" : [ ]
}, {
"type" : "list item",
"page number" : 1,
"bounding box" : [ 66.257, 374.548, 71.261, 384.603 ],
"font" : "ArialMT",
"font size" : 9.0,
"text color" : "[0.0, 0.0, 0.0, 1.0]",
"content" : "2",
"kids" : [ ]
}, {
"type" : "list item",
"page number" : 1,
"bounding box" : [ 66.257, 348.196, 71.261, 358.251 ],
"font" : "ArialMT",
"font size" : 9.0,
"text color" : "[0.0, 0.0, 0.0, 1.0]",
"content" : "3",
"kids" : [ ]
}, {
"type" : "list item",
"page number" : 1,
"bounding box" : [ 66.257, 321.844, 71.261, 331.898 ],
"font" : "ArialMT",
"font size" : 9.0,
"text color" : "[0.0, 0.0, 0.0, 1.0]",
"content" : "4",
"kids" : [ ]
}, {
"type" : "list item",
"page number" : 1,
"bounding box" : [ 66.257, 295.491, 71.261, 305.546 ],
"font" : "ArialMT",
"font size" : 9.0,
"text color" : "[0.0, 0.0, 0.0, 1.0]",
"content" : "5",
"kids" : [ ]
}, {
"type" : "list item",
"page number" : 1,
"bounding box" : [ 66.257, 278.939, 71.261, 288.993 ],
"font" : "ArialMT",
"font size" : 9.0,
"text color" : "[0.0, 0.0, 0.0, 1.0]",
"content" : "6",
"kids" : [ ]
}, {
"type" : "list item",
"page number" : 1,
"bounding box" : [ 66.257, 263.386, 71.261, 273.441 ],
"font" : "ArialMT",
"font size" : 9.0,
"text color" : "[0.0, 0.0, 0.0, 1.0]",
"content" : "7",
"kids" : [ ]
} ]
} ]
}, {
"type" : "table cell",
"page number" : 1,
"bounding box" : [ 83.52, 249.993, 273.472, 413.86 ],
"row number" : 2,
"column number" : 2,
"row span" : 1,
"column span" : 1,
"kids" : [ {
"type" : "paragraph",
"id" : 5,
"page number" : 1,
"bounding box" : [ 87.52, 390.1, 249.615, 410.955 ],
"font" : "ArialMT",
"font size" : 9.0,
"text color" : "[0.0, 0.0, 0.0, 1.0]",
"content" : "Union of Youth Federations of Cambodia (UYFC)"
}, {
"type" : "paragraph",
"id" : 6,
"page number" : 1,
"bounding box" : [ 87.52, 363.748, 225.426, 384.603 ],
"font" : "ArialMT",
"font size" : 9.0,
"text color" : "[0.0, 0.0, 0.0, 1.0]",
"content" : "Cambodian Women for Peace and Development"
}, {
"type" : "paragraph",
"id" : 7,
"page number" : 1,
"bounding box" : [ 87.52, 337.396, 239.584, 358.251 ],
"font" : "ArialMT",
"font size" : 9.0,
"text color" : "[0.0, 0.0, 0.0, 1.0]",
"content" : "Association of Democratic Students of Cambodia"
}, {
"type" : "paragraph",
"id" : 8,
"page number" : 1,
"bounding box" : [ 87.52, 311.044, 231.623, 331.898 ],
"font" : "ArialMT",
"font size" : 9.0,
"text color" : "[0.0, 0.0, 0.0, 1.0]",
"content" : "Association of Intellectual and Youth Volunteer"
}, {
"type" : "paragraph",
"id" : 9,
"page number" : 1,
"bounding box" : [ 87.52, 252.586, 237.745, 305.546 ],
"font" : "ArialMT",
"font size" : 9.0,
"text color" : "[0.0, 0.0, 0.0, 1.0]",
"content" : "Our Friends Association COMFREL Traditional and Modern Mental Health Organization"
} ]
}, {
"type" : "table cell",
"page number" : 1,
"bounding box" : [ 273.472, 249.993, 372.352, 413.86 ],
"row number" : 2,
"column number" : 3,
"row span" : 1,
"column span" : 1,
"kids" : [ {
"type" : "paragraph",
"id" : 10,
"page number" : 1,
"bounding box" : [ 309.336, 400.9, 336.858, 410.955 ],
"font" : "ArialMT",
"font size" : 9.0,
"text color" : "[0.0, 0.0, 0.0, 1.0]",
"content" : "17,266"
}, {
"type" : "paragraph",
"id" : 11,
"page number" : 1,
"bounding box" : [ 311.839, 374.548, 334.357, 384.603 ],
"font" : "ArialMT",
"font size" : 9.0,
"text color" : "[0.0, 0.0, 0.0, 1.0]",
"content" : "9,835"
}, {
"type" : "paragraph",
"id" : 12,
"page number" : 1,
"bounding box" : [ 315.926, 348.196, 329.608, 358.251 ],
"font" : "ArialMT",
"font size" : 9.0,
"text color" : "[0.0, 0.0, 0.0, 1.0]",
"content" : "711"
}, {
"type" : "paragraph",
"id" : 13,
"page number" : 1,
"bounding box" : [ 318.095, 321.844, 328.103, 331.898 ],
"font" : "ArialMT",
"font size" : 9.0,
"text color" : "[0.0, 0.0, 0.0, 1.0]",
"content" : "46"
}, {
"type" : "paragraph",
"id" : 14,
"page number" : 1,
"bounding box" : [ 318.095, 263.386, 328.103, 305.546 ],
"font" : "ArialMT",
"font size" : 9.0,
"text color" : "[0.0, 0.0, 0.0, 1.0]",
"content" : "27 26 15"
} ]
} ]
}, {
"type" : "table row",
"row number" : 3,
"cells" : [ {
"type" : "table cell",
"page number" : 1,
"bounding box" : [ 54.375, 234.566, 83.52, 249.993 ],
"row number" : 3,
"column number" : 1,
"row span" : 1,
"column span" : 1,
"kids" : [ ]
}, {
"type" : "table cell",
"page number" : 1,
"bounding box" : [ 83.52, 234.566, 273.472, 249.993 ],
"row number" : 3,
"column number" : 2,
"row span" : 1,
"column span" : 1,
"kids" : [ {
"type" : "paragraph",
"id" : 15,
"page number" : 1,
"bounding box" : [ 87.52, 237.034, 108.351, 247.089 ],
"font" : "Arial-BoldMT",
"font size" : 9.0,
"text color" : "[0.0, 0.0, 0.0, 1.0]",
"content" : "Total"
} ]
}, {
"type" : "table cell",
"page number" : 1,
"bounding box" : [ 273.472, 234.566, 372.352, 249.993 ],
"row number" : 3,
"column number" : 3,
"row span" : 1,
"column span" : 1,
"kids" : [ {
"type" : "paragraph",
"id" : 16,
"page number" : 1,
"bounding box" : [ 309.336, 237.034, 336.858, 247.089 ],
"font" : "Arial-BoldMT",
"font size" : 9.0,
"text color" : "[0.0, 0.0, 0.0, 1.0]",
"content" : "27,926"
} ]
} ]
} ]
}, {
"type" : "image",
"id" : 22,
"page number" : 1,
"bounding box" : [ 54.0, 68.275, 126.0, 68.525 ]
}, {
"type" : "paragraph",
"id" : 23,
"page number" : 1,
"bounding box" : [ 54.0, 52.729, 185.287, 59.432 ],
"font" : "ArialMT",
"font size" : 6.0,
"text color" : "[0.86, 0.57, 0.0, 0.16]",
"content" : "15 https://www.nec.gov.kh/khmer/content/5524"
}, {
"type" : "paragraph",
"id" : 18,
"page number" : 1,
"bounding box" : [ 363.829, 34.305, 372.725, 43.242 ],
"font" : "ArialMT",
"font size" : 8.0,
"text color" : "[0.0, 0.0, 0.0, 0.75]",
"content" : "17"
} ]
}
docs/hybrid/research/opendataloader-sample-response.md
+16
View File
@@ -0,0 +1,16 @@
Civil Society Engagement
election integrity. The registration of local election observers runs until 25 May, and the NEC is still reviewing the application of nearly 5,000 observers.
# Table: The number of accredited observers as of 28 April 202215
|No.|Name of organization|Number of accredited observers|
|---|---|---|
|1<br>2<br>3<br>4<br>5<br>6<br>7<br>|Union of Youth Federations of Cambodia (UYFC)<br><br>Cambodian Women for Peace and Development<br><br>Association of Democratic Students of Cambodia<br><br>Association of Intellectual and Youth Volunteer<br><br>Our Friends Association COMFREL Traditional and Modern Mental Health Organization|17,266<br><br>9,835<br><br>711<br><br>46<br><br>27 26 15|
||Total|27,926|
15 https://www.nec.gov.kh/khmer/content/5524
17
docs/superpowers/plans/2026-03-16-cid-font-detection.md
+630
View File
@@ -0,0 +1,630 @@
# CID Font Extraction Failure Detection — Implementation Plan
> **For agentic workers:** REQUIRED: Use superpowers:subagent-driven-development (if subagents available) or superpowers:executing-plans to implement this plan. Steps use checkbox (`- [ ]`) syntax for tracking.
**Goal:** Detect pages where CID font extraction failed (high U+FFFD ratio), emit warning logs, and auto-route to OCR backend in hybrid mode.
**Architecture:** Measure replacement character ratio in ContentFilterProcessor before replacement, store in StaticLayoutContainers, consume in TriageProcessor as highest-priority signal. Warning log fires regardless of hybrid mode.
**Tech Stack:** Java 11+, JUnit Jupiter, veraPDF API (`ChunkParser.REPLACEMENT_CHARACTER_STRING`)
---
## File Structure
| File | Responsibility |
|---|---|
| `TextProcessor.java` | New `measureReplacementCharRatio()` method |
| `StaticLayoutContainers.java` | Per-page replacement ratio storage |
| `ContentFilterProcessor.java` | Measure + warn + store before replacement |
| `TriageProcessor.java` | Signal 0: route high-ratio pages to BACKEND |
| `TextProcessorTest.java` | Unit tests for measurement |
| `TriageProcessorTest.java` | Unit tests for Signal 0 routing |
| `CidFontDetectionTest.java` (new) | e2e test with synthetic PDF |
| `test/resources/cid-font-no-tounicode.pdf` (new) | Test fixture |
| `test/resources/generate-cid-test-pdf.py` (new) | Generation script (reference) |
All paths below are relative to `java/opendataloader-pdf-core/src/`.
---
## Chunk 1: Measurement + Storage
### Task 1: Add per-page ratio storage to StaticLayoutContainers
**Files:**
- Modify: `main/java/org/opendataloader/pdf/containers/StaticLayoutContainers.java`
- [ ] **Step 1: Add ThreadLocal map field and imports**
Add after line 40 (`imageFormat` field):
```java
private static final ThreadLocal<Map<Integer, Double>> replacementCharRatios = ThreadLocal.withInitial(HashMap::new);
```
Add to imports:
```java
import java.util.HashMap;
import java.util.Map;
```
- [ ] **Step 2: Add getter and setter**
Add after `setImageFormat()` (after line 145):
```java
public static void setReplacementCharRatio(int pageNumber, double ratio) {
replacementCharRatios.get().put(pageNumber, ratio);
}
public static double getReplacementCharRatio(int pageNumber) {
return replacementCharRatios.get().getOrDefault(pageNumber, 0.0);
}
```
- [ ] **Step 3: Clear in clearContainers()**
Add inside `clearContainers()` method, after line 51 (`imageFormat.set(...)`):
```java
replacementCharRatios.get().clear();
```
- [ ] **Step 4: Compile check**
Run: `cd java && mvn compile -pl opendataloader-pdf-core -q`
Expected: BUILD SUCCESS
- [ ] **Step 5: Commit**
```bash
git add java/opendataloader-pdf-core/src/main/java/org/opendataloader/pdf/containers/StaticLayoutContainers.java
git commit -m "feat: add per-page replacement char ratio storage to StaticLayoutContainers"
```
### Task 2: Add measureReplacementCharRatio to TextProcessor
**Files:**
- Modify: `main/java/org/opendataloader/pdf/processors/TextProcessor.java`
- Test: `test/java/org/opendataloader/pdf/processors/TextProcessorTest.java`
- [ ] **Step 1: Write failing tests**
Add to `TextProcessorTest.java` after the last test method (before closing `}`):
```java
@Test
public void testMeasureReplacementCharRatio_allReplacement() {
List<IObject> contents = new ArrayList<>();
contents.add(new TextChunk(new BoundingBox(1, 10.0, 10.0, 100.0, 20.0),
"\uFFFD\uFFFD\uFFFD", 10, 10.0));
double ratio = TextProcessor.measureReplacementCharRatio(contents);
Assertions.assertEquals(1.0, ratio, 0.001);
}
@Test
public void testMeasureReplacementCharRatio_noReplacement() {
List<IObject> contents = new ArrayList<>();
contents.add(new TextChunk(new BoundingBox(1, 10.0, 10.0, 100.0, 20.0),
"Hello World", 10, 10.0));
double ratio = TextProcessor.measureReplacementCharRatio(contents);
Assertions.assertEquals(0.0, ratio, 0.001);
}
@Test
public void testMeasureReplacementCharRatio_mixed() {
List<IObject> contents = new ArrayList<>();
// 3 replacement chars out of 10 total = 0.3
contents.add(new TextChunk(new BoundingBox(1, 10.0, 10.0, 100.0, 20.0),
"\uFFFD\uFFFD\uFFFDAbcdefg", 10, 10.0));
double ratio = TextProcessor.measureReplacementCharRatio(contents);
Assertions.assertEquals(0.3, ratio, 0.001);
}
@Test
public void testMeasureReplacementCharRatio_emptyContents() {
List<IObject> contents = new ArrayList<>();
double ratio = TextProcessor.measureReplacementCharRatio(contents);
Assertions.assertEquals(0.0, ratio, 0.001);
}
@Test
public void testMeasureReplacementCharRatio_nonTextChunksIgnored() {
List<IObject> contents = new ArrayList<>();
contents.add(new ImageChunk(new BoundingBox(1, 10.0, 10.0, 100.0, 20.0)));
contents.add(new TextChunk(new BoundingBox(1, 10.0, 30.0, 100.0, 40.0),
"\uFFFD\uFFFD\uFFFD\uFFFD\uFFFD", 10, 10.0));
double ratio = TextProcessor.measureReplacementCharRatio(contents);
// Only TextChunks counted: 5/5 = 1.0
Assertions.assertEquals(1.0, ratio, 0.001);
}
```
- [ ] **Step 2: Run tests to verify they fail**
Run: `cd java && mvn test -pl opendataloader-pdf-core -Dtest=TextProcessorTest#testMeasureReplacementCharRatio_allReplacement -q`
Expected: FAIL — `measureReplacementCharRatio` method not found
- [ ] **Step 3: Implement measureReplacementCharRatio**
Add to `TextProcessor.java` after `replaceUndefinedCharacters()` method (after line 53):
```java
public static double measureReplacementCharRatio(List<IObject> contents) {
char replacementChar = ChunkParser.REPLACEMENT_CHARACTER_STRING.charAt(0);
int totalChars = 0;
int replacementChars = 0;
for (IObject object : contents) {
if (object instanceof TextChunk) {
String value = ((TextChunk) object).getValue();
totalChars += value.length();
for (int i = 0; i < value.length(); i++) {
if (value.charAt(i) == replacementChar) {
replacementChars++;
}
}
}
}
if (totalChars == 0) {
return 0.0;
}
return (double) replacementChars / totalChars;
}
```
- [ ] **Step 4: Run tests to verify they pass**
Run: `cd java && mvn test -pl opendataloader-pdf-core -Dtest=TextProcessorTest -q`
Expected: All tests PASS
- [ ] **Step 5: Commit**
```bash
git add java/opendataloader-pdf-core/src/main/java/org/opendataloader/pdf/processors/TextProcessor.java
git add java/opendataloader-pdf-core/src/test/java/org/opendataloader/pdf/processors/TextProcessorTest.java
git commit -m "feat: add measureReplacementCharRatio to TextProcessor
Counts U+FFFD replacement characters across TextChunks and returns
the ratio. Returns 0.0 for empty contents or pages with no text."
```
---
## Chunk 2: Warning Log + Triage Routing
### Task 3: Add warning log in ContentFilterProcessor
**Files:**
- Modify: `main/java/org/opendataloader/pdf/processors/ContentFilterProcessor.java`
- [ ] **Step 1: Add import for StaticLayoutContainers**
Add to imports:
```java
import org.opendataloader.pdf.containers.StaticLayoutContainers;
```
- [ ] **Step 2: Add measurement + warning before replaceUndefinedCharacters**
Insert immediately before line 74 (`TextProcessor.replaceUndefinedCharacters(...)`) in `getFilteredContents()`:
```java
double replacementCharRatio = TextProcessor.measureReplacementCharRatio(pageContents);
StaticLayoutContainers.setReplacementCharRatio(pageNumber, replacementCharRatio);
if (replacementCharRatio >= 0.3) {
LOGGER.log(Level.WARNING,
"Page {0}: {1,number,#.#%} of characters are replacement characters (U+FFFD). "
+ "This PDF likely contains CID-keyed fonts without ToUnicode mappings. "
+ "Text extraction may be incomplete. Consider using --hybrid-mode for OCR fallback.",
new Object[]{pageNumber + 1, replacementCharRatio});
}
```
- [ ] **Step 3: Compile check**
Run: `cd java && mvn compile -pl opendataloader-pdf-core -q`
Expected: BUILD SUCCESS
- [ ] **Step 4: Commit**
```bash
git add java/opendataloader-pdf-core/src/main/java/org/opendataloader/pdf/processors/ContentFilterProcessor.java
git commit -m "feat: detect CID font extraction failure and emit warning log
Measures U+FFFD ratio before replacement. Warns when >= 30% of
characters are replacement characters, suggesting hybrid mode."
```
### Task 4: Add Signal 0 to TriageProcessor
**Files:**
- Modify: `main/java/org/opendataloader/pdf/hybrid/TriageProcessor.java`
- Test: `test/java/org/opendataloader/pdf/hybrid/TriageProcessorTest.java`
- [ ] **Step 1: Write failing tests**
Add to `TriageProcessorTest.java` before the `// Helper methods` comment:
```java
@Test
public void testClassifyPage_highReplacementRatio_routesToBackend() {
StaticLayoutContainers.clearContainers();
StaticLayoutContainers.setReplacementCharRatio(0, 0.5);
List<IObject> contents = new ArrayList<>();
contents.add(createTextChunk(10, 100, 200, 120, "text"));
TriageResult result = TriageProcessor.classifyPage(contents, 0, new HybridConfig());
Assertions.assertEquals(TriageDecision.BACKEND, result.getDecision());
Assertions.assertEquals(1.0, result.getConfidence(), 0.001);
}
@Test
public void testClassifyPage_lowReplacementRatio_noEffect() {
StaticLayoutContainers.clearContainers();
StaticLayoutContainers.setReplacementCharRatio(0, 0.1);
List<IObject> contents = new ArrayList<>();
contents.add(createTextChunk(10, 100, 200, 120, "normal text"));
TriageResult result = TriageProcessor.classifyPage(contents, 0, new HybridConfig());
Assertions.assertEquals(TriageDecision.JAVA, result.getDecision());
}
@Test
public void testClassifyPage_exactThreshold_routesToBackend() {
StaticLayoutContainers.clearContainers();
StaticLayoutContainers.setReplacementCharRatio(0, 0.3);
List<IObject> contents = new ArrayList<>();
contents.add(createTextChunk(10, 100, 200, 120, "text"));
TriageResult result = TriageProcessor.classifyPage(contents, 0, new HybridConfig());
Assertions.assertEquals(TriageDecision.BACKEND, result.getDecision());
Assertions.assertEquals(1.0, result.getConfidence(), 0.001);
}
```
- [ ] **Step 2: Run tests to verify they fail**
Run: `cd java && mvn test -pl opendataloader-pdf-core -Dtest=TriageProcessorTest#testClassifyPage_highReplacementRatio_routesToBackend -q`
Expected: FAIL — returns JAVA instead of BACKEND
- [ ] **Step 3: Add Signal 0 to classifyPage**
In `TriageProcessor.java`, in the `classifyPage()` method with `TriageThresholds` parameter, insert before the TableBorder check (before `// Signal 1: TableBorder presence`):
```java
// Signal 0: CID font extraction failure (highest priority)
// Only fires in hybrid mode (classifyPage is only called from HybridDocumentProcessor)
double replacementRatio = StaticLayoutContainers.getReplacementCharRatio(pageNumber);
if (replacementRatio >= 0.3) {
return TriageResult.backend(pageNumber, 1.0, signals);
}
```
Add import at top of file:
```java
import org.opendataloader.pdf.containers.StaticLayoutContainers;
```
Also update the `classifyPage()` Javadoc (around line 617) to list Signal 0:
```
* <p>Signal priority (highest to lowest):
* <ol>
* <li>CID font extraction failure (replacement char ratio >= 30%)</li>
* <li>TableBorder presence</li>
* ...
```
- [ ] **Step 4: Run tests to verify they pass**
Run: `cd java && mvn test -pl opendataloader-pdf-core -Dtest=TriageProcessorTest -q`
Expected: All tests PASS
- [ ] **Step 5: Run full test suite**
Run: `cd java && mvn test -pl opendataloader-pdf-core -q`
Expected: All tests PASS — no regressions
- [ ] **Step 6: Commit**
```bash
git add java/opendataloader-pdf-core/src/main/java/org/opendataloader/pdf/hybrid/TriageProcessor.java
git add java/opendataloader-pdf-core/src/test/java/org/opendataloader/pdf/hybrid/TriageProcessorTest.java
git commit -m "feat: add CID font detection signal to TriageProcessor
Signal 0 (highest priority): routes pages with >= 30% replacement
characters to backend for OCR fallback in hybrid mode."
```
---
## Chunk 3: e2e Test + Test Fixture
### Task 5: Generate synthetic CID PDF test fixture
**Files:**
- Create: `java/opendataloader-pdf-core/src/test/resources/generate-cid-test-pdf.py`
- Create: `java/opendataloader-pdf-core/src/test/resources/cid-font-no-tounicode.pdf`
- [ ] **Step 1: Create test resources directory**
```bash
mkdir -p java/opendataloader-pdf-core/src/test/resources
```
- [ ] **Step 2: Write the generation script**
Create `java/opendataloader-pdf-core/src/test/resources/generate-cid-test-pdf.py`.
This script generates a minimal PDF with a Type0 (CID) font that has no ToUnicode CMap. The PDF must cause veraPDF to emit `\uFFFD` for the majority of text characters when parsed.
Approach: Use raw PDF syntax to embed a CID font referencing CID values without a ToUnicode mapping. Alternatively, use `reportlab` which provides low-level CID font control.
The implementer should:
1. Generate the PDF
2. Verify it with opendataloader-pdf: `cd java && mvn exec:java -pl opendataloader-pdf-cli -Dexec.mainClass="org.opendataloader.pdf.cli.CLIMain" -Dexec.args="../src/test/resources/cid-font-no-tounicode.pdf -f text" 2>&1`
3. Confirm output is mostly empty/whitespace (indicating `\uFFFD` → space replacement)
4. Confirm WARNING log about replacement characters appears
If generating a proper CID PDF proves difficult, search `odl-test-fixtures` for an existing PDF with CID font issues, or use a known CID-problematic PDF from the Korean pharmaceutical fixtures (pdf-003 through pdf-007).
- [ ] **Step 3: Commit fixture**
```bash
git add java/opendataloader-pdf-core/src/test/resources/cid-font-no-tounicode.pdf
git add java/opendataloader-pdf-core/src/test/resources/generate-cid-test-pdf.py
git commit -m "test: add synthetic CID font PDF test fixture
PDF with CID-keyed font without ToUnicode mapping for testing
replacement character detection. Generation script included."
```
### Task 6: Write e2e integration test
**Files:**
- Create: `test/java/org/opendataloader/pdf/processors/CidFontDetectionTest.java`
This test follows the pattern from `TriageProcessorIntegrationTest.java`:
`DocumentProcessor.preprocessing()` → `ContentFilterProcessor.getFilteredContents()` → assert ratio and warning.
- [ ] **Step 1: Write the integration test**
Create `java/opendataloader-pdf-core/src/test/java/org/opendataloader/pdf/processors/CidFontDetectionTest.java`:
```java
/*
* Copyright 2025-2026 Hancom Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.opendataloader.pdf.processors;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Assumptions;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.Test;
import org.opendataloader.pdf.api.Config;
import org.opendataloader.pdf.containers.StaticLayoutContainers;
import org.verapdf.wcag.algorithms.entities.IObject;
import org.verapdf.wcag.algorithms.entities.content.TextChunk;
import org.verapdf.wcag.algorithms.entities.geometry.BoundingBox;
import org.verapdf.wcag.algorithms.semanticalgorithms.containers.StaticContainers;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.List;
import java.util.logging.Handler;
import java.util.logging.Level;
import java.util.logging.LogRecord;
import java.util.logging.Logger;
/**
* Integration test for CID font extraction failure detection.
*
* Tests the full pipeline: PDF parsing → ContentFilterProcessor →
* measurement → StaticLayoutContainers storage → warning log.
*/
public class CidFontDetectionTest {
private static final Path CID_PDF_PATH = Paths.get(
"src/test/resources/cid-font-no-tounicode.pdf");
private static boolean pdfAvailable = false;
@BeforeAll
static void checkFixture() {
pdfAvailable = Files.exists(CID_PDF_PATH) && Files.isRegularFile(CID_PDF_PATH);
if (!pdfAvailable) {
System.out.println("CID font test PDF not found: " + CID_PDF_PATH.toAbsolutePath());
System.out.println("Skipping integration tests. Generate fixture first.");
}
}
@Test
public void testCidPdf_highReplacementRatio_detected() throws IOException {
Assumptions.assumeTrue(pdfAvailable, "CID font test PDF not available");
String pdfPath = CID_PDF_PATH.toAbsolutePath().toString();
Config config = new Config();
DocumentProcessor.preprocessing(pdfPath, config);
StaticLayoutContainers.clearContainers();
int numPages = StaticContainers.getDocument().getNumberOfPages();
Assertions.assertTrue(numPages > 0, "PDF should have at least 1 page");
// Process page 0 through ContentFilterProcessor
List<IObject> filteredContents = ContentFilterProcessor.getFilteredContents(
pdfPath,
StaticContainers.getDocument().getArtifacts(0),
0,
config
);
// Verify ratio was stored
double ratio = StaticLayoutContainers.getReplacementCharRatio(0);
Assertions.assertTrue(ratio >= 0.3,
"CID font PDF should have >= 30% replacement characters, got "
+ String.format("%.1f%%", ratio * 100));
}
@Test
public void testCidPdf_warningLogEmitted() throws IOException {
Assumptions.assumeTrue(pdfAvailable, "CID font test PDF not available");
// Capture warning logs
Logger logger = Logger.getLogger(ContentFilterProcessor.class.getCanonicalName());
List<String> warnings = new ArrayList<>();
Handler handler = new Handler() {
@Override public void publish(LogRecord r) {
if (r.getLevel() == Level.WARNING) {
warnings.add(r.getMessage());
}
}
@Override public void flush() {}
@Override public void close() {}
};
logger.addHandler(handler);
try {
String pdfPath = CID_PDF_PATH.toAbsolutePath().toString();
Config config = new Config();
DocumentProcessor.preprocessing(pdfPath, config);
StaticLayoutContainers.clearContainers();
ContentFilterProcessor.getFilteredContents(
pdfPath,
StaticContainers.getDocument().getArtifacts(0),
0,
config
);
boolean hasReplacementWarning = warnings.stream()
.anyMatch(w -> w.contains("replacement characters"));
Assertions.assertTrue(hasReplacementWarning,
"Expected WARNING log about replacement characters");
} finally {
logger.removeHandler(handler);
}
}
/**
* Unit-level boundary tests (no PDF fixture needed).
*/
@Test
public void testBoundary_belowThreshold_29percent() {
// 29 replacement chars out of 100 = 0.29 (below threshold)
StringBuilder sb = new StringBuilder();
for (int i = 0; i < 29; i++) sb.append('\uFFFD');
for (int i = 0; i < 71; i++) sb.append('A');
List<IObject> contents = new ArrayList<>();
contents.add(new TextChunk(new BoundingBox(1, 10.0, 10.0, 500.0, 20.0),
sb.toString(), 10, 10.0));
double ratio = TextProcessor.measureReplacementCharRatio(contents);
Assertions.assertTrue(ratio < 0.3,
"29% should be below threshold, got " + ratio);
}
@Test
public void testBoundary_atThreshold_30percent() {
// 30 replacement chars out of 100 = 0.30 (at threshold)
StringBuilder sb = new StringBuilder();
for (int i = 0; i < 30; i++) sb.append('\uFFFD');
for (int i = 0; i < 70; i++) sb.append('A');
List<IObject> contents = new ArrayList<>();
contents.add(new TextChunk(new BoundingBox(1, 10.0, 10.0, 500.0, 20.0),
sb.toString(), 10, 10.0));
double ratio = TextProcessor.measureReplacementCharRatio(contents);
Assertions.assertTrue(ratio >= 0.3,
"30% should be at threshold, got " + ratio);
}
}
```
- [ ] **Step 2: Run tests**
Run: `cd java && mvn test -pl opendataloader-pdf-core -Dtest=CidFontDetectionTest -q`
Expected: Integration tests pass (or skip if fixture not yet generated). Boundary tests always pass.
- [ ] **Step 3: Run full test suite**
Run: `cd java && mvn test -pl opendataloader-pdf-core -q`
Expected: All tests PASS
- [ ] **Step 4: Commit**
```bash
git add java/opendataloader-pdf-core/src/test/java/org/opendataloader/pdf/processors/CidFontDetectionTest.java
git commit -m "test: add integration + boundary tests for CID font detection
Integration tests load cid-font-no-tounicode.pdf through full pipeline.
Boundary tests verify 29%/30% threshold behavior."
```
---
## Chunk 4: Final Verification
### Task 7: Full test suite + benchmark regression check
- [ ] **Step 1: Run full Java test suite**
Run: `cd java && mvn test -q`
Expected: All tests PASS across both core and cli modules
- [ ] **Step 2: Check benchmark regression (if benchmark script exists)**
Run: `./scripts/bench.sh --check-regression 2>/dev/null || echo "No benchmark script — skip"`
Expected: PASS or skip
- [ ] **Step 3: Review all changes**
Run: `git log --oneline main..HEAD`
Expected commits (newest first):
1. `test: add e2e tests for CID font detection pipeline`
2. `test: add synthetic CID font PDF test fixture`
3. `feat: add CID font detection signal to TriageProcessor`
4. `feat: detect CID font extraction failure and emit warning log`
5. `feat: add measureReplacementCharRatio to TextProcessor`
6. `feat: add per-page replacement char ratio storage to StaticLayoutContainers`
- [ ] **Step 4: Update issue #286 comment**
Update the previously posted enhancement comment with the actual fix status. The earlier `enhancement` label should be removed since this is now a fix.
```bash
gh issue edit 286 --repo opendataloader-project/opendataloader-pdf --remove-label enhancement
```
docs/superpowers/plans/2026-04-18-hancom-ai-mock-server.md
+1710
View File
File diff suppressed because it is too large Load Diff
docs/superpowers/plans/2026-04-29-hybrid-hancom-ai-options.md
+1265
View File
File diff suppressed because it is too large Load Diff
docs/superpowers/specs/2026-03-16-cid-font-detection-design.md
+151
View File
@@ -0,0 +1,151 @@
# CID Font Extraction Failure Detection
Issue: [#286](https://github.com/opendataloader-project/opendataloader-pdf/issues/286)
## Problem
PDFs with CID-keyed fonts that lack ToUnicode mappings produce no usable text from veraPDF extraction. veraPDF replaces unmappable characters with U+FFFD (replacement character), which `TextProcessor.replaceUndefinedCharacters()` then converts to spaces. The result is empty or whitespace-only output with no indication to the user of what went wrong.
Users currently resort to external tools (e.g., pdfplumber) to pre-screen PDFs for CID issues before passing them to opendataloader-pdf.
## Solution
Detect pages with high replacement character ratios and:
1. **Always**: emit a WARNING log explaining the problem and suggesting `--hybrid-mode`
2. **When hybrid mode is on**: automatically route affected pages to OCR backend via TriageProcessor
No new CLI options. Hybrid mode setting is respected as-is.
## Design
### Detection: Replacement Character Ratio
`TextProcessor.measureReplacementCharRatio(List<IObject>)` counts `\uFFFD` characters across all TextChunks on a page and returns the ratio (0.0–1.0).
**Threshold**: 30%. CID-affected pages typically show 90%+ replacement characters. 30% catches real problems while avoiding false positives from PDFs with occasional unmappable glyphs.
**Measurement point**: Inside `ContentFilterProcessor.getFilteredContents()`, immediately before `replaceUndefinedCharacters()` is called (line 74). At this point veraPDF has already inserted `\uFFFD` but the characters haven't been replaced with spaces yet, so measurement is accurate.
**Safety of measurement point**: The prior processing steps (`mergeCloseTextChunks`, `trimTextChunksWhiteSpaces`, `filterConsecutiveSpaces`, `splitTextChunksByWhiteSpaces`) do not affect U+FFFD characters. U+FFFD is not whitespace, so it is not trimmed, compressed, or used as a split boundary. The count is accurate at this position.
**Zero-text pages**: When a page has no TextChunk objects (e.g., image-only pages), the method returns 0.0 to avoid division by zero. This correctly avoids triggering the CID warning on non-text pages.
The method uses `ChunkParser.REPLACEMENT_CHARACTER_STRING` constant (not a hardcoded `"\uFFFD"` literal) to stay consistent with `replaceUndefinedCharacters()`.
### Data Flow
The measured ratio is stored in `StaticLayoutContainers` per page:
```
ContentFilterProcessor.getFilteredContents()
│
├─ TextProcessor.measureReplacementCharRatio() → ratio
├─ StaticLayoutContainers.setReplacementCharRatio(pageNumber, ratio)
├─ if ratio >= 0.3: LOGGER.warning(...)
└─ TextProcessor.replaceUndefinedCharacters() // existing call
```
Note: `StaticLayoutContainers` currently stores global `ThreadLocal` scalars and lists, not per-page maps. Per-page data (e.g., bounding boxes) lives in `DocumentProcessor`. This change introduces a new per-page `Map<Integer, Double>` pattern to `StaticLayoutContainers`. We place it here rather than `DocumentProcessor` because it is layout-metadata consumed by `TriageProcessor`, keeping the triage data path self-contained. The existing `clearContainers()` method **must** be updated to clear this map to prevent cross-document data leakage in multi-document processing.
### Warning Log
Emitted from `ContentFilterProcessor` when ratio >= 0.3:
```
WARNING: Page 3: 94% of characters are replacement characters (U+FFFD).
This PDF likely contains CID-keyed fonts without ToUnicode mappings.
Text extraction may be incomplete. Consider using --hybrid-mode for OCR fallback.
```
This fires regardless of hybrid mode setting.
### Triage Routing
In `TriageProcessor.classifyPage()`, a new **Signal 0** is inserted before all existing signals (before TableBorder check). This signal only fires when hybrid mode is active, since `classifyPage()` is only called from `HybridDocumentProcessor`. In non-hybrid mode, only the warning log (from `ContentFilterProcessor`) is emitted:
```java
double replacementRatio = StaticLayoutContainers.getReplacementCharRatio(pageNumber);
if (replacementRatio >= 0.3) {
return TriageResult.backend(pageNumber, 1.0, signals);
}
```
Priority is highest (confidence 1.0) because a page with mostly broken text extraction gains nothing from Java-path processing.
### Behavior Matrix
| Hybrid Mode | Ratio >= 30% | Result |
|---|---|---|
| OFF | Yes | Warning log. Java path produces incomplete text. |
| OFF | No | No change. Normal processing. |
| ON (auto) | Yes | Warning log + auto-route to BACKEND (OCR). |
| ON (auto) | No | No change. Normal triage. |
| ON (full) | Yes | Warning log. All pages already go to BACKEND. |
| ON (full) | No | No change. All pages already go to BACKEND. |
## Changes
### Modified Files
| File | Change |
|---|---|
| `TextProcessor.java` | Add `measureReplacementCharRatio()` static method |
| `ContentFilterProcessor.java` | Call measurement before `replaceUndefinedCharacters()`, store result, emit warning |
| `StaticLayoutContainers.java` | Add `replacementCharRatios` map with getter/setter, clear in `clearContainers()` |
| `TriageProcessor.java` | Add Signal 0: replacement ratio check before TableBorder signal |
### New Files
| File | Purpose |
|---|---|
| `java/opendataloader-pdf-core/src/test/java/org/opendataloader/pdf/processors/CidFontDetectionTest.java` | e2e test using synthetic CID PDF |
| `java/opendataloader-pdf-core/src/test/resources/cid-font-no-tounicode.pdf` | Pre-generated test fixture (CID font, no ToUnicode) |
| `java/opendataloader-pdf-core/src/test/resources/generate-cid-test-pdf.py` | Generation script for reference |
### Modified Test Files
| File | Change |
|---|---|
| `TextProcessorTest.java` | 5 unit tests for `measureReplacementCharRatio()` |
| `TriageProcessorTest.java` | 3 unit tests for Signal 0 routing |
## Test Plan
### Unit Tests (TextProcessorTest)
- `testMeasureReplacementCharRatio_allReplacement` — all U+FFFD → 1.0
- `testMeasureReplacementCharRatio_noReplacement` — normal text → 0.0
- `testMeasureReplacementCharRatio_mixed` — 30% U+FFFD → 0.3
- `testMeasureReplacementCharRatio_emptyContents` — empty list → 0.0
- `testMeasureReplacementCharRatio_nonTextChunksIgnored` — non-text objects skipped
### Unit Tests (TriageProcessorTest)
- `testClassifyPage_highReplacementRatio_routesToBackend` — ratio 0.5 → BACKEND
- `testClassifyPage_lowReplacementRatio_noEffect` — ratio 0.1 → JAVA (default)
- `testClassifyPage_exactThreshold_routesToBackend` — ratio 0.3 → BACKEND
### Boundary Tests
- `testWarningNotEmitted_belowThreshold` — ratio 0.29 → no warning log emitted
- `testWarningEmitted_atThreshold` — ratio 0.30 → warning log emitted
### e2e Test (CidFontDetectionTest)
- Load pre-generated `cid-font-no-tounicode.pdf`
- Run through `ContentFilterProcessor.getFilteredContents()`
- Assert: `StaticLayoutContainers.getReplacementCharRatio(0) >= 0.3`
- Assert: warning log contains "replacement characters"
### Benchmark Regression
- Existing benchmark PDFs are normal documents with near-zero replacement ratios
- New logic does not affect existing test/benchmark results
## Not In Scope
- New CLI options (no `--cid-fallback` or similar)
- `npm run sync` not required (no CLI option changes)
- API signature changes (backward compatible)
- Benchmark threshold changes
docs/superpowers/specs/2026-04-29-hybrid-hancom-ai-options-design.md
+240
View File
@@ -0,0 +1,240 @@
# Hybrid hancom-ai Backend Options & CLI Refactoring
Issue: not yet filed — this design precedes the issue/PRs.
Scope: opendataloader-pdf (core CLI) + opendataloader-pdfua (downstream CLI)
## Problem
Five hancom-ai backend behaviors are already wired through `HybridConfig` but not exposed on any CLI:
| HybridConfig field | Default | Effect when set |
|---|---|---|
| `regionlistStrategy` | `table-first` | How to handle DLA label 7 (regionlist) |
| `ocrStrategy` | `auto` | Stream-only / fallback / OCR-only |
| `imageCache` | `memory` | Page image cache backing |
| `saveCrops` | `false` | Persist cropped figures (debug) |
| `cropOutputDir` | `null` | Output dir for saved crops (debug) |
`HancomAISchemaTransformer` and `HancomAIClient` already read these via `HybridConfig`, but `CLIOptions.java` has no flags and `applyHybridOptions()` never sets them. They are reachable only through programmatic `Config` use.
A second, structural problem makes adding these flags painful:
- **opendataloader-pdf** defines all CLI options in `CLIOptions.OPTION_DEFINITIONS` (private, single source for `options.json`/Python/Node bindings).
- **opendataloader-pdfua** defines its own `--hybrid`, `--hybrid-url`, `--hybrid-mode` separately in `Main.java`, with different defaults, then forwards them through `RemediationConfig` (3 hybrid fields, 9 constructor overloads).
Adding 5 new flags to both CLIs the current way means duplicating definitions, expanding `RemediationConfig` to 8 hybrid fields, and creating yet more constructor overloads. The two CLIs will drift further apart.
## Solution
Two coupled changes, executed in order:
1. **Add the 5 hancom-ai-specific options** to core `CLIOptions` under a `--hybrid-hancom-ai-*` prefix, with a guard that rejects them when `--hybrid` is not `hancom-ai`.
2. **Refactor core `CLIOptions` to be reusable** so opendataloader-pdfua imports the full core option set and adds only its own pdfua-specific options on top. `RemediationConfig` embeds a core `Config` instead of carrying parallel hybrid fields.
After this, adding any future core CLI option propagates to pdfua with a rebuild — no Main.java edits.
## Design
### Option naming: `--hybrid-hancom-ai-*` (full path)
Decision rationale (recorded for future contributors):
- `--hybrid-*` alone (e.g. `--hybrid-regionlist-strategy`) was rejected because docling-fast and other backends will never support these knobs; the name would lie about scope.
- `--hancom-ai-*` alone (e.g. `--hancom-ai-regionlist-strategy`) was rejected because it breaks the existing `--hybrid-mode/url/timeout/fallback` grouping and gives users two mental models.
- `--hybrid-hancom-ai-*` mirrors the `gh pr create` / `git remote add` full-path convention: every option name encodes the context it belongs to. `--help` alphabetic sort keeps all hybrid options in one block.
- A single comma-separated `--hybrid-hancom-ai-config` mega-option was rejected: defeats Apache Commons CLI typo detection, breaks `options.json` codegen for Python/Node bindings, complicates Windows path escaping.
The five new options:
| Long option | Type | Default | Exported | Description |
|---|---|---|---|---|
| `--hybrid-hancom-ai-regionlist-strategy` | string | `table-first` | yes | DLA label 7 handling. Values: `table-first`, `list-only` |
| `--hybrid-hancom-ai-ocr-strategy` | string | `auto` | yes | OCR strategy. Values: `off`, `auto`, `force` |
| `--hybrid-hancom-ai-image-cache` | string | `memory` | yes | Page image cache. Values: `memory`, `disk` |
| `--hybrid-hancom-ai-save-crops` | boolean | `false` | **no** | Persist cropped figures (debug only) |
| `--hybrid-hancom-ai-crop-output-dir` | string | `null` | **no** | Output directory for `--hybrid-hancom-ai-save-crops` |
`exported=false` for the two debug options keeps them out of `options.json` and the auto-generated Python/Node bindings, while remaining usable on the Java CLI. Same pattern as existing legacy options at `CLIOptions.java:201-208`.
### Validation
Inside `applyHybridOptions()`, after `--hybrid` is parsed, before returning:
```java
boolean usesHancomAiOnly =
commandLine.hasOption(HYBRID_HANCOM_AI_REGIONLIST_STRATEGY_LONG_OPTION) ||
commandLine.hasOption(HYBRID_HANCOM_AI_OCR_STRATEGY_LONG_OPTION) ||
commandLine.hasOption(HYBRID_HANCOM_AI_IMAGE_CACHE_LONG_OPTION) ||
commandLine.hasOption(HYBRID_HANCOM_AI_SAVE_CROPS_LONG_OPTION) ||
commandLine.hasOption(HYBRID_HANCOM_AI_CROP_OUTPUT_DIR_LONG_OPTION);
if (usesHancomAiOnly && !Config.HYBRID_HANCOM_AI.equals(config.getHybrid())) {
throw new IllegalArgumentException(
"Options --hybrid-hancom-ai-* require --hybrid=hancom-ai");
}
```
Per-value validation (e.g. `regionlistStrategy` must be `table-first`/`list-only`) is already implemented in `HybridConfig` setters and re-thrown as `IllegalArgumentException`. CLI passes the raw value through; HybridConfig is the validation authority.
### Core CLIOptions refactoring
Goal: pdfua can register every core option in its own `Options` and ask core to apply them to a `Config`.
Two new public static methods on `CLIOptions`:
```java
/** Register every core option onto an external Options. Used by downstream CLIs (pdfua). */
public static void addAllTo(Options options) {
for (OptionDefinition def : OPTION_DEFINITIONS) {
options.addOption(def.toOption());
}
}
/** Apply parsed core options to a Config. Used by downstream CLIs after parse. */
public static void applyAllTo(Config config, CommandLine commandLine) {
// body identical to current createConfigFromCommandLine,
// minus `new Config()` and minus the positional-arg output-folder fallback
}
```
The existing `defineOptions()` and `createConfigFromCommandLine()` keep their signatures and behavior; internally they delegate to the two new methods. Backward compatibility for any existing callers is preserved.
`OPTION_DEFINITIONS` stays private. `OptionDefinition` stays private. We expose only the two operations downstream CLIs actually need.
### pdfua/Main.java refactoring
Replace the self-defined hybrid option block (`Main.java:60-62, 99-101`) with:
```java
Options options = new Options();
CLIOptions.addAllTo(options); // all core options
options.addOption(null, "lang", true, ...); // pdfua-specific only
options.addOption(null, "audit-bundle-mode", true, ...);
options.addOption(null, "font-embed-mode", true, ...);
options.addOption(null, "conformance", true, ...);
// ... other pdfua-only options
CommandLine cmd = parser.parse(options, args);
Config config = new Config();
CLIOptions.applyAllTo(config, cmd);
applyPdfuaDefaults(config); // pdfua's hybrid defaults
```
### pdfua's hybrid defaults
pdfua currently hardcodes different defaults from core: `hybrid=hancom-ai`, `hybrid-url=http://localhost:18008`, `hybrid-mode=full` (`Main.java:60-62`). After refactoring, these become an explicit override block:
```java
private static void applyPdfuaDefaults(Config config) {
if (Config.HYBRID_OFF.equals(config.getHybrid())) {
config.setHybrid(Config.HYBRID_HANCOM_AI);
}
if (config.getHybridConfig().getUrl() == null) {
config.getHybridConfig().setUrl("http://localhost:18008");
}
if (Config.HYBRID_MODE_AUTO.equals(config.getHybridConfig().getMode())) {
config.getHybridConfig().setMode(Config.HYBRID_MODE_FULL);
}
}
```
This makes pdfua's deviation from core defaults explicit and grep-able. User-supplied flags still win — the override only fires when the user did not specify a value.
### RemediationConfig refactoring (Hard break)
Current state: 3 flat hybrid fields (`hybrid`, `hybridUrl`, `hybridMode`) and 9 constructor overloads (`RemediationConfig.java:42-105`).
Target state: embed core `Config` directly. Keep only pdfua-specific fields. Builder replaces the constructor overloads.
```java
public class RemediationConfig {
private final Config coreConfig; // ← core options live here
// pdfua-only fields
private final String input, output, lang;
private final AuditBundleMode auditBundleMode;
private final FontEmbedMode fontEmbedMode;
private final List<String> conformances;
private final int threads;
private final boolean enrichPictureDescription;
private RemediationConfig(Builder b) { ... }
public Config getCoreConfig() { return coreConfig; }
public String getHybrid() { return coreConfig.getHybrid(); }
public HybridConfig getHybridConfig() { return coreConfig.getHybridConfig(); }
// ... pdfua-only getters
public static Builder builder() { return new Builder(); }
public static class Builder { ... }
}
```
`getHybridUrl()` and `getHybridMode()` are **removed** (Hard break). All call sites move to `getHybridConfig().getUrl()` / `getHybridConfig().getMode()`. Verified call sites are confined to test code (`AuditBundleEmitterTest`, `CertificateIssuerTest`, `AuditManifestBuilderTest`, `RemediationConfigAuditBundleTest`) and `RemediationProcessor.java:165-170`. No external SDK consumers identified.
`RemediationProcessor.java:163-170` simplifies to:
```java
Config config = remediationConfig.getCoreConfig();
// (no per-field copy needed — Config already carries all hybrid state)
```
### Files touched
**opendataloader-pdf (core)**
- `java/opendataloader-pdf-cli/src/main/java/org/opendataloader/pdf/cli/CLIOptions.java`
- Add 5 option constants (long names + descriptions)
- Add 5 entries to `OPTION_DEFINITIONS` (3 exported, 2 not)
- Extend `applyHybridOptions()`: parse, set on `HybridConfig`, validate hancom-ai gate
- Add public `addAllTo(Options)` and `applyAllTo(Config, CommandLine)`
- `defineOptions()` and `createConfigFromCommandLine()` delegate to the new methods
- `java/opendataloader-pdf-cli/src/test/java/org/opendataloader/pdf/cli/CLIOptionsTest.java`
- Tests for parsing each new option
- Tests for the hancom-ai gate (using flag without `--hybrid=hancom-ai` throws)
- After core changes: run `npm run sync` to regenerate `options.json` + Python/Node bindings
**opendataloader-pdfua**
- `src/main/java/org/opendataloader/pdf/Main.java`
- Replace self-defined hybrid options with `CLIOptions.addAllTo(options)`
- Add `applyPdfuaDefaults(config)` helper
- Pass `coreConfig` to `RemediationConfig.Builder` instead of individual hybrid strings
- `src/main/java/org/opendataloader/pdf/remediation/RemediationConfig.java`
- Embed `Config coreConfig`, drop `hybrid/hybridUrl/hybridMode` fields
- Replace 9 constructors with one `Builder`
- Update getters: `getHybrid()` delegates, `getHybridUrl()`/`getHybridMode()` removed
- `src/main/java/org/opendataloader/pdf/remediation/RemediationProcessor.java`
- Simplify the lines 163-170 hybrid forwarding block
- All test files instantiating `RemediationConfig` (5 files identified): switch to `Builder`
### Build & verification
Per `opendataloader-pdfua/CLAUDE.md`:
1. Core changes first: `cd opendataloader-pdf/java && mvn install -DskipTests`
2. Then pdfua: `cd opendataloader-pdfua && mvn clean package`
3. Run both test suites
4. Manually verify:
- `opendataloader-pdf input.pdf --hybrid=hancom-ai --hybrid-hancom-ai-regionlist-strategy=list-only` works
- `opendataloader-pdf input.pdf --hybrid-hancom-ai-regionlist-strategy=list-only` (no `--hybrid`) fails with clear error
- `opendataloader-pdfua input.pdf --hybrid-hancom-ai-ocr-strategy=force` works (inherits core option, pdfua defaults still apply)
### Phasing
The work splits cleanly into independent commits/PRs:
1. **PR 1 (core)**: Add 5 options + gate validation + tests + `npm run sync`. Self-contained, mergeable alone.
2. **PR 2 (core)**: Extract `addAllTo` / `applyAllTo` public API. Pure refactoring, no behavior change.
3. **PR 3 (pdfua)**: Switch `Main.java` to use core's API; refactor `RemediationConfig` to Builder + embedded `Config`; update tests.
PR 1 unblocks immediate user value. PRs 2 & 3 are coupled (PR 3 depends on PR 2 merge → core artifact rebuild) but neither blocks PR 1.
## Out of scope
- Migrating other parts of pdfua to a Builder-style config (only `RemediationConfig` is touched).
- Adding flags to docling-fast or other hybrid backends. Spec is hancom-ai-specific.
- Changing `HybridConfig` setter validation (already strict).
- Renaming or repackaging existing `--hybrid-*` options.
- Documentation site updates — `opendataloader.org` reference docs are CI-generated from `options.json` after core merge.
examples/python/batch/README.md
+45
View File
@@ -0,0 +1,45 @@
# Batch Processing Example
Demonstrates processing multiple PDFs in a single invocation to avoid repeated Java JVM startup overhead.
## Prerequisites
- Python 3.10+
- Java 11+ (on PATH)
## Example
[`batch_processing.py`](batch_processing.py) shows two methods for batch conversion:
1. **File list** — Pass multiple PDF paths as a list
2. **Directory** — Pass a directory path (recursively finds all PDFs)
Both methods use a single JVM invocation, which is significantly faster than calling the CLI once per file.
**Run:**
```bash
pip install -r requirements.txt
python batch_processing.py
```
## Sample Output
```
Found 4 PDFs in pdf/
==========================================================
Method 1: Batch convert with file list
==========================================================
Document Pages Top-level
----------------------------------------------------------
1901.03003 15 241
2408.02509v1 14 365
chinese_scan 1 1
lorem 1 2
----------------------------------------------------------
Total 31 609
Processed 4 documents
Time: 7.95s (single JVM invocation)
```
examples/python/batch/batch_processing.py
+119
View File
@@ -0,0 +1,119 @@
#!/usr/bin/env python3
"""
Batch Processing Example
Demonstrates processing multiple PDFs in a single invocation to avoid
repeated Java JVM startup overhead. This is the recommended approach
for large-scale document pipelines.
Requires Python 3.10+.
Usage:
pip install opendataloader-pdf
python batch_processing.py
"""
from __future__ import annotations
import json
import tempfile
import time
from pathlib import Path
import opendataloader_pdf
def batch_convert(pdf_paths: list[str], output_dir: str) -> list[Path]:
"""Convert multiple PDFs in a single JVM invocation."""
opendataloader_pdf.convert(
input_path=pdf_paths,
output_dir=output_dir,
format="json,markdown",
quiet=True,
)
# Collect output JSON files
return sorted(Path(output_dir).glob("*.json"))
def convert_directory(directory: str, output_dir: str) -> list[Path]:
"""Convert all PDFs in a directory (recursive)."""
opendataloader_pdf.convert(
input_path=directory,
output_dir=output_dir,
format="json,markdown",
quiet=True,
)
return sorted(Path(output_dir).glob("*.json"))
def summarize_results(json_files: list[Path]) -> None:
"""Print a summary of all converted documents."""
total_pages = 0
total_elements = 0
print(f"\n{'Document':<40} {'Pages':>6} {'Top-level':>9}")
print("-" * 58)
for json_path in json_files:
with open(json_path, encoding="utf-8") as f:
doc = json.load(f)
pages = doc.get("number of pages", 0)
elements = len(doc.get("kids", []))
total_pages += pages
total_elements += elements
print(f"{json_path.stem:<40} {pages:>6} {elements:>9}")
print("-" * 58)
print(f"{'Total':<40} {total_pages:>6} {total_elements:>9}")
print(f"\nProcessed {len(json_files)} documents")
def main():
# Find sample PDFs relative to this script
script_dir = Path(__file__).resolve().parent
repo_root = script_dir.parent.parent.parent
samples_dir = repo_root / "samples" / "pdf"
pdf_files = sorted(samples_dir.glob("*.pdf"))
if not pdf_files:
print(f"No sample PDFs found at: {samples_dir}")
return
print(f"Found {len(pdf_files)} PDFs in {samples_dir.name}/")
for p in pdf_files:
print(f" - {p.name}")
# --- Method 1: Pass a list of files ---
print("\n" + "=" * 58)
print("Method 1: Batch convert with file list")
print("=" * 58)
with tempfile.TemporaryDirectory() as temp_dir:
start = time.perf_counter()
json_files = batch_convert(
[str(p) for p in pdf_files],
temp_dir,
)
elapsed = time.perf_counter() - start
summarize_results(json_files)
print(f"Time: {elapsed:.2f}s (single JVM invocation)")
# --- Method 2: Pass a directory ---
# Note: directory input recursively finds PDFs in subdirectories,
# so the file count may differ from Method 1 (which uses top-level glob).
print("\n" + "=" * 58)
print("Method 2: Convert entire directory")
print("=" * 58)
with tempfile.TemporaryDirectory() as temp_dir:
start = time.perf_counter()
json_files = convert_directory(str(samples_dir), temp_dir)
elapsed = time.perf_counter() - start
summarize_results(json_files)
print(f"Time: {elapsed:.2f}s (single JVM invocation)")
if __name__ == "__main__":
main()
examples/python/batch/requirements.txt
+2
View File
@@ -0,0 +1,2 @@
# Requires Python 3.10+
opendataloader-pdf>=2.2.1
examples/python/rag/README.md
+95
View File
@@ -0,0 +1,95 @@
# RAG Examples for OpenDataLoader PDF
Working examples demonstrating how to use OpenDataLoader PDF in RAG (Retrieval-Augmented Generation) pipelines.
## Prerequisites
- Python 3.10+
- Java 11+ (on PATH)
## Sample PDF
Examples use `samples/pdf/1901.03003.pdf` - a multi-page academic paper (arXiv:1901.03003) with:
- Two-column layout
- Multiple sections and headings
- Tables and figures
- Complex reading order
## Examples
### 1. Basic Chunking (No External Dependencies)
[`basic_chunking.py`](basic_chunking.py) demonstrates PDF-to-chunks conversion using only `opendataloader-pdf` and Python standard library. No external embedding or vector store dependencies.
**Features:**
- PDF to JSON conversion with reading order
- Three chunking strategies:
1. By element (paragraph, heading, list)
2. By section (grouped under headings)
3. Merged chunks (minimum size threshold)
- Bounding box metadata for citations
**Run:**
```bash
pip install opendataloader-pdf
python basic_chunking.py
```
### 2. LangChain Integration
[`langchain_example.py`](langchain_example.py) shows integration with the official LangChain loader.
**Features:**
- OpenDataLoaderPDFLoader usage
- Returns LangChain Document objects
- Ready for any LangChain pipeline
**Run:**
```bash
pip install -r requirements.txt
python langchain_example.py
```
## Sample Output
```
Processing: 1901.03003.pdf
==================================================
Document: 1901.03003.pdf
Pages: 9
Elements: 187
--- Strategy 1: Chunk by Element ---
Created 156 chunks
[1] RoBERTa: A Robustly Optimized BERT Pretraining Approach
Source: 1901.03003.pdf, Page 1, Position (108, 655)
[2] Yinhan Liu† Myle Ott† Naman Goyal† Jingfei Du† ...
Source: 1901.03003.pdf, Page 1, Position (142, 603)
--- Strategy 2: Chunk by Section ---
Created 12 chunks
Section: RoBERTa: A Robustly Optimized BERT Pretraining Approach
Section: 1 Introduction
Section: 2 Background
...
```
## Next Steps
After chunking, integrate with your preferred:
- **Embedding model**: OpenAI, Cohere, HuggingFace, etc.
- **Vector store**: Chroma, FAISS, Pinecone, Weaviate, etc.
Each chunk includes `text` and `metadata` ready for embedding:
```python
{
"text": "Language model pretraining has led to significant...",
"metadata": {
"type": "paragraph",
"page": 1,
"bbox": [108.0, 526.2, 286.5, 592.8],
"source": "1901.03003.pdf"
}
}
```
examples/python/rag/basic_chunking.py
+227
View File
@@ -0,0 +1,227 @@
#!/usr/bin/env python3
"""
Basic RAG Chunking Example - No External Dependencies
Demonstrates PDF-to-chunks conversion using only opendataloader-pdf
and Python standard library. Ready for integration with any embedding
model or vector store.
Usage:
pip install opendataloader-pdf
python basic_chunking.py
"""
import json
import tempfile
from pathlib import Path
import opendataloader_pdf
def convert_pdf_to_json(pdf_path: str, output_dir: str) -> Path:
"""Convert PDF to JSON and Markdown with reading order enabled."""
opendataloader_pdf.convert(
input_path=pdf_path,
output_dir=output_dir,
format="json,markdown",
reading_order="xycut",
quiet=True,
)
pdf_name = Path(pdf_path).stem
return Path(output_dir) / f"{pdf_name}.json"
def load_document(json_path: Path) -> dict:
"""Load the JSON output from OpenDataLoader."""
with open(json_path, encoding="utf-8") as f:
return json.load(f)
def chunk_by_element(doc: dict) -> list[dict]:
"""
Strategy 1: Chunk by semantic element.
Creates one chunk per paragraph, heading, or list element.
Best for: Fine-grained retrieval, precise citations.
"""
chunks = []
for element in doc.get("kids", []):
if element.get("type") in ("paragraph", "heading", "list"):
chunks.append({
"text": element.get("content", ""),
"metadata": {
"type": element["type"],
"page": element.get("page number"),
"bbox": element.get("bounding box"),
"source": doc.get("file name"),
}
})
return chunks
def chunk_by_section(doc: dict) -> list[dict]:
"""
Strategy 2: Chunk by heading/section.
Groups content under headings into coherent sections.
Best for: Context-rich retrieval, topic-based search.
"""
chunks = []
current_heading = None
current_content: list[str] = []
current_start_page = None
for element in doc.get("kids", []):
element_type = element.get("type")
if element_type == "heading":
# Save previous section
if current_content:
chunks.append({
"text": "\n".join(current_content),
"metadata": {
"heading": current_heading,
"page": current_start_page,
"source": doc.get("file name"),
}
})
current_heading = element.get("content", "")
current_content = [current_heading]
current_start_page = element.get("page number")
elif element_type in ("paragraph", "list"):
content = element.get("content", "")
if content:
current_content.append(content)
# Save the last section
if current_content:
chunks.append({
"text": "\n".join(current_content),
"metadata": {
"heading": current_heading,
"page": current_start_page,
"source": doc.get("file name"),
}
})
return chunks
def chunk_with_min_size(doc: dict, min_chars: int = 200) -> list[dict]:
"""
Strategy 3: Merge adjacent elements until minimum size.
Combines small paragraphs to avoid overly fragmented chunks.
Best for: Balanced chunk sizes, reducing noise.
"""
chunks = []
buffer_text = ""
buffer_pages: list[int] = []
for element in doc.get("kids", []):
if element.get("type") in ("paragraph", "heading", "list"):
content = element.get("content", "")
page = element.get("page number")
buffer_text += content + "\n"
if page and page not in buffer_pages:
buffer_pages.append(page)
if len(buffer_text) >= min_chars:
chunks.append({
"text": buffer_text.strip(),
"metadata": {
"pages": buffer_pages.copy(),
"source": doc.get("file name"),
}
})
buffer_text = ""
buffer_pages = []
# Save remaining buffer
if buffer_text.strip():
chunks.append({
"text": buffer_text.strip(),
"metadata": {
"pages": buffer_pages,
"source": doc.get("file name"),
}
})
return chunks
def format_citation(metadata: dict) -> str:
"""Generate a citation string from chunk metadata."""
source = metadata.get("source", "unknown")
page = metadata.get("page") or (metadata.get("pages", [None]) or [None])[0]
bbox = metadata.get("bbox")
citation = f"Source: {source}"
if page:
citation += f", Page {page}"
if bbox:
citation += f", Position ({bbox[0]:.0f}, {bbox[1]:.0f})"
return citation
def main():
# Find sample PDF relative to this script
# Using 1901.03003.pdf - a multi-page academic paper with complex layout
script_dir = Path(__file__).resolve().parent
repo_root = script_dir.parent.parent.parent
sample_pdf = repo_root / "samples" / "pdf" / "1901.03003.pdf"
if not sample_pdf.exists():
print(f"Sample PDF not found at: {sample_pdf}")
print("Make sure you're running from the repository.")
return
print(f"Processing: {sample_pdf.name}")
print("=" * 50)
# Convert PDF to JSON in a temp directory
with tempfile.TemporaryDirectory() as temp_dir:
json_path = convert_pdf_to_json(str(sample_pdf), temp_dir)
doc = load_document(json_path)
print(f"Document: {doc.get('file name')}")
print(f"Pages: {doc.get('number of pages')}")
print(f"Elements: {len(doc.get('kids', []))}")
# Strategy 1: By element
print("\n--- Strategy 1: Chunk by Element ---")
element_chunks = chunk_by_element(doc)
print(f"Created {len(element_chunks)} chunks")
for i, chunk in enumerate(element_chunks[:3]):
text_preview = chunk["text"][:60] + "..." if len(chunk["text"]) > 60 else chunk["text"]
print(f" [{i+1}] {text_preview}")
print(f" {format_citation(chunk['metadata'])}")
# Strategy 2: By section
print("\n--- Strategy 2: Chunk by Section ---")
section_chunks = chunk_by_section(doc)
print(f"Created {len(section_chunks)} chunks")
for i, chunk in enumerate(section_chunks[:2]):
heading = chunk["metadata"].get("heading", "No heading")
print(f" Section: {heading}")
print(f" Text: {chunk['text'][:60]}...")
# Strategy 3: Merged
print("\n--- Strategy 3: Merged Chunks (min 200 chars) ---")
merged_chunks = chunk_with_min_size(doc, min_chars=200)
print(f"Created {len(merged_chunks)} chunks")
for i, chunk in enumerate(merged_chunks[:2]):
print(f" [{i+1}] {len(chunk['text'])} chars: {chunk['text'][:50]}...")
# Show example chunk structure
print("\n--- Example Chunk Structure ---")
print("Each chunk has 'text' and 'metadata' ready for embedding:")
if element_chunks:
print(json.dumps(element_chunks[0], indent=2, ensure_ascii=False))
if __name__ == "__main__":
main()
examples/python/rag/langchain_example.py
+79
View File
@@ -0,0 +1,79 @@
#!/usr/bin/env python3
"""
LangChain Integration Example
Demonstrates using the official langchain-opendataloader-pdf package
for seamless RAG pipeline integration.
Usage:
pip install langchain-opendataloader-pdf
python langchain_example.py
"""
from pathlib import Path
from langchain_opendataloader_pdf import OpenDataLoaderPDFLoader
def main():
# Find sample PDF relative to this script
# Using 1901.03003.pdf - a multi-page academic paper with complex layout
script_dir = Path(__file__).resolve().parent
repo_root = script_dir.parent.parent.parent
sample_pdf = repo_root / "samples" / "pdf" / "1901.03003.pdf"
if not sample_pdf.exists():
print(f"Sample PDF not found at: {sample_pdf}")
print("Make sure you're running from the repository.")
return
print(f"Loading: {sample_pdf.name}")
print("=" * 50)
# Create loader with LangChain integration
loader = OpenDataLoaderPDFLoader(
file_path=[str(sample_pdf)],
format="text",
quiet=True,
)
# Load documents (returns LangChain Document objects)
documents = loader.load()
print(f"Loaded {len(documents)} document(s)\n")
for i, doc in enumerate(documents):
print(f"--- Document {i+1} ---")
print(f"Metadata: {doc.metadata}")
content_preview = doc.page_content[:200] + "..." if len(doc.page_content) > 200 else doc.page_content
print(f"Content:\n{content_preview}\n")
# Show integration points
print("--- LangChain Integration ---")
print("These Document objects work directly with:")
print(" - Text splitters: RecursiveCharacterTextSplitter, etc.")
print(" - Vector stores: Chroma, FAISS, Pinecone, etc.")
print(" - Retrievers: vectorstore.as_retriever()")
print(" - Chains: RetrievalQA, ConversationalRetrievalChain, etc.")
# Example: Using with a text splitter
print("\n--- Example: Text Splitting ---")
try:
from langchain_text_splitters import RecursiveCharacterTextSplitter
splitter = RecursiveCharacterTextSplitter(
chunk_size=500,
chunk_overlap=50,
)
chunks = splitter.split_documents(documents)
print(f"Split into {len(chunks)} chunks")
if chunks:
print(f"First chunk ({len(chunks[0].page_content)} chars):")
print(f" {chunks[0].page_content[:100]}...")
except ImportError:
print("Install langchain-text-splitters to see this example:")
print(" pip install langchain-text-splitters")
if __name__ == "__main__":
main()
examples/python/rag/requirements.txt
+3
View File
@@ -0,0 +1,3 @@
opendataloader-pdf>=2.2.1
langchain-opendataloader-pdf>=2.0.0
langchain-text-splitters>=1.1.2
java/.run/OpenDataLoaderCli.run.xml
+14
View File
@@ -0,0 +1,14 @@
<component name="ProjectRunConfigurationManager">
<configuration default="false" name="OpenDataLoaderCli" type="Application"
factoryName="Application">
<option name="ALTERNATIVE_JRE_PATH" value="corretto-21" />
<option name="ALTERNATIVE_JRE_PATH_ENABLED" value="true" />
<option name="MAIN_CLASS_NAME" value="org.opendataloader.pdf.cli.CLIMain" />
<module name="opendataloader-pdf-cli" />
<option name="PROGRAM_PARAMETERS"
value="../samples/pdf/1901.03003.pdf --markdown --html --pdf" />
<method v="2">
<option name="Make" enabled="true" />
</method>
</configuration>
</component>
java/checkstyle.xml
+13
View File
@@ -0,0 +1,13 @@
<?xml version="1.0"?>
<!DOCTYPE module PUBLIC "-//Checkstyle//DTD Checkstyle Configuration 1.3//EN" "https://checkstyle.org/dtds/configuration_1_3.dtd">
<module name="Checker">
<module name="TreeWalker">
<module name="UnusedImports"/>
<module name="WhitespaceAfter"/>
<module name="WhitespaceAround"/>
<module name="NoWhitespaceAfter"/>
<module name="NoWhitespaceBefore"/>
<module name="NeedBraces"/>
<module name="EmptyBlock"/>
</module>
</module>
java/opendataloader-pdf-cli/pom.xml
+137
View File
@@ -0,0 +1,137 @@
<?xml version="1.0" encoding="UTF-8"?>
<!--
Copyright 2025 Hancom Inc.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.opendataloader</groupId>
<artifactId>opendataloader-pdf-parent</artifactId>
<version>0.0.0</version>
<relativePath>../pom.xml</relativePath>
</parent>
<artifactId>opendataloader-pdf-cli</artifactId>
<packaging>jar</packaging>
<name>OpenDataLoader PDF CLI</name>
<description>OpenDataLoader PDF CLI</description>
<dependencies>
<dependency>
<groupId>org.opendataloader</groupId>
<artifactId>opendataloader-pdf-core</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>commons-cli</groupId>
<artifactId>commons-cli</artifactId>
</dependency>
<dependency>
<groupId>org.assertj</groupId>
<artifactId>assertj-core</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.junit.jupiter</groupId>
<artifactId>junit-jupiter</artifactId>
<scope>test</scope>
</dependency>
</dependencies>
<build>
<plugins>
<plugin>
<artifactId>maven-compiler-plugin</artifactId>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-shade-plugin</artifactId>
<executions>
<execution>
<phase>package</phase>
<goals>
<goal>shade</goal>
</goals>
<configuration>
<filters>
<filter>
<artifact>*:*</artifact>
<excludes>
<exclude>about.html</exclude>
<exclude>module-info.class</exclude>
<exclude>META-INF/*.MF</exclude>
<exclude>META-INF/*.SF</exclude>
<exclude>META-INF/*.DSA</exclude>
<exclude>META-INF/*.RSA</exclude>
<exclude>META-INF/versions/**</exclude>
<exclude>META-INF/LICENSE</exclude>
<exclude>META-INF/LICENSE.txt</exclude>
<exclude>META-INF/LICENSE.md</exclude>
<exclude>META-INF/NOTICE</exclude>
<exclude>META-INF/NOTICE.txt</exclude>
<exclude>META-INF/NOTICE.md</exclude>
</excludes>
</filter>
<filter>
<artifact>com.sun.xml.bind:jaxb-impl</artifact>
<excludes>
<exclude>com/sun/xml/bind/**</exclude>
</excludes>
</filter>
<filter>
<artifact>org.jacoco:*</artifact>
<excludes>
<exclude>META-INF/maven/**</exclude>
</excludes>
</filter>
</filters>
<transformers>
<transformer
implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
<mainClass>org.opendataloader.pdf.cli.CLIMain</mainClass>
</transformer>
<transformer
implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer"/>
<transformer
implementation="org.apache.maven.plugins.shade.resource.ComponentsXmlResourceTransformer"/>
<transformer
implementation="org.apache.maven.plugins.shade.resource.AppendingTransformer">
<resource>META-INF/DEPENDENCIES</resource>
</transformer>
<transformer
implementation="org.apache.maven.plugins.shade.resource.IncludeResourceTransformer">
<resource>META-INF/LICENSE</resource>
<file>${project.basedir}/../../LICENSE</file>
</transformer>
<transformer
implementation="org.apache.maven.plugins.shade.resource.IncludeResourceTransformer">
<resource>META-INF/NOTICE</resource>
<file>${project.basedir}/../../NOTICE</file>
</transformer>
</transformers>
</configuration>
</execution>
</executions>
</plugin>
</plugins>
</build>
</project>
java/opendataloader-pdf-cli/src/main/java/org/opendataloader/pdf/cli/CLIMain.java
+262
View File
@@ -0,0 +1,262 @@
/*
* Copyright 2025-2026 Hancom Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.opendataloader.pdf.cli;
import org.apache.commons.cli.*;
import org.opendataloader.pdf.api.Config;
import org.opendataloader.pdf.api.OpenDataLoaderPDF;
import org.opendataloader.pdf.api.cli.CLIOptions;
import org.opendataloader.pdf.exceptions.EncryptedTaggedPdfNotSupportedException;
import org.opendataloader.pdf.exceptions.InvalidPdfFileException;
import org.verapdf.exceptions.InvalidPasswordException;
import org.verapdf.wcag.algorithms.semanticalgorithms.containers.StaticContainers;
import java.io.File;
import java.util.Locale;
import java.util.logging.Handler;
import java.util.logging.Level;
import java.util.logging.Logger;
public class CLIMain {
private static final Logger LOGGER = Logger.getLogger(CLIMain.class.getCanonicalName());
private static final String HELP = "[options] <INPUT FILE OR FOLDER>...\n Options:";
private enum InputSource { CLI_ARGUMENT, DIRECTORY_CHILD }
/**
* Result of processing a path: whether all files succeeded, and how many
* PDF files were processed under it (counted recursively for directories,
* 1 or 0 for a single file). Used by {@link #processDirectory} to print a
* clear summary when a user-supplied folder contains no PDFs (PDFDLOSP-15).
*/
private static final class PathResult {
final boolean allSucceeded;
final int pdfCount;
PathResult(boolean allSucceeded, int pdfCount) {
this.allSucceeded = allSucceeded;
this.pdfCount = pdfCount;
}
}
public static void main(String[] args) {
int exitCode = run(args);
if (exitCode != 0) {
System.exit(exitCode);
}
}
/**
* Runs the CLI with the given arguments and returns the exit code.
*
* @param args command-line arguments
* @return 0 on success, non-zero on failure
*/
static int run(String[] args) {
Options options = CLIOptions.defineOptions();
HelpFormatter formatter = new HelpFormatter();
CommandLine commandLine;
try {
commandLine = new DefaultParser().parse(options, args);
} catch (ParseException e) {
System.out.println(e.getMessage());
formatter.printHelp(HELP, options);
return 2;
}
// Handle --export-options before requiring input files
if (commandLine.hasOption(CLIOptions.EXPORT_OPTIONS_LONG_OPTION)) {
CLIOptions.exportOptionsAsJson(System.out);
return 0;
}
if (commandLine.getArgs().length < 1) {
formatter.printHelp(HELP, options);
return 0;
}
String[] arguments = commandLine.getArgs();
Config config;
boolean quiet;
try {
config = CLIOptions.createConfigFromCommandLine(commandLine);
quiet = commandLine.hasOption(CLIOptions.QUIET_OPTION) || commandLine.hasOption("quiet");
} catch (IllegalArgumentException exception) {
System.out.println(exception.getMessage());
formatter.printHelp(HELP, options);
return 2;
}
configureLogging(quiet);
boolean hasFailure = false;
try {
for (String argument : arguments) {
if (!processPath(new File(argument), config, InputSource.CLI_ARGUMENT).allSucceeded) {
hasFailure = true;
}
}
} finally {
// Release resources (e.g., hybrid client thread pools)
OpenDataLoaderPDF.shutdown();
}
return hasFailure ? 1 : 0;
}
private static void configureLogging(boolean quiet) {
if (!quiet) {
return;
}
Logger rootLogger = Logger.getLogger("");
rootLogger.setLevel(Level.OFF);
for (Handler handler : rootLogger.getHandlers()) {
handler.setLevel(Level.OFF);
}
LOGGER.setLevel(Level.OFF);
}
/**
* Processes a file or directory, returning true if all files succeeded.
*
* <p>{@code source} distinguishes user-provided arguments
* ({@link InputSource#CLI_ARGUMENT}) from files discovered during directory
* traversal ({@link InputSource#DIRECTORY_CHILD}): a non-PDF given directly
* on the command line is reported as an error, while non-PDF files inside a
* directory are silently skipped (preserves batch-folder processing).
*/
private static PathResult processPath(File file, Config config, InputSource source) {
if (!file.exists()) {
LOGGER.log(Level.WARNING, "File or folder " + file.getAbsolutePath() + " not found.");
return new PathResult(false, 0);
}
if (file.isDirectory()) {
return processDirectory(file, config, source);
}
if (file.isFile()) {
boolean isPdf = isPdfFile(file);
if (source == InputSource.CLI_ARGUMENT && !isPdf) {
System.out.println("Error: '" + file.getName()
+ "' is not a PDF file. Input must be a PDF file or a folder containing PDF files.");
return new PathResult(false, 0);
}
return new PathResult(processFile(file, config, source), isPdf ? 1 : 0);
}
return new PathResult(true, 0);
}
/**
* Counts PDF files processed under the directory rooted at {@code file}
* (recursively), so the CLI can surface a clear summary when a
* user-supplied folder contains no PDFs. Without this feedback the program
* would exit silently with status 0 and the user could not distinguish
* "wrong folder", "empty folder", and "successful run" (PDFDLOSP-15).
*
* <p>The summary is only printed for folders given directly on the command
* line ({@link InputSource#CLI_ARGUMENT}) — nested subdirectories aggregate
* upward into the top-level count rather than each printing their own line.
*
* <p>The summary line is the final <em>result</em> of the run, not a log
* entry, and is therefore intentionally emitted on stdout even under
* {@code --quiet}. {@code --quiet} suppresses processing logs; users
* still need to see whether anything was actually processed. The path
* shown is {@link File#getPath()} (the literal argument the user typed,
* e.g. {@code .} or {@code basic_images}) rather than {@link File#getName()},
* which would be empty for {@code .} or trailing-slash inputs.
*/
private static PathResult processDirectory(File file, Config config, InputSource source) {
File[] children = file.listFiles();
if (children == null) {
LOGGER.log(Level.WARNING, "Unable to read folder " + file.getAbsolutePath());
return new PathResult(false, 0);
}
boolean allSucceeded = true;
int pdfCount = 0;
for (File child : children) {
PathResult childResult = processPath(child, config, InputSource.DIRECTORY_CHILD);
if (!childResult.allSucceeded) {
allSucceeded = false;
}
pdfCount += childResult.pdfCount;
}
if (source == InputSource.CLI_ARGUMENT) {
if (pdfCount == 0) {
System.out.println("No PDF files found in '" + file.getPath() + "'.");
} else {
System.out.println("Processed " + pdfCount + " PDF file"
+ (pdfCount == 1 ? "" : "s") + " in '" + file.getPath() + "'.");
}
}
return new PathResult(allSucceeded, pdfCount);
}
/**
* Processes a single PDF file.
*
* <p>{@code source} controls how an {@link InvalidPdfFileException} from
* the magic-number guard is surfaced: {@link InputSource#CLI_ARGUMENT}
* routes it to stdout as a user-facing error and fails the run;
* {@link InputSource#DIRECTORY_CHILD} logs a WARNING and treats the file
* as silently skipped so batch-folder runs can still exit 0.
*
* @param file the file to process
* @param config the processing configuration
* @param source whether the file came from a CLI argument or directory traversal
* @return true if processing succeeded (or the file was a silently
* skipped non-PDF inside a directory), false on error.
*/
private static boolean processFile(File file, Config config, InputSource source) {
if (!isPdfFile(file)) {
LOGGER.log(Level.FINE, "Skipping non-PDF file " + file.getAbsolutePath());
return true;
}
try {
OpenDataLoaderPDF.processFile(file.getAbsolutePath(), config);
return true;
} catch (InvalidPdfFileException invalid) {
if (source == InputSource.CLI_ARGUMENT) {
System.out.println("Error: " + invalid.getMessage());
return false;
}
LOGGER.log(Level.WARNING, invalid.getMessage() + " Skipping.");
return true;
} catch (InvalidPasswordException exception) {
String password = config.getPassword();
String message = (password == null || password.isEmpty())
? "Error: '" + file.getName() + "' is password-protected. Use --password option."
: "Error: Incorrect password for '" + file.getName() + "'.";
System.out.println(message);
return false;
} catch (EncryptedTaggedPdfNotSupportedException exception) {
System.out.println("Error: " + exception.getMessage());
return false;
} catch (Exception exception) {
LOGGER.log(Level.SEVERE, "Exception during processing file " + file.getAbsolutePath() + ": " +
exception.getMessage(), exception);
return false;
} finally {
StaticContainers.closeImagesUtils();
}
}
private static boolean isPdfFile(File file) {
if (!file.isFile()) {
return false;
}
String name = file.getName();
return name.toLowerCase(Locale.ROOT).endsWith(".pdf");
}
}
java/opendataloader-pdf-cli/src/test/java/org/opendataloader/pdf/cli/CLIMainTest.java
+672
View File
@@ -0,0 +1,672 @@
/*
* Copyright 2025-2026 Hancom Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.opendataloader.pdf.cli;
import org.apache.pdfbox.pdmodel.PDDocument;
import org.apache.pdfbox.pdmodel.PDPage;
import org.apache.pdfbox.pdmodel.encryption.AccessPermission;
import org.apache.pdfbox.pdmodel.encryption.StandardProtectionPolicy;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.io.TempDir;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.PrintStream;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.ArrayList;
import java.util.List;
import java.util.logging.Handler;
import java.util.logging.Level;
import java.util.logging.LogRecord;
import java.util.logging.Logger;
import static org.junit.jupiter.api.Assertions.*;
class CLIMainTest {
@TempDir
Path tempDir;
/**
* When processing a PDF file throws any exception, CLIMain.run() must return
* a non-zero exit code. This test uses a malformed PDF with hybrid mode
* targeting an unreachable server, which triggers an exception during processing.
*
* <p>Before this fix, processFile() caught all exceptions and logged them at
* SEVERE level but never propagated the failure to the exit code.
*
* <p>Regression test for https://github.com/opendataloader-project/opendataloader-pdf/issues/287
*/
@Test
void testProcessingFailureReturnsNonZeroExitCode() throws IOException {
// Create a minimal PDF file so processFile is actually invoked
// (the file must exist and end in .pdf to pass the isPdfFile check)
Path testPdf = tempDir.resolve("test.pdf");
Files.write(testPdf, "%PDF-1.4 minimal".getBytes());
// Use an unreachable hybrid URL — the processing will fail either at
// the hybrid availability check or during PDF parsing, both of which
// must result in a non-zero exit code.
int exitCode = CLIMain.run(new String[]{
"--hybrid", "docling-fast",
"--hybrid-url", "http://127.0.0.1:59999",
testPdf.toString()
});
assertNotEquals(0, exitCode,
"Exit code must be non-zero when file processing fails");
}
/**
* When a directory contains a file that fails processing, run() must return
* non-zero, even though other files in the directory may succeed. The
* "failure" here must be a genuine processing error (unreachable hybrid
* backend), not a corrupt-content classification — corrupt PDFs inside a
* directory are silently skipped by design (see
* {@link #testCorruptPdfInsideDirectoryIsSilentlySkipped()}), so the test
* uses a parseable PDF and lets the hybrid availability check fail.
*/
@Test
void testDirectoryWithFailingFileReturnsNonZeroExitCode() throws IOException {
Path dir = tempDir.resolve("docs");
Files.createDirectory(dir);
Path testPdf = dir.resolve("bad.pdf");
try (PDDocument doc = new PDDocument()) {
doc.addPage(new PDPage());
doc.save(testPdf.toFile());
}
int exitCode = CLIMain.run(new String[]{
"--hybrid", "docling-fast",
"--hybrid-url", "http://127.0.0.1:59999",
dir.toString()
});
assertNotEquals(0, exitCode,
"Exit code must be non-zero when any file in directory fails");
}
/**
* Normal invocation with no arguments should return 0 (just prints help).
*/
@Test
void testNoArgumentsReturnsZero() {
int exitCode = CLIMain.run(new String[]{});
assertEquals(0, exitCode);
}
/**
* Invalid CLI arguments (e.g., unrecognized option) must return exit code 2,
* following POSIX convention for command-line usage errors.
*/
@Test
void testInvalidArgumentsReturnsExitCode2() {
int exitCode = CLIMain.run(new String[]{"--no-such-option"});
assertEquals(2, exitCode);
}
/**
* Non-existent input file must return non-zero exit code.
*/
@Test
void testNonExistentFileReturnsNonZeroExitCode() {
int exitCode = CLIMain.run(new String[]{"/nonexistent/path/file.pdf"});
assertNotEquals(0, exitCode,
"Exit code must be non-zero when input file does not exist");
}
/**
* A non-PDF file given directly on the command line must produce a clear
* error message on stdout and a non-zero exit code, instead of silently
* exiting with success.
*
* <p>Regression test for PDFDLOSP-5.
*/
@Test
void testNonPdfTopLevelArgumentEmitsErrorAndFails() throws IOException {
Path png = tempDir.resolve("abcd.png");
Files.write(png, new byte[]{(byte) 0x89, 0x50, 0x4E, 0x47});
String stdout = captureStdoutOf(() -> CLIMain.run(new String[]{png.toString()}));
assertTrue(stdout.contains("'abcd.png' is not a PDF file"),
"stdout must mention the offending file name and that it is not a PDF; got: " + stdout);
assertTrue(stdout.contains("Input must be a PDF file or a folder containing PDF files"),
"stdout must guide the user to valid input; got: " + stdout);
}
@Test
void testNonPdfTopLevelArgumentReturnsNonZeroExitCode() throws IOException {
Path png = tempDir.resolve("abcd.png");
Files.write(png, new byte[]{(byte) 0x89, 0x50, 0x4E, 0x47});
int exitCode = CLIMain.run(new String[]{png.toString()});
assertNotEquals(0, exitCode,
"Exit code must be non-zero when the top-level argument is not a PDF");
}
/**
* Non-PDF files inside a directory must continue to be silently skipped —
* batch-folder processing with mixed file types is a legitimate use case
* and must not regress.
*/
@Test
void testNonPdfFileInsideDirectoryIsSilentlySkipped() throws IOException {
Path dir = tempDir.resolve("mixed");
Files.createDirectory(dir);
Files.write(dir.resolve("note.png"), new byte[]{(byte) 0x89, 0x50, 0x4E, 0x47});
Files.write(dir.resolve("readme.txt"), "hello".getBytes(StandardCharsets.UTF_8));
String stdout = captureStdoutOf(() -> CLIMain.run(new String[]{dir.toString()}));
assertFalse(stdout.contains("is not a PDF file"),
"Files discovered during directory traversal must not trigger the top-level error; got: " + stdout);
}
@Test
void testDirectoryWithOnlyNonPdfFilesReturnsZero() throws IOException {
Path dir = tempDir.resolve("non-pdf-only");
Files.createDirectory(dir);
Files.write(dir.resolve("note.png"), new byte[]{(byte) 0x89, 0x50, 0x4E, 0x47});
int exitCode = CLIMain.run(new String[]{dir.toString()});
assertEquals(0, exitCode,
"Directories containing only non-PDF files must succeed (silent skip)");
}
/**
* A folder containing zero processable PDFs must emit a clear "No PDF files
* found" message instead of exiting silently with status 0 — without this
* the user cannot distinguish "wrong folder", "empty folder", and
* "successful run". The path shown is the literal argument the user typed
* (File#getPath), so {@code .} and trailing-slash inputs render correctly.
*
* <p>Regression test for PDFDLOSP-15.
*/
@Test
void testEmptyDirectoryEmitsNoPdfFoundMessage() throws IOException {
Path dir = tempDir.resolve("empty");
Files.createDirectory(dir);
String[] stdoutHolder = new String[1];
int exitCode = (int) runCapturingStdout(
() -> CLIMain.run(new String[]{dir.toString()}),
stdoutHolder);
assertEquals(0, exitCode, "Empty folder is not an error");
assertTrue(stdoutHolder[0].contains("No PDF files found in '" + dir + "'"),
"stdout must report that no PDFs were found; got: " + stdoutHolder[0]);
}
@Test
void testDirectoryWithOnlyNonPdfFilesEmitsNoPdfFoundMessage() throws IOException {
Path dir = tempDir.resolve("basic_images");
Files.createDirectory(dir);
Files.write(dir.resolve("a.png"), new byte[]{(byte) 0x89, 0x50, 0x4E, 0x47});
Files.write(dir.resolve("b.jpg"), new byte[]{(byte) 0xFF, (byte) 0xD8});
String[] stdoutHolder = new String[1];
int exitCode = (int) runCapturingStdout(
() -> CLIMain.run(new String[]{dir.toString()}),
stdoutHolder);
assertEquals(0, exitCode);
assertTrue(stdoutHolder[0].contains("No PDF files found in '" + dir + "'"),
"stdout must report that no PDFs were found; got: " + stdoutHolder[0]);
}
/**
* Subdirectories must aggregate into the top-level summary rather than
* emit their own "No PDF files found" / "Processed N..." lines — only the
* user-supplied folder path appears in the output.
*/
@Test
void testNestedSubdirectoriesAggregateIntoTopLevelSummary() throws IOException {
Path top = tempDir.resolve("top");
Path sub = top.resolve("sub");
Files.createDirectories(sub);
Files.write(sub.resolve("nested.pdf"), "%PDF-1.4 minimal".getBytes(StandardCharsets.UTF_8));
String[] stdoutHolder = new String[1];
runCapturingStdout(
() -> CLIMain.run(new String[]{top.toString()}),
stdoutHolder);
assertTrue(stdoutHolder[0].contains("Processed 1 PDF file in '" + top + "'"),
"stdout must summarize at the top-level folder including nested PDFs; got: "
+ stdoutHolder[0]);
assertFalse(stdoutHolder[0].contains("in '" + sub + "'"),
"stdout must not emit a separate summary for nested subdirectories; got: "
+ stdoutHolder[0]);
}
@Test
void testDirectoryWithMultiplePdfsEmitsProcessedSummary() throws IOException {
Path dir = tempDir.resolve("docs");
Files.createDirectory(dir);
Files.write(dir.resolve("a.pdf"), "%PDF-1.4 minimal".getBytes(StandardCharsets.UTF_8));
Files.write(dir.resolve("b.pdf"), "%PDF-1.4 minimal".getBytes(StandardCharsets.UTF_8));
String[] stdoutHolder = new String[1];
runCapturingStdout(
() -> CLIMain.run(new String[]{dir.toString()}),
stdoutHolder);
assertTrue(stdoutHolder[0].contains("Processed 2 PDF files in '" + dir + "'"),
"stdout must summarize with plural 'files' when count > 1; got: "
+ stdoutHolder[0]);
}
/**
* When the command line mixes a valid PDF argument with a top-level non-PDF
* argument, the run must fail overall but only the non-PDF entry should
* produce the user-facing error message.
*/
@Test
void testMixedTopLevelArgumentsReportOnlyNonPdfAndFailOverall() throws IOException {
Path png = tempDir.resolve("note.png");
Files.write(png, new byte[]{(byte) 0x89, 0x50, 0x4E, 0x47});
Path pdf = tempDir.resolve("doc.pdf");
Files.write(pdf, "%PDF-1.4 minimal".getBytes(StandardCharsets.UTF_8));
String[] stdoutHolder = new String[1];
int exitCode = (int) runCapturingStdout(
() -> CLIMain.run(new String[]{pdf.toString(), png.toString()}),
stdoutHolder);
assertNotEquals(0, exitCode,
"Exit code must be non-zero when any top-level argument is not a PDF");
assertTrue(stdoutHolder[0].contains("'note.png' is not a PDF file"),
"stdout must call out the non-PDF argument by name; got: " + stdoutHolder[0]);
}
/**
* Password-protected PDF with no password supplied must produce a single
* user-friendly error message (no verapdf stack trace) and exit non-zero.
*
* <p>Regression test for PDFDLOSP-9.
*/
@Test
void testPasswordProtectedWithoutPasswordEmitsFriendlyError() throws IOException {
Path pdf = createPasswordProtectedPdf(tempDir.resolve("locked.pdf"), "1234");
String[] holder = new String[1];
int exitCode = (int) runCapturingStdout(
() -> CLIMain.run(new String[]{pdf.toString()}),
holder);
assertNotEquals(0, exitCode, "exit code must be non-zero");
assertTrue(holder[0].contains("'locked.pdf' is password-protected"),
"stdout must call out the file by name and explain it is password-protected; got: " + holder[0]);
assertTrue(holder[0].contains("--password"),
"stdout must guide the user to the --password option; got: " + holder[0]);
assertFalse(holder[0].contains("InvalidPasswordException"),
"stdout must not expose internal exception type; got: " + holder[0]);
assertFalse(holder[0].contains("at org.verapdf"),
"stdout must not expose a stack trace; got: " + holder[0]);
}
/**
* Password-protected PDF with an incorrect password must produce a single
* user-friendly error message (no verapdf stack trace) and exit non-zero.
*
* <p>Regression test for PDFDLOSP-9.
*/
@Test
void testPasswordProtectedWithWrongPasswordEmitsFriendlyError() throws IOException {
Path pdf = createPasswordProtectedPdf(tempDir.resolve("locked.pdf"), "1234");
String[] holder = new String[1];
int exitCode = (int) runCapturingStdout(
() -> CLIMain.run(new String[]{pdf.toString(), "--password", "wrongpw"}),
holder);
assertNotEquals(0, exitCode, "exit code must be non-zero");
assertTrue(holder[0].contains("Incorrect password for 'locked.pdf'"),
"stdout must report incorrect password and the file name; got: " + holder[0]);
assertFalse(holder[0].contains("InvalidPasswordException"),
"stdout must not expose internal exception type; got: " + holder[0]);
assertFalse(holder[0].contains("at org.verapdf"),
"stdout must not expose a stack trace; got: " + holder[0]);
}
/**
* Correct password must keep the existing success path: exit code 0 and
* the JSON output produced next to the input. Regression guard for the
* happy case while fixing PDFDLOSP-9.
*/
@Test
void testPasswordProtectedWithCorrectPasswordSucceeds() throws IOException {
Path pdf = createPasswordProtectedPdf(tempDir.resolve("locked.pdf"), "1234");
int exitCode = CLIMain.run(new String[]{pdf.toString(), "--password", "1234"});
assertEquals(0, exitCode, "correct password must keep exit code 0");
assertTrue(Files.exists(tempDir.resolve("locked.json")),
"JSON output must be produced next to the input PDF");
}
private static Path createPasswordProtectedPdf(Path target, String password) throws IOException {
try (PDDocument document = new PDDocument()) {
document.addPage(new PDPage());
AccessPermission permissions = new AccessPermission();
StandardProtectionPolicy policy = new StandardProtectionPolicy(password, password, permissions);
policy.setEncryptionKeyLength(128);
document.protect(policy);
document.save(target.toFile());
}
return target;
}
// --- PDFDLOSP-21: hybrid backend unavailable, stack-trace guard -------
/**
* When the hybrid backend is unreachable and {@code --hybrid-fallback} is
* NOT supplied, CLIMain must surface the friendly fail-fast message
* (including the new {@code --hybrid-fallback} hint) via the SEVERE log
* record and must never leak a Java stack trace to stdout. Regression for
* PDFDLOSP-21.
*/
@Test
void testHybridUnavailableWithoutFallbackEmitsFriendlyMessageNoStackTrace() throws IOException {
Path testPdf = tempDir.resolve("ok.pdf");
// Minimal but valid-enough header so the file reaches HybridDocumentProcessor.
// PDF parsing happens after health-check on this branch.
try (PDDocument doc = new PDDocument()) {
doc.addPage(new PDPage());
doc.save(testPdf.toFile());
}
// Reserve an ephemeral port and release it so connect() is refused.
int closedPort;
try (java.net.ServerSocket socket = new java.net.ServerSocket(0)) {
closedPort = socket.getLocalPort();
}
String[] stdoutHolder = new String[1];
StringBuilder logCapture = new StringBuilder();
java.util.logging.Logger cliLogger = java.util.logging.Logger.getLogger(
"org.opendataloader.pdf.cli.CLIMain");
java.util.logging.Handler captureHandler = new java.util.logging.Handler() {
@Override public void publish(java.util.logging.LogRecord record) {
logCapture.append(record.getMessage()).append('\n');
Throwable thrown = record.getThrown();
while (thrown != null) {
logCapture.append(thrown).append('\n');
for (StackTraceElement frame : thrown.getStackTrace()) {
logCapture.append("\tat ").append(frame).append('\n');
}
thrown = thrown.getCause();
}
}
@Override public void flush() {}
@Override public void close() {}
};
cliLogger.addHandler(captureHandler);
int exitCode;
try {
exitCode = (int) runCapturingStdout(() -> CLIMain.run(new String[]{
"--hybrid", "docling-fast",
"--hybrid-mode", "full",
"--hybrid-url", "http://127.0.0.1:" + closedPort,
"--output", tempDir.toString(),
testPdf.toString()
}), stdoutHolder);
} finally {
cliLogger.removeHandler(captureHandler);
}
assertNotEquals(0, exitCode,
"Exit code must be non-zero when hybrid backend is unreachable and fallback is off");
String out = stdoutHolder[0];
assertFalse(out.contains("\tat "),
"stdout must not contain a Java stack trace ('\\tat '); got: " + out);
String logged = logCapture.toString();
assertTrue(logged.contains("Hybrid server is not available"),
"Log output should explain that the hybrid backend is unreachable; got: " + logged);
assertTrue(logged.contains("--hybrid-fallback"),
"Log output should include the --hybrid-fallback recovery hint; got: " + logged);
}
// --- PDFDLOSP-14: magic-number guard regressions ----------------------
private static final byte[] JPEG_PREFIX = new byte[]{
(byte) 0xFF, (byte) 0xD8, (byte) 0xFF, (byte) 0xE0,
0x00, 0x10, 'J', 'F', 'I', 'F', 0x00
};
/**
* A .pdf-named file whose content is not actually PDF must produce the
* user-facing magic-number error on stdout, fail with non-zero exit, AND
* must not leak any veraPDF internal stack trace. Regression for
* PDFDLOSP-14.
*/
@Test
void testPdfExtensionWithNonPdfContentEmitsMagicNumberError() throws IOException {
Path fakePdf = tempDir.resolve("fake.pdf");
Files.write(fakePdf, JPEG_PREFIX);
String[] stdoutHolder = new String[1];
int exitCode = (int) runCapturingStdout(
() -> CLIMain.run(new String[]{fakePdf.toString()}),
stdoutHolder);
assertNotEquals(0, exitCode,
"Exit code must be non-zero when input content is not PDF");
assertTrue(stdoutHolder[0].contains("'fake.pdf' is not a valid PDF file (missing %PDF- header)"),
"stdout must contain the magic-number error; got: " + stdoutHolder[0]);
}
/**
* Stack-trace leakage guard: the veraPDF internals must NEVER appear on
* stdout for the non-PDF-content case. This is the core regression we
* are preventing — if the catch branch is reverted or the guard moves
* back behind PDDocument construction, this test fails.
*/
@Test
void testPdfExtensionWithNonPdfContentDoesNotLeakStackTrace() throws IOException {
Path fakePdf = tempDir.resolve("fake.pdf");
Files.write(fakePdf, JPEG_PREFIX);
String[] stdoutHolder = new String[1];
runCapturingStdout(
() -> CLIMain.run(new String[]{fakePdf.toString()}),
stdoutHolder);
String out = stdoutHolder[0];
assertFalse(out.toLowerCase(java.util.Locale.ROOT).contains("verapdf"),
"Output must not contain 'verapdf'; got: " + out);
assertFalse(out.contains("java.io.IOException:"),
"Output must not contain 'java.io.IOException:'; got: " + out);
assertFalse(out.contains("\tat "),
"Output must not contain a Java stack trace ('\\tat '); got: " + out);
}
/**
* A directory containing a .pdf-named non-PDF file alongside another
* non-PDF file must (a) NOT print the top-level magic-number error to
* stdout, (b) still exit 0 (silent skip preserves PR #496's batch-folder
* semantics), and (c) emit a single WARNING log line so operators can
* investigate why a file was skipped. Captures CLIMain's JUL logger
* directly because java.util.logging routes WARNING to a ConsoleHandler
* on stderr by default and may be suppressed entirely under --quiet.
*/
@Test
void testPdfExtensionNonPdfInsideDirectoryIsSilentlySkipped() throws IOException {
Path dir = tempDir.resolve("mixed");
Files.createDirectory(dir);
Files.write(dir.resolve("fake.pdf"), JPEG_PREFIX);
Files.write(dir.resolve("note.png"), new byte[]{(byte) 0x89, 0x50, 0x4E, 0x47});
List<LogRecord> records = new ArrayList<>();
String[] stdoutHolder = new String[1];
int exitCode = (int) captureLogsOf(records, () -> runCapturingStdout(
() -> CLIMain.run(new String[]{dir.toString()}),
stdoutHolder)).intValue();
assertEquals(0, exitCode,
"Directory containing only invalid files must still exit 0 (silent skip)");
assertFalse(stdoutHolder[0].contains("Error:"),
"Directory traversal must not surface 'Error:' on stdout; got: " + stdoutHolder[0]);
assertFalse(stdoutHolder[0].contains("missing %PDF- header"),
"Directory traversal must not surface the magic-number error on stdout; got: "
+ stdoutHolder[0]);
long warningMatches = records.stream().filter(r ->
r.getLevel() == Level.WARNING
&& r.getMessage() != null
&& r.getMessage().contains("'fake.pdf'")
&& r.getMessage().contains("missing %PDF- header")).count();
assertEquals(1L, warningMatches,
"Exactly one WARNING log must name the offending file and the missing header; "
+ "got: " + records);
}
/**
* A PDF with a valid {@code %PDF-} header but a corrupt or truncated body
* must surface the friendly "corrupted or truncated content" error on
* stdout, exit non-zero, AND must not leak a veraPDF stack trace. The
* wording must also be distinct from the missing-header case so users
* can tell the two failure modes apart.
*/
@Test
void testCorruptPdfShowsCorruptedFileErrorWithoutStackTrace() throws IOException {
Path corrupt = tempDir.resolve("corrupt.pdf");
// Valid header, but nothing else — veraPDF will fail at parse time.
Files.write(corrupt, "%PDF-1.4\n<garbage>".getBytes(StandardCharsets.UTF_8));
String[] stdoutHolder = new String[1];
int exitCode = (int) runCapturingStdout(
() -> CLIMain.run(new String[]{corrupt.toString()}),
stdoutHolder);
String out = stdoutHolder[0];
assertNotEquals(0, exitCode,
"Corrupt PDF must still fail (non-zero exit); got stdout: " + out);
assertTrue(out.contains("'corrupt.pdf' is not a valid PDF file (corrupted or truncated content)"),
"stdout must surface the friendly corrupted-content error; got: " + out);
assertFalse(out.contains("missing %PDF- header"),
"Corrupt-but-headered PDFs must NOT be misreported as missing the header; got: "
+ out);
assertFalse(out.toLowerCase(java.util.Locale.ROOT).contains("verapdf"),
"Output must not contain 'verapdf'; got: " + out);
assertFalse(out.contains("java.io.IOException:"),
"Output must not contain 'java.io.IOException:'; got: " + out);
assertFalse(out.contains("\tat "),
"Output must not contain a Java stack trace ('\\tat '); got: " + out);
}
/**
* A corrupt-but-headered PDF inside a directory must follow the same
* batch-folder semantics as a non-PDF-content file: WARNING log, no
* {@code Error:} on stdout, exit 0. This mirrors
* {@link #testPdfExtensionNonPdfInsideDirectoryIsSilentlySkipped()} for
* the corrupted-body branch so both failure modes route through the
* same {@link InvalidPdfFileException} handler in CLIMain.
*/
@Test
void testCorruptPdfInsideDirectoryIsSilentlySkipped() throws IOException {
Path dir = tempDir.resolve("mixed-corrupt");
Files.createDirectory(dir);
Files.write(dir.resolve("corrupt.pdf"),
"%PDF-1.4\n<garbage>".getBytes(StandardCharsets.UTF_8));
Files.write(dir.resolve("note.png"), new byte[]{(byte) 0x89, 0x50, 0x4E, 0x47});
List<LogRecord> records = new ArrayList<>();
String[] stdoutHolder = new String[1];
int exitCode = (int) captureLogsOf(records, () -> runCapturingStdout(
() -> CLIMain.run(new String[]{dir.toString()}),
stdoutHolder)).intValue();
assertEquals(0, exitCode,
"Directory traversal must exit 0 even when a child is a corrupt PDF "
+ "(silent-skip semantics); got stdout: " + stdoutHolder[0]);
assertFalse(stdoutHolder[0].contains("Error:"),
"Directory traversal must not surface 'Error:' on stdout; got: " + stdoutHolder[0]);
long warningMatches = records.stream().filter(r ->
r.getLevel() == Level.WARNING
&& r.getMessage() != null
&& r.getMessage().contains("'corrupt.pdf'")
&& r.getMessage().contains("corrupted or truncated")).count();
assertEquals(1L, warningMatches,
"Exactly one WARNING log must name the offending file and describe "
+ "the corrupted-content failure; got: " + records);
}
/**
* Runs {@code action} with a temporary {@link Handler} attached to
* {@link CLIMain}'s JUL logger, collecting every {@link LogRecord} into
* {@code sink}. Restores the prior handler list and level on exit.
*
* <p>JUL is global; this helper assumes sequential test execution
* (JUnit 5 + Surefire default). Returns whatever {@code action} returned.
*/
private static <T> T captureLogsOf(List<LogRecord> sink, java.util.concurrent.Callable<T> action) {
Logger logger = Logger.getLogger(CLIMain.class.getCanonicalName());
Level priorLevel = logger.getLevel();
boolean priorUseParent = logger.getUseParentHandlers();
Handler capture = new Handler() {
@Override public void publish(LogRecord record) { sink.add(record); }
@Override public void flush() { }
@Override public void close() { }
};
capture.setLevel(Level.ALL);
logger.addHandler(capture);
logger.setLevel(Level.ALL);
logger.setUseParentHandlers(false);
try {
return action.call();
} catch (Exception exception) {
throw new RuntimeException(exception);
} finally {
logger.removeHandler(capture);
logger.setLevel(priorLevel);
logger.setUseParentHandlers(priorUseParent);
}
}
private static String captureStdoutOf(Runnable action) {
String[] holder = new String[1];
runCapturingStdout(() -> {
action.run();
return 0;
}, holder);
return holder[0];
}
// System.setOut is JVM-global; this helper assumes sequential test
// execution (JUnit 5 + maven-surefire default). Revisit if parallel
// test execution is enabled — concurrent captures would interleave.
private static long runCapturingStdout(java.util.concurrent.Callable<Integer> action, String[] stdoutHolder) {
PrintStream originalOut = System.out;
ByteArrayOutputStream buffer = new ByteArrayOutputStream();
System.setOut(new PrintStream(buffer, true, StandardCharsets.UTF_8));
try {
int exitCode = action.call();
stdoutHolder[0] = buffer.toString(StandardCharsets.UTF_8);
return exitCode;
} catch (Exception exception) {
throw new RuntimeException(exception);
} finally {
System.setOut(originalOut);
}
}
}
java/opendataloader-pdf-cli/src/test/java/org/opendataloader/pdf/cli/FormatLogRegressionTest.java
+124
View File
@@ -0,0 +1,124 @@
/*
* Copyright 2025-2026 Hancom Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.opendataloader.pdf.cli;
import org.apache.pdfbox.pdmodel.PDDocument;
import org.apache.pdfbox.pdmodel.PDPage;
import org.junit.jupiter.api.io.TempDir;
import org.junit.jupiter.params.ParameterizedTest;
import org.junit.jupiter.params.provider.ValueSource;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.Callable;
import java.util.logging.Handler;
import java.util.logging.Level;
import java.util.logging.LogRecord;
import java.util.logging.Logger;
import static org.junit.jupiter.api.Assertions.assertEquals;
/**
* Regression test for PDFDLOSP-26: every {@code --format} value must emit
* exactly one INFO-level "Created &lt;path&gt;" log line after writing its
* output. Tagged-PDF previously skipped this log because
* {@code AutoTaggingProcessor.createTaggedPDF()} had no {@code Logger}.
*/
class FormatLogRegressionTest {
@TempDir
Path tempDir;
@ParameterizedTest(name = "--format {0} emits exactly one Created log")
@ValueSource(strings = {"json", "text", "html", "pdf", "markdown", "tagged-pdf"})
void formatEmitsExactlyOneCreatedLog(String format) throws IOException {
Path pdf = createMinimalPdf(tempDir.resolve("in.pdf"));
Path outDir = tempDir.resolve(format);
Files.createDirectories(outDir);
List<LogRecord> records = new ArrayList<>();
int exitCode = captureAllLogsOf(records, () -> CLIMain.run(new String[]{
"--format", format,
"--output", outDir.toString(),
pdf.toString()
}));
assertEquals(0, exitCode, () ->
"Exit code must be 0 for --format " + format + "; records: " + summarize(records));
long createdCount = records.stream()
.filter(r -> r.getLevel() == Level.INFO)
.filter(r -> r.getMessage() != null && r.getMessage().contains("Created"))
.count();
assertEquals(1L, createdCount, () ->
"Exactly one INFO 'Created ...' log expected for --format " + format
+ "; got " + createdCount + ". Records: " + summarize(records));
}
private static Path createMinimalPdf(Path target) throws IOException {
try (PDDocument document = new PDDocument()) {
document.addPage(new PDPage());
document.save(target.toFile());
}
return target;
}
/**
* Captures every {@link LogRecord} emitted by any logger under the
* {@code org.opendataloader.pdf} hierarchy. Attaching to the root logger
* would also include third-party noise; attaching to the package root
* picks up CLIMain plus every generator/processor in core (JUL records
* propagate to parent handlers).
*
* <p>JUL is global; this assumes sequential test execution
* (JUnit 5 + Surefire default).
*/
private static <T> T captureAllLogsOf(List<LogRecord> sink, Callable<T> action) {
Logger logger = Logger.getLogger("org.opendataloader.pdf");
Level priorLevel = logger.getLevel();
boolean priorUseParent = logger.getUseParentHandlers();
Handler capture = new Handler() {
@Override public void publish(LogRecord record) { sink.add(record); }
@Override public void flush() { }
@Override public void close() { }
};
capture.setLevel(Level.ALL);
logger.addHandler(capture);
logger.setLevel(Level.ALL);
logger.setUseParentHandlers(false);
try {
return action.call();
} catch (Exception exception) {
throw new RuntimeException(exception);
} finally {
logger.removeHandler(capture);
logger.setLevel(priorLevel);
logger.setUseParentHandlers(priorUseParent);
}
}
private static String summarize(List<LogRecord> records) {
StringBuilder sb = new StringBuilder("[");
for (LogRecord r : records) {
sb.append('(').append(r.getLevel()).append(": ").append(r.getMessage()).append(") ");
}
return sb.append(']').toString();
}
}
java/opendataloader-pdf-core/pom.xml
+245
View File
@@ -0,0 +1,245 @@
<?xml version="1.0" encoding="UTF-8"?>
<!--
Copyright 2025-2026 Hancom Inc.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.opendataloader</groupId>
<artifactId>opendataloader-pdf-parent</artifactId>
<version>0.0.0</version>
<relativePath>../pom.xml</relativePath>
</parent>
<artifactId>opendataloader-pdf-core</artifactId>
<packaging>jar</packaging>
<name>OpenDataLoader PDF Core</name>
<description>OpenDataLoader PDF Core</description>
<dependencies>
<dependency>
<groupId>org.verapdf</groupId>
<artifactId>validation-model</artifactId>
<version>${verapdf.version}</version>
<exclusions>
<exclusion>
<groupId>org.verapdf</groupId>
<artifactId>feature-reporting</artifactId>
</exclusion>
<exclusion>
<groupId>org.jacoco</groupId>
<artifactId>jacoco-maven-plugin</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.verapdf</groupId>
<artifactId>wcag-validation</artifactId>
<version>${verapdf.version}</version>
<exclusions>
<exclusion>
<groupId>org.jacoco</groupId>
<artifactId>jacoco-maven-plugin</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.verapdf</groupId>
<artifactId>wcag-algorithms</artifactId>
<version>${verapdf.wcag.algs.version}</version>
<exclusions>
<exclusion>
<groupId>org.jacoco</groupId>
<artifactId>jacoco-maven-plugin</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-databind</artifactId>
<version>${jackson.databind.version}</version>
</dependency>
<dependency>
<groupId>commons-cli</groupId>
<artifactId>commons-cli</artifactId>
</dependency>
<!--
okhttp 5's plain `okhttp` artifact is an empty Kotlin-Multiplatform
metadata jar; the JVM classes live in `okhttp-jvm`, and Maven cannot
resolve the JVM variant from Gradle module metadata — so depend on
`okhttp-jvm` explicitly. Version is managed by the okhttp-bom in the
parent pom (see java/pom.xml dependencyManagement).
-->
<dependency>
<groupId>com.squareup.okhttp3</groupId>
<artifactId>okhttp-jvm</artifactId>
</dependency>
<dependency>
<groupId>org.junit.jupiter</groupId>
<artifactId>junit-jupiter</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>com.squareup.okhttp3</groupId>
<artifactId>mockwebserver3</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.assertj</groupId>
<artifactId>assertj-core</artifactId>
<scope>test</scope>
</dependency>
</dependencies>
<build>
<resources>
<resource>
<directory>${project.basedir}/../../</directory>
<includes>
<include>LICENSE</include>
<include>NOTICE</include>
</includes>
<targetPath>META-INF</targetPath>
<filtering>false</filtering>
</resource>
<resource>
<directory>${project.basedir}/../../THIRD_PARTY</directory>
<includes>
<include>**/*</include>
</includes>
<targetPath>META-INF/THIRD_PARTY</targetPath>
<filtering>false</filtering>
</resource>
</resources>
<plugins>
<plugin>
<groupId>org.codehaus.mojo</groupId>
<artifactId>flatten-maven-plugin</artifactId>
<configuration>
<updatePomFile>true</updatePomFile>
<flattenMode>oss</flattenMode>
<pomElements>
<name/>
<description/>
<url/>
<scm/>
<developers/>
</pomElements>
</configuration>
<executions>
<execution>
<id>flatten</id>
<phase>process-resources</phase>
<goals>
<goal>flatten</goal>
</goals>
</execution>
<execution>
<id>flatten.clean</id>
<phase>clean</phase>
<goals>
<goal>clean</goal>
</goals>
</execution>
</executions>
</plugin>
<plugin>
<artifactId>maven-compiler-plugin</artifactId>
</plugin>
<plugin>
<groupId>org.jacoco</groupId>
<artifactId>jacoco-maven-plugin</artifactId>
<executions>
<execution>
<goals>
<goal>prepare-agent</goal>
</goals>
</execution>
<execution>
<id>report</id>
<phase>test</phase>
<goals>
<goal>report</goal>
</goals>
</execution>
</executions>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-source-plugin</artifactId>
<executions>
<execution>
<id>attach-sources</id>
<goals>
<goal>jar-no-fork</goal>
</goals>
</execution>
</executions>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-javadoc-plugin</artifactId>
<executions>
<execution>
<id>attach-javadocs</id>
<goals>
<goal>jar</goal>
</goals>
</execution>
</executions>
</plugin>
</plugins>
</build>
<profiles>
<profile>
<id>release</id>
<build>
<plugins>
<plugin>
<groupId>org.sonatype.central</groupId>
<artifactId>central-publishing-maven-plugin</artifactId>
<extensions>true</extensions>
<configuration>
<publishingServerId>central</publishingServerId>
<autoPublish>true</autoPublish>
<waitUntil>published</waitUntil>
</configuration>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-gpg-plugin</artifactId>
<executions>
<execution>
<id>sign-artifacts</id>
<phase>verify</phase>
<goals>
<goal>sign</goal>
</goals>
</execution>
</executions>
</plugin>
</plugins>
</build>
</profile>
</profiles>
</project>
java/opendataloader-pdf-core/src/main/java/org/opendataloader/pdf/api/AutoTagger.java
+92
View File
@@ -0,0 +1,92 @@
/*
* Copyright 2025-2026 Hancom Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.opendataloader.pdf.api;
import org.opendataloader.pdf.processors.AutoTaggingProcessor;
import org.opendataloader.pdf.processors.DocumentProcessor;
import org.opendataloader.pdf.processors.ExtractionResult;
import org.verapdf.pd.PDDocument;
import org.verapdf.tools.StaticResources;
import java.io.IOException;
/**
* Public API for standalone PDF auto-tagging.
*
* <p>Returns an in-memory tagged {@link PDDocument} without writing intermediate files.
* Use {@link OpenDataLoaderPDF#processFile} for the full extraction + output pipeline.
*
* <p>Example usage:
* <pre>{@code
* Config config = new Config();
* config.setHybrid("docling-fast");
*
* try (TaggingResult result = AutoTagger.tag("input.pdf", config)) {
* PDDocument tagged = result.getDocument();
* // use tagged document...
* result.saveTo("output_tagged.pdf");
* }
* AutoTagger.shutdown();
* }</pre>
*/
public final class AutoTagger {
private AutoTagger() {
}
/**
* Extract content from a PDF and produce a tagged PDF document in-memory.
* Output format flags in config are ignored — this method only produces
* a tagged PDDocument.
*
* @param inputPdf path to the input PDF file
* @param config configuration (extraction + hybrid fields are used;
* output format flags are ignored)
* @return result containing the tagged PDDocument and timing metadata
* @throws IOException if unable to read or process the PDF
*/
public static TaggingResult tag(String inputPdf, Config config, Float pdfVersion) throws IOException {
ExtractionResult extraction = DocumentProcessor.extractContents(inputPdf, config);
return tag(extraction, pdfVersion);
}
/**
* Tag a PDF document from a previously computed {@link ExtractionResult}.
* Use this when you need both tagged PDF and other output formats from the
* same extraction — call {@link DocumentProcessor#extractContents} once,
* then pass the result here and to other output generators.
*
* @param extraction pre-computed extraction result
* @return result containing the tagged PDDocument and timing metadata
* @throws IOException if unable to tag the document
*/
public static TaggingResult tag(ExtractionResult extraction, Float pdfVersion) throws IOException {
long t0 = System.nanoTime();
PDDocument document = StaticResources.getDocument();
AutoTaggingProcessor.tagDocument(document, extraction.getContents(), pdfVersion);
long taggingNs = System.nanoTime() - t0;
return new TaggingResult(document, extraction.getExtractionNs(), taggingNs,
extraction.getHybridTimings(), extraction.getElementMetadata());
}
/**
* Release hybrid client resources. Call when processing is complete.
*/
public static void shutdown() {
OpenDataLoaderPDF.shutdown();
}
}
java/opendataloader-pdf-core/src/main/java/org/opendataloader/pdf/api/Config.java
+922
View File
@@ -0,0 +1,922 @@
/*
* Copyright 2025-2026 Hancom Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.opendataloader.pdf.api;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
import java.util.Locale;
import java.util.Set;
import java.util.logging.Level;
import java.util.logging.Logger;
import org.opendataloader.pdf.hybrid.HybridConfig;
/**
* Configuration class for the PDF processing.
* Use this class to specify output formats, text processing options, and other settings.
*/
public class Config {
private static final Logger LOGGER = Logger.getLogger(Config.class.getCanonicalName());
/** Reading order option: no sorting, keeps PDF COS object order. */
public static final String READING_ORDER_OFF = "off";
/** Reading order option: XY-Cut++ algorithm for layout-aware sorting. */
public static final String READING_ORDER_XYCUT = "xycut";
private static Set<String> readingOrderOptions = new HashSet<>();
/** Hybrid mode: off (Java-only processing, no external dependency). */
public static final String HYBRID_OFF = "off";
/** Hybrid mode: docling backend (Docling FastAPI server). */
public static final String HYBRID_DOCLING = "docling";
/** Hybrid mode: docling-fast backend (deprecated alias for docling). */
public static final String HYBRID_DOCLING_FAST = "docling-fast";
/** Hybrid mode: hancom backend (Hancom Document AI). */
public static final String HYBRID_HANCOM = "hancom";
/** Hybrid mode: hancom-ai backend (Hancom AI HOCR SDK — individual modules). */
public static final String HYBRID_HANCOM_AI = "hancom-ai";
/** Hybrid mode: azure backend (Azure Document Intelligence). */
public static final String HYBRID_AZURE = "azure";
/** Hybrid mode: google backend (Google Document AI). */
public static final String HYBRID_GOOGLE = "google";
private static Set<String> hybridOptions = new HashSet<>();
/** Hybrid triage mode: auto (dynamic triage based on page content). */
public static final String HYBRID_MODE_AUTO = "auto";
/** Hybrid triage mode: full (skip triage, send all pages to backend). */
public static final String HYBRID_MODE_FULL = "full";
private static Set<String> hybridModeOptions = new HashSet<>();
/** Placeholder string for page number in separators. */
public static final String PAGE_NUMBER_STRING = "%page-number%";
private String password;
private boolean isGenerateMarkdown = false;
private boolean isGenerateHtml = false;
private boolean isGeneratePDF = false;
private boolean keepLineBreaks = false;
private boolean isGenerateJSON = true;
private boolean isGenerateText = false;
private boolean isGenerateTaggedPDF = false;
private boolean useStructTree = false;
private boolean useHTMLInMarkdown = false;
private boolean addImageToMarkdown = false;
private String replaceInvalidChars = " ";
private String outputFolder;
private String tableMethod = TABLE_METHOD_DEFAULT;
private String readingOrder = READING_ORDER_XYCUT;
private String markdownPageSeparator = "";
private String textPageSeparator = "";
private String htmlPageSeparator = "";
private String imageOutput = IMAGE_OUTPUT_EXTERNAL;
private String imageFormat = IMAGE_FORMAT_PNG;
private String imageDir;
private String pages;
private List<Integer> cachedPageNumbers;
private final FilterConfig filterConfig = new FilterConfig();
private String hybrid = HYBRID_OFF;
private final HybridConfig hybridConfig = new HybridConfig();
private boolean includeHeaderFooter = false;
private boolean detectStrikethrough = false;
/** Table detection method: default (border-based detection). */
public static final String TABLE_METHOD_DEFAULT = "default";
/** Table detection method: cluster-based detection (includes border-based). */
public static final String TABLE_METHOD_CLUSTER = "cluster";
private static Set<String> tableMethodOptions = new HashSet<>();
/** Image format: PNG. */
public static final String IMAGE_FORMAT_PNG = "png";
/** Image format: JPEG. */
public static final String IMAGE_FORMAT_JPEG = "jpeg";
private static Set<String> imageFormatOptions = new HashSet<>();
/** Image output mode: no image extraction. */
public static final String IMAGE_OUTPUT_OFF = "off";
/** Image output mode: embedded as Base64 data URIs. */
public static final String IMAGE_OUTPUT_EMBEDDED = "embedded";
/** Image output mode: external file references. */
public static final String IMAGE_OUTPUT_EXTERNAL = "external";
private static Set<String> imageOutputOptions = new HashSet<>();
static {
readingOrderOptions.add(READING_ORDER_OFF);
readingOrderOptions.add(READING_ORDER_XYCUT);
tableMethodOptions.add(TABLE_METHOD_DEFAULT);
tableMethodOptions.add(TABLE_METHOD_CLUSTER);
imageFormatOptions.add(IMAGE_FORMAT_PNG);
imageFormatOptions.add(IMAGE_FORMAT_JPEG);
imageOutputOptions.add(IMAGE_OUTPUT_OFF);
imageOutputOptions.add(IMAGE_OUTPUT_EMBEDDED);
imageOutputOptions.add(IMAGE_OUTPUT_EXTERNAL);
hybridOptions.add(HYBRID_OFF);
hybridOptions.add(HYBRID_DOCLING);
hybridOptions.add(HYBRID_DOCLING_FAST); // deprecated alias
hybridOptions.add(HYBRID_HANCOM);
hybridOptions.add(HYBRID_HANCOM_AI);
// azure, google added when implemented
hybridModeOptions.add(HYBRID_MODE_AUTO);
hybridModeOptions.add(HYBRID_MODE_FULL);
}
/**
* Gets the filter config.
*
* @return The FilterConfig.
*/
public FilterConfig getFilterConfig() {
return filterConfig;
}
/**
* Default constructor initializing the configuration with default values.
*/
public Config() {
}
/**
* Gets the password for opening encrypted PDF files.
*
* @return The password, or null if not set.
*/
public String getPassword() {
return password;
}
/**
* Sets the password for opening encrypted PDF files.
*
* @param password The password to use.
*/
public void setPassword(String password) {
this.password = password;
}
/**
* Checks if Markdown output generation is enabled.
* <p>
* Markdown generation is automatically enabled if {@link #isAddImageToMarkdown()} or
* {@link #isUseHTMLInMarkdown()} is true.
*
* @return true if Markdown output should be generated, false otherwise.
*/
public boolean isGenerateMarkdown() {
return isGenerateMarkdown || isAddImageToMarkdown() || isUseHTMLInMarkdown();
}
/**
* Enables or disables Markdown output generation.
*
* @param generateMarkdown true to enable, false to disable.
*/
public void setGenerateMarkdown(boolean generateMarkdown) {
isGenerateMarkdown = generateMarkdown;
}
/**
* Checks if HTML output generation is enabled.
*
* @return true if HTML output should be generated, false otherwise.
*/
public boolean isGenerateHtml() {
return isGenerateHtml;
}
/**
* Enables or disables HTML output generation.
*
* @param generateHtml true to enable, false to disable.
*/
public void setGenerateHtml(boolean generateHtml) {
isGenerateHtml = generateHtml;
}
/**
* Checks if a new PDF with tagged structure is generated.
*
* @return true if PDF generation is enabled, false otherwise.
*/
public boolean isGeneratePDF() {
return isGeneratePDF;
}
/**
* Enables or disables generation of a new, tagged PDF.
*
* @param generatePDF true to enable, false to disable.
*/
public void setGeneratePDF(boolean generatePDF) {
isGeneratePDF = generatePDF;
}
/**
* Checks if original line breaks within text blocks should be preserved.
*
* @return true if line breaks are preserved, false otherwise.
*/
public boolean isKeepLineBreaks() {
return keepLineBreaks;
}
/**
* Sets whether to preserve original line breaks within text blocks.
*
* @param keepLineBreaks true to preserve line breaks, false to merge lines into paragraphs.
*/
public void setKeepLineBreaks(boolean keepLineBreaks) {
this.keepLineBreaks = keepLineBreaks;
}
/**
* Checks if JSON output generation is enabled. Defaults to true.
*
* @return true if JSON output should be generated, false otherwise.
*/
public boolean isGenerateJSON() {
return isGenerateJSON;
}
/**
* Enables or disables JSON output generation.
*
* @param generateJSON true to enable, false to disable.
*/
public void setGenerateJSON(boolean generateJSON) {
isGenerateJSON = generateJSON;
}
/**
* Checks if plain text output generation is enabled.
*
* @return true if plain text output should be generated, false otherwise.
*/
public boolean isGenerateText() {
return isGenerateText;
}
/**
* Enables or disables plain text output generation.
*
* @param generateText true to enable, false to disable.
*/
public void setGenerateText(boolean generateText) {
isGenerateText = generateText;
}
public boolean isGenerateTaggedPDF() {
return isGenerateTaggedPDF;
}
public void setGenerateTaggedPDF(boolean generateTaggedPDF) {
isGenerateTaggedPDF = generateTaggedPDF;
}
/**
* Checks if HTML tags should be used within the Markdown output for complex structures like tables.
*
* @return true if HTML is used in Markdown, false otherwise.
*/
public boolean isUseHTMLInMarkdown() {
return useHTMLInMarkdown;
}
/**
* Enables or disables the use of HTML tags in Markdown output.
* Enabling this will also enable {@link #isGenerateMarkdown()}.
*
* @param useHTMLInMarkdown true to use HTML, false for pure Markdown.
*/
public void setUseHTMLInMarkdown(boolean useHTMLInMarkdown) {
this.useHTMLInMarkdown = useHTMLInMarkdown;
}
/**
* Checks if images should be extracted and included in the Markdown output.
*
* @return true if images are included in Markdown, false otherwise.
*/
public boolean isAddImageToMarkdown() {
return addImageToMarkdown;
}
/**
* Enables or disables the inclusion of extracted images in Markdown output.
* Enabling this will also enable {@link #isGenerateMarkdown()}.
*
* @param addImageToMarkdown true to include images, false otherwise.
*/
public void setAddImageToMarkdown(boolean addImageToMarkdown) {
this.addImageToMarkdown = addImageToMarkdown;
}
/**
* Gets the path to the output folder where generated files will be saved.
*
* @return The output folder path.
*/
public String getOutputFolder() {
return outputFolder;
}
/**
* Sets the path to the output folder where generated files will be saved.
* The directory will be created if it does not exist.
*
* @param outputFolder The path to the output folder.
*/
public void setOutputFolder(String outputFolder) {
this.outputFolder = outputFolder;
}
/**
* Gets the character, that replaces invalid or unrecognized characters (e.g., �, \u0000).
*
* @return The specified replacement character.
*/
public String getReplaceInvalidChars() {
return replaceInvalidChars;
}
/**
* Sets the character, that replaces invalid or unrecognized characters (e.g., �, \u0000).
*
* @param replaceInvalidChars The specified replacement character.
*/
public void setReplaceInvalidChars(String replaceInvalidChars) {
this.replaceInvalidChars = replaceInvalidChars;
}
/**
* Checks if the PDF structure tree should be used for document parsing.
*
* @return true if structure tree should be used, false otherwise.
*/
public boolean isUseStructTree() {
return useStructTree;
}
/**
* Enables or disables use of PDF structure tree for document parsing.
*
* @param useStructTree true to use structure tree, false otherwise.
*/
public void setUseStructTree(boolean useStructTree) {
this.useStructTree = useStructTree;
}
/**
* Checks if cluster-based table detection is enabled.
*
* @return true if cluster table detection is enabled, false otherwise.
*/
public boolean isClusterTableMethod() {
return TABLE_METHOD_CLUSTER.equals(tableMethod);
}
/**
* Gets the table detection method.
*
* @return The table detection method (default or cluster).
*/
public String getTableMethod() {
return tableMethod;
}
/**
* Sets the table detection method.
*
* @param tableMethod The table detection method (default or cluster).
* @throws IllegalArgumentException if the method is not supported.
*/
public void setTableMethod(String tableMethod) {
if (tableMethod != null && !isValidTableMethod(tableMethod)) {
throw new IllegalArgumentException(
String.format("Unsupported table method '%s'. Supported values: %s",
tableMethod, getTableMethodOptions(", ")));
}
this.tableMethod = tableMethod != null ? tableMethod.toLowerCase(Locale.ROOT) : TABLE_METHOD_DEFAULT;
}
/**
* Gets the list of methods of table detection.
*
* @param delimiter the delimiter to use between options
* @return the string with methods separated by the delimiter
*/
public static String getTableMethodOptions(CharSequence delimiter) {
return String.join(delimiter, tableMethodOptions);
}
/**
* Checks if the given table method is valid.
*
* @param method The table method to check.
* @return true if the method is valid, false otherwise.
*/
public static boolean isValidTableMethod(String method) {
return method != null && tableMethodOptions.contains(method.toLowerCase(Locale.ROOT));
}
/**
* Gets the reading order, that states in which order content should be processed.
*
* @return The specified order.
*/
public String getReadingOrder() {
return readingOrder;
}
/**
* Sets the reading order, that states in which order content should be processed.
*
* @param readingOrder The specified order (off or xycut).
* @throws IllegalArgumentException if the order is not supported.
*/
public void setReadingOrder(String readingOrder) {
if (readingOrder != null && !isValidReadingOrder(readingOrder)) {
throw new IllegalArgumentException(
String.format("Unsupported reading order '%s'. Supported values: %s",
readingOrder, getReadingOrderOptions(", ")));
}
this.readingOrder = readingOrder != null ? readingOrder.toLowerCase(Locale.ROOT) : READING_ORDER_XYCUT;
}
/**
* Gets the list of reading order options.
*
* @param delimiter The delimiter to use between options.
* @return The string with reading orders separated by the delimiter.
*/
public static String getReadingOrderOptions(CharSequence delimiter) {
return String.join(delimiter, readingOrderOptions);
}
/**
* Checks if the given reading order is valid.
*
* @param order The reading order to check.
* @return true if the order is valid, false otherwise.
*/
public static boolean isValidReadingOrder(String order) {
return order != null && readingOrderOptions.contains(order.toLowerCase(Locale.ROOT));
}
/**
* Gets the string, that separates content from different pages in markdown.
*
* @return The specified string.
*/
public String getMarkdownPageSeparator() {
return markdownPageSeparator;
}
/**
* Sets the string, that separates content from different pages in markdown.
*
* @param markdownPageSeparator The specified string.
*/
public void setMarkdownPageSeparator(String markdownPageSeparator) {
this.markdownPageSeparator = markdownPageSeparator;
}
/**
* Gets the string, that separates content from different pages in text.
*
* @return The specified string.
*/
public String getTextPageSeparator() {
return textPageSeparator;
}
/**
* Sets the string, that separates content from different pages in text.
*
* @param textPageSeparator The specified string.
*/
public void setTextPageSeparator(String textPageSeparator) {
this.textPageSeparator = textPageSeparator;
}
/**
* Gets the string, that separates content from different pages in html.
*
* @return The specified string.
*/
public String getHtmlPageSeparator() {
return htmlPageSeparator;
}
/**
* Sets the string, that separates content from different pages in html.
*
* @param htmlPageSeparator The specified string.
*/
public void setHtmlPageSeparator(String htmlPageSeparator) {
this.htmlPageSeparator = htmlPageSeparator;
}
/**
* Checks if images should be embedded as Base64 data URIs in the output.
*
* @return true if images should be embedded as Base64, false for file path references.
*/
public boolean isEmbedImages() {
return IMAGE_OUTPUT_EMBEDDED.equals(imageOutput);
}
/**
* Checks if image extraction is disabled.
*
* @return true if image output is off, false otherwise.
*/
public boolean isImageOutputOff() {
return IMAGE_OUTPUT_OFF.equals(imageOutput);
}
/**
* Gets the image output mode.
*
* @return The image output mode (off, embedded, or external).
*/
public String getImageOutput() {
return imageOutput;
}
/**
* Sets the image output mode.
*
* @param imageOutput The image output mode (off, embedded, or external).
* @throws IllegalArgumentException if the mode is not supported.
*/
public void setImageOutput(String imageOutput) {
if (imageOutput != null && !isValidImageOutput(imageOutput)) {
throw new IllegalArgumentException(
String.format("Unsupported image output mode '%s'. Supported values: %s",
imageOutput, getImageOutputOptions(", ")));
}
this.imageOutput = imageOutput != null ? imageOutput.toLowerCase(Locale.ROOT) : IMAGE_OUTPUT_EXTERNAL;
}
/**
* Gets the list of supported image output options.
*
* @param delimiter The delimiter to use between options.
* @return The string with image output modes separated by the delimiter.
*/
public static String getImageOutputOptions(CharSequence delimiter) {
return String.join(delimiter, imageOutputOptions);
}
/**
* Checks if the given image output mode is valid.
*
* @param mode The image output mode to check.
* @return true if the mode is valid, false otherwise.
*/
public static boolean isValidImageOutput(String mode) {
return mode != null && imageOutputOptions.contains(mode.toLowerCase(Locale.ROOT));
}
/**
* Gets the image format for extracted images.
*
* @return The image format (png or jpeg).
*/
public String getImageFormat() {
return imageFormat;
}
/**
* Sets the image format for extracted images.
*
* @param imageFormat The image format (png or jpeg).
* @throws IllegalArgumentException if the format is not supported.
*/
public void setImageFormat(String imageFormat) {
if (imageFormat != null && !isValidImageFormat(imageFormat)) {
throw new IllegalArgumentException(
String.format("Unsupported image format '%s'. Supported values: %s",
imageFormat, getImageFormatOptions(", ")));
}
this.imageFormat = imageFormat != null ? imageFormat.toLowerCase(Locale.ROOT) : IMAGE_FORMAT_PNG;
}
/**
* Gets the list of supported image format options.
*
* @param delimiter The delimiter to use between options.
* @return The string with image formats separated by the delimiter.
*/
public static String getImageFormatOptions(CharSequence delimiter) {
return String.join(delimiter, imageFormatOptions);
}
/**
* Checks if the given image format is valid.
*
* @param format The image format to check.
* @return true if the format is valid, false otherwise.
*/
public static boolean isValidImageFormat(String format) {
return format != null && imageFormatOptions.contains(format.toLowerCase(Locale.ROOT));
}
/**
* Gets the directory for extracted images.
*
* @return The image directory path, or null for default.
*/
public String getImageDir() {
return imageDir;
}
/**
* Sets the directory for extracted images.
* Empty or whitespace-only strings are treated as null (use default).
*
* @param imageDir The directory path for extracted images.
*/
public void setImageDir(String imageDir) {
if (imageDir != null && imageDir.trim().isEmpty()) {
this.imageDir = null;
} else {
this.imageDir = imageDir;
}
}
private static final String INVALID_PAGE_RANGE_FORMAT = "Invalid page range format: '%s'. Expected format: 1,3,5-7";
/** Split limit to preserve trailing empty strings (e.g., "5-" splits to ["5", ""]). */
private static final int SPLIT_KEEP_EMPTY_TRAILING = -1;
/**
* Gets the pages to extract from the PDF.
*
* @return The page specification string (e.g., "1,3,5-7"), or null for all pages.
*/
public String getPages() {
return pages;
}
/**
* Sets the pages to extract from the PDF.
*
* @param pages The page specification (e.g., "1,3,5-7"). Use null or empty for all pages.
* @throws IllegalArgumentException if the format is invalid.
*/
public void setPages(String pages) {
if (pages != null && !pages.trim().isEmpty()) {
this.cachedPageNumbers = parsePageRanges(pages);
} else {
this.cachedPageNumbers = null;
}
this.pages = pages;
}
/**
* Gets the list of page numbers to extract.
*
* @return List of 1-based page numbers, or empty list if all pages should be extracted.
*/
public List<Integer> getPageNumbers() {
if (cachedPageNumbers == null) {
return new ArrayList<>();
}
return new ArrayList<>(cachedPageNumbers);
}
/**
* Parses a page range specification into a list of page numbers.
*
* @param pages The page specification (e.g., "1,3,5-7").
* @return List of 1-based page numbers.
* @throws IllegalArgumentException if the format is invalid.
*/
private static List<Integer> parsePageRanges(String pages) {
List<Integer> result = new ArrayList<>();
String[] parts = pages.split(",");
for (String part : parts) {
String trimmed = part.trim();
if (trimmed.isEmpty()) {
throw new IllegalArgumentException(String.format(INVALID_PAGE_RANGE_FORMAT, pages));
}
if (trimmed.contains("-")) {
parseRange(trimmed, pages, result);
} else {
parseSinglePage(trimmed, pages, result);
}
}
return result;
}
private static void parseRange(String range, String fullInput, List<Integer> result) {
String[] parts = range.split("-", SPLIT_KEEP_EMPTY_TRAILING);
if (parts.length != 2 || parts[0].isEmpty() || parts[1].isEmpty()) {
throw new IllegalArgumentException(String.format(INVALID_PAGE_RANGE_FORMAT, fullInput));
}
try {
int start = Integer.parseInt(parts[0].trim());
int end = Integer.parseInt(parts[1].trim());
if (start < 1 || end < 1) {
throw new IllegalArgumentException(
String.format("Page numbers must be positive: '%s'", fullInput));
}
if (start > end) {
throw new IllegalArgumentException(
String.format("Invalid page range '%s': start page cannot be greater than end page", range));
}
for (int i = start; i <= end; i++) {
result.add(i);
}
} catch (NumberFormatException e) {
throw new IllegalArgumentException(String.format(INVALID_PAGE_RANGE_FORMAT, fullInput));
}
}
private static void parseSinglePage(String page, String fullInput, List<Integer> result) {
try {
int pageNum = Integer.parseInt(page);
if (pageNum < 1) {
throw new IllegalArgumentException(
String.format("Page numbers must be positive: '%s'", fullInput));
}
result.add(pageNum);
} catch (NumberFormatException e) {
throw new IllegalArgumentException(String.format(INVALID_PAGE_RANGE_FORMAT, fullInput));
}
}
/**
* Gets the hybrid backend name.
*
* @return The hybrid backend (off, docling, hancom, azure, google).
*/
public String getHybrid() {
return hybrid;
}
/**
* Sets the hybrid backend.
*
* @param hybrid The hybrid backend (off, docling, hancom, azure, google).
* @throws IllegalArgumentException if the backend is not supported.
*/
public void setHybrid(String hybrid) {
if (hybrid != null && !isValidHybrid(hybrid)) {
throw new IllegalArgumentException(
String.format("Unsupported hybrid backend '%s'. Supported values: %s",
hybrid, getHybridOptions(", ")));
}
this.hybrid = hybrid != null ? hybrid.toLowerCase(Locale.ROOT) : HYBRID_OFF;
}
/**
* Gets the list of supported hybrid backend options.
*
* @param delimiter The delimiter to use between options.
* @return The string with hybrid backends separated by the delimiter.
*/
public static String getHybridOptions(CharSequence delimiter) {
return String.join(delimiter, hybridOptions);
}
/**
* Checks if the given hybrid backend is valid.
*
* @param hybrid The hybrid backend to check.
* @return true if the backend is valid, false otherwise.
*/
public static boolean isValidHybrid(String hybrid) {
return hybrid != null && hybridOptions.contains(hybrid.toLowerCase(Locale.ROOT));
}
/**
* Checks if hybrid processing is enabled.
*
* @return true if hybrid mode is not off, false otherwise.
*/
public boolean isHybridEnabled() {
return !HYBRID_OFF.equals(hybrid);
}
/**
* Gets the hybrid configuration.
*
* @return The HybridConfig instance.
*/
public HybridConfig getHybridConfig() {
return hybridConfig;
}
/**
* Gets the list of supported hybrid mode options.
*
* @param delimiter The delimiter to use between options.
* @return The string with hybrid modes separated by the delimiter.
*/
public static String getHybridModeOptions(CharSequence delimiter) {
return String.join(delimiter, hybridModeOptions);
}
/**
* Checks if the given hybrid mode is valid.
*
* @param mode The hybrid mode to check.
* @return true if the mode is valid, false otherwise.
*/
public static boolean isValidHybridMode(String mode) {
return mode != null && hybridModeOptions.contains(mode.toLowerCase(Locale.ROOT));
}
/**
* Checks if page headers and footers should be included in output.
*
* @return true if headers and footers should be included, false otherwise.
*/
public boolean isIncludeHeaderFooter() {
return includeHeaderFooter;
}
/**
* Enables or disables inclusion of page headers and footers in output.
*
* @param includeHeaderFooter true to include headers and footers, false to exclude.
*/
public void setIncludeHeaderFooter(boolean includeHeaderFooter) {
this.includeHeaderFooter = includeHeaderFooter;
}
public boolean isDetectStrikethrough() {
return detectStrikethrough;
}
public void setDetectStrikethrough(boolean detectStrikethrough) {
this.detectStrikethrough = detectStrikethrough;
}
private boolean outputStdout = false;
public boolean isOutputStdout() {
return outputStdout;
}
public void setOutputStdout(boolean outputStdout) {
this.outputStdout = outputStdout;
}
private int threads = 1;
public int getThreads() {
return threads;
}
public void setThreads(int threads) {
if (threads < 1) {
throw new IllegalArgumentException("threads must be >= 1, got " + threads);
}
this.threads = Math.min(threads, Runtime.getRuntime().availableProcessors());
}
/**
* Returns true if any output format requires structured content
* (reading order, heading levels, list detection, etc.).
* Text-only output does not need these expensive processing steps.
*/
public boolean needsStructuredProcessing() {
return isGenerateMarkdown() || isGenerateHtml() || isGenerateJSON() || isGeneratePDF();
}
/**
* Resolves conflicts between individually valid option values.
* Call once after all setters, before passing the Config to a processor.
* Currently: in hybrid mode, forces {@code threads} to 1 because the hybrid
* pipeline runs sequentially regardless of this value.
*/
public void normalize() {
if (isHybridEnabled() && threads > 1) {
LOGGER.log(Level.WARNING,
"--threads={0} ignored in hybrid mode (forcing threads=1); "
+ "the hybrid pipeline processes pages sequentially",
threads);
threads = 1;
}
}
}
java/opendataloader-pdf-core/src/main/java/org/opendataloader/pdf/api/FilterConfig.java
+205
View File
@@ -0,0 +1,205 @@
/*
* Copyright 2025-2026 Hancom Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.opendataloader.pdf.api;
import org.opendataloader.pdf.utils.SanitizationRule;
import java.util.ArrayList;
import java.util.List;
import java.util.regex.Pattern;
/**
* Configuration class for content filtering options.
* Controls filtering of hidden text, out-of-page content, tiny text, and hidden OCGs.
*/
public class FilterConfig {
private boolean filterHiddenText = false;
private boolean filterOutOfPage = true;
private boolean filterTinyText = true;
private boolean filterHiddenOCG = true;
private boolean filterSensitiveData = false;
private final List<SanitizationRule> filterRules;
/** Default rules */
private void initializeDefaultRules() {
filterRules.add(new SanitizationRule(
Pattern.compile("[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\\.[A-Za-z]{2,}"),
"email@example.com"
));
filterRules.add(new SanitizationRule(
Pattern.compile("[+]\\d+(?:-\\d+)+"),
"+00-0000-0000"
));
filterRules.add(new SanitizationRule(
Pattern.compile("[A-Z]{1,2}\\d{6,9}"),
"AA0000000"
));
filterRules.add(new SanitizationRule(
Pattern.compile("\\b\\d{4}-?\\d{4}-?\\d{4}-?\\d{4}\\b"),
"0000-0000-0000-0000"
));
filterRules.add(new SanitizationRule(
Pattern.compile("\\b\\d{10,18}\\b"),
"0000000000000000"
));
filterRules.add(new SanitizationRule(
Pattern.compile("\\b(?:\\d{1,3}\\.){3}\\d{1,3}\\b"),
"0.0.0.0"
));
filterRules.add(new SanitizationRule(
Pattern.compile("\\b([0-9a-fA-F]{0,4}:){2,7}[0-9a-fA-F]{0,4}\\b"),
"0.0.0.0::1"
));
filterRules.add(new SanitizationRule(
Pattern.compile("\\b(?:[0-9A-Fa-f]{2}:){5}[0-9A-Fa-f]{2}\\b"),
"00:00:00:00:00:00"
));
filterRules.add(new SanitizationRule(
Pattern.compile("\\b\\d{15}\\b"),
"000000000000000"
));
filterRules.add(new SanitizationRule(
Pattern.compile("https?://[A-Za-z0-9.-]+(:\\d+)?(/\\S*)?"),
"https://example.com"
));
}
/**
* Constructor initializing the configuration of filter.
*/
public FilterConfig() {
this.filterRules = new ArrayList<>();
initializeDefaultRules();
}
/**
* Enables or disables filter of hidden text.
*
* @param filterHiddenText true to enable filter, false to disable.
*/
public void setFilterHiddenText(boolean filterHiddenText) {
this.filterHiddenText = filterHiddenText;
}
/**
* Checks if the processor should attempt to find and extract hidden text.
*
* @return true if hidden text is filtered, false otherwise.
*/
public boolean isFilterHiddenText() {
return filterHiddenText;
}
/**
* Enables or disables checking content that exceeds MediaBox or CropBox.
*
* @param filterOutOfPage true to enable, false to disable.
*/
public void setFilterOutOfPage(boolean filterOutOfPage) {
this.filterOutOfPage = filterOutOfPage;
}
/**
* Checks if the processor should filter out of page content.
*
* @return true if filter is enabled, false otherwise.
*/
public boolean isFilterOutOfPage() {
return filterOutOfPage;
}
/**
* Checks if the processor should filter out tiny text.
*
* @return true if filter is enabled, false otherwise.
*/
public boolean isFilterTinyText() {
return filterTinyText;
}
/**
* Enables or disables filter of tiny text.
*
* @param filterTinyText true to enable filter, false to disable.
*/
public void setFilterTinyText(boolean filterTinyText) {
this.filterTinyText = filterTinyText;
}
/**
* Checks if the processor should filter out hidden OCGs.
*
* @return true if filter is enabled, false otherwise.
*/
public boolean isFilterHiddenOCG() {
return filterHiddenOCG;
}
/**
* Enables or disables filter of hidden OCGs.
*
* @param filterHiddenOCG true to enable filter, false to disable.
*/
public void setFilterHiddenOCG(boolean filterHiddenOCG) {
this.filterHiddenOCG = filterHiddenOCG;
}
/**
* Checks if the processor should filter out sensitive data.
*
* @return true if filter is enabled, false otherwise.
*/
public boolean isFilterSensitiveData() {
return filterSensitiveData;
}
/**
* Enables or disables filter of sensitive data.
*
* @param filterSensitiveData true to enable filter, false to disable.
*/
public void setFilterSensitiveData(boolean filterSensitiveData) {
this.filterSensitiveData = filterSensitiveData;
}
/**
* Gets custom filter sanitization rules.
*
* @return List of sanitization rules.
*/
public List<SanitizationRule> getFilterRules() {
return filterRules;
}
/**
* Add custom filter sanitization rule.
*
* @param pattern pattern string.
* @param replacement pattern replacement string.
*/
public void addFilterRule(String pattern, String replacement) {
filterRules.add(new SanitizationRule(Pattern.compile(pattern), replacement));
}
/**
* Remove filter sanitization rule.
*
* @param pattern pattern string.
*/
public void removeFilterRule(String pattern) {
filterRules.removeIf(rule -> rule.getPattern().pattern().equals(pattern));
}
}
java/opendataloader-pdf-core/src/main/java/org/opendataloader/pdf/api/OpenDataLoaderPDF.java
+52
View File
@@ -0,0 +1,52 @@
/*
* Copyright 2025-2026 Hancom Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.opendataloader.pdf.api;
import org.opendataloader.pdf.hybrid.HybridClientFactory;
import org.opendataloader.pdf.processors.DocumentProcessor;
import java.io.IOException;
/**
* The main entry point for the opendataloader-pdf library.
* Use the static method {@link #processFile(String, Config)} to process a PDF.
*/
public final class OpenDataLoaderPDF {
private OpenDataLoaderPDF() {
}
/**
* Processes a PDF file to extract its content and structure based on the provided configuration.
*
* @param inputPdfName The path to the input PDF file.
* @param config The configuration object specifying output formats and other options.
* @throws IOException If an error occurs during file reading or processing.
*/
public static void processFile(String inputPdfName, Config config) throws IOException {
DocumentProcessor.processFile(inputPdfName, config);
}
/**
* Shuts down any cached resources used by the library.
*
* <p>This method should be called when processing is complete, typically at CLI exit.
* It releases resources such as HTTP client thread pools used for hybrid mode backends.
*/
public static void shutdown() {
HybridClientFactory.shutdown();
}
}
java/opendataloader-pdf-core/src/main/java/org/opendataloader/pdf/api/OutputWriter.java
+83
View File
@@ -0,0 +1,83 @@
/*
* Copyright 2025-2026 Hancom Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.opendataloader.pdf.api;
import org.opendataloader.pdf.processors.DocumentProcessor;
import org.opendataloader.pdf.processors.ExtractionResult;
import java.io.IOException;
/**
* Writes configured output files (JSON, Markdown, HTML, PDF, text, images,
* tagged PDF) from a pre-computed {@link ExtractionResult}.
*
* <p>Use this when you have already run extraction once (e.g. via
* {@link AutoTagger#tag(ExtractionResult, Float)}) and want to emit file
* outputs from that same result without re-extracting.
*
* <p>Typical two-phase usage:
* <pre>{@code
* Config config = new Config();
* config.setOutputFolder("/out");
* config.setGenerateJSON(true);
* config.setGenerateMarkdown(true);
*
* // Phase 1: extract once
* ExtractionResult extraction =
* org.opendataloader.pdf.processors.DocumentProcessor.extractContents(
* "input.pdf", config);
*
* // Phase 2a: write output files
* OutputWriter.writeOutputs("input.pdf", extraction, config);
*
* // Phase 2b: tag in-memory and reuse the same extraction
* try (TaggingResult tagged = AutoTagger.tag("input.pdf", extraction)) {
* // ... use tagged.getDocument()
* }
* }</pre>
*
* <p>For the single-call extraction-and-output pipeline, use
* {@link OpenDataLoaderPDF#processFile} instead.
*/
public final class OutputWriter {
private OutputWriter() {
}
/**
* Writes the output files configured on {@code config} (e.g.
* {@code generateJSON}, {@code generateMarkdown}, {@code generateHtml},
* {@code generatePDF}, {@code generateTaggedPDF}, {@code generateText})
* using the supplied pre-computed extraction.
*
* <p>This method does <em>not</em> re-run extraction. Output behaviour is
* identical to {@link OpenDataLoaderPDF#processFile} for the same
* {@link Config}, including stdout mode, image directory resolution, and
* tagged-PDF generation.
*
* @param inputPdfName path to the input PDF file (used for filename derivation
* and tagged-PDF / annotated-PDF re-saves; not re-parsed)
* @param extraction pre-computed extraction result (from
* {@code DocumentProcessor.extractContents})
* @param config configuration controlling which output formats to emit
* @throws IOException if writing any output file fails
*/
public static void writeOutputs(String inputPdfName, ExtractionResult extraction, Config config)
throws IOException {
DocumentProcessor.generateOutputs(inputPdfName, extraction.getContents(), config,
extraction.getElementMetadata());
}
}
java/opendataloader-pdf-core/src/main/java/org/opendataloader/pdf/api/TaggingResult.java
+83
View File
@@ -0,0 +1,83 @@
package org.opendataloader.pdf.api;
import com.fasterxml.jackson.databind.JsonNode;
import org.opendataloader.pdf.hybrid.ElementMetadata;
import org.verapdf.pd.PDDocument;
import java.io.IOException;
import java.util.Collections;
import java.util.Map;
/**
* Result of {@link AutoTagger#tag}. Contains the tagged PDF document in-memory
* and processing timing metadata.
*
* <p>Implements {@link AutoCloseable} for use with try-with-resources.
* The caller is responsible for closing this result, which releases the
* underlying PDDocument resources.
*/
public class TaggingResult implements AutoCloseable {
private final PDDocument document;
private final long extractionNs;
private final long taggingNs;
private final JsonNode hybridTimings;
private final Map<Long, ElementMetadata> elementMetadata;
public TaggingResult(PDDocument document, long extractionNs, long taggingNs,
JsonNode hybridTimings, Map<Long, ElementMetadata> elementMetadata) {
if (document == null) {
throw new IllegalArgumentException("document must not be null");
}
this.document = document;
this.extractionNs = extractionNs;
this.taggingNs = taggingNs;
this.hybridTimings = hybridTimings;
this.elementMetadata = elementMetadata != null ? elementMetadata : Collections.emptyMap();
}
public TaggingResult(PDDocument document, long extractionNs, long taggingNs, JsonNode hybridTimings) {
this(document, extractionNs, taggingNs, hybridTimings, Collections.emptyMap());
}
/** The tagged PDF document. Do not close this directly — close the TaggingResult instead. */
public PDDocument getDocument() {
return document;
}
/** Time spent on extraction (parsing + layout + content extraction) in nanoseconds. */
public long getExtractionNs() {
return extractionNs;
}
/** Time spent on auto-tagging (structure tree creation) in nanoseconds. */
public long getTaggingNs() {
return taggingNs;
}
/** Per-step hybrid server timings, or null if hybrid mode was not used. */
public JsonNode getHybridTimings() {
return hybridTimings;
}
/** Element metadata from hybrid backend, or empty map if not available. */
public Map<Long, ElementMetadata> getElementMetadata() {
return elementMetadata;
}
/**
* Save the tagged PDF to a file.
*
* @param outputPath the output file path
*/
public void saveTo(String outputPath) throws IOException {
document.saveAs(outputPath);
}
@Override
public void close() {
if (document != null) {
document.close();
}
}
}
java/opendataloader-pdf-core/src/main/java/org/opendataloader/pdf/api/cli/CLIOptions.java
+819
View File
@@ -0,0 +1,819 @@
/*
* Copyright 2025-2026 Hancom Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.opendataloader.pdf.api.cli;
import org.apache.commons.cli.CommandLine;
import org.apache.commons.cli.Option;
import org.apache.commons.cli.Options;
import org.opendataloader.pdf.api.Config;
import org.opendataloader.pdf.hybrid.HybridConfig;
import java.io.File;
import java.io.PrintStream;
import java.util.Arrays;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Locale;
import java.util.Set;
import java.util.stream.Collectors;
/**
* Adapter that maps Apache Commons CLI options to {@link Config} / {@link HybridConfig}.
*
* <p><b>Stable API for downstream tools</b> (e.g. opendataloader-pdfua) — these four
* members are the supported integration surface and will not break compatibly:
* <ul>
* <li>{@link #defineOptions()} — get a fully populated {@code Options} instance</li>
* <li>{@link #addAllTo(Options)} — add core options into an externally-built {@code Options}</li>
* <li>{@link #applyAllTo(Config, CommandLine)} — populate a {@code Config} from a parsed line</li>
* <li>{@link #FOLDER_OPTION} — short option name for {@code --output-dir}</li>
* </ul>
*
* <p><b>Everything else is internal.</b> The numerous other public {@code static} members
* (option-name constants, helpers like {@code createConfigFromCommandLine},
* {@code exportOptionsAsJson}) exist for the CLI module ({@code CLIMain}) and the
* options-export tooling that drives Node/Python binding generation. Their visibility
* is {@code public} only for cross-package access within this codebase; they are
* <i>not</i> part of the supported API and may be renamed, moved, or removed in any
* release. Downstream consumers depending on them do so at their own risk.
*
* <p>Pdfua's usage pattern (build your own {@code Options}, add core's, parse, then
* populate {@code Config}):
* <pre>{@code
* Options options = new Options();
* options.addOption(...); // your tool-specific options
* CLIOptions.addAllTo(options); // add core's options
* CommandLine cmd = parser.parse(options, args);
* Config core = new Config();
* CLIOptions.applyAllTo(core, cmd);
* }</pre>
*/
public class CLIOptions {
// ===== Output Directory =====
public static final String FOLDER_OPTION = "o";
private static final String FOLDER_LONG_OPTION = "output-dir";
private static final String FOLDER_DESC = "Directory where output files are written. Default: input file directory";
// ===== Password =====
public static final String PASSWORD_OPTION = "p";
private static final String PASSWORD_LONG_OPTION = "password";
private static final String PASSWORD_DESC = "Password for encrypted PDF files";
// ===== Format =====
public static final String FORMAT_OPTION = "f";
public static final String FORMAT_LONG_OPTION = "format";
private static final String FORMAT_DESC = "Output formats (comma-separated). "
+ "Values: json, text, html, pdf, markdown, tagged-pdf. Default: json. "
+ "For HTML inside Markdown use --markdown-with-html. "
+ "For image extraction control use --image-output.";
// ===== Quiet =====
public static final String QUIET_OPTION = "q";
private static final String QUIET_LONG_OPTION = "quiet";
private static final String QUIET_DESC = "Suppress console logging output";
// ===== Content Safety =====
private static final String CONTENT_SAFETY_OFF_LONG_OPTION = "content-safety-off";
private static final String CONTENT_SAFETY_OFF_DESC = "Disable content safety filters. "
+ "Values: all, hidden-text, off-page, tiny, hidden-ocg";
// ===== Sanitize =====
private static final String SANITIZE_LONG_OPTION = "sanitize";
private static final String SANITIZE_DESC = "Enable sensitive data sanitization. "
+ "Replaces emails, phone numbers, IPs, credit cards, and URLs with placeholders";
// ===== Keep Line Breaks =====
private static final String KEEP_LINE_BREAKS_LONG_OPTION = "keep-line-breaks";
private static final String KEEP_LINE_BREAKS_DESC = "Preserve original line breaks in extracted text";
// ===== Replace Invalid Chars =====
private static final String REPLACE_INVALID_CHARS_LONG_OPTION = "replace-invalid-chars";
private static final String REPLACE_INVALID_CHARS_DESC = "Replacement character for invalid/unrecognized characters. Default: space";
// ===== Use Struct Tree =====
private static final String USE_STRUCT_TREE_LONG_OPTION = "use-struct-tree";
private static final String USE_STRUCT_TREE_DESC = "Use PDF structure tree (tagged PDF) for reading order and semantic structure. Output quality depends on tag quality";
// ===== Table Method =====
private static final String TABLE_METHOD_LONG_OPTION = "table-method";
private static final String TABLE_METHOD_DESC = "Table detection method. Values: default (border-based), cluster (border + cluster). Default: default";
// ===== Reading Order =====
private static final String READING_ORDER_LONG_OPTION = "reading-order";
private static final String READING_ORDER_DESC = "Reading order algorithm. Values: off, xycut. Default: xycut";
// ===== Page Separators =====
private static final String MARKDOWN_PAGE_SEPARATOR_LONG_OPTION = "markdown-page-separator";
private static final String MARKDOWN_PAGE_SEPARATOR_DESC = "Separator between pages in Markdown output. Use %page-number% for page numbers. Default: none";
private static final String TEXT_PAGE_SEPARATOR_LONG_OPTION = "text-page-separator";
private static final String TEXT_PAGE_SEPARATOR_DESC = "Separator between pages in text output. Use %page-number% for page numbers. Default: none";
private static final String HTML_PAGE_SEPARATOR_LONG_OPTION = "html-page-separator";
private static final String HTML_PAGE_SEPARATOR_DESC = "Separator between pages in HTML output. Use %page-number% for page numbers. Default: none";
// ===== Image Options =====
private static final String IMAGE_OUTPUT_LONG_OPTION = "image-output";
private static final String IMAGE_OUTPUT_DESC = "Image output mode. Values: off (no images), embedded (Base64 data URIs), external (file references). Default: external";
private static final String IMAGE_FORMAT_LONG_OPTION = "image-format";
private static final String IMAGE_FORMAT_DESC = "Output format for extracted images. Values: png, jpeg. Default: png";
private static final String IMAGE_DIR_LONG_OPTION = "image-dir";
private static final String IMAGE_DIR_DESC = "Directory for extracted images (applies only with --image-output external)";
// ===== Pages =====
private static final String PAGES_LONG_OPTION = "pages";
private static final String PAGES_DESC = "Pages to extract (e.g., \"1,3,5-7\"). Default: all pages";
// ===== Include Header Footer =====
private static final String INCLUDE_HEADER_FOOTER_LONG_OPTION = "include-header-footer";
private static final String INCLUDE_HEADER_FOOTER_DESC = "Include page headers and footers in output";
// ===== Detect Strikethrough =====
private static final String DETECT_STRIKETHROUGH_LONG_OPTION = "detect-strikethrough";
private static final String DETECT_STRIKETHROUGH_DESC = "Detect strikethrough text and wrap with ~~ in Markdown output or <del></del> tag in HTML output (experimental)";
// ===== Hybrid Mode =====
private static final String HYBRID_LONG_OPTION = "hybrid";
private static final String HYBRID_DESC = "Hybrid backend (requires a running server). "
+ "Quick start: pip install \"opendataloader-pdf[hybrid]\" && opendataloader-pdf-hybrid --port 5002. "
+ "For remote servers use --hybrid-url. Values: off (default), docling-fast, hancom-ai";
private static final String HYBRID_MODE_LONG_OPTION = "hybrid-mode";
private static final String HYBRID_MODE_DESC = "Hybrid triage mode. Values: auto (default, dynamic triage), full (skip triage, all pages to backend)";
// Deprecated: OCR settings are now configured on the hybrid server
private static final String HYBRID_OCR_LONG_OPTION = "hybrid-ocr";
private static final String HYBRID_OCR_DESC = "[Deprecated] OCR settings are now configured on the hybrid server (--ocr-lang, --force-ocr)";
private static final String HYBRID_URL_LONG_OPTION = "hybrid-url";
private static final String HYBRID_URL_DESC = "Hybrid backend server URL (overrides default)";
private static final String HYBRID_TIMEOUT_LONG_OPTION = "hybrid-timeout";
private static final String HYBRID_TIMEOUT_DESC = "Hybrid backend request timeout in milliseconds (0 = no timeout). Default: 0";
private static final String HYBRID_FALLBACK_LONG_OPTION = "hybrid-fallback";
private static final String HYBRID_FALLBACK_DESC = "Opt in to Java fallback on hybrid backend error (default: disabled)";
// ===== Hybrid hancom-ai backend-specific =====
private static final String HYBRID_HANCOM_AI_REGIONLIST_STRATEGY_LONG_OPTION =
"hybrid-hancom-ai-regionlist-strategy";
private static final String HYBRID_HANCOM_AI_REGIONLIST_STRATEGY_DESC =
"DLA label 7 (regionlist) handling. Requires --hybrid=hancom-ai. "
+ "Values: table-first (default; check TSR overlap), list-only (skip TSR, always treat as list)";
private static final String HYBRID_HANCOM_AI_OCR_STRATEGY_LONG_OPTION =
"hybrid-hancom-ai-ocr-strategy";
private static final String HYBRID_HANCOM_AI_OCR_STRATEGY_DESC =
"OCR strategy. Requires --hybrid=hancom-ai. "
+ "Values: off (stream-only), auto (default; stream first, OCR fallback), force (OCR-only)";
private static final String HYBRID_HANCOM_AI_IMAGE_CACHE_LONG_OPTION =
"hybrid-hancom-ai-image-cache";
private static final String HYBRID_HANCOM_AI_IMAGE_CACHE_DESC =
"Page image cache backing. Requires --hybrid=hancom-ai. "
+ "Values: memory (default), disk";
private static final String HYBRID_HANCOM_AI_SAVE_CROPS_LONG_OPTION =
"hybrid-hancom-ai-save-crops";
private static final String HYBRID_HANCOM_AI_SAVE_CROPS_DESC =
"Persist cropped figure images to disk for debugging. Requires --hybrid=hancom-ai";
private static final String HYBRID_HANCOM_AI_CROP_OUTPUT_DIR_LONG_OPTION =
"hybrid-hancom-ai-crop-output-dir";
private static final String HYBRID_HANCOM_AI_CROP_OUTPUT_DIR_DESC =
"Output directory for --hybrid-hancom-ai-save-crops. Requires --hybrid=hancom-ai";
// ===== Stdout Output =====
private static final String TO_STDOUT_LONG_OPTION = "to-stdout";
private static final String TO_STDOUT_DESC = "Write output to stdout instead of file (single format only)";
// ===== Threads =====
private static final String THREADS_LONG_OPTION = "threads";
private static final String THREADS_DESC = "Number of worker threads for per-page processing. "
+ "Default: 1 (sequential, stable). Values >1 (experimental) run pages in parallel for faster throughput; "
+ "output may vary slightly on some PDFs. Capped at the number of available CPU cores. "
+ "Applies to the native Java pipeline only; ignored in --hybrid mode";
// ===== Markdown modifiers =====
public static final String HTML_IN_MARKDOWN_LONG_OPTION = "markdown-with-html";
private static final String HTML_IN_MARKDOWN_DESC =
"Allow HTML tags inside Markdown output for complex structures such as multi-row-span tables. "
+ "Implies --format markdown.";
// ===== Export Options (internal) =====
public static final String EXPORT_OPTIONS_LONG_OPTION = "export-options";
// ===== Legacy Options (hidden, backward compatibility) =====
public static final String PDF_REPORT_LONG_OPTION = "pdf";
public static final String MARKDOWN_REPORT_LONG_OPTION = "markdown";
public static final String HTML_REPORT_LONG_OPTION = "html";
private static final String MARKDOWN_IMAGE_LONG_OPTION = "markdown-with-images";
public static final String NO_JSON_REPORT_LONG_OPTION = "no-json";
/**
* Single source of truth for all CLI option definitions.
* Add new options here - they will automatically be available in both CLI and
* JSON export.
*/
private static final List<OptionDefinition> OPTION_DEFINITIONS = Arrays.asList(
// Primary options (exported to JSON)
new OptionDefinition(FOLDER_LONG_OPTION, FOLDER_OPTION, "string", null, FOLDER_DESC, true),
new OptionDefinition(PASSWORD_LONG_OPTION, PASSWORD_OPTION, "string", null, PASSWORD_DESC, true),
new OptionDefinition(FORMAT_LONG_OPTION, FORMAT_OPTION, "string", null, FORMAT_DESC, true),
new OptionDefinition(QUIET_LONG_OPTION, QUIET_OPTION, "boolean", false, QUIET_DESC, true),
new OptionDefinition(CONTENT_SAFETY_OFF_LONG_OPTION, null, "string", null, CONTENT_SAFETY_OFF_DESC, true),
new OptionDefinition(SANITIZE_LONG_OPTION, null, "boolean", false, SANITIZE_DESC, true),
new OptionDefinition(KEEP_LINE_BREAKS_LONG_OPTION, null, "boolean", false, KEEP_LINE_BREAKS_DESC, true),
new OptionDefinition(REPLACE_INVALID_CHARS_LONG_OPTION, null, "string", " ", REPLACE_INVALID_CHARS_DESC,
true),
new OptionDefinition(USE_STRUCT_TREE_LONG_OPTION, null, "boolean", false, USE_STRUCT_TREE_DESC, true),
new OptionDefinition(TABLE_METHOD_LONG_OPTION, null, "string", "default", TABLE_METHOD_DESC, true),
new OptionDefinition(READING_ORDER_LONG_OPTION, null, "string", "xycut", READING_ORDER_DESC, true),
new OptionDefinition(MARKDOWN_PAGE_SEPARATOR_LONG_OPTION, null, "string", null,
MARKDOWN_PAGE_SEPARATOR_DESC, true),
new OptionDefinition(HTML_IN_MARKDOWN_LONG_OPTION, null, "boolean", false,
HTML_IN_MARKDOWN_DESC, true),
new OptionDefinition(TEXT_PAGE_SEPARATOR_LONG_OPTION, null, "string", null, TEXT_PAGE_SEPARATOR_DESC, true),
new OptionDefinition(HTML_PAGE_SEPARATOR_LONG_OPTION, null, "string", null, HTML_PAGE_SEPARATOR_DESC, true),
new OptionDefinition(IMAGE_OUTPUT_LONG_OPTION, null, "string", "external", IMAGE_OUTPUT_DESC, true),
new OptionDefinition(IMAGE_FORMAT_LONG_OPTION, null, "string", "png", IMAGE_FORMAT_DESC, true),
new OptionDefinition(IMAGE_DIR_LONG_OPTION, null, "string", null, IMAGE_DIR_DESC, true),
new OptionDefinition(PAGES_LONG_OPTION, null, "string", null, PAGES_DESC, true),
new OptionDefinition(INCLUDE_HEADER_FOOTER_LONG_OPTION, null, "boolean", false,
INCLUDE_HEADER_FOOTER_DESC, true),
new OptionDefinition(DETECT_STRIKETHROUGH_LONG_OPTION, null, "boolean", false,
DETECT_STRIKETHROUGH_DESC, true),
new OptionDefinition(HYBRID_LONG_OPTION, null, "string", "off", HYBRID_DESC, true),
new OptionDefinition(HYBRID_MODE_LONG_OPTION, null, "string", "auto", HYBRID_MODE_DESC, true),
new OptionDefinition(HYBRID_URL_LONG_OPTION, null, "string", null, HYBRID_URL_DESC, true),
new OptionDefinition(HYBRID_TIMEOUT_LONG_OPTION, null, "string", "0", HYBRID_TIMEOUT_DESC, true),
new OptionDefinition(HYBRID_FALLBACK_LONG_OPTION, null, "boolean", false, HYBRID_FALLBACK_DESC, true),
new OptionDefinition(HYBRID_HANCOM_AI_REGIONLIST_STRATEGY_LONG_OPTION, null, "string",
"table-first", HYBRID_HANCOM_AI_REGIONLIST_STRATEGY_DESC, true),
new OptionDefinition(HYBRID_HANCOM_AI_OCR_STRATEGY_LONG_OPTION, null, "string",
"auto", HYBRID_HANCOM_AI_OCR_STRATEGY_DESC, true),
new OptionDefinition(HYBRID_HANCOM_AI_IMAGE_CACHE_LONG_OPTION, null, "string",
"memory", HYBRID_HANCOM_AI_IMAGE_CACHE_DESC, true),
new OptionDefinition(TO_STDOUT_LONG_OPTION, null, "boolean", false, TO_STDOUT_DESC, true),
new OptionDefinition(THREADS_LONG_OPTION, null, "string", "1", THREADS_DESC, true),
new OptionDefinition(EXPORT_OPTIONS_LONG_OPTION, null, "boolean", null, null, false),
// Legacy options (not exported, for backward compatibility)
new OptionDefinition(HYBRID_OCR_LONG_OPTION, null, "string", null, HYBRID_OCR_DESC, false),
new OptionDefinition(PDF_REPORT_LONG_OPTION, null, "boolean", null, null, false),
new OptionDefinition(MARKDOWN_REPORT_LONG_OPTION, null, "boolean", null, null, false),
new OptionDefinition(HTML_REPORT_LONG_OPTION, null, "boolean", null, null, false),
new OptionDefinition(MARKDOWN_IMAGE_LONG_OPTION, null, "boolean", null, null, false),
new OptionDefinition(NO_JSON_REPORT_LONG_OPTION, null, "boolean", null, null, false),
new OptionDefinition(HYBRID_HANCOM_AI_SAVE_CROPS_LONG_OPTION, null, "boolean",
false, HYBRID_HANCOM_AI_SAVE_CROPS_DESC, false),
new OptionDefinition(HYBRID_HANCOM_AI_CROP_OUTPUT_DIR_LONG_OPTION, null, "string",
null, HYBRID_HANCOM_AI_CROP_OUTPUT_DIR_DESC, false));
public static Options defineOptions() {
Options options = new Options();
addAllTo(options);
return options;
}
/**
* Registers every core CLI option onto an external {@link Options} instance.
* Used by downstream CLIs (e.g. opendataloader-pdfua) that want to inherit
* the entire core option set and add their own options on top.
*
* @param options the Options instance to populate
*/
public static void addAllTo(Options options) {
for (OptionDefinition def : OPTION_DEFINITIONS) {
options.addOption(def.toOption());
}
}
public static Config createConfigFromCommandLine(CommandLine commandLine) {
Config config = new Config();
if (commandLine.hasOption(CLIOptions.FOLDER_OPTION)) {
config.setOutputFolder(commandLine.getOptionValue(CLIOptions.FOLDER_OPTION));
} else {
String argument = commandLine.getArgs()[0];
File file = new File(argument);
file = new File(file.getAbsolutePath());
config.setOutputFolder(file.isDirectory() ? file.getAbsolutePath() : file.getParent());
}
applyAllTo(config, commandLine);
return config;
}
/**
* Applies every core CLI option from the parsed command line onto the given Config.
* Caller is responsible for setting required Config state that is not represented
* by a CLI option (e.g. output folder when no positional input file is provided).
*
* Used by downstream CLIs that build their own Options + Config and want core
* options applied without paying for the positional-arg-based output-folder
* fallback that {@link #createConfigFromCommandLine} performs.
*
* @param config Config to populate
* @param commandLine parsed CommandLine
*/
public static void applyAllTo(Config config, CommandLine commandLine) {
if (commandLine.hasOption(CLIOptions.PASSWORD_OPTION)) {
config.setPassword(commandLine.getOptionValue(CLIOptions.PASSWORD_OPTION));
}
if (commandLine.hasOption(CLIOptions.KEEP_LINE_BREAKS_LONG_OPTION)) {
config.setKeepLineBreaks(true);
}
if (commandLine.hasOption(CLIOptions.PDF_REPORT_LONG_OPTION)) {
config.setGeneratePDF(true);
}
if (commandLine.hasOption(CLIOptions.MARKDOWN_REPORT_LONG_OPTION)) {
config.setGenerateMarkdown(true);
}
if (commandLine.hasOption(CLIOptions.HTML_REPORT_LONG_OPTION)) {
config.setGenerateHtml(true);
}
if (commandLine.hasOption(CLIOptions.HTML_IN_MARKDOWN_LONG_OPTION)) {
config.setUseHTMLInMarkdown(true);
}
if (commandLine.hasOption(CLIOptions.MARKDOWN_IMAGE_LONG_OPTION)) {
config.setGenerateMarkdown(true);
}
if (commandLine.hasOption(CLIOptions.NO_JSON_REPORT_LONG_OPTION)) {
config.setGenerateJSON(false);
}
if (commandLine.hasOption(CLIOptions.REPLACE_INVALID_CHARS_LONG_OPTION)) {
config.setReplaceInvalidChars(commandLine.getOptionValue(CLIOptions.REPLACE_INVALID_CHARS_LONG_OPTION));
}
if (commandLine.hasOption(CLIOptions.USE_STRUCT_TREE_LONG_OPTION)) {
config.setUseStructTree(true);
}
if (commandLine.hasOption(INCLUDE_HEADER_FOOTER_LONG_OPTION)) {
config.setIncludeHeaderFooter(true);
}
if (commandLine.hasOption(DETECT_STRIKETHROUGH_LONG_OPTION)) {
config.setDetectStrikethrough(true);
}
if (commandLine.hasOption(CLIOptions.READING_ORDER_LONG_OPTION)) {
config.setReadingOrder(commandLine.getOptionValue(CLIOptions.READING_ORDER_LONG_OPTION));
}
if (commandLine.hasOption(CLIOptions.MARKDOWN_PAGE_SEPARATOR_LONG_OPTION)) {
config.setMarkdownPageSeparator(commandLine.getOptionValue(CLIOptions.MARKDOWN_PAGE_SEPARATOR_LONG_OPTION));
}
if (commandLine.hasOption(CLIOptions.TEXT_PAGE_SEPARATOR_LONG_OPTION)) {
config.setTextPageSeparator(commandLine.getOptionValue(CLIOptions.TEXT_PAGE_SEPARATOR_LONG_OPTION));
}
if (commandLine.hasOption(CLIOptions.HTML_PAGE_SEPARATOR_LONG_OPTION)) {
config.setHtmlPageSeparator(commandLine.getOptionValue(CLIOptions.HTML_PAGE_SEPARATOR_LONG_OPTION));
}
applyContentSafetyOption(config, commandLine);
applySanitizeOption(config, commandLine);
applyFormatOption(config, commandLine);
applyTableMethodOption(config, commandLine);
applyImageOptions(config, commandLine);
applyPagesOption(config, commandLine);
applyHybridOptions(config, commandLine);
applyThreadsOption(config, commandLine);
config.normalize();
}
private static void applyThreadsOption(Config config, CommandLine commandLine) {
if (!commandLine.hasOption(THREADS_LONG_OPTION)) {
return;
}
String value = commandLine.getOptionValue(THREADS_LONG_OPTION);
int requested;
try {
requested = Integer.parseInt(value.trim());
} catch (NumberFormatException e) {
throw new IllegalArgumentException(
String.format("Option --threads requires an integer >= 1, got '%s'", value));
}
if (requested < 1) {
throw new IllegalArgumentException(
String.format("Option --threads requires an integer >= 1, got %d", requested));
}
config.setThreads(requested);
int applied = config.getThreads();
if (applied < requested) {
System.err.println(String.format(
"Warning: --threads=%d exceeds available CPU cores; capped to %d.",
requested, applied));
}
}
private static void applyImageOptions(Config config, CommandLine commandLine) {
if (commandLine.hasOption(IMAGE_OUTPUT_LONG_OPTION)) {
String outputValue = commandLine.getOptionValue(IMAGE_OUTPUT_LONG_OPTION);
if (outputValue == null || outputValue.trim().isEmpty()) {
throw new IllegalArgumentException(
String.format("Option --image-output requires a value. Supported values: %s",
Config.getImageOutputOptions(", ")));
}
String output = outputValue.trim().toLowerCase(Locale.ROOT);
if (!Config.isValidImageOutput(output)) {
throw new IllegalArgumentException(
String.format("Unsupported image output mode '%s'. Supported values: %s",
output, Config.getImageOutputOptions(", ")));
}
config.setImageOutput(output);
}
if (commandLine.hasOption(IMAGE_FORMAT_LONG_OPTION)) {
String formatValue = commandLine.getOptionValue(IMAGE_FORMAT_LONG_OPTION);
if (formatValue == null || formatValue.trim().isEmpty()) {
throw new IllegalArgumentException(
"Option --image-format requires a value. Supported values: png, jpeg");
}
String format = formatValue.trim().toLowerCase(Locale.ROOT);
if (!Config.isValidImageFormat(format)) {
throw new IllegalArgumentException(
String.format("Unsupported image format '%s'. Supported values: png, jpeg", format));
}
config.setImageFormat(format);
}
if (commandLine.hasOption(IMAGE_DIR_LONG_OPTION)) {
config.setImageDir(commandLine.getOptionValue(IMAGE_DIR_LONG_OPTION));
}
}
private static void applyPagesOption(Config config, CommandLine commandLine) {
if (commandLine.hasOption(PAGES_LONG_OPTION)) {
config.setPages(commandLine.getOptionValue(PAGES_LONG_OPTION));
}
}
private static void applyTableMethodOption(Config config, CommandLine commandLine) {
if (commandLine.hasOption(TABLE_METHOD_LONG_OPTION)) {
String methodValue = commandLine.getOptionValue(TABLE_METHOD_LONG_OPTION);
if (methodValue == null || methodValue.trim().isEmpty()) {
throw new IllegalArgumentException(
String.format("Option --table-method requires a value. Supported values: %s",
Config.getTableMethodOptions(", ")));
}
String method = methodValue.trim().toLowerCase(Locale.ROOT);
if (!Config.isValidTableMethod(method)) {
throw new IllegalArgumentException(
String.format("Unsupported table method '%s'. Supported values: %s",
method, Config.getTableMethodOptions(", ")));
}
config.setTableMethod(method);
}
}
private static void applyContentSafetyOption(Config config, CommandLine commandLine) {
if (!commandLine.hasOption(CONTENT_SAFETY_OFF_LONG_OPTION)) {
return;
}
String[] optionValues = commandLine.getOptionValues(CONTENT_SAFETY_OFF_LONG_OPTION);
if (optionValues == null || optionValues.length == 0) {
throw new IllegalArgumentException(
"Option --content-safety-off requires at least one value. Supported values: all, hidden-text, off-page, tiny, hidden-ocg");
}
Set<String> values = parseOptionValues(optionValues);
if (values.isEmpty()) {
throw new IllegalArgumentException(
"Option --content-safety-off requires at least one value. Supported values: all, hidden-text, off-page, tiny, hidden-ocg");
}
for (String value : values) {
switch (value) {
case "hidden-text":
config.getFilterConfig().setFilterHiddenText(false);
break;
case "off-page":
config.getFilterConfig().setFilterOutOfPage(false);
break;
case "tiny":
config.getFilterConfig().setFilterTinyText(false);
break;
case "hidden-ocg":
config.getFilterConfig().setFilterHiddenOCG(false);
break;
case "sensitive-data":
System.err.println("Warning: '--content-safety-off sensitive-data' is deprecated and has no effect. "
+ "Sensitive data sanitization is now opt-in. "
+ "Use '--sanitize' to enable masking.");
break;
case "all":
config.getFilterConfig().setFilterHiddenText(false);
config.getFilterConfig().setFilterOutOfPage(false);
config.getFilterConfig().setFilterTinyText(false);
config.getFilterConfig().setFilterHiddenOCG(false);
break;
default:
throw new IllegalArgumentException(String.format(
"Unsupported value '%s'. Supported values: all, hidden-text, off-page, tiny, hidden-ocg",
value));
}
}
}
private static void applySanitizeOption(Config config, CommandLine commandLine) {
if (commandLine.hasOption(SANITIZE_LONG_OPTION)) {
config.getFilterConfig().setFilterSensitiveData(true);
}
}
private static void applyFormatOption(Config config, CommandLine commandLine) {
if (!commandLine.hasOption(FORMAT_OPTION)) {
return;
}
String[] optionValues = commandLine.getOptionValues(FORMAT_OPTION);
if (optionValues == null || optionValues.length == 0) {
throw new IllegalArgumentException(
"Option --format requires at least one value. Supported values: json, text, html, pdf, markdown, tagged-pdf");
}
Set<String> values = parseOptionValues(optionValues);
if (values.isEmpty()) {
throw new IllegalArgumentException(
"Option --format requires at least one value. Supported values: json, text, html, pdf, markdown, tagged-pdf");
}
config.setGenerateJSON(false);
for (String value : values) {
switch (value) {
case "json":
config.setGenerateJSON(true);
break;
case "html":
config.setGenerateHtml(true);
break;
case "text":
config.setGenerateText(true);
break;
case "pdf":
config.setGeneratePDF(true);
break;
case "markdown":
config.setGenerateMarkdown(true);
break;
case "markdown-with-html":
System.err.println("[WARN] --format markdown-with-html is deprecated and will be removed "
+ "in the next major release. Use --format markdown --markdown-with-html instead.");
config.setUseHTMLInMarkdown(true);
break;
case "markdown-with-images":
System.err.println("[WARN] --format markdown-with-images is deprecated and will be removed "
+ "in the next major release. Use --format markdown with --image-output "
+ "(off|embedded|external) instead.");
config.setGenerateMarkdown(true);
break;
case "tagged-pdf":
config.setGenerateTaggedPDF(true);
break;
default:
throw new IllegalArgumentException(String.format(
"Unsupported format '%s'. Supported values: json, text, html, pdf, markdown, tagged-pdf",
value));
}
}
}
private static Set<String> parseOptionValues(String[] optionValues) {
Set<String> values = new LinkedHashSet<>();
for (String rawValue : optionValues) {
if (rawValue == null) {
continue;
}
String[] splitValues = rawValue.split(",");
for (String candidate : splitValues) {
String format = candidate.trim().toLowerCase(Locale.ROOT);
if (!format.isEmpty()) {
values.add(format);
}
}
}
return values;
}
private static void applyHybridOptions(Config config, CommandLine commandLine) {
if (commandLine.hasOption(HYBRID_LONG_OPTION)) {
String hybridValue = commandLine.getOptionValue(HYBRID_LONG_OPTION);
if (hybridValue == null || hybridValue.trim().isEmpty()) {
throw new IllegalArgumentException(
String.format("Option --hybrid requires a value. Supported values: %s",
Config.getHybridOptions(", ")));
}
String hybrid = hybridValue.trim().toLowerCase(Locale.ROOT);
if (!Config.isValidHybrid(hybrid)) {
throw new IllegalArgumentException(
String.format("Unsupported hybrid backend '%s'. Supported values: %s",
hybrid, Config.getHybridOptions(", ")));
}
config.setHybrid(hybrid);
}
if (commandLine.hasOption(HYBRID_MODE_LONG_OPTION)) {
String modeValue = commandLine.getOptionValue(HYBRID_MODE_LONG_OPTION);
if (modeValue == null || modeValue.trim().isEmpty()) {
throw new IllegalArgumentException(
String.format("Option --hybrid-mode requires a value. Supported values: %s",
Config.getHybridModeOptions(", ")));
}
String mode = modeValue.trim().toLowerCase(Locale.ROOT);
if (!Config.isValidHybridMode(mode)) {
throw new IllegalArgumentException(
String.format("Unsupported hybrid mode '%s'. Supported values: %s",
mode, Config.getHybridModeOptions(", ")));
}
config.getHybridConfig().setMode(mode);
}
if (commandLine.hasOption(HYBRID_OCR_LONG_OPTION)) {
// Deprecated: OCR settings are now configured on the hybrid server
System.err.println("Warning: --hybrid-ocr is deprecated. "
+ "Configure OCR settings on the hybrid server instead (--ocr-lang, --force-ocr).");
}
if (commandLine.hasOption(HYBRID_URL_LONG_OPTION)) {
String url = commandLine.getOptionValue(HYBRID_URL_LONG_OPTION);
if (url != null && !url.trim().isEmpty()) {
config.getHybridConfig().setUrl(url.trim());
}
}
if (commandLine.hasOption(HYBRID_TIMEOUT_LONG_OPTION)) {
String timeoutValue = commandLine.getOptionValue(HYBRID_TIMEOUT_LONG_OPTION);
if (timeoutValue != null && !timeoutValue.trim().isEmpty()) {
try {
int timeout = Integer.parseInt(timeoutValue.trim());
config.getHybridConfig().setTimeoutMs(timeout);
} catch (NumberFormatException e) {
throw new IllegalArgumentException(
String.format("Invalid timeout value '%s'. Must be a non-negative integer.", timeoutValue));
}
}
}
if (commandLine.hasOption(HYBRID_FALLBACK_LONG_OPTION)) {
config.getHybridConfig().setFallbackToJava(true);
}
if (commandLine.hasOption(HYBRID_HANCOM_AI_REGIONLIST_STRATEGY_LONG_OPTION)) {
String value = commandLine.getOptionValue(HYBRID_HANCOM_AI_REGIONLIST_STRATEGY_LONG_OPTION);
if (value != null && !value.trim().isEmpty()) {
String normalized = value.trim().toLowerCase(Locale.ROOT);
if (!HybridConfig.REGIONLIST_TABLE_FIRST.equals(normalized)
&& !HybridConfig.REGIONLIST_LIST_ONLY.equals(normalized)) {
throw new IllegalArgumentException(String.format(
"Option --%s: unsupported value '%s'. Supported values: %s, %s",
HYBRID_HANCOM_AI_REGIONLIST_STRATEGY_LONG_OPTION, normalized,
HybridConfig.REGIONLIST_TABLE_FIRST, HybridConfig.REGIONLIST_LIST_ONLY));
}
config.getHybridConfig().setRegionlistStrategy(normalized);
}
}
if (commandLine.hasOption(HYBRID_HANCOM_AI_OCR_STRATEGY_LONG_OPTION)) {
String value = commandLine.getOptionValue(HYBRID_HANCOM_AI_OCR_STRATEGY_LONG_OPTION);
if (value != null && !value.trim().isEmpty()) {
String normalized = value.trim().toLowerCase(Locale.ROOT);
if (!HybridConfig.OCR_OFF.equals(normalized)
&& !HybridConfig.OCR_AUTO.equals(normalized)
&& !HybridConfig.OCR_FORCE.equals(normalized)) {
throw new IllegalArgumentException(String.format(
"Option --%s: unsupported value '%s'. Supported values: %s, %s, %s",
HYBRID_HANCOM_AI_OCR_STRATEGY_LONG_OPTION, normalized,
HybridConfig.OCR_OFF, HybridConfig.OCR_AUTO, HybridConfig.OCR_FORCE));
}
config.getHybridConfig().setOcrStrategy(normalized);
}
}
if (commandLine.hasOption(HYBRID_HANCOM_AI_IMAGE_CACHE_LONG_OPTION)) {
String value = commandLine.getOptionValue(HYBRID_HANCOM_AI_IMAGE_CACHE_LONG_OPTION);
if (value != null && !value.trim().isEmpty()) {
String normalized = value.trim().toLowerCase(Locale.ROOT);
if (!"memory".equals(normalized) && !"disk".equals(normalized)) {
throw new IllegalArgumentException(String.format(
"Option --%s: unsupported value '%s'. Supported values: memory, disk",
HYBRID_HANCOM_AI_IMAGE_CACHE_LONG_OPTION, normalized));
}
config.getHybridConfig().setImageCache(normalized);
}
}
if (commandLine.hasOption(HYBRID_HANCOM_AI_SAVE_CROPS_LONG_OPTION)) {
config.getHybridConfig().setSaveCrops(true);
}
if (commandLine.hasOption(HYBRID_HANCOM_AI_CROP_OUTPUT_DIR_LONG_OPTION)) {
String value = commandLine.getOptionValue(HYBRID_HANCOM_AI_CROP_OUTPUT_DIR_LONG_OPTION);
if (value != null && !value.trim().isEmpty()) {
config.getHybridConfig().setCropOutputDir(value.trim());
}
}
if (commandLine.hasOption(TO_STDOUT_LONG_OPTION)) {
config.setOutputStdout(true);
}
// Keep in sync with all HYBRID_HANCOM_AI_*_LONG_OPTION constants above.
boolean usesHancomAiOnly =
commandLine.hasOption(HYBRID_HANCOM_AI_REGIONLIST_STRATEGY_LONG_OPTION) ||
commandLine.hasOption(HYBRID_HANCOM_AI_OCR_STRATEGY_LONG_OPTION) ||
commandLine.hasOption(HYBRID_HANCOM_AI_IMAGE_CACHE_LONG_OPTION) ||
commandLine.hasOption(HYBRID_HANCOM_AI_SAVE_CROPS_LONG_OPTION) ||
commandLine.hasOption(HYBRID_HANCOM_AI_CROP_OUTPUT_DIR_LONG_OPTION);
if (usesHancomAiOnly && !Config.HYBRID_HANCOM_AI.equals(config.getHybrid())) {
throw new IllegalArgumentException(
"Options --hybrid-hancom-ai-* require --hybrid=hancom-ai (got --hybrid="
+ config.getHybrid() + ")");
}
}
/**
* Exports CLI option definitions as JSON for code generation.
* This is used to generate Node.js, Python, and documentation from a single
* source of truth.
*
* @param out The output stream to write JSON to
*/
public static void exportOptionsAsJson(PrintStream out) {
List<OptionDefinition> exportable = OPTION_DEFINITIONS.stream()
.filter(d -> d.exported)
.collect(Collectors.toList());
// Build JSON manually to avoid external dependencies
StringBuilder json = new StringBuilder();
json.append("{\n");
json.append(" \"options\": [\n");
for (int i = 0; i < exportable.size(); i++) {
OptionDefinition opt = exportable.get(i);
json.append(" {\n");
json.append(" \"name\": \"").append(opt.longName).append("\",\n");
json.append(" \"shortName\": ").append(opt.shortName == null ? "null" : "\"" + opt.shortName + "\"")
.append(",\n");
json.append(" \"type\": \"").append(opt.type).append("\",\n");
json.append(" \"required\": false,\n");
if (opt.defaultValue == null) {
json.append(" \"default\": null,\n");
} else if (opt.defaultValue instanceof Boolean) {
json.append(" \"default\": ").append(opt.defaultValue).append(",\n");
} else {
json.append(" \"default\": \"").append(escapeJson(opt.defaultValue.toString())).append("\",\n");
}
json.append(" \"description\": \"").append(escapeJson(opt.description)).append("\"\n");
json.append(" }");
if (i < exportable.size() - 1) {
json.append(",");
}
json.append("\n");
}
json.append(" ]\n");
json.append("}\n");
out.print(json.toString());
}
private static String escapeJson(String value) {
if (value == null) {
return "";
}
return value
.replace("\\", "\\\\")
.replace("\"", "\\\"")
.replace("\n", "\\n")
.replace("\r", "\\r")
.replace("\t", "\\t");
}
/**
* Internal class to hold option definition for both CLI and JSON export.
* Single source of truth for all option metadata.
*/
private static class OptionDefinition {
final String longName;
final String shortName;
final String type; // "string" | "boolean"
final Object defaultValue;
final String description;
final boolean exported; // Whether to include in JSON export
OptionDefinition(String longName, String shortName, String type, Object defaultValue, String description,
boolean exported) {
this.longName = longName;
this.shortName = shortName;
this.type = type;
this.defaultValue = defaultValue;
this.description = description;
this.exported = exported;
}
/** Creates an Apache Commons CLI Option from this definition. */
Option toOption() {
boolean hasArg = "string".equals(type);
return new Option(shortName, longName, hasArg, description);
}
}
}
java/opendataloader-pdf-core/src/main/java/org/opendataloader/pdf/autotagging/ChunksWriter.java
+323
View File
@@ -0,0 +1,323 @@
package org.opendataloader.pdf.autotagging;
import org.opendataloader.pdf.processors.AutoTaggingProcessor;
import org.verapdf.as.ASAtom;
import org.verapdf.as.io.ASInputStream;
import org.verapdf.cos.*;
import org.verapdf.gf.model.factory.chunks.ChunkParser;
import org.verapdf.gf.model.factory.chunks.GraphicsState;
import org.verapdf.gf.model.impl.sa.util.ResourceHandler;
import org.verapdf.operator.Operator;
import org.verapdf.parser.Operators;
import org.verapdf.parser.PDFStreamParser;
import org.verapdf.pd.PDContentStream;
import org.verapdf.pd.PDExtGState;
import org.verapdf.pd.images.PDXForm;
import org.verapdf.pd.images.PDXObject;
import org.verapdf.tools.StaticResources;
import org.verapdf.tools.TaggedPDFConstants;
import org.verapdf.wcag.algorithms.semanticalgorithms.containers.StaticContainers;
import org.verapdf.wcag.algorithms.semanticalgorithms.utils.StreamInfo;
import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.util.*;
public class ChunksWriter {
private static final java.util.logging.Logger CHUNKS_LOGGER = java.util.logging.Logger.getLogger(ChunksWriter.class.getName());
private final ResourceHandler resourceHandler;
private final GraphicsState graphicsState;
public ChunksWriter(GraphicsState inheritedGraphicState, ResourceHandler resourceHandler) {
this.graphicsState = inheritedGraphicState.clone();
this.resourceHandler = resourceHandler;
}
public static List<Object> getTokens(PDContentStream pdContentStream) {
if (pdContentStream != null) {
try {
COSObject contentStream = pdContentStream.getContents();
if (contentStream.getType() == COSObjType.COS_STREAM || contentStream.getType() == COSObjType.COS_ARRAY) {
try (ASInputStream opStream = contentStream.getDirectBase().getData(COSStream.FilterFlags.DECODE)) {
try (PDFStreamParser streamParser = new PDFStreamParser(opStream)) {
streamParser.parseTokens();
return streamParser.getTokens();
}
}
}
} catch (IOException e) {
CHUNKS_LOGGER.warning("Failed to read content stream tokens: " + e.getMessage());
}
}
return Collections.emptyList();
}
public List<Object> processTokens(List<Object> processTokens, OperatorStreamKey operatorStreamKey) throws IOException {
Map<Integer, Set<StreamInfo>> operatorIndexesToStreamInfosMap = AutoTaggingProcessor.getOperatorIndexesToStreamInfosMap().get(operatorStreamKey);
if (operatorIndexesToStreamInfosMap == null) {
operatorIndexesToStreamInfosMap = Collections.emptyMap();
}
List<Object> result = new ArrayList<>();
List<COSBase> arguments = new ArrayList<>();
for (int index = 0; index < processTokens.size(); index++) {
Object token = processTokens.get(index);
if (token instanceof COSBase) {
arguments.add((COSBase) token);
} else if (token instanceof Operator) {
processOperator(result, (Operator)token, arguments, index, operatorIndexesToStreamInfosMap, operatorStreamKey);
arguments.clear();
}
}
return result;
}
private void processOperator(List<Object> result, Operator rawOperator, List<COSBase> arguments, int operatorIndex,
Map<Integer, Set<StreamInfo>> operatorIndexesToStreamInfosMap,
OperatorStreamKey operatorStreamKey) throws IOException {
String operatorName = rawOperator.getOperator();
switch (operatorName) {
case Operators.BMC:
case Operators.EMC:
case Operators.BDC:
break;
case Operators.DO:
OperatorStreamKey xObjectOperatorStreamKey = new OperatorStreamKey(operatorStreamKey.getPageNumber(), arguments.get(0).getString());
Integer xObjectStructParent = AutoTaggingProcessor.getStructParentsIntegers().get(xObjectOperatorStreamKey);
if (AutoTaggingProcessor.getOperatorIndexesToStreamInfosMap().containsKey(xObjectOperatorStreamKey)
&& xObjectStructParent != null) {
COSName xObjectName = getLastCOSName(arguments);
PDXObject pdxObject = resourceHandler.getXObject(xObjectName);
if (pdxObject == null) {
processContentOperator(result, rawOperator, arguments, operatorIndex, operatorIndexesToStreamInfosMap, operatorName, operatorStreamKey);
break;
}
pdxObject.setKey(ASAtom.STRUCT_PARENTS,
COSInteger.construct(xObjectStructParent));
StaticResources.getDocument().getDocument().addChangedObject(pdxObject.getObject());
PDXForm pdxForm = (PDXForm)pdxObject;
GraphicsState xFormGraphicsState = graphicsState.clone();
AutoTaggingProcessor.setUpContents(pdxForm.getObject(), new ChunksWriter(xFormGraphicsState,
resourceHandler.getExtendedResources(pdxForm.getResources())).processTokens(
ChunksWriter.getTokens(pdxForm), xObjectOperatorStreamKey));
// Preserve the Do operator in the parent stream so the XObject is still invoked
result.addAll(arguments);
result.add(rawOperator);
} else {
processContentOperator(result, rawOperator, arguments, operatorIndex, operatorIndexesToStreamInfosMap, operatorName, operatorStreamKey);
}
break;
case Operators.TJ_SHOW:
case Operators.TJ_SHOW_POS:
case Operators.QUOTE:
case Operators.DOUBLE_QUOTE:
case Operators.BI:
case Operators.F_FILL:
case Operators.F_FILL_OBSOLETE:
case Operators.F_STAR_FILL:
case Operators.B_CLOSEPATH_FILL_STROKE:
case Operators.B_STAR_CLOSEPATH_EOFILL_STROKE:
case Operators.S_CLOSE_STROKE:
case Operators.S_STROKE:
processContentOperator(result, rawOperator, arguments, operatorIndex, operatorIndexesToStreamInfosMap, operatorName, operatorStreamKey);
break;
case org.verapdf.model.tools.constants.Operators.GS:
PDExtGState extGState = this.resourceHandler.getExtGState(getLastCOSName(arguments));
this.graphicsState.copyPropertiesFromExtGState(extGState);
result.addAll(arguments);
result.add(rawOperator);
break;
case org.verapdf.model.tools.constants.Operators.TF:
this.graphicsState.getTextState().setTextFont(resourceHandler.getFont(getFirstCOSName(arguments)));
result.addAll(arguments);
result.add(rawOperator);
break;
default:
// if (!Operators.D_SET_DASH.equals(operatorName)) {
result.addAll(arguments);
result.add(rawOperator);
// }
break;
}
}
private void processContentOperator(List<Object> result, Operator rawOperator, List<COSBase> arguments, int operatorIndex,
Map<Integer, Set<StreamInfo>> operatorIndexesToStreamInfosMap, String operatorName,
OperatorStreamKey operatorStreamKey) {
Set<StreamInfo> streamInfos = operatorIndexesToStreamInfosMap.get(operatorIndex);
if (streamInfos == null || streamInfos.isEmpty()) {
writeMarkedContent(result, arguments, rawOperator, operatorName, null, null);
} else {
if (streamInfos.size() == 1) {
Integer mcid = streamInfos.iterator().next().getMcid();
writeMarkedContent(result, arguments, rawOperator, operatorName, mcid, operatorStreamKey);
} else {
List<StreamInfo> streamInfosList = updateStreamInfos(streamInfos);
Map<StreamInfo, COSObject> newArguments = getArguments(arguments.get(arguments.size() - 1), streamInfosList);
for (Map.Entry<StreamInfo, COSObject> entry : newArguments.entrySet()) {
arguments.set(arguments.size() - 1, entry.getValue().get());
writeMarkedContent(result, arguments, rawOperator, operatorName, entry.getKey().getMcid(), operatorStreamKey);
}
}
}
}
private static String getStructureType(Integer mcid, OperatorStreamKey operatorStreamKey) {
if (mcid == null || operatorStreamKey == null) return null;
List<COSObject> parents = AutoTaggingProcessor.getStructParents().get(operatorStreamKey);
if (parents == null) {
CHUNKS_LOGGER.warning("structParents: no entry for key page=" + operatorStreamKey.getPageNumber() + " xobj=" + operatorStreamKey.getXObjectName() + " (available keys: " + AutoTaggingProcessor.getStructParents().keySet().stream().map(k -> "p"+k.getPageNumber()+"x"+k.getXObjectName()).collect(java.util.stream.Collectors.joining(",")) + ")");
return null;
}
if (mcid >= parents.size()) {
CHUNKS_LOGGER.warning("structParents: mcid=" + mcid + " out of range (size=" + parents.size() + ") for key page=" + operatorStreamKey.getPageNumber() + " xobj=" + operatorStreamKey.getXObjectName());
return null;
}
COSObject structElem = parents.get(mcid);
if (structElem == null) return null;
COSObject typeObj = structElem.getKey(ASAtom.S);
if (typeObj == null || typeObj.empty()) return null;
return typeObj.getString();
}
private static void writeMarkedContent(List<Object> result, List<COSBase> arguments, Operator token,
String operatorName, Integer mcid, OperatorStreamKey operatorStreamKey) {
if (mcid == null) {
result.add(COSName.construct(TaggedPDFConstants.ARTIFACT).getDirectBase());
result.add(Operator.getOperator(Operators.BMC));
} else {
String tagName;
if (Operators.BI.equals(operatorName) || Operators.DO.equals(operatorName)) {
tagName = TaggedPDFConstants.FIGURE;
} else {
String structType = getStructureType(mcid, operatorStreamKey);
tagName = (structType != null) ? structType : TaggedPDFConstants.SPAN;
}
result.add(COSName.construct(tagName).getDirectBase());
COSObject dictionary = COSDictionary.construct();
dictionary.setKey(ASAtom.MCID, COSInteger.construct(mcid));
result.add(dictionary.getDirectBase());
result.add(Operator.getOperator(Operators.BDC));
}
result.addAll(arguments);
result.add(token);
result.add(Operator.getOperator(Operators.EMC));
}
private Map<StreamInfo, COSObject> getArguments(COSBase object, List<StreamInfo> streamInfos) {
Map<StreamInfo, COSObject> map = new TreeMap<>();
if (object.getType() == COSObjType.COS_STRING) {
Queue<StreamInfo> streamInfoQueue = new LinkedList<>(streamInfos);
processString((COSString)object, map, streamInfoQueue, null, 0);
} else if (object.getType() == COSObjType.COS_ARRAY) {
int stringIndex = 0;
List<COSObject> array = new ArrayList<>();
Queue<StreamInfo> streamInfoQueue = new LinkedList<>(streamInfos);
for (COSObject element : (COSArray)object.getDirectBase()) {
if (element.getType() == COSObjType.COS_STRING) {
stringIndex = processString((COSString)element.get(), map, streamInfoQueue, array, stringIndex);
} else if (element.getType().isNumber()) {
array.add(element);
}
}
if (!array.isEmpty()) {
StreamInfo peekInfo = streamInfoQueue.peek();
COSObject target = peekInfo != null ? map.get(peekInfo) : null;
if (target != null && target.getType() == COSObjType.COS_ARRAY) {
COSArray currentArray = (COSArray) target.get();
for (COSObject element : array) {
currentArray.add(element);
}
} else {
CHUNKS_LOGGER.warning("Issue during text operator processing");
}
}
}
return map;
}
public int processString(COSString string, Map<StreamInfo, COSObject> map, Queue<StreamInfo> streamInfoQueue,
List<COSObject> array, int stringIndex) {
int currentStringIndex = stringIndex;
int currentBytesIndex = 0;
int dif = 0;
byte[] bytes = string.get();
try (InputStream inputStream = new ByteArrayInputStream(bytes)) {
int available = inputStream.available();
while (inputStream.available() > 0) {
int code = graphicsState.getTextState().getTextFont().readCode(inputStream);
String value = graphicsState.getTextState().getTextFont().toUnicode(code);
if (value == null) {
value = StaticContainers.getIsIgnoreCharactersWithoutUnicode() ? "" : ChunkParser.REPLACEMENT_CHARACTER_STRING;
}
int newAvailable = inputStream.available();
dif += available - newAvailable;
available = newAvailable;
int length = streamInfoQueue.peek().getEndIndex() - currentStringIndex;
currentStringIndex += value.length();
if (length <= value.length()) {
COSObject cosString = COSString.construct(Arrays.copyOfRange(bytes, currentBytesIndex, currentBytesIndex + dif), string.isHexadecimal());
if (array != null) {
array.add(cosString);
map.put(streamInfoQueue.peek(), new COSObject(new COSArray(array)));
} else {
map.put(streamInfoQueue.peek(), cosString);
}
currentBytesIndex += dif;
dif = 0;
if (array != null) {
array.clear();
}
if (streamInfoQueue.size() > 1) {
streamInfoQueue.poll();
}
}
}
} catch (IOException e) {
CHUNKS_LOGGER.warning("Failed to process font string: " + e.getMessage());
}
if (array != null && dif > 0) {
array.add(COSString.construct(Arrays.copyOfRange(bytes, currentBytesIndex, currentBytesIndex + dif), string.isHexadecimal()));
}
return currentStringIndex;
}
private static List<StreamInfo> updateStreamInfos(Set<StreamInfo> streamInfos) {
Iterator<StreamInfo> streamInfoIterator = streamInfos.iterator();
List<StreamInfo> newStreamInfos = new ArrayList<>();
StreamInfo streamInfo = null;
int currentIndex = 0;
while (streamInfoIterator.hasNext()) {
streamInfo = streamInfoIterator.next();
if (currentIndex < streamInfo.getStartIndex()) {
newStreamInfos.add(new StreamInfo(streamInfo.getOperatorIndex(), streamInfo.getXObjectName(),
streamInfo.getXImageObjectKey(), currentIndex, streamInfo.getStartIndex(), streamInfo.getLength(), null));
}
currentIndex = streamInfo.getEndIndex();
newStreamInfos.add(streamInfo);
}
if (currentIndex < streamInfo.getLength()) {
newStreamInfos.add(new StreamInfo(streamInfo.getOperatorIndex(), streamInfo.getXObjectName(),
streamInfo.getXImageObjectKey(), streamInfo.getEndIndex(), streamInfo.getLength(), streamInfo.getLength(), null));
}
return newStreamInfos;
}
private static COSName getFirstCOSName(List<COSBase> arguments) {
COSBase lastElement = arguments.isEmpty() ? null : arguments.get(0);
if (lastElement instanceof COSName) {
return (COSName) lastElement;
}
return null;
}
private static COSName getLastCOSName(List<COSBase> arguments) {
COSBase lastElement = arguments.isEmpty() ? null : arguments.get(arguments.size() - 1);
if (lastElement instanceof COSName) {
return (COSName) lastElement;
}
return null;
}
}
java/opendataloader-pdf-core/src/main/java/org/opendataloader/pdf/autotagging/OperatorStreamKey.java
+39
View File
@@ -0,0 +1,39 @@
package org.opendataloader.pdf.autotagging;
import java.util.Objects;
public class OperatorStreamKey {
private final int pageNumber;
private final String xObjectName;
public OperatorStreamKey(int pageNumber, String xObjectName) {
this.pageNumber = pageNumber;
this.xObjectName = xObjectName;
}
public int getPageNumber() {
return pageNumber;
}
public String getXObjectName() {
return xObjectName;
}
@Override
public boolean equals(Object o) {
if (this == o) {
return true;
}
if (o == null || getClass() != o.getClass()) {
return false;
}
OperatorStreamKey that = (OperatorStreamKey) o;
return pageNumber == that.pageNumber &&
Objects.equals(xObjectName, that.xObjectName);
}
@Override
public int hashCode() {
return Objects.hash(pageNumber, xObjectName);
}
}
java/opendataloader-pdf-core/src/main/java/org/opendataloader/pdf/containers/StaticLayoutContainers.java
+164
View File
@@ -0,0 +1,164 @@
/*
* Copyright 2025-2026 Hancom Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.opendataloader.pdf.containers;
import org.opendataloader.pdf.api.Config;
import org.verapdf.wcag.algorithms.entities.SemanticHeading;
import java.io.File;
import java.util.Collections;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
import java.util.logging.Logger;
public class StaticLayoutContainers {
protected static final Logger LOGGER = Logger.getLogger(StaticLayoutContainers.class.getCanonicalName());
private static final ThreadLocal<Long> currentContentId = new ThreadLocal<>();
private static final ThreadLocal<List<SemanticHeading>> headings = new ThreadLocal<>();
private static final ThreadLocal<Integer> imageIndex = new ThreadLocal<>();
private static final ThreadLocal<Boolean> isUseStructTree = new ThreadLocal<>();
private static final ThreadLocal<String> imagesDirectory = new ThreadLocal<>();
private static final ThreadLocal<Boolean> embedImages = new ThreadLocal<>();
private static final ThreadLocal<String> imageFormat = new ThreadLocal<>();
private static final ThreadLocal<Map<Integer, Double>> replacementCharRatios = ThreadLocal.withInitial(ConcurrentHashMap::new);
private static final ThreadLocal<Map<String, byte[]>> embeddedImageBytes = ThreadLocal.withInitial(ConcurrentHashMap::new);
public static void clearContainers() {
currentContentId.set(1L);
headings.set(Collections.synchronizedList(new LinkedList<>()));
imageIndex.set(1);
isUseStructTree.set(false);
imagesDirectory.set("");
embedImages.set(false);
imageFormat.set(Config.IMAGE_FORMAT_PNG);
replacementCharRatios.get().clear();
embeddedImageBytes.get().clear();
}
public static long getCurrentContentId() {
return currentContentId.get();
}
public static long incrementContentId() {
long id = getCurrentContentId();
StaticLayoutContainers.setCurrentContentId(id + 1);
return id;
}
public static void setCurrentContentId(long currentContentId) {
StaticLayoutContainers.currentContentId.set(currentContentId);
}
public static String getImagesDirectory() {
return imagesDirectory.get();
}
public static String getImagesDirectoryName() {
String dir = imagesDirectory.get();
return dir != null && !dir.isEmpty() ? new File(dir).getName() : "";
}
public static void setImagesDirectory(String imagesDirectory) {
StaticLayoutContainers.imagesDirectory.set(imagesDirectory);
}
public static List<SemanticHeading> getHeadings() {
return headings.get();
}
public static void setHeadings(List<SemanticHeading> headings) {
StaticLayoutContainers.headings.set(headings);
}
public static Boolean isUseStructTree() {
return isUseStructTree.get();
}
public static void setIsUseStructTree(Boolean isUseStructTree) {
StaticLayoutContainers.isUseStructTree.set(isUseStructTree);
}
public static int incrementImageIndex() {
int imageIndex = StaticLayoutContainers.imageIndex.get();
StaticLayoutContainers.imageIndex.set(imageIndex + 1);
return imageIndex;
}
public static void resetImageIndex() {
StaticLayoutContainers.imageIndex.set(1);
}
public static boolean isEmbedImages() {
return Boolean.TRUE.equals(embedImages.get());
}
public static void setEmbedImages(boolean embedImages) {
StaticLayoutContainers.embedImages.set(embedImages);
}
public static String getImageFormat() {
String format = imageFormat.get();
return format != null ? format : Config.IMAGE_FORMAT_PNG;
}
public static void setImageFormat(String format) {
StaticLayoutContainers.imageFormat.set(format);
}
public static void setReplacementCharRatio(int pageNumber, double ratio) {
replacementCharRatios.get().put(pageNumber, ratio);
}
public static double getReplacementCharRatio(int pageNumber) {
return replacementCharRatios.get().getOrDefault(pageNumber, 0.0);
}
public static void cacheEmbeddedImageBytes(String absolutePath, byte[] bytes) {
embeddedImageBytes.get().put(normalizeImageKey(absolutePath), bytes);
}
public static byte[] getEmbeddedImageBytes(String absolutePath) {
return embeddedImageBytes.get().get(normalizeImageKey(absolutePath));
}
public static boolean hasEmbeddedImageBytes(String absolutePath) {
return embeddedImageBytes.get().containsKey(normalizeImageKey(absolutePath));
}
// Map-level accessors are used by DocumentProcessor.propagateState so worker threads
// share the main thread's cache instance. Today the cache is only touched on the main
// thread, but propagating it eliminates a silent-data-loss trap if generators ever
// run on workers — matches the CLAUDE.md ThreadLocal-propagation gotcha.
public static Map<String, byte[]> getEmbeddedImageBytesMap() {
return embeddedImageBytes.get();
}
public static void setEmbeddedImageBytesMap(Map<String, byte[]> map) {
embeddedImageBytes.set(map);
}
// Image paths flow through String.format with File.separator (cache write side) and
// through File.getPath() (cache read side via Base64ImageUtils.toDataUri). The two
// forms can differ on Windows when the user passes forward-slash output paths
// (e.g. /tmp/x). Collapse both forms to a single canonical key so writers and readers
// agree on cache identity.
private static String normalizeImageKey(String path) {
return path == null ? null : new File(path).getPath();
}
}
java/opendataloader-pdf-core/src/main/java/org/opendataloader/pdf/entities/EnrichedImageChunk.java
+106
View File
@@ -0,0 +1,106 @@
/*
* Copyright 2025-2026 Hancom Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.opendataloader.pdf.entities;
import org.verapdf.wcag.algorithms.entities.content.ImageChunk;
/**
* An ImageChunk enriched with an AI-generated description (alt text).
*
* <p>Created when the hybrid backend returns a SemanticPicture whose bounding
* box overlaps a Java-extracted ImageChunk. The description is matched by
* bounding-box IoU in HybridDocumentProcessor and propagated to:
* <ul>
* <li>AutoTaggingProcessor — inserts /Alt into the Figure struct element</li>
* <li>ImageSerializer — writes "alt" field to JSON output</li>
* <li>MarkdownGenerator / HtmlGenerator — uses description as alt text</li>
* </ul>
*/
public class EnrichedImageChunk extends ImageChunk {
/**
* Where the alt text came from. Drives the {@code alt_source} JSON field
* and downstream evidence-report badging — a reviewer treats a human-authored
* /Alt very differently from an AI-generated caption.
*/
public enum AltSource {
/** Pulled from the source PDF's struct-tree /Alt entry. */
ORIGINAL,
/** Generated by the hybrid backend (Hancom AI / docling SmolVLM). */
AI_GENERATED
}
private final String description;
private final AltSource altSource;
/**
* Convenience constructor for author-authored /Alt sourced from the input
* PDF's struct tree (used by {@link org.opendataloader.pdf.processors.TaggedDocumentProcessor}).
* AI / backend-generated captions must use the 3-arg form with
* {@link AltSource#AI_GENERATED} so downstream consumers can distinguish
* human alt text from synthesised descriptions.
*/
public EnrichedImageChunk(ImageChunk source, String description) {
this(source, description, AltSource.ORIGINAL);
}
public EnrichedImageChunk(ImageChunk source, String description, AltSource altSource) {
super(source.getBoundingBox());
// Copy index so serializers can reference the image file
setIndex(source.getIndex());
// Copy stream infos so MCID / struct-tree linkage is preserved
getStreamInfos().addAll(source.getStreamInfos());
this.description = description;
this.altSource = altSource;
}
public String getDescription() {
return description != null ? description : "";
}
public boolean hasDescription() {
return description != null && !description.isEmpty();
}
public AltSource getAltSource() {
return altSource;
}
/**
* Sanitized description safe for use as PDF /Alt, Markdown alt text,
* HTML alt attribute, and JSON value.
*/
public String sanitizeDescription() {
if (!hasDescription()) return "";
return description
.replace("\r\n", " ")
.replace("\n", " ")
.replace("\r", " ")
.replace("\"", "")
.replace("[", "")
.replace("]", "")
.replace("<", "")
.replace(">", "")
.replace("&", "")
.replace("\u0000", "")
// PDF/UA-2 clause 8.4.3.3 forbids Private Use Area code points in /Alt.
// Strip BMP PUA (U+E000–U+F8FF) and supplementary PUA (encoded as surrogate pairs
// U+DB80–U+DBFF paired with U+DC00–U+DFFF).
.replaceAll("[\\uE000-\\uF8FF]|[\\uDB80-\\uDBFF][\\uDC00-\\uDFFF]", "")
.replaceAll("\\s{2,}", " ")
.trim();
}
}
java/opendataloader-pdf-core/src/main/java/org/opendataloader/pdf/entities/SemanticFootnote.java
+34
View File
@@ -0,0 +1,34 @@
/*
* Copyright 2025-2026 Hancom Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.opendataloader.pdf.entities;
import org.verapdf.wcag.algorithms.entities.SemanticParagraph;
import org.verapdf.wcag.algorithms.entities.enums.SemanticType;
/**
* Represents a footnote/endnote element.
* Maps to PDF 2.0 FENote structure element in AutoTaggingProcessor.
*/
public class SemanticFootnote extends SemanticParagraph {
public SemanticFootnote() {
super();
}
public SemanticFootnote(SemanticParagraph node) {
super(node);
this.setSemanticType(SemanticType.NOTE);
}
}
java/opendataloader-pdf-core/src/main/java/org/opendataloader/pdf/entities/SemanticFormula.java
+52
View File
@@ -0,0 +1,52 @@
/*
* Copyright 2025-2026 Hancom Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.opendataloader.pdf.entities;
import org.verapdf.wcag.algorithms.entities.BaseObject;
import org.verapdf.wcag.algorithms.entities.geometry.BoundingBox;
/**
* Represents a mathematical formula element with LaTeX content.
*
* <p>This class stores formula content in LaTeX format, which can be rendered
* using MathJax, KaTeX, or similar libraries in the output formats.
*
* <p>Extends BaseObject to leverage the standard IObject implementation.
*/
public class SemanticFormula extends BaseObject {
private final String latex;
/**
* Creates a SemanticFormula with the given bounding box and LaTeX content.
*
* @param boundingBox The bounding box of the formula
* @param latex The LaTeX representation of the formula
*/
public SemanticFormula(BoundingBox boundingBox, String latex) {
super(boundingBox);
this.latex = latex;
}
/**
* Gets the LaTeX representation of the formula.
*
* @return The LaTeX string, or empty string if null
*/
public String getLatex() {
return latex != null ? latex : "";
}
}
java/opendataloader-pdf-core/src/main/java/org/opendataloader/pdf/entities/SemanticPicture.java
+120
View File
@@ -0,0 +1,120 @@
/*
* Copyright 2025-2026 Hancom Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.opendataloader.pdf.entities;
import org.verapdf.wcag.algorithms.entities.BaseObject;
import org.verapdf.wcag.algorithms.entities.geometry.BoundingBox;
/**
* Represents a picture element with optional description (alt text).
*
* <p>This class stores picture metadata including AI-generated descriptions
* for accessibility purposes. Descriptions are generated using vision-language
* models when the hybrid server is configured with --enrich-picture-description.
*
* <p>Extends BaseObject to leverage the standard IObject implementation.
*/
public class SemanticPicture extends BaseObject {
private final int index;
private final String description;
/**
* Creates a SemanticPicture with the given bounding box and index.
*
* @param boundingBox The bounding box of the picture
* @param index The sequential index of the picture
*/
public SemanticPicture(BoundingBox boundingBox, int index) {
this(boundingBox, index, null);
}
/**
* Creates a SemanticPicture with the given bounding box, index, and description.
*
* @param boundingBox The bounding box of the picture
* @param index The sequential index of the picture
* @param description The AI-generated description (alt text) for accessibility
*/
public SemanticPicture(BoundingBox boundingBox, int index, String description) {
super(boundingBox);
this.index = index;
this.description = description;
}
/**
* Gets the sequential index of this picture.
*
* @return The picture index
*/
public int getPictureIndex() {
return index;
}
/**
* Gets the description (alt text) of this picture.
*
* @return The description string, or empty string if null
*/
public String getDescription() {
return description != null ? description : "";
}
/**
* Checks if this picture has a description.
*
* @return true if description is non-null and non-empty
*/
public boolean hasDescription() {
return description != null && !description.isEmpty();
}
/**
* Returns a sanitized version of the description safe for use as alt text
* across all output formats (Markdown, HTML, JSON) without format-specific escaping.
*
* <p>Removes characters that are structurally significant in at least one output format:
* <ul>
* <li>{@code "} — HTML attribute delimiter</li>
* <li>{@code [}, {@code ]} — Markdown alt text delimiters</li>
* <li>{@code <}, {@code >} — HTML tag delimiters</li>
* <li>{@code &} — HTML entity prefix</li>
* <li>{@code \u0000} — null character</li>
* <li>Newlines ({@code \n}, {@code \r}) — replaced with a space</li>
* </ul>
* Consecutive whitespace is collapsed to a single space and the result is trimmed.
*
* @return sanitized description string, or empty string if no description
*/
public String sanitizeDescription() {
if (!hasDescription()) {
return "";
}
return description
.replace("\r\n", " ")
.replace("\n", " ")
.replace("\r", " ")
.replace("\"", "")
.replace("[", "")
.replace("]", "")
.replace("<", "")
.replace(">", "")
.replace("&", "")
.replace("\u0000", "")
.replaceAll("\\s{2,}", " ")
.trim();
}
}
java/opendataloader-pdf-core/src/main/java/org/opendataloader/pdf/exceptions/EncryptedTaggedPdfNotSupportedException.java
+36
View File
@@ -0,0 +1,36 @@
/*
* Copyright 2025-2026 Hancom Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.opendataloader.pdf.exceptions;
import java.io.IOException;
/**
* Thrown when tagged-pdf generation is requested for an encrypted document.
*
* <p>Encrypted PDFs (those with an {@code /Encrypt} dictionary in the trailer)
* are not supported for tagged-pdf output. Writing a tagged copy requires
* re-serializing the document, which interacts poorly with the document's
* encryption state and is also not permitted by the PDF specification for
* documents whose permissions deny modification.
*/
public class EncryptedTaggedPdfNotSupportedException extends IOException {
private static final long serialVersionUID = 1L;
public EncryptedTaggedPdfNotSupportedException(String message) {
super(message);
}
}
java/opendataloader-pdf-core/src/main/java/org/opendataloader/pdf/exceptions/InvalidPdfFileException.java
+60
View File
@@ -0,0 +1,60 @@
/*
* Copyright 2025-2026 Hancom Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.opendataloader.pdf.exceptions;
import java.io.IOException;
/**
* Thrown when an input file cannot be processed as a PDF. Two failure modes
* share this exception type, distinguished by message wording:
* <ul>
* <li><em>Missing header</em> — the {@code %PDF-} magic number is absent
* from the first 1024 bytes (JPEG renamed to {@code .pdf}, empty file,
* arbitrary text). Detected before veraPDF is invoked.</li>
* <li><em>Corrupted or truncated content</em> — the magic number is
* present but the body fails to parse (interrupted download, missing
* trailing xref, garbage payload). Detected when {@code new PDDocument}
* throws {@link IOException}; the original veraPDF exception is
* preserved via {@link #getCause()} for diagnostics.</li>
* </ul>
*
* <p>This is a checked subtype of {@link IOException} so callers that already
* handle {@code IOException} keep compiling, while callers that want to
* distinguish "not a usable PDF" from other I/O failures can catch this type
* specifically.
*
* <p>Public entry points that may surface this exception:
* <ul>
* <li>{@code OpenDataLoaderPDF.processFile(String, Config)}</li>
* <li>{@code DocumentProcessor.processFile(String, Config)}</li>
* <li>{@code DocumentProcessor.processFileWithResult(String, Config)}</li>
* <li>{@code DocumentProcessor.extractContents(String, Config)}</li>
* <li>{@code DocumentProcessor.preprocessing(String, Config)}</li>
* <li>{@code AutoTagger.tag(String, Config, Float)}</li>
* </ul>
*/
public class InvalidPdfFileException extends IOException {
private static final long serialVersionUID = 1L;
public InvalidPdfFileException(String message) {
super(message);
}
public InvalidPdfFileException(String message, Throwable cause) {
super(message, cause);
}
}
java/opendataloader-pdf-core/src/main/java/org/opendataloader/pdf/html/HtmlGenerator.java
+550
View File
@@ -0,0 +1,550 @@
/*
* Copyright 2025-2026 Hancom Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.opendataloader.pdf.html;
import org.opendataloader.pdf.api.Config;
import org.opendataloader.pdf.containers.StaticLayoutContainers;
import org.opendataloader.pdf.entities.SemanticFormula;
import org.opendataloader.pdf.entities.EnrichedImageChunk;
import org.opendataloader.pdf.entities.SemanticPicture;
import org.opendataloader.pdf.markdown.MarkdownSyntax;
import org.opendataloader.pdf.utils.Base64ImageUtils;
import org.opendataloader.pdf.utils.GeneratorUtils;
import org.opendataloader.pdf.utils.ImagesUtils;
import org.opendataloader.pdf.utils.OutputType;
import org.verapdf.wcag.algorithms.entities.*;
import org.verapdf.wcag.algorithms.entities.content.*;
import org.verapdf.wcag.algorithms.entities.lists.ListItem;
import org.verapdf.wcag.algorithms.entities.lists.PDFList;
import org.verapdf.wcag.algorithms.entities.tables.tableBorders.TableBorder;
import org.verapdf.wcag.algorithms.entities.tables.tableBorders.TableBorderCell;
import org.verapdf.wcag.algorithms.entities.tables.tableBorders.TableBorderRow;
import org.verapdf.wcag.algorithms.semanticalgorithms.containers.StaticContainers;
import org.verapdf.wcag.algorithms.semanticalgorithms.utils.NodeUtils;
import java.awt.Color;
import java.io.Closeable;
import java.io.File;
import java.io.FileWriter;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.nio.file.Path;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import java.util.logging.Level;
import java.util.logging.Logger;
/**
* Generates HTML output from PDF document content.
* Converts semantic elements like paragraphs, headings, tables, and images into HTML format.
*/
public class HtmlGenerator implements Closeable {
/** Logger for this class. */
protected static final Logger LOGGER = Logger.getLogger(HtmlGenerator.class.getCanonicalName());
/** Writer for the HTML output file. */
protected final FileWriter htmlWriter;
/** Name of the input PDF file. */
protected final String pdfFileName;
/** Absolute path to the input PDF file. */
protected final Path pdfFilePath;
/** Name of the output HTML file. */
protected final String htmlFileName;
/** Absolute path to the output HTML file. */
protected final Path htmlFilePath;
/** Current table nesting level for tracking nested tables. */
protected int tableNesting = 0;
/** String to insert between pages in HTML output. */
protected String htmlPageSeparator = "";
/**
* Page numbers (1-based) selected by --pages; an empty set means all pages.
* Sourced from the raw {@link Config#getPageNumbers()} list (not the
* validated set built by {@code DocumentProcessor.getValidPageNumbers}).
* Safe to compare against {@code pageNumber + 1} because the surrounding
* loop is bounded by the document's actual page count, so out-of-range
* values from the raw list are never tested for membership.
*/
protected final Set<Integer> selectedPageNumbers;
/** Whether to embed images as Base64 data URIs. */
protected boolean embedImages = false;
/** Format for extracted images (png or jpeg). */
protected String imageFormat = Config.IMAGE_FORMAT_PNG;
/** Whether to include page headers and footers in output. */
protected boolean includeHeaderFooter = false;
/** Opening tag for strikethrough text*/
protected static final String strikethroughTextHtmlOpeningTag = "<del>";
/** Closing tag for strikethrough text*/
protected static final String strikethroughTextHtmlClosingTag = "</del>";;
/**
* Creates a new HtmlGenerator for the specified PDF file.
*
* @param inputPdf the input PDF file
* @param config the configuration settings
* @throws IOException if unable to create the output file
*/
public HtmlGenerator(File inputPdf, Config config) throws IOException {
this.pdfFileName = inputPdf.getName();
this.pdfFilePath = inputPdf.toPath().toAbsolutePath();
this.htmlFileName = pdfFileName.substring(0, pdfFileName.length() - 3) + "html";
this.htmlFilePath = Path.of(config.getOutputFolder(), htmlFileName);
this.htmlWriter = new FileWriter(htmlFilePath.toFile(), StandardCharsets.UTF_8);
this.htmlPageSeparator = escapeHtmlAttribute(config.getHtmlPageSeparator());
this.selectedPageNumbers = new HashSet<>(config.getPageNumbers());
this.embedImages = config.isEmbedImages();
this.imageFormat = config.getImageFormat();
this.includeHeaderFooter = config.isIncludeHeaderFooter();
}
/**
* Writes the document contents to HTML format.
*
* @param contents the document contents organized by page
*/
public void writeToHtml(List<List<IObject>> contents) {
try {
htmlWriter.write("<!DOCTYPE html>\n");
htmlWriter.write("<html lang=\"und\">\n<head>\n<meta charset=\"utf-8\">\n");
htmlWriter.write("<title>" + escapeHtmlText(pdfFileName) + "</title>\n");
htmlWriter.write("</head>\n<body>\n");
for (int pageNumber = 0; pageNumber < StaticContainers.getDocument().getNumberOfPages(); pageNumber++) {
if (selectedPageNumbers.isEmpty() || selectedPageNumbers.contains(pageNumber + 1)) {
writePageSeparator(pageNumber);
}
for (IObject content : contents.get(pageNumber)) {
this.write(content);
}
}
htmlWriter.write("\n</body>\n</html>");
LOGGER.log(Level.INFO, "Created {0}", htmlFilePath);
} catch (Exception e) {
LOGGER.log(Level.WARNING, "Unable to create html output: " + e.getMessage());
}
}
/**
* Writes a page separator to the HTML output if configured.
*
* @param pageNumber the current page number (0-indexed)
* @throws IOException if unable to write to the output
*/
protected void writePageSeparator(int pageNumber) throws IOException {
if (!htmlPageSeparator.isEmpty()) {
htmlWriter.write(htmlPageSeparator.contains(Config.PAGE_NUMBER_STRING)
? htmlPageSeparator.replace(Config.PAGE_NUMBER_STRING, String.valueOf(pageNumber + 1))
: htmlPageSeparator);
htmlWriter.write("\n");
}
}
/**
* Writes a single content object to the HTML output.
*
* @param object the content object to write
* @throws IOException if unable to write to the output
*/
protected void write(IObject object) throws IOException {
if (object instanceof SemanticHeaderOrFooter) {
if (includeHeaderFooter) {
writeHeaderOrFooter((SemanticHeaderOrFooter) object);
}
return;
} else if (object instanceof SemanticPicture) {
writePicture((SemanticPicture) object);
} else if (object instanceof ImageChunk) {
writeImage((ImageChunk) object);
} else if (object instanceof SemanticFormula) {
writeFormula((SemanticFormula) object);
} else if (object instanceof SemanticHeading) {
writeHeading((SemanticHeading) object);
} else if (object instanceof SemanticParagraph) {
writeParagraph((SemanticParagraph) object);
} else if (object instanceof SemanticTextNode) {
writeSemanticTextNode((SemanticTextNode) object);
} else if (object instanceof TableBorder) {
writeTable((TableBorder) object);
} else if (object instanceof PDFList) {
writeList((PDFList) object);
} else if (object instanceof SemanticTOC) {
writeTOC((SemanticTOC) object);
} else {
return;
}
if (!isInsideTable()) {
htmlWriter.write(HtmlSyntax.HTML_LINE_BREAK);
}
}
/**
* Writes a header or footer element to the HTML output.
*
* @param headerOrFooter the header or footer to write
* @throws IOException if unable to write to the output
*/
protected void writeHeaderOrFooter(SemanticHeaderOrFooter headerOrFooter) throws IOException {
for (IObject content : headerOrFooter.getContents()) {
write(content);
}
}
/**
* Writes a formula element to the HTML output using MathJax-compatible markup.
*
* @param formula the formula to write
* @throws IOException if unable to write to the output
*/
protected void writeFormula(SemanticFormula formula) throws IOException {
htmlWriter.write(HtmlSyntax.HTML_MATH_DISPLAY_TAG);
htmlWriter.write("\\[");
htmlWriter.write(escapeHtmlText(formula.getLatex()));
htmlWriter.write("\\]");
htmlWriter.write(HtmlSyntax.HTML_MATH_DISPLAY_CLOSE_TAG);
htmlWriter.write(HtmlSyntax.HTML_LINE_BREAK);
}
/**
* Writes an image element to the HTML output.
*
* @param image the image chunk to write
*/
protected void writeImage(ImageChunk image) {
try {
String absolutePath = String.format(MarkdownSyntax.IMAGE_FILE_NAME_FORMAT, StaticLayoutContainers.getImagesDirectory(), File.separator, image.getIndex(), imageFormat);
String relativePath = String.format(MarkdownSyntax.IMAGE_FILE_NAME_FORMAT, StaticLayoutContainers.getImagesDirectoryName(), "/", image.getIndex(), imageFormat);
if (ImagesUtils.isImageFileExists(absolutePath)) {
String imageSource;
if (embedImages) {
File imageFile = new File(absolutePath);
imageSource = Base64ImageUtils.toDataUri(imageFile, imageFormat);
if (imageSource == null) {
LOGGER.log(Level.WARNING, "Failed to convert image to Base64: {0}", absolutePath);
}
} else {
imageSource = relativePath;
}
if (imageSource != null) {
String escapedSource = escapeHtmlAttribute(imageSource);
// Empty alt is correct HTML for "missing description": screen
// readers skip it, and our evidence-report flags it as
// alt_source="missing". Never synthesize "figureN".
String altText = (image instanceof EnrichedImageChunk && ((EnrichedImageChunk) image).hasDescription())
? ((EnrichedImageChunk) image).sanitizeDescription()
: "";
String imageString = String.format("<img src=\"%s\" alt=\"%s\">", escapedSource, escapeHtmlAttribute(altText));
htmlWriter.write(imageString);
htmlWriter.write(HtmlSyntax.HTML_LINE_BREAK);
}
}
} catch (IOException e) {
LOGGER.log(Level.WARNING, "Unable to write image for html output: " + e.getMessage());
}
}
/**
* Writes a SemanticPicture element with figure/figcaption for description.
*
* @param picture the picture to write
*/
protected void writePicture(SemanticPicture picture) {
try {
String absolutePath = String.format(MarkdownSyntax.IMAGE_FILE_NAME_FORMAT, StaticLayoutContainers.getImagesDirectory(), File.separator, picture.getPictureIndex(), imageFormat);
String relativePath = String.format(MarkdownSyntax.IMAGE_FILE_NAME_FORMAT, StaticLayoutContainers.getImagesDirectoryName(), "/", picture.getPictureIndex(), imageFormat);
if (ImagesUtils.isImageFileExists(absolutePath)) {
String imageSource;
if (embedImages) {
File imageFile = new File(absolutePath);
imageSource = Base64ImageUtils.toDataUri(imageFile, imageFormat);
if (imageSource == null) {
LOGGER.log(Level.WARNING, "Failed to convert image to Base64: {0}", absolutePath);
}
} else {
imageSource = relativePath;
}
if (imageSource != null) {
String altText = picture.hasDescription()
? picture.sanitizeDescription()
: "";
String escapedSource = escapeHtmlAttribute(imageSource);
htmlWriter.write(HtmlSyntax.HTML_FIGURE_TAG);
htmlWriter.write(HtmlSyntax.HTML_LINE_BREAK);
String imageString = String.format("<img src=\"%s\" alt=\"%s\">", escapedSource, escapeHtmlAttribute(altText));
htmlWriter.write(imageString);
htmlWriter.write(HtmlSyntax.HTML_LINE_BREAK);
htmlWriter.write(HtmlSyntax.HTML_FIGURE_CLOSE_TAG);
htmlWriter.write(HtmlSyntax.HTML_LINE_BREAK);
}
}
} catch (IOException e) {
LOGGER.log(Level.WARNING, "Unable to write picture for html output: " + e.getMessage());
}
}
/**
* Writes a list element to the HTML output.
*
* @param list the PDF list to write
* @throws IOException if unable to write to the output
*/
protected void writeList(PDFList list) throws IOException {
htmlWriter.write(HtmlSyntax.HTML_UNORDERED_LIST_TAG);
htmlWriter.write(HtmlSyntax.HTML_LINE_BREAK);
for (ListItem item : list.getListItems()) {
htmlWriter.write(HtmlSyntax.HTML_LIST_ITEM_TAG);
htmlWriter.write(HtmlSyntax.HTML_PARAGRAPH_TAG);
String value = GeneratorUtils.getTextFromLines(item.getLines(), OutputType.HTML);
htmlWriter.write(value);
htmlWriter.write(HtmlSyntax.HTML_PARAGRAPH_CLOSE_TAG);
for (IObject object : item.getContents()) {
write(object);
}
htmlWriter.write(HtmlSyntax.HTML_LIST_ITEM_CLOSE_TAG);
htmlWriter.write(HtmlSyntax.HTML_LINE_BREAK);
}
htmlWriter.write(HtmlSyntax.HTML_UNORDERED_LIST_CLOSE_TAG);
htmlWriter.write(HtmlSyntax.HTML_LINE_BREAK);
}
protected void writeTOC(SemanticTOC toc) throws IOException {
htmlWriter.write(HtmlSyntax.HTML_UNORDERED_LIST_TAG);
htmlWriter.write(HtmlSyntax.HTML_LINE_BREAK);
for (IObject item : toc.getTOCItems()) {
if (item instanceof SemanticTOC) {
htmlWriter.write(HtmlSyntax.HTML_LIST_ITEM_TAG);
htmlWriter.write(HtmlSyntax.HTML_LINE_BREAK);
writeTOC((SemanticTOC) item);
htmlWriter.write(HtmlSyntax.HTML_LIST_ITEM_CLOSE_TAG);
} else if (item instanceof SemanticTOCI) {
SemanticTOCI tocItem = (SemanticTOCI) item;
htmlWriter.write(HtmlSyntax.HTML_LIST_ITEM_TAG);
htmlWriter.write(HtmlSyntax.HTML_PARAGRAPH_TAG);
String value = GeneratorUtils.getTextFromLines(tocItem.getLines(), OutputType.HTML);
htmlWriter.write(value);
htmlWriter.write(HtmlSyntax.HTML_PARAGRAPH_CLOSE_TAG);
for (IObject object : tocItem.getContents()) {
write(object);
}
htmlWriter.write(HtmlSyntax.HTML_LIST_ITEM_CLOSE_TAG);
}
htmlWriter.write(HtmlSyntax.HTML_LINE_BREAK);
}
htmlWriter.write(HtmlSyntax.HTML_UNORDERED_LIST_CLOSE_TAG);
htmlWriter.write(HtmlSyntax.HTML_LINE_BREAK);
}
/**
* Writes a semantic text node as a figure caption to the HTML output.
*
* @param textNode the text node to write
* @throws IOException if unable to write to the output
*/
protected void writeSemanticTextNode(SemanticTextNode textNode) throws IOException {
htmlWriter.write(HtmlSyntax.HTML_FIGURE_CAPTION_TAG);
htmlWriter.write(GeneratorUtils.getTextFromTextNode(textNode, OutputType.HTML));
htmlWriter.write(HtmlSyntax.HTML_FIGURE_CAPTION_CLOSE_TAG);
htmlWriter.write(HtmlSyntax.HTML_LINE_BREAK);
}
/**
* Writes a table element to the HTML output.
*
* @param table the table border to write
* @throws IOException if unable to write to the output
*/
protected void writeTable(TableBorder table) throws IOException {
enterTable();
htmlWriter.write(HtmlSyntax.HTML_TABLE_TAG);
htmlWriter.write(HtmlSyntax.HTML_LINE_BREAK);
for (int rowNumber = 0; rowNumber < table.getNumberOfRows(); rowNumber++) {
TableBorderRow row = table.getRow(rowNumber);
htmlWriter.write(HtmlSyntax.HTML_TABLE_ROW_TAG);
htmlWriter.write(HtmlSyntax.HTML_LINE_BREAK);
for (int colNumber = 0; colNumber < table.getNumberOfColumns(); colNumber++) {
TableBorderCell cell = row.getCell(colNumber);
if (cell.getRowNumber() == rowNumber && cell.getColNumber() == colNumber) {
writeCellTag(cell);
List<IObject> contents = cell.getContents();
if (!contents.isEmpty()) {
for (IObject contentItem : contents) {
this.write(contentItem);
}
}
if (cell.isHeaderCell()) {
htmlWriter.write(HtmlSyntax.HTML_TABLE_HEADER_CLOSE_TAG);
} else {
htmlWriter.write(HtmlSyntax.HTML_TABLE_CELL_CLOSE_TAG);
}
htmlWriter.write(HtmlSyntax.HTML_LINE_BREAK);
}
}
htmlWriter.write(HtmlSyntax.HTML_TABLE_ROW_CLOSE_TAG);
htmlWriter.write(HtmlSyntax.HTML_LINE_BREAK);
}
htmlWriter.write(HtmlSyntax.HTML_TABLE_CLOSE_TAG);
htmlWriter.write(HtmlSyntax.HTML_LINE_BREAK);
leaveTable();
}
/**
* Writes a paragraph element to the HTML output.
*
* @param paragraph the semantic paragraph to write
* @throws IOException if unable to write to the output
*/
protected void writeParagraph(SemanticParagraph paragraph) throws IOException {
double paragraphIndent = paragraph.getColumns().get(0).getBlocks().get(0).getFirstLineIndent();
htmlWriter.write(HtmlSyntax.HTML_PARAGRAPH_TAG);
if (paragraphIndent > 0) {
htmlWriter.write(HtmlSyntax.HTML_INDENT);
}
String paragraphValue = GeneratorUtils.getTextFromTextNode(paragraph, OutputType.HTML);
if (isInsideTable() && StaticContainers.isKeepLineBreaks()) {
paragraphValue = paragraphValue.replace(HtmlSyntax.HTML_LINE_BREAK, HtmlSyntax.HTML_LINE_BREAK_TAG);
}
htmlWriter.write(paragraphValue);
htmlWriter.write(HtmlSyntax.HTML_PARAGRAPH_CLOSE_TAG);
htmlWriter.write(HtmlSyntax.HTML_LINE_BREAK);
}
/**
* Writes a heading element to the HTML output.
*
* @param heading the semantic heading to write
* @throws IOException if unable to write to the output
*/
protected void writeHeading(SemanticHeading heading) throws IOException {
int headingLevel = Math.min(6, Math.max(1, heading.getHeadingLevel()));
htmlWriter.write("<h" + headingLevel + ">");
htmlWriter.write(GeneratorUtils.getTextFromTextNode(heading, OutputType.HTML));
htmlWriter.write("</h" + headingLevel + ">");
htmlWriter.write(HtmlSyntax.HTML_LINE_BREAK);
}
private void writeCellTag(TableBorderCell cell) throws IOException {
String tag = cell.isHeaderCell() ? "<th" : "<td";
StringBuilder cellTag = new StringBuilder(tag);
int colSpan = cell.getColSpan();
if (colSpan != 1) {
cellTag.append(" colspan=\"").append(colSpan).append("\"");
}
int rowSpan = cell.getRowSpan();
if (rowSpan != 1) {
cellTag.append(" rowspan=\"").append(rowSpan).append("\"");
}
cellTag.append(">");
htmlWriter.write(cellTag.toString());
}
/**
* Increments the table nesting level when entering a table.
*/
protected void enterTable() {
tableNesting++;
}
/**
* Decrements the table nesting level when leaving a table.
*/
protected void leaveTable() {
if (tableNesting > 0) {
tableNesting--;
}
}
/**
* Checks whether currently writing inside a table.
*
* @return true if inside a table, false otherwise
*/
protected boolean isInsideTable() {
return tableNesting > 0;
}
/**
* Escapes special characters for use in HTML attributes.
* Handles quotes, ampersands, less-than, greater-than, and newlines.
*
* @param value the string to escape
* @return the escaped string safe for HTML attribute values
*/
protected static String escapeHtmlAttribute(String value) {
if (value == null) {
return "";
}
value = escapeHtmlText(value);
return value
.replace("\"", "&quot;")
.replace("\n", " ")
.replace("\r", "");
}
protected static String escapeHtmlText(String value) {
if (value == null) {
return "";
}
return value
.replace("\u0000", "")
.replace("&", "&amp;")
.replace("<", "&lt;")
.replace(">", "&gt;");
}
public static void getTextFromLineForHTML(TextLine line, StringBuilder stringBuilder) {
for (TextChunk chunk : line.getTextChunks()) {
String style = getTextStyle(chunk);
if (!style.isEmpty()) {
String styleAttribute = String.format(HtmlSyntax.HTML_STYLE_ATTRIBUTE, style.trim());
stringBuilder.append(String.format(HtmlSyntax.HTML_SPAN_START_TAG, styleAttribute));
stringBuilder.append(escapeHtmlText(chunk.getValue()));
stringBuilder.append(HtmlSyntax.HTML_SPAN_CLOSE_TAG);
} else {
stringBuilder.append(escapeHtmlText(chunk.getValue()));
}
}
}
private static String getTextStyle(TextChunk chunk) {
StringBuilder style = new StringBuilder();
if (chunk.getIsStrikethroughText() || chunk.getIsUnderlinedText()) {
style.append(String.format(HtmlSyntax.HTML_TEXT_DECORATION_STYLE_PROPERTY,
(chunk.getIsStrikethroughText() ? HtmlSyntax.HTML_STRIKETHROUGH_VALUE : "") +
(chunk.getIsUnderlinedText() ? HtmlSyntax.HTML_UNDERLINE_VALUE : "")));
}
return style.toString();
}
@Override
public void close() throws IOException {
if (htmlWriter != null) {
htmlWriter.close();
}
}
}
java/opendataloader-pdf-core/src/main/java/org/opendataloader/pdf/html/HtmlGeneratorFactory.java
+39
View File
@@ -0,0 +1,39 @@
/*
* Copyright 2025-2026 Hancom Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.opendataloader.pdf.html;
import org.opendataloader.pdf.api.Config;
import java.io.File;
import java.io.IOException;
/**
* Factory class for creating HtmlGenerator instances.
*/
public class HtmlGeneratorFactory {
/**
* Creates a new HtmlGenerator for the specified PDF file.
*
* @param inputPdf the input PDF file
* @param config the configuration settings
* @return a new HtmlGenerator instance
* @throws IOException if unable to create the generator
*/
public static HtmlGenerator getHtmlGenerator(File inputPdf, Config config) throws IOException {
return new HtmlGenerator(inputPdf, config);
}
}
java/opendataloader-pdf-core/src/main/java/org/opendataloader/pdf/html/HtmlSyntax.java
+86
View File
@@ -0,0 +1,86 @@
/*
* Copyright 2025-2026 Hancom Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.opendataloader.pdf.html;
/**
* Constants for HTML syntax elements used in HTML output generation.
*/
public class HtmlSyntax {
/** Format string for image file names. */
public static final String IMAGE_FILE_NAME_FORMAT = "figure%d.png";
/** Line break character for HTML output. */
public static final String HTML_LINE_BREAK = "\n";
/** Opening table tag with border. */
public static final String HTML_TABLE_TAG = "<table border=\"1\">";
/** Closing table tag. */
public static final String HTML_TABLE_CLOSE_TAG = "</table>";
/** Opening table row tag. */
public static final String HTML_TABLE_ROW_TAG = "<tr>";
/** Closing table row tag. */
public static final String HTML_TABLE_ROW_CLOSE_TAG = "</tr>";
/** Opening table cell tag. */
public static final String HTML_TABLE_CELL_TAG = "<td>";
/** Closing table cell tag. */
public static final String HTML_TABLE_CELL_CLOSE_TAG = "</td>";
/** Opening table header cell tag. */
public static final String HTML_TABLE_HEADER_TAG = "<th>";
/** Closing table header cell tag. */
public static final String HTML_TABLE_HEADER_CLOSE_TAG = "</th>";
/** Opening ordered list tag. */
public static final String HTML_ORDERED_LIST_TAG = "<ol>";
/** Closing ordered list tag. */
public static final String HTML_ORDERED_LIST_CLOSE_TAG = "</ol>";
/** Opening unordered list tag. */
public static final String HTML_UNORDERED_LIST_TAG = "<ul>";
/** Closing unordered list tag. */
public static final String HTML_UNORDERED_LIST_CLOSE_TAG = "</ul>";
/** Opening list item tag. */
public static final String HTML_LIST_ITEM_TAG = "<li>";
/** Closing list item tag. */
public static final String HTML_LIST_ITEM_CLOSE_TAG = "</li>";
/** HTML line break tag. */
public static final String HTML_LINE_BREAK_TAG = "<br>";
/** Indentation string for paragraphs. */
public static final String HTML_INDENT = "";
/** Opening paragraph tag. */
public static final String HTML_PARAGRAPH_TAG = "<p>";
/** Closing paragraph tag. */
public static final String HTML_PARAGRAPH_CLOSE_TAG = "</p>";
/** Opening figure tag. */
public static final String HTML_FIGURE_TAG = "<figure>";
/** Closing figure tag. */
public static final String HTML_FIGURE_CLOSE_TAG = "</figure>";
/** Opening figure caption tag. */
public static final String HTML_FIGURE_CAPTION_TAG = "<figcaption>";
/** Closing figure caption tag. */
public static final String HTML_FIGURE_CAPTION_CLOSE_TAG = "</figcaption>";
/** Opening math display block tag for MathJax/KaTeX rendering. */
public static final String HTML_MATH_DISPLAY_TAG = "<div class=\"math-display\">";
/** Closing math display block tag. */
public static final String HTML_MATH_DISPLAY_CLOSE_TAG = "</div>";
/** Span opening tag. */
public static final String HTML_SPAN_START_TAG = "<span%s>";
/** Style attribute */
public static final String HTML_STYLE_ATTRIBUTE = " style=\"%s\"";
/** Span closing tag. */
public static final String HTML_SPAN_CLOSE_TAG = "</span>";
/** Text decoration property. */
public static final String HTML_TEXT_DECORATION_STYLE_PROPERTY = "text-decoration:%s; ";
/** Strikethrough text property. */
public static final String HTML_STRIKETHROUGH_VALUE = " line-through";
/** Underline text property. */
public static final String HTML_UNDERLINE_VALUE = " underline";
}
java/opendataloader-pdf-core/src/main/java/org/opendataloader/pdf/hybrid/DiskPageImageCache.java
+97
View File
@@ -0,0 +1,97 @@
/*
* Copyright 2025-2026 Hancom Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.opendataloader.pdf.hybrid;
import javax.imageio.ImageIO;
import java.awt.image.BufferedImage;
import java.io.IOException;
import java.nio.file.DirectoryStream;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.logging.Level;
import java.util.logging.Logger;
/**
* Disk-backed page image cache. Stores page images as PNG files in a
* temporary directory. evict() is a no-op (keeps files for potential re-read).
* close() deletes the temp directory and all files.
*/
public class DiskPageImageCache implements PageImageCache {
private static final Logger LOGGER = Logger.getLogger(DiskPageImageCache.class.getCanonicalName());
private final Path tempDir;
public DiskPageImageCache() throws IOException {
this.tempDir = Files.createTempDirectory("odl-pages-");
}
// Visible for testing
DiskPageImageCache(Path tempDir) {
this.tempDir = tempDir;
}
@Override
public BufferedImage getOrFetch(int pageIndex, PageImageFetcher fetcher) throws IOException {
Path file = tempDir.resolve("page-" + pageIndex + ".png");
if (Files.exists(file)) {
BufferedImage cached = ImageIO.read(file.toFile());
if (cached != null) {
return cached;
}
// Cached file is unreadable (corrupt or no ImageReader) — re-fetch.
LOGGER.log(Level.WARNING, "Cached page image is unreadable, re-fetching: {0}", file);
Files.deleteIfExists(file);
}
BufferedImage image = fetcher.fetch(pageIndex);
if (image == null) {
throw new IOException("Page image fetcher returned null for page " + pageIndex);
}
if (!ImageIO.write(image, "png", file.toFile())) {
throw new IOException("No ImageIO writer accepted PNG output for page " + pageIndex);
}
return image;
}
@Override
public void evict(int pageIndex) {
// no-op: keep on disk for potential re-read
}
@Override
public void close() throws IOException {
if (!Files.exists(tempDir)) {
return;
}
try (DirectoryStream<Path> stream = Files.newDirectoryStream(tempDir)) {
for (Path entry : stream) {
try {
Files.deleteIfExists(entry);
} catch (IOException e) {
LOGGER.log(Level.WARNING, "Failed to delete temp file: {0}", entry);
}
}
}
Files.deleteIfExists(tempDir);
}
/**
* Returns the temp directory path (for testing).
*/
Path getTempDir() {
return tempDir;
}
}
java/opendataloader-pdf-core/src/main/java/org/opendataloader/pdf/hybrid/DoclingFastServerClient.java
+315
View File
@@ -0,0 +1,315 @@
/*
* Copyright 2025-2026 Hancom Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.opendataloader.pdf.hybrid;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import okhttp3.MediaType;
import okhttp3.MultipartBody;
import okhttp3.OkHttpClient;
import okhttp3.Request;
import okhttp3.RequestBody;
import okhttp3.Response;
import okhttp3.ResponseBody;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.TimeUnit;
import java.util.logging.Level;
import java.util.logging.Logger;
/**
* HTTP client for docling-fast-server API.
*
* <p>This client communicates with the optimized FastAPI server (opendataloader-pdf-hybrid)
* which provides 3.3x faster performance than docling-serve by using a DocumentConverter
* singleton pattern.
*
* <p>The API is compatible with docling-serve, using the same /v1/convert/file endpoint
* and response format.
*
* @see HybridClient
* @see HybridConfig
*/
public class DoclingFastServerClient implements HybridClient {
private static final Logger LOGGER = Logger.getLogger(DoclingFastServerClient.class.getCanonicalName());
/** Default URL for docling-fast-server. */
public static final String DEFAULT_URL = "http://localhost:5002";
private static final String CONVERT_ENDPOINT = "/v1/convert/file";
private static final String HEALTH_ENDPOINT = "/health";
private static final int HEALTH_CHECK_TIMEOUT_MS = 3000;
private static final String DEFAULT_FILENAME = "document.pdf";
private static final MediaType MEDIA_TYPE_PDF = MediaType.parse("application/pdf");
private final String baseUrl;
private final OkHttpClient httpClient;
private final ObjectMapper objectMapper;
/**
* Creates a new DoclingFastServerClient with the specified configuration.
*
* @param config The hybrid configuration containing URL and timeout settings.
*/
public DoclingFastServerClient(HybridConfig config) {
this.baseUrl = config.getEffectiveUrl("docling-fast");
this.objectMapper = new ObjectMapper();
this.httpClient = new OkHttpClient.Builder()
.connectTimeout(config.getTimeoutMs(), TimeUnit.MILLISECONDS)
.readTimeout(config.getTimeoutMs(), TimeUnit.MILLISECONDS)
.writeTimeout(config.getTimeoutMs(), TimeUnit.MILLISECONDS)
.build();
}
/**
* Creates a new DoclingFastServerClient with a custom OkHttpClient (for testing).
*
* @param baseUrl The base URL of the docling-fast-server instance.
* @param httpClient The OkHttp client to use for requests.
* @param objectMapper The Jackson ObjectMapper for JSON parsing.
*/
DoclingFastServerClient(String baseUrl, OkHttpClient httpClient, ObjectMapper objectMapper) {
this.baseUrl = baseUrl;
this.httpClient = httpClient;
this.objectMapper = objectMapper;
}
@Override
public void checkAvailability() throws IOException {
OkHttpClient healthClient = httpClient.newBuilder()
.connectTimeout(HEALTH_CHECK_TIMEOUT_MS, TimeUnit.MILLISECONDS)
.readTimeout(HEALTH_CHECK_TIMEOUT_MS, TimeUnit.MILLISECONDS)
.build();
Request healthRequest = new Request.Builder()
.url(baseUrl + HEALTH_ENDPOINT)
.get()
.build();
Response response;
try {
response = healthClient.newCall(healthRequest).execute();
} catch (IOException e) {
throw new IOException(
"Hybrid server is not available at " + baseUrl + "\n"
+ "To start the local hybrid server:\n"
+ " 1. Install: pip install \"opendataloader-pdf[hybrid]\"\n"
+ " 2. Start: opendataloader-pdf-hybrid --port 5002\n"
+ "To use a remote server or custom port: --hybrid-url http://host:port\n"
+ "Or pass --hybrid-fallback to fall back to Java-only output for this run.\n"
+ "Or run without --hybrid flag for Java-only processing.", e);
}
try (response) {
if (!response.isSuccessful()) {
throw new IOException(
"Hybrid server at " + baseUrl + " returned HTTP " + response.code()
+ " during health check.\n"
+ "The server is reachable but may be starting up or unhealthy.");
}
}
}
@Override
public HybridResponse convert(HybridRequest request) throws IOException {
Request httpRequest = buildConvertRequest(request);
LOGGER.log(Level.FINE, "Sending request to {0}", baseUrl + CONVERT_ENDPOINT);
try (Response response = httpClient.newCall(httpRequest).execute()) {
return parseResponse(response);
}
}
@Override
public CompletableFuture<HybridResponse> convertAsync(HybridRequest request) {
return CompletableFuture.supplyAsync(() -> {
try {
return convert(request);
} catch (IOException e) {
throw new IllegalStateException("Failed to convert", e);
}
});
}
/**
* Gets the base URL of this client.
*
* @return The base URL.
*/
public String getBaseUrl() {
return baseUrl;
}
/**
* Builds a multipart/form-data HTTP request for the convert endpoint.
*/
private Request buildConvertRequest(HybridRequest request) {
MultipartBody.Builder bodyBuilder = new MultipartBody.Builder()
.setType(MultipartBody.FORM)
.addFormDataPart("files", DEFAULT_FILENAME,
RequestBody.create(request.getPdfBytes(), MEDIA_TYPE_PDF));
// Add page range if specified
if (request.getPageNumbers() != null && !request.getPageNumbers().isEmpty()) {
int minPage = request.getPageNumbers().stream().min(Integer::compareTo).orElse(1);
int maxPage = request.getPageNumbers().stream().max(Integer::compareTo).orElse(Integer.MAX_VALUE);
bodyBuilder.addFormDataPart("page_ranges", minPage + "-" + maxPage);
}
return new Request.Builder()
.url(baseUrl + CONVERT_ENDPOINT)
.post(bodyBuilder.build())
.build();
}
/**
* Parses the HTTP response into a HybridResponse.
*/
private HybridResponse parseResponse(Response response) throws IOException {
if (!response.isSuccessful()) {
ResponseBody body = response.body();
String bodyStr = body != null ? body.string() : "";
throw new IOException("Docling Fast Server request failed with status " + response.code() +
": " + bodyStr);
}
ResponseBody body = response.body();
if (body == null) {
throw new IOException("Empty response body");
}
String responseStr = body.string();
JsonNode root = objectMapper.readTree(responseStr);
// Check for API error status
JsonNode statusNode = root.get("status");
String status = statusNode != null ? statusNode.asText() : "";
if ("failure".equals(status)) {
JsonNode errorsNode = root.get("errors");
String errorMessage = errorsNode != null ? errorsNode.toString() : "Unknown error";
throw new IOException("Docling Fast Server processing failed: " + errorMessage);
}
// Log partial_success status
if ("partial_success".equals(status)) {
JsonNode errorsNode = root.get("errors");
LOGGER.log(Level.WARNING, "Backend returned partial_success: {0}",
errorsNode != null ? errorsNode.toString() : "no error details");
}
// Extract document content
JsonNode documentNode = root.get("document");
if (documentNode == null) {
throw new IOException("Invalid response: missing 'document' field");
}
JsonNode jsonContent = documentNode.get("json_content");
// Extract per-page content from json_content if available
Map<Integer, JsonNode> pageContents = extractPageContents(jsonContent);
// Extract failed pages (1-indexed) from partial_success responses
List<Integer> failedPages = extractFailedPages(root);
// Extract per-step pipeline timings (layout, ocr, table_structure, etc.)
JsonNode timingsNode = root.get("timings");
return new HybridResponse(null, null, jsonContent, pageContents, failedPages, timingsNode);
}
/**
* Extracts per-page content from the DoclingDocument JSON structure.
*
* <p>The DoclingDocument stores page information in the "pages" object,
* keyed by page number (as string). This method extracts the content
* elements for each page based on the "prov" (provenance) information.
*/
private Map<Integer, JsonNode> extractPageContents(JsonNode jsonContent) {
Map<Integer, JsonNode> pageContents = new HashMap<>();
if (jsonContent == null) {
return pageContents;
}
// The pages node contains page metadata keyed by page number
JsonNode pagesNode = jsonContent.get("pages");
if (pagesNode != null && pagesNode.isObject()) {
Iterator<String> fieldNames = pagesNode.fieldNames();
while (fieldNames.hasNext()) {
String pageNumStr = fieldNames.next();
try {
int pageNum = Integer.parseInt(pageNumStr);
pageContents.put(pageNum, pagesNode.get(pageNumStr));
} catch (NumberFormatException ignored) {
// Skip non-numeric page keys
}
}
}
return pageContents;
}
/**
* Extracts the list of failed page numbers from the response.
*
* <p>When the backend returns partial_success, the failed_pages array contains
* 1-indexed page numbers that failed during processing (e.g., due to Invalid code point
* errors in PDF font encoding).
*/
private List<Integer> extractFailedPages(JsonNode root) {
JsonNode failedPagesNode = root.get("failed_pages");
if (failedPagesNode == null || !failedPagesNode.isArray() || failedPagesNode.isEmpty()) {
return Collections.emptyList();
}
List<Integer> failedPages = new ArrayList<>();
for (JsonNode pageNode : failedPagesNode) {
if (pageNode.isNumber() && pageNode.canConvertToInt()) {
failedPages.add(pageNode.asInt());
}
}
return failedPages;
}
/**
* Shuts down the HTTP client and releases all resources.
*
* <p>This gracefully shuts down the dispatcher's executor service,
* allowing the JVM to exit cleanly. Idle connections are evicted
* from the connection pool.
*/
public void shutdown() {
// Gracefully shutdown the dispatcher - allows pending requests to complete
httpClient.dispatcher().executorService().shutdown();
// Evict idle connections from pool (does not affect the server)
httpClient.connectionPool().evictAll();
// Close the cache if present
if (httpClient.cache() != null) {
try {
httpClient.cache().close();
} catch (Exception ignored) {
// Ignore cache close errors
}
}
}
}
java/opendataloader-pdf-core/src/main/java/org/opendataloader/pdf/hybrid/DoclingSchemaTransformer.java
+621
View File
@@ -0,0 +1,621 @@
/*
* Copyright 2025-2026 Hancom Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.opendataloader.pdf.hybrid;
import com.fasterxml.jackson.databind.JsonNode;
import org.opendataloader.pdf.containers.StaticLayoutContainers;
import org.opendataloader.pdf.entities.SemanticFormula;
import org.opendataloader.pdf.entities.SemanticPicture;
import org.opendataloader.pdf.hybrid.HybridClient.HybridResponse;
import org.verapdf.wcag.algorithms.entities.IObject;
import org.verapdf.wcag.algorithms.entities.SemanticHeading;
import org.verapdf.wcag.algorithms.entities.SemanticParagraph;
import org.verapdf.wcag.algorithms.entities.content.TextChunk;
import org.verapdf.wcag.algorithms.entities.content.TextLine;
import org.verapdf.wcag.algorithms.entities.enums.SemanticType;
import org.verapdf.wcag.algorithms.entities.geometry.BoundingBox;
import org.verapdf.wcag.algorithms.entities.tables.tableBorders.TableBorder;
import org.verapdf.wcag.algorithms.entities.tables.tableBorders.TableBorderCell;
import org.verapdf.wcag.algorithms.entities.tables.tableBorders.TableBorderRow;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Comparator;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.logging.Level;
import java.util.logging.Logger;
/**
* Transforms Docling JSON output to OpenDataLoader IObject hierarchy.
*
* <p>This transformer handles the DoclingDocument JSON format and converts
* its elements (texts, tables, pictures) to the equivalent IObject types
* used by OpenDataLoader's downstream processors and generators.
*
* <h2>Schema Mapping</h2>
* <ul>
* <li>texts (label: text) → SemanticParagraph</li>
* <li>texts (label: section_header) → SemanticHeading</li>
* <li>texts (label: caption, footnote) → SemanticParagraph</li>
* <li>texts (label: page_header, page_footer) → Filtered out (furniture)</li>
* <li>tables → TableBorder with rows and cells</li>
* <li>pictures → SemanticPicture (with optional description)</li>
* </ul>
*
* <h2>Coordinate System</h2>
* <p>Both Docling and OpenDataLoader use BOTTOMLEFT origin. Docling provides
* bbox as {l, t, r, b} while OpenDataLoader uses [left, bottom, right, top].
* When Docling uses TOPLEFT origin, coordinates are converted appropriately.
*
* <h2>Thread Safety</h2>
* <p>This class is NOT thread-safe. The {@code transform()} method updates
* internal state (pictureIndex) during each call. Concurrent calls
* to transform() on the same instance may produce incorrect results.
* Use separate instances for concurrent transformations.
*/
public class DoclingSchemaTransformer implements HybridSchemaTransformer {
private static final Logger LOGGER = Logger.getLogger(DoclingSchemaTransformer.class.getCanonicalName());
private static final String BACKEND_TYPE = "docling";
// Picture index counter — accumulates across transform() calls on the same instance
// to ensure document-unique indices when processing chunked responses (#352).
private int pictureIndex;
// Docling text labels
private static final String LABEL_TEXT = "text";
private static final String LABEL_SECTION_HEADER = "section_header";
private static final String LABEL_CAPTION = "caption";
private static final String LABEL_FOOTNOTE = "footnote";
private static final String LABEL_PAGE_HEADER = "page_header";
private static final String LABEL_PAGE_FOOTER = "page_footer";
private static final String LABEL_LIST_ITEM = "list_item";
private static final String LABEL_FORMULA = "formula";
// Docling coordinate origins
private static final String COORD_ORIGIN_BOTTOMLEFT = "BOTTOMLEFT";
private static final String COORD_ORIGIN_TOPLEFT = "TOPLEFT";
@Override
public String getBackendType() {
return BACKEND_TYPE;
}
@Override
public List<List<IObject>> transform(HybridResponse response, Map<Integer, Double> pageHeights) {
JsonNode json = response.getJson();
if (json == null) {
LOGGER.log(Level.WARNING, "HybridResponse JSON is null, returning empty result");
return Collections.emptyList();
}
// Note: pictureIndex is NOT reset here — it must accumulate across
// multiple transform() calls when processing chunked responses (#352).
// Each transformer instance starts with pictureIndex=0 (field default),
// so single-call usage is unaffected.
// Determine number of pages from page info or content
int numPages = determinePageCount(json, pageHeights);
// Initialize result list
List<List<IObject>> result = new ArrayList<>(numPages);
for (int i = 0; i < numPages; i++) {
result.add(new ArrayList<>());
}
// Transform texts
JsonNode texts = json.get("texts");
if (texts != null && texts.isArray()) {
for (JsonNode textNode : texts) {
transformText(textNode, result, pageHeights);
}
}
// Transform tables
JsonNode tables = json.get("tables");
if (tables != null && tables.isArray()) {
for (JsonNode tableNode : tables) {
transformTable(tableNode, result, pageHeights);
}
}
// Transform pictures
JsonNode pictures = json.get("pictures");
if (pictures != null && pictures.isArray()) {
for (JsonNode pictureNode : pictures) {
transformPicture(pictureNode, result, pageHeights);
}
}
// Sort each page's contents by reading order (top to bottom, left to right)
for (List<IObject> pageContents : result) {
sortByReadingOrder(pageContents);
}
return result;
}
@Override
public List<IObject> transformPage(int pageNumber, JsonNode pageContent, double pageHeight) {
Map<Integer, Double> pageHeights = new HashMap<>();
pageHeights.put(pageNumber, pageHeight);
// Create a wrapper response with just this page's content
HybridResponse singlePageResponse = new HybridResponse("", pageContent, Collections.emptyMap());
List<List<IObject>> result = transform(singlePageResponse, pageHeights);
if (result.isEmpty()) {
return Collections.emptyList();
}
// Find the page in the result
int pageIndex = pageNumber - 1;
if (pageIndex >= 0 && pageIndex < result.size()) {
return result.get(pageIndex);
}
return Collections.emptyList();
}
/**
* Determines the number of pages from the JSON response.
*/
private int determinePageCount(JsonNode json, Map<Integer, Double> pageHeights) {
// First check pageHeights if provided
if (pageHeights != null && !pageHeights.isEmpty()) {
return pageHeights.keySet().stream().mapToInt(Integer::intValue).max().orElse(0);
}
// Check pages array in JSON
JsonNode pages = json.get("pages");
if (pages != null && pages.isArray()) {
return pages.size();
}
// Check page_dimensions or similar
JsonNode pageDimensions = json.get("page_dimensions");
if (pageDimensions != null && pageDimensions.isObject()) {
int maxPage = 0;
Iterator<String> fieldNames = pageDimensions.fieldNames();
while (fieldNames.hasNext()) {
try {
int pageNum = Integer.parseInt(fieldNames.next());
maxPage = Math.max(maxPage, pageNum);
} catch (NumberFormatException e) {
// ignore
}
}
return maxPage;
}
// Default to scanning content
return scanContentForPageCount(json);
}
/**
* Scans content elements to determine page count.
*/
private int scanContentForPageCount(JsonNode json) {
int maxPage = 0;
JsonNode texts = json.get("texts");
if (texts != null && texts.isArray()) {
for (JsonNode text : texts) {
maxPage = Math.max(maxPage, getPageNumberFromProv(text));
}
}
JsonNode tables = json.get("tables");
if (tables != null && tables.isArray()) {
for (JsonNode table : tables) {
maxPage = Math.max(maxPage, getPageNumberFromProv(table));
}
}
return maxPage;
}
/**
* Extracts page number from provenance info.
*/
private int getPageNumberFromProv(JsonNode node) {
JsonNode prov = node.get("prov");
if (prov != null && prov.isArray() && prov.size() > 0) {
JsonNode firstProv = prov.get(0);
JsonNode pageNo = firstProv.get("page_no");
if (pageNo != null && pageNo.isInt()) {
return pageNo.asInt();
}
}
return 0;
}
/**
* Transforms a Docling text element to an IObject.
*/
private void transformText(JsonNode textNode, List<List<IObject>> result, Map<Integer, Double> pageHeights) {
String label = getTextValue(textNode, "label");
// Skip furniture elements (page headers/footers)
if (LABEL_PAGE_HEADER.equals(label) || LABEL_PAGE_FOOTER.equals(label)) {
return;
}
// Get provenance for position info
JsonNode prov = textNode.get("prov");
if (prov == null || !prov.isArray() || prov.size() == 0) {
LOGGER.log(Level.FINE, "Text element missing provenance, skipping");
return;
}
JsonNode firstProv = prov.get(0);
int pageNo = firstProv.has("page_no") ? firstProv.get("page_no").asInt() : 1;
int pageIndex = pageNo - 1;
// Ensure result list is large enough
while (result.size() <= pageIndex) {
result.add(new ArrayList<>());
}
// Get bounding box
BoundingBox bbox = extractBoundingBox(firstProv.get("bbox"), pageIndex, pageHeights.get(pageNo));
// Get text content
String text = getTextValue(textNode, "text");
if (text == null || text.isEmpty()) {
text = getTextValue(textNode, "orig");
}
// Create appropriate IObject based on label
IObject object;
if (LABEL_SECTION_HEADER.equals(label)) {
object = createHeading(text, bbox, textNode);
} else if (LABEL_FORMULA.equals(label)) {
object = createFormula(text, bbox);
} else {
object = createParagraph(text, bbox);
}
if (object != null) {
result.get(pageIndex).add(object);
}
}
/**
* Creates a SemanticHeading from Docling section_header.
*/
private SemanticHeading createHeading(String text, BoundingBox bbox, JsonNode textNode) {
int level = 1; // Default level
// Try to extract level from node metadata
JsonNode meta = textNode.get("meta");
if (meta != null && meta.has("level")) {
level = meta.get("level").asInt(1);
}
// Create a text chunk and wrap in TextLine
TextChunk textChunk = new TextChunk(bbox, text, 12.0, 12.0);
textChunk.adjustSymbolEndsToBoundingBox(null);
TextLine textLine = new TextLine(textChunk);
// Create heading using default constructor and add content
SemanticHeading heading = new SemanticHeading();
heading.add(textLine);
heading.setRecognizedStructureId(StaticLayoutContainers.incrementContentId());
heading.setHeadingLevel(level);
return heading;
}
/**
* Creates a SemanticParagraph from Docling text element.
*/
private SemanticParagraph createParagraph(String text, BoundingBox bbox) {
// Create a text chunk and wrap in TextLine
TextChunk textChunk = new TextChunk(bbox, text, 12.0, 12.0);
textChunk.adjustSymbolEndsToBoundingBox(null);
TextLine textLine = new TextLine(textChunk);
// Create paragraph using default constructor and add content
SemanticParagraph paragraph = new SemanticParagraph();
paragraph.add(textLine);
paragraph.setRecognizedStructureId(StaticLayoutContainers.incrementContentId());
return paragraph;
}
/**
* Creates a SemanticFormula from Docling formula element.
*
* @param latex The LaTeX representation of the formula
* @param bbox The bounding box
* @return A SemanticFormula object
*/
private SemanticFormula createFormula(String latex, BoundingBox bbox) {
SemanticFormula formula = new SemanticFormula(bbox, latex);
formula.setRecognizedStructureId(StaticLayoutContainers.incrementContentId());
return formula;
}
/**
* Transforms a Docling picture element to a SemanticPicture.
*/
private void transformPicture(JsonNode pictureNode, List<List<IObject>> result, Map<Integer, Double> pageHeights) {
// Get provenance for position info
JsonNode prov = pictureNode.get("prov");
if (prov == null || !prov.isArray() || prov.size() == 0) {
LOGGER.log(Level.FINE, "Picture element missing provenance, skipping");
return;
}
JsonNode firstProv = prov.get(0);
int pageNo = firstProv.has("page_no") ? firstProv.get("page_no").asInt() : 1;
int pageIndex = pageNo - 1;
// Ensure result list is large enough
while (result.size() <= pageIndex) {
result.add(new ArrayList<>());
}
// Get bounding box
BoundingBox bbox = extractBoundingBox(firstProv.get("bbox"), pageIndex, pageHeights.get(pageNo));
// Extract description from annotations (if available)
String description = extractPictureDescription(pictureNode);
// Create SemanticPicture with description
SemanticPicture picture = new SemanticPicture(bbox, ++pictureIndex, description);
picture.setRecognizedStructureId(StaticLayoutContainers.incrementContentId());
result.get(pageIndex).add(picture);
}
/**
* Extracts picture description from annotations array.
*
* <p>Docling stores picture descriptions in the annotations array with kind="description".
*
* @param pictureNode The picture JSON node
* @return The description text, or null if not available
*/
private String extractPictureDescription(JsonNode pictureNode) {
JsonNode annotations = pictureNode.get("annotations");
if (annotations != null && annotations.isArray()) {
for (JsonNode annotation : annotations) {
String kind = getTextValue(annotation, "kind");
if ("description".equals(kind)) {
return getTextValue(annotation, "text");
}
}
}
return null;
}
/**
* Transforms a Docling table element to a TableBorder.
*/
private void transformTable(JsonNode tableNode, List<List<IObject>> result, Map<Integer, Double> pageHeights) {
// Get provenance for position info
JsonNode prov = tableNode.get("prov");
if (prov == null || !prov.isArray() || prov.size() == 0) {
LOGGER.log(Level.FINE, "Table element missing provenance, skipping");
return;
}
JsonNode firstProv = prov.get(0);
int pageNo = firstProv.has("page_no") ? firstProv.get("page_no").asInt() : 1;
int pageIndex = pageNo - 1;
// Ensure result list is large enough
while (result.size() <= pageIndex) {
result.add(new ArrayList<>());
}
// Get table data
JsonNode data = tableNode.get("data");
if (data == null) {
LOGGER.log(Level.FINE, "Table element missing data, skipping");
return;
}
// Get grid dimensions
JsonNode gridNode = data.get("grid");
if (gridNode == null || !gridNode.isArray()) {
LOGGER.log(Level.FINE, "Table missing grid data, skipping");
return;
}
int numRows = gridNode.size();
int numCols = 0;
if (numRows > 0 && gridNode.get(0).isArray()) {
numCols = gridNode.get(0).size();
}
if (numRows == 0 || numCols == 0) {
return;
}
// Get table bounding box
BoundingBox tableBbox = extractBoundingBox(firstProv.get("bbox"), pageIndex, pageHeights.get(pageNo));
// Create TableBorder
TableBorder table = new TableBorder(numRows, numCols);
table.setBoundingBox(tableBbox);
table.setRecognizedStructureId(StaticLayoutContainers.incrementContentId());
// Get table cells from data
JsonNode tableCells = data.get("table_cells");
Map<String, JsonNode> cellMap = new HashMap<>();
if (tableCells != null && tableCells.isArray()) {
for (JsonNode cell : tableCells) {
int startRow = cell.has("start_row_offset_idx") ? cell.get("start_row_offset_idx").asInt() : 0;
int startCol = cell.has("start_col_offset_idx") ? cell.get("start_col_offset_idx").asInt() : 0;
String key = startRow + "," + startCol;
cellMap.put(key, cell);
}
}
// Build table structure
double rowHeight = (tableBbox.getTopY() - tableBbox.getBottomY()) / numRows;
double colWidth = (tableBbox.getRightX() - tableBbox.getLeftX()) / numCols;
for (int row = 0; row < numRows; row++) {
TableBorderRow borderRow = new TableBorderRow(row, numCols, 0L);
double rowTop = tableBbox.getTopY() - (row * rowHeight);
double rowBottom = rowTop - rowHeight;
borderRow.setBoundingBox(new BoundingBox(pageIndex, tableBbox.getLeftX(), rowBottom, tableBbox.getRightX(), rowTop));
table.getRows()[row] = borderRow;
}
for (int row = 0; row < numRows; row++) {
for (int col = 0; col < numCols; col++) {
// Slot already covered by a spanning cell from an earlier
// iteration — do not overwrite with a fresh placeholder.
if (table.getRows()[row].getCells()[col] != null) continue;
String key = row + "," + col;
JsonNode cellNode = cellMap.get(key);
int rowSpan = 1;
int colSpan = 1;
String cellText = "";
if (cellNode != null) {
rowSpan = cellNode.has("row_span") ? cellNode.get("row_span").asInt(1) : 1;
colSpan = cellNode.has("col_span") ? cellNode.get("col_span").asInt(1) : 1;
cellText = getTextValue(cellNode, "text");
if (cellText == null) {
cellText = "";
}
}
// Clamp spans to the declared table bounds. Docling occasionally
// emits col_span/row_span values that, combined with the start
// index, run past the table; let the spanning area be exactly
// what stays inside the grid. Math.max(1, ...) defends against
// malformed zero or negative span values that would otherwise
// leave the slot unfilled.
int effectiveColSpan = Math.max(1, Math.min(colSpan, numCols - col));
int effectiveRowSpan = Math.max(1, Math.min(rowSpan, numRows - row));
TableBorderCell cell = new TableBorderCell(row, col, effectiveRowSpan, effectiveColSpan, 0L);
cell.setSemanticType(row == 0 ? SemanticType.TABLE_HEADER : SemanticType.TABLE_CELL);
double cellLeft = tableBbox.getLeftX() + (col * colWidth);
double cellRight = cellLeft + (effectiveColSpan * colWidth);
double cellTop = tableBbox.getTopY() - (row * rowHeight);
double cellBottom = cellTop - (effectiveRowSpan * rowHeight);
cell.setBoundingBox(new BoundingBox(pageIndex, cellLeft, cellBottom, cellRight, cellTop));
if (!cellText.isEmpty()) {
SemanticParagraph content = createParagraph(cellText, cell.getBoundingBox());
cell.addContentObject(content);
}
// Mark every slot covered by this cell's span with the same
// instance so AutoTaggingProcessor.addTableRow skips the
// duplicates. Without this Docling produces redundant
// 1x1 placeholders next to spanning cells, breaking
// PDF/UA-1 §7.2 (test 43) / PDF/UA-2 §8.2.5.26 "Table rows
// shall have the same number of columns (taking into account
// column spans)".
for (int r = row; r < row + effectiveRowSpan; r++) {
for (int c = col; c < col + effectiveColSpan; c++) {
table.getRows()[r].getCells()[c] = cell;
}
}
}
}
result.get(pageIndex).add(table);
}
/**
* Extracts a BoundingBox from Docling bbox JSON.
*
* @param bboxNode The bbox JSON node with l, t, r, b, coord_origin fields
* @param pageIndex The 0-indexed page number
* @param pageHeight The page height for coordinate transformation
* @return A BoundingBox in OpenDataLoader format
*/
private BoundingBox extractBoundingBox(JsonNode bboxNode, int pageIndex, Double pageHeight) {
if (bboxNode == null) {
return new BoundingBox(pageIndex, 0, 0, 0, 0);
}
double l = bboxNode.has("l") ? bboxNode.get("l").asDouble() : 0;
double t = bboxNode.has("t") ? bboxNode.get("t").asDouble() : 0;
double r = bboxNode.has("r") ? bboxNode.get("r").asDouble() : 0;
double b = bboxNode.has("b") ? bboxNode.get("b").asDouble() : 0;
String coordOrigin = bboxNode.has("coord_origin") ?
bboxNode.get("coord_origin").asText() : COORD_ORIGIN_BOTTOMLEFT;
double left, bottom, right, top;
if (COORD_ORIGIN_TOPLEFT.equals(coordOrigin) && pageHeight != null) {
// Convert from TOPLEFT to BOTTOMLEFT
// In TOPLEFT: t is distance from top, b is distance from top (t < b since t is higher)
// In BOTTOMLEFT: bottom is distance from bottom, top is distance from bottom
left = l;
right = r;
top = pageHeight - t; // t was distance from top
bottom = pageHeight - b; // b was distance from top
} else {
// BOTTOMLEFT origin - Docling uses {l, t, r, b} where t=top, b=bottom
left = l;
bottom = b;
right = r;
top = t;
}
return new BoundingBox(pageIndex, left, bottom, right, top);
}
/**
* Gets a text value from a JSON node.
*/
private String getTextValue(JsonNode node, String fieldName) {
if (node != null && node.has(fieldName)) {
JsonNode field = node.get(fieldName);
if (field.isTextual()) {
return field.asText();
}
}
return null;
}
/**
* Sorts page contents by reading order (top to bottom, left to right).
*/
private void sortByReadingOrder(List<IObject> contents) {
contents.sort(new Comparator<IObject>() {
@Override
public int compare(IObject o1, IObject o2) {
// Sort by top Y (descending - higher on page first)
double topDiff = o2.getTopY() - o1.getTopY();
if (Math.abs(topDiff) > 5.0) { // Use tolerance for same-line detection
return topDiff > 0 ? 1 : -1;
}
// Same line, sort by left X (ascending)
return Double.compare(o1.getLeftX(), o2.getLeftX());
}
});
}
}

Some files were not shown because too many files have changed in this diff Show More