chore: import upstream snapshot with attribution
This commit is contained in:
wehub-resource-sync
@@ -0,0 +1,5 @@
|
||||
# Owner for all documents
|
||||
*.md @bdoubrov @hnc-sujicho
|
||||
|
||||
# Default owners for everything else in the repository
|
||||
* @MaximPlusov @LonelyMidoriya @hyunhee-jo @bundolee @hnc-jglee
|
||||
@@ -0,0 +1,33 @@
|
||||
---
|
||||
name: Bug report
|
||||
about: Report an issue
|
||||
title: ""
|
||||
labels: bug
|
||||
assignees: ""
|
||||
---
|
||||
|
||||
### Bug
|
||||
|
||||
<!-- Describe the buggy behavior you have observed. -->
|
||||
|
||||
...
|
||||
|
||||
### Steps to reproduce
|
||||
|
||||
<!-- Describe the sequence of steps for reproducing the bug. -->
|
||||
|
||||
...
|
||||
|
||||
### Version
|
||||
|
||||
<!-- Copy version. -->
|
||||
|
||||
...
|
||||
|
||||
### Java version
|
||||
|
||||
<!-- Copy the output of `java --version`. -->
|
||||
|
||||
...
|
||||
|
||||
<!-- ⚠️ ATTENTION: When sharing screenshots, attachments, or other data make sure not to include any sensitive information. -->
|
||||
@@ -0,0 +1 @@
|
||||
blank_issues_enabled: false
|
||||
@@ -0,0 +1,21 @@
|
||||
---
|
||||
name: Feature request
|
||||
about: Suggest an idea
|
||||
title: ""
|
||||
labels: enhancement
|
||||
assignees: ""
|
||||
---
|
||||
|
||||
### Requested feature
|
||||
|
||||
<!-- Describe the feature you have in mind and the user need it addresses. -->
|
||||
|
||||
...
|
||||
|
||||
### Alternatives
|
||||
|
||||
<!-- Describe any alternatives you have considered. -->
|
||||
|
||||
...
|
||||
|
||||
<!-- ⚠️ ATTENTION: When sharing screenshots, attachments, or other data make sure not to include any sensitive information. -->
|
||||
@@ -0,0 +1,15 @@
|
||||
---
|
||||
name: Question
|
||||
about: Ask a question
|
||||
title: ""
|
||||
labels: question
|
||||
assignees: ""
|
||||
---
|
||||
|
||||
### Question
|
||||
|
||||
<!-- Describe what you would like to achieve and which part you need help with. -->
|
||||
|
||||
...
|
||||
|
||||
<!-- ⚠️ ATTENTION: When sharing screenshots, attachments, or other data make sure not to include any sensitive information. -->
|
||||
@@ -0,0 +1,19 @@
|
||||
<!-- Thank you for your contribution! -->
|
||||
|
||||
<!-- STEPS TO FOLLOW:
|
||||
1. Add a description of the changes (frequently the same as the commit description)
|
||||
2. Enter the issue number next to "Resolves #" below (if there is no tracking issue resolved, **remove that section**)
|
||||
3. Make sure the PR title follows the **Commit Message Formatting**: https://www.conventionalcommits.org/en/v1.0.0/#summary.
|
||||
4. Follow the steps in the checklist below, starting with the **Commit Message Formatting**.
|
||||
-->
|
||||
|
||||
<!-- Uncomment this section with the issue number if an issue is being resolved
|
||||
**Issue resolved by this Pull Request:**
|
||||
Resolves #
|
||||
--->
|
||||
|
||||
**Checklist:**
|
||||
|
||||
- [ ] Documentation has been updated, if necessary.
|
||||
- [ ] Examples have been added, if necessary.
|
||||
- [ ] Tests have been added, if necessary.
|
||||
@@ -0,0 +1,24 @@
|
||||
# Security Policy
|
||||
|
||||
## Reporting a Vulnerability
|
||||
|
||||
If you think you've identified a security issue in the project repository, please DO NOT report the issue publicly via
|
||||
the GitHub issue tracker, etc.
|
||||
|
||||
Instead, send an email with as many details as possible. This is a private mailing list for the maintainers team.
|
||||
|
||||
Please do not create a public issue.
|
||||
|
||||
### Security Vulnerability Response
|
||||
|
||||
Each report is acknowledged and analyzed by the core maintainers within 3 working days.
|
||||
|
||||
Any vulnerability information shared with core maintainers stays within the project and will not be disseminated to
|
||||
other projects unless it is necessary to get the issue fixed.
|
||||
|
||||
After the initial reply to your report, the security team will keep you informed of the progress towards a fix and full
|
||||
announcement, and may ask for additional information or guidance.
|
||||
|
||||
## Security Alerts
|
||||
|
||||
We will send announcements of security vulnerabilities and steps to remediate on the project announcements.
|
||||
@@ -0,0 +1,71 @@
|
||||
version: 2
|
||||
updates:
|
||||
# Java — multi-module Maven (parent /java declares core/cli child modules; Dependabot auto-detects them)
|
||||
- package-ecosystem: "maven"
|
||||
directory: "/java"
|
||||
schedule:
|
||||
interval: "weekly"
|
||||
day: "monday"
|
||||
time: "09:00"
|
||||
timezone: "Asia/Seoul"
|
||||
open-pull-requests-limit: 10
|
||||
labels:
|
||||
- "dependencies"
|
||||
groups:
|
||||
maven-minor-patch:
|
||||
update-types:
|
||||
- "minor"
|
||||
- "patch"
|
||||
|
||||
# Node.js SDK (pnpm; Dependabot handles pnpm under the "npm" ecosystem).
|
||||
# Root package.json is a scripts-only workspace meta-file with no deps, so it is omitted.
|
||||
- package-ecosystem: "npm"
|
||||
directory: "/node/opendataloader-pdf"
|
||||
schedule:
|
||||
interval: "weekly"
|
||||
day: "monday"
|
||||
time: "09:00"
|
||||
timezone: "Asia/Seoul"
|
||||
open-pull-requests-limit: 10
|
||||
labels:
|
||||
- "dependencies"
|
||||
groups:
|
||||
npm-minor-patch:
|
||||
update-types:
|
||||
- "minor"
|
||||
- "patch"
|
||||
|
||||
# Python SDK + MCP server (example requirements.txt excluded to avoid noise)
|
||||
- package-ecosystem: "pip"
|
||||
directories:
|
||||
- "/python/opendataloader-pdf"
|
||||
- "/python/opendataloader-pdf-mcp"
|
||||
schedule:
|
||||
interval: "weekly"
|
||||
day: "monday"
|
||||
time: "09:00"
|
||||
timezone: "Asia/Seoul"
|
||||
open-pull-requests-limit: 10
|
||||
labels:
|
||||
- "dependencies"
|
||||
groups:
|
||||
pip-minor-patch:
|
||||
update-types:
|
||||
- "minor"
|
||||
- "patch"
|
||||
|
||||
# GitHub Actions workflows
|
||||
- package-ecosystem: "github-actions"
|
||||
directory: "/"
|
||||
schedule:
|
||||
interval: "weekly"
|
||||
day: "monday"
|
||||
time: "09:00"
|
||||
timezone: "Asia/Seoul"
|
||||
labels:
|
||||
- "dependencies"
|
||||
groups:
|
||||
github-actions:
|
||||
update-types:
|
||||
- "minor"
|
||||
- "patch"
|
||||
@@ -0,0 +1,167 @@
|
||||
name: Release
|
||||
|
||||
on:
|
||||
push:
|
||||
tags:
|
||||
- 'v*'
|
||||
workflow_dispatch:
|
||||
|
||||
jobs:
|
||||
release:
|
||||
runs-on: ubuntu-latest
|
||||
env:
|
||||
VERSION: '0.0.0'
|
||||
permissions:
|
||||
contents: write
|
||||
id-token: write
|
||||
|
||||
steps:
|
||||
# =================================================================
|
||||
# 1. SETUP
|
||||
# =================================================================
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v7
|
||||
|
||||
- name: Initialize VERSION
|
||||
run: |
|
||||
if [[ "${GITHUB_REF}" == refs/tags/v* ]]; then
|
||||
echo "VERSION=${GITHUB_REF_NAME#v}" >> $GITHUB_ENV
|
||||
else
|
||||
echo "VERSION=0.0.0" >> $GITHUB_ENV
|
||||
fi
|
||||
|
||||
- name: Set up Java
|
||||
uses: actions/setup-java@v5
|
||||
with:
|
||||
java-version: '21'
|
||||
distribution: 'temurin'
|
||||
cache: 'maven'
|
||||
server-id: central
|
||||
server-username: MAVEN_CENTRAL_USERNAME
|
||||
server-password: MAVEN_CENTRAL_PASSWORD
|
||||
gpg-private-key: ${{ secrets.MAVEN_GPG_KEY }}
|
||||
gpg-passphrase: ${{ secrets.MAVEN_GPG_PASSPHRASE }}
|
||||
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@v6
|
||||
with:
|
||||
python-version: '3.12'
|
||||
|
||||
- name: Install uv
|
||||
uses: astral-sh/setup-uv@v7
|
||||
|
||||
- name: Set up Node.js and pnpm
|
||||
uses: actions/setup-node@v6
|
||||
with:
|
||||
node-version: '20'
|
||||
registry-url: 'https://registry.npmjs.org'
|
||||
|
||||
- name: Install pnpm
|
||||
uses: pnpm/action-setup@v6
|
||||
with:
|
||||
version: 9
|
||||
|
||||
# =================================================================
|
||||
# 2. BUILD & TEST
|
||||
# =================================================================
|
||||
- name: Build and test all packages
|
||||
run: ./scripts/build-all.sh ${{ env.VERSION }}
|
||||
|
||||
# =================================================================
|
||||
# 3. VERIFY CLI OPTIONS (before deploy)
|
||||
# =================================================================
|
||||
- name: Install opendataloader-pdf CLI
|
||||
# --system installs into the setup-python interpreter; without it (or an
|
||||
# active venv) `uv pip install` aborts with "No virtual environment found".
|
||||
run: uv pip install --system ./python/opendataloader-pdf/dist/*.whl
|
||||
|
||||
- name: Run CLI verification
|
||||
# ci-verify.py writes its own markdown table to $GITHUB_STEP_SUMMARY,
|
||||
# so no separate summary step is needed here.
|
||||
run: python verification/ci-verify.py
|
||||
|
||||
- name: Upload verification report
|
||||
if: always()
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: verification-report-release
|
||||
path: verification/verification-report-ci.txt
|
||||
retention-days: 30
|
||||
|
||||
# =================================================================
|
||||
# 4. DEPLOY (only on tag push)
|
||||
# =================================================================
|
||||
- name: '[Java] Deploy to Maven Central'
|
||||
if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v')
|
||||
run: mvn -B -pl opendataloader-pdf-core deploy -P release
|
||||
working-directory: ./java
|
||||
env:
|
||||
MAVEN_CENTRAL_USERNAME: ${{ secrets.MAVEN_CENTRAL_USERNAME }}
|
||||
MAVEN_CENTRAL_PASSWORD: ${{ secrets.MAVEN_CENTRAL_PASSWORD }}
|
||||
MAVEN_GPG_KEY: ${{ secrets.MAVEN_GPG_KEY }}
|
||||
MAVEN_GPG_PASSPHRASE: ${{ secrets.MAVEN_GPG_PASSPHRASE }}
|
||||
|
||||
- name: '[Python] Publish to PyPI'
|
||||
if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v')
|
||||
uses: pypa/gh-action-pypi-publish@release/v1
|
||||
with:
|
||||
packages-dir: ./python/opendataloader-pdf/dist
|
||||
|
||||
- name: '[Node.js] Publish to npm'
|
||||
if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v')
|
||||
run: pnpm publish --no-git-checks
|
||||
working-directory: ./node/opendataloader-pdf
|
||||
env:
|
||||
NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }}
|
||||
|
||||
# =================================================================
|
||||
# 5. GITHUB RELEASE (only on tag push)
|
||||
# =================================================================
|
||||
- name: Package CLI as ZIP
|
||||
if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v')
|
||||
run: |
|
||||
cd java/opendataloader-pdf-cli/target
|
||||
mkdir -p release
|
||||
cp "opendataloader-pdf-cli-${{ env.VERSION }}.jar" release/
|
||||
cp ../../../README.md release/
|
||||
cp ../../../LICENSE release/
|
||||
cp ../../../NOTICE release/
|
||||
cp -r ../../../THIRD_PARTY release/
|
||||
cd release
|
||||
zip -r "../opendataloader-pdf-cli-${{ env.VERSION }}.zip" .
|
||||
cd ../..
|
||||
|
||||
- name: Create GitHub Release
|
||||
if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v')
|
||||
uses: softprops/action-gh-release@v3
|
||||
with:
|
||||
tag_name: ${{ github.ref_name }}
|
||||
name: Release ${{ github.ref_name }}
|
||||
generate_release_notes: true
|
||||
files: |
|
||||
java/opendataloader-pdf-cli/target/opendataloader-pdf-cli-${{ env.VERSION }}.zip
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
# =================================================================
|
||||
# 6. SYNC DOCS TO HOMEPAGE (only on tag push)
|
||||
# =================================================================
|
||||
- name: Generate reference docs
|
||||
if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v')
|
||||
run: |
|
||||
mkdir -p content/docs/reference
|
||||
node scripts/generate-options.mjs
|
||||
node scripts/generate-schema.mjs
|
||||
|
||||
- name: Push reference docs to homepage
|
||||
if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v')
|
||||
# Pinned to v1.7.3 for security - verify before updating
|
||||
uses: cpina/github-action-push-to-another-repository@55306faa4ed53b815ae49e564af8cfb359d32ae2 # v1.7.3
|
||||
with:
|
||||
source-directory: 'content/docs/reference'
|
||||
destination-github-username: 'opendataloader-project'
|
||||
destination-repository-name: 'opendataloader.org'
|
||||
target-directory: 'apps/v1/content/docs/reference'
|
||||
target-branch: main
|
||||
env:
|
||||
API_TOKEN_GITHUB: ${{ secrets.HOMEPAGE_SYNC_TOKEN }}
|
||||
@@ -0,0 +1,200 @@
|
||||
name: Test & Benchmark
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
branches: [main]
|
||||
paths:
|
||||
- 'java/**'
|
||||
- 'python/**'
|
||||
- 'node/**'
|
||||
- 'scripts/**'
|
||||
- 'verification/**'
|
||||
- 'samples/**'
|
||||
- '.github/workflows/**'
|
||||
workflow_dispatch:
|
||||
|
||||
concurrency:
|
||||
group: ci-${{ github.event.pull_request.number || github.ref }}
|
||||
cancel-in-progress: true
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
test:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v7
|
||||
|
||||
- name: Setup Java
|
||||
uses: actions/setup-java@v5
|
||||
with:
|
||||
distribution: 'temurin'
|
||||
java-version: '21'
|
||||
|
||||
- name: Setup Python
|
||||
uses: actions/setup-python@v6
|
||||
with:
|
||||
python-version: '3.12'
|
||||
|
||||
- name: Setup uv
|
||||
uses: astral-sh/setup-uv@v7
|
||||
|
||||
- name: Setup Node.js
|
||||
uses: actions/setup-node@v6
|
||||
with:
|
||||
node-version: '20'
|
||||
|
||||
- name: Setup pnpm
|
||||
run: npm install -g pnpm
|
||||
|
||||
- name: Build & Test All
|
||||
run: ./scripts/build-all.sh
|
||||
|
||||
- name: Install opendataloader-pdf CLI
|
||||
# --system installs into the setup-python interpreter; without it (or an
|
||||
# active venv) `uv pip install` aborts with "No virtual environment found".
|
||||
run: uv pip install --system ./python/opendataloader-pdf/dist/*.whl
|
||||
|
||||
- name: Run CLI verification
|
||||
# ci-verify.py writes its own markdown table to $GITHUB_STEP_SUMMARY,
|
||||
# so no separate summary step is needed here.
|
||||
run: python verification/ci-verify.py
|
||||
|
||||
- name: Upload verification report
|
||||
if: always()
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: verification-report-ci
|
||||
path: verification/verification-report-ci.txt
|
||||
retention-days: 7
|
||||
|
||||
- name: Upload coverage to Codecov
|
||||
uses: codecov/codecov-action@v7
|
||||
with:
|
||||
files: java/opendataloader-pdf-core/target/site/jacoco/jacoco.xml
|
||||
fail_ci_if_error: false
|
||||
token: ${{ secrets.CODECOV_TOKEN }}
|
||||
|
||||
- name: Upload build artifacts
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: java-build
|
||||
path: java/opendataloader-pdf-cli/target/*.jar
|
||||
retention-days: 1
|
||||
|
||||
benchmark:
|
||||
needs: test
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v7
|
||||
|
||||
- name: Setup Java
|
||||
uses: actions/setup-java@v5
|
||||
with:
|
||||
distribution: 'temurin'
|
||||
java-version: '21'
|
||||
|
||||
- name: Download build artifacts
|
||||
uses: actions/download-artifact@v8
|
||||
with:
|
||||
name: java-build
|
||||
path: java/opendataloader-pdf-cli/target/
|
||||
|
||||
- name: Setup Python
|
||||
uses: actions/setup-python@v6
|
||||
with:
|
||||
python-version: '3.13'
|
||||
|
||||
- name: Setup uv
|
||||
uses: astral-sh/setup-uv@v7
|
||||
|
||||
- name: Run benchmark
|
||||
run: ./scripts/bench.sh --skip-build --check-regression
|
||||
|
||||
- name: Benchmark summary
|
||||
if: always()
|
||||
run: |
|
||||
python3 << 'PYEOF'
|
||||
import json, os, sys
|
||||
from pathlib import Path
|
||||
|
||||
eval_path = Path("/tmp/opendataloader-bench/prediction/opendataloader/evaluation.json")
|
||||
thresh_path = Path("/tmp/opendataloader-bench/thresholds.json")
|
||||
|
||||
summary_file = os.environ.get("GITHUB_STEP_SUMMARY", "/dev/null")
|
||||
|
||||
if not eval_path.exists() or not thresh_path.exists():
|
||||
with open(summary_file, "a") as f:
|
||||
f.write("## Benchmark Results\n\nBenchmark did not produce evaluation results.\n")
|
||||
sys.exit(0)
|
||||
|
||||
try:
|
||||
with open(eval_path) as f:
|
||||
eval_data = json.load(f)
|
||||
with open(thresh_path) as f:
|
||||
thresholds = json.load(f)
|
||||
except json.JSONDecodeError as e:
|
||||
with open(summary_file, "a") as f:
|
||||
f.write(f"## Benchmark Results\n\nFailed to parse results: {e}\n")
|
||||
sys.exit(0)
|
||||
|
||||
scores = eval_data.get("metrics", {}).get("score", {})
|
||||
table_detection = eval_data.get("table_detection", {})
|
||||
speed = eval_data.get("speed", {})
|
||||
triage = eval_data.get("triage", {})
|
||||
tol = thresholds.get("regression_tolerance", 0)
|
||||
|
||||
rows = []
|
||||
|
||||
for key, label, src in [
|
||||
("nid", "NID", scores.get("nid_mean")),
|
||||
("teds", "TEDS", scores.get("teds_mean")),
|
||||
("mhs", "MHS", scores.get("mhs_mean")),
|
||||
("table_detection_f1", "Table Detection F1", table_detection.get("f1")),
|
||||
]:
|
||||
t = thresholds.get(key)
|
||||
if src is not None and t is not None:
|
||||
effective = t - tol
|
||||
status = "✅" if src >= effective else "❌"
|
||||
rows.append(f"| {label} | {src:.4f} | ≥ {effective:.2f} | {status} |")
|
||||
|
||||
elapsed = speed.get("elapsed_per_doc")
|
||||
elapsed_thresh = thresholds.get("elapsed_per_doc")
|
||||
if elapsed is not None and elapsed_thresh is not None:
|
||||
status = "✅" if elapsed <= elapsed_thresh else "❌"
|
||||
rows.append(f"| Speed | {elapsed:.2f}s/doc | ≤ {elapsed_thresh}s/doc | {status} |")
|
||||
|
||||
if triage:
|
||||
tr_recall = triage.get("recall")
|
||||
tr_thresh = thresholds.get("triage_recall")
|
||||
if tr_recall is not None and tr_thresh is not None:
|
||||
effective = tr_thresh - tol
|
||||
status = "✅" if tr_recall >= effective else "❌"
|
||||
rows.append(f"| Triage Recall | {tr_recall:.4f} | ≥ {effective:.2f} | {status} |")
|
||||
|
||||
tr_fn = triage.get("fn_count")
|
||||
tr_fn_max = thresholds.get("triage_fn_max")
|
||||
if tr_fn is not None and tr_fn_max is not None:
|
||||
status = "✅" if tr_fn <= tr_fn_max else "❌"
|
||||
rows.append(f"| Triage FN | {tr_fn} | ≤ {tr_fn_max} | {status} |")
|
||||
|
||||
with open(summary_file, "a") as f:
|
||||
f.write("## Benchmark Results\n\n")
|
||||
f.write("| Metric | Score | Threshold | Status |\n")
|
||||
f.write("|--------|-------|-----------|--------|\n")
|
||||
for row in rows:
|
||||
f.write(row + "\n")
|
||||
|
||||
if not rows:
|
||||
f.write("| (no metrics found) | | | |\n")
|
||||
PYEOF
|
||||
|
||||
- name: Upload evaluation results
|
||||
uses: actions/upload-artifact@v7
|
||||
if: always()
|
||||
with:
|
||||
name: benchmark-results
|
||||
path: /tmp/opendataloader-bench/prediction/opendataloader/evaluation.json
|
||||
Reference in New Issue
Block a user