chore: import upstream snapshot with attribution

2026-07-13 13:08:55 +08:00
commit 7a0da7932b
2985 changed files with 1049377 additions and 0 deletions
@@ -0,0 +1,79 @@
+name: Journal Quality Data Integration
+
+# Validates that the five external journal-quality data sources can
+# still be downloaded and the read-only reference DB can be built from
+# them end to end. Catches upstream schema breaks (OpenAlex renaming a
+# field, DOAJ changing their dump format, etc.) BEFORE we cut a release.
+#
+# This is intentionally a separate workflow rather than a job inside the
+# main test suite — it pulls ~25 MB from third-party APIs and takes
+# ~30–60 seconds, so it shouldn't run on every PR. It runs:
+#   - Weekly (Mondays 4 AM UTC) — catches drift between releases
+#   - On the release-gate workflow_call — blocks publishing if upstreams
+#     have broken our build
+#   - Manually via workflow_dispatch — for ad-hoc validation
+
+on:
+  workflow_call:  # called by release-gate.yml
+  workflow_dispatch:
+  schedule:
+    # Mondays 4 AM UTC — staggered after release-gate (2 AM) so the two
+    # don't collide if upstream rate-limits are touchy.
+    - cron: '0 4 * * 1'
+
+permissions: {}  # minimal top-level for OSSF Scorecard
+
+jobs:
+  download-and-build:
+    name: Download external sources + build reference DB
+    runs-on: ubuntu-latest
+    # 45 min job timeout: parallel downloads are bounded by the slowest
+    # source (OpenAlex Institutions, ~10 min for ~110K rows via cursor
+    # pagination). Plus build_db (~15s) + setup overhead. Generous
+    # headroom so transient API slowness doesn't false-fail the gate.
+    timeout-minutes: 45
+    permissions:
+      contents: read
+
+    steps:
+      - name: Harden the runner (Audit all outbound calls)
+        uses: step-security/harden-runner@9af89fc71515a100421586dfdb3dc9c984fbf411 # v2.19.4
+        with:
+          egress-policy: audit
+
+      - name: Checkout code
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
+        with:
+          persist-credentials: false
+
+      - name: Set up Python
+        uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0
+        with:
+          python-version: '3.11'
+
+      - name: Install system dependencies for SQLCipher
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y libsqlcipher-dev
+
+      - name: Set up PDM
+        uses: pdm-project/setup-pdm@973541a5febeafcfdadf8a51211435be6ecfd90f # v4.5
+        with:
+          python-version: '3.11'
+
+      - name: Install dependencies
+        run: pdm install --dev
+
+      - name: Run journal-quality release-gate integration test
+        # The test is marked `integration and slow` so it's skipped by
+        # the regular suite. We opt in here. The session fixture
+        # downloads all 5 sources in PARALLEL via ThreadPoolExecutor;
+        # wall-clock is bounded by the slowest source (institutions,
+        # ~10 min). Per-test pytest-timeout=2400s (40 min) overrides
+        # the global 60s default for these slow tests.
+        run: |
+          pdm run pytest tests/integration/test_journal_quality_release_gate.py \
+            -m "integration and slow" \
+            -v --tb=short \
+            --timeout=2400
+        timeout-minutes: 42