learningcircuit--local-deep-research/.github/workflows/journal-data-integration.yml

name: Journal Quality Data Integration

# Validates that the five external journal-quality data sources can
# still be downloaded and the read-only reference DB can be built from
# them end to end. Catches upstream schema breaks (OpenAlex renaming a
# field, DOAJ changing their dump format, etc.) BEFORE we cut a release.
#
# This is intentionally a separate workflow rather than a job inside the
# main test suite — it pulls ~25 MB from third-party APIs and takes
# ~30–60 seconds, so it shouldn't run on every PR. It runs:
#   - Weekly (Mondays 4 AM UTC) — catches drift between releases
#   - On the release-gate workflow_call — blocks publishing if upstreams
#     have broken our build
#   - Manually via workflow_dispatch — for ad-hoc validation

on:
  workflow_call:  # called by release-gate.yml
  workflow_dispatch:
  schedule:
    # Mondays 4 AM UTC — staggered after release-gate (2 AM) so the two
    # don't collide if upstream rate-limits are touchy.
    - cron: '0 4 * * 1'

permissions: {}  # minimal top-level for OSSF Scorecard

jobs:
  download-and-build:
    name: Download external sources + build reference DB
    runs-on: ubuntu-latest
    # 45 min job timeout: parallel downloads are bounded by the slowest
    # source (OpenAlex Institutions, ~10 min for ~110K rows via cursor
    # pagination). Plus build_db (~15s) + setup overhead. Generous
    # headroom so transient API slowness doesn't false-fail the gate.
    timeout-minutes: 45
    permissions:
      contents: read

    steps:
      - name: Harden the runner (Audit all outbound calls)
        uses: step-security/harden-runner@9af89fc71515a100421586dfdb3dc9c984fbf411 # v2.19.4
        with:
          egress-policy: audit

      - name: Checkout code
        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
        with:
          persist-credentials: false

      - name: Set up Python
        uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0
        with:
          python-version: '3.11'

      - name: Install system dependencies for SQLCipher
        run: |
          sudo apt-get update
          sudo apt-get install -y libsqlcipher-dev

      - name: Set up PDM
        uses: pdm-project/setup-pdm@973541a5febeafcfdadf8a51211435be6ecfd90f # v4.5
        with:
          python-version: '3.11'

      - name: Install dependencies
        run: pdm install --dev

      - name: Run journal-quality release-gate integration test
        # The test is marked `integration and slow` so it's skipped by
        # the regular suite. We opt in here. The session fixture
        # downloads all 5 sources in PARALLEL via ThreadPoolExecutor;
        # wall-clock is bounded by the slowest source (institutions,
        # ~10 min). Per-test pytest-timeout=2400s (40 min) overrides
        # the global 60s default for these slow tests.
        run: |
          pdm run pytest tests/integration/test_journal_quality_release_gate.py \
            -m "integration and slow" \
            -v --tb=short \
            --timeout=2400
        timeout-minutes: 42