name: Release Gate # IMPORTANT: This workflow is the ONLY path to publishing releases. # # Security model: # - All security scans must pass before any release artifacts are published # - Docker and PyPI publish workflows are triggered ONLY via repository_dispatch # from release.yml, which depends on this gate passing # - Direct release creation via GitHub UI will NOT trigger publishing # (this is intentional - it prevents bypassing security checks) # # If you need to publish manually, use workflow_dispatch on release.yml # which will run this security gate first. # # This workflow runs all security scans. # Used by release workflow and also runs daily to catch issues early. # # Philosophy: # - Secrets (gitleaks) run on PRs because secrets in main = permanent exposure # - Everything else runs at release gate only - can be fixed before release on: workflow_call: # Called by release.yml workflow_dispatch: # Manual trigger schedule: # Run daily at 2 AM UTC to catch security issues early - cron: '0 2 * * *' permissions: {} # Minimal top-level for OSSF Scorecard # Serialize direct workflow_dispatch/schedule runs so two long gate runs # (notably the qemu TCG emulation job) don't execute side by side. # # The group MUST use a literal prefix, not ${{ github.workflow }}: inside a # workflow_call invocation github.workflow resolves to the *caller's* name # (release.yml = "Create Release"), so ${{ github.workflow }} would collide # with release.yml's own "Create Release-" group. The parent holds that # group and waits for this called workflow, which can't acquire the same group # (cancel-in-progress: false) — GitHub detects the deadlock and cancels the # whole release run. A literal "release-gate-" prefix keeps the called group # distinct from the parent while still serializing standalone runs. # # cancel-in-progress stays false — this gate is the only path to publishing, # so an in-flight run must never be killed by a newer trigger. concurrency: group: release-gate-${{ github.ref }} cancel-in-progress: false jobs: # ============================================ # Existing security scans # ============================================ # NOTE: container-security, dockle, and grype each build the Docker image # independently. Do NOT combine them into a shared build — this gate runs # infrequently (daily + releases) so the ~3-6 min savings isn't worth the # added complexity, and scanners rely on repo config files (.trivyignore, # .grype.yaml) that require a checkout step tied to each scanner's own job. security-tests: uses: ./.github/workflows/security-tests.yml permissions: contents: read security-events: write semgrep-scan: uses: ./.github/workflows/semgrep.yml permissions: contents: read security-events: write actions: read container-security: uses: ./.github/workflows/container-security.yml permissions: contents: read security-events: write actions: read owasp-zap-scan: uses: ./.github/workflows/owasp-zap-scan.yml permissions: contents: read security-events: write # ============================================ # SAST and code analysis # ============================================ codeql-scan: uses: ./.github/workflows/codeql.yml permissions: contents: read security-events: write packages: read actions: read devskim-scan: uses: ./.github/workflows/devskim.yml permissions: contents: read security-events: write actions: read bearer-scan: uses: ./.github/workflows/bearer.yml permissions: contents: read security-events: write actions: read # ============================================ # Dependency vulnerability scanning # ============================================ # NOTE: OSV-Scanner is intentionally NOT included in this gate. # # GitHub Actions limits reusable workflow nesting to 2 levels. Since: # release.yml → release-gate.yml → osv-scanner.yml → google/osv-scanner-reusable.yml # would create 4 levels, this causes startup_failure. # # OSV-Scanner runs on PRs instead (see osv-scanner.yml), which is better # for shift-left security - catching vulnerable dependencies before merge. # # See: https://docs.github.com/en/actions/using-workflows/reusing-workflows#nesting-reusable-workflows npm-audit: uses: ./.github/workflows/npm-audit.yml permissions: contents: read retirejs-scan: uses: ./.github/workflows/retirejs.yml permissions: contents: read security-events: write # ============================================ # Container and IaC security # ============================================ dockle-scan: uses: ./.github/workflows/dockle.yml permissions: contents: read security-events: write hadolint-scan: uses: ./.github/workflows/hadolint.yml permissions: contents: read security-events: write checkov-scan: uses: ./.github/workflows/checkov.yml permissions: contents: read security-events: write docker-multiarch-test: uses: ./.github/workflows/docker-multiarch-test.yml permissions: contents: read # ============================================ # Workflow security # ============================================ zizmor-scan: uses: ./.github/workflows/zizmor-security.yml permissions: contents: read security-events: write actions: read # ============================================ # Dependency/container scanning (overlapping coverage) # ============================================ grype-scan: uses: ./.github/workflows/grype.yml permissions: contents: read security-events: write actions: read # ============================================ # Secret detection - scans only main branch history # ============================================ gitleaks-scan: uses: ./.github/workflows/gitleaks-main.yml permissions: contents: read security-events: write actions: read # ============================================ # DAST (overlapping coverage with ZAP) # ============================================ # TEMPORARILY DISABLED: nuclei-scan — causing slowdowns # nuclei-scan: # uses: ./.github/workflows/nuclei.yml # permissions: # contents: read # security-events: write # actions: read # ============================================ # License compliance (catches direct-to-main commits) # ============================================ license-check: name: License Compliance Check runs-on: ubuntu-latest timeout-minutes: 15 permissions: contents: read steps: - name: Harden the runner (Audit all outbound calls) uses: step-security/harden-runner@9af89fc71515a100421586dfdb3dc9c984fbf411 # v2.19.4 with: egress-policy: audit - name: Checkout code uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: persist-credentials: false - name: Set up Python uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0 with: python-version: '3.12' - name: Install system dependencies for SQLCipher run: | sudo apt-get update sudo apt-get install -y libsqlcipher-dev - name: Set up PDM uses: pdm-project/setup-pdm@973541a5febeafcfdadf8a51211435be6ecfd90f # v4.5 with: python-version: '3.12' - name: Install dependencies run: pdm sync -d - name: Check Python dependency licenses run: | set -euo pipefail pdm run pip install pip-licenses echo "=== Python Dependency Licenses ===" pdm run pip-licenses --format=plain echo "" echo "=== Checking against ASF Category A + B allowlist ===" # --partial-match: substring matching (pip-licenses reports both SPDX IDs and full names) # --allow-only: fails on any license not in the list (including UNKNOWN) # --ignore-packages: exceptions for packages with metadata issues or tri-licensing # chardet: LGPL-only orphan (not in pdm.lock, nothing imports it) # text-unidecode: tri-licensed Artistic/GPL/GPLv2+ (accepted under Artistic, # but combined string contains "GPL" which can't be allowlisted safely) # cookiecutter: actually BSD-3-Clause but reports UNKNOWN (metadata bug, dev-only dep) # pyphen: tri-licensed GPLv2+/LGPLv2+/MPL-1.1 (accepted under MPL-1.1, # but combined string contains "GPL" which can't be allowlisted safely) # NOTE: CC-BY uses version-specific entries to avoid matching CC-BY-NC/CC-BY-SA # NOTE: "Mozilla Public License 2.0" avoids matching MPL-1.0/1.1 (more restrictive) # NOTE: "Artistic-2.0" avoids matching Artistic-1.0 (ASF Category X) # See also: .github/workflows/dependency-review.yml (PR-time SPDX check) pdm run pip-licenses \ --partial-match \ --allow-only="Apache-2.0;Apache Software License;Apache License;Apache 2.0;MIT;MIT License;BSD-2-Clause;BSD-3-Clause;BSD License;ISC;ISC License;0BSD;Zlib;PSF-2.0;Python Software Foundation;Unlicense;CC0-1.0;CC-BY-3.0;CC-BY-4.0;Artistic-2.0;MPL-2.0;Mozilla Public License 2.0;HPND;Historical Permission Notice;MIT-CMU;OFL-1.1;BlueOak-1.0.0;Public Domain;Boost;Unicode" \ --ignore-packages chardet text-unidecode cookiecutter pyphen - name: Check npm dependency licenses run: | set -euo pipefail # Use allowlist (--onlyAllow) not denylist, consistent with Python check # UNLICENSED: the root package itself (private, no license field) NPM_ALLOWED="MIT;ISC;BSD-2-Clause;BSD-3-Clause;Apache-2.0;MPL-2.0;CC-BY-4.0;OFL-1.1;Zlib;Unlicense;CC0-1.0;0BSD;Python-2.0;BlueOak-1.0.0;UNLICENSED" echo "=== Root npm dependencies ===" npm ci --ignore-scripts npx license-checker --summary --onlyAllow "$NPM_ALLOWED" echo "" echo "=== Test npm dependencies ===" cd tests npm ci --ignore-scripts npx license-checker --summary --onlyAllow "$NPM_ALLOWED" # ============================================ # Runtime security validation # ============================================ security-headers-scan: uses: ./.github/workflows/security-headers-validation.yml permissions: contents: read # Backwards compatibility & migration tests # ============================================ backwards-compatibility: uses: ./.github/workflows/backwards-compatibility.yml permissions: contents: read # ============================================ # External data integration — guards against upstream API/format # changes silently breaking the journal-quality build # ============================================ journal-data-integration: uses: ./.github/workflows/journal-data-integration.yml permissions: contents: read # ============================================ # pip install verification (CRITICAL — DO NOT REMOVE OR WEAKEN) # ============================================ # ┌─────────────────────────────────────────────────────────────────┐ # │ WHY THIS TEST EXISTS AND MUST NEVER BE SKIPPED │ # │ │ # │ PDM uses [tool.pdm.resolution.overrides] to force specific │ # │ dependency versions (e.g. requests>=2.33 for CVE fixes). │ # │ These overrides LEAK into wheel metadata when `pdm build` │ # │ runs, creating constraints that pip cannot satisfy. │ # │ │ # │ Example: v1.4.0 shipped with requests>=2.33 in metadata │ # │ (from PDM override), but arxiv~=2.4 requires requests<2.33. │ # │ pip install failed for ALL users. PDM installs worked fine │ # │ because PDM applies overrides differently. │ # │ │ # │ This test catches that BEFORE publishing to PyPI. │ # │ If this test fails, the release MUST be blocked. │ # │ │ # │ DO NOT: │ # │ - Add continue-on-error to this job │ # │ - Remove it from release-gate-summary needs │ # │ - Replace pip install with pdm install │ # │ - Skip dependency resolution (--no-deps) │ # │ │ # │ If it fails, fix the dependency conflict in pyproject.toml │ # │ before releasing. │ # └─────────────────────────────────────────────────────────────────┘ pip-install-check: name: pip Install Verification runs-on: ubuntu-latest timeout-minutes: 15 permissions: contents: read steps: - name: Harden the runner (Audit all outbound calls) uses: step-security/harden-runner@9af89fc71515a100421586dfdb3dc9c984fbf411 # v2.19.4 with: egress-policy: audit - name: Checkout code uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: persist-credentials: false - name: Set up Python uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0 with: python-version: '3.12' - name: Install system dependencies for SQLCipher run: | sudo apt-get update sudo apt-get install -y libsqlcipher-dev - name: Install Node.js for Vite build uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 with: # Match package.json's `engines: { node: ">=24.0.0" }` and # publish.yml's node:24-alpine. Building the wheel-tested # frontend on Node 20 here while the real publish runs on # Node 24 means the gate validates a different runtime than # ships, defeating the gate's stated purpose. node-version: '24' - name: Set up PDM (build only) uses: pdm-project/setup-pdm@973541a5febeafcfdadf8a51211435be6ecfd90f # v4.5 with: python-version: '3.12' - name: Build Vite frontend assets run: | npm ci npm run build - name: Build wheel with PDM run: pdm build --no-sdist - name: Verify pip install in clean environment run: | # ============================================================ # CRITICAL: This verifies the WHEEL can be installed with pip. # # PDM overrides in pyproject.toml can inject dependency # constraints into the wheel metadata that pip cannot resolve. # This catches that failure BEFORE we publish to PyPI. # # A failure here means users running `pip install local-deep-research` # would get a dependency resolution error. # ============================================================ echo "=== Creating clean virtual environment ===" python -m venv /tmp/pip-verify-env . /tmp/pip-verify-env/bin/activate echo "" echo "=== Installing wheel with pip (NOT pdm) ===" echo "This simulates what end users experience with: pip install local-deep-research" echo "" WHEEL=$(find dist/ -maxdepth 1 -name '*.whl' -print -quit) if [ -z "$WHEEL" ]; then echo "::error::No wheel found in dist/" exit 1 fi echo "Wheel: $WHEEL" echo "" # --no-cache-dir: force fresh resolution, no cached metadata # DO NOT add --no-deps — that defeats the entire purpose of this check if ! pip install --no-cache-dir "$WHEEL" 2>&1; then echo "" echo "::error::pip install FAILED — the built wheel has unresolvable dependencies." echo "::error::This means the PyPI release would be broken for all pip users." echo "::error::Check [tool.pdm.resolution.overrides] in pyproject.toml for conflicts." exit 1 fi echo "" echo "=== Verifying import works ===" python -c "import local_deep_research; print(f'OK: v{local_deep_research.__version__}')" echo "" echo "=== Verifying dependency tree is consistent ===" pip check echo "" echo "=== twine check (PyPI metadata + long_description rendering) ===" # Twine validates that the wheel's PKG-INFO is well-formed and the # README renders cleanly on PyPI. pdm build accepts metadata twine # rejects (e.g. malformed long_description), so this catches a class # of "PyPI project page silently broken" failures pip-install-check # cannot see. Installed transiently inside the throwaway venv. pip install --quiet twine twine check --strict "$WHEEL" echo "" echo "=== Installing [mcp] extra so ldr-mcp can resolve ===" # `ldr-mcp`'s target (local_deep_research.mcp.server) unconditionally # imports the `mcp` package, which is declared in the [mcp] extra # rather than core deps — intentional, per mcp/__init__.py docstring. # Incremental install on top of the base install above preserves the # base-install verification while letting the entry-point smoke # exercise both scripts. pip install --quiet --no-cache-dir "${WHEEL}[mcp]" echo "" echo "=== CLI entry-point resolution ===" # `import local_deep_research` above only proves the package is # importable — it doesn't exercise the [project.scripts] entries. # We can't invoke `ldr-web`/`ldr-mcp` directly: neither parses # --help, both start long-running servers and would hang the job. # Instead, replicate exactly what the console_script wrapper does # at runtime — import the target module and resolve the attribute # — via entry_points().load(), without calling the function. # Catches dead module refs (the original `main:main` bug) and # missing/renamed target attributes. python - <<'PY' import importlib.metadata as md targets = {"ldr-web", "ldr-mcp"} found = set() for ep in md.entry_points(group="console_scripts"): if ep.name in targets: fn = ep.load() assert callable(fn), f"{ep.name} -> {ep.value} is not callable" print(f"OK {ep.name} -> {ep.value}") found.add(ep.name) missing = targets - found assert not missing, f"missing console_scripts: {missing}" PY echo "" echo "✅ pip install verification passed — wheel is safe to publish" # ============================================ # CPU baseline smoke test under qemu-emulated CPUs # # Catches dependency wheels that silently raise the x86-64 # instruction-set baseline (e.g. AVX2) and would crash with SIGILL # at import time on older CPUs. Added after faiss-cpu 1.14.2 shipped # an AVX2 instruction in a static initializer that killed every # AVX-only deployment at container startup with exit code 132 # (#4480, upstream facebookresearch/faiss#5296). # # Release-gate only by design: full-system TCG emulation is far too # slow for per-PR CI, and the failure mode only ships via dependency # bumps, which the gate sees before any release. # ============================================ cpu-baseline-smoke: name: CPU Baseline Smoke (qemu) runs-on: ubuntu-latest timeout-minutes: 45 permissions: contents: read steps: - name: Harden the runner (Audit all outbound calls) uses: step-security/harden-runner@9af89fc71515a100421586dfdb3dc9c984fbf411 # v2.19.4 with: egress-policy: audit - name: Checkout code uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: persist-credentials: false - name: Set up Python id: setup-python uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0 with: # Keep in sync with requires-python in pyproject.toml — smoke-test # the wheels for the oldest interpreter the project supports. python-version: '3.12' - name: Install system dependencies (SQLCipher, qemu user-mode) run: | sudo apt-get update sudo apt-get install -y libsqlcipher-dev qemu-user # Fail here with a clear message rather than as a confusing # "command not found" inside the per-module loop below. command -v qemu-x86_64 - name: Set up PDM uses: pdm-project/setup-pdm@973541a5febeafcfdadf8a51211435be6ecfd90f # v4.5 with: # Keep in sync with requires-python in pyproject.toml (see above). python-version: '3.12' - name: Cache pdm venv (keyed on pdm.lock) # A cold pdm sync re-downloads the full production lock — the torch # wheel alone is several hundred MB — adding minutes before the # emulation even starts. The exact interpreter patch version is part # of the key because the venv hard-links the toolcache interpreter: # a runner-image Python bump would otherwise restore a venv with a # dangling interpreter path. No restore-keys on purpose: pdm.lock IS # the contract this job tests, so a venv from any other lock must # never be restored — on a lock change a cold, exact install is the # correct behavior. pdm sync still runs below either way and # verifies the restored venv against the lock. uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 with: path: .venv key: cpu-smoke-venv-${{ runner.os }}-py${{ steps.setup-python.outputs.python-version }}-${{ hashFiles('pdm.lock') }} - name: Install locked production dependencies # lock --check first: the whole point is to test the exact wheels # the Docker image ships, so fail loudly if pdm.lock has drifted # from pyproject.toml instead of silently testing a stale set. # --no-self: only the dependency wheels matter here; skipping the # project build avoids needing the Vite frontend. pdm sync installs # the exact pins from pdm.lock — the same wheels the Docker image # ships. run: | pdm lock --check pdm sync --prod --no-self - name: Import native-heavy modules under emulated CPUs run: | set -euo pipefail # SIMD-heavy native extensions on the app's startup path. These # are the wheels with per-arch optimized builds, i.e. the ones # that can SIGILL on older CPUs. Pure-Python deps can't fail # this way and would only slow the emulation down. MODULES=(numpy pandas pyarrow sklearn faiss torch) # SandyBridge = AVX without AVX2 — the supported x86-64 floor # and the CPU class that hit #4480. Haswell = AVX2+FMA positive # control proving the dispatch path still selects the fast # kernels. Pre-AVX models (Nehalem/Westmere) are deliberately # absent: the pandas and scikit-learn wheels already require # AVX (verified to SIGILL on emulated Nehalem/Westmere), so # AVX is LDR's de-facto minimum and gating on older CPUs would # block every release. No AVX512 model: qemu TCG cannot emulate # AVX512, so an emulated Skylake-Server would just mask those # bits and retest the Haswell path. CPUS=(SandyBridge Haswell) # Keep emulated imports single-threaded — thread pools multiply # TCG overhead without adding coverage. export OMP_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 MKL_NUM_THREADS=1 VENV_PY="$(pdm run python -c 'import sys; print(sys.executable)')" # Per-import ceiling so a hang surfaces as "timed out on # module X / CPU Y" (exit 124) instead of the job dying at its # 45-minute wall with no signal. The slowest module (torch) # imports in well under 2 minutes at typical ~30x TCG slowdown; # 300s leaves generous headroom. IMPORT_TIMEOUT=300 echo "=== Native control run (host CPU) ===" # Proves each module imports at all, so any qemu failure below # is attributable to the emulated CPU's instruction set. for mod in "${MODULES[@]}"; do rc=0 timeout "$IMPORT_TIMEOUT" "$VENV_PY" -c "import $mod" || rc=$? if [ "$rc" -ne 0 ]; then echo "::error::import $mod failed natively (exit $rc) — the module is broken regardless of CPU; the emulated runs below were not reached." exit 1 fi echo "OK $mod (native)" done # CPU-outer / module-inner with fail-fast (exit 1 on the first # failing pair) is deliberate: Haswell is a strict superset of # SandyBridge, so once a module fails on the older CPU the newer # CPU's result adds no signal — don't "fix" this into a # full-matrix report. for cpu in "${CPUS[@]}"; do echo "" echo "=== Emulated CPU: $cpu ===" for mod in "${MODULES[@]}"; do rc=0 timeout "$IMPORT_TIMEOUT" qemu-x86_64 -cpu "$cpu" "$VENV_PY" -c "import $mod" || rc=$? if [ "$rc" -ne 0 ]; then if [ "$rc" -eq 132 ]; then echo "::error::import $mod died with SIGILL (exit 132) on emulated $cpu — a dependency wheel executes instructions this CPU lacks. Find the offending package and pin it (see #4480 for the faiss-cpu precedent and debugging walkthrough)." elif [ "$rc" -eq 124 ]; then echo "::error::import $mod timed out after ${IMPORT_TIMEOUT}s on emulated $cpu — likely a hang specific to TCG emulation rather than an instruction-set issue." else echo "::error::import $mod failed (exit $rc) on emulated $cpu" fi exit 1 fi echo "OK $mod ($cpu)" done done echo "" echo "✅ All native-heavy imports succeeded on every emulated CPU" # ============================================ # Check for open code scanning alerts # This runs after all SARIF uploads complete and fails if # high/critical/medium severity alerts exist from ANY tool # ============================================ check-code-scanning-alerts: name: Check Code Scanning Alerts runs-on: ubuntu-latest needs: # Wait for all SARIF-uploading scans to complete - codeql-scan - semgrep-scan - devskim-scan - bearer-scan - security-tests - container-security # grype-scan uploads the grype-filesystem / grype-container SARIF. Without # this edge the alert query below can run before Grype's SARIF is indexed, # so a fresh Grype finding for this commit could slip past the gate. (A # grype-scan *failure* was already caught by release-gate-summary; this # makes its *findings* reliably gate too.) - grype-scan - owasp-zap-scan - retirejs-scan - dockle-scan - hadolint-scan - checkov-scan - zizmor-scan - gitleaks-scan permissions: security-events: read steps: - name: Harden the runner (Audit all outbound calls) uses: step-security/harden-runner@9af89fc71515a100421586dfdb3dc9c984fbf411 # v2.19.4 with: egress-policy: audit - name: Wait for SARIF indexing run: | echo "Waiting 120 seconds for GitHub to index SARIF uploads..." sleep 120 - name: Check for open security alerts env: GH_TOKEN: ${{ github.token }} run: | echo "Checking for open code scanning alerts..." # Query all open code scanning alerts with pagination ALERTS="[]" PAGE=1 while true; do RESPONSE=$(gh api \ -H "Accept: application/vnd.github+json" \ -H "X-GitHub-Api-Version: 2022-11-28" \ "/repos/${{ github.repository }}/code-scanning/alerts?state=open&per_page=100&page=${PAGE}" \ 2>&1) || { echo "::error::Failed to query code scanning API (page ${PAGE}): ${RESPONSE}" exit 1 } # Validate response is a JSON array if ! echo "$RESPONSE" | jq -e 'type == "array"' > /dev/null 2>&1; then echo "::error::Unexpected API response (not a JSON array): ${RESPONSE}" exit 1 fi COUNT=$(echo "$RESPONSE" | jq 'length') if [ "$COUNT" -eq 0 ]; then break fi # Merge page results into ALERTS ALERTS=$(echo "$ALERTS" "$RESPONSE" | jq -s '.[0] + .[1]') PAGE=$((PAGE + 1)) done TOTAL_ALERTS=$(echo "$ALERTS" | jq 'length') echo "Fetched ${TOTAL_ALERTS} open alert(s) across $((PAGE - 1)) page(s)." # Count alerts by severity CRITICAL=$(echo "$ALERTS" | jq '[.[] | select(.rule.security_severity_level == "critical")] | length') HIGH=$(echo "$ALERTS" | jq '[.[] | select(.rule.security_severity_level == "high")] | length') MEDIUM=$(echo "$ALERTS" | jq '[.[] | select(.rule.security_severity_level == "medium")] | length') { echo "" echo "## Code Scanning Alert Summary" echo "" echo "| Severity | Count |" echo "|----------|-------|" echo "| Critical | $CRITICAL |" echo "| High | $HIGH |" echo "| Medium | $MEDIUM |" echo "" } >> "$GITHUB_STEP_SUMMARY" TOTAL=$((CRITICAL + HIGH + MEDIUM)) if [ "$TOTAL" -gt 0 ]; then BLOCKING_FILTER='select(.rule.security_severity_level == "critical" or .rule.security_severity_level == "high" or .rule.security_severity_level == "medium")' { echo "❌ **Found $TOTAL blocking alerts (Critical: $CRITICAL, High: $HIGH, Medium: $MEDIUM)**" echo "" echo "### Blocking Alerts:" echo "$ALERTS" | jq -r ".[] | ${BLOCKING_FILTER} | \"- [\\(.rule.security_severity_level | ascii_upcase)] \\(.rule.description) ([#\\(.number)](\\(.html_url)))\"" } >> "$GITHUB_STEP_SUMMARY" echo "" echo "::error::Security gate failed: Found $TOTAL open code scanning alerts (Critical: $CRITICAL, High: $HIGH, Medium: $MEDIUM)" exit 1 else echo "✅ **No critical, high, or medium severity alerts found**" >> "$GITHUB_STEP_SUMMARY" echo "No blocking code scanning alerts found." fi # ============================================ # Summary job that reports overall status # ============================================ release-gate-summary: name: Release Gate Summary runs-on: ubuntu-latest needs: # Existing scans - security-tests - semgrep-scan - container-security - owasp-zap-scan # SAST and code analysis - codeql-scan - devskim-scan - bearer-scan # Dependency scanning (OSV-Scanner runs on PRs, not here - see comment above) - npm-audit - retirejs-scan # Dependency/container scanning (overlapping coverage) - grype-scan # Container and IaC - dockle-scan - hadolint-scan - checkov-scan - docker-multiarch-test # Workflow security - zizmor-scan # Secret detection - gitleaks-scan # DAST (overlapping coverage) — nuclei temporarily disabled # - nuclei-scan # Runtime security - security-headers-scan # License compliance - license-check # Backwards compatibility & migration tests - backwards-compatibility # External data integration (journal_quality) - journal-data-integration # CRITICAL — DO NOT REMOVE: pip install verification catches PDM override # leaks that break `pip install` for all users. See pip-install-check job # comments for the v1.4.0 incident. Removing this dependency silently # allows broken wheels to be published to PyPI. - pip-install-check # CPU baseline smoke — catches dependency wheels that raise the # x86 instruction-set baseline and SIGILL on older CPUs (#4480) - cpu-baseline-smoke # Alert check (fails if any critical/high/medium alerts exist) - check-code-scanning-alerts if: always() permissions: contents: read steps: - name: Harden the runner (Audit all outbound calls) uses: step-security/harden-runner@9af89fc71515a100421586dfdb3dc9c984fbf411 # v2.19.4 with: egress-policy: audit - name: Check security scan results run: | # Redirect all output to GITHUB_STEP_SUMMARY (fixes SC2129) exec >> "$GITHUB_STEP_SUMMARY" # Count results first FAILED="" PASS_COUNT=0 FAIL_COUNT=0 check_result() { local result="$1" if [ "$result" = "success" ]; then PASS_COUNT=$((PASS_COUNT + 1)) return 0 else FAIL_COUNT=$((FAIL_COUNT + 1)) FAILED="true" return 1 fi } # Check all results silently first check_result "${{ needs.security-tests.result }}" || true check_result "${{ needs.semgrep-scan.result }}" || true check_result "${{ needs.container-security.result }}" || true check_result "${{ needs.owasp-zap-scan.result }}" || true check_result "${{ needs.codeql-scan.result }}" || true check_result "${{ needs.devskim-scan.result }}" || true check_result "${{ needs.bearer-scan.result }}" || true check_result "${{ needs.npm-audit.result }}" || true check_result "${{ needs.retirejs-scan.result }}" || true check_result "${{ needs.grype-scan.result }}" || true # nuclei-scan temporarily disabled check_result "${{ needs.dockle-scan.result }}" || true check_result "${{ needs.hadolint-scan.result }}" || true check_result "${{ needs.checkov-scan.result }}" || true check_result "${{ needs.docker-multiarch-test.result }}" || true check_result "${{ needs.zizmor-scan.result }}" || true check_result "${{ needs.gitleaks-scan.result }}" || true check_result "${{ needs.security-headers-scan.result }}" || true check_result "${{ needs.license-check.result }}" || true check_result "${{ needs.backwards-compatibility.result }}" || true check_result "${{ needs.journal-data-integration.result }}" || true # CRITICAL — DO NOT REMOVE: without this line, broken wheels pass the gate check_result "${{ needs.pip-install-check.result }}" || true check_result "${{ needs.cpu-baseline-smoke.result }}" || true check_result "${{ needs.check-code-scanning-alerts.result }}" || true TOTAL=$((PASS_COUNT + FAIL_COUNT)) # ============================================ # BIG STATUS BANNER # ============================================ if [ -z "$FAILED" ]; then echo "# :white_check_mark: RELEASE GATE: PASSED" echo "" echo "> **All $TOTAL security scans passed successfully.**" echo "> This release is approved from a security perspective." else echo "# :x: RELEASE GATE: FAILED" echo "" echo "> **$FAIL_COUNT of $TOTAL checks failed.** Release is blocked." echo "> Review the failures below and fix before releasing." fi echo "" echo "---" echo "" echo "## Detailed Results" echo "" # Reset for detailed output FAILED="" # ============================================ # Existing scans # ============================================ echo "### Core Security Scans" if [ "${{ needs.security-tests.result }}" = "success" ]; then echo "✅ **Security Tests**: Passed" else echo "❌ **Security Tests**: ${{ needs.security-tests.result }}" FAILED="true" fi if [ "${{ needs.semgrep-scan.result }}" = "success" ]; then echo "✅ **Semgrep Scan**: Passed" else echo "❌ **Semgrep Scan**: ${{ needs.semgrep-scan.result }}" FAILED="true" fi if [ "${{ needs.container-security.result }}" = "success" ]; then echo "✅ **Container Security**: Passed" else echo "❌ **Container Security**: ${{ needs.container-security.result }}" FAILED="true" fi if [ "${{ needs.owasp-zap-scan.result }}" = "success" ]; then echo "✅ **OWASP ZAP Scan**: Passed" else echo "❌ **OWASP ZAP Scan**: ${{ needs.owasp-zap-scan.result }}" FAILED="true" fi # nuclei-scan temporarily disabled # ============================================ # SAST and code analysis # ============================================ echo "" echo "### SAST & Code Analysis" if [ "${{ needs.codeql-scan.result }}" = "success" ]; then echo "✅ **CodeQL**: Passed" else echo "❌ **CodeQL**: ${{ needs.codeql-scan.result }}" FAILED="true" fi if [ "${{ needs.devskim-scan.result }}" = "success" ]; then echo "✅ **DevSkim**: Passed" else echo "❌ **DevSkim**: ${{ needs.devskim-scan.result }}" FAILED="true" fi if [ "${{ needs.bearer-scan.result }}" = "success" ]; then echo "✅ **Bearer**: Passed" else echo "❌ **Bearer**: ${{ needs.bearer-scan.result }}" FAILED="true" fi # ============================================ # Dependency scanning # ============================================ echo "" echo "### Dependency Vulnerability Scanning" echo "ℹ️ **OSV-Scanner**: Runs on PRs (excluded from gate due to workflow nesting limits)" if [ "${{ needs.npm-audit.result }}" = "success" ]; then echo "✅ **npm Audit**: Passed" else echo "❌ **npm Audit**: ${{ needs.npm-audit.result }}" FAILED="true" fi if [ "${{ needs.retirejs-scan.result }}" = "success" ]; then echo "✅ **Retire.js**: Passed" else echo "❌ **Retire.js**: ${{ needs.retirejs-scan.result }}" FAILED="true" fi if [ "${{ needs.grype-scan.result }}" = "success" ]; then echo "✅ **Grype**: Passed" else echo "❌ **Grype**: ${{ needs.grype-scan.result }}" FAILED="true" fi # ============================================ # Container and IaC security # ============================================ echo "" echo "### Container & IaC Security" if [ "${{ needs.dockle-scan.result }}" = "success" ]; then echo "✅ **Dockle**: Passed" else echo "❌ **Dockle**: ${{ needs.dockle-scan.result }}" FAILED="true" fi if [ "${{ needs.hadolint-scan.result }}" = "success" ]; then echo "✅ **Hadolint**: Passed" else echo "❌ **Hadolint**: ${{ needs.hadolint-scan.result }}" FAILED="true" fi if [ "${{ needs.checkov-scan.result }}" = "success" ]; then echo "✅ **Checkov**: Passed" else echo "❌ **Checkov**: ${{ needs.checkov-scan.result }}" FAILED="true" fi if [ "${{ needs.docker-multiarch-test.result }}" = "success" ]; then echo "✅ **Docker Multiarch Build**: Passed" else echo "❌ **Docker Multiarch Build**: ${{ needs.docker-multiarch-test.result }}" FAILED="true" fi # ============================================ # Workflow security # ============================================ echo "" echo "### Workflow Security" if [ "${{ needs.zizmor-scan.result }}" = "success" ]; then echo "✅ **Zizmor**: Passed" else echo "❌ **Zizmor**: ${{ needs.zizmor-scan.result }}" FAILED="true" fi # ============================================ # Secret detection # ============================================ echo "" echo "### Secret Detection" if [ "${{ needs.gitleaks-scan.result }}" = "success" ]; then echo "✅ **Gitleaks**: Passed" else echo "❌ **Gitleaks**: ${{ needs.gitleaks-scan.result }}" FAILED="true" fi # ============================================ # Runtime security validation # ============================================ echo "" echo "### Runtime Security Validation" if [ "${{ needs.security-headers-scan.result }}" = "success" ]; then echo "✅ **Security Headers**: Passed" else echo "❌ **Security Headers**: ${{ needs.security-headers-scan.result }}" FAILED="true" fi # ============================================ # License compliance # ============================================ echo "" echo "### License Compliance" if [ "${{ needs.license-check.result }}" = "success" ]; then echo "✅ **License Check**: Passed" else echo "❌ **License Check**: ${{ needs.license-check.result }}" FAILED="true" fi # ============================================ # Backwards compatibility & migration tests # ============================================ echo "" echo "### Backwards Compatibility & Migrations" if [ "${{ needs.backwards-compatibility.result }}" = "success" ]; then echo "✅ **Backwards Compatibility**: Passed" else echo "❌ **Backwards Compatibility**: ${{ needs.backwards-compatibility.result }}" FAILED="true" fi # ============================================ # Journal-quality external data integration # Catches upstream schema breaks (OpenAlex/DOAJ/etc.) before # they ship in a release. # ============================================ echo "" echo "### External Data Integration" if [ "${{ needs.journal-data-integration.result }}" = "success" ]; then echo "✅ **Journal Data Integration**: Passed — all 5 sources downloaded and DB built" else echo "❌ **Journal Data Integration**: ${{ needs.journal-data-integration.result }}" echo " > Upstream API/format change broke the build. Check the test log for which source failed." FAILED="true" fi # ============================================ # pip install verification (CRITICAL — DO NOT REMOVE) # Without this block, a broken wheel can pass the release gate. # See pip-install-check job comments for the v1.4.0 incident. # ============================================ echo "" echo "### pip Install Verification" if [ "${{ needs.pip-install-check.result }}" = "success" ]; then echo "✅ **pip Install Check**: Passed — wheel installs cleanly with pip" else echo "❌ **pip Install Check**: ${{ needs.pip-install-check.result }} — ⚠️ WHEEL HAS BROKEN DEPENDENCIES" echo " > Check \`[tool.pdm.resolution.overrides]\` in pyproject.toml for conflicts that leak into wheel metadata" FAILED="true" fi # ============================================ # CPU baseline smoke (qemu) # Catches dependency wheels compiled past the supported x86 # baseline before they ship in a release (#4480). # ============================================ echo "" echo "### CPU Baseline Compatibility" if [ "${{ needs.cpu-baseline-smoke.result }}" = "success" ]; then echo "✅ **CPU Baseline Smoke**: Passed — native imports OK on SandyBridge (AVX floor) and Haswell (AVX2)" else echo "❌ **CPU Baseline Smoke**: ${{ needs.cpu-baseline-smoke.result }} — ⚠️ A DEPENDENCY WHEEL CRASHES ON OLDER CPUs" echo " > Some wheel executes instructions (e.g. AVX2) not present on the emulated CPU. See the job log for which module/CPU, and #4480 for the faiss-cpu precedent." FAILED="true" fi # ============================================ # Code scanning alerts check # ============================================ echo "" echo "### Code Scanning Alerts" if [ "${{ needs.check-code-scanning-alerts.result }}" = "success" ]; then echo "✅ **Alert Check**: No critical/high/medium alerts" else echo "❌ **Alert Check**: ${{ needs.check-code-scanning-alerts.result }} - Open alerts found!" FAILED="true" fi # ============================================ # Final result with prominent summary # ============================================ echo "" echo "---" echo "" if [ -n "$FAILED" ]; then echo "## :rotating_light: Action Required" echo "" echo "| Status | Result |" echo "|--------|--------|" echo "| **Gate** | :x: **BLOCKED** |" echo "| **Passed** | $PASS_COUNT |" echo "| **Failed** | $FAIL_COUNT |" echo "" echo "_Fix the failing checks above before releasing._" exit 1 else echo "## :tada: Ready for Release" echo "" echo "| Status | Result |" echo "|--------|--------|" echo "| **Gate** | :white_check_mark: **APPROVED** |" echo "| **Passed** | $PASS_COUNT / $TOTAL |" echo "" echo "_All security scans passed. E2E tests run as separate gate in release pipeline._" fi