learningcircuit--local-deep-research/tests/ci/test_release_gate_integrity.py

"""
CRITICAL — DO NOT MODIFY OR DELETE THIS FILE

These tests verify that the release gate workflow contains the pip install
verification job and all its required wiring. This job prevents publishing
broken wheels to PyPI.

Background (v1.4.0 incident):
  PDM resolution overrides (pyproject.toml [tool.pdm.resolution.overrides])
  leak into wheel metadata when `pdm build` runs. This creates dependency
  constraints that pip cannot resolve, making `pip install local-deep-research`
  fail for ALL users. PDM installs work fine because PDM handles overrides
  differently, so the problem is invisible during development and CI — it
  only surfaces after publishing to PyPI, when it's too late.

  Version 1.4.0 shipped with requests>=2.33 in its metadata (from a PDM
  override for a CVE fix), but arxiv~=2.4 requires requests~=2.32.0.
  pip could not resolve this conflict. Every user who ran
  `pip install local-deep-research` got an error.

Why guardian tests:
  The pip-install-check job in release-gate.yml is the ONLY thing that
  catches this class of bug before it reaches users. If someone removes
  it (e.g. to "simplify" the workflow, reduce CI time, or fix an
  unrelated failure), the safety net disappears silently. These tests
  make that removal loud and obvious.

DO NOT:
  - Delete this file
  - Mark these tests as xfail or skip
  - Weaken the assertions (e.g. checking substring instead of exact job name)
  - Move the pip-install-check job to a non-blocking position in the workflow
"""

from pathlib import Path

import yaml
import pytest

RELEASE_GATE = (
    Path(__file__).resolve().parents[2]
    / ".github"
    / "workflows"
    / "release-gate.yml"
)


@pytest.fixture(scope="module")
def workflow():
    """Load the release-gate workflow YAML.

    CRITICAL — DO NOT MODIFY: this fixture is used by all guardian tests below.
    """
    assert RELEASE_GATE.exists(), (
        f"release-gate.yml not found at {RELEASE_GATE}. "
        "If you moved it, update the path — do NOT delete the tests."
    )
    return yaml.safe_load(RELEASE_GATE.read_text())


class TestPipInstallCheckJobExists:
    """
    CRITICAL — DO NOT DELETE OR WEAKEN THESE TESTS.

    They verify that the pip-install-check job exists in the release gate
    and cannot be published without catching PDM override leaks.
    See module docstring for the full incident report.
    """

    def test_pip_install_check_job_is_defined(self, workflow):
        """The pip-install-check job must exist in release-gate.yml.

        CRITICAL — DO NOT REMOVE: without this job, broken wheels reach PyPI.
        """
        jobs = workflow.get("jobs", {})
        assert "pip-install-check" in jobs, (
            "CRITICAL: 'pip-install-check' job is missing from release-gate.yml! "
            "This job prevents publishing wheels with broken dependencies to PyPI. "
            "It was removed — this MUST be restored before any release. "
            "See tests/ci/test_release_gate_integrity.py docstring for background."
        )

    def test_pip_install_check_is_in_summary_needs(self, workflow):
        """The summary job must depend on pip-install-check so failures block releases.

        CRITICAL — DO NOT REMOVE from needs: removing the dependency means the
        release gate passes even when pip install fails.
        """
        summary_job = workflow.get("jobs", {}).get("release-gate-summary", {})
        needs = summary_job.get("needs", [])
        assert "pip-install-check" in needs, (
            "CRITICAL: 'pip-install-check' is missing from release-gate-summary.needs! "
            "The release gate will pass even if pip install fails. "
            "Add it back to the needs list."
        )

    def test_pip_install_check_has_no_continue_on_error(self, workflow):
        """The pip-install-check job must NOT have continue-on-error.

        CRITICAL — DO NOT ADD continue-on-error: it would make pip install
        failures non-blocking, allowing broken wheels to be published.
        """
        job = workflow.get("jobs", {}).get("pip-install-check", {})
        assert job.get("continue-on-error") is not True, (
            "CRITICAL: pip-install-check has continue-on-error: true! "
            "This makes dependency failures non-blocking — broken wheels "
            "will be published to PyPI. Remove continue-on-error."
        )

    def test_pip_install_check_uses_pip_not_pdm(self, workflow):
        """The verification step must use pip, not pdm.

        CRITICAL — DO NOT CHANGE to pdm install: PDM resolves overrides
        differently and will NOT catch the dependency conflicts that break
        pip users. The whole point is to test what pip users experience.
        """
        job = workflow.get("jobs", {}).get("pip-install-check", {})
        steps = job.get("steps", [])

        # Find the verification step
        verify_step = None
        for step in steps:
            run = step.get("run", "")
            if "pip install" in run and "no-cache-dir" in run:
                verify_step = step
                break

        assert verify_step is not None, (
            "CRITICAL: Cannot find the pip install verification step in pip-install-check! "
            "The step must run 'pip install --no-cache-dir' on the built wheel."
        )

        # Make sure it doesn't use pdm install
        run_content = verify_step.get("run", "")
        assert "pdm install" not in run_content, (
            "CRITICAL: Verification step uses 'pdm install' instead of 'pip install'! "
            "PDM handles overrides differently and won't catch dependency conflicts. "
            "Use 'pip install' to simulate what end users experience."
        )

    def test_pip_install_check_does_not_use_no_deps(self, workflow):
        """The pip install must NOT use --no-deps.

        CRITICAL — DO NOT ADD --no-deps: it skips dependency resolution
        entirely, which is the exact thing we're trying to verify.
        """
        job = workflow.get("jobs", {}).get("pip-install-check", {})
        steps = job.get("steps", [])

        for step in steps:
            run = step.get("run", "")
            if "pip install" in run and ".whl" in run:
                # Check actual pip install commands, not comments
                for line in run.splitlines():
                    stripped = line.strip()
                    if stripped.startswith("#"):
                        continue
                    if "pip install" in stripped and "--no-deps" in stripped:
                        raise AssertionError(
                            "CRITICAL: pip install command uses --no-deps! "
                            "This skips dependency resolution — the exact check we need. "
                            "Remove --no-deps."
                        )

    def test_pip_install_check_runs_pip_check(self, workflow):
        """The verification must run `pip check` to catch inconsistencies.

        CRITICAL — DO NOT REMOVE: pip check catches dependency version
        mismatches that pip install might not flag during resolution.
        """
        job = workflow.get("jobs", {}).get("pip-install-check", {})
        steps = job.get("steps", [])

        has_pip_check = False
        for step in steps:
            if "pip check" in step.get("run", ""):
                has_pip_check = True
                break

        assert has_pip_check, (
            "CRITICAL: pip-install-check job does not run 'pip check'! "
            "This catches dependency inconsistencies after installation. "
            "Add 'pip check' to the verification step."
        )