"""Tests for the CI audit scripts. Covers `scripts/check_workflow_permissions.py` and `scripts/check_coverage_thresholds.py`. """ from __future__ import annotations import shlex import subprocess import sys import textwrap from pathlib import Path import pytest pytestmark = pytest.mark.repo_lint REPO_ROOT = Path(__file__).resolve().parents[2] SCRIPTS = REPO_ROOT / "scripts" def _run(args: list[str]) -> subprocess.CompletedProcess[str]: return subprocess.run( [sys.executable, *args], capture_output=True, text=True, cwd=REPO_ROOT, check=False, timeout=30, ) # --------------------------------------------------------------------------- # check_workflow_permissions.py # --------------------------------------------------------------------------- def test_workflow_permissions_passes_on_real_state(): """Current .github/workflows passes the check.""" result = _run([str(SCRIPTS / "check_workflow_permissions.py")]) assert result.returncode == 0, f"stderr: {result.stderr}\nstdout: {result.stdout}" def test_workflow_permissions_detects_missing_block(tmp_path): """Synthetic workflow without top-level permissions → exit 1.""" wf_dir = tmp_path / "workflows" wf_dir.mkdir() (wf_dir / "bad.yml").write_text( textwrap.dedent( """\ name: Bad on: push: branches: [main] jobs: x: runs-on: ubuntu-latest steps: - run: echo hi """ ) ) result = _run([str(SCRIPTS / "check_workflow_permissions.py"), "--workflow-dir", str(wf_dir)]) assert result.returncode == 1 assert "bad.yml" in result.stderr @pytest.mark.parametrize( "body", [ "permissions: write-all\n", # inline string value "permissions: read-all\n", # inline — still not a scoped block "permissions: {}\n", # inline empty map "permissions:\n contents: write\n", # non-read value "permissions:\n contents: read\n issues: write\n", # mixed "permissions:\n unknown-scope: read\n", # unknown scope ], ) def test_workflow_permissions_rejects_bypass_attempts(tmp_path, body): """Inline strings, write-all, mixed scopes, and unknown scopes all fail.""" wf_dir = tmp_path / "workflows" wf_dir.mkdir() (wf_dir / "sneaky.yml").write_text( "name: Sneaky\non: push\n" + body + "jobs:\n x:\n runs-on: ubuntu-latest\n" ) result = _run([str(SCRIPTS / "check_workflow_permissions.py"), "--workflow-dir", str(wf_dir)]) assert result.returncode == 1 assert "sneaky.yml" in result.stderr def test_workflow_permissions_accepts_quoted_values(tmp_path): """`contents: 'read'` and `contents: "read"` are valid YAML and must pass.""" wf_dir = tmp_path / "workflows" wf_dir.mkdir() (wf_dir / "quoted.yml").write_text( "name: Quoted\non: push\n" "permissions:\n" " contents: 'read'\n" ' issues: "none"\n' "jobs:\n x:\n runs-on: ubuntu-latest\n" ) result = _run([str(SCRIPTS / "check_workflow_permissions.py"), "--workflow-dir", str(wf_dir)]) assert result.returncode == 0, f"stderr: {result.stderr}" def test_workflow_permissions_globs_yaml_extension(tmp_path): """`.yaml` files are also scanned (GitHub Actions accepts both).""" wf_dir = tmp_path / "workflows" wf_dir.mkdir() (wf_dir / "missing.yaml").write_text( "name: NoPerms\non: push\njobs:\n x:\n runs-on: ubuntu-latest\n" ) result = _run([str(SCRIPTS / "check_workflow_permissions.py"), "--workflow-dir", str(wf_dir)]) assert result.returncode == 1 assert "missing.yaml" in result.stderr def test_workflow_permissions_allowlist(tmp_path): """codeql.yml is allowlisted; a workflow without block but named codeql.yml passes.""" wf_dir = tmp_path / "workflows" wf_dir.mkdir() (wf_dir / "codeql.yml").write_text( textwrap.dedent( """\ name: CodeQL on: push jobs: analyze: runs-on: ubuntu-latest permissions: security-events: write steps: - run: echo hi """ ) ) result = _run([str(SCRIPTS / "check_workflow_permissions.py"), "--workflow-dir", str(wf_dir)]) assert result.returncode == 0 def test_verify_package_e2e_retry_is_scoped_to_last_failed_e2e_tests(): """The protected E2E retry must fail closed instead of running an unintended suite.""" workflow = REPO_ROOT / ".github" / "workflows" / "verify-package.yml" text = workflow.read_text(encoding="utf-8") marker = "Retry failed E2E tests after 10-min cool-down" assert marker in text retry_step = text.split(marker, 1)[1].split("\n - name:", 1)[0] retry_args = shlex.split(retry_step) assert "uv run pytest tests/e2e" in retry_step assert "--last-failed" in retry_args assert "--last-failed-no-failures=none" in retry_step assert "pytest --last-failed -v" not in retry_step # --------------------------------------------------------------------------- # check_coverage_thresholds.py # --------------------------------------------------------------------------- def test_coverage_thresholds_passes_on_real_state(): """Current pyproject.toml + test.yml agree on the coverage threshold.""" result = _run([str(SCRIPTS / "check_coverage_thresholds.py")]) assert result.returncode == 0, f"stderr: {result.stderr}\nstdout: {result.stdout}" def test_coverage_thresholds_ignores_commented_cov_fail_under(tmp_path): """A commented `# --cov-fail-under=70` must not shadow the active value.""" pyproject = tmp_path / "pyproject.toml" pyproject.write_text( textwrap.dedent( """\ [tool.coverage.report] fail_under = 90 """ ) ) workflow = tmp_path / "test.yml" # The commented line should be ignored; the live line agrees with pyproject. workflow.write_text( "jobs:\n" " x:\n" " steps:\n" " # Historical: --cov-fail-under=70\n" " - run: pytest --cov-fail-under=90\n" ) result = _run( [ str(SCRIPTS / "check_coverage_thresholds.py"), "--pyproject", str(pyproject), "--workflow", str(workflow), ] ) assert result.returncode == 0, f"stderr: {result.stderr}" def test_coverage_thresholds_catches_divergent_second_occurrence(tmp_path): """Multiple --cov-fail-under occurrences must all match (not just the first).""" pyproject = tmp_path / "pyproject.toml" pyproject.write_text("[tool.coverage.report]\nfail_under = 90\n") workflow = tmp_path / "test.yml" # First occurrence matches; second diverges. Old re.search would miss it. workflow.write_text( "jobs:\n" " a:\n steps:\n - run: pytest --cov-fail-under=90\n" " b:\n steps:\n - run: pytest --cov-fail-under=70\n" ) result = _run( [ str(SCRIPTS / "check_coverage_thresholds.py"), "--pyproject", str(pyproject), "--workflow", str(workflow), ] ) assert result.returncode == 1 assert "DRIFT" in result.stderr assert "--cov-fail-under=70" in result.stderr def test_coverage_thresholds_detects_drift(tmp_path): """Synthetic mismatched thresholds → exit 1 with drift message.""" pyproject = tmp_path / "pyproject.toml" pyproject.write_text( textwrap.dedent( """\ [tool.coverage.report] fail_under = 90 """ ) ) workflow = tmp_path / "test.yml" workflow.write_text("jobs:\n x:\n steps:\n - run: pytest --cov-fail-under=70\n") result = _run( [ str(SCRIPTS / "check_coverage_thresholds.py"), "--pyproject", str(pyproject), "--workflow", str(workflow), ] ) assert result.returncode == 1 assert "DRIFT" in result.stderr assert "fail_under=90" in result.stderr assert "--cov-fail-under=70" in result.stderr