learningcircuit--local-deep-research/.pre-commit-hooks/check-weak-tests.py

#!/usr/bin/env python3
"""
Pre-commit hook to detect tautological / no-op test functions.

These patterns pass regardless of whether the system-under-test (SUT) works,
so they provide false confidence and inflate coverage without catching
regressions. A 2026 test-suite audit removed ~3,700 such tests
(see docs/processes/test-review/README.md); this hook prevents reintroduction.

Patterns flagged (each is *never* a legitimate assertion):

  1. ASSERT_TRUE — a test whose only assertion(s) are ``assert True``
     (or ``assert <constant-truthy-literal>``). Verifies nothing.

  2. IMPORT_EXISTENCE — body is exactly ``from M import X`` + ``assert X is
     not None``. The import already raises ImportError if X is missing, so
     the assert pins nothing the import didn't already.

  3. TAUTOLOGY_OR — ``assert <anything> or True`` (always True).

  4. NULLCHECK_TAUTOLOGY — ``assert x is None or x is not None`` (or the
     reverse ordering) on the same operand. Accepts every possible value.

Opt-out: add ``# allow: weak-test`` on the ``def`` line or the assert line.
Use sparingly and only with a comment explaining why the pattern is
intentional (e.g. a deliberately-minimal smoke test).
"""

import ast
import re
import sys

SUPPRESS_RE = re.compile(r"#\s*allow:\s*weak-test(?:\s|$)")


def _strip_docstring(body: list[ast.stmt]) -> list[ast.stmt]:
    if (
        body
        and isinstance(body[0], ast.Expr)
        and isinstance(body[0].value, ast.Constant)
        and isinstance(body[0].value.value, str)
    ):
        return body[1:]
    return body


def _is_truthy_constant(node: ast.expr) -> bool:
    """True for ``True`` / non-empty literal constants that are always truthy."""
    return isinstance(node, ast.Constant) and bool(node.value)


def _is_none(node: ast.expr) -> bool:
    return isinstance(node, ast.Constant) and node.value is None


def _operand_src(node: ast.expr) -> str:
    try:
        return ast.unparse(node)
    except Exception:  # pragma: no cover - unparse is stable on 3.9+
        return repr(node)


def _is_nullcheck_tautology(test: ast.expr) -> bool:
    """Detect ``x is None or x is not None`` (either ordering, same operand)."""
    if not (isinstance(test, ast.BoolOp) and isinstance(test.op, ast.Or)):
        return False
    if len(test.values) != 2:
        return False
    left, right = test.values
    if not (isinstance(left, ast.Compare) and isinstance(right, ast.Compare)):
        return False
    if not (len(left.ops) == 1 and len(right.ops) == 1):
        return False
    left_is = isinstance(left.ops[0], ast.Is)
    left_isnot = isinstance(left.ops[0], ast.IsNot)
    right_is = isinstance(right.ops[0], ast.Is)
    right_isnot = isinstance(right.ops[0], ast.IsNot)
    # Both comparisons must be against None
    if not (_is_none(left.comparators[0]) and _is_none(right.comparators[0])):
        return False
    # Same operand on both sides
    if _operand_src(left.left) != _operand_src(right.left):
        return False
    # One "is None" and one "is not None" => covers all values
    return (left_is and right_isnot) or (left_isnot and right_is)


def _has_or_true(test: ast.expr) -> bool:
    """Detect ``<anything> or True`` (or ``True or <anything>``)."""
    if not (isinstance(test, ast.BoolOp) and isinstance(test.op, ast.Or)):
        return False
    return any(_is_truthy_constant(v) for v in test.values)


class WeakTestChecker(ast.NodeVisitor):
    def __init__(self, filepath: str, lines: list[str]):
        self.filepath = filepath
        self.lines = lines
        self.issues: list[tuple[int, str]] = []

    def _suppressed(self, lineno: int) -> bool:
        idx = lineno - 1
        return 0 <= idx < len(self.lines) and bool(
            SUPPRESS_RE.search(self.lines[idx])
        )

    def _has_skip_marker(self, node: ast.FunctionDef) -> bool:
        for dec in node.decorator_list:
            src = _operand_src(dec)
            if "skip" in src or "xfail" in src:
                return True
        return False

    def visit_FunctionDef(self, node: ast.FunctionDef) -> None:
        if node.name.startswith("test_") and not self._has_skip_marker(node):
            self._check_test(node)
        self.generic_visit(node)

    def _check_test(self, node: ast.FunctionDef) -> None:
        # Function-level opt-out (on the def line).
        if self._suppressed(node.lineno):
            return

        body = _strip_docstring(node.body)
        asserts = [s for s in body if isinstance(s, ast.Assert)]

        # Pattern 3 & 4: per-assert tautologies (anywhere in the body).
        for a in asserts:
            if self._suppressed(a.lineno):
                continue
            if _has_or_true(a.test):
                self.issues.append(
                    (a.lineno, "assertion is always true (`... or True`)")
                )
            elif _is_nullcheck_tautology(a.test):
                self.issues.append(
                    (
                        a.lineno,
                        "assertion `x is None or x is not None` accepts any value",
                    )
                )

        if not asserts:
            # Pattern 2: import-existence tautology (no Assert means it can't
            # match here; handled below only when an assert exists).
            return

        # Pattern 1: every assertion is a truthy literal (e.g. `assert True`).
        if all(_is_truthy_constant(a.test) for a in asserts):
            if not self._suppressed(node.lineno):
                self.issues.append(
                    (
                        node.lineno,
                        f"test '{node.name}' has no real assertion "
                        "(only `assert <truthy-literal>`)",
                    )
                )

        # Pattern 2: body is exactly import(s) + `assert X is not None`.
        non_import = [
            s for s in body if not isinstance(s, (ast.Import, ast.ImportFrom))
        ]
        if (
            len(non_import) == 1
            and isinstance(non_import[0], ast.Assert)
            and len(body) > 1  # at least one import present
        ):
            test = non_import[0].test
            if (
                isinstance(test, ast.Compare)
                and len(test.ops) == 1
                and isinstance(test.ops[0], ast.IsNot)
                and _is_none(test.comparators[0])
            ):
                if not self._suppressed(node.lineno):
                    self.issues.append(
                        (
                            node.lineno,
                            f"test '{node.name}' is an import-existence "
                            "tautology (the import already fails if the "
                            "symbol is missing)",
                        )
                    )


def check_file(filepath: str) -> list[tuple[int, str]]:
    try:
        with open(filepath, encoding="utf-8") as fh:
            source = fh.read()
    except (OSError, UnicodeDecodeError):
        return []
    try:
        tree = ast.parse(source)
    except SyntaxError:
        # Leave syntax errors to other tools (ruff / py_compile).
        return []
    checker = WeakTestChecker(filepath, source.splitlines())
    checker.visit(tree)
    return sorted(checker.issues)


def main(argv: list[str]) -> int:
    files = [
        f
        for f in argv
        if f.endswith(".py")
        and ("/test_" in f or f.startswith("test_") or "/tests/" in f)
    ]
    found = False
    for filepath in files:
        for lineno, message in check_file(filepath):
            found = True
            print(f"{filepath}:{lineno}: {message}")
    if found:
        print(
            "\nWeak/tautological test patterns detected (see above).\n"
            "These pass regardless of whether the code works. Either assert a "
            "real outcome, or — if the no-op is intentional (e.g. a minimal "
            "smoke test) — add `# allow: weak-test` with a justifying comment.\n"
            "Background: docs/processes/test-review/README.md"
        )
        return 1
    return 0


if __name__ == "__main__":
    sys.exit(main(sys.argv[1:]))