tracer-cloud--opensre/tests/integrations/test_validator_sentry_capture.py

"""AST checks: every migrated broad-except in integrations calls report_validation_failure.

Issue #1470 acceptance criterion: every listed validator must invoke
``report_validation_failure`` on its broad ``except Exception`` block, with
tags carrying ``integration`` and ``method`` so Sentry events are filterable
by vendor.

This test parses each integration module and asserts, for every listed
``(function, line)`` pair, that:

1. The function's broad ``except Exception`` handler contains a call to
   ``report_validation_failure``.
2. The ``integration=`` and ``method=`` keyword arguments are string literals
   matching the expected tag values.

Two intentional exceptions are excluded — both are inner-loop ``except`` blocks
that ``continue`` or ``raise``; their failures already reach Sentry via the
outer except (or are deliberate version-detection fallbacks):

- ``mysql.get_replication_status`` inner ``stmt_err`` (MySQL <8.0.22 fallback)
- ``mariadb.get_replication_status`` inner ``stmt_err`` (analogous)
"""

from __future__ import annotations

import ast
from collections.abc import Iterator
from dataclasses import dataclass
from pathlib import Path

import pytest

_REPO_ROOT = Path(__file__).resolve().parents[2]


@dataclass(frozen=True)
class MigrationCase:
    module_path: str  # repo-relative file path
    function: str  # outer function name (may include "."-separated nested suffix)
    integration: str  # expected tag.integration
    method: str  # expected tag.method


CASES: tuple[MigrationCase, ...] = (
    # trello
    MigrationCase(
        "integrations/trello/verifier.py",
        "validate_trello_config",
        "trello",
        "validate_trello_config",
    ),
    # kafka
    MigrationCase(
        "integrations/kafka.py", "validate_kafka_config", "kafka", "validate_kafka_config"
    ),
    MigrationCase("integrations/kafka.py", "get_topic_health", "kafka", "get_topic_health"),
    MigrationCase(
        "integrations/kafka.py", "get_consumer_group_lag", "kafka", "get_consumer_group_lag"
    ),
    # clickhouse
    MigrationCase(
        "integrations/clickhouse.py",
        "validate_clickhouse_config",
        "clickhouse",
        "validate_clickhouse_config",
    ),
    MigrationCase(
        "integrations/clickhouse.py", "get_query_activity", "clickhouse", "get_query_activity"
    ),
    MigrationCase(
        "integrations/clickhouse.py", "get_system_health", "clickhouse", "get_system_health"
    ),
    MigrationCase("integrations/clickhouse.py", "get_table_stats", "clickhouse", "get_table_stats"),
    # github_mcp
    MigrationCase(
        "integrations/github/mcp.py",
        "validate_github_mcp_config",
        "github_mcp",
        "validate_github_mcp_config",
    ),
    # airflow (validate + loop site)
    MigrationCase(
        "integrations/airflow/config.py",
        "validate_airflow_config",
        "airflow",
        "validate_airflow_config",
    ),
    MigrationCase(
        "integrations/airflow/config.py",
        "get_recent_airflow_failures",
        "airflow",
        "get_recent_airflow_failures.task_instances",
    ),
    # posthog
    MigrationCase(
        "integrations/posthog/verifier.py",
        "validate_posthog_config",
        "posthog",
        "validate_posthog_config",
    ),
    # azure_sql
    MigrationCase(
        "integrations/azure_sql.py",
        "validate_azure_sql_config",
        "azure_sql",
        "validate_azure_sql_config",
    ),
    MigrationCase(
        "integrations/azure_sql.py", "get_server_status", "azure_sql", "get_server_status"
    ),
    MigrationCase(
        "integrations/azure_sql.py", "get_current_queries", "azure_sql", "get_current_queries"
    ),
    MigrationCase(
        "integrations/azure_sql.py", "get_resource_stats", "azure_sql", "get_resource_stats"
    ),
    MigrationCase("integrations/azure_sql.py", "get_slow_queries", "azure_sql", "get_slow_queries"),
    MigrationCase("integrations/azure_sql.py", "get_wait_stats", "azure_sql", "get_wait_stats"),
    # openclaw
    MigrationCase(
        "integrations/openclaw.py",
        "validate_openclaw_config",
        "openclaw",
        "validate_openclaw_config",
    ),
    # betterstack
    MigrationCase(
        "integrations/betterstack.py",
        "validate_betterstack_config",
        "betterstack",
        "validate_betterstack_config",
    ),
    MigrationCase("integrations/betterstack.py", "query_logs", "betterstack", "query_logs"),
    # gitlab
    MigrationCase(
        "integrations/gitlab.py", "validate_gitlab_config", "gitlab", "validate_gitlab_config"
    ),
    # bitbucket
    MigrationCase(
        "integrations/bitbucket.py",
        "validate_bitbucket_config",
        "bitbucket",
        "validate_bitbucket_config",
    ),
    MigrationCase("integrations/bitbucket.py", "list_commits", "bitbucket", "list_commits"),
    MigrationCase(
        "integrations/bitbucket.py", "get_file_contents", "bitbucket", "get_file_contents"
    ),
    MigrationCase("integrations/bitbucket.py", "search_code", "bitbucket", "search_code"),
    # mongodb
    MigrationCase(
        "integrations/mongodb.py",
        "validate_mongodb_config",
        "mongodb",
        "validate_mongodb_config",
    ),
    MigrationCase("integrations/mongodb.py", "get_server_status", "mongodb", "get_server_status"),
    MigrationCase("integrations/mongodb.py", "get_current_ops", "mongodb", "get_current_ops"),
    MigrationCase("integrations/mongodb.py", "get_rs_status", "mongodb", "get_rs_status"),
    MigrationCase("integrations/mongodb.py", "get_profiler_data", "mongodb", "get_profiler_data"),
    MigrationCase(
        "integrations/mongodb.py", "get_collection_stats", "mongodb", "get_collection_stats"
    ),
    # postgresql
    MigrationCase(
        "integrations/postgresql.py",
        "validate_postgresql_config",
        "postgresql",
        "validate_postgresql_config",
    ),
    MigrationCase(
        "integrations/postgresql.py", "get_server_status", "postgresql", "get_server_status"
    ),
    MigrationCase(
        "integrations/postgresql.py", "get_current_queries", "postgresql", "get_current_queries"
    ),
    MigrationCase(
        "integrations/postgresql.py",
        "get_replication_status",
        "postgresql",
        "get_replication_status",
    ),
    MigrationCase(
        "integrations/postgresql.py", "get_slow_queries", "postgresql", "get_slow_queries"
    ),
    MigrationCase("integrations/postgresql.py", "get_table_stats", "postgresql", "get_table_stats"),
    # mysql
    MigrationCase(
        "integrations/mysql.py", "validate_mysql_config", "mysql", "validate_mysql_config"
    ),
    MigrationCase("integrations/mysql.py", "get_server_status", "mysql", "get_server_status"),
    MigrationCase(
        "integrations/mysql.py", "get_current_processes", "mysql", "get_current_processes"
    ),
    MigrationCase(
        "integrations/mysql.py", "get_replication_status", "mysql", "get_replication_status"
    ),
    MigrationCase("integrations/mysql.py", "get_slow_queries", "mysql", "get_slow_queries"),
    MigrationCase("integrations/mysql.py", "get_table_stats", "mysql", "get_table_stats"),
    # mariadb
    MigrationCase(
        "integrations/mariadb.py",
        "validate_mariadb_config",
        "mariadb",
        "validate_mariadb_config",
    ),
    MigrationCase("integrations/mariadb.py", "get_process_list", "mariadb", "get_process_list"),
    MigrationCase("integrations/mariadb.py", "get_global_status", "mariadb", "get_global_status"),
    MigrationCase("integrations/mariadb.py", "get_innodb_status", "mariadb", "get_innodb_status"),
    MigrationCase("integrations/mariadb.py", "get_slow_queries", "mariadb", "get_slow_queries"),
    MigrationCase(
        "integrations/mariadb.py", "get_replication_status", "mariadb", "get_replication_status"
    ),
    # rabbitmq
    MigrationCase(
        "integrations/rabbitmq.py",
        "validate_rabbitmq_config",
        "rabbitmq",
        "validate_rabbitmq_config",
    ),
    MigrationCase("integrations/rabbitmq.py", "get_queue_backlog", "rabbitmq", "get_queue_backlog"),
    MigrationCase(
        "integrations/rabbitmq.py", "get_consumer_health", "rabbitmq", "get_consumer_health"
    ),
    MigrationCase(
        "integrations/rabbitmq.py", "get_broker_overview", "rabbitmq", "get_broker_overview"
    ),
    MigrationCase("integrations/rabbitmq.py", "get_node_health", "rabbitmq", "get_node_health"),
    MigrationCase(
        "integrations/rabbitmq.py", "get_connection_stats", "rabbitmq", "get_connection_stats"
    ),
    # mongodb_atlas
    MigrationCase(
        "integrations/mongodb_atlas.py",
        "validate_mongodb_atlas_config",
        "mongodb_atlas",
        "validate_mongodb_atlas_config",
    ),
    MigrationCase("integrations/mongodb_atlas.py", "get_clusters", "mongodb_atlas", "get_clusters"),
    MigrationCase("integrations/mongodb_atlas.py", "get_alerts", "mongodb_atlas", "get_alerts"),
    MigrationCase(
        "integrations/mongodb_atlas.py",
        "get_cluster_metrics",
        "mongodb_atlas",
        "get_cluster_metrics",
    ),
    MigrationCase(
        "integrations/mongodb_atlas.py",
        "get_performance_advisor",
        "mongodb_atlas",
        "get_performance_advisor",
    ),
    MigrationCase(
        "integrations/mongodb_atlas.py",
        "get_cluster_events",
        "mongodb_atlas",
        "get_cluster_events",
    ),
    # sentry (its own validator captures into OpenSRE's Sentry)
    MigrationCase(
        "integrations/sentry.py", "validate_sentry_config", "sentry", "validate_sentry_config"
    ),
    # adjacent
    MigrationCase(
        "integrations/daily_update.py",
        "summarize_highlights",
        "daily_update",
        "summarize_highlights",
    ),
    MigrationCase(
        "integrations/llm_cli/kimi.py",
        "_check_kimi_auth_fallback",
        "kimi",
        "_check_kimi_auth_fallback",
    ),
)


def _walk_funcs(tree: ast.AST) -> Iterator[ast.FunctionDef]:
    for node in ast.walk(tree):
        if isinstance(node, ast.FunctionDef):
            yield node


def _find_function(tree: ast.AST, name: str) -> ast.FunctionDef | None:
    for fn in _walk_funcs(tree):
        if fn.name == name:
            return fn
    return None


def _broad_except_handlers(fn: ast.FunctionDef) -> list[ast.ExceptHandler]:
    """Return the *outermost* ``except Exception`` handlers in the function body.

    Skips handlers that are themselves nested inside another broad-except — a
    `report_validation_failure` placed in an inner handler wouldn't compensate
    for a missing call in the outer one, which is the false-positive Greptile
    flagged in PR #1869.
    """
    all_handlers: list[ast.ExceptHandler] = []
    for node in ast.walk(fn):
        if isinstance(node, ast.ExceptHandler):
            t = node.type
            if isinstance(t, ast.Name) and t.id == "Exception":
                all_handlers.append(node)

    nested: set[int] = set()
    for outer in all_handlers:
        for descendant in ast.walk(outer):
            if descendant is outer:
                continue
            if isinstance(descendant, ast.ExceptHandler):
                t = descendant.type
                if isinstance(t, ast.Name) and t.id == "Exception":
                    nested.add(id(descendant))

    return [h for h in all_handlers if id(h) not in nested]


def _calls_to(handler: ast.ExceptHandler, func_name: str) -> list[ast.Call]:
    """Find ``func_name(...)`` calls inside an except handler body.

    Does not descend into nested ``except`` handler bodies — a call inside a
    nested broad-except does not satisfy coverage of the outer one.
    """
    matches: list[ast.Call] = []
    stack: list[ast.AST] = list(handler.body)
    while stack:
        node = stack.pop()
        if isinstance(node, ast.ExceptHandler):
            # Don't descend into another except's body; its calls aren't ours.
            continue
        if (
            isinstance(node, ast.Call)
            and isinstance(node.func, ast.Name)
            and node.func.id == func_name
        ):
            matches.append(node)
        stack.extend(ast.iter_child_nodes(node))
    return matches


def _kwarg_str(call: ast.Call, key: str) -> str | None:
    for kw in call.keywords:
        if kw.arg == key and isinstance(kw.value, ast.Constant) and isinstance(kw.value.value, str):
            return kw.value.value
    return None


def _source_path(module_path: str) -> Path:
    path = _REPO_ROOT / module_path
    if path.exists():
        return path
    if module_path.startswith("integrations/") and module_path.endswith(".py"):
        package_path = _REPO_ROOT / module_path.removesuffix(".py") / "__init__.py"
        if package_path.exists():
            return package_path
    return path


@pytest.mark.parametrize(
    "case",
    CASES,
    ids=lambda c: f"{Path(c.module_path).stem}::{c.function}",
)
def test_broad_except_calls_report_validation_failure(case: MigrationCase) -> None:
    source = _source_path(case.module_path).read_text(encoding="utf-8")
    tree = ast.parse(source)
    fn = _find_function(tree, case.function)
    assert fn is not None, f"function {case.function} not found in {case.module_path}"

    handlers = _broad_except_handlers(fn)
    assert handlers, f"no `except Exception` handlers in {case.module_path}::{case.function}"

    matching_calls: list[ast.Call] = []
    for h in handlers:
        for call in _calls_to(h, "report_validation_failure"):
            integration = _kwarg_str(call, "integration")
            method = _kwarg_str(call, "method")
            if integration == case.integration and method == case.method:
                matching_calls.append(call)

    assert matching_calls, (
        f"{case.module_path}::{case.function} has no `report_validation_failure` call "
        f"with integration={case.integration!r} and method={case.method!r}"
    )
    # Each tagged (integration, method) pair should appear exactly once per function
    # to prevent accidental double-capture in the same except block.
    assert len(matching_calls) == 1, (
        f"{case.module_path}::{case.function} reports the same "
        f"(integration={case.integration!r}, method={case.method!r}) "
        f"{len(matching_calls)} times; expected exactly once"
    )


def test_every_migrated_module_imports_the_helper() -> None:
    """Sanity guard: every file we touched should import report_validation_failure."""
    seen_modules = {case.module_path for case in CASES}
    for module_path in seen_modules:
        source = _source_path(module_path).read_text(encoding="utf-8")
        # Accept both a standalone import line and a combined import that also
        # brings in other helpers (e.g. report_classify_failure).
        imported = (
            "from integrations._validation_helpers import report_validation_failure" in source
            or (
                "from integrations._validation_helpers import" in source
                and "report_validation_failure" in source
            )
        )
        assert imported, (
            f"{module_path} migration is incomplete: missing import of report_validation_failure"
        )


def test_broad_except_handlers_skips_nested_handlers() -> None:
    """The helper must ignore broad-except handlers nested inside another broad-except.

    Otherwise a `report_validation_failure` call in the inner handler would
    falsely satisfy the assertion when the outer handler is uncovered (the
    false-positive path flagged by Greptile on PR #1869).
    """
    source = """
def f():
    try:
        do_thing()
    except Exception:
        try:
            cleanup()
        except Exception:
            report_validation_failure(err, integration="x", method="f")
"""
    tree = ast.parse(source)
    fn = _find_function(tree, "f")
    assert fn is not None
    handlers = _broad_except_handlers(fn)
    assert len(handlers) == 1, "should return only the outer handler"
    # The call exists only in the inner handler; helper must not surface it.
    assert _calls_to(handlers[0], "report_validation_failure") == []


def test_broad_except_handlers_finds_handler_inside_normal_control_flow() -> None:
    """Handlers inside for/while/if are still surfaced — only nested-except is excluded.

    Airflow's per-DAG-run capture lives inside a ``for`` loop's try block; the
    helper must not regress that case.
    """
    source = """
def f():
    for x in items:
        try:
            work(x)
        except Exception:
            report_validation_failure(err, integration="x", method="f")
            continue
"""
    tree = ast.parse(source)
    fn = _find_function(tree, "f")
    assert fn is not None
    handlers = _broad_except_handlers(fn)
    assert len(handlers) == 1
    assert _calls_to(handlers[0], "report_validation_failure")