tracer-cloud--opensre/tests/integrations/test_catalog_silent_fallback_elimination.py

"""Tests for #1468: eliminate silent env-loader / classify fallbacks.

Three patterns previously dropped exceptions to ``(None, None)``, ``pass``,
or ``logger.debug(..., exc_info=True)`` with no Sentry trace:

  A. ``_classify_service_instance`` per-vendor ``try/except Exception``
     blocks (33 sites covering grafana .. supabase).
  B. ``load_env_integrations`` argocd + helm ``except Exception: pass``.
  C. ``load_env_integrations`` debug-only env loaders (incident_io,
     openclaw, mariadb, rabbitmq, rds, betterstack, alertmanager,
     victoria_logs, supabase).

After the fix every site routes through ``report_classify_failure`` (in
``integrations._validation_helpers``) or ``_report_env_loader_failure``
(in ``integrations._catalog_impl``), which call ``report_exception`` with
``surface=integration``, ``event=classify_failed`` / ``event=env_loader_failed``,
and the vendor tag — preserving the historic "skip the integration" caller
contract.
"""

from __future__ import annotations

import os
from typing import Any
from unittest.mock import patch

import pytest

from integrations._catalog_impl import (
    _classify_service_instance,
    _report_env_loader_failure,
    load_env_integrations,
)
from integrations._validation_helpers import report_classify_failure


@pytest.fixture(autouse=True)
def _quiet_sentry(monkeypatch: pytest.MonkeyPatch) -> None:
    monkeypatch.setenv("OPENSRE_NO_TELEMETRY", "1")


# ---------------------------------------------------------------------------
# Helper smoke tests
# ---------------------------------------------------------------------------


def test_report_classify_failure_forwards_tags() -> None:
    exc = ValueError("bad config")
    with patch("integrations._validation_helpers.report_exception") as mock_report:
        report_classify_failure(
            exc,
            logger=__import__("logging").getLogger(__name__),
            integration="datadog",
            record_id="rec-1",
        )
    mock_report.assert_called_once()
    kwargs = mock_report.call_args.kwargs
    assert kwargs["severity"] == "warning"
    assert kwargs["tags"] == {
        "surface": "integration",
        "component": "integrations",
        "integration": "datadog",
        "event": "classify_failed",
    }
    assert kwargs["extras"] == {"record_id": "rec-1"}


def test_report_env_loader_failure_forwards_tags() -> None:
    exc = RuntimeError("env missing")
    with patch("integrations._catalog_impl.report_exception") as mock_report:
        _report_env_loader_failure(exc, integration="rabbitmq")
    mock_report.assert_called_once()
    kwargs = mock_report.call_args.kwargs
    assert kwargs["severity"] == "warning"
    assert kwargs["tags"] == {
        "surface": "integration",
        "component": "integrations._catalog_impl",
        "integration": "rabbitmq",
        "event": "env_loader_failed",
    }


# ---------------------------------------------------------------------------
# Pattern A: _classify_service_instance routes failures to Sentry
# ---------------------------------------------------------------------------


# Per-vendor: the module path and symbol whose constructor we force to raise,
# so we can prove the surrounding try/except now routes the failure through
# ``report_exception`` instead of returning ``(None, None)`` silently.
_CLASSIFY_PATCH_TARGETS: list[tuple[str, str, str]] = [
    ("grafana", "integrations.grafana", "GrafanaIntegrationConfig"),
    ("aws", "integrations.aws", "AWSIntegrationConfig"),
    ("datadog", "integrations.datadog", "DatadogIntegrationConfig"),
    ("honeycomb", "integrations.honeycomb", "HoneycombIntegrationConfig"),
    ("coralogix", "integrations.coralogix", "CoralogixIntegrationConfig"),
    ("github", "integrations.github.mcp", "build_github_mcp_config"),
    ("sentry", "integrations.sentry", "build_sentry_config"),
    ("gitlab", "integrations.gitlab", "build_gitlab_config"),
    ("mongodb", "integrations.mongodb", "build_mongodb_config"),
    ("postgresql", "integrations.postgresql", "build_postgresql_config"),
    ("mongodb_atlas", "integrations.mongodb_atlas", "build_mongodb_atlas_config"),
    ("mariadb", "integrations.mariadb", "build_mariadb_config"),
    ("vercel", "integrations.vercel", "VercelConfig"),
    ("opsgenie", "integrations.opsgenie", "OpsGenieIntegrationConfig"),
    ("incident_io", "integrations.incident_io", "IncidentIoIntegrationConfig"),
    ("jira", "integrations.jira", "JiraIntegrationConfig"),
    ("discord", "integrations.discord", "DiscordBotConfig"),
    ("telegram", "integrations.telegram", "TelegramBotConfig"),
    ("openclaw", "integrations.openclaw", "build_openclaw_config"),
    ("mysql", "integrations.mysql", "build_mysql_config"),
    ("rabbitmq", "integrations.rabbitmq", "build_rabbitmq_config"),
    ("rds", "integrations.rds", "build_rds_config"),
    ("airflow", "integrations.airflow.config", "build_airflow_config"),
    ("betterstack", "integrations.betterstack", "build_betterstack_config"),
    ("azure_sql", "integrations.azure_sql", "build_azure_sql_config"),
    ("alertmanager", "integrations.alertmanager", "AlertmanagerIntegrationConfig"),
    ("argocd", "integrations.argocd", "ArgoCDIntegrationConfig"),
    ("helm", "integrations.helm", "HelmIntegrationConfig"),
    ("victoria_logs", "integrations.victoria_logs", "VictoriaLogsIntegrationConfig"),
    ("splunk", "integrations.splunk", "SplunkIntegrationConfig"),
    ("supabase", "integrations.supabase", "build_supabase_config"),
    ("smtp", "integrations.smtp", "SMTPIntegrationConfig"),
]


@pytest.mark.parametrize(("integration", "patch_module", "patch_symbol"), _CLASSIFY_PATCH_TARGETS)
def test_classify_failure_skips_integration_and_reports(
    integration: str,
    patch_module: str,
    patch_symbol: str,
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    """Every Pattern A site (``_classify_service_instance``) must still return
    ``(None, None)`` *and* now route the exception through
    ``report_exception`` with ``event=classify_failed`` and the vendor tag."""

    def _boom(*_args: Any, **_kwargs: Any) -> None:
        raise RuntimeError(f"forced {integration} failure")

    monkeypatch.setattr(f"{patch_module}.{patch_symbol}", _boom)

    with patch("integrations._validation_helpers.report_exception") as mock_report:
        result = _classify_service_instance(
            integration,
            {"endpoint": "https://x", "api_key": "k", "bot_token": "fake-token"},
            record_id=f"rec-{integration}",
        )

    assert result == (None, None), (
        f"caller contract broken for {integration}: must still skip integration"
    )
    assert mock_report.call_count == 1, (
        f"{integration} silently swallowed exception (no report_exception call)"
    )
    tags = mock_report.call_args.kwargs["tags"]
    assert tags["integration"] == integration
    assert tags["event"] == "classify_failed"
    assert tags["surface"] == "integration"
    assert mock_report.call_args.kwargs["severity"] == "warning"
    assert mock_report.call_args.kwargs["extras"]["record_id"] == f"rec-{integration}"


@pytest.mark.parametrize("bot_token", ["", " ", None])
@pytest.mark.parametrize("integration", ["discord", "telegram"])
def test_classify_empty_bot_token_silently_skips(
    integration: str,
    bot_token: str | None,
) -> None:
    """Discord/Telegram records with empty bot_token are silently skipped without
    reporting to Sentry — empty token means 'not configured', not misconfigured."""
    credentials: dict[str, Any] = {"endpoint": "https://x"}
    if bot_token is not None:
        credentials["bot_token"] = bot_token

    with patch("integrations._validation_helpers.report_exception") as mock_report:
        result = _classify_service_instance(
            integration,
            credentials,
            record_id="rec-test",
        )

    assert result == (None, None)
    assert mock_report.call_count == 0, (
        f"{integration} with empty bot_token should not report to Sentry"
    )


# ---------------------------------------------------------------------------
# Pattern B + C: load_env_integrations routes loader failures to Sentry
# ---------------------------------------------------------------------------


# For each env-loader case: env vars to enable the loader path + the symbol in
# ``integrations._catalog_impl`` to patch so its constructor raises. This
# proves the exception is *caught* and *routed* — without depending on which
# fields the real builder rejects (validator details drift independently).
_ENV_LOADER_CASES: list[tuple[str, dict[str, str], str]] = [
    # Pattern B
    (
        "argocd",
        {"ARGOCD_BASE_URL": "https://argo.example", "ARGOCD_AUTH_TOKEN": "t"},
        "ArgoCDIntegrationConfig",
    ),
    (
        "helm",
        {"OSRE_HELM_INTEGRATION": "1"},
        "HelmIntegrationConfig",
    ),
    # Pattern C
    (
        "incident_io",
        {"INCIDENT_IO_API_KEY": "k"},
        "IncidentIoIntegrationConfig",
    ),
    (
        "mariadb",
        {"MARIADB_HOST": "h", "MARIADB_DATABASE": "d"},
        "build_mariadb_config",
    ),
    (
        "rabbitmq",
        {"RABBITMQ_HOST": "h", "RABBITMQ_USERNAME": "u"},
        "build_rabbitmq_config",
    ),
    (
        "betterstack",
        {"BETTERSTACK_QUERY_ENDPOINT": "https://bs.example", "BETTERSTACK_USERNAME": "u"},
        "build_betterstack_config",
    ),
    (
        "alertmanager",
        {"ALERTMANAGER_URL": "https://am.example"},
        "AlertmanagerIntegrationConfig",
    ),
    (
        "victoria_logs",
        {"VICTORIA_LOGS_URL": "https://vl.example"},
        "VictoriaLogsIntegrationConfig",
    ),
    (
        "supabase",
        {"SUPABASE_URL": "https://s.example", "SUPABASE_SERVICE_KEY": "k"},
        "build_supabase_config",
    ),
    (
        "openclaw",
        {"OPENCLAW_MCP_URL": "https://oc.example"},
        "build_openclaw_config",
    ),
    (
        "rds",
        {},
        "rds_config_from_env",
    ),
    # Pattern C — added in this PR (was the #2036 follow-up scope).
    (
        "vercel",
        {"VERCEL_API_TOKEN": "t"},
        "VercelConfig",
    ),
    (
        "opsgenie",
        {"OPSGENIE_API_KEY": "k"},
        "OpsGenieIntegrationConfig",
    ),
    (
        "jira",
        {
            "JIRA_BASE_URL": "https://jira.example",
            "JIRA_EMAIL": "e@example.com",
            "JIRA_API_TOKEN": "t",
        },
        "JiraIntegrationConfig",
    ),
    (
        "discord",
        {"DISCORD_BOT_TOKEN": "t"},
        "DiscordBotConfig",
    ),
    (
        "telegram",
        {"TELEGRAM_BOT_TOKEN": "t"},
        "TelegramBotConfig",
    ),
    (
        "mongodb_atlas",
        {
            "MONGODB_ATLAS_PUBLIC_KEY": "pub",
            "MONGODB_ATLAS_PRIVATE_KEY": "priv",
            "MONGODB_ATLAS_PROJECT_ID": "proj",
        },
        "build_mongodb_atlas_config",
    ),
    # Early env-loader blocks that #1468 never guarded — a malformed cred here
    # aborted discovery of every later integration (the #2036 class of bug).
    (
        "grafana",
        {"GRAFANA_INSTANCE_URL": "https://g.example", "GRAFANA_READ_TOKEN": "t"},
        "GrafanaIntegrationConfig",
    ),
    (
        "datadog",
        {"DD_API_KEY": "k", "DD_APP_KEY": "a"},
        "DatadogIntegrationConfig",
    ),
    (
        "honeycomb",
        {"HONEYCOMB_API_KEY": "k"},
        "HoneycombIntegrationConfig",
    ),
    (
        "coralogix",
        {"CORALOGIX_API_KEY": "k"},
        "CoralogixIntegrationConfig",
    ),
    (
        "aws",
        {"AWS_ROLE_ARN": "arn:aws:iam::123456789012:role/x"},
        "AWSIntegrationConfig",
    ),
    (
        # The access-key path is a separate elif branch with its own guard.
        "aws",
        {"AWS_ACCESS_KEY_ID": "key", "AWS_SECRET_ACCESS_KEY": "secret"},
        "AWSIntegrationConfig",
    ),
    (
        "whatsapp",
        {
            "TWILIO_ACCOUNT_SID": "sid",
            "TWILIO_AUTH_TOKEN": "tok",
            "TWILIO_WHATSAPP_FROM": "+15551234567",
        },
        "WhatsAppConfig",
    ),
    (
        "splunk",
        {"SPLUNK_URL": "https://s.example", "SPLUNK_TOKEN": "t"},
        "SplunkIntegrationConfig",
    ),
    # Stragglers still on the pre-#1468 swallow pattern: signoz/jenkins/tempo
    # logged only at debug (exc_info=True); twilio dropped to None silently.
    (
        "signoz",
        {},
        "signoz_config_from_env",
    ),
    (
        "jenkins",
        {},
        "jenkins_config_from_env",
    ),
    (
        "tempo",
        {},
        "tempo_config_from_env",
    ),
    (
        "twilio",
        {
            "TWILIO_ACCOUNT_SID": "sid",
            "TWILIO_AUTH_TOKEN": "tok",
            "TWILIO_SMS_FROM": "+15551234567",
        },
        "TwilioIntegrationConfig",
    ),
]


@pytest.mark.parametrize(("integration", "env", "patch_symbol"), _ENV_LOADER_CASES)
def test_env_loader_failure_reports_and_skips(
    integration: str,
    env: dict[str, str],
    patch_symbol: str,
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    """Pattern B + C: when a vendor's env-derived builder raises, the
    integration must be skipped (preserving the historic caller contract)
    *and* the failure must reach Sentry via ``report_exception``."""
    # Clear ambient env so other integrations on the dev box don't enable
    # unrelated paths and inflate the assertion.
    for var in list(os.environ):
        monkeypatch.delenv(var, raising=False)
    for var, value in env.items():
        monkeypatch.setenv(var, value)

    def _boom(*_args: Any, **_kwargs: Any) -> None:
        raise RuntimeError(f"forced {integration} failure")

    monkeypatch.setattr(f"integrations._catalog_impl.{patch_symbol}", _boom)

    with patch("integrations._catalog_impl.report_exception") as mock_report:
        result = load_env_integrations()

    services = {entry["service"] for entry in result}
    assert integration not in services, f"{integration} must be skipped when its env loader fails"

    matching = [
        call
        for call in mock_report.call_args_list
        if call.kwargs.get("tags", {}).get("integration") == integration
    ]
    assert matching, (
        f"{integration} env-loader failure was swallowed silently — expected "
        "report_exception call with event=env_loader_failed"
    )
    tags = matching[0].kwargs["tags"]
    assert tags["event"] == "env_loader_failed"
    assert tags["surface"] == "integration"
    assert matching[0].kwargs["severity"] == "warning"


def test_one_failing_env_loader_does_not_abort_remaining_integrations(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    """The regression #2036 was filed for: a single failing ``model_validate``
    in ``load_env_integrations`` must not abort discovery for every later
    vendor. The survivor must run *after* the failing loader — otherwise the
    assertion passes even when the loop aborts on vercel (caught by
    @VibhorGautam in review).

    incident_io is loaded after vercel in ``load_env_integrations``, so its
    presence in the result proves the loop continued past the vercel failure.
    """
    for var in list(os.environ):
        monkeypatch.delenv(var, raising=False)
    monkeypatch.setenv("VERCEL_API_TOKEN", "tkn")
    monkeypatch.setenv("INCIDENT_IO_API_KEY", "tkn")

    def _boom(*_args: Any, **_kwargs: Any) -> None:
        raise RuntimeError("forced vercel failure")

    monkeypatch.setattr("integrations._catalog_impl.VercelConfig", _boom)

    with patch("integrations._catalog_impl.report_exception"):
        result = load_env_integrations()

    services = {entry["service"] for entry in result}
    assert "vercel" not in services
    assert "incident_io" in services, (
        "incident_io was dropped — discovery aborted on the vercel failure instead of "
        "continuing past it (this is exactly the bug #2036 is about)"
    )