tracer-cloud--opensre/tests/hermes/test_correlating_sink.py

"""Multi-channel delivery tests for :class:`CorrelatingSink`.

The correlator/sink logic itself (dedup, escalation, metrics, close
semantics) is covered by ``test_correlator.py``. This module focuses on the
*delivery* seam: a :class:`CorrelatingSink` fanning incidents out to more than
one downstream channel, with the real vendor transports mocked.

Hermes only ships a :class:`~integrations.hermes.sinks.TelegramSink` today and
its :class:`~integrations.hermes.correlator.RouteDestination` set has no SLACK
or DISCORD members. So, we exercise multi-channel fan-out the way a real
deployment would wire extra channels *now*: by registering thin, test-local
channel adapters on existing destinations and asserting each vendor transport
is (or is not) called with the right payload.

Transports mocked:

* Telegram → ``integrations.telegram.alarms.post_telegram_message``
  (reached through the production :class:`TelegramSink`/:class:`AlarmDispatcher`).
* Slack    → ``integrations.slack.delivery.send_slack_webhook_message``.
* Discord  → ``integrations.discord.delivery.post_discord_message``.

The Slack/Discord adapters are deliberately small — just enough to prove
routing reaches the vendor transport. They are test fixtures, not a proposal
for production Hermes sinks.
"""

from __future__ import annotations

from datetime import datetime, timedelta
from typing import Any

import pytest

from integrations.discord import delivery as discord_delivery
from integrations.hermes.correlating_sink import CorrelatingSink
from integrations.hermes.correlator import IncidentCorrelator, RouteDestination
from integrations.hermes.incident import HermesIncident, IncidentSeverity, LogLevel, LogRecord
from integrations.hermes.sinks import TelegramSink
from integrations.slack import delivery as slack_delivery
from integrations.telegram.alarms import AlarmDispatcher
from integrations.telegram.credentials import TelegramCredentials

_TS = datetime(2026, 5, 12, 12, 0, 0)


# ---------------------------------------------------------------------------
# Fixtures / builders


def _record(seconds: int = 0) -> LogRecord:
    ts = _TS + timedelta(seconds=seconds)
    return LogRecord(
        timestamp=ts,
        level=LogLevel.ERROR,
        logger="hermes.agent",
        message="boom",
        raw=f"{ts.isoformat()} ERROR hermes.agent: boom",
    )


def _incident(
    *,
    rule: str = "error_severity",
    severity: IncidentSeverity = IncidentSeverity.HIGH,
    fingerprint: str = "fp-1",
    title: str = "ERROR from hermes.agent",
    seconds: int = 0,
) -> HermesIncident:
    return HermesIncident(
        rule=rule,
        severity=severity,
        title=title,
        detected_at=_TS + timedelta(seconds=seconds),
        logger="hermes.agent",
        fingerprint=fingerprint,
        records=(_record(seconds=seconds),),
    )


class _Channels:
    """Captured calls for each mocked vendor transport."""

    def __init__(self) -> None:
        self.telegram: list[dict[str, Any]] = []
        self.slack: list[dict[str, Any]] = []
        self.discord: list[dict[str, Any]] = []

    @property
    def counts(self) -> tuple[int, int, int]:
        return len(self.telegram), len(self.slack), len(self.discord)


def _mock_channels(monkeypatch: pytest.MonkeyPatch) -> _Channels:
    """Patch all three vendor transports and record their calls."""
    channels = _Channels()

    def _fake_telegram(
        chat_id: str,
        text: str,
        bot_token: str,
        parse_mode: str = "",
        reply_to_message_id: str = "",
        reply_markup: dict[str, Any] | None = None,
    ) -> tuple[bool, str, str]:
        channels.telegram.append({"chat_id": chat_id, "text": text, "bot_token": bot_token})
        return True, "", "1"

    def _fake_slack(text: str, **kwargs: Any) -> tuple[bool, str]:
        channels.slack.append({"text": text, "kwargs": kwargs})
        return True, ""

    def _fake_discord(
        channel_id: str,
        embeds: list[dict[str, Any]],
        bot_token: str,
        content: str = "",
    ) -> tuple[bool, str, str]:
        channels.discord.append({"channel_id": channel_id, "content": content, "embeds": embeds})
        return True, "", "42"

    monkeypatch.setattr("integrations.telegram.alarms.post_telegram_message", _fake_telegram)
    monkeypatch.setattr(slack_delivery, "send_slack_webhook_message", _fake_slack)
    monkeypatch.setattr(discord_delivery, "post_discord_message", _fake_discord)
    return channels


def _telegram_sink() -> TelegramSink:
    creds = TelegramCredentials(bot_token="tok", chat_id="chat-1")
    # cooldown 0 so dedup is driven purely by the correlator, not the dispatcher.
    return TelegramSink(AlarmDispatcher(creds, cooldown_seconds=0.0))


def _slack_sink() -> Any:
    """A minimal Slack channel adapter: format + post via the vendor transport.

    Resolved at call time (module attribute access) so ``monkeypatch.setattr``
    on ``slack_delivery.send_slack_webhook_message`` is honoured.
    """

    def _sink(incident: HermesIncident) -> None:
        text = f"[{incident.severity.value.upper()}] {incident.title} ({incident.rule})"
        slack_delivery.send_slack_webhook_message(text, webhook_url="https://hooks.example/x")

    return _sink


def _discord_sink() -> Any:
    def _sink(incident: HermesIncident) -> None:
        content = f"[{incident.severity.value.upper()}] {incident.title} ({incident.rule})"
        discord_delivery.post_discord_message(
            channel_id="chan-1", embeds=[], bot_token="bot-tok", content=content
        )

    return _sink


def _three_channel_sink(
    monkeypatch: pytest.MonkeyPatch,
    *,
    correlator: IncidentCorrelator | None = None,
) -> tuple[CorrelatingSink, _Channels]:
    channels = _mock_channels(monkeypatch)
    corr = correlator if correlator is not None else IncidentCorrelator()
    sink = CorrelatingSink(
        correlator=corr,
        routes={
            RouteDestination.TELEGRAM_WITH_RCA: _telegram_sink(),
            RouteDestination.TELEGRAM: _slack_sink(),
            RouteDestination.PAGER: _discord_sink(),
        },
    )
    return sink, channels


# ---------------------------------------------------------------------------
# Fan-out routing


class TestChannelFanOut:
    def test_rca_route_hits_only_telegram(self, monkeypatch: pytest.MonkeyPatch) -> None:
        sink, channels = _three_channel_sink(monkeypatch)

        # error_severity → TELEGRAM_WITH_RCA in the default matrix.
        sink(_incident(rule="error_severity", severity=IncidentSeverity.HIGH))

        assert channels.counts == (1, 0, 0)
        assert "Hermes incident" in channels.telegram[0]["text"]

    def test_warning_burst_route_hits_only_slack(self, monkeypatch: pytest.MonkeyPatch) -> None:
        sink, channels = _three_channel_sink(monkeypatch)

        # warning_burst → TELEGRAM (wired here to the Slack adapter).
        sink(
            _incident(
                rule="warning_burst",
                severity=IncidentSeverity.MEDIUM,
                fingerprint="fp-warn",
            )
        )

        assert channels.counts == (0, 1, 0)
        assert "warning_burst" in channels.slack[0]["text"]
        assert channels.slack[0]["kwargs"]["webhook_url"] == "https://hooks.example/x"

    def test_pager_route_hits_only_discord(self, monkeypatch: pytest.MonkeyPatch) -> None:
        sink, channels = _three_channel_sink(monkeypatch)

        # crash_loop → PAGER (wired here to the Discord adapter).
        sink(
            _incident(
                rule="crash_loop",
                severity=IncidentSeverity.HIGH,
                fingerprint="fp-crash",
            )
        )

        assert channels.counts == (0, 0, 1)
        assert "crash_loop" in channels.discord[0]["content"]

    def test_distinct_incidents_fan_out_to_distinct_channels(
        self, monkeypatch: pytest.MonkeyPatch
    ) -> None:
        """A mixed batch must land each incident on exactly its routed channel
        and leave the others untouched."""
        sink, channels = _three_channel_sink(monkeypatch)

        sink(_incident(rule="error_severity", severity=IncidentSeverity.HIGH, fingerprint="a"))
        sink(_incident(rule="warning_burst", severity=IncidentSeverity.MEDIUM, fingerprint="b"))
        sink(_incident(rule="crash_loop", severity=IncidentSeverity.HIGH, fingerprint="c"))

        assert channels.counts == (1, 1, 1)
        assert sink.metrics_snapshot()["delivered"] == 3


class TestNonDeliveringDecisions:
    def test_dropped_incident_touches_no_channel(self, monkeypatch: pytest.MonkeyPatch) -> None:
        sink, channels = _three_channel_sink(monkeypatch)

        # Unknown rule + MEDIUM severity → DROP in the fallback route.
        sink(_incident(rule="unknown_rule", severity=IncidentSeverity.MEDIUM))

        assert channels.counts == (0, 0, 0)
        assert sink.metrics_snapshot()["dropped"] == 1

    def test_suppressed_duplicate_touches_no_channel(self, monkeypatch: pytest.MonkeyPatch) -> None:
        sink, channels = _three_channel_sink(monkeypatch)

        sink(_incident(fingerprint="dup", seconds=0))
        sink(_incident(fingerprint="dup", seconds=10))  # within dedup window

        assert channels.counts == (1, 0, 0)
        snap = sink.metrics_snapshot()
        assert snap["delivered"] == 1
        assert snap["suppressed"] == 1

    def test_unrouted_destination_touches_no_channel(self, monkeypatch: pytest.MonkeyPatch) -> None:
        """A PAGER-routed incident with no PAGER handler must be counted
        unrouted and reach none of the wired channels."""
        channels = _mock_channels(monkeypatch)
        corr = IncidentCorrelator()
        sink = CorrelatingSink(
            correlator=corr,
            routes={
                RouteDestination.TELEGRAM_WITH_RCA: _telegram_sink(),
                RouteDestination.TELEGRAM: _slack_sink(),
                # PAGER intentionally omitted.
            },
        )

        sink(_incident(rule="crash_loop", severity=IncidentSeverity.HIGH, fingerprint="fp-x"))

        assert channels.counts == (0, 0, 0)
        assert sink.metrics_snapshot()["unrouted"] == 1

    def test_default_route_catches_unregistered_destination(
        self, monkeypatch: pytest.MonkeyPatch
    ) -> None:
        channels = _mock_channels(monkeypatch)
        corr = IncidentCorrelator()
        sink = CorrelatingSink(
            correlator=corr,
            routes={RouteDestination.TELEGRAM_WITH_RCA: _telegram_sink()},
            default_route=_discord_sink(),
        )

        # crash_loop → PAGER, unregistered, so the default (Discord) catches it.
        sink(_incident(rule="crash_loop", severity=IncidentSeverity.HIGH, fingerprint="fp-d"))

        assert channels.counts == (0, 0, 1)
        assert sink.metrics_snapshot()["delivered"] == 1


class TestChannelIsolation:
    def test_one_channel_raising_does_not_starve_the_others(
        self, monkeypatch: pytest.MonkeyPatch
    ) -> None:
        """A downstream channel that raises must be isolated: its incident is
        counted as a sink error, but incidents routed to healthy channels keep
        flowing."""
        channels = _mock_channels(monkeypatch)

        def _boom_slack(incident: HermesIncident) -> None:
            raise RuntimeError("slack webhook exploded")

        corr = IncidentCorrelator()
        sink = CorrelatingSink(
            correlator=corr,
            routes={
                RouteDestination.TELEGRAM_WITH_RCA: _telegram_sink(),
                RouteDestination.TELEGRAM: _boom_slack,
            },
        )

        # Route to the broken Slack channel first…
        sink(_incident(rule="warning_burst", severity=IncidentSeverity.MEDIUM, fingerprint="w"))
        # …then to the healthy Telegram channel.
        sink(_incident(rule="error_severity", severity=IncidentSeverity.HIGH, fingerprint="e"))

        assert channels.counts == (1, 0, 0)  # telegram delivered, slack never captured
        snap = sink.metrics_snapshot()
        assert snap["sink_errors"] == 1
        assert snap["delivered"] == 1

    def test_escalated_fingerprint_reaches_downstream_transport(
        self, monkeypatch: pytest.MonkeyPatch
    ) -> None:
        """Escalation must break through dedup AND carry the ':escalated'
        fingerprint all the way to the Telegram transport so a downstream
        AlarmDispatcher cooldown cannot re-suppress it. We assert two distinct
        deliveries reached the wire for the same underlying event."""
        corr = IncidentCorrelator(dedup_window_s=0, escalation_window_s=60, escalation_threshold=2)
        channels = _mock_channels(monkeypatch)
        # A dedicated dispatcher with a real cooldown; distinct fingerprints
        # are what let the second (escalated) send through.
        creds = TelegramCredentials(bot_token="tok", chat_id="chat-1")
        telegram = TelegramSink(AlarmDispatcher(creds, cooldown_seconds=300.0))
        sink = CorrelatingSink(
            correlator=corr,
            routes={
                RouteDestination.TELEGRAM_WITH_RCA: telegram,
                RouteDestination.PAGER: telegram,
            },
        )

        sink(_incident(fingerprint="esc", seconds=0))  # first, plain fingerprint
        sink(_incident(fingerprint="esc", seconds=5))  # escalates → CRITICAL → PAGER

        # Both reached the transport: the cooldown did not swallow the second
        # because its fingerprint was suffixed with ':escalated'.
        assert len(channels.telegram) == 2


class TestCloseFanOut:
    def test_close_closes_every_channel_and_resets_dedup(
        self, monkeypatch: pytest.MonkeyPatch
    ) -> None:
        channels = _mock_channels(monkeypatch)

        closed: list[str] = []

        class _ClosableSlack:
            def __call__(self, incident: HermesIncident) -> None:
                slack_delivery.send_slack_webhook_message(incident.title)

            def close(self) -> None:
                closed.append("slack")

        corr = IncidentCorrelator()
        sink = CorrelatingSink(
            correlator=corr,
            routes={
                RouteDestination.TELEGRAM_WITH_RCA: _telegram_sink(),
                RouteDestination.TELEGRAM: _ClosableSlack(),
            },
        )

        sink(_incident(fingerprint="c", seconds=0))
        sink(_incident(fingerprint="c", seconds=10))  # suppressed by dedup
        assert channels.counts == (1, 0, 0)

        sink.close()
        assert closed == ["slack"]

        # After close(), correlator dedup state is cleared, so the same
        # fingerprint delivers again instead of being suppressed.
        sink(_incident(fingerprint="c", seconds=20))
        assert channels.counts == (2, 0, 0)