"""Tests for IncidentCorrelator and CorrelatingSink.""" from __future__ import annotations from datetime import datetime, timedelta import pytest from integrations.hermes.correlating_sink import CorrelatingSink from integrations.hermes.correlator import ( DEFAULT_DEDUP_WINDOW_S, IncidentCorrelator, RouteDestination, correlate_all, default_routing_matrix, ) from integrations.hermes.incident import HermesIncident, IncidentSeverity, LogLevel, LogRecord def _record(seconds: int = 0) -> LogRecord: return LogRecord( timestamp=datetime(2026, 5, 12, 12, 0, 0) + timedelta(seconds=seconds), level=LogLevel.ERROR, logger="hermes.agent", message="boom", raw="ERROR hermes.agent: boom", ) def _incident( *, rule: str = "error_severity", severity: IncidentSeverity = IncidentSeverity.HIGH, fingerprint: str = "fp-1", seconds: int = 0, ) -> HermesIncident: return HermesIncident( rule=rule, severity=severity, title=f"{severity.value} from hermes.agent", detected_at=datetime(2026, 5, 12, 12, 0, 0) + timedelta(seconds=seconds), logger="hermes.agent", fingerprint=fingerprint, records=(_record(seconds=seconds),), ) class TestCorrelatorDedup: def test_first_incident_always_delivered(self) -> None: corr = IncidentCorrelator() decision = corr.correlate(_incident()) assert not decision.suppressed assert decision.repeat_count == 1 assert decision.escalated_from is None def test_second_within_dedup_window_is_suppressed(self) -> None: corr = IncidentCorrelator() corr.correlate(_incident(seconds=0)) decision = corr.correlate(_incident(seconds=10)) assert decision.suppressed assert decision.destination is RouteDestination.DROP def test_after_dedup_window_delivers_again(self) -> None: corr = IncidentCorrelator(dedup_window_s=30, escalation_window_s=15) corr.correlate(_incident(seconds=0)) decision = corr.correlate(_incident(seconds=60)) assert not decision.suppressed # The second incident is on its own in the escalation window. assert decision.repeat_count == 1 def test_different_fingerprints_do_not_dedupe(self) -> None: corr = IncidentCorrelator() corr.correlate(_incident(fingerprint="a")) decision = corr.correlate(_incident(fingerprint="b")) assert not decision.suppressed def test_idle_fingerprint_evicted_after_long_gap(self) -> None: corr = IncidentCorrelator(dedup_window_s=60, escalation_window_s=60, escalation_threshold=3) corr.correlate(_incident(fingerprint="a", seconds=0)) corr.correlate(_incident(fingerprint="b", seconds=800)) d = corr.correlate(_incident(fingerprint="a", seconds=801)) assert not d.suppressed assert d.repeat_count == 1 class TestCorrelatorEscalation: def test_escalates_after_threshold(self) -> None: corr = IncidentCorrelator(dedup_window_s=0, escalation_window_s=60, escalation_threshold=3) d1 = corr.correlate(_incident(seconds=0)) d2 = corr.correlate(_incident(seconds=10)) d3 = corr.correlate(_incident(seconds=20)) assert d1.escalated_from is None assert d2.escalated_from is None assert d3.escalated_from is IncidentSeverity.HIGH assert d3.deliver.severity is IncidentSeverity.CRITICAL assert "ESCALATED" in d3.deliver.title def test_critical_does_not_escalate_further(self) -> None: corr = IncidentCorrelator(dedup_window_s=0, escalation_window_s=60, escalation_threshold=2) corr.correlate(_incident(severity=IncidentSeverity.CRITICAL)) d2 = corr.correlate(_incident(severity=IncidentSeverity.CRITICAL, seconds=5)) # Repeat count triggers escalation logic but severity is already top. assert d2.deliver.severity is IncidentSeverity.CRITICAL assert d2.escalated_from is None def test_escalation_breaks_through_dedup(self) -> None: # Dedup window large; escalation threshold low → escalated incidents # should still be delivered. corr = IncidentCorrelator( dedup_window_s=300, escalation_window_s=60, escalation_threshold=3 ) corr.correlate(_incident(seconds=0)) corr.correlate(_incident(seconds=10)) decision = corr.correlate(_incident(seconds=20)) assert not decision.suppressed assert decision.escalated_from is IncidentSeverity.HIGH class TestCorrelatorRouting: def test_default_matrix_routes_crash_loop_to_pager(self) -> None: corr = IncidentCorrelator() decision = corr.correlate(_incident(rule="crash_loop")) assert decision.destination is RouteDestination.PAGER def test_unknown_rule_high_severity_goes_to_telegram(self) -> None: corr = IncidentCorrelator() decision = corr.correlate(_incident(rule="unknown_rule")) assert decision.destination is RouteDestination.TELEGRAM def test_unknown_rule_medium_drops(self) -> None: corr = IncidentCorrelator() decision = corr.correlate(_incident(rule="unknown_rule", severity=IncidentSeverity.MEDIUM)) assert decision.destination is RouteDestination.DROP def test_escalation_to_critical_promotes_telegram_to_pager(self) -> None: corr = IncidentCorrelator( dedup_window_s=0, escalation_window_s=60, escalation_threshold=2, routing_matrix={"warning_burst": RouteDestination.TELEGRAM}, ) corr.correlate( _incident( rule="warning_burst", severity=IncidentSeverity.HIGH, fingerprint="burst", ) ) d2 = corr.correlate( _incident( rule="warning_burst", severity=IncidentSeverity.HIGH, fingerprint="burst", seconds=10, ) ) assert d2.escalated_from is IncidentSeverity.HIGH # Escalated to CRITICAL → routing promoted from TELEGRAM to PAGER. assert d2.destination is RouteDestination.PAGER class TestCorrelatorValidation: def test_rejects_invalid_dedup_window(self) -> None: with pytest.raises(ValueError): IncidentCorrelator(dedup_window_s=-1) def test_rejects_zero_escalation_window(self) -> None: with pytest.raises(ValueError): IncidentCorrelator(escalation_window_s=0) def test_rejects_low_escalation_threshold(self) -> None: with pytest.raises(ValueError): IncidentCorrelator(escalation_threshold=1) class TestCorrelateAllAndDefaults: def test_correlate_all_batch(self) -> None: corr = IncidentCorrelator() decisions = correlate_all( corr, [ _incident(fingerprint="a"), _incident(fingerprint="b"), _incident(fingerprint="a", seconds=5), ], ) assert len(decisions) == 3 assert decisions[2].suppressed # dedup'd def test_default_routing_matrix_has_expected_rules(self) -> None: matrix = default_routing_matrix() assert matrix["crash_loop"] is RouteDestination.PAGER assert matrix["disk_full"] is RouteDestination.PAGER assert matrix["oom_killed"] is RouteDestination.TELEGRAM_WITH_RCA assert matrix["rate_limit"] is RouteDestination.TELEGRAM class TestCorrelatingSink: def test_delivers_to_routed_sink(self) -> None: delivered: list[HermesIncident] = [] corr = IncidentCorrelator() sink = CorrelatingSink( correlator=corr, routes={RouteDestination.TELEGRAM_WITH_RCA: delivered.append}, ) sink(_incident()) assert len(delivered) == 1 def test_suppressed_incident_is_not_delivered(self) -> None: delivered: list[HermesIncident] = [] corr = IncidentCorrelator() sink = CorrelatingSink( correlator=corr, routes={RouteDestination.TELEGRAM_WITH_RCA: delivered.append}, ) sink(_incident(seconds=0)) sink(_incident(seconds=10)) # within dedup window assert len(delivered) == 1 snapshot = sink.metrics_snapshot() assert snapshot["delivered"] == 1 assert snapshot["suppressed"] == 1 def test_missing_route_increments_unrouted_metric(self, caplog) -> None: corr = IncidentCorrelator() sink = CorrelatingSink(correlator=corr, routes={}) with caplog.at_level("WARNING", logger="integrations.hermes.correlating_sink"): sink(_incident()) snapshot = sink.metrics_snapshot() assert snapshot["delivered"] == 0 assert snapshot["unrouted"] == 1 assert any("no sink registered" in r.message for r in caplog.records) def test_downstream_sink_exception_does_not_propagate(self) -> None: def boom(_: HermesIncident) -> None: raise RuntimeError("downstream broke") corr = IncidentCorrelator() sink = CorrelatingSink(correlator=corr, routes={RouteDestination.TELEGRAM_WITH_RCA: boom}) # Must not raise: sink(_incident()) assert sink.metrics_snapshot()["sink_errors"] == 1 assert sink.metrics_snapshot()["delivered"] == 0 def test_dedup_window_default_constant_is_documented(self) -> None: # Catches anyone tightening the default below the AlarmDispatcher cooldown. assert DEFAULT_DEDUP_WINDOW_S == 300.0 def test_escalation_metric_tracks_correctly(self) -> None: delivered: list[HermesIncident] = [] corr = IncidentCorrelator(dedup_window_s=0, escalation_window_s=60, escalation_threshold=2) sink = CorrelatingSink( correlator=corr, routes={ RouteDestination.TELEGRAM_WITH_RCA: delivered.append, RouteDestination.PAGER: delivered.append, }, ) sink(_incident(seconds=0)) sink(_incident(seconds=5)) # escalates assert sink.metrics_snapshot()["escalated"] == 1 def test_escalated_incident_uses_distinct_fingerprint_key(self) -> None: """Greptile P1: escalated incidents must reach the sink with an ':escalated'-suffixed fingerprint so AlarmDispatcher's cooldown does not suppress them under the first-occurrence bucket.""" delivered: list[HermesIncident] = [] corr = IncidentCorrelator(dedup_window_s=0, escalation_window_s=60, escalation_threshold=2) sink = CorrelatingSink( correlator=corr, routes={ RouteDestination.TELEGRAM_WITH_RCA: delivered.append, RouteDestination.PAGER: delivered.append, }, ) sink(_incident(seconds=0)) # first occurrence — plain fingerprint sink(_incident(seconds=5)) # escalates assert len(delivered) == 2 fp_first = delivered[0].fingerprint fp_escalated = delivered[1].fingerprint assert not fp_first.endswith(":escalated"), "first occurrence must use plain fingerprint" assert fp_escalated == f"{fp_first}:escalated", ( "escalated incident must use ':escalated'-suffixed fingerprint" ) def test_close_resets_correlator_state(self) -> None: delivered: list[HermesIncident] = [] corr = IncidentCorrelator() sink = CorrelatingSink( correlator=corr, routes={RouteDestination.TELEGRAM_WITH_RCA: delivered.append}, ) sink(_incident(seconds=0)) sink(_incident(seconds=10)) # suppressed by dedup sink.close() sink(_incident(seconds=20)) assert len(delivered) == 2 def test_close_propagates_once_for_shared_downstream_sink(self) -> None: class _Closeable: def __init__(self) -> None: self.closed = 0 def __call__(self, _incident: HermesIncident) -> None: return None def close(self) -> None: self.closed += 1 closeable = _Closeable() corr = IncidentCorrelator() sink = CorrelatingSink( correlator=corr, routes={ RouteDestination.TELEGRAM: closeable, RouteDestination.TELEGRAM_WITH_RCA: closeable, }, default_route=closeable, ) sink.close() assert closeable.closed == 1 def test_close_calls_all_sinks_even_when_one_raises(self) -> None: """A raising close() on one downstream sink must not skip the others or leave _correlator.reset() uncalled (which would leak TelegramSink thread-pool workers).""" class _RaisesOnClose: def __call__(self, _incident: HermesIncident) -> None: return None def close(self) -> None: raise RuntimeError("close failed") class _Closeable: def __init__(self) -> None: self.closed = 0 def __call__(self, _incident: HermesIncident) -> None: return None def close(self) -> None: self.closed += 1 raiser = _RaisesOnClose() good = _Closeable() corr = IncidentCorrelator() sink = CorrelatingSink( correlator=corr, routes={ RouteDestination.TELEGRAM: raiser, RouteDestination.TELEGRAM_WITH_RCA: good, }, ) # Seed correlator state so we can verify reset() ran. sink(_incident(seconds=0)) with pytest.raises(RuntimeError, match="close failed"): sink.close() assert good.closed == 1, "good sink must be closed even when a sibling raises" def test_close_always_resets_correlator_even_when_sink_raises(self) -> None: """_correlator.reset() must run in the finally clause so dedup state is cleared even when a downstream close() raises.""" class _RaisesOnClose: def __call__(self, _incident: HermesIncident) -> None: return None def close(self) -> None: raise RuntimeError("sink boom") delivered: list[HermesIncident] = [] corr = IncidentCorrelator() sink = CorrelatingSink( correlator=corr, routes={ RouteDestination.TELEGRAM: _RaisesOnClose(), RouteDestination.TELEGRAM_WITH_RCA: delivered.append, }, ) sink(_incident(seconds=0)) sink(_incident(seconds=5)) # suppressed by dedup window with pytest.raises(RuntimeError): sink.close() # After close(), dedup state was reset so the next incident is not # suppressed. This proves reset() ran despite the exception. sink(_incident(seconds=20)) assert len(delivered) == 2, ( "incident after close must not be deduplicated — reset() must have run" )