"""Foreground watchdog loop and alarm formatting."""
from __future__ import annotations
import html
import socket
import time
from collections import deque
from collections.abc import Callable
from dataclasses import dataclass
from datetime import UTC, datetime
from typing import Protocol
import click
from integrations.telegram.alarms import AlarmDispatcher
from integrations.telegram.credentials import load_credentials_from_env
from platform.common.exit_codes import ERROR, SUCCESS
from tools.system.watch_dog.config import WatchdogConfig
from tools.system.watch_dog.process_monitor import ProcessMonitor, ProcessSample, Sampler
class Dispatcher(Protocol):
"""Minimal alarm dispatcher protocol used by the runner tests."""
def dispatch(self, threshold_name: str, message: str) -> bool:
"""Dispatch one threshold alarm."""
@dataclass(frozen=True)
class ThresholdBreach:
"""A threshold violation observed for a sample."""
name: str
limit: float
observed: float
window_seconds: float | None = None
def run_watchdog(
config: WatchdogConfig,
*,
sampler: Sampler | None = None,
dispatcher: Dispatcher | None = None,
_sleep: Callable[[float], None] = time.sleep,
_clock: Callable[[], float] = time.monotonic,
) -> int:
"""Run the watchdog loop until target exit, SIGINT, or --once alarm."""
active_sampler = sampler or ProcessMonitor(config)
active_dispatcher = dispatcher or _build_dispatcher(config)
cpu_window = _CpuWindow()
try:
while True:
sample = active_sampler.sample()
if not sample.alive:
if config.verbose:
click.echo(f"watchdog: target exited pid={sample.pid}")
return SUCCESS
now = _clock()
breaches = _evaluate_thresholds(config, sample, cpu_window=cpu_window, now=now)
if config.verbose:
click.echo(_format_sample_log(sample, breaches))
for breach in breaches:
active_dispatcher.dispatch(
breach.name,
_format_alarm_message(sample, breach),
)
if breaches and config.once:
return ERROR
_sleep(config.interval)
except KeyboardInterrupt:
return SUCCESS
def _build_dispatcher(config: WatchdogConfig) -> AlarmDispatcher:
creds = load_credentials_from_env(chat_id_override=config.chat_id)
return AlarmDispatcher(creds, cooldown_seconds=config.cooldown, parse_mode="HTML")
def _evaluate_thresholds(
config: WatchdogConfig,
sample: ProcessSample,
*,
cpu_window: _CpuWindow,
now: float,
) -> tuple[ThresholdBreach, ...]:
breaches: list[ThresholdBreach] = []
if config.max_cpu is not None:
observed_cpu = cpu_window.add(now, sample.cpu_percent, window_seconds=config.cpu_window)
if observed_cpu >= config.max_cpu:
breaches.append(
ThresholdBreach(
name="max_cpu",
limit=config.max_cpu,
observed=observed_cpu,
window_seconds=config.cpu_window,
)
)
if config.max_runtime is not None and sample.runtime_seconds >= config.max_runtime:
breaches.append(
ThresholdBreach(
name="max_runtime",
limit=config.max_runtime,
observed=sample.runtime_seconds,
)
)
if config.max_rss is not None and sample.rss_bytes >= config.max_rss:
breaches.append(
ThresholdBreach(
name="max_rss",
limit=float(config.max_rss),
observed=float(sample.rss_bytes),
)
)
return tuple(breaches)
class _CpuWindow:
"""Rolling CPU average over a time window."""
def __init__(self) -> None:
self._samples: deque[tuple[float, float]] = deque()
def add(self, now: float, value: float, *, window_seconds: float) -> float:
self._samples.append((now, value))
oldest_allowed = now - window_seconds
while len(self._samples) > 1 and self._samples[0][0] < oldest_allowed:
self._samples.popleft()
return sum(cpu for _, cpu in self._samples) / len(self._samples)
def _format_sample_log(
sample: ProcessSample,
breaches: tuple[ThresholdBreach, ...],
) -> str:
status = "alarm" if breaches else "ok"
return (
"watchdog: "
f"status={status} pid={sample.pid} name={sample.name} "
f"cpu={sample.cpu_percent:.1f}% rss={_format_bytes(sample.rss_bytes)} "
f"runtime={_format_duration(sample.runtime_seconds)}"
)
def _format_alarm_message(sample: ProcessSample, breach: ThresholdBreach) -> str:
started = "-"
if sample.started_at is not None:
started = datetime.fromtimestamp(sample.started_at, tz=UTC).isoformat()
started = started.replace("+00:00", "Z")
command = sample.command or "-"
if len(command) > 180:
command = f"{command[:177]}..."
return "\n".join(
[
"🚨 OpenSRE Watchdog Alarm",
f"host {html.escape(socket.gethostname())}",
f"pid {sample.pid} "
f"({html.escape(sample.name or '-')})",
f"cmd {html.escape(command)}",
f"threshold {html.escape(_format_threshold_breach(breach))}",
f"runtime {html.escape(_format_duration(sample.runtime_seconds))}",
f"started {html.escape(started)}",
]
)
def _format_threshold_breach(breach: ThresholdBreach) -> str:
if breach.name == "max_cpu":
window = f" window={_format_duration(breach.window_seconds or 0)}"
return f"max_cpu limit={breach.limit:.1f}% observed={breach.observed:.1f}%{window}"
if breach.name == "max_runtime":
return (
"max_runtime "
f"limit={_format_duration(breach.limit)} "
f"observed={_format_duration(breach.observed)}"
)
if breach.name == "max_rss":
return (
"max_rss "
f"limit={_format_bytes(breach.limit)} "
f"observed={_format_bytes(breach.observed)}"
)
return f"{breach.name} limit={breach.limit} observed={breach.observed}"
def _format_duration(seconds: float) -> str:
remaining = max(0, int(round(seconds)))
hours, remaining = divmod(remaining, 3600)
minutes, secs = divmod(remaining, 60)
if hours:
return f"{hours}h{minutes:02d}m{secs:02d}s"
if minutes:
return f"{minutes}m{secs:02d}s"
return f"{secs}s"
def _format_bytes(value: float) -> str:
amount = float(value)
for unit in ("B", "KiB", "MiB", "GiB", "TiB"):
if abs(amount) < 1024.0 or unit == "TiB":
if unit == "B":
return f"{int(amount)}B"
return f"{amount:.1f}{unit}"
amount /= 1024.0
return f"{amount:.1f}TiB"