import asyncio import logging import time from typing import Dict, Optional from ray.serve._private.constants import ( RAY_SERVE_EVENT_LOOP_MONITORING_INTERVAL_S, SERVE_EVENT_LOOP_LATENCY_HISTOGRAM_BOUNDARIES_MS, SERVE_LOGGER_NAME, ) from ray.util import metrics logger = logging.getLogger(SERVE_LOGGER_NAME) def setup_event_loop_monitoring( loop: asyncio.AbstractEventLoop, scheduling_latency: metrics.Histogram, iterations: metrics.Counter, tasks: metrics.Gauge, tags: Dict[str, str], interval_s: Optional[float] = None, ) -> asyncio.Task: """Start monitoring an event loop and recording metrics. This function creates a background task that periodically measures: - How long it takes for the event loop to wake up after sleeping (scheduling latency / event loop lag) - The number of pending asyncio tasks Args: loop: The asyncio event loop to monitor. scheduling_latency: Histogram metric to record scheduling latency. iterations: Counter metric to track monitoring iterations. tasks: Gauge metric to track number of pending tasks. tags: Dictionary of tags to apply to all metrics. interval_s: Optional override for the monitoring interval. Defaults to RAY_SERVE_EVENT_LOOP_MONITORING_INTERVAL_S. Returns: The asyncio Task running the monitoring loop. """ if interval_s is None: interval_s = RAY_SERVE_EVENT_LOOP_MONITORING_INTERVAL_S return loop.create_task( _run_monitoring_loop( loop=loop, schedule_latency=scheduling_latency, iterations=iterations, task_gauge=tasks, tags=tags, interval_s=interval_s, ), name="serve_event_loop_monitoring", ) async def _run_monitoring_loop( loop: asyncio.AbstractEventLoop, schedule_latency: metrics.Histogram, iterations: metrics.Counter, task_gauge: metrics.Gauge, tags: Dict[str, str], interval_s: float, ) -> None: """Internal monitoring loop that runs until the event loop stops. The scheduling latency is measured by comparing the actual elapsed time after sleeping to the expected sleep duration. In an ideal scenario with no blocking, the latency should be close to zero. """ while loop.is_running(): iterations.inc(1, tags) num_tasks = len(asyncio.all_tasks(loop)) task_gauge.set(num_tasks, tags) yield_time = time.monotonic() await asyncio.sleep(interval_s) elapsed_time = time.monotonic() - yield_time # Historically, Ray's implementation of histograms are extremely finicky # with non-positive values (https://github.com/ray-project/ray/issues/26698). # Technically it shouldn't be possible for this to be negative, add the # max just to be safe. # Convert to milliseconds for the metric. latency_ms = max(0.0, (elapsed_time - interval_s) * 1000) schedule_latency.observe(latency_ms, tags) class EventLoopMonitor: TAG_KEY_COMPONENT = "component" TAG_KEY_LOOP_TYPE = "loop_type" TAG_KEY_ACTOR_ID = "actor_id" # Component types COMPONENT_PROXY = "proxy" COMPONENT_REPLICA = "replica" COMPONENT_UNKNOWN = "unknown" # Loop types LOOP_TYPE_MAIN = "main" LOOP_TYPE_USER_CODE = "user_code" LOOP_TYPE_ROUTER = "router" def __init__( self, component: str, loop_type: str, actor_id: str, interval_s: float = RAY_SERVE_EVENT_LOOP_MONITORING_INTERVAL_S, extra_tags: Optional[Dict[str, str]] = None, ): """Initialize the event loop monitor. Args: component: The component type ("proxy" or "replica"). loop_type: The type of event loop ("main", "user_code", or "router"). actor_id: The ID of the actor where this event loop runs. interval_s: Optional override for the monitoring interval. extra_tags: Optional dictionary of additional tags to include in metrics. """ self._interval_s = interval_s self._tags = { self.TAG_KEY_COMPONENT: component, self.TAG_KEY_LOOP_TYPE: loop_type, self.TAG_KEY_ACTOR_ID: actor_id, } if extra_tags: self._tags.update(extra_tags) self._tag_keys = tuple(self._tags.keys()) # Create metrics self._scheduling_latency = metrics.Histogram( "serve_event_loop_scheduling_latency_ms", description=( "Latency of getting yielded control on the event loop in milliseconds. " "High values indicate the event loop is blocked." ), boundaries=SERVE_EVENT_LOOP_LATENCY_HISTOGRAM_BOUNDARIES_MS, tag_keys=self._tag_keys, ) self._scheduling_latency.set_default_tags(self._tags) self._iterations = metrics.Counter( "serve_event_loop_monitoring_iterations", description=( "Number of times the event loop monitoring task has run. " "Can be used as a heartbeat." ), tag_keys=self._tag_keys, ) self._iterations.set_default_tags(self._tags) self._tasks = metrics.Gauge( "serve_event_loop_tasks", description="Number of pending asyncio tasks on the event loop.", tag_keys=self._tag_keys, ) self._tasks.set_default_tags(self._tags) self._monitoring_task: Optional[asyncio.Task] = None def start(self, loop: asyncio.AbstractEventLoop) -> asyncio.Task: """Start monitoring the given event loop. Args: loop: The asyncio event loop to monitor. Returns: The asyncio Task running the monitoring loop. """ self._monitoring_task = setup_event_loop_monitoring( loop=loop, scheduling_latency=self._scheduling_latency, iterations=self._iterations, tasks=self._tasks, tags=self._tags, interval_s=self._interval_s, ) logger.debug( f"Started event loop monitoring for {self._tags[self.TAG_KEY_COMPONENT]} " f"({self._tags[self.TAG_KEY_LOOP_TYPE]}) actor {self._tags[self.TAG_KEY_ACTOR_ID]}" ) return self._monitoring_task def stop(self): if self._monitoring_task is not None and not self._monitoring_task.done(): self._monitoring_task.cancel() self._monitoring_task = None @property def tags(self) -> Dict[str, str]: return self._tags.copy()