chore: import upstream snapshot with attribution

python/ray/serve/_private/event_loop_monitoring.py

@@ -0,0 +1,193 @@
+import asyncio
+import logging
+import time
+from typing import Dict, Optional
+
+from ray.serve._private.constants import (
+    RAY_SERVE_EVENT_LOOP_MONITORING_INTERVAL_S,
+    SERVE_EVENT_LOOP_LATENCY_HISTOGRAM_BOUNDARIES_MS,
+    SERVE_LOGGER_NAME,
+)
+from ray.util import metrics
+
+logger = logging.getLogger(SERVE_LOGGER_NAME)
+
+
+def setup_event_loop_monitoring(
+    loop: asyncio.AbstractEventLoop,
+    scheduling_latency: metrics.Histogram,
+    iterations: metrics.Counter,
+    tasks: metrics.Gauge,
+    tags: Dict[str, str],
+    interval_s: Optional[float] = None,
+) -> asyncio.Task:
+    """Start monitoring an event loop and recording metrics.
+
+    This function creates a background task that periodically measures:
+    - How long it takes for the event loop to wake up after sleeping
+      (scheduling latency / event loop lag)
+    - The number of pending asyncio tasks
+
+    Args:
+        loop: The asyncio event loop to monitor.
+        scheduling_latency: Histogram metric to record scheduling latency.
+        iterations: Counter metric to track monitoring iterations.
+        tasks: Gauge metric to track number of pending tasks.
+        tags: Dictionary of tags to apply to all metrics.
+        interval_s: Optional override for the monitoring interval.
+            Defaults to RAY_SERVE_EVENT_LOOP_MONITORING_INTERVAL_S.
+
+    Returns:
+        The asyncio Task running the monitoring loop.
+    """
+    if interval_s is None:
+        interval_s = RAY_SERVE_EVENT_LOOP_MONITORING_INTERVAL_S
+
+    return loop.create_task(
+        _run_monitoring_loop(
+            loop=loop,
+            schedule_latency=scheduling_latency,
+            iterations=iterations,
+            task_gauge=tasks,
+            tags=tags,
+            interval_s=interval_s,
+        ),
+        name="serve_event_loop_monitoring",
+    )
+
+
+async def _run_monitoring_loop(
+    loop: asyncio.AbstractEventLoop,
+    schedule_latency: metrics.Histogram,
+    iterations: metrics.Counter,
+    task_gauge: metrics.Gauge,
+    tags: Dict[str, str],
+    interval_s: float,
+) -> None:
+    """Internal monitoring loop that runs until the event loop stops.
+
+    The scheduling latency is measured by comparing the actual elapsed time
+    after sleeping to the expected sleep duration. In an ideal scenario
+    with no blocking, the latency should be close to zero.
+    """
+    while loop.is_running():
+        iterations.inc(1, tags)
+        num_tasks = len(asyncio.all_tasks(loop))
+        task_gauge.set(num_tasks, tags)
+        yield_time = time.monotonic()
+        await asyncio.sleep(interval_s)
+        elapsed_time = time.monotonic() - yield_time
+
+        # Historically, Ray's implementation of histograms are extremely finicky
+        # with non-positive values (https://github.com/ray-project/ray/issues/26698).
+        # Technically it shouldn't be possible for this to be negative, add the
+        # max just to be safe.
+        # Convert to milliseconds for the metric.
+        latency_ms = max(0.0, (elapsed_time - interval_s) * 1000)
+        schedule_latency.observe(latency_ms, tags)
+
+
+class EventLoopMonitor:
+    TAG_KEY_COMPONENT = "component"
+    TAG_KEY_LOOP_TYPE = "loop_type"
+    TAG_KEY_ACTOR_ID = "actor_id"
+
+    # Component types
+    COMPONENT_PROXY = "proxy"
+    COMPONENT_REPLICA = "replica"
+    COMPONENT_UNKNOWN = "unknown"
+
+    # Loop types
+    LOOP_TYPE_MAIN = "main"
+    LOOP_TYPE_USER_CODE = "user_code"
+    LOOP_TYPE_ROUTER = "router"
+
+    def __init__(
+        self,
+        component: str,
+        loop_type: str,
+        actor_id: str,
+        interval_s: float = RAY_SERVE_EVENT_LOOP_MONITORING_INTERVAL_S,
+        extra_tags: Optional[Dict[str, str]] = None,
+    ):
+        """Initialize the event loop monitor.
+
+        Args:
+            component: The component type ("proxy" or "replica").
+            loop_type: The type of event loop ("main", "user_code", or "router").
+            actor_id: The ID of the actor where this event loop runs.
+            interval_s: Optional override for the monitoring interval.
+            extra_tags: Optional dictionary of additional tags to include in metrics.
+        """
+        self._interval_s = interval_s
+        self._tags = {
+            self.TAG_KEY_COMPONENT: component,
+            self.TAG_KEY_LOOP_TYPE: loop_type,
+            self.TAG_KEY_ACTOR_ID: actor_id,
+        }
+        if extra_tags:
+            self._tags.update(extra_tags)
+        self._tag_keys = tuple(self._tags.keys())
+
+        # Create metrics
+        self._scheduling_latency = metrics.Histogram(
+            "serve_event_loop_scheduling_latency_ms",
+            description=(
+                "Latency of getting yielded control on the event loop in milliseconds. "
+                "High values indicate the event loop is blocked."
+            ),
+            boundaries=SERVE_EVENT_LOOP_LATENCY_HISTOGRAM_BOUNDARIES_MS,
+            tag_keys=self._tag_keys,
+        )
+        self._scheduling_latency.set_default_tags(self._tags)
+
+        self._iterations = metrics.Counter(
+            "serve_event_loop_monitoring_iterations",
+            description=(
+                "Number of times the event loop monitoring task has run. "
+                "Can be used as a heartbeat."
+            ),
+            tag_keys=self._tag_keys,
+        )
+        self._iterations.set_default_tags(self._tags)
+
+        self._tasks = metrics.Gauge(
+            "serve_event_loop_tasks",
+            description="Number of pending asyncio tasks on the event loop.",
+            tag_keys=self._tag_keys,
+        )
+        self._tasks.set_default_tags(self._tags)
+
+        self._monitoring_task: Optional[asyncio.Task] = None
+
+    def start(self, loop: asyncio.AbstractEventLoop) -> asyncio.Task:
+        """Start monitoring the given event loop.
+
+        Args:
+            loop: The asyncio event loop to monitor.
+
+        Returns:
+            The asyncio Task running the monitoring loop.
+        """
+        self._monitoring_task = setup_event_loop_monitoring(
+            loop=loop,
+            scheduling_latency=self._scheduling_latency,
+            iterations=self._iterations,
+            tasks=self._tasks,
+            tags=self._tags,
+            interval_s=self._interval_s,
+        )
+        logger.debug(
+            f"Started event loop monitoring for {self._tags[self.TAG_KEY_COMPONENT]} "
+            f"({self._tags[self.TAG_KEY_LOOP_TYPE]}) actor {self._tags[self.TAG_KEY_ACTOR_ID]}"
+        )
+        return self._monitoring_task
+
+    def stop(self):
+        if self._monitoring_task is not None and not self._monitoring_task.done():
+            self._monitoring_task.cancel()
+            self._monitoring_task = None
+
+    @property
+    def tags(self) -> Dict[str, str]:
+        return self._tags.copy()