chore: import upstream snapshot with attribution

python/ray/serve/experimental/round_robin_router.py

@@ -0,0 +1,90 @@
+"""Round-robin request router.
+
+RoundRobinRouter cycles through the candidate replicas passed in by Serve and
+returns strictly ordered singleton ranks. Each request starts at the current
+round-robin cursor; if that replica cannot fulfill the request, Serve tries the
+next replica in order, wrapping around the candidate list.
+"""
+
+import random
+from collections.abc import Sequence
+from typing import List, Optional
+
+from ray.serve._private.request_router.common import (
+    PendingRequest,
+)
+from ray.serve._private.request_router.replica_wrapper import (
+    RunningReplica,
+)
+from ray.serve._private.request_router.request_router import (
+    FIFOMixin,
+    RequestRouter,
+)
+
+
+class _RoundRobinReplicaRanks(Sequence[List[RunningReplica]]):
+    """Lazy ordered singleton ranks for a strict round-robin attempt."""
+
+    def __init__(
+        self,
+        replicas: List[RunningReplica],
+        start_index: int,
+    ):
+        self._replicas = replicas
+        self._start_index = start_index
+
+    def __len__(self) -> int:
+        return len(self._replicas)
+
+    def __getitem__(self, index):
+        if isinstance(index, slice):
+            return list(self)[index]
+
+        num_replicas = len(self._replicas)
+        if index < 0:
+            index += num_replicas
+        if index < 0 or index >= num_replicas:
+            raise IndexError(index)
+
+        return [self._replicas[(self._start_index + index) % num_replicas]]
+
+    def __iter__(self):
+        num_replicas = len(self._replicas)
+        for offset in range(num_replicas):
+            yield [self._replicas[(self._start_index + offset) % num_replicas]]
+
+
+class RoundRobinRouter(FIFOMixin, RequestRouter):
+    """Routes requests by cycling through candidate replicas.
+
+    Each call to ``choose_replicas`` advances a shared cursor by one position
+    and returns ordered singleton ranks starting from that cursor. This upholds
+    strict round-robin ordering while still allowing Serve's existing selector
+    to continue to the next replica if the current one is full.
+    """
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self._round_robin_counter = random.randrange(2**31)
+
+    def initialize_state(self, **kwargs) -> None:
+        pass
+
+    async def choose_replicas(
+        self,
+        candidate_replicas: List[RunningReplica],
+        pending_request: Optional[PendingRequest] = None,
+    ) -> Sequence[List[RunningReplica]]:
+        if not candidate_replicas:
+            return []
+
+        if pending_request is not None:
+            # Enable exponential-backoff sleep between outer retry iterations.
+            # Without this, the base class tight-loops calling choose_replicas
+            # when every replica is at capacity.
+            pending_request.routing_context.should_backoff = True
+
+        index = self._round_robin_counter % len(candidate_replicas)
+        self._round_robin_counter += 1
+
+        return _RoundRobinReplicaRanks(candidate_replicas, index)