chore: import upstream snapshot with attribution

python/ray/serve/tests/unit/BUILD.bazel

@@ -0,0 +1,115 @@
+load("@rules_python//python:defs.bzl", "py_library")
+load("//bazel:python.bzl", "py_test_module_list", "py_test_module_list_with_env_variants", "py_test_run_all_subdirectory")
+
+py_library(
+    name = "conftest",
+    srcs = ["conftest.py"],
+)
+
+py_test_run_all_subdirectory(
+    size = "small",
+    include = glob(["test_*.py"]),
+    exclude = [
+        "test_haproxy_binary.py",
+        "test_round_robin_router.py",
+    ],
+    extra_srcs = [],
+    tags = ["team:serve"],
+    deps = [
+        ":conftest",
+        "//python/ray/serve:serve_lib",
+        "//python/ray/serve/tests:common",
+    ],
+)
+
+py_test_module_list(
+    size = "small",
+    files = ["test_round_robin_router.py"],
+    tags = ["team:serve"],
+    deps = [
+        ":conftest",
+        "//python/ray/serve:serve_lib",
+        "//python/ray/serve/tests:common",
+    ],
+)
+
+py_test_module_list(
+    size = "small",
+    files = ["test_haproxy_binary.py"],
+    tags = [
+        "no_windows",
+        "team:serve",
+    ],
+    deps = [
+        ":conftest",
+        "//python/ray/serve:serve_lib",
+        "//python/ray/serve/tests:common",
+    ],
+)
+
+py_test_module_list(
+    size = "small",
+    timeout = "short",
+    env = {
+        "RAY_SERVE_USE_PACK_SCHEDULING_STRATEGY": "1",
+        "RAY_SERVE_FAIL_ON_RANK_ERROR": "1",
+    },
+    files = [
+        "test_deployment_scheduler.py",
+        "test_deployment_state.py",
+    ],
+    name_suffix = "_with_pack_scheduling",
+    tags = [
+        "no_windows",
+        "team:serve",
+    ],
+    deps = [
+        ":conftest",
+        "//python/ray/serve:serve_lib",
+        "//python/ray/serve/tests:common",
+    ],
+)
+
+py_test_module_list_with_env_variants(
+    size = "small",
+    env_variants = {
+        "metr_disab": {
+            "env": {
+                "RAY_SERVE_COLLECT_AUTOSCALING_METRICS_ON_HANDLE": "0",
+                "RAY_SERVE_FAIL_ON_RANK_ERROR": "1",
+            },
+            "name_suffix": "_metr_disab",
+        },
+        "metr_agg_at_controller": {
+            "env": {
+                "RAY_SERVE_AGGREGATE_METRICS_AT_CONTROLLER": "1",
+                "RAY_SERVE_COLLECT_AUTOSCALING_METRICS_ON_HANDLE": "1",
+                "RAY_SERVE_FAIL_ON_RANK_ERROR": "1",
+            },
+            "name_suffix": "_metr_agg_at_controller",
+        },
+        "metr_agg_at_controller_and_replicas": {
+            "env": {
+                "RAY_SERVE_AGGREGATE_METRICS_AT_CONTROLLER": "1",
+                "RAY_SERVE_COLLECT_AUTOSCALING_METRICS_ON_HANDLE": "0",
+                "RAY_SERVE_FAIL_ON_RANK_ERROR": "1",
+            },
+            "name_suffix": "_metr_agg_at_controller_and_replicas",
+        },
+    },
+    files = [
+        "test_autoscaling_policy.py",
+        "test_controller.py",
+        "test_deployment_state.py",
+        "test_router.py",
+    ],
+    tags = [
+        "no_windows",
+        "team:serve",
+    ],
+    deps = [
+        ":conftest",
+        "//python/ray/serve:serve_lib",
+        "//python/ray/serve/tests:common",
+    ],
+)

python/ray/serve/tests/unit/README.md

@@ -0,0 +1 @@
+This directory should only contain unit tests that do not depend on running a Ray instance.

python/ray/serve/tests/unit/conftest.py

@@ -0,0 +1,94 @@
+from typing import Tuple
+from unittest.mock import Mock, patch
+
+import pytest
+
+import ray
+from ray._raylet import NodeID
+from ray.serve._private.autoscaling_state import AutoscalingStateManager
+from ray.serve._private.deployment_state import DeploymentStateManager
+from ray.serve._private.test_utils import (
+    MockClusterNodeInfoCache,
+    MockDeploymentActorWrapper,
+    MockKVStore,
+    MockReplicaActorWrapper,
+    MockTimer,
+    dead_replicas_context,
+    replica_rank_context,
+    uninitialized_replicas_context,
+)
+
+
+@pytest.fixture(autouse=True)
+def disallow_ray_init(monkeypatch):
+    def raise_on_init():
+        raise RuntimeError("Unit tests should not depend on Ray being initialized.")
+
+    monkeypatch.setattr(ray, "init", raise_on_init)
+    # Unit tests don't run on a Ray cluster, so stub the runtime context
+    # that callers consult for worker/actor ids.
+    monkeypatch.setattr(ray, "get_runtime_context", Mock())
+
+
+@pytest.fixture
+def mock_deployment_state_manager(
+    request,
+) -> Tuple[DeploymentStateManager, MockTimer, Mock, Mock]:
+    """Fully mocked deployment state manager.
+
+    i.e kv store and gcs client is mocked so we don't need to initialize
+    ray. Also, since this is used for some recovery tests, this yields a
+    method for creating a new mocked deployment state manager.
+    """
+
+    timer = MockTimer()
+    with patch(
+        "ray.serve._private.deployment_state.ActorReplicaWrapper",
+        new=MockReplicaActorWrapper,
+    ), patch(
+        "ray.serve._private.deployment_state.DeploymentActorWrapper",
+        new=MockDeploymentActorWrapper,
+    ), patch(
+        "time.time", new=timer.time
+    ), patch(
+        "ray.serve._private.long_poll.LongPollHost"
+    ) as mock_long_poll, patch(
+        "ray.get_runtime_context"
+    ):
+        kv_store = MockKVStore()
+        cluster_node_info_cache = MockClusterNodeInfoCache()
+        cluster_node_info_cache.add_node(NodeID.from_random().hex())
+        autoscaling_state_manager = AutoscalingStateManager()
+
+        def create_deployment_state_manager(
+            actor_names=None,
+            placement_group_names=None,
+            create_placement_group_fn_override=None,
+        ):
+            if actor_names is None:
+                actor_names = []
+
+            if placement_group_names is None:
+                placement_group_names = []
+
+            return DeploymentStateManager(
+                kv_store,
+                mock_long_poll,
+                actor_names,
+                placement_group_names,
+                cluster_node_info_cache,
+                autoscaling_state_manager,
+                head_node_id_override="fake-head-node-id",
+                create_placement_group_fn_override=create_placement_group_fn_override,
+            )
+
+        yield (
+            create_deployment_state_manager,
+            timer,
+            cluster_node_info_cache,
+            autoscaling_state_manager,
+        )
+
+        dead_replicas_context.clear()
+        replica_rank_context.clear()
+        uninitialized_replicas_context.clear()

python/ray/serve/tests/unit/test_batching.py

@@ -0,0 +1,959 @@
+import asyncio
+import logging
+import time
+from typing import List
+
+import pytest
+
+import ray
+from ray import serve
+from ray._common.utils import get_or_create_event_loop
+from ray.serve._private.common import DeploymentID, ReplicaID
+from ray.serve._private.config import DeploymentConfig
+from ray.serve._private.constants import SERVE_LOGGER_NAME
+from ray.serve.batching import _BatchQueue
+from ray.serve.exceptions import RayServeException
+
+# Setup the global replica context for the test.
+default_deployment_config = DeploymentConfig()
+ray.serve.context._set_internal_replica_context(
+    replica_id=ReplicaID(unique_id="test", deployment_id=DeploymentID(name="test")),
+    servable_object=None,
+    _deployment_config=default_deployment_config,
+    rank=0,
+    world_size=1,
+)
+
+
+class FakeStream:
+    def __init__(self):
+        self.messages = []
+
+    def write(self, buf):
+        self.messages.append(buf)
+
+    def reset_message(self):
+        self.messages = []
+
+
+# We use a single event loop for the entire test session. Without this
+# fixture, the event loop is sometimes prematurely terminated by pytest.
+@pytest.fixture(scope="session")
+def event_loop():
+    loop = get_or_create_event_loop()
+    yield loop
+    loop.close()
+
+
+@pytest.mark.asyncio
+async def test_decorator_validation():
+    @serve.batch
+    async def function():
+        pass
+
+    @serve.batch(max_batch_size=10, batch_wait_timeout_s=1.5)
+    async def function2():
+        pass
+
+    class Class:
+        @serve.batch
+        async def method(self):
+            pass
+
+    class Class2:
+        @serve.batch(max_batch_size=10, batch_wait_timeout_s=1.5)
+        async def method(self):
+            pass
+
+    with pytest.raises(TypeError, match="async def"):
+
+        @serve.batch
+        def non_async_function():
+            pass
+
+    with pytest.raises(TypeError, match="async def"):
+
+        class NotAsync:
+            @serve.batch
+            def method(self, requests):
+                pass
+
+    with pytest.raises(ValueError):
+
+        class ZeroBatch:
+            @serve.batch(max_batch_size=0)
+            async def method(self, requests):
+                pass
+
+    with pytest.raises(TypeError):
+
+        class FloatNonIntBatch:
+            @serve.batch(max_batch_size=1.1)
+            async def method(self, requests):
+                pass
+
+    class FloatIntegerBatch:
+        @serve.batch(max_batch_size=1.0)
+        async def method(self, requests):
+            pass
+
+    with pytest.raises(ValueError):
+
+        class NegativeTimeout:
+            @serve.batch(batch_wait_timeout_s=-0.1)
+            async def method(self, requests):
+                pass
+
+    class FloatZeroTimeout:
+        @serve.batch(batch_wait_timeout_s=0.0)
+        async def method(self, requests):
+            pass
+
+    class IntZeroTimeout:
+        @serve.batch(batch_wait_timeout_s=0)
+        async def method(self, requests):
+            pass
+
+    with pytest.raises(TypeError):
+
+        class NonTimeout:
+            @serve.batch(batch_wait_timeout_s="a")
+            async def method(self, requests):
+                pass
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("use_class", [True, False])
+async def test_batch_size_one_long_timeout(use_class):
+    @serve.batch(max_batch_size=1, batch_wait_timeout_s=1000)
+    async def long_timeout(requests):
+        if "raise" in requests:
+            _ = 1 / 0
+        return requests
+
+    class LongTimeout:
+        @serve.batch(max_batch_size=1, batch_wait_timeout_s=1000)
+        async def long_timeout(self, requests):
+            if "raise" in requests:
+                _ = 1 / 0
+            return requests
+
+    cls = LongTimeout()
+
+    async def call(arg):
+        if use_class:
+            return await cls.long_timeout(arg)
+        else:
+            return await long_timeout(arg)
+
+    assert await call("hi") == "hi"
+    with pytest.raises(ZeroDivisionError):
+        await call("raise")
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("use_class", [True, False])
+async def test_batch_size_multiple_zero_timeout(use_class):
+    block_execution_event = asyncio.Event()
+
+    @serve.batch(max_batch_size=2, batch_wait_timeout_s=0)
+    async def zero_timeout(requests):
+        await block_execution_event.wait()
+        if "raise" in requests:
+            _ = 1 / 0
+        return requests
+
+    class ZeroTimeout:
+        @serve.batch(max_batch_size=2, batch_wait_timeout_s=0)
+        async def zero_timeout(self, requests):
+            await block_execution_event.wait()
+            if "raise" in requests:
+                _ = 1 / 0
+            return requests
+
+    cls = ZeroTimeout()
+
+    async def call(arg):
+        if use_class:
+            return await cls.zero_timeout(arg)
+        else:
+            return await zero_timeout(arg)
+
+    block_execution_event.set()
+    assert await call("hi") == "hi"
+    with pytest.raises(ZeroDivisionError):
+        await call("raise")
+    block_execution_event.clear()
+
+    # Check that 2 requests will be executed together if available.
+    # The first should cause a size-one batch to be executed, then
+    # the next two should be executed together (signaled by both
+    # having the exception).
+    t1 = get_or_create_event_loop().create_task(call("hi1"))
+    with pytest.raises(asyncio.TimeoutError):
+        await asyncio.wait_for(asyncio.shield(t1), timeout=0.001)
+
+    t2 = get_or_create_event_loop().create_task(call("hi2"))
+    t3 = get_or_create_event_loop().create_task(call("raise"))
+
+    block_execution_event.set()
+    assert await t1 == "hi1"
+
+    with pytest.raises(ZeroDivisionError):
+        await t2
+    with pytest.raises(ZeroDivisionError):
+        await t3
+
+
+@pytest.mark.asyncio
+async def test_batch_timeout_empty_queue():
+    """Check that Serve waits when creating batches.
+
+    Serve should wait a full batch_wait_timeout_s after receiving the first
+    request in the next batch before processing the batch.
+    """
+
+    @serve.batch(max_batch_size=10, batch_wait_timeout_s=0.25)
+    async def no_op(requests):
+        return ["No-op"] * len(requests)
+
+    num_iterations = 2
+    for iteration in range(num_iterations):
+        tasks = [get_or_create_event_loop().create_task(no_op(None)) for _ in range(9)]
+        done, _ = await asyncio.wait(tasks, timeout=0.05)
+
+        # Due to the long timeout, none of the tasks should finish until a tenth
+        # request is submitted
+        assert len(done) == 0
+
+        tasks.append(get_or_create_event_loop().create_task(no_op(None)))
+        done, _ = await asyncio.wait(tasks, timeout=0.05)
+
+        # All the timeout tasks should be finished
+        assert set(tasks) == set(done)
+        assert all(t.result() == "No-op" for t in tasks)
+
+        if iteration < num_iterations - 1:
+            # Leave queue empty for batch_wait_timeout_s between batches
+            time.sleep(0.25)
+
+
+@pytest.mark.asyncio
+async def test_batch_wait_queue_exceeds_batch_size_race_condition():
+    """Check that the wait queue can exceed the batch size.
+
+    This test was added to guard against a race condition documented in
+    https://github.com/ray-project/ray/pull/42705#discussion_r1466653910.
+    """
+
+    @serve.batch(max_batch_size=2, batch_wait_timeout_s=10000)
+    async def no_op(requests):
+        return ["No-op"] * len(requests)
+
+    tasks = [get_or_create_event_loop().create_task(no_op(None)) for _ in range(10)]
+
+    # All the tasks should finish.
+    done, pending = await asyncio.wait(tasks, timeout=0.5)
+    assert len(done) == len(tasks)
+    assert len(pending) == 0
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("use_class", [True, False])
+async def test_batch_size_multiple_long_timeout(use_class):
+    @serve.batch(max_batch_size=3, batch_wait_timeout_s=1000)
+    async def long_timeout(requests):
+        if "raise" in requests:
+            _ = 1 / 0
+        return requests
+
+    class LongTimeout:
+        @serve.batch(max_batch_size=3, batch_wait_timeout_s=1000)
+        async def long_timeout(self, requests):
+            if "raise" in requests:
+                _ = 1 / 0
+            return requests
+
+    cls = LongTimeout()
+
+    async def call(arg):
+        if use_class:
+            return await cls.long_timeout(arg)
+        else:
+            return await long_timeout(arg)
+
+    t1 = get_or_create_event_loop().create_task(call("hi1"))
+    t2 = get_or_create_event_loop().create_task(call("hi2"))
+    done, pending = await asyncio.wait([t1, t2], timeout=0.1)
+    assert len(done) == 0
+    t3 = get_or_create_event_loop().create_task(call("hi3"))
+    done, pending = await asyncio.wait([t1, t2, t3], timeout=100)
+    assert set(done) == {t1, t2, t3}
+    assert [t1.result(), t2.result(), t3.result()] == ["hi1", "hi2", "hi3"]
+
+    t1 = get_or_create_event_loop().create_task(call("hi1"))
+    t2 = get_or_create_event_loop().create_task(call("raise"))
+    done, pending = await asyncio.wait([t1, t2], timeout=0.1)
+    assert len(done) == 0
+    t3 = get_or_create_event_loop().create_task(call("hi3"))
+    done, pending = await asyncio.wait([t1, t2, t3], timeout=100)
+    assert set(done) == {t1, t2, t3}
+    assert all(isinstance(t.exception(), ZeroDivisionError) for t in done)
+    with pytest.raises(ZeroDivisionError):
+        t1.result()
+    with pytest.raises(ZeroDivisionError):
+        t2.result()
+    with pytest.raises(ZeroDivisionError):
+        t3.result()
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("mode", ["args", "kwargs", "mixed", "out-of-order"])
+@pytest.mark.parametrize("use_class", [True, False])
+async def test_batch_args_kwargs(mode, use_class):
+    if use_class:
+
+        class MultipleArgs:
+            @serve.batch(max_batch_size=2, batch_wait_timeout_s=1000)
+            async def method(self, key1, key2):
+                return [(key1[i], key2[i]) for i in range(len(key1))]
+
+        instance = MultipleArgs()
+        func = instance.method
+
+    else:
+
+        @serve.batch(max_batch_size=2, batch_wait_timeout_s=1000)
+        async def func(key1, key2):
+            return [(key1[i], key2[i]) for i in range(len(key1))]
+
+    if mode == "args":
+        coros = [func("hi1", "hi2"), func("hi3", "hi4")]
+    elif mode == "kwargs":
+        coros = [func(key1="hi1", key2="hi2"), func(key1="hi3", key2="hi4")]
+    elif mode == "mixed":
+        coros = [func("hi1", key2="hi2"), func("hi3", key2="hi4")]
+    elif mode == "out-of-order":
+        coros = [func(key2="hi2", key1="hi1"), func(key2="hi4", key1="hi3")]
+
+    result = await asyncio.gather(*coros)
+    assert result == [("hi1", "hi2"), ("hi3", "hi4")]
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("use_class", [True, False])
+@pytest.mark.parametrize("use_gen", [True, False])
+async def test_batch_cancellation(use_class, use_gen):
+    block_requests = asyncio.Event()
+    request_was_cancelled = True
+
+    async def unary_implementation(key1, key2):
+        nonlocal block_requests, request_was_cancelled
+        await block_requests.wait()
+        request_was_cancelled = False
+        return [(key1[i], key2[i]) for i in range(len(key1))]
+
+    async def streaming_implementation(key1, key2):
+        nonlocal block_requests, request_was_cancelled
+        await block_requests.wait()
+        request_was_cancelled = False
+        yield [(key1[i], key2[i]) for i in range(len(key1))]
+
+    if use_class:
+
+        class MultipleArgs:
+            @serve.batch(max_batch_size=2, batch_wait_timeout_s=1000)
+            async def unary_method(self, key1, key2):
+                return await unary_implementation(key1, key2)
+
+            @serve.batch(max_batch_size=2, batch_wait_timeout_s=1000)
+            async def streaming_method(self, key1, key2):
+                async for value in streaming_implementation(key1, key2):
+                    yield value
+
+        instance = MultipleArgs()
+
+        if use_gen:
+            func = instance.streaming_method
+        else:
+            func = instance.unary_method
+
+    else:
+
+        @serve.batch(max_batch_size=2, batch_wait_timeout_s=1000)
+        async def unary_func(key1, key2):
+            return await unary_implementation(key1, key2)
+
+        @serve.batch(max_batch_size=2, batch_wait_timeout_s=1000)
+        async def streaming_func(key1, key2):
+            async for value in streaming_implementation(key1, key2):
+                yield value
+
+        if use_gen:
+            func = streaming_func
+        else:
+            func = unary_func
+
+    if use_gen:
+        gens = [func("hi1", "hi2"), func("hi3", "hi4")]
+        tasks = [asyncio.create_task(gen.__anext__()) for gen in gens]
+    else:
+        tasks = [
+            asyncio.create_task(func("hi1", "hi2")),
+            asyncio.create_task(func("hi3", "hi4")),
+        ]
+
+    print("Submitted requests.")
+
+    # The requests should be blocked on the long request_timeout
+    done, pending = await asyncio.wait(tasks, timeout=0.01)
+    assert len(done) == 0
+    assert len(pending) == 2
+
+    print("Requests are blocked, as expected.")
+
+    # Cancel the first request. The second request should still be blocked on
+    # the long request_timeout
+    tasks[0].cancel()
+    pending, done = await asyncio.wait(tasks, timeout=0.01)
+    assert len(done) == 1
+    assert len(pending) == 1
+
+    print("Cancelled first request.")
+
+    # Cancel the second request. Both requests should be done.
+    tasks[1].cancel()
+    done, pending = await asyncio.wait(tasks, timeout=0.01)
+    assert len(done) == 2
+    assert len(pending) == 0
+
+    print("Cancelled second request. Sending new requests with no timeout.")
+
+    # Sanity check that the request was actually cancelled.
+    assert request_was_cancelled
+
+    # Unblock requests. The requests should succeed.
+    block_requests.set()
+
+    if use_gen:
+        gens = [func("hi1", "hi2"), func("hi3", "hi4")]
+        tasks = [asyncio.create_task(gen.__anext__()) for gen in gens]
+    else:
+        tasks = [
+            asyncio.create_task(func("hi1", "hi2")),
+            asyncio.create_task(func("hi3", "hi4")),
+        ]
+
+    result = await asyncio.gather(*tasks)
+    assert result == [("hi1", "hi2"), ("hi3", "hi4")]
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("use_class", [True, False])
+@pytest.mark.parametrize("use_gen", [True, False])
+async def test_cancellation_after_error(use_class, use_gen):
+    """Cancelling a request after it errors should be supported."""
+
+    raise_error = asyncio.Event()
+
+    async def unary_implementation(key1, key2):
+        if not raise_error.is_set():
+            raise ValueError()
+        return [(key1[i], key2[i]) for i in range(len(key1))]
+
+    async def streaming_implementation(key1, key2):
+        if not raise_error.is_set():
+            raise ValueError()
+        yield [(key1[i], key2[i]) for i in range(len(key1))]
+
+    if use_class:
+
+        class MultipleArgs:
+            @serve.batch(max_batch_size=2, batch_wait_timeout_s=1000)
+            async def unary_method(self, key1, key2):
+                return await unary_implementation(key1, key2)
+
+            @serve.batch(max_batch_size=2, batch_wait_timeout_s=1000)
+            async def streaming_method(self, key1, key2):
+                async for value in streaming_implementation(key1, key2):
+                    yield value
+
+        instance = MultipleArgs()
+
+        if use_gen:
+            func = instance.streaming_method
+        else:
+            func = instance.unary_method
+
+    else:
+
+        @serve.batch(max_batch_size=2, batch_wait_timeout_s=1000)
+        async def unary_func(key1, key2):
+            return await unary_implementation(key1, key2)
+
+        @serve.batch(max_batch_size=2, batch_wait_timeout_s=1000)
+        async def streaming_func(key1, key2):
+            async for value in streaming_implementation(key1, key2):
+                yield value
+
+        if use_gen:
+            func = streaming_func
+        else:
+            func = unary_func
+
+    # Submit requests and then cancel them.
+    if use_gen:
+        gens = [func("hi1", "hi2"), func("hi3", "hi4")]
+        tasks = [asyncio.create_task(gen.__anext__()) for gen in gens]
+    else:
+        tasks = [
+            asyncio.create_task(func("hi1", "hi2")),
+            asyncio.create_task(func("hi3", "hi4")),
+        ]
+
+    print("Submitted initial batch of requests.")
+
+    for task in tasks:
+        task.cancel()
+
+    print("Closed initial batch of requests.")
+
+    raise_error.set()
+
+    # Submit requests and check that they still work.
+    if use_gen:
+        gens = [func("hi1", "hi2"), func("hi3", "hi4")]
+        tasks = [asyncio.create_task(gen.__anext__()) for gen in gens]
+    else:
+        tasks = [
+            asyncio.create_task(func("hi1", "hi2")),
+            asyncio.create_task(func("hi3", "hi4")),
+        ]
+
+    print("Submitted new batch of requests.")
+
+    result = await asyncio.gather(*tasks)
+    assert result == [("hi1", "hi2"), ("hi3", "hi4")]
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("use_class", [True, False])
+async def test_batch_setters(use_class):
+    if use_class:
+
+        class C:
+            @serve.batch(max_batch_size=2, batch_wait_timeout_s=1000)
+            async def method(self, key1, key2):
+                return [(key1[i], key2[i]) for i in range(len(key1))]
+
+        instance = C()
+        func = instance.method
+
+    else:
+
+        @serve.batch(max_batch_size=2, batch_wait_timeout_s=1000)
+        async def func(key1, key2):
+            return [(key1[i], key2[i]) for i in range(len(key1))]
+
+    assert func._get_max_batch_size() == 2
+    assert func._get_batch_wait_timeout_s() == 1000
+
+    # @serve.batch should create batches of size 2
+    tasks = [
+        get_or_create_event_loop().create_task(func("hi1", "hi2")),
+        get_or_create_event_loop().create_task(func("hi3", "hi4")),
+    ]
+    done, pending = await asyncio.wait(tasks, timeout=0.1)
+    assert len(pending) == 0
+    assert {task.result() for task in done} == {("hi1", "hi2"), ("hi3", "hi4")}
+
+    # Set new values
+    func.set_max_batch_size(3)
+    func.set_batch_wait_timeout_s(15000)
+
+    assert func._get_max_batch_size() == 3
+    assert func._get_batch_wait_timeout_s() == 15000
+
+    # @serve.batch should create batches of size 3
+    tasks = [
+        get_or_create_event_loop().create_task(func("hi1", "hi2")),
+        get_or_create_event_loop().create_task(func("hi3", "hi4")),
+        get_or_create_event_loop().create_task(func("hi5", "hi6")),
+    ]
+    done, pending = await asyncio.wait(tasks, timeout=0.1)
+    assert len(pending) == 0
+    assert {task.result() for task in done} == {
+        ("hi1", "hi2"),
+        ("hi3", "hi4"),
+        ("hi5", "hi6"),
+    }
+
+
+@pytest.mark.asyncio
+async def test_batch_use_earliest_setters():
+    """@serve.batch should use the right settings when constructing a batch.
+
+    When the @serve.batch setters get called before a batch has started
+    accumulating, the next batch should use the setters' values. When they
+    get called while a batch is accumulating, the previous values should be
+    used.
+    """
+
+    @serve.batch(max_batch_size=2, batch_wait_timeout_s=1000)
+    async def func(key1, key2):
+        return [(key1[i], key2[i]) for i in range(len(key1))]
+
+    assert func._get_max_batch_size() == 2
+    assert func._get_batch_wait_timeout_s() == 1000
+
+    # Set new values
+    func.set_max_batch_size(3)
+    func.set_batch_wait_timeout_s(15000)
+
+    assert func._get_max_batch_size() == 3
+    assert func._get_batch_wait_timeout_s() == 15000
+
+    # Should create batches of size 3, even if setters are called while
+    # batch is accumulated
+    tasks = [
+        get_or_create_event_loop().create_task(func("hi1", "hi2")),
+        get_or_create_event_loop().create_task(func("hi3", "hi4")),
+    ]
+
+    # Batch should be waiting for last request
+    done, pending = await asyncio.wait(tasks, timeout=0.1)
+    assert len(done) == 0 and len(pending) == 2
+
+    func.set_max_batch_size(1)
+    func.set_batch_wait_timeout_s(0)
+
+    assert func._get_max_batch_size() == 1
+    assert func._get_batch_wait_timeout_s() == 0
+
+    # Batch should still be waiting for last request
+    done, pending = await asyncio.wait(pending, timeout=0.1)
+    assert len(done) == 0 and len(pending) == 2
+
+    # Batch should execute after last request
+    pending.add(get_or_create_event_loop().create_task(func("hi5", "hi6")))
+    done, pending = await asyncio.wait(pending, timeout=0.1)
+    assert len(pending) == 0
+    assert {task.result() for task in done} == {
+        ("hi1", "hi2"),
+        ("hi3", "hi4"),
+        ("hi5", "hi6"),
+    }
+
+    # Next batch should use updated values
+    tasks = [get_or_create_event_loop().create_task(func("hi1", "hi2"))]
+    done, pending = await asyncio.wait(tasks, timeout=0.1)
+    assert len(done) == 1 and len(pending) == 0
+    assert done.pop().result() == ("hi1", "hi2")
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("mode", ["args", "kwargs", "mixed", "out-of-order"])
+@pytest.mark.parametrize("use_class", [True, False])
+@pytest.mark.parametrize("generator_length", [0, 2, 5])
+async def test_batch_generator_basic(mode, use_class, generator_length):
+    if use_class:
+
+        class MultipleArgs:
+            @serve.batch(max_batch_size=2, batch_wait_timeout_s=1000)
+            async def method(self, key1, key2):
+                for gen_idx in range(generator_length):
+                    yield [(gen_idx, key1[i], key2[i]) for i in range(len(key1))]
+
+        instance = MultipleArgs()
+        func = instance.method
+
+    else:
+
+        @serve.batch(max_batch_size=2, batch_wait_timeout_s=1000)
+        async def func(key1, key2):
+            for gen_idx in range(generator_length):
+                yield [(gen_idx, key1[i], key2[i]) for i in range(len(key1))]
+
+    if mode == "args":
+        generators = [func("hi1", "hi2"), func("hi3", "hi4")]
+    elif mode == "kwargs":
+        generators = [func(key1="hi1", key2="hi2"), func(key1="hi3", key2="hi4")]
+    elif mode == "mixed":
+        generators = [func("hi1", key2="hi2"), func("hi3", key2="hi4")]
+    elif mode == "out-of-order":
+        generators = [func(key2="hi2", key1="hi1"), func(key2="hi4", key1="hi3")]
+
+    results = [
+        [result async for result in generators[0]],
+        [result async for result in generators[1]],
+    ]
+
+    assert results == [
+        [(gen_idx, "hi1", "hi2") for gen_idx in range(generator_length)],
+        [(gen_idx, "hi3", "hi4") for gen_idx in range(generator_length)],
+    ]
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("error_type", ["runtime_error", "mismatched_lengths"])
+async def test_batch_generator_exceptions(error_type):
+    GENERATOR_LENGTH = 5
+    ERROR_IDX = 2
+    ERROR_MSG = "Testing error"
+
+    @serve.batch(max_batch_size=2, batch_wait_timeout_s=1000)
+    async def func(key1, key2):
+        for gen_idx in range(GENERATOR_LENGTH):
+            results = [(gen_idx, key1[i], key2[i]) for i in range(len(key1))]
+            if gen_idx == ERROR_IDX:
+                if error_type == "runtime_error":
+                    raise RuntimeError(ERROR_MSG)
+                elif error_type == "mismatched_lengths":
+                    yield results * 2
+            yield results
+
+    generators = [func("hi1", "hi2"), func("hi3", "hi4")]
+
+    for generator in generators:
+        for _ in range(ERROR_IDX):
+            await generator.__anext__()
+
+        if error_type == "runtime_error":
+            with pytest.raises(RuntimeError, match=ERROR_MSG):
+                await generator.__anext__()
+        elif error_type == "mismatched_lengths":
+            with pytest.raises(RayServeException):
+                await generator.__anext__()
+
+        with pytest.raises(StopAsyncIteration):
+            await generator.__anext__()
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("stop_token", [StopAsyncIteration, StopIteration])
+async def test_batch_generator_early_termination(stop_token):
+    NUM_CALLERS = 4
+
+    event = asyncio.Event()
+
+    @serve.batch(max_batch_size=NUM_CALLERS, batch_wait_timeout_s=1000)
+    async def sequential_terminator(ids: List[int]):
+        """Terminates callers one-after-another in order of call."""
+
+        for num_finished_callers in range(1, NUM_CALLERS + 1):
+            event.clear()
+            responses = [stop_token for _ in range(num_finished_callers)]
+            responses += [ids[idx] for idx in range(num_finished_callers, NUM_CALLERS)]
+            yield responses
+            await event.wait()
+
+    ids = list(range(NUM_CALLERS))
+    generators = [sequential_terminator(id) for id in ids]
+    for id, generator in zip(ids, generators):
+        async for result in generator:
+            assert result == id
+
+        # Each terminated caller frees the sequential_terminator to process
+        # another iteration.
+        event.set()
+
+
+@pytest.mark.asyncio
+async def test_batch_generator_setters():
+    """@serve.batch setters should succeed while the current batch streams."""
+
+    @serve.batch(max_batch_size=2, batch_wait_timeout_s=1000)
+    async def yield_three_times(key1, key2):
+        for _ in range(3):
+            yield [(key1[i], key2[i]) for i in range(len(key1))]
+
+    assert yield_three_times._get_max_batch_size() == 2
+    assert yield_three_times._get_batch_wait_timeout_s() == 1000
+
+    args_list = [("hi1", "hi2"), ("hi3", "hi4")]
+    coros = [yield_three_times(*args) for args in args_list]
+
+    # Partially consume generators
+    for coro, expected_result in zip(coros, args_list):
+        for _ in range(2):
+            await coro.__anext__() == expected_result
+
+    # Set new values
+    yield_three_times.set_max_batch_size(3)
+    yield_three_times.set_batch_wait_timeout_s(15000)
+
+    assert yield_three_times._get_max_batch_size() == 3
+    assert yield_three_times._get_batch_wait_timeout_s() == 15000
+
+    # Execute three more requests
+    args_list_2 = [("hi1", "hi2"), ("hi3", "hi4"), ("hi5", "hi6")]
+    coros_2 = [yield_three_times(*args) for args in args_list_2]
+
+    # Finish consuming original requests
+    for coro, expected_result in zip(coros, args_list):
+        await coro.__anext__() == expected_result
+        with pytest.raises(StopAsyncIteration):
+            await coro.__anext__()
+
+    # Consume new requests
+    for coro, expected_result in zip(coros_2, args_list_2):
+        for _ in range(3):
+            await coro.__anext__() == expected_result
+        with pytest.raises(StopAsyncIteration):
+            await coro.__anext__()
+
+
+@pytest.mark.asyncio
+async def test_batch_size_fn_deferred_item_early_break():
+    batches_processed = []
+
+    @serve.batch(
+        max_batch_size=10,
+        batch_wait_timeout_s=0.05,
+        batch_size_fn=lambda items: sum(item["size"] for item in items),
+    )
+    async def batch_handler(requests):
+        batches_processed.append([req["value"] for req in requests])
+        return [req["value"] for req in requests]
+
+    # Request 1: size=6 (fits in batch)
+    # Request 2: size=6 (would make total 12 > 10, should be deferred)
+    # Each should be processed in its own batch
+    t1 = get_or_create_event_loop().create_task(
+        batch_handler({"size": 6, "value": "first"})
+    )
+    t2 = get_or_create_event_loop().create_task(
+        batch_handler({"size": 6, "value": "second"})
+    )
+
+    done, pending = await asyncio.wait([t1, t2], timeout=1.0)
+
+    assert len(done) == 2, "Both tasks should complete"
+    assert len(pending) == 0
+
+    results = {t1.result(), t2.result()}
+    assert results == {"first", "second"}
+
+    # Verify they were processed in separate batches due to size constraint
+    assert (
+        len(batches_processed) == 2
+    ), f"Expected 2 separate batches, got {batches_processed}"
+
+
+@pytest.mark.asyncio
+async def test_batch_size_fn_fail_to_fit():
+    batches_processed = []
+
+    @serve.batch(
+        max_batch_size=10,
+        batch_wait_timeout_s=0.05,
+        batch_size_fn=lambda items: sum(item["size"] for item in items),
+    )
+    async def batch_handler(requests):
+        batches_processed.append([req["value"] for req in requests])
+        return [req["value"] for req in requests]
+
+    # Request 1: size=6 (fits in batch)
+    # Request 2: size=6 (would make total 12 > 10, should be deferred)
+    # Each should be processed in its own batch
+    t1 = get_or_create_event_loop().create_task(
+        batch_handler({"size": 6, "value": "first"})
+    )
+    t2 = get_or_create_event_loop().create_task(
+        batch_handler({"size": 12, "value": "second"})
+    )
+
+    t1_result = await t1
+    assert t1_result == "first"
+    with pytest.raises(RuntimeError):
+        await t2
+
+
+@pytest.mark.asyncio
+async def test_batch_size_fn_multiple_items_fit():
+    """Test that multiple items are batched together when they fit within max_batch_size."""
+    batches_seen = []
+
+    @serve.batch(
+        max_batch_size=20,
+        batch_wait_timeout_s=0.1,
+        batch_size_fn=lambda items: sum(item["size"] for item in items),
+    )
+    async def batch_handler(requests):
+        batch_values = [req["value"] for req in requests]
+        batches_seen.append(batch_values)
+        return batch_values
+
+    # All three requests fit: 5 + 6 + 7 = 18 <= 20
+    t1 = get_or_create_event_loop().create_task(
+        batch_handler({"size": 5, "value": "a"})
+    )
+    t2 = get_or_create_event_loop().create_task(
+        batch_handler({"size": 6, "value": "b"})
+    )
+    t3 = get_or_create_event_loop().create_task(
+        batch_handler({"size": 7, "value": "c"})
+    )
+
+    done, pending = await asyncio.wait([t1, t2, t3], timeout=1.0)
+    assert len(done) == 3
+    assert len(pending) == 0
+
+    # All three should be in the same batch since they fit
+    assert len(batches_seen) == 1, f"Expected 1 batch, got {len(batches_seen)}"
+    assert set(batches_seen[0]) == {"a", "b", "c"}
+
+
+def test_warn_if_max_batch_size_exceeds_max_ongoing_requests():
+    """Test warn_if_max_batch_size_exceeds_max_ongoing_requests() logged the warning
+     message correctly.
+
+    When the queue starts with or updated `max_batch_size` to be larger than
+    max_ongoing_requests, log the warning to suggest configuring `max_ongoing_requests`.
+    When the queue starts with or updated `max_batch_size` to be smaller or equal than
+    max_ongoing_requests, no warning should be logged.
+    """
+    logger = logging.getLogger(SERVE_LOGGER_NAME)
+    stream = FakeStream()
+    stream_handler = logging.StreamHandler(stream)
+    logger.addHandler(stream_handler)
+    bound = default_deployment_config.max_ongoing_requests
+    over_bound = bound + 1
+    under_bound = bound - 1
+    over_bound_warning_message = (
+        f"`max_batch_size` ({over_bound}) * `max_concurrent_batches` "
+        f"({1}) is larger than `max_ongoing_requests` "
+        f"({bound}). This means the replica will never achieve "
+        "the configured `max_batch_size` concurrently. Please update "
+        "`max_ongoing_requests` to be >= `max_batch_size` * `max_concurrent_batches`.\n"
+    )
+
+    # Start queue above the bound will log warning. Start at under or at the bound will
+    # not log warning
+    for max_batch_size in [over_bound, under_bound, bound]:
+        queue = _BatchQueue(
+            max_batch_size=max_batch_size,
+            batch_wait_timeout_s=1000,
+            max_concurrent_batches=1,
+        )
+        if max_batch_size > bound:
+            assert over_bound_warning_message in stream.messages
+        else:
+            assert over_bound_warning_message not in stream.messages
+        stream.reset_message()
+
+    # Update queue above the bound will log warning. Update at under or at the bound
+    # will not log warning
+    for max_batch_size in [over_bound, under_bound, bound]:
+        queue.set_max_batch_size(max_batch_size)
+        if max_batch_size > bound:
+            assert over_bound_warning_message in stream.messages
+        else:
+            assert over_bound_warning_message not in stream.messages
+        stream.reset_message()
+
+
+if __name__ == "__main__":
+    import sys
+
+    sys.exit(pytest.main(["-v", "-s", __file__]))

python/ray/serve/tests/unit/test_build_app.py

@@ -0,0 +1,857 @@
+import sys
+from typing import Any, Dict, List, Optional
+
+import pytest
+from fastapi import FastAPI
+
+from ray import serve
+from ray.serve._private.build_app import (
+    CUSTOM_INGRESS_REQUEST_ROUTER_UNSUPPORTED_ERROR,
+    BuiltApplication,
+    build_app,
+)
+from ray.serve._private.client import ServeControllerClient
+from ray.serve._private.common import DeploymentID
+from ray.serve.config import RequestRouterConfig
+from ray.serve.deployment import Application, Deployment
+from ray.serve.exceptions import RayServeException
+from ray.serve.experimental.round_robin_router import RoundRobinRouter
+from ray.serve.handle import DeploymentHandle
+
+
+class FakeDeploymentHandle:
+    def __init__(self, deployment_name: str, app_name: str):
+        self.deployment_id = DeploymentID(deployment_name, app_name)
+
+    @classmethod
+    def from_deployment(cls, deployment, app_name: str) -> "FakeDeploymentHandle":
+        return cls(deployment.name, app_name)
+
+    def __eq__(self, other: Any) -> bool:
+        return (
+            isinstance(other, FakeDeploymentHandle)
+            and self.deployment_id == other.deployment_id
+        )
+
+
+def _build_and_check(
+    app: Application,
+    *,
+    expected_ingress_name: str,
+    expected_deployments: List[Deployment],
+    app_name: str = "default",
+    default_runtime_env: Optional[Dict[str, Any]] = None,
+):
+    built_app: BuiltApplication = build_app(
+        app,
+        name=app_name,
+        # Each real DeploymentHandle has a unique ID (intentionally), so the below
+        # equality checks don't work. Use a fake implementation instead.
+        make_deployment_handle=FakeDeploymentHandle.from_deployment,
+        default_runtime_env=default_runtime_env,
+    )
+    assert built_app.name == app_name
+    assert built_app.ingress_deployment_name == expected_ingress_name
+    assert len(built_app.deployments) == len(expected_deployments)
+
+    # Check that the returned deployment_handles are populated properly.
+    assert len(built_app.deployment_handles) == len(expected_deployments)
+    for d in expected_deployments:
+        h = built_app.deployment_handles.get(d.name, None)
+        assert h is not None, f"No handle returned for deployment {d.name}."
+        assert isinstance(h, FakeDeploymentHandle)
+        assert h.deployment_id == DeploymentID(d.name, app_name=app_name)
+
+    for expected_deployment in expected_deployments:
+        generated_deployment = None
+        for d in built_app.deployments:
+            if d.name == expected_deployment.name:
+                generated_deployment = d
+
+        assert generated_deployment is not None, (
+            f"Expected a deployment with name '{expected_deployment.name}' "
+            "to be generated but none was found. All generated names: "
+            + str([d.name for d in built_app.deployments])
+        )
+
+        assert expected_deployment == generated_deployment
+
+
+def test_real_deployment_handle_default():
+    """Other tests inject a FakeDeploymentHandle, so check the default behavior."""
+
+    @serve.deployment
+    class D:
+        pass
+
+    built_app: BuiltApplication = build_app(
+        D.bind(D.options(name="Inner").bind()),
+        name="app-name",
+    )
+    assert len(built_app.deployments) == 2
+    assert len(built_app.deployments[1].init_args) == 1
+    assert isinstance(built_app.deployments[1].init_args[0], DeploymentHandle)
+    assert built_app.deployments[1].init_args[0].deployment_id == DeploymentID(
+        "Inner", app_name="app-name"
+    )
+
+
+def test_single_deployment_basic():
+    @serve.deployment(
+        num_replicas=123,
+        max_ongoing_requests=10,
+        max_queued_requests=10,
+    )
+    class D:
+        pass
+
+    app = D.bind("hello world!", hello="world")
+    _build_and_check(
+        app,
+        expected_ingress_name="D",
+        expected_deployments=[
+            D.options(
+                name="D", _init_args=("hello world!",), _init_kwargs={"hello": "world"}
+            )
+        ],
+    )
+
+
+def test_single_deployment_custom_name():
+    @serve.deployment(
+        num_replicas=123,
+        max_ongoing_requests=10,
+        max_queued_requests=10,
+    )
+    class D:
+        pass
+
+    app = D.options(name="foobar123").bind("hello world!", hello="world")
+    _build_and_check(
+        app,
+        expected_ingress_name="foobar123",
+        expected_deployments=[
+            D.options(
+                name="foobar123",
+                _init_args=("hello world!",),
+                _init_kwargs={"hello": "world"},
+            )
+        ],
+    )
+
+    # Change to `with pytest.raises(ValueError)` when the warning is removed.
+    with pytest.warns(UserWarning):
+
+        @serve.deployment(name="test#deployment")
+        def my_deployment():
+            return "Hello!"
+
+    with pytest.warns(UserWarning):
+
+        @serve.deployment()
+        def my_deployment():
+            return "Hello!"
+
+        my_deployment.options(name="test#deployment")
+
+
+def test_multi_deployment_basic():
+    @serve.deployment(num_replicas=3)
+    class Inner:
+        pass
+
+    @serve.deployment(num_replicas=1)
+    class Outer:
+        pass
+
+    app = Outer.bind(Inner.bind(), other=Inner.options(name="Other").bind())
+    _build_and_check(
+        app,
+        expected_ingress_name="Outer",
+        expected_deployments=[
+            Inner.options(name="Inner", _init_args=tuple(), _init_kwargs={}),
+            Inner.options(name="Other", _init_args=tuple(), _init_kwargs={}),
+            Outer.options(
+                name="Outer",
+                _init_args=(
+                    FakeDeploymentHandle(
+                        "Inner",
+                        app_name="default",
+                    ),
+                ),
+                _init_kwargs={
+                    "other": FakeDeploymentHandle(
+                        "Other",
+                        app_name="default",
+                    ),
+                },
+            ),
+        ],
+    )
+
+
+def test_multi_deployment_handle_in_nested_obj():
+    @serve.deployment(num_replicas=3)
+    class Inner:
+        pass
+
+    @serve.deployment(num_replicas=1)
+    class Outer:
+        pass
+
+    app = Outer.bind([Inner.bind()])
+    _build_and_check(
+        app,
+        expected_ingress_name="Outer",
+        expected_deployments=[
+            Inner.options(name="Inner", _init_args=tuple(), _init_kwargs={}),
+            Outer.options(
+                name="Outer",
+                _init_args=(
+                    [
+                        FakeDeploymentHandle(
+                            "Inner",
+                            app_name="default",
+                        ),
+                    ],
+                ),
+                _init_kwargs={},
+            ),
+        ],
+    )
+
+
+def test_multi_deployment_custom_app_name():
+    @serve.deployment(num_replicas=3)
+    class Inner:
+        pass
+
+    @serve.deployment(num_replicas=1)
+    class Outer:
+        pass
+
+    app = Outer.bind(Inner.bind())
+    _build_and_check(
+        app,
+        app_name="custom",
+        expected_ingress_name="Outer",
+        expected_deployments=[
+            Inner.options(name="Inner", _init_args=tuple(), _init_kwargs={}),
+            Outer.options(
+                name="Outer",
+                _init_args=(
+                    FakeDeploymentHandle(
+                        "Inner",
+                        app_name="custom",
+                    ),
+                ),
+                _init_kwargs={},
+            ),
+        ],
+    )
+
+
+def test_multi_deployment_name_collision():
+    @serve.deployment
+    class Inner:
+        pass
+
+    @serve.deployment
+    class Outer:
+        pass
+
+    app = Outer.bind(
+        Inner.bind("arg1"),
+        Inner.bind("arg2"),
+    )
+    _build_and_check(
+        app,
+        expected_ingress_name="Outer",
+        expected_deployments=[
+            Inner.options(name="Inner", _init_args=("arg1",), _init_kwargs={}),
+            Inner.options(name="Inner_1", _init_args=("arg2",), _init_kwargs={}),
+            Outer.options(
+                name="Outer",
+                _init_args=(
+                    FakeDeploymentHandle(
+                        "Inner",
+                        app_name="default",
+                    ),
+                    FakeDeploymentHandle(
+                        "Inner_1",
+                        app_name="default",
+                    ),
+                ),
+                _init_kwargs={},
+            ),
+        ],
+    )
+
+
+def test_multi_deployment_same_app_passed_twice():
+    @serve.deployment
+    class Shared:
+        pass
+
+    @serve.deployment(num_replicas=3)
+    class Inner:
+        pass
+
+    @serve.deployment(num_replicas=1)
+    class Outer:
+        pass
+
+    shared = Shared.bind()
+    app = Outer.bind(Inner.bind(shared), shared)
+    shared_handle = FakeDeploymentHandle(
+        "Shared",
+        app_name="default",
+    )
+    _build_and_check(
+        app,
+        expected_ingress_name="Outer",
+        expected_deployments=[
+            Shared.options(
+                name="Shared",
+                _init_args=tuple(),
+                _init_kwargs={},
+            ),
+            Inner.options(name="Inner", _init_args=(shared_handle,), _init_kwargs={}),
+            Outer.options(
+                name="Outer",
+                _init_args=(
+                    FakeDeploymentHandle(
+                        "Inner",
+                        app_name="default",
+                    ),
+                    shared_handle,
+                ),
+                _init_kwargs={},
+            ),
+        ],
+    )
+
+
+def test_default_runtime_env():
+    @serve.deployment
+    class NoRayActorOptions:
+        pass
+
+    @serve.deployment(ray_actor_options={"num_cpus": 0, "num_gpus": 1})
+    class NoRuntimeEnv:
+        pass
+
+    @serve.deployment(ray_actor_options={"runtime_env": {"env_vars": {"ENV": "VAR"}}})
+    class RuntimeEnvNoWorkingDir:
+        pass
+
+    @serve.deployment(
+        ray_actor_options={"runtime_env": {"working_dir": "s3://test.zip"}}
+    )
+    class RuntimeEnvWithWorkingDir:
+        pass
+
+    @serve.deployment
+    class Outer:
+        pass
+
+    app = Outer.bind(
+        NoRayActorOptions.bind(),
+        NoRuntimeEnv.bind(),
+        RuntimeEnvNoWorkingDir.bind(),
+        RuntimeEnvWithWorkingDir.bind(),
+    )
+
+    handles = tuple(
+        FakeDeploymentHandle(name, app_name="default")
+        for name in [
+            "NoRayActorOptions",
+            "NoRuntimeEnv",
+            "RuntimeEnvNoWorkingDir",
+            "RuntimeEnvWithWorkingDir",
+        ]
+    )
+
+    # 1) Test behavior when no default_runtime_env is passed.
+    _build_and_check(
+        app,
+        expected_ingress_name="Outer",
+        expected_deployments=[
+            Outer.options(name="Outer", _init_args=handles, _init_kwargs={}),
+            NoRayActorOptions.options(
+                name="NoRayActorOptions", _init_args=tuple(), _init_kwargs={}
+            ),
+            NoRuntimeEnv.options(
+                name="NoRuntimeEnv", _init_args=tuple(), _init_kwargs={}
+            ),
+            RuntimeEnvNoWorkingDir.options(
+                name="RuntimeEnvNoWorkingDir", _init_args=tuple(), _init_kwargs={}
+            ),
+            RuntimeEnvWithWorkingDir.options(
+                name="RuntimeEnvWithWorkingDir", _init_args=tuple(), _init_kwargs={}
+            ),
+        ],
+    )
+
+    # 2) Test behavior when a default_runtime_env is passed without a working_dir.
+    default_runtime_env = {"env_vars": {"DEFAULT": "ENV"}}
+    _build_and_check(
+        app,
+        expected_ingress_name="Outer",
+        expected_deployments=[
+            Outer.options(
+                name="Outer",
+                _init_args=handles,
+                _init_kwargs={},
+                ray_actor_options={"num_cpus": 1, "runtime_env": default_runtime_env},
+            ),
+            NoRayActorOptions.options(
+                name="NoRayActorOptions",
+                _init_args=tuple(),
+                _init_kwargs={},
+                ray_actor_options={"num_cpus": 1, "runtime_env": default_runtime_env},
+            ),
+            NoRuntimeEnv.options(
+                name="NoRuntimeEnv",
+                _init_args=tuple(),
+                _init_kwargs={},
+                ray_actor_options={
+                    "num_cpus": 0,
+                    "num_gpus": 1,
+                    "runtime_env": default_runtime_env,
+                },
+            ),
+            # ray_actor_options shouldn't be affected.
+            RuntimeEnvNoWorkingDir.options(
+                name="RuntimeEnvNoWorkingDir", _init_args=tuple(), _init_kwargs={}
+            ),
+            # ray_actor_options shouldn't be affected.
+            RuntimeEnvWithWorkingDir.options(
+                name="RuntimeEnvWithWorkingDir", _init_args=tuple(), _init_kwargs={}
+            ),
+        ],
+        default_runtime_env=default_runtime_env,
+    )
+
+    # 3) Test behavior when a default_runtime_env is passed with a working_dir.
+    default_runtime_env = {
+        "working_dir": "s3://default.zip",
+        "env_vars": {"DEFAULT": "ENV"},
+    }
+    _build_and_check(
+        app,
+        expected_ingress_name="Outer",
+        expected_deployments=[
+            Outer.options(
+                name="Outer",
+                _init_args=handles,
+                _init_kwargs={},
+                ray_actor_options={"num_cpus": 1, "runtime_env": default_runtime_env},
+            ),
+            NoRayActorOptions.options(
+                name="NoRayActorOptions",
+                _init_args=tuple(),
+                _init_kwargs={},
+                ray_actor_options={
+                    "num_cpus": 1,
+                    "runtime_env": default_runtime_env,
+                },
+            ),
+            NoRuntimeEnv.options(
+                name="NoRuntimeEnv",
+                _init_args=tuple(),
+                _init_kwargs={},
+                ray_actor_options={
+                    "num_cpus": 0,
+                    "num_gpus": 1,
+                    "runtime_env": default_runtime_env,
+                },
+            ),
+            # Only the working_dir field should be overridden.
+            RuntimeEnvNoWorkingDir.options(
+                name="RuntimeEnvNoWorkingDir",
+                _init_args=tuple(),
+                _init_kwargs={},
+                ray_actor_options={
+                    "num_cpus": 1,
+                    "runtime_env": {
+                        "working_dir": "s3://default.zip",
+                        **RuntimeEnvNoWorkingDir.ray_actor_options["runtime_env"],
+                    },
+                },
+            ),
+            # ray_actor_options shouldn't be affected.
+            RuntimeEnvWithWorkingDir.options(
+                name="RuntimeEnvWithWorkingDir", _init_args=tuple(), _init_kwargs={}
+            ),
+        ],
+        default_runtime_env=default_runtime_env,
+    )
+
+
+def test_external_scaler_enabled():
+    """Test that external_scaler_enabled is correctly set in BuiltApplication.
+
+    This test verifies that when build_app is called with external_scaler_enabled=True
+    or external_scaler_enabled=False, the resulting BuiltApplication object correctly
+    stores the external_scaler_enabled value.
+    """
+
+    @serve.deployment
+    class D:
+        pass
+
+    app = D.bind()
+
+    # Test with external_scaler_enabled=True
+    built_app_with_scaler: BuiltApplication = build_app(
+        app,
+        name="app-with-scaler",
+        external_scaler_enabled=True,
+    )
+    assert built_app_with_scaler.external_scaler_enabled is True
+    assert built_app_with_scaler.name == "app-with-scaler"
+
+    # Test with external_scaler_enabled=False (explicit)
+    built_app_without_scaler: BuiltApplication = build_app(
+        app,
+        name="app-without-scaler",
+        external_scaler_enabled=False,
+    )
+    assert built_app_without_scaler.external_scaler_enabled is False
+    assert built_app_without_scaler.name == "app-without-scaler"
+
+    # Test with default value (should be False)
+    built_app_default: BuiltApplication = build_app(
+        app,
+        name="app-default",
+    )
+    assert built_app_default.external_scaler_enabled is False
+    assert built_app_default.name == "app-default"
+
+
+def test_ingress_name_got_modified():
+    """Test that the ingress deployment name is modified when there are name
+    collisions.
+    """
+
+    @serve.deployment
+    class D:
+        pass
+
+    app = D.bind(D.bind(D.bind()))
+    built_app: BuiltApplication = build_app(
+        app,
+        name="default",
+        make_deployment_handle=FakeDeploymentHandle.from_deployment,
+    )
+
+    # The ingress deployment name should be modified to avoid collision.
+    assert built_app.ingress_deployment_name == "D_2"
+
+    # The ingress deployment should be the last one.
+    assert len(built_app.deployments) == 3
+    assert built_app.deployments[-1].name == "D_2"
+
+
+def test_build_app_keeps_ingress_request_router_separate_from_app_deployments(
+    monkeypatch,
+):
+    monkeypatch.setattr("ray.serve._private.build_app.RAY_SERVE_ENABLE_HA_PROXY", True)
+
+    @serve.deployment
+    class Ingress:
+        pass
+
+    @serve.deployment
+    class IngressRequestRouter:
+        pass
+
+    ingress_app = Ingress.bind()
+    app = ingress_app._with_ingress_request_router(
+        IngressRequestRouter.bind(llm_deployment=ingress_app)
+    )
+
+    built_app: BuiltApplication = build_app(
+        app,
+        name="default",
+        make_deployment_handle=FakeDeploymentHandle.from_deployment,
+    )
+
+    assert [deployment.name for deployment in built_app.deployments] == ["Ingress"]
+    assert set(built_app.deployment_handles) == {"Ingress"}
+    assert built_app.ingress_request_router_deployment is not None
+    assert built_app.ingress_request_router_deployment.name == "IngressRequestRouter"
+
+
+def test_build_app_requires_ingress_request_router_to_be_single_deployment(
+    monkeypatch,
+):
+    monkeypatch.setattr("ray.serve._private.build_app.RAY_SERVE_ENABLE_HA_PROXY", True)
+
+    @serve.deployment
+    class Ingress:
+        pass
+
+    @serve.deployment
+    class RouterChild:
+        pass
+
+    @serve.deployment
+    class IngressRequestRouter:
+        def __init__(self, child):
+            self._child = child
+
+    with pytest.raises(
+        ValueError,
+        match=(
+            "`ingress_request_router` to build into exactly one standalone "
+            "deployment"
+        ),
+    ):
+        ingress_app = Ingress.bind()
+        app = ingress_app._with_ingress_request_router(
+            IngressRequestRouter.bind(RouterChild.bind())
+        )
+        build_app(
+            app,
+            name="default",
+            make_deployment_handle=FakeDeploymentHandle.from_deployment,
+        )
+
+
+def test_build_app_rejects_ingress_request_router_in_main_app_graph(monkeypatch):
+    monkeypatch.setattr("ray.serve._private.build_app.RAY_SERVE_ENABLE_HA_PROXY", True)
+
+    @serve.deployment
+    class IngressRequestRouter:
+        pass
+
+    @serve.deployment
+    class Ingress:
+        def __init__(self, router):
+            self._router = router
+
+    ingress_request_router = IngressRequestRouter.bind()
+    ingress_app = Ingress.bind(ingress_request_router)
+    app = ingress_app._with_ingress_request_router(ingress_request_router)
+
+    with pytest.raises(
+        ValueError,
+        match="did not produce any new deployments",
+    ):
+        build_app(
+            app,
+            name="default",
+            make_deployment_handle=FakeDeploymentHandle.from_deployment,
+        )
+
+
+def test_ingress_validation_excludes_ingress_request_router_fastapi_app(monkeypatch):
+    monkeypatch.setattr("ray.serve._private.build_app.RAY_SERVE_ENABLE_HA_PROXY", True)
+
+    ingress_api = FastAPI()
+    router_api = FastAPI()
+
+    @serve.deployment
+    @serve.ingress(ingress_api)
+    class Ingress:
+        pass
+
+    @serve.deployment
+    @serve.ingress(router_api)
+    class IngressRequestRouter:
+        pass
+
+    ingress_app = Ingress.bind()
+    app = ingress_app._with_ingress_request_router(
+        IngressRequestRouter.bind(llm_deployment=ingress_app)
+    )
+
+    built_app: BuiltApplication = build_app(
+        app,
+        name="default",
+        make_deployment_handle=FakeDeploymentHandle.from_deployment,
+    )
+
+    client = object.__new__(ServeControllerClient)
+    client._check_ingress_deployments([built_app])
+
+
+def test_callable_uses_multiplexing_static():
+    """Detects `@serve.multiplexed` on a method/function statically (class input)."""
+    from ray.serve._private.utils import _callable_uses_multiplexing
+
+    class MultiplexedClass:
+        @serve.multiplexed(max_num_models_per_replica=2)
+        async def load_model(self, model_id: str) -> str:
+            return model_id
+
+    @serve.multiplexed
+    async def standalone(model_id: str) -> str:
+        return model_id
+
+    class Plain:
+        async def __call__(self, request) -> str:
+            return "ok"
+
+    assert _callable_uses_multiplexing(MultiplexedClass)
+    assert _callable_uses_multiplexing(standalone)
+    assert not _callable_uses_multiplexing(Plain)
+
+
+def test_callable_uses_multiplexing_dynamic_instance():
+    """Detects multiplexing wired up dynamically at init (e.g. LLMServer LoRA).
+
+    This is only visible on a constructed instance, not on the class statically.
+    """
+    from ray.serve._private.utils import _callable_uses_multiplexing
+
+    class DynamicMultiplexed:
+        def __init__(self):
+            async def _load(model_id: str) -> str:
+                return model_id
+
+            self._load_model = serve.multiplexed(max_num_models_per_replica=2)(_load)
+
+    # Not detectable on the class (the wrapper only exists after construction)...
+    assert not _callable_uses_multiplexing(DynamicMultiplexed)
+    # ...but detectable on the constructed instance.
+    assert _callable_uses_multiplexing(DynamicMultiplexed())
+
+
+def test_callable_uses_multiplexing_ignores_handle_attrs():
+    """A composed ingress holding DeploymentHandles must not be flagged.
+
+    `DeploymentHandle.__getattr__` returns a (truthy) handle for any attribute name,
+    so the instance-attribute scan must check the marker by identity (`is True`) to
+    avoid false positives that would break every composed app under direct ingress.
+    """
+    from ray.serve._private.utils import _callable_uses_multiplexing
+
+    class FakeHandle:
+        # Mirrors DeploymentHandle: returns a truthy object for any attribute.
+        def __getattr__(self, name):
+            return self
+
+    class Ingress:
+        def __init__(self):
+            self._backend = FakeHandle()
+
+        async def __call__(self, request):
+            return await self._backend.remote()
+
+    assert not _callable_uses_multiplexing(Ingress())
+
+
+@pytest.mark.parametrize(
+    "haproxy_enabled, request_router_class, rejected",
+    [
+        # A custom router on the ingress is rejected only under HAProxy, which
+        # load-balances ingress traffic and bypasses the Serve request router.
+        (True, RoundRobinRouter, True),
+        (False, RoundRobinRouter, False),
+        # The default router (left unset) is not custom.
+        (True, None, False),
+    ],
+)
+def test_build_app_rejects_only_custom_ingress_request_router_under_haproxy(
+    monkeypatch, haproxy_enabled, request_router_class, rejected
+):
+    """A custom ingress router is rejected only under HAProxy. The default
+    router and the no-HAProxy case build."""
+    monkeypatch.setattr(
+        "ray.serve._private.build_app.RAY_SERVE_ENABLE_HA_PROXY", haproxy_enabled
+    )
+
+    options = {}
+    if request_router_class is not None:
+        options["request_router_config"] = RequestRouterConfig(
+            request_router_class=request_router_class
+        )
+
+    @serve.deployment
+    class Ingress:
+        pass
+
+    app = Ingress.options(**options).bind()
+
+    def build():
+        return build_app(
+            app,
+            name="default",
+            make_deployment_handle=FakeDeploymentHandle.from_deployment,
+        )
+
+    if rejected:
+        with pytest.raises(
+            RayServeException, match=CUSTOM_INGRESS_REQUEST_ROUTER_UNSUPPORTED_ERROR
+        ):
+            build()
+    else:
+        assert [deployment.name for deployment in build().deployments] == ["Ingress"]
+
+
+def test_build_app_allows_custom_ingress_request_router_in_direct_streaming(
+    monkeypatch,
+):
+    """Direct streaming attaches an ingress_request_router that delegates replica
+    selection back to the ingress deployment's router, so a custom router is
+    allowed even under HAProxy."""
+    monkeypatch.setattr("ray.serve._private.build_app.RAY_SERVE_ENABLE_HA_PROXY", True)
+
+    @serve.deployment(
+        request_router_config=RequestRouterConfig(request_router_class=RoundRobinRouter)
+    )
+    class Ingress:
+        pass
+
+    @serve.deployment
+    class IngressRequestRouter:
+        pass
+
+    ingress_app = Ingress.bind()
+    app = ingress_app._with_ingress_request_router(
+        IngressRequestRouter.bind(server=ingress_app)
+    )
+
+    built_app: BuiltApplication = build_app(
+        app,
+        name="default",
+        make_deployment_handle=FakeDeploymentHandle.from_deployment,
+    )
+
+    assert [deployment.name for deployment in built_app.deployments] == ["Ingress"]
+    assert built_app.ingress_request_router_deployment is not None
+
+
+def test_build_app_haproxy_allows_custom_router_on_non_ingress_deployment(monkeypatch):
+    """The guard targets only the ingress, so a custom router on a downstream
+    deployment is honored under HAProxy."""
+    monkeypatch.setattr("ray.serve._private.build_app.RAY_SERVE_ENABLE_HA_PROXY", True)
+
+    @serve.deployment(
+        request_router_config=RequestRouterConfig(request_router_class=RoundRobinRouter)
+    )
+    class Downstream:
+        pass
+
+    @serve.deployment
+    class Ingress:
+        def __init__(self, child):
+            pass
+
+    built_app: BuiltApplication = build_app(
+        Ingress.bind(Downstream.bind()),
+        name="default",
+        make_deployment_handle=FakeDeploymentHandle.from_deployment,
+    )
+
+    assert {deployment.name for deployment in built_app.deployments} == {
+        "Ingress",
+        "Downstream",
+    }
+
+
+if __name__ == "__main__":
+    sys.exit(pytest.main(["-v", "-s", __file__]))

python/ray/serve/tests/unit/test_capacity_queue.py

@@ -0,0 +1,662 @@
+import asyncio
+import sys
+from typing import List
+
+import pytest
+
+from ray.serve._private.common import DeploymentID
+from ray.serve.experimental.capacity_queue import (
+    CapacityQueue,
+    CapacityQueueStats,
+    ReplicaCapacityInfo,
+)
+
+
+def _get_raw_class():
+    """Get the underlying class without the @ray.remote decorator."""
+    return CapacityQueue.__ray_actor_class__
+
+
+def _create_queue(acquire_timeout_s=30.0, token_ttl_s=None):
+    """Create a CapacityQueue instance for local testing (no Ray)."""
+    raw_class = _get_raw_class()
+    return raw_class(
+        acquire_timeout_s=acquire_timeout_s,
+        token_ttl_s=token_ttl_s,
+        _enable_long_poll=False,
+    )
+
+
+class TestCapacityQueueStats:
+    """Tests for CapacityQueueStats dataclass."""
+
+    def test_stats_fields(self):
+        stats = CapacityQueueStats(
+            queue_size=10,
+            num_waiters=2,
+            num_replicas=3,
+            total_capacity=15,
+            total_in_flight=5,
+            total_acquires=100,
+            total_releases=95,
+            total_timeouts=1,
+            max_waiters_seen=5,
+        )
+
+        assert stats.queue_size == 10
+        assert stats.num_waiters == 2
+        assert stats.num_replicas == 3
+        assert stats.total_capacity == 15
+        assert stats.total_in_flight == 5
+        assert stats.total_acquires == 100
+        assert stats.total_releases == 95
+        assert stats.total_timeouts == 1
+        assert stats.max_waiters_seen == 5
+
+
+class TestReplicaCapacityInfo:
+    """Tests for ReplicaCapacityInfo dataclass."""
+
+    def test_default_values(self):
+        info = ReplicaCapacityInfo(replica_id="replica1", max_capacity=5)
+        assert info.replica_id == "replica1"
+        assert info.max_capacity == 5
+        assert info.in_flight == 0
+
+
+class TestCapacityQueueLocal:
+    """Tests for CapacityQueue without Ray (local logic only)."""
+
+    def test_initialization(self):
+        queue = _create_queue(acquire_timeout_s=10.0)
+        assert queue._acquire_timeout_s == 10.0
+        assert queue.get_queue_length() == 0
+        assert len(queue._waiters) == 0
+        assert len(queue._replicas) == 0
+
+    def test_register_replica(self):
+        queue = _create_queue()
+        queue.register_replica("replica1", 5)
+        assert queue.get_queue_length() == 5
+        assert "replica1" in queue._replicas
+        assert queue._replicas["replica1"].max_capacity == 5
+        assert queue._replicas["replica1"].in_flight == 0
+
+    def test_register_replica_duplicate(self):
+        queue = _create_queue()
+        queue.register_replica("replica1", 5)
+        queue.register_replica("replica1", 10)  # Should be ignored
+        assert queue.get_queue_length() == 5
+        assert queue._replicas["replica1"].max_capacity == 5
+
+    def test_register_multiple_replicas(self):
+        queue = _create_queue()
+        queue.register_replica("replica1", 3)
+        queue.register_replica("replica2", 5)
+        queue.register_replica("replica3", 2)
+        assert queue.get_queue_length() == 10
+        assert len(queue._replicas) == 3
+
+    def test_unregister_replica(self):
+        queue = _create_queue()
+        queue.register_replica("replica1", 5)
+        queue.register_replica("replica2", 3)
+        queue.unregister_replica("replica1")
+        assert queue.get_queue_length() == 3
+        assert "replica1" not in queue._replicas
+        assert "replica2" in queue._replicas
+
+    def test_unregister_nonexistent_replica(self):
+        queue = _create_queue()
+        queue.register_replica("replica1", 5)
+        queue.unregister_replica("nonexistent")  # Should not raise
+        assert queue.get_queue_length() == 5
+
+    def test_release_to_unregistered_replica(self):
+        queue = _create_queue()
+        queue.register_replica("replica1", 5)
+        initial_tokens = queue.get_queue_length()
+        queue.release("nonexistent")
+        assert queue.get_queue_length() == initial_tokens
+
+    def test_get_stats(self):
+        queue = _create_queue()
+        queue.register_replica("replica1", 5)
+        queue.register_replica("replica2", 3)
+        stats = queue.get_stats()
+        assert stats.queue_size == 8
+        assert stats.num_replicas == 2
+        assert stats.total_capacity == 8
+        assert stats.total_in_flight == 0
+
+    def test_get_registered_replicas(self):
+        queue = _create_queue()
+        queue.register_replica("replica1", 5)
+        queue.register_replica("replica2", 3)
+        replicas = queue.get_registered_replicas()
+        assert set(replicas) == {"replica1", "replica2"}
+
+
+class TestCapacityQueueAsync:
+    """Async tests for CapacityQueue."""
+
+    @pytest.mark.asyncio
+    async def test_acquire_immediate(self):
+        queue = _create_queue()
+        queue.register_replica("replica1", 5)
+        replica_id = await queue.acquire()
+        assert replica_id == "replica1"
+        assert queue._replicas["replica1"].in_flight == 1
+        assert queue.get_queue_length() == 4
+
+    @pytest.mark.asyncio
+    async def test_acquire_and_release(self):
+        queue = _create_queue()
+        queue.register_replica("replica1", 2)
+
+        r1 = await queue.acquire()
+        r2 = await queue.acquire()
+        assert queue.get_queue_length() == 0
+        assert queue._replicas["replica1"].in_flight == 2
+
+        queue.release(r1)
+        assert queue.get_queue_length() == 1
+        assert queue._replicas["replica1"].in_flight == 1
+
+        queue.release(r2)
+        assert queue.get_queue_length() == 2
+        assert queue._replicas["replica1"].in_flight == 0
+
+    @pytest.mark.asyncio
+    async def test_acquire_timeout(self):
+        queue = _create_queue(acquire_timeout_s=0.1)
+        # No replicas registered
+        replica_id = await queue.acquire()
+        assert replica_id is None
+        assert queue._total_timeouts == 1
+
+    @pytest.mark.asyncio
+    async def test_acquire_timeout_custom(self):
+        queue = _create_queue(acquire_timeout_s=10.0)
+        replica_id = await queue.acquire(timeout_s=0.1)
+        assert replica_id is None
+
+    @pytest.mark.asyncio
+    async def test_acquire_waits_for_release(self):
+        queue = _create_queue()
+        queue.register_replica("replica1", 1)
+
+        await queue.acquire()
+        assert queue.get_queue_length() == 0
+
+        acquire_task = asyncio.create_task(queue.acquire(timeout_s=5.0))
+        await asyncio.sleep(0.01)
+        assert queue.get_num_waiters() == 1
+
+        queue.release("replica1")
+        replica_id = await acquire_task
+        assert replica_id == "replica1"
+        assert queue.get_num_waiters() == 0
+
+    @pytest.mark.asyncio
+    async def test_acquire_waits_for_registration(self):
+        queue = _create_queue()
+
+        acquire_task = asyncio.create_task(queue.acquire(timeout_s=5.0))
+        await asyncio.sleep(0.01)
+        assert queue.get_num_waiters() == 1
+
+        queue.register_replica("replica1", 3)
+        replica_id = await acquire_task
+        assert replica_id == "replica1"
+        assert queue.get_num_waiters() == 0
+
+    @pytest.mark.asyncio
+    async def test_acquire_cancellation(self):
+        queue = _create_queue()
+
+        acquire_task = asyncio.create_task(queue.acquire(timeout_s=5.0))
+        await asyncio.sleep(0.01)
+        assert queue.get_num_waiters() == 1
+
+        acquire_task.cancel()
+        try:
+            await acquire_task
+        except asyncio.CancelledError:
+            pass
+
+        await asyncio.sleep(0.01)
+        assert queue.get_num_waiters() == 0
+
+    @pytest.mark.asyncio
+    async def test_least_loaded_distribution(self):
+        queue = _create_queue()
+        queue.register_replica("replica1", 3)
+        queue.register_replica("replica2", 3)
+
+        # Acquire 2 tokens - should distribute evenly
+        await queue.acquire()
+        await queue.acquire()
+
+        assert queue._replicas["replica1"].in_flight == 1
+        assert queue._replicas["replica2"].in_flight == 1
+
+        # Acquire 2 more - again evenly
+        await queue.acquire()
+        await queue.acquire()
+
+        assert queue._replicas["replica1"].in_flight == 2
+        assert queue._replicas["replica2"].in_flight == 2
+
+    @pytest.mark.asyncio
+    async def test_max_waiters_tracking(self):
+        queue = _create_queue()
+
+        tasks = [asyncio.create_task(queue.acquire(timeout_s=0.2)) for _ in range(5)]
+        await asyncio.sleep(0.05)
+        await asyncio.gather(*tasks)
+
+        stats = queue.get_stats()
+        assert stats.max_waiters_seen >= 5
+
+
+class TestCapacityQueueStatsTracking:
+    """Tests for statistics tracking during operations."""
+
+    @pytest.mark.asyncio
+    async def test_total_acquires_increments(self):
+        queue = _create_queue(acquire_timeout_s=0.1)
+        queue.register_replica("replica1", 2)
+
+        assert queue._total_acquires == 0
+
+        await queue.acquire()
+        assert queue._total_acquires == 1
+
+        await queue.acquire()
+        assert queue._total_acquires == 2
+
+        # Timeout also counts
+        await queue.acquire()
+        assert queue._total_acquires == 3
+
+    @pytest.mark.asyncio
+    async def test_total_releases_increments(self):
+        queue = _create_queue()
+        queue.register_replica("replica1", 3)
+
+        assert queue._total_releases == 0
+
+        r1 = await queue.acquire()
+        r2 = await queue.acquire()
+
+        queue.release(r1)
+        assert queue._total_releases == 1
+
+        queue.release(r2)
+        assert queue._total_releases == 2
+
+    @pytest.mark.asyncio
+    async def test_in_flight_tracking(self):
+        queue = _create_queue()
+        queue.register_replica("replica1", 3)
+
+        assert queue._replicas["replica1"].in_flight == 0
+
+        r1 = await queue.acquire()
+        assert queue._replicas["replica1"].in_flight == 1
+
+        r2 = await queue.acquire()
+        assert queue._replicas["replica1"].in_flight == 2
+
+        queue.release(r1)
+        assert queue._replicas["replica1"].in_flight == 1
+
+        queue.release(r2)
+        assert queue._replicas["replica1"].in_flight == 0
+
+    @pytest.mark.asyncio
+    async def test_stats_reflect_operations(self):
+        queue = _create_queue(acquire_timeout_s=0.1)
+        queue.register_replica("replica1", 2)
+        queue.register_replica("replica2", 1)
+
+        r1 = await queue.acquire()
+        await queue.acquire()
+
+        stats = queue.get_stats()
+        assert stats.total_capacity == 3
+        assert stats.total_in_flight == 2
+        assert stats.queue_size == 1
+        assert stats.total_acquires == 2
+
+        queue.release(r1)
+
+        stats = queue.get_stats()
+        assert stats.total_in_flight == 1
+        assert stats.queue_size == 2
+        assert stats.total_releases == 1
+
+
+class TestCapacityQueueEdgeCases:
+    """Tests for edge cases and error handling."""
+
+    def test_register_zero_capacity(self):
+        queue = _create_queue()
+        queue.register_replica("replica1", 0)
+        assert "replica1" in queue._replicas
+        assert queue._replicas["replica1"].max_capacity == 0
+        assert queue.get_queue_length() == 0
+
+    @pytest.mark.asyncio
+    async def test_unregister_while_in_flight(self):
+        queue = _create_queue()
+        queue.register_replica("replica1", 3)
+
+        r1 = await queue.acquire()
+        assert queue._replicas["replica1"].in_flight == 1
+
+        queue.unregister_replica("replica1")
+        assert "replica1" not in queue._replicas
+        assert queue.get_queue_length() == 0
+
+        # Release should be handled gracefully (discarded)
+        queue.release(r1)
+        assert queue.get_queue_length() == 0
+
+    @pytest.mark.asyncio
+    async def test_release_same_token_twice(self):
+        """Releasing the same token twice doesn't inflate capacity."""
+        queue = _create_queue()
+        queue.register_replica("replica1", 1)
+
+        r1 = await queue.acquire()
+        assert queue.get_queue_length() == 0
+
+        queue.release(r1)
+        assert queue.get_queue_length() == 1
+
+        # Release again - in_flight clamped to 0
+        queue.release(r1)
+        assert queue.get_queue_length() == 1  # Correctly doesn't inflate
+
+    @pytest.mark.asyncio
+    async def test_fifo_waiter_ordering(self):
+        """Waiters are fulfilled in FIFO order."""
+        queue = _create_queue()
+
+        completion_order: List[int] = []
+
+        async def waiter(waiter_id: int):
+            await queue.acquire(timeout_s=5.0)
+            completion_order.append(waiter_id)
+
+        tasks = [asyncio.create_task(waiter(i)) for i in range(3)]
+        await asyncio.sleep(0.05)
+        assert queue.get_num_waiters() == 3
+
+        queue.register_replica("replica1", 3)
+        await asyncio.gather(*tasks)
+
+        assert completion_order == [0, 1, 2]
+
+    @pytest.mark.asyncio
+    async def test_multiple_replicas_least_loaded(self):
+        """Least-loaded replica is selected."""
+        queue = _create_queue()
+        queue.register_replica("replica1", 2)
+        queue.register_replica("replica2", 2)
+
+        r1 = await queue.acquire()
+        r2 = await queue.acquire()
+
+        # Should be different replicas (load balanced)
+        assert r1 != r2
+
+        # Both should have 1 in-flight
+        assert queue._replicas["replica1"].in_flight == 1
+        assert queue._replicas["replica2"].in_flight == 1
+
+        # Acquire two more
+        await queue.acquire()
+        await queue.acquire()
+
+        assert queue._replicas["replica1"].in_flight == 2
+        assert queue._replicas["replica2"].in_flight == 2
+
+        # Clean up
+        queue.release(r1)
+        queue.release(r2)
+
+    @pytest.mark.asyncio
+    async def test_waiter_timeout_cleanup(self):
+        """Timed-out waiters are properly cleaned up."""
+        queue = _create_queue(acquire_timeout_s=0.1)
+
+        tasks = [asyncio.create_task(queue.acquire()) for _ in range(3)]
+        results = await asyncio.gather(*tasks)
+
+        assert all(r is None for r in results)
+        assert queue.get_num_waiters() == 0
+        assert queue._total_timeouts == 3
+
+    @pytest.mark.asyncio
+    async def test_cancel_before_token_acquired(self):
+        """Cancelling a request while waiting for a token (tokens exhausted)
+        should clean up the waiter without affecting capacity."""
+        queue = _create_queue()
+        queue.register_replica("replica1", 1)
+
+        # Exhaust the single token.
+        r1 = await queue.acquire()
+        assert queue.get_queue_length() == 0
+
+        # Second acquire blocks — no capacity available.
+        waiting_task = asyncio.create_task(queue.acquire(timeout_s=5.0))
+        await asyncio.sleep(0.01)
+        assert queue.get_num_waiters() == 1
+
+        # Cancel before a token is granted.
+        waiting_task.cancel()
+        with pytest.raises(asyncio.CancelledError):
+            await waiting_task
+
+        await asyncio.sleep(0.01)
+        assert queue.get_num_waiters() == 0
+        # Capacity unchanged — still 0 available (1 in-flight).
+        assert queue.get_queue_length() == 0
+        assert queue._replicas["replica1"].in_flight == 1
+
+        # Release the held token — full capacity restored.
+        queue.release(r1)
+        assert queue.get_queue_length() == 1
+        assert queue._replicas["replica1"].in_flight == 0
+
+    @pytest.mark.asyncio
+    async def test_cancel_after_token_acquired(self):
+        """Cancelling a request after a token was acquired should still allow
+        the caller to release the token, preserving capacity."""
+        queue = _create_queue()
+        queue.register_replica("replica1", 2)
+
+        # Acquire a token normally.
+        r1 = await queue.acquire()
+        assert r1 == "replica1"
+        assert queue._replicas["replica1"].in_flight == 1
+        assert queue.get_queue_length() == 1
+
+        # Simulate the caller being cancelled after acquiring — the token is
+        # held. The caller (router) is responsible for releasing it.
+        # Verify capacity is reduced while the token is held.
+        assert queue._replicas["replica1"].in_flight == 1
+
+        # Release the token (as the router's on_request_cancelled would do).
+        queue.release(r1)
+        assert queue._replicas["replica1"].in_flight == 0
+        assert queue.get_queue_length() == 2
+
+
+class TestReplicaLifecycleOnDeploymentTargetUpdate:
+    """Tests that the queue reflects replica changes from the controller.
+
+    The controller sends deployment target updates (via long poll) when replicas
+    are added or removed.  The queue must unregister removed replicas and
+    register new ones so that tokens are only issued for live replicas.
+    """
+
+    @staticmethod
+    def _make_target_info(replica_specs):
+        """Build a DeploymentTargetInfo from a list of (unique_id, capacity) tuples."""
+        from ray.serve._private.common import (
+            DeploymentTargetInfo,
+            ReplicaID,
+            RunningReplicaInfo,
+        )
+
+        dep_id = DeploymentID(name="test_deployment", app_name="test_app")
+        replicas = []
+        for uid, cap in replica_specs:
+            replicas.append(
+                RunningReplicaInfo(
+                    replica_id=ReplicaID(unique_id=uid, deployment_id=dep_id),
+                    node_id="node1",
+                    node_ip="127.0.0.1",
+                    availability_zone=None,
+                    actor_name=f"actor_{uid}",
+                    max_ongoing_requests=cap,
+                    is_cross_language=False,
+                )
+            )
+        return DeploymentTargetInfo(is_available=True, running_replicas=replicas)
+
+    def test_update_adds_new_replicas(self):
+        """New replicas in the update are registered in the queue."""
+        queue = _create_queue()
+        target = self._make_target_info([("r1", 5)])
+
+        queue._update_deployment_targets(target)
+
+        assert set(queue.get_registered_replicas()) == {"r1"}
+        assert queue.get_queue_length() == 5
+
+    def test_update_removes_old_replicas(self):
+        """Replicas absent from the update are unregistered."""
+        queue = _create_queue()
+        queue.register_replica("r1", 3)
+        queue.register_replica("r2", 2)
+        assert len(queue.get_registered_replicas()) == 2
+
+        # Update with only r1 — r2 should be removed
+        target = self._make_target_info([("r1", 3)])
+        queue._update_deployment_targets(target)
+
+        assert set(queue.get_registered_replicas()) == {"r1"}
+        assert queue.get_queue_length() == 3
+
+    def test_update_empty_removes_all(self):
+        """An update with no replicas clears the queue."""
+        queue = _create_queue()
+        queue.register_replica("r1", 3)
+
+        target = self._make_target_info([])
+        queue._update_deployment_targets(target)
+
+        assert queue.get_registered_replicas() == []
+        assert queue.get_queue_length() == 0
+
+    def test_update_is_idempotent(self):
+        """Calling update twice with the same replicas is a no-op."""
+        queue = _create_queue()
+        target = self._make_target_info([("r1", 5)])
+
+        queue._update_deployment_targets(target)
+        queue._update_deployment_targets(target)
+
+        assert set(queue.get_registered_replicas()) == {"r1"}
+        assert queue.get_queue_length() == 5
+
+    @pytest.mark.asyncio
+    async def test_scale_down_frees_dead_capacity(self):
+        """After scale-down, tokens for removed replicas are no longer issued."""
+        queue = _create_queue()
+        queue.register_replica("r1", 1)
+        queue.register_replica("r2", 1)
+
+        # Saturate both replicas
+        t1 = await queue.acquire()
+        t2 = await queue.acquire()
+        assert queue.get_queue_length() == 0
+
+        # Scale down: remove r2
+        target = self._make_target_info([("r1", 1)])
+        queue._update_deployment_targets(target)
+
+        # Release both tokens — only r1's should restore capacity
+        queue.release(t1)
+        queue.release(t2)  # r2 is gone, release is discarded
+
+        assert queue.get_queue_length() == 1
+        assert set(queue.get_registered_replicas()) == {"r1"}
+
+        # Next acquire must go to r1 (no phantom r2)
+        token = await queue.acquire(timeout_s=1.0)
+        assert token == "r1"
+
+
+class TestTokenTTL:
+    """Tests for the token TTL auto-reclaim feature."""
+
+    @pytest.mark.asyncio
+    async def test_expired_token_reclaimed(self):
+        """Tokens exceeding the TTL are automatically reclaimed."""
+        queue = _create_queue(token_ttl_s=0.2)
+        queue.register_replica("replica1", 2)
+
+        # Acquire a token (never release it).
+        r = await queue.acquire()
+        assert r == "replica1"
+        assert queue._replicas["replica1"].in_flight == 1
+        assert queue.get_queue_length() == 1
+
+        # Wait for the TTL reaper to reclaim it.
+        await asyncio.sleep(0.5)
+
+        assert queue._replicas["replica1"].in_flight == 0
+        assert queue.get_queue_length() == 2
+        assert queue._total_ttl_reclaims == 1
+
+    @pytest.mark.asyncio
+    async def test_ttl_does_not_reclaim_released_tokens(self):
+        """Properly released tokens are not double-reclaimed by the reaper."""
+        queue = _create_queue(token_ttl_s=0.2)
+        queue.register_replica("replica1", 2)
+
+        r = await queue.acquire()
+        queue.release(r)
+
+        assert queue._replicas["replica1"].in_flight == 0
+
+        # Wait past TTL — reaper should find nothing to reclaim.
+        await asyncio.sleep(0.5)
+
+        assert queue._replicas["replica1"].in_flight == 0
+        assert queue.get_queue_length() == 2
+        assert queue._total_ttl_reclaims == 0
+
+    @pytest.mark.asyncio
+    async def test_ttl_reclaim_wakes_waiters(self):
+        """Reclaimed capacity from TTL should wake blocked waiters."""
+        queue = _create_queue(token_ttl_s=0.2)
+        queue.register_replica("replica1", 1)
+
+        # Exhaust capacity.
+        await queue.acquire()
+        assert queue.get_queue_length() == 0
+
+        # This waiter should be woken once the TTL reaper reclaims.
+        result = await asyncio.wait_for(queue.acquire(), timeout=2.0)
+        assert result == "replica1"
+
+
+if __name__ == "__main__":
+    sys.exit(pytest.main([__file__, "-v"]))

python/ray/serve/tests/unit/test_cli.py

@@ -0,0 +1,194 @@
+import json
+import os
+import pathlib
+import sys
+from tempfile import NamedTemporaryFile
+from typing import Dict, Optional
+from unittest.mock import Mock, patch
+
+import click
+import pytest
+import yaml
+from click.testing import CliRunner
+
+from ray.serve.schema import ServeApplicationSchema
+from ray.serve.scripts import build, convert_args_to_dict, deploy
+
+
+def test_convert_args_to_dict():
+    assert convert_args_to_dict(tuple()) == {}
+
+    with pytest.raises(
+        click.ClickException, match="Invalid application argument 'bad_arg'"
+    ):
+        convert_args_to_dict(("bad_arg",))
+
+    with pytest.raises(
+        click.ClickException, match="Invalid application argument 'bad_arg='"
+    ):
+        convert_args_to_dict(("bad_arg=",))
+
+    assert convert_args_to_dict(("key1=val1", "key2=val2", "key3=nested=val")) == {
+        "key1": "val1",
+        "key2": "val2",
+        "key3": "nested=val",
+    }
+
+
+class FakeServeSubmissionClient:
+    def __init__(self):
+        self._deployed_config: Optional[Dict] = None
+
+    @property
+    def deployed_config(self) -> Optional[Dict]:
+        return self._deployed_config
+
+    def deploy_applications(self, config: Dict):
+        self._deployed_config = config
+
+
+@pytest.fixture
+def fake_serve_client() -> FakeServeSubmissionClient:
+    fake_client = FakeServeSubmissionClient()
+    with patch(
+        "ray.serve.scripts.ServeSubmissionClient",
+        new=Mock(return_value=fake_client),
+    ):
+        yield fake_client
+
+
+class TestDeploy:
+    def test_deploy_basic(self, fake_serve_client):
+        runner = CliRunner()
+        result = runner.invoke(deploy, ["my_module:my_app"])
+        assert result.exit_code == 0, result.output
+
+        assert fake_serve_client.deployed_config["applications"] == [
+            ServeApplicationSchema(
+                import_path="my_module:my_app",
+                args={},
+                runtime_env={},
+            ).model_dump(exclude_unset=True)
+        ]
+
+    def test_deploy_with_name(self, fake_serve_client):
+        runner = CliRunner()
+        result = runner.invoke(deploy, ["my_module:my_app", "--name", "test-name"])
+        assert result.exit_code == 0, result.output
+
+        assert fake_serve_client.deployed_config["applications"] == [
+            ServeApplicationSchema(
+                import_path="my_module:my_app",
+                name="test-name",
+                args={},
+                runtime_env={},
+            ).model_dump(exclude_unset=True)
+        ]
+
+    def test_deploy_with_args(self, fake_serve_client):
+        runner = CliRunner()
+        result = runner.invoke(deploy, ["my_module:my_app", "arg1=val1", "arg2=val2"])
+        assert result.exit_code == 0, result.output
+
+        assert fake_serve_client.deployed_config["applications"] == [
+            ServeApplicationSchema(
+                import_path="my_module:my_app",
+                args={"arg1": "val1", "arg2": "val2"},
+                runtime_env={},
+            ).model_dump(exclude_unset=True)
+        ]
+
+    @pytest.mark.skipif(sys.platform == "win32", reason="Tempfile not working.")
+    @pytest.mark.parametrize(
+        "runtime_env",
+        [
+            {"env_vars": {"hi": "123"}},
+            {
+                "working_dir": "s3://some_bucket/pkg.zip",
+                "py_modules": ["s3://some_other_bucket/pkg.zip"],
+            },
+        ],
+    )
+    @pytest.mark.parametrize("use_json", [False, True])
+    def test_deploy_with_runtime_env(
+        self, fake_serve_client, runtime_env: Dict, use_json: bool
+    ):
+        runner = CliRunner()
+        with NamedTemporaryFile("w") as f:
+            if use_json:
+                runtime_env_args = ["--runtime-env-json", json.dumps(runtime_env)]
+            else:
+                yaml.dump(runtime_env, f, default_flow_style=False)
+                runtime_env_args = ["--runtime-env", f.name]
+
+            result = runner.invoke(deploy, ["my_module:my_app"] + runtime_env_args)
+
+        assert result.exit_code == 0, result.output
+
+        assert fake_serve_client.deployed_config["applications"] == [
+            ServeApplicationSchema(
+                import_path="my_module:my_app",
+                args={},
+                runtime_env=runtime_env,
+            ).model_dump(exclude_unset=True)
+        ]
+
+
+class TestEnumSerialization:
+    """Test that enum representer correctly serializes enums in YAML dumps."""
+
+    def test_build_command_with_enum_serialization(self):
+        """Test that serve build correctly serializes AggregationFunction enum."""
+        runner = CliRunner()
+        with NamedTemporaryFile(mode="w", suffix=".py", delete=False) as f:
+            f.write(
+                "from ray import serve\n"
+                "from ray.serve.config import AggregationFunction, AutoscalingConfig\n\n"
+                "@serve.deployment(\n"
+                "    autoscaling_config=AutoscalingConfig(\n"
+                "        min_replicas=1,\n"
+                "        max_replicas=2,\n"
+                "        aggregation_function=AggregationFunction.MEAN,\n"
+                "    )\n"
+                ")\n"
+                "def test_deployment():\n"
+                "    return 'ok'\n\n"
+                "app = test_deployment.bind()\n"
+            )
+            temp_path = f.name
+
+        output_path = None
+        try:
+            import_path = f"{pathlib.Path(temp_path).stem}:app"
+            with NamedTemporaryFile(
+                mode="w", suffix=".yaml", delete=False
+            ) as output_file:
+                output_path = output_file.name
+
+            result = runner.invoke(
+                build,
+                [
+                    import_path,
+                    "--app-dir",
+                    str(pathlib.Path(temp_path).parent),
+                    "--output-path",
+                    output_path,
+                ],
+            )
+            assert result.exit_code == 0, result.output
+
+            with open(output_path, "r") as f:
+                config = yaml.safe_load(f)
+            agg_func = config["applications"][0]["deployments"][0][
+                "autoscaling_config"
+            ]["aggregation_function"]
+            assert agg_func == "mean"
+            assert isinstance(agg_func, str)
+        finally:
+            os.unlink(temp_path)
+            if output_path:
+                os.unlink(output_path)
+
+
+if __name__ == "__main__":
+    sys.exit(pytest.main(["-v", "-s", __file__]))

python/ray/serve/tests/unit/test_common.py

@@ -0,0 +1,257 @@
+import pytest
+
+from ray.serve._private.common import (
+    REPLICA_ID_FULL_ID_STR_PREFIX,
+    DeploymentID,
+    DeploymentStatus,
+    DeploymentStatusInfo,
+    DeploymentStatusInternalTrigger,
+    DeploymentStatusTrigger,
+    ReplicaID,
+    RunningReplicaInfo,
+)
+from ray.serve._private.constants import SERVE_DEPLOYMENT_ACTOR_PREFIX
+from ray.serve._private.utils import get_deployment_actor_name, get_random_string
+from ray.serve.generated.serve_pb2 import (
+    DeploymentStatusInfo as DeploymentStatusInfoProto,
+)
+
+
+def test_get_deployment_actor_name():
+    """Test deterministic actor name for deployment-scoped actors."""
+    dep_id = DeploymentID(name="MyDeployment", app_name="my_app")
+    assert get_deployment_actor_name(dep_id, "prefix_tree", "v1") == (
+        f"{SERVE_DEPLOYMENT_ACTOR_PREFIX}my_app::MyDeployment::v1::prefix_tree"
+    )
+
+    dep_id_default_app = DeploymentID(name="Other")  # app_name="default"
+    assert get_deployment_actor_name(dep_id_default_app, "x", "abc") == (
+        f"{SERVE_DEPLOYMENT_ACTOR_PREFIX}default::Other::abc::x"
+    )
+
+
+def test_replica_id_formatting():
+    deployment = "DeploymentA"
+    unique_id = get_random_string()
+
+    app = "my_app"
+    replica_id = ReplicaID(
+        unique_id, deployment_id=DeploymentID(name=deployment, app_name=app)
+    )
+    assert (
+        replica_id.to_full_id_str()
+        == f"{REPLICA_ID_FULL_ID_STR_PREFIX}{app}#{deployment}#{unique_id}"
+    )
+    assert (
+        str(replica_id)
+        == f"Replica(id='{unique_id}', deployment='{deployment}', app='{app}')"
+    )
+
+
+def test_replica_id_from_str():
+    unique_id = get_random_string()
+
+    # Test without app name.
+    full_id_str = f"{REPLICA_ID_FULL_ID_STR_PREFIX}DeploymentA#{unique_id}"
+    replica_id = ReplicaID.from_full_id_str(full_id_str)
+    assert replica_id.unique_id == unique_id
+    assert replica_id.deployment_id == DeploymentID(name="DeploymentA", app_name="")
+
+    # Test with app name.
+    full_id_str = f"{REPLICA_ID_FULL_ID_STR_PREFIX}App1#DeploymentA#{unique_id}"
+    replica_id = ReplicaID.from_full_id_str(full_id_str)
+    assert replica_id.unique_id == unique_id
+    assert replica_id.deployment_id == DeploymentID(name="DeploymentA", app_name="App1")
+
+
+def test_invalid_id_from_str():
+    unique_id = get_random_string()
+
+    # Missing prefix.
+    full_id_str = f"DeploymentA#{unique_id}"
+    with pytest.raises(AssertionError):
+        ReplicaID.from_full_id_str(full_id_str)
+
+    # Too many delimiters.
+    full_id_str = f"{REPLICA_ID_FULL_ID_STR_PREFIX}DeploymentA###{unique_id}"
+    with pytest.raises(ValueError):
+        ReplicaID.from_full_id_str(full_id_str)
+
+
+def test_is_replica_id():
+    unique_id = get_random_string()
+
+    assert not ReplicaID.is_full_id_str(f"DeploymentA##{unique_id}")
+    assert not ReplicaID.is_full_id_str(f"DeploymentA#{unique_id}")
+    assert ReplicaID.is_full_id_str(
+        f"{REPLICA_ID_FULL_ID_STR_PREFIX}DeploymentA#{unique_id}"
+    )
+    assert ReplicaID.is_full_id_str(
+        f"{REPLICA_ID_FULL_ID_STR_PREFIX}#App1#DeploymentA#{unique_id}"
+    )
+
+
+class TestDeploymentStatusInfo:
+    def test_name_required(self):
+        with pytest.raises(TypeError):
+            DeploymentStatusInfo(
+                status=DeploymentStatus.HEALTHY,
+                status_trigger=DeploymentStatusTrigger.CONFIG_UPDATE_STARTED,
+            )
+
+    def test_deployment_status_required(self):
+        with pytest.raises(TypeError):
+            DeploymentStatusInfo(
+                name="test_name",
+                status_trigger=DeploymentStatusTrigger.CONFIG_UPDATE_STARTED,
+            )
+
+    @pytest.mark.parametrize(
+        "status,status_trigger",
+        list(zip(list(DeploymentStatus), list(DeploymentStatusTrigger))),
+    )
+    def test_proto(self, status, status_trigger):
+        deployment_status_info = DeploymentStatusInfo(
+            name="test_name",
+            status=status,
+            status_trigger=status_trigger,
+            message="context about status",
+        )
+        serialized_proto = deployment_status_info.to_proto().SerializeToString()
+        deserialized_proto = DeploymentStatusInfoProto.FromString(serialized_proto)
+        reconstructed_info = DeploymentStatusInfo.from_proto(deserialized_proto)
+
+        assert deployment_status_info == reconstructed_info
+
+    def test_handle_transition_deployment_actor_failed_when_already_deploy_failed(
+        self,
+    ):
+        """DEPLOYMENT_ACTOR_FAILED must be handled in DEPLOY_FAILED block.
+
+        When status is already DEPLOY_FAILED, repeated ticks call handle_transition
+        with DEPLOYMENT_ACTOR_FAILED. Without handling, the trigger falls through
+        and returns self (old message). With handling, returns updated copy.
+        """
+        info = DeploymentStatusInfo(
+            name="test",
+            status=DeploymentStatus.DEPLOY_FAILED,
+            status_trigger=DeploymentStatusTrigger.DEPLOYMENT_ACTOR_FAILED,
+            message="original error message",
+        )
+        new_message = (
+            "The deployment failed to start deployment actors 2 times in a row."
+        )
+        result = info.handle_transition(
+            trigger=DeploymentStatusInternalTrigger.DEPLOYMENT_ACTOR_FAILED,
+            message=new_message,
+        )
+        assert result is not None
+        assert result.status == DeploymentStatus.DEPLOY_FAILED
+        assert result.status_trigger == DeploymentStatusTrigger.DEPLOYMENT_ACTOR_FAILED
+        assert result.message == new_message
+
+
+def test_running_replica_info():
+    """Test hash value of RunningReplicaInfo"""
+
+    replica_id = ReplicaID("asdf123", deployment_id=DeploymentID(name="my_deployment"))
+    actor_name = replica_id.to_full_id_str()
+
+    # Test that replicas with same attributes have same hash
+    replica1 = RunningReplicaInfo(
+        replica_id=replica_id,
+        node_id="node_id",
+        node_ip="node_ip",
+        availability_zone="some-az",
+        actor_name=actor_name,
+        max_ongoing_requests=1,
+        is_cross_language=False,
+    )
+    replica2 = RunningReplicaInfo(
+        replica_id=replica_id,
+        node_id="node_id",
+        node_ip="node_ip",
+        availability_zone="some-az",
+        actor_name=actor_name,
+        max_ongoing_requests=1,
+        is_cross_language=False,
+    )
+    # Test that cross-language setting affects hash
+    replica3 = RunningReplicaInfo(
+        replica_id=replica_id,
+        node_id="node_id",
+        node_ip="node_ip",
+        availability_zone="some-az",
+        actor_name=actor_name,
+        max_ongoing_requests=1,
+        is_cross_language=True,
+    )
+    assert replica1._hash == replica2._hash
+    assert replica3._hash != replica1._hash
+
+    # Test that backend_http_port affects hash so long-poll updates
+    # propagate when the backend HTTP port changes.
+    replica4 = RunningReplicaInfo(
+        replica_id=replica_id,
+        node_id="node_id",
+        node_ip="node_ip",
+        availability_zone="some-az",
+        actor_name=actor_name,
+        max_ongoing_requests=1,
+        is_cross_language=False,
+        backend_http_port=8001,
+    )
+    replica5 = RunningReplicaInfo(
+        replica_id=replica_id,
+        node_id="node_id",
+        node_ip="node_ip",
+        availability_zone="some-az",
+        actor_name=actor_name,
+        max_ongoing_requests=1,
+        is_cross_language=False,
+        backend_http_port=8002,
+    )
+    assert replica4._hash != replica1._hash
+    assert replica4._hash != replica5._hash
+
+    # Test that network endpoint changes affect hash so wrappers and
+    # long-poll consumers refresh when the replica moves or its gRPC
+    # port changes.
+    replica6 = RunningReplicaInfo(
+        replica_id=replica_id,
+        node_id="node_id",
+        node_ip="node_ip_a",
+        availability_zone="some-az",
+        actor_name=actor_name,
+        max_ongoing_requests=1,
+        is_cross_language=False,
+        port=9000,
+    )
+    replica7 = RunningReplicaInfo(
+        replica_id=replica_id,
+        node_id="node_id",
+        node_ip="node_ip_b",
+        availability_zone="some-az",
+        actor_name=actor_name,
+        max_ongoing_requests=1,
+        is_cross_language=False,
+        port=9000,
+    )
+    replica8 = RunningReplicaInfo(
+        replica_id=replica_id,
+        node_id="node_id",
+        node_ip="node_ip_a",
+        availability_zone="some-az",
+        actor_name=actor_name,
+        max_ongoing_requests=1,
+        is_cross_language=False,
+        port=9001,
+    )
+    assert replica6._hash != replica7._hash
+    assert replica6._hash != replica8._hash
+
+
+if __name__ == "__main__":
+    import sys
+
+    sys.exit(pytest.main(["-v", "-s", __file__]))

python/ray/serve/tests/unit/test_consistent_hash_router.py

@@ -0,0 +1,618 @@
+import asyncio
+import sys
+from collections import Counter
+from typing import Optional
+
+import pytest
+
+from ray._common.utils import get_or_create_event_loop
+from ray.exceptions import ActorDiedError, ActorUnavailableError
+from ray.serve._private.common import (
+    DeploymentHandleSource,
+    RequestMetadata,
+)
+from ray.serve._private.request_router import PendingRequest
+from ray.serve._private.test_utils import (
+    FAKE_REPLICA_DEFAULT_MAX_ONGOING_REQUESTS as DEFAULT_MAX_ONGOING_REQUESTS,
+    FAKE_REPLICA_DEPLOYMENT_ID as DEPLOYMENT_ID,
+    FakeRunningReplica,
+    MockTimer,
+)
+from ray.serve._private.utils import generate_request_id
+from ray.serve.experimental.consistent_hash_router import (
+    DEFAULT_NUM_FALLBACK_REPLICAS,
+    DEFAULT_NUM_VIRTUAL_NODES,
+    ConsistentHashRouter,
+    _hash_bytes,
+)
+
+TIMER = MockTimer()
+
+
+def _make_request(
+    session_id: str = "", request_id: Optional[str] = None
+) -> PendingRequest:
+    internal_id = request_id if request_id is not None else generate_request_id()
+    return PendingRequest(
+        args=[],
+        kwargs={},
+        metadata=RequestMetadata(
+            request_id=generate_request_id(),
+            internal_request_id=internal_id,
+            session_id=session_id,
+        ),
+    )
+
+
+@pytest.fixture
+def router(request) -> ConsistentHashRouter:
+    """Construct a ConsistentHashRouter on its own loop so its
+    asyncio.Event attaches to the correct loop, then return it to
+    the active test loop."""
+    params = getattr(request, "param", {})
+
+    async def build(loop: asyncio.AbstractEventLoop) -> ConsistentHashRouter:
+        r = ConsistentHashRouter(
+            deployment_id=DEPLOYMENT_ID,
+            handle_source=DeploymentHandleSource.REPLICA,
+            self_actor_id="fake-actor-id",
+            self_actor_handle=None,
+            use_replica_queue_len_cache=False,
+            get_curr_time_s=TIMER.time,
+        )
+        r.initialize_state(**params)
+        r.initial_backoff_s = 0.001
+        r.backoff_multiplier = 1
+        r.max_backoff_s = 0.001
+        return r
+
+    s = asyncio.new_event_loop().run_until_complete(build(get_or_create_event_loop()))
+    TIMER.reset()
+    yield s
+    # Confirm routing loop exits cleanly when there's no residual work.
+    assert s.curr_num_routing_tasks == 0
+    assert s.num_pending_requests == 0
+
+
+def test_hash_bytes():
+    assert _hash_bytes(b"hello") == _hash_bytes(b"hello")
+    assert _hash_bytes(b"sess_0000") != _hash_bytes(b"sess_0001")
+
+    h = _hash_bytes(b"arbitrary")
+    assert isinstance(h, int)
+    assert 0 <= h < (1 << 64)
+
+
+@pytest.mark.parametrize(
+    "kwargs",
+    [
+        {"num_virtual_nodes": 0},
+        {"num_virtual_nodes": -5},
+        {"num_virtual_nodes": "many"},
+        {"num_fallback_replicas": -1},
+    ],
+)
+def test_initialize_state_invalid_args(kwargs):
+    r = ConsistentHashRouter(
+        deployment_id=DEPLOYMENT_ID,
+        handle_source=DeploymentHandleSource.REPLICA,
+        self_actor_id="a",
+        self_actor_handle=None,
+    )
+    with pytest.raises(ValueError):
+        r.initialize_state(**kwargs)
+
+
+def test_initialize_state_valid_args():
+    r = ConsistentHashRouter(
+        deployment_id=DEPLOYMENT_ID,
+        handle_source=DeploymentHandleSource.REPLICA,
+        self_actor_id="a",
+        self_actor_handle=None,
+    )
+    r.initialize_state(num_virtual_nodes=42, num_fallback_replicas=7)
+    assert r._num_virtual_nodes == 42
+    assert r._num_fallback_replicas == 7
+
+
+def test_ring_defaults(router):
+    assert router._num_virtual_nodes == DEFAULT_NUM_VIRTUAL_NODES
+    assert router._num_fallback_replicas == DEFAULT_NUM_FALLBACK_REPLICAS
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("router", [{"num_fallback_replicas": 5}], indirect=True)
+async def test_explicit_num_fallback_replicas_caps_gracefully(router):
+    """When the operator explicitly requests more fallbacks than there
+    are replicas, the lookup must stop early gracefully."""
+    # 3 replicas with num_fallback_replicas=5 means we'd want a rank list
+    # of 6 but only 3 distinct replicas exist; the loop must cap at 3.
+    replicas = [FakeRunningReplica(f"r{i}") for i in range(3)]
+    router.update_replicas(replicas)
+
+    ranked = router._lookup_ranked_replicas("any_session")
+    assert len(ranked) == 3
+    assert len(set(ranked)) == 3
+
+
+@pytest.mark.asyncio
+async def test_default_num_fallback_replicas_caps_gracefully(
+    router,
+):
+    """Defaulted num_fallback_replicas must also stop early gracefully when
+    there are fewer replicas than 1 + DEFAULT_NUM_FALLBACK_REPLICAS."""
+    assert router._num_fallback_replicas == DEFAULT_NUM_FALLBACK_REPLICAS
+
+    replicas = [
+        FakeRunningReplica(f"r{i}") for i in range(DEFAULT_NUM_FALLBACK_REPLICAS)
+    ]
+    router.update_replicas(replicas)
+
+    ranked = router._lookup_ranked_replicas("any_session")
+    assert len(ranked) == len(replicas)
+    assert len(set(ranked)) == len(replicas)
+
+
+@pytest.mark.asyncio
+async def test_choose_replicas_no_replicas_returns_empty(router):
+    result = await router.choose_replicas(
+        candidate_replicas=[], pending_request=_make_request("sess_x")
+    )
+    assert result == []
+
+
+@pytest.mark.asyncio
+async def test_choose_replicas_stickiness(router):
+    """Repeated calls with the same session_id must map to the same primary replica."""
+    replicas = [FakeRunningReplica(f"r{i}") for i in range(4)]
+    router.update_replicas(replicas)
+
+    sess = "user_123"
+    first = await router.choose_replicas(
+        candidate_replicas=replicas, pending_request=_make_request(sess)
+    )
+    for _ in range(10):
+        again = await router.choose_replicas(
+            candidate_replicas=replicas, pending_request=_make_request(sess)
+        )
+        assert again[0][0].replica_id == first[0][0].replica_id
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("router", [{"num_fallback_replicas": 0}], indirect=True)
+async def test_choose_replicas_no_fallbacks(router):
+    """K=0 -> a single rank containing only the primary."""
+    replicas = [FakeRunningReplica(f"r{i}") for i in range(5)]
+    router.update_replicas(replicas)
+
+    ranks = await router.choose_replicas(
+        candidate_replicas=replicas, pending_request=_make_request("sess_x")
+    )
+    assert len(ranks) == 1
+    assert len(ranks[0]) == 1
+
+
+@pytest.mark.asyncio
+async def test_choose_replicas_fallback_caps_at_replica_count(router):
+    replicas = [FakeRunningReplica("ra"), FakeRunningReplica("rb")]
+    router.update_replicas(replicas)
+
+    ranks = await router.choose_replicas(
+        candidate_replicas=replicas, pending_request=_make_request("sess_x")
+    )
+    assert len(ranks) == 2
+    flat = [rep.replica_id for rank in ranks for rep in rank]
+    assert len(flat) == len(set(flat)) == 2
+
+
+@pytest.mark.asyncio
+async def test_choose_replicas_missing_session_spreads_across_replicas(router):
+    replicas = [FakeRunningReplica(f"r{i}") for i in range(3)]
+    router.update_replicas(replicas)
+
+    primaries = set()
+    for _ in range(20):
+        ranks = await router.choose_replicas(
+            candidate_replicas=replicas,
+            pending_request=_make_request(session_id=""),
+        )
+        primaries.add(ranks[0][0].replica_id)
+
+    assert len(primaries) > 1
+
+
+@pytest.mark.asyncio
+async def test_lookup_wraps_around_past_max_vnode(router, monkeypatch):
+    replicas = [FakeRunningReplica(f"r{i}") for i in range(3)]
+    router.update_replicas(replicas)
+
+    max_vnode = max(router._ring_hashes)
+    monkeypatch.setattr(
+        "ray.serve.experimental.consistent_hash_router._hash_bytes",
+        lambda key: max_vnode + 1,
+    )
+
+    ranked = router._lookup_ranked_replicas("anything")
+    # Wrapping around sends us to index 0 so the primary is whichever replica
+    # owns the smallest vnode.
+    assert ranked[0] == router._ring_replicas[0]
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("key_hash", [0, (1 << 64) - 1])
+async def test_lookup_at_hash_space_boundaries(router, monkeypatch, key_hash):
+    """Key hashing to exactly 0 or 2**64 - 1 must still produce a valid owner."""
+    replicas = [FakeRunningReplica(f"r{i}") for i in range(3)]
+    router.update_replicas(replicas)
+
+    monkeypatch.setattr(
+        "ray.serve.experimental.consistent_hash_router._hash_bytes",
+        lambda key: key_hash,
+    )
+    ranked = router._lookup_ranked_replicas("anything")
+
+    assert len(ranked) == 1 + DEFAULT_NUM_FALLBACK_REPLICAS
+    assert ranked[0] in {r.replica_id for r in replicas}
+
+
+@pytest.mark.asyncio
+async def test_balanced_distribution(router):
+    """100 vnodes per replica should distribute random sessions close to uniform."""
+    replicas = [FakeRunningReplica(f"r{i}") for i in range(5)]
+    router.update_replicas(replicas)
+
+    counts: Counter = Counter()
+    for i in range(2000):
+        ranks = await router.choose_replicas(
+            candidate_replicas=replicas,
+            pending_request=_make_request(f"sess_{i}"),
+        )
+        counts[ranks[0][0].replica_id.unique_id] += 1
+
+    fair = 2000 / 5
+    for c in counts.values():
+        assert 0.5 * fair < c < 2 * fair
+
+
+@pytest.mark.asyncio
+async def test_scale_up_remaps_small_fraction(router):
+    """Adding the Nth replica should remap ~1/N of sessions."""
+    initial = [FakeRunningReplica(f"r{i}") for i in range(4)]
+    router.update_replicas(initial)
+
+    sessions = [f"sess_{i}" for i in range(1000)]
+    mapping_before = {}
+    for s in sessions:
+        ranks = await router.choose_replicas(
+            candidate_replicas=initial, pending_request=_make_request(s)
+        )
+        mapping_before[s] = ranks[0][0].replica_id
+
+    # Scale from 4 to 5.
+    scaled = initial + [FakeRunningReplica("r_new")]
+    router.update_replicas(scaled)
+
+    moved = 0
+    for s in sessions:
+        ranks = await router.choose_replicas(
+            candidate_replicas=scaled, pending_request=_make_request(s)
+        )
+        if ranks[0][0].replica_id != mapping_before[s]:
+            moved += 1
+
+    # Expected: 1/5 = 20%. Upper bound generous to avoid flake on
+    # random-cluster draws of the hash.
+    assert moved < 0.4 * len(
+        sessions
+    ), f"Scale-up remapped {moved}/{len(sessions)} sessions; expected ~200"
+
+    assert moved > 0
+
+
+@pytest.mark.asyncio
+async def test_scale_down_remaps_only_owned_sessions(router):
+    """Removing one replica out of R should remap roughly 1/R sessions.
+    Sessions that were not on the removed replica must stay put."""
+    initial = [FakeRunningReplica(f"r{i}") for i in range(4)]
+    router.update_replicas(initial)
+
+    sessions = [f"sess_{i}" for i in range(1000)]
+    mapping_before = {}
+    for s in sessions:
+        ranks = await router.choose_replicas(
+            candidate_replicas=initial, pending_request=_make_request(s)
+        )
+        mapping_before[s] = ranks[0][0].replica_id
+
+    # Drop the first replica from the set.
+    dropped_id = initial[0].replica_id
+    remaining = initial[1:]
+    router.update_replicas(remaining)
+
+    moved_owned = 0
+    moved_other = 0
+    for s in sessions:
+        ranks = await router.choose_replicas(
+            candidate_replicas=remaining, pending_request=_make_request(s)
+        )
+        new = ranks[0][0].replica_id
+        prev = mapping_before[s]
+        if prev == dropped_id:
+            moved_owned += 1
+        elif new != prev:
+            moved_other += 1
+
+    # Every session that was on the dropped replica must move.
+    expected_owned = sum(1 for v in mapping_before.values() if v == dropped_id)
+    assert moved_owned == expected_owned
+    # Sessions not owned by the dropped replica stay put.
+    assert moved_other == 0
+
+
+@pytest.mark.asyncio
+async def test_fallback_when_primary_at_max_ongoing(router):
+    """Primary saturated -> the outer retry loop must walk to fallback_1."""
+    loop = get_or_create_event_loop()
+    replicas = [FakeRunningReplica(f"r{i}") for i in range(4)]
+    router.update_replicas(replicas)
+
+    ranks = await router.choose_replicas(
+        candidate_replicas=replicas, pending_request=_make_request("user_42")
+    )
+    primary_id = ranks[0][0].replica_id
+    fallback_id = ranks[1][0].replica_id
+
+    # Saturate the primary, leave fallbacks free.
+    for r in replicas:
+        if r.replica_id == primary_id:
+            r.set_queue_len_response(DEFAULT_MAX_ONGOING_REQUESTS)
+        else:
+            r.set_queue_len_response(0)
+
+    chosen = await loop.create_task(
+        router._choose_replica_for_request(_make_request("user_42"))
+    )
+    assert chosen.replica_id == fallback_id
+
+
+@pytest.mark.asyncio
+async def test_backoff_when_all_exhausted_then_retries_on_drain(router):
+    """When every ranked replica is at max_ongoing_requests, the routing
+    task backs off and loops. When capacity opens up, it picks the newly
+    freed replica on the next iteration -- without us having to call
+    update_replicas."""
+    loop = get_or_create_event_loop()
+    # K+1 = 3 replicas so every one is in the ranked set.
+    replicas = [FakeRunningReplica(f"r{i}") for i in range(3)]
+    for r in replicas:
+        r.set_queue_len_response(DEFAULT_MAX_ONGOING_REQUESTS)
+    router.update_replicas(replicas)
+
+    task = loop.create_task(
+        router._choose_replica_for_request(_make_request("user_42"))
+    )
+    # Nothing should be ready: the task is stuck in the backoff loop
+    # because every replica reports itself as full.
+    done, _ = await asyncio.wait([task], timeout=0.02)
+    assert len(done) == 0
+
+    # Drain the primary. The retry loop's next probe sees the open slot
+    # and the task completes.
+    ranks = await router.choose_replicas(
+        candidate_replicas=replicas, pending_request=_make_request("user_42")
+    )
+    primary_id = ranks[0][0].replica_id
+    for r in replicas:
+        if r.replica_id == primary_id:
+            r.set_queue_len_response(0)
+
+    chosen = await task
+    assert chosen.replica_id == primary_id
+
+
+@pytest.mark.asyncio
+async def test_dead_primary_routes_to_fallback(router):
+    """``ActorDiedError`` drops the primary from ``_replicas``; the
+    ring's stale reference is filtered out and the request lands on
+    the fallback instead of hanging."""
+    replicas = [FakeRunningReplica(f"r{i}") for i in range(3)]
+    router.update_replicas(replicas)
+
+    # Find the primary + fallback for "user_42" through the ring.
+    ranks = await router.choose_replicas(
+        candidate_replicas=replicas, pending_request=_make_request("user_42")
+    )
+    primary_id = ranks[0][0].replica_id
+    fallback_id = ranks[1][0].replica_id
+
+    # Primary's probe raises ActorDiedError; fallbacks respond normally.
+    for r in replicas:
+        if r.replica_id == primary_id:
+            r.set_queue_len_response(queue_len=0, exception=ActorDiedError())
+        else:
+            r.set_queue_len_response(0)
+
+    chosen = await router._choose_replica_for_request(_make_request("user_42"))
+    assert chosen.replica_id == fallback_id
+
+    # The dead replica should be gone from _replicas, so subsequent
+    # requests on this session never probe it again.
+    assert primary_id not in router._replicas
+
+
+@pytest.mark.asyncio
+async def test_unavailable_primary_falls_over_then_recovers(router):
+    """``ActorUnavailableError`` is transient: request falls over to
+    the fallback, the replica stays in ``_replicas``, and sticky
+    traffic returns to the primary once it recovers."""
+    replicas = [FakeRunningReplica(f"r{i}") for i in range(3)]
+    router.update_replicas(replicas)
+
+    ranks = await router.choose_replicas(
+        candidate_replicas=replicas, pending_request=_make_request("user_42")
+    )
+    primary_id = ranks[0][0].replica_id
+    fallback_id = ranks[1][0].replica_id
+
+    primary = next(r for r in replicas if r.replica_id == primary_id)
+
+    # Primary raises ActorUnavailableError; fallbacks accept.
+    primary.set_queue_len_response(
+        queue_len=0,
+        exception=ActorUnavailableError(
+            error_message="temporarily unavailable",
+            actor_id=b"a" * 16,
+        ),
+    )
+    for r in replicas:
+        if r.replica_id != primary_id:
+            r.set_queue_len_response(0)
+
+    chosen = await router._choose_replica_for_request(_make_request("user_42"))
+    assert chosen.replica_id == fallback_id
+    # Unlike the died case, the replica must still be in _replicas --
+    # ActorUnavailableError is transient.
+    assert primary_id in router._replicas
+
+    # Recover the primary. Next sticky request must return to it.
+    primary.set_queue_len_response(queue_len=0, exception=None)
+    chosen2 = await router._choose_replica_for_request(_make_request("user_42"))
+    assert chosen2.replica_id == primary_id
+
+
+@pytest.mark.asyncio
+async def test_backoff_engaged_when_all_ranks_saturated(router):
+    router.initial_backoff_s = 999
+    router.backoff_multiplier = 1
+    router.max_backoff_s = 999
+
+    loop = get_or_create_event_loop()
+
+    # 1 + DEFAULT_NUM_FALLBACK_REPLICAS replicas -> the full ranked set is
+    # saturated, so _choose_replicas_with_backoff exhausts every yielded rank
+    # and falls through to the backoff branch.
+    replicas = [
+        FakeRunningReplica(f"r{i}") for i in range(1 + DEFAULT_NUM_FALLBACK_REPLICAS)
+    ]
+    for r in replicas:
+        r.set_queue_len_response(DEFAULT_MAX_ONGOING_REQUESTS)
+    router.update_replicas(replicas)
+
+    task = loop.create_task(
+        router._choose_replica_for_request(_make_request("user_42"))
+    )
+    done, _ = await asyncio.wait([task], timeout=0.1)
+    assert len(done) == 0
+
+    # The routing task must have entered the backoff branch.
+    assert router.num_routing_tasks_in_backoff >= 1
+
+    # Free up the primary. With a 999s backoff sleep already in flight, the
+    # task must remain pending.
+    ranks = await router.choose_replicas(
+        candidate_replicas=replicas, pending_request=_make_request("user_42")
+    )
+    primary_id = ranks[0][0].replica_id
+    for r in replicas:
+        if r.replica_id == primary_id:
+            r.set_queue_len_response(0)
+
+    done, _ = await asyncio.wait([task], timeout=0.1)
+    assert len(done) == 0, (
+        "routing task completed within the sleep window -- "
+        "exponential backoff is not being applied"
+    )
+
+    task.cancel()
+    try:
+        await task
+    except BaseException:
+        pass
+    router._routing_tasks.clear()
+    router._pending_requests_to_fulfill.clear()
+    router._pending_requests_to_route.clear()
+
+
+@pytest.mark.asyncio
+async def test_concurrent_same_session_does_not_orphan(router):
+    """Concurrent same-session requests at the rank-list size must all
+    resolve, not livelock.
+
+    Regression: the routing task that pops a pending from the route deque
+    is the one responsible for fulfilling it. If the inner retry loop
+    breaks early under load (e.g., on the routing-task-shedding check after
+    a probe miss) without fulfilling, that pending used to become an
+    orphan: still in `_pending_requests_to_fulfill` / `_pending_requests_by_id`
+    but with no task actively routing it. Surviving routing tasks would
+    then loop with `request_metadata=None`, and because ConsistentHashRouter
+    uses strict-metadata-match fulfillment (no FIFO fallback like pow2),
+    they couldn't pick up the orphan, livelocking the router.
+    """
+    loop = get_or_create_event_loop()
+
+    # 1 + DEFAULT_NUM_FALLBACK_REPLICAS replicas so the full rank list is
+    # exactly the same set across requests (no replica outside the ranks).
+    replicas = [
+        FakeRunningReplica(f"r{i}") for i in range(1 + DEFAULT_NUM_FALLBACK_REPLICAS)
+    ]
+    for r in replicas:
+        r.set_queue_len_response(0)
+    router.update_replicas(replicas)
+
+    # Fire (1 + num_fallback_replicas + 2) concurrent same-session requests.
+    # That's strictly more than the rank-list size, exercising the path
+    # where the routing task that owns a popped pending might temporarily
+    # exceed the dynamically computed task target.
+    burst_size = 1 + DEFAULT_NUM_FALLBACK_REPLICAS + 2
+    tasks = [
+        loop.create_task(
+            router._choose_replica_for_request(_make_request("burst-session"))
+        )
+        for _ in range(burst_size)
+    ]
+    done, pending = await asyncio.wait(tasks, timeout=2.0)
+    assert len(pending) == 0, (
+        f"{len(pending)}/{burst_size} concurrent same-session requests "
+        "did not resolve -- ConsistentHashRouter livelocked under load."
+    )
+
+    chosen_replicas = {t.result().replica_id for t in done}
+    # Affinity preserved: every burst request lands on the same primary.
+    assert len(chosen_replicas) == 1
+
+
+@pytest.mark.asyncio
+async def test_two_routers_route_same_session_to_same_replica(router):
+    """Two independent routers with the same replica set must pick the same
+    replica for the same session_id."""
+    replicas_a = [FakeRunningReplica(f"r{i}") for i in range(5)]
+    replicas_b = [FakeRunningReplica(f"r{i}") for i in range(5)]
+    router.update_replicas(replicas_a)
+
+    other = ConsistentHashRouter(
+        deployment_id=DEPLOYMENT_ID,
+        handle_source=DeploymentHandleSource.REPLICA,
+        self_actor_id="other",
+        self_actor_handle=None,
+    )
+    other.initialize_state()
+    other.update_replicas(replicas_b)
+
+    for i in range(20):
+        sess = f"user_{i}"
+        ranks_a = await router.choose_replicas(
+            candidate_replicas=replicas_a,
+            pending_request=_make_request(sess),
+        )
+        ranks_b = await other.choose_replicas(
+            candidate_replicas=replicas_b,
+            pending_request=_make_request(sess),
+        )
+        assert ranks_a[0][0].replica_id == ranks_b[0][0].replica_id, (
+            f"Session {sess} diverged: {ranks_a[0][0].replica_id} vs "
+            f"{ranks_b[0][0].replica_id}"
+        )
+
+
+if __name__ == "__main__":
+    sys.exit(pytest.main(["-v", "-s", __file__]))

python/ray/serve/tests/unit/test_constants_utils.py

@@ -0,0 +1,240 @@
+import os
+from unittest.mock import patch
+
+import pytest
+
+from ray.serve._private.constants_utils import (
+    _validate_name,
+    get_env_bool,
+    get_env_float,
+    get_env_float_non_negative,
+    get_env_float_positive,
+    get_env_int,
+    get_env_int_non_negative,
+    get_env_int_positive,
+    get_env_str,
+    parse_latency_buckets,
+    str_to_list,
+)
+
+
+class TestStrToList:
+    def test_str_to_list_basic(self):
+        assert str_to_list("a,b,c") == ["a", "b", "c"]
+
+    def test_str_to_list_with_whitespace(self):
+        assert str_to_list(" a , b , c ") == ["a", "b", "c"]
+
+    def test_str_to_list_empty_string(self):
+        assert str_to_list("") == []
+
+    def test_str_to_list_with_empty_entries(self):
+        assert str_to_list("a,,b,c,") == ["a", "b", "c"]
+
+    def test_str_to_list_only_whitespace(self):
+        assert str_to_list("   ") == []
+
+    def test_str_to_list_single_entry(self):
+        assert str_to_list("single") == ["single"]
+
+    def test_str_to_list_only_commas(self):
+        assert str_to_list(",,,,") == []
+
+    def test_str_to_list_whitespace_entries(self):
+        assert str_to_list("a, ,b") == ["a", "b"]
+
+
+class TestParseLatencyBuckets:
+    def test_parse_latency_buckets(self):
+        # Test valid inputs with different formats
+        assert parse_latency_buckets("1,2,3", []) == [1.0, 2.0, 3.0]
+        assert parse_latency_buckets("1,2,3,4 ", []) == [1.0, 2.0, 3.0, 4.0]
+        assert parse_latency_buckets("  1,2,3,4,5", []) == [1.0, 2.0, 3.0, 4.0, 5.0]
+        assert parse_latency_buckets(" 1, 2,3  ,4,5 ,6 ", []) == [
+            1.0,
+            2.0,
+            3.0,
+            4.0,
+            5.0,
+            6.0,
+        ]
+
+        # Test decimal numbers
+        assert parse_latency_buckets("0.5,1.5,2.5", []) == [0.5, 1.5, 2.5]
+
+    def test_parse_latency_buckets_invalid(self):
+        # Test negative numbers
+        with pytest.raises(ValueError, match=".*must be positive.*"):
+            parse_latency_buckets("-1,1,2,3,4", [])
+
+        # Test non-ascending order
+        with pytest.raises(ValueError, match=".*be in strictly ascending order*"):
+            parse_latency_buckets("4,3,2,1", [])
+
+        # Test duplicate values
+        with pytest.raises(ValueError, match=".*be in strictly ascending order.*"):
+            parse_latency_buckets("1,2,2,3,4", [])
+
+        # Test invalid number format
+        with pytest.raises(ValueError, match=".*Invalid.*format.*"):
+            parse_latency_buckets("1,2,3,4,a", [])
+
+        # Test empty list
+        with pytest.raises(ValueError, match=".*could not convert.*"):
+            parse_latency_buckets(",,,", [])
+
+        # Test invalid separators
+        with pytest.raises(ValueError, match=".*could not convert.*"):
+            parse_latency_buckets("1;2;3;4", [])
+
+
+@pytest.fixture
+def mock_environ():
+    with patch.dict(os.environ, {}, clear=True) as mock_env:
+        yield mock_env
+
+
+class TestEnvValueFunctions:
+    def test_get_env_int(self, mock_environ):
+        assert get_env_int("RAY_SERVE_TEST_VAR", 0) == 0
+
+        mock_environ["RAY_SERVE_TEST_VAR"] = "42"
+        assert get_env_int("RAY_SERVE_TEST_VAR", 0) == 42
+
+        mock_environ["RAY_SERVE_TEST_VAR"] = "-1"
+        assert get_env_int("RAY_SERVE_TEST_VAR", 0) == -1
+
+        mock_environ["RAY_SERVE_TEST_VAR"] = "0.1"
+        with pytest.raises(ValueError, match=".*`0.1` cannot be converted to `int`!*"):
+            get_env_int_positive("RAY_SERVE_TEST_VAR", 5)
+
+        mock_environ["RAY_SERVE_TEST_VAR"] = "abc"
+        with pytest.raises(ValueError, match=".*`abc` cannot be converted to `int`!*"):
+            get_env_int_positive("RAY_SERVE_TEST_VAR", 5)
+
+        with pytest.raises(ValueError, match=".*require prefix `RAY_SERVE_`*"):
+            get_env_int_positive("NO_PREFIX", 5)
+
+    def test_get_env_int_positive(self, mock_environ):
+        assert get_env_int_positive("RAY_SERVE_TEST_VAR", 1) == 1
+
+        mock_environ["RAY_SERVE_TEST_VAR"] = "42"
+        assert get_env_int_positive("RAY_SERVE_TEST_VAR", 1) == 42
+
+        mock_environ["RAY_SERVE_TEST_VAR"] = "-1"
+        with pytest.raises(ValueError, match=".*Expected positive `int`.*"):
+            get_env_int_positive("RAY_SERVE_TEST_VAR", 5)
+
+    def test_get_env_int_non_negative(self, mock_environ):
+        assert get_env_int_non_negative("RAY_SERVE_TEST_VAR", 0) == 0
+        assert get_env_int_non_negative("RAY_SERVE_TEST_VAR", 1) == 1
+
+        mock_environ["RAY_SERVE_TEST_VAR"] = "42"
+        assert get_env_int_non_negative("RAY_SERVE_TEST_VAR", 0) == 42
+
+        mock_environ["RAY_SERVE_TEST_VAR"] = "-1"
+        with pytest.raises(ValueError, match=".*Expected non negative `int`.*"):
+            get_env_int_non_negative("RAY_SERVE_TEST_VAR", 5)
+
+        with pytest.raises(ValueError, match=".*Expected non negative `int`.*"):
+            get_env_int_non_negative("RAY_SERVE_TEST_VAR_FROM_DEFAULT", -1)
+
+    def test_get_env_float(self, mock_environ):
+        assert get_env_float("RAY_SERVE_TEST_VAR", 0.0) == 0.0
+
+        mock_environ["RAY_SERVE_TEST_VAR"] = "3.14"
+        assert get_env_float("RAY_SERVE_TEST_VAR", 0.0) == 3.14
+
+        mock_environ["RAY_SERVE_TEST_VAR"] = "-2.5"
+        assert get_env_float("RAY_SERVE_TEST_VAR", 0.0) == -2.5
+
+        mock_environ["RAY_SERVE_TEST_VAR"] = "abc"
+        with pytest.raises(
+            ValueError, match=".*`abc` cannot be converted to `float`!*"
+        ):
+            get_env_float("RAY_SERVE_TEST_VAR", 0.0)
+
+    def test_get_env_float_positive(self, mock_environ):
+        assert get_env_float_positive("RAY_SERVE_TEST_VAR", 1.5) == 1.5
+        assert get_env_float_positive("RAY_SERVE_TEST_VAR", None) is None
+
+        mock_environ["RAY_SERVE_TEST_VAR"] = "42.5"
+        assert get_env_float_positive("RAY_SERVE_TEST_VAR", 1.0) == 42.5
+
+        mock_environ["RAY_SERVE_TEST_VAR"] = "-1.2"
+        with pytest.raises(ValueError, match=".*Expected positive `float`.*"):
+            get_env_float_positive("RAY_SERVE_TEST_VAR", 5.0)
+
+        with pytest.raises(ValueError, match=".*Expected positive `float`.*"):
+            get_env_float_positive("RAY_SERVE_TEST_VAR_FROM_DEFAULT", 0.0)
+
+        with pytest.raises(ValueError, match=".*Expected positive `float`.*"):
+            get_env_float_positive("RAY_SERVE_TEST_VAR_FROM_DEFAULT", -1)
+
+    def test_get_env_float_non_negative(self, mock_environ):
+        assert get_env_float_non_negative("RAY_SERVE_TEST_VAR", 0.0) == 0.0
+        assert get_env_float_non_negative("RAY_SERVE_TEST_VAR", 1.5) == 1.5
+
+        mock_environ["RAY_SERVE_TEST_VAR"] = "42.5"
+        assert get_env_float_non_negative("RAY_SERVE_TEST_VAR", 0.0) == 42.5
+
+        mock_environ["RAY_SERVE_TEST_VAR"] = "-1.2"
+        with pytest.raises(ValueError, match=".*Expected non negative `float`.*"):
+            get_env_float_non_negative("RAY_SERVE_TEST_VAR", 5.0)
+
+    def test_get_env_str(self, mock_environ):
+        mock_environ["RAY_SERVE_TEST_STR"] = "hello"
+        assert get_env_str("RAY_SERVE_TEST_STR", "default") == "hello"
+
+        assert get_env_str("RAY_SERVE_NONEXISTENT_VAR", "default_str") == "default_str"
+
+        assert get_env_str("RAY_SERVE_NONEXISTENT_VAR", None) is None
+
+    def test_get_env_bool(self, mock_environ):
+        mock_environ["RAY_SERVE_TEST_BOOL_TRUE"] = "1"
+        assert get_env_bool("RAY_SERVE_TEST_BOOL_TRUE", "0") is True
+
+        # Test with any other value (False)
+        mock_environ["RAY_SERVE_TEST_BOOL_FALSE"] = "true"
+        assert get_env_bool("RAY_SERVE_TEST_BOOL_FALSE", "0") is False
+        mock_environ["RAY_SERVE_TEST_BOOL_FALSE2"] = "yes"
+        assert get_env_bool("RAY_SERVE_TEST_BOOL_FALSE2", "0") is False
+
+        # Test with default when environment variable not set
+        assert get_env_bool("RAY_SERVE_NONEXISTENT_VAR", "1") is True
+        assert get_env_bool("RAY_SERVE_NONEXISTENT_VAR", "0") is False
+
+
+class TestValidation:
+    @pytest.mark.parametrize(
+        "name",
+        [
+            "RAY_SERVE_FOO",
+            "RAY_SERVE__DOUBLE_UNDERSCORE",
+            "RAY_SERVE_123",
+            "RAY_SERVE_VAR_NAME",
+        ],
+    )
+    def test_validate_name_accepts_valid_prefix(self, name):
+        # Should not raise
+        assert _validate_name(name) is None
+
+    @pytest.mark.parametrize(
+        "name",
+        [
+            "",
+            "RAY_SERVE",  # missing trailing underscore and name
+            "SERVE_VAR",
+            "ray_SERVE_BAR",
+            "RAY_service_VAR",
+        ],
+    )
+    def test_validate_name_rejects_invalid_prefix(self, name):
+        with pytest.raises(ValueError, match=".*require prefix `RAY_SERVE_`*"):
+            _validate_name(name)
+
+
+if __name__ == "__main__":
+    import sys
+
+    sys.exit(pytest.main(["-v", "-s", __file__]))

python/ray/serve/tests/unit/test_controller.py

@@ -0,0 +1,720 @@
+from copy import deepcopy
+
+import pytest
+
+from ray.serve._private.common import TargetCapacityDirection
+from ray.serve._private.controller import (
+    applications_match,
+    calculate_target_capacity_direction,
+)
+from ray.serve._private.controller_health_metrics_tracker import (
+    _HEALTH_METRICS_HISTORY_SIZE,
+    ControllerHealthMetricsTracker,
+)
+from ray.serve.schema import (
+    ControllerHealthMetrics,
+    DurationStats,
+    HTTPOptionsSchema,
+    ServeApplicationSchema,
+    ServeDeploySchema,
+)
+
+
+def create_app_config(name: str) -> ServeApplicationSchema:
+    return ServeApplicationSchema(
+        name=name, import_path=f"fake.{name}", route_prefix=f"/{name}"
+    )
+
+
+class TestApplicationsMatch:
+    def test_config_with_self(self):
+        config = ServeDeploySchema(
+            applications=[
+                ServeApplicationSchema(
+                    name="app1",
+                    import_path="fake.import",
+                    route_prefix="/",
+                ),
+            ],
+        )
+
+        assert applications_match(config, config) is True
+
+    def test_configs_with_matching_app_names(self):
+        config1 = ServeDeploySchema(
+            applications=[
+                ServeApplicationSchema(
+                    name="app1",
+                    import_path="fake.import",
+                    route_prefix="/app1",
+                ),
+                ServeApplicationSchema(
+                    name="app2",
+                    import_path="fake.import2",
+                    route_prefix="/app2",
+                ),
+                ServeApplicationSchema(
+                    name="app3",
+                    import_path="fake.import3",
+                    route_prefix="/app3",
+                ),
+            ],
+        )
+
+        # Configs contain apps with same name but different import paths.
+        config2 = deepcopy(config1)
+        config2.applications[0].import_path = "different_fake.import"
+        assert applications_match(config1, config2) is True
+
+        config2.applications[0].import_path = "extended.fake.import"
+        assert applications_match(config1, config2) is True
+
+        config2.applications[0].import_path = "fake:import"
+        assert applications_match(config1, config2) is True
+
+        # Configs contain apps with same name but different route_prefixes.
+        config2 = deepcopy(config1)
+        config2.applications[0].route_prefix += "_suffix"
+        assert applications_match(config1, config2) is True
+
+        # Configs contain apps with same name but different runtime_envs.
+        config2 = deepcopy(config1)
+        config2.applications[1].runtime_env = {"working_dir": "https://fake/uri"}
+        assert applications_match(config1, config2) is True
+
+        # Configs contain apps with same name but different target_capacities.
+        config2 = deepcopy(config1)
+        config2.target_capacity = 50
+        assert applications_match(config1, config2) is True
+
+        # Configs contain apps with same name but different http options.
+        config2 = deepcopy(config1)
+        config2.http_options = HTTPOptionsSchema(host="62.79.45.100")
+        assert applications_match(config1, config2) is True
+
+    def test_configs_with_different_app_names(self):
+        config1 = ServeDeploySchema(
+            applications=[
+                ServeApplicationSchema(
+                    name="app1",
+                    import_path="fake.import",
+                    route_prefix="/app1",
+                ),
+                ServeApplicationSchema(
+                    name="app2",
+                    import_path="fake.import2",
+                    route_prefix="/app2",
+                ),
+                ServeApplicationSchema(
+                    name="app3",
+                    import_path="fake.import3",
+                    route_prefix="/app3",
+                ),
+            ],
+        )
+
+        # Configs contain apps with different names but same import paths.
+        config2 = deepcopy(config1)
+        config2.applications[0].name = "different_app1"
+        assert applications_match(config1, config2) is False
+
+        # Configs contain different number of apps.
+        config2 = deepcopy(config1)
+        config2.applications.pop()
+        assert applications_match(config1, config2) is False
+
+
+class TestCalculateScaleDirection:
+    @pytest.mark.parametrize(
+        "curr_direction",
+        [TargetCapacityDirection.UP, TargetCapacityDirection.DOWN],
+    )
+    @pytest.mark.parametrize(
+        "new_direction",
+        [TargetCapacityDirection.UP, TargetCapacityDirection.DOWN],
+    )
+    def test_change_target_capacity_numeric(self, curr_direction, new_direction):
+        curr_target_capacity = 5
+
+        curr_config = ServeDeploySchema(
+            target_capacity=curr_target_capacity,
+            applications=[
+                create_app_config(name="app1"),
+                create_app_config(name="app2"),
+            ],
+        )
+
+        if new_direction == TargetCapacityDirection.UP:
+            new_target_capacity = curr_target_capacity * 3.3
+        elif new_direction == TargetCapacityDirection.DOWN:
+            new_target_capacity = curr_target_capacity / 3.3
+
+        new_config = deepcopy(curr_config)
+        new_config.target_capacity = new_target_capacity
+
+        # The new direction must be returned, regardless of the current direction.
+        assert (
+            calculate_target_capacity_direction(
+                curr_config,
+                new_config,
+                curr_direction,
+            )
+            == new_direction
+        )
+
+    @pytest.mark.parametrize("target_capacity", [0, 50, 100, None])
+    @pytest.mark.parametrize(
+        "curr_direction",
+        [TargetCapacityDirection.UP, TargetCapacityDirection.DOWN, None],
+    )
+    def test_no_change_target_capacity(self, target_capacity, curr_direction):
+        """When target_capacity doesn't change, return the current direction."""
+
+        if target_capacity is None:
+            # When target_capacity is None, the current direction must be None.
+            curr_direction = None
+
+        curr_config = ServeDeploySchema(
+            target_capacity=target_capacity,
+            applications=[
+                create_app_config(name="app1"),
+                create_app_config(name="app2"),
+                create_app_config(name="app3"),
+            ],
+        )
+
+        assert (
+            calculate_target_capacity_direction(
+                curr_config,
+                curr_config,
+                curr_direction,
+            )
+            == curr_direction
+        )
+
+    @pytest.mark.parametrize("curr_target_capacity", [0, 50, 100, None])
+    @pytest.mark.parametrize(
+        "curr_direction",
+        [TargetCapacityDirection.UP, TargetCapacityDirection.DOWN, None],
+    )
+    def test_enter_null_target_capacity(self, curr_target_capacity, curr_direction):
+        """When target capacity becomes null, scale up/down behavior must stop."""
+
+        if curr_target_capacity is None:
+            # When target_capacity is None, the current direction must be None.
+            curr_direction = None
+
+        curr_config = ServeDeploySchema(
+            target_capacity=curr_target_capacity,
+            applications=[
+                create_app_config(name="app1"),
+                create_app_config(name="app2"),
+                create_app_config(name="app3"),
+            ],
+        )
+
+        new_config = deepcopy(curr_config)
+        new_config.target_capacity = None
+
+        assert (
+            calculate_target_capacity_direction(
+                curr_config,
+                new_config,
+                curr_direction,
+            )
+            is None
+        )
+
+    @pytest.mark.parametrize("new_target_capacity", [0, 50, 100])
+    def test_exit_null_target_capacity(self, new_target_capacity):
+        """When target capacity goes null -> non-null, scale down must start."""
+
+        curr_config = ServeDeploySchema(
+            target_capacity=None,
+            applications=[create_app_config(name="app1")],
+        )
+
+        new_config = deepcopy(curr_config)
+        new_config.target_capacity = new_target_capacity
+
+        # When Serve is already running the applications at target_capacity
+        # None, and then a target_capacity is applied, the direction must
+        # become DOWN.
+        assert (
+            calculate_target_capacity_direction(
+                curr_config,
+                new_config,
+                None,
+            )
+            == TargetCapacityDirection.DOWN
+        )
+
+    def test_scale_up_first_config(self):
+        """Check how Serve handles the first config that's applied."""
+
+        # Case 1: target_capacity is set. Serve should transition to scaling up.
+
+        new_config = ServeDeploySchema(
+            target_capacity=20,
+            applications=[create_app_config(name="app1")],
+        )
+        assert (
+            calculate_target_capacity_direction(
+                None,
+                new_config,
+                None,
+            )
+            == TargetCapacityDirection.UP
+        )
+
+        # Case 2: target_capacity is not set. Serve should not be scaling.
+
+        new_config = ServeDeploySchema(
+            target_capacity=None,
+            applications=[create_app_config(name="app1")],
+        )
+        assert (
+            calculate_target_capacity_direction(
+                None,
+                new_config,
+                None,
+            )
+            is None
+        )
+
+    @pytest.mark.parametrize("curr_target_capacity", [0, 50, 100, None])
+    @pytest.mark.parametrize(
+        "curr_direction",
+        [TargetCapacityDirection.UP, TargetCapacityDirection.DOWN, None],
+    )
+    def test_config_live_apps_mismatch(self, curr_target_capacity, curr_direction):
+        """Apply a config with apps that don't match the live apps.
+
+        Serve should treat this like applying the first config. Its scaling
+        direction should not be based on the previous config's target_capacity.
+        """
+
+        if curr_target_capacity is None:
+            # When target_capacity is None, the current direction must be None.
+            curr_direction = None
+
+        curr_config = ServeDeploySchema(
+            target_capacity=curr_target_capacity,
+            applications=[
+                create_app_config(name="app1"),
+                create_app_config(name="app2"),
+                create_app_config(name="app3"),
+            ],
+        )
+
+        # Case 1: target_capacity is set. Serve should transition to scaling up.
+        new_config = ServeDeploySchema(
+            target_capacity=30,
+            applications=[
+                create_app_config(name="new_app1"),
+                create_app_config(name="app2"),
+            ],
+        )
+
+        assert (
+            calculate_target_capacity_direction(
+                curr_config,
+                new_config,
+                curr_direction,
+            )
+            is TargetCapacityDirection.UP
+        )
+
+        # Case 2: target_capacity is not set. Serve should not be scaling.
+        new_config = ServeDeploySchema(
+            target_capacity=None,
+            applications=[
+                create_app_config(name="new_app1"),
+                create_app_config(name="app2"),
+            ],
+        )
+
+        assert (
+            calculate_target_capacity_direction(
+                curr_config,
+                new_config,
+                curr_direction,
+            )
+            is None
+        )
+
+
+class TestDurationStats:
+    """Tests for DurationStats model."""
+
+    def test_empty_values(self):
+        """Test statistics from empty list."""
+        stats = DurationStats.from_values([])
+        assert stats.mean == 0.0
+        assert stats.std == 0.0
+        assert stats.min == 0.0
+        assert stats.max == 0.0
+
+    def test_single_value(self):
+        """Test statistics from single value."""
+        stats = DurationStats.from_values([5.0])
+        assert stats.mean == 5.0
+        assert stats.std == 0.0
+        assert stats.min == 5.0
+        assert stats.max == 5.0
+
+    def test_multiple_values(self):
+        """Test statistics from multiple values."""
+        values = [1.0, 2.0, 3.0, 4.0, 5.0]
+        stats = DurationStats.from_values(values)
+        assert stats.mean == 3.0
+        assert stats.min == 1.0
+        assert stats.max == 5.0
+        # std for [1,2,3,4,5] with population std = sqrt(2)
+        assert abs(stats.std - 1.4142135623730951) < 0.0001
+
+    def test_dict_serialization(self):
+        """Test that DurationStats serializes to dict."""
+        stats = DurationStats(mean=1.0, std=0.5, min=0.5, max=1.5)
+        result = stats.model_dump()
+        assert result == {"mean": 1.0, "std": 0.5, "min": 0.5, "max": 1.5}
+
+
+class TestControllerHealthMetrics:
+    """Tests for ControllerHealthMetrics dataclass."""
+
+    def test_default_values(self):
+        """Test that all default values are initialized correctly."""
+        metrics = ControllerHealthMetrics()
+        assert metrics.timestamp == 0.0
+        assert metrics.controller_start_time == 0.0
+        assert metrics.uptime_s == 0.0
+        assert metrics.num_control_loops == 0
+        assert metrics.last_control_loop_time == 0.0
+        assert metrics.loop_duration_s is None
+        assert metrics.event_loop_delay_s == 0.0
+        assert metrics.num_asyncio_tasks == 0
+        assert metrics.process_memory_mb == 0.0
+
+    def test_model_dump(self):
+        """Test serialization to dictionary."""
+        loop_stats = DurationStats(mean=0.3, std=0.1, min=0.1, max=0.5)
+        metrics = ControllerHealthMetrics(
+            timestamp=1000.0,
+            controller_start_time=900.0,
+            uptime_s=100.0,
+            num_control_loops=50,
+            loop_duration_s=loop_stats,
+        )
+        result = metrics.model_dump()
+
+        assert isinstance(result, dict)
+        assert result["timestamp"] == 1000.0
+        assert result["controller_start_time"] == 900.0
+        assert result["uptime_s"] == 100.0
+        assert result["num_control_loops"] == 50
+        assert result["loop_duration_s"]["mean"] == 0.3
+
+    def test_all_fields_in_model_dump(self):
+        """Ensure model_dump() includes all fields."""
+        metrics = ControllerHealthMetrics()
+        result = metrics.model_dump()
+
+        expected_keys = [
+            "timestamp",
+            "controller_start_time",
+            "uptime_s",
+            "num_control_loops",
+            "last_control_loop_time",
+            "loop_duration_s",
+            "loops_per_second",
+            "last_sleep_duration_s",
+            "expected_sleep_duration_s",
+            "event_loop_delay_s",
+            "num_asyncio_tasks",
+            "deployment_state_update_duration_s",
+            "application_state_update_duration_s",
+            "proxy_state_update_duration_s",
+            "node_update_duration_s",
+            "handle_metrics_delay_ms",
+            "replica_metrics_delay_ms",
+            "process_memory_mb",
+        ]
+
+        for key in expected_keys:
+            assert key in result, f"Missing key: {key}"
+
+
+class TestCollectHealthMetrics:
+    """Tests for the health metrics collection logic."""
+
+    def test_loop_statistics_computation(self):
+        """Test that loop statistics are computed correctly from tracker data."""
+        tracker = ControllerHealthMetricsTracker()
+
+        # Record some loop durations
+        durations = [0.1, 0.2, 0.3, 0.4, 0.5]
+        for d in durations:
+            tracker.record_loop_duration(d)
+
+        # Verify tracker state
+        assert len(tracker.loop_durations) == 5
+        assert tracker.loop_durations[-1] == 0.5
+
+        # Collect metrics and verify DurationStats
+        metrics = tracker.collect_metrics()
+        assert metrics.loop_duration_s is not None
+        assert metrics.loop_duration_s.mean == 0.3
+        assert metrics.loop_duration_s.min == 0.1
+        assert metrics.loop_duration_s.max == 0.5
+        assert metrics.loop_duration_s.std > 0
+
+    def test_metrics_delay_statistics(self):
+        """Test that metrics delay statistics are computed correctly."""
+        tracker = ControllerHealthMetricsTracker()
+
+        # Record handle metrics delays
+        handle_delays = [10.0, 20.0, 30.0, 40.0, 50.0]
+        for d in handle_delays:
+            tracker.record_handle_metrics_delay(d)
+
+        # Record replica metrics delays
+        replica_delays = [5.0, 15.0, 25.0, 35.0, 45.0]
+        for d in replica_delays:
+            tracker.record_replica_metrics_delay(d)
+
+        # Collect metrics and verify DurationStats
+        metrics = tracker.collect_metrics()
+
+        assert metrics.handle_metrics_delay_ms is not None
+        assert metrics.handle_metrics_delay_ms.mean == 30.0
+        assert metrics.handle_metrics_delay_ms.min == 10.0
+        assert metrics.handle_metrics_delay_ms.max == 50.0
+
+        assert metrics.replica_metrics_delay_ms is not None
+        assert metrics.replica_metrics_delay_ms.mean == 25.0
+        assert metrics.replica_metrics_delay_ms.min == 5.0
+        assert metrics.replica_metrics_delay_ms.max == 45.0
+
+    def test_empty_metrics_delays(self):
+        """Test handling of empty metrics delay lists."""
+        tracker = ControllerHealthMetricsTracker()
+
+        # When no delays recorded, DurationStats should have zero values
+        metrics = tracker.collect_metrics()
+
+        assert metrics.handle_metrics_delay_ms is not None
+        assert metrics.handle_metrics_delay_ms.mean == 0.0
+        assert metrics.handle_metrics_delay_ms.max == 0.0
+
+        assert metrics.replica_metrics_delay_ms is not None
+        assert metrics.replica_metrics_delay_ms.mean == 0.0
+        assert metrics.replica_metrics_delay_ms.max == 0.0
+
+    def test_event_loop_delay_calculation(self):
+        """Test event loop delay is calculated correctly."""
+        from ray.serve._private.constants import CONTROL_LOOP_INTERVAL_S
+
+        tracker = ControllerHealthMetricsTracker()
+
+        # Case 1: Sleep took longer than expected (overloaded)
+        tracker.last_sleep_duration_s = CONTROL_LOOP_INTERVAL_S + 0.5
+        delay = max(0.0, tracker.last_sleep_duration_s - CONTROL_LOOP_INTERVAL_S)
+        assert delay == 0.5
+
+        # Case 2: Sleep took expected time (healthy)
+        tracker.last_sleep_duration_s = CONTROL_LOOP_INTERVAL_S
+        delay = max(0.0, tracker.last_sleep_duration_s - CONTROL_LOOP_INTERVAL_S)
+        assert delay == 0.0
+
+        # Case 3: Sleep was shorter (shouldn't happen, but handle it)
+        tracker.last_sleep_duration_s = CONTROL_LOOP_INTERVAL_S - 0.1
+        delay = max(0.0, tracker.last_sleep_duration_s - CONTROL_LOOP_INTERVAL_S)
+        assert delay == 0.0
+
+    def test_loops_per_second_calculation(self):
+        """Test loops per second calculation."""
+        import time
+
+        tracker = ControllerHealthMetricsTracker()
+        tracker.controller_start_time = time.time() - 10.0  # Started 10 seconds ago
+        tracker.num_control_loops = 5
+
+        now = time.time()
+        uptime = now - tracker.controller_start_time
+        loops_per_second = tracker.num_control_loops / uptime if uptime > 0 else 0.0
+
+        # Should be approximately 0.5 loops per second
+        assert 0.4 < loops_per_second < 0.6
+
+    def test_last_control_loop_time_propagated(self):
+        """Test that last_control_loop_time on the tracker is propagated through
+        collect_metrics."""
+        tracker = ControllerHealthMetricsTracker()
+
+        # Default: tracker has not recorded a control loop yet.
+        metrics = tracker.collect_metrics()
+        assert metrics.last_control_loop_time == 0.0
+
+        # Set a specific timestamp and verify it is reflected in collected metrics.
+        tracker.last_control_loop_time = 12345.6789
+        metrics = tracker.collect_metrics()
+        assert metrics.last_control_loop_time == 12345.6789
+
+        # Update it again and verify the latest value is returned.
+        tracker.last_control_loop_time = 99999.0
+        metrics = tracker.collect_metrics()
+        assert metrics.last_control_loop_time == 99999.0
+
+    def test_component_update_durations_tracked(self):
+        """Test that component update durations are tracked with DurationStats."""
+        tracker = ControllerHealthMetricsTracker()
+
+        # Record some component update durations
+        dsm_durations = [0.1, 0.2, 0.3, 0.4, 0.5]
+        asm_durations = [0.2, 0.3, 0.4, 0.5, 0.6]
+        proxy_durations = [0.3, 0.4, 0.5, 0.6, 0.7]
+        node_durations = [0.05, 0.06, 0.07, 0.08, 0.09]
+
+        for d in dsm_durations:
+            tracker.record_dsm_update_duration(d)
+        for d in asm_durations:
+            tracker.record_asm_update_duration(d)
+        for d in proxy_durations:
+            tracker.record_proxy_update_duration(d)
+        for d in node_durations:
+            tracker.record_node_update_duration(d)
+
+        # Collect metrics and verify DurationStats
+        metrics = tracker.collect_metrics()
+
+        assert metrics.deployment_state_update_duration_s is not None
+        assert metrics.deployment_state_update_duration_s.mean == 0.3
+        assert metrics.deployment_state_update_duration_s.min == 0.1
+        assert metrics.deployment_state_update_duration_s.max == 0.5
+
+        assert metrics.application_state_update_duration_s is not None
+        assert metrics.application_state_update_duration_s.mean == 0.4
+        assert metrics.application_state_update_duration_s.min == 0.2
+        assert metrics.application_state_update_duration_s.max == 0.6
+
+        assert metrics.proxy_state_update_duration_s is not None
+        assert metrics.proxy_state_update_duration_s.mean == 0.5
+        assert metrics.proxy_state_update_duration_s.min == 0.3
+        assert metrics.proxy_state_update_duration_s.max == 0.7
+
+        assert metrics.node_update_duration_s is not None
+        assert abs(metrics.node_update_duration_s.mean - 0.07) < 0.0001
+        assert metrics.node_update_duration_s.min == 0.05
+        assert metrics.node_update_duration_s.max == 0.09
+
+
+class TestControllerHealthMetricsTracker:
+    """Tests for ControllerHealthMetricsTracker."""
+
+    def test_record_loop_duration(self):
+        """Test recording loop durations."""
+        tracker = ControllerHealthMetricsTracker()
+
+        tracker.record_loop_duration(0.5)
+        assert len(tracker.loop_durations) == 1
+        assert tracker.loop_durations[-1] == 0.5
+
+        tracker.record_loop_duration(0.3)
+        assert len(tracker.loop_durations) == 2
+        assert tracker.loop_durations[-1] == 0.3
+
+    def test_rolling_window_size(self):
+        """Test that rolling window doesn't exceed max size."""
+        tracker = ControllerHealthMetricsTracker()
+
+        # Record more than the max history size
+        for i in range(_HEALTH_METRICS_HISTORY_SIZE + 50):
+            tracker.record_loop_duration(float(i))
+
+        assert len(tracker.loop_durations) == _HEALTH_METRICS_HISTORY_SIZE
+        # The oldest values should have been dropped
+        assert tracker.loop_durations[0] == 50.0
+
+    def test_record_handle_metrics_delay(self):
+        """Test recording handle metrics delays."""
+        tracker = ControllerHealthMetricsTracker()
+
+        tracker.record_handle_metrics_delay(100.0)
+        assert len(tracker.handle_metrics_delays) == 1
+        assert tracker.handle_metrics_delays[-1] == 100.0
+
+    def test_record_replica_metrics_delay(self):
+        """Test recording replica metrics delays."""
+        tracker = ControllerHealthMetricsTracker()
+
+        tracker.record_replica_metrics_delay(50.0)
+        assert len(tracker.replica_metrics_delays) == 1
+        assert tracker.replica_metrics_delays[-1] == 50.0
+
+    def test_multiple_delay_records(self):
+        """Test recording multiple metrics delays."""
+        tracker = ControllerHealthMetricsTracker()
+
+        for i in range(10):
+            tracker.record_handle_metrics_delay(float(i * 10))
+            tracker.record_replica_metrics_delay(float(i * 5))
+
+        assert len(tracker.handle_metrics_delays) == 10
+        assert len(tracker.replica_metrics_delays) == 10
+        assert tracker.handle_metrics_delays[-1] == 90.0
+        assert tracker.replica_metrics_delays[-1] == 45.0
+
+    def test_record_dsm_update_duration(self):
+        """Test recording deployment state manager update durations."""
+        tracker = ControllerHealthMetricsTracker()
+
+        tracker.record_dsm_update_duration(0.1)
+        assert len(tracker.dsm_update_durations) == 1
+        assert tracker.dsm_update_durations[-1] == 0.1
+
+        tracker.record_dsm_update_duration(0.2)
+        assert len(tracker.dsm_update_durations) == 2
+        assert tracker.dsm_update_durations[-1] == 0.2
+
+    def test_record_asm_update_duration(self):
+        """Test recording application state manager update durations."""
+        tracker = ControllerHealthMetricsTracker()
+
+        tracker.record_asm_update_duration(0.15)
+        assert len(tracker.asm_update_durations) == 1
+        assert tracker.asm_update_durations[-1] == 0.15
+
+    def test_record_proxy_update_duration(self):
+        """Test recording proxy state update durations."""
+        tracker = ControllerHealthMetricsTracker()
+
+        tracker.record_proxy_update_duration(0.25)
+        assert len(tracker.proxy_update_durations) == 1
+        assert tracker.proxy_update_durations[-1] == 0.25
+
+    def test_record_node_update_duration(self):
+        """Test recording node update durations."""
+        tracker = ControllerHealthMetricsTracker()
+
+        tracker.record_node_update_duration(0.05)
+        assert len(tracker.node_update_durations) == 1
+        assert tracker.node_update_durations[-1] == 0.05
+
+    def test_component_duration_rolling_window(self):
+        """Test that component duration rolling windows respect max size."""
+        tracker = ControllerHealthMetricsTracker()
+
+        # Record more than the max history size
+        for i in range(_HEALTH_METRICS_HISTORY_SIZE + 50):
+            tracker.record_dsm_update_duration(float(i))
+
+        assert len(tracker.dsm_update_durations) == _HEALTH_METRICS_HISTORY_SIZE
+        # The oldest values should have been dropped
+        assert tracker.dsm_update_durations[0] == 50.0
+
+
+if __name__ == "__main__":
+    import sys
+
+    sys.exit(pytest.main(["-v", "-s", __file__]))

python/ray/serve/tests/unit/test_controller_haproxy.py

@@ -0,0 +1,299 @@
+import pytest
+
+from ray.serve._private.common import (
+    DeploymentID,
+    ReplicaID,
+    RequestProtocol,
+    RunningReplicaInfo,
+)
+from ray.serve._private.long_poll import LongPollNamespace
+from ray.serve.schema import (
+    Target,
+    TargetGroup,
+)
+from ray.serve.tests.unit.test_controller_direct_ingress import (
+    FakeApplicationStateManager,
+    FakeDeploymentStateManager,
+    FakeDirectIngressController,
+    FakeKVStore,
+    FakeLongPollHost,
+    FakeProxyStateManager,
+)
+
+
+# Test Controller that overrides methods and dependencies for HAProxy testing
+class FakeHAProxyController(FakeDirectIngressController):
+    def __init__(
+        self,
+        kv_store,
+        long_poll_host,
+        application_state_manager,
+        deployment_state_manager,
+        proxy_state_manager,
+    ):
+        super().__init__(
+            kv_store=kv_store,
+            long_poll_host=long_poll_host,
+            application_state_manager=application_state_manager,
+            deployment_state_manager=deployment_state_manager,
+            proxy_state_manager=proxy_state_manager,
+        )
+
+        self._ha_proxy_enabled = True
+
+
+@pytest.fixture
+def haproxy_controller():
+    kv_store = FakeKVStore()
+    long_poll_host = FakeLongPollHost()
+    app_state_manager = FakeApplicationStateManager({}, {}, {})
+    deployment_state_manager = FakeDeploymentStateManager({})
+    proxy_state_manager = FakeProxyStateManager()
+
+    controller = FakeHAProxyController(
+        kv_store=kv_store,
+        long_poll_host=long_poll_host,
+        application_state_manager=app_state_manager,
+        deployment_state_manager=deployment_state_manager,
+        proxy_state_manager=proxy_state_manager,
+    )
+
+    yield controller
+
+
+@pytest.mark.parametrize("from_proxy_manager", [True, False])
+@pytest.mark.parametrize("ha_proxy_enabled", [True, False])
+def test_get_target_groups_haproxy(
+    haproxy_controller: FakeHAProxyController,
+    from_proxy_manager: bool,
+    ha_proxy_enabled: bool,
+):
+    """Tests get_target_groups returns the appropriate target groups based on the
+    ha_proxy_enabled and from_proxy_manager parameters."""
+
+    haproxy_controller._ha_proxy_enabled = ha_proxy_enabled
+
+    # Setup test data with running applications
+    app_statuses = {"app1": {}}
+    route_prefixes = {"app1": "/app1"}
+    ingress_deployments = {"app1": "app1_ingress"}
+
+    deployment_id1 = DeploymentID(name="app1_ingress", app_name="app1")
+
+    # Create replica info
+    replica_id1 = ReplicaID(unique_id="replica1", deployment_id=deployment_id1)
+    replica_info1 = RunningReplicaInfo(
+        replica_id=replica_id1,
+        node_id="node1",
+        node_ip="10.0.0.1",
+        availability_zone="az1",
+        actor_name="replica1",
+        max_ongoing_requests=100,
+    )
+
+    running_replica_infos = {deployment_id1: [replica_info1]}
+
+    # Setup test application state manager
+    haproxy_controller.application_state_manager = FakeApplicationStateManager(
+        app_statuses=app_statuses,
+        route_prefixes=route_prefixes,
+        ingress_deployments=ingress_deployments,
+    )
+
+    # Setup test deployment state manager
+    haproxy_controller.deployment_state_manager = FakeDeploymentStateManager(
+        running_replica_infos=running_replica_infos,
+    )
+
+    # Setup proxy state manager
+    haproxy_controller.proxy_state_manager.add_proxy_details(
+        "proxy_node1", "10.0.1.1", "proxy1"
+    )
+    haproxy_controller.proxy_state_manager.add_proxy_details(
+        "proxy_node2", "10.0.1.2", "proxy2"
+    )
+
+    # Allocate ports for replicas using controller's methods
+    http_port1 = haproxy_controller.allocate_replica_port(
+        "node1", replica_id1.unique_id, RequestProtocol.HTTP
+    )
+    grpc_port1 = haproxy_controller.allocate_replica_port(
+        "node1", replica_id1.unique_id, RequestProtocol.GRPC
+    )
+
+    target_groups = haproxy_controller.get_target_groups(
+        from_proxy_manager=from_proxy_manager
+    )
+
+    # Create expected target groups
+    if ha_proxy_enabled and not from_proxy_manager:
+        expected_target_groups = [
+            TargetGroup(
+                protocol=RequestProtocol.HTTP,
+                route_prefix="/",
+                app_name="",
+                targets=[
+                    Target(ip="10.0.1.1", port=8000, instance_id="", name="proxy1"),
+                    Target(ip="10.0.1.2", port=8000, instance_id="", name="proxy2"),
+                ],
+            ),
+            TargetGroup(
+                protocol=RequestProtocol.GRPC,
+                route_prefix="/",
+                app_name="",
+                targets=[
+                    Target(ip="10.0.1.1", port=9000, instance_id="", name="proxy1"),
+                    Target(ip="10.0.1.2", port=9000, instance_id="", name="proxy2"),
+                ],
+            ),
+        ]
+    else:
+        expected_target_groups = [
+            TargetGroup(
+                protocol=RequestProtocol.HTTP,
+                route_prefix="/app1",
+                app_name="app1",
+                targets=[
+                    Target(
+                        ip="10.0.0.1", port=http_port1, instance_id="", name="replica1"
+                    ),
+                ],
+            ),
+            TargetGroup(
+                protocol=RequestProtocol.GRPC,
+                route_prefix="/app1",
+                app_name="app1",
+                targets=[
+                    Target(
+                        ip="10.0.0.1", port=grpc_port1, instance_id="", name="replica1"
+                    ),
+                ],
+            ),
+        ]
+
+    # Sort both lists to ensure consistent comparison
+    target_groups.sort(key=lambda g: (g.protocol, g.route_prefix))
+    expected_target_groups.sort(key=lambda g: (g.protocol, g.route_prefix))
+
+    assert target_groups == expected_target_groups
+
+
+def test_get_target_groups_app_with_no_running_replicas(
+    haproxy_controller: FakeHAProxyController,
+):
+    """Tests get_target_groups returns the appropriate target groups when an app
+    has no running replicas."""
+
+    # Setup test data with running applications
+    app_statuses = {"app1": {}}
+    route_prefixes = {"app1": "/app1"}
+    ingress_deployments = {"app1": "app1_ingress"}
+
+    deployment_id1 = DeploymentID(name="app1_ingress", app_name="app1")
+
+    running_replica_infos = {deployment_id1: []}
+
+    # Setup test application state manager
+    haproxy_controller.application_state_manager = FakeApplicationStateManager(
+        app_statuses=app_statuses,
+        route_prefixes=route_prefixes,
+        ingress_deployments=ingress_deployments,
+    )
+
+    # Setup test deployment state manager
+    haproxy_controller.deployment_state_manager = FakeDeploymentStateManager(
+        running_replica_infos=running_replica_infos,
+    )
+
+    target_groups = haproxy_controller.get_target_groups(
+        from_proxy_manager=True,
+    )
+
+    assert target_groups == [
+        TargetGroup(
+            protocol=RequestProtocol.HTTP,
+            route_prefix="/app1",
+            app_name="app1",
+            targets=[],
+        ),
+        TargetGroup(
+            protocol=RequestProtocol.GRPC,
+            route_prefix="/app1",
+            app_name="app1",
+            targets=[],
+        ),
+    ]
+
+
+def test_broadcast_fallback_targets(
+    haproxy_controller: FakeHAProxyController,
+):
+    """Tests get_fallback_proxy_targets returns the appropriate fallback proxy targets."""
+
+    lph = haproxy_controller.long_poll_host
+    assert LongPollNamespace.FALLBACK_TARGETS not in lph.notified_changes
+
+    # Ensure there's no broadcast
+    haproxy_controller.broadcast_fallback_targets_if_changed()
+    assert LongPollNamespace.FALLBACK_TARGETS not in lph.notified_changes
+
+    # Add fallback proxy target
+    haproxy_controller.proxy_state_manager.add_fallback_proxy_target(
+        node_ip="10.0.0.1",
+        port=8500,
+        node_instance_id="instance1",
+        actor_name="proxy1",
+        protocol=RequestProtocol.HTTP,
+    )
+
+    # Ensure there is a broadcast
+    haproxy_controller.broadcast_fallback_targets_if_changed()
+    assert LongPollNamespace.FALLBACK_TARGETS in lph.notified_changes
+    assert lph.notified_changes[LongPollNamespace.FALLBACK_TARGETS] == {
+        RequestProtocol.HTTP: Target(
+            ip="10.0.0.1", port=8500, instance_id="instance1", name="proxy1"
+        ),
+    }
+
+    # Change the fallback proxy target
+    haproxy_controller.proxy_state_manager.add_fallback_proxy_target(
+        node_ip="10.0.0.2",
+        port=8500,
+        node_instance_id="instance1",
+        actor_name="proxy1",
+        protocol=RequestProtocol.HTTP,
+    )
+
+    # Ensure there is a broadcast
+    haproxy_controller.broadcast_fallback_targets_if_changed()
+    assert LongPollNamespace.FALLBACK_TARGETS in lph.notified_changes
+    assert lph.notified_changes[LongPollNamespace.FALLBACK_TARGETS] == {
+        RequestProtocol.HTTP: Target(
+            ip="10.0.0.2", port=8500, instance_id="instance1", name="proxy1"
+        ),
+    }
+
+    # Add fallback proxy target
+    haproxy_controller.proxy_state_manager.add_fallback_proxy_target(
+        node_ip="10.0.0.1",
+        port=9500,
+        node_instance_id="instance1",
+        actor_name="proxy1",
+        protocol=RequestProtocol.GRPC,
+    )
+
+    # Ensure there's another broadcast
+    haproxy_controller.broadcast_fallback_targets_if_changed()
+    assert LongPollNamespace.FALLBACK_TARGETS in lph.notified_changes
+    assert lph.notified_changes[LongPollNamespace.FALLBACK_TARGETS] == {
+        RequestProtocol.HTTP: Target(
+            ip="10.0.0.2", port=8500, instance_id="instance1", name="proxy1"
+        ),
+        RequestProtocol.GRPC: Target(
+            ip="10.0.0.1", port=9500, instance_id="instance1", name="proxy1"
+        ),
+    }
+
+
+if __name__ == "__main__":
+    pytest.main()

python/ray/serve/tests/unit/test_deployment_class.py

@@ -0,0 +1,261 @@
+import itertools
+import random
+import sys
+from typing import Dict, List
+
+import pytest
+
+from ray import serve
+from ray.serve._private.config import DeploymentConfig
+
+
+def get_random_dict_combos(d: Dict, n: int) -> List[Dict]:
+    """Gets n random combinations of dictionary d.
+
+    Args:
+        d: The source dictionary to draw combinations from.
+        n: The maximum number of combinations to return.
+
+    Returns:
+        List of dictionary combinations of lengths from 0 to len(d). List
+        contains n random combinations of d's elements.
+    """
+
+    # Shuffle dictionary without modifying original dictionary
+    d = dict(random.sample(list(d.items()), len(d)))
+
+    combos = []
+
+    # Sample random combos of random size
+    subset_sizes = list(range(len(d) + 1))
+    random.shuffle(subset_sizes)
+
+    for subset_size in subset_sizes:
+        subset_combo_iterator = map(
+            dict, itertools.combinations(d.items(), subset_size)
+        )
+        if len(combos) < n:
+            subset_combos = list(
+                itertools.islice(subset_combo_iterator, n - len(combos))
+            )
+            combos.extend(subset_combos)
+        else:
+            break
+
+    return combos
+
+
+class TestGetDictCombos:
+    def test_empty(self):
+        assert get_random_dict_combos({}, 1) == [{}]
+
+    def test_basic(self):
+        d = {"a": 1, "b": 2, "c": 3}
+        combos = get_random_dict_combos(d, 8)
+
+        # Sort combos for comparison (sort by length, break ties by value sum)
+        combos.sort(key=lambda d: len(d) * 100 + sum(d.values()))
+
+        assert combos == [
+            # Dictionaries of length 0
+            {},
+            # Dictionaries of length 1
+            *({"a": 1}, {"b": 2}, {"c": 3}),
+            # Dictionaries of length 2
+            *({"a": 1, "b": 2}, {"a": 1, "c": 3}, {"b": 2, "c": 3}),
+            # Dictionaries of length 3
+            {"a": 1, "b": 2, "c": 3},
+        ]
+
+    def test_len(self):
+        d = {i: i + 1 for i in range(50)}
+        assert len(get_random_dict_combos(d, 1000)) == 1000
+
+    def test_randomness(self):
+        d = {i: i + 1 for i in range(1000)}
+        combo1 = get_random_dict_combos(d, 1000)[0]
+        combo2 = get_random_dict_combos(d, 1000)[0]
+        assert combo1 != combo2
+
+
+class TestDeploymentOptions:
+    # Deployment options mapped to sample input
+    deployment_options = {
+        "name": "test",
+        "num_replicas": 1,
+        "ray_actor_options": {},
+        "user_config": {},
+        "max_ongoing_requests": 10,
+        "autoscaling_config": None,
+        "graceful_shutdown_wait_loop_s": 10,
+        "graceful_shutdown_timeout_s": 10,
+        "health_check_period_s": 10,
+        "health_check_timeout_s": 10,
+    }
+
+    deployment_option_combos = get_random_dict_combos(deployment_options, 1000)
+
+    @pytest.mark.parametrize("options", deployment_option_combos)
+    def test_user_configured_option_names(self, options: Dict):
+        """Check that user_configured_option_names tracks the correct options.
+
+        Args:
+            options: Maps deployment option strings (e.g. "name",
+                "num_replicas", etc.) to sample inputs. Pairs come from
+                TestDeploymentOptions.deployment_options.
+        """
+
+        @serve.deployment(**options)
+        def f():
+            pass
+
+        assert f._deployment_config.user_configured_option_names == set(options.keys())
+
+    @pytest.mark.parametrize("options", deployment_option_combos)
+    def test_user_configured_option_names_serialized(self, options: Dict):
+        """Check user_configured_option_names after serialization.
+
+        Args:
+            options: Maps deployment option strings (e.g. "name",
+                "num_replicas", etc.) to sample inputs. Pairs come from
+                TestDeploymentOptions.deployment_options.
+        """
+
+        # init_kwargs requires independent serialization, so we omit it.
+        if "init_kwargs" in options:
+            del options["init_kwargs"]
+
+        @serve.deployment(**options)
+        def f():
+            pass
+
+        serialized_config = f._deployment_config.to_proto_bytes()
+        deserialized_config = DeploymentConfig.from_proto_bytes(serialized_config)
+
+        assert deserialized_config.user_configured_option_names == set(options.keys())
+
+    @pytest.mark.parametrize(
+        "option",
+        [
+            "num_replicas",
+            "autoscaling_config",
+            "user_config",
+        ],
+    )
+    def test_nullable_options(self, option: str):
+        """Check that nullable options can be set to None."""
+
+        deployment_options = {option: None}
+
+        # One of "num_replicas" or "autoscaling_config" must be provided.
+        if option == "num_replicas":
+            deployment_options["autoscaling_config"] = {
+                "min_replicas": 1,
+                "max_replicas": 5,
+                "target_ongoing_requests": 5,
+            }
+        elif option == "autoscaling_config":
+            deployment_options["num_replicas"] = 5
+
+        # Deployment should be created without error.
+        @serve.deployment(**deployment_options)
+        def f():
+            pass
+
+    @pytest.mark.parametrize("options", deployment_option_combos)
+    def test_options(self, options):
+        """Check that updating options also updates user_configured_options_names."""
+
+        @serve.deployment
+        def f():
+            pass
+
+        f = f.options(**options)
+        assert f._deployment_config.user_configured_option_names == set(options.keys())
+
+    def test_deployment_decorator_version_removed(self):
+        with pytest.raises(
+            ValueError,
+            match=r"`version` in `@serve\.deployment` has been removed",
+        ):
+
+            @serve.deployment(version="abcd")
+            def f():
+                pass
+
+    def test_deployment_options_version_removed(self):
+        @serve.deployment
+        def f():
+            pass
+
+        with pytest.raises(
+            ValueError,
+            match=r"`version` in `Deployment\.options\(\)` has been removed",
+        ):
+            f.options(version="abcd")
+
+    def test_deployment_route_prefix_removed(self):
+        @serve.deployment
+        def f():
+            pass
+
+        assert not hasattr(f, "route_prefix")
+        with pytest.raises(AttributeError):
+            _ = f.route_prefix
+
+        with pytest.raises(TypeError, match="route_prefix"):
+
+            @serve.deployment(route_prefix="/prefix")
+            def g():
+                pass
+
+        with pytest.raises(TypeError, match="route_prefix"):
+            f.options(route_prefix="/prefix")
+
+    def test_eager_placement_group_validation(self):
+        """Check that placement groups are validated early.
+
+        Placement group bundles should be validated when the deployment is
+        defined, not when it's deployed.
+        """
+
+        with pytest.raises(ValueError):
+
+            # PG bundle with empty resources is invalid.
+            @serve.deployment(
+                placement_group_bundles=[{"CPU": 0, "GPU": 0}],
+                ray_actor_options={"num_cpus": 0, "num_gpus": 0},
+            )
+            def f():
+                pass
+
+    def test_deployment_url_removed(self):
+        @serve.deployment
+        def f():
+            pass
+
+        assert not hasattr(f, "url")
+        with pytest.raises(AttributeError):
+            _ = f.url
+
+    def test_placement_group_strategy_without_bundles(self):
+        """Check that specifying strategy requires also specifying bundles."""
+
+        with pytest.raises(ValueError):
+
+            # PG strategy without bundles is invalid.
+            @serve.deployment(placement_group_strategy="PACK")
+            def f():
+                pass
+
+        # PG strategy with bundles is valid.
+        @serve.deployment(
+            placement_group_strategy="PACK",
+            placement_group_bundles=[{"CPU": 10}],
+        )
+        def g():
+            pass
+
+
+if __name__ == "__main__":
+    sys.exit(pytest.main(["-v", "-s", __file__]))

python/ray/serve/tests/unit/test_deployment_info.py

@@ -0,0 +1,89 @@
+from typing import Optional
+from unittest.mock import patch
+
+import pytest
+
+from ray.serve._private.common import TargetCapacityDirection
+from ray.serve._private.config import DeploymentConfig, ReplicaConfig
+from ray.serve._private.deployment_info import DeploymentInfo
+
+
+@patch("ray.serve._private.deployment_info.ray.get_runtime_context")
+@pytest.mark.parametrize("target_capacity", [0, 25.809, 73, 100.0, None])
+@pytest.mark.parametrize(
+    "target_capacity_direction",
+    [TargetCapacityDirection.UP, TargetCapacityDirection.DOWN, None],
+)
+def test_deployment_info_serialization(
+    mock_runtime_context,
+    target_capacity: Optional[float],
+    target_capacity_direction: Optional[TargetCapacityDirection],
+):
+    """Checks that deploymet infos can be serialized without losing data."""
+    # Mock out the runtime_context call, so Ray doesn't start.
+    class MockRuntimeContext:
+        def get_job_id(self) -> str:
+            return ""
+
+    mock_runtime_context.return_value = MockRuntimeContext()
+
+    def compare_replica_configs(config1: ReplicaConfig, config2: ReplicaConfig):
+        """Check whether two replica configs contain the same data."""
+
+        assert config1.serialized_deployment_def == config2.serialized_deployment_def
+        assert config1.serialized_init_args == config2.serialized_init_args
+        assert config1.serialized_init_kwargs == config2.serialized_init_kwargs
+        assert config1.ray_actor_options == config2.ray_actor_options
+        assert config1.placement_group_bundles == config2.placement_group_bundles
+        assert config1.placement_group_strategy == config2.placement_group_strategy
+        assert config1.max_replicas_per_node == config2.max_replicas_per_node
+
+    def compare_deployment_info(info1: DeploymentInfo, info2: DeploymentInfo):
+        """Checks that two deployment infos contain the same data.
+
+        Does not compare job_ids because those are mocked out to avoid starting
+        Ray
+        """
+
+        assert info1.version == info2.version
+        assert info1.deployment_config == info2.deployment_config
+        compare_replica_configs(info1.replica_config, info2.replica_config)
+        assert info1.start_time_ms == info2.start_time_ms
+        assert info1.target_capacity == info2.target_capacity
+        assert info1.target_capacity_direction == info2.target_capacity_direction
+
+    deployment_info_args = dict(
+        version="123",
+        deployment_config=DeploymentConfig(num_replicas=1),
+        replica_config=ReplicaConfig.create(lambda x: x),
+        start_time_ms=0,
+        deployer_job_id="",
+    )
+
+    # Check that serialization works correctly when the DeploymentInfo is
+    # initialized with the target_capacity info directly.
+    info = DeploymentInfo(
+        target_capacity=target_capacity,
+        target_capacity_direction=target_capacity_direction,
+        **deployment_info_args
+    )
+    serialized_info = info.to_proto()
+    reconstructed_info = DeploymentInfo.from_proto(serialized_info)
+    compare_deployment_info(reconstructed_info, info)
+
+    # Check that serialization works correctly when the DeploymentInfo's
+    # target_capacity has been set using set_target_capacity().
+    info = DeploymentInfo(**deployment_info_args)
+    info.set_target_capacity(
+        new_target_capacity=target_capacity,
+        new_target_capacity_direction=target_capacity_direction,
+    )
+    serialized_info = info.to_proto()
+    reconstructed_info = DeploymentInfo.from_proto(serialized_info)
+    compare_deployment_info(reconstructed_info, info)
+
+
+if __name__ == "__main__":
+    import sys
+
+    sys.exit(pytest.main(["-v", "-s", __file__]))

python/ray/serve/tests/unit/test_deployment_version.py

@@ -0,0 +1,412 @@
+import pytest
+
+from ray.serve._private.config import DeploymentConfig
+from ray.serve._private.deployment_state import DeploymentVersion
+
+
+def test_validation():
+    # Code version must be a string.
+    with pytest.raises(TypeError):
+        DeploymentVersion(123, DeploymentConfig(), {})
+
+
+def test_other_type_equality():
+    v = DeploymentVersion("1", DeploymentConfig(), {})
+
+    assert v is not None
+    assert v != "1"
+    assert v != None  # noqa: E711
+
+
+def test_code_version():
+    v1 = DeploymentVersion("1", DeploymentConfig(), {})
+    v2 = DeploymentVersion("1", DeploymentConfig(), {})
+    v3 = DeploymentVersion("2", DeploymentConfig(), {})
+
+    assert v1 == v2
+    assert hash(v1) == hash(v2)
+    assert v1 != v3
+    assert hash(v1) != hash(v3)
+
+
+def test_deployment_config_basic():
+    v1 = DeploymentVersion("1", DeploymentConfig(user_config="1"), {})
+    v2 = DeploymentVersion("1", DeploymentConfig(user_config="1"), {})
+    v3 = DeploymentVersion("1", DeploymentConfig(user_config="2"), {})
+
+    assert v1 == v2
+    assert hash(v1) == hash(v2)
+    assert v1 != v3
+    assert hash(v1) != hash(v3)
+
+
+def test_user_config_hashable():
+    v1 = DeploymentVersion("1", DeploymentConfig(user_config=("1", "2")), {})
+    v2 = DeploymentVersion("1", DeploymentConfig(user_config=("1", "2")), {})
+    v3 = DeploymentVersion("1", DeploymentConfig(user_config=("1", "3")), {})
+
+    assert v1 == v2
+    assert hash(v1) == hash(v2)
+    assert v1 != v3
+    assert hash(v1) != hash(v3)
+
+
+def test_user_config_list():
+    v1 = DeploymentVersion("1", DeploymentConfig(user_config=["1", "2"]), {})
+    v2 = DeploymentVersion("1", DeploymentConfig(user_config=["1", "2"]), {})
+    v3 = DeploymentVersion("1", DeploymentConfig(user_config=["1", "3"]), {})
+
+    assert v1 == v2
+    assert hash(v1) == hash(v2)
+    assert v1 != v3
+    assert hash(v1) != hash(v3)
+
+
+def test_user_config_dict_keys():
+    v1 = DeploymentVersion("1", DeploymentConfig(user_config={"1": "1"}), {})
+    v2 = DeploymentVersion("1", DeploymentConfig(user_config={"1": "1"}), {})
+    v3 = DeploymentVersion("1", DeploymentConfig(user_config={"2": "1"}), {})
+
+    assert v1 == v2
+    assert hash(v1) == hash(v2)
+    assert v1 != v3
+    assert hash(v1) != hash(v3)
+
+
+def test_user_config_dict_vals():
+    v1 = DeploymentVersion("1", DeploymentConfig(user_config={"1": "1"}), {})
+    v2 = DeploymentVersion("1", DeploymentConfig(user_config={"1": "1"}), {})
+    v3 = DeploymentVersion("1", DeploymentConfig(user_config={"1": "2"}), {})
+
+    assert v1 == v2
+    assert hash(v1) == hash(v2)
+    assert v1 != v3
+    assert hash(v1) != hash(v3)
+
+
+def test_user_config_nested():
+    v1 = DeploymentVersion(
+        "1", DeploymentConfig(user_config=[{"1": "2"}, {"1": "2"}]), {}
+    )
+    v2 = DeploymentVersion(
+        "1", DeploymentConfig(user_config=[{"1": "2"}, {"1": "2"}]), {}
+    )
+    v3 = DeploymentVersion(
+        "1", DeploymentConfig(user_config=[{"1": "2"}, {"1": "3"}]), {}
+    )
+
+    assert v1 == v2
+    assert hash(v1) == hash(v2)
+    assert v1 != v3
+    assert hash(v1) != hash(v3)
+
+
+def test_user_config_nested_in_hashable():
+    v1 = DeploymentVersion(
+        "1", DeploymentConfig(user_config=([{"1": "2"}, {"1": "2"}])), {}
+    )
+    v2 = DeploymentVersion(
+        "1", DeploymentConfig(user_config=([{"1": "2"}, {"1": "2"}])), {}
+    )
+    v3 = DeploymentVersion(
+        "1", DeploymentConfig(user_config=([{"1": "2"}, {"1": "3"}])), {}
+    )
+
+    assert v1 == v2
+    assert hash(v1) == hash(v2)
+    assert v1 != v3
+    assert hash(v1) != hash(v3)
+
+
+def test_num_replicas():
+    v1 = DeploymentVersion("1", DeploymentConfig(num_replicas=1), {})
+    v2 = DeploymentVersion("1", DeploymentConfig(num_replicas=2), {})
+
+    assert v1 == v2
+    assert hash(v1) == hash(v2)
+
+
+def test_autoscaling_config():
+    v1 = DeploymentVersion(
+        "1",
+        DeploymentConfig(
+            autoscaling_config={"max_replicas": 2, "metrics_interval_s": 10}
+        ),
+        {},
+    )
+    v2 = DeploymentVersion(
+        "1",
+        DeploymentConfig(
+            autoscaling_config={"max_replicas": 5, "metrics_interval_s": 10}
+        ),
+        {},
+    )
+    v3 = DeploymentVersion(
+        "1",
+        DeploymentConfig(
+            autoscaling_config={"max_replicas": 2, "metrics_interval_s": 3}
+        ),
+        {},
+    )
+
+    assert v1 == v2
+    assert hash(v1) == hash(v2)
+    assert v1 != v3
+    assert hash(v1) != hash(v3)
+
+
+def test_max_ongoing_requests():
+    v1 = DeploymentVersion("1", DeploymentConfig(max_ongoing_requests=5), {})
+    v2 = DeploymentVersion("1", DeploymentConfig(max_ongoing_requests=5), {})
+    v3 = DeploymentVersion("1", DeploymentConfig(max_ongoing_requests=10), {})
+
+    assert v1 == v2
+    assert hash(v1) == hash(v2)
+    assert not v1.requires_actor_reconfigure(v2)
+    assert v1 != v3
+    assert hash(v1) != hash(v3)
+    assert v1.requires_actor_reconfigure(v3)
+
+
+def test_health_check_period_s():
+    v1 = DeploymentVersion("1", DeploymentConfig(health_check_period_s=5), {})
+    v2 = DeploymentVersion("1", DeploymentConfig(health_check_period_s=5), {})
+    v3 = DeploymentVersion("1", DeploymentConfig(health_check_period_s=10), {})
+
+    assert v1 == v2
+    assert hash(v1) == hash(v2)
+    assert v1 != v3
+    assert hash(v1) != hash(v3)
+
+
+def test_health_check_timeout_s():
+    v1 = DeploymentVersion("1", DeploymentConfig(health_check_timeout_s=5), {})
+    v2 = DeploymentVersion("1", DeploymentConfig(health_check_timeout_s=5), {})
+    v3 = DeploymentVersion("1", DeploymentConfig(health_check_timeout_s=10), {})
+
+    assert v1 == v2
+    assert hash(v1) == hash(v2)
+    assert v1 != v3
+    assert hash(v1) != hash(v3)
+
+
+def test_graceful_shutdown_timeout_s():
+    v1 = DeploymentVersion("1", DeploymentConfig(graceful_shutdown_timeout_s=5), {})
+    v2 = DeploymentVersion("1", DeploymentConfig(graceful_shutdown_timeout_s=5), {})
+    v3 = DeploymentVersion("1", DeploymentConfig(graceful_shutdown_timeout_s=10), {})
+
+    assert v1 == v2
+    assert hash(v1) == hash(v2)
+    assert v1 != v3
+    assert hash(v1) != hash(v3)
+
+
+def test_graceful_shutdown_wait_loop_s():
+    v1 = DeploymentVersion("1", DeploymentConfig(graceful_shutdown_wait_loop_s=5), {})
+    v2 = DeploymentVersion("1", DeploymentConfig(graceful_shutdown_wait_loop_s=5), {})
+    v3 = DeploymentVersion("1", DeploymentConfig(graceful_shutdown_wait_loop_s=10), {})
+
+    assert v1 == v2
+    assert hash(v1) == hash(v2)
+    assert v1 != v3
+    assert hash(v1) != hash(v3)
+
+
+def test_ray_actor_options():
+    v1 = DeploymentVersion("1", DeploymentConfig(), {"num_cpus": 0.1})
+    v2 = DeploymentVersion("1", DeploymentConfig(), {"num_cpus": 0.1})
+    v3 = DeploymentVersion("1", DeploymentConfig(), {"num_gpus": 0.1})
+
+    assert v1 == v2
+    assert hash(v1) == hash(v2)
+    assert v1 != v3
+    assert hash(v1) != hash(v3)
+
+
+def test_max_replicas_per_node():
+    v1 = DeploymentVersion("1", DeploymentConfig(), {"num_cpus": 0.1})
+    v2 = DeploymentVersion(
+        "1",
+        DeploymentConfig(),
+        {"num_cpus": 0.1},
+        max_replicas_per_node=1,
+    )
+    v3 = DeploymentVersion(
+        "1",
+        DeploymentConfig(),
+        {"num_cpus": 0.1},
+        max_replicas_per_node=1,
+    )
+    v4 = DeploymentVersion(
+        "1",
+        DeploymentConfig(),
+        {"num_cpus": 0.1},
+        max_replicas_per_node=2,
+    )
+
+    assert v1 != v2
+    assert hash(v1) != hash(v2)
+    assert v1.requires_actor_restart(v2)
+
+    assert v2 == v3
+    assert hash(v2) == hash(v3)
+    assert not v2.requires_actor_restart(v3)
+
+    assert v3 != v4
+    assert hash(v3) != hash(v4)
+    assert v3.requires_actor_restart(v4)
+
+
+def test_placement_group_options():
+    v1 = DeploymentVersion("1", DeploymentConfig(), {"num_cpus": 0.1})
+    v2 = DeploymentVersion(
+        "1",
+        DeploymentConfig(),
+        {"num_cpus": 0.1},
+        placement_group_bundles=[{"CPU": 0.1}],
+    )
+    v3 = DeploymentVersion(
+        "1",
+        DeploymentConfig(),
+        {"num_cpus": 0.1},
+        placement_group_bundles=[{"CPU": 0.1}],
+    )
+    v4 = DeploymentVersion(
+        "1",
+        DeploymentConfig(),
+        {"num_cpus": 0.1},
+        placement_group_bundles=[{"GPU": 0.1}],
+    )
+
+    assert v1 != v2
+    assert hash(v1) != hash(v2)
+    assert v1.requires_actor_restart(v2)
+
+    assert v2 == v3
+    assert hash(v2) == hash(v3)
+    assert not v2.requires_actor_restart(v3)
+
+    assert v3 != v4
+    assert hash(v3) != hash(v4)
+    assert v3.requires_actor_restart(v4)
+
+    v5 = DeploymentVersion(
+        "1",
+        DeploymentConfig(),
+        {"num_cpus": 0.1},
+        placement_group_bundles=[{"CPU": 0.1}],
+        placement_group_strategy="PACK",
+    )
+    v6 = DeploymentVersion(
+        "1",
+        DeploymentConfig(),
+        {"num_cpus": 0.1},
+        placement_group_bundles=[{"CPU": 0.1}],
+        placement_group_strategy="PACK",
+    )
+    v7 = DeploymentVersion(
+        "1",
+        DeploymentConfig(),
+        {"num_cpus": 0.1},
+        placement_group_bundles=[{"CPU": 0.1}],
+        placement_group_strategy="SPREAD",
+    )
+
+    assert v5 == v6
+    assert hash(v5) == hash(v6)
+    assert not v5.requires_actor_restart(v6)
+
+    assert v6 != v7
+    assert hash(v6) != hash(v7)
+    assert v6.requires_actor_restart(v7)
+
+
+def test_requires_actor_restart():
+    # Code version different
+    v1 = DeploymentVersion("1", DeploymentConfig(), {"num_cpus": 0.1})
+    v2 = DeploymentVersion("2", DeploymentConfig(), {"num_cpus": 0.1})
+    assert v1.requires_actor_restart(v2)
+
+    # Runtime env different
+    v1 = DeploymentVersion("1", DeploymentConfig(), {"num_cpus": 0.1})
+    v2 = DeploymentVersion("1", DeploymentConfig(), {"num_cpus": 0.2})
+    assert v1.requires_actor_restart(v2)
+
+    # Placement group bundles different
+    v1 = DeploymentVersion(
+        "1", DeploymentConfig(), {}, placement_group_bundles=[{"CPU": 0.1}]
+    )
+    v2 = DeploymentVersion(
+        "1", DeploymentConfig(), {}, placement_group_bundles=[{"CPU": 0.2}]
+    )
+    assert v1.requires_actor_restart(v2)
+
+    # Placement group strategy different
+    v1 = DeploymentVersion("1", DeploymentConfig(), {}, placement_group_strategy="PACK")
+    v2 = DeploymentVersion(
+        "1", DeploymentConfig(), {}, placement_group_strategy="SPREAD"
+    )
+    assert v1.requires_actor_restart(v2)
+
+    # Both code version and runtime env different
+    v1 = DeploymentVersion("1", DeploymentConfig(), {"num_cpus": 0.1})
+    v2 = DeploymentVersion("2", DeploymentConfig(), {"num_cpus": 0.2})
+    assert v1.requires_actor_restart(v2)
+
+    # Num replicas is different
+    v1 = DeploymentVersion("1", DeploymentConfig(num_replicas=1), {})
+    v2 = DeploymentVersion("1", DeploymentConfig(num_replicas=2), {})
+    assert not v1.requires_actor_restart(v2)
+
+    # Graceful shutdown timeout is different
+    v1 = DeploymentVersion("1", DeploymentConfig(graceful_shutdown_timeout_s=5), {})
+    v2 = DeploymentVersion("1", DeploymentConfig(graceful_shutdown_timeout_s=10), {})
+    assert not v1.requires_actor_restart(v2)
+
+
+def test_requires_actor_reconfigure():
+    # Replicas need the updated user config to call the user-defined
+    # reconfigure method
+    v1 = DeploymentVersion("1", DeploymentConfig(user_config=1), {})
+    v2 = DeploymentVersion("1", DeploymentConfig(user_config=2), {})
+    assert v1.requires_actor_reconfigure(v2)
+
+    # Graceful shutdown loop requires actor reconfigure, since the
+    # replica needs the updated value to correctly execute graceful
+    # shutdown.
+    v1 = DeploymentVersion("1", DeploymentConfig(graceful_shutdown_wait_loop_s=1), {})
+    v2 = DeploymentVersion("1", DeploymentConfig(graceful_shutdown_wait_loop_s=2), {})
+    assert v1.requires_actor_reconfigure(v2)
+
+    # Graceful shutdown timeout shouldn't require actor reconfigure, as
+    # it's only used by the controller to decide when to force-kill a
+    # replica
+    v1 = DeploymentVersion("1", DeploymentConfig(graceful_shutdown_timeout_s=5), {})
+    v2 = DeploymentVersion("1", DeploymentConfig(graceful_shutdown_timeout_s=10), {})
+    assert not v1.requires_actor_reconfigure(v2)
+
+    # Num replicas shouldn't require actor reconfigure, as it's only
+    # by the controller to decide when to start or stop replicas.
+    v1 = DeploymentVersion("1", DeploymentConfig(num_replicas=1), {})
+    v2 = DeploymentVersion("1", DeploymentConfig(num_replicas=2), {})
+    assert not v1.requires_actor_reconfigure(v2)
+
+
+def test_requires_long_poll_broadcast():
+    # If max concurrent queries is updated, it needs to be broadcasted
+    # to all routers.
+    v1 = DeploymentVersion("1", DeploymentConfig(max_ongoing_requests=5), {})
+    v2 = DeploymentVersion("1", DeploymentConfig(max_ongoing_requests=10), {})
+    assert v1.requires_long_poll_broadcast(v2)
+
+    # Something random like health check timeout doesn't require updating
+    # any info on routers.
+    v1 = DeploymentVersion("1", DeploymentConfig(health_check_timeout_s=5), {})
+    v2 = DeploymentVersion("1", DeploymentConfig(health_check_timeout_s=10), {})
+    assert not v1.requires_long_poll_broadcast(v2)
+
+
+if __name__ == "__main__":
+    import sys
+
+    sys.exit(pytest.main(["-v", "-s", __file__]))

python/ray/serve/tests/unit/test_endpoint_state.py

@@ -0,0 +1,66 @@
+import sys
+from typing import Any, Tuple
+from unittest.mock import Mock, patch
+
+import pytest
+
+from ray.serve._private.endpoint_state import EndpointState
+
+
+class MockKVStore:
+    def __init__(self):
+        self.store = dict()
+
+    def put(self, key: str, val: Any) -> bool:
+        if not isinstance(key, str):
+            raise TypeError("key must be a string, got: {}.".format(type(key)))
+        self.store[key] = val
+        return True
+
+    def get(self, key: str) -> Any:
+        if not isinstance(key, str):
+            raise TypeError("key must be a string, got: {}.".format(type(key)))
+        return self.store.get(key, None)
+
+    def delete(self, key: str) -> bool:
+        if not isinstance(key, str):
+            raise TypeError("key must be a string, got: {}.".format(type(key)))
+
+        if key in self.store:
+            del self.store[key]
+            return True
+
+        return False
+
+
+@pytest.fixture
+def mock_endpoint_state() -> Tuple[EndpointState, Mock]:
+    with patch("ray.serve._private.long_poll.LongPollHost") as mock_long_poll:
+        endpoint_state = EndpointState(
+            kv_store=MockKVStore(),
+            long_poll_host=mock_long_poll,
+        )
+        yield endpoint_state
+
+
+def test_is_ready_for_shutdown(mock_endpoint_state):
+    """Test `is_ready_for_shutdown()` returns the correct state.
+
+    Before shutting down endpoint `is_ready_for_shutdown()` should return False.
+    After shutting down endpoint `is_ready_for_shutdown()` should return True.
+    """
+    # Setup endpoint state with checkpoint
+    endpoint_state = mock_endpoint_state
+    endpoint_state._checkpoint()
+
+    # Before shutdown is called, `is_ready_for_shutdown()` should return False
+    assert not endpoint_state.is_ready_for_shutdown()
+
+    endpoint_state.shutdown()
+
+    # After shutdown is called, `is_ready_for_shutdown()` should return True
+    assert endpoint_state.is_ready_for_shutdown()
+
+
+if __name__ == "__main__":
+    sys.exit(pytest.main(["-v", "-s", __file__]))

python/ray/serve/tests/unit/test_extract_route_patterns.py

@@ -0,0 +1,374 @@
+"""Unit tests for extract_route_patterns function."""
+import pytest
+from fastapi import APIRouter, FastAPI
+from starlette.applications import Starlette
+from starlette.routing import Mount, Route
+
+from ray.serve._private.thirdparty.get_asgi_route_name import (
+    extract_route_patterns,
+)
+
+
+def has_path(patterns, path):
+    """Helper to check if a path exists in patterns list."""
+    return any(pattern.path == path for pattern in patterns)
+
+
+def get_methods_for_path(patterns, path):
+    """Helper to get methods for a specific path."""
+    for pattern in patterns:
+        if pattern.path == path:
+            return pattern.methods
+    return None
+
+
+def test_extract_route_patterns_fastapi_simple():
+    """Test extracting route patterns from a simple FastAPI app."""
+    app = FastAPI()
+
+    @app.get("/")
+    def root():
+        return {"message": "root"}
+
+    @app.get("/users/{user_id}")
+    def get_user(user_id: str):
+        return {"user_id": user_id}
+
+    @app.post("/items/{item_id}")
+    def create_item(item_id: str):
+        return {"item_id": item_id}
+
+    patterns = extract_route_patterns(app)
+
+    # FastAPI automatically adds some default routes
+    assert has_path(patterns, "/")
+    assert has_path(patterns, "/users/{user_id}")
+    assert has_path(patterns, "/items/{item_id}")
+    # FastAPI adds OpenAPI routes
+    assert has_path(patterns, "/openapi.json")
+    assert has_path(patterns, "/docs")
+
+
+def test_extract_route_patterns_nested_paths():
+    """Test extracting nested parameterized routes."""
+    app = FastAPI()
+
+    @app.get("/api/v1/users/{user_id}/posts/{post_id}")
+    def get_post(user_id: str, post_id: str):
+        return {"user_id": user_id, "post_id": post_id}
+
+    @app.get("/api/v1/users/{user_id}/settings")
+    def get_settings(user_id: str):
+        return {"user_id": user_id}
+
+    patterns = extract_route_patterns(app)
+
+    assert has_path(patterns, "/api/v1/users/{user_id}/posts/{post_id}")
+    assert has_path(patterns, "/api/v1/users/{user_id}/settings")
+
+
+def test_extract_route_patterns_with_mounts():
+    """Test extracting route patterns from apps with mounted sub-apps."""
+    # Create a sub-app
+    sub_app = Starlette(
+        routes=[
+            Route("/health", lambda request: None),
+            Route("/status", lambda request: None),
+        ]
+    )
+
+    # Create main app with mounted sub-app
+    app = Starlette(
+        routes=[
+            Route("/", lambda request: None),
+            Mount("/admin", app=sub_app),
+        ]
+    )
+
+    patterns = extract_route_patterns(app)
+
+    assert has_path(patterns, "/")
+    assert has_path(patterns, "/admin/health")
+    assert has_path(patterns, "/admin/status")
+
+
+def test_extract_route_patterns_included_router():
+    """Routes registered via `include_router` must be extracted.
+
+    On FastAPI >= 0.137 these routes are nested under an `_IncludedRouter` node
+    instead of being flattened into `app.routes` (see #64475).
+    """
+    app = FastAPI()
+
+    @app.get("/direct")
+    def direct():
+        return {}
+
+    # Router with its own prefix, included without an include-time prefix.
+    router = APIRouter(prefix="/prefix")
+
+    @router.get("/routed/{user_id}")
+    def routed():
+        return {}
+
+    @router.websocket("/ws")
+    async def ws():
+        pass
+
+    # Router included with an include-time prefix.
+    other = APIRouter()
+
+    @other.post("/create")
+    def create():
+        return {}
+
+    app.include_router(router)
+    app.include_router(other, prefix="/other")
+
+    patterns = extract_route_patterns(app)
+
+    assert has_path(patterns, "/direct")
+    assert has_path(patterns, "/prefix/routed/{user_id}")
+    assert get_methods_for_path(patterns, "/prefix/routed/{user_id}") == ["GET"]
+    assert has_path(patterns, "/other/create")
+    assert get_methods_for_path(patterns, "/other/create") == ["POST"]
+    # WebSocket routes have no method restrictions.
+    assert has_path(patterns, "/prefix/ws")
+    assert get_methods_for_path(patterns, "/prefix/ws") is None
+
+
+def test_extract_route_patterns_nested_mounts():
+    """Test extracting patterns from deeply nested mounts."""
+    # Innermost app
+    inner_app = Starlette(
+        routes=[
+            Route("/details", lambda request: None),
+        ]
+    )
+
+    # Middle app
+    middle_app = Starlette(
+        routes=[
+            Route("/list", lambda request: None),
+            Mount("/item", app=inner_app),
+        ]
+    )
+
+    # Main app
+    app = Starlette(
+        routes=[
+            Route("/", lambda request: None),
+            Mount("/api/v1", app=middle_app),
+        ]
+    )
+
+    patterns = extract_route_patterns(app)
+
+    assert has_path(patterns, "/")
+    assert has_path(patterns, "/api/v1/list")
+    assert has_path(patterns, "/api/v1/item/details")
+
+
+def test_extract_route_patterns_with_root_path():
+    """Test extracting patterns from apps with root_path set."""
+    app = FastAPI(root_path="/v1")
+
+    @app.get("/")
+    def root():
+        return {}
+
+    @app.get("/users")
+    def get_users():
+        return []
+
+    @app.get("/items/{item_id}")
+    def get_item(item_id: str):
+        return {"item_id": item_id}
+
+    patterns = extract_route_patterns(app)
+
+    # Root path should be prepended to all routes
+    assert has_path(patterns, "/v1/")  # Root route
+    assert has_path(patterns, "/v1/users")
+    assert has_path(patterns, "/v1/items/{item_id}")
+
+
+def test_extract_route_patterns_empty_app():
+    """Test extracting patterns from an app with no user-defined routes."""
+    app = FastAPI()
+    # Don't define any routes
+
+    patterns = extract_route_patterns(app)
+
+    # Should still have FastAPI defaults
+    assert has_path(patterns, "/openapi.json")
+    assert has_path(patterns, "/docs")
+    # May or may not have "/" depending on FastAPI version
+
+
+def test_extract_route_patterns_starlette():
+    """Test extracting patterns from a pure Starlette app."""
+
+    async def homepage(request):
+        return None
+
+    async def user_detail(request):
+        return None
+
+    app = Starlette(
+        routes=[
+            Route("/", homepage),
+            Route("/users/{user_id}", user_detail),
+        ]
+    )
+
+    patterns = extract_route_patterns(app)
+
+    assert has_path(patterns, "/")
+    assert has_path(patterns, "/users/{user_id}")
+    # Starlette shouldn't have OpenAPI routes
+    assert not has_path(patterns, "/openapi.json")
+
+
+def test_extract_route_patterns_multiple_methods_same_path():
+    """Test that methods are grouped when multiple methods use same path."""
+    app = FastAPI()
+
+    @app.get("/items/{item_id}")
+    def get_item(item_id: str):
+        return {"item_id": item_id}
+
+    @app.put("/items/{item_id}")
+    def update_item(item_id: str):
+        return {"item_id": item_id}
+
+    @app.delete("/items/{item_id}")
+    def delete_item(item_id: str):
+        return {"item_id": item_id}
+
+    patterns = extract_route_patterns(app)
+
+    # Path should appear only once with all methods grouped
+    path_count = sum(1 for pattern in patterns if pattern.path == "/items/{item_id}")
+    assert path_count == 1
+
+    # Check that all methods are present
+    methods = get_methods_for_path(patterns, "/items/{item_id}")
+    assert methods is not None
+    assert "GET" in methods
+    assert "PUT" in methods
+    assert "DELETE" in methods
+
+
+def test_extract_route_patterns_invalid_app():
+    """Test that invalid apps return empty list gracefully."""
+
+    class FakeApp:
+        """An app without routes attribute."""
+
+        pass
+
+    fake_app = FakeApp()
+
+    # Should return empty list without raising exception
+    patterns = extract_route_patterns(fake_app)
+    assert patterns == []
+
+
+def test_extract_route_patterns_mount_without_routes():
+    """Test handling mounts that don't have sub-routes."""
+    from starlette.responses import PlainTextResponse
+
+    async def custom_mount(scope, receive, send):
+        response = PlainTextResponse("Custom mount")
+        await response(scope, receive, send)
+
+    app = Starlette(
+        routes=[
+            Route("/", lambda request: None),
+            Mount("/custom", app=custom_mount),
+        ]
+    )
+
+    patterns = extract_route_patterns(app)
+
+    assert has_path(patterns, "/")
+    assert has_path(patterns, "/custom")
+    # Custom mount has no method restrictions
+    assert get_methods_for_path(patterns, "/custom") is None
+
+
+def test_extract_route_patterns_sorted_output():
+    """Test that output is sorted by path."""
+    app = FastAPI()
+
+    @app.get("/zebra")
+    def zebra():
+        return {}
+
+    @app.get("/apple")
+    def apple():
+        return {}
+
+    @app.get("/banana")
+    def banana():
+        return {}
+
+    patterns = extract_route_patterns(app)
+
+    # Extract just the paths
+    paths = [pattern.path for pattern in patterns]
+
+    # Find the user-defined routes
+    user_routes = [p for p in paths if p in ["/zebra", "/apple", "/banana"]]
+
+    # Should be sorted
+    assert user_routes == ["/apple", "/banana", "/zebra"]
+
+
+def test_extract_route_patterns_special_characters():
+    """Test routes with special regex characters."""
+    app = FastAPI()
+
+    @app.get("/users/{user_id:path}")
+    def get_user_path(user_id: str):
+        return {"user_id": user_id}
+
+    @app.get("/items/{item_id:int}")
+    def get_item_int(item_id: int):
+        return {"item_id": item_id}
+
+    patterns = extract_route_patterns(app)
+
+    # Extract just the paths
+    paths = [pattern.path for pattern in patterns]
+
+    # FastAPI converts these to standard patterns
+    assert any("user_id" in p for p in paths)
+    assert any("item_id" in p for p in paths)
+
+
+def test_extract_route_patterns_websocket_routes():
+    """Test that WebSocket routes are also extracted."""
+    app = FastAPI()
+
+    @app.get("/http")
+    def http_route():
+        return {}
+
+    @app.websocket("/ws")
+    async def websocket_route(websocket):
+        await websocket.accept()
+        await websocket.close()
+
+    patterns = extract_route_patterns(app)
+
+    assert has_path(patterns, "/http")
+    assert has_path(patterns, "/ws")
+
+    # WebSocket route should have no method restrictions
+    assert get_methods_for_path(patterns, "/ws") is None
+
+
+if __name__ == "__main__":
+    pytest.main([__file__, "-v"])

python/ray/serve/tests/unit/test_get_asgi_app_route.py

@@ -0,0 +1,286 @@
+import sys
+
+import pytest
+from fastapi import APIRouter, FastAPI
+
+from ray.serve._private.thirdparty.get_asgi_route_name import get_asgi_route_name
+
+
+def test_single_path():
+    app = FastAPI()
+
+    @app.get("/")
+    def root():
+        pass
+
+    # Match.
+    assert (
+        get_asgi_route_name(app, {"type": "http", "method": "GET", "path": "/"}) == "/"
+    )
+    # Mismatched subpath.
+    assert (
+        get_asgi_route_name(app, {"type": "http", "method": "GET", "path": "/subpath"})
+        is None
+    )
+
+
+def test_methods():
+    app = FastAPI()
+
+    @app.get("/")
+    def root_get():
+        pass
+
+    @app.post("/")
+    def root_post():
+        pass
+
+    # Match GET.
+    assert (
+        get_asgi_route_name(app, {"type": "http", "method": "GET", "path": "/"}) == "/"
+    )
+    # Match POST.
+    assert (
+        get_asgi_route_name(app, {"type": "http", "method": "GET", "path": "/"}) == "/"
+    )
+    # Missing PUT.
+    assert (
+        get_asgi_route_name(app, {"type": "http", "method": "PUT", "path": "/"}) is None
+    )
+
+
+def test_subpath():
+    app = FastAPI()
+
+    @app.get("/subpath")
+    def subpath():
+        pass
+
+    # Match.
+    assert (
+        get_asgi_route_name(app, {"type": "http", "method": "GET", "path": "/subpath"})
+        == "/subpath"
+    )
+    # Missing subpath.
+    assert (
+        get_asgi_route_name(app, {"type": "http", "method": "GET", "path": "/"}) is None
+    )
+
+
+def test_wildcard():
+    app = FastAPI()
+
+    @app.get("/{user_id}")
+    def dynamic_subpath():
+        pass
+
+    # Match.
+    assert (
+        get_asgi_route_name(app, {"type": "http", "method": "GET", "path": "/abc123"})
+        == "/{user_id}"
+    )
+    # Missing subpath.
+    assert (
+        get_asgi_route_name(app, {"type": "http", "method": "GET", "path": "/"}) is None
+    )
+
+
+def test_mounted_app():
+    app = FastAPI()
+
+    @app.get("/")
+    def root():
+        pass
+
+    mounted_app = FastAPI()
+
+    @mounted_app.get("/")
+    def mounted_root():
+        pass
+
+    @mounted_app.get("/subpath")
+    def mounted_subpath():
+        pass
+
+    @mounted_app.get("/subpath/{user_id}")
+    def mounted_dynamic_subpath():
+        pass
+
+    app.mount("/mounted", mounted_app)
+
+    # Match base app root.
+    assert (
+        get_asgi_route_name(app, {"type": "http", "method": "GET", "path": "/"}) == "/"
+    )
+    # Match mounted app root.
+    assert (
+        get_asgi_route_name(app, {"type": "http", "method": "GET", "path": "/mounted"})
+        == "/mounted"
+    )
+    # Match mounted app subpath.
+    assert (
+        get_asgi_route_name(
+            app, {"type": "http", "method": "GET", "path": "/mounted/subpath"}
+        )
+        == "/mounted/subpath"
+    )
+    # Match mounted app dynamic subpath.
+    assert (
+        get_asgi_route_name(
+            app, {"type": "http", "method": "GET", "path": "/mounted/subpath/abc123"}
+        )
+        == "/mounted/subpath/{user_id}"
+    )
+    # Missing mounted app route.
+    assert (
+        get_asgi_route_name(
+            app, {"type": "http", "method": "GET", "path": "/mounted/some-other-path"}
+        )
+        is None
+    )
+
+
+def test_included_router():
+    """Routes registered via `include_router` must resolve their route name.
+
+    On FastAPI >= 0.137 these routes are nested under an `_IncludedRouter` node
+    instead of being flattened into `app.routes` (see #64475).
+    """
+    app = FastAPI()
+
+    @app.get("/direct")
+    def direct():
+        pass
+
+    # Router with its own prefix, included without an include-time prefix.
+    router = APIRouter(prefix="/prefix")
+
+    @router.get("/routed/{user_id}")
+    def routed():
+        pass
+
+    @router.websocket("/ws")
+    async def ws():
+        pass
+
+    # Router included with an include-time prefix.
+    other = APIRouter()
+
+    @other.post("/create")
+    def create():
+        pass
+
+    app.include_router(router)
+    app.include_router(other, prefix="/other")
+
+    # Directly decorated route still resolves.
+    assert (
+        get_asgi_route_name(app, {"type": "http", "method": "GET", "path": "/direct"})
+        == "/direct"
+    )
+    # Route from an included router (with dynamic segment).
+    assert (
+        get_asgi_route_name(
+            app, {"type": "http", "method": "GET", "path": "/prefix/routed/abc123"}
+        )
+        == "/prefix/routed/{user_id}"
+    )
+    # WebSocket route from an included router.
+    assert (
+        get_asgi_route_name(app, {"type": "websocket", "path": "/prefix/ws"})
+        == "/prefix/ws"
+    )
+    # Route from a router included with an include-time prefix.
+    assert (
+        get_asgi_route_name(
+            app, {"type": "http", "method": "POST", "path": "/other/create"}
+        )
+        == "/other/create"
+    )
+    # Unknown path still returns None.
+    assert (
+        get_asgi_route_name(app, {"type": "http", "method": "GET", "path": "/nope"})
+        is None
+    )
+
+
+def test_root_path():
+    app = FastAPI(root_path="/some/root")
+
+    @app.get("/subpath")
+    def subpath():
+        pass
+
+    assert (
+        get_asgi_route_name(
+            app,
+            {
+                "type": "http",
+                "method": "GET",
+                "path": "/subpath",
+                "root_path": "/some/root",
+            },
+        )
+        == "/some/root/subpath"
+    )
+
+
+@pytest.mark.parametrize("redirect_slashes", [False, True])
+def test_redirect_slashes(redirect_slashes: bool):
+    app = FastAPI(redirect_slashes=redirect_slashes)
+
+    @app.get("/subpath")
+    def subpath():
+        pass
+
+    # Should always match.
+    assert (
+        get_asgi_route_name(app, {"type": "http", "method": "GET", "path": "/subpath"})
+        == "/subpath"
+    )
+    # Should match depending on redirect_slashes behavior.
+    if redirect_slashes:
+        assert (
+            get_asgi_route_name(
+                app, {"type": "http", "method": "GET", "path": "/subpath/"}
+            )
+            == "/subpath/"
+        )
+    else:
+        assert (
+            get_asgi_route_name(
+                app, {"type": "http", "method": "GET", "path": "/subpath/"}
+            )
+            is None
+        )
+
+    @app.get("/other/{user_id}")
+    def dynamic_subpath():
+        pass
+
+    # Should always match.
+    assert (
+        get_asgi_route_name(
+            app, {"type": "http", "method": "GET", "path": "/other/abc123"}
+        )
+        == "/other/{user_id}"
+    )
+    # Should match depending on redirect_slashes behavior.
+    if redirect_slashes:
+        assert (
+            get_asgi_route_name(
+                app, {"type": "http", "method": "GET", "path": "/other/abc123/"}
+            )
+            == "/other/{user_id}/"
+        )
+    else:
+        assert (
+            get_asgi_route_name(
+                app, {"type": "http", "method": "GET", "path": "/other/abc123/"}
+            )
+            is None
+        )
+
+
+if __name__ == "__main__":
+    sys.exit(pytest.main(["-v", "-s", __file__]))

python/ray/serve/tests/unit/test_grpc_replica_result.py

@@ -0,0 +1,391 @@
+import asyncio
+import sys
+import threading
+
+import pytest
+
+from ray import ActorID, cloudpickle
+from ray._common.test_utils import wait_for_condition
+from ray.serve._private.common import RequestMetadata
+from ray.serve._private.replica_result import gRPCReplicaResult
+from ray.serve.generated import serve_pb2
+
+
+class FakegRPCUnaryCall:
+    def __init__(self, item, is_error: bool = False):
+        self._loop = asyncio.get_running_loop()
+        self._item = item
+        self._is_error = is_error
+
+    def __await__(self):
+        if asyncio.get_running_loop() != self._loop:
+            raise RuntimeError("Tried to fetch from a different loop!")
+
+        yield
+        return serve_pb2.ASGIResponse(
+            serialized_message=cloudpickle.dumps(self._item), is_error=self._is_error
+        )
+
+    def add_done_callback(self, cb):
+        pass
+
+
+class FakegRPCStreamCall:
+    def __init__(self, items, *, event: threading.Event = None):
+        self._loop = asyncio.get_running_loop()
+        self._items = items
+        self._event = event
+
+    def is_empty(self) -> bool:
+        assert len(self._items) == 0
+        return True
+
+    def __aiter__(self):
+        return self
+
+    async def __anext__(self):
+        if asyncio.get_running_loop() != self._loop:
+            raise RuntimeError("Tried to fetch from a different loop!")
+
+        if not self._items:
+            raise StopAsyncIteration
+
+        if self._event:
+            await self._loop.run_in_executor(None, self._event.wait)
+
+        item, is_error = self._items.pop(0)
+        return serve_pb2.ASGIResponse(
+            serialized_message=cloudpickle.dumps(item),
+            is_error=is_error,
+        )
+
+    def add_done_callback(self, cb):
+        pass
+
+
+@pytest.fixture
+def create_asyncio_event_loop_in_thread():
+    async_loop = asyncio.new_event_loop()
+    thread = threading.Thread(daemon=True, target=async_loop.run_forever)
+    thread.start()
+    event = threading.Event()
+
+    yield async_loop, event
+
+    # Unblock event in case it's blocking shutdown
+    event.set()
+
+
+@pytest.mark.asyncio
+class TestSameLoop:
+    def make_fake_call(self, is_streaming: bool, *, data=None, error=None):
+        if is_streaming:
+            fake_call = FakegRPCStreamCall(data)
+        else:
+            if error:
+                fake_call = FakegRPCUnaryCall(error, is_error=True)
+            else:
+                fake_call = FakegRPCUnaryCall(data, is_error=False)
+
+        return gRPCReplicaResult(
+            fake_call,
+            metadata=RequestMetadata(
+                request_id="",
+                internal_request_id="",
+                is_streaming=False,
+                _on_separate_loop=False,
+            ),
+            actor_id=ActorID(b"2" * 16),
+            loop=asyncio.get_running_loop(),
+        )
+
+    async def test_unary(self):
+        replica_result = self.make_fake_call(is_streaming=False, data="hello")
+        assert await replica_result.get_async() == "hello"
+
+    async def test_streaming(self):
+        replica_result = self.make_fake_call(
+            is_streaming=True, data=[(1, False), (2, False), (3, False), (4, False)]
+        )
+        assert [r async for r in replica_result] == [1, 2, 3, 4]
+
+    async def test_unary_with_gen(self):
+        replica_result = self.make_fake_call(is_streaming=True, data=[("hello", False)])
+        assert await replica_result.get_async() == "hello"
+
+    async def test_unary_error(self):
+        """Test error is raised correctly."""
+
+        replica_result = self.make_fake_call(
+            is_streaming=False, error=RuntimeError("oh no!")
+        )
+        with pytest.raises(RuntimeError, match="oh no!"):
+            await replica_result.get_async()
+
+    async def test_streaming_error(self):
+        """Test error is raised correctly."""
+
+        replica_result = self.make_fake_call(
+            is_streaming=True, data=[(RuntimeError("oh no!"), True)]
+        )
+        with pytest.raises(RuntimeError, match="oh no!"):
+            await replica_result.__anext__()
+
+
+class TestSeparateLoop:
+    async def make_fake_unary_request(self, data, loop: asyncio.AbstractEventLoop):
+        fake_call = FakegRPCUnaryCall(data)
+        replica_result = gRPCReplicaResult(
+            fake_call,
+            metadata=RequestMetadata(
+                request_id="",
+                internal_request_id="",
+                is_streaming=False,
+                _on_separate_loop=True,
+            ),
+            actor_id=ActorID(b"2" * 16),
+            loop=loop,
+        )
+        return replica_result
+
+    async def make_fake_streaming_request(
+        self,
+        data,
+        loop: asyncio.AbstractEventLoop,
+        on_separate_loop: bool,
+        *,
+        is_streaming: bool = True,
+        event: threading.Event = None,
+        error=None,
+    ):
+        if error:
+            fake_call = FakegRPCStreamCall([(error, True)], event=event)
+        else:
+            fake_call = FakegRPCStreamCall([(d, False) for d in data], event=event)
+        return gRPCReplicaResult(
+            fake_call,
+            metadata=RequestMetadata(
+                request_id="",
+                internal_request_id="",
+                is_streaming=is_streaming,
+                _on_separate_loop=on_separate_loop,
+            ),
+            actor_id=ActorID(b"2" * 16),
+            loop=loop,
+        )
+
+    def test_unary_sync(self, create_asyncio_event_loop_in_thread):
+        loop, _ = create_asyncio_event_loop_in_thread
+
+        fut = asyncio.run_coroutine_threadsafe(
+            self.make_fake_unary_request("hello", loop), loop=loop
+        )
+        replica_result = fut.result()
+
+        assert replica_result.get(None) == "hello"
+
+    @pytest.mark.asyncio
+    async def test_unary_async(self, create_asyncio_event_loop_in_thread):
+        loop, _ = create_asyncio_event_loop_in_thread
+
+        fut = asyncio.run_coroutine_threadsafe(
+            self.make_fake_unary_request("hello", loop), loop=loop
+        )
+        replica_result = fut.result()
+
+        assert await replica_result.get_async() == "hello"
+
+    def test_streaming_sync(self, create_asyncio_event_loop_in_thread):
+        loop, _ = create_asyncio_event_loop_in_thread
+
+        # Instantiate gRPCReplicaResult with FakegRPCStreamCall. This needs
+        # to be run on the "other loop"
+        fut = asyncio.run_coroutine_threadsafe(
+            self.make_fake_streaming_request([1, 2, 3, 4], loop, on_separate_loop=True),
+            loop=loop,
+        )
+        replica_result = fut.result()
+
+        # The async generator should be consumed even if we don't fetch
+        # the items explicitly through the ReplicaResult object
+        wait_for_condition(replica_result._call.is_empty, retry_interval_ms=10)
+
+        # Finally, check results given by gRPCReplicaResult fetched from
+        # the queue are correct
+        assert list(replica_result) == [1, 2, 3, 4]
+
+    @pytest.mark.asyncio
+    async def test_streaming_async(self, create_asyncio_event_loop_in_thread):
+        loop, _ = create_asyncio_event_loop_in_thread
+
+        fut = asyncio.run_coroutine_threadsafe(
+            self.make_fake_streaming_request([1, 2, 3, 4], loop, on_separate_loop=True),
+            loop=loop,
+        )
+        replica_result = fut.result()
+
+        # Check async generator is consumed on its own
+        wait_for_condition(replica_result._call.is_empty, retry_interval_ms=10)
+        assert [r async for r in replica_result] == [1, 2, 3, 4]
+
+    @pytest.mark.asyncio
+    async def test_streaming_blocked(self, create_asyncio_event_loop_in_thread):
+        """Use threading event to block async generator, check everything works"""
+
+        loop, event = create_asyncio_event_loop_in_thread
+
+        fut = asyncio.run_coroutine_threadsafe(
+            self.make_fake_streaming_request(
+                [1, 2, 3, 4], loop, on_separate_loop=True, event=event
+            ),
+            loop=loop,
+        )
+        replica_result = fut.result()
+
+        async def fetch():
+            return [r async for r in replica_result]
+
+        t = asyncio.create_task(fetch())
+
+        with pytest.raises(asyncio.TimeoutError):
+            await asyncio.wait_for(asyncio.shield(t), 0.01)
+
+        event.set()
+        assert await t == [1, 2, 3, 4]
+
+    def test_unary_with_gen_sync(self, create_asyncio_event_loop_in_thread):
+        loop, _ = create_asyncio_event_loop_in_thread
+
+        fut = asyncio.run_coroutine_threadsafe(
+            self.make_fake_streaming_request(
+                ["hello"], loop, on_separate_loop=True, is_streaming=False
+            ),
+            loop=loop,
+        )
+        replica_result = fut.result()
+
+        # Check async generator is consumed on its own
+        wait_for_condition(replica_result._call.is_empty, retry_interval_ms=10)
+        assert replica_result.get(None) == "hello"
+
+    @pytest.mark.asyncio
+    async def test_unary_with_gen_async(self, create_asyncio_event_loop_in_thread):
+        loop, _ = create_asyncio_event_loop_in_thread
+
+        fut = asyncio.run_coroutine_threadsafe(
+            self.make_fake_streaming_request(
+                ["hello"], loop, on_separate_loop=True, is_streaming=False
+            ),
+            loop=loop,
+        )
+        replica_result = fut.result()
+
+        # Check async generator is consumed on its own
+        wait_for_condition(replica_result._call.is_empty, retry_interval_ms=10)
+        assert await replica_result.get_async() == "hello"
+
+    @pytest.mark.asyncio
+    async def test_unary_with_gen_blocked(self, create_asyncio_event_loop_in_thread):
+        """Use threading event to block async generator, check everything works"""
+
+        loop, event = create_asyncio_event_loop_in_thread
+
+        fut = asyncio.run_coroutine_threadsafe(
+            self.make_fake_streaming_request(
+                ["hello"], loop, on_separate_loop=True, event=event
+            ),
+            loop=loop,
+        )
+        replica_result = fut.result()
+
+        t = asyncio.create_task(replica_result.get_async())
+        with pytest.raises(asyncio.TimeoutError):
+            await asyncio.wait_for(asyncio.shield(t), 0.01)
+
+        event.set()
+        assert await t == "hello"
+
+    def test_unary_with_timeout(self, create_asyncio_event_loop_in_thread):
+        """Test get() with timeout."""
+
+        loop, event = create_asyncio_event_loop_in_thread
+
+        fut = asyncio.run_coroutine_threadsafe(
+            self.make_fake_streaming_request(
+                ["hello"], loop, on_separate_loop=True, event=event
+            ),
+            loop=loop,
+        )
+        replica_result = fut.result()
+
+        with pytest.raises(TimeoutError):
+            replica_result.get(timeout_s=0.01)
+
+        event.set()
+        assert replica_result.get(timeout_s=0.01) == "hello"
+
+    def test_unary_error_sync(self, create_asyncio_event_loop_in_thread):
+        """Test error is raised correctly."""
+
+        loop, _ = create_asyncio_event_loop_in_thread
+        fut = asyncio.run_coroutine_threadsafe(
+            self.make_fake_streaming_request(
+                None, loop, on_separate_loop=True, error=RuntimeError("oh no!")
+            ),
+            loop=loop,
+        )
+        replica_result = fut.result()
+
+        with pytest.raises(RuntimeError, match="oh no!"):
+            replica_result.get(None)
+
+    @pytest.mark.asyncio
+    async def test_unary_error_async(self, create_asyncio_event_loop_in_thread):
+        """Test error is raised correctly."""
+
+        loop, _ = create_asyncio_event_loop_in_thread
+        fut = asyncio.run_coroutine_threadsafe(
+            self.make_fake_streaming_request(
+                None, loop, on_separate_loop=True, error=RuntimeError("oh no!")
+            ),
+            loop=loop,
+        )
+        replica_result = fut.result()
+
+        with pytest.raises(RuntimeError, match="oh no!"):
+            await replica_result.get_async()
+
+    def test_streaming_error_sync(self, create_asyncio_event_loop_in_thread):
+        """Test error is raised correctly."""
+
+        loop, _ = create_asyncio_event_loop_in_thread
+        fut = asyncio.run_coroutine_threadsafe(
+            self.make_fake_streaming_request(
+                None, loop, on_separate_loop=True, error=RuntimeError("oh no!")
+            ),
+            loop=loop,
+        )
+        replica_result = fut.result()
+
+        with pytest.raises(RuntimeError, match="oh no!"):
+            replica_result.__next__()
+
+    @pytest.mark.asyncio
+    async def test_streaming_error_async(self, create_asyncio_event_loop_in_thread):
+        """Test error is raised correctly."""
+
+        loop, _ = create_asyncio_event_loop_in_thread
+        fut = asyncio.run_coroutine_threadsafe(
+            self.make_fake_streaming_request(
+                None, loop, on_separate_loop=True, error=RuntimeError("oh no!")
+            ),
+            loop=loop,
+        )
+        replica_result = fut.result()
+
+        with pytest.raises(RuntimeError, match="oh no!"):
+            await replica_result.__anext__()
+
+
+if __name__ == "__main__":
+    sys.exit(pytest.main(["-v", "-s", __file__]))

python/ray/serve/tests/unit/test_grpc_util.py

@@ -0,0 +1,224 @@
+import pickle
+from typing import Callable
+from unittest.mock import Mock
+
+import grpc
+import pytest
+from google.protobuf.any_pb2 import Any as AnyProto
+
+from ray import cloudpickle
+from ray.serve._private.default_impl import add_grpc_address
+from ray.serve._private.grpc_util import (
+    GRPC_MAX_STATUS_DETAILS_LENGTH,
+    _truncate_message,
+    get_grpc_response_status,
+    gRPCGenericServer,
+)
+from ray.serve._private.proxy_request_response import gRPCStreamingType
+from ray.serve._private.test_utils import FakeGrpcContext
+from ray.serve.exceptions import BackPressureError, gRPCStatusError
+from ray.serve.grpc_util import RayServegRPCContext
+
+
+class FakeGrpcServer:
+    def __init__(self):
+        self.address = None
+
+    def add_insecure_port(self, address):
+        self.address = address
+
+
+def fake_service_handler_factory(
+    service_method: str, streaming_type: gRPCStreamingType
+) -> Callable:
+    def foo() -> bytes:
+        return f"{streaming_type.value} call from {service_method}".encode()
+
+    return foo
+
+
+def test_grpc_server():
+    """Test `gRPCGenericServer` did the correct overrides.
+
+    When a add_servicer_to_server function is called on an instance of `gRPCGenericServer`,
+    it correctly overrides `response_serializer` to None, and `unary_unary`,
+    `unary_stream`, `stream_unary`, and `stream_stream` to be generated from the
+    factory function.
+    """
+    service_name = "ray.serve.ServeAPIService"
+    method_name = "ServeRoutes"
+
+    def add_test_servicer_to_server(servicer, server):
+        rpc_method_handlers = {
+            method_name: grpc.unary_unary_rpc_method_handler(
+                servicer.ServeRoutes,
+                request_deserializer=AnyProto.FromString,
+                response_serializer=AnyProto.SerializeToString,
+            ),
+        }
+        generic_handler = grpc.method_handlers_generic_handler(
+            service_name, rpc_method_handlers
+        )
+        server.add_generic_rpc_handlers((generic_handler,))
+
+    grpc_server = gRPCGenericServer(fake_service_handler_factory)
+    dummy_servicer = Mock()
+
+    # Ensure `generic_rpc_handlers` is not populated before calling
+    # the add_servicer_to_server function.
+    assert grpc_server.generic_rpc_handlers == []
+
+    add_test_servicer_to_server(dummy_servicer, grpc_server)
+
+    # `generic_rpc_handlers` should be populated after add_servicer_to_server is called.
+    assert len(grpc_server.generic_rpc_handlers) == 1
+
+    # The populated rpc handler should have the correct service name.
+    rpc_handler = grpc_server.generic_rpc_handlers[0][0]
+    assert rpc_handler.service_name() == service_name
+
+    # The populated method handlers should have the correct response_serializer,
+    # unary_unary, unary_stream, stream_unary, and stream_stream.
+    service_method = f"/{service_name}/{method_name}"
+    method_handlers = rpc_handler._method_handlers.get(service_method)
+    assert method_handlers.response_serializer is None
+    assert (
+        method_handlers.unary_unary()
+        == f"unary_unary call from {service_method}".encode()
+    )
+    assert (
+        method_handlers.unary_stream()
+        == f"unary_stream call from {service_method}".encode()
+    )
+    assert (
+        method_handlers.stream_unary()
+        == f"stream_unary call from {service_method}".encode()
+    )
+    assert (
+        method_handlers.stream_stream()
+        == f"stream_stream call from {service_method}".encode()
+    )
+
+
+def test_ray_serve_grpc_context_serializable():
+    """RayServegRPCContext should be serializable."""
+    context = RayServegRPCContext(FakeGrpcContext())
+    pickled_context = pickle.dumps(context)
+    deserialized_context = pickle.loads(pickled_context)
+    assert deserialized_context.__dict__ == context.__dict__
+
+    cloudpickled_context = cloudpickle.dumps(context)
+    deserialized_context = pickle.loads(cloudpickled_context)
+    assert deserialized_context.__dict__ == context.__dict__
+
+
+def test_add_grpc_address():
+    """Test `add_grpc_address` adds the address to the gRPC server."""
+    fake_grpc_server = FakeGrpcServer()
+    grpc_address = "fake_address:50051"
+    assert fake_grpc_server.address is None
+    add_grpc_address(fake_grpc_server, grpc_address)
+    assert fake_grpc_server.address == grpc_address
+
+
+def test_get_grpc_response_status_backpressure_error():
+    """Test that BackPressureError returns RESOURCE_EXHAUSTED status."""
+    backpressure_error = BackPressureError(
+        num_queued_requests=10, max_queued_requests=5
+    )
+
+    status = get_grpc_response_status(
+        exc=backpressure_error, request_timeout_s=30.0, request_id="test_request_123"
+    )
+
+    assert status.code == grpc.StatusCode.RESOURCE_EXHAUSTED
+    assert status.is_error is True
+    assert status.message == backpressure_error.message
+
+
+def test_get_grpc_response_status_grpc_status_error():
+    """Test that gRPCStatusError preserves user-set status code."""
+    original_error = RuntimeError("test error")
+    user_status_code = grpc.StatusCode.INVALID_ARGUMENT
+    user_details = "Invalid argument provided"
+
+    grpc_status_error = gRPCStatusError(
+        original_exception=original_error,
+        code=user_status_code,
+        details=user_details,
+    )
+
+    status = get_grpc_response_status(
+        exc=grpc_status_error, request_timeout_s=30.0, request_id="test_request_123"
+    )
+
+    assert status.code == user_status_code
+    assert status.is_error is True
+    assert status.message == user_details
+
+
+def test_get_grpc_response_status_grpc_status_error_no_details():
+    """Test that gRPCStatusError without details uses original exception message."""
+    original_error = RuntimeError("original error message")
+    user_status_code = grpc.StatusCode.RESOURCE_EXHAUSTED
+
+    grpc_status_error = gRPCStatusError(
+        original_exception=original_error,
+        code=user_status_code,
+        details=None,
+    )
+
+    status = get_grpc_response_status(
+        exc=grpc_status_error, request_timeout_s=30.0, request_id="test_request_123"
+    )
+
+    assert status.code == user_status_code
+    assert status.is_error is True
+    assert "original error message" in status.message
+
+
+def test_truncate_message_short():
+    """Test that short messages are not truncated."""
+    short_message = "short error message"
+    result = _truncate_message(short_message)
+    assert result == short_message
+
+
+def test_truncate_message_long():
+    """Test that long messages are truncated."""
+    # Create a message longer than the max length
+    long_message = "a" * (GRPC_MAX_STATUS_DETAILS_LENGTH + 1000)
+    result = _truncate_message(long_message)
+
+    assert len(result) <= GRPC_MAX_STATUS_DETAILS_LENGTH
+    assert result.endswith("... [truncated]")
+
+
+def test_truncate_message_at_boundary():
+    """Test truncation at the exact boundary."""
+    # Create a message exactly at the limit
+    exact_message = "a" * GRPC_MAX_STATUS_DETAILS_LENGTH
+    result = _truncate_message(exact_message)
+    assert result == exact_message
+    assert len(result) == GRPC_MAX_STATUS_DETAILS_LENGTH
+
+
+def test_get_grpc_response_status_truncates_long_message():
+    """Test that long error messages are truncated in INTERNAL errors."""
+    long_message = "a" * (GRPC_MAX_STATUS_DETAILS_LENGTH + 1000)
+    long_error = RuntimeError(long_message)
+
+    status = get_grpc_response_status(
+        exc=long_error, request_timeout_s=30.0, request_id="test_request_123"
+    )
+
+    assert status.code == grpc.StatusCode.INTERNAL
+    assert status.is_error is True
+    assert len(status.message) <= GRPC_MAX_STATUS_DETAILS_LENGTH
+    assert status.message.endswith("... [truncated]")
+
+
+if __name__ == "__main__":
+    import sys
+
+    sys.exit(pytest.main(["-v", "-s", __file__]))

python/ray/serve/tests/unit/test_handle_options.py

@@ -0,0 +1,102 @@
+import sys
+
+import pytest
+
+from ray.serve._private.common import DeploymentHandleSource
+from ray.serve._private.handle_options import DynamicHandleOptions, InitHandleOptions
+from ray.serve._private.utils import DEFAULT
+
+
+def test_dynamic_handle_options():
+    default_options = DynamicHandleOptions()
+    assert default_options.method_name == "__call__"
+    assert default_options.multiplexed_model_id == ""
+    assert default_options.session_id == ""
+    assert default_options.stream is False
+
+    # Test setting method name.
+    only_set_method = default_options.copy_and_update(method_name="hi")
+    assert only_set_method.method_name == "hi"
+    assert only_set_method.multiplexed_model_id == ""
+    assert only_set_method.session_id == ""
+    assert only_set_method.stream is False
+
+    # Existing options should be unmodified.
+    assert default_options.method_name == "__call__"
+    assert default_options.multiplexed_model_id == ""
+    assert default_options.session_id == ""
+    assert default_options.stream is False
+
+    # Test setting model ID.
+    only_set_model_id = default_options.copy_and_update(multiplexed_model_id="hi")
+    assert only_set_model_id.method_name == "__call__"
+    assert only_set_model_id.multiplexed_model_id == "hi"
+    assert only_set_model_id.session_id == ""
+    assert only_set_model_id.stream is False
+
+    # Existing options should be unmodified.
+    assert default_options.method_name == "__call__"
+    assert default_options.multiplexed_model_id == ""
+    assert default_options.session_id == ""
+    assert default_options.stream is False
+
+    # Test setting stream.
+    only_set_stream = default_options.copy_and_update(stream=True)
+    assert only_set_stream.method_name == "__call__"
+    assert only_set_stream.multiplexed_model_id == ""
+    assert only_set_stream.session_id == ""
+    assert only_set_stream.stream is True
+
+    # Existing options should be unmodified.
+    assert default_options.method_name == "__call__"
+    assert default_options.multiplexed_model_id == ""
+    assert default_options.session_id == ""
+    assert default_options.stream is False
+
+    # Test setting session ID.
+    only_set_session_id = default_options.copy_and_update(session_id="sess_abc")
+    assert only_set_session_id.method_name == "__call__"
+    assert only_set_session_id.multiplexed_model_id == ""
+    assert only_set_session_id.session_id == "sess_abc"
+    assert only_set_session_id.stream is False
+
+    # Existing options should be unmodified.
+    assert default_options.method_name == "__call__"
+    assert default_options.multiplexed_model_id == ""
+    assert default_options.session_id == ""
+    assert default_options.stream is False
+
+    # Test setting multiple.
+    set_multiple = default_options.copy_and_update(method_name="hi", stream=True)
+    assert set_multiple.method_name == "hi"
+    assert set_multiple.multiplexed_model_id == ""
+    assert set_multiple.session_id == ""
+    assert set_multiple.stream is True
+
+
+def test_init_handle_options():
+    default_options = InitHandleOptions.create()
+    assert default_options._prefer_local_routing is False
+    assert default_options._source == DeploymentHandleSource.UNKNOWN
+
+    default1 = InitHandleOptions.create(_prefer_local_routing=DEFAULT.VALUE)
+    assert default1._prefer_local_routing is False
+    assert default1._source == DeploymentHandleSource.UNKNOWN
+
+    default2 = InitHandleOptions.create(_source=DEFAULT.VALUE)
+    assert default2._prefer_local_routing is False
+    assert default2._source == DeploymentHandleSource.UNKNOWN
+
+    prefer_local = InitHandleOptions.create(
+        _prefer_local_routing=True, _source=DEFAULT.VALUE
+    )
+    assert prefer_local._prefer_local_routing is True
+    assert prefer_local._source == DeploymentHandleSource.UNKNOWN
+
+    proxy_options = InitHandleOptions.create(_source=DeploymentHandleSource.PROXY)
+    assert proxy_options._prefer_local_routing is False
+    assert proxy_options._source == DeploymentHandleSource.PROXY
+
+
+if __name__ == "__main__":
+    sys.exit(pytest.main(["-v", "-s", __file__]))

python/ray/serve/tests/unit/test_haproxy_binary.py

@@ -0,0 +1,103 @@
+"""Unit tests for get_haproxy_binary() resolution logic.
+
+get_haproxy_binary() resolves an HAProxy binary path with this priority:
+  1. Explicit RAY_SERVE_HAPROXY_BINARY_PATH override, validated and returned.
+  2. Bundled binary from the ``ray-haproxy`` PyPI package.
+  3. System ``haproxy`` on PATH.
+  4. FileNotFoundError with an actionable message.
+"""
+
+import stat
+from unittest.mock import MagicMock, patch
+
+import pytest
+
+from ray.serve._private.haproxy import get_haproxy_binary
+
+# Patch targets: module-level constants imported at the top of haproxy.py.
+HAPROXY_MODULE = "ray.serve._private.haproxy"
+BINARY_PATH_PATCH = f"{HAPROXY_MODULE}.RAY_SERVE_HAPROXY_BINARY_PATH"
+WHICH_PATCH = f"{HAPROXY_MODULE}.shutil.which"
+
+
+def _make_executable_binary(tmp_path):
+    """Create an empty executable file under tmp_path and return its path."""
+    binary = tmp_path / "haproxy"
+    binary.write_bytes(b"")
+    binary.chmod(binary.stat().st_mode | stat.S_IXUSR)
+    return binary
+
+
+def test_explicit_path_takes_precedence_over_bundled(tmp_path):
+    """An explicit RAY_SERVE_HAPROXY_BINARY_PATH wins over the bundled
+    ray-haproxy package, so an operator's custom binary is not silently
+    overridden."""
+    binary = _make_executable_binary(tmp_path)
+
+    mock_module = MagicMock()
+    mock_module.get_haproxy_binary.return_value = "/bundled/haproxy"
+    with patch.dict("sys.modules", {"ray_haproxy": mock_module}), patch(
+        BINARY_PATH_PATCH, str(binary)
+    ):
+        assert get_haproxy_binary() == str(binary)
+        mock_module.get_haproxy_binary.assert_not_called()
+
+
+def test_explicit_path_validates_executable(tmp_path):
+    """An explicit RAY_SERVE_HAPROXY_BINARY_PATH is validated before use: it must
+    exist and be executable, otherwise resolution raises."""
+    binary = _make_executable_binary(tmp_path)
+
+    with patch(BINARY_PATH_PATCH, str(binary)):
+        assert get_haproxy_binary() == str(binary)
+
+    # Same file without the execute bit is rejected.
+    binary.chmod(0o644)
+    with patch(BINARY_PATH_PATCH, str(binary)):
+        with pytest.raises(FileNotFoundError, match="HAPROXY_BINARY_PATH"):
+            get_haproxy_binary()
+
+    # A nonexistent path is rejected.
+    with patch(BINARY_PATH_PATCH, str(tmp_path / "no_such_file")):
+        with pytest.raises(FileNotFoundError, match="HAPROXY_BINARY_PATH"):
+            get_haproxy_binary()
+
+
+@patch(BINARY_PATH_PATCH, "")
+def test_bundled_package_used_when_no_explicit_path():
+    """With no explicit path set, the bundled ray-haproxy binary is used."""
+    mock_module = MagicMock()
+    mock_module.get_haproxy_binary.return_value = "/bundled/haproxy"
+    with patch.dict("sys.modules", {"ray_haproxy": mock_module}):
+        assert get_haproxy_binary() == "/bundled/haproxy"
+
+
+@patch(WHICH_PATCH, return_value="/usr/sbin/haproxy")
+@patch(BINARY_PATH_PATCH, "")
+def test_pip_package_oserror_falls_through(_mock_which):
+    """If ray-haproxy is installed but its binary is broken (e.g. missing file,
+    bad permissions), the function should fall through to the system haproxy
+    rather than crashing the proxy actor. We catch OSError (not just
+    FileNotFoundError) so PermissionError is also handled."""
+    mock_module = MagicMock()
+    mock_module.get_haproxy_binary.side_effect = OSError("bad permissions")
+
+    with patch.dict("sys.modules", {"ray_haproxy": mock_module}):
+        assert get_haproxy_binary() == "/usr/sbin/haproxy"
+
+
+@patch(WHICH_PATCH, return_value=None)
+@patch(BINARY_PATH_PATCH, "")
+def test_nothing_available_raises_with_instructions(_mock_which):
+    """When no binary is available from any source, the error message must
+    tell the user how to fix it (install ray[serve], set the env var, or
+    put haproxy on PATH)."""
+    with patch.dict("sys.modules", {"ray_haproxy": None}):
+        with pytest.raises(FileNotFoundError, match=r"ray\[serve\]"):
+            get_haproxy_binary()
+
+
+if __name__ == "__main__":
+    import sys
+
+    sys.exit(pytest.main(["-v", "-s", __file__]))

python/ray/serve/tests/unit/test_haproxy_grpc_healthcheck.py

@@ -0,0 +1,195 @@
+"""Unit tests for build_grpc_healthcheck_request_hex().
+
+The function returns a hex string passed verbatim to HAProxy's
+`tcp-check send-binary`. It must encode a complete, valid HTTP/2 client
+message stream for a unary gRPC `Healthz` call:
+
+  1. H2 connection preface
+  2. empty SETTINGS frame
+  3. HEADERS frame (HPACK, END_HEADERS only) carrying :method, :scheme,
+     :path, :authority, te, content-type
+  4. DATA frame carrying a 5-byte empty gRPC message and END_STREAM
+
+This test verifies all four pieces by parsing the bytes back out.
+"""
+
+from typing import Tuple
+
+import pytest
+
+from ray.serve._private.haproxy import build_grpc_healthcheck_request_hex
+
+H2_PREFACE = b"PRI * HTTP/2.0\r\n\r\nSM\r\n\r\n"
+
+# HTTP/2 frame type codes (RFC 7540 §6).
+FRAME_DATA = 0x0
+FRAME_HEADERS = 0x1
+FRAME_SETTINGS = 0x4
+
+# HTTP/2 frame flags relevant to this test.
+FLAG_END_STREAM = 0x1
+FLAG_END_HEADERS = 0x4
+
+
+def _parse_frame(buf: bytes, offset: int) -> Tuple[int, int, int, bytes, int]:
+    """Parse a single HTTP/2 frame starting at offset.
+
+    Returns (frame_type, flags, stream_id, payload, next_offset).
+    """
+    length = int.from_bytes(buf[offset : offset + 3], "big")
+    frame_type = buf[offset + 3]
+    flags = buf[offset + 4]
+    stream_id = int.from_bytes(buf[offset + 5 : offset + 9], "big") & 0x7FFFFFFF
+    payload = buf[offset + 9 : offset + 9 + length]
+    assert len(payload) == length, "frame truncated"
+    return frame_type, flags, stream_id, payload, offset + 9 + length
+
+
+def _parse_hpack_literals(block: bytes) -> dict:
+    """Parse an HPACK block built by build_grpc_healthcheck_request_hex.
+
+    The producer only emits 0x00-prefixed literal-without-indexing entries
+    with new names, so we don't need a real HPACK decoder. Every entry is:
+      0x00 <name_len> <name bytes> <value_len> <value bytes>
+    All lengths are < 127 (single byte). Returns {name: value}.
+    """
+    out: dict = {}
+    i = 0
+    while i < len(block):
+        assert (
+            block[i] == 0x00
+        ), f"expected literal-no-index marker at {i}, got {block[i]:#x}"
+        i += 1
+        name_len = block[i]
+        assert name_len < 128, "unexpected huffman / long-length encoding"
+        i += 1
+        name = block[i : i + name_len].decode()
+        i += name_len
+        value_len = block[i]
+        assert value_len < 128
+        i += 1
+        value = block[i : i + value_len].decode()
+        i += value_len
+        out[name] = value
+    return out
+
+
+@pytest.fixture
+def healthz_bytes() -> bytes:
+    hex_str = build_grpc_healthcheck_request_hex(
+        "/ray.serve.RayServeAPIService/Healthz"
+    )
+    # Sanity: result is a hex string, decodes cleanly.
+    return bytes.fromhex(hex_str)
+
+
+def test_starts_with_h2_preface(healthz_bytes: bytes):
+    """Every HTTP/2 client connection must start with the 24-byte preface."""
+    assert healthz_bytes.startswith(H2_PREFACE)
+
+
+def test_emits_empty_settings_then_headers_then_data(healthz_bytes: bytes):
+    """Frame sequence must be SETTINGS (empty) → HEADERS → DATA."""
+    offset = len(H2_PREFACE)
+
+    ftype, flags, stream_id, payload, offset = _parse_frame(healthz_bytes, offset)
+    assert ftype == FRAME_SETTINGS
+    assert payload == b"", "client SETTINGS must be empty (no settings sent)"
+    assert stream_id == 0
+
+    ftype, flags, _, headers_payload, offset = _parse_frame(healthz_bytes, offset)
+    assert ftype == FRAME_HEADERS
+
+    ftype, flags, _, data_payload, offset = _parse_frame(healthz_bytes, offset)
+    assert ftype == FRAME_DATA
+
+    # No trailing bytes after the DATA frame.
+    assert offset == len(
+        healthz_bytes
+    ), f"unexpected trailing bytes: {healthz_bytes[offset:]!r}"
+
+
+def test_headers_frame_carries_required_grpc_pseudo_headers(healthz_bytes: bytes):
+    """The Python gRPC server rejects requests missing :authority with a
+    Trailers-Only error before the handler runs. Verify every required
+    pseudo-header (and te/content-type) is present.
+    """
+    offset = len(H2_PREFACE)
+    _, _, _, _, offset = _parse_frame(healthz_bytes, offset)  # skip SETTINGS
+    ftype, flags, stream_id, payload, _ = _parse_frame(healthz_bytes, offset)
+    assert ftype == FRAME_HEADERS
+
+    # HEADERS frame must end the headers block but NOT the stream, since the
+    # gRPC message is still coming in the following DATA frame.
+    assert flags & FLAG_END_HEADERS, "END_HEADERS must be set"
+    assert not (
+        flags & FLAG_END_STREAM
+    ), "END_STREAM on HEADERS would suppress the DATA frame"
+    assert stream_id == 1, "client stream ids start at 1"
+
+    headers = _parse_hpack_literals(payload)
+    assert headers[":method"] == "POST"
+    assert headers[":scheme"] == "http"
+    assert headers[":path"] == "/ray.serve.RayServeAPIService/Healthz"
+    # `:authority` is mandatory; without it, the server returns Trailers-Only
+    # "Missing :authority header" and the check never matches.
+    assert headers[":authority"], ":authority must be present"
+    assert headers["content-type"] == "application/grpc"
+    assert headers["te"] == "trailers"
+
+
+def test_data_frame_is_empty_grpc_message_with_end_stream(healthz_bytes: bytes):
+    """The DATA frame must carry exactly the 5-byte empty gRPC frame
+    (uncompressed flag + 4-byte length=0) so the server's framing layer
+    deserializes an empty HealthzRequest and runs the handler. END_STREAM
+    must be on this frame, not on HEADERS.
+    """
+    offset = len(H2_PREFACE)
+    _, _, _, _, offset = _parse_frame(healthz_bytes, offset)  # SETTINGS
+    _, _, _, _, offset = _parse_frame(healthz_bytes, offset)  # HEADERS
+    ftype, flags, stream_id, payload, _ = _parse_frame(healthz_bytes, offset)
+
+    assert ftype == FRAME_DATA
+    assert stream_id == 1
+    assert flags & FLAG_END_STREAM, "DATA frame must close the request stream"
+    assert (
+        payload == b"\x00\x00\x00\x00\x00"
+    ), f"expected empty gRPC frame, got {payload.hex()}"
+
+
+@pytest.mark.parametrize(
+    "path",
+    [
+        "/ray.serve.RayServeAPIService/Healthz",
+        "/grpc.health.v1.Health/Check",
+        "/a/b",
+    ],
+)
+def test_path_is_threaded_through_to_pseudo_header(path: str):
+    """The :path pseudo-header must reflect the argument verbatim — the
+    template renders one healthcheck per backend, all with the same
+    health_check_path, and rendering the wrong path would silently route
+    the check away from the handler."""
+    raw = bytes.fromhex(build_grpc_healthcheck_request_hex(path))
+    offset = len(H2_PREFACE)
+    _, _, _, _, offset = _parse_frame(raw, offset)  # SETTINGS
+    _, _, _, headers_payload, _ = _parse_frame(raw, offset)
+    headers = _parse_hpack_literals(headers_payload)
+    assert headers[":path"] == path
+
+
+def test_output_is_lowercase_hex_with_no_separators():
+    """`tcp-check send-binary` accepts a hex string with no separators;
+    HAProxy will reject the config if we emit anything else. Guard against
+    accidental formatting changes (uppercase, spaces, 0x-prefixes)."""
+    out = build_grpc_healthcheck_request_hex("/x")
+    assert out == out.lower(), "hex must be lowercase"
+    assert all(
+        c in "0123456789abcdef" for c in out
+    ), f"non-hex character in output: {out!r}"
+
+
+if __name__ == "__main__":
+    import sys
+
+    sys.exit(pytest.main(["-v", "-s", __file__]))

python/ray/serve/tests/unit/test_haproxy_process_manager.py

@@ -0,0 +1,164 @@
+"""Unit tests for HAProxyApi reload takeover verification and -sf signaling.
+
+Regression coverage for the orphaned-worker incident: during a reload the
+admin socket answers through its previous owner, so a socket-only readiness
+check declares a dead spawn ready and strands the worker it was meant to
+stop, which keeps serving its stale config indefinitely.
+"""
+import asyncio
+import os
+import sys
+from typing import Optional
+
+import pytest
+
+from ray.serve._private.haproxy import HAProxyApi, HAProxyConfig
+
+
+class FakeProc:
+    """Minimal stand-in for asyncio.subprocess.Process."""
+
+    def __init__(
+        self,
+        pid: int,
+        returncode: Optional[int] = None,
+        stdout_path: str = "",
+        stderr_path: str = "",
+    ):
+        self.pid = pid
+        self.returncode = returncode
+        self._stdout_path = stdout_path
+        self._stderr_path = stderr_path
+
+
+@pytest.fixture
+def api(tmp_path) -> HAProxyApi:
+    cfg = HAProxyConfig(
+        socket_path=str(tmp_path / "admin.sock"),
+        server_state_base=str(tmp_path),
+        server_state_file=str(tmp_path / "server-state"),
+        enable_hap_optimization=False,
+    )
+    return HAProxyApi(cfg=cfg, config_file_path=str(tmp_path / "haproxy.cfg"))
+
+
+def _make_stream_files(tmp_path, stderr_text: str = ""):
+    stdout = tmp_path / "spawn.stdout.log"
+    stderr = tmp_path / "spawn.stderr.log"
+    stdout.write_text("")
+    stderr.write_text(stderr_text)
+    return str(stdout), str(stderr)
+
+
+def _answer_show_info(api: HAProxyApi, response: str) -> None:
+    async def fake_send(cmd: str) -> str:
+        assert cmd == "show info"
+        return response
+
+    api._send_socket_command = fake_send
+
+
+class TestWaitForHapAvailability:
+    def test_rejects_answer_from_previous_socket_owner(self, api, tmp_path):
+        """Readiness requires the answering pid to be the spawn itself, not
+        whichever previous worker still owns the socket path."""
+        stdout, stderr = _make_stream_files(tmp_path, "fd transfer failed")
+        # The socket answers, but from the OLD worker (pid 111).
+        _answer_show_info(api, "Name: HAProxy\nPid: 111\nUptime: 1d\n")
+        proc = FakeProc(pid=222, stdout_path=stdout, stderr_path=stderr)
+
+        with pytest.raises(RuntimeError, match="did not take over") as exc_info:
+            asyncio.run(api._wait_for_hap_availability(proc, timeout_s=1))
+
+        # The spawn's stderr is surfaced at failure time.
+        assert "fd transfer failed" in str(exc_info.value)
+
+    def test_passes_when_answering_pid_matches(self, api, tmp_path):
+        stdout, stderr = _make_stream_files(tmp_path)
+        _answer_show_info(api, "Name: HAProxy\nPid: 222\nUptime: 0d\n")
+        proc = FakeProc(pid=222, stdout_path=stdout, stderr_path=stderr)
+
+        asyncio.run(api._wait_for_hap_availability(proc, timeout_s=1))
+
+    def test_crashed_spawn_raises_with_stderr(self, api, tmp_path):
+        stdout, stderr = _make_stream_files(tmp_path, "cannot bind socket")
+        _answer_show_info(api, "Name: HAProxy\nPid: 111\n")
+        proc = FakeProc(pid=222, returncode=1, stdout_path=stdout, stderr_path=stderr)
+
+        with pytest.raises(RuntimeError, match="crashed during startup"):
+            asyncio.run(api._wait_for_hap_availability(proc, timeout_s=1))
+
+
+class TestGetRunningPid:
+    @pytest.mark.parametrize(
+        "response,expected",
+        [
+            ("Name: HAProxy\nVersion: 2.8\nPid: 4242\nUptime: 0d\n", 4242),
+            ("Name: HAProxy\nPid: not-a-pid\n", None),
+            ("Name: HAProxy\nVersion: 2.8\n", None),
+        ],
+    )
+    def test_parses_show_info(self, api, response, expected):
+        _answer_show_info(api, response)
+        assert asyncio.run(api._get_running_pid()) == expected
+
+    def test_returns_none_when_socket_unavailable(self, api):
+        async def fake_send(cmd: str) -> str:
+            raise RuntimeError("socket does not exist")
+
+        api._send_socket_command = fake_send
+        assert asyncio.run(api._get_running_pid()) is None
+
+
+class TestIsOurHaproxy:
+    """`_is_our_haproxy` keeps a recycled `-sf` signal from hitting an unrelated
+    process: only pids whose /proc cmdline still carries our config path pass."""
+
+    @pytest.mark.skipif(
+        not sys.platform.startswith("linux"), reason="/proc is Linux-only"
+    )
+    def test_true_when_cmdline_matches(self, api):
+        # Point config_file_path at a real token in this process's cmdline.
+        with open(f"/proc/{os.getpid()}/cmdline", "rb") as f:
+            argv = [t for t in f.read().split(b"\0") if t]
+        api.config_file_path = argv[0].decode()
+        assert api._is_our_haproxy(os.getpid()) is True
+
+    @pytest.mark.skipif(
+        not sys.platform.startswith("linux"), reason="/proc is Linux-only"
+    )
+    def test_false_when_cmdline_does_not_match(self, api):
+        api.config_file_path = "/tmp/not-in-any-cmdline/haproxy.cfg"
+        assert api._is_our_haproxy(os.getpid()) is False
+
+    def test_false_for_nonexistent_pid(self, api):
+        # Missing /proc entry (or any non-Linux platform) -> OSError -> False.
+        api.config_file_path = "/tmp/whatever/haproxy.cfg"
+        assert api._is_our_haproxy(2_000_000_000) is False
+
+
+class TestCountHaproxyProcesses:
+    """`count_haproxy_processes` scans /proc for processes whose cmdline carries
+    our config path, so the count spans live, draining, and leaked workers."""
+
+    @pytest.mark.skipif(
+        not sys.platform.startswith("linux"), reason="/proc is Linux-only"
+    )
+    def test_counts_matching_processes(self, api):
+        # Point config_file_path at a real token in this process's cmdline so at
+        # least this process is counted.
+        with open(f"/proc/{os.getpid()}/cmdline", "rb") as f:
+            argv = [t for t in f.read().split(b"\0") if t]
+        api.config_file_path = argv[0].decode()
+        assert api.count_haproxy_processes() >= 1
+
+    @pytest.mark.skipif(
+        not sys.platform.startswith("linux"), reason="/proc is Linux-only"
+    )
+    def test_returns_zero_when_no_match(self, api):
+        api.config_file_path = "/tmp/not-in-any-cmdline/haproxy.cfg"
+        assert api.count_haproxy_processes() == 0
+
+
+if __name__ == "__main__":
+    sys.exit(pytest.main(["-v", __file__]))

python/ray/serve/tests/unit/test_http_util.py

@@ -0,0 +1,358 @@
+import asyncio
+import pickle
+import sys
+from typing import Generator, Tuple
+from unittest.mock import MagicMock, patch
+
+import pytest
+from starlette.middleware import Middleware
+from starlette.middleware.base import BaseHTTPMiddleware
+
+from ray._common.utils import get_or_create_event_loop
+from ray.serve import HTTPOptions
+from ray.serve._private.http_util import (
+    ASGIReceiveProxy,
+    MessageQueue,
+    configure_http_middlewares,
+    configure_http_options_with_defaults,
+)
+
+
+@pytest.mark.asyncio
+async def test_message_queue_nowait():
+    queue = MessageQueue()
+
+    # Check that wait_for_message hangs until a message is sent.
+    with pytest.raises(asyncio.TimeoutError):
+        await asyncio.wait_for(queue.wait_for_message(), 0.001)
+
+    assert len(list(queue.get_messages_nowait())) == 0
+
+    await queue({"type": "http.response.start"})
+    await queue.wait_for_message()
+    assert len(list(queue.get_messages_nowait())) == 1
+
+    # Check that messages are cleared after being consumed.
+    assert len(list(queue.get_messages_nowait())) == 0
+    with pytest.raises(asyncio.TimeoutError):
+        await asyncio.wait_for(queue.wait_for_message(), 0.001)
+
+    # Check that consecutive messages are returned in order.
+    await queue({"type": "http.response.start", "idx": 0})
+    await queue({"type": "http.response.start", "idx": 1})
+    await queue.wait_for_message()
+    messages = list(queue.get_messages_nowait())
+    assert len(messages) == 2
+    assert messages[0]["idx"] == 0
+    assert messages[1]["idx"] == 1
+
+    assert len(list(queue.get_messages_nowait())) == 0
+    with pytest.raises(asyncio.TimeoutError):
+        await asyncio.wait_for(queue.wait_for_message(), 0.001)
+
+    # Check that a concurrent waiter is notified when a message is available.
+    loop = asyncio.get_running_loop()
+    waiting_task = loop.create_task(queue.wait_for_message())
+    for _ in range(1000):
+        assert not waiting_task.done()
+
+    await queue({"type": "http.response.start"})
+    await waiting_task
+    assert len(list(queue.get_messages_nowait())) == 1
+
+    # Check that once the queue is closed, new messages should be rejected and
+    # ongoing and subsequent calls to wait for messages should return immediately.
+    waiting_task = loop.create_task(queue.wait_for_message())
+    queue.close()
+    await waiting_task  # Ongoing call should return.
+
+    for _ in range(100):
+        with pytest.raises(RuntimeError):
+            await queue({"hello": "world"})
+        await queue.wait_for_message()
+        assert queue.get_messages_nowait() == []
+
+
+@pytest.mark.asyncio
+async def test_message_queue_wait():
+    queue = MessageQueue()
+
+    with pytest.raises(asyncio.TimeoutError):
+        await asyncio.wait_for(queue.get_one_message(), 0.001)
+
+    queue.put_nowait("A")
+    assert await queue.get_one_message() == "A"
+
+    # Check that messages are cleared after being consumed.
+    with pytest.raises(asyncio.TimeoutError):
+        await asyncio.wait_for(queue.get_one_message(), 0.001)
+
+    # Check that consecutive messages are returned in order.
+    queue.put_nowait("B")
+    queue.put_nowait("C")
+    assert await queue.get_one_message() == "B"
+    assert await queue.get_one_message() == "C"
+
+    # Check that messages are cleared after being consumed.
+    with pytest.raises(asyncio.TimeoutError):
+        await asyncio.wait_for(queue.get_one_message(), 0.001)
+
+    # Check that a concurrent waiter is notified when a message is available.
+    loop = asyncio.get_running_loop()
+    fetch_task = loop.create_task(queue.get_one_message())
+    for _ in range(1000):
+        assert not fetch_task.done()
+    queue.put_nowait("D")
+    assert await fetch_task == "D"
+
+
+@pytest.mark.asyncio
+async def test_message_queue_wait_closed():
+    queue = MessageQueue()
+
+    queue.put_nowait("A")
+    assert await queue.get_one_message() == "A"
+
+    # Check that once the queue is closed, ongoing and subsequent calls
+    # to get_one_message should raise an exception
+    loop = asyncio.get_running_loop()
+    fetch_task = loop.create_task(queue.get_one_message())
+    queue.close()
+    with pytest.raises(StopAsyncIteration):
+        await fetch_task
+
+    for _ in range(10):
+        with pytest.raises(StopAsyncIteration):
+            await queue.get_one_message()
+
+
+@pytest.mark.asyncio
+async def test_message_queue_wait_error():
+    queue = MessageQueue()
+
+    queue.put_nowait("A")
+    assert await queue.get_one_message() == "A"
+
+    # Check setting an error
+    loop = asyncio.get_running_loop()
+    fetch_task = loop.create_task(queue.get_one_message())
+    queue.set_error(TypeError("uh oh! something went wrong."))
+    with pytest.raises(TypeError, match="uh oh! something went wrong"):
+        await fetch_task
+
+    for _ in range(10):
+        with pytest.raises(TypeError, match="uh oh! something went wrong"):
+            await queue.get_one_message()
+
+
+@pytest.fixture
+@pytest.mark.asyncio
+def setup_receive_proxy(
+    request,
+) -> Generator[Tuple[ASGIReceiveProxy, MessageQueue], None, None]:
+    # Param can be 'http' (default) or 'websocket' (ASGI scope type).
+    type = getattr(request, "param", "http")
+
+    queue = MessageQueue()
+
+    async def receive_asgi_messages(request_id: str) -> bytes:
+        await queue.wait_for_message()
+        messages = queue.get_messages_nowait()
+        for message in messages:
+            if isinstance(message, Exception):
+                raise message
+
+        return pickle.dumps(messages)
+
+    loop = get_or_create_event_loop()
+    asgi_receive_proxy = ASGIReceiveProxy({"type": type}, "", receive_asgi_messages)
+    receiver_task = loop.create_task(asgi_receive_proxy.fetch_until_disconnect())
+    try:
+        yield asgi_receive_proxy, queue
+    except Exception:
+        receiver_task.cancel()
+
+
+@pytest.mark.asyncio
+class TestASGIReceiveProxy:
+    async def test_basic(
+        self, setup_receive_proxy: Tuple[ASGIReceiveProxy, MessageQueue]
+    ):
+        asgi_receive_proxy, queue = setup_receive_proxy
+
+        queue.put_nowait({"type": "foo"})
+        queue.put_nowait({"type": "bar"})
+        assert await asgi_receive_proxy() == {"type": "foo"}
+        assert await asgi_receive_proxy() == {"type": "bar"}
+
+        assert asgi_receive_proxy._queue.empty()
+
+        # Once disconnect is received, it should be returned repeatedly.
+        queue.put_nowait({"type": "http.disconnect"})
+        for _ in range(100):
+            assert await asgi_receive_proxy() == {"type": "http.disconnect"}
+
+        # Subsequent messages should be ignored.
+        queue.put_nowait({"type": "baz"})
+        assert await asgi_receive_proxy() == {"type": "http.disconnect"}
+
+    async def test_raises_exception(
+        self, setup_receive_proxy: Tuple[ASGIReceiveProxy, MessageQueue]
+    ):
+        asgi_receive_proxy, queue = setup_receive_proxy
+
+        queue.put_nowait({"type": "foo"})
+        queue.put_nowait({"type": "bar"})
+        assert await asgi_receive_proxy() == {"type": "foo"}
+        assert await asgi_receive_proxy() == {"type": "bar"}
+
+        queue.put_nowait(RuntimeError("oopsies"))
+        with pytest.raises(RuntimeError, match="oopsies"):
+            await asgi_receive_proxy()
+
+    @pytest.mark.parametrize(
+        "setup_receive_proxy",
+        ["http", "websocket"],
+        indirect=True,
+    )
+    async def test_return_disconnect_on_key_error(
+        self, setup_receive_proxy: Tuple[ASGIReceiveProxy, MessageQueue]
+    ):
+        """If the proxy is no longer handling a given request, it raises a KeyError.
+
+        In these cases, the ASGI receive proxy should return a disconnect message.
+
+        See https://github.com/ray-project/ray/pull/44647 for details.
+        """
+        asgi_receive_proxy, queue = setup_receive_proxy
+
+        queue.put_nowait({"type": "foo"})
+        queue.put_nowait({"type": "bar"})
+        assert await asgi_receive_proxy() == {"type": "foo"}
+        assert await asgi_receive_proxy() == {"type": "bar"}
+
+        queue.put_nowait(KeyError("not found"))
+        for _ in range(100):
+            if asgi_receive_proxy._type == "http":
+                assert await asgi_receive_proxy() == {"type": "http.disconnect"}
+            else:
+                assert await asgi_receive_proxy() == {
+                    "type": "websocket.disconnect",
+                    "code": 1005,
+                }
+
+    async def test_receive_asgi_messages_raises(self):
+        async def receive_asgi_messages(request_id: str) -> bytes:
+            raise RuntimeError("maybe actor crashed")
+
+        loop = get_or_create_event_loop()
+        asgi_receive_proxy = ASGIReceiveProxy(
+            {"type": "http"}, "", receive_asgi_messages
+        )
+        receiver_task = loop.create_task(asgi_receive_proxy.fetch_until_disconnect())
+
+        try:
+            with pytest.raises(RuntimeError, match="maybe actor crashed"):
+                await asgi_receive_proxy()
+        finally:
+            receiver_task.cancel()
+
+
+class MockMiddleware:
+    """Mock middleware class for testing."""
+
+    def __init__(self, name):
+        self.name = name
+
+    def __eq__(self, other):
+        return isinstance(other, MockMiddleware) and self.name == other.name
+
+    def __repr__(self):
+        return f"MockMiddleware({self.name})"
+
+
+@pytest.fixture
+def base_http_options():
+    """Provides basic HTTPOptions for testing."""
+    return HTTPOptions(
+        host="0.0.0.0",
+        port=8000,
+        request_timeout_s=30.0,
+        keep_alive_timeout_s=5.0,
+        middlewares=[],
+    )
+
+
+class TestConfigureHttpOptionsWithDefaults:
+    """Test suite for configure_http_options_with_defaults function."""
+
+    def test_basic_configuration(self, base_http_options):
+        """Test basic configuration preserves settings."""
+        result = configure_http_options_with_defaults(base_http_options)
+
+        # Request timeout should be preserved
+        assert result.request_timeout_s == 30.0
+        # Keep alive timeout should be preserved (no env override)
+        assert result.keep_alive_timeout_s == 5.0
+        # Should initialize middlewares list
+        assert result.middlewares == []
+        # Original should not be modified
+        assert base_http_options.request_timeout_s == 30.0
+
+    @patch("ray.serve._private.http_util.call_function_from_import_path")
+    @patch(
+        "ray.serve._private.http_util.RAY_SERVE_HTTP_PROXY_CALLBACK_IMPORT_PATH",
+        "my.module.callback",
+    )
+    def test_callback_middleware_injection(self, mock_call_function, base_http_options):
+        """Test that the callback middleware is injected correctly."""
+
+        # Arrange: Create a valid middleware by wrapping it with Starlette's Middleware class
+        class CustomMiddleware(BaseHTTPMiddleware):
+            async def dispatch(self, request, call_next):
+                response = await call_next(request)  # Simply pass the request through
+                return response
+
+        # Mock the app argument
+        mock_app = MagicMock()
+
+        wrapped_middleware = Middleware(CustomMiddleware, app=mock_app)
+        mock_call_function.return_value = [
+            wrapped_middleware
+        ]  # Return list of wrapped middleware
+
+        # Act
+        result = configure_http_middlewares(base_http_options)
+
+        # Assert
+        mock_call_function.assert_called_once_with(
+            "my.module.callback"
+        )  # Verify callback execution
+        assert len(result.middlewares) == 1  # Ensure one middleware was injected
+        assert isinstance(result.middlewares[0], Middleware)
+
+    def test_callback_middleware_disabled(self, base_http_options):
+        """Test that callback middleware is not loaded when disabled."""
+        with patch(
+            "ray.serve._private.http_util.RAY_SERVE_HTTP_PROXY_CALLBACK_IMPORT_PATH",
+            "",
+        ):
+            result = configure_http_options_with_defaults(base_http_options)
+
+            # Assert that no callback middleware is added
+            assert result.middlewares == []
+
+    def test_deep_copy_behavior(self, base_http_options):
+        """Test that an original HTTPOptions object is not modified."""
+        original_timeout = base_http_options.request_timeout_s
+
+        result = configure_http_options_with_defaults(base_http_options)
+
+        # Original should remain unchanged
+        assert base_http_options.request_timeout_s == original_timeout
+        # Result should be a different object
+        assert result is not base_http_options
+
+
+if __name__ == "__main__":
+    sys.exit(pytest.main(["-v", "-s", __file__]))

python/ray/serve/tests/unit/test_llm_imports.py

@@ -0,0 +1,37 @@
+import sys
+
+import pytest
+
+# skip the test if vllm is installed
+HAS_VLLM = False
+try:
+    import vllm  # noqa: F401
+
+    HAS_VLLM = True
+except ImportError:
+    pass
+
+
+@pytest.mark.skipif(not HAS_VLLM, reason="vllm is not installed")
+def test_serve_llm_import_does_not_error():
+    # expected ImportError because of missing
+    # dependencies without ray[llm] dependencies
+    with pytest.raises(ImportError):
+        import ray.serve.llm  # noqa: F401
+    with pytest.raises(ImportError):
+        from ray.serve.llm import (
+            LLMConfig,  # noqa: F401
+        )
+    with pytest.raises(ImportError):
+        from ray.serve.llm.deployment import (
+            LLMServer,  # noqa: F401
+        )
+    with pytest.raises(ImportError):
+        from ray.serve.llm import (
+            build_llm_deployment,  # noqa: F401
+            build_openai_app,  # noqa: F401
+        )
+
+
+if __name__ == "__main__":
+    sys.exit(pytest.main(["-v", __file__]))

python/ray/serve/tests/unit/test_local_testing_mode.py

@@ -0,0 +1,246 @@
+import logging
+import os
+import re
+import sys
+
+import pytest
+
+from ray import serve
+from ray.serve._private.constants import SERVE_LOGGER_NAME
+from ray.serve.exceptions import RayServeException
+from ray.serve.handle import DeploymentHandle
+
+
+def test_basic_composition():
+    @serve.deployment
+    class Inner:
+        def __init__(self, my_name: str):
+            self._my_name = my_name
+
+        def __call__(self):
+            return self._my_name
+
+    @serve.deployment
+    class Outer:
+        def __init__(self, my_name: str, inner_handle: DeploymentHandle):
+            assert isinstance(inner_handle, DeploymentHandle)
+
+            self._my_name = my_name
+            self._inner_handle = inner_handle
+
+        async def __call__(self, name: str):
+            inner_name = await self._inner_handle.remote()
+            return f"Hello {name} from {self._my_name} and {inner_name}!"
+
+    h = serve.run(Outer.bind("Theodore", Inner.bind("Kevin")), _local_testing_mode=True)
+    assert isinstance(h, DeploymentHandle)
+    assert h.remote("Edith").result() == "Hello Edith from Theodore and Kevin!"
+
+
+@pytest.mark.parametrize("deployment", ["Inner", "Outer"])
+def test_exception_raised_in_constructor(deployment: str):
+    @serve.deployment
+    class Inner:
+        def __init__(self, should_raise: bool):
+            if should_raise:
+                raise RuntimeError("Exception in Inner constructor.")
+
+    @serve.deployment
+    class Outer:
+        def __init__(self, h: DeploymentHandle, should_raise: bool):
+            if should_raise:
+                raise RuntimeError("Exception in Outer constructor.")
+
+    with pytest.raises(RuntimeError, match=f"Exception in {deployment} constructor."):
+        serve.run(
+            Outer.bind(Inner.bind(deployment == "Inner"), deployment == "Outer"),
+            _local_testing_mode=True,
+        )
+
+
+def test_to_object_ref_error_message():
+    def _get_error_match(by_reference: bool) -> str:
+        if by_reference:
+            return (
+                "Converting DeploymentResponses to ObjectRefs "
+                "is not supported in local testing mode."
+            )
+        else:
+            return re.escape(
+                "Converting by-value DeploymentResponses to ObjectRefs is not supported. "
+                "Use handle.options(_by_reference=True) to enable it."
+            )
+
+    @serve.deployment
+    class Inner:
+        pass
+
+    @serve.deployment
+    class Outer:
+        def __init__(self, h: DeploymentHandle):
+            self._h = h
+
+        async def __call__(self):
+            match = _get_error_match(self._h.handle_options._by_reference)
+            with pytest.raises(
+                RuntimeError,
+                match=match,
+            ):
+                await self._h.remote()._to_object_ref()
+
+    h = serve.run(Outer.bind(Inner.bind()), _local_testing_mode=True)
+    match = _get_error_match(h.handle_options._by_reference)
+    with pytest.raises(
+        RuntimeError,
+        match=match,
+    ):
+        h.remote()._to_object_ref_sync()
+
+    # Test the inner handle case (this would raise if it failed).
+    h.remote().result()
+
+
+def test_dictionary_logging_config_with_local_mode():
+    """Test that the logging config can be passed as a dictionary.
+
+    See: https://github.com/ray-project/ray/issues/50052
+    """
+
+    @serve.deployment
+    class MyApp:
+        def __call__(self):
+            logger = logging.getLogger(SERVE_LOGGER_NAME)
+            return logger.level
+
+    app = MyApp.bind()
+    logging_config = {"log_level": "WARNING"}
+
+    # This should not raise exception.
+    h = serve.run(app, logging_config=logging_config, _local_testing_mode=True)
+
+    # The logger should be setup with WARNING level.
+    assert h.remote().result() == logging.WARNING
+
+
+def test_ingress_request_router_requires_haproxy(monkeypatch):
+    @serve.deployment
+    class LLMServer:
+        pass
+
+    @serve.deployment
+    class IngressRequestRouter:
+        pass
+
+    monkeypatch.setattr("ray.serve._private.build_app.RAY_SERVE_ENABLE_HA_PROXY", False)
+
+    with pytest.raises(
+        RayServeException,
+        match="Ray controller's environment",
+    ):
+        llm_server = LLMServer.bind()
+        app = llm_server._with_ingress_request_router(
+            IngressRequestRouter.bind(llm_deployment=llm_server)
+        )
+        serve.run(
+            app,
+            _local_testing_mode=True,
+        )
+
+
+def test_deploy_multiple_apps_batched() -> None:
+    @serve.deployment
+    class A:
+        def __call__(self):
+            return "a"
+
+    @serve.deployment
+    class B:
+        def __call__(self):
+            return "b"
+
+    a, b = serve.run_many(
+        [
+            serve.RunTarget(A.bind(), name="a", route_prefix="/a"),
+            serve.RunTarget(B.bind(), name="b", route_prefix="/b"),
+        ],
+        _local_testing_mode=True,
+    )
+
+    assert a.remote().result() == "a"
+    assert b.remote().result() == "b"
+
+
+def test_redeploy_multiple_apps_batched() -> None:
+    @serve.deployment
+    class A:
+        def __call__(self):
+            return "a", os.getpid()
+
+    @serve.deployment
+    class V1:
+        def __call__(self):
+            return "version 1", os.getpid()
+
+    @serve.deployment
+    class V2:
+        def __call__(self):
+            return "version 2", os.getpid()
+
+    a_handle, v1_handle = serve.run_many(
+        [
+            serve.RunTarget(A.bind(), name="a", route_prefix="/a"),
+            serve.RunTarget(V1.bind(), name="v", route_prefix="/v"),
+        ],
+        _local_testing_mode=True,
+    )
+
+    a1, pida1 = a_handle.remote().result()
+
+    assert a1 == "a"
+
+    v1, pid1 = v1_handle.remote().result()
+
+    assert v1 == "version 1"
+
+    (v2_handle,) = serve.run_many(
+        [
+            serve.RunTarget(V2.bind(), name="v", route_prefix="/v"),
+        ],
+        _local_testing_mode=True,
+    )
+
+    v2, pid2 = v2_handle.remote().result()
+
+    assert v2 == "version 2"
+    assert pid1 == pid2  # because local testing mode, so it's all in-process
+
+    # Redeploying "v" should not have affected "a"
+    a2, pida2 = a_handle.remote().result()
+
+    assert a1 == a2
+    assert pida1 == pida2
+
+
+def test_reconfigure():
+    @serve.deployment
+    class Hello:
+        def __init__(self):
+            self.user_config = {"name": "Bob"}
+
+        async def reconfigure(self, user_config) -> None:
+            print("Reconfiguring...")
+            self.user_config = user_config
+
+        def __call__(self):
+            name = self.user_config["name"]
+            return f"Hi {name}!"
+
+    h = serve.run(
+        Hello.options(user_config={"name": "Charles"}).bind(), _local_testing_mode=True
+    )
+    assert isinstance(h, DeploymentHandle)
+    assert h.remote().result() == "Hi Charles!"
+
+
+if __name__ == "__main__":
+    sys.exit(pytest.main(["-v", "-s", __file__]))

python/ray/serve/tests/unit/test_node_port_manager.py

@@ -0,0 +1,356 @@
+import sys
+from typing import Set
+
+import pytest
+
+from ray._common.test_utils import wait_for_condition
+from ray.serve._private.common import RequestProtocol
+from ray.serve._private.node_port_manager import NoAvailablePortError, NodePortManager
+
+
+@pytest.fixture
+def port_range_constants():
+    """Fixture to set up port range constants for testing."""
+    return {
+        "RAY_SERVE_DIRECT_INGRESS_MIN_HTTP_PORT": 8000,
+        "RAY_SERVE_DIRECT_INGRESS_MAX_HTTP_PORT": 8005,
+        "RAY_SERVE_DIRECT_INGRESS_MIN_GRPC_PORT": 9000,
+        "RAY_SERVE_DIRECT_INGRESS_MAX_GRPC_PORT": 9005,
+    }
+
+
+@pytest.fixture(autouse=True)
+def setup_port_constants(monkeypatch, port_range_constants):
+    monkeypatch.setattr(
+        "ray.serve._private.node_port_manager.RAY_SERVE_DIRECT_INGRESS_MIN_HTTP_PORT",
+        port_range_constants["RAY_SERVE_DIRECT_INGRESS_MIN_HTTP_PORT"],
+    )
+    monkeypatch.setattr(
+        "ray.serve._private.node_port_manager.RAY_SERVE_DIRECT_INGRESS_MAX_HTTP_PORT",
+        port_range_constants["RAY_SERVE_DIRECT_INGRESS_MAX_HTTP_PORT"],
+    )
+    monkeypatch.setattr(
+        "ray.serve._private.node_port_manager.RAY_SERVE_DIRECT_INGRESS_MIN_GRPC_PORT",
+        port_range_constants["RAY_SERVE_DIRECT_INGRESS_MIN_GRPC_PORT"],
+    )
+    monkeypatch.setattr(
+        "ray.serve._private.node_port_manager.RAY_SERVE_DIRECT_INGRESS_MAX_GRPC_PORT",
+        port_range_constants["RAY_SERVE_DIRECT_INGRESS_MAX_GRPC_PORT"],
+    )
+    # Disable quarantine so existing tests keep their immediate-reuse semantics.
+    monkeypatch.setattr(
+        "ray.serve._private.node_port_manager.RAY_SERVE_PORT_QUARANTINE_S",
+        0.0,
+    )
+    yield
+    # _node_managers is class-level and persists across tests; reset for isolation.
+    NodePortManager._node_managers.clear()
+
+
+def test_http_port_allocation_and_release(port_range_constants):
+    manager = NodePortManager.get_node_manager("node-1")
+    port = manager.allocate_port("replica-1", RequestProtocol.HTTP)
+    assert (
+        port_range_constants["RAY_SERVE_DIRECT_INGRESS_MIN_HTTP_PORT"]
+        <= port
+        < port_range_constants["RAY_SERVE_DIRECT_INGRESS_MAX_HTTP_PORT"]
+    )
+
+    # Port should be reusable after release
+    manager.release_port("replica-1", port, RequestProtocol.HTTP)
+    port2 = manager.allocate_port("replica-2", RequestProtocol.HTTP)
+    assert port2 == port
+
+
+def test_grpc_port_allocation_and_blocking():
+    manager = NodePortManager.get_node_manager("node-2")
+    port = manager.allocate_port("replica-a", RequestProtocol.GRPC)
+    manager.release_port("replica-a", port, RequestProtocol.GRPC, block_port=True)
+
+    # New allocation should skip blocked port
+    allocated_ports = set()
+    for i in range(4):
+        new_port = manager.allocate_port(f"replica-b{i}", RequestProtocol.GRPC)
+        assert new_port != port
+        allocated_ports.add(new_port)
+
+    assert len(allocated_ports) == 4
+
+
+def test_reuse_existing_port_if_already_allocated():
+    manager = NodePortManager.get_node_manager("node-3")
+    port1 = manager.allocate_port("replica-x", RequestProtocol.HTTP)
+    port2 = manager.allocate_port("replica-x", RequestProtocol.HTTP)
+    assert port1 == port2
+
+
+def test_cleanup_releases_ports():
+    manager = NodePortManager.get_node_manager("node-4")
+    allocated = set()
+
+    # Allocate all HTTP ports
+    for i in range(5):
+        port = manager.allocate_port(f"replica-{i}", RequestProtocol.HTTP)
+        allocated.add(port)
+
+    # Only keep some replicas active
+    active_replica_ids: Set[str] = {"replica-0", "replica-1"}
+    NodePortManager.prune({"node-4": active_replica_ids})
+
+    # We should be able to reallocate some of the cleaned-up ports
+    new_port = manager.allocate_port("replica-new", RequestProtocol.HTTP)
+    assert new_port in allocated - {
+        manager.get_port("replica-0", RequestProtocol.HTTP),
+        manager.get_port("replica-1", RequestProtocol.HTTP),
+    }
+
+
+def test_node_manager_cleanup():
+    NodePortManager.get_node_manager("node-5")
+    assert "node-5" in NodePortManager._node_managers
+
+    NodePortManager.prune(node_id_to_alive_replica_ids={})
+    assert "node-5" not in NodePortManager._node_managers
+
+
+def test_port_exhaustion():
+    """Test behavior when all ports are exhausted."""
+    manager = NodePortManager.get_node_manager("node-6")
+
+    # Allocate all HTTP ports
+    allocated_ports = set()
+    for i in range(5):  # 5 ports available (8000-8004)
+        port = manager.allocate_port(f"replica-{i}", RequestProtocol.HTTP)
+        allocated_ports.add(port)
+
+    # Try to allocate one more port
+    with pytest.raises(NoAvailablePortError, match="No available"):
+        manager.allocate_port("replica-extra", RequestProtocol.HTTP)
+
+
+def test_invalid_port_release():
+    """Test error handling when releasing invalid ports."""
+    manager = NodePortManager.get_node_manager("node-7")
+
+    # Allocate a port
+    port = manager.allocate_port("replica-1", RequestProtocol.HTTP)
+
+    # Try to release with wrong port number
+    with pytest.raises(AssertionError, match="port mismatch"):
+        manager.release_port("replica-1", port + 1, RequestProtocol.HTTP)
+
+
+def test_concurrent_port_allocation():
+    """Test concurrent port allocation from multiple coroutines."""
+    manager = NodePortManager.get_node_manager("node-8")
+
+    def allocate_port(replica_id: str):
+        return manager.allocate_port(replica_id, RequestProtocol.HTTP)
+
+    # Allocate ports concurrently
+    ports = [allocate_port(f"replica-{i}") for i in range(5)]
+
+    # Verify all ports are unique and in range
+    assert len(set(ports)) == len(ports)
+    assert all(8000 <= port < 8005 for port in ports)
+
+
+def test_mixed_protocol_port_allocation():
+    """Test allocating both HTTP and gRPC ports for the same replica."""
+    manager = NodePortManager.get_node_manager("node-9")
+
+    http_port = manager.allocate_port("replica-1", RequestProtocol.HTTP)
+    grpc_port = manager.allocate_port("replica-1", RequestProtocol.GRPC)
+
+    assert 8000 <= http_port < 8005
+    assert 9000 <= grpc_port < 9005
+    assert http_port != grpc_port
+
+
+def test_port_blocking_persistence():
+    """Test that blocked ports remain blocked across multiple allocations."""
+    manager = NodePortManager.get_node_manager("node-10")
+
+    # Block a port
+    port = manager.allocate_port("replica-1", RequestProtocol.HTTP)
+    manager.release_port("replica-1", port, RequestProtocol.HTTP, block_port=True)
+
+    # Try to allocate ports multiple times
+    for _ in range(3):
+        new_port = manager.allocate_port("replica-2", RequestProtocol.HTTP)
+        assert new_port != port
+        manager.release_port("replica-2", new_port, RequestProtocol.HTTP)
+
+
+def test_check_replica_port_allocated():
+    """Test the check_replica_port_allocated method."""
+    manager = NodePortManager.get_node_manager("node-11")
+
+    # Test case where port is allocated
+    port = manager.allocate_port("replica-1", RequestProtocol.HTTP)
+    assert manager.get_port("replica-1", RequestProtocol.HTTP) == port
+
+    # Test case where port is not allocated
+    with pytest.raises(
+        ValueError,
+        match="HTTP port not allocated for replica non-existent on node node-11",
+    ):
+        manager.get_port("non-existent", RequestProtocol.HTTP)
+
+    # Clean up
+    manager.release_port("replica-1", port, RequestProtocol.HTTP)
+
+
+def test_quarantine_holds_released_port(monkeypatch, port_range_constants):
+    """A released port is held in quarantine and not immediately reusable."""
+    monkeypatch.setattr(
+        "ray.serve._private.node_port_manager.RAY_SERVE_PORT_QUARANTINE_S",
+        60.0,
+    )
+    manager = NodePortManager.get_node_manager("node-quarantine-1")
+
+    first = manager.allocate_port("replica-q1", RequestProtocol.HTTP)
+    manager.release_port("replica-q1", first, RequestProtocol.HTTP)
+
+    next_port = manager.allocate_port("replica-q2", RequestProtocol.HTTP)
+    assert next_port != first
+    assert first not in manager._http_allocator._available_ports
+    assert first in manager._http_allocator._quarantined_ports
+
+
+def test_quarantine_expiry_returns_port_to_pool(monkeypatch, port_range_constants):
+    """Once a quarantined port's timer expires, it becomes available again."""
+    import time
+
+    monkeypatch.setattr(
+        "ray.serve._private.node_port_manager.RAY_SERVE_PORT_QUARANTINE_S",
+        0.05,
+    )
+    manager = NodePortManager.get_node_manager("node-quarantine-2")
+
+    first = manager.allocate_port("replica-q1", RequestProtocol.HTTP)
+    manager.release_port("replica-q1", first, RequestProtocol.HTTP)
+    assert first in manager._http_allocator._quarantined_ports
+
+    time.sleep(0.1)
+    # Drain explicitly so we don't rely on the allocate-side drain firing.
+    manager._http_allocator._drain_expired_quarantine()
+    assert first not in manager._http_allocator._quarantined_ports
+    assert first in manager._http_allocator._available_ports
+
+
+def test_quarantine_skipped_for_block_port(monkeypatch, port_range_constants):
+    """block_port=True bypasses quarantine."""
+    monkeypatch.setattr(
+        "ray.serve._private.node_port_manager.RAY_SERVE_PORT_QUARANTINE_S",
+        60.0,
+    )
+    manager = NodePortManager.get_node_manager("node-quarantine-3")
+    port = manager.allocate_port("replica-q", RequestProtocol.HTTP)
+    manager.release_port("replica-q", port, RequestProtocol.HTTP, block_port=True)
+
+    assert port in manager._http_allocator._blocked_ports
+    assert port not in manager._http_allocator._quarantined_ports
+
+
+def test_quarantine_disabled_when_zero(monkeypatch, port_range_constants):
+    """Quarantine at 0 preserves immediate-reuse behavior."""
+    monkeypatch.setattr(
+        "ray.serve._private.node_port_manager.RAY_SERVE_PORT_QUARANTINE_S",
+        0.0,
+    )
+    manager = NodePortManager.get_node_manager("node-quarantine-4")
+    port = manager.allocate_port("replica-q1", RequestProtocol.HTTP)
+    manager.release_port("replica-q1", port, RequestProtocol.HTTP)
+    next_port = manager.allocate_port("replica-q2", RequestProtocol.HTTP)
+    assert next_port == port
+
+
+def test_allocate_skips_recovered_port(port_range_constants):
+    """update_port_if_missing recovers a port but leaves it in the heap;
+    a subsequent allocate must not hand the same port to another replica."""
+    manager = NodePortManager.get_node_manager("node-recover")
+    # Recover a replica at the smallest port in the range — heappop would
+    # otherwise return it first.
+    recovered = port_range_constants["RAY_SERVE_DIRECT_INGRESS_MIN_HTTP_PORT"]
+    manager._http_allocator.update_port_if_missing("replica-r", recovered)
+
+    next_port = manager.allocate_port("replica-new", RequestProtocol.HTTP)
+    assert next_port != recovered
+
+
+def test_allocate_skips_quarantined_recovered_port(monkeypatch, port_range_constants):
+    """A port recovered via update_port_if_missing stays in the heap; once
+    released into quarantine it must not be handed out again until the
+    quarantine expires, even though it's still in _available_ports."""
+    monkeypatch.setattr(
+        "ray.serve._private.node_port_manager.RAY_SERVE_PORT_QUARANTINE_S",
+        60.0,
+    )
+    manager = NodePortManager.get_node_manager("node-recover-quarantine")
+    alloc = manager._http_allocator
+    # Recover at the smallest port so heappop would otherwise return it first.
+    recovered = port_range_constants["RAY_SERVE_DIRECT_INGRESS_MIN_HTTP_PORT"]
+    alloc.update_port_if_missing("replica-r", recovered)
+
+    # Release it: goes into quarantine but is still in the heap (recovery
+    # never popped it).
+    manager.release_port("replica-r", recovered, RequestProtocol.HTTP)
+    assert recovered in alloc._quarantined_ports
+    assert recovered in alloc._available_ports  # the invariant-violating state
+
+    # allocate() must not hand back the still-quarantined port.
+    next_port = manager.allocate_port("replica-new", RequestProtocol.HTTP)
+    assert next_port != recovered
+    assert recovered in alloc._quarantined_ports  # still held
+
+
+def test_prune_keeps_manager_while_port_quarantined(monkeypatch, port_range_constants):
+    """A node losing its last replica must NOT have its manager (and quarantine
+    state) dropped while a freed port is still quarantined, and the quarantined
+    port must not be handed to the next replica scheduled there."""
+    monkeypatch.setattr(
+        "ray.serve._private.node_port_manager.RAY_SERVE_PORT_QUARANTINE_S",
+        60.0,
+    )
+    node_id = "node-prune-quarantine"
+    manager = NodePortManager.get_node_manager(node_id)
+
+    port = manager.allocate_port("replica-old", RequestProtocol.HTTP)
+    manager.release_port("replica-old", port, RequestProtocol.HTTP)
+    assert port in manager._http_allocator._quarantined_ports
+
+    # Node now has zero alive replicas; the manager must survive the prune.
+    NodePortManager.prune(node_id_to_alive_replica_ids={})
+    assert node_id in NodePortManager._node_managers
+    assert manager.has_pending_quarantine()
+
+    # The next replica on this node must not reuse the quarantined port.
+    new_port = manager.allocate_port("replica-new", RequestProtocol.HTTP)
+    assert new_port != port
+
+
+def test_prune_drops_empty_manager_after_quarantine(monkeypatch, port_range_constants):
+    """Once the quarantine drains, an empty node's manager is reclaimed."""
+    monkeypatch.setattr(
+        "ray.serve._private.node_port_manager.RAY_SERVE_PORT_QUARANTINE_S",
+        0.05,
+    )
+    node_id = "node-prune-quarantine-expire"
+    manager = NodePortManager.get_node_manager(node_id)
+    port = manager.allocate_port("replica-old", RequestProtocol.HTTP)
+    manager.release_port("replica-old", port, RequestProtocol.HTTP)
+
+    NodePortManager.prune(node_id_to_alive_replica_ids={})
+    assert node_id in NodePortManager._node_managers  # still quarantined -> kept
+
+    def _manager_reclaimed():
+        # Drives prune each poll; True once the expired quarantine lets it go.
+        NodePortManager.prune(node_id_to_alive_replica_ids={})
+        return node_id not in NodePortManager._node_managers
+
+    wait_for_condition(_manager_reclaimed, timeout=5)
+
+
+if __name__ == "__main__":
+    sys.exit(pytest.main(["-v", "-s", __file__]))

python/ray/serve/tests/unit/test_proxy_request_response.py

@@ -0,0 +1,344 @@
+import pickle
+from unittest.mock import MagicMock
+
+import pytest
+
+import ray.serve._private.logging_utils as logging_utils_mod
+from ray.serve._private.common import gRPCRequest
+from ray.serve._private.logging_utils import access_log_msg, format_client_address
+from ray.serve._private.proxy_request_response import (
+    ASGIProxyRequest,
+    ProxyRequest,
+    gRPCProxyRequest,
+)
+from ray.serve._private.test_utils import FakeGrpcContext
+from ray.serve.generated import serve_pb2
+
+
+class TestASGIProxyRequest:
+    def create_asgi_proxy_request(self, scope: dict) -> ASGIProxyRequest:
+        receive = MagicMock()
+        send = MagicMock()
+        return ASGIProxyRequest(scope=scope, receive=receive, send=send)
+
+    def test_request_type(self):
+        """Test calling request_type on an instance of ASGIProxyRequest.
+
+        When the request_type is not passed into the scope, it returns empty string.
+        When the request_type is passed into the scope, it returns the correct value.
+        """
+        proxy_request = self.create_asgi_proxy_request(scope={})
+        assert isinstance(proxy_request, ProxyRequest)
+        assert proxy_request.request_type == ""
+
+        request_type = "fake-request_type"
+        proxy_request = self.create_asgi_proxy_request(scope={"type": request_type})
+        assert isinstance(proxy_request, ProxyRequest)
+        assert proxy_request.request_type == request_type
+
+    def test_client(self):
+        """Test calling client on an instance of ASGIProxyRequest.
+
+        When the client is not passed into the scope, it returns empty string.
+        When the request_type is passed into the scope, it returns the correct value.
+        """
+        proxy_request = self.create_asgi_proxy_request(scope={})
+        assert isinstance(proxy_request, ProxyRequest)
+        assert proxy_request.client == ""
+
+        client = "fake-client"
+        proxy_request = self.create_asgi_proxy_request(scope={"client": client})
+        assert isinstance(proxy_request, ProxyRequest)
+        assert proxy_request.client == client
+
+    def test_method(self):
+        """Test calling method on an instance of ASGIProxyRequest.
+
+        When the method is not passed into the scope, it returns "WEBSOCKET". When
+        the method is passed into the scope, it returns the correct value.
+        """
+        proxy_request = self.create_asgi_proxy_request(scope={})
+        assert isinstance(proxy_request, ProxyRequest)
+        assert proxy_request.method == "WS"
+
+        method = "fake-method"
+        proxy_request = self.create_asgi_proxy_request(scope={"method": method})
+        assert isinstance(proxy_request, ProxyRequest)
+        assert proxy_request.method == method.upper()
+
+    def test_root_path(self):
+        """Test calling root_path on an instance of ASGIProxyRequest.
+
+        When the root_path is not passed into the scope, it returns empty string.
+        When calling set_root_path, it correctly sets the root_path. When the
+        root_path is passed into the scope, it returns the correct value.
+        """
+        proxy_request = self.create_asgi_proxy_request(scope={})
+        assert isinstance(proxy_request, ProxyRequest)
+        assert proxy_request.root_path == ""
+
+        root_path = "fake-root_path"
+        proxy_request.set_root_path(root_path)
+        assert proxy_request.root_path == root_path
+
+        proxy_request = self.create_asgi_proxy_request(scope={"root_path": root_path})
+        assert isinstance(proxy_request, ProxyRequest)
+        assert proxy_request.root_path == root_path
+
+    def test_path(self):
+        """Test calling path on an instance of ASGIProxyRequest.
+
+        When the path is not passed into the scope, it returns empty string.
+        When calling set_path, it correctly sets the path. When the
+        path is passed into the scope, it returns the correct value.
+        """
+        proxy_request = self.create_asgi_proxy_request(scope={})
+        assert isinstance(proxy_request, ProxyRequest)
+        assert proxy_request.path == ""
+
+        path = "fake-path"
+        proxy_request.set_path(path)
+        assert proxy_request.path == path
+
+        proxy_request = self.create_asgi_proxy_request(scope={"path": path})
+        assert isinstance(proxy_request, ProxyRequest)
+        assert proxy_request.path == path
+
+    def test_headers(self):
+        """Test calling headers on an instance of ASGIProxyRequest.
+
+        When the headers are not passed into the scope, it returns empty list.
+        When the headers are passed into the scope, it returns the correct value.
+        """
+        proxy_request = self.create_asgi_proxy_request(scope={})
+        assert isinstance(proxy_request, ProxyRequest)
+        assert proxy_request.headers == []
+
+        headers = [(b"fake-header-key", b"fake-header-value")]
+        proxy_request = self.create_asgi_proxy_request(scope={"headers": headers})
+        assert isinstance(proxy_request, ProxyRequest)
+        assert proxy_request.headers == headers
+
+    def test_is_route_request(self):
+        """Test calling is_route_request on an instance of ASGIProxyRequest.
+
+        When the is_route_request is called with `/-/routes`, it returns true.
+        When the is_route_request is called with other path, it returns false.
+        """
+        scope = {"path": "/-/routes"}
+        proxy_request = self.create_asgi_proxy_request(scope=scope)
+        assert proxy_request.is_route_request is True
+
+        scope = {"path": "/foo"}
+        proxy_request = self.create_asgi_proxy_request(scope=scope)
+        assert proxy_request.is_route_request is False
+
+    def test_is_health_request(self):
+        """Test calling is_health_request on an instance of ASGIProxyRequest.
+
+        When the is_health_request is called with `/-/healthz`, it returns true.
+        When the is_health_request is called with other path, it returns false.
+        """
+        scope = {"path": "/-/healthz"}
+        proxy_request = self.create_asgi_proxy_request(scope=scope)
+        assert proxy_request.is_health_request is True
+
+        scope = {"path": "/foo"}
+        proxy_request = self.create_asgi_proxy_request(scope=scope)
+        assert proxy_request.is_health_request is False
+
+
+class TestgRPCProxyRequest:
+    def test_calling_list_applications_method(self):
+        """Test initialize gRPCProxyRequest with list applications service method.
+
+        When the gRPCProxyRequest is initialized with list application service method,
+        calling is_route_request should return true and calling is_health_request
+        should return false.
+        """
+        context = FakeGrpcContext()
+        request_proto = serve_pb2.ListApplicationsRequest()
+        service_method = "/ray.serve.RayServeAPIService/ListApplications"
+        proxy_request = gRPCProxyRequest(
+            request_proto=request_proto,
+            context=context,
+            service_method=service_method,
+            stream=False,
+        )
+        assert isinstance(proxy_request, ProxyRequest)
+        assert proxy_request.is_route_request is True
+        assert proxy_request.is_health_request is False
+
+    def test_calling_healthz_method(self):
+        """Test initialize gRPCProxyRequest with healthz service method.
+
+        When the gRPCProxyRequest is initialized with healthz service method, calling
+        is_route_request should return false and calling is_health_request
+        should return true.
+        """
+        context = FakeGrpcContext()
+        request_proto = serve_pb2.HealthzRequest()
+        service_method = "/ray.serve.RayServeAPIService/Healthz"
+        proxy_request = gRPCProxyRequest(
+            request_proto=request_proto,
+            context=context,
+            service_method=service_method,
+            stream=False,
+        )
+        assert isinstance(proxy_request, ProxyRequest)
+        assert proxy_request.is_route_request is False
+        assert proxy_request.is_health_request is True
+
+    def test_calling_user_defined_method(self):
+        """Test initialize gRPCProxyRequest with user defined service method.
+
+        When the gRPCProxyRequest is initialized with user defined service method,
+        all attributes should be setup accordingly. Calling both is_route_request
+        and is_health_request should return false. `send_request_id()` should
+        also work accordingly to be able to send the into back to the client.
+        `request_object()` generates a gRPCRequest object with the correct attributes.
+        """
+        request_proto = serve_pb2.UserDefinedMessage(name="foo", num=30, foo="bar")
+        application = "fake-application"
+        request_id = "fake-request_id"
+        multiplexed_model_id = "fake-multiplexed_model_id"
+        session_id = "fake-session_id"
+        metadata = (
+            ("foo", "bar"),
+            ("application", application),
+            ("request_id", request_id),
+            ("multiplexed_model_id", multiplexed_model_id),
+            ("session_id", session_id),
+        )
+        context = MagicMock()
+        context.invocation_metadata.return_value = metadata
+        method_name = "Method1"
+        service_method = f"/custom.defined.Service/{method_name}"
+
+        proxy_request = gRPCProxyRequest(
+            request_proto=request_proto,
+            context=context,
+            service_method=service_method,
+            stream=MagicMock(),
+        )
+        assert isinstance(proxy_request, ProxyRequest)
+        assert proxy_request.route_path == application
+        assert proxy_request.method_name == method_name
+        assert proxy_request.app_name == application
+        assert proxy_request.request_id == request_id
+        assert proxy_request.multiplexed_model_id == multiplexed_model_id
+        assert proxy_request.session_id == session_id
+        assert proxy_request.is_route_request is False
+        assert proxy_request.is_health_request is False
+
+        proxy_request.send_request_id(request_id=request_id)
+        assert proxy_request.ray_serve_grpc_context.trailing_metadata() == [
+            ("request_id", request_id)
+        ]
+
+        serialized_arg = proxy_request.serialized_replica_arg()
+        assert isinstance(serialized_arg, bytes)
+        request_object = pickle.loads(serialized_arg)
+        assert isinstance(request_object, gRPCRequest)
+        assert request_object.user_request_proto == request_proto
+
+
+class TestGRPCProxyRequestClient:
+    """Tests for gRPCProxyRequest.client property."""
+
+    def _make_request(self, peer_value):
+        context = MagicMock()
+        context.peer.return_value = peer_value
+        context.invocation_metadata.return_value = ()
+        return gRPCProxyRequest(
+            request_proto=MagicMock(),
+            context=context,
+            service_method="/ray.serve.RayServeAPIService/Healthz",
+            stream=False,
+        )
+
+    def test_ipv4_peer(self):
+        req = self._make_request("ipv4:127.0.0.1:54321")
+        assert req.client == "127.0.0.1:54321"
+
+    def test_ipv6_peer(self):
+        # gRPC URL-encodes brackets in IPv6 peer addresses
+        req = self._make_request("ipv6:%5B::1%5D:54321")
+        assert req.client == "[::1]:54321"
+
+    def test_none_peer(self):
+        req = self._make_request(None)
+        assert req.client == ""
+
+    def test_empty_peer(self):
+        req = self._make_request("")
+        assert req.client == ""
+
+
+class TestFormatClientAddress:
+    """Tests for format_client_address in proxy.py."""
+
+    def test_tuple(self):
+        assert format_client_address(("10.0.0.1", 54321)) == "10.0.0.1:54321"
+
+    def test_list(self):
+        assert format_client_address(["10.0.0.1", 54321]) == "10.0.0.1:54321"
+
+    def test_string(self):
+        assert format_client_address("10.0.0.1:54321") == "10.0.0.1:54321"
+
+    def test_empty_string(self):
+        assert format_client_address("") == ""
+
+    def test_none(self):
+        assert format_client_address(None) == ""
+
+    def test_ipv6_tuple(self):
+        assert format_client_address(("::1", 54321)) == "[::1]:54321"
+
+    def test_ipv6_full_tuple(self):
+        assert format_client_address(("2001:db8::1", 8080)) == "[2001:db8::1]:8080"
+
+    def test_ipv6_string_passthrough(self):
+        assert format_client_address("[::1]:54321") == "[::1]:54321"
+
+
+class TestAccessLogMsg:
+    """Tests for access_log_msg formatting."""
+
+    def test_without_client(self):
+        msg = access_log_msg(method="GET", route="/", status="200", latency_ms=1.0)
+        assert msg == "GET / 200 1.0ms"
+
+    def test_with_client_flag_enabled(self, monkeypatch):
+        monkeypatch.setattr(logging_utils_mod, "RAY_SERVE_LOG_CLIENT_ADDRESS", True)
+        msg = access_log_msg(
+            method="GET",
+            route="/",
+            status="200",
+            latency_ms=1.0,
+            client="10.0.0.1:54321",
+        )
+        assert msg == "10.0.0.1:54321 GET / 200 1.0ms"
+
+    def test_with_client_flag_disabled(self):
+        msg = access_log_msg(
+            method="GET",
+            route="/",
+            status="200",
+            latency_ms=1.0,
+            client="10.0.0.1:54321",
+        )
+        assert msg == "GET / 200 1.0ms"
+
+    def test_with_empty_client(self):
+        msg = access_log_msg(
+            method="POST", route="/api", status="500", latency_ms=25.3, client=""
+        )
+        assert msg == "POST /api 500 25.3ms"
+
+
+if __name__ == "__main__":
+    import sys
+
+    sys.exit(pytest.main(["-v", "-s", __file__]))

python/ray/serve/tests/unit/test_proxy_router.py

@@ -0,0 +1,227 @@
+import pytest
+
+from ray.serve._private.common import DeploymentID, EndpointInfo
+from ray.serve._private.proxy_router import (
+    NO_REPLICAS_MESSAGE,
+    NO_ROUTES_MESSAGE,
+    ProxyRouter,
+)
+from ray.serve._private.test_utils import MockDeploymentHandle
+
+
+@pytest.fixture
+def mock_router():
+    def mock_get_handle(endpoint, info):
+        return MockDeploymentHandle(endpoint.name, endpoint.app_name)
+
+    yield ProxyRouter(mock_get_handle)
+
+
+def test_no_match(mock_router: ProxyRouter):
+    router = mock_router
+    router.update_routes(
+        {
+            DeploymentID(name="endpoint", app_name="default"): EndpointInfo(
+                route="/hello"
+            ),
+            DeploymentID(name="endpoint2", app_name="app2"): EndpointInfo(
+                route="/hello2"
+            ),
+        },
+    )
+    assert router.match_route("/nonexistent") is None
+    assert router.get_handle_for_endpoint("/nonexistent") is None
+
+
+def test_default_route(mock_router: ProxyRouter):
+    router = mock_router
+    router.update_routes(
+        {
+            DeploymentID(name="endpoint", app_name="default"): EndpointInfo(
+                route="/endpoint"
+            ),
+            DeploymentID(name="endpoint2", app_name="app2"): EndpointInfo(
+                route="/endpoint2"
+            ),
+        },
+    )
+
+    # Route based matching
+    assert router.match_route("/nonexistent") is None
+
+    route, handle, app_is_cross_language = router.match_route("/endpoint")
+    assert route == "/endpoint"
+    assert handle == ("endpoint", "default")
+    assert not app_is_cross_language
+
+    # Endpoint based matching
+    assert router.get_handle_for_endpoint("/nonexistent") is None
+
+    route, handle, app_is_cross_language = router.get_handle_for_endpoint("default")
+    assert route == "/endpoint"
+    assert handle == ("endpoint", "default")
+    assert not app_is_cross_language
+
+
+def test_trailing_slash(mock_router: ProxyRouter):
+    router = mock_router
+    router.update_routes(
+        {
+            DeploymentID(name="endpoint", app_name="default"): EndpointInfo(
+                route="/test"
+            )
+        },
+    )
+
+    route, handle, _ = router.match_route("/test/")
+    assert route == "/test" and handle == ("endpoint", "default")
+
+    router.update_routes(
+        {
+            DeploymentID(name="endpoint", app_name="default"): EndpointInfo(
+                route="/test/"
+            )
+        },
+    )
+
+    assert router.match_route("/test") is None
+
+
+def test_prefix_match(mock_router):
+    router = mock_router
+    router.update_routes(
+        {
+            DeploymentID(name="endpoint1", app_name="default"): EndpointInfo(
+                route="/test/test2"
+            ),
+            DeploymentID(name="endpoint2", app_name="default"): EndpointInfo(
+                route="/test"
+            ),
+            DeploymentID(name="endpoint3", app_name="default"): EndpointInfo(route="/"),
+        },
+    )
+
+    route, handle, _ = router.match_route("/test/test2/subpath")
+    assert route == "/test/test2" and handle == ("endpoint1", "default")
+    route, handle, _ = router.match_route("/test/test2/")
+    assert route == "/test/test2" and handle == ("endpoint1", "default")
+    route, handle, _ = router.match_route("/test/test2")
+    assert route == "/test/test2" and handle == ("endpoint1", "default")
+
+    route, handle, _ = router.match_route("/test/subpath")
+    assert route == "/test" and handle == ("endpoint2", "default")
+    route, handle, _ = router.match_route("/test/")
+    assert route == "/test" and handle == ("endpoint2", "default")
+    route, handle, _ = router.match_route("/test")
+    assert route == "/test" and handle == ("endpoint2", "default")
+
+    route, handle, _ = router.match_route("/test2")
+    assert route == "/" and handle == ("endpoint3", "default")
+    route, handle, _ = router.match_route("/")
+    assert route == "/" and handle == ("endpoint3", "default")
+
+
+def test_update_routes(mock_router):
+    router = mock_router
+    router.update_routes(
+        {DeploymentID("endpoint", "app1"): EndpointInfo(route="/endpoint")},
+    )
+
+    route, handle, app_is_cross_language = router.match_route("/endpoint")
+    assert route == "/endpoint"
+    assert handle == ("endpoint", "app1")
+    assert not app_is_cross_language
+
+    route, handle, app_is_cross_language = router.get_handle_for_endpoint("app1")
+    assert route == "/endpoint"
+    assert handle == ("endpoint", "app1")
+    assert not app_is_cross_language
+
+    router.update_routes(
+        {
+            DeploymentID(name="endpoint2", app_name="app2"): EndpointInfo(
+                route="/endpoint2",
+                app_is_cross_language=True,
+            ),
+            DeploymentID(name="endpoint3", app_name="app3"): EndpointInfo(
+                route="/endpoint3",
+                app_is_cross_language=True,
+            ),
+        },
+    )
+
+    assert router.match_route("/endpoint") is None
+    assert router.match_route("app1") is None
+
+    route, handle, app_is_cross_language = router.match_route("/endpoint2")
+    assert route == "/endpoint2"
+    assert handle == ("endpoint2", "app2")
+    assert app_is_cross_language
+
+    route, handle, app_is_cross_language = router.get_handle_for_endpoint("app2")
+    assert route == "/endpoint2"
+    assert handle == ("endpoint2", "app2")
+    assert app_is_cross_language
+
+
+class TestReadyForTraffic:
+    @pytest.mark.parametrize("is_head", [False, True])
+    def test_route_table_not_populated(self, mock_router, is_head: bool):
+        """Proxy router should NOT be ready for traffic if:
+        - it has not received route table from controller
+        """
+
+        ready_for_traffic, msg = mock_router.ready_for_traffic(is_head=is_head)
+        assert not ready_for_traffic
+        assert msg == NO_ROUTES_MESSAGE
+
+    def test_head_route_table_populated_no_replicas(self, mock_router):
+        """Proxy router should be ready for traffic if:
+        - it has received route table from controller
+        - it hasn't received any replicas yet
+        - it lives on head node
+        """
+
+        d_id = DeploymentID(name="A", app_name="B")
+        mock_router.update_routes({d_id: EndpointInfo(route="/")})
+        mock_router.handles[d_id].set_running_replicas_populated(False)
+
+        ready_for_traffic, msg = mock_router.ready_for_traffic(is_head=True)
+        assert ready_for_traffic
+        assert not msg
+
+    def test_worker_route_table_populated_no_replicas(self, mock_router):
+        """Proxy router should NOT be ready for traffic if:
+        - it has received route table from controller
+        - it hasn't received any replicas yet
+        - it lives on a worker node
+        """
+
+        d_id = DeploymentID(name="A", app_name="B")
+        mock_router.update_routes({d_id: EndpointInfo(route="/")})
+        mock_router.handles[d_id].set_running_replicas_populated(False)
+
+        ready_for_traffic, msg = mock_router.ready_for_traffic(is_head=False)
+        assert not ready_for_traffic
+        assert msg == NO_REPLICAS_MESSAGE
+
+    @pytest.mark.parametrize("is_head", [False, True])
+    def test_route_table_populated_with_replicas(self, mock_router, is_head: bool):
+        """Proxy router should be ready for traffic if:
+        - it has received route table from controller
+        - it has received replicas from controller
+        """
+
+        d_id = DeploymentID(name="A", app_name="B")
+        mock_router.update_routes({d_id: EndpointInfo(route="/")})
+        mock_router.handles[d_id].set_running_replicas_populated(True)
+
+        ready_for_traffic, msg = mock_router.ready_for_traffic(is_head=is_head)
+        assert ready_for_traffic
+        assert not msg
+
+
+if __name__ == "__main__":
+    import sys
+
+    sys.exit(pytest.main(["-v", "-s", __file__]))

python/ray/serve/tests/unit/test_proxy_state.py

@@ -0,0 +1,977 @@
+import json
+from typing import List, Optional, Tuple
+from unittest import mock
+from unittest.mock import patch
+
+import pytest
+
+from ray._common.test_utils import wait_for_condition
+from ray._common.utils import Timer
+from ray.serve._private.cluster_node_info_cache import ClusterNodeInfoCache
+from ray.serve._private.common import RequestProtocol
+from ray.serve._private.constants import (
+    PROXY_HEALTH_CHECK_UNHEALTHY_THRESHOLD,
+    RAY_SERVE_FALLBACK_PROXY_GRPC_PORT,
+    RAY_SERVE_FALLBACK_PROXY_HTTP_PORT,
+)
+from ray.serve._private.proxy_state import ProxyState, ProxyStateManager, ProxyWrapper
+from ray.serve._private.test_utils import MockTimer
+from ray.serve.config import HTTPOptions, ProxyLocation
+from ray.serve.schema import LoggingConfig, ProxyStatus
+
+HEAD_NODE_ID = "node_id-index-head"
+
+
+class MockClusterNodeInfoCache:
+    def __init__(self):
+        self.alive_nodes = []
+
+    def get_alive_nodes(self):
+        return self.alive_nodes
+
+    def get_alive_node_ids(self):
+        return {node_id for node_id, _, _ in self.alive_nodes}
+
+
+class FakeProxyActor:
+    def __init__(self, *args, **kwargs):
+        pass
+
+    def ready(self):
+        return json.dumps(["mock_worker_id", "mock_log_file_path"])
+
+    def check_health(self):
+        pass
+
+
+class FakeProxyWrapper(ProxyWrapper):
+    def __init__(self, *args, **kwargs):
+        self.actor_handle = FakeProxyActor(*args, **kwargs)
+        self.is_ready_response = None
+        self.is_healthy_response = None
+        self.is_drained_response = False
+        self.worker_id = "mock_worker_id"
+        self.log_file_path = "mock_log_file_path"
+        self.shutdown = False
+        self.num_health_checks = 0
+        self.num_drain_checks = 0
+
+    @property
+    def actor_id(self) -> str:
+        pass
+
+    def is_ready(self, timeout_s: float) -> Optional[bool]:
+        return self.is_ready_response
+
+    def is_healthy(self, timeout_s: float) -> Optional[bool]:
+        self.num_health_checks += 1
+        return self.is_healthy_response
+
+    def is_drained(self, timeout_s: float) -> Optional[bool]:
+        self.num_drain_checks += 1
+        return self.is_drained_response
+
+    def is_shutdown(self):
+        return self.shutdown
+
+    def update_draining(self, draining: bool):
+        pass
+
+    def kill(self):
+        self.shutdown = True
+
+    def get_num_health_checks(self):
+        return self.num_health_checks
+
+    def get_num_drain_checks(self):
+        return self.num_health_checks
+
+
+def _create_proxy_state_manager(
+    http_options: HTTPOptions = HTTPOptions(),
+    head_node_id: str = HEAD_NODE_ID,
+    cluster_node_info_cache=MockClusterNodeInfoCache(),
+    actor_proxy_wrapper_class=FakeProxyWrapper,
+    timer=Timer(),
+    running_native_proxies: bool = False,
+    proxy_location=None,
+) -> (ProxyStateManager, ClusterNodeInfoCache):
+    return (
+        ProxyStateManager(
+            http_options=http_options,
+            head_node_id=head_node_id,
+            cluster_node_info_cache=cluster_node_info_cache,
+            logging_config=LoggingConfig(),
+            actor_proxy_wrapper_class=actor_proxy_wrapper_class,
+            timer=timer,
+            running_native_proxies=running_native_proxies,
+            proxy_location=proxy_location,
+        ),
+        cluster_node_info_cache,
+    )
+
+
+def _create_proxy_state(
+    actor_proxy_wrapper_class=FakeProxyWrapper,
+    status: ProxyStatus = ProxyStatus.STARTING,
+    node_id: str = "mock_node_id",
+    timer=Timer(),
+    **kwargs,
+) -> ProxyState:
+    state = ProxyState(
+        actor_proxy_wrapper=actor_proxy_wrapper_class(),
+        actor_name="alice",
+        node_id=node_id,
+        node_ip="mock_node_ip",
+        node_instance_id="mock_instance_id",
+        timer=timer,
+    )
+    state._set_status(status=status)
+    return state
+
+
+@pytest.fixture
+def number_of_worker_nodes() -> int:
+    return 100
+
+
+@pytest.fixture
+def all_nodes(number_of_worker_nodes) -> List[Tuple[str, str, str]]:
+    return [(HEAD_NODE_ID, "fake-head-ip", "fake-head-instance-id")] + [
+        (f"worker-node-id-{i}", f"fake-worker-ip-{i}", f"fake-instance-id-{i}")
+        for i in range(number_of_worker_nodes)
+    ]
+
+
+def _reconcile_and_check_proxy_status(state: ProxyState, status: ProxyStatus):
+    state.reconcile()
+    assert state.status == status
+    return True
+
+
+def _update_and_check_proxy_state_manager(
+    proxy_state_manager: ProxyStateManager,
+    node_ids: List[str],
+    statuses: List[ProxyStatus],
+    **kwargs,
+):
+    proxy_state_manager.update(**kwargs)
+    proxy_states = proxy_state_manager._proxy_states
+    assert all(
+        [
+            proxy_states[node_ids[idx]].status == statuses[idx]
+            for idx in range(len(node_ids))
+        ]
+    ), [proxy_state.status for proxy_state in proxy_states.values()]
+    return True
+
+
+def test_node_selection_via_proxy_location(all_nodes):
+    # `proxy_location` is the placement authority (threaded separately from the
+    # deprecated HTTPOptions.location); covers HTTP and gRPC (single ProxyActor).
+    all_node_ids = {node_id for node_id, _, _ in all_nodes}
+
+    def mgr(**kwargs):
+        psm, cache = _create_proxy_state_manager(**kwargs)
+        cache.alive_nodes = all_nodes
+        return psm
+
+    assert (
+        mgr(proxy_location=ProxyLocation.Disabled)._get_target_nodes(all_node_ids) == []
+    )
+    assert (
+        mgr(proxy_location=ProxyLocation.HeadOnly)._get_target_nodes(all_node_ids)
+        == all_nodes[:1]
+    )
+    assert (
+        mgr(proxy_location=ProxyLocation.EveryNode)._get_target_nodes(all_node_ids)
+        == all_nodes
+    )
+
+    # Default (neither location nor proxy_location) resolves to EveryNode.
+    assert mgr()._get_target_nodes(all_node_ids) == all_nodes
+    assert mgr().get_proxy_location() == ProxyLocation.EveryNode
+
+    # Explicit (deprecated) HTTPOptions.location overrides proxy_location.
+    with pytest.warns(DeprecationWarning, match="`location` in HTTPOptions"):
+        override = HTTPOptions(location=ProxyLocation.HeadOnly)
+    psm = mgr(http_options=override, proxy_location=ProxyLocation.EveryNode)
+    assert psm._get_target_nodes(all_node_ids) == all_nodes[:1]
+    assert psm.get_proxy_location() == ProxyLocation.HeadOnly
+
+
+def test_node_selection(all_nodes):
+    all_node_ids = {node_id for node_id, _, _ in all_nodes}
+    # Test NoServer
+    proxy_state_manager, cluster_node_info_cache = _create_proxy_state_manager(
+        HTTPOptions(location=ProxyLocation.Disabled)
+    )
+    cluster_node_info_cache.alive_nodes = all_nodes
+    assert proxy_state_manager._get_target_nodes(all_node_ids) == []
+
+    # Test HeadOnly
+    proxy_state_manager, cluster_node_info_cache = _create_proxy_state_manager(
+        HTTPOptions(location=ProxyLocation.HeadOnly)
+    )
+    cluster_node_info_cache.alive_nodes = all_nodes
+    assert proxy_state_manager._get_target_nodes(all_node_ids) == all_nodes[:1]
+
+    # Test EveryNode
+    proxy_state_manager, cluster_node_info_cache = _create_proxy_state_manager(
+        HTTPOptions(location=ProxyLocation.EveryNode)
+    )
+    cluster_node_info_cache.alive_nodes = all_nodes
+    assert proxy_state_manager._get_target_nodes(all_node_ids) == all_nodes
+
+    # Test specific nodes
+    proxy_state_manager, cluster_node_info_cache = _create_proxy_state_manager(
+        HTTPOptions(location=ProxyLocation.EveryNode)
+    )
+    cluster_node_info_cache.alive_nodes = all_nodes
+    assert proxy_state_manager._get_target_nodes({HEAD_NODE_ID}) == [
+        (HEAD_NODE_ID, "fake-head-ip", "fake-head-instance-id")
+    ]
+
+
+@patch("ray.serve._private.proxy_state.PROXY_HEALTH_CHECK_PERIOD_S", 5)
+def test_proxy_state_manager_restarts_unhealthy_proxies(all_nodes):
+    """Test the update method in ProxyStateManager would
+    kill and restart unhealthy proxies.
+    """
+
+    timer = MockTimer()
+    proxy_state_manager, cluster_node_info_cache = _create_proxy_state_manager(
+        timer=timer
+    )
+
+    cluster_node_info_cache.alive_nodes = all_nodes
+    # First iteration, refresh state
+    proxy_state_manager.update(proxy_nodes={HEAD_NODE_ID})
+
+    prev_proxy_state = proxy_state_manager._proxy_states[HEAD_NODE_ID]
+    # Mark existing head-node proxy UNHEALTHY
+    prev_proxy_state._set_status(ProxyStatus.UNHEALTHY)
+    old_proxy = prev_proxy_state.actor_handle
+
+    # Continuously trigger update and wait for status to be changed to HEALTHY.
+    for _ in range(1):
+        proxy_state_manager.update(proxy_nodes={HEAD_NODE_ID})
+        # Advance timer by 5 (to perform a health-check)
+        timer.advance(5)
+
+    new_proxy_state = proxy_state_manager._proxy_states[HEAD_NODE_ID]
+    # Previous proxy's state stays UNHEALTHY
+    assert prev_proxy_state.status == ProxyStatus.UNHEALTHY
+    # Ensure the old proxy is getting shutdown.
+    assert prev_proxy_state._shutting_down
+    # New proxy's state should be STARTING
+    assert new_proxy_state.status == ProxyStatus.STARTING
+    assert new_proxy_state.proxy_restart_count == 1
+
+    new_proxy = new_proxy_state.actor_handle
+    # Ensure the new proxy is completely different object than old proxy.
+    assert new_proxy != old_proxy
+
+
+def test_proxy_state_reconcile_shutting_down():
+    proxy_state = _create_proxy_state()
+    previous_status = proxy_state.status
+    proxy_state.shutdown()
+    # This should be no-op. The status of the http proxy state will not be changed.
+    proxy_state.reconcile()
+    current_status = proxy_state.status
+
+    # Ensure the proxy state is in the shutting down state.
+    assert proxy_state._shutting_down
+
+    # Ensure the status didn't change.
+    assert previous_status == current_status
+
+
+def test_proxy_state_reconcile_readiness_check_succeed():
+    proxy_state = _create_proxy_state()
+
+    # Configure is_ready to be true
+    proxy_state._actor_proxy_wrapper.is_ready_response = True
+
+    # Ensure the proxy status before update is STARTING.
+    assert proxy_state.status == ProxyStatus.STARTING
+
+    # Ensure actor_details are set to the initial state when the proxy_state is created.
+    assert proxy_state.actor_details.worker_id is None
+    assert proxy_state.actor_details.log_file_path is None
+    assert proxy_state.actor_details.status == ProxyStatus.STARTING.value
+
+    # Continuously trigger update and wait for status to be changed.
+    proxy_state.reconcile()
+    assert proxy_state.status == ProxyStatus.HEALTHY
+
+    # Ensure actor_details are updated.
+    assert proxy_state.actor_details.worker_id == "mock_worker_id"
+    assert proxy_state.actor_details.log_file_path == "mock_log_file_path"
+    assert proxy_state.actor_details.status == ProxyStatus.HEALTHY.value
+
+
+def test_proxy_state_reconcile_readiness_check_pending():
+    proxy_state = _create_proxy_state()
+
+    # Ensure the proxy status before update is STARTING.
+    assert proxy_state.status == ProxyStatus.STARTING
+
+    # When the proxy readiness check is pending, the proxy wrapper is_ready
+    # will return None
+    proxy_state._actor_proxy_wrapper.is_ready_response = None
+
+    # Trigger update. The status do not change, while readiness check is pending
+    for _ in range(10):
+        proxy_state.reconcile()
+        assert proxy_state.status == ProxyStatus.STARTING
+
+    # Unblock is_ready call, trigger update, and wait for status change to HEALTHY.
+    proxy_state._actor_proxy_wrapper.is_ready_response = True
+
+    proxy_state.reconcile()
+    assert proxy_state.status == ProxyStatus.HEALTHY
+
+
+def test_proxy_state_reconcile_readiness_check_fails():
+    proxy_state = _create_proxy_state()
+
+    # Emulate readiness check failure
+    proxy_state._actor_proxy_wrapper.is_ready_response = False
+    # Ensure the proxy status before update is STARTING.
+    assert proxy_state.status == ProxyStatus.STARTING
+
+    # First failure shouldn't trigger state-transition to UNHEALTHY
+    proxy_state.reconcile()
+    assert proxy_state.status == ProxyStatus.STARTING
+
+    # After PROXY_HEALTH_CHECK_UNHEALTHY_THRESHOLD failures, state should
+    # transition to UNHEALTHY
+    for _ in range(PROXY_HEALTH_CHECK_UNHEALTHY_THRESHOLD - 1):
+        proxy_state.reconcile()
+
+    assert proxy_state.status == ProxyStatus.UNHEALTHY
+
+
+@patch("ray.serve._private.proxy_state.PROXY_READY_CHECK_TIMEOUT_S", 5)
+def test_proxy_state_reconcile_health_check_succeed():
+    timer = MockTimer()
+    proxy_state = _create_proxy_state(time=timer)
+
+    # Emulate readiness check succeeding
+    proxy_state._actor_proxy_wrapper.is_ready_response = True
+    # Should transition to HEALTHY
+    proxy_state.reconcile()
+    assert proxy_state.status == ProxyStatus.HEALTHY
+
+    # Trigger update few more times and the status continue to be HEALTHY.
+    for _ in range(10):
+        _reconcile_and_check_proxy_status(proxy_state, ProxyStatus.HEALTHY)
+        # Advance timer by 5s
+        timer.advance(5)
+
+    # Health-checks should have been performed on every iteration
+    assert proxy_state._actor_proxy_wrapper.num_health_checks == 10
+
+
+@patch("ray.serve._private.proxy_state.PROXY_HEALTH_CHECK_PERIOD_S", 5)
+def test_proxy_state_reconcile_health_check_transient_failures():
+    timer = MockTimer()
+    # Start with HEALTHY state
+    proxy_state = _create_proxy_state(status=ProxyStatus.HEALTHY, timer=timer)
+    # Simulate health-checks failing
+    proxy_state._actor_proxy_wrapper.is_healthy_response = False
+
+    # Reconcile PROXY_HEALTH_CHECK_UNHEALTHY_THRESHOLD - 1 times, state should
+    # continue to stay HEALTHY
+    for _ in range(PROXY_HEALTH_CHECK_UNHEALTHY_THRESHOLD - 1):
+        _reconcile_and_check_proxy_status(proxy_state, ProxyStatus.HEALTHY)
+        # Advance timer by 5 (to trigger new health-check)
+        timer.advance(5)
+
+    assert (
+        proxy_state._actor_proxy_wrapper.get_num_health_checks()
+        == PROXY_HEALTH_CHECK_UNHEALTHY_THRESHOLD - 1
+    )
+
+    # Simulate health-checks passing
+    proxy_state._actor_proxy_wrapper.is_healthy_response = True
+
+    wait_for_condition(
+        condition_predictor=_reconcile_and_check_proxy_status,
+        state=proxy_state,
+        status=ProxyStatus.HEALTHY,
+    )
+    # Ensure _consecutive_health_check_failures is reset
+    assert proxy_state._consecutive_health_check_failures == 0
+
+
+@patch("ray.serve._private.proxy_state.PROXY_HEALTH_CHECK_PERIOD_S", 5)
+def test_proxy_state_reconcile_health_check_persistent_failures():
+    timer = MockTimer()
+    # Start with HEALTHY state
+    proxy_state = _create_proxy_state(status=ProxyStatus.HEALTHY, timer=timer)
+    # Simulate health-checks failing
+    proxy_state._actor_proxy_wrapper.is_healthy_response = False
+
+    # Reconcile PROXY_HEALTH_CHECK_UNHEALTHY_THRESHOLD - 1 times, state should
+    # continue to stay HEALTHY
+    for _ in range(PROXY_HEALTH_CHECK_UNHEALTHY_THRESHOLD - 1):
+        _reconcile_and_check_proxy_status(proxy_state, ProxyStatus.HEALTHY)
+        # Advance timer by 5 (to trigger new health-check)
+        timer.advance(5)
+
+    assert (
+        proxy_state._actor_proxy_wrapper.get_num_health_checks()
+        == PROXY_HEALTH_CHECK_UNHEALTHY_THRESHOLD - 1
+    )
+
+    # On PROXY_HEALTH_CHECK_UNHEALTHY_THRESHOLD iteration, state transitions to
+    # UNHEALTHY
+    _reconcile_and_check_proxy_status(proxy_state, ProxyStatus.UNHEALTHY)
+    # Ensure _consecutive_health_check_failures is correct
+    assert proxy_state._consecutive_health_check_failures == 3
+
+
+@patch("ray.serve._private.proxy_state.PROXY_HEALTH_CHECK_PERIOD_S", 0)
+@pytest.mark.parametrize("number_of_worker_nodes", [0, 1, 2, 3])
+def test_proxy_manager_update_proxies_states(all_nodes, number_of_worker_nodes):
+    """Test update draining logics.
+
+    When update nodes to inactive, head node http proxy should never be draining while
+    worker node http proxy should change to draining. When update nodes to active, head
+    node http proxy should continue to be healthy while worker node http proxy should
+    be healthy.
+    """
+    manager, cluster_node_info_cache = _create_proxy_state_manager(
+        HTTPOptions(location=ProxyLocation.EveryNode)
+    )
+    cluster_node_info_cache.alive_nodes = all_nodes
+
+    for node_id, _, _ in all_nodes:
+        manager._proxy_states[node_id] = _create_proxy_state(
+            status=ProxyStatus.HEALTHY,
+            node_id=node_id,
+        )
+
+    node_ids = [node_id for node_id, _, _ in all_nodes]
+
+    # No target proxy nodes
+    proxy_nodes = {HEAD_NODE_ID}
+
+    # Head node proxy should continue to be HEALTHY.
+    # Worker node proxy should turn DRAINING.
+    wait_for_condition(
+        condition_predictor=_update_and_check_proxy_state_manager,
+        proxy_state_manager=manager,
+        node_ids=node_ids,
+        statuses=[ProxyStatus.HEALTHY]
+        + [ProxyStatus.DRAINING] * number_of_worker_nodes,
+        proxy_nodes=proxy_nodes,
+    )
+
+    # All nodes are target proxy nodes
+    proxy_nodes = set(node_ids)
+
+    # Head node proxy should continue to be HEALTHY.
+    # Worker node proxy should turn HEALTHY.
+    wait_for_condition(
+        condition_predictor=_update_and_check_proxy_state_manager,
+        proxy_state_manager=manager,
+        node_ids=node_ids,
+        statuses=[ProxyStatus.HEALTHY] * (number_of_worker_nodes + 1),
+        proxy_nodes=proxy_nodes,
+    )
+
+
+@patch("ray.serve._private.proxy_state.PROXY_HEALTH_CHECK_PERIOD_S", 5)
+def test_proxy_state_reconcile_draining_success():
+    """Test that the proxy will remain DRAINING even if health check succeeds."""
+    timer = MockTimer(start_time=0)
+    # Start with HEALTHY state
+    proxy_state = _create_proxy_state(status=ProxyStatus.HEALTHY, timer=timer)
+    # Simulate health-checks passing
+    proxy_state._actor_proxy_wrapper.is_healthy_response = True
+    # Simulate is_drained returning false
+    proxy_state._actor_proxy_wrapper.is_drained_response = False
+
+    for _ in range(10):
+        proxy_state.reconcile(draining=True)
+        assert proxy_state.status == ProxyStatus.DRAINING
+        # Advance timer by 5 (to trigger new health-check, drain-check)
+        timer.advance(5)
+
+    assert proxy_state._actor_proxy_wrapper.get_num_health_checks() == 10
+    assert proxy_state._actor_proxy_wrapper.get_num_drain_checks() == 10
+
+    # Make sure the status is still DRAINING
+    assert proxy_state.status == ProxyStatus.DRAINING
+
+    # Simulate is_drained request to ProxyActor pending (for 5 iterations)
+    proxy_state._actor_proxy_wrapper.is_drained_response = None
+
+    for _ in range(5):
+        proxy_state.reconcile(draining=True)
+        assert proxy_state.status == ProxyStatus.DRAINING
+        # Advance timer by 5 (to trigger new health-check, drain-check)
+        timer.advance(5)
+
+    assert proxy_state._actor_proxy_wrapper.get_num_health_checks() == 15
+    # No new drain checks will occur, since there's a pending one (not completed yet)
+    assert proxy_state._actor_proxy_wrapper.get_num_drain_checks() == 15
+
+    # Simulate draining completed
+    proxy_state._actor_proxy_wrapper.is_drained_response = True
+    # Advance timer by 5 (to trigger new health-check, drain-check on next iteration)
+    timer.advance(5)
+
+    proxy_state.reconcile(draining=True)
+    # State should transition to DRAINED
+    assert proxy_state.status == ProxyStatus.DRAINED
+
+
+@patch("ray.serve._private.proxy_state.PROXY_DRAIN_CHECK_PERIOD_S", 5)
+@pytest.mark.parametrize("number_of_worker_nodes", [1])
+def test_proxy_actor_manager_removing_proxies(all_nodes, number_of_worker_nodes):
+    """Test the state transition from DRAINING to UNHEALTHY for the proxy actor."""
+
+    assert len(all_nodes) == 2, "There should be 2 nodes in this test"
+
+    timer = MockTimer(start_time=0)
+
+    manager, cluster_node_info_cache = _create_proxy_state_manager(
+        HTTPOptions(location=ProxyLocation.EveryNode),
+        timer=timer,
+    )
+    cluster_node_info_cache.alive_nodes = all_nodes
+
+    for node_id, _, _ in all_nodes:
+        manager._proxy_states[node_id] = _create_proxy_state(
+            status=ProxyStatus.STARTING,
+            node_id=node_id,
+            timer=timer,
+        )
+
+        manager._proxy_states[node_id]._actor_proxy_wrapper.is_ready_response = True
+
+    # All nodes are target proxy nodes
+    node_ids = [node_id for node_id, _, _ in all_nodes]
+
+    worker_node_id = node_ids[1]
+    worker_proxy_state = manager._proxy_states[worker_node_id]
+
+    # Reconcile all proxies states
+    manager.update(
+        proxy_nodes=(set(node_ids)),
+    )
+
+    # Assert all proxies are HEALTHY
+    proxy_statuses = [manager._proxy_states[node_id].status for node_id in node_ids]
+    assert [ProxyStatus.HEALTHY, ProxyStatus.HEALTHY] == proxy_statuses
+
+    # Reconcile proxies with empty set of target nodes (ie only proxy on the head-node
+    # should be preserved all the other should be drained)
+    worker_proxy_state._actor_proxy_wrapper.is_drained_response = False
+    N = 10
+    for _ in range(N):
+        manager.update(
+            proxy_nodes={HEAD_NODE_ID},
+        )
+        timer.advance(5)
+        # Assert that
+        #   - Head-node proxy is HEALTHY4
+        #   - Worker node proxy is DRAINING
+        proxy_statuses = [manager._proxy_states[node_id].status for node_id in node_ids]
+        assert [ProxyStatus.HEALTHY, ProxyStatus.DRAINING] == proxy_statuses
+
+    assert worker_proxy_state._actor_proxy_wrapper.get_num_drain_checks() == N
+
+    # Mark target proxy as fully drained
+    worker_proxy_state._actor_proxy_wrapper.is_drained_response = True
+
+    # Reconcile proxies with empty set of target nodes (worker node proxy
+    # will be shutdown by now)
+    manager.update(
+        proxy_nodes={HEAD_NODE_ID},
+    )
+
+    assert len(manager._proxy_states) == 1
+    assert manager._proxy_states[HEAD_NODE_ID].status == ProxyStatus.HEALTHY
+
+
+def test_is_ready_for_shutdown(all_nodes):
+    """Test `is_ready_for_shutdown()` returns True the correct state.
+
+    Before `shutdown()` is called, `is_ready_for_shutdown()` should return false. After
+    `shutdown()` is called and all proxy actor are killed, `is_ready_for_shutdown()`
+    should return true.
+    """
+    manager, cluster_node_info_cache = _create_proxy_state_manager(
+        HTTPOptions(location=ProxyLocation.EveryNode)
+    )
+    cluster_node_info_cache.alive_nodes = all_nodes
+
+    for node_id, _, _ in all_nodes:
+        manager._proxy_states[node_id] = _create_proxy_state(
+            status=ProxyStatus.HEALTHY,
+            node_id=node_id,
+        )
+
+    # Ensure before shutdown, manager is not shutdown
+    assert not manager.is_ready_for_shutdown()
+
+    manager.shutdown()
+
+    # Ensure after shutdown, manager is shutdown and all proxy states are shutdown
+    def check_is_ready_for_shutdown():
+        return manager.is_ready_for_shutdown()
+
+    wait_for_condition(check_is_ready_for_shutdown)
+
+
+@patch("ray.serve._private.proxy_state.PROXY_HEALTH_CHECK_UNHEALTHY_THRESHOLD", 1)
+@pytest.mark.parametrize("number_of_worker_nodes", [1])
+def test_proxy_state_manager_timing_out_on_start(number_of_worker_nodes, all_nodes):
+    """Test update method on ProxyStateManager when the proxy state is STARTING and
+    when the ready call takes longer than PROXY_READY_CHECK_TIMEOUT_S.
+
+    The proxy state started with STARTING. After update is called, ready calls takes
+    some time to finish. The proxy state manager will restart the proxy state after
+    PROXY_READY_CHECK_TIMEOUT_S. After the next period of check_health call,
+    the proxy state manager will check on backoff timeout, not immediately
+    restarting the proxy states, and eventually set the proxy state to HEALTHY.
+    """
+    fake_time = MockTimer()
+    proxy_state_manager, cluster_node_info_cache = _create_proxy_state_manager(
+        http_options=HTTPOptions(location=ProxyLocation.EveryNode),
+        timer=fake_time,
+    )
+    cluster_node_info_cache.alive_nodes = all_nodes
+
+    node_ids = {node[0] for node in all_nodes}
+
+    # Run update to create proxy states.
+    proxy_state_manager.update(proxy_nodes=node_ids)
+
+    # Ensure 2 proxies are created, one for the head node and another for the worker.
+    assert len(proxy_state_manager._proxy_states) == len(node_ids)
+
+    # Ensure the proxy state statuses before update are STARTING, set the
+    # readiness check to failing
+    for node_id in node_ids:
+        proxy_state = proxy_state_manager._proxy_states[node_id]
+
+        assert proxy_state.status == ProxyStatus.STARTING
+        proxy_state._actor_proxy_wrapper.is_ready_response = False
+
+    # Capture current proxy states (prior to updating)
+    prev_proxy_states = dict(proxy_state_manager._proxy_states)
+
+    # Trigger PSM to reconcile
+    proxy_state_manager.update(proxy_nodes=node_ids)
+
+    # Ensure the proxy state statuses before update are STARTING, set the
+    # readiness check to failing
+    for node_id in node_ids:
+        proxy_state = proxy_state_manager._proxy_states[node_id]
+        prev_proxy_state = prev_proxy_states[node_id]
+        # Assert
+        #   - All proxies are restarted
+        #   - Previous proxy states are UNHEALTHY
+        #   - New proxy states are STARTING
+        assert proxy_state != prev_proxy_state
+        assert prev_proxy_state.status == ProxyStatus.UNHEALTHY
+        assert proxy_state.status == ProxyStatus.STARTING
+        # Mark new proxy readiness checks as passing
+        proxy_state._actor_proxy_wrapper.is_ready_response = True
+
+    # Capture current proxy states again (prior to updating)
+    prev_proxy_states = dict(proxy_state_manager._proxy_states)
+
+    # Trigger PSM to reconcile
+    proxy_state_manager.update(proxy_nodes=node_ids)
+
+    # Ensure the proxy state statuses before update are STARTING, set the
+    # readiness check to failing
+    for node_id in node_ids:
+        proxy_state = proxy_state_manager._proxy_states[node_id]
+        prev_proxy_state = prev_proxy_states[node_id]
+        # Assert
+        #   - All proxies are restarted
+        #   - Previous proxy states are UNHEALTHY
+        #   - New proxy states are STARTING
+        assert proxy_state == prev_proxy_state
+        assert prev_proxy_state.status == ProxyStatus.HEALTHY
+        assert proxy_state.status == ProxyStatus.HEALTHY
+
+
+def test_proxy_state_manager_get_targets(all_nodes):
+    """Test the get_targets method on ProxyStateManager."""
+    manager, cluster_node_info_cache = _create_proxy_state_manager(
+        HTTPOptions(location=ProxyLocation.EveryNode)
+    )
+    cluster_node_info_cache.alive_nodes = all_nodes
+
+    for node_id, _, _ in all_nodes:
+        manager._proxy_states[node_id] = _create_proxy_state(
+            status=ProxyStatus.HEALTHY,
+            node_id=node_id,
+        )
+
+    manager._proxy_states[all_nodes[-1][0]].try_update_status(ProxyStatus.DRAINED)
+
+    targets = manager.get_targets(RequestProtocol.HTTP)
+    assert len(targets) == len(all_nodes) - 1
+    assert targets[0].ip == "mock_node_ip"
+    assert targets[0].port == 8000
+    assert targets[0].instance_id == "mock_instance_id"
+    assert targets[0].name == "alice"
+
+    targets = manager.get_targets(RequestProtocol.GRPC)
+    assert len(targets) == 0
+
+    with pytest.raises(ValueError):
+        manager.get_targets("invalid_protocol")
+
+
+class TestFallbackProxy:
+    """Tests for fallback proxy lifecycle management in ProxyStateManager."""
+
+    HEAD_NODE = (HEAD_NODE_ID, "fake-head-ip", "fake-head-instance-id")
+    WORKER_NODE = ("worker-node-id-0", "fake-worker-ip-0", "fake-worker-instance-id-0")
+
+    def _create_manager_with_fallback(self, timer=None):
+        """Helper to create a ProxyStateManager with running_native_proxies=True."""
+        timer = timer or Timer()
+        cache = MockClusterNodeInfoCache()
+        cache.alive_nodes = [self.HEAD_NODE, self.WORKER_NODE]
+        manager, cluster_node_info_cache = _create_proxy_state_manager(
+            http_options=HTTPOptions(location=ProxyLocation.HeadOnly),
+            cluster_node_info_cache=cache,
+            running_native_proxies=True,
+            timer=timer,
+        )
+        return manager, cluster_node_info_cache
+
+    def test_fallback_proxy_starts_on_head_node(self):
+        """When running_native_proxies=True, a fallback proxy should be
+        started on the head node."""
+        manager, _ = self._create_manager_with_fallback()
+
+        assert manager._fallback_proxy_state is None
+
+        manager.update(proxy_nodes={HEAD_NODE_ID})
+
+        assert manager._fallback_proxy_state is not None
+        assert manager._fallback_proxy_state._node_id == HEAD_NODE_ID
+        assert manager._fallback_proxy_state._status == ProxyStatus.STARTING
+
+    def test_no_fallback_proxy_when_not_native(self):
+        """When running_native_proxies=False, no fallback proxy should be started."""
+        cache = MockClusterNodeInfoCache()
+        cache.alive_nodes = [self.HEAD_NODE]
+        manager, _ = _create_proxy_state_manager(
+            http_options=HTTPOptions(location=ProxyLocation.HeadOnly),
+            cluster_node_info_cache=cache,
+            running_native_proxies=False,
+        )
+
+        manager.update(proxy_nodes={HEAD_NODE_ID})
+
+        assert manager._fallback_proxy_state is None
+
+    def test_fallback_proxy_uses_correct_ports(self):
+        """The fallback proxy should use the fallback ports (8500/9500),
+        not the default proxy ports (8000/9000)."""
+        cache = MockClusterNodeInfoCache()
+        cache.alive_nodes = [self.HEAD_NODE, self.WORKER_NODE]
+        manager, _ = _create_proxy_state_manager(
+            http_options=HTTPOptions(location=ProxyLocation.HeadOnly),
+            cluster_node_info_cache=cache,
+            running_native_proxies=True,
+        )
+
+        with mock.patch.object(
+            manager, "_start_proxy", wraps=manager._start_proxy
+        ) as mock_start:
+            manager.update(proxy_nodes={HEAD_NODE_ID})
+
+            fallback_calls = [
+                call
+                for call in mock_start.call_args_list
+                if "fallback" in call.kwargs.get("name", "")
+            ]
+            assert len(fallback_calls) == 1
+
+            kwargs = fallback_calls[0].kwargs
+            assert kwargs["http_options"].port == RAY_SERVE_FALLBACK_PROXY_HTTP_PORT
+            assert kwargs["grpc_options"].port == RAY_SERVE_FALLBACK_PROXY_GRPC_PORT
+
+    def test_fallback_proxy_not_started_twice(self):
+        """Calling update multiple times should not create a second fallback proxy."""
+        manager, _ = self._create_manager_with_fallback()
+
+        manager.update(proxy_nodes={HEAD_NODE_ID})
+        first_fallback = manager._fallback_proxy_state
+
+        manager.update(proxy_nodes={HEAD_NODE_ID})
+        assert manager._fallback_proxy_state is first_fallback
+
+    @patch("ray.serve._private.proxy_state.PROXY_HEALTH_CHECK_UNHEALTHY_THRESHOLD", 1)
+    def test_stop_unhealthy_fallback_proxy(self):
+        """When the fallback proxy becomes unhealthy, it should be stopped
+        and the restart count incremented."""
+        manager, _ = self._create_manager_with_fallback()
+
+        manager.update(proxy_nodes={HEAD_NODE_ID})
+        assert manager._fallback_proxy_state is not None
+        assert manager._fallback_proxy_restart_count == 0
+
+        manager._fallback_proxy_state._set_status(ProxyStatus.UNHEALTHY)
+
+        manager.update(proxy_nodes={HEAD_NODE_ID})
+
+        # Unhealthy fallback proxy should be stopped and cleared
+        # A new one should be started in the same update cycle
+        assert manager._fallback_proxy_restart_count == 1
+        assert manager._fallback_proxy_state is not None
+        assert manager._fallback_proxy_state.status == ProxyStatus.STARTING
+
+    def test_stop_drained_fallback_proxy(self):
+        """When the fallback proxy enters DRAINED status, it should be removed."""
+        manager, _ = self._create_manager_with_fallback()
+
+        manager.update(proxy_nodes={HEAD_NODE_ID})
+        assert manager._fallback_proxy_state is not None
+
+        manager._fallback_proxy_state._set_status(ProxyStatus.DRAINED)
+
+        manager.update(proxy_nodes={HEAD_NODE_ID})
+
+        # Drained fallback proxy is stopped, then a new one is started
+        assert manager._fallback_proxy_restart_count == 1
+        assert manager._fallback_proxy_state is not None
+        assert manager._fallback_proxy_state.status == ProxyStatus.STARTING
+
+    def test_fallback_proxy_included_in_handles(self):
+        """get_proxy_handles() should include the fallback proxy."""
+        manager, _ = self._create_manager_with_fallback()
+
+        manager.update(proxy_nodes={HEAD_NODE_ID})
+
+        handles = manager.get_proxy_handles()
+        assert f"fallback-{HEAD_NODE_ID}" in handles
+        assert (
+            handles[f"fallback-{HEAD_NODE_ID}"]
+            is manager._fallback_proxy_state.actor_handle
+        )
+
+    def test_fallback_proxy_included_in_names(self):
+        """get_proxy_names() should include the fallback proxy."""
+        manager, _ = self._create_manager_with_fallback()
+
+        manager.update(proxy_nodes={HEAD_NODE_ID})
+
+        names = manager.get_proxy_names()
+        assert f"fallback-{HEAD_NODE_ID}" in names
+        assert (
+            names[f"fallback-{HEAD_NODE_ID}"]
+            == manager._fallback_proxy_state.actor_name
+        )
+
+    def test_fallback_proxy_included_in_alive_actor_ids(self):
+        """get_alive_proxy_actor_ids() should include the fallback proxy's actor ID."""
+        manager, _ = self._create_manager_with_fallback()
+
+        manager.update(proxy_nodes={HEAD_NODE_ID})
+
+        actor_ids = manager.get_alive_proxy_actor_ids()
+        assert manager._fallback_proxy_state.actor_id in actor_ids
+
+    def test_fallback_proxy_blocks_shutdown_readiness(self):
+        """is_ready_for_shutdown() should return False until the fallback
+        proxy is also shut down."""
+        manager, _ = self._create_manager_with_fallback()
+
+        manager.update(proxy_nodes={HEAD_NODE_ID})
+
+        # Shut down regular proxies but not the fallback
+        for proxy_state in manager._proxy_states.values():
+            proxy_state.shutdown()
+            proxy_state._actor_proxy_wrapper.shutdown = True
+
+        # Fallback proxy is still alive, so not ready for shutdown
+        assert not manager.is_ready_for_shutdown()
+
+        # Now shut down the fallback proxy
+        manager._fallback_proxy_state.shutdown()
+        manager._fallback_proxy_state._actor_proxy_wrapper.shutdown = True
+
+        assert manager.is_ready_for_shutdown()
+
+    def test_shutdown_includes_fallback_proxy(self):
+        """shutdown() should also shut down the fallback proxy."""
+        manager, _ = self._create_manager_with_fallback()
+
+        manager.update(proxy_nodes={HEAD_NODE_ID})
+        assert manager._fallback_proxy_state is not None
+
+        fallback_wrapper = manager._fallback_proxy_state._actor_proxy_wrapper
+
+        manager.shutdown()
+
+        assert manager._fallback_proxy_state._shutting_down
+        assert fallback_wrapper.shutdown is True
+
+    def test_fallback_proxy_reconciled_during_update(self):
+        """The fallback proxy should be reconciled (health checked) during each
+        update cycle."""
+        timer = MockTimer()
+        manager, _ = self._create_manager_with_fallback(timer=timer)
+
+        manager.update(proxy_nodes={HEAD_NODE_ID})
+
+        # Make fallback proxy ready
+        manager._fallback_proxy_state._actor_proxy_wrapper.is_ready_response = True
+        manager.update(proxy_nodes={HEAD_NODE_ID})
+        assert manager._fallback_proxy_state.status == ProxyStatus.HEALTHY
+
+        # Set health check to pass and verify it runs
+        manager._fallback_proxy_state._actor_proxy_wrapper.is_healthy_response = True
+        timer.advance(5)
+        manager.update(proxy_nodes={HEAD_NODE_ID})
+        assert manager._fallback_proxy_state.status == ProxyStatus.HEALTHY
+        assert manager._fallback_proxy_state._actor_proxy_wrapper.num_health_checks > 0
+
+    def test_get_fallback_proxy_targets(self):
+        """get_fallback_proxy_targets() should return the fallback proxy targets
+        when the fallback proxy is set and healthy."""
+        manager, _ = self._create_manager_with_fallback()
+
+        # The fallback proxy is not set.
+        targets = manager.get_fallback_proxy_targets()
+        assert len(targets) == 0
+
+        # The fallback proxy is not healthy.
+        state = _create_proxy_state()
+        manager._fallback_proxy_state = state
+        targets = manager.get_fallback_proxy_targets()
+        assert len(targets) == 0
+
+        # The fallback proxy is healthy.
+        state.try_update_status(ProxyStatus.HEALTHY)
+        targets = manager.get_fallback_proxy_targets()
+        assert len(targets) == 1
+
+        target = targets[RequestProtocol.HTTP]
+        assert target.ip == state.actor_details.node_ip
+        assert target.port == 8500
+        assert target.instance_id == state.actor_details.node_instance_id
+        assert target.name == state.actor_name
+
+
+if __name__ == "__main__":
+    import sys
+
+    sys.exit(pytest.main(["-v", "-s", __file__]))

python/ray/serve/tests/unit/test_replica_backpressure.py

@@ -0,0 +1,79 @@
+import asyncio
+import sys
+from unittest.mock import MagicMock
+
+import pytest
+
+from ray.serve._private.common import RequestMetadata
+from ray.serve._private.replica import Replica
+from ray.serve._private.utils import Semaphore
+
+
+def _make_metadata() -> RequestMetadata:
+    return RequestMetadata(
+        request_id="test-request",
+        internal_request_id="test-internal-request",
+        is_direct_ingress=True,
+    )
+
+
+def _make_fake_replica(max_ongoing_requests: int):
+    """Minimal stand-in exposing the real slot/queue accounting methods."""
+    fake = MagicMock()
+    fake._num_queued_requests = 0
+    fake._semaphore = Semaphore(lambda: max_ongoing_requests)
+    fake._start_request = Replica._start_request.__get__(fake, Replica)
+    fake._track_queued_request = Replica._track_queued_request.__get__(fake, Replica)
+    return fake
+
+
+async def _wait_for_semaphore_waiter(semaphore: Semaphore):
+    for _ in range(1000):
+        await asyncio.sleep(0)
+        if semaphore._waiters:
+            return
+    raise AssertionError("no coroutine blocked on the semaphore")
+
+
+class TestTrackQueuedRequest:
+    def test_release_decrements_exactly_once(self):
+        fake = _make_fake_replica(max_ongoing_requests=1)
+
+        with fake._track_queued_request() as release:
+            assert fake._num_queued_requests == 1
+            release()
+            assert fake._num_queued_requests == 0
+            # Extra calls are a no-op.
+            release()
+            assert fake._num_queued_requests == 0
+
+        # Exiting the block must not decrement below zero.
+        assert fake._num_queued_requests == 0
+
+    @pytest.mark.asyncio
+    async def test_count_released_when_cancelled_while_waiting(self):
+        """A request cancelled while blocked on the slot is released on block
+        exit, as the direct-ingress handlers wire it up."""
+        fake = _make_fake_replica(max_ongoing_requests=1)
+
+        # Hold the only slot so the request blocks while queued.
+        await fake._semaphore.acquire()
+
+        async def handler():
+            with fake._track_queued_request() as release:
+                async with fake._start_request(_make_metadata()):
+                    release()
+
+        task = asyncio.ensure_future(handler())
+        await _wait_for_semaphore_waiter(fake._semaphore)
+        assert fake._num_queued_requests == 1
+
+        task.cancel()
+        with pytest.raises(asyncio.CancelledError):
+            await task
+
+        assert fake._num_queued_requests == 0
+
+
+if __name__ == "__main__":
+    sys.exit(pytest.main(["-v", "-s", __file__]))

python/ray/serve/tests/unit/test_replica_quiesce.py

@@ -0,0 +1,235 @@
+import asyncio
+import sys
+from unittest.mock import MagicMock
+
+import pytest
+
+from ray.serve._private.common import RequestMetadata
+from ray.serve._private.replica import Replica
+
+
+def _make_metadata(*, is_direct_ingress: bool) -> RequestMetadata:
+    return RequestMetadata(
+        request_id="test-request",
+        internal_request_id="test-internal-request",
+        is_direct_ingress=is_direct_ingress,
+    )
+
+
+class FakeUvicornServer:
+    """Stand-in for `uvicorn.Server` that records when `should_exit` is set."""
+
+    def __init__(self, events):
+        # Write through __dict__ to avoid recursing into __setattr__.
+        self.__dict__["_events"] = events
+        self.__dict__["should_exit"] = False
+
+    def __setattr__(self, name, value):
+        if name == "should_exit":
+            self._events.append(("http_should_exit", value))
+        self.__dict__[name] = value
+
+
+class FakeGrpcServer:
+    """Stand-in for a `grpc.aio` server that records graceful stops."""
+
+    def __init__(self, events, name):
+        self._events = events
+        self._name = name
+
+    async def stop(self, grace):
+        self._events.append((self._name, grace))
+
+
+def _make_shutdown_fake(
+    *,
+    initialized: bool = True,
+    with_http_server: bool = False,
+    http_task=None,
+    with_grpc_server: bool = False,
+    grpc_task=None,
+    internal_grpc_port=12345,
+    grace_period_s: float = 1.0,
+):
+    """Builds a minimal stand-in for `Replica` for `perform_graceful_shutdown`.
+
+    Returns the fake and the ordered list of events it records.
+    """
+    events = []
+
+    fake = MagicMock()
+    fake._shutting_down = False
+    fake._quiescing = False
+    fake._user_callable_initialized = initialized
+    fake._ingress = False
+    fake._deployment_config.graceful_shutdown_timeout_s = grace_period_s
+    fake._direct_ingress_http_server = (
+        FakeUvicornServer(events) if with_http_server else None
+    )
+    fake._direct_ingress_http_server_task = http_task
+    fake._direct_ingress_grpc_server = (
+        FakeGrpcServer(events, "direct_ingress_grpc_stop") if with_grpc_server else None
+    )
+    fake._direct_ingress_grpc_server_task = grpc_task
+    fake._internal_grpc_port = internal_grpc_port
+    fake._server = FakeGrpcServer(events, "inter_deployment_stop")
+
+    async def drain(min_draining_period_s):
+        # Quiescing must only start AFTER the drain: during the drain the
+        # replica must keep serving normally.
+        assert fake._quiescing is False
+        events.append(("drain", min_draining_period_s))
+
+    fake._drain_ongoing_requests = drain
+
+    async def shutdown():
+        events.append(("shutdown",))
+
+    fake.shutdown = shutdown
+
+    return fake, events
+
+
+class TestCanAcceptRequestWhileQuiescing:
+    @staticmethod
+    def _bind_quiescing(fake):
+        # _can_accept_request delegates the quiescing decision to
+        # _is_replica_quiescing; wire the real helper onto the fake so it runs
+        # instead of returning a truthy MagicMock.
+        fake._is_replica_quiescing = lambda rm: Replica._is_replica_quiescing(fake, rm)
+
+    def test_handle_path_rejected_when_quiescing(self):
+        fake = MagicMock()
+        fake._quiescing = True
+        self._bind_quiescing(fake)
+
+        assert (
+            Replica._can_accept_request(fake, _make_metadata(is_direct_ingress=False))
+            is False
+        )
+
+    def test_direct_ingress_not_rejected_when_quiescing(self):
+        """Direct ingress requests must NOT be rejected while quiescing.
+
+        The HTTP/gRPC servers are shut down gracefully during quiescing, so
+        any direct ingress request that still arrives is served to
+        completion; rejecting it would surface a client-visible error
+        instead of a safe retry.
+        """
+        fake = MagicMock()
+        fake._quiescing = True
+        fake.max_queued_requests = -1
+        fake._num_queued_requests = 0
+        self._bind_quiescing(fake)
+
+        assert (
+            Replica._can_accept_request(fake, _make_metadata(is_direct_ingress=True))
+            is True
+        )
+
+    def test_handle_path_accepted_when_not_quiescing(self):
+        fake = MagicMock()
+        fake._quiescing = False
+        fake._semaphore.locked.return_value = False
+        self._bind_quiescing(fake)
+
+        assert (
+            Replica._can_accept_request(fake, _make_metadata(is_direct_ingress=False))
+            is True
+        )
+
+
+class TestPerformGracefulShutdown:
+    @pytest.mark.asyncio
+    async def test_quiesces_and_stops_servers_in_order(self):
+        """Drain → quiesce → graceful server stops → shutdown, in order."""
+        loop = asyncio.get_running_loop()
+        http_task = loop.create_future()
+        # Simulate the uvicorn serve task exiting promptly once
+        # `should_exit` is set.
+        http_task.set_result(None)
+        grpc_task = MagicMock()
+
+        fake, events = _make_shutdown_fake(
+            with_http_server=True,
+            http_task=http_task,
+            with_grpc_server=True,
+            grpc_task=grpc_task,
+            grace_period_s=1.0,
+        )
+
+        await Replica.perform_graceful_shutdown(fake)
+
+        assert fake._shutting_down is True
+        assert fake._quiescing is True
+        assert [e[0] for e in events] == [
+            "drain",
+            "http_should_exit",
+            "direct_ingress_grpc_stop",
+            "inter_deployment_stop",
+            "shutdown",
+        ]
+        # The grace passed to each stop is the REMAINING shutdown budget at
+        # that step (deadline-based), so it must be positive and within the
+        # configured budget.
+        for name, *args in events:
+            if name.endswith("_stop"):
+                assert 0.0 < args[0] <= 1.0
+        # The direct ingress gRPC server task is cancelled after the
+        # graceful stop completes.
+        assert grpc_task.cancel.called
+
+    @pytest.mark.asyncio
+    async def test_http_graceful_close_timeout_falls_back_to_cancel(self):
+        """If the HTTP server doesn't exit within the grace period, the
+        server task is cancelled (the previous abrupt behavior)."""
+        loop = asyncio.get_running_loop()
+        # A serve task that never exits on its own.
+        http_task = loop.create_future()
+
+        fake, events = _make_shutdown_fake(
+            with_http_server=True,
+            http_task=http_task,
+            grace_period_s=0.05,
+        )
+
+        await Replica.perform_graceful_shutdown(fake)
+
+        # `asyncio.wait_for` cancels the awaited task on timeout.
+        assert http_task.cancelled()
+        # Shutdown still completes despite the timeout.
+        assert fake._quiescing is True
+        assert events[-1] == ("shutdown",)
+
+    @pytest.mark.asyncio
+    async def test_abrupt_cancel_when_server_object_missing(self):
+        """If only the server task exists (no server object), fall back to
+        the abrupt cancel."""
+        http_task = MagicMock()
+        fake, events = _make_shutdown_fake(http_task=http_task)
+
+        await Replica.perform_graceful_shutdown(fake)
+
+        assert http_task.cancel.called
+        assert [e[0] for e in events] == [
+            "drain",
+            "inter_deployment_stop",
+            "shutdown",
+        ]
+
+    @pytest.mark.asyncio
+    async def test_uninitialized_replica_skips_drain_and_server_stops(self):
+        """A replica that never initialized has no servers to stop and never
+        served traffic, so only the final shutdown step runs."""
+        fake, events = _make_shutdown_fake(
+            initialized=False,
+            internal_grpc_port=None,
+        )
+
+        await Replica.perform_graceful_shutdown(fake)
+
+        assert events == [("shutdown",)]
+
+
+if __name__ == "__main__":
+    sys.exit(pytest.main(["-v", "-s", __file__]))

python/ray/serve/tests/unit/test_request_metadata.py

@@ -0,0 +1,50 @@
+import pytest
+
+from ray.serve._private.common import RequestMetadata, RequestProtocol
+
+
+def test_request_metadata():
+    """Test logic on RequestMetadata.
+
+    Ensure the default values are set correctly and both is_http_request and
+    is_grpc_request returns the correct value. When the _request_protocol is set to
+    HTTP, is_http_request should return True and is_grpc_request should return False.
+    When the _request_protocol is set to gRPC, is_http_request should return False and
+    is_grpc_request should return True.
+    """
+    request_id = "request-id"
+    internal_request_id = "internal-request-id"
+    request_metadata = RequestMetadata(
+        request_id=request_id,
+        internal_request_id=internal_request_id,
+    )
+
+    # Ensure the default values are set correctly.
+    assert request_metadata.request_id == request_id
+    assert request_metadata.call_method == "__call__"
+    assert request_metadata.route == ""
+    assert request_metadata.app_name == ""
+    assert request_metadata.multiplexed_model_id == ""
+    assert request_metadata.session_id == ""
+    assert request_metadata.is_streaming is False
+    assert request_metadata._request_protocol == RequestProtocol.UNDEFINED
+    assert request_metadata.is_http_request is False
+    assert request_metadata.is_grpc_request is False
+
+    # is_http_request and is_grpc_request returns the correct values when the
+    # _request_protocol is set to HTTP.
+    request_metadata._request_protocol = RequestProtocol.HTTP
+    assert request_metadata.is_http_request is True
+    assert request_metadata.is_grpc_request is False
+
+    # is_http_request and is_grpc_request returns the correct values when the
+    # _request_protocol is set to gRPC.
+    request_metadata._request_protocol = RequestProtocol.GRPC
+    assert request_metadata.is_http_request is False
+    assert request_metadata.is_grpc_request is True
+
+
+if __name__ == "__main__":
+    import sys
+
+    sys.exit(pytest.main(["-v", "-s", __file__]))

python/ray/serve/tests/unit/test_round_robin_router.py

@@ -0,0 +1,191 @@
+import asyncio
+import sys
+
+import pytest
+
+from ray.serve._private.common import (
+    DeploymentHandleSource,
+    RequestMetadata,
+)
+from ray.serve._private.request_router import PendingRequest
+from ray.serve._private.test_utils import (
+    FAKE_REPLICA_DEPLOYMENT_ID as DEPLOYMENT_ID,
+    FakeRunningReplica,
+    MockTimer,
+)
+from ray.serve._private.utils import generate_request_id
+from ray.serve.experimental.round_robin_router import RoundRobinRouter
+
+TIMER = MockTimer()
+
+
+def _make_request(model_id: str = "") -> PendingRequest:
+    return PendingRequest(
+        args=[],
+        kwargs={},
+        metadata=RequestMetadata(
+            request_id=generate_request_id(),
+            internal_request_id=generate_request_id(),
+            multiplexed_model_id=model_id,
+        ),
+    )
+
+
+@pytest.fixture
+def router(request) -> RoundRobinRouter:
+    if not hasattr(request, "param"):
+        request.param = {}
+
+    params = dict(request.param)
+    use_replica_queue_len_cache = params.pop("use_replica_queue_len_cache", False)
+
+    TIMER.reset()
+    request_router = RoundRobinRouter(
+        deployment_id=DEPLOYMENT_ID,
+        handle_source=DeploymentHandleSource.REPLICA,
+        self_actor_id="fake-actor-id",
+        self_actor_handle=None,
+        use_replica_queue_len_cache=use_replica_queue_len_cache,
+        get_curr_time_s=TIMER.time,
+        initial_backoff_s=0.001,
+        backoff_multiplier=1,
+        max_backoff_s=0.001,
+    )
+    request_router.initialize_state(**params)
+
+    yield request_router
+
+    assert request_router.curr_num_routing_tasks == 0
+    assert request_router.num_pending_requests == 0
+
+
+def _make_replicas(*replica_ids: str, **kwargs):
+    replicas = [FakeRunningReplica(replica_id, **kwargs) for replica_id in replica_ids]
+    for replica in replicas:
+        replica.set_queue_len_response(0)
+    return replicas
+
+
+def _ranked_replica_unique_ids(ranks):
+    return [[replica.replica_id.unique_id for replica in rank] for rank in ranks]
+
+
+def test_initialize_state_ignores_router_kwargs(router):
+    router.initialize_state(unused=True)
+
+
+def test_round_robin_counter_initialized_randomly(router):
+    assert 0 <= router._round_robin_counter < 2**31
+
+
+@pytest.mark.asyncio
+async def test_choose_replicas_round_robin_order(router):
+    replicas = _make_replicas("r2", "r0", "r1")
+    router.update_replicas(replicas)
+    router._round_robin_counter = 0
+    await asyncio.sleep(0)
+
+    assert _ranked_replica_unique_ids(
+        await router.choose_replicas(replicas, _make_request())
+    ) == [["r2"], ["r0"], ["r1"]]
+    assert _ranked_replica_unique_ids(
+        await router.choose_replicas(replicas, _make_request())
+    ) == [["r0"], ["r1"], ["r2"]]
+    assert _ranked_replica_unique_ids(
+        await router.choose_replicas(replicas, _make_request())
+    ) == [["r1"], ["r2"], ["r0"]]
+    assert _ranked_replica_unique_ids(
+        await router.choose_replicas(replicas, _make_request())
+    ) == [["r2"], ["r0"], ["r1"]]
+
+
+@pytest.mark.asyncio
+async def test_choose_replicas_ignores_request_metadata(router):
+    replicas = [
+        FakeRunningReplica("r0", model_ids={"model-a"}),
+        FakeRunningReplica("r1"),
+        FakeRunningReplica("r2", model_ids={"model-a"}),
+    ]
+    for replica in replicas:
+        replica.set_queue_len_response(0)
+    router.update_replicas(replicas)
+    router._round_robin_counter = 0
+    await asyncio.sleep(0)
+
+    assert _ranked_replica_unique_ids(
+        await router.choose_replicas(replicas, _make_request(model_id="model-a"))
+    ) == [["r0"], ["r1"], ["r2"]]
+    assert _ranked_replica_unique_ids(
+        await router.choose_replicas(replicas, _make_request(model_id="model-a"))
+    ) == [["r1"], ["r2"], ["r0"]]
+    assert _ranked_replica_unique_ids(
+        await router.choose_replicas(replicas, _make_request(model_id="model-a"))
+    ) == [["r2"], ["r0"], ["r1"]]
+
+
+@pytest.mark.parametrize(
+    "router", [{"use_replica_queue_len_cache": True}], indirect=True
+)
+@pytest.mark.asyncio
+async def test_choose_replicas_includes_cached_full_replica_first(router):
+    replicas = [
+        FakeRunningReplica("r0", max_ongoing_requests=1),
+        FakeRunningReplica("r1", max_ongoing_requests=1),
+        FakeRunningReplica("r2", max_ongoing_requests=1),
+    ]
+    for replica in replicas:
+        replica.set_queue_len_response(0)
+    router.update_replicas(replicas)
+    router.replica_queue_len_cache.update(replicas[0].replica_id, 1)
+    router._round_robin_counter = 0
+    await asyncio.sleep(0)
+
+    # Selection should preserve strict round-robin order even if fulfillment
+    # later rejects the cached-full replica and advances to the next rank.
+    assert _ranked_replica_unique_ids(
+        await router.choose_replicas(replicas, _make_request())
+    ) == [["r0"], ["r1"], ["r2"]]
+
+
+@pytest.mark.asyncio
+async def test_choose_replica_falls_back_when_current_candidate_full(router):
+    replicas = [
+        FakeRunningReplica("r0", max_ongoing_requests=1),
+        FakeRunningReplica("r1"),
+        FakeRunningReplica("r2"),
+    ]
+    replicas[0].set_queue_len_response(1)
+    replicas[1].set_queue_len_response(0)
+    replicas[2].set_queue_len_response(0)
+    router.update_replicas(replicas)
+    router._round_robin_counter = 0
+    await asyncio.sleep(0)
+
+    chosen = await router._choose_replica_for_request(_make_request())
+    assert chosen.replica_id.unique_id == "r1"
+    await asyncio.sleep(0)
+
+
+@pytest.mark.asyncio
+async def test_choose_replicas_empty_candidates(router):
+    counter = router._round_robin_counter
+    assert await router.choose_replicas([], _make_request()) == []
+    assert router._round_robin_counter == counter
+
+
+@pytest.mark.asyncio
+async def test_choose_replicas_sets_should_backoff(router):
+    # Without should_backoff, the base class tight-loops choose_replicas when
+    # every replica is at capacity (see _choose_replicas_with_backoff).
+    replicas = _make_replicas("r0", "r1")
+    router.update_replicas(replicas)
+    await asyncio.sleep(0)
+
+    request = _make_request()
+    assert request.routing_context.should_backoff is False
+    await router.choose_replicas(replicas, request)
+    assert request.routing_context.should_backoff is True
+
+
+if __name__ == "__main__":
+    sys.exit(pytest.main(["-v", "-s", __file__]))

python/ray/serve/tests/unit/test_task_consumer.py

@@ -0,0 +1,319 @@
+import sys
+import uuid
+from typing import Any, Dict, List
+from unittest.mock import MagicMock, call
+
+import pytest
+
+from ray.serve.api import deployment
+from ray.serve.schema import (
+    CeleryAdapterConfig,
+    TaskProcessorAdapter,
+    TaskProcessorConfig,
+    TaskResult,
+)
+from ray.serve.task_consumer import task_consumer, task_handler
+
+
+class MockTaskProcessorAdapter(TaskProcessorAdapter):
+    """Mock adapter for testing task processor functionality."""
+
+    def __init__(self, config: TaskProcessorConfig):
+        self._config = config
+        self._start_consumer_received: bool = False
+        self._stop_consumer_received: bool = False
+        self.register_task_handle_mock = MagicMock()
+
+    def initialize(self, consumer_concurrency: int = 3):
+        pass
+
+    def register_task_handle(self, func, name=None):
+        self.register_task_handle_mock(func, name=name)
+
+    def enqueue_task_sync(
+        self, task_name, args=None, kwargs=None, **options
+    ) -> TaskResult:
+        pass
+
+    def get_task_status_sync(self, task_id) -> TaskResult:
+        pass
+
+    def start_consumer(self, **kwargs):
+        self._start_consumer_received = True
+
+    def stop_consumer(self, timeout: float = 10.0):
+        self._stop_consumer_received = True
+
+    def cancel_task_sync(self, task_id) -> bool:
+        pass
+
+    def get_metrics_sync(self) -> Dict[str, Any]:
+        pass
+
+    def health_check_sync(self) -> List[Dict]:
+        pass
+
+
+@pytest.fixture
+def config():
+    """Provides a mock TaskProcessorConfig."""
+    queue_name = f"test_queue_{uuid.uuid4().hex}"
+    return TaskProcessorConfig(
+        queue_name=queue_name,
+        adapter_config=CeleryAdapterConfig(
+            broker_url="fake://",
+            backend_url="fake://",
+        ),
+        adapter=MockTaskProcessorAdapter,
+    )
+
+
+class TestTaskHandlerDecorator:
+    """Test the task_handler decorator."""
+
+    def _create_and_test_handler(self, decorator_args=None, expected_name=None):
+        """Helper to create and test a task handler."""
+        mock = MagicMock()
+
+        if decorator_args is None:
+
+            @task_handler
+            def test_handler():
+                mock()
+
+        else:
+
+            @task_handler(**decorator_args)
+            def test_handler():
+                mock()
+
+        test_handler()
+
+        assert mock.call_count == 1
+        assert test_handler._task_name == expected_name
+
+    def test_task_handler_decorator_with_name(self):
+        self._create_and_test_handler(
+            decorator_args={"name": "my_task"}, expected_name="my_task"
+        )
+
+    def test_task_handler_decorator_without_name(self):
+        self._create_and_test_handler(expected_name="test_handler")
+
+    @pytest.mark.parametrize("invalid_name", ["", "   ", 123])
+    def test_task_handler_decorator_invalid_name(self, invalid_name):
+        """Test various invalid task names."""
+        with pytest.raises(
+            ValueError,
+            match=f"Task name must be a non-empty string, got {invalid_name}",
+        ):
+
+            @task_handler(name=invalid_name)
+            def my_task_handler():
+                pass
+
+    def test_task_handler_on_callable_object_without_name_attr(self):
+        """Test that AttributeError is raised for callables with no __name__."""
+
+        class MyCallable:
+            """A simple callable class without a __name__ attribute on instances."""
+
+            def __call__(self):
+                pass
+
+        with pytest.raises(AttributeError):
+            task_handler(MyCallable())
+
+
+class TestTaskConsumerDecorator:
+    """Test the task_consumer decorator."""
+
+    def _verify_and_cleanup(self, instance, expected_calls=None):
+        """Verify consumer and cleanup instance."""
+        instance.initialize_callable(5)
+        adapter = instance._adapter
+        assert adapter._start_consumer_received
+
+        if expected_calls is not None:
+            if expected_calls:
+                calls = [call(method, name=name) for method, name in expected_calls]
+                adapter.register_task_handle_mock.assert_has_calls(
+                    calls, any_order=False
+                )
+                assert adapter.register_task_handle_mock.call_count == len(
+                    expected_calls
+                )
+            else:
+                adapter.register_task_handle_mock.assert_not_called()
+
+        del instance
+
+    def _run_consumer_test(
+        self, config, consumer_class_factory, expected_calls_factory=None
+    ):
+        """Run a consumer test with factory functions."""
+        consumer_class = consumer_class_factory(config)
+        instance = consumer_class()
+
+        expected_calls = (
+            expected_calls_factory(instance) if expected_calls_factory else None
+        )
+
+        self._verify_and_cleanup(instance, expected_calls)
+
+    def test_task_consumer_basic(self, config):
+        """Test basic functionality of the task_consumer decorator."""
+
+        def make_consumer(cfg):
+            @task_consumer(task_processor_config=cfg)
+            class MyConsumer:
+                @task_handler
+                def my_task(self):
+                    pass
+
+            return MyConsumer
+
+        self._run_consumer_test(
+            config, make_consumer, lambda inst: [(inst.my_task, "my_task")]
+        )
+
+    def test_task_consumer_multiple_handlers(self, config):
+        """Test with multiple task handlers."""
+
+        def make_consumer(cfg):
+            @task_consumer(task_processor_config=cfg)
+            class MyConsumer:
+                @task_handler
+                def task1(self):
+                    pass
+
+                @task_handler
+                def task2(self):
+                    pass
+
+            return MyConsumer
+
+        self._run_consumer_test(
+            config,
+            make_consumer,
+            lambda inst: [(inst.task1, "task1"), (inst.task2, "task2")],
+        )
+
+    def test_task_consumer_custom_names(self, config):
+        """Test task handlers with and without custom names."""
+
+        def make_consumer(cfg):
+            @task_consumer(task_processor_config=cfg)
+            class MyConsumer:
+                @task_handler(name="custom_task")
+                def task1(self):
+                    pass
+
+                @task_handler
+                def task2(self):
+                    pass
+
+            return MyConsumer
+
+        self._run_consumer_test(
+            config,
+            make_consumer,
+            lambda inst: [(inst.task1, "custom_task"), (inst.task2, "task2")],
+        )
+
+    def test_task_consumer_init_args(self, config):
+        """Test that __init__ arguments are passed correctly."""
+
+        @task_consumer(task_processor_config=config)
+        class MyConsumer:
+            def __init__(self, value):
+                self.value = value
+
+        instance = MyConsumer(value=42)
+        assert instance.value == 42
+        self._verify_and_cleanup(instance)
+
+    def test_task_consumer_no_handlers(self, config):
+        """Test with a class that has no task handlers."""
+
+        def make_consumer(cfg):
+            @task_consumer(task_processor_config=cfg)
+            class MyConsumer:
+                def some_method(self):
+                    pass
+
+            return MyConsumer
+
+        self._run_consumer_test(config, make_consumer, lambda inst: [])
+
+    def test_task_consumer_inheritance(self, config):
+        """Test that inherited task handlers are registered."""
+
+        def make_consumer(cfg):
+            class BaseConsumer:
+                @task_handler
+                def base_task(self):
+                    pass
+
+            @task_consumer(task_processor_config=cfg)
+            class DerivedConsumer(BaseConsumer):
+                @task_handler
+                def derived_task(self):
+                    pass
+
+            return DerivedConsumer
+
+        self._run_consumer_test(
+            config,
+            make_consumer,
+            lambda inst: [
+                (inst.base_task, "base_task"),
+                (inst.derived_task, "derived_task"),
+            ],
+        )
+
+    def test_task_consumer_no_args_decorator(self):
+        """Test using @task_consumer without arguments raises TypeError."""
+        with pytest.raises(TypeError):
+
+            @task_consumer
+            class MyConsumer:
+                pass
+
+
+def test_default_deployment_name_stays_same_with_task_consumer(config):
+    """Test that the default deployment name is the class name when using task_consumer with serve.deployment."""
+
+    @deployment
+    @task_consumer(task_processor_config=config)
+    class MyTaskConsumer:
+        @task_handler
+        def my_task(self):
+            pass
+
+    # The deployment name should default to the class name
+    assert MyTaskConsumer.name == "MyTaskConsumer"
+
+
+def test_task_consumer_preserves_metadata(config):
+    class OriginalConsumer:
+        """Docstring for a task consumer."""
+
+        value: int
+
+    wrapped_cls = task_consumer(task_processor_config=config)(OriginalConsumer)
+
+    assert wrapped_cls.__name__ == OriginalConsumer.__name__
+    assert wrapped_cls.__qualname__ == OriginalConsumer.__qualname__
+    assert wrapped_cls.__module__ == OriginalConsumer.__module__
+    assert wrapped_cls.__doc__ == OriginalConsumer.__doc__
+    assert (
+        wrapped_cls.__annotations__["value"]
+        == OriginalConsumer.__annotations__["value"]
+    )
+    assert wrapped_cls.__annotations__["_adapter"] is TaskProcessorAdapter
+    assert getattr(wrapped_cls, "__wrapped__", None) is OriginalConsumer
+
+
+if __name__ == "__main__":
+    sys.exit(pytest.main(["-v", "-s", __file__]))

python/ray/serve/tests/unit/test_user_callable_wrapper.py

@@ -0,0 +1,858 @@
+import asyncio
+import pickle
+import sys
+import threading
+from dataclasses import dataclass
+from typing import Any, AsyncGenerator, Callable, Dict, Generator, Optional, Tuple
+
+import pytest
+from fastapi import FastAPI
+from starlette.requests import Request
+from starlette.responses import PlainTextResponse
+
+import ray.serve._private.replica as replica_module
+from ray import serve
+from ray.serve._private.common import (
+    DeploymentID,
+    RequestMetadata,
+    RequestProtocol,
+)
+from ray.serve._private.config import DeploymentConfig
+from ray.serve._private.http_util import ASGIReceiveProxy
+from ray.serve._private.replica import UserCallableWrapper
+from ray.serve.generated import serve_pb2
+
+
+class BasicClass:
+    def __call__(self, suffix: Optional[str] = None, raise_exception: bool = False):
+        if raise_exception:
+            raise RuntimeError("uh-oh!")
+
+        return "hi" + (suffix if suffix is not None else "")
+
+    async def call_async(
+        self, suffix: Optional[str] = None, raise_exception: bool = False
+    ):
+        if raise_exception:
+            raise RuntimeError("uh-oh!")
+
+        return "hi" + (suffix if suffix is not None else "")
+
+    def call_generator(
+        self, n: int, raise_exception: bool = False
+    ) -> Generator[int, None, None]:
+        for i in range(n):
+            yield i
+
+            if raise_exception:
+                raise RuntimeError("uh-oh!")
+
+    async def call_async_generator(
+        self, n: int, raise_exception: bool = False
+    ) -> AsyncGenerator[int, None]:
+        for i in range(n):
+            yield i
+
+            if raise_exception:
+                raise RuntimeError("uh-oh!")
+
+
+def basic_sync_function(suffix: Optional[str] = None, raise_exception: bool = False):
+    if raise_exception:
+        raise RuntimeError("uh-oh!")
+
+    return "hi" + (suffix if suffix is not None else "")
+
+
+async def basic_async_function(
+    suffix: Optional[str] = None, raise_exception: bool = False
+):
+    if raise_exception:
+        raise RuntimeError("uh-oh!")
+
+    return "hi" + (suffix if suffix is not None else "")
+
+
+def basic_sync_generator(n: int, raise_exception: bool = False):
+    for i in range(n):
+        yield i
+
+        if raise_exception:
+            raise RuntimeError("uh-oh!")
+
+
+async def basic_async_generator(n: int, raise_exception: bool = False):
+    for i in range(n):
+        yield i
+
+        if raise_exception:
+            raise RuntimeError("uh-oh!")
+
+
+def _make_user_callable_wrapper(
+    callable: Optional[Callable] = None,
+    *,
+    init_args: Optional[Tuple[Any]] = None,
+    init_kwargs: Optional[Dict[str, Any]] = None,
+    run_sync_methods_in_threadpool: bool = False,
+    run_user_code_in_separate_thread: bool = True,
+) -> UserCallableWrapper:
+    return UserCallableWrapper(
+        callable if callable is not None else BasicClass,
+        init_args or tuple(),
+        init_kwargs or dict(),
+        deployment_id=DeploymentID(name="test_name"),
+        run_sync_methods_in_threadpool=run_sync_methods_in_threadpool,
+        run_user_code_in_separate_thread=run_user_code_in_separate_thread,
+        local_testing_mode=False,
+        deployment_config=DeploymentConfig(max_ongoing_requests=100),
+        actor_id="test-actor-id",
+        ray_actor_options={},
+    )
+
+
+def _make_request_metadata(
+    *,
+    call_method: Optional[str] = None,
+    is_http_request: bool = False,
+    is_grpc_request: bool = False,
+    is_streaming: bool = False,
+) -> RequestMetadata:
+    protocol = RequestProtocol.UNDEFINED
+    if is_http_request:
+        protocol = RequestProtocol.HTTP
+    if is_grpc_request:
+        protocol = RequestProtocol.GRPC
+
+    return RequestMetadata(
+        request_id="test_request",
+        internal_request_id="test_internal_request",
+        call_method=call_method if call_method is not None else "__call__",
+        _request_protocol=protocol,
+        is_streaming=is_streaming,
+    )
+
+
+@pytest.mark.parametrize("run_user_code_in_separate_thread", [False, True])
+@pytest.mark.asyncio
+async def test_calling_initialize_twice(run_user_code_in_separate_thread: bool):
+    user_callable_wrapper = _make_user_callable_wrapper(
+        run_user_code_in_separate_thread=run_user_code_in_separate_thread
+    )
+
+    await user_callable_wrapper.initialize_callable()
+    assert isinstance(user_callable_wrapper.user_callable, BasicClass)
+    with pytest.raises(RuntimeError):
+        await user_callable_wrapper.initialize_callable()
+
+
+@pytest.mark.parametrize("run_user_code_in_separate_thread", [False, True])
+@pytest.mark.asyncio
+async def test_calling_methods_before_initialize(
+    run_user_code_in_separate_thread: bool,
+):
+    user_callable_wrapper = _make_user_callable_wrapper(
+        run_user_code_in_separate_thread=run_user_code_in_separate_thread
+    )
+
+    with pytest.raises(RuntimeError):
+        await user_callable_wrapper.call_user_method(None, tuple(), dict())
+
+    with pytest.raises(RuntimeError):
+        await user_callable_wrapper.call_user_health_check()
+
+    with pytest.raises(RuntimeError):
+        await user_callable_wrapper.call_reconfigure(None, rank=0)
+
+
+@pytest.mark.parametrize("run_user_code_in_separate_thread", [False, True])
+@pytest.mark.parametrize("run_sync_methods_in_threadpool", [False, True])
+@pytest.mark.asyncio
+async def test_basic_class_callable(
+    run_user_code_in_separate_thread: bool, run_sync_methods_in_threadpool: bool
+):
+    user_callable_wrapper = _make_user_callable_wrapper(
+        run_sync_methods_in_threadpool=run_sync_methods_in_threadpool,
+        run_user_code_in_separate_thread=run_user_code_in_separate_thread,
+    )
+
+    await user_callable_wrapper.initialize_callable()
+
+    # Call non-generator method with is_streaming.
+    request_metadata = _make_request_metadata(is_streaming=True)
+    with pytest.raises(TypeError, match="did not return a generator."):
+        async for _ in user_callable_wrapper.call_user_generator(
+            request_metadata, tuple(), dict()
+        ):
+            pass
+
+    # Test calling default sync `__call__` method.
+    request_metadata = _make_request_metadata()
+    assert (
+        await user_callable_wrapper.call_user_method(request_metadata, tuple(), dict())
+    ) == "hi"
+    assert (
+        await user_callable_wrapper.call_user_method(
+            request_metadata, ("-arg",), dict()
+        )
+        == "hi-arg"
+    )
+    assert (
+        await user_callable_wrapper.call_user_method(
+            request_metadata, tuple(), {"suffix": "-kwarg"}
+        )
+        == "hi-kwarg"
+    )
+    with pytest.raises(RuntimeError, match="uh-oh"):
+        await user_callable_wrapper.call_user_method(
+            request_metadata, tuple(), {"raise_exception": True}
+        )
+
+    # Call non-generator async method with is_streaming.
+    request_metadata = _make_request_metadata(
+        call_method="call_async", is_streaming=True
+    )
+    with pytest.raises(TypeError, match="did not return a generator."):
+        async for _ in user_callable_wrapper.call_user_generator(
+            request_metadata, tuple(), dict()
+        ):
+            pass
+
+    # Test calling `call_async` method.
+    request_metadata = _make_request_metadata(call_method="call_async")
+    assert (
+        await user_callable_wrapper.call_user_method(request_metadata, tuple(), dict())
+        == "hi"
+    )
+    assert (
+        await user_callable_wrapper.call_user_method(
+            request_metadata, ("-arg",), dict()
+        )
+        == "hi-arg"
+    )
+    assert (
+        await user_callable_wrapper.call_user_method(
+            request_metadata, tuple(), {"suffix": "-kwarg"}
+        )
+        == "hi-kwarg"
+    )
+    with pytest.raises(RuntimeError, match="uh-oh"):
+        await user_callable_wrapper.call_user_method(
+            request_metadata, tuple(), {"raise_exception": True}
+        )
+
+
+@pytest.mark.parametrize("run_user_code_in_separate_thread", [False, True])
+@pytest.mark.parametrize("run_sync_methods_in_threadpool", [False, True])
+@pytest.mark.asyncio
+async def test_basic_class_callable_generators(
+    run_sync_methods_in_threadpool: bool, run_user_code_in_separate_thread: bool
+):
+    user_callable_wrapper = _make_user_callable_wrapper(
+        run_sync_methods_in_threadpool=run_sync_methods_in_threadpool,
+        run_user_code_in_separate_thread=run_user_code_in_separate_thread,
+    )
+    await user_callable_wrapper.initialize_callable()
+
+    result_list = []
+
+    # Call sync generator without is_streaming.
+    request_metadata = _make_request_metadata(
+        call_method="call_generator", is_streaming=False
+    )
+    with pytest.raises(
+        TypeError, match="Method 'call_generator' returned a generator."
+    ):
+        await user_callable_wrapper.call_user_method(
+            request_metadata,
+            (10,),
+            dict(),
+        )
+
+    # Call sync generator.
+    request_metadata = _make_request_metadata(
+        call_method="call_generator", is_streaming=True
+    )
+    async for result in user_callable_wrapper.call_user_generator(
+        request_metadata, (10,), dict()
+    ):
+        result_list.append(result)
+    assert result_list == list(range(10))
+    result_list.clear()
+
+    # Call sync generator raising exception.
+    with pytest.raises(RuntimeError, match="uh-oh"):
+        async for result in user_callable_wrapper.call_user_generator(
+            request_metadata,
+            (10,),
+            {"raise_exception": True},
+        ):
+            result_list.append(result)
+    assert result_list == [0]
+    result_list.clear()
+
+    # Call async generator without is_streaming.
+    request_metadata = _make_request_metadata(
+        call_method="call_async_generator", is_streaming=False
+    )
+    with pytest.raises(
+        TypeError, match="Method 'call_async_generator' returned a generator."
+    ):
+        await user_callable_wrapper.call_user_method(
+            request_metadata,
+            (10,),
+            dict(),
+        )
+
+    # Call async generator.
+    request_metadata = _make_request_metadata(
+        call_method="call_async_generator", is_streaming=True
+    )
+    async for result in user_callable_wrapper.call_user_generator(
+        request_metadata, (10,), dict()
+    ):
+        result_list.append(result)
+    assert result_list == list(range(10))
+    result_list.clear()
+
+    # Call async generator raising exception.
+    with pytest.raises(RuntimeError, match="uh-oh"):
+        async for result in user_callable_wrapper.call_user_generator(
+            request_metadata,
+            (10,),
+            {"raise_exception": True},
+        ):
+            result_list.append(result)
+    assert result_list == [0]
+
+
+@pytest.mark.parametrize("run_user_code_in_separate_thread", [False, True])
+@pytest.mark.parametrize("run_sync_methods_in_threadpool", [False, True])
+@pytest.mark.parametrize("fn", [basic_sync_function, basic_async_function])
+@pytest.mark.asyncio
+async def test_basic_function_callable(
+    fn: Callable,
+    run_sync_methods_in_threadpool: bool,
+    run_user_code_in_separate_thread: bool,
+):
+    user_callable_wrapper = _make_user_callable_wrapper(
+        fn,
+        run_sync_methods_in_threadpool=run_sync_methods_in_threadpool,
+        run_user_code_in_separate_thread=run_user_code_in_separate_thread,
+    )
+    await user_callable_wrapper.initialize_callable()
+
+    # Call non-generator function with is_streaming.
+    request_metadata = _make_request_metadata(is_streaming=True)
+    with pytest.raises(TypeError, match="did not return a generator."):
+        async for _ in user_callable_wrapper.call_user_generator(
+            request_metadata, tuple(), dict()
+        ):
+            pass
+
+    request_metadata = _make_request_metadata()
+    assert (
+        await user_callable_wrapper.call_user_method(request_metadata, tuple(), dict())
+    ) == "hi"
+    assert (
+        await user_callable_wrapper.call_user_method(
+            request_metadata, ("-arg",), dict()
+        )
+    ) == "hi-arg"
+    assert (
+        await user_callable_wrapper.call_user_method(
+            request_metadata, tuple(), {"suffix": "-kwarg"}
+        )
+    ) == "hi-kwarg"
+    with pytest.raises(RuntimeError, match="uh-oh"):
+        await user_callable_wrapper.call_user_method(
+            request_metadata, tuple(), {"raise_exception": True}
+        )
+
+
+@pytest.mark.parametrize("run_user_code_in_separate_thread", [False, True])
+@pytest.mark.parametrize("run_sync_methods_in_threadpool", [False, True])
+@pytest.mark.parametrize("fn", [basic_sync_generator, basic_async_generator])
+@pytest.mark.asyncio
+async def test_basic_function_callable_generators(
+    fn: Callable,
+    run_sync_methods_in_threadpool: bool,
+    run_user_code_in_separate_thread: bool,
+):
+    user_callable_wrapper = _make_user_callable_wrapper(
+        fn,
+        run_sync_methods_in_threadpool=run_sync_methods_in_threadpool,
+        run_user_code_in_separate_thread=run_user_code_in_separate_thread,
+    )
+    await user_callable_wrapper.initialize_callable()
+
+    result_list = []
+
+    # Call generator function without is_streaming.
+    request_metadata = _make_request_metadata(is_streaming=False)
+    with pytest.raises(
+        TypeError, match=f"Method '{fn.__name__}' returned a generator."
+    ):
+        await user_callable_wrapper.call_user_method(
+            request_metadata,
+            (10,),
+            dict(),
+        )
+
+    # Call generator function.
+    request_metadata = _make_request_metadata(
+        call_method="call_generator", is_streaming=True
+    )
+    async for result in user_callable_wrapper.call_user_generator(
+        request_metadata, (10,), dict()
+    ):
+        result_list.append(result)
+    assert result_list == list(range(10))
+    result_list.clear()
+
+    # Call generator function raising exception.
+    with pytest.raises(RuntimeError, match="uh-oh"):
+        async for result in user_callable_wrapper.call_user_generator(
+            request_metadata,
+            (10,),
+            {"raise_exception": True},
+        ):
+            result_list.append(result)
+    assert result_list == [0]
+
+
+@pytest.mark.parametrize("run_user_code_in_separate_thread", [False, True])
+@pytest.mark.asyncio
+async def test_callable_with_async_init(run_user_code_in_separate_thread: bool):
+    class AsyncInitializer:
+        async def __init__(self, msg: str):
+            await asyncio.sleep(0.001)
+            self._msg = msg
+
+        def __call__(self) -> str:
+            return self._msg
+
+    msg = "hello world"
+    user_callable_wrapper = _make_user_callable_wrapper(
+        AsyncInitializer,
+        init_args=(msg,),
+        run_user_code_in_separate_thread=run_user_code_in_separate_thread,
+    )
+    await user_callable_wrapper.initialize_callable()
+    request_metadata = _make_request_metadata()
+    assert (
+        await user_callable_wrapper.call_user_method(request_metadata, tuple(), dict())
+    ) == msg
+
+
+@pytest.mark.parametrize("run_user_code_in_separate_thread", [False, True])
+@pytest.mark.parametrize("async_del", [False, True])
+@pytest.mark.asyncio
+async def test_destructor_only_called_once(
+    async_del: bool, run_user_code_in_separate_thread: bool
+):
+    num_destructor_calls = 0
+
+    if async_del:
+
+        class DestroyerOfNothing:
+            async def __del__(self) -> str:
+                nonlocal num_destructor_calls
+                num_destructor_calls += 1
+
+    else:
+
+        class DestroyerOfNothing:
+            def __del__(self) -> str:
+                nonlocal num_destructor_calls
+                num_destructor_calls += 1
+
+    user_callable_wrapper = _make_user_callable_wrapper(
+        DestroyerOfNothing,
+        run_user_code_in_separate_thread=run_user_code_in_separate_thread,
+    )
+    await user_callable_wrapper.initialize_callable()
+
+    # Call `call_destructor` many times in parallel; only the first one should actually
+    # run the `__del__` method.
+    await asyncio.gather(*[user_callable_wrapper.call_destructor() for _ in range(100)])
+    assert num_destructor_calls == 1
+
+
+class gRPCClass:
+    def greet(self, msg: serve_pb2.UserDefinedMessage):
+        return serve_pb2.UserDefinedResponse(greeting=f"Hello {msg.greeting}!")
+
+    def stream(self, msg: serve_pb2.UserDefinedMessage):
+        for i in range(10):
+            yield serve_pb2.UserDefinedResponse(greeting=f"Hello {msg.greeting} {i}!")
+
+
+@pytest.mark.parametrize("run_user_code_in_separate_thread", [False, True])
+@pytest.mark.parametrize("run_sync_methods_in_threadpool", [False, True])
+@pytest.mark.asyncio
+async def test_grpc_unary_request(
+    run_sync_methods_in_threadpool: bool, run_user_code_in_separate_thread: bool
+):
+    user_callable_wrapper = _make_user_callable_wrapper(
+        gRPCClass,
+        run_sync_methods_in_threadpool=run_sync_methods_in_threadpool,
+        run_user_code_in_separate_thread=run_user_code_in_separate_thread,
+    )
+    await user_callable_wrapper.initialize_callable()
+
+    request_metadata = _make_request_metadata(call_method="greet", is_grpc_request=True)
+    result = await user_callable_wrapper.call_user_method(
+        request_metadata, (serve_pb2.UserDefinedResponse(greeting="world"),), dict()
+    )
+    assert isinstance(result, serve_pb2.UserDefinedResponse)
+    assert result.greeting == "Hello world!"
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("run_user_code_in_separate_thread", [False, True])
+@pytest.mark.parametrize("run_sync_methods_in_threadpool", [False, True])
+async def test_grpc_streaming_request(
+    run_sync_methods_in_threadpool: bool, run_user_code_in_separate_thread: bool
+):
+    user_callable_wrapper = _make_user_callable_wrapper(
+        gRPCClass,
+        run_sync_methods_in_threadpool=run_sync_methods_in_threadpool,
+        run_user_code_in_separate_thread=run_user_code_in_separate_thread,
+    )
+    await user_callable_wrapper.initialize_callable()
+
+    result_list = []
+
+    request_metadata = _make_request_metadata(
+        call_method="stream", is_grpc_request=True, is_streaming=True
+    )
+    async for result in user_callable_wrapper.call_user_generator(
+        request_metadata,
+        (serve_pb2.UserDefinedResponse(greeting="world"),),
+        dict(),
+    ):
+        result_list.append(result)
+
+    assert len(result_list) == 10
+    for i, result in enumerate(result_list):
+        assert isinstance(result, serve_pb2.UserDefinedResponse)
+        assert result.greeting == f"Hello world {i}!"
+
+
+class RawRequestHandler:
+    async def __call__(self, request: Request) -> str:
+        msg = await request.body()
+        return PlainTextResponse(f"Hello {msg}!")
+
+
+app = FastAPI()
+
+
+@serve.ingress(app)
+class FastAPIRequestHandler:
+    @app.get("/")
+    async def handle_root(self, request: Request) -> str:
+        msg = await request.body()
+        return PlainTextResponse(f"Hello {msg}!")
+
+
+@pytest.mark.parametrize("run_user_code_in_separate_thread", [False, True])
+@pytest.mark.parametrize("callable", [RawRequestHandler, FastAPIRequestHandler])
+@pytest.mark.asyncio
+async def test_http_handler(
+    callable: Callable, monkeypatch, run_user_code_in_separate_thread: bool
+):
+    user_callable_wrapper = _make_user_callable_wrapper(
+        callable, run_user_code_in_separate_thread=run_user_code_in_separate_thread
+    )
+    await user_callable_wrapper.initialize_callable()
+
+    @dataclass
+    class MockReplicaContext:
+        servable_object: Callable
+
+    monkeypatch.setattr(
+        serve,
+        "get_replica_context",
+        lambda: MockReplicaContext(user_callable_wrapper.user_callable),
+    )
+
+    asgi_scope = {
+        "type": "http",
+        "asgi": {"version": "3.0", "spec_version": "2.1"},
+        "http_version": "1.1",
+        "server": ("127.0.0.1", 8000),
+        "client": ("127.0.0.1", 51517),
+        "scheme": "http",
+        "method": "GET",
+        "root_path": "",
+        "path": "/",
+        "raw_path": b"/",
+        "query_string": b"",
+        "headers": [
+            (b"host", b"localhost:8000"),
+            (b"user-agent", b"curl/8.1.2"),
+            (b"accept", b"*/*"),
+            (b"x-request-id", b"e45c04ad-bcd8-434f-8998-05689227e103"),
+        ],
+    }
+
+    asgi_messages = [
+        {"type": "http.request", "body": b'"world"', "more_body": False},
+        {"type": "http.disconnect"},
+    ]
+
+    async def receive_asgi_messages(_: str):
+        return pickle.dumps(asgi_messages)
+
+    result_list = []
+
+    request_metadata = _make_request_metadata(is_http_request=True, is_streaming=True)
+    async for result in user_callable_wrapper.call_http_entrypoint(
+        request_metadata,
+        lambda *args: None,
+        asgi_scope,
+        ASGIReceiveProxy(asgi_scope, request_metadata, receive_asgi_messages),
+    ):
+        result_list.extend(result)
+
+    assert result_list[0]["type"] == "http.response.start"
+    assert result_list[0]["status"] == 200
+    assert "headers" in result_list[0]
+    assert result_list[1] == {
+        "type": "http.response.body",
+        "body": b"Hello b'\"world\"'!",
+    }
+
+
+failing_init_app = FastAPI()
+
+
+@serve.ingress(failing_init_app)
+class FailingFastAPIRequestHandler:
+    def __init__(self):
+        raise RuntimeError("init failed")
+
+
+@pytest.mark.parametrize("run_user_code_in_separate_thread", [False, True])
+@pytest.mark.asyncio
+async def test_fastapi_init_failure_destructor_is_noop(
+    monkeypatch, run_user_code_in_separate_thread: bool
+):
+    logged_messages = []
+
+    def mock_exception(msg, *args, **kwargs):
+        logged_messages.append(msg)
+
+    monkeypatch.setattr(replica_module.logger, "exception", mock_exception)
+
+    user_callable_wrapper = _make_user_callable_wrapper(
+        FailingFastAPIRequestHandler,
+        run_user_code_in_separate_thread=run_user_code_in_separate_thread,
+    )
+
+    with pytest.raises(RuntimeError, match="init failed"):
+        await user_callable_wrapper.initialize_callable()
+
+    await user_callable_wrapper.call_destructor()
+
+    assert logged_messages == []
+
+
+class TestSeparateThread:
+    @pytest.mark.asyncio
+    @pytest.mark.parametrize("run_sync_methods_in_threadpool", [False, True])
+    async def test_user_code_runs_on_separate_loop(
+        self, run_sync_methods_in_threadpool: bool
+    ):
+        main_loop = asyncio.get_running_loop()
+
+        class GetLoop:
+            def __init__(self):
+                self._constructor_loop = asyncio.get_running_loop()
+
+            async def check_health(self):
+                check_health_loop = asyncio.get_running_loop()
+                assert (
+                    check_health_loop == self._constructor_loop
+                ), "User constructor and health check should run on the same loop."
+                return check_health_loop
+
+            async def call_async(self) -> Optional[asyncio.AbstractEventLoop]:
+                user_method_loop = asyncio.get_running_loop()
+                assert (
+                    user_method_loop == self._constructor_loop
+                ), "User constructor and other methods should run on the same loop."
+
+                return user_method_loop
+
+            def call_sync(self):
+                if run_sync_methods_in_threadpool:
+                    with pytest.raises(RuntimeError, match="no running event loop"):
+                        asyncio.get_running_loop()
+
+                    user_method_loop = None
+                else:
+                    user_method_loop = asyncio.get_running_loop()
+                    assert (
+                        user_method_loop == self._constructor_loop
+                    ), "User constructor and other methods should run on the same loop."
+
+                return user_method_loop
+
+        user_callable_wrapper = _make_user_callable_wrapper(
+            GetLoop,
+            run_sync_methods_in_threadpool=run_sync_methods_in_threadpool,
+            run_user_code_in_separate_thread=True,
+        )
+        await user_callable_wrapper.initialize_callable()
+
+        # Async methods should all run on the same loop.
+        request_metadata = _make_request_metadata(call_method="call_async")
+        user_code_loop = await user_callable_wrapper.call_user_method(
+            request_metadata, tuple(), dict()
+        )
+        assert isinstance(user_code_loop, asyncio.AbstractEventLoop)
+        assert user_code_loop != main_loop
+
+        # Sync methods should run on the same loop if run_sync_methods_in_threadpool is off,
+        # else run in no asyncio loop.
+        request_metadata = _make_request_metadata(call_method="call_sync")
+        user_code_loop = await user_callable_wrapper.call_user_method(
+            request_metadata, tuple(), dict()
+        )
+        if run_sync_methods_in_threadpool:
+            assert user_code_loop is None
+        else:
+            assert isinstance(user_code_loop, asyncio.AbstractEventLoop)
+            assert user_code_loop != main_loop
+
+        # `check_health` method asserts that it runs on the correct loop.
+        await user_callable_wrapper.call_user_health_check()
+
+    @pytest.mark.asyncio
+    async def test_no_user_health_check_not_blocked(self):
+        """
+        If there is no user-defined health check, call_user_health_check() returns None
+        and does not interact with the user code event loop at all. The watchdog
+        (RAY_SERVE_USER_HEALTH_CHECK_PROBE_MAX_FAIL > 0) handles loop-stall detection
+        separately. Without it enabled, a blocked loop does not affect health checks.
+        """
+        sync_event = threading.Event()
+
+        class LoopBlocker:
+            async def __call__(self) -> str:
+                # Block the loop until the event is set.
+                sync_event.wait()
+                return "Sorry I got stuck!"
+
+        user_callable_wrapper = _make_user_callable_wrapper(
+            LoopBlocker,
+            run_user_code_in_separate_thread=True,
+        )
+        await user_callable_wrapper.initialize_callable()
+        request_metadata = _make_request_metadata()
+        blocked_future = user_callable_wrapper.call_user_method(
+            request_metadata, tuple(), dict()
+        )
+        _, pending = await asyncio.wait([blocked_future], timeout=0.01)
+        assert len(pending) == 1
+
+        for _ in range(100):
+            # If this called something on the event loop, it'd be blocked.
+            # Instead, call_user_health_check returns None when there's no user
+            # health check configured and the probe counter hasn't exceeded the
+            # threshold.
+            assert user_callable_wrapper.call_user_health_check() is None
+
+        sync_event.set()
+        assert await blocked_future == "Sorry I got stuck!"
+
+
+@pytest.mark.asyncio
+async def test_user_loop_watchdog_not_started_when_max_fail_zero(monkeypatch):
+    monkeypatch.setattr(replica_module, "USER_HEALTH_CHECK_PROBE_MAX_FAIL", 0)
+    w = _make_user_callable_wrapper(
+        run_sync_methods_in_threadpool=False,
+        run_user_code_in_separate_thread=True,
+    )
+    await w.initialize_callable()
+    w.start_user_loop_watchdog(asyncio.get_running_loop())
+    assert w._user_loop_probe_task is None
+
+
+@pytest.mark.asyncio
+async def test_user_loop_watchdog_started_when_eligible(monkeypatch):
+    monkeypatch.setattr(replica_module, "USER_HEALTH_CHECK_PROBE_MAX_FAIL", 1)
+    monkeypatch.setattr(replica_module, "USER_HEALTH_CHECK_PROBE_INTERVAL_S", 3600.0)
+    w = _make_user_callable_wrapper(
+        run_sync_methods_in_threadpool=False,
+        run_user_code_in_separate_thread=True,
+    )
+    await w.initialize_callable()
+    w.start_user_loop_watchdog(asyncio.get_running_loop())
+    assert w._user_loop_probe_task is not None
+    w.stop_user_loop_watchdog()
+    await asyncio.sleep(0.01)
+    assert w._user_loop_probe_task is None
+
+
+@pytest.mark.asyncio
+async def test_user_loop_watchdog_not_started_with_user_check_health(monkeypatch):
+    monkeypatch.setattr(replica_module, "USER_HEALTH_CHECK_PROBE_MAX_FAIL", 2)
+
+    class WithHealth:
+        async def check_health(self):
+            pass
+
+        def __call__(self):
+            return "ok"
+
+    w = _make_user_callable_wrapper(
+        WithHealth,
+        run_sync_methods_in_threadpool=False,
+        run_user_code_in_separate_thread=True,
+    )
+    await w.initialize_callable()
+    w.start_user_loop_watchdog(asyncio.get_running_loop())
+    assert w._user_loop_probe_task is None
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("run_sync_methods_in_threadpool", [False, True])
+async def test_call_user_health_check_raises_when_probe_failures_exhausted(
+    monkeypatch, run_sync_methods_in_threadpool: bool
+):
+    monkeypatch.setattr(replica_module, "USER_HEALTH_CHECK_PROBE_MAX_FAIL", 2)
+    w = _make_user_callable_wrapper(
+        run_sync_methods_in_threadpool=run_sync_methods_in_threadpool,
+        run_user_code_in_separate_thread=True,
+    )
+    await w.initialize_callable()
+    w._user_loop_probe_consecutive_fail_count = 2
+    with pytest.raises(RuntimeError, match="User event loop unresponsive"):
+        w.call_user_health_check()
+
+
+@pytest.mark.asyncio
+async def test_user_loop_watchdog_started_when_sync_in_threadpool(monkeypatch):
+    monkeypatch.setattr(replica_module, "USER_HEALTH_CHECK_PROBE_MAX_FAIL", 2)
+    monkeypatch.setattr(replica_module, "USER_HEALTH_CHECK_PROBE_INTERVAL_S", 3600.0)
+    w = _make_user_callable_wrapper(
+        run_sync_methods_in_threadpool=True,
+        run_user_code_in_separate_thread=True,
+    )
+    await w.initialize_callable()
+    w.start_user_loop_watchdog(asyncio.get_running_loop())
+    assert w._user_loop_probe_task is not None
+    w.stop_user_loop_watchdog()
+    await asyncio.sleep(0.01)
+    assert w._user_loop_probe_task is None
+
+
+if __name__ == "__main__":
+    # Tests are timing out on Windows for an unknown reason. Given this is just a unit
+    # test, running on Linux and Mac should be sufficient.
+    if sys.platform != "win32":
+        sys.exit(pytest.main(["-v", "-s", __file__]))