chore: import upstream snapshot with attribution

python/ray/llm/tests/serve/cpu/deployments/routers/kv/test_config_files/llm_kv_aware_deployment.yaml

@@ -0,0 +1,21 @@
+applications:
+  - name: kv-llm
+    route_prefix: /
+    import_path: ray.serve.llm:build_openai_app
+    runtime_env:
+      env_vars:
+        RAY_SERVE_LLM_ENABLE_DIRECT_STREAMING: "1"
+        RAY_SERVE_ENABLE_HA_PROXY: "1"
+    args:
+      llm_configs:
+        - model_loading_config:
+            model_id: qwen3-0.6b
+            model_source: Qwen/Qwen3-0.6B
+          accelerator_type: null
+          deployment_config:
+            autoscaling_config:
+              min_replicas: 0
+              initial_replicas: 0
+              max_replicas: 1
+            request_router_config:
+              request_router_class: ray.serve.llm.request_router.KVAwareRouter

python/ray/llm/tests/serve/cpu/deployments/routers/kv/test_kv_aware_router.py

@@ -0,0 +1,529 @@
+"""KVRouterActor attachment and live replica-membership tracking.
+
+Attachment is covered two ways: ``build_openai_app`` with a Python ``LLMConfig``,
+and a declarative YAML config deployed via ``serve deploy`` (the dotted-string
+router class only YAML can express). Membership tracking is covered by deploying
+a dummy multi-replica deployment and asserting the actor's LongPoll listener
+stays in sync with the live replicas across scale up/down.
+"""
+
+import os
+import subprocess
+import sys
+from typing import List
+from unittest import mock
+
+import pytest
+
+import ray
+from ray import serve
+from ray._common.test_utils import wait_for_condition
+from ray.llm._internal.serve.core.configs.llm_config import LLMConfig
+from ray.llm._internal.serve.core.ingress.builder import (
+    LLMServingArgs,
+    build_openai_app,
+)
+from ray.llm._internal.serve.core.ingress.tokenizer import REQUEST_TOKEN_IDS_KWARG
+from ray.llm._internal.serve.routing_policies.kv_aware.kv_aware_actor import (
+    KV_ROUTER_ACTOR_NAME,
+    KVRouterActor,
+    get_worker_id,
+)
+from ray.serve._private.common import (
+    REPLICA_ID_FULL_ID_STR_PREFIX,
+    DeploymentID,
+    DeploymentTargetInfo,
+    ReplicaID,
+    RequestMetadata,
+    RunningReplicaInfo,
+)
+from ray.serve._private.constants import SERVE_DEPLOYMENT_ACTOR_PREFIX, SERVE_NAMESPACE
+from ray.serve._private.request_router import PendingRequest
+from ray.serve.config import DeploymentActorConfig
+from ray.serve.llm.request_router import KVAwareRouter
+from ray.util.state import list_actors
+
+
+def get_kv_actor_configs(deployment):
+    return [
+        cfg
+        for cfg in (deployment._deployment_config.deployment_actors or [])
+        if (cfg["name"] if isinstance(cfg, dict) else cfg.name) == KV_ROUTER_ACTOR_NAME
+    ]
+
+
+def build_test_llm_config(experimental_configs=None) -> LLMConfig:
+    return LLMConfig(
+        model_loading_config={
+            "model_id": "qwen3-0.6b",
+            "model_source": "Qwen/Qwen3-0.6B",
+        },
+        accelerator_type=None,
+        deployment_config={
+            "autoscaling_config": {"min_replicas": 1, "max_replicas": 1},
+            "request_router_config": {"request_router_class": KVAwareRouter},
+        },
+        experimental_configs=experimental_configs or {},
+    )
+
+
+def build_non_kv_llm_config(**engine_kwargs) -> LLMConfig:
+    """An LLMConfig whose request router is the default (not a KVAwareRouter)."""
+    return LLMConfig(
+        model_loading_config={
+            "model_id": "qwen3-0.6b",
+            "model_source": "Qwen/Qwen3-0.6B",
+        },
+        accelerator_type=None,
+        deployment_config={
+            "autoscaling_config": {"min_replicas": 1, "max_replicas": 1}
+        },
+        engine_kwargs=engine_kwargs,
+    )
+
+
+def get_kv_actor_names(app_name: str) -> list:
+    prefix = f"{SERVE_DEPLOYMENT_ACTOR_PREFIX}{app_name}::"
+    suffix = f"::{KV_ROUTER_ACTOR_NAME}"
+    return [
+        a["name"]
+        for a in list_actors(filters=[("state", "=", "ALIVE")])
+        if a["name"] and a["name"].startswith(prefix) and a["name"].endswith(suffix)
+    ]
+
+
+def discover_deployment_actor(app_name, deployment_name, actor_name):
+    """Handle to a deployment-scoped actor by app/deployment/logical name."""
+    prefix = f"{SERVE_DEPLOYMENT_ACTOR_PREFIX}{app_name}::{deployment_name}::"
+    suffix = f"::{actor_name}"
+    for entry in ray.util.list_named_actors(all_namespaces=True):
+        name = entry.get("name") or ""
+        if (
+            entry.get("namespace") == SERVE_NAMESPACE
+            and name.startswith(prefix)
+            and (name.endswith(suffix))
+        ):
+            return ray.get_actor(name, namespace=SERVE_NAMESPACE)
+    return None
+
+
+def get_candidate_ids(app_name):
+    handle = discover_deployment_actor(
+        app_name, "ReplicaTrackingDeployment", KV_ROUTER_ACTOR_NAME
+    )
+    assert handle is not None
+    return ray.get(handle.get_candidate_worker_ids.remote())
+
+
+def get_live_replica_worker_ids(app_name, deployment_name="ReplicaTrackingDeployment"):
+    """Worker ids derived directly from the deployment's alive replica actors."""
+    prefix = f"{REPLICA_ID_FULL_ID_STR_PREFIX}{app_name}#{deployment_name}#"
+    return {
+        get_worker_id(a["name"][len(prefix) :])
+        for a in list_actors(filters=[("state", "=", "ALIVE")])
+        if a["name"] and a["name"].startswith(prefix)
+    }
+
+
+@pytest.fixture(autouse=True)
+def enable_direct_streaming(monkeypatch):
+    monkeypatch.setattr(
+        "ray.llm._internal.serve.core.ingress.builder."
+        "RAY_SERVE_LLM_ENABLE_DIRECT_STREAMING",
+        True,
+    )
+
+
+@pytest.fixture(scope="module")
+def serve_instance():
+    if not ray.is_initialized():
+        ray.init(address="auto")
+    yield
+    serve.shutdown()
+
+
+def test_build_openai_app_attaches_kv_actor():
+    """A KVAwareRouter on the LLMConfig attaches the KVRouterActor."""
+    app = build_openai_app(LLMServingArgs(llm_configs=[build_test_llm_config()]))
+
+    configs = get_kv_actor_configs(app._bound_deployment)
+    assert len(configs) == 1
+    actor_cfg = configs[0]
+    assert actor_cfg.get_actor_class().__ray_actor_class__ is KVRouterActor
+    assert actor_cfg.actor_options["num_cpus"] == 0
+    assert actor_cfg.init_kwargs == {"indexer_threads": 4}
+
+
+def test_configurable_indexer_threads():
+    llm_config = build_test_llm_config(experimental_configs={"KV_INDEXER_THREADS": 8})
+    app = build_openai_app(LLMServingArgs(llm_configs=[llm_config]))
+
+    actor_cfg = get_kv_actor_configs(app._bound_deployment)[0]
+    assert actor_cfg.init_kwargs["indexer_threads"] == 8
+
+
+def test_non_kv_router_warns_kv_events_config():
+    """Without a KVAwareRouter no KVRouterActor is attached and a user-provided
+    kv_events_config is left untouched (just unused), with a warning pointing at
+    how to consume the engine's KV events."""
+    kv_events_config = {
+        "enable_kv_cache_events": True,
+        "publisher": "zmq",
+        "endpoint": "tcp://*:5557",
+    }
+    llm_config = build_non_kv_llm_config(kv_events_config=kv_events_config)
+
+    with mock.patch(
+        "ray.llm._internal.serve.routing_policies.kv_aware.utils.logger"
+    ) as logger:
+        app = build_openai_app(LLMServingArgs(llm_configs=[llm_config]))
+
+    assert get_kv_actor_configs(app._bound_deployment) == []
+    assert llm_config.engine_kwargs["kv_events_config"] == kv_events_config
+    logger.warning.assert_called_once()
+    assert "KVAwareRouter" in logger.warning.call_args.args[0]
+
+
+def test_yaml_config_attaches_kv_actor(serve_instance):
+    """Deploying a YAML config that selects KVAwareRouter creates the KVRouterActor."""
+    config_file = os.path.join(
+        os.path.dirname(__file__), "test_config_files", "llm_kv_aware_deployment.yaml"
+    )
+    app_name = "kv-llm"
+
+    subprocess.check_output(["serve", "deploy", config_file], stderr=subprocess.STDOUT)
+    try:
+        wait_for_condition(lambda: len(get_kv_actor_names(app_name)) == 1, timeout=60)
+    finally:
+        serve.delete(app_name, _blocking=True)
+
+
+class _TestKVRouterActor(KVRouterActor):
+    """KVRouterActor augmented with test-only introspection."""
+
+    async def get_candidate_worker_ids(self) -> List[int]:
+        """The workers currently tracked from running replicas.
+
+        Async so it runs on the actor's event loop, serialized with
+        ``_on_deployment_targets`` which mutates the same map on that loop.
+        """
+        return sorted(self._replica_id_by_worker)
+
+
+@serve.deployment(
+    num_replicas=4,
+    deployment_actors=[
+        DeploymentActorConfig(
+            name=KV_ROUTER_ACTOR_NAME,
+            actor_class=ray.remote(_TestKVRouterActor),
+            actor_options={"num_cpus": 0},
+            init_kwargs={},
+        ),
+    ],
+)
+class ReplicaTrackingDeployment:
+    """Dummy deployment with a KVRouterActor deployment actor.
+
+    Advertises a per-replica KV-events endpoint via ``record_routing_stats`` as a
+    real engine would, so the selection service tracks each replica as a worker.
+    """
+
+    async def __call__(self) -> str:
+        return "ok"
+
+    async def record_routing_stats(self) -> dict:
+        rank = serve.get_replica_context().rank.local_rank
+        return {
+            "kv_event_metadata": {
+                "endpoint": f"tcp://{ray.util.get_node_ip_address()}:{25000 + rank}",
+                "block_size": 16,
+                "max_num_batched_tokens": 8192,
+                "dp_rank": 0,
+            }
+        }
+
+
+class TestReplicaTrackingIntegration:
+    def test_tracks_running_replicas(self, serve_instance):
+        """KVRouterActor's LongPollClient receives the running replicas."""
+        app_name = "kv-replica-tracking"
+        serve.run(
+            ReplicaTrackingDeployment.bind(), name=app_name, route_prefix="/kv_track"
+        )
+        try:
+            wait_for_condition(
+                lambda: len(get_candidate_ids(app_name)) == 4, timeout=30
+            )
+            # The tracked workers are exactly those of the live replica actors.
+            assert set(get_candidate_ids(app_name)) == get_live_replica_worker_ids(
+                app_name
+            )
+        finally:
+            serve.delete(app_name, _blocking=True)
+
+    def test_membership_broadcast_on_scale(self, serve_instance):
+        """A scale up then down is broadcast over LongPoll; the actor re-syncs to
+        exactly the live replica set each time.
+        """
+        app_name = "kv-replica-scale"
+
+        def tracks_live_replicas(expected):
+            # The tracked workers match the live replica actors by their actual
+            # ids (a stale handle is possible while the deployment is updated).
+            try:
+                tracked = set(get_candidate_ids(app_name))
+            except ray.exceptions.RayActorError:
+                return False
+            return len(tracked) == expected and tracked == get_live_replica_worker_ids(
+                app_name
+            )
+
+        def scale(num_replicas):
+            serve.run(
+                ReplicaTrackingDeployment.options(num_replicas=num_replicas).bind(),
+                name=app_name,
+                route_prefix="/kv_scale",
+            )
+
+        scale(2)
+        try:
+            wait_for_condition(lambda: tracks_live_replicas(2), timeout=30)
+            scale(4)  # upscale: the new replicas are picked up over LongPoll.
+            wait_for_condition(lambda: tracks_live_replicas(4), timeout=30)
+            scale(2)  # downscale: the departed replicas are dropped.
+            wait_for_condition(lambda: tracks_live_replicas(2), timeout=30)
+        finally:
+            serve.delete(app_name, _blocking=True)
+
+
+class _LocalKVRouterActor(_TestKVRouterActor):
+    """In-process KVRouterActor with the selection service and LongPoll disabled,
+    to drive ``_on_deployment_targets`` directly with synthetic snapshots.
+    """
+
+    def _create_selection_service(self) -> None:
+        self._svc = None  # reconcile membership without dynamo
+
+    def _start_replica_tracking(self) -> None:
+        pass
+
+    def _schedule(self, coro) -> None:
+        coro.close()  # _svc is None, so the scheduled upsert is a no-op
+
+
+def make_target_info(unique_ids):
+    """A DeploymentTargetInfo whose replicas advertise a KV-events endpoint via
+    routing_stats, exactly as the controller broadcasts it over LongPoll."""
+    deployment_id = DeploymentID(name="d", app_name="app")
+    running_replicas = [
+        RunningReplicaInfo(
+            replica_id=ReplicaID(unique_id=uid, deployment_id=deployment_id),
+            node_id="node",
+            node_ip="10.0.0.1",
+            availability_zone="az",
+            actor_name=f"actor-{uid}",
+            max_ongoing_requests=1,
+            routing_stats={
+                "kv_event_metadata": {
+                    "endpoint": "tcp://10.0.0.1:25000",
+                    "block_size": 16,
+                    "max_num_batched_tokens": 8192,
+                    "dp_rank": 0,
+                }
+            },
+        )
+        for uid in unique_ids
+    ]
+    return DeploymentTargetInfo(is_available=True, running_replicas=running_replicas)
+
+
+class TestOnDeploymentTargets:
+    async def test_reconciles_added_and_removed_workers(self):
+        actor = _LocalKVRouterActor()
+        actor._on_deployment_targets(make_target_info(["a", "b"]))
+        assert set(await actor.get_candidate_worker_ids()) == {
+            get_worker_id("a"),
+            get_worker_id("b"),
+        }
+        # "a" departs and "c" joins: the tracked set follows the new snapshot.
+        actor._on_deployment_targets(make_target_info(["b", "c"]))
+        assert set(await actor.get_candidate_worker_ids()) == {
+            get_worker_id("b"),
+            get_worker_id("c"),
+        }
+
+
+class _StubReplica:
+    """RunningReplica stand-in exposing only replica_id.unique_id."""
+
+    def __init__(self, unique_id: str):
+        self.replica_id = ReplicaID(
+            unique_id=unique_id, deployment_id=DeploymentID(name="d", app_name="app")
+        )
+
+
+class _SelectWorkerStub:
+    def __init__(self, worker_id: int):
+        self._worker_id = worker_id
+        self.token_ids = None
+        self.allowed = None
+
+    async def remote(self, request_id, token_ids, allowed_worker_ids):
+        self.token_ids = token_ids
+        self.allowed = allowed_worker_ids
+        return {
+            "worker_id": self._worker_id,
+            "dp_rank": 0,
+            "overlap_tokens": 1,
+            "effective_prefill_tokens": len(token_ids),
+        }
+
+
+class _KVRouterActorStub:
+    def __init__(self, worker_id: int):
+        self.select_worker = _SelectWorkerStub(worker_id)
+
+
+class _StubKVAwareRouter(KVAwareRouter):
+    """KVAwareRouter with the scorer actor injected, bypassing actor discovery."""
+
+    def __init__(self, kv_router_actor):
+        self._kv_router_actor = kv_router_actor
+
+
+def _build_kv_aware_router(worker_id: int) -> KVAwareRouter:
+    return _StubKVAwareRouter(_KVRouterActorStub(worker_id))
+
+
+@pytest.mark.asyncio
+async def test_select_worker_requires_tokens():
+    actor = KVRouterActor.__new__(KVRouterActor)
+    actor._svc = object()
+
+    with pytest.raises(ValueError, match="non-empty token_ids"):
+        await actor.select_worker("req-empty", [], [get_worker_id("r1")])
+
+
+@pytest.mark.asyncio
+async def test_select_worker_without_dynamo_raises():
+    """Without ai-dynamo the actor cannot score, so it raises a clear error
+    instead of silently degrading to a non-KV-aware pick."""
+    actor = KVRouterActor.__new__(KVRouterActor)
+    actor._svc = None
+
+    with pytest.raises(RuntimeError, match="ai-dynamo is not installed"):
+        await actor.select_worker("req", [1, 2, 3], [get_worker_id("r1")])
+
+
+@pytest.mark.asyncio
+async def test_choose_replicas_routes_to_selected_worker():
+    """choose_replicas maps candidates to worker ids, asks the actor to select,
+    and returns the chosen worker's replica."""
+    replicas = [_StubReplica("r1"), _StubReplica("r2")]
+    worker_ids = [get_worker_id("r1"), get_worker_id("r2")]
+
+    router = _build_kv_aware_router(worker_ids[1])
+    pending = PendingRequest(
+        args=[],
+        kwargs={REQUEST_TOKEN_IDS_KWARG: [10, 11, 12]},
+        metadata=RequestMetadata(request_id="req-1", internal_request_id="int-1"),
+    )
+
+    groups = await router.choose_replicas(replicas, pending)
+
+    # The actor selected r2's worker, so r2 is returned.
+    assert groups == [[replicas[1]]]
+    # choose_replicas forwarded the prompt token ids and the full candidate set.
+    select = router._kv_router_actor.select_worker
+    assert select.token_ids == [10, 11, 12]
+    assert sorted(select.allowed) == sorted(worker_ids)
+
+
+@pytest.mark.asyncio
+async def test_missing_token_ids_picks_random_replica():
+    """Token-less requests (batch prompts, truncated bodies) route to a single
+    random replica so they spread."""
+    replicas = [_StubReplica("r1"), _StubReplica("r2")]
+    router = _build_kv_aware_router(get_worker_id("r1"))
+
+    picked = set()
+    for _ in range(50):
+        pending = PendingRequest(
+            args=[],
+            kwargs={},
+            metadata=RequestMetadata(request_id="req", internal_request_id="int"),
+        )
+        groups = await router.choose_replicas(replicas, pending)
+        assert len(groups) == 1 and len(groups[0]) == 1
+        assert groups[0][0] in replicas
+        picked.add(groups[0][0].replica_id.unique_id)
+
+    # The picked replica varies across calls, so load spreads (not stuck on one).
+    assert picked == {"r1", "r2"}
+    assert router._kv_router_actor.select_worker.token_ids is None
+
+
+@pytest.mark.asyncio
+async def test_tokenize_call_picks_random_replica():
+    """The pre-routing /tokenize RPC is routed through choose_replicas before any
+    token ids exist; it must resolve so KV routing can bootstrap, and picks a random
+    replica without scoring."""
+    replicas = [_StubReplica("r1"), _StubReplica("r2")]
+
+    router = _build_kv_aware_router(get_worker_id("r2"))
+    pending = PendingRequest(
+        args=[],
+        kwargs={},
+        metadata=RequestMetadata(
+            request_id="req-tokenize",
+            internal_request_id="int-tokenize",
+            call_method="tokenize",
+        ),
+    )
+
+    groups = await router.choose_replicas(replicas, pending)
+
+    assert len(groups) == 1 and len(groups[0]) == 1
+    assert groups[0][0] in replicas
+    assert router._kv_router_actor.select_worker.token_ids is None
+
+
+@pytest.mark.asyncio
+async def test_empty_token_ids_picks_random_replica():
+    """Empty token ids carry no KV signal, so pick a random replica instead of
+    handing an empty prompt to the Dynamo selection service (which rejects it)."""
+    replicas = [_StubReplica("r1"), _StubReplica("r2")]
+
+    router = _build_kv_aware_router(get_worker_id("r2"))
+    pending = PendingRequest(
+        args=[],
+        kwargs={REQUEST_TOKEN_IDS_KWARG: []},
+        metadata=RequestMetadata(
+            request_id="req-empty", internal_request_id="int-empty"
+        ),
+    )
+
+    groups = await router.choose_replicas(replicas, pending)
+
+    assert len(groups) == 1 and len(groups[0]) == 1
+    assert groups[0][0] in replicas
+    assert router._kv_router_actor.select_worker.token_ids is None
+
+
+@pytest.mark.asyncio
+async def test_no_pending_request_picks_random_replica():
+    """Serve may ask again after route metadata has been consumed; pick a random
+    replica (nothing to score on)."""
+    replicas = [_StubReplica("r1"), _StubReplica("r2")]
+
+    router = _build_kv_aware_router(get_worker_id("r1"))
+
+    groups = await router.choose_replicas(replicas, pending_request=None)
+
+    assert len(groups) == 1 and len(groups[0]) == 1
+    assert groups[0][0] in replicas
+    assert router._kv_router_actor.select_worker.token_ids is None
+
+
+if __name__ == "__main__":
+    sys.exit(pytest.main(["-v", __file__]))

python/ray/llm/tests/serve/cpu/deployments/routers/kv/test_kv_events.py

@@ -0,0 +1,122 @@
+import sys
+from types import SimpleNamespace
+from unittest import mock
+
+import pytest
+
+import ray
+from ray.llm._internal.serve.core.configs.llm_config import LLMConfig
+from ray.llm._internal.serve.routing_policies.kv_aware.vllm.kv_events import (
+    assign_replica_kv_events_endpoint,
+    configure_kv_events_for_kv_routing,
+    get_kv_event_routing_stats,
+    resolve_kv_event_source_endpoint,
+)
+from ray.serve.llm.request_router import KVAwareRouter
+
+
+def make_kv_aware_llm_config(**kwargs) -> LLMConfig:
+    return LLMConfig(
+        model_loading_config={
+            "model_id": "qwen3-0.6b",
+            "model_source": "Qwen/Qwen3-0.6B",
+        },
+        accelerator_type=None,
+        deployment_config={
+            "autoscaling_config": {"min_replicas": 1, "max_replicas": 1},
+            "request_router_config": {"request_router_class": KVAwareRouter},
+        },
+        **kwargs,
+    )
+
+
+@pytest.fixture(scope="module")
+def ray_instance():
+    started = not ray.is_initialized()
+    if started:
+        ray.init()
+    yield
+    if started:
+        ray.shutdown()
+
+
+class TestConfigureKvEvents:
+    def test_configure_enables_events_and_pins_seed(self):
+        """KV-aware config turns on engine ZMQ KV events and pins the hash seed."""
+        llm_config = make_kv_aware_llm_config()
+        configure_kv_events_for_kv_routing(llm_config)
+
+        assert llm_config.engine_kwargs["kv_events_config"] == {
+            "enable_kv_cache_events": True,
+            "publisher": "zmq",
+            "endpoint": "tcp://*:5557",
+            "replay_endpoint": "tcp://*:6557",
+        }
+        assert llm_config.runtime_env["env_vars"]["PYTHONHASHSEED"] == "0"
+
+    @pytest.mark.parametrize(
+        "engine_kwargs, local_rank, expected_port, expected_replay_port",
+        [
+            # Non-DP: offset the base port by the replica's node-local rank so
+            # colocated replicas don't bind the same ZMQ PUB port.
+            ({}, 2, 5559, 6559),
+            # DP: data_parallel_rank set -> offset 0 (the engine offsets the
+            # bound port by dp_rank itself), so local_rank must be ignored.
+            ({"data_parallel_rank": 2}, 2, 5557, 6557),
+        ],
+    )
+    def test_assign_replica_endpoint_offsets_port(
+        self, engine_kwargs, local_rank, expected_port, expected_replay_port
+    ):
+        """Per-replica endpoint offset: by node-local rank without DP, 0 with DP."""
+        llm_config = make_kv_aware_llm_config(engine_kwargs=dict(engine_kwargs))
+        configure_kv_events_for_kv_routing(llm_config)  # base ports 5557 / 6557
+        replica_context = SimpleNamespace(rank=SimpleNamespace(local_rank=local_rank))
+        with mock.patch("ray.serve.get_replica_context", return_value=replica_context):
+            assign_replica_kv_events_endpoint(llm_config)
+        kv_events_config = llm_config.engine_kwargs["kv_events_config"]
+        assert kv_events_config["endpoint"] == f"tcp://*:{expected_port}"
+        assert kv_events_config["replay_endpoint"] == f"tcp://*:{expected_replay_port}"
+
+    def test_resolve_endpoint_is_node_routable(self, ray_instance):
+        """The advertised endpoint is the replica's node IP."""
+        llm_config = make_kv_aware_llm_config()
+        configure_kv_events_for_kv_routing(llm_config)
+
+        endpoint = resolve_kv_event_source_endpoint(llm_config)
+        node_ip = ray.util.get_node_ip_address()
+        assert endpoint == f"tcp://{node_ip}:5557"
+
+    def test_routing_stats_advertise_endpoint(self, ray_instance):
+        """The replica advertises its node-routable endpoint plus the engine
+        facts the selection service needs to schedule it via record_routing_stats."""
+        llm_config = make_kv_aware_llm_config()
+        configure_kv_events_for_kv_routing(llm_config)
+
+        stats = get_kv_event_routing_stats(
+            llm_config, block_size=16, max_num_batched_tokens=4096
+        )
+        node_ip = ray.util.get_node_ip_address()
+        assert stats == {
+            "kv_event_metadata": {
+                "endpoint": f"tcp://{node_ip}:5557",
+                "block_size": 16,
+                "max_num_batched_tokens": 4096,
+                "dp_rank": 0,
+                "replay_endpoint": f"tcp://{node_ip}:6557",
+            }
+        }
+
+    def test_routing_stats_empty_without_kv_events(self):
+        """Nothing to advertise when KV-cache events are not enabled."""
+        llm_config = make_kv_aware_llm_config()
+        assert (
+            get_kv_event_routing_stats(
+                llm_config, block_size=16, max_num_batched_tokens=4096
+            )
+            == {}
+        )
+
+
+if __name__ == "__main__":
+    sys.exit(pytest.main(["-v", __file__]))

python/ray/llm/tests/serve/cpu/deployments/routers/kv/test_request_lifecycle_tracking.py

@@ -0,0 +1,807 @@
+import asyncio
+import random
+import sys
+from collections import OrderedDict
+from dataclasses import asdict
+from types import SimpleNamespace
+
+import pytest
+from vllm.outputs import CompletionOutput, RequestOutput
+from vllm.sampling_params import RequestOutputKind, SamplingParams
+
+import ray
+import ray.cloudpickle
+from ray import serve
+from ray.llm._internal.serve.core.configs.llm_config import LLMConfig
+from ray.llm._internal.serve.routing_policies.kv_aware.constants import (
+    REQUEST_TRACKING_TTL_S,
+)
+from ray.llm._internal.serve.routing_policies.kv_aware.kv_aware_actor import (
+    KV_ROUTER_ACTOR_NAME,
+    KVRouterActor,
+    get_worker_id,
+)
+from ray.llm._internal.serve.routing_policies.kv_aware.kv_aware_router import (
+    is_kv_aware,
+)
+from ray.llm._internal.serve.routing_policies.kv_aware.vllm.token_tracking import (
+    enable_token_tracking,
+)
+from ray.llm.tests.serve.mocks.mock_vllm_engine import MockAsyncLLM
+from ray.serve.llm.request_router import KVAwareRouter
+
+# The @ray.remote actors below are pickled by reference, so a worker must import
+# this module -- but pytest imports it under a bare name the worker cannot import
+# (ModuleNotFoundError). Pickling by value ships the class bodies instead.
+ray.cloudpickle.register_pickle_by_value(sys.modules[__name__])
+
+REPLICA_UNIQUE_ID = "test-replica-uid"
+WORKER_ID = get_worker_id(REPLICA_UNIQUE_ID)
+# A pre-tokenized prompt, as vLLM's serving layer always passes to generate.
+PROMPT = {"prompt_token_ids": [1, 2, 3]}
+# SamplingParams.max_tokens the engine reports as the request's expected output.
+MAX_TOKENS = 20
+
+
+@pytest.fixture(scope="module", autouse=True)
+def ray_cluster():
+    if not ray.is_initialized():
+        ray.init()
+
+
+def request_output(token_counts, prompt_len=5, finished=False):
+    """A real vLLM RequestOutput: one CompletionOutput per entry in token_counts."""
+    return RequestOutput(
+        request_id="r",
+        prompt=None,
+        prompt_token_ids=list(range(prompt_len)),
+        prompt_logprobs=None,
+        outputs=[
+            CompletionOutput(
+                index=i,
+                text="",
+                token_ids=list(range(n)),
+                cumulative_logprob=None,
+                logprobs=None,
+            )
+            for i, n in enumerate(token_counts)
+        ],
+        finished=finished,
+    )
+
+
+def delta_steps(num_tokens, prompt_len=5):
+    """A DELTA-kind stream: one new token per step, last step finished."""
+    return [
+        request_output([1], prompt_len=prompt_len, finished=i == num_tokens - 1)
+        for i in range(num_tokens)
+    ]
+
+
+class MockSelectionService:
+    """Records the selection-service reservation calls the actor's lifecycle
+    hooks make, standing in for the Dynamo selection service. ``add_output_block``
+    is synchronous as in the real binding; the rest are async."""
+
+    def __init__(self):
+        self.calls = []
+        self.reservations = []
+
+    async def create_reservation(self, request):
+        self.reservations.append(dict(request))
+        self.calls.append(
+            (
+                "create_reservation",
+                request["reservation_id"],
+                request["worker_id"],
+                len(request["token_ids"]),
+                request.get("expected_output_tokens"),
+            )
+        )
+
+    async def prefill_complete(self, reservation_id):
+        self.calls.append(("prefill_complete", reservation_id))
+
+    def add_output_block(self, reservation_id, *, decay_fraction=None):
+        self.calls.append(("add_output_block", reservation_id, decay_fraction))
+
+    async def free_reservation(self, reservation_id):
+        self.calls.append(("free_reservation", reservation_id))
+
+    async def delete_worker(self, worker_id):
+        self.calls.append(("delete_worker", worker_id))
+
+
+@ray.remote(num_cpus=0)
+class RecordingKVRouterActor(KVRouterActor):
+    """KVRouterActor that records the lifecycle events it receives for tests."""
+
+    def __init__(self, block_size):
+        self._block_size = block_size
+        self._replica_id_by_worker = {}
+        self._requests = OrderedDict()
+        self._request_ids_by_worker = {}
+        self._effective_prefill_tokens_by_request = {}
+        self._pending_tasks = set()
+        self._svc = MockSelectionService()
+        self._event_log = []
+
+    async def on_lifecycle_events(self, events):
+        self._event_log.extend(events)
+        await super().on_lifecycle_events(events)
+
+    def get_event_log(self):
+        return self._event_log
+
+    def get_selection_service_calls(self):
+        return self._svc.calls
+
+    async def get_request_lifecycle(self, request_id):
+        """Return a snapshot of an in-flight request's state, or ``None``."""
+        state = self._requests.get(request_id)
+        if state is None:
+            return None
+        snapshot = asdict(state)
+        snapshot.pop("created_at", None)  # internal TTL bookkeeping, not asserted on
+        return snapshot
+
+    async def get_active_request_ids(self):
+        """Return ids of the in-flight requests."""
+        return list(self._requests)
+
+    async def get_worker_active_load(self, worker_id):
+        """Return the number of in-flight requests attributed to ``worker_id``."""
+        return sum(1 for s in self._requests.values() if s.worker_id == worker_id)
+
+
+@ray.remote(num_cpus=0)
+class RaisingActor:
+    """A KV-router stand-in whose event ingest always raises, to prove the
+    engine token stream is never disrupted."""
+
+    async def on_lifecycle_events(self, events):
+        raise RuntimeError("actor down")
+
+
+class LocalKVRouterActor(KVRouterActor):
+    """In-process KVRouterActor with the event plane + Serve LongPoll stripped."""
+
+    def __init__(self, block_size):
+        self._block_size = block_size
+        self._replica_id_by_worker = {}
+        self._requests = OrderedDict()
+        self._request_ids_by_worker = {}
+        self._effective_prefill_tokens_by_request = {}
+        self._pending_tasks = set()
+        self._svc = MockSelectionService()
+
+    async def get_request_lifecycle(self, request_id):
+        """Return a snapshot of an in-flight request's state, or ``None``."""
+        state = self._requests.get(request_id)
+        if state is None:
+            return None
+        snapshot = asdict(state)
+        snapshot.pop("created_at", None)  # internal TTL bookkeeping, not asserted on
+        return snapshot
+
+    async def get_active_request_ids(self):
+        """Return ids of the in-flight requests."""
+        return list(self._requests)
+
+    async def get_worker_active_load(self, worker_id):
+        """Return the number of in-flight requests attributed to ``worker_id``."""
+        return sum(1 for s in self._requests.values() if s.worker_id == worker_id)
+
+
+@pytest.fixture
+def build_token_tracking_engine(monkeypatch):
+    def _build(script, actor, **engine_kwargs):
+        def get_deployment_actor(name):
+            assert name == KV_ROUTER_ACTOR_NAME
+            return actor
+
+        monkeypatch.setattr(serve, "get_deployment_actor", get_deployment_actor)
+        monkeypatch.setattr(
+            serve,
+            "get_replica_context",
+            lambda: SimpleNamespace(
+                replica_id=SimpleNamespace(unique_id=REPLICA_UNIQUE_ID)
+            ),
+        )
+        return enable_token_tracking(MockAsyncLLM)(script, **engine_kwargs)
+
+    return _build
+
+
+async def consume(stream, limit=None):
+    """Drain ``stream``, optionally closing it early after ``limit`` outputs."""
+    outputs = []
+    async for output in stream:
+        outputs.append(output)
+        if limit is not None and len(outputs) == limit:
+            await stream.aclose()
+    return outputs
+
+
+async def drain(engine):
+    """Wait for the engine forwarder's queued lifecycle batches to land."""
+    await engine._lifecycle_forwarder.flush()
+
+
+def decode_counts(events):
+    return [args[1] for name, args in events if name == "on_decode_progress"]
+
+
+def op_names(calls):
+    return [c[0] for c in calls]
+
+
+@pytest.mark.asyncio
+async def test_basic_lifecycle(build_token_tracking_engine):
+    """A streamed request reports add -> prefill -> exact decode counts -> done."""
+    actor = RecordingKVRouterActor.remote(block_size=16)
+    engine = build_token_tracking_engine(delta_steps(3, prompt_len=10), actor)
+
+    prompt = {"prompt_token_ids": list(range(10))}
+    outputs = await consume(
+        engine.generate(
+            prompt,
+            SamplingParams(output_kind=RequestOutputKind.DELTA, max_tokens=MAX_TOKENS),
+            "req-1",
+        )
+    )
+    await drain(engine)
+
+    assert ray.get(actor.get_event_log.remote()) == [
+        ("on_request_added", ("req-1", WORKER_ID, list(range(10)), MAX_TOKENS)),
+        ("on_prefill_complete", ("req-1",)),
+        ("on_decode_progress", ("req-1", 1)),
+        ("on_decode_progress", ("req-1", 2)),
+        ("on_decode_progress", ("req-1", 3)),
+        ("on_request_completed", ("req-1",)),
+    ]
+    assert outputs == engine.script
+
+
+@pytest.mark.asyncio
+async def test_lifecycle_uses_serve_request_id(build_token_tracking_engine):
+    """Lifecycle events use the same Serve request id used by routing, even if
+    vLLM's engine-level id is different."""
+    actor = RecordingKVRouterActor.remote(block_size=16)
+    engine = build_token_tracking_engine(delta_steps(1, prompt_len=10), actor)
+
+    prompt = {"prompt_token_ids": list(range(10))}
+    serve.context._serve_request_context.set(
+        serve.context._RequestContext(request_id="serve-route-id")
+    )
+    try:
+        await consume(
+            engine.generate(
+                prompt,
+                SamplingParams(
+                    output_kind=RequestOutputKind.DELTA, max_tokens=MAX_TOKENS
+                ),
+                "chatcmpl-serve-route-id",
+            )
+        )
+    finally:
+        serve.context._serve_request_context.set(serve.context._RequestContext())
+    await drain(engine)
+
+    assert ray.get(actor.get_event_log.remote()) == [
+        (
+            "on_request_added",
+            ("serve-route-id", WORKER_ID, list(range(10)), MAX_TOKENS),
+        ),
+        ("on_prefill_complete", ("serve-route-id",)),
+        ("on_decode_progress", ("serve-route-id", 1)),
+        ("on_request_completed", ("serve-route-id",)),
+    ]
+
+
+@pytest.mark.asyncio
+async def test_in_order_reports(build_token_tracking_engine):
+    """Back-to-back reports reach the actor in submission order."""
+    actor = RecordingKVRouterActor.remote(block_size=16)
+    engine = build_token_tracking_engine(delta_steps(200), actor)
+
+    await consume(
+        engine.generate(
+            PROMPT, SamplingParams(output_kind=RequestOutputKind.DELTA), "r"
+        )
+    )
+    await drain(engine)
+
+    events = ray.get(actor.get_event_log.remote())
+    assert events[0][0] == "on_request_added"
+    assert events[1][0] == "on_prefill_complete"
+    assert events[-1] == ("on_request_completed", ("r",))
+    assert decode_counts(events) == list(range(1, 201))
+
+
+@pytest.mark.asyncio
+async def test_streaming_accumulates_decode_progress(build_token_tracking_engine):
+    """A DELTA (streaming) request sums each step's new tokens into a running
+    total reported as decode progress."""
+    actor = RecordingKVRouterActor.remote(block_size=16)
+    # Steps carry only new tokens: 1, then 2, then 1 -> cumulative 1, 3, 4.
+    script = [request_output([n]) for n in (1, 2, 1)]
+    engine = build_token_tracking_engine(script, actor)
+
+    await consume(
+        engine.generate(
+            PROMPT, SamplingParams(output_kind=RequestOutputKind.DELTA), "r"
+        )
+    )
+    await drain(engine)
+
+    assert decode_counts(ray.get(actor.get_event_log.remote())) == [1, 3, 4]
+
+
+@pytest.mark.asyncio
+async def test_non_streaming_reports_full_output_once(build_token_tracking_engine):
+    """A FINAL_ONLY (non-streaming) request arrives as one finished chunk, with a
+    CompletionOutput per candidate, so progress is reported once at the summed
+    token count across candidates."""
+    actor = RecordingKVRouterActor.remote(block_size=16)
+    # FINAL_ONLY n=3: a single finished chunk carrying every candidate's output.
+    script = [request_output([2, 3, 4], finished=True)]
+    engine = build_token_tracking_engine(script, actor)
+
+    await consume(
+        engine.generate(
+            PROMPT, SamplingParams(output_kind=RequestOutputKind.FINAL_ONLY), "r"
+        )
+    )
+    await drain(engine)
+
+    events = ray.get(actor.get_event_log.remote())
+    assert decode_counts(events) == [9]  # 2 + 3 + 4 summed across candidates
+    assert [name for name, _ in events] == [
+        "on_request_added",
+        "on_prefill_complete",
+        "on_decode_progress",
+        "on_request_completed",
+    ]
+
+
+@pytest.mark.asyncio
+async def test_cumulative_skips_tracking(build_token_tracking_engine):
+    """CUMULATIVE repeats output-so-far each chunk; tracking is skipped (not
+    summed) to avoid over-counting, so no lifecycle events are reported."""
+    actor = RecordingKVRouterActor.remote(block_size=16)
+    engine = build_token_tracking_engine(delta_steps(3), actor)
+
+    outputs = await consume(
+        engine.generate(
+            PROMPT, SamplingParams(output_kind=RequestOutputKind.CUMULATIVE), "r"
+        )
+    )
+
+    assert len(outputs) == 3  # stream still passes through untouched
+    assert ray.get(actor.get_event_log.remote()) == []
+
+
+@pytest.mark.asyncio
+async def test_empty_steps_ignored(build_token_tracking_engine):
+    """Token-less outputs (e.g. a finish-only chunk) emit no progress hooks."""
+    actor = RecordingKVRouterActor.remote(block_size=16)
+    script = [
+        request_output([1]),
+        request_output([0]),  # structural chunk: no new tokens
+        request_output([1], finished=True),
+    ]
+    engine = build_token_tracking_engine(script, actor)
+
+    await consume(
+        engine.generate(
+            PROMPT, SamplingParams(output_kind=RequestOutputKind.DELTA), "r"
+        )
+    )
+    await drain(engine)
+
+    events = ray.get(actor.get_event_log.remote())
+    assert decode_counts(events) == [1, 2]
+    assert [e for e in events if e[0] == "on_prefill_complete"] == [
+        ("on_prefill_complete", ("r",))
+    ]
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("early_drop", [False, True])
+async def test_completed_exactly_once(early_drop, build_token_tracking_engine):
+    """Completion fires exactly once on normal end and on early stream close."""
+    actor = RecordingKVRouterActor.remote(block_size=16)
+    engine = build_token_tracking_engine(delta_steps(3), actor)
+
+    stream = engine.generate(
+        PROMPT, SamplingParams(output_kind=RequestOutputKind.DELTA), "r"
+    )
+    await consume(stream, limit=1 if early_drop else None)
+    await drain(engine)
+
+    events = ray.get(actor.get_event_log.remote())
+    assert [e for e in events if e[0] == "on_request_completed"] == [
+        ("on_request_completed", ("r",))
+    ]
+
+
+@pytest.mark.asyncio
+async def test_engine_error_still_completes(build_token_tracking_engine):
+    """A mid-stream engine error propagates but still frees the request."""
+    actor = RecordingKVRouterActor.remote(block_size=16)
+    engine = build_token_tracking_engine(delta_steps(3), actor, error_after=1)
+
+    with pytest.raises(RuntimeError, match="engine failure"):
+        await consume(
+            engine.generate(
+                PROMPT, SamplingParams(output_kind=RequestOutputKind.DELTA), "r"
+            )
+        )
+    await drain(engine)
+
+    events = ray.get(actor.get_event_log.remote())
+    assert events[-1] == ("on_request_completed", ("r",))
+
+
+@pytest.mark.asyncio
+async def test_zero_token_request(build_token_tracking_engine):
+    """An output-less request (e.g. validation abort) is added and freed only."""
+    actor = RecordingKVRouterActor.remote(block_size=16)
+    engine = build_token_tracking_engine([request_output([0], finished=True)], actor)
+
+    prompt = {"prompt_token_ids": [1, 2, 3]}
+    await consume(
+        engine.generate(
+            prompt,
+            SamplingParams(output_kind=RequestOutputKind.DELTA, max_tokens=MAX_TOKENS),
+            "r",
+        )
+    )
+    await drain(engine)
+
+    assert ray.get(actor.get_event_log.remote()) == [
+        ("on_request_added", ("r", WORKER_ID, [1, 2, 3], MAX_TOKENS)),
+        ("on_request_completed", ("r",)),
+    ]
+
+
+@pytest.mark.asyncio
+async def test_actor_failure_isolation(build_token_tracking_engine):
+    """A failing actor never disrupts the engine's output stream."""
+    engine = build_token_tracking_engine(delta_steps(2), RaisingActor.remote())
+
+    outputs = await consume(
+        engine.generate(
+            PROMPT, SamplingParams(output_kind=RequestOutputKind.DELTA), "r"
+        )
+    )
+    await drain(engine)  # the failed batches are dropped without raising
+
+    assert len(outputs) == 2
+
+
+def test_decorator_returns_subclass():
+    """The decorator returns an isinstance-compatible subclass."""
+    assert issubclass(enable_token_tracking(MockAsyncLLM), MockAsyncLLM)
+
+
+@pytest.mark.asyncio
+async def test_passthrough_without_actor(monkeypatch):
+    """Outside a replica (no actor resolvable) the engine is a pure pass-through."""
+
+    def _raise(name):
+        raise RuntimeError("no actor")
+
+    monkeypatch.setattr(serve, "get_deployment_actor", _raise)
+    engine = enable_token_tracking(MockAsyncLLM)(delta_steps(2))
+
+    outputs = await consume(
+        engine.generate(
+            PROMPT, SamplingParams(output_kind=RequestOutputKind.DELTA), "r"
+        )
+    )
+
+    assert len(outputs) == 2
+    assert engine._lifecycle_forwarder is None  # resolution failed; retried next call
+
+
+@pytest.mark.parametrize(
+    "request_router_config, expected",
+    [
+        ({"request_router_class": KVAwareRouter}, True),
+        ({}, False),  # default (non-KV) router
+        (None, False),  # no router configured
+    ],
+)
+def test_is_kv_aware(request_router_config, expected):
+    """The engine wraps with token tracking only on KVAwareRouter deployments,
+    so non-KV deployments never retry a missing actor lookup per request."""
+    deployment_config = {}
+    if request_router_config is not None:
+        deployment_config["request_router_config"] = request_router_config
+    llm_config = LLMConfig(
+        model_loading_config={
+            "model_id": "qwen3-0.6b",
+            "model_source": "Qwen/Qwen3-0.6B",
+        },
+        accelerator_type=None,
+        deployment_config=deployment_config,
+    )
+    assert is_kv_aware(llm_config) is expected
+
+
+@pytest.mark.asyncio
+async def test_block_boundary_crossings():
+    """Each ceil((prompt+output)/block_size) increase advances total_blocks."""
+    actor = LocalKVRouterActor(block_size=16)
+    await actor.on_request_added("r", 1, list(range(10)))
+    assert (await actor.get_request_lifecycle("r"))["total_blocks"] == 1  # ceil(10/16)
+
+    await actor.on_decode_progress("r", 6)  # 10+6=16 -> still 1 block
+    assert (await actor.get_request_lifecycle("r"))["total_blocks"] == 1
+
+    await actor.on_decode_progress("r", 7)  # 17 -> crosses into block 2
+    assert (await actor.get_request_lifecycle("r"))["total_blocks"] == 2
+
+    await actor.on_decode_progress("r", 39)  # 49 -> ceil=4, crosses two more at once
+    snapshot = await actor.get_request_lifecycle("r")
+    assert snapshot["total_blocks"] == 4
+    assert snapshot["output_tokens"] == 39
+
+
+@pytest.mark.asyncio
+async def test_active_load_tracking():
+    """Active load is per-worker; completion evicts the request entirely."""
+    actor = LocalKVRouterActor(block_size=16)
+    await actor.on_request_added("a", 1, list(range(8)))
+    await actor.on_request_added("b", 1, [])
+    await actor.on_request_added("c", 2, [])
+    assert await actor.get_worker_active_load(1) == 2
+    assert await actor.get_worker_active_load(2) == 1
+
+    await actor.on_prefill_complete("a")
+    await actor.on_decode_progress("a", 5)
+    assert await actor.get_worker_active_load(1) == 2  # still active while decoding
+    assert await actor.get_request_lifecycle("a") == {
+        "worker_id": 1,
+        "prompt_tokens": 8,
+        "expected_output_tokens": None,
+        "prefill_completed": True,
+        "output_tokens": 5,
+        "total_blocks": 1,
+    }
+
+    # Completion evicts (bounding memory to in-flight requests).
+    await actor.on_request_completed("a")
+    assert await actor.get_worker_active_load(1) == 1
+    assert set(await actor.get_active_request_ids()) == {"b", "c"}
+    assert await actor.get_request_lifecycle("a") is None
+
+    # Hooks for an unknown request id are ignored.
+    await actor.on_prefill_complete("missing")
+    await actor.on_decode_progress("missing", 3)
+    await actor.on_request_completed("missing")
+    assert await actor.get_request_lifecycle("missing") is None
+
+
+@pytest.mark.asyncio
+async def test_tracking_drains_under_churn():
+    """Memory chaos: a long run of interleaved submissions and completions across
+    workers grows the in-flight state and drains it back to nothing."""
+    rng = random.Random(20240708)
+    actor = LocalKVRouterActor(block_size=16)
+    workers = [1, 2, 3]
+    total = 400
+    inflight: set = set()
+    launched = 0
+    peak = 0
+
+    # Randomly interleave admitting new requests and completing live ones.
+    for _ in range(total * 3):
+        if launched < total and (not inflight or rng.random() < 0.6):
+            request_id = f"r{launched}"
+            launched += 1
+            # A routed request carries its effective prefill tokens from select();
+            # admission must drain that map too, not just _requests.
+            actor._effective_prefill_tokens_by_request[request_id] = rng.randint(0, 40)
+            await actor.on_request_added(
+                request_id,
+                rng.choice(workers),
+                list(range(rng.randint(1, 40))),
+                expected_output_tokens=rng.choice([None, 32]),
+            )
+            inflight.add(request_id)
+        elif inflight:
+            request_id = rng.choice(list(inflight))
+            if rng.random() < 0.5:
+                await actor.on_prefill_complete(request_id)
+                await actor.on_decode_progress(request_id, rng.randint(1, 80))
+            await actor.on_request_completed(request_id)
+            inflight.discard(request_id)
+        peak = max(peak, len(actor._requests))
+
+    for request_id in list(inflight):
+        await actor.on_request_completed(request_id)
+
+    assert launched == total
+    assert peak > 1  # state actually accumulated under concurrent load
+    # Everything drained: no request state, no index entries, no active load.
+    assert actor._requests == {}
+    assert actor._request_ids_by_worker == {}
+    assert actor._effective_prefill_tokens_by_request == {}
+    assert await actor.get_active_request_ids() == []
+    for worker_id in workers:
+        assert await actor.get_worker_active_load(worker_id) == 0
+
+
+@pytest.mark.asyncio
+async def test_remove_worker_evicts_requests():
+    """A departed replica's in-flight request state is purged: its completion
+    events can never arrive, so the entries would otherwise leak forever."""
+    actor = LocalKVRouterActor(block_size=16)
+    await actor.on_request_added("a", 1, list(range(8)))
+    await actor.on_request_added("b", 1, list(range(8)))
+    await actor.on_request_added("c", 2, list(range(8)))
+
+    actor.remove_worker(1)
+
+    assert set(await actor.get_active_request_ids()) == {"c"}
+    assert await actor.get_worker_active_load(1) == 0
+    assert await actor.get_worker_active_load(2) == 1
+    # The reverse index drops the departed worker and keeps the survivor.
+    assert 1 not in actor._request_ids_by_worker
+    assert actor._request_ids_by_worker.get(2) == {"c"}
+    # remove_worker schedules delete_worker; drain the task and confirm.
+    await asyncio.gather(*list(actor._pending_tasks))
+    assert ("delete_worker", 1) in actor._svc.calls
+
+
+@pytest.mark.asyncio
+async def test_stale_request_evicted_after_ttl():
+    """Backstop for a lost completion on a live replica: a request tracked past
+    the TTL is evicted and its reservation freed, so it cannot accumulate."""
+    actor = LocalKVRouterActor(block_size=16)
+    await actor.on_request_added("stale", 1, list(range(8)))
+    await actor.on_request_added("fresh_untriggered", 1, list(range(8)))
+    # Backdate the stale request's admission beyond the TTL (lost completion).
+    actor._requests["stale"].created_at -= REQUEST_TRACKING_TTL_S + 1
+
+    # A new admission triggers the lazy sweep of the oldest entries.
+    await actor.on_request_added("trigger", 2, list(range(8)))
+
+    assert "stale" not in await actor.get_active_request_ids()
+    assert ("free_reservation", "stale") in actor._svc.calls
+    # A still-fresh request is left untouched (sweep stops at the first fresh one).
+    assert set(await actor.get_active_request_ids()) == {"fresh_untriggered", "trigger"}
+    assert ("free_reservation", "fresh_untriggered") not in actor._svc.calls
+
+
+@pytest.mark.asyncio
+async def test_admission_race_frees_reservation():
+    """If remove_worker evicts a request while create_reservation is in flight,
+    the orphaned reservation is freed rather than leaking in the service."""
+    actor = LocalKVRouterActor(block_size=16)
+
+    # Simulate the LongPoll remove_worker firing during the create_reservation await.
+    book_reservation = actor._svc.create_reservation
+
+    async def racing_create_reservation(request):
+        await book_reservation(request)
+        actor.remove_worker(request["worker_id"])
+
+    actor._svc.create_reservation = racing_create_reservation
+
+    await actor.on_request_added("r", 1, list(range(8)))
+
+    assert await actor.get_active_request_ids() == []  # evicted mid-flight
+    assert ("free_reservation", "r") in actor._svc.calls  # reservation not orphaned
+
+
+@pytest.mark.asyncio
+async def test_tracks_streamed_request_state(build_token_tracking_engine):
+    """End-to-end: exact token counts land as actor block state over ``.remote``."""
+    actor = RecordingKVRouterActor.remote(block_size=8)
+    # prompt 12, block_size 8: baseline ceil(12/8)=2; boundaries at cumulative
+    # output 5 and 13 -> 9 generated tokens cross only the first.
+    engine = build_token_tracking_engine(delta_steps(9, prompt_len=12), actor)
+
+    prompt = {"prompt_token_ids": list(range(12))}
+    stream = engine.generate(
+        prompt,
+        SamplingParams(output_kind=RequestOutputKind.DELTA, max_tokens=MAX_TOKENS),
+        "req-e2e",
+    )
+    for _ in range(9):
+        await stream.__anext__()
+    await drain(engine)
+
+    # All outputs consumed but the stream is still open: the request is
+    # tracked with its exact final counts.
+    assert await actor.get_request_lifecycle.remote("req-e2e") == {
+        "worker_id": WORKER_ID,
+        "prompt_tokens": 12,
+        "expected_output_tokens": MAX_TOKENS,
+        "prefill_completed": True,
+        "output_tokens": 9,
+        "total_blocks": 3,
+    }
+    assert await actor.get_worker_active_load.remote(WORKER_ID) == 1
+
+    # Stream end fires completion, which evicts the request.
+    with pytest.raises(StopAsyncIteration):
+        await stream.__anext__()
+    await drain(engine)
+    assert await actor.get_request_lifecycle.remote("req-e2e") is None
+    assert await actor.get_active_request_ids.remote() == []
+    assert await actor.get_worker_active_load.remote(WORKER_ID) == 0
+
+
+@pytest.mark.asyncio
+async def test_lifecycle_books_selection_service_load(build_token_tracking_engine):
+    """A streamed request books create_reservation -> prefill_complete -> one
+    add_output_block per crossed decode block -> free_reservation, in order."""
+    actor = RecordingKVRouterActor.remote(block_size=8)
+    # prompt 12 (2 blocks); cumulative output crosses into block 3 at 5 tokens
+    # and block 4 at 13 tokens -> exactly two output blocks over 20 tokens.
+    engine = build_token_tracking_engine(delta_steps(20, prompt_len=12), actor)
+
+    prompt = {"prompt_token_ids": list(range(12))}
+    await consume(
+        engine.generate(
+            prompt,
+            SamplingParams(output_kind=RequestOutputKind.DELTA, max_tokens=MAX_TOKENS),
+            "req-1",
+        )
+    )
+    await drain(engine)
+
+    calls = ray.get(actor.get_selection_service_calls.remote())
+    assert op_names(calls) == (
+        ["create_reservation", "prefill_complete"]
+        + ["add_output_block"] * 2
+        + ["free_reservation"]
+    )
+    assert calls[0] == ("create_reservation", "req-1", WORKER_ID, 12, MAX_TOKENS)
+    assert calls[-1] == ("free_reservation", "req-1")
+
+
+@pytest.mark.asyncio
+async def test_decode_blocks_book_add_output_block():
+    """Each crossed decode block books one add_output_block in the service."""
+    actor = LocalKVRouterActor(block_size=16)
+    await actor.on_request_added("r", WORKER_ID, list(range(10)))  # 1 prompt block
+    await actor.on_decode_progress("r", 6)  # 16 -> still 1 block
+    await actor.on_decode_progress("r", 7)  # 17 -> crosses into block 2
+    await actor.on_decode_progress("r", 39)  # 49 -> ceil=4, crosses two more
+
+    assert (
+        op_names(actor._svc.calls) == ["create_reservation"] + ["add_output_block"] * 3
+    )
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize(
+    "expected_output_tokens, expected_decay",
+    [
+        pytest.param(40, pytest.approx(1.0 - 8 / 40), id="with-estimate"),
+        pytest.param(None, None, id="no-estimate"),
+    ],
+)
+async def test_expected_output_tokens_sets_decay_fraction(
+    expected_output_tokens, expected_decay
+):
+    """With an output-length estimate each booked decode block decays by the
+    remaining fraction; without one the block carries no decay."""
+    actor = LocalKVRouterActor(block_size=8)
+    await actor.on_request_added(
+        "r", WORKER_ID, list(range(8)), expected_output_tokens=expected_output_tokens
+    )
+    await actor.on_decode_progress("r", 8)  # total 16 -> crosses into block 2
+
+    block_calls = [c for c in actor._svc.calls if c[0] == "add_output_block"]
+    assert block_calls == [("add_output_block", "r", expected_decay)]
+
+
+if __name__ == "__main__":
+    sys.exit(pytest.main(["-v", __file__]))

python/ray/llm/tests/serve/cpu/deployments/routers/test_builder_ingress.py

@@ -0,0 +1,805 @@
+import os
+import re
+import signal
+import subprocess
+import sys
+import tempfile
+
+import pytest
+import yaml
+
+from ray import serve
+from ray._common.test_utils import wait_for_condition
+from ray.llm._internal.serve.constants import DEFAULT_MAX_TARGET_ONGOING_REQUESTS
+from ray.llm._internal.serve.core.configs.llm_config import (
+    LLMConfig,
+    ModelLoadingConfig,
+)
+from ray.llm._internal.serve.core.ingress.builder import (
+    IngressClsConfig,
+    LLMServingArgs,
+    build_openai_app,
+)
+from ray.llm._internal.serve.core.ingress.ingress import OpenAiIngress
+from ray.llm._internal.serve.serving_patterns.data_parallel.builder import (
+    build_dp_openai_app,
+)
+from ray.llm._internal.serve.serving_patterns.data_parallel.dp_server import (
+    DPServer,
+)
+from ray.llm._internal.serve.serving_patterns.prefill_decode.builder import (
+    build_pd_openai_app,
+)
+from ray.llm._internal.serve.serving_patterns.prefill_decode.pd_server import (
+    DPPDDecodeServer,
+    DPPDPrefillServer,
+    PDDecodeServer,
+    PDPrefillServer,
+)
+from ray.serve._private.http_util import ASGIAppReplicaWrapper
+from ray.serve.config import AutoscalingConfig, RequestRouterConfig
+from ray.serve.experimental.consistent_hash_router import ConsistentHashRouter
+from ray.serve.experimental.round_robin_router import RoundRobinRouter
+
+
+@pytest.fixture
+def get_llm_serve_args(llm_config_with_mock_engine):
+    yield LLMServingArgs(llm_configs=[llm_config_with_mock_engine])
+
+
+@pytest.fixture()
+def serve_config_separate_model_config_files():
+    config_dir = tempfile.mkdtemp()
+    serve_config_filename = "llm_app_separate_model_config_files.yaml"
+    config_root = os.path.join(os.path.dirname(__file__), "test_config_files")
+    serve_config_src = os.path.join(config_root, serve_config_filename)
+    serve_config_dst = os.path.join(config_dir, serve_config_filename)
+
+    with open(serve_config_src, "r") as f:
+        serve_config_yaml = yaml.safe_load(f)
+
+    for application in serve_config_yaml["applications"]:
+        llm_configs = application["args"]["llm_configs"]
+        tmp_llm_config_files = []
+        for llm_config in llm_configs:
+            llm_config_src = llm_config.replace(".", config_root, 1)
+            llm_config_dst = llm_config.replace(".", config_dir, 1)
+            tmp_llm_config_files.append(llm_config_dst)
+
+            with open(llm_config_src, "r") as f:
+                llm_config_yaml = yaml.safe_load(f)
+
+            # Make sure engine is mocked.
+            if llm_config_yaml.get("runtime_env", None) is None:
+                llm_config_yaml["runtime_env"] = {}
+            llm_config_yaml["runtime_env"]["env_vars"] = {
+                "RAYLLM_VLLM_ENGINE_CLS": "ray.llm.tests.serve.mocks.mock_vllm_engine.MockVLLMEngine"
+            }
+
+            # Explicitly set accelerator_type to None to avoid GPU placement groups
+            llm_config_yaml["accelerator_type"] = None
+
+            # Use placement_group_config to specify CPU-only bundles
+            llm_config_yaml["placement_group_config"] = {
+                "bundles": [{"CPU": 1, "GPU": 0}]
+            }
+
+            os.makedirs(os.path.dirname(llm_config_dst), exist_ok=True)
+            with open(llm_config_dst, "w") as f:
+                yaml.dump(llm_config_yaml, f)
+
+        application["args"]["llm_configs"] = tmp_llm_config_files
+
+    with open(serve_config_dst, "w") as f:
+        yaml.dump(serve_config_yaml, f)
+
+    yield serve_config_dst
+
+
+class TestLLMServingArgs:
+    """Test suite for LLMServingArgs data model."""
+
+    @pytest.fixture
+    def llm_config(self):
+        """Basic LLMConfig for testing."""
+        return LLMConfig(
+            model_loading_config=ModelLoadingConfig(
+                model_id="test-model", model_source="test-source"
+            )
+        )
+
+    def test_basic_creation_and_defaults(self, llm_config):
+        """Test creation with minimal config and verify defaults."""
+        args = LLMServingArgs(llm_configs=[llm_config])
+
+        # Verify llm_configs
+        assert len(args.llm_configs) == 1
+        assert isinstance(args.llm_configs[0], LLMConfig)
+
+        # Verify defaults
+        assert isinstance(args.ingress_cls_config, IngressClsConfig)
+        assert args.ingress_cls_config.ingress_cls == OpenAiIngress
+        assert args.ingress_deployment_config == {}
+
+    def test_flexible_input_types(self, llm_config):
+        """Test accepts dicts, objects, and mixed types for llm_configs."""
+        config_dict = {
+            "model_loading_config": {
+                "model_id": "test-model-2",
+                "model_source": "test-source-2",
+            }
+        }
+        args = LLMServingArgs(llm_configs=[llm_config, config_dict])
+        assert len(args.llm_configs) == 2
+        assert all(isinstance(c, LLMConfig) for c in args.llm_configs)
+
+    def test_ingress_config_flexibility(self, llm_config):
+        """Test ingress_cls_config: defaults, dict input, object input, and class loading."""
+        # Test defaults
+        args_default = LLMServingArgs(llm_configs=[llm_config])
+        assert isinstance(args_default.ingress_cls_config, IngressClsConfig)
+        assert args_default.ingress_cls_config.ingress_cls == OpenAiIngress
+        assert args_default.ingress_cls_config.ingress_extra_kwargs == {}
+
+        # Test as dict with custom kwargs
+        args_dict = LLMServingArgs(
+            llm_configs=[llm_config],
+            ingress_cls_config={"ingress_extra_kwargs": {"key": "value"}},
+        )
+        assert isinstance(args_dict.ingress_cls_config, IngressClsConfig)
+        assert args_dict.ingress_cls_config.ingress_extra_kwargs == {"key": "value"}
+
+        # Test as object
+        args_obj = LLMServingArgs(
+            llm_configs=[llm_config],
+            ingress_cls_config=IngressClsConfig(ingress_extra_kwargs={"key": "value"}),
+        )
+        assert isinstance(args_obj.ingress_cls_config, IngressClsConfig)
+        assert args_obj.ingress_cls_config.ingress_extra_kwargs == {"key": "value"}
+
+        # Test class loading from string
+        args_str = LLMServingArgs(
+            llm_configs=[llm_config],
+            ingress_cls_config={
+                "ingress_cls": "ray.llm._internal.serve.core.ingress.ingress:OpenAiIngress"
+            },
+        )
+        assert args_str.ingress_cls_config.ingress_cls == OpenAiIngress
+
+    def test_validation_rules(self):
+        """Test validation: unique model IDs and non-empty list."""
+        # Duplicate model IDs
+        config1 = LLMConfig(
+            model_loading_config=ModelLoadingConfig(
+                model_id="same-id", model_source="source1"
+            )
+        )
+        config2 = LLMConfig(
+            model_loading_config=ModelLoadingConfig(
+                model_id="same-id", model_source="source2"
+            )
+        )
+        with pytest.raises(ValueError, match="Duplicate models found"):
+            LLMServingArgs(llm_configs=[config1, config2])
+
+        # Empty list
+        with pytest.raises(ValueError, match="List of models is empty"):
+            LLMServingArgs(llm_configs=[])
+
+
+class TestBuildOpenaiApp:
+    @pytest.fixture
+    def llm_config(self):
+        """Basic LLMConfig for testing."""
+        return LLMConfig(
+            model_loading_config=ModelLoadingConfig(
+                model_id="test-model", model_source="test-source"
+            )
+        )
+
+    def test_build_openai_app(
+        self, get_llm_serve_args, shutdown_ray_and_serve, disable_placement_bundles
+    ):
+        """Test `build_openai_app` can build app and run it with Serve."""
+
+        app = build_openai_app(
+            get_llm_serve_args,
+        )
+        assert isinstance(app, serve.Application)
+        serve.run(app)
+
+    def test_build_openai_app_with_config(
+        self,
+        serve_config_separate_model_config_files,
+        shutdown_ray_and_serve,
+        disable_placement_bundles,
+    ):
+        """Test `build_openai_app` can be used in serve config."""
+
+        def deployments_healthy():
+            status_response = subprocess.check_output(["serve", "status"])
+            print("[TEST] Status response: ", status_response)
+            applications = extract_applications_from_output(status_response)
+
+            if "llm-endpoint" not in applications:
+                print("[TEST] Application 'llm-endpoint' not found.")
+                return False
+
+            llm_endpoint_status = applications["llm-endpoint"]
+            if len(llm_endpoint_status["deployments"]) != 2:
+                print(
+                    f"[TEST] Expected 2 deployments, found {len(llm_endpoint_status['deployments'])}"
+                )
+                return False
+
+            deployment_status = llm_endpoint_status["deployments"].values()
+            if not all([status["status"] == "HEALTHY" for status in deployment_status]):
+                print(f"[TEST] Not all deployments healthy: {deployment_status}")
+                return False
+
+            print("[TEST] All deployments healthy.")
+            return True
+
+        p = subprocess.Popen(["serve", "run", serve_config_separate_model_config_files])
+        wait_for_condition(deployments_healthy, timeout=60, retry_interval_ms=1000)
+
+        p.send_signal(signal.SIGINT)  # Equivalent to ctrl-C
+        p.wait()
+
+    def test_router_built_with_autoscaling_configs(self, disable_placement_bundles):
+        """Test that the router is built with the correct autoscaling configs that
+        will scale.
+        """
+        llm_config_no_autoscaling_configured = LLMConfig(
+            model_loading_config=ModelLoadingConfig(model_id="model_id_1"),
+            accelerator_type="L4",
+        )
+        llm_config_autoscaling_default = LLMConfig(
+            model_loading_config=ModelLoadingConfig(model_id="model_id_2"),
+            accelerator_type="L4",
+            deployment_config={"autoscaling_config": AutoscalingConfig()},
+        )
+        llm_config_autoscaling_non_default = LLMConfig(
+            model_loading_config=ModelLoadingConfig(model_id="model_id_3"),
+            accelerator_type="L4",
+            deployment_config={
+                "autoscaling_config": AutoscalingConfig(
+                    min_replicas=2,
+                    initial_replicas=3,
+                    max_replicas=4,
+                )
+            },
+        )
+
+        app = build_openai_app(
+            LLMServingArgs(
+                llm_configs=[
+                    llm_config_no_autoscaling_configured,
+                    llm_config_autoscaling_default,
+                    llm_config_autoscaling_non_default,
+                ],
+                ingress_deployment_config={
+                    "autoscaling_config": {
+                        "min_replicas": 8,
+                        "initial_replicas": 10,
+                        "max_replicas": 12,
+                        "target_ongoing_requests": 10,
+                    }
+                },
+            )
+        )
+        router_autoscaling_config = (
+            app._bound_deployment._deployment_config.autoscaling_config
+        )
+        assert router_autoscaling_config.min_replicas == 8  # (1 + 1 + 2) * 2
+        assert router_autoscaling_config.initial_replicas == 10  # (1 + 1 + 3) * 2
+        assert router_autoscaling_config.max_replicas == 12  # (1 + 1 + 4) * 2
+        assert router_autoscaling_config.target_ongoing_requests == 10
+
+    def test_ingress_deployment_config_merging(
+        self, llm_config, disable_placement_bundles
+    ):
+        """Test that ingress_deployment_config is properly merged with default options.
+
+        This test ensures that deep_merge_dicts return value is properly assigned
+        and that nested dictionaries are properly deep-merged without losing default values.
+        """
+        # Build app with custom ingress deployment config including nested options
+        app = build_openai_app(
+            dict(
+                llm_configs=[llm_config],
+                ingress_deployment_config={
+                    "num_replicas": 3,
+                    "ray_actor_options": {
+                        "num_cpus": 4,
+                        "memory": 1024,
+                    },
+                    "max_ongoing_requests": 200,  # Override default
+                },
+            )
+        )
+
+        # Verify the custom config was applied
+        deployment = app._bound_deployment
+        assert deployment._deployment_config.num_replicas == 3
+        assert deployment.ray_actor_options["num_cpus"] == 4
+        assert deployment.ray_actor_options["memory"] == 1024
+        assert deployment._deployment_config.max_ongoing_requests == 200
+
+    def test_default_autoscaling_config_included_without_num_replicas(
+        self, llm_config, disable_placement_bundles
+    ):
+        """Test that default autoscaling_config with target_ongoing_requests is included
+        when num_replicas is not specified.
+        """
+        app = build_openai_app(
+            dict(
+                llm_configs=[llm_config],
+            )
+        )
+
+        deployment = app._bound_deployment
+        autoscaling_config = deployment._deployment_config.autoscaling_config
+        assert autoscaling_config is not None
+        assert (
+            autoscaling_config.target_ongoing_requests
+            == DEFAULT_MAX_TARGET_ONGOING_REQUESTS
+        )
+
+    def test_autoscaling_config_removed_from_defaults_when_num_replicas_specified(
+        self, llm_config, disable_placement_bundles
+    ):
+        """Test that autoscaling_config from defaults is removed when user specifies
+        num_replicas, since Ray Serve does not allow both.
+        """
+        app = build_openai_app(
+            dict(
+                llm_configs=[llm_config],
+                ingress_deployment_config={
+                    "num_replicas": 2,
+                },
+            )
+        )
+
+        deployment = app._bound_deployment
+        assert deployment._deployment_config.num_replicas == 2
+        # autoscaling_config should be None since num_replicas is set
+        assert deployment._deployment_config.autoscaling_config is None
+
+    def test_user_target_ongoing_requests_respected(
+        self, llm_config, disable_placement_bundles
+    ):
+        """Test that user-specified target_ongoing_requests is respected and not
+        overridden by defaults.
+        """
+        user_target = 50
+        app = build_openai_app(
+            dict(
+                llm_configs=[llm_config],
+                ingress_deployment_config={
+                    "autoscaling_config": {
+                        "target_ongoing_requests": user_target,
+                    },
+                },
+            )
+        )
+
+        deployment = app._bound_deployment
+        autoscaling_config = deployment._deployment_config.autoscaling_config
+        assert autoscaling_config is not None
+        assert autoscaling_config.target_ongoing_requests == user_target
+
+    def test_direct_streaming_builds_ingress_with_router_attached(
+        self, llm_config, disable_placement_bundles, monkeypatch
+    ):
+        monkeypatch.setattr(
+            "ray.llm._internal.serve.core.ingress.builder."
+            "RAY_SERVE_LLM_ENABLE_DIRECT_STREAMING",
+            True,
+        )
+
+        app = build_openai_app(LLMServingArgs(llm_configs=[llm_config]))
+        ingress_request_router = app._ingress_request_router
+
+        assert app._bound_deployment.name == "LLMServer:test-model"
+        assert issubclass(app._bound_deployment.func_or_class, ASGIAppReplicaWrapper)
+        assert ingress_request_router is not None
+        assert ingress_request_router._bound_deployment.name == "LLMRouter"
+        assert ingress_request_router._bound_deployment.init_kwargs["server"] is app
+
+        # `RequestRouterConfig._serialize_request_router_cls` normalizes the
+        # class to its import path at config-build time.
+        request_router_config = (
+            app._bound_deployment._deployment_config.request_router_config
+        )
+        assert request_router_config.request_router_class == (
+            f"{RoundRobinRouter.__module__}.{RoundRobinRouter.__name__}"
+        )
+
+    def test_direct_streaming_user_request_router_config_wins(
+        self, llm_config, disable_placement_bundles, monkeypatch
+    ):
+        """A user-supplied ``request_router_config`` on ``LLMConfig`` must
+        survive direct-streaming wiring rather than being overwritten with the
+        default.
+        """
+        monkeypatch.setattr(
+            "ray.llm._internal.serve.core.ingress.builder."
+            "RAY_SERVE_LLM_ENABLE_DIRECT_STREAMING",
+            True,
+        )
+        llm_config.deployment_config["request_router_config"] = RequestRouterConfig(
+            request_router_class=ConsistentHashRouter,
+        )
+
+        app = build_openai_app(LLMServingArgs(llm_configs=[llm_config]))
+        request_router_config = (
+            app._bound_deployment._deployment_config.request_router_config
+        )
+        assert request_router_config.request_router_class == (
+            f"{ConsistentHashRouter.__module__}.{ConsistentHashRouter.__name__}"
+        )
+
+    def test_direct_streaming_rejects_multiple_llm_configs(
+        self, llm_config, disable_placement_bundles, monkeypatch
+    ):
+        monkeypatch.setattr(
+            "ray.llm._internal.serve.core.ingress.builder."
+            "RAY_SERVE_LLM_ENABLE_DIRECT_STREAMING",
+            True,
+        )
+        other_llm_config = LLMConfig(
+            model_loading_config=ModelLoadingConfig(model_id="other-model")
+        )
+
+        with pytest.raises(
+            ValueError,
+            match="currently supports exactly one LLM config",
+        ):
+            build_openai_app(LLMServingArgs(llm_configs=[llm_config, other_llm_config]))
+
+    @pytest.mark.parametrize(
+        ("builder_kwargs", "match"),
+        [
+            (
+                {"ingress_deployment_config": {"num_replicas": 2}},
+                "does not support ingress_deployment_config",
+            ),
+            (
+                {"ingress_cls_config": {"ingress_extra_kwargs": {"key": "value"}}},
+                "does not support ingress_cls_config",
+            ),
+        ],
+    )
+    def test_direct_streaming_rejects_ingress_config(
+        self,
+        llm_config,
+        disable_placement_bundles,
+        monkeypatch,
+        builder_kwargs,
+        match,
+    ):
+        monkeypatch.setattr(
+            "ray.llm._internal.serve.core.ingress.builder."
+            "RAY_SERVE_LLM_ENABLE_DIRECT_STREAMING",
+            True,
+        )
+
+        with pytest.raises(ValueError, match=match):
+            build_openai_app(LLMServingArgs(llm_configs=[llm_config], **builder_kwargs))
+
+
+class TestDirectStreamingDP:
+    """Direct-streaming wiring tests for the data-parallel builder.
+
+    Mirrors the ``test_direct_streaming_*`` tests on ``TestBuildOpenaiApp``
+    but exercises ``build_dp_openai_app`` so that regressions in the DP
+    wiring (deployment class, default request router) are caught at CPU
+    unit-test speed instead of in GPU integration / release tests.
+    """
+
+    @pytest.fixture
+    def llm_config(self):
+        return LLMConfig(
+            model_loading_config=ModelLoadingConfig(
+                model_id="test-model", model_source="test-source"
+            )
+        )
+
+    def _enable_direct_streaming(self, monkeypatch):
+        monkeypatch.setattr(
+            "ray.llm._internal.serve.serving_patterns.data_parallel.builder."
+            "RAY_SERVE_LLM_ENABLE_DIRECT_STREAMING",
+            True,
+        )
+
+    def test_dp_builds_dpserver_ingress_with_router_attached(
+        self, llm_config, disable_placement_bundles, monkeypatch
+    ):
+        self._enable_direct_streaming(monkeypatch)
+
+        app = build_dp_openai_app({"llm_config": llm_config})
+        ingress_request_router = app._ingress_request_router
+
+        assert app._bound_deployment.name == "DPServer:test-model"
+        assert issubclass(app._bound_deployment.func_or_class, ASGIAppReplicaWrapper)
+        assert issubclass(app._bound_deployment.func_or_class, DPServer)
+        assert ingress_request_router is not None
+        assert ingress_request_router._bound_deployment.name == "LLMRouter"
+        assert ingress_request_router._bound_deployment.init_kwargs["server"] is app
+
+        request_router_config = (
+            app._bound_deployment._deployment_config.request_router_config
+        )
+        assert request_router_config.request_router_class == (
+            f"{RoundRobinRouter.__module__}.{RoundRobinRouter.__name__}"
+        )
+
+    def test_dp_user_request_router_config_wins(
+        self, llm_config, disable_placement_bundles, monkeypatch
+    ):
+        """A user-supplied ``request_router_config`` on ``LLMConfig`` must
+        survive DP direct-streaming wiring rather than being overwritten with
+        the default ``RoundRobinRouter``.
+        """
+        self._enable_direct_streaming(monkeypatch)
+        llm_config.deployment_config["request_router_config"] = RequestRouterConfig(
+            request_router_class=ConsistentHashRouter,
+        )
+
+        app = build_dp_openai_app({"llm_config": llm_config})
+        request_router_config = (
+            app._bound_deployment._deployment_config.request_router_config
+        )
+        assert request_router_config.request_router_class == (
+            f"{ConsistentHashRouter.__module__}.{ConsistentHashRouter.__name__}"
+        )
+
+
+class TestDirectStreamingPD:
+    """Direct-streaming wiring tests for the prefill/decode builder.
+
+    Covers the decode-class selection (``PDDecodeServer`` vs
+    ``DPPDDecodeServer`` based on ``decode_dp_size``), the prefill binding
+    into decode's init kwargs, and the ``LLMRouter`` ingress-request-router
+    hookup.
+    """
+
+    @pytest.fixture
+    def pd_configs(self):
+        """Prefill and decode configs with required kv_transfer_config."""
+        base_config = {
+            "model_loading_config": {
+                "model_id": "test-model",
+                "model_source": "test-source",
+            },
+            "engine_kwargs": {
+                "kv_transfer_config": {
+                    "kv_connector": "NixlConnector",
+                    "kv_role": "kv_both",
+                },
+            },
+        }
+        prefill = LLMConfig.model_validate(base_config)
+        decode = LLMConfig.model_validate(base_config)
+        return prefill, decode
+
+    def _enable_direct_streaming(self, monkeypatch):
+        monkeypatch.setattr(
+            "ray.llm._internal.serve.serving_patterns.prefill_decode.builder."
+            "RAY_SERVE_LLM_ENABLE_DIRECT_STREAMING",
+            True,
+        )
+
+    @staticmethod
+    def _set_dp_size(llm_config, size):
+        llm_config.engine_kwargs["data_parallel_size"] = size
+
+    @pytest.mark.parametrize(
+        ("prefill_dp", "decode_dp", "expected_prefill_cls", "expected_decode_cls"),
+        [
+            (1, 1, PDPrefillServer, PDDecodeServer),
+            (1, 4, PDPrefillServer, DPPDDecodeServer),
+            (4, 1, DPPDPrefillServer, PDDecodeServer),
+            (4, 4, DPPDPrefillServer, DPPDDecodeServer),
+        ],
+    )
+    def test_pd_decode_class_selection(
+        self,
+        pd_configs,
+        disable_placement_bundles,
+        monkeypatch,
+        prefill_dp,
+        decode_dp,
+        expected_prefill_cls,
+        expected_decode_cls,
+    ):
+        """Verify the DP-vs-non-DP variants are picked based on
+        ``data_parallel_size`` for both prefill and decode legs.
+        """
+        self._enable_direct_streaming(monkeypatch)
+        prefill, decode = pd_configs
+        self._set_dp_size(prefill, prefill_dp)
+        self._set_dp_size(decode, decode_dp)
+
+        app = build_pd_openai_app({"prefill_config": prefill, "decode_config": decode})
+
+        decode_deployment = app._bound_deployment
+        assert issubclass(decode_deployment.func_or_class, ASGIAppReplicaWrapper)
+        assert issubclass(decode_deployment.func_or_class, expected_decode_cls)
+
+        prefill_app = decode_deployment.init_kwargs["prefill_server"]
+        prefill_deployment = prefill_app._bound_deployment
+        assert prefill_deployment.func_or_class is expected_prefill_cls
+
+    def test_pd_ingress_request_router_is_llmrouter(
+        self, pd_configs, disable_placement_bundles, monkeypatch
+    ):
+        self._enable_direct_streaming(monkeypatch)
+        prefill, decode = pd_configs
+
+        app = build_pd_openai_app({"prefill_config": prefill, "decode_config": decode})
+        ingress_request_router = app._ingress_request_router
+
+        assert ingress_request_router is not None
+        assert ingress_request_router._bound_deployment.name == "LLMRouter"
+        assert ingress_request_router._bound_deployment.init_kwargs["server"] is app
+
+        request_router_config = (
+            app._bound_deployment._deployment_config.request_router_config
+        )
+        assert request_router_config.request_router_class == (
+            f"{RoundRobinRouter.__module__}.{RoundRobinRouter.__name__}"
+        )
+
+    def test_pd_user_request_router_config_wins(
+        self, pd_configs, disable_placement_bundles, monkeypatch
+    ):
+        """A user-supplied ``request_router_config`` on the decode
+        ``LLMConfig`` must survive PD direct-streaming wiring rather than
+        being overwritten with the default ``RoundRobinRouter``.
+        """
+        self._enable_direct_streaming(monkeypatch)
+        prefill, decode = pd_configs
+        decode.deployment_config["request_router_config"] = RequestRouterConfig(
+            request_router_class=ConsistentHashRouter,
+        )
+
+        app = build_pd_openai_app({"prefill_config": prefill, "decode_config": decode})
+        request_router_config = (
+            app._bound_deployment._deployment_config.request_router_config
+        )
+        assert request_router_config.request_router_class == (
+            f"{ConsistentHashRouter.__module__}.{ConsistentHashRouter.__name__}"
+        )
+
+
+class TestIngressScaleToZero:
+    """Tests for ingress scale-to-zero behavior when all models have min_replicas=0."""
+
+    def test_all_models_scale_to_zero(self, disable_placement_bundles):
+        """When all models have min_replicas=0, ingress should also have min_replicas=0."""
+        llm_cfg_dict_autoscaling = LLMConfig(
+            model_loading_config=ModelLoadingConfig(model_id="model_a"),
+            accelerator_type="L4",
+            deployment_config={
+                "autoscaling_config": {
+                    "min_replicas": 0,
+                    "max_replicas": 2,
+                }
+            },
+        )
+        llm_cfg_obj_autoscaling = LLMConfig(
+            model_loading_config=ModelLoadingConfig(model_id="model_b"),
+            accelerator_type="L4",
+            deployment_config={
+                "autoscaling_config": AutoscalingConfig(
+                    min_replicas=0,
+                    max_replicas=4,
+                )
+            },
+        )
+
+        app = build_openai_app(
+            LLMServingArgs(
+                llm_configs=[llm_cfg_dict_autoscaling, llm_cfg_obj_autoscaling],
+            )
+        )
+        autoscaling_config = app._bound_deployment._deployment_config.autoscaling_config
+        assert autoscaling_config.min_replicas == 0
+
+    def test_mixed_min_replicas_keeps_default(self, disable_placement_bundles):
+        """When some models have min_replicas>0, ingress should keep default min_replicas."""
+        llm_cfg_zero = LLMConfig(
+            model_loading_config=ModelLoadingConfig(model_id="model_a"),
+            accelerator_type="L4",
+            deployment_config={
+                "autoscaling_config": {
+                    "min_replicas": 0,
+                    "max_replicas": 2,
+                }
+            },
+        )
+        llm_cfg_nonzero = LLMConfig(
+            model_loading_config=ModelLoadingConfig(model_id="model_b"),
+            accelerator_type="L4",
+            deployment_config={
+                "autoscaling_config": AutoscalingConfig(
+                    min_replicas=1,
+                    max_replicas=4,
+                )
+            },
+        )
+
+        app = build_openai_app(
+            LLMServingArgs(
+                llm_configs=[llm_cfg_zero, llm_cfg_nonzero],
+            )
+        )
+        autoscaling_config = app._bound_deployment._deployment_config.autoscaling_config
+        # Default min_replicas from AutoscalingConfig is 1
+        assert autoscaling_config.min_replicas == 1
+
+    def test_no_autoscaling_config_keeps_default(self, disable_placement_bundles):
+        """When models don't have autoscaling_config, ingress should keep default."""
+        llm_cfg = LLMConfig(
+            model_loading_config=ModelLoadingConfig(model_id="model_a"),
+            accelerator_type="L4",
+        )
+
+        app = build_openai_app(
+            LLMServingArgs(llm_configs=[llm_cfg]),
+        )
+        autoscaling_config = app._bound_deployment._deployment_config.autoscaling_config
+        assert autoscaling_config.min_replicas == 1
+
+    def test_user_override_takes_precedence(self, disable_placement_bundles):
+        """User-specified ingress min_replicas should override scale-to-zero logic."""
+        llm_cfg = LLMConfig(
+            model_loading_config=ModelLoadingConfig(model_id="model_a"),
+            accelerator_type="L4",
+            deployment_config={
+                "autoscaling_config": {
+                    "min_replicas": 0,
+                    "max_replicas": 2,
+                }
+            },
+        )
+
+        app = build_openai_app(
+            LLMServingArgs(
+                llm_configs=[llm_cfg],
+                ingress_deployment_config={
+                    "autoscaling_config": {
+                        "min_replicas": 3,
+                        "max_replicas": 5,
+                    }
+                },
+            )
+        )
+        autoscaling_config = app._bound_deployment._deployment_config.autoscaling_config
+        assert autoscaling_config.min_replicas == 3
+
+
+def extract_applications_from_output(output: bytes) -> dict:
+    """
+    Extracts the 'applications' block from mixed output and returns it as a dict.
+    """
+    # 1. Decode bytes to string
+    text = output.decode("utf-8", errors="ignore")
+
+    # 2. Regex to find the 'applications:' block and its indented content
+    #    This matches 'applications:' and all following lines that are indented (YAML block)
+    match = re.search(r"(^applications:\n(?:^(?: {2,}|\t).*\n?)+)", text, re.MULTILINE)
+    if not match:
+        raise ValueError("Could not find 'applications:' block in output.")
+
+    applications_block = match.group(1)
+
+    # 3. Parse the YAML block
+    applications_dict = yaml.safe_load(applications_block)
+    return applications_dict["applications"]
+
+
+if __name__ == "__main__":
+    sys.exit(pytest.main(["-v", __file__]))

python/ray/llm/tests/serve/cpu/deployments/routers/test_config_files/llm_app_separate_model_config_files.yaml

@@ -0,0 +1,7 @@
+applications:
+  - args:
+      llm_configs:
+        - ./model_config/llm_config.yaml
+    import_path: ray.serve.llm:build_openai_app
+    name: llm-endpoint
+    route_prefix: /

python/ray/llm/tests/serve/cpu/deployments/routers/test_config_files/model_config/llm_config.yaml

@@ -0,0 +1,2 @@
+model_loading_config:
+  model_id: model1

python/ray/llm/tests/serve/cpu/deployments/routers/test_dev_ingress.py

@@ -0,0 +1,322 @@
+"""Tests for DevIngress control plane endpoints.
+
+This module tests the HTTP endpoints exposed by DevIngress:
+- POST /sleep, POST /wakeup, GET /is_sleeping
+- POST /pause, POST /resume, GET /is_paused
+- POST /reset_prefix_cache
+
+These tests verify:
+1. Endpoints are correctly registered and accessible
+2. Broadcast API correctly broadcasts to replicas
+3. Sleep/wakeup and pause/resume isolation between different models
+"""
+
+import sys
+
+import httpx
+import pytest
+
+import ray
+from ray import serve
+from ray.llm._internal.serve.core.configs.openai_api_models import to_model_metadata
+from ray.llm._internal.serve.core.ingress.dev_ingress import DEV_ENDPOINTS, DevIngress
+from ray.llm._internal.serve.core.ingress.ingress import make_fastapi_ingress
+from ray.llm._internal.serve.core.server.llm_server import LLMServer
+from ray.llm.tests.serve.mocks.mock_vllm_engine import MockVLLMEngine
+from ray.serve.llm import LLMConfig, ModelLoadingConfig
+
+
+@pytest.fixture(scope="module")
+def ray_instance():
+    """Initialize Ray for the module."""
+    if not ray.is_initialized():
+        ray.init()
+    yield
+    serve.shutdown()
+    ray.shutdown()
+
+
+@pytest.fixture
+def single_model_dev_ingress(ray_instance, disable_placement_bundles):
+    """Start a Serve app with one model and DevIngress endpoints."""
+    model_id = "test-model-1"
+    llm_config = LLMConfig(
+        model_loading_config=ModelLoadingConfig(model_id=model_id),
+        runtime_env={},
+        log_engine_metrics=False,
+    )
+
+    # Create LLMServer deployment with mock engine
+    llm_deployment = serve.deployment(LLMServer).bind(
+        llm_config, engine_cls=MockVLLMEngine
+    )
+
+    # Create DevIngress with the dev endpoints
+    ingress_cls = make_fastapi_ingress(DevIngress, endpoint_map=DEV_ENDPOINTS)
+    ingress_options = DevIngress.get_deployment_options([llm_config])
+    ingress_app = serve.deployment(ingress_cls, **ingress_options).bind(
+        llm_deployments={model_id: llm_deployment},
+        model_cards={model_id: to_model_metadata(model_id, llm_config)},
+    )
+
+    serve.run(ingress_app, name="single-model-app")
+    yield model_id
+    serve.delete("single-model-app", _blocking=True)
+
+
+@pytest.fixture
+def two_model_dev_ingress(ray_instance, disable_placement_bundles):
+    """Start a Serve app with TWO model deployments to test isolation."""
+    model_id_1 = "test-model-1"
+    model_id_2 = "test-model-2"
+
+    llm_config_1 = LLMConfig(
+        model_loading_config=ModelLoadingConfig(model_id=model_id_1),
+        runtime_env={},
+        log_engine_metrics=False,
+    )
+    llm_config_2 = LLMConfig(
+        model_loading_config=ModelLoadingConfig(model_id=model_id_2),
+        runtime_env={},
+        log_engine_metrics=False,
+    )
+
+    # Create LLMServer deployments with mock engine
+    llm_deployment_1 = serve.deployment(LLMServer).bind(
+        llm_config_1, engine_cls=MockVLLMEngine
+    )
+    llm_deployment_2 = serve.deployment(LLMServer).bind(
+        llm_config_2, engine_cls=MockVLLMEngine
+    )
+
+    # Create DevIngress with the dev endpoints
+    ingress_cls = make_fastapi_ingress(DevIngress, endpoint_map=DEV_ENDPOINTS)
+    ingress_options = DevIngress.get_deployment_options([llm_config_1, llm_config_2])
+    ingress_app = serve.deployment(ingress_cls, **ingress_options).bind(
+        llm_deployments={
+            model_id_1: llm_deployment_1,
+            model_id_2: llm_deployment_2,
+        },
+        model_cards={
+            model_id_1: to_model_metadata(model_id_1, llm_config_1),
+            model_id_2: to_model_metadata(model_id_2, llm_config_2),
+        },
+    )
+
+    serve.run(ingress_app, name="two-model-app")
+    yield model_id_1, model_id_2
+    serve.delete("two-model-app", _blocking=True)
+
+
+class TestDevIngressEndpoints:
+    """Test DevIngress endpoints."""
+
+    @pytest.mark.asyncio
+    async def test_reset_prefix_cache_endpoint(self, single_model_dev_ingress):
+        """Test POST /reset_prefix_cache endpoint."""
+        model_id = single_model_dev_ingress
+
+        async with httpx.AsyncClient(timeout=60.0) as client:
+            response = await client.post(
+                "http://localhost:8000/reset_prefix_cache",
+                json={"model": model_id},
+            )
+            assert response.status_code == 200
+
+    @pytest.mark.asyncio
+    async def test_sleep_wakeup_cycle(self, single_model_dev_ingress):
+        """Test full sleep -> is_sleeping -> wakeup -> is_sleeping cycle."""
+        model_id = single_model_dev_ingress
+
+        async with httpx.AsyncClient(timeout=60.0) as client:
+            # Initial state - should not be sleeping
+            response = await client.get(
+                f"http://localhost:8000/is_sleeping?model={model_id}",
+            )
+            assert response.status_code == 200
+            assert response.json().get("is_sleeping") is False
+
+            # Sleep the engine
+            response = await client.post(
+                "http://localhost:8000/sleep",
+                json={"model": model_id, "options": {"level": 1}},
+            )
+            assert response.status_code == 200
+
+            # Check is_sleeping - should be True
+            response = await client.get(
+                f"http://localhost:8000/is_sleeping?model={model_id}",
+            )
+            assert response.status_code == 200
+            assert response.json().get("is_sleeping") is True
+
+            # Wake up the engine
+            response = await client.post(
+                "http://localhost:8000/wakeup",
+                json={"model": model_id, "options": {}},
+            )
+            assert response.status_code == 200
+
+            # Check is_sleeping - should be False again
+            response = await client.get(
+                f"http://localhost:8000/is_sleeping?model={model_id}",
+            )
+            assert response.status_code == 200
+            assert response.json().get("is_sleeping") is False
+
+    @pytest.mark.asyncio
+    async def test_pause_resume_cycle(self, single_model_dev_ingress):
+        """Test full pause -> is_paused -> resume -> is_paused cycle."""
+        model_id = single_model_dev_ingress
+
+        async with httpx.AsyncClient(timeout=60.0) as client:
+            # Initial state - should not be paused
+            response = await client.get(
+                f"http://localhost:8000/is_paused?model={model_id}",
+            )
+            assert response.status_code == 200
+            assert response.json().get("is_paused") is False
+
+            # Pause the engine
+            response = await client.post(
+                "http://localhost:8000/pause",
+                json={"model": model_id, "options": {"clear_cache": True}},
+            )
+            assert response.status_code == 200
+
+            # Check is_paused - should be True
+            response = await client.get(
+                f"http://localhost:8000/is_paused?model={model_id}",
+            )
+            assert response.status_code == 200
+            assert response.json().get("is_paused") is True
+
+            # Resume the engine
+            response = await client.post(
+                "http://localhost:8000/resume",
+                json={"model": model_id, "options": {}},
+            )
+            assert response.status_code == 200
+
+            # Check is_paused - should be False again
+            response = await client.get(
+                f"http://localhost:8000/is_paused?model={model_id}",
+            )
+            assert response.status_code == 200
+            assert response.json().get("is_paused") is False
+
+
+class TestDevIngressModelIsolation:
+    """Test that control plane operations are isolated per model."""
+
+    @pytest.mark.asyncio
+    async def test_sleep_wakeup_isolation(self, two_model_dev_ingress):
+        """Test that sleeping model_1 does NOT affect model_2."""
+        model_1, model_2 = two_model_dev_ingress
+
+        async with httpx.AsyncClient(timeout=60.0) as client:
+            # Both models should start awake
+            response = await client.get(
+                f"http://localhost:8000/is_sleeping?model={model_1}",
+            )
+            assert response.json().get("is_sleeping") is False
+
+            response = await client.get(
+                f"http://localhost:8000/is_sleeping?model={model_2}",
+            )
+            assert response.json().get("is_sleeping") is False
+
+            # Sleep model_1 only
+            response = await client.post(
+                "http://localhost:8000/sleep",
+                json={"model": model_1, "options": {"level": 1}},
+            )
+            assert response.status_code == 200
+
+            # model_1 should be sleeping
+            response = await client.get(
+                f"http://localhost:8000/is_sleeping?model={model_1}",
+            )
+            assert response.json().get("is_sleeping") is True
+
+            # model_2 should NOT be sleeping
+            response = await client.get(
+                f"http://localhost:8000/is_sleeping?model={model_2}",
+            )
+            assert response.json().get("is_sleeping") is False
+
+            # Wake up model_1
+            response = await client.post(
+                "http://localhost:8000/wakeup",
+                json={"model": model_1, "options": {}},
+            )
+            assert response.status_code == 200
+
+            # Both should now be awake
+            response = await client.get(
+                f"http://localhost:8000/is_sleeping?model={model_1}",
+            )
+            assert response.json().get("is_sleeping") is False
+
+            response = await client.get(
+                f"http://localhost:8000/is_sleeping?model={model_2}",
+            )
+            assert response.json().get("is_sleeping") is False
+
+    @pytest.mark.asyncio
+    async def test_pause_resume_isolation(self, two_model_dev_ingress):
+        """Test that pausing model_1 does NOT affect model_2."""
+        model_1, model_2 = two_model_dev_ingress
+
+        async with httpx.AsyncClient(timeout=60.0) as client:
+            # Both models should start unpaused
+            response = await client.get(
+                f"http://localhost:8000/is_paused?model={model_1}",
+            )
+            assert response.json().get("is_paused") is False
+
+            response = await client.get(
+                f"http://localhost:8000/is_paused?model={model_2}",
+            )
+            assert response.json().get("is_paused") is False
+
+            # Pause model_1 only
+            response = await client.post(
+                "http://localhost:8000/pause",
+                json={"model": model_1, "options": {"clear_cache": True}},
+            )
+            assert response.status_code == 200
+
+            # model_1 should be paused
+            response = await client.get(
+                f"http://localhost:8000/is_paused?model={model_1}",
+            )
+            assert response.json().get("is_paused") is True
+
+            # model_2 should NOT be paused
+            response = await client.get(
+                f"http://localhost:8000/is_paused?model={model_2}",
+            )
+            assert response.json().get("is_paused") is False
+
+            # Resume model_1
+            response = await client.post(
+                "http://localhost:8000/resume",
+                json={"model": model_1, "options": {}},
+            )
+            assert response.status_code == 200
+
+            # Both should now be unpaused
+            response = await client.get(
+                f"http://localhost:8000/is_paused?model={model_1}",
+            )
+            assert response.json().get("is_paused") is False
+
+            response = await client.get(
+                f"http://localhost:8000/is_paused?model={model_2}",
+            )
+            assert response.json().get("is_paused") is False
+
+
+if __name__ == "__main__":
+    sys.exit(pytest.main(["-v", __file__]))

python/ray/llm/tests/serve/cpu/deployments/routers/test_make_fastapi_ingress.py

@@ -0,0 +1,163 @@
+"""Tests for make_fastapi_ingress function."""
+
+import inspect
+import sys
+
+import pytest
+from fastapi import FastAPI, Request
+from fastapi.routing import APIRoute
+
+from ray.llm._internal.serve.core.ingress.ingress import (
+    DEFAULT_ENDPOINTS,
+    OpenAiIngress,
+    make_fastapi_ingress,
+)
+
+
+class TestMakeFastapiIngress:
+    """Test suite for make_fastapi_ingress."""
+
+    def test_subclass_inherits_endpoints(self):
+        """Test that subclassing OpenAiIngress works with make_fastapi_ingress."""
+
+        class MyCustomIngress(OpenAiIngress):
+            """Custom ingress that inherits all OpenAI endpoints."""
+
+            pass
+
+        app = FastAPI()
+        # Create the ingress class - should not raise
+        ingress_cls = make_fastapi_ingress(MyCustomIngress, app=app)
+
+        # Verify the ingress class was created successfully
+        assert ingress_cls is not None
+
+        # Verify routes are registered (inherited from OpenAiIngress)
+        route_paths = [
+            route.path for route in app.routes if isinstance(route, APIRoute)
+        ]
+        assert "/v1/models" in route_paths
+        assert "/v1/completions" in route_paths
+
+    def test_subclass_with_custom_method(self):
+        """Test that custom methods added by subclass are also properly handled."""
+
+        class MyCustomIngress(OpenAiIngress):
+            """Custom ingress with an additional endpoint."""
+
+            async def custom_endpoint(self, request: Request):
+                """A custom endpoint added by the subclass."""
+                return {"status": "ok"}
+
+        custom_endpoints = {
+            "custom_endpoint": lambda app: app.post("/custom"),
+            **DEFAULT_ENDPOINTS,
+        }
+
+        app = FastAPI()
+        ingress_cls = make_fastapi_ingress(
+            MyCustomIngress, endpoint_map=custom_endpoints, app=app
+        )
+
+        # Verify the class was created and the custom route is registered
+        assert ingress_cls is not None
+        route_paths = [
+            route.path for route in app.routes if isinstance(route, APIRoute)
+        ]
+        assert "/custom" in route_paths
+
+    def test_routes_registered_correctly(self):
+        """Test that routes are registered with the FastAPI app."""
+
+        class MyCustomIngress(OpenAiIngress):
+            pass
+
+        app = FastAPI()
+        make_fastapi_ingress(MyCustomIngress, app=app)
+
+        # Get all registered routes
+        route_paths = [
+            route.path for route in app.routes if isinstance(route, APIRoute)
+        ]
+
+        # Check that default endpoints are registered
+        assert "/v1/models" in route_paths
+        assert "/v1/completions" in route_paths
+        assert "/v1/chat/completions" in route_paths
+
+    def test_custom_endpoint_map_overrides_defaults(self):
+        """Test that custom endpoint_map can override default endpoints."""
+
+        class MyCustomIngress(OpenAiIngress):
+            async def models(self):
+                """Override the models endpoint."""
+                return {"custom": True}
+
+        # Only register models endpoint with a custom path
+        custom_endpoints = {
+            "models": lambda app: app.get("/custom/models"),
+        }
+
+        app = FastAPI()
+        make_fastapi_ingress(MyCustomIngress, endpoint_map=custom_endpoints, app=app)
+
+        route_paths = [
+            route.path for route in app.routes if isinstance(route, APIRoute)
+        ]
+
+        # Should have custom path, not default
+        assert "/custom/models" in route_paths
+        assert "/v1/models" not in route_paths
+
+    def test_deeply_nested_inheritance(self):
+        """Test that deeply nested inheritance works correctly."""
+
+        class IntermediateIngress(OpenAiIngress):
+            """Intermediate class in inheritance chain."""
+
+            async def intermediate_method(self, request: Request):
+                return {"level": "intermediate"}
+
+        class FinalIngress(IntermediateIngress):
+            """Final class in inheritance chain."""
+
+            async def final_method(self, request: Request):
+                return {"level": "final"}
+
+        custom_endpoints = {
+            "intermediate_method": lambda app: app.post("/intermediate"),
+            "final_method": lambda app: app.post("/final"),
+            **DEFAULT_ENDPOINTS,
+        }
+
+        app = FastAPI()
+        make_fastapi_ingress(FinalIngress, endpoint_map=custom_endpoints, app=app)
+
+        # Verify all routes are registered
+        route_paths = [
+            route.path for route in app.routes if isinstance(route, APIRoute)
+        ]
+        assert "/intermediate" in route_paths
+        assert "/final" in route_paths
+        assert "/v1/completions" in route_paths
+
+    def test_method_signature_preserved(self):
+        """Test that method signatures are preserved after decoration."""
+
+        class MyCustomIngress(OpenAiIngress):
+            pass
+
+        ingress_cls = make_fastapi_ingress(MyCustomIngress)
+
+        # Get the completions method and check its signature
+        completions_method = ingress_cls.completions
+        sig = inspect.signature(completions_method)
+        param_names = list(sig.parameters.keys())
+
+        # Should have 'self' and 'body' parameters
+        assert "self" in param_names
+        assert "body" in param_names
+
+
+if __name__ == "__main__":
+    sys.exit(pytest.main(["-v", __file__]))

python/ray/llm/tests/serve/cpu/deployments/routers/test_router.py

@@ -0,0 +1,563 @@
+import sys
+from contextlib import asynccontextmanager
+from types import SimpleNamespace
+from typing import Optional
+from unittest.mock import AsyncMock, MagicMock, patch
+
+import openai
+import pytest
+from fastapi import HTTPException
+from starlette.datastructures import Headers
+
+from ray import serve
+from ray.llm._internal.serve.core.configs.llm_config import (
+    LLMConfig,
+    ModelLoadingConfig,
+)
+from ray.llm._internal.serve.core.configs.openai_api_models import to_model_metadata
+from ray.llm._internal.serve.core.ingress import router as router_module
+from ray.llm._internal.serve.core.ingress.ingress import (
+    OpenAiIngress,
+    make_fastapi_ingress,
+)
+from ray.llm._internal.serve.core.ingress.router import (
+    LLMRouter,
+    _parse_routing_payload,
+)
+from ray.llm._internal.serve.core.server.llm_server import LLMServer
+from ray.llm.tests.serve.mocks.mock_vllm_engine import MockVLLMEngine
+from ray.serve._private.common import DeploymentID
+from ray.serve.exceptions import DeploymentUnavailableError
+
+
+class _DirectRouterReplicaId:
+    def __init__(self, unique_id: str, full_id: Optional[str] = None):
+        self.unique_id = unique_id
+        self._full_id = full_id or unique_id
+
+    def to_full_id_str(self) -> str:
+        return self._full_id
+
+
+class _FakeRequest:
+    def __init__(self, body: bytes, headers: Optional[dict] = None):
+        self._body = body
+        self.headers = Headers(headers or {})
+
+    async def body(self) -> bytes:
+        return self._body
+
+
+class _DirectRouterReplica:
+    """RunningReplica stand-in for ``LLMRouter._pick_replica`` tests."""
+
+    def __init__(
+        self,
+        unique_id: str,
+        full_id: Optional[str] = None,
+        endpoint: Optional[tuple] = ("127.0.0.1", 8000),
+    ):
+        self.replica_id = _DirectRouterReplicaId(unique_id, full_id)
+        self.backend_http_endpoint = endpoint
+
+
+def _new_direct_router(handle=None):
+    router = LLMRouter.__new__(LLMRouter)
+    router._handle = handle or MagicMock()
+    # Routing tests don't exercise tokenization; that lives in test_tokenizer.py.
+    router._tokenizer = None
+    return router
+
+
+def _selection_for(replica):
+    """Build a ``ReplicaSelection``-shaped mock that ``_pick_replica`` reads."""
+    return MagicMock(replica_id=replica.replica_id.unique_id, _replica=replica)
+
+
+def _choose_replica_returning(*replicas):
+    """Patch ``handle.choose_replica`` to yield the given replicas in order.
+
+    Each call to ``choose_replica`` consumes one replica from the sequence and
+    yields its ``_DirectRouterReplica`` wrapped as a selection.
+    """
+    selections = iter(_selection_for(r) for r in replicas)
+
+    @asynccontextmanager
+    async def fake_choose_replica(*args, **kwargs):
+        yield next(selections)
+
+    return fake_choose_replica
+
+
+@pytest.fixture(name="llm_config")
+def create_llm_config(stream_batching_interval_ms: Optional[int] = None):
+
+    if stream_batching_interval_ms is not None:
+        return LLMConfig(
+            model_loading_config=ModelLoadingConfig(
+                model_id="llm_model_id",
+            ),
+            experimental_configs={
+                "stream_batching_interval_ms": stream_batching_interval_ms,
+            },
+        )
+    else:
+        return LLMConfig(
+            model_loading_config=ModelLoadingConfig(
+                model_id="llm_model_id",
+            ),
+        )
+
+
+@pytest.fixture(name="client")
+def create_oai_client(llm_config: LLMConfig):
+    ServerDeployment = serve.deployment(LLMServer)
+
+    ingress_options = OpenAiIngress.get_deployment_options(llm_configs=[llm_config])
+    ingress_cls = make_fastapi_ingress(OpenAiIngress)
+    RouterDeployment = serve.deployment(ingress_cls, **ingress_options)
+    server = ServerDeployment.bind(llm_config, engine_cls=MockVLLMEngine)
+    router = RouterDeployment.bind(
+        llm_deployments={llm_config.model_id: server},
+        model_cards={
+            llm_config.model_id: to_model_metadata(llm_config.model_id, llm_config)
+        },
+    )
+    serve.run(router)
+
+    client = openai.Client(base_url="http://localhost:8000/v1", api_key="foo")
+    yield client
+
+    serve.shutdown()
+
+
+class TestDirectStreamingLLMRouter:
+    @pytest.mark.asyncio
+    async def test_route_parses_body_into_routing_payload(self):
+        """A parseable body becomes a routing payload passed positionally."""
+        router = _new_direct_router()
+        router._pick_replica = AsyncMock(
+            return_value=("127.0.0.1", 9001, "DeploymentName#replica")
+        )
+
+        body = b'{"model":"x","messages":[{"role":"user","content":"hi"}]}'
+        request = _FakeRequest(body)
+
+        result = await router.route(request)
+
+        assert result == {
+            "host": "127.0.0.1",
+            "port": 9001,
+            "replica_id": "DeploymentName#replica",
+        }
+        _, kwargs = router._pick_replica.call_args
+        assert kwargs["handle"] is router._handle
+        payload = kwargs["routing_payload"]
+        assert isinstance(payload, SimpleNamespace)
+        assert payload.messages == [{"role": "user", "content": "hi"}]
+        # The whole body is exposed, so a router can read any field.
+        assert payload.model == "x"
+        assert not hasattr(payload, "prompt")
+        # A parseable body must not trip the "no routing key" warning.
+        assert router._warned_no_routing_key is False
+
+    @pytest.mark.asyncio
+    async def test_route_truncated_body_yields_no_payload_and_warns_once(self):
+        """A truncated body derives no key. ``route`` forwards ``None`` and
+        warns once per replica."""
+        router = _new_direct_router()
+        router._pick_replica = AsyncMock(
+            return_value=("127.0.0.1", 9001, "DeploymentName#replica")
+        )
+
+        # Truncated prefix is not valid JSON so json.loads fails.
+        body = b'{"model":"x","prompt":"' + (b"x" * 1024)
+        request = _FakeRequest(body, headers={"x-body-truncated": "1058/90000"})
+
+        with patch.object(router_module.logger, "warning") as mock_warning:
+            await router.route(request)
+            await router.route(request)
+
+        # routing_payload is None on both calls. Warning fires once.
+        for call in router._pick_replica.call_args_list:
+            assert call.kwargs["routing_payload"] is None
+        assert mock_warning.call_count == 1
+        assert router._warned_no_routing_key is True
+
+    @pytest.mark.asyncio
+    async def test_route_returns_503_on_pick_failure(self):
+        router = _new_direct_router()
+        router._pick_replica = AsyncMock(side_effect=RuntimeError("no replicas"))
+
+        with pytest.raises(HTTPException) as exc_info:
+            await router.route(_FakeRequest(b"{}"))
+        assert exc_info.value.status_code == 503
+        assert "no replicas" in exc_info.value.detail
+
+    @pytest.mark.asyncio
+    async def test_route_returns_400_on_bad_routing_request(self):
+        router = _new_direct_router()
+        router._pick_replica = AsyncMock(side_effect=ValueError("empty prompt"))
+
+        with pytest.raises(HTTPException) as exc_info:
+            await router.route(_FakeRequest(b"{}"))
+        assert exc_info.value.status_code == 400
+        assert "empty prompt" in exc_info.value.detail
+
+    @pytest.mark.asyncio
+    async def test_route_returns_503_on_deployment_unavailable(self):
+        err = DeploymentUnavailableError(DeploymentID(name="LLMServer:test"))
+        router = _new_direct_router()
+        router._pick_replica = AsyncMock(side_effect=err)
+
+        with pytest.raises(HTTPException) as exc_info:
+            await router.route(_FakeRequest(b"{}"))
+        assert exc_info.value.status_code == 503
+        assert "LLMServer:test" in exc_info.value.detail
+
+    @pytest.mark.asyncio
+    async def test_pick_replica_returns_backend_endpoint_from_handle(self):
+        """``_pick_replica`` reads the endpoint off the selection's replica."""
+        replica = _DirectRouterReplica(
+            "r1",
+            full_id="DeploymentName#r1",
+            endpoint=("10.0.0.1", 8123),
+        )
+        handle = MagicMock()
+        handle.choose_replica = _choose_replica_returning(replica)
+        router = _new_direct_router(handle)
+
+        host, port, replica_id = await router._pick_replica(handle=handle)
+
+        assert (host, port, replica_id) == ("10.0.0.1", 8123, "DeploymentName#r1")
+
+    @pytest.mark.asyncio
+    async def test_pick_replica_forwards_payload_positionally(self):
+        """A routing payload reaches ``choose_replica`` as the first positional
+        arg, alongside the ``_reserve=False`` fast-path flag."""
+        replica = _DirectRouterReplica("r1", full_id="d#r1")
+
+        captured = {}
+
+        @asynccontextmanager
+        async def fake_choose_replica(*args, **kwargs):
+            captured["args"] = args
+            captured["kwargs"] = kwargs
+            yield _selection_for(replica)
+
+        handle = MagicMock()
+        handle.choose_replica = fake_choose_replica
+        router = _new_direct_router(handle)
+
+        payload = SimpleNamespace(messages=[{"role": "user", "content": "hi"}])
+        await router._pick_replica(handle=handle, routing_payload=payload)
+
+        assert captured["args"] == (payload,)
+        assert captured["kwargs"] == {"_reserve": False}
+
+    @pytest.mark.asyncio
+    async def test_pick_replica_omits_positional_arg_when_no_payload(self):
+        """With no routing payload, nothing is forwarded positionally. The
+        configured router then sees empty args and load-balances."""
+        replica = _DirectRouterReplica("r1", full_id="d#r1")
+
+        captured = {}
+
+        @asynccontextmanager
+        async def fake_choose_replica(*args, **kwargs):
+            captured["args"] = args
+            captured["kwargs"] = kwargs
+            yield _selection_for(replica)
+
+        handle = MagicMock()
+        handle.choose_replica = fake_choose_replica
+        router = _new_direct_router(handle)
+
+        await router._pick_replica(handle=handle, routing_payload=None)
+
+        assert captured["args"] == ()
+        assert captured["kwargs"] == {"_reserve": False}
+
+    @pytest.mark.asyncio
+    async def test_pick_replica_raises_when_endpoint_missing(self):
+        """If the picked replica has no backend HTTP endpoint, surface a 503
+        via ``RuntimeError`` (same error contract as before)."""
+        replica = _DirectRouterReplica("r1", endpoint=None)
+        handle = MagicMock()
+        handle.choose_replica = _choose_replica_returning(replica)
+        router = _new_direct_router(handle)
+
+        with pytest.raises(RuntimeError, match="no backend HTTP endpoint"):
+            await router._pick_replica(handle=handle)
+
+
+class TestRoutingPayload:
+    """Unit coverage for wrapping a body as a routing namespace."""
+
+    def test_parses_chat_messages(self):
+        body = b'{"model":"x","messages":[{"role":"user","content":"hi"}]}'
+        payload = _parse_routing_payload(body)
+        assert isinstance(payload, SimpleNamespace)
+        assert payload.messages == [{"role": "user", "content": "hi"}]
+        # A chat body exposes no `prompt`, so `_extract_text_from_request`
+        # resolves it as a chat request. Other fields are still exposed.
+        assert not hasattr(payload, "prompt")
+        assert payload.model == "x"
+
+    def test_parses_completion_prompt(self):
+        payload = _parse_routing_payload(b'{"model":"x","prompt":"hello"}')
+        assert isinstance(payload, SimpleNamespace)
+        assert payload.prompt == "hello"
+        assert not hasattr(payload, "messages")
+
+    @pytest.mark.parametrize(
+        "body",
+        [
+            b"",  # empty
+            b'{"model":"x","prompt":"' + (b"x" * 64),  # truncated, invalid JSON
+            b"not json",  # unparseable
+            b"[1, 2, 3]",  # valid JSON but not an object
+            b'{"model":"x","max_tokens":8}',  # object without messages or prompt
+            b'{"messages":[]}',  # empty messages carry no routing signal
+            b'{"prompt":""}',  # empty prompt carries no routing signal
+            b'{"model":"x","input":"hello"}',  # other request type, no routing key
+        ],
+    )
+    def test_returns_none_when_no_key_derivable(self, body):
+        assert _parse_routing_payload(body) is None
+
+    @pytest.mark.asyncio
+    async def test_payload_satisfies_prefix_router_contract(self):
+        """The normalized payload is read by the real
+        ``PrefixCacheAffinityRouter._extract_text_from_request``, the consumer
+        that regressed in #64326.
+
+        Async so a running event loop exists for the ``PendingRequest`` default
+        ``asyncio.Future``.
+        """
+        from ray.llm._internal.serve.routing_policies.prefix_aware.prefix_aware_router import (  # noqa: E501
+            PrefixCacheAffinityRouter,
+        )
+        from ray.serve._private.request_router.common import PendingRequest
+
+        # __new__ avoids the tree-actor setup in __init__. The method under test
+        # only uses self for the pure `_normalize_prompt_to_string` helper.
+        router = PrefixCacheAffinityRouter.__new__(PrefixCacheAffinityRouter)
+
+        chat = _parse_routing_payload(
+            b'{"messages":[{"role":"user","content":"hello world"}]}'
+        )
+        pr = PendingRequest(args=[chat], kwargs={}, metadata=MagicMock())
+        assert router._extract_text_from_request(pr) == "hello world"
+
+        completion = _parse_routing_payload(b'{"prompt":"hello world"}')
+        pr = PendingRequest(args=[completion], kwargs={}, metadata=MagicMock())
+        assert router._extract_text_from_request(pr) == "hello world"
+
+
+class TestOpenAiIngress:
+    @pytest.mark.parametrize("stream_batching_interval_ms", [None, 0, 10000])
+    @pytest.mark.parametrize("stream", [True, False])
+    @pytest.mark.asyncio
+    async def test_chat(self, stream_batching_interval_ms, client, stream):
+        """Tests chat streaming with different stream_batching_interval_ms values.
+
+        0ms super fast batching (no batching)
+        10000ms basically should be equivalent to non-streaming
+        None is default, which is some fixed non-zero value.
+        """
+
+        # Generate 1000 chunks
+        n_tokens = 1000
+
+        response = client.chat.completions.create(
+            model="llm_model_id",
+            messages=[dict(role="user", content="Hello")],
+            stream=stream,
+            max_tokens=n_tokens,
+        )
+
+        if stream:
+            text = ""
+            role = None
+            for chunk in response:
+                if chunk.choices[0].delta.role is not None and role is None:
+                    role = chunk.choices[0].delta.role
+                if chunk.choices[0].delta.content:
+                    text += chunk.choices[0].delta.content
+        else:
+            text = response.choices[0].message.content
+            role = response.choices[0].message.role
+
+        assert role == "assistant"
+        assert text.strip() == " ".join([f"test_{i}" for i in range(n_tokens)])
+
+    @pytest.mark.parametrize("stream_batching_interval_ms", [None, 0, 10000])
+    @pytest.mark.parametrize("stream", [True, False])
+    @pytest.mark.asyncio
+    async def test_completion(self, stream_batching_interval_ms, client, stream):
+        """Tests text completions streaming with different stream_batching_interval_ms values."""
+
+        # Generate tokens
+        n_tokens = 1000
+
+        response = client.completions.create(
+            model="llm_model_id",
+            prompt="Hello",
+            stream=stream,
+            max_tokens=n_tokens,
+        )
+
+        if stream:
+            text = ""
+            for chunk in response:
+                text += chunk.choices[0].text
+        else:
+            text = response.choices[0].text
+
+        # The mock engine produces "test_0 test_1 test_2 ..." pattern
+        expected_text = " ".join([f"test_{i}" for i in range(n_tokens)])
+        assert text.strip() == expected_text
+
+    @pytest.mark.parametrize("stream", [True, False])
+    @pytest.mark.asyncio
+    async def test_tool_call(self, client, stream):
+        response = client.chat.completions.create(
+            model="llm_model_id",
+            messages=[
+                {
+                    "role": "user",
+                    "content": "Can you tell me what the temperate will be in Dallas, in fahrenheit?",
+                },
+                {
+                    "content": None,
+                    "role": "assistant",
+                    "tool_calls": [
+                        {
+                            "id": "RBS92VTjJ",
+                            "function": {
+                                "arguments": '{"city": "Dallas", "state": "TX", "unit": "fahrenheit"}',
+                                "name": "get_current_weather",
+                            },
+                            "type": "function",
+                        }
+                    ],
+                },
+                {
+                    "role": "tool",
+                    "content": "The weather in Dallas, TX is 85 degrees fahrenheit. It is partly cloudly, with highs in the 90's.",
+                    "tool_call_id": "n3OMUpydP",
+                },
+            ],
+            stream=stream,
+            max_tokens=200,
+        )
+
+        if stream:
+            text = ""
+            role = None
+            for chunk in response:
+                if chunk.choices[0].delta.role is not None and role is None:
+                    role = chunk.choices[0].delta.role
+                if chunk.choices[0].delta.content:
+                    text += chunk.choices[0].delta.content
+        else:
+            text = response.choices[0].message.content
+            role = response.choices[0].message.role
+
+        assert text
+
+    @pytest.mark.asyncio
+    async def test_check_health(self, llm_config: LLMConfig):
+        """Test health check functionality."""
+
+        server = MagicMock()
+        server.check_health = MagicMock()
+        server.check_health.remote = AsyncMock()
+
+        router = OpenAiIngress(
+            llm_deployments={llm_config.model_id: server},
+            model_cards={
+                llm_config.model_id: to_model_metadata(llm_config.model_id, llm_config)
+            },
+        )
+
+        await router.check_health()
+
+    @pytest.mark.asyncio
+    async def test_raw_request_info_passed_to_deployment_handle(
+        self, llm_config: LLMConfig
+    ):
+        """Test that raw_request_info is passed to the deployment handle."""
+        from ray.llm._internal.serve.core.configs.openai_api_models import (
+            ChatCompletionRequest,
+            ChatCompletionResponse,
+        )
+        from ray.llm._internal.serve.core.protocol import RawRequestInfo
+
+        # Track if raw_request_info was received
+        captured_raw_request_infos = []
+
+        # Create a mock deployment handle that captures raw_request_info
+        async def mock_chat_generator(request, raw_request_info):
+            captured_raw_request_infos.append(raw_request_info)
+            # Return a valid response
+            yield ChatCompletionResponse(
+                id="test_id",
+                choices=[
+                    {
+                        "index": 0,
+                        "message": {"role": "assistant", "content": "Hello!"},
+                        "finish_reason": "stop",
+                    }
+                ],
+                model="llm_model_id",
+                object="chat.completion",
+                usage={
+                    "prompt_tokens": 1,
+                    "completion_tokens": 1,
+                    "total_tokens": 2,
+                },
+            )
+
+        mock_handle = MagicMock()
+        mock_handle.chat = MagicMock()
+        mock_handle.chat.remote = mock_chat_generator
+        # Make options() return the same mock so chat.remote is preserved
+        mock_handle.options.return_value = mock_handle
+
+        # Create router with mock handle
+        router = OpenAiIngress(
+            llm_deployments={llm_config.model_id: mock_handle},
+            model_cards={
+                llm_config.model_id: to_model_metadata(llm_config.model_id, llm_config)
+            },
+        )
+
+        # Create a mock FastAPI request
+        from starlette.datastructures import Headers
+
+        mock_request = MagicMock()
+        mock_headers = {
+            "content-type": "application/json",
+            "x-ray-serve-llm-test-header": "router-raw-request-info",
+        }
+        mock_request.headers = Headers(mock_headers)
+
+        # Make a request through the router
+        request_body = ChatCompletionRequest(
+            model="llm_model_id",
+            messages=[{"role": "user", "content": "Hello"}],
+            stream=False,
+        )
+
+        await router.chat(request_body, mock_request)
+
+        # Verify that raw_request_info was passed to the deployment handle
+        assert len(captured_raw_request_infos) == 1
+        assert isinstance(captured_raw_request_infos[0], RawRequestInfo)
+        assert captured_raw_request_infos[0].headers == mock_headers
+
+
+if __name__ == "__main__":
+    sys.exit(pytest.main(["-v", __file__]))

python/ray/llm/tests/serve/cpu/deployments/routers/test_router_direct_streaming.py

@@ -0,0 +1,65 @@
+import sys
+
+import pytest
+
+from ray.llm._internal.serve.core.configs.llm_config import (
+    LLMConfig,
+    ModelLoadingConfig,
+)
+from ray.llm._internal.serve.core.ingress.builder import (
+    LLMServingArgs,
+    build_openai_app,
+)
+from ray.llm.tests.serve.cpu.deployments.utils.direct_streaming_utils import (
+    consistent_hash_deployment_config,
+    requires_direct_streaming,
+    run_app_through_haproxy,
+    session_chat_response,
+)
+
+
+@requires_direct_streaming
+class TestDirectStreamingConsistentHashRouting:
+    """Session affinity over the full direct-streaming path.
+
+    A request flows through HAProxy and the LLMRouter ``/internal/route``
+    decision (ConsistentHashRouter) to a backend replica. The session id
+    reaches the chosen replica, and one session pins to one replica.
+    """
+
+    @pytest.fixture(name="llm_config")
+    def _llm_config(self):
+        return LLMConfig(model_loading_config=ModelLoadingConfig(model_id="test-model"))
+
+    @pytest.fixture(name="base_url")
+    def run_direct_streaming_app(
+        self,
+        llm_config_with_mock_engine,
+        shutdown_ray_and_serve,
+        disable_placement_bundles,
+    ):
+        llm_config = llm_config_with_mock_engine
+        llm_config.deployment_config = consistent_hash_deployment_config()
+        yield run_app_through_haproxy(
+            build_openai_app(LLMServingArgs(llm_configs=[llm_config]))
+        )
+
+    def test_session_affinity(self, base_url):
+        replicas = {
+            session_chat_response(base_url, "test-session-id").headers["x-replica-id"]
+            for _ in range(10)
+        }
+        assert len(replicas) == 1
+
+    def test_different_sessions_spread(self, base_url):
+        replicas = {
+            session_chat_response(base_url, f"test-session-id-{i}").headers[
+                "x-replica-id"
+            ]
+            for i in range(10)
+        }
+        assert len(replicas) > 1
+
+
+if __name__ == "__main__":
+    sys.exit(pytest.main(["-v", __file__]))

python/ray/llm/tests/serve/cpu/deployments/routers/test_tokenizer.py

@@ -0,0 +1,282 @@
+import sys
+from typing import Optional
+from unittest.mock import AsyncMock, MagicMock
+
+import pytest
+from fastapi import HTTPException
+from starlette.datastructures import Headers
+
+from ray.llm._internal.serve.core.configs.llm_config import LLMConfig
+from ray.llm._internal.serve.core.configs.openai_api_models import (
+    ErrorInfo,
+    ErrorResponse,
+    TokenizeChatRequest,
+    TokenizeCompletionRequest,
+)
+from ray.llm._internal.serve.core.ingress.builder import (
+    LLMServingArgs,
+    build_openai_app,
+)
+from ray.llm._internal.serve.core.ingress.router import LLMRouter
+from ray.llm._internal.serve.core.ingress.tokenizer import TokenizeError, Tokenizer
+from ray.serve.experimental.round_robin_router import RoundRobinRouter
+from ray.serve.llm.request_router import KVAwareRouter
+
+
+class _TokenizeResponse:
+    def __init__(self, tokens):
+        self.tokens = tokens
+
+
+async def _tokenize_stream(response):
+    yield response
+
+
+def _handle_returning(response):
+    """A DeploymentHandle whose /tokenize streams ``response``; captures the
+    Tokenize* request it was called with under ``captured``."""
+    captured = {}
+
+    def tokenize_remote(tok_req, _):
+        captured["request"] = tok_req
+        return _tokenize_stream(response)
+
+    handle = MagicMock()
+    handle.options.return_value.tokenize.remote = tokenize_remote
+    return handle, captured
+
+
+class TestTokenizer:
+    @pytest.mark.parametrize(
+        "payload",
+        [
+            {"model": "m", "prompt": ["a", "b"]},  # batch of prompts
+            {"model": "m", "prompt": [1, 2, 3]},  # pre-tokenized token ids
+            {"model": "m"},  # neither messages nor prompt
+        ],
+    )
+    @pytest.mark.asyncio
+    async def test_untokenizable_payload_returns_none(self, payload):
+        """A parsed payload with no single-string prompt yields None."""
+        assert await Tokenizer(MagicMock()).tokenize(payload) is None
+
+    @pytest.mark.parametrize(
+        "payload, expected_request_type",
+        [
+            (
+                {"model": "m", "messages": [{"role": "user", "content": "hi"}]},
+                TokenizeChatRequest,
+            ),
+            ({"model": "m", "prompt": "hello"}, TokenizeCompletionRequest),
+        ],
+    )
+    @pytest.mark.asyncio
+    async def test_tokenizes_chat_and_completion(self, payload, expected_request_type):
+        """A chat or completion payload is sent to /tokenize as the right
+        Tokenize* request and its returned token ids are surfaced."""
+        handle, captured = _handle_returning(_TokenizeResponse([5, 6, 7]))
+        tokens = await Tokenizer(handle).tokenize(payload)
+        assert tokens == [5, 6, 7]
+        assert isinstance(captured["request"], expected_request_type)
+
+    @pytest.mark.parametrize(
+        "payload, expected",
+        [
+            (  # chat: template-rendering fields + request-provided prompt flags
+                {
+                    "model": "m",
+                    "messages": [{"role": "user", "content": "hi"}],
+                    "tools": [
+                        {
+                            "type": "function",
+                            "function": {"name": "f", "parameters": {}},
+                        }
+                    ],
+                    "chat_template": "TEMPLATE",
+                    "chat_template_kwargs": {"enable_thinking": False},
+                    "mm_processor_kwargs": {"num_crops": 4},
+                    "add_generation_prompt": False,
+                    "continue_final_message": True,
+                    "temperature": 0.7,
+                },
+                {
+                    "chat_template": "TEMPLATE",
+                    "chat_template_kwargs": {"enable_thinking": False},
+                    "mm_processor_kwargs": {"num_crops": 4},
+                    "add_generation_prompt": False,
+                    "continue_final_message": True,
+                },
+            ),
+            (  # completion: add_special_tokens comes from the request
+                {
+                    "model": "m",
+                    "prompt": "hi",
+                    "add_special_tokens": False,
+                    "temperature": 0.7,
+                },
+                {"add_special_tokens": False},
+            ),
+        ],
+    )
+    @pytest.mark.asyncio
+    async def test_forwards_prompt_fields_only(self, payload, expected):
+        """Prompt-rendering fields come from the request (not hardcoded) and
+        sampling params are dropped, so routing ids match prefill."""
+        handle, captured = _handle_returning(_TokenizeResponse([1, 2]))
+        await Tokenizer(handle).tokenize(payload)
+        request = captured["request"]
+        for attr, value in expected.items():
+            assert getattr(request, attr) == value
+        assert "temperature" not in (request.model_extra or {})
+
+    @pytest.mark.asyncio
+    async def test_error_response_raises(self):
+        """A /tokenize ErrorResponse surfaces as a TokenizeError carrying vLLM's
+        status code, message, and type."""
+        err = ErrorResponse(
+            error=ErrorInfo(message="bad model", type="NotFoundError", code=404)
+        )
+        handle, _ = _handle_returning(err)
+        with pytest.raises(TokenizeError) as exc_info:
+            await Tokenizer(handle).tokenize({"model": "m", "prompt": "hi"})
+        assert exc_info.value.status_code == 404
+        assert exc_info.value.message == "bad model"
+        assert exc_info.value.type == "NotFoundError"
+
+    @pytest.mark.asyncio
+    async def test_empty_response_raises(self):
+        """An empty /tokenize stream raises rather than returning no tokens."""
+
+        async def _empty(*_args):
+            for _ in ():
+                yield
+
+        handle = MagicMock()
+        handle.options.return_value.tokenize.remote = _empty
+        with pytest.raises(TokenizeError) as exc_info:
+            await Tokenizer(handle).tokenize({"model": "m", "prompt": "hi"})
+        assert exc_info.value.status_code == 500
+
+
+class TestRoute:
+    @pytest.mark.asyncio
+    async def test_no_tokenizer_forwards_none(self):
+        # A non-KV router has no tokenizer, so route forwards request_token_ids=None.
+        router = LLMRouter.__new__(LLMRouter)
+        router._handle = MagicMock()
+        router._tokenizer = None
+        router._pick_replica = AsyncMock(return_value=("h", 1, "rid"))
+
+        request = MagicMock()
+        request.body = AsyncMock(return_value=b'{"model": "m", "prompt": "hi"}')
+        request.headers = Headers({})
+        await router.route(request)
+        assert router._pick_replica.call_args.kwargs["request_token_ids"] is None
+
+    @pytest.mark.asyncio
+    async def test_forwards_token_ids(self):
+        # A successful tokenization forwards its token ids to _pick_replica.
+        router = LLMRouter.__new__(LLMRouter)
+        router._handle = MagicMock()
+        router._tokenizer = MagicMock()
+        router._tokenizer.tokenize = AsyncMock(return_value=[5, 6, 7])
+        router._pick_replica = AsyncMock(return_value=("h", 1, "rid"))
+
+        request = MagicMock()
+        request.body = AsyncMock(return_value=b'{"model": "m", "prompt": "hi"}')
+        request.headers = Headers({})
+        await router.route(request)
+        assert router._pick_replica.call_args.kwargs["request_token_ids"] == [5, 6, 7]
+
+    @pytest.mark.asyncio
+    async def test_unparseable_body_skips_tokenization(self):
+        # A truncated/unparseable body derives no routing payload, so the
+        # tokenizer is never called and request_token_ids stays None.
+        router = LLMRouter.__new__(LLMRouter)
+        router._handle = MagicMock()
+        router._tokenizer = MagicMock()
+        router._tokenizer.tokenize = AsyncMock(return_value=[5, 6, 7])
+        router._pick_replica = AsyncMock(return_value=("h", 1, "rid"))
+
+        request = MagicMock()
+        # Truncated prefix: not valid JSON, so it can't be parsed or tokenized.
+        request.body = AsyncMock(return_value=b'{"model": "m", "prompt": "' + b"x" * 8)
+        request.headers = Headers({"x-body-truncated": "8/90000"})
+        await router.route(request)
+
+        router._tokenizer.tokenize.assert_not_called()
+        assert router._pick_replica.call_args.kwargs["request_token_ids"] is None
+
+    @pytest.mark.asyncio
+    async def test_tokenize_error_becomes_http_error(self):
+        # A /tokenize rejection becomes an HTTPException with the same status
+        # code, and routing is not attempted.
+        router = LLMRouter.__new__(LLMRouter)
+        router._handle = MagicMock()
+        router._tokenizer = MagicMock()
+        router._tokenizer.tokenize = AsyncMock(
+            side_effect=TokenizeError(
+                "bad model", status_code=404, type="NotFoundError"
+            )
+        )
+        router._pick_replica = AsyncMock()
+
+        request = MagicMock()
+        request.body = AsyncMock(return_value=b'{"model": "m", "prompt": "hi"}')
+        request.headers = Headers({})
+        with pytest.raises(HTTPException) as exc_info:
+            await router.route(request)
+        assert exc_info.value.status_code == 404
+        assert exc_info.value.detail == "bad model"
+        router._pick_replica.assert_not_called()
+
+
+def _build_llm_app(request_router_class):
+    """Build a direct-streaming OpenAI app, optionally pinning a router class."""
+    deployment_config = {"autoscaling_config": {"min_replicas": 1, "max_replicas": 1}}
+    if request_router_class is not None:
+        deployment_config["request_router_config"] = {
+            "request_router_class": request_router_class
+        }
+    llm_config = LLMConfig(
+        model_loading_config={
+            "model_id": "qwen3-0.6b",
+            "model_source": "Qwen/Qwen3-0.6B",
+        },
+        accelerator_type=None,
+        deployment_config=deployment_config,
+    )
+    return build_openai_app(LLMServingArgs(llm_configs=[llm_config]))
+
+
+def _pre_routing_tokenization(app) -> Optional[bool]:
+    init_kwargs = app._ingress_request_router._bound_deployment.init_kwargs
+    return init_kwargs["pre_routing_tokenization"]
+
+
+class TestPreRoutingTokenization:
+    """build_openai_app enables pre-routing tokenization iff the router is KV-aware."""
+
+    @pytest.fixture(autouse=True)
+    def enable_direct_streaming(self, monkeypatch):
+        monkeypatch.setattr(
+            "ray.llm._internal.serve.core.ingress.builder."
+            "RAY_SERVE_LLM_ENABLE_DIRECT_STREAMING",
+            True,
+        )
+
+    @pytest.mark.parametrize(
+        "request_router_class, expected",
+        [
+            (KVAwareRouter, True),
+            (None, False),
+            (RoundRobinRouter, False),
+        ],
+    )
+    def test_enabled_only_for_kv_aware_router(self, request_router_class, expected):
+        app = _build_llm_app(request_router_class)
+        assert _pre_routing_tokenization(app) is expected
+
+
+if __name__ == "__main__":
+    sys.exit(pytest.main(["-v", __file__]))