chore: import upstream snapshot with attribution

python/ray/serve/.claude/CLAUDE.md

@@ -0,0 +1,10 @@
+<!-- Loaded on-demand when Claude works on Ray Serve files. -->
+<!-- Keep under 50 lines. Multi-step procedures → skills. Code style → rules/. -->
+
+# Ray Serve
+
+## Key Modules
+<!-- Entry points, important abstractions, non-obvious dependencies -->
+
+## Gotchas
+<!-- Non-obvious behaviors, common mistakes, things that break silently -->

python/ray/serve/.claude/rules/README.md

@@ -0,0 +1,9 @@
+<!-- Add Ray Serve team-specific rules here as .md files. -->
+<!-- Rules with paths: frontmatter only load when matching files are edited. -->
+<!-- Example:
+---
+paths:
+  - "python/ray/serve/**/*.py"
+---
+- Follow the deployment graph pattern for new features
+-->

python/ray/serve/BUILD.bazel

@@ -0,0 +1,35 @@
+load("@rules_python//python:defs.bzl", "py_library")
+load("//bazel:python.bzl", "doctest")
+
+doctest(
+    size = "small",
+    data = glob(["**/*.lua.tmpl"]),
+    files = glob(
+        ["**/*.py"],
+        exclude = [
+            "tests/**",
+            # FIXME: Add the llm tests back with a diff tag.
+            "llm/**",
+            # FIXME: Failing on Windows
+            "gradio_integrations.py",
+            "_private/benchmarks/**",
+        ],
+    ),
+    tags = ["team:serve"],
+)
+
+# This is a dummy test dependency that causes the above tests to be
+# re-run if any of these files changes.
+py_library(
+    name = "serve_lib",
+    srcs = glob(
+        ["**/*.py"],
+        exclude = ["tests/**/*.py"],
+    ),
+    data = glob(["**/*.lua.tmpl"]),
+    visibility = [
+        "//python/ray/serve:__pkg__",
+        "//python/ray/serve:__subpackages__",
+        "//release:__pkg__",
+    ],
+)

python/ray/serve/__init__.py

@@ -0,0 +1,72 @@
+import ray._private.worker
+
+try:
+    from ray.serve._private.logging_utils import configure_default_serve_logger
+    from ray.serve.api import (
+        Application,
+        Deployment,
+        RunTarget,
+        _run,
+        _run_many,
+        delete,
+        deployment,
+        get_app_handle,
+        get_deployment_actor,
+        get_deployment_actor_context,
+        get_deployment_handle,
+        get_multiplexed_model_id,
+        get_replica_context,
+        ingress,
+        multiplexed,
+        run,
+        run_many,
+        shutdown,
+        shutdown_async,
+        start,
+        status,
+    )
+    from ray.serve.batching import batch
+    from ray.serve.config import ControllerOptions, HTTPOptions
+    from ray.serve.utils import get_trace_context
+
+except ModuleNotFoundError as e:
+    e.msg += (
+        '. You can run `pip install "ray[serve]"` to install all Ray Serve'
+        " dependencies."
+    )
+    raise e
+
+# Setup default ray.serve logger to ensure all serve module logs are captured.
+configure_default_serve_logger()
+
+# Mute the warning because Serve sometimes intentionally calls
+# ray.get inside async actors.
+ray._private.worker.blocking_get_inside_async_warned = True
+
+__all__ = [
+    "_run",
+    "_run_many",
+    "batch",
+    "start",
+    "ControllerOptions",
+    "HTTPOptions",
+    "get_replica_context",
+    "get_deployment_actor",
+    "get_deployment_actor_context",
+    "get_trace_context",
+    "shutdown",
+    "shutdown_async",
+    "ingress",
+    "deployment",
+    "run",
+    "run_many",
+    "RunTarget",
+    "delete",
+    "Application",
+    "Deployment",
+    "multiplexed",
+    "get_multiplexed_model_id",
+    "status",
+    "get_app_handle",
+    "get_deployment_handle",
+]

python/ray/serve/_private/api.py

@@ -0,0 +1,380 @@
+import asyncio
+import inspect
+import logging
+from types import FunctionType
+from typing import Any, Dict, List, Tuple, Union
+
+from pydantic import BaseModel
+
+import ray
+from ray import ObjectRef
+from ray._common.usage import usage_lib
+from ray.actor import ActorHandle
+from ray.serve._private.client import ServeControllerClient
+from ray.serve._private.constants import (
+    HTTP_PROXY_TIMEOUT,
+    SERVE_LOGGER_NAME,
+    SERVE_NAMESPACE,
+)
+from ray.serve._private.default_impl import get_controller_impl
+from ray.serve.config import (
+    ControllerOptions,
+    HTTPOptions,
+    ProxyLocation,
+    gRPCOptions,
+)
+from ray.serve.context import (
+    _check_cached_client_alive,
+    _get_global_client,
+    _set_global_client,
+)
+from ray.serve.deployment import Application
+from ray.serve.exceptions import RayServeException
+from ray.serve.schema import LoggingConfig
+
+logger = logging.getLogger(SERVE_LOGGER_NAME)
+
+
+def _coerce_controller_options(
+    controller_options: Union[None, dict, ControllerOptions],
+) -> ControllerOptions:
+    """Normalize an optional dict / model into a validated ControllerOptions."""
+    if controller_options is None:
+        return ControllerOptions()
+    if isinstance(controller_options, ControllerOptions):
+        return controller_options
+    if isinstance(controller_options, dict):
+        return ControllerOptions.model_validate(controller_options)
+    raise TypeError(
+        "controller_options must be a dict, ControllerOptions, or None; got "
+        f"{type(controller_options).__name__}."
+    )
+
+
+def _check_http_options(
+    client: ServeControllerClient, http_options: Union[dict, HTTPOptions]
+) -> None:
+    if http_options:
+        client_http_options = client.http_config
+        new_http_options = (
+            http_options
+            if isinstance(http_options, HTTPOptions)
+            else HTTPOptions.model_validate(http_options)
+        )
+        different_fields = []
+        all_http_option_fields = new_http_options.__dict__
+        for field in all_http_option_fields:
+            if getattr(new_http_options, field) != getattr(client_http_options, field):
+                different_fields.append(field)
+
+        if len(different_fields):
+            logger.warning(
+                "The new client HTTP config differs from the existing one "
+                f"in the following fields: {different_fields}. "
+                "The new HTTP config is ignored."
+            )
+
+
+def _create_controller_and_proxy_refs(
+    http_options: Union[None, dict, HTTPOptions],
+    grpc_options: Union[None, dict, gRPCOptions],
+    global_logging_config: Union[None, dict, LoggingConfig],
+    controller_options: ControllerOptions,
+    proxy_location: Union[None, str, ProxyLocation] = None,
+    **kwargs,
+) -> Tuple[ActorHandle, List[ObjectRef]]:
+    """Create the detached ServeController actor in the caller process.
+
+    Runs everything the old ``_start_controller`` remote task did, but inline:
+    ray.init if needed, arg validation, controller actor creation, and resolving
+    ``get_proxies``. Returns the controller handle (owned locally by the caller,
+    following normal ref-counting — no cross-process transfer) plus the list of
+    proxy-readiness ObjectRefs the caller must still wait on.
+
+    Caller resolves ``proxy_ready_refs`` synchronously (``ray.get``) or
+    asynchronously (``await asyncio.gather``). Splitting the wait out of this
+    helper is what lets sync and async share one creation path without
+    duplication (per @edoakes's suggestion on #63597).
+
+    Parameters are same as ray.serve._private.api.serve_start(), except
+    ``controller_options`` must already be a validated ``ControllerOptions``
+    (callers coerce eagerly so a bad value raises locally, not from a task).
+    """
+
+    # Initialize ray if needed.
+    ray._private.worker.global_worker._filter_logs_by_job = False
+    if not ray.is_initialized():
+        ray.init(namespace=SERVE_NAMESPACE)
+
+    # Legacy http proxy actor check
+    http_deprecated_args = ["http_host", "http_port", "http_middlewares"]
+    for key in http_deprecated_args:
+        if key in kwargs:
+            raise ValueError(
+                f"{key} is deprecated, please use serve.start(http_options="
+                f'{{"{key}": {kwargs[key]}}}) instead.'
+            )
+
+    if isinstance(http_options, dict):
+        http_options = HTTPOptions.model_validate(http_options)
+    if http_options is None:
+        http_options = HTTPOptions()
+
+    proxy_location = ProxyLocation._normalize(proxy_location)
+
+    if isinstance(grpc_options, dict):
+        grpc_options = gRPCOptions(**grpc_options)
+
+    if global_logging_config is None:
+        global_logging_config = LoggingConfig()
+    elif isinstance(global_logging_config, dict):
+        global_logging_config = LoggingConfig(**global_logging_config)
+
+    controller_impl = get_controller_impl(controller_options=controller_options)
+    controller = controller_impl.remote(
+        http_options=http_options,
+        grpc_options=grpc_options,
+        global_logging_config=global_logging_config,
+        proxy_location=proxy_location,
+    )
+
+    proxy_handles = ray.get(controller.get_proxies.remote())
+    proxy_ready_refs = (
+        [handle.ready.remote() for handle in proxy_handles.values()]
+        if len(proxy_handles) > 0
+        else []
+    )
+    return controller, proxy_ready_refs
+
+
+async def serve_start_async(
+    http_options: Union[None, dict, HTTPOptions] = None,
+    grpc_options: Union[None, dict, gRPCOptions] = None,
+    global_logging_config: Union[None, dict, LoggingConfig] = None,
+    controller_options: Union[None, dict, ControllerOptions] = None,
+    proxy_location: Union[None, str, ProxyLocation] = None,
+    **kwargs,
+) -> ServeControllerClient:
+    """Initialize a serve instance asynchronously.
+
+    This function is not thread-safe. The caller should maintain the async lock in order
+    to start the serve instance asynchronously.
+
+    This function has the same functionality as ray.serve._private.api.serve_start().
+
+    Parameters & Returns are same as ray.serve._private.api.serve_start().
+    """
+
+    usage_lib.record_library_usage("serve")
+
+    # Validate eagerly in the caller so a bad ``controller_options`` raises
+    # here, at the call site, rather than after a round-trip into the helper.
+    controller_options = _coerce_controller_options(controller_options)
+
+    client, _ = _check_cached_client_alive()
+    if client is None:
+        try:
+            client = _get_global_client()
+        except RayServeException:
+            client = None
+    if client is not None:
+        logger.info(
+            f'Connecting to existing Serve app in namespace "{SERVE_NAMESPACE}".'
+            " New http_options/controller_options will not be applied."
+        )
+        if http_options:
+            _check_http_options(client, http_options)
+        return client
+
+    # Run the blocking controller-creation helper in a worker thread so its
+    # ray.get(controller.get_proxies.remote()) does not stall this event loop.
+    # serve_start_async exists (vs serve_start) precisely to keep long-lived
+    # callers like the Dashboard Agent responsive while Serve starts up.
+    # Safe because the helper only touches ray.init/ray.get/actor.remote, all
+    # thread-safe via the CoreWorker C++ API, and the Dashboard Agent uses sync
+    # ray.init (so the helper's ray.init branch is skipped). If the dashboard
+    # ever switched to async-mode ray.init this would need revisiting.
+    controller, proxy_ready_refs = await asyncio.to_thread(
+        _create_controller_and_proxy_refs,
+        http_options,
+        grpc_options,
+        global_logging_config,
+        controller_options,
+        proxy_location,
+        **kwargs,
+    )
+
+    if proxy_ready_refs:
+        try:
+            await asyncio.wait_for(
+                asyncio.gather(*proxy_ready_refs),
+                timeout=HTTP_PROXY_TIMEOUT,
+            )
+        except asyncio.TimeoutError:
+            raise TimeoutError(
+                f"HTTP proxies not available after {HTTP_PROXY_TIMEOUT}s."
+            )
+
+    client = ServeControllerClient(controller)
+    _set_global_client(client)
+    logger.info(f'Started Serve in namespace "{SERVE_NAMESPACE}".')
+    return client
+
+
+def serve_start(
+    http_options: Union[None, dict, HTTPOptions] = None,
+    grpc_options: Union[None, dict, gRPCOptions] = None,
+    global_logging_config: Union[None, dict, LoggingConfig] = None,
+    controller_options: Union[None, dict, ControllerOptions] = None,
+    proxy_location: Union[None, str, ProxyLocation] = None,
+    **kwargs,
+) -> ServeControllerClient:
+    """Initialize a serve instance.
+
+    By default, the instance will be scoped to the lifetime of the returned
+    Client object (or when the script exits). This is
+    only relevant if connecting to a long-running Ray cluster (e.g., with
+    ray.init(address="auto") or ray.init("ray://<remote_addr>")).
+
+    Args:
+        http_options: Configuration options for HTTP proxy. You can pass in a
+          dictionary or HTTPOptions object with fields:
+
+            - host(str, None): Host for HTTP servers to listen on. Defaults to
+              localhost. To expose Serve publicly, you probably want to set
+              this to "0.0.0.0" for IPv4 or "::" for IPv6.
+            - port(int): Port for HTTP server. Defaults to 8000.
+            - root_path(str): Root path to mount the serve application
+              (for example, "/serve"). All deployment routes will be prefixed
+              with this path. Defaults to "".
+            - middlewares(list): A list of Starlette middlewares that will be
+              applied to the HTTP servers in the cluster. Defaults to [].
+            - location(str, serve.config.ProxyLocation): The deployment
+              location of HTTP servers:
+
+                - "HeadOnly": start one HTTP server on the head node. Serve
+                  assumes the head node is the node you executed serve.start
+                  on. This is the default.
+                - "EveryNode": start one HTTP server per node.
+                - "Disabled" (or legacy "NoServer") or None: disable HTTP server.
+            - num_cpus (int): The number of CPU cores to reserve for each
+              internal Serve HTTP proxy actor.  Defaults to 0.
+        grpc_options: Configuration options for gRPC proxy.
+          You can pass in a gRPCOptions object with fields:
+
+            - port(int): Port for gRPC server. Defaults to 9000.
+            - grpc_servicer_functions(list): List of import paths for gRPC
+                `add_servicer_to_server` functions to add to Serve's gRPC proxy.
+                Default empty list, meaning not to start the gRPC server.
+        global_logging_config: Optional ``LoggingConfig`` (or dict) applied as
+            the default logging configuration for the Serve controller and all
+            proxies/replicas in this Serve instance.
+        controller_options: Optional ``ControllerOptions`` (or dict) for the
+            Serve controller actor. Currently only ``runtime_env.env_vars``
+            is honored; see ``ray.serve.config.ControllerOptions``. Only
+            applied on first controller creation -- ignored if the controller
+            is already running in this Ray cluster (a log line is emitted).
+        proxy_location: Where to run proxies that handle ingress traffic to
+            the cluster. See ``ProxyLocation`` for supported options. Defaults
+            to ``EveryNode`` when unspecified. An explicit (deprecated)
+            ``HTTPOptions.location`` overrides this.
+        **kwargs: Reserved for forwarding to internal controller-start hooks;
+            no public keys are currently supported and unknown keys may raise.
+
+    Returns:
+        A ``ServeControllerClient`` connected to the Serve controller (either
+        newly started or an already-running one in this Ray cluster).
+    """
+
+    usage_lib.record_library_usage("serve")
+
+    controller_options = _coerce_controller_options(controller_options)
+
+    client, _ = _check_cached_client_alive()
+    if client is None:
+        try:
+            client = _get_global_client()
+        except RayServeException:
+            client = None
+    if client is not None:
+        logger.info(
+            f'Connecting to existing Serve app in namespace "{SERVE_NAMESPACE}".'
+            " New http_options/controller_options will not be applied."
+        )
+        if http_options:
+            _check_http_options(client, http_options)
+        return client
+
+    controller, proxy_ready_refs = _create_controller_and_proxy_refs(
+        http_options,
+        grpc_options,
+        global_logging_config,
+        controller_options,
+        proxy_location,
+        **kwargs,
+    )
+
+    if proxy_ready_refs:
+        try:
+            ray.get(proxy_ready_refs, timeout=HTTP_PROXY_TIMEOUT)
+        except ray.exceptions.GetTimeoutError:
+            raise TimeoutError(
+                f"HTTP proxies not available after {HTTP_PROXY_TIMEOUT}s."
+            )
+
+    client = ServeControllerClient(controller)
+    _set_global_client(client)
+    logger.info(f'Started Serve in namespace "{SERVE_NAMESPACE}".')
+    return client
+
+
+def call_user_app_builder_with_args_if_necessary(
+    builder: Union[Application, FunctionType],
+    args: Dict[str, Any],
+) -> Application:
+    """Calls a user-provided function that returns Serve application.
+
+    If an Application object is passed, this is a no-op.
+
+    Else, we validate the signature of the function, convert the args dictionary to
+    the user-annotated Pydantic model if provided, and call the function.
+
+    The output of the function is returned (must be an Application).
+    """
+    if isinstance(builder, Application):
+        if len(args) > 0:
+            raise ValueError(
+                "Arguments can only be passed to an application builder function, "
+                "not an already built application."
+            )
+        return builder
+    elif not isinstance(builder, FunctionType):
+        raise TypeError(
+            "Expected a built Serve application or an application builder function "
+            f"but got: {type(builder)}."
+        )
+
+    # Check that the builder only takes a single argument.
+    # TODO(edoakes): we may want to loosen this to allow optional kwargs in the future.
+    signature = inspect.signature(builder)
+    if len(signature.parameters) != 1:
+        raise TypeError(
+            "Application builder functions should take exactly one parameter, "
+            "a dictionary containing the passed arguments."
+        )
+
+    # If the sole argument to the builder is a pydantic model, convert the args dict to
+    # that model. This will perform standard pydantic validation (e.g., raise an
+    # exception if required fields are missing).
+    param = signature.parameters[list(signature.parameters.keys())[0]]
+    if inspect.isclass(param.annotation) and issubclass(param.annotation, BaseModel):
+        args = param.annotation.model_validate(args)
+
+    app = builder(args)
+    if not isinstance(app, Application):
+        raise TypeError(
+            "Application builder functions must return an `Application` returned "
+            f"`from `Deployment.bind()`, but got: {type(app)}."
+        )
+
+    return app

python/ray/serve/_private/benchmarks/common.py

@@ -0,0 +1,688 @@
+import asyncio
+import inspect
+import logging
+import random
+import string
+import time
+from functools import partial
+from typing import Any, Callable, Coroutine, Dict, List, Optional, Tuple
+
+import aiohttp
+import aiohttp.client_exceptions
+import grpc
+import numpy as np
+import pandas as pd
+from starlette.responses import StreamingResponse
+from tqdm import tqdm
+
+import ray
+from ray import serve
+from ray._common.test_utils import SignalActor as _SignalActor
+from ray.serve._private.common import DeploymentStatus
+from ray.serve.generated import serve_pb2, serve_pb2_grpc
+from ray.serve.handle import DeploymentHandle
+
+
+async def run_latency_benchmark(
+    f: Callable, num_requests: int, *, num_warmup_requests: int = 100
+) -> pd.Series:
+    if inspect.iscoroutinefunction(f):
+        to_call = f
+    else:
+
+        async def to_call():
+            f()
+
+    latencies = []
+    for i in tqdm(range(num_requests + num_warmup_requests)):
+        start = time.perf_counter()
+        await to_call()
+        end = time.perf_counter()
+
+        # Don't include warm-up requests.
+        if i >= num_warmup_requests:
+            latencies.append(1000 * (end - start))
+
+    return pd.Series(latencies)
+
+
+async def run_throughput_benchmark(
+    fn: Callable[[], List[float]],
+    multiplier: int = 1,
+    num_trials: int = 10,
+    trial_runtime: float = 1,
+) -> Tuple[float, float, pd.Series]:
+    """Benchmarks throughput of a function.
+
+    Args:
+        fn: The function to benchmark. If this returns anything, it must
+            return a list of latencies.
+        multiplier: The number of requests or tokens (or whatever unit
+            is appropriate for this throughput benchmark) that is
+            completed in one call to `fn`.
+        num_trials: The number of trials to run.
+        trial_runtime: How long each trial should run for. During the
+            duration of one trial, `fn` will be repeatedly called.
+
+    Returns:
+        A tuple ``(mean, stddev, latencies)`` summarizing per-trial throughput
+        across ``num_trials`` runs.
+    """
+    # Warmup
+    start = time.time()
+    while time.time() - start < 0.1:
+        await fn()
+
+    # Benchmark
+    stats = []
+    latencies = []
+    for _ in tqdm(range(num_trials)):
+        start = time.perf_counter()
+        count = 0
+        while time.perf_counter() - start < trial_runtime:
+            res = await fn()
+            if res:
+                latencies.extend(res)
+
+            count += 1
+        end = time.perf_counter()
+        stats.append(multiplier * count / (end - start))
+
+    return round(np.mean(stats), 2), round(np.std(stats), 2), pd.Series(latencies)
+
+
+async def do_single_http_batch(
+    *,
+    batch_size: int = 100,
+    url: str = "http://localhost:8000",
+    stream: bool = False,
+) -> List[float]:
+    """Sends a batch of http requests and returns e2e latencies."""
+
+    # By default, aiohttp limits the number of client connections to 100.
+    # We need to use TCPConnector to configure the limit if batch size
+    # is greater than 100.
+    connector = aiohttp.TCPConnector(limit=batch_size)
+    async with aiohttp.ClientSession(
+        connector=connector, raise_for_status=True
+    ) as session:
+
+        async def do_query():
+            start = time.perf_counter()
+            try:
+                async with session.get(url) as r:
+                    if stream:
+                        async for chunk, _ in r.content.iter_chunks():
+                            pass
+                    else:
+                        # Read the response to ensure it's consumed
+                        await r.read()
+            except aiohttp.client_exceptions.ClientConnectionError:
+                pass
+
+            end = time.perf_counter()
+            return 1000 * (end - start)
+
+        return await asyncio.gather(*[do_query() for _ in range(batch_size)])
+
+
+async def do_single_grpc_batch(
+    *, batch_size: int = 100, target: str = "localhost:9000"
+):
+    channel = grpc.aio.insecure_channel(target)
+    stub = serve_pb2_grpc.RayServeBenchmarkServiceStub(channel)
+    payload = serve_pb2.StringData(data="")
+
+    async def do_query():
+        start = time.perf_counter()
+
+        await stub.grpc_call(payload)
+
+        end = time.perf_counter()
+        return 1000 * (end - start)
+
+    return await asyncio.gather(*[do_query() for _ in range(batch_size)])
+
+
+async def collect_profile_events(coro: Coroutine):
+    """Collects profiling events using Viztracer"""
+
+    from viztracer import VizTracer
+
+    tracer = VizTracer()
+    tracer.start()
+
+    await coro
+
+    tracer.stop()
+    tracer.save()
+
+
+def generate_payload(size: int = 100, chars=string.ascii_uppercase + string.digits):
+    return "".join(random.choice(chars) for _ in range(size))
+
+
+class Blackhole:
+    def sink(self, o):
+        pass
+
+
+@serve.deployment
+class Noop:
+    def __init__(self):
+        logging.getLogger("ray.serve").setLevel(logging.WARNING)
+
+    def __call__(self, *args, **kwargs):
+        return b""
+
+
+@serve.deployment
+class ModelComp:
+    def __init__(self, child):
+        logging.getLogger("ray.serve").setLevel(logging.WARNING)
+        self._child = child
+
+    async def __call__(self, *args, **kwargs):
+        return await self._child.remote()
+
+
+@serve.deployment
+class GrpcDeployment:
+    def __init__(self):
+        logging.getLogger("ray.serve").setLevel(logging.WARNING)
+
+    async def grpc_call(self, user_message):
+        return serve_pb2.ModelOutput(output=9)
+
+    async def call_with_string(self, user_message):
+        return serve_pb2.ModelOutput(output=9)
+
+
+@serve.deployment
+class GrpcModelComp:
+    def __init__(self, child):
+        logging.getLogger("ray.serve").setLevel(logging.WARNING)
+        self._child = child
+
+    async def grpc_call(self, user_message):
+        await self._child.remote()
+        return serve_pb2.ModelOutput(output=9)
+
+    async def call_with_string(self, user_message):
+        await self._child.remote()
+        return serve_pb2.ModelOutput(output=9)
+
+
+@serve.deployment
+class Streamer:
+    def __init__(self, tokens_per_request: int, inter_token_delay_ms: int = 10):
+        logging.getLogger("ray.serve").setLevel(logging.WARNING)
+        self._tokens_per_request = tokens_per_request
+        self._inter_token_delay_s = inter_token_delay_ms / 1000
+
+    async def stream(self):
+        for _ in range(self._tokens_per_request):
+            await asyncio.sleep(self._inter_token_delay_s)
+            yield b"hi"
+
+    async def __call__(self):
+        return StreamingResponse(self.stream())
+
+
+@serve.deployment
+class IntermediateRouter:
+    def __init__(self, handle: DeploymentHandle):
+        logging.getLogger("ray.serve").setLevel(logging.WARNING)
+        self._handle = handle.options(stream=True)
+
+    async def stream(self):
+        async for token in self._handle.stream.remote():
+            yield token
+
+    def __call__(self):
+        return StreamingResponse(self.stream())
+
+
+@serve.deployment
+class Benchmarker:
+    def __init__(
+        self,
+        handle: DeploymentHandle,
+        stream: bool = False,
+    ):
+        logging.getLogger("ray.serve").setLevel(logging.WARNING)
+        self._handle = handle.options(stream=stream)
+        self._stream = stream
+
+    async def do_single_request(self, payload: Any = None) -> float:
+        """Completes a single unary request. Returns e2e latency in ms."""
+        start = time.perf_counter()
+
+        if payload is None:
+            await self._handle.remote()
+        else:
+            await self._handle.remote(payload)
+
+        end = time.perf_counter()
+        return 1000 * (end - start)
+
+    async def do_single_choose_dispatch(self, payload: Any = None) -> float:
+        """Completes a single unary request via choose_replica + dispatch.
+
+        Returns e2e latency in ms. With SingletonThreadRouter this involves
+        two run_coroutine_threadsafe round-trips (one for __aenter__, one
+        for _dispatch_to_marked_selection) vs. one for ``remote``.
+        """
+        start = time.perf_counter()
+
+        if payload is None:
+            async with self._handle.choose_replica() as sel:
+                await self._handle.dispatch(sel)
+        else:
+            async with self._handle.choose_replica(payload) as sel:
+                await self._handle.dispatch(sel, payload)
+
+        end = time.perf_counter()
+        return 1000 * (end - start)
+
+    async def _do_single_stream(self) -> float:
+        """Consumes a single streaming request. Returns e2e latency in ms."""
+        start = time.perf_counter()
+
+        async for r in self._handle.stream.remote():
+            pass
+
+        end = time.perf_counter()
+        return 1000 * (end - start)
+
+    async def _do_single_batch(self, batch_size: int) -> List[float]:
+        if self._stream:
+            return await asyncio.gather(
+                *[self._do_single_stream() for _ in range(batch_size)]
+            )
+        else:
+            return await asyncio.gather(
+                *[self.do_single_request() for _ in range(batch_size)]
+            )
+
+    async def run_latency_benchmark(
+        self,
+        *,
+        num_requests: int,
+        payload: Any = None,
+        mode: str = "remote",
+    ) -> pd.Series:
+        if mode == "remote":
+
+            async def f():
+                await self.do_single_request(payload)
+
+        elif mode == "choose_dispatch":
+
+            async def f():
+                await self.do_single_choose_dispatch(payload)
+
+        else:
+            raise ValueError(f"Unknown mode {mode!r}")
+
+        return await run_latency_benchmark(f, num_requests=num_requests)
+
+    async def run_throughput_benchmark(
+        self,
+        *,
+        batch_size: int,
+        num_trials: int,
+        trial_runtime: float,
+        tokens_per_request: Optional[float] = None,
+    ) -> Tuple[float, float]:
+        if self._stream:
+            assert tokens_per_request
+            multiplier = tokens_per_request * batch_size
+        else:
+            multiplier = batch_size
+
+        return await run_throughput_benchmark(
+            fn=partial(
+                self._do_single_batch,
+                batch_size=batch_size,
+            ),
+            multiplier=multiplier,
+            num_trials=num_trials,
+            trial_runtime=trial_runtime,
+        )
+
+
+# =============================================================================
+# Controller Benchmark
+# =============================================================================
+# See https://github.com/ray-project/ray/issues/60680 for more details.
+
+CONTROLLER_BENCH_CONFIG = {
+    "checkpoints": [1, 2, 4, 8, 16, 32, 64, 128, 256, 512, 1024, 2048, 3072, 4096],
+    "marination_period_s": 180,
+    "sample_interval_s": 5,
+}
+
+_CONTROLLER_AUTOSCALING_CONFIG = {
+    "min_replicas": 1,
+    "max_replicas": 4096,
+    "target_ongoing_requests": 1,
+    "upscale_delay_s": 1,
+}
+
+_CONTROLLER_WAITER_TIMEOUT_S = 1200
+
+# SignalActor from ray._common.test_utils; use high max_concurrency for many
+# concurrent waiters (up to 4096 in controller benchmark).
+_SignalActorForController = _SignalActor.options(max_concurrency=100000)
+
+
+@serve.deployment(
+    graceful_shutdown_timeout_s=1,
+    ray_actor_options={"num_cpus": 0.2},
+    max_ongoing_requests=100000,
+    autoscaling_config={
+        "min_replicas": 5,
+        "max_replicas": 10,
+        "target_ongoing_requests": 100000,
+        "upscale_delay_s": 1,
+    },
+)
+class ControllerBenchHelloWorld:
+    def __init__(self, signal_actor):
+        self.signal = signal_actor
+
+    async def __call__(self):
+        await self.signal.wait.remote()
+        return "hello"
+
+
+@serve.deployment(
+    autoscaling_config=_CONTROLLER_AUTOSCALING_CONFIG,
+    max_ongoing_requests=2,
+    graceful_shutdown_timeout_s=1,
+    ray_actor_options={"num_cpus": 0.4},
+)
+class ControllerBenchMetricsGenerator:
+    """Autoscaling deployment that generates handle metrics to stress the controller."""
+
+    def __init__(self, hello_world: DeploymentHandle):
+        self.hello_world = hello_world
+
+    async def __call__(self):
+        return await self.hello_world.remote()
+
+
+def _controller_get_active_nodes() -> int:
+    """Get number of active nodes in the cluster."""
+    return len([n for n in ray.nodes() if n.get("Alive", False)])
+
+
+async def _controller_get_replica_count(
+    deployment_name: str = "ControllerBenchMetricsGenerator",
+) -> int:
+    """Get current number of running replicas for the specified deployment."""
+    status = serve.status()
+    for app in status.applications.values():
+        for name, deployment in app.deployments.items():
+            if name == deployment_name:
+                return deployment.replica_states.get("RUNNING", 0)
+    return 0
+
+
+async def _controller_get_health_metrics() -> Dict[str, Any]:
+    """Get controller health metrics. Fails the run if unavailable."""
+    client = serve.context._global_client
+    if client is None:
+        raise RuntimeError(
+            "Serve is not connected. get_health_metrics requires an active Serve "
+            "controller. Ensure Serve is started before running the controller benchmark."
+        )
+    controller = client._controller
+    if not hasattr(controller, "get_health_metrics"):
+        raise RuntimeError(
+            "Controller does not have get_health_metrics. This API is required for "
+            "the controller benchmark. Please use a Ray version that supports "
+            "controller health metrics."
+        )
+    return await controller.get_health_metrics.remote()
+
+
+def _controller_extract_metrics_row(
+    health_metrics: Dict[str, Any],
+    checkpoint: int,
+    sample: int,
+    target_replicas: int,
+    actual_replicas: int,
+    num_nodes: int,
+    autoscale_duration_s: float,
+) -> Dict[str, Any]:
+    """Extract a flat row from health metrics with all available fields."""
+
+    def get_stat(d: dict, key: str, stat: str, default=0):
+        return d.get(key, {}).get(stat, default)
+
+    return {
+        "checkpoint": checkpoint,
+        "sample": sample,
+        "target_replicas": target_replicas,
+        "actual_replicas": actual_replicas,
+        "num_nodes": num_nodes,
+        "autoscale_duration_s": round(autoscale_duration_s, 3),
+        "loop_duration_mean_s": get_stat(health_metrics, "loop_duration_s", "mean"),
+        "loop_duration_std_s": get_stat(health_metrics, "loop_duration_s", "std"),
+        "loops_per_second": health_metrics.get("loops_per_second", 0),
+        "event_loop_delay_s": health_metrics.get("event_loop_delay_s", 0),
+        "num_asyncio_tasks": health_metrics.get("num_asyncio_tasks", 0),
+        "deployment_state_update_mean_s": get_stat(
+            health_metrics, "deployment_state_update_duration_s", "mean"
+        ),
+        "application_state_update_mean_s": get_stat(
+            health_metrics, "application_state_update_duration_s", "mean"
+        ),
+        "proxy_state_update_mean_s": get_stat(
+            health_metrics, "proxy_state_update_duration_s", "mean"
+        ),
+        "proxy_state_update_std_s": get_stat(
+            health_metrics, "proxy_state_update_duration_s", "std"
+        ),
+        "node_update_mean_s": get_stat(
+            health_metrics, "node_update_duration_s", "mean"
+        ),
+        "node_update_std_s": get_stat(health_metrics, "node_update_duration_s", "std"),
+        "node_update_min_s": get_stat(health_metrics, "node_update_duration_s", "min"),
+        "handle_metrics_delay_mean_ms": get_stat(
+            health_metrics, "handle_metrics_delay_ms", "mean"
+        ),
+        "replica_metrics_delay_mean_ms": get_stat(
+            health_metrics, "replica_metrics_delay_ms", "mean"
+        ),
+        "process_memory_mb": health_metrics.get("process_memory_mb", 0),
+    }
+
+
+async def _controller_wait_for_replicas_up(target: int, timeout: float = 300) -> float:
+    start = time.time()
+    while time.time() - start < timeout:
+        actual = await _controller_get_replica_count()
+        if actual >= target:
+            return time.time() - start
+        if int(time.time() - start) % 10 == 0:
+            logging.info(f"Waiting for {target} replicas... {actual}/{target}")
+        await asyncio.sleep(0.5)
+    actual = await _controller_get_replica_count()
+    raise RuntimeError(
+        f"Timeout: Only {actual}/{target} replicas after {timeout}s. Ending experiment."
+    )
+
+
+async def _controller_wait_for_waiters(
+    signal_actor, expected: int, timeout: float = 300
+) -> float:
+    start = time.time()
+    while time.time() - start < timeout:
+        num_waiters = await signal_actor.cur_num_waiters.remote()
+        if num_waiters >= expected:
+            return time.time() - start
+        await asyncio.sleep(0.5)
+        if int(time.time() - start) % 10 == 0:
+            logging.info(f"Waiting for {expected} waiters... {num_waiters}/{expected}")
+    num_waiters = await signal_actor.cur_num_waiters.remote()
+    raise RuntimeError(
+        f"Timeout: Only {num_waiters}/{expected} requests reached replicas after "
+        f"{timeout}s. Ending experiment."
+    )
+
+
+async def _controller_wait_for_deployment_healthy(
+    deployment_name: str = "ControllerBenchMetricsGenerator",
+    app_name: str = "default",
+    timeout: float = 60,
+) -> None:
+    """Wait for the deployment to enter HEALTHY status via serve.status()."""
+    start = time.time()
+    while time.time() - start < timeout:
+        status = serve.status()
+        app = status.applications.get(app_name)
+        dep_status = None
+        if app and deployment_name in app.deployments:
+            dep = app.deployments[deployment_name]
+            dep_status = dep.status
+            if dep_status == DeploymentStatus.HEALTHY:
+                return
+            if dep_status == DeploymentStatus.UNHEALTHY:
+                raise RuntimeError(
+                    f"Deployment {deployment_name} is UNHEALTHY: {getattr(dep, 'message', '')}"
+                )
+        if int(time.time() - start) % 10 == 0:
+            logging.info(
+                f"Waiting for {deployment_name} to be healthy, current: {dep_status}."
+            )
+        await asyncio.sleep(0.5)
+
+    raise RuntimeError(
+        f"Deployment {deployment_name} did not become HEALTHY after {timeout}s."
+    )
+
+
+_BATCH_SIZE = 64
+
+
+async def _controller_run_checkpoint(
+    handle: DeploymentHandle,
+    signal_actor,
+    checkpoint: int,
+    target_replicas: int,
+    marination_period_s: int,
+    sample_interval_s: int,
+) -> List[Dict[str, Any]]:
+    """Run a single checkpoint and collect metrics."""
+    start_time = time.time()
+    num_requests = int(target_replicas)
+
+    pending_requests: List[Any] = []
+    pending_requests.extend([handle.remote() for _ in range(num_requests)])
+    logging.info(f"Waiting for {num_requests} requests to be up...")
+    await _controller_wait_for_waiters(
+        signal_actor, len(pending_requests), timeout=_CONTROLLER_WAITER_TIMEOUT_S
+    )
+    logging.info(f"Waiting for {target_replicas} replicas to be up...")
+    # TODO: This is a hack to allow for some tolerance in the number of replicas.
+    # This is because the controller may not scale exactly to the target number of replicas.
+    # This is a bug in the controller autoscaling metrics aggregation logic, needs
+    # to be investigated further.
+    # This has the potential to introduce noise in the results from this benchmark.
+    replica_tolerance = 0.8
+    await _controller_wait_for_replicas_up(
+        int(target_replicas * replica_tolerance), timeout=_CONTROLLER_WAITER_TIMEOUT_S
+    )
+    logging.info(f"All {target_replicas} replicas are up.")
+    logging.info("Waiting for deployment to be healthy...")
+    await _controller_wait_for_deployment_healthy(timeout=_CONTROLLER_WAITER_TIMEOUT_S)
+    logging.info("Deployment is healthy.")
+    logging.info(f"Waiting for {marination_period_s} seconds to collect metrics...")
+
+    autoscale_duration_s = time.time() - start_time
+
+    samples = []
+    num_samples = marination_period_s // sample_interval_s
+    for sample_idx in range(num_samples):
+        health_metrics = await _controller_get_health_metrics()
+        actual_replicas = await _controller_get_replica_count()
+        num_nodes = _controller_get_active_nodes()
+        row = _controller_extract_metrics_row(
+            health_metrics=health_metrics,
+            checkpoint=checkpoint,
+            sample=sample_idx,
+            target_replicas=target_replicas,
+            actual_replicas=actual_replicas,
+            num_nodes=num_nodes,
+            autoscale_duration_s=autoscale_duration_s,
+        )
+        samples.append(row)
+        if sample_idx < num_samples - 1:
+            await asyncio.sleep(sample_interval_s)
+
+    await signal_actor.send.remote(clear=True)
+    try:
+        await asyncio.wait_for(
+            asyncio.gather(*pending_requests, return_exceptions=True),
+            timeout=30.0,
+        )
+    except asyncio.TimeoutError:
+        pass
+
+    return samples
+
+
+async def run_controller_benchmark(
+    config: Optional[Dict[str, Any]] = None,
+) -> List[Dict[str, Any]]:
+    """
+    Run the controller health metrics benchmark and return raw samples.
+
+    Uses MetricsGenerator (autoscaling) -> HelloWorld (fixed) -> SignalActor
+    to stress the controller as replicas scale. Fails if get_health_metrics
+    is unavailable.
+
+    Args:
+        config: Optional benchmark config (checkpoints, marination_period_s,
+            sample_interval_s). Uses CONTROLLER_BENCH_CONFIG if None.
+
+    Returns:
+        List of sample dicts (one per marination sample). Each sample has
+        target_replicas, autoscale_duration_s, loop_duration_mean_s,
+        loops_per_second, event_loop_delay_s, num_asyncio_tasks, etc.
+        Caller converts to perf_metrics via convert_controller_samples_to_perf_metrics.
+    """
+    cfg = config or CONTROLLER_BENCH_CONFIG
+    checkpoints = cfg["checkpoints"]
+    marination_period_s = cfg["marination_period_s"]
+    sample_interval_s = cfg["sample_interval_s"]
+
+    if not ray.is_initialized():
+        ray.init()
+
+    signal_actor = _SignalActorForController.remote()
+    all_samples: List[Dict[str, Any]] = []
+
+    try:
+        for checkpoint_idx, target_replicas in enumerate(checkpoints):
+            hello_world = ControllerBenchHelloWorld.bind(signal_actor)
+            app = ControllerBenchMetricsGenerator.bind(hello_world)
+            handle = serve.run(app, name="default", route_prefix=None)
+
+            samples = await _controller_run_checkpoint(
+                handle=handle,
+                signal_actor=signal_actor,
+                checkpoint=checkpoint_idx,
+                target_replicas=target_replicas,
+                marination_period_s=marination_period_s,
+                sample_interval_s=sample_interval_s,
+            )
+            all_samples.extend(samples)
+            serve.shutdown()
+    finally:
+        serve.shutdown()
+
+    return all_samples

python/ray/serve/_private/benchmarks/handle_noop_latency.py

@@ -0,0 +1,39 @@
+import time
+
+import click
+
+from ray import serve
+from ray.serve._private.benchmarks.common import Benchmarker, Noop
+from ray.serve.handle import DeploymentHandle
+
+
+@click.command(help="Benchmark no-op DeploymentHandle latency.")
+@click.option("--num-replicas", type=int, default=1)
+@click.option("--num-requests", type=int, default=100)
+@click.option(
+    "--mode",
+    type=click.Choice(["remote", "choose_dispatch"]),
+    default="remote",
+    help="Which call pattern to benchmark.",
+)
+def main(num_replicas: int, num_requests: int, mode: str):
+    h: DeploymentHandle = serve.run(
+        Benchmarker.bind(Noop.options(num_replicas=num_replicas).bind())
+    )
+
+    latencies = h.run_latency_benchmark.remote(
+        num_requests=num_requests, mode=mode
+    ).result()
+
+    # Let the logs flush to avoid interwoven output.
+    time.sleep(1)
+
+    print(
+        f"Latency (ms) for noop DeploymentHandle requests via {mode!r} "
+        f"(num_replicas={num_replicas},num_requests={num_requests}):"
+    )
+    print(latencies.describe(percentiles=[0.5, 0.9, 0.95, 0.99]))
+
+
+if __name__ == "__main__":
+    main()

python/ray/serve/_private/benchmarks/handle_throughput.py

@@ -0,0 +1,62 @@
+import click
+
+from ray import serve
+from ray.serve._private.benchmarks.common import Benchmarker, Hello
+from ray.serve.handle import DeploymentHandle
+
+
+@click.command(help="Benchmark deployment handle throughput.")
+@click.option(
+    "--batch-size",
+    type=int,
+    default=100,
+    help="Number of requests to send to downstream deployment in each trial.",
+)
+@click.option(
+    "--num-replicas",
+    type=int,
+    default=1,
+    help="Number of replicas in the downstream deployment.",
+)
+@click.option(
+    "--num-trials",
+    type=int,
+    default=5,
+    help="Number of trials of the benchmark to run.",
+)
+@click.option(
+    "--trial-runtime",
+    type=int,
+    default=1,
+    help="Duration to run each trial of the benchmark for (seconds).",
+)
+def main(
+    batch_size: int,
+    num_replicas: int,
+    num_trials: int,
+    trial_runtime: float,
+):
+    app = Benchmarker.bind(
+        Hello.options(
+            num_replicas=num_replicas, ray_actor_options={"num_cpus": 0}
+        ).bind(),
+    )
+    h: DeploymentHandle = serve.run(app)
+
+    mean, stddev = h.run_throughput_benchmark.remote(
+        batch_size=batch_size,
+        num_trials=num_trials,
+        trial_runtime=trial_runtime,
+    ).result()
+
+    print(
+        "DeploymentHandle throughput {}: {} +- {} requests/s".format(
+            f"(num_replicas={num_replicas}, batch_size={batch_size})",
+            mean,
+            stddev,
+        )
+    )
+
+
+if __name__ == "__main__":
+    main()

python/ray/serve/_private/benchmarks/http_noop_latency.py

@@ -0,0 +1,32 @@
+import asyncio
+
+import click
+import pandas as pd
+import requests
+
+from ray import serve
+from ray.serve._private.benchmarks.common import Noop, run_latency_benchmark
+
+
+@click.command(help="Benchmark no-op HTTP latency.")
+@click.option("--num-replicas", type=int, default=1)
+@click.option("--num-requests", type=int, default=100)
+def main(num_replicas: int, num_requests: int):
+    serve.run(Noop.options(num_replicas=num_replicas).bind())
+
+    latencies: pd.Series = asyncio.new_event_loop().run_until_complete(
+        run_latency_benchmark(
+            lambda: requests.get("http://localhost:8000"),
+            num_requests=num_requests,
+        )
+    )
+
+    print(
+        "Latency (ms) for noop HTTP requests "
+        f"(num_replicas={num_replicas},num_requests={num_requests}):"
+    )
+    print(latencies.describe(percentiles=[0.5, 0.9, 0.95, 0.99]))
+
+
+if __name__ == "__main__":
+    main()

python/ray/serve/_private/benchmarks/locust_utils.py

@@ -0,0 +1,374 @@
+import argparse
+import logging
+import time
+from dataclasses import asdict, dataclass
+from typing import Any, Dict, List, NamedTuple
+
+from ray.serve._private.utils import generate_request_id
+
+logger = logging.getLogger(__file__)
+logging.basicConfig(level=logging.INFO)
+
+MASTER_PORT = 5557
+
+
+@dataclass
+class LocustStage:
+    duration_s: int
+    users: int
+    spawn_rate: float
+
+
+@dataclass
+class PerformanceStats:
+    p50_latency: float
+    p90_latency: float
+    p99_latency: float
+    rps: float
+
+
+@dataclass
+class LocustTestResults:
+    history: List[Dict]
+    total_requests: int
+    num_failures: int
+    avg_latency: float
+    p50_latency: float
+    p90_latency: float
+    p99_latency: float
+    avg_rps: float
+    stats_in_stages: List[PerformanceStats]
+
+
+@dataclass
+class FailedRequest:
+    request_id: str
+    status_code: int
+    exception: str
+    response_time_ms: float
+    start_time_s: float
+
+
+class LocustClient:
+    def __init__(
+        self,
+        host_url: str,
+        token: str,
+        data: Dict[str, Any] = None,
+    ):
+        from locust import FastHttpUser, constant, events, task
+        from locust.contrib.fasthttp import FastResponse
+
+        self.errors = []
+        self.stats_in_stages: List[PerformanceStats] = []
+
+        class EndpointUser(FastHttpUser):
+            wait_time = constant(0)
+            failed_requests = []
+            host = host_url
+
+            @task
+            def test(self):
+                request_id = generate_request_id()
+                headers = (
+                    {"Authorization": f"Bearer {token}", "X-Request-ID": request_id}
+                    if token
+                    else None
+                )
+                start = time.perf_counter()
+                with self.client.get(
+                    "", headers=headers, json=data, catch_response=True
+                ) as r:
+                    # locust<=2.18 FastHttp truncates response_time to whole ms;
+                    # re-measure so the 0.1ms buckets see sub-ms differences.
+                    r.request_meta["response_time"] = (
+                        time.perf_counter() - start
+                    ) * 1000
+                    r.request_meta["context"]["request_id"] = request_id
+
+            @events.request.add_listener
+            def on_request(
+                response: FastResponse,
+                exception,
+                context,
+                start_time: float,
+                response_time: float,
+                **kwargs,
+            ):
+                if exception and response.status_code != 0:
+                    request_id = context["request_id"]
+                    print(
+                        f"Request '{request_id}' failed with exception:\n"
+                        f"{exception}\n{response.text}"
+                    )
+
+                    if response.status_code != 0:
+                        response.encoding = "utf-8"
+                        err = FailedRequest(
+                            request_id=request_id,
+                            status_code=response.status_code,
+                            exception=response.text,
+                            response_time_ms=response_time,
+                            start_time_s=start_time,
+                        )
+                        self.errors.append(err)
+                        print(
+                            f"Request '{request_id}' failed with exception:\n"
+                            f"{exception}\n{response.text}"
+                        )
+
+        self.user_class = EndpointUser
+
+
+class ResponseTimeSnapshot(NamedTuple):
+    # Cumulative {rounded_response_time: count} histogram + request count.
+    response_times: Dict[float, int]
+    num_requests: int
+
+
+def _fine_bucket_response_time(response_time):
+    """0.1ms resolution below 100ms (vs locust's 1ms floor), coarser above."""
+    if response_time < 100:
+        return round(response_time, 1)
+    elif response_time < 1000:
+        return round(response_time)
+    else:
+        return int(round(response_time, -1))
+
+
+def _install_fine_response_time_bucketing():
+    """Swap in the finer bucketer; must run in every response-logging process.
+
+    Released locust (through at least 2.41) inlines the rounding in
+    StatsEntry._log_response_time, so patching requires overriding the whole
+    method. Unreleased locust factors it into stats.bucket_response_time."""
+    import locust.stats
+
+    if hasattr(locust.stats, "bucket_response_time"):
+        locust.stats.bucket_response_time = _fine_bucket_response_time
+        return
+
+    def _log_response_time(self, response_time):
+        # Copy of locust 2.x StatsEntry._log_response_time with the inline
+        # rounding replaced by the fine bucketer.
+        if response_time is None:
+            self.num_none_requests += 1
+            return
+
+        self.total_response_time += response_time
+
+        if self.min_response_time is None:
+            self.min_response_time = response_time
+        self.min_response_time = min(self.min_response_time, response_time)
+        self.max_response_time = max(self.max_response_time, response_time)
+
+        rounded_response_time = _fine_bucket_response_time(response_time)
+        # setdefault keeps this compatible with both the plain dict (<=2.18)
+        # and defaultdict (>=2.33) versions of response_times.
+        self.response_times.setdefault(rounded_response_time, 0)
+        self.response_times[rounded_response_time] += 1
+
+    locust.stats.StatsEntry._log_response_time = _log_response_time
+
+
+def on_stage_finished(master_runner, stats_in_stages, stage_duration_s, prev_snapshot):
+    """Per-stage stats by differencing cumulative snapshots; returns the
+    snapshot to seed the next stage. Percentiles use locust's own
+    calculate_response_time_percentile so they match its end-of-test report."""
+    from locust.stats import (
+        calculate_response_time_percentile,
+        diff_response_time_dicts,
+    )
+
+    stats_entry = master_runner.stats.entries.get(("", "GET"))
+    snapshot = ResponseTimeSnapshot(
+        dict(stats_entry.response_times), stats_entry.num_requests
+    )
+
+    stage_hist = diff_response_time_dicts(
+        snapshot.response_times, prev_snapshot.response_times
+    )
+    stage_requests = snapshot.num_requests - prev_snapshot.num_requests
+
+    stats_in_stages.append(
+        PerformanceStats(
+            p50_latency=calculate_response_time_percentile(
+                stage_hist, stage_requests, 0.5
+            ),
+            p90_latency=calculate_response_time_percentile(
+                stage_hist, stage_requests, 0.9
+            ),
+            p99_latency=calculate_response_time_percentile(
+                stage_hist, stage_requests, 0.99
+            ),
+            rps=stage_requests / stage_duration_s if stage_duration_s else 0.0,
+        )
+    )
+    return snapshot
+
+
+def run_locust_worker(
+    master_address: str, host_url: str, token: str, data: Dict[str, Any]
+):
+    import locust
+    from locust.env import Environment
+    from locust.log import setup_logging
+
+    setup_logging("INFO")
+    # Workers log response times, so the finer bucketer must be installed here.
+    _install_fine_response_time_bucketing()
+    client = LocustClient(host_url=host_url, token=token, data=data)
+    env = Environment(user_classes=[client.user_class], events=locust.events)
+
+    runner = env.create_worker_runner(
+        master_host=master_address, master_port=MASTER_PORT
+    )
+    runner.greenlet.join()
+
+    if client.errors:
+        raise RuntimeError(f"There were {len(client.errors)} errors: {client.errors}")
+
+
+def run_locust_master(
+    host_url: str,
+    token: str,
+    expected_num_workers: int,
+    stages: List[LocustStage],
+    wait_for_workers_timeout_s: float,
+):
+    import gevent
+    import locust
+    from locust import LoadTestShape
+    from locust.env import Environment
+    from locust.stats import (
+        get_error_report_summary,
+        get_percentile_stats_summary,
+        get_stats_summary,
+        stats_history,
+        stats_printer,
+    )
+
+    _install_fine_response_time_bucketing()
+    client = LocustClient(host_url, token)
+
+    class StagesShape(LoadTestShape):
+        curr_stage_ix = 0
+        # Cumulative response-time snapshot at the start of the current stage;
+        # on_stage_finished diffs against it to get per-stage stats.
+        prev_snapshot = ResponseTimeSnapshot({}, 0)
+
+        def tick(cls):
+            run_time = cls.get_run_time()
+            prefix_time = 0
+            for i, stage in enumerate(stages):
+                prefix_time += stage.duration_s
+
+                if run_time < prefix_time:
+                    if i != cls.curr_stage_ix:
+                        cls.prev_snapshot = on_stage_finished(
+                            master_runner,
+                            client.stats_in_stages,
+                            stages[cls.curr_stage_ix].duration_s,
+                            cls.prev_snapshot,
+                        )
+                        cls.curr_stage_ix = i
+
+                    current_stage = stages[cls.curr_stage_ix]
+                    return current_stage.users, current_stage.spawn_rate
+
+            # End of stage test
+            cls.prev_snapshot = on_stage_finished(
+                master_runner,
+                client.stats_in_stages,
+                stages[cls.curr_stage_ix].duration_s,
+                cls.prev_snapshot,
+            )
+
+    master_env = Environment(
+        user_classes=[client.user_class],
+        shape_class=StagesShape(),
+        events=locust.events,
+    )
+    master_runner = master_env.create_master_runner("*", MASTER_PORT)
+
+    start = time.time()
+    while len(master_runner.clients.ready) < expected_num_workers:
+        if time.time() - start > wait_for_workers_timeout_s:
+            raise RuntimeError(
+                f"Timed out waiting for {expected_num_workers} workers to "
+                "connect to Locust master."
+            )
+
+        print(
+            f"Waiting for workers to be ready, "
+            f"{len(master_runner.clients.ready)} "
+            f"of {expected_num_workers} ready."
+        )
+        time.sleep(1)
+
+    # Periodically output current stats (each entry is aggregated
+    # stats over the past 10 seconds, by default)
+    gevent.spawn(stats_printer(master_env.stats))
+    gevent.spawn(stats_history, master_runner)
+
+    # Start test & wait for the shape test to finish
+    master_runner.start_shape()
+    master_runner.shape_greenlet.join()
+    # Send quit signal to all locust workers
+    master_runner.quit()
+
+    # Print stats
+    for line in get_stats_summary(master_runner.stats, current=False):
+        print(line)
+    # Print percentile stats
+    for line in get_percentile_stats_summary(master_runner.stats):
+        print(line)
+    # Print error report
+    if master_runner.stats.errors:
+        for line in get_error_report_summary(master_runner.stats):
+            print(line)
+
+    stats_entry_key = ("", "GET")
+    stats_entry = master_runner.stats.entries.get(stats_entry_key)
+    results = LocustTestResults(
+        history=master_runner.stats.history,
+        total_requests=master_runner.stats.num_requests,
+        num_failures=master_runner.stats.num_failures,
+        avg_latency=stats_entry.avg_response_time,
+        p50_latency=stats_entry.get_response_time_percentile(0.5),
+        p90_latency=stats_entry.get_response_time_percentile(0.9),
+        p99_latency=stats_entry.get_response_time_percentile(0.99),
+        avg_rps=stats_entry.total_rps,
+        stats_in_stages=client.stats_in_stages,
+    )
+    return asdict(results)
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--worker-type", type=str, required=True)
+    parser.add_argument("--host-url", type=str, required=True)
+    parser.add_argument("--token", type=str, required=True)
+    parser.add_argument("--master-address", type=str, required=False)
+    parser.add_argument("--expected-num-workers", type=int, required=False)
+    parser.add_argument("--stages", type=str, required=False)
+    parser.add_argument("--wait-for-workers-timeout-s", type=float, required=False)
+    args = parser.parse_args()
+    host_url = args.host_url
+    token = args.token
+    if args.worker_type == "master":
+        results = run_locust_master(
+            host_url,
+            token,
+            args.expected_num_workers,
+            args.stages,
+            args.wait_for_workers_timeout_s,
+        )
+    else:
+        results = run_locust_worker(args.master_address, host_url, token, args.data)
+
+    print(results)
+
+
+if __name__ == "__main__":
+    main()

python/ray/serve/_private/benchmarks/microbenchmark.py

@@ -0,0 +1,182 @@
+# Runs several scenarios with varying max batch size, max concurrent queries,
+# number of replicas, and with intermediate serve handles (to simulate ensemble
+# models) either on or off.
+
+import asyncio
+import logging
+from pprint import pprint
+from typing import Dict, Union
+
+import aiohttp
+from starlette.requests import Request
+
+import ray
+from ray import serve
+from ray.serve._private.benchmarks.common import run_throughput_benchmark
+from ray.serve.handle import DeploymentHandle
+
+NUM_CLIENTS = 8
+CALLS_PER_BATCH = 100
+
+
+async def fetch(session, data):
+    async with session.get("http://localhost:8000/", data=data) as response:
+        response = await response.text()
+        assert response == "ok", response
+
+
+@ray.remote
+class Client:
+    def ready(self):
+        return "ok"
+
+    async def do_queries(self, num, data):
+        async with aiohttp.ClientSession() as session:
+            for _ in range(num):
+                await fetch(session, data)
+
+
+def build_app(
+    intermediate_handles: bool,
+    num_replicas: int,
+    max_batch_size: int,
+    max_ongoing_requests: int,
+):
+    @serve.deployment(max_ongoing_requests=1000)
+    class Upstream:
+        def __init__(self, handle: DeploymentHandle):
+            self._handle = handle
+
+            # Turn off access log.
+            logging.getLogger("ray.serve").setLevel(logging.WARNING)
+
+        async def __call__(self, req: Request):
+            return await self._handle.remote(await req.body())
+
+    @serve.deployment(
+        num_replicas=num_replicas,
+        max_ongoing_requests=max_ongoing_requests,
+    )
+    class Downstream:
+        def __init__(self):
+            # Turn off access log.
+            logging.getLogger("ray.serve").setLevel(logging.WARNING)
+
+        @serve.batch(max_batch_size=max_batch_size)
+        async def batch(self, reqs):
+            return [b"ok"] * len(reqs)
+
+        async def __call__(self, req: Union[bytes, Request]):
+            if max_batch_size > 1:
+                return await self.batch(req)
+            else:
+                return b"ok"
+
+    if intermediate_handles:
+        return Upstream.bind(Downstream.bind())
+    else:
+        return Downstream.bind()
+
+
+async def trial(
+    intermediate_handles: bool,
+    num_replicas: int,
+    max_batch_size: int,
+    max_ongoing_requests: int,
+    data_size: str,
+) -> Dict[str, float]:
+    results = {}
+
+    trial_key_base = (
+        f"replica:{num_replicas}/batch_size:{max_batch_size}/"
+        f"concurrent_queries:{max_ongoing_requests}/"
+        f"data_size:{data_size}/intermediate_handle:{intermediate_handles}"
+    )
+
+    print(
+        f"intermediate_handles={intermediate_handles},"
+        f"num_replicas={num_replicas},"
+        f"max_batch_size={max_batch_size},"
+        f"max_ongoing_requests={max_ongoing_requests},"
+        f"data_size={data_size}"
+    )
+
+    app = build_app(
+        intermediate_handles, num_replicas, max_batch_size, max_ongoing_requests
+    )
+    serve.run(app)
+
+    if data_size == "small":
+        data = None
+    elif data_size == "large":
+        data = b"a" * 1024 * 1024
+    else:
+        raise ValueError("data_size should be 'small' or 'large'.")
+
+    async with aiohttp.ClientSession() as session:
+
+        async def single_client():
+            for _ in range(CALLS_PER_BATCH):
+                await fetch(session, data)
+
+        single_client_avg_tps, single_client_std_tps = await run_throughput_benchmark(
+            single_client,
+            multiplier=CALLS_PER_BATCH,
+        )
+        print(
+            "\t{} {} +- {} requests/s".format(
+                "single client {} data".format(data_size),
+                single_client_avg_tps,
+                single_client_std_tps,
+            )
+        )
+        key = f"num_client:1/{trial_key_base}"
+        results[key] = single_client_avg_tps
+
+    clients = [Client.remote() for _ in range(NUM_CLIENTS)]
+    ray.get([client.ready.remote() for client in clients])
+
+    async def many_clients():
+        ray.get([a.do_queries.remote(CALLS_PER_BATCH, data) for a in clients])
+
+    multi_client_avg_tps, _ = await run_throughput_benchmark(
+        many_clients,
+        multiplier=CALLS_PER_BATCH * len(clients),
+    )
+
+    results[f"num_client:{len(clients)}/{trial_key_base}"] = multi_client_avg_tps
+    return results
+
+
+async def main():
+    results = {}
+    for intermediate_handles in [False, True]:
+        for num_replicas in [1, 8]:
+            for max_batch_size, max_ongoing_requests in [
+                (1, 1),
+                (1, 10000),
+                (10000, 10000),
+            ]:
+                # TODO(edoakes): large data causes broken pipe errors.
+                for data_size in ["small"]:
+                    results.update(
+                        await trial(
+                            intermediate_handles,
+                            num_replicas,
+                            max_batch_size,
+                            max_ongoing_requests,
+                            data_size,
+                        )
+                    )
+
+    print("Results from all conditions:")
+    pprint(results)
+    return results
+
+
+if __name__ == "__main__":
+    ray.init()
+    serve.start()
+    loop = asyncio.new_event_loop()
+    asyncio.set_event_loop(loop)
+    loop.run_until_complete(main())

python/ray/serve/_private/benchmarks/proxy_benchmark.py

@@ -0,0 +1,294 @@
+# Runs some request ping to compare HTTP and gRPC performances in TPS and latency.
+# Note: this takes around 1 hour to run.
+
+import asyncio
+import json
+import logging
+import time
+from random import random
+from typing import Callable, Dict
+
+import aiohttp
+import numpy as np
+import pandas as pd
+from grpc import aio
+from starlette.requests import Request
+
+import ray
+from ray import serve
+from ray.serve._private.common import RequestProtocol
+from ray.serve.config import gRPCOptions
+from ray.serve.generated import serve_pb2, serve_pb2_grpc
+from ray.serve.handle import DeploymentHandle
+
+CALLS_PER_BATCH = 100
+DELTA = 10**-7
+
+
+async def get_query_tps(name: str, fn: Callable, multiplier: int = CALLS_PER_BATCH):
+    """Get query TPS.
+
+    Run the function for 0.5 seconds 10 times to calculate how many requests can
+    be completed. And use those stats to calculate the mean and std of TPS.
+    """
+    # warmup
+    start = time.time()
+    while time.time() - start < 0.1:
+        await fn()
+    # real run
+    stats = []
+    for _ in range(10):
+        count = 0
+        start = time.time()
+        while time.time() - start < 0.5:
+            await fn()
+            count += 1
+        end = time.time()
+        stats.append(multiplier * count / (end - start))
+    tps_mean = round(np.mean(stats), 2)
+    tps_std = round(np.std(stats), 2)
+    print(f"\t{name} {tps_mean} +- {tps_std} requests/s")
+
+    return tps_mean, tps_std
+
+
+async def get_query_latencies(name: str, fn: Callable):
+    """Get query latencies.
+
+    Take all the latencies from the function and calculate the mean and std.
+    """
+    many_client_results = np.asarray(await fn())
+    many_client_results.flatten()
+    latency_ms_mean = round(np.mean(many_client_results) * 1000, 2)
+    latency_ms_std = round(np.std(many_client_results) * 1000, 2)
+    print(f"\t{name} {latency_ms_mean} +- {latency_ms_std} ms")
+
+    return latency_ms_mean, latency_ms_std
+
+
+async def fetch_http(session, data):
+    data_json = {"nums": data}
+    response = await session.get("http://localhost:8000/", json=data_json)
+    response_text = await response.read()
+    float(response_text.decode())
+
+
+async def fetch_grpc(stub, data):
+    result = await stub.grpc_call(serve_pb2.RawData(nums=data))
+    _ = result.output
+
+
+@ray.remote
+class HTTPClient:
+    def ready(self):
+        return "ok"
+
+    async def do_queries(self, num, data):
+        async with aiohttp.ClientSession() as session:
+            for _ in range(num):
+                await fetch_http(session, data)
+
+    async def time_queries(self, num, data):
+        stats = []
+        async with aiohttp.ClientSession() as session:
+            for _ in range(num):
+                start = time.time()
+                await fetch_http(session, data)
+                end = time.time()
+                stats.append(end - start)
+
+        return stats
+
+
+@ray.remote
+class gRPCClient:
+    def __init__(self):
+        channel = aio.insecure_channel("localhost:9000")
+        self.stub = serve_pb2_grpc.RayServeBenchmarkServiceStub(channel)
+
+    def ready(self):
+        return "ok"
+
+    async def do_queries(self, num, data):
+        for _ in range(num):
+            await fetch_grpc(self.stub, data)
+
+    async def time_queries(self, num, data):
+        stats = []
+        for _ in range(num):
+            start = time.time()
+            await fetch_grpc(self.stub, data)
+            end = time.time()
+            stats.append(end - start)
+        return stats
+
+
+def build_app(
+    num_replicas: int,
+    max_ongoing_requests: int,
+    data_size: int,
+):
+    @serve.deployment(max_ongoing_requests=1000)
+    class DataPreprocessing:
+        def __init__(self, handle: DeploymentHandle):
+            self._handle = handle
+
+            # Turn off access log.
+            logging.getLogger("ray.serve").setLevel(logging.WARNING)
+
+        def normalize(self, raw: np.ndarray) -> np.ndarray:
+            return (raw - np.min(raw)) / (np.max(raw) - np.min(raw) + DELTA)
+
+        async def __call__(self, req: Request):
+            """HTTP entrypoint.
+
+            It parses the request, normalize the data, and send to model for inference.
+            """
+            body = json.loads(await req.body())
+            raw = np.asarray(body["nums"])
+            processed = self.normalize(raw)
+            return await self._handle.remote(processed)
+
+        async def grpc_call(self, raq_data):
+            """gRPC entrypoint.
+
+            It parses the request, normalize the data, and send to model for inference.
+            """
+            raw = np.asarray(raq_data.nums)
+            processed = self.normalize(raw)
+            output = await self._handle.remote(processed)
+            return serve_pb2.ModelOutput(output=output)
+
+        async def call_with_string(self, raq_data):
+            """gRPC entrypoint."""
+            return serve_pb2.ModelOutput(output=0)
+
+    @serve.deployment(
+        num_replicas=num_replicas,
+        max_ongoing_requests=max_ongoing_requests,
+    )
+    class ModelInference:
+        def __init__(self):
+            # Turn off access log.
+            logging.getLogger("ray.serve").setLevel(logging.WARNING)
+            self._model = np.random.randn(data_size, data_size)
+
+        async def __call__(self, processed: np.ndarray) -> float:
+            # Run a dot product with a random matrix to simulate a model inference.
+            model_output = np.dot(processed, self._model)
+            return sum(model_output)
+
+    return DataPreprocessing.bind(ModelInference.bind())
+
+
+async def trial(
+    num_replicas: int,
+    max_ongoing_requests: int,
+    data_size: int,
+    num_clients: int,
+    proxy: RequestProtocol,
+) -> Dict[str, float]:
+    # Generate input data as array of random floats.
+    data = [random() for _ in range(data_size)]
+
+    # Build and deploy the app.
+    app = build_app(
+        num_replicas=num_replicas,
+        max_ongoing_requests=max_ongoing_requests,
+        data_size=data_size,
+    )
+    serve.run(app)
+
+    # Start clients.
+    if proxy == RequestProtocol.GRPC:
+        clients = [gRPCClient.remote() for _ in range(num_clients)]
+    elif proxy == RequestProtocol.HTTP:
+        clients = [HTTPClient.remote() for _ in range(num_clients)]
+    ray.get([client.ready.remote() for client in clients])
+
+    async def client_time_queries():
+        return ray.get([a.time_queries.remote(CALLS_PER_BATCH, data) for a in clients])
+
+    async def client_do_queries():
+        ray.get([a.do_queries.remote(CALLS_PER_BATCH, data) for a in clients])
+
+    trial_key_base = (
+        f"proxy:{proxy}/"
+        f"num_client:{num_clients}/"
+        f"replica:{num_replicas}/"
+        f"concurrent_queries:{max_ongoing_requests}/"
+        f"data_size:{data_size}"
+    )
+    tps_mean, tps_sdt = await get_query_tps(
+        trial_key_base,
+        client_do_queries,
+    )
+    latency_ms_mean, latency_ms_std = await get_query_latencies(
+        trial_key_base,
+        client_time_queries,
+    )
+
+    results = {
+        "proxy": proxy.value,
+        "num_client": num_clients,
+        "replica": num_replicas,
+        "concurrent_queries": max_ongoing_requests,
+        "data_size": data_size,
+        "tps_mean": tps_mean,
+        "tps_sdt": tps_sdt,
+        "latency_ms_mean": latency_ms_mean,
+        "latency_ms_std": latency_ms_std,
+    }
+
+    return results
+
+
+async def main():
+    start_time = time.time()
+    results = []
+    for num_replicas in [1, 8]:
+        for max_ongoing_requests in [1, 10_000]:
+            for data_size in [1, 100, 10_000]:
+                for num_clients in [1, 8]:
+                    for proxy in [RequestProtocol.GRPC, RequestProtocol.HTTP]:
+                        results.append(
+                            await trial(
+                                num_replicas=num_replicas,
+                                max_ongoing_requests=max_ongoing_requests,
+                                data_size=data_size,
+                                num_clients=num_clients,
+                                proxy=proxy,
+                            )
+                        )
+
+    print(f"Total time: {time.time() - start_time}s")
+    print("results", results)
+
+    df = pd.DataFrame.from_dict(results)
+    df = df.sort_values(
+        by=["proxy", "num_client", "replica", "concurrent_queries", "data_size"]
+    )
+    print("Results from all conditions:")
+    # Print the results in with tab separated so we can copy into google sheets.
+    for i in range(len(df.index)):
+        row = list(df.iloc[i])
+        print("\t".join(map(str, row)))
+
+
+if __name__ == "__main__":
+    ray.init()
+
+    grpc_port = 9000
+    grpc_servicer_functions = [
+        "ray.serve.generated.serve_pb2_grpc."
+        "add_RayServeBenchmarkServiceServicer_to_server",
+    ]
+    serve.start(
+        grpc_options=gRPCOptions(
+            port=grpc_port,
+            grpc_servicer_functions=grpc_servicer_functions,
+        )
+    )
+    loop = asyncio.new_event_loop()
+    asyncio.set_event_loop(loop)
+    loop.run_until_complete(main())

python/ray/serve/_private/benchmarks/serialization/common.py

@@ -0,0 +1,29 @@
+from dataclasses import dataclass
+from typing import List, Optional
+
+from pydantic import BaseModel
+
+#
+# NOTE: PLEASE READ CAREFULLY BEFORE CHANGING
+#
+# Payloads in this module are purposefully extracted from benchmark file to force
+# Ray's cloudpickle behavior when it does NOT serialize the class definition itself
+# along with its payload (instead relying on it being imported)
+#
+
+
+class PayloadPydantic(BaseModel):
+    text: Optional[str] = None
+    floats: Optional[List[float]] = None
+    ints: Optional[List[int]] = None
+    ts: Optional[float] = None
+    reason: Optional[str] = None
+
+
+@dataclass
+class PayloadDataclass:
+    text: Optional[str] = None
+    floats: Optional[List[float]] = None
+    ints: Optional[List[int]] = None
+    ts: Optional[float] = None
+    reason: Optional[str] = None

python/ray/serve/_private/benchmarks/streaming/_grpc/grpc_server.py

@@ -0,0 +1,60 @@
+import grpc
+
+from ray.serve._private.benchmarks.streaming._grpc import (
+    test_server_pb2,
+    test_server_pb2_grpc,
+)
+
+
+async def _async_list(async_iterator):
+    items = []
+    async for item in async_iterator:
+        items.append(item)
+
+    return items
+
+
+class TestGRPCServer(test_server_pb2_grpc.GRPCTestServerServicer):
+    def __init__(self, tokens_per_request):
+        self._tokens_per_request = tokens_per_request
+
+    async def Unary(self, request, context):
+        if request.request_data == "error":
+            await context.abort(
+                code=grpc.StatusCode.INTERNAL,
+                details="unary rpc error",
+            )
+
+        return test_server_pb2.Response(response_data="OK")
+
+    async def ClientStreaming(self, request_iterator, context):
+        data = await _async_list(request_iterator)
+
+        if data and data[0].request_data == "error":
+            await context.abort(
+                code=grpc.StatusCode.INTERNAL,
+                details="client streaming rpc error",
+            )
+
+        return test_server_pb2.Response(response_data="OK")
+
+    async def ServerStreaming(self, request, context):
+        if request.request_data == "error":
+            await context.abort(
+                code=grpc.StatusCode.INTERNAL,
+                details="OK",
+            )
+
+        for i in range(self._tokens_per_request):
+            yield test_server_pb2.Response(response_data="OK")
+
+    async def BidiStreaming(self, request_iterator, context):
+        data = await _async_list(request_iterator)
+        if data and data[0].request_data == "error":
+            await context.abort(
+                code=grpc.StatusCode.INTERNAL,
+                details="bidi-streaming rpc error",
+            )
+
+        for i in range(self._tokens_per_request):
+            yield test_server_pb2.Response(response_data="OK")

python/ray/serve/_private/benchmarks/streaming/_grpc/test_server.proto

@@ -0,0 +1,16 @@
+syntax = "proto3";
+
+message Request {
+  string request_data = 2;
+}
+
+message Response {
+  string response_data = 2;
+}
+
+service GRPCTestServer {
+  rpc Unary(Request) returns (Response);
+  rpc ClientStreaming(stream Request) returns (Response);
+  rpc ServerStreaming(Request) returns (stream Response);
+  rpc BidiStreaming(stream Request) returns (stream Response);
+}

python/ray/serve/_private/benchmarks/streaming/_grpc/test_server_pb2_grpc.py

@@ -0,0 +1,216 @@
+# Generated by the gRPC Python protocol compiler plugin. DO NOT EDIT!
+"""Client and server classes corresponding to protobuf-defined services."""
+import grpc
+
+from ray.serve._private.benchmarks.streaming._grpc import (
+    test_server_pb2 as backend_dot_server_dot_common_dot_clients_dot_grpc_dot_proto_dot_test__server__pb2,
+)
+
+
+class GRPCTestServerStub(object):
+    """Missing associated documentation comment in .proto file."""
+
+    def __init__(self, channel: grpc.Channel):
+        """Constructor.
+
+        Args:
+            channel: A grpc.Channel.
+        """
+        self.Unary = channel.unary_unary(
+            "/GRPCTestServer/Unary",
+            request_serializer=backend_dot_server_dot_common_dot_clients_dot_grpc_dot_proto_dot_test__server__pb2.Request.SerializeToString,
+            response_deserializer=backend_dot_server_dot_common_dot_clients_dot_grpc_dot_proto_dot_test__server__pb2.Response.FromString,
+        )
+        self.ClientStreaming = channel.stream_unary(
+            "/GRPCTestServer/ClientStreaming",
+            request_serializer=backend_dot_server_dot_common_dot_clients_dot_grpc_dot_proto_dot_test__server__pb2.Request.SerializeToString,
+            response_deserializer=backend_dot_server_dot_common_dot_clients_dot_grpc_dot_proto_dot_test__server__pb2.Response.FromString,
+        )
+        self.ServerStreaming = channel.unary_stream(
+            "/GRPCTestServer/ServerStreaming",
+            request_serializer=backend_dot_server_dot_common_dot_clients_dot_grpc_dot_proto_dot_test__server__pb2.Request.SerializeToString,
+            response_deserializer=backend_dot_server_dot_common_dot_clients_dot_grpc_dot_proto_dot_test__server__pb2.Response.FromString,
+        )
+        self.BidiStreaming = channel.stream_stream(
+            "/GRPCTestServer/BidiStreaming",
+            request_serializer=backend_dot_server_dot_common_dot_clients_dot_grpc_dot_proto_dot_test__server__pb2.Request.SerializeToString,
+            response_deserializer=backend_dot_server_dot_common_dot_clients_dot_grpc_dot_proto_dot_test__server__pb2.Response.FromString,
+        )
+
+
+class GRPCTestServerServicer(object):
+    """Missing associated documentation comment in .proto file."""
+
+    def Unary(self, request, context):
+        """Missing associated documentation comment in .proto file."""
+        context.set_code(grpc.StatusCode.UNIMPLEMENTED)
+        context.set_details("Method not implemented!")
+        raise NotImplementedError("Method not implemented!")
+
+    def ClientStreaming(self, request_iterator, context):
+        """Missing associated documentation comment in .proto file."""
+        context.set_code(grpc.StatusCode.UNIMPLEMENTED)
+        context.set_details("Method not implemented!")
+        raise NotImplementedError("Method not implemented!")
+
+    def ServerStreaming(self, request, context):
+        """Missing associated documentation comment in .proto file."""
+        context.set_code(grpc.StatusCode.UNIMPLEMENTED)
+        context.set_details("Method not implemented!")
+        raise NotImplementedError("Method not implemented!")
+
+    def BidiStreaming(self, request_iterator, context):
+        """Missing associated documentation comment in .proto file."""
+        context.set_code(grpc.StatusCode.UNIMPLEMENTED)
+        context.set_details("Method not implemented!")
+        raise NotImplementedError("Method not implemented!")
+
+
+def add_GRPCTestServerServicer_to_server(servicer, server):
+    rpc_method_handlers = {
+        "Unary": grpc.unary_unary_rpc_method_handler(
+            servicer.Unary,
+            request_deserializer=backend_dot_server_dot_common_dot_clients_dot_grpc_dot_proto_dot_test__server__pb2.Request.FromString,
+            response_serializer=backend_dot_server_dot_common_dot_clients_dot_grpc_dot_proto_dot_test__server__pb2.Response.SerializeToString,
+        ),
+        "ClientStreaming": grpc.stream_unary_rpc_method_handler(
+            servicer.ClientStreaming,
+            request_deserializer=backend_dot_server_dot_common_dot_clients_dot_grpc_dot_proto_dot_test__server__pb2.Request.FromString,
+            response_serializer=backend_dot_server_dot_common_dot_clients_dot_grpc_dot_proto_dot_test__server__pb2.Response.SerializeToString,
+        ),
+        "ServerStreaming": grpc.unary_stream_rpc_method_handler(
+            servicer.ServerStreaming,
+            request_deserializer=backend_dot_server_dot_common_dot_clients_dot_grpc_dot_proto_dot_test__server__pb2.Request.FromString,
+            response_serializer=backend_dot_server_dot_common_dot_clients_dot_grpc_dot_proto_dot_test__server__pb2.Response.SerializeToString,
+        ),
+        "BidiStreaming": grpc.stream_stream_rpc_method_handler(
+            servicer.BidiStreaming,
+            request_deserializer=backend_dot_server_dot_common_dot_clients_dot_grpc_dot_proto_dot_test__server__pb2.Request.FromString,
+            response_serializer=backend_dot_server_dot_common_dot_clients_dot_grpc_dot_proto_dot_test__server__pb2.Response.SerializeToString,
+        ),
+    }
+    generic_handler = grpc.method_handlers_generic_handler(
+        "GRPCTestServer", rpc_method_handlers
+    )
+    server.add_generic_rpc_handlers((generic_handler,))
+
+
+# This class is part of an EXPERIMENTAL API.
+class GRPCTestServer(object):
+    """Missing associated documentation comment in .proto file."""
+
+    @staticmethod
+    def Unary(
+        request,
+        target,
+        options=(),
+        channel_credentials=None,
+        call_credentials=None,
+        insecure=False,
+        compression=None,
+        wait_for_ready=None,
+        timeout=None,
+        metadata=None,
+    ):
+        return grpc.experimental.unary_unary(
+            request,
+            target,
+            "/GRPCTestServer/Unary",
+            backend_dot_server_dot_common_dot_clients_dot_grpc_dot_proto_dot_test__server__pb2.Request.SerializeToString,
+            backend_dot_server_dot_common_dot_clients_dot_grpc_dot_proto_dot_test__server__pb2.Response.FromString,
+            options,
+            channel_credentials,
+            insecure,
+            call_credentials,
+            compression,
+            wait_for_ready,
+            timeout,
+            metadata,
+        )
+
+    @staticmethod
+    def ClientStreaming(
+        request_iterator,
+        target,
+        options=(),
+        channel_credentials=None,
+        call_credentials=None,
+        insecure=False,
+        compression=None,
+        wait_for_ready=None,
+        timeout=None,
+        metadata=None,
+    ):
+        return grpc.experimental.stream_unary(
+            request_iterator,
+            target,
+            "/GRPCTestServer/ClientStreaming",
+            backend_dot_server_dot_common_dot_clients_dot_grpc_dot_proto_dot_test__server__pb2.Request.SerializeToString,
+            backend_dot_server_dot_common_dot_clients_dot_grpc_dot_proto_dot_test__server__pb2.Response.FromString,
+            options,
+            channel_credentials,
+            insecure,
+            call_credentials,
+            compression,
+            wait_for_ready,
+            timeout,
+            metadata,
+        )
+
+    @staticmethod
+    def ServerStreaming(
+        request,
+        target,
+        options=(),
+        channel_credentials=None,
+        call_credentials=None,
+        insecure=False,
+        compression=None,
+        wait_for_ready=None,
+        timeout=None,
+        metadata=None,
+    ):
+        return grpc.experimental.unary_stream(
+            request,
+            target,
+            "/GRPCTestServer/ServerStreaming",
+            backend_dot_server_dot_common_dot_clients_dot_grpc_dot_proto_dot_test__server__pb2.Request.SerializeToString,
+            backend_dot_server_dot_common_dot_clients_dot_grpc_dot_proto_dot_test__server__pb2.Response.FromString,
+            options,
+            channel_credentials,
+            insecure,
+            call_credentials,
+            compression,
+            wait_for_ready,
+            timeout,
+            metadata,
+        )
+
+    @staticmethod
+    def BidiStreaming(
+        request_iterator,
+        target,
+        options=(),
+        channel_credentials=None,
+        call_credentials=None,
+        insecure=False,
+        compression=None,
+        wait_for_ready=None,
+        timeout=None,
+        metadata=None,
+    ):
+        return grpc.experimental.stream_stream(
+            request_iterator,
+            target,
+            "/GRPCTestServer/BidiStreaming",
+            backend_dot_server_dot_common_dot_clients_dot_grpc_dot_proto_dot_test__server__pb2.Request.SerializeToString,
+            backend_dot_server_dot_common_dot_clients_dot_grpc_dot_proto_dot_test__server__pb2.Response.FromString,
+            options,
+            channel_credentials,
+            insecure,
+            call_credentials,
+            compression,
+            wait_for_ready,
+            timeout,
+            metadata,
+        )

python/ray/serve/_private/benchmarks/streaming/common.py

@@ -0,0 +1,123 @@
+import abc
+import asyncio
+import enum
+import logging
+import time
+from typing import Tuple, Union
+
+import numpy as np
+
+from ray.actor import ActorHandle
+from ray.runtime_env import RuntimeEnv
+from ray.serve._private.benchmarks.common import Blackhole, run_throughput_benchmark
+from ray.serve._private.benchmarks.serialization.common import PayloadPydantic
+from ray.serve.handle import DeploymentHandle
+
+GRPC_DEBUG_RUNTIME_ENV = RuntimeEnv(
+    env_vars={"GRPC_TRACE": "http", "GRPC_VERBOSITY": "debug"},
+)
+
+
+class IOMode(enum.Enum):
+    SYNC = "SYNC"
+    ASYNC = "ASYNC"
+
+
+class Endpoint:
+    def __init__(self, tokens_per_request: int):
+        self._tokens_per_request = tokens_per_request
+        # Switch off logging to minimize its impact
+        logging.getLogger("ray").setLevel(logging.WARNING)
+        logging.getLogger("ray.serve").setLevel(logging.WARNING)
+
+    def stream(self):
+        payload = PayloadPydantic(
+            text="Test output",
+            floats=[float(f) for f in range(1, 100)],
+            ints=list(range(1, 100)),
+            ts=time.time(),
+            reason="Success!",
+        )
+
+        for i in range(self._tokens_per_request):
+            yield payload
+
+    async def aio_stream(self):
+        payload = PayloadPydantic(
+            text="Test output",
+            floats=[float(f) for f in range(1, 100)],
+            ints=list(range(1, 100)),
+            ts=time.time(),
+            reason="Success!",
+        )
+
+        for i in range(self._tokens_per_request):
+            yield payload
+
+
+class Caller(Blackhole):
+    def __init__(
+        self,
+        downstream: Union[ActorHandle, DeploymentHandle],
+        *,
+        mode: IOMode,
+        tokens_per_request: int,
+        batch_size: int,
+        num_trials: int,
+        trial_runtime: float,
+    ):
+        self._h = downstream
+        self._mode = mode
+        self._tokens_per_request = tokens_per_request
+        self._batch_size = batch_size
+        self._num_trials = num_trials
+        self._trial_runtime = trial_runtime
+        self._durations = []
+
+        # Switch off logging to minimize its impact
+        logging.getLogger("ray").setLevel(logging.WARNING)
+        logging.getLogger("ray.serve").setLevel(logging.WARNING)
+
+    def _get_remote_method(self):
+        if self._mode == IOMode.SYNC:
+            return self._h.stream
+        elif self._mode == IOMode.ASYNC:
+            return self._h.aio_stream
+        else:
+            raise NotImplementedError(f"Streaming mode not supported ({self._mode})")
+
+    @abc.abstractmethod
+    async def _consume_single_stream(self):
+        pass
+
+    async def _do_single_batch(self):
+        durations = await asyncio.gather(
+            *[
+                self._execute(self._consume_single_stream)
+                for _ in range(self._batch_size)
+            ]
+        )
+
+        self._durations.extend(durations)
+
+    async def _execute(self, fn):
+        start = time.monotonic()
+        await fn()
+        dur_s = time.monotonic() - start
+        return dur_s * 1000  # ms
+
+    async def run_benchmark(self) -> Tuple[float, float]:
+        coro = run_throughput_benchmark(
+            fn=self._do_single_batch,
+            multiplier=self._batch_size * self._tokens_per_request,
+            num_trials=self._num_trials,
+            trial_runtime=self._trial_runtime,
+        )
+        # total_runtime = await collect_profile_events(coro)
+        total_runtime = await coro
+
+        p50, p75, p99 = np.percentile(self._durations, [50, 75, 99])
+
+        print(f"Individual request quantiles:\n\tP50={p50}\n\tP75={p75}\n\tP99={p99}")
+
+        return total_runtime

python/ray/serve/_private/benchmarks/streaming/streaming_core_throughput.py

@@ -0,0 +1,95 @@
+import click
+
+import ray
+from ray.serve._private.benchmarks.streaming.common import Caller, Endpoint, IOMode
+
+
+# @ray.remote(runtime_env=GRPC_DEBUG_RUNTIME_ENV)
+@ray.remote
+class EndpointActor(Endpoint):
+    pass
+
+
+# @ray.remote(runtime_env=GRPC_DEBUG_RUNTIME_ENV)
+@ray.remote
+class CallerActor(Caller):
+    async def _consume_single_stream(self):
+        method = self._get_remote_method()
+        async for ref in method.options(num_returns="streaming").remote():
+            r = ray.get(ref)
+
+            # self.sink(str(r, 'utf-8'))
+            self.sink(r)
+
+
+@click.command(help="Benchmark streaming deployment handle throughput.")
+@click.option(
+    "--tokens-per-request",
+    type=int,
+    default=1000,
+    help="Number of tokens (per request) to stream from downstream deployment",
+)
+@click.option(
+    "--batch-size",
+    type=int,
+    default=10,
+    help="Number of requests to send to downstream deployment in each batch.",
+)
+@click.option(
+    "--num-replicas",
+    type=int,
+    default=1,
+    help="Number of replicas in the downstream deployment.",
+)
+@click.option(
+    "--num-trials",
+    type=int,
+    default=5,
+    help="Number of trials of the benchmark to run.",
+)
+@click.option(
+    "--trial-runtime",
+    type=int,
+    default=5,
+    help="Duration to run each trial of the benchmark for (seconds).",
+)
+@click.option(
+    "--io-mode",
+    type=str,
+    default="async",
+    help="Controls mode of the streaming generation (either 'sync' or 'async')",
+)
+def main(
+    tokens_per_request: int,
+    batch_size: int,
+    num_replicas: int,
+    num_trials: int,
+    trial_runtime: float,
+    io_mode: str,
+):
+    h = CallerActor.remote(
+        EndpointActor.remote(
+            tokens_per_request=tokens_per_request,
+        ),
+        mode=IOMode(io_mode.upper()),
+        tokens_per_request=tokens_per_request,
+        batch_size=batch_size,
+        num_trials=num_trials,
+        trial_runtime=trial_runtime,
+    )
+
+    mean, stddev = ray.get(h.run_benchmark.remote())
+    print(
+        "Core Actors streaming throughput ({}) {}: {} +- {} tokens/s".format(
+            io_mode.upper(),
+            f"(num_replicas={num_replicas}, "
+            f"tokens_per_request={tokens_per_request}, "
+            f"batch_size={batch_size})",
+            mean,
+            stddev,
+        )
+    )
+
+
+if __name__ == "__main__":
+    main()

python/ray/serve/_private/benchmarks/streaming/streaming_grpc_throughput.py

@@ -0,0 +1,218 @@
+import asyncio
+import logging
+import time
+from concurrent import futures
+from tempfile import TemporaryDirectory
+
+import click
+import grpc
+
+import ray
+from ray.serve._private.benchmarks.streaming._grpc import (
+    test_server_pb2,
+    test_server_pb2_grpc,
+)
+from ray.serve._private.benchmarks.streaming._grpc.grpc_server import TestGRPCServer
+from ray.serve._private.benchmarks.streaming.common import Caller, IOMode
+
+
+# @ray.remote(runtime_env=GRPC_DEBUG_RUNTIME_ENV)
+@ray.remote
+class EndpointActor:
+    async def __init__(self, tokens_per_request, socket_type, tempdir):
+        # Switch off logging to minimize its impact
+        logging.getLogger("ray").setLevel(logging.WARNING)
+        logging.getLogger("ray.serve").setLevel(logging.WARNING)
+
+        self.server = await self.start_server(tokens_per_request, socket_type, tempdir)
+
+        print("gRPC server started!")
+
+    @staticmethod
+    async def start_server(tokens_per_request, socket_type, tempdir):
+        server = grpc.aio.server(futures.ThreadPoolExecutor(max_workers=1))
+
+        addr, server_creds, _ = _gen_addr_creds(socket_type, tempdir)
+
+        server.add_secure_port(addr, server_creds)
+
+        await server.start()
+
+        test_server_pb2_grpc.add_GRPCTestServerServicer_to_server(
+            TestGRPCServer(tokens_per_request), server
+        )
+
+        return server
+
+
+# @ray.remote(runtime_env=GRPC_DEBUG_RUNTIME_ENV)
+@ray.remote
+class GrpcCallerActor(Caller):
+    def __init__(
+        self,
+        tempdir,
+        socket_type,
+        *,
+        mode: IOMode,
+        tokens_per_request: int,
+        batch_size: int,
+        num_trials: int,
+        trial_runtime: float,
+    ):
+        super().__init__(
+            self.create_downstream(socket_type, tempdir),
+            mode=mode,
+            tokens_per_request=tokens_per_request,
+            batch_size=batch_size,
+            num_trials=num_trials,
+            trial_runtime=trial_runtime,
+        )
+
+    @staticmethod
+    def create_downstream(socket_type, tempdir):
+        addr, _, channel_creds = _gen_addr_creds(socket_type, tempdir)
+
+        channel = grpc.aio.secure_channel(
+            addr, credentials=channel_creds, interceptors=[]
+        )
+
+        return test_server_pb2_grpc.GRPCTestServerStub(channel)
+
+    async def _consume_single_stream(self):
+        try:
+            async for r in self._h.ServerStreaming(test_server_pb2.Request()):
+                self.sink(r)
+        except Exception as e:
+            print(str(e))
+
+
+def _gen_addr_creds(socket_type, tempdir):
+    if socket_type == "uds":
+        addr = f"unix://{tempdir}/server.sock"
+        server_creds = grpc.local_server_credentials(grpc.LocalConnectionType.UDS)
+        channel_creds = grpc.local_channel_credentials(grpc.LocalConnectionType.UDS)
+    elif socket_type == "local_tcp":
+        addr = "127.0.0.1:5432"
+        server_creds = grpc.local_server_credentials(grpc.LocalConnectionType.LOCAL_TCP)
+        channel_creds = grpc.local_channel_credentials(
+            grpc.LocalConnectionType.LOCAL_TCP
+        )
+    else:
+        raise NotImplementedError(f"Not supported socket type ({socket_type})")
+
+    return addr, server_creds, channel_creds
+
+
+async def run_grpc_benchmark(
+    batch_size,
+    io_mode,
+    socket_type,
+    num_replicas,
+    num_trials,
+    tokens_per_request,
+    trial_runtime,
+):
+    with TemporaryDirectory() as tempdir:
+        _ = EndpointActor.remote(
+            tokens_per_request=tokens_per_request,
+            socket_type=socket_type,
+            tempdir=tempdir,
+        )
+
+        ca = GrpcCallerActor.remote(
+            tempdir,
+            socket_type,
+            mode=IOMode(io_mode.upper()),
+            tokens_per_request=tokens_per_request,
+            batch_size=batch_size,
+            num_trials=num_trials,
+            trial_runtime=trial_runtime,
+        )
+
+        # TODO make starting server a method (to make synchronization explicit)
+        time.sleep(5)
+
+        mean, stddev = await ca.run_benchmark.remote()
+
+        print(
+            "gRPC streaming throughput ({}) {}: {} +- {} tokens/s".format(
+                io_mode.upper(),
+                f"(num_replicas={num_replicas}, "
+                f"tokens_per_request={tokens_per_request}, "
+                f"batch_size={batch_size})",
+                mean,
+                stddev,
+            )
+        )
+
+
+@click.command(help="Benchmark streaming deployment handle throughput.")
+@click.option(
+    "--tokens-per-request",
+    type=int,
+    default=1000,
+    help="Number of tokens (per request) to stream from downstream deployment",
+)
+@click.option(
+    "--batch-size",
+    type=int,
+    default=10,
+    help="Number of requests to send to downstream deployment in each batch.",
+)
+@click.option(
+    "--num-replicas",
+    type=int,
+    default=1,
+    help="Number of replicas in the downstream deployment.",
+)
+@click.option(
+    "--num-trials",
+    type=int,
+    default=5,
+    help="Number of trials of the benchmark to run.",
+)
+@click.option(
+    "--trial-runtime",
+    type=int,
+    default=5,
+    help="Duration to run each trial of the benchmark for (seconds).",
+)
+@click.option(
+    "--io-mode",
+    type=str,
+    default="async",
+    help="Controls mode of the streaming generation (either 'sync' or 'async')",
+)
+@click.option(
+    "--socket-type",
+    type=str,
+    default="local_tcp",
+    help="Controls type of socket used as underlying transport (either 'uds' or "
+    "'local_tcp')",
+)
+def main(
+    tokens_per_request: int,
+    batch_size: int,
+    num_replicas: int,
+    num_trials: int,
+    trial_runtime: float,
+    io_mode: str,
+    socket_type: grpc.LocalConnectionType,
+):
+    """Reference benchmark for vanilla Python (w/ C-based core) gRPC implementation"""
+
+    asyncio.run(
+        run_grpc_benchmark(
+            batch_size,
+            io_mode,
+            socket_type,
+            num_replicas,
+            num_trials,
+            tokens_per_request,
+            trial_runtime,
+        )
+    )
+
+
+if __name__ == "__main__":
+    main()

python/ray/serve/_private/benchmarks/streaming/streaming_handle_throughput.py

@@ -0,0 +1,94 @@
+import click
+
+from ray import serve
+from ray.serve._private.benchmarks.streaming.common import Caller, Endpoint, IOMode
+
+
+@serve.deployment(ray_actor_options={"num_cpus": 0})
+class EndpointDeployment(Endpoint):
+    pass
+
+
+@serve.deployment
+class CallerDeployment(Caller):
+    async def _consume_single_stream(self):
+        method = self._get_remote_method().options(
+            stream=True,
+        )
+
+        async for r in method.remote():
+            # Blackhole the response
+            # self.sink(str(r, 'utf-8'))
+            self.sink(r)
+
+
+@click.command(help="Benchmark streaming deployment handle throughput.")
+@click.option(
+    "--tokens-per-request",
+    type=int,
+    default=1000,
+    help="Number of requests to send to downstream deployment in each trial.",
+)
+@click.option(
+    "--batch-size",
+    type=int,
+    default=10,
+    help="Number of requests to send to downstream deployment in each trial.",
+)
+@click.option(
+    "--num-replicas",
+    type=int,
+    default=1,
+    help="Number of replicas in the downstream deployment.",
+)
+@click.option(
+    "--num-trials",
+    type=int,
+    default=5,
+    help="Number of trials of the benchmark to run.",
+)
+@click.option(
+    "--trial-runtime",
+    type=int,
+    default=1,
+    help="Duration to run each trial of the benchmark for (seconds).",
+)
+@click.option(
+    "--io-mode",
+    type=str,
+    default="async",
+    help="Controls mode of the streaming generation (either 'sync' or 'async')",
+)
+def main(
+    tokens_per_request: int,
+    batch_size: int,
+    num_replicas: int,
+    num_trials: int,
+    trial_runtime: float,
+    io_mode: str,
+):
+    app = CallerDeployment.bind(
+        EndpointDeployment.options(num_replicas=num_replicas).bind(tokens_per_request),
+        mode=IOMode(io_mode.upper()),
+        tokens_per_request=tokens_per_request,
+        batch_size=batch_size,
+        num_trials=num_trials,
+        trial_runtime=trial_runtime,
+    )
+    h = serve.run(app)
+
+    mean, stddev = h.run_benchmark.remote().result()
+    print(
+        "DeploymentHandle streaming throughput ({}) {}: {} +- {} tokens/s".format(
+            io_mode.upper(),
+            f"(num_replicas={num_replicas}, "
+            f"tokens_per_request={tokens_per_request}, "
+            f"batch_size={batch_size})",
+            mean,
+            stddev,
+        )
+    )
+
+
+if __name__ == "__main__":
+    main()

python/ray/serve/_private/benchmarks/streaming/streaming_http_throughput.py

@@ -0,0 +1,140 @@
+import asyncio
+import logging
+from typing import Tuple
+
+import aiohttp
+import click
+from starlette.responses import StreamingResponse
+
+from ray import serve
+from ray.serve._private.benchmarks.common import run_throughput_benchmark
+from ray.serve.handle import DeploymentHandle
+
+
+@serve.deployment(ray_actor_options={"num_cpus": 0})
+class Downstream:
+    def __init__(self, tokens_per_request: int):
+        logging.getLogger("ray.serve").setLevel(logging.WARNING)
+
+        self._tokens_per_request = tokens_per_request
+
+    async def stream(self):
+        for i in range(self._tokens_per_request):
+            yield "hi"
+
+    def __call__(self, *args):
+        return StreamingResponse(self.stream())
+
+
+@serve.deployment(ray_actor_options={"num_cpus": 0})
+class Intermediate:
+    def __init__(self, downstream: DeploymentHandle):
+        logging.getLogger("ray.serve").setLevel(logging.WARNING)
+
+        self._h = downstream.options(stream=True)
+
+    async def stream(self):
+        async for token in self._h.stream.remote():
+            yield token
+
+    def __call__(self, *args):
+        return StreamingResponse(self.stream())
+
+
+async def _consume_single_stream():
+    async with aiohttp.ClientSession(raise_for_status=True) as session:
+        async with session.get("http://localhost:8000") as r:
+            async for line in r.content:
+                pass
+
+
+async def run_benchmark(
+    tokens_per_request: int,
+    batch_size: int,
+    num_trials: int,
+    trial_runtime: float,
+) -> Tuple[float, float]:
+    async def _do_single_batch():
+        await asyncio.gather(*[_consume_single_stream() for _ in range(batch_size)])
+
+    return await run_throughput_benchmark(
+        fn=_do_single_batch,
+        multiplier=batch_size * tokens_per_request,
+        num_trials=num_trials,
+        trial_runtime=trial_runtime,
+    )
+
+
+@click.command(help="Benchmark streaming HTTP throughput.")
+@click.option(
+    "--tokens-per-request",
+    type=int,
+    default=1000,
+    help="Number of requests to send to downstream deployment in each trial.",
+)
+@click.option(
+    "--batch-size",
+    type=int,
+    default=10,
+    help="Number of requests to send to downstream deployment in each trial.",
+)
+@click.option(
+    "--num-replicas",
+    type=int,
+    default=1,
+    help="Number of replicas in the downstream deployment.",
+)
+@click.option(
+    "--num-trials",
+    type=int,
+    default=5,
+    help="Number of trials of the benchmark to run.",
+)
+@click.option(
+    "--trial-runtime",
+    type=int,
+    default=1,
+    help="Duration to run each trial of the benchmark for (seconds).",
+)
+@click.option(
+    "--use-intermediate-deployment",
+    is_flag=True,
+    default=False,
+    help="Whether to run an intermediate deployment proxying the requests.",
+)
+def main(
+    tokens_per_request: int,
+    batch_size: int,
+    num_replicas: int,
+    num_trials: int,
+    trial_runtime: float,
+    use_intermediate_deployment: bool,
+):
+    app = Downstream.options(num_replicas=num_replicas).bind(tokens_per_request)
+    if use_intermediate_deployment:
+        app = Intermediate.bind(app)
+
+    serve.run(app)
+
+    mean, stddev = asyncio.new_event_loop().run_until_complete(
+        run_benchmark(
+            tokens_per_request,
+            batch_size,
+            num_trials,
+            trial_runtime,
+        )
+    )
+    print(
+        "HTTP streaming throughput {}: {} +- {} tokens/s".format(
+            f"(num_replicas={num_replicas}, "
+            f"tokens_per_request={tokens_per_request}, "
+            f"batch_size={batch_size}, "
+            f"use_intermediate_deployment={use_intermediate_deployment})",
+            mean,
+            stddev,
+        )
+    )
+
+
+if __name__ == "__main__":
+    main()

python/ray/serve/_private/broker.py

@@ -0,0 +1,285 @@
+# This module provides broker clients for querying queue lengths from message brokers.
+# Adapted from Flower's broker.py (https://github.com/mher/flower/blob/master/flower/utils/broker.py)
+# with the following modification:
+# - Added close() method to BrokerBase and RedisBase for resource cleanup
+
+import json
+import logging
+import numbers
+import socket
+from urllib.parse import quote, unquote, urljoin, urlparse
+
+from tornado import httpclient, ioloop
+
+from ray.serve._private.constants import SERVE_LOGGER_NAME
+
+try:
+    import redis
+except ImportError:
+    redis = None
+
+
+logger = logging.getLogger(SERVE_LOGGER_NAME)
+
+
+class BrokerBase:
+    def __init__(self, broker_url, *_, **__):
+        purl = urlparse(broker_url)
+        self.host = purl.hostname
+        self.port = purl.port
+        self.vhost = purl.path[1:]
+
+        username = purl.username
+        password = purl.password
+
+        self.username = unquote(username) if username else username
+        self.password = unquote(password) if password else password
+
+    async def queues(self, names):
+        raise NotImplementedError
+
+    def close(self):
+        """Close any open connections. Override in subclasses as needed."""
+        pass
+
+
+class RabbitMQ(BrokerBase):
+    def __init__(self, broker_url, http_api, io_loop=None, **__):
+        super().__init__(broker_url)
+        self.io_loop = io_loop or ioloop.IOLoop.instance()
+
+        self.host = self.host or "localhost"
+        self.port = self.port or 15672
+        self.vhost = quote(self.vhost, "") or "/" if self.vhost != "/" else self.vhost
+        self.username = self.username or "guest"
+        self.password = self.password or "guest"
+
+        if not http_api:
+            http_api = f"http://{self.username}:{self.password}@{self.host}:{self.port}/api/{self.vhost}"
+
+        try:
+            self.validate_http_api(http_api)
+        except ValueError:
+            logger.error("Invalid broker api url: %s", http_api)
+
+        self.http_api = http_api
+
+    async def queues(self, names):
+        url = urljoin(self.http_api, "queues/" + self.vhost)
+        api_url = urlparse(self.http_api)
+        username = unquote(api_url.username or "") or self.username
+        password = unquote(api_url.password or "") or self.password
+
+        http_client = httpclient.AsyncHTTPClient()
+        try:
+            response = await http_client.fetch(
+                url,
+                auth_username=username,
+                auth_password=password,
+                connect_timeout=1.0,
+                request_timeout=2.0,
+                validate_cert=False,
+            )
+        except (socket.error, httpclient.HTTPError) as e:
+            logger.error("RabbitMQ management API call failed: %s", e)
+            return []
+        finally:
+            http_client.close()
+
+        if response.code == 200:
+            info = json.loads(response.body.decode())
+            return [x for x in info if x["name"] in names]
+        response.rethrow()
+
+    @classmethod
+    def validate_http_api(cls, http_api):
+        url = urlparse(http_api)
+        if url.scheme not in ("http", "https"):
+            raise ValueError(f"Invalid http api schema: {url.scheme}")
+
+
+class RedisBase(BrokerBase):
+    DEFAULT_SEP = "\x06\x16"
+    DEFAULT_PRIORITY_STEPS = [0, 3, 6, 9]
+
+    def __init__(self, broker_url, *_, **kwargs):
+        super().__init__(broker_url)
+        self.redis = None
+
+        if not redis:
+            raise ImportError("redis library is required")
+
+        broker_options = kwargs.get("broker_options", {})
+        self.priority_steps = broker_options.get(
+            "priority_steps", self.DEFAULT_PRIORITY_STEPS
+        )
+        self.sep = broker_options.get("sep", self.DEFAULT_SEP)
+        self.broker_prefix = broker_options.get("global_keyprefix", "")
+
+    def _q_for_pri(self, queue, pri):
+        if pri not in self.priority_steps:
+            raise ValueError("Priority not in priority steps")
+        # pylint: disable=consider-using-f-string
+        return "{0}{1}{2}".format(*((queue, self.sep, pri) if pri else (queue, "", "")))
+
+    async def queues(self, names):
+        queue_stats = []
+        for name in names:
+            priority_names = [
+                self.broker_prefix + self._q_for_pri(name, pri)
+                for pri in self.priority_steps
+            ]
+            queue_stats.append(
+                {
+                    "name": name,
+                    "messages": sum((self.redis.llen(x) for x in priority_names)),
+                }
+            )
+        return queue_stats
+
+    def close(self):
+        """Close the Redis connection."""
+        if self.redis is not None:
+            self.redis.close()
+            self.redis = None
+
+
+class Redis(RedisBase):
+    def __init__(self, broker_url, *args, **kwargs):
+        super().__init__(broker_url, *args, **kwargs)
+        self.host = self.host or "localhost"
+        self.port = self.port or 6379
+        self.vhost = self._prepare_virtual_host(self.vhost)
+        self.redis = self._get_redis_client()
+
+    def _prepare_virtual_host(self, vhost):
+        if not isinstance(vhost, numbers.Integral):
+            if not vhost or vhost == "/":
+                vhost = 0
+            elif vhost.startswith("/"):
+                vhost = vhost[1:]
+            try:
+                vhost = int(vhost)
+            except ValueError as exc:
+                raise ValueError(
+                    f"Database is int between 0 and limit - 1, not {vhost}"
+                ) from exc
+        return vhost
+
+    def _get_redis_client_args(self):
+        return {
+            "host": self.host,
+            "port": self.port,
+            "db": self.vhost,
+            "username": self.username,
+            "password": self.password,
+        }
+
+    def _get_redis_client(self):
+        return redis.Redis(**self._get_redis_client_args())
+
+
+class RedisSentinel(RedisBase):
+    def __init__(self, broker_url, *args, **kwargs):
+        super().__init__(broker_url, *args, **kwargs)
+        broker_options = kwargs.get("broker_options", {})
+        broker_use_ssl = kwargs.get("broker_use_ssl", None)
+        self.host = self.host or "localhost"
+        self.port = self.port or 26379
+        self.vhost = self._prepare_virtual_host(self.vhost)
+        self.master_name = self._prepare_master_name(broker_options)
+        self.redis = self._get_redis_client(broker_options, broker_use_ssl)
+
+    def _prepare_virtual_host(self, vhost):
+        if not isinstance(vhost, numbers.Integral):
+            if not vhost or vhost == "/":
+                vhost = 0
+            elif vhost.startswith("/"):
+                vhost = vhost[1:]
+            try:
+                vhost = int(vhost)
+            except ValueError as exc:
+                raise ValueError(
+                    f"Database is int between 0 and limit - 1, not {vhost}"
+                ) from exc
+        return vhost
+
+    def _prepare_master_name(self, broker_options):
+        try:
+            master_name = broker_options["master_name"]
+        except KeyError as exc:
+            raise ValueError("master_name is required for Sentinel broker") from exc
+        return master_name
+
+    def _get_redis_client(self, broker_options, broker_use_ssl):
+        connection_kwargs = {
+            "password": self.password,
+            "sentinel_kwargs": broker_options.get("sentinel_kwargs"),
+        }
+        if isinstance(broker_use_ssl, dict):
+            connection_kwargs["ssl"] = True
+            connection_kwargs.update(broker_use_ssl)
+        # get all sentinel hosts from Celery App config and use them to initialize Sentinel
+        sentinel = redis.sentinel.Sentinel(
+            [(self.host, self.port)], **connection_kwargs
+        )
+        redis_client = sentinel.master_for(self.master_name)
+        return redis_client
+
+
+class RedisSocket(RedisBase):
+    def __init__(self, broker_url, *args, **kwargs):
+        super().__init__(broker_url, *args, **kwargs)
+        self.redis = redis.Redis(
+            unix_socket_path="/" + self.vhost, password=self.password
+        )
+
+
+class RedisSsl(Redis):
+    """
+    Redis SSL class offering connection to the broker over SSL.
+    This does not currently support SSL settings through the url, only through
+    the broker_use_ssl celery configuration.
+    """
+
+    def __init__(self, broker_url, *args, **kwargs):
+        if "broker_use_ssl" not in kwargs:
+            raise ValueError("rediss broker requires broker_use_ssl")
+        self.broker_use_ssl = kwargs.get("broker_use_ssl", {})
+        super().__init__(broker_url, *args, **kwargs)
+
+    def _get_redis_client_args(self):
+        client_args = super()._get_redis_client_args()
+        client_args["ssl"] = True
+        if isinstance(self.broker_use_ssl, dict):
+            client_args.update(self.broker_use_ssl)
+        return client_args
+
+
+class Broker:
+    """Factory returning the appropriate broker client based on URL scheme.
+
+    Supported schemes:
+    ``amqp`` or ``amqps``  -> :class:`RabbitMQ`
+    ``redis``              -> :class:`Redis`
+    ``rediss``             -> :class:`RedisSsl`
+    ``redis+socket``       -> :class:`RedisSocket`
+    ``sentinel``           -> :class:`RedisSentinel`
+    """
+
+    def __new__(cls, broker_url, *args, **kwargs):
+        scheme = urlparse(broker_url).scheme
+        if scheme in ("amqp", "amqps"):
+            return RabbitMQ(broker_url, *args, **kwargs)
+        if scheme == "redis":
+            return Redis(broker_url, *args, **kwargs)
+        if scheme == "rediss":
+            return RedisSsl(broker_url, *args, **kwargs)
+        if scheme == "redis+socket":
+            return RedisSocket(broker_url, *args, **kwargs)
+        if scheme == "sentinel":
+            return RedisSentinel(broker_url, *args, **kwargs)
+        raise NotImplementedError
+
+    async def queues(self, names):
+        raise NotImplementedError

python/ray/serve/_private/build_app.py

@@ -0,0 +1,340 @@
+import inspect
+import logging
+from copy import deepcopy
+from dataclasses import dataclass
+from typing import Any, Callable, Dict, Generic, List, Optional, TypeVar, Union
+
+from ray.dag.py_obj_scanner import _PyObjScanner
+from ray.serve._private.constants import (
+    RAY_SERVE_ENABLE_HA_PROXY,
+    SERVE_LOGGER_NAME,
+)
+from ray.serve._private.http_util import ASGIAppReplicaWrapper
+from ray.serve.deployment import Application, Deployment
+from ray.serve.exceptions import RayServeException
+from ray.serve.handle import DeploymentHandle
+from ray.serve.schema import LoggingConfig
+
+logger = logging.getLogger(SERVE_LOGGER_NAME)
+
+K = TypeVar("K")
+V = TypeVar("V")
+
+INGRESS_REQUEST_ROUTER_REQUIRES_HAPROXY_ERROR = (
+    "`ingress_request_router` requires HAProxy. "
+    "Set `RAY_SERVE_ENABLE_HA_PROXY=1` in the Ray controller's environment."
+)
+
+CUSTOM_INGRESS_REQUEST_ROUTER_UNSUPPORTED_ERROR = (
+    "A custom `request_router_config.request_router_class` is not supported on "
+    "the ingress deployment when HAProxy is enabled. HAProxy load-balances "
+    "ingress traffic with its own algorithm and bypasses the Serve request "
+    "router, so the custom router would be silently ignored. Remove the custom "
+    "`request_router_class` from the ingress deployment, or configure HAProxy's "
+    "load-balancing algorithm instead."
+)
+
+
+class IDDict(dict, Generic[K, V]):
+    """Dictionary that uses id() for keys instead of hash().
+
+    This is necessary because Application objects aren't hashable and we want each
+    instance to map to a unique key.
+    """
+
+    def __getitem__(self, key: K) -> V:
+        if not isinstance(key, int):
+            key = id(key)
+        return super().__getitem__(key)
+
+    def __setitem__(self, key: K, value: V):
+        if not isinstance(key, int):
+            key = id(key)
+        return super().__setitem__(key, value)
+
+    def __delitem__(self, key: K):
+        if not isinstance(key, int):
+            key = id(key)
+        return super().__delitem__(key)
+
+    def __contains__(self, key: object):
+        if not isinstance(key, int):
+            key = id(key)
+        return super().__contains__(key)
+
+
+@dataclass(frozen=True)
+class BuiltApplication:
+    # Name of the application.
+    name: str
+    route_prefix: Optional[str]
+    logging_config: Optional[LoggingConfig]
+    # Name of the application's 'ingress' deployment
+    # (the one exposed over gRPC/HTTP/handle).
+    ingress_deployment_name: str
+    # List of unique deployments comprising the app.
+    deployments: List[Deployment]
+    # Dict[name, DeploymentHandle] mapping deployment names to the handles that replaced
+    # them in other deployments' init args/kwargs.
+    deployment_handles: Dict[str, DeploymentHandle]
+    external_scaler_enabled: bool
+    # Optional ingress request router deployment for ingress bypass mode.
+    # When set, this deployment serves /internal/route for HAProxy Lua routing.
+    ingress_request_router_deployment: Optional[Deployment] = None
+
+    def validate_single_fastapi_ingress(self) -> None:
+        """Validate that the application has at most one FastAPI ingress."""
+        num_ingress_deployments = sum(
+            inspect.isclass(deployment.func_or_class)
+            and issubclass(deployment.func_or_class, ASGIAppReplicaWrapper)
+            for deployment in self.deployments
+        )
+        if num_ingress_deployments > 1:
+            raise RayServeException(
+                f'Found multiple FastAPI deployments in application "{self.name}". '
+                "Please only include one deployment with @serve.ingress "
+                "in your application to avoid this issue."
+            )
+
+
+def _has_custom_request_router(deployment: Deployment) -> bool:
+    """Whether the deployment configures a non-default request router class."""
+    request_router_config = deployment._deployment_config.request_router_config
+    return not request_router_config.is_default_request_router()
+
+
+def _make_deployment_handle_default(
+    deployment: Deployment, app_name: str
+) -> DeploymentHandle:
+    return DeploymentHandle(
+        deployment.name,
+        app_name=app_name,
+    )
+
+
+def build_app(
+    app: Application,
+    *,
+    name: str,
+    route_prefix: Optional[str] = None,
+    logging_config: Optional[Union[Dict, LoggingConfig]] = None,
+    default_runtime_env: Optional[Dict[str, Any]] = None,
+    make_deployment_handle: Optional[
+        Callable[[Deployment, str], DeploymentHandle]
+    ] = None,
+    external_scaler_enabled: bool = False,
+) -> BuiltApplication:
+    """Builds the application into a list of finalized deployments.
+
+    The following transformations are made:
+        - Application objects in constructor args/kwargs are converted to
+          DeploymentHandles for injection at runtime.
+        - Name conflicts from deployments that use the same class are handled
+          by appending a monotonically increasing suffix (e.g., SomeClass_1).
+
+    Returns: BuiltApplication
+    """
+    if make_deployment_handle is None:
+        make_deployment_handle = _make_deployment_handle_default
+
+    ingress_request_router = app._ingress_request_router
+    if ingress_request_router is not None and not isinstance(
+        ingress_request_router, Application
+    ):
+        raise TypeError(
+            "`ingress_request_router` must be an `Application` returned by "
+            "`Deployment.bind()`."
+        )
+    if ingress_request_router is not None and not RAY_SERVE_ENABLE_HA_PROXY:
+        raise RayServeException(INGRESS_REQUEST_ROUTER_REQUIRES_HAPROXY_ERROR)
+
+    # Under HAProxy, ingress traffic is load-balanced by HAProxy and bypasses
+    # the ingress deployment's Serve request router, so a custom router there is
+    # silently ignored. Reject it unless an `ingress_request_router` is attached
+    # (the Serve LLM direct-streaming path), where HAProxy delegates replica
+    # selection back to that router.
+    if (
+        RAY_SERVE_ENABLE_HA_PROXY
+        and ingress_request_router is None
+        and _has_custom_request_router(app._bound_deployment)
+    ):
+        raise RayServeException(CUSTOM_INGRESS_REQUEST_ROUTER_UNSUPPORTED_ERROR)
+
+    handles = IDDict()
+    deployment_names = IDDict()
+    deployments = _build_app_recursive(
+        app,
+        app_name=name,
+        handles=handles,
+        deployment_names=deployment_names,
+        default_runtime_env=default_runtime_env,
+        make_deployment_handle=make_deployment_handle,
+    )
+    ingress_request_router_deployment = None
+    if ingress_request_router is not None:
+        ingress_request_router_deployments = _build_app_recursive(
+            ingress_request_router,
+            app_name=name,
+            handles=handles,
+            deployment_names=deployment_names,
+            default_runtime_env=default_runtime_env,
+            make_deployment_handle=make_deployment_handle,
+        )
+        # TODO(eicherseiji): The current ingress-bypass design only supports a
+        # standalone single-deployment router. Revisit this once routers can
+        # compose helper deployments.
+        if len(ingress_request_router_deployments) == 0:
+            raise ValueError(
+                "Expected `ingress_request_router` to build into one standalone "
+                "deployment, but it did not produce any new deployments. This "
+                "usually means the same bound router deployment is also reachable "
+                "from the main application graph; attach it only as "
+                "`ingress_request_router`."
+            )
+        if len(ingress_request_router_deployments) > 1:
+            raise ValueError(
+                "Expected `ingress_request_router` to build into exactly one "
+                "standalone deployment, got "
+                f"{len(ingress_request_router_deployments)}."
+            )
+        ingress_request_router_deployment = ingress_request_router_deployments[0]
+
+    main_deployment_names = {deployment.name for deployment in deployments}
+
+    return BuiltApplication(
+        name=name,
+        route_prefix=route_prefix,
+        logging_config=logging_config,
+        ingress_deployment_name=deployment_names[app],
+        deployments=deployments,
+        deployment_handles={
+            deployment_names[app]: handle
+            for app, handle in handles.items()
+            if deployment_names[app] in main_deployment_names
+        },
+        external_scaler_enabled=external_scaler_enabled,
+        ingress_request_router_deployment=ingress_request_router_deployment,
+    )
+
+
+def _build_app_recursive(
+    app: Application,
+    *,
+    app_name: str,
+    deployment_names: IDDict[Application, str],
+    handles: IDDict[Application, DeploymentHandle],
+    default_runtime_env: Optional[Dict[str, Any]] = None,
+    make_deployment_handle: Callable[[Deployment, str], DeploymentHandle],
+) -> List[Deployment]:
+    """Recursively traverses the graph of Application objects.
+
+    Each Application will have an associated DeploymentHandle created that will replace
+    it in any occurrences in other Applications' args or kwargs.
+
+    Also collects a list of the unique Applications encountered and returns them as
+    deployable Deployment objects.
+    """
+    # This application has already been encountered.
+    # There's no need to recurse into its child args and we don't want to create
+    # a duplicate entry for it in the list of deployments.
+    if app in handles:
+        return []
+
+    deployments = []
+    scanner = _PyObjScanner(source_type=Application)
+    try:
+        # Recursively traverse any Application objects bound to init args/kwargs.
+        child_apps = scanner.find_nodes(
+            (app._bound_deployment.init_args, app._bound_deployment.init_kwargs)
+        )
+        for child_app in child_apps:
+            deployments.extend(
+                _build_app_recursive(
+                    child_app,
+                    app_name=app_name,
+                    handles=handles,
+                    deployment_names=deployment_names,
+                    make_deployment_handle=make_deployment_handle,
+                    default_runtime_env=default_runtime_env,
+                )
+            )
+
+        # Replace Application objects with their corresponding DeploymentHandles.
+        new_init_args, new_init_kwargs = scanner.replace_nodes(handles)
+        final_deployment = app._bound_deployment.options(
+            name=_get_unique_deployment_name_memoized(app, deployment_names),
+            _init_args=new_init_args,
+            _init_kwargs=new_init_kwargs,
+        )
+        final_deployment = _set_default_runtime_env(
+            final_deployment, default_runtime_env
+        )
+
+        # Create the DeploymentHandle that will be used to replace this application
+        # in the arguments of its parent(s).
+        handles[app] = make_deployment_handle(
+            final_deployment,
+            app_name,
+        )
+
+        return deployments + [final_deployment]
+    finally:
+        scanner.clear()
+
+
+def _set_default_runtime_env(
+    d: Deployment, default_runtime_env: Optional[Dict[str, Any]]
+) -> Deployment:
+    """Configures the deployment with the provided default runtime_env.
+
+    If the deployment does not have a runtime_env configured, the default will be set.
+
+    If it does have a runtime_env configured but that runtime_env does not have a
+    working_dir, only the working_dir field will be set.
+
+    Else the deployment's runtime_env will be left untouched.
+    """
+    if not default_runtime_env:
+        return d
+
+    ray_actor_options = deepcopy(d.ray_actor_options or {})
+    default_working_dir = default_runtime_env.get("working_dir", None)
+    if "runtime_env" not in ray_actor_options:
+        ray_actor_options["runtime_env"] = default_runtime_env
+    elif default_working_dir is not None:
+        ray_actor_options["runtime_env"].setdefault("working_dir", default_working_dir)
+
+    return d.options(ray_actor_options=ray_actor_options)
+
+
+def _get_unique_deployment_name_memoized(
+    app: Application, deployment_names: IDDict[Application, str]
+) -> str:
+    """Generates a name for the deployment.
+
+    This is used to handle collisions when the user does not specify a name
+    explicitly, so typically we'd use the class name as the default.
+
+    In that case, we append a monotonically increasing suffix to the name, e.g.,
+    Deployment, then Deployment_1, then Deployment_2, ...
+
+    Names are memoized in the `deployment_names` dict, which should be passed to
+    subsequent calls to this function.
+    """
+    if app in deployment_names:
+        return deployment_names[app]
+
+    idx = 1
+    name = app._bound_deployment.name
+    while name in deployment_names.values():
+        name = f"{app._bound_deployment.name}_{idx}"
+        idx += 1
+
+    if idx != 1:
+        logger.warning(
+            "There are multiple deployments with the same name "
+            f"'{app._bound_deployment.name}'. Renaming one to '{name}'."
+        )
+
+    deployment_names[app] = name
+    return name

python/ray/serve/_private/client.py

@@ -0,0 +1,650 @@
+import asyncio
+import logging
+import random
+import time
+from collections.abc import Sequence
+from functools import wraps
+from typing import Callable, Dict, List, Optional, Tuple, Union
+
+import ray
+from ray.actor import ActorHandle
+from ray.serve._private.application_state import StatusOverview
+from ray.serve._private.build_app import BuiltApplication
+from ray.serve._private.common import (
+    DeploymentID,
+    DeploymentStatus,
+    DeploymentStatusInfo,
+    RequestRoutingInfo,
+)
+from ray.serve._private.constants import (
+    CLIENT_CHECK_CREATION_POLLING_INTERVAL_S,
+    CLIENT_POLLING_INTERVAL_S,
+    HTTP_PROXY_TIMEOUT,
+    MAX_CACHED_HANDLES,
+    SERVE_DEFAULT_APP_NAME,
+    SERVE_LOGGER_NAME,
+)
+from ray.serve._private.controller import ServeController
+from ray.serve._private.deploy_utils import get_deploy_args
+from ray.serve._private.deployment_info import DeploymentInfo
+from ray.serve._private.utils import _callable_uses_multiplexing, get_random_string
+from ray.serve.config import HTTPOptions
+from ray.serve.exceptions import RayServeException
+from ray.serve.generated.serve_pb2 import (
+    ApplicationArgs,
+    DeploymentArgs,
+    DeploymentRoute,
+    DeploymentStatusInfo as DeploymentStatusInfoProto,
+    StatusOverview as StatusOverviewProto,
+)
+from ray.serve.handle import DeploymentHandle
+from ray.serve.schema import (
+    ApplicationStatus,
+    LoggingConfig,
+    ServeApplicationSchema,
+    ServeDeploySchema,
+)
+
+logger = logging.getLogger(SERVE_LOGGER_NAME)
+
+
+def _ensure_connected(f: Callable) -> Callable:
+    @wraps(f)
+    def check(self, *args, **kwargs):
+        if self._shutdown:
+            raise RayServeException("Client has already been shut down.")
+        return f(self, *args, **kwargs)
+
+    return check
+
+
+class ServeControllerClient:
+    def __init__(
+        self,
+        controller: ActorHandle,
+    ):
+        self._controller: ServeController = controller
+        self._shutdown = False
+        self._http_config: HTTPOptions = ray.get(controller.get_http_config.remote())
+        self._root_url = ray.get(controller.get_root_url.remote())
+
+        # Each handle has the overhead of long poll client, therefore cached.
+        self.handle_cache = dict()
+        self._evicted_handle_keys = set()
+
+    @property
+    def root_url(self):
+        return self._root_url
+
+    @property
+    def http_config(self):
+        return self._http_config
+
+    def __reduce__(self):
+        raise RayServeException(("Ray Serve client cannot be serialized."))
+
+    def shutdown_cached_handles(self):
+        """Shuts down all cached handles.
+
+        Remove the reference to the cached handles so that they can be
+        garbage collected.
+        """
+        for cache_key in list(self.handle_cache):
+            self.handle_cache[cache_key].shutdown()
+            del self.handle_cache[cache_key]
+
+    async def shutdown_cached_handles_async(self):
+        """Shuts down all cached handles asynchronously.
+
+        Remove the reference to the cached handles so that they can be
+        garbage collected.
+        """
+
+        async def shutdown_task(cache_key):
+            await self.handle_cache[cache_key].shutdown_async()
+            del self.handle_cache[cache_key]
+
+        await asyncio.gather(
+            *[shutdown_task(cache_key) for cache_key in list(self.handle_cache)]
+        )
+
+    def shutdown(self, timeout_s: float = 30.0) -> None:
+        """Completely shut down the connected Serve instance.
+
+        Shuts down all processes and deletes all state associated with the
+        instance.
+        """
+        self.shutdown_cached_handles()
+
+        if ray.is_initialized() and not self._shutdown:
+            try:
+                ray.get(self._controller.graceful_shutdown.remote(), timeout=timeout_s)
+            except ray.exceptions.RayActorError:
+                # Controller has been shut down.
+                pass
+            except TimeoutError:
+                logger.warning(
+                    f"Controller failed to shut down within {timeout_s}s. "
+                    "Check controller logs for more details."
+                )
+            self._shutdown = True
+
+    async def shutdown_async(self, timeout_s: float = 30.0) -> None:
+        """Completely shut down the connected Serve instance.
+
+        Shuts down all processes and deletes all state associated with the
+        instance.
+        """
+        await self.shutdown_cached_handles_async()
+
+        if ray.is_initialized() and not self._shutdown:
+            try:
+                await asyncio.wait_for(
+                    self._controller.graceful_shutdown.remote(), timeout=timeout_s
+                )
+            except ray.exceptions.RayActorError:
+                # Controller has been shut down.
+                pass
+            except TimeoutError:
+                logger.warning(
+                    f"Controller failed to shut down within {timeout_s}s. "
+                    "Check controller logs for more details."
+                )
+            self._shutdown = True
+
+    def _wait_for_deployment_healthy(self, name: str, timeout_s: int = -1):
+        """Waits for the named deployment to enter "HEALTHY" status.
+
+        Raises RuntimeError if the deployment enters the "UNHEALTHY" status
+        instead.
+
+        Raises TimeoutError if this doesn't happen before timeout_s.
+        """
+        start = time.time()
+        while time.time() - start < timeout_s or timeout_s < 0:
+            status_bytes = ray.get(self._controller.get_deployment_status.remote(name))
+
+            if status_bytes is None:
+                raise RuntimeError(
+                    f"Waiting for deployment {name} to be HEALTHY, "
+                    "but deployment doesn't exist."
+                )
+
+            status = DeploymentStatusInfo.from_proto(
+                DeploymentStatusInfoProto.FromString(status_bytes)
+            )
+
+            if status.status == DeploymentStatus.HEALTHY:
+                break
+            elif status.status == DeploymentStatus.UNHEALTHY:
+                raise RuntimeError(
+                    f"Deployment {name} is UNHEALTHY: " f"{status.message}"
+                )
+            else:
+                # Guard against new unhandled statuses being added.
+                assert status.status == DeploymentStatus.UPDATING
+
+            logger.debug(
+                f"Waiting for {name} to be healthy, current status: "
+                f"{status.status}."
+            )
+            time.sleep(CLIENT_POLLING_INTERVAL_S)
+        else:
+            raise TimeoutError(
+                f"Deployment {name} did not become HEALTHY after {timeout_s}s."
+            )
+
+    def _wait_for_deployment_deleted(
+        self, name: str, app_name: str, timeout_s: int = 60
+    ):
+        """Waits for the named deployment to be shut down and deleted.
+
+        Raises TimeoutError if this doesn't happen before timeout_s.
+        """
+        start = time.time()
+        while time.time() - start < timeout_s:
+            curr_status_bytes = ray.get(
+                self._controller.get_deployment_status.remote(name)
+            )
+            if curr_status_bytes is None:
+                break
+            curr_status = DeploymentStatusInfo.from_proto(
+                DeploymentStatusInfoProto.FromString(curr_status_bytes)
+            )
+            logger.debug(
+                f"Waiting for {name} to be deleted, current status: {curr_status}."
+            )
+            time.sleep(CLIENT_POLLING_INTERVAL_S)
+        else:
+            raise TimeoutError(f"Deployment {name} wasn't deleted after {timeout_s}s.")
+
+    def _wait_for_deployment_created(
+        self, deployment_name: str, app_name: str, timeout_s: int = -1
+    ):
+        """Waits for the named deployment to be created.
+
+        A deployment being created simply means that its been registered
+        with the deployment state manager. The deployment state manager
+        will then continue to reconcile the deployment towards its
+        target state.
+
+        Raises TimeoutError if this doesn't happen before timeout_s.
+        """
+
+        start = time.time()
+        while time.time() - start < timeout_s or timeout_s < 0:
+            status_bytes = ray.get(
+                self._controller.get_deployment_status.remote(deployment_name, app_name)
+            )
+
+            if status_bytes is not None:
+                break
+
+            logger.debug(
+                f"Waiting for deployment '{deployment_name}' in application "
+                f"'{app_name}' to be created."
+            )
+            time.sleep(CLIENT_CHECK_CREATION_POLLING_INTERVAL_S)
+        else:
+            raise TimeoutError(
+                f"Deployment '{deployment_name}' in application '{app_name}' "
+                f"did not become HEALTHY after {timeout_s}s."
+            )
+
+    def _wait_for_application_running(self, name: str, timeout_s: int = -1):
+        """Waits for the named application to enter "RUNNING" status.
+
+        Args:
+            name: the application name to wait on.
+            timeout_s: maximum time to wait, in seconds. A negative value waits
+                indefinitely.
+
+        Raises:
+            RuntimeError: if the application enters the "DEPLOY_FAILED" status instead.
+            TimeoutError: if this doesn't happen before timeout_s.
+        """
+        start = time.time()
+        while time.time() - start < timeout_s or timeout_s < 0:
+            status_bytes = ray.get(self._controller.get_serve_status.remote(name))
+            if status_bytes is None:
+                raise RuntimeError(
+                    f"Waiting for application {name} to be RUNNING, "
+                    "but application doesn't exist."
+                )
+
+            status = StatusOverview.from_proto(
+                StatusOverviewProto.FromString(status_bytes)
+            )
+
+            if status.app_status.status == ApplicationStatus.RUNNING:
+                break
+            elif status.app_status.status == ApplicationStatus.DEPLOY_FAILED:
+                raise RuntimeError(
+                    f"Deploying application {name} failed: {status.app_status.message}"
+                )
+
+            logger.debug(
+                f"Waiting for {name} to be RUNNING, current status: "
+                f"{status.app_status.status}."
+            )
+            time.sleep(CLIENT_POLLING_INTERVAL_S)
+        else:
+            raise TimeoutError(
+                f"Application {name} did not become RUNNING after {timeout_s}s."
+            )
+
+    @_ensure_connected
+    def wait_for_proxies_serving(
+        self, wait_for_applications_running: bool = True
+    ) -> None:
+        """Wait for the proxies to be ready to serve requests."""
+        proxy_handles = ray.get(self._controller.get_proxies.remote())
+
+        if not proxy_handles:
+            return
+
+        serving_refs = [
+            handle.serving.remote(
+                wait_for_applications_running=wait_for_applications_running
+            )
+            for handle in proxy_handles.values()
+        ]
+
+        done, pending = ray.wait(
+            serving_refs,
+            timeout=HTTP_PROXY_TIMEOUT,
+            num_returns=len(serving_refs),
+        )
+
+        if len(pending) > 0:
+            raise TimeoutError(f"Proxies not available after {HTTP_PROXY_TIMEOUT}s.")
+
+        # Ensure the proxies are either serving or dead.
+        for ref in done:
+            try:
+                ray.get(ref, timeout=1)
+            except ray.exceptions.RayActorError:
+                pass
+            except Exception:
+                raise TimeoutError(
+                    f"Proxies not available after {HTTP_PROXY_TIMEOUT}s."
+                )
+
+    @_ensure_connected
+    def deploy_applications(
+        self,
+        built_apps: Sequence[BuiltApplication],
+        *,
+        wait_for_ingress_deployment_creation: bool = True,
+        wait_for_applications_running: bool = True,
+    ) -> List[DeploymentHandle]:
+        name_to_deployment_args_list = {}
+        name_to_application_args = {}
+        for app in built_apps:
+            deployment_args_list = []
+            deployments_to_deploy = list(app.deployments)
+            if app.ingress_request_router_deployment is not None:
+                deployments_to_deploy.append(app.ingress_request_router_deployment)
+
+            for deployment in deployments_to_deploy:
+                if deployment.logging_config is None and app.logging_config:
+                    deployment = deployment.options(logging_config=app.logging_config)
+
+                is_ingress = deployment.name == app.ingress_deployment_name
+                is_ingress_request_router = (
+                    app.ingress_request_router_deployment is not None
+                    and deployment.name == app.ingress_request_router_deployment.name
+                )
+                deployment_args = get_deploy_args(
+                    deployment.name,
+                    ingress=is_ingress,
+                    ingress_request_router=is_ingress_request_router,
+                    replica_config=deployment._replica_config,
+                    deployment_config=deployment._deployment_config,
+                    version=deployment._version or get_random_string(),
+                    route_prefix=app.route_prefix if is_ingress else None,
+                    uses_multiplexing=_callable_uses_multiplexing(
+                        deployment.func_or_class
+                    ),
+                )
+
+                deployment_args_proto = DeploymentArgs()
+                deployment_args_proto.deployment_name = deployment_args[
+                    "deployment_name"
+                ]
+                deployment_args_proto.deployment_config = deployment_args[
+                    "deployment_config_proto_bytes"
+                ]
+                deployment_args_proto.replica_config = deployment_args[
+                    "replica_config_proto_bytes"
+                ]
+                deployment_args_proto.deployer_job_id = deployment_args[
+                    "deployer_job_id"
+                ]
+                if deployment_args["route_prefix"]:
+                    deployment_args_proto.route_prefix = deployment_args["route_prefix"]
+                deployment_args_proto.ingress = deployment_args["ingress"]
+                deployment_args_proto.ingress_request_router = deployment_args[
+                    "ingress_request_router"
+                ]
+                deployment_args_proto.uses_multiplexing = deployment_args[
+                    "uses_multiplexing"
+                ]
+
+                deployment_args_list.append(deployment_args_proto.SerializeToString())
+
+            application_args_proto = ApplicationArgs()
+            application_args_proto.external_scaler_enabled = app.external_scaler_enabled
+
+            name_to_deployment_args_list[app.name] = deployment_args_list
+            name_to_application_args[
+                app.name
+            ] = application_args_proto.SerializeToString()
+
+        # Validate applications before sending to controller.
+        self._check_ingress_deployments(built_apps)
+
+        ray.get(
+            self._controller.deploy_applications.remote(
+                name_to_deployment_args_list, name_to_application_args
+            )
+        )
+
+        handles = []
+        ready_apps = []
+        for app in built_apps:
+            # The deployment state is not guaranteed to be created after
+            # deploy_application returns; the application state manager will
+            # need another reconcile iteration to create it.
+            if wait_for_ingress_deployment_creation:
+                self._wait_for_deployment_created(app.ingress_deployment_name, app.name)
+
+            if wait_for_applications_running:
+                self._wait_for_application_running(app.name)
+                ready_apps.append(app)
+
+            handles.append(
+                self.get_handle(
+                    app.ingress_deployment_name, app.name, check_exists=False
+                )
+            )
+
+        # Wait for the proxies to be serving before declaring the applications
+        # ready, so the "is ready" log line only prints once requests can
+        # actually be routed to the applications.
+        self.wait_for_proxies_serving(
+            wait_for_applications_running=wait_for_applications_running
+        )
+
+        for app in ready_apps:
+            if app.route_prefix is not None:
+                url_part = " at " + self._root_url + app.route_prefix
+            else:
+                url_part = ""
+            logger.info(f"Application '{app.name}' is ready{url_part}.")
+
+        return handles
+
+    @_ensure_connected
+    def deploy_apps(
+        self,
+        config: Union[ServeApplicationSchema, ServeDeploySchema],
+        _blocking: bool = False,
+    ) -> None:
+        """Starts a task on the controller that deploys application(s) from a config.
+
+        Args:
+            config: A single-application config (ServeApplicationSchema) or a
+                multi-application config (ServeDeploySchema)
+            _blocking: Whether to block until the application is running.
+
+        Raises:
+            RayTaskError: If the deploy task on the controller fails. This can be
+                because a single-app config was deployed after deploying a multi-app
+                config, or vice versa.
+        """
+        ray.get(self._controller.apply_config.remote(config))
+
+        if _blocking:
+            timeout_s = 60
+
+            if isinstance(config, ServeDeploySchema):
+                app_names = {app.name for app in config.applications}
+            else:
+                app_names = {config.name}
+
+            start = time.time()
+            while time.time() - start < timeout_s:
+                statuses = self.list_serve_statuses()
+                app_to_status = {
+                    status.name: status.app_status.status
+                    for status in statuses
+                    if status.name in app_names
+                }
+                if len(app_names) == len(app_to_status) and set(
+                    app_to_status.values()
+                ) == {ApplicationStatus.RUNNING}:
+                    break
+
+                time.sleep(CLIENT_POLLING_INTERVAL_S)
+            else:
+                raise TimeoutError(
+                    f"Serve application isn't running after {timeout_s}s."
+                )
+
+            self.wait_for_proxies_serving(wait_for_applications_running=True)
+
+    def _check_ingress_deployments(
+        self, built_apps: Sequence[BuiltApplication]
+    ) -> None:
+        """Check @serve.ingress of deployments across applications.
+
+        Raises: RayServeException if more than one @serve.ingress
+            is found among deployments in any single application.
+        """
+        for app in built_apps:
+            app.validate_single_fastapi_ingress()
+
+    @_ensure_connected
+    def delete_apps(self, names: List[str], blocking: bool = True):
+        if not names:
+            return
+
+        logger.info(f"Deleting app {names}")
+        self._controller.delete_apps.remote(names)
+        if blocking:
+            start = time.time()
+            while time.time() - start < 60:
+                curr_statuses_bytes = ray.get(
+                    self._controller.get_serve_statuses.remote(names)
+                )
+                all_deleted = True
+                for cur_status_bytes in curr_statuses_bytes:
+                    cur_status = StatusOverview.from_proto(
+                        StatusOverviewProto.FromString(cur_status_bytes)
+                    )
+                    if cur_status.app_status.status != ApplicationStatus.NOT_STARTED:
+                        all_deleted = False
+                if all_deleted:
+                    return
+                time.sleep(CLIENT_POLLING_INTERVAL_S)
+            else:
+                raise TimeoutError(
+                    f"Some of these applications weren't deleted after 60s: {names}"
+                )
+
+    @_ensure_connected
+    def delete_all_apps(self, blocking: bool = True):
+        """Delete all applications"""
+        all_apps = []
+        for status_bytes in ray.get(self._controller.list_serve_statuses.remote()):
+            proto = StatusOverviewProto.FromString(status_bytes)
+            status = StatusOverview.from_proto(proto)
+            all_apps.append(status.name)
+        self.delete_apps(all_apps, blocking)
+
+    @_ensure_connected
+    def get_deployment_info(
+        self, name: str, app_name: str
+    ) -> Tuple[DeploymentInfo, str]:
+        deployment_route = DeploymentRoute.FromString(
+            ray.get(self._controller.get_deployment_info.remote(name, app_name))
+        )
+        return (
+            DeploymentInfo.from_proto(deployment_route.deployment_info),
+            deployment_route.route if deployment_route.route != "" else None,
+        )
+
+    @_ensure_connected
+    def get_serve_status(self, name: str = SERVE_DEFAULT_APP_NAME) -> StatusOverview:
+        proto = StatusOverviewProto.FromString(
+            ray.get(self._controller.get_serve_status.remote(name))
+        )
+        return StatusOverview.from_proto(proto)
+
+    @_ensure_connected
+    def list_serve_statuses(self) -> List[StatusOverview]:
+        statuses_bytes = ray.get(self._controller.list_serve_statuses.remote())
+        return [
+            StatusOverview.from_proto(StatusOverviewProto.FromString(status_bytes))
+            for status_bytes in statuses_bytes
+        ]
+
+    @_ensure_connected
+    def get_all_deployment_statuses(self) -> List[DeploymentStatusInfo]:
+        statuses_bytes = ray.get(self._controller.get_all_deployment_statuses.remote())
+        return [
+            DeploymentStatusInfo.from_proto(
+                DeploymentStatusInfoProto.FromString(status_bytes)
+            )
+            for status_bytes in statuses_bytes
+        ]
+
+    @_ensure_connected
+    def get_serve_details(self) -> Dict:
+        return ray.get(self._controller.get_serve_instance_details.remote())
+
+    @_ensure_connected
+    def get_handle(
+        self,
+        deployment_name: str,
+        app_name: Optional[str] = SERVE_DEFAULT_APP_NAME,
+        check_exists: bool = True,
+    ) -> DeploymentHandle:
+        """Construct a handle for the specified deployment.
+
+        Args:
+            deployment_name: Deployment name.
+            app_name: Application name.
+            check_exists: If False, then Serve won't check the deployment
+                is registered. True by default.
+
+        Returns:
+            DeploymentHandle
+        """
+        deployment_id = DeploymentID(name=deployment_name, app_name=app_name)
+        cache_key = (deployment_name, app_name, check_exists)
+        if cache_key in self.handle_cache:
+            return self.handle_cache[cache_key]
+
+        if check_exists:
+            all_deployments = ray.get(self._controller.list_deployment_ids.remote())
+            if deployment_id not in all_deployments:
+                raise KeyError(f"{deployment_id} does not exist.")
+
+        handle = DeploymentHandle(deployment_name, app_name)
+        self.handle_cache[cache_key] = handle
+        if cache_key in self._evicted_handle_keys:
+            logger.warning(
+                "You just got a ServeHandle that was evicted from internal "
+                "cache. This means you are getting too many ServeHandles in "
+                "the same process, this will bring down Serve's performance. "
+                "Please post a github issue at "
+                "https://github.com/ray-project/ray/issues to let the Serve "
+                "team to find workaround for your use case."
+            )
+
+        if len(self.handle_cache) > MAX_CACHED_HANDLES:
+            # Perform random eviction to keep the handle cache from growing
+            # infinitely. We used use WeakValueDictionary but hit
+            # https://github.com/ray-project/ray/issues/18980.
+            evict_key = random.choice(list(self.handle_cache.keys()))
+            self._evicted_handle_keys.add(evict_key)
+            self.handle_cache.pop(evict_key)
+
+        return handle
+
+    @_ensure_connected
+    def record_request_routing_info(self, info: RequestRoutingInfo):
+        """Record replica routing information for a replica.
+
+        Args:
+            info: RequestRoutingInfo including deployment name, replica tag,
+                multiplex model ids, and routing stats.
+        """
+        self._controller.record_request_routing_info.remote(info)
+
+    @_ensure_connected
+    def update_global_logging_config(self, logging_config: LoggingConfig):
+        """Reconfigure the logging config for the controller & proxies."""
+        self._controller.reconfigure_global_logging_config.remote(logging_config)

python/ray/serve/_private/cluster_node_info_cache.py

@@ -0,0 +1,145 @@
+import logging
+from abc import ABC, abstractmethod
+from typing import Dict, FrozenSet, List, Optional, Set, Tuple, Union
+
+import ray
+from ray._common.utils import binary_to_hex
+from ray._raylet import GcsClient
+from ray.serve._private.constants import RAY_GCS_RPC_TIMEOUT_S, SERVE_LOGGER_NAME
+
+logger = logging.getLogger(SERVE_LOGGER_NAME)
+
+
+class ClusterNodeInfoCache(ABC):
+    """Provide access to cached node information in the cluster."""
+
+    def __init__(self, gcs_client: GcsClient):
+        self._gcs_client = gcs_client
+        self._cached_alive_nodes = None
+        self._cached_node_labels = dict()
+        self._cached_total_resources_per_node = dict()
+        self._cached_available_resources_per_node = dict()
+        # Track alive node IDs to detect cluster membership changes and skip
+        # rebuilding labels / total resources when nothing changed.
+        self._alive_node_id_set: FrozenSet[str] = frozenset()
+
+    def update(self):
+        """Update the cache by fetching latest node information from GCS.
+
+        This should be called once in each update cycle.
+        Within an update cycle, everyone will see the same
+        cached node info avoiding any potential issues
+        caused by inconsistent node info seen by different components.
+        """
+        nodes = self._gcs_client.get_all_node_info(timeout=RAY_GCS_RPC_TIMEOUT_S)
+        alive_nodes = [
+            (node_id.hex(), node.node_name, node.instance_id)
+            for (node_id, node) in nodes.items()
+            if node.state == ray.core.generated.gcs_pb2.GcsNodeInfo.ALIVE
+        ]
+
+        # Sort on NodeID to ensure the ordering is deterministic across the cluster.
+        alive_nodes.sort()
+        self._cached_alive_nodes = alive_nodes
+
+        # Detect whether the set of alive nodes has changed. Rebuild labels
+        # and total resources only when it has, since they are static per-node
+        # properties that don't change while a node stays alive.
+        current_alive_ids = frozenset(node_id for node_id, _, _ in alive_nodes)
+        if current_alive_ids != self._alive_node_id_set:
+            self._alive_node_id_set = current_alive_ids
+            self._cached_node_labels = {
+                node_id.hex(): dict(node.labels)
+                for (node_id, node) in nodes.items()
+                if node_id.hex() in current_alive_ids
+            }
+            self._cached_total_resources_per_node = {
+                node_id.hex(): dict(node.resources_total)
+                for (node_id, node) in nodes.items()
+                if node_id.hex() in current_alive_ids
+            }
+
+        # Fetch available resources using the existing GCS client rather than
+        # the legacy GlobalStateAccessor path (which opens a second connection
+        # and performs redundant protobuf deserialization).
+        self._cached_available_resources_per_node = (
+            self._fetch_available_resources_per_node()
+        )
+
+    def _fetch_available_resources_per_node(self) -> Dict[str, Dict[str, float]]:
+        """Fetch available resources per alive node via get_all_resource_usage()."""
+        try:
+            reply = self._gcs_client.get_all_resource_usage(
+                timeout=RAY_GCS_RPC_TIMEOUT_S
+            )
+        except Exception:
+            logger.warning(
+                "Failed to fetch resource usage from GCS. "
+                "Available resources cache will be stale.",
+                exc_info=True,
+            )
+            return self._cached_available_resources_per_node
+
+        return {
+            node_id: dict(resource_data.resources_available)
+            for resource_data in reply.resource_usage_data.batch
+            if (node_id := binary_to_hex(resource_data.node_id))
+            in self._alive_node_id_set
+        }
+
+    def get_alive_nodes(self) -> List[Tuple[str, str, str]]:
+        """Get IDs, IPs, and Instance IDs for all live nodes in the cluster.
+
+        Returns a list of (node_id: str, node_ip: str, instance_id: str).
+        The node_id can be passed into the Ray SchedulingPolicy API.
+        """
+        return self._cached_alive_nodes
+
+    def get_total_resources_per_node(self) -> Dict[str, Dict]:
+        """Get total resources for alive nodes."""
+        return self._cached_total_resources_per_node
+
+    def get_alive_node_ids(self) -> Set[str]:
+        """Get IDs of all live nodes in the cluster."""
+        return {node_id for node_id, _, _ in self.get_alive_nodes()}
+
+    @abstractmethod
+    def get_draining_nodes(self) -> Dict[str, int]:
+        """Get draining nodes in the cluster and their deadlines."""
+        raise NotImplementedError
+
+    @abstractmethod
+    def get_node_az(self, node_id: str) -> Optional[str]:
+        """Get availability zone of a node."""
+        raise NotImplementedError
+
+    def get_active_node_ids(self) -> Set[str]:
+        """Get IDs of all active nodes in the cluster.
+
+        A node is active if it's schedulable for new tasks and actors.
+        """
+        return self.get_alive_node_ids() - set(self.get_draining_nodes())
+
+    def get_available_resources_per_node(self) -> Dict[str, Union[float, Dict]]:
+        """Get available resources per node.
+
+        Returns a map from (node_id -> Dict of resources).
+        """
+
+        return self._cached_available_resources_per_node
+
+    def get_node_labels(self, node_id: str) -> Dict[str, str]:
+        """Get the labels for a specific node from the cache."""
+        return self._cached_node_labels.get(node_id, {})
+
+
+class DefaultClusterNodeInfoCache(ClusterNodeInfoCache):
+    def __init__(self, gcs_client: GcsClient):
+        super().__init__(gcs_client)
+
+    def get_draining_nodes(self) -> Dict[str, int]:
+        return dict()
+
+    def get_node_az(self, node_id: str) -> Optional[str]:
+        """Get availability zone of a node."""
+        return None

python/ray/serve/_private/constants_utils.py

@@ -0,0 +1,314 @@
+import os
+import warnings
+from typing import Callable, List, Optional, Type, TypeVar
+
+
+def str_to_list(s: str) -> List[str]:
+    """Return a list from a comma-separated string.
+
+    Trims whitespace and skips empty entries.
+    """
+    return [part for part in (part.strip() for part in s.split(",")) if part]
+
+
+def parse_latency_buckets(bucket_str: str, default_buckets: List[float]) -> List[float]:
+    """Parse a comma-separated string of latency bucket values.
+
+    Args:
+        bucket_str: A comma-separated string of positive numbers in ascending order.
+        default_buckets: Default bucket values to use if bucket_str is empty.
+
+    Returns:
+        A list of parsed float values.
+
+    Raises:
+        ValueError: If the format is invalid or values don't meet requirements.
+    """
+    if bucket_str.strip() == "":
+        return default_buckets
+    try:
+        # Convert string to list of floats
+        buckets = [float(x.strip()) for x in bucket_str.split(",")]
+
+        if not buckets:
+            raise ValueError("Empty bucket list")
+        if any(x <= 0 for x in buckets):
+            raise ValueError("Bucket values must be positive")
+        if sorted(set(buckets)) != buckets:
+            raise ValueError("Bucket values must be in strictly ascending order")
+
+        return buckets
+    except Exception as e:
+        raise ValueError(
+            f"Invalid format for `{bucket_str}`. "
+            f"Expected comma-separated positive numbers in ascending order. Error: {str(e)}"
+        ) from e
+
+
+T = TypeVar("T")
+
+# todo: remove for the '3.0.0' release.
+_wrong_names_white_list = {
+    "REQUEST_LATENCY_BUCKETS_MS",
+    "MODEL_LOAD_LATENCY_BUCKETS_MS",
+    "MAX_CACHED_HANDLES",
+    "SERVE_REQUEST_PROCESSING_TIMEOUT_S",
+}
+
+
+def _validate_name(name: str) -> None:
+    """Validate Ray Serve environment variable name."""
+    required_prefix = "RAY_SERVE_"
+
+    if not name.startswith(required_prefix):
+        if name in _wrong_names_white_list:
+            return
+
+        raise ValueError(
+            f"Got unexpected environment variable name `{name}`! "
+            f"Ray Serve environment variables require prefix `{required_prefix}`. "
+        )
+
+
+def _get_env_value(
+    name: str,
+    default: Optional[T],
+    value_type: Type[T],
+    validation_func: Optional[Callable[[T], bool]] = None,
+    expected_value_description: Optional[str] = None,
+) -> Optional[T]:
+    """Get environment variable with type conversion and validation.
+
+    This function retrieves an environment variable, converts it to the specified type,
+    and optionally validates the converted value.
+
+    Args:
+        name: The name of the environment variable.
+        default: Default value to use if the environment variable is not set.
+            If None, the function will return None without validation.
+        value_type: Type to convert the environment variable value to (e.g., int, float, str).
+        validation_func: Optional function that takes the converted value and returns
+            a boolean indicating whether the value is valid.
+        expected_value_description: Description of the expected value characteristics
+            (e.g., "positive", "non-negative") used in error messages.
+            Optional, expected only if validation_func is provided.
+
+    Returns:
+        The environment variable value converted to the specified type and validated,
+        or the default value if the environment variable is not set.
+
+    Raises:
+        ValueError: If the environment variable value cannot be converted to the specified
+            type, or if it fails the optional validation check. Also, if name validation fails.
+    """
+    _validate_name(name)
+
+    explicitly_defined_value = os.environ.get(name)
+    if explicitly_defined_value is None:
+        if default is None:
+            return None
+        else:
+            raw = default
+    else:
+        _deprecation_warning(name)
+        raw = explicitly_defined_value
+
+    try:
+        value = value_type(raw)
+    except ValueError as e:
+        raise ValueError(
+            f"Environment variable `{name}` value `{raw}` cannot be converted to `{value_type.__name__}`!"
+        ) from e
+
+    if validation_func and not validation_func(value):
+        raise ValueError(
+            f"Got unexpected value `{value}` for `{name}` environment variable! "
+            f"Expected {expected_value_description} `{value_type.__name__}`."
+        )
+
+    return value
+
+
+def get_env_int(name: str, default: Optional[int]) -> Optional[int]:
+    """Get environment variable as an integer.
+
+    Args:
+        name: The name of the environment variable.
+        default: Default value to use if the environment variable is not set.
+
+    Returns:
+        The environment variable value as an integer.
+
+    Raises:
+        ValueError: If the value cannot be converted to an integer.
+    """
+    return _get_env_value(name, default, int)
+
+
+def get_env_int_positive(name: str, default: Optional[int]) -> Optional[int]:
+    """Get environment variable as a positive integer.
+
+    Args:
+        name: The name of the environment variable.
+        default: Default value to use if the environment variable is not set.
+
+    Returns:
+        The environment variable value as a positive integer.
+
+    Raises:
+        ValueError: If the value cannot be converted to an integer or is not positive.
+    """
+    return _get_env_value(name, default, int, lambda x: x > 0, "positive")
+
+
+def get_env_int_non_negative(name: str, default: Optional[int]) -> Optional[int]:
+    """Get environment variable as a non-negative integer.
+
+    Args:
+        name: The name of the environment variable.
+        default: Default value to use if the environment variable is not set.
+
+    Returns:
+        The environment variable value as a non-negative integer.
+
+    Raises:
+        ValueError: If the value cannot be converted to an integer or is negative.
+    """
+    return _get_env_value(name, default, int, lambda x: x >= 0, "non negative")
+
+
+def get_env_float(name: str, default: Optional[float]) -> Optional[float]:
+    """Get environment variable as a float.
+
+    Args:
+        name: The name of the environment variable.
+        default: Default value to use if the environment variable is not set.
+
+    Returns:
+        The environment variable value as a float.
+
+    Raises:
+        ValueError: If the value cannot be converted to a float.
+    """
+    return _get_env_value(name, default, float)
+
+
+def get_env_float_positive(name: str, default: Optional[float]) -> Optional[float]:
+    """Get environment variable as a positive float.
+
+    Args:
+        name: The name of the environment variable.
+        default: Default value to use if the environment variable is not set.
+
+    Returns:
+        The environment variable value as a positive float.
+
+    Raises:
+        ValueError: If the value cannot be converted to a float or is not positive.
+    """
+    return _get_env_value(name, default, float, lambda x: x > 0, "positive")
+
+
+def get_env_float_non_negative(name: str, default: Optional[float]) -> Optional[float]:
+    """Get environment variable as a non-negative float.
+
+    Args:
+        name: The name of the environment variable.
+        default: Default value to use if the environment variable is not set.
+
+    Returns:
+        The environment variable value as a non-negative float.
+
+    Raises:
+        ValueError: If the value cannot be converted to a float or is negative.
+    """
+    return _get_env_value(name, default, float, lambda x: x >= 0, "non negative")
+
+
+def get_env_str(name: str, default: Optional[str]) -> Optional[str]:
+    """Get environment variable as a string.
+
+    Args:
+        name: The name of the environment variable.
+        default: Default value to use if the environment variable is not set.
+
+    Returns:
+        The environment variable value as a string.
+        Returns `None` if default is `None` and value not found.
+    """
+    return _get_env_value(name, default, str)
+
+
+def get_env_bool(name: str, default: str) -> bool:
+    """Get environment variable as a boolean.
+
+    Environment variable values of "1" are interpreted as True, all others as False.
+
+    Args:
+        name: The name of the environment variable.
+        default: Default value to use if the environment variable is not set.
+            Expects "0" or "1".
+
+    Returns:
+        True if the environment variable value is "1", False otherwise.
+    """
+    env_value_str = _get_env_value(name, default, str)
+    return env_value_str == "1"
+
+
+# Environment variables that are fully deprecated and will be ignored.
+_fully_deprecated_env_vars = {
+    "RAY_SERVE_HTTP_KEEP_ALIVE_TIMEOUT_S": "http_options.keep_alive_timeout_s",
+    "RAY_SERVE_ROUTER_RETRY_INITIAL_BACKOFF_S": "request_router_config.initial_backoff_s",
+    "RAY_SERVE_ROUTER_RETRY_BACKOFF_MULTIPLIER": "request_router_config.backoff_multiplier",
+    "RAY_SERVE_ROUTER_RETRY_MAX_BACKOFF_S": "request_router_config.max_backoff_s",
+}
+
+
+def _deprecation_warning(name: str) -> None:
+    """Log replacement warning for wrong or legacy environment variables.
+
+    TODO: remove this function for the '3.0.0' release.
+
+    Args:
+        name: Environment variable name.
+    """
+
+    def get_new_name(name: str) -> str:
+        if name == "RAY_SERVE_HANDLE_METRIC_PUSH_INTERVAL_S":
+            return "RAY_SERVE_HANDLE_AUTOSCALING_METRIC_PUSH_INTERVAL_S"
+        elif name == "SERVE_REQUEST_PROCESSING_TIMEOUT_S":
+            return "RAY_SERVE_REQUEST_PROCESSING_TIMEOUT_S"
+        else:
+            return f"{required_prefix}{name}"
+
+    change_version = "3.0.0"
+    required_prefix = "RAY_SERVE_"
+
+    if (
+        name in _wrong_names_white_list
+        or name == "RAY_SERVE_HANDLE_METRIC_PUSH_INTERVAL_S"
+    ):
+        new_name = get_new_name(name)
+        warnings.warn(
+            f"Starting from version `{change_version}` environment variable "
+            f"`{name}` will be deprecated. Please use `{new_name}` instead.",
+            FutureWarning,
+            stacklevel=4,
+        )
+
+
+def warn_if_deprecated_env_var_set(name: str) -> None:
+    """Warn if a fully deprecated environment variable is set.
+
+    Args:
+        name: Environment variable name.
+    """
+    if name in _fully_deprecated_env_vars and os.environ.get(name):
+        config_option = _fully_deprecated_env_vars[name]
+        warnings.warn(
+            f"`{name}` environment variable will be deprecated in the future. "
+            f"Use `{config_option}` in the Serve config instead.",
+            DeprecationWarning,
+            stacklevel=2,
+        )

python/ray/serve/_private/controller_avatar.py

@@ -0,0 +1,45 @@
+import ray
+from ray.serve._private.constants import SERVE_CONTROLLER_NAME, SERVE_NAMESPACE
+from ray.serve._private.default_impl import get_controller_impl
+from ray.serve.config import HTTPOptions, ProxyLocation
+from ray.serve.schema import LoggingConfig
+
+
+@ray.remote(num_cpus=0)
+class ServeControllerAvatar:
+    """A hack that proxy the creation of async actors from Java.
+
+    To be removed after https://github.com/ray-project/ray/pull/26037
+
+    Java api can not support python async actor. If we use java api create
+    python async actor. The async init method won't be executed. The async
+    method will fail with pickle error. And the run_control_loop of controller
+    actor can't be executed too. We use this proxy actor create python async
+    actor to avoid the above problem.
+    """
+
+    def __init__(
+        self,
+        http_proxy_port: int = 8000,
+    ):
+        try:
+            self._controller = ray.get_actor(
+                SERVE_CONTROLLER_NAME, namespace=SERVE_NAMESPACE
+            )
+        except ValueError:
+            self._controller = None
+        if self._controller is None:
+            controller_impl = get_controller_impl()
+            # This Java bootstrap builds HTTPOptions directly and previously
+            # relied on the (now removed) HeadOnly default of
+            # HTTPOptions.location. Pass proxy_location explicitly to preserve
+            # head-only proxy placement on multi-node clusters.
+            self._controller = controller_impl.remote(
+                http_options=HTTPOptions(port=http_proxy_port),
+                proxy_location=ProxyLocation.HeadOnly,
+                global_logging_config=LoggingConfig(),
+            )
+
+    def check_alive(self) -> None:
+        """No-op to check if this actor is alive."""
+        return

python/ray/serve/_private/controller_health_metrics_tracker.py

@@ -0,0 +1,147 @@
+import asyncio
+import sys
+import time
+from collections import deque
+from dataclasses import dataclass, field
+from typing import Deque
+
+from ray.serve._private.constants import CONTROL_LOOP_INTERVAL_S
+from ray.serve.schema import ControllerHealthMetrics, DurationStats
+
+# Number of recent loop iterations to track for rolling averages
+_HEALTH_METRICS_HISTORY_SIZE = 100
+
+
+@dataclass
+class ControllerHealthMetricsTracker:
+    """Tracker for collecting controller health metrics over time."""
+
+    controller_start_time: float = field(default_factory=time.time)
+
+    # Rolling history of loop durations
+    loop_durations: Deque[float] = field(
+        default_factory=lambda: deque(maxlen=_HEALTH_METRICS_HISTORY_SIZE)
+    )
+
+    # Rolling history of metrics delays
+    handle_metrics_delays: Deque[float] = field(
+        default_factory=lambda: deque(maxlen=_HEALTH_METRICS_HISTORY_SIZE)
+    )
+    replica_metrics_delays: Deque[float] = field(
+        default_factory=lambda: deque(maxlen=_HEALTH_METRICS_HISTORY_SIZE)
+    )
+
+    # Rolling history of component update durations
+    dsm_update_durations: Deque[float] = field(
+        default_factory=lambda: deque(maxlen=_HEALTH_METRICS_HISTORY_SIZE)
+    )
+    asm_update_durations: Deque[float] = field(
+        default_factory=lambda: deque(maxlen=_HEALTH_METRICS_HISTORY_SIZE)
+    )
+    proxy_update_durations: Deque[float] = field(
+        default_factory=lambda: deque(maxlen=_HEALTH_METRICS_HISTORY_SIZE)
+    )
+    node_update_durations: Deque[float] = field(
+        default_factory=lambda: deque(maxlen=_HEALTH_METRICS_HISTORY_SIZE)
+    )
+
+    # Latest values (used in collect_metrics)
+    last_sleep_duration_s: float = 0.0
+    num_control_loops: int = 0
+    last_control_loop_time: float = 0.0
+
+    def record_loop_duration(self, duration: float):
+        self.loop_durations.append(duration)
+
+    def record_handle_metrics_delay(self, delay_ms: float):
+        self.handle_metrics_delays.append(delay_ms)
+
+    def record_replica_metrics_delay(self, delay_ms: float):
+        self.replica_metrics_delays.append(delay_ms)
+
+    def record_dsm_update_duration(self, duration: float):
+        self.dsm_update_durations.append(duration)
+
+    def record_asm_update_duration(self, duration: float):
+        self.asm_update_durations.append(duration)
+
+    def record_proxy_update_duration(self, duration: float):
+        self.proxy_update_durations.append(duration)
+
+    def record_node_update_duration(self, duration: float):
+        self.node_update_durations.append(duration)
+
+    def collect_metrics(self) -> ControllerHealthMetrics:
+        """Collect and return current health metrics."""
+        now = time.time()
+
+        # Calculate loop statistics from rolling history
+        loop_duration_stats = DurationStats.from_values(list(self.loop_durations))
+
+        # Calculate loops per second based on uptime and total loops
+        uptime = now - self.controller_start_time
+        loops_per_second = self.num_control_loops / uptime if uptime > 0 else 0.0
+
+        # Calculate event loop delay (actual sleep - expected sleep)
+        # Positive values indicate the event loop is overloaded
+        event_loop_delay = max(
+            0.0, self.last_sleep_duration_s - CONTROL_LOOP_INTERVAL_S
+        )
+
+        # Get asyncio task count
+        try:
+            loop = asyncio.get_event_loop()
+            num_asyncio_tasks = len(asyncio.all_tasks(loop))
+        except RuntimeError:
+            num_asyncio_tasks = 0
+
+        # Calculate metrics delay statistics
+        handle_delay_stats = DurationStats.from_values(list(self.handle_metrics_delays))
+        replica_delay_stats = DurationStats.from_values(
+            list(self.replica_metrics_delays)
+        )
+
+        # Calculate component update duration statistics
+        dsm_update_stats = DurationStats.from_values(list(self.dsm_update_durations))
+        asm_update_stats = DurationStats.from_values(list(self.asm_update_durations))
+        proxy_update_stats = DurationStats.from_values(
+            list(self.proxy_update_durations)
+        )
+        node_update_stats = DurationStats.from_values(list(self.node_update_durations))
+
+        # Get memory usage in MB
+        # Note: ru_maxrss is in bytes on macOS but kilobytes on Linux
+        # The resource module is Unix-only, so we handle Windows gracefully
+        try:
+            import resource
+
+            rusage = resource.getrusage(resource.RUSAGE_SELF)
+            process_memory_mb = (
+                rusage.ru_maxrss / (1024 * 1024)  # Convert bytes to MB on macOS
+                if sys.platform == "darwin"
+                else rusage.ru_maxrss / 1024  # Convert KB to MB on Linux
+            )
+        except ImportError:
+            # resource module not available on Windows
+            process_memory_mb = 0.0
+
+        return ControllerHealthMetrics(
+            timestamp=now,
+            controller_start_time=self.controller_start_time,
+            uptime_s=uptime,
+            last_control_loop_time=self.last_control_loop_time,
+            num_control_loops=self.num_control_loops,
+            loop_duration_s=loop_duration_stats,
+            loops_per_second=loops_per_second,
+            last_sleep_duration_s=self.last_sleep_duration_s,
+            expected_sleep_duration_s=CONTROL_LOOP_INTERVAL_S,
+            event_loop_delay_s=event_loop_delay,
+            num_asyncio_tasks=num_asyncio_tasks,
+            deployment_state_update_duration_s=dsm_update_stats,
+            application_state_update_duration_s=asm_update_stats,
+            proxy_state_update_duration_s=proxy_update_stats,
+            node_update_duration_s=node_update_stats,
+            handle_metrics_delay_ms=handle_delay_stats,
+            replica_metrics_delay_ms=replica_delay_stats,
+            process_memory_mb=process_memory_mb,
+        )

python/ray/serve/_private/default_impl.py

@@ -0,0 +1,271 @@
+import asyncio
+from typing import Callable, Optional, Tuple
+
+import ray
+from ray._common.constants import HEAD_NODE_RESOURCE_NAME
+from ray._raylet import GcsClient
+from ray.serve._private.cluster_node_info_cache import (
+    ClusterNodeInfoCache,
+    DefaultClusterNodeInfoCache,
+)
+from ray.serve._private.common import (
+    CreatePlacementGroupRequest,
+    DeploymentHandleSource,
+    DeploymentID,
+    EndpointInfo,
+    RequestMetadata,
+    RequestProtocol,
+)
+from ray.serve._private.constants import (
+    CONTROLLER_MAX_CONCURRENCY,
+    RAY_SERVE_ENABLE_TASK_EVENTS,
+    RAY_SERVE_PROXY_PREFER_LOCAL_NODE_ROUTING,
+    RAY_SERVE_PROXY_USE_GRPC,
+    RAY_SERVE_RUN_ROUTER_IN_SEPARATE_LOOP,
+    SERVE_CONTROLLER_NAME,
+    SERVE_NAMESPACE,
+)
+from ray.serve._private.deployment_scheduler import (
+    DefaultDeploymentScheduler,
+    DeploymentScheduler,
+)
+from ray.serve._private.event_loop_monitoring import EventLoopMonitor
+from ray.serve._private.grpc_util import gRPCGenericServer
+from ray.serve._private.handle_options import DynamicHandleOptions, InitHandleOptions
+from ray.serve._private.router import CurrentLoopRouter, Router, SingletonThreadRouter
+from ray.serve._private.utils import (
+    asyncio_grpc_exception_handler,
+    generate_request_id,
+    get_current_actor_id,
+    get_head_node_id,
+    inside_ray_client_context,
+    resolve_deployment_response,
+)
+from ray.serve.config import ControllerOptions
+from ray.util.placement_group import PlacementGroup
+
+# NOTE: Please read carefully before changing!
+#
+# These methods are common extension points, therefore these should be
+# changed as a Developer API, ie methods should not be renamed, have their
+# API modified w/o substantial enough justification
+
+
+def create_cluster_node_info_cache(gcs_client: GcsClient) -> ClusterNodeInfoCache:
+    return DefaultClusterNodeInfoCache(gcs_client)
+
+
+CreatePlacementGroupFn = Callable[[CreatePlacementGroupRequest], PlacementGroup]
+
+
+def _default_create_placement_group(
+    request: CreatePlacementGroupRequest,
+) -> PlacementGroup:
+    return ray.util.placement_group(
+        request.bundles,
+        request.strategy,
+        _soft_target_node_id=request.target_node_id,
+        name=request.name,
+        lifetime="detached",
+        bundle_label_selector=request.bundle_label_selector,
+    )
+
+
+def create_deployment_scheduler(
+    cluster_node_info_cache: ClusterNodeInfoCache,
+    head_node_id_override: Optional[str] = None,
+    create_placement_group_fn_override: Optional[CreatePlacementGroupFn] = None,
+) -> DeploymentScheduler:
+    head_node_id = head_node_id_override or get_head_node_id()
+    return DefaultDeploymentScheduler(
+        cluster_node_info_cache,
+        head_node_id,
+        create_placement_group_fn=create_placement_group_fn_override
+        or _default_create_placement_group,
+    )
+
+
+def create_replica_impl(**kwargs):
+    from ray.serve._private.replica import Replica
+
+    return Replica(**kwargs)
+
+
+def create_replica_metrics_manager(**kwargs):
+    from ray.serve._private.replica import ReplicaMetricsManager
+
+    return ReplicaMetricsManager(**kwargs)
+
+
+def create_dynamic_handle_options(**kwargs):
+    return DynamicHandleOptions(**kwargs)
+
+
+def create_init_handle_options(**kwargs):
+    return InitHandleOptions.create(**kwargs)
+
+
+def get_request_metadata(init_options, handle_options):
+    _request_context = ray.serve.context._get_serve_request_context()
+
+    request_protocol = RequestProtocol.UNDEFINED
+    if init_options and init_options._source == DeploymentHandleSource.PROXY:
+        if _request_context.is_http_request:
+            request_protocol = RequestProtocol.HTTP
+        elif _request_context.grpc_context:
+            request_protocol = RequestProtocol.GRPC
+
+    return RequestMetadata(
+        request_id=_request_context.request_id
+        if _request_context.request_id
+        else generate_request_id(),
+        internal_request_id=_request_context._internal_request_id
+        if _request_context._internal_request_id
+        else generate_request_id(),
+        call_method=handle_options.method_name,
+        route=_request_context.route,
+        app_name=_request_context.app_name,
+        multiplexed_model_id=handle_options.multiplexed_model_id,
+        session_id=handle_options.session_id,
+        is_streaming=handle_options.stream,
+        _request_protocol=request_protocol,
+        grpc_context=_request_context.grpc_context,
+        _client=_request_context._client,
+        _by_reference=handle_options._by_reference,
+        _on_separate_loop=init_options._run_router_in_separate_loop,
+        request_serialization=handle_options.request_serialization,
+        response_serialization=handle_options.response_serialization,
+    )
+
+
+def _get_node_id_and_az() -> Tuple[str, Optional[str]]:
+    node_id = ray.get_runtime_context().get_node_id()
+    try:
+        cluster_node_info_cache = create_cluster_node_info_cache(
+            GcsClient(address=ray.get_runtime_context().gcs_address)
+        )
+        cluster_node_info_cache.update()
+        az = cluster_node_info_cache.get_node_az(node_id)
+    except Exception:
+        az = None
+
+    return node_id, az
+
+
+# Interface definition for create_router.
+CreateRouterCallable = Callable[[str, DeploymentID, InitHandleOptions], Router]
+
+
+def create_router(
+    handle_id: str,
+    deployment_id: DeploymentID,
+    handle_options: InitHandleOptions,
+    request_router_class: Optional[Callable] = None,
+) -> Router:
+    # NOTE(edoakes): this is lazy due to a nasty circular import that should be fixed.
+    from ray.serve.context import _get_global_client
+
+    actor_id = get_current_actor_id()
+    node_id, availability_zone = _get_node_id_and_az()
+    controller_handle = _get_global_client()._controller
+    is_inside_ray_client_context = inside_ray_client_context()
+
+    if handle_options._run_router_in_separate_loop:
+        router_wrapper_cls = SingletonThreadRouter
+        # Determine the component for the event loop monitor
+        if handle_options._source == DeploymentHandleSource.REPLICA:
+            component = EventLoopMonitor.COMPONENT_REPLICA
+        elif handle_options._source == DeploymentHandleSource.PROXY:
+            component = EventLoopMonitor.COMPONENT_PROXY
+        else:
+            component = EventLoopMonitor.COMPONENT_UNKNOWN
+        SingletonThreadRouter._get_singleton_asyncio_loop(
+            component
+        ).set_exception_handler(asyncio_grpc_exception_handler)
+    else:
+        try:
+            asyncio.get_running_loop()
+        except RuntimeError:
+            raise RuntimeError(
+                "No event loop running. You cannot use a handle initialized with "
+                "`_run_router_in_separate_loop=False` when not inside an asyncio event "
+                "loop."
+            )
+
+        router_wrapper_cls = CurrentLoopRouter
+
+    return router_wrapper_cls(
+        controller_handle=controller_handle,
+        deployment_id=deployment_id,
+        handle_id=handle_id,
+        self_actor_id=actor_id,
+        handle_source=handle_options._source,
+        request_router_class=request_router_class,
+        # Streaming ObjectRefGenerators are not supported in Ray Client
+        enable_strict_max_ongoing_requests=not is_inside_ray_client_context,
+        resolve_request_arg_func=resolve_deployment_response,
+        node_id=node_id,
+        availability_zone=availability_zone,
+        prefer_local_node_routing=handle_options._prefer_local_routing,
+    )
+
+
+def add_grpc_address(grpc_server: gRPCGenericServer, server_address: str):
+    """Helper function to add an address to a gRPC server."""
+    grpc_server.add_insecure_port(server_address)
+
+
+def get_proxy_handle(endpoint: DeploymentID, info: EndpointInfo):
+    # NOTE(zcin): needs to be lazy import due to a circular dependency.
+    # We should not be importing from application_state in context.
+    from ray.serve.context import _get_global_client
+
+    client = _get_global_client()
+    handle = client.get_handle(endpoint.name, endpoint.app_name, check_exists=True)
+
+    # NOTE(zcin): It's possible that a handle is already initialized
+    # if a deployment with the same name and application name was
+    # deleted, then redeployed later. However this is not an issue since
+    # we initialize all handles with the same init options.
+    if not handle.is_initialized:
+        # NOTE(zcin): since the router is eagerly initialized here, the
+        # proxy will receive the replica set from the controller early.
+        handle._init(
+            _prefer_local_routing=RAY_SERVE_PROXY_PREFER_LOCAL_NODE_ROUTING,
+            _source=DeploymentHandleSource.PROXY,
+            _run_router_in_separate_loop=RAY_SERVE_RUN_ROUTER_IN_SEPARATE_LOOP,
+        )
+
+    return handle.options(
+        stream=not info.app_is_cross_language,
+        _by_reference=not RAY_SERVE_PROXY_USE_GRPC,
+    )
+
+
+def get_controller_impl(controller_options: Optional[ControllerOptions] = None):
+    """Build the Ray actor class for the Serve controller.
+
+    ``controller_options`` is the validated ``ControllerOptions`` model from
+    ``serve.start`` / ``serve.run`` / the YAML schema. Today only its
+    ``runtime_env`` field is consumed; future fields (num_cpus, resources,
+    max_concurrency overrides) slot in here.
+    """
+    from ray.serve._private.controller import ServeController
+
+    actor_options = dict(
+        name=SERVE_CONTROLLER_NAME,
+        namespace=SERVE_NAMESPACE,
+        num_cpus=0,
+        lifetime="detached",
+        max_restarts=-1,
+        max_task_retries=-1,
+        resources={HEAD_NODE_RESOURCE_NAME: 0.001},
+        max_concurrency=CONTROLLER_MAX_CONCURRENCY,
+        enable_task_events=RAY_SERVE_ENABLE_TASK_EVENTS,
+    )
+    if controller_options is not None and controller_options.runtime_env:
+        # The validator on ControllerOptions guarantees this is a dict
+        # containing only the ``env_vars`` key with str->str entries.
+        actor_options["runtime_env"] = controller_options.runtime_env
+
+    return ray.remote(**actor_options)(ServeController)

python/ray/serve/_private/deploy_utils.py

@@ -0,0 +1,180 @@
+import hashlib
+import json
+import logging
+import time
+from typing import Any, Dict, Optional, Union
+
+import ray
+import ray.util.serialization_addons
+from ray.serve._private.common import DeploymentID
+from ray.serve._private.config import DeploymentConfig, ReplicaConfig
+from ray.serve._private.constants import (
+    RAY_SERVE_DIRECT_INGRESS_MIN_DRAINING_PERIOD_S,
+    RAY_SERVE_DIRECT_INGRESS_SHUTDOWN_BUFFER_S,
+    RAY_SERVE_ENABLE_DIRECT_INGRESS,
+    SERVE_LOGGER_NAME,
+)
+from ray.serve._private.deployment_info import DeploymentInfo
+from ray.serve.exceptions import RayServeException
+from ray.serve.schema import ServeApplicationSchema
+
+logger = logging.getLogger(SERVE_LOGGER_NAME)
+
+
+def get_deploy_args(
+    name: str,
+    replica_config: ReplicaConfig,
+    ingress: bool = False,
+    ingress_request_router: bool = False,
+    deployment_config: Optional[Union[DeploymentConfig, Dict[str, Any]]] = None,
+    version: Optional[str] = None,
+    route_prefix: Optional[str] = None,
+    serialized_autoscaling_policy_def: Optional[bytes] = None,
+    serialized_request_router_cls: Optional[bytes] = None,
+    serialized_deployment_actors: Optional[Dict[str, bytes]] = None,
+    uses_multiplexing: bool = False,
+) -> Dict:
+    """
+    Takes a deployment's configuration, and returns the arguments needed
+    for the controller to deploy it.
+    """
+    if deployment_config is None:
+        deployment_config = {}
+
+    if isinstance(deployment_config, dict):
+        deployment_config = DeploymentConfig.model_validate(deployment_config)
+    elif not isinstance(deployment_config, DeploymentConfig):
+        raise TypeError("config must be a DeploymentConfig or a dictionary.")
+
+    deployment_config.version = version
+
+    controller_deploy_args = {
+        "deployment_name": name,
+        "deployment_config_proto_bytes": deployment_config.to_proto_bytes(),
+        "replica_config_proto_bytes": replica_config.to_proto_bytes(),
+        "route_prefix": route_prefix,
+        "deployer_job_id": ray.get_runtime_context().get_job_id(),
+        "ingress": ingress,
+        "ingress_request_router": ingress_request_router,
+        "serialized_autoscaling_policy_def": serialized_autoscaling_policy_def,
+        "serialized_request_router_cls": serialized_request_router_cls,
+        "serialized_deployment_actors": serialized_deployment_actors,
+        "uses_multiplexing": uses_multiplexing,
+    }
+
+    return controller_deploy_args
+
+
+def deploy_args_to_deployment_info(
+    deployment_name: str,
+    deployment_config_proto_bytes: bytes,
+    replica_config_proto_bytes: bytes,
+    deployer_job_id: Union[str, bytes],
+    app_name: Optional[str] = None,
+    ingress: bool = False,
+    ingress_request_router: bool = False,
+    route_prefix: Optional[str] = None,
+    uses_multiplexing: bool = False,
+    **kwargs,
+) -> DeploymentInfo:
+    """Takes deployment args passed to the controller after building an application and
+    constructs a DeploymentInfo object.
+    """
+
+    deployment_config = DeploymentConfig.from_proto_bytes(deployment_config_proto_bytes)
+
+    if ingress and RAY_SERVE_ENABLE_DIRECT_INGRESS:
+        # Model multiplexing relies on the multiplexed model ID being propagated through
+        # the proxy, which direct ingress bypasses (the model ID is never populated).
+        # Only the *statically* detectable case is caught here; dynamically-initialized
+        # multiplexing is caught at replica initialization.
+        if uses_multiplexing:
+            raise RayServeException(
+                f'Ingress deployment "{deployment_name}" in application "{app_name}" uses '
+                "model multiplexing (`@serve.multiplexed`), which is not supported on the "
+                "ingress deployment when direct ingress or HAProxy is enabled."
+            )
+
+        # Floor the timeout so the controller's force-kill can't cut the
+        # direct-ingress drain (min draining period) short.
+        floor_s = (
+            RAY_SERVE_DIRECT_INGRESS_MIN_DRAINING_PERIOD_S
+            + RAY_SERVE_DIRECT_INGRESS_SHUTDOWN_BUFFER_S
+        )
+        if deployment_config.graceful_shutdown_timeout_s < floor_s:
+            logger.info(
+                f"Raising graceful_shutdown_timeout_s for ingress deployment "
+                f"'{deployment_name}' from "
+                f"{deployment_config.graceful_shutdown_timeout_s}s to {floor_s}s so "
+                f"the force-kill deadline covers the direct-ingress drain period."
+            )
+            deployment_config.graceful_shutdown_timeout_s = floor_s
+
+    version = deployment_config.version
+    replica_config = ReplicaConfig.from_proto_bytes(
+        replica_config_proto_bytes, deployment_config.needs_pickle()
+    )
+
+    # Java API passes in JobID as bytes
+    if isinstance(deployer_job_id, bytes):
+        deployer_job_id = ray.JobID.from_int(
+            int.from_bytes(deployer_job_id, "little")
+        ).hex()
+
+    return DeploymentInfo(
+        actor_name=DeploymentID(
+            name=deployment_name, app_name=app_name
+        ).to_replica_actor_class_name(),
+        version=version,
+        deployment_config=deployment_config,
+        replica_config=replica_config,
+        deployer_job_id=deployer_job_id,
+        start_time_ms=int(time.time() * 1000),
+        route_prefix=route_prefix,
+        ingress=ingress,
+        ingress_request_router=ingress_request_router,
+    )
+
+
+def get_app_code_version(app_config: ServeApplicationSchema) -> str:
+    """Returns the code version of an application.
+
+    Args:
+        app_config: The application config.
+
+    Returns:
+        str: A hash of the import path and (application level) runtime env
+            representing the code version of the application.
+    """
+    request_router_configs = [
+        deployment.request_router_config
+        for deployment in app_config.deployments
+        if isinstance(deployment.request_router_config, dict)
+    ]
+    deployment_autoscaling_policies = [
+        deployment_config.autoscaling_config.get("policy", None)
+        for deployment_config in app_config.deployments
+        if isinstance(deployment_config.autoscaling_config, dict)
+    ]
+    deployment_actors_configs = [
+        deployment.deployment_actors
+        for deployment in app_config.deployments
+        if isinstance(deployment.deployment_actors, list)
+    ]
+
+    encoded = json.dumps(
+        {
+            "import_path": app_config.import_path,
+            "runtime_env": app_config.runtime_env,
+            "args": app_config.args,
+            # NOTE: trigger a change in the code version when
+            # application level autoscaling policy is changed or
+            # any one of the deployment level autoscaling policy is changed
+            "autoscaling_policy": app_config.autoscaling_policy,
+            "deployment_autoscaling_policies": deployment_autoscaling_policies,
+            "request_router_configs": request_router_configs,
+            "deployment_actors": deployment_actors_configs,
+        },
+        sort_keys=True,
+    ).encode("utf-8")
+    return hashlib.sha256(encoded).hexdigest()

python/ray/serve/_private/deployment_info.py

@@ -0,0 +1,189 @@
+from typing import Any, Dict, Optional
+
+import ray
+from ray.serve._private.common import TargetCapacityDirection
+from ray.serve._private.config import DeploymentConfig, ReplicaConfig
+from ray.serve.generated.serve_pb2 import (
+    DeploymentInfo as DeploymentInfoProto,
+    TargetCapacityDirection as TargetCapacityDirectionProto,
+)
+
+
+class DeploymentInfo:
+    def __init__(
+        self,
+        deployment_config: DeploymentConfig,
+        replica_config: ReplicaConfig,
+        start_time_ms: int,
+        deployer_job_id: str,
+        actor_name: Optional[str] = None,
+        version: Optional[str] = None,
+        end_time_ms: Optional[int] = None,
+        route_prefix: str = None,
+        ingress: bool = False,
+        ingress_request_router: bool = False,
+        target_capacity: Optional[float] = None,
+        target_capacity_direction: Optional[TargetCapacityDirection] = None,
+    ):
+        self.deployment_config = deployment_config
+        self.replica_config = replica_config
+        # The time when .deploy() was first called for this deployment.
+        self.start_time_ms = start_time_ms
+        self.actor_name = actor_name
+        self.version = version
+        self.deployer_job_id = deployer_job_id
+        # The time when this deployment was deleted.
+        self.end_time_ms = end_time_ms
+
+        # ephermal state
+        self._cached_actor_def = None
+
+        self.route_prefix = route_prefix
+        self.ingress = ingress
+        self.ingress_request_router = ingress_request_router
+
+        self.target_capacity = target_capacity
+        self.target_capacity_direction = target_capacity_direction
+
+    def __getstate__(self) -> Dict[Any, Any]:
+        clean_dict = self.__dict__.copy()
+        del clean_dict["_cached_actor_def"]
+        return clean_dict
+
+    def __setstate__(self, d: Dict[Any, Any]) -> None:
+        self.__dict__ = d
+        self._cached_actor_def = None
+
+    def update(
+        self,
+        deployment_config: DeploymentConfig = None,
+        replica_config: ReplicaConfig = None,
+        version: str = None,
+        route_prefix: str = None,
+    ) -> "DeploymentInfo":
+        return DeploymentInfo(
+            deployment_config=deployment_config or self.deployment_config,
+            replica_config=replica_config or self.replica_config,
+            start_time_ms=self.start_time_ms,
+            deployer_job_id=self.deployer_job_id,
+            actor_name=self.actor_name,
+            version=version or self.version,
+            end_time_ms=self.end_time_ms,
+            route_prefix=route_prefix or self.route_prefix,
+            ingress=self.ingress,
+            ingress_request_router=self.ingress_request_router,
+            target_capacity=self.target_capacity,
+            target_capacity_direction=self.target_capacity_direction,
+        )
+
+    def set_target_capacity(
+        self,
+        new_target_capacity: Optional[float],
+        new_target_capacity_direction: Optional[TargetCapacityDirection],
+    ):
+        self.target_capacity = new_target_capacity
+        self.target_capacity_direction = new_target_capacity_direction
+
+    def config_changed(self, other) -> bool:
+        return (
+            self.deployment_config != other.deployment_config
+            or self.replica_config.ray_actor_options
+            != other.replica_config.ray_actor_options
+            or other.version is None
+            or self.version != other.version
+        )
+
+    @property
+    def actor_def(self):
+        if self._cached_actor_def is None:
+            assert self.actor_name is not None
+
+            # Break circular import :(.
+            from ray.serve._private.replica import ReplicaActor
+
+            # Dynamically create a new class with custom name here so Ray picks it up
+            # correctly in actor metadata table and observability stack.
+            self._cached_actor_def = ray.remote(
+                type(
+                    self.actor_name,
+                    (ReplicaActor,),
+                    dict(ReplicaActor.__dict__),
+                )
+            )
+
+        return self._cached_actor_def
+
+    @classmethod
+    def from_proto(cls, proto: DeploymentInfoProto):
+        deployment_config = (
+            DeploymentConfig.from_proto(proto.deployment_config)
+            if proto.deployment_config
+            else None
+        )
+
+        target_capacity = proto.target_capacity if proto.target_capacity != -1 else None
+
+        target_capacity_direction = TargetCapacityDirectionProto.Name(
+            proto.target_capacity_direction
+        )
+        if target_capacity_direction == "UNSET":
+            target_capacity_direction = None
+        else:
+            target_capacity_direction = TargetCapacityDirection(
+                target_capacity_direction
+            )
+
+        data = {
+            "deployment_config": deployment_config,
+            "replica_config": ReplicaConfig.from_proto(
+                proto.replica_config,
+                deployment_config.needs_pickle() if deployment_config else True,
+            ),
+            "start_time_ms": proto.start_time_ms,
+            "actor_name": proto.actor_name if proto.actor_name != "" else None,
+            "version": proto.version if proto.version != "" else None,
+            "end_time_ms": proto.end_time_ms if proto.end_time_ms != 0 else None,
+            "deployer_job_id": ray.get_runtime_context().get_job_id(),
+            "target_capacity": target_capacity,
+            "target_capacity_direction": target_capacity_direction,
+            "ingress_request_router": proto.ingress_request_router,
+        }
+
+        return cls(**data)
+
+    def to_proto(self):
+        data = {
+            "start_time_ms": self.start_time_ms,
+            "actor_name": self.actor_name,
+            "version": self.version,
+            "end_time_ms": self.end_time_ms,
+        }
+        if self.deployment_config:
+            data["deployment_config"] = self.deployment_config.to_proto()
+        if self.replica_config:
+            data["replica_config"] = self.replica_config.to_proto()
+        if self.target_capacity is None:
+            data["target_capacity"] = -1
+        else:
+            data["target_capacity"] = self.target_capacity
+        if self.target_capacity_direction is None:
+            data["target_capacity_direction"] = TargetCapacityDirectionProto.UNSET
+        else:
+            data["target_capacity_direction"] = self.target_capacity_direction.name
+        data["ingress_request_router"] = self.ingress_request_router
+        return DeploymentInfoProto(**data)
+
+    def to_dict(self):
+        # only use for logging purposes
+        return {
+            "deployment_config": (
+                self.deployment_config.to_dict() if self.deployment_config else None
+            ),
+            "replica_config": (
+                self.replica_config.to_dict() if self.replica_config else None
+            ),
+            "start_time_ms": self.start_time_ms,
+            "actor_name": self.actor_name,
+            "version": self.version,
+            "end_time_ms": self.end_time_ms,
+        }

python/ray/serve/_private/deployment_node.py

@@ -0,0 +1,56 @@
+from typing import Any, Dict, List, Optional, Tuple
+
+from ray.dag import DAGNode
+from ray.dag.format_utils import get_dag_node_str
+from ray.serve.deployment import Deployment
+from ray.serve.handle import DeploymentHandle
+
+
+class DeploymentNode(DAGNode):
+    """Represents a deployment node in a DAG authored Ray DAG API."""
+
+    def __init__(
+        self,
+        # For serve structured deployment, deployment body can be import path
+        # to the class or function instead.
+        deployment: Deployment,
+        app_name: str,
+        deployment_init_args: Tuple[Any],
+        deployment_init_kwargs: Dict[str, Any],
+        ray_actor_options: Dict[str, Any],
+        other_args_to_resolve: Optional[Dict[str, Any]] = None,
+    ):
+        # Assign instance variables in base class constructor.
+        super().__init__(
+            deployment_init_args,
+            deployment_init_kwargs,
+            ray_actor_options,
+            other_args_to_resolve=other_args_to_resolve,
+        )
+        self._app_name = app_name
+        self._deployment = deployment
+        self._deployment_handle = DeploymentHandle(
+            self._deployment.name, self._app_name
+        )
+
+    def _copy_impl(
+        self,
+        new_args: List[Any],
+        new_kwargs: Dict[str, Any],
+        new_options: Dict[str, Any],
+        new_other_args_to_resolve: Dict[str, Any],
+    ):
+        return DeploymentNode(
+            self._deployment,
+            self._app_name,
+            new_args,
+            new_kwargs,
+            new_options,
+            other_args_to_resolve=new_other_args_to_resolve,
+        )
+
+    def __str__(self) -> str:
+        return get_dag_node_str(self, str(self._deployment))
+
+    def get_deployment_name(self):
+        return self._deployment.name

python/ray/serve/_private/direct_ingress_grpc_util.py

@@ -0,0 +1,105 @@
+import asyncio
+from typing import Any, AsyncIterator, Optional, TypeVar
+
+from ray._common.utils import get_or_create_event_loop
+
+T = TypeVar("T")
+
+# Sentinel placed on the queue to signal that the request stream has ended.
+_STREAM_END = object()
+
+
+class gRPCDIReceiveStream(AsyncIterator[T]):
+    """Exposes a native gRPC request iterator to user code for direct ingress.
+
+    The native iterator is bound to the replica's server event loop, but user code
+    may run on a separate event loop. When the loops differ, a fetch task drains the
+    iterator on the server loop and hands messages to user code across loops; when
+    they're the same, user code iterates the native iterator directly.
+    """
+
+    def __init__(
+        self,
+        request_iterator: AsyncIterator[T],
+        user_event_loop: asyncio.AbstractEventLoop,
+        *,
+        cancel_event: Optional[asyncio.Event] = None,
+    ):
+        self._request_iterator = request_iterator
+        self._user_event_loop = user_event_loop
+        self._same_event_loop = user_event_loop is get_or_create_event_loop()
+        # Set when the client disconnects/errors mid-stream so the consumer's
+        # gRPCInputStream reports is_cancelled() and ends gracefully.
+        self._cancel_event = cancel_event
+        # Lazily created so it binds to the user-code event loop (where it is
+        # always accessed), never the server loop.
+        self._queue: Optional[asyncio.Queue] = None
+        self._fetch_task: Optional[asyncio.Task] = None
+
+    @property
+    def _message_queue(self) -> asyncio.Queue:
+        if self._queue is None:
+            self._queue = asyncio.Queue()
+        return self._queue
+
+    def _put(self, item: Any):
+        self._message_queue.put_nowait(item)
+
+    def start(self) -> Optional[asyncio.Task]:
+        """Start draining the native iterator. Must be called on the server loop."""
+        # We don't create another task to consume the input stream if we run on the
+        # same event loop.
+        if self._same_event_loop:
+            return
+
+        self._fetch_task = asyncio.ensure_future(self._fetch_until_done())
+        return self._fetch_task
+
+    def cancel(self):
+        """Stop draining the native iterator (e.g. if the consumer finished)."""
+        if self._fetch_task is not None:
+            self._fetch_task.cancel()
+
+    async def _fetch_until_done(self):
+        """Drive the native iterator on the server loop, forwarding each message.
+
+        A stream error (e.g. client disconnect) is treated as cancellation: the
+        cancel event is set and the stream ends gracefully -- mirroring the proxy
+        path (`receive_grpc_messages`) -- rather than surfacing the raw gRPC error
+        to user code. A sentinel is always enqueued at the end so the consumer
+        terminates.
+        """
+        try:
+            async for message in self._request_iterator:
+                # Stop draining if the consumer cancelled (e.g. user code called
+                # input_stream.cancel()).
+                if self._cancel_event is not None and self._cancel_event.is_set():
+                    break
+                self._user_event_loop.call_soon_threadsafe(self._put, message)
+        except asyncio.CancelledError:
+            raise
+        except Exception:
+            if self._cancel_event is not None:
+                self._user_event_loop.call_soon_threadsafe(self._cancel_event.set)
+        finally:
+            self._user_event_loop.call_soon_threadsafe(self._put, _STREAM_END)
+
+    def __aiter__(self) -> "gRPCDIReceiveStream":
+        return self
+
+    async def __anext__(self) -> T:
+        """Return the next request message. Runs on the user-code loop."""
+        if self._same_event_loop:
+            try:
+                return await anext(self._request_iterator)
+            except (asyncio.CancelledError, StopAsyncIteration):
+                raise
+            except Exception:
+                if self._cancel_event is not None:
+                    self._cancel_event.set()
+                raise StopAsyncIteration
+
+        item = await self._message_queue.get()
+        if item is _STREAM_END:
+            raise StopAsyncIteration
+        return item

python/ray/serve/_private/direct_ingress_http_util.py

@@ -0,0 +1,104 @@
+import asyncio
+import logging
+
+from starlette.types import Message, Receive, Scope
+
+from ray.serve._private.constants import (
+    SERVE_LOGGER_NAME,
+)
+
+logger = logging.getLogger(SERVE_LOGGER_NAME)
+
+
+class ASGIDIReceiveProxy:
+    """Proxies ASGI receive from an actor.
+
+    The `receive_asgi_messages` callback will be called repeatedly to fetch messages
+    until a disconnect message is received.
+    """
+
+    def __init__(
+        self,
+        scope: Scope,
+        receive: Receive,
+        user_event_loop: asyncio.AbstractEventLoop,
+    ):
+        self._type = scope["type"]  # Either 'http' or 'websocket'.
+        # Lazy init the queue to ensure it is created in the user code event loop.
+        self._queue = None
+        self._receive = receive
+        self._user_event_loop = user_event_loop
+        self._disconnect_message = None
+
+    def _get_default_disconnect_message(self) -> Message:
+        """Return the appropriate disconnect message based on the connection type.
+
+        HTTP ASGI spec:
+            https://asgi.readthedocs.io/en/latest/specs/www.html#disconnect-receive-event
+
+        WS ASGI spec:
+            https://asgi.readthedocs.io/en/latest/specs/www.html#disconnect-receive-event-ws
+        """
+        if self._type == "websocket":
+            return {
+                "type": "websocket.disconnect",
+                # 1005 is the default disconnect code according to the ASGI spec.
+                "code": 1005,
+            }
+        else:
+            return {"type": "http.disconnect"}
+
+    @property
+    def queue(self) -> asyncio.Queue:
+        if self._queue is None:
+            self._queue = asyncio.Queue()
+
+        return self._queue
+
+    def put_message(self, msg: Message):
+        self.queue.put_nowait(msg)
+
+    def close_queue(self):
+        self.queue.close()
+
+    def fetch_until_disconnect_task(self) -> asyncio.Task:
+        return asyncio.create_task(self._fetch_until_disconnect())
+
+    async def _fetch_until_disconnect(self):
+        """Fetch messages repeatedly until a disconnect message is received.
+
+        If a disconnect message is received, this function exits and returns it.
+
+        If an exception occurs, it will be raised on the next __call__ and no more
+        messages will be received.
+
+        Note that this is meant to be called in the system event loop.
+        """
+        while True:
+            msg = await self._receive()
+            if asyncio.get_running_loop() == self._user_event_loop:
+                await self.queue.put(msg)
+            else:
+                self._user_event_loop.call_soon_threadsafe(self.put_message, msg)
+
+            if msg["type"] == "http.disconnect":
+                self._disconnect_message = msg
+                return None
+
+            if msg["type"] == "websocket.disconnect":
+                self._disconnect_message = msg
+                return msg["code"]
+
+    async def __call__(self) -> Message:
+        """Return the next message once available.
+
+        This will repeatedly return a disconnect message once it's been received.
+        """
+        if self.queue.empty() and self._disconnect_message is not None:
+            return self._disconnect_message
+
+        message = await self.queue.get()
+        if isinstance(message, Exception):
+            raise message
+
+        return message

python/ray/serve/_private/endpoint_state.py

@@ -0,0 +1,109 @@
+import logging
+from typing import Any, Dict, Optional
+
+from ray import cloudpickle
+from ray.serve._private.common import DeploymentID, EndpointInfo
+from ray.serve._private.constants import SERVE_LOGGER_NAME
+from ray.serve._private.long_poll import LongPollHost, LongPollNamespace
+from ray.serve._private.storage.kv_store import KVStoreBase
+
+CHECKPOINT_KEY = "serve-endpoint-state-checkpoint"
+
+logger = logging.getLogger(SERVE_LOGGER_NAME)
+
+
+class EndpointState:
+    """Manages all state for endpoints in the system.
+
+    This class is *not* thread safe, so any state-modifying methods should be
+    called with a lock held.
+    """
+
+    def __init__(self, kv_store: KVStoreBase, long_poll_host: LongPollHost):
+        self._kv_store = kv_store
+        self._long_poll_host = long_poll_host
+        self._endpoints: Dict[DeploymentID, EndpointInfo] = dict()
+
+        checkpoint = self._kv_store.get(CHECKPOINT_KEY)
+        if checkpoint is not None:
+            self._endpoints = cloudpickle.loads(checkpoint)
+
+        self._notify_route_table_changed()
+
+    def shutdown(self):
+        self._kv_store.delete(CHECKPOINT_KEY)
+
+    def is_ready_for_shutdown(self) -> bool:
+        """Returns whether the endpoint checkpoint has been deleted.
+
+        Get the endpoint checkpoint from the kv store. If it is None, then it has been
+        deleted.
+        """
+        return self._kv_store.get(CHECKPOINT_KEY) is None
+
+    def _checkpoint(self):
+        self._kv_store.put(CHECKPOINT_KEY, cloudpickle.dumps(self._endpoints))
+
+    def _notify_route_table_changed(self):
+        self._long_poll_host.notify_changed(
+            {LongPollNamespace.ROUTE_TABLE: self._endpoints}
+        )
+
+    def _get_endpoint_for_route(self, route: str) -> Optional[DeploymentID]:
+        for endpoint, info in self._endpoints.items():
+            if info.route == route:
+                return endpoint
+
+        return None
+
+    def update_endpoint(
+        self, endpoint: DeploymentID, endpoint_info: EndpointInfo
+    ) -> None:
+        """Create or update the given endpoint.
+
+        This method is idempotent - if the endpoint already exists it will be
+        updated to match the given parameters. Calling this twice with the same
+        arguments is a no-op.
+        """
+
+        if self._endpoints.get(endpoint) == endpoint_info:
+            return
+
+        existing_route_endpoint = self._get_endpoint_for_route(endpoint_info.route)
+        if existing_route_endpoint is not None and existing_route_endpoint != endpoint:
+            logger.debug(
+                f'route_prefix "{endpoint_info.route}" is currently '
+                f'registered to deployment "{existing_route_endpoint.name}". '
+                f'Re-registering route_prefix "{endpoint_info.route}" to '
+                f'deployment "{endpoint.name}".'
+            )
+            del self._endpoints[existing_route_endpoint]
+
+        self._endpoints[endpoint] = endpoint_info
+
+        self._checkpoint()
+        self._notify_route_table_changed()
+
+    def get_endpoint_route(self, endpoint: DeploymentID) -> Optional[str]:
+        if endpoint in self._endpoints:
+            return self._endpoints[endpoint].route
+        return None
+
+    def get_endpoints(self) -> Dict[DeploymentID, Dict[str, Any]]:
+        endpoints = {}
+        for endpoint, info in self._endpoints.items():
+            endpoints[endpoint] = {
+                "route": info.route,
+            }
+        return endpoints
+
+    def delete_endpoint(self, endpoint: DeploymentID) -> None:
+        # This method must be idempotent. We should validate that the
+        # specified endpoint exists on the client.
+        if endpoint not in self._endpoints:
+            return
+
+        del self._endpoints[endpoint]
+
+        self._checkpoint()
+        self._notify_route_table_changed()

python/ray/serve/_private/event_loop_monitoring.py

@@ -0,0 +1,193 @@
+import asyncio
+import logging
+import time
+from typing import Dict, Optional
+
+from ray.serve._private.constants import (
+    RAY_SERVE_EVENT_LOOP_MONITORING_INTERVAL_S,
+    SERVE_EVENT_LOOP_LATENCY_HISTOGRAM_BOUNDARIES_MS,
+    SERVE_LOGGER_NAME,
+)
+from ray.util import metrics
+
+logger = logging.getLogger(SERVE_LOGGER_NAME)
+
+
+def setup_event_loop_monitoring(
+    loop: asyncio.AbstractEventLoop,
+    scheduling_latency: metrics.Histogram,
+    iterations: metrics.Counter,
+    tasks: metrics.Gauge,
+    tags: Dict[str, str],
+    interval_s: Optional[float] = None,
+) -> asyncio.Task:
+    """Start monitoring an event loop and recording metrics.
+
+    This function creates a background task that periodically measures:
+    - How long it takes for the event loop to wake up after sleeping
+      (scheduling latency / event loop lag)
+    - The number of pending asyncio tasks
+
+    Args:
+        loop: The asyncio event loop to monitor.
+        scheduling_latency: Histogram metric to record scheduling latency.
+        iterations: Counter metric to track monitoring iterations.
+        tasks: Gauge metric to track number of pending tasks.
+        tags: Dictionary of tags to apply to all metrics.
+        interval_s: Optional override for the monitoring interval.
+            Defaults to RAY_SERVE_EVENT_LOOP_MONITORING_INTERVAL_S.
+
+    Returns:
+        The asyncio Task running the monitoring loop.
+    """
+    if interval_s is None:
+        interval_s = RAY_SERVE_EVENT_LOOP_MONITORING_INTERVAL_S
+
+    return loop.create_task(
+        _run_monitoring_loop(
+            loop=loop,
+            schedule_latency=scheduling_latency,
+            iterations=iterations,
+            task_gauge=tasks,
+            tags=tags,
+            interval_s=interval_s,
+        ),
+        name="serve_event_loop_monitoring",
+    )
+
+
+async def _run_monitoring_loop(
+    loop: asyncio.AbstractEventLoop,
+    schedule_latency: metrics.Histogram,
+    iterations: metrics.Counter,
+    task_gauge: metrics.Gauge,
+    tags: Dict[str, str],
+    interval_s: float,
+) -> None:
+    """Internal monitoring loop that runs until the event loop stops.
+
+    The scheduling latency is measured by comparing the actual elapsed time
+    after sleeping to the expected sleep duration. In an ideal scenario
+    with no blocking, the latency should be close to zero.
+    """
+    while loop.is_running():
+        iterations.inc(1, tags)
+        num_tasks = len(asyncio.all_tasks(loop))
+        task_gauge.set(num_tasks, tags)
+        yield_time = time.monotonic()
+        await asyncio.sleep(interval_s)
+        elapsed_time = time.monotonic() - yield_time
+
+        # Historically, Ray's implementation of histograms are extremely finicky
+        # with non-positive values (https://github.com/ray-project/ray/issues/26698).
+        # Technically it shouldn't be possible for this to be negative, add the
+        # max just to be safe.
+        # Convert to milliseconds for the metric.
+        latency_ms = max(0.0, (elapsed_time - interval_s) * 1000)
+        schedule_latency.observe(latency_ms, tags)
+
+
+class EventLoopMonitor:
+    TAG_KEY_COMPONENT = "component"
+    TAG_KEY_LOOP_TYPE = "loop_type"
+    TAG_KEY_ACTOR_ID = "actor_id"
+
+    # Component types
+    COMPONENT_PROXY = "proxy"
+    COMPONENT_REPLICA = "replica"
+    COMPONENT_UNKNOWN = "unknown"
+
+    # Loop types
+    LOOP_TYPE_MAIN = "main"
+    LOOP_TYPE_USER_CODE = "user_code"
+    LOOP_TYPE_ROUTER = "router"
+
+    def __init__(
+        self,
+        component: str,
+        loop_type: str,
+        actor_id: str,
+        interval_s: float = RAY_SERVE_EVENT_LOOP_MONITORING_INTERVAL_S,
+        extra_tags: Optional[Dict[str, str]] = None,
+    ):
+        """Initialize the event loop monitor.
+
+        Args:
+            component: The component type ("proxy" or "replica").
+            loop_type: The type of event loop ("main", "user_code", or "router").
+            actor_id: The ID of the actor where this event loop runs.
+            interval_s: Optional override for the monitoring interval.
+            extra_tags: Optional dictionary of additional tags to include in metrics.
+        """
+        self._interval_s = interval_s
+        self._tags = {
+            self.TAG_KEY_COMPONENT: component,
+            self.TAG_KEY_LOOP_TYPE: loop_type,
+            self.TAG_KEY_ACTOR_ID: actor_id,
+        }
+        if extra_tags:
+            self._tags.update(extra_tags)
+        self._tag_keys = tuple(self._tags.keys())
+
+        # Create metrics
+        self._scheduling_latency = metrics.Histogram(
+            "serve_event_loop_scheduling_latency_ms",
+            description=(
+                "Latency of getting yielded control on the event loop in milliseconds. "
+                "High values indicate the event loop is blocked."
+            ),
+            boundaries=SERVE_EVENT_LOOP_LATENCY_HISTOGRAM_BOUNDARIES_MS,
+            tag_keys=self._tag_keys,
+        )
+        self._scheduling_latency.set_default_tags(self._tags)
+
+        self._iterations = metrics.Counter(
+            "serve_event_loop_monitoring_iterations",
+            description=(
+                "Number of times the event loop monitoring task has run. "
+                "Can be used as a heartbeat."
+            ),
+            tag_keys=self._tag_keys,
+        )
+        self._iterations.set_default_tags(self._tags)
+
+        self._tasks = metrics.Gauge(
+            "serve_event_loop_tasks",
+            description="Number of pending asyncio tasks on the event loop.",
+            tag_keys=self._tag_keys,
+        )
+        self._tasks.set_default_tags(self._tags)
+
+        self._monitoring_task: Optional[asyncio.Task] = None
+
+    def start(self, loop: asyncio.AbstractEventLoop) -> asyncio.Task:
+        """Start monitoring the given event loop.
+
+        Args:
+            loop: The asyncio event loop to monitor.
+
+        Returns:
+            The asyncio Task running the monitoring loop.
+        """
+        self._monitoring_task = setup_event_loop_monitoring(
+            loop=loop,
+            scheduling_latency=self._scheduling_latency,
+            iterations=self._iterations,
+            tasks=self._tasks,
+            tags=self._tags,
+            interval_s=self._interval_s,
+        )
+        logger.debug(
+            f"Started event loop monitoring for {self._tags[self.TAG_KEY_COMPONENT]} "
+            f"({self._tags[self.TAG_KEY_LOOP_TYPE]}) actor {self._tags[self.TAG_KEY_ACTOR_ID]}"
+        )
+        return self._monitoring_task
+
+    def stop(self):
+        if self._monitoring_task is not None and not self._monitoring_task.done():
+            self._monitoring_task.cancel()
+            self._monitoring_task = None
+
+    @property
+    def tags(self) -> Dict[str, str]:
+        return self._tags.copy()

python/ray/serve/_private/exceptions.py

@@ -0,0 +1,10 @@
+class DeploymentIsBeingDeletedError(Exception):
+    """Raised when an operation is attempted on a deployment that is being deleted."""
+
+    pass
+
+
+class ExternalScalerDisabledError(Exception):
+    """Raised when the external scaling API is used but external_scaler_enabled is False."""
+
+    pass

python/ray/serve/_private/gang_scheduling_autoscaling_policy.py

@@ -0,0 +1,36 @@
+import math
+from typing import Any, Callable, Dict, Tuple
+
+from ray.serve.config import AutoscalingContext
+
+
+class GangSchedulingAutoscalingPolicy:
+    """Autoscaling policy that aligns replica counts to gang size multiples.
+
+    When gang scheduling is enabled, the number of replicas must always be a
+    multiple of gang_size so that complete gangs can be scheduled or released
+    atomically. This policy wraps a base scaling policy (e.g.
+    replica_queue_length_autoscaling_policy or user's custom policy) and rounds
+    up to the next gang-aligned multiple.
+
+    Always rounding up ensures the deployment never operates below the capacity
+    the base policy requested, avoiding capacity deficits. The result is
+    deterministic — the same desired count always produces the same output
+    regardless of the current replica count, which prevents oscillation.
+
+    This class is not intended to be configured directly by users. It is
+    automatically injected with a gang-scheduled deployment with autoscaling
+    enabled.
+    """
+
+    def __init__(self, base_scaling_policy: Callable, gang_size: int):
+        self._base_scaling_policy = base_scaling_policy
+        self._gang_size = gang_size
+
+    def __call__(self, ctx: AutoscalingContext) -> Tuple[int, Dict[str, Any]]:
+        num_replicas, policy_state = self._base_scaling_policy(ctx)
+
+        if self._gang_size > 1 and num_replicas > 0:
+            num_replicas = math.ceil(num_replicas / self._gang_size) * self._gang_size
+
+        return num_replicas, policy_state

python/ray/serve/_private/grpc_util.py

@@ -0,0 +1,231 @@
+import asyncio
+import logging
+from copy import deepcopy
+from typing import Callable, List, Optional, Sequence, Tuple
+from unittest.mock import Mock
+
+import grpc
+from grpc.aio._server import Server
+
+from ray.exceptions import RayActorError, RayTaskError
+from ray.serve._private.constants import (
+    DEFAULT_GRPC_SERVER_OPTIONS,
+    RAY_SERVE_REQUEST_PROCESSING_TIMEOUT_S,
+    SERVE_LOGGER_NAME,
+)
+from ray.serve._private.proxy_request_response import ResponseStatus, gRPCStreamingType
+from ray.serve.config import gRPCOptions
+from ray.serve.exceptions import (
+    BackPressureError,
+    DeploymentUnavailableError,
+    gRPCStatusError,
+)
+from ray.serve.generated.serve_pb2_grpc import add_RayServeAPIServiceServicer_to_server
+
+# Maximum length for gRPC status details to avoid hitting HTTP/2 trailer limits.
+# gRPC default max metadata size is 8KB, so we use a conservative limit.
+GRPC_MAX_STATUS_DETAILS_LENGTH = 4096
+
+logger = logging.getLogger(SERVE_LOGGER_NAME)
+
+
+class gRPCGenericServer(Server):
+    """Custom gRPC server that will override all service method handlers.
+
+    Original implementation see: https://github.com/grpc/grpc/blob/
+        60c1701f87cacf359aa1ad785728549eeef1a4b0/src/python/grpcio/grpc/aio/_server.py
+    """
+
+    def __init__(
+        self,
+        service_handler_factory: Callable,
+        *,
+        extra_options: Optional[List[Tuple[str, str]]] = None,
+    ):
+        super().__init__(
+            thread_pool=None,
+            generic_handlers=(),
+            interceptors=(),
+            maximum_concurrent_rpcs=None,
+            compression=None,
+            options=DEFAULT_GRPC_SERVER_OPTIONS + (extra_options or []),
+        )
+        self.generic_rpc_handlers = []
+        self.service_handler_factory = service_handler_factory
+
+    def add_generic_rpc_handlers(
+        self, generic_rpc_handlers: Sequence[grpc.GenericRpcHandler]
+    ):
+        """Override generic_rpc_handlers before adding to the gRPC server.
+
+        This function will override all user defined handlers to have
+            1. None `response_serializer` so the server can pass back the
+            raw protobuf bytes to the user.
+            2. `unary_unary` is always calling the unary function generated via
+            `self.service_handler_factory`
+            3. `unary_stream` is always calling the streaming function generated via
+            `self.service_handler_factory`
+            4. `stream_unary` for client streaming requests
+            5. `stream_stream` for bidirectional streaming requests
+        """
+        serve_rpc_handlers = {}
+        rpc_handler = generic_rpc_handlers[0]
+        for service_method, method_handler in rpc_handler._method_handlers.items():
+            serve_method_handler = method_handler._replace(
+                response_serializer=None,
+                unary_unary=self.service_handler_factory(
+                    service_method=service_method,
+                    streaming_type=gRPCStreamingType.UNARY_UNARY,
+                ),
+                unary_stream=self.service_handler_factory(
+                    service_method=service_method,
+                    streaming_type=gRPCStreamingType.UNARY_STREAM,
+                ),
+                stream_unary=self.service_handler_factory(
+                    service_method=service_method,
+                    streaming_type=gRPCStreamingType.STREAM_UNARY,
+                ),
+                stream_stream=self.service_handler_factory(
+                    service_method=service_method,
+                    streaming_type=gRPCStreamingType.STREAM_STREAM,
+                ),
+            )
+            serve_rpc_handlers[service_method] = serve_method_handler
+        generic_rpc_handlers[0]._method_handlers = serve_rpc_handlers
+        self.generic_rpc_handlers.append(generic_rpc_handlers)
+        super().add_generic_rpc_handlers(generic_rpc_handlers)
+
+
+async def start_grpc_server(
+    service_handler_factory: Callable,
+    grpc_options: gRPCOptions,
+    *,
+    event_loop: asyncio.AbstractEventLoop,
+    enable_so_reuseport: bool = False,
+) -> Tuple[asyncio.Task, gRPCGenericServer]:
+    """Start a gRPC server that handles requests with the service handler factory.
+
+    Returns a task that blocks until the server exits (e.g., due to error) and
+    the server object itself (so callers can shut it down gracefully).
+    """
+    from ray.serve._private.default_impl import add_grpc_address
+
+    server = gRPCGenericServer(
+        service_handler_factory,
+        extra_options=[("grpc.so_reuseport", str(int(enable_so_reuseport)))],
+    )
+    add_grpc_address(server, f"[::]:{grpc_options.port}")
+
+    # Add built-in gRPC service and user-defined services to the server.
+    # We pass a mock servicer because the actual implementation will be overwritten
+    # in the gRPCGenericServer implementation.
+    mock_servicer = Mock()
+    for servicer_fn in [
+        add_RayServeAPIServiceServicer_to_server
+    ] + grpc_options.grpc_servicer_func_callable:
+        servicer_fn(mock_servicer, server)
+
+    await server.start()
+    return event_loop.create_task(server.wait_for_termination()), server
+
+
+def _truncate_message(
+    message: str, max_length: int = GRPC_MAX_STATUS_DETAILS_LENGTH
+) -> str:
+    """Truncate a message to avoid exceeding HTTP/2 trailer limits.
+
+    gRPC status details are sent as part of HTTP/2 trailers, which have a fixed size limit.
+    If the message (e.g., a stack trace) is too long, it can cause issues on the client side.
+    """
+    if len(message) <= max_length:
+        return message
+    truncation_notice = "... [truncated]"
+    return message[: max_length - len(truncation_notice)] + truncation_notice
+
+
+def get_grpc_response_status(
+    exc: BaseException, request_timeout_s: float, request_id: str
+) -> ResponseStatus:
+    if isinstance(exc, TimeoutError):
+        message = f"Request timed out after {request_timeout_s}s."
+        return ResponseStatus(
+            code=grpc.StatusCode.DEADLINE_EXCEEDED,
+            is_error=True,
+            message=message,
+        )
+    elif isinstance(exc, asyncio.CancelledError):
+        message = f"Client for request {request_id} disconnected."
+        return ResponseStatus(
+            code=grpc.StatusCode.CANCELLED,
+            is_error=True,
+            message=message,
+        )
+    elif isinstance(exc, BackPressureError):
+        return ResponseStatus(
+            code=grpc.StatusCode.RESOURCE_EXHAUSTED,
+            is_error=True,
+            message=exc.message,
+        )
+    elif isinstance(exc, DeploymentUnavailableError):
+        if isinstance(exc, RayTaskError):
+            logger.warning(f"Request failed: {exc}", extra={"log_to_stderr": False})
+        return ResponseStatus(
+            code=grpc.StatusCode.UNAVAILABLE,
+            is_error=True,
+            message=exc.message,
+        )
+    elif isinstance(exc, gRPCStatusError):
+        # User set a gRPC status code before raising the exception.
+        # Respect the user's status code instead of returning INTERNAL.
+        original_exc = exc.original_exception
+        if isinstance(original_exc, (RayActorError, RayTaskError)):
+            logger.warning(
+                f"Request failed: {original_exc}", extra={"log_to_stderr": False}
+            )
+        else:
+            logger.exception(
+                f"Request failed with user-set gRPC status code {exc.grpc_code}."
+            )
+        # Use user-set details if provided, otherwise use the original exception message.
+        message = exc.grpc_details if exc.grpc_details else str(original_exc)
+        return ResponseStatus(
+            code=exc.grpc_code,
+            is_error=True,
+            message=_truncate_message(message),
+        )
+    else:
+        if isinstance(exc, (RayActorError, RayTaskError)):
+            logger.warning(f"Request failed: {exc}", extra={"log_to_stderr": False})
+        else:
+            logger.exception("Request failed due to unexpected error.")
+        return ResponseStatus(
+            code=grpc.StatusCode.INTERNAL,
+            is_error=True,
+            message=_truncate_message(str(exc)),
+        )
+
+
+def set_grpc_code_and_details(
+    context: grpc._cython.cygrpc._ServicerContext, status: ResponseStatus
+):
+    # Only the latest code and details will take effect. If the user already
+    # set them to a truthy value in the context, skip setting them with Serve's
+    # default values. By default, if nothing is set, the code is 0 and the
+    # details is "", which both are falsy. So if the user did not set them or
+    # if they're explicitly set to falsy values, such as None, Serve will
+    # continue to set them with our default values.
+    if not context.code():
+        context.set_code(status.code)
+    if not context.details():
+        context.set_details(status.message)
+
+
+def set_proxy_default_grpc_options(grpc_options) -> gRPCOptions:
+    grpc_options = deepcopy(grpc_options) or gRPCOptions()
+
+    if grpc_options.request_timeout_s or RAY_SERVE_REQUEST_PROCESSING_TIMEOUT_S:
+        grpc_options.request_timeout_s = (
+            grpc_options.request_timeout_s or RAY_SERVE_REQUEST_PROCESSING_TIMEOUT_S
+        )
+
+    return grpc_options

python/ray/serve/_private/handle_options.py

@@ -0,0 +1,82 @@
+from abc import ABC, abstractmethod
+from dataclasses import dataclass, fields
+
+import ray
+from ray.serve._private.common import DeploymentHandleSource
+from ray.serve._private.constants import (
+    RAY_SERVE_RUN_ROUTER_IN_SEPARATE_LOOP,
+    RAY_SERVE_USE_GRPC_BY_DEFAULT,
+)
+from ray.serve._private.utils import DEFAULT
+
+
+@dataclass(frozen=True)
+class InitHandleOptionsBase(ABC):
+    """Init options for each ServeHandle instance.
+
+    These fields can be set by calling `.init()` on a handle before
+    sending the first request.
+    """
+
+    _prefer_local_routing: bool = False
+    _source: DeploymentHandleSource = DeploymentHandleSource.UNKNOWN
+    _run_router_in_separate_loop: bool = RAY_SERVE_RUN_ROUTER_IN_SEPARATE_LOOP
+
+    @classmethod
+    @abstractmethod
+    def create(cls, **kwargs) -> "InitHandleOptionsBase":
+        raise NotImplementedError
+
+
+@dataclass(frozen=True)
+class InitHandleOptions(InitHandleOptionsBase):
+    @classmethod
+    def create(cls, **kwargs) -> "InitHandleOptions":
+        for k in list(kwargs.keys()):
+            if kwargs[k] == DEFAULT.VALUE:
+                # Use default value
+                del kwargs[k]
+
+        # Detect replica source for handles
+        if (
+            "_source" not in kwargs
+            and ray.serve.context._get_internal_replica_context() is not None
+        ):
+            kwargs["_source"] = DeploymentHandleSource.REPLICA
+
+        return cls(**kwargs)
+
+
+@dataclass(frozen=True)
+class DynamicHandleOptionsBase(ABC):
+    """Dynamic options for each ServeHandle instance.
+
+    These fields can be changed by calling `.options()` on a handle.
+    """
+
+    method_name: str = "__call__"
+    multiplexed_model_id: str = ""
+    session_id: str = ""
+    stream: bool = False
+
+    @abstractmethod
+    def copy_and_update(self, **kwargs) -> "DynamicHandleOptionsBase":
+        pass
+
+
+@dataclass(frozen=True)
+class DynamicHandleOptions(DynamicHandleOptionsBase):
+    _by_reference: bool = not RAY_SERVE_USE_GRPC_BY_DEFAULT
+    request_serialization: str = "cloudpickle"
+    response_serialization: str = "cloudpickle"
+
+    def copy_and_update(self, **kwargs) -> "DynamicHandleOptions":
+        new_kwargs = {}
+
+        for f in fields(self):
+            if f.name not in kwargs or kwargs[f.name] == DEFAULT.VALUE:
+                new_kwargs[f.name] = getattr(self, f.name)
+            else:
+                new_kwargs[f.name] = kwargs[f.name]
+
+        return DynamicHandleOptions(**new_kwargs)

python/ray/serve/_private/haproxy_metrics.py

@@ -0,0 +1,514 @@
+"""Metrics collection for the HAProxy ingress request router data path.
+
+HAProxy is configured to emit one RFC 5424 syslog line per request to a
+dedicated Unix dgram socket. Existing rfc3164 log targets are unaffected.
+`HAProxyMetricsCollector` owns the parsing and the `ray.util.metrics`
+objects; `_DatagramHandler` is the asyncio glue that hands datagrams in.
+"""
+
+import asyncio
+import logging
+import os
+import re
+import socket
+from dataclasses import dataclass
+from typing import Optional
+
+from ray.serve._private.common import RequestProtocol
+from ray.serve._private.haproxy import HAProxyApi
+from ray.serve._private.request_ingress_metrics import RequestIngressMetrics
+from ray.util import metrics
+
+logger = logging.getLogger(__name__)
+
+# SD-ID we publish under. The leading bracket + this string is what the
+# parser anchors on; only lines containing this section are processed.
+_SD_ID = "serve@1"
+
+# RFC 5424 SD element looks like `[serve@1 key="value" key="value"]`. Capture
+# the body between the SD-ID and the closing `]`. A `]` inside a value is
+# escaped as `\]`, so the body matches escaped pairs and stops at the first
+# unescaped `]`.
+_SD_SECTION_RE = re.compile(r"\[" + re.escape(_SD_ID) + r"(?P<body>(?:[^\]\\]|\\.)*)\]")
+
+# Capture `key=value` pairs where the value is either RFC 5424 quoted (may
+# contain spaces; backslash escapes allowed) or a bare token. HAProxy quotes
+# sample-fetch values (`%[var(...)]`) and string aliases like `%HM`, but renders
+# numeric aliases (`%ST`, `%Ta`) unquoted, so the parser must accept both.
+_KV_RE = re.compile(r'(\w+)=(?:"((?:[^"\\]|\\.)*)"|(\S+))')
+
+# HAProxy renders unset txn vars as an empty string in log-format. We map
+# that to None so callers don't have to distinguish "unset" from "empty".
+_UNSET = ""
+
+# HAProxy escapes some characters in a quoted log value with a leading
+# backslash. Drop the backslash so the tag holds the original name.
+# Example: `a\"b` -> `a"b`.
+_UNESCAPE_RE = re.compile(r"\\(.)")
+
+
+def _unescape(value: str) -> str:
+    return _UNESCAPE_RE.sub(r"\1", value)
+
+
+@dataclass
+class ParsedMetrics:
+    """One per-request observation, parsed from the SD section.
+
+    The first group is the general per-request ingress data present on every
+    HTTP request through the frontend; it feeds the `serve_num_http_*` /
+    `serve_http_request_latency_ms` families. The `ingress_request_*` fields
+    are router-specific and only populated when ingress-request-router metrics
+    are enabled and the request went through (or attempted) the router.
+    """
+
+    app: Optional[str] = None
+    ingress_request_intended_server: Optional[str] = None
+    ingress_request_actual_server: Optional[str] = None
+    ingress_request_router_latency_us: Optional[int] = None
+    ingress_request_body_truncated_full_length: Optional[int] = None
+    ingress_request_via_router: bool = False
+    ingress_request_failed: Optional[str] = None
+    route: Optional[str] = None
+    method: Optional[str] = None
+    status_code: Optional[str] = None
+    latency_ms: Optional[int] = None
+    deployment: Optional[str] = None
+    # HAProxy 2-char session termination state (%ts). A leading "C" means the
+    # client aborted the connection; the recorder maps that to status 499.
+    termination_state: Optional[str] = None
+
+
+class HAProxyMetricsCollector:
+    """Owns every `serve_haproxy_*` metric for one proxy node.
+
+    Two families of metrics live here:
+
+    - Per-request, push-based ingress-request-router metrics (Counters /
+      Histogram), fed by HAProxy datagrams. `parse_line` and `record` are
+      exposed for unit tests that want to drive these without binding
+      anything; `bind_and_attach` wires an `AF_UNIX` dgram socket to the
+      loop and `close` tears it down.
+    - Node-level, poll-based gauges (process count and broadcasted-vs-reported
+      target mismatch), sampled from an `HAProxyApi` on a periodic loop started
+      by `start_node_metrics_polling`.
+
+    The node-level gauges are always emitted; the datagram reader is only
+    bound when ingress-request-router metrics are enabled.
+    """
+
+    # Sub-millisecond to 1s, biased toward the expected sub-10ms range for
+    # a healthy local router consultation.
+    _LATENCY_BUCKETS_MS = [
+        0.5,
+        1.0,
+        2.0,
+        5.0,
+        10.0,
+        25.0,
+        50.0,
+        100.0,
+        250.0,
+        500.0,
+        1000.0,
+    ]
+
+    def __init__(
+        self,
+        haproxy_api: HAProxyApi,
+        node_id: str,
+        node_ip_address: str = "",
+    ) -> None:
+        self._transport: Optional[asyncio.DatagramTransport] = None
+        self._socket_path: Optional[str] = None
+
+        # Source for the node-level poll loop (process count + target mismatch).
+        self._haproxy_api = haproxy_api
+        self._node_id = node_id
+        self._node_metrics_task: Optional[asyncio.Task] = None
+
+        # Per-request HTTP ingress metrics (serve_num_http_requests, latency,
+        # errors). In HAProxy mode these are emitted here from HAProxy log
+        # datagrams rather than by a Python proxy, so requests HAProxy terminates
+        # itself (e.g. 404, /-/routes, health checks) are still counted. HTTP
+        # only -- HAProxy does not expose gRPC status (it lives in HTTP/2
+        # trailers), so gRPC ingress metrics stay on the replica. The
+        # ongoing-requests gauge is not driven here: it can't be derived from
+        # per-request log lines.
+        self.request_ingress_metrics = RequestIngressMetrics(
+            RequestProtocol.HTTP,
+            source="proxy",
+            node_id=node_id,
+            node_ip_address=node_ip_address,
+        )
+
+        self.truncated_bodies_counter = metrics.Counter(
+            "serve_haproxy_ingress_router_truncations",
+            description=(
+                "Count of requests whose body was truncated by HAProxy "
+                "(exceeded tune.bufsize) before being forwarded to the "
+                "ingress request router."
+            ),
+            tag_keys=("application",),
+        )
+        self.latency_histogram = metrics.Histogram(
+            "serve_haproxy_ingress_router_latency_ms",
+            description=(
+                "Wall-clock time (in milliseconds) HAProxy spent to resolve "
+                "the request to a server via the ingress request router. "
+                "Only includes successful routing attempts."
+            ),
+            boundaries=self._LATENCY_BUCKETS_MS,
+            tag_keys=("application", "outcome"),
+        )
+        self.replica_mismatches_counter = metrics.Counter(
+            "serve_haproxy_ingress_router_server_mismatch",
+            description=(
+                "Count of requests where HAProxy ultimately routed to a "
+                "different replica than the one the ingress request router "
+                "returned (typically because the named replica was DOWN and "
+                "option redispatch picked another)."
+            ),
+            tag_keys=("application",),
+        )
+        self.failures_counter = metrics.Counter(
+            "serve_haproxy_ingress_router_failures",
+            description=(
+                "Count of ingress-request-router consultations that failed "
+                "to pin a replica, broken down by reason. Possible reasons: "
+                "'router_unreachable' (socket connect/send/recv failed), "
+                "'router_non_200' (router returned a non-200 status), "
+                "'unparseable_replica_id' (router 200 but response body "
+                "did not contain a string replica_id), "
+                "'unknown_replica_id' (router returned a replica_id not "
+                "present in the current replica map). All reasons return 503 "
+                "to the client except 'unknown_replica_id', which routes to the "
+                "fallback proxy when one is available (otherwise 503)."
+            ),
+            tag_keys=("application", "reason"),
+        )
+        self.requests_counter = metrics.Counter(
+            "serve_haproxy_ingress_router_requests",
+            description=(
+                "The number of requests that have been processed by "
+                "the ingress request router. This includes both successful "
+                "and failed requests."
+            ),
+            tag_keys=("application",),
+        )
+
+        # Node-level gauges, sampled by _report_node_metrics_forever.
+        self.process_count_gauge = metrics.Gauge(
+            "serve_haproxy_process_count",
+            description=(
+                "Number of HAProxy processes running on the node for this proxy, "
+                "spanning the live worker, draining workers from prior reloads, "
+                "and any leaked/orphaned workers. A value persistently above 1 "
+                "indicates HAProxy processes are not being reaped."
+            ),
+            tag_keys=("node_id",),
+        )
+        self.target_mismatch_gauge = metrics.Gauge(
+            "serve_haproxy_target_mismatch",
+            description=(
+                "Number of targets that differ between the controller's "
+                "broadcasted target set and the targets HAProxy actually reports "
+                "in its stats on this node (symmetric set difference). A non-zero "
+                "value means the HAProxy config has not yet converged to the "
+                "broadcasted targets."
+            ),
+            tag_keys=("node_id",),
+        )
+        self.process_count_gauge.set_default_tags({"node_id": node_id})
+        self.target_mismatch_gauge.set_default_tags({"node_id": node_id})
+
+    @staticmethod
+    def parse_line(line: bytes) -> Optional[ParsedMetrics]:
+        """Extract metric fields from one RFC 5424 log datagram.
+
+        Returns `None` when the SD section is absent or unparseable. The
+        rest of the syslog line (priority, timestamp, message) is ignored.
+        """
+        try:
+            text = line.decode("utf-8", errors="replace")
+        except Exception:
+            return None
+
+        match = _SD_SECTION_RE.search(text)
+        if not match:
+            return None
+
+        kv: dict = {}
+        for key, quoted, bare in _KV_RE.findall(match.group("body")):
+            # Bare tokens (numeric aliases) are used as-is; quoted values are
+            # unescaped so the tag holds the original string. An empty quoted
+            # value ("") becomes None.
+            value = bare if bare else _unescape(quoted)
+            kv[key] = value if value != _UNSET else None
+
+        def as_int(key: str) -> Optional[int]:
+            raw = kv.get(key)
+            if raw is None:
+                return None
+            try:
+                return int(raw)
+            except ValueError:
+                return None
+
+        return ParsedMetrics(
+            app=kv.get("app"),
+            ingress_request_intended_server=kv.get("intended"),
+            ingress_request_actual_server=kv.get("actual"),
+            ingress_request_router_latency_us=as_int("router_latency_us"),
+            ingress_request_body_truncated_full_length=as_int(
+                "body_truncated_full_length"
+            ),
+            # HAProxy renders booleans as "1"/"0"; absence as "" -> False.
+            ingress_request_via_router=kv.get("via_router") == "1",
+            ingress_request_failed=kv.get("failed"),
+            route=kv.get("route"),
+            method=kv.get("method"),
+            status_code=kv.get("status"),
+            latency_ms=as_int("latency_ms"),
+            deployment=kv.get("deployment"),
+            termination_state=kv.get("term_state"),
+        )
+
+    def _record_ingress_request(self, parsed: ParsedMetrics) -> None:
+        """Emit the per-request RequestIngressMetrics for one observation.
+
+        Mirrors what the Python proxy records per request (the
+        `serve_num_http_*` and `serve_http_request_latency_ms` families) --
+        for every request, including ones HAProxy terminates itself (404,
+        `/-/routes`, health checks), matching the proxy's tags (`application`
+        and `route` are empty for those). Skips lines with no status, which
+        aren't real request observations.
+        """
+        if parsed.status_code is None:
+            return
+
+        # A client abort (HAProxy termination state with a leading "C") is
+        # recorded as 499, matching the Python proxy's client-disconnect
+        # convention.
+        status_code = parsed.status_code
+        if parsed.termination_state and parsed.termination_state.startswith("C"):
+            status_code = "499"
+
+        try:
+            is_error = int(status_code) >= 400
+        except ValueError:
+            # Non-numeric status (shouldn't happen for HTTP); treat as non-error.
+            is_error = False
+
+        self.request_ingress_metrics.record_request(
+            route=parsed.route or "",
+            method=parsed.method or "",
+            application=parsed.app or "",
+            status_code=status_code,
+            # %Ta is integer-ms resolution; sub-ms requests round to 0.
+            latency_ms=float(parsed.latency_ms or 0),
+            is_error=is_error,
+            deployment_name=parsed.deployment or "",
+        )
+
+    def record(self, parsed: ParsedMetrics) -> None:
+        """Update metrics from one parsed observation.
+
+        First records the general per-request ingress metrics (every HTTP
+        request). Then records the router-specific metrics, which only apply to
+        requests that went through (or attempted) the ingress request router:
+        - `ingress_request_failed` set: the Lua action set `txn.ingress_request_router_failed`
+          and returned early. Bump the failures counter with the reason; no
+          replica was pinned, so other router metrics don't apply.
+        - `ingress_request_via_router` true: the Lua action successfully pinned a replica.
+          Record latency, truncation, and replica-mismatch as applicable.
+        - Neither: the request didn't go through the router path at all
+          (no router-bearing app matched, or router state not yet pushed).
+          No router metrics to record.
+        """
+        # General per-request ingress metrics, independent of the router path.
+        self._record_ingress_request(parsed)
+
+        # `application` tag is required by the metric definitions; default
+        # to "unknown" rather than dropping the observation, so misconfigured
+        # frontends still show up in the data.
+        app_tag = parsed.app or "unknown"
+        tags = {"application": app_tag}
+
+        if parsed.ingress_request_via_router and not parsed.ingress_request_failed:
+            self.requests_counter.inc(tags=tags)
+
+            if parsed.ingress_request_body_truncated_full_length is not None:
+                self.truncated_bodies_counter.inc(tags=tags)
+
+            # Only count mismatch when we have both sides AND the request actually
+            # reached a server (ingress_request_actual_server is not None / "<NOSRV>"). If the
+            # router pinned a replica but the request was rejected upstream of
+            # server selection (e.g. queued and aborted), HAProxy logs "<NOSRV>"
+            # for %s — we treat that as "not a mismatch, not a match".
+            if (
+                parsed.ingress_request_intended_server
+                and parsed.ingress_request_actual_server
+                and parsed.ingress_request_actual_server != "<NOSRV>"
+                and parsed.ingress_request_intended_server
+                != parsed.ingress_request_actual_server
+            ):
+                self.replica_mismatches_counter.inc(tags=tags)
+        elif parsed.ingress_request_failed:
+            self.requests_counter.inc(tags=tags)
+            self.failures_counter.inc(
+                tags={**tags, "reason": parsed.ingress_request_failed}
+            )
+        else:
+            return
+
+        if parsed.ingress_request_router_latency_us is not None:
+            self.latency_histogram.observe(
+                parsed.ingress_request_router_latency_us / 1_000.0,
+                tags={
+                    **tags,
+                    "outcome": "failure"
+                    if parsed.ingress_request_failed
+                    else "success",
+                },
+            )
+
+    async def _report_node_metrics_forever(self, interval_s: float) -> None:
+        """Background task to emit the node-level HAProxy observability gauges."""
+        consecutive_errors = 0
+        while True:
+            try:
+                await asyncio.sleep(interval_s)
+                # count_haproxy_processes does blocking /proc IO that scales
+                # with the node's process count; run it in a thread so the
+                # actor's event loop (health checks, reloads) isn't stalled.
+                loop = asyncio.get_running_loop()
+                count = await loop.run_in_executor(
+                    None, self._haproxy_api.count_haproxy_processes
+                )
+                self.process_count_gauge.set(count)
+                self.target_mismatch_gauge.set(
+                    await self._haproxy_api.compute_target_mismatch()
+                )
+                # num_ongoing_requests can't be derived from the per-request log
+                # lines, so it's sampled here from HAProxy's backend `scur`.
+                self.request_ingress_metrics.set_num_ongoing_requests(
+                    await self._haproxy_api.count_ongoing_http_requests()
+                )
+                consecutive_errors = 0
+            except Exception:
+                logger.exception("Unexpected error reporting HAProxy node metrics.")
+
+                # Exponential backoff starting at 1s and capping at 10s.
+                backoff_time_s = min(10, 2**consecutive_errors)
+                consecutive_errors += 1
+                await asyncio.sleep(backoff_time_s)
+
+    def start_node_metrics_polling(
+        self,
+        loop: asyncio.AbstractEventLoop,
+        interval_s: float,
+    ) -> None:
+        """Start the periodic loop that emits the node-level gauges."""
+        if self._node_metrics_task is None:
+            self._node_metrics_task = loop.create_task(
+                self._report_node_metrics_forever(interval_s)
+            )
+
+    def start(
+        self,
+        loop: asyncio.AbstractEventLoop,
+        *,
+        poll_interval_s: float,
+        metrics_socket_path: Optional[str] = None,
+    ) -> asyncio.Task:
+        """Start all metric collection for this proxy node.
+
+        Always starts the node-level gauge poll loop. Also always creates the
+        dgram socket directory and binds the per-request reader at `metrics_socket_path`
+        (where HAProxy writes one RFC 5424 line per request), returning the
+        bind task so the caller can await it (e.g. to surface bind failures at
+        actor-readiness time).
+        """
+        self.start_node_metrics_polling(loop, poll_interval_s)
+        os.makedirs(os.path.dirname(metrics_socket_path), exist_ok=True)
+        return loop.create_task(self.bind_and_attach(metrics_socket_path, loop=loop))
+
+    async def bind_and_attach(
+        self,
+        socket_path: str,
+        loop: Optional[asyncio.AbstractEventLoop] = None,
+    ) -> None:
+        """Bind a Unix dgram socket at `socket_path` and register the
+        asyncio reader on `loop`.
+
+        Many HAProxy frontends can write to the same socket; dgram
+        delivery preserves message boundaries so the reader gets one
+        observation per `recvfrom`.
+
+        Idempotent: if a transport is already attached, it is closed
+        first. On failure, the collector is left in an unbound state and
+        `close()` is still safe to call.
+        """
+        if self._transport is not None:
+            self.close()
+
+        if loop is None:
+            loop = asyncio.get_event_loop()
+
+        try:
+            os.unlink(socket_path)
+        except FileNotFoundError:
+            pass
+
+        sock = socket.socket(socket.AF_UNIX, socket.SOCK_DGRAM)
+        try:
+            sock.bind(socket_path)
+            # Match the existing admin socket pattern (mode 666) so HAProxy
+            # processes running as a different user can still send to it.
+            os.chmod(socket_path, 0o666)
+            transport, _ = await loop.create_datagram_endpoint(
+                lambda: _DatagramHandler(self), sock=sock
+            )
+        except Exception:
+            sock.close()
+            raise
+
+        self._socket_path = socket_path
+        self._transport = transport
+
+    def close(self) -> None:
+        """Tear down the node-metrics loop, the dgram transport, and the
+        socket file.
+
+        Safe to call multiple times; safe to call without ever having
+        bound or started polling. The metric objects survive close — they
+        are owned by Ray's metric registry, not this instance.
+        """
+        if self._node_metrics_task is not None:
+            self._node_metrics_task.cancel()
+            self._node_metrics_task = None
+        if self._transport is not None:
+            self._transport.close()
+            self._transport = None
+        if self._socket_path is not None:
+            try:
+                os.unlink(self._socket_path)
+            except FileNotFoundError:
+                pass
+            self._socket_path = None
+
+
+class _DatagramHandler(asyncio.DatagramProtocol):
+    def __init__(self, collector: HAProxyMetricsCollector) -> None:
+        self._collector = collector
+
+    def datagram_received(self, data: bytes, addr) -> None:  # noqa: D401
+        try:
+            parsed = self._collector.parse_line(data)
+            if parsed is not None:
+                self._collector.record(parsed)
+        except Exception:
+            # A malformed datagram must never crash the proxy actor. Log
+            # once per occurrence at debug to keep busy frontends quiet.
+            logger.debug("Failed to handle HAProxy metrics datagram", exc_info=True)

python/ray/serve/_private/haproxy_templates.py

@@ -0,0 +1,435 @@
+HAPROXY_HEALTHZ_RULES_TEMPLATE = """    # Health check endpoint
+    acl healthcheck path -i {{ config.health_check_endpoint }}
+    # Keep health checks out of the access log but still record their metric:
+    # tag them `debug` so the access-log target (level info) drops them while the
+    # metrics socket (level debug) keeps them. Mirrors the proxy, which records
+    # the healthz metric with should_record_access_log=False.
+    http-request set-log-level debug if healthcheck
+    {%- if config.metrics_enabled %}
+    http-request set-var-fmt(txn.serve_route) {{ config.health_check_endpoint | haproxy_fmt }} if healthcheck
+    {%- endif %}
+{%- if not health_info.healthy %}
+    # Override: force health checks to fail (used by drain/disable)
+    http-request return status {{ health_info.status }} content-type text/plain string "{{ health_info.health_message }}" if healthcheck
+{%- elif backends %}
+    # 200 if any backend has at least one server UP
+{%-   for backend in backends %}
+    acl backend_{{ backend.name or 'unknown' }}_server_up nbsrv({{ backend.name or 'unknown' }}) ge 1
+{%-   endfor %}
+    # Any backend with a server UP passes the health check (OR logic)
+{%-   for backend in backends %}
+    http-request return status {{ health_info.status }} content-type text/plain string "{{ health_info.health_message }}" if healthcheck backend_{{ backend.name or 'unknown' }}_server_up
+{%-   endfor %}
+    http-request return status 503 content-type text/plain string "Service Unavailable" if healthcheck
+{%- elif config.is_head %}
+    # Head is the always-on ingress endpoint, so it stays ready with no backends.
+    # Mirrors is_head in ProxyRouter.ready_for_traffic.
+    http-request return status {{ health_info.status }} content-type text/plain string "{{ health_info.health_message }}" if healthcheck
+{%- endif %}
+"""
+
+# Same shape as HAPROXY_HEALTHZ_RULES_TEMPLATE, but emits gRPC trailers-only
+# responses. The OK / UNAVAILABLE distinction maps to grpc-status 0 / 14
+# (gRPC clients map HTTP 503 to UNAVAILABLE, which is why every shape here
+# uses HTTP 200 and signals state through grpc-status).
+HAPROXY_GRPC_HEALTHZ_RULES_TEMPLATE = """    # Health check endpoint (gRPC `Healthz`)
+    acl is_healthz path /ray.serve.RayServeAPIService/Healthz
+    # Suppress logging for health checks
+    http-request set-log-level silent if is_healthz
+{%- if not health_info.healthy %}
+    # Override: force health checks to fail (used by drain/disable)
+    http-request return status 200 content-type application/grpc hdr grpc-status 14 hdr grpc-message "{{ health_info.health_message }}" if is_healthz
+{%- elif backends %}
+    # OK if any backend has at least one server UP
+{%-   for backend in backends %}
+    acl backend_{{ backend.name or 'unknown' }}_server_up nbsrv({{ backend.name or 'unknown' }}) ge 1
+{%-   endfor %}
+    # Any backend with a server UP passes the health check (OR logic)
+{%-   for backend in backends %}
+    http-request return status 200 content-type application/grpc hdr grpc-status 0 if is_healthz backend_{{ backend.name or 'unknown' }}_server_up
+{%-   endfor %}
+    http-request return status 200 content-type application/grpc hdr grpc-status 14 hdr grpc-message "Service Unavailable" if is_healthz
+{%- elif config.is_head %}
+    # Head is the always-on ingress endpoint, so it stays ready with no backends.
+    # Mirrors is_head in ProxyRouter.ready_for_traffic.
+    http-request return status 200 content-type application/grpc hdr grpc-status 0 if is_healthz
+{%- endif %}
+"""
+
+HAPROXY_CONFIG_TEMPLATE = """global
+    # Access/event log at `info`. The system endpoints (/-/healthz, /-/routes)
+    # tag their logs `debug` so they are dropped here but still reach the
+    # rfc5424 metrics socket (level `debug`) when metrics are enabled -- mirroring
+    # the proxy, which records their metric but not their access log.
+    log {{ config.log_target }} local0 info
+    stats socket {{ config.socket_path }} mode 666 level admin expose-fd listeners
+    stats timeout 30s
+    maxconn {{ config.maxconn }}
+    nbthread {{ config.nbthread }}
+    {%- if has_ingress_request_router %}
+    lua-load-per-thread {{ ingress_request_router_lua_path }}
+    {%- endif %}
+    {%- if has_ingress_request_router and ingress_request_router_forward_body %}
+    tune.bufsize {{ ingress_request_router_bufsize }}
+    {%- else %}
+    tune.bufsize {{ config.bufsize }}
+    {%- endif %}
+    {%- if config.enable_hap_optimization %}
+    server-state-base {{ config.server_state_base }}
+    server-state-file {{ config.server_state_file }}
+    {%- endif %}
+    {%- if config.hard_stop_after_s is not none %}
+    hard-stop-after {{ config.hard_stop_after_s }}s
+    {%- endif %}
+    {%- if config.grpc_enabled %}
+    tune.h2.max-frame-size {{ config.h2_max_frame_size }}
+    tune.h2.be.initial-window-size {{config.h2_be_initial_window_size}}
+    tune.h2.be.max-concurrent-streams {{config.h2_be_max_concurrent_streams}}
+    tune.h2.fe.initial-window-size {{config.h2_fe_initial_window_size}}
+    tune.h2.fe.max-concurrent-streams {{config.h2_fe_max_concurrent_streams}}
+    {%- endif %}
+defaults
+    mode http
+    option log-health-checks
+    {% if config.timeout_connect_s is not none %}timeout connect {{ config.timeout_connect_s }}s{% endif %}
+    {% if config.timeout_client_s is not none %}timeout client {{ config.timeout_client_s }}s{% endif %}
+    {% if config.timeout_server_s is not none %}timeout server {{ config.timeout_server_s }}s{% endif %}
+    {% if config.timeout_http_request_s is not none %}timeout http-request {{ config.timeout_http_request_s }}s{% endif %}
+    {% if config.timeout_http_keep_alive_s is not none %}timeout http-keep-alive {{ config.timeout_http_keep_alive_s }}s{% endif %}
+    {% if config.timeout_queue_s is not none %}timeout queue {{ config.timeout_queue_s }}s{% endif %}
+    log global
+    option httplog
+    option abortonclose
+    option splice-request
+    option splice-response
+    # On a retry, use a different slot (`1`). retry-on defaults to connect
+    # failures only (nothing was sent → safe to replay); override globally via
+    # RAY_SERVE_HAPROXY_RETRY_ON. Inherited by every backend.
+    option redispatch 1
+    retry-on {{ config.retry_on }}
+    {%- if config.retries is not none %}
+    retries {{ config.retries }}
+    {%- endif %}
+    {%- if config.tcp_nodelay %}
+    # Set TCP_NODELAY on all connections
+    option http-no-delay
+    {%- endif %}
+    {%- if config.enable_hap_optimization %}
+    option idle-close-on-response
+    {%- endif %}
+    # Normalize 502/503/504 to 500 per Serve's default behavior. 503
+    # covers HAProxy's own "all retries exhausted / no server" response.
+    {%- if config.error_file_path %}
+    errorfile 502 {{ config.error_file_path }}
+    errorfile 503 {{ config.error_file_path }}
+    errorfile 504 {{ config.error_file_path }}
+    {%- endif %}
+    {%- if config.enable_hap_optimization %}
+    load-server-state-from-file global
+    {%- endif %}
+    balance {{ config.balance_algorithm }}
+frontend prometheus
+    bind :{{ config.metrics_port }}
+    mode http
+    http-request use-service prometheus-exporter if { path {{ config.metrics_uri }} }
+    no log
+frontend http_frontend
+    bind {{ config.frontend_host }}:{{ config.frontend_port }}
+    {%- if config.metrics_enabled %}
+    log global
+    # Per-request HTTP ingress metrics. One RFC 5424 line per request matched to
+    # a Serve app backend, scraped into the serve_num_http_* /
+    # serve_http_request_latency_ms families (the metrics the Python proxy emits
+    # in non-HAProxy mode). Goes only to the rfc5424 target below; the inherited
+    # rfc3164 targets do not include the SD section, so their byte stream is
+    # unchanged. The general fields come from txn.serve_* vars set per backend
+    # below; %ST/%Ta/%ts render unquoted (HAProxy does not quote those aliases).
+    # term_state (%ts) is HAProxy's 2-char session termination state; a leading "C"
+    # means the client aborted, which the collector maps to status 499 to match the
+    # Python proxy's client-disconnect convention. When ingress-request-router
+    # metrics are also enabled, the router-specific fields are appended to the same
+    # line.
+    log {{ metrics_socket_path }} len 8192 format rfc5424 local1 debug
+    log-format-sd "%{+Q,+E}o [serve@1 app=%[var(txn.serve_app)] route=%[var(txn.serve_route)] method=%HM status=%ST latency_ms=%Ta deployment=%[var(txn.serve_deployment)] term_state=%ts{% if ingress_request_router_metrics_enabled and has_ingress_request_router %} intended=%[var(txn.ingress_request_router_target)] actual=%s router_latency_us=%[var(txn.ingress_request_router_latency_us)] body_truncated_full_length=%[var(txn.ingress_request_router_truncated_full_length)] via_router=%[var(txn.via_ingress_request_router)] failed=%[var(txn.ingress_request_router_failed)]{% endif %}]"
+    {%- endif %}
+    {%- if config.root_path %}
+    # Strip the configured global root_path so the health/routes endpoints, the
+    # per-backend path ACLs, and the path forwarded to replicas are all
+    # root_path-agnostic. Mirrors the native Serve proxy, which mounts the app
+    # under root_path. An exact root_path match becomes "/", and paths outside
+    # root_path are left unchanged.
+    http-request set-path / if { path {{ config.root_path }} }
+    http-request set-path %[path,regsub(^{{ config.root_path }}/,/)] if { path_beg {{ config.root_path }}/ }
+    {%- endif %}
+{{ healthz_rules|safe }}
+    # Routes endpoint
+    acl routes path -i /-/routes
+    # Like health checks: kept out of the access log (tagged `debug`); its metric
+    # is recorded (route=/-/routes, app unset) when metrics are enabled.
+    http-request set-log-level debug if routes
+    {%- if config.metrics_enabled %}
+    http-request set-var-fmt(txn.serve_route) {{ '/-/routes' | haproxy_fmt }} if routes
+    {%- endif %}
+    http-request return status {{ route_info.status }} content-type {{ route_info.routes_content_type }} string "{{ route_info.routes_message }}" if routes
+
+    {%- if config.inject_process_id_header and config.reload_id %}
+    # Inject unique reload ID as header to track which HAProxy instance handled the request (testing only)
+    http-request set-header x-haproxy-reload-id {{ config.reload_id }}
+    {%- endif %}
+    # Per-backend path ACLs (used for both ingress-request-router dispatch
+    # and static use_backend selection below).
+{%- for backend in backends %}
+    acl is_{{ backend.name or 'unknown' }} path_beg {{ '/' if not backend.path_prefix or backend.path_prefix == '/' else backend.path_prefix ~ '/' }}
+    acl is_{{ backend.name or 'unknown' }} path {{ backend.path_prefix or '/' }}
+{%- endfor %}
+    {%- if config.metrics_enabled %}
+    # Per-request HTTP metric vars (app / route / ingress deployment), set on the
+    # first matching backend. Backends are sorted longest-prefix-first and the
+    # !found guard makes the longest match win, mirroring the use_backend rules
+    # below. Requests that match no app backend (e.g. /-/routes, 404s) leave
+    # these unset, so the collector can skip them.
+    {%- for backend in backends %}
+    http-request set-var-fmt(txn.serve_app) {{ (backend.app_name or 'unknown') | haproxy_fmt }} if is_{{ backend.name or 'unknown' }} !{ var(txn.serve_app) -m found }
+    http-request set-var-fmt(txn.serve_route) {{ (backend.path_prefix or '/') | haproxy_fmt }} if is_{{ backend.name or 'unknown' }} !{ var(txn.serve_route) -m found }
+    {%- if backend.ingress_deployment_name %}
+    http-request set-var-fmt(txn.serve_deployment) {{ backend.ingress_deployment_name | haproxy_fmt }} if is_{{ backend.name or 'unknown' }} !{ var(txn.serve_deployment) -m found }
+    {%- endif %}
+    {%- endfor %}
+    {%- endif %}
+    {%- if has_ingress_request_router %}
+    # Set txn.ingress_request_router_app to the first matching router-bearing
+    # backend. Backends are sorted longest-prefix-first, and the !found guard
+    # ensures only the longest match wins.
+    {%- for backend in backends %}
+    {%- if backend.ingress_request_router_servers %}
+    http-request set-var(txn.ingress_request_router_app) str({{ backend.name or 'unknown' }}) if is_{{ backend.name or 'unknown' }} !{ var(txn.ingress_request_router_app) -m found }
+    {%- endif %}
+    {%- endfor %}
+    acl has_ingress_request_router_app var(txn.ingress_request_router_app) -m found
+    {%- if ingress_request_router_forward_body %}
+    http-request wait-for-body time {{ ingress_request_router_timeout_s }}s if METH_POST has_ingress_request_router_app
+    {%- endif %}
+    http-request lua.route_via_ingress_request_router if METH_POST has_ingress_request_router_app
+    # A pin-miss is recoverable only if its app has a fallback proxy. Mark it
+    # per app so the 503 below fails loud for apps with none.
+    {%- for backend in backends %}
+    {%- if backend.ingress_request_router_servers and backend.fallback_server %}
+    http-request set-var(txn.ingress_request_router_recoverable) str(1) if { var(txn.ingress_request_router_app) -m str "{{ backend.name or 'unknown' }}" } { var(txn.ingress_request_router_failed) -m str "unknown_replica_id" }
+    {%- endif %}
+    {%- endfor %}
+    # 503 on any router failure except a recoverable pin-miss. Must precede the
+    # use_backend rules so failures never fall through to the primary backend.
+    http-request return status 503 content-type text/plain lf-string "Ingress request router failed: %[var(txn.ingress_request_router_failed)]" hdr X-Serve-Reason %[var(txn.ingress_request_router_failed)] if { var(txn.ingress_request_router_failed) -m found } !{ var(txn.ingress_request_router_recoverable) -m found }
+    {%- endif %}
+    # Static routing based on path prefixes in decreasing length then alphabetical order
+{%- for backend in backends %}
+    {%- if has_ingress_request_router and backend.ingress_request_router_servers %}
+    use_backend {{ backend.name or 'unknown' }}-via-ingress-request-router if is_{{ backend.name or 'unknown' }} { var(txn.via_ingress_request_router) -m found }
+    {%- if backend.fallback_server %}
+    # Pin-miss recovery: route into the router backend, which picks the fallback.
+    use_backend {{ backend.name or 'unknown' }}-via-ingress-request-router if is_{{ backend.name or 'unknown' }} { var(txn.ingress_request_router_failed) -m str "unknown_replica_id" }
+    {%- endif %}
+    {%- endif %}
+    use_backend {{ backend.name or 'unknown' }} if is_{{ backend.name or 'unknown' }}
+{%- endfor %}
+    default_backend default_backend
+backend default_backend
+    http-request return status 404 content-type text/plain lf-string "Path \'%[path]\' not found. Ping http://.../-/routes for available routes."
+{%- for item in backends_with_health_config %}
+{%- set backend = item.backend %}
+{%- set hc = item.health_config %}
+backend {{ backend.name or 'unknown' }}
+    log global
+    # Enable HTTP connection reuse for better performance
+    http-reuse always
+    # Set backend-specific timeouts, overriding defaults if specified
+    {%- if backend.timeout_connect_s is not none %}
+    timeout connect {{ backend.timeout_connect_s }}s
+    {%- endif %}
+    {%- if backend.timeout_server_s is not none %}
+    timeout server {{ backend.timeout_server_s }}s
+    {%- endif %}
+    {%- if backend.timeout_client_s is not none %}
+    timeout client {{ backend.timeout_client_s }}s
+    {%- endif %}
+    {%- if backend.timeout_http_request_s is not none %}
+    timeout http-request {{ backend.timeout_http_request_s }}s
+    {%- endif %}
+    {%- if backend.timeout_queue_s is not none %}
+    timeout queue {{ backend.timeout_queue_s }}s
+    {%- endif %}
+    # Set timeouts to support keep-alive connections
+    {%- if backend.timeout_http_keep_alive_s is not none %}
+    timeout http-keep-alive {{ backend.timeout_http_keep_alive_s }}s
+    {%- endif %}
+    {%- if backend.timeout_tunnel_s is not none %}
+    timeout tunnel {{ backend.timeout_tunnel_s }}s
+    {%- endif %}
+    # Health check configuration - use backend-specific or global defaults
+    {%- if hc.health_path %}
+    # HTTP health check with custom path
+    option httpchk GET {{ hc.health_path }}
+    http-check expect status 200
+    {%- endif %}
+    {{ hc.default_server_directive }}
+    # Servers in this backend
+    {%- for server in backend.servers %}
+    server {{ server.name }} {{ server.host }}:{{ server.port }} check
+    {%- endfor %}
+    {%- if backend.fallback_server %}
+    # Fallback to head node's Serve proxy when no ingress replicas are available
+    server {{ backend.fallback_server.name }} {{ backend.fallback_server.host }}:{{ backend.fallback_server.port }} check backup
+    {%- endif %}
+{%- if has_ingress_request_router and backend.ingress_request_router_servers %}
+backend {{ backend.name or 'unknown' }}-via-ingress-request-router
+    log global
+    # Keep the pinned data-plane path on the same connection policy as the
+    # primary backend. For streamed responses, forcing server-close can leave
+    # HAProxy holding unread server-side FINs under a burst while worker
+    # threads are still routing other requests.
+    http-reuse always
+    # Inherits the defaults block's `option redispatch 1` + retry-on, so a
+    # DOWN/slow pinned server falls through to a different replica instead of
+    # head-of-line-blocking on the original pick. One retry policy everywhere.
+    {%- if backend.timeout_connect_s is not none %}
+    timeout connect {{ backend.timeout_connect_s }}s
+    {%- endif %}
+    {%- if config.ingress_timeout_server_s is not none %}
+    timeout server {{ config.ingress_timeout_server_s }}s
+    {%- elif backend.timeout_server_s is not none %}
+    timeout server {{ backend.timeout_server_s }}s
+    {%- endif %}
+    {%- if backend.timeout_http_keep_alive_s is not none %}
+    timeout http-keep-alive {{ backend.timeout_http_keep_alive_s }}s
+    {%- endif %}
+    {%- for server in backend.servers %}
+    use-server {{ server.name }} if { var(txn.ingress_request_router_target) -m str "{{ server.name }}" }
+    {%- endfor %}
+    {%- if backend.fallback_server %}
+    # Pin-miss: route to the fallback Serve proxy, which re-pins via its own
+    # router. If the fallback is DOWN this use-server is skipped and the request
+    # load-balances onto a primary replica in this backend, so affinity lapses
+    # until the fallback's health check passes. That is plain selection-time
+    # fallthrough, not `option redispatch` (which only re-picks after a
+    # connection failure to an already-selected server).
+    use-server {{ backend.fallback_server.name }} if { var(txn.ingress_request_router_failed) -m str "unknown_replica_id" }
+    {%- endif %}
+    # `track` allows us to mirror primary-backend health and avoid double-checking.
+    {%- for server in backend.servers %}
+    server {{ server.name }} {{ server.host }}:{{ server.port }} track {{ backend.name or 'unknown' }}/{{ server.name }}
+    {%- endfor %}
+    {%- if backend.fallback_server %}
+    server {{ backend.fallback_server.name }} {{ backend.fallback_server.host }}:{{ backend.fallback_server.port }} track {{ backend.name or 'unknown' }}/{{ backend.fallback_server.name }} backup
+    {%- endif %}
+{%- endif %}
+{%- endfor %}
+{%- if config.grpc_enabled %}
+frontend grpc_frontend
+    # gRPC requires HTTP/2. HAProxy decodes H2 frames into HTTP request
+    # semantics in `mode http` when `proto h2` is on the bind line.
+    bind {{ config.grpc_frontend_host }}:{{ config.grpc_frontend_port }} proto h2
+    mode http
+    log global
+    # No per-request ingress-metrics log line here (unlike http_frontend): a
+    # gRPC call's real status is `grpc-status`, sent in HTTP/2 trailers on
+    # success, and HAProxy cannot read trailers in log-format (haproxy/haproxy#112).
+    # gRPC ingress metrics are therefore emitted by the replica instead.
+
+{{ grpc_healthz_rules|safe }}
+
+    # ListApplications must aggregate across all apps, so it goes to the
+    # head-node fallback Serve proxy rather than an individual replica.
+    acl is_list_applications path /ray.serve.RayServeAPIService/ListApplications
+{%- if grpc_fallback_backend_with_health_config %}
+    use_backend grpc_fallback_backend if is_list_applications
+{%- else %}
+    http-request return status 200 content-type application/grpc hdr grpc-status 14 hdr grpc-message "ListApplications is unavailable" if is_list_applications
+{%- endif %}
+
+    # Route per-app on the `application` metadata that Ray Serve clients attach.
+{%- for backend in grpc_backends %}
+    acl is_{{ backend.name or 'unknown' }} req.hdr(application) -m str {{ backend.app_name }}
+    use_backend {{ backend.name or 'unknown' }} if is_{{ backend.name or 'unknown' }}
+{%- endfor %}
+{%- if grpc_backends|length == 1 %}
+    # With exactly one app deployed, route there regardless of metadata so
+    # clients can call it without setting the `application` header.
+    default_backend {{ grpc_backends[0].name or 'unknown' }}
+{%- else %}
+    # Zero apps, or multiple apps without a matching `application` header.
+    default_backend default_grpc_backend
+{%- endif %}
+
+{%- if grpc_fallback_backend_with_health_config %}
+{%- set backend = grpc_fallback_backend_with_health_config.backend %}
+{%- set hc = grpc_fallback_backend_with_health_config.health_config %}
+{%- if backend.servers %}
+{%- set server = backend.servers[0] %}
+backend grpc_fallback_backend
+    mode http
+    log global
+    # gRPC health check: replay a complete unary `Healthz` request via
+    # `tcp-check send-binary` and match the healthy message in the response.
+    # `http-check` can't be used because its body is truncated at the first NUL
+    # byte and a gRPC frame always starts with the NUL compression flag, so the
+    # server would get a message-less unary and stall until timeout.
+    option tcp-check
+    tcp-check connect
+    tcp-check send-binary {{ hc.grpc_healthcheck_request_hex }}
+    tcp-check expect binary {{ hc.grpc_healthcheck_expect_hex }}
+    {{ hc.default_server_directive }}
+    # `proto h2` makes HAProxy speak HTTP/2 cleartext to the fallback gRPC server.
+    server {{ server.name }} {{ server.host }}:{{ server.port }} proto h2 check
+{%- endif %}
+{%- endif %}
+{%- if grpc_backends|length != 1 %}
+backend default_grpc_backend
+    mode http
+    log global
+    # Trailers-only NOT_FOUND. gRPC clients surface this as
+    # grpc.StatusCode.NOT_FOUND; an HTTP 503 would map to UNAVAILABLE instead.
+    acl has_application_header req.hdr(application) -m found
+    http-request return status 200 content-type application/grpc hdr grpc-status 5 hdr grpc-message "Application '%[req.hdr(application)]' not found. Ping /ray.serve.RayServeAPIService/ListApplications for available applications." if has_application_header
+    http-request return status 200 content-type application/grpc hdr grpc-status 5 hdr grpc-message "Application metadata not set. Ping /ray.serve.RayServeAPIService/ListApplications for available applications."
+{%- endif %}
+{%- for item in grpc_backends_with_health_config %}
+{%- set backend = item.backend %}
+{%- set hc = item.health_config %}
+backend {{ backend.name or 'unknown' }}
+    mode http
+    log global
+    http-reuse always
+    {%- if backend.timeout_connect_s is not none %}
+    timeout connect {{ backend.timeout_connect_s }}s
+    {%- endif %}
+    {%- if backend.timeout_server_s is not none %}
+    timeout server {{ backend.timeout_server_s }}s
+    {%- endif %}
+    {%- if backend.timeout_client_s is not none %}
+    timeout client {{ backend.timeout_client_s }}s
+    {%- endif %}
+    # gRPC health check: replay a complete unary `Healthz` request via
+    # `tcp-check send-binary` and match the healthy message in the response.
+    # `http-check` can't be used because its body is truncated at the first NUL
+    # byte and a gRPC frame always starts with the NUL compression flag, so the
+    # server would get a message-less unary and stall until timeout.
+    option tcp-check
+    tcp-check connect
+    tcp-check send-binary {{ hc.grpc_healthcheck_request_hex }}
+    tcp-check expect binary {{ hc.grpc_healthcheck_expect_hex }}
+    {{ hc.default_server_directive }}
+    # `proto h2` makes HAProxy speak HTTP/2 cleartext to backend gRPC servers.
+    {%- for server in backend.servers %}
+    server {{ server.name }} {{ server.host }}:{{ server.port }} proto h2 check
+    {%- endfor %}
+    {%- if backend.fallback_server %}
+    server {{ backend.fallback_server.name }} {{ backend.fallback_server.host }}:{{ backend.fallback_server.port }} proto h2 check backup
+    {%- endif %}
+{%- endfor %}
+{%- endif %}
+listen stats
+  bind *:{{ config.stats_port }}
+  stats enable
+  stats uri {{ config.stats_uri }}
+  stats refresh 1s
+"""

python/ray/serve/_private/ingress_request_router.lua.tmpl

@@ -0,0 +1,207 @@
+-- HAProxy Lua action that picks a backend replica via the ingress request
+-- router. Templated at config-reload time; placeholders are filled in by
+-- _write_ingress_request_router_lua in haproxy.py.
+--
+-- Bodies exceeding tune.bufsize are truncated; we forward what we have with
+-- X-Body-Truncated since prefix-routing only needs the head of the body.
+-- See RAY_SERVE_INGRESS_REQUEST_ROUTER_FORWARD_BODY in constants.py.
+
+local ROUTER_REQUEST_TIMEOUT_S = ${TIMEOUT_S}
+local FORWARD_BODY = ${FORWARD_BODY}
+-- Name of the client header that carries the session id Serve should
+-- pin on. Filled from SERVE_SESSION_ID (set via the env var
+-- RAY_SERVE_SESSION_ID_HEADER_KEY) in lowercase form. Forwarded on the
+-- same name to /internal/route so session-aware routers (e.g.
+-- ConsistentHashRouter) can use it.
+local SESSION_HEADER = "${SESSION_HEADER}"
+
+-- Per-app state. The frontend sets txn.ingress_request_router_app to the
+-- backend name whose path prefix matched, and we look up that app's
+-- router pool and replica map here. Each app routes through its own
+-- router; replica IDs are scoped to the app to avoid cross-app pinning.
+local ROUTERS = ${ROUTERS}
+local REPLICA_TARGETS = ${REPLICA_TARGETS}
+
+local function call_router(router, body, truncated_length, session_id)
+    local sock = core.tcp()
+    sock:settimeout(ROUTER_REQUEST_TIMEOUT_S)
+
+    if not sock:connect(router.host, router.port) then
+        return nil
+    end
+
+    local truncation_header = ""
+    if truncated_length then
+        truncation_header = "X-Body-Truncated: "
+            .. #body .. "/" .. truncated_length .. "\r\n"
+    end
+
+    local session_header = ""
+    if session_id and session_id ~= "" then
+        session_header = SESSION_HEADER .. ": " .. session_id .. "\r\n"
+    end
+
+    -- Connection: close so sock:receive("*a") terminates on EOF.
+    local req = "POST /internal/route HTTP/1.0\r\n"
+        .. "Host: " .. router.host_header .. "\r\n"
+        .. "Connection: close\r\n"
+        .. "Content-Type: application/json\r\n"
+        .. "Content-Length: " .. #body .. "\r\n"
+        .. truncation_header
+        .. session_header
+        .. "\r\n"
+        .. body
+
+    if not sock:send(req) then
+        sock:close()
+        return nil
+    end
+
+    local response = sock:receive("*a")
+    sock:close()
+    return response
+end
+
+local function is_http_200(response)
+    return response:match("^HTTP/[%d%.]+ 200") ~= nil
+end
+
+local function http_response_body(response)
+    local _, separator_end = response:find("\r\n\r\n", 1, true)
+    if not separator_end then
+        return ""
+    end
+    return response:sub(separator_end + 1)
+end
+
+-- Router contract: response MUST be a flat JSON object with `replica_id` as a
+-- plain string actor name (no escaped quotes, no nested objects). The regex
+-- relies on this; broaden the parser if the contract grows.
+local function extract_replica_id(json_body)
+    return json_body:match('"replica_id"%s*:%s*"([^"]+)"')
+end
+
+-- Returns the original Content-Length when `body` is shorter than the
+-- advertised length (truncated by tune.bufsize), or nil otherwise.
+local function truncated_full_length(txn, body)
+    local cl = tonumber(txn.sf:hdr("content-length"))
+    if cl and #body < cl then
+        return cl
+    end
+end
+
+-- Core routing decision. Sets either txn.ingress_request_router_target +
+-- txn.via_ingress_request_router (success) or txn.ingress_request_router_failed
+-- (failure). Both outcomes are timed by the caller; only the silent early
+-- returns in the action handler skip timing because no routing was attempted.
+local function _route_via_ingress_request_router(txn, router, replica_map)
+    -- FORWARD_BODY=false: don't read or forward the body; call the router
+    -- with body="" so a Content-Length: 0 POST still goes through routing
+    -- (any policy that needs the body must opt in via FORWARD_BODY=true).
+    -- Empty body in FORWARD_BODY=true mode is treated the same -- a
+    -- legitimate input the router can accept or reject on its own terms;
+    -- we don't synthesize a sentinel for it here.
+    local body = ""
+    local truncated = nil
+    if FORWARD_BODY then
+        body = txn.sf:req_body() or ""
+        if body ~= "" then
+            truncated = truncated_full_length(txn, body)
+            if truncated then
+                core.log(core.warning,
+                    "ingress_request_router: forwarding truncated body to router ("
+                        .. #body .. "/" .. truncated .. " bytes)")
+                ${METRICS_SET_TRUNCATED}
+            end
+        end
+    end
+
+    -- Look up the configured session-id header from the client request.
+    -- req_get_headers() returns a table keyed by lowercase header name
+    -- with array values (header can appear multiple times); take the
+    -- first occurrence. Missing header -> nil -> call_router skips the
+    -- forwarded header line entirely.
+    --
+    -- Also probe the `-`/`_` variant so an intermediate proxy that rewrites
+    -- the separator doesn't silently drop session affinity, matching the
+    -- Python-side `_matches_session_id_header` behavior in http_util.py.
+    local session_id = nil
+    if SESSION_HEADER ~= "" then
+        local hdrs = txn.http:req_get_headers()
+        local entry = hdrs and hdrs[SESSION_HEADER]
+        if hdrs and not entry then
+            local alt = SESSION_HEADER:gsub("-", "_")
+            if alt == SESSION_HEADER then
+                alt = SESSION_HEADER:gsub("_", "-")
+            end
+            entry = hdrs[alt]
+        end
+        if entry then
+            session_id = entry[0]
+        end
+    end
+
+    local response = call_router(router, body, truncated, session_id)
+    if not response then
+        txn:set_var("txn.ingress_request_router_failed", "router_unreachable")
+        return
+    end
+    if not is_http_200(response) then
+        txn:set_var("txn.ingress_request_router_failed", "router_non_200")
+        return
+    end
+
+    local replica_id = extract_replica_id(http_response_body(response))
+    if not replica_id then
+        txn:set_var("txn.ingress_request_router_failed", "unparseable_replica_id")
+        return
+    end
+
+    local server_name = replica_map[replica_id]
+    if not server_name then
+        -- Pin-miss: router named a replica not in HAProxy's server map, a
+        -- transient gap between the router view and HAProxy's config snapshot.
+        -- Arm `failed`. The frontend recovers it via the fallback proxy.
+        txn:set_var("txn.ingress_request_router_failed", "unknown_replica_id")
+        return
+    end
+
+    txn:set_var("txn.ingress_request_router_target", server_name)
+    txn:set_var("txn.via_ingress_request_router", true)
+end
+
+-- Failure semantics: every path that reaches a router decision but cannot pin
+-- a replica must arm txn.ingress_request_router_failed so the frontend's 503
+-- rule fires instead of letting the request silently fall through to the
+-- primary backend. The product invariant is: requests to a router-bearing app
+-- are served via the router or fail; there is no quiet alternative path.
+-- Exception: `unknown_replica_id` is recovered via the fallback proxy when the
+-- app has one, otherwise 503ed. The fallback re-pins via its own router so the
+-- policy is not bypassed. While the fallback is DOWN the request load-balances
+-- onto a primary replica in the router backend instead of 503ing, so affinity
+-- lapses for that window. Still counted as a failure.
+-- Two silent returns are correct: (1) the request didn't target a
+-- router-bearing app (no txn var set, no app entry in our maps), and
+-- (2) the controller hasn't pushed router/replica state for this app yet.
+-- Failure-mode bucketing belongs in observability (response header label,
+-- metric label), not in the data plane.
+core.register_action("route_via_ingress_request_router", {"http-req"}, function(txn)
+    -- Time the routing attempt regardless of outcome: both successful pins
+    -- and explicit failures (router_unreachable, router_non_200,
+    -- unparseable_replica_id, unknown_replica_id) record latency_us so
+    -- failure paths are visible in the metrics stream.
+    ${METRICS_PRE_CALL_ROUTER}
+
+    local app = txn:get_var("txn.ingress_request_router_app")
+    if not app then
+        return
+    end
+    local router = ROUTERS[app]
+    local replica_map = REPLICA_TARGETS[app]
+    if not router or not replica_map then
+        return
+    end
+
+    _route_via_ingress_request_router(txn, router, replica_map)
+    ${METRICS_POST_CALL_ROUTER}
+end, 0)

python/ray/serve/_private/local_testing_mode.py

@@ -0,0 +1,406 @@
+import asyncio
+import concurrent.futures
+import inspect
+import logging
+import queue
+import time
+from contextlib import asynccontextmanager
+from functools import wraps
+from typing import (
+    Any,
+    AsyncIterator,
+    Callable,
+    Coroutine,
+    Dict,
+    List,
+    Optional,
+    Tuple,
+    Union,
+)
+
+import ray
+from ray import cloudpickle
+from ray.serve._private.common import DeploymentID, RequestMetadata
+from ray.serve._private.constants import (
+    RAY_SERVE_RUN_SYNC_IN_THREADPOOL,
+    SERVE_LOGGER_NAME,
+)
+from ray.serve._private.replica import UserCallableWrapper
+from ray.serve._private.replica_result import ReplicaResult
+from ray.serve._private.request_router.replica_wrapper import ReplicaSelection
+from ray.serve._private.router import Router
+from ray.serve._private.utils import GENERATOR_COMPOSITION_NOT_SUPPORTED_ERROR
+from ray.serve.deployment import Deployment
+from ray.serve.exceptions import RequestCancelledError
+from ray.serve.handle import (
+    DeploymentHandle,
+    DeploymentResponse,
+    DeploymentResponseGenerator,
+)
+
+logger = logging.getLogger(SERVE_LOGGER_NAME)
+
+
+def _validate_deployment_options(
+    deployment: Deployment,
+    deployment_id: DeploymentID,
+):
+    if "num_gpus" in deployment.ray_actor_options:
+        logger.warning(
+            f"Deployment {deployment_id} has num_gpus configured. "
+            "CUDA_VISIBLE_DEVICES is not managed automatically in local testing mode. "
+        )
+
+    if "runtime_env" in deployment.ray_actor_options:
+        logger.warning(
+            f"Deployment {deployment_id} has runtime_env configured. "
+            "runtime_envs are ignored in local testing mode."
+        )
+
+
+def make_local_deployment_handle(
+    deployment: Deployment,
+    app_name: str,
+) -> DeploymentHandle:
+    """Constructs an in-process DeploymentHandle.
+
+    This is used in the application build process for local testing mode,
+    where all deployments of an app run in the local process which enables
+    faster dev iterations and use of tooling like PDB.
+
+    The user callable will be run on an asyncio loop in a separate thread
+    (sharing the same code that's used in the replica).
+
+    The constructor for the user callable is run eagerly in this function to
+    ensure that any exceptions are raised during `serve.run`.
+    """
+    deployment_id = DeploymentID(deployment.name, app_name)
+    _validate_deployment_options(deployment, deployment_id)
+    user_callable_wrapper = UserCallableWrapper(
+        deployment.func_or_class,
+        deployment.init_args,
+        deployment.init_kwargs,
+        deployment_id=deployment_id,
+        run_sync_methods_in_threadpool=RAY_SERVE_RUN_SYNC_IN_THREADPOOL,
+        run_user_code_in_separate_thread=True,
+        local_testing_mode=True,
+        deployment_config=deployment._deployment_config,
+        actor_id="local",
+        ray_actor_options=deployment.ray_actor_options,
+    )
+    try:
+        logger.info(f"Initializing local replica class for {deployment_id}.")
+        user_callable_wrapper.initialize_callable().result()
+        user_callable_wrapper.call_reconfigure(deployment.user_config, rank=0)
+    except Exception:
+        logger.exception(f"Failed to initialize deployment {deployment_id}.")
+        raise
+
+    def _create_local_router(
+        handle_id: str, deployment_id: DeploymentID, handle_options: Any
+    ) -> Router:
+        return LocalRouter(
+            user_callable_wrapper,
+            deployment_id=deployment_id,
+            handle_options=handle_options,
+        )
+
+    return DeploymentHandle(
+        deployment.name,
+        app_name,
+        _create_router=_create_local_router,
+    )
+
+
+class LocalReplicaResult(ReplicaResult):
+    """ReplicaResult used by in-process Deployment Handles."""
+
+    OBJ_REF_NOT_SUPPORTED_ERROR = RuntimeError(
+        "Converting DeploymentResponses to ObjectRefs is not supported "
+        "in local testing mode."
+    )
+    REJECTION_NOT_SUPPORTED_ERROR = RuntimeError(
+        "Request rejection is not supported in local testing mode."
+    )
+
+    def __init__(
+        self,
+        future: concurrent.futures.Future,
+        *,
+        request_id: str,
+        is_streaming: bool = False,
+        generator_result_queue: Optional[queue.Queue] = None,
+    ):
+        self._future = future
+        self._lazy_asyncio_future = None
+        self._request_id = request_id
+        self._is_streaming = is_streaming
+
+        # For streaming requests, results must be written to this queue.
+        # The queue will be consumed until the future is completed.
+        self._generator_result_queue = generator_result_queue
+        if self._is_streaming:
+            assert (
+                self._generator_result_queue is not None
+            ), "generator_result_queue must be provided for streaming results."
+
+    @property
+    def _asyncio_future(self) -> asyncio.Future:
+        if self._lazy_asyncio_future is None:
+            self._lazy_asyncio_future = asyncio.wrap_future(self._future)
+
+        return self._lazy_asyncio_future
+
+    def _process_response(f: Union[Callable, Coroutine]):
+        @wraps(f)
+        def wrapper(self, *args, **kwargs):
+            try:
+                return f(self, *args, **kwargs)
+            except (asyncio.CancelledError, concurrent.futures.CancelledError):
+                raise RequestCancelledError(self._request_id)
+
+        @wraps(f)
+        async def async_wrapper(self, *args, **kwargs):
+            try:
+                return await f(self, *args, **kwargs)
+            except (asyncio.CancelledError, concurrent.futures.CancelledError):
+                raise RequestCancelledError(self._request_id)
+
+        if inspect.iscoroutinefunction(f):
+            return async_wrapper
+        else:
+            return wrapper
+
+    @_process_response
+    async def get_rejection_response(self):
+        raise self.REJECTION_NOT_SUPPORTED_ERROR
+
+    @_process_response
+    def get(self, timeout_s: Optional[float]):
+        assert (
+            not self._is_streaming
+        ), "get() can only be called on a non-streaming result."
+
+        try:
+            return self._future.result(timeout=timeout_s)
+        except concurrent.futures.TimeoutError:
+            raise TimeoutError("Timed out waiting for result.")
+
+    @_process_response
+    async def get_async(self):
+        assert (
+            not self._is_streaming
+        ), "get_async() can only be called on a non-streaming result."
+
+        return await self._asyncio_future
+
+    @_process_response
+    def __next__(self):
+        assert self._is_streaming, "next() can only be called on a streaming result."
+
+        while True:
+            if self._future.done() and self._generator_result_queue.empty():
+                if self._future.exception():
+                    raise self._future.exception()
+                else:
+                    raise StopIteration
+
+            try:
+                return self._generator_result_queue.get(timeout=0.01)
+            except queue.Empty:
+                pass
+
+    @_process_response
+    async def __anext__(self):
+        assert self._is_streaming, "anext() can only be called on a streaming result."
+
+        # This callback does not pull from the queue, only checks that a result is
+        # available, else there is a race condition where the future finishes and the
+        # queue is empty, but this function hasn't returned the result yet.
+        def _wait_for_result():
+            while True:
+                if self._future.done() or not self._generator_result_queue.empty():
+                    return
+                time.sleep(0.01)
+
+        wait_for_result_task = asyncio.get_running_loop().create_task(
+            asyncio.to_thread(_wait_for_result),
+        )
+        done, _ = await asyncio.wait(
+            [self._asyncio_future, wait_for_result_task],
+            return_when=asyncio.FIRST_COMPLETED,
+        )
+
+        if not self._generator_result_queue.empty():
+            return self._generator_result_queue.get()
+
+        if self._asyncio_future.exception():
+            raise self._asyncio_future.exception()
+
+        raise StopAsyncIteration
+
+    def add_done_callback(self, callback: Callable):
+        self._future.add_done_callback(callback)
+
+    def cancel(self):
+        self._future.cancel()
+
+    def to_object_ref(self, timeout_s: Optional[float]) -> ray.ObjectRef:
+        raise self.OBJ_REF_NOT_SUPPORTED_ERROR
+
+    async def to_object_ref_async(self) -> ray.ObjectRef:
+        raise self.OBJ_REF_NOT_SUPPORTED_ERROR
+
+    def to_object_ref_gen(self) -> ray.ObjectRefGenerator:
+        raise self.OBJ_REF_NOT_SUPPORTED_ERROR
+
+
+class LocalRouter(Router):
+    def __init__(
+        self,
+        user_callable_wrapper: UserCallableWrapper,
+        deployment_id: DeploymentID,
+        handle_options: Any,
+    ):
+        self._deployment_id = deployment_id
+        self._user_callable_wrapper = user_callable_wrapper
+        assert (
+            self._user_callable_wrapper._callable is not None
+        ), "User callable must already be initialized."
+
+    def running_replicas_populated(self) -> bool:
+        return True
+
+    def _resolve_deployment_responses(
+        self, request_args: Tuple[Any], request_kwargs: Dict[str, Any]
+    ) -> Tuple[Tuple[Any], Dict[str, Any]]:
+        """Replace DeploymentResponse objects with their results.
+
+        NOTE(edoakes): this currently calls the blocking `.result()` method
+        on the responses to resolve them to their values. This is a divergence
+        from the remote codepath where they're resolved concurrently.
+        """
+
+        def _new_arg(arg: Any) -> Any:
+            if isinstance(arg, DeploymentResponse):
+                new_arg = arg.result(_skip_asyncio_check=True)
+            elif isinstance(arg, DeploymentResponseGenerator):
+                raise GENERATOR_COMPOSITION_NOT_SUPPORTED_ERROR
+            else:
+                new_arg = arg
+
+            return new_arg
+
+        # Serialize and deserialize the arguments to mimic remote call behavior.
+        return cloudpickle.loads(
+            cloudpickle.dumps(
+                (
+                    tuple(_new_arg(arg) for arg in request_args),
+                    {k: _new_arg(v) for k, v in request_kwargs.items()},
+                )
+            )
+        )
+
+    def assign_request(
+        self,
+        request_meta: RequestMetadata,
+        *request_args,
+        **request_kwargs,
+    ) -> concurrent.futures.Future[LocalReplicaResult]:
+        request_args, request_kwargs = self._resolve_deployment_responses(
+            request_args, request_kwargs
+        )
+
+        if request_meta.is_streaming:
+            generator_result_queue = queue.Queue()
+
+            def generator_result_callback(item: Any):
+                generator_result_queue.put_nowait(item)
+
+        else:
+            generator_result_queue = None
+            generator_result_callback = None
+
+        # Conform to the router interface of returning a future to the ReplicaResult.
+        if request_meta.is_http_request:
+            fut = self._user_callable_wrapper._call_http_entrypoint(
+                request_meta,
+                request_args,
+                request_kwargs,
+                generator_result_callback=generator_result_callback,
+            )
+        elif request_meta.is_streaming:
+            fut = self._user_callable_wrapper._call_user_generator(
+                request_meta,
+                request_args,
+                request_kwargs,
+                enqueue=generator_result_callback,
+            )
+        else:
+            fut = self._user_callable_wrapper.call_user_method(
+                request_meta,
+                request_args,
+                request_kwargs,
+            )
+        noop_future = concurrent.futures.Future()
+        noop_future.set_result(
+            LocalReplicaResult(
+                fut,
+                request_id=request_meta.request_id,
+                is_streaming=request_meta.is_streaming,
+                generator_result_queue=generator_result_queue,
+            )
+        )
+        return noop_future
+
+    @asynccontextmanager
+    async def choose_replica(
+        self,
+        request_meta: RequestMetadata,
+        *request_args,
+        **request_kwargs,
+    ) -> AsyncIterator[ReplicaSelection]:
+        """Choose replica is not supported in local testing mode.
+
+        This is a stub implementation to satisfy the Router ABC interface.
+        """
+        raise NotImplementedError(
+            "choose_replica is not supported in local testing mode. "
+            "Use assign_request instead."
+        )
+        yield  # Make this a generator for asynccontextmanager
+
+    def dispatch(
+        self,
+        selection: ReplicaSelection,
+        request_meta: RequestMetadata,
+        *request_args,
+        **request_kwargs,
+    ) -> concurrent.futures.Future[ReplicaResult]:
+        """Dispatch is not supported in local testing mode.
+
+        This is a stub implementation to satisfy the Router ABC interface.
+        """
+        raise NotImplementedError(
+            "dispatch is not supported in local testing mode. "
+            "Use assign_request instead."
+        )
+
+    async def broadcast(
+        self,
+        request_meta: RequestMetadata,
+        *request_args,
+        **request_kwargs,
+    ) -> List[ReplicaResult]:
+        """Broadcast in local testing mode calls the single local replica."""
+        result_future = self.assign_request(
+            request_meta, *request_args, **request_kwargs
+        )
+        # In local testing mode there is only one replica.
+        replica_result = result_future.result()
+        return [replica_result]
+
+    def shutdown(self):
+        noop_future = concurrent.futures.Future()
+        noop_future.set_result(None)
+        return noop_future

python/ray/serve/_private/logging_utils.py

@@ -0,0 +1,552 @@
+import builtins
+import logging
+import os
+import sys
+import traceback
+from typing import Any, Optional
+
+import ray
+from ray._common.filters import CoreContextFilter
+from ray._common.formatters import JSONFormatter, TextFormatter
+from ray._common.ray_constants import LOGGING_ROTATE_BACKUP_COUNT, LOGGING_ROTATE_BYTES
+from ray.serve._private.common import ServeComponentType
+from ray.serve._private.constants import (
+    RAY_SERVE_ENABLE_MEMORY_PROFILING,
+    RAY_SERVE_LOG_CLIENT_ADDRESS,
+    RAY_SERVE_LOG_TO_STDERR,
+    SERVE_LOG_APPLICATION,
+    SERVE_LOG_COMPONENT,
+    SERVE_LOG_COMPONENT_ID,
+    SERVE_LOG_DEPLOYMENT,
+    SERVE_LOG_LEVEL_NAME,
+    SERVE_LOG_MESSAGE,
+    SERVE_LOG_RECORD_FORMAT,
+    SERVE_LOG_REPLICA,
+    SERVE_LOG_REQUEST_ID,
+    SERVE_LOG_ROUTE,
+    SERVE_LOG_TIME,
+    SERVE_LOG_UNWANTED_ATTRS,
+    SERVE_LOGGER_NAME,
+)
+from ray.serve._private.utils import get_component_file_name
+from ray.serve.schema import EncodingType, LoggingConfig
+
+buildin_print = builtins.print
+
+
+def should_skip_context_filter(record: logging.LogRecord) -> bool:
+    """Check if the log record should skip the context filter."""
+    return getattr(record, "skip_context_filter", False)
+
+
+class ServeCoreContextFilter(CoreContextFilter):
+    def filter(self, record: logging.LogRecord) -> bool:
+        if should_skip_context_filter(record):
+            return True
+        return super().filter(record)
+
+
+class ServeComponentFilter(logging.Filter):
+    """Serve component filter.
+
+    The filter will add the component name, id, and type to the log record.
+    """
+
+    def __init__(
+        self,
+        component_name: str,
+        component_id: str,
+        component_type: Optional[ServeComponentType] = None,
+    ):
+        self.component_name = component_name
+        self.component_id = component_id
+        self.component_type = component_type
+
+    def filter(self, record: logging.LogRecord) -> bool:
+        """Add component attributes to the log record.
+
+        Note: the filter doesn't do any filtering, it only adds the component
+        attributes.
+        """
+        if should_skip_context_filter(record):
+            return True
+        if self.component_type and self.component_type == ServeComponentType.REPLICA:
+            setattr(record, SERVE_LOG_DEPLOYMENT, self.component_name)
+            setattr(record, SERVE_LOG_REPLICA, self.component_id)
+            setattr(record, SERVE_LOG_COMPONENT, self.component_type)
+        else:
+            setattr(record, SERVE_LOG_COMPONENT, self.component_name)
+            setattr(record, SERVE_LOG_COMPONENT_ID, self.component_id)
+
+        return True
+
+
+class ServeContextFilter(logging.Filter):
+    """Serve context filter.
+
+    The filter will add the route, request id, app name to the log record.
+
+    Note: the filter doesn't do any filtering, it only adds the serve request context
+    attributes.
+    """
+
+    def filter(self, record):
+        if should_skip_context_filter(record):
+            return True
+
+        request_context = ray.serve.context._get_serve_request_context()
+        if request_context.route:
+            setattr(record, SERVE_LOG_ROUTE, request_context.route)
+        if request_context.request_id:
+            setattr(record, SERVE_LOG_REQUEST_ID, request_context.request_id)
+        if request_context.app_name:
+            setattr(record, SERVE_LOG_APPLICATION, request_context.app_name)
+        return True
+
+
+class ServeLogAttributeRemovalFilter(logging.Filter):
+    """Serve log attribute removal filter.
+
+    The filter will remove unwanted attributes on the log record so they won't be
+    included in the structured logs.
+
+    Note: the filter doesn't do any filtering, it only removes unwanted attributes.
+    """
+
+    def filter(self, record):
+        for key in SERVE_LOG_UNWANTED_ATTRS:
+            if hasattr(record, key):
+                delattr(record, key)
+
+        return True
+
+
+class ServeFormatter(TextFormatter):
+    """Serve Logging Formatter
+
+    The formatter will generate the log format on the fly based on the field of record.
+    Optimized to pre-compute format strings and formatters for better performance.
+    """
+
+    COMPONENT_LOG_FMT = f"%({SERVE_LOG_LEVEL_NAME})s %({SERVE_LOG_TIME})s {{{SERVE_LOG_COMPONENT}}} {{{SERVE_LOG_COMPONENT_ID}}} "  # noqa:E501
+
+    def __init__(
+        self,
+        component_name: str,
+        component_id: str,
+        fmt: Optional[str] = None,
+        datefmt: Optional[str] = None,
+        style: str = "%",
+        validate: bool = True,
+    ):
+        super().__init__(fmt, datefmt, style, validate)
+        self.component_log_fmt = ServeFormatter.COMPONENT_LOG_FMT.format(
+            component_name=component_name, component_id=component_id
+        )
+
+        # Pre-compute format strings and formatters for performance
+        self._precompute_formatters()
+
+    def set_additional_log_standard_attrs(self, *args, **kwargs):
+        super().set_additional_log_standard_attrs(*args, **kwargs)
+        self._precompute_formatters()
+
+    def _precompute_formatters(self):
+        self.base_formatter = self._create_formatter([])
+        self.request_formatter = self._create_formatter(
+            [SERVE_LOG_RECORD_FORMAT[SERVE_LOG_REQUEST_ID]]
+        )
+
+    def _create_formatter(self, initial_attrs: list) -> logging.Formatter:
+        attrs = initial_attrs.copy()
+        attrs.extend([f"%({k})s" for k in self.additional_log_standard_attrs])
+        attrs.append(SERVE_LOG_RECORD_FORMAT[SERVE_LOG_MESSAGE])
+
+        format_string = self.component_log_fmt + " ".join(attrs)
+        return logging.Formatter(format_string)
+
+    def format(self, record: logging.LogRecord) -> str:
+        """Format the log record into the format string.
+
+        Args:
+            record: The log record to be formatted.
+
+        Returns:
+            The formatted log record in string format.
+        """
+        # Use pre-computed formatters for better performance
+        if SERVE_LOG_REQUEST_ID in record.__dict__:
+            return self.request_formatter.format(record)
+        else:
+            return self.base_formatter.format(record)
+
+
+def format_grpc_peer_address(peer: str) -> str:
+    """Extract the client address from a gRPC peer() string.
+
+    gRPC peer() returns "ipv4:host:port" or "ipv6:%5Bhost%5D:port".
+    Strips the protocol prefix and URL-decodes IPv6 brackets.
+    """
+    if not peer:
+        return ""
+    for prefix in ("ipv4:", "ipv6:"):
+        if peer.startswith(prefix):
+            addr = peer[len(prefix) :]
+            return addr.replace("%5B", "[").replace("%5D", "]")
+    return peer
+
+
+def format_client_address(client) -> str:
+    """Format a raw ASGI scope client value into a string."""
+    if isinstance(client, (tuple, list)):
+        if len(client) != 2:
+            return ":".join(str(x) for x in client)
+        host, port = str(client[0]), str(client[1])
+        # Wrap IPv6 addresses in brackets to avoid ambiguity (e.g. [::1]:54321).
+        if ":" in host:
+            return f"[{host}]:{port}"
+        return f"{host}:{port}"
+    elif isinstance(client, str):
+        return client
+    return str(client) if client else ""
+
+
+def access_log_msg(
+    *, method: str, route: str, status: str, latency_ms: float, client: str = ""
+):
+    """Returns a formatted message for an HTTP or ServeHandle access log."""
+    if client and RAY_SERVE_LOG_CLIENT_ADDRESS:
+        return f"{client} {method} {route} {status} {latency_ms:.1f}ms"
+    return f"{method} {route} {status} {latency_ms:.1f}ms"
+
+
+def log_to_stderr_filter(record: logging.LogRecord) -> bool:
+    """Filters log records based on a parameter in the `extra` dictionary."""
+    if not hasattr(record, "log_to_stderr") or record.log_to_stderr is None:
+        return True
+
+    return record.log_to_stderr
+
+
+def log_access_log_filter(record: logging.LogRecord) -> bool:
+    """Filters ray serve access log based on 'serve_access_log' key in `extra` dict."""
+    if not hasattr(record, "serve_access_log") or record.serve_access_log is None:
+        return True
+
+    return not record.serve_access_log
+
+
+def get_component_logger_file_path() -> Optional[str]:
+    """Returns the relative file path for the Serve logger, if it exists.
+
+    If a logger was configured through configure_component_logger() for the Serve
+    component that's calling this function, this returns the location of the log file
+    relative to the ray logs directory.
+    """
+    logger = logging.getLogger(SERVE_LOGGER_NAME)
+    for handler in logger.handlers:
+        if isinstance(handler, logging.handlers.MemoryHandler):
+            absolute_path = handler.target.baseFilename
+            ray_logs_dir = ray._private.worker._global_node.get_logs_dir_path()
+            if absolute_path.startswith(ray_logs_dir):
+                return absolute_path[len(ray_logs_dir) :]
+
+
+class StreamToLogger(object):
+    """
+    Fake file-like stream object that redirects writes to a logger instance.
+
+    This comes from https://stackoverflow.com/a/36296215 directly.
+    """
+
+    def __init__(self, logger: logging.Logger, log_level: int, original_object: Any):
+        self._logger = logger
+        self._log_level = log_level
+        self._original_object = original_object
+        self._linebuf = ""
+
+    def __getattr__(self, attr: str) -> Any:
+        # getting attributes from the original object
+        return getattr(self._original_object, attr)
+
+    @staticmethod
+    def get_stacklevel() -> int:
+        """Rewind stack to get the stacklevel for the user code.
+
+        Going from the back of the traceback and traverse until it's no longer in
+        logging_utils.py or site-packages.
+        """
+        reverse_traces = traceback.extract_stack()[::-1]
+        for index, trace in enumerate(reverse_traces):
+            if (
+                "logging_utils.py" not in trace.filename
+                and "site-packages" not in trace.filename
+            ):
+                return index
+        return 1
+
+    def write(self, buf: str):
+        temp_linebuf = self._linebuf + buf
+        self._linebuf = ""
+        for line in temp_linebuf.splitlines(True):
+            # From the io.TextIOWrapper docs:
+            #   On output, if newline is None, any '\n' characters written
+            #   are translated to the system default line separator.
+            # By default sys.stdout.write() expects '\n' newlines and then
+            # translates them so this is still cross-platform.
+            if line[-1] == "\n":
+                self._logger.log(
+                    self._log_level,
+                    line.rstrip(),
+                    stacklevel=self.get_stacklevel(),
+                )
+            else:
+                self._linebuf += line
+
+    def flush(self):
+        if self._linebuf != "":
+            self._logger.log(
+                self._log_level,
+                self._linebuf.rstrip(),
+                stacklevel=self.get_stacklevel(),
+            )
+        self._linebuf = ""
+
+    def isatty(self) -> bool:
+        return True
+
+
+def redirected_print(*objects, sep=" ", end="\n", file=None, flush=False):
+    """Implement python's print function to redirect logs to Serve's logger.
+
+    If the file is set to anything other than stdout, stderr, or None, call the
+    builtin print. Else, construct the message and redirect to Serve's logger.
+
+    See https://docs.python.org/3/library/functions.html#print
+    """
+    if file not in [sys.stdout, sys.stderr, None]:
+        return buildin_print(objects, sep=sep, end=end, file=file, flush=flush)
+
+    serve_logger = logging.getLogger(SERVE_LOGGER_NAME)
+    message = sep.join(map(str, objects)) + end
+    # We monkey patched print function, so this is always at stack level 2.
+    serve_logger.log(logging.INFO, message, stacklevel=2)
+
+
+def configure_component_logger(
+    *,
+    component_name: str,
+    component_id: str,
+    logging_config: LoggingConfig,
+    component_type: Optional[ServeComponentType] = None,
+    max_bytes: Optional[int] = None,
+    backup_count: Optional[int] = None,
+    stream_handler_only: bool = False,
+    buffer_size: int = 1,
+):
+    """Configure a logger to be used by a Serve component.
+
+    The logger will log using a standard format to make components identifiable
+    using the provided name and unique ID for this instance (e.g., replica ID).
+
+    This logger will *not* propagate its log messages to the parent logger(s).
+    """
+    logger = logging.getLogger(SERVE_LOGGER_NAME)
+    logger.propagate = False
+    logger.setLevel(logging_config.log_level)
+    logger.handlers.clear()
+
+    serve_formatter = ServeFormatter(component_name, component_id)
+    json_formatter = JSONFormatter()
+    if logging_config.additional_log_standard_attrs:
+        json_formatter.set_additional_log_standard_attrs(
+            logging_config.additional_log_standard_attrs
+        )
+        serve_formatter.set_additional_log_standard_attrs(
+            logging_config.additional_log_standard_attrs
+        )
+
+    # Only add stream handler if RAY_SERVE_LOG_TO_STDERR is True or if
+    # `stream_handler_only` is set to True.
+    if RAY_SERVE_LOG_TO_STDERR or stream_handler_only:
+        stream_handler = logging.StreamHandler()
+        stream_handler.setFormatter(serve_formatter)
+        stream_handler.addFilter(log_to_stderr_filter)
+        stream_handler.addFilter(ServeContextFilter())
+        if logging_config.enable_access_log is False:
+            stream_handler.addFilter(log_access_log_filter)
+        logger.addHandler(stream_handler)
+
+    # Skip setting up file handler and stdout/stderr redirect if `stream_handler_only`
+    # is set to True. Logger such as default serve logger can be configured outside the
+    # context of a Serve component, we don't want those logs to redirect into serve's
+    # logger and log files.
+    if stream_handler_only:
+        return
+
+    if logging_config.logs_dir:
+        logs_dir = logging_config.logs_dir
+    else:
+        logs_dir = get_serve_logs_dir()
+    os.makedirs(logs_dir, exist_ok=True)
+
+    if max_bytes is None:
+        max_bytes = ray._private.worker._global_node.max_bytes
+    if backup_count is None:
+        backup_count = ray._private.worker._global_node.backup_count
+
+    log_file_name = get_component_file_name(
+        component_name=component_name,
+        component_id=component_id,
+        component_type=component_type,
+        suffix=".log",
+    )
+
+    file_handler = logging.handlers.RotatingFileHandler(
+        os.path.join(logs_dir, log_file_name),
+        maxBytes=max_bytes,
+        backupCount=backup_count,
+    )
+    # Create a memory handler that buffers log records and flushes to file handler
+    # Buffer capacity: buffer_size records
+    # Flush triggers: buffer full, ERROR messages, or explicit flush
+    memory_handler = logging.handlers.MemoryHandler(
+        capacity=buffer_size,
+        target=file_handler,
+        flushLevel=logging.ERROR,  # Auto-flush on ERROR/CRITICAL
+    )
+    # Add filters directly to the memory handler effective for both buffered and non buffered cases
+    if logging_config.encoding == EncodingType.JSON:
+        memory_handler.addFilter(ServeCoreContextFilter())
+        memory_handler.addFilter(ServeContextFilter())
+        memory_handler.addFilter(
+            ServeComponentFilter(component_name, component_id, component_type)
+        )
+        file_handler.setFormatter(json_formatter)
+    else:
+        file_handler.setFormatter(serve_formatter)
+
+    if logging_config.enable_access_log is False:
+        memory_handler.addFilter(log_access_log_filter)
+    else:
+        memory_handler.addFilter(ServeContextFilter())
+
+    # Remove unwanted attributes from the log record.
+    memory_handler.addFilter(ServeLogAttributeRemovalFilter())
+
+    # Redirect print, stdout, and stderr to Serve logger, only when it's on the replica.
+    if not RAY_SERVE_LOG_TO_STDERR and component_type == ServeComponentType.REPLICA:
+        builtins.print = redirected_print
+        sys.stdout = StreamToLogger(logger, logging.INFO, sys.stdout)
+        sys.stderr = StreamToLogger(logger, logging.INFO, sys.stderr)
+
+    # Add the memory handler instead of the file handler directly
+    logger.addHandler(memory_handler)
+
+
+def configure_default_serve_logger():
+    """Helper function to configure the default Serve logger that's used outside of
+    individual Serve components."""
+    configure_component_logger(
+        component_name="serve",
+        component_id=str(os.getpid()),
+        logging_config=LoggingConfig(),
+        max_bytes=LOGGING_ROTATE_BYTES,
+        backup_count=LOGGING_ROTATE_BACKUP_COUNT,
+        stream_handler_only=True,
+    )
+
+
+def configure_component_memory_profiler(
+    component_name: str,
+    component_id: str,
+    component_type: Optional[ServeComponentType] = None,
+):
+    """Configures the memory logger for this component.
+
+    Does nothing if RAY_SERVE_ENABLE_MEMORY_PROFILING is disabled.
+    """
+
+    if RAY_SERVE_ENABLE_MEMORY_PROFILING:
+        logger = logging.getLogger(SERVE_LOGGER_NAME)
+
+        try:
+            import memray
+
+            logs_dir = get_serve_logs_dir()
+            memray_file_name = get_component_file_name(
+                component_name=component_name,
+                component_id=component_id,
+                component_type=component_type,
+                suffix="_memray_0.bin",
+            )
+            memray_file_path = os.path.join(logs_dir, memray_file_name)
+
+            # If the actor restarted, memray requires a new file to start
+            # tracking memory.
+            restart_counter = 1
+            while os.path.exists(memray_file_path):
+                memray_file_name = get_component_file_name(
+                    component_name=component_name,
+                    component_id=component_id,
+                    component_type=component_type,
+                    suffix=f"_memray_{restart_counter}.bin",
+                )
+                memray_file_path = os.path.join(logs_dir, memray_file_name)
+                restart_counter += 1
+
+            # Memray usually tracks the memory usage of only a block of code
+            # within a context manager. We explicitly call __enter__ here
+            # instead of using a context manager to track memory usage across
+            # all of the caller's code instead.
+            memray.Tracker(memray_file_path, native_traces=True).__enter__()
+
+            logger.info(
+                "RAY_SERVE_ENABLE_MEMORY_PROFILING is enabled. Started a "
+                "memray tracker on this actor. Tracker file located at "
+                f'"{memray_file_path}"'
+            )
+
+        except ImportError:
+            logger.warning(
+                "RAY_SERVE_ENABLE_MEMORY_PROFILING is enabled, but memray "
+                "is not installed. No memory profiling is happening. "
+                "`pip install memray` to enable memory profiling."
+            )
+
+
+def get_serve_logs_dir() -> str:
+    """Get the directory that stores Serve log files.
+
+    If `ray._private.worker._global_node` is None (running outside the context of Ray),
+    then the current working directory with subdirectory of serve is used as the logs
+    directory. Otherwise, the logs directory is determined by the global node's logs
+    directory path.
+    """
+    if ray._private.worker._global_node is None:
+        return os.path.join(os.getcwd(), "serve")
+
+    return os.path.join(ray._private.worker._global_node.get_logs_dir_path(), "serve")
+
+
+class LoggingContext:
+    """
+    Context manager to manage logging behaviors within a particular block, such as:
+    1) Overriding logging level
+
+    Source (python3 official documentation)
+    https://docs.python.org/3/howto/logging-cookbook.html#using-a-context-manager-for-selective-logging # noqa: E501
+    """
+
+    def __init__(self, logger, level=None):
+        self.logger = logger
+        self.level = level
+
+    def __enter__(self):
+        if self.level is not None:
+            self.old_level = self.logger.level
+            self.logger.setLevel(self.level)
+
+    def __exit__(self, et, ev, tb):
+        if self.level is not None:
+            self.logger.setLevel(self.old_level)

python/ray/serve/_private/long_poll.py

@@ -0,0 +1,697 @@
+import asyncio
+import contextvars
+import logging
+import os
+import random
+import time
+from asyncio import sleep
+from asyncio.events import AbstractEventLoop
+from collections import defaultdict
+from collections.abc import Mapping
+from dataclasses import dataclass
+from enum import Enum, auto
+from typing import (
+    Any,
+    Callable,
+    DefaultDict,
+    Dict,
+    Iterable,
+    Optional,
+    Set,
+    Tuple,
+    Union,
+)
+
+import ray
+from ray._common.utils import get_or_create_event_loop
+from ray.serve._private.constants import (
+    DEFAULT_LATENCY_BUCKET_MS,
+    RAY_SERVE_COMPACT_LONG_POLL_METRIC_TAGS,
+    SERVE_LOGGER_NAME,
+)
+from ray.serve.generated.serve_pb2 import (
+    DeploymentTargetInfo,
+    EndpointInfo as EndpointInfoProto,
+    EndpointSet,
+    LongPollRequest,
+    LongPollResult,
+    UpdatedObject as UpdatedObjectProto,
+)
+from ray.util import metrics
+
+logger = logging.getLogger(SERVE_LOGGER_NAME)
+
+# Each LongPollClient will send requests to LongPollHost to poll changes
+# as blocking awaitable. This doesn't scale if we have many client instances
+# that will slow down, or even block controller actor's event loop if near
+# its max_concurrency limit. Therefore we timeout a polling request after
+# a few seconds and let each client retry on their end.
+# We randomly select a timeout within this range to avoid a "thundering herd"
+# when there are many clients subscribing at the same time.
+LISTEN_FOR_CHANGE_REQUEST_TIMEOUT_S = (
+    float(os.environ.get("LISTEN_FOR_CHANGE_REQUEST_TIMEOUT_S_LOWER_BOUND", "30")),
+    float(os.environ.get("LISTEN_FOR_CHANGE_REQUEST_TIMEOUT_S_UPPER_BOUND", "60")),
+)
+
+
+class LongPollNamespace(Enum):
+    def __repr__(self):
+        return f"{self.__class__.__name__}.{self.name}"
+
+    DEPLOYMENT_TARGETS = auto()
+    ROUTE_TABLE = auto()
+    GLOBAL_LOGGING_CONFIG = auto()
+    DEPLOYMENT_CONFIG = auto()
+    TARGET_GROUPS = auto()
+    FALLBACK_TARGETS = auto()
+
+
+@dataclass
+class UpdatedObject:
+    object_snapshot: Any
+    # The identifier for the object's version. There is not sequential relation
+    # among different object's snapshot_ids.
+    snapshot_id: int
+    # Timestamp (in seconds since epoch) when notify_changed was called.
+    # Used by clients to measure end-to-end propagation latency.
+    notify_timestamp: float
+
+
+# Type signature for the update state callbacks. E.g.
+# async def update_state(updated_object: Any):
+#     do_something(updated_object)
+UpdateStateCallable = Callable[[Any], None]
+KeyType = Union[str, LongPollNamespace, Tuple[LongPollNamespace, str]]
+
+
+def _get_metric_namespace_tag(key: KeyType) -> str:
+    """Extract the namespace string from a long poll key for metric tags.
+
+    When RAY_SERVE_COMPACT_LONG_POLL_METRIC_TAGS is enabled, returns only
+    the LongPollNamespace enum name (e.g., "DEPLOYMENT_CONFIG") for tuple keys
+    like (LongPollNamespace.DEPLOYMENT_CONFIG, "deployment_name"), avoiding
+    high-cardinality metric labels that would otherwise include per-deployment
+    identifiers. When disabled (default), returns str(key) preserving
+    deployment-level metric granularity.
+    """
+    if not RAY_SERVE_COMPACT_LONG_POLL_METRIC_TAGS:
+        return str(key)
+    if isinstance(key, tuple):
+        return key[0].name if isinstance(key[0], LongPollNamespace) else str(key[0])
+    elif isinstance(key, LongPollNamespace):
+        return key.name
+    else:
+        return str(key)
+
+
+class LongPollState(Enum):
+    TIME_OUT = auto()
+
+
+class LongPollClient:
+    """The asynchronous long polling client.
+
+    Args:
+        host_actor: handle to actor embedding LongPollHost.
+        key_listeners: a dictionary mapping keys to
+          callbacks to be called on state update for the corresponding keys.
+        call_in_event_loop: an asyncio event loop
+          to post the callback into.
+        client_id: identifier reported back to the host if this client
+          disables itself.
+    """
+
+    def __init__(
+        self,
+        host_actor: Any,
+        key_listeners: Dict[KeyType, UpdateStateCallable],
+        call_in_event_loop: AbstractEventLoop,
+        client_id: str,
+    ) -> None:
+        # We used to allow this to be optional, but due to Ray Client issue
+        # we now enforce all long poll client to post callback to event loop
+        # See https://github.com/ray-project/ray/issues/20971
+        assert call_in_event_loop is not None
+
+        self.host_actor = host_actor
+        self.key_listeners = key_listeners
+        self.event_loop = call_in_event_loop
+        self.client_id = client_id
+        # The initial snapshot id for each key is < 0,
+        # but real snapshot keys in the long poll host are always >= 0,
+        # so this will always trigger an initial update.
+        self.snapshot_ids: Dict[KeyType, int] = dict.fromkeys(
+            self.key_listeners.keys(), -1
+        )
+        self.is_running = True
+
+        # Metric to track end-to-end latency from controller to client
+        self.long_poll_latency_histogram = metrics.Histogram(
+            "serve_long_poll_latency_ms",
+            description=(
+                "The time in milliseconds for updates to propagate from "
+                "controller to clients."
+            ),
+            boundaries=DEFAULT_LATENCY_BUCKET_MS,
+            tag_keys=("namespace",),
+        )
+
+        # NOTE(edoakes): we schedule the initial _poll_next call with an empty context
+        # so that Ray will not recursively cancel the underlying `listen_for_change`
+        # task. See: https://github.com/ray-project/ray/issues/52476.
+        self.event_loop.call_soon_threadsafe(
+            self._poll_next, context=contextvars.Context()
+        )
+
+    def stop(self) -> None:
+        """Stop the long poll client after the next RPC returns."""
+        self.is_running = False
+
+    def add_key_listeners(
+        self, key_listeners: Dict[KeyType, UpdateStateCallable]
+    ) -> None:
+        """Add more key listeners to the client.
+        The new listeners will only be included in the *next* long poll request;
+        the current request will continue with the existing listeners.
+
+        If a key is already in the client, the new listener will replace the old one,
+        but the snapshot ID will be preserved, so the new listener will only be called
+        on the *next* update to that key.
+        """
+        # We need to run the underlying method in the same event loop that runs
+        # the long poll loop, because we need to mutate the mapping of snapshot IDs,
+        # which also needs to be serialized by the long poll's RPC to the
+        # Serve Controller. If those happened concurrently in different threads,
+        # we could get a `RuntimeError: dictionary changed size during iteration`.
+        # See https://github.com/ray-project/ray/pull/52793 for more details.
+        self.event_loop.call_soon_threadsafe(self._add_key_listeners, key_listeners)
+
+    def _add_key_listeners(
+        self, key_listeners: Dict[KeyType, UpdateStateCallable]
+    ) -> None:
+        """Inner method that actually adds the key listeners, to be called
+        via call_soon_threadsafe for thread safety.
+        """
+        # Only initialize snapshot ids for *new* keys.
+        self.snapshot_ids.update(
+            {key: -1 for key in key_listeners.keys() if key not in self.key_listeners}
+        )
+        self.key_listeners.update(key_listeners)
+
+    def _on_callback_completed(self, trigger_at: int):
+        """Called after a single callback is completed.
+
+        When the total number of callback completed equals to trigger_at,
+        _poll_next() will be called. This is designed to make sure we only
+        _poll_next() after all the state callbacks completed. This is a
+        way to serialize the callback invocations between object versions.
+        """
+        self._callbacks_processed_count += 1
+        if self._callbacks_processed_count == trigger_at:
+            self._poll_next()
+
+    def _poll_next(self):
+        """Poll the update. The callback is expected to scheduler another
+        _poll_next call.
+        """
+        if not self.is_running:
+            return
+
+        self._callbacks_processed_count = 0
+        self._current_ref = self.host_actor.listen_for_change.remote(self.snapshot_ids)
+        self._current_ref._on_completed(lambda update: self._process_update(update))
+
+    def _schedule_to_event_loop(self, callback):
+        # Schedule the next iteration only if the loop is running.
+        # The event loop might not be running if users used a cached
+        # version across loops.
+        if not self.is_running:
+            return
+
+        if self.event_loop.is_running():
+            self.event_loop.call_soon_threadsafe(callback)
+        else:
+            reason = "Bound asyncio event loop is not running; controller updates cannot be delivered."
+            logger.error(
+                f"LongPollClient {self.client_id!r} has been disabled: {reason} "
+                f"Keep the loop running for the lifetime of this process."
+            )
+            self.is_running = False
+            # Fire-and-forget notify so the controller logs this client as disabled.
+            try:
+                self.host_actor.notify_long_poll_client_disabled.remote(
+                    self.client_id, reason
+                )
+            except Exception:
+                logger.exception(
+                    "Failed to notify host that LongPollClient "
+                    f"{self.client_id!r} disabled itself."
+                )
+
+    def _process_update(self, updates: Dict[str, UpdatedObject]):
+        if isinstance(updates, (ray.exceptions.RayActorError)):
+            # This can happen during shutdown where the controller is
+            # intentionally killed, the client should just gracefully
+            # exit.
+            logger.debug("LongPollClient failed to connect to host. Shutting down.")
+            self.is_running = False
+            return
+
+        if isinstance(updates, ConnectionError):
+            logger.warning("LongPollClient connection failed, shutting down.")
+            self.is_running = False
+            return
+
+        if isinstance(updates, (ray.exceptions.RayTaskError)):
+            # Some error happened in the controller. It could be a bug or
+            # some undesired state.
+            logger.error("LongPollHost errored\n" + updates.traceback_str)
+            # We must call this in event loop so it works in Ray Client.
+            # See https://github.com/ray-project/ray/issues/20971
+            self._schedule_to_event_loop(self._poll_next)
+            return
+
+        if updates == LongPollState.TIME_OUT:
+            logger.debug("LongPollClient polling timed out. Retrying.")
+            self._schedule_to_event_loop(self._poll_next)
+            return
+
+        logger.debug(
+            f"LongPollClient {self} received updates for keys: {list(updates.keys())}.",
+            extra={"log_to_stderr": False},
+        )
+        if not updates:  # no updates, no callbacks to run, just poll again
+            self._schedule_to_event_loop(self._poll_next)
+
+        # Record latency metrics for received updates.
+        # Skip observations that exceed twice the maximum long poll timeout —
+        # these are catch-up updates received by a client that was offline or
+        # missed several polling cycles, and including them would distort the
+        # metric (e.g. a 10-minute spike just because a new replica connected).
+        max_valid_latency_ms = LISTEN_FOR_CHANGE_REQUEST_TIMEOUT_S[1] * 2 * 1000
+        receive_time = time.time()
+        for key, update in updates.items():
+            latency_ms = (receive_time - update.notify_timestamp) * 1000
+            if latency_ms <= max_valid_latency_ms:
+                self.long_poll_latency_histogram.observe(
+                    latency_ms,
+                    tags={"namespace": _get_metric_namespace_tag(key)},
+                )
+            else:
+                logger.debug(
+                    f"Skipping long poll latency observation of {latency_ms:.0f}ms "
+                    f"for key {key} (exceeds threshold {max_valid_latency_ms:.0f}ms)."
+                )
+
+            self.snapshot_ids[key] = update.snapshot_id
+            callback = self.key_listeners[key]
+
+            # Bind the parameters because closures are late-binding.
+            # https://docs.python-guide.org/writing/gotchas/#late-binding-closures # noqa: E501
+            def chained(callback=callback, arg=update.object_snapshot):
+                callback(arg)
+                self._on_callback_completed(trigger_at=len(updates))
+
+            self._schedule_to_event_loop(chained)
+
+
+class LongPollHost:
+    """The server side object that manages long pulling requests.
+
+    The desired use case is to embed this in an Ray actor. Client will be
+    expected to call actor.listen_for_change.remote(...). On the host side,
+    you can call host.notify_changed({key: object}) to update the state and
+    potentially notify whoever is polling for these values.
+
+    Internally, we use snapshot_ids for each object to identify client with
+    outdated object and immediately return the result. If the client has the
+    up-to-date version, then the listen_for_change call will only return when
+    the object is updated.
+    """
+
+    def __init__(
+        self,
+        listen_for_change_request_timeout_s: Tuple[
+            int, int
+        ] = LISTEN_FOR_CHANGE_REQUEST_TIMEOUT_S,
+    ):
+        # Map object_key -> int
+        self.snapshot_ids: Dict[KeyType, int] = {}
+        # Map object_key -> object
+        self.object_snapshots: Dict[KeyType, Any] = {}
+        # Map object_key -> set(asyncio.Event waiting for updates)
+        self.notifier_events: DefaultDict[KeyType, Set[asyncio.Event]] = defaultdict(
+            set
+        )
+        # Map object_key -> timestamp when notify_changed was called
+        # Used to track latency for propagating updates to clients
+        self._notify_timestamps: Dict[KeyType, float] = {}
+        # Aggregate count of pending clients per namespace tag. Needed because
+        # multiple keys can map to the same low-cardinality namespace tag, so
+        # we must track the total rather than setting per-key counts.
+        self._pending_clients_by_namespace: DefaultDict[str, int] = defaultdict(int)
+
+        self._listen_for_change_request_timeout_s = listen_for_change_request_timeout_s
+        self.transmission_counter = metrics.Counter(
+            "serve_long_poll_host_transmission_counter",
+            description="The number of times the long poll host transmits data.",
+            tag_keys=("namespace_or_state",),
+        )
+        self.pending_clients_gauge = metrics.Gauge(
+            "serve_long_poll_pending_clients",
+            description=("The number of clients waiting for updates per namespace."),
+            tag_keys=("namespace",),
+        )
+
+    def _get_num_notifier_events(self, key: Optional[KeyType] = None):
+        """Used for testing."""
+        if key is not None:
+            return len(self.notifier_events[key])
+        else:
+            return sum(len(events) for events in self.notifier_events.values())
+
+    def _get_pending_clients_by_namespace(self, namespace_tag: str) -> int:
+        """Used for testing. Returns the aggregate pending client count."""
+        return self._pending_clients_by_namespace.get(namespace_tag, 0)
+
+    def notify_client_disabled(self, client_id: str, reason: str) -> None:
+        """Fire-and-forget hook for clients that are shutting themselves down.
+
+        LongPollClient calls this before flipping ``is_running`` to False when
+        it cannot keep delivering updates (e.g. its bound asyncio loop is no
+        longer running).
+        """
+        logger.error(
+            f"LongPollClient {client_id!r} disabled itself and will no longer "
+            f"receive controller updates. Reason: {reason}"
+        )
+
+    def _count_send(
+        self, timeout_or_data: Union[LongPollState, Dict[KeyType, UpdatedObject]]
+    ):
+        """Helper method that tracks the data sent by listen_for_change.
+
+        Records number of times long poll host sends data in the
+        ray_serve_long_poll_host_send_counter metric.
+        """
+
+        if isinstance(timeout_or_data, LongPollState):
+            # The only LongPollState is TIME_OUT– the long poll
+            # connection has timed out.
+            self.transmission_counter.inc(
+                value=1, tags={"namespace_or_state": "TIMEOUT"}
+            )
+        else:
+            data = timeout_or_data
+            for key in data.keys():
+                self.transmission_counter.inc(
+                    value=1,
+                    tags={"namespace_or_state": _get_metric_namespace_tag(key)},
+                )
+
+    async def listen_for_change(
+        self,
+        keys_to_snapshot_ids: Dict[KeyType, int],
+    ) -> Union[LongPollState, Dict[KeyType, UpdatedObject]]:
+        """Listen for changed objects.
+
+        This method will return a dictionary of updated objects. It returns
+        immediately if any of the snapshot_ids are outdated,
+        otherwise it will block until there's an update.
+        """
+        # If there are no keys to listen for,
+        # just wait for a short time to provide backpressure,
+        # then return an empty update.
+        if not keys_to_snapshot_ids:
+            await sleep(1)
+
+            updated_objects = {}
+            self._count_send(updated_objects)
+            return updated_objects
+
+        # If there are any keys with outdated snapshot ids,
+        # return their updated values immediately.
+        updated_objects = {}
+        for key, client_snapshot_id in keys_to_snapshot_ids.items():
+            try:
+                existing_id = self.snapshot_ids[key]
+            except KeyError:
+                # The caller may ask for keys that we don't know about (yet),
+                # just ignore them.
+                # This can happen when, for example,
+                # a deployment handle is manually created for an app
+                # that hasn't been deployed yet (by bypassing the safety checks).
+                continue
+
+            if existing_id != client_snapshot_id:
+                updated_objects[key] = UpdatedObject(
+                    self.object_snapshots[key],
+                    existing_id,
+                    self._notify_timestamps[key],
+                )
+        if len(updated_objects) > 0:
+            self._count_send(updated_objects)
+            return updated_objects
+
+        # Otherwise, register asyncio events to be waited.
+        async_task_to_events = {}
+        async_task_to_watched_keys = {}
+        for key in keys_to_snapshot_ids.keys():
+            # Create a new asyncio event for this key.
+            event = asyncio.Event()
+
+            # Make sure future caller of notify_changed will unblock this
+            # asyncio Event.
+            self.notifier_events[key].add(event)
+
+            # Update aggregate pending clients gauge for this namespace
+            namespace_tag = _get_metric_namespace_tag(key)
+            self._pending_clients_by_namespace[namespace_tag] += 1
+            self.pending_clients_gauge.set(
+                self._pending_clients_by_namespace[namespace_tag],
+                tags={"namespace": namespace_tag},
+            )
+
+            task = get_or_create_event_loop().create_task(event.wait())
+            async_task_to_events[task] = event
+            async_task_to_watched_keys[task] = key
+
+        done, not_done = await asyncio.wait(
+            async_task_to_watched_keys.keys(),
+            return_when=asyncio.FIRST_COMPLETED,
+            timeout=random.uniform(*self._listen_for_change_request_timeout_s),
+        )
+
+        # Collect per-namespace decrements, flush the gauge once per
+        # unique tag after the loop — a single timed-out poll over many
+        # keys can otherwise do N redundant metric writes for the same
+        # namespace.
+        affected_namespaces = set()
+        for task in not_done:
+            task.cancel()
+            event = async_task_to_events[task]
+            key = async_task_to_watched_keys[task]
+            # .get() avoids resurrecting a defaultdict entry for a key
+            # evicted while we were parked.
+            events_set = self.notifier_events.get(key)
+            if events_set is None:
+                continue
+            try:
+                events_set.remove(event)
+            except KeyError:
+                # FIRST_COMPLETED: a sibling wake may have popped this
+                # event via notify_changed.
+                continue
+            if not events_set:
+                self.notifier_events.pop(key, None)
+            namespace_tag = _get_metric_namespace_tag(key)
+            self._pending_clients_by_namespace[namespace_tag] -= 1
+            affected_namespaces.add(namespace_tag)
+        for namespace_tag in affected_namespaces:
+            self.pending_clients_gauge.set(
+                self._pending_clients_by_namespace[namespace_tag],
+                tags={"namespace": namespace_tag},
+            )
+
+        if len(done) == 0:
+            self._count_send(LongPollState.TIME_OUT)
+            return LongPollState.TIME_OUT
+        else:
+            updated_objects = {}
+            for task in done:
+                updated_object_key = async_task_to_watched_keys[task]
+                # Evicted via remove_keys while parked; skip.
+                if updated_object_key not in self.snapshot_ids:
+                    continue
+                updated_objects[updated_object_key] = UpdatedObject(
+                    self.object_snapshots[updated_object_key],
+                    self.snapshot_ids[updated_object_key],
+                    self._notify_timestamps[updated_object_key],
+                )
+            self._count_send(updated_objects)
+            return updated_objects
+
+    async def listen_for_change_java(
+        self,
+        keys_to_snapshot_ids_bytes: bytes,
+    ) -> bytes:
+        """Listen for changed objects. only call by java proxy/router now.
+
+        Args:
+            keys_to_snapshot_ids_bytes: the protobuf bytes of
+              keys_to_snapshot_ids (Dict[str, int]).
+
+        Returns:
+            The serialized protobuf bytes of the update payload.
+        """
+        request_proto = LongPollRequest.FromString(keys_to_snapshot_ids_bytes)
+        keys_to_snapshot_ids = {
+            self._parse_xlang_key(xlang_key): snapshot_id
+            for xlang_key, snapshot_id in request_proto.keys_to_snapshot_ids.items()
+        }
+        result = await self.listen_for_change(keys_to_snapshot_ids)
+
+        # Java long-poll protocol currently doesn't encode LongPollState.
+        # Convert timeout to an empty update payload so the Java client can retry.
+        if result is LongPollState.TIME_OUT:
+            result = {}
+
+        return self._listen_result_to_proto_bytes(result)
+
+    def _parse_poll_namespace(self, name: str):
+        if name == LongPollNamespace.ROUTE_TABLE.name:
+            return LongPollNamespace.ROUTE_TABLE
+        elif name == LongPollNamespace.DEPLOYMENT_TARGETS.name:
+            return LongPollNamespace.DEPLOYMENT_TARGETS
+        else:
+            return name
+
+    def _parse_xlang_key(self, xlang_key: str) -> KeyType:
+        if xlang_key is None:
+            raise ValueError("func _parse_xlang_key: xlang_key is None")
+        if xlang_key.startswith("(") and xlang_key.endswith(")"):
+            fields = xlang_key[1:-1].split(",")
+            if len(fields) == 2:
+                enum_field = self._parse_poll_namespace(fields[0].strip())
+                if isinstance(enum_field, LongPollNamespace):
+                    return enum_field, fields[1].strip()
+        else:
+            return self._parse_poll_namespace(xlang_key)
+        raise ValueError("can not parse key type from xlang_key {}".format(xlang_key))
+
+    def _build_xlang_key(self, key: KeyType) -> str:
+        if isinstance(key, tuple):
+            return "(" + key[0].name + "," + key[1] + ")"
+        elif isinstance(key, LongPollNamespace):
+            return key.name
+        else:
+            return key
+
+    def _object_snapshot_to_proto_bytes(
+        self, key: KeyType, object_snapshot: Any
+    ) -> bytes:
+        if key == LongPollNamespace.ROUTE_TABLE:
+            # object_snapshot is Dict[DeploymentID, EndpointInfo]
+            # NOTE(zcin): the endpoint dictionary broadcasted to Java
+            # HTTP proxies should use string as key because Java does
+            # not yet support 2.x or applications
+            xlang_endpoints = {
+                str(endpoint_tag): EndpointInfoProto(route=endpoint_info.route)
+                for endpoint_tag, endpoint_info in object_snapshot.items()
+            }
+            return EndpointSet(endpoints=xlang_endpoints).SerializeToString()
+        elif isinstance(key, tuple) and key[0] == LongPollNamespace.DEPLOYMENT_TARGETS:
+            # object_snapshot.running_replicas is List[RunningReplicaInfo]
+            actor_name_list = [
+                replica_info.replica_id.to_full_id_str()
+                for replica_info in object_snapshot.running_replicas
+            ]
+            return DeploymentTargetInfo(
+                replica_names=actor_name_list,
+                is_available=object_snapshot.is_available,
+            ).SerializeToString()
+        else:
+            return str.encode(str(object_snapshot))
+
+    def _listen_result_to_proto_bytes(
+        self, keys_to_updated_objects: Dict[KeyType, UpdatedObject]
+    ) -> bytes:
+        xlang_keys_to_updated_objects = {
+            self._build_xlang_key(key): UpdatedObjectProto(
+                snapshot_id=updated_object.snapshot_id,
+                object_snapshot=self._object_snapshot_to_proto_bytes(
+                    key, updated_object.object_snapshot
+                ),
+            )
+            for key, updated_object in keys_to_updated_objects.items()
+        }
+        data = {
+            "updated_objects": xlang_keys_to_updated_objects,
+        }
+        proto = LongPollResult(**data)
+        return proto.SerializeToString()
+
+    def notify_changed(self, updates: Mapping[KeyType, Any]) -> None:
+        """
+        Update the current snapshot of some objects
+        and notify any long poll clients.
+        """
+        notify_time = time.time()
+        for object_key, updated_object in updates.items():
+            try:
+                self.snapshot_ids[object_key] += 1
+            except KeyError:
+                # Initial snapshot id must be >= 0, so that the long poll client
+                # can send a negative initial snapshot id to get a fast update.
+                # They should also be randomized; see
+                # https://github.com/ray-project/ray/pull/45881#discussion_r1645243485
+                self.snapshot_ids[object_key] = random.randint(0, 1_000_000)
+            self.object_snapshots[object_key] = updated_object
+            # Record timestamp for latency tracking
+            self._notify_timestamps[object_key] = notify_time
+            logger.debug(f"LongPollHost: Notify change for key {object_key}.")
+
+            events_to_notify = self.notifier_events.pop(object_key, set())
+            if events_to_notify:
+                # Decrement aggregate count by the number of events popped
+                namespace_tag = _get_metric_namespace_tag(object_key)
+                self._pending_clients_by_namespace[namespace_tag] -= len(
+                    events_to_notify
+                )
+                self.pending_clients_gauge.set(
+                    self._pending_clients_by_namespace[namespace_tag],
+                    tags={"namespace": namespace_tag},
+                )
+            for event in events_to_notify:
+                event.set()
+
+    def remove_keys(self, keys: Iterable[KeyType]) -> None:
+        """Evict per-key state and wake any parked listeners.
+
+        Do NOT evict a key in the same sync call as ``notify_changed``
+        on it — the waiter only runs after the call returns, by which
+        point ``listen_for_change``'s done-branch guard would drop the
+        payload.
+        """
+        affected_namespaces = set()
+        for key in keys:
+            self.snapshot_ids.pop(key, None)
+            self.object_snapshots.pop(key, None)
+            self._notify_timestamps.pop(key, None)
+            events_to_notify = self.notifier_events.pop(key, set())
+            if events_to_notify:
+                # Decrement before waking: the listen_for_change timeout
+                # cleanup would otherwise skip the decrement on the now-
+                # missing notifier_events entry.
+                namespace_tag = _get_metric_namespace_tag(key)
+                self._pending_clients_by_namespace[namespace_tag] -= len(
+                    events_to_notify
+                )
+                affected_namespaces.add(namespace_tag)
+            for event in events_to_notify:
+                event.set()
+        for namespace_tag in affected_namespaces:
+            self.pending_clients_gauge.set(
+                self._pending_clients_by_namespace[namespace_tag],
+                tags={"namespace": namespace_tag},
+            )

python/ray/serve/_private/metrics_utils.py

@@ -0,0 +1,416 @@
+import asyncio
+import bisect
+import logging
+import statistics
+from collections import defaultdict
+from dataclasses import dataclass
+from itertools import chain
+from typing import (
+    Awaitable,
+    Callable,
+    DefaultDict,
+    Dict,
+    Hashable,
+    Iterable,
+    List,
+    Optional,
+    Tuple,
+    Union,
+)
+
+from ray._raylet import (
+    merge_instantaneous_total_cython,
+    time_weighted_average_cython,
+)
+from ray.serve._private.common import TimeSeries, TimeStampedValue
+from ray.serve._private.constants import (
+    METRICS_PUSHER_GRACEFUL_SHUTDOWN_TIMEOUT_S,
+    SERVE_LOGGER_NAME,
+)
+from ray.serve.config import AggregationFunction
+
+QUEUED_REQUESTS_KEY = "queued"
+
+logger = logging.getLogger(SERVE_LOGGER_NAME)
+
+
+@dataclass
+class _MetricsTask:
+    task_func: Union[Callable, Callable[[], Awaitable]]
+    interval_s: float
+
+
+class MetricsPusher:
+    """Periodically runs registered asyncio tasks."""
+
+    def __init__(
+        self,
+        *,
+        async_sleep: Optional[Callable[[int], None]] = None,
+    ):
+        self._async_sleep = async_sleep or asyncio.sleep
+        self._tasks: Dict[str, _MetricsTask] = dict()
+        self._async_tasks: Dict[str, asyncio.Task] = dict()
+
+        # The event needs to be lazily initialized because this class may be constructed
+        # on the main thread but its methods called on a separate asyncio loop.
+        self._stop_event: Optional[asyncio.Event] = None
+
+    @property
+    def stop_event(self) -> asyncio.Event:
+        if self._stop_event is None:
+            self._stop_event = asyncio.Event()
+
+        return self._stop_event
+
+    def start(self):
+        self.stop_event.clear()
+
+    async def metrics_task(self, name: str):
+        """Periodically runs `task_func` every `interval_s` until `stop_event` is set.
+
+        If `task_func` raises an error, an exception will be logged.
+        Supports both sync and async task functions.
+        """
+
+        wait_for_stop_event = asyncio.create_task(self.stop_event.wait())
+        while True:
+            if wait_for_stop_event.done():
+                return
+
+            try:
+                task_func = self._tasks[name].task_func
+                # Check if the function is a coroutine function
+                if asyncio.iscoroutinefunction(task_func):
+                    await task_func()
+                else:
+                    task_func()
+            except Exception as e:
+                logger.exception(f"Failed to run metrics task '{name}': {e}")
+
+            sleep_task = asyncio.create_task(
+                self._async_sleep(self._tasks[name].interval_s)
+            )
+            await asyncio.wait(
+                [sleep_task, wait_for_stop_event],
+                return_when=asyncio.FIRST_COMPLETED,
+            )
+
+            if not sleep_task.done():
+                sleep_task.cancel()
+
+    def register_or_update_task(
+        self,
+        name: str,
+        task_func: Union[Callable, Callable[[], Awaitable]],
+        interval_s: int,
+    ) -> None:
+        """Register a sync or async task under the provided name, or update it.
+
+        This method is idempotent - if a task is already registered with
+        the specified name, it will update it with the most recent info.
+
+        Args:
+            name: Unique name for the task.
+            task_func: Either a sync function or async function (coroutine function).
+            interval_s: Interval in seconds between task executions.
+        """
+
+        self._tasks[name] = _MetricsTask(task_func, interval_s)
+        if name not in self._async_tasks or self._async_tasks[name].done():
+            self._async_tasks[name] = asyncio.create_task(self.metrics_task(name))
+
+    def stop_tasks(self):
+        self.stop_event.set()
+        self._tasks.clear()
+        self._async_tasks.clear()
+
+    async def graceful_shutdown(self):
+        """Shutdown metrics pusher gracefully.
+
+        This method will ensure idempotency of shutdown call.
+        """
+
+        self.stop_event.set()
+        if self._async_tasks:
+            await asyncio.wait(
+                list(self._async_tasks.values()),
+                timeout=METRICS_PUSHER_GRACEFUL_SHUTDOWN_TIMEOUT_S,
+            )
+
+        self._tasks.clear()
+        self._async_tasks.clear()
+
+
+class InMemoryMetricsStore:
+    """A very simple, in memory time series database"""
+
+    def __init__(self):
+        self.data: DefaultDict[Hashable, TimeSeries] = defaultdict(list)
+
+    def add_metrics_point(self, data_points: Dict[Hashable, float], timestamp: float):
+        """Push new data points to the store.
+
+        Args:
+            data_points: dictionary containing the metrics values. The
+              key should uniquely identify this time series
+              and to be used to perform aggregation.
+            timestamp: the unix epoch timestamp the metrics are
+              collected at.
+        """
+
+        for name, value in data_points.items():
+            # Using in-sort to insert while maintaining sorted ordering.
+            bisect.insort(a=self.data[name], x=TimeStampedValue(timestamp, value))
+
+    def prune_keys_and_compact_data(self, start_timestamp_s: float):
+        """Prune keys and compact data that are outdated.
+
+        For keys that haven't had new data recorded after the timestamp,
+        remove them from the database.
+        For keys that have, compact the datapoints that were recorded
+        before the timestamp.
+        """
+        for key, datapoints in list(self.data.items()):
+            if len(datapoints) == 0 or datapoints[-1].timestamp < start_timestamp_s:
+                del self.data[key]
+            else:
+                self.data[key] = self._get_datapoints(key, start_timestamp_s)
+
+    def _get_datapoints(
+        self, key: Hashable, window_start_timestamp_s: float
+    ) -> TimeSeries:
+        """Get all data points given key after window_start_timestamp_s"""
+
+        datapoints = self.data[key]
+
+        idx = bisect.bisect(
+            a=datapoints,
+            x=TimeStampedValue(
+                timestamp=window_start_timestamp_s, value=0  # dummy value
+            ),
+        )
+        return datapoints[idx:]
+
+    def _aggregate_reduce(
+        self,
+        keys: Iterable[Hashable],
+        aggregate_fn: Callable[[Iterable[float]], float],
+    ) -> Tuple[Optional[float], int]:
+        """Reduce the entire set of timeseries values across the specified keys.
+
+        Args:
+            keys: Iterable of keys to aggregate across.
+            aggregate_fn: Function to apply across all float values, e.g., sum, max.
+
+        Returns:
+            A tuple of (float, int) where the first element is the aggregated value
+            and the second element is the number of valid keys used.
+            Returns (None, 0) if no valid keys have data.
+
+        Example:
+        Suppose the store contains:
+        >>> store = InMemoryMetricsStore()
+        >>> store.data.update({
+        ...     "a": [TimeStampedValue(0, 1.0), TimeStampedValue(1, 2.0)],
+        ...     "b": [],
+        ...     "c": [TimeStampedValue(0, 10.0)],
+        ... })
+
+        Using sum across keys:
+
+        >>> store._aggregate_reduce(keys=["a", "b", "c"], aggregate_fn=sum)
+        (13.0, 2)
+
+        Here:
+        - The aggregated value is 1.0 + 2.0 + 10.0 = 13.0
+        - Only keys "a" and "c" contribute values, so report_count = 2
+        """
+        valid_key_count = 0
+
+        def _values_generator():
+            """Generator that yields values from valid keys without storing them all in memory."""
+            nonlocal valid_key_count
+            for key in keys:
+                series = self.data.get(key, [])
+                if not series:
+                    continue
+
+                valid_key_count += 1
+                for timestamp_value in series:
+                    yield timestamp_value.value
+
+        # Create the generator and check if it has any values
+        values_gen = _values_generator()
+        try:
+            first_value = next(values_gen)
+        except StopIteration:
+            # No valid data found
+            return None, 0
+
+        # Apply aggregation to the generator (memory efficient)
+        aggregated_result = aggregate_fn(chain([first_value], values_gen))
+        return aggregated_result, valid_key_count
+
+    def get_latest(
+        self,
+        key: Hashable,
+    ) -> Optional[float]:
+        """Get the latest value for a given key."""
+        if not self.data.get(key, None):
+            return None
+        return self.data[key][-1].value
+
+    def aggregate_sum(
+        self,
+        keys: Iterable[Hashable],
+    ) -> Tuple[Optional[float], int]:
+        """Sum the entire set of timeseries values across the specified keys.
+        Args:
+            keys: Iterable of keys to aggregate across.
+        Returns:
+            A tuple of (float, int) where the first element is the sum across
+            all values found at `keys`, and the second is the number of valid
+            keys used to compute the sum.
+            Returns (None, 0) if no valid keys have data.
+        """
+        return self._aggregate_reduce(keys, sum)
+
+    def aggregate_avg(
+        self,
+        keys: Iterable[Hashable],
+    ) -> Tuple[Optional[float], int]:
+        """Average the entire set of timeseries values across the specified keys.
+
+        Args:
+            keys: Iterable of keys to aggregate across.
+        Returns:
+            A tuple of (float, int) where the first element is the mean across
+            all values found at `keys`, and the second is the number of valid
+            keys used to compute the mean.
+            Returns (None, 0) if no valid keys have data.
+        """
+        return self._aggregate_reduce(keys, statistics.mean)
+
+    def timeseries_count(
+        self,
+        key: Hashable,
+    ) -> int:
+        """Count the number of values across all timeseries values at the specified keys."""
+        series = self.data.get(key, [])
+        if not series:
+            return 0
+        return len(series)
+
+
+def time_weighted_average(
+    step_series: TimeSeries,
+    window_start: Optional[float] = None,
+    window_end: Optional[float] = None,
+    last_window_s: float = 1.0,
+) -> Optional[float]:
+    """
+    Compute time-weighted average of a step function over a time interval.
+
+    This function uses a Cython-optimized implementation for improved performance.
+
+    Args:
+        step_series: Step function as list of (timestamp, value) points, sorted by time.
+            Values are right-continuous (constant until next change).
+        window_start: Start of averaging window (inclusive). If None, uses the start of the series.
+        window_end: End of averaging window (exclusive). If None, uses the end of the series.
+        last_window_s: when window_end is None, uses the last_window_s to compute the end of the window.
+    Returns:
+        Time-weighted average over the interval, or None if no data overlaps.
+    """
+    # Convert None to negative infinity for Cython (C doesn't have None)
+    # Using -inf instead of a specific value like -1.0 ensures any valid float
+    # (including -1.0) can be used as a window boundary.
+    ws = window_start if window_start is not None else float("-inf")
+    we = window_end if window_end is not None else float("-inf")
+    return time_weighted_average_cython(step_series, ws, we, last_window_s)
+
+
+def aggregate_timeseries(
+    timeseries: TimeSeries,
+    aggregation_function: AggregationFunction,
+    last_window_s: float = 1.0,
+    window_start: Optional[float] = None,
+) -> Optional[float]:
+    """Aggregate the values in a timeseries using a specified function."""
+    if aggregation_function == AggregationFunction.MEAN:
+        return time_weighted_average(
+            timeseries, window_start=window_start, last_window_s=last_window_s
+        )
+    elif aggregation_function == AggregationFunction.MAX:
+        values = (
+            ts.value
+            for ts in timeseries
+            if window_start is None or ts.timestamp >= window_start
+        )
+        return max(values, default=None)
+    elif aggregation_function == AggregationFunction.MIN:
+        values = (
+            ts.value
+            for ts in timeseries
+            if window_start is None or ts.timestamp >= window_start
+        )
+        return min(values, default=None)
+    else:
+        raise ValueError(f"Invalid aggregation function: {aggregation_function}")
+
+
+def merge_instantaneous_total(
+    replicas_timeseries: List[TimeSeries],
+) -> TimeSeries:
+    """
+    Merge multiple gauge time series (right-continuous, LOCF) into an
+    instantaneous total time series as a step function.
+
+    This function uses a Cython-optimized implementation for 5-10x performance
+    improvement over pure Python.
+
+    This approach treats each replica's gauge as right-continuous, last-observation-
+    carried-forward (LOCF), which matches gauge semantics. It produces an exact
+    instantaneous total across replicas without bias from arbitrary windowing.
+
+    Uses a k-way merge algorithm for O(n log k) complexity where k is the number
+    of timeseries and n is the total number of events.
+
+    Timestamps are rounded to 10ms precision (2 decimal places) and datapoints
+    with the same rounded timestamp are combined, keeping the most recent value.
+
+    Args:
+        replicas_timeseries: List of time series, one per replica. Each time series
+            is a list of TimeStampedValue objects sorted by timestamp.
+
+    Returns:
+        A list of TimeStampedValue representing the instantaneous total at event times.
+        Between events, the total remains constant (step function). Timestamps are
+        rounded to 10ms precision and duplicate timestamps are combined.
+    """
+    # Handle trivial cases in Python to avoid type conversion overhead
+    active_series = [series for series in replicas_timeseries if series]
+    if not active_series:
+        return []
+    if len(active_series) == 1:
+        return active_series[0]
+
+    # Cython returns list of (timestamp, value) tuples; convert to TimeStampedValue
+    merged_tuples = merge_instantaneous_total_cython(active_series)
+    return [TimeStampedValue(ts, val) for ts, val in merged_tuples]
+
+
+def merge_timeseries_dicts(
+    *timeseries_dicts: DefaultDict[Hashable, TimeSeries],
+) -> DefaultDict[Hashable, TimeSeries]:
+    """
+    Merge multiple time-series dictionaries using instantaneous merge approach.
+    """
+    merged: DefaultDict[Hashable, TimeSeries] = defaultdict(list)
+
+    for ts_dict in timeseries_dicts:
+        for key, ts in ts_dict.items():
+            merged[key].append(ts)
+
+    return {key: merge_instantaneous_total(ts_list) for key, ts_list in merged.items()}

python/ray/serve/_private/node_port_manager.py

@@ -0,0 +1,348 @@
+import heapq
+import logging
+import time
+from typing import Dict, List, Optional, Set, Tuple
+
+from ray.serve._private.common import RequestProtocol
+from ray.serve._private.constants import (
+    RAY_SERVE_DIRECT_INGRESS_MAX_GRPC_PORT,
+    RAY_SERVE_DIRECT_INGRESS_MAX_HTTP_PORT,
+    RAY_SERVE_DIRECT_INGRESS_MIN_GRPC_PORT,
+    RAY_SERVE_DIRECT_INGRESS_MIN_HTTP_PORT,
+    RAY_SERVE_PORT_QUARANTINE_S,
+    SERVE_LOGGER_NAME,
+)
+
+logger = logging.getLogger(SERVE_LOGGER_NAME)
+
+
+class NoAvailablePortError(Exception):
+    def __init__(self, protocol: str, node_id: str):
+        message = f"No available ports on node {node_id} for {protocol} protocol."
+        super().__init__(message)
+
+
+class PortAllocator:
+    """Manages a pool of ports for a specific protocol (e.g., HTTP or gRPC)."""
+
+    def __init__(self, min_port: int, max_port: int, protocol: str, node_id: str):
+        self._protocol = protocol
+        self._node_id = node_id
+        # TODO(abrar): add a validation here to ensure min_port and max_port dont overlap with
+        # ray params min_worker_port and max_worker_port.
+        self._available_ports = list(range(min_port, max_port))
+        heapq.heapify(self._available_ports)
+
+        self._allocated_ports: Dict[str, int] = {}
+        self._blocked_ports: Set[int] = set()
+        # port -> monotonic deadline at which it leaves quarantine.
+        self._quarantined_ports: Dict[int, float] = {}
+        # Min-heap of (deadline, port) mirroring _quarantined_ports so a no-op drain
+        # is O(1) (peek) instead of O(quarantined). Lazy-deleted (P2).
+        self._quarantine_heap: List[Tuple[float, int]] = []
+
+    def _drain_expired_quarantine(self) -> None:
+        """Return quarantined ports past their expiry to the available pool.
+
+        Drains via the min-heap keyed on expiry, so when nothing is due this is a
+        single O(1) peek instead of an O(quarantined) dict scan. Lazy deletion: a
+        popped heap entry is honored only if it still matches the live deadline in
+        _quarantined_ports (entries left stale by update_port_if_missing's reclaim
+        pop, or by a re-quarantine, are skipped).
+        """
+        if not self._quarantine_heap:
+            return
+        now = time.monotonic()
+        while self._quarantine_heap and self._quarantine_heap[0][0] <= now:
+            deadline, port = heapq.heappop(self._quarantine_heap)
+            if self._quarantined_ports.get(port) == deadline:
+                heapq.heappush(self._available_ports, port)
+                del self._quarantined_ports[port]
+                logger.info(
+                    f"Released {self._protocol} port {port} from quarantine on "
+                    f"node {self._node_id}; returning it to the available pool."
+                )
+
+    def has_pending_quarantine(self) -> bool:
+        """True if any port is still quarantined (drains expired entries first)."""
+        self._drain_expired_quarantine()
+        return bool(self._quarantined_ports)
+
+    def update_port_if_missing(self, replica_id: str, port: Optional[int]):
+        """Update port value for a replica."""
+        if replica_id in self._allocated_ports:
+            return
+        # Reclaim trumps quarantine: the port is back in active use.
+        if port is not None:
+            self._quarantined_ports.pop(port, None)
+        assert (
+            port is not None
+        ), f"Port is None for {self._protocol} protocol on replica {replica_id} on node {self._node_id}"
+        if self._protocol == RequestProtocol.HTTP:
+            if not (
+                RAY_SERVE_DIRECT_INGRESS_MIN_HTTP_PORT
+                <= port
+                <= RAY_SERVE_DIRECT_INGRESS_MAX_HTTP_PORT
+            ):
+                logger.warning(f"HTTP port out of range: {port}")
+        elif self._protocol == RequestProtocol.GRPC:
+            if not (
+                RAY_SERVE_DIRECT_INGRESS_MIN_GRPC_PORT
+                <= port
+                <= RAY_SERVE_DIRECT_INGRESS_MAX_GRPC_PORT
+            ):
+                logger.warning(f"GRPC port out of range: {port}")
+        self._allocated_ports[replica_id] = port
+
+        logger.info(
+            f"Recovered {self._protocol} port {port} for replica {replica_id} on node {self._node_id}"
+        )
+        return port
+
+    def allocate(self, replica_id: str) -> int:
+        if replica_id in self._allocated_ports:
+            logger.warning(
+                f"{self._protocol} port already allocated for replica {replica_id}"
+            )
+            return self._allocated_ports[replica_id]
+
+        self._drain_expired_quarantine()
+
+        # Recovered replicas live in _allocated_ports but their ports are
+        # still in _available_ports, so guard against re-handing them out.
+        # A recovered port released into quarantine is also still in the heap
+        # (update_port_if_missing never pops it), so guard against handing out
+        # a still-quarantined port too.
+        allocated = set(self._allocated_ports.values())
+        while self._available_ports:
+            port = heapq.heappop(self._available_ports)
+            if (
+                port not in self._blocked_ports
+                and port not in allocated
+                and port not in self._quarantined_ports
+            ):
+                self._allocated_ports[replica_id] = port
+                logger.info(
+                    f"Allocated {self._protocol} port {port} to replica {replica_id} on node {self._node_id}"
+                )
+                return port
+
+        raise NoAvailablePortError(self._protocol, self._node_id)
+
+    def release(self, replica_id: str, port: int, block_port: bool = False):
+        """
+        Releases a port for a replica.
+
+        Args:
+            replica_id: The ID of the replica to release the port for.
+            port: The port to release.
+            block_port: Whether to block the port from being allocated again. Use this in
+                situations where the port is being released due some other process is using it.
+        """
+        if replica_id not in self._allocated_ports:
+            raise ValueError(
+                f"{self._protocol} port not allocated for replica {replica_id} on node {self._node_id}"
+            )
+        expected_port = self._allocated_ports[replica_id]
+        assert expected_port == port, (
+            f"{self._protocol} port mismatch for replica {replica_id} on node {self._node_id}: "
+            f"expected {expected_port}, got {port}"
+        )
+
+        del self._allocated_ports[replica_id]
+
+        if block_port:
+            # Block is a stronger guarantee than quarantine; skip quarantine.
+            self._blocked_ports.add(port)
+            heapq.heappush(self._available_ports, port)
+            logger.info(
+                f"Released and blocked {self._protocol} port {port} for replica "
+                f"{replica_id} on node {self._node_id}"
+            )
+        elif RAY_SERVE_PORT_QUARANTINE_S > 0:
+            deadline = time.monotonic() + RAY_SERVE_PORT_QUARANTINE_S
+            self._quarantined_ports[port] = deadline
+            heapq.heappush(self._quarantine_heap, (deadline, port))
+            logger.info(
+                f"Quarantined {self._protocol} port {port} for "
+                f"{RAY_SERVE_PORT_QUARANTINE_S}s after releasing for replica "
+                f"{replica_id} on node {self._node_id}"
+            )
+        else:
+            heapq.heappush(self._available_ports, port)
+            logger.info(
+                f"Released {self._protocol} port {port} for replica {replica_id} on node {self._node_id}"
+            )
+
+    def prune(self, active_replica_ids: Set[str]):
+        for replica_id in list(self._allocated_ports.keys()):
+            if replica_id not in active_replica_ids:
+                port = self._allocated_ports[replica_id]
+                logger.info(
+                    f"Cleaning up {self._protocol} port {port} for stale replica {replica_id} on node {self._node_id}"
+                )
+                self.release(replica_id, port)
+
+    def get_port(self, replica_id: str) -> int:
+        if replica_id not in self._allocated_ports:
+            raise ValueError(
+                f"{self._protocol} port not allocated for replica {replica_id} on node {self._node_id}"
+            )
+        return self._allocated_ports[replica_id]
+
+    def is_port_allocated(self, replica_id: str) -> bool:
+        return replica_id in self._allocated_ports
+
+
+class NodePortManager:
+    """
+    This class is responsible for managing replica-specific port allocations on a node,
+    and is only used in direct ingress mode, where each Serve replica is exposed individually
+    via a Kubernetes or GCP or AWS Ingress.
+
+    The primary goal of this class is to assign ports in a consistent and efficient manner,
+    minimizing EndpointSlice fragmentation in Kubernetes. It uses a min-heap strategy to
+    allocate ports incrementally, ensuring that all nodes tend to reuse the same port numbers.
+
+    Background:
+    Kubernetes groups endpoints into EndpointSlices based on the set of ports exposed by each Pod.
+    If Pods expose different port combinations (e.g., due to random port assignment), Kubernetes
+    generates separate EndpointSlices per unique port list. This leads to unnecessary fragmentation
+    and increased resource consumption.
+
+    By allocating ports deterministically, we ensure:
+    - Consistent port usage across all nodes
+    - Fewer unique port lists, reducing the number of EndpointSlices created
+    - Improved performance and resource utilization
+
+    Although Kubernetes does not allow users to explicitly configure the ports included in
+    EndpointSlices, maintaining a uniform port layout across nodes is still beneficial.
+
+    Port lifecycle:
+    - Replicas are expected to release their ports when stopped
+    - If a replica crashes without releasing its port, the controller loop will detect and
+    reclaim leaked ports during reconciliation
+
+    Note:
+    Although this strategy is designed with Kubernetes in mind, it is applied uniformly
+    across all platforms for consistency.
+    """
+
+    _node_managers: Dict[str, "NodePortManager"] = {}
+
+    @classmethod
+    def get_node_manager(cls, node_id: str) -> "NodePortManager":
+        # this doesn't need to be behind a lock because it will already be called from same thread
+        if node_id not in cls._node_managers:
+            logger.info(f"Creating node manager for node {node_id}")
+            cls._node_managers[node_id] = cls(node_id)
+        return cls._node_managers[node_id]
+
+    @classmethod
+    def prune(cls, node_id_to_alive_replica_ids: Dict[str, Set[str]]):
+        # this doesn't need to be behind a lock because it will already be called from same thread
+        for node_id in list(cls._node_managers):
+            manager = cls._node_managers[node_id]
+            alive = node_id_to_alive_replica_ids.get(node_id, set())
+            # Skip the reclaim scan for a node whose set of alive replicas is unchanged
+            # since the last prune. Comparing the whole alive set means a replica that
+            # left, arrived, or moved nodes is still caught; skipping only defers reclaim
+            # and never reuses a live port (allocate() guards that). The emptied-manager
+            # teardown below still runs every tick.
+            if alive == manager._last_pruned_alive:
+                pass
+            else:
+                # Release ports of replicas no longer alive (quarantines them).
+                manager._prune_replica_ports(alive)
+                manager._last_pruned_alive = set(alive)
+            # Keep the manager until its quarantine drains; dropping it early
+            # would discard the deadline and let the port be reused immediately.
+            if not alive and not manager.has_pending_quarantine():
+                logger.info(f"Removing node manager for node {node_id}")
+                del cls._node_managers[node_id]
+
+    @classmethod
+    def update_ports(cls, ingress_replicas_info: List[Tuple[str, str, int, int]]):
+        """Update port values for ingress replicas."""
+        for node_id, replica_id, http_port, grpc_port in ingress_replicas_info:
+            if node_id is None:
+                continue
+            node_port_manager = cls.get_node_manager(node_id)
+            if http_port is not None:
+                node_port_manager._http_allocator.update_port_if_missing(
+                    replica_id,
+                    http_port,
+                )
+            if grpc_port is not None:
+                node_port_manager._grpc_allocator.update_port_if_missing(
+                    replica_id,
+                    grpc_port,
+                )
+
+    def __init__(self, node_id: str):
+        self._node_id = node_id
+        # Set of alive replicas this manager last pruned against; when unchanged, the
+        # reclaim scan is skipped. None means never pruned, so the first prune runs.
+        self._last_pruned_alive: Optional[Set[str]] = None
+
+        self._http_allocator = PortAllocator(
+            RAY_SERVE_DIRECT_INGRESS_MIN_HTTP_PORT,
+            RAY_SERVE_DIRECT_INGRESS_MAX_HTTP_PORT,
+            protocol=RequestProtocol.HTTP,
+            node_id=node_id,
+        )
+        self._grpc_allocator = PortAllocator(
+            RAY_SERVE_DIRECT_INGRESS_MIN_GRPC_PORT,
+            RAY_SERVE_DIRECT_INGRESS_MAX_GRPC_PORT,
+            protocol=RequestProtocol.GRPC,
+            node_id=node_id,
+        )
+
+    def _prune_replica_ports(self, active_replica_ids: Set[str]):
+        self._http_allocator.prune(active_replica_ids)
+        self._grpc_allocator.prune(active_replica_ids)
+
+    def allocate_port(self, replica_id: str, protocol: RequestProtocol) -> int:
+        if protocol == RequestProtocol.HTTP:
+            return self._http_allocator.allocate(replica_id)
+        elif protocol == RequestProtocol.GRPC:
+            return self._grpc_allocator.allocate(replica_id)
+        else:
+            raise ValueError(f"Unsupported protocol: {protocol}")
+
+    def release_port(
+        self,
+        replica_id: str,
+        port: int,
+        protocol: RequestProtocol,
+        block_port: bool = False,
+    ):
+        if protocol == RequestProtocol.HTTP:
+            self._http_allocator.release(replica_id, port, block_port)
+        elif protocol == RequestProtocol.GRPC:
+            self._grpc_allocator.release(replica_id, port, block_port)
+        else:
+            raise ValueError(f"Unsupported protocol: {protocol}")
+
+    def get_port(self, replica_id: str, protocol: RequestProtocol) -> int:
+        if protocol == RequestProtocol.HTTP:
+            return self._http_allocator.get_port(replica_id)
+        elif protocol == RequestProtocol.GRPC:
+            return self._grpc_allocator.get_port(replica_id)
+        else:
+            raise ValueError(f"Unsupported protocol: {protocol}")
+
+    def is_port_allocated(self, replica_id: str, protocol: RequestProtocol) -> bool:
+        if protocol == RequestProtocol.HTTP:
+            return self._http_allocator.is_port_allocated(replica_id)
+        elif protocol == RequestProtocol.GRPC:
+            return self._grpc_allocator.is_port_allocated(replica_id)
+        else:
+            raise ValueError(f"Unsupported protocol: {protocol}")
+
+    def has_pending_quarantine(self) -> bool:
+        """True if either allocator still holds a quarantined port."""
+        # Evaluate both (no short-circuit) so each allocator drains expired ports.
+        http_pending = self._http_allocator.has_pending_quarantine()
+        grpc_pending = self._grpc_allocator.has_pending_quarantine()
+        return http_pending or grpc_pending

python/ray/serve/_private/proxy_request_response.py

@@ -0,0 +1,302 @@
+import logging
+import pickle
+from abc import ABC, abstractmethod
+from dataclasses import dataclass
+from enum import Enum
+from typing import Any, AsyncIterator, List, Optional, Tuple, Union
+
+import grpc
+from starlette.types import Receive, Scope, Send
+
+from ray.serve._private.common import StreamingHTTPRequest, gRPCRequest
+from ray.serve._private.constants import SERVE_LOGGER_NAME
+from ray.serve._private.logging_utils import format_grpc_peer_address
+from ray.serve._private.tracing_utils import (
+    extract_propagated_context,
+    is_tracing_enabled,
+    set_trace_context,
+)
+from ray.serve._private.utils import DEFAULT
+from ray.serve.grpc_util import RayServegRPCContext
+
+logger = logging.getLogger(SERVE_LOGGER_NAME)
+
+
+class gRPCStreamingType(str, Enum):
+    """Enum representing the gRPC streaming type."""
+
+    UNARY_UNARY = "unary_unary"  # Single request, single response
+    UNARY_STREAM = (
+        "unary_stream"  # Single request, streaming response (server streaming)
+    )
+    STREAM_UNARY = (
+        "stream_unary"  # Streaming request, single response (client streaming)
+    )
+    STREAM_STREAM = (
+        "stream_stream"  # Streaming request, streaming response (bidirectional)
+    )
+
+
+class ProxyRequest(ABC):
+    """Base ProxyRequest class to use in the common interface among proxies"""
+
+    @property
+    @abstractmethod
+    def request_type(self) -> str:
+        raise NotImplementedError
+
+    @property
+    @abstractmethod
+    def method(self) -> str:
+        raise NotImplementedError
+
+    @property
+    @abstractmethod
+    def route_path(self) -> str:
+        raise NotImplementedError
+
+    @property
+    @abstractmethod
+    def is_route_request(self) -> bool:
+        raise NotImplementedError
+
+    @property
+    @abstractmethod
+    def is_health_request(self) -> bool:
+        raise NotImplementedError
+
+    @property
+    def client(self) -> str:
+        return ""
+
+    @abstractmethod
+    def populate_tracing_context(self):
+        """Implement this method to populate tracing context so the parent and
+        child spans will be connected into a single trace."""
+        raise NotImplementedError
+
+
+class ASGIProxyRequest(ProxyRequest):
+    """ProxyRequest implementation to wrap ASGI scope, receive, and send."""
+
+    def __init__(self, scope: Scope, receive: Receive, send: Send):
+        self.scope = scope
+        self.receive = receive
+        self.send = send
+
+    @property
+    def request_type(self) -> str:
+        return self.scope.get("type", "")
+
+    @property
+    def method(self) -> str:
+        # WebSocket messages don't have a 'method' field.
+        return self.scope.get("method", "WS").upper()
+
+    @property
+    def route_path(self) -> str:
+        return self.scope.get("path", "")[len(self.root_path) :]
+
+    @property
+    def is_route_request(self) -> bool:
+        return self.route_path == "/-/routes"
+
+    @property
+    def is_health_request(self) -> bool:
+        return self.route_path == "/-/healthz"
+
+    @property
+    def client(self) -> str:
+        return self.scope.get("client", "")
+
+    @property
+    def root_path(self) -> str:
+        return self.scope.get("root_path", "")
+
+    @property
+    def path(self) -> str:
+        return self.scope.get("path", "")
+
+    @property
+    def headers(self) -> List[Tuple[bytes, bytes]]:
+        return self.scope.get("headers", [])
+
+    def set_path(self, path: str):
+        self.scope["path"] = path
+
+    def set_root_path(self, root_path: str):
+        self.scope["root_path"] = root_path
+
+    def serialized_replica_arg(self, proxy_actor_name: str) -> bytes:
+        # NOTE(edoakes): it's important that the request is sent as raw bytes to
+        # skip the Ray cloudpickle serialization codepath for performance.
+        return pickle.dumps(
+            StreamingHTTPRequest(
+                asgi_scope=self.scope,
+                proxy_actor_name=proxy_actor_name,
+            )
+        )
+
+    def populate_tracing_context(self):
+        """Populate tracing context for ASGI requests.
+
+        This method extracts the "traceparent" header from the request headers and sets
+        the tracing context from it.
+        """
+        if not is_tracing_enabled():
+            return
+
+        for key, value in self.headers:
+            if key.decode() == "traceparent":
+                trace_context = extract_propagated_context(
+                    {key.decode(): value.decode()}
+                )
+                set_trace_context(trace_context)
+
+
+class gRPCProxyRequest(ProxyRequest):
+    """ProxyRequest implementation to wrap gRPC request protobuf and metadata."""
+
+    def __init__(
+        self,
+        request_proto: Any,
+        context: grpc._cython.cygrpc._ServicerContext,
+        service_method: str,
+        stream: bool,
+        *,
+        streaming_type: gRPCStreamingType = None,
+        request_iterator: Optional[AsyncIterator[Any]] = None,
+    ):
+        self._request_proto = request_proto
+        self._request_iterator = request_iterator
+        self.context = context
+        self.service_method = service_method
+        self.stream = stream
+        # Determine streaming type based on parameters
+        if streaming_type is not None:
+            self.streaming_type = streaming_type
+        elif request_iterator is not None:
+            # Has input stream
+            self.streaming_type = (
+                gRPCStreamingType.STREAM_STREAM
+                if stream
+                else gRPCStreamingType.STREAM_UNARY
+            )
+        else:
+            # No input stream
+            self.streaming_type = (
+                gRPCStreamingType.UNARY_STREAM
+                if stream
+                else gRPCStreamingType.UNARY_UNARY
+            )
+        self.app_name = ""
+        self.request_id = None
+        self.method_name = "__call__"
+        self.multiplexed_model_id = DEFAULT.VALUE
+        self.session_id = DEFAULT.VALUE
+        # ray_serve_grpc_context is a class implemented by us to be able to serialize
+        # the object and pass it into the deployment.
+        self.ray_serve_grpc_context = RayServegRPCContext(context)
+        self.setup_variables()
+
+    def setup_variables(self):
+        if not self.is_route_request and not self.is_health_request:
+            service_method_split = self.service_method.split("/")
+            self.method_name = service_method_split[-1]
+            for key, value in self.context.invocation_metadata():
+                if key == "application":
+                    self.app_name = value
+                elif key == "request_id":
+                    self.request_id = value
+                elif key == "multiplexed_model_id":
+                    self.multiplexed_model_id = value
+                elif key == "session_id":
+                    self.session_id = value
+
+    @property
+    def request_type(self) -> str:
+        return "grpc"
+
+    @property
+    def method(self) -> str:
+        return self.service_method
+
+    @property
+    def route_path(self) -> str:
+        return self.app_name
+
+    @property
+    def is_route_request(self) -> bool:
+        return self.service_method == "/ray.serve.RayServeAPIService/ListApplications"
+
+    @property
+    def is_health_request(self) -> bool:
+        return self.service_method == "/ray.serve.RayServeAPIService/Healthz"
+
+    @property
+    def client(self) -> str:
+        return format_grpc_peer_address(self.context.peer())
+
+    @property
+    def has_input_stream(self) -> bool:
+        """Returns True if this request has a streaming input (client/bidi streaming)."""
+        return self.streaming_type in (
+            gRPCStreamingType.STREAM_UNARY,
+            gRPCStreamingType.STREAM_STREAM,
+        )
+
+    @property
+    def request_iterator(self) -> Optional[AsyncIterator[Any]]:
+        """Returns the request iterator for client/bidi streaming, or None."""
+        return self._request_iterator
+
+    def send_request_id(self, request_id: str):
+        # Setting the trailing metadata on the ray_serve_grpc_context object, so it's
+        # not overriding the ones set from the user and will be sent back to the
+        # client altogether.
+        self.ray_serve_grpc_context.set_trailing_metadata([("request_id", request_id)])
+
+    def serialized_replica_arg(self) -> bytes:
+        # NOTE(edoakes): it's important that the request is sent as raw bytes to
+        # skip the Ray cloudpickle serialization codepath for performance.
+        return pickle.dumps(gRPCRequest(user_request_proto=self._request_proto))
+
+    def populate_tracing_context(self):
+        """Populate tracing context for gRPC requests.
+
+        This method extracts the "traceparent" metadata from the request headers and
+        sets the tracing context from it.
+        """
+        if not is_tracing_enabled():
+            return
+
+        for key, value in self.context.invocation_metadata():
+            if key == "traceparent":
+                trace_context = extract_propagated_context({key: value})
+                set_trace_context(trace_context)
+
+
+@dataclass(frozen=True)
+class ResponseStatus:
+    code: Union[str, grpc.StatusCode]  # Must be convertible to a string.
+    is_error: bool = False
+    message: str = ""
+
+
+# Yields protocol-specific messages followed by a final `ResponseStatus`.
+ResponseGenerator = AsyncIterator[Union[Any, ResponseStatus]]
+
+
+@dataclass(frozen=True)
+class HandlerMetadata:
+    application_name: str = ""
+    deployment_name: str = ""
+    route: str = ""
+
+
+@dataclass(frozen=True)
+class ResponseHandlerInfo:
+    response_generator: ResponseGenerator
+    metadata: HandlerMetadata
+    should_record_access_log: bool
+    should_increment_ongoing_requests: bool

python/ray/serve/_private/proxy_response_generator.py

@@ -0,0 +1,181 @@
+import asyncio
+import logging
+import time
+from abc import ABC, abstractmethod
+from asyncio.tasks import FIRST_COMPLETED
+from typing import Any, Callable, Optional, Union
+
+from ray.serve._private.constants import SERVE_LOGGER_NAME
+from ray.serve._private.utils import calculate_remaining_timeout
+from ray.serve.exceptions import RequestCancelledError
+from ray.serve.handle import DeploymentResponse, DeploymentResponseGenerator
+
+logger = logging.getLogger(SERVE_LOGGER_NAME)
+
+
+class _ProxyResponseGeneratorBase(ABC):
+    def __init__(
+        self,
+        *,
+        timeout_s: Optional[float] = None,
+        disconnected_task: Optional[asyncio.Task] = None,
+        result_callback: Optional[Callable[[Any], Any]] = None,
+    ):
+        """Implements a generator wrapping a deployment response.
+
+        Args:
+            timeout_s: an end-to-end timeout for the request. If this expires and the
+                response is not completed, the request will be cancelled. If `None`,
+                there's no timeout.
+            disconnected_task: a task whose completion signals that the client has
+                disconnected. When this happens, the request will be cancelled. If `None`,
+                disconnects will not be detected.
+            result_callback: will be called on each result before it's returned. If
+                `None`, the unmodified result is returned.
+        """
+        self._timeout_s = timeout_s
+        self._start_time_s = time.time()
+        self._disconnected_task = disconnected_task
+        self._result_callback = result_callback
+
+    def __aiter__(self):
+        return self
+
+    @abstractmethod
+    async def __anext__(self):
+        """Return the next message in the stream.
+
+        Raises:
+            TimeoutError: On timeout.
+            asyncio.CancelledError: On disconnect.
+            StopAsyncIteration: When the stream is completed.
+        """
+        pass
+
+    def stop_checking_for_disconnect(self):
+        """Once this is called, the disconnected_task will be ignored."""
+        self._disconnected_task = None
+
+
+def swallow_cancelled(task: asyncio.Task):
+    try:
+        task.result()
+    except (RequestCancelledError, asyncio.CancelledError):
+        # We expect RequestCancelledError to be raised because for disconnect or
+        # timeouts, we explicitly call resp.cancel(). To avoid "Task exception
+        # was never retrieved" errors from spamming the proxy logs, swallow
+        # them here.
+        pass
+    except Exception:
+        # For all other exceptions, do not catch and instead re-raise here so that
+        # they will be logged properly.
+        raise
+
+
+class ProxyResponseGenerator(_ProxyResponseGeneratorBase):
+    """Wraps a unary DeploymentResponse or streaming DeploymentResponseGenerator.
+
+    In the case of a unary DeploymentResponse, __anext__ will only ever return one
+    result.
+    """
+
+    def __init__(
+        self,
+        response: Union[DeploymentResponse, DeploymentResponseGenerator],
+        *,
+        timeout_s: Optional[float] = None,
+        disconnected_task: Optional[asyncio.Task] = None,
+        result_callback: Optional[Callable[[Any], Any]] = None,
+    ):
+        super().__init__(
+            timeout_s=timeout_s,
+            disconnected_task=disconnected_task,
+            result_callback=result_callback,
+        )
+        self._done = False
+        self._response = response
+
+    def cancelled(self) -> bool:
+        return self._response.cancelled()
+
+    async def __anext__(self):
+        if self._done:
+            raise StopAsyncIteration
+
+        try:
+            if isinstance(self._response, DeploymentResponseGenerator):
+                result = await self._get_next_streaming_result()
+            else:
+                result = await self._get_unary_result()
+                self._done = True
+
+            if self._result_callback is not None:
+                result = self._result_callback(result)
+        except asyncio.CancelledError as e:
+            # This is specifically for gRPC. The cancellation can happen from client
+            # dropped connection before the request is completed. If self._response is
+            # not already cancelled, we want to explicitly cancel the task, so it
+            # doesn't waste cluster resource in this case and can be terminated
+            # gracefully.
+            if not self._response.cancelled():
+                self._response.cancel()
+                self._done = True
+
+            raise e from None
+        except Exception as e:
+            self._done = True
+            raise e from None
+
+        return result
+
+    async def _await_response_anext(self) -> Any:
+        return await self._response.__anext__()
+
+    async def _get_next_streaming_result(self) -> Any:
+        next_result_task = asyncio.create_task(self._await_response_anext())
+        tasks = [next_result_task]
+        if self._disconnected_task is not None:
+            tasks.append(self._disconnected_task)
+
+        done, _ = await asyncio.wait(
+            tasks,
+            return_when=FIRST_COMPLETED,
+            timeout=calculate_remaining_timeout(
+                timeout_s=self._timeout_s,
+                start_time_s=self._start_time_s,
+                curr_time_s=time.time(),
+            ),
+        )
+        if next_result_task in done:
+            return next_result_task.result()
+        elif self._disconnected_task in done:
+            next_result_task.cancel()
+            next_result_task.add_done_callback(swallow_cancelled)
+            self._response.cancel()
+            raise asyncio.CancelledError()
+        else:
+            next_result_task.cancel()
+            next_result_task.add_done_callback(swallow_cancelled)
+            self._response.cancel()
+            raise TimeoutError()
+
+    async def _await_response(self) -> Any:
+        return await self._response
+
+    async def _get_unary_result(self) -> Any:
+        result_task = asyncio.create_task(self._await_response())
+        tasks = [result_task]
+        if self._disconnected_task is not None:
+            tasks.append(self._disconnected_task)
+
+        done, _ = await asyncio.wait(
+            tasks, return_when=FIRST_COMPLETED, timeout=self._timeout_s
+        )
+        if result_task in done:
+            return result_task.result()
+        elif self._disconnected_task in done:
+            self._response.cancel()
+            raise asyncio.CancelledError()
+        else:
+            self._response.cancel()
+            raise TimeoutError()

python/ray/serve/_private/proxy_router.py

@@ -0,0 +1,252 @@
+import logging
+from typing import Any, Callable, Dict, List, Optional, Tuple
+
+from starlette.applications import Starlette
+from starlette.requests import Request
+from starlette.routing import Route
+from starlette.types import Scope
+
+from ray.serve._private.common import ApplicationName, DeploymentID, EndpointInfo
+from ray.serve._private.constants import SERVE_LOGGER_NAME
+from ray.serve._private.thirdparty.get_asgi_route_name import (
+    RoutePattern,
+    get_asgi_route_name,
+)
+from ray.serve.handle import DeploymentHandle
+
+logger = logging.getLogger(SERVE_LOGGER_NAME)
+
+NO_ROUTES_MESSAGE = "Route table is not populated yet."
+NO_REPLICAS_MESSAGE = "No replicas are available yet."
+
+
+class ProxyRouter:
+    """Router interface for the proxy to use."""
+
+    def __init__(
+        self,
+        get_handle: Callable[[str, str], DeploymentHandle],
+    ):
+        # Function to get a handle given a name. Used to mock for testing.
+        self._get_handle = get_handle
+        # Contains a ServeHandle for each endpoint.
+        self.handles: Dict[DeploymentID, DeploymentHandle] = dict()
+        # Flipped to `True` once the route table has been updated at least once.
+        # The proxy router is not ready for traffic until the route table is populated
+        self._route_table_populated = False
+
+        # Info used for HTTP proxy
+        # Routes sorted in order of decreasing length.
+        self.sorted_routes: List[str] = list()
+        # Endpoints associated with the routes.
+        self.route_info: Dict[str, DeploymentID] = dict()
+        # Map of application name to is_cross_language.
+        self.app_to_is_cross_language: Dict[ApplicationName, bool] = dict()
+
+        # Info used for gRPC proxy
+        # Endpoints info associated with endpoints.
+        self.endpoints: Dict[DeploymentID, EndpointInfo] = dict()
+
+        # Map of route prefix to list of route patterns for that endpoint
+        # Used to match incoming requests to ASGI route patterns for metrics
+        # Route patterns are tuples of (methods, path) where methods can be None
+        self.route_patterns: Dict[str, List[RoutePattern]] = dict()
+        # Cache of mock Starlette apps for route pattern matching
+        # Key: route prefix, Value: pre-built Starlette app with routes
+        self._route_pattern_apps: Dict[str, Any] = dict()
+
+    def ready_for_traffic(self, is_head: bool) -> Tuple[bool, str]:
+        """Whether the proxy router is ready to serve traffic.
+
+        The first return value will be false if any of the following hold:
+        - The route table has not been populated yet with a non-empty set of routes
+        - The route table has been populated, but none of the handles
+          have received running replicas yet AND it lives on a worker node.
+
+        Otherwise, the first return value will be true.
+        """
+
+        if not self._route_table_populated:
+            return False, NO_ROUTES_MESSAGE
+
+        # NOTE(zcin): For the proxy on the head node, even if none of its handles have
+        # been populated with running replicas yet, we MUST mark the proxy as ready for
+        # traffic. This is to handle the case when all deployments have scaled to zero.
+        # If the deployments (more precisely, ingress deployments) have all scaled down
+        # to zero, at least one proxy needs to be able to receive incoming requests to
+        # trigger upscale.
+        if is_head:
+            return True, ""
+
+        for handle in self.handles.values():
+            if handle.running_replicas_populated():
+                return True, ""
+
+        return False, NO_REPLICAS_MESSAGE
+
+    def update_routes(self, endpoints: Dict[DeploymentID, EndpointInfo]):
+        logger.info(
+            f"Got updated endpoints: {endpoints}.", extra={"log_to_stderr": True}
+        )
+        if endpoints:
+            self._route_table_populated = True
+
+        self.endpoints = endpoints
+
+        existing_handles = set(self.handles.keys())
+        routes = []
+        route_info = {}
+        app_to_is_cross_language = {}
+        route_patterns = {}
+        for endpoint, info in endpoints.items():
+            routes.append(info.route)
+            route_info[info.route] = endpoint
+            app_to_is_cross_language[endpoint.app_name] = info.app_is_cross_language
+            if info.route_patterns:
+                route_patterns[info.route] = info.route_patterns
+            if endpoint in self.handles:
+                existing_handles.remove(endpoint)
+            else:
+                self.handles[endpoint] = self._get_handle(endpoint, info)
+
+        # Clean up any handles that are no longer used.
+        if len(existing_handles) > 0:
+            logger.info(
+                f"Deleting {len(existing_handles)} unused handles.",
+                extra={"log_to_stderr": False},
+            )
+        for endpoint in existing_handles:
+            del self.handles[endpoint]
+
+        # Routes are sorted in order of decreasing length to enable longest
+        # prefix matching.
+        self.sorted_routes = sorted(routes, key=lambda x: len(x), reverse=True)
+        self.route_info = route_info
+        self.app_to_is_cross_language = app_to_is_cross_language
+        self.route_patterns = route_patterns
+        # Invalidate cached mock apps when route patterns change
+        self._route_pattern_apps.clear()
+
+    def match_route(
+        self, target_route: str
+    ) -> Optional[Tuple[str, DeploymentHandle, bool]]:
+        """Return the longest prefix match among existing routes for the route.
+        Args:
+            target_route: route to match against.
+        Returns:
+            (route, handle, is_cross_language) if found, else None.
+        """
+
+        for route in self.sorted_routes:
+            if target_route.startswith(route):
+                matched = False
+                # If the route we matched on ends in a '/', then so does the
+                # target route and this must be a match.
+                if route.endswith("/"):
+                    matched = True
+                # If the route we matched on doesn't end in a '/', we need to
+                # do another check to ensure that either this is an exact match
+                # or the next character in the target route is a '/'. This is
+                # to guard against the scenario where we have '/route' as a
+                # prefix and there's a request to '/routesuffix'. In this case,
+                # it should *not* be a match.
+                elif len(target_route) == len(route) or target_route[len(route)] == "/":
+                    matched = True
+
+                if matched:
+                    endpoint = self.route_info[route]
+                    return (
+                        route,
+                        self.handles[endpoint],
+                        self.app_to_is_cross_language[endpoint.app_name],
+                    )
+
+        return None
+
+    def get_handle_for_endpoint(
+        self, target_app_name: str
+    ) -> Optional[Tuple[str, DeploymentHandle, bool]]:
+        """Return the handle that matches with endpoint.
+
+        Args:
+            target_app_name: app_name to match against.
+        Returns:
+            (route, handle, is_cross_language) for the single app if there
+            is only one, else find the app and handle for exact match. Else return None.
+        """
+        for endpoint_tag, handle in self.handles.items():
+            # If the target_app_name matches with the endpoint or if
+            # there is only one endpoint.
+            if target_app_name == endpoint_tag.app_name or len(self.handles) == 1:
+                endpoint_info = self.endpoints[endpoint_tag]
+                return (
+                    endpoint_info.route,
+                    handle,
+                    endpoint_info.app_is_cross_language,
+                )
+
+        return None
+
+    def match_route_pattern(self, route_prefix: str, asgi_scope: Scope) -> str:
+        """Match an incoming request to a specific route pattern.
+
+        This attempts to match the request path to a route pattern (e.g., /api/{user_id})
+        rather than just the route prefix. This provides more granular metrics.
+
+        The mock Starlette app is cached per route_prefix for performance, avoiding
+        the overhead of recreating the app and routes on every request.
+
+        Args:
+            route_prefix: The matched route prefix from match_route()
+            asgi_scope: The ASGI scope containing the request path and method
+
+        Returns:
+            The matched route pattern if available, otherwise the route_prefix
+        """
+        # If we don't have route patterns for this prefix, return the prefix
+        if route_prefix not in self.route_patterns:
+            return route_prefix
+
+        patterns = self.route_patterns[route_prefix]
+        if not patterns:
+            return route_prefix
+
+        # Get or create the cached mock app for this route_prefix
+        mock_app = self._route_pattern_apps.get(route_prefix)
+        if mock_app is None:
+            try:
+                # Create routes from patterns
+                # We use a dummy endpoint since we only need pattern matching
+                async def dummy_endpoint(request: Request):
+                    pass
+
+                routes = [
+                    Route(pattern.path, dummy_endpoint, methods=pattern.methods)
+                    for pattern in patterns
+                ]
+                mock_app = Starlette(routes=routes)
+
+                # Cache the mock app for future requests
+                self._route_pattern_apps[route_prefix] = mock_app
+            except Exception:
+                # If app creation fails, fall back to route prefix
+                logger.debug(
+                    f"Failed to create mock app for route pattern matching: {route_prefix}",
+                    exc_info=True,
+                )
+                return route_prefix
+
+        # Use the cached mock app to match the route pattern
+        try:
+            matched = get_asgi_route_name(mock_app, asgi_scope)
+            if matched:
+                return matched
+        except Exception:
+            # If matching fails for any reason, fall back to route prefix
+            logger.debug(
+                f"Failed to match route pattern for {route_prefix}",
+                exc_info=True,
+            )
+
+        # Fall back to route prefix if no pattern matched
+        return route_prefix

python/ray/serve/_private/queue_monitor.py

@@ -0,0 +1,224 @@
+import logging
+import time
+
+import ray
+from ray._common.constants import HEAD_NODE_RESOURCE_NAME
+from ray.actor import ActorHandle
+from ray.serve._private.broker import Broker
+from ray.serve._private.common import (
+    AsyncInferenceTaskQueueMetricReport,
+    DeploymentID,
+)
+from ray.serve._private.constants import (
+    RAY_SERVE_ASYNC_INFERENCE_TASK_QUEUE_METRIC_PUSH_INTERVAL_S,
+    SERVE_LOGGER_NAME,
+)
+from ray.serve._private.metrics_utils import MetricsPusher
+
+logger = logging.getLogger(SERVE_LOGGER_NAME)
+
+# Actor name prefix for QueueMonitor actors
+QUEUE_MONITOR_ACTOR_PREFIX = "QUEUE_MONITOR::"
+
+
+def get_queue_monitor_actor_name(deployment_id: DeploymentID) -> str:
+    """Get the Ray actor name for a deployment's QueueMonitor.
+
+    Args:
+        deployment_id: ID of the deployment (contains app_name and name)
+
+    Returns:
+        The full actor name in format "QUEUE_MONITOR::<app_name>#<deployment_name>#"
+    """
+    return f"{QUEUE_MONITOR_ACTOR_PREFIX}{deployment_id.app_name}#{deployment_id.name}#"
+
+
+@ray.remote(num_cpus=0)
+class QueueMonitorActor:
+    """
+    Actor that monitors queue length by directly querying the broker.
+
+    Returns pending tasks in the queue.
+
+    Uses native broker clients:
+        - Redis: Uses redis-py library with LLEN command
+        - RabbitMQ: Uses HTTP management API
+
+    Periodically pushes queue length metrics to the controller for autoscaling.
+    """
+
+    PUSH_METRICS_TO_CONTROLLER_TASK_NAME = "push_metrics_to_controller"
+
+    async def __init__(
+        self,
+        broker_url: str,
+        queue_name: str,
+        deployment_id: DeploymentID,
+        controller_handle: ActorHandle,
+        rabbitmq_http_url: str = "http://guest:guest@localhost:15672/api/",
+    ):
+        self._broker_url = broker_url
+        self._queue_name = queue_name
+        self._deployment_id = deployment_id
+        self._controller_handle = controller_handle
+        self._rabbitmq_http_url = rabbitmq_http_url
+
+        self._broker = Broker(self._broker_url, http_api=self._rabbitmq_http_url)
+
+        self._metrics_pusher = MetricsPusher()
+        self._start_metrics_pusher()
+
+    def _start_metrics_pusher(self):
+        """Start the metrics pusher to periodically push metrics to the controller."""
+        self._metrics_pusher.register_or_update_task(
+            self.PUSH_METRICS_TO_CONTROLLER_TASK_NAME,
+            self._push_metrics_to_controller,
+            RAY_SERVE_ASYNC_INFERENCE_TASK_QUEUE_METRIC_PUSH_INTERVAL_S,
+        )
+        self._metrics_pusher.start()
+
+    def __ray_shutdown__(self):
+        # Note: This must be synchronous (not async) because Ray's core code
+        # in _raylet.pyx calls __ray_shutdown__() without awaiting.
+        if self._metrics_pusher is not None:
+            self._metrics_pusher.stop_tasks()
+            self._metrics_pusher = None
+        if self._broker is not None:
+            self._broker.close()
+            self._broker = None
+
+    async def get_queue_length(self) -> int:
+        """
+        Fetch queue length from the broker.
+
+        Returns:
+            Number of pending tasks in the queue.
+
+        Raises:
+            ValueError: If queue is not found in broker response or
+                if queue data is missing the 'messages' field.
+        """
+        queues = await self._broker.queues([self._queue_name])
+        if queues is not None:
+            for q in queues:
+                if q.get("name") == self._queue_name:
+                    queue_length = q.get("messages")
+                    if queue_length is None:
+                        raise ValueError(
+                            f"Queue '{self._queue_name}' is missing 'messages' field"
+                        )
+                    return queue_length
+
+        raise ValueError(f"Queue '{self._queue_name}' not found in broker response")
+
+    async def _push_metrics_to_controller(self) -> None:
+        """Push queue length metrics to the controller for autoscaling."""
+        try:
+            queue_length = await self.get_queue_length()
+        except Exception as e:
+            logger.warning(
+                f"[{self._deployment_id}] Failed to get queue length for metrics push: {e}"
+            )
+            raise e
+
+        report = AsyncInferenceTaskQueueMetricReport(
+            deployment_id=self._deployment_id,
+            queue_length=queue_length,
+            timestamp_s=time.time(),
+        )
+        # Fire-and-forget push to controller
+        self._controller_handle.record_autoscaling_metrics_from_async_inference_task_queue.remote(
+            report
+        )
+
+
+def create_queue_monitor_actor(
+    deployment_id: DeploymentID,
+    broker_url: str,
+    queue_name: str,
+    controller_handle: ActorHandle,
+    rabbitmq_http_url: str = "http://guest:guest@localhost:15672/api/",
+    namespace: str = "serve",
+) -> ray.actor.ActorHandle:
+    """
+    Create a named QueueMonitor Ray actor.
+
+    Args:
+        deployment_id: ID of the deployment (contains name and app_name)
+        broker_url: URL of the message broker
+        queue_name: Name of the queue to monitor
+        controller_handle: Handle to the Serve controller for pushing metrics
+        rabbitmq_http_url: HTTP API URL for RabbitMQ management (only for RabbitMQ)
+        namespace: Ray namespace for the actor
+
+    Returns:
+        ActorHandle for the QueueMonitor actor
+    """
+    try:
+        existing = get_queue_monitor_actor(deployment_id, namespace=namespace)
+        logger.info(
+            f"QueueMonitor actor for deployment '{deployment_id}' already exists, reusing"
+        )
+        return existing
+    except ValueError:
+        actor_name = get_queue_monitor_actor_name(deployment_id)
+        actor = QueueMonitorActor.options(
+            name=actor_name,
+            namespace=namespace,
+            max_restarts=-1,
+            max_task_retries=-1,
+            resources={HEAD_NODE_RESOURCE_NAME: 0.001},
+        ).remote(
+            broker_url=broker_url,
+            queue_name=queue_name,
+            deployment_id=deployment_id,
+            controller_handle=controller_handle,
+            rabbitmq_http_url=rabbitmq_http_url,
+        )
+
+        logger.info(
+            f"Created QueueMonitor actor '{actor_name}' in namespace '{namespace}'"
+        )
+        return actor
+
+
+def get_queue_monitor_actor(
+    deployment_id: DeploymentID,
+    namespace: str = "serve",
+) -> ray.actor.ActorHandle:
+    """
+    Get an existing QueueMonitor actor by name.
+
+    Args:
+        deployment_id: ID of the deployment (contains app_name and name)
+        namespace: Ray namespace
+
+    Returns:
+        ActorHandle for the QueueMonitor actor
+
+    Raises:
+        ValueError: If actor doesn't exist
+    """
+    actor_name = get_queue_monitor_actor_name(deployment_id)
+    return ray.get_actor(actor_name, namespace=namespace)
+
+
+def kill_queue_monitor_actor(
+    deployment_id: DeploymentID,
+    namespace: str = "serve",
+) -> None:
+    """
+    Delete a QueueMonitor actor by name.
+
+    Args:
+        deployment_id: ID of the deployment (contains app_name and name)
+        namespace: Ray namespace
+
+    Raises:
+        ValueError: If actor doesn't exist
+    """
+    actor_name = get_queue_monitor_actor_name(deployment_id)
+    actor = get_queue_monitor_actor(deployment_id, namespace=namespace)
+
+    ray.kill(actor, no_restart=True)
+    logger.info(f"Deleted QueueMonitor actor '{actor_name}'")

python/ray/serve/_private/replica_response_generator.py

@@ -0,0 +1,91 @@
+import asyncio
+import time
+from typing import Any, AsyncGenerator, Callable, Optional
+
+from ray.serve._private.proxy_response_generator import (
+    _ProxyResponseGeneratorBase,
+    swallow_cancelled,
+)
+from ray.serve._private.utils import calculate_remaining_timeout
+
+
+class ReplicaResponseGenerator(_ProxyResponseGeneratorBase):
+    """Generic wrapper that adds disconnect detection to any async generator.
+
+    This can be used to wrap any async generator and add timeout and disconnect
+    detection capabilities. When a disconnect is detected, the generator will
+    raise asyncio.CancelledError.
+    """
+
+    def __init__(
+        self,
+        async_generator: AsyncGenerator[Any, None],
+        *,
+        timeout_s: Optional[float] = None,
+        disconnected_task: Optional[asyncio.Task] = None,
+        result_callback: Optional[Callable[[Any], Any]] = None,
+    ):
+        super().__init__(
+            timeout_s=timeout_s,
+            disconnected_task=disconnected_task,
+            result_callback=result_callback,
+        )
+        self._async_generator = async_generator
+        self._done = False
+
+    async def __anext__(self):
+        if self._done:
+            raise StopAsyncIteration
+
+        try:
+            result = await self._get_next_result()
+            if self._result_callback is not None:
+                result = self._result_callback(result)
+            return result
+        except (StopAsyncIteration, asyncio.CancelledError) as e:
+            self._done = True
+            raise e from None
+        except Exception as e:
+            self._done = True
+            raise e from None
+
+    async def _await_response_anext(self) -> Any:
+        return await self._async_generator.__anext__()
+
+    async def _get_next_result(self) -> Any:
+        """Get the next result from the async generator with disconnect detection."""
+        # If there's no disconnect detection needed, use direct await to preserve
+        # cancellation propagation (important for gRPC cancellation)
+        remaining_timeout = calculate_remaining_timeout(
+            timeout_s=self._timeout_s,
+            start_time_s=self._start_time_s,
+            curr_time_s=time.time(),
+        )
+        if self._disconnected_task is None:
+            try:
+                return await asyncio.wait_for(
+                    self._await_response_anext(), timeout=remaining_timeout
+                )
+            except asyncio.TimeoutError:
+                raise TimeoutError()
+        # Otherwise use asyncio.wait for disconnect detection
+        next_result_task = asyncio.create_task(self._await_response_anext())
+        tasks = [next_result_task, self._disconnected_task]
+
+        done, _ = await asyncio.wait(
+            tasks,
+            return_when=asyncio.FIRST_COMPLETED,
+            timeout=remaining_timeout,
+        )
+
+        if next_result_task in done:
+            return next_result_task.result()
+        elif self._disconnected_task in done:
+            next_result_task.cancel()
+            next_result_task.add_done_callback(swallow_cancelled)
+            raise asyncio.CancelledError()
+        else:
+            # Timeout occurred
+            next_result_task.cancel()
+            next_result_task.add_done_callback(swallow_cancelled)
+            raise TimeoutError()

python/ray/serve/_private/replica_result.py

@@ -0,0 +1,590 @@
+import asyncio
+import concurrent.futures
+import inspect
+import logging
+import pickle
+import threading
+import time
+from abc import ABC, abstractmethod
+from asyncio import run_coroutine_threadsafe
+from functools import wraps
+from typing import Any, AsyncIterator, Callable, Coroutine, Iterator, Optional, Union
+
+import grpc
+
+import ray
+from ray.exceptions import ActorUnavailableError, RayTaskError, TaskCancelledError
+from ray.serve._private.common import (
+    OBJ_REF_NOT_SUPPORTED_ERROR,
+    ReplicaQueueLengthInfo,
+    RequestMetadata,
+)
+from ray.serve._private.constants import SERVE_LOGGER_NAME
+from ray.serve._private.http_util import MessageQueue
+from ray.serve._private.serialization import RPCSerializer
+from ray.serve._private.utils import calculate_remaining_timeout, generate_request_id
+from ray.serve.exceptions import RequestCancelledError
+from ray.serve.generated.serve_pb2 import ASGIResponse
+
+logger = logging.getLogger(SERVE_LOGGER_NAME)
+
+
+def is_running_in_asyncio_loop() -> bool:
+    try:
+        asyncio.get_running_loop()
+        return True
+    except RuntimeError:
+        return False
+
+
+class ReplicaResult(ABC):
+    @abstractmethod
+    async def get_rejection_response(self) -> Optional[ReplicaQueueLengthInfo]:
+        raise NotImplementedError
+
+    @abstractmethod
+    def get(self, timeout_s: Optional[float]):
+        raise NotImplementedError
+
+    @abstractmethod
+    async def get_async(self):
+        raise NotImplementedError
+
+    @abstractmethod
+    def __next__(self):
+        raise NotImplementedError
+
+    @abstractmethod
+    async def __anext__(self):
+        raise NotImplementedError
+
+    @abstractmethod
+    def add_done_callback(self, callback: Callable):
+        raise NotImplementedError
+
+    @abstractmethod
+    def cancel(self):
+        raise NotImplementedError
+
+    @abstractmethod
+    def to_object_ref(self, timeout_s: Optional[float]) -> ray.ObjectRef:
+        raise NotImplementedError
+
+    @abstractmethod
+    async def to_object_ref_async(self) -> ray.ObjectRef:
+        raise NotImplementedError
+
+    @abstractmethod
+    def to_object_ref_gen(self) -> ray.ObjectRefGenerator:
+        # NOTE(edoakes): there is only a sync version of this method because it
+        # does not block like `to_object_ref` (so there's also no timeout argument).
+        raise NotImplementedError
+
+
+class ActorReplicaResult(ReplicaResult):
+    def __init__(
+        self,
+        obj_ref_or_gen: Union[ray.ObjectRef, ray.ObjectRefGenerator],
+        metadata: RequestMetadata,
+        *,
+        with_rejection: bool = False,
+    ):
+        self._obj_ref: Optional[ray.ObjectRef] = None
+        self._obj_ref_gen: Optional[ray.ObjectRefGenerator] = None
+        self._is_streaming: bool = metadata.is_streaming
+        self._request_id: str = metadata.request_id
+        self._object_ref_or_gen_sync_lock = threading.Lock()
+        self._with_rejection = with_rejection
+        self._rejection_response = None
+
+        if isinstance(obj_ref_or_gen, ray.ObjectRefGenerator):
+            self._obj_ref_gen = obj_ref_or_gen
+        else:
+            self._obj_ref = obj_ref_or_gen
+
+        if self._is_streaming:
+            assert (
+                self._obj_ref_gen is not None
+            ), "An ObjectRefGenerator must be passed for streaming requests."
+
+        request_context = ray.serve.context._get_serve_request_context()
+        if request_context.cancel_on_parent_request_cancel:
+            # Keep track of in-flight requests.
+            self._response_id = generate_request_id()
+            ray.serve.context._add_in_flight_request(
+                request_context._internal_request_id, self._response_id, self
+            )
+            self.add_done_callback(
+                lambda _: ray.serve.context._remove_in_flight_request(
+                    request_context._internal_request_id, self._response_id
+                )
+            )
+
+    def _process_response(f: Union[Callable, Coroutine]):
+        @wraps(f)
+        def wrapper(self, *args, **kwargs):
+            try:
+                return f(self, *args, **kwargs)
+            except ray.exceptions.TaskCancelledError:
+                raise RequestCancelledError(self._request_id)
+
+        @wraps(f)
+        async def async_wrapper(self, *args, **kwargs):
+            try:
+                return await f(self, *args, **kwargs)
+            except ray.exceptions.TaskCancelledError:
+                raise asyncio.CancelledError()
+
+        if inspect.iscoroutinefunction(f):
+            return async_wrapper
+        else:
+            return wrapper
+
+    @_process_response
+    async def get_rejection_response(self) -> Optional[ReplicaQueueLengthInfo]:
+        """Get the queue length info from the replica to handle rejection."""
+        assert (
+            self._with_rejection and self._obj_ref_gen is not None
+        ), "get_rejection_response() can only be called when request rejection is enabled."
+
+        try:
+            if self._rejection_response is None:
+                response = await (await self._obj_ref_gen.__anext__())
+                self._rejection_response = pickle.loads(response)
+
+            return self._rejection_response
+        except asyncio.CancelledError as e:
+            # HTTP client disconnected or request was explicitly canceled.
+            logger.info(
+                "Cancelling request that has already been assigned to a replica."
+            )
+            self.cancel()
+            raise e from None
+        except TaskCancelledError:
+            raise asyncio.CancelledError()
+
+    @_process_response
+    def get(self, timeout_s: Optional[float]):
+        assert (
+            not self._is_streaming
+        ), "get() can only be called on a unary ActorReplicaResult."
+
+        start_time_s = time.time()
+        object_ref = self.to_object_ref(timeout_s=timeout_s)
+        remaining_timeout_s = calculate_remaining_timeout(
+            timeout_s=timeout_s,
+            start_time_s=start_time_s,
+            curr_time_s=time.time(),
+        )
+        return ray.get(object_ref, timeout=remaining_timeout_s)
+
+    @_process_response
+    async def get_async(self):
+        assert (
+            not self._is_streaming
+        ), "get_async() can only be called on a unary ActorReplicaResult."
+
+        return await (await self.to_object_ref_async())
+
+    @_process_response
+    def __next__(self):
+        assert (
+            self._is_streaming
+        ), "next() can only be called on a streaming ActorReplicaResult."
+
+        next_obj_ref = self._obj_ref_gen.__next__()
+        return ray.get(next_obj_ref)
+
+    @_process_response
+    async def __anext__(self):
+        assert (
+            self._is_streaming
+        ), "__anext__() can only be called on a streaming ActorReplicaResult."
+
+        next_obj_ref = await self._obj_ref_gen.__anext__()
+        return await next_obj_ref
+
+    def add_done_callback(self, callback: Callable):
+        if self._obj_ref_gen is not None:
+            self._obj_ref_gen.completed()._on_completed(callback)
+        else:
+            self._obj_ref._on_completed(callback)
+
+    def cancel(self):
+        if self._obj_ref_gen is not None:
+            ray.cancel(self._obj_ref_gen)
+        else:
+            ray.cancel(self._obj_ref)
+
+    def to_object_ref(self, *, timeout_s: Optional[float] = None) -> ray.ObjectRef:
+        assert (
+            not self._is_streaming
+        ), "to_object_ref can only be called on a unary ReplicaActorResult."
+
+        # NOTE(edoakes): this section needs to be guarded with a lock and the resulting
+        # object ref cached in order to avoid calling `__next__()` to
+        # resolve to the underlying object ref more than once.
+        # See: https://github.com/ray-project/ray/issues/43879.
+        with self._object_ref_or_gen_sync_lock:
+            if self._obj_ref is None:
+                obj_ref = self._obj_ref_gen._next_sync(timeout_s=timeout_s)
+                if obj_ref.is_nil():
+                    raise TimeoutError("Timed out resolving to ObjectRef.")
+
+                self._obj_ref = obj_ref
+
+        return self._obj_ref
+
+    async def to_object_ref_async(self) -> ray.ObjectRef:
+        assert (
+            not self._is_streaming
+        ), "to_object_ref_async can only be called on a unary ReplicaActorResult."
+
+        # NOTE(edoakes): this section needs to be guarded with a lock and the resulting
+        # object ref cached in order to avoid calling `__anext__()` to
+        # resolve to the underlying object ref more than once.
+        # See: https://github.com/ray-project/ray/issues/43879.
+        #
+        # IMPORTANT: We use a threading lock instead of asyncio.Lock because this method
+        # can be called from multiple event loops concurrently:
+        # 1. From the user's code (on the replica's event loop) when awaiting a response
+        # 2. From the router's event loop when resolving a DeploymentResponse argument
+        # asyncio.Lock is NOT thread-safe and NOT designed for cross-loop usage, which
+        # causes deadlocks.
+        #
+        # We use a non-blocking acquire pattern to avoid blocking the event loop:
+        # - Try to acquire the lock without blocking
+        # - If already held, yield and retry (allows other async tasks to run)
+        # - Once acquired, check if result is already available (double-check pattern)
+        while True:
+            # Fast path: already computed
+            if self._obj_ref is not None:
+                return self._obj_ref
+
+            acquired = self._object_ref_or_gen_sync_lock.acquire(blocking=False)
+            if acquired:
+                try:
+                    # Double-check under lock
+                    if self._obj_ref is None:
+                        self._obj_ref = await self._obj_ref_gen.__anext__()
+                    return self._obj_ref
+                finally:
+                    self._object_ref_or_gen_sync_lock.release()
+            else:
+                # Lock is held by another task/thread, yield and retry
+                await asyncio.sleep(0)
+
+    def to_object_ref_gen(self) -> ray.ObjectRefGenerator:
+        assert (
+            self._is_streaming
+        ), "to_object_ref_gen can only be called on a streaming ReplicaActorResult."
+
+        return self._obj_ref_gen
+
+
+class gRPCReplicaResult(ReplicaResult):
+    def __init__(
+        self,
+        call: grpc.aio.Call,
+        metadata: RequestMetadata,
+        actor_id: ray.ActorID,
+        loop: asyncio.AbstractEventLoop = None,
+        *,
+        with_rejection: bool = False,
+    ):
+        self._call: grpc.aio.Call = call
+        self._actor_id: ray.ActorID = actor_id
+        self._metadata: RequestMetadata = metadata  # Store metadata for serialization
+        self._result_queue: MessageQueue = MessageQueue()
+        # This is the asyncio event loop that the gRPC Call object is attached to
+        self._grpc_call_loop = loop or asyncio._get_running_loop()
+        self._is_streaming = metadata.is_streaming
+        self._with_rejection = with_rejection
+        self._rejection_response = None
+
+        self._gen = None
+        self._fut = None
+
+        # NOTE(zcin): for now, these two concepts will be synonymous.
+        # In other words, using a queue means the router is running on
+        # a separate thread/event loop, and vice versa not using a queue
+        # means the router is running on the main event loop, where the
+        # DeploymentHandle lives.
+        self._calling_from_same_loop = not metadata._on_separate_loop
+        if hasattr(self._call, "__aiter__"):
+            self._gen = self._call.__aiter__()
+            # If the grpc call IS streaming, AND it was created on a
+            # a separate loop, then use a queue to fetch the objects
+            self._use_queue = metadata._on_separate_loop
+        else:
+            self._use_queue = False
+
+        # Start a background task that continuously fetches from the
+        # streaming grpc call. This way callbacks will actually be
+        # called when the request finishes even without the user
+        # explicitly consuming the response.
+        self._consume_task = None
+        if self._use_queue:
+            self._consume_task = self._grpc_call_loop.create_task(
+                self.consume_messages_from_gen()
+            )
+
+        # Keep track of in-flight requests.
+        self._response_id = generate_request_id()
+        request_context = ray.serve.context._get_serve_request_context()
+        ray.serve.context._add_in_flight_request(
+            request_context._internal_request_id, self._response_id, self
+        )
+        self.add_done_callback(
+            lambda _: ray.serve.context._remove_in_flight_request(
+                request_context._internal_request_id, self._response_id
+            )
+        )
+
+    def _process_grpc_response(f: Union[Callable, Coroutine]):
+        def deserialize_or_raise_error(
+            grpc_response: ASGIResponse,
+            metadata: RequestMetadata,
+        ):
+            # Create serializer with options from metadata
+            serializer = RPCSerializer(
+                metadata.request_serialization,
+                metadata.response_serialization,
+            )
+
+            if grpc_response.is_error:
+                err = serializer.loads_response(grpc_response.serialized_message)
+                if isinstance(err, RayTaskError):
+                    raise err.as_instanceof_cause()
+                else:
+                    raise err
+            else:
+                # If it's an HTTP request, then the proxy response generator is
+                # expecting a pickled dictionary, so we return result directly
+                # without deserializing. Otherwise, we deserialize the result.
+                if ray.serve.context._get_serve_request_context().is_http_request:
+                    return grpc_response.serialized_message
+                else:
+                    return serializer.loads_response(grpc_response.serialized_message)
+
+        @wraps(f)
+        def wrapper(self, *args, **kwargs):
+            try:
+                grpc_response = f(self, *args, **kwargs)
+            except grpc.aio.AioRpcError as e:
+                if e.code() == grpc.StatusCode.UNAVAILABLE:
+                    raise ActorUnavailableError(
+                        "Actor is unavailable.",
+                        self._actor_id.binary(),
+                    )
+                raise
+            except concurrent.futures.CancelledError:
+                raise RequestCancelledError from None
+
+            return deserialize_or_raise_error(grpc_response, self._metadata)
+
+        @wraps(f)
+        async def async_wrapper(self, *args, **kwargs):
+            try:
+                grpc_response = await f(self, *args, **kwargs)
+            except grpc.aio.AioRpcError as e:
+                if e.code() == grpc.StatusCode.UNAVAILABLE:
+                    raise ActorUnavailableError(
+                        "Actor is unavailable.",
+                        self._actor_id.binary(),
+                    )
+                raise
+
+            return deserialize_or_raise_error(grpc_response, self._metadata)
+
+        if inspect.iscoroutinefunction(f):
+            return async_wrapper
+        else:
+            return wrapper
+
+    def __aiter__(self) -> AsyncIterator[Any]:
+        return self
+
+    def __iter__(self) -> Iterator[Any]:
+        return self
+
+    async def consume_messages_from_gen(self):
+        try:
+            async for resp in self._gen:
+                self._result_queue.put_nowait(resp)
+        except BaseException as e:
+            self._result_queue.set_error(e)
+        finally:
+            self._result_queue.close()
+
+    async def _get_internal(self):
+        """Gets the result from the gRPC call object.
+
+        If the call object is a UnaryUnaryCall, we await the call.
+        Otherwise the call object is a UnaryStreamCall.
+          - If the request was sent on a separate loop, then the
+          streamed results are being consumed and put onto the in-memory
+          queue, so we read from that queue.
+          - Otherwise the request was sent on the current loop, so we
+          fetch the next object from the async generator.
+        """
+
+        if self._gen is None:
+            return await self._call
+        elif self._use_queue:
+            return await self._result_queue.get_one_message()
+        else:
+            return await self._gen.__anext__()
+
+    async def get_rejection_response(self) -> Optional[ReplicaQueueLengthInfo]:
+        """Get the queue length info from the replica to handle rejection."""
+        assert (
+            self._with_rejection
+        ), "get_rejection_response() can only be called when request rejection is enabled."
+
+        try:
+            if self._rejection_response is None:
+                # NOTE(edoakes): this is required for gRPC to raise an AioRpcError if something
+                # goes wrong establishing the connection (for example, a bug in our code).
+                await self._call.wait_for_connection()
+                metadata = await self._call.initial_metadata()
+
+                accepted = metadata.get("accepted", None)
+                num_ongoing_requests = metadata.get("num_ongoing_requests", None)
+                if accepted is None or num_ongoing_requests is None:
+                    code = await self._call.code()
+                    details = await self._call.details()
+                    raise RuntimeError(f"Unexpected error ({code}): {details}.")
+
+                self._rejection_response = ReplicaQueueLengthInfo(
+                    accepted=bool(int(accepted)),
+                    num_ongoing_requests=int(num_ongoing_requests),
+                )
+
+            return self._rejection_response
+        except asyncio.CancelledError as e:
+            # HTTP client disconnected or request was explicitly canceled.
+            logger.info(
+                "Cancelling request that has already been assigned to a replica."
+            )
+            self.cancel()
+            raise e from None
+        except grpc.aio.AioRpcError as e:
+            # If we received an `UNAVAILABLE` grpc error, that is
+            # equivalent to `RayActorError`, although we don't know
+            # whether it's `ActorDiedError` or `ActorUnavailableError`.
+            # Conservatively, we assume it is `ActorUnavailableError`,
+            # and we raise it here so that it goes through the unified
+            # code path for handling RayActorErrors.
+            # The router will retry scheduling the request with the
+            # cache invalidated, at which point if the actor is actually
+            # dead, the router will realize through active probing.
+            if not self._is_streaming:
+                # In UnaryUnary calls, initial metadata is sent back with the request
+                # response, so we can't determine if the request was accepted until
+                # after the request is handled. If the replica crashed while handling
+                # the request, we can still get initial metadata via the AioRpcError,
+                # since the server sets the metadata before handling the request.
+                # If there is no metadata, we know the replica was already unavailable
+                # prior to the request being sent. We only raise an ActorUnavailableError
+                # (and thus retry the request) if the request was rejected or if the
+                # replica was already unavailable.
+                metadata = e.initial_metadata()
+                accepted = metadata.get("accepted", None)
+                if accepted is not None and bool(int(accepted)):
+                    num_ongoing_requests = metadata.get("num_ongoing_requests", None)
+                    if num_ongoing_requests is None:
+                        raise RuntimeError(
+                            f"Unexpected error ({e.code()}): {e.details()}."
+                        )
+
+                    return ReplicaQueueLengthInfo(
+                        accepted=True,
+                        num_ongoing_requests=int(num_ongoing_requests),
+                    )
+
+            # Peer-sent CANCELLED means the replica's gRPC server cancelled
+            # the call before the handler ran (graceful-shutdown window).
+            # Without `accepted` metadata (checked above) the request never
+            # executed, so it is safe to retry, same as UNAVAILABLE. A local
+            # cancellation raises asyncio.CancelledError, not AioRpcError.
+            if e.code() in (
+                grpc.StatusCode.UNAVAILABLE,
+                grpc.StatusCode.CANCELLED,
+            ):
+                raise ActorUnavailableError(
+                    "Actor is unavailable.",
+                    self._actor_id.binary(),
+                )
+
+            raise e from None
+
+    @_process_grpc_response
+    def get(self, timeout_s: Optional[float]):
+        if is_running_in_asyncio_loop():
+            raise RuntimeError(
+                "Sync method `get()` should not be called from within an `asyncio` "
+                "event loop. Use `get_async()` instead."
+            )
+
+        if self._fut is None:
+            self._fut = run_coroutine_threadsafe(
+                self._get_internal(), self._grpc_call_loop
+            )
+
+        try:
+            return self._fut.result(timeout=timeout_s)
+        except concurrent.futures.TimeoutError:
+            raise TimeoutError("Timed out waiting for result.") from None
+
+    @_process_grpc_response
+    async def get_async(self):
+        if self._fut is None:
+            if self._calling_from_same_loop:
+                return await self._get_internal()
+            else:
+                self._fut = run_coroutine_threadsafe(
+                    self._get_internal(), self._grpc_call_loop
+                )
+
+        return await asyncio.wrap_future(self._fut)
+
+    @_process_grpc_response
+    def __next__(self):
+        if is_running_in_asyncio_loop():
+            raise RuntimeError(
+                "Sync method `__next__()` should not be called from within an "
+                "`asyncio` event loop. Use `__anext__()` instead."
+            )
+
+        fut = run_coroutine_threadsafe(self._get_internal(), loop=self._grpc_call_loop)
+        try:
+            return fut.result()
+        except StopAsyncIteration:
+            # We need to raise the synchronous version, StopIteration
+            raise StopIteration
+
+    @_process_grpc_response
+    async def __anext__(self):
+        if self._calling_from_same_loop:
+            return await self._get_internal()
+        else:
+            fut = run_coroutine_threadsafe(
+                self._get_internal(), loop=self._grpc_call_loop
+            )
+            return await asyncio.wrap_future(fut)
+
+    def add_done_callback(self, callback: Callable):
+        self._call.add_done_callback(callback)
+
+    def cancel(self):
+        self._call.cancel()
+
+    def to_object_ref(self, timeout_s: Optional[float]) -> ray.ObjectRef:
+        raise OBJ_REF_NOT_SUPPORTED_ERROR
+
+    async def to_object_ref_async(self) -> ray.ObjectRef:
+        raise OBJ_REF_NOT_SUPPORTED_ERROR
+
+    def to_object_ref_gen(self) -> ray.ObjectRefGenerator:
+        raise OBJ_REF_NOT_SUPPORTED_ERROR

python/ray/serve/_private/request_ingress_metrics.py

@@ -0,0 +1,180 @@
+from typing import Dict
+
+from ray.serve._private.common import RequestProtocol
+from ray.serve._private.constants import REQUEST_LATENCY_BUCKETS_MS
+from ray.util import metrics
+
+
+class RequestIngressMetrics:
+    """E2E request metrics shared by the proxies and direct-ingress replicas.
+
+    Defines and emits the standard ``serve_num_{protocol}_requests`` family of
+    metrics. Both the proxy (which sees all proxy-routed traffic) and a
+    direct-ingress replica (which sees traffic that bypasses the proxy) record
+    them, so the series are disjoint at runtime.
+    """
+
+    def __init__(
+        self,
+        protocol: RequestProtocol,
+        *,
+        source: str,
+        node_id: str,
+        node_ip_address: str,
+    ):
+        """Create the metric objects.
+
+        Args:
+            protocol: Request protocol these metrics describe (e.g. HTTP).
+            source: Human-readable origin of the metrics, "proxy" or "ingress".
+                Only affects the metric descriptions so that they remain
+                accurate for each call site.
+            node_id: Default tag value for the ongoing-requests gauge.
+            node_ip_address: Default tag value for the ongoing-requests gauge.
+        """
+        protocol_name = protocol.lower()
+
+        self.request_counter = metrics.Counter(
+            f"serve_num_{protocol_name}_requests",
+            description=f"The number of {protocol.value} requests processed.",
+            tag_keys=("route", "method", "application", "status_code"),
+        )
+
+        self.request_error_counter = metrics.Counter(
+            f"serve_num_{protocol_name}_error_requests",
+            description=f"The number of errored {protocol.value} responses.",
+            tag_keys=(
+                "route",
+                "error_code",
+                "method",
+                "application",
+            ),
+        )
+
+        self.deployment_request_error_counter = metrics.Counter(
+            f"serve_num_deployment_{protocol_name}_error_requests",
+            description=(
+                f"The number of errored {protocol.value} "
+                "responses returned by each deployment."
+            ),
+            tag_keys=(
+                "deployment",
+                "error_code",
+                "method",
+                "route",
+                "application",
+            ),
+        )
+
+        self.processing_latency_tracker = metrics.Histogram(
+            f"serve_{protocol_name}_request_latency_ms",
+            description=(
+                f"The end-to-end latency of {protocol.value} requests "
+                f"(measured from the Serve {protocol.value} {source})."
+            ),
+            boundaries=REQUEST_LATENCY_BUCKETS_MS,
+            tag_keys=(
+                "method",
+                "route",
+                "application",
+                "status_code",
+            ),
+        )
+
+        self.num_ongoing_requests_gauge = metrics.Gauge(
+            name=f"serve_num_ongoing_{protocol_name}_requests",
+            description=(
+                f"The number of ongoing requests in this {protocol.value} {source}."
+            ),
+            tag_keys=("node_id", "node_ip_address"),
+        ).set_default_tags(
+            {
+                "node_id": node_id,
+                "node_ip_address": node_ip_address,
+            }
+        )
+
+    @staticmethod
+    def request_tags(
+        *, route: str, method: str, application: str, status_code: str
+    ) -> Dict[str, str]:
+        """Tags for the request counter and processing-latency tracker."""
+        return {
+            "route": route,
+            "method": method,
+            "application": application,
+            "status_code": status_code,
+        }
+
+    @staticmethod
+    def request_error_tags(
+        *, route: str, method: str, application: str, status_code: str
+    ) -> Dict[str, str]:
+        """Tags for the request error counter."""
+        return {
+            "route": route,
+            "method": method,
+            "application": application,
+            "error_code": status_code,
+        }
+
+    @staticmethod
+    def deployment_error_tags(
+        *,
+        route: str,
+        method: str,
+        application: str,
+        status_code: str,
+        deployment: str,
+    ) -> Dict[str, str]:
+        """Tags for the per-deployment request error counter."""
+        return {
+            "route": route,
+            "method": method,
+            "application": application,
+            "error_code": status_code,
+            "deployment": deployment,
+        }
+
+    def record_request(
+        self,
+        *,
+        route: str,
+        method: str,
+        application: str,
+        status_code: str,
+        latency_ms: float,
+        is_error: bool,
+        deployment_name: str,
+    ):
+        """Emit the per-request metrics directly (no batching)."""
+        request_tags = self.request_tags(
+            route=route,
+            method=method,
+            application=application,
+            status_code=status_code,
+        )
+        self.request_counter.inc(tags=request_tags)
+        self.processing_latency_tracker.observe(latency_ms, tags=request_tags)
+        if is_error:
+            self.request_error_counter.inc(
+                tags=self.request_error_tags(
+                    route=route,
+                    method=method,
+                    application=application,
+                    status_code=status_code,
+                )
+            )
+            self.deployment_request_error_counter.inc(
+                tags=self.deployment_error_tags(
+                    route=route,
+                    method=method,
+                    application=application,
+                    status_code=status_code,
+                    deployment=deployment_name,
+                )
+            )
+
+    def set_num_ongoing_requests(self, num_ongoing_requests: int):
+        """Set the ongoing-requests gauge."""
+        self.num_ongoing_requests_gauge.set(num_ongoing_requests)

python/ray/serve/_private/request_router/__init__.py

@@ -0,0 +1,11 @@
+from ray.serve._private.request_router.common import PendingRequest  # noqa: F401
+from ray.serve._private.request_router.pow_2_router import (  # noqa: F401
+    PowerOfTwoChoicesRequestRouter,
+)
+from ray.serve._private.request_router.replica_wrapper import (  # noqa: F401
+    ReplicaSelection,
+    RunningReplica,
+)
+from ray.serve._private.request_router.request_router import (  # noqa: F401
+    RequestRouter,
+)

python/ray/serve/_private/request_router/common.py

@@ -0,0 +1,107 @@
+import asyncio
+import logging
+import time
+from dataclasses import dataclass, field
+from typing import Any, Callable, Dict, List, Optional, Set
+
+from ray.serve._private.common import ReplicaID, RequestMetadata
+from ray.serve._private.constants import (
+    RAY_SERVE_QUEUE_LENGTH_CACHE_TIMEOUT_S,
+    SERVE_LOGGER_NAME,
+)
+from ray.util.annotations import PublicAPI
+
+logger = logging.getLogger(SERVE_LOGGER_NAME)
+
+
+@dataclass()
+class RequestRoutingContext:
+    multiplexed_start_matching_time: Optional[float] = None
+    tried_fewest_multiplexed_models: bool = False
+    tried_first_multiplexed_models: bool = False
+    tried_same_node: bool = False
+    tried_same_az: bool = False
+    should_backoff: bool = False
+
+
+@PublicAPI(stability="alpha")
+@dataclass
+class PendingRequest:
+    """A request that is pending execution by a replica."""
+
+    args: List[Any]
+    """Positional arguments for the request."""
+
+    kwargs: Dict[Any, Any]
+    """Keyword arguments for the request."""
+
+    metadata: RequestMetadata
+    """Metadata for the request, including request ID and whether it's streaming."""
+
+    created_at: float = field(default_factory=lambda: time.time())
+    """Timestamp when the request was created."""
+
+    future: asyncio.Future = field(default_factory=lambda: asyncio.Future())
+    """An asyncio Future that will be set when the request is routed."""
+
+    routing_context: RequestRoutingContext = field(
+        default_factory=RequestRoutingContext
+    )
+    """Context for request routing, used to track routing attempts and backoff."""
+
+    resolved: bool = False
+    """Whether the arguments have been resolved."""
+
+    def reset_future(self):
+        """Reset the `asyncio.Future`, must be called if this request is re-used."""
+        self.future = asyncio.Future()
+
+
+@dataclass(frozen=True)
+class ReplicaQueueLengthCacheEntry:
+    queue_len: int
+    timestamp: float
+
+
+class ReplicaQueueLengthCache:
+    def __init__(
+        self,
+        *,
+        staleness_timeout_s: float = RAY_SERVE_QUEUE_LENGTH_CACHE_TIMEOUT_S,
+        get_curr_time_s: Optional[Callable[[], float]] = None,
+    ):
+        self._cache: Dict[ReplicaID, ReplicaQueueLengthCacheEntry] = {}
+        self._staleness_timeout_s = staleness_timeout_s
+        self._get_curr_time_s = (
+            get_curr_time_s if get_curr_time_s is not None else lambda: time.time()
+        )
+
+    def _is_timed_out(self, timestamp_s: int) -> bool:
+        return self._get_curr_time_s() - timestamp_s > self._staleness_timeout_s
+
+    def get(self, replica_id: ReplicaID) -> Optional[int]:
+        """Get the queue length for a replica.
+
+        Returns `None` if the replica ID is not present or the entry is timed out.
+        """
+        entry = self._cache.get(replica_id)
+        if entry is None or self._is_timed_out(entry.timestamp):
+            return None
+
+        return entry.queue_len
+
+    def update(self, replica_id: ReplicaID, queue_len: int):
+        """Set (or update) the queue length for a replica ID."""
+        self._cache[replica_id] = ReplicaQueueLengthCacheEntry(
+            queue_len, self._get_curr_time_s()
+        )
+
+    def invalidate_key(self, replica_id: ReplicaID):
+        self._cache.pop(replica_id, None)
+
+    def remove_inactive_replicas(self, *, active_replica_ids: Set[ReplicaID]):
+        """Removes entries for all replica IDs not in the provided active set."""
+        # NOTE: the size of the cache dictionary changes during this loop.
+        for replica_id in list(self._cache.keys()):
+            if replica_id not in active_replica_ids:
+                self._cache.pop(replica_id)

python/ray/serve/_private/request_router/pow_2_router.py

@@ -0,0 +1,106 @@
+import logging
+import random
+from typing import (
+    List,
+    Optional,
+)
+
+from ray.serve._private.constants import (
+    SERVE_LOGGER_NAME,
+)
+from ray.serve._private.request_router.common import (
+    PendingRequest,
+)
+from ray.serve._private.request_router.replica_wrapper import (
+    RunningReplica,
+)
+from ray.serve._private.request_router.request_router import (
+    FIFOMixin,
+    LocalityMixin,
+    MultiplexMixin,
+    RequestRouter,
+)
+
+logger = logging.getLogger(SERVE_LOGGER_NAME)
+
+
+class PowerOfTwoChoicesRequestRouter(
+    FIFOMixin, LocalityMixin, MultiplexMixin, RequestRouter
+):
+    """Chooses a replica for each request using the "power of two choices" procedure.
+
+    Requests are routed in FIFO order.
+
+    When a request comes in, two candidate replicas are chosen randomly. Each replica
+    is sent a control message to fetch its queue length.
+
+    The replica responds with two items: (queue_len, accepted). Only replicas that
+    accept the request are considered; between those, the one with the lower queue
+    length is chosen.
+
+    In the case when neither replica accepts the request (e.g., their queues are full),
+    the procedure is repeated with backoff. This backoff repeats indefinitely until a
+    replica is chosen, so the caller should use timeouts and cancellation to avoid
+    hangs.
+
+    Each request being routed may spawn an independent task that runs the routing
+    procedure concurrently. This task will not necessarily satisfy the request that
+    started it (in order to maintain the FIFO order). The total number of tasks is
+    capped at (2 * num_replicas).
+    """
+
+    async def choose_replicas(
+        self,
+        candidate_replicas: List[RunningReplica],
+        pending_request: Optional[PendingRequest] = None,
+    ) -> List[List[RunningReplica]]:
+        """One iteration of the power of two choices procedure that chooses
+         (at most) two random available replicas.
+
+        For multiplexing, this will first attempt to choose replicas that have the
+        requested model ID for a configured timeout. If no replicas with the matching
+        model ID are available after that timeout, it will fall back to the regular
+        procedure.
+        """
+        if (
+            pending_request is not None
+            and pending_request.metadata.multiplexed_model_id
+        ):
+            # Get candidates for multiplexed model ID.
+            candidate_replica_ids = self.apply_multiplex_routing(
+                pending_request=pending_request,
+            )
+        else:
+            # Get candidates for locality preference.
+            candidate_replica_ids = self.apply_locality_routing(
+                pending_request=pending_request,
+            )
+
+        if not candidate_replica_ids:
+            return []
+
+        # Optimized selection: use direct randrange for k=2 instead of random.sample.
+        # This is ~1.9x faster for the common case of selecting 2 replicas.
+        #
+        # Correctness proof: We pick i uniformly from [0, n), then j uniformly from
+        # [0, n-1) and shift j up if j >= i. Every ordered pair (i, j) with i != j
+        # has probability: Pr(i,j) = 1/n * 1/(n-1) = 1/(n(n-1))
+        # This matches random.sample(k=2): uniform among all 2-permutations.
+        candidates = tuple(candidate_replica_ids)
+        n = len(candidates)
+        if n == 1:
+            chosen_ids = [candidates[0]]
+        elif n == 2:
+            # Randomize order to ensure fair selection when queue lengths are equal
+            if random.getrandbits(1):
+                chosen_ids = [candidates[0], candidates[1]]
+            else:
+                chosen_ids = [candidates[1], candidates[0]]
+        else:
+            i = random.randrange(n)
+            j = random.randrange(n - 1)
+            if j >= i:
+                j += 1
+            chosen_ids = [candidates[i], candidates[j]]
+
+        return [[self._replicas[chosen_id] for chosen_id in chosen_ids]]

python/ray/serve/_private/request_router/replica_wrapper.py

@@ -0,0 +1,477 @@
+import asyncio
+import logging
+import pickle
+import uuid
+from abc import ABC, abstractmethod
+from dataclasses import dataclass, field
+from typing import Any, Dict, Optional, Set, Tuple
+
+import grpc
+
+import ray
+from ray.actor import ActorHandle
+from ray.serve._private.common import (
+    DeploymentID,
+    ReplicaID,
+    ReplicaQueueLengthInfo,
+    RequestMetadata,
+    RunningReplicaInfo,
+)
+from ray.serve._private.constants import (
+    RAY_SERVE_REPLICA_GRPC_MAX_MESSAGE_LENGTH,
+    SERVE_LOGGER_NAME,
+)
+from ray.serve._private.replica_result import (
+    ActorReplicaResult,
+    ReplicaResult,
+    gRPCReplicaResult,
+)
+from ray.serve._private.request_router.common import PendingRequest
+from ray.serve._private.serialization import RPCSerializer
+from ray.serve._private.utils import JavaActorHandleProxy
+from ray.serve.generated.serve_pb2 import (
+    ASGIRequest,
+    RequestMetadata as RequestMetadataProto,
+)
+from ray.serve.generated.serve_pb2_grpc import ASGIServiceStub
+from ray.util.annotations import PublicAPI
+from ray.util.tracing.tracing_helper import (
+    _DictPropagator,
+    _is_tracing_enabled,
+)
+
+logger = logging.getLogger(SERVE_LOGGER_NAME)
+
+
+class ReplicaWrapper(ABC):
+    """This is used to abstract away details of the transport layer
+    when communicating with the replica.
+    """
+
+    @abstractmethod
+    def send_request_java(self, pr: PendingRequest) -> ReplicaResult:
+        """Send request to Java replica."""
+        pass
+
+    @abstractmethod
+    def send_request_python(
+        self, pr: PendingRequest, *, with_rejection: bool
+    ) -> ReplicaResult:
+        """Send request to Python replica.
+
+        If sending request with rejection, the replica will yield a
+        system message (ReplicaQueueLengthInfo) before executing the
+        actual request. This can cause it to reject the request. The
+        result will *always* be a generator, so for non-streaming
+        requests it's up to the caller to resolve it to its first (and
+        only) ObjectRef.
+        """
+        pass
+
+
+class ActorReplicaWrapper(ReplicaWrapper):
+    def __init__(self, actor_handle):
+        self._actor_handle = actor_handle
+
+    def send_request_java(self, pr: PendingRequest) -> ActorReplicaResult:
+        """Send the request to a Java replica.
+        Does not currently support streaming.
+        """
+        if pr.metadata.is_streaming:
+            raise RuntimeError("Streaming not supported for Java.")
+
+        if len(pr.args) != 1:
+            raise ValueError("Java handle calls only support a single argument.")
+
+        return ActorReplicaResult(
+            self._actor_handle.handle_request.remote(
+                RequestMetadataProto(
+                    request_id=pr.metadata.request_id,
+                    # Default call method in java is "call," not "__call__" like Python.
+                    call_method="call"
+                    if pr.metadata.call_method == "__call__"
+                    else pr.metadata.call_method,
+                ).SerializeToString(),
+                pr.args,
+            ),
+            pr.metadata,
+        )
+
+    def send_request_python(
+        self, pr: PendingRequest, *, with_rejection: bool
+    ) -> ActorReplicaResult:
+        """Send the request to a Python replica."""
+        if with_rejection:
+            # Call a separate handler that may reject the request.
+            # This handler is *always* a streaming call and the first message will
+            # be a system message that accepts or rejects.
+            method = self._actor_handle.handle_request_with_rejection.options(
+                num_returns="streaming"
+            )
+        elif pr.metadata.is_streaming:
+            method = self._actor_handle.handle_request_streaming.options(
+                num_returns="streaming"
+            )
+        else:
+            method = self._actor_handle.handle_request
+
+        obj_ref_gen = method.remote(pickle.dumps(pr.metadata), *pr.args, **pr.kwargs)
+        return ActorReplicaResult(
+            obj_ref_gen, pr.metadata, with_rejection=with_rejection
+        )
+
+
+class gRPCReplicaWrapper(ReplicaWrapper):
+    def __init__(self, stub, actor_id):
+        self._stub = stub
+        self._actor_id = actor_id
+        self._loop = asyncio.get_running_loop()
+
+    def send_request_java(self, pr: PendingRequest):
+        raise RuntimeError("gRPC requests not supported for Java.")
+
+    def send_request_python(
+        self, pr: PendingRequest, *, with_rejection: bool
+    ) -> gRPCReplicaResult:
+        """Send the request to a Python replica."""
+
+        # Get serialization options from request metadata
+        request_serialization = pr.metadata.request_serialization
+        response_serialization = pr.metadata.response_serialization
+
+        # Get cached serializer for this request to avoid per-request instantiation overhead
+        serializer = RPCSerializer.get_cached_serializer(
+            request_serialization, response_serialization
+        )
+
+        # When using gRPC transport, requests go over the network rather than through
+        # Ray's actor RPC. Ray's tracing decorators inject _ray_trace_ctx for actor
+        # calls, but that doesn't apply here. We must manually inject the current
+        # trace context so it propagates to the replica (matching the actor path).
+        if _is_tracing_enabled():
+            pr.kwargs["_ray_trace_ctx"] = _DictPropagator.inject_current_context()
+
+        asgi_request = ASGIRequest(
+            pickled_request_metadata=pickle.dumps(pr.metadata),
+            request_args=serializer.dumps_request(pr.args),
+            request_kwargs=serializer.dumps_request(pr.kwargs),
+        )
+        if with_rejection and pr.metadata.is_streaming:
+            # Call a separate handler that may reject the request.
+            # This handler is *always* a streaming call and the first message will
+            # be a system message that accepts or rejects.
+            call = self._stub.HandleRequestWithRejectionStreaming(asgi_request)
+        elif with_rejection and not pr.metadata.is_streaming:
+            # Call a separate handler that may reject the request.
+            # This handler is *always* a unary call and the first message will
+            # be a system message that accepts or rejects.
+            call = self._stub.HandleRequestWithRejection(asgi_request)
+        elif pr.metadata.is_streaming:
+            call = self._stub.HandleRequestStreaming(asgi_request)
+        else:
+            call = self._stub.HandleRequest(asgi_request)
+
+        return gRPCReplicaResult(
+            call,
+            pr.metadata,
+            self._actor_id,
+            loop=self._loop,
+            with_rejection=with_rejection,
+        )
+
+
+@PublicAPI(stability="alpha")
+class RunningReplica:
+    """Contains info on a running replica.
+    Also defines the interface for a request router to talk to a replica.
+    """
+
+    def __init__(self, replica_info: RunningReplicaInfo):
+        self._replica_info = replica_info
+        self._multiplexed_model_ids = set(replica_info.multiplexed_model_ids)
+
+        # Fetch and cache the actor handle once per RunningReplica instance.
+        # This avoids the borrower-of-borrower pattern while minimizing GCS lookups.
+        actor_handle = replica_info.get_actor_handle()
+        if replica_info.is_cross_language:
+            self._actor_handle = JavaActorHandleProxy(actor_handle)
+        else:
+            self._actor_handle = actor_handle
+
+        # Lazily created
+        self._channel = None
+        self._stub = None
+
+        # Replica wrappers
+        self._actor_replica_wrapper = ActorReplicaWrapper(self._actor_handle)
+        self._grpc_replica_wrapper = None
+
+    def update_replica_info(self, replica_info: RunningReplicaInfo) -> None:
+        """Update mutable fields from a new RunningReplicaInfo.
+
+        Called when reusing an existing wrapper in _update_running_replicas.
+        Replicas dynamically load/unload models via record_multiplexed_model_ids,
+        which triggers a broadcast with updated RunningReplicaInfo. Without this
+        update, the router would use stale multiplexed_model_ids and break
+        multiplexed model routing.
+
+        Because we reassign _replica_info, any property that reads from it
+        (including max_ongoing_requests, node_id, availability_zone, etc.)
+        will reflect the new values. Fields that are cached separately
+        (e.g., _actor_handle) are NOT refreshed here because they are tied
+        to the replica's identity and should never change for a live replica.
+        """
+        self._replica_info = replica_info
+        self._multiplexed_model_ids = set(replica_info.multiplexed_model_ids)
+
+    @property
+    def replica_id(self) -> ReplicaID:
+        """ID of this replica."""
+        return self._replica_info.replica_id
+
+    @property
+    def actor_id(self) -> ray.ActorID:
+        """Actor ID of this replica."""
+        return self._actor_handle._actor_id
+
+    @property
+    def node_id(self) -> str:
+        """Node ID of the node this replica is running on."""
+        return self._replica_info.node_id
+
+    @property
+    def availability_zone(self) -> Optional[str]:
+        """Availability zone of the node this replica is running on."""
+        return self._replica_info.availability_zone
+
+    @property
+    def multiplexed_model_ids(self) -> Set[str]:
+        """Set of model IDs on this replica."""
+        return self._multiplexed_model_ids
+
+    @property
+    def routing_stats(self) -> Dict[str, Any]:
+        """Dictionary of routing stats."""
+        return self._replica_info.routing_stats
+
+    @property
+    def replica_metadata(self) -> Dict[str, Any]:
+        """Static per-replica metadata captured once when the replica became ready."""
+        # Return a copy so callers can't mutate the RunningReplicaInfo's dict.
+        return self._replica_info.replica_metadata.copy()
+
+    @property
+    def max_ongoing_requests(self) -> int:
+        """Max concurrent requests that can be sent to this replica."""
+        return self._replica_info.max_ongoing_requests
+
+    @property
+    def is_cross_language(self) -> bool:
+        """Whether this replica is cross-language (Java)."""
+        return self._replica_info.is_cross_language
+
+    @property
+    def backend_http_endpoint(self) -> Optional[Tuple[str, int]]:
+        """Return (host, port) of the replica's backend HTTP server."""
+        port = self._replica_info.backend_http_port
+        host = self._replica_info.node_ip
+        if host is not None and port is not None:
+            return (host, port)
+        return None
+
+    @property
+    def stub(self):
+        if self._stub is None:
+            self._channel = grpc.aio.insecure_channel(
+                f"{self._replica_info.node_ip}:{self._replica_info.port}",
+                options=[
+                    (
+                        "grpc.max_receive_message_length",
+                        RAY_SERVE_REPLICA_GRPC_MAX_MESSAGE_LENGTH,
+                    )
+                ],
+            )
+            self._stub = ASGIServiceStub(self._channel)
+
+        return self._stub
+
+    def _get_replica_wrapper(self, pr: PendingRequest) -> ReplicaWrapper:
+        if self._grpc_replica_wrapper is None:
+            self._grpc_replica_wrapper = gRPCReplicaWrapper(
+                self.stub, self._actor_handle._actor_id
+            )
+
+        return (
+            self._actor_replica_wrapper
+            if pr.metadata._by_reference
+            else self._grpc_replica_wrapper
+        )
+
+    def push_proxy_handle(self, handle: ActorHandle):
+        """When on proxy, push proxy's self handle to replica"""
+        self._actor_handle.push_proxy_handle.remote(handle)
+
+    async def get_queue_len(self, *, deadline_s: float) -> int:
+        """Returns current queue len for the replica.
+        `deadline_s` is passed to verify backoff for testing.
+        """
+        # NOTE(edoakes): the `get_num_ongoing_requests` method name is shared by
+        # the Python and Java replica implementations. If you change it, you need to
+        # change both (or introduce a branch here).
+        obj_ref = self._actor_handle.get_num_ongoing_requests.remote()
+        try:
+            return await obj_ref
+        except asyncio.CancelledError:
+            ray.cancel(obj_ref)
+            raise
+
+    def try_send_request(
+        self, pr: PendingRequest, with_rejection: bool
+    ) -> ReplicaResult:
+        """Try to send the request to this replica. It may be rejected."""
+        wrapper = self._get_replica_wrapper(pr)
+        if self._replica_info.is_cross_language:
+            assert not with_rejection, "Request rejection not supported for Java."
+            return wrapper.send_request_java(pr)
+
+        return wrapper.send_request_python(pr, with_rejection=with_rejection)
+
+    async def reserve_slot(
+        self, request_metadata: RequestMetadata
+    ) -> Tuple[str, ReplicaQueueLengthInfo]:
+        """Reserve a slot on this replica for an upcoming request.
+
+        Returns a unique token that can be used to release the slot later.
+        This is used in the choose_replica/dispatch pattern to track
+        reservations that haven't been dispatched yet.
+        """
+        if self._replica_info.is_cross_language:
+            raise RuntimeError("Slot reservation not supported for Java.")
+
+        slot_token = str(uuid.uuid4())
+        obj_ref = self._actor_handle.reserve_slot.remote(request_metadata, slot_token)
+        try:
+            accepted, num_ongoing_requests = await obj_ref
+        except asyncio.CancelledError:
+            ray.cancel(obj_ref)
+            self._actor_handle.release_slot.remote(slot_token)
+            raise
+        except Exception:
+            # The actor may have reserved the slot before the reply was lost
+            # (e.g. ActorUnavailableError). `release_slot` is idempotent for unknown
+            # tokens, so this is safe even when the reservation never actually happened.
+            self._actor_handle.release_slot.remote(slot_token)
+            raise
+
+        return slot_token, ReplicaQueueLengthInfo(
+            accepted=accepted,
+            num_ongoing_requests=num_ongoing_requests,
+        )
+
+    async def release_slot(self, slot_token: str) -> int:
+        """Release a previously reserved slot.
+
+        This should be called if a request is not dispatched after
+        reserving a slot (e.g., due to an error or cancellation).
+
+        Returns the replica's reported num_ongoing_requests after the release.
+        """
+        if self._replica_info.is_cross_language:
+            raise RuntimeError("Slot reservation not supported for Java.")
+
+        _, num_ongoing_requests = await self._actor_handle.release_slot.remote(
+            slot_token
+        )
+        return num_ongoing_requests
+
+
+@dataclass
+class ReplicaSelection:
+    """Represents a selected replica, holding information for dispatch or coordination.
+
+    This class is returned by the choose_replica() context manager.
+    The slot reservation lifecycle is managed by the context manager.
+    """
+
+    # Public, user-accessible fields
+    replica_id: str
+    """Unique identifier for the selected replica."""
+
+    node_ip: str
+    """IP address of the node running this replica."""
+
+    port: Optional[int]
+    """Port number for direct communication (if configured)."""
+
+    node_id: str
+    """Ray node ID where the replica is running."""
+
+    availability_zone: Optional[str]
+    """Cloud availability zone of the replica's node."""
+
+    replica_metadata: Dict[str, Any]
+    """Static, immutable per-replica metadata published by the deployment's
+    ``record_replica_metadata`` hook (captured once when the replica became
+    ready). Empty dict if the deployment does not define the hook."""
+
+    # Internal fields (not part of public API)
+    _replica: RunningReplica
+    _deployment_id: Optional[DeploymentID]
+    _request_metadata: RequestMetadata
+    _method_name: str
+    # Token to be used for replica reservation;
+    # Can be None when created via the pick-only path
+    _slot_token: Optional[str]
+    _dispatched: bool = field(
+        default=False, init=False
+    )  # Tracks if dispatch was called
+
+    # Set by dispatch once the result's done-callback is wired up. Read by
+    # choose_replica's finally to decide whether to fire on_request_completed
+    # manually (only one of the two paths should fire it).
+    _completion_callback_registered: bool = field(default=False, init=False)
+
+    @property
+    def address(self) -> str:
+        """Returns the replica address in host:port format."""
+        if self.port:
+            return f"{self.node_ip}:{self.port}"
+        return self.node_ip
+
+    def to_dict(self) -> Dict[str, Any]:
+        """Serialize public fields to a dictionary."""
+        return {
+            "replica_id": self.replica_id,
+            "node_ip": self.node_ip,
+            "port": self.port,
+            "node_id": self.node_id,
+            "availability_zone": self.availability_zone,
+            "replica_metadata": self.replica_metadata,
+        }
+
+    def _mark_dispatched(self) -> None:
+        """Internal: Mark this selection as dispatched (slot consumed).
+
+        Raises:
+            RuntimeError: If the selection has already been dispatched.
+        """
+        if self._dispatched:
+            raise RuntimeError(
+                f"ReplicaSelection for {self.replica_id} has already been dispatched. "
+                "Each selection can only be dispatched once."
+            )
+        self._dispatched = True
+
+    async def _release_slot(self, *, force: bool = False) -> Optional[int]:
+        """Internal: Release the reserved slot.
+
+        Returns the replica's reported num_ongoing_requests after the release,
+        or None if dispatch already consumed the slot (and ``force`` is False),
+        or None if this selection was created without a reservation.
+        """
+        if self._slot_token is None:
+            return
+        if self._dispatched and not force:
+            return
+
+        return await self._replica.release_slot(self._slot_token)

python/ray/serve/_private/rolling_window.py

@@ -0,0 +1,288 @@
+import threading
+import time
+from typing import List
+
+
+class _ThreadBuckets:
+    """Per-thread bucket storage for rolling window.
+
+    Each thread gets its own instance to avoid lock contention on the hot path.
+    """
+
+    # This is a performance optimization to avoid creating a dictionary for the instance.
+    __slots__ = ("buckets", "current_bucket_idx", "last_rotation_time")
+
+    def __init__(self, num_buckets: int):
+        self.buckets = [0.0] * num_buckets
+        self.current_bucket_idx = 0
+        self.last_rotation_time = time.time()
+
+
+class _ThreadLocalRef(threading.local):
+    """Thread-local reference to the thread's _ThreadBuckets instance."""
+
+    def __init__(self):
+        super().__init__()
+        # by using threading.local, each thread gets its own instance of _ThreadBuckets.
+        self.data: _ThreadBuckets = None
+
+
+class _RollingWindowBase:
+    """Base class for rolling window trackers.
+
+    Provides the shared infrastructure: bucketing, rotation, thread-local
+    storage, and thread registration. Subclasses define how values are
+    recorded into buckets and how buckets are aggregated.
+
+    Uses bucketing for memory efficiency - divides the window into N buckets
+    and rotates them as time passes. This allows efficient tracking of values
+    over a sliding window without storing individual data points.
+    """
+
+    def __init__(
+        self,
+        window_duration_s: float,
+        num_buckets: int = 60,
+    ):
+        if window_duration_s <= 0:
+            raise ValueError(
+                f"window_duration_s must be positive, got {window_duration_s}"
+            )
+        if num_buckets <= 0:
+            raise ValueError(f"num_buckets must be positive, got {num_buckets}")
+
+        self._window_duration_s = window_duration_s
+        self._num_buckets = num_buckets
+        self._bucket_duration_s = window_duration_s / num_buckets
+
+        # Thread-local reference to per-thread bucket data
+        self._local = _ThreadLocalRef()
+
+        # Track all per-thread bucket instances for aggregation
+        self._all_thread_data: List[_ThreadBuckets] = []
+        self._registry_lock = threading.Lock()
+
+    @property
+    def window_duration_s(self) -> float:
+        """The total duration of the rolling window in seconds."""
+        return self._window_duration_s
+
+    @property
+    def num_buckets(self) -> int:
+        """The number of buckets in the rolling window."""
+        return self._num_buckets
+
+    @property
+    def bucket_duration_s(self) -> float:
+        """The duration of each bucket in seconds."""
+        return self._bucket_duration_s
+
+    def _ensure_initialized(self) -> _ThreadBuckets:
+        """Ensure thread-local storage is initialized for the current thread.
+
+        This is called on every add() but the fast path (already initialized)
+        is just a single attribute check with no locking.
+
+        Returns:
+            The _ThreadBuckets instance for the current thread.
+        """
+        data = self._local.data
+        if data is not None:
+            return data
+
+        # Slow path: first call from this thread
+        data = _ThreadBuckets(self._num_buckets)
+        self._local.data = data
+
+        # Register for aggregation (only happens once per thread)
+        with self._registry_lock:
+            self._all_thread_data.append(data)
+
+        return data
+
+    def _rotate_buckets_if_needed(self, data: _ThreadBuckets) -> None:
+        """Rotate buckets for the given thread's storage.
+
+        Advances the current bucket index and clears old buckets as time passes.
+        """
+        now = time.time()
+        elapsed = now - data.last_rotation_time
+        buckets_to_advance = int(elapsed / self._bucket_duration_s)
+
+        if buckets_to_advance > 0:
+            if buckets_to_advance >= self._num_buckets:
+                # All buckets have expired, reset everything
+                data.buckets = [0.0] * self._num_buckets
+                data.current_bucket_idx = 0
+            else:
+                # Clear old buckets as we advance
+                for _ in range(buckets_to_advance):
+                    data.current_bucket_idx = (
+                        data.current_bucket_idx + 1
+                    ) % self._num_buckets
+                    data.buckets[data.current_bucket_idx] = 0.0
+
+            data.last_rotation_time = now
+
+    def get_num_registered_threads(self) -> int:
+        """Get the number of threads that have called add().
+
+        Useful for debugging and testing.
+
+        Returns:
+            The number of threads registered with this accumulator.
+        """
+        with self._registry_lock:
+            return len(self._all_thread_data)
+
+
+class RollingWindowAccumulator(_RollingWindowBase):
+    """Tracks cumulative values over a rolling time window.
+
+    Uses thread-local storage for lock-free writes on the hot path (add()).
+    Only get_total() requires synchronization to aggregate across threads.
+
+    Example:
+        # Create a 10-minute rolling window with 60 buckets (10s each)
+        accumulator = RollingWindowAccumulator(
+            window_duration_s=600.0,
+            num_buckets=60,
+        )
+
+        # Add values (lock-free, safe from multiple threads)
+        accumulator.add(100.0)
+        accumulator.add(50.0)
+
+        # Get total (aggregates across all threads)
+        total = accumulator.get_total()
+
+    Thread Safety:
+        - add() is lock-free after the first call from each thread
+        - get_total() acquires a lock to aggregate across threads
+        - Safe to call from multiple threads concurrently
+    """
+
+    def add(self, value: float) -> None:
+        """Add a value to the current bucket.
+
+        This operation is lock-free for the calling thread after the first call.
+        Safe to call from multiple threads concurrently.
+
+        Args:
+            value: The value to add to the accumulator.
+        """
+        # Fast path: just check if initialized (no lock)
+        data = self._ensure_initialized()
+
+        # Lock-free: only touches thread-local data
+        self._rotate_buckets_if_needed(data)
+        data.buckets[data.current_bucket_idx] += value
+
+    def get_total(self) -> float:
+        """Get total value across all buckets in the window.
+
+        This aggregates values from all threads that have called add().
+        Expired buckets (older than window_duration_s) are not included.
+
+        Note: We are accepting some inaccuracy in the total value to avoid the overhead of a lock.
+        This is acceptable because we are only using this for utilization metrics, which are not
+        critical for the overall system. Given that the default window duration is 600s and the
+        default report interval is 10s, the inaccuracy is less than 0.16%.
+
+        Returns:
+            The sum of all non-expired values in the rolling window.
+        """
+        total = 0.0
+        now = time.time()
+
+        with self._registry_lock:
+            for data in self._all_thread_data:
+                # Calculate which buckets are still valid for this thread's data
+                elapsed = now - data.last_rotation_time
+                buckets_expired = int(elapsed / self._bucket_duration_s)
+
+                if buckets_expired >= self._num_buckets:
+                    # All buckets have expired for this thread
+                    continue
+
+                # Sum buckets that haven't expired
+                # Buckets are arranged in a circular buffer, with current_bucket_idx
+                # being the most recent. We need to skip buckets that have expired.
+                for i in range(self._num_buckets - buckets_expired):
+                    # Go backwards from current bucket
+                    idx = (data.current_bucket_idx - i) % self._num_buckets
+                    total += data.buckets[idx]
+
+        return total
+
+
+class RollingWindowMax(_RollingWindowBase):
+    """Tracks the maximum value over a rolling time window.
+
+    Uses the same bucketed rolling window approach as RollingWindowAccumulator,
+    but each bucket stores the maximum observed value instead of a cumulative
+    sum. Querying returns the max across all non-expired buckets.
+
+    Example:
+        # Create a 30-second rolling window with 6 buckets (5s each)
+        tracker = RollingWindowMax(
+            window_duration_s=30.0,
+            num_buckets=6,
+        )
+
+        # Record values (lock-free, safe from multiple threads)
+        tracker.add(100.0)
+        tracker.add(500.0)
+        tracker.add(50.0)
+
+        # Get max in the window (aggregates across all threads)
+        maximum = tracker.get_max()  # returns 500.0
+
+    Thread Safety:
+        - add() is lock-free after the first call from each thread
+        - get_max() acquires a lock to aggregate across threads
+        - Safe to call from multiple threads concurrently
+    """
+
+    def add(self, value: float) -> None:
+        """Record a value, updating the current bucket's max if exceeded.
+
+        This operation is lock-free for the calling thread after the first call.
+        Safe to call from multiple threads concurrently.
+
+        Args:
+            value: The value to record.
+        """
+        data = self._ensure_initialized()
+
+        self._rotate_buckets_if_needed(data)
+        if value > data.buckets[data.current_bucket_idx]:
+            data.buckets[data.current_bucket_idx] = value
+
+    def get_max(self) -> float:
+        """Get max value across all non-expired buckets in the window.
+
+        This aggregates values from all threads that have called add().
+        Expired buckets (older than window_duration_s) are not included.
+
+        Returns:
+            The maximum value observed in the rolling window, or 0.0
+            if no values have been recorded.
+        """
+        result = 0.0
+        now = time.time()
+
+        with self._registry_lock:
+            for data in self._all_thread_data:
+                elapsed = now - data.last_rotation_time
+                buckets_expired = int(elapsed / self._bucket_duration_s)
+
+                if buckets_expired >= self._num_buckets:
+                    continue
+
+                for i in range(self._num_buckets - buckets_expired):
+                    idx = (data.current_bucket_idx - i) % self._num_buckets
+                    if data.buckets[idx] > result:
+                        result = data.buckets[idx]
+
+        return result

python/ray/serve/_private/serialization.py

@@ -0,0 +1,207 @@
+import logging
+import pickle
+from typing import Any, Dict, Tuple
+
+from ray import cloudpickle
+from ray.serve._private.constants import SERVE_LOGGER_NAME
+
+try:
+    import orjson
+except ImportError:
+    orjson = None
+try:
+    import ormsgpack
+except ImportError:
+    ormsgpack = None
+
+
+logger = logging.getLogger(SERVE_LOGGER_NAME)
+
+
+class SerializationMethod:
+    """Available serialization methods for RPC communication."""
+
+    CLOUDPICKLE = "cloudpickle"
+    PICKLE = "pickle"
+    MSGPACK = "msgpack"
+    ORJSON = "orjson"
+    NOOP = "noop"
+
+
+# Global cache for serializer instances to avoid per-request instantiation overhead
+_serializer_cache: Dict[Tuple[str, str], "RPCSerializer"] = {}
+
+
+class RPCSerializer:
+    """Serializer for RPC communication with configurable serialization methods."""
+
+    def __init__(
+        self,
+        request_method: str = SerializationMethod.CLOUDPICKLE,
+        response_method: str = SerializationMethod.CLOUDPICKLE,
+    ):
+        self.request_method = request_method.lower()
+        self.response_method = response_method.lower()
+        self._validate_methods()
+        self._setup_serializers()
+
+    @classmethod
+    def get_cached_serializer(
+        cls,
+        request_method: str = SerializationMethod.CLOUDPICKLE,
+        response_method: str = SerializationMethod.CLOUDPICKLE,
+    ) -> "RPCSerializer":
+        """Get a cached serializer instance to avoid per-request instantiation overhead.
+
+        This method maintains a cache of serializer instances based on
+        (request_method, response_method) pairs, significantly reducing overhead
+        in high-throughput systems.
+        """
+        # Normalize method names
+        req_method = request_method.lower()
+        resp_method = response_method.lower()
+        cache_key = (req_method, resp_method)
+
+        if cache_key not in _serializer_cache:
+            _serializer_cache[cache_key] = cls(req_method, resp_method)
+
+        return _serializer_cache[cache_key]
+
+    def _validate_methods(self):
+        """Validate that the serialization methods are supported."""
+        valid_methods = {
+            SerializationMethod.CLOUDPICKLE,
+            SerializationMethod.PICKLE,
+            SerializationMethod.MSGPACK,
+            SerializationMethod.ORJSON,
+            SerializationMethod.NOOP,
+        }
+
+        if self.request_method not in valid_methods:
+            raise ValueError(
+                f"Unsupported request serialization method: {self.request_method}. "
+                f"Valid options: {valid_methods}"
+            )
+
+        if self.response_method not in valid_methods:
+            raise ValueError(
+                f"Unsupported response serialization method: {self.response_method}. "
+                f"Valid options: {valid_methods}"
+            )
+
+    def _setup_serializers(self):
+        """Setup the serialization functions based on the selected methods."""
+        self._request_dumps, self._request_loads = self._get_serializer_funcs(
+            self.request_method
+        )
+        self._response_dumps, self._response_loads = self._get_serializer_funcs(
+            self.response_method
+        )
+
+    def _get_serializer_funcs(self, method: str) -> Tuple[Any, Any]:
+        """Get dumps and loads functions for a given serialization method."""
+        if method == SerializationMethod.CLOUDPICKLE:
+            return cloudpickle.dumps, cloudpickle.loads
+        elif method == SerializationMethod.PICKLE:
+            return self._get_pickle_funcs()
+        elif method == SerializationMethod.MSGPACK:
+            return self._get_msgpack_funcs()
+        elif method == SerializationMethod.ORJSON:
+            return self._get_orjson_funcs()
+        elif method == SerializationMethod.NOOP:
+            return self._get_noop_funcs()
+
+    def _get_noop_funcs(self) -> Tuple[Any, Any]:
+        """Get no-op serialization functions for binary data."""
+
+        def _noop_dumps(obj: Any) -> bytes:
+            if not isinstance(obj, bytes):
+                raise TypeError(
+                    f"a bytes-like object is required, got {type(obj).__name__}. "
+                    "Use a different serialization method for non-binary data."
+                )
+            return obj
+
+        def _noop_loads(data: bytes) -> Any:
+            if not isinstance(data, bytes):
+                raise TypeError(
+                    f"a bytes-like object is required, got {type(data).__name__}. "
+                    "Use a different serialization method for non-binary data."
+                )
+            return data
+
+        return _noop_dumps, _noop_loads
+
+    def _get_pickle_funcs(self) -> Tuple[Any, Any]:
+        """Get pickle serialization functions with highest protocol."""
+
+        def _pickle_dumps(obj: Any) -> bytes:
+            return pickle.dumps(obj, protocol=pickle.HIGHEST_PROTOCOL)
+
+        def _pickle_loads(data: bytes) -> Any:
+            return pickle.loads(data)
+
+        return _pickle_dumps, _pickle_loads
+
+    def _get_msgpack_funcs(self) -> Tuple[Any, Any]:
+        """Get msgpack serialization functions."""
+
+        if ormsgpack is None:
+            raise ImportError(
+                "ormsgpack is not installed. Please install it with `pip install ormsgpack`."
+            )
+
+        # Configure ormsgpack with appropriate options
+        def _msgpack_dumps(obj: Any) -> bytes:
+            return ormsgpack.packb(obj)
+
+        def _msgpack_loads(data: bytes) -> Any:
+            return ormsgpack.unpackb(data)
+
+        return _msgpack_dumps, _msgpack_loads
+
+    def _get_orjson_funcs(self) -> Tuple[Any, Any]:
+        """Get orjson serialization functions."""
+
+        if orjson is None:
+            raise ImportError(
+                "orjson is not installed. Please install it with `pip install orjson`."
+            )
+
+        # orjson only supports JSON-serializable types
+        def _orjson_dumps(obj: Any) -> bytes:
+            try:
+                return orjson.dumps(obj)
+            except TypeError as e:
+                raise TypeError(
+                    f"orjson serialization failed: {e}. "
+                    "Only JSON-serializable types are supported with orjson. "
+                    "Consider using 'cloudpickle' or 'pickle' for complex objects."
+                )
+
+        def _orjson_loads(data: bytes) -> Any:
+            return orjson.loads(data)
+
+        return _orjson_dumps, _orjson_loads
+
+    def dumps_request(self, obj: Any) -> bytes:
+        """Serialize a request object to bytes."""
+        return self._request_dumps(obj)
+
+    def loads_request(self, data: bytes) -> Any:
+        """Deserialize bytes to a request object."""
+        return self._request_loads(data)
+
+    def dumps_response(self, obj: Any) -> bytes:
+        """Serialize a response object to bytes."""
+        return self._response_dumps(obj)
+
+    def loads_response(self, data: bytes) -> Any:
+        """Deserialize bytes to a response object."""
+        return self._response_loads(data)
+
+
+def clear_serializer_cache():
+    """Clear the cached serializer instances. Useful for testing or memory management."""
+    global _serializer_cache
+    _serializer_cache.clear()

python/ray/serve/_private/storage/kv_store.py

@@ -0,0 +1,116 @@
+import logging
+from typing import Optional
+
+import ray
+import ray.serve._private.constants as serve_constants
+from ray._raylet import GcsClient
+from ray.serve._private.storage.kv_store_base import KVStoreBase
+
+logger = logging.getLogger(serve_constants.SERVE_LOGGER_NAME)
+
+SERVE_INTERNAL_KV_NAMESPACE = b"serve"
+
+
+def get_storage_key(namespace: str, storage_key: str) -> str:
+    """In case we need to access kvstore"""
+    return "{ns}-{key}".format(ns=namespace, key=storage_key)
+
+
+class KVStoreError(Exception):
+    def __init__(self, rpc_code):
+        self.rpc_code = rpc_code
+
+
+class RayInternalKVStore(KVStoreBase):
+    """Wraps ray's internal_kv with a namespace to avoid collisions.
+
+    Supports string keys and bytes values, caller must handle serialization.
+    """
+
+    def __init__(
+        self,
+        namespace: Optional[str] = None,
+        gcs_client: Optional[GcsClient] = None,
+    ):
+        if namespace is not None and not isinstance(namespace, str):
+            raise TypeError("namespace must a string, got: {}.".format(type(namespace)))
+        if gcs_client is not None:
+            self.gcs_client = gcs_client
+        else:
+            self.gcs_client = GcsClient(address=ray.get_runtime_context().gcs_address)
+        self.timeout = serve_constants.RAY_SERVE_KV_TIMEOUT_S
+        self.namespace = namespace or ""
+
+    def get_storage_key(self, key: str) -> str:
+        return "{ns}-{key}".format(ns=self.namespace, key=key)
+
+    def put(self, key: str, val: bytes) -> bool:
+        """Put the key-value pair into the store.
+
+        Args:
+            key: The key to store.
+            val: The value to store.
+
+        Returns:
+            True if the value was newly inserted, False if it overwrote an
+            existing value.
+        """
+        if not isinstance(key, str):
+            raise TypeError("key must be a string, got: {}.".format(type(key)))
+        if not isinstance(val, bytes):
+            raise TypeError("val must be bytes, got: {}.".format(type(val)))
+
+        try:
+            return self.gcs_client.internal_kv_put(
+                self.get_storage_key(key).encode(),
+                val,
+                overwrite=True,
+                namespace=SERVE_INTERNAL_KV_NAMESPACE,
+                timeout=self.timeout,
+            )
+        except ray.exceptions.RpcError as e:
+            raise KVStoreError(e.rpc_code)
+
+    def get(self, key: str) -> Optional[bytes]:
+        """Get the value associated with the given key from the store.
+
+        Args:
+            key: The key to retrieve.
+
+        Returns:
+            Optional[bytes]: The bytes value. If the key wasn't found, returns None.
+        """
+        if not isinstance(key, str):
+            raise TypeError("key must be a string, got: {}.".format(type(key)))
+
+        try:
+            return self.gcs_client.internal_kv_get(
+                self.get_storage_key(key).encode(),
+                namespace=SERVE_INTERNAL_KV_NAMESPACE,
+                timeout=self.timeout,
+            )
+        except ray.exceptions.RpcError as e:
+            raise KVStoreError(e.rpc_code)
+
+    def delete(self, key: str):
+        """Delete the value associated with the given key from the store.
+
+        Args:
+            key: The key to delete.
+
+        Returns:
+            The number of keys deleted (0 if the key did not exist).
+        """
+
+        if not isinstance(key, str):
+            raise TypeError("key must be a string, got: {}.".format(type(key)))
+
+        try:
+            return self.gcs_client.internal_kv_del(
+                self.get_storage_key(key).encode(),
+                False,
+                namespace=SERVE_INTERNAL_KV_NAMESPACE,
+                timeout=self.timeout,
+            )
+        except ray.exceptions.RpcError as e:
+            raise KVStoreError(e.rpc_code)

python/ray/serve/_private/storage/kv_store_base.py

@@ -0,0 +1,60 @@
+import abc
+from abc import abstractmethod
+from typing import Optional
+
+from ray.util.annotations import DeveloperAPI
+
+
+@DeveloperAPI
+class KVStoreBase(metaclass=abc.ABCMeta):
+    """Abstract class for KVStore defining APIs needed for ray serve
+    use cases, currently (8/6/2021) controller state checkpointing.
+    """
+
+    @abstractmethod
+    def get_storage_key(self, key: str) -> str:
+        """Get internal key for storage.
+
+        Args:
+            key: User provided key
+
+        Returns:
+            storage_key: Formatted key for storage, usually by
+                prepending namespace.
+        """
+        raise NotImplementedError("get_storage_key() has to be implemented")
+
+    @abstractmethod
+    def put(self, key: str, val: bytes) -> bool:
+        """Put object into kv store, bytes only.
+
+        Args:
+            key: Key for object to be stored.
+            val: Byte value of object.
+
+        Returns:
+            True if the value was newly inserted, False if it overwrote an
+            existing value.
+        """
+        raise NotImplementedError("put() has to be implemented")
+
+    @abstractmethod
+    def get(self, key: str) -> Optional[bytes]:
+        """Get object from storage.
+
+        Args:
+            key: Key for object to be retrieved.
+
+        Returns:
+            val: Byte value of object from storage.
+        """
+        raise NotImplementedError("get() has to be implemented")
+
+    @abstractmethod
+    def delete(self, key: str) -> None:
+        """Delete an object.
+
+        Args:
+            key: Key for object to be deleted.
+        """
+        raise NotImplementedError("delete() has to be implemented")

python/ray/serve/_private/task_consumer.py

@@ -0,0 +1,12 @@
+from abc import ABC
+
+
+class TaskConsumerWrapper(ABC):
+    def __init__(self, *args, **kwargs):
+        pass
+
+    def initialize_callable(self, consumer_concurrency: int):
+        pass
+
+    def __del__(self):
+        pass

python/ray/serve/_private/thirdparty/get_asgi_route_name.py

@@ -0,0 +1,241 @@
+# This code is adapted from:
+# https://github.com/elastic/apm-agent-python/blob/f570e8c2b68a8714628acac815aebcc3518b44c7/elasticapm/contrib/starlette/__init__.py
+#
+#  BSD 3-Clause License
+#
+#  Copyright (c) 2012, the Sentry Team, see AUTHORS for more details
+#  Copyright (c) 2019, Elasticsearch BV
+#  All rights reserved.
+#
+#  Redistribution and use in source and binary forms, with or without
+#  modification, are permitted provided that the following conditions are met:
+#
+#  * Redistributions of source code must retain the above copyright notice, this
+#    list of conditions and the following disclaimer.
+#
+#  * Redistributions in binary form must reproduce the above copyright notice,
+#    this list of conditions and the following disclaimer in the documentation
+#    and/or other materials provided with the distribution.
+#
+#  * Neither the name of the copyright holder nor the names of its
+#    contributors may be used to endorse or promote products derived from
+#    this software without specific prior written permission.
+#
+#  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+#  AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+#  IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+#  DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+#  FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+#  DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+#  SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+#  CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+#  OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+
+from dataclasses import dataclass
+from typing import Dict, List, Optional, Set
+
+from starlette.routing import Match, Mount, Route
+from starlette.types import ASGIApp, Scope
+
+try:
+    # `_IncludedRouter` only exists on FastAPI >= 0.137, where routes registered
+    # via `include_router` are nested under it instead of being flattened into
+    # the parent's `routes` list. It is `None` on older versions.
+    from fastapi.routing import _IncludedRouter
+except ImportError:  # FastAPI < 0.137
+    _IncludedRouter = None
+
+
+def _included_router_prefix(route: "Route") -> str:
+    """Return the URL prefix an ``_IncludedRouter`` node applies to its routes.
+
+    Accesses ``_IncludedRouter``'s private attributes directly (rather than
+    defensively) so that a structural change in a future FastAPI version raises
+    loudly instead of silently returning a wrong route name. Both callers are
+    wrapped in ``try``/``except`` that fall back to the route prefix and log the
+    error, so a failure here degrades gracefully while staying visible.
+    """
+    return route.include_context.prefix
+
+
+@dataclass(frozen=True)
+class RoutePattern:
+    """Represents a route pattern with optional HTTP method restrictions.
+
+    Attributes:
+        methods: List of HTTP methods (e.g., ["GET", "POST"]), or None if the route
+                 accepts all methods (e.g., WebSocket routes, ASGI apps).
+        path: The route path pattern (e.g., "/", "/users/{user_id}").
+    """
+
+    methods: Optional[List[str]]
+    path: str
+
+
+def _get_route_name(
+    scope: Scope, routes: List[Route], *, route_name: Optional[str] = None
+) -> Optional[str]:
+    for route in routes:
+        if _IncludedRouter is not None and isinstance(route, _IncludedRouter):
+            # FastAPI >= 0.137: routes added via `include_router` live on
+            # `route.original_router.routes` with paths relative to the include
+            # prefix. Unlike `Mount`, `_IncludedRouter.matches()` doesn't return
+            # a prefix-stripped child scope, so we strip it ourselves and prepend
+            # the prefix back to the resolved child name. This mirrors the Mount
+            # handling below and works for both HTTP and WebSocket routes.
+            prefix = _included_router_prefix(route)
+            path = scope.get("path", "")
+            if prefix:
+                if not path.startswith(prefix):
+                    continue
+                child_scope = {**scope, "path": path[len(prefix) :]}
+            else:
+                child_scope = scope
+            child_route_name = _get_route_name(
+                child_scope, route.original_router.routes
+            )
+            if child_route_name is not None:
+                return prefix + child_route_name
+            continue
+
+        match, child_scope = route.matches(scope)
+        if match == Match.FULL:
+            route_name = route.path
+            child_scope = {**scope, **child_scope}
+            if isinstance(route, Mount) and route.routes:
+                child_route_name = _get_route_name(
+                    child_scope, route.routes, route_name=route_name
+                )
+                if child_route_name is None:
+                    route_name = None
+                else:
+                    route_name += child_route_name
+            return route_name
+        elif match == Match.PARTIAL and route_name is None:
+            route_name = route.path
+
+    return None
+
+
+def get_asgi_route_name(app: ASGIApp, scope: Scope) -> Optional[str]:
+    """Gets route name for given request taking mounts into account."""
+
+    routes = app.routes
+    route_name = _get_route_name(scope, routes)
+
+    # Starlette magically redirects requests if the path matches a route name
+    # with a trailing slash appended or removed. To not spam the transaction
+    # names list, we do the same here and put these redirects all in the
+    # same "redirect trailing slashes" transaction name.
+    if not route_name and app.router.redirect_slashes and scope["path"] != "/":
+        redirect_scope = dict(scope)
+        if scope["path"].endswith("/"):
+            redirect_scope["path"] = scope["path"][:-1]
+            trim = True
+        else:
+            redirect_scope["path"] = scope["path"] + "/"
+            trim = False
+
+        route_name = _get_route_name(redirect_scope, routes)
+        if route_name is not None:
+            route_name = route_name + "/" if trim else route_name[:-1]
+
+    if route_name:
+        root_path = scope.get("root_path", "")
+        if root_path:
+            route_name = root_path.rstrip("/") + "/" + route_name.lstrip("/")
+
+    return route_name
+
+
+def extract_route_patterns(app: ASGIApp) -> List[RoutePattern]:
+    """Extracts all route patterns from an ASGI app.
+
+    This function recursively traverses the app's routes (including mounted apps)
+    and returns a list of all route patterns. This is used to communicate available
+    routes from build time to proxies for accurate metrics tagging.
+
+    Args:
+        app: The ASGI application (typically FastAPI or Starlette)
+
+    Returns:
+        List of RoutePattern objects. Examples:
+        - RoutePattern(methods=["GET", "POST"], path="/"): GET and POST to root
+        - RoutePattern(methods=["GET"], path="/users/{id}"): GET to users endpoint
+        - RoutePattern(methods=None, path="/websocket"): No method restrictions
+    """
+    # Use a dict to store path -> set of methods mapping
+    # This allows us to track which methods apply to each path
+    # Use None as a sentinel value to indicate "no method restrictions"
+    path_methods: Dict[str, Optional[Set[str]]] = {}
+
+    def _extract_from_routes(routes: List[Route], prefix: str = "") -> None:
+        for route in routes:
+            if _IncludedRouter is not None and isinstance(route, _IncludedRouter):
+                # FastAPI >= 0.137: recurse into routes registered via
+                # `include_router`, applying the include prefix (see
+                # `_get_route_name` for details).
+                _extract_from_routes(
+                    route.original_router.routes,
+                    prefix + _included_router_prefix(route),
+                )
+                continue
+
+            route_path = prefix + route.path
+
+            if isinstance(route, Mount):
+                # Recursively extract patterns from mounted apps
+                if hasattr(route, "routes") and route.routes:
+                    _extract_from_routes(route.routes, route_path)
+                else:
+                    # Mount without sub-routes - no method restrictions
+                    if route_path not in path_methods:
+                        path_methods[route_path] = None
+            else:
+                # Regular route - extract methods if available
+                if hasattr(route, "methods") and route.methods:
+                    # Route has specific methods
+                    if route_path not in path_methods:
+                        path_methods[route_path] = set()
+                    # Only add methods if we haven't already marked this path as "all methods"
+                    if path_methods[route_path] is not None:
+                        path_methods[route_path].update(route.methods)
+                else:
+                    # Route has no method restrictions (accepts all methods)
+                    # Mark this path as accepting all methods (None)
+                    path_methods[route_path] = None
+
+    try:
+        if hasattr(app, "routes"):
+            _extract_from_routes(app.routes)
+
+            # Handle root_path if present
+            if hasattr(app, "root_path") and app.root_path:
+                root_path = app.root_path.rstrip("/")
+                adjusted_path_methods = {}
+                for path, methods in path_methods.items():
+                    adjusted_path = (
+                        root_path + "/" + path.lstrip("/")
+                        if path != "/"
+                        else root_path + path
+                    )
+                    adjusted_path_methods[adjusted_path] = methods
+                path_methods = adjusted_path_methods
+    except Exception:
+        # If extraction fails for any reason, return empty list
+        # This shouldn't break the system
+        return []
+
+    # Convert path_methods dict to list of RoutePattern objects
+    patterns: List[RoutePattern] = []
+    for path, methods in path_methods.items():
+        if methods is None:
+            # No method restrictions
+            patterns.append(RoutePattern(methods=None, path=path))
+        else:
+            # Convert set to sorted list for consistent ordering
+            methods_list = sorted(methods)
+            patterns.append(RoutePattern(methods=methods_list, path=path))
+
+    # Sort by path for consistent ordering
+    return sorted(patterns, key=lambda x: x.path)

python/ray/serve/_private/tracing_utils.py

@@ -0,0 +1,509 @@
+import inspect
+import os
+import threading
+from contextvars import ContextVar, Token
+from functools import wraps
+from typing import Any, Callable, Dict, List, Optional
+
+from ray._common.utils import import_attr
+from ray.serve._private.constants import (
+    DEFAULT_TRACING_EXPORTER_IMPORT_PATH,
+    RAY_SERVE_TRACING_EXPORTER_IMPORT_PATH,
+    RAY_SERVE_TRACING_SAMPLING_RATIO,
+)
+
+try:
+    from opentelemetry import trace
+    from opentelemetry.context import attach, detach, get_current
+    from opentelemetry.sdk.trace import SpanProcessor, TracerProvider
+    from opentelemetry.sdk.trace.export import ConsoleSpanExporter, SimpleSpanProcessor
+    from opentelemetry.sdk.trace.sampling import ParentBasedTraceIdRatio
+    from opentelemetry.semconv.trace import SpanAttributes
+    from opentelemetry.trace import SpanKind
+    from opentelemetry.trace.propagation import set_span_in_context
+    from opentelemetry.trace.propagation.tracecontext import (
+        TraceContextTextMapPropagator,
+    )
+    from opentelemetry.trace.status import Status, StatusCode
+
+except ImportError:
+    SpanProcessor = None
+    ConsoleSpanExporter = None
+    SimpleSpanProcessor = None
+    trace = None
+    SpanKind = None
+    TracerProvider = None
+    TraceIdRatioBased = None
+    Status = None
+    StatusCode = None
+    set_span_in_context = None
+    TraceContextTextMapPropagator = None
+    get_current = None
+    attach = None
+    detach = None
+    SpanAttributes = None
+    ParentBasedTraceIdRatio = None
+
+
+TRACE_STACK: ContextVar[List[Any]] = ContextVar(
+    "trace_stack"
+)  # Create tracer once at module level
+
+_tracer = None
+_tracer_lock = threading.Lock()
+
+
+def get_tracer():
+    global _tracer
+    if _tracer is None:
+        with _tracer_lock:
+            if _tracer is None:
+                _tracer = trace.get_tracer(__name__)
+    return _tracer
+
+
+# Default tracing exporter needs to map to DEFAULT_TRACING_EXPORTER_IMPORT_PATH
+# defined in "python/ray/serve/_private/constants.py"
+def default_tracing_exporter(tracing_file_name):
+    from ray.serve._private.logging_utils import get_serve_logs_dir
+
+    serve_logs_dir = get_serve_logs_dir()
+    spans_dir = os.path.join(serve_logs_dir, "spans")
+    os.makedirs(spans_dir, exist_ok=True)
+    spans_file = os.path.join(spans_dir, tracing_file_name)
+
+    out_file = open(spans_file, "a")
+
+    class FileConsoleSpanExporter(ConsoleSpanExporter):
+        def shutdown(self):
+            if not out_file.closed:
+                out_file.flush()
+                out_file.close()
+
+    return [SimpleSpanProcessor(FileConsoleSpanExporter(out=out_file))]
+
+
+class TraceContextManager:
+    def __init__(
+        self, trace_name, span_kind=None, trace_context: Optional[Dict[str, str]] = None
+    ):
+        self.span = None
+        self.trace_name = trace_name
+        self.span_kind = span_kind
+        self.trace_context = trace_context
+
+        self.is_tracing_enabled = is_tracing_enabled()
+
+    def __enter__(self):
+        if self.is_tracing_enabled:
+            self.span_kind = self.span_kind or SpanKind.SERVER
+
+            tracer = get_tracer()
+            ctx = self.trace_context if self.trace_context else get_trace_context()
+
+            self.span = tracer.start_span(
+                self.trace_name,
+                kind=self.span_kind,
+                context=ctx,
+            )
+            if not self.span.get_span_context().trace_flags.sampled:
+                return self
+            new_ctx = set_span_in_context(self.span)
+            set_trace_context(new_ctx)
+            _append_trace_stack(self.span)
+            set_span_name(self.trace_name)
+
+        return self
+
+    def __exit__(self, exc_type, exc_value, traceback):
+        if self.is_tracing_enabled and self.span is not None:
+            # if exc_type is not None, we have made a explicit decision
+            # to not set the span status to error. This is because
+            # errors are spans internal to Ray Serve and should not
+            # be reported as errors in the trace. They cause noise
+            # in the trace and are not meaningful to the user.
+            self.span.end()
+            _pop_trace_stack()
+
+        return False
+
+
+class BatchTraceContextManager:
+    """Attach/detach a tracing context around a block to scope the span of a batch."""
+
+    def __init__(self, trace_context: Optional[object]):
+        self._enabled = is_tracing_enabled() and trace_context is not None
+        self._trace_context = trace_context
+        self._token: Optional[Token] = None
+
+    def __enter__(self):
+        if self._enabled:
+            self._token = set_trace_context(self._trace_context)
+
+        return self
+
+    def __exit__(self, exc_type, exc, tb):
+        if self._enabled and self._token is not None:
+            detach_trace_context(self._token)
+        return False
+
+
+def tracing_decorator_factory(
+    trace_name: str, span_kind: Optional[Any] = None
+) -> Callable:
+    """
+    Factory function to create a tracing decorator for instrumenting functions/methods
+    with distributed tracing.
+
+    Args:
+        trace_name: The name of the trace.
+        span_kind: The kind of span to create
+            (e.g., SERVER, CLIENT). Defaults to trace.SpanKind.SERVER.
+
+    Returns:
+        Callable: A decorator function that can be used to wrap
+            functions/methods with distributed tracing.
+
+    Example Usage:
+    ```python
+    @tracing_decorator_factory(
+        "my_trace",
+        span_kind=trace.SpanKind.CLIENT,
+    )
+    def my_function(obj):
+        # Function implementation
+    ```
+    """
+
+    def tracing_decorator(func):
+        if not is_tracing_enabled():
+            # if tracing is not enabled, we don't want to wrap the function
+            # with the tracing decorator.
+            return func
+
+        @wraps(func)
+        def synchronous_wrapper(*args, **kwargs):
+            with TraceContextManager(trace_name, span_kind):
+                result = func(*args, **kwargs)
+
+            return result
+
+        @wraps(func)
+        def generator_wrapper(*args, **kwargs):
+            with TraceContextManager(trace_name, span_kind):
+                for item in func(*args, **kwargs):
+                    yield item
+
+        @wraps(func)
+        async def asynchronous_wrapper(*args, **kwargs):
+            with TraceContextManager(trace_name, span_kind):
+                result = await func(*args, **kwargs)
+            return result
+
+        @wraps(func)
+        async def asyc_generator_wrapper(*args, **kwargs):
+            with TraceContextManager(trace_name, span_kind):
+                async for item in func(*args, **kwargs):
+                    yield item
+
+        is_generator = _is_generator_function(func)
+        is_async = _is_async_function(func)
+        if is_generator and is_async:
+            return asyc_generator_wrapper
+        elif is_async:
+            return asynchronous_wrapper
+        elif is_generator:
+            return generator_wrapper
+        else:
+            return synchronous_wrapper
+
+    return tracing_decorator
+
+
+def setup_tracing(
+    component_name: str,
+    component_id: str,
+    component_type: Optional["ServeComponentType"] = None,  # noqa: F821
+    tracing_exporter_import_path: Optional[
+        str
+    ] = RAY_SERVE_TRACING_EXPORTER_IMPORT_PATH,
+    tracing_sampling_ratio: Optional[float] = RAY_SERVE_TRACING_SAMPLING_RATIO,
+) -> bool:
+    """
+    Set up tracing for a specific Serve component.
+
+    Args:
+        component_name: The name of the component.
+        component_id: The unique identifier of the component.
+        component_type: The type of the component.
+        tracing_exporter_import_path: Path to tracing exporter function.
+        tracing_sampling_ratio: Sampling ratio for traces (0.0 to 1.0).
+
+    Returns:
+        bool: True if tracing setup is successful, False otherwise.
+    """
+    if tracing_exporter_import_path == "":
+        return False
+
+    # Check dependencies
+    if not trace:
+        raise ImportError(
+            "You must `pip install opentelemetry-api` and "
+            "`pip install opentelemetry-sdk` "
+            "to enable tracing on Ray Serve."
+        )
+
+    from ray.serve._private.utils import get_component_file_name
+
+    tracing_file_name = get_component_file_name(
+        component_name=component_name,
+        component_id=component_id,
+        component_type=component_type,
+        suffix="_tracing.json",
+    )
+
+    span_processors = _load_span_processors(
+        tracing_exporter_import_path, tracing_file_name
+    )
+
+    # Intialize tracing
+    # Sets the tracer_provider. This is only allowed once~ per execution
+    # context and will log a warning if attempted multiple times.
+    # use ParentBasedTraceIdRatio to respect the parent span's sampling decision
+    # and sample probabilistically based on the tracing_sampling_ratio
+    sampler = ParentBasedTraceIdRatio(tracing_sampling_ratio)
+
+    trace.set_tracer_provider(TracerProvider(sampler=sampler))
+
+    for span_processor in span_processors:
+        trace.get_tracer_provider().add_span_processor(span_processor)
+
+    return True
+
+
+def create_propagated_context() -> Dict[str, str]:
+    """Create context that can be used across services and processes.
+
+    This function retrieves the current context and converts it
+    into a dictionary that can be used across actors and tasks since
+    it is serializable.
+
+    Returns:
+    - Trace Context Propagator (dict or None): A dictionary containing the propagated
+    trace context if available, otherwise None.
+    """
+    trace_context = get_trace_context()
+    if trace_context and TraceContextTextMapPropagator:
+        ctx = {}
+        TraceContextTextMapPropagator().inject(ctx, trace_context)
+        return ctx
+
+    return None
+
+
+def extract_propagated_context(
+    propagated_context: Optional[Dict[str, str]] = None
+) -> Optional[Dict[str, str]]:
+    """Extract the trace context from a Trace Context Propagator."""
+    if is_tracing_enabled() and propagated_context and TraceContextTextMapPropagator:
+        return TraceContextTextMapPropagator().extract(carrier=propagated_context)
+
+    return None
+
+
+def set_trace_context(trace_context: Dict[str, str]) -> Optional[Token]:
+    """Set the current trace context."""
+    if attach is None:
+        return
+
+    return attach(trace_context)
+
+
+def detach_trace_context(token: Token):
+    """Detach the current trace context corresponding to the token."""
+    if detach is None:
+        return
+
+    detach(token)
+
+
+def get_trace_context() -> Optional[Dict[str, str]]:
+    """Retrieve the current trace context."""
+    if get_current is None:
+        return None
+
+    context = get_current()
+    return context if context else None
+
+
+def set_span_name(name: str):
+    """Set the name for the current span in context."""
+    if TRACE_STACK:
+        trace_stack = TRACE_STACK.get([])
+        if trace_stack:
+            trace_stack[-1].update_name(name)
+    # this is added specifically for Datadog tracing.
+    # See https://docs.datadoghq.com/tracing/guide/configuring-primary-operation/#opentracing
+    set_span_attributes({"resource.name": name})
+
+
+def set_rpc_span_attributes(
+    system: str = "grpc",
+    method: Optional[str] = None,
+    status_code: Optional[str] = None,
+    service: Optional[str] = None,
+):
+    """
+    Use this function to set attributes for RPC spans.
+    Only include attributes that are in the OpenTelemetry
+    RPC span attributes spec https://opentelemetry.io/docs/specs/semconv/attributes-registry/rpc/.
+    """
+    if not is_tracing_enabled():
+        return
+    attributes = {
+        SpanAttributes.RPC_SYSTEM: system,
+        SpanAttributes.RPC_METHOD: method,
+        SpanAttributes.RPC_GRPC_STATUS_CODE: status_code,
+        SpanAttributes.RPC_SERVICE: service,
+    }
+    set_span_attributes(attributes)
+
+
+def set_http_span_attributes(
+    method: Optional[str] = None,
+    status_code: Optional[str] = None,
+    route: Optional[str] = None,
+):
+    """
+    Use this function to set attributes for HTTP spans.
+    Only include attributes that are in the OpenTelemetry
+    HTTP span attributes spec https://opentelemetry.io/docs/specs/semconv/attributes-registry/http/.
+    """
+    if not is_tracing_enabled():
+        return
+    attributes = {
+        SpanAttributes.HTTP_METHOD: method,
+        SpanAttributes.HTTP_STATUS_CODE: status_code,
+        SpanAttributes.HTTP_ROUTE: route,
+    }
+    set_span_attributes(attributes)
+
+
+def set_span_attributes(attributes: Dict[str, Any]):
+    """Set attributes for the current span in context."""
+    if TRACE_STACK:
+        trace_stack = TRACE_STACK.get([])
+        if trace_stack:
+            # filter attribute values that are None, otherwise they
+            # will show up as warning logs on the console.
+            attributes = {k: v for k, v in attributes.items() if v is not None}
+            trace_stack[-1].set_attributes(attributes)
+
+
+def set_trace_status(is_error: bool, description: str = ""):
+    """Set the status for the current span in context."""
+    trace_stack = TRACE_STACK.get([])
+    if trace_stack:
+        if is_error:
+            status_code = StatusCode.ERROR
+        else:
+            status_code = StatusCode.OK
+            description = None
+        trace_stack[-1].set_status(
+            Status(status_code=status_code, description=description)
+        )
+
+
+def set_span_exception(exc: Exception, escaped: bool = False):
+    """Set the exception for the current span in context."""
+    trace_stack = TRACE_STACK.get([])
+    if trace_stack:
+        trace_stack[-1].record_exception(exc, escaped=escaped)
+
+
+def is_tracing_enabled() -> bool:
+    return RAY_SERVE_TRACING_EXPORTER_IMPORT_PATH != "" and trace is not None
+
+
+def is_span_recording() -> bool:
+    if TRACE_STACK:
+        trace_stack = TRACE_STACK.get([])
+        if trace_stack:
+            return True
+    return False
+
+
+def _append_trace_stack(span):
+    """Append span to global trace stack."""
+    trace_stack = TRACE_STACK.get([])
+    trace_stack.append(span)
+    TRACE_STACK.set(trace_stack)
+
+
+def _pop_trace_stack():
+    """Pop span to global trace stack."""
+    trace_stack = TRACE_STACK.get([])
+    if trace_stack:
+        trace_stack.pop()
+        TRACE_STACK.set(trace_stack)
+
+
+def _validate_tracing_exporter(func: Callable) -> None:
+    """Validate that the custom tracing exporter
+    is a function that takes no arguments.
+    """
+    if inspect.isfunction(func) is False:
+        raise TypeError("Tracing exporter must be a function.")
+
+    signature = inspect.signature(func)
+
+    if len(signature.parameters) != 0:
+        raise TypeError("Tracing exporter cannot take any arguments.")
+
+
+def _validate_tracing_exporter_processors(span_processors: List[Any]):
+    """Validate that the output of a custom tracing exporter
+    returns type List[SpanProcessor].
+    """
+    if not isinstance(span_processors, list):
+        raise TypeError(
+            "Output of tracing exporter needs to be of type "
+            f"List[SpanProcessor], but received type {type(span_processors)}."
+        )
+
+    for span_processor in span_processors:
+        if not isinstance(span_processor, SpanProcessor):
+            raise TypeError(
+                "Output of tracing exporter needs to be of "
+                "type List[SpanProcessor], "
+                f"but received type {type(span_processor)}."
+            )
+
+
+def _load_span_processors(
+    tracing_exporter_import_path: str,
+    tracing_file_name: str,
+):
+    """Load span processors from a custome tracing
+    exporter function.
+    """
+    tracing_exporter_def = import_attr(tracing_exporter_import_path)
+
+    if tracing_exporter_import_path == DEFAULT_TRACING_EXPORTER_IMPORT_PATH:
+        return tracing_exporter_def(tracing_file_name)
+    else:
+        # Validate tracing exporter function
+        _validate_tracing_exporter(tracing_exporter_def)
+        # Validate tracing exporter processors
+        span_processors = tracing_exporter_def()
+        _validate_tracing_exporter_processors(span_processors)
+
+    return span_processors
+
+
+def _is_generator_function(func):
+    return inspect.isgeneratorfunction(func) or inspect.isasyncgenfunction(func)
+
+
+def _is_async_function(func):
+    return inspect.iscoroutinefunction(func) or inspect.isasyncgenfunction(func)

python/ray/serve/_private/usage.py

@@ -0,0 +1,55 @@
+from enum import Enum
+from typing import Dict, Optional
+
+from ray._common.usage.usage_lib import TagKey, record_extra_usage_tag
+
+
+class ServeUsageTag(Enum):
+    API_VERSION = TagKey.SERVE_API_VERSION
+    NUM_DEPLOYMENTS = TagKey.SERVE_NUM_DEPLOYMENTS
+    GCS_STORAGE = TagKey.GCS_STORAGE
+    NUM_GPU_DEPLOYMENTS = TagKey.SERVE_NUM_GPU_DEPLOYMENTS
+    FASTAPI_USED = TagKey.SERVE_FASTAPI_USED
+    DAG_DRIVER_USED = TagKey.SERVE_DAG_DRIVER_USED
+    HTTP_ADAPTER_USED = TagKey.SERVE_HTTP_ADAPTER_USED
+    GRPC_INGRESS_USED = TagKey.SERVE_GRPC_INGRESS_USED
+    REST_API_VERSION = TagKey.SERVE_REST_API_VERSION
+    NUM_APPS = TagKey.SERVE_NUM_APPS
+    NUM_REPLICAS_LIGHTWEIGHT_UPDATED = TagKey.SERVE_NUM_REPLICAS_LIGHTWEIGHT_UPDATED
+    USER_CONFIG_LIGHTWEIGHT_UPDATED = TagKey.SERVE_USER_CONFIG_LIGHTWEIGHT_UPDATED
+    AUTOSCALING_CONFIG_LIGHTWEIGHT_UPDATED = (
+        TagKey.SERVE_AUTOSCALING_CONFIG_LIGHTWEIGHT_UPDATED
+    )
+    DEPLOYMENT_HANDLE_API_USED = TagKey.SERVE_DEPLOYMENT_HANDLE_API_USED
+    DEPLOYMENT_HANDLE_TO_OBJECT_REF_API_USED = (
+        TagKey.SERVE_DEPLOYMENT_HANDLE_TO_OBJECT_REF_API_USED
+    )
+    MULTIPLEXED_API_USED = TagKey.SERVE_MULTIPLEXED_API_USED
+    HTTP_PROXY_USED = TagKey.SERVE_HTTP_PROXY_USED
+    GRPC_PROXY_USED = TagKey.SERVE_GRPC_PROXY_USED
+    SERVE_STATUS_API_USED = TagKey.SERVE_STATUS_API_USED
+    SERVE_GET_APP_HANDLE_API_USED = TagKey.SERVE_GET_APP_HANDLE_API_USED
+    SERVE_GET_DEPLOYMENT_HANDLE_API_USED = TagKey.SERVE_GET_DEPLOYMENT_HANDLE_API_USED
+    APP_CONTAINER_RUNTIME_ENV_USED = TagKey.SERVE_APP_CONTAINER_RUNTIME_ENV_USED
+    DEPLOYMENT_CONTAINER_RUNTIME_ENV_USED = (
+        TagKey.SERVE_DEPLOYMENT_CONTAINER_RUNTIME_ENV_USED
+    )
+    NUM_NODE_COMPACTIONS = TagKey.SERVE_NUM_NODE_COMPACTIONS
+    AUTO_NUM_REPLICAS_USED = TagKey.SERVE_AUTO_NUM_REPLICAS_USED
+    CUSTOM_REQUEST_ROUTER_USED = TagKey.SERVE_CUSTOM_REQUEST_ROUTER_USED
+    NUM_REPLICAS_VIA_API_CALL_UPDATED = TagKey.SERVE_NUM_REPLICAS_VIA_API_CALL_UPDATED
+    NUM_REPLICAS_USING_ASYNCHRONOUS_INFERENCE = (
+        TagKey.SERVE_NUM_REPLICAS_USING_ASYNCHRONOUS_INFERENCE
+    )
+    CUSTOM_AUTOSCALING_POLICY_USED = TagKey.SERVE_CUSTOM_AUTOSCALING_POLICY_USED
+
+    def record(self, value: str):
+        """Record telemetry value."""
+        record_extra_usage_tag(self.value, value)
+
+    def get_value_from_report(self, report: Dict) -> Optional[str]:
+        """Returns `None` if the tag isn't in the report."""
+        if "extra_usage_tags" not in report:
+            return None
+
+        return report["extra_usage_tags"].get(TagKey.Name(self.value).lower(), None)

python/ray/serve/_private/utils.py

@@ -0,0 +1,892 @@
+import asyncio
+import collections
+import copy
+import errno
+import importlib
+import inspect
+import logging
+import random
+import re
+import time
+import uuid
+import zlib
+from decimal import ROUND_HALF_UP, Decimal
+from enum import Enum
+from functools import wraps
+from typing import Any, Callable, Dict, List, Optional, Set, TypeVar, Union
+
+import requests
+
+import ray
+import ray.util.serialization_addons
+from ray import cloudpickle
+from ray._common.constants import HEAD_NODE_RESOURCE_NAME
+from ray._common.utils import get_random_alphanumeric_string, import_attr
+from ray._raylet import MessagePackSerializer
+from ray.actor import ActorHandle
+from ray.serve._private.common import DeploymentID, RequestMetadata, ServeComponentType
+from ray.serve._private.constants import (
+    HTTP_PROXY_TIMEOUT,
+    SERVE_DEPLOYMENT_ACTOR_PREFIX,
+    SERVE_LOGGER_NAME,
+    SERVE_NAMESPACE,
+)
+from ray.types import ObjectRef
+from ray.util.serialization import StandaloneSerializationContext
+
+try:
+    import pandas as pd
+except ImportError:
+    pd = None
+
+try:
+    import numpy as np
+except ImportError:
+    np = None
+
+FILE_NAME_REGEX = r"[^\x20-\x7E]|[<>:\"/\\|?*]"
+
+MESSAGE_PACK_OFFSET = 9
+
+# Attribute set on functions/methods decorated with `@serve.multiplexed`. The
+# `__serve_multiplex_wrapper` is only created lazily on the first call, so this
+# marker is used to detect multiplexing statically (e.g. at replica startup)
+# without invoking user code.
+MULTIPLEXED_FUNCTION_MARKER_ATTR = "_serve_multiplexed_function"
+
+
+def _callable_uses_multiplexing(callable_obj: Any) -> bool:
+    """Whether `callable_obj` is or defines an `@serve.multiplexed` function.
+
+    Accepts a standalone function, a class, or a class instance, so it can be used
+    both at build time (where the deployment's `func_or_class` is available) and at
+    runtime (where an initialized instance is available).
+
+    For an instance it also inspects instance attributes, so multiplexing that is
+    wired up dynamically at init time (e.g. ``self._load_model =
+    serve.multiplexed(...)(fn)``) is detected. This case can only be caught at
+    runtime, since it is not visible on the class statically.
+    """
+    # NOTE: the marker is checked with `is True` rather than truthiness because some
+    # objects (e.g. `DeploymentHandle`, whose `__getattr__` returns a handle for any
+    # name) return a truthy value for an arbitrary attribute. The decorator always
+    # sets the marker to the literal `True`, so this stays exact without false
+    # positives.
+    def _has_marker(obj: Any) -> bool:
+        return getattr(obj, MULTIPLEXED_FUNCTION_MARKER_ATTR, False) is True
+
+    # Standalone function deployment decorated with `@serve.multiplexed`.
+    if _has_marker(callable_obj):
+        return True
+
+    # A class (or instance of one) with a method decorated with `@serve.multiplexed`.
+    klass = callable_obj if isinstance(callable_obj, type) else type(callable_obj)
+    for base in klass.__mro__:
+        for attr in base.__dict__.values():
+            if _has_marker(attr):
+                return True
+
+    # An instance that stored a multiplexed wrapper as an instance attribute.
+    if not isinstance(callable_obj, type):
+        for attr in getattr(callable_obj, "__dict__", {}).values():
+            if _has_marker(attr):
+                return True
+
+    return False
+
+
+def asyncio_grpc_exception_handler(loop, context):
+    """Exception handler to filter out false positive BlockingIOErrors from gRPC."""
+    exc = context.get("exception")
+    msg = context.get("message")
+    if (
+        exc
+        and isinstance(exc, BlockingIOError)
+        and exc.errno == errno.EAGAIN
+        and "PollerCompletionQueue._handle_events" in msg
+    ):
+        return
+
+    loop.default_exception_handler(context)
+
+
+def validate_ssl_config(
+    ssl_certfile: Optional[str], ssl_keyfile: Optional[str]
+) -> None:
+    """Validate SSL configuration for HTTPS support.
+
+    Args:
+        ssl_certfile: Path to SSL certificate file
+        ssl_keyfile: Path to SSL private key file
+
+    Raises:
+        ValueError: If only one of ssl_certfile or ssl_keyfile is provided
+    """
+    if (ssl_certfile and not ssl_keyfile) or (ssl_keyfile and not ssl_certfile):
+        raise ValueError(
+            "Both ssl_keyfile and ssl_certfile must be provided together "
+            "to enable HTTPS."
+        )
+
+
+def get_deployment_actor_name(
+    deployment_id: DeploymentID,
+    actor_name: str,
+    code_version: str,
+) -> str:
+    """Return the deterministic Ray actor name for a deployment-scoped actor.
+
+    The name is versioned by code_version to allow old and new replicas to
+    coexist during rollout (each uses its version's actors). Actors serve as
+    central state for replicas, so we version by code_version to ensure fresh
+    actors when a new code version is deployed.
+    """
+    base = (
+        f"{SERVE_DEPLOYMENT_ACTOR_PREFIX}{deployment_id.app_name}"
+        f"::{deployment_id.name}"
+    )
+    return f"{base}::{code_version}::{actor_name}"
+
+
+GENERATOR_COMPOSITION_NOT_SUPPORTED_ERROR = RuntimeError(
+    "Streaming deployment handle results cannot be passed to "
+    "downstream handle calls. If you have a use case requiring "
+    "this feature, please file a feature request on GitHub."
+)
+
+
+# Use a global singleton enum to emulate default options. We cannot use None
+# for those option because None is a valid new value.
+class DEFAULT(Enum):
+    VALUE = 1
+
+
+class DeploymentOptionUpdateType(str, Enum):
+    # Nothing needs to be done other than setting the target state.
+    LightWeight = "LightWeight"
+    # Each DeploymentReplica instance (tracked in DeploymentState) uses certain options
+    # from the deployment config. These values need to be updated in DeploymentReplica.
+    NeedsReconfigure = "NeedsReconfigure"
+    # Options that are sent to the replica actor. If changed, reconfigure() on the actor
+    # needs to be called to update these values.
+    NeedsActorReconfigure = "NeedsActorReconfigure"
+    # If changed, restart all replicas.
+    HeavyWeight = "HeavyWeight"
+
+
+# Type alias: objects that can be DEFAULT.VALUE have type Default[T]
+T = TypeVar("T")
+Default = Union[DEFAULT, T]
+
+logger = logging.getLogger(SERVE_LOGGER_NAME)
+
+# Format for component files
+FILE_FMT = "{component_name}_{component_id}{suffix}"
+
+
+class _ServeCustomEncoders:
+    """Group of custom encoders for common types that's not handled by FastAPI."""
+
+    @staticmethod
+    def encode_np_array(obj):
+        assert isinstance(obj, np.ndarray)
+        if obj.dtype.kind == "f":  # floats
+            obj = obj.astype(float)
+        if obj.dtype.kind in {"i", "u"}:  # signed and unsigned integers.
+            obj = obj.astype(int)
+        return obj.tolist()
+
+    @staticmethod
+    def encode_np_scaler(obj):
+        assert isinstance(obj, np.generic)
+        return obj.item()
+
+    @staticmethod
+    def encode_exception(obj):
+        assert isinstance(obj, Exception)
+        return str(obj)
+
+    @staticmethod
+    def encode_pandas_dataframe(obj):
+        assert isinstance(obj, pd.DataFrame)
+        return obj.to_dict(orient="records")
+
+
+serve_encoders = {Exception: _ServeCustomEncoders.encode_exception}
+
+if np is not None:
+    serve_encoders[np.ndarray] = _ServeCustomEncoders.encode_np_array
+    serve_encoders[np.generic] = _ServeCustomEncoders.encode_np_scaler
+
+if pd is not None:
+    serve_encoders[pd.DataFrame] = _ServeCustomEncoders.encode_pandas_dataframe
+
+
+@ray.remote(num_cpus=0)
+def block_until_http_ready(
+    http_endpoint,
+    backoff_time_s=1,
+    check_ready=None,
+    timeout=HTTP_PROXY_TIMEOUT,
+):
+    http_is_ready = False
+    start_time = time.time()
+
+    while not http_is_ready:
+        try:
+            resp = requests.get(http_endpoint)
+            assert resp.status_code == 200
+            if check_ready is None:
+                http_is_ready = True
+            else:
+                http_is_ready = check_ready(resp)
+        except Exception:
+            pass
+
+        if 0 < timeout < time.time() - start_time:
+            raise TimeoutError("HTTP proxy not ready after {} seconds.".format(timeout))
+
+        time.sleep(backoff_time_s)
+
+
+def get_random_string(length: int = 8):
+    return get_random_alphanumeric_string(length)
+
+
+def format_actor_name(actor_name, *modifiers):
+    name = actor_name
+    for modifier in modifiers:
+        name += "-{}".format(modifier)
+
+    return name
+
+
+CLASS_WRAPPER_METADATA_ATTRS = (
+    "__name__",
+    "__qualname__",
+    "__module__",
+    "__doc__",
+    "__annotations__",
+)
+
+
+def copy_class_metadata(wrapper_cls, target_cls) -> None:
+    """Copy common class-level metadata onto a wrapper class."""
+    for attr in CLASS_WRAPPER_METADATA_ATTRS:
+        if attr == "__annotations__":
+            target_annotations = getattr(target_cls, "__annotations__", None)
+            if target_annotations:
+                merged_annotations = dict(
+                    wrapper_cls.__dict__.get("__annotations__", {})
+                )
+                for key, value in target_annotations.items():
+                    merged_annotations.setdefault(key, value)
+                wrapper_cls.__annotations__ = merged_annotations
+            continue
+
+        if hasattr(target_cls, attr):
+            setattr(wrapper_cls, attr, getattr(target_cls, attr))
+    wrapper_cls.__wrapped__ = target_cls
+
+
+def ensure_serialization_context():
+    """Ensure the serialization addons on registered, even when Ray has not
+    been started."""
+    ctx = StandaloneSerializationContext()
+    ray.util.serialization_addons.apply(ctx)
+
+
+def msgpack_serialize(obj):
+    ctx = ray._private.worker.global_worker.get_serialization_context()
+    buffer = ctx.serialize(obj)
+    serialized = buffer.to_bytes()
+    return serialized
+
+
+def msgpack_deserialize(data):
+    # todo: Ray does not provide a msgpack deserialization api.
+    try:
+        obj = MessagePackSerializer.loads(data[MESSAGE_PACK_OFFSET:], None)
+    except Exception:
+        raise
+    return obj
+
+
+def merge_dict(dict1, dict2):
+    if dict1 is None and dict2 is None:
+        return None
+    if dict1 is None:
+        dict1 = dict()
+    if dict2 is None:
+        dict2 = dict()
+    result = dict()
+    for key in dict1.keys() | dict2.keys():
+        result[key] = sum([e.get(key, 0) for e in (dict1, dict2)])
+    return result
+
+
+def parse_import_path(import_path: str):
+    """
+    Takes in an import_path of form:
+
+    [subdirectory 1].[subdir 2]...[subdir n].[file name].[attribute name]
+
+    Parses this path and returns the module name (everything before the last
+    dot) and attribute name (everything after the last dot), such that the
+    attribute can be imported using "from module_name import attr_name".
+    """
+
+    nodes = import_path.split(".")
+    if len(nodes) < 2:
+        raise ValueError(
+            f"Got {import_path} as import path. The import path "
+            f"should at least specify the file name and "
+            f"attribute name connected by a dot."
+        )
+
+    return ".".join(nodes[:-1]), nodes[-1]
+
+
+def override_runtime_envs_except_env_vars(parent_env: Dict, child_env: Dict) -> Dict:
+    """Creates a runtime_env dict by merging a parent and child environment.
+
+    This method is not destructive. It leaves the parent and child envs
+    the same.
+
+    The merge is a shallow update where the child environment inherits the
+    parent environment's settings. If the child environment specifies any
+    env settings, those settings take precdence over the parent.
+        - Note: env_vars are a special case. The child's env_vars are combined
+            with the parent.
+
+    Args:
+        parent_env: The environment to inherit settings from.
+        child_env: The environment with override settings.
+
+    Returns:
+        A new dictionary containing the merged runtime_env settings.
+
+    Raises:
+        TypeError: If a dictionary is not passed in for parent_env or child_env.
+    """
+
+    if not isinstance(parent_env, Dict):
+        raise TypeError(
+            f'Got unexpected type "{type(parent_env)}" for parent_env. '
+            "parent_env must be a dictionary."
+        )
+    if not isinstance(child_env, Dict):
+        raise TypeError(
+            f'Got unexpected type "{type(child_env)}" for child_env. '
+            "child_env must be a dictionary."
+        )
+
+    defaults = copy.deepcopy(parent_env)
+    overrides = copy.deepcopy(child_env)
+
+    default_env_vars = defaults.get("env_vars", {})
+    override_env_vars = overrides.get("env_vars", {})
+
+    defaults.update(overrides)
+    default_env_vars.update(override_env_vars)
+
+    defaults["env_vars"] = default_env_vars
+
+    return defaults
+
+
+class JavaActorHandleProxy:
+    """Wraps actor handle and translate snake_case to camelCase."""
+
+    def __init__(self, handle: ActorHandle):
+        self.handle = handle
+        self._available_attrs = set(dir(self.handle))
+
+    def __getattr__(self, key: str):
+        if key in self._available_attrs:
+            camel_case_key = key
+        else:
+            components = key.split("_")
+            camel_case_key = components[0] + "".join(x.title() for x in components[1:])
+        return getattr(self.handle, camel_case_key)
+
+
+def require_packages(packages: List[str]):
+    """Decorator making sure function run in specified environments
+
+    Examples:
+        >>> from ray.serve._private.utils import require_packages
+        >>> @require_packages(["numpy", "package_a"]) # doctest: +SKIP
+        ... def func(): # doctest: +SKIP
+        ...     import numpy as np # doctest: +SKIP
+        ...     ... # doctest: +SKIP
+        >>> func() # doctest: +SKIP
+        ImportError: func requires ["numpy", "package_a"] but
+        ["package_a"] are not available, please pip install them.
+
+    Args:
+        packages: The list of package names that must be importable when the
+            decorated function is invoked.
+
+    Returns:
+        A decorator that wraps the target function with the package check.
+    """
+
+    def decorator(func):
+        def check_import_once():
+            if not hasattr(func, "_require_packages_checked"):
+                missing_packages = []
+                for package in packages:
+                    try:
+                        importlib.import_module(package)
+                    except ModuleNotFoundError:
+                        missing_packages.append(package)
+                if len(missing_packages) > 0:
+                    raise ImportError(
+                        f"{func} requires packages {packages} to run but "
+                        f"{missing_packages} are missing. Please "
+                        "`pip install` them or add them to "
+                        "`runtime_env`."
+                    )
+                func._require_packages_checked = True
+
+        if inspect.iscoroutinefunction(func):
+
+            @wraps(func)
+            async def wrapped(*args, **kwargs):
+                check_import_once()
+                return await func(*args, **kwargs)
+
+        elif inspect.isroutine(func):
+
+            @wraps(func)
+            def wrapped(*args, **kwargs):
+                check_import_once()
+                return func(*args, **kwargs)
+
+        else:
+            raise ValueError("Decorator expect callable functions.")
+
+        return wrapped
+
+    return decorator
+
+
+def in_interactive_shell():
+    # Taken from:
+    # https://stackoverflow.com/questions/15411967/how-can-i-check-if-code-is-executed-in-the-ipython-notebook
+    import __main__ as main
+
+    return not hasattr(main, "__file__")
+
+
+def snake_to_camel_case(snake_str: str) -> str:
+    """Convert a snake case string to camel case."""
+
+    words = snake_str.strip("_").split("_")
+    return words[0] + "".join(word[:1].upper() + word[1:] for word in words[1:])
+
+
+def check_obj_ref_ready_nowait(obj_ref: ObjectRef) -> bool:
+    """Check if ray object reference is ready without waiting for it."""
+    finished, _ = ray.wait([obj_ref], timeout=0)
+    return len(finished) == 1
+
+
+def compress_metric_report(report: Any) -> bytes:
+    """Compress a metric report (HandleMetricReport or ReplicaMetricReport) for RPC.
+
+    Uses zlib level 9 (stdlib, no extra deps). ~75KB uncompressed -> ~5KB for 1000 replicas.
+    """
+    return zlib.compress(cloudpickle.dumps(report), level=9)
+
+
+def decompress_metric_report(compressed: bytes) -> Any:
+    """Decompress a metric report from RPC."""
+    return cloudpickle.loads(zlib.decompress(compressed))
+
+
+def extract_self_if_method_call(args: List[Any], func: Callable) -> Optional[object]:
+    """Check if this is a method rather than a function.
+
+    Does this by checking to see if `func` is the attribute of the first
+    (`self`) argument under `func.__name__`. Unfortunately, this is the most
+    robust solution to this I was able to find. It would also be preferable
+    to do this check when the decorator runs, rather than when the method is.
+
+    Arguments:
+        args: arguments to the function/method call.
+        func: the unbound function that was called.
+
+    Returns:
+        The ``self`` object if it's a method call, else ``None``.
+    """
+    if len(args) > 0:
+        method = getattr(args[0], func.__name__, False)
+        if method:
+            wrapped = getattr(method, "__wrapped__", False)
+            if wrapped and wrapped == func:
+                return args[0]
+
+    return None
+
+
+def call_function_from_import_path(import_path: str) -> Any:
+    """Call the function given import path.
+
+    Args:
+        import_path: The import path of the function to call.
+    Raises:
+        ValueError: If the import path is invalid.
+        TypeError: If the import path is not callable.
+        RuntimeError: if the function raise exeception during execution.
+    Returns:
+        The result of the function call.
+    """
+    try:
+        callback_func = import_attr(import_path)
+    except Exception as e:
+        raise ValueError(f"The import path {import_path} cannot be imported: {e}")
+
+    if not callable(callback_func):
+        raise TypeError(f"The import path {import_path} is not callable.")
+
+    try:
+        return callback_func()
+    except Exception as e:
+        raise RuntimeError(f"The function {import_path} raised an exception: {e}")
+
+
+def get_head_node_id() -> str:
+    """Get the head node id.
+
+    Iterate through all nodes in the ray cluster and return the node id of the first
+    alive node with head node resource.
+    """
+    head_node_id = None
+    for node in ray.nodes():
+        if HEAD_NODE_RESOURCE_NAME in node["Resources"] and node["Alive"]:
+            head_node_id = node["NodeID"]
+            break
+    assert head_node_id is not None, "Cannot find alive head node."
+
+    return head_node_id
+
+
+def calculate_remaining_timeout(
+    *,
+    timeout_s: Optional[float],
+    start_time_s: float,
+    curr_time_s: float,
+) -> Optional[float]:
+    """Get the timeout remaining given an overall timeout, start time, and curr time.
+
+    If the timeout passed in was `None` or negative, will always return that timeout
+    directly.
+
+    If the timeout is >= 0, the returned remaining timeout always be >= 0.
+    """
+    if timeout_s is None or timeout_s < 0:
+        return timeout_s
+
+    time_since_start_s = curr_time_s - start_time_s
+    return max(0, timeout_s - time_since_start_s)
+
+
+def get_all_live_placement_group_names() -> List[str]:
+    """Fetch and parse the Ray placement group table for live placement group names.
+
+    Placement groups are filtered based on their `scheduling_state`; any placement
+    group not in the "REMOVED" state is considered live.
+    """
+    placement_group_table = ray.util.placement_group_table()
+
+    live_pg_names = []
+    for entry in placement_group_table.values():
+        pg_name = entry.get("name", "")
+        if (
+            pg_name
+            and entry.get("stats", {}).get("scheduling_state", "UNKNOWN") != "REMOVED"
+        ):
+            live_pg_names.append(pg_name)
+
+    return live_pg_names
+
+
+def get_active_placement_group_ids() -> Set[str]:
+    """
+    Retrieve the set of placement group IDs referenced by alive Serve actors.
+
+    Returns:
+        The set of placement group IDs referenced by alive Serve actors.
+    """
+    # TODO (jeffreywang): Move the imports to the top of the file.
+    # https://github.com/ray-project/ray/issues/61330
+    from ray.util.state import list_actors
+    from ray.util.state.common import RAY_MAX_LIMIT_FROM_API_SERVER
+
+    actors = list_actors(
+        filters=[
+            ("ray_namespace", "=", SERVE_NAMESPACE),
+            ("state", "=", "ALIVE"),
+        ],
+        limit=RAY_MAX_LIMIT_FROM_API_SERVER,
+        detail=True,
+        raise_on_missing_output=False,
+    )
+
+    return {
+        actor.placement_group_id
+        for actor in actors
+        if actor.placement_group_id is not None
+    }
+
+
+def get_current_actor_id() -> str:
+    """Gets the ID of the calling actor.
+
+    If this is called in a driver, returns "DRIVER."
+
+    If otherwise called outside of an actor, returns an empty string.
+
+    This function hangs when GCS is down due to the `ray.get_runtime_context()`
+    call.
+    """
+
+    worker_mode = ray.get_runtime_context().worker.mode
+    if worker_mode == ray.SCRIPT_MODE:
+        return "DRIVER"
+    else:
+        try:
+            actor_id = ray.get_runtime_context().get_actor_id()
+            if actor_id is None:
+                return ""
+            else:
+                return actor_id
+        except Exception:
+            return ""
+
+
+def is_running_in_asyncio_loop() -> bool:
+    try:
+        asyncio.get_running_loop()
+        return True
+    except RuntimeError:
+        return False
+
+
+def get_capacity_adjusted_num_replicas(
+    num_replicas: int, target_capacity: Optional[float]
+) -> int:
+    """Return the `num_replicas` adjusted by the `target_capacity`.
+
+    The output will only ever be 0 if `target_capacity` is 0 or `num_replicas` is
+    0 (to support autoscaling deployments using scale-to-zero).
+
+    Rather than using the default `round` behavior in Python, which rounds half to
+    even, uses the `decimal` module to round half up (standard rounding behavior).
+    """
+    if target_capacity is None or target_capacity == 100:
+        return num_replicas
+
+    if target_capacity == 0 or num_replicas == 0:
+        return 0
+
+    adjusted_num_replicas = Decimal(num_replicas * target_capacity) / Decimal(100.0)
+    rounded_adjusted_num_replicas = adjusted_num_replicas.to_integral_value(
+        rounding=ROUND_HALF_UP
+    )
+    return max(1, int(rounded_adjusted_num_replicas))
+
+
+def generate_request_id() -> str:
+    # NOTE(edoakes): we use random.getrandbits because it reduces CPU overhead
+    # significantly. This is less cryptographically secure but should be ok for
+    # request ID generation.
+    # See https://bugs.python.org/issue45556 for discussion.
+    return str(uuid.UUID(int=random.getrandbits(128), version=4))
+
+
+def inside_ray_client_context() -> bool:
+    return ray.util.client.ray.is_connected()
+
+
+def get_component_file_name(
+    component_name: str,
+    component_id: str,
+    component_type: Optional[ServeComponentType],
+    suffix: str = "",
+) -> str:
+    """Get the component's file name. Replaces special characters with underscores."""
+    component_name = re.sub(FILE_NAME_REGEX, "_", component_name)
+
+    # For DEPLOYMENT component type, we want to log the deployment name
+    # instead of adding the component type to the component name.
+    component_log_file_name = component_name
+    if component_type is not None:
+        component_log_file_name = f"{component_type.value}_{component_name}"
+        if component_type != ServeComponentType.REPLICA:
+            component_name = f"{component_type}_{component_name}"
+    file_name = FILE_FMT.format(
+        component_name=component_log_file_name,
+        component_id=component_id,
+        suffix=suffix,
+    )
+    return file_name
+
+
+def validate_route_prefix(route_prefix: Union[DEFAULT, None, str]):
+    if route_prefix is DEFAULT.VALUE or route_prefix is None:
+        return
+
+    if not route_prefix.startswith("/"):
+        raise ValueError(
+            f"Invalid route_prefix '{route_prefix}', "
+            "must start with a forward slash ('/')."
+        )
+
+    if route_prefix != "/" and route_prefix.endswith("/"):
+        raise ValueError(
+            f"Invalid route_prefix '{route_prefix}', "
+            "may not end with a trailing '/'."
+        )
+
+    if "{" in route_prefix or "}" in route_prefix:
+        raise ValueError(
+            f"Invalid route_prefix '{route_prefix}', may not contain wildcards."
+        )
+
+
+async def await_deployment_response(deployment_response):
+    return await deployment_response
+
+
+async def resolve_deployment_response(obj: Any, request_metadata: RequestMetadata):
+    """Resolve `DeploymentResponse` objects to underlying object references.
+
+    This enables composition without explicitly calling `_to_object_ref`.
+    """
+    from ray.serve.handle import DeploymentResponse, DeploymentResponseGenerator
+
+    if isinstance(obj, DeploymentResponseGenerator):
+        raise GENERATOR_COMPOSITION_NOT_SUPPORTED_ERROR
+    elif isinstance(obj, DeploymentResponse):
+        if request_metadata._by_reference and obj.by_reference:
+            # If sending requests by reference, launch async task to
+            # convert DeploymentResponse to an object ref
+            return asyncio.create_task(obj._to_object_ref())
+        else:
+            # Otherwise, resolve DeploymentResponse directly to result
+            return asyncio.create_task(await_deployment_response(obj))
+    elif not request_metadata._by_reference and isinstance(obj, ray.ObjectRef):
+        # If the router is sending requests by value (i.e. using gRPC),
+        # resolve all Ray objects to mirror Ray behavior
+        return asyncio.wrap_future(obj.future())
+
+
+def wait_for_interrupt() -> None:
+    try:
+        while True:
+            # Block, letting Ray print logs to the terminal.
+            time.sleep(10)
+    except KeyboardInterrupt:
+        logger.warning("Got KeyboardInterrupt, exiting...")
+        # We need to re-raise KeyboardInterrupt, so serve components can be shutdown
+        # from the main script.
+        raise
+
+
+def is_grpc_enabled(grpc_config) -> bool:
+    return grpc_config.port > 0 and len(grpc_config.grpc_servicer_functions) > 0
+
+
+class Semaphore:
+    """Based on asyncio.Semaphore.
+
+    This is a semaphore that can be used to limit the number of concurrent requests.
+    Its maximum value is dynamic and is determined by the `get_value_fn` function.
+    """
+
+    def __init__(self, get_value_fn: Callable[[], int]):
+        self._waiters = None
+        self._value = 0
+        self._get_value_fn = get_value_fn
+
+    def __repr__(self):
+        res = super().__repr__()
+        extra = "locked" if self.locked() else f"unlocked, value:{self._value}"
+        if self._waiters:
+            extra = f"{extra}, waiters:{len(self._waiters)}"
+        return f"<{res[1:-1]} [{extra}]>"
+
+    async def __aenter__(self):
+        await self.acquire()
+        # We have no use for the "as ..."  clause in the with
+        # statement for locks.
+        return None
+
+    async def __aexit__(self, exc_type, exc, tb):
+        self.release()
+
+    def get_max_value(self):
+        return self._get_value_fn()
+
+    def locked(self):
+        """Returns True if semaphore cannot be acquired immediately."""
+        return self._value >= self.get_max_value() or (
+            any(not w.cancelled() for w in (self._waiters or ()))
+        )
+
+    async def acquire(self):
+        """Acquire a semaphore.
+        If the internal counter is larger than zero on entry,
+        decrement it by one and return True immediately.  If it is
+        zero on entry, block, waiting until some other coroutine has
+        called release() to make it larger than 0, and then return
+        True.
+        """
+        if not self.locked():
+            self._value += 1
+            return True
+
+        if self._waiters is None:
+            self._waiters = collections.deque()
+        fut = asyncio.Future()
+        self._waiters.append(fut)
+
+        # Finally block should be called before the CancelledError
+        # handling as we don't want CancelledError to call
+        # _wake_up_first() and attempt to wake up itself.
+        try:
+            try:
+                await fut
+            finally:
+                self._waiters.remove(fut)
+        except asyncio.CancelledError:
+            if not fut.cancelled():
+                self._value -= 1
+                self._wake_up_next()
+            raise
+
+        if self._value < self.get_max_value():
+            self._wake_up_next()
+        return True
+
+    def release(self):
+        """Release a semaphore, incrementing the internal counter by one.
+        When it was zero on entry and another coroutine is waiting for it to
+        become larger than zero again, wake up that coroutine.
+        """
+        self._value -= 1
+        self._wake_up_next()
+
+    def _wake_up_next(self):
+        """Wake up the first waiter that isn't done."""
+        if not self._waiters:
+            return
+
+        for fut in self._waiters:
+            if not fut.done():
+                self._value += 1
+                fut.set_result(True)
+                return

python/ray/serve/_private/version.py

@@ -0,0 +1,282 @@
+import json
+import logging
+from copy import deepcopy
+from typing import Any, Dict, List, Optional
+from zlib import crc32
+
+from pydantic import BaseModel
+
+from ray.serve._private.config import DeploymentConfig
+from ray.serve._private.constants import SERVE_LOGGER_NAME
+from ray.serve._private.utils import DeploymentOptionUpdateType, get_random_string
+from ray.serve.config import AutoscalingConfig
+from ray.serve.generated.serve_pb2 import DeploymentVersion as DeploymentVersionProto
+
+logger = logging.getLogger(SERVE_LOGGER_NAME)
+
+
+class DeploymentVersion:
+    def __init__(
+        self,
+        code_version: Optional[str],
+        deployment_config: DeploymentConfig,
+        ray_actor_options: Optional[Dict],
+        placement_group_bundles: Optional[List[Dict[str, float]]] = None,
+        placement_group_strategy: Optional[str] = None,
+        placement_group_bundle_label_selector: Optional[List[Dict[str, str]]] = None,
+        placement_group_fallback_strategy: Optional[List[Dict[str, Any]]] = None,
+        max_replicas_per_node: Optional[int] = None,
+        route_prefix: Optional[str] = None,
+    ):
+        if code_version is not None and not isinstance(code_version, str):
+            raise TypeError(f"code_version must be str, got {type(code_version)}.")
+        if code_version is None:
+            self.code_version = get_random_string()
+        else:
+            self.code_version = code_version
+
+        # Options for this field may be mutated over time, so any logic that uses this
+        # should access this field directly.
+        self.deployment_config = deployment_config
+        self.ray_actor_options = ray_actor_options
+        self.placement_group_bundles = placement_group_bundles
+        self.placement_group_strategy = placement_group_strategy
+        self.placement_group_bundle_label_selector = (
+            placement_group_bundle_label_selector
+        )
+        self.placement_group_fallback_strategy = placement_group_fallback_strategy
+        self.max_replicas_per_node = max_replicas_per_node
+        self.route_prefix = route_prefix
+        self.compute_hashes()
+
+    @classmethod
+    def from_deployment_version(
+        cls, deployment_version, deployment_config, route_prefix: Optional[str] = None
+    ):
+        version_copy = deepcopy(deployment_version)
+        version_copy.deployment_config = deployment_config
+        version_copy.route_prefix = route_prefix
+        version_copy.compute_hashes()
+        return version_copy
+
+    def __hash__(self) -> int:
+        return self._hash
+
+    def __eq__(self, other: Any) -> bool:
+        if not isinstance(other, DeploymentVersion):
+            return False
+        return self._hash == other._hash
+
+    def requires_actor_restart(self, new_version):
+        """Determines whether the new version requires actors of the current version to
+        be restarted.
+        """
+        return (
+            self.code_version != new_version.code_version
+            or self.ray_actor_options_hash != new_version.ray_actor_options_hash
+            or self.placement_group_options_hash
+            != new_version.placement_group_options_hash
+            or self.max_replicas_per_node != new_version.max_replicas_per_node
+            or self.gang_scheduling_config_hash
+            != new_version.gang_scheduling_config_hash
+        )
+
+    def requires_actor_reconfigure(self, new_version):
+        """Determines whether the new version requires calling reconfigure() on the
+        replica actor.
+        """
+        return self.reconfigure_actor_hash != new_version.reconfigure_actor_hash
+
+    def requires_long_poll_broadcast(self, new_version):
+        """Determines whether lightweightly updating an existing replica to the new
+        version requires broadcasting through long poll that the running replicas has
+        changed.
+        """
+        return (
+            self.deployment_config.max_ongoing_requests
+            != new_version.deployment_config.max_ongoing_requests
+        )
+
+    def compute_hashes(self):
+        # If these change, the controller will rolling upgrade existing replicas.
+        serialized_ray_actor_options = _serialize(self.ray_actor_options or {})
+        self.ray_actor_options_hash = crc32(serialized_ray_actor_options)
+        combined_placement_group_options = {}
+        if self.placement_group_bundles is not None:
+            combined_placement_group_options["bundles"] = self.placement_group_bundles
+        if self.placement_group_strategy is not None:
+            combined_placement_group_options["strategy"] = self.placement_group_strategy
+        if self.placement_group_bundle_label_selector is not None:
+            combined_placement_group_options[
+                "bundle_label_selector"
+            ] = self.placement_group_bundle_label_selector
+        if self.placement_group_fallback_strategy is not None:
+            combined_placement_group_options[
+                "fallback_strategy"
+            ] = self.placement_group_fallback_strategy
+        serialized_placement_group_options = _serialize(
+            combined_placement_group_options
+        )
+        self.placement_group_options_hash = crc32(serialized_placement_group_options)
+        serialized_gang_scheduling_config = _serialize(
+            self.deployment_config.gang_scheduling_config.model_dump()
+            if self.deployment_config.gang_scheduling_config is not None
+            else {}
+        )
+        self.gang_scheduling_config_hash = crc32(serialized_gang_scheduling_config)
+        # Include app-level route prefix in the version hashes so changing
+        # it triggers an in-place reconfigure of running replicas.
+        serialized_route_prefix = _serialize(self.route_prefix)
+
+        # If this changes, DeploymentReplica.reconfigure() will call reconfigure on the
+        # actual replica actor
+        self.reconfigure_actor_hash = crc32(
+            serialized_route_prefix
+            + self._get_serialized_options(
+                [DeploymentOptionUpdateType.NeedsActorReconfigure]
+            )
+        )
+
+        # Used by __eq__ in deployment state to either reconfigure the replicas or
+        # stop and restart them
+        self._hash = crc32(
+            self.code_version.encode("utf-8")
+            + serialized_ray_actor_options
+            + serialized_placement_group_options
+            + str(self.max_replicas_per_node).encode("utf-8")
+            + serialized_route_prefix
+            + self._get_serialized_options(
+                [
+                    DeploymentOptionUpdateType.NeedsReconfigure,
+                    DeploymentOptionUpdateType.NeedsActorReconfigure,
+                ]
+            )
+            + serialized_gang_scheduling_config
+        )
+
+    def to_proto(self) -> bytes:
+        # TODO(simon): enable cross language user config
+
+        placement_group_bundles = (
+            json.dumps(self.placement_group_bundles)
+            if self.placement_group_bundles is not None
+            else ""
+        )
+
+        placement_group_bundle_label_selector = (
+            json.dumps(self.placement_group_bundle_label_selector)
+            if self.placement_group_bundle_label_selector is not None
+            else ""
+        )
+
+        placement_group_fallback_strategy = (
+            json.dumps(self.placement_group_fallback_strategy)
+            if self.placement_group_fallback_strategy is not None
+            else ""
+        )
+
+        placement_group_strategy = (
+            self.placement_group_strategy
+            if self.placement_group_strategy is not None
+            else ""
+        )
+        max_replicas_per_node = (
+            self.max_replicas_per_node if self.max_replicas_per_node is not None else 0
+        )
+
+        return DeploymentVersionProto(
+            code_version=self.code_version,
+            deployment_config=self.deployment_config.to_proto(),
+            ray_actor_options=json.dumps(self.ray_actor_options),
+            placement_group_bundles=placement_group_bundles,
+            placement_group_strategy=placement_group_strategy,
+            placement_group_bundle_label_selector=placement_group_bundle_label_selector,
+            placement_group_fallback_strategy=placement_group_fallback_strategy,
+            max_replicas_per_node=max_replicas_per_node,
+        )
+
+    @classmethod
+    def from_proto(cls, proto: DeploymentVersionProto):
+        return DeploymentVersion(
+            proto.code_version,
+            DeploymentConfig.from_proto(proto.deployment_config),
+            json.loads(proto.ray_actor_options),
+            placement_group_bundles=(
+                json.loads(proto.placement_group_bundles)
+                if proto.placement_group_bundles
+                else None
+            ),
+            placement_group_bundle_label_selector=(
+                json.loads(proto.placement_group_bundle_label_selector)
+                if proto.placement_group_bundle_label_selector
+                else None
+            ),
+            placement_group_fallback_strategy=(
+                json.loads(proto.placement_group_fallback_strategy)
+                if proto.placement_group_fallback_strategy
+                else None
+            ),
+            placement_group_strategy=(
+                proto.placement_group_strategy
+                if proto.placement_group_strategy
+                else None
+            ),
+            max_replicas_per_node=(
+                proto.max_replicas_per_node if proto.max_replicas_per_node else None
+            ),
+        )
+
+    def _get_serialized_options(
+        self, update_types: List[DeploymentOptionUpdateType]
+    ) -> bytes:
+        """Returns a serialized dictionary containing fields of a deployment config that
+        should prompt a deployment version update.
+        """
+        reconfigure_dict = {}
+        # In pydantic 2.0, `__fields__` has been renamed to `model_fields`.
+        # Access from class, not instance, to avoid deprecation warning.
+        fields = DeploymentConfig.model_fields
+        for option_name, field in fields.items():
+            # In Pydantic v2, extra kwargs passed to Field() are in json_schema_extra
+            json_schema_extra = field.json_schema_extra
+            option_weight = (
+                json_schema_extra.get("update_type")
+                if isinstance(json_schema_extra, dict)
+                else None
+            )
+            if option_weight in update_types:
+                reconfigure_dict[option_name] = getattr(
+                    self.deployment_config, option_name
+                )
+                # If autoscaling config was changed, only broadcast to
+                # replicas if metrics_interval_s or look_back_period_s
+                # was changed, because the rest of the fields are only
+                # used in deployment state manager
+                if isinstance(reconfigure_dict[option_name], AutoscalingConfig):
+                    reconfigure_dict[option_name] = reconfigure_dict[
+                        option_name
+                    ].model_dump(include={"metrics_interval_s", "look_back_period_s"})
+                elif isinstance(reconfigure_dict[option_name], BaseModel):
+                    reconfigure_dict[option_name] = reconfigure_dict[
+                        option_name
+                    ].model_dump()
+
+        # Can't serialize bytes. The request router class is already
+        # included in the serialized config as request_router_class.
+        if "request_router_config" in reconfigure_dict:
+            reconfigure_dict["request_router_config"].pop(
+                "_serialized_request_router_cls", None
+            )
+
+        if (
+            isinstance(self.deployment_config.user_config, bytes)
+            and "user_config" in reconfigure_dict
+        ):
+            del reconfigure_dict["user_config"]
+            return self.deployment_config.user_config + _serialize(reconfigure_dict)
+
+        return _serialize(reconfigure_dict)
+
+
+def _serialize(json_object):
+    return str.encode(json.dumps(json_object, sort_keys=True))

python/ray/serve/async_inference_autoscaling_policy.py

@@ -0,0 +1,137 @@
+import asyncio
+import logging
+import time
+from typing import Any, Dict, Optional, Tuple, Union
+
+from ray.serve._private.broker import Broker
+from ray.serve._private.constants import SERVE_LOGGER_NAME
+from ray.serve.config import AutoscalingContext
+
+logger = logging.getLogger(SERVE_LOGGER_NAME)
+
+DEFAULT_ASYNC_INFERENCE_QUEUE_POLL_INTERVAL_S = 10.0
+
+
+class AsyncInferenceAutoscalingPolicy:
+    """Autoscaling policy that scales replicas based on message queue length.
+
+    Polls a message broker (Redis or RabbitMQ) for queue length and combines
+    it with HTTP request load to compute the desired number of replicas.
+
+    Polling uses one-shot async tasks instead of an infinite background loop.
+    An infinite ``while True`` coroutine holds a strong reference to ``self``
+    through the coroutine, and the event loop keeps the task alive, so
+    ``__del__`` would never fire after the framework drops the policy on
+    redeploy/deregistration — leaking both the poller and the broker
+    connection. Instead, each poll is a single one-shot task kicked off from
+    ``__call__`` when the poll interval has elapsed. The task completes
+    naturally after one poll, so there is at most one short-lived in-flight
+    task at any time and no cleanup is needed when the policy is
+    garbage-collected.
+
+    This policy is intended for use with ``@task_consumer`` deployments.
+    Pass it as a class-based policy via ``AutoscalingPolicy``:
+
+    .. code-block:: python
+
+        from ray.serve.config import AutoscalingConfig, AutoscalingPolicy
+
+        @serve.deployment(
+            autoscaling_config=AutoscalingConfig(
+                min_replicas=1,
+                max_replicas=10,
+                policy=AutoscalingPolicy(
+                    policy_function=AsyncInferenceAutoscalingPolicy,
+                    policy_kwargs={
+                        "broker_url": "redis://localhost:6379/0",
+                        "queue_name": "my_queue",
+                    },
+                ),
+            ),
+        )
+        @task_consumer(task_processor_config=config)
+        class MyConsumer: ...
+
+    Args:
+        broker_url: URL of the message broker (e.g. ``redis://localhost:6379/0``
+            or ``amqp://guest:guest@localhost:5672//``).
+        queue_name: Name of the queue to monitor.
+        rabbitmq_management_url: RabbitMQ HTTP management API URL. Only required
+            for RabbitMQ brokers (e.g. ``http://guest:guest@localhost:15672/api/``).
+        poll_interval_s: How often (in seconds) to poll the broker for queue
+            length. Defaults to 10s. Lower values increase responsiveness
+            but add broker load.
+    """
+
+    def __init__(
+        self,
+        broker_url: str,
+        queue_name: str,
+        rabbitmq_management_url: Optional[str] = None,
+        poll_interval_s: float = DEFAULT_ASYNC_INFERENCE_QUEUE_POLL_INTERVAL_S,
+    ):
+        self._broker_url = broker_url
+        self._queue_name = queue_name
+        self._rabbitmq_management_url = rabbitmq_management_url
+        self._poll_interval_s = poll_interval_s
+
+        self._queue_length: int = 0
+        self._broker: Optional[Broker] = None
+        self._task: Optional[asyncio.Task] = None
+        self._last_poll_time: float = 0.0
+
+    def _ensure_broker(self) -> None:
+        """Lazily initialize the broker connection."""
+        if self._broker is not None:
+            return
+
+        if self._rabbitmq_management_url is not None:
+            self._broker = Broker(
+                self._broker_url, http_api=self._rabbitmq_management_url
+            )
+        else:
+            self._broker = Broker(self._broker_url)
+
+    async def _poll_once(self) -> None:
+        """Single one-shot poll of the broker for queue length."""
+        try:
+            queues = await self._broker.queues([self._queue_name])
+            if queues is not None:
+                for q in queues:
+                    if q.get("name") == self._queue_name:
+                        queue_length = q.get("messages")
+                        if queue_length is not None:
+                            self._queue_length = queue_length
+                        break
+        except Exception as e:
+            logger.warning(f"Failed to get queue length for '{self._queue_name}': {e}")
+
+    def __call__(
+        self, ctx: AutoscalingContext
+    ) -> Tuple[Union[int, float], Dict[str, Any]]:
+        self._ensure_broker()
+
+        # Clear completed poll task so a new one can be started.
+        if self._task is not None and self._task.done():
+            self._task = None
+
+        # Start a new poll if the interval has elapsed and no poll is in-flight.
+        now = time.monotonic()
+        if self._task is None and (now - self._last_poll_time) >= self._poll_interval_s:
+            self._last_poll_time = now
+            self._task = asyncio.get_running_loop().create_task(self._poll_once())
+
+        num_running_replicas = ctx.current_num_replicas
+        total_workload = ctx.total_num_requests + self._queue_length
+        config = ctx.config
+
+        if num_running_replicas == 0:
+            return 1 if total_workload > 0 else 0, {"queue_length": self._queue_length}
+
+        target_num_requests = (
+            config.get_target_ongoing_requests() * num_running_replicas
+        )
+        error_ratio = total_workload / target_num_requests
+        desired_num_replicas = num_running_replicas * error_ratio
+
+        return desired_num_replicas, {"queue_length": self._queue_length}

python/ray/serve/autoscaling_policy.py

@@ -0,0 +1,359 @@
+import functools
+import logging
+import math
+import time
+from typing import Any, Callable, Dict, Optional, Tuple, Union
+
+from ray.serve._private.common import DeploymentID
+from ray.serve._private.constants import (
+    SERVE_AUTOSCALING_DECISION_COUNTERS_KEY,
+    SERVE_AUTOSCALING_DECISION_TIMESTAMP_KEY,
+    SERVE_LOGGER_NAME,
+)
+from ray.serve.config import AutoscalingConfig, AutoscalingContext
+from ray.util.annotations import PublicAPI
+
+logger = logging.getLogger(SERVE_LOGGER_NAME)
+
+# Tolerance for delay elapsed-time comparisons.  Subtracting two large
+# time.time() values (or test fake clocks derived from tick counters) can
+# drift slightly below the true elapsed interval in IEEE 754 (e.g. 400.0s
+# configured delay may compare as 399.9999999999999 >= 400.0).
+_DELAY_ELAPSED_EPS_S = 1e-6
+
+
+def _apply_scaling_factors(
+    desired_num_replicas: Union[int, float],
+    current_num_replicas: int,
+    autoscaling_config: AutoscalingConfig,
+) -> int:
+    """Apply scaling factors to the desired number of replicas.
+    Returns the scaled number of replicas depending on the scaling factor.
+    The computation uses the difference between desired and current to scale.
+
+    """
+    # When scaling from zero, the scaling factor is not meaningful: the
+    # entire desired count would be treated as the delta and amplified,
+    # creating a feedback loop that compounds every control-loop tick.
+    # Return the raw desired value and let bounds handle the rest.
+    if current_num_replicas == 0:
+        return math.ceil(desired_num_replicas)
+
+    replicas_delta = desired_num_replicas - current_num_replicas
+    scaling_factor = (
+        autoscaling_config.get_upscaling_factor()
+        if replicas_delta > 0
+        else autoscaling_config.get_downscaling_factor()
+    )
+    scaled_num_replicas = math.ceil(
+        current_num_replicas + scaling_factor * replicas_delta
+    )
+    # If the scaled_replicas are stuck during downscaling because of scaling factor, decrement by 1.
+    if (
+        math.ceil(float(desired_num_replicas)) < current_num_replicas
+        and scaled_num_replicas == current_num_replicas
+    ):
+        scaled_num_replicas -= 1
+    return scaled_num_replicas
+
+
+def _apply_delay_logic(
+    desired_num_replicas: int,
+    curr_target_num_replicas: int,
+    config: AutoscalingConfig,
+    policy_state: Dict[str, Any],
+    _now: Optional[float] = None,
+) -> Tuple[int, Dict[str, Any]]:
+
+    """Apply delay logic to the desired number of replicas.
+
+    Uses wall-clock timestamps to measure delay instead of counting iterations,
+    so the effective delay matches the configured delay_s regardless of how long
+    each control loop iteration takes.
+    """
+    now = _now if _now is not None else time.time()
+    decision_num_replicas = curr_target_num_replicas
+    # decision_counter encodes direction: >0 means upscale, <0 means downscale.
+    # We keep it for backward-compatible state transitions but the actual delay
+    # check uses the timestamp.
+    decision_counter = policy_state.get(SERVE_AUTOSCALING_DECISION_COUNTERS_KEY, 0)
+    decision_timestamp = policy_state.get(
+        SERVE_AUTOSCALING_DECISION_TIMESTAMP_KEY, None
+    )
+
+    # Scale up.
+    if desired_num_replicas > curr_target_num_replicas:
+        # If the previous decision was to scale down, reset.
+        if decision_counter < 0:
+            decision_counter = 0
+            decision_timestamp = None
+        decision_counter += 1
+
+        # Record the timestamp when we first start wanting to scale up.
+        if decision_timestamp is None:
+            decision_timestamp = now
+
+        # Only actually scale the replicas if enough wall-clock time has
+        # elapsed since the first consecutive scale-up decision.
+        if now - decision_timestamp + _DELAY_ELAPSED_EPS_S >= config.upscale_delay_s:
+            decision_counter = 0
+            decision_timestamp = None
+            decision_num_replicas = desired_num_replicas
+
+    # Scale down.
+    elif desired_num_replicas < curr_target_num_replicas:
+        # If the previous decision was to scale up, reset.
+        if decision_counter > 0:
+            decision_counter = 0
+            decision_timestamp = None
+        decision_counter -= 1
+
+        # Downscaling to zero is only allowed from 1 -> 0
+        is_scaling_to_zero = curr_target_num_replicas == 1
+        # Determine the delay to use
+        if is_scaling_to_zero:
+            if config.downscale_to_zero_delay_s is not None:
+                delay_s = config.downscale_to_zero_delay_s
+            else:
+                delay_s = config.downscale_delay_s
+        else:
+            delay_s = config.downscale_delay_s
+            # The desired_num_replicas>0 for downscaling cases other than 1->0
+            desired_num_replicas = max(1, desired_num_replicas)
+
+        # Record the timestamp when we first start wanting to scale down.
+        if decision_timestamp is None:
+            decision_timestamp = now
+
+        # Only actually scale the replicas if enough wall-clock time has
+        # elapsed since the first consecutive scale-down decision.
+        if now - decision_timestamp + _DELAY_ELAPSED_EPS_S >= delay_s:
+            decision_counter = 0
+            decision_timestamp = None
+            decision_num_replicas = desired_num_replicas
+
+    # Do nothing.
+    else:
+        decision_counter = 0
+        decision_timestamp = None
+
+    policy_state[SERVE_AUTOSCALING_DECISION_COUNTERS_KEY] = decision_counter
+    policy_state[SERVE_AUTOSCALING_DECISION_TIMESTAMP_KEY] = decision_timestamp
+    return decision_num_replicas, policy_state
+
+
+def _apply_default_params(
+    desired_num_replicas: Union[int, float],
+    ctx: AutoscalingContext,
+    policy_state: Dict[str, Any],
+) -> Tuple[int, Dict[str, Any]]:
+    """Apply the default parameters to the desired number of replicas."""
+
+    desired_num_replicas = _apply_scaling_factors(
+        desired_num_replicas, ctx.current_num_replicas, ctx.config
+    )
+
+    # If curr num replicas is 0 and the policy wants to scale up (e.g. based on internal
+    # signals like queue length), bypass the delay logic for immediate scale-up.
+    if ctx.current_num_replicas == 0 and desired_num_replicas > 0:
+        return desired_num_replicas, policy_state
+
+    # Apply delay logic
+    # Only send the internal state here to avoid overwriting the custom policy state.
+    final_num_replicas, updated_state = _apply_delay_logic(
+        max(0, desired_num_replicas), ctx.target_num_replicas, ctx.config, policy_state
+    )
+
+    return final_num_replicas, updated_state
+
+
+def _extract_internal_policy_state(policy_state: Dict[str, Any]) -> Dict[str, Any]:
+    """Extract the internal states from a policy state dict."""
+    return {
+        SERVE_AUTOSCALING_DECISION_COUNTERS_KEY: policy_state.get(
+            SERVE_AUTOSCALING_DECISION_COUNTERS_KEY, 0
+        ),
+        SERVE_AUTOSCALING_DECISION_TIMESTAMP_KEY: policy_state.get(
+            SERVE_AUTOSCALING_DECISION_TIMESTAMP_KEY, None
+        ),
+    }
+
+
+def _apply_default_params_and_merge_state(
+    policy_state: Dict[str, Any],
+    user_policy_state: Dict[str, Any],
+    desired_num_replicas: Union[int, float],
+    ctx: AutoscalingContext,
+) -> Tuple[int, Dict[str, Any]]:
+
+    internal_policy_state = _extract_internal_policy_state(policy_state)
+    # Only pass the internal state used for delay counters so we don't
+    # overwrite any custom user state.
+    final_num_replicas, updated_state = _apply_default_params(
+        desired_num_replicas, ctx, internal_policy_state
+    )
+    # Merge internal updated_state with the user's custom policy state.
+    if updated_state:
+        user_policy_state.update(updated_state)
+    return final_num_replicas, user_policy_state
+
+
+def _merge_user_state_with_internal_state(
+    policy_state: Dict[str, Any],
+    user_policy_state: Dict[str, Any],
+) -> Dict[str, Any]:
+    """Merge user state with previous policy state, preserving internal keys.
+
+    This mutates and returns `user_policy_state`.
+    """
+    internal_policy_state = _extract_internal_policy_state(policy_state)
+    user_policy_state.update(internal_policy_state)
+    return user_policy_state
+
+
+def _get_cold_start_scale_up_replicas(ctx: AutoscalingContext) -> Optional[int]:
+    """
+    Returns the desired number of replicas if the cold start fast path applies, otherwise returns None.
+    """
+    if ctx.current_num_replicas == 0 and ctx.total_num_requests > 0:
+        return max(
+            math.ceil(1 * ctx.config.get_upscaling_factor()),
+            ctx.target_num_replicas,
+        )
+
+    return None
+
+
+def _apply_autoscaling_config(
+    policy_func: Callable[
+        [AutoscalingContext], Tuple[Union[int, float], Dict[str, Any]]
+    ]
+) -> Callable[[AutoscalingContext], Tuple[int, Dict[str, Any]]]:
+    """
+    Wraps a custom policy function to automatically apply:
+    - upscaling_factor / downscaling_factor
+    - min_replicas / max_replicas bounds
+    - upscale_delay_s / downscale_delay_s / downscale_to_zero_delay_s
+    """
+
+    @functools.wraps(policy_func)
+    def wrapped_policy(ctx: AutoscalingContext) -> Tuple[int, Dict[str, Any]]:
+
+        # Cold start fast path: 0 replicas bypasses delay logic for immediate scale-up
+        cold_start_replicas = _get_cold_start_scale_up_replicas(ctx)
+        if cold_start_replicas is not None:
+            return cold_start_replicas, ctx.policy_state
+        policy_state = ctx.policy_state.copy()
+        desired_num_replicas, updated_custom_policy_state = policy_func(ctx)
+        final_num_replicas, final_state = _apply_default_params_and_merge_state(
+            policy_state, updated_custom_policy_state, desired_num_replicas, ctx
+        )
+
+        return final_num_replicas, final_state
+
+    return wrapped_policy
+
+
+def _apply_app_level_autoscaling_config(
+    policy_func: Callable[
+        [Dict[DeploymentID, AutoscalingContext]],
+        Tuple[
+            Dict[DeploymentID, Union[int, float]],
+            Optional[Dict[DeploymentID, Dict]],
+        ],
+    ]
+) -> Callable[
+    [Dict[DeploymentID, AutoscalingContext]],
+    Tuple[Dict[DeploymentID, int], Dict[DeploymentID, Dict]],
+]:
+    """
+    Wraps an application-level custom policy function to automatically apply per-deployment:
+    - upscaling_factor / downscaling_factor
+    - min_replicas / max_replicas bounds
+    - upscale_delay_s / downscale_delay_s / downscale_to_zero_delay_s
+    """
+
+    @functools.wraps(policy_func)
+    def wrapped_policy(
+        contexts: Dict[DeploymentID, AutoscalingContext]
+    ) -> Tuple[Dict[DeploymentID, int], Dict[DeploymentID, Dict]]:
+
+        # Store the policy state per deployment
+        state_per_deployment = {}
+        for dep_id, ctx in contexts.items():
+            state_per_deployment[dep_id] = ctx.policy_state.copy()
+
+        # Send to the actual policy
+        desired_num_replicas_dict, updated_custom_policy_state = policy_func(contexts)
+        updated_custom_policy_state = updated_custom_policy_state or {}
+
+        # Build per-deployment replicas count and state dictionary.
+        final_decisions: Dict[DeploymentID, int] = {}
+        final_state: Dict[DeploymentID, Dict] = {}
+        for dep_id, ctx in contexts.items():
+            custom_policy_state_per_deployment = (
+                updated_custom_policy_state.get(dep_id) or {}
+            ).copy()
+            if dep_id not in desired_num_replicas_dict:
+                final_state[dep_id] = _merge_user_state_with_internal_state(
+                    state_per_deployment[dep_id],
+                    custom_policy_state_per_deployment,
+                )
+                continue
+            # Cold start fast path: 0 replicas bypasses delay logic for immediate scale-up
+            cold_start_replicas = _get_cold_start_scale_up_replicas(ctx)
+            if cold_start_replicas is not None:
+                final_decisions[dep_id] = cold_start_replicas
+                # Merge user policy state with internal policy state
+                final_state[dep_id] = _merge_user_state_with_internal_state(
+                    state_per_deployment[dep_id],
+                    custom_policy_state_per_deployment,
+                )
+                continue
+            final_num_replicas, final_dep_state = _apply_default_params_and_merge_state(
+                state_per_deployment[dep_id],
+                custom_policy_state_per_deployment,
+                desired_num_replicas_dict[dep_id],
+                ctx,
+            )
+            final_decisions[dep_id] = final_num_replicas
+            final_state[dep_id] = final_dep_state
+        return final_decisions, final_state
+
+    return wrapped_policy
+
+
+def _core_replica_queue_length_policy(
+    ctx: AutoscalingContext,
+) -> Tuple[float, Dict[str, Any]]:
+    num_running_replicas = ctx.current_num_replicas
+    config = ctx.config
+    if num_running_replicas == 0:
+        return ctx.target_num_replicas, {}
+    target_num_requests = config.get_target_ongoing_requests() * num_running_replicas
+    error_ratio = ctx.total_num_requests / target_num_requests
+    desired_num_replicas = num_running_replicas * error_ratio
+    return desired_num_replicas, {}
+
+
+@PublicAPI(stability="stable")
+def replica_queue_length_autoscaling_policy(
+    ctx: AutoscalingContext,
+) -> Tuple[Union[int, float], Dict[str, Any]]:
+    """The default autoscaling policy based on basic thresholds for scaling.
+    There is a minimum threshold for the average queue length in the cluster
+    to scale up and a maximum threshold to scale down. Each period, a 'scale
+    up' or 'scale down' decision is made. This decision must be made for a
+    specified number of periods in a row before the number of replicas is
+    actually scaled. See config options for more details.  Assumes
+    `get_decision_num_replicas` is called once every CONTROL_LOOP_PERIOD_S
+    seconds.
+    """
+    # Adding this guard makes the public policy safe to call directly.
+    cold_start_replicas = _get_cold_start_scale_up_replicas(ctx)
+    if cold_start_replicas is not None:
+        return cold_start_replicas, ctx.policy_state
+    return _core_replica_queue_length_policy(ctx)
+
+
+default_autoscaling_policy = replica_queue_length_autoscaling_policy

python/ray/serve/context.py

@@ -0,0 +1,475 @@
+"""
+This file stores global state for a Serve application. Deployment replicas
+can use this state to access metadata or the Serve controller.
+"""
+
+import asyncio
+import contextvars
+import logging
+import os
+from collections import defaultdict
+from dataclasses import dataclass
+from typing import Callable, Dict, List, Optional
+
+import ray
+from ray.serve._private.client import ServeControllerClient
+from ray.serve._private.common import DeploymentID, ReplicaID
+from ray.serve._private.config import DeploymentConfig
+from ray.serve._private.constants import (
+    RAY_SERVE_INTERNAL_DEPLOYMENT_ACTOR_NAME_ENV_VAR,
+    RAY_SERVE_INTERNAL_DEPLOYMENT_APP_NAME_ENV_VAR,
+    RAY_SERVE_INTERNAL_DEPLOYMENT_CODE_VERSION_ENV_VAR,
+    RAY_SERVE_INTERNAL_DEPLOYMENT_NAME_ENV_VAR,
+    SERVE_CONTROLLER_NAME,
+    SERVE_LOGGER_NAME,
+    SERVE_NAMESPACE,
+)
+from ray.serve._private.replica_result import ReplicaResult
+from ray.serve._private.utils import get_deployment_actor_name
+from ray.serve.exceptions import RayServeException
+from ray.serve.gang import GangContext
+from ray.serve.grpc_util import RayServegRPCContext
+from ray.serve.schema import ReplicaRank
+from ray.util.annotations import DeveloperAPI
+
+logger = logging.getLogger(SERVE_LOGGER_NAME)
+
+_INTERNAL_REPLICA_CONTEXT: "ReplicaContext" = None
+_INTERNAL_DEPLOYMENT_ACTOR_CONTEXT: "DeploymentActorContext" = None
+_global_client: ServeControllerClient = None
+# Ray job id (driver session) that created the cached client above. ray.shutdown()
+# leaves this module global intact, so after a reconnect (new job id) the cached
+# handle is unusable and must not be reused. See #64647.
+_global_client_job_id: Optional[str] = None
+
+
+@DeveloperAPI
+@dataclass
+class ReplicaContext:
+    """Stores runtime context info for replicas.
+
+    Fields:
+        - app_name: name of the application the replica is a part of.
+        - deployment: name of the deployment the replica is a part of.
+        - replica_tag: unique ID for the replica.
+        - servable_object: instance of the user class/function this replica is running.
+        - rank: the rank of the replica.
+        - world_size: the number of replicas in the deployment.
+        - gang_context: context information for the gang the replica is part of.
+        - code_version: code version of the deployment (for get_deployment_actor).
+    """
+
+    replica_id: ReplicaID
+    servable_object: Callable
+    _deployment_config: DeploymentConfig
+    rank: ReplicaRank
+    world_size: int
+    _handle_registration_callback: Optional[Callable[[DeploymentID], None]] = None
+    gang_context: Optional[GangContext] = None
+    code_version: Optional[str] = None
+
+    @property
+    def app_name(self) -> str:
+        return self.replica_id.deployment_id.app_name
+
+    @property
+    def deployment(self) -> str:
+        return self.replica_id.deployment_id.name
+
+    @property
+    def replica_tag(self) -> str:
+        return self.replica_id.unique_id
+
+
+@DeveloperAPI
+@dataclass
+class DeploymentActorContext:
+    """Stores runtime context info for deployment-scoped actors."""
+
+    deployment_id: DeploymentID
+    actor_name: str
+    code_version: Optional[str] = None
+
+    @property
+    def app_name(self) -> str:
+        return self.deployment_id.app_name
+
+    @property
+    def deployment(self) -> str:
+        return self.deployment_id.name
+
+
+def _get_global_client(
+    raise_if_no_controller_running: bool = True,
+) -> Optional[ServeControllerClient]:
+    """Gets the global client, which stores the controller's handle.
+
+    Args:
+        raise_if_no_controller_running: Whether to raise an exception if
+            there is no currently running Serve controller.
+
+    Returns:
+        ServeControllerClient to the running Serve controller. If there
+        is no running controller and raise_if_no_controller_running is
+        set to False, returns None.
+
+    Raises:
+        RayServeException: If there is no running Serve controller actor
+            and raise_if_no_controller_running is set to True.
+    """
+
+    if _cached_client_from_current_session():
+        return _global_client
+
+    # No usable cache (never set, or left over from a previous driver session).
+    # Drop any stale handle and rediscover the controller by name.
+    _disconnect()
+    return _connect(raise_if_no_controller_running)
+
+
+def _check_cached_client_alive() -> tuple:
+    """Health-check the cached controller client.
+
+    Returns:
+        (client, had_cached) tuple.
+        - ``(client, True)``: cached client from the current session is alive.
+        - ``(None, True)``: cached client from the current session existed but
+          is unreachable; the cache has been cleared. Callers should **not**
+          attempt to reconnect via ``_connect()`` because GCS is likely dead and
+          ``ray.get_actor()`` would hang until the 60-second C++ GCS
+          reconnection timeout kills the process.
+        - ``(None, False)``: no usable cached client (never set, or left over
+          from a previous driver session). Callers may safely call
+          ``_get_global_client()`` to discover a running controller.
+    """
+
+    if not _cached_client_from_current_session():
+        # Either nothing cached, or a handle left behind by a previous driver
+        # session. In the latter case GCS is alive (we are connected now), so it
+        # is safe for callers to reconnect. Drop any stale handle.
+        _disconnect()
+        return None, False
+
+    try:
+        ray.get(_global_client._controller.check_alive.remote(), timeout=5)
+        return _global_client, True
+    except Exception as e:
+        logger.info(f"The cached controller has died or is unreachable: {e}.")
+        _disconnect()
+        return None, True
+
+
+def _set_global_client(client):
+    global _global_client, _global_client_job_id
+    _global_client = client
+    _global_client_job_id = (
+        ray.get_runtime_context().get_job_id() if client is not None else None
+    )
+
+
+def _disconnect():
+    """Forget the cached controller client for this driver session.
+
+    Mirrors ``_connect()``. This does **not** shut Serve down on the cluster; it
+    only drops the local cached handle so the next call rediscovers the
+    controller by name.
+    """
+    _set_global_client(None)
+
+
+def _cached_client_from_current_session() -> bool:
+    """Whether a usable cached client exists for the current Ray session.
+
+    ``_global_client`` is a module global that survives ``ray.shutdown()``, so a
+    handle cached by a previous driver session cannot be used after the driver
+    reconnects (which yields a new job id). Comparing job ids detects that case.
+    """
+    if _global_client is None or not ray.is_initialized():
+        return False
+    return _global_client_job_id == ray.get_runtime_context().get_job_id()
+
+
+def _get_internal_replica_context():
+    return _INTERNAL_REPLICA_CONTEXT
+
+
+def _get_internal_deployment_actor_context():
+    global _INTERNAL_DEPLOYMENT_ACTOR_CONTEXT
+
+    if _INTERNAL_DEPLOYMENT_ACTOR_CONTEXT is not None:
+        return _INTERNAL_DEPLOYMENT_ACTOR_CONTEXT
+
+    app_name = os.environ.get(RAY_SERVE_INTERNAL_DEPLOYMENT_APP_NAME_ENV_VAR)
+    deployment_name = os.environ.get(RAY_SERVE_INTERNAL_DEPLOYMENT_NAME_ENV_VAR)
+    actor_name = os.environ.get(RAY_SERVE_INTERNAL_DEPLOYMENT_ACTOR_NAME_ENV_VAR)
+
+    if app_name is None or deployment_name is None or actor_name is None:
+        return None
+
+    _INTERNAL_DEPLOYMENT_ACTOR_CONTEXT = DeploymentActorContext(
+        deployment_id=DeploymentID(name=deployment_name, app_name=app_name),
+        actor_name=actor_name,
+        code_version=os.environ.get(RAY_SERVE_INTERNAL_DEPLOYMENT_CODE_VERSION_ENV_VAR),
+    )
+    return _INTERNAL_DEPLOYMENT_ACTOR_CONTEXT
+
+
+def _get_deployment_actor(actor_name: str):
+    """Get a handle to a deployment-scoped actor by name.
+
+    Thin wrapper around ``ray.get_actor`` with the Serve deployment-actor naming
+    convention. See ``serve.get_deployment_actor`` docstring for behavior,
+    ``ValueError``/``RayActorError`` expectations, and refresh patterns.
+    """
+    internal_context = _get_internal_replica_context()
+    if internal_context is None:
+        raise RayServeException(
+            "`serve.get_deployment_actor()` may only be called from within "
+            "a Ray Serve deployment replica."
+        )
+    deployment_id = internal_context.replica_id.deployment_id
+    return ray.get_actor(
+        get_deployment_actor_name(
+            deployment_id,
+            actor_name,
+            code_version=internal_context.code_version,
+        ),
+        namespace=SERVE_NAMESPACE,
+    )
+
+
+def _set_internal_replica_context(
+    *,
+    replica_id: ReplicaID,
+    servable_object: Callable,
+    _deployment_config: DeploymentConfig,
+    rank: ReplicaRank,
+    world_size: int,
+    handle_registration_callback: Optional[Callable[[str, str], None]] = None,
+    gang_context: Optional[GangContext] = None,
+    code_version: Optional[str] = None,
+):
+    global _INTERNAL_REPLICA_CONTEXT
+    _INTERNAL_REPLICA_CONTEXT = ReplicaContext(
+        replica_id=replica_id,
+        servable_object=servable_object,
+        _deployment_config=_deployment_config,
+        rank=rank,
+        world_size=world_size,
+        _handle_registration_callback=handle_registration_callback,
+        gang_context=gang_context,
+        code_version=code_version,
+    )
+
+
+def _connect(raise_if_no_controller_running: bool = True) -> ServeControllerClient:
+    """Connect to an existing Serve application on this Ray cluster.
+
+    If called from within a replica, this will connect to the same Serve
+    app that the replica is running in.
+
+    Args:
+        raise_if_no_controller_running: If ``True``, raise when no Serve
+            controller actor is found. If ``False``, return ``None`` instead.
+
+    Returns:
+        ServeControllerClient that encapsulates a Ray actor handle to the
+        existing Serve application's Serve Controller. None if there is
+        no running Serve controller actor and raise_if_no_controller_running
+        is set to False.
+
+    Raises:
+        RayServeException: If there is no running Serve controller actor
+            and raise_if_no_controller_running is set to True.
+    """
+
+    # Initialize ray if needed.
+    ray._private.worker.global_worker._filter_logs_by_job = False
+    if not ray.is_initialized():
+        ray.init(namespace=SERVE_NAMESPACE)
+
+    # Try to get serve controller if it exists
+    try:
+        controller = ray.get_actor(SERVE_CONTROLLER_NAME, namespace=SERVE_NAMESPACE)
+    except ValueError:
+        if raise_if_no_controller_running:
+            raise RayServeException(
+                "There is no Serve instance running on this Ray cluster."
+            )
+        return
+
+    client = ServeControllerClient(
+        controller,
+    )
+    _set_global_client(client)
+    return client
+
+
+# Serve request context var which is used for storing the internal
+# request context information.
+# route_prefix: http url route path, e.g. http://127.0.0.1:/app
+#     the route is "/app". When you send requests by handle,
+#     the route is empty.
+# request_id: the request id is generated from http proxy, the value
+#     shouldn't be changed when the variable is set.
+#     This can be from the client and is used for logging.
+# _internal_request_id: the request id is generated from the proxy. Used to track the
+#     request objects in the system.
+# note:
+#   The request context is readonly to avoid potential
+#       async task conflicts when using it concurrently.
+
+
+@dataclass(frozen=True)
+class _RequestContext:
+    route: str = ""
+    request_id: str = ""
+    _internal_request_id: str = ""
+    app_name: str = ""
+    multiplexed_model_id: str = ""
+    session_id: str = ""
+    grpc_context: Optional[RayServegRPCContext] = None
+    is_http_request: bool = False
+    cancel_on_parent_request_cancel: bool = False
+    # The client address in "host:port" format, if available.
+    _client: str = ""
+    # Ray tracing context for this request (if tracing is enabled)
+    # This is extracted from _ray_trace_ctx kwarg at the replica entry point
+    # Advanced users can access this to propagate tracing to external systems
+    _ray_trace_ctx: Optional[dict] = None
+
+
+_serve_request_context = contextvars.ContextVar(
+    "Serve internal request context variable", default=None
+)
+
+_serve_batch_request_context = contextvars.ContextVar(
+    "Serve internal batching request context variable", default=None
+)
+
+
+def _get_serve_request_context():
+    """Get the current request context.
+
+    Returns:
+        The current request context
+    """
+
+    if _serve_request_context.get() is None:
+        _serve_request_context.set(_RequestContext())
+    return _serve_request_context.get()
+
+
+def _get_serve_batch_request_context():
+    """Get the list of request contexts for the current batch."""
+    if _serve_batch_request_context.get() is None:
+        _serve_batch_request_context.set([])
+    return _serve_batch_request_context.get()
+
+
+def _set_request_context(
+    route: str = "",
+    request_id: str = "",
+    _internal_request_id: str = "",
+    app_name: str = "",
+    multiplexed_model_id: str = "",
+):
+    """Set the request context. If the value is not set,
+    the current context value will be used."""
+
+    current_request_context = _get_serve_request_context()
+
+    _serve_request_context.set(
+        _RequestContext(
+            route=route or current_request_context.route,
+            request_id=request_id or current_request_context.request_id,
+            _internal_request_id=_internal_request_id
+            or current_request_context._internal_request_id,
+            app_name=app_name or current_request_context.app_name,
+            multiplexed_model_id=multiplexed_model_id
+            or current_request_context.multiplexed_model_id,
+        )
+    )
+
+
+def _unset_request_context():
+    """Unset the request context."""
+    _serve_request_context.set(_RequestContext())
+
+
+def _set_batch_request_context(request_contexts: List[_RequestContext]):
+    """Add the request context to the batch request context."""
+    _serve_batch_request_context.set(request_contexts)
+
+
+# `_requests_pending_assignment` is a map from request ID to a
+# dictionary of asyncio tasks.
+# The request ID points to an ongoing request that is executing on the
+# current replica, and the asyncio tasks are ongoing tasks started on
+# the router to assign child requests to downstream replicas.
+
+# A dictionary is used over a set to track the asyncio tasks for more
+# efficient addition and deletion time complexity. A uniquely generated
+# `response_id` is used to identify each task.
+
+_requests_pending_assignment: Dict[str, Dict[str, asyncio.Task]] = defaultdict(dict)
+
+
+# Note that the functions below that manipulate
+# `_requests_pending_assignment` are NOT thread-safe. They are only
+# expected to be called from the same thread/asyncio event-loop.
+
+
+def _get_requests_pending_assignment(parent_request_id: str) -> Dict[str, asyncio.Task]:
+    if parent_request_id in _requests_pending_assignment:
+        return _requests_pending_assignment[parent_request_id]
+
+    return {}
+
+
+def _add_request_pending_assignment(parent_request_id: str, response_id: str, task):
+    # NOTE: `parent_request_id` is the `internal_request_id` corresponding
+    # to an ongoing Serve request, so it is always non-empty.
+    _requests_pending_assignment[parent_request_id][response_id] = task
+
+
+def _remove_request_pending_assignment(parent_request_id: str, response_id: str):
+    if response_id in _requests_pending_assignment[parent_request_id]:
+        del _requests_pending_assignment[parent_request_id][response_id]
+
+    if len(_requests_pending_assignment[parent_request_id]) == 0:
+        del _requests_pending_assignment[parent_request_id]
+
+
+# `_in_flight_requests` is a map from request ID to a dictionary of replica results.
+# The request ID points to an ongoing Serve request, and the replica results are
+# in-flight child requests that have been assigned to a downstream replica.
+
+# A dictionary is used over a set to track the replica results for more
+# efficient addition and deletion time complexity. A uniquely generated
+# `response_id` is used to identify each replica result.
+
+_in_flight_requests: Dict[str, Dict[str, ReplicaResult]] = defaultdict(dict)
+
+# Note that the functions below that manipulate `_in_flight_requests`
+# are NOT thread-safe. They are only expected to be called from the
+# same thread/asyncio event-loop.
+
+
+def _get_in_flight_requests(parent_request_id):
+    if parent_request_id in _in_flight_requests:
+        return _in_flight_requests[parent_request_id]
+
+    return {}
+
+
+def _add_in_flight_request(parent_request_id, response_id, replica_result):
+    _in_flight_requests[parent_request_id][response_id] = replica_result
+
+
+def _remove_in_flight_request(parent_request_id, response_id):
+    if response_id in _in_flight_requests[parent_request_id]:
+        del _in_flight_requests[parent_request_id][response_id]
+
+    if len(_in_flight_requests[parent_request_id]) == 0:
+        del _in_flight_requests[parent_request_id]

python/ray/serve/dag.py

@@ -0,0 +1,5 @@
+from ray.dag.input_node import InputNode
+
+__all__ = [
+    "InputNode",
+]

python/ray/serve/deployment.py

@@ -0,0 +1,590 @@
+import warnings
+from copy import deepcopy
+from typing import Any, Callable, Dict, List, Optional, Tuple, Union
+
+from ray.serve._private.config import (
+    DeploymentConfig,
+    ReplicaConfig,
+    RequestRouterConfig,
+    handle_num_replicas_auto,
+)
+from ray.serve._private.usage import ServeUsageTag
+from ray.serve._private.utils import DEFAULT, Default
+from ray.serve.config import (
+    AutoscalingConfig,
+    DeploymentActorConfig,
+    GangSchedulingConfig,
+)
+from ray.serve.schema import DeploymentSchema, LoggingConfig, RayActorOptionsSchema
+from ray.util.annotations import PublicAPI
+
+
+@PublicAPI(stability="stable")
+class Application:
+    """One or more deployments bound with arguments that can be deployed together.
+
+    Can be passed into another `Deployment.bind()` to compose multiple deployments in a
+    single application, passed to `serve.run`, or deployed via a Serve config file.
+
+    For example, to define an Application and run it in Python:
+
+        .. code-block:: python
+
+            from ray import serve
+            from ray.serve import Application
+
+            @serve.deployment
+            class MyDeployment:
+                pass
+
+            app: Application = MyDeployment.bind(OtherDeployment.bind())
+            serve.run(app)
+
+    To run the same app using the command line interface (CLI):
+
+        .. code-block:: bash
+
+            serve run python_file:app
+
+    To deploy the same app via a config file:
+
+        .. code-block:: yaml
+
+            applications:
+                my_app:
+                    import_path: python_file:app
+
+    """
+
+    def __init__(self, bound_deployment: "Deployment"):
+        # This is used by `build_app`, but made private so users don't use it.
+        self._bound_deployment = bound_deployment
+        # Optional peer ingress request router for ingress bypass mode.
+        self._ingress_request_router: Optional["Application"] = None
+
+    def _with_ingress_request_router(
+        self, ingress_request_router: "Application"
+    ) -> "Application":
+        # Internal-only, unstable hook for the Serve LLM direct-ingress stack.
+        # This is not a stable public Serve API.
+        self._ingress_request_router = ingress_request_router
+        return self
+
+
+@PublicAPI(stability="stable")
+class Deployment:
+    """Class (or function) decorated with the `@serve.deployment` decorator.
+
+    This is run on a number of replica actors. Requests to those replicas call
+    this class.
+
+    One or more deployments can be composed together into an `Application` which is
+    then run via `serve.run` or a config file.
+
+    Example:
+
+    .. code-block:: python
+
+        @serve.deployment
+        class MyDeployment:
+            def __init__(self, name: str):
+                self._name = name
+
+            def __call__(self, request):
+                return "Hello world!"
+
+        app = MyDeployment.bind()
+        # Run via `serve.run` or the `serve run` CLI command.
+        serve.run(app)
+
+    """
+
+    def __init__(
+        self,
+        name: str,
+        deployment_config: DeploymentConfig,
+        replica_config: ReplicaConfig,
+        version: Optional[str] = None,
+        _internal: bool = False,
+    ) -> None:
+        """Construct a Deployment. Should only be called by Serve internals.
+
+        Args:
+            name: Unique name of this deployment.
+            deployment_config: Serve-level configuration (number of replicas,
+                user config, autoscaling, etc.).
+            replica_config: Replica-level configuration (actor options, init
+                args/kwargs, etc.).
+            version: Optional opaque deployment version used to determine
+                whether replicas need to be restarted on update.
+            _internal: Internal flag; ``Deployment`` instances must be created
+                via the ``@serve.deployment`` decorator, which sets this to
+                ``True``.
+        """
+        if not _internal:
+            raise RuntimeError(
+                "The Deployment constructor should not be called "
+                "directly. Use `@serve.deployment` instead."
+            )
+        self._validate_name(name)
+        if not (version is None or isinstance(version, str)):
+            raise TypeError("version must be a string.")
+
+        self._name = name
+        self._version = version
+        self._deployment_config = deployment_config
+        self._replica_config = replica_config
+
+    def _validate_name(self, name: str):
+        if not isinstance(name, str):
+            raise TypeError("name must be a string.")
+
+        # name does not contain #
+        if "#" in name:
+            warnings.warn(
+                f"Deployment names should not contain the '#' character, this will raise an error starting in Ray 2.46.0. "
+                f"Current name: {name}."
+            )
+
+    @property
+    def name(self) -> str:
+        """Unique name of this deployment."""
+        return self._name
+
+    @property
+    def version(self) -> Optional[str]:
+        return self._version
+
+    @property
+    def func_or_class(self) -> Union[Callable, str]:
+        """Underlying class or function that this deployment wraps."""
+        return self._replica_config.deployment_def
+
+    @property
+    def num_replicas(self) -> int:
+        """Target number of replicas."""
+        return self._deployment_config.num_replicas
+
+    @property
+    def user_config(self) -> Any:
+        """Dynamic user-provided config options."""
+        return self._deployment_config.user_config
+
+    @property
+    def max_ongoing_requests(self) -> int:
+        """Max number of requests a replica can handle at once."""
+        return self._deployment_config.max_ongoing_requests
+
+    @property
+    def max_queued_requests(self) -> int:
+        """Max number of requests that can be queued in each deployment handle."""
+        return self._deployment_config.max_queued_requests
+
+    @property
+    def ray_actor_options(self) -> Optional[Dict]:
+        """Actor options such as resources required for each replica."""
+        return self._replica_config.ray_actor_options
+
+    @property
+    def init_args(self) -> Tuple[Any]:
+        return self._replica_config.init_args
+
+    @property
+    def init_kwargs(self) -> Tuple[Any]:
+        return self._replica_config.init_kwargs
+
+    @property
+    def logging_config(self) -> Dict:
+        return self._deployment_config.logging_config
+
+    def set_logging_config(self, logging_config: Dict):
+        self._deployment_config.logging_config = logging_config
+
+    def __call__(self):
+        raise RuntimeError(
+            "Deployments cannot be constructed directly. "
+            "Use `deployment.deploy() instead.`"
+        )
+
+    def bind(self, *args, **kwargs) -> Application:
+        """Bind the arguments to the deployment and return an Application.
+
+        The returned Application can be deployed using `serve.run` (or via
+        config file) or bound to another deployment for composition.
+        """
+        return Application(self.options(_init_args=args, _init_kwargs=kwargs))
+
+    def options(
+        self,
+        func_or_class: Optional[Callable] = None,
+        name: Default[str] = DEFAULT.VALUE,
+        version: Default[str] = DEFAULT.VALUE,
+        num_replicas: Default[Optional[Union[int, str]]] = DEFAULT.VALUE,
+        ray_actor_options: Default[Optional[Dict]] = DEFAULT.VALUE,
+        placement_group_bundles: Default[List[Dict[str, float]]] = DEFAULT.VALUE,
+        placement_group_strategy: Default[str] = DEFAULT.VALUE,
+        placement_group_bundle_label_selector: Default[
+            List[Dict[str, str]]
+        ] = DEFAULT.VALUE,
+        max_replicas_per_node: Default[int] = DEFAULT.VALUE,
+        user_config: Default[Optional[Any]] = DEFAULT.VALUE,
+        max_ongoing_requests: Default[int] = DEFAULT.VALUE,
+        max_queued_requests: Default[int] = DEFAULT.VALUE,
+        autoscaling_config: Default[
+            Union[Dict, AutoscalingConfig, None]
+        ] = DEFAULT.VALUE,
+        graceful_shutdown_wait_loop_s: Default[float] = DEFAULT.VALUE,
+        graceful_shutdown_timeout_s: Default[float] = DEFAULT.VALUE,
+        health_check_period_s: Default[float] = DEFAULT.VALUE,
+        health_check_timeout_s: Default[float] = DEFAULT.VALUE,
+        logging_config: Default[Union[Dict, LoggingConfig, None]] = DEFAULT.VALUE,
+        request_router_config: Default[
+            Union[Dict, RequestRouterConfig, None]
+        ] = DEFAULT.VALUE,
+        _init_args: Default[Tuple[Any]] = DEFAULT.VALUE,
+        _init_kwargs: Default[Dict[Any, Any]] = DEFAULT.VALUE,
+        _internal: bool = False,
+        max_constructor_retry_count: Default[int] = DEFAULT.VALUE,
+        gang_scheduling_config: Default[
+            Union[Dict, GangSchedulingConfig, None]
+        ] = DEFAULT.VALUE,
+        deployment_actors: Default[
+            Optional[List[Union[Dict, DeploymentActorConfig]]]
+        ] = DEFAULT.VALUE,
+    ) -> "Deployment":
+        """Return a copy of this deployment with updated options.
+
+        Only those options passed in will be updated, all others will remain
+        unchanged from the existing deployment.
+
+        Refer to the `@serve.deployment` decorator docs for available arguments.
+        """
+        if not _internal and version is not DEFAULT.VALUE:
+            raise ValueError(
+                "`version` in `Deployment.options()` has been removed. "
+                "Serve manages deployment versions internally."
+            )
+
+        # Modify max_ongoing_requests and autoscaling_config if
+        # `num_replicas="auto"`
+        if max_ongoing_requests is None:
+            raise ValueError("`max_ongoing_requests` must be non-null, got None.")
+
+        if num_replicas == "auto":
+            max_ongoing_requests, autoscaling_config = handle_num_replicas_auto(
+                max_ongoing_requests, autoscaling_config
+            )
+
+            ServeUsageTag.AUTO_NUM_REPLICAS_USED.record("1")
+
+        # NOTE: The user_configured_option_names should be the first thing that's
+        # defined in this method. It depends on the locals() dictionary storing
+        # only the function args/kwargs.
+        # Create list of all user-configured options from keyword args
+        user_configured_option_names = [
+            option
+            for option, value in locals().items()
+            if option not in {"self", "func_or_class", "_internal"}
+            and value is not DEFAULT.VALUE
+        ]
+
+        new_deployment_config = deepcopy(self._deployment_config)
+        if not _internal:
+            new_deployment_config.user_configured_option_names.update(
+                user_configured_option_names
+            )
+
+        if num_replicas not in [
+            DEFAULT.VALUE,
+            None,
+            "auto",
+        ] and autoscaling_config not in [
+            DEFAULT.VALUE,
+            None,
+        ]:
+            raise ValueError(
+                "Manually setting num_replicas is not allowed when "
+                "autoscaling_config is provided."
+            )
+
+        if num_replicas == 0:
+            raise ValueError("num_replicas is expected to larger than 0")
+
+        if num_replicas not in [DEFAULT.VALUE, None, "auto"]:
+            new_deployment_config.num_replicas = num_replicas
+
+        if user_config is not DEFAULT.VALUE:
+            new_deployment_config.user_config = user_config
+
+        if max_ongoing_requests is not DEFAULT.VALUE:
+            new_deployment_config.max_ongoing_requests = max_ongoing_requests
+
+        if max_queued_requests is not DEFAULT.VALUE:
+            new_deployment_config.max_queued_requests = max_queued_requests
+
+        if max_constructor_retry_count is not DEFAULT.VALUE:
+            new_deployment_config.max_constructor_retry_count = (
+                max_constructor_retry_count
+            )
+
+        if func_or_class is None:
+            func_or_class = self._replica_config.deployment_def
+
+        if name is DEFAULT.VALUE:
+            name = self._name
+
+        if version is DEFAULT.VALUE:
+            version = self._version
+
+        if _init_args is DEFAULT.VALUE:
+            _init_args = self._replica_config.init_args
+
+        if _init_kwargs is DEFAULT.VALUE:
+            _init_kwargs = self._replica_config.init_kwargs
+
+        if ray_actor_options is DEFAULT.VALUE:
+            ray_actor_options = self._replica_config.ray_actor_options
+
+        if placement_group_bundles is DEFAULT.VALUE:
+            placement_group_bundles = self._replica_config.placement_group_bundles
+
+        if placement_group_strategy is DEFAULT.VALUE:
+            placement_group_strategy = self._replica_config.placement_group_strategy
+
+        if placement_group_bundle_label_selector is DEFAULT.VALUE:
+            placement_group_bundle_label_selector = (
+                self._replica_config.placement_group_bundle_label_selector
+            )
+
+        # TODO(ryanaoleary@): Add conditional check once fallback_strategy is
+        # added to placement group options.
+        placement_group_fallback_strategy = (
+            self._replica_config.placement_group_fallback_strategy
+        )
+
+        if max_replicas_per_node is DEFAULT.VALUE:
+            max_replicas_per_node = self._replica_config.max_replicas_per_node
+
+        if autoscaling_config is not DEFAULT.VALUE:
+            new_deployment_config.autoscaling_config = autoscaling_config
+
+        if request_router_config is not DEFAULT.VALUE:
+            new_deployment_config.request_router_config = request_router_config
+
+        if graceful_shutdown_wait_loop_s is not DEFAULT.VALUE:
+            new_deployment_config.graceful_shutdown_wait_loop_s = (
+                graceful_shutdown_wait_loop_s
+            )
+
+        if graceful_shutdown_timeout_s is not DEFAULT.VALUE:
+            new_deployment_config.graceful_shutdown_timeout_s = (
+                graceful_shutdown_timeout_s
+            )
+
+        if health_check_period_s is not DEFAULT.VALUE:
+            new_deployment_config.health_check_period_s = health_check_period_s
+
+        if health_check_timeout_s is not DEFAULT.VALUE:
+            new_deployment_config.health_check_timeout_s = health_check_timeout_s
+
+        if logging_config is not DEFAULT.VALUE:
+            if isinstance(logging_config, LoggingConfig):
+                logging_config = logging_config.model_dump()
+            new_deployment_config.logging_config = logging_config
+
+        if gang_scheduling_config is not DEFAULT.VALUE:
+            new_deployment_config.gang_scheduling_config = gang_scheduling_config
+
+        if deployment_actors is not DEFAULT.VALUE:
+            new_deployment_config.deployment_actors = deployment_actors
+
+        gc = new_deployment_config.gang_scheduling_config
+        if (
+            gc is not None
+            and isinstance(new_deployment_config.num_replicas, int)
+            and new_deployment_config.autoscaling_config is None
+        ):
+            # When autoscaling is enabled, num_replicas defaults to 1
+            if new_deployment_config.num_replicas % gc.gang_size != 0:
+                raise ValueError(
+                    f"num_replicas ({new_deployment_config.num_replicas}) must "
+                    f"be a multiple of gang_size ({gc.gang_size})."
+                )
+
+        if gc is not None and max_replicas_per_node is not None:
+            raise ValueError(
+                "Setting max_replicas_per_node is not allowed when "
+                "gang_scheduling_config is provided."
+            )
+
+        if gc is not None and placement_group_strategy is not None:
+            raise ValueError(
+                "Setting placement_group_strategy is not allowed when "
+                "gang_scheduling_config is provided. Use "
+                "gang_scheduling_config.gang_placement_strategy instead."
+            )
+
+        new_replica_config = ReplicaConfig.create(
+            func_or_class,
+            init_args=_init_args,
+            init_kwargs=_init_kwargs,
+            ray_actor_options=ray_actor_options,
+            placement_group_bundles=placement_group_bundles,
+            placement_group_strategy=placement_group_strategy,
+            placement_group_bundle_label_selector=placement_group_bundle_label_selector,
+            placement_group_fallback_strategy=placement_group_fallback_strategy,
+            max_replicas_per_node=max_replicas_per_node,
+        )
+
+        return Deployment(
+            name,
+            new_deployment_config,
+            new_replica_config,
+            version=version,
+            _internal=True,
+        )
+
+    def __eq__(self, other):
+        return all(
+            [
+                self._name == other._name,
+                self._version == other._version,
+                self._deployment_config == other._deployment_config,
+                self._replica_config.init_args == other._replica_config.init_args,
+                self._replica_config.init_kwargs == other._replica_config.init_kwargs,
+                self._replica_config.ray_actor_options
+                == other._replica_config.ray_actor_options,
+            ]
+        )
+
+    def __str__(self):
+        return f"Deployment(name={self._name})"
+
+    def __repr__(self):
+        return str(self)
+
+
+def deployment_to_schema(d: Deployment) -> DeploymentSchema:
+    """Converts a live deployment object to a corresponding structured schema.
+
+    Args:
+        d: Deployment object to convert
+
+    Returns:
+        The structured ``DeploymentSchema`` representing ``d``.
+    """
+
+    if d.ray_actor_options is not None:
+        ray_actor_options_schema = RayActorOptionsSchema.model_validate(
+            d.ray_actor_options
+        )
+    else:
+        ray_actor_options_schema = None
+
+    deployment_options = {
+        "name": d.name,
+        "num_replicas": None
+        if d._deployment_config.autoscaling_config
+        else d.num_replicas,
+        "max_ongoing_requests": d.max_ongoing_requests,
+        "max_queued_requests": d.max_queued_requests,
+        "user_config": d.user_config,
+        "autoscaling_config": d._deployment_config.autoscaling_config,
+        "graceful_shutdown_wait_loop_s": d._deployment_config.graceful_shutdown_wait_loop_s,  # noqa: E501
+        "graceful_shutdown_timeout_s": d._deployment_config.graceful_shutdown_timeout_s,
+        "health_check_period_s": d._deployment_config.health_check_period_s,
+        "health_check_timeout_s": d._deployment_config.health_check_timeout_s,
+        "ray_actor_options": ray_actor_options_schema,
+        "placement_group_strategy": d._replica_config.placement_group_strategy,
+        "placement_group_bundles": d._replica_config.placement_group_bundles,
+        "max_replicas_per_node": d._replica_config.max_replicas_per_node,
+        "logging_config": d._deployment_config.logging_config,
+        "request_router_config": d._deployment_config.request_router_config,
+        "gang_scheduling_config": d._deployment_config.gang_scheduling_config,
+        "deployment_actors": d._deployment_config.deployment_actors,
+        "rolling_update_percentage": d._deployment_config.rolling_update_percentage,
+    }
+
+    # Let non-user-configured options be set to defaults. If the schema
+    # is converted back to a deployment, this lets Serve continue tracking
+    # which options were set by the user. Name is a required field in the
+    # schema, so it should be passed in explicitly.
+    for option in list(deployment_options.keys()):
+        if (
+            option != "name"
+            and option not in d._deployment_config.user_configured_option_names
+        ):
+            del deployment_options[option]
+
+    # TODO(Sihan) DeploymentConfig num_replicas and auto_config can be set together
+    # because internally we use these two field for autoscale and deploy.
+    # We can improve the code after we separate the user faced deployment config and
+    # internal deployment config.
+    return DeploymentSchema(**deployment_options)
+
+
+def schema_to_deployment(s: DeploymentSchema) -> Deployment:
+    """Creates a deployment with parameters specified in schema.
+
+    The returned deployment CANNOT be deployed immediately. It's func_or_class
+    value is an empty string (""), which is not a valid import path. The
+    func_or_class value must be overwritten with a valid function or class
+    before the deployment can be deployed.
+    """
+
+    if s.ray_actor_options is DEFAULT.VALUE:
+        ray_actor_options = None
+    else:
+        ray_actor_options = s.ray_actor_options.model_dump(exclude_unset=True)
+
+    if s.placement_group_bundles is DEFAULT.VALUE:
+        placement_group_bundles = None
+    else:
+        placement_group_bundles = s.placement_group_bundles
+
+    if s.placement_group_strategy is DEFAULT.VALUE:
+        placement_group_strategy = None
+    else:
+        placement_group_strategy = s.placement_group_strategy
+
+    if s.max_replicas_per_node is DEFAULT.VALUE:
+        max_replicas_per_node = None
+    else:
+        max_replicas_per_node = s.max_replicas_per_node
+
+    deployment_config = DeploymentConfig.from_default(
+        num_replicas=s.num_replicas,
+        user_config=s.user_config,
+        max_ongoing_requests=s.max_ongoing_requests,
+        max_queued_requests=s.max_queued_requests,
+        autoscaling_config=s.autoscaling_config,
+        graceful_shutdown_wait_loop_s=s.graceful_shutdown_wait_loop_s,
+        graceful_shutdown_timeout_s=s.graceful_shutdown_timeout_s,
+        health_check_period_s=s.health_check_period_s,
+        health_check_timeout_s=s.health_check_timeout_s,
+        logging_config=s.logging_config,
+        request_router_config=s.request_router_config,
+        gang_scheduling_config=s.gang_scheduling_config,
+        deployment_actors=s.deployment_actors,
+        rolling_update_percentage=s.rolling_update_percentage,
+    )
+    deployment_config.user_configured_option_names = (
+        s._get_user_configured_option_names()
+    )
+
+    replica_config = ReplicaConfig.create(
+        deployment_def="",
+        init_args=(),
+        init_kwargs={},
+        ray_actor_options=ray_actor_options,
+        placement_group_bundles=placement_group_bundles,
+        placement_group_strategy=placement_group_strategy,
+        max_replicas_per_node=max_replicas_per_node,
+    )
+
+    return Deployment(
+        name=s.name,
+        deployment_config=deployment_config,
+        replica_config=replica_config,
+        _internal=True,
+    )