chore: import upstream snapshot with attribution

python/ray/serve/_private/benchmarks/common.py

@@ -0,0 +1,688 @@
+import asyncio
+import inspect
+import logging
+import random
+import string
+import time
+from functools import partial
+from typing import Any, Callable, Coroutine, Dict, List, Optional, Tuple
+
+import aiohttp
+import aiohttp.client_exceptions
+import grpc
+import numpy as np
+import pandas as pd
+from starlette.responses import StreamingResponse
+from tqdm import tqdm
+
+import ray
+from ray import serve
+from ray._common.test_utils import SignalActor as _SignalActor
+from ray.serve._private.common import DeploymentStatus
+from ray.serve.generated import serve_pb2, serve_pb2_grpc
+from ray.serve.handle import DeploymentHandle
+
+
+async def run_latency_benchmark(
+    f: Callable, num_requests: int, *, num_warmup_requests: int = 100
+) -> pd.Series:
+    if inspect.iscoroutinefunction(f):
+        to_call = f
+    else:
+
+        async def to_call():
+            f()
+
+    latencies = []
+    for i in tqdm(range(num_requests + num_warmup_requests)):
+        start = time.perf_counter()
+        await to_call()
+        end = time.perf_counter()
+
+        # Don't include warm-up requests.
+        if i >= num_warmup_requests:
+            latencies.append(1000 * (end - start))
+
+    return pd.Series(latencies)
+
+
+async def run_throughput_benchmark(
+    fn: Callable[[], List[float]],
+    multiplier: int = 1,
+    num_trials: int = 10,
+    trial_runtime: float = 1,
+) -> Tuple[float, float, pd.Series]:
+    """Benchmarks throughput of a function.
+
+    Args:
+        fn: The function to benchmark. If this returns anything, it must
+            return a list of latencies.
+        multiplier: The number of requests or tokens (or whatever unit
+            is appropriate for this throughput benchmark) that is
+            completed in one call to `fn`.
+        num_trials: The number of trials to run.
+        trial_runtime: How long each trial should run for. During the
+            duration of one trial, `fn` will be repeatedly called.
+
+    Returns:
+        A tuple ``(mean, stddev, latencies)`` summarizing per-trial throughput
+        across ``num_trials`` runs.
+    """
+    # Warmup
+    start = time.time()
+    while time.time() - start < 0.1:
+        await fn()
+
+    # Benchmark
+    stats = []
+    latencies = []
+    for _ in tqdm(range(num_trials)):
+        start = time.perf_counter()
+        count = 0
+        while time.perf_counter() - start < trial_runtime:
+            res = await fn()
+            if res:
+                latencies.extend(res)
+
+            count += 1
+        end = time.perf_counter()
+        stats.append(multiplier * count / (end - start))
+
+    return round(np.mean(stats), 2), round(np.std(stats), 2), pd.Series(latencies)
+
+
+async def do_single_http_batch(
+    *,
+    batch_size: int = 100,
+    url: str = "http://localhost:8000",
+    stream: bool = False,
+) -> List[float]:
+    """Sends a batch of http requests and returns e2e latencies."""
+
+    # By default, aiohttp limits the number of client connections to 100.
+    # We need to use TCPConnector to configure the limit if batch size
+    # is greater than 100.
+    connector = aiohttp.TCPConnector(limit=batch_size)
+    async with aiohttp.ClientSession(
+        connector=connector, raise_for_status=True
+    ) as session:
+
+        async def do_query():
+            start = time.perf_counter()
+            try:
+                async with session.get(url) as r:
+                    if stream:
+                        async for chunk, _ in r.content.iter_chunks():
+                            pass
+                    else:
+                        # Read the response to ensure it's consumed
+                        await r.read()
+            except aiohttp.client_exceptions.ClientConnectionError:
+                pass
+
+            end = time.perf_counter()
+            return 1000 * (end - start)
+
+        return await asyncio.gather(*[do_query() for _ in range(batch_size)])
+
+
+async def do_single_grpc_batch(
+    *, batch_size: int = 100, target: str = "localhost:9000"
+):
+    channel = grpc.aio.insecure_channel(target)
+    stub = serve_pb2_grpc.RayServeBenchmarkServiceStub(channel)
+    payload = serve_pb2.StringData(data="")
+
+    async def do_query():
+        start = time.perf_counter()
+
+        await stub.grpc_call(payload)
+
+        end = time.perf_counter()
+        return 1000 * (end - start)
+
+    return await asyncio.gather(*[do_query() for _ in range(batch_size)])
+
+
+async def collect_profile_events(coro: Coroutine):
+    """Collects profiling events using Viztracer"""
+
+    from viztracer import VizTracer
+
+    tracer = VizTracer()
+    tracer.start()
+
+    await coro
+
+    tracer.stop()
+    tracer.save()
+
+
+def generate_payload(size: int = 100, chars=string.ascii_uppercase + string.digits):
+    return "".join(random.choice(chars) for _ in range(size))
+
+
+class Blackhole:
+    def sink(self, o):
+        pass
+
+
+@serve.deployment
+class Noop:
+    def __init__(self):
+        logging.getLogger("ray.serve").setLevel(logging.WARNING)
+
+    def __call__(self, *args, **kwargs):
+        return b""
+
+
+@serve.deployment
+class ModelComp:
+    def __init__(self, child):
+        logging.getLogger("ray.serve").setLevel(logging.WARNING)
+        self._child = child
+
+    async def __call__(self, *args, **kwargs):
+        return await self._child.remote()
+
+
+@serve.deployment
+class GrpcDeployment:
+    def __init__(self):
+        logging.getLogger("ray.serve").setLevel(logging.WARNING)
+
+    async def grpc_call(self, user_message):
+        return serve_pb2.ModelOutput(output=9)
+
+    async def call_with_string(self, user_message):
+        return serve_pb2.ModelOutput(output=9)
+
+
+@serve.deployment
+class GrpcModelComp:
+    def __init__(self, child):
+        logging.getLogger("ray.serve").setLevel(logging.WARNING)
+        self._child = child
+
+    async def grpc_call(self, user_message):
+        await self._child.remote()
+        return serve_pb2.ModelOutput(output=9)
+
+    async def call_with_string(self, user_message):
+        await self._child.remote()
+        return serve_pb2.ModelOutput(output=9)
+
+
+@serve.deployment
+class Streamer:
+    def __init__(self, tokens_per_request: int, inter_token_delay_ms: int = 10):
+        logging.getLogger("ray.serve").setLevel(logging.WARNING)
+        self._tokens_per_request = tokens_per_request
+        self._inter_token_delay_s = inter_token_delay_ms / 1000
+
+    async def stream(self):
+        for _ in range(self._tokens_per_request):
+            await asyncio.sleep(self._inter_token_delay_s)
+            yield b"hi"
+
+    async def __call__(self):
+        return StreamingResponse(self.stream())
+
+
+@serve.deployment
+class IntermediateRouter:
+    def __init__(self, handle: DeploymentHandle):
+        logging.getLogger("ray.serve").setLevel(logging.WARNING)
+        self._handle = handle.options(stream=True)
+
+    async def stream(self):
+        async for token in self._handle.stream.remote():
+            yield token
+
+    def __call__(self):
+        return StreamingResponse(self.stream())
+
+
+@serve.deployment
+class Benchmarker:
+    def __init__(
+        self,
+        handle: DeploymentHandle,
+        stream: bool = False,
+    ):
+        logging.getLogger("ray.serve").setLevel(logging.WARNING)
+        self._handle = handle.options(stream=stream)
+        self._stream = stream
+
+    async def do_single_request(self, payload: Any = None) -> float:
+        """Completes a single unary request. Returns e2e latency in ms."""
+        start = time.perf_counter()
+
+        if payload is None:
+            await self._handle.remote()
+        else:
+            await self._handle.remote(payload)
+
+        end = time.perf_counter()
+        return 1000 * (end - start)
+
+    async def do_single_choose_dispatch(self, payload: Any = None) -> float:
+        """Completes a single unary request via choose_replica + dispatch.
+
+        Returns e2e latency in ms. With SingletonThreadRouter this involves
+        two run_coroutine_threadsafe round-trips (one for __aenter__, one
+        for _dispatch_to_marked_selection) vs. one for ``remote``.
+        """
+        start = time.perf_counter()
+
+        if payload is None:
+            async with self._handle.choose_replica() as sel:
+                await self._handle.dispatch(sel)
+        else:
+            async with self._handle.choose_replica(payload) as sel:
+                await self._handle.dispatch(sel, payload)
+
+        end = time.perf_counter()
+        return 1000 * (end - start)
+
+    async def _do_single_stream(self) -> float:
+        """Consumes a single streaming request. Returns e2e latency in ms."""
+        start = time.perf_counter()
+
+        async for r in self._handle.stream.remote():
+            pass
+
+        end = time.perf_counter()
+        return 1000 * (end - start)
+
+    async def _do_single_batch(self, batch_size: int) -> List[float]:
+        if self._stream:
+            return await asyncio.gather(
+                *[self._do_single_stream() for _ in range(batch_size)]
+            )
+        else:
+            return await asyncio.gather(
+                *[self.do_single_request() for _ in range(batch_size)]
+            )
+
+    async def run_latency_benchmark(
+        self,
+        *,
+        num_requests: int,
+        payload: Any = None,
+        mode: str = "remote",
+    ) -> pd.Series:
+        if mode == "remote":
+
+            async def f():
+                await self.do_single_request(payload)
+
+        elif mode == "choose_dispatch":
+
+            async def f():
+                await self.do_single_choose_dispatch(payload)
+
+        else:
+            raise ValueError(f"Unknown mode {mode!r}")
+
+        return await run_latency_benchmark(f, num_requests=num_requests)
+
+    async def run_throughput_benchmark(
+        self,
+        *,
+        batch_size: int,
+        num_trials: int,
+        trial_runtime: float,
+        tokens_per_request: Optional[float] = None,
+    ) -> Tuple[float, float]:
+        if self._stream:
+            assert tokens_per_request
+            multiplier = tokens_per_request * batch_size
+        else:
+            multiplier = batch_size
+
+        return await run_throughput_benchmark(
+            fn=partial(
+                self._do_single_batch,
+                batch_size=batch_size,
+            ),
+            multiplier=multiplier,
+            num_trials=num_trials,
+            trial_runtime=trial_runtime,
+        )
+
+
+# =============================================================================
+# Controller Benchmark
+# =============================================================================
+# See https://github.com/ray-project/ray/issues/60680 for more details.
+
+CONTROLLER_BENCH_CONFIG = {
+    "checkpoints": [1, 2, 4, 8, 16, 32, 64, 128, 256, 512, 1024, 2048, 3072, 4096],
+    "marination_period_s": 180,
+    "sample_interval_s": 5,
+}
+
+_CONTROLLER_AUTOSCALING_CONFIG = {
+    "min_replicas": 1,
+    "max_replicas": 4096,
+    "target_ongoing_requests": 1,
+    "upscale_delay_s": 1,
+}
+
+_CONTROLLER_WAITER_TIMEOUT_S = 1200
+
+# SignalActor from ray._common.test_utils; use high max_concurrency for many
+# concurrent waiters (up to 4096 in controller benchmark).
+_SignalActorForController = _SignalActor.options(max_concurrency=100000)
+
+
+@serve.deployment(
+    graceful_shutdown_timeout_s=1,
+    ray_actor_options={"num_cpus": 0.2},
+    max_ongoing_requests=100000,
+    autoscaling_config={
+        "min_replicas": 5,
+        "max_replicas": 10,
+        "target_ongoing_requests": 100000,
+        "upscale_delay_s": 1,
+    },
+)
+class ControllerBenchHelloWorld:
+    def __init__(self, signal_actor):
+        self.signal = signal_actor
+
+    async def __call__(self):
+        await self.signal.wait.remote()
+        return "hello"
+
+
+@serve.deployment(
+    autoscaling_config=_CONTROLLER_AUTOSCALING_CONFIG,
+    max_ongoing_requests=2,
+    graceful_shutdown_timeout_s=1,
+    ray_actor_options={"num_cpus": 0.4},
+)
+class ControllerBenchMetricsGenerator:
+    """Autoscaling deployment that generates handle metrics to stress the controller."""
+
+    def __init__(self, hello_world: DeploymentHandle):
+        self.hello_world = hello_world
+
+    async def __call__(self):
+        return await self.hello_world.remote()
+
+
+def _controller_get_active_nodes() -> int:
+    """Get number of active nodes in the cluster."""
+    return len([n for n in ray.nodes() if n.get("Alive", False)])
+
+
+async def _controller_get_replica_count(
+    deployment_name: str = "ControllerBenchMetricsGenerator",
+) -> int:
+    """Get current number of running replicas for the specified deployment."""
+    status = serve.status()
+    for app in status.applications.values():
+        for name, deployment in app.deployments.items():
+            if name == deployment_name:
+                return deployment.replica_states.get("RUNNING", 0)
+    return 0
+
+
+async def _controller_get_health_metrics() -> Dict[str, Any]:
+    """Get controller health metrics. Fails the run if unavailable."""
+    client = serve.context._global_client
+    if client is None:
+        raise RuntimeError(
+            "Serve is not connected. get_health_metrics requires an active Serve "
+            "controller. Ensure Serve is started before running the controller benchmark."
+        )
+    controller = client._controller
+    if not hasattr(controller, "get_health_metrics"):
+        raise RuntimeError(
+            "Controller does not have get_health_metrics. This API is required for "
+            "the controller benchmark. Please use a Ray version that supports "
+            "controller health metrics."
+        )
+    return await controller.get_health_metrics.remote()
+
+
+def _controller_extract_metrics_row(
+    health_metrics: Dict[str, Any],
+    checkpoint: int,
+    sample: int,
+    target_replicas: int,
+    actual_replicas: int,
+    num_nodes: int,
+    autoscale_duration_s: float,
+) -> Dict[str, Any]:
+    """Extract a flat row from health metrics with all available fields."""
+
+    def get_stat(d: dict, key: str, stat: str, default=0):
+        return d.get(key, {}).get(stat, default)
+
+    return {
+        "checkpoint": checkpoint,
+        "sample": sample,
+        "target_replicas": target_replicas,
+        "actual_replicas": actual_replicas,
+        "num_nodes": num_nodes,
+        "autoscale_duration_s": round(autoscale_duration_s, 3),
+        "loop_duration_mean_s": get_stat(health_metrics, "loop_duration_s", "mean"),
+        "loop_duration_std_s": get_stat(health_metrics, "loop_duration_s", "std"),
+        "loops_per_second": health_metrics.get("loops_per_second", 0),
+        "event_loop_delay_s": health_metrics.get("event_loop_delay_s", 0),
+        "num_asyncio_tasks": health_metrics.get("num_asyncio_tasks", 0),
+        "deployment_state_update_mean_s": get_stat(
+            health_metrics, "deployment_state_update_duration_s", "mean"
+        ),
+        "application_state_update_mean_s": get_stat(
+            health_metrics, "application_state_update_duration_s", "mean"
+        ),
+        "proxy_state_update_mean_s": get_stat(
+            health_metrics, "proxy_state_update_duration_s", "mean"
+        ),
+        "proxy_state_update_std_s": get_stat(
+            health_metrics, "proxy_state_update_duration_s", "std"
+        ),
+        "node_update_mean_s": get_stat(
+            health_metrics, "node_update_duration_s", "mean"
+        ),
+        "node_update_std_s": get_stat(health_metrics, "node_update_duration_s", "std"),
+        "node_update_min_s": get_stat(health_metrics, "node_update_duration_s", "min"),
+        "handle_metrics_delay_mean_ms": get_stat(
+            health_metrics, "handle_metrics_delay_ms", "mean"
+        ),
+        "replica_metrics_delay_mean_ms": get_stat(
+            health_metrics, "replica_metrics_delay_ms", "mean"
+        ),
+        "process_memory_mb": health_metrics.get("process_memory_mb", 0),
+    }
+
+
+async def _controller_wait_for_replicas_up(target: int, timeout: float = 300) -> float:
+    start = time.time()
+    while time.time() - start < timeout:
+        actual = await _controller_get_replica_count()
+        if actual >= target:
+            return time.time() - start
+        if int(time.time() - start) % 10 == 0:
+            logging.info(f"Waiting for {target} replicas... {actual}/{target}")
+        await asyncio.sleep(0.5)
+    actual = await _controller_get_replica_count()
+    raise RuntimeError(
+        f"Timeout: Only {actual}/{target} replicas after {timeout}s. Ending experiment."
+    )
+
+
+async def _controller_wait_for_waiters(
+    signal_actor, expected: int, timeout: float = 300
+) -> float:
+    start = time.time()
+    while time.time() - start < timeout:
+        num_waiters = await signal_actor.cur_num_waiters.remote()
+        if num_waiters >= expected:
+            return time.time() - start
+        await asyncio.sleep(0.5)
+        if int(time.time() - start) % 10 == 0:
+            logging.info(f"Waiting for {expected} waiters... {num_waiters}/{expected}")
+    num_waiters = await signal_actor.cur_num_waiters.remote()
+    raise RuntimeError(
+        f"Timeout: Only {num_waiters}/{expected} requests reached replicas after "
+        f"{timeout}s. Ending experiment."
+    )
+
+
+async def _controller_wait_for_deployment_healthy(
+    deployment_name: str = "ControllerBenchMetricsGenerator",
+    app_name: str = "default",
+    timeout: float = 60,
+) -> None:
+    """Wait for the deployment to enter HEALTHY status via serve.status()."""
+    start = time.time()
+    while time.time() - start < timeout:
+        status = serve.status()
+        app = status.applications.get(app_name)
+        dep_status = None
+        if app and deployment_name in app.deployments:
+            dep = app.deployments[deployment_name]
+            dep_status = dep.status
+            if dep_status == DeploymentStatus.HEALTHY:
+                return
+            if dep_status == DeploymentStatus.UNHEALTHY:
+                raise RuntimeError(
+                    f"Deployment {deployment_name} is UNHEALTHY: {getattr(dep, 'message', '')}"
+                )
+        if int(time.time() - start) % 10 == 0:
+            logging.info(
+                f"Waiting for {deployment_name} to be healthy, current: {dep_status}."
+            )
+        await asyncio.sleep(0.5)
+
+    raise RuntimeError(
+        f"Deployment {deployment_name} did not become HEALTHY after {timeout}s."
+    )
+
+
+_BATCH_SIZE = 64
+
+
+async def _controller_run_checkpoint(
+    handle: DeploymentHandle,
+    signal_actor,
+    checkpoint: int,
+    target_replicas: int,
+    marination_period_s: int,
+    sample_interval_s: int,
+) -> List[Dict[str, Any]]:
+    """Run a single checkpoint and collect metrics."""
+    start_time = time.time()
+    num_requests = int(target_replicas)
+
+    pending_requests: List[Any] = []
+    pending_requests.extend([handle.remote() for _ in range(num_requests)])
+    logging.info(f"Waiting for {num_requests} requests to be up...")
+    await _controller_wait_for_waiters(
+        signal_actor, len(pending_requests), timeout=_CONTROLLER_WAITER_TIMEOUT_S
+    )
+    logging.info(f"Waiting for {target_replicas} replicas to be up...")
+    # TODO: This is a hack to allow for some tolerance in the number of replicas.
+    # This is because the controller may not scale exactly to the target number of replicas.
+    # This is a bug in the controller autoscaling metrics aggregation logic, needs
+    # to be investigated further.
+    # This has the potential to introduce noise in the results from this benchmark.
+    replica_tolerance = 0.8
+    await _controller_wait_for_replicas_up(
+        int(target_replicas * replica_tolerance), timeout=_CONTROLLER_WAITER_TIMEOUT_S
+    )
+    logging.info(f"All {target_replicas} replicas are up.")
+    logging.info("Waiting for deployment to be healthy...")
+    await _controller_wait_for_deployment_healthy(timeout=_CONTROLLER_WAITER_TIMEOUT_S)
+    logging.info("Deployment is healthy.")
+    logging.info(f"Waiting for {marination_period_s} seconds to collect metrics...")
+
+    autoscale_duration_s = time.time() - start_time
+
+    samples = []
+    num_samples = marination_period_s // sample_interval_s
+    for sample_idx in range(num_samples):
+        health_metrics = await _controller_get_health_metrics()
+        actual_replicas = await _controller_get_replica_count()
+        num_nodes = _controller_get_active_nodes()
+        row = _controller_extract_metrics_row(
+            health_metrics=health_metrics,
+            checkpoint=checkpoint,
+            sample=sample_idx,
+            target_replicas=target_replicas,
+            actual_replicas=actual_replicas,
+            num_nodes=num_nodes,
+            autoscale_duration_s=autoscale_duration_s,
+        )
+        samples.append(row)
+        if sample_idx < num_samples - 1:
+            await asyncio.sleep(sample_interval_s)
+
+    await signal_actor.send.remote(clear=True)
+    try:
+        await asyncio.wait_for(
+            asyncio.gather(*pending_requests, return_exceptions=True),
+            timeout=30.0,
+        )
+    except asyncio.TimeoutError:
+        pass
+
+    return samples
+
+
+async def run_controller_benchmark(
+    config: Optional[Dict[str, Any]] = None,
+) -> List[Dict[str, Any]]:
+    """
+    Run the controller health metrics benchmark and return raw samples.
+
+    Uses MetricsGenerator (autoscaling) -> HelloWorld (fixed) -> SignalActor
+    to stress the controller as replicas scale. Fails if get_health_metrics
+    is unavailable.
+
+    Args:
+        config: Optional benchmark config (checkpoints, marination_period_s,
+            sample_interval_s). Uses CONTROLLER_BENCH_CONFIG if None.
+
+    Returns:
+        List of sample dicts (one per marination sample). Each sample has
+        target_replicas, autoscale_duration_s, loop_duration_mean_s,
+        loops_per_second, event_loop_delay_s, num_asyncio_tasks, etc.
+        Caller converts to perf_metrics via convert_controller_samples_to_perf_metrics.
+    """
+    cfg = config or CONTROLLER_BENCH_CONFIG
+    checkpoints = cfg["checkpoints"]
+    marination_period_s = cfg["marination_period_s"]
+    sample_interval_s = cfg["sample_interval_s"]
+
+    if not ray.is_initialized():
+        ray.init()
+
+    signal_actor = _SignalActorForController.remote()
+    all_samples: List[Dict[str, Any]] = []
+
+    try:
+        for checkpoint_idx, target_replicas in enumerate(checkpoints):
+            hello_world = ControllerBenchHelloWorld.bind(signal_actor)
+            app = ControllerBenchMetricsGenerator.bind(hello_world)
+            handle = serve.run(app, name="default", route_prefix=None)
+
+            samples = await _controller_run_checkpoint(
+                handle=handle,
+                signal_actor=signal_actor,
+                checkpoint=checkpoint_idx,
+                target_replicas=target_replicas,
+                marination_period_s=marination_period_s,
+                sample_interval_s=sample_interval_s,
+            )
+            all_samples.extend(samples)
+            serve.shutdown()
+    finally:
+        serve.shutdown()
+
+    return all_samples

python/ray/serve/_private/benchmarks/handle_noop_latency.py

@@ -0,0 +1,39 @@
+import time
+
+import click
+
+from ray import serve
+from ray.serve._private.benchmarks.common import Benchmarker, Noop
+from ray.serve.handle import DeploymentHandle
+
+
+@click.command(help="Benchmark no-op DeploymentHandle latency.")
+@click.option("--num-replicas", type=int, default=1)
+@click.option("--num-requests", type=int, default=100)
+@click.option(
+    "--mode",
+    type=click.Choice(["remote", "choose_dispatch"]),
+    default="remote",
+    help="Which call pattern to benchmark.",
+)
+def main(num_replicas: int, num_requests: int, mode: str):
+    h: DeploymentHandle = serve.run(
+        Benchmarker.bind(Noop.options(num_replicas=num_replicas).bind())
+    )
+
+    latencies = h.run_latency_benchmark.remote(
+        num_requests=num_requests, mode=mode
+    ).result()
+
+    # Let the logs flush to avoid interwoven output.
+    time.sleep(1)
+
+    print(
+        f"Latency (ms) for noop DeploymentHandle requests via {mode!r} "
+        f"(num_replicas={num_replicas},num_requests={num_requests}):"
+    )
+    print(latencies.describe(percentiles=[0.5, 0.9, 0.95, 0.99]))
+
+
+if __name__ == "__main__":
+    main()

python/ray/serve/_private/benchmarks/handle_throughput.py

@@ -0,0 +1,62 @@
+import click
+
+from ray import serve
+from ray.serve._private.benchmarks.common import Benchmarker, Hello
+from ray.serve.handle import DeploymentHandle
+
+
+@click.command(help="Benchmark deployment handle throughput.")
+@click.option(
+    "--batch-size",
+    type=int,
+    default=100,
+    help="Number of requests to send to downstream deployment in each trial.",
+)
+@click.option(
+    "--num-replicas",
+    type=int,
+    default=1,
+    help="Number of replicas in the downstream deployment.",
+)
+@click.option(
+    "--num-trials",
+    type=int,
+    default=5,
+    help="Number of trials of the benchmark to run.",
+)
+@click.option(
+    "--trial-runtime",
+    type=int,
+    default=1,
+    help="Duration to run each trial of the benchmark for (seconds).",
+)
+def main(
+    batch_size: int,
+    num_replicas: int,
+    num_trials: int,
+    trial_runtime: float,
+):
+    app = Benchmarker.bind(
+        Hello.options(
+            num_replicas=num_replicas, ray_actor_options={"num_cpus": 0}
+        ).bind(),
+    )
+    h: DeploymentHandle = serve.run(app)
+
+    mean, stddev = h.run_throughput_benchmark.remote(
+        batch_size=batch_size,
+        num_trials=num_trials,
+        trial_runtime=trial_runtime,
+    ).result()
+
+    print(
+        "DeploymentHandle throughput {}: {} +- {} requests/s".format(
+            f"(num_replicas={num_replicas}, batch_size={batch_size})",
+            mean,
+            stddev,
+        )
+    )
+
+
+if __name__ == "__main__":
+    main()

python/ray/serve/_private/benchmarks/http_noop_latency.py

@@ -0,0 +1,32 @@
+import asyncio
+
+import click
+import pandas as pd
+import requests
+
+from ray import serve
+from ray.serve._private.benchmarks.common import Noop, run_latency_benchmark
+
+
+@click.command(help="Benchmark no-op HTTP latency.")
+@click.option("--num-replicas", type=int, default=1)
+@click.option("--num-requests", type=int, default=100)
+def main(num_replicas: int, num_requests: int):
+    serve.run(Noop.options(num_replicas=num_replicas).bind())
+
+    latencies: pd.Series = asyncio.new_event_loop().run_until_complete(
+        run_latency_benchmark(
+            lambda: requests.get("http://localhost:8000"),
+            num_requests=num_requests,
+        )
+    )
+
+    print(
+        "Latency (ms) for noop HTTP requests "
+        f"(num_replicas={num_replicas},num_requests={num_requests}):"
+    )
+    print(latencies.describe(percentiles=[0.5, 0.9, 0.95, 0.99]))
+
+
+if __name__ == "__main__":
+    main()

python/ray/serve/_private/benchmarks/locust_utils.py

@@ -0,0 +1,374 @@
+import argparse
+import logging
+import time
+from dataclasses import asdict, dataclass
+from typing import Any, Dict, List, NamedTuple
+
+from ray.serve._private.utils import generate_request_id
+
+logger = logging.getLogger(__file__)
+logging.basicConfig(level=logging.INFO)
+
+MASTER_PORT = 5557
+
+
+@dataclass
+class LocustStage:
+    duration_s: int
+    users: int
+    spawn_rate: float
+
+
+@dataclass
+class PerformanceStats:
+    p50_latency: float
+    p90_latency: float
+    p99_latency: float
+    rps: float
+
+
+@dataclass
+class LocustTestResults:
+    history: List[Dict]
+    total_requests: int
+    num_failures: int
+    avg_latency: float
+    p50_latency: float
+    p90_latency: float
+    p99_latency: float
+    avg_rps: float
+    stats_in_stages: List[PerformanceStats]
+
+
+@dataclass
+class FailedRequest:
+    request_id: str
+    status_code: int
+    exception: str
+    response_time_ms: float
+    start_time_s: float
+
+
+class LocustClient:
+    def __init__(
+        self,
+        host_url: str,
+        token: str,
+        data: Dict[str, Any] = None,
+    ):
+        from locust import FastHttpUser, constant, events, task
+        from locust.contrib.fasthttp import FastResponse
+
+        self.errors = []
+        self.stats_in_stages: List[PerformanceStats] = []
+
+        class EndpointUser(FastHttpUser):
+            wait_time = constant(0)
+            failed_requests = []
+            host = host_url
+
+            @task
+            def test(self):
+                request_id = generate_request_id()
+                headers = (
+                    {"Authorization": f"Bearer {token}", "X-Request-ID": request_id}
+                    if token
+                    else None
+                )
+                start = time.perf_counter()
+                with self.client.get(
+                    "", headers=headers, json=data, catch_response=True
+                ) as r:
+                    # locust<=2.18 FastHttp truncates response_time to whole ms;
+                    # re-measure so the 0.1ms buckets see sub-ms differences.
+                    r.request_meta["response_time"] = (
+                        time.perf_counter() - start
+                    ) * 1000
+                    r.request_meta["context"]["request_id"] = request_id
+
+            @events.request.add_listener
+            def on_request(
+                response: FastResponse,
+                exception,
+                context,
+                start_time: float,
+                response_time: float,
+                **kwargs,
+            ):
+                if exception and response.status_code != 0:
+                    request_id = context["request_id"]
+                    print(
+                        f"Request '{request_id}' failed with exception:\n"
+                        f"{exception}\n{response.text}"
+                    )
+
+                    if response.status_code != 0:
+                        response.encoding = "utf-8"
+                        err = FailedRequest(
+                            request_id=request_id,
+                            status_code=response.status_code,
+                            exception=response.text,
+                            response_time_ms=response_time,
+                            start_time_s=start_time,
+                        )
+                        self.errors.append(err)
+                        print(
+                            f"Request '{request_id}' failed with exception:\n"
+                            f"{exception}\n{response.text}"
+                        )
+
+        self.user_class = EndpointUser
+
+
+class ResponseTimeSnapshot(NamedTuple):
+    # Cumulative {rounded_response_time: count} histogram + request count.
+    response_times: Dict[float, int]
+    num_requests: int
+
+
+def _fine_bucket_response_time(response_time):
+    """0.1ms resolution below 100ms (vs locust's 1ms floor), coarser above."""
+    if response_time < 100:
+        return round(response_time, 1)
+    elif response_time < 1000:
+        return round(response_time)
+    else:
+        return int(round(response_time, -1))
+
+
+def _install_fine_response_time_bucketing():
+    """Swap in the finer bucketer; must run in every response-logging process.
+
+    Released locust (through at least 2.41) inlines the rounding in
+    StatsEntry._log_response_time, so patching requires overriding the whole
+    method. Unreleased locust factors it into stats.bucket_response_time."""
+    import locust.stats
+
+    if hasattr(locust.stats, "bucket_response_time"):
+        locust.stats.bucket_response_time = _fine_bucket_response_time
+        return
+
+    def _log_response_time(self, response_time):
+        # Copy of locust 2.x StatsEntry._log_response_time with the inline
+        # rounding replaced by the fine bucketer.
+        if response_time is None:
+            self.num_none_requests += 1
+            return
+
+        self.total_response_time += response_time
+
+        if self.min_response_time is None:
+            self.min_response_time = response_time
+        self.min_response_time = min(self.min_response_time, response_time)
+        self.max_response_time = max(self.max_response_time, response_time)
+
+        rounded_response_time = _fine_bucket_response_time(response_time)
+        # setdefault keeps this compatible with both the plain dict (<=2.18)
+        # and defaultdict (>=2.33) versions of response_times.
+        self.response_times.setdefault(rounded_response_time, 0)
+        self.response_times[rounded_response_time] += 1
+
+    locust.stats.StatsEntry._log_response_time = _log_response_time
+
+
+def on_stage_finished(master_runner, stats_in_stages, stage_duration_s, prev_snapshot):
+    """Per-stage stats by differencing cumulative snapshots; returns the
+    snapshot to seed the next stage. Percentiles use locust's own
+    calculate_response_time_percentile so they match its end-of-test report."""
+    from locust.stats import (
+        calculate_response_time_percentile,
+        diff_response_time_dicts,
+    )
+
+    stats_entry = master_runner.stats.entries.get(("", "GET"))
+    snapshot = ResponseTimeSnapshot(
+        dict(stats_entry.response_times), stats_entry.num_requests
+    )
+
+    stage_hist = diff_response_time_dicts(
+        snapshot.response_times, prev_snapshot.response_times
+    )
+    stage_requests = snapshot.num_requests - prev_snapshot.num_requests
+
+    stats_in_stages.append(
+        PerformanceStats(
+            p50_latency=calculate_response_time_percentile(
+                stage_hist, stage_requests, 0.5
+            ),
+            p90_latency=calculate_response_time_percentile(
+                stage_hist, stage_requests, 0.9
+            ),
+            p99_latency=calculate_response_time_percentile(
+                stage_hist, stage_requests, 0.99
+            ),
+            rps=stage_requests / stage_duration_s if stage_duration_s else 0.0,
+        )
+    )
+    return snapshot
+
+
+def run_locust_worker(
+    master_address: str, host_url: str, token: str, data: Dict[str, Any]
+):
+    import locust
+    from locust.env import Environment
+    from locust.log import setup_logging
+
+    setup_logging("INFO")
+    # Workers log response times, so the finer bucketer must be installed here.
+    _install_fine_response_time_bucketing()
+    client = LocustClient(host_url=host_url, token=token, data=data)
+    env = Environment(user_classes=[client.user_class], events=locust.events)
+
+    runner = env.create_worker_runner(
+        master_host=master_address, master_port=MASTER_PORT
+    )
+    runner.greenlet.join()
+
+    if client.errors:
+        raise RuntimeError(f"There were {len(client.errors)} errors: {client.errors}")
+
+
+def run_locust_master(
+    host_url: str,
+    token: str,
+    expected_num_workers: int,
+    stages: List[LocustStage],
+    wait_for_workers_timeout_s: float,
+):
+    import gevent
+    import locust
+    from locust import LoadTestShape
+    from locust.env import Environment
+    from locust.stats import (
+        get_error_report_summary,
+        get_percentile_stats_summary,
+        get_stats_summary,
+        stats_history,
+        stats_printer,
+    )
+
+    _install_fine_response_time_bucketing()
+    client = LocustClient(host_url, token)
+
+    class StagesShape(LoadTestShape):
+        curr_stage_ix = 0
+        # Cumulative response-time snapshot at the start of the current stage;
+        # on_stage_finished diffs against it to get per-stage stats.
+        prev_snapshot = ResponseTimeSnapshot({}, 0)
+
+        def tick(cls):
+            run_time = cls.get_run_time()
+            prefix_time = 0
+            for i, stage in enumerate(stages):
+                prefix_time += stage.duration_s
+
+                if run_time < prefix_time:
+                    if i != cls.curr_stage_ix:
+                        cls.prev_snapshot = on_stage_finished(
+                            master_runner,
+                            client.stats_in_stages,
+                            stages[cls.curr_stage_ix].duration_s,
+                            cls.prev_snapshot,
+                        )
+                        cls.curr_stage_ix = i
+
+                    current_stage = stages[cls.curr_stage_ix]
+                    return current_stage.users, current_stage.spawn_rate
+
+            # End of stage test
+            cls.prev_snapshot = on_stage_finished(
+                master_runner,
+                client.stats_in_stages,
+                stages[cls.curr_stage_ix].duration_s,
+                cls.prev_snapshot,
+            )
+
+    master_env = Environment(
+        user_classes=[client.user_class],
+        shape_class=StagesShape(),
+        events=locust.events,
+    )
+    master_runner = master_env.create_master_runner("*", MASTER_PORT)
+
+    start = time.time()
+    while len(master_runner.clients.ready) < expected_num_workers:
+        if time.time() - start > wait_for_workers_timeout_s:
+            raise RuntimeError(
+                f"Timed out waiting for {expected_num_workers} workers to "
+                "connect to Locust master."
+            )
+
+        print(
+            f"Waiting for workers to be ready, "
+            f"{len(master_runner.clients.ready)} "
+            f"of {expected_num_workers} ready."
+        )
+        time.sleep(1)
+
+    # Periodically output current stats (each entry is aggregated
+    # stats over the past 10 seconds, by default)
+    gevent.spawn(stats_printer(master_env.stats))
+    gevent.spawn(stats_history, master_runner)
+
+    # Start test & wait for the shape test to finish
+    master_runner.start_shape()
+    master_runner.shape_greenlet.join()
+    # Send quit signal to all locust workers
+    master_runner.quit()
+
+    # Print stats
+    for line in get_stats_summary(master_runner.stats, current=False):
+        print(line)
+    # Print percentile stats
+    for line in get_percentile_stats_summary(master_runner.stats):
+        print(line)
+    # Print error report
+    if master_runner.stats.errors:
+        for line in get_error_report_summary(master_runner.stats):
+            print(line)
+
+    stats_entry_key = ("", "GET")
+    stats_entry = master_runner.stats.entries.get(stats_entry_key)
+    results = LocustTestResults(
+        history=master_runner.stats.history,
+        total_requests=master_runner.stats.num_requests,
+        num_failures=master_runner.stats.num_failures,
+        avg_latency=stats_entry.avg_response_time,
+        p50_latency=stats_entry.get_response_time_percentile(0.5),
+        p90_latency=stats_entry.get_response_time_percentile(0.9),
+        p99_latency=stats_entry.get_response_time_percentile(0.99),
+        avg_rps=stats_entry.total_rps,
+        stats_in_stages=client.stats_in_stages,
+    )
+    return asdict(results)
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--worker-type", type=str, required=True)
+    parser.add_argument("--host-url", type=str, required=True)
+    parser.add_argument("--token", type=str, required=True)
+    parser.add_argument("--master-address", type=str, required=False)
+    parser.add_argument("--expected-num-workers", type=int, required=False)
+    parser.add_argument("--stages", type=str, required=False)
+    parser.add_argument("--wait-for-workers-timeout-s", type=float, required=False)
+    args = parser.parse_args()
+    host_url = args.host_url
+    token = args.token
+    if args.worker_type == "master":
+        results = run_locust_master(
+            host_url,
+            token,
+            args.expected_num_workers,
+            args.stages,
+            args.wait_for_workers_timeout_s,
+        )
+    else:
+        results = run_locust_worker(args.master_address, host_url, token, args.data)
+
+    print(results)
+
+
+if __name__ == "__main__":
+    main()

python/ray/serve/_private/benchmarks/microbenchmark.py

@@ -0,0 +1,182 @@
+# Runs several scenarios with varying max batch size, max concurrent queries,
+# number of replicas, and with intermediate serve handles (to simulate ensemble
+# models) either on or off.
+
+import asyncio
+import logging
+from pprint import pprint
+from typing import Dict, Union
+
+import aiohttp
+from starlette.requests import Request
+
+import ray
+from ray import serve
+from ray.serve._private.benchmarks.common import run_throughput_benchmark
+from ray.serve.handle import DeploymentHandle
+
+NUM_CLIENTS = 8
+CALLS_PER_BATCH = 100
+
+
+async def fetch(session, data):
+    async with session.get("http://localhost:8000/", data=data) as response:
+        response = await response.text()
+        assert response == "ok", response
+
+
+@ray.remote
+class Client:
+    def ready(self):
+        return "ok"
+
+    async def do_queries(self, num, data):
+        async with aiohttp.ClientSession() as session:
+            for _ in range(num):
+                await fetch(session, data)
+
+
+def build_app(
+    intermediate_handles: bool,
+    num_replicas: int,
+    max_batch_size: int,
+    max_ongoing_requests: int,
+):
+    @serve.deployment(max_ongoing_requests=1000)
+    class Upstream:
+        def __init__(self, handle: DeploymentHandle):
+            self._handle = handle
+
+            # Turn off access log.
+            logging.getLogger("ray.serve").setLevel(logging.WARNING)
+
+        async def __call__(self, req: Request):
+            return await self._handle.remote(await req.body())
+
+    @serve.deployment(
+        num_replicas=num_replicas,
+        max_ongoing_requests=max_ongoing_requests,
+    )
+    class Downstream:
+        def __init__(self):
+            # Turn off access log.
+            logging.getLogger("ray.serve").setLevel(logging.WARNING)
+
+        @serve.batch(max_batch_size=max_batch_size)
+        async def batch(self, reqs):
+            return [b"ok"] * len(reqs)
+
+        async def __call__(self, req: Union[bytes, Request]):
+            if max_batch_size > 1:
+                return await self.batch(req)
+            else:
+                return b"ok"
+
+    if intermediate_handles:
+        return Upstream.bind(Downstream.bind())
+    else:
+        return Downstream.bind()
+
+
+async def trial(
+    intermediate_handles: bool,
+    num_replicas: int,
+    max_batch_size: int,
+    max_ongoing_requests: int,
+    data_size: str,
+) -> Dict[str, float]:
+    results = {}
+
+    trial_key_base = (
+        f"replica:{num_replicas}/batch_size:{max_batch_size}/"
+        f"concurrent_queries:{max_ongoing_requests}/"
+        f"data_size:{data_size}/intermediate_handle:{intermediate_handles}"
+    )
+
+    print(
+        f"intermediate_handles={intermediate_handles},"
+        f"num_replicas={num_replicas},"
+        f"max_batch_size={max_batch_size},"
+        f"max_ongoing_requests={max_ongoing_requests},"
+        f"data_size={data_size}"
+    )
+
+    app = build_app(
+        intermediate_handles, num_replicas, max_batch_size, max_ongoing_requests
+    )
+    serve.run(app)
+
+    if data_size == "small":
+        data = None
+    elif data_size == "large":
+        data = b"a" * 1024 * 1024
+    else:
+        raise ValueError("data_size should be 'small' or 'large'.")
+
+    async with aiohttp.ClientSession() as session:
+
+        async def single_client():
+            for _ in range(CALLS_PER_BATCH):
+                await fetch(session, data)
+
+        single_client_avg_tps, single_client_std_tps = await run_throughput_benchmark(
+            single_client,
+            multiplier=CALLS_PER_BATCH,
+        )
+        print(
+            "\t{} {} +- {} requests/s".format(
+                "single client {} data".format(data_size),
+                single_client_avg_tps,
+                single_client_std_tps,
+            )
+        )
+        key = f"num_client:1/{trial_key_base}"
+        results[key] = single_client_avg_tps
+
+    clients = [Client.remote() for _ in range(NUM_CLIENTS)]
+    ray.get([client.ready.remote() for client in clients])
+
+    async def many_clients():
+        ray.get([a.do_queries.remote(CALLS_PER_BATCH, data) for a in clients])
+
+    multi_client_avg_tps, _ = await run_throughput_benchmark(
+        many_clients,
+        multiplier=CALLS_PER_BATCH * len(clients),
+    )
+
+    results[f"num_client:{len(clients)}/{trial_key_base}"] = multi_client_avg_tps
+    return results
+
+
+async def main():
+    results = {}
+    for intermediate_handles in [False, True]:
+        for num_replicas in [1, 8]:
+            for max_batch_size, max_ongoing_requests in [
+                (1, 1),
+                (1, 10000),
+                (10000, 10000),
+            ]:
+                # TODO(edoakes): large data causes broken pipe errors.
+                for data_size in ["small"]:
+                    results.update(
+                        await trial(
+                            intermediate_handles,
+                            num_replicas,
+                            max_batch_size,
+                            max_ongoing_requests,
+                            data_size,
+                        )
+                    )
+
+    print("Results from all conditions:")
+    pprint(results)
+    return results
+
+
+if __name__ == "__main__":
+    ray.init()
+    serve.start()
+    loop = asyncio.new_event_loop()
+    asyncio.set_event_loop(loop)
+    loop.run_until_complete(main())

python/ray/serve/_private/benchmarks/proxy_benchmark.py

@@ -0,0 +1,294 @@
+# Runs some request ping to compare HTTP and gRPC performances in TPS and latency.
+# Note: this takes around 1 hour to run.
+
+import asyncio
+import json
+import logging
+import time
+from random import random
+from typing import Callable, Dict
+
+import aiohttp
+import numpy as np
+import pandas as pd
+from grpc import aio
+from starlette.requests import Request
+
+import ray
+from ray import serve
+from ray.serve._private.common import RequestProtocol
+from ray.serve.config import gRPCOptions
+from ray.serve.generated import serve_pb2, serve_pb2_grpc
+from ray.serve.handle import DeploymentHandle
+
+CALLS_PER_BATCH = 100
+DELTA = 10**-7
+
+
+async def get_query_tps(name: str, fn: Callable, multiplier: int = CALLS_PER_BATCH):
+    """Get query TPS.
+
+    Run the function for 0.5 seconds 10 times to calculate how many requests can
+    be completed. And use those stats to calculate the mean and std of TPS.
+    """
+    # warmup
+    start = time.time()
+    while time.time() - start < 0.1:
+        await fn()
+    # real run
+    stats = []
+    for _ in range(10):
+        count = 0
+        start = time.time()
+        while time.time() - start < 0.5:
+            await fn()
+            count += 1
+        end = time.time()
+        stats.append(multiplier * count / (end - start))
+    tps_mean = round(np.mean(stats), 2)
+    tps_std = round(np.std(stats), 2)
+    print(f"\t{name} {tps_mean} +- {tps_std} requests/s")
+
+    return tps_mean, tps_std
+
+
+async def get_query_latencies(name: str, fn: Callable):
+    """Get query latencies.
+
+    Take all the latencies from the function and calculate the mean and std.
+    """
+    many_client_results = np.asarray(await fn())
+    many_client_results.flatten()
+    latency_ms_mean = round(np.mean(many_client_results) * 1000, 2)
+    latency_ms_std = round(np.std(many_client_results) * 1000, 2)
+    print(f"\t{name} {latency_ms_mean} +- {latency_ms_std} ms")
+
+    return latency_ms_mean, latency_ms_std
+
+
+async def fetch_http(session, data):
+    data_json = {"nums": data}
+    response = await session.get("http://localhost:8000/", json=data_json)
+    response_text = await response.read()
+    float(response_text.decode())
+
+
+async def fetch_grpc(stub, data):
+    result = await stub.grpc_call(serve_pb2.RawData(nums=data))
+    _ = result.output
+
+
+@ray.remote
+class HTTPClient:
+    def ready(self):
+        return "ok"
+
+    async def do_queries(self, num, data):
+        async with aiohttp.ClientSession() as session:
+            for _ in range(num):
+                await fetch_http(session, data)
+
+    async def time_queries(self, num, data):
+        stats = []
+        async with aiohttp.ClientSession() as session:
+            for _ in range(num):
+                start = time.time()
+                await fetch_http(session, data)
+                end = time.time()
+                stats.append(end - start)
+
+        return stats
+
+
+@ray.remote
+class gRPCClient:
+    def __init__(self):
+        channel = aio.insecure_channel("localhost:9000")
+        self.stub = serve_pb2_grpc.RayServeBenchmarkServiceStub(channel)
+
+    def ready(self):
+        return "ok"
+
+    async def do_queries(self, num, data):
+        for _ in range(num):
+            await fetch_grpc(self.stub, data)
+
+    async def time_queries(self, num, data):
+        stats = []
+        for _ in range(num):
+            start = time.time()
+            await fetch_grpc(self.stub, data)
+            end = time.time()
+            stats.append(end - start)
+        return stats
+
+
+def build_app(
+    num_replicas: int,
+    max_ongoing_requests: int,
+    data_size: int,
+):
+    @serve.deployment(max_ongoing_requests=1000)
+    class DataPreprocessing:
+        def __init__(self, handle: DeploymentHandle):
+            self._handle = handle
+
+            # Turn off access log.
+            logging.getLogger("ray.serve").setLevel(logging.WARNING)
+
+        def normalize(self, raw: np.ndarray) -> np.ndarray:
+            return (raw - np.min(raw)) / (np.max(raw) - np.min(raw) + DELTA)
+
+        async def __call__(self, req: Request):
+            """HTTP entrypoint.
+
+            It parses the request, normalize the data, and send to model for inference.
+            """
+            body = json.loads(await req.body())
+            raw = np.asarray(body["nums"])
+            processed = self.normalize(raw)
+            return await self._handle.remote(processed)
+
+        async def grpc_call(self, raq_data):
+            """gRPC entrypoint.
+
+            It parses the request, normalize the data, and send to model for inference.
+            """
+            raw = np.asarray(raq_data.nums)
+            processed = self.normalize(raw)
+            output = await self._handle.remote(processed)
+            return serve_pb2.ModelOutput(output=output)
+
+        async def call_with_string(self, raq_data):
+            """gRPC entrypoint."""
+            return serve_pb2.ModelOutput(output=0)
+
+    @serve.deployment(
+        num_replicas=num_replicas,
+        max_ongoing_requests=max_ongoing_requests,
+    )
+    class ModelInference:
+        def __init__(self):
+            # Turn off access log.
+            logging.getLogger("ray.serve").setLevel(logging.WARNING)
+            self._model = np.random.randn(data_size, data_size)
+
+        async def __call__(self, processed: np.ndarray) -> float:
+            # Run a dot product with a random matrix to simulate a model inference.
+            model_output = np.dot(processed, self._model)
+            return sum(model_output)
+
+    return DataPreprocessing.bind(ModelInference.bind())
+
+
+async def trial(
+    num_replicas: int,
+    max_ongoing_requests: int,
+    data_size: int,
+    num_clients: int,
+    proxy: RequestProtocol,
+) -> Dict[str, float]:
+    # Generate input data as array of random floats.
+    data = [random() for _ in range(data_size)]
+
+    # Build and deploy the app.
+    app = build_app(
+        num_replicas=num_replicas,
+        max_ongoing_requests=max_ongoing_requests,
+        data_size=data_size,
+    )
+    serve.run(app)
+
+    # Start clients.
+    if proxy == RequestProtocol.GRPC:
+        clients = [gRPCClient.remote() for _ in range(num_clients)]
+    elif proxy == RequestProtocol.HTTP:
+        clients = [HTTPClient.remote() for _ in range(num_clients)]
+    ray.get([client.ready.remote() for client in clients])
+
+    async def client_time_queries():
+        return ray.get([a.time_queries.remote(CALLS_PER_BATCH, data) for a in clients])
+
+    async def client_do_queries():
+        ray.get([a.do_queries.remote(CALLS_PER_BATCH, data) for a in clients])
+
+    trial_key_base = (
+        f"proxy:{proxy}/"
+        f"num_client:{num_clients}/"
+        f"replica:{num_replicas}/"
+        f"concurrent_queries:{max_ongoing_requests}/"
+        f"data_size:{data_size}"
+    )
+    tps_mean, tps_sdt = await get_query_tps(
+        trial_key_base,
+        client_do_queries,
+    )
+    latency_ms_mean, latency_ms_std = await get_query_latencies(
+        trial_key_base,
+        client_time_queries,
+    )
+
+    results = {
+        "proxy": proxy.value,
+        "num_client": num_clients,
+        "replica": num_replicas,
+        "concurrent_queries": max_ongoing_requests,
+        "data_size": data_size,
+        "tps_mean": tps_mean,
+        "tps_sdt": tps_sdt,
+        "latency_ms_mean": latency_ms_mean,
+        "latency_ms_std": latency_ms_std,
+    }
+
+    return results
+
+
+async def main():
+    start_time = time.time()
+    results = []
+    for num_replicas in [1, 8]:
+        for max_ongoing_requests in [1, 10_000]:
+            for data_size in [1, 100, 10_000]:
+                for num_clients in [1, 8]:
+                    for proxy in [RequestProtocol.GRPC, RequestProtocol.HTTP]:
+                        results.append(
+                            await trial(
+                                num_replicas=num_replicas,
+                                max_ongoing_requests=max_ongoing_requests,
+                                data_size=data_size,
+                                num_clients=num_clients,
+                                proxy=proxy,
+                            )
+                        )
+
+    print(f"Total time: {time.time() - start_time}s")
+    print("results", results)
+
+    df = pd.DataFrame.from_dict(results)
+    df = df.sort_values(
+        by=["proxy", "num_client", "replica", "concurrent_queries", "data_size"]
+    )
+    print("Results from all conditions:")
+    # Print the results in with tab separated so we can copy into google sheets.
+    for i in range(len(df.index)):
+        row = list(df.iloc[i])
+        print("\t".join(map(str, row)))
+
+
+if __name__ == "__main__":
+    ray.init()
+
+    grpc_port = 9000
+    grpc_servicer_functions = [
+        "ray.serve.generated.serve_pb2_grpc."
+        "add_RayServeBenchmarkServiceServicer_to_server",
+    ]
+    serve.start(
+        grpc_options=gRPCOptions(
+            port=grpc_port,
+            grpc_servicer_functions=grpc_servicer_functions,
+        )
+    )
+    loop = asyncio.new_event_loop()
+    asyncio.set_event_loop(loop)
+    loop.run_until_complete(main())

python/ray/serve/_private/benchmarks/serialization/common.py

@@ -0,0 +1,29 @@
+from dataclasses import dataclass
+from typing import List, Optional
+
+from pydantic import BaseModel
+
+#
+# NOTE: PLEASE READ CAREFULLY BEFORE CHANGING
+#
+# Payloads in this module are purposefully extracted from benchmark file to force
+# Ray's cloudpickle behavior when it does NOT serialize the class definition itself
+# along with its payload (instead relying on it being imported)
+#
+
+
+class PayloadPydantic(BaseModel):
+    text: Optional[str] = None
+    floats: Optional[List[float]] = None
+    ints: Optional[List[int]] = None
+    ts: Optional[float] = None
+    reason: Optional[str] = None
+
+
+@dataclass
+class PayloadDataclass:
+    text: Optional[str] = None
+    floats: Optional[List[float]] = None
+    ints: Optional[List[int]] = None
+    ts: Optional[float] = None
+    reason: Optional[str] = None

python/ray/serve/_private/benchmarks/streaming/_grpc/grpc_server.py

@@ -0,0 +1,60 @@
+import grpc
+
+from ray.serve._private.benchmarks.streaming._grpc import (
+    test_server_pb2,
+    test_server_pb2_grpc,
+)
+
+
+async def _async_list(async_iterator):
+    items = []
+    async for item in async_iterator:
+        items.append(item)
+
+    return items
+
+
+class TestGRPCServer(test_server_pb2_grpc.GRPCTestServerServicer):
+    def __init__(self, tokens_per_request):
+        self._tokens_per_request = tokens_per_request
+
+    async def Unary(self, request, context):
+        if request.request_data == "error":
+            await context.abort(
+                code=grpc.StatusCode.INTERNAL,
+                details="unary rpc error",
+            )
+
+        return test_server_pb2.Response(response_data="OK")
+
+    async def ClientStreaming(self, request_iterator, context):
+        data = await _async_list(request_iterator)
+
+        if data and data[0].request_data == "error":
+            await context.abort(
+                code=grpc.StatusCode.INTERNAL,
+                details="client streaming rpc error",
+            )
+
+        return test_server_pb2.Response(response_data="OK")
+
+    async def ServerStreaming(self, request, context):
+        if request.request_data == "error":
+            await context.abort(
+                code=grpc.StatusCode.INTERNAL,
+                details="OK",
+            )
+
+        for i in range(self._tokens_per_request):
+            yield test_server_pb2.Response(response_data="OK")
+
+    async def BidiStreaming(self, request_iterator, context):
+        data = await _async_list(request_iterator)
+        if data and data[0].request_data == "error":
+            await context.abort(
+                code=grpc.StatusCode.INTERNAL,
+                details="bidi-streaming rpc error",
+            )
+
+        for i in range(self._tokens_per_request):
+            yield test_server_pb2.Response(response_data="OK")

python/ray/serve/_private/benchmarks/streaming/_grpc/test_server.proto

@@ -0,0 +1,16 @@
+syntax = "proto3";
+
+message Request {
+  string request_data = 2;
+}
+
+message Response {
+  string response_data = 2;
+}
+
+service GRPCTestServer {
+  rpc Unary(Request) returns (Response);
+  rpc ClientStreaming(stream Request) returns (Response);
+  rpc ServerStreaming(Request) returns (stream Response);
+  rpc BidiStreaming(stream Request) returns (stream Response);
+}

python/ray/serve/_private/benchmarks/streaming/_grpc/test_server_pb2_grpc.py

@@ -0,0 +1,216 @@
+# Generated by the gRPC Python protocol compiler plugin. DO NOT EDIT!
+"""Client and server classes corresponding to protobuf-defined services."""
+import grpc
+
+from ray.serve._private.benchmarks.streaming._grpc import (
+    test_server_pb2 as backend_dot_server_dot_common_dot_clients_dot_grpc_dot_proto_dot_test__server__pb2,
+)
+
+
+class GRPCTestServerStub(object):
+    """Missing associated documentation comment in .proto file."""
+
+    def __init__(self, channel: grpc.Channel):
+        """Constructor.
+
+        Args:
+            channel: A grpc.Channel.
+        """
+        self.Unary = channel.unary_unary(
+            "/GRPCTestServer/Unary",
+            request_serializer=backend_dot_server_dot_common_dot_clients_dot_grpc_dot_proto_dot_test__server__pb2.Request.SerializeToString,
+            response_deserializer=backend_dot_server_dot_common_dot_clients_dot_grpc_dot_proto_dot_test__server__pb2.Response.FromString,
+        )
+        self.ClientStreaming = channel.stream_unary(
+            "/GRPCTestServer/ClientStreaming",
+            request_serializer=backend_dot_server_dot_common_dot_clients_dot_grpc_dot_proto_dot_test__server__pb2.Request.SerializeToString,
+            response_deserializer=backend_dot_server_dot_common_dot_clients_dot_grpc_dot_proto_dot_test__server__pb2.Response.FromString,
+        )
+        self.ServerStreaming = channel.unary_stream(
+            "/GRPCTestServer/ServerStreaming",
+            request_serializer=backend_dot_server_dot_common_dot_clients_dot_grpc_dot_proto_dot_test__server__pb2.Request.SerializeToString,
+            response_deserializer=backend_dot_server_dot_common_dot_clients_dot_grpc_dot_proto_dot_test__server__pb2.Response.FromString,
+        )
+        self.BidiStreaming = channel.stream_stream(
+            "/GRPCTestServer/BidiStreaming",
+            request_serializer=backend_dot_server_dot_common_dot_clients_dot_grpc_dot_proto_dot_test__server__pb2.Request.SerializeToString,
+            response_deserializer=backend_dot_server_dot_common_dot_clients_dot_grpc_dot_proto_dot_test__server__pb2.Response.FromString,
+        )
+
+
+class GRPCTestServerServicer(object):
+    """Missing associated documentation comment in .proto file."""
+
+    def Unary(self, request, context):
+        """Missing associated documentation comment in .proto file."""
+        context.set_code(grpc.StatusCode.UNIMPLEMENTED)
+        context.set_details("Method not implemented!")
+        raise NotImplementedError("Method not implemented!")
+
+    def ClientStreaming(self, request_iterator, context):
+        """Missing associated documentation comment in .proto file."""
+        context.set_code(grpc.StatusCode.UNIMPLEMENTED)
+        context.set_details("Method not implemented!")
+        raise NotImplementedError("Method not implemented!")
+
+    def ServerStreaming(self, request, context):
+        """Missing associated documentation comment in .proto file."""
+        context.set_code(grpc.StatusCode.UNIMPLEMENTED)
+        context.set_details("Method not implemented!")
+        raise NotImplementedError("Method not implemented!")
+
+    def BidiStreaming(self, request_iterator, context):
+        """Missing associated documentation comment in .proto file."""
+        context.set_code(grpc.StatusCode.UNIMPLEMENTED)
+        context.set_details("Method not implemented!")
+        raise NotImplementedError("Method not implemented!")
+
+
+def add_GRPCTestServerServicer_to_server(servicer, server):
+    rpc_method_handlers = {
+        "Unary": grpc.unary_unary_rpc_method_handler(
+            servicer.Unary,
+            request_deserializer=backend_dot_server_dot_common_dot_clients_dot_grpc_dot_proto_dot_test__server__pb2.Request.FromString,
+            response_serializer=backend_dot_server_dot_common_dot_clients_dot_grpc_dot_proto_dot_test__server__pb2.Response.SerializeToString,
+        ),
+        "ClientStreaming": grpc.stream_unary_rpc_method_handler(
+            servicer.ClientStreaming,
+            request_deserializer=backend_dot_server_dot_common_dot_clients_dot_grpc_dot_proto_dot_test__server__pb2.Request.FromString,
+            response_serializer=backend_dot_server_dot_common_dot_clients_dot_grpc_dot_proto_dot_test__server__pb2.Response.SerializeToString,
+        ),
+        "ServerStreaming": grpc.unary_stream_rpc_method_handler(
+            servicer.ServerStreaming,
+            request_deserializer=backend_dot_server_dot_common_dot_clients_dot_grpc_dot_proto_dot_test__server__pb2.Request.FromString,
+            response_serializer=backend_dot_server_dot_common_dot_clients_dot_grpc_dot_proto_dot_test__server__pb2.Response.SerializeToString,
+        ),
+        "BidiStreaming": grpc.stream_stream_rpc_method_handler(
+            servicer.BidiStreaming,
+            request_deserializer=backend_dot_server_dot_common_dot_clients_dot_grpc_dot_proto_dot_test__server__pb2.Request.FromString,
+            response_serializer=backend_dot_server_dot_common_dot_clients_dot_grpc_dot_proto_dot_test__server__pb2.Response.SerializeToString,
+        ),
+    }
+    generic_handler = grpc.method_handlers_generic_handler(
+        "GRPCTestServer", rpc_method_handlers
+    )
+    server.add_generic_rpc_handlers((generic_handler,))
+
+
+# This class is part of an EXPERIMENTAL API.
+class GRPCTestServer(object):
+    """Missing associated documentation comment in .proto file."""
+
+    @staticmethod
+    def Unary(
+        request,
+        target,
+        options=(),
+        channel_credentials=None,
+        call_credentials=None,
+        insecure=False,
+        compression=None,
+        wait_for_ready=None,
+        timeout=None,
+        metadata=None,
+    ):
+        return grpc.experimental.unary_unary(
+            request,
+            target,
+            "/GRPCTestServer/Unary",
+            backend_dot_server_dot_common_dot_clients_dot_grpc_dot_proto_dot_test__server__pb2.Request.SerializeToString,
+            backend_dot_server_dot_common_dot_clients_dot_grpc_dot_proto_dot_test__server__pb2.Response.FromString,
+            options,
+            channel_credentials,
+            insecure,
+            call_credentials,
+            compression,
+            wait_for_ready,
+            timeout,
+            metadata,
+        )
+
+    @staticmethod
+    def ClientStreaming(
+        request_iterator,
+        target,
+        options=(),
+        channel_credentials=None,
+        call_credentials=None,
+        insecure=False,
+        compression=None,
+        wait_for_ready=None,
+        timeout=None,
+        metadata=None,
+    ):
+        return grpc.experimental.stream_unary(
+            request_iterator,
+            target,
+            "/GRPCTestServer/ClientStreaming",
+            backend_dot_server_dot_common_dot_clients_dot_grpc_dot_proto_dot_test__server__pb2.Request.SerializeToString,
+            backend_dot_server_dot_common_dot_clients_dot_grpc_dot_proto_dot_test__server__pb2.Response.FromString,
+            options,
+            channel_credentials,
+            insecure,
+            call_credentials,
+            compression,
+            wait_for_ready,
+            timeout,
+            metadata,
+        )
+
+    @staticmethod
+    def ServerStreaming(
+        request,
+        target,
+        options=(),
+        channel_credentials=None,
+        call_credentials=None,
+        insecure=False,
+        compression=None,
+        wait_for_ready=None,
+        timeout=None,
+        metadata=None,
+    ):
+        return grpc.experimental.unary_stream(
+            request,
+            target,
+            "/GRPCTestServer/ServerStreaming",
+            backend_dot_server_dot_common_dot_clients_dot_grpc_dot_proto_dot_test__server__pb2.Request.SerializeToString,
+            backend_dot_server_dot_common_dot_clients_dot_grpc_dot_proto_dot_test__server__pb2.Response.FromString,
+            options,
+            channel_credentials,
+            insecure,
+            call_credentials,
+            compression,
+            wait_for_ready,
+            timeout,
+            metadata,
+        )
+
+    @staticmethod
+    def BidiStreaming(
+        request_iterator,
+        target,
+        options=(),
+        channel_credentials=None,
+        call_credentials=None,
+        insecure=False,
+        compression=None,
+        wait_for_ready=None,
+        timeout=None,
+        metadata=None,
+    ):
+        return grpc.experimental.stream_stream(
+            request_iterator,
+            target,
+            "/GRPCTestServer/BidiStreaming",
+            backend_dot_server_dot_common_dot_clients_dot_grpc_dot_proto_dot_test__server__pb2.Request.SerializeToString,
+            backend_dot_server_dot_common_dot_clients_dot_grpc_dot_proto_dot_test__server__pb2.Response.FromString,
+            options,
+            channel_credentials,
+            insecure,
+            call_credentials,
+            compression,
+            wait_for_ready,
+            timeout,
+            metadata,
+        )

python/ray/serve/_private/benchmarks/streaming/common.py

@@ -0,0 +1,123 @@
+import abc
+import asyncio
+import enum
+import logging
+import time
+from typing import Tuple, Union
+
+import numpy as np
+
+from ray.actor import ActorHandle
+from ray.runtime_env import RuntimeEnv
+from ray.serve._private.benchmarks.common import Blackhole, run_throughput_benchmark
+from ray.serve._private.benchmarks.serialization.common import PayloadPydantic
+from ray.serve.handle import DeploymentHandle
+
+GRPC_DEBUG_RUNTIME_ENV = RuntimeEnv(
+    env_vars={"GRPC_TRACE": "http", "GRPC_VERBOSITY": "debug"},
+)
+
+
+class IOMode(enum.Enum):
+    SYNC = "SYNC"
+    ASYNC = "ASYNC"
+
+
+class Endpoint:
+    def __init__(self, tokens_per_request: int):
+        self._tokens_per_request = tokens_per_request
+        # Switch off logging to minimize its impact
+        logging.getLogger("ray").setLevel(logging.WARNING)
+        logging.getLogger("ray.serve").setLevel(logging.WARNING)
+
+    def stream(self):
+        payload = PayloadPydantic(
+            text="Test output",
+            floats=[float(f) for f in range(1, 100)],
+            ints=list(range(1, 100)),
+            ts=time.time(),
+            reason="Success!",
+        )
+
+        for i in range(self._tokens_per_request):
+            yield payload
+
+    async def aio_stream(self):
+        payload = PayloadPydantic(
+            text="Test output",
+            floats=[float(f) for f in range(1, 100)],
+            ints=list(range(1, 100)),
+            ts=time.time(),
+            reason="Success!",
+        )
+
+        for i in range(self._tokens_per_request):
+            yield payload
+
+
+class Caller(Blackhole):
+    def __init__(
+        self,
+        downstream: Union[ActorHandle, DeploymentHandle],
+        *,
+        mode: IOMode,
+        tokens_per_request: int,
+        batch_size: int,
+        num_trials: int,
+        trial_runtime: float,
+    ):
+        self._h = downstream
+        self._mode = mode
+        self._tokens_per_request = tokens_per_request
+        self._batch_size = batch_size
+        self._num_trials = num_trials
+        self._trial_runtime = trial_runtime
+        self._durations = []
+
+        # Switch off logging to minimize its impact
+        logging.getLogger("ray").setLevel(logging.WARNING)
+        logging.getLogger("ray.serve").setLevel(logging.WARNING)
+
+    def _get_remote_method(self):
+        if self._mode == IOMode.SYNC:
+            return self._h.stream
+        elif self._mode == IOMode.ASYNC:
+            return self._h.aio_stream
+        else:
+            raise NotImplementedError(f"Streaming mode not supported ({self._mode})")
+
+    @abc.abstractmethod
+    async def _consume_single_stream(self):
+        pass
+
+    async def _do_single_batch(self):
+        durations = await asyncio.gather(
+            *[
+                self._execute(self._consume_single_stream)
+                for _ in range(self._batch_size)
+            ]
+        )
+
+        self._durations.extend(durations)
+
+    async def _execute(self, fn):
+        start = time.monotonic()
+        await fn()
+        dur_s = time.monotonic() - start
+        return dur_s * 1000  # ms
+
+    async def run_benchmark(self) -> Tuple[float, float]:
+        coro = run_throughput_benchmark(
+            fn=self._do_single_batch,
+            multiplier=self._batch_size * self._tokens_per_request,
+            num_trials=self._num_trials,
+            trial_runtime=self._trial_runtime,
+        )
+        # total_runtime = await collect_profile_events(coro)
+        total_runtime = await coro
+
+        p50, p75, p99 = np.percentile(self._durations, [50, 75, 99])
+
+        print(f"Individual request quantiles:\n\tP50={p50}\n\tP75={p75}\n\tP99={p99}")
+
+        return total_runtime

python/ray/serve/_private/benchmarks/streaming/streaming_core_throughput.py

@@ -0,0 +1,95 @@
+import click
+
+import ray
+from ray.serve._private.benchmarks.streaming.common import Caller, Endpoint, IOMode
+
+
+# @ray.remote(runtime_env=GRPC_DEBUG_RUNTIME_ENV)
+@ray.remote
+class EndpointActor(Endpoint):
+    pass
+
+
+# @ray.remote(runtime_env=GRPC_DEBUG_RUNTIME_ENV)
+@ray.remote
+class CallerActor(Caller):
+    async def _consume_single_stream(self):
+        method = self._get_remote_method()
+        async for ref in method.options(num_returns="streaming").remote():
+            r = ray.get(ref)
+
+            # self.sink(str(r, 'utf-8'))
+            self.sink(r)
+
+
+@click.command(help="Benchmark streaming deployment handle throughput.")
+@click.option(
+    "--tokens-per-request",
+    type=int,
+    default=1000,
+    help="Number of tokens (per request) to stream from downstream deployment",
+)
+@click.option(
+    "--batch-size",
+    type=int,
+    default=10,
+    help="Number of requests to send to downstream deployment in each batch.",
+)
+@click.option(
+    "--num-replicas",
+    type=int,
+    default=1,
+    help="Number of replicas in the downstream deployment.",
+)
+@click.option(
+    "--num-trials",
+    type=int,
+    default=5,
+    help="Number of trials of the benchmark to run.",
+)
+@click.option(
+    "--trial-runtime",
+    type=int,
+    default=5,
+    help="Duration to run each trial of the benchmark for (seconds).",
+)
+@click.option(
+    "--io-mode",
+    type=str,
+    default="async",
+    help="Controls mode of the streaming generation (either 'sync' or 'async')",
+)
+def main(
+    tokens_per_request: int,
+    batch_size: int,
+    num_replicas: int,
+    num_trials: int,
+    trial_runtime: float,
+    io_mode: str,
+):
+    h = CallerActor.remote(
+        EndpointActor.remote(
+            tokens_per_request=tokens_per_request,
+        ),
+        mode=IOMode(io_mode.upper()),
+        tokens_per_request=tokens_per_request,
+        batch_size=batch_size,
+        num_trials=num_trials,
+        trial_runtime=trial_runtime,
+    )
+
+    mean, stddev = ray.get(h.run_benchmark.remote())
+    print(
+        "Core Actors streaming throughput ({}) {}: {} +- {} tokens/s".format(
+            io_mode.upper(),
+            f"(num_replicas={num_replicas}, "
+            f"tokens_per_request={tokens_per_request}, "
+            f"batch_size={batch_size})",
+            mean,
+            stddev,
+        )
+    )
+
+
+if __name__ == "__main__":
+    main()

python/ray/serve/_private/benchmarks/streaming/streaming_grpc_throughput.py

@@ -0,0 +1,218 @@
+import asyncio
+import logging
+import time
+from concurrent import futures
+from tempfile import TemporaryDirectory
+
+import click
+import grpc
+
+import ray
+from ray.serve._private.benchmarks.streaming._grpc import (
+    test_server_pb2,
+    test_server_pb2_grpc,
+)
+from ray.serve._private.benchmarks.streaming._grpc.grpc_server import TestGRPCServer
+from ray.serve._private.benchmarks.streaming.common import Caller, IOMode
+
+
+# @ray.remote(runtime_env=GRPC_DEBUG_RUNTIME_ENV)
+@ray.remote
+class EndpointActor:
+    async def __init__(self, tokens_per_request, socket_type, tempdir):
+        # Switch off logging to minimize its impact
+        logging.getLogger("ray").setLevel(logging.WARNING)
+        logging.getLogger("ray.serve").setLevel(logging.WARNING)
+
+        self.server = await self.start_server(tokens_per_request, socket_type, tempdir)
+
+        print("gRPC server started!")
+
+    @staticmethod
+    async def start_server(tokens_per_request, socket_type, tempdir):
+        server = grpc.aio.server(futures.ThreadPoolExecutor(max_workers=1))
+
+        addr, server_creds, _ = _gen_addr_creds(socket_type, tempdir)
+
+        server.add_secure_port(addr, server_creds)
+
+        await server.start()
+
+        test_server_pb2_grpc.add_GRPCTestServerServicer_to_server(
+            TestGRPCServer(tokens_per_request), server
+        )
+
+        return server
+
+
+# @ray.remote(runtime_env=GRPC_DEBUG_RUNTIME_ENV)
+@ray.remote
+class GrpcCallerActor(Caller):
+    def __init__(
+        self,
+        tempdir,
+        socket_type,
+        *,
+        mode: IOMode,
+        tokens_per_request: int,
+        batch_size: int,
+        num_trials: int,
+        trial_runtime: float,
+    ):
+        super().__init__(
+            self.create_downstream(socket_type, tempdir),
+            mode=mode,
+            tokens_per_request=tokens_per_request,
+            batch_size=batch_size,
+            num_trials=num_trials,
+            trial_runtime=trial_runtime,
+        )
+
+    @staticmethod
+    def create_downstream(socket_type, tempdir):
+        addr, _, channel_creds = _gen_addr_creds(socket_type, tempdir)
+
+        channel = grpc.aio.secure_channel(
+            addr, credentials=channel_creds, interceptors=[]
+        )
+
+        return test_server_pb2_grpc.GRPCTestServerStub(channel)
+
+    async def _consume_single_stream(self):
+        try:
+            async for r in self._h.ServerStreaming(test_server_pb2.Request()):
+                self.sink(r)
+        except Exception as e:
+            print(str(e))
+
+
+def _gen_addr_creds(socket_type, tempdir):
+    if socket_type == "uds":
+        addr = f"unix://{tempdir}/server.sock"
+        server_creds = grpc.local_server_credentials(grpc.LocalConnectionType.UDS)
+        channel_creds = grpc.local_channel_credentials(grpc.LocalConnectionType.UDS)
+    elif socket_type == "local_tcp":
+        addr = "127.0.0.1:5432"
+        server_creds = grpc.local_server_credentials(grpc.LocalConnectionType.LOCAL_TCP)
+        channel_creds = grpc.local_channel_credentials(
+            grpc.LocalConnectionType.LOCAL_TCP
+        )
+    else:
+        raise NotImplementedError(f"Not supported socket type ({socket_type})")
+
+    return addr, server_creds, channel_creds
+
+
+async def run_grpc_benchmark(
+    batch_size,
+    io_mode,
+    socket_type,
+    num_replicas,
+    num_trials,
+    tokens_per_request,
+    trial_runtime,
+):
+    with TemporaryDirectory() as tempdir:
+        _ = EndpointActor.remote(
+            tokens_per_request=tokens_per_request,
+            socket_type=socket_type,
+            tempdir=tempdir,
+        )
+
+        ca = GrpcCallerActor.remote(
+            tempdir,
+            socket_type,
+            mode=IOMode(io_mode.upper()),
+            tokens_per_request=tokens_per_request,
+            batch_size=batch_size,
+            num_trials=num_trials,
+            trial_runtime=trial_runtime,
+        )
+
+        # TODO make starting server a method (to make synchronization explicit)
+        time.sleep(5)
+
+        mean, stddev = await ca.run_benchmark.remote()
+
+        print(
+            "gRPC streaming throughput ({}) {}: {} +- {} tokens/s".format(
+                io_mode.upper(),
+                f"(num_replicas={num_replicas}, "
+                f"tokens_per_request={tokens_per_request}, "
+                f"batch_size={batch_size})",
+                mean,
+                stddev,
+            )
+        )
+
+
+@click.command(help="Benchmark streaming deployment handle throughput.")
+@click.option(
+    "--tokens-per-request",
+    type=int,
+    default=1000,
+    help="Number of tokens (per request) to stream from downstream deployment",
+)
+@click.option(
+    "--batch-size",
+    type=int,
+    default=10,
+    help="Number of requests to send to downstream deployment in each batch.",
+)
+@click.option(
+    "--num-replicas",
+    type=int,
+    default=1,
+    help="Number of replicas in the downstream deployment.",
+)
+@click.option(
+    "--num-trials",
+    type=int,
+    default=5,
+    help="Number of trials of the benchmark to run.",
+)
+@click.option(
+    "--trial-runtime",
+    type=int,
+    default=5,
+    help="Duration to run each trial of the benchmark for (seconds).",
+)
+@click.option(
+    "--io-mode",
+    type=str,
+    default="async",
+    help="Controls mode of the streaming generation (either 'sync' or 'async')",
+)
+@click.option(
+    "--socket-type",
+    type=str,
+    default="local_tcp",
+    help="Controls type of socket used as underlying transport (either 'uds' or "
+    "'local_tcp')",
+)
+def main(
+    tokens_per_request: int,
+    batch_size: int,
+    num_replicas: int,
+    num_trials: int,
+    trial_runtime: float,
+    io_mode: str,
+    socket_type: grpc.LocalConnectionType,
+):
+    """Reference benchmark for vanilla Python (w/ C-based core) gRPC implementation"""
+
+    asyncio.run(
+        run_grpc_benchmark(
+            batch_size,
+            io_mode,
+            socket_type,
+            num_replicas,
+            num_trials,
+            tokens_per_request,
+            trial_runtime,
+        )
+    )
+
+
+if __name__ == "__main__":
+    main()

python/ray/serve/_private/benchmarks/streaming/streaming_handle_throughput.py

@@ -0,0 +1,94 @@
+import click
+
+from ray import serve
+from ray.serve._private.benchmarks.streaming.common import Caller, Endpoint, IOMode
+
+
+@serve.deployment(ray_actor_options={"num_cpus": 0})
+class EndpointDeployment(Endpoint):
+    pass
+
+
+@serve.deployment
+class CallerDeployment(Caller):
+    async def _consume_single_stream(self):
+        method = self._get_remote_method().options(
+            stream=True,
+        )
+
+        async for r in method.remote():
+            # Blackhole the response
+            # self.sink(str(r, 'utf-8'))
+            self.sink(r)
+
+
+@click.command(help="Benchmark streaming deployment handle throughput.")
+@click.option(
+    "--tokens-per-request",
+    type=int,
+    default=1000,
+    help="Number of requests to send to downstream deployment in each trial.",
+)
+@click.option(
+    "--batch-size",
+    type=int,
+    default=10,
+    help="Number of requests to send to downstream deployment in each trial.",
+)
+@click.option(
+    "--num-replicas",
+    type=int,
+    default=1,
+    help="Number of replicas in the downstream deployment.",
+)
+@click.option(
+    "--num-trials",
+    type=int,
+    default=5,
+    help="Number of trials of the benchmark to run.",
+)
+@click.option(
+    "--trial-runtime",
+    type=int,
+    default=1,
+    help="Duration to run each trial of the benchmark for (seconds).",
+)
+@click.option(
+    "--io-mode",
+    type=str,
+    default="async",
+    help="Controls mode of the streaming generation (either 'sync' or 'async')",
+)
+def main(
+    tokens_per_request: int,
+    batch_size: int,
+    num_replicas: int,
+    num_trials: int,
+    trial_runtime: float,
+    io_mode: str,
+):
+    app = CallerDeployment.bind(
+        EndpointDeployment.options(num_replicas=num_replicas).bind(tokens_per_request),
+        mode=IOMode(io_mode.upper()),
+        tokens_per_request=tokens_per_request,
+        batch_size=batch_size,
+        num_trials=num_trials,
+        trial_runtime=trial_runtime,
+    )
+    h = serve.run(app)
+
+    mean, stddev = h.run_benchmark.remote().result()
+    print(
+        "DeploymentHandle streaming throughput ({}) {}: {} +- {} tokens/s".format(
+            io_mode.upper(),
+            f"(num_replicas={num_replicas}, "
+            f"tokens_per_request={tokens_per_request}, "
+            f"batch_size={batch_size})",
+            mean,
+            stddev,
+        )
+    )
+
+
+if __name__ == "__main__":
+    main()

python/ray/serve/_private/benchmarks/streaming/streaming_http_throughput.py

@@ -0,0 +1,140 @@
+import asyncio
+import logging
+from typing import Tuple
+
+import aiohttp
+import click
+from starlette.responses import StreamingResponse
+
+from ray import serve
+from ray.serve._private.benchmarks.common import run_throughput_benchmark
+from ray.serve.handle import DeploymentHandle
+
+
+@serve.deployment(ray_actor_options={"num_cpus": 0})
+class Downstream:
+    def __init__(self, tokens_per_request: int):
+        logging.getLogger("ray.serve").setLevel(logging.WARNING)
+
+        self._tokens_per_request = tokens_per_request
+
+    async def stream(self):
+        for i in range(self._tokens_per_request):
+            yield "hi"
+
+    def __call__(self, *args):
+        return StreamingResponse(self.stream())
+
+
+@serve.deployment(ray_actor_options={"num_cpus": 0})
+class Intermediate:
+    def __init__(self, downstream: DeploymentHandle):
+        logging.getLogger("ray.serve").setLevel(logging.WARNING)
+
+        self._h = downstream.options(stream=True)
+
+    async def stream(self):
+        async for token in self._h.stream.remote():
+            yield token
+
+    def __call__(self, *args):
+        return StreamingResponse(self.stream())
+
+
+async def _consume_single_stream():
+    async with aiohttp.ClientSession(raise_for_status=True) as session:
+        async with session.get("http://localhost:8000") as r:
+            async for line in r.content:
+                pass
+
+
+async def run_benchmark(
+    tokens_per_request: int,
+    batch_size: int,
+    num_trials: int,
+    trial_runtime: float,
+) -> Tuple[float, float]:
+    async def _do_single_batch():
+        await asyncio.gather(*[_consume_single_stream() for _ in range(batch_size)])
+
+    return await run_throughput_benchmark(
+        fn=_do_single_batch,
+        multiplier=batch_size * tokens_per_request,
+        num_trials=num_trials,
+        trial_runtime=trial_runtime,
+    )
+
+
+@click.command(help="Benchmark streaming HTTP throughput.")
+@click.option(
+    "--tokens-per-request",
+    type=int,
+    default=1000,
+    help="Number of requests to send to downstream deployment in each trial.",
+)
+@click.option(
+    "--batch-size",
+    type=int,
+    default=10,
+    help="Number of requests to send to downstream deployment in each trial.",
+)
+@click.option(
+    "--num-replicas",
+    type=int,
+    default=1,
+    help="Number of replicas in the downstream deployment.",
+)
+@click.option(
+    "--num-trials",
+    type=int,
+    default=5,
+    help="Number of trials of the benchmark to run.",
+)
+@click.option(
+    "--trial-runtime",
+    type=int,
+    default=1,
+    help="Duration to run each trial of the benchmark for (seconds).",
+)
+@click.option(
+    "--use-intermediate-deployment",
+    is_flag=True,
+    default=False,
+    help="Whether to run an intermediate deployment proxying the requests.",
+)
+def main(
+    tokens_per_request: int,
+    batch_size: int,
+    num_replicas: int,
+    num_trials: int,
+    trial_runtime: float,
+    use_intermediate_deployment: bool,
+):
+    app = Downstream.options(num_replicas=num_replicas).bind(tokens_per_request)
+    if use_intermediate_deployment:
+        app = Intermediate.bind(app)
+
+    serve.run(app)
+
+    mean, stddev = asyncio.new_event_loop().run_until_complete(
+        run_benchmark(
+            tokens_per_request,
+            batch_size,
+            num_trials,
+            trial_runtime,
+        )
+    )
+    print(
+        "HTTP streaming throughput {}: {} +- {} tokens/s".format(
+            f"(num_replicas={num_replicas}, "
+            f"tokens_per_request={tokens_per_request}, "
+            f"batch_size={batch_size}, "
+            f"use_intermediate_deployment={use_intermediate_deployment})",
+            mean,
+            stddev,
+        )
+    )
+
+
+if __name__ == "__main__":
+    main()