chore: import upstream snapshot with attribution

python/ray/serve/_private/benchmarks/streaming/_grpc/grpc_server.py

@@ -0,0 +1,60 @@
+import grpc
+
+from ray.serve._private.benchmarks.streaming._grpc import (
+    test_server_pb2,
+    test_server_pb2_grpc,
+)
+
+
+async def _async_list(async_iterator):
+    items = []
+    async for item in async_iterator:
+        items.append(item)
+
+    return items
+
+
+class TestGRPCServer(test_server_pb2_grpc.GRPCTestServerServicer):
+    def __init__(self, tokens_per_request):
+        self._tokens_per_request = tokens_per_request
+
+    async def Unary(self, request, context):
+        if request.request_data == "error":
+            await context.abort(
+                code=grpc.StatusCode.INTERNAL,
+                details="unary rpc error",
+            )
+
+        return test_server_pb2.Response(response_data="OK")
+
+    async def ClientStreaming(self, request_iterator, context):
+        data = await _async_list(request_iterator)
+
+        if data and data[0].request_data == "error":
+            await context.abort(
+                code=grpc.StatusCode.INTERNAL,
+                details="client streaming rpc error",
+            )
+
+        return test_server_pb2.Response(response_data="OK")
+
+    async def ServerStreaming(self, request, context):
+        if request.request_data == "error":
+            await context.abort(
+                code=grpc.StatusCode.INTERNAL,
+                details="OK",
+            )
+
+        for i in range(self._tokens_per_request):
+            yield test_server_pb2.Response(response_data="OK")
+
+    async def BidiStreaming(self, request_iterator, context):
+        data = await _async_list(request_iterator)
+        if data and data[0].request_data == "error":
+            await context.abort(
+                code=grpc.StatusCode.INTERNAL,
+                details="bidi-streaming rpc error",
+            )
+
+        for i in range(self._tokens_per_request):
+            yield test_server_pb2.Response(response_data="OK")

python/ray/serve/_private/benchmarks/streaming/_grpc/test_server.proto

@@ -0,0 +1,16 @@
+syntax = "proto3";
+
+message Request {
+  string request_data = 2;
+}
+
+message Response {
+  string response_data = 2;
+}
+
+service GRPCTestServer {
+  rpc Unary(Request) returns (Response);
+  rpc ClientStreaming(stream Request) returns (Response);
+  rpc ServerStreaming(Request) returns (stream Response);
+  rpc BidiStreaming(stream Request) returns (stream Response);
+}

python/ray/serve/_private/benchmarks/streaming/_grpc/test_server_pb2_grpc.py

@@ -0,0 +1,216 @@
+# Generated by the gRPC Python protocol compiler plugin. DO NOT EDIT!
+"""Client and server classes corresponding to protobuf-defined services."""
+import grpc
+
+from ray.serve._private.benchmarks.streaming._grpc import (
+    test_server_pb2 as backend_dot_server_dot_common_dot_clients_dot_grpc_dot_proto_dot_test__server__pb2,
+)
+
+
+class GRPCTestServerStub(object):
+    """Missing associated documentation comment in .proto file."""
+
+    def __init__(self, channel: grpc.Channel):
+        """Constructor.
+
+        Args:
+            channel: A grpc.Channel.
+        """
+        self.Unary = channel.unary_unary(
+            "/GRPCTestServer/Unary",
+            request_serializer=backend_dot_server_dot_common_dot_clients_dot_grpc_dot_proto_dot_test__server__pb2.Request.SerializeToString,
+            response_deserializer=backend_dot_server_dot_common_dot_clients_dot_grpc_dot_proto_dot_test__server__pb2.Response.FromString,
+        )
+        self.ClientStreaming = channel.stream_unary(
+            "/GRPCTestServer/ClientStreaming",
+            request_serializer=backend_dot_server_dot_common_dot_clients_dot_grpc_dot_proto_dot_test__server__pb2.Request.SerializeToString,
+            response_deserializer=backend_dot_server_dot_common_dot_clients_dot_grpc_dot_proto_dot_test__server__pb2.Response.FromString,
+        )
+        self.ServerStreaming = channel.unary_stream(
+            "/GRPCTestServer/ServerStreaming",
+            request_serializer=backend_dot_server_dot_common_dot_clients_dot_grpc_dot_proto_dot_test__server__pb2.Request.SerializeToString,
+            response_deserializer=backend_dot_server_dot_common_dot_clients_dot_grpc_dot_proto_dot_test__server__pb2.Response.FromString,
+        )
+        self.BidiStreaming = channel.stream_stream(
+            "/GRPCTestServer/BidiStreaming",
+            request_serializer=backend_dot_server_dot_common_dot_clients_dot_grpc_dot_proto_dot_test__server__pb2.Request.SerializeToString,
+            response_deserializer=backend_dot_server_dot_common_dot_clients_dot_grpc_dot_proto_dot_test__server__pb2.Response.FromString,
+        )
+
+
+class GRPCTestServerServicer(object):
+    """Missing associated documentation comment in .proto file."""
+
+    def Unary(self, request, context):
+        """Missing associated documentation comment in .proto file."""
+        context.set_code(grpc.StatusCode.UNIMPLEMENTED)
+        context.set_details("Method not implemented!")
+        raise NotImplementedError("Method not implemented!")
+
+    def ClientStreaming(self, request_iterator, context):
+        """Missing associated documentation comment in .proto file."""
+        context.set_code(grpc.StatusCode.UNIMPLEMENTED)
+        context.set_details("Method not implemented!")
+        raise NotImplementedError("Method not implemented!")
+
+    def ServerStreaming(self, request, context):
+        """Missing associated documentation comment in .proto file."""
+        context.set_code(grpc.StatusCode.UNIMPLEMENTED)
+        context.set_details("Method not implemented!")
+        raise NotImplementedError("Method not implemented!")
+
+    def BidiStreaming(self, request_iterator, context):
+        """Missing associated documentation comment in .proto file."""
+        context.set_code(grpc.StatusCode.UNIMPLEMENTED)
+        context.set_details("Method not implemented!")
+        raise NotImplementedError("Method not implemented!")
+
+
+def add_GRPCTestServerServicer_to_server(servicer, server):
+    rpc_method_handlers = {
+        "Unary": grpc.unary_unary_rpc_method_handler(
+            servicer.Unary,
+            request_deserializer=backend_dot_server_dot_common_dot_clients_dot_grpc_dot_proto_dot_test__server__pb2.Request.FromString,
+            response_serializer=backend_dot_server_dot_common_dot_clients_dot_grpc_dot_proto_dot_test__server__pb2.Response.SerializeToString,
+        ),
+        "ClientStreaming": grpc.stream_unary_rpc_method_handler(
+            servicer.ClientStreaming,
+            request_deserializer=backend_dot_server_dot_common_dot_clients_dot_grpc_dot_proto_dot_test__server__pb2.Request.FromString,
+            response_serializer=backend_dot_server_dot_common_dot_clients_dot_grpc_dot_proto_dot_test__server__pb2.Response.SerializeToString,
+        ),
+        "ServerStreaming": grpc.unary_stream_rpc_method_handler(
+            servicer.ServerStreaming,
+            request_deserializer=backend_dot_server_dot_common_dot_clients_dot_grpc_dot_proto_dot_test__server__pb2.Request.FromString,
+            response_serializer=backend_dot_server_dot_common_dot_clients_dot_grpc_dot_proto_dot_test__server__pb2.Response.SerializeToString,
+        ),
+        "BidiStreaming": grpc.stream_stream_rpc_method_handler(
+            servicer.BidiStreaming,
+            request_deserializer=backend_dot_server_dot_common_dot_clients_dot_grpc_dot_proto_dot_test__server__pb2.Request.FromString,
+            response_serializer=backend_dot_server_dot_common_dot_clients_dot_grpc_dot_proto_dot_test__server__pb2.Response.SerializeToString,
+        ),
+    }
+    generic_handler = grpc.method_handlers_generic_handler(
+        "GRPCTestServer", rpc_method_handlers
+    )
+    server.add_generic_rpc_handlers((generic_handler,))
+
+
+# This class is part of an EXPERIMENTAL API.
+class GRPCTestServer(object):
+    """Missing associated documentation comment in .proto file."""
+
+    @staticmethod
+    def Unary(
+        request,
+        target,
+        options=(),
+        channel_credentials=None,
+        call_credentials=None,
+        insecure=False,
+        compression=None,
+        wait_for_ready=None,
+        timeout=None,
+        metadata=None,
+    ):
+        return grpc.experimental.unary_unary(
+            request,
+            target,
+            "/GRPCTestServer/Unary",
+            backend_dot_server_dot_common_dot_clients_dot_grpc_dot_proto_dot_test__server__pb2.Request.SerializeToString,
+            backend_dot_server_dot_common_dot_clients_dot_grpc_dot_proto_dot_test__server__pb2.Response.FromString,
+            options,
+            channel_credentials,
+            insecure,
+            call_credentials,
+            compression,
+            wait_for_ready,
+            timeout,
+            metadata,
+        )
+
+    @staticmethod
+    def ClientStreaming(
+        request_iterator,
+        target,
+        options=(),
+        channel_credentials=None,
+        call_credentials=None,
+        insecure=False,
+        compression=None,
+        wait_for_ready=None,
+        timeout=None,
+        metadata=None,
+    ):
+        return grpc.experimental.stream_unary(
+            request_iterator,
+            target,
+            "/GRPCTestServer/ClientStreaming",
+            backend_dot_server_dot_common_dot_clients_dot_grpc_dot_proto_dot_test__server__pb2.Request.SerializeToString,
+            backend_dot_server_dot_common_dot_clients_dot_grpc_dot_proto_dot_test__server__pb2.Response.FromString,
+            options,
+            channel_credentials,
+            insecure,
+            call_credentials,
+            compression,
+            wait_for_ready,
+            timeout,
+            metadata,
+        )
+
+    @staticmethod
+    def ServerStreaming(
+        request,
+        target,
+        options=(),
+        channel_credentials=None,
+        call_credentials=None,
+        insecure=False,
+        compression=None,
+        wait_for_ready=None,
+        timeout=None,
+        metadata=None,
+    ):
+        return grpc.experimental.unary_stream(
+            request,
+            target,
+            "/GRPCTestServer/ServerStreaming",
+            backend_dot_server_dot_common_dot_clients_dot_grpc_dot_proto_dot_test__server__pb2.Request.SerializeToString,
+            backend_dot_server_dot_common_dot_clients_dot_grpc_dot_proto_dot_test__server__pb2.Response.FromString,
+            options,
+            channel_credentials,
+            insecure,
+            call_credentials,
+            compression,
+            wait_for_ready,
+            timeout,
+            metadata,
+        )
+
+    @staticmethod
+    def BidiStreaming(
+        request_iterator,
+        target,
+        options=(),
+        channel_credentials=None,
+        call_credentials=None,
+        insecure=False,
+        compression=None,
+        wait_for_ready=None,
+        timeout=None,
+        metadata=None,
+    ):
+        return grpc.experimental.stream_stream(
+            request_iterator,
+            target,
+            "/GRPCTestServer/BidiStreaming",
+            backend_dot_server_dot_common_dot_clients_dot_grpc_dot_proto_dot_test__server__pb2.Request.SerializeToString,
+            backend_dot_server_dot_common_dot_clients_dot_grpc_dot_proto_dot_test__server__pb2.Response.FromString,
+            options,
+            channel_credentials,
+            insecure,
+            call_credentials,
+            compression,
+            wait_for_ready,
+            timeout,
+            metadata,
+        )

python/ray/serve/_private/benchmarks/streaming/common.py

@@ -0,0 +1,123 @@
+import abc
+import asyncio
+import enum
+import logging
+import time
+from typing import Tuple, Union
+
+import numpy as np
+
+from ray.actor import ActorHandle
+from ray.runtime_env import RuntimeEnv
+from ray.serve._private.benchmarks.common import Blackhole, run_throughput_benchmark
+from ray.serve._private.benchmarks.serialization.common import PayloadPydantic
+from ray.serve.handle import DeploymentHandle
+
+GRPC_DEBUG_RUNTIME_ENV = RuntimeEnv(
+    env_vars={"GRPC_TRACE": "http", "GRPC_VERBOSITY": "debug"},
+)
+
+
+class IOMode(enum.Enum):
+    SYNC = "SYNC"
+    ASYNC = "ASYNC"
+
+
+class Endpoint:
+    def __init__(self, tokens_per_request: int):
+        self._tokens_per_request = tokens_per_request
+        # Switch off logging to minimize its impact
+        logging.getLogger("ray").setLevel(logging.WARNING)
+        logging.getLogger("ray.serve").setLevel(logging.WARNING)
+
+    def stream(self):
+        payload = PayloadPydantic(
+            text="Test output",
+            floats=[float(f) for f in range(1, 100)],
+            ints=list(range(1, 100)),
+            ts=time.time(),
+            reason="Success!",
+        )
+
+        for i in range(self._tokens_per_request):
+            yield payload
+
+    async def aio_stream(self):
+        payload = PayloadPydantic(
+            text="Test output",
+            floats=[float(f) for f in range(1, 100)],
+            ints=list(range(1, 100)),
+            ts=time.time(),
+            reason="Success!",
+        )
+
+        for i in range(self._tokens_per_request):
+            yield payload
+
+
+class Caller(Blackhole):
+    def __init__(
+        self,
+        downstream: Union[ActorHandle, DeploymentHandle],
+        *,
+        mode: IOMode,
+        tokens_per_request: int,
+        batch_size: int,
+        num_trials: int,
+        trial_runtime: float,
+    ):
+        self._h = downstream
+        self._mode = mode
+        self._tokens_per_request = tokens_per_request
+        self._batch_size = batch_size
+        self._num_trials = num_trials
+        self._trial_runtime = trial_runtime
+        self._durations = []
+
+        # Switch off logging to minimize its impact
+        logging.getLogger("ray").setLevel(logging.WARNING)
+        logging.getLogger("ray.serve").setLevel(logging.WARNING)
+
+    def _get_remote_method(self):
+        if self._mode == IOMode.SYNC:
+            return self._h.stream
+        elif self._mode == IOMode.ASYNC:
+            return self._h.aio_stream
+        else:
+            raise NotImplementedError(f"Streaming mode not supported ({self._mode})")
+
+    @abc.abstractmethod
+    async def _consume_single_stream(self):
+        pass
+
+    async def _do_single_batch(self):
+        durations = await asyncio.gather(
+            *[
+                self._execute(self._consume_single_stream)
+                for _ in range(self._batch_size)
+            ]
+        )
+
+        self._durations.extend(durations)
+
+    async def _execute(self, fn):
+        start = time.monotonic()
+        await fn()
+        dur_s = time.monotonic() - start
+        return dur_s * 1000  # ms
+
+    async def run_benchmark(self) -> Tuple[float, float]:
+        coro = run_throughput_benchmark(
+            fn=self._do_single_batch,
+            multiplier=self._batch_size * self._tokens_per_request,
+            num_trials=self._num_trials,
+            trial_runtime=self._trial_runtime,
+        )
+        # total_runtime = await collect_profile_events(coro)
+        total_runtime = await coro
+
+        p50, p75, p99 = np.percentile(self._durations, [50, 75, 99])
+
+        print(f"Individual request quantiles:\n\tP50={p50}\n\tP75={p75}\n\tP99={p99}")
+
+        return total_runtime

python/ray/serve/_private/benchmarks/streaming/streaming_core_throughput.py

@@ -0,0 +1,95 @@
+import click
+
+import ray
+from ray.serve._private.benchmarks.streaming.common import Caller, Endpoint, IOMode
+
+
+# @ray.remote(runtime_env=GRPC_DEBUG_RUNTIME_ENV)
+@ray.remote
+class EndpointActor(Endpoint):
+    pass
+
+
+# @ray.remote(runtime_env=GRPC_DEBUG_RUNTIME_ENV)
+@ray.remote
+class CallerActor(Caller):
+    async def _consume_single_stream(self):
+        method = self._get_remote_method()
+        async for ref in method.options(num_returns="streaming").remote():
+            r = ray.get(ref)
+
+            # self.sink(str(r, 'utf-8'))
+            self.sink(r)
+
+
+@click.command(help="Benchmark streaming deployment handle throughput.")
+@click.option(
+    "--tokens-per-request",
+    type=int,
+    default=1000,
+    help="Number of tokens (per request) to stream from downstream deployment",
+)
+@click.option(
+    "--batch-size",
+    type=int,
+    default=10,
+    help="Number of requests to send to downstream deployment in each batch.",
+)
+@click.option(
+    "--num-replicas",
+    type=int,
+    default=1,
+    help="Number of replicas in the downstream deployment.",
+)
+@click.option(
+    "--num-trials",
+    type=int,
+    default=5,
+    help="Number of trials of the benchmark to run.",
+)
+@click.option(
+    "--trial-runtime",
+    type=int,
+    default=5,
+    help="Duration to run each trial of the benchmark for (seconds).",
+)
+@click.option(
+    "--io-mode",
+    type=str,
+    default="async",
+    help="Controls mode of the streaming generation (either 'sync' or 'async')",
+)
+def main(
+    tokens_per_request: int,
+    batch_size: int,
+    num_replicas: int,
+    num_trials: int,
+    trial_runtime: float,
+    io_mode: str,
+):
+    h = CallerActor.remote(
+        EndpointActor.remote(
+            tokens_per_request=tokens_per_request,
+        ),
+        mode=IOMode(io_mode.upper()),
+        tokens_per_request=tokens_per_request,
+        batch_size=batch_size,
+        num_trials=num_trials,
+        trial_runtime=trial_runtime,
+    )
+
+    mean, stddev = ray.get(h.run_benchmark.remote())
+    print(
+        "Core Actors streaming throughput ({}) {}: {} +- {} tokens/s".format(
+            io_mode.upper(),
+            f"(num_replicas={num_replicas}, "
+            f"tokens_per_request={tokens_per_request}, "
+            f"batch_size={batch_size})",
+            mean,
+            stddev,
+        )
+    )
+
+
+if __name__ == "__main__":
+    main()

python/ray/serve/_private/benchmarks/streaming/streaming_grpc_throughput.py

@@ -0,0 +1,218 @@
+import asyncio
+import logging
+import time
+from concurrent import futures
+from tempfile import TemporaryDirectory
+
+import click
+import grpc
+
+import ray
+from ray.serve._private.benchmarks.streaming._grpc import (
+    test_server_pb2,
+    test_server_pb2_grpc,
+)
+from ray.serve._private.benchmarks.streaming._grpc.grpc_server import TestGRPCServer
+from ray.serve._private.benchmarks.streaming.common import Caller, IOMode
+
+
+# @ray.remote(runtime_env=GRPC_DEBUG_RUNTIME_ENV)
+@ray.remote
+class EndpointActor:
+    async def __init__(self, tokens_per_request, socket_type, tempdir):
+        # Switch off logging to minimize its impact
+        logging.getLogger("ray").setLevel(logging.WARNING)
+        logging.getLogger("ray.serve").setLevel(logging.WARNING)
+
+        self.server = await self.start_server(tokens_per_request, socket_type, tempdir)
+
+        print("gRPC server started!")
+
+    @staticmethod
+    async def start_server(tokens_per_request, socket_type, tempdir):
+        server = grpc.aio.server(futures.ThreadPoolExecutor(max_workers=1))
+
+        addr, server_creds, _ = _gen_addr_creds(socket_type, tempdir)
+
+        server.add_secure_port(addr, server_creds)
+
+        await server.start()
+
+        test_server_pb2_grpc.add_GRPCTestServerServicer_to_server(
+            TestGRPCServer(tokens_per_request), server
+        )
+
+        return server
+
+
+# @ray.remote(runtime_env=GRPC_DEBUG_RUNTIME_ENV)
+@ray.remote
+class GrpcCallerActor(Caller):
+    def __init__(
+        self,
+        tempdir,
+        socket_type,
+        *,
+        mode: IOMode,
+        tokens_per_request: int,
+        batch_size: int,
+        num_trials: int,
+        trial_runtime: float,
+    ):
+        super().__init__(
+            self.create_downstream(socket_type, tempdir),
+            mode=mode,
+            tokens_per_request=tokens_per_request,
+            batch_size=batch_size,
+            num_trials=num_trials,
+            trial_runtime=trial_runtime,
+        )
+
+    @staticmethod
+    def create_downstream(socket_type, tempdir):
+        addr, _, channel_creds = _gen_addr_creds(socket_type, tempdir)
+
+        channel = grpc.aio.secure_channel(
+            addr, credentials=channel_creds, interceptors=[]
+        )
+
+        return test_server_pb2_grpc.GRPCTestServerStub(channel)
+
+    async def _consume_single_stream(self):
+        try:
+            async for r in self._h.ServerStreaming(test_server_pb2.Request()):
+                self.sink(r)
+        except Exception as e:
+            print(str(e))
+
+
+def _gen_addr_creds(socket_type, tempdir):
+    if socket_type == "uds":
+        addr = f"unix://{tempdir}/server.sock"
+        server_creds = grpc.local_server_credentials(grpc.LocalConnectionType.UDS)
+        channel_creds = grpc.local_channel_credentials(grpc.LocalConnectionType.UDS)
+    elif socket_type == "local_tcp":
+        addr = "127.0.0.1:5432"
+        server_creds = grpc.local_server_credentials(grpc.LocalConnectionType.LOCAL_TCP)
+        channel_creds = grpc.local_channel_credentials(
+            grpc.LocalConnectionType.LOCAL_TCP
+        )
+    else:
+        raise NotImplementedError(f"Not supported socket type ({socket_type})")
+
+    return addr, server_creds, channel_creds
+
+
+async def run_grpc_benchmark(
+    batch_size,
+    io_mode,
+    socket_type,
+    num_replicas,
+    num_trials,
+    tokens_per_request,
+    trial_runtime,
+):
+    with TemporaryDirectory() as tempdir:
+        _ = EndpointActor.remote(
+            tokens_per_request=tokens_per_request,
+            socket_type=socket_type,
+            tempdir=tempdir,
+        )
+
+        ca = GrpcCallerActor.remote(
+            tempdir,
+            socket_type,
+            mode=IOMode(io_mode.upper()),
+            tokens_per_request=tokens_per_request,
+            batch_size=batch_size,
+            num_trials=num_trials,
+            trial_runtime=trial_runtime,
+        )
+
+        # TODO make starting server a method (to make synchronization explicit)
+        time.sleep(5)
+
+        mean, stddev = await ca.run_benchmark.remote()
+
+        print(
+            "gRPC streaming throughput ({}) {}: {} +- {} tokens/s".format(
+                io_mode.upper(),
+                f"(num_replicas={num_replicas}, "
+                f"tokens_per_request={tokens_per_request}, "
+                f"batch_size={batch_size})",
+                mean,
+                stddev,
+            )
+        )
+
+
+@click.command(help="Benchmark streaming deployment handle throughput.")
+@click.option(
+    "--tokens-per-request",
+    type=int,
+    default=1000,
+    help="Number of tokens (per request) to stream from downstream deployment",
+)
+@click.option(
+    "--batch-size",
+    type=int,
+    default=10,
+    help="Number of requests to send to downstream deployment in each batch.",
+)
+@click.option(
+    "--num-replicas",
+    type=int,
+    default=1,
+    help="Number of replicas in the downstream deployment.",
+)
+@click.option(
+    "--num-trials",
+    type=int,
+    default=5,
+    help="Number of trials of the benchmark to run.",
+)
+@click.option(
+    "--trial-runtime",
+    type=int,
+    default=5,
+    help="Duration to run each trial of the benchmark for (seconds).",
+)
+@click.option(
+    "--io-mode",
+    type=str,
+    default="async",
+    help="Controls mode of the streaming generation (either 'sync' or 'async')",
+)
+@click.option(
+    "--socket-type",
+    type=str,
+    default="local_tcp",
+    help="Controls type of socket used as underlying transport (either 'uds' or "
+    "'local_tcp')",
+)
+def main(
+    tokens_per_request: int,
+    batch_size: int,
+    num_replicas: int,
+    num_trials: int,
+    trial_runtime: float,
+    io_mode: str,
+    socket_type: grpc.LocalConnectionType,
+):
+    """Reference benchmark for vanilla Python (w/ C-based core) gRPC implementation"""
+
+    asyncio.run(
+        run_grpc_benchmark(
+            batch_size,
+            io_mode,
+            socket_type,
+            num_replicas,
+            num_trials,
+            tokens_per_request,
+            trial_runtime,
+        )
+    )
+
+
+if __name__ == "__main__":
+    main()

python/ray/serve/_private/benchmarks/streaming/streaming_handle_throughput.py

@@ -0,0 +1,94 @@
+import click
+
+from ray import serve
+from ray.serve._private.benchmarks.streaming.common import Caller, Endpoint, IOMode
+
+
+@serve.deployment(ray_actor_options={"num_cpus": 0})
+class EndpointDeployment(Endpoint):
+    pass
+
+
+@serve.deployment
+class CallerDeployment(Caller):
+    async def _consume_single_stream(self):
+        method = self._get_remote_method().options(
+            stream=True,
+        )
+
+        async for r in method.remote():
+            # Blackhole the response
+            # self.sink(str(r, 'utf-8'))
+            self.sink(r)
+
+
+@click.command(help="Benchmark streaming deployment handle throughput.")
+@click.option(
+    "--tokens-per-request",
+    type=int,
+    default=1000,
+    help="Number of requests to send to downstream deployment in each trial.",
+)
+@click.option(
+    "--batch-size",
+    type=int,
+    default=10,
+    help="Number of requests to send to downstream deployment in each trial.",
+)
+@click.option(
+    "--num-replicas",
+    type=int,
+    default=1,
+    help="Number of replicas in the downstream deployment.",
+)
+@click.option(
+    "--num-trials",
+    type=int,
+    default=5,
+    help="Number of trials of the benchmark to run.",
+)
+@click.option(
+    "--trial-runtime",
+    type=int,
+    default=1,
+    help="Duration to run each trial of the benchmark for (seconds).",
+)
+@click.option(
+    "--io-mode",
+    type=str,
+    default="async",
+    help="Controls mode of the streaming generation (either 'sync' or 'async')",
+)
+def main(
+    tokens_per_request: int,
+    batch_size: int,
+    num_replicas: int,
+    num_trials: int,
+    trial_runtime: float,
+    io_mode: str,
+):
+    app = CallerDeployment.bind(
+        EndpointDeployment.options(num_replicas=num_replicas).bind(tokens_per_request),
+        mode=IOMode(io_mode.upper()),
+        tokens_per_request=tokens_per_request,
+        batch_size=batch_size,
+        num_trials=num_trials,
+        trial_runtime=trial_runtime,
+    )
+    h = serve.run(app)
+
+    mean, stddev = h.run_benchmark.remote().result()
+    print(
+        "DeploymentHandle streaming throughput ({}) {}: {} +- {} tokens/s".format(
+            io_mode.upper(),
+            f"(num_replicas={num_replicas}, "
+            f"tokens_per_request={tokens_per_request}, "
+            f"batch_size={batch_size})",
+            mean,
+            stddev,
+        )
+    )
+
+
+if __name__ == "__main__":
+    main()

python/ray/serve/_private/benchmarks/streaming/streaming_http_throughput.py

@@ -0,0 +1,140 @@
+import asyncio
+import logging
+from typing import Tuple
+
+import aiohttp
+import click
+from starlette.responses import StreamingResponse
+
+from ray import serve
+from ray.serve._private.benchmarks.common import run_throughput_benchmark
+from ray.serve.handle import DeploymentHandle
+
+
+@serve.deployment(ray_actor_options={"num_cpus": 0})
+class Downstream:
+    def __init__(self, tokens_per_request: int):
+        logging.getLogger("ray.serve").setLevel(logging.WARNING)
+
+        self._tokens_per_request = tokens_per_request
+
+    async def stream(self):
+        for i in range(self._tokens_per_request):
+            yield "hi"
+
+    def __call__(self, *args):
+        return StreamingResponse(self.stream())
+
+
+@serve.deployment(ray_actor_options={"num_cpus": 0})
+class Intermediate:
+    def __init__(self, downstream: DeploymentHandle):
+        logging.getLogger("ray.serve").setLevel(logging.WARNING)
+
+        self._h = downstream.options(stream=True)
+
+    async def stream(self):
+        async for token in self._h.stream.remote():
+            yield token
+
+    def __call__(self, *args):
+        return StreamingResponse(self.stream())
+
+
+async def _consume_single_stream():
+    async with aiohttp.ClientSession(raise_for_status=True) as session:
+        async with session.get("http://localhost:8000") as r:
+            async for line in r.content:
+                pass
+
+
+async def run_benchmark(
+    tokens_per_request: int,
+    batch_size: int,
+    num_trials: int,
+    trial_runtime: float,
+) -> Tuple[float, float]:
+    async def _do_single_batch():
+        await asyncio.gather(*[_consume_single_stream() for _ in range(batch_size)])
+
+    return await run_throughput_benchmark(
+        fn=_do_single_batch,
+        multiplier=batch_size * tokens_per_request,
+        num_trials=num_trials,
+        trial_runtime=trial_runtime,
+    )
+
+
+@click.command(help="Benchmark streaming HTTP throughput.")
+@click.option(
+    "--tokens-per-request",
+    type=int,
+    default=1000,
+    help="Number of requests to send to downstream deployment in each trial.",
+)
+@click.option(
+    "--batch-size",
+    type=int,
+    default=10,
+    help="Number of requests to send to downstream deployment in each trial.",
+)
+@click.option(
+    "--num-replicas",
+    type=int,
+    default=1,
+    help="Number of replicas in the downstream deployment.",
+)
+@click.option(
+    "--num-trials",
+    type=int,
+    default=5,
+    help="Number of trials of the benchmark to run.",
+)
+@click.option(
+    "--trial-runtime",
+    type=int,
+    default=1,
+    help="Duration to run each trial of the benchmark for (seconds).",
+)
+@click.option(
+    "--use-intermediate-deployment",
+    is_flag=True,
+    default=False,
+    help="Whether to run an intermediate deployment proxying the requests.",
+)
+def main(
+    tokens_per_request: int,
+    batch_size: int,
+    num_replicas: int,
+    num_trials: int,
+    trial_runtime: float,
+    use_intermediate_deployment: bool,
+):
+    app = Downstream.options(num_replicas=num_replicas).bind(tokens_per_request)
+    if use_intermediate_deployment:
+        app = Intermediate.bind(app)
+
+    serve.run(app)
+
+    mean, stddev = asyncio.new_event_loop().run_until_complete(
+        run_benchmark(
+            tokens_per_request,
+            batch_size,
+            num_trials,
+            trial_runtime,
+        )
+    )
+    print(
+        "HTTP streaming throughput {}: {} +- {} tokens/s".format(
+            f"(num_replicas={num_replicas}, "
+            f"tokens_per_request={tokens_per_request}, "
+            f"batch_size={batch_size}, "
+            f"use_intermediate_deployment={use_intermediate_deployment})",
+            mean,
+            stddev,
+        )
+    )
+
+
+if __name__ == "__main__":
+    main()