import asyncio import sys from typing import Any import grpc # coding: utf-8 import pytest import ray from ray import serve from ray._common.test_utils import SignalActor from ray.serve._private.constants import ( RAY_SERVE_ENABLE_DIRECT_INGRESS, RAY_SERVE_ENABLE_HA_PROXY, SERVE_NAMESPACE, ) from ray.serve._private.test_utils import ( get_application_url, ping_fruit_stand, ping_grpc_another_method, ping_grpc_call_method, ping_grpc_healthz, ping_grpc_list_applications, ping_grpc_model_multiplexing, ping_grpc_streaming, send_signal_on_cancellation, ) from ray.serve.config import gRPCOptions from ray.serve.generated import serve_pb2, serve_pb2_grpc from ray.serve.grpc_util import RayServegRPCContext, gRPCInputStream from ray.serve.tests.test_config_files.grpc_deployment import g, g2, multiplexed_g def test_serving_grpc_requests(ray_cluster): """Test serving gRPC requests. When Serve runs with a gRPC deployment, the app should be deployed successfully, both ListApplications and Healthz methods returning successful responses, and registered gRPC methods are routing to the correct replica and return the correct response. """ cluster = ray_cluster cluster.add_node(num_cpus=2) cluster.connect(namespace=SERVE_NAMESPACE) grpc_port = 9000 grpc_servicer_functions = [ "ray.serve.generated.serve_pb2_grpc.add_UserDefinedServiceServicer_to_server", "ray.serve.generated.serve_pb2_grpc.add_FruitServiceServicer_to_server", ] serve.start( grpc_options=gRPCOptions( port=grpc_port, grpc_servicer_functions=grpc_servicer_functions, ), ) channel = grpc.insecure_channel("localhost:9000") # Ensures the not found is responding correctly. app_name = "default" ping_grpc_call_method(channel, app_name, test_not_found=True) serve.run(g) # Get the ingress address dynamically grpc_url = get_application_url("gRPC", use_localhost=True) channel = grpc.insecure_channel(grpc_url) # Ensures ListApplications method succeeding. ping_grpc_list_applications(channel, [app_name]) # Ensures Healthz method succeeding. ping_grpc_healthz(channel) # Ensures a custom defined method is responding correctly. ping_grpc_call_method(channel, app_name) # Ensures another custom defined method is responding correctly. ping_grpc_another_method(channel, app_name) if not RAY_SERVE_ENABLE_HA_PROXY: # Ensure Streaming method is responding correctly. ping_grpc_streaming(channel, app_name) serve.run(g2) # Ensure model composition is responding correctly. grpc_url = get_application_url("gRPC", use_localhost=True) channel = grpc.insecure_channel(grpc_url) ping_fruit_stand(channel, app_name) @pytest.mark.skipif( RAY_SERVE_ENABLE_DIRECT_INGRESS, reason="Model multiplexing is not supported on the ingress deployment when " "direct ingress / HAProxy is enabled (the multiplexed model ID is not " "propagated to the replica).", ) def test_grpc_model_multiplexing(ray_cluster): """Model multiplexing over gRPC routes requests to the correct model.""" cluster = ray_cluster cluster.add_node(num_cpus=2) cluster.connect(namespace=SERVE_NAMESPACE) serve.start( grpc_options=gRPCOptions( port=9000, grpc_servicer_functions=[ "ray.serve.generated.serve_pb2_grpc." "add_UserDefinedServiceServicer_to_server", ], ), ) serve.run(multiplexed_g) channel = grpc.insecure_channel("localhost:9000") ping_grpc_model_multiplexing(channel, "default") def test_serve_start_dictionary_grpc_options(ray_cluster): """Test serve able to start with dictionary grpc_options. When Serve starts with dictionary grpc_options, it should not throw errors and able to serve health check and list applications gRPC requests. """ cluster = ray_cluster cluster.add_node(num_cpus=2) cluster.connect(namespace=SERVE_NAMESPACE) grpc_port = 9000 grpc_servicer_functions = [ "ray.serve.generated.serve_pb2_grpc.add_UserDefinedServiceServicer_to_server", "ray.serve.generated.serve_pb2_grpc.add_FruitServiceServicer_to_server", ] serve.start( grpc_options={ "port": grpc_port, "grpc_servicer_functions": grpc_servicer_functions, }, ) serve.run(g) url = get_application_url("gRPC", use_localhost=True) channel = grpc.insecure_channel(url) # Ensures ListApplications method succeeding. ping_grpc_list_applications(channel, ["default"]) # Ensures Healthz method succeeding. ping_grpc_healthz(channel) def test_grpc_routing_without_metadata(ray_cluster): """Test metadata are not required when calling gRPC proxy with only one app. When there is only one app deployed, gRPC proxy will route the request to the app with or without the metadata. If there are multiple app deployed, without metadata will return a notfound response. """ # Routing doesn't happen in direct ingress mode. if RAY_SERVE_ENABLE_DIRECT_INGRESS: pytest.skip() cluster = ray_cluster cluster.add_node(num_cpus=2) cluster.connect(namespace=SERVE_NAMESPACE) grpc_port = 9000 grpc_servicer_functions = [ "ray.serve.generated.serve_pb2_grpc.add_UserDefinedServiceServicer_to_server", "ray.serve.generated.serve_pb2_grpc.add_FruitServiceServicer_to_server", ] serve.start( grpc_options=gRPCOptions( port=grpc_port, grpc_servicer_functions=grpc_servicer_functions, ), ) app1 = "app1" serve.run(g, name=app1, route_prefix=f"/{app1}") url = get_application_url("gRPC", app_name=app1, use_localhost=True) channel = grpc.insecure_channel(url) stub = serve_pb2_grpc.UserDefinedServiceStub(channel) request = serve_pb2.UserDefinedMessage(name="foo", num=30, foo="bar") # Ensures the gRPC Proxy responding correctly without metadata. response, call = stub.__call__.with_call(request=request) assert call.code() == grpc.StatusCode.OK assert response.greeting == "Hello foo from bar" # Ensures the gRPC Proxy responding correctly with metadata. metadata = (("application", app1),) response, call = stub.__call__.with_call(request=request, metadata=metadata) assert call.code() == grpc.StatusCode.OK assert response.greeting == "Hello foo from bar" # Deploy another app. app2 = "app2" serve.run(g2, name=app2, route_prefix=f"/{app2}") # Ensure the gRPC request without metadata will now return not found response. with pytest.raises(grpc.RpcError) as exception_info: _, _ = stub.__call__.with_call(request=request) rpc_error = exception_info.value assert rpc_error.code() == grpc.StatusCode.NOT_FOUND assert "Application metadata not set" in rpc_error.details() def test_grpc_request_with_request_id(ray_cluster): """Test gRPC request with and without request id. When no request id is passed, gRPC proxy will respond with a random request id in the trailing metadata. When request id is passed, gRPC proxy will respond with the original request id. """ cluster = ray_cluster cluster.add_node(num_cpus=2) cluster.connect(namespace=SERVE_NAMESPACE) grpc_port = 9000 grpc_servicer_functions = [ "ray.serve.generated.serve_pb2_grpc.add_UserDefinedServiceServicer_to_server", "ray.serve.generated.serve_pb2_grpc.add_FruitServiceServicer_to_server", ] serve.start( grpc_options=gRPCOptions( port=grpc_port, grpc_servicer_functions=grpc_servicer_functions, ), ) app1 = "app1" serve.run(g, name=app1, route_prefix=f"/{app1}") url = get_application_url("gRPC", app_name=app1, use_localhost=True) channel = grpc.insecure_channel(url) stub = serve_pb2_grpc.UserDefinedServiceStub(channel) request = serve_pb2.UserDefinedMessage(name="foo", num=30, foo="bar") # Ensures the gRPC Proxy returning the original request id. custom_request_id = "fake-request-id" metadata = (("request_id", custom_request_id),) response, call = stub.__call__.with_call(request=request, metadata=metadata) response_request_id = None for key, value in call.trailing_metadata(): if key == "request_id": response_request_id = value break assert custom_request_id == response_request_id # Ensures the gRPC Proxy returning a new request id. response, call = stub.__call__.with_call(request=request) response_request_id = None for key, value in call.trailing_metadata(): if key == "request_id": response_request_id = value break assert custom_request_id != response_request_id @pytest.mark.parametrize("streaming", [False, True]) def test_grpc_request_timeouts(ray_instance, ray_shutdown, streaming: bool): """Test gRPC request timed out. When the request timed out, gRPC proxy should return timeout response for both unary and streaming request. """ grpc_port = 9000 grpc_servicer_functions = [ "ray.serve.generated.serve_pb2_grpc.add_UserDefinedServiceServicer_to_server", "ray.serve.generated.serve_pb2_grpc.add_FruitServiceServicer_to_server", ] grpc_request_timeout_s = 0.1 serve.start( grpc_options=gRPCOptions( port=grpc_port, grpc_servicer_functions=grpc_servicer_functions, request_timeout_s=grpc_request_timeout_s, ), ) signal_actor = SignalActor.remote() @serve.deployment class HelloModel: async def __call__(self, user_message): await signal_actor.wait.remote() return serve_pb2.UserDefinedResponse(greeting="hello") async def Streaming(self, user_message): for i in range(10): await signal_actor.wait.remote() yield serve_pb2.UserDefinedResponse(greeting="hello") serve.run(HelloModel.bind()) url = get_application_url("gRPC", use_localhost=True) channel = grpc.insecure_channel(url) stub = serve_pb2_grpc.UserDefinedServiceStub(channel) request = serve_pb2.UserDefinedMessage(name="foo", num=30, foo="bar") timeout_response = "timed out after 0.1s." if streaming: # Ensure streaming request respond with timeout response when running longer # than the serve request timeout setting with pytest.raises(grpc.RpcError) as exception_info: list(stub.Streaming(request=request)) else: # Ensure unary request respond with timeout response when running longer than # the serve request timeout setting with pytest.raises(grpc.RpcError) as exception_info: stub.__call__(request=request) rpc_error = exception_info.value assert rpc_error.code() == grpc.StatusCode.DEADLINE_EXCEEDED assert timeout_response in rpc_error.details() # Unblock the handlers to avoid graceful shutdown time. ray.get(signal_actor.send.remote(clear=True)) @pytest.mark.parametrize("streaming", [False, True]) def test_grpc_request_internal_error(ray_instance, ray_shutdown, streaming: bool): """Test gRPC request error out. When the request error out, gRPC proxy should return INTERNAL status and the error message in the response for both unary and streaming request. """ grpc_port = 9000 grpc_servicer_functions = [ "ray.serve.generated.serve_pb2_grpc.add_UserDefinedServiceServicer_to_server", ] serve.start( grpc_options=gRPCOptions( port=grpc_port, grpc_servicer_functions=grpc_servicer_functions, ), ) error_message = "test error case" @serve.deployment() class HelloModel: def __call__(self, user_message): raise RuntimeError(error_message) def Streaming(self, user_message): raise RuntimeError(error_message) model = HelloModel.bind() app_name = "app1" serve.run(model, name=app_name) url = get_application_url("gRPC", app_name=app_name, use_localhost=True) channel = grpc.insecure_channel(url) stub = serve_pb2_grpc.UserDefinedServiceStub(channel) request = serve_pb2.UserDefinedMessage(name="foo", num=30, foo="bar") if streaming: rpc_error = stub.Streaming(request=request) else: with pytest.raises(grpc.RpcError) as exception_info: _ = stub.__call__(request=request) rpc_error = exception_info.value assert rpc_error.code() == grpc.StatusCode.INTERNAL assert error_message in rpc_error.details() @pytest.mark.asyncio @pytest.mark.parametrize("streaming", [False, True]) async def test_grpc_request_cancellation(ray_instance, ray_shutdown, streaming: bool): """Test gRPC request client cancelled. When the request is canceled, gRPC proxy should cancel the underlying task. """ grpc_port = 9000 grpc_servicer_functions = [ "ray.serve.generated.serve_pb2_grpc.add_UserDefinedServiceServicer_to_server", ] serve.start( grpc_options=gRPCOptions( port=grpc_port, grpc_servicer_functions=grpc_servicer_functions, ), ) running_signal_actor = SignalActor.remote() cancelled_signal_actor = SignalActor.remote() @serve.deployment class Downstream: async def wait_for_singal(self): async with send_signal_on_cancellation(cancelled_signal_actor): await running_signal_actor.send.remote() async def __call__(self, *args): await self.wait_for_singal() return serve_pb2.UserDefinedResponse(greeting="hello") async def Streaming(self, *args): await self.wait_for_singal() yield serve_pb2.UserDefinedResponse(greeting="hello") downstream = Downstream.bind() serve.run(downstream, name="downstream", route_prefix="/downstream") # Send a request and wait for it to start executing. url = get_application_url("gRPC", app_name="downstream", use_localhost=True) channel = grpc.insecure_channel(url) stub = serve_pb2_grpc.UserDefinedServiceStub(channel) metadata = (("application", "downstream"),) request = serve_pb2.UserDefinedMessage(name="foo", num=30, foo="bar") if streaming: r = stub.Streaming(request=request, metadata=metadata) else: r = stub.__call__.future(request=request, metadata=metadata) await running_signal_actor.wait.remote() # Cancel it and verify that it is cancelled via signal. r.cancel() await cancelled_signal_actor.wait.remote() with pytest.raises(grpc.FutureCancelledError): r.result() ray.get(running_signal_actor.send.remote(clear=True)) ray.get(cancelled_signal_actor.send.remote(clear=True)) @pytest.mark.parametrize("streaming", [False, True]) def test_using_grpc_context(ray_instance, ray_shutdown, streaming: bool): """Test using gRPC context. When the deployment sets code, details, and trailing metadata in the gRPC context, the response will reflect those values. """ grpc_port = 9000 grpc_servicer_functions = [ "ray.serve.generated.serve_pb2_grpc.add_UserDefinedServiceServicer_to_server", ] serve.start( grpc_options=gRPCOptions( port=grpc_port, grpc_servicer_functions=grpc_servicer_functions, ), ) error_code = grpc.StatusCode.DATA_LOSS error_message = "my specific error message" trailing_metadata = ("foo", "bar") @serve.deployment() class HelloModel: def __call__( self, user_message: serve_pb2.UserDefinedMessage, grpc_context: RayServegRPCContext, ): grpc_context.set_code(error_code) grpc_context.set_details(error_message) grpc_context.set_trailing_metadata([trailing_metadata]) return serve_pb2.UserDefinedResponse(greeting="hello") def Streaming( self, user_message: serve_pb2.UserDefinedMessage, grpc_context: RayServegRPCContext, ): grpc_context.set_code(error_code) grpc_context.set_details(error_message) grpc_context.set_trailing_metadata([trailing_metadata]) yield serve_pb2.UserDefinedResponse(greeting="hello") model = HelloModel.bind() app_name = "app1" serve.run(model, name=app_name) url = get_application_url("gRPC", app_name=app_name, use_localhost=True) channel = grpc.insecure_channel(url) stub = serve_pb2_grpc.UserDefinedServiceStub(channel) request = serve_pb2.UserDefinedMessage(name="foo", num=30, foo="bar") with pytest.raises(grpc.RpcError) as exception_info: if streaming: list(stub.Streaming(request=request)) else: _ = stub.__call__(request=request) rpc_error = exception_info.value assert rpc_error.code() == error_code assert error_message == rpc_error.details() assert trailing_metadata in rpc_error.trailing_metadata() # request_id should always be set in the trailing metadata. assert any([key == "request_id" for key, _ in rpc_error.trailing_metadata()]) @pytest.mark.parametrize("streaming", [False, True]) def test_using_grpc_context_exception(ray_instance, ray_shutdown, streaming: bool): """Test setting code on gRPC context then raised exception. When the deployment sets a status code on the gRPC context and then raises an exception, the user-defined status code should be preserved in the response. """ grpc_port = 9000 grpc_servicer_functions = [ "ray.serve.generated.serve_pb2_grpc.add_UserDefinedServiceServicer_to_server", ] serve.start( grpc_options=gRPCOptions( port=grpc_port, grpc_servicer_functions=grpc_servicer_functions, ), ) user_defined_error_code = grpc.StatusCode.DATA_LOSS real_error_message = "test error" @serve.deployment() class HelloModel: def __call__( self, user_message: serve_pb2.UserDefinedMessage, grpc_context: RayServegRPCContext, ): grpc_context.set_code(user_defined_error_code) raise RuntimeError(real_error_message) def Streaming( self, user_message: serve_pb2.UserDefinedMessage, grpc_context: RayServegRPCContext, ): grpc_context.set_code(user_defined_error_code) raise RuntimeError(real_error_message) model = HelloModel.bind() app_name = "app1" serve.run(model, name=app_name) url = get_application_url("gRPC", app_name=app_name, use_localhost=True) channel = grpc.insecure_channel(url) stub = serve_pb2_grpc.UserDefinedServiceStub(channel) request = serve_pb2.UserDefinedMessage(name="foo", num=30, foo="bar") with pytest.raises(grpc.RpcError) as exception_info: if streaming: list(stub.Streaming(request=request)) else: _ = stub.__call__(request=request) rpc_error = exception_info.value # User-defined status code should be preserved instead of INTERNAL assert rpc_error.code() == user_defined_error_code assert real_error_message in rpc_error.details() @pytest.mark.parametrize("streaming", [False, True]) def test_exception_without_grpc_context_code( ray_instance, ray_shutdown, streaming: bool ): """Test raising exception without setting gRPC status code. When the deployment raises an exception without setting a status code on the gRPC context, the response should be INTERNAL error. """ grpc_port = 9000 grpc_servicer_functions = [ "ray.serve.generated.serve_pb2_grpc.add_UserDefinedServiceServicer_to_server", ] serve.start( grpc_options=gRPCOptions( port=grpc_port, grpc_servicer_functions=grpc_servicer_functions, ), ) real_error_message = "test error without status code" @serve.deployment() class HelloModel: def __call__( self, user_message: serve_pb2.UserDefinedMessage, grpc_context: RayServegRPCContext, ): # Don't set any status code, just raise exception raise RuntimeError(real_error_message) def Streaming( self, user_message: serve_pb2.UserDefinedMessage, grpc_context: RayServegRPCContext, ): # Don't set any status code, just raise exception raise RuntimeError(real_error_message) model = HelloModel.bind() app_name = "app1" serve.run(model, name=app_name) url = get_application_url("gRPC", app_name=app_name, use_localhost=True) channel = grpc.insecure_channel(url) stub = serve_pb2_grpc.UserDefinedServiceStub(channel) request = serve_pb2.UserDefinedMessage(name="foo", num=30, foo="bar") with pytest.raises(grpc.RpcError) as exception_info: if streaming: list(stub.Streaming(request=request)) else: _ = stub.__call__(request=request) rpc_error = exception_info.value # Without user-defined status code, should be INTERNAL assert rpc_error.code() == grpc.StatusCode.INTERNAL assert real_error_message in rpc_error.details() @pytest.mark.parametrize("streaming", [False, True]) @pytest.mark.parametrize("issue", ["incorrect_spelling", "more_args"]) def test_using_grpc_context_bad_function_signature( ray_instance, ray_shutdown, streaming: bool, issue: str ): """Test using gRPC context with bad function signature. When the deployment sets code, details, and trailing metadata in the gRPC context, the response will reflect those values. """ grpc_port = 9000 grpc_servicer_functions = [ "ray.serve.generated.serve_pb2_grpc.add_UserDefinedServiceServicer_to_server", ] serve.start( grpc_options=gRPCOptions( port=grpc_port, grpc_servicer_functions=grpc_servicer_functions, ), ) error_code = grpc.StatusCode.DATA_LOSS error_message = "my specific error message" trailing_metadata = ("foo", "bar") if issue == "incorrect_spelling": @serve.deployment() class HelloModel: def __call__( self, user_message: serve_pb2.UserDefinedMessage, grpc_context_incorrect_spelling: RayServegRPCContext, ): grpc_context_incorrect_spelling.set_code(error_code) grpc_context_incorrect_spelling.set_details(error_message) grpc_context_incorrect_spelling.set_trailing_metadata( [trailing_metadata] ) return serve_pb2.UserDefinedResponse(greeting="hello") def Streaming( self, user_message: serve_pb2.UserDefinedMessage, grpc_context_incorrect_spelling: RayServegRPCContext, ): grpc_context_incorrect_spelling.set_code(error_code) grpc_context_incorrect_spelling.set_details(error_message) grpc_context_incorrect_spelling.set_trailing_metadata( [trailing_metadata] ) yield serve_pb2.UserDefinedResponse(greeting="hello") elif issue == "more_args": @serve.deployment() class HelloModel: def __call__( self, user_message: serve_pb2.UserDefinedMessage, grpc_context: RayServegRPCContext, extra_required_arg: Any, ): grpc_context.set_code(error_code) grpc_context.set_details(error_message) grpc_context.set_trailing_metadata([trailing_metadata]) return serve_pb2.UserDefinedResponse(greeting="hello") def Streaming( self, user_message: serve_pb2.UserDefinedMessage, grpc_context: RayServegRPCContext, extra_required_arg: Any, ): grpc_context.set_code(error_code) grpc_context.set_details(error_message) grpc_context.set_trailing_metadata([trailing_metadata]) yield serve_pb2.UserDefinedResponse(greeting="hello") model = HelloModel.bind() app_name = "app1" serve.run(model, name=app_name) url = get_application_url("gRPC", app_name=app_name, use_localhost=True) channel = grpc.insecure_channel(url) stub = serve_pb2_grpc.UserDefinedServiceStub(channel) request = serve_pb2.UserDefinedMessage(name="foo", num=30, foo="bar") with pytest.raises(grpc.RpcError) as exception_info: if streaming: list(stub.Streaming(request=request)) else: _ = stub.__call__(request=request) rpc_error = exception_info.value assert rpc_error.code() == grpc.StatusCode.INTERNAL assert "missing 1 required positional argument:" in rpc_error.details() if issue == "incorrect_spelling": assert "grpc_context_incorrect_spelling" in rpc_error.details() elif issue == "more_args": assert "extra_required_arg" in rpc_error.details() def test_grpc_client_sending_large_payload(ray_instance, ray_shutdown): """Test gRPC client sending large payload. Serve's gRPC proxy should be configured to allow the client to send large payloads without error. """ grpc_port = 9000 grpc_servicer_functions = [ "ray.serve.generated.serve_pb2_grpc.add_UserDefinedServiceServicer_to_server", ] serve.start( grpc_options=gRPCOptions( port=grpc_port, grpc_servicer_functions=grpc_servicer_functions, ), ) serve.run(g) # This option allows the client to pass larger message. options = [ ("grpc.max_receive_message_length", 1024 * 1024 * 1024), ] url = get_application_url("gRPC", use_localhost=True) channel = grpc.insecure_channel(url, options=options) stub = serve_pb2_grpc.UserDefinedServiceStub(channel) # This is a large payload that exists gRPC's default message limit. large_payload = "foobar" * 1_000_000 request = serve_pb2.UserDefinedMessage(name=large_payload, num=30, foo="bar") metadata = (("application", "default"),) response = stub.__call__(request=request, metadata=metadata) assert response == serve_pb2.UserDefinedResponse( greeting=f"Hello {large_payload} from bar", num_x2=60, ) @pytest.mark.skipif( RAY_SERVE_ENABLE_HA_PROXY, reason="HAProxy does not support streaming." ) def test_grpc_client_streaming(ray_instance, ray_shutdown): """Test gRPC client streaming (stream-unary) requests. When a client sends a stream of requests, the deployment should receive all messages and return a single response. """ grpc_port = 9000 grpc_servicer_functions = [ "ray.serve.generated.serve_pb2_grpc.add_UserDefinedServiceServicer_to_server", ] serve.start( grpc_options=gRPCOptions( port=grpc_port, grpc_servicer_functions=grpc_servicer_functions, ), ) @serve.deployment class ClientStreamingService: async def ClientStreaming(self, request_stream: gRPCInputStream): """Receives stream of requests, returns sum of all num values.""" total = 0 count = 0 async for request in request_stream: total += request.num count += 1 return serve_pb2.UserDefinedResponse( greeting=f"Received {count} messages", num_x2=total * 2, ) serve.run(ClientStreamingService.bind()) url = get_application_url("gRPC", use_localhost=True) channel = grpc.insecure_channel(url) stub = serve_pb2_grpc.UserDefinedServiceStub(channel) # Create a generator of requests def request_generator(): for i in range(5): yield serve_pb2.UserDefinedMessage(name=f"msg_{i}", num=i + 1, foo="bar") # Call client streaming method response = stub.ClientStreaming(request_generator()) assert response.greeting == "Received 5 messages" assert response.num_x2 == 30 # (1+2+3+4+5) * 2 = 30 def test_grpc_unary_not_found(ray_instance, ray_shutdown): """Test gRPC unary returns clean NOT_FOUND when app doesn't exist. When proxy_request yields only ResponseStatus (no body), returning None would cause serialization errors. This verifies unary_unary returns empty bytes and the client receives a clean gRPC error. """ grpc_port = 9000 grpc_servicer_functions = [ "ray.serve.generated.serve_pb2_grpc.add_UserDefinedServiceServicer_to_server", ] serve.start( grpc_options=gRPCOptions( port=grpc_port, grpc_servicer_functions=grpc_servicer_functions, ), ) # Deploy two apps so routing requires metadata; single-app routes to default serve.run(g, name="app1", route_prefix="/app1") serve.run(g, name="app2", route_prefix="/app2") url = get_application_url("gRPC", app_name="app1", use_localhost=True) channel = grpc.insecure_channel(url) stub = serve_pb2_grpc.UserDefinedServiceStub(channel) request = serve_pb2.UserDefinedMessage(name="foo", num=30, foo="bar") # Call with non-existent app metadata - triggers not_found_response (only # ResponseStatus yielded, no body). Before fix, returning None caused # server-side serialization error. with pytest.raises(grpc.RpcError) as exc_info: stub.__call__.with_call( request=request, metadata=(("application", "nonexistent_app"),), ) assert exc_info.value.code() == grpc.StatusCode.NOT_FOUND def test_grpc_client_streaming_not_found(ray_instance, ray_shutdown): """Test gRPC client streaming returns clean NOT_FOUND when app doesn't exist. When proxy_request yields only ResponseStatus (no body), returning None would cause serialization errors. This verifies stream_unary returns empty bytes and the client receives a clean gRPC error. """ grpc_port = 9000 grpc_servicer_functions = [ "ray.serve.generated.serve_pb2_grpc.add_UserDefinedServiceServicer_to_server", ] serve.start( grpc_options=gRPCOptions( port=grpc_port, grpc_servicer_functions=grpc_servicer_functions, ), ) @serve.deployment class ClientStreamingService: async def ClientStreaming(self, request_stream: gRPCInputStream): async for _ in request_stream: pass return serve_pb2.UserDefinedResponse(greeting="ok", num_x2=0) # Deploy two apps so routing requires metadata; single-app routes to default serve.run(ClientStreamingService.bind(), name="app1", route_prefix="/app1") serve.run(ClientStreamingService.bind(), name="app2", route_prefix="/app2") url = get_application_url("gRPC", app_name="app1", use_localhost=True) channel = grpc.insecure_channel(url) stub = serve_pb2_grpc.UserDefinedServiceStub(channel) def request_generator(): yield serve_pb2.UserDefinedMessage(name="x", num=1, foo="bar") # Call with non-existent app metadata - triggers not_found_response (only # ResponseStatus yielded, no body). Before fix, returning None caused # server-side serialization error. with pytest.raises(grpc.RpcError) as exc_info: stub.ClientStreaming( request_generator(), metadata=(("application", "nonexistent_app"),), ) assert exc_info.value.code() == grpc.StatusCode.NOT_FOUND @pytest.mark.skipif( RAY_SERVE_ENABLE_HA_PROXY, reason="HAProxy does not support streaming." ) def test_grpc_bidirectional_streaming(ray_instance, ray_shutdown): """Test gRPC bidirectional streaming (stream-stream) requests. When a client sends a stream of requests, the deployment should yield responses for each request. """ grpc_port = 9000 grpc_servicer_functions = [ "ray.serve.generated.serve_pb2_grpc.add_UserDefinedServiceServicer_to_server", ] serve.start( grpc_options=gRPCOptions( port=grpc_port, grpc_servicer_functions=grpc_servicer_functions, ), ) @serve.deployment class BidiStreamingService: async def BidiStreaming(self, request_stream: gRPCInputStream): """Receives stream of requests, yields response for each.""" async for request in request_stream: yield serve_pb2.UserDefinedResponse( greeting=f"Hello {request.name}", num_x2=request.num * 2, ) serve.run(BidiStreamingService.bind()) url = get_application_url("gRPC", use_localhost=True) channel = grpc.insecure_channel(url) stub = serve_pb2_grpc.UserDefinedServiceStub(channel) # Create a generator of requests def request_generator(): for i in range(3): yield serve_pb2.UserDefinedMessage(name=f"user_{i}", num=i * 10, foo="bar") # Call bidirectional streaming method responses = list(stub.BidiStreaming(request_generator())) assert len(responses) == 3 assert responses[0].greeting == "Hello user_0" assert responses[0].num_x2 == 0 assert responses[1].greeting == "Hello user_1" assert responses[1].num_x2 == 20 assert responses[2].greeting == "Hello user_2" assert responses[2].num_x2 == 40 @pytest.mark.skipif( RAY_SERVE_ENABLE_HA_PROXY, reason="HAProxy does not support streaming." ) def test_grpc_client_streaming_with_grpc_context(ray_instance, ray_shutdown): """Test gRPC client streaming with gRPC context. The deployment should be able to access and modify the gRPC context during client streaming. """ grpc_port = 9000 grpc_servicer_functions = [ "ray.serve.generated.serve_pb2_grpc.add_UserDefinedServiceServicer_to_server", ] serve.start( grpc_options=gRPCOptions( port=grpc_port, grpc_servicer_functions=grpc_servicer_functions, ), ) custom_code = grpc.StatusCode.DATA_LOSS custom_message = "Custom error from client streaming" @serve.deployment class ClientStreamingWithContext: async def ClientStreaming( self, request_stream: gRPCInputStream, grpc_context: RayServegRPCContext, ): """Receives stream and sets custom status code.""" count = 0 async for request in request_stream: count += 1 grpc_context.set_code(custom_code) grpc_context.set_details(custom_message) grpc_context.set_trailing_metadata([("custom-key", "custom-value")]) return serve_pb2.UserDefinedResponse( greeting=f"Processed {count} messages", num_x2=count * 2, ) serve.run(ClientStreamingWithContext.bind()) url = get_application_url("gRPC", use_localhost=True) channel = grpc.insecure_channel(url) stub = serve_pb2_grpc.UserDefinedServiceStub(channel) def request_generator(): for i in range(3): yield serve_pb2.UserDefinedMessage(name=f"msg_{i}", num=i, foo="bar") with pytest.raises(grpc.RpcError) as exception_info: stub.ClientStreaming(request_generator()) rpc_error = exception_info.value assert rpc_error.code() == custom_code assert custom_message == rpc_error.details() assert ("custom-key", "custom-value") in rpc_error.trailing_metadata() @pytest.mark.skipif( RAY_SERVE_ENABLE_HA_PROXY, reason="HAProxy does not support streaming." ) def test_grpc_bidirectional_streaming_with_grpc_context(ray_instance, ray_shutdown): """Test gRPC bidirectional streaming with gRPC context. The deployment should be able to access and modify the gRPC context during bidirectional streaming. """ grpc_port = 9000 grpc_servicer_functions = [ "ray.serve.generated.serve_pb2_grpc.add_UserDefinedServiceServicer_to_server", ] serve.start( grpc_options=gRPCOptions( port=grpc_port, grpc_servicer_functions=grpc_servicer_functions, ), ) custom_code = grpc.StatusCode.DATA_LOSS custom_message = "Custom error from bidi streaming" @serve.deployment class BidiStreamingWithContext: async def BidiStreaming( self, request_stream: gRPCInputStream, grpc_context: RayServegRPCContext, ): """Receives stream and yields responses with custom status.""" grpc_context.set_code(custom_code) grpc_context.set_details(custom_message) grpc_context.set_trailing_metadata([("bidi-key", "bidi-value")]) async for request in request_stream: yield serve_pb2.UserDefinedResponse( greeting=f"Echo: {request.name}", num_x2=request.num * 2, ) serve.run(BidiStreamingWithContext.bind()) url = get_application_url("gRPC", use_localhost=True) channel = grpc.insecure_channel(url) stub = serve_pb2_grpc.UserDefinedServiceStub(channel) def request_generator(): for i in range(2): yield serve_pb2.UserDefinedMessage(name=f"test_{i}", num=i * 5, foo="bar") with pytest.raises(grpc.RpcError) as exception_info: list(stub.BidiStreaming(request_generator())) rpc_error = exception_info.value assert rpc_error.code() == custom_code assert custom_message == rpc_error.details() assert ("bidi-key", "bidi-value") in rpc_error.trailing_metadata() @pytest.mark.skipif( RAY_SERVE_ENABLE_HA_PROXY, reason="HAProxy does not support streaming." ) @pytest.mark.parametrize("streaming_type", ["client", "bidi"]) def test_grpc_streaming_internal_error(ray_instance, ray_shutdown, streaming_type: str): """Test gRPC streaming request with internal error. When the handler raises an exception, it should return INTERNAL status. """ grpc_port = 9000 grpc_servicer_functions = [ "ray.serve.generated.serve_pb2_grpc.add_UserDefinedServiceServicer_to_server", ] serve.start( grpc_options=gRPCOptions( port=grpc_port, grpc_servicer_functions=grpc_servicer_functions, ), ) error_message = "test streaming error" @serve.deployment class ErrorStreamingService: async def ClientStreaming(self, request_stream: gRPCInputStream): async for _ in request_stream: pass raise RuntimeError(error_message) async def BidiStreaming(self, request_stream: gRPCInputStream): async for _ in request_stream: raise RuntimeError(error_message) serve.run(ErrorStreamingService.bind()) url = get_application_url("gRPC", use_localhost=True) channel = grpc.insecure_channel(url) stub = serve_pb2_grpc.UserDefinedServiceStub(channel) def request_generator(): for i in range(3): yield serve_pb2.UserDefinedMessage(name=f"msg_{i}", num=i, foo="bar") if streaming_type == "client": with pytest.raises(grpc.RpcError) as exception_info: stub.ClientStreaming(request_generator()) rpc_error = exception_info.value else: with pytest.raises(grpc.RpcError) as exception_info: list(stub.BidiStreaming(request_generator())) rpc_error = exception_info.value assert rpc_error.code() == grpc.StatusCode.INTERNAL assert error_message in rpc_error.details() @pytest.mark.skipif( RAY_SERVE_ENABLE_HA_PROXY, reason="HAProxy does not support streaming." ) @pytest.mark.parametrize("streaming_type", ["client", "bidi"]) def test_grpc_streaming_timeout(ray_instance, ray_shutdown, streaming_type: str): """Test gRPC streaming request timeout. When the request takes longer than the timeout, it should return DEADLINE_EXCEEDED status. """ grpc_port = 9000 grpc_servicer_functions = [ "ray.serve.generated.serve_pb2_grpc.add_UserDefinedServiceServicer_to_server", ] grpc_request_timeout_s = 0.1 serve.start( grpc_options=gRPCOptions( port=grpc_port, grpc_servicer_functions=grpc_servicer_functions, request_timeout_s=grpc_request_timeout_s, ), ) signal_actor = SignalActor.remote() @serve.deployment class SlowStreamingService: async def ClientStreaming(self, request_stream: gRPCInputStream): async for _ in request_stream: pass # Wait for signal that never comes (causes timeout) await signal_actor.wait.remote() return serve_pb2.UserDefinedResponse(greeting="done") async def BidiStreaming(self, request_stream: gRPCInputStream): async for request in request_stream: await signal_actor.wait.remote() yield serve_pb2.UserDefinedResponse(greeting=f"Hello {request.name}") serve.run(SlowStreamingService.bind()) url = get_application_url("gRPC", use_localhost=True) channel = grpc.insecure_channel(url) stub = serve_pb2_grpc.UserDefinedServiceStub(channel) def request_generator(): for i in range(2): yield serve_pb2.UserDefinedMessage(name=f"msg_{i}", num=i, foo="bar") timeout_response = f"timed out after {grpc_request_timeout_s}s." if streaming_type == "client": with pytest.raises(grpc.RpcError) as exception_info: stub.ClientStreaming(request_generator()) else: with pytest.raises(grpc.RpcError) as exception_info: list(stub.BidiStreaming(request_generator())) rpc_error = exception_info.value assert rpc_error.code() == grpc.StatusCode.DEADLINE_EXCEEDED assert timeout_response in rpc_error.details() # Unblock the handlers to avoid graceful shutdown time. ray.get(signal_actor.send.remote(clear=True)) @pytest.mark.skipif( RAY_SERVE_ENABLE_HA_PROXY, reason="HAProxy does not support streaming." ) def test_grpc_client_streaming_empty_stream(ray_instance, ray_shutdown): """Test gRPC client streaming with empty stream. When the client sends no messages, the handler should still work. """ grpc_port = 9000 grpc_servicer_functions = [ "ray.serve.generated.serve_pb2_grpc.add_UserDefinedServiceServicer_to_server", ] serve.start( grpc_options=gRPCOptions( port=grpc_port, grpc_servicer_functions=grpc_servicer_functions, ), ) @serve.deployment class EmptyStreamService: async def ClientStreaming(self, request_stream: gRPCInputStream): count = 0 async for _ in request_stream: count += 1 return serve_pb2.UserDefinedResponse( greeting=f"Received {count} messages", num_x2=count * 2, ) serve.run(EmptyStreamService.bind()) url = get_application_url("gRPC", use_localhost=True) channel = grpc.insecure_channel(url) stub = serve_pb2_grpc.UserDefinedServiceStub(channel) # Empty generator - sends no messages def empty_generator(): return yield # Never reached, but makes it a generator response = stub.ClientStreaming(empty_generator()) assert response.greeting == "Received 0 messages" assert response.num_x2 == 0 @pytest.mark.skipif( RAY_SERVE_ENABLE_HA_PROXY, reason="HAProxy does not support streaming." ) def test_grpc_bidi_streaming_empty_stream(ray_instance, ray_shutdown): """Test gRPC bidirectional streaming with empty stream. When the client sends no messages, the handler should yield no responses. """ grpc_port = 9000 grpc_servicer_functions = [ "ray.serve.generated.serve_pb2_grpc.add_UserDefinedServiceServicer_to_server", ] serve.start( grpc_options=gRPCOptions( port=grpc_port, grpc_servicer_functions=grpc_servicer_functions, ), ) @serve.deployment class EmptyBidiService: async def BidiStreaming(self, request_stream: gRPCInputStream): async for request in request_stream: yield serve_pb2.UserDefinedResponse( greeting=f"Hello {request.name}", num_x2=request.num * 2, ) serve.run(EmptyBidiService.bind()) url = get_application_url("gRPC", use_localhost=True) channel = grpc.insecure_channel(url) stub = serve_pb2_grpc.UserDefinedServiceStub(channel) # Empty generator - sends no messages def empty_generator(): return yield # Never reached, but makes it a generator responses = list(stub.BidiStreaming(empty_generator())) assert len(responses) == 0 @pytest.mark.asyncio @pytest.mark.skipif( RAY_SERVE_ENABLE_HA_PROXY, reason="HAProxy does not support streaming." ) @pytest.mark.parametrize("streaming_type", ["client", "bidi"]) async def test_grpc_streaming_cancellation( ray_instance, ray_shutdown, streaming_type: str ): """Test gRPC streaming request client cancellation. When the client cancels the request, it should propagate to the handler. """ grpc_port = 9000 grpc_servicer_functions = [ "ray.serve.generated.serve_pb2_grpc.add_UserDefinedServiceServicer_to_server", ] serve.start( grpc_options=gRPCOptions( port=grpc_port, grpc_servicer_functions=grpc_servicer_functions, ), ) running_signal_actor = SignalActor.remote() cancelled_signal_actor = SignalActor.remote() @serve.deployment class CancellableStreamingService: async def ClientStreaming(self, request_stream: gRPCInputStream): async with send_signal_on_cancellation(cancelled_signal_actor): async for _ in request_stream: pass await running_signal_actor.send.remote() # Wait indefinitely - should be cancelled await asyncio.sleep(10000) return serve_pb2.UserDefinedResponse(greeting="done") async def BidiStreaming(self, request_stream: gRPCInputStream): async with send_signal_on_cancellation(cancelled_signal_actor): await running_signal_actor.send.remote() async for request in request_stream: yield serve_pb2.UserDefinedResponse( greeting=f"Hello {request.name}" ) serve.run(CancellableStreamingService.bind()) url = get_application_url("gRPC", use_localhost=True) channel = grpc.insecure_channel(url) stub = serve_pb2_grpc.UserDefinedServiceStub(channel) def request_generator(): for i in range(2): yield serve_pb2.UserDefinedMessage(name=f"msg_{i}", num=i, foo="bar") if streaming_type == "client": r = stub.ClientStreaming.future(request_generator()) else: r = stub.BidiStreaming(request_generator()) await running_signal_actor.wait.remote() # Cancel it and verify that it is cancelled via signal. r.cancel() await cancelled_signal_actor.wait.remote() with pytest.raises(grpc.FutureCancelledError): r.result() ray.get(running_signal_actor.send.remote(clear=True)) ray.get(cancelled_signal_actor.send.remote(clear=True)) @pytest.mark.skipif( RAY_SERVE_ENABLE_HA_PROXY, reason="HAProxy does not support streaming." ) @pytest.mark.parametrize("streaming_type", ["client", "bidi"]) def test_grpc_streaming_context_with_exception( ray_instance, ray_shutdown, streaming_type: str ): """Test setting gRPC context then raising exception in streaming. When the handler sets a custom gRPC status code on the context then raises an exception, the response should preserve the user-set status code. """ grpc_port = 9000 grpc_servicer_functions = [ "ray.serve.generated.serve_pb2_grpc.add_UserDefinedServiceServicer_to_server", ] serve.start( grpc_options=gRPCOptions( port=grpc_port, grpc_servicer_functions=grpc_servicer_functions, ), ) user_defined_error_code = grpc.StatusCode.DATA_LOSS real_error_message = "real streaming error" @serve.deployment class ContextExceptionService: async def ClientStreaming( self, request_stream: gRPCInputStream, grpc_context: RayServegRPCContext, ): grpc_context.set_code(user_defined_error_code) async for _ in request_stream: pass raise RuntimeError(real_error_message) async def BidiStreaming( self, request_stream: gRPCInputStream, grpc_context: RayServegRPCContext, ): grpc_context.set_code(user_defined_error_code) async for _ in request_stream: raise RuntimeError(real_error_message) serve.run(ContextExceptionService.bind()) url = get_application_url("gRPC", use_localhost=True) channel = grpc.insecure_channel(url) stub = serve_pb2_grpc.UserDefinedServiceStub(channel) def request_generator(): for i in range(2): yield serve_pb2.UserDefinedMessage(name=f"msg_{i}", num=i, foo="bar") if streaming_type == "client": with pytest.raises(grpc.RpcError) as exception_info: stub.ClientStreaming(request_generator()) else: with pytest.raises(grpc.RpcError) as exception_info: list(stub.BidiStreaming(request_generator())) rpc_error = exception_info.value assert rpc_error.code() == user_defined_error_code assert real_error_message in rpc_error.details() @pytest.mark.skipif( RAY_SERVE_ENABLE_HA_PROXY, reason="HAProxy does not support streaming." ) @pytest.mark.parametrize("streaming_type", ["client", "bidi"]) def test_grpc_streaming_backpressure(ray_instance, ray_shutdown, streaming_type: str): """Test gRPC streaming with slow consumer (backpressure). When the server processes messages slower than the client sends them, all messages should still be received correctly without loss. """ grpc_port = 9000 grpc_servicer_functions = [ "ray.serve.generated.serve_pb2_grpc.add_UserDefinedServiceServicer_to_server", ] serve.start( grpc_options=gRPCOptions( port=grpc_port, grpc_servicer_functions=grpc_servicer_functions, ), ) num_messages = 20 process_delay_s = 0.05 # 50ms delay per message @serve.deployment class SlowConsumerService: async def ClientStreaming(self, request_stream: gRPCInputStream): """Slow consumer - processes each message with a delay.""" received_messages = [] async for request in request_stream: await asyncio.sleep(process_delay_s) received_messages.append(request.name) return serve_pb2.UserDefinedResponse( greeting=f"Received {len(received_messages)} messages", num_x2=len(received_messages), ) async def BidiStreaming(self, request_stream: gRPCInputStream): """Slow consumer - processes each message with a delay.""" count = 0 async for request in request_stream: await asyncio.sleep(process_delay_s) count += 1 yield serve_pb2.UserDefinedResponse( greeting=f"Processed: {request.name}", num_x2=count, ) serve.run(SlowConsumerService.bind()) url = get_application_url("gRPC", use_localhost=True) channel = grpc.insecure_channel(url) stub = serve_pb2_grpc.UserDefinedServiceStub(channel) # Fast producer - sends messages as quickly as possible def fast_request_generator(): for i in range(num_messages): yield serve_pb2.UserDefinedMessage(name=f"msg_{i}", num=i, foo="bar") if streaming_type == "client": response = stub.ClientStreaming(fast_request_generator()) assert response.greeting == f"Received {num_messages} messages" assert response.num_x2 == num_messages else: responses = list(stub.BidiStreaming(fast_request_generator())) assert len(responses) == num_messages # Verify all messages were processed in order for i, response in enumerate(responses): assert response.greeting == f"Processed: msg_{i}" assert response.num_x2 == i + 1 @pytest.mark.skipif( RAY_SERVE_ENABLE_HA_PROXY, reason="HAProxy does not support streaming." ) @pytest.mark.parametrize("streaming_type", ["client", "bidi"]) def test_grpc_streaming_client_error_mid_stream( ray_instance, ray_shutdown, streaming_type: str ): """Test gRPC streaming when client raises error mid-stream. When the client generator raises an exception while streaming, the error should propagate to the client as a gRPC error. """ grpc_port = 9000 grpc_servicer_functions = [ "ray.serve.generated.serve_pb2_grpc.add_UserDefinedServiceServicer_to_server", ] serve.start( grpc_options=gRPCOptions( port=grpc_port, grpc_servicer_functions=grpc_servicer_functions, ), ) messages_before_error = 2 @serve.deployment class ErrorMidStreamService: async def ClientStreaming(self, request_stream: gRPCInputStream): """Receives messages until client errors.""" received = [] async for request in request_stream: received.append(request.name) return serve_pb2.UserDefinedResponse( greeting=f"Received {len(received)} messages", num_x2=len(received), ) async def BidiStreaming(self, request_stream: gRPCInputStream): """Receives and responds to messages until client errors.""" count = 0 async for request in request_stream: count += 1 yield serve_pb2.UserDefinedResponse( greeting=f"Echo: {request.name}", num_x2=count, ) serve.run(ErrorMidStreamService.bind()) url = get_application_url("gRPC", use_localhost=True) channel = grpc.insecure_channel(url) stub = serve_pb2_grpc.UserDefinedServiceStub(channel) # Generator that sends some messages then raises an error def error_request_generator(): for i in range(messages_before_error): yield serve_pb2.UserDefinedMessage(name=f"msg_{i}", num=i, foo="bar") # Simulate client-side error mid-stream raise RuntimeError("Client error during streaming") # Client error should result in a gRPC error (or RuntimeError propagating) if streaming_type == "client": with pytest.raises((grpc.RpcError,)): stub.ClientStreaming(error_request_generator()) else: with pytest.raises((grpc.RpcError,)): # Consume responses until error propagates list(stub.BidiStreaming(error_request_generator())) @pytest.mark.skipif( RAY_SERVE_ENABLE_HA_PROXY, reason="HAProxy does not support streaming." ) def test_grpc_streaming_client_closes_channel_mid_stream(ray_instance, ray_shutdown): """Test gRPC streaming when client closes channel mid-stream. When the client closes the gRPC channel while streaming, the server should detect this and stop processing. """ grpc_port = 9000 grpc_servicer_functions = [ "ray.serve.generated.serve_pb2_grpc.add_UserDefinedServiceServicer_to_server", ] serve.start( grpc_options=gRPCOptions( port=grpc_port, grpc_servicer_functions=grpc_servicer_functions, ), ) client_ready_signal = SignalActor.remote() server_received_signal = SignalActor.remote() server_cancelled_signal = SignalActor.remote() @serve.deployment class ChannelCloseService: async def BidiStreaming(self, request_stream: gRPCInputStream): """Receives messages and signals when first one arrives.""" async with send_signal_on_cancellation(server_cancelled_signal): count = 0 async for request in request_stream: count += 1 await server_received_signal.send.remote() yield serve_pb2.UserDefinedResponse( greeting=f"Got: {request.name}", num_x2=count, ) # Wait for more messages (will be cancelled) await asyncio.sleep(10000) serve.run(ChannelCloseService.bind()) url = get_application_url("gRPC", use_localhost=True) channel = grpc.insecure_channel(url) stub = serve_pb2_grpc.UserDefinedServiceStub(channel) # Generator that waits for signal before sending more messages def slow_request_generator(): yield serve_pb2.UserDefinedMessage(name="first", num=1, foo="bar") # Wait for server to receive the message ray.get(client_ready_signal.wait.remote()) # This message may or may not be sent depending on channel close timing yield serve_pb2.UserDefinedMessage(name="second", num=2, foo="bar") # Start the streaming call response_iterator = stub.BidiStreaming(slow_request_generator()) # Get first response first_response = next(response_iterator) assert first_response.greeting == "Got: first" # Wait for server to process the first message ray.get(server_received_signal.wait.remote(), timeout=5) # Close the channel abruptly channel.close() # Signal the generator to continue (it will fail to send) ray.get(client_ready_signal.send.remote()) # The iterator should raise an error since channel is closed with pytest.raises(grpc.RpcError): list(response_iterator) # Verify the server received the cancellation ray.get(server_cancelled_signal.wait.remote(), timeout=5) if __name__ == "__main__": sys.exit(pytest.main(["-v", "-s", __file__]))