wehub/ray-project--ray
1
0
Fork 0
Code Issues Pull Requests Actions Deployments Models Agent Notes Packages Projects Releases Wiki Activity

chore: import upstream snapshot with attribution

Browse Source
This commit is contained in:
wehub-resource-sync
2026-07-13 13:17:40 +08:00
commit f1825c8ceb
10096 changed files with 2364182 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
python/ray/serve/tests/BUILD.bazel
+886
View File
@@ -0,0 +1,886 @@
load("@rules_python//python:defs.bzl", "py_library", "py_test")
load("//bazel:python.bzl", "py_test_module_list", "py_test_module_list_with_env_variants")
py_library(
name = "conftest",
srcs = ["conftest.py"],
)
py_library(
name = "common",
srcs = glob(["common/*.py"]),
visibility = [
"//python/ray/serve/tests:__subpackages__",
],
)
# Minimal installation test (should *not* include conftest).
py_test_module_list(
size = "small",
files = [
"test_minimal_installation.py",
],
tags = [
"exclusive",
"minimal",
"team:serve",
],
deps = [
"//python/ray/serve:serve_lib",
],
)
# Custom metrics tests.
py_test_module_list_with_env_variants(
size = "medium",
env_variants = {
"metr_agg_at_controller": {
"env": {
"RAY_SERVE_AGGREGATE_METRICS_AT_CONTROLLER": "1",
"RAY_SERVE_COLLECT_AUTOSCALING_METRICS_ON_HANDLE": "0",
"RAY_SERVE_AUTOSCALING_METRIC_RECORD_INTERVAL_FACTOR": "0.001",
"RAY_SERVE_RECORD_AUTOSCALING_STATS_TIMEOUT_S": "3",
},
"name_suffix": "_metr_agg_at_controller",
},
"metr_agg_at_replicas": {
"env": {
"RAY_SERVE_AGGREGATE_METRICS_AT_CONTROLLER": "0",
"RAY_SERVE_COLLECT_AUTOSCALING_METRICS_ON_HANDLE": "0",
"RAY_SERVE_AUTOSCALING_METRIC_RECORD_INTERVAL_FACTOR": "0.001",
"RAY_SERVE_RECORD_AUTOSCALING_STATS_TIMEOUT_S": "3",
},
"name_suffix": "_metr_agg_at_replicas",
},
},
files = [
"test_custom_autoscaling_metrics.py",
],
tags = [
"exclusive",
"team:serve",
],
deps = [
":common",
":conftest",
"//python/ray/serve:serve_lib",
],
)
# Small tests.
py_test_module_list(
size = "small",
files = [
"test_advanced.py",
"test_cluster_node_info_cache.py",
"test_constructor_failure.py",
"test_deployment_version.py",
"test_enable_task_events.py",
"test_expected_versions.py",
"test_http_cancellation.py",
"test_kv_store.py",
"test_long_poll.py",
"test_persistence.py",
"test_proxy_actor_wrapper.py",
"test_replica_request_context.py",
"test_serve_with_tracing.py",
"test_websockets.py",
],
tags = [
"exclusive",
"team:serve",
],
deps = [
":common",
":conftest",
"//python/ray/serve:serve_lib",
],
)
# Medium tests.
py_test_module_list(
size = "medium",
files = [
"test_actor_replica_wrapper.py",
"test_backpressure.py",
"test_backpressure_grpc.py",
"test_batching.py",
"test_broadcast.py",
"test_callback.py",
"test_cluster.py",
"test_controller.py",
"test_controller_benchmark.py",
"test_controller_recovery.py",
"test_deploy_2.py",
"test_deployment_scheduler_downscale.py",
"test_deployment_topology.py",
"test_failure.py",
"test_failure_2.py",
"test_grpc_e2e.py",
"test_grpc_replica_wrapper.py",
"test_handle.py",
"test_handle_1.py",
"test_handle_2.py",
"test_handle_cancellation.py",
"test_handle_streaming.py",
"test_healthcheck.py",
"test_http_headers.py",
"test_http_routes.py",
"test_https_proxy.py",
"test_list_outbound_deployments.py",
"test_max_replicas_per_node.py",
"test_multiplex.py",
"test_proxy.py",
"test_proxy_response_generator.py",
"test_queue_monitor.py",
"test_ray_client.py",
"test_record_replica_metadata.py",
"test_record_routing_stats.py",
"test_regression.py",
"test_replica_placement_group.py",
"test_request_timeout.py",
"test_round_robin_router.py",
"test_streaming_response.py",
"test_task_consumer_autoscaling.py",
"test_task_processor.py",
"test_util.py",
],
tags = [
"exclusive",
"team:serve",
],
deps = [
":common",
":conftest",
"//python/ray/serve:serve_lib",
],
)
# Large tests.
py_test_module_list(
size = "large",
files = [
"test_deployment_scheduler.py",
],
tags = [
"exclusive",
"team:serve",
],
deps = [
":common",
":conftest",
"//python/ray/serve:serve_lib",
],
)
# Medium tests, don't run on windows.
py_test_module_list(
size = "medium",
env = {
"RAY_SERVE_FAIL_ON_RANK_ERROR": "1",
},
files = [
"test_gcs_failure.py",
"test_gradio.py",
"test_replica_ranks.py",
],
tags = [
"exclusive",
"no_windows",
"team:serve",
],
deps = [
":common",
":conftest",
"//python/ray/serve:serve_lib",
],
)
# Tracing tests (require fixture data and tracing env var for subprocesses).
py_test_module_list(
size = "medium",
data = glob(["fixtures/*.*"]),
env = {
"RAY_SERVE_TRACING_EXPORTER_IMPORT_PATH": "ray.serve._private.tracing_utils:default_tracing_exporter",
"RAY_SERVE_TRACING_SAMPLING_RATIO": "1.0",
},
files = [
"test_tracing_utils.py",
],
tags = [
"exclusive",
"team:serve",
],
deps = [
":common",
":conftest",
"//python/ray/serve:serve_lib",
],
)
# Large tests.
py_test_module_list(
size = "enormous",
data = glob(["test_config_files/**/*"]),
files = [
"test_autoscaling_policy.py",
"test_deploy.py",
"test_fastapi.py",
"test_grpc.py",
"test_logging.py",
"test_logging_2.py",
"test_shutdown.py",
"test_standalone.py",
"test_standalone_3.py",
"test_target_capacity.py",
"test_telemetry.py",
"test_telemetry_1.py",
"test_telemetry_2.py",
],
tags = [
"exclusive",
"team:serve",
],
deps = [
":common",
":conftest",
"//python/ray/serve:serve_lib",
],
)
# Logging tests with client IP logging enabled.
py_test_module_list(
size = "enormous",
env = {"RAY_SERVE_LOG_CLIENT_ADDRESS": "1"},
files = [
"test_logging.py",
"test_logging_2.py",
],
name_suffix = "_with_client_address_logging",
tags = [
"exclusive",
"team:serve",
],
deps = [
":common",
":conftest",
"//python/ray/serve:serve_lib",
],
)
# Don't run gang scheduling tests on Windows
py_test_module_list(
size = "enormous",
files = [
"test_gang_scheduling.py",
],
tags = [
"exclusive",
"no_windows",
"team:serve",
],
deps = [
":common",
":conftest",
"//python/ray/serve:serve_lib",
],
)
# Large tests requiring `test_config_files/`.
py_test_module_list(
size = "large",
data = glob(["test_config_files/**/*"]),
files = [
"test_capacity_queue_router.py",
"test_cli.py",
"test_cli_2.py",
"test_cli_3.py",
"test_cli_4.py",
"test_consistent_hash_router.py",
],
tags = [
"exclusive",
"team:serve",
],
deps = [
":common",
":conftest",
"//python/ray/serve:serve_lib",
],
)
# Large tests require `test_config_files/`, no windows.
py_test_module_list(
size = "large",
data = glob(["test_config_files/**/*"]),
files = [
"test_standalone_2.py",
],
tags = [
"exclusive",
"no_windows",
"team:serve",
],
deps = [
":common",
":conftest",
"//python/ray/serve:serve_lib",
],
)
# Run serially on Windows.
py_test_module_list(
size = "medium",
timeout = "long",
data = glob(["test_config_files/**/*"]),
env = {
"RAY_SERVE_ROUTER_QUEUE_LEN_GAUGE_THROTTLE_S": "0",
"RAY_SERVE_RUN_SYNC_IN_THREADPOOL": "1",
"RAY_SERVE_REPLICA_UTILIZATION_WINDOW_S": "5",
"RAY_SERVE_REPLICA_UTILIZATION_REPORT_INTERVAL_S": "1",
"RAY_SERVE_REPLICA_UTILIZATION_NUM_BUCKETS": "10",
"RAY_SERVE_REPLICA_MAX_PROCESSING_LATENCY_WINDOW_S": "5",
"RAY_SERVE_REPLICA_MAX_PROCESSING_LATENCY_REPORT_INTERVAL_S": "1",
"RAY_SERVE_REPLICA_MAX_PROCESSING_LATENCY_NUM_BUCKETS": "5",
"RAY_SERVE_STATUS_GAUGE_REPORT_INTERVAL_S": "0.1",
},
files = [
"test_deploy_app.py",
"test_deploy_app_2.py",
"test_metrics.py",
"test_metrics_2.py",
"test_metrics_3.py",
],
tags = [
"exclusive",
"team:serve",
"use_all_core_windows",
],
deps = [
":common",
":conftest",
"//python/ray/serve:serve_lib",
],
)
# Run the controller metric high-cardinality opt-out test with the flag disabled.
py_test_module_list(
size = "medium",
args = [
"-k",
"test_disable_high_cardinality_controller_metrics",
],
env = {
"RAY_SERVE_CONTROLLER_METRICS_INCLUDE_HIGH_CARDINALITY_TAGS": "0",
},
files = [
"test_metrics.py",
],
name_suffix = "_controller_metrics_without_high_cardinality",
tags = [
"exclusive",
"team:serve",
"use_all_core_windows",
],
deps = [
":common",
":conftest",
"//python/ray/serve:serve_lib",
],
)
# Deployment actors tests: split into own target with size=large because
# the test file has 39 tests including heavyweight crash-recovery tests
# with 120s waits that exceed the medium/long (900s) timeout under CI load.
py_test_module_list(
size = "large",
data = glob(["test_config_files/**/*"]),
env = {
"RAY_SERVE_ROUTER_QUEUE_LEN_GAUGE_THROTTLE_S": "0",
"RAY_SERVE_RUN_SYNC_IN_THREADPOOL": "1",
"RAY_SERVE_REPLICA_UTILIZATION_WINDOW_S": "5",
"RAY_SERVE_REPLICA_UTILIZATION_REPORT_INTERVAL_S": "1",
"RAY_SERVE_REPLICA_UTILIZATION_NUM_BUCKETS": "10",
"RAY_SERVE_STATUS_GAUGE_REPORT_INTERVAL_S": "0.1",
},
files = [
"test_deployment_actors.py",
"test_deployment_actors_recovery.py",
],
tags = [
"exclusive",
"team:serve",
"use_all_core_windows",
],
deps = [
":common",
":conftest",
"//python/ray/serve:serve_lib",
],
)
# Minimal tests
py_test_module_list(
size = "large",
files = [
"test_api.py",
],
tags = [
"exclusive",
"minimal",
"team:serve",
],
deps = [
":common",
":conftest",
"//python/ray/serve:serve_lib",
],
)
# API tests that run faster
py_test_module_list(
size = "small",
files = [
"test_api_2.py",
],
tags = [
"exclusive",
"minimal",
"team:serve",
],
deps = [
":common",
":conftest",
"//python/ray/serve:serve_lib",
],
)
# Model composition test that needs medium size
py_test_module_list(
size = "medium",
timeout = "moderate",
files = [
"test_model_composition.py",
],
tags = [
"exclusive",
"minimal",
"team:serve",
],
deps = [
":common",
":conftest",
"//python/ray/serve:serve_lib",
],
)
# Post-wheel-build tests.
py_test_module_list(
size = "large",
files = [
"test_runtime_env.py",
"test_runtime_env_2.py",
],
tags = [
"custom_setup",
"exclusive",
"post_wheel_build",
"team:serve",
],
deps = [
":common",
":conftest",
"//python/ray/serve:serve_lib",
],
)
# Runs test_api and test_failure with injected failures in the controller.
py_test(
name = "test_controller_crashes",
size = "large",
srcs = [
"test_api.py",
"test_api_2.py",
"test_controller_crashes.py",
"test_failure.py",
"test_failure_2.py",
],
tags = [
"exclusive",
"team:serve",
],
deps = [
":common",
":conftest",
"//python/ray/serve:serve_lib",
],
)
# Serve HA.
py_test(
name = "test_serve_ha",
size = "medium",
srcs = ["test_serve_ha.py"],
tags = [
"custom_setup",
"exclusive",
"ha_integration",
"team:serve",
],
deps = [
":common",
":conftest",
"//python/ray/serve:serve_lib",
],
)
# ----- TEST FEATURE FLAGS -----
# Test autoscaling with different metric collection configurations
AUTOSCALING_METRIC_ENV_VARIANTS = {
"metr_disab": {
"env": {
"RAY_SERVE_COLLECT_AUTOSCALING_METRICS_ON_HANDLE": "0",
# Make sure queued metrics are cleared out quickly.
"RAY_SERVE_HANDLE_AUTOSCALING_METRIC_PUSH_INTERVAL_S": "0.1",
},
"name_suffix": "_metr_disab",
},
"metr_agg_at_controller": {
"env": {
"RAY_SERVE_AGGREGATE_METRICS_AT_CONTROLLER": "1",
# Make sure queued metrics are cleared out quickly.
"RAY_SERVE_HANDLE_AUTOSCALING_METRIC_PUSH_INTERVAL_S": "0.1",
},
"name_suffix": "_metr_agg_at_controller",
},
"metr_agg_at_controller_and_replicas": {
"env": {
"RAY_SERVE_AGGREGATE_METRICS_AT_CONTROLLER": "1",
"RAY_SERVE_COLLECT_AUTOSCALING_METRICS_ON_HANDLE": "0",
# Make sure queued metrics are cleared out quickly.
"RAY_SERVE_HANDLE_AUTOSCALING_METRIC_PUSH_INTERVAL_S": "0.1",
},
"name_suffix": "_metr_agg_at_controller_and_replicas",
},
}
py_test_module_list_with_env_variants(
size = "large",
env_variants = AUTOSCALING_METRIC_ENV_VARIANTS,
files = [
"test_autoscaling_policy.py",
"test_deploy.py",
"test_standalone_3.py",
"test_target_capacity.py",
],
tags = [
"autoscaling",
"exclusive",
"team:serve",
],
deps = [
":common",
":conftest",
"//python/ray/serve:serve_lib",
],
)
# Test autoscaling with streaming and unary configurations.
# Regression test for https://github.com/ray-project/ray/issues/61551
AUTOSCALING_ENV_VARIANTS = {
"env_overrides": {
"env": {
"RAY_SERVE_LOG_TO_STDERR": "0",
"RAY_SERVE_RUN_ROUTER_IN_SEPARATE_LOOP": "0",
"RAY_SERVE_USE_GRPC_BY_DEFAULT": "1",
},
"name_suffix": "_env_overrides",
},
"throughput_optimized": {
"env": {
"RAY_SERVE_THROUGHPUT_OPTIMIZED": "1",
},
"name_suffix": "_throughput_optimized",
},
}
py_test_module_list_with_env_variants(
size = "large",
args = [
"-k",
"TestAutoscalingWithRejection",
],
env_variants = AUTOSCALING_ENV_VARIANTS,
files = [
"test_autoscaling_policy.py",
],
tags = [
"exclusive",
"team:serve",
],
deps = [
":common",
":conftest",
"//python/ray/serve:serve_lib",
],
)
# Test feature flag for task events.
py_test_module_list(
size = "small",
data = glob(["test_config_files/**/*"]),
env = {"RAY_SERVE_ENABLE_TASK_EVENTS": "1"},
files = [
"test_enable_task_events.py",
],
name_suffix = "_with_task_events_enabled",
tags = [
"exclusive",
"no_windows",
"team:serve",
],
deps = [
":common",
":conftest",
"//python/ray/serve:serve_lib",
],
)
# Medium tests with pack scheduling
py_test_module_list(
size = "medium",
data = glob(["test_config_files/**/*"]),
env = {"RAY_SERVE_USE_PACK_SCHEDULING_STRATEGY": "1"},
files = [
"test_cluster.py",
"test_controller_recovery.py",
"test_gcs_failure.py",
"test_max_replicas_per_node.py",
"test_replica_placement_group.py",
],
name_suffix = "_with_pack_scheduling",
tags = [
"exclusive",
"no_windows",
"team:serve",
],
deps = [
":common",
":conftest",
"//python/ray/serve:serve_lib",
],
)
# Large tests with pack scheduling
py_test_module_list(
size = "enormous",
env = {"RAY_SERVE_USE_PACK_SCHEDULING_STRATEGY": "1"},
files = [
"test_standalone.py",
"test_standalone_3.py",
],
name_suffix = "_with_pack_sche",
tags = [
"exclusive",
"team:serve",
],
deps = [
":common",
":conftest",
"//python/ray/serve:serve_lib",
],
)
# Large tests with pack scheduling, no windows
py_test_module_list(
size = "large",
data = glob(["test_config_files/**/*"]),
env = {"RAY_SERVE_USE_PACK_SCHEDULING_STRATEGY": "1"},
files = [
"test_deployment_scheduler.py",
"test_deployment_scheduler_downscale.py",
"test_gang_scheduling.py",
"test_standalone_2.py",
],
name_suffix = "_with_pack_scheduling",
tags = [
"exclusive",
"no_windows",
"team:serve",
],
deps = [
":common",
":conftest",
"//python/ray/serve:serve_lib",
],
)
# Test handle API with local testing mode.
py_test_module_list(
size = "small",
env = {"RAY_SERVE_FORCE_LOCAL_TESTING_MODE": "1"},
files = [
"test_handle_1.py",
"test_handle_2.py",
"test_handle_cancellation.py",
"test_handle_streaming.py",
],
name_suffix = "_with_local_testing_mode",
tags = [
"exclusive",
"no_windows",
"team:serve",
],
deps = [
":common",
":conftest",
"//python/ray/serve:serve_lib",
],
)
# Test currently off-by-default behavior to run replica sync methods in a threadpool.
# TODO(edoakes): remove this once the FF is flipped on by default.
py_test_module_list(
size = "medium",
env = {"RAY_SERVE_RUN_SYNC_IN_THREADPOOL": "1"},
files = [
"test_replica_sync_methods.py",
],
name_suffix = "_with_run_sync_in_threadpool",
tags = [
"exclusive",
"no_windows",
"team:serve",
],
deps = [
":common",
":conftest",
"//python/ray/serve:serve_lib",
],
)
py_test_module_list(
size = "medium",
env = {"RAY_SERVE_RUN_ROUTER_IN_SEPARATE_LOOP": "0"},
files = [
"test_handle_same_loop.py",
"test_proxy.py",
],
name_suffix = "_with_router_in_same_loop",
tags = [
"exclusive",
"no_windows",
"team:serve",
],
deps = [
":common",
":conftest",
"//python/ray/serve:serve_lib",
],
)
py_test_module_list(
size = "large",
files = [
"test_direct_ingress.py",
],
tags = [
"direct_ingress",
"exclusive",
"team:serve",
],
deps = [
":common",
":conftest",
"//python/ray/serve:serve_lib",
],
)
py_test_module_list(
size = "large",
timeout = "eternal",
files = [
"test_per_request_headers.py",
],
tags = [
"exclusive",
"team:serve",
],
deps = [
":common",
":conftest",
"//python/ray/serve:serve_lib",
],
)
py_test_module_list(
size = "large",
env = {"RAY_SERVE_USE_GRPC_BY_DEFAULT": "1"},
files = [
"test_direct_ingress.py",
],
name_suffix = "_with_grpc",
tags = [
"direct_ingress",
"exclusive",
"team:serve",
],
deps = [
":common",
":conftest",
"//python/ray/serve:serve_lib",
],
)
py_test_module_list(
size = "large",
timeout = "eternal",
env = {"RAY_SERVE_USE_GRPC_BY_DEFAULT": "1"},
files = [
"test_per_request_headers.py",
],
name_suffix = "_with_grpc",
tags = [
"exclusive",
"team:serve",
],
deps = [
":common",
":conftest",
"//python/ray/serve:serve_lib",
],
)
# HAProxy tests need RAY_SERVE_ENABLE_HA_PROXY=1. The HAProxy binary comes from
# the ray-haproxy package in the serve test dependencies.
py_test_module_list_with_env_variants(
size = "large",
env_variants = {
"system": {
"env": {
"RAY_SERVE_ENABLE_HA_PROXY": "1",
"RAY_SERVE_DIRECT_INGRESS_MIN_DRAINING_PERIOD_S": "0.01",
"RAY_SERVE_STATUS_GAUGE_REPORT_INTERVAL_S": "0.1",
},
"name_suffix": "",
},
},
files = [
"test_haproxy.py",
"test_haproxy_api.py",
"test_haproxy_metrics.py",
"test_metrics_haproxy.py",
],
tags = [
"exclusive",
"haproxy",
"no_windows",
"team:serve",
],
deps = [
":common",
":conftest",
"//python/ray/serve:serve_lib",
],
)
python/ray/serve/tests/__init__.py
View File
python/ray/serve/tests/common/remote_uris.py
+11
View File
@@ -0,0 +1,11 @@
# ruff: noqa
"""
This file contains links to pinned versions of remote URIs used in testing.
All tests should use pinned versions to avoid accidental breakages.
"""
TEST_DAG_PINNED_URI = "https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip"
TEST_DEPLOY_GROUP_PINNED_URI = "https://github.com/ray-project/test_deploy_group/archive/67971777e225600720f91f618cdfe71fc47f60ee.zip"
TEST_MODULE_PINNED_URI = "https://github.com/ray-project/test_module/archive/aa6f366f7daa78c98408c27d917a983caa9f888b.zip"
TEST_RUNTIME_ENV_PINNED_URI = "https://github.com/ray-project/test_runtime_env/archive/a82f5fb9e5ddd417aebd18fd6b28caeabf252a37.zip"
python/ray/serve/tests/common/test_modules.py
+96
View File
@@ -0,0 +1,96 @@
"""
Ray decorated classes and functions defined at top of file, importable with
fully qualified name as import_path to test DAG building, artifact generation
and structured deployment.
"""
import asyncio
from typing import Dict, Union
from ray import serve
from ray.actor import ActorHandle
from ray.serve.handle import DeploymentHandle
NESTED_HANDLE_KEY = "nested_handle"
@serve.deployment
class ClassHello:
def __init__(self):
pass
def hello(self):
return "hello"
@serve.deployment
class Model:
def __init__(self, weight: int, ratio: float = None):
self.weight = weight
self.ratio = ratio or 1
def forward(self, input: int):
return self.ratio * self.weight * input
def __call__(self, request):
input_data = request
return self.ratio * self.weight * input_data
@serve.deployment
class Combine:
def __init__(
self,
m1: DeploymentHandle,
m2: Union[DeploymentHandle, Dict[str, DeploymentHandle]],
m2_nested: bool = False,
):
self.m1 = m1
self.m2 = m2.get(NESTED_HANDLE_KEY) if m2_nested else m2
async def __call__(self, req):
if isinstance(self.m1, ActorHandle) and isinstance(self.m2, ActorHandle):
r1_ref = self.m1.forward.remote(req)
r2_ref = self.m2.forward.remote(req)
else:
r1_ref = await self.m1.forward.remote(req)._to_object_ref()
r2_ref = await self.m2.forward.remote(req)._to_object_ref()
return sum(await asyncio.gather(r1_ref, r2_ref))
@serve.deployment
class Counter:
def __init__(self, val):
self.val = val
def get(self):
return self.val
def inc(self, inc):
self.val += inc
@serve.deployment
def fn_hello():
return "hello"
@serve.deployment
def fn(val, incr=0):
return val + incr
@serve.deployment
def combine(m1_output, m2_output, kwargs_output=0):
return m1_output + m2_output + kwargs_output
def class_factory():
class MyInlineClass:
def __init__(self, val):
self.val = val
def get(self):
return self.val
return MyInlineClass
python/ray/serve/tests/conftest.py
+458
View File
@@ -0,0 +1,458 @@
import os
import random
import socket
import subprocess
import tempfile
from contextlib import contextmanager
from copy import deepcopy
from typing import Any, Dict, Generator
import httpx
import pytest
import pytest_asyncio
import ray
from ray import serve
from ray._common.test_utils import SignalActor, wait_for_condition
from ray._common.usage import usage_lib
from ray._common.utils import reset_ray_address
from ray.cluster_utils import AutoscalingCluster, Cluster
from ray.serve._private.test_utils import (
TELEMETRY_ROUTE_PREFIX,
TEST_METRICS_EXPORT_PORT,
check_ray_started,
check_ray_stopped,
start_telemetry_app,
)
from ray.serve.config import HTTPOptions, ProxyLocation, gRPCOptions
from ray.serve.context import _get_global_client
from ray.tests.conftest import ( # noqa
external_redis,
propagate_logs,
pytest_runtest_makereport,
)
# https://tools.ietf.org/html/rfc6335#section-6
MIN_DYNAMIC_PORT = 49152
MAX_DYNAMIC_PORT = 65535
TEST_GRPC_SERVICER_FUNCTIONS = [
"ray.serve.generated.serve_pb2_grpc.add_UserDefinedServiceServicer_to_server",
"ray.serve.generated.serve_pb2_grpc.add_FruitServiceServicer_to_server",
]
if os.environ.get("RAY_SERVE_INTENTIONALLY_CRASH", False) == 1:
serve.controller._CRASH_AFTER_CHECKPOINT_PROBABILITY = 0.5
@pytest.fixture(autouse=True)
def _clear_stale_ray_address():
# Serve CI runs several test targets per container sharing /tmp/ray; a target
# killed mid-run can leave a ray_current_cluster pointing at a dead cluster.
# Drop it before each test so an address-less ray.init() starts fresh.
reset_ray_address()
yield
@pytest.fixture
def ray_shutdown():
serve.shutdown()
if ray.is_initialized():
ray.shutdown()
yield
serve.shutdown()
if ray.is_initialized():
ray.shutdown()
@pytest.fixture
def ray_cluster():
cluster = Cluster()
yield cluster
serve.shutdown()
ray.shutdown()
cluster.shutdown()
@pytest.fixture
def ray_autoscaling_cluster(request):
# NOTE(zcin): We have to make a deepcopy here because AutoscalingCluster
# modifies the dictionary that's passed in.
params = deepcopy(request.param)
cluster = AutoscalingCluster(**params)
cluster.start()
yield
serve.shutdown()
ray.shutdown()
cluster.shutdown()
@pytest.fixture
def ray_start(scope="module"):
port = random.randint(MIN_DYNAMIC_PORT, MAX_DYNAMIC_PORT)
subprocess.check_output(
[
"ray",
"start",
"--head",
"--num-cpus",
"16",
"--ray-client-server-port",
f"{port}",
]
)
try:
yield f"localhost:{port}"
finally:
subprocess.check_output(["ray", "stop", "--force"])
def _check_ray_stop():
try:
httpx.get("http://localhost:8265/api/ray/version")
return False
except Exception:
return True
@contextmanager
def start_and_shutdown_ray_cli():
subprocess.check_output(["ray", "stop", "--force"])
wait_for_condition(_check_ray_stop, timeout=15)
subprocess.check_output(["ray", "start", "--head"])
yield
subprocess.check_output(["ray", "stop", "--force"])
wait_for_condition(_check_ray_stop, timeout=15)
@pytest.fixture(scope="module")
def start_and_shutdown_ray_cli_module():
with start_and_shutdown_ray_cli():
yield
@pytest.fixture
def tmp_dir():
with tempfile.TemporaryDirectory() as tmp_dir:
old_dir = os.getcwd()
os.chdir(tmp_dir)
yield tmp_dir
os.chdir(old_dir)
@pytest.fixture(scope="session")
def _shared_serve_instance():
# Note(simon):
# This line should be not turned on on master because it leads to very
# spammy and not useful log in case of a failure in CI.
# To run locally, please use this instead.
# SERVE_DEBUG_LOG=1 pytest -v -s test_api.py
# os.environ["SERVE_DEBUG_LOG"] = "1" <- Do not uncomment this.
# Overriding task_retry_delay_ms to relaunch actors more quickly
ray.init(
address="local",
num_cpus=36,
namespace="default_test_namespace",
_metrics_export_port=9999,
_system_config={"metrics_report_interval_ms": 1000, "task_retry_delay_ms": 50},
)
serve.start(
proxy_location=ProxyLocation.HeadOnly,
http_options={"host": "0.0.0.0"},
grpc_options={
"port": 9000,
"grpc_servicer_functions": TEST_GRPC_SERVICER_FUNCTIONS,
},
)
yield _get_global_client()
# Shutdown Serve and Ray when the session ends so that proxy actors
# (e.g. HAProxyManager) run their shutdown logic and stop subprocesses.
serve.shutdown()
@pytest_asyncio.fixture
async def serve_instance_async(_shared_serve_instance):
yield _shared_serve_instance
# Clear all state for 2.x applications and deployments.
_shared_serve_instance.delete_all_apps()
# Clear the ServeHandle cache between tests to avoid them piling up.
await _shared_serve_instance.shutdown_cached_handles_async()
@pytest.fixture
def serve_instance(_shared_serve_instance):
yield _shared_serve_instance
# Clear all state for 2.x applications and deployments.
_shared_serve_instance.delete_all_apps()
# Clear the ServeHandle cache between tests to avoid them piling up.
_shared_serve_instance.shutdown_cached_handles()
@pytest.fixture
def serve_instance_with_signal(serve_instance):
client = serve_instance
signal = SignalActor.options(name="signal123").remote()
yield client, signal
# Delete signal actor so there is no conflict between tests
ray.kill(signal)
def check_ray_stop():
try:
httpx.get("http://localhost:8265/api/ray/version")
return False
except Exception:
return True
@pytest.fixture(scope="function")
def ray_start_stop():
subprocess.check_output(["ray", "stop", "--force"])
ray.shutdown()
wait_for_condition(
check_ray_stop,
timeout=15,
)
subprocess.check_output(["ray", "start", "--head"])
wait_for_condition(
lambda: httpx.get("http://localhost:8265/api/ray/version").status_code == 200,
timeout=15,
)
ray.init("auto")
yield
serve.shutdown()
ray.shutdown()
subprocess.check_output(["ray", "stop", "--force"])
wait_for_condition(
check_ray_stop,
timeout=15,
)
@pytest.fixture(scope="function")
def ray_start_stop_in_specific_directory(request):
original_working_dir = os.getcwd()
# Change working directory so Ray will start in the requested directory.
new_working_dir = request.param
os.chdir(new_working_dir)
print(f"\nChanged working directory to {new_working_dir}\n")
subprocess.check_output(["ray", "start", "--head"])
wait_for_condition(
lambda: httpx.get("http://localhost:8265/api/ray/version").status_code == 200,
timeout=15,
)
try:
yield
finally:
# Change the directory back to the original one.
os.chdir(original_working_dir)
print(f"\nChanged working directory back to {original_working_dir}\n")
subprocess.check_output(["ray", "stop", "--force"])
wait_for_condition(
check_ray_stop,
timeout=15,
)
@pytest.fixture
def ray_instance(
request: pytest.FixtureRequest,
) -> Generator[Dict[str, Any], None, None]:
"""Starts and stops a Ray instance for this test.
Args:
request: request.param should contain a dictionary of env vars and
their values. The Ray instance will be started with these env vars.
Yields:
Dict[str, Any]: The dict returned by ``ray.init`` for the started cluster.
"""
original_env_vars = os.environ.copy()
try:
requested_env_vars = request.param
except AttributeError:
requested_env_vars = {}
os.environ.update(requested_env_vars)
yield ray.init(
address="local",
_metrics_export_port=9999,
_system_config={
"metrics_report_interval_ms": 1000,
"task_retry_delay_ms": 50,
},
)
serve.shutdown()
ray.shutdown()
os.environ.clear()
os.environ.update(original_env_vars)
@pytest.fixture
def manage_ray_with_telemetry(monkeypatch):
with monkeypatch.context() as m:
m.setenv("RAY_USAGE_STATS_ENABLED", "1")
m.setenv(
"RAY_USAGE_STATS_REPORT_URL",
f"http://127.0.0.1:8000{TELEMETRY_ROUTE_PREFIX}",
)
m.setenv("RAY_USAGE_STATS_REPORT_INTERVAL_S", "1")
subprocess.check_output(["ray", "stop", "--force"])
wait_for_condition(check_ray_stopped, timeout=5)
subprocess.check_output(["ray", "start", "--head"])
wait_for_condition(check_ray_started, timeout=5)
storage = start_telemetry_app()
wait_for_condition(
lambda: ray.get(storage.get_reports_received.remote()) > 1, timeout=15
)
yield storage
# Call Python API shutdown() methods to clear global variable state
serve.shutdown()
ray.shutdown()
# Reset global state (any keys that may have been set and cached while the
# workload was running).
usage_lib.reset_global_state()
# Shut down Ray cluster with CLI
subprocess.check_output(["ray", "stop", "--force"])
wait_for_condition(check_ray_stopped, timeout=5)
def wait_for_metrics_port_free(port=TEST_METRICS_EXPORT_PORT, timeout=30):
"""
Ensures the metrics export port is freed.
"""
def port_free():
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
try:
s.bind(("", port))
return True
except OSError:
return False
finally:
s.close()
wait_for_condition(port_free, timeout=timeout, retry_interval_ms=200)
def wait_for_metrics_endpoint(session_name, port=TEST_METRICS_EXPORT_PORT, timeout=30):
"""
Ensures the current dashboard agent is serving the metrics endpoint. A
timeout indicates another agent is still running and holding the port.
"""
def ready():
try:
resp = httpx.get(f"http://localhost:{port}/metrics", timeout=1.0)
except Exception:
return False
return resp.status_code == 200 and f'SessionName="{session_name}"' in resp.text
wait_for_condition(ready, timeout=timeout, retry_interval_ms=500)
@pytest.fixture
def metrics_start_shutdown(request):
param = request.param if hasattr(request, "param") else None
request_timeout_s = param if param else None
"""Fixture provides a fresh Ray cluster to prevent metrics state sharing."""
wait_for_metrics_port_free()
ray.init(
address="local",
_metrics_export_port=TEST_METRICS_EXPORT_PORT,
_system_config={
"metrics_report_interval_ms": 100,
"task_retry_delay_ms": 50,
},
)
try:
session_name = ray._private.worker._global_node.session_name
wait_for_metrics_endpoint(session_name)
grpc_port = 9000
grpc_servicer_functions = [
"ray.serve.generated.serve_pb2_grpc.add_UserDefinedServiceServicer_to_server",
"ray.serve.generated.serve_pb2_grpc.add_FruitServiceServicer_to_server",
]
yield serve.start(
grpc_options=gRPCOptions(
port=grpc_port,
grpc_servicer_functions=grpc_servicer_functions,
request_timeout_s=request_timeout_s,
),
http_options=HTTPOptions(
host="0.0.0.0",
request_timeout_s=request_timeout_s,
),
)
finally:
serve.shutdown()
ray.shutdown()
reset_ray_address()
# Helper function to return the node ID of a remote worker.
@ray.remote(num_cpus=0)
def _get_node_id():
return ray.get_runtime_context().get_node_id()
# Test fixture to start a Serve instance in a RayCluster with two labeled nodes
@pytest.fixture(scope="module")
def serve_instance_with_labeled_nodes():
cluster = Cluster()
# Unlabeled default node.
cluster.add_node(num_cpus=3, resources={"worker0": 1})
# Node 1 - labeled A100 node in us-west.
cluster.add_node(
num_cpus=3,
resources={"worker1": 1},
labels={"region": "us-west", "gpu-type": "A100"},
)
# Node 2 - labeled H100 node in us-east.
cluster.add_node(
num_cpus=3,
resources={"worker2": 1},
labels={"region": "us-east", "gpu-type": "H100"},
)
cluster.wait_for_nodes()
if ray.is_initialized():
ray.shutdown()
ray.init(address=cluster.address)
node_1_id = ray.get(_get_node_id.options(resources={"worker1": 1}).remote())
node_2_id = ray.get(_get_node_id.options(resources={"worker2": 1}).remote())
serve.start()
yield _get_global_client(), node_1_id, node_2_id, cluster
serve.shutdown()
ray.shutdown()
cluster.shutdown()
python/ray/serve/tests/fixtures/basic_proxy_spans.json
Vendored
+46
View File
@@ -0,0 +1,46 @@
[
{
"name": "route_to_replica BasicModel __call__",
"kind": "SpanKind.SERVER",
"status": {
"status_code": "UNSET"
},
"attributes": {
"request_id": "",
"deployment": "BasicModel",
"app": "default",
"call_method": "__call__",
"route": "/",
"multiplexed_model_id": "",
"is_streaming": true,
"is_http_request": true,
"is_grpc_request": false,
"resource.name": "route_to_replica BasicModel __call__",
"http.method": "__call__",
"http.route": "/"
},
"events": [],
"links": []
},
{
"name": "proxy_http_request BasicModel POST /",
"kind": "SpanKind.SERVER",
"status": {
"status_code": "OK"
},
"attributes": {
"request_id": "",
"deployment": "BasicModel",
"app": "default",
"request_type": "http",
"request_method": "POST",
"request_route_path": "/",
"resource.name": "proxy_http_request BasicModel POST /",
"http.method": "POST",
"http.status_code": 200,
"http.route": "/"
},
"events": [],
"links": []
}
]
python/ray/serve/tests/fixtures/basic_replica_spans.json
Vendored
+38
View File
@@ -0,0 +1,38 @@
[
{
"name": "application_span",
"kind": "SpanKind.INTERNAL",
"status": {
"status_code": "UNSET"
},
"attributes": {
"deployment": "BasicModel",
"replica_id": ""
},
"events": [],
"links": []
},
{
"name": "replica_handle_request BasicModel __call__",
"kind": "SpanKind.SERVER",
"status": {
"status_code": "UNSET"
},
"attributes": {
"request_id": "",
"replica_id": "",
"deployment": "BasicModel",
"app": "default",
"call_method": "__call__",
"route": "/",
"multiplexed_model_id": "",
"is_streaming": true,
"resource.name": "replica_handle_request BasicModel __call__",
"http.method": "POST",
"http.status_code": "200",
"http.route": "/"
},
"events": [],
"links": []
}
]
python/ray/serve/tests/fixtures/basic_upstream_spans.json
Vendored
+12
View File
@@ -0,0 +1,12 @@
[
{
"name": "upstream_app",
"kind": "SpanKind.INTERNAL",
"status": {
"status_code": "UNSET"
},
"attributes": {},
"events": [],
"links": []
}
]
python/ray/serve/tests/fixtures/grpc_proxy_spans.json
Vendored
+45
View File
@@ -0,0 +1,45 @@
[
{
"name": "route_to_replica grpc-deployment __call__",
"kind": "SpanKind.SERVER",
"status": {
"status_code": "UNSET"
},
"attributes": {
"request_id": "",
"deployment": "grpc-deployment",
"app": "default",
"call_method": "__call__",
"route": "default",
"multiplexed_model_id": "",
"is_streaming": false,
"is_http_request": false,
"is_grpc_request": true,
"resource.name": "route_to_replica grpc-deployment __call__",
"rpc.system": "gRPC",
"rpc.method": "__call__",
"rpc.service": "grpc-deployment"
},
"events": [],
"links": []
},
{
"name": "proxy_grpc_request grpc-deployment /ray.serve.UserDefinedService/__call__",
"kind": "SpanKind.SERVER",
"status": {
"status_code": "OK"
},
"attributes": {
"request_id": "",
"deployment": "grpc-deployment",
"app": "default",
"request_type": "grpc",
"resource.name": "proxy_grpc_request grpc-deployment /ray.serve.UserDefinedService/__call__",
"rpc.system": "grpc",
"rpc.method": "/ray.serve.UserDefinedService/__call__",
"rpc.grpc.status_code": "OK"
},
"events": [],
"links": []
}
]
python/ray/serve/tests/fixtures/grpc_replica_spans.json
Vendored
+38
View File
@@ -0,0 +1,38 @@
[
{
"name": "application_span",
"kind": "SpanKind.INTERNAL",
"status": {
"status_code": "UNSET"
},
"attributes": {
"deployment": "grpc-deployment",
"replica_id": ""
},
"events": [],
"links": []
},
{
"name": "replica_handle_request grpc-deployment __call__",
"kind": "SpanKind.SERVER",
"status": {
"status_code": "UNSET"
},
"attributes": {
"request_id": "",
"replica_id": "",
"deployment": "grpc-deployment",
"app": "default",
"call_method": "__call__",
"route": "default",
"multiplexed_model_id": "",
"is_streaming": false,
"resource.name": "replica_handle_request grpc-deployment __call__",
"rpc.system": "gRPC",
"rpc.method": "__call__",
"rpc.service": "grpc-deployment"
},
"events": [],
"links": []
}
]
python/ray/serve/tests/fixtures/grpc_upstream_spans.json
Vendored
+12
View File
@@ -0,0 +1,12 @@
[
{
"name": "upstream_app",
"kind": "SpanKind.INTERNAL",
"status": {
"status_code": "UNSET"
},
"attributes": {},
"events": [],
"links": []
}
]
python/ray/serve/tests/fixtures/streaming_proxy_spans.json
Vendored
+46
View File
@@ -0,0 +1,46 @@
[
{
"name": "route_to_replica StreamingModel __call__",
"kind": "SpanKind.SERVER",
"status": {
"status_code": "UNSET"
},
"attributes": {
"request_id": "",
"deployment": "StreamingModel",
"app": "default",
"call_method": "__call__",
"route": "/",
"multiplexed_model_id": "",
"is_streaming": true,
"is_http_request": true,
"is_grpc_request": false,
"resource.name": "route_to_replica StreamingModel __call__",
"http.method": "__call__",
"http.route": "/"
},
"events": [],
"links": []
},
{
"name": "proxy_http_request StreamingModel GET /",
"kind": "SpanKind.SERVER",
"status": {
"status_code": "OK"
},
"attributes": {
"request_id": "",
"deployment": "StreamingModel",
"app": "default",
"request_type": "http",
"request_method": "GET",
"request_route_path": "/",
"resource.name": "proxy_http_request StreamingModel GET /",
"http.method": "GET",
"http.status_code": 200,
"http.route": "/"
},
"events": [],
"links": []
}
]
python/ray/serve/tests/fixtures/streaming_replica_spans.json
Vendored
+38
View File
@@ -0,0 +1,38 @@
[
{
"name": "application_span",
"kind": "SpanKind.INTERNAL",
"status": {
"status_code": "UNSET"
},
"attributes": {
"deployment": "StreamingModel",
"replica_id": ""
},
"events": [],
"links": []
},
{
"name": "replica_handle_request StreamingModel __call__",
"kind": "SpanKind.SERVER",
"status": {
"status_code": "UNSET"
},
"attributes": {
"request_id": "",
"replica_id": "",
"deployment": "StreamingModel",
"app": "default",
"call_method": "__call__",
"route": "/",
"multiplexed_model_id": "",
"is_streaming": true,
"resource.name": "replica_handle_request StreamingModel __call__",
"http.method": "GET",
"http.status_code": "200",
"http.route": "/"
},
"events": [],
"links": []
}
]
python/ray/serve/tests/fixtures/streaming_upstream_spans.json
Vendored
+12
View File
@@ -0,0 +1,12 @@
[
{
"name": "upstream_app",
"kind": "SpanKind.INTERNAL",
"status": {
"status_code": "UNSET"
},
"attributes": {},
"events": [],
"links": []
}
]
python/ray/serve/tests/test_actor_replica_wrapper.py
+458
View File
@@ -0,0 +1,458 @@
import asyncio
import pickle
import sys
from types import SimpleNamespace
from typing import Union
import pytest
import ray
from ray import ObjectRef, ObjectRefGenerator
from ray._common.test_utils import SignalActor, async_wait_for_condition
from ray._common.utils import get_or_create_event_loop
from ray.exceptions import ActorDiedError, ActorUnavailableError, TaskCancelledError
from ray.serve._private.common import (
DeploymentID,
ReplicaID,
ReplicaQueueLengthInfo,
RequestMetadata,
RunningReplicaInfo,
)
from ray.serve._private.constants import SERVE_NAMESPACE
from ray.serve._private.request_router.common import PendingRequest
from ray.serve._private.request_router.replica_wrapper import RunningReplica
from ray.serve._private.test_utils import send_signal_on_cancellation
from ray.serve._private.utils import Semaphore
class _IntMetricsManager:
"""Minimal metrics manager that tracks only the in-flight count."""
def __init__(self):
self._n = 0
def get_num_ongoing_requests(self):
return self._n
def inc_num_ongoing_requests(self, _):
self._n += 1
def dec_num_ongoing_requests(self, _):
self._n -= 1
@ray.remote(num_cpus=0)
class SlotReservationActor:
"""Ray actor wrapping the real Replica.reserve_slot / release_slot.
Used by integration tests that need production slot-reservation logic
running under Ray's actor concurrency model — unit tests share one event
loop and can't observe sync/async ordering on a real ReplicaActor.
"""
def __init__(self, max_ongoing_requests: int):
from ray.serve._private.replica import Replica
replica = Replica.__new__(Replica)
replica._deployment_config = SimpleNamespace(
max_ongoing_requests=max_ongoing_requests
)
replica._reserved_slots = set()
replica._semaphore = Semaphore(lambda: max_ongoing_requests)
replica._metrics_manager = _IntMetricsManager()
# __init__ is bypassed; set the quiesce flag read by
# _can_accept_request (reservations are rejected once quiescing).
replica._quiescing = False
self._replica = replica
async def reserve_slot(self, request_metadata, slot_token: str):
return await self._replica.reserve_slot(request_metadata, slot_token)
def release_slot(self, slot_token: str):
return self._replica.release_slot(slot_token)
def get_num_ongoing_requests(self) -> int:
return self._replica.get_num_ongoing_requests()
@ray.remote(num_cpus=0)
class BlockingReserveActor:
"""Actor whose reserve_slot blocks on a SignalActor.
Records every release_slot token it receives so a test can verify the
cancellation cleanup path in RunningReplica.reserve_slot.
"""
def __init__(self, signal_actor):
self._signal = signal_actor
self._released_tokens = []
async def reserve_slot(self, request_metadata, slot_token: str):
await self._signal.wait.remote()
return True, 1
def release_slot(self, slot_token: str):
self._released_tokens.append(slot_token)
return True, 0
def get_released_tokens(self):
return list(self._released_tokens)
@ray.remote(num_cpus=0)
class FakeReplicaActor:
def __init__(self):
self._replica_queue_length_info = None
def set_replica_queue_length_info(self, info: ReplicaQueueLengthInfo):
self._replica_queue_length_info = info
async def handle_request(
self,
request_metadata: Union[bytes, RequestMetadata],
message: str,
*,
is_streaming: bool,
):
if isinstance(request_metadata, bytes):
request_metadata = pickle.loads(request_metadata)
assert not is_streaming and not request_metadata.is_streaming
return message
async def handle_request_streaming(
self,
request_metadata: Union[bytes, RequestMetadata],
message: str,
*,
is_streaming: bool,
):
if isinstance(request_metadata, bytes):
request_metadata = pickle.loads(request_metadata)
assert is_streaming and request_metadata.is_streaming
for i in range(5):
yield f"{message}-{i}"
async def handle_request_with_rejection(
self,
pickled_request_metadata: bytes,
*args,
**kwargs,
):
cancelled_signal_actor = kwargs.pop("cancelled_signal_actor", None)
if cancelled_signal_actor is not None:
executing_signal_actor = kwargs.pop("executing_signal_actor")
async with send_signal_on_cancellation(cancelled_signal_actor):
await executing_signal_actor.send.remote()
return
# Special case: if "raise_task_cancelled_error" is in kwargs, raise TaskCancelledError
# This simulates the scenario where the underlying Ray task gets cancelled
if kwargs.pop("raise_task_cancelled_error", False):
raise TaskCancelledError()
yield pickle.dumps(self._replica_queue_length_info)
if not self._replica_queue_length_info.accepted:
return
request_metadata = pickle.loads(pickled_request_metadata)
if request_metadata.is_streaming:
async for result in self.handle_request_streaming(
request_metadata, *args, **kwargs
):
yield result
else:
yield await self.handle_request(request_metadata, *args, **kwargs)
@pytest.fixture
def setup_fake_replica(ray_instance) -> RunningReplica:
replica_id = ReplicaID(
"fake_replica", deployment_id=DeploymentID(name="fake_deployment")
)
actor_name = replica_id.to_full_id_str()
# Create actor with a name so it can be retrieved by get_actor_handle()
_ = FakeReplicaActor.options(
name=actor_name, namespace=SERVE_NAMESPACE, lifetime="detached"
).remote()
return RunningReplicaInfo(
replica_id=replica_id,
node_id=None,
node_ip=None,
availability_zone=None,
actor_name=actor_name,
max_ongoing_requests=10,
is_cross_language=False,
)
def test_update_replica_info_refreshes_backend_http_endpoint(setup_fake_replica):
replica = RunningReplica(setup_fake_replica)
assert replica.backend_http_endpoint is None
updated_info = RunningReplicaInfo(
replica_id=setup_fake_replica.replica_id,
node_id=setup_fake_replica.node_id,
node_ip="127.0.0.1",
availability_zone=setup_fake_replica.availability_zone,
actor_name=setup_fake_replica.actor_name,
max_ongoing_requests=setup_fake_replica.max_ongoing_requests,
is_cross_language=setup_fake_replica.is_cross_language,
backend_http_port=8001,
)
replica.update_replica_info(updated_info)
assert replica.backend_http_endpoint == ("127.0.0.1", 8001)
def test_backend_http_endpoint_requires_host_and_port(setup_fake_replica):
replica = RunningReplica(setup_fake_replica)
updated_info = RunningReplicaInfo(
replica_id=setup_fake_replica.replica_id,
node_id=setup_fake_replica.node_id,
node_ip=None,
availability_zone=setup_fake_replica.availability_zone,
actor_name=setup_fake_replica.actor_name,
max_ongoing_requests=setup_fake_replica.max_ongoing_requests,
is_cross_language=setup_fake_replica.is_cross_language,
backend_http_port=8001,
)
replica.update_replica_info(updated_info)
assert replica.backend_http_endpoint is None
@pytest.mark.asyncio
@pytest.mark.parametrize("is_streaming", [False, True])
async def test_send_request_without_rejection(setup_fake_replica, is_streaming: bool):
replica = RunningReplica(setup_fake_replica)
pr = PendingRequest(
args=["Hello"],
kwargs={"is_streaming": is_streaming},
metadata=RequestMetadata(
request_id="abc",
internal_request_id="def",
is_streaming=is_streaming,
),
)
replica_result = replica.try_send_request(pr, with_rejection=False)
if is_streaming:
assert isinstance(replica_result.to_object_ref_gen(), ObjectRefGenerator)
for i in range(5):
assert await replica_result.__anext__() == f"Hello-{i}"
else:
assert isinstance(replica_result.to_object_ref(), ObjectRef)
assert isinstance(await replica_result.to_object_ref_async(), ObjectRef)
assert await replica_result.get_async() == "Hello"
@pytest.mark.asyncio
@pytest.mark.parametrize("accepted", [False, True])
@pytest.mark.parametrize("is_streaming", [False, True])
async def test_send_request_with_rejection(
setup_fake_replica, accepted: bool, is_streaming: bool
):
actor_handle = setup_fake_replica.get_actor_handle()
replica = RunningReplica(setup_fake_replica)
ray.get(
actor_handle.set_replica_queue_length_info.remote(
ReplicaQueueLengthInfo(accepted=accepted, num_ongoing_requests=10),
)
)
pr = PendingRequest(
args=["Hello"],
kwargs={"is_streaming": is_streaming},
metadata=RequestMetadata(
request_id="abc",
internal_request_id="def",
is_streaming=is_streaming,
),
)
replica_result = replica.try_send_request(pr, with_rejection=True)
info = await replica_result.get_rejection_response()
assert info.accepted == accepted
assert info.num_ongoing_requests == 10
if not accepted:
pass
elif is_streaming:
assert isinstance(replica_result.to_object_ref_gen(), ObjectRefGenerator)
for i in range(5):
assert await replica_result.__anext__() == f"Hello-{i}"
else:
assert isinstance(replica_result.to_object_ref(), ObjectRef)
assert isinstance(await replica_result.to_object_ref_async(), ObjectRef)
assert await replica_result.get_async() == "Hello"
@pytest.mark.asyncio
async def test_send_request_with_rejection_cancellation(setup_fake_replica):
"""
Verify that the downstream actor method call is cancelled if the call to send the
request to the replica is cancelled.
"""
replica = RunningReplica(setup_fake_replica)
executing_signal_actor = SignalActor.remote()
cancelled_signal_actor = SignalActor.remote()
pr = PendingRequest(
args=["Hello"],
kwargs={
"cancelled_signal_actor": cancelled_signal_actor,
"executing_signal_actor": executing_signal_actor,
},
metadata=RequestMetadata(
request_id="abc",
internal_request_id="def",
),
)
# Send request should hang because the downstream actor method call blocks
# before sending the system message.
replica_result = replica.try_send_request(pr, with_rejection=True)
request_task = get_or_create_event_loop().create_task(
replica_result.get_rejection_response()
)
# Check that the downstream actor method call has started.
await executing_signal_actor.wait.remote()
_, pending = await asyncio.wait([request_task], timeout=0.001)
assert len(pending) == 1
# Cancel the task. This should cause the downstream actor method call to
# be cancelled (verified via signal actor).
request_task.cancel()
with pytest.raises(asyncio.CancelledError):
await request_task
await cancelled_signal_actor.wait.remote()
@pytest.mark.asyncio
async def test_send_request_with_rejection_task_cancelled_error(setup_fake_replica):
"""
Test that TaskCancelledError from the underlying Ray task gets converted to
asyncio.CancelledError when sending request with rejection.
"""
actor_handle = setup_fake_replica.get_actor_handle()
replica = RunningReplica(setup_fake_replica)
# Set up the replica to accept the request
ray.get(
actor_handle.set_replica_queue_length_info.remote(
ReplicaQueueLengthInfo(accepted=True, num_ongoing_requests=5),
)
)
pr = PendingRequest(
args=["Hello"],
kwargs={
"raise_task_cancelled_error": True
}, # This will trigger TaskCancelledError
metadata=RequestMetadata(
request_id="abc",
internal_request_id="def",
),
)
# The TaskCancelledError should be caught and converted to asyncio.CancelledError
replica_result = replica.try_send_request(pr, with_rejection=True)
with pytest.raises(asyncio.CancelledError):
await replica_result.get_rejection_response()
def _spawn_running_replica(actor_cls, replica_id_str: str, *actor_args, **actor_kwargs):
"""Spawn a named actor and wrap it in a RunningReplica.
Returns ``(running_replica, actor_handle)``. The actor must be created
with the canonical replica-id name so RunningReplica can resolve it
through its normal GCS lookup.
"""
replica_id = ReplicaID(
replica_id_str, deployment_id=DeploymentID(name="slot_reservation_test")
)
actor_name = replica_id.to_full_id_str()
actor_handle = actor_cls.options(
name=actor_name, namespace=SERVE_NAMESPACE, lifetime="detached"
).remote(*actor_args, **actor_kwargs)
info = RunningReplicaInfo(
replica_id=replica_id,
node_id=None,
node_ip=None,
availability_zone=None,
actor_name=actor_name,
max_ongoing_requests=10,
is_cross_language=False,
)
return RunningReplica(info), actor_handle
def _dummy_request_metadata() -> RequestMetadata:
return RequestMetadata(request_id="abc", internal_request_id="def")
@pytest.mark.asyncio
async def test_reserve_slot_cancellation_releases_slot_on_actor(ray_instance):
"""If the awaiting reserve_slot task is cancelled, the wrapper must fire a
follow-up release_slot.remote(token) so the actor doesn't leak the slot.
"""
signal = SignalActor.remote()
replica, actor = _spawn_running_replica(
BlockingReserveActor, "blocking-replica", signal
)
task = get_or_create_event_loop().create_task(
replica.reserve_slot(_dummy_request_metadata())
)
# Let the actor enter reserve_slot and start awaiting the signal.
_, pending = await asyncio.wait([task], timeout=0.5)
assert len(pending) == 1
task.cancel()
with pytest.raises(asyncio.CancelledError):
await task
# Unblock the actor so it can process the follow-up release_slot.remote().
await signal.send.remote()
# The wrapper's cancellation cleanup fires release_slot.remote(token)
# without awaiting it; wait until the actor records the call.
async def _release_received():
return bool(await actor.get_released_tokens.remote())
await async_wait_for_condition(_release_received, timeout=5)
released_tokens = await actor.get_released_tokens.remote()
assert len(released_tokens) == 1
@pytest.mark.asyncio
async def test_reserve_slot_propagates_actor_died_error(ray_instance):
"""If the replica actor is dead, RunningReplica.reserve_slot must raise
ActorDiedError so AsyncioRouter.choose_replica can retry against another
replica. ActorUnavailableError is also acceptable on the brief window
before the actor failure has propagated.
"""
replica, actor = _spawn_running_replica(
SlotReservationActor, "doomed-replica", max_ongoing_requests=1
)
# Confirm liveness via a successful reservation first.
_, info = await replica.reserve_slot(_dummy_request_metadata())
assert info.accepted
ray.kill(actor)
with pytest.raises((ActorDiedError, ActorUnavailableError)):
await replica.reserve_slot(_dummy_request_metadata())
if __name__ == "__main__":
sys.exit(pytest.main(["-v", "-s", __file__]))
python/ray/serve/tests/test_advanced.py
+146
View File
@@ -0,0 +1,146 @@
import sys
import time
import httpx
import pytest
from starlette.requests import Request
import ray
from ray import serve
from ray._common.test_utils import SignalActor
from ray.serve._private.constants import SERVE_DEFAULT_APP_NAME
from ray.serve._private.test_utils import Barrier
from ray.serve.handle import DeploymentHandle
from ray.util.state import list_objects
def test_serve_forceful_shutdown(serve_instance):
@serve.deployment(graceful_shutdown_timeout_s=0.1)
def sleeper():
while True:
time.sleep(1000)
handle = serve.run(sleeper.bind())
response = handle.remote()
serve.delete(SERVE_DEFAULT_APP_NAME)
with pytest.raises(ray.exceptions.RayActorError):
response.result()
def test_serve_graceful_shutdown(serve_instance):
signal = SignalActor.remote()
@serve.deployment(
name="wait",
max_ongoing_requests=10,
graceful_shutdown_timeout_s=1000,
graceful_shutdown_wait_loop_s=0.5,
)
class Wait:
async def __call__(self, signal_actor):
await signal_actor.wait.remote()
handle = serve.run(Wait.bind())
responses = [handle.remote(signal) for _ in range(10)]
# Wait for all the queries to be enqueued
with pytest.raises(TimeoutError):
responses[0].result(timeout_s=1)
@ray.remote(num_cpus=0)
def do_blocking_delete():
serve.delete(SERVE_DEFAULT_APP_NAME)
# Now delete the deployment. This should trigger the shutdown sequence.
delete_ref = do_blocking_delete.remote()
# The queries should be enqueued but not executed becuase they are blocked
# by signal actor.
with pytest.raises(TimeoutError):
responses[0].result(timeout_s=1)
signal.send.remote()
# All the queries should be drained and executed without error.
[r.result() for r in responses]
# Blocking delete should complete.
ray.get(delete_ref)
def test_parallel_start(serve_instance):
# Test the ability to start multiple replicas in parallel.
# In the past, when Serve scale up a deployment, it does so one by one and
# wait for each replica to initialize. This test avoid this by preventing
# the first replica to finish initialization unless the second replica is
# also started.
barrier = Barrier.remote(n=2)
@serve.deployment(num_replicas=2)
class LongStartingServable:
def __init__(self):
ray.get(barrier.wait.remote(), timeout=10)
def __call__(self):
return "Ready"
handle = serve.run(LongStartingServable.bind())
handle.remote().result(timeout_s=10)
def test_passing_object_ref_to_deployment_not_pinned_to_memory(serve_instance):
"""Passing object refs to deployments should not pin the refs in memory.
We had issue that passing object ref to a deployment will result in memory leak
due to _PyObjScanner/ cloudpickler pinning the object to memory. This test will
ensure the object ref is released after the request is done.
See: https://github.com/ray-project/ray/issues/43248
"""
def _obj_ref_exists_in_state_api(obj_ref_hex: str) -> bool:
return (
len(
list_objects(
filters=[("object_id", "=", obj_ref_hex)],
raise_on_missing_output=False,
)
)
> 0
)
@serve.deployment
class Dep1:
def multiply_by_two(self, length: int):
return length * 2
@serve.deployment
class Gateway:
def __init__(self, dep1: DeploymentHandle):
self.dep1: DeploymentHandle = dep1
async def __call__(self, http_request: Request) -> str:
length = int(http_request.query_params.get("length"))
length_ref = ray.put(length)
# Sanity check that the ObjectRef exists in the state API.
assert _obj_ref_exists_in_state_api(length_ref.hex())
return {
"length": length,
"result": await self.dep1.multiply_by_two.remote(length_ref),
"length_ref_hex": length_ref.hex(),
}
serve.run(Gateway.bind(Dep1.bind()))
length = 10
response = httpx.get(f"http://localhost:8000?length={length}").json()
assert response["length"] == length
assert response["result"] == length * 2
# Ensure the object ref is not in the memory anymore.
assert not _obj_ref_exists_in_state_api(response["length_ref_hex"])
if __name__ == "__main__":
sys.exit(pytest.main(["-v", "-s", __file__]))
python/ray/serve/tests/test_api.py
+1579
View File
File diff suppressed because it is too large Load Diff
python/ray/serve/tests/test_api_2.py
+67
View File
@@ -0,0 +1,67 @@
import pytest
import ray
from ray import serve
from ray._common.network_utils import build_address
from ray.serve._private.common import RequestProtocol
from ray.serve._private.test_utils import get_application_urls
def test_get_application_urls(serve_instance):
@serve.deployment
def f():
return "Hello, world!"
serve.run(f.bind())
controller_details = ray.get(serve_instance._controller.get_actor_details.remote())
node_ip = controller_details.node_ip
assert get_application_urls(use_localhost=False) == [
f"http://{build_address(node_ip, 8000)}"
]
assert get_application_urls("gRPC", use_localhost=False) == [
build_address(node_ip, 9000)
]
assert get_application_urls(RequestProtocol.HTTP, use_localhost=False) == [
f"http://{build_address(node_ip, 8000)}"
]
assert get_application_urls(RequestProtocol.GRPC, use_localhost=False) == [
build_address(node_ip, 9000)
]
def test_get_application_urls_with_app_name(serve_instance):
@serve.deployment
def f():
return "Hello, world!"
serve.run(f.bind(), name="app1", route_prefix="/")
controller_details = ray.get(serve_instance._controller.get_actor_details.remote())
node_ip = controller_details.node_ip
assert get_application_urls("HTTP", app_name="app1", use_localhost=False) == [
f"http://{node_ip}:8000"
]
assert get_application_urls("gRPC", app_name="app1", use_localhost=False) == [
f"{node_ip}:9000"
]
def test_get_application_urls_with_route_prefix(serve_instance):
@serve.deployment
def f():
return "Hello, world!"
serve.run(f.bind(), name="app1", route_prefix="/app1")
controller_details = ray.get(serve_instance._controller.get_actor_details.remote())
node_ip = controller_details.node_ip
assert get_application_urls("HTTP", app_name="app1", use_localhost=False) == [
f"http://{node_ip}:8000/app1"
]
assert get_application_urls("gRPC", app_name="app1", use_localhost=False) == [
f"{node_ip}:9000"
]
if __name__ == "__main__":
import sys
sys.exit(pytest.main(["-v", "-s", __file__]))
python/ray/serve/tests/test_autoscaling_policy.py
+2560
View File
File diff suppressed because it is too large Load Diff
python/ray/serve/tests/test_backpressure.py
+238
View File
@@ -0,0 +1,238 @@
import sys
from concurrent.futures import FIRST_COMPLETED, ThreadPoolExecutor, wait
from urllib.parse import urljoin
import httpx
import pytest
from fastapi import FastAPI
from fastapi.responses import PlainTextResponse
from starlette.requests import Request
import ray
from ray import serve
from ray._common.test_utils import SignalActor, wait_for_condition
from ray.serve._private.test_utils import get_application_url
from ray.serve.exceptions import BackPressureError
def test_handle_backpressure(serve_instance):
"""Requests should raise a BackPressureError once the limit is reached."""
signal_actor = SignalActor.remote()
@serve.deployment(max_ongoing_requests=1, max_queued_requests=1)
class Deployment:
async def __call__(self, msg: str) -> str:
await signal_actor.wait.remote()
return msg
handle = serve.run(Deployment.bind())
# First response should block. Until the signal is sent, all subsequent requests
# will be queued in the handle.
first_response = handle.remote("hi-1")
wait_for_condition(lambda: ray.get(signal_actor.cur_num_waiters.remote()) == 1)
# Check that beyond the 1st queued request, others are dropped due to backpressure.
second_response = handle.remote("hi-2")
# Wait until "hi-2" is actually registered as queued in the router before
# sending more requests. The router processes the request asynchronously on
# its own event loop thread, so without this the requests below can race
# ahead and get queued themselves (blocking forever) instead of being
# rejected with backpressure.
wait_for_condition(
lambda: handle._router._asyncio_router._metrics_manager.num_queued_requests == 1
)
for _ in range(10):
with pytest.raises(BackPressureError):
handle.remote().result()
# Send the signal; the first request will be unblocked and the second should
# subsequently get scheduled and executed.
ray.get(signal_actor.send.remote())
assert first_response.result() == "hi-1"
assert second_response.result() == "hi-2"
ray.get(signal_actor.send.remote(clear=True))
wait_for_condition(lambda: ray.get(signal_actor.cur_num_waiters.remote()) == 0)
def test_http_backpressure(serve_instance):
"""Requests should return a 503 once the limit is reached."""
signal_actor = SignalActor.remote()
@serve.deployment(max_ongoing_requests=1, max_queued_requests=1)
class Deployment:
async def __call__(self, request: Request) -> str:
msg = (await request.json())["msg"]
await signal_actor.wait.remote()
return msg
serve.run(Deployment.bind())
def send_request(msg: str = "hi"):
application_url = get_application_url()
r = httpx.request("GET", application_url, json={"msg": msg}, timeout=30.0)
return r.status_code, r.text
with ThreadPoolExecutor(max_workers=5) as exc:
# First response should block. Until the signal is sent, all subsequent
# requests will be queued in the handle.
first_fut = exc.submit(send_request, "hi-1")
wait_for_condition(lambda: ray.get(signal_actor.cur_num_waiters.remote()) == 1)
done, _ = wait([first_fut], timeout=0.1, return_when=FIRST_COMPLETED)
assert len(done) == 0
# Second request should get queued.
second_fut = exc.submit(send_request, "hi-2")
done, _ = wait(
[first_fut, second_fut], timeout=0.1, return_when=FIRST_COMPLETED
)
assert len(done) == 0
# Check that beyond the 1st queued request, others are dropped due to
# backpressure.
for _ in range(10):
rejected_fut = exc.submit(send_request, "hi-err")
status_code, text = rejected_fut.result()
assert status_code == 503
assert text.startswith("Request dropped due to backpressure")
# Send the signal; the first request will be unblocked and the second
# should subsequently get scheduled and executed.
ray.get(signal_actor.send.remote())
assert first_fut.result() == (200, "hi-1")
assert second_fut.result() == (200, "hi-2")
ray.get(signal_actor.send.remote(clear=True))
wait_for_condition(lambda: ray.get(signal_actor.cur_num_waiters.remote()) == 0)
def test_model_composition_backpressure(serve_instance):
signal_actor = SignalActor.remote()
@serve.deployment(max_ongoing_requests=1, max_queued_requests=1)
class Child:
async def __call__(self):
await signal_actor.wait.remote()
return "ok"
@serve.deployment
class Parent:
def __init__(self, child):
self.child = child
async def __call__(self):
return await self.child.remote()
def send_request():
return httpx.get(get_application_url())
serve.run(Parent.bind(child=Child.bind()))
with ThreadPoolExecutor(max_workers=3) as exc:
# Send first request, wait for it to be blocked while executing.
executing_fut = exc.submit(send_request)
wait_for_condition(lambda: ray.get(signal_actor.cur_num_waiters.remote()) == 1)
done, _ = wait([executing_fut], timeout=0.1, return_when=FIRST_COMPLETED)
assert len(done) == 0
# Send second request, it should get queued.
queued_fut = exc.submit(send_request)
done, _ = wait(
[executing_fut, queued_fut], timeout=0.1, return_when=FIRST_COMPLETED
)
assert len(done) == 0
# Send third request, it should get rejected.
rejected_fut = exc.submit(send_request)
assert rejected_fut.result().status_code == 503
# Send signal, check the two requests succeed.
ray.get(signal_actor.send.remote(clear=False))
assert executing_fut.result().status_code == 200
assert executing_fut.result().text == "ok"
assert queued_fut.result().status_code == 200
assert queued_fut.result().text == "ok"
ray.get(signal_actor.send.remote(clear=True))
wait_for_condition(lambda: ray.get(signal_actor.cur_num_waiters.remote()) == 0)
@pytest.mark.parametrize("request_type", ["async_non_gen", "sync_non_gen"])
def test_model_composition_backpressure_with_fastapi(serve_instance, request_type):
"""Tests backpressure behavior with FastAPI model composition.
Tests that when a Child deployment with max_ongoing_requests=1 and max_queued_requests=1
is called through a Parent FastAPI deployment:
1. First request blocks while executing
2. Second request gets queued
3. Third request gets rejected with 503 status code
4. After unblocking, first two requests complete successfully
Tests both async and sync non-generator endpoints.
"""
signal_actor = SignalActor.remote()
app = FastAPI()
@serve.deployment(max_ongoing_requests=1, max_queued_requests=1)
class Child:
async def __call__(self):
await signal_actor.wait.remote()
return "ok"
@serve.deployment
@serve.ingress(app)
class Parent:
def __init__(self, child):
self.child = child
@app.get("/async_non_gen")
async def async_non_gen(self):
result = await self.child.remote()
return PlainTextResponse(result)
@app.get("/sync_non_gen")
def sync_non_gen(self):
result = self.child.remote().result()
return PlainTextResponse(result)
def send_request():
url_map = {
"async_non_gen": urljoin(get_application_url(), "async_non_gen"),
"sync_non_gen": urljoin(get_application_url(), "sync_non_gen"),
}
resp = httpx.get(url_map[request_type])
return resp
serve.run(Parent.bind(child=Child.bind()))
with ThreadPoolExecutor(max_workers=3) as exc:
executing_fut = exc.submit(send_request)
wait_for_condition(lambda: ray.get(signal_actor.cur_num_waiters.remote()) == 1)
done, _ = wait([executing_fut], timeout=0.1, return_when=FIRST_COMPLETED)
assert len(done) == 0
queued_fut = exc.submit(send_request)
done, _ = wait(
[executing_fut, queued_fut], timeout=0.1, return_when=FIRST_COMPLETED
)
assert len(done) == 0
rejected_fut = exc.submit(send_request)
assert rejected_fut.result().status_code == 503
# Send signal, let the two requests succeed.
ray.get(signal_actor.send.remote())
assert executing_fut.result().status_code == 200
assert executing_fut.result().text == "ok"
assert queued_fut.result().status_code == 200
assert queued_fut.result().text == "ok"
wait_for_condition(lambda: ray.get(signal_actor.cur_num_waiters.remote()) == 0)
if __name__ == "__main__":
sys.exit(pytest.main(["-v", "-s", __file__]))
python/ray/serve/tests/test_backpressure_grpc.py
+79
View File
@@ -0,0 +1,79 @@
import sys
from typing import Tuple
import grpc
import pytest
import ray
from ray import serve
from ray._common.test_utils import SignalActor, wait_for_condition
from ray.serve._private.common import RequestProtocol
from ray.serve._private.test_utils import get_application_url
from ray.serve.generated import serve_pb2, serve_pb2_grpc
def test_grpc_backpressure(serve_instance):
"""Requests should return UNAVAILABLE once the limit is reached."""
signal_actor = SignalActor.remote()
@serve.deployment(max_ongoing_requests=1, max_queued_requests=1)
class Deployment:
async def __call__(self, request: serve_pb2.UserDefinedMessage):
await signal_actor.wait.remote()
return serve_pb2.UserDefinedResponse(greeting=request.name)
serve.run(Deployment.bind())
@ray.remote(num_cpus=0)
def do_request(msg: str) -> Tuple[grpc.StatusCode, str]:
channel = grpc.insecure_channel(
get_application_url(protocol=RequestProtocol.GRPC)
)
stub = serve_pb2_grpc.UserDefinedServiceStub(channel)
try:
response, call = stub.__call__.with_call(
serve_pb2.UserDefinedMessage(name=msg)
)
return call.code(), response.greeting
except grpc.RpcError as e:
return e.code(), e.details()
# First response should block. Until the signal is sent, all subsequent requests
# will be queued in the handle.
first_ref = do_request.remote("hi-1")
wait_for_condition(lambda: ray.get(signal_actor.cur_num_waiters.remote()) == 1)
_, pending = ray.wait([first_ref], timeout=0.1)
assert len(pending) == 1
# Check that beyond the 1st queued request, others are dropped due to backpressure.
num_requests = 10
burst_refs = [do_request.remote("hi-err") for _ in range(num_requests)]
def num_rejected() -> int:
ready, _ = ray.wait(burst_refs, num_returns=len(burst_refs), timeout=0)
rejected = 0
for status_code, text in ray.get(ready):
if status_code == grpc.StatusCode.RESOURCE_EXHAUSTED:
assert text.startswith("Request dropped due to backpressure")
rejected += 1
return rejected
# All but the single queued request should be rejected with backpressure.
wait_for_condition(lambda: num_rejected() == num_requests - 1)
# Send the signal; the ongoing request and the single queued request both
# get unblocked and complete successfully.
ray.get(signal_actor.send.remote())
assert ray.get(first_ref) == (grpc.StatusCode.OK, "hi-1")
num_ok = sum(
1 for status_code, _ in ray.get(burst_refs) if status_code == grpc.StatusCode.OK
)
assert num_ok == 1
ray.get(signal_actor.send.remote(clear=True))
wait_for_condition(lambda: ray.get(signal_actor.cur_num_waiters.remote()) == 0)
if __name__ == "__main__":
sys.exit(pytest.main(["-v", "-s", __file__]))
python/ray/serve/tests/test_batching.py
+751
View File
@@ -0,0 +1,751 @@
import asyncio
import math
from collections.abc import Callable
from concurrent.futures.thread import ThreadPoolExecutor
from functools import partial
from threading import Thread
from typing import List, Optional
import httpx
import pytest
from fastapi import FastAPI, Request
from starlette.responses import StreamingResponse
from ray import serve
from ray._common.test_utils import SignalActor, async_wait_for_condition
from ray.serve._private.test_utils import get_application_url
from ray.serve.batching import _RuntimeSummaryStatistics
from ray.serve.context import (
_get_serve_batch_request_context,
_get_serve_request_context,
)
def test_batching(serve_instance):
@serve.deployment
class BatchingExample:
def __init__(self):
self.count = 0
@serve.batch(max_batch_size=5, batch_wait_timeout_s=1)
async def handle_batch(self, requests):
self.count += 1
batch_size = len(requests)
return [self.count] * batch_size
async def __call__(self, request):
return await self.handle_batch(request)
handle = serve.run(BatchingExample.bind())
result_list = [handle.remote(1) for _ in range(20)]
# since count is only updated per batch of queries
# If there atleast one __call__ fn call with batch size greater than 1
# counter result will always be less than 20
assert max([r.result() for r in result_list]) < 20
def test_concurrent_batching(serve_instance):
BATCHES_IN_FLIGHT = 2
MAX_BATCH_SIZE = 5
BATCH_WAIT_TIMEOUT_S = 1
MAX_REQUESTS_IN_FLIGHT = BATCHES_IN_FLIGHT * MAX_BATCH_SIZE
@serve.deployment(max_ongoing_requests=MAX_REQUESTS_IN_FLIGHT * 2)
class BatchingExample:
def __init__(self):
self.n_batches_in_flight = 0
self.n_requests_in_flight = 0
@serve.batch(
max_batch_size=MAX_BATCH_SIZE,
batch_wait_timeout_s=BATCH_WAIT_TIMEOUT_S,
max_concurrent_batches=BATCHES_IN_FLIGHT,
)
async def handle_batch(self, requests):
self.n_batches_in_flight += 1
self.n_requests_in_flight += len(requests)
await asyncio.sleep(0.5)
out = [
(req_idx, self.n_batches_in_flight, self.n_requests_in_flight)
for req_idx in requests
]
await asyncio.sleep(0.5)
self.n_requests_in_flight -= len(requests)
self.n_batches_in_flight -= 1
return out
async def __call__(self, request):
return await self.handle_batch(request)
handle = serve.run(BatchingExample.bind())
idxs = set(range(20))
result_futures = [handle.remote(i) for i in idxs]
result_list = [future.result() for future in result_futures]
out_idxs = set()
for idx, batches_in_flight, requests_in_flight in result_list:
out_idxs.add(idx)
assert (
batches_in_flight == BATCHES_IN_FLIGHT
), f"Should have been {BATCHES_IN_FLIGHT} batches in flight at all times, got {batches_in_flight}"
assert (
requests_in_flight == MAX_REQUESTS_IN_FLIGHT
), f"Should have been {MAX_REQUESTS_IN_FLIGHT} requests in flight at all times, got {requests_in_flight}"
assert idxs == out_idxs, "All requests should be processed"
def test_batching_exception(serve_instance):
@serve.deployment
class NoListReturned:
def __init__(self):
self.count = 0
@serve.batch(max_batch_size=5)
async def handle_batch(self, requests):
return len(requests)
async def __call__(self, request):
return await self.handle_batch(request)
# Set the max batch size.
handle = serve.run(NoListReturned.bind())
with pytest.raises(TypeError):
assert handle.remote(1).result()
@pytest.mark.asyncio
async def test_batch_generator_streaming_response_integration_test(serve_instance):
NUM_YIELDS = 10
@serve.deployment
class Textgen:
@serve.batch(max_batch_size=4, batch_wait_timeout_s=1000)
async def batch_handler(self, prompts: List[str]):
for _ in range(NUM_YIELDS):
# Check that the batch handler can yield unhashable types
prompt_responses = [{"value": prompt} for prompt in prompts]
yield prompt_responses
async def value_extractor(self, prompt_responses):
async for prompt_response in prompt_responses:
yield prompt_response["value"]
async def __call__(self, request):
prompt = request.query_params["prompt"]
response_values = self.value_extractor(self.batch_handler(prompt))
return StreamingResponse(response_values)
serve.run(Textgen.bind())
prompt_prefix = "hola"
url = f"{get_application_url()}/?prompt={prompt_prefix}"
with ThreadPoolExecutor() as pool:
futs = [pool.submit(partial(httpx.get, url + str(idx))) for idx in range(4)]
responses = [fut.result() for fut in futs]
for idx, response in enumerate(responses):
assert response.status_code == 200
assert response.text == "".join([prompt_prefix + str(idx)] * NUM_YIELDS)
def test_batching_client_dropped_unary(serve_instance):
"""Test unary batching with clients that drops the connection.
After requests are dropped. The next request should succeed.
"""
@serve.deployment
class ModelUnary:
@serve.batch(max_batch_size=5)
async def handle_batch(self, requests):
await asyncio.sleep(0.05)
return ["fake-response" for _ in range(len(requests))]
async def __call__(self, request):
return await self.handle_batch(request)
serve.run(ModelUnary.bind())
url = f"{get_application_url()}/"
# Sending requests with clients that drops the connection.
for _ in range(3):
with pytest.raises(httpx.ReadTimeout):
httpx.get(url, timeout=0.005)
# The following request should succeed.
resp = httpx.get(url, timeout=1)
assert resp.status_code == 200
assert resp.text == "fake-response"
def test_batching_client_dropped_streaming(serve_instance):
"""Test streaming batching with clients that drops the connection.
After requests are dropped. The next request should succeed.
"""
@serve.deployment
class ModelStreaming:
@serve.batch(max_batch_size=3)
async def handle_batch(self, requests):
await asyncio.sleep(0.05)
for i in range(10):
yield [str(i) for _ in range(len(requests))]
async def __call__(self, request):
return StreamingResponse(self.handle_batch(request))
serve.run(ModelStreaming.bind())
url = "http://localhost:8000/"
# Sending requests with clients that drops the connection.
for _ in range(3):
with pytest.raises((httpx.ReadTimeout, httpx.ConnectError)):
httpx.get(url, timeout=0.005)
# The following request should succeed.
resp = httpx.get(url, timeout=1)
assert resp.status_code == 200
assert resp.text == "0123456789"
@pytest.mark.asyncio
@pytest.mark.parametrize("max_concurrent_batches", [1, 10])
@pytest.mark.parametrize("max_batch_size", [1, 10])
@pytest.mark.parametrize("n_requests", [1, 10])
async def test_observability_helpers(
serve_instance, n_requests: int, max_batch_size: int, max_concurrent_batches: int
) -> None:
"""Checks observability helper methods that are used for batching.
Tests three observability helper methods:
* _get_curr_iteration_start_times: gets the current iteration's start
time.
* _is_batching_task_alive: returns whether the batch-handler task is
alive.
* _get_handling_task_stack: returns the stack for the batch-handler task.
"""
signal_actor = SignalActor.remote()
@serve.deployment(
name="batcher", max_ongoing_requests=max_concurrent_batches * max_batch_size
)
class Batcher:
@serve.batch(
max_batch_size=max_batch_size,
max_concurrent_batches=max_concurrent_batches,
batch_wait_timeout_s=0.1,
)
async def handle_batch(self, requests):
await signal_actor.wait.remote() # wait until the outer signal actor is released
return [0] * len(requests)
async def __call__(self, request):
return await self.handle_batch(request)
async def _get_curr_iteration_start_times(self) -> _RuntimeSummaryStatistics:
return self.handle_batch._get_curr_iteration_start_times()
async def _is_batching_task_alive(self) -> bool:
return await self.handle_batch._is_batching_task_alive()
async def _get_handling_task_stack(self) -> Optional[str]:
return await self.handle_batch._get_handling_task_stack()
serve.run(target=Batcher.bind(), name="app_name")
handle = serve.get_deployment_handle(deployment_name="batcher", app_name="app_name")
assert await handle._is_batching_task_alive.remote()
min_num_batches = min(
math.ceil(n_requests / max_batch_size), max_concurrent_batches
)
async with httpx.AsyncClient() as client:
tasks1 = await send_k_requests(
signal_actor,
n_requests,
min_num_batches,
app_name="app_name",
client=client,
)
prev_iter_times = await handle._get_curr_iteration_start_times.remote()
await signal_actor.send.remote() # unblock the batch handler now that we have the iter times
assert len(prev_iter_times.start_times) >= min_num_batches
assert len(await handle._get_handling_task_stack.remote()) is not None
assert await handle._is_batching_task_alive.remote()
tasks2 = await send_k_requests(
signal_actor,
n_requests,
min_num_batches,
app_name="app_name",
client=client,
)
new_iter_times = await handle._get_curr_iteration_start_times.remote()
await signal_actor.send.remote() # unblock the batch handler now that we have the iter times
assert len(new_iter_times.start_times) >= min_num_batches
assert len(await handle._get_handling_task_stack.remote()) is not None
assert await handle._is_batching_task_alive.remote()
assert new_iter_times.min_start_time > prev_iter_times.max_start_time
# Cancel and await all tasks to avoid "Task exception was never retrieved" warning.
# We don't need the HTTP responses, just need to clean up the tasks properly.
for task in tasks1 + tasks2:
task.cancel()
await asyncio.gather(*tasks1, *tasks2, return_exceptions=True)
async def send_k_requests(
signal_actor: SignalActor,
k: int,
min_num_batches: float,
app_name: str,
client: httpx.AsyncClient,
) -> List[asyncio.Task]:
"""Send k requests and wait until at least min_num_batches are waiting.
Returns the list of request tasks so they can be awaited by the caller
after unblocking the batch handler.
"""
await signal_actor.send.remote(True) # type: ignore[attr-defined]
tasks = []
for _ in range(k):
tasks.append(
asyncio.create_task(
client.get(f"{get_application_url(app_name=app_name)}/")
)
)
await wait_for_n_waiters(
signal_actor, lambda num_waiters: num_waiters >= min_num_batches
)
return tasks
async def wait_for_n_waiters(
signal_actor: SignalActor, condition: Callable[[int], bool]
) -> None:
async def poll() -> bool:
num_waiters: int = await signal_actor.cur_num_waiters.remote() # type: ignore[attr-defined]
return condition(num_waiters)
return await async_wait_for_condition(poll)
def test_batching_request_context(serve_instance):
"""Test that _get_serve_batch_request_context() works correctly with batching.
With 6 requests and max_batch_size=3, Serve should create 2 batches processed in parallel.
Each batch should have access to the request contexts of all requests in that batch,
and context should be properly unset after processing.
"""
@serve.deployment(max_ongoing_requests=10)
class BatchContextTester:
def __init__(self):
self.batch_results = []
@serve.batch(
max_batch_size=3, batch_wait_timeout_s=1.0, max_concurrent_batches=2
)
async def handle_batch(self, batch):
# Store results for verification
batch_result = {
"batch_size": len(batch),
"batch_request_contexts": _get_serve_batch_request_context(),
"current_request_context": _get_serve_request_context(),
}
self.batch_results.append(batch_result)
return ["ok" for _ in range(len(batch))]
async def __call__(self, request):
return await self.handle_batch(1)
async def get_results(self):
return self.batch_results
handle = serve.run(BatchContextTester.bind())
def do_request():
"""Make a request with a specific request ID."""
url = get_application_url()
r = httpx.post(f"{url}/")
r.raise_for_status()
# Launch 6 requests. Expect 2 batches of 3 requests each.
threads = [Thread(target=do_request) for _ in range(6)]
for t in threads:
t.start()
for t in threads:
t.join()
# Get results from the deployment
batch_results = handle.get_results.remote().result()
# Verify each batch has correct size and context
total_requests_processed = 0
request_ids_in_batch_context = set()
for result in batch_results:
# Batch context should contain all 3 request contexts
assert (
len(result["batch_request_contexts"]) == 3
), f"Expected 3 contexts in batch, got {result['batch_request_contexts']}"
req_ids_in_batch_context = [
ctx.request_id for ctx in result["batch_request_contexts"]
]
assert (
len(req_ids_in_batch_context) == 3
), f"Expected 3 batch request IDs, got {len(req_ids_in_batch_context)}"
request_ids_in_batch_context.update(req_ids_in_batch_context)
# Current request context read within the batcher should be a default empty context.
current_request_context = result["current_request_context"]
assert current_request_context.request_id == ""
assert current_request_context.route == ""
assert current_request_context.app_name == ""
assert current_request_context.multiplexed_model_id == ""
total_requests_processed += result["batch_size"]
# Verify all 6 requests were processed
assert (
total_requests_processed == 6
), f"Expected 6 total requests processed, got {total_requests_processed}"
assert (
len(request_ids_in_batch_context) == 6
), f"Expected 6 unique request IDs, got {len(request_ids_in_batch_context)}"
def test_batch_size_fn_simple(serve_instance):
"""Test batch_size_fn with a simple custom batch size metric."""
@serve.deployment
class BatchSizeFnExample:
def __init__(self):
self.batches_received = []
@serve.batch(
max_batch_size=100, # Set based on total size, not count
batch_wait_timeout_s=0.5,
batch_size_fn=lambda items: sum(item["size"] for item in items),
)
async def handle_batch(self, requests: List):
# Record the batch for verification
self.batches_received.append(requests)
# Return results
return [req["value"] * 2 for req in requests]
async def __call__(self, request):
return await self.handle_batch(request)
def get_batches(self):
return self.batches_received
handle = serve.run(BatchSizeFnExample.bind())
# Send requests with different sizes
# Request 1: size=30, value=1
# Request 2: size=40, value=2
# Request 3: size=20, value=3
# Request 4: size=25, value=4
# Total of first 3 = 90 (< 100), but adding 4th would be 115 (> 100)
requests = [
{"size": 30, "value": 1},
{"size": 40, "value": 2},
{"size": 20, "value": 3},
{"size": 25, "value": 4},
]
result_futures = [handle.remote(req) for req in requests]
results = [future.result() for future in result_futures]
# Verify results are correct
assert results == [2, 4, 6, 8]
# Verify batching behavior
batches = handle.get_batches.remote().result()
# Should have created at least one batch
assert len(batches) > 0
def test_batch_size_fn_graph_nodes(serve_instance):
"""Test batch_size_fn with a GNN-style use case (batching by total nodes)."""
class Graph:
def __init__(self, num_nodes: int, graph_id: int):
self.num_nodes = num_nodes
self.graph_id = graph_id
@serve.deployment
class GraphBatcher:
def __init__(self):
self.batch_sizes = []
@serve.batch(
max_batch_size=100, # Max 100 nodes per batch
batch_wait_timeout_s=0.5,
batch_size_fn=lambda graphs: sum(g.num_nodes for g in graphs),
)
async def process_graphs(self, graphs: List[Graph]):
# Record batch size (total nodes)
total_nodes = sum(g.num_nodes for g in graphs)
self.batch_sizes.append(total_nodes)
# Return graph_id * num_nodes as result
return [g.graph_id * g.num_nodes for g in graphs]
async def __call__(self, graph):
return await self.process_graphs(graph)
def get_batch_sizes(self):
return self.batch_sizes
handle = serve.run(GraphBatcher.bind())
# Create graphs with different node counts
# Graph 1: 30 nodes, Graph 2: 40 nodes, Graph 3: 35 nodes, Graph 4: 50 nodes
# First 3 total = 105 nodes (> 100), so should be 2 batches
graphs = [
Graph(num_nodes=30, graph_id=1),
Graph(num_nodes=40, graph_id=2),
Graph(num_nodes=35, graph_id=3),
Graph(num_nodes=50, graph_id=4),
]
result_futures = [handle.remote(g) for g in graphs]
results = [future.result() for future in result_futures]
# Verify results
assert results == [30, 80, 105, 200]
# Verify batch sizes respect the limit
batch_sizes = handle.get_batch_sizes.remote().result()
for batch_size in batch_sizes:
# Each batch should have <= 100 nodes
assert batch_size <= 100, f"Batch size {batch_size} exceeds limit of 100"
def test_batch_size_fn_token_count(serve_instance):
"""Test batch_size_fn with an NLP-style use case (batching by total tokens)."""
@serve.deployment
class TokenBatcher:
@serve.batch(
max_batch_size=1000, # Max 1000 tokens per batch
batch_wait_timeout_s=0.5,
batch_size_fn=lambda sequences: sum(len(s.split()) for s in sequences),
)
async def process_sequences(self, sequences: List[str]):
# Return word count for each sequence
return [len(s.split()) for s in sequences]
async def __call__(self, sequence):
return await self.process_sequences(sequence)
handle = serve.run(TokenBatcher.bind())
# Create sequences with different token counts
sequences = [
"This is a short sequence", # 5 tokens
"This is a much longer sequence with many more words in it", # 12 tokens
"Short", # 1 token
"A B C D E F G H I J", # 10 tokens
]
result_futures = [handle.remote(s) for s in sequences]
results = [future.result() for future in result_futures]
# Verify results are correct
assert results == [5, 12, 1, 10]
def test_batch_size_fn_validation():
"""Test that batch_size_fn validation works correctly."""
from ray.serve.batching import batch
# Test with non-callable batch_size_fn
with pytest.raises(TypeError, match="batch_size_fn must be a callable or None"):
@batch(batch_size_fn="not_a_function")
async def my_batch_handler(items):
return items
def test_batch_size_fn_default_behavior(serve_instance):
"""Test that default behavior (batch_size_fn=None) still works as expected."""
@serve.deployment
class DefaultBatcher:
@serve.batch(max_batch_size=5, batch_wait_timeout_s=0.5)
async def handle_batch(self, requests):
return [r * 2 for r in requests]
async def __call__(self, request):
return await self.handle_batch(request)
handle = serve.run(DefaultBatcher.bind())
# Send 10 requests
result_futures = [handle.remote(i) for i in range(10)]
results = [future.result() for future in result_futures]
# Verify all results are correct
assert results == [i * 2 for i in range(10)]
def test_batch_size_fn_oversized_item_raises_error(serve_instance):
app = FastAPI()
@serve.deployment
@serve.ingress(app)
class OversizedItemBatcher:
@serve.batch(
max_batch_size=10,
batch_wait_timeout_s=0.5,
batch_size_fn=lambda items: sum(item["size"] for item in items),
)
async def handle_batch(self, requests: List):
return [req["value"] for req in requests]
@app.post("/")
async def f(self, request: Request):
body = await request.json()
return await self.handle_batch(body)
serve.run(OversizedItemBatcher.bind())
# Send a request with size > max_batch_size (15 > 10)
# This should return a 500 error with RuntimeError message
url = f"{get_application_url(use_localhost=True)}/"
response = httpx.post(url, json={"size": 15, "value": "too_large"}, timeout=5)
assert response.status_code == 500
def test_batch_size_fn_deferred_item_processed(serve_instance):
@serve.deployment(max_ongoing_requests=15)
class DeferredItemBatcher:
def __init__(self):
self.batch_sizes = []
@serve.batch(
max_batch_size=10,
batch_wait_timeout_s=0.5,
batch_size_fn=lambda items: sum(item["size"] for item in items),
)
async def handle_batch(self, requests: List):
# Record actual batch sizes for verification
total_size = sum(req["size"] for req in requests)
self.batch_sizes.append(total_size)
return [req["value"] for req in requests]
async def __call__(self, request):
return await self.handle_batch(request)
def get_batch_sizes(self):
return self.batch_sizes
handle = serve.run(DeferredItemBatcher.bind())
# Send requests where some will need to be deferred:
# Request 1: size=6 (fits)
# Request 2: size=6 (would make total 12 > 10, deferred)
# Request 3: size=3 (fits with request 1, total 9)
# Request 4: size=4 (would make total 13 > 10, deferred)
requests = [
{"size": 6, "value": "a"},
{"size": 6, "value": "b"},
{"size": 3, "value": "c"},
{"size": 4, "value": "d"},
]
result_futures = [handle.remote(req) for req in requests]
results = [future.result() for future in result_futures]
# All requests should be processed successfully
assert set(results) == {"a", "b", "c", "d"}
# Verify total size processed equals sum of all request sizes
batch_sizes = handle.get_batch_sizes.remote().result()
total_processed = sum(batch_sizes)
expected_total = sum(req["size"] for req in requests) # 6 + 6 + 3 + 4 = 19
assert (
total_processed == expected_total
), f"Total processed {total_processed} != expected {expected_total}"
def test_batch_size_fn_mixed_normal_and_large_items(serve_instance):
@serve.deployment
class MixedSizeBatcher:
def __init__(self):
self.batches_processed = []
@serve.batch(
max_batch_size=100,
batch_wait_timeout_s=0.5,
batch_size_fn=lambda items: sum(item["tokens"] for item in items),
)
async def handle_batch(self, requests: List):
batch_info = {
"total_tokens": sum(req["tokens"] for req in requests),
"num_items": len(requests),
}
self.batches_processed.append(batch_info)
return [f"processed_{req['id']}" for req in requests]
async def __call__(self, request):
return await self.handle_batch(request)
def get_batches(self):
return self.batches_processed
handle = serve.run(MixedSizeBatcher.bind())
# Mix of small and larger items
requests = [
{"id": 1, "tokens": 10}, # Small
{"id": 2, "tokens": 20}, # Small
{"id": 3, "tokens": 50}, # Medium
{"id": 4, "tokens": 15}, # Small
{"id": 5, "tokens": 90}, # Large (near limit)
{"id": 6, "tokens": 5}, # Small
]
result_futures = [handle.remote(req) for req in requests]
results = [future.result() for future in result_futures]
# All requests should be processed
expected_results = [f"processed_{i}" for i in range(1, 7)]
assert set(results) == set(expected_results)
# Verify total tokens processed equals sum of all request tokens
batches = handle.get_batches.remote().result()
total_tokens_processed = sum(batch["total_tokens"] for batch in batches)
expected_total = sum(req["tokens"] for req in requests) # 10+20+50+15+90+5 = 190
assert (
total_tokens_processed == expected_total
), f"Total tokens {total_tokens_processed} != expected {expected_total}"
# Verify total items processed equals number of requests
total_items = sum(batch["num_items"] for batch in batches)
assert total_items == len(
requests
), f"Total items {total_items} != expected {len(requests)}"
if __name__ == "__main__":
import sys
sys.exit(pytest.main(["-v", "-s", __file__]))
python/ray/serve/tests/test_broadcast.py
+206
View File
@@ -0,0 +1,206 @@
import os
import sys
import pytest
from ray import serve
def test_broadcast_basic(serve_instance):
"""Test that broadcast() calls every replica."""
@serve.deployment(num_replicas=3)
class D:
def get_pid(self):
return os.getpid()
serve.run(D.bind())
handle = serve.get_deployment_handle("D", "default")
# broadcast() internally waits for replicas to be available.
pids = handle.broadcast("get_pid").results(timeout_s=10)
assert len(pids) == 3
# Each replica should have a unique PID.
assert len(set(pids)) == 3
def test_broadcast_with_args(serve_instance):
"""Test broadcast with positional and keyword arguments."""
@serve.deployment(num_replicas=2)
class D:
def add(self, a, b=0):
return a + b
serve.run(D.bind())
handle = serve.get_deployment_handle("D", "default")
results = handle.broadcast("add", 1, b=2).results(timeout_s=10)
assert len(results) == 2
assert all(r == 3 for r in results)
def test_broadcast_stateful(serve_instance):
"""Test broadcast for state mutation (the cache-reset use case)."""
@serve.deployment(num_replicas=2)
class D:
def __init__(self):
self.cache = {"key": "value"}
def reset_cache(self):
self.cache.clear()
return "cleared"
def get_cache_size(self):
return len(self.cache)
serve.run(D.bind())
handle = serve.get_deployment_handle("D", "default")
# All replicas should start with cache size 1.
sizes = handle.broadcast("get_cache_size").results(timeout_s=10)
assert all(s == 1 for s in sizes)
# Broadcast cache reset.
results = handle.broadcast("reset_cache").results(timeout_s=10)
assert all(r == "cleared" for r in results)
# All replicas should now have empty caches.
sizes = handle.broadcast("get_cache_size").results(timeout_s=10)
assert all(s == 0 for s in sizes)
@pytest.mark.asyncio
async def test_broadcast_async(serve_instance):
"""Test the async results path."""
@serve.deployment(num_replicas=2)
class D:
def get_pid(self):
return os.getpid()
serve.run(D.bind())
handle = serve.get_deployment_handle("D", "default")
pids = await handle.broadcast("get_pid").results_async()
assert len(pids) == 2
assert len(set(pids)) == 2
def test_broadcast_single_replica(serve_instance):
"""Test broadcast with a single replica."""
@serve.deployment(num_replicas=1)
class D:
def ping(self):
return "pong"
serve.run(D.bind())
handle = serve.get_deployment_handle("D", "default")
results = handle.broadcast("ping").results(timeout_s=10)
assert results == ["pong"]
def test_broadcast_ignores_streaming_handle_option(serve_instance):
"""Test broadcast on a handle configured with stream=True."""
@serve.deployment(num_replicas=2)
class D:
def ping(self):
return "pong"
serve.run(D.bind())
handle = serve.get_deployment_handle("D", "default").options(stream=True)
results = handle.broadcast("ping").results(timeout_s=10)
assert len(results) == 2
assert all(r == "pong" for r in results)
def test_broadcast_return_exceptions_sync(serve_instance):
"""Test best-effort sync result collection with exceptions."""
@serve.deployment(num_replicas=2)
class D:
def fail(self):
raise RuntimeError("boom")
serve.run(D.bind())
handle = serve.get_deployment_handle("D", "default")
with pytest.raises(Exception):
handle.broadcast("fail").results(timeout_s=10)
results = handle.broadcast("fail").results(
timeout_s=10,
return_exceptions=True,
)
assert len(results) == 2
assert all(isinstance(r, Exception) for r in results)
@pytest.mark.asyncio
async def test_broadcast_return_exceptions_async(serve_instance):
"""Test best-effort async result collection with exceptions."""
@serve.deployment(num_replicas=2)
class D:
def fail(self):
raise RuntimeError("boom")
serve.run(D.bind())
handle = serve.get_deployment_handle("D", "default")
with pytest.raises(Exception):
await handle.broadcast("fail").results_async()
results = await handle.broadcast("fail").results_async(return_exceptions=True)
assert len(results) == 2
assert all(isinstance(r, Exception) for r in results)
def test_broadcast_sync_timeout(serve_instance):
"""Test sync timeout handling for broadcast results collection."""
@serve.deployment(num_replicas=2)
class D:
def slow(self):
import time
time.sleep(10)
return "done"
serve.run(D.bind())
handle = serve.get_deployment_handle("D", "default")
with pytest.raises(TimeoutError):
handle.broadcast("slow").results(timeout_s=0.5)
@pytest.mark.asyncio
async def test_broadcast_async_timeout(serve_instance):
"""Test async timeout handling for broadcast results collection."""
@serve.deployment(num_replicas=2)
class D:
def slow(self):
import time
time.sleep(10)
return "done"
serve.run(D.bind())
handle = serve.get_deployment_handle("D", "default")
with pytest.raises(TimeoutError):
await handle.broadcast("slow").results_async(timeout_s=0.5)
if __name__ == "__main__":
sys.exit(pytest.main(["-v", "-s", __file__]))
python/ray/serve/tests/test_callback.py
+272
View File
@@ -0,0 +1,272 @@
import importlib
import logging
import os
import sys
from typing import Any, Dict, Generator
import httpx
import pytest
import starlette
from starlette.middleware import Middleware
import ray
from ray import serve
from ray._common.test_utils import wait_for_condition
from ray.exceptions import RayActorError
from ray.serve._private.test_utils import get_application_url
from ray.serve._private.utils import call_function_from_import_path
from ray.serve.config import HTTPOptions, gRPCOptions
from ray.serve.context import _get_global_client
from ray.serve.schema import (
LoggingConfig,
ProxyStatus,
ServeInstanceDetails,
)
# ==== Callbacks used in this test ====
class ASGIMiddleware:
def __init__(self, app):
self.app = app
async def __call__(self, scope, receive, send):
scope.get("headers").append((b"custom_header_key", "custom_header_value"))
await self.app(scope, receive, send)
def add_middleware():
return [Middleware(ASGIMiddleware)]
class MyServeFormatter(logging.Formatter):
def format(self, record: logging.LogRecord):
log_msg = super().format(record)
return "MyCustom message: hello " + log_msg
def add_logger():
ray_logger = logging.getLogger("ray.serve")
handler = logging.StreamHandler()
handler.setFormatter(MyServeFormatter())
ray_logger.addHandler(handler)
def raise_error_callback():
raise RuntimeError("this is from raise_error_callback")
def return_bad_objects():
return [1, 2, 3]
ADD_MIDDLEWARE_IMPORT_PATH = "ray.serve.tests.test_callback.add_middleware"
ADD_LOGGER_IMPORT_PATH = "ray.serve.tests.test_callback.add_logger"
RAISE_ERROR_IMPORT_PATH = "ray.serve.tests.test_callback.raise_error_callback"
RETURN_BAD_OBJECTS_IMPORT_PATH = "ray.serve.tests.test_callback.return_bad_objects"
NOT_CALLABLE_OBJECT = 1
# ==== end ====
@pytest.fixture()
def ray_instance(
request: pytest.FixtureRequest,
) -> Generator[Dict[str, Any], None, None]:
"""Starts and stops a Ray instance for this test.
Args:
request: request.param should contain a dictionary of env vars and
their values. The Ray instance will be started with these env vars.
Yields:
Dict[str, Any]: The dict returned by ``ray.init`` for the started cluster.
"""
original_env_vars = os.environ.copy()
try:
requested_env_vars = request.param
except AttributeError:
requested_env_vars = {}
os.environ.update(requested_env_vars)
importlib.reload(ray.serve._private.constants)
importlib.reload(ray.serve._private.controller)
importlib.reload(ray.serve._private.proxy)
yield ray.init()
serve.shutdown()
# wait_for_processes=True blocks until the raylet/GCS/etc. subprocesses
# have fully exited. Without it, this teardown races the next test's
# ray.init() and the new raylet can fail to register the driver.
ray.shutdown(wait_for_processes=True)
os.environ.clear()
os.environ.update(original_env_vars)
def test_call_function_from_import_path():
"""Basic test for call_function_from_import_path"""
# basic
assert [1, 2, 3] == call_function_from_import_path(RETURN_BAD_OBJECTS_IMPORT_PATH)
# rasie exception when callback function raise exception
with pytest.raises(RuntimeError, match="this is from raise_error_callback"):
call_function_from_import_path(RAISE_ERROR_IMPORT_PATH)
# raise exception when providing invalid import path
with pytest.raises(ValueError, match="cannot be imported"):
call_function_from_import_path("not_exist")
# raise exception when providing non callable object
with pytest.raises(TypeError, match="is not callable"):
call_function_from_import_path(
"ray.serve.tests.test_callback.NOT_CALLABLE_OBJECT"
)
@pytest.mark.parametrize(
"ray_instance",
[
{
"RAY_SERVE_CONTROLLER_CALLBACK_IMPORT_PATH": ADD_LOGGER_IMPORT_PATH,
"RAY_SERVE_HTTP_PROXY_CALLBACK_IMPORT_PATH": ADD_MIDDLEWARE_IMPORT_PATH,
},
],
indirect=True,
)
def test_callback(ray_instance, capsys):
"""Test callback function works in http proxy and controller"""
serve.start(
http_options=HTTPOptions(
host="0.0.0.0",
request_timeout_s=500,
),
)
@serve.deployment
class Model:
def __call__(self, request: starlette.requests.Request):
headers = request.scope.get("headers")
for k, v in headers:
if k == b"custom_header_key":
return v
return "Not found custom headers"
serve.run(Model.bind())
url = get_application_url()
resp = httpx.get(url)
assert resp.text == "custom_header_value"
captured = capsys.readouterr()
assert "MyCustom message: hello" in captured.err
@pytest.mark.parametrize(
"ray_instance",
[
{
"RAY_SERVE_CONTROLLER_CALLBACK_IMPORT_PATH": "not_exist",
"RAY_SERVE_HTTP_PROXY_CALLBACK_IMPORT_PATH": RAISE_ERROR_IMPORT_PATH,
},
],
indirect=True,
)
def test_callback_fail(ray_instance):
"""Test actor call call_function_from_import_path rasing exception.
Actor will fail to be started and further call will raise RayActorError.
"""
actor_def = ray.serve._private.proxy.ProxyActor
handle = actor_def.remote(
http_options=HTTPOptions(host="http_proxy", root_path="/", port=123),
grpc_options=gRPCOptions(),
node_ip_address="127.0.0.1",
node_id="123",
logging_config=LoggingConfig(),
long_poll_client="fake_client",
)
with pytest.raises(RayActorError, match="this is from raise_error_callback"):
ray.get(handle.ready.remote())
serve_controller = ray.serve._private.controller.ServeController
actor_def = ray.actor._make_actor(serve_controller, {})
handle = actor_def.remote(
http_options=HTTPOptions(),
grpc_options=gRPCOptions(),
global_logging_config=LoggingConfig(),
)
with pytest.raises(RayActorError, match="cannot be imported"):
ray.get(handle.check_alive.remote())
@pytest.mark.parametrize(
"ray_instance",
[
{
"RAY_SERVE_HTTP_PROXY_CALLBACK_IMPORT_PATH": RETURN_BAD_OBJECTS_IMPORT_PATH,
},
],
indirect=True,
)
def test_http_proxy_return_aribitary_objects(ray_instance):
"""Test invalid callback path in http proxy"""
actor_def = ray.serve._private.proxy.ProxyActor
handle = actor_def.remote(
http_options=HTTPOptions(host="http_proxy", root_path="/", port=123),
grpc_options=gRPCOptions(),
node_ip_address="127.0.0.1",
node_id="123",
logging_config=LoggingConfig(),
long_poll_client="fake_client",
)
with pytest.raises(
RayActorError, match="must return a list of Starlette middlewares"
):
ray.get(handle.ready.remote())
@pytest.mark.parametrize(
"ray_instance",
[
{
"RAY_SERVE_HTTP_PROXY_CALLBACK_IMPORT_PATH": RAISE_ERROR_IMPORT_PATH,
},
],
indirect=True,
)
def test_http_proxy_callback_failures(ray_instance, capsys):
"""Test http proxy keeps restarting when callback function fails"""
try:
serve.start()
except RayActorError:
# serve.start will fail because the http proxy is not started successfully
# and client use proxy handle to check the proxy readiness, so it will raise
# RayActorError.
pass
client = _get_global_client()
def check_http_proxy_keep_restarting():
# The proxy will be under "STARTING" status and keep restarting.
prev_actor_id = None
while True:
serve_details = ServeInstanceDetails(**client.get_serve_details())
for _, proxy_info in serve_details.proxies.items():
if proxy_info.status != ProxyStatus.STARTING:
return False
if prev_actor_id is None:
prev_actor_id = proxy_info.actor_id
break
elif prev_actor_id != proxy_info.actor_id:
return True
wait_for_condition(check_http_proxy_keep_restarting)
if __name__ == "__main__":
sys.exit(pytest.main(["-v", "-s", __file__]))
python/ray/serve/tests/test_capacity_queue_router.py
+880
View File
@@ -0,0 +1,880 @@
import sys
import uuid
from collections import Counter
import pytest
import ray
from ray import serve
from ray._common.test_utils import SignalActor, wait_for_condition
from ray.serve._private.constants import SERVE_DEPLOYMENT_ACTOR_PREFIX, SERVE_NAMESPACE
from ray.serve._private.test_utils import check_running
from ray.serve.config import DeploymentActorConfig, RequestRouterConfig
from ray.serve.context import _get_internal_replica_context
from ray.serve.experimental.capacity_queue import (
CapacityQueue,
)
def _deploy_capacity_queue_app(
num_replicas: int = 3,
max_ongoing_requests: int = 5,
acquire_timeout_s: float = 0.5,
token_ttl_s: float = 5,
):
"""Deploy a simple app with CapacityQueue deployment actor and CapacityQueueRouter."""
@serve.deployment(
deployment_actors=[
DeploymentActorConfig(
name="capacity_queue",
actor_class=CapacityQueue,
init_kwargs={
"acquire_timeout_s": acquire_timeout_s,
"token_ttl_s": token_ttl_s,
},
actor_options={"num_cpus": 0},
),
],
request_router_config=RequestRouterConfig(
request_router_class=(
"ray.serve.experimental.capacity_queue_router:CapacityQueueRouter"
),
request_router_kwargs={
"capacity_queue_actor_name": "capacity_queue",
},
initial_backoff_s=0.01,
backoff_multiplier=2.0,
max_backoff_s=0.1,
),
num_replicas=num_replicas,
max_ongoing_requests=max_ongoing_requests,
ray_actor_options={"num_cpus": 0},
)
class App:
def __init__(self):
context = _get_internal_replica_context()
self.replica_id = context.replica_id
self.unique_id = context.replica_id.unique_id
async def __call__(self):
return self.unique_id
handle = serve.run(App.bind())
return handle
def _deploy_blocking_capacity_queue_app(
signal_actor_name: str,
num_replicas: int = 2,
max_ongoing_requests: int = 5,
):
"""Deploy an app whose requests block until a SignalActor is triggered."""
@serve.deployment(
deployment_actors=[
DeploymentActorConfig(
name="capacity_queue",
actor_class=CapacityQueue,
init_kwargs={
"acquire_timeout_s": 0.5,
"token_ttl_s": 5,
},
actor_options={"num_cpus": 0},
),
],
request_router_config=RequestRouterConfig(
request_router_class=(
"ray.serve.experimental.capacity_queue_router:CapacityQueueRouter"
),
request_router_kwargs={
"capacity_queue_actor_name": "capacity_queue",
},
initial_backoff_s=0.01,
backoff_multiplier=2.0,
max_backoff_s=0.1,
),
num_replicas=num_replicas,
max_ongoing_requests=max_ongoing_requests,
ray_actor_options={"num_cpus": 0},
)
class BlockingApp:
def __init__(self):
context = _get_internal_replica_context()
self.unique_id = context.replica_id.unique_id
async def __call__(self):
signal = ray.get_actor(signal_actor_name)
await signal.wait.remote()
return self.unique_id
handle = serve.run(BlockingApp.bind())
return handle
def _find_capacity_queue_handle():
"""Find the CapacityQueue deployment actor."""
actors = ray.util.list_named_actors(all_namespaces=True)
for actor_info in actors:
if (
actor_info["namespace"] == SERVE_NAMESPACE
and "capacity_queue" in actor_info["name"]
and SERVE_DEPLOYMENT_ACTOR_PREFIX in actor_info["name"]
):
return ray.get_actor(actor_info["name"], namespace=SERVE_NAMESPACE)
return None
class TestCapacityQueueRouterBasic:
"""Basic integration tests for the capacity queue router."""
def test_single_request(self, serve_instance):
"""A single request should be routed to one of the replicas."""
handle = _deploy_capacity_queue_app(num_replicas=2)
# Wait for deployment to be healthy
wait_for_condition(check_running, timeout=30)
response = handle.remote().result(timeout_s=10)
assert isinstance(response, str)
assert len(response) > 0
def test_multiple_requests_distributed(self, serve_instance):
"""Requests should be distributed across replicas."""
num_replicas = 3
handle = _deploy_capacity_queue_app(num_replicas=num_replicas)
wait_for_condition(check_running, timeout=30)
# Send enough requests that all replicas should receive at least one
num_requests = 30
responses = []
for _ in range(num_requests):
r = handle.remote().result(timeout_s=10)
responses.append(r)
unique_replicas = set(responses)
# All replicas should have received at least one request
assert len(unique_replicas) == num_replicas, (
f"Expected {num_replicas} unique replicas, got {len(unique_replicas)}: "
f"{unique_replicas}"
)
def test_concurrent_requests(self, serve_instance):
"""Concurrent requests should be distributed across replicas."""
num_replicas = 3
handle = _deploy_capacity_queue_app(num_replicas=num_replicas)
wait_for_condition(check_running, timeout=30)
# Send concurrent requests
refs = [handle.remote() for _ in range(30)]
responses = [ref.result(timeout_s=30) for ref in refs]
unique_replicas = set(responses)
assert len(unique_replicas) == num_replicas
def test_capacity_queue_stats(self, serve_instance):
"""The capacity queue should track stats correctly.
Some early requests may fall back to power-of-two-choices before the
router discovers the queue, so we assert >= rather than exact counts.
"""
handle = _deploy_capacity_queue_app(num_replicas=2)
wait_for_condition(check_running, timeout=30)
queue_handle = _find_capacity_queue_handle()
assert queue_handle is not None, "CapacityQueue deployment actor not found"
# Wait for queue to have replicas before sending requests so most
# go through the queue path (not power-of-two-choices fallback).
wait_for_condition(
lambda: ray.get(queue_handle.get_stats.remote()).num_replicas == 2,
timeout=15,
)
# Send some requests
for _ in range(10):
handle.remote().result(timeout_s=10)
# Wait for all releases to settle (on_request_completed is async)
def _stats_settled():
stats = ray.get(queue_handle.get_stats.remote())
assert stats.num_replicas == 2
assert stats.total_in_flight == 0
# Most requests should go through the queue. Some may fall back
# to power-of-two-choices, so use >= with a lower bound.
assert stats.total_acquires >= 5
assert stats.total_releases >= 5
return True
wait_for_condition(_stats_settled, timeout=10)
class TestCapacityQueueRouterLoadBalancing:
"""Tests for load balancing behavior."""
def test_least_loaded_balancing(self, serve_instance):
"""Requests should be balanced across replicas (least-loaded)."""
num_replicas = 3
handle = _deploy_capacity_queue_app(num_replicas=num_replicas)
wait_for_condition(check_running, timeout=30)
# Send sequential requests - should round-robin approximately
num_requests = 60
responses = []
for _ in range(num_requests):
r = handle.remote().result(timeout_s=10)
responses.append(r)
counter = Counter(responses)
# Each replica should get roughly equal share
expected_per_replica = num_requests / num_replicas
for replica_id, count in counter.items():
assert (
count >= expected_per_replica * 0.3
), f"Replica {replica_id} got {count} requests, expected ~{expected_per_replica}"
class TestCapacityQueueRouterWithSingleReplica:
"""Tests with a single replica to verify basic token flow."""
def test_single_replica_all_requests(self, serve_instance):
"""With one replica, all requests should go to the same replica."""
handle = _deploy_capacity_queue_app(num_replicas=1)
wait_for_condition(check_running, timeout=30)
responses = set()
for _ in range(10):
r = handle.remote().result(timeout_s=10)
responses.add(r)
assert len(responses) == 1
class TestCapacityQueueRouterPowerOfTwoFallback:
"""Tests that the router falls back to power-of-two-choices when the
queue is unavailable."""
def test_requests_succeed_without_queue(self, serve_instance):
"""Requests succeed via power-of-two-choices even when the queue is
killed immediately."""
handle = _deploy_capacity_queue_app(num_replicas=2)
wait_for_condition(check_running, timeout=30)
queue = _find_capacity_queue_handle()
wait_for_condition(
lambda: ray.get(queue.get_stats.remote()).num_replicas == 2,
timeout=15,
)
# Kill the queue so all subsequent requests must use fallback.
ray.kill(queue)
for _ in range(5):
resp = handle.remote().result(timeout_s=15)
assert isinstance(resp, str)
def test_requests_distributed_without_queue(self, serve_instance):
"""In fallback mode, requests are still distributed across replicas."""
num_replicas = 3
handle = _deploy_capacity_queue_app(num_replicas=num_replicas)
wait_for_condition(check_running, timeout=30)
queue = _find_capacity_queue_handle()
wait_for_condition(
lambda: ray.get(queue.get_stats.remote()).num_replicas == num_replicas,
timeout=15,
)
# Kill the queue.
ray.kill(queue)
responses = []
for _ in range(30):
r = handle.remote().result(timeout_s=15)
responses.append(r)
unique_replicas = set(responses)
assert len(unique_replicas) == num_replicas
class TestCapacityQueueRouterFailures:
def test_unreleased_token_recovered_by_ttl(self, serve_instance):
"""Leaked tokens are automatically reclaimed after the TTL expires.
When a token is acquired but never released (e.g. a router process
dies between acquire() and release()), the queue's in_flight count
stays elevated. With token_ttl_s configured, a background reaper
reclaims expired tokens and restores full capacity.
"""
token_ttl_s = 2.0
@serve.deployment(
deployment_actors=[
DeploymentActorConfig(
name="capacity_queue",
actor_class=CapacityQueue,
init_kwargs={
"acquire_timeout_s": 0.5,
"token_ttl_s": token_ttl_s,
},
actor_options={"num_cpus": 0},
),
],
request_router_config=RequestRouterConfig(
request_router_class=(
"ray.serve.experimental.capacity_queue_router:CapacityQueueRouter"
),
request_router_kwargs={
"capacity_queue_actor_name": "capacity_queue",
},
),
num_replicas=1,
max_ongoing_requests=3,
ray_actor_options={"num_cpus": 0},
)
class TtlApp:
def __init__(self):
context = _get_internal_replica_context()
self.unique_id = context.replica_id.unique_id
async def __call__(self):
return self.unique_id
handle = serve.run(TtlApp.bind())
wait_for_condition(check_running, timeout=30)
queue = _find_capacity_queue_handle()
wait_for_condition(
lambda: ray.get(queue.get_stats.remote()).num_replicas == 1,
timeout=15,
)
# Simulate a router acquiring a token then crashing (never releases).
leaked = ray.get(queue.acquire.remote(timeout_s=5))
assert leaked is not None
stats = ray.get(queue.get_stats.remote())
assert stats.total_in_flight == 1
assert stats.queue_size == 2 # 3 capacity - 1 leaked
# Remaining capacity still serves requests.
resp = handle.remote().result(timeout_s=10)
assert isinstance(resp, str)
# After the TTL expires, the reaper reclaims the leaked token and
# full capacity is restored.
def _capacity_restored():
s = ray.get(queue.get_stats.remote())
return s.total_in_flight == 0 and s.queue_size == 3
wait_for_condition(_capacity_restored, timeout=token_ttl_s + 5)
def test_replica_death_releases_token_and_recovers(self, serve_instance):
"""When a replica dies mid-request, its token is released and
the queue stops routing to it after the long-poll update."""
@serve.deployment(
deployment_actors=[
DeploymentActorConfig(
name="capacity_queue",
actor_class=CapacityQueue,
init_kwargs={
"acquire_timeout_s": 0.5,
"token_ttl_s": 5,
},
actor_options={"num_cpus": 0},
),
],
request_router_config=RequestRouterConfig(
request_router_class=(
"ray.serve.experimental.capacity_queue_router:CapacityQueueRouter"
),
request_router_kwargs={
"capacity_queue_actor_name": "capacity_queue",
},
),
num_replicas=2,
max_ongoing_requests=2,
ray_actor_options={"num_cpus": 0},
)
class CrashApp:
def __init__(self):
context = _get_internal_replica_context()
self.unique_id = context.replica_id.unique_id
async def __call__(self, crash: bool = False):
if crash:
import os
os._exit(1)
return self.unique_id
handle = serve.run(CrashApp.bind())
wait_for_condition(check_running, timeout=30)
queue = _find_capacity_queue_handle()
wait_for_condition(
lambda: ray.get(queue.get_stats.remote()).num_replicas == 2,
timeout=15,
)
# Crash one replica by sending a request that exits the process.
try:
handle.remote(crash=True).result(timeout_s=5)
except Exception:
pass # Expected — the replica died.
# The controller detects the death, removes the replica, and starts
# a replacement. Long poll updates the queue. Eventually the queue
# should recover to 2 replicas (the survivor + the replacement)
# with full capacity and no leaked in-flight counts.
def _cluster_fully_recovered():
stats = ray.get(queue.get_stats.remote())
assert stats.num_replicas == 2
assert stats.total_capacity == 4 # 2 replicas * max_ongoing_requests=2
assert stats.total_in_flight == 0
return True
wait_for_condition(_cluster_fully_recovered, timeout=30)
# Requests still succeed — routed to the surviving / replacement replica.
resp = handle.remote().result(timeout_s=15)
assert isinstance(resp, str)
def test_capacity_queue_death_and_recovery(self, serve_instance):
"""When the CapacityQueue actor dies, the router falls back to
power-of-two-choices and requests continue to succeed. Once the
controller recreates the queue, the router rediscovers it and
resumes token-based routing.
"""
handle = _deploy_capacity_queue_app(num_replicas=2)
wait_for_condition(check_running, timeout=30)
queue = _find_capacity_queue_handle()
wait_for_condition(
lambda: ray.get(queue.get_stats.remote()).num_replicas == 2,
timeout=15,
)
# Verify requests work before the kill.
resp = handle.remote().result(timeout_s=10)
assert isinstance(resp, str)
# Kill the capacity queue actor.
ray.kill(queue)
# Requests should STILL succeed via power-of-two-choices fallback
# even while the queue is dead.
resp = handle.remote().result(timeout_s=15)
assert isinstance(resp, str)
# The controller recreates the deployment actor. The new queue starts
# fresh and gets replicas via long poll. Wait for it to appear.
def _queue_recovered():
new_q = _find_capacity_queue_handle()
if new_q is None:
return False
stats = ray.get(new_q.get_stats.remote())
return stats.num_replicas == 2
wait_for_condition(_queue_recovered, timeout=30)
# After recovery, requests go through the queue again. Verify the
# new queue is being used by checking that acquires increase.
new_queue = _find_capacity_queue_handle()
stats_before = ray.get(new_queue.get_stats.remote())
for _ in range(3):
handle.remote().result(timeout_s=10)
def _queue_used():
stats = ray.get(new_queue.get_stats.remote())
return stats.total_acquires > stats_before.total_acquires
wait_for_condition(_queue_used, timeout=10)
def test_capacity_queue_restarts_with_full_capacity(self, serve_instance):
"""
After a queue restart, it bootstraps with full capacity even though
replicas may have in-flight requests from before the crash.
"""
signal_name = f"block_signal_{uuid.uuid4().hex[:8]}"
signal = SignalActor.options(name=signal_name).remote()
handle = _deploy_blocking_capacity_queue_app(
signal_actor_name=signal_name,
num_replicas=1,
max_ongoing_requests=2,
)
wait_for_condition(check_running, timeout=30)
queue = _find_capacity_queue_handle()
wait_for_condition(
lambda: ray.get(queue.get_stats.remote()).num_replicas == 1,
timeout=15,
)
# Send a blocking request — occupies 1 of 2 slots on the replica.
ref = handle.remote()
wait_for_condition(
lambda: ray.get(queue.get_stats.remote()).total_in_flight == 1,
timeout=10,
)
# Kill the capacity queue.
ray.kill(queue)
# Wait for the controller to recreate it.
def _new_queue_ready():
q = _find_capacity_queue_handle()
if q is None:
return False
stats = ray.get(q.get_stats.remote())
return stats.num_replicas == 1
wait_for_condition(_new_queue_ready, timeout=30)
# The new queue shows full capacity (2) even though the replica still
# has 1 in-flight request from before the crash.
new_queue = _find_capacity_queue_handle()
stats = ray.get(new_queue.get_stats.remote())
assert stats.total_capacity == 2
assert stats.total_in_flight == 0 # Queue doesn't know about the old request
# Release the signal so the blocked request finishes.
ray.get(signal.send.remote())
try:
ref.result(timeout_s=10)
except Exception:
pass # May fail since the queue died mid-request
# Cleanup
ray.kill(signal)
def test_queue_converges_after_restart(self, serve_instance):
"""After the queue restarts, its per-replica token view converges to
match actual replica capacity.
Setup: 1 replica, max_ongoing_requests=5, 3 blocked requests.
1. Send 3 blocking requests occupying 3/5 slots. Queue correctly
shows in_flight=3, available_tokens=2 for the replica.
2. Kill the queue — it restarts with in_flight=0, thinking the
replica has 5 available tokens (stale).
3. The router sends requests via the stale queue. Tokens for the
3 occupied slots get rejected. Unreleased rejection tokens
ratchet in_flight up, teaching the queue the correct state.
4. Release the blocking requests — replica frees all 5 slots.
5. TTL reaper clears phantom in_flight entries from rejections.
6. Assert per-replica convergence: available_tokens == max_capacity.
"""
token_ttl_s = 2.0
max_ongoing = 5
num_blocked = 3
signal_name = f"block_signal_{uuid.uuid4().hex[:8]}"
signal = SignalActor.options(name=signal_name).remote()
@serve.deployment(
deployment_actors=[
DeploymentActorConfig(
name="capacity_queue",
actor_class=CapacityQueue,
init_kwargs={
"acquire_timeout_s": 0.5,
"token_ttl_s": token_ttl_s,
},
actor_options={"num_cpus": 0},
),
],
request_router_config=RequestRouterConfig(
request_router_class=(
"ray.serve.experimental.capacity_queue_router:CapacityQueueRouter"
),
request_router_kwargs={
"capacity_queue_actor_name": "capacity_queue",
},
),
num_replicas=1,
max_ongoing_requests=max_ongoing,
ray_actor_options={"num_cpus": 0},
)
class ConvergeApp:
def __init__(self):
context = _get_internal_replica_context()
self.unique_id = context.replica_id.unique_id
async def __call__(self, block: bool = False):
if block:
sig = ray.get_actor(signal_name)
await sig.wait.remote()
return self.unique_id
handle = serve.run(ConvergeApp.bind())
wait_for_condition(check_running, timeout=30)
queue = _find_capacity_queue_handle()
wait_for_condition(
lambda: ray.get(queue.get_stats.remote()).num_replicas == 1,
timeout=15,
)
# Step 1: Occupy 3 of 5 slots with blocking requests.
blocking_refs = [handle.remote(block=True) for _ in range(num_blocked)]
wait_for_condition(
lambda: ray.get(queue.get_stats.remote()).total_in_flight == num_blocked,
timeout=15,
)
# Verify pre-crash per-replica state: in_flight=3, capacity=5.
replica_info = ray.get(queue.get_replica_in_flight.remote())
assert len(replica_info) == 1
for rid, (in_flight, max_cap) in replica_info.items():
assert max_cap == max_ongoing
assert in_flight == num_blocked
assert max_cap - in_flight == 2 # 2 available tokens
# Step 2: Kill the queue. It restarts with in_flight=0.
ray.kill(queue)
def _new_queue_ready():
q = _find_capacity_queue_handle()
if q is None:
return False
stats = ray.get(q.get_stats.remote())
return stats.num_replicas == 1
wait_for_condition(_new_queue_ready, timeout=30)
new_queue = _find_capacity_queue_handle()
# Verify stale state: queue thinks replica has 5 available tokens.
stale_info = ray.get(new_queue.get_replica_in_flight.remote())
for rid, (in_flight, max_cap) in stale_info.items():
assert in_flight == 0
assert max_cap == max_ongoing
# Step 3 & 4: Release the blocking requests so the replica frees up.
ray.get(signal.send.remote())
for ref in blocking_refs:
try:
ref.result(timeout_s=15)
except Exception:
pass # May fail — queue died while these were in flight.
# Send requests to exercise the queue and trigger any rejection-based
# learning for the stale window.
for _ in range(5):
handle.remote().result(timeout_s=15)
# Step 5 & 6: Wait for TTL reaper, then verify per-replica convergence.
# available_tokens (max_capacity - in_flight) must equal max_capacity
# because the replica has 0 real in-flight after the signal release.
def _per_replica_converged():
info = ray.get(new_queue.get_replica_in_flight.remote())
if len(info) != 1:
return False
for in_flight, max_cap in info.values():
if max_cap - in_flight != max_ongoing:
return False
return True
wait_for_condition(_per_replica_converged, timeout=token_ttl_s + 10)
# Final assertion: in_flight is exactly 0, all 5 tokens available.
final_info = ray.get(new_queue.get_replica_in_flight.remote())
for rid, (in_flight, max_cap) in final_info.items():
assert (
in_flight == 0
), f"Replica {rid}: expected converged in_flight=0, got {in_flight}"
assert max_cap - in_flight == max_ongoing
ray.kill(signal)
def test_capacity_depleted_backoff_and_recovery(self, serve_instance):
"""
When all replicas are at capacity, the router backs off and
retries until capacity frees up.
"""
signal_name = f"block_signal_{uuid.uuid4().hex[:8]}"
signal = SignalActor.options(name=signal_name).remote()
@serve.deployment(
deployment_actors=[
DeploymentActorConfig(
name="capacity_queue",
actor_class=CapacityQueue,
init_kwargs={
"acquire_timeout_s": 0.5,
"token_ttl_s": 5,
},
actor_options={"num_cpus": 0},
),
],
request_router_config=RequestRouterConfig(
request_router_class=(
"ray.serve.experimental.capacity_queue_router:CapacityQueueRouter"
),
request_router_kwargs={
"capacity_queue_actor_name": "capacity_queue",
},
),
num_replicas=1,
max_ongoing_requests=2,
ray_actor_options={"num_cpus": 0},
)
class DepletedApp:
def __init__(self):
context = _get_internal_replica_context()
self.unique_id = context.replica_id.unique_id
async def __call__(self, block: bool = False):
if block:
sig = ray.get_actor(signal_name)
await sig.wait.remote()
return self.unique_id
handle = serve.run(DepletedApp.bind())
wait_for_condition(check_running, timeout=30)
queue = _find_capacity_queue_handle()
wait_for_condition(
lambda: ray.get(queue.get_stats.remote()).num_replicas == 1,
timeout=10,
)
# Fill both slots with blocking requests.
blocking_refs = [handle.remote(block=True) for _ in range(2)]
wait_for_condition(
lambda: ray.get(queue.get_stats.remote()).total_in_flight == 2,
timeout=10,
)
# Send a third request — will be blocked waiting for capacity.
waiting_ref = handle.remote(block=False)
# Wait for at least one CQ timeout — proves the router hit the
# depleted path and backed off (total_timeouts is 0 before this
# since the blocking requests were acquired via the fast path).
wait_for_condition(
lambda: ray.get(queue.get_stats.remote()).total_timeouts >= 1,
timeout=30,
)
# Release the blockers so capacity frees up.
ray.get(signal.send.remote())
for ref in blocking_refs:
ref.result(timeout_s=15)
# The waiting request should complete once capacity is available.
result = waiting_ref.result(timeout_s=15)
assert isinstance(result, str)
ray.kill(signal)
def test_rejection_teaches_cq_after_restart(self, serve_instance):
"""
After a CQ restart, rejected tokens are NOT released back to the
CQ, so in_flight stays elevated and the CQ learns the replica is busy.
"""
signal_name = f"block_signal_{uuid.uuid4().hex[:8]}"
signal = SignalActor.options(name=signal_name).remote()
@serve.deployment(
deployment_actors=[
DeploymentActorConfig(
name="capacity_queue",
actor_class=CapacityQueue,
init_kwargs={
"acquire_timeout_s": 0.5,
"token_ttl_s": 5,
},
actor_options={"num_cpus": 0},
),
],
request_router_config=RequestRouterConfig(
request_router_class=(
"ray.serve.experimental.capacity_queue_router:CapacityQueueRouter"
),
request_router_kwargs={
"capacity_queue_actor_name": "capacity_queue",
},
),
num_replicas=1,
max_ongoing_requests=2,
ray_actor_options={"num_cpus": 0},
)
class RejectApp:
def __init__(self):
context = _get_internal_replica_context()
self.unique_id = context.replica_id.unique_id
async def __call__(self, block: bool = False):
if block:
sig = ray.get_actor(signal_name)
await sig.wait.remote()
return self.unique_id
handle = serve.run(RejectApp.bind())
wait_for_condition(check_running, timeout=30)
queue = _find_capacity_queue_handle()
wait_for_condition(
lambda: ray.get(queue.get_stats.remote()).num_replicas == 1,
timeout=15,
)
# Step 1: Saturate the replica — 2/2 slots occupied.
blocking_refs = [handle.remote(block=True) for _ in range(2)]
wait_for_condition(
lambda: ray.get(queue.get_stats.remote()).total_in_flight == 2,
timeout=10,
)
# Step 2: Kill the CQ. It restarts with in_flight=0.
ray.kill(queue)
def _new_queue_ready():
q = _find_capacity_queue_handle()
if q is None:
return False
stats = ray.get(q.get_stats.remote())
return stats.num_replicas == 1
wait_for_condition(_new_queue_ready, timeout=30)
new_queue = _find_capacity_queue_handle()
stale_stats = ray.get(new_queue.get_stats.remote())
assert stale_stats.total_in_flight == 0 # Stale: thinks replica is idle
# Step 3: Send a non-blocking request. The stale CQ issues a token,
# the replica rejects (full), the router retries. The rejected token
# is NOT released, so the CQ's in_flight ratchets up.
new_ref = handle.remote(block=False)
# Step 4: The CQ should have learned — in_flight > 0 because the
# rejected token was not released.
def _cq_learned():
stats = ray.get(new_queue.get_stats.remote())
return stats.total_in_flight > 0
wait_for_condition(_cq_learned, timeout=15)
# Step 5: Release the blockers so all requests complete.
ray.get(signal.send.remote())
for ref in blocking_refs:
try:
ref.result(timeout_s=15)
except Exception:
pass
result = new_ref.result(timeout_s=15)
assert isinstance(result, str)
ray.kill(signal)
if __name__ == "__main__":
sys.exit(pytest.main([__file__, "-v", "-s"]))
python/ray/serve/tests/test_certs/ca.crt
+21
View File
@@ -0,0 +1,21 @@
-----BEGIN CERTIFICATE-----
MIIDfTCCAmWgAwIBAgIUYcUOt0aN1Ml/1WnFPB9gveNNniQwDQYJKoZIhvcNAQEL
BQAwZzELMAkGA1UEBhMCVVMxEzARBgNVBAgMCkNhbGlmb3JuaWExFjAUBgNVBAcM
DVNhbiBGcmFuY2lzY28xFzAVBgNVBAoMDlJheSBTZXJ2ZSBUZXN0MRIwEAYDVQQD
DAlsb2NhbGhvc3QwHhcNMjUwODIwMTgxODUzWhcNMjYwODIwMTgxODUzWjBnMQsw
CQYDVQQGEwJVUzETMBEGA1UECAwKQ2FsaWZvcm5pYTEWMBQGA1UEBwwNU2FuIEZy
YW5jaXNjbzEXMBUGA1UECgwOUmF5IFNlcnZlIFRlc3QxEjAQBgNVBAMMCWxvY2Fs
aG9zdDCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAKYXcIirTR5AHb5V
T6yijOR8mvc6AXSKkmIKu7n2vaJ3Jrt7d6mPz/ScXlLYxq+mgt4avX/VozES0ARM
NcbqlHOcahgfyyN+/02q/Aimwbaf/FwiS5qyQfMXzFg70kydqlDlUsyE49qdFHEv
xx4ostLnTeyIpS7AS14qJXGeg5NE9Pm+XSs0HVBPZBaM6VCJl8/Pjog0qqffovGo
/qN8gVxnydg4ayTZ9nl+NNMivFJ/f5MUXmJiuFYAoZnwMiCy2QAU9TmdA5mCOGNZ
pv/KSSdqkVh7X6JNGB6OLgikCsObWxAJqq7WZgiHoc2WlXuN+U2SLuA0JLZZZr+t
zpw1DH0CAwEAAaMhMB8wHQYDVR0OBBYEFIey4ZBoVICZ7kAJv7K5kY/SHP6wMA0G
CSqGSIb3DQEBCwUAA4IBAQAg47MfYFykzDdynJnKf/Aqlp4bnT3GVEW3lRk8AMv9
yrjwQeVKihiQLgC6b7ChyLUQWxcxJPqhzAIe/+sn9bAxz448oGMtU6ghHtxt13T2
9VKsyyrjgZ3fbiFT5AFMYxwYlcaf1hJPE+PKKU3oUhYxUlEBKweDjTw7+7xym/Ix
hNYv36lDst/zwA1HKmvorDhCVOT3Y90deVA31NxFQbqNpeCjG6uiURAtO3jMan50
m9U60cHjJBkSxCKCw4SQXOan9VKePIsHnZgIiDPmO25KYSJxeat92sHVtI3FZfrh
pN3cjQaXhMbJFO9ySv5tqr0KxUbymN56ynWkScMGbI0W
-----END CERTIFICATE-----
python/ray/serve/tests/test_certs/server.crt
+21
View File
@@ -0,0 +1,21 @@
-----BEGIN CERTIFICATE-----
MIIDfTCCAmWgAwIBAgIUYcUOt0aN1Ml/1WnFPB9gveNNniQwDQYJKoZIhvcNAQEL
BQAwZzELMAkGA1UEBhMCVVMxEzARBgNVBAgMCkNhbGlmb3JuaWExFjAUBgNVBAcM
DVNhbiBGcmFuY2lzY28xFzAVBgNVBAoMDlJheSBTZXJ2ZSBUZXN0MRIwEAYDVQQD
DAlsb2NhbGhvc3QwHhcNMjUwODIwMTgxODUzWhcNMjYwODIwMTgxODUzWjBnMQsw
CQYDVQQGEwJVUzETMBEGA1UECAwKQ2FsaWZvcm5pYTEWMBQGA1UEBwwNU2FuIEZy
YW5jaXNjbzEXMBUGA1UECgwOUmF5IFNlcnZlIFRlc3QxEjAQBgNVBAMMCWxvY2Fs
aG9zdDCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAKYXcIirTR5AHb5V
T6yijOR8mvc6AXSKkmIKu7n2vaJ3Jrt7d6mPz/ScXlLYxq+mgt4avX/VozES0ARM
NcbqlHOcahgfyyN+/02q/Aimwbaf/FwiS5qyQfMXzFg70kydqlDlUsyE49qdFHEv
xx4ostLnTeyIpS7AS14qJXGeg5NE9Pm+XSs0HVBPZBaM6VCJl8/Pjog0qqffovGo
/qN8gVxnydg4ayTZ9nl+NNMivFJ/f5MUXmJiuFYAoZnwMiCy2QAU9TmdA5mCOGNZ
pv/KSSdqkVh7X6JNGB6OLgikCsObWxAJqq7WZgiHoc2WlXuN+U2SLuA0JLZZZr+t
zpw1DH0CAwEAAaMhMB8wHQYDVR0OBBYEFIey4ZBoVICZ7kAJv7K5kY/SHP6wMA0G
CSqGSIb3DQEBCwUAA4IBAQAg47MfYFykzDdynJnKf/Aqlp4bnT3GVEW3lRk8AMv9
yrjwQeVKihiQLgC6b7ChyLUQWxcxJPqhzAIe/+sn9bAxz448oGMtU6ghHtxt13T2
9VKsyyrjgZ3fbiFT5AFMYxwYlcaf1hJPE+PKKU3oUhYxUlEBKweDjTw7+7xym/Ix
hNYv36lDst/zwA1HKmvorDhCVOT3Y90deVA31NxFQbqNpeCjG6uiURAtO3jMan50
m9U60cHjJBkSxCKCw4SQXOan9VKePIsHnZgIiDPmO25KYSJxeat92sHVtI3FZfrh
pN3cjQaXhMbJFO9ySv5tqr0KxUbymN56ynWkScMGbI0W
-----END CERTIFICATE-----
python/ray/serve/tests/test_certs/server.csr
+17
View File
@@ -0,0 +1,17 @@
-----BEGIN CERTIFICATE REQUEST-----
MIICrDCCAZQCAQAwZzELMAkGA1UEBhMCVVMxEzARBgNVBAgMCkNhbGlmb3JuaWEx
FjAUBgNVBAcMDVNhbiBGcmFuY2lzY28xFzAVBgNVBAoMDlJheSBTZXJ2ZSBUZXN0
MRIwEAYDVQQDDAlsb2NhbGhvc3QwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEK
AoIBAQCmF3CIq00eQB2+VU+soozkfJr3OgF0ipJiCru59r2idya7e3epj8/0nF5S
2MavpoLeGr1/1aMxEtAETDXG6pRznGoYH8sjfv9NqvwIpsG2n/xcIkuaskHzF8xY
O9JMnapQ5VLMhOPanRRxL8ceKLLS503siKUuwEteKiVxnoOTRPT5vl0rNB1QT2QW
jOlQiZfPz46INKqn36LxqP6jfIFcZ8nYOGsk2fZ5fjTTIrxSf3+TFF5iYrhWAKGZ
8DIgstkAFPU5nQOZgjhjWab/ykknapFYe1+iTRgeji4IpArDm1sQCaqu1mYIh6HN
lpV7jflNki7gNCS2WWa/rc6cNQx9AgMBAAGgADANBgkqhkiG9w0BAQsFAAOCAQEA
igYR2ZQ4fmp339T/BGvXSDIjQQkecd9MeifdcXuN/2FZ7dhyfDWHjQadtohgXSZw
LwfUx43L+JcebMY8GyN/4JIAKA5hVqqvAiaMb+vRUItgku5M2WIpnPLVKQJHTUGC
aaDq6u7aS4eFcvuYGaFTUD7tNMOfRP8SfQL/sk2UqZVOCIxCFX9gLS/p4IyorUsb
VjdQBHRvOZnZCFMwmisquXXeGxtAPabUWMPLvSqcP/93WdjFwtrcscyY68s+AC6o
9sx1x3qjnTxnx+a8ho5f0p/JSUqye+G/gzqzB5WMZK5U7oiYgP0rEajU9odGIPSK
AqzWpVDtZBSr8FFamw4uqQ==
-----END CERTIFICATE REQUEST-----
python/ray/serve/tests/test_certs/server.key
+28
View File
@@ -0,0 +1,28 @@
-----BEGIN PRIVATE KEY-----
MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQCmF3CIq00eQB2+
VU+soozkfJr3OgF0ipJiCru59r2idya7e3epj8/0nF5S2MavpoLeGr1/1aMxEtAE
TDXG6pRznGoYH8sjfv9NqvwIpsG2n/xcIkuaskHzF8xYO9JMnapQ5VLMhOPanRRx
L8ceKLLS503siKUuwEteKiVxnoOTRPT5vl0rNB1QT2QWjOlQiZfPz46INKqn36Lx
qP6jfIFcZ8nYOGsk2fZ5fjTTIrxSf3+TFF5iYrhWAKGZ8DIgstkAFPU5nQOZgjhj
Wab/ykknapFYe1+iTRgeji4IpArDm1sQCaqu1mYIh6HNlpV7jflNki7gNCS2WWa/
rc6cNQx9AgMBAAECggEAFj7SHLaiJ+i7KHCcBj7ok1Bjyl8OLizCebfUTY5QTH/x
mRoVUd7oIcFbxMmHUE6t/STPDV3GHgmAq5gFeonqrigHRwnjFvL91h+OOB5q7ZSJ
+VEX7TVDg1VEUkEDjq1t+qhsVDuBmm3VfL9tx4qjQNTSvq536UYUvMefp5MX2P54
/7IDM9osP5VgeFIUx/d7QYymhgmVaSv+xcxxlZCwT3ib/wW7eU964FjkuRG8eein
zlyOwRufmg+eEvOUHN/4Fth0AUUirCMpflgRdcQtKs77FARiG8LybMGyDDsE7YBt
5f/UBZea2TQG9q4aGNUIHA869CCNKg2R27AtBpTtBQKBgQDd95GDIZMlEmR3GzpJ
6rqRHtfXj7BHUlzew+RCI1KhWkjRZqv2bavmeqRLpRdKd36aC+l+QallSW/MME+7
JSgRMqqdQK2myLJnZOIcONjMlOn9xzEQGYUsKL4IiPkdP0lWdzJ6iqAHm/Xq7GxE
BJF5XkYD1NP2+y3dlZYNrmUGHwKBgQC/jrOCV7Y34IriUPHSQA1JaPZQDBBxwiNo
ifPYkRc5C0zwskiELnJGF34Y/fK06n73JyBh6WqMdu7+V/QKNCEgcKU45n+pnlAL
vx+xflfMknWEOhLdT31ca0kvxtGEomOD1MNV+b1cRYBlL/oMC2IpIKd0N/HFa3Nc
pDmLcBWB4wKBgAIHXD4dlXG2TFLGXe8FBTWEWaavuoW8W/rxQWnVVtEAuT+ot5Om
BvcxUcUbOi5FD1QrHbQ4t2qklDAClQf52/bkRqjvSWcH2JGXW3W0k06zYbwfEPS7
tvrjWHFNhzFcPbhbmIuELthC9alzBb5NaGL6mJs6W8GbJB0tW9S+LlAzAoGBAIlB
h/B6Rs+s7fcSBuQfDyYttmhO7K2GbPan+niQJfKy3TOOm5VS7oC4rprbw7/MUqNn
frWJmdYCFmdawDtbdO0Yqdqmlo0EKdjw3pXAsMqdmuTe88tt/KZvHWbFcDU4YlQA
7OI662slRcW7ZdChi3lqs3H78BoETwnvhmgaLN7/AoGBAIVtEVcieOsasQ3Cje4L
mZxo9WFwtX4llH/CTZZeyek6VZBEWP8b3i1uh0uOzeiR7nDiwGEbHfXdvIvWrZqf
IC9Lo1D24uzE14XcKypFsYL5GAwtNhTAuP52tfV9V7DlS2QmxQt6hzx0/MhtdM3X
1XCsMrmi/WleIy611H2j0gUj
-----END PRIVATE KEY-----
python/ray/serve/tests/test_cli.py
+1019
View File
File diff suppressed because it is too large Load Diff
python/ray/serve/tests/test_cli_2.py
+377
View File
@@ -0,0 +1,377 @@
import json
import os
import re
import signal
import subprocess
import sys
from typing import Dict, Optional, Pattern
import httpx
import pytest
import yaml
from ray import serve
from ray._common.test_utils import wait_for_condition
from ray.serve._private.constants import (
RAY_SERVE_ENABLE_HA_PROXY,
SERVE_DEFAULT_APP_NAME,
)
from ray.serve._private.test_utils import get_application_url
from ray.util.state import list_actors
CONNECTION_ERROR_MSG = "connection error"
def ping_endpoint(app_name: str = SERVE_DEFAULT_APP_NAME, params: str = ""):
try:
url = get_application_url("HTTP", app_name=app_name)
return httpx.get(f"{url}/{params}").text
except httpx.HTTPError:
return CONNECTION_ERROR_MSG
def check_app_status(app_name: str, expected_status: str):
status_response = subprocess.check_output(["serve", "status"])
status = yaml.safe_load(status_response)["applications"]
assert status[app_name]["status"] == expected_status
return True
def check_app_running(app_name: str):
return check_app_status(app_name, "RUNNING")
def check_http_response(expected_text: str, json: Optional[Dict] = None):
url = get_application_url("HTTP")
resp = httpx.post(url, json=json)
assert resp.text == expected_text
return True
def test_start_shutdown(ray_start_stop):
subprocess.check_output(["serve", "start"])
# deploy a simple app
import_path = "ray.serve.tests.test_config_files.arg_builders.build_echo_app"
deploy_response = subprocess.check_output(["serve", "deploy", import_path])
assert b"Sent deploy request successfully." in deploy_response
wait_for_condition(
check_http_response,
expected_text="DEFAULT",
timeout=15,
)
ret = subprocess.check_output(["serve", "shutdown", "-y"])
assert b"Sent shutdown request; applications will be deleted asynchronously" in ret
def check_no_apps():
status = subprocess.check_output(["serve", "status"])
return b"applications: {}" in status
wait_for_condition(check_no_apps, timeout=15)
# Test shutdown when no Serve instance is running
ret = subprocess.check_output(["serve", "shutdown", "-y"], stderr=subprocess.STDOUT)
assert b"No Serve instance found running" in ret
def test_start_shutdown_without_serve_running(ray_start_stop):
# Test shutdown when no Serve instance is running
ret = subprocess.check_output(["serve", "shutdown", "-y"], stderr=subprocess.STDOUT)
assert b"No Serve instance found running" in ret
# def test_start_shutdown_without_ray_running():
# # Test shutdown when Ray is not running
# ret = subprocess.check_output(["serve", "shutdown", "-y"], stderr=subprocess.STDOUT)
# assert b"Unable to shutdown Serve on the cluster" in ret
@pytest.mark.skipif(sys.platform == "win32", reason="File path incorrect on Windows.")
def test_shutdown(ray_start_stop):
"""Deploys a config file and shuts down the Serve application."""
# Check that `serve shutdown` works even if no Serve app is running
subprocess.check_output(["serve", "shutdown", "-y"])
def num_live_deployments():
status_response = subprocess.check_output(["serve", "status"])
serve_status = yaml.safe_load(status_response)["applications"][
SERVE_DEFAULT_APP_NAME
]
return len(serve_status["deployments"])
config_file_name = os.path.join(
os.path.dirname(__file__), "test_config_files", "basic_graph.yaml"
)
# Check idempotence
num_iterations = 2
for iteration in range(1, num_iterations + 1):
print(f"*** Starting Iteration {iteration}/{num_iterations} ***\n")
print("Deploying config.")
subprocess.check_output(["serve", "deploy", config_file_name])
wait_for_condition(lambda: num_live_deployments() == 2, timeout=15)
print("Deployment successful. Deployments are live.")
# `serve config` and `serve status` should print non-empty schemas
config_response = subprocess.check_output(["serve", "config"])
yaml.safe_load(config_response)
status_response = subprocess.check_output(["serve", "status"])
status = yaml.safe_load(status_response)
assert len(status["applications"])
print("`serve config` and `serve status` print non-empty responses.\n")
print("Deleting Serve app.")
subprocess.check_output(["serve", "shutdown", "-y"])
# `serve config` and `serve status` should print messages indicating
# nothing is deployed
def serve_config_empty_warning():
config_response = subprocess.check_output(["serve", "config"]).decode(
"utf-8"
)
return config_response == "No configuration was found.\n"
def serve_status_empty():
status_response = subprocess.check_output(["serve", "status"])
status = yaml.safe_load(status_response)
return len(status["applications"]) == 0
wait_for_condition(serve_config_empty_warning)
wait_for_condition(serve_status_empty)
print("`serve config` and `serve status` print empty responses.\n")
@pytest.mark.skipif(sys.platform == "win32", reason="File path incorrect on Windows.")
def test_deploy_with_http_options(ray_start_stop):
"""Deploys config with host and port options specified"""
f1 = os.path.join(
os.path.dirname(__file__), "test_config_files", "basic_graph_http.yaml"
)
success_message_fragment = b"Sent deploy request successfully."
with open(f1, "r") as config_file:
config = yaml.safe_load(config_file)
deploy_response = subprocess.check_output(["serve", "deploy", f1])
assert success_message_fragment in deploy_response
wait_for_condition(
lambda: httpx.post("http://localhost:8005/", json=None).text
== "wonderful world",
timeout=15,
)
# Config should contain matching host and port options
info_response = subprocess.check_output(["serve", "config"])
info = yaml.safe_load(info_response)
# TODO(zcin): the assertion should just be `info == config` here but the output
# formatting removes a lot of info.
assert info == config["applications"][0]
@pytest.mark.skipif(sys.platform == "win32", reason="File path incorrect on Windows.")
@pytest.mark.parametrize("use_command", [True, False])
def test_idempotence_after_controller_death(ray_start_stop, use_command: bool):
"""Check that CLI is idempotent even if controller dies."""
config_file_name = os.path.join(
os.path.dirname(__file__), "test_config_files", "basic_graph.yaml"
)
success_message_fragment = b"Sent deploy request successfully."
deploy_response = subprocess.check_output(["serve", "deploy", config_file_name])
assert success_message_fragment in deploy_response
expected_num_actors = 4
if RAY_SERVE_ENABLE_HA_PROXY:
# fallback proxy
expected_num_actors += 1
serve.start()
wait_for_condition(
lambda: len(list_actors(filters=[("state", "=", "ALIVE")]))
== expected_num_actors,
timeout=15,
)
# Kill controller
if use_command:
subprocess.check_output(["serve", "shutdown", "-y"])
else:
serve.shutdown()
status_response = subprocess.check_output(["serve", "status"])
status_info = yaml.safe_load(status_response)
assert len(status_info["applications"]) == 0
deploy_response = subprocess.check_output(["serve", "deploy", config_file_name])
assert success_message_fragment in deploy_response
# Restore testing controller
serve.start()
wait_for_condition(
lambda: len(list_actors(filters=[("state", "=", "ALIVE")]))
== expected_num_actors,
timeout=15,
)
@pytest.mark.skipif(sys.platform == "win32", reason="File path incorrect on Windows.")
def test_control_c_shutdown_serve_components(ray_start_stop):
"""Test ctrl+c after `serve run` shuts down serve components."""
p = subprocess.Popen(["serve", "run", "ray.serve.tests.test_cli_3.echo_app"])
# Make sure Serve components are up and running
wait_for_condition(check_app_running, app_name=SERVE_DEFAULT_APP_NAME)
assert httpx.get("http://localhost:8000/-/healthz").text == "success"
assert json.loads(httpx.get("http://localhost:8000/-/routes").text) == {
"/": "default"
}
assert httpx.get("http://localhost:8000/").text == "hello"
# Send ctrl+c to shutdown Serve components
p.send_signal(signal.SIGINT)
p.wait()
# Make sure Serve components are shutdown
status_response = subprocess.check_output(["serve", "status"])
status = yaml.safe_load(status_response)
assert status == {"applications": {}, "proxies": {}, "target_capacity": None}
@pytest.mark.skipif(sys.platform == "win32", reason="File path incorrect on Windows.")
@pytest.mark.parametrize(
"ray_start_stop_in_specific_directory",
[
os.path.join(os.path.dirname(__file__), "test_config_files"),
],
indirect=True,
)
def test_deploy_with_access_to_current_directory(ray_start_stop_in_specific_directory):
"""Test serve deploy using modules in the current directory succeeds.
There was an issue where dashboard client doesn't add the current directory to
the sys.path and failed to deploy a Serve app defined in the directory. This
test ensures that files in the current directory can be accessed and deployed.
See: https://github.com/ray-project/ray/issues/43889
"""
# Deploy Serve application with a config in the current directory.
subprocess.check_output(["serve", "deploy", "use_current_working_directory.yaml"])
# Ensure serve deploy eventually succeeds.
def check_deploy_successfully():
status_response = subprocess.check_output(["serve", "status"])
assert b"RUNNING" in status_response
return True
wait_for_condition(check_deploy_successfully, timeout=5)
@pytest.mark.skipif(sys.platform == "win32", reason="File path incorrect on Windows.")
class TestRayReinitialization:
@pytest.fixture
def import_file_name(self) -> str:
return "ray.serve.tests.test_config_files.ray_already_initialized:app"
@pytest.fixture
def pattern(self) -> Pattern:
return re.compile(r"Connecting to existing Ray cluster at address: (.*)\.\.\.")
@pytest.fixture
def ansi_escape(self) -> Pattern:
return re.compile(r"\x1B(?:[@-Z\\-_]|\[[0-?]*[ -/]*[@-~])")
def test_run_without_address(self, import_file_name, ray_start_stop):
"""Test serve run with ray already initialized and run without address argument.
When the imported file already initialized a ray instance and serve doesn't run
with address argument, then serve does not reinitialize another ray instance and
cause error.
"""
p = subprocess.Popen(["serve", "run", import_file_name])
wait_for_condition(lambda: ping_endpoint() == "foobar", timeout=10)
p.send_signal(signal.SIGINT)
p.wait()
def test_run_with_address_same_address(self, import_file_name, ray_start_stop):
"""Test serve run with ray already initialized and run with address argument
that has the same address as existing ray instance.
When the imported file already initialized a ray instance and serve runs with
address argument same as the ray instance, then serve does not reinitialize
another ray instance and cause error.
"""
p = subprocess.Popen(
["serve", "run", "--address=127.0.0.1:6379", import_file_name]
)
wait_for_condition(lambda: ping_endpoint() == "foobar", timeout=10)
p.send_signal(signal.SIGINT)
p.wait()
def test_run_with_address_different_address(
self, import_file_name, pattern, ansi_escape, ray_start_stop
):
"""Test serve run with ray already initialized and run with address argument
that has the different address as existing ray instance.
When the imported file already initialized a ray instance and serve runs with
address argument different as the ray instance, then serve does not reinitialize
another ray instance and cause error and logs warning to the user.
"""
p = subprocess.Popen(
["serve", "run", "--address=ray://123.45.67.89:50005", import_file_name],
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
)
wait_for_condition(lambda: ping_endpoint() == "foobar", timeout=10)
p.send_signal(signal.SIGINT)
p.wait()
process_output, _ = p.communicate()
logs = process_output.decode("utf-8").strip()
ray_address = ansi_escape.sub("", pattern.search(logs).group(1))
expected_warning_message = (
"An address was passed to `serve run` but the imported module also "
f"connected to Ray at a different address: '{ray_address}'. You do not "
"need to call `ray.init` in your code when using `serve run`."
)
assert expected_warning_message in logs
def test_run_with_auto_address(
self, import_file_name, pattern, ansi_escape, ray_start_stop
):
"""Test serve run with ray already initialized and run with "auto" address
argument.
When the imported file already initialized a ray instance and serve runs with
address argument same as the ray instance, then serve does not reinitialize
another ray instance and cause error.
"""
p = subprocess.Popen(
["serve", "run", "--address=auto", import_file_name],
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
)
wait_for_condition(lambda: ping_endpoint() == "foobar", timeout=10)
p.send_signal(signal.SIGINT)
p.wait()
process_output, _ = p.communicate()
logs = process_output.decode("utf-8").strip()
ray_address = ansi_escape.sub("", pattern.search(logs).group(1))
expected_warning_message = (
"An address was passed to `serve run` but the imported module also "
f"connected to Ray at a different address: '{ray_address}'. You do not "
"need to call `ray.init` in your code when using `serve run`."
)
assert expected_warning_message not in logs
if __name__ == "__main__":
sys.exit(pytest.main(["-v", "-s", __file__]))
python/ray/serve/tests/test_cli_3.py
+613
View File
@@ -0,0 +1,613 @@
import json
import os
import signal
import subprocess
import sys
import time
from typing import Union
import httpx
import pytest
import yaml
from pydantic import BaseModel
from ray import serve
from ray._common.test_utils import wait_for_condition
from ray.serve._private.constants import SERVE_DEFAULT_APP_NAME
from ray.serve.handle import DeploymentHandle
from ray.serve.tests.common.remote_uris import (
TEST_DAG_PINNED_URI,
TEST_DEPLOY_GROUP_PINNED_URI,
)
CONNECTION_ERROR_MSG = "connection error"
def ping_endpoint(endpoint: str, params: str = ""):
endpoint = endpoint.lstrip("/")
try:
return httpx.get(f"http://localhost:8000/{endpoint}{params}").text
except httpx.HTTPError:
return CONNECTION_ERROR_MSG
def check_app_status(app_name: str, expected_status: str):
status_response = subprocess.check_output(["serve", "status"])
status = yaml.safe_load(status_response)["applications"]
assert status[app_name]["status"] == expected_status
return True
def check_app_running(app_name: str):
return check_app_status(app_name, "RUNNING")
@serve.deployment
def parrot(request):
return request.query_params["sound"]
parrot_node = parrot.bind()
@serve.deployment
class MetalDetector:
def __call__(self, *args):
return os.environ.get("buried_item", "no dice")
metal_detector_node = MetalDetector.bind()
@serve.deployment
class ConstructorFailure:
def __init__(self):
raise RuntimeError("Intentionally failing.")
constructor_failure_node = ConstructorFailure.bind()
@serve.deployment
class Macaw:
def __init__(self, color, name="Mulligan", surname=None):
self.color = color
self.name = name
self.surname = surname
def __call__(self):
if self.surname is not None:
return f"{self.name} {self.surname} is {self.color}!"
else:
return f"{self.name} is {self.color}!"
molly_macaw = Macaw.bind("green", name="Molly")
@serve.deployment
def global_f(*args):
return "wonderful world"
@serve.deployment
class NoArgDriver:
def __init__(self, h: DeploymentHandle):
self._h = h
async def __call__(self):
return await self._h.remote()
TestBuildFNode = global_f.bind()
TestBuildDagNode = NoArgDriver.bind(TestBuildFNode)
TestApp1Node = global_f.options(name="app1").bind()
TestApp2Node = NoArgDriver.options(name="app2").bind(global_f.bind())
@serve.deployment
class Echo:
def __init__(self, message: str):
print("Echo message:", message)
self._message = message
def __call__(self, *args):
return self._message
echo_app = Echo.bind("hello")
def build_echo_app(args):
return Echo.bind(args.get("message", "DEFAULT"))
class TypedArgs(BaseModel):
message: str = "DEFAULT"
def build_echo_app_typed(args: TypedArgs):
return Echo.bind(args.message)
k8sFNode = global_f.options(
num_replicas=2, ray_actor_options={"num_cpus": 2, "num_gpus": 1}
).bind()
class TestRun:
@pytest.mark.skipif(
sys.platform == "win32", reason="File path incorrect on Windows."
)
@pytest.mark.parametrize(
"proxy_location,expected",
[
(
None,
"EveryNode",
), # default ProxyLocation `EveryNode` is used as http_options.location is not specified
("EveryNode", "EveryNode"),
("HeadOnly", "HeadOnly"),
("Disabled", "Disabled"),
],
)
def test_proxy_location(self, ray_start_stop, tmp_path, proxy_location, expected):
# when the `serve run` cli command is executed
# without serve already running (for the first time)
# `proxy_location` should be set from the config file if specified
def is_proxy_location_correct(expected_proxy_location: str) -> bool:
try:
response = httpx.get(
"http://localhost:8265/api/serve/applications/"
).text
response_json = json.loads(response)
print("response_json")
print(response_json)
return response_json["proxy_location"] == expected_proxy_location
except httpx.HTTPError:
return False
def arithmetic_config(with_proxy_location: Union[str, None]) -> str:
config_file_name = os.path.join(
os.path.dirname(__file__), "test_config_files", "arithmetic.yaml"
)
with open(config_file_name, "r") as config_file:
arithmetic_config_dict = yaml.safe_load(config_file)
config_path = tmp_path / "config.yaml"
if with_proxy_location:
arithmetic_config_dict["proxy_location"] = with_proxy_location
with open(config_path, "w") as f:
yaml.dump(arithmetic_config_dict, f)
return str(config_path)
config_path = arithmetic_config(with_proxy_location=proxy_location)
p = subprocess.Popen(["serve", "run", config_path])
wait_for_condition(
lambda: is_proxy_location_correct(expected_proxy_location=expected),
timeout=10,
)
p.send_signal(signal.SIGINT)
p.wait()
@pytest.mark.parametrize("number_of_kill_signals", (1, 2))
@pytest.mark.skipif(
sys.platform == "win32", reason="File path incorrect on Windows."
)
def test_run_application(self, ray_start_stop, number_of_kill_signals):
"""Deploys valid config file and import path via `serve run`."""
# Deploy via config file
config_file_name = os.path.join(
os.path.dirname(__file__), "test_config_files", "arithmetic.yaml"
)
print('Running config file "arithmetic.yaml".')
p = subprocess.Popen(["serve", "run", "--address=auto", config_file_name])
wait_for_condition(
lambda: httpx.post("http://localhost:8000/", json=["ADD", 0]).json() == 1,
timeout=15,
)
wait_for_condition(
lambda: httpx.post("http://localhost:8000/", json=["SUB", 5]).json() == 3,
timeout=15,
)
print(
"Run successful! Deployments are live and reachable over HTTP. Killing run."
)
for _ in range(number_of_kill_signals):
p.send_signal(signal.SIGINT)
# Mimic realistic human Ctrl-C timing. Without a gap, two
# back-to-back SIGINTs can land before the KeyboardInterrupt
# handler in `serve run` has a chance to run, aborting the
# process before graceful shutdown begins.
time.sleep(0.1)
p.wait()
with pytest.raises(httpx.HTTPError):
httpx.post("http://localhost:8000/", json=["ADD", 0]).json()
print("Kill successful! Deployments are not reachable over HTTP.")
print('Running node at import path "ray.serve.tests.test_cli_3.parrot_node".')
# Deploy via import path
p = subprocess.Popen(
["serve", "run", "--address=auto", "ray.serve.tests.test_cli_3.parrot_node"]
)
wait_for_condition(
lambda: ping_endpoint("/", params="?sound=squawk") == "squawk"
)
print(
"Run successful! Deployment is live and reachable over HTTP. Killing run."
)
p.send_signal(signal.SIGINT) # Equivalent to ctrl-C
p.wait()
assert ping_endpoint("/", params="?sound=squawk") == CONNECTION_ERROR_MSG
print("Kill successful! Deployment is not reachable over HTTP.")
@pytest.mark.skipif(
sys.platform == "win32", reason="File path incorrect on Windows."
)
def test_run_multi_app(self, ray_start_stop):
"""Deploys valid multi-app config file via `serve run`."""
# Deploy via config file
config_file_name = os.path.join(
os.path.dirname(__file__), "test_config_files", "pizza_world.yaml"
)
print('Running config file "pizza_world.yaml".')
p = subprocess.Popen(["serve", "run", "--address=auto", config_file_name])
wait_for_condition(
lambda: httpx.post("http://localhost:8000/app1").text == "wonderful world",
timeout=15,
)
print('Application "app1" is reachable over HTTP.')
wait_for_condition(
lambda: httpx.post("http://localhost:8000/app2", json=["ADD", 2]).text
== "12 pizzas please!",
timeout=15,
)
wait_for_condition(
lambda: httpx.post("http://localhost:8000/app2", json=["MUL", 2]).text
== "20 pizzas please!",
timeout=15,
)
print(
"Run successful! Deployments are live and reachable over HTTP. Killing run."
)
p.send_signal(signal.SIGINT) # Equivalent to ctrl-C
p.wait()
with pytest.raises(httpx.HTTPError):
_ = httpx.post("http://localhost:8000/app1").text
with pytest.raises(httpx.HTTPError):
_ = httpx.post("http://localhost:8000/app2", json=["ADD", 0]).text
print("Kill successful! Deployments are not reachable over HTTP.")
@pytest.mark.skipif(
sys.platform == "win32", reason="File path incorrect on Windows."
)
def test_run_deployment_node(self, ray_start_stop):
"""Test `serve run` with bound args and kwargs."""
# Deploy via import path
p = subprocess.Popen(
[
"serve",
"run",
"--address=auto",
"ray.serve.tests.test_cli_3.molly_macaw",
]
)
wait_for_condition(lambda: ping_endpoint("/") == "Molly is green!", timeout=10)
p.send_signal(signal.SIGINT)
p.wait()
assert ping_endpoint("/") == CONNECTION_ERROR_MSG
@pytest.mark.skipif(
sys.platform == "win32", reason="File path incorrect on Windows."
)
@pytest.mark.parametrize(
"import_path",
[
"ray.serve.tests.test_cli_3.build_echo_app",
"ray.serve.tests.test_cli_3.build_echo_app_typed",
],
)
def test_run_builder_with_args(self, ray_start_stop, import_path: str):
"""Test `serve run` with args passed into a builder function.
Tests both the untyped and typed args cases.
"""
# First deploy without any arguments, should get default response.
p = subprocess.Popen(
[
"serve",
"run",
"--address=auto",
import_path,
]
)
wait_for_condition(lambda: ping_endpoint("") == "DEFAULT", timeout=10)
p.send_signal(signal.SIGINT)
p.wait()
assert ping_endpoint("/") == CONNECTION_ERROR_MSG
# Now deploy passing a message as an argument, should get passed message.
p = subprocess.Popen(
[
"serve",
"run",
"--address=auto",
import_path,
"message=hello world",
]
)
wait_for_condition(lambda: ping_endpoint("") == "hello world", timeout=10)
p.send_signal(signal.SIGINT)
p.wait()
assert ping_endpoint("/") == CONNECTION_ERROR_MSG
@pytest.mark.skipif(
sys.platform == "win32", reason="File path incorrect on Windows."
)
def test_run_runtime_env(self, ray_start_stop):
"""Test `serve run` with runtime_env passed in."""
# With import path
p = subprocess.Popen(
[
"serve",
"run",
"--address=auto",
"ray.serve.tests.test_cli_3.metal_detector_node",
"--runtime-env-json",
('{"env_vars": {"buried_item": "lucky coin"} }'),
]
)
wait_for_condition(
lambda: ping_endpoint("MetalDetector") == "lucky coin", timeout=10
)
p.send_signal(signal.SIGINT)
p.wait()
# With config
p = subprocess.Popen(
[
"serve",
"run",
"--address=auto",
os.path.join(
os.path.dirname(__file__),
"test_config_files",
"missing_runtime_env.yaml",
),
"--runtime-env-json",
json.dumps(
{
"py_modules": [TEST_DEPLOY_GROUP_PINNED_URI],
"working_dir": "http://nonexistentlink-q490123950ni34t",
}
),
"--working-dir",
TEST_DAG_PINNED_URI,
]
)
wait_for_condition(lambda: ping_endpoint("") == "wonderful world", timeout=15)
p.send_signal(signal.SIGINT)
p.wait()
@pytest.mark.skipif(
sys.platform == "win32", reason="File path incorrect on Windows."
)
@pytest.mark.parametrize("config_file", ["basic_graph.yaml", "basic_multi.yaml"])
def test_run_config_port1(self, ray_start_stop, config_file):
"""Test that `serve run` defaults to port 8000."""
config_file_name = os.path.join(
os.path.dirname(__file__), "test_config_files", config_file
)
p = subprocess.Popen(["serve", "run", config_file_name])
wait_for_condition(
lambda: httpx.post("http://localhost:8000/").text == "wonderful world",
timeout=15,
)
p.send_signal(signal.SIGINT)
p.wait()
@pytest.mark.skipif(
sys.platform == "win32", reason="File path incorrect on Windows."
)
@pytest.mark.parametrize(
"config_file", ["basic_graph_http.yaml", "basic_multi_http.yaml"]
)
def test_run_config_port2(self, ray_start_stop, config_file):
"""If config file specifies a port, the default port value should not be used."""
config_file_name = os.path.join(
os.path.dirname(__file__), "test_config_files", config_file
)
p = subprocess.Popen(["serve", "run", config_file_name])
wait_for_condition(
lambda: httpx.post("http://localhost:8005/").text == "wonderful world",
timeout=15,
)
p.send_signal(signal.SIGINT)
p.wait()
@pytest.mark.skipif(
sys.platform == "win32", reason="File path incorrect on Windows."
)
def test_run_teardown(self, ray_start_stop):
"""Consecutive serve runs should tear down controller so logs can always be seen."""
logs = subprocess.check_output(
["serve", "run", "ray.serve.tests.test_cli_3.constructor_failure_node"],
stderr=subprocess.STDOUT,
timeout=30,
).decode()
assert "Intentionally failing." in logs
logs = subprocess.check_output(
["serve", "run", "ray.serve.tests.test_cli_3.constructor_failure_node"],
stderr=subprocess.STDOUT,
timeout=30,
).decode()
assert "Intentionally failing." in logs
@pytest.mark.skipif(
sys.platform == "win32", reason="File path incorrect on Windows."
)
def test_run_route_prefix_and_name_default(self, ray_start_stop):
"""Test `serve run` without route_prefix and name options."""
p = subprocess.Popen(
[
"serve",
"run",
"--address=auto",
"ray.serve.tests.test_cli_3.echo_app",
]
)
wait_for_condition(check_app_running, app_name=SERVE_DEFAULT_APP_NAME)
assert ping_endpoint("/") == "hello"
p.send_signal(signal.SIGINT)
p.wait()
@pytest.mark.skipif(
sys.platform == "win32", reason="File path incorrect on Windows."
)
def test_run_route_prefix_and_name_override(self, ray_start_stop):
"""Test `serve run` with route prefix option."""
p = subprocess.Popen(
[
"serve",
"run",
"--address=auto",
"--route-prefix=/hello",
"--name=hello_app",
"ray.serve.tests.test_cli_3.echo_app",
],
)
wait_for_condition(check_app_running, app_name="hello_app")
assert "Path '/' not found" in ping_endpoint("/")
assert ping_endpoint("/hello") == "hello"
p.send_signal(signal.SIGINT)
p.wait()
@pytest.mark.skipif(
sys.platform == "win32", reason="File path incorrect on Windows."
)
def test_run_config_request_timeout(self, ray_start_stop):
"""Test running serve with request timeout in http_options.
The config file has 0.1s as the `request_timeout_s` in the `http_options`. First
case checks that when the query runs longer than the 0.1s, the deployment returns a
task failed message. The second case checks that when the query takes less than
0.1s, the deployment returns a success message.
"""
config_file_name = os.path.join(
os.path.dirname(__file__),
"test_config_files",
"http_option_request_timeout_s.yaml",
)
p = subprocess.Popen(["serve", "run", config_file_name])
# Ensure the http request is killed and failed when the deployment runs longer than
# the 0.1 request_timeout_s set in in the config yaml
wait_for_condition(
lambda: httpx.get("http://localhost:8000/app1?sleep_s=0.11").status_code
== 408,
)
# Ensure the http request returned the correct response when the deployment runs
# shorter than the 0.1 request_timeout_s set up in the config yaml
wait_for_condition(
lambda: httpx.get("http://localhost:8000/app1?sleep_s=0.09").text
== "Task Succeeded!",
)
p.send_signal(signal.SIGINT)
p.wait()
@pytest.mark.skipif(
sys.platform == "win32", reason="File path incorrect on Windows."
)
def test_run_reload_basic(self, ray_start_stop, tmp_path):
"""Test `serve run` with reload."""
code_template = """
from ray import serve
@serve.deployment
class MessageDeployment:
def __init__(self, msg):
{invalid_suffix}
self.msg = msg
def __call__(self):
return self.msg
msg_app = MessageDeployment.bind("Hello {message}!")
"""
def write_file(message: str, invalid_suffix: str = ""):
with open(os.path.join(tmp_path, "reload_serve.py"), "w") as f:
code = code_template.format(
invalid_suffix=invalid_suffix, message=message
)
print(f"Writing updated code:\n{code}")
f.write(code)
f.flush()
write_file("World")
p = subprocess.Popen(
[
"serve",
"run",
"--address=auto",
"--app-dir",
tmp_path,
"--reload",
"reload_serve:msg_app",
]
)
wait_for_condition(lambda: ping_endpoint("") == "Hello World!", timeout=10)
# Sleep to ensure the `serve run` command is in the file watching loop when we
# write the change, else it won't be picked up.
time.sleep(5)
# Write the file: an update should be auto-triggered.
write_file("Updated")
wait_for_condition(lambda: ping_endpoint("") == "Hello Updated!", timeout=10)
# Ensure a bad change doesn't shut down serve and serve reports deploy failed.
write_file(message="update1", invalid_suffix="foobar")
wait_for_condition(
condition_predictor=check_app_status,
app_name="default",
expected_status="DEPLOY_FAILED",
)
# Ensure the following reload happens as expected.
write_file("Updated2")
wait_for_condition(lambda: ping_endpoint("") == "Hello Updated2!", timeout=10)
p.send_signal(signal.SIGINT)
p.wait()
assert ping_endpoint("") == CONNECTION_ERROR_MSG
if __name__ == "__main__":
sys.exit(pytest.main(["-v", "-s", __file__]))
python/ray/serve/tests/test_cli_4.py
+190
View File
@@ -0,0 +1,190 @@
import os
import subprocess
import sys
from tempfile import NamedTemporaryFile
import grpc
import httpx
import pytest
from ray._common.test_utils import wait_for_condition
from ray.serve._private.constants import (
RAY_SERVE_ENABLE_DIRECT_INGRESS,
)
from ray.serve._private.test_utils import (
get_application_url,
ping_fruit_stand,
ping_grpc_another_method,
ping_grpc_call_method,
ping_grpc_healthz,
ping_grpc_list_applications,
ping_grpc_model_multiplexing,
ping_grpc_streaming,
)
from ray.serve.generated import serve_pb2, serve_pb2_grpc
from ray.serve.tests.test_cli_2 import check_app_running, ping_endpoint
@pytest.mark.skipif(sys.platform == "win32", reason="File path incorrect on Windows.")
def test_build_multi_app(ray_start_stop):
with NamedTemporaryFile(mode="w+", suffix=".yaml") as tmp:
print('Building nodes "TestApp1Node" and "TestApp2Node".')
# Build an app
grpc_servicer_func_root = "ray.serve.generated.serve_pb2_grpc"
subprocess.check_output(
[
"serve",
"build",
"ray.serve.tests.test_cli_3.TestApp1Node",
"ray.serve.tests.test_cli_3.TestApp2Node",
"ray.serve.tests.test_config_files.grpc_deployment.g",
"--grpc-servicer-functions",
f"{grpc_servicer_func_root}.add_UserDefinedServiceServicer_to_server",
"-o",
tmp.name,
]
)
print("Build succeeded! Deploying node.")
subprocess.check_output(["serve", "deploy", tmp.name])
print("Deploy succeeded!")
wait_for_condition(
lambda: ping_endpoint("app1") == "wonderful world", timeout=15
)
print("App 1 is live and reachable over HTTP.")
wait_for_condition(
lambda: ping_endpoint("app2") == "wonderful world", timeout=15
)
print("App 2 is live and reachable over HTTP.")
app_name = "app3"
channel = grpc.insecure_channel(get_application_url("gRPC", app_name=app_name))
stub = serve_pb2_grpc.UserDefinedServiceStub(channel)
request = serve_pb2.UserDefinedMessage(name="foo", num=30, foo="bar")
metadata = (("application", app_name),)
response = stub.__call__(request=request, metadata=metadata)
assert response.greeting == "Hello foo from bar"
print("App 3 is live and reachable over gRPC.")
print("Deleting applications.")
app_urls = [
get_application_url("HTTP", app_name=app) for app in ["app1", "app2"]
]
subprocess.check_output(["serve", "shutdown", "-y"])
def check_no_apps():
for url in app_urls:
with pytest.raises(httpx.HTTPError):
_ = httpx.get(url).text
return True
wait_for_condition(check_no_apps, timeout=15)
print("Delete succeeded! Node is no longer reachable over HTTP.")
@pytest.mark.skipif(sys.platform == "win32", reason="File path incorrect on Windows.")
def test_serving_request_through_grpc_proxy(ray_start_stop):
"""Test serving request through gRPC proxy
When Serve runs with a gRPC deployment, the app should be deployed successfully,
both ListApplications and Healthz methods returning success response, and registered
gRPC methods are routing to the correct replica and return the correct response.
"""
config_file = os.path.join(
os.path.dirname(__file__),
"test_config_files",
"deploy_grpc_app.yaml",
)
subprocess.check_output(["serve", "deploy", config_file], stderr=subprocess.STDOUT)
app1 = "app1"
app_names = [app1]
# Wait for the application to be RUNNING before sending requests.
wait_for_condition(check_app_running, app_name=app1)
channel = grpc.insecure_channel(get_application_url("gRPC", app_name=app1))
# Ensures ListApplications method succeeding.
ping_grpc_list_applications(channel, app_names)
# Ensures Healthz method succeeding.
ping_grpc_healthz(channel)
# Ensures a custom defined method is responding correctly.
ping_grpc_call_method(channel, app1)
# Ensures another custom defined method is responding correctly.
ping_grpc_another_method(channel, app1)
# TODO: gRPC streaming is not supported in direct ingress / haproxy
if not RAY_SERVE_ENABLE_DIRECT_INGRESS:
# Ensure Streaming method is responding correctly.
ping_grpc_streaming(channel, app1)
@pytest.mark.skipif(sys.platform == "win32", reason="File path incorrect on Windows.")
@pytest.mark.skipif(
RAY_SERVE_ENABLE_DIRECT_INGRESS,
reason="Model multiplexing is not supported on the ingress deployment when "
"direct ingress / HAProxy is enabled (the multiplexed model ID is not "
"propagated to the replica).",
)
def test_serving_grpc_proxy_model_multiplexing(ray_start_stop):
"""Test model multiplexing over gRPC when deployed via the CLI."""
config_file = os.path.join(
os.path.dirname(__file__),
"test_config_files",
"deploy_grpc_multiplexed_app.yaml",
)
subprocess.check_output(["serve", "deploy", config_file], stderr=subprocess.STDOUT)
app1 = "app1"
# Wait for the application to be RUNNING before sending requests.
wait_for_condition(check_app_running, app_name=app1)
channel = grpc.insecure_channel(get_application_url("gRPC", app_name=app1))
# Ensures model multiplexing is responding correctly.
ping_grpc_model_multiplexing(channel, app1)
@pytest.mark.skipif(sys.platform == "win32", reason="File path incorrect on Windows.")
def test_grpc_proxy_model_composition(ray_start_stop):
"""Test serving request through gRPC proxy
When Serve runs with a gRPC deployment, the app should be deployed successfully,
both ListApplications and Healthz methods returning success response, and model
composition should work correctly.
"""
config_file = os.path.join(
os.path.dirname(__file__),
"test_config_files",
"deploy_grpc_model_composition.yaml",
)
subprocess.check_output(["serve", "deploy", config_file], stderr=subprocess.STDOUT)
app = "app1"
app_names = [app]
# Wait for the application to be RUNNING before sending requests.
wait_for_condition(check_app_running, app_name=app)
channel = grpc.insecure_channel(get_application_url("gRPC", app_name=app))
# Ensures ListApplications method succeeding.
ping_grpc_list_applications(channel, app_names)
# Ensures Healthz method succeeding.
ping_grpc_healthz(channel)
# Ensure model composition is responding correctly.
ping_fruit_stand(channel, app)
if __name__ == "__main__":
sys.exit(pytest.main(["-v", "-s", __file__]))
python/ray/serve/tests/test_cluster.py
+723
View File
@@ -0,0 +1,723 @@
import os
import sys
import time
from collections import defaultdict
import httpx
import pytest
import ray
from ray import serve
from ray._common.test_utils import SignalActor, wait_for_condition
from ray.cluster_utils import Cluster
from ray.exceptions import RayActorError
from ray.serve._private.common import DeploymentID, DeploymentStatus, ReplicaState
from ray.serve._private.constants import (
RAY_SERVE_USE_PACK_SCHEDULING_STRATEGY,
SERVE_DEFAULT_APP_NAME,
SERVE_NAMESPACE,
)
from ray.serve._private.deployment_state import ReplicaStartupStatus
from ray.serve._private.test_utils import (
check_deployment_status,
expected_proxy_actors,
skip_if_haproxy,
)
from ray.serve._private.utils import calculate_remaining_timeout, get_head_node_id
from ray.serve.config import GangSchedulingConfig
from ray.serve.context import _get_global_client
from ray.serve.handle import DeploymentHandle
from ray.serve.schema import ServeDeploySchema
from ray.util.state import list_actors
def get_pids(expected, deployment_name="D", app_name="default", timeout=30):
handle = serve.get_deployment_handle(deployment_name, app_name)
pids = set()
start = time.time()
while len(pids) < expected:
for r in [handle.remote() for _ in range(10)]:
try:
pids.add(
r.result(
timeout_s=calculate_remaining_timeout(
timeout_s=timeout,
start_time_s=start,
curr_time_s=time.time(),
)
)
)
except RayActorError:
# Handle sent request to dead actor before running replicas were updated
# This can happen because health check period = 1s
pass
if time.time() - start >= timeout:
raise TimeoutError("Timed out waiting for pids.")
return pids
@serve.deployment(health_check_period_s=1, max_ongoing_requests=1)
def pid():
time.sleep(0.1)
return os.getpid()
pid_app = pid.bind()
def test_scale_up(ray_cluster):
cluster = ray_cluster
cluster.add_node(num_cpus=1)
cluster.connect(namespace=SERVE_NAMESPACE)
# By default, Serve controller and proxy actors use 0 CPUs,
# so initially there should only be room for 1 replica.
app_config = {
"name": "default",
"import_path": "ray.serve.tests.test_cluster.pid_app",
"deployments": [{"name": "pid", "num_replicas": 1}],
}
serve.start()
client = serve.context._connect()
client.deploy_apps(ServeDeploySchema(**{"applications": [app_config]}))
client._wait_for_application_running("default")
pids1 = get_pids(1, deployment_name="pid", app_name="default")
app_config["deployments"][0]["num_replicas"] = 3
client.deploy_apps(ServeDeploySchema(**{"applications": [app_config]}))
# Check that a new replica has not started in 1.0 seconds. This
# doesn't guarantee that a new replica won't ever be started, but
# 1.0 seconds is a reasonable upper bound on replica startup time.
with pytest.raises(TimeoutError):
client._wait_for_application_running("default", timeout_s=1)
assert get_pids(1, deployment_name="pid", app_name="default") == pids1
# Add a node with another CPU, another replica should get placed.
cluster.add_node(num_cpus=1)
with pytest.raises(TimeoutError):
client._wait_for_application_running("default", timeout_s=1)
pids2 = get_pids(2, deployment_name="pid", app_name="default")
assert pids1.issubset(pids2)
# Add a node with another CPU, the final replica should get placed
# and the deploy goal should be done.
cluster.add_node(num_cpus=1)
client._wait_for_application_running("default")
pids3 = get_pids(3, deployment_name="pid", app_name="default")
assert pids2.issubset(pids3)
@pytest.mark.skipif(sys.platform == "win32", reason="Flaky on Windows.")
def test_node_failure(ray_cluster):
cluster = ray_cluster
cluster.add_node(num_cpus=3)
cluster.connect(namespace=SERVE_NAMESPACE)
# NOTE(edoakes): we need to start serve before adding the worker node to
# guarantee that the controller is placed on the head node (we should be
# able to tolerate being placed on workers, but there's currently a bug).
# We should add an explicit test for that in the future when it's fixed.
serve.start()
worker_node = cluster.add_node(num_cpus=2)
@serve.deployment(num_replicas=5, health_check_period_s=1, max_ongoing_requests=1)
def D(*args):
time.sleep(0.1)
return os.getpid()
print("Initial deploy.")
serve.run(D.bind())
pids1 = get_pids(5)
# Remove the node. There should still be three replicas running.
print("Kill node.")
cluster.remove_node(worker_node)
pids2 = get_pids(3)
assert pids2.issubset(pids1)
# Add a worker node back. One replica should get placed.
print("Add back first node.")
cluster.add_node(num_cpus=1)
pids3 = get_pids(4)
assert pids2.issubset(pids3)
# Add another worker node. One more replica should get placed.
print("Add back second node.")
cluster.add_node(num_cpus=1)
pids4 = get_pids(5)
assert pids3.issubset(pids4)
@pytest.mark.skipif(sys.platform == "win32", reason="Flaky on Windows.")
def test_replica_startup_status_transitions(ray_cluster):
cluster = ray_cluster
cluster.add_node(num_cpus=1)
cluster.connect(namespace=SERVE_NAMESPACE)
serve.start()
client = _get_global_client()
signal = SignalActor.remote()
@serve.deployment(ray_actor_options={"num_cpus": 2})
class E:
async def __init__(self):
await signal.wait.remote()
serve._run(E.bind(), _blocking=False)
def get_replicas(replica_state):
controller = client._controller
replicas = ray.get(
controller._dump_replica_states_for_testing.remote(
DeploymentID(name=E.name)
)
)
return replicas.get([replica_state])
# wait for serve to start the replica
wait_for_condition(lambda: len(get_replicas(ReplicaState.STARTING)) > 0)
# currently there are no resources to allocate the replica
def get_starting_replica():
replicas = get_replicas(ReplicaState.STARTING)
return replicas[0] if replicas else None
def is_pending_allocation():
replica = get_starting_replica()
if replica is None:
return False
return replica.check_started()[0] == ReplicaStartupStatus.PENDING_ALLOCATION
wait_for_condition(is_pending_allocation)
# add the necessary resources to allocate the replica
cluster.add_node(num_cpus=4)
wait_for_condition(lambda: (ray.cluster_resources().get("CPU", 0) >= 4))
wait_for_condition(lambda: (ray.available_resources().get("CPU", 0) >= 2))
def is_replica_pending_initialization():
replica = get_starting_replica()
if replica is None:
return False
status, _ = replica.check_started()
return status == ReplicaStartupStatus.PENDING_INITIALIZATION
wait_for_condition(is_replica_pending_initialization, timeout=25)
# send signal to complete replica initialization
ray.get(signal.send.remote())
def check_succeeded():
# After initialization succeeds, replica transitions to RUNNING state
# So check both STARTING and RUNNING states
replica = get_starting_replica()
if replica:
status, _ = replica.check_started()
if status == ReplicaStartupStatus.SUCCEEDED:
return True
# Check if replica has moved to RUNNING state (which means it succeeded)
running_replicas = get_replicas(ReplicaState.RUNNING)
if running_replicas and len(running_replicas) > 0:
return True
return False
wait_for_condition(check_succeeded)
@pytest.mark.skipif(sys.platform == "win32", reason="Flaky on Windows.")
def test_gang_replica_startup_status_transitions(ray_cluster):
cluster = ray_cluster
# Start with only 1 CPU — not enough for a gang of 2 replicas each needing 0.75 CPUs.
cluster.add_node(num_cpus=1)
cluster.connect(namespace=SERVE_NAMESPACE)
serve.start()
client = _get_global_client()
signal = SignalActor.remote()
@serve.deployment(
ray_actor_options={"num_cpus": 0.75},
num_replicas=2,
gang_scheduling_config=GangSchedulingConfig(gang_size=2),
)
class GangDeployment:
async def __init__(self):
await signal.wait.remote()
serve._run(GangDeployment.bind(), _blocking=False)
def get_replicas(replica_state):
controller = client._controller
replicas = ray.get(
controller._dump_replica_states_for_testing.remote(
DeploymentID(name="GangDeployment")
)
)
return replicas.get([replica_state])
# Wait for replicas to be created in STARTING state.
wait_for_condition(lambda: len(get_replicas(ReplicaState.STARTING)) > 0)
# With only 1 CPU available and each replica needing 0.75, replicas should
# be stuck in PENDING_ALLOCATION.
def is_pending_allocation():
replicas = get_replicas(ReplicaState.STARTING)
if not replicas:
return False
return all(
r.check_started()[0] == ReplicaStartupStatus.PENDING_ALLOCATION
for r in replicas
)
wait_for_condition(is_pending_allocation)
# Add enough resources for the gang
cluster.add_node(num_cpus=1)
wait_for_condition(lambda: ray.cluster_resources().get("CPU", 0) == 2)
# Replicas should transition to PENDING_INITIALIZATION
def is_pending_initialization():
replicas = get_replicas(ReplicaState.STARTING)
if not replicas:
return False
return all(
r.check_started()[0] == ReplicaStartupStatus.PENDING_INITIALIZATION
for r in replicas
)
wait_for_condition(is_pending_initialization, timeout=30)
# Complete initialization
ray.get(signal.send.remote())
# Replicas should transition to RUNNING
def check_running():
running_replicas = get_replicas(ReplicaState.RUNNING)
return len(running_replicas) == 2
wait_for_condition(check_running, timeout=30)
@serve.deployment
def f():
pass
f_app = f.bind()
def test_intelligent_scale_down(ray_cluster):
cluster = ray_cluster
# Head node
cluster.add_node(num_cpus=0)
cluster.connect(namespace=SERVE_NAMESPACE)
cluster.add_node(num_cpus=2)
cluster.add_node(num_cpus=2)
serve.start()
client = _get_global_client()
def get_actor_distributions():
node_to_actors = defaultdict(list)
for actor in list_actors(
address=cluster.address, filters=[("STATE", "=", "ALIVE")]
):
if "ServeReplica" not in actor.class_name:
continue
node_to_actors[actor.node_id].append(actor)
return set(map(len, node_to_actors.values()))
def check_app_running_with_replicas(num_replicas):
status = serve.status().applications["default"]
assert status.status == "RUNNING"
assert status.deployments["f"].replica_states["RUNNING"] == num_replicas
return True
app_config = {
"name": "default",
"import_path": "ray.serve.tests.test_cluster.f_app",
"deployments": [{"name": "f", "num_replicas": 3}],
}
client.deploy_apps(ServeDeploySchema(**{"applications": [app_config]}))
wait_for_condition(check_app_running_with_replicas, num_replicas=3)
assert get_actor_distributions() == {2, 1}
app_config["deployments"][0]["num_replicas"] = 2
client.deploy_apps(ServeDeploySchema(**{"applications": [app_config]}))
wait_for_condition(check_app_running_with_replicas, num_replicas=2)
assert get_actor_distributions() == {2}
@pytest.mark.skipif(sys.platform == "win32", reason="Flaky on Windows.")
@pytest.mark.skipif(
RAY_SERVE_USE_PACK_SCHEDULING_STRATEGY, reason="Needs spread strategy."
)
def test_replica_spread(ray_cluster):
cluster = ray_cluster
cluster.add_node(num_cpus=2)
# NOTE(edoakes): we need to start serve before adding the worker node to
# guarantee that the controller is placed on the head node (we should be
# able to tolerate being placed on workers, but there's currently a bug).
# We should add an explicit test for that in the future when it's fixed.
cluster.connect(namespace=SERVE_NAMESPACE)
serve.start()
worker_node = cluster.add_node(num_cpus=2)
@serve.deployment(
num_replicas=2,
health_check_period_s=1,
)
def D():
return "hi"
serve.run(D.bind())
def get_num_nodes():
client = _get_global_client()
details = client.get_serve_details()
dep = details["applications"]["default"]["deployments"]["D"]
nodes = {r["node_id"] for r in dep["replicas"]}
print("replica nodes", nodes)
return len(nodes)
# Check that the two replicas are spread across the two nodes.
wait_for_condition(lambda: get_num_nodes() == 2)
# Kill the worker node. The second replica should get rescheduled on
# the head node.
print("Removing worker node. Replica should be rescheduled.")
cluster.remove_node(worker_node)
# Check that the replica on the dead node can be rescheduled.
wait_for_condition(lambda: get_num_nodes() == 1)
def test_autoscale_upscaling_stuck_then_healthy(ray_cluster):
"""Test that deployment stuck in upscaling (due to insufficient cluster resources)
recovers to healthy when ongoing requests drop to zero.
Setup: Head with 0 CPUs + 1 worker with 1 CPU. 1 replica using 1 CPU,
target_ongoing_requests=1. Send 2 requests via handle -> autoscaler wants 2
replicas but can't add one (no CPU). Deployment stuck in UPSCALING.
Release requests -> deployment HEALTHY.
"""
cluster = ray_cluster
cluster.add_node(num_cpus=0) # Head node (controller/proxy use 0 CPU)
cluster.connect(namespace=SERVE_NAMESPACE)
serve.start() # Start before adding worker so controller goes on head
cluster.add_node(num_cpus=1) # Worker with 1 CPU for replica
cluster.wait_for_nodes()
signal = SignalActor.remote()
@serve.deployment(
autoscaling_config={
"min_replicas": 1,
"max_replicas": 2,
"target_ongoing_requests": 1,
"metrics_interval_s": 0.1,
"look_back_period_s": 0.5,
"upscale_delay_s": 0,
# If delay is large then the test will be stuck in UPSCALING state.
"downscale_delay_s": 1,
},
max_ongoing_requests=1,
ray_actor_options={"num_cpus": 1},
graceful_shutdown_timeout_s=2,
)
def blocking_replica():
ray.get(signal.wait.remote())
return "ok"
handle = serve.run(blocking_replica.bind())
wait_for_condition(
check_deployment_status,
name="blocking_replica",
expected_status=DeploymentStatus.HEALTHY,
)
# Send 2 requests - first occupies the replica, second queues. With
# target_ongoing_requests=1 and 1 replica, 2 requests triggers scale to 2.
responses = [handle.remote() for _ in range(2)]
# Deployment should get stuck in UPSCALING: autoscaler wants 2 replicas
# but cluster only has 1 CPU (replica uses it all).
wait_for_condition(
check_deployment_status,
name="blocking_replica",
expected_status=DeploymentStatus.UPSCALING,
timeout=15,
)
# Release the signal so running requests complete and go to zero.
ray.get(signal.send.remote())
for r in responses:
assert r.result() == "ok"
# Deployment should recover to HEALTHY as load drops (may go through
# DOWNSCALING first if a second replica was briefly added).
wait_for_condition(
check_deployment_status,
name="blocking_replica",
expected_status=DeploymentStatus.HEALTHY,
timeout=30,
)
def test_handle_prefers_replicas_on_same_node(ray_cluster):
"""Verify that handle calls prefer replicas on the same node when possible.
If all replicas on the same node are occupied (at `max_ongoing_requests` limit),
requests should spill to other nodes.
"""
cluster = ray_cluster
cluster.add_node(num_cpus=1)
cluster.add_node(num_cpus=1)
signal = SignalActor.remote()
@serve.deployment(num_replicas=2, max_ongoing_requests=1)
def inner(block_on_signal):
if block_on_signal:
ray.get(signal.wait.remote())
return ray.get_runtime_context().get_node_id()
@serve.deployment(num_replicas=1, ray_actor_options={"num_cpus": 0})
class Outer:
def __init__(self, inner_handle: DeploymentHandle):
self._h = inner_handle.options(_prefer_local_routing=True)
def get_node_id(self) -> str:
return ray.get_runtime_context().get_node_id()
async def call_inner(self, block_on_signal: bool = False) -> str:
return await self._h.remote(block_on_signal)
# The inner deployment's two replicas will be spread across the two nodes and
# the outer deployment's single replica will be placed on one of them.
h = serve.run(Outer.bind(inner.bind()))
# When sending requests sequentially, all requests to the inner deployment should
# go to the replica on the same node as the outer deployment replica.
outer_node_id = h.get_node_id.remote().result()
for _ in range(10):
assert h.call_inner.remote().result() == outer_node_id
# Make a blocking request to the inner deployment replica on the same node.
blocked_response = h.call_inner.remote(block_on_signal=True)
with pytest.raises(TimeoutError):
blocked_response.result(timeout_s=1)
# Because there's a blocking request and `max_ongoing_requests` is set to 1, all
# requests should now spill to the other node.
for _ in range(10):
assert h.call_inner.remote().result() != outer_node_id
ray.get(signal.send.remote())
assert blocked_response.result() == outer_node_id
# TODO: HAProxy's default ingress balances across all replicas with no
# node-local preference. prefer-local routing could be wired under HAProxy via
# the ingress_request_router use-server delegation, then this skip dropped.
@skip_if_haproxy("balances across replicas without node-local preference")
@pytest.mark.parametrize("set_flag", [True, False])
def test_proxy_prefers_replicas_on_same_node(ray_cluster: Cluster, set_flag):
"""When the feature flag is turned on via env var, verify that http proxy routes to
replicas on the same node when possible. Otherwise if env var is not set, http proxy
should route to all replicas equally.
"""
if not set_flag:
os.environ["RAY_SERVE_PROXY_PREFER_LOCAL_NODE_ROUTING"] = "0"
cluster = ray_cluster
cluster.add_node(num_cpus=1)
cluster.add_node(num_cpus=1)
# Only start one HTTP proxy on the head node.
serve.start(http_options={"location": "HeadOnly"})
head_node_id = get_head_node_id()
@serve.deployment(num_replicas=2, max_ongoing_requests=1)
def f():
return ray.get_runtime_context().get_node_id()
# The deployment's two replicas will be spread across the two nodes
serve.run(f.bind())
# Since they're sent sequentially, all requests should be routed to
# the replica on the head node
responses = [httpx.post("http://localhost:8000").text for _ in range(10)]
if set_flag:
assert all(resp == head_node_id for resp in responses)
else:
assert len(set(responses)) == 2
if "RAY_SERVE_PROXY_PREFER_LOCAL_NODE_ROUTING" in os.environ:
del os.environ["RAY_SERVE_PROXY_PREFER_LOCAL_NODE_ROUTING"]
class TestHealthzAndRoutes:
def test_head_node_proxy_healthy(self, ray_cluster: Cluster):
"""When a new cluster is started with no replicas, head node proxy should
respond with 200 at /-/healthz and /-/routes"""
cluster = ray_cluster
cluster.add_node(num_cpus=0) # Head node
cluster.wait_for_nodes()
ray.init(address=cluster.address)
serve.start(http_options={"location": "EveryNode"})
@serve.deployment(ray_actor_options={"num_cpus": 0})
class Dummy:
pass
serve.run(Dummy.bind())
# Head node proxy /-/healthz and /-/routes should return 200
r = httpx.post("http://localhost:8000/-/healthz")
assert r.status_code == 200
r = httpx.post("http://localhost:8000/-/routes")
assert r.status_code == 200
def test_head_and_worker_nodes_no_replicas(self, ray_cluster: Cluster):
"""Test `/-/healthz` and `/-/routes` return the correct responses for head and
worker nodes.
When there are replicas on all nodes, `/-/healthz` and `/-/routes` on all nodes
should return 200. When there are no replicas on any nodes, `/-/healthz` and
`/-/routes` on the head node should continue to return 200. `/-/healthz` and
`/-/routes` on the worker node should start to return 503
"""
# Setup worker http proxy to be pointing to port 8001. Head node http proxy will
# continue to be pointing to the default port 8000.
cluster = ray_cluster
cluster.add_node(num_cpus=0)
cluster.add_node(
num_cpus=2, env_vars={"RAY_SERVE_WORKER_PROXY_HTTP_PORT": "8001"}
)
cluster.wait_for_nodes()
ray.init(address=cluster.address)
serve.start(http_options={"location": "EveryNode"})
# Deploy 2 replicas, both should be on the worker node.
@serve.deployment(num_replicas=2)
class HelloModel:
def __call__(self):
return "hello"
model = HelloModel.bind()
serve.run(target=model)
# Ensure worker node has both replicas.
def check_replicas_on_worker_nodes():
return (
len(
{
a.node_id
for a in list_actors(address=cluster.address)
if a.class_name.startswith("ServeReplica")
}
)
== 1
)
wait_for_condition(check_replicas_on_worker_nodes)
# Total alive actors: EveryNode proxies on both nodes + 1 controller +
# 2 replicas. Under HAProxy each proxy node runs an HAProxyManager and
# the head node adds a fallback ProxyActor.
expected_num_actors = (
sum(expected_proxy_actors(num_proxy_nodes=2).values()) + 1 + 2
)
wait_for_condition(
lambda: len(list_actors(address=cluster.address)) == expected_num_actors
)
assert len(ray.nodes()) == 2
# Ensure `/-/healthz` and `/-/routes` return 200 and expected responses
# on both nodes.
def check_request(url: str, expected_code: int, expected_text: str):
req = httpx.get(url)
assert req.status_code == expected_code
assert req.text == expected_text
return True
wait_for_condition(
condition_predictor=check_request,
url="http://127.0.0.1:8000/-/healthz",
expected_code=200,
expected_text="success",
)
assert httpx.get("http://127.0.0.1:8000/-/routes").status_code == 200
assert httpx.get("http://127.0.0.1:8000/-/routes").text == '{"/":"default"}'
wait_for_condition(
condition_predictor=check_request,
url="http://127.0.0.1:8001/-/healthz",
expected_code=200,
expected_text="success",
)
assert httpx.get("http://127.0.0.1:8001/-/routes").status_code == 200
assert httpx.get("http://127.0.0.1:8001/-/routes").text == '{"/":"default"}'
# Deleting the deployment drops the replicas on all nodes. The proxies and
# controller stay alive (the worker proxy drains), so the count is the
# pre-delete total minus the 2 replicas.
serve.delete(name=SERVE_DEFAULT_APP_NAME)
expected_num_actors_after_delete = (
sum(expected_proxy_actors(num_proxy_nodes=2).values()) + 1
)
wait_for_condition(
lambda: len(
list_actors(address=cluster.address, filters=[("STATE", "=", "ALIVE")])
)
== expected_num_actors_after_delete,
)
# Ensure head node `/-/healthz` and `/-/routes` continue to
# return 200 and expected responses. Also, the worker node
# `/-/healthz` and `/-/routes` should return 503 and unavailable
# responses.
wait_for_condition(
condition_predictor=check_request,
url="http://127.0.0.1:8000/-/healthz",
expected_code=200,
expected_text="success",
)
wait_for_condition(
condition_predictor=check_request,
url="http://127.0.0.1:8000/-/routes",
expected_code=200,
expected_text="{}",
)
wait_for_condition(
condition_predictor=check_request,
url="http://127.0.0.1:8001/-/healthz",
expected_code=503,
expected_text="This node is being drained.",
)
wait_for_condition(
condition_predictor=check_request,
url="http://127.0.0.1:8001/-/routes",
expected_code=503,
expected_text="This node is being drained.",
)
if __name__ == "__main__":
sys.exit(pytest.main(["-v", "-s", __file__]))
python/ray/serve/tests/test_cluster_node_info_cache.py
+54
View File
@@ -0,0 +1,54 @@
import pytest
import ray
from ray._raylet import GcsClient
from ray.serve._private.default_impl import create_cluster_node_info_cache
from ray.serve._private.test_utils import get_node_id
from ray.tests.conftest import * # noqa
def test_get_alive_nodes(ray_start_cluster):
cluster = ray_start_cluster
cluster.add_node(resources={"head": 1})
ray.init(address=cluster.address)
worker_node = cluster.add_node(resources={"worker": 1})
cluster.wait_for_nodes()
head_node_id = ray.get(get_node_id.options(resources={"head": 1}).remote())
worker_node_id = ray.get(get_node_id.options(resources={"worker": 1}).remote())
gcs_client = GcsClient(address=ray.get_runtime_context().gcs_address)
cluster_node_info_cache = create_cluster_node_info_cache(gcs_client)
cluster_node_info_cache.update()
assert set(cluster_node_info_cache.get_alive_nodes()) == {
(head_node_id, ray.nodes()[0]["NodeName"], ""),
(worker_node_id, ray.nodes()[0]["NodeName"], ""),
}
assert cluster_node_info_cache.get_alive_node_ids() == {
head_node_id,
worker_node_id,
}
assert (
cluster_node_info_cache.get_alive_node_ids()
== cluster_node_info_cache.get_active_node_ids()
)
cluster.remove_node(worker_node)
cluster.wait_for_nodes()
# The killed worker node shouldn't show up in the alive node list.
cluster_node_info_cache.update()
assert cluster_node_info_cache.get_alive_nodes() == [
(head_node_id, ray.nodes()[0]["NodeName"], "")
]
assert cluster_node_info_cache.get_alive_node_ids() == {head_node_id}
assert (
cluster_node_info_cache.get_alive_node_ids()
== cluster_node_info_cache.get_active_node_ids()
)
if __name__ == "__main__":
import sys
sys.exit(pytest.main(["-v", "-s", __file__]))
python/ray/serve/tests/test_config_files/apps_with_args.yaml
+17
View File
@@ -0,0 +1,17 @@
applications:
- name: untyped_default
route_prefix: /untyped_default
import_path: ray.serve.tests.test_config_files.arg_builders.build_echo_app
- name: untyped_hello
route_prefix: /untyped_hello
import_path: ray.serve.tests.test_config_files.arg_builders.build_echo_app
args:
message: hello
- name: typed_default
route_prefix: /typed_default
import_path: ray.serve.tests.test_config_files.arg_builders.build_echo_app_typed
- name: typed_hello
route_prefix: /typed_hello
import_path: ray.serve.tests.test_config_files.arg_builders.build_echo_app_typed
args:
message: hello
python/ray/serve/tests/test_config_files/arg_builders.py
+25
View File
@@ -0,0 +1,25 @@
from pydantic import BaseModel
from ray import serve
class TypedArgs(BaseModel):
message: str = "DEFAULT"
@serve.deployment(ray_actor_options={"num_cpus": 0})
class Echo:
def __init__(self, message: str):
print("Echo message:", message)
self._message = message
def __call__(self, *args):
return self._message
def build_echo_app(args):
return Echo.bind(args.get("message", "DEFAULT"))
def build_echo_app_typed(args: TypedArgs):
return Echo.bind(args.message)
python/ray/serve/tests/test_config_files/arithmetic.yaml
+20
View File
@@ -0,0 +1,20 @@
applications:
- name: default
import_path: "dir.subdir.a.add_and_sub.serve_dag"
runtime_env:
# Keep these pinned remote URIs in sync with tests/common/remote_uris.py.
working_dir: "https://github.com/ray-project/test_dag/archive/203140040e4ab50b9d35b4773ec5c22615c034b3.zip"
deployments:
- name: "Router"
graceful_shutdown_timeout_s: 0.0001
- name: "Add"
graceful_shutdown_timeout_s: 0.0001
- name: "Subtract"
graceful_shutdown_timeout_s: 0.0001
ray_actor_options:
runtime_env:
# Keep these pinned remote URIs in sync with tests/common/remote_uris.py.
py_modules:
- "https://github.com/ray-project/test_module/archive/aa6f366f7daa78c98408c27d917a983caa9f888b.zip"
python/ray/serve/tests/test_config_files/bad_multi_config.yaml
+25
View File
@@ -0,0 +1,25 @@
applications:
- name: "app1"
route_prefix: "/app1"
import_path: ray.serve.tests.test_config_files.pizza.serve_dag
deployments:
- name: Multiplier
user_config:
factor: 1
- name: Adder
user_config:
increment: 1
- name: "app2"
# Route prefixes should be unique across all apps!
route_prefix: "/app1"
import_path: ray.serve.tests.test_config_files.pizza.serve_dag
deployments:
- name: Multiplier
user_config:
factor: 2
- name: Adder
user_config:
increment: 3
python/ray/serve/tests/test_config_files/bad_runtime_env.yaml
+4
View File
@@ -0,0 +1,4 @@
applications:
- name: default
import_path: ray.serve.tests.test_config_files.test_dag.basic_dag.DagNode
runtime_env: {"working_dir": "s3://does_not_exist.zip"}
python/ray/serve/tests/test_config_files/basic_graph.yaml
+4
View File
@@ -0,0 +1,4 @@
applications:
- name: default
route_prefix: /
import_path: ray.serve.tests.test_config_files.test_dag.basic_dag.DagNode
python/ray/serve/tests/test_config_files/basic_graph_http.yaml
+7
View File
@@ -0,0 +1,7 @@
http_options:
host: 127.0.0.1
port: 8005
applications:
- name: default
import_path: ray.serve.tests.test_config_files.test_dag.basic_dag.DagNode
python/ray/serve/tests/test_config_files/basic_multi.yaml
+3
View File
@@ -0,0 +1,3 @@
applications:
- name: "app1"
import_path: ray.serve.tests.test_config_files.test_dag.basic_dag.DagNode
python/ray/serve/tests/test_config_files/basic_multi_http.yaml
+6
View File
@@ -0,0 +1,6 @@
http_options:
port: 8005
applications:
- name: "app1"
import_path: ray.serve.tests.test_config_files.test_dag.basic_dag.DagNode
python/ray/serve/tests/test_config_files/broken_app.py
+26
View File
@@ -0,0 +1,26 @@
import ray.cloudpickle as pickle
from ray import serve
class NonserializableException(Exception):
"""This exception cannot be serialized."""
def __reduce__(self):
raise RuntimeError("This exception cannot be serialized!")
# Confirm that NonserializableException cannot be serialized.
try:
pickle.dumps(NonserializableException())
except RuntimeError as e:
assert "This exception cannot be serialized!" in repr(e)
raise NonserializableException("custom exception info")
@serve.deployment
def f():
pass
app = f.bind()
python/ray/serve/tests/test_config_files/config_only_deployment_actor.py
+20
View File
@@ -0,0 +1,20 @@
"""Deployment that uses deployment actors but defines them only via config override.
The deployment has no deployment_actors in the @serve.deployment decorator;
they are added purely through the declarative config override.
"""
import ray
from ray import serve
@serve.deployment(ray_actor_options={"num_cpus": 0.1})
class ConfigOnlyDriver:
"""Uses get_deployment_actor; deployment_actors come from config only."""
def __call__(self):
counter = serve.get_deployment_actor("counter")
return str(ray.get(counter.get.remote()))
app = ConfigOnlyDriver.bind()
python/ray/serve/tests/test_config_files/delete_blocked.py
+13
View File
@@ -0,0 +1,13 @@
import asyncio
from ray import serve
@serve.deployment
class A:
async def __del__(self):
while True:
await asyncio.sleep(0.1)
app = A.bind()
python/ray/serve/tests/test_config_files/deploy_grpc_app.yaml
+8
View File
@@ -0,0 +1,8 @@
grpc_options:
port: 9000
grpc_servicer_functions:
- ray.serve.generated.serve_pb2_grpc.add_UserDefinedServiceServicer_to_server
applications:
- name: app1
import_path: ray.serve.tests.test_config_files.grpc_deployment:g
python/ray/serve/tests/test_config_files/deploy_grpc_model_composition.yaml
+8
View File
@@ -0,0 +1,8 @@
grpc_options:
port: 9000
grpc_servicer_functions:
- ray.serve.generated.serve_pb2_grpc.add_FruitServiceServicer_to_server
applications:
- name: app1
import_path: ray.serve.tests.test_config_files.grpc_deployment:g2
python/ray/serve/tests/test_config_files/deploy_grpc_multiplexed_app.yaml
+8
View File
@@ -0,0 +1,8 @@
grpc_options:
port: 9000
grpc_servicer_functions:
- ray.serve.generated.serve_pb2_grpc.add_UserDefinedServiceServicer_to_server
applications:
- name: app1
import_path: ray.serve.tests.test_config_files.grpc_deployment:multiplexed_g
python/ray/serve/tests/test_config_files/deployment_actors.yaml
+11
View File
@@ -0,0 +1,11 @@
applications:
- name: default
route_prefix: /
import_path: ray.serve.tests.test_config_files.config_only_deployment_actor:app
deployments:
- name: ConfigOnlyDriver
deployment_actors:
- name: counter
actor_class: ray.serve.tests.test_deployment_actors:SharedCounter
init_kwargs:
start: 88
python/ray/serve/tests/test_config_files/deployment_fail.yaml
+3
View File
@@ -0,0 +1,3 @@
applications:
- name: default
import_path: ray.serve.tests.test_config_files.fail.node
python/ray/serve/tests/test_config_files/deployment_fail_2.yaml
+3
View File
@@ -0,0 +1,3 @@
applications:
- name: default
import_path: ray.serve.tests.test_config_files.fail_2.node
python/ray/serve/tests/test_config_files/deployment_uses_utils_module.yaml
+4
View File
@@ -0,0 +1,4 @@
applications:
- name: default
route_prefix: /
import_path: ray.serve.tests.test_config_files.test_dag.hello_serve:model
python/ray/serve/tests/test_config_files/duplicate_app_names.yaml
+8
View File
@@ -0,0 +1,8 @@
applications:
- name: app1
route_prefix: /a
import_path: ray.serve.tests.test_config_files.test_dag.basic_dag.DagNode
- name: app1
route_prefix: /b
import_path: ray.serve.tests.test_config_files.test_dag.basic_dag.DagNode
python/ray/serve/tests/test_config_files/duplicate_app_routes.yaml
+8
View File
@@ -0,0 +1,8 @@
applications:
- name: app1
route_prefix: /alice
import_path: ray.serve.tests.test_config_files.test_dag.basic_dag.DagNode
- name: app2
route_prefix: /alice
import_path: ray.serve.tests.test_config_files.test_dag.basic_dag.DagNode
python/ray/serve/tests/test_config_files/fail.py
+10
View File
@@ -0,0 +1,10 @@
from ray import serve
@serve.deployment
class A:
def __init__(self):
_ = 1 / 0
node = A.bind()
python/ray/serve/tests/test_config_files/fail_2.py
+13
View File
@@ -0,0 +1,13 @@
import time
from ray import serve
@serve.deployment
class A:
def __init__(self):
time.sleep(5)
_ = 1 / 0
node = A.bind()
python/ray/serve/tests/test_config_files/fastapi_deployment.py
+16
View File
@@ -0,0 +1,16 @@
from fastapi import FastAPI
from ray import serve
app = FastAPI(docs_url="/my_docs")
@serve.deployment
@serve.ingress(app)
class FastAPIDeployment:
@app.get("/hello")
def incr(self):
return "Hello world!"
node = FastAPIDeployment.bind()
python/ray/serve/tests/test_config_files/flaky_build.py
+29
View File
@@ -0,0 +1,29 @@
"""Test fixture: raises on first FLAKY_BUILD_FAIL_COUNT imports, then succeeds.
A counter file persists state across the fresh worker processes that
``build_serve_application``'s ``max_calls=1`` decorator spawns on each retry.
"""
import os
from ray import serve
_COUNTER_FILE = os.environ["FLAKY_BUILD_COUNTER_FILE"]
_FAIL_COUNT = int(os.environ.get("FLAKY_BUILD_FAIL_COUNT", "3"))
with open(_COUNTER_FILE, "r") as f:
_attempts = int(f.read().strip() or "0")
with open(_COUNTER_FILE, "w") as f:
f.write(str(_attempts + 1))
if _attempts < _FAIL_COUNT:
raise RuntimeError(f"flaky build failure on attempt {_attempts + 1}")
@serve.deployment
class FlakyApp:
def __call__(self):
return "ok"
node = FlakyApp.bind()
python/ray/serve/tests/test_config_files/flex_deployment_actor.py
+21
View File
@@ -0,0 +1,21 @@
"""Flexible driver that works with or without deployment actors.
Used by declarative redeployment tests that need to transition from
no-deployment-actors to having them.
"""
import ray
from ray import serve
@serve.deployment(ray_actor_options={"num_cpus": 0.1})
class FlexDriver:
def __call__(self):
try:
actor = serve.get_deployment_actor("counter")
return str(ray.get(actor.get.remote()))
except Exception:
return "no_actor"
app = FlexDriver.bind()
python/ray/serve/tests/test_config_files/gang_scheduling.py
+15
View File
@@ -0,0 +1,15 @@
import json
import os
from ray import serve
@serve.deployment
class GangApp:
def __call__(self, *args):
ctx = serve.context._get_internal_replica_context()
gc = ctx.gang_context
return json.dumps({"pid": os.getpid(), "gang_id": gc.gang_id if gc else None})
app = GangApp.bind()
python/ray/serve/tests/test_config_files/gang_scheduling.yaml
+12
View File
@@ -0,0 +1,12 @@
applications:
- name: gang_app
route_prefix: /
import_path: ray.serve.tests.test_config_files.gang_scheduling:app
deployments:
- name: GangApp
num_replicas: 4
ray_actor_options:
num_cpus: 0.25
gang_scheduling_config:
gang_size: 2
gang_placement_strategy: PACK
python/ray/serve/tests/test_config_files/gang_scheduling_2_replicas.yaml
+12
View File
@@ -0,0 +1,12 @@
applications:
- name: gang_app
route_prefix: /
import_path: ray.serve.tests.test_config_files.gang_scheduling:app
deployments:
- name: GangApp
num_replicas: 2
ray_actor_options:
num_cpus: 0.25
gang_scheduling_config:
gang_size: 2
gang_placement_strategy: PACK
python/ray/serve/tests/test_config_files/gang_scheduling_4_replicas.yaml
+12
View File
@@ -0,0 +1,12 @@
applications:
- name: gang_app
route_prefix: /
import_path: ray.serve.tests.test_config_files.gang_scheduling:app
deployments:
- name: GangApp
num_replicas: 4
ray_actor_options:
num_cpus: 0.25
gang_scheduling_config:
gang_size: 2
gang_placement_strategy: PACK
python/ray/serve/tests/test_config_files/get_multi_deployment_signal_app.py
+29
View File
@@ -0,0 +1,29 @@
import os
import ray
from ray import serve
from ray.serve.handle import DeploymentHandle
@serve.deployment
class A:
def __init__(self, b: DeploymentHandle):
self.b = b
self.signal = ray.get_actor("signal_A", namespace="default_test_namespace")
async def __call__(self):
await self.signal.wait.remote()
return os.getpid()
@serve.deployment
class B:
def __init__(self):
self.signal = ray.get_actor("signal_B", namespace="default_test_namespace")
async def __call__(self):
await self.signal.wait.remote()
return os.getpid()
app = A.bind(B.bind())
python/ray/serve/tests/test_config_files/get_signal.py
+15
View File
@@ -0,0 +1,15 @@
import os
import ray
from ray import serve
@serve.deployment
class A:
async def __call__(self):
signal = ray.get_actor("signal123")
await signal.wait.remote()
return os.getpid()
app = A.bind()
python/ray/serve/tests/test_config_files/grpc_deployment.py
+125
View File
@@ -0,0 +1,125 @@
from typing import Dict
# Users need to include their custom message type which will be embedded in the request.
from ray import serve
from ray.serve.generated import serve_pb2
from ray.serve.handle import DeploymentHandle
@serve.deployment
class GrpcDeployment:
def __call__(self, user_message):
greeting = f"Hello {user_message.name} from {user_message.foo}"
num_x2 = user_message.num * 2
user_response = serve_pb2.UserDefinedResponse(
greeting=greeting,
num_x2=num_x2,
)
return user_response
def Method1(self, user_message):
greeting = f"Hello {user_message.name} from method1"
num_x2 = user_message.num * 3
user_response = serve_pb2.UserDefinedResponse(
greeting=greeting,
num_x2=num_x2,
)
return user_response
def Streaming(self, user_message):
for i in range(10):
greeting = f"{i}: Hello {user_message.name} from {user_message.foo}"
num_x2 = user_message.num * 2 + i
user_response = serve_pb2.UserDefinedResponse(
greeting=greeting,
num_x2=num_x2,
)
yield user_response
g = GrpcDeployment.options(name="grpc-deployment").bind()
# NOTE: model multiplexing is kept on a separate deployment (not the shared `g`)
# because it is not supported on the ingress deployment when direct ingress /
# HAProxy is enabled (the multiplexed model ID is not propagated to the replica).
# Tests that exercise it must be skipped under those modes.
@serve.deployment
class MultiplexedGrpcDeployment:
def __call__(self, user_message):
greeting = f"Hello {user_message.name} from {user_message.foo}"
return serve_pb2.UserDefinedResponse(greeting=greeting)
@serve.multiplexed(max_num_models_per_replica=1)
async def get_model(self, model_id: str) -> str:
return f"loading model: {model_id}"
async def Method2(self, user_message):
model_id = serve.get_multiplexed_model_id()
model = await self.get_model(model_id)
user_response = serve_pb2.UserDefinedResponse(
greeting=f"Method2 called model, {model}",
)
return user_response
multiplexed_g = MultiplexedGrpcDeployment.options(name="grpc-deployment").bind()
@serve.deployment(ray_actor_options={"num_cpus": 0})
class FruitMarket:
def __init__(
self,
_orange_stand: DeploymentHandle,
_apple_stand: DeploymentHandle,
):
self.directory = {
"ORANGE": _orange_stand,
"APPLE": _apple_stand,
}
async def FruitStand(self, fruit_amounts_proto):
fruit_amounts = {}
if fruit_amounts_proto.orange:
fruit_amounts["ORANGE"] = fruit_amounts_proto.orange
if fruit_amounts_proto.apple:
fruit_amounts["APPLE"] = fruit_amounts_proto.apple
if fruit_amounts_proto.banana:
fruit_amounts["BANANA"] = fruit_amounts_proto.banana
costs = await self.check_price(fruit_amounts)
return serve_pb2.FruitCosts(costs=costs)
async def check_price(self, inputs: Dict[str, int]) -> float:
costs = 0
for fruit, amount in inputs.items():
if fruit not in self.directory:
return
fruit_stand = self.directory[fruit]
costs += await fruit_stand.remote(int(amount))
return costs
@serve.deployment(ray_actor_options={"num_cpus": 0})
class OrangeStand:
def __init__(self):
self.price = 2.0
def __call__(self, num_oranges: int):
return num_oranges * self.price
@serve.deployment(ray_actor_options={"num_cpus": 0})
class AppleStand:
def __init__(self):
self.price = 3.0
def __call__(self, num_oranges: int):
return num_oranges * self.price
orange_stand = OrangeStand.bind()
apple_stand = AppleStand.bind()
g2 = FruitMarket.options(name="grpc-deployment-model-composition").bind(
orange_stand, apple_stand
)
python/ray/serve/tests/test_config_files/hangs.py
+13
View File
@@ -0,0 +1,13 @@
import ray
from ray import serve
signal = ray.get_actor("signal123")
ray.get(signal.wait.remote())
@serve.deployment(ray_actor_options={"num_cpus": 0.1})
def f():
return "hello world"
app = f.bind()
python/ray/serve/tests/test_config_files/http_option_app_sleeps.py
+14
View File
@@ -0,0 +1,14 @@
import time
from ray import serve
@serve.deployment
def sleep(request):
sleep_s = float(request.query_params.get("sleep_s", 0))
print(f"sleep_s: {sleep_s}")
time.sleep(sleep_s)
return "Task Succeeded!"
sleep_node = sleep.bind()
python/ray/serve/tests/test_config_files/http_option_request_timeout_s.yaml
+7
View File
@@ -0,0 +1,7 @@
http_options:
request_timeout_s: 0.1
applications:
- name: "app1"
import_path: ray.serve.tests.test_config_files.http_option_app_sleeps.sleep_node
route_prefix: /app1
python/ray/serve/tests/test_config_files/import_error.py
+11
View File
@@ -0,0 +1,11 @@
from ray import serve
_ = 1 / 0
@serve.deployment(ray_actor_options={"num_cpus": 0.1})
def f(*args):
return "hello world"
app = f.bind()
python/ray/serve/tests/test_config_files/logging_config_test.py
+62
View File
@@ -0,0 +1,62 @@
import logging
import ray
from ray import serve
from ray.exceptions import RayActorError
from ray.serve.context import _get_global_client
logger = logging.getLogger("ray.serve")
@serve.deployment
class Model:
def __call__(self):
logger.debug("this_is_debug_info")
logger.info("this_is_access_log", extra={"serve_access_log": True})
log_file = logger.handlers[1].target.baseFilename
return {
"log_file": log_file,
"replica": serve.get_replica_context().replica_id.to_full_id_str(),
"log_level": logger.level,
"num_handlers": len(logger.handlers),
}
@serve.deployment
class Router:
def __init__(self, handle):
self.handle = handle
async def __call__(self):
logger.debug("this_is_debug_info_from_router")
log_info = await self.handle.remote()
if len(logger.handlers) == 2:
log_info["router_log_file"] = logger.handlers[1].target.baseFilename
else:
log_info["router_log_file"] = None
log_info["router_log_level"] = logger.level
try:
# Add controller log file path
client = _get_global_client()
_, log_file_path = ray.get(client._controller._get_logging_config.remote())
except RayActorError:
log_file_path = None
log_info["controller_log_file"] = log_file_path
return log_info
model = Router.bind(Model.bind())
@serve.deployment(logging_config={"log_level": "DEBUG"})
class ModelWithConfig:
def __call__(self):
logger.debug("this_is_debug_info")
log_file = logger.handlers[1].target.baseFilename
return {"log_file": log_file}
model2 = ModelWithConfig.bind()
python/ray/serve/tests/test_config_files/max_replicas_per_node.py
+10
View File
@@ -0,0 +1,10 @@
from ray import serve
@serve.deployment
class D:
def __call__(self, *args):
return "hi"
app = D.bind()
python/ray/serve/tests/test_config_files/max_replicas_per_node.yaml
+15
View File
@@ -0,0 +1,15 @@
applications:
- name: valid
route_prefix: /valid
import_path: ray.serve.tests.test_config_files.max_replicas_per_node.app
deployments:
- name: D
max_replicas_per_node: 2
- name: invalid
route_prefix: /invalid
import_path: ray.serve.tests.test_config_files.max_replicas_per_node.app
deployments:
- name: D
# Non-positive max_replicas_per_node.
max_replicas_per_node: 0
python/ray/serve/tests/test_config_files/missing_runtime_env.yaml
+4
View File
@@ -0,0 +1,4 @@
applications:
- name: default
import_path: "basic_dag.DagNode"
route_prefix: /
python/ray/serve/tests/test_config_files/multi_fastapi.py
+28
View File
@@ -0,0 +1,28 @@
from fastapi import FastAPI
from ray import serve
from ray.serve.handle import DeploymentHandle
app1 = FastAPI()
app2 = FastAPI()
@serve.deployment
@serve.ingress(app2)
class SubModel:
def add(self, a: int):
return a + 1
@serve.deployment
@serve.ingress(app1)
class Model:
def __init__(self, submodel: DeploymentHandle):
self.submodel = submodel
@app1.get("/{a}")
async def func(self, a: int):
return await self.submodel.add.remote(a)
invalid_model = Model.bind(SubModel.bind())
python/ray/serve/tests/test_config_files/pid.py
+78
View File
@@ -0,0 +1,78 @@
import asyncio
import os
import time
from ray import serve
@serve.deployment
class f:
def __init__(self, async_wait: bool = False):
# whether to use wait() (busy spin) or async_wait() (busy spin
# while yielding event loop)
self._async = async_wait
# to be updated through reconfigure()
self.name = "default_name"
# used to block calls to __call__()
self.ready = True
# used to check how many times __call__() has been called
self.counter = 0
# used to block calls to health_check()
self.health_check_ready = True
# used to check how many times health_check() has been called
self.health_check_counter = 0
async def get_counter(self, health_check=False) -> int:
if health_check:
return self.health_check_counter
else:
return self.counter
def send(self, clear=False, health_check=False):
if health_check:
self.health_check_ready = not clear
else:
self.ready = not clear
def wait(self, _health_check=False):
if _health_check:
while not self.health_check_ready:
time.sleep(0.1)
else:
while not self.ready:
time.sleep(0.1)
async def async_wait(self, _health_check=False):
if _health_check:
while not self.health_check_ready:
await asyncio.sleep(0.1)
else:
while not self.ready:
await asyncio.sleep(0.1)
def reconfigure(self, config: dict):
self.name = config.get("name", "default_name")
async def __call__(self):
self.counter += 1
if self._async:
await self.async_wait()
else:
self.wait()
return os.getpid(), self.name
async def check_health(self):
self.health_check_counter += 1
if self._async:
await self.async_wait(_health_check=True)
else:
self.wait(_health_check=True)
node = f.bind()
dup_node = f.bind()
async_node = f.bind(async_wait=True)
python/ray/serve/tests/test_config_files/pizza.py
+79
View File
@@ -0,0 +1,79 @@
from enum import Enum
from typing import Dict, List
import starlette.requests
from ray import serve
from ray.serve.handle import DeploymentHandle
class Operation(str, Enum):
ADDITION = "ADD"
MULTIPLICATION = "MUL"
@serve.deployment(ray_actor_options={"num_cpus": 0.15})
class Router:
def __init__(self, multiplier: DeploymentHandle, adder: DeploymentHandle):
self.adder = adder
self.multiplier = multiplier
async def route(self, op: Operation, input: int) -> str:
if op == Operation.ADDITION:
amount = await self.adder.add.remote(input)
elif op == Operation.MULTIPLICATION:
amount = await self.multiplier.multiply.remote(input)
return f"{amount} pizzas please!"
async def __call__(self, request: starlette.requests.Request) -> str:
op, input = await request.json()
return await self.route(op, input)
@serve.deployment(
user_config={
"factor": 3,
},
ray_actor_options={"num_cpus": 0.15},
)
class Multiplier:
def __init__(self, factor: int):
self.factor = factor
def reconfigure(self, config: Dict):
self.factor = config.get("factor", -1)
def multiply(self, input_factor: int) -> int:
return input_factor * self.factor
@serve.deployment(
user_config={
"increment": 2,
},
ray_actor_options={"num_cpus": 0.15},
)
class Adder:
def __init__(self, increment: int):
self.increment = increment
def reconfigure(self, config: Dict):
self.increment = config.get("increment", -1)
def add(self, input: int) -> int:
return input + self.increment
async def json_resolver(request: starlette.requests.Request) -> List:
return await request.json()
# Overwritten by user_config
ORIGINAL_INCREMENT = 1
ORIGINAL_FACTOR = 1
multiplier = Multiplier.bind(ORIGINAL_FACTOR)
adder = Adder.bind(ORIGINAL_INCREMENT)
serve_dag = Router.bind(multiplier, adder)
python/ray/serve/tests/test_config_files/pizza.yaml
+20
View File
@@ -0,0 +1,20 @@
applications:
- name: default
route_prefix: /
import_path: ray.serve.tests.test_config_files.test_dag.conditional_dag.serve_dag
deployments:
- name: Router
graceful_shutdown_timeout_s: 0.0001
- name: Multiplier
graceful_shutdown_timeout_s: 0.0001
user_config:
factor: 1
- name: Adder
graceful_shutdown_timeout_s: 0.0001
ray_actor_options:
runtime_env:
env_vars:
override_increment: '1'
python/ray/serve/tests/test_config_files/pizza_world.yaml
+16
View File
@@ -0,0 +1,16 @@
applications:
- name: "app1"
route_prefix: "/app1"
import_path: ray.serve.tests.test_config_files.world.DagNode
- name: "app2"
route_prefix: "/app2"
import_path: ray.serve.tests.test_config_files.pizza.serve_dag
deployments:
- name: Multiplier
user_config:
factor: 10
- name: Adder
user_config:
increment: 10
python/ray/serve/tests/test_config_files/ray_already_initialized.py
+12
View File
@@ -0,0 +1,12 @@
import ray
from ray import serve
ray.init(address="auto")
@serve.deployment
def f():
return "foobar"
app = f.bind()
python/ray/serve/tests/test_config_files/replica_placement_groups.py
+10
View File
@@ -0,0 +1,10 @@
from ray import serve
@serve.deployment
class D:
def __call__(self, *args):
return "hi"
app = D.bind()
python/ray/serve/tests/test_config_files/replica_placement_groups.yaml
+27
View File
@@ -0,0 +1,27 @@
applications:
- name: valid
route_prefix: /valid
import_path: ray.serve.tests.test_config_files.replica_placement_groups.app
deployments:
- name: D
placement_group_bundles:
- {"CPU": 1}
placement_group_strategy: STRICT_PACK
- name: invalid_bundles
route_prefix: /invalid_bundles
import_path: ray.serve.tests.test_config_files.replica_placement_groups.app
deployments:
- name: D
# Insufficient resources for the replica actor.
placement_group_bundles:
- {"CPU": 0.1}
- name: invalid_strategy
route_prefix: /invalid_strategy
import_path: ray.serve.tests.test_config_files.replica_placement_groups.app
deployments:
- name: D
placement_group_bundles:
- {"CPU": 1}
placement_group_strategy: FAKE_NEWS
python/ray/serve/tests/test_config_files/sqlalchemy.py
+15
View File
@@ -0,0 +1,15 @@
from ray import serve
@serve.deployment
class TestDeployment:
def __init__(self):
import pymysql
from sqlalchemy import create_engine
pymysql.install_as_MySQLdb()
create_engine("mysql://some_wrong_url:3306").connect()
app = TestDeployment.bind()
python/ray/serve/tests/test_config_files/sqlalchemy.yaml
+11
View File
@@ -0,0 +1,11 @@
applications:
- name: default
import_path: ray.serve.tests.test_config_files.sqlalchemy.app
deployments:
- name: TestDeployment
num_replicas: 1
ray_actor_options:
runtime_env:
pip:
- PyMySQL
- sqlalchemy==1.3.19
python/ray/serve/tests/test_config_files/syntax_error.py
+1
View File
@@ -0,0 +1 @@
x = (1 + 2
python/ray/serve/tests/test_config_files/syntax_error.yaml
+3
View File
@@ -0,0 +1,3 @@
applications:
- name: default
import_path: ray.serve.tests.test_config_files.syntax_error.app
python/ray/serve/tests/test_config_files/test_dag/basic_dag.py
+28
View File
@@ -0,0 +1,28 @@
from ray import serve
from ray.serve.handle import DeploymentHandle
@serve.deployment(
ray_actor_options={
"num_cpus": 0.1,
}
)
def f(*args):
return "wonderful world"
@serve.deployment(
ray_actor_options={
"num_cpus": 0.1,
}
)
class BasicDriver:
def __init__(self, h: DeploymentHandle):
self._h = h
async def __call__(self):
return await self._h.remote()
FNode = f.bind()
DagNode = BasicDriver.bind(FNode)
python/ray/serve/tests/test_config_files/test_dag/conditional_dag.py
+97
View File
@@ -0,0 +1,97 @@
import os
from enum import Enum
from typing import Dict
import starlette.requests
from ray import serve
from ray.serve.handle import DeploymentHandle
class Operation(str, Enum):
ADDITION = "ADD"
MULTIPLICATION = "MUL"
@serve.deployment(
ray_actor_options={
"num_cpus": 0.1,
}
)
class Router:
def __init__(self, multiplier: DeploymentHandle, adder: DeploymentHandle):
self.adder = adder
self.multiplier = multiplier
async def route(self, op: Operation, input: int) -> int:
if op == Operation.ADDITION:
amount = await self.adder.add.remote(input)
elif op == Operation.MULTIPLICATION:
amount = await self.multiplier.multiply.remote(input)
return f"{amount} pizzas please!"
async def __call__(self, request: starlette.requests.Request) -> str:
op, input = await request.json()
return await self.route(op, input)
@serve.deployment(
user_config={
"factor": 3,
},
ray_actor_options={
"num_cpus": 0.1,
"runtime_env": {
"env_vars": {
"override_factor": "-2",
}
},
},
)
class Multiplier:
def __init__(self, factor: int):
self.factor = factor
def reconfigure(self, config: Dict):
self.factor = config.get("factor", -1)
def multiply(self, input_factor: int) -> int:
if os.getenv("override_factor") is not None:
return input_factor * int(os.getenv("override_factor"))
return input_factor * self.factor
@serve.deployment(
user_config={
"increment": 2,
},
ray_actor_options={
"num_cpus": 0.1,
"runtime_env": {
"env_vars": {
"override_increment": "-2",
}
},
},
)
class Adder:
def __init__(self, increment: int):
self.increment = increment
def reconfigure(self, config: Dict):
self.increment = config.get("increment", -1)
def add(self, input: int) -> int:
if os.getenv("override_increment") is not None:
return input + int(os.getenv("override_increment"))
return input + self.increment
# Overwritten by user_config
ORIGINAL_INCREMENT = 1
ORIGINAL_FACTOR = 1
multiplier = Multiplier.bind(ORIGINAL_FACTOR)
adder = Adder.bind(ORIGINAL_INCREMENT)
serve_dag = Router.bind(multiplier, adder)
python/ray/serve/tests/test_config_files/test_dag/dir/subdir/a/add_and_sub.py
+65
View File
@@ -0,0 +1,65 @@
from enum import Enum
import starlette.requests
from ray import serve
from ray.serve.handle import DeploymentHandle
class Operation(str, Enum):
ADD = "ADD"
SUBTRACT = "SUB"
@serve.deployment(
ray_actor_options={
"num_cpus": 0.1,
}
)
class Add:
# Requires the test_dag repo as a py_module:
# https://github.com/ray-project/test_dag
def add(self, input: int) -> int:
from dir2.library import add_one
return add_one(input)
@serve.deployment(
ray_actor_options={
"num_cpus": 0.1,
}
)
class Subtract:
# Requires the test_module repo as a py_module:
# https://github.com/ray-project/test_module
def subtract(self, input: int) -> int:
from test_module.test import one
return input - one() # Returns input - 2
@serve.deployment(
ray_actor_options={
"num_cpus": 0.1,
}
)
class Router:
def __init__(self, adder: DeploymentHandle, subtractor: DeploymentHandle):
self.adder = adder
self.subtractor = subtractor
async def __call__(self, request: starlette.requests.Request) -> int:
op, input = await request.json()
if op == Operation.ADD:
return await self.adder.add.remote(input)
elif op == Operation.SUBTRACT:
return await self.subtractor.subtract.remote(input)
adder = Add.bind()
subtractor = Subtract.bind()
serve_dag = Router.bind(adder, subtractor)
python/ray/serve/tests/test_config_files/test_dag/dir2/library.py
+2
View File
@@ -0,0 +1,2 @@
def add_one(inp):
return inp + 1
python/ray/serve/tests/test_config_files/test_dag/hello_serve.py
+13
View File
@@ -0,0 +1,13 @@
from starlette.requests import Request
from ray import serve
from ray.serve.tests.test_config_files.test_dag.utils.test import hello
@serve.deployment
class HelloModel:
async def __call__(self, starlette_request: Request) -> None:
return hello()
model = HelloModel.bind()
python/ray/serve/tests/test_config_files/test_dag/utils/test.py
+2
View File
@@ -0,0 +1,2 @@
def hello():
return "hello_from_utils"
python/ray/serve/tests/test_config_files/two_pizzas.yaml
+24
View File
@@ -0,0 +1,24 @@
applications:
- name: "app1"
route_prefix: "/app1"
import_path: ray.serve.tests.test_config_files.pizza.serve_dag
deployments:
- name: Multiplier
user_config:
factor: 1
- name: Adder
user_config:
increment: 1
- name: "app2"
route_prefix: "/app2"
import_path: ray.serve.tests.test_config_files.pizza.serve_dag
deployments:
- name: Multiplier
user_config:
factor: 2
- name: Adder
user_config:
increment: 3
python/ray/serve/tests/test_config_files/use_current_working_directory.py
+9
View File
@@ -0,0 +1,9 @@
from ray import serve
@serve.deployment
def f():
return "hi"
app = f.bind()
python/ray/serve/tests/test_config_files/use_current_working_directory.yaml
+6
View File
@@ -0,0 +1,6 @@
applications:
- name: app1
route_prefix: /
import_path: use_current_working_directory:app
deployments:
- name: f
python/ray/serve/tests/test_config_files/use_custom_autoscaling.yaml
+16
View File
@@ -0,0 +1,16 @@
applications:
- name: app1
route_prefix: /
import_path: ray.serve.tests.test_config_files.use_custom_autoscaling_policy:app
deployments:
- name: CustomAutoscalingPolicy
num_replicas: auto
ray_actor_options:
num_cpus: 0.0
autoscaling_config:
min_replicas: 1
max_replicas: 2
upscale_delay_s: 1
downscale_delay_s: 2
policy:
policy_function: ray.serve.tests.test_config_files.use_custom_autoscaling_policy.custom_autoscaling_policy
python/ray/serve/tests/test_config_files/use_custom_autoscaling_policy.py
+16
View File
@@ -0,0 +1,16 @@
from ray import serve
from ray.serve.config import AutoscalingContext
def custom_autoscaling_policy(ctx: AutoscalingContext):
print("custom_autoscaling_policy")
return 2, {}
@serve.deployment
class CustomAutoscalingPolicy:
def __call__(self):
return "hello_from_custom_autoscaling_policy"
app = CustomAutoscalingPolicy.bind()
python/ray/serve/tests/test_config_files/use_custom_request_router.py
+47
View File
@@ -0,0 +1,47 @@
import random
from typing import (
List,
Optional,
)
from ray import serve
from ray.serve.context import _get_internal_replica_context
from ray.serve.request_router import (
PendingRequest,
ReplicaID,
ReplicaResult,
RequestRouter,
RunningReplica,
)
class UniformRequestRouter(RequestRouter):
async def choose_replicas(
self,
candidate_replicas: List[RunningReplica],
pending_request: Optional[PendingRequest] = None,
) -> List[List[RunningReplica]]:
print("UniformRequestRouter routing request")
index = random.randint(0, len(candidate_replicas) - 1)
return [[candidate_replicas[index]]]
def on_request_routed(
self,
pending_request: PendingRequest,
replica_id: ReplicaID,
result: ReplicaResult,
):
print("on_request_routed callback is called!!")
@serve.deployment
class UniformRequestRouterApp:
def __init__(self):
context = _get_internal_replica_context()
self.replica_id: ReplicaID = context.replica_id
async def __call__(self):
return "hello_from_custom_request_router"
app = UniformRequestRouterApp.bind()

Some files were not shown because too many files have changed in this diff Show More