load("@rules_python//python:defs.bzl", "py_library", "py_test") load("//bazel:python.bzl", "py_test_module_list", "py_test_module_list_with_env_variants") py_library( name = "conftest", srcs = ["conftest.py"], ) py_library( name = "common", srcs = glob(["common/*.py"]), visibility = [ "//python/ray/serve/tests:__subpackages__", ], ) # Minimal installation test (should *not* include conftest). py_test_module_list( size = "small", files = [ "test_minimal_installation.py", ], tags = [ "exclusive", "minimal", "team:serve", ], deps = [ "//python/ray/serve:serve_lib", ], ) # Custom metrics tests. py_test_module_list_with_env_variants( size = "medium", env_variants = { "metr_agg_at_controller": { "env": { "RAY_SERVE_AGGREGATE_METRICS_AT_CONTROLLER": "1", "RAY_SERVE_COLLECT_AUTOSCALING_METRICS_ON_HANDLE": "0", "RAY_SERVE_AUTOSCALING_METRIC_RECORD_INTERVAL_FACTOR": "0.001", "RAY_SERVE_RECORD_AUTOSCALING_STATS_TIMEOUT_S": "3", }, "name_suffix": "_metr_agg_at_controller", }, "metr_agg_at_replicas": { "env": { "RAY_SERVE_AGGREGATE_METRICS_AT_CONTROLLER": "0", "RAY_SERVE_COLLECT_AUTOSCALING_METRICS_ON_HANDLE": "0", "RAY_SERVE_AUTOSCALING_METRIC_RECORD_INTERVAL_FACTOR": "0.001", "RAY_SERVE_RECORD_AUTOSCALING_STATS_TIMEOUT_S": "3", }, "name_suffix": "_metr_agg_at_replicas", }, }, files = [ "test_custom_autoscaling_metrics.py", ], tags = [ "exclusive", "team:serve", ], deps = [ ":common", ":conftest", "//python/ray/serve:serve_lib", ], ) # Small tests. py_test_module_list( size = "small", files = [ "test_advanced.py", "test_cluster_node_info_cache.py", "test_constructor_failure.py", "test_deployment_version.py", "test_enable_task_events.py", "test_expected_versions.py", "test_http_cancellation.py", "test_kv_store.py", "test_long_poll.py", "test_persistence.py", "test_proxy_actor_wrapper.py", "test_replica_request_context.py", "test_serve_with_tracing.py", "test_websockets.py", ], tags = [ "exclusive", "team:serve", ], deps = [ ":common", ":conftest", "//python/ray/serve:serve_lib", ], ) # Medium tests. py_test_module_list( size = "medium", files = [ "test_actor_replica_wrapper.py", "test_backpressure.py", "test_backpressure_grpc.py", "test_batching.py", "test_broadcast.py", "test_callback.py", "test_cluster.py", "test_controller.py", "test_controller_benchmark.py", "test_controller_recovery.py", "test_deploy_2.py", "test_deployment_scheduler_downscale.py", "test_deployment_topology.py", "test_failure.py", "test_failure_2.py", "test_grpc_e2e.py", "test_grpc_replica_wrapper.py", "test_handle.py", "test_handle_1.py", "test_handle_2.py", "test_handle_cancellation.py", "test_handle_streaming.py", "test_healthcheck.py", "test_http_headers.py", "test_http_routes.py", "test_https_proxy.py", "test_list_outbound_deployments.py", "test_max_replicas_per_node.py", "test_multiplex.py", "test_proxy.py", "test_proxy_response_generator.py", "test_queue_monitor.py", "test_ray_client.py", "test_record_replica_metadata.py", "test_record_routing_stats.py", "test_regression.py", "test_replica_placement_group.py", "test_request_timeout.py", "test_round_robin_router.py", "test_streaming_response.py", "test_task_consumer_autoscaling.py", "test_task_processor.py", "test_util.py", ], tags = [ "exclusive", "team:serve", ], deps = [ ":common", ":conftest", "//python/ray/serve:serve_lib", ], ) # Large tests. py_test_module_list( size = "large", files = [ "test_deployment_scheduler.py", ], tags = [ "exclusive", "team:serve", ], deps = [ ":common", ":conftest", "//python/ray/serve:serve_lib", ], ) # Medium tests, don't run on windows. py_test_module_list( size = "medium", env = { "RAY_SERVE_FAIL_ON_RANK_ERROR": "1", }, files = [ "test_gcs_failure.py", "test_gradio.py", "test_replica_ranks.py", ], tags = [ "exclusive", "no_windows", "team:serve", ], deps = [ ":common", ":conftest", "//python/ray/serve:serve_lib", ], ) # Tracing tests (require fixture data and tracing env var for subprocesses). py_test_module_list( size = "medium", data = glob(["fixtures/*.*"]), env = { "RAY_SERVE_TRACING_EXPORTER_IMPORT_PATH": "ray.serve._private.tracing_utils:default_tracing_exporter", "RAY_SERVE_TRACING_SAMPLING_RATIO": "1.0", }, files = [ "test_tracing_utils.py", ], tags = [ "exclusive", "team:serve", ], deps = [ ":common", ":conftest", "//python/ray/serve:serve_lib", ], ) # Large tests. py_test_module_list( size = "enormous", data = glob(["test_config_files/**/*"]), files = [ "test_autoscaling_policy.py", "test_deploy.py", "test_fastapi.py", "test_grpc.py", "test_logging.py", "test_logging_2.py", "test_shutdown.py", "test_standalone.py", "test_standalone_3.py", "test_target_capacity.py", "test_telemetry.py", "test_telemetry_1.py", "test_telemetry_2.py", ], tags = [ "exclusive", "team:serve", ], deps = [ ":common", ":conftest", "//python/ray/serve:serve_lib", ], ) # Logging tests with client IP logging enabled. py_test_module_list( size = "enormous", env = {"RAY_SERVE_LOG_CLIENT_ADDRESS": "1"}, files = [ "test_logging.py", "test_logging_2.py", ], name_suffix = "_with_client_address_logging", tags = [ "exclusive", "team:serve", ], deps = [ ":common", ":conftest", "//python/ray/serve:serve_lib", ], ) # Don't run gang scheduling tests on Windows py_test_module_list( size = "enormous", files = [ "test_gang_scheduling.py", ], tags = [ "exclusive", "no_windows", "team:serve", ], deps = [ ":common", ":conftest", "//python/ray/serve:serve_lib", ], ) # Large tests requiring `test_config_files/`. py_test_module_list( size = "large", data = glob(["test_config_files/**/*"]), files = [ "test_capacity_queue_router.py", "test_cli.py", "test_cli_2.py", "test_cli_3.py", "test_cli_4.py", "test_consistent_hash_router.py", ], tags = [ "exclusive", "team:serve", ], deps = [ ":common", ":conftest", "//python/ray/serve:serve_lib", ], ) # Large tests require `test_config_files/`, no windows. py_test_module_list( size = "large", data = glob(["test_config_files/**/*"]), files = [ "test_standalone_2.py", ], tags = [ "exclusive", "no_windows", "team:serve", ], deps = [ ":common", ":conftest", "//python/ray/serve:serve_lib", ], ) # Run serially on Windows. py_test_module_list( size = "medium", timeout = "long", data = glob(["test_config_files/**/*"]), env = { "RAY_SERVE_ROUTER_QUEUE_LEN_GAUGE_THROTTLE_S": "0", "RAY_SERVE_RUN_SYNC_IN_THREADPOOL": "1", "RAY_SERVE_REPLICA_UTILIZATION_WINDOW_S": "5", "RAY_SERVE_REPLICA_UTILIZATION_REPORT_INTERVAL_S": "1", "RAY_SERVE_REPLICA_UTILIZATION_NUM_BUCKETS": "10", "RAY_SERVE_REPLICA_MAX_PROCESSING_LATENCY_WINDOW_S": "5", "RAY_SERVE_REPLICA_MAX_PROCESSING_LATENCY_REPORT_INTERVAL_S": "1", "RAY_SERVE_REPLICA_MAX_PROCESSING_LATENCY_NUM_BUCKETS": "5", "RAY_SERVE_STATUS_GAUGE_REPORT_INTERVAL_S": "0.1", }, files = [ "test_deploy_app.py", "test_deploy_app_2.py", "test_metrics.py", "test_metrics_2.py", "test_metrics_3.py", ], tags = [ "exclusive", "team:serve", "use_all_core_windows", ], deps = [ ":common", ":conftest", "//python/ray/serve:serve_lib", ], ) # Run the controller metric high-cardinality opt-out test with the flag disabled. py_test_module_list( size = "medium", args = [ "-k", "test_disable_high_cardinality_controller_metrics", ], env = { "RAY_SERVE_CONTROLLER_METRICS_INCLUDE_HIGH_CARDINALITY_TAGS": "0", }, files = [ "test_metrics.py", ], name_suffix = "_controller_metrics_without_high_cardinality", tags = [ "exclusive", "team:serve", "use_all_core_windows", ], deps = [ ":common", ":conftest", "//python/ray/serve:serve_lib", ], ) # Deployment actors tests: split into own target with size=large because # the test file has 39 tests including heavyweight crash-recovery tests # with 120s waits that exceed the medium/long (900s) timeout under CI load. py_test_module_list( size = "large", data = glob(["test_config_files/**/*"]), env = { "RAY_SERVE_ROUTER_QUEUE_LEN_GAUGE_THROTTLE_S": "0", "RAY_SERVE_RUN_SYNC_IN_THREADPOOL": "1", "RAY_SERVE_REPLICA_UTILIZATION_WINDOW_S": "5", "RAY_SERVE_REPLICA_UTILIZATION_REPORT_INTERVAL_S": "1", "RAY_SERVE_REPLICA_UTILIZATION_NUM_BUCKETS": "10", "RAY_SERVE_STATUS_GAUGE_REPORT_INTERVAL_S": "0.1", }, files = [ "test_deployment_actors.py", "test_deployment_actors_recovery.py", ], tags = [ "exclusive", "team:serve", "use_all_core_windows", ], deps = [ ":common", ":conftest", "//python/ray/serve:serve_lib", ], ) # Minimal tests py_test_module_list( size = "large", files = [ "test_api.py", ], tags = [ "exclusive", "minimal", "team:serve", ], deps = [ ":common", ":conftest", "//python/ray/serve:serve_lib", ], ) # API tests that run faster py_test_module_list( size = "small", files = [ "test_api_2.py", ], tags = [ "exclusive", "minimal", "team:serve", ], deps = [ ":common", ":conftest", "//python/ray/serve:serve_lib", ], ) # Model composition test that needs medium size py_test_module_list( size = "medium", timeout = "moderate", files = [ "test_model_composition.py", ], tags = [ "exclusive", "minimal", "team:serve", ], deps = [ ":common", ":conftest", "//python/ray/serve:serve_lib", ], ) # Post-wheel-build tests. py_test_module_list( size = "large", files = [ "test_runtime_env.py", "test_runtime_env_2.py", ], tags = [ "custom_setup", "exclusive", "post_wheel_build", "team:serve", ], deps = [ ":common", ":conftest", "//python/ray/serve:serve_lib", ], ) # Runs test_api and test_failure with injected failures in the controller. py_test( name = "test_controller_crashes", size = "large", srcs = [ "test_api.py", "test_api_2.py", "test_controller_crashes.py", "test_failure.py", "test_failure_2.py", ], tags = [ "exclusive", "team:serve", ], deps = [ ":common", ":conftest", "//python/ray/serve:serve_lib", ], ) # Serve HA. py_test( name = "test_serve_ha", size = "medium", srcs = ["test_serve_ha.py"], tags = [ "custom_setup", "exclusive", "ha_integration", "team:serve", ], deps = [ ":common", ":conftest", "//python/ray/serve:serve_lib", ], ) # ----- TEST FEATURE FLAGS ----- # Test autoscaling with different metric collection configurations AUTOSCALING_METRIC_ENV_VARIANTS = { "metr_disab": { "env": { "RAY_SERVE_COLLECT_AUTOSCALING_METRICS_ON_HANDLE": "0", # Make sure queued metrics are cleared out quickly. "RAY_SERVE_HANDLE_AUTOSCALING_METRIC_PUSH_INTERVAL_S": "0.1", }, "name_suffix": "_metr_disab", }, "metr_agg_at_controller": { "env": { "RAY_SERVE_AGGREGATE_METRICS_AT_CONTROLLER": "1", # Make sure queued metrics are cleared out quickly. "RAY_SERVE_HANDLE_AUTOSCALING_METRIC_PUSH_INTERVAL_S": "0.1", }, "name_suffix": "_metr_agg_at_controller", }, "metr_agg_at_controller_and_replicas": { "env": { "RAY_SERVE_AGGREGATE_METRICS_AT_CONTROLLER": "1", "RAY_SERVE_COLLECT_AUTOSCALING_METRICS_ON_HANDLE": "0", # Make sure queued metrics are cleared out quickly. "RAY_SERVE_HANDLE_AUTOSCALING_METRIC_PUSH_INTERVAL_S": "0.1", }, "name_suffix": "_metr_agg_at_controller_and_replicas", }, } py_test_module_list_with_env_variants( size = "large", env_variants = AUTOSCALING_METRIC_ENV_VARIANTS, files = [ "test_autoscaling_policy.py", "test_deploy.py", "test_standalone_3.py", "test_target_capacity.py", ], tags = [ "autoscaling", "exclusive", "team:serve", ], deps = [ ":common", ":conftest", "//python/ray/serve:serve_lib", ], ) # Test autoscaling with streaming and unary configurations. # Regression test for https://github.com/ray-project/ray/issues/61551 AUTOSCALING_ENV_VARIANTS = { "env_overrides": { "env": { "RAY_SERVE_LOG_TO_STDERR": "0", "RAY_SERVE_RUN_ROUTER_IN_SEPARATE_LOOP": "0", "RAY_SERVE_USE_GRPC_BY_DEFAULT": "1", }, "name_suffix": "_env_overrides", }, "throughput_optimized": { "env": { "RAY_SERVE_THROUGHPUT_OPTIMIZED": "1", }, "name_suffix": "_throughput_optimized", }, } py_test_module_list_with_env_variants( size = "large", args = [ "-k", "TestAutoscalingWithRejection", ], env_variants = AUTOSCALING_ENV_VARIANTS, files = [ "test_autoscaling_policy.py", ], tags = [ "exclusive", "team:serve", ], deps = [ ":common", ":conftest", "//python/ray/serve:serve_lib", ], ) # Test feature flag for task events. py_test_module_list( size = "small", data = glob(["test_config_files/**/*"]), env = {"RAY_SERVE_ENABLE_TASK_EVENTS": "1"}, files = [ "test_enable_task_events.py", ], name_suffix = "_with_task_events_enabled", tags = [ "exclusive", "no_windows", "team:serve", ], deps = [ ":common", ":conftest", "//python/ray/serve:serve_lib", ], ) # Medium tests with pack scheduling py_test_module_list( size = "medium", data = glob(["test_config_files/**/*"]), env = {"RAY_SERVE_USE_PACK_SCHEDULING_STRATEGY": "1"}, files = [ "test_cluster.py", "test_controller_recovery.py", "test_gcs_failure.py", "test_max_replicas_per_node.py", "test_replica_placement_group.py", ], name_suffix = "_with_pack_scheduling", tags = [ "exclusive", "no_windows", "team:serve", ], deps = [ ":common", ":conftest", "//python/ray/serve:serve_lib", ], ) # Large tests with pack scheduling py_test_module_list( size = "enormous", env = {"RAY_SERVE_USE_PACK_SCHEDULING_STRATEGY": "1"}, files = [ "test_standalone.py", "test_standalone_3.py", ], name_suffix = "_with_pack_sche", tags = [ "exclusive", "team:serve", ], deps = [ ":common", ":conftest", "//python/ray/serve:serve_lib", ], ) # Large tests with pack scheduling, no windows py_test_module_list( size = "large", data = glob(["test_config_files/**/*"]), env = {"RAY_SERVE_USE_PACK_SCHEDULING_STRATEGY": "1"}, files = [ "test_deployment_scheduler.py", "test_deployment_scheduler_downscale.py", "test_gang_scheduling.py", "test_standalone_2.py", ], name_suffix = "_with_pack_scheduling", tags = [ "exclusive", "no_windows", "team:serve", ], deps = [ ":common", ":conftest", "//python/ray/serve:serve_lib", ], ) # Test handle API with local testing mode. py_test_module_list( size = "small", env = {"RAY_SERVE_FORCE_LOCAL_TESTING_MODE": "1"}, files = [ "test_handle_1.py", "test_handle_2.py", "test_handle_cancellation.py", "test_handle_streaming.py", ], name_suffix = "_with_local_testing_mode", tags = [ "exclusive", "no_windows", "team:serve", ], deps = [ ":common", ":conftest", "//python/ray/serve:serve_lib", ], ) # Test currently off-by-default behavior to run replica sync methods in a threadpool. # TODO(edoakes): remove this once the FF is flipped on by default. py_test_module_list( size = "medium", env = {"RAY_SERVE_RUN_SYNC_IN_THREADPOOL": "1"}, files = [ "test_replica_sync_methods.py", ], name_suffix = "_with_run_sync_in_threadpool", tags = [ "exclusive", "no_windows", "team:serve", ], deps = [ ":common", ":conftest", "//python/ray/serve:serve_lib", ], ) py_test_module_list( size = "medium", env = {"RAY_SERVE_RUN_ROUTER_IN_SEPARATE_LOOP": "0"}, files = [ "test_handle_same_loop.py", "test_proxy.py", ], name_suffix = "_with_router_in_same_loop", tags = [ "exclusive", "no_windows", "team:serve", ], deps = [ ":common", ":conftest", "//python/ray/serve:serve_lib", ], ) py_test_module_list( size = "large", files = [ "test_direct_ingress.py", ], tags = [ "direct_ingress", "exclusive", "team:serve", ], deps = [ ":common", ":conftest", "//python/ray/serve:serve_lib", ], ) py_test_module_list( size = "large", timeout = "eternal", files = [ "test_per_request_headers.py", ], tags = [ "exclusive", "team:serve", ], deps = [ ":common", ":conftest", "//python/ray/serve:serve_lib", ], ) py_test_module_list( size = "large", env = {"RAY_SERVE_USE_GRPC_BY_DEFAULT": "1"}, files = [ "test_direct_ingress.py", ], name_suffix = "_with_grpc", tags = [ "direct_ingress", "exclusive", "team:serve", ], deps = [ ":common", ":conftest", "//python/ray/serve:serve_lib", ], ) py_test_module_list( size = "large", timeout = "eternal", env = {"RAY_SERVE_USE_GRPC_BY_DEFAULT": "1"}, files = [ "test_per_request_headers.py", ], name_suffix = "_with_grpc", tags = [ "exclusive", "team:serve", ], deps = [ ":common", ":conftest", "//python/ray/serve:serve_lib", ], ) # HAProxy tests need RAY_SERVE_ENABLE_HA_PROXY=1. The HAProxy binary comes from # the ray-haproxy package in the serve test dependencies. py_test_module_list_with_env_variants( size = "large", env_variants = { "system": { "env": { "RAY_SERVE_ENABLE_HA_PROXY": "1", "RAY_SERVE_DIRECT_INGRESS_MIN_DRAINING_PERIOD_S": "0.01", "RAY_SERVE_STATUS_GAUGE_REPORT_INTERVAL_S": "0.1", }, "name_suffix": "", }, }, files = [ "test_haproxy.py", "test_haproxy_api.py", "test_haproxy_metrics.py", "test_metrics_haproxy.py", ], tags = [ "exclusive", "haproxy", "no_windows", "team:serve", ], deps = [ ":common", ":conftest", "//python/ray/serve:serve_lib", ], )