import math import sys import time from typing import Dict, List, Optional, Tuple from unittest.mock import Mock, PropertyMock, patch import cloudpickle import pytest from fastapi import FastAPI from pydantic import ValidationError from ray import serve from ray.exceptions import RayTaskError from ray.serve._private.application_state import ( CHECKPOINT_KEY, ApplicationState, ApplicationStateManager, ApplicationStatusInfo, BuildAppStatus, StatusOverview, build_serve_application, override_deployment_info, ) from ray.serve._private.autoscaling_state import AutoscalingStateManager from ray.serve._private.build_app import CUSTOM_INGRESS_REQUEST_ROUTER_UNSUPPORTED_ERROR from ray.serve._private.common import ( RUNNING_REQUESTS_KEY, DeploymentHandleSource, DeploymentID, DeploymentStatus, DeploymentStatusInfo, DeploymentStatusTrigger, HandleMetricReport, ReplicaID, ReplicaMetricReport, TimeStampedValue, ) from ray.serve._private.config import DeploymentConfig, ReplicaConfig from ray.serve._private.constants import ( CONTROL_LOOP_INTERVAL_S, RAY_SERVE_COLLECT_AUTOSCALING_METRICS_ON_HANDLE, ) from ray.serve._private.deploy_utils import deploy_args_to_deployment_info from ray.serve._private.deployment_info import DeploymentInfo from ray.serve._private.test_utils import MockKVStore from ray.serve._private.utils import get_random_string from ray.serve._private.version import DeploymentVersion from ray.serve.config import ( AutoscalingConfig, DeploymentActorConfig, GangSchedulingConfig, RequestRouterConfig, ) from ray.serve.exceptions import RayServeException from ray.serve.experimental.round_robin_router import RoundRobinRouter from ray.serve.generated.serve_pb2 import ( ApplicationArgs as ApplicationArgsProto, ApplicationStatusInfo as ApplicationStatusInfoProto, StatusOverview as StatusOverviewProto, ) from ray.serve.schema import ( APIType, ApplicationStatus, DeploymentSchema, LoggingConfig, ServeApplicationSchema, ) class MockEndpointState: def __init__(self): self.endpoints = dict() def update_endpoint(self, endpoint, endpoint_info): self.endpoints[endpoint] = endpoint_info def delete_endpoint(self, endpoint): if endpoint in self.endpoints: del self.endpoints[endpoint] class MockDeploymentStateManager: def __init__(self, kv_store): self.kv_store = kv_store self.deployment_infos: Dict[DeploymentID, DeploymentInfo] = dict() self.deployment_statuses: Dict[DeploymentID, DeploymentStatusInfo] = dict() self.deleting: Dict[DeploymentID, bool] = dict() # Recover recovered_deployments = self.kv_store.get("fake_deployment_state_checkpoint") if recovered_deployments is not None: for name, checkpointed_data in recovered_deployments.items(): (info, deleting) = checkpointed_data self.deployment_infos[name] = info self.deployment_statuses[name] = DeploymentStatusInfo( name=name, status=DeploymentStatus.UPDATING, status_trigger=DeploymentStatusTrigger.CONFIG_UPDATE_STARTED, message="", ) self.deleting[name] = deleting self._scaling_decisions = {} def deploy( self, deployment_id: DeploymentID, deployment_info: DeploymentInfo, ): existing_info = self.deployment_infos.get(deployment_id) self.deleting[deployment_id] = False self.deployment_infos[deployment_id] = deployment_info if not existing_info or existing_info.version != deployment_info.version: self.deployment_statuses[deployment_id] = DeploymentStatusInfo( name=deployment_id.name, status=DeploymentStatus.UPDATING, status_trigger=DeploymentStatusTrigger.CONFIG_UPDATE_STARTED, message="", ) self.kv_store.put( "fake_deployment_state_checkpoint", dict( zip( self.deployment_infos.keys(), zip(self.deployment_infos.values(), self.deleting.values()), ) ), ) @property def deployments(self) -> List[str]: return list(self.deployment_infos.keys()) def get_deployment_statuses(self, ids: List[DeploymentID]): return [self.deployment_statuses[id] for id in ids] def get_deployment(self, deployment_id: DeploymentID) -> DeploymentInfo: if deployment_id in self.deployment_statuses: # Return dummy deployment info object return DeploymentInfo( deployment_config=DeploymentConfig( num_replicas=self.deployment_infos[ deployment_id ].deployment_config.num_replicas, user_config={}, ), replica_config=ReplicaConfig.create(lambda x: x), start_time_ms=0, deployer_job_id="", ) def get_deployments_in_application(self, app_name: str): deployments = [] for deployment_id in self.deployment_infos: if deployment_id.app_name == app_name: deployments.append(deployment_id.name) return deployments def set_deployment_unhealthy(self, id: DeploymentID): self.deployment_statuses[id].status = DeploymentStatus.UNHEALTHY def set_deployment_deploy_failed(self, id: DeploymentID): self.deployment_statuses[id].status = DeploymentStatus.DEPLOY_FAILED def set_deployment_healthy(self, id: DeploymentID): self.deployment_statuses[id].status = DeploymentStatus.HEALTHY def set_deployment_updating(self, id: DeploymentID): self.deployment_statuses[id].status = DeploymentStatus.UPDATING def set_deployment_deleted(self, id: str): if not self.deployment_infos[id]: raise ValueError( f"Tried to mark deployment {id} as deleted, but {id} not found" ) if not self.deleting[id]: raise ValueError( f"Tried to mark deployment {id} as deleted, but delete_deployment()" f"hasn't been called for {id} yet" ) del self.deployment_infos[id] del self.deployment_statuses[id] del self.deleting[id] def delete_deployment(self, id: DeploymentID): self.deleting[id] = True def get_deployment_target_num_replicas(self, id: DeploymentID) -> Optional[int]: return self.deployment_infos[id].deployment_config.num_replicas def save_checkpoint(self): """Mock save checkpoint method.""" pass def autoscale(self, id: DeploymentID, target_num_replicas: int): self._scaling_decisions[id] = target_num_replicas return True def get_deployment_route_patterns(self, id: DeploymentID) -> Optional[List[str]]: return None def get_deployment_outbound_deployments( self, id: DeploymentID ) -> Optional[List[DeploymentID]]: """Mock method to return outbound deployments for a deployment.""" # Return None by default, tests can override this return getattr(self, f"_outbound_deps_{id.name}_{id.app_name}", None) @pytest.fixture def mocked_application_state_manager() -> ( Tuple[ApplicationStateManager, MockDeploymentStateManager] ): kv_store = MockKVStore() deployment_state_manager = MockDeploymentStateManager(kv_store) application_state_manager = ApplicationStateManager( deployment_state_manager, AutoscalingStateManager(), MockEndpointState(), kv_store, LoggingConfig(), ) yield application_state_manager, deployment_state_manager, kv_store def deployment_params( name: str, route_prefix: str = None, autoscaling_config: AutoscalingConfig = None, num_replicas: int = 1, ingress_request_router: bool = False, ): return { "deployment_name": name, "deployment_config_proto_bytes": DeploymentConfig( num_replicas=num_replicas, user_config={}, version=get_random_string(), autoscaling_config=autoscaling_config, ).to_proto_bytes(), "replica_config_proto_bytes": ReplicaConfig.create( lambda x: x ).to_proto_bytes(), "deployer_job_id": "random", "route_prefix": route_prefix, "ingress": route_prefix is not None, "ingress_request_router": ingress_request_router, "serialized_autoscaling_policy_def": None, "serialized_request_router_cls": None, } def deployment_info( name: str, route_prefix: str = None, autoscaling_config: AutoscalingConfig = None, num_replicas: int = 1, ingress_request_router: bool = False, ): params = deployment_params( name, route_prefix, autoscaling_config, num_replicas, ingress_request_router, ) return deploy_args_to_deployment_info(**params, app_name="test_app") class TestGracefulShutdownTimeoutFloor: """deploy_args_to_deployment_info floors graceful_shutdown_timeout_s to the direct-ingress min draining period (plus buffer) for ingress deployments, so the controller's force-kill deadline can't cut the replica's drain short.""" @staticmethod def _params(*, graceful_shutdown_timeout_s, ingress): return { "deployment_name": "d", "deployment_config_proto_bytes": DeploymentConfig( graceful_shutdown_timeout_s=graceful_shutdown_timeout_s, version=get_random_string(), ).to_proto_bytes(), "replica_config_proto_bytes": ReplicaConfig.create( lambda x: x ).to_proto_bytes(), "deployer_job_id": "random", "route_prefix": "/" if ingress else None, "ingress": ingress, } def _timeout(self, **kwargs): info = deploy_args_to_deployment_info(**self._params(**kwargs), app_name="app") return info.deployment_config.graceful_shutdown_timeout_s @patch.multiple( "ray.serve._private.deploy_utils", RAY_SERVE_ENABLE_DIRECT_INGRESS=True, RAY_SERVE_DIRECT_INGRESS_MIN_DRAINING_PERIOD_S=30, RAY_SERVE_DIRECT_INGRESS_SHUTDOWN_BUFFER_S=5, ) def test_ingress_below_floor_is_raised(self): # max(10, 30 + 5) == 35 assert self._timeout(graceful_shutdown_timeout_s=10, ingress=True) == 35 @patch.multiple( "ray.serve._private.deploy_utils", RAY_SERVE_ENABLE_DIRECT_INGRESS=True, RAY_SERVE_DIRECT_INGRESS_MIN_DRAINING_PERIOD_S=30, RAY_SERVE_DIRECT_INGRESS_SHUTDOWN_BUFFER_S=5, ) def test_ingress_above_floor_is_unchanged(self): assert self._timeout(graceful_shutdown_timeout_s=60, ingress=True) == 60 @patch.multiple( "ray.serve._private.deploy_utils", RAY_SERVE_ENABLE_DIRECT_INGRESS=True, RAY_SERVE_DIRECT_INGRESS_MIN_DRAINING_PERIOD_S=30, RAY_SERVE_DIRECT_INGRESS_SHUTDOWN_BUFFER_S=5, ) def test_non_ingress_is_not_floored(self): assert self._timeout(graceful_shutdown_timeout_s=10, ingress=False) == 10 @patch.multiple( "ray.serve._private.deploy_utils", RAY_SERVE_ENABLE_DIRECT_INGRESS=False, RAY_SERVE_DIRECT_INGRESS_MIN_DRAINING_PERIOD_S=30, RAY_SERVE_DIRECT_INGRESS_SHUTDOWN_BUFFER_S=5, ) def test_not_floored_when_direct_ingress_disabled(self): assert self._timeout(graceful_shutdown_timeout_s=10, ingress=True) == 10 def test_build_serve_application_excludes_router_from_fastapi_ingress_count(): ingress_api = FastAPI() router_api = FastAPI() @serve.deployment @serve.ingress(ingress_api) class LLMServer: pass @serve.deployment @serve.ingress(router_api) class IngressRequestRouter: pass llm_server = LLMServer.bind() app = llm_server._with_ingress_request_router( IngressRequestRouter.bind(llm_deployment=llm_server) ) runtime_context = Mock() runtime_context.runtime_env = {} runtime_context.get_job_id.return_value = "job-id" with ( patch("ray.serve._private.application_state.import_attr", return_value=app), patch( "ray.serve._private.application_state.ray.get_runtime_context", return_value=runtime_context, ), patch("ray.serve._private.application_state.configure_component_logger"), patch("ray.serve._private.build_app.RAY_SERVE_ENABLE_HA_PROXY", True), ): _, deploy_args, error = build_serve_application._function( "module.app", "code-version", "default", {}, LoggingConfig(), None, {}, {}, {}, ) assert error is None assert [ (args["deployment_name"], args["ingress_request_router"]) for args in deploy_args ] == [ ("LLMServer", False), ("IngressRequestRouter", True), ] @pytest.fixture def mocked_application_state() -> Tuple[ApplicationState, MockDeploymentStateManager]: kv_store = MockKVStore() deployment_state_manager = MockDeploymentStateManager(kv_store) application_state = ApplicationState( name="test_app", deployment_state_manager=deployment_state_manager, autoscaling_state_manager=AutoscalingStateManager(), endpoint_state=MockEndpointState(), logging_config=LoggingConfig(), external_scaler_enabled=False, ) yield application_state, deployment_state_manager def test_application_state_clears_stale_ingress_request_router( mocked_application_state, ): application_state, _ = mocked_application_state application_state._set_target_state( {"Router": deployment_info("Router", ingress_request_router=True)}, api_type=APIType.IMPERATIVE, code_version="1", target_config=None, ) assert application_state.ingress_request_router_deployment == "Router" application_state._set_target_state( {"Main": deployment_info("Main")}, api_type=APIType.IMPERATIVE, code_version="2", target_config=None, ) assert application_state.ingress_request_router_deployment is None class TestApplicationStatusInfo: def test_application_status_required(self): with pytest.raises(TypeError): ApplicationStatusInfo( message="context about status", deployment_timestamp=time.time() ) @pytest.mark.parametrize("status", list(ApplicationStatus)) def test_proto(self, status): serve_application_status_info = ApplicationStatusInfo( status=status, message="context about status", deployment_timestamp=time.time(), ) serialized_proto = serve_application_status_info.to_proto().SerializeToString() deserialized_proto = ApplicationStatusInfoProto.FromString(serialized_proto) reconstructed_info = ApplicationStatusInfo.from_proto(deserialized_proto) assert serve_application_status_info == reconstructed_info class TestStatusOverview: def get_valid_serve_application_status_info(self): return ApplicationStatusInfo( status=ApplicationStatus.RUNNING, message="", deployment_timestamp=time.time(), ) def test_app_status_required(self): with pytest.raises(TypeError): StatusOverview(deployment_statuses=[]) def test_empty_list_valid(self): """Should be able to create StatusOverview with no deployment statuses.""" # Check default is empty list status_info = StatusOverview( app_status=self.get_valid_serve_application_status_info() ) status_info.deployment_statuses == [] # Ensure empty list can be passed in explicitly status_info = StatusOverview( app_status=self.get_valid_serve_application_status_info(), deployment_statuses=[], ) status_info.deployment_statuses == [] def test_equality_mismatched_deployment_statuses(self): """Check that StatusOverviews with different numbers of statuses are unequal.""" status_info_few_deployments = StatusOverview( app_status=self.get_valid_serve_application_status_info(), deployment_statuses=[ DeploymentStatusInfo( name="1", status=DeploymentStatus.HEALTHY, status_trigger=DeploymentStatusTrigger.CONFIG_UPDATE_STARTED, ), DeploymentStatusInfo( name="2", status=DeploymentStatus.UNHEALTHY, status_trigger=DeploymentStatusTrigger.CONFIG_UPDATE_STARTED, ), ], ) status_info_many_deployments = StatusOverview( app_status=self.get_valid_serve_application_status_info(), deployment_statuses=[ DeploymentStatusInfo( name="1", status=DeploymentStatus.HEALTHY, status_trigger=DeploymentStatusTrigger.CONFIG_UPDATE_STARTED, ), DeploymentStatusInfo( name="2", status=DeploymentStatus.UNHEALTHY, status_trigger=DeploymentStatusTrigger.CONFIG_UPDATE_STARTED, ), DeploymentStatusInfo( name="3", status=DeploymentStatus.UNHEALTHY, status_trigger=DeploymentStatusTrigger.CONFIG_UPDATE_STARTED, ), DeploymentStatusInfo( name="4", status=DeploymentStatus.UPDATING, status_trigger=DeploymentStatusTrigger.CONFIG_UPDATE_STARTED, ), ], ) assert status_info_few_deployments != status_info_many_deployments @pytest.mark.parametrize("application_status", list(ApplicationStatus)) def test_proto(self, application_status): status_info = StatusOverview( app_status=ApplicationStatusInfo( status=application_status, message="context about this status", deployment_timestamp=time.time(), ), deployment_statuses=[ DeploymentStatusInfo( name="name1", status=DeploymentStatus.UPDATING, message="deployment updating", status_trigger=DeploymentStatusTrigger.CONFIG_UPDATE_STARTED, ), DeploymentStatusInfo( name="name2", status=DeploymentStatus.HEALTHY, message="", status_trigger=DeploymentStatusTrigger.CONFIG_UPDATE_STARTED, ), DeploymentStatusInfo( name="name3", status=DeploymentStatus.UNHEALTHY, message="this deployment is unhealthy", status_trigger=DeploymentStatusTrigger.CONFIG_UPDATE_STARTED, ), ], ) serialized_proto = status_info.to_proto().SerializeToString() deserialized_proto = StatusOverviewProto.FromString(serialized_proto) reconstructed_info = StatusOverview.from_proto(deserialized_proto) assert status_info == reconstructed_info @patch.object( ApplicationState, "target_deployments", PropertyMock(return_value=["a", "b", "c"]) ) class TestDetermineAppStatus: @patch.object(ApplicationState, "get_deployments_statuses") def test_running(self, get_deployments_statuses, mocked_application_state): app_state, _ = mocked_application_state get_deployments_statuses.return_value = [ DeploymentStatusInfo( "a", DeploymentStatus.HEALTHY, DeploymentStatusTrigger.CONFIG_UPDATE_STARTED, ), DeploymentStatusInfo( "b", DeploymentStatus.HEALTHY, DeploymentStatusTrigger.CONFIG_UPDATE_STARTED, ), DeploymentStatusInfo( "c", DeploymentStatus.HEALTHY, DeploymentStatusTrigger.CONFIG_UPDATE_STARTED, ), ] assert app_state._determine_app_status() == (ApplicationStatus.RUNNING, "") @patch.object(ApplicationState, "get_deployments_statuses") def test_stay_running(self, get_deployments_statuses, mocked_application_state): app_state, _ = mocked_application_state app_state._status = ApplicationStatus.RUNNING get_deployments_statuses.return_value = [ DeploymentStatusInfo( "a", DeploymentStatus.HEALTHY, DeploymentStatusTrigger.CONFIG_UPDATE_STARTED, ), DeploymentStatusInfo( "b", DeploymentStatus.HEALTHY, DeploymentStatusTrigger.CONFIG_UPDATE_STARTED, ), DeploymentStatusInfo( "c", DeploymentStatus.HEALTHY, DeploymentStatusTrigger.CONFIG_UPDATE_STARTED, ), ] assert app_state._determine_app_status() == (ApplicationStatus.RUNNING, "") @patch.object(ApplicationState, "get_deployments_statuses") def test_deploying(self, get_deployments_statuses, mocked_application_state): app_state, _ = mocked_application_state get_deployments_statuses.return_value = [ DeploymentStatusInfo( "a", DeploymentStatus.UPDATING, DeploymentStatusTrigger.CONFIG_UPDATE_STARTED, ), DeploymentStatusInfo( "b", DeploymentStatus.HEALTHY, DeploymentStatusTrigger.CONFIG_UPDATE_STARTED, ), DeploymentStatusInfo( "c", DeploymentStatus.HEALTHY, DeploymentStatusTrigger.CONFIG_UPDATE_STARTED, ), ] assert app_state._determine_app_status() == (ApplicationStatus.DEPLOYING, "") @patch.object(ApplicationState, "get_deployments_statuses") def test_deploy_failed(self, get_deployments_statuses, mocked_application_state): app_state, _ = mocked_application_state get_deployments_statuses.return_value = [ DeploymentStatusInfo( "a", DeploymentStatus.UPDATING, DeploymentStatusTrigger.CONFIG_UPDATE_STARTED, ), DeploymentStatusInfo( "b", DeploymentStatus.HEALTHY, DeploymentStatusTrigger.CONFIG_UPDATE_STARTED, ), DeploymentStatusInfo( "c", DeploymentStatus.DEPLOY_FAILED, DeploymentStatusTrigger.CONFIG_UPDATE_STARTED, ), ] status, error_msg = app_state._determine_app_status() assert status == ApplicationStatus.DEPLOY_FAILED assert error_msg @patch.object(ApplicationState, "get_deployments_statuses") def test_unhealthy(self, get_deployments_statuses, mocked_application_state): app_state, _ = mocked_application_state app_state._status = ApplicationStatus.RUNNING get_deployments_statuses.return_value = [ DeploymentStatusInfo( "a", DeploymentStatus.HEALTHY, DeploymentStatusTrigger.CONFIG_UPDATE_STARTED, ), DeploymentStatusInfo( "b", DeploymentStatus.HEALTHY, DeploymentStatusTrigger.CONFIG_UPDATE_STARTED, ), DeploymentStatusInfo( "c", DeploymentStatus.UNHEALTHY, DeploymentStatusTrigger.CONFIG_UPDATE_STARTED, ), ] status, error_msg = app_state._determine_app_status() assert status == ApplicationStatus.UNHEALTHY assert error_msg @patch.object(ApplicationState, "get_deployments_statuses") def test_autoscaling(self, get_deployments_statuses, mocked_application_state): app_state, _ = mocked_application_state app_state._status = ApplicationStatus.RUNNING get_deployments_statuses.return_value = [ DeploymentStatusInfo( "a", DeploymentStatus.HEALTHY, DeploymentStatusTrigger.CONFIG_UPDATE_STARTED, ), DeploymentStatusInfo( "b", DeploymentStatus.UPSCALING, DeploymentStatusTrigger.AUTOSCALING ), DeploymentStatusInfo( "c", DeploymentStatus.DOWNSCALING, DeploymentStatusTrigger.AUTOSCALING ), ] status, error_msg = app_state._determine_app_status() assert status == ApplicationStatus.RUNNING @patch.object(ApplicationState, "get_deployments_statuses") def test_manual_scale_num_replicas( self, get_deployments_statuses, mocked_application_state ): app_state, _ = mocked_application_state app_state._status = ApplicationStatus.RUNNING get_deployments_statuses.return_value = [ DeploymentStatusInfo( "a", DeploymentStatus.HEALTHY, DeploymentStatusTrigger.CONFIG_UPDATE_STARTED, ), DeploymentStatusInfo( "b", DeploymentStatus.UPSCALING, DeploymentStatusTrigger.CONFIG_UPDATE_STARTED, ), DeploymentStatusInfo( "c", DeploymentStatus.DOWNSCALING, DeploymentStatusTrigger.CONFIG_UPDATE_STARTED, ), ] status, error_msg = app_state._determine_app_status() assert status == ApplicationStatus.DEPLOYING def test_deploy_and_delete_app(mocked_application_state): """Deploy app with 2 deployments, transition DEPLOYING -> RUNNING -> DELETING. This tests the basic typical workflow. """ app_state, deployment_state_manager = mocked_application_state # DEPLOY application with deployments {d1, d2} d1_id = DeploymentID(name="d1", app_name="test_app") d2_id = DeploymentID(name="d2", app_name="test_app") app_state.deploy_app( { "d1": deployment_info("d1", "/hi"), "d2": deployment_info("d2"), }, ApplicationArgsProto(external_scaler_enabled=False), ) assert app_state.route_prefix == "/hi" app_status = app_state.get_application_status_info() assert app_status.status == ApplicationStatus.DEPLOYING assert app_status.deployment_timestamp > 0 app_state.update() # After one update, deployments {d1, d2} should be created assert deployment_state_manager.get_deployment(d1_id) assert deployment_state_manager.get_deployment(d2_id) assert app_state.status == ApplicationStatus.DEPLOYING # Until both deployments are healthy, app should be deploying app_state.update() assert app_state.status == ApplicationStatus.DEPLOYING # Mark deployment d1 healthy (app should be deploying) deployment_state_manager.set_deployment_healthy(d1_id) app_state.update() assert app_state.status == ApplicationStatus.DEPLOYING # Mark deployment d2 healthy (app should be running) deployment_state_manager.set_deployment_healthy(d2_id) app_state.update() assert app_state.status == ApplicationStatus.RUNNING # Rerun update, status shouldn't change (still running) app_state.update() assert app_state.status == ApplicationStatus.RUNNING # Delete application (app should be deleting) app_state.delete() assert app_state.status == ApplicationStatus.DELETING app_state.update() deployment_state_manager.set_deployment_deleted(d1_id) ready_to_be_deleted, _ = app_state.update() assert not ready_to_be_deleted assert app_state.status == ApplicationStatus.DELETING # Once both deployments are deleted, the app should be ready to delete deployment_state_manager.set_deployment_deleted(d2_id) ready_to_be_deleted = app_state.update() assert ready_to_be_deleted def test_app_deploy_failed_and_redeploy(mocked_application_state): """Test DEPLOYING -> DEPLOY_FAILED -> (redeploy) -> DEPLOYING -> RUNNING""" app_state, deployment_state_manager = mocked_application_state d1_id = DeploymentID(name="d1", app_name="test_app") d2_id = DeploymentID(name="d2", app_name="test_app") app_state.deploy_app( {"d1": deployment_info("d1")}, ApplicationArgsProto(external_scaler_enabled=False), ) assert app_state.status == ApplicationStatus.DEPLOYING # Before status of deployment changes, app should still be DEPLOYING app_state.update() assert app_state.status == ApplicationStatus.DEPLOYING # Mark deployment unhealthy -> app should be DEPLOY_FAILED deployment_state_manager.set_deployment_deploy_failed(d1_id) app_state.update() assert app_state.status == ApplicationStatus.DEPLOY_FAILED # Message and status should not change deploy_failed_msg = app_state._status_msg assert len(deploy_failed_msg) != 0 app_state.update() assert app_state.status == ApplicationStatus.DEPLOY_FAILED assert app_state._status_msg == deploy_failed_msg app_state.deploy_app( {"d1": deployment_info("d1"), "d2": deployment_info("d2")}, ApplicationArgsProto(external_scaler_enabled=False), ) assert app_state.status == ApplicationStatus.DEPLOYING assert app_state._status_msg != deploy_failed_msg # After one update, deployments {d1, d2} should be created app_state.update() assert deployment_state_manager.get_deployment(d1_id) assert deployment_state_manager.get_deployment(d2_id) assert app_state.status == ApplicationStatus.DEPLOYING deployment_state_manager.set_deployment_healthy(d1_id) deployment_state_manager.set_deployment_healthy(d2_id) app_state.update() assert app_state.status == ApplicationStatus.RUNNING # Message and status should not change running_msg = app_state._status_msg assert running_msg != deploy_failed_msg app_state.update() assert app_state.status == ApplicationStatus.RUNNING assert app_state._status_msg == running_msg def test_app_deploy_failed_and_recover(mocked_application_state): """Test DEPLOYING -> DEPLOY_FAILED -> (self recovered) -> RUNNING If while the application is deploying a deployment becomes unhealthy, the app is marked as deploy failed. But if the deployment recovers, the application status should update to running. """ app_state, deployment_state_manager = mocked_application_state deployment_id = DeploymentID(name="d1", app_name="test_app") app_state.deploy_app( {"d1": deployment_info("d1")}, ApplicationArgsProto(external_scaler_enabled=False), ) assert app_state.status == ApplicationStatus.DEPLOYING # Before status of deployment changes, app should still be DEPLOYING app_state.update() assert app_state.status == ApplicationStatus.DEPLOYING # Mark deployment unhealthy -> app should be DEPLOY_FAILED deployment_state_manager.set_deployment_deploy_failed(deployment_id) app_state.update() assert app_state.status == ApplicationStatus.DEPLOY_FAILED app_state.update() assert app_state.status == ApplicationStatus.DEPLOY_FAILED # Deployment recovers to healthy -> app should be RUNNING deployment_state_manager.set_deployment_healthy(deployment_id) app_state.update() assert app_state.status == ApplicationStatus.RUNNING app_state.update() assert app_state.status == ApplicationStatus.RUNNING def test_app_unhealthy(mocked_application_state): """Test DEPLOYING -> RUNNING -> UNHEALTHY -> RUNNING. Even after an application becomes running, if a deployment becomes unhealthy at some point, the application status should also be updated to unhealthy. """ app_state, deployment_state_manager = mocked_application_state id_a, id_b = DeploymentID(name="a", app_name="test_app"), DeploymentID( name="b", app_name="test_app" ) app_state.deploy_app( {"a": deployment_info("a"), "b": deployment_info("b")}, ApplicationArgsProto(external_scaler_enabled=False), ) assert app_state.status == ApplicationStatus.DEPLOYING app_state.update() assert app_state.status == ApplicationStatus.DEPLOYING # Once both deployments become healthy, app should be running deployment_state_manager.set_deployment_healthy(id_a) deployment_state_manager.set_deployment_healthy(id_b) app_state.update() assert app_state.status == ApplicationStatus.RUNNING # If a deployment becomes unhealthy, application should become unhealthy deployment_state_manager.set_deployment_unhealthy(id_a) app_state.update() assert app_state.status == ApplicationStatus.UNHEALTHY # Rerunning update shouldn't make a difference app_state.update() assert app_state.status == ApplicationStatus.UNHEALTHY # If the deployment recovers, the application should also recover deployment_state_manager.set_deployment_healthy(id_a) app_state.update() assert app_state.status == ApplicationStatus.RUNNING @patch("ray.serve._private.application_state.build_serve_application", Mock()) @patch("ray.get", Mock(return_value=(None, [deployment_params("a", "/old")], None))) @patch("ray.serve._private.application_state.check_obj_ref_ready_nowait") def test_apply_app_configs_succeed(check_obj_ref_ready_nowait): """Test deploying through config successfully. Deploy obj ref finishes successfully, so status should transition to running. """ kv_store = MockKVStore() deployment_id = DeploymentID(name="a", app_name="test_app") deployment_state_manager = MockDeploymentStateManager(kv_store) app_state_manager = ApplicationStateManager( deployment_state_manager, AutoscalingStateManager(), MockEndpointState(), kv_store, LoggingConfig(), ) # Deploy config app_config = ServeApplicationSchema( name="test_app", import_path="fa.ke", route_prefix="/new" ) app_state_manager.apply_app_configs([app_config]) app_state = app_state_manager._application_states["test_app"] assert app_state.status == ApplicationStatus.DEPLOYING # Before object ref is ready check_obj_ref_ready_nowait.return_value = False app_state.update() assert app_state._build_app_task_info assert app_state.status == ApplicationStatus.DEPLOYING app_state.update() assert app_state.status == ApplicationStatus.DEPLOYING # Object ref is ready check_obj_ref_ready_nowait.return_value = True app_state.update() assert app_state.status == ApplicationStatus.DEPLOYING assert app_state.target_deployments == ["a"] assert app_state.route_prefix == "/new" # Set healthy deployment_state_manager.set_deployment_healthy(deployment_id) app_state.update() assert app_state.status == ApplicationStatus.RUNNING @patch( "ray.serve._private.application_state.get_app_code_version", Mock(return_value="123"), ) @patch("ray.serve._private.application_state.build_serve_application", Mock()) @patch("ray.get", Mock(side_effect=RayTaskError(None, "intentionally failed", None))) @patch("ray.serve._private.application_state.check_obj_ref_ready_nowait") def test_apply_app_configs_fail(check_obj_ref_ready_nowait): """Test fail to deploy through config. Deploy obj ref errors out, so status should transition to deploy failed. """ kv_store = MockKVStore() deployment_state_manager = MockDeploymentStateManager(kv_store) app_state_manager = ApplicationStateManager( deployment_state_manager, AutoscalingStateManager(), MockEndpointState(), kv_store, LoggingConfig(), ) # Deploy config app_config = ServeApplicationSchema( name="test_app", import_path="fa.ke", route_prefix="/new" ) app_state_manager.apply_app_configs([app_config]) app_state = app_state_manager._application_states["test_app"] assert app_state.status == ApplicationStatus.DEPLOYING # Before object ref is ready check_obj_ref_ready_nowait.return_value = False app_state.update() assert app_state._build_app_task_info assert app_state.status == ApplicationStatus.DEPLOYING app_state.update() assert app_state.status == ApplicationStatus.DEPLOYING # Object ref is ready, and the task has called deploy_app check_obj_ref_ready_nowait.return_value = True app_state.update() assert app_state.status == ApplicationStatus.DEPLOY_FAILED assert "failed" in app_state._status_msg or "error" in app_state._status_msg @patch( "ray.serve._private.application_state.get_app_code_version", Mock(return_value="123"), ) @patch("ray.serve._private.application_state.build_serve_application", Mock()) @patch("ray.get", Mock(return_value=(None, [deployment_params("a", "/old")], None))) @patch("ray.serve._private.application_state.check_obj_ref_ready_nowait") def test_apply_app_configs_deletes_existing(check_obj_ref_ready_nowait): """Test that apply_app_configs deletes existing apps that aren't in the new list. This should *not* apply to apps that were deployed via `deploy_app` (which is an imperative API). """ kv_store = MockKVStore() deployment_state_manager = MockDeploymentStateManager(kv_store) app_state_manager = ApplicationStateManager( deployment_state_manager, AutoscalingStateManager(), MockEndpointState(), kv_store, LoggingConfig(), ) # Deploy an app via `deploy_app` - should not be affected. a_id = DeploymentID(name="a", app_name="imperative_app") app_state_manager.deploy_app( "imperative_app", [deployment_params("a", "/hi")], ApplicationArgsProto(external_scaler_enabled=False), ) imperative_app_state = app_state_manager._application_states["imperative_app"] assert imperative_app_state.api_type == APIType.IMPERATIVE assert imperative_app_state.status == ApplicationStatus.DEPLOYING imperative_app_state.update() deployment_state_manager.set_deployment_healthy(a_id) imperative_app_state.update() assert imperative_app_state.status == ApplicationStatus.RUNNING # Now deploy an initial version of the config with app 1 and app 2. app1_config = ServeApplicationSchema( name="app1", import_path="fa.ke", route_prefix="/1" ) app2_config = ServeApplicationSchema( name="app2", import_path="fa.ke", route_prefix="/2" ) app_state_manager.apply_app_configs([app1_config, app2_config]) app1_state = app_state_manager._application_states["app1"] assert app1_state.api_type == APIType.DECLARATIVE app2_state = app_state_manager._application_states["app2"] assert app2_state.api_type == APIType.DECLARATIVE app1_state.update() app2_state.update() assert app1_state.status == ApplicationStatus.DEPLOYING assert app2_state.status == ApplicationStatus.DEPLOYING # Now redeploy a new config that removes app 1 and adds app 3. app3_config = ServeApplicationSchema( name="app3", import_path="fa.ke", route_prefix="/3" ) app_state_manager.apply_app_configs([app3_config, app2_config]) app3_state = app_state_manager._application_states["app3"] assert app3_state.api_type == APIType.DECLARATIVE app1_state.update() app2_state.update() app3_state.update() assert app1_state.status == ApplicationStatus.DELETING assert app2_state.status == ApplicationStatus.DEPLOYING assert app3_state.status == ApplicationStatus.DEPLOYING @patch( "ray.serve._private.application_state.get_app_code_version", Mock(return_value="123"), ) @patch("ray.serve._private.application_state.build_serve_application", Mock()) @patch("ray.get", Mock(return_value=(None, [deployment_params("d1", "/route1")], None))) @patch("ray.serve._private.application_state.check_obj_ref_ready_nowait") def test_apply_app_configs_with_external_scaler_enabled(check_obj_ref_ready_nowait): """Test that apply_app_configs correctly sets external_scaler_enabled. This test verifies that when apply_app_configs is called with app configs that have external_scaler_enabled=True or False, the ApplicationState is correctly initialized with the appropriate external_scaler_enabled value. """ kv_store = MockKVStore() deployment_state_manager = MockDeploymentStateManager(kv_store) app_state_manager = ApplicationStateManager( deployment_state_manager, AutoscalingStateManager(), MockEndpointState(), kv_store, LoggingConfig(), ) # Deploy app with external_scaler_enabled=True app_config_with_scaler = ServeApplicationSchema( name="app_with_scaler", import_path="fa.ke", route_prefix="/with_scaler", external_scaler_enabled=True, ) # Deploy app with external_scaler_enabled=False (default) app_config_without_scaler = ServeApplicationSchema( name="app_without_scaler", import_path="fa.ke", route_prefix="/without_scaler", external_scaler_enabled=False, ) # Apply both configs app_state_manager.apply_app_configs( [app_config_with_scaler, app_config_without_scaler] ) # Verify that external_scaler_enabled is correctly set for both apps assert app_state_manager.get_external_scaler_enabled("app_with_scaler") is True assert app_state_manager.get_external_scaler_enabled("app_without_scaler") is False # Verify the internal state is also correct app_state_with_scaler = app_state_manager._application_states["app_with_scaler"] app_state_without_scaler = app_state_manager._application_states[ "app_without_scaler" ] assert app_state_with_scaler.external_scaler_enabled is True assert app_state_without_scaler.external_scaler_enabled is False # Simulate the build task completing check_obj_ref_ready_nowait.return_value = True app_state_with_scaler.update() app_state_without_scaler.update() # After update, external_scaler_enabled should still be preserved assert app_state_manager.get_external_scaler_enabled("app_with_scaler") is True assert app_state_manager.get_external_scaler_enabled("app_without_scaler") is False def test_redeploy_same_app(mocked_application_state): """Test redeploying same application with updated deployments.""" app_state, deployment_state_manager = mocked_application_state a_id = DeploymentID(name="a", app_name="test_app") b_id = DeploymentID(name="b", app_name="test_app") c_id = DeploymentID(name="c", app_name="test_app") app_state.deploy_app( {"a": deployment_info("a"), "b": deployment_info("b")}, ApplicationArgsProto(external_scaler_enabled=False), ) assert app_state.status == ApplicationStatus.DEPLOYING # Update app_state.update() assert app_state.status == ApplicationStatus.DEPLOYING assert set(app_state.target_deployments) == {"a", "b"} # Transition to running deployment_state_manager.set_deployment_healthy(a_id) app_state.update() assert app_state.status == ApplicationStatus.DEPLOYING deployment_state_manager.set_deployment_healthy(b_id) app_state.update() assert app_state.status == ApplicationStatus.RUNNING # Deploy the same app with different deployments app_state.deploy_app( {"b": deployment_info("b"), "c": deployment_info("c")}, ApplicationArgsProto(external_scaler_enabled=False), ) assert app_state.status == ApplicationStatus.DEPLOYING # Target state should be updated immediately assert "a" not in app_state.target_deployments # Remove deployment `a` app_state.update() deployment_state_manager.set_deployment_deleted(a_id) app_state.update() assert app_state.status == ApplicationStatus.DEPLOYING # Move to running deployment_state_manager.set_deployment_healthy(c_id) app_state.update() assert app_state.status == ApplicationStatus.DEPLOYING deployment_state_manager.set_deployment_healthy(b_id) app_state.update() assert app_state.status == ApplicationStatus.RUNNING def test_deploy_with_route_prefix_conflict(mocked_application_state_manager): """Test that an application with a route prefix conflict fails to deploy""" app_state_manager, _, _ = mocked_application_state_manager app_state_manager.deploy_app( "app1", [deployment_params("a", "/hi")], ApplicationArgsProto(external_scaler_enabled=False), ) with pytest.raises(RayServeException): app_state_manager.deploy_app( "app2", [deployment_params("b", "/hi")], ApplicationArgsProto(external_scaler_enabled=False), ) def test_deploy_with_renamed_app(mocked_application_state_manager): """ Test that an application deploys successfully when there is a route prefix conflict with an old app running on the cluster. """ app_state_manager, deployment_state_manager, _ = mocked_application_state_manager a_id, b_id = DeploymentID(name="a", app_name="app1"), DeploymentID( name="b", app_name="app2" ) # deploy app1 app_state_manager.deploy_app( "app1", [deployment_params("a", "/url1")], ApplicationArgsProto(external_scaler_enabled=False), ) app_state = app_state_manager._application_states["app1"] assert app_state_manager.get_app_status("app1") == ApplicationStatus.DEPLOYING # Update app_state_manager.update() assert app_state_manager.get_app_status("app1") == ApplicationStatus.DEPLOYING assert set(app_state.target_deployments) == {"a"} # Once its single deployment is healthy, app1 should be running deployment_state_manager.set_deployment_healthy(a_id) app_state_manager.update() assert app_state_manager.get_app_status("app1") == ApplicationStatus.RUNNING # delete app1 app_state_manager.delete_app("app1") assert app_state_manager.get_app_status("app1") == ApplicationStatus.DELETING app_state_manager.update() # deploy app2 app_state_manager.deploy_app( "app2", [deployment_params("b", "/url1")], ApplicationArgsProto(external_scaler_enabled=False), ) assert app_state_manager.get_app_status("app2") == ApplicationStatus.DEPLOYING app_state_manager.update() # app2 deploys before app1 finishes deleting deployment_state_manager.set_deployment_healthy(b_id) app_state_manager.update() assert app_state_manager.get_app_status("app2") == ApplicationStatus.RUNNING assert app_state_manager.get_app_status("app1") == ApplicationStatus.DELETING # app1 finally finishes deleting deployment_state_manager.set_deployment_deleted(a_id) app_state_manager.update() assert app_state_manager.get_app_status("app1") == ApplicationStatus.NOT_STARTED assert app_state_manager.get_app_status("app2") == ApplicationStatus.RUNNING def test_application_state_recovery(mocked_application_state_manager): """Test DEPLOYING -> RUNNING -> (controller crash) -> DEPLOYING -> RUNNING""" ( app_state_manager, deployment_state_manager, kv_store, ) = mocked_application_state_manager deployment_id = DeploymentID(name="d1", app_name="test_app") app_name = "test_app" # DEPLOY application with deployments {d1, d2} params = deployment_params("d1") app_state_manager.deploy_app( app_name, [params], ApplicationArgsProto(external_scaler_enabled=False) ) app_state = app_state_manager._application_states[app_name] assert app_state.status == ApplicationStatus.DEPLOYING # Once deployment is healthy, app should be running app_state_manager.update() assert deployment_state_manager.get_deployment(deployment_id) deployment_state_manager.set_deployment_healthy(deployment_id) app_state_manager.update() assert app_state.status == ApplicationStatus.RUNNING # In real code this checkpoint would be done by the caller of the deploys app_state_manager.save_checkpoint() # Simulate controller crashed!! Create new deployment state manager, # which should recover target state for deployment "d1" from kv store new_deployment_state_manager = MockDeploymentStateManager(kv_store) version1 = new_deployment_state_manager.deployment_infos[deployment_id].version # Create new application state manager, and it should call _recover_from_checkpoint new_app_state_manager = ApplicationStateManager( new_deployment_state_manager, AutoscalingStateManager(), MockEndpointState(), kv_store, LoggingConfig(), ) app_state = new_app_state_manager._application_states[app_name] assert app_state.status == ApplicationStatus.DEPLOYING assert app_state._target_state.deployment_infos["d1"].version == version1 new_deployment_state_manager.set_deployment_healthy(deployment_id) new_app_state_manager.update() assert app_state.status == ApplicationStatus.RUNNING def test_recover_during_update(mocked_application_state_manager): """Test that application and deployment states are recovered if controller crashed in the middle of a redeploy. Target state is checkpointed in the application state manager, but not yet the deployment state manager when the controller crashes Then the deployment state manager should recover an old version of the deployment during initial recovery, but the application state manager should eventually reconcile this. """ ( app_state_manager, deployment_state_manager, kv_store, ) = mocked_application_state_manager deployment_id = DeploymentID(name="d1", app_name="test_app") app_name = "test_app" # DEPLOY application with deployment "d1" params = deployment_params("d1") app_state_manager.deploy_app( app_name, [params], ApplicationArgsProto(external_scaler_enabled=False) ) app_state = app_state_manager._application_states[app_name] assert app_state.status == ApplicationStatus.DEPLOYING # Once deployment is healthy, app should be running app_state_manager.update() assert deployment_state_manager.get_deployment(deployment_id) deployment_state_manager.set_deployment_healthy(deployment_id) app_state_manager.update() assert app_state.status == ApplicationStatus.RUNNING # Deploy new version of "d1" (this auto generates new random version) params2 = deployment_params("d1") app_state_manager.deploy_app( app_name, [params2], ApplicationArgsProto(external_scaler_enabled=False) ) assert app_state.status == ApplicationStatus.DEPLOYING # In real code this checkpoint would be done by the caller of the deploys app_state_manager.save_checkpoint() # Before application state manager could propagate new version to # deployment state manager, controller crashes. # Create new deployment state manager. It should recover the old # version of the deployment from the kv store new_deployment_state_manager = MockDeploymentStateManager(kv_store) dr_version = new_deployment_state_manager.deployment_infos[deployment_id].version # Create new application state manager, and it should call _recover_from_checkpoint new_app_state_manager = ApplicationStateManager( new_deployment_state_manager, AutoscalingStateManager(), MockEndpointState(), kv_store, LoggingConfig(), ) app_state = new_app_state_manager._application_states[app_name] ar_version = app_state._target_state.deployment_infos["d1"].version assert app_state.status == ApplicationStatus.DEPLOYING assert ar_version != dr_version new_app_state_manager.update() assert ( new_deployment_state_manager.deployment_infos[deployment_id].version == ar_version ) assert app_state.status == ApplicationStatus.DEPLOYING new_deployment_state_manager.set_deployment_healthy(deployment_id) new_app_state_manager.update() assert app_state.status == ApplicationStatus.RUNNING def test_is_ready_for_shutdown(mocked_application_state_manager): """Test `is_ready_for_shutdown()` returns the correct state. When shutting down applications before deployments are deleted, application state `is_deleted()` should return False and `is_ready_for_shutdown()` should return False. When shutting down applications after deployments are deleted, application state `is_deleted()` should return True and `is_ready_for_shutdown()` should return True. """ ( app_state_manager, deployment_state_manager, kv_store, ) = mocked_application_state_manager app_name = "test_app" deployment_name = "d1" deployment_id = DeploymentID(name=deployment_name, app_name=app_name) # DEPLOY application with deployment "d1" params = deployment_params(deployment_name) app_state_manager.deploy_app( app_name, [params], ApplicationArgsProto(external_scaler_enabled=False) ) app_state = app_state_manager._application_states[app_name] assert app_state.status == ApplicationStatus.DEPLOYING # Once deployment is healthy, app should be running app_state_manager.update() assert deployment_state_manager.get_deployment(deployment_id) deployment_state_manager.set_deployment_healthy(deployment_id) app_state_manager.update() assert app_state.status == ApplicationStatus.RUNNING # When shutting down applications before deployments are deleted, application state # `is_deleted()` should return False and `is_ready_for_shutdown()` should return # False app_state_manager.shutdown() assert not app_state.is_deleted() assert not app_state_manager.is_ready_for_shutdown() # When shutting down applications after deployments are deleted, application state # `is_deleted()` should return True and `is_ready_for_shutdown()` should return True deployment_state_manager.delete_deployment(deployment_id) deployment_state_manager.set_deployment_deleted(deployment_id) app_state_manager.update() assert app_state.is_deleted() assert app_state_manager.is_ready_for_shutdown() def test_shutdown_does_not_delete_checkpoint(mocked_application_state_manager): """Tests checkpoint survives `shutdown() and `is_ready_for_shutdown(). Only an explicit `delete_checkpoint() call should remove it. """ ( app_state_manager, deployment_state_manager, kv_store, ) = mocked_application_state_manager app_name = "test_app" deployment_name = "d1" deployment_id = DeploymentID(name=deployment_name, app_name=app_name) # Deploy an application and bring it to RUNNING. params = deployment_params(deployment_name) app_state_manager.deploy_app( app_name, [params], ApplicationArgsProto(external_scaler_enabled=False) ) app_state_manager.update() deployment_state_manager.set_deployment_healthy(deployment_id) app_state_manager.update() app_state_manager.save_checkpoint() # Checkpoint should exist after save. assert kv_store.get(CHECKPOINT_KEY) is not None # shutdown() must NOT delete the checkpoint. app_state_manager.shutdown() assert kv_store.get(CHECKPOINT_KEY) is not None # save_checkpoint() after shutdown should be a no-op. pre_shutdown_checkpoint = kv_store.get(CHECKPOINT_KEY) app_state_manager.save_checkpoint() assert kv_store.get(CHECKPOINT_KEY) is pre_shutdown_checkpoint # Delete deployments so is_ready_for_shutdown() is True. deployment_state_manager.delete_deployment(deployment_id) deployment_state_manager.set_deployment_deleted(deployment_id) app_state_manager.update() assert app_state_manager.is_ready_for_shutdown() # is_ready_for_shutdown() must NOT delete the checkpoint. assert kv_store.get(CHECKPOINT_KEY) is not None # Only delete_checkpoint() should remove it. app_state_manager.delete_checkpoint() assert kv_store.get(CHECKPOINT_KEY) is None class TestOverrideDeploymentInfo: @pytest.fixture def info(self): return DeploymentInfo( route_prefix="/", version="123", deployment_config=DeploymentConfig(num_replicas=1), replica_config=ReplicaConfig.create(lambda x: x), start_time_ms=0, deployer_job_id="", ) @staticmethod def _make_info(ingress=False, ingress_request_router=False): return DeploymentInfo( route_prefix="/" if ingress else None, version="123", deployment_config=DeploymentConfig(num_replicas=1), replica_config=ReplicaConfig.create(lambda x: x), start_time_ms=0, deployer_job_id="", ingress=ingress, ingress_request_router=ingress_request_router, ) def test_override_deployment_config(self, info): config = ServeApplicationSchema( name="default", import_path="test.import.path", deployments=[ DeploymentSchema( name="A", num_replicas=3, max_ongoing_requests=200, user_config={"price": "4"}, graceful_shutdown_wait_loop_s=4, graceful_shutdown_timeout_s=40, health_check_period_s=20, health_check_timeout_s=60, ) ], ) updated_infos = override_deployment_info({"A": info}, config) updated_info = updated_infos["A"] assert updated_info.route_prefix == "/" assert updated_info.version == "123" assert updated_info.deployment_config.max_ongoing_requests == 200 assert updated_info.deployment_config.user_config == {"price": "4"} assert updated_info.deployment_config.graceful_shutdown_wait_loop_s == 4 assert updated_info.deployment_config.graceful_shutdown_timeout_s == 40 assert updated_info.deployment_config.health_check_period_s == 20 assert updated_info.deployment_config.health_check_timeout_s == 60 def test_noop_deployment_override_keeps_actor_options(self, info): original_actor_options = dict(info.replica_config.ray_actor_options) config = ServeApplicationSchema( name="default", import_path="test.import.path", deployments=[DeploymentSchema(name="A")], ) updated_infos = override_deployment_info({"A": info}, config) updated_info = updated_infos["A"] assert updated_info.replica_config.ray_actor_options == original_actor_options assert "runtime_env" not in updated_info.replica_config.ray_actor_options old_version = DeploymentVersion( info.version, info.deployment_config, info.replica_config.ray_actor_options, ) new_version = DeploymentVersion( updated_info.version, updated_info.deployment_config, updated_info.replica_config.ray_actor_options, ) assert old_version.ray_actor_options_hash == new_version.ray_actor_options_hash assert not old_version.requires_actor_restart(new_version) def test_override_autoscaling_config(self, info): config = ServeApplicationSchema( name="default", import_path="test.import.path", deployments=[ DeploymentSchema( name="A", autoscaling_config={ "min_replicas": 1, "initial_replicas": 12, "max_replicas": 79, }, ) ], ) updated_infos = override_deployment_info({"A": info}, config) updated_info = updated_infos["A"] assert updated_info.route_prefix == "/" assert updated_info.version == "123" assert updated_info.deployment_config.autoscaling_config.min_replicas == 1 assert updated_info.deployment_config.autoscaling_config.initial_replicas == 12 assert updated_info.deployment_config.autoscaling_config.max_replicas == 79 def test_override_route_prefix(self, info): config = ServeApplicationSchema( name="default", import_path="test.import.path", route_prefix="/bob", deployments=[ DeploymentSchema( name="A", ) ], ) updated_infos = override_deployment_info({"A": info}, config) updated_info = updated_infos["A"] assert updated_info.route_prefix == "/bob" assert updated_info.version == "123" @pytest.mark.parametrize( "haproxy_enabled, attach_ingress_request_router, rejected", [ # A custom router added by config override is rejected only under # HAProxy, since build_app cannot see the override... (True, False, True), (False, False, False), # ...unless an ingress request router is attached (direct streaming), # where routing flows through the Serve router. (True, True, False), ], ) def test_override_custom_ingress_request_router_under_haproxy( self, monkeypatch, haproxy_enabled, attach_ingress_request_router, rejected ): monkeypatch.setattr( "ray.serve._private.application_state.RAY_SERVE_ENABLE_HA_PROXY", haproxy_enabled, ) infos = {"Ingress": self._make_info(ingress=True)} if attach_ingress_request_router: infos["Router"] = self._make_info(ingress_request_router=True) config = ServeApplicationSchema( name="default", import_path="test.import.path", deployments=[ DeploymentSchema( name="Ingress", request_router_config=RequestRouterConfig( request_router_class=RoundRobinRouter ), ) ], ) if rejected: with pytest.raises( RayServeException, match=CUSTOM_INGRESS_REQUEST_ROUTER_UNSUPPORTED_ERROR ): override_deployment_info(infos, config) else: router_config = override_deployment_info(infos, config)[ "Ingress" ].deployment_config.request_router_config assert not router_config.is_default_request_router() def test_override_allows_custom_router_on_non_ingress_under_haproxy( self, monkeypatch ): """The guard targets only the ingress, so a custom router on a downstream deployment is honored under HAProxy.""" monkeypatch.setattr( "ray.serve._private.application_state.RAY_SERVE_ENABLE_HA_PROXY", True ) infos = { "Ingress": self._make_info(ingress=True), "Downstream": self._make_info(ingress=False), } config = ServeApplicationSchema( name="default", import_path="test.import.path", deployments=[ DeploymentSchema( name="Downstream", request_router_config=RequestRouterConfig( request_router_class=RoundRobinRouter ), ) ], ) router_config = override_deployment_info(infos, config)[ "Downstream" ].deployment_config.request_router_config assert not router_config.is_default_request_router() def test_override_ray_actor_options_1(self, info): """Test runtime env specified in config at deployment level.""" config = ServeApplicationSchema( name="default", import_path="test.import.path", deployments=[ DeploymentSchema( name="A", ray_actor_options={"runtime_env": {"working_dir": "s3://B"}}, ) ], ) updated_infos = override_deployment_info({"A": info}, config) updated_info = updated_infos["A"] assert updated_info.route_prefix == "/" assert updated_info.version == "123" assert ( updated_info.replica_config.ray_actor_options["runtime_env"]["working_dir"] == "s3://B" ) def test_override_ray_actor_options_2(self, info): """Test application runtime env is propagated to deployments.""" config = ServeApplicationSchema( name="default", import_path="test.import.path", runtime_env={"working_dir": "s3://C"}, deployments=[ DeploymentSchema( name="A", ) ], ) updated_infos = override_deployment_info({"A": info}, config) updated_info = updated_infos["A"] assert updated_info.route_prefix == "/" assert updated_info.version == "123" assert ( updated_info.replica_config.ray_actor_options["runtime_env"]["working_dir"] == "s3://C" ) def test_override_ray_actor_options_3(self, info): """If runtime env is specified in the config at the deployment level, it should override the application-level runtime env. """ config = ServeApplicationSchema( name="default", import_path="test.import.path", runtime_env={"working_dir": "s3://C"}, deployments=[ DeploymentSchema( name="A", ray_actor_options={"runtime_env": {"working_dir": "s3://B"}}, ) ], ) updated_infos = override_deployment_info({"A": info}, config) updated_info = updated_infos["A"] assert updated_info.route_prefix == "/" assert updated_info.version == "123" assert ( updated_info.replica_config.ray_actor_options["runtime_env"]["working_dir"] == "s3://B" ) def test_override_ray_actor_options_4(self): """If runtime env is specified for the deployment in code, it should override the application-level runtime env. """ info = DeploymentInfo( route_prefix="/", version="123", deployment_config=DeploymentConfig(num_replicas=1), replica_config=ReplicaConfig.create( lambda x: x, ray_actor_options={"runtime_env": {"working_dir": "s3://A"}}, ), start_time_ms=0, deployer_job_id="", ) config = ServeApplicationSchema( name="default", import_path="test.import.path", runtime_env={"working_dir": "s3://C"}, deployments=[ DeploymentSchema( name="A", ) ], ) updated_infos = override_deployment_info({"A": info}, config) updated_info = updated_infos["A"] assert updated_info.route_prefix == "/" assert updated_info.version == "123" assert ( updated_info.replica_config.ray_actor_options["runtime_env"]["working_dir"] == "s3://A" ) def test_override_ray_actor_options_5(self): """If runtime env is specified in all three places: - In code - In the config at the deployment level - In the config at the application level The one specified in the config at the deployment level should take precedence. """ info = DeploymentInfo( route_prefix="/", version="123", deployment_config=DeploymentConfig(num_replicas=1), replica_config=ReplicaConfig.create( lambda x: x, ray_actor_options={"runtime_env": {"working_dir": "s3://A"}}, ), start_time_ms=0, deployer_job_id="", ) config = ServeApplicationSchema( name="default", import_path="test.import.path", runtime_env={"working_dir": "s3://C"}, deployments=[ DeploymentSchema( name="A", ray_actor_options={"runtime_env": {"working_dir": "s3://B"}}, ) ], ) updated_infos = override_deployment_info({"A": info}, config) updated_info = updated_infos["A"] assert updated_info.route_prefix == "/" assert updated_info.version == "123" assert ( updated_info.replica_config.ray_actor_options["runtime_env"]["working_dir"] == "s3://B" ) def test_override_bundle_label_selector(self, info): """Test placement_group_bundle_label_selector is propagated from config.""" config = ServeApplicationSchema( name="default", import_path="test.import.path", deployments=[ DeploymentSchema( name="A", placement_group_bundles=[{"CPU": 1}], placement_group_bundle_label_selector=[ {"accelerator-type": "A100"} ], ) ], ) updated_infos = override_deployment_info({"A": info}, config) updated_info = updated_infos["A"] assert updated_info.replica_config.placement_group_bundle_label_selector == [ {"accelerator-type": "A100"} ] def test_override_fallback_strategy(self, info): """Test placement_group_fallback_strategy is preserved when config is updated. placement_group_fallback_strategy is not yet part of the public DeploymentSchema, so we cannot set it via the override config. Instead, we verify that an existing value in the ReplicaConfig is preserved when other fields are updated via the config. """ initial_info = DeploymentInfo( route_prefix="/", version="123", deployment_config=DeploymentConfig(num_replicas=1), replica_config=ReplicaConfig.create( lambda x: x, placement_group_bundles=[{"CPU": 1}], placement_group_fallback_strategy=[{"bundles": [{"CPU": 1}]}], ), start_time_ms=0, deployer_job_id="", ) config = ServeApplicationSchema( name="default", import_path="test.import.path", deployments=[ DeploymentSchema( name="A", num_replicas=5, # Update a different field ) ], ) updated_infos = override_deployment_info({"A": initial_info}, config) updated_info = updated_infos["A"] assert updated_info.deployment_config.num_replicas == 5 assert updated_info.replica_config.placement_group_fallback_strategy == [ {"bundles": [{"CPU": 1}]} ] def test_override_gang_scheduling_config(self, info): """Test gang_scheduling_config dict is converted to GangSchedulingConfig.""" config = ServeApplicationSchema( name="default", import_path="test.import.path", deployments=[ DeploymentSchema( name="A", num_replicas=4, gang_scheduling_config={ "gang_size": 2, "gang_placement_strategy": "SPREAD", }, ) ], ) updated_infos = override_deployment_info({"A": info}, config) updated_info = updated_infos["A"] gang_config = updated_info.deployment_config.gang_scheduling_config assert isinstance(gang_config, GangSchedulingConfig) assert gang_config.gang_size == 2 assert gang_config.gang_placement_strategy.value == "SPREAD" assert updated_info.deployment_config.num_replicas == 4 def test_override_num_replicas_rejects_invalid_gang_multiple(self): """Test that changing num_replicas to a value not divisible by the existing gang_size is rejected.""" initial_info = DeploymentInfo( route_prefix="/", version="123", deployment_config=DeploymentConfig( num_replicas=4, gang_scheduling_config=GangSchedulingConfig(gang_size=2), ), replica_config=ReplicaConfig.create(lambda x: x), start_time_ms=0, deployer_job_id="", ) config = ServeApplicationSchema( name="default", import_path="test.import.path", deployments=[ DeploymentSchema( name="A", num_replicas=5, ) ], ) with pytest.raises(ValidationError, match="must be a multiple of gang_size"): override_deployment_info({"A": initial_info}, config) def test_override_deployment_info_injects_serialized_deployment_actors(self): """Config-only deployment actors: serialized bytes from build task are injected into deployment config via override_deployment_info. """ # Minimal actor class for serialization (must be at module level for pickle) class _TestActorForSerialization: pass serialized = cloudpickle.dumps(_TestActorForSerialization) deployment_to_serialized = {"A": {"counter": serialized}} initial_info = DeploymentInfo( route_prefix="/", version="123", deployment_config=DeploymentConfig(num_replicas=1), replica_config=ReplicaConfig.create(lambda x: x), start_time_ms=0, deployer_job_id="", ) config = ServeApplicationSchema( name="default", import_path="test.import.path", deployments=[ DeploymentSchema( name="A", deployment_actors=[ { "name": "counter", "actor_class": "test.module:SomeActor", "init_kwargs": {"start": 0}, }, ], ) ], ) updated_infos = override_deployment_info( {"A": initial_info}, config, deployment_to_serialized_deployment_actors=deployment_to_serialized, ) updated_info = updated_infos["A"] assert updated_info.deployment_config.deployment_actors is not None assert len(updated_info.deployment_config.deployment_actors) == 1 actor_cfg = updated_info.deployment_config.deployment_actors[0] assert actor_cfg.name == "counter" assert actor_cfg._serialized_actor_class == serialized # Can deserialize and get the class resolved = actor_cfg.get_actor_class() assert resolved.__name__ == "_TestActorForSerialization" def test_override_deployment_info_deployment_actors_no_serialized_provided(self): """When deployment has deployment_actors but no serialized bytes provided, override still succeeds (actor_class stays as import path string). """ initial_info = DeploymentInfo( route_prefix="/", version="123", deployment_config=DeploymentConfig( num_replicas=1, deployment_actors=[ DeploymentActorConfig( name="counter", actor_class="ray.serve.tests.test_deployment_actors:SharedCounter", init_kwargs={"start": 0}, ), ], ), replica_config=ReplicaConfig.create(lambda x: x), start_time_ms=0, deployer_job_id="", ) config = ServeApplicationSchema( name="default", import_path="test.import.path", deployments=[ DeploymentSchema( name="A", num_replicas=2, ) ], ) updated_infos = override_deployment_info( {"A": initial_info}, config, deployment_to_serialized_deployment_actors=None, ) updated_info = updated_infos["A"] assert updated_info.deployment_config.deployment_actors is not None assert len(updated_info.deployment_config.deployment_actors) == 1 assert updated_info.deployment_config.deployment_actors[0].name == "counter" assert updated_info.deployment_config.num_replicas == 2 def test_override_deployment_info_deployment_actors_partial_match(self): """Only actor names present in serialized map get _serialized_actor_class.""" class _Actor1: pass class _Actor2: pass serialized_1 = cloudpickle.dumps(_Actor1) deployment_to_serialized = {"A": {"actor1": serialized_1}} initial_info = DeploymentInfo( route_prefix="/", version="123", deployment_config=DeploymentConfig( num_replicas=1, deployment_actors=[ DeploymentActorConfig( name="actor1", actor_class="test:Actor1", init_kwargs={}, ), DeploymentActorConfig( name="actor2", actor_class="test:Actor2", init_kwargs={}, ), ], ), replica_config=ReplicaConfig.create(lambda x: x), start_time_ms=0, deployer_job_id="", ) config = ServeApplicationSchema( name="default", import_path="test.import.path", deployments=[ DeploymentSchema( name="A", deployment_actors=[ { "name": "actor1", "actor_class": "test:Actor1", "init_kwargs": {}, }, { "name": "actor2", "actor_class": "test:Actor2", "init_kwargs": {}, }, ], ) ], ) updated_infos = override_deployment_info( {"A": initial_info}, config, deployment_to_serialized_deployment_actors=deployment_to_serialized, ) updated_info = updated_infos["A"] actors = updated_info.deployment_config.deployment_actors assert actors[0]._serialized_actor_class == serialized_1 assert actors[1]._serialized_actor_class == b"" # Not in map, stays empty @patch( "ray.serve._private.application_state.get_app_code_version", Mock(return_value="123"), ) @patch("ray.serve._private.application_state.check_obj_ref_ready_nowait") def test_apply_app_config_extracts_deployment_actor_classes(check_obj_ref_ready_nowait): """When config has deployment_actors (dict format), they are extracted and passed to build_serve_application as deployment_to_deployment_actor_classes. """ with patch( "ray.serve._private.application_state.build_serve_application" ) as mock_build: mock_build.options.return_value.remote.return_value = Mock() kv_store = MockKVStore() deployment_state_manager = MockDeploymentStateManager(kv_store) app_state_manager = ApplicationStateManager( deployment_state_manager, AutoscalingStateManager(), MockEndpointState(), kv_store, LoggingConfig(), ) app_config = ServeApplicationSchema( name="test_app", import_path="test.import.path", route_prefix="/", deployments=[ DeploymentSchema( name="MyDeployment", deployment_actors=[ { "name": "counter", "actor_class": "ray.serve.tests.test_deployment_actors:SharedCounter", "init_kwargs": {"start": 0}, }, { "name": "cache", "actor_class": "ray.serve.tests.test_deployment_actors:SharedCache", "init_kwargs": {}, }, ], ), ], ) app_state_manager.apply_app_configs([app_config]) app_state = app_state_manager._application_states["test_app"] assert app_state.status == ApplicationStatus.DEPLOYING check_obj_ref_ready_nowait.return_value = False app_state.update() mock_build.options.return_value.remote.assert_called_once() call_kwargs = mock_build.options.return_value.remote.call_args # deployment_to_deployment_actor_classes is the 9th positional arg (index 8) deployment_to_deployment_actor_classes = call_kwargs[0][8] assert deployment_to_deployment_actor_classes == { "MyDeployment": { "counter": "ray.serve.tests.test_deployment_actors:SharedCounter", "cache": "ray.serve.tests.test_deployment_actors:SharedCache", }, } class TestAutoscale: def test_autoscale(self, mocked_application_state_manager): """Test autoscaling behavior with two deployments under load.""" ( app_state_manager, deployment_state_manager, _, ) = mocked_application_state_manager # Setup: Create autoscaling configuration autoscaling_config = { "target_ongoing_requests": 1, "min_replicas": 1, "max_replicas": 5, "initial_replicas": 1, "upscale_delay_s": 0, "downscale_delay_s": 0, "metrics_interval_s": 0.1, } # Setup: Deploy two deployments d1_id, d2_id = self._deploy_test_deployments( app_state_manager, deployment_state_manager, autoscaling_config ) # Setup: Register deployments with autoscaling manager asm = app_state_manager._autoscaling_state_manager self._register_deployments_with_asm(asm, d1_id, d2_id, autoscaling_config) # Setup: Create running replicas self._create_running_replicas(asm, d1_id, d2_id) # Test: Simulate load metrics self._simulate_load_metrics(asm, d1_id, d2_id) # Verify: Check autoscaling decisions app_state_manager.update() assert app_state_manager.get_app_status("test_app") == ApplicationStatus.RUNNING assert deployment_state_manager._scaling_decisions == {d1_id: 4, d2_id: 2} def test_should_autoscale_with_autoscaling_deployments( self, mocked_application_state_manager ): """Test should_autoscale returns True when app has autoscaling deployments.""" ( app_state_manager, deployment_state_manager, _, ) = mocked_application_state_manager # Setup: Create autoscaling configuration autoscaling_config = { "target_ongoing_requests": 1, "min_replicas": 1, "max_replicas": 5, "initial_replicas": 1, } # Deploy app with autoscaling enabled d1_id, d2_id = self._deploy_test_deployments( app_state_manager, deployment_state_manager, autoscaling_config ) # Register with autoscaling manager asm = app_state_manager._autoscaling_state_manager self._register_deployments_with_asm(asm, d1_id, d2_id, autoscaling_config) # Get the application state app_state = app_state_manager._application_states["test_app"] # Verify should_autoscale returns True assert app_state.should_autoscale() is True def test_should_autoscale_without_autoscaling_deployments( self, mocked_application_state_manager ): """Test should_autoscale returns False when app has no autoscaling deployments.""" ( app_state_manager, deployment_state_manager, _, ) = mocked_application_state_manager # Deploy app without autoscaling configuration d1_id = DeploymentID(name="d1", app_name="test_app") d1_params = deployment_params("d1", "/hi") # No autoscaling config app_state_manager.deploy_app( "test_app", [d1_params], ApplicationArgsProto(external_scaler_enabled=False) ) app_state_manager.update() deployment_state_manager.set_deployment_healthy(d1_id) app_state_manager.update() # Get the application state app_state = app_state_manager._application_states["test_app"] # Verify should_autoscale returns False assert app_state.should_autoscale() is False def test_autoscale_with_no_deployments(self, mocked_application_state_manager): """Test autoscale returns False when app has no target deployments.""" app_state_manager, _, _ = mocked_application_state_manager # Create app state without any deployments app_state = ApplicationState( name="empty_app", deployment_state_manager=MockDeploymentStateManager(MockKVStore()), autoscaling_state_manager=AutoscalingStateManager(), endpoint_state=MockEndpointState(), logging_config=LoggingConfig(), external_scaler_enabled=False, ) # Verify autoscale returns False assert app_state.autoscale() is False def test_autoscale_with_deployment_details_none( self, mocked_application_state_manager ): """Test autoscale handles None deployment details gracefully.""" ( app_state_manager, deployment_state_manager, _, ) = mocked_application_state_manager # Setup: Deploy app with autoscaling autoscaling_config = { "target_ongoing_requests": 1, "min_replicas": 1, "max_replicas": 5, "initial_replicas": 1, } d1_id, d2_id = self._deploy_test_deployments( app_state_manager, deployment_state_manager, autoscaling_config ) # Mock get_deployment_target_num_replicas to return None deployment_state_manager.get_deployment_target_num_replicas = Mock( return_value=None ) app_state = app_state_manager._application_states["test_app"] # Verify autoscale returns False when deployment details are None assert app_state.autoscale() is False def test_autoscale_applies_decisions_correctly( self, mocked_application_state_manager ): """Test autoscale applies autoscaling decisions to deployment state manager.""" ( app_state_manager, deployment_state_manager, _, ) = mocked_application_state_manager # Setup: Deploy app with autoscaling autoscaling_config = { "target_ongoing_requests": 1, "min_replicas": 1, "max_replicas": 5, "initial_replicas": 1, "upscale_delay_s": 0, "downscale_delay_s": 0, "metrics_interval_s": 0.1, } d1_id, d2_id = self._deploy_test_deployments( app_state_manager, deployment_state_manager, autoscaling_config ) # Register with autoscaling manager and create replicas asm = app_state_manager._autoscaling_state_manager self._register_deployments_with_asm(asm, d1_id, d2_id, autoscaling_config) self._create_running_replicas(asm, d1_id, d2_id) # Simulate load: d1 has 3x target load, d2 has 0.5x target load self._simulate_load_metrics(asm, d1_id, d2_id, d1_load=3, d2_load=0) app_state = app_state_manager._application_states["test_app"] # Call autoscale result = app_state.autoscale() # Verify it returns True (target state changed) assert result is True # Verify scaling decisions were applied # d1 should scale up (high load), d2 should scale down (low load) assert d1_id in deployment_state_manager._scaling_decisions assert d2_id in deployment_state_manager._scaling_decisions def test_autoscale_no_decisions_returns_false( self, mocked_application_state_manager ): """Test autoscale returns False when no autoscaling decisions are made.""" ( app_state_manager, deployment_state_manager, _, ) = mocked_application_state_manager # Setup: Deploy app with autoscaling autoscaling_config = { "target_ongoing_requests": 1, "min_replicas": 1, "max_replicas": 5, "initial_replicas": 1, "upscale_delay_s": 0, "downscale_delay_s": 0, "metrics_interval_s": 0.1, } d1_id, d2_id = self._deploy_test_deployments( app_state_manager, deployment_state_manager, autoscaling_config ) # Register with autoscaling manager and create replicas asm = app_state_manager._autoscaling_state_manager self._register_deployments_with_asm(asm, d1_id, d2_id, autoscaling_config) self._create_running_replicas(asm, d1_id, d2_id) # Simulate balanced load (exactly at target, so no scaling needed) self._simulate_load_metrics(asm, d1_id, d2_id, d1_load=1, d2_load=1) app_state = app_state_manager._application_states["test_app"] # Call autoscale result = app_state.autoscale() # Verify it returns False (no scaling decisions needed) # When load exactly matches target, autoscaler shouldn't make changes assert result is False or deployment_state_manager._scaling_decisions == { d1_id: 2, d2_id: 2, } def test_application_state_manager_autoscaling_integration( self, mocked_application_state_manager ): """Test autoscaling integration in ApplicationStateManager.update().""" ( app_state_manager, deployment_state_manager, _, ) = mocked_application_state_manager # Setup: Deploy app with autoscaling autoscaling_config = { "target_ongoing_requests": 1, "min_replicas": 1, "max_replicas": 5, "initial_replicas": 1, "upscale_delay_s": 0, "downscale_delay_s": 0, "metrics_interval_s": 0.1, } d1_id, d2_id = self._deploy_test_deployments( app_state_manager, deployment_state_manager, autoscaling_config ) # Register with autoscaling manager and create replicas asm = app_state_manager._autoscaling_state_manager self._register_deployments_with_asm(asm, d1_id, d2_id, autoscaling_config) self._create_running_replicas(asm, d1_id, d2_id) # Simulate high load on d1, moderate load on d2 self._simulate_load_metrics(asm, d1_id, d2_id, d1_load=4, d2_load=2) # Clear any existing scaling decisions deployment_state_manager._scaling_decisions.clear() # Call ApplicationStateManager.update() app_state_manager.update() # Verify autoscaling decisions were applied during update # Both deployments should have scaling decisions due to load assert len(deployment_state_manager._scaling_decisions) > 0 def test_autoscaling_with_mixed_deployment_types( self, mocked_application_state_manager ): """Test autoscaling behavior with mix of autoscaling and non-autoscaling deployments.""" ( app_state_manager, deployment_state_manager, _, ) = mocked_application_state_manager # Setup: Deploy app with one autoscaling and one non-autoscaling deployment autoscaling_config = { "target_ongoing_requests": 1, "min_replicas": 1, "max_replicas": 5, "initial_replicas": 1, "upscale_delay_s": 0, "downscale_delay_s": 0, "metrics_interval_s": 0.1, } d1_id = DeploymentID(name="d1", app_name="test_app") d2_id = DeploymentID(name="d2", app_name="test_app") # d1 has autoscaling, d2 doesn't d1_params = deployment_params( "d1", "/hi", autoscaling_config=autoscaling_config ) d2_params = deployment_params("d2") # No autoscaling config app_state_manager.deploy_app( "test_app", [d1_params, d2_params], ApplicationArgsProto(external_scaler_enabled=False), ) app_state_manager.update() deployment_state_manager.set_deployment_healthy(d1_id) deployment_state_manager.set_deployment_healthy(d2_id) app_state_manager.update() # Register only d1 with autoscaling manager and create replicas asm = app_state_manager._autoscaling_state_manager d1_info = deployment_info("d1", "/hi", autoscaling_config=autoscaling_config) asm.register_deployment(d1_id, d1_info, 1) # Create replicas for d1 only d1_replicas = [ ReplicaID(unique_id=f"replica_{i}", deployment_id=d1_id) for i in [1, 2] ] asm.update_running_replica_ids(d1_id, d1_replicas) # Simulate high load on d1 current_time = time.time() timestamp_offset = current_time - 0.1 if RAY_SERVE_COLLECT_AUTOSCALING_METRICS_ON_HANDLE: r1 = ReplicaID(unique_id="replica_1", deployment_id=d1_id) r2 = ReplicaID(unique_id="replica_2", deployment_id=d1_id) d1_handle_report = HandleMetricReport( deployment_id=d1_id, handle_id="random", actor_id="actor_id", handle_source=DeploymentHandleSource.UNKNOWN, queued_requests=[TimeStampedValue(timestamp_offset, 0)], aggregated_queued_requests=0, aggregated_metrics={ RUNNING_REQUESTS_KEY: { r1.to_full_id_str(): 3, r2.to_full_id_str(): 3, } }, metrics={ RUNNING_REQUESTS_KEY: { r1.to_full_id_str(): [TimeStampedValue(timestamp_offset, 3)], r2.to_full_id_str(): [TimeStampedValue(timestamp_offset, 3)], } }, timestamp=time.time(), ) asm.record_request_metrics_for_handle(d1_handle_report) else: for i in [1, 2]: replica_report = ReplicaMetricReport( replica_id=ReplicaID(unique_id=f"replica_{i}", deployment_id=d1_id), aggregated_metrics={RUNNING_REQUESTS_KEY: 3}, metrics={ RUNNING_REQUESTS_KEY: [TimeStampedValue(timestamp_offset, 3)] }, timestamp=time.time(), ) asm.record_request_metrics_for_replica(replica_report) app_state = app_state_manager._application_states["test_app"] # Call autoscale result = app_state.autoscale() # Verify only d1's decision was applied (d2 has no autoscaling) assert result is True assert d1_id in deployment_state_manager._scaling_decisions assert d2_id not in deployment_state_manager._scaling_decisions def test_autoscale_multiple_apps_independent( self, mocked_application_state_manager ): """Test that autoscaling decisions for one app don't affect another app.""" ( app_state_manager, deployment_state_manager, _, ) = mocked_application_state_manager # Setup: Create autoscaling configuration autoscaling_config = { "target_ongoing_requests": 1, "min_replicas": 1, "max_replicas": 5, "initial_replicas": 1, "upscale_delay_s": 0, "downscale_delay_s": 0, "metrics_interval_s": 0.1, } # Deploy app1 with two deployments app1_d1_id = DeploymentID(name="d1", app_name="app1") app1_d2_id = DeploymentID(name="d2", app_name="app1") app1_d1_params = deployment_params( "d1", "/app1", autoscaling_config=autoscaling_config ) app1_d2_params = deployment_params("d2", autoscaling_config=autoscaling_config) app_state_manager.deploy_app( "app1", [app1_d1_params, app1_d2_params], ApplicationArgsProto(external_scaler_enabled=False), ) app_state_manager.update() deployment_state_manager.set_deployment_healthy(app1_d1_id) deployment_state_manager.set_deployment_healthy(app1_d2_id) app_state_manager.update() # Deploy app2 with two deployments app2_d1_id = DeploymentID(name="d1", app_name="app2") app2_d2_id = DeploymentID(name="d2", app_name="app2") app2_d1_params = deployment_params( "d1", "/app2", autoscaling_config=autoscaling_config ) app2_d2_params = deployment_params("d2", autoscaling_config=autoscaling_config) app_state_manager.deploy_app( "app2", [app2_d1_params, app2_d2_params], ApplicationArgsProto(external_scaler_enabled=False), ) app_state_manager.update() deployment_state_manager.set_deployment_healthy(app2_d1_id) deployment_state_manager.set_deployment_healthy(app2_d2_id) app_state_manager.update() # Register app1 deployments with autoscaling manager asm = app_state_manager._autoscaling_state_manager app1_d1_info = deployment_info( "d1", "/app1", autoscaling_config=autoscaling_config ) app1_d2_info = deployment_info("d2", autoscaling_config=autoscaling_config) app1_d1_info.app_name = "app1" app1_d2_info.app_name = "app1" asm.register_deployment(app1_d1_id, app1_d1_info, 1) asm.register_deployment(app1_d2_id, app1_d2_info, 1) # Register app2 deployments with autoscaling manager app2_d1_info = deployment_info( "d1", "/app2", autoscaling_config=autoscaling_config ) app2_d2_info = deployment_info("d2", autoscaling_config=autoscaling_config) app2_d1_info.app_name = "app2" app2_d2_info.app_name = "app2" asm.register_deployment(app2_d1_id, app2_d1_info, 1) asm.register_deployment(app2_d2_id, app2_d2_info, 1) # Create replicas for both apps app1_d1_replicas = [ ReplicaID(unique_id=f"app1_d1_replica_{i}", deployment_id=app1_d1_id) for i in [1, 2] ] app1_d2_replicas = [ ReplicaID(unique_id=f"app1_d2_replica_{i}", deployment_id=app1_d2_id) for i in [3, 4] ] asm.update_running_replica_ids(app1_d1_id, app1_d1_replicas) asm.update_running_replica_ids(app1_d2_id, app1_d2_replicas) app2_d1_replicas = [ ReplicaID(unique_id=f"app2_d1_replica_{i}", deployment_id=app2_d1_id) for i in [5, 6] ] app2_d2_replicas = [ ReplicaID(unique_id=f"app2_d2_replica_{i}", deployment_id=app2_d2_id) for i in [7, 8] ] asm.update_running_replica_ids(app2_d1_id, app2_d1_replicas) asm.update_running_replica_ids(app2_d2_id, app2_d2_replicas) # Simulate high load on app1, low load on app2 current_time = time.time() timestamp_offset = current_time - 0.1 # App1: High load for replica_id in app1_d1_replicas + app1_d2_replicas: replica_report = ReplicaMetricReport( replica_id=replica_id, aggregated_metrics={RUNNING_REQUESTS_KEY: 3}, metrics={RUNNING_REQUESTS_KEY: [TimeStampedValue(timestamp_offset, 3)]}, timestamp=time.time(), ) asm.record_request_metrics_for_replica(replica_report) # App2: Low load for replica_id in app2_d1_replicas + app2_d2_replicas: replica_report = ReplicaMetricReport( replica_id=replica_id, aggregated_metrics={RUNNING_REQUESTS_KEY: 0}, metrics={RUNNING_REQUESTS_KEY: [TimeStampedValue(timestamp_offset, 0)]}, timestamp=time.time(), ) asm.record_request_metrics_for_replica(replica_report) # Clear scaling decisions deployment_state_manager._scaling_decisions.clear() # Call update which triggers autoscaling for both apps app_state_manager.update() # Verify app1 deployments scaled up (high load) assert app1_d1_id in deployment_state_manager._scaling_decisions assert app1_d2_id in deployment_state_manager._scaling_decisions assert deployment_state_manager._scaling_decisions[app1_d1_id] > 2 # Verify app2 deployments scaled down (low load) assert app2_d1_id in deployment_state_manager._scaling_decisions assert app2_d2_id in deployment_state_manager._scaling_decisions assert deployment_state_manager._scaling_decisions[app2_d1_id] == 1 def test_autoscale_with_partial_deployment_details( self, mocked_application_state_manager ): """Test autoscale when some deployments have details and others return None.""" ( app_state_manager, deployment_state_manager, _, ) = mocked_application_state_manager # Setup: Deploy app with autoscaling autoscaling_config = { "target_ongoing_requests": 1, "min_replicas": 1, "max_replicas": 5, "initial_replicas": 1, "upscale_delay_s": 0, "downscale_delay_s": 0, "metrics_interval_s": 0.1, } d1_id, d2_id = self._deploy_test_deployments( app_state_manager, deployment_state_manager, autoscaling_config ) # Register only d1 with autoscaling manager (d2 won't be registered) asm = app_state_manager._autoscaling_state_manager d1_info = deployment_info("d1", "/hi", autoscaling_config=autoscaling_config) asm.register_deployment(d1_id, d1_info, 1) # Create replicas for d1 only d1_replicas = [ ReplicaID(unique_id=f"d1_replica_{i}", deployment_id=d1_id) for i in [1, 2] ] asm.update_running_replica_ids(d1_id, d1_replicas) # Simulate load for d1 current_time = time.time() timestamp_offset = current_time - 0.1 for i in [1, 2]: replica_report = ReplicaMetricReport( replica_id=ReplicaID(unique_id=f"d1_replica_{i}", deployment_id=d1_id), aggregated_metrics={RUNNING_REQUESTS_KEY: 3}, metrics={RUNNING_REQUESTS_KEY: [TimeStampedValue(timestamp_offset, 3)]}, timestamp=time.time(), ) asm.record_request_metrics_for_replica(replica_report) # Mock get_deployment_target_num_replicas to return None for d2 only original_get_details = ( deployment_state_manager.get_deployment_target_num_replicas ) def selective_get_details(dep_id) -> Optional[int]: if dep_id == d2_id: return None return original_get_details(dep_id) deployment_state_manager.get_deployment_target_num_replicas = ( selective_get_details ) app_state = app_state_manager._application_states["test_app"] # Call autoscale result = app_state.autoscale() # Verify it returns True (d1 has scaling decision) assert result is True # Verify only d1 has scaling decision (d2 was skipped due to None details) assert d1_id in deployment_state_manager._scaling_decisions assert d2_id not in deployment_state_manager._scaling_decisions def test_autoscale_single_deployment_in_app(self, mocked_application_state_manager): """Test autoscaling with only one deployment in the app.""" ( app_state_manager, deployment_state_manager, _, ) = mocked_application_state_manager # Setup: Create autoscaling configuration autoscaling_config = { "target_ongoing_requests": 1, "min_replicas": 1, "max_replicas": 5, "initial_replicas": 1, "upscale_delay_s": 0, "downscale_delay_s": 0, "metrics_interval_s": 0.1, } # Deploy single deployment d1_id = DeploymentID(name="d1", app_name="test_app") d1_params = deployment_params( "d1", "/hi", autoscaling_config=autoscaling_config ) app_state_manager.deploy_app( "test_app", [d1_params], ApplicationArgsProto(external_scaler_enabled=False) ) app_state_manager.update() deployment_state_manager.set_deployment_healthy(d1_id) app_state_manager.update() # Register with autoscaling manager asm = app_state_manager._autoscaling_state_manager d1_info = deployment_info("d1", "/hi", autoscaling_config=autoscaling_config) asm.register_deployment(d1_id, d1_info, 1) # Create replicas d1_replicas = [ ReplicaID(unique_id=f"replica_{i}", deployment_id=d1_id) for i in [1, 2] ] asm.update_running_replica_ids(d1_id, d1_replicas) # Simulate high load current_time = time.time() timestamp_offset = current_time - 0.1 for i in [1, 2]: replica_report = ReplicaMetricReport( replica_id=ReplicaID(unique_id=f"replica_{i}", deployment_id=d1_id), aggregated_metrics={RUNNING_REQUESTS_KEY: 4}, metrics={RUNNING_REQUESTS_KEY: [TimeStampedValue(timestamp_offset, 4)]}, timestamp=time.time(), ) asm.record_request_metrics_for_replica(replica_report) app_state = app_state_manager._application_states["test_app"] # Call autoscale result = app_state.autoscale() # Verify it returns True assert result is True # Verify scaling decision was made assert d1_id in deployment_state_manager._scaling_decisions assert deployment_state_manager._scaling_decisions[d1_id] > 2 def test_autoscale_during_app_deletion(self, mocked_application_state_manager): """Test autoscaling behavior when app is being deleted.""" ( app_state_manager, deployment_state_manager, _, ) = mocked_application_state_manager # Setup: Deploy app with autoscaling autoscaling_config = { "target_ongoing_requests": 1, "min_replicas": 1, "max_replicas": 5, "initial_replicas": 1, "upscale_delay_s": 0, "downscale_delay_s": 0, "metrics_interval_s": 0.1, } d1_id, d2_id = self._deploy_test_deployments( app_state_manager, deployment_state_manager, autoscaling_config ) # Register with autoscaling manager and create replicas asm = app_state_manager._autoscaling_state_manager self._register_deployments_with_asm(asm, d1_id, d2_id, autoscaling_config) self._create_running_replicas(asm, d1_id, d2_id) # Simulate load self._simulate_load_metrics(asm, d1_id, d2_id, d1_load=5, d2_load=5) # Delete the app app_state_manager.delete_app("test_app") # Get app state app_state = app_state_manager._application_states["test_app"] # Verify app status is DELETING assert app_state.status == ApplicationStatus.DELETING # Clear scaling decisions deployment_state_manager._scaling_decisions.clear() # Call update (should not autoscale deleting apps) app_state_manager.update() # Verify no autoscaling decisions were made (app is deleting) assert len(deployment_state_manager._scaling_decisions) == 0 def test_autoscale_many_deployments_in_app(self, mocked_application_state_manager): """Test autoscaling with many (15+) deployments in single app.""" ( app_state_manager, deployment_state_manager, _, ) = mocked_application_state_manager # Setup: Create autoscaling configuration autoscaling_config = { "target_ongoing_requests": 1, "min_replicas": 1, "max_replicas": 3, "initial_replicas": 1, "upscale_delay_s": 0, "downscale_delay_s": 0, "metrics_interval_s": 0.1, } # Deploy 15 deployments num_deployments = 15 deployment_ids = [] deployment_params_list = [] for i in range(num_deployments): deployment_ids.append(DeploymentID(name=f"d{i}", app_name="test_app")) deployment_params_list.append( deployment_params(f"d{i}", autoscaling_config=autoscaling_config) ) app_state_manager.deploy_app( "test_app", deployment_params_list, ApplicationArgsProto(external_scaler_enabled=False), ) app_state_manager.update() # Mark all as healthy for dep_id in deployment_ids: deployment_state_manager.set_deployment_healthy(dep_id) app_state_manager.update() # Register all with autoscaling manager asm = app_state_manager._autoscaling_state_manager for i, dep_id in enumerate(deployment_ids): info = deployment_info(f"d{i}", autoscaling_config=autoscaling_config) asm.register_deployment(dep_id, info, 1) # Create replicas replicas = [ ReplicaID(unique_id=f"d{i}_replica_{j}", deployment_id=dep_id) for j in [1, 2] ] asm.update_running_replica_ids(dep_id, replicas) # Simulate load (alternating high/low) load = 3 if i % 2 == 0 else 0 current_time = time.time() timestamp_offset = current_time - 0.1 for replica in replicas: replica_report = ReplicaMetricReport( replica_id=replica, aggregated_metrics={RUNNING_REQUESTS_KEY: load}, metrics={ RUNNING_REQUESTS_KEY: [TimeStampedValue(timestamp_offset, load)] }, timestamp=time.time(), ) asm.record_request_metrics_for_replica(replica_report) # Clear scaling decisions deployment_state_manager._scaling_decisions.clear() # Call update app_state_manager.update() # Verify all deployments have scaling decisions assert len(deployment_state_manager._scaling_decisions) == num_deployments # Verify high-load deployments scaled up for i in range(0, num_deployments, 2): # Even indices have high load assert deployment_state_manager._scaling_decisions[deployment_ids[i]] == 3 # Verify low-load deployments scaled down for i in range(1, num_deployments, 2): # Odd indices have low load assert deployment_state_manager._scaling_decisions[deployment_ids[i]] == 1 def test_autoscale_with_min_equals_max_replicas( self, mocked_application_state_manager ): """Test autoscaling when min_replicas equals max_replicas (no room to scale).""" ( app_state_manager, deployment_state_manager, _, ) = mocked_application_state_manager # Setup: Create autoscaling configuration with no scaling room autoscaling_config = { "target_ongoing_requests": 1, "min_replicas": 3, "max_replicas": 3, # Same as min "initial_replicas": 3, "upscale_delay_s": 0, "downscale_delay_s": 0, "metrics_interval_s": 0.1, } d1_id = DeploymentID(name="d1", app_name="test_app") d1_params = deployment_params( "d1", "/hi", autoscaling_config=autoscaling_config ) app_state_manager.deploy_app( "test_app", [d1_params], ApplicationArgsProto(external_scaler_enabled=False) ) app_state_manager.update() deployment_state_manager.set_deployment_healthy(d1_id) app_state_manager.update() # Register with autoscaling manager asm = app_state_manager._autoscaling_state_manager d1_info = deployment_info("d1", "/hi", autoscaling_config=autoscaling_config) asm.register_deployment(d1_id, d1_info, 3) # Create replicas d1_replicas = [ ReplicaID(unique_id=f"replica_{i}", deployment_id=d1_id) for i in range(3) ] asm.update_running_replica_ids(d1_id, d1_replicas) # Simulate extreme load (should want to scale up but can't) current_time = time.time() timestamp_offset = current_time - 0.1 for i in range(3): replica_report = ReplicaMetricReport( replica_id=ReplicaID(unique_id=f"replica_{i}", deployment_id=d1_id), aggregated_metrics={RUNNING_REQUESTS_KEY: 10}, metrics={ RUNNING_REQUESTS_KEY: [TimeStampedValue(timestamp_offset, 10)] }, timestamp=time.time(), ) asm.record_request_metrics_for_replica(replica_report) app_state = app_state_manager._application_states["test_app"] # Call autoscale _ = app_state.autoscale() # Decision should be made but capped at max_replicas (3) assert d1_id in deployment_state_manager._scaling_decisions assert deployment_state_manager._scaling_decisions[d1_id] == 3 def test_autoscale_multiple_updates_stable_load( self, mocked_application_state_manager ): """Test multiple update() calls with stable load don't cause thrashing.""" ( app_state_manager, deployment_state_manager, _, ) = mocked_application_state_manager # Setup: Deploy app with autoscaling autoscaling_config = { "target_ongoing_requests": 1, "min_replicas": 1, "max_replicas": 5, "initial_replicas": 2, "upscale_delay_s": 0, "downscale_delay_s": 0, "metrics_interval_s": 0.1, } d1_id, d2_id = self._deploy_test_deployments( app_state_manager, deployment_state_manager, autoscaling_config ) # Register with autoscaling manager and create replicas asm = app_state_manager._autoscaling_state_manager self._register_deployments_with_asm(asm, d1_id, d2_id, autoscaling_config) self._create_running_replicas(asm, d1_id, d2_id) # Simulate stable load at target self._simulate_load_metrics(asm, d1_id, d2_id, d1_load=1, d2_load=1) # Clear scaling decisions deployment_state_manager._scaling_decisions.clear() # Call update multiple times for _ in range(5): app_state_manager.update() # Verify decisions are stable (should be 2 replicas - no change) # If decisions keep changing, that's thrashing if deployment_state_manager._scaling_decisions: assert deployment_state_manager._scaling_decisions.get(d1_id, 2) == 2 assert deployment_state_manager._scaling_decisions.get(d2_id, 2) == 2 def _deploy_test_deployments( self, app_state_manager, deployment_state_manager, autoscaling_config ): """Deploy two test deployments and mark them as healthy.""" d1_id = DeploymentID(name="d1", app_name="test_app") d2_id = DeploymentID(name="d2", app_name="test_app") d1_params = deployment_params( "d1", "/hi", autoscaling_config=autoscaling_config ) d2_params = deployment_params("d2", autoscaling_config=autoscaling_config) app_state_manager.deploy_app( "test_app", [d1_params, d2_params], ApplicationArgsProto(external_scaler_enabled=False), ) app_state_manager.update() deployment_state_manager.set_deployment_healthy(d1_id) deployment_state_manager.set_deployment_healthy(d2_id) app_state_manager.update() assert app_state_manager.get_app_status("test_app") == ApplicationStatus.RUNNING return d1_id, d2_id def _register_deployments_with_asm(self, asm, d1_id, d2_id, autoscaling_config): """Register deployments with the autoscaling state manager.""" d1_info = deployment_info("d1", "/hi", autoscaling_config=autoscaling_config) d2_info = deployment_info("d2", autoscaling_config=autoscaling_config) asm.register_deployment(d1_id, d1_info, 1) asm.register_deployment(d2_id, d2_info, 1) def _create_running_replicas(self, asm, d1_id, d2_id): """Create running replicas for both deployments.""" # d1 gets 2 replicas d1_replicas = [ ReplicaID(unique_id=f"replica_{i}", deployment_id=d1_id) for i in [1, 2] ] asm.update_running_replica_ids(d1_id, d1_replicas) # d2 gets 2 replicas d2_replicas = [ ReplicaID(unique_id=f"replica_{i}", deployment_id=d2_id) for i in [3, 4] ] asm.update_running_replica_ids(d2_id, d2_replicas) def _simulate_load_metrics(self, asm, d1_id, d2_id, d1_load=2, d2_load=1): current_time = time.time() timestamp_offset = current_time - 0.1 if RAY_SERVE_COLLECT_AUTOSCALING_METRICS_ON_HANDLE: self._record_handle_metrics( asm, d1_id, d2_id, timestamp_offset, d1_load, d2_load ) else: self._record_replica_metrics( asm, d1_id, d2_id, timestamp_offset, d1_load, d2_load ) def _record_handle_metrics( self, asm, d1_id, d2_id, timestamp_offset, d1_load=2, d2_load=1 ): """Record metrics using handle-based reporting.""" # d1: Load based on d1_load parameter d1_r1 = ReplicaID(unique_id="replica_1", deployment_id=d1_id) d1_r2 = ReplicaID(unique_id="replica_2", deployment_id=d1_id) d1_handle_report = HandleMetricReport( deployment_id=d1_id, handle_id="random", actor_id="actor_id", handle_source=DeploymentHandleSource.UNKNOWN, queued_requests=[TimeStampedValue(timestamp_offset, 0)], aggregated_queued_requests=0, aggregated_metrics={ RUNNING_REQUESTS_KEY: { d1_r1.to_full_id_str(): d1_load, d1_r2.to_full_id_str(): d1_load, } }, metrics={ RUNNING_REQUESTS_KEY: { d1_r1.to_full_id_str(): [ TimeStampedValue(timestamp_offset, d1_load) ], d1_r2.to_full_id_str(): [ TimeStampedValue(timestamp_offset, d1_load) ], } }, timestamp=time.time(), ) asm.record_request_metrics_for_handle(d1_handle_report) # d2: Load based on d2_load parameter d2_r3 = ReplicaID(unique_id="replica_3", deployment_id=d2_id) d2_r4 = ReplicaID(unique_id="replica_4", deployment_id=d2_id) d2_handle_report = HandleMetricReport( deployment_id=d2_id, handle_id="random", actor_id="actor_id", handle_source=DeploymentHandleSource.UNKNOWN, queued_requests=[TimeStampedValue(timestamp_offset, 0)], aggregated_queued_requests=0, aggregated_metrics={ RUNNING_REQUESTS_KEY: { d2_r3.to_full_id_str(): d2_load, d2_r4.to_full_id_str(): d2_load, } }, metrics={ RUNNING_REQUESTS_KEY: { d2_r3.to_full_id_str(): [ TimeStampedValue(timestamp_offset, d2_load) ], d2_r4.to_full_id_str(): [ TimeStampedValue(timestamp_offset, d2_load) ], } }, timestamp=time.time(), ) asm.record_request_metrics_for_handle(d2_handle_report) def _record_replica_metrics( self, asm, d1_id, d2_id, timestamp_offset, d1_load=2, d2_load=1 ): """Record metrics using replica-based reporting.""" # d1: Load based on d1_load parameter for i in [1, 2]: replica_report = ReplicaMetricReport( replica_id=ReplicaID(unique_id=f"replica_{i}", deployment_id=d1_id), aggregated_metrics={RUNNING_REQUESTS_KEY: d1_load}, metrics={ RUNNING_REQUESTS_KEY: [TimeStampedValue(timestamp_offset, d1_load)] }, timestamp=time.time(), ) asm.record_request_metrics_for_replica(replica_report) # d2: Load based on d2_load parameter for i in [3, 4]: replica_report = ReplicaMetricReport( replica_id=ReplicaID(unique_id=f"replica_{i}", deployment_id=d2_id), aggregated_metrics={RUNNING_REQUESTS_KEY: d2_load}, metrics={ RUNNING_REQUESTS_KEY: [TimeStampedValue(timestamp_offset, d2_load)] }, timestamp=time.time(), ) asm.record_request_metrics_for_replica(replica_report) def simple_app_level_policy(contexts): """Simple policy that scales all deployments to 3 replicas.""" decisions = {} for deployment_id, _ in contexts.items(): decisions[deployment_id] = 3 return decisions, {} def stateful_app_level_policy(contexts): """Stateful application level policy that increments a counter in policy_state. Used in tests to verify that application level autoscaling policy state is persisted and passed back into subsequent policy invocations. """ # Increment the internal state everytime the policy is called new_state = {} for deployment_id, ctx in contexts.items(): prev_counter = 0 if ctx.policy_state: prev_counter = ctx.policy_state.get("counter", 0) new_state[deployment_id] = {"counter": prev_counter + 1} # Scale all deployments to 3 replicas decisions = {deployment_id: 3 for deployment_id in contexts.keys()} # Persist updated counter for next iteration. return decisions, new_state def app_level_policy_with_decorator(contexts): """App-level policy used to verify that the decorator applies delay logic.""" decisions = {} for dep_id, ctx in contexts.items(): curr = ctx.target_num_replicas if curr < 5: decisions[dep_id] = 5 elif curr > 1: decisions[dep_id] = 1 else: decisions[dep_id] = curr return decisions, {} def partial_app_level_policy(contexts): """Policy that returns decisions for only a subset of deployments.""" decisions = {} for deployment_id in contexts.keys(): if deployment_id.name == "d1": decisions[deployment_id] = 4 return decisions, {} def partial_decisions_app_level_policy(contexts): """ The decison for deployment "d1" is skipped but state for each deployment is always provided """ decisions = {} new_state = {} for deployment_id, ctx in contexts.items(): prev_counter = 0 if ctx.policy_state: prev_counter = ctx.policy_state.get("counter", 0) if deployment_id.name != "d1": decisions[deployment_id] = 3 # Pass the state regardless new_state[deployment_id] = {"counter": prev_counter + 1} return decisions, new_state class TestApplicationLevelAutoscaling: """Test application-level autoscaling policy registration, execution, and lifecycle.""" def _create_app_config( self, app_name="test_app", has_policy=True, deployments=None ): """Helper to create a ServeApplicationSchema with optional autoscaling policy.""" if deployments is None: deployments = [ DeploymentSchema( name="d1", autoscaling_config={ "target_ongoing_requests": 1, "min_replicas": 1, "max_replicas": 5, "initial_replicas": 1, }, ) ] # Overriding the default delay values for deterministic behavior for unit tests for d in deployments: if d.autoscaling_config is None: continue d.autoscaling_config.setdefault("upscale_delay_s", 0.0) d.autoscaling_config.setdefault("downscale_delay_s", 0.0) return ServeApplicationSchema( name=app_name, import_path="fake.import.path", route_prefix="/hi", autoscaling_policy={ "policy_function": "ray.serve.tests.unit.test_application_state:simple_app_level_policy" } if has_policy else None, deployments=deployments, ) def _deploy_app_with_mocks(self, app_state_manager, app_config): """Helper to deploy an app with proper mocking to avoid Ray initialization.""" with patch( "ray.serve._private.application_state.build_serve_application" ) as mock_build: mock_build.return_value = Mock() app_state_manager.apply_app_configs([app_config]) app_state = app_state_manager._application_states[app_config.name] app_state._build_app_task_info = Mock() app_state._build_app_task_info.code_version = "test_version" app_state._build_app_task_info.config = app_config app_state._build_app_task_info.target_capacity = None app_state._build_app_task_info.target_capacity_direction = None # Mock reconcile to succeed with patch.object(app_state, "_reconcile_build_app_task") as mock_reconcile: deployment_infos = {} for deployment in app_config.deployments: deployment_infos[deployment.name] = deployment_info( deployment.name, "/hi" if deployment.name == "d1" else None, autoscaling_config=deployment.autoscaling_config, ) mock_reconcile.return_value = ( None, deployment_infos, BuildAppStatus.SUCCEEDED, "", ) app_state.update() return app_state def _register_deployments(self, app_state_manager, app_config): """Helper to register deployments with autoscaling manager.""" # Pick autoscaling config from the app config asm = app_state_manager._autoscaling_state_manager for deployment in app_config.deployments: deployment_id = DeploymentID(name=deployment.name, app_name=app_config.name) deployment_info_obj = deployment_info( deployment.name, "/hi" if deployment.name == "d1" else None, autoscaling_config=deployment.autoscaling_config, ) asm.register_deployment(deployment_id, deployment_info_obj, 1) return asm def _deploy_multiple_apps_with_mocks(self, app_state_manager, app_configs): """Helper to deploy multiple apps simultaneously with proper mocking.""" # Deploy all apps at once with patch( "ray.serve._private.application_state.build_serve_application" ) as mock_build: mock_build.return_value = Mock() app_state_manager.apply_app_configs(app_configs) # Mock the build app tasks for all apps for app_config in app_configs: app_state = app_state_manager._application_states[app_config.name] app_state._build_app_task_info = Mock() app_state._build_app_task_info.code_version = "test_version" app_state._build_app_task_info.config = app_config app_state._build_app_task_info.target_capacity = None app_state._build_app_task_info.target_capacity_direction = None # Mock reconcile to succeed with patch.object(app_state, "_reconcile_build_app_task") as mock_reconcile: deployment_infos = {} for deployment in app_config.deployments: deployment_infos[deployment.name] = deployment_info( deployment.name, "/hi" if deployment.name == "d1" else None, autoscaling_config={ "target_ongoing_requests": 1, "min_replicas": 1, "max_replicas": 5, "initial_replicas": 1, }, ) mock_reconcile.return_value = ( None, deployment_infos, BuildAppStatus.SUCCEEDED, "", ) app_state.update() return app_state_manager._autoscaling_state_manager def test_app_level_autoscaling_policy_registration_and_execution( self, mocked_application_state_manager ): """Test that application-level autoscaling policy is registered and executed when set in config.""" ( app_state_manager, deployment_state_manager, _, ) = mocked_application_state_manager # Create app config with policy app_config = self._create_app_config() # Deploy app app_state = self._deploy_app_with_mocks(app_state_manager, app_config) # Register deployments asm = self._register_deployments(app_state_manager, app_config) # Verify policy was registered assert asm._application_has_policy("test_app") is True assert app_state.should_autoscale() is True assert asm.should_autoscale_application("test_app") is True # Create replicas and test autoscaling d1_id = DeploymentID(name="d1", app_name="test_app") d1_replicas = [ ReplicaID(unique_id=f"d1_replica_{i}", deployment_id=d1_id) for i in [1, 2] ] asm.update_running_replica_ids(d1_id, d1_replicas) # Clear scaling decisions and test autoscaling deployment_state_manager._scaling_decisions.clear() app_state_manager.update() # Verify policy was executed (scales to 3 replicas) assert deployment_state_manager._scaling_decisions[d1_id] == 3 @pytest.mark.parametrize( "policy_import_path", [ "ray.serve.tests.unit.test_application_state:partial_app_level_policy", ], ) def test_app_level_autoscaling_policy_can_return_partial_decisions( self, mocked_application_state_manager, policy_import_path ): """Omitted deployments from decisions should not be autoscaled.""" ( app_state_manager, deployment_state_manager, _, ) = mocked_application_state_manager # Create app config with two deployments and override to use the partial policy. deployments = [ DeploymentSchema( name="d1", autoscaling_config={ "target_ongoing_requests": 1, "min_replicas": 1, "max_replicas": 5, "initial_replicas": 1, "upscale_delay_s": 0.0, "downscale_delay_s": 0.0, "metrics_interval_s": 0.1, }, ), DeploymentSchema( name="d2", autoscaling_config={ "target_ongoing_requests": 1, "min_replicas": 1, "max_replicas": 5, "initial_replicas": 1, "upscale_delay_s": 0.0, "downscale_delay_s": 0.0, "metrics_interval_s": 0.1, }, ), ] app_config = self._create_app_config(deployments=deployments) app_config.autoscaling_policy = {"policy_function": policy_import_path} _ = self._deploy_app_with_mocks(app_state_manager, app_config) asm = self._register_deployments(app_state_manager, app_config) d1_id = DeploymentID(name="d1", app_name="test_app") d2_id = DeploymentID(name="d2", app_name="test_app") # Create replicas so autoscaling runs. d1_replicas = [ ReplicaID(unique_id=f"d1_replica_{i}", deployment_id=d1_id) for i in [1, 2] ] d2_replicas = [ ReplicaID(unique_id=f"d2_replica_{i}", deployment_id=d2_id) for i in [1, 2] ] asm.update_running_replica_ids(d1_id, d1_replicas) asm.update_running_replica_ids(d2_id, d2_replicas) # Add a previous decision for both depoloyments deployment_state_manager._scaling_decisions[d1_id] = 2 deployment_state_manager._scaling_decisions[d2_id] = 99 app_state_manager.update() assert deployment_state_manager._scaling_decisions[d1_id] == 4 assert deployment_state_manager._scaling_decisions[d2_id] == 99 def test_app_level_autoscaling_policy_recovery( self, mocked_application_state_manager ): """Test that application-level autoscaling policy is registered when recovered from checkpoint.""" ( app_state_manager, deployment_state_manager, kv_store, ) = mocked_application_state_manager # Deploy app with policy app_config = self._create_app_config() _ = self._deploy_app_with_mocks(app_state_manager, app_config) asm = self._register_deployments(app_state_manager, app_config) # Save checkpoint app_state_manager.update() # Simulate controller crash - create new managers new_deployment_state_manager = MockDeploymentStateManager(kv_store) new_app_state_manager = ApplicationStateManager( new_deployment_state_manager, asm, MockEndpointState(), kv_store, LoggingConfig(), ) # Recovery happens automatically during initialization # Verify app-level policy was recovered assert asm._application_has_policy("test_app") is True # Test that recovered policy still works d1_id = DeploymentID(name="d1", app_name="test_app") d1_replicas = [ ReplicaID(unique_id=f"d1_replica_{i}", deployment_id=d1_id) for i in [1, 2] ] asm.update_running_replica_ids(d1_id, d1_replicas) new_deployment_state_manager._scaling_decisions.clear() new_app_state_manager.update() assert new_deployment_state_manager._scaling_decisions[d1_id] == 3 def test_app_level_autoscaling_policy_deregistration_on_deletion( self, mocked_application_state_manager ): """Test that application-level autoscaling policy is deregistered when application is deleted.""" ( app_state_manager, deployment_state_manager, _, ) = mocked_application_state_manager # Deploy app with policy app_config = self._create_app_config() _ = self._deploy_app_with_mocks(app_state_manager, app_config) asm = self._register_deployments(app_state_manager, app_config) # Verify app is registered assert asm._application_has_policy("test_app") is True # Delete the application deployment_state_manager.delete_deployment( DeploymentID(name="d1", app_name="test_app") ) deployment_state_manager.set_deployment_deleted( DeploymentID(name="d1", app_name="test_app") ) app_state_manager.delete_app("test_app") app_state_manager.update() # Verify app-level policy is deregistered assert asm._application_has_policy("test_app") is False assert asm.should_autoscale_application("test_app") is False def test_app_level_autoscaling_policy_add_and_remove_from_config( self, mocked_application_state_manager ): """Test that application-level autoscaling policy is registered when added and deregistered when removed.""" ( app_state_manager, deployment_state_manager, _, ) = mocked_application_state_manager # Deploy app without policy initially app_config_no_policy = self._create_app_config(has_policy=False) _ = self._deploy_app_with_mocks(app_state_manager, app_config_no_policy) asm = self._register_deployments(app_state_manager, app_config_no_policy) # Verify no app-level policy initially # Note: The app might be registered but without a policy assert asm._application_has_policy("test_app") is False # Now add app-level autoscaling policy app_config_with_policy = self._create_app_config(has_policy=True) _ = self._deploy_app_with_mocks(app_state_manager, app_config_with_policy) # Verify app-level policy is registered assert asm._application_has_policy("test_app") is True # Now remove app-level autoscaling policy app_config_no_policy_again = self._create_app_config(has_policy=False) _ = self._deploy_app_with_mocks(app_state_manager, app_config_no_policy_again) # Verify app-level policy is deregistered # Note: The app might still exist but without a policy assert asm._application_has_policy("test_app") is False assert asm.should_autoscale_application("test_app") is False def test_app_level_autoscaling_policy_with_multiple_deployments( self, mocked_application_state_manager ): """Test that app-level autoscaling policy works correctly with multiple deployments.""" ( app_state_manager, deployment_state_manager, _, ) = mocked_application_state_manager # Create app with multiple deployments deployments = [ DeploymentSchema( name="d1", autoscaling_config={ "target_ongoing_requests": 1, "min_replicas": 1, "max_replicas": 10, "initial_replicas": 1, }, ), DeploymentSchema( name="d2", autoscaling_config={ "target_ongoing_requests": 1, "min_replicas": 1, "max_replicas": 10, "initial_replicas": 1, }, ), DeploymentSchema( name="d3", autoscaling_config={ "target_ongoing_requests": 1, "min_replicas": 1, "max_replicas": 10, "initial_replicas": 1, }, ), ] app_config = self._create_app_config(deployments=deployments) _ = self._deploy_app_with_mocks(app_state_manager, app_config) asm = self._register_deployments(app_state_manager, app_config) # Verify policy was registered assert asm._application_has_policy("test_app") is True # Create replicas for all deployments deployment_ids = [ DeploymentID(name=f"d{i}", app_name="test_app") for i in range(1, 4) ] for i, deployment_id in enumerate(deployment_ids): replicas = [ ReplicaID(unique_id=f"d{i+1}_replica_{j}", deployment_id=deployment_id) for j in [1, 2] ] asm.update_running_replica_ids(deployment_id, replicas) # Test autoscaling deployment_state_manager._scaling_decisions.clear() app_state_manager.update() # Verify all deployments were scaled to 3 (our policy scales all to 3) assert asm.should_autoscale_application("test_app") is True for deployment_id in deployment_ids: assert deployment_id in deployment_state_manager._scaling_decisions assert deployment_state_manager._scaling_decisions[deployment_id] == 3 def test_app_level_autoscaling_policy_state_persistence( self, mocked_application_state_manager ): """Test that app-level autoscaling policy state is maintained across multiple calls.""" ( app_state_manager, deployment_state_manager, _, ) = mocked_application_state_manager # Deploy app with policy app_config = self._create_app_config() _ = self._deploy_app_with_mocks(app_state_manager, app_config) asm = self._register_deployments(app_state_manager, app_config) # Create replicas d1_id = DeploymentID(name="d1", app_name="test_app") d1_replicas = [ ReplicaID(unique_id=f"d1_replica_{i}", deployment_id=d1_id) for i in [1, 2] ] asm.update_running_replica_ids(d1_id, d1_replicas) # Test multiple autoscaling calls for i in range(3): deployment_state_manager._scaling_decisions.clear() app_state_manager.update() assert asm.should_autoscale_application("test_app") is True assert deployment_state_manager._scaling_decisions[d1_id] == 3 def test_autoscaling_state_manager_helper_methods( self, mocked_application_state_manager ): """Test the new helper methods in AutoscalingStateManager.""" ( app_state_manager, deployment_state_manager, _, ) = mocked_application_state_manager asm = app_state_manager._autoscaling_state_manager # Test with no applications registered assert asm._application_has_policy("nonexistent_app") is False assert asm.should_autoscale_application("nonexistent_app") is False # Deploy app with policy app_config = self._create_app_config() _ = self._deploy_app_with_mocks(app_state_manager, app_config) asm = self._register_deployments(app_state_manager, app_config) # Test helper methods assert asm._application_has_policy("test_app") is True assert asm.should_autoscale_application("test_app") is True d1_id = DeploymentID(name="d1", app_name="test_app") assert asm.should_autoscale_deployment(d1_id) is True # Test with app without policy app_config_no_policy = self._create_app_config(has_policy=False) _ = self._deploy_app_with_mocks(app_state_manager, app_config_no_policy) asm_no_policy = self._register_deployments( app_state_manager, app_config_no_policy ) assert asm_no_policy._application_has_policy("test_app") is False assert ( asm_no_policy.should_autoscale_application("test_app") is True ) # App exists but no policy assert asm_no_policy.should_autoscale_deployment(d1_id) is True def test_get_decision_num_replicas_method(self, mocked_application_state_manager): """Test the get_decision_num_replicas method in AutoscalingStateManager.""" ( app_state_manager, deployment_state_manager, _, ) = mocked_application_state_manager # Deploy app with policy app_config = self._create_app_config() _ = self._deploy_app_with_mocks(app_state_manager, app_config) asm = self._register_deployments(app_state_manager, app_config) # Create replicas d1_id = DeploymentID(name="d1", app_name="test_app") d1_replicas = [ ReplicaID(unique_id=f"d1_replica_{i}", deployment_id=d1_id) for i in [1, 2] ] asm.update_running_replica_ids(d1_id, d1_replicas) # Test get_decision_num_replicas deployment_to_target_num_replicas = {d1_id: 2} decisions = asm.get_decision_num_replicas( "test_app", deployment_to_target_num_replicas ) assert d1_id in decisions assert decisions[d1_id] == 3 # Our policy scales to 3 def test_multiple_applications_autoscaling_isolation( self, mocked_application_state_manager ): """Test that autoscaling works correctly with multiple applications.""" ( app_state_manager, deployment_state_manager, _, ) = mocked_application_state_manager # Deploy both apps simultaneously app_config1 = self._create_app_config(app_name="app1") app_config2 = self._create_app_config(app_name="app2", has_policy=False) # Deploy both apps using new helper asm = self._deploy_multiple_apps_with_mocks( app_state_manager, [app_config1, app_config2] ) # Register deployments for both apps using existing helper asm = self._register_deployments(app_state_manager, app_config1) asm = self._register_deployments(app_state_manager, app_config2) # Test isolation assert asm._application_has_policy("app1") is True assert asm._application_has_policy("app2") is False assert asm.should_autoscale_application("app1") is True assert asm.should_autoscale_application("app2") is True # Test deployment-level isolation d1_app1_id = DeploymentID(name="d1", app_name="app1") d1_app2_id = DeploymentID(name="d1", app_name="app2") asm.update_running_replica_ids( d1_app1_id, [ ReplicaID(unique_id=f"d1_app1_replica_{i}", deployment_id=d1_app1_id) for i in [1, 2] ], ) asm.update_running_replica_ids( d1_app2_id, [ ReplicaID(unique_id=f"d1_app2_replica_{i}", deployment_id=d1_app2_id) for i in [1, 2] ], ) assert asm.should_autoscale_deployment(d1_app1_id) is True assert asm.should_autoscale_deployment(d1_app2_id) is True deployment_state_manager._scaling_decisions.clear() app_state_manager.update() # Both apps should be autoscaled, but with different behaviors: # app1 has an app-level policy, so it scales to 3 replicas # app2 doesn't have an app-level policy, so it uses deployment-level autoscaling (scales to 1) assert d1_app1_id in deployment_state_manager._scaling_decisions assert deployment_state_manager._scaling_decisions[d1_app1_id] == 3 assert d1_app2_id in deployment_state_manager._scaling_decisions assert deployment_state_manager._scaling_decisions[d1_app2_id] == 1 def test_autoscaling_state_manager_edge_cases( self, mocked_application_state_manager ): """Test edge cases for AutoscalingStateManager methods.""" ( app_state_manager, deployment_state_manager, _, ) = mocked_application_state_manager asm = app_state_manager._autoscaling_state_manager # Test with empty app name assert asm._application_has_policy("") is False assert asm.should_autoscale_application("") is False # Test with None app name assert asm._application_has_policy(None) is False assert asm.should_autoscale_application(None) is False # Test get_decision_num_replicas with nonexistent app with pytest.raises(KeyError): asm.get_decision_num_replicas("nonexistent_app", {}) # Test should_autoscale_deployment with nonexistent deployment nonexistent_deployment_id = DeploymentID( name="nonexistent", app_name="nonexistent_app" ) assert asm.should_autoscale_deployment(nonexistent_deployment_id) is False def test_autoscaling_with_deployment_level_configs( self, mocked_application_state_manager ): """Test that app-level autoscaling respects deployment-level autoscaling configs.""" ( app_state_manager, deployment_state_manager, _, ) = mocked_application_state_manager # Create app with deployments that have different autoscaling configs deployments = [ DeploymentSchema( name="d1", autoscaling_config={ "target_ongoing_requests": 1, "min_replicas": 1, "max_replicas": 3, # Lower max "initial_replicas": 1, }, ), DeploymentSchema( name="d2", autoscaling_config={ "target_ongoing_requests": 1, "min_replicas": 1, "max_replicas": 10, # Higher max "initial_replicas": 1, }, ), ] app_config = self._create_app_config(deployments=deployments) _ = self._deploy_app_with_mocks(app_state_manager, app_config) asm = self._register_deployments(app_state_manager, app_config) # Create replicas d1_id = DeploymentID(name="d1", app_name="test_app") d2_id = DeploymentID(name="d2", app_name="test_app") d1_replicas = [ ReplicaID(unique_id=f"d1_replica_{i}", deployment_id=d1_id) for i in [1, 2] ] d2_replicas = [ ReplicaID(unique_id=f"d2_replica_{i}", deployment_id=d2_id) for i in [1, 2] ] asm.update_running_replica_ids(d1_id, d1_replicas) asm.update_running_replica_ids(d2_id, d2_replicas) # Test autoscaling deployment_state_manager._scaling_decisions.clear() app_state_manager.update() # Verify both deployments were scaled, but d1 should be capped at max_replicas=3 assert d1_id in deployment_state_manager._scaling_decisions assert d2_id in deployment_state_manager._scaling_decisions assert ( deployment_state_manager._scaling_decisions[d1_id] == 3 ) # Capped by max_replicas assert ( deployment_state_manager._scaling_decisions[d2_id] == 3 ) # Our policy scales to 3 def test_app_level_autoscaling_policy_state_persistence_with_stateful_policy( self, mocked_application_state_manager ): """Test that app-level autoscaling policy state is maintained across multiple calls.""" ( app_state_manager, deployment_state_manager, _, ) = mocked_application_state_manager # Create app config but override to use the stateful policy. app_config = self._create_app_config() app_config.autoscaling_policy = { "policy_function": "ray.serve.tests.unit.test_application_state:stateful_app_level_policy" } # Deploy app and register deployments with autoscaling manager. _ = self._deploy_app_with_mocks(app_state_manager, app_config) asm = self._register_deployments(app_state_manager, app_config) # Create replicas so autoscaling runs. d1_id = DeploymentID(name="d1", app_name="test_app") d1_replicas = [ ReplicaID(unique_id=f"d1_replica_{i}", deployment_id=d1_id) for i in [1, 2] ] asm.update_running_replica_ids(d1_id, d1_replicas) for _ in range(3): deployment_state_manager._scaling_decisions.clear() app_state_manager.update() assert asm.should_autoscale_application("test_app") is True assert deployment_state_manager._scaling_decisions[d1_id] == 3 # The stateful policy should have incremented its counter across calls. # Since update() was called 3 times, the counter should be 3. app_autoscaling_state = asm._app_autoscaling_states["test_app"] assert app_autoscaling_state._policy_state is not None assert len(app_autoscaling_state._policy_state) != 0 for _, state in app_autoscaling_state._policy_state.items(): assert state.get("counter") == 3 def test_validate_policy_state(self, mocked_application_state_manager): """Test that _validate_policy_state correctly validates application-level policy state.""" ( app_state_manager, deployment_state_manager, _, ) = mocked_application_state_manager # Deploy app and register deployments with autoscaling manager. app_config = self._create_app_config() _ = self._deploy_app_with_mocks(app_state_manager, app_config) asm = self._register_deployments(app_state_manager, app_config) app_autoscaling_state = asm._app_autoscaling_states["test_app"] d1_id = DeploymentID(name="d1", app_name="test_app") # Valid cases # None should pass (no validation) app_autoscaling_state._validate_policy_state(None) # Valid dict with valid deployment ID and dict value should pass valid_state = {d1_id: {"key": "value"}} app_autoscaling_state._validate_policy_state(valid_state) # Invalid cases # Not a dict should fail with pytest.raises(AssertionError, match="must return policy_state as Dict"): app_autoscaling_state._validate_policy_state("deployment") with pytest.raises(AssertionError, match="must return policy_state as Dict"): app_autoscaling_state._validate_policy_state(1) # Invalid deployment ID should fail invalid_deployment_id = DeploymentID(name="invalid", app_name="test_app") invalid_state = {invalid_deployment_id: {"key": "value"}} with pytest.raises(AssertionError, match="contains invalid deployment ID"): app_autoscaling_state._validate_policy_state(invalid_state) # Non dict value should fail invalid_value_state = {d1_id: "not a dict"} with pytest.raises(AssertionError, match="must be a dictionary"): app_autoscaling_state._validate_policy_state(invalid_value_state) def test_policy_state_persitence_for_skipped_deployments( self, mocked_application_state_manager ): """ Test that when an app-level policy returns decisions for only a subset of deployments, the skipped deployment's user state is still maintained across multiple calls """ ( app_state_manager, deployment_state_manager, _, ) = mocked_application_state_manager # Create app config with two deployments and override to use the stateful policy. deployments = [ DeploymentSchema( name="d1", autoscaling_config={ "target_ongoing_requests": 1, "min_replicas": 1, "max_replicas": 5, "initial_replicas": 1, }, ), DeploymentSchema( name="d2", autoscaling_config={ "target_ongoing_requests": 1, "min_replicas": 1, "max_replicas": 5, "initial_replicas": 1, }, ), ] app_config = self._create_app_config(deployments=deployments) app_config.autoscaling_policy = { "policy_function": "ray.serve.tests.unit.test_application_state:partial_decisions_app_level_policy" } # Deploy app and register deployments with autoscaling manager. _ = self._deploy_app_with_mocks(app_state_manager, app_config) asm = self._register_deployments(app_state_manager, app_config) # Create replicas so autoscaling runs. d1_id = DeploymentID(name="d1", app_name="test_app") d2_id = DeploymentID(name="d2", app_name="test_app") d1_replicas = [ ReplicaID(unique_id=f"d1_replica_{i}", deployment_id=d1_id) for i in [1, 2] ] d2_replicas = [ ReplicaID(unique_id=f"d2_replica_{i}", deployment_id=d2_id) for i in [1, 2] ] asm.update_running_replica_ids(d1_id, d1_replicas) asm.update_running_replica_ids(d2_id, d2_replicas) for i in range(3): deployment_state_manager._scaling_decisions.clear() app_state_manager.update() # The scaling decisions will not contain d1 assert d1_id not in deployment_state_manager._scaling_decisions assert deployment_state_manager._scaling_decisions[d2_id] == 3 # State still exists and increments correctly according to the policy for each deployment app_autoscaling_state = asm._app_autoscaling_states["test_app"] for _, state in app_autoscaling_state._policy_state.items(): assert state.get("counter") == i + 1 def test_app_level_autoscaling_with_decorator_applies_delays( self, mocked_application_state_manager ): """Delay logic uses wall-clock time in _apply_delay_logic, not iteration count. Autoscale() runs in a tight test loop, so real time between calls is ~0. We patch ``ray.serve.autoscaling_policy.time`` so each policy evaluation advances a fake clock by CONTROL_LOOP_INTERVAL_S (matching controller cadence). Wait counts use math.ceil(delay / interval): int() undercounts when float division yields 5.999... for 0.6/0.1, and wall-clock needs enough ticks that elapsed >= delay_s. Fake times use tick/ticks_per_second (not tick * interval) so timestamp subtraction stays exact in IEEE 754. """ ( app_state_manager, deployment_state_manager, _, ) = mocked_application_state_manager deployments = [ DeploymentSchema( name="d1", autoscaling_config={ "target_ongoing_requests": 1, "min_replicas": 1, "max_replicas": 5, "initial_replicas": 1, "upscale_delay_s": 0.4, "downscale_delay_s": 0.6, "metrics_interval_s": 0.1, }, ) ] app_config = self._create_app_config(deployments=deployments) app_config.autoscaling_policy = { "policy_function": "ray.serve.tests.unit.test_application_state:app_level_policy_with_decorator" } _ = self._deploy_app_with_mocks(app_state_manager, app_config) asm = self._register_deployments(app_state_manager, app_config) d1_id = DeploymentID(name="d1", app_name="test_app") upscale_delay_s = deployments[0].autoscaling_config["upscale_delay_s"] downscale_delay_s = deployments[0].autoscaling_config["downscale_delay_s"] ticks_per_second = round(1.0 / CONTROL_LOOP_INTERVAL_S) wait_ticks_before_upscale = math.ceil(upscale_delay_s * ticks_per_second) wait_ticks_before_downscale = math.ceil(downscale_delay_s * ticks_per_second) d1_replicas = [ ReplicaID(unique_id=f"d1_replica_{i}", deployment_id=d1_id) for i in [1, 2] ] asm.update_running_replica_ids(d1_id, d1_replicas) app_state = app_state_manager._application_states["test_app"] fake_tick = [0] def _advance_time(): t = fake_tick[0] / ticks_per_second fake_tick[0] += 1 return t with patch("ray.serve.autoscaling_policy.time") as mock_time: mock_time.time = _advance_time for _ in range(wait_ticks_before_upscale): app_state.autoscale() assert deployment_state_manager._scaling_decisions[d1_id] == 1 app_state.autoscale() assert deployment_state_manager._scaling_decisions[d1_id] == 5 deployment_state_manager.deployment_infos[ d1_id ].deployment_config.num_replicas = 5 for _ in range(wait_ticks_before_downscale): app_state.autoscale() assert deployment_state_manager._scaling_decisions[d1_id] == 5 app_state.autoscale() assert deployment_state_manager._scaling_decisions[d1_id] == 1 def test_get_external_scaler_enabled(mocked_application_state_manager): """Test get_external_scaler_enabled returns correct value based on app config. Test that get_external_scaler_enabled returns True when an app is deployed with external_scaler_enabled=True, False when deployed with external_scaler_enabled=False, and False for non-existent apps. """ app_state_manager, _, _ = mocked_application_state_manager # Deploy app with external_scaler_enabled=True app_state_manager.deploy_app( "app_with_external_scaler", [deployment_params("deployment1", "/route1")], ApplicationArgsProto(external_scaler_enabled=True), ) # Deploy app with external_scaler_enabled=False app_state_manager.deploy_app( "app_without_external_scaler", [deployment_params("deployment2", "/route2")], ApplicationArgsProto(external_scaler_enabled=False), ) # Test that get_external_scaler_enabled returns True for app with external scaler enabled assert ( app_state_manager.get_external_scaler_enabled("app_with_external_scaler") is True ) # Test that get_external_scaler_enabled returns False for app without external scaler assert ( app_state_manager.get_external_scaler_enabled("app_without_external_scaler") is False ) # Test that get_external_scaler_enabled returns False for non-existent app assert app_state_manager.get_external_scaler_enabled("non_existent_app") is False def test_deploy_apps_with_external_scaler_enabled(mocked_application_state_manager): """Test that deploy_apps correctly uses external_scaler_enabled from name_to_application_args. This test verifies that when deploy_apps is called with name_to_application_args containing external_scaler_enabled values, the ApplicationState is correctly initialized with the appropriate external_scaler_enabled value for each app. """ ( app_state_manager, deployment_state_manager, kv_store, ) = mocked_application_state_manager # Deploy multiple apps with different external_scaler_enabled settings name_to_deployment_args = { "app_with_scaler": [deployment_params("d1", "/with_scaler")], "app_without_scaler": [deployment_params("d2", "/without_scaler")], "app_default": [deployment_params("d3", "/default")], } name_to_application_args = { "app_with_scaler": ApplicationArgsProto(external_scaler_enabled=True), "app_without_scaler": ApplicationArgsProto(external_scaler_enabled=False), "app_default": ApplicationArgsProto(external_scaler_enabled=False), } # Call deploy_apps app_state_manager.deploy_apps(name_to_deployment_args, name_to_application_args) # Verify that external_scaler_enabled is correctly set for each app assert app_state_manager.get_external_scaler_enabled("app_with_scaler") is True assert app_state_manager.get_external_scaler_enabled("app_without_scaler") is False assert app_state_manager.get_external_scaler_enabled("app_default") is False # Verify the internal state is also correct app_state_with_scaler = app_state_manager._application_states["app_with_scaler"] app_state_without_scaler = app_state_manager._application_states[ "app_without_scaler" ] app_state_default = app_state_manager._application_states["app_default"] assert app_state_with_scaler.external_scaler_enabled is True assert app_state_without_scaler.external_scaler_enabled is False assert app_state_default.external_scaler_enabled is False # Verify that all apps are in the correct state assert app_state_with_scaler.status == ApplicationStatus.DEPLOYING assert app_state_without_scaler.status == ApplicationStatus.DEPLOYING assert app_state_default.status == ApplicationStatus.DEPLOYING def test_external_scaler_enabled_recovery_from_checkpoint( mocked_application_state_manager, ): """Test that external_scaler_enabled is correctly recovered from checkpoint. This test verifies that after a controller crash and recovery, the external_scaler_enabled flag is correctly restored from the checkpoint for both apps with external_scaler_enabled=True and external_scaler_enabled=False. """ ( app_state_manager, deployment_state_manager, kv_store, ) = mocked_application_state_manager app_name_with_scaler = "app_with_external_scaler" app_name_without_scaler = "app_without_external_scaler" deployment_id_with_scaler = DeploymentID(name="d1", app_name=app_name_with_scaler) deployment_id_without_scaler = DeploymentID( name="d2", app_name=app_name_without_scaler ) # Deploy app with external_scaler_enabled=True app_state_manager.deploy_app( app_name_with_scaler, [deployment_params("d1")], ApplicationArgsProto(external_scaler_enabled=True), ) # Deploy app with external_scaler_enabled=False app_state_manager.deploy_app( app_name_without_scaler, [deployment_params("d2")], ApplicationArgsProto(external_scaler_enabled=False), ) # Verify initial state assert app_state_manager.get_external_scaler_enabled(app_name_with_scaler) is True assert ( app_state_manager.get_external_scaler_enabled(app_name_without_scaler) is False ) # Make deployments healthy and update app_state_manager.update() deployment_state_manager.set_deployment_healthy(deployment_id_with_scaler) deployment_state_manager.set_deployment_healthy(deployment_id_without_scaler) app_state_manager.update() # Save checkpoint app_state_manager.save_checkpoint() # Simulate controller crash - create new managers with the same kv_store new_deployment_state_manager = MockDeploymentStateManager(kv_store) new_app_state_manager = ApplicationStateManager( new_deployment_state_manager, AutoscalingStateManager(), MockEndpointState(), kv_store, LoggingConfig(), ) # Verify that external_scaler_enabled is correctly recovered from checkpoint assert ( new_app_state_manager.get_external_scaler_enabled(app_name_with_scaler) is True ) assert ( new_app_state_manager.get_external_scaler_enabled(app_name_without_scaler) is False ) # Verify the internal state is also correct app_state_with_scaler = new_app_state_manager._application_states[ app_name_with_scaler ] app_state_without_scaler = new_app_state_manager._application_states[ app_name_without_scaler ] assert app_state_with_scaler.external_scaler_enabled is True assert app_state_without_scaler.external_scaler_enabled is False class TestDeploymentDAG: """Test deployment DAG building and retrieval functionality.""" def test_build_dag_single_deployment(self, mocked_application_state): """Test building DAG with a single deployment.""" app_state, deployment_state_manager = mocked_application_state d1_id = DeploymentID(name="d1", app_name="test_app") # Deploy single deployment app_state.deploy_app( {"d1": deployment_info("d1", "/hi")}, ApplicationArgsProto(external_scaler_enabled=False), ) app_state.update() deployment_state_manager.set_deployment_healthy(d1_id) app_state.update() # Get topology topology = app_state.get_deployment_topology() assert topology is not None assert topology.app_name == "test_app" assert topology.ingress_deployment == "d1" assert "d1" in topology.nodes assert topology.nodes["d1"].name == "d1" assert topology.nodes["d1"].is_ingress is True assert topology.nodes["d1"].outbound_deployments == [] def test_build_dag_multiple_deployments_no_deps(self, mocked_application_state): """Test building DAG with multiple deployments without dependencies.""" app_state, deployment_state_manager = mocked_application_state d1_id = DeploymentID(name="d1", app_name="test_app") d2_id = DeploymentID(name="d2", app_name="test_app") d3_id = DeploymentID(name="d3", app_name="test_app") # Deploy multiple deployments app_state.deploy_app( { "d1": deployment_info("d1", "/hi"), "d2": deployment_info("d2"), "d3": deployment_info("d3"), }, ApplicationArgsProto(external_scaler_enabled=False), ) app_state.update() deployment_state_manager.set_deployment_healthy(d1_id) deployment_state_manager.set_deployment_healthy(d2_id) deployment_state_manager.set_deployment_healthy(d3_id) app_state.update() # Get topology topology = app_state.get_deployment_topology() assert topology is not None assert topology.app_name == "test_app" assert topology.ingress_deployment == "d1" assert len(topology.nodes) == 3 assert "d1" in topology.nodes assert "d2" in topology.nodes assert "d3" in topology.nodes assert topology.nodes["d1"].is_ingress is True assert topology.nodes["d2"].is_ingress is False assert topology.nodes["d3"].is_ingress is False def test_build_dag_with_outbound_dependencies(self, mocked_application_state): """Test building DAG with outbound dependencies.""" app_state, deployment_state_manager = mocked_application_state d1_id = DeploymentID(name="d1", app_name="test_app") d2_id = DeploymentID(name="d2", app_name="test_app") d3_id = DeploymentID(name="d3", app_name="test_app") # Set up outbound dependencies: d1 -> d2, d3; d2 -> d3 deployment_state_manager._outbound_deps_d1_test_app = [d2_id, d3_id] deployment_state_manager._outbound_deps_d2_test_app = [d3_id] deployment_state_manager._outbound_deps_d3_test_app = [] # Deploy deployments app_state.deploy_app( { "d1": deployment_info("d1", "/hi"), "d2": deployment_info("d2"), "d3": deployment_info("d3"), }, ApplicationArgsProto(external_scaler_enabled=False), ) app_state.update() deployment_state_manager.set_deployment_healthy(d1_id) deployment_state_manager.set_deployment_healthy(d2_id) deployment_state_manager.set_deployment_healthy(d3_id) app_state.update() # Get topology topology = app_state.get_deployment_topology() assert topology is not None assert topology.app_name == "test_app" assert len(topology.nodes) == 3 # Verify d1 has outbound to d2 and d3 assert len(topology.nodes["d1"].outbound_deployments) == 2 d1_outbound = topology.nodes["d1"].outbound_deployments assert {"name": "d2", "app_name": "test_app"} in d1_outbound assert {"name": "d3", "app_name": "test_app"} in d1_outbound # Verify d2 has outbound to d3 assert len(topology.nodes["d2"].outbound_deployments) == 1 d2_outbound = topology.nodes["d2"].outbound_deployments assert ( d3_id in d2_outbound or {"name": "d3", "app_name": "test_app"} in d2_outbound ) # Verify d3 has no outbound dependencies assert len(topology.nodes["d3"].outbound_deployments) == 0 def test_build_dag_with_cross_app_dependencies(self, mocked_application_state): """Test building DAG with dependencies to deployments in other apps.""" app_state, deployment_state_manager = mocked_application_state d1_id = DeploymentID(name="d1", app_name="test_app") d2_id = DeploymentID(name="d2", app_name="test_app") # Create dependencies to deployments in another app external_d1_id = DeploymentID(name="ext_d1", app_name="other_app") external_d2_id = DeploymentID(name="ext_d2", app_name="other_app") # Set up outbound dependencies: d1 -> d2, ext_d1; d2 -> ext_d2 deployment_state_manager._outbound_deps_d1_test_app = [d2_id, external_d1_id] deployment_state_manager._outbound_deps_d2_test_app = [external_d2_id] # Deploy deployments app_state.deploy_app( { "d1": deployment_info("d1", "/hi"), "d2": deployment_info("d2"), }, ApplicationArgsProto(external_scaler_enabled=False), ) app_state.update() deployment_state_manager.set_deployment_healthy(d1_id) deployment_state_manager.set_deployment_healthy(d2_id) app_state.update() # Get topology topology = app_state.get_deployment_topology() assert topology is not None assert topology.app_name == "test_app" assert len(topology.nodes) == 2 # Verify d1 has outbound to both internal d2 and external ext_d1 assert len(topology.nodes["d1"].outbound_deployments) == 2 d1_outbound = topology.nodes["d1"].outbound_deployments assert {"name": "d2", "app_name": "test_app"} in d1_outbound assert {"name": "ext_d1", "app_name": "other_app"} in d1_outbound # Verify d2 has outbound to external ext_d2 assert len(topology.nodes["d2"].outbound_deployments) == 1 d2_outbound = topology.nodes["d2"].outbound_deployments assert {"name": "ext_d2", "app_name": "other_app"} in d2_outbound def test_get_dag_before_deployment(self, mocked_application_state): """Test getting topology before any deployments are created.""" app_state, _ = mocked_application_state # Topology should be None before any deployments topology = app_state.get_deployment_topology() assert topology is None def test_dag_updates_on_redeploy(self, mocked_application_state): """Test that topology updates when deployments are redeployed.""" app_state, deployment_state_manager = mocked_application_state d1_id = DeploymentID(name="d1", app_name="test_app") d2_id = DeploymentID(name="d2", app_name="test_app") d3_id = DeploymentID(name="d3", app_name="test_app") # Initial deployment: d1, d2 app_state.deploy_app( { "d1": deployment_info("d1", "/hi"), "d2": deployment_info("d2"), }, ApplicationArgsProto(external_scaler_enabled=False), ) app_state.update() deployment_state_manager.set_deployment_healthy(d1_id) deployment_state_manager.set_deployment_healthy(d2_id) app_state.update() # Verify initial topology topology = app_state.get_deployment_topology() assert topology is not None assert len(topology.nodes) == 2 assert "d1" in topology.nodes assert "d2" in topology.nodes # Redeploy with d2, d3 (removing d1, adding d3) app_state.deploy_app( { "d2": deployment_info("d2", "/hi"), "d3": deployment_info("d3"), }, ApplicationArgsProto(external_scaler_enabled=False), ) app_state.update() deployment_state_manager.set_deployment_deleted(d1_id) deployment_state_manager.set_deployment_healthy(d3_id) app_state.update() # Verify updated topology topology = app_state.get_deployment_topology() assert topology is not None assert len(topology.nodes) == 2 assert "d2" in topology.nodes assert "d3" in topology.nodes assert "d1" not in topology.nodes assert topology.ingress_deployment == "d2" def test_dag_with_changing_dependencies(self, mocked_application_state): """Test that topology updates when outbound dependencies change.""" app_state, deployment_state_manager = mocked_application_state d1_id = DeploymentID(name="d1", app_name="test_app") d2_id = DeploymentID(name="d2", app_name="test_app") # Initial: d1 -> d2 deployment_state_manager._outbound_deps_d1_test_app = [d2_id] app_state.deploy_app( { "d1": deployment_info("d1", "/hi"), "d2": deployment_info("d2"), }, ApplicationArgsProto(external_scaler_enabled=False), ) app_state.update() deployment_state_manager.set_deployment_healthy(d1_id) deployment_state_manager.set_deployment_healthy(d2_id) app_state.update() # Verify initial dependencies topology = app_state.get_deployment_topology() assert len(topology.nodes["d1"].outbound_deployments) == 1 d1_outbound = topology.nodes["d1"].outbound_deployments assert {"name": "d2", "app_name": "test_app"} in d1_outbound # Change dependencies: d1 -> [] (no dependencies) deployment_state_manager._outbound_deps_d1_test_app = [] # Trigger update to rebuild topology app_state.update() # Verify updated dependencies topology = app_state.get_deployment_topology() assert len(topology.nodes["d1"].outbound_deployments) == 0 def test_application_state_manager_get_deployment_topology( self, mocked_application_state_manager ): """Test getting topology via ApplicationStateManager.""" ( app_state_manager, deployment_state_manager, _, ) = mocked_application_state_manager d1_id = DeploymentID(name="d1", app_name="test_app") # Deploy app app_state_manager.deploy_app( "test_app", [deployment_params("d1", "/hi")], ApplicationArgsProto(external_scaler_enabled=False), ) app_state_manager.update() deployment_state_manager.set_deployment_healthy(d1_id) app_state_manager.update() # Get topology via ApplicationStateManager topology = app_state_manager.get_deployment_topology("test_app") assert topology is not None assert topology.app_name == "test_app" assert "d1" in topology.nodes def test_application_state_manager_get_topology_nonexistent_app( self, mocked_application_state_manager ): """Test getting topology for non-existent app returns None.""" app_state_manager, _, _ = mocked_application_state_manager # Get topology for non-existent app topology = app_state_manager.get_deployment_topology("nonexistent_app") assert topology is None def test_dag_with_multiple_apps(self, mocked_application_state_manager): """Test that each app has its own independent topology.""" ( app_state_manager, deployment_state_manager, _, ) = mocked_application_state_manager # Deploy app1 app1_d1_id = DeploymentID(name="d1", app_name="app1") app1_d2_id = DeploymentID(name="d2", app_name="app1") app_state_manager.deploy_app( "app1", [ deployment_params("d1", "/app1"), deployment_params("d2"), ], ApplicationArgsProto(external_scaler_enabled=False), ) app_state_manager.update() deployment_state_manager.set_deployment_healthy(app1_d1_id) deployment_state_manager.set_deployment_healthy(app1_d2_id) app_state_manager.update() # Deploy app2 app2_d1_id = DeploymentID(name="d1", app_name="app2") app2_d2_id = DeploymentID(name="d2", app_name="app2") app_state_manager.deploy_app( "app2", [ deployment_params("d1", "/app2"), deployment_params("d2"), ], ApplicationArgsProto(external_scaler_enabled=False), ) app_state_manager.update() deployment_state_manager.set_deployment_healthy(app2_d1_id) deployment_state_manager.set_deployment_healthy(app2_d2_id) app_state_manager.update() # Get topologies for both apps topology1 = app_state_manager.get_deployment_topology("app1") topology2 = app_state_manager.get_deployment_topology("app2") assert topology1 is not None assert topology2 is not None assert topology1.app_name == "app1" assert topology2.app_name == "app2" assert len(topology1.nodes) == 2 assert len(topology2.nodes) == 2 assert topology1.ingress_deployment == "d1" assert topology2.ingress_deployment == "d1" def test_dag_with_complex_dependency_graph(self, mocked_application_state): """Test building topology with a complex dependency graph.""" app_state, deployment_state_manager = mocked_application_state # Create a complex topology: # ingress -> orchestrator -> [worker1, worker2] # worker1 -> database # worker2 -> [database, cache] ingress_id = DeploymentID(name="ingress", app_name="test_app") orchestrator_id = DeploymentID(name="orchestrator", app_name="test_app") worker1_id = DeploymentID(name="worker1", app_name="test_app") worker2_id = DeploymentID(name="worker2", app_name="test_app") database_id = DeploymentID(name="database", app_name="test_app") cache_id = DeploymentID(name="cache", app_name="test_app") # Set up dependencies deployment_state_manager._outbound_deps_ingress_test_app = [orchestrator_id] deployment_state_manager._outbound_deps_orchestrator_test_app = [ worker1_id, worker2_id, ] deployment_state_manager._outbound_deps_worker1_test_app = [database_id] deployment_state_manager._outbound_deps_worker2_test_app = [ database_id, cache_id, ] deployment_state_manager._outbound_deps_database_test_app = [] deployment_state_manager._outbound_deps_cache_test_app = [] # Deploy all deployments app_state.deploy_app( { "ingress": deployment_info("ingress", "/api"), "orchestrator": deployment_info("orchestrator"), "worker1": deployment_info("worker1"), "worker2": deployment_info("worker2"), "database": deployment_info("database"), "cache": deployment_info("cache"), }, ApplicationArgsProto(external_scaler_enabled=False), ) app_state.update() # Mark all as healthy for dep_id in [ ingress_id, orchestrator_id, worker1_id, worker2_id, database_id, cache_id, ]: deployment_state_manager.set_deployment_healthy(dep_id) app_state.update() # Get and verify topology topology = app_state.get_deployment_topology() assert topology is not None assert len(topology.nodes) == 6 assert topology.ingress_deployment == "ingress" # Verify ingress node assert topology.nodes["ingress"].is_ingress is True assert len(topology.nodes["ingress"].outbound_deployments) == 1 ingress_outbound = topology.nodes["ingress"].outbound_deployments assert {"name": "orchestrator", "app_name": "test_app"} in ingress_outbound # Verify orchestrator node assert len(topology.nodes["orchestrator"].outbound_deployments) == 2 orchestrator_outbound = topology.nodes["orchestrator"].outbound_deployments assert {"name": "worker1", "app_name": "test_app"} in orchestrator_outbound assert {"name": "worker2", "app_name": "test_app"} in orchestrator_outbound # Verify worker nodes assert len(topology.nodes["worker1"].outbound_deployments) == 1 worker1_outbound = topology.nodes["worker1"].outbound_deployments assert {"name": "database", "app_name": "test_app"} in worker1_outbound assert len(topology.nodes["worker2"].outbound_deployments) == 2 worker2_outbound = topology.nodes["worker2"].outbound_deployments assert {"name": "database", "app_name": "test_app"} in worker2_outbound assert {"name": "cache", "app_name": "test_app"} in worker2_outbound # Verify leaf nodes have no dependencies assert len(topology.nodes["database"].outbound_deployments) == 0 assert len(topology.nodes["cache"].outbound_deployments) == 0 if __name__ == "__main__": sys.exit(pytest.main(["-v", "-s", __file__]))