import itertools import json import logging import math import os import random import time import traceback from collections import defaultdict, deque from copy import copy from dataclasses import dataclass from enum import Enum from typing import Any, Callable, Deque, Dict, List, Optional, Set, Tuple import ray from ray import ObjectRef, cloudpickle from ray._common import ray_constants from ray.actor import ActorHandle from ray.exceptions import ( RayActorError, RayError, RayTaskError, RuntimeEnvSetupError, ) from ray.serve import metrics from ray.serve._private import default_impl from ray.serve._private.autoscaling_state import AutoscalingStateManager from ray.serve._private.cluster_node_info_cache import ClusterNodeInfoCache from ray.serve._private.common import ( GANG_PG_NAME_PREFIX, DeploymentID, DeploymentStatus, DeploymentStatusInfo, DeploymentStatusInternalTrigger, DeploymentStatusTrigger, DeploymentTargetInfo, Duration, GangPlacementGroupRequest, GangReservationResult, ReplicaID, ReplicaState, RequestRoutingInfo, RunningReplicaInfo, ) from ray.serve._private.config import DeploymentConfig, GangSchedulingConfig from ray.serve._private.constants import ( DEFAULT_LATENCY_BUCKET_MS, DEPLOYMENT_ACTOR_HEALTH_CHECK_PERIOD_S, DEPLOYMENT_ACTOR_HEALTH_CHECK_TIMEOUT_S, DEPLOYMENT_ACTOR_HEALTH_CHECK_UNHEALTHY_THRESHOLD, MAX_PER_REPLICA_RETRY_COUNT, RAY_SERVE_CONTROLLER_METRICS_INCLUDE_HIGH_CARDINALITY_TAGS, RAY_SERVE_DIRECT_INGRESS_MIN_DRAINING_PERIOD_S, RAY_SERVE_ENABLE_DIRECT_INGRESS, RAY_SERVE_ENABLE_TASK_EVENTS, RAY_SERVE_FAIL_ON_RANK_ERROR, RAY_SERVE_FORCE_STOP_UNHEALTHY_REPLICAS, RAY_SERVE_INTERNAL_DEPLOYMENT_ACTOR_NAME_ENV_VAR, RAY_SERVE_INTERNAL_DEPLOYMENT_APP_NAME_ENV_VAR, RAY_SERVE_INTERNAL_DEPLOYMENT_CODE_VERSION_ENV_VAR, RAY_SERVE_INTERNAL_DEPLOYMENT_NAME_ENV_VAR, RAY_SERVE_RETAINED_DEAD_REPLICAS, RAY_SERVE_STATUS_GAUGE_REPORT_INTERVAL_S, RAY_SERVE_USE_PACK_SCHEDULING_STRATEGY, REPLICA_HEALTH_CHECK_UNHEALTHY_THRESHOLD, REPLICA_STARTUP_SHUTDOWN_LATENCY_BUCKETS_MS, REQUEST_LATENCY_BUCKETS_MS, SERVE_LOGGER_NAME, SERVE_NAMESPACE, ) from ray.serve._private.deployment_info import DeploymentInfo from ray.serve._private.deployment_scheduler import ( DeploymentDownscaleRequest, DeploymentScheduler, ReplicaSchedulingRequest, ReplicaSchedulingRequestStatus, SpreadDeploymentSchedulingPolicy, ) from ray.serve._private.exceptions import DeploymentIsBeingDeletedError from ray.serve._private.long_poll import LongPollHost, LongPollNamespace from ray.serve._private.storage.kv_store import KVStoreBase from ray.serve._private.usage import ServeUsageTag from ray.serve._private.utils import ( JavaActorHandleProxy, check_obj_ref_ready_nowait, get_active_placement_group_ids, get_capacity_adjusted_num_replicas, get_deployment_actor_name, get_random_string, msgpack_deserialize, msgpack_serialize, override_runtime_envs_except_env_vars, ) from ray.serve._private.version import DeploymentVersion from ray.serve.config import DeploymentActorConfig, GangRuntimeFailurePolicy from ray.serve.gang import GangContext from ray.serve.generated.serve_pb2 import DeploymentLanguage from ray.serve.schema import ( DeploymentDetails, ReplicaDetails, ReplicaRank, _deployment_info_to_schema, ) from ray.util import metrics as ray_metrics from ray.util.placement_group import PlacementGroup logger = logging.getLogger(SERVE_LOGGER_NAME) _RESERVED_INTERNAL_DEPLOYMENT_CONTEXT_ENV_VARS = { RAY_SERVE_INTERNAL_DEPLOYMENT_APP_NAME_ENV_VAR, RAY_SERVE_INTERNAL_DEPLOYMENT_NAME_ENV_VAR, RAY_SERVE_INTERNAL_DEPLOYMENT_ACTOR_NAME_ENV_VAR, RAY_SERVE_INTERNAL_DEPLOYMENT_CODE_VERSION_ENV_VAR, } def _validate_no_reserved_internal_deployment_context_env_vars( env_vars: Dict[str, Any], ) -> None: conflicting_keys = sorted( _RESERVED_INTERNAL_DEPLOYMENT_CONTEXT_ENV_VARS.intersection(env_vars) ) if conflicting_keys: raise ValueError( "Users may not set reserved Ray Serve deployment actor context env vars: " + ", ".join(conflicting_keys) ) def _inject_internal_deployment_context_env_vars( runtime_env: Optional[Dict[str, Any]], deployment_id: DeploymentID, actor_name: str, code_version: str, ) -> Dict[str, Any]: runtime_env = copy(runtime_env) if runtime_env else {} env_vars = dict(runtime_env.get("env_vars", {})) _validate_no_reserved_internal_deployment_context_env_vars(env_vars) env_vars.update( { RAY_SERVE_INTERNAL_DEPLOYMENT_APP_NAME_ENV_VAR: deployment_id.app_name, RAY_SERVE_INTERNAL_DEPLOYMENT_NAME_ENV_VAR: deployment_id.name, RAY_SERVE_INTERNAL_DEPLOYMENT_ACTOR_NAME_ENV_VAR: actor_name, RAY_SERVE_INTERNAL_DEPLOYMENT_CODE_VERSION_ENV_VAR: code_version, } ) runtime_env["env_vars"] = env_vars return runtime_env class DeploymentActorState(Enum): STARTING = 1 RECOVERING = 2 RUNNING = 3 ALL_DEPLOYMENT_ACTOR_STATES = list(DeploymentActorState) class DeploymentActorWrapper: """Lifecycle wrapper for a single deployment-scoped actor. The controller reconciles failed actors (no Ray auto-restart): it polls ``__ray_ready__`` on a schedule and recreates the actor after repeated failures, same pattern as replica health checks. """ def __init__( self, deployment_id: DeploymentID, config: DeploymentActorConfig, code_version: str, recovered_handle: Optional[ActorHandle] = None, ): self._deployment_id = deployment_id self._config = config self._code_version = code_version self._actor_name = get_deployment_actor_name( self._deployment_id, self._config.name, code_version=self._code_version ) self._handle: Optional[ActorHandle] = recovered_handle self._ready_ref: Optional[ObjectRef] = None if recovered_handle is not None and hasattr(recovered_handle, "__ray_ready__"): self._ready_ref = recovered_handle.__ray_ready__.remote() self._health_check_ref: Optional[ObjectRef] = None self._last_health_check_time: float = 0.0 self._consecutive_health_check_failures: int = 0 self._healthy: bool = True @property def actor_logical_name(self) -> str: return self._config.name @property def code_version(self) -> str: return self._code_version def start( self, deployment_runtime_env: Optional[Dict[str, Any]] = None ) -> Tuple[bool, Optional[str]]: """Start this deployment actor. Args: deployment_runtime_env: Runtime env inherited from the deployment. Returns: (started_successfully, error_msg). `started_successfully` indicates whether actor start/setup succeeded. Callers should check wrapper state to decide if readiness polling is needed. """ deployment_runtime_env = deployment_runtime_env or {} try: actor_cls = self._config.get_actor_class() logger.info( f"Creating deployment actor '{self._config.name}' with " f"name={self._actor_name}" ) actor_options = { k: v for k, v in (self._config.actor_options or {}).items() if k not in ("name", "max_restarts") } # Inherit runtime_env from deployment; actor_options override. actor_runtime_env = actor_options.pop("runtime_env", {}) merged_runtime_env = override_runtime_envs_except_env_vars( deployment_runtime_env, actor_runtime_env ) merged_runtime_env = _inject_internal_deployment_context_env_vars( merged_runtime_env, deployment_id=self._deployment_id, actor_name=self._config.name, code_version=self._code_version, ) actor_options["runtime_env"] = merged_runtime_env # Serve recreates deployment actors after failed health checks instead # of relying on Ray actor restarts. actor_options["max_restarts"] = 0 self._handle = actor_cls.options( name=self._actor_name, namespace=SERVE_NAMESPACE, lifetime="detached", get_if_exists=True, **actor_options, ).remote( *(self._config.init_args or ()), **(self._config.init_kwargs or {}), ) # Keep both handle and __ray_ready__ ref so pending creation can be # cancelled on delete while still waiting for resources. self._ready_ref = self._handle.__ray_ready__.remote() return True, None except Exception as e: logger.exception( f"Failed to create deployment actor '{self._config.name}' " f"for {self._deployment_id}: {e}" ) return False, str(e) def check_ready(self) -> Tuple[bool, Optional[str]]: """Check readiness for this actor without blocking.""" # Already ready: _ready_ref cleared after successful wait if self._ready_ref is None and self._handle is not None: return True, None if self._ready_ref is None: return False, None if not check_obj_ref_ready_nowait(self._ready_ref): return False, None try: ray.get(self._ready_ref) self._ready_ref = None return True, None except Exception as e: return False, f"Deployment actor '{self._config.name}' failed: {e}" def reset_health_state_after_running(self) -> None: """Reset health-check bookkeeping when the actor reaches RUNNING.""" self._health_check_ref = None self._last_health_check_time = 0.0 self._consecutive_health_check_failures = 0 self._healthy = True def _check_active_deployment_actor_health_check( self, ) -> "ReplicaHealthCheckResponse": """Resolve the outstanding ``__ray_ready__`` health check ref, if any.""" if self._health_check_ref is None: return ReplicaHealthCheckResponse.NONE if check_obj_ref_ready_nowait(self._health_check_ref): try: ray.get(self._health_check_ref) return ReplicaHealthCheckResponse.SUCCEEDED except RayActorError: return ReplicaHealthCheckResponse.ACTOR_CRASHED except RayError as e: logger.warning( f"Health check for deployment actor " f"'{self._config.name}' ({self._deployment_id}) failed: {e}" ) return ReplicaHealthCheckResponse.APP_FAILURE elif ( time.time() - self._last_health_check_time > DEPLOYMENT_ACTOR_HEALTH_CHECK_TIMEOUT_S ): logger.warning( "Didn't receive health check response for deployment actor " f"'{self._config.name}' ({self._deployment_id}) after " f"{DEPLOYMENT_ACTOR_HEALTH_CHECK_TIMEOUT_S}s, treating as failed." ) return ReplicaHealthCheckResponse.APP_FAILURE return ReplicaHealthCheckResponse.NONE def _should_start_new_deployment_actor_health_check(self) -> bool: # Do not poll a handle we already marked unhealthy: ``check_health`` clears # the ref before returning False, and the controller will ``kill`` this # wrapper—starting another ``__ray_ready__`` would target a dead actor and # orphan the new ObjectRef when ``kill`` clears ``_health_check_ref``. if not self._healthy: return False if self._health_check_ref is not None: return False if self._handle is None: return False time_since_last = time.time() - self._last_health_check_time randomized_period = DEPLOYMENT_ACTOR_HEALTH_CHECK_PERIOD_S * random.uniform( 0.9, 1.1 ) return time_since_last > randomized_period def check_health(self) -> bool: """Poll ``__ray_ready__`` like replica health checks; update ``_healthy``.""" response = self._check_active_deployment_actor_health_check() if response is ReplicaHealthCheckResponse.NONE: pass elif response is ReplicaHealthCheckResponse.SUCCEEDED: if self._consecutive_health_check_failures > 0: logger.info( f"Deployment actor '{self._config.name}' ({self._deployment_id}) " "passed the health check after " f"{self._consecutive_health_check_failures} consecutive failures." ) self._consecutive_health_check_failures = 0 self._healthy = True elif response is ReplicaHealthCheckResponse.APP_FAILURE: self._consecutive_health_check_failures += 1 if ( self._consecutive_health_check_failures >= DEPLOYMENT_ACTOR_HEALTH_CHECK_UNHEALTHY_THRESHOLD ): logger.warning( f"Deployment actor '{self._config.name}' ({self._deployment_id}) " "failed the health check " f"{self._consecutive_health_check_failures} times in a row, " "marking unhealthy." ) self._healthy = False elif response is ReplicaHealthCheckResponse.ACTOR_CRASHED: logger.warning( f"Deployment actor '{self._config.name}' ({self._deployment_id}) " "actor crashed during health check, marking unhealthy immediately." ) self._healthy = False else: assert False, f"Unknown response type: {response}." if response is not ReplicaHealthCheckResponse.NONE: self._health_check_ref = None if self._should_start_new_deployment_actor_health_check(): self._last_health_check_time = time.time() self._health_check_ref = self._handle.__ray_ready__.remote() return self._healthy def kill(self) -> None: """Kill this deployment actor by deterministic actor name.""" self._health_check_ref = None try: if not self._handle: self._handle = ray.get_actor( self._actor_name, namespace=SERVE_NAMESPACE ) ray.kill(self._handle, no_restart=True) except (ValueError, RayActorError): logger.warning( f"Deployment actor '{self._config.name}' for {self._deployment_id} " f"was already stopped" ) @dataclass class DeploymentActorEntry: """Encapsulates a deployment actor with its version, name, and wrapper. Similar to how DeploymentReplica is stored in ReplicaStateContainer. State is tracked by the container via which bucket the entry lives in. """ code_version: str name: str wrapper: DeploymentActorWrapper class DeploymentActorContainer: """Manages deployment-scoped actor wrappers for a single deployment. Entries are keyed by state (like ReplicaStateContainer). Each entry encapsulates version, name, wrapper, and state. """ def __init__(self, deployment_id: DeploymentID): self._deployment_id = deployment_id self._actors_by_state: Dict[ DeploymentActorState, List[DeploymentActorEntry] ] = defaultdict(list) self._actors_index: Dict[ Tuple[str, str], Tuple[DeploymentActorState, DeploymentActorEntry] ] = {} # (code_version, name) -> (state, entry) def add( self, state: DeploymentActorState, wrapper: DeploymentActorWrapper, ) -> None: """Add or move a wrapper under the given state.""" code_version = wrapper.code_version name = wrapper.actor_logical_name key = (code_version, name) # Remove from old state bucket if present (e.g. STARTING -> READY) existing = self._actors_index.get(key) if existing is not None: old_state, old_entry = existing bucket = self._actors_by_state[old_state] bucket.remove(old_entry) if not bucket: del self._actors_by_state[old_state] entry = DeploymentActorEntry( code_version=code_version, name=name, wrapper=wrapper, ) self._actors_by_state[state].append(entry) self._actors_index[key] = (state, entry) def get( self, code_version: Optional[str] = None, states: Optional[List[DeploymentActorState]] = None, ) -> List[DeploymentActorWrapper]: if states is None: states = ALL_DEPLOYMENT_ACTOR_STATES entries = list( itertools.chain.from_iterable(self._actors_by_state[s] for s in states) ) if code_version is not None: entries = [e for e in entries if e.code_version == code_version] return [e.wrapper for e in entries] def pop( self, code_version: Optional[str] = None, states: Optional[List[DeploymentActorState]] = None, ) -> List[Tuple[DeploymentActorState, DeploymentActorEntry]]: """Remove and return (state, entry) pairs.""" if code_version is None: removed: List[Tuple[DeploymentActorState, DeploymentActorEntry]] = [] for version in list( {t[1].code_version for t in self._actors_index.values()} ): removed.extend(self.pop(version, states=states)) return removed if states is None: states = ALL_DEPLOYMENT_ACTOR_STATES to_remove: List[Tuple[DeploymentActorState, DeploymentActorEntry]] = [] for state in states: bucket = self._actors_by_state.get(state, []) for entry in bucket[:]: if entry.code_version == code_version: bucket.remove(entry) to_remove.append((state, entry)) self._actors_index.pop((entry.code_version, entry.name), None) if not bucket: self._actors_by_state.pop(state, None) return to_remove def count( self, code_version: Optional[str] = None, states: Optional[List[DeploymentActorState]] = None, ) -> int: return len(self.get(code_version, states=states)) def get_wrapper( self, code_version: str, name: str ) -> Optional[DeploymentActorWrapper]: """Get wrapper by (code_version, name), or None if not found.""" existing = self._actors_index.get((code_version, name)) return existing[1].wrapper if existing is not None else None def get_code_versions(self) -> Set[str]: """Return the set of code versions currently in the container.""" return {entry.code_version for _, entry in self._actors_index.values()} def is_empty(self) -> bool: """O(1): True if no actor wrappers are tracked in any state.""" return not self._actors_index class ReplicaStartupStatus(Enum): PENDING_ALLOCATION = 1 PENDING_INITIALIZATION = 2 SUCCEEDED = 3 FAILED = 4 class ReplicaHealthCheckResponse(Enum): NONE = 1 SUCCEEDED = 2 APP_FAILURE = 3 ACTOR_CRASHED = 4 @dataclass class DeploymentTargetState: """The current goal state for a deployment. info: contains the information needed to initialize a replica. target_num_replicas: the number of replicas to run. This should already be adjusted by the target_capacity. version: the goal version of the deployment. deleting: whether the deployment is being deleted. """ info: Optional[DeploymentInfo] target_num_replicas: int version: Optional[DeploymentVersion] deleting: bool @classmethod def default(cls) -> "DeploymentTargetState": return cls(None, -1, None, False) @classmethod def create( cls, info: DeploymentInfo, target_num_replicas: int, *, deleting: bool = False, ) -> "DeploymentTargetState": if deleting: if target_num_replicas != 0: raise ValueError( "target_num_replicas must be 0 when setting target state " f"to deleting. Got {target_num_replicas} instead." ) version = DeploymentVersion( info.version, deployment_config=info.deployment_config, ray_actor_options=info.replica_config.ray_actor_options, placement_group_bundles=info.replica_config.placement_group_bundles, placement_group_strategy=info.replica_config.placement_group_strategy, max_replicas_per_node=info.replica_config.max_replicas_per_node, route_prefix=info.route_prefix, placement_group_bundle_label_selector=( info.replica_config.placement_group_bundle_label_selector ), placement_group_fallback_strategy=( info.replica_config.placement_group_fallback_strategy ), ) return cls(info, target_num_replicas, version, deleting) def is_scaled_copy_of(self, other_target_state: "DeploymentTargetState") -> bool: """Checks if this target state is a scaled copy of another target state. A target state is a scaled copy of another target state if all configurable info is identical, other than target_num_replicas. Returns: True if this target state contains a non-None DeploymentInfo and is a scaled copy of the other target state. """ if other_target_state.info is None: return False if self.info is None: return False actor_options_match = ( self.info.replica_config.ray_actor_options == other_target_state.info.replica_config.ray_actor_options ) bundles_match = ( self.info.replica_config.placement_group_bundles == other_target_state.info.replica_config.placement_group_bundles ) strategy_match = ( self.info.replica_config.placement_group_strategy == other_target_state.info.replica_config.placement_group_strategy ) max_replicas_match = ( self.info.replica_config.max_replicas_per_node == other_target_state.info.replica_config.max_replicas_per_node ) deployment_config_match = self.info.deployment_config.model_dump( exclude={"num_replicas"} ) == other_target_state.info.deployment_config.model_dump( exclude={"num_replicas"} ) # Backward compatibility check for older versions of Ray without these fields. current_bundle_label_selector = getattr( self.info.replica_config, "placement_group_bundle_label_selector", None ) other_bundle_label_selector = getattr( other_target_state.info.replica_config, "placement_group_bundle_label_selector", None, ) bundle_label_selector_match = ( current_bundle_label_selector == other_bundle_label_selector ) current_fallback = getattr( self.info.replica_config, "placement_group_fallback_strategy", None ) other_fallback = getattr( other_target_state.info.replica_config, "placement_group_fallback_strategy", None, ) fallback_match = current_fallback == other_fallback # TODO(zcin): version can be None, this is from an outdated codepath. # We should remove outdated code, so version can never be None. version_match = ( self.version is not None and self.version == other_target_state.version ) return all( [ actor_options_match, bundles_match, strategy_match, bundle_label_selector_match, fallback_match, max_replicas_match, deployment_config_match, version_match, ] ) @dataclass class DeploymentStateUpdateResult: deleted: bool any_replicas_recovering: bool upscale: List[ReplicaSchedulingRequest] downscale: Optional[DeploymentDownscaleRequest] CHECKPOINT_KEY = "serve-deployment-state-checkpoint" SLOW_STARTUP_WARNING_S = int( os.environ.get( "RAY_SERVE_SLOW_STARTUP_WARNING_S", os.environ.get("SERVE_SLOW_STARTUP_WARNING_S", 30), ) ) SLOW_STARTUP_WARNING_PERIOD_S = int( os.environ.get( "RAY_SERVE_SLOW_STARTUP_WARNING_PERIOD_S", os.environ.get("SERVE_SLOW_STARTUP_WARNING_PERIOD_S", 30), ) ) ALL_REPLICA_STATES = list(ReplicaState) _SCALING_LOG_ENABLED = os.environ.get("SERVE_ENABLE_SCALING_LOG", "0") != "0" def print_verbose_scaling_log(): assert _SCALING_LOG_ENABLED log_path = "/tmp/ray/session_latest/logs/monitor.log" last_n_lines = 50 autoscaler_log_last_n_lines = [] if os.path.exists(log_path): with open(log_path) as f: autoscaler_log_last_n_lines = f.readlines()[-last_n_lines:] debug_info = { "nodes": ray.nodes(), "available_resources": ray.available_resources(), "total_resources": ray.cluster_resources(), "autoscaler_logs": autoscaler_log_last_n_lines, } logger.error(f"Scaling information\n{json.dumps(debug_info, indent=2)}") class ActorReplicaWrapper: """Wraps a Ray actor for a deployment replica. This is primarily defined so that we can mock out actual Ray operations for unit testing. *All Ray API calls should be made here, not in DeploymentState.* """ def __init__( self, replica_id: ReplicaID, version: DeploymentVersion, ): self._replica_id = replica_id self._deployment_id = replica_id.deployment_id self._actor_name = replica_id.to_full_id_str() # Populated in either self.start() or self.recover() self._allocated_obj_ref: ObjectRef = None self._ready_obj_ref: ObjectRef = None # Populated in self.recover() for non-cross-language replicas to # asynchronously verify the actor finished its initial setup before # we trigger `initialize_and_get_metadata`. self._was_initialized_obj_ref: Optional[ObjectRef] = None # Set to True when `check_ready()` determines the actor cannot be # recovered (e.g., the previous controller crashed before the actor # finished its initial setup). The reconciler treats this case as a # silent drop / replace rather than a deploy failure, since the # underlying cause is a controller-side crash, not user code. self._unrecoverable: bool = False self._actor_resources: Dict[str, float] = None # If the replica is being started, this will be the true version # If the replica is being recovered, this will be the target # version, which may be inconsistent with the actual replica # version. If so, the actual version will be updated later after # recover() and check_ready() self._version: DeploymentVersion = version self._healthy: bool = True self._health_check_ref: Optional[ObjectRef] = None self._last_health_check_time: float = 0.0 self._consecutive_health_check_failures = 0 self._last_health_check_latency_ms: Optional[float] = None self._last_health_check_failed: Optional[bool] = None self._initialization_latency_s: Optional[float] = None self._reconfigure_start_time: Optional[float] = None self._internal_grpc_port: Optional[int] = None self._docs_path: Optional[str] = None self._route_patterns: Optional[List[str]] = None # Rank assigned to the replica. self._assign_rank_callback: Optional[Callable[[ReplicaID], ReplicaRank]] = None self._rank: Optional[ReplicaRank] = None # Gang context for the replica. self._gang_context: Optional[GangContext] = None # Populated in `on_scheduled` or `recover`. self._actor_handle: ActorHandle = None self._placement_group: PlacementGroup = None # Populated after replica is allocated. self._pid: int = None self._actor_id: str = None self._worker_id: str = None self._node_id: str = None self._node_ip: str = None self._node_instance_id: str = None self._log_file_path: str = None self._http_port: int = None self._grpc_port: int = None # Populated in self.stop(). self._graceful_shutdown_ref: ObjectRef = None # todo: will be confused with deployment_config.is_cross_language self._is_cross_language = False self._deployment_is_cross_language = False self._routing_stats: Dict[str, Any] = {} self._record_routing_stats_ref: Optional[ObjectRef] = None self._last_record_routing_stats_time: float = 0.0 self._has_user_routing_stats_method: bool = False # Static per-replica metadata captured once when the replica became # ready (via the user's `record_replica_metadata` hook). self._replica_metadata: Dict[str, Any] = {} self._ingress: bool = False # Outbound deployments polling state self._outbound_deployments: Optional[List[DeploymentID]] = None # Histogram to track routing stats delay from replica to controller self._routing_stats_delay_histogram = metrics.Histogram( "serve_routing_stats_delay_ms", description=( "The delay in milliseconds for routing stats to propagate " "from replica to controller." ), boundaries=DEFAULT_LATENCY_BUCKET_MS, tag_keys=("deployment", "application"), ) self._routing_stats_delay_histogram.set_default_tags( { "deployment": self._deployment_id.name, "application": self._deployment_id.app_name, } ) # Counter to track exceptions/timeouts when getting routing stats self._routing_stats_error_counter = metrics.Counter( "serve_routing_stats_error", description=( "The number of errors (exceptions or timeouts) when getting " "routing stats from replica." ), tag_keys=("deployment", "replica", "application", "error_type"), ) self._routing_stats_error_counter.set_default_tags( { "deployment": self._deployment_id.name, "replica": self._replica_id.unique_id, "application": self._deployment_id.app_name, } ) @property def replica_id(self) -> str: return self._replica_id @property def deployment_name(self) -> str: return self._deployment_id.name @property def rank(self) -> Optional[ReplicaRank]: return self._rank @property def gang_context(self) -> Optional[GangContext]: return self._gang_context @property def replica_metadata(self) -> Dict[str, Any]: return self._replica_metadata @property def unrecoverable(self) -> bool: return self._unrecoverable @property def app_name(self) -> str: return self._deployment_id.app_name @property def is_cross_language(self) -> bool: return self._is_cross_language @property def actor_handle(self) -> Optional[ActorHandle]: if not self._actor_handle: try: self._actor_handle = ray.get_actor( self._actor_name, namespace=SERVE_NAMESPACE ) except ValueError: self._actor_handle = None if self._is_cross_language: assert isinstance(self._actor_handle, JavaActorHandleProxy) return self._actor_handle.handle return self._actor_handle @property def placement_group_bundles(self) -> Optional[List[Dict[str, float]]]: if not self._placement_group: return None return self._placement_group.bundle_specs @property def version(self) -> DeploymentVersion: """Replica version. This can be incorrect during state recovery. If the controller crashes and the deployment state is being recovered, this will temporarily be the deployment-wide target version, which may be inconsistent with the actual version running on the replica actor. If so, the actual version will be updated when the replica transitions from RECOVERING -> RUNNING """ return self._version @property def deployment_config(self) -> DeploymentConfig: """Deployment config. This can return an incorrect config during state recovery. If the controller hasn't yet recovered the up-to-date version from the running replica actor, this property will return the current target config for the deployment. """ return self._version.deployment_config @property def docs_path(self) -> Optional[str]: return self._docs_path @property def route_patterns(self) -> Optional[List[str]]: return self._route_patterns @property def max_ongoing_requests(self) -> int: return self.deployment_config.max_ongoing_requests @property def max_queued_requests(self) -> int: return self.deployment_config.max_queued_requests @property def graceful_shutdown_timeout_s(self) -> float: return self.deployment_config.graceful_shutdown_timeout_s @property def health_check_period_s(self) -> float: return self.deployment_config.health_check_period_s @property def health_check_timeout_s(self) -> float: return self.deployment_config.health_check_timeout_s @property def http_port(self) -> Optional[int]: return self._http_port @property def grpc_port(self) -> Optional[int]: return self._grpc_port @property def request_routing_stats_period_s(self) -> float: return ( self.deployment_config.request_router_config.request_routing_stats_period_s ) @property def request_routing_stats_timeout_s(self) -> float: return ( self.deployment_config.request_router_config.request_routing_stats_timeout_s ) @property def pid(self) -> Optional[int]: """Returns the pid of the actor, None if not started.""" return self._pid @property def actor_id(self) -> Optional[str]: """Returns the actor id, None if not started.""" return self._actor_id @property def worker_id(self) -> Optional[str]: """Returns the worker id, None if not started.""" return self._worker_id @property def node_id(self) -> Optional[str]: """Returns the node id of the actor, None if not placed.""" return self._node_id @property def node_ip(self) -> Optional[str]: """Returns the node ip of the actor, None if not placed.""" return self._node_ip @property def node_instance_id(self) -> Optional[str]: """Returns the node instance id of the actor, None if not placed.""" return self._node_instance_id @property def log_file_path(self) -> Optional[str]: """Returns the relative log file path of the actor, None if not placed.""" return self._log_file_path @property def initialization_latency_s(self) -> Optional[float]: """Returns the initialization latency for the replica actor. Returns None if the replica hasn't started yet. Note: this value isn't checkpointed, so if the controller restarts, this value goes back to None. """ return self._initialization_latency_s @property def reconfigure_start_time(self) -> Optional[float]: """Returns the start time of the last reconfigure operation. Returns None if no reconfigure operation has started. """ return self._reconfigure_start_time @property def last_health_check_latency_ms(self) -> Optional[float]: """Returns the latency of the last completed health check in milliseconds. Returns None if no health check has completed in the current check cycle. """ return self._last_health_check_latency_ms @property def last_health_check_failed(self) -> Optional[bool]: """Returns whether the last completed health check failed. Returns False if no health check has completed in the current check cycle. """ return self._last_health_check_failed def start( self, deployment_info: DeploymentInfo, assign_rank_callback: Callable[[ReplicaID], ReplicaRank], gang_placement_group: Optional[PlacementGroup] = None, gang_pg_index: Optional[int] = None, gang_context: Optional[GangContext] = None, ) -> ReplicaSchedulingRequest: """Start the current DeploymentReplica instance. The replica will be in the STARTING and PENDING_ALLOCATION states until the deployment scheduler schedules the underlying actor. Args: deployment_info: Configuration info for the deployment. assign_rank_callback: Callback to assign rank to the replica. gang_placement_group: Pre-created gang PG to schedule this replica on. gang_pg_index: Bundle index within the gang PG for this replica. gang_context: Gang context for this replica. Returns: ReplicaSchedulingRequest: The scheduling request for the replica. """ self._assign_rank_callback = assign_rank_callback self._actor_resources = deployment_info.replica_config.resource_dict self._ingress = deployment_info.ingress self._gang_placement_group = gang_placement_group self._gang_pg_index = gang_pg_index self._gang_context = gang_context # it is currently not possible to create a placement group # with no resources (https://github.com/ray-project/ray/issues/20401) self._deployment_is_cross_language = ( deployment_info.deployment_config.is_cross_language ) logger.info( f"Starting {self.replica_id}.", extra={"log_to_stderr": False}, ) actor_def = deployment_info.actor_def if ( deployment_info.deployment_config.deployment_language == DeploymentLanguage.PYTHON ): if deployment_info.replica_config.serialized_init_args is None: serialized_init_args = cloudpickle.dumps(()) else: serialized_init_args = ( cloudpickle.dumps( msgpack_deserialize( deployment_info.replica_config.serialized_init_args ) ) if self._deployment_is_cross_language else deployment_info.replica_config.serialized_init_args ) init_args = ( self.replica_id, cloudpickle.dumps(deployment_info.replica_config.deployment_def) if self._deployment_is_cross_language else deployment_info.replica_config.serialized_deployment_def, serialized_init_args, deployment_info.replica_config.serialized_init_kwargs if deployment_info.replica_config.serialized_init_kwargs else cloudpickle.dumps({}), deployment_info.deployment_config.to_proto_bytes(), self._version, deployment_info.ingress, deployment_info.route_prefix, deployment_info.ingress_request_router, ) # TODO(simon): unify the constructor arguments across language elif ( deployment_info.deployment_config.deployment_language == DeploymentLanguage.JAVA ): self._is_cross_language = True actor_def = ray.cross_language.java_actor_class( "io.ray.serve.replica.RayServeWrappedReplica" ) init_args = ( # String deploymentName, self.deployment_name, # String replicaID, self.replica_id.to_full_id_str(), # String deploymentDef deployment_info.replica_config.deployment_def_name, # byte[] initArgsbytes msgpack_serialize( cloudpickle.loads( deployment_info.replica_config.serialized_init_args ) ) if self._deployment_is_cross_language else deployment_info.replica_config.serialized_init_args, # byte[] deploymentConfigBytes, deployment_info.deployment_config.to_proto_bytes(), # byte[] deploymentVersionBytes, self._version.to_proto().SerializeToString(), # String controllerName # String appName self.app_name, ) actor_options = { "name": self._actor_name, "namespace": SERVE_NAMESPACE, "lifetime": "detached", "enable_task_events": RAY_SERVE_ENABLE_TASK_EVENTS, } actor_options.update(deployment_info.replica_config.ray_actor_options) # A replica's default `max_concurrency` value can prevent it from # respecting the configured `max_ongoing_requests`. To avoid this # unintentional behavior, use `max_ongoing_requests` to override # the Actor's `max_concurrency` if it is larger. if ( deployment_info.deployment_config.max_ongoing_requests > ray_constants.DEFAULT_MAX_CONCURRENCY_ASYNC ): actor_options[ "max_concurrency" ] = deployment_info.deployment_config.max_ongoing_requests return ReplicaSchedulingRequest( replica_id=self.replica_id, actor_def=actor_def, actor_resources=self._actor_resources, actor_options=actor_options, actor_init_args=init_args, placement_group_bundles=( deployment_info.replica_config.placement_group_bundles ), placement_group_strategy=( deployment_info.replica_config.placement_group_strategy ), placement_group_bundle_label_selector=( deployment_info.replica_config.placement_group_bundle_label_selector ), placement_group_fallback_strategy=( deployment_info.replica_config.placement_group_fallback_strategy ), max_replicas_per_node=( deployment_info.replica_config.max_replicas_per_node ), on_scheduled=self.on_scheduled, gang_placement_group=self._gang_placement_group, gang_pg_index=self._gang_pg_index, ) def on_scheduled( self, actor_handle: ActorHandle, placement_group: Optional[PlacementGroup] = None, ): self._actor_handle = actor_handle self._placement_group = placement_group if self._is_cross_language: self._actor_handle = JavaActorHandleProxy(self._actor_handle) self._allocated_obj_ref = self._actor_handle.is_allocated.remote() else: self._allocated_obj_ref = self._actor_handle.is_allocated.remote() def _format_user_config(self, user_config: Any): temp = copy(user_config) if user_config is not None and self._deployment_is_cross_language: if self._is_cross_language: temp = msgpack_serialize(temp) else: temp = msgpack_deserialize(temp) return temp def reconfigure(self, version: DeploymentVersion, rank: ReplicaRank) -> bool: """ Update replica version. Also, updates the deployment config on the actor behind this DeploymentReplica instance if necessary. Returns: whether the actor is being updated. """ updating = False # Determine if we need heavyweight reconfiguration # vs lightweight updates needs_actor_reconfigure = self._version.requires_actor_reconfigure(version) has_rank_changes = self._rank != rank if needs_actor_reconfigure or has_rank_changes: # Call into replica actor reconfigure() with updated user config and # graceful_shutdown_wait_loop_s # Setting updating=True because we want to transition to UPDATING state # when rank is updated or deployment config changes. updating = True self._reconfigure_start_time = time.time() deployment_config = copy(version.deployment_config) deployment_config.user_config = self._format_user_config( deployment_config.user_config ) self._ready_obj_ref = self._actor_handle.reconfigure.remote( deployment_config, rank, version.route_prefix, ) self._version = version self._rank = rank return updating def recover(self, ingress: bool = False) -> bool: """Recover replica version from a live replica actor. When controller dies, the deployment state loses the info on the version that's running on each individual replica actor, so as part of the recovery process, we need to recover the version that is running on the replica actor. Also confirm that actor is allocated and initialized before marking as running. This call is non-blocking: any RPCs needed to query the actor are fired here as ObjectRefs and observed in `check_ready()` from the reconcile loop. Args: ingress: Whether this replica is an ingress replica. Returns: False if the replica actor is no longer alive; the caller drops the replica from tracking. Otherwise True. Replicas that are alive but never finished their initial setup are detected asynchronously in `check_ready()` rather than here so that controller recovery does not block. """ logger.info(f"Recovering {self.replica_id}.") self._ingress = ingress try: self._actor_handle = ray.get_actor( self._actor_name, namespace=SERVE_NAMESPACE ) except ValueError: logger.warning( f"Failed to get handle to replica {self._actor_name} " "during controller recovery. Marking as dead." ) return False try: self._placement_group = ray.util.get_placement_group( self._actor_name, ) except ValueError: # ValueError is raised if the placement group does not exist. self._placement_group = None # Re-fetch initialization proof self._allocated_obj_ref = self._actor_handle.is_allocated.remote() # Running actor handle already has all info needed, thus successful # starting simply means retrieving replica version hash from actor if self._is_cross_language: self._ready_obj_ref = self._actor_handle.check_health.remote() else: # For non-cross-language replicas, asynchronously probe whether # the actor finished its initial setup before triggering # `initialize_and_get_metadata`. If the previous controller # crashed between actor creation and the first # `initialize_and_get_metadata(rank=...)` call, the actor has # neither a rank nor an initialized user callable. Recovering it # would silently complete its initialization with `rank=None`, # leaving rank tracking permanently broken for that replica. # `check_ready()` waits for this probe and replaces the replica # if it reports `False`. We defer firing # `initialize_and_get_metadata` until the probe passes so we # don't accidentally drive the bad actor through initialization # only to kill it afterwards. self._was_initialized_obj_ref = self._actor_handle.was_initialized.remote() return True def _kill_unrecoverable_actor(self) -> None: """Force-kill an actor that cannot be recovered. Best-effort: any failure to kill the actor is logged but ignored. """ try: ray.kill(self._actor_handle, no_restart=True) except Exception: logger.exception( f"Failed to kill unrecoverable replica actor " f"{self._replica_id} during controller recovery." ) def check_ready(self) -> Tuple[ReplicaStartupStatus, Optional[str]]: """ Check if current replica has started by making ray API calls on relevant actor / object ref. Replica initialization calls __init__(), reconfigure(), and check_health(). Returns: state (ReplicaStartupStatus): PENDING_ALLOCATION: replica is waiting for a worker to start PENDING_INITIALIZATION: replica initialization hasn't finished. FAILED: replica initialization failed. SUCCEEDED: replica initialization succeeded. error_msg: None: for PENDING_ALLOCATION, PENDING_INITIALIZATION or SUCCEEDED states str: for FAILED state """ # Check whether the replica has been allocated. if self._allocated_obj_ref is None or not check_obj_ref_ready_nowait( self._allocated_obj_ref ): return ReplicaStartupStatus.PENDING_ALLOCATION, None if not self._is_cross_language: try: ( self._pid, self._actor_id, self._worker_id, self._node_id, self._node_ip, self._node_instance_id, self._log_file_path, ) = ray.get(self._allocated_obj_ref) except RayTaskError as e: logger.exception( f"Exception in {self._replica_id}, the replica will be stopped." ) return ReplicaStartupStatus.FAILED, str(e.as_instanceof_cause()) except RuntimeEnvSetupError as e: msg = f"Exception when allocating {self._replica_id}: {str(e)}" logger.exception(msg) return ReplicaStartupStatus.FAILED, msg except Exception: msg = ( f"Exception when allocating {self._replica_id}:\n" + traceback.format_exc() ) logger.exception(msg) return ReplicaStartupStatus.FAILED, msg # If we issued a `was_initialized` probe in `recover()`, wait for it # before triggering `initialize_and_get_metadata`. If the actor was # never initialized previously (the previous controller crashed # mid-startup), kill it and signal an unrecoverable drop so the # reconciler replaces it without counting a deploy failure. if self._was_initialized_obj_ref is not None: if not check_obj_ref_ready_nowait(self._was_initialized_obj_ref): return ReplicaStartupStatus.PENDING_INITIALIZATION, None probe_ref = self._was_initialized_obj_ref self._was_initialized_obj_ref = None try: was_initialized = ray.get(probe_ref) except Exception as e: msg = ( f"Failed to probe initialization state of " f"{self._replica_id} during controller recovery ({e!r}). " "Replacing with a fresh replica." ) logger.warning(msg) self._kill_unrecoverable_actor() self._unrecoverable = True return ReplicaStartupStatus.FAILED, msg if not was_initialized: msg = ( f"{self._replica_id} was found alive but never finished " "its initial setup; the previous controller likely " "crashed during replica startup. Replacing with a fresh " "replica." ) logger.warning(msg) self._kill_unrecoverable_actor() self._unrecoverable = True return ReplicaStartupStatus.FAILED, msg # Probe succeeded; safe to drive the actor through recovery. self._ready_obj_ref = ( self._actor_handle.initialize_and_get_metadata.remote() ) if self._ready_obj_ref is None: # Perform auto method name translation for java handles. # See https://github.com/ray-project/ray/issues/21474 deployment_config = copy(self._version.deployment_config) deployment_config.user_config = self._format_user_config( deployment_config.user_config ) if self._is_cross_language: self._ready_obj_ref = self._actor_handle.is_initialized.remote( deployment_config.to_proto_bytes() ) else: replica_ready_check_func = ( self._actor_handle.initialize_and_get_metadata ) # this guarantees that node_id is set before rank is assigned self._rank = self._assign_rank_callback( self._replica_id.unique_id, self._node_id ) self._ready_obj_ref = replica_ready_check_func.remote( deployment_config, self._rank, self._gang_context ) return ReplicaStartupStatus.PENDING_INITIALIZATION, None # Check whether replica initialization has completed. replica_ready = check_obj_ref_ready_nowait(self._ready_obj_ref) # In case of deployment constructor failure, ray.get will help to # surface exception to each update() cycle. if not replica_ready: return ReplicaStartupStatus.PENDING_INITIALIZATION, None else: try: # TODO(simon): fully implement reconfigure for Java replicas. if self._is_cross_language: return ReplicaStartupStatus.SUCCEEDED, None # todo: The replica's userconfig whitch java client created # is different from the controller's userconfig if not self._deployment_is_cross_language: # This should only update version if the replica is being recovered. # If this is checking on a replica that is newly started, this # should return a version that is identical to what's already stored ( _, self._version, self._initialization_latency_s, self._internal_grpc_port, self._docs_path, self._http_port, self._grpc_port, self._rank, self._route_patterns, self._outbound_deployments, self._has_user_routing_stats_method, self._gang_context, self._replica_metadata, ) = ray.get(self._ready_obj_ref) except RayTaskError as e: logger.exception( f"Exception in {self._replica_id}, the replica will be stopped." ) # NOTE(zcin): we should use str(e) instead of traceback.format_exc() # here because the full details of the error is not displayed properly # with traceback.format_exc(). return ReplicaStartupStatus.FAILED, str(e.as_instanceof_cause()) except Exception as e: logger.exception( f"Exception in {self._replica_id}, the replica will be stopped." ) return ReplicaStartupStatus.FAILED, repr(e) return ReplicaStartupStatus.SUCCEEDED, None @property def actor_resources(self) -> Optional[Dict[str, float]]: return self._actor_resources @property def available_resources(self) -> Dict[str, float]: return ray.available_resources() def graceful_stop(self) -> Duration: """Request the actor to exit gracefully. Returns the timeout after which to kill the actor. """ try: handle = ray.get_actor(self._actor_name, namespace=SERVE_NAMESPACE) if self._is_cross_language: handle = JavaActorHandleProxy(handle) self._graceful_shutdown_ref = handle.perform_graceful_shutdown.remote() except ValueError: # ValueError thrown from ray.get_actor means actor has already been deleted. pass return self.graceful_shutdown_timeout_s def check_stopped(self) -> bool: """Check if the actor has exited.""" stopped = False try: handle = ray.get_actor(self._actor_name, namespace=SERVE_NAMESPACE) if self._graceful_shutdown_ref is None: # graceful_stop() failed to set the shutdown ref (e.g., the # actor was not found at that time). Treat as not yet stopped; # the next reconcile iteration will retry. stopped = False else: stopped = check_obj_ref_ready_nowait(self._graceful_shutdown_ref) if stopped: try: ray.get(self._graceful_shutdown_ref) except Exception: logger.exception( "Exception when trying to gracefully shutdown replica:\n" + traceback.format_exc() ) ray.kill(handle, no_restart=True) except ValueError: # ValueError thrown from ray.get_actor means actor has already been deleted. stopped = True finally: # Remove the placement group both if the actor has already been deleted or # it was just killed above. if stopped and self._placement_group is not None: try: ray.util.remove_placement_group(self._placement_group) except ValueError: # ValueError thrown from ray.util.remove_placement_group means the # placement group has already been removed. logger.debug( f"Placement group for {self._replica_id} was already removed." ) return stopped def _check_active_health_check(self) -> ReplicaHealthCheckResponse: """Check the active health check (if any). self._health_check_ref will be reset to `None` when the active health check is deemed to have succeeded or failed. This method *does not* start a new health check, that's up to the caller. Returns: - NONE if there's no active health check, or it hasn't returned yet and the timeout is not up. - SUCCEEDED if the active health check succeeded. - APP_FAILURE if the active health check failed (or didn't return before the timeout). - ACTOR_CRASHED if the underlying actor crashed. """ # Reset the last health check status for this check cycle. # We do this because _check_active_health_check is being called in a loop, # and we want to avoid accumulating latency and failure metrics over multiple # check cycles. self._last_health_check_latency_ms = None self._last_health_check_failed = None if self._health_check_ref is None: # There is no outstanding health check. response = ReplicaHealthCheckResponse.NONE elif check_obj_ref_ready_nowait(self._health_check_ref): # Object ref is ready, ray.get it to check for exceptions. try: ray.get(self._health_check_ref) # Calculate health check latency. self._last_health_check_latency_ms = ( time.time() - self._last_health_check_time ) * 1000 self._last_health_check_failed = False # Health check succeeded without exception. response = ReplicaHealthCheckResponse.SUCCEEDED except RayActorError: # Health check failed due to actor crashing. response = ReplicaHealthCheckResponse.ACTOR_CRASHED self._last_health_check_failed = True except RayError as e: # Health check failed due to application-level exception. logger.warning(f"Health check for {self._replica_id} failed: {e}") response = ReplicaHealthCheckResponse.APP_FAILURE self._last_health_check_failed = True elif time.time() - self._last_health_check_time > self.health_check_timeout_s: # Health check hasn't returned and the timeout is up, consider it failed. logger.warning( "Didn't receive health check response for replica " f"{self._replica_id} after " f"{self.health_check_timeout_s}s, marking it unhealthy." ) response = ReplicaHealthCheckResponse.APP_FAILURE # Calculate latency for timeout case. self._last_health_check_latency_ms = ( time.time() - self._last_health_check_time ) * 1000 self._last_health_check_failed = True else: # Health check hasn't returned and the timeout isn't up yet. response = ReplicaHealthCheckResponse.NONE if response is not ReplicaHealthCheckResponse.NONE: self._health_check_ref = None return response def _should_start_new_health_check(self) -> bool: """Determines if a new health check should be kicked off. A health check will be started if: 1) There is not already an active health check. 2) It has been more than health_check_period_s since the previous health check was *started*. This assumes that self._health_check_ref is reset to `None` when an active health check succeeds or fails (due to returning or timeout). """ if self._health_check_ref is not None: # There's already an active health check. return False # If there's no active health check, kick off another and reset # the timer if it's been long enough since the last health # check. Add some randomness to avoid synchronizing across all # replicas. time_since_last = time.time() - self._last_health_check_time randomized_period = self.health_check_period_s * random.uniform(0.9, 1.1) return time_since_last > randomized_period def _should_record_routing_stats(self) -> bool: """Determines if a new record routing stats should be kicked off. A record routing stats will be started if: 1) The user has defined a record_routing_stats method on their deployment class. If not, skip entirely to avoid unnecessary remote calls that would just return empty dicts. 2) There is not already an active record routing stats. 3) It has been more than request_routing_stats_period_s since the previous record routing stats was *started*. This assumes that self._record_routing_stats_ref is reset to `None` when an active record routing stats succeeds or fails (due to returning or timeout). """ if not self._has_user_routing_stats_method: # The user hasn't defined a record_routing_stats method, so # there's no point in making remote calls to collect stats. return False if self._record_routing_stats_ref is not None: # There's already an active record routing stats. return False # If there's no active record routing stats, kick off another and # reset the timer if it's been long enough since the last record # routing stats. Add some randomness to avoid synchronizing across # all replicas. time_since_last = time.time() - self._last_record_routing_stats_time randomized_period = self.request_routing_stats_period_s * random.uniform( 0.9, 1.1 ) return time_since_last > randomized_period def check_health(self) -> bool: """Check if the actor is healthy. self._healthy should *only* be modified in this method. This is responsible for: 1) Checking the outstanding health check (if any). 2) Determining the replica health based on the health check results. 3) Kicking off a new health check if needed. """ response: ReplicaHealthCheckResponse = self._check_active_health_check() if response is ReplicaHealthCheckResponse.NONE: # No info; don't update replica health. pass elif response is ReplicaHealthCheckResponse.SUCCEEDED: # Health check succeeded. Reset the consecutive failure counter # and mark the replica healthy. if self._consecutive_health_check_failures > 0: logger.info( f"{self._replica_id} passed the health check after " f"{self._consecutive_health_check_failures} consecutive failures." ) self._consecutive_health_check_failures = 0 self._healthy = True elif response is ReplicaHealthCheckResponse.APP_FAILURE: # Health check failed. If it has failed more than N times in a row, # mark the replica unhealthy. self._consecutive_health_check_failures += 1 if ( self._consecutive_health_check_failures >= REPLICA_HEALTH_CHECK_UNHEALTHY_THRESHOLD ): logger.warning( f"Replica {self._replica_id} failed the health " f"check {self._consecutive_health_check_failures} " "times in a row, marking it unhealthy." ) self._healthy = False elif response is ReplicaHealthCheckResponse.ACTOR_CRASHED: # Actor crashed, mark the replica unhealthy immediately. logger.warning( f"Actor for {self._replica_id} crashed, marking " "it unhealthy immediately." ) self._healthy = False else: assert False, f"Unknown response type: {response}." if self._should_start_new_health_check(): self._last_health_check_time = time.time() self._health_check_ref = self._actor_handle.check_health.remote() return self._healthy def get_routing_stats(self) -> Dict[str, Any]: """Get the routing stats for the replica.""" if self._record_routing_stats_ref is None: # There's no active record routing stats. pass elif check_obj_ref_ready_nowait(self._record_routing_stats_ref): # Object ref is ready, ray.get it to check for exceptions. try: self._routing_stats = ray.get(self._record_routing_stats_ref) # Record the round-trip delay for routing stats delay_ms = (time.time() - self._last_record_routing_stats_time) * 1000 self._routing_stats_delay_histogram.observe(delay_ms) except Exception: logger.exception( "Exception when trying to get routing stats:\n" + traceback.format_exc() ) self._routing_stats_error_counter.inc(tags={"error_type": "exception"}) self._record_routing_stats_ref = None elif ( time.time() - self._last_record_routing_stats_time > self.request_routing_stats_timeout_s ): # Record routing stats hasn't returned and the timeout is up, retrying. logger.warning( "Didn't receive routing stats response for replica " f"{self._replica_id} after " f"{self.request_routing_stats_timeout_s}s, retrying." ) self._routing_stats_error_counter.inc(tags={"error_type": "timeout"}) self._record_routing_stats_ref = None if self._should_record_routing_stats(): self._last_record_routing_stats_time = time.time() self._record_routing_stats_ref = ( self._actor_handle.record_routing_stats.remote() ) return self._routing_stats def force_stop(self): """Force the actor to exit without shutting down gracefully.""" try: ray.kill(ray.get_actor(self._actor_name, namespace=SERVE_NAMESPACE)) except ValueError: pass def get_outbound_deployments(self) -> Optional[List[DeploymentID]]: return self._outbound_deployments class DeploymentReplica: """Manages state transitions for deployment replicas. This is basically a checkpointable lightweight state machine. """ def __init__( self, replica_id: ReplicaID, version: DeploymentVersion, ): self._replica_id = replica_id self._actor = ActorReplicaWrapper(replica_id, version) self._start_time = None self._shutdown_start_time: Optional[float] = None self._actor_details = ReplicaDetails( actor_name=replica_id.to_full_id_str(), replica_id=self._replica_id.unique_id, state=ReplicaState.STARTING, start_time_s=0, ) self._multiplexed_model_ids: List[str] = [] self._routing_stats: Dict[str, Any] = {} def get_running_replica_info( self, cluster_node_info_cache: ClusterNodeInfoCache ) -> RunningReplicaInfo: return RunningReplicaInfo( replica_id=self._replica_id, node_id=self.actor_node_id, node_ip=self._actor.node_ip, availability_zone=cluster_node_info_cache.get_node_az(self.actor_node_id), actor_name=self._actor._actor_name, max_ongoing_requests=self._actor.max_ongoing_requests, is_cross_language=self._actor.is_cross_language, multiplexed_model_ids=self.multiplexed_model_ids, routing_stats=self.routing_stats, replica_metadata=self.replica_metadata, port=self._actor._internal_grpc_port, backend_http_port=self._actor._http_port or None, ) def record_multiplexed_model_ids(self, multiplexed_model_ids: List[str]): """Record the multiplexed model ids for this replica.""" self._multiplexed_model_ids = multiplexed_model_ids def record_routing_stats(self, routing_stats: Optional[Dict[str, Any]]): """Record the routing stats for this replica. Recording routing_stats as an empty dictionary is valid. But skip update if the routing_stats is None. """ if routing_stats is not None: self._routing_stats = routing_stats @property def multiplexed_model_ids(self) -> List[str]: return self._multiplexed_model_ids @property def routing_stats(self) -> Dict[str, Any]: return self._routing_stats @property def replica_metadata(self) -> Dict[str, Any]: # Captured by the actor wrapper from the ready handshake (and re-captured # on a new replica incarnation), so no separate restore path is needed. return getattr(self._actor, "replica_metadata", {}) @property def actor_details(self) -> ReplicaDetails: return self._actor_details @property def replica_id(self) -> ReplicaID: return self._replica_id @property def deployment_name(self) -> str: return self._replica_id.deployment_id.name @property def app_name(self) -> str: return self._replica_id.deployment_id.app_name @property def version(self): return self._actor.version @property def docs_path(self) -> Optional[str]: return self._actor.docs_path @property def route_patterns(self) -> Optional[List[str]]: return self._actor.route_patterns @property def actor_id(self) -> str: return self._actor.actor_id @property def actor_handle(self) -> ActorHandle: return self._actor.actor_handle @property def actor_node_id(self) -> Optional[str]: """Returns the node id of the actor, None if not placed.""" return self._actor.node_id @property def actor_http_port(self) -> Optional[int]: return self._actor.http_port @property def actor_grpc_port(self) -> Optional[int]: return self._actor.grpc_port @property def actor_pid(self) -> Optional[int]: """Returns the node id of the actor, None if not placed.""" return self._actor.pid @property def initialization_latency_s(self) -> Optional[float]: """Returns how long the replica took to initialize.""" return self._actor.initialization_latency_s @property def reconfigure_start_time(self) -> Optional[float]: """Returns the start time of the last reconfigure operation.""" return self._actor.reconfigure_start_time @property def last_health_check_latency_ms(self) -> Optional[float]: """Returns the latency of the last completed health check in milliseconds.""" return self._actor.last_health_check_latency_ms @property def last_health_check_failed(self) -> Optional[bool]: """Returns whether the last completed health check failed.""" return self._actor.last_health_check_failed @property def shutdown_start_time(self) -> Optional[float]: """Returns the start time of the shutdown operation.""" return self._shutdown_start_time def start( self, deployment_info: DeploymentInfo, assign_rank_callback: Callable[[ReplicaID], ReplicaRank], gang_placement_group: Optional[PlacementGroup] = None, gang_pg_index: Optional[int] = None, gang_context: Optional[GangContext] = None, ) -> ReplicaSchedulingRequest: """ Start a new actor for current DeploymentReplica instance. Args: deployment_info: Configuration info for the deployment. assign_rank_callback: Callback to assign rank to the replica. gang_placement_group: Pre-created gang PG to schedule this replica on. gang_pg_index: Bundle index within the gang PG for this replica. gang_context: Gang context for this replica. Returns: ReplicaSchedulingRequest: The scheduling request for the replica. """ replica_scheduling_request = self._actor.start( deployment_info, assign_rank_callback=assign_rank_callback, gang_placement_group=gang_placement_group, gang_pg_index=gang_pg_index, gang_context=gang_context, ) self._start_time = time.time() self.update_actor_details(start_time_s=self._start_time) return replica_scheduling_request def reconfigure( self, version: DeploymentVersion, rank: ReplicaRank, ) -> bool: """ Update replica version. Also, updates the deployment config on the actor behind this DeploymentReplica instance if necessary. Returns: whether the actor is being updated. """ return self._actor.reconfigure(version, rank=rank) def recover(self, deployment_info: DeploymentInfo) -> bool: """ Recover states in DeploymentReplica instance by fetching running actor status Args: deployment_info: The deployment info for this replica. Returns: True if the replica actor is alive and recovered successfully. False if the replica actor is no longer alive. """ # If replica is no longer alive if not self._actor.recover(ingress=deployment_info.ingress): return False self._start_time = time.time() self.update_actor_details(start_time_s=self._start_time) return True @property def rank(self) -> Optional[ReplicaRank]: """Get the rank assigned to the replica.""" return self._actor.rank @property def gang_context(self) -> Optional[GangContext]: """Get the gang context for this replica.""" return self._actor.gang_context @property def unrecoverable(self) -> bool: """Whether `check_ready()` determined the actor cannot be recovered. When True, the reconciler should drop and replace the replica without counting it as a deploy failure (the underlying cause is a previous controller crash, not user code). """ return self._actor.unrecoverable def check_started( self, ) -> Tuple[ReplicaStartupStatus, Optional[str], Optional[float]]: """Check if the replica has started. If so, transition to RUNNING. Should handle the case where the replica has already stopped. Returns: status: Most recent state of replica by querying actor obj ref """ is_ready = self._actor.check_ready() self.update_actor_details( pid=self._actor.pid, node_id=self._actor.node_id, node_ip=self._actor.node_ip, node_instance_id=self._actor.node_instance_id, actor_id=self._actor.actor_id, worker_id=self._actor.worker_id, log_file_path=self._actor.log_file_path, ) return is_ready def stop(self, graceful: bool = True) -> None: """Stop the replica. Should handle the case where the replica is already stopped. """ state = self._actor_details.state logger.info( f"Stopping {self.replica_id} (currently {state}).", extra={"log_to_stderr": False}, ) self._shutdown_start_time = time.time() timeout_s = self._actor.graceful_stop() if not graceful: timeout_s = 0 elif self._actor._ingress and RAY_SERVE_ENABLE_DIRECT_INGRESS: # In direct ingress mode, ensure we wait at least # RAY_SERVE_DIRECT_INGRESS_MIN_DRAINING_PERIOD_S to give external # load balancers (e.g., ALB) time to deregister the replica. timeout_s = max(timeout_s, RAY_SERVE_DIRECT_INGRESS_MIN_DRAINING_PERIOD_S) self._shutdown_deadline = time.time() + timeout_s def check_stopped(self) -> bool: """Check if the replica has finished stopping.""" if self._actor.check_stopped(): return True timeout_passed = time.time() >= self._shutdown_deadline if timeout_passed: logger.info( f"{self.replica_id} did not shut down after grace " "period, force-killing it." ) self._actor.force_stop() return False def check_health(self) -> bool: """Check if the replica is healthy. Returns `True` if the replica is healthy, else `False`. """ return self._actor.check_health() def pull_routing_stats(self) -> Optional[Dict[str, Any]]: """Get the latest response from the routing stats on the replica. Returns None if the replica is still calculating the stats. """ return self._actor.get_routing_stats() def update_state(self, state: ReplicaState) -> None: """Updates state in actor details.""" self.update_actor_details(state=state) _SENTINEL = object() def update_actor_details(self, **kwargs) -> None: # Fast path: skip if all provided values are already current. # This avoids unnecessary object creation on every tick when the # pop-iterate-readd pattern re-adds replicas without state changes. # We use _SENTINEL (not None) as the getattr default so that an # invalid field name always fails the check and falls through to # .copy(), which will raise an appropriate error. if all( getattr(self._actor_details, k, self._SENTINEL) == v for k, v in kwargs.items() ): return # Use .model_copy(update=...) instead of .model_dump() + reconstruction # to avoid full Pydantic serialization and validation on every update. self._actor_details = self._actor_details.model_copy(update=kwargs) def resource_requirements(self) -> Tuple[str, str]: """Returns required and currently available resources. Only resources with nonzero requirements will be included in the required dict and only resources in the required dict will be included in the available dict (filtered for relevance). """ if self._actor.actor_resources is None: return "UNKNOWN", "UNKNOWN" if self._actor.placement_group_bundles is not None: required = self._actor.placement_group_bundles else: required = { k: v for k, v in self._actor.actor_resources.items() if v is not None and v > 0 } available = { k: v for k, v in self._actor.available_resources.items() if k in required } # Use json.dumps() instead of str() here to avoid double-quoting keys # when dumping these objects. See # https://github.com/ray-project/ray/issues/26210 for the issue. return json.dumps(required), json.dumps(available) def get_outbound_deployments(self) -> Optional[List[DeploymentID]]: return self._actor.get_outbound_deployments() class ReplicaStateContainer: """Container for mapping ReplicaStates to lists of DeploymentReplicas.""" def __init__(self, on_replica_state_change=None): self._replicas: Dict[ReplicaState, List[DeploymentReplica]] = defaultdict(list) self._replica_id_index: Dict[ReplicaID, DeploymentReplica] = {} self._on_replica_state_change = on_replica_state_change def __getstate__(self): # Exclude the callback to keep the container picklable (the callback # is a bound method on DeploymentState which holds unpicklable objects # like asyncio futures). Used by _dump_replica_states_for_testing. state = self.__dict__.copy() state["_on_replica_state_change"] = None return state def add(self, state: ReplicaState, replica: DeploymentReplica): """Add the provided replica under the provided state. Args: state: state to add the replica under. replica: replica to add. """ assert isinstance(state, ReplicaState), f"Type: {type(state)}" actor_details = getattr(replica, "actor_details", None) old_state = actor_details.state if actor_details is not None else None replica.update_state(state) self._replicas[state].append(replica) self._replica_id_index[replica.replica_id] = replica if self._on_replica_state_change and state != old_state: self._on_replica_state_change(old_state, state) def get( self, states: Optional[List[ReplicaState]] = None ) -> List[DeploymentReplica]: """Get all replicas of the given states. This does not remove them from the container. Replicas are returned in order of state as passed in. Args: states: states to consider. If not specified, all replicas are considered. Returns: The matching replicas, in the same order as ``states``. """ if states is None: return list(self._replica_id_index.values()) assert isinstance(states, list) return list( itertools.chain.from_iterable(self._replicas[state] for state in states) ) def get_by_id(self, replica_id: ReplicaID) -> Optional[DeploymentReplica]: """Get a replica by its ID in O(1) time. Args: replica_id: the ID of the replica to look up. Returns: The DeploymentReplica if found, else None. """ return self._replica_id_index.get(replica_id) def pop( self, exclude_version: Optional[DeploymentVersion] = None, states: Optional[List[ReplicaState]] = None, max_replicas: Optional[int] = math.inf, ) -> List[DeploymentReplica]: """Get and remove all replicas of the given states. This removes the replicas from the container. Replicas are returned in order of state as passed in. Args: exclude_version: if specified, replicas of the provided version will *not* be removed. states: states to consider. If not specified, all replicas are considered. max_replicas: max number of replicas to return. If not specified, will pop all replicas matching the criteria. Returns: The removed replicas, in the same order as ``states``. """ if states is None: states = ALL_REPLICA_STATES assert exclude_version is None or isinstance(exclude_version, DeploymentVersion) assert isinstance(states, list) replicas = [] for state in states: popped = [] remaining = [] for replica in self._replicas[state]: if len(replicas) + len(popped) == max_replicas: remaining.append(replica) elif exclude_version is not None and replica.version == exclude_version: remaining.append(replica) else: popped.append(replica) self._replicas[state] = remaining replicas.extend(popped) for replica in replicas: self._replica_id_index.pop(replica.replica_id, None) return replicas def count( self, exclude_version: Optional[DeploymentVersion] = None, version: Optional[DeploymentVersion] = None, states: Optional[List[ReplicaState]] = None, ): """Get the total count of replicas of the given states. Args: exclude_version: version to exclude. If not specified, all versions are considered. version: version to filter to. If not specified, all versions are considered. states: states to consider. If not specified, all replicas are considered. Returns: The number of replicas matching the criteria. """ if states is None: states = ALL_REPLICA_STATES assert isinstance(states, list) assert exclude_version is None or isinstance(exclude_version, DeploymentVersion) assert version is None or isinstance(version, DeploymentVersion) if exclude_version is None and version is None: return sum(len(self._replicas[state]) for state in states) elif exclude_version is None and version is not None: return sum( sum(1 for r in self._replicas[state] if r.version == version) for state in states ) elif exclude_version is not None and version is None: return sum( sum(1 for r in self._replicas[state] if r.version != exclude_version) for state in states ) else: raise ValueError( "Only one of `version` or `exclude_version` may be provided." ) def remove(self, replica_ids: Set[ReplicaID]) -> List[DeploymentReplica]: """Remove and return all replicas whose IDs are in the given set. Performs a single pass over the container. Non-matching replicas stay in place without being re-added (so no spurious ``update_state`` / ``update_actor_details`` calls). Args: replica_ids: collection of ReplicaIDs to remove. Returns: The list of removed DeploymentReplicas. """ replica_ids = set(replica_ids) removed = [] remaining_to_find = len(replica_ids) for state in ALL_REPLICA_STATES: if remaining_to_find == 0: break found_any = False remaining = [] for replica in self._replicas[state]: if remaining_to_find > 0 and replica.replica_id in replica_ids: removed.append(replica) remaining_to_find -= 1 found_any = True else: remaining.append(replica) if found_any: self._replicas[state] = remaining return removed def __str__(self): return str(self._replicas) def __repr__(self): return repr(self._replicas) class RankManager: """Manages ranks for a single node.""" def __init__(self): self._ranks: Dict[str, int] = {} self._released_ranks: Set[int] = set() self._next_rank: int = 0 def assign_rank(self, key: str) -> int: if key in self._ranks: raise RuntimeError(f"Rank for {key} already assigned: {self._ranks[key]}") if self._released_ranks: # Reuse the smallest released rank rank = min(self._released_ranks) self._released_ranks.remove(rank) else: # Assign the next available rank # This is the first time we're assigning a rank to this replica rank = self._next_rank self._next_rank += 1 self._ranks[key] = rank return rank def release_rank(self, key: str) -> None: if key not in self._ranks: raise RuntimeError(f"Rank for {key} not assigned") rank = self._ranks.pop(key) # Add the released rank to the set of released ranks # This rank can be reused for a new replica self._released_ranks.add(rank) def recover_rank(self, key: str, rank: int) -> None: if key in self._ranks: raise RuntimeError(f"Rank for {key} already assigned: {self._ranks[key]}") self._ranks[key] = rank self._released_ranks.discard(rank) if rank >= self._next_rank: self._next_rank = rank + 1 def get_rank(self, key: str) -> int: if key not in self._ranks: raise RuntimeError(f"Rank for {key} not assigned") return self._ranks[key] def has_rank(self, key: str) -> bool: return key in self._ranks def get_ranks_mapping(self) -> Dict[str, int]: return self._ranks.copy() def clear(self) -> None: self._ranks.clear() self._released_ranks.clear() self._next_rank = 0 def check_rank_consistency_and_reassign_minimally( self, active_keys: List[str], ) -> List[str]: """Verify rank system invariants and reassign ranks when needed. This method ensures: 1. All active keys have ranks 2. No duplicate ranks exist 3. Ranks are contiguous when at target count Args: active_keys: List of currently active keys Returns: List of keys that need to be reconfigured with new ranks Raises: RuntimeError: If rank system invariants are violated and fail_on_error=True """ if not active_keys: return [] active_keys_set = set(active_keys) # Check for stale ranks - this should never happen stale_keys = set(self._ranks.keys()) - active_keys_set if stale_keys: logger.error( f"Found stale ranks for keys: {stale_keys}. " "This should never happen. Please report this as a bug." ) raise RuntimeError("Rank system is in an invalid state.") # Verify system invariants - all active keys must have ranks unranked_keys = active_keys_set - set(self._ranks.keys()) if unranked_keys: logger.error( f"Found active keys without ranks: {unranked_keys}. " "This should never happen. Please report this as a bug." ) raise RuntimeError("Rank system is in an invalid state.") # Check for duplicate ranks - this should never happen rank_counts = {} for key, rank in self._ranks.copy().items(): if key in active_keys_set: # Only check active keys rank_counts[rank] = rank_counts.get(rank, 0) + 1 if rank_counts[rank] > 1: logger.error( f"Found duplicate rank {rank} assigned to multiple keys. " "This should never happen. Please report this as a bug." ) raise RuntimeError("Rank system is in an invalid state.") # Check if we need to reassign ranks for contiguity # Only force contiguity when at target count (e.g., after autoscaling down) current_ranks = sorted(self._ranks.values()) expected_ranks = list(range(len(active_keys))) keys_needing_reconfiguration_from_reassignment = [] if current_ranks != expected_ranks: logger.debug( f"At target count but ranks are not contiguous. " f"Current: {current_ranks}, Expected: {expected_ranks}. " "Performing minimal reassignment." ) keys_needing_reconfiguration_from_reassignment = ( self._perform_minimal_rank_reassignment(active_keys) ) return keys_needing_reconfiguration_from_reassignment def _perform_minimal_rank_reassignment(self, active_keys: List[str]) -> List[str]: """Perform minimal rank reassignment to achieve contiguity. This method reassigns ranks while minimizing the number of keys that need to be reconfigured. It prioritizes keeping existing ranks when possible. Args: active_keys: List of currently active keys Returns: List of keys that need to be reconfigured with new ranks """ target_ranks_set = set(range(len(active_keys))) # Find which keys need new ranks keys_needing_ranks = [] keys_keeping_ranks = [] for key in active_keys: current_rank = self.get_rank(key) if current_rank in target_ranks_set: # This key can keep its rank target_ranks_set.remove(current_rank) # O(1) operation keys_keeping_ranks.append(key) else: # This key needs a new rank keys_needing_ranks.append(key) # Convert remaining target ranks to sorted list for deterministic assignment available_ranks = sorted(target_ranks_set) # Assign new ranks to keys that need them for i, key in enumerate(keys_needing_ranks): new_rank = available_ranks[i] # O(1) operation # Store the old rank before updating old_rank = self._ranks[key] logger.debug(f"Reassigning key {key}: rank {old_rank} -> {new_rank}") # Update the rank mapping self._ranks[key] = new_rank # Remove the newly assigned rank from available ranks self._released_ranks.discard(new_rank) # Add the old rank back to available ranks for reuse self._released_ranks.add(old_rank) # Log the reassignment summary logger.debug( f"Minimal reassignment complete: {len(keys_keeping_ranks)} keys kept ranks, " f"{len(keys_needing_ranks)} keys reassigned" ) return keys_needing_ranks class DeploymentRankManager: """Manages replica ranks for a deployment. This class handles rank assignment, release, consistency checking, and reassignment. It maintains the rank system invariants and provides a clean interface for rank operations. Maintains three levels of rank tracking: - Global rank: Replica-level rank across all nodes (0, 1, 2, ...) - Local rank: Replica's rank within its node (0, 1, 2, ... per node) - Node rank ID: Index assigned to each node (0, 1, 2, ...) """ def __init__(self, fail_on_rank_error: bool = True): # Global rank manager (existing replica-level rank) self._replica_rank_manager = RankManager() self._fail_on_rank_error = fail_on_rank_error # Node rank manager (assigns rank IDs to nodes) self._node_rank_manager = RankManager() # Local rank managers (one per node, manages replica ranks within each node) self._local_rank_managers: Dict[str, RankManager] = {} # Track which node each replica is on self._replica_to_node: Dict[str, str] = {} def _execute_with_error_handling(self, func, safe_default, *args, **kwargs): if self._fail_on_rank_error: # Let exceptions propagate return func(*args, **kwargs) else: # Catch exceptions and return safe default try: return func(*args, **kwargs) except Exception as e: logger.error(f"Error executing function {func.__name__}: {e}") return safe_default def assign_rank(self, replica_id: str, node_id: str) -> ReplicaRank: """Assign a rank to a new replica. Args: replica_id: The unique ID of the replica node_id: The unique ID of the node Returns: ReplicaRank object with the assigned rank Raises: RuntimeError: If the replica already has a rank assigned """ def _assign_rank_impl(): if self.has_replica_rank(replica_id): raise RuntimeError( f"Rank for {replica_id} already assigned: {self._replica_rank_manager.get_rank(replica_id)}" ) # Track the replica-to-node mapping self._replica_to_node[replica_id] = node_id # Assign global rank rank = self._replica_rank_manager.assign_rank(replica_id) # Assign node rank if this node doesn't have one yet if node_id not in self._local_rank_managers: self._node_rank_manager.assign_rank(node_id) self._local_rank_managers[node_id] = RankManager() node_rank = self._node_rank_manager.get_rank(node_id) # Assign local rank within the node local_rank = self._local_rank_managers[node_id].assign_rank(replica_id) return ReplicaRank(rank=rank, node_rank=node_rank, local_rank=local_rank) return self._execute_with_error_handling( _assign_rank_impl, ReplicaRank(rank=0, node_rank=0, local_rank=0) ) def release_rank(self, replica_id: str) -> None: """Release rank for a replica. Args: replica_id: ID of the replica Raises: RuntimeError: If replica doesn't have ranks """ def _release_rank_impl(): if not self.has_replica_rank(replica_id): raise RuntimeError(f"Rank for {replica_id} not assigned") # Get the node_id from the replica mapping node_id = self._replica_to_node[replica_id] # Release global rank self._replica_rank_manager.release_rank(replica_id) # Release local rank self._local_rank_managers[node_id].release_rank(replica_id) # Release node rank if this was the last replica on the node if len(self._local_rank_managers[node_id].get_ranks_mapping()) == 0: self._node_rank_manager.release_rank(node_id) del self._local_rank_managers[node_id] # Remove replica from node mapping del self._replica_to_node[replica_id] return self._execute_with_error_handling(_release_rank_impl, None) def recover_rank( self, replica_id: str, node_id: str, rank: ReplicaRank, ) -> None: """Recover rank for a replica (e.g., after controller restart). Args: replica_id: ID of the replica node_id: ID of the node rank: The rank to recover Raises: RuntimeError: If replica already has ranks assigned """ def _recover_rank_impl(): if self.has_replica_rank(replica_id): raise RuntimeError( f"Rank for {replica_id} already assigned: {self._replica_rank_manager.get_rank(replica_id)}" ) # Recover global rank self._replica_rank_manager.recover_rank(replica_id, rank.rank) # Recover node rank only if this node doesn't already have one if not self._node_rank_manager.has_rank(node_id): self._node_rank_manager.recover_rank(node_id, rank.node_rank) # Recover local rank if node_id not in self._local_rank_managers: self._local_rank_managers[node_id] = RankManager() self._local_rank_managers[node_id].recover_rank(replica_id, rank.local_rank) # Track the replica-to-node mapping self._replica_to_node[replica_id] = node_id return self._execute_with_error_handling(_recover_rank_impl, None) def has_replica_rank(self, replica_id: str) -> bool: """Check if replica has a rank assigned. Args: replica_id: The unique ID of the replica Returns: True if the replica has a rank assigned, False otherwise Raises: RuntimeError: If the replica doesn't have ranks assigned """ if replica_id not in self._replica_to_node: return False node_id = self._replica_to_node[replica_id] return ( self._replica_rank_manager.has_rank(replica_id) and node_id in self._local_rank_managers and self._node_rank_manager.has_rank(node_id) and self._local_rank_managers[node_id].has_rank(replica_id) ) def get_replica_rank(self, replica_id: str) -> ReplicaRank: """Get the rank for a replica. Args: replica_id: ID of the replica Returns: ReplicaRank object Raises: RuntimeError: If replica doesn't have ranks assigned """ def _get_replica_rank_impl(): if not self.has_replica_rank(replica_id): raise RuntimeError(f"Rank for {replica_id} not assigned") global_rank = self._replica_rank_manager.get_rank(replica_id) node_id = self._replica_to_node[replica_id] node_rank = self._node_rank_manager.get_rank(node_id) local_rank = self._local_rank_managers[node_id].get_rank(replica_id) return ReplicaRank( rank=global_rank, node_rank=node_rank, local_rank=local_rank ) return self._execute_with_error_handling( _get_replica_rank_impl, ReplicaRank(rank=0, node_rank=0, local_rank=0) ) def check_rank_consistency_and_reassign_minimally( self, active_replicas: List["DeploymentReplica"], ) -> List["DeploymentReplica"]: """Verify rank system invariants and reassign ranks when needed across all three levels. This method ensures: 1. Global ranks are contiguous [0, N-1] for N replicas 2. Node ranks are contiguous [0, M-1] for M nodes 3. Local ranks are contiguous [0, K-1] for K replicas on each node Args: active_replicas: List of currently active replicas Returns: List of replicas that need to be reconfigured with new ranks """ def _check_rank_consistency_impl(): if not active_replicas: return [] # Extract replica IDs from replicas active_replica_ids = [ replica.replica_id.unique_id for replica in active_replicas ] # Create a mapping from replica ID to replica object for quick lookup replica_id_to_replica = { replica.replica_id.unique_id: replica for replica in active_replicas } # Track all replicas needing reconfiguration from any rank system all_replica_ids_needing_reconfiguration = set() # STEP 1: Check global rank consistency replica_ids_from_global = self._replica_rank_manager.check_rank_consistency_and_reassign_minimally( active_replica_ids ) all_replica_ids_needing_reconfiguration.update(replica_ids_from_global) # STEP 2: Group replicas by node and check local rank consistency per node replicas_by_node: Dict[str, List[str]] = {} for replica_id in active_replica_ids: node_id = self._replica_to_node.get(replica_id) assert ( node_id is not None ), f"Replica {replica_id} not assigned to any node" if node_id not in replicas_by_node: replicas_by_node[node_id] = [] replicas_by_node[node_id].append(replica_id) for node_id, replica_ids_on_node in replicas_by_node.items(): replica_ids_from_local = self._local_rank_managers[ node_id ].check_rank_consistency_and_reassign_minimally(replica_ids_on_node) all_replica_ids_needing_reconfiguration.update(replica_ids_from_local) # STEP 3: Check node rank consistency active_node_ids = list(replicas_by_node.keys()) if active_node_ids: node_ids_needing_reassignment = self._node_rank_manager.check_rank_consistency_and_reassign_minimally( active_node_ids, ) # If any nodes were reassigned, all replicas on those nodes need reconfiguration for node_id in node_ids_needing_reassignment: all_replica_ids_needing_reconfiguration.update( replicas_by_node[node_id] ) # Convert replica IDs back to replica objects # Filter out stale replicas that are not in the active set replicas_needing_reconfiguration = [ replica_id_to_replica[replica_id] for replica_id in all_replica_ids_needing_reconfiguration if replica_id in replica_id_to_replica ] return replicas_needing_reconfiguration return self._execute_with_error_handling(_check_rank_consistency_impl, []) def clear(self) -> None: self._replica_rank_manager.clear() self._node_rank_manager.clear() self._local_rank_managers.clear() self._replica_to_node.clear() def get_replica_ranks_mapping(self) -> Dict[str, ReplicaRank]: """Get the current mapping of replica IDs to ReplicaRank objects. Returns: Dict mapping replica_id to ReplicaRank object """ result = {} for replica_id in self._replica_rank_manager.get_ranks_mapping().keys(): result[replica_id] = self.get_replica_rank(replica_id) return result class DeploymentState: """Manages the target state and replicas for a single deployment.""" FORCE_STOP_UNHEALTHY_REPLICAS = RAY_SERVE_FORCE_STOP_UNHEALTHY_REPLICAS def __init__( self, id: DeploymentID, long_poll_host: LongPollHost, deployment_scheduler: DeploymentScheduler, cluster_node_info_cache: ClusterNodeInfoCache, autoscaling_state_manager: AutoscalingStateManager, ): self._id = id self._long_poll_host: LongPollHost = long_poll_host self._deployment_scheduler = deployment_scheduler self._cluster_node_info_cache = cluster_node_info_cache self._autoscaling_state_manager = autoscaling_state_manager # Each time we set a new deployment goal, we're trying to save new # DeploymentInfo and bring current deployment to meet new status. self._target_state: DeploymentTargetState = DeploymentTargetState.default() self._prev_startup_warning: float = time.time() self._replica_constructor_error_msg: Optional[str] = None # Counter for how many times replicas failed to start. This is reset to 0 when: # (1) The deployment is deployed / re-deployed. # (2) The deployment reaches the HEALTHY state. self._replica_constructor_retry_counter: int = 0 # Flag for whether any replicas of the target version has successfully started. # This is reset to False when the deployment is re-deployed. self._replica_has_started: bool = False # Set when a deployment-scoped actor fails to start (constructor error). # Checked in check_curr_status to transition to DEPLOY_FAILED. self._deployment_actor_failed: Optional[str] = None # Counter for consecutive deployment actor start failures. Reset on # successful actor readiness or re-deploy. After exceeding the # threshold, the deployment is considered terminally failed. self._deployment_actor_retry_counter: int = 0 self._replicas: ReplicaStateContainer = ReplicaStateContainer( on_replica_state_change=self._on_replica_state_change ) self._recent_dead_replicas: Deque[ReplicaDetails] = deque( maxlen=RAY_SERVE_RETAINED_DEAD_REPLICAS ) self._curr_status_info: DeploymentStatusInfo = DeploymentStatusInfo( self._id.name, DeploymentStatus.UPDATING, DeploymentStatusTrigger.CONFIG_UPDATE_STARTED, ) self._rank_manager = DeploymentRankManager( fail_on_rank_error=RAY_SERVE_FAIL_ON_RANK_ERROR ) self.replica_average_ongoing_requests: Dict[str, float] = {} # Cache the last-reported health gauge value and timestamp per replica. # This avoids redundant Gauge.set() calls on every control loop # iteration, which are expensive at scale (O(num_replicas) Cython # FFI calls per loop). We only call Gauge.set() when the value # changes or the cache entry is older than _HEALTH_GAUGE_REPORT_INTERVAL_S # (to ensure the metric is re-exported within each Prometheus scrape window). self._health_gauge_cache: Dict[str, Tuple[int, float]] = {} self._last_health_check_healthy_replica_ids: Set[str] = set() # Maintain gang membership bookkeeping to avoid O(num_replicas) lookups when stopping gangs. # Updated on replica creation during upscaling and permanent removal during downscaling. self._gang_id_by_replica: Dict[ReplicaID, str] = {} self._replicas_by_gang_id: Dict[str, Set[ReplicaID]] = defaultdict(set) # Deployment-scoped actor lifecycle (per deployment) self._deployment_actors = DeploymentActorContainer(self._id) replica_lifecycle_metric_tag_keys = ( ("deployment", "replica", "application") if RAY_SERVE_CONTROLLER_METRICS_INCLUDE_HIGH_CARDINALITY_TAGS else ("deployment", "application") ) self.health_check_gauge = metrics.Gauge( "serve_deployment_replica_healthy", description=( "Tracks healthy replicas. When source tags are enabled, each " "replica series is 1 for healthy and 0 for unhealthy; otherwise, " "the deployment/application series is the healthy replica count." ), tag_keys=replica_lifecycle_metric_tag_keys, ) self.health_check_gauge.set_default_tags( {"deployment": self._id.name, "application": self._id.app_name} ) # Histogram for replica startup latency (time from creation to ready state). self.replica_startup_latency_histogram = metrics.Histogram( "serve_replica_startup_latency_ms", description=("Time from replica creation to ready state in milliseconds."), boundaries=REPLICA_STARTUP_SHUTDOWN_LATENCY_BUCKETS_MS, tag_keys=("deployment", "application"), ) self.replica_startup_latency_histogram.set_default_tags( {"deployment": self._id.name, "application": self._id.app_name} ) # Histogram for replica initialization latency. self.replica_initialization_latency_histogram = metrics.Histogram( "serve_replica_initialization_latency_ms", description=("Time for replica to initialize in milliseconds."), boundaries=REPLICA_STARTUP_SHUTDOWN_LATENCY_BUCKETS_MS, tag_keys=("deployment", "application"), ) self.replica_initialization_latency_histogram.set_default_tags( {"deployment": self._id.name, "application": self._id.app_name} ) # Histogram for replica reconfigure latency. # NOTE(abrar): value of this metric represents reconfigure + time until next controller loop self.replica_reconfigure_latency_histogram = metrics.Histogram( "serve_replica_reconfigure_latency_ms", description=("Time for replica to complete reconfigure in milliseconds."), boundaries=REQUEST_LATENCY_BUCKETS_MS, tag_keys=("deployment", "application"), ) self.replica_reconfigure_latency_histogram.set_default_tags( {"deployment": self._id.name, "application": self._id.app_name} ) # Histogram for health check latency. self.health_check_latency_histogram = metrics.Histogram( "serve_health_check_latency_ms", description=("Duration of health check calls in milliseconds."), boundaries=REQUEST_LATENCY_BUCKETS_MS, tag_keys=("deployment", "application"), ) self.health_check_latency_histogram.set_default_tags( {"deployment": self._id.name, "application": self._id.app_name} ) # Counter for health check failures. self.health_check_failures_counter = metrics.Counter( "serve_health_check_failures_total", description=("Count of failed health checks."), tag_keys=replica_lifecycle_metric_tag_keys, ) self.health_check_failures_counter.set_default_tags( {"deployment": self._id.name, "application": self._id.app_name} ) # Histogram for replica shutdown duration. self.replica_shutdown_duration_histogram = metrics.Histogram( "serve_replica_shutdown_duration_ms", description=( "Time from shutdown signal to replica fully stopped in milliseconds." ), boundaries=REPLICA_STARTUP_SHUTDOWN_LATENCY_BUCKETS_MS, tag_keys=("deployment", "application"), ) self.replica_shutdown_duration_histogram.set_default_tags( {"deployment": self._id.name, "application": self._id.app_name} ) self.target_replicas_gauge = metrics.Gauge( "serve_autoscaling_target_replicas", description=( "The target number of replicas for this deployment. " "This is the number the autoscaler is trying to reach." ), tag_keys=("deployment", "application"), ) self.target_replicas_gauge.set_default_tags( {"deployment": self._id.name, "application": self._id.app_name} ) # Whether the request routing info have been updated since the last # time we checked. self._request_routing_info_updated = False # Dirty flag: set when replicas transition state (start, stop, health # check fail, migration) or when availability-related fields change. # When False *and* _request_routing_info_updated is False, the # broadcast_running_replicas_if_changed() method can skip all work. self._broadcasted_replicas_set_changed = True # Reconciliation flag: when False the deployment is in steady state # (all replicas RUNNING at target version, status HEALTHY) and # expensive per-tick work in check_curr_status(), # scale_deployment_replicas(), and the startup/stopping sections of # check_and_update_replicas() can be skipped. Health checks on # RUNNING/PENDING_MIGRATION replicas always run regardless. # Cleared only when check_curr_status() confirms steady state. self._in_transition = True self._last_broadcasted_running_replica_infos: List[RunningReplicaInfo] = [] self._last_broadcasted_availability: Optional[bool] = None self._last_broadcasted_deployment_config = None self._docs_path: Optional[str] = None self._route_patterns: Optional[List[str]] = None _BROADCAST_STATES = frozenset( {ReplicaState.RUNNING, ReplicaState.PENDING_MIGRATION} ) def _on_replica_state_change( self, old_state: ReplicaState, new_state: ReplicaState ) -> None: """Called by ReplicaStateContainer.add() when a replica transitions.""" broadcast_set_changed = (old_state in self._BROADCAST_STATES) != ( new_state in self._BROADCAST_STATES ) if broadcast_set_changed: self._broadcasted_replicas_set_changed = True self._in_transition = True def should_autoscale(self) -> bool: """ Check if the deployment is under autoscaling """ return self._autoscaling_state_manager.should_autoscale_deployment(self._id) def get_checkpoint_data(self) -> DeploymentTargetState: """ Return deployment's target state submitted by user's deployment call. Should be persisted and outlive current ray cluster. """ return self._target_state def recover_target_state_from_checkpoint( self, target_state_checkpoint: DeploymentTargetState ): logger.info(f"Recovering target state for {self._id} from checkpoint.") self._target_state = target_state_checkpoint self._deployment_scheduler.on_deployment_deployed( self._id, self._target_state.info.replica_config ) if self._target_state.info.deployment_config.autoscaling_config: self._autoscaling_state_manager.register_deployment( self._id, self._target_state.info, self._target_state.target_num_replicas, ) self._recover_deployment_actors() def recover_current_state_from_replica_actor_names( self, replica_actor_names: List[str] ): """Recover deployment state from live replica actors found in the cluster.""" assert self._target_state is not None, ( "Target state should be recovered successfully first before " "recovering current state from replica actor names." ) logger.info( f"Recovering current state for {self._id} " f"from {len(replica_actor_names)} live actors." ) # All current states use default value, only attach running replicas. for replica_actor_name in replica_actor_names: replica_id = ReplicaID.from_full_id_str(replica_actor_name) new_deployment_replica = DeploymentReplica( replica_id, self._target_state.version, ) # If replica is no longer alive, simply don't add it to the # deployment state manager to track. if not new_deployment_replica.recover(self._target_state.info): logger.warning(f"{replica_id} died before controller could recover it.") continue self._replicas.add(ReplicaState.RECOVERING, new_deployment_replica) self._deployment_scheduler.on_replica_recovering(replica_id) logger.debug(f"RECOVERING {replica_id}.") def _recover_deployment_actors(self): """Recover deployment-scoped actors that survived a controller restart. Deployment actors are created with ``lifetime="detached"`` and ``get_if_exists=True``, so they survive across controller restarts. After restoring the target state from a checkpoint, probe for each expected actor by name. If found, add a wrapper in READY state to avoid the unnecessary STARTING delay that would occur if we let ``start_deployment_actors()`` recreate them via ``get_if_exists``. """ version = self._target_state.version if version is None: return deployment_actors_configs = self._get_deployment_actors_configs(version) if not deployment_actors_configs: return code_ver = version.code_version recovered = 0 for cfg in deployment_actors_configs: actor_name = get_deployment_actor_name( self._id, cfg.name, code_version=code_ver ) try: handle = ray.get_actor(actor_name, namespace=SERVE_NAMESPACE) except ValueError: logger.info( f"Deployment actor '{cfg.name}' for {self._id} " f"(code_version={code_ver}) not found during recovery; " "will be recreated." ) continue wrapper = DeploymentActorWrapper( deployment_id=self._id, config=cfg, code_version=code_ver, recovered_handle=handle, ) self._deployment_actors.add(DeploymentActorState.RECOVERING, wrapper) recovered += 1 if recovered > 0: logger.info( f"Recovered {recovered}/{len(deployment_actors_configs)} deployment actors " f"for {self._id} (code_version={code_ver})." ) @property def target_info(self) -> DeploymentInfo: return self._target_state.info @property def target_version(self) -> DeploymentVersion: return self._target_state.version @property def target_num_replicas(self) -> int: return self._target_state.target_num_replicas @property def curr_status_info(self) -> DeploymentStatusInfo: return self._curr_status_info @property def deployment_name(self) -> str: return self._id.name @property def app_name(self) -> str: return self._id.app_name @property def docs_path(self) -> Optional[str]: return self._docs_path @property def route_patterns(self) -> Optional[List[str]]: return self._route_patterns @property def _failed_to_start_threshold(self) -> int: return min( self._target_state.info.deployment_config.max_constructor_retry_count, self._target_state.target_num_replicas * MAX_PER_REPLICA_RETRY_COUNT, ) @property def _deployment_actor_failed_to_start_threshold(self) -> int: """Deployment actor failure threshold. Uses max_constructor_retry_count only. Unlike replica threshold, deployment actors are deployment-scoped so we don't scale by target_num_replicas (which would be 0 during scale-to-zero/deletion). """ return self._target_state.info.deployment_config.max_constructor_retry_count def _get_deployment_actors_configs( self, version: Optional[DeploymentVersion] = None ) -> List[DeploymentActorConfig]: """Return deployment actor configs for the given version, or [] if None.""" v = version if version is not None else self._target_state.version if v is None: return [] return v.deployment_config.deployment_actors or [] def _deployment_actors_satisfied_for_target(self) -> bool: """True when every configured deployment-scoped actor is RUNNING for the target. STARTING/RECOVERING do not count: otherwise ``check_curr_status`` could mark HEALTHY and clear ``_in_transition`` while ``scale_deployment_replicas`` still needs to run ``check_deployment_actors_ready`` to promote pending actors (e.g. after a health-check recreation). """ configs = self._get_deployment_actors_configs() if not configs: return True target_version = self._target_state.version if target_version is None: return False code_ver = target_version.code_version expected_names = {cfg.name for cfg in configs} running_names = { w.actor_logical_name for w in self._deployment_actors.get( code_ver, states=[DeploymentActorState.RUNNING], ) } return expected_names == running_names def _replica_startup_failing(self) -> bool: """Check whether replicas are currently failing and the number of failures has exceeded a threshold. """ return ( self._target_state.target_num_replicas > 0 and self._replica_constructor_retry_counter >= self._failed_to_start_threshold ) def _terminally_failed(self) -> bool: """Check whether the current version is terminally errored. The version is considered terminally errored if the number of replica failures has exceeded a threshold (and there hasn't been any replicas of the target version that has successfully started), or if deployment-scoped actors have permanently failed to start. """ replica_failed = ( not self._replica_has_started and self._replica_startup_failing() ) return replica_failed or self.deployment_actor_terminally_failed() def get_alive_replica_actor_ids(self) -> Set[str]: return {replica.actor_id for replica in self._replicas.get()} def get_running_replica_ids(self) -> List[ReplicaID]: return [ replica.replica_id for replica in self._replicas.get( [ReplicaState.RUNNING, ReplicaState.PENDING_MIGRATION] ) ] def get_running_replica_infos(self) -> List[RunningReplicaInfo]: return [ replica.get_running_replica_info(self._cluster_node_info_cache) for replica in self._replicas.get( [ReplicaState.RUNNING, ReplicaState.PENDING_MIGRATION] ) ] def get_num_running_replicas(self, version: DeploymentVersion = None) -> int: return self._replicas.count(states=[ReplicaState.RUNNING], version=version) def get_gang_config(self): return ( self._target_state.info.deployment_config.gang_scheduling_config if self._target_state is not None else None ) @property def _is_gang_deployment(self) -> bool: """Returns True if this deployment uses gang scheduling.""" return self.get_gang_config() is not None def _get_target_replica_delta(self) -> int: """Calculate delta between target replicas and active replicas.""" current_replicas = self._replicas.count( states=[ReplicaState.STARTING, ReplicaState.UPDATING, ReplicaState.RUNNING] ) recovering_replicas = self._replicas.count(states=[ReplicaState.RECOVERING]) return ( self._target_state.target_num_replicas - current_replicas - recovering_replicas ) def get_active_node_ids(self) -> Set[str]: """Get the node ids of all running replicas in this deployment. This is used to determine which node has replicas. Only nodes with replicas and head node should have active proxies. """ active_states = [ ReplicaState.STARTING, ReplicaState.UPDATING, ReplicaState.RECOVERING, ReplicaState.RUNNING, # NOTE(zcin): We still want a proxy to run on a draining # node before all the replicas are migrated. ReplicaState.PENDING_MIGRATION, ] return { replica.actor_node_id for replica in self._replicas.get(active_states) if replica.actor_node_id is not None } def list_replica_details(self) -> List[ReplicaDetails]: return [replica.actor_details for replica in self._replicas.get()] def list_recent_dead_replicas(self) -> List[ReplicaDetails]: return list(self._recent_dead_replicas) def broadcast_running_replicas_if_changed(self) -> None: """Broadcasts the set of running replicas over long poll if it has changed. Keeps an in-memory record of the last set of running replicas that was broadcast to determine if it has changed. The set will also be broadcast if any replicas have an updated set of multiplexed model IDs. Uses a dirty flag (_broadcasted_replicas_set_changed) to skip all work in steady state when no replicas have transitioned and no routing info has been updated. RunningReplicaInfo objects are only constructed when a broadcast may actually be needed. """ # Fast path: nothing could have changed, skip entirely. if ( not self._broadcasted_replicas_set_changed and not self._request_routing_info_updated ): return # Hold off on broadcasting while replicas are in RECOVERING state to avoid sending # partial or empty routable set. if self._replicas.count(states=[ReplicaState.RECOVERING]) > 0: return running_replica_infos = self.get_running_replica_infos() is_available = not self._terminally_failed() running_broadcasted_replicas_set_changed = ( set(self._last_broadcasted_running_replica_infos) != set(running_replica_infos) or self._request_routing_info_updated ) availability_changed = is_available != self._last_broadcasted_availability # Clear the dirty flag now that we've done the comparison. self._broadcasted_replicas_set_changed = False if not running_broadcasted_replicas_set_changed and not availability_changed: return deployment_metadata = DeploymentTargetInfo( is_available=is_available, running_replicas=running_replica_infos, ) self._long_poll_host.notify_changed( { ( LongPollNamespace.DEPLOYMENT_TARGETS, self._id, ): deployment_metadata, # NOTE(zcin): notify changed for Java routers. Since Java only # supports 1.x API, there is no concept of applications in Java, # so the key should remain a string describing the deployment # name. If there are no Java routers, this is a no-op. ( LongPollNamespace.DEPLOYMENT_TARGETS, self._id.name, ): deployment_metadata, } ) self._last_broadcasted_running_replica_infos = running_replica_infos self._last_broadcasted_availability = is_available self._request_routing_info_updated = False def broadcast_deployment_config_if_changed(self) -> None: """Broadcasts the deployment config over long poll if it has changed. Keeps an in-memory record of the last config that was broadcast to determine if it has changed. """ current_deployment_config = self._target_state.info.deployment_config if self._last_broadcasted_deployment_config == current_deployment_config: return self._long_poll_host.notify_changed( {(LongPollNamespace.DEPLOYMENT_CONFIG, self._id): current_deployment_config} ) self._last_broadcasted_deployment_config = current_deployment_config def _set_target_state_deleting(self) -> None: """Set the target state for the deployment to be deleted.""" target_state = DeploymentTargetState.create( info=self._target_state.info, target_num_replicas=0, deleting=True, ) self._target_state = target_state self._curr_status_info = self._curr_status_info.handle_transition( trigger=DeploymentStatusInternalTrigger.DELETE ) self._broadcasted_replicas_set_changed = True self._in_transition = True logger.info( f"Deleting {self._id}", extra={"log_to_stderr": False}, ) def _set_target_state( self, target_info: DeploymentInfo, target_num_replicas: int, updated_via_api: bool = False, ) -> None: """Set the target state for the deployment to the provided info. Args: target_info: The info with which to set the target state. target_num_replicas: The number of replicas that this deployment should attempt to run. updated_via_api: Whether the target state update was triggered via API. """ new_target_state = DeploymentTargetState.create( target_info, target_num_replicas, deleting=False ) if self._target_state.version == new_target_state.version: # Record either num replica or autoscaling config lightweight update if ( self._target_state.version.deployment_config.autoscaling_config != new_target_state.version.deployment_config.autoscaling_config ): ServeUsageTag.AUTOSCALING_CONFIG_LIGHTWEIGHT_UPDATED.record("True") elif updated_via_api: ServeUsageTag.NUM_REPLICAS_VIA_API_CALL_UPDATED.record("True") elif ( self._target_state.version.deployment_config.num_replicas != new_target_state.version.deployment_config.num_replicas ): ServeUsageTag.NUM_REPLICAS_LIGHTWEIGHT_UPDATED.record("True") self._target_state = new_target_state self._broadcasted_replicas_set_changed = True self._in_transition = True # Emit target replicas metric self.target_replicas_gauge.set(target_num_replicas) def deploy(self, deployment_info: DeploymentInfo) -> bool: """Deploy the deployment. If the deployment already exists with the same version, config, target_capacity, and target_capacity_direction, this method returns False. Args: deployment_info: The target deployment info to apply. Returns: bool: Whether the target state has changed. """ curr_deployment_info = self._target_state.info if curr_deployment_info is not None: # Redeploying should not reset the deployment's start time. if not self._target_state.deleting: deployment_info.start_time_ms = curr_deployment_info.start_time_ms deployment_settings_changed = ( self._target_state.deleting or curr_deployment_info.deployment_config != deployment_info.deployment_config or curr_deployment_info.replica_config.ray_actor_options != deployment_info.replica_config.ray_actor_options or curr_deployment_info.route_prefix != deployment_info.route_prefix or deployment_info.version is None or curr_deployment_info.version != deployment_info.version ) target_capacity_changed = ( curr_deployment_info.target_capacity != deployment_info.target_capacity or curr_deployment_info.target_capacity_direction != deployment_info.target_capacity_direction ) else: deployment_settings_changed = True target_capacity_changed = True # Exit early if the deployment info hasn't changed. Ensures this method # is idempotent. if not deployment_settings_changed and not target_capacity_changed: # Emit target replicas metric when the deployment info hasn't changed. self.target_replicas_gauge.set(self._target_state.target_num_replicas) return False logger.debug(f"Deploying '{self._id}': {deployment_info.to_dict()}") logger.debug( f"Current target state for '{self._id}': " f"{self._target_state.info.to_dict() if self._target_state.info is not None else None}" ) if deployment_info.deployment_config.autoscaling_config: target_num_replicas = self._autoscaling_state_manager.register_deployment( self._id, deployment_info, self._target_state.target_num_replicas ) else: self._autoscaling_state_manager.deregister_deployment(self._id) target_num_replicas = get_capacity_adjusted_num_replicas( deployment_info.deployment_config.num_replicas, deployment_info.target_capacity, ) old_target_state = self._target_state self._set_target_state(deployment_info, target_num_replicas=target_num_replicas) self._deployment_scheduler.on_deployment_deployed( self._id, deployment_info.replica_config ) # Determine if the updated target state simply scales the current state. # Although the else branch handles the CONFIG_UPDATE, we also take this branch # for a config update whose only effect is changing `num_replicas`. # Treating it as a scaling event keeps the user-visible deployment status more # consistent for observability. if self._target_state.is_scaled_copy_of(old_target_state): old_num = old_target_state.target_num_replicas new_num = self._target_state.target_num_replicas if new_num > old_num: self._curr_status_info = self._curr_status_info.handle_transition( trigger=DeploymentStatusInternalTrigger.MANUALLY_INCREASE_NUM_REPLICAS, # noqa: E501 message=f"Upscaling from {old_num} to {new_num} replicas.", ) elif new_num < old_num: self._curr_status_info = self._curr_status_info.handle_transition( trigger=DeploymentStatusInternalTrigger.MANUALLY_DECREASE_NUM_REPLICAS, # noqa: E501 message=f"Downscaling from {old_num} to {new_num} replicas.", ) else: # Otherwise, the deployment configuration has actually been updated. self._curr_status_info = self._curr_status_info.handle_transition( trigger=DeploymentStatusInternalTrigger.CONFIG_UPDATE ) logger.info( f"Deploying new version of {self._id} " f"(initial target replicas: {target_num_replicas})." ) self._replica_constructor_retry_counter = 0 self._replica_has_started = False self._deployment_actor_failed = None self._deployment_actor_retry_counter = 0 return True def autoscale(self, decision_num_replicas: int) -> bool: """ Apply the given scaling decision by updating the target replica count. Skips if deleting, if `decision_num_replicas` is None, or matches the current target. Otherwise updates the state and logs an up/down scaling. Args: decision_num_replicas: target replica count to apply. Returns: bool: True if the target state was updated, False if no change occurred. """ if self._target_state.deleting: return False if decision_num_replicas == self._target_state.target_num_replicas: return False new_info = copy(self._target_state.info) new_info.version = self._target_state.version.code_version old_num = self._target_state.target_num_replicas self._set_target_state(new_info, decision_num_replicas) # The deployment should only transition to UPSCALING/DOWNSCALING # if it's within the autoscaling bounds if not self._autoscaling_state_manager.is_within_bounds( self._id, self._replicas.count( states=[ReplicaState.RUNNING], version=self._target_state.version ), ): return True curr_stats_str = ( f"Current ongoing requests: " f"{self._autoscaling_state_manager.get_total_num_requests_for_deployment(self._id):.2f}, " f"current running replicas: " f"{self._replicas.count(states=[ReplicaState.RUNNING])}." ) new_num = self._target_state.target_num_replicas if new_num > old_num: logger.info( f"Upscaling {self._id} from {old_num} to {new_num} replicas. " f"{curr_stats_str}" ) self._curr_status_info = self._curr_status_info.handle_transition( trigger=DeploymentStatusInternalTrigger.AUTOSCALE_UP, message=f"Upscaling from {old_num} to {new_num} replicas.", ) self._autoscaling_state_manager.record_scale_up(self._id) elif new_num < old_num: logger.info( f"Downscaling {self._id} from {old_num} to {new_num} replicas. " f"{curr_stats_str}" ) self._curr_status_info = self._curr_status_info.handle_transition( trigger=DeploymentStatusInternalTrigger.AUTOSCALE_DOWN, message=f"Downscaling from {old_num} to {new_num} replicas.", ) self._autoscaling_state_manager.record_scale_down(self._id) return True def delete(self) -> bool: if not self._target_state.deleting: self._set_target_state_deleting() return True return False def set_target_num_replicas( self, target_num_replicas: int, ) -> None: """Set the target state for the deployment to the provided info.""" self._set_target_state( self._target_state.info, target_num_replicas, updated_via_api=True ) def _stop_or_update_outdated_version_replicas( self, max_to_stop: float = math.inf ) -> bool: """Stop or update replicas with outdated versions. Stop replicas with versions that require the actor to be restarted, and reconfigure replicas that require refreshing deployment config values. For gang-scheduled deployments, replicas that need restarting are grouped by gang_id and stopped in complete gangs so that we never leave a gang partially torn down. Args: max_to_stop: max number of replicas to stop, by default, it stops all replicas with an outdated version. Returns: Whether any replicas were stopped or reconfigured. """ replicas_to_update = self._replicas.pop( exclude_version=self._target_state.version, states=[ ReplicaState.STARTING, ReplicaState.PENDING_MIGRATION, ReplicaState.RUNNING, ], ) replicas_changed = False code_version_changes = 0 reconfigure_changes = 0 # Process gang-grouped replicas if self._is_gang_deployment: need_restart: List[DeploymentReplica] = [] remaining: List[DeploymentReplica] = [] for replica in replicas_to_update: if replica.version.requires_actor_restart(self._target_state.version): need_restart.append(replica) else: remaining.append(replica) # Group restart-candidates by gang_id. gangs: Dict[str, List[DeploymentReplica]] = defaultdict(list) for replica in need_restart: gangs[replica.gang_context.gang_id].append(replica) # Stop complete gangs atomically within the budget for _, gang_replicas in gangs.items(): expected_size = gang_replicas[0].gang_context.world_size if len(gang_replicas) != expected_size: # Gang is incomplete (members may be RECOVERING/UPDATING); # wait for them to stabilize before tearing down. for replica in gang_replicas: self._replicas.add(replica.actor_details.state, replica) elif code_version_changes + len(gang_replicas) <= max_to_stop: for replica in gang_replicas: code_version_changes += 1 # Forcefully stop siblings to avoid partial gangs self._stop_replica(replica, graceful_stop=False) replicas_changed = True else: # Not enough budget for this gang; put replicas back for replica in gang_replicas: self._replicas.add(replica.actor_details.state, replica) # In gang deployments, replicas that only require reconfiguration are processed below. # All gang replicas that require actor restart and fit within the max_to_stop budget have already been stopped. replicas_to_update = remaining # Per-replica restart/reconfigure handling for replica in replicas_to_update: if (code_version_changes + reconfigure_changes) >= max_to_stop: self._replicas.add(replica.actor_details.state, replica) # If the new version requires the actors to be restarted, stop the replica. # A new one with the correct version will be started later as part of the # normal scale-up process. elif replica.version.requires_actor_restart(self._target_state.version): code_version_changes += 1 # If the replica is still `STARTING`, we don't need to go through the # graceful stop period. graceful_stop = replica.actor_details.state == ReplicaState.RUNNING self._stop_replica(replica, graceful_stop=graceful_stop) replicas_changed = True # Otherwise, only lightweight options in deployment config is a mismatch, so # we update it dynamically without restarting the replica. elif replica.actor_details.state == ReplicaState.RUNNING: reconfigure_changes += 1 if replica.version.requires_long_poll_broadcast( self._target_state.version ): replicas_changed = True self._broadcasted_replicas_set_changed = True self._in_transition = True # Get current rank for the replica current_rank = self._rank_manager.get_replica_rank( replica.replica_id.unique_id ) actor_updating = replica.reconfigure( self._target_state.version, rank=current_rank ) if actor_updating: self._replicas.add(ReplicaState.UPDATING, replica) else: self._replicas.add(ReplicaState.RUNNING, replica) # We don't allow going from STARTING, PENDING_MIGRATION to UPDATING. else: self._replicas.add(replica.actor_details.state, replica) if code_version_changes > 0: logger.info( f"Stopping {code_version_changes} replicas of {self._id} " "with outdated versions." ) if reconfigure_changes > 0: logger.info( f"Updating {reconfigure_changes} replicas of {self._id} " "with outdated deployment configs." ) # Record user config lightweight update ServeUsageTag.USER_CONFIG_LIGHTWEIGHT_UPDATED.record("True") return replicas_changed def _check_and_stop_outdated_version_replicas(self) -> bool: """Stops replicas with outdated versions to implement rolling updates. This includes both explicit code version updates and changes to the user_config. Returns whether any replicas were stopped. """ # Short circuit if target replicas is 0 (the deployment is being # deleted) because this will be handled in the main loop. if self._target_state.target_num_replicas == 0: return False # We include STARTING and UPDATING replicas here # because if there are replicas still pending startup, we may as well # terminate them and start new version replicas instead. old_running_replicas = self._replicas.count( exclude_version=self._target_state.version, states=[ ReplicaState.STARTING, ReplicaState.UPDATING, ReplicaState.RUNNING, ], ) old_stopping_replicas = self._replicas.count( exclude_version=self._target_state.version, states=[ReplicaState.STOPPING] ) new_running_replicas = self._replicas.count( version=self._target_state.version, states=[ReplicaState.RUNNING] ) # If the deployment is currently scaling down, let the scale down # complete before doing a rolling update. if ( self._target_state.target_num_replicas < old_running_replicas + old_stopping_replicas ): return False # The number of replicas that are currently in transition between # an old version and the new version. Note that we cannot directly # count the number of stopping replicas because once replicas finish # stopping, they are removed from the data structure. pending_replicas = ( self._target_state.target_num_replicas - new_running_replicas - old_running_replicas ) # Maximum number of replicas that can be updating at any given time. # There should never be more than rollout_size old replicas stopping # or rollout_size new replicas starting. rolling_update_percentage = ( self._target_state.info.deployment_config.rolling_update_percentage ) rollout_size = max( int(rolling_update_percentage * self._target_state.target_num_replicas), 1 ) # For gang deployments, ensure rollout_size is at least a multiple of # gang_size so that we always stop and start complete gangs. if self._is_gang_deployment: gang_config = self.get_gang_config() gs = gang_config.gang_size rollout_size = max(rollout_size, gs) rollout_size = math.ceil(rollout_size / gs) * gs max_to_stop = max(rollout_size - pending_replicas, 0) return self._stop_or_update_outdated_version_replicas(max_to_stop) def scale_deployment_replicas( self, gang_placement_groups: Optional[ Dict[DeploymentID, GangReservationResult] ] = None, ) -> Tuple[List[ReplicaSchedulingRequest], DeploymentDownscaleRequest]: """Scale the given deployment to the number of replicas. Args: gang_placement_groups: Reserved gang placement groups. If this deployment uses gang scheduling and PGs were reserved, replicas will be scheduled onto these PGs. Returns: Tuple[List[ReplicaSchedulingRequest], DeploymentDownscaleRequest]: The scheduling requests for the new replicas and the downscale request. """ # Fast path: already at target count with all replicas at target # version — no scaling or version updates needed. if not self._in_transition: return [], None assert ( self._target_state.target_num_replicas >= 0 ), "Target number of replicas must be greater than or equal to 0." upscale = [] downscale = None self._check_and_stop_outdated_version_replicas() self.stop_deployment_actors_if_needed() # When deployment_actors are configured, start them and wait until ready # before creating replicas. Skip during deletion. deployment_actors_configs = self._get_deployment_actors_configs() if deployment_actors_configs and not self._target_state.deleting: self.start_deployment_actors() if not self.check_deployment_actors_ready(): # Deployment actors not ready yet; defer replica creation. return (upscale, downscale) delta_replicas = self._get_target_replica_delta() if delta_replicas == 0: return (upscale, downscale) elif delta_replicas > 0: to_add = delta_replicas upscale = self._get_upscale_replicas( to_add=to_add, gang_placement_groups=gang_placement_groups ) elif delta_replicas < 0: to_remove = -delta_replicas gang_id_by_replica = None replicas_by_gang_id = None if self._is_gang_deployment: gang_id_by_replica = self._gang_id_by_replica replicas_by_gang_id = self._replicas_by_gang_id removed_replicas = f"{to_remove} replica{'s' if to_remove > 1 else ''}" logger.info(f"Removing {removed_replicas} from {self._id}.") downscale = DeploymentDownscaleRequest( deployment_id=self._id, num_to_stop=to_remove, gang_id_by_replica=gang_id_by_replica, replicas_by_gang_id=replicas_by_gang_id, gang_size=self.get_gang_config().gang_size if self._is_gang_deployment else None, ) return upscale, downscale def _get_upscale_replicas( self, to_add: int, gang_placement_groups: Optional[ Dict[DeploymentID, GangReservationResult] ] = None, ) -> List[ReplicaSchedulingRequest]: """Add replicas for this deployment, using gang scheduling when configured.""" upscale = [] if to_add <= 0 or self._terminally_failed(): return upscale if not self._is_gang_deployment: return self._add_upscale_replicas(to_add) gang_reservation_result = ( gang_placement_groups.get(self._id) if gang_placement_groups else None ) return self._add_upscale_gang_replicas( self.get_gang_config(), gang_reservation_result ) def _add_upscale_replicas(self, to_add: int) -> List[ReplicaSchedulingRequest]: """Add replicas for deployments that adopt single-replica (non-gang) scheduling.""" upscale = [] logger.info(f"Adding {to_add} replica{'s' * (to_add > 1)} to {self._id}.") for _ in range(to_add): replica_id = ReplicaID(get_random_string(), deployment_id=self._id) new_deployment_replica = DeploymentReplica( replica_id, self._target_state.version, ) scheduling_request = new_deployment_replica.start( self._target_state.info, assign_rank_callback=self._rank_manager.assign_rank, ) upscale.append(scheduling_request) self._replicas.add(ReplicaState.STARTING, new_deployment_replica) return upscale def _add_upscale_gang_replicas( self, gang_config: GangSchedulingConfig, gang_reservation_result: Optional[GangReservationResult], ) -> List[ReplicaSchedulingRequest]: """Add replicas using gang scheduling with reserved placement groups. Args: gang_config: Gang scheduling configuration. gang_reservation_result: Gang reservation result with reserved PGs. Returns: List of ReplicaSchedulingRequests for the new replicas. """ upscale = [] if gang_reservation_result is None: logger.info( f"Gang placement group reservation was skipped for {self._id}. " "Will retry in the next reconciliation loop." ) return upscale if not gang_reservation_result.success: logger.error( f"Gang scheduling failed for {self._id}: {gang_reservation_result.error_message}. " "Skipping replica creation." ) self.record_replica_startup_failure( f"Gang scheduling failed: {gang_reservation_result.error_message} " "See Serve controller logs for more details." ) return upscale gang_pgs = gang_reservation_result.gang_pgs gang_ids = gang_reservation_result.gang_ids gang_pg_names = gang_reservation_result.gang_pg_names gang_size = gang_config.gang_size num_gangs = len(gang_pgs) replicas_to_add = num_gangs * gang_size # When per-replica PG bundles are defined, each replica occupies multiple # consecutive bundles in the gang PG. The actor for replica i is placed at # bundle i * bundles_per_replica. pg_bundles = self._target_state.info.replica_config.placement_group_bundles bundles_per_replica = len(pg_bundles) if pg_bundles else 1 logger.info( f"Adding {replicas_to_add} replica{'s' * (replicas_to_add > 1)} " f"to {self._id} using gang scheduling " f"(gang_size={gang_size}, {num_gangs} gang(s))." ) for gang_pg, gang_id, pg_name in zip(gang_pgs, gang_ids, gang_pg_names): member_replica_ids = [ ReplicaID(get_random_string(), deployment_id=self._id) for _ in range(gang_size) ] for bundle_index, replica_id in enumerate(member_replica_ids): gang_context = GangContext( gang_id=gang_id, rank=bundle_index, world_size=gang_size, member_replica_ids=[r.unique_id for r in member_replica_ids], pg_name=pg_name, ) new_deployment_replica = DeploymentReplica( replica_id, self._target_state.version, ) scheduling_request = new_deployment_replica.start( self._target_state.info, assign_rank_callback=self._rank_manager.assign_rank, gang_placement_group=gang_pg, gang_pg_index=bundle_index * bundles_per_replica, gang_context=gang_context, ) upscale.append(scheduling_request) self._replicas.add(ReplicaState.STARTING, new_deployment_replica) self._register_gang_replica(replica_id, gang_id) return upscale def check_curr_status(self) -> Tuple[bool, bool]: """Check the current deployment status. Checks the difference between the target vs. running replica count for the target version. This will update the current deployment status depending on the state of the replicas. Returns (deleted, any_replicas_recovering). """ # Fast path: deployment is in steady state — status is already # HEALTHY, all replicas are RUNNING at target version. Nothing to do. if not self._in_transition: return False, False target_version = self._target_state.version any_replicas_recovering = ( self._replicas.count(states=[ReplicaState.RECOVERING]) > 0 ) all_running_replica_cnt = self._replicas.count(states=[ReplicaState.RUNNING]) all_active_deployment_actors_cnt = self._deployment_actors.count( states=[ DeploymentActorState.RUNNING, DeploymentActorState.STARTING, DeploymentActorState.RECOVERING, ] ) running_at_target_version_replica_cnt = self._replicas.count( states=[ReplicaState.RUNNING], version=target_version ) # Got to make a call to complete current deploy() goal after # start failure threshold reached, while we might still have # pending replicas in current goal. if running_at_target_version_replica_cnt > 0: # At least one RUNNING replica at target state, partial # success; We can stop tracking constructor failures and # leave it to the controller to fully scale to target # number of replicas and only return as completed once # reached target replica count self._replica_has_started = True # Deployment-scoped actor failed after threshold exceeded (consistent # with replica startup: transition only when retries exhausted). elif self.deployment_actor_terminally_failed(): msg = self._deployment_actor_failed self._curr_status_info = self._curr_status_info.handle_transition( trigger=DeploymentStatusInternalTrigger.DEPLOYMENT_ACTOR_FAILED, message=( "The deployment failed to start deployment actors " f"{self._deployment_actor_retry_counter} times " "in a row. See controller logs for details. Error:\n" f"{msg}" ), ) return False, any_replicas_recovering elif self._replica_startup_failing(): self._curr_status_info = self._curr_status_info.handle_transition( trigger=DeploymentStatusInternalTrigger.REPLICA_STARTUP_FAILED, message=( "The deployment failed to start " f"{self._replica_constructor_retry_counter} times " "in a row. This may be due to a problem with its " "constructor or initial health check failing. See " "controller logs for details. Error:\n" f"{self._replica_constructor_error_msg}" ), ) return False, any_replicas_recovering # If we have pending ops, the current goal is *not* ready. if ( self._replicas.count( states=[ ReplicaState.STARTING, ReplicaState.UPDATING, ReplicaState.RECOVERING, ReplicaState.STOPPING, ] ) == 0 ): # Deletion can complete when no replicas and no deployment actors # (READY or STARTING) remain. if ( self._target_state.deleting and all_running_replica_cnt == 0 and all_active_deployment_actors_cnt == 0 ): return True, any_replicas_recovering if ( # not self._target_state.deleting is important to avoid transitioning to HEALTHY # when the deployment is being deleted. This can happen if all replicas are running at the target version, # but the deployment is still in the process of being deleted because the deployment actors are not yet # deleted. not self._target_state.deleting and ( self._target_state.target_num_replicas == running_at_target_version_replica_cnt and running_at_target_version_replica_cnt == all_running_replica_cnt ) # Stay in transition until deployment actors for obsolete code versions # are dropped (see stop_deployment_actors_if_needed); otherwise # scale_deployment_replicas would never run cleanup after _in_transition # is cleared. and not self._orphaned_deployment_actor_code_versions() # Require all deployment-scoped actors (target version) to exist so we # do not clear _in_transition while actors are missing after a health # recycle or similar. and self._deployment_actors_satisfied_for_target() ): self._curr_status_info = self._curr_status_info.handle_transition( trigger=DeploymentStatusInternalTrigger.HEALTHY ) self._replica_constructor_retry_counter = 0 # Deployment is in steady state: all replicas RUNNING at # target version, no pending operations. self._in_transition = False return False, any_replicas_recovering return False, any_replicas_recovering def _check_startup_replicas( self, original_state: ReplicaState, stop_on_slow: bool = False ) -> List[Tuple[DeploymentReplica, ReplicaStartupStatus]]: """ Common helper function for startup actions tracking and status transition: STARTING, UPDATING and RECOVERING. Args: original_state: The state replicas are transitioning out of. stop_on_slow: If we consider a replica failed upon observing it's slow to reach running state. Returns: The list of replicas considered slow, along with their startup status. """ slow_replicas = [] failed_gang_ids: Set[str] = set() for replica in self._replicas.pop(states=[original_state]): start_status, error_msg = replica.check_started() if start_status == ReplicaStartupStatus.SUCCEEDED: if original_state == ReplicaState.RECOVERING: # If the previous state was RECOVERING, that mean the replica # crashed and is now starting up again. We need to recover the rank # from the replica actor. The invariant is that the rank is assigned # during startup and before the replica is added to the replicas # data structure with RUNNING state. # Skip if the rank is already assigned (e.g., health-check failure # put the replica into RECOVERING without a controller crash, so the # rank was never released). replica_id = replica.replica_id.unique_id if not self._rank_manager.has_replica_rank(replica_id): self._rank_manager.recover_rank( replica_id, replica.actor_node_id, replica.rank ) # Register recovered gang replicas in the incremental # bookkeeping (newly created gang replicas are already # registered in _add_upscale_gang_replicas). if ( original_state == ReplicaState.RECOVERING and replica.gang_context is not None ): self._register_gang_replica( replica.replica_id, replica.gang_context.gang_id ) # This replica should be now be added to handle's replica # set. self._replicas.add(ReplicaState.RUNNING, replica) self._deployment_scheduler.on_replica_running( replica.replica_id, replica.actor_node_id ) # if replica version is the same as the target version, # we update the docs path and route patterns if replica.version == self._target_state.version: self._docs_path = replica.docs_path self._route_patterns = replica.route_patterns # Log the startup latency. e2e_replica_start_latency = time.time() - replica._start_time replica_startup_message = ( f"{replica.replica_id} started successfully " f"on node '{replica.actor_node_id}' after " f"{e2e_replica_start_latency:.1f}s (PID: {replica.actor_pid})." ) if replica.initialization_latency_s is not None: # This condition should always be True. The initialization # latency is only None before the replica has initialized. replica_startup_message += ( " Replica constructor, " "reconfigure method, and initial health check took " f"{replica.initialization_latency_s:.1f}s." ) logger.info(replica_startup_message, extra={"log_to_stderr": False}) # Record startup or reconfigure latency metrics. if original_state == ReplicaState.STARTING: # Record replica startup latency (end-to-end from creation to ready). # This includes the time taken from starting a node, scheduling the replica, # and the replica constructor. e2e_replica_start_latency_ms = e2e_replica_start_latency * 1000 self.replica_startup_latency_histogram.observe( e2e_replica_start_latency_ms ) # Record replica initialization latency. if replica.initialization_latency_s is not None: initialization_latency_ms = ( replica.initialization_latency_s * 1000 ) self.replica_initialization_latency_histogram.observe( initialization_latency_ms ) elif original_state == ReplicaState.UPDATING: # Record replica reconfigure latency. if replica.reconfigure_start_time is not None: reconfigure_latency_ms = ( time.time() - replica.reconfigure_start_time ) * 1000 self.replica_reconfigure_latency_histogram.observe( reconfigure_latency_ms ) elif start_status == ReplicaStartupStatus.FAILED: # Replica reconfigure (deploy / upgrade) failed. # When `check_ready()` flagged the replica as unrecoverable # (e.g., the previous controller crashed before assigning a # rank to it), don't bump the deploy failure counter -- the # underlying cause is controller-side, not user code, and a # fresh replica will be started in its place. The actor was # already killed in `check_ready()`, so force-stop to avoid # issuing a graceful shutdown RPC to a dead actor. We still # propagate gang failure tracking so siblings get cleaned up. if replica.unrecoverable: self._stop_replica(replica, graceful_stop=False) if replica.gang_context is not None: failed_gang_ids.add(replica.gang_context.gang_id) continue # For gang replicas, count the failure once per gang and not per replica so the # retry counter isn't inflated by gang_size on every cycle. if ( replica.gang_context is None or replica.gang_context.gang_id not in failed_gang_ids ): self.record_replica_startup_failure(error_msg) self._stop_replica(replica) # Track failed gang IDs for sibling cleanup below. if replica.gang_context is not None: failed_gang_ids.add(replica.gang_context.gang_id) elif start_status in [ ReplicaStartupStatus.PENDING_ALLOCATION, ReplicaStartupStatus.PENDING_INITIALIZATION, ]: is_slow = time.time() - replica._start_time > SLOW_STARTUP_WARNING_S if is_slow: slow_replicas.append((replica, start_status)) # Does it make sense to stop replicas in PENDING_ALLOCATION # state? if is_slow and stop_on_slow: self._stop_replica(replica, graceful_stop=False) else: self._replicas.add(original_state, replica) # If any gang member failed during startup, stop all other members of # that gang so partial gangs never exist. if failed_gang_ids: for state in {original_state, ReplicaState.RUNNING}: for replica in self._replicas.pop(states=[state]): if ( replica.gang_context is not None and replica.gang_context.gang_id in failed_gang_ids ): logger.info( f"Stopping {replica.replica_id} because a gang " f"member failed during startup " f"(gang_id={replica.gang_context.gang_id})." ) # Forcefully stop siblings to avoid partial gangs self._stop_replica(replica, graceful_stop=False) else: self._replicas.add(state, replica) return slow_replicas def record_replica_startup_failure(self, error_msg: str): """Record that a replica failed to start.""" # There is no need to record replica failures if the target is 0. if self._target_state.target_num_replicas == 0: return # Increase startup failure counter (may change _terminally_failed() # result, which affects broadcasted availability). self._replica_constructor_retry_counter += 1 self._broadcasted_replicas_set_changed = True self._in_transition = True self._replica_constructor_error_msg = error_msg # Update the deployment message only if replicas are failing during # the very first time the controller is trying to start replicas of # this version. retrying_msg = "" if not self._replica_has_started: remaining_retries = max( self._failed_to_start_threshold - self._replica_constructor_retry_counter, 0, ) retrying_msg = f" {remaining_retries} more time(s)" message = ( f"A replica failed to start with exception. Retrying{retrying_msg}. " f"Error:\n{error_msg}" ) self._curr_status_info = self._curr_status_info.update_message(message) def _set_health_gauge(self, replica_unique_id: str, value: int) -> None: """Set the health-check gauge for *replica_unique_id*, skipping the (expensive) Cython ``Gauge.set()`` call when the value hasn't changed and was recently reported. In large clusters this avoids O(num_replicas) redundant FFI calls on every control-loop iteration while still refreshing the metric often enough for Prometheus export. """ if not RAY_SERVE_CONTROLLER_METRICS_INCLUDE_HIGH_CARDINALITY_TAGS: return now = time.time() cached = self._health_gauge_cache.get(replica_unique_id) if ( cached is not None and cached[0] == value and (now - cached[1]) < RAY_SERVE_STATUS_GAUGE_REPORT_INTERVAL_S ): return self.health_check_gauge.set(value, tags={"replica": replica_unique_id}) self._health_gauge_cache[replica_unique_id] = (value, now) def _register_gang_replica(self, replica_id: ReplicaID, gang_id: str) -> None: """Register a replica in the gang membership bookkeeping.""" self._gang_id_by_replica[replica_id] = gang_id self._replicas_by_gang_id[gang_id].add(replica_id) def _unregister_gang_replica(self, replica_id: ReplicaID) -> None: """Remove a replica from the gang membership bookkeeping.""" gang_id = self._gang_id_by_replica.pop(replica_id, None) if gang_id is not None: members = self._replicas_by_gang_id.get(gang_id) if members is not None: members.discard(replica_id) if not members: self._replicas_by_gang_id.pop(gang_id, None) def _clear_health_gauge_cache(self, replica_unique_id: str) -> None: """Remove a replica from the health-gauge cache (after it has fully stopped and been removed from tracking).""" self._health_gauge_cache.pop(replica_unique_id, None) def stop_replicas(self, replicas_to_stop: Set[ReplicaID]) -> None: for replica in self._replicas.remove(replicas_to_stop): self._stop_replica(replica) def _stop_replica(self, replica: DeploymentReplica, graceful_stop=True): """Stop replica 1. Stop the replica. 2. Change the replica into stopping state. 3. Set the health replica stats to 0. """ logger.debug(f"Adding STOPPING to replica: {replica.replica_id}.") replica.stop(graceful=graceful_stop) self._replicas.add(ReplicaState.STOPPING, replica) self._deployment_scheduler.on_replica_stopping(replica.replica_id) if RAY_SERVE_CONTROLLER_METRICS_INCLUDE_HIGH_CARDINALITY_TAGS: self._set_health_gauge(replica.replica_id.unique_id, 0) else: self._last_health_check_healthy_replica_ids.discard( replica.replica_id.unique_id ) self.health_check_gauge.set( len(self._last_health_check_healthy_replica_ids) ) def _stop_replica_mark_unhealthy_if_target_version( self, replica: DeploymentReplica, graceful_stop: bool ): """Stop the replica and mark deployment as UNHEALTHY if the replica is the target version.""" self._stop_replica(replica, graceful_stop=graceful_stop) if replica.version == self._target_state.version: self._curr_status_info = self._curr_status_info.handle_transition( trigger=DeploymentStatusInternalTrigger.HEALTH_CHECK_FAILED, message="A replica's health check failed. This " "deployment will be UNHEALTHY until the replica " "recovers or a new deploy happens.", ) def _forcefully_stop_gang_replicas( self, healthy_replicas: List[DeploymentReplica], unhealthy_replicas: List[DeploymentReplica], ) -> Tuple[List[DeploymentReplica], List[DeploymentReplica]]: """Forcefully stop all replicas in gangs that have any unhealthy or missing member. Under the RESTART_GANG policy, when any replica in a gang fails its health check, every member of that gang, including healthy ones, must be torn down so the gang can be rescheduled atomically. A gang is also restarted when any of its expected members, listed in gang_context.member_replica_ids, are missing entirely from the replica manager. This handles the case where a gang member dies while the controller is down: on recovery, the dead member is never tracked, leaving an incomplete gang that would block upscaling (the replica deficit would not be divisible by gang_size). This differs from normal (single-replica scheduling) unhealthy-replica handling in two ways: 1. Healthy siblings are also force-stopped. 2. Force-stop is always used (graceful_stop=False), regardless of the FORCE_STOP_UNHEALTHY_REPLICAS setting. Args: healthy_replicas: A list of healthy replicas. unhealthy_replicas: A list of unhealthy replicas. Returns: A (remaining_healthy, remaining_unhealthy) tuple containing only replicas that follow single-replica scheduling logic. """ gang_ids_to_restart: Set[str] = { replica.gang_context.gang_id for replica in unhealthy_replicas if replica.gang_context is not None } # Detect incomplete gangs: gang members that are missing entirely from # the replica manager. The healthy/unhealthy lists were popped from # self._replicas in check_and_update_replicas, so we must include them # explicitly to get the full set of tracked replica IDs. all_tracked_replica_ids: Set[str] = ( {replica.replica_id.unique_id for replica in self._replicas.get()} | {replica.replica_id.unique_id for replica in healthy_replicas} | {replica.replica_id.unique_id for replica in unhealthy_replicas} ) for replica in healthy_replicas: gc = replica.gang_context if gc is None or gc.gang_id in gang_ids_to_restart: continue for member_id in gc.member_replica_ids: if member_id not in all_tracked_replica_ids: gang_ids_to_restart.add(gc.gang_id) break if len(gang_ids_to_restart) == 0: return healthy_replicas, unhealthy_replicas remaining_healthy: List[DeploymentReplica] = [] for replica in healthy_replicas: if ( replica.gang_context is not None and replica.gang_context.gang_id in gang_ids_to_restart ): logger.warning( f"Replica {replica.replica_id} belongs to gang " f"(gang_id={replica.gang_context.gang_id}) that has an " "unhealthy or missing member. Forcefully stopping it " "because RESTART_GANG runtime failure policy is enabled." ) self._stop_replica_mark_unhealthy_if_target_version(replica, False) else: remaining_healthy.append(replica) remaining_unhealthy: List[DeploymentReplica] = [] for replica in unhealthy_replicas: if ( replica.gang_context is not None and replica.gang_context.gang_id in gang_ids_to_restart ): logger.warning( f"Replica {replica.replica_id} failed health check, " "forcefully stopping it as part of gang restart " f"(gang_id={replica.gang_context.gang_id})." ) self._stop_replica_mark_unhealthy_if_target_version(replica, False) else: remaining_unhealthy.append(replica) return remaining_healthy, remaining_unhealthy def _record_health_check_metrics(self, replica) -> None: """Record health-check latency + failure metrics for one replica. Shared by the in-place path and the pop/re-add (gang) path in ``check_and_update_replicas``. """ if replica.last_health_check_latency_ms is not None: self.health_check_latency_histogram.observe( replica.last_health_check_latency_ms ) if replica.last_health_check_failed: if RAY_SERVE_CONTROLLER_METRICS_INCLUDE_HIGH_CARDINALITY_TAGS: self.health_check_failures_counter.inc( tags={"replica": replica.replica_id.unique_id} ) else: self.health_check_failures_counter.inc() def _process_healthy_replica(self, replica) -> None: """Set the health gauge and pull/broadcast routing stats for a healthy replica. Container re-bucketing differs between the two reconcile paths, so it is left to the caller. """ self._set_health_gauge(replica.replica_id.unique_id, 1) routing_stats = replica.pull_routing_stats() if routing_stats is not None and routing_stats != replica.routing_stats: self._broadcasted_replicas_set_changed = True replica.record_routing_stats(routing_stats) def _stop_unhealthy_replica(self, replica) -> None: """Log and stop a replica that failed its health check. The caller removes it from ``self._replicas`` first -- the pop/re-add path already popped it; the in-place path batch-removes the whole unhealthy set. """ logger.warning( f"Replica {replica.replica_id} failed health check, stopping it." ) graceful = not self.FORCE_STOP_UNHEALTHY_REPLICAS self._stop_replica_mark_unhealthy_if_target_version(replica, graceful) def check_and_update_replicas(self): """ Check current state of all DeploymentReplica being tracked, and compare with state container from previous update() cycle to see if any state transition happened. """ healthy_replicas: List[DeploymentReplica] = [] unhealthy_replicas: List[DeploymentReplica] = [] # Profile-guided: for the common non-gang case, iterate # RUNNING/PENDING_MIGRATION IN PLACE. Healthy replicas that stay in their state # bucket are never popped+re-added -> eliminates the O(num_replicas) container # churn on the control loop at scale. Gang deployments fall back to the # original pop/re-add path (their force-stop reshuffles the lists). if not self._is_gang_deployment: origin: List[ReplicaState] = [] pairs = [ (replica, st) for st in (ReplicaState.RUNNING, ReplicaState.PENDING_MIGRATION) for replica in self._replicas.get([st]) ] healths = [replica.check_health() for replica, _ in pairs] for (replica, st), is_healthy in zip(pairs, healths): self._record_health_check_metrics(replica) if is_healthy: healthy_replicas.append(replica) origin.append(st) else: unhealthy_replicas.append(replica) for replica, st in zip(healthy_replicas, origin): self._process_healthy_replica(replica) # Re-bucket a healthy replica only if its state changed -- avoiding the # pop/re-add churn is the whole point of the in-place path. # actor_details.state is only set via ReplicaStateContainer.add() and # check_health() never transitions state, so for RUNNING/PENDING_MIGRATION # this is a no-op today; kept as a defensive guard so the in-place path # stays behavior-identical to the pop/re-add path if that ever changes. if replica.actor_details.state != st: self._replicas.remove({replica.replica_id}) self._replicas.add(replica.actor_details.state, replica) # Batch-remove all unhealthy replicas in a single O(num_replicas) pass; # a per-replica remove() would be O(unhealthy * num_replicas) -> O(N^2) # during mass health-check failures (e.g. a node/AZ outage). self._replicas.remove( {replica.replica_id for replica in unhealthy_replicas} ) for replica in unhealthy_replicas: self._stop_unhealthy_replica(replica) else: for replica in self._replicas.pop( states=[ReplicaState.RUNNING, ReplicaState.PENDING_MIGRATION] ): is_healthy = replica.check_health() self._record_health_check_metrics(replica) if is_healthy: healthy_replicas.append(replica) else: unhealthy_replicas.append(replica) # Under the RESTART_GANG policy, force-stop all members of any gang that has at # least one unhealthy replica. Replicas handled here are removed from the lists; # remaining replicas continue to respect FORCE_STOP_UNHEALTHY_REPLICAS. if ( self._is_gang_deployment and self.get_gang_config().runtime_failure_policy == GangRuntimeFailurePolicy.RESTART_GANG ): ( healthy_replicas, unhealthy_replicas, ) = self._forcefully_stop_gang_replicas( healthy_replicas, unhealthy_replicas ) for replica in healthy_replicas: self._replicas.add(replica.actor_details.state, replica) self._process_healthy_replica(replica) # Only single-replica scheduling replicas remain. for replica in unhealthy_replicas: self._stop_unhealthy_replica(replica) # In steady state there are no STARTING/UPDATING/RECOVERING/STOPPING # replicas, so skip startup/stopping checks. The rank consistency # check below still runs (it has its own lightweight guard). if self._in_transition: self._check_and_update_transitioning_replicas() if not RAY_SERVE_CONTROLLER_METRICS_INCLUDE_HIGH_CARDINALITY_TAGS: # When the replica tag is disabled, this is a single # deployment/application series. Emit the count of replicas that # passed health checks in this iteration so newly promoted replicas # are not counted before their first successful health check. self._last_health_check_healthy_replica_ids = { replica.replica_id.unique_id for replica in healthy_replicas } self.health_check_gauge.set( len(self._last_health_check_healthy_replica_ids) ) # After replica state updates, check rank consistency and perform minimal reassignment if needed # This ensures ranks are continuous after lifecycle events # Only do consistency check when deployment is stable (not during active updates) # maybe this constraint need to be relaxed in the future. The implication is that # if we delay the rank reassignment, the rank system will be in an invalid state # for a longer period of time. Abrar made this decision because he is not confident # about how rollouts work in the deployment state machine. active_replicas = self._replicas.get() if ( active_replicas and self._curr_status_info.status == DeploymentStatus.HEALTHY # Skip consistency check if there are STARTING replicas. During node # migration, new replicas are created in STARTING state (without ranks) # after the status is set to HEALTHY. Running the consistency check # with STARTING replicas causes "active keys without ranks" error. and self._replicas.count(states=[ReplicaState.STARTING]) == 0 ): replicas_to_reconfigure = ( self._rank_manager.check_rank_consistency_and_reassign_minimally( active_replicas, ) ) # Reconfigure replicas that had their ranks reassigned self._reconfigure_replicas_with_new_ranks(replicas_to_reconfigure) def _handle_deployment_actor_failed_health_check( self, wrapper: DeploymentActorWrapper, *, actor_name: str, ) -> None: """Stop an actor after failed health polling; separate from startup retries. Matches replica semantics in ``_stop_replica_mark_unhealthy_if_target_version``: target-version failures mark the deployment UNHEALTHY and set ``_in_transition`` so ``start_deployment_actors`` can recreate; older code versions are only stopped (no ``_deployment_actor_retry_counter``). """ detail = ( f"Deployment actor '{actor_name}' failed health checks " f"({DEPLOYMENT_ACTOR_HEALTH_CHECK_UNHEALTHY_THRESHOLD} consecutive " "failures or actor crash); recreating. " "Replicas should call serve.get_deployment_actor() again if they " "cached a stale ActorHandle." ) logger.warning(f"{detail} deployment_id={self._id}") wrapper.kill() target_version = self._target_state.version if ( target_version is not None and wrapper.code_version == target_version.code_version ): self._in_transition = True self._curr_status_info = self._curr_status_info.handle_transition( trigger=DeploymentStatusInternalTrigger.HEALTH_CHECK_FAILED, message=( "A deployment actor's health check failed. This deployment will be " "UNHEALTHY until the actor is recreated or a new deploy happens." ), ) def check_and_update_deployment_actors(self) -> None: """Poll all RUNNING deployment-scoped actors (every code version), like replicas. Failed health checks kill the actor. Target-version actors are recreated via ``start_deployment_actors`` without consuming ``_deployment_actor_retry_counter``. """ if self._target_state.deleting: return if not self._get_deployment_actors_configs(): return if self.deployment_actor_terminally_failed(): return removed = self._deployment_actors.pop( states=[DeploymentActorState.RUNNING], ) for _, entry in removed: wrapper = entry.wrapper if wrapper.check_health(): self._deployment_actors.add(DeploymentActorState.RUNNING, wrapper) else: self._handle_deployment_actor_failed_health_check( wrapper, actor_name=wrapper.actor_logical_name, ) def _check_and_update_transitioning_replicas(self): """Check STARTING/UPDATING/RECOVERING/STOPPING replicas for state transitions.""" slow_start_replicas = [] slow_start = self._check_startup_replicas(ReplicaState.STARTING) slow_update = self._check_startup_replicas(ReplicaState.UPDATING) slow_recover = self._check_startup_replicas( ReplicaState.RECOVERING, stop_on_slow=True ) slow_start_replicas = slow_start + slow_update + slow_recover if ( len(slow_start_replicas) and time.time() - self._prev_startup_warning > SLOW_STARTUP_WARNING_PERIOD_S ): pending_allocation = [] pending_initialization = [] for replica, startup_status in slow_start_replicas: if startup_status == ReplicaStartupStatus.PENDING_ALLOCATION: pending_allocation.append(replica) if startup_status == ReplicaStartupStatus.PENDING_INITIALIZATION: pending_initialization.append(replica) if len(pending_allocation) > 0: required, available = pending_allocation[0].resource_requirements() message = ( f"Deployment '{self.deployment_name}' in application " f"'{self.app_name}' has {len(pending_allocation)} replicas that " f"have taken more than {SLOW_STARTUP_WARNING_S}s to be scheduled. " "This may be due to waiting for the cluster to auto-scale or for a " "runtime environment to be installed. " f"Resources required for each replica: {required}, " f"total resources available: {available}. " "Use `ray status` for more details." ) logger.warning(message) if _SCALING_LOG_ENABLED: print_verbose_scaling_log() # If status is UNHEALTHY, leave the status and message as is. # The issue that caused the deployment to be unhealthy should be # prioritized over this resource availability issue. if self._curr_status_info.status not in [ DeploymentStatus.UNHEALTHY, DeploymentStatus.DEPLOY_FAILED, ]: self._curr_status_info = self._curr_status_info.update_message( message ) if len(pending_initialization) > 0: message = ( f"Deployment '{self.deployment_name}' in application " f"'{self.app_name}' has {len(pending_initialization)} replicas " f"that have taken more than {SLOW_STARTUP_WARNING_S}s to " "initialize.\n" "This may be caused by a slow __init__ or reconfigure method." ) logger.warning(message) # If status is UNHEALTHY, leave the status and message as is. # The issue that caused the deployment to be unhealthy should be # prioritized over this resource availability issue. if self._curr_status_info.status not in [ DeploymentStatus.UNHEALTHY, DeploymentStatus.DEPLOY_FAILED, ]: self._curr_status_info = self._curr_status_info.update_message( message ) self._prev_startup_warning = time.time() for replica in self._replicas.pop(states=[ReplicaState.STOPPING]): stopped = replica.check_stopped() if not stopped: self._replicas.add(ReplicaState.STOPPING, replica) else: logger.info(f"{replica.replica_id} is stopped.") # Retain replicas that allocated a log file so the dashboard can # still show their logs after the actor is gone. if replica.actor_details.log_file_path is not None: self._recent_dead_replicas.append( replica.actor_details.model_copy( update={"state": ReplicaState.STOPPED} ) ) # Record shutdown duration metric. if replica.shutdown_start_time is not None: shutdown_duration_ms = ( time.time() - replica.shutdown_start_time ) * 1000 self.replica_shutdown_duration_histogram.observe( shutdown_duration_ms ) # Release rank only after replica is successfully stopped # This ensures rank is available during draining/graceful shutdown replica_id = replica.replica_id.unique_id self._clear_health_gauge_cache(replica_id) if self._rank_manager.has_replica_rank(replica_id): # Only release rank if assigned. Replicas that failed allocation # or never reached RUNNING state won't have ranks. self._rank_manager.release_rank(replica_id) logger.debug( f"Released rank from replica {replica_id} in deployment {self._id}" ) self._autoscaling_state_manager.on_replica_stopped(replica.replica_id) self._unregister_gang_replica(replica.replica_id) def _reconfigure_replicas_with_new_ranks( self, replicas_to_reconfigure: List["DeploymentReplica"] ): """Reconfigure replicas with their new ranks after reassignment. This uses the reconfigure() mechanism to update replicas with their new ranks. """ if not replicas_to_reconfigure: return logger.debug( f"Reconfiguring {len(replicas_to_reconfigure)} replicas with rank changes in deployment {self._id}" ) updated_count = 0 for replica in replicas_to_reconfigure: replica_id = replica.replica_id.unique_id new_rank = self._rank_manager.get_replica_rank(replica_id) # Use reconfigure() to update rank # World size is calculated automatically from deployment config _ = replica.reconfigure( self._target_state.version, rank=new_rank, ) updated_count += 1 logger.debug( f"Successfully reconfigured {updated_count} replicas with new ranks in deployment {self._id}" ) def _get_replica_ranks_mapping(self) -> Dict[str, ReplicaRank]: """Get the current mapping of replica IDs to ReplicaRank objects. Returns: Dictionary mapping replica_id to ReplicaRank object (with rank, node_rank, local_rank). """ return self._rank_manager.get_replica_ranks_mapping() @staticmethod def _group_effective_deadline( group: List[DeploymentReplica], deadlines: Dict[str, int], ) -> float: """Return the effective deadline for a group of replicas. Uses the earliest deadline among members on draining nodes, and falls back to infinity if no member is on a draining node. """ member_deadlines = [ deadlines[r.actor_node_id] for r in group if r.actor_node_id in deadlines ] return min(member_deadlines) if member_deadlines else float("inf") @staticmethod def _group_shutdown_timeout_ms( group: List[DeploymentReplica], ) -> float: """Return the graceful shutdown timeout (in ms) for the group.""" return group[0]._actor.graceful_shutdown_timeout_s * 1000 def _choose_pending_migration_replicas_to_stop( self, replicas: List[DeploymentReplica], deadlines: Dict[str, int], min_replicas_to_stop: int, ) -> Tuple[List[DeploymentReplica], List[DeploymentReplica]]: """Returns a partition of replicas to stop and to keep. Each replica is treated as an independent unit. Args: replicas: The current list of replicas pending migration. deadlines: The current draining node deadlines. min_replicas_to_stop: The minimum number of replicas to stop. Returns: A tuple ``(replicas_to_stop, replicas_to_keep)``. """ # Treat each replica as a group of one. groups = [[r] for r in replicas] return self._partition_groups_to_stop(groups, deadlines, min_replicas_to_stop) def _group_replicas_by_gang_id( self, replicas: List[DeploymentReplica] ) -> Dict[str, List[DeploymentReplica]]: """Group replicas by their gang_id.""" gangs: Dict[str, List[DeploymentReplica]] = defaultdict(list) for replica in replicas: gangs[replica.gang_context.gang_id].append(replica) return gangs def _choose_pending_migration_gangs_to_stop( self, replicas: List[DeploymentReplica], deadlines: Dict[str, int], min_replicas_to_stop: int, ) -> Tuple[List[DeploymentReplica], List[DeploymentReplica]]: """Gang-aware variant: stop complete gangs atomically. A gang is considered deadline-expired if ANY member's deadline is up. For excess stopping, gangs are sorted by their earliest member deadline. """ gangs = self._group_replicas_by_gang_id(replicas) return self._partition_groups_to_stop( list(gangs.values()), deadlines, min_replicas_to_stop ) def _partition_groups_to_stop( self, groups: List[List[DeploymentReplica]], deadlines: Dict[str, int], min_replicas_to_stop: int, ) -> Tuple[List[DeploymentReplica], List[DeploymentReplica]]: """Partition replica groups into those to stop and those to keep. A group (single replica or full gang) is the atomic unit of stopping. 1. Groups whose deadline is up are stopped unconditionally. 2. Remaining groups are stopped greedily (earliest deadline first) until min_replicas_to_stop is satisfied. """ to_stop: List[DeploymentReplica] = [] remaining_groups: List[Tuple[float, List[DeploymentReplica]]] = [] curr_timestamp_ms = time.time() * 1000 for group in groups: effective_deadline = self._group_effective_deadline(group, deadlines) timeout_ms = self._group_shutdown_timeout_ms(group) if curr_timestamp_ms >= effective_deadline - timeout_ms: to_stop.extend(group) else: remaining_groups.append((effective_deadline, group)) # Stop excess groups, earliest deadline first. # NOTE: num_excess can be negative when deadline-forced stops in the # loop above already exceed min_replicas_to_stop. That's fine — no # extra groups are stopped because of the guard num_excess >= len(group). remaining_groups.sort(key=lambda x: x[0]) num_excess = min_replicas_to_stop - len(to_stop) remaining: List[DeploymentReplica] = [] for _, group in remaining_groups: if num_excess >= len(group): to_stop.extend(group) num_excess -= len(group) else: remaining.extend(group) return to_stop, remaining def migrate_replicas_on_draining_nodes(self, draining_nodes: Dict[str, int]): # Fast path: no draining nodes and deployment is in steady state — # no PENDING_MIGRATION replicas to move back and no replicas to # migrate, so skip the O(N) pop-and-readd. if not draining_nodes and not self._in_transition: return # Move replicas back to RUNNING if they are no longer on a draining node. # If this causes the number of replicas to exceed the target state, # they will be scaled down because `scale_deployment_replicas` is called on # each deployment after this. pending_migration_replicas = self._replicas.pop( states=[ReplicaState.PENDING_MIGRATION] ) if self._is_gang_deployment: # For gangs, only move back to RUNNING if ALL members' nodes are no # longer draining. gangs = self._group_replicas_by_gang_id(pending_migration_replicas) gangs_still_draining: Set[str] = set() for gang_id, gang_replicas in gangs.items(): if any( replica.actor_node_id in draining_nodes for replica in gang_replicas ): gangs_still_draining.add(gang_id) def still_draining(r): return r.gang_context.gang_id in gangs_still_draining else: def still_draining(r): return r.actor_node_id in draining_nodes for replica in pending_migration_replicas: if still_draining(replica): self._replicas.add(ReplicaState.PENDING_MIGRATION, replica) else: self._replicas.add(ReplicaState.RUNNING, replica) # Migrate replicas on draining nodes all_replicas = self._replicas.pop( states=[ ReplicaState.UPDATING, ReplicaState.RUNNING, ReplicaState.STARTING, ] ) if self._is_gang_deployment: # For gangs, if ANY member is on a draining node the entire gang migrates. gangs_to_migrate: Set[str] = set() for replica in all_replicas: if replica.actor_node_id in draining_nodes: gangs_to_migrate.add(replica.gang_context.gang_id) def needs_migration(r): return r.gang_context.gang_id in gangs_to_migrate else: def needs_migration(r): return r.actor_node_id in draining_nodes for replica in all_replicas: if not needs_migration(replica): self._replicas.add(replica.actor_details.state, replica) # For RUNNING replicas, migrate them safely by starting # a replacement replica first. elif replica.actor_details.state == ReplicaState.RUNNING: logger.info( f"Migrating {replica.replica_id} from draining node " f"'{replica.actor_node_id}'. A new replica will be " "created on another node." ) self._replicas.add(ReplicaState.PENDING_MIGRATION, replica) # For replicas that are STARTING or UPDATING, might as # well terminate them immediately to allow replacement # replicas to start. Otherwise we need to wait for them # to transition to RUNNING before starting migration. else: self._stop_replica( replica, # Always force-stop gang members to avoid leaving partial gangs. graceful_stop=not self._is_gang_deployment, ) num_running = self._replicas.count(states=[ReplicaState.RUNNING]) num_draining = self._replicas.count(states=[ReplicaState.PENDING_MIGRATION]) num_pending_migration_replicas_to_stop = ( num_running + num_draining - self._target_state.target_num_replicas ) choose_pending_migration_to_stop_fn = ( self._choose_pending_migration_gangs_to_stop if self._is_gang_deployment else self._choose_pending_migration_replicas_to_stop ) replicas_to_stop, replicas_to_keep = choose_pending_migration_to_stop_fn( self._replicas.pop(states=[ReplicaState.PENDING_MIGRATION]), draining_nodes, num_pending_migration_replicas_to_stop, ) for replica in replicas_to_stop: logger.info( f"Stopping {replica.replica_id} " f"on draining node {replica.actor_node_id}." ) self._stop_replica(replica, graceful_stop=True) for replica in replicas_to_keep: self._replicas.add(ReplicaState.PENDING_MIGRATION, replica) def record_request_routing_info(self, info: RequestRoutingInfo) -> None: """Records the multiplexed model IDs of a replica. Args: info: RequestRoutingInfo including deployment name, replica tag, multiplex model ids, and routing stats. """ # O(1) lookup via replica_id index. replica = self._replicas.get_by_id(info.replica_id) if replica is not None: if info.multiplexed_model_ids is not None: replica.record_multiplexed_model_ids(info.multiplexed_model_ids) if info.routing_stats is not None: replica.record_routing_stats(info.routing_stats) self._request_routing_info_updated = True else: logger.warning(f"{info.replica_id} not found.") def _stop_one_running_replica_for_testing(self): running_replicas = self._replicas.pop(states=[ReplicaState.RUNNING]) replica_to_stop = running_replicas.pop() replica_to_stop.stop(graceful=False) self._replicas.add(ReplicaState.STOPPING, replica_to_stop) for replica in running_replicas: self._replicas.add(ReplicaState.RUNNING, replica) def is_ingress(self) -> bool: return self._target_state.info.ingress def is_ingress_request_router(self) -> bool: return self._target_state.info.ingress_request_router def get_outbound_deployments(self) -> Optional[List[DeploymentID]]: """Get the outbound deployments. Returns: Sorted list of deployment IDs that this deployment calls. None if outbound deployments are not yet polled. """ result: Set[DeploymentID] = set() has_outbound_deployments = False for replica in self._replicas.get([ReplicaState.RUNNING]): if replica.version != self._target_state.version: # Only consider replicas of the target version continue outbound_deployments = replica.get_outbound_deployments() if outbound_deployments is not None: result.update(outbound_deployments) has_outbound_deployments = True if not has_outbound_deployments: return None return sorted(result, key=lambda d: (d.name)) def deployment_actor_terminally_failed(self) -> bool: """True when deployment actors have failed too many times to keep retrying.""" if self._target_state.deleting: return False deployment_actors_configs = self._get_deployment_actors_configs() if not deployment_actors_configs: return False return ( self._deployment_actor_retry_counter >= self._deployment_actor_failed_to_start_threshold ) def record_deployment_actor_startup_failure(self, error_msg: str) -> None: """Record constructor/start failure for a deployment actor (not health checks). Increments ``_deployment_actor_retry_counter`` toward DEPLOY_FAILED, like ``record_replica_startup_failure``. Health-driven kills use ``_handle_deployment_actor_failed_health_check`` instead so rolling updates are not coupled to startup retries. """ deployment_actors_configs = self._get_deployment_actors_configs() if not deployment_actors_configs: return self._deployment_actor_retry_counter += 1 self._deployment_actor_failed = error_msg self._broadcasted_replicas_set_changed = True self._in_transition = True remaining_retries = max( self._deployment_actor_failed_to_start_threshold - self._deployment_actor_retry_counter, 0, ) retrying_msg = f" {remaining_retries} more time(s)" if remaining_retries else "" message = ( f"A deployment actor failed to start. Retrying{retrying_msg}. " f"Error:\n{error_msg}" ) self._curr_status_info = self._curr_status_info.update_message(message) def start_deployment_actors(self) -> None: """Start deployment-scoped actors for the target version. Initiates creation if not already started or ready. On failure, records `_deployment_actor_failed` for status transition. Stops retrying after `_deployment_actor_failed_to_start_threshold` consecutive failures. """ if self._target_state.deleting: return version = self._target_state.version if version is None: return deployment_actors_configs = self._get_deployment_actors_configs(version) if not deployment_actors_configs: return code_ver = version.code_version if self.deployment_actor_terminally_failed(): return deployment_runtime_env = (version.ray_actor_options or {}).get( "runtime_env", {} ) # Create only missing actors (supports partial recovery after controller restart). configs_to_create = [ cfg for cfg in deployment_actors_configs if self._deployment_actors.get_wrapper(code_ver, cfg.name) is None ] if not configs_to_create: return logger.info( f"Creating {len(configs_to_create)} deployment actor(s) for {self._id}" ) for cfg in configs_to_create: wrapper = DeploymentActorWrapper( deployment_id=self._id, config=cfg, code_version=code_ver, ) started_successfully, error_msg = wrapper.start(deployment_runtime_env) if started_successfully is False: logger.warning( f"Deployment actor creation failed for {self._id}: {error_msg} " f"(attempt {self._deployment_actor_retry_counter + 1}/" f"{self._deployment_actor_failed_to_start_threshold})" ) self.record_deployment_actor_startup_failure(error_msg) return self._deployment_actors.add(DeploymentActorState.STARTING, wrapper) return def check_deployment_actors_ready(self) -> bool: """Check if deployment-scoped actors are ready for the target version. Polls pending refs without blocking. Returns True when all configured actors are ready. Sets _deployment_actor_failed and returns False when an actor constructor fails. """ version = self._target_state.version if version is None: return True deployment_actors_configs = self._get_deployment_actors_configs(version) if not deployment_actors_configs: return True code_ver = version.code_version ready_count = self._deployment_actors.count( code_ver, states=[DeploymentActorState.RUNNING] ) pending_count = self._deployment_actors.count( code_ver, states=[DeploymentActorState.STARTING, DeploymentActorState.RECOVERING], ) if ready_count == len(deployment_actors_configs) and pending_count == 0: # Align with the counter reset when we promote the last pending actor # below; otherwise a health-recreate can leave the counter elevated while # all actors are already RUNNING on the next tick. self._deployment_actor_retry_counter = 0 return True pending_wrappers = self._deployment_actors.get( code_ver, states=[DeploymentActorState.STARTING, DeploymentActorState.RECOVERING], ) if not pending_wrappers: return False not_ready_wrappers = [] for wrapper in pending_wrappers: ready, error_msg = wrapper.check_ready() if error_msg is not None: logger.warning( f"Deployment actor '{wrapper.actor_logical_name}' for {self._id} " f"failed to become ready: {error_msg}" ) self.record_deployment_actor_startup_failure(error_msg) self._deployment_actors.pop( code_ver, states=[ DeploymentActorState.STARTING, DeploymentActorState.RECOVERING, ], ) return False if ready: self._deployment_actors.add(DeploymentActorState.RUNNING, wrapper) wrapper.reset_health_state_after_running() else: not_ready_wrappers.append(wrapper) if not_ready_wrappers: return False # Verify total RUNNING count equals configured count. When # start_deployment_actors fails partway through, some actors are # never added; we must not return True or reset retry counter. actual_running = self._deployment_actors.count( code_ver, states=[DeploymentActorState.RUNNING] ) if actual_running == len(deployment_actors_configs): self._deployment_actor_retry_counter = 0 return True return False def _orphaned_deployment_actor_code_versions(self) -> Set[str]: """Code versions still tracked for deployment actors that no replica needs. Matches the retention rule in ``stop_deployment_actors_if_needed``: keep actors for every replica's ``code_version``, and (when not deleting) for the target deployment version. """ target_version = self._target_state.version if target_version is None: return set() # Fast path: with no deployment-scoped actors tracked, nothing can be orphaned # (get_code_versions() is empty, and empty - anything == empty). Skips the O(N) # self._replicas.get() materialization on every scale_deployment_replicas tick # for the common deployment that has no deployment-scoped actors. if self._deployment_actors.is_empty(): return set() versions_to_keep = {r.version.code_version for r in self._replicas.get()} if not self._target_state.deleting: versions_to_keep.add(target_version.code_version) return self._deployment_actors.get_code_versions() - versions_to_keep def stop_deployment_actors_if_needed(self) -> None: """Stop deployment-scoped actors when no longer needed. During normal operation, keeps actors for versions that have replicas in any state (STARTING, UPDATING, RECOVERING, RUNNING, STOPPING, PENDING_MIGRATION), since all of these may need deployment actors. During deletion, actors are kept only while replicas still exist. """ if self._target_state.version is None: return wrappers_to_stop: List[DeploymentActorWrapper] = [] for code_version in self._orphaned_deployment_actor_code_versions(): entries = self._deployment_actors.pop( code_version=code_version, states=[ DeploymentActorState.RUNNING, DeploymentActorState.STARTING, DeploymentActorState.RECOVERING, ], ) wrappers_to_stop.extend(entry.wrapper for _, entry in entries) self._stop_deployment_actor_wrappers(wrappers_to_stop) def _stop_deployment_actor_wrappers( self, wrappers: List[DeploymentActorWrapper] ) -> None: """Best-effort stop for deployment-scoped actor wrappers.""" for wrapper in wrappers: logger.info( f"Stopping deployment actor '{wrapper.actor_logical_name}' " f"for {self._id} code_version={wrapper.code_version}" ) wrapper.kill() class DeploymentStateManager: """Manages all state for deployments in the system. This class is *not* thread safe, so any state-modifying methods should be called with a lock held. """ def __init__( self, kv_store: KVStoreBase, long_poll_host: LongPollHost, all_current_actor_names: List[str], all_current_placement_group_names: List[str], cluster_node_info_cache: ClusterNodeInfoCache, autoscaling_state_manager: AutoscalingStateManager, head_node_id_override: Optional[str] = None, create_placement_group_fn_override: Optional[Callable] = None, ): self._kv_store = kv_store self._long_poll_host = long_poll_host self._cluster_node_info_cache = cluster_node_info_cache self._deployment_scheduler = default_impl.create_deployment_scheduler( cluster_node_info_cache, head_node_id_override, create_placement_group_fn_override, ) self._autoscaling_state_manager = autoscaling_state_manager self._shutting_down = False self._deployment_states: Dict[DeploymentID, DeploymentState] = {} self._app_deployment_mapping: Dict[str, Set[str]] = defaultdict(set) # Metric for tracking deployment status self._deployment_status_gauge = ray_metrics.Gauge( "serve_deployment_status", description=( "Numeric status of deployment. " "0=UNKNOWN, 1=DEPLOY_FAILED, 2=UNHEALTHY, 3=UPDATING, " "4=UPSCALING, 5=DOWNSCALING, 6=HEALTHY." ), tag_keys=("deployment", "application"), ) self._recover_from_checkpoint( all_current_actor_names, all_current_placement_group_names ) def _create_deployment_state(self, deployment_id): self._deployment_scheduler.on_deployment_created( deployment_id, SpreadDeploymentSchedulingPolicy() ) # Evict any stale long-poll snapshots for this deployment_id from a # prior deletion. The delete path tombstones DEPLOYMENT_TARGETS # (is_available=False) so existing handles fail fast, but the # tombstone persists in LongPollHost. A re-created DeploymentState's # _last_broadcasted_* defaults match its own (True, []) state, so # broadcast_running_replicas_if_changed short-circuits and never # overwrites the tombstone — freshly-subscribed routers would then # see is_available=False and reject every request. self._long_poll_host.remove_keys( [ (LongPollNamespace.DEPLOYMENT_TARGETS, deployment_id), (LongPollNamespace.DEPLOYMENT_TARGETS, deployment_id.name), (LongPollNamespace.DEPLOYMENT_CONFIG, deployment_id), ] ) return DeploymentState( deployment_id, self._long_poll_host, self._deployment_scheduler, self._cluster_node_info_cache, self._autoscaling_state_manager, ) def _map_actor_names_to_deployment( self, all_current_actor_names: List[str] ) -> Dict[str, List[str]]: """ Given a list of all actor names queried from current ray cluster, map them to corresponding deployments. Example: Args: [A#zxc123, B#xcv234, A#qwe234] Returns: { A: [A#zxc123, A#qwe234] B: [B#xcv234] } Args: all_current_actor_names: Actor names currently registered with Ray. Returns: A mapping from deployment ID to the list of replica actor names associated with that deployment. """ all_replica_names = [ actor_name for actor_name in all_current_actor_names if ReplicaID.is_full_id_str(actor_name) ] deployment_to_current_replicas = defaultdict(list) if len(all_replica_names) > 0: for replica_name in all_replica_names: replica_id = ReplicaID.from_full_id_str(replica_name) deployment_to_current_replicas[replica_id.deployment_id].append( replica_name ) return deployment_to_current_replicas def _detect_and_remove_leaked_placement_groups( self, all_current_actor_names: List[str], all_current_placement_group_names: List[str], ): """Detect and remove any placement groups not associated with a replica. For per-replica PGs, a PG is leaked if no actor with that name exists. This can happen under certain rare circumstances: - The controller creates a placement group then crashes before creating the associated replica actor. - While the controller is down, a replica actor crashes but its placement group still exists. In both of these (or any other unknown cases), we simply need to remove the leaked placement groups. For gang PGs, a PG is leaked only if no alive actor references its placement group ID. Gang PGs that still have live actors are preserved to avoid releasing their resource reservations. """ leaked_pg_names = [] for pg_name in all_current_placement_group_names: if ( ReplicaID.is_full_id_str(pg_name) and pg_name not in all_current_actor_names ): leaked_pg_names.append(pg_name) gang_pg_names_in_cluster = [ name for name in all_current_placement_group_names if name.startswith(GANG_PG_NAME_PREFIX) ] if gang_pg_names_in_cluster: pg_table = ray.util.placement_group_table() gang_pg_name_to_id: Dict[str, str] = {} for pg_id_hex, entry in pg_table.items(): name = entry.get("name", "") if name.startswith(GANG_PG_NAME_PREFIX): gang_pg_name_to_id[name] = pg_id_hex occupied_pg_ids = get_active_placement_group_ids() for gang_pg_name in gang_pg_names_in_cluster: pg_id = gang_pg_name_to_id.get(gang_pg_name) if pg_id is not None and pg_id not in occupied_pg_ids: leaked_pg_names.append(gang_pg_name) if len(leaked_pg_names) > 0: logger.warning( f"Detected leaked placement groups: {leaked_pg_names}. " "The placement groups will be removed. This can happen in rare " "circumstances when the controller crashes and should not cause any " "issues. If this happens repeatedly, please file an issue on GitHub." ) for leaked_pg_name in leaked_pg_names: try: pg = ray.util.get_placement_group(leaked_pg_name) ray.util.remove_placement_group(pg) except Exception: logger.exception( f"Failed to remove leaked placement group {leaked_pg_name}." ) def _recover_from_checkpoint( self, all_current_actor_names: List[str], all_current_placement_group_names: List[str], ): """ Recover from checkpoint upon controller failure with all actor names found in current cluster. Each deployment resumes target state from checkpoint if available. For current state it will prioritize reconstructing from current actor names found that matches deployment tag if applicable. """ self._detect_and_remove_leaked_placement_groups( all_current_actor_names, all_current_placement_group_names, ) deployment_to_current_replicas = self._map_actor_names_to_deployment( all_current_actor_names ) checkpoint = self._kv_store.get(CHECKPOINT_KEY) if checkpoint is not None: deployment_state_info = cloudpickle.loads(checkpoint) for deployment_id, checkpoint_data in deployment_state_info.items(): deployment_state = self._create_deployment_state(deployment_id) deployment_state.recover_target_state_from_checkpoint(checkpoint_data) if len(deployment_to_current_replicas[deployment_id]) > 0: deployment_state.recover_current_state_from_replica_actor_names( # noqa: E501 deployment_to_current_replicas[deployment_id] ) self._deployment_states[deployment_id] = deployment_state self._app_deployment_mapping[deployment_id.app_name].add( deployment_id.name ) def shutdown(self): """ Shutdown all running replicas by notifying the controller, and leave it to the controller event loop to take actions afterwards. Once shutdown signal is received, it will also prevent any new deployments or replicas from being created. One can send multiple shutdown signals but won't effectively make any difference compare to calling it once. """ self._shutting_down = True for deployment_state in self._deployment_states.values(): deployment_state.delete() def is_ready_for_shutdown(self) -> bool: """Return whether all deployments are shutdown. Check there are no deployment states. """ return self._shutting_down and len(self._deployment_states) == 0 def delete_checkpoint(self) -> None: """Delete the deployment state checkpoint from KV store.""" self._kv_store.delete(CHECKPOINT_KEY) def save_checkpoint(self) -> None: """Write a checkpoint of all deployment states.""" if self._shutting_down: # Once we're told to shut down, stop writing checkpoints. # Calling .shutdown() deletes any existing checkpoint. return deployment_state_info = { deployment_id: deployment_state.get_checkpoint_data() for deployment_id, deployment_state in self._deployment_states.items() } self._kv_store.put( CHECKPOINT_KEY, cloudpickle.dumps(deployment_state_info), ) def get_running_replica_infos( self, ) -> Dict[DeploymentID, List[RunningReplicaInfo]]: return { id: deployment_state.get_running_replica_infos() for id, deployment_state in self._deployment_states.items() } def get_deployment_infos(self) -> Dict[DeploymentID, DeploymentInfo]: infos: Dict[DeploymentID, DeploymentInfo] = {} for deployment_id, deployment_state in self._deployment_states.items(): infos[deployment_id] = deployment_state.target_info return infos def get_deployment(self, deployment_id: DeploymentID) -> Optional[DeploymentInfo]: if deployment_id in self._deployment_states: return self._deployment_states[deployment_id].target_info else: return None def get_deployment_docs_path(self, deployment_id: DeploymentID) -> Optional[str]: if deployment_id in self._deployment_states: return self._deployment_states[deployment_id].docs_path def get_deployment_route_patterns( self, deployment_id: DeploymentID ) -> Optional[List[str]]: """Get route patterns for a deployment if available.""" if deployment_id in self._deployment_states: return self._deployment_states[deployment_id].route_patterns return None def get_deployment_target_num_replicas( self, deployment_id: DeploymentID ) -> Optional[int]: if deployment_id not in self._deployment_states: return None return self._deployment_states[deployment_id].target_num_replicas def get_deployment_details(self, id: DeploymentID) -> Optional[DeploymentDetails]: """Gets detailed info on a deployment. Args: id: The ID of the deployment to look up. Returns: DeploymentDetails: if the deployment is live. None: if the deployment is deleted. """ statuses = self.get_deployment_statuses([id]) if len(statuses) == 0: return None else: status_info = statuses[0] deployment_state = self._deployment_states[id] return DeploymentDetails( name=id.name, status=status_info.status, status_trigger=status_info.status_trigger, message=status_info.message, deployment_config=_deployment_info_to_schema( id.name, self.get_deployment(id) ), target_num_replicas=deployment_state._target_state.target_num_replicas, required_resources=deployment_state.target_info.replica_config.resource_dict, replicas=deployment_state.list_replica_details(), recent_dead_replicas=deployment_state.list_recent_dead_replicas(), ) def get_deployment_statuses( self, ids: Optional[List[DeploymentID]] = None ) -> List[DeploymentStatusInfo]: """ Return the statuses of the deployments with the given `ids`. If `ids` is `None`, returns the status of all deployments. """ if ids is None: # fast path for returning all deployments, # avoids checking `if ids is None` in a loop return [ state.curr_status_info for state in self._deployment_states.values() ] else: statuses = [] for id in ids: state = self._deployment_states.get(id) if state is not None: statuses.append(state.curr_status_info) return statuses def get_alive_replica_actor_ids(self) -> Set[str]: alive_replica_actor_ids = set() for ds in self._deployment_states.values(): alive_replica_actor_ids |= ds.get_alive_replica_actor_ids() return alive_replica_actor_ids def get_deployment_ids(self) -> List[DeploymentID]: return list(self._deployment_states.keys()) def get_node_id_to_alive_replica_ids(self) -> Dict[str, Set[str]]: node_id_to_alive_replica_ids = defaultdict(set) for deployment_state in self._deployment_states.values(): # Keep replicas that are alive even if they are not yet fully running so # ingress cleanup does not aggressively prune their allocated ports. for replica in deployment_state._replicas.get(): if replica.actor_node_id is not None: node_id_to_alive_replica_ids[replica.actor_node_id].add( replica.replica_id.unique_id ) return dict(node_id_to_alive_replica_ids) def _dump_replica_states_for_testing( self, deployment_id: DeploymentID ) -> ReplicaStateContainer: return self._deployment_states[deployment_id]._replicas def _stop_one_running_replica_for_testing(self, deployment_id: DeploymentID): self._deployment_states[deployment_id]._stop_one_running_replica_for_testing() def deploy( self, deployment_id: DeploymentID, deployment_info: DeploymentInfo, ) -> bool: """Deploy the deployment. If the deployment already exists with the same version and config, this is a no-op and returns False. Args: deployment_id: The ID of the deployment to apply. deployment_info: The target deployment info to apply. Returns: bool: Whether the target state has changed. """ if self._shutting_down: logger.warning( f"Ignoring deploy request for {deployment_id} " "because deployment state manager is shutting down." ) return False if deployment_id not in self._deployment_states: self._deployment_states[deployment_id] = self._create_deployment_state( deployment_id ) self._app_deployment_mapping[deployment_id.app_name].add(deployment_id.name) self._record_deployment_usage() return self._deployment_states[deployment_id].deploy(deployment_info) def get_deployments_in_application(self, app_name: str) -> List[str]: """Return list of deployment names in application.""" return list(self._app_deployment_mapping[app_name]) def delete_deployment(self, id: DeploymentID): # This method must be idempotent. We should validate that the # specified deployment exists on the client. if id in self._deployment_states: return self._deployment_states[id].delete() return False def _validate_deployment_state_for_num_replica_update( self, deployment_id: DeploymentID ): """Validate the state of a deployment for num replica update.""" statuses = self.get_deployment_statuses([deployment_id]) if statuses is None or len(statuses) == 0: raise ValueError(f"Deployment {deployment_id} not found") elif statuses[0].status_trigger == DeploymentStatusTrigger.DELETING: raise DeploymentIsBeingDeletedError( f"Deployment {deployment_id} is being deleted. Scaling operations are not allowed." ) def set_target_num_replicas( self, deployment_id: DeploymentID, target_num_replicas: int ): """Set target number of replicas for a deployment.""" if self._shutting_down: logger.warning( f"Ignoring set_target_num_replicas request for {deployment_id} " "because deployment state manager is shutting down." ) return self._validate_deployment_state_for_num_replica_update(deployment_id) deployment_state = self._deployment_states[deployment_id] if target_num_replicas != deployment_state.target_num_replicas: logger.info( f"Target number of replicas changed from {deployment_state.target_num_replicas} to {target_num_replicas} for deployment {deployment_id}" ) deployment_state.set_target_num_replicas(target_num_replicas) self.save_checkpoint() else: logger.info( f"Skipping updating target number of replicas as it did not change for deployment {deployment_id}" ) def update(self) -> bool: """Updates the state of all deployments to match their goal state. Returns True if any of the deployments have replicas in the RECOVERING state. """ deleted_ids = [] any_recovering = False upscales: Dict[DeploymentID, List[ReplicaSchedulingRequest]] = {} downscales: Dict[DeploymentID, DeploymentDownscaleRequest] = {} target_state_changed = False # STEP 1: Update current state for deployment_state in self._deployment_states.values(): deployment_state.check_and_update_replicas() deployment_state.check_and_update_deployment_actors() # STEP 2: Check current status for deployment_state in self._deployment_states.values(): deployment_state.check_curr_status() # STEP 3: Drain nodes draining_nodes = self._cluster_node_info_cache.get_draining_nodes() allow_new_compaction = len(draining_nodes) == 0 and all( ds.curr_status_info.status == DeploymentStatus.HEALTHY # TODO(zcin): Make sure that status should never be healthy if # the number of running replicas at target version is not at # target number, so we can remove this defensive check. and ds.get_num_running_replicas(ds.target_version) == ds.target_num_replicas # To be extra conservative, only actively compact if there # are no non-running replicas and ds._replicas.count() == ds.target_num_replicas for ds in self._deployment_states.values() ) if RAY_SERVE_USE_PACK_SCHEDULING_STRATEGY: # Tuple of target node to compact, and its draining deadline node_info: Optional[ Tuple[str, float] ] = self._deployment_scheduler.get_node_to_compact( allow_new_compaction=allow_new_compaction ) if node_info: target_node_id, deadline = node_info draining_nodes = {target_node_id: deadline} for deployment_id, deployment_state in self._deployment_states.items(): deployment_state.migrate_replicas_on_draining_nodes(draining_nodes) # STEP 3: Reserve gang placement groups gang_placement_groups = self._reserve_gang_placement_groups() # STEP 4: Scale replicas for deployment_id, deployment_state in self._deployment_states.items(): upscale, downscale = deployment_state.scale_deployment_replicas( gang_placement_groups=gang_placement_groups, ) if upscale: upscales[deployment_id] = upscale if downscale: downscales[deployment_id] = downscale # STEP 5: Update status for deployment_id, deployment_state in self._deployment_states.items(): deleted, any_replicas_recovering = deployment_state.check_curr_status() if deleted: deleted_ids.append(deployment_id) any_recovering |= any_replicas_recovering # STEP 6: Schedule all STARTING replicas and stop all STOPPING replicas # (Replicas are only added in scale_deployment_replicas when deployment # actors are ready, so no additional gate needed here.) deployment_to_replicas_to_stop = self._deployment_scheduler.schedule( upscales, downscales ) for deployment_id, replicas_to_stop in deployment_to_replicas_to_stop.items(): self._deployment_states[deployment_id].stop_replicas(replicas_to_stop) for deployment_id, scheduling_requests in upscales.items(): self._handle_scheduling_request_failures(deployment_id, scheduling_requests) # STEP 7: Broadcast long poll information for deployment_id, deployment_state in self._deployment_states.items(): deployment_state.broadcast_running_replicas_if_changed() deployment_state.broadcast_deployment_config_if_changed() if deployment_state.should_autoscale(): self._autoscaling_state_manager.update_running_replica_ids( deployment_id=deployment_id, running_replicas=deployment_state.get_running_replica_ids(), ) # STEP 8: Record deployment status metrics for deployment_id, deployment_state in self._deployment_states.items(): status = deployment_state.curr_status_info.status self._deployment_status_gauge.set( status.to_numeric(), tags={ "deployment": deployment_id.name, "application": deployment_id.app_name, }, ) # STEP 9: Cleanup for deployment_id in deleted_ids: self._deployment_scheduler.on_deployment_deleted(deployment_id) self._autoscaling_state_manager.deregister_deployment(deployment_id) del self._deployment_states[deployment_id] if ( deployment_id.app_name in self._app_deployment_mapping and deployment_id.name in self._app_deployment_mapping[deployment_id.app_name] ): self._app_deployment_mapping[deployment_id.app_name].remove( deployment_id.name ) # Clean up the app_name entry if no deployments are left if not self._app_deployment_mapping[deployment_id.app_name]: del self._app_deployment_mapping[deployment_id.app_name] # Tombstone TARGETS so routers fail fast on a deleted deployment. # Don't evict in the same tick: the waiter wakes after update() # returns and listen_for_change's guard would drop the payload. tombstone = DeploymentTargetInfo(is_available=False, running_replicas=[]) self._long_poll_host.notify_changed( { (LongPollNamespace.DEPLOYMENT_TARGETS, deployment_id): tombstone, ( LongPollNamespace.DEPLOYMENT_TARGETS, deployment_id.name, ): tombstone, } ) self._long_poll_host.remove_keys( [(LongPollNamespace.DEPLOYMENT_CONFIG, deployment_id)] ) if len(deleted_ids): self._record_deployment_usage() if target_state_changed: self.save_checkpoint() return any_recovering def autoscale(self, deployment_id: DeploymentID, target_num_replicas: int) -> bool: """Autoscale the deployment to the target number of replicas. Args: deployment_id: The deployment ID. target_num_replicas: The target number of replicas. Returns: True if the deployment was autoscaled, False otherwise. """ if self._shutting_down: logger.warning( f"Ignoring autoscale request for {deployment_id} " "because deployment state manager is shutting down." ) return False if deployment_id not in self._deployment_states: return False return self._deployment_states[deployment_id].autoscale(target_num_replicas) def _handle_scheduling_request_failures( self, deployment_id: DeploymentID, scheduling_requests: List[ReplicaSchedulingRequest], ): """Updates internal datastructures when replicas fail to be scheduled.""" failed_replicas: List[ReplicaID] = [] for scheduling_request in scheduling_requests: if ( scheduling_request.status == ReplicaSchedulingRequestStatus.PLACEMENT_GROUP_CREATION_FAILED ): failed_replicas.append(scheduling_request.replica_id) self._deployment_states[deployment_id].record_replica_startup_failure( "Replica scheduling failed. Failed to create a placement " f"group for replica {scheduling_request.replica_id}. " "See Serve controller logs for more details." ) elif ( scheduling_request.status == ReplicaSchedulingRequestStatus.ACTOR_CREATION_FAILED ): failed_replicas.append(scheduling_request.replica_id) self._deployment_states[deployment_id].record_replica_startup_failure( "Replica scheduling failed. Failed to create an actor " f"for replica {scheduling_request.replica_id}. " "See Serve controller logs for more details." ) if failed_replicas: self._deployment_states[deployment_id].stop_replicas(failed_replicas) def _record_deployment_usage(self): ServeUsageTag.NUM_DEPLOYMENTS.record(str(len(self._deployment_states))) num_gpu_deployments = 0 for deployment_state in self._deployment_states.values(): if ( deployment_state.target_info is not None and deployment_state.target_info.replica_config is not None and deployment_state.target_info.replica_config.ray_actor_options is not None and ( deployment_state.target_info.replica_config.ray_actor_options.get( "num_gpus", 0 ) > 0 ) ): num_gpu_deployments += 1 ServeUsageTag.NUM_GPU_DEPLOYMENTS.record(str(num_gpu_deployments)) def _reserve_gang_placement_groups( self, ) -> Dict[DeploymentID, GangReservationResult]: """Reserve gang placement groups for deployments. Returns: Map of deployment_id to GangReservationResult containing the created placement groups or error information. """ gang_requests: Dict[DeploymentID, GangPlacementGroupRequest] = {} for deployment_id, deployment_state in self._deployment_states.items(): if not deployment_state._is_gang_deployment: continue gang_config = deployment_state.get_gang_config() num_replicas_to_add = deployment_state._get_target_replica_delta() if num_replicas_to_add <= 0: # Only reserve PGs if we need to add replicas continue # Skip if deployment is terminally failed if deployment_state._terminally_failed(): continue # Skip if deployment has replicas still stopping. Their resources # haven't been released yet, so PG creation would likely fail or # block waiting for resources. We'll retry next reconciliation loop. if deployment_state._replicas.count(states=[ReplicaState.STOPPING]) > 0: continue # Skip if deployment actors are configured but not yet ready. # scale_deployment_replicas() defers replica creation until actors # are ready, so PGs created here would be orphaned. Orphaned PGs # accumulate every tick (~100ms) and consume cluster resources. if ( deployment_state._get_deployment_actors_configs() and not deployment_state._deployment_actors_satisfied_for_target() ): continue replica_config = deployment_state._target_state.info.replica_config gang_requests[deployment_id] = GangPlacementGroupRequest( deployment_id=deployment_id, gang_size=gang_config.gang_size, gang_placement_strategy=gang_config.gang_placement_strategy.value, num_replicas_to_add=num_replicas_to_add, replica_resource_dict=replica_config.resource_dict.copy(), replica_placement_group_bundles=( replica_config.placement_group_bundles ), replica_pg_bundle_label_selector=( replica_config.placement_group_bundle_label_selector ), replica_pg_fallback_strategy=( replica_config.placement_group_fallback_strategy ), ) if not gang_requests: return {} return self._deployment_scheduler.schedule_gang_placement_groups(gang_requests) def record_request_routing_info(self, info: RequestRoutingInfo) -> None: """ Record request routing information for a replica. Args: info: Request routing info including deployment name, replica tag, multiplex model ids, and routing stats. """ deployment_id = info.replica_id.deployment_id if deployment_id not in self._deployment_states: app_msg = f" in application '{deployment_id.app_name}'" logger.error( f"Deployment '{deployment_id.name}'{app_msg} not found in state " "manager." ) return self._deployment_states[deployment_id].record_request_routing_info(info) def get_active_node_ids(self) -> Set[str]: """Return set of node ids with running replicas of any deployment. This is used to determine which node has replicas. Only nodes with replicas and head node should have active proxies. """ node_ids = set() for deployment_state in self._deployment_states.values(): node_ids.update(deployment_state.get_active_node_ids()) return node_ids def get_ingress_replicas_info(self) -> List[Tuple[str, str, int, int]]: """Get replicas that own direct-ingress ports. Includes both ingress deployments and ingress request router deployments. """ ingress_replicas_list = [ deployment_state._replicas.get() for deployment_state in self._deployment_states.values() if deployment_state.is_ingress() or deployment_state.is_ingress_request_router() ] ingress_replicas_info = [] for replicas in ingress_replicas_list: for replica in replicas: ingress_replicas_info.append( ( replica.actor_node_id, replica.replica_id.unique_id, replica.actor_http_port, replica.actor_grpc_port, ) ) return ingress_replicas_info def _get_replica_ranks_mapping( self, deployment_id: DeploymentID ) -> Dict[str, ReplicaRank]: """Get the current rank mapping for all replicas in a deployment. Args: deployment_id: The deployment ID to get ranks for. Returns: Dictionary mapping replica_id to ReplicaRank object (with rank, node_rank, local_rank). """ deployment_state = self._deployment_states.get(deployment_id) if deployment_state is None: return {} return deployment_state._get_replica_ranks_mapping() def get_deployment_outbound_deployments( self, deployment_id: DeploymentID ) -> Optional[List[DeploymentID]]: """Get the cached outbound deployments for a specific deployment. Args: deployment_id: The deployment ID to get outbound deployments for. Returns: List of deployment IDs that this deployment calls, or None if the deployment doesn't exist or hasn't been polled yet. """ deployment_state = self._deployment_states.get(deployment_id) if deployment_state is None: return None return deployment_state.get_outbound_deployments()