chore: import upstream snapshot with attribution
This commit is contained in:
wehub-resource-sync
@@ -0,0 +1,31 @@
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from .autoscaling_actor_pool import ActorPoolScalingRequest, AutoscalingActorPool
|
||||
from .base_actor_autoscaler import ActorAutoscaler
|
||||
from .default_actor_autoscaler import DefaultActorAutoscaler, _get_max_scale_up
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from ray.data._internal.execution.resource_manager import ResourceManager
|
||||
from ray.data._internal.execution.streaming_executor_state import Topology
|
||||
from ray.data.context import AutoscalingConfig
|
||||
|
||||
|
||||
def create_actor_autoscaler(
|
||||
topology: "Topology",
|
||||
resource_manager: "ResourceManager",
|
||||
config: "AutoscalingConfig",
|
||||
) -> ActorAutoscaler:
|
||||
return DefaultActorAutoscaler(
|
||||
topology,
|
||||
resource_manager,
|
||||
config=config,
|
||||
)
|
||||
|
||||
|
||||
__all__ = [
|
||||
"ActorAutoscaler",
|
||||
"ActorPoolScalingRequest",
|
||||
"AutoscalingActorPool",
|
||||
"create_actor_autoscaler",
|
||||
"_get_max_scale_up",
|
||||
]
|
||||
@@ -0,0 +1,284 @@
|
||||
from abc import ABC, abstractmethod
|
||||
from dataclasses import dataclass, field
|
||||
from typing import List, Optional
|
||||
|
||||
from ray import ObjectRef
|
||||
from ray.actor import ActorHandle
|
||||
from ray.data._internal.execution.interfaces.common import NodeIdStr
|
||||
from ray.data._internal.execution.interfaces.execution_options import ExecutionResources
|
||||
from ray.data._internal.execution.interfaces.ref_bundle import RefBundle
|
||||
from ray.util.annotations import DeveloperAPI
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ActorPoolScalingRequest:
|
||||
|
||||
delta: int
|
||||
force: bool = field(default=False)
|
||||
reason: Optional[str] = field(default=None)
|
||||
|
||||
@classmethod
|
||||
def no_op(cls, *, reason: Optional[str] = None) -> "ActorPoolScalingRequest":
|
||||
return ActorPoolScalingRequest(delta=0, reason=reason)
|
||||
|
||||
@classmethod
|
||||
def upscale(cls, *, delta: int, reason: Optional[str] = None):
|
||||
assert delta > 0
|
||||
return ActorPoolScalingRequest(delta=delta, reason=reason)
|
||||
|
||||
@classmethod
|
||||
def downscale(
|
||||
cls, *, delta: int, force: bool = False, reason: Optional[str] = None
|
||||
):
|
||||
assert delta < 0, "For scale down delta is expected to be negative!"
|
||||
return ActorPoolScalingRequest(delta=delta, force=force, reason=reason)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class AutoscalingActorConfig:
|
||||
"""
|
||||
per_actor_resource_usage: The resource usage per actor.
|
||||
min_size: The minimum number of running actors to be maintained
|
||||
in the pool. Note, that this constraint could be violated when
|
||||
no new work is available for scheduling in the actor pool (ie
|
||||
when operator completes execution).
|
||||
max_size: The maximum number of running actors to be maintained
|
||||
in the pool.
|
||||
initial_size: The initial number of actors to start with.
|
||||
max_actor_concurrency: The maximum number of concurrent tasks a
|
||||
single actor can execute (derived from `ray_remote_args`
|
||||
passed to the operator).
|
||||
max_tasks_in_flight_per_actor: The maximum number of tasks that can
|
||||
be submitted to a single actor at any given time.
|
||||
"""
|
||||
|
||||
min_size: int
|
||||
max_size: int
|
||||
initial_size: int
|
||||
max_tasks_in_flight_per_actor: int
|
||||
max_actor_concurrency: int
|
||||
per_actor_resource_usage: ExecutionResources
|
||||
|
||||
def __post_init__(self):
|
||||
assert self.min_size >= 1
|
||||
assert self.max_size >= self.min_size
|
||||
assert self.initial_size <= self.max_size
|
||||
assert self.initial_size >= self.min_size
|
||||
assert self.max_tasks_in_flight_per_actor >= 1
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ActorPoolInfo:
|
||||
"""Breakdown of the state of the actors used by the ``PhysicalOperator``"""
|
||||
|
||||
running: int
|
||||
pending: int
|
||||
restarting: int
|
||||
active: int = 0
|
||||
idle: int = 0
|
||||
pool_utilization: float = 0.0
|
||||
tasks_in_flight: int = 0
|
||||
|
||||
def __str__(self):
|
||||
return (
|
||||
f"running={self.running}, restarting={self.restarting}, "
|
||||
f"pending={self.pending}, active={self.active}, idle={self.idle}, "
|
||||
f"util={self.pool_utilization:.3f}, tasks_in_flight={self.tasks_in_flight}"
|
||||
)
|
||||
|
||||
|
||||
@DeveloperAPI
|
||||
class AutoscalingActorPool(ABC):
|
||||
"""Abstract interface of an autoscaling actor pool.
|
||||
|
||||
A `PhysicalOperator` can manage one or more `AutoscalingActorPool`s.
|
||||
`Autoscaler` is responsible for deciding autoscaling of these actor
|
||||
pools.
|
||||
"""
|
||||
|
||||
_LOGICAL_ACTOR_ID_LABEL_KEY = "__ray_data_logical_actor_id"
|
||||
_DEFAULT_POOL_UTILIZATION = 0
|
||||
|
||||
def __init__(self, config: AutoscalingActorConfig):
|
||||
self._config = config
|
||||
|
||||
@abstractmethod
|
||||
def num_running_actors(self) -> int:
|
||||
"""Number of running actors."""
|
||||
...
|
||||
|
||||
@abstractmethod
|
||||
def num_restarting_actors(self) -> int:
|
||||
"""Number of restarting actors"""
|
||||
...
|
||||
|
||||
@abstractmethod
|
||||
def num_active_actors(self) -> int:
|
||||
"""Number of actors with at least one active task."""
|
||||
...
|
||||
|
||||
@abstractmethod
|
||||
def num_pending_actors(self) -> int:
|
||||
"""Number of actors pending creation."""
|
||||
...
|
||||
|
||||
@abstractmethod
|
||||
def num_tasks_in_flight(self) -> int:
|
||||
"""Number of current in-flight tasks (ie total nubmer of tasks that have been
|
||||
submitted to the actor pool)."""
|
||||
...
|
||||
|
||||
def can_schedule_task(self) -> bool:
|
||||
"""Returns `True` iff the actor pool has an available actor that can run a task."""
|
||||
return self.select_actors() is not None
|
||||
|
||||
@abstractmethod
|
||||
def scale(self, req: ActorPoolScalingRequest):
|
||||
"""Applies autoscaling action"""
|
||||
...
|
||||
|
||||
@abstractmethod
|
||||
def refresh_actor_state(self):
|
||||
"""Refreshes the actor pool state (for, example, running, restarting, pending)"""
|
||||
...
|
||||
|
||||
@abstractmethod
|
||||
def on_task_submitted(self, actor: ActorHandle):
|
||||
"""Callback when an actor is picked for running a task"""
|
||||
...
|
||||
|
||||
@abstractmethod
|
||||
def on_task_completed(self, actor: ActorHandle):
|
||||
"""Called when a task completes. Returns the provided actor to the pool."""
|
||||
...
|
||||
|
||||
@abstractmethod
|
||||
def select_actors(
|
||||
self,
|
||||
bundle: Optional[RefBundle] = None,
|
||||
actor_locality_enabled: bool = False,
|
||||
) -> Optional[ActorHandle]:
|
||||
"""Select an actor to process the given bundle.
|
||||
|
||||
When ``bundle`` is ``None``, returns any available actor with spare
|
||||
capacity (used by ``can_schedule_task`` to probe schedulability).
|
||||
When ``bundle`` is provided, returns the best actor for that bundle
|
||||
(considering locality when ``actor_locality_enabled`` is True).
|
||||
|
||||
Args:
|
||||
bundle: The bundle to find an actor for. If ``None``, returns any
|
||||
available actor with spare capacity.
|
||||
actor_locality_enabled: Whether to consider locality when selecting
|
||||
an actor.
|
||||
|
||||
Returns:
|
||||
An actor handle if an actor with capacity is available, otherwise
|
||||
``None``.
|
||||
"""
|
||||
...
|
||||
|
||||
@abstractmethod
|
||||
def get_pending_actor_refs(self) -> List[ObjectRef]:
|
||||
"""Return the list of object refs for actors that are pending creation."""
|
||||
...
|
||||
|
||||
@abstractmethod
|
||||
def pending_to_running(self, ready_ref: ObjectRef) -> Optional[ActorHandle]:
|
||||
"""Mark the actor corresponding to the provided ready future as running.
|
||||
|
||||
Args:
|
||||
ready_ref: The ready future for the actor to mark as running.
|
||||
|
||||
Returns:
|
||||
The actor handle if the actor is still alive, otherwise ``None``.
|
||||
"""
|
||||
...
|
||||
|
||||
@abstractmethod
|
||||
def get_actor_location(self, actor: ActorHandle) -> NodeIdStr:
|
||||
"""Get the node_id of the actor"""
|
||||
...
|
||||
|
||||
@abstractmethod
|
||||
def shutdown(self, force: bool = False):
|
||||
"""Kills all actors, including running/active actors.
|
||||
|
||||
This is called once the operator is shutting down.
|
||||
"""
|
||||
...
|
||||
|
||||
def get_logical_id_label_key(self) -> str:
|
||||
"""Get the label key for the logical actor ID.
|
||||
|
||||
Actors launched by this pool should have this label.
|
||||
"""
|
||||
return self._LOGICAL_ACTOR_ID_LABEL_KEY
|
||||
|
||||
def get_actor_info(self) -> ActorPoolInfo:
|
||||
"""Returns current snapshot of actors' being used in the pool"""
|
||||
pool_util = self.get_pool_util()
|
||||
# Handle infinite utilization case (no actors)
|
||||
if pool_util == float("inf"):
|
||||
pool_util = self._DEFAULT_POOL_UTILIZATION
|
||||
|
||||
return ActorPoolInfo(
|
||||
running=self.num_alive_actors(),
|
||||
pending=self.num_pending_actors(),
|
||||
restarting=self.num_restarting_actors(),
|
||||
active=self.num_active_actors(),
|
||||
idle=self.num_idle_actors(),
|
||||
pool_utilization=pool_util,
|
||||
tasks_in_flight=self.num_tasks_in_flight(),
|
||||
)
|
||||
|
||||
def num_alive_actors(self) -> int:
|
||||
"""Alive actors are all the running actors in ALIVE state."""
|
||||
return self.num_running_actors() - self.num_restarting_actors()
|
||||
|
||||
def num_idle_actors(self) -> int:
|
||||
"""Return the number of idle actors in the pool."""
|
||||
return self.num_running_actors() - self.num_active_actors()
|
||||
|
||||
def per_actor_resource_usage(self) -> ExecutionResources:
|
||||
"""Per actor resource usage."""
|
||||
return self._config.per_actor_resource_usage
|
||||
|
||||
def max_actor_concurrency(self) -> int:
|
||||
"""Returns max number of tasks single actor could run concurrently."""
|
||||
return self._config.max_actor_concurrency
|
||||
|
||||
def max_tasks_in_flight_per_actor(self) -> int:
|
||||
"""Max number of in-flight tasks per actor."""
|
||||
return self._config.max_tasks_in_flight_per_actor
|
||||
|
||||
def initial_size(self) -> int:
|
||||
return self._config.initial_size
|
||||
|
||||
def current_size(self) -> int:
|
||||
return self.num_pending_actors() + self.num_running_actors()
|
||||
|
||||
def min_size(self) -> int:
|
||||
"""Min size of the actor pool."""
|
||||
return self._config.min_size
|
||||
|
||||
def max_size(self) -> int:
|
||||
"""Max size of the actor pool."""
|
||||
return self._config.max_size
|
||||
|
||||
def get_pool_util(self) -> float:
|
||||
"""Calculate the utilization of the given actor pool."""
|
||||
|
||||
# If there are no running actors, we set the utilization to indicate that the pool should be scaled up immediately.
|
||||
if self.current_size() == 0:
|
||||
return float("inf")
|
||||
else:
|
||||
# We compute utilization as a ratio of
|
||||
# - Number of submitted tasks over
|
||||
# - Max number of tasks that Actor Pool could currently run
|
||||
#
|
||||
# This value could exceed 100%, since by default actors are allowed
|
||||
# to queue tasks (to pipeline task execution by overlapping block
|
||||
# fetching with the execution of the previous task)
|
||||
return self.num_tasks_in_flight() / (
|
||||
self.max_actor_concurrency() * self.current_size()
|
||||
)
|
||||
@@ -0,0 +1,31 @@
|
||||
from abc import ABC, abstractmethod
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from ray.util.annotations import DeveloperAPI
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from ray.data._internal.execution.resource_manager import ResourceManager
|
||||
from ray.data._internal.execution.streaming_executor_state import Topology
|
||||
|
||||
|
||||
@DeveloperAPI
|
||||
class ActorAutoscaler(ABC):
|
||||
"""Abstract interface for Ray Data actor autoscaler."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
topology: "Topology",
|
||||
resource_manager: "ResourceManager",
|
||||
):
|
||||
self._topology = topology
|
||||
self._resource_manager = resource_manager
|
||||
|
||||
@abstractmethod
|
||||
def try_trigger_scaling(self):
|
||||
"""Try trigger autoscaling.
|
||||
|
||||
This method will be called each time when StreamingExecutor makes
|
||||
a scheduling decision. A subclass should override this method to
|
||||
handle the autoscaling of `AutoscalingActorPool`s.
|
||||
"""
|
||||
...
|
||||
@@ -0,0 +1,326 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import math
|
||||
import sys
|
||||
from typing import TYPE_CHECKING, Optional
|
||||
|
||||
from .autoscaling_actor_pool import ActorPoolScalingRequest, AutoscalingActorPool
|
||||
from .base_actor_autoscaler import ActorAutoscaler
|
||||
from ray.data._internal.execution.interfaces.execution_options import ExecutionResources
|
||||
from ray.data.context import WARN_PREFIX, AutoscalingConfig
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from ray.data._internal.execution.interfaces import PhysicalOperator
|
||||
from ray.data._internal.execution.resource_manager import ResourceManager
|
||||
from ray.data._internal.execution.streaming_executor_state import OpState, Topology
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class DefaultActorAutoscaler(ActorAutoscaler):
|
||||
def __init__(
|
||||
self,
|
||||
topology: "Topology",
|
||||
resource_manager: "ResourceManager",
|
||||
*,
|
||||
config: AutoscalingConfig,
|
||||
):
|
||||
super().__init__(topology, resource_manager)
|
||||
|
||||
self._actor_pool_scaling_up_threshold: float = (
|
||||
config.actor_pool_util_upscaling_threshold
|
||||
)
|
||||
self._actor_pool_scaling_down_threshold: float = (
|
||||
config.actor_pool_util_downscaling_threshold
|
||||
)
|
||||
self._actor_pool_max_upscaling_delta: Optional[
|
||||
int
|
||||
] = config.actor_pool_max_upscaling_delta
|
||||
|
||||
self._validate_autoscaling_config()
|
||||
|
||||
def try_trigger_scaling(self):
|
||||
for op, state in self._topology.items():
|
||||
actor_pools = op.get_autoscaling_actor_pools()
|
||||
for actor_pool in actor_pools:
|
||||
# Trigger auto-scaling
|
||||
actor_pool.scale(
|
||||
self._derive_target_scaling_config(actor_pool, op, state)
|
||||
)
|
||||
|
||||
def _derive_target_scaling_config(
|
||||
self,
|
||||
actor_pool: AutoscalingActorPool,
|
||||
op: "PhysicalOperator",
|
||||
op_state: "OpState",
|
||||
) -> ActorPoolScalingRequest:
|
||||
# If all inputs have been consumed, short-circuit
|
||||
if op.has_completed() or (
|
||||
op._inputs_complete and op_state.total_enqueued_input_blocks() == 0
|
||||
):
|
||||
num_to_scale_down = self._compute_downscale_delta(actor_pool)
|
||||
return ActorPoolScalingRequest.downscale(
|
||||
delta=-num_to_scale_down, force=True, reason="consumed all inputs"
|
||||
)
|
||||
|
||||
if actor_pool.current_size() < actor_pool.min_size():
|
||||
# Scale up, if the actor pool is below min size.
|
||||
return ActorPoolScalingRequest.upscale(
|
||||
delta=actor_pool.min_size() - actor_pool.current_size(),
|
||||
reason="pool below min size",
|
||||
)
|
||||
elif actor_pool.current_size() > actor_pool.max_size():
|
||||
return ActorPoolScalingRequest.downscale(
|
||||
delta=-(actor_pool.current_size() - actor_pool.max_size()),
|
||||
reason="pool exceeding max size",
|
||||
)
|
||||
|
||||
allocation = self._resource_manager.get_allocation(op)
|
||||
op_usage = self._resource_manager.get_op_usage(op)
|
||||
if allocation is not None and op_usage is not None:
|
||||
over_budget_scale_down = _get_required_scale_down(
|
||||
actor_pool, allocation.subtract(op_usage)
|
||||
)
|
||||
if over_budget_scale_down > 0:
|
||||
max_can_release = actor_pool.current_size() - actor_pool.min_size()
|
||||
num_to_scale_down = min(over_budget_scale_down, max_can_release)
|
||||
if num_to_scale_down > 0:
|
||||
return ActorPoolScalingRequest.downscale(
|
||||
delta=-num_to_scale_down,
|
||||
reason="actor pool exceeds resource allocation",
|
||||
)
|
||||
return ActorPoolScalingRequest.no_op(
|
||||
reason="actor pool exceeds resource allocation "
|
||||
"but cannot scale below min size",
|
||||
)
|
||||
|
||||
# To prevent unexpected downscaling from the initial size, short-circuit if
|
||||
# the operator hasn't received any inputs.
|
||||
if op.metrics.num_inputs_received == 0:
|
||||
return ActorPoolScalingRequest.no_op(reason="no inputs received")
|
||||
|
||||
# Determine whether to scale up based on the actor pool utilization.
|
||||
util = actor_pool.get_pool_util()
|
||||
|
||||
if util >= self._actor_pool_scaling_up_threshold:
|
||||
# Do not scale up if either
|
||||
# - Actor Pool is at max size already
|
||||
# - Op is throttled (ie exceeding allocated resource quota)
|
||||
if actor_pool.current_size() >= actor_pool.max_size():
|
||||
return ActorPoolScalingRequest.no_op(reason="reached max size")
|
||||
if not op_state._scheduling_status.under_resource_limits:
|
||||
return ActorPoolScalingRequest.no_op(
|
||||
reason="operator exceeding resource quota"
|
||||
)
|
||||
|
||||
budget = self._resource_manager.get_budget(op)
|
||||
budget_max_scale_up = (
|
||||
_get_max_scale_up(actor_pool, budget) if budget else sys.maxsize
|
||||
)
|
||||
|
||||
# Determine maximum available scale up based on
|
||||
# - Maximum available resource budget
|
||||
# - Configured max scale-up delta (or "+inf" if not configured)
|
||||
# - Total # of actors needed to reach `max_size`
|
||||
max_scale_up: int = min(
|
||||
budget_max_scale_up,
|
||||
self._get_actor_pool_max_upscaling_delta(),
|
||||
actor_pool.max_size() - actor_pool.current_size(),
|
||||
)
|
||||
|
||||
if max_scale_up == 0:
|
||||
return ActorPoolScalingRequest.no_op(reason="exceeded resource limits")
|
||||
|
||||
if util == float("inf"):
|
||||
return ActorPoolScalingRequest.upscale(
|
||||
delta=1, reason="no running actors, scale up immediately"
|
||||
)
|
||||
|
||||
delta = self._compute_upscale_delta(actor_pool, op_state)
|
||||
# At least scale up by 1
|
||||
delta = max(1, delta)
|
||||
# Cap delta
|
||||
delta = min(delta, max_scale_up)
|
||||
|
||||
return ActorPoolScalingRequest.upscale(
|
||||
delta=delta,
|
||||
reason=(
|
||||
f"utilization of {util} >= "
|
||||
f"{self._actor_pool_scaling_up_threshold}"
|
||||
),
|
||||
)
|
||||
elif util <= self._actor_pool_scaling_down_threshold:
|
||||
if actor_pool.num_pending_actors() > 0:
|
||||
return ActorPoolScalingRequest.no_op(
|
||||
reason="no downscaling while actors are pending"
|
||||
)
|
||||
if actor_pool.current_size() <= actor_pool.min_size():
|
||||
return ActorPoolScalingRequest.no_op(reason="reached min size")
|
||||
|
||||
max_can_release = actor_pool.current_size() - actor_pool.min_size()
|
||||
num_to_scale_down = min(
|
||||
self._compute_downscale_delta(actor_pool), max_can_release
|
||||
)
|
||||
|
||||
return ActorPoolScalingRequest.downscale(
|
||||
delta=-num_to_scale_down,
|
||||
reason=(
|
||||
f"utilization of {util} <= "
|
||||
f"{self._actor_pool_scaling_down_threshold}"
|
||||
),
|
||||
)
|
||||
else:
|
||||
return ActorPoolScalingRequest.no_op(
|
||||
reason=(
|
||||
f"utilization of {util} w/in limits "
|
||||
f"[{self._actor_pool_scaling_down_threshold}, "
|
||||
f"{self._actor_pool_scaling_up_threshold}]"
|
||||
)
|
||||
)
|
||||
|
||||
def _get_actor_pool_max_upscaling_delta(self) -> int:
|
||||
return (
|
||||
self._actor_pool_max_upscaling_delta
|
||||
if self._actor_pool_max_upscaling_delta is not None
|
||||
else sys.maxsize
|
||||
)
|
||||
|
||||
def _validate_autoscaling_config(self):
|
||||
# Validate that max upscaling delta is positive to prevent override by safeguard
|
||||
if (
|
||||
self._actor_pool_max_upscaling_delta is not None
|
||||
and self._actor_pool_max_upscaling_delta <= 0
|
||||
):
|
||||
raise ValueError(
|
||||
f"actor_pool_max_upscaling_delta must be positive, "
|
||||
f"got {self._actor_pool_max_upscaling_delta}"
|
||||
)
|
||||
# Validate that upscaling threshold is positive to prevent division by zero
|
||||
# and incorrect scaling calculations
|
||||
if self._actor_pool_scaling_up_threshold <= 0:
|
||||
raise ValueError(
|
||||
f"actor_pool_util_upscaling_threshold must be positive, "
|
||||
f"got {self._actor_pool_scaling_up_threshold}"
|
||||
)
|
||||
|
||||
for op, state in self._topology.items():
|
||||
for actor_pool in op.get_autoscaling_actor_pools():
|
||||
self._validate_actor_pool_autoscaling_config(actor_pool, op)
|
||||
|
||||
def _validate_actor_pool_autoscaling_config(
|
||||
self,
|
||||
actor_pool: AutoscalingActorPool,
|
||||
op: "PhysicalOperator",
|
||||
) -> None:
|
||||
"""Validate autoscaling configuration.
|
||||
|
||||
Args:
|
||||
actor_pool: Actor pool to validate configuration thereof.
|
||||
op: ``PhysicalOperator`` using target actor pool.
|
||||
"""
|
||||
# Fixed-size pools don't autoscale by design
|
||||
if actor_pool.min_size() == actor_pool.max_size():
|
||||
return
|
||||
|
||||
max_tasks_in_flight_per_actor = actor_pool.max_tasks_in_flight_per_actor()
|
||||
max_concurrency = actor_pool.max_actor_concurrency()
|
||||
|
||||
if (
|
||||
max_tasks_in_flight_per_actor / max_concurrency
|
||||
< self._actor_pool_scaling_up_threshold
|
||||
):
|
||||
logger.warning(
|
||||
f"{WARN_PREFIX} Actor Pool configuration of the {op} will not allow it to scale up: "
|
||||
f"configured utilization threshold ({self._actor_pool_scaling_up_threshold * 100}%) "
|
||||
f"couldn't be reached with configured max_concurrency={max_concurrency} "
|
||||
f"and max_tasks_in_flight_per_actor={max_tasks_in_flight_per_actor} "
|
||||
f"(max utilization will be max_tasks_in_flight_per_actor / max_concurrency = {(max_tasks_in_flight_per_actor / max_concurrency) * 100:g}%)"
|
||||
)
|
||||
|
||||
def _compute_upscale_delta(
|
||||
self, actor_pool: AutoscalingActorPool, op_state: OpState
|
||||
) -> int:
|
||||
# Calculate desired delta based on utilization
|
||||
return math.ceil(
|
||||
actor_pool.current_size()
|
||||
* (actor_pool.get_pool_util() / self._actor_pool_scaling_up_threshold - 1)
|
||||
)
|
||||
|
||||
def _compute_downscale_delta(self, actor_pool: "AutoscalingActorPool") -> int:
|
||||
return 1
|
||||
|
||||
|
||||
def _estimate_total_available_task_slots(actor_pool: "AutoscalingActorPool") -> int:
|
||||
# Estimates number of available task slots to schedule new tasks
|
||||
#
|
||||
# NOTE: This must include pending actors for estimation to make sure
|
||||
# autoscaler appropriately accounts task slots that will be available
|
||||
# once pending actors become running.
|
||||
return (
|
||||
actor_pool.max_tasks_in_flight_per_actor() * actor_pool.current_size()
|
||||
- actor_pool.num_tasks_in_flight()
|
||||
)
|
||||
|
||||
|
||||
def _get_max_scale_up(
|
||||
actor_pool: AutoscalingActorPool,
|
||||
budget: ExecutionResources,
|
||||
) -> int:
|
||||
"""Get the maximum number of actors that can be scaled up.
|
||||
|
||||
Args:
|
||||
actor_pool: The actor pool to scale up.
|
||||
budget: The budget to scale up.
|
||||
|
||||
Returns:
|
||||
The maximum number of actors that can be scaled up, or `None` if you can
|
||||
scale up infinitely.
|
||||
"""
|
||||
assert budget.cpu >= 0 and budget.gpu >= 0 and budget.memory >= 0
|
||||
|
||||
per_actor = actor_pool.per_actor_resource_usage()
|
||||
assert per_actor.cpu >= 0 and per_actor.gpu >= 0 and per_actor.memory >= 0
|
||||
|
||||
# floordiv handles per_actor.x == 0 → inf (no constraint from that resource)
|
||||
# and budget.x == inf → inf. We ignore object_store_memory since it is not
|
||||
# a per-actor declared resource.
|
||||
divisions = budget.floordiv(per_actor)
|
||||
max_scale_up = min(divisions.cpu, divisions.gpu, divisions.memory)
|
||||
if math.isinf(max_scale_up):
|
||||
return sys.maxsize
|
||||
return int(max_scale_up)
|
||||
|
||||
|
||||
def _get_required_scale_down(
|
||||
actor_pool: AutoscalingActorPool,
|
||||
budget: ExecutionResources,
|
||||
) -> int:
|
||||
"""Get the number of actors that must be removed to fit within budget.
|
||||
|
||||
Args:
|
||||
actor_pool: The actor pool to scale down.
|
||||
budget: The net remaining budget (allocation - usage). Can be negative
|
||||
if the operator is over its allocation.
|
||||
|
||||
Returns:
|
||||
The number of actors that need to be removed, or 0 if the pool
|
||||
is within budget.
|
||||
"""
|
||||
per_actor = actor_pool.per_actor_resource_usage()
|
||||
|
||||
required_cpu_scale_down = 0
|
||||
if per_actor.cpu > 0 and budget.cpu < 0:
|
||||
required_cpu_scale_down = math.ceil(abs(budget.cpu) / per_actor.cpu)
|
||||
|
||||
required_gpu_scale_down = 0
|
||||
if per_actor.gpu > 0 and budget.gpu < 0:
|
||||
required_gpu_scale_down = math.ceil(abs(budget.gpu) / per_actor.gpu)
|
||||
|
||||
required_memory_scale_down = 0
|
||||
if per_actor.memory > 0 and budget.memory < 0:
|
||||
required_memory_scale_down = math.ceil(abs(budget.memory) / per_actor.memory)
|
||||
|
||||
return max(
|
||||
required_cpu_scale_down, required_gpu_scale_down, required_memory_scale_down
|
||||
)
|
||||
Reference in New Issue
Block a user