chore: import upstream snapshot with attribution
This commit is contained in:
wehub-resource-sync
@@ -0,0 +1,71 @@
|
||||
import enum
|
||||
import logging
|
||||
import os
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from .base_autoscaling_coordinator import (
|
||||
AutoscalingCoordinator,
|
||||
ResourceDict,
|
||||
ResourceRequestPriority,
|
||||
)
|
||||
from .base_cluster_autoscaler import ClusterAutoscaler
|
||||
from .default_autoscaling_coordinator import (
|
||||
DefaultAutoscalingCoordinator,
|
||||
get_or_create_autoscaling_coordinator,
|
||||
)
|
||||
from .default_cluster_autoscaler_v2 import DefaultClusterAutoscalerV2
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from ray.data._internal.execution.resource_manager import ResourceManager
|
||||
from ray.data._internal.execution.streaming_executor_state import Topology
|
||||
from ray.data.context import DataContext
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
CLUSTER_AUTOSCALER_ENV_KEY = "RAY_DATA_CLUSTER_AUTOSCALER"
|
||||
DEFAULT_CLUSTER_AUTOSCALER_VERSION = "V2"
|
||||
|
||||
|
||||
class ClusterAutoscalerVersion(str, enum.Enum):
|
||||
V2 = "V2"
|
||||
|
||||
|
||||
def create_cluster_autoscaler(
|
||||
topology: "Topology",
|
||||
resource_manager: "ResourceManager",
|
||||
data_context: "DataContext",
|
||||
*,
|
||||
execution_id: str,
|
||||
) -> ClusterAutoscaler:
|
||||
resource_limits = data_context.execution_options.resource_limits
|
||||
label_selector = data_context.execution_options.label_selector
|
||||
cluster_autoscaler_version = os.environ.get(
|
||||
CLUSTER_AUTOSCALER_ENV_KEY, DEFAULT_CLUSTER_AUTOSCALER_VERSION
|
||||
)
|
||||
logger.debug(f"Using cluster autoscaler version: {cluster_autoscaler_version!r}")
|
||||
|
||||
if cluster_autoscaler_version == ClusterAutoscalerVersion.V2:
|
||||
return DefaultClusterAutoscalerV2(
|
||||
resource_manager,
|
||||
execution_id=execution_id,
|
||||
resource_limits=resource_limits,
|
||||
label_selector=label_selector,
|
||||
)
|
||||
|
||||
else:
|
||||
valid_values = [version.value for version in ClusterAutoscalerVersion]
|
||||
raise ValueError(
|
||||
f"Cluster autoscaler version of {cluster_autoscaler_version} isn't a valid "
|
||||
f"option. Valid options are: {valid_values}."
|
||||
)
|
||||
|
||||
|
||||
__all__ = [
|
||||
"ClusterAutoscaler",
|
||||
# Objects related to the `AutoscalingCoordinator`.
|
||||
"AutoscalingCoordinator",
|
||||
"DefaultAutoscalingCoordinator",
|
||||
"get_or_create_autoscaling_coordinator",
|
||||
"ResourceDict",
|
||||
"ResourceRequestPriority",
|
||||
]
|
||||
@@ -0,0 +1,58 @@
|
||||
import abc
|
||||
from enum import Enum
|
||||
from typing import Dict, List, Optional
|
||||
|
||||
ResourceDict = Dict[str, float]
|
||||
|
||||
|
||||
class ResourceRequestPriority(Enum):
|
||||
"""Priority of a resource request."""
|
||||
|
||||
LOW = -10
|
||||
MEDIUM = 0
|
||||
HIGH = 10
|
||||
|
||||
|
||||
class AutoscalingCoordinator(abc.ABC):
|
||||
@abc.abstractmethod
|
||||
def request_resources(
|
||||
self,
|
||||
resources: List[ResourceDict],
|
||||
expire_after_s: float,
|
||||
request_remaining: bool = False,
|
||||
priority: ResourceRequestPriority = ResourceRequestPriority.MEDIUM,
|
||||
label_selectors: Optional[List[Dict[str, str]]] = None,
|
||||
) -> None:
|
||||
"""Request cluster resources.
|
||||
|
||||
The requested resources should represent the full set of resources needed,
|
||||
not just the incremental amount.
|
||||
|
||||
Args:
|
||||
resources: The requested resources. This should match the format accepted
|
||||
by `ray.autoscaler.sdk.request_resources`.
|
||||
expire_after_s: Time in seconds after which this request will expire.
|
||||
The requester is responsible for periodically sending new requests
|
||||
to avoid the request being purged.
|
||||
request_remaining: If true, after allocating requested resources to each
|
||||
requester, remaining resources will also be allocated to this requester.
|
||||
priority: The priority of the request. Higher value means higher priority.
|
||||
label_selectors: Optional per-bundle label selectors, one per entry in
|
||||
``resources``. Forwarded to the autoscaler as
|
||||
``bundle_label_selectors``.
|
||||
"""
|
||||
...
|
||||
|
||||
@abc.abstractmethod
|
||||
def cancel_request(self) -> None:
|
||||
"""Cancel the resource request from the requester."""
|
||||
...
|
||||
|
||||
@abc.abstractmethod
|
||||
def get_allocated_resources(self) -> List[ResourceDict]:
|
||||
"""Get the allocated resources for the requester.
|
||||
|
||||
Returns:
|
||||
A list of dictionaries representing the allocated resources bundles.
|
||||
"""
|
||||
...
|
||||
@@ -0,0 +1,34 @@
|
||||
from abc import ABC, abstractmethod
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from ray.util.annotations import DeveloperAPI
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from ray.data._internal.execution.interfaces.execution_options import (
|
||||
ExecutionResources,
|
||||
)
|
||||
|
||||
|
||||
@DeveloperAPI
|
||||
class ClusterAutoscaler(ABC):
|
||||
"""Abstract interface for Ray Data cluster autoscaler."""
|
||||
|
||||
@abstractmethod
|
||||
def try_trigger_scaling(self):
|
||||
"""Try trigger autoscaling.
|
||||
|
||||
This method will be called each time when StreamingExecutor makes
|
||||
a scheduling decision. A subclass should override this method to
|
||||
handle the autoscaling of the cluster.
|
||||
"""
|
||||
...
|
||||
|
||||
@abstractmethod
|
||||
def on_executor_shutdown(self):
|
||||
"""Callback when the StreamingExecutor is shutting down."""
|
||||
...
|
||||
|
||||
@abstractmethod
|
||||
def get_total_resources(self) -> "ExecutionResources":
|
||||
"""Get the total resources that are available to this data execution."""
|
||||
...
|
||||
@@ -0,0 +1,577 @@
|
||||
import copy
|
||||
import functools
|
||||
import logging
|
||||
import threading
|
||||
import time
|
||||
from dataclasses import dataclass
|
||||
from typing import Callable, Dict, List, Optional
|
||||
|
||||
import ray
|
||||
import ray.exceptions
|
||||
from .base_autoscaling_coordinator import (
|
||||
AutoscalingCoordinator,
|
||||
ResourceDict,
|
||||
ResourceRequestPriority,
|
||||
)
|
||||
from ray._common.utils import env_bool
|
||||
from ray.data._internal.execution.util import memory_string
|
||||
from ray.util.scheduling_strategies import NodeAffinitySchedulingStrategy
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
HEAD_NODE_RESOURCE_LABEL = "node:__internal_head__"
|
||||
_RESOURCE_LOG_KEYS = ("CPU", "GPU", "memory", "object_store_memory")
|
||||
_RESOURCE_LOG_MEMORY_KEYS = {"memory", "object_store_memory"}
|
||||
# Label key the cluster autoscaler uses to bucket nodes by subcluster.
|
||||
# Hardcoded so all components agree without per-Dataset configuration.
|
||||
SUBCLUSTER_LABEL_KEY = "ray-subcluster"
|
||||
# Sentinel for "no subcluster" — used as both a node-label fallback and
|
||||
# the bucket key for unlabeled nodes in ``_cluster_node_resources``.
|
||||
DEFAULT_SUBCLUSTER: Optional[str] = None
|
||||
|
||||
|
||||
RAY_DATA_AUTOSCALING_COORDINATOR_LOG_TRACEBACK = env_bool(
|
||||
"RAY_DATA_AUTOSCALING_COORDINATOR_LOG_TRACEBACK", True
|
||||
)
|
||||
|
||||
|
||||
def _format_resource_value_for_log(resource_name: str, value: float) -> str:
|
||||
"""Format a numerical resource value to a human-readable string.
|
||||
|
||||
Args:
|
||||
resource_name: The resource name.
|
||||
value: The resource value.
|
||||
|
||||
Returns:
|
||||
A human-readable string.
|
||||
"""
|
||||
if resource_name in _RESOURCE_LOG_MEMORY_KEYS:
|
||||
return memory_string(value)
|
||||
if isinstance(value, float) and value.is_integer():
|
||||
return str(int(value))
|
||||
return str(value)
|
||||
|
||||
|
||||
def _format_resource_bundle_for_log(bundle: ResourceDict) -> str:
|
||||
"""Format a resource bundle to a human-readable string.
|
||||
|
||||
Drops custom resource keys (e.g. ``anyscale/...``, ``node:...``) and
|
||||
zero-valued resources, keeping only the standard keys in ``_RESOURCE_LOG_KEYS``.
|
||||
|
||||
Args:
|
||||
bundle: The resource bundle to format.
|
||||
|
||||
Returns:
|
||||
A human-readable string, e.g. ``"{CPU: 8, memory: 32.0GiB}"``.
|
||||
|
||||
Example:
|
||||
>>> from ray.data._internal.util import GiB
|
||||
>>> _format_resource_bundle_for_log({"CPU": 8, "GPU": 0, "memory": 32 * GiB})
|
||||
'{CPU: 8, memory: 32.0GiB}'
|
||||
"""
|
||||
resources = []
|
||||
for resource_name in _RESOURCE_LOG_KEYS:
|
||||
value = bundle.get(resource_name, 0)
|
||||
if value == 0:
|
||||
continue
|
||||
resources.append(
|
||||
f"{resource_name}: {_format_resource_value_for_log(resource_name, value)}"
|
||||
)
|
||||
return "{" + ", ".join(resources) + "}"
|
||||
|
||||
|
||||
def _format_resources_for_log(resources: List[ResourceDict]) -> str:
|
||||
"""Format and aggregate resource bundles for logging.
|
||||
|
||||
Bundles that format to the same string (after dropping custom/zero-valued
|
||||
resources) are collapsed into a single ``N x {...}`` entry.
|
||||
|
||||
Args:
|
||||
resources: The resource bundles to format.
|
||||
|
||||
Returns:
|
||||
A human-readable string, e.g. ``"[2 x {CPU: 1}, 1 x {GPU: 1}]"``.
|
||||
|
||||
Example:
|
||||
>>> _format_resources_for_log([{"CPU": 1}, {"CPU": 1}, {"GPU": 1}])
|
||||
'[2 x {CPU: 1}, 1 x {GPU: 1}]'
|
||||
"""
|
||||
bundle_counts: Dict[str, int] = {}
|
||||
for resource in resources:
|
||||
bundle = _format_resource_bundle_for_log(resource)
|
||||
if bundle == "{}":
|
||||
continue
|
||||
bundle_counts[bundle] = bundle_counts.get(bundle, 0) + 1
|
||||
|
||||
return (
|
||||
"["
|
||||
+ ", ".join(f"{count} x {bundle}" for bundle, count in bundle_counts.items())
|
||||
+ "]"
|
||||
)
|
||||
|
||||
|
||||
@dataclass
|
||||
class OngoingRequest:
|
||||
"""Represents an ongoing resource request from a requester."""
|
||||
|
||||
# The time when the request was first received.
|
||||
first_request_time: float
|
||||
# Requested resources.
|
||||
requested_resources: List[ResourceDict]
|
||||
# The expiration time of the request.
|
||||
expiration_time: float
|
||||
# If true, after allocating requested resources to each requester,
|
||||
# remaining resources will also be allocated to this requester.
|
||||
request_remaining: bool
|
||||
# The priority of the request, higher value means higher priority.
|
||||
priority: int
|
||||
# Resources that are already allocated to the requester.
|
||||
allocated_resources: List[ResourceDict]
|
||||
# Per-bundle label selectors, parallel to ``requested_resources``.
|
||||
# Empty dicts mean no label constraint on that bundle. Required to have
|
||||
# the same length as ``requested_resources``.
|
||||
requested_label_selectors: List[Dict[str, str]]
|
||||
|
||||
def __lt__(self, other):
|
||||
"""Used to sort requests when allocating resources.
|
||||
|
||||
Higher priority first, then earlier first_request_time first.
|
||||
"""
|
||||
if self.priority != other.priority:
|
||||
return self.priority > other.priority
|
||||
return self.first_request_time < other.first_request_time
|
||||
|
||||
|
||||
class DefaultAutoscalingCoordinator(AutoscalingCoordinator):
|
||||
"""Non-blocking client-side proxy for the _AutoscalingCoordinatorActor.
|
||||
|
||||
Not thread-safe; all methods must be called from a single thread.
|
||||
|
||||
Create one instance per requester. Multiple instances sharing the same
|
||||
``requester_id`` will have diverging caches and break the FIFO ordering
|
||||
guarantee that ``request_resources`` and ``get_allocated_resources`` rely on.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
requester_id: str,
|
||||
autoscaling_coordinator_actor=None, # For testing only: injects an actor instead of using the shared named singleton.
|
||||
subcluster_selector: Optional[Dict[str, str]] = None,
|
||||
):
|
||||
self._requester_id = requester_id
|
||||
# Label selector keyed by ``SUBCLUSTER_LABEL_KEY`` pinning this
|
||||
# requester to a single subcluster.
|
||||
self._subcluster_selector = subcluster_selector
|
||||
self._cached_allocated_resources: List[ResourceDict] = []
|
||||
# In-flight get_allocated_resources ref, or None if no request is pending.
|
||||
self._pending_allocated_resources: Optional[ray.ObjectRef] = None
|
||||
if autoscaling_coordinator_actor is not None:
|
||||
# Bypass the cached_property by injecting the actor directly.
|
||||
# Used in tests to avoid the shared named actor.
|
||||
self.__dict__["_autoscaling_coordinator"] = autoscaling_coordinator_actor
|
||||
|
||||
@functools.cached_property
|
||||
def _autoscaling_coordinator(self):
|
||||
# Lazy: avoids creating the actor in __init__.
|
||||
return get_or_create_autoscaling_coordinator()
|
||||
|
||||
def request_resources(
|
||||
self,
|
||||
resources: List[ResourceDict],
|
||||
expire_after_s: float,
|
||||
request_remaining: bool = False,
|
||||
priority: ResourceRequestPriority = ResourceRequestPriority.MEDIUM,
|
||||
label_selectors: Optional[List[Dict[str, str]]] = None,
|
||||
) -> None:
|
||||
"""Fire-and-forget: submit a resource request to the coordinator actor.
|
||||
|
||||
Actor-side errors are not surfaced to the caller.
|
||||
"""
|
||||
self._autoscaling_coordinator.request_resources.remote(
|
||||
requester_id=self._requester_id,
|
||||
resources=resources,
|
||||
expire_after_s=expire_after_s,
|
||||
request_remaining=request_remaining,
|
||||
priority=priority,
|
||||
label_selectors=label_selectors,
|
||||
subcluster_selector=self._subcluster_selector,
|
||||
)
|
||||
|
||||
def cancel_request(self) -> None:
|
||||
"""Fire-and-forget: cancel a resource request on the coordinator actor.
|
||||
|
||||
Also clears client-side state (pending ref and cached allocation) so
|
||||
a subsequent ``get_allocated_resources`` call returns a fresh result
|
||||
rather than stale data from a prior pipeline run.
|
||||
"""
|
||||
self._pending_allocated_resources = None
|
||||
self._cached_allocated_resources = []
|
||||
self._autoscaling_coordinator.cancel_request.remote(self._requester_id)
|
||||
|
||||
def get_allocated_resources(self) -> List[ResourceDict]:
|
||||
"""Return allocated resources without blocking.
|
||||
|
||||
Submits an async RPC and immediately returns the last cached result.
|
||||
The cache is updated the next time the pending RPC completes.
|
||||
|
||||
Because the actor processes calls in FIFO order, the result always
|
||||
reflects state after all previously submitted ``request_resources`` calls
|
||||
to the same actor.
|
||||
|
||||
On actor errors, returns the cached value and logs a warning; never raises.
|
||||
"""
|
||||
ref = self._pending_allocated_resources
|
||||
if ref is not None:
|
||||
ready, _ = ray.wait([ref], timeout=0)
|
||||
if ready:
|
||||
self._pending_allocated_resources = None
|
||||
try:
|
||||
self._cached_allocated_resources = ray.get(ref, timeout=0)
|
||||
except ray.exceptions.RayError:
|
||||
logger.warning(
|
||||
f"Failed to get allocated resources for {self._requester_id};"
|
||||
" falling back to the cached value."
|
||||
" If this persists, file a GitHub issue.",
|
||||
exc_info=RAY_DATA_AUTOSCALING_COORDINATOR_LOG_TRACEBACK,
|
||||
)
|
||||
|
||||
# Submit a new request if none is currently in-flight
|
||||
# (first call, or the previous request completed or errored).
|
||||
if self._pending_allocated_resources is None:
|
||||
self._pending_allocated_resources = (
|
||||
self._autoscaling_coordinator.get_allocated_resources.remote(
|
||||
self._requester_id,
|
||||
)
|
||||
)
|
||||
|
||||
return self._cached_allocated_resources
|
||||
|
||||
|
||||
def _default_send_resources_request(
|
||||
bundles: List[ResourceDict],
|
||||
label_selectors: Optional[List[Dict[str, str]]] = None,
|
||||
) -> None:
|
||||
"""Default ``send_resources_request`` implementation for the actor."""
|
||||
ray.autoscaler.sdk.request_resources(
|
||||
bundles=bundles, bundle_label_selectors=label_selectors
|
||||
)
|
||||
|
||||
|
||||
class _AutoscalingCoordinatorActor:
|
||||
"""An actor to coordinate autoscaling resource requests from different components.
|
||||
|
||||
This actor is responsible for:
|
||||
* Merging received requests and dispatching them to Ray Autoscaler.
|
||||
* Allocating cluster resources to the requesters.
|
||||
"""
|
||||
|
||||
TICK_INTERVAL_S = 20
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
get_current_time: Callable[[], float] = time.time,
|
||||
send_resources_request: Callable[
|
||||
[List[ResourceDict], Optional[List[Dict[str, str]]]], None
|
||||
] = _default_send_resources_request,
|
||||
get_cluster_nodes: Callable[[], List[Dict]] = ray.nodes,
|
||||
):
|
||||
self._get_current_time = get_current_time
|
||||
self._send_resources_request = send_resources_request
|
||||
self._get_cluster_nodes = get_cluster_nodes
|
||||
|
||||
self._ongoing_reqs: Dict[str, OngoingRequest] = {}
|
||||
# Map from requester id to its subcluster selector.
|
||||
self._subcluster_selectors: Dict[str, Optional[Dict[str, str]]] = {}
|
||||
# Node resources bucketed by their ``SUBCLUSTER_LABEL_KEY`` value.
|
||||
# Nodes without the key fall under ``DEFAULT_SUBCLUSTER``.
|
||||
self._cluster_node_resources: Dict[Optional[str], List[ResourceDict]] = {}
|
||||
# Lock for thread-safe access to shared state from the background
|
||||
self._lock = threading.Lock()
|
||||
self._update_cluster_node_resources()
|
||||
|
||||
# This is an actor, so the following check should always be True.
|
||||
# It's only needed for unit tests.
|
||||
if ray.is_initialized():
|
||||
# Start a thread to perform periodical operations.
|
||||
def tick_thread_run():
|
||||
while True:
|
||||
time.sleep(self.TICK_INTERVAL_S)
|
||||
self._tick()
|
||||
|
||||
self._tick_thread = threading.Thread(target=tick_thread_run, daemon=True)
|
||||
self._tick_thread.start()
|
||||
|
||||
def _tick(self):
|
||||
"""Used to perform periodical operations, e.g., purge expired requests,
|
||||
merge and send requests, check cluster resource updates, etc."""
|
||||
with self._lock:
|
||||
self._merge_and_send_requests()
|
||||
self._update_cluster_node_resources()
|
||||
self._reallocate_resources()
|
||||
|
||||
def request_resources(
|
||||
self,
|
||||
requester_id: str,
|
||||
resources: List[ResourceDict],
|
||||
expire_after_s: float,
|
||||
request_remaining: bool = False,
|
||||
priority: ResourceRequestPriority = ResourceRequestPriority.MEDIUM,
|
||||
label_selectors: Optional[List[Dict[str, str]]] = None,
|
||||
subcluster_selector: Optional[Dict[str, str]] = None,
|
||||
) -> None:
|
||||
logger.debug(
|
||||
"Received request from %s: %s "
|
||||
"(label_selectors=%s, subcluster_selector=%s).",
|
||||
requester_id,
|
||||
resources,
|
||||
label_selectors,
|
||||
subcluster_selector,
|
||||
)
|
||||
if label_selectors is None:
|
||||
label_selectors = [{} for _ in resources]
|
||||
elif len(label_selectors) != len(resources):
|
||||
raise ValueError(
|
||||
f"label_selectors length ({len(label_selectors)}) must match "
|
||||
f"resources length ({len(resources)})."
|
||||
)
|
||||
if subcluster_selector and label_selectors:
|
||||
req_subcluster = subcluster_selector.get(SUBCLUSTER_LABEL_KEY)
|
||||
for i, sel in enumerate(label_selectors):
|
||||
bundle_subcluster = sel.get(SUBCLUSTER_LABEL_KEY)
|
||||
if (
|
||||
bundle_subcluster is not None
|
||||
and bundle_subcluster != req_subcluster
|
||||
):
|
||||
raise ValueError(
|
||||
f"Bundle {i} label_selector targets subcluster "
|
||||
f"{bundle_subcluster!r}, but requester is registered to "
|
||||
f"{req_subcluster!r}. Per-bundle cross-subcluster "
|
||||
f"allocation is not supported."
|
||||
)
|
||||
with self._lock:
|
||||
now = self._get_current_time()
|
||||
request_updated = False
|
||||
old_req = self._ongoing_reqs.get(requester_id)
|
||||
if old_req is not None:
|
||||
if request_remaining != old_req.request_remaining:
|
||||
raise ValueError(
|
||||
"Cannot change request_remaining flag of an ongoing request."
|
||||
)
|
||||
if priority.value != old_req.priority:
|
||||
raise ValueError("Cannot change priority of an ongoing request.")
|
||||
if (
|
||||
requester_id in self._subcluster_selectors
|
||||
and self._subcluster_selectors[requester_id] != subcluster_selector
|
||||
):
|
||||
raise ValueError(
|
||||
"Cannot change subcluster_selector of an ongoing request "
|
||||
f"from {self._subcluster_selectors[requester_id]!r} to "
|
||||
f"{subcluster_selector!r}."
|
||||
)
|
||||
|
||||
request_updated = (
|
||||
resources != old_req.requested_resources
|
||||
or label_selectors != old_req.requested_label_selectors
|
||||
)
|
||||
old_req.requested_resources = resources
|
||||
old_req.requested_label_selectors = label_selectors
|
||||
old_req.expiration_time = now + expire_after_s
|
||||
else:
|
||||
request_updated = True
|
||||
self._ongoing_reqs[requester_id] = OngoingRequest(
|
||||
first_request_time=now,
|
||||
requested_resources=resources,
|
||||
requested_label_selectors=label_selectors,
|
||||
request_remaining=request_remaining,
|
||||
priority=priority.value,
|
||||
expiration_time=now + expire_after_s,
|
||||
allocated_resources=[],
|
||||
)
|
||||
# Write subcluster after all validations so a rejected call
|
||||
# never leaves the registry on a new subcluster.
|
||||
self._subcluster_selectors[requester_id] = subcluster_selector
|
||||
if request_updated:
|
||||
# If the request has updated, immediately send
|
||||
# a new request and reallocate resources.
|
||||
self._merge_and_send_requests()
|
||||
self._reallocate_resources()
|
||||
|
||||
def cancel_request(
|
||||
self,
|
||||
requester_id: str,
|
||||
):
|
||||
logger.debug("Canceling request for %s.", requester_id)
|
||||
with self._lock:
|
||||
if requester_id not in self._ongoing_reqs:
|
||||
return
|
||||
del self._ongoing_reqs[requester_id]
|
||||
self._subcluster_selectors.pop(requester_id, None)
|
||||
self._merge_and_send_requests()
|
||||
self._reallocate_resources()
|
||||
|
||||
def _purge_expired_requests(self):
|
||||
now = self._get_current_time()
|
||||
live = {
|
||||
requester_id: req
|
||||
for requester_id, req in self._ongoing_reqs.items()
|
||||
if req.expiration_time > now
|
||||
}
|
||||
for expired_id in self._ongoing_reqs.keys() - live.keys():
|
||||
self._subcluster_selectors.pop(expired_id, None)
|
||||
self._ongoing_reqs = live
|
||||
|
||||
def _merge_and_send_requests(self):
|
||||
"""Merge requests and send them to Ray Autoscaler.
|
||||
|
||||
Each bundle's forwarded selector is the union of its per-bundle
|
||||
``requested_label_selectors`` entry and the requester's
|
||||
``subcluster_selector``. The subcluster pin wins on key conflict,
|
||||
so the autoscaler always sees the correct subcluster regardless
|
||||
of what the per-bundle selectors contain.
|
||||
"""
|
||||
self._purge_expired_requests()
|
||||
merged_req: List[ResourceDict] = []
|
||||
merged_selectors: List[Dict[str, str]] = []
|
||||
for requester_id, req in self._ongoing_reqs.items():
|
||||
merged_req.extend(req.requested_resources)
|
||||
subcluster_selector = self._subcluster_selectors.get(requester_id) or {}
|
||||
for per_bundle in req.requested_label_selectors:
|
||||
merged_selectors.append({**per_bundle, **subcluster_selector})
|
||||
if any(merged_selectors):
|
||||
self._send_resources_request(merged_req, label_selectors=merged_selectors)
|
||||
else:
|
||||
self._send_resources_request(merged_req)
|
||||
|
||||
def get_allocated_resources(self, requester_id: str) -> List[ResourceDict]:
|
||||
"""Get the allocated resources for the requester."""
|
||||
with self._lock:
|
||||
if requester_id not in self._ongoing_reqs:
|
||||
return []
|
||||
return self._ongoing_reqs[requester_id].allocated_resources
|
||||
|
||||
def _maybe_subtract_resources(self, res1: ResourceDict, res2: ResourceDict) -> bool:
|
||||
"""If res2<=res1, subtract res2 from res1 in-place, and return True.
|
||||
Otherwise return False."""
|
||||
if any(res1.get(key, 0) < res2[key] for key in res2):
|
||||
return False
|
||||
for key in res2:
|
||||
if key in res1:
|
||||
res1[key] -= res2[key]
|
||||
return True
|
||||
|
||||
def _update_cluster_node_resources(self) -> bool:
|
||||
"""Update cluster resources bucketed by subcluster. Return True if changed."""
|
||||
|
||||
def _is_node_eligible(node):
|
||||
# Exclude dead nodes.
|
||||
if not node["Alive"]:
|
||||
return False
|
||||
resources = node["Resources"]
|
||||
# Exclude the head node if it doesn't have CPUs and GPUs,
|
||||
# because the object store is not usable.
|
||||
if HEAD_NODE_RESOURCE_LABEL in resources and (
|
||||
resources.get("CPU", 0) == 0 and resources.get("GPU", 0) == 0
|
||||
):
|
||||
return False
|
||||
return True
|
||||
|
||||
nodes = list(filter(_is_node_eligible, self._get_cluster_nodes()))
|
||||
nodes = sorted(nodes, key=lambda node: node.get("NodeID", ""))
|
||||
cluster_node_resources: Dict[Optional[str], List[ResourceDict]] = {}
|
||||
for node in nodes:
|
||||
# Safeguard against case where the value of Labels is None.
|
||||
labels = node.get("Labels") or {}
|
||||
subcluster = labels.get(SUBCLUSTER_LABEL_KEY, DEFAULT_SUBCLUSTER)
|
||||
cluster_node_resources.setdefault(subcluster, []).append(node["Resources"])
|
||||
if cluster_node_resources == self._cluster_node_resources:
|
||||
return False
|
||||
logger.debug("Cluster resources updated: %s.", cluster_node_resources)
|
||||
self._cluster_node_resources = cluster_node_resources
|
||||
return True
|
||||
|
||||
def _reallocate_resources(self):
|
||||
"""Reallocate cluster resources.
|
||||
|
||||
Each requester's subcluster comes from its ``subcluster_selector``.
|
||||
A requester without one is eligible only for the ``None`` bucket.
|
||||
"""
|
||||
now = self._get_current_time()
|
||||
cluster_node_resources: Dict[Optional[str], List[ResourceDict]] = copy.deepcopy(
|
||||
self._cluster_node_resources
|
||||
)
|
||||
live_items = [
|
||||
(req_id, req)
|
||||
for req_id, req in self._ongoing_reqs.items()
|
||||
if req.expiration_time >= now
|
||||
]
|
||||
live_items.sort(key=lambda item: item[1])
|
||||
|
||||
def _subcluster_of(requester_id: str) -> Optional[str]:
|
||||
selector = self._subcluster_selectors.get(requester_id)
|
||||
return (selector or {}).get(SUBCLUSTER_LABEL_KEY, DEFAULT_SUBCLUSTER)
|
||||
|
||||
# TODO(hchen): Optimize the following triple loop.
|
||||
for requester_id, ongoing_req in live_items:
|
||||
ongoing_req.allocated_resources = []
|
||||
subcluster = _subcluster_of(requester_id)
|
||||
for bundle in ongoing_req.requested_resources:
|
||||
for node_resource in cluster_node_resources.get(subcluster, []):
|
||||
if self._maybe_subtract_resources(node_resource, bundle):
|
||||
ongoing_req.allocated_resources.append(bundle)
|
||||
break
|
||||
|
||||
# Allocate remaining resources. Multiple concurrent requesters in
|
||||
# the same subcluster split that subcluster's leftovers equally.
|
||||
remaining_items = [
|
||||
(req_id, req) for req_id, req in live_items if req.request_remaining
|
||||
]
|
||||
for subcluster, node_resources in cluster_node_resources.items():
|
||||
eligible = [
|
||||
req
|
||||
for req_id, req in remaining_items
|
||||
if _subcluster_of(req_id) == subcluster
|
||||
]
|
||||
if not eligible:
|
||||
continue
|
||||
for node_resource in node_resources:
|
||||
# Integer division may leave some resources unallocated.
|
||||
divided = {k: v // len(eligible) for k, v in node_resource.items()}
|
||||
if not any(v > 0 for v in divided.values()):
|
||||
continue
|
||||
for r in eligible:
|
||||
r.allocated_resources.append(divided)
|
||||
|
||||
if logger.isEnabledFor(logging.DEBUG):
|
||||
msg = "Allocated resources:\n"
|
||||
for requester_id, ongoing_req in self._ongoing_reqs.items():
|
||||
allocated_resources_log_str = _format_resources_for_log(
|
||||
ongoing_req.allocated_resources
|
||||
)
|
||||
msg += f"Requester {requester_id}: {allocated_resources_log_str}\n"
|
||||
logger.debug(msg)
|
||||
|
||||
|
||||
_get_or_create_lock = threading.Lock()
|
||||
|
||||
|
||||
def get_or_create_autoscaling_coordinator():
|
||||
"""Get or create the AutoscalingCoordinator actor."""
|
||||
# Create the actor on the local node,
|
||||
# to reduce network overhead.
|
||||
scheduling_strategy = NodeAffinitySchedulingStrategy(
|
||||
ray.get_runtime_context().get_node_id(),
|
||||
soft=False,
|
||||
)
|
||||
actor_cls = ray.remote(num_cpus=0, max_restarts=-1, max_task_retries=-1)(
|
||||
_AutoscalingCoordinatorActor
|
||||
).options(
|
||||
name="AutoscalingCoordinator",
|
||||
namespace="AutoscalingCoordinator",
|
||||
get_if_exists=True,
|
||||
lifetime="detached",
|
||||
scheduling_strategy=scheduling_strategy,
|
||||
)
|
||||
# NOTE: Need the following lock, because Ray Core doesn't allow creating the same
|
||||
# actor from multiple threads simultaneously.
|
||||
with _get_or_create_lock:
|
||||
return actor_cls.remote()
|
||||
@@ -0,0 +1,428 @@
|
||||
import logging
|
||||
import math
|
||||
import time
|
||||
from collections import Counter, defaultdict
|
||||
from dataclasses import dataclass
|
||||
from typing import TYPE_CHECKING, Any, Callable, Dict, List, Optional
|
||||
|
||||
import ray
|
||||
from .base_autoscaling_coordinator import AutoscalingCoordinator, ResourceDict
|
||||
from .default_autoscaling_coordinator import (
|
||||
DEFAULT_SUBCLUSTER,
|
||||
SUBCLUSTER_LABEL_KEY,
|
||||
DefaultAutoscalingCoordinator,
|
||||
)
|
||||
from .resource_utilization_gauge import (
|
||||
ResourceUtilizationGauge,
|
||||
RollingLogicalUtilizationGauge,
|
||||
)
|
||||
from .util import cap_resource_request_to_limits, is_autoscaling_enabled
|
||||
from ray._common.utils import env_bool, env_float, env_integer
|
||||
from ray.data._internal.cluster_autoscaler import ClusterAutoscaler
|
||||
from ray.data._internal.execution.interfaces.execution_options import ExecutionResources
|
||||
from ray.data._internal.execution.util import memory_string
|
||||
from ray.data._internal.util import GiB
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from ray.data._internal.execution.resource_manager import ResourceManager
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class _NodeResourceSpec:
|
||||
cpu: int
|
||||
gpu: int
|
||||
mem: int
|
||||
|
||||
def __post_init__(self):
|
||||
assert isinstance(self.cpu, int)
|
||||
assert self.cpu >= 0
|
||||
assert isinstance(self.gpu, int)
|
||||
assert self.gpu >= 0
|
||||
assert isinstance(self.mem, int)
|
||||
assert self.mem >= 0
|
||||
|
||||
def __str__(self):
|
||||
return (
|
||||
"{"
|
||||
+ f"CPU: {self.cpu}, GPU: {self.gpu}, memory: {memory_string(self.mem)}"
|
||||
+ "}"
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def of(cls, *, cpu=0, gpu=0, mem=0):
|
||||
cpu = math.floor(cpu)
|
||||
gpu = math.floor(gpu)
|
||||
# Round memory to the nearest 0.1 GiB so that nodes of the same type
|
||||
# with slightly different reported physical memory are grouped together.
|
||||
mem = int(round(mem / GiB, 1) * GiB) if mem > 0 else 0
|
||||
return cls(cpu=cpu, gpu=gpu, mem=mem)
|
||||
|
||||
@classmethod
|
||||
def from_bundle(cls, bundle: Dict[str, Any]) -> "_NodeResourceSpec":
|
||||
return _NodeResourceSpec.of(
|
||||
cpu=bundle.get("CPU", 0),
|
||||
gpu=bundle.get("GPU", 0),
|
||||
mem=bundle.get("memory", 0),
|
||||
)
|
||||
|
||||
def to_bundle(self):
|
||||
return {"CPU": self.cpu, "GPU": self.gpu, "memory": self.mem}
|
||||
|
||||
|
||||
def _get_node_resource_spec_and_count(
|
||||
subcluster: Optional[str] = DEFAULT_SUBCLUSTER,
|
||||
) -> Dict[_NodeResourceSpec, int]:
|
||||
"""Get the unique node resource specs and their count in the cluster,
|
||||
scoped to a single subcluster.
|
||||
|
||||
The returned specs are the scalable worker shapes used to build scale-up
|
||||
requests, so the head node is deliberately excluded:
|
||||
|
||||
* Head node *instances* are not counted (they can't be used to schedule tasks).
|
||||
* A node group *config* dedicated to the head node is dropped as well
|
||||
(``max_count == 1`` and a shape matching the running head node).
|
||||
Otherwise its shape would be requested as an extra node even though the
|
||||
head can't be scaled. Groups that can also host workers
|
||||
(``max_count > 1``) or that have a non-head shape are kept, including
|
||||
worker types with zero running instances (scale-from-zero).
|
||||
|
||||
Quirk: the returned dict also contains a ``node_type: 0`` (ex: `"m5.xlarge": 0`) entry for every
|
||||
node type registered in ``cluster_config.node_group_configs`` that
|
||||
isn't included in this subcluster. ``get_cluster_config()``
|
||||
reports node types but not labels, so the only way to know a
|
||||
shape's subcluster is to inspect live nodes. Harmless: for example,
|
||||
if m5.xlarge nodes only exist in the training subcluster, the validation
|
||||
dataset will emit pending-bundle scale-up demand for foo nodes
|
||||
stamped with the validation label, which the autoscaler can never
|
||||
satisfy.
|
||||
TODO: get labels from cluster config so the catalog can be filtered.
|
||||
|
||||
Args:
|
||||
subcluster: The value at ``SUBCLUSTER_LABEL_KEY`` to match against.
|
||||
The default ``DEFAULT_SUBCLUSTER`` (None) selects nodes with no
|
||||
subcluster label.
|
||||
|
||||
Returns:
|
||||
A mapping from each scalable worker node shape to its current count of
|
||||
running instances (0 for shapes discovered only from the cluster
|
||||
config).
|
||||
"""
|
||||
nodes_resource_spec_count = defaultdict(int)
|
||||
|
||||
# Split nodes into the head node and worker nodes. There is exactly one head
|
||||
# node, and it can't be scaled up, so it's excluded from the counts and used
|
||||
# below to identify a node group dedicated to the head. Worker nodes are
|
||||
# further scoped to the requester's subcluster, so foreign subclusters'
|
||||
# shapes and counts don't leak into this requester's active / pending
|
||||
# bundles. Head detection is intentionally not subcluster-scoped: the head
|
||||
# node group is global.
|
||||
head_node_spec = None
|
||||
worker_node_resources = []
|
||||
for node in ray.nodes():
|
||||
if not node["Alive"]:
|
||||
continue
|
||||
if "node:__internal_head__" in node["Resources"]:
|
||||
head_node_spec = _NodeResourceSpec.from_bundle(node["Resources"])
|
||||
continue
|
||||
if (node.get("Labels") or {}).get(
|
||||
SUBCLUSTER_LABEL_KEY, DEFAULT_SUBCLUSTER
|
||||
) == subcluster:
|
||||
worker_node_resources.append(node["Resources"])
|
||||
|
||||
cluster_config = ray._private.state.state.get_cluster_config()
|
||||
if cluster_config and cluster_config.node_group_configs:
|
||||
for node_group_config in cluster_config.node_group_configs:
|
||||
if not node_group_config.resources or node_group_config.max_count == 0:
|
||||
continue
|
||||
|
||||
node_resource_spec = _NodeResourceSpec.from_bundle(
|
||||
node_group_config.resources
|
||||
)
|
||||
# Skip a node group dedicated to the head node, since it can't be scaled up and thus shouldn't be counted towards the current cluster capacity or used as a template for scaling up.
|
||||
if (
|
||||
node_group_config.max_count == 1
|
||||
and node_resource_spec == head_node_spec
|
||||
):
|
||||
continue
|
||||
|
||||
nodes_resource_spec_count[node_resource_spec] = 0
|
||||
|
||||
for r in worker_node_resources:
|
||||
node_resource_spec = _NodeResourceSpec.from_bundle(r)
|
||||
nodes_resource_spec_count[node_resource_spec] += 1
|
||||
|
||||
return nodes_resource_spec_count
|
||||
|
||||
|
||||
class DefaultClusterAutoscalerV2(ClusterAutoscaler):
|
||||
"""Ray Data's second cluster autoscaler implementation.
|
||||
|
||||
It works in the following way:
|
||||
|
||||
* Check the average cluster utilization (CPU and memory)
|
||||
in a time window (by default 10s). If the utilization is above a threshold (by
|
||||
default 0.75), send a request to Ray's autoscaler to scale up the cluster.
|
||||
* Unlike previous implementation, each resource bundle in the request is a node
|
||||
resource spec, rather than an `incremental_resource_usage()`. This allows us
|
||||
to directly scale up nodes.
|
||||
* Cluster scaling down isn't handled here. It depends on the idle node
|
||||
termination.
|
||||
"""
|
||||
|
||||
# Default cluster utilization threshold to trigger scaling up.
|
||||
DEFAULT_CLUSTER_SCALING_UP_UTIL_THRESHOLD: float = env_float(
|
||||
"RAY_DATA_CLUSTER_SCALING_UP_UTIL_THRESHOLD",
|
||||
0.75,
|
||||
)
|
||||
# Default time window in seconds to calculate the average of cluster utilization.
|
||||
DEFAULT_CLUSTER_UTIL_AVG_WINDOW_S: int = env_integer(
|
||||
"RAY_DATA_CLUSTER_UTIL_AVG_WINDOW_S",
|
||||
10,
|
||||
)
|
||||
# Default number of nodes to add per node type.
|
||||
DEFAULT_CLUSTER_SCALING_UP_DELTA: int = env_integer(
|
||||
"RAY_DATA_CLUSTER_SCALING_UP_DELTA",
|
||||
1,
|
||||
)
|
||||
|
||||
# Min number of seconds between two autoscaling requests.
|
||||
MIN_GAP_BETWEEN_AUTOSCALING_REQUESTS: int = env_integer(
|
||||
"RAY_DATA_MIN_GAP_BETWEEN_AUTOSCALING_REQUESTS",
|
||||
10,
|
||||
)
|
||||
# The time in seconds after which an autoscaling request will expire.
|
||||
AUTOSCALING_REQUEST_EXPIRE_TIME_S: int = env_integer(
|
||||
"RAY_DATA_AUTOSCALING_REQUEST_EXPIRE_TIME_S",
|
||||
180,
|
||||
)
|
||||
# When utilization drops below the scale-up threshold, keep renewing the last
|
||||
# explicit request for a short time before releasing it.
|
||||
DEFAULT_LOW_UTIL_REQUEST_RELEASE_DELAY_S: float = env_float(
|
||||
"RAY_DATA_LOW_UTIL_REQUEST_RELEASE_DELAY_S",
|
||||
180,
|
||||
)
|
||||
# Whether to disable INFO-level logs.
|
||||
RAY_DATA_DISABLE_AUTOSCALER_LOGGING = env_bool(
|
||||
"RAY_DATA_DISABLE_AUTOSCALER_LOGGING", False
|
||||
)
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
resource_manager: "ResourceManager",
|
||||
execution_id: str,
|
||||
resource_limits: ExecutionResources = ExecutionResources.inf(),
|
||||
resource_utilization_calculator: Optional[ResourceUtilizationGauge] = None,
|
||||
cluster_scaling_up_util_threshold: float = DEFAULT_CLUSTER_SCALING_UP_UTIL_THRESHOLD, # noqa: E501
|
||||
cluster_scaling_up_delta: float = DEFAULT_CLUSTER_SCALING_UP_DELTA,
|
||||
cluster_util_avg_window_s: float = DEFAULT_CLUSTER_UTIL_AVG_WINDOW_S,
|
||||
min_gap_between_autoscaling_requests_s: float = MIN_GAP_BETWEEN_AUTOSCALING_REQUESTS, # noqa: E501
|
||||
low_util_request_release_delay_s: float = DEFAULT_LOW_UTIL_REQUEST_RELEASE_DELAY_S, # noqa: E501
|
||||
autoscaling_coordinator: Optional[AutoscalingCoordinator] = None,
|
||||
get_node_counts: Optional[Callable[[], Dict[_NodeResourceSpec, int]]] = None,
|
||||
get_time: Callable[[], float] = time.time,
|
||||
label_selector: Optional[Dict[str, str]] = None,
|
||||
):
|
||||
assert cluster_scaling_up_delta > 0
|
||||
assert cluster_util_avg_window_s > 0
|
||||
assert min_gap_between_autoscaling_requests_s >= 0
|
||||
assert low_util_request_release_delay_s >= 0
|
||||
|
||||
if resource_utilization_calculator is None:
|
||||
resource_utilization_calculator = RollingLogicalUtilizationGauge(
|
||||
resource_manager,
|
||||
cluster_util_avg_window_s=cluster_util_avg_window_s,
|
||||
execution_id=execution_id,
|
||||
)
|
||||
|
||||
self._resource_limits = resource_limits
|
||||
self._label_selector = label_selector or {}
|
||||
self._resource_utilization_calculator = resource_utilization_calculator
|
||||
# Threshold of cluster utilization to trigger scaling up.
|
||||
self._cluster_scaling_up_util_threshold = cluster_scaling_up_util_threshold
|
||||
self._cluster_scaling_up_delta = int(math.ceil(cluster_scaling_up_delta))
|
||||
self._min_gap_between_autoscaling_requests_s = (
|
||||
min_gap_between_autoscaling_requests_s
|
||||
)
|
||||
self._low_util_request_release_delay_s = low_util_request_release_delay_s
|
||||
# Last time when a request was sent to Ray's autoscaler.
|
||||
self._last_request_time = 0
|
||||
# Track the last non-empty explicit request so low-utilization heartbeats
|
||||
# can keep it alive briefly without turning allocated remaining-share
|
||||
# resources into explicit autoscaler demand.
|
||||
self._last_non_empty_resource_request: List[ResourceDict] = []
|
||||
self._last_non_empty_request_time: Optional[float] = None
|
||||
# Unique identifier for the cluster autoscaler as a requester for
|
||||
# the autoscaling coordinator.
|
||||
self._requester_id = f"data-{execution_id}"
|
||||
if autoscaling_coordinator is None:
|
||||
autoscaling_coordinator = DefaultAutoscalingCoordinator(
|
||||
requester_id=self._requester_id,
|
||||
subcluster_selector=label_selector,
|
||||
)
|
||||
self._autoscaling_coordinator = autoscaling_coordinator
|
||||
if get_node_counts is None:
|
||||
# Scope node-shape/count discovery to this requester's subcluster
|
||||
# so try_trigger_scaling doesn't pull node shapes / counts from
|
||||
# other subclusters into ``active_bundles`` / ``pending_bundles``.
|
||||
subcluster = self._label_selector.get(
|
||||
SUBCLUSTER_LABEL_KEY, DEFAULT_SUBCLUSTER
|
||||
)
|
||||
get_node_counts = lambda: _get_node_resource_spec_and_count( # noqa: E731
|
||||
subcluster=subcluster
|
||||
)
|
||||
self._get_node_counts = get_node_counts
|
||||
self._get_time = get_time
|
||||
self._autoscaling_enabled = is_autoscaling_enabled()
|
||||
|
||||
# Register with the coordinator immediately so the actor knows about this
|
||||
# requester before the first ``get_allocated_resources call``. The cached value
|
||||
# returned by ``get_allocated_resources`` (and thus ``get_total_resources``) will
|
||||
# be empty until the actor responds with the first allocation (cold-start).
|
||||
self._send_resource_request([])
|
||||
|
||||
def try_trigger_scaling(self):
|
||||
# Note, should call this method before checking `_last_request_time`,
|
||||
# in order to update the average cluster utilization.
|
||||
self._resource_utilization_calculator.observe()
|
||||
|
||||
# Limit the frequency of autoscaling requests.
|
||||
now = self._get_time()
|
||||
if now - self._last_request_time < self._min_gap_between_autoscaling_requests_s:
|
||||
return
|
||||
|
||||
util = self._resource_utilization_calculator.get()
|
||||
if (
|
||||
util.cpu < self._cluster_scaling_up_util_threshold
|
||||
and util.gpu < self._cluster_scaling_up_util_threshold
|
||||
and util.memory < self._cluster_scaling_up_util_threshold
|
||||
and util.object_store_memory < self._cluster_scaling_up_util_threshold
|
||||
):
|
||||
logger.debug(
|
||||
"Cluster utilization is below threshold: "
|
||||
f"CPU={util.cpu:.2f}, GPU={util.gpu:.2f}, memory={util.memory:.2f}, "
|
||||
f"object_store_memory={util.object_store_memory:.2f}."
|
||||
)
|
||||
self._send_resource_request(None)
|
||||
return
|
||||
|
||||
# We separate active bundles (existing nodes) from pending bundles (scale-up delta)
|
||||
# to ensure existing nodes' resources are never crowded out by scale-up requests.
|
||||
# TODO(hchen): We scale up all nodes by the same delta for now.
|
||||
# We may want to distinguish different node types based on their individual
|
||||
# utilization.
|
||||
active_bundles = []
|
||||
pending_bundles = []
|
||||
node_resource_spec_count = self._get_node_counts()
|
||||
for node_resource_spec, count in node_resource_spec_count.items():
|
||||
bundle = node_resource_spec.to_bundle()
|
||||
# Bundles for existing nodes -> active (must include)
|
||||
active_bundles.extend([bundle] * count)
|
||||
# Bundles for scale-up delta -> pending (best-effort)
|
||||
pending_bundles.extend([bundle] * self._cluster_scaling_up_delta)
|
||||
|
||||
# Cap the resource request to respect user-configured limits.
|
||||
# Active bundles (existing nodes) are always included; pending bundles
|
||||
# (scale-up requests) are best-effort.
|
||||
resource_request = cap_resource_request_to_limits(
|
||||
active_bundles, pending_bundles, self._resource_limits
|
||||
)
|
||||
|
||||
if resource_request != active_bundles:
|
||||
self._log_resource_request(util, active_bundles, resource_request)
|
||||
|
||||
self._send_resource_request(resource_request)
|
||||
|
||||
def _log_resource_request(
|
||||
self,
|
||||
current_utilization: ExecutionResources,
|
||||
active_bundles: List[Dict[str, float]],
|
||||
resource_request: List[Dict[str, float]],
|
||||
) -> None:
|
||||
message = (
|
||||
"The utilization of one or more logical resource is higher than the "
|
||||
f"specified threshold of {self._cluster_scaling_up_util_threshold:.0%}: "
|
||||
f"CPU={current_utilization.cpu:.0%}, GPU={current_utilization.gpu:.0%}, "
|
||||
f"memory={current_utilization.memory:.0%}, "
|
||||
f"object_store_memory={current_utilization.object_store_memory:.0%}. "
|
||||
f"Requesting {self._cluster_scaling_up_delta} node(s) of each shape:"
|
||||
)
|
||||
|
||||
current_node_counts = Counter(
|
||||
[_NodeResourceSpec.from_bundle(bundle) for bundle in active_bundles]
|
||||
)
|
||||
requested_node_counts = Counter(
|
||||
[_NodeResourceSpec.from_bundle(bundle) for bundle in resource_request]
|
||||
)
|
||||
for node_spec, requested_count in requested_node_counts.items():
|
||||
current_count = current_node_counts.get(node_spec, 0)
|
||||
message += f" [{node_spec}: {current_count} -> {requested_count}]"
|
||||
|
||||
if self.RAY_DATA_DISABLE_AUTOSCALER_LOGGING or not self._autoscaling_enabled:
|
||||
level = logging.DEBUG
|
||||
else:
|
||||
level = logging.INFO
|
||||
|
||||
logger.log(level, message)
|
||||
|
||||
def _should_keep_non_empty_request(self, now: float) -> bool:
|
||||
return (
|
||||
self._last_non_empty_request_time is not None
|
||||
and now - self._last_non_empty_request_time
|
||||
< self._low_util_request_release_delay_s
|
||||
)
|
||||
|
||||
def _send_resource_request(
|
||||
self,
|
||||
resource_request: Optional[List[ResourceDict]],
|
||||
):
|
||||
now = self._get_time()
|
||||
update_non_empty_request_state = True
|
||||
if resource_request is None:
|
||||
if self._should_keep_non_empty_request(now):
|
||||
resource_request = self._last_non_empty_resource_request
|
||||
update_non_empty_request_state = False
|
||||
else:
|
||||
# Renew our registration on AutoscalingCoordinator without
|
||||
# keeping explicit autoscaler demand alive.
|
||||
resource_request = []
|
||||
|
||||
# Make autoscaler resource request.
|
||||
self._autoscaling_coordinator.request_resources(
|
||||
resources=resource_request,
|
||||
expire_after_s=self.AUTOSCALING_REQUEST_EXPIRE_TIME_S,
|
||||
request_remaining=True,
|
||||
)
|
||||
if resource_request and update_non_empty_request_state:
|
||||
self._last_non_empty_resource_request = [
|
||||
bundle.copy() for bundle in resource_request
|
||||
]
|
||||
self._last_non_empty_request_time = now
|
||||
elif not resource_request:
|
||||
self._last_non_empty_resource_request = []
|
||||
self._last_non_empty_request_time = None
|
||||
self._last_request_time = now
|
||||
|
||||
def on_executor_shutdown(self):
|
||||
# Cancel the resource request when the executor is shutting down.
|
||||
try:
|
||||
self._autoscaling_coordinator.cancel_request()
|
||||
except Exception:
|
||||
# cancel_request is fire-and-forget and shouldn't raise, but guard
|
||||
# against unexpected Ray Core errors at submit time. At shutdown
|
||||
# there's nothing useful to do except log and let the request expire.
|
||||
msg = (
|
||||
f"Failed to cancel resource request for {self._requester_id}."
|
||||
" The request will still expire after the timeout of"
|
||||
f" {self._min_gap_between_autoscaling_requests_s} seconds."
|
||||
)
|
||||
logger.warning(msg, exc_info=True)
|
||||
|
||||
def get_total_resources(self) -> ExecutionResources:
|
||||
"""Get total resources available from the autoscaling coordinator."""
|
||||
resources = self._autoscaling_coordinator.get_allocated_resources()
|
||||
total = ExecutionResources.zero()
|
||||
for res in resources:
|
||||
total = total.add(ExecutionResources.from_resource_dict(res))
|
||||
return total
|
||||
@@ -0,0 +1,83 @@
|
||||
import time
|
||||
from dataclasses import dataclass
|
||||
from typing import Callable, Dict, List, Optional
|
||||
|
||||
from .base_autoscaling_coordinator import (
|
||||
AutoscalingCoordinator,
|
||||
ResourceDict,
|
||||
ResourceRequestPriority,
|
||||
)
|
||||
|
||||
|
||||
class FakeAutoscalingCoordinator(AutoscalingCoordinator):
|
||||
"""A lightweight implementation for testing.
|
||||
|
||||
This implementation always allocates the requested resources to the
|
||||
requester. It doesn't support the `priority` parameter.
|
||||
"""
|
||||
|
||||
@dataclass
|
||||
class Allocation:
|
||||
resources: List[ResourceDict]
|
||||
expiration_time_s: float
|
||||
request_remaining: bool
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
get_time: Callable[[], float] = time.time,
|
||||
initial_cluster_resources: Optional[List[ResourceDict]] = None,
|
||||
):
|
||||
"""Initialize the coordinator.
|
||||
|
||||
Args:
|
||||
get_time: A function that returns the current time in seconds. This is a
|
||||
seam for testing.
|
||||
initial_cluster_resources: If the requester sends an empty request and
|
||||
``request_remaining`` is True, the coordinator allocates these resources
|
||||
to the requester. Otherwise, the coordinator allocates the requested
|
||||
resources.
|
||||
"""
|
||||
if initial_cluster_resources is None:
|
||||
initial_cluster_resources = []
|
||||
|
||||
self._get_time = get_time
|
||||
self._initial_cluster_resources = initial_cluster_resources
|
||||
self._allocation: Optional[FakeAutoscalingCoordinator.Allocation] = None
|
||||
|
||||
def request_resources(
|
||||
self,
|
||||
resources: List[ResourceDict],
|
||||
expire_after_s: float,
|
||||
request_remaining: bool = False,
|
||||
priority: ResourceRequestPriority = ResourceRequestPriority.MEDIUM,
|
||||
label_selectors: Optional[List[Dict[str, str]]] = None,
|
||||
subcluster_selector: Optional[Dict[str, str]] = None,
|
||||
) -> None:
|
||||
if priority != ResourceRequestPriority.MEDIUM:
|
||||
raise NotImplementedError(
|
||||
"This fake implementation doesn't support the `priority` parameter."
|
||||
)
|
||||
|
||||
if not resources and request_remaining:
|
||||
resources = [r.copy() for r in self._initial_cluster_resources]
|
||||
|
||||
# Always accept the request and record it.
|
||||
self._allocation = self.Allocation(
|
||||
resources=resources,
|
||||
expiration_time_s=self._get_time() + expire_after_s,
|
||||
request_remaining=request_remaining,
|
||||
)
|
||||
|
||||
def cancel_request(self) -> None:
|
||||
self._allocation = None
|
||||
|
||||
def get_allocated_resources(self) -> List[ResourceDict]:
|
||||
"""Return the allocated resources if they haven't expired."""
|
||||
if self._allocation is None:
|
||||
return []
|
||||
|
||||
if self._allocation.expiration_time_s < self._get_time():
|
||||
self._allocation = None
|
||||
return []
|
||||
|
||||
return [r.copy() for r in self._allocation.resources]
|
||||
@@ -0,0 +1,141 @@
|
||||
import abc
|
||||
import math
|
||||
from dataclasses import dataclass
|
||||
from typing import Optional
|
||||
|
||||
from ray.data._internal.average_calculator import TimeWindowAverageCalculator
|
||||
from ray.data._internal.execution.resource_manager import ResourceManager
|
||||
from ray.util.metrics import Gauge
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ClusterUtil:
|
||||
cpu: float = 0.0
|
||||
gpu: float = 0.0
|
||||
memory: float = 0.0
|
||||
object_store_memory: float = 0.0
|
||||
|
||||
def __post_init__(self):
|
||||
# If we overcommit tasks, the logical utilization can exceed 1.0.
|
||||
assert math.isfinite(self.cpu) and 0 <= self.cpu, self.cpu
|
||||
assert math.isfinite(self.gpu) and 0 <= self.gpu, self.gpu
|
||||
assert math.isfinite(self.memory) and 0 <= self.memory, self.memory
|
||||
assert (
|
||||
math.isfinite(self.object_store_memory) and 0 <= self.object_store_memory
|
||||
), self.object_store_memory
|
||||
|
||||
|
||||
class ResourceUtilizationGauge(abc.ABC):
|
||||
@abc.abstractmethod
|
||||
def observe(self):
|
||||
"""Observe the cluster utilization."""
|
||||
...
|
||||
|
||||
@abc.abstractmethod
|
||||
def get(self) -> ClusterUtil:
|
||||
"""Get the resource cluster utilization."""
|
||||
...
|
||||
|
||||
|
||||
class RollingLogicalUtilizationGauge(ResourceUtilizationGauge):
|
||||
# Default time window in seconds to calculate the average of cluster utilization.
|
||||
DEFAULT_CLUSTER_UTIL_AVG_WINDOW_S: int = 10
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
resource_manager: ResourceManager,
|
||||
*,
|
||||
cluster_util_avg_window_s: float = DEFAULT_CLUSTER_UTIL_AVG_WINDOW_S,
|
||||
execution_id: Optional[str] = None,
|
||||
):
|
||||
self._resource_manager = resource_manager
|
||||
self._execution_id = execution_id
|
||||
|
||||
self._cluster_cpu_util_calculator = TimeWindowAverageCalculator(
|
||||
cluster_util_avg_window_s
|
||||
)
|
||||
self._cluster_gpu_util_calculator = TimeWindowAverageCalculator(
|
||||
cluster_util_avg_window_s
|
||||
)
|
||||
self._cluster_mem_util_calculator = TimeWindowAverageCalculator(
|
||||
cluster_util_avg_window_s
|
||||
)
|
||||
self._cluster_obj_mem_util_calculator = TimeWindowAverageCalculator(
|
||||
cluster_util_avg_window_s
|
||||
)
|
||||
|
||||
self._cluster_cpu_utilization_gauge = None
|
||||
self._cluster_gpu_utilization_gauge = None
|
||||
self._cluster_mem_utilization_gauge = None
|
||||
self._cluster_object_store_memory_utilization_gauge = None
|
||||
|
||||
if self._execution_id is not None:
|
||||
self._cluster_cpu_utilization_gauge = Gauge(
|
||||
"data_cluster_cpu_utilization",
|
||||
description="Cluster utilization % (CPU)",
|
||||
tag_keys=("dataset",),
|
||||
)
|
||||
self._cluster_gpu_utilization_gauge = Gauge(
|
||||
"data_cluster_gpu_utilization",
|
||||
description="Cluster utilization % (GPU)",
|
||||
tag_keys=("dataset",),
|
||||
)
|
||||
self._cluster_mem_utilization_gauge = Gauge(
|
||||
"data_cluster_mem_utilization",
|
||||
description="Cluster utilization % (Memory)",
|
||||
tag_keys=("dataset",),
|
||||
)
|
||||
self._cluster_object_store_memory_utilization_gauge = Gauge(
|
||||
"data_cluster_object_store_memory_utilization",
|
||||
description="Cluster utilization % (Object Store Memory)",
|
||||
tag_keys=("dataset",),
|
||||
)
|
||||
|
||||
def observe(self):
|
||||
"""Report the cluster utilization based on global usage / global limits."""
|
||||
|
||||
def save_div(numerator, denominator):
|
||||
if not denominator:
|
||||
return 0
|
||||
else:
|
||||
return numerator / denominator
|
||||
|
||||
global_usage = self._resource_manager.get_global_usage()
|
||||
global_limits = self._resource_manager.get_global_limits()
|
||||
|
||||
cpu_util = save_div(global_usage.cpu, global_limits.cpu)
|
||||
gpu_util = save_div(global_usage.gpu, global_limits.gpu)
|
||||
mem_util = save_div(global_usage.memory, global_limits.memory)
|
||||
obj_store_mem_util = save_div(
|
||||
global_usage.object_store_memory, global_limits.object_store_memory
|
||||
)
|
||||
|
||||
self._cluster_cpu_util_calculator.report(cpu_util)
|
||||
self._cluster_gpu_util_calculator.report(gpu_util)
|
||||
self._cluster_mem_util_calculator.report(mem_util)
|
||||
self._cluster_obj_mem_util_calculator.report(obj_store_mem_util)
|
||||
|
||||
if self._execution_id is not None:
|
||||
tags = {"dataset": self._execution_id}
|
||||
if self._cluster_cpu_utilization_gauge is not None:
|
||||
self._cluster_cpu_utilization_gauge.set(cpu_util * 100, tags=tags)
|
||||
if self._cluster_gpu_utilization_gauge is not None:
|
||||
self._cluster_gpu_utilization_gauge.set(gpu_util * 100, tags=tags)
|
||||
if self._cluster_mem_utilization_gauge is not None:
|
||||
self._cluster_mem_utilization_gauge.set(mem_util * 100, tags=tags)
|
||||
if self._cluster_object_store_memory_utilization_gauge is not None:
|
||||
self._cluster_object_store_memory_utilization_gauge.set(
|
||||
obj_store_mem_util * 100, tags=tags
|
||||
)
|
||||
|
||||
def get(self) -> ClusterUtil:
|
||||
"""Get the average cluster utilization based on global usage / global limits."""
|
||||
# Clamp to 0 to handle floating-point drift in the rolling average.
|
||||
return ClusterUtil(
|
||||
cpu=max(0, self._cluster_cpu_util_calculator.get_average() or 0),
|
||||
gpu=max(0, self._cluster_gpu_util_calculator.get_average() or 0),
|
||||
memory=max(0, self._cluster_mem_util_calculator.get_average() or 0),
|
||||
object_store_memory=max(
|
||||
0, self._cluster_obj_mem_util_calculator.get_average() or 0
|
||||
),
|
||||
)
|
||||
@@ -0,0 +1,127 @@
|
||||
from typing import Dict, TypeVar
|
||||
|
||||
from ray.data._internal.execution.interfaces import ExecutionResources
|
||||
|
||||
# The math functions defined in this module use a generic type rather than
|
||||
# `PhysicalOperator` so it's easier to test. We already pass in all of the necessary
|
||||
# inputs, so the actual type doesn't matter.
|
||||
T = TypeVar("T")
|
||||
|
||||
|
||||
_SCHEDULABLE_RESOURCE_NAMES = ("cpu", "gpu", "memory")
|
||||
|
||||
|
||||
def allocate_resources(
|
||||
throughput: float,
|
||||
*,
|
||||
rates: Dict[T, float],
|
||||
resource_requirements: Dict[T, ExecutionResources],
|
||||
) -> Dict[T, ExecutionResources]:
|
||||
"""Allocate resources for a pipeline to sustain the given throughput.
|
||||
|
||||
Key insight: in a pipeline, all operators must sustain the same throughput T.
|
||||
Operator i with per-task rate r_i needs T/r_i tasks to sustain T. So maximizing
|
||||
throughput is equivalent to finding the largest feasible T, then deriving task
|
||||
counts from it.
|
||||
|
||||
Args:
|
||||
throughput: The throughput for the pipeline in the same units as the rates.
|
||||
rates: The rate at which a task or actor produces outputs for each operator.
|
||||
resource_requirements: The logical resources required to schedule a task or
|
||||
actor for each operator.
|
||||
|
||||
Returns:
|
||||
A dictionary mapping operators to the allocated resources.
|
||||
"""
|
||||
assert throughput >= 0, "Throughput must be non-negative"
|
||||
assert all(rate > 0 for rate in rates.values()), "Rates must be positive"
|
||||
|
||||
if not rates:
|
||||
return {}
|
||||
|
||||
if throughput == 0:
|
||||
return {op: ExecutionResources.zero() for op in rates}
|
||||
|
||||
# NOTE: This implementation computes fractional task counts. In practice, you
|
||||
# can't schedule a fractional task or actor, so the allocations might be infeasible.
|
||||
task_counts = {op: throughput / rate for op, rate in rates.items()}
|
||||
return {op: resource_requirements[op].scale(task_counts[op]) for op in rates}
|
||||
|
||||
|
||||
def compute_optimal_throughput(
|
||||
*,
|
||||
rates: Dict[T, float],
|
||||
resource_requirements: Dict[T, ExecutionResources],
|
||||
resource_limits: ExecutionResources,
|
||||
concurrency_limits: Dict[T, int | None],
|
||||
) -> float:
|
||||
"""Compute the optimal throughput for a pipeline.
|
||||
|
||||
The optimal throughput is bounded by two constraints (we take the tightest):
|
||||
1. Resource limits — total resource usage across all operators must fit the
|
||||
budget.
|
||||
2. Concurrency limits — each operator's task count cannot exceed its limit.
|
||||
|
||||
Args:
|
||||
rates: The rate at which a task or actor produces outputs for each operator.
|
||||
resource_requirements: The logical resources required to schedule a task or
|
||||
actor for each operator.
|
||||
resource_limits: The resource limits for the cluster.
|
||||
concurrency_limits: The maximum number of tasks or actors that can be scheduled
|
||||
concurrently for each operator.
|
||||
|
||||
Returns:
|
||||
The optimal throughput for the pipeline in the same units as the rates.
|
||||
"""
|
||||
assert rates, "Rates must be non-empty"
|
||||
return min(
|
||||
_max_throughput_from_resources(rates, resource_requirements, resource_limits),
|
||||
_max_throughput_from_concurrency(rates, concurrency_limits),
|
||||
)
|
||||
|
||||
|
||||
def _max_throughput_from_resources(
|
||||
rates: Dict[T, float],
|
||||
resource_requirements: Dict[T, ExecutionResources],
|
||||
resource_limits: ExecutionResources,
|
||||
) -> float:
|
||||
"""For each resource type, compute the max throughput the resource budget allows."""
|
||||
assert rates, "Rates must be non-empty"
|
||||
assert all(rate > 0 for rate in rates.values()), "Rates must be positive"
|
||||
assert (
|
||||
rates.keys() <= resource_requirements.keys()
|
||||
), "You must provide a resource requirement for each operator with a rate."
|
||||
|
||||
max_throughput = float("inf")
|
||||
|
||||
for resource_name in _SCHEDULABLE_RESOURCE_NAMES:
|
||||
resource_limit = getattr(resource_limits, resource_name)
|
||||
resource_cost_per_unit_throughput = sum(
|
||||
getattr(resource_requirements[op], resource_name) / rates[op]
|
||||
for op in rates
|
||||
)
|
||||
if resource_cost_per_unit_throughput > 0:
|
||||
max_throughput = min(
|
||||
max_throughput, resource_limit / resource_cost_per_unit_throughput
|
||||
)
|
||||
|
||||
assert max_throughput >= 0, "Max throughput must be non-negative"
|
||||
return max_throughput
|
||||
|
||||
|
||||
def _max_throughput_from_concurrency(
|
||||
rates: Dict[T, float],
|
||||
concurrency_limits: Dict[T, int | None],
|
||||
) -> float:
|
||||
"""Each operator's throughput is capped at rate * concurrency_limit."""
|
||||
assert rates, "Rates must be non-empty"
|
||||
assert (
|
||||
rates.keys() <= concurrency_limits.keys()
|
||||
), "You must provide a concurrency limit for each operator with a rate."
|
||||
|
||||
# Convert `None` to float("inf") for operators with no concurrency limit
|
||||
normalized_concurrency_limits: Dict[T, float] = {
|
||||
op: limit if limit is not None else float("inf")
|
||||
for op, limit in concurrency_limits.items()
|
||||
}
|
||||
return min(rates[op] * normalized_concurrency_limits[op] for op in rates)
|
||||
@@ -0,0 +1,97 @@
|
||||
import logging
|
||||
from typing import Dict, List
|
||||
|
||||
from ray.data._internal.execution.interfaces import ExecutionResources
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def is_autoscaling_enabled() -> bool:
|
||||
"""Check if any node type has autoscaling enabled (can scale up).
|
||||
|
||||
A node type is autoscalable if max_count == -1 (unlimited) or
|
||||
max_count > min_count. If no cluster config is available or no node type
|
||||
is autoscalable, returns False.
|
||||
"""
|
||||
import ray._private.state
|
||||
|
||||
cluster_config = ray._private.state.state.get_cluster_config()
|
||||
if not cluster_config or not cluster_config.node_group_configs:
|
||||
return False
|
||||
return any(
|
||||
ngc.max_count == -1 or ngc.max_count > ngc.min_count
|
||||
for ngc in cluster_config.node_group_configs
|
||||
if ngc.resources and ngc.max_count != 0
|
||||
)
|
||||
|
||||
|
||||
def cap_resource_request_to_limits(
|
||||
active_bundles: List[Dict],
|
||||
pending_bundles: List[Dict],
|
||||
resource_limits: ExecutionResources,
|
||||
) -> List[Dict]:
|
||||
"""Cap the resource request to not exceed user-configured resource limits.
|
||||
|
||||
Active bundles (for running tasks or existing nodes) are always included first
|
||||
since they represent resources already in use. Pending bundles (for future work
|
||||
or scale-up requests) are then added best-effort, sorted smallest-first to
|
||||
maximize packing within limits.
|
||||
|
||||
This ensures that resources for already-running tasks are never crowded out
|
||||
by pending work from smaller operators.
|
||||
|
||||
Args:
|
||||
active_bundles: Bundles for already-running tasks or existing nodes
|
||||
(must include - these represent current resource usage).
|
||||
pending_bundles: Bundles for pending work or scale-up requests
|
||||
(best-effort - only added if within limits).
|
||||
resource_limits: The user-configured resource limits.
|
||||
|
||||
Returns:
|
||||
A list of resource bundles that respects user limits, with active bundles
|
||||
always included first.
|
||||
"""
|
||||
# If no explicit limits are set (all infinite), return everything
|
||||
if resource_limits == ExecutionResources.inf():
|
||||
return active_bundles + pending_bundles
|
||||
|
||||
# Always include active bundles first - they're already running/allocated
|
||||
capped_request = list(active_bundles)
|
||||
total = ExecutionResources.zero()
|
||||
for bundle in active_bundles:
|
||||
total = total.add(ExecutionResources.from_resource_dict(bundle))
|
||||
|
||||
# Sort pending bundles by size (smallest first) to maximize packing.
|
||||
# This ensures smaller bundles aren't excluded due to larger bundles
|
||||
# appearing earlier in arbitrary iteration order.
|
||||
def bundle_sort_key(bundle: Dict) -> tuple:
|
||||
return (
|
||||
bundle.get("CPU", 0),
|
||||
bundle.get("GPU", 0),
|
||||
bundle.get("memory", 0),
|
||||
)
|
||||
|
||||
sorted_pending = sorted(pending_bundles, key=bundle_sort_key)
|
||||
|
||||
for bundle in sorted_pending:
|
||||
new_total = total.add(ExecutionResources.from_resource_dict(bundle))
|
||||
|
||||
# Skip bundles that don't fit, continue checking smaller ones
|
||||
if not new_total.satisfies_limit(resource_limits):
|
||||
continue
|
||||
|
||||
capped_request.append(bundle)
|
||||
total = new_total
|
||||
|
||||
total_input = len(active_bundles) + len(pending_bundles)
|
||||
if len(capped_request) < total_input:
|
||||
logger.debug(
|
||||
f"Capped autoscaling resource request from {total_input} "
|
||||
f"bundles to {len(capped_request)} bundles to respect "
|
||||
f"user-configured resource limits: {resource_limits}. "
|
||||
f"({len(active_bundles)} active bundles kept, "
|
||||
f"{len(capped_request) - len(active_bundles)}/{len(pending_bundles)} "
|
||||
f"pending bundles included)."
|
||||
)
|
||||
|
||||
return capped_request
|
||||
Reference in New Issue
Block a user