chore: import upstream snapshot with attribution
This commit is contained in:
wehub-resource-sync
@@ -0,0 +1,709 @@
|
||||
import time
|
||||
from contextlib import contextmanager
|
||||
from types import MethodType
|
||||
from typing import Optional
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import pytest
|
||||
|
||||
import ray
|
||||
from ray.data import ExecutionResources
|
||||
from ray.data._internal.actor_autoscaler import (
|
||||
ActorPoolScalingRequest,
|
||||
DefaultActorAutoscaler,
|
||||
)
|
||||
from ray.data._internal.actor_autoscaler.default_actor_autoscaler import (
|
||||
_get_max_scale_up,
|
||||
)
|
||||
from ray.data._internal.execution.operators.actor_pool_map_operator import _ActorPool
|
||||
from ray.data._internal.execution.operators.base_physical_operator import (
|
||||
InternalQueueOperatorMixin,
|
||||
)
|
||||
from ray.data._internal.execution.resource_manager import ResourceManager
|
||||
from ray.data._internal.execution.streaming_executor_state import OpState
|
||||
from ray.data.context import (
|
||||
AutoscalingConfig,
|
||||
)
|
||||
|
||||
|
||||
def test_actor_pool_scaling():
|
||||
"""Test `_actor_pool_should_scale_up` and `_actor_pool_should_scale_down`
|
||||
in `DefaultAutoscaler`"""
|
||||
|
||||
resource_manager = MagicMock(
|
||||
spec=ResourceManager,
|
||||
get_budget=MagicMock(return_value=None),
|
||||
get_allocation=MagicMock(return_value=None),
|
||||
)
|
||||
autoscaler = DefaultActorAutoscaler(
|
||||
topology=MagicMock(),
|
||||
resource_manager=resource_manager,
|
||||
config=AutoscalingConfig(
|
||||
actor_pool_util_upscaling_threshold=1.0,
|
||||
actor_pool_util_downscaling_threshold=0.5,
|
||||
actor_pool_max_upscaling_delta=None,
|
||||
),
|
||||
)
|
||||
|
||||
# Current actor pool utilization is 0.9, which is above the threshold.
|
||||
actor_pool: _ActorPool = MagicMock(
|
||||
spec=_ActorPool,
|
||||
min_size=MagicMock(return_value=5),
|
||||
max_size=MagicMock(return_value=15),
|
||||
current_size=MagicMock(return_value=10),
|
||||
num_active_actors=MagicMock(return_value=10),
|
||||
num_running_actors=MagicMock(return_value=10),
|
||||
num_pending_actors=MagicMock(return_value=0),
|
||||
num_tasks_in_flight=MagicMock(return_value=15),
|
||||
per_actor_resource_usage=MagicMock(return_value=ExecutionResources(cpu=1)),
|
||||
max_tasks_in_flight_per_actor=MagicMock(return_value=2),
|
||||
max_actor_concurrency=MagicMock(return_value=1),
|
||||
get_pool_util=MagicMock(
|
||||
# NOTE: Unittest mocking library doesn't support proxying to actual
|
||||
# non-mocked methods so we have emulate it by directly binding existing
|
||||
# method of `get_pool_util` to a mocked object
|
||||
side_effect=lambda: MethodType(_ActorPool.get_pool_util, actor_pool)()
|
||||
),
|
||||
)
|
||||
|
||||
op = MagicMock(
|
||||
spec=InternalQueueOperatorMixin,
|
||||
has_completed=MagicMock(return_value=False),
|
||||
_inputs_complete=False,
|
||||
input_dependencies=[MagicMock()],
|
||||
internal_input_queue_num_blocks=MagicMock(return_value=1),
|
||||
metrics=MagicMock(average_num_inputs_per_task=1, num_inputs_received=1),
|
||||
num_output_splits=MagicMock(return_value=1),
|
||||
)
|
||||
op_state = OpState(
|
||||
op, inqueues=[MagicMock(__len__=MagicMock(return_value=10), num_blocks=10)]
|
||||
)
|
||||
op_state._scheduling_status = MagicMock(under_resource_limits=True)
|
||||
|
||||
@contextmanager
|
||||
def patch(mock, attr, value, is_method=True):
|
||||
original = getattr(mock, attr)
|
||||
if is_method:
|
||||
value = MagicMock(return_value=value)
|
||||
setattr(mock, attr, value)
|
||||
yield
|
||||
setattr(mock, attr, original)
|
||||
|
||||
def assert_autoscaling_action(
|
||||
*, delta: int, expected_reason: Optional[str], force: bool = False
|
||||
):
|
||||
nonlocal actor_pool, op, op_state
|
||||
|
||||
assert autoscaler._derive_target_scaling_config(
|
||||
actor_pool=actor_pool,
|
||||
op=op,
|
||||
op_state=op_state,
|
||||
) == ActorPoolScalingRequest(delta=delta, force=force, reason=expected_reason)
|
||||
|
||||
# Should scale up since the util above the threshold.
|
||||
assert actor_pool.get_pool_util() == 1.5
|
||||
assert_autoscaling_action(
|
||||
delta=5,
|
||||
expected_reason="utilization of 1.5 >= 1.0",
|
||||
)
|
||||
|
||||
# Should scale up immediately when the actor pool has no running actors.
|
||||
with patch(actor_pool, "num_running_actors", 0):
|
||||
with patch(actor_pool, "get_pool_util", float("inf")):
|
||||
assert_autoscaling_action(
|
||||
delta=1,
|
||||
expected_reason="no running actors, scale up immediately",
|
||||
)
|
||||
|
||||
# Should be no-op since the util is below the threshold.
|
||||
with patch(actor_pool, "num_tasks_in_flight", 9):
|
||||
assert actor_pool.get_pool_util() == 0.9
|
||||
assert_autoscaling_action(
|
||||
delta=0, expected_reason="utilization of 0.9 w/in limits [0.5, 1.0]"
|
||||
)
|
||||
|
||||
# Should be no-op since there are pending actors (no downscaling while pending)
|
||||
with patch(actor_pool, "num_pending_actors", 1):
|
||||
with patch(actor_pool, "num_tasks_in_flight", 4):
|
||||
assert actor_pool.get_pool_util() == 0.4
|
||||
assert_autoscaling_action(
|
||||
delta=0,
|
||||
expected_reason="no downscaling while actors are pending",
|
||||
)
|
||||
|
||||
# Should be no-op since we have reached the max size (ie could not scale
|
||||
# up even though utilization > threshold)
|
||||
with patch(actor_pool, "current_size", 15):
|
||||
with patch(actor_pool, "num_tasks_in_flight", 20):
|
||||
assert_autoscaling_action(
|
||||
delta=0,
|
||||
expected_reason="reached max size",
|
||||
)
|
||||
|
||||
# Should be no-op since we have reached the min size (ie could not scale
|
||||
# down even though utilization < threshold)
|
||||
with patch(actor_pool, "current_size", 5):
|
||||
with patch(actor_pool, "num_tasks_in_flight", 2):
|
||||
assert_autoscaling_action(
|
||||
delta=0,
|
||||
expected_reason="reached min size",
|
||||
)
|
||||
|
||||
# Should scale up since the pool is below the min size.
|
||||
with patch(actor_pool, "current_size", 4):
|
||||
assert_autoscaling_action(
|
||||
delta=1,
|
||||
expected_reason="pool below min size",
|
||||
)
|
||||
|
||||
# Should scale down since if the op is completed, or
|
||||
# the op has no more inputs.
|
||||
with patch(op, "has_completed", True):
|
||||
# NOTE: We simulate actor pool dipping below min size upon
|
||||
# completion (to verify that it will be able to scale to 0)
|
||||
with patch(actor_pool, "current_size", 5):
|
||||
assert_autoscaling_action(
|
||||
delta=-1,
|
||||
expected_reason="consumed all inputs",
|
||||
force=True,
|
||||
)
|
||||
|
||||
# Should scale down only once all inputs have been already dispatched AND
|
||||
# no new inputs ar expected
|
||||
with patch(op_state.input_queues[0], "num_blocks", 0, is_method=False):
|
||||
with patch(op, "internal_input_queue_num_blocks", 0):
|
||||
with patch(op, "_inputs_complete", True, is_method=False):
|
||||
assert_autoscaling_action(
|
||||
delta=-1,
|
||||
force=True,
|
||||
expected_reason="consumed all inputs",
|
||||
)
|
||||
|
||||
# With no enqueued inputs but inputs not being complete still,
|
||||
# the autoscaler should still scale up based on utilization
|
||||
assert_autoscaling_action(
|
||||
delta=5,
|
||||
expected_reason="utilization of 1.5 >= 1.0",
|
||||
)
|
||||
|
||||
# Should be no-op since the op doesn't have enough resources.
|
||||
with patch(
|
||||
op_state._scheduling_status,
|
||||
"under_resource_limits",
|
||||
False,
|
||||
is_method=False,
|
||||
):
|
||||
assert_autoscaling_action(
|
||||
delta=0,
|
||||
expected_reason="operator exceeding resource quota",
|
||||
)
|
||||
|
||||
# Should be a no-op since the op has enough available concurrency slots for
|
||||
# the existing inputs.
|
||||
with patch(actor_pool, "num_tasks_in_flight", 7):
|
||||
assert_autoscaling_action(
|
||||
delta=0,
|
||||
expected_reason="utilization of 0.7 w/in limits [0.5, 1.0]",
|
||||
)
|
||||
|
||||
# Should scale down since the util is below the threshold.
|
||||
with patch(actor_pool, "num_tasks_in_flight", 4):
|
||||
assert actor_pool.get_pool_util() == 0.4
|
||||
assert_autoscaling_action(
|
||||
delta=-1,
|
||||
expected_reason="utilization of 0.4 <= 0.5",
|
||||
)
|
||||
|
||||
# Should scale down since the pool is above the max size.
|
||||
with patch(actor_pool, "current_size", 16):
|
||||
assert_autoscaling_action(
|
||||
delta=-1,
|
||||
expected_reason="pool exceeding max size",
|
||||
)
|
||||
|
||||
# Should no-op because the op has no budget.
|
||||
with patch(resource_manager, "get_budget", ExecutionResources.zero()):
|
||||
assert_autoscaling_action(
|
||||
delta=0,
|
||||
expected_reason="exceeded resource limits",
|
||||
)
|
||||
|
||||
# Should no-op because the op has not received any inputs.
|
||||
with patch(op.metrics, "num_inputs_received", 0, is_method=False):
|
||||
assert_autoscaling_action(
|
||||
delta=0,
|
||||
expected_reason="no inputs received",
|
||||
)
|
||||
|
||||
# --- Resource budget enforcement (downscaling) ---
|
||||
# get_allocation and get_op_usage are patched to simulate an operator that
|
||||
# has exceeded its total resource allocation. The over-budget check fires
|
||||
# before utilization logic, so even high utilization (1.5x) is overridden.
|
||||
|
||||
# CPU over-budget by 2 actors: allocation=8 CPUs, usage=10 CPUs, 1 CPU/actor.
|
||||
# allocation - usage = -2 → scale down by ceil(2/1) = 2.
|
||||
with patch(resource_manager, "get_allocation", ExecutionResources(cpu=8)):
|
||||
with patch(resource_manager, "get_op_usage", ExecutionResources(cpu=10)):
|
||||
assert_autoscaling_action(
|
||||
delta=-2,
|
||||
expected_reason="actor pool exceeds resource allocation",
|
||||
)
|
||||
|
||||
# Over-budget but current_size=6 (min_size+1): required=2 but can only
|
||||
# release 1 actor (max_can_release = 6 - 5 = 1).
|
||||
with patch(resource_manager, "get_allocation", ExecutionResources(cpu=8)):
|
||||
with patch(resource_manager, "get_op_usage", ExecutionResources(cpu=10)):
|
||||
with patch(actor_pool, "current_size", 6):
|
||||
assert_autoscaling_action(
|
||||
delta=-1,
|
||||
expected_reason="actor pool exceeds resource allocation",
|
||||
)
|
||||
|
||||
# Over-budget but pool is at min_size (current=5): cannot release any actors.
|
||||
with patch(resource_manager, "get_allocation", ExecutionResources(cpu=8)):
|
||||
with patch(resource_manager, "get_op_usage", ExecutionResources(cpu=10)):
|
||||
with patch(actor_pool, "current_size", 5):
|
||||
assert_autoscaling_action(
|
||||
delta=0,
|
||||
expected_reason="actor pool exceeds resource allocation "
|
||||
"but cannot scale below min size",
|
||||
)
|
||||
|
||||
# GPU pool: allocation=3 GPUs, usage=6 GPUs, 1 GPU/actor.
|
||||
# allocation - usage = -3 → scale down by 3.
|
||||
with patch(actor_pool, "per_actor_resource_usage", ExecutionResources(gpu=1)):
|
||||
with patch(resource_manager, "get_allocation", ExecutionResources(gpu=3)):
|
||||
with patch(resource_manager, "get_op_usage", ExecutionResources(gpu=6)):
|
||||
assert_autoscaling_action(
|
||||
delta=-3,
|
||||
expected_reason="actor pool exceeds resource allocation",
|
||||
)
|
||||
|
||||
# Cross-resource: GPU-only pool (per_actor.cpu=0) with negative CPU budget
|
||||
# but positive GPU budget. CPU over-budget doesn't trigger since the pool
|
||||
# doesn't consume CPU. GPU headroom = floor(5/1)=5, capped by
|
||||
# max_size(15)-current_size(10)=5.
|
||||
with patch(actor_pool, "per_actor_resource_usage", ExecutionResources(gpu=1)):
|
||||
with patch(
|
||||
resource_manager, "get_allocation", ExecutionResources(cpu=8, gpu=10)
|
||||
):
|
||||
with patch(
|
||||
resource_manager, "get_op_usage", ExecutionResources(cpu=10, gpu=5)
|
||||
):
|
||||
assert_autoscaling_action(
|
||||
delta=5,
|
||||
expected_reason="utilization of 1.5 >= 1.0",
|
||||
)
|
||||
|
||||
# Memory bottleneck: allocation=4 GB, usage=5 GB, 500 MB/actor.
|
||||
# allocation - usage = -1 GB → ceil(1 GB / 500 MB) = 2 actors to remove.
|
||||
# CPU is within budget (allocation.cpu > usage.cpu), so CPU does not trigger.
|
||||
with patch(
|
||||
actor_pool,
|
||||
"per_actor_resource_usage",
|
||||
ExecutionResources(cpu=1, memory=500_000_000),
|
||||
):
|
||||
with patch(
|
||||
resource_manager,
|
||||
"get_allocation",
|
||||
ExecutionResources(cpu=15, memory=4_000_000_000),
|
||||
):
|
||||
with patch(
|
||||
resource_manager,
|
||||
"get_op_usage",
|
||||
ExecutionResources(cpu=10, memory=5_000_000_000),
|
||||
):
|
||||
assert_autoscaling_action(
|
||||
delta=-2,
|
||||
expected_reason="actor pool exceeds resource allocation",
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def autoscaler_max_upscaling_delta_setup():
|
||||
resource_manager = MagicMock(
|
||||
spec=ResourceManager,
|
||||
get_budget=MagicMock(return_value=None),
|
||||
get_allocation=MagicMock(return_value=None),
|
||||
)
|
||||
|
||||
actor_pool = MagicMock(
|
||||
spec=_ActorPool,
|
||||
min_size=MagicMock(return_value=5),
|
||||
max_size=MagicMock(return_value=20),
|
||||
current_size=MagicMock(return_value=10),
|
||||
get_current_size=MagicMock(return_value=10),
|
||||
num_pending_actors=MagicMock(return_value=0),
|
||||
num_tasks_in_flight=MagicMock(return_value=40),
|
||||
max_tasks_in_flight_per_actor=MagicMock(return_value=4),
|
||||
get_pool_util=MagicMock(return_value=2.0),
|
||||
)
|
||||
|
||||
op = MagicMock(
|
||||
spec=InternalQueueOperatorMixin,
|
||||
has_completed=MagicMock(return_value=False),
|
||||
_inputs_complete=False,
|
||||
metrics=MagicMock(average_num_inputs_per_task=1, num_inputs_received=1),
|
||||
)
|
||||
op_state = MagicMock(
|
||||
spec=OpState,
|
||||
total_enqueued_input_blocks=MagicMock(return_value=1),
|
||||
)
|
||||
op_state.op = op
|
||||
op_state._scheduling_status = MagicMock(under_resource_limits=True)
|
||||
return resource_manager, actor_pool, op, op_state
|
||||
|
||||
|
||||
def test_actor_pool_scaling_respects_small_max_upscaling_delta(
|
||||
autoscaler_max_upscaling_delta_setup,
|
||||
):
|
||||
resource_manager, actor_pool, op, op_state = autoscaler_max_upscaling_delta_setup
|
||||
autoscaler = DefaultActorAutoscaler(
|
||||
topology=MagicMock(),
|
||||
resource_manager=resource_manager,
|
||||
config=AutoscalingConfig(
|
||||
actor_pool_util_upscaling_threshold=1.0,
|
||||
actor_pool_util_downscaling_threshold=0.5,
|
||||
actor_pool_max_upscaling_delta=3,
|
||||
),
|
||||
)
|
||||
request = autoscaler._derive_target_scaling_config(
|
||||
actor_pool=actor_pool,
|
||||
op=op,
|
||||
op_state=op_state,
|
||||
)
|
||||
# With current_size=10, util=2.0, threshold=1.0:
|
||||
# plan_delta = ceil(10 * (2.0/1.0 - 1)) = ceil(10) = 10
|
||||
# However, delta is limited by max_upscaling_delta=3, so delta = min(10, 3) = 3
|
||||
assert request.delta == 3
|
||||
|
||||
|
||||
def test_actor_pool_scaling_respects_large_max_upscaling_delta(
|
||||
autoscaler_max_upscaling_delta_setup,
|
||||
):
|
||||
resource_manager, actor_pool, op, op_state = autoscaler_max_upscaling_delta_setup
|
||||
autoscaler = DefaultActorAutoscaler(
|
||||
topology=MagicMock(),
|
||||
resource_manager=resource_manager,
|
||||
config=AutoscalingConfig(
|
||||
actor_pool_util_upscaling_threshold=1.0,
|
||||
actor_pool_util_downscaling_threshold=0.5,
|
||||
actor_pool_max_upscaling_delta=100,
|
||||
),
|
||||
)
|
||||
request = autoscaler._derive_target_scaling_config(
|
||||
actor_pool=actor_pool,
|
||||
op=op,
|
||||
op_state=op_state,
|
||||
)
|
||||
# With current_size=10, util=2.0, threshold=1.0:
|
||||
# plan_delta = ceil(10 * (2.0/1.0 - 1)) = ceil(10) = 10
|
||||
# max_upscaling_delta=100 is large enough, but delta is limited by max_size:
|
||||
# max_size(20) - current_size(10) = 10, so delta = min(10, 100, 10) = 10
|
||||
assert request.delta == 10
|
||||
|
||||
|
||||
class BarrierWaiter:
|
||||
def __init__(self, barrier):
|
||||
self._barrier = barrier
|
||||
|
||||
def __call__(self, x):
|
||||
ray.get(self._barrier.wait.remote(), timeout=10)
|
||||
return x
|
||||
|
||||
|
||||
@ray.remote(max_concurrency=10)
|
||||
class Barrier:
|
||||
def __init__(self, n, delay=0):
|
||||
self.n = n
|
||||
self.delay = delay
|
||||
self.max_waiters = 0
|
||||
self.cur_waiters = 0
|
||||
|
||||
def wait(self):
|
||||
self.cur_waiters += 1
|
||||
if self.cur_waiters > self.max_waiters:
|
||||
self.max_waiters = self.cur_waiters
|
||||
self.n -= 1
|
||||
print("wait", self.n)
|
||||
while self.n > 0:
|
||||
time.sleep(0.1)
|
||||
time.sleep(self.delay)
|
||||
print("wait done")
|
||||
self.cur_waiters -= 1
|
||||
|
||||
def get_max_waiters(self):
|
||||
return self.max_waiters
|
||||
|
||||
|
||||
def test_actor_pool_scales_up(ray_start_10_cpus_shared, restore_data_context):
|
||||
# The Ray cluster started by the fixture might not have much object store memory.
|
||||
# To prevent the actor pool from getting backpressured, we decrease the max block
|
||||
# size.
|
||||
ctx = ray.data.DataContext.get_current()
|
||||
ctx.target_max_block_size = 1 * 1024**2
|
||||
|
||||
# The `BarrierWaiter` UDF blocks until there are 2 actors running. If we don't
|
||||
# scale up, the UDF raises a timeout.
|
||||
barrier = Barrier.remote(2)
|
||||
# We produce 3 blocks (1 elem each) such that
|
||||
# - We start wiht actor pool of min_size
|
||||
# - 2 tasks could be submitted to an actor (utilization reaches 200%)
|
||||
# - Autoscaler kicks in and creates another actor
|
||||
# - 3 task is submitted to a new actor (unblocking the barrier)
|
||||
ray.data.range(3, override_num_blocks=3).map(
|
||||
BarrierWaiter,
|
||||
fn_constructor_args=(barrier,),
|
||||
compute=ray.data.ActorPoolStrategy(
|
||||
min_size=1, max_size=2, max_tasks_in_flight_per_actor=2
|
||||
),
|
||||
).take_all()
|
||||
|
||||
|
||||
def test_actor_pool_respects_max_size(ray_start_10_cpus_shared, restore_data_context):
|
||||
# The Ray cluster started by the fixture might not have much object store memory.
|
||||
# To prevent the actor pool from getting backpressured, we decrease the max block
|
||||
# size.
|
||||
ctx = ray.data.DataContext.get_current()
|
||||
ctx.target_max_block_size = 1 * 1024**2
|
||||
|
||||
# The `BarrierWaiter` UDF blocks until there are 3 actors running. Since the max
|
||||
# pool size is 2, the UDF should eventually timeout.
|
||||
barrier = Barrier.remote(3)
|
||||
with pytest.raises(ray.exceptions.RayTaskError):
|
||||
ray.data.range(2, override_num_blocks=2).map(
|
||||
BarrierWaiter,
|
||||
fn_constructor_args=(barrier,),
|
||||
compute=ray.data.ActorPoolStrategy(min_size=1, max_size=2),
|
||||
).take_all()
|
||||
|
||||
|
||||
def test_autoscaling_config_validation_warnings(
|
||||
ray_start_10_cpus_shared, restore_data_context
|
||||
):
|
||||
"""Test that validation warnings are emitted when actor pool config won't allow scaling up."""
|
||||
|
||||
class SimpleMapper:
|
||||
"""Simple callable class for testing autoscaling validation."""
|
||||
|
||||
def __call__(self, row):
|
||||
# Map operates on rows which are dicts
|
||||
return {"value": row["id"] * 2}
|
||||
|
||||
# Test #1: Invalid config (should warn)
|
||||
# - max_tasks_in_flight / max_concurrency == 1
|
||||
# - Default upscaling threshold (200%)
|
||||
with patch(
|
||||
"ray.data._internal.actor_autoscaler.default_actor_autoscaler.logger.warning"
|
||||
) as mock_warning:
|
||||
ds = ray.data.range(2, override_num_blocks=2).map_batches(
|
||||
SimpleMapper,
|
||||
compute=ray.data.ActorPoolStrategy(
|
||||
max_tasks_in_flight_per_actor=1,
|
||||
),
|
||||
max_concurrency=1,
|
||||
)
|
||||
# Take just one item to minimize execution time
|
||||
ds.take_all()
|
||||
|
||||
# Check that warning was called with expected message
|
||||
warn_log_args_str = str(mock_warning.call_args_list)
|
||||
expected_message = (
|
||||
"⚠️ Actor Pool configuration of the "
|
||||
"ActorPoolMapOperator[MapBatches(SimpleMapper)] will not allow it to scale up: "
|
||||
"configured utilization threshold (175.0%) couldn't be reached with "
|
||||
"configured max_concurrency=1 and max_tasks_in_flight_per_actor=1 "
|
||||
"(max utilization will be max_tasks_in_flight_per_actor / max_concurrency = 100%)"
|
||||
)
|
||||
|
||||
assert expected_message in warn_log_args_str
|
||||
|
||||
# Test #2: Provided config is valid (no warnings)
|
||||
# - max_tasks_in_flight / max_concurrency == 2 (default)
|
||||
# - Default upscaling threshold (200%)
|
||||
with patch(
|
||||
"ray.data._internal.actor_autoscaler.default_actor_autoscaler.logger.warning"
|
||||
) as mock_warning:
|
||||
ds = ray.data.range(2, override_num_blocks=2).map_batches(
|
||||
SimpleMapper,
|
||||
compute=ray.data.ActorPoolStrategy(
|
||||
max_tasks_in_flight_per_actor=2,
|
||||
),
|
||||
max_concurrency=1,
|
||||
)
|
||||
ds.take_all()
|
||||
|
||||
# Check that this warning hasn't been emitted
|
||||
warn_log_args_str = str(mock_warning.call_args_list)
|
||||
expected_message = (
|
||||
"⚠️ Actor Pool configuration of the "
|
||||
"ActorPoolMapOperator[MapBatches(SimpleMapper)] will not allow it to scale up: "
|
||||
)
|
||||
|
||||
assert expected_message not in warn_log_args_str
|
||||
|
||||
# Test #3: Default config is valid (no warnings)
|
||||
# - max_tasks_in_flight / max_concurrency == 4 (default)
|
||||
# - Default upscaling threshold (200%)
|
||||
with patch(
|
||||
"ray.data._internal.actor_autoscaler.default_actor_autoscaler.logger.warning"
|
||||
) as mock_warning:
|
||||
ds = ray.data.range(2, override_num_blocks=2).map_batches(
|
||||
SimpleMapper, compute=ray.data.ActorPoolStrategy()
|
||||
)
|
||||
ds.take_all()
|
||||
|
||||
# Check that this warning hasn't been emitted
|
||||
warn_log_args_str = str(mock_warning.call_args_list)
|
||||
expected_message = (
|
||||
"⚠️ Actor Pool configuration of the "
|
||||
"ActorPoolMapOperator[MapBatches(SimpleMapper)] will not allow it to scale up: "
|
||||
)
|
||||
|
||||
assert expected_message not in warn_log_args_str
|
||||
|
||||
# Test #4: Fixed-size pool with invalid config (no warnings)
|
||||
# - max_tasks_in_flight / max_concurrency == 1
|
||||
# - Default upscaling threshold (200%)
|
||||
# - Even though config would normally trigger warning, fixed-size pools
|
||||
# don't scale up by design, so warning should not be emitted
|
||||
with patch(
|
||||
"ray.data._internal.actor_autoscaler.default_actor_autoscaler.logger.warning"
|
||||
) as mock_warning:
|
||||
ds = ray.data.range(2, override_num_blocks=2).map_batches(
|
||||
SimpleMapper,
|
||||
compute=ray.data.ActorPoolStrategy(
|
||||
size=2,
|
||||
max_tasks_in_flight_per_actor=1,
|
||||
),
|
||||
max_concurrency=1,
|
||||
)
|
||||
ds.take_all()
|
||||
|
||||
# Check that this warning hasn't been emitted for fixed-size pool
|
||||
warn_log_args_str = str(mock_warning.call_args_list)
|
||||
expected_message = (
|
||||
"⚠️ Actor Pool configuration of the "
|
||||
"ActorPoolMapOperator[MapBatches(SimpleMapper)] will not allow it to scale up: "
|
||||
)
|
||||
|
||||
assert expected_message not in warn_log_args_str
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def autoscaler_config_mocks():
|
||||
resource_manager = MagicMock(spec=ResourceManager)
|
||||
topology = MagicMock()
|
||||
topology.items = MagicMock(return_value=[])
|
||||
return resource_manager, topology
|
||||
|
||||
|
||||
def test_autoscaling_config_validation_zero_delta(autoscaler_config_mocks):
|
||||
resource_manager, topology = autoscaler_config_mocks
|
||||
|
||||
with pytest.raises(
|
||||
ValueError, match="actor_pool_max_upscaling_delta must be positive"
|
||||
):
|
||||
DefaultActorAutoscaler(
|
||||
topology=topology,
|
||||
resource_manager=resource_manager,
|
||||
config=AutoscalingConfig(
|
||||
actor_pool_util_upscaling_threshold=1.0,
|
||||
actor_pool_util_downscaling_threshold=0.5,
|
||||
actor_pool_max_upscaling_delta=0,
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def test_autoscaling_config_validation_negative_delta(autoscaler_config_mocks):
|
||||
resource_manager, topology = autoscaler_config_mocks
|
||||
|
||||
with pytest.raises(
|
||||
ValueError, match="actor_pool_max_upscaling_delta must be positive"
|
||||
):
|
||||
DefaultActorAutoscaler(
|
||||
topology=topology,
|
||||
resource_manager=resource_manager,
|
||||
config=AutoscalingConfig(
|
||||
actor_pool_util_upscaling_threshold=1.0,
|
||||
actor_pool_util_downscaling_threshold=0.5,
|
||||
actor_pool_max_upscaling_delta=-1,
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def test_autoscaling_config_validation_positive_delta(autoscaler_config_mocks):
|
||||
resource_manager, topology = autoscaler_config_mocks
|
||||
|
||||
autoscaler = DefaultActorAutoscaler(
|
||||
topology=topology,
|
||||
resource_manager=resource_manager,
|
||||
config=AutoscalingConfig(
|
||||
actor_pool_util_upscaling_threshold=1.0,
|
||||
actor_pool_util_downscaling_threshold=0.5,
|
||||
actor_pool_max_upscaling_delta=5,
|
||||
),
|
||||
)
|
||||
assert autoscaler._actor_pool_max_upscaling_delta == 5
|
||||
|
||||
|
||||
def test_autoscaling_config_validation_zero_upscaling_threshold(
|
||||
autoscaler_config_mocks,
|
||||
):
|
||||
resource_manager, topology = autoscaler_config_mocks
|
||||
|
||||
with pytest.raises(
|
||||
ValueError, match="actor_pool_util_upscaling_threshold must be positive"
|
||||
):
|
||||
DefaultActorAutoscaler(
|
||||
topology=topology,
|
||||
resource_manager=resource_manager,
|
||||
config=AutoscalingConfig(
|
||||
actor_pool_util_upscaling_threshold=0,
|
||||
actor_pool_util_downscaling_threshold=0.5,
|
||||
actor_pool_max_upscaling_delta=5,
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def test_autoscaling_config_validation_negative_upscaling_threshold(
|
||||
autoscaler_config_mocks,
|
||||
):
|
||||
resource_manager, topology = autoscaler_config_mocks
|
||||
|
||||
with pytest.raises(
|
||||
ValueError, match="actor_pool_util_upscaling_threshold must be positive"
|
||||
):
|
||||
DefaultActorAutoscaler(
|
||||
topology=topology,
|
||||
resource_manager=resource_manager,
|
||||
config=AutoscalingConfig(
|
||||
actor_pool_util_upscaling_threshold=-1.0,
|
||||
actor_pool_util_downscaling_threshold=0.5,
|
||||
actor_pool_max_upscaling_delta=5,
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def test_get_max_scale_up_tolerates_float_drift():
|
||||
"""Regression test for #64291.
|
||||
|
||||
A budget can carry tiny float drift (e.g. ``gpu=-1e-16``) from chained
|
||||
arithmetic. ``_get_max_scale_up`` reads raw fields (non-negativity assert +
|
||||
``floordiv``), so this must not trip the assert or yield a negative
|
||||
scale-up. ``ExecutionResources`` rounds at construction, so the drift
|
||||
collapses to 0.
|
||||
"""
|
||||
actor_pool = MagicMock()
|
||||
actor_pool.per_actor_resource_usage = MagicMock(
|
||||
return_value=ExecutionResources(cpu=1.0, gpu=0.25, memory=0.0)
|
||||
)
|
||||
# gpu drift rounds to 0 -> 0 actors fit on the gpu dimension -> scale-up 0.
|
||||
budget = ExecutionResources(cpu=4, gpu=-1e-16, memory=0.0)
|
||||
assert _get_max_scale_up(actor_pool, budget) == 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
import sys
|
||||
|
||||
sys.exit(pytest.main(["-v", __file__]))
|
||||
Reference in New Issue
Block a user