import os import sys import time import pytest from ray.autoscaler.v2.instance_manager.config import NodeTypeConfig from ray.autoscaler.v2.instance_manager.subscribers.cloud_resource_monitor import ( CloudResourceMonitor, ) from ray.autoscaler.v2.scheduler import ( ResourceDemandScheduler, ResourceRequestSource, SchedulingNode, SchedulingNodeStatus, SchedulingRequest, ) from ray.core.generated.autoscaler_pb2 import ResourceRequest as PBResourceRequest from ray.core.generated.instance_manager_pb2 import ( Instance, InstanceUpdateEvent, NodeKind, ) def test_recovery_scoring(): monitor = CloudResourceMonitor() node_type = "gpu-node" # Mock failure event = InstanceUpdateEvent( instance_type=node_type, new_instance_status=Instance.ALLOCATION_TIMEOUT ) monitor.notify([event]) # Immediately after failure, score should be 0.0 scores = monitor.get_recoverable_resource_availabilities() assert scores[node_type] == 0.0 # After safety floor (e.g., 11s, default safety floor is 10s) monitor._last_unavailable_timestamp[node_type] = time.time() - 11 scores = monitor.get_recoverable_resource_availabilities() assert 0.0 < scores[node_type] < 0.1 # Halfway through recovery window (600s / 2 = 300s) monitor._last_unavailable_timestamp[node_type] = time.time() - 300 scores = monitor.get_recoverable_resource_availabilities() # 300 / 600 = 0.5 assert pytest.approx(scores[node_type], 0.01) == 0.5 # After recovery window monitor._last_unavailable_timestamp[node_type] = time.time() - 601 scores = monitor.get_recoverable_resource_availabilities() assert scores[node_type] == 1.0 def test_scheduler_priority_tie_breaking(): # Two node types with identical resources resources = {"CPU": 4} node_type_1 = "high-priority" node_type_2 = "low-priority" config_1 = NodeTypeConfig( name=node_type_1, min_worker_nodes=0, max_worker_nodes=10, resources=resources, priority=10, ) config_2 = NodeTypeConfig( name=node_type_2, min_worker_nodes=0, max_worker_nodes=10, resources=resources, priority=0, ) node_1 = SchedulingNode.from_node_config( config_1, status=SchedulingNodeStatus.TO_LAUNCH, node_kind=NodeKind.WORKER ) node_2 = SchedulingNode.from_node_config( config_2, status=SchedulingNodeStatus.TO_LAUNCH, node_kind=NodeKind.WORKER ) request = PBResourceRequest(resources_bundle={"CPU": 1}) # Utilization and Availability are equal (all perfect) # Priority should break the tie best_node, infeasible, remaining_nodes = ResourceDemandScheduler._sched_best_node( [request], [node_2, node_1], ResourceRequestSource.PENDING_DEMAND, {}, {} ) assert best_node.node_type == node_type_1 def test_schedule_context_propagation(): resources = {"CPU": 4} node_type = "gpu-node" config = NodeTypeConfig( name=node_type, min_worker_nodes=0, max_worker_nodes=10, resources=resources, priority=7, ) # Mock cloud availabilities cloud_availabilities = {node_type: 0.1} # failure recency recoverable_availabilities = {node_type: 0.5} req = SchedulingRequest( node_type_configs={node_type: config}, cloud_resource_availabilities=cloud_availabilities, recoverable_resource_availabilities=recoverable_availabilities, disable_launch_config_check=True, ) ctx = ResourceDemandScheduler.ScheduleContext.from_schedule_request(req) # Check if a new node created from this context has the correct priority node_pools = [ SchedulingNode.from_node_config( ctx.get_node_type_configs()[nt], status=SchedulingNodeStatus.TO_LAUNCH, node_kind=NodeKind.WORKER, ) for nt, num_available in ctx.get_node_type_available().items() ] assert len(node_pools) == 1 node = node_pools[0] assert node.priority == 7 # Dynamic scores are in the context, not the node assert ctx.get_recoverable_resource_availabilities()[node_type] == 0.5 assert ctx.get_cloud_resource_availabilities()[node_type] == 0.1 def test_scheduler_availability_over_priority(): # High priority node is recovering (score 0.5) # Low priority node is available (score 1.0) resources = {"CPU": 4} node_type_1 = "high-priority" node_type_2 = "low-priority" config_1 = NodeTypeConfig( name=node_type_1, min_worker_nodes=0, max_worker_nodes=10, resources=resources, priority=10, ) config_2 = NodeTypeConfig( name=node_type_2, min_worker_nodes=0, max_worker_nodes=10, resources=resources, priority=0, ) node_1 = SchedulingNode.from_node_config( config_1, status=SchedulingNodeStatus.TO_LAUNCH, node_kind=NodeKind.WORKER ) node_2 = SchedulingNode.from_node_config( config_2, status=SchedulingNodeStatus.TO_LAUNCH, node_kind=NodeKind.WORKER ) request = PBResourceRequest(resources_bundle={"CPU": 1}) # Recoverable Availability is higher for node_2, so it should be chosen despite lower priority best_node, infeasible, remaining_nodes = ResourceDemandScheduler._sched_best_node( [request], [node_1, node_2], ResourceRequestSource.PENDING_DEMAND, cloud_resource_availabilities={}, recoverable_resource_availabilities={node_type_1: 0.5, node_type_2: 1.0}, ) assert best_node.node_type == node_type_2 def test_scheduler_failure_recency_tie_breaking(): # Same priority, same recoverable availability (1.0) # One has an older failure than the other. resources = {"CPU": 4} node_type_1 = "older-failure" node_type_2 = "newer-failure" config_1 = NodeTypeConfig( name=node_type_1, min_worker_nodes=0, max_worker_nodes=10, resources=resources, priority=5, ) config_2 = NodeTypeConfig( name=node_type_2, min_worker_nodes=0, max_worker_nodes=10, resources=resources, priority=5, ) node_1 = SchedulingNode.from_node_config( config_1, status=SchedulingNodeStatus.TO_LAUNCH, node_kind=NodeKind.WORKER ) node_2 = SchedulingNode.from_node_config( config_2, status=SchedulingNodeStatus.TO_LAUNCH, node_kind=NodeKind.WORKER ) request = PBResourceRequest(resources_bundle={"CPU": 1}) best_node, infeasible, remaining_nodes = ResourceDemandScheduler._sched_best_node( [request], [node_1, node_2], ResourceRequestSource.PENDING_DEMAND, cloud_resource_availabilities={node_type_1: 0.9, node_type_2: 0.1}, recoverable_resource_availabilities={node_type_1: 1.0, node_type_2: 1.0}, ) assert best_node.node_type == node_type_1 def test_recovery_integration(): monitor = CloudResourceMonitor() node_type_1 = "high-priority" node_type_2 = "low-priority" # Mock failure for node_1 (high priority) event = InstanceUpdateEvent( instance_type=node_type_1, new_instance_status=Instance.ALLOCATION_TIMEOUT ) monitor.notify([event]) # Score should be 0.0 for node_1 immediately scores = monitor.get_recoverable_resource_availabilities() assert scores[node_type_1] == 0.0 # Setup scheduler structures resources = {"CPU": 4} config_1 = NodeTypeConfig( name=node_type_1, min_worker_nodes=0, max_worker_nodes=10, resources=resources, priority=10, ) config_2 = NodeTypeConfig( name=node_type_2, min_worker_nodes=0, max_worker_nodes=10, resources=resources, priority=0, ) node_1 = SchedulingNode.from_node_config( config_1, status=SchedulingNodeStatus.TO_LAUNCH, node_kind=NodeKind.WORKER ) node_2 = SchedulingNode.from_node_config( config_2, status=SchedulingNodeStatus.TO_LAUNCH, node_kind=NodeKind.WORKER ) request = PBResourceRequest(resources_bundle={"CPU": 1}) # Pass scores from monitor to scheduler best_node, infeasible, remaining_nodes = ResourceDemandScheduler._sched_best_node( [request], [node_1, node_2], ResourceRequestSource.PENDING_DEMAND, cloud_resource_availabilities={}, recoverable_resource_availabilities=scores, ) # Node 2 should be chosen because Node 1 is recovering, despite Node 1 having higher priority assert best_node.node_type == node_type_2 def test_scheduler_utilization_over_priority(): # Node 1: 4 CPUs, priority 10 # Node 2: 2 CPUs, priority 0 # Request: 2 CPUs # Node 2 should be selected because it fits perfectly (utilization score is higher), # even though Node 1 has higher priority. resources_1 = {"CPU": 4} resources_2 = {"CPU": 2} node_type_1 = "large-node" node_type_2 = "small-node" config_1 = NodeTypeConfig( name=node_type_1, min_worker_nodes=0, max_worker_nodes=10, resources=resources_1, priority=10, ) config_2 = NodeTypeConfig( name=node_type_2, min_worker_nodes=0, max_worker_nodes=10, resources=resources_2, priority=0, ) node_1 = SchedulingNode.from_node_config( config_1, status=SchedulingNodeStatus.TO_LAUNCH, node_kind=NodeKind.WORKER ) node_2 = SchedulingNode.from_node_config( config_2, status=SchedulingNodeStatus.TO_LAUNCH, node_kind=NodeKind.WORKER ) request = PBResourceRequest(resources_bundle={"CPU": 2}) best_node, infeasible, remaining_nodes = ResourceDemandScheduler._sched_best_node( [request], [node_1, node_2], ResourceRequestSource.PENDING_DEMAND, cloud_resource_availabilities={}, recoverable_resource_availabilities={}, ) assert best_node.node_type == node_type_2 if __name__ == "__main__": if os.environ.get("PARALLEL_CI"): sys.exit(pytest.main(["-n", "auto", "--boxed", "-vs", __file__])) else: sys.exit(pytest.main(["-sv", __file__]))