ray-project--ray/python/ray/tests/test_autoscaler_e2e.py

import subprocess
import sys

import pytest

import ray
from ray._common.network_utils import build_address
from ray._common.test_utils import (
    MetricSamplePattern,
    PrometheusTimeseries,
    SignalActor,
    wait_for_condition,
)
from ray._private.test_utils import get_metric_check_condition
from ray.autoscaler._private.constants import AUTOSCALER_METRIC_PORT
from ray.autoscaler.node_launch_exception import NodeLaunchException


@pytest.mark.parametrize(
    "local_autoscaling_cluster",
    [
        (
            {"CPU": 0},
            {
                "type-i": {
                    "resources": {"CPU": 4, "fun": 1},
                    "node_config": {},
                    "min_workers": 1,
                    "max_workers": 1,
                },
                "type-ii": {
                    "resources": {"CPU": 3, "fun": 100},
                    "node_config": {},
                    "min_workers": 1,
                    "max_workers": 1,
                },
            },
            None,
        )
    ],
    indirect=["local_autoscaling_cluster"],
)
@pytest.mark.parametrize("enable_v2", [True, False], ids=["v2", "v1"])
def test_ray_status_activity(local_autoscaling_cluster, shutdown_only, enable_v2):

    ray.init(address="auto")
    if enable_v2:
        wait_for_condition(
            lambda: subprocess.run(
                "ray status --verbose", shell=True, capture_output=True
            )
            .stdout.decode()
            .count("Idle: ")
            > 0
        )

    @ray.remote(num_cpus=2, resources={"fun": 2})
    class Actor:
        def ping(self):
            return None

    actor = Actor.remote()
    ray.get(actor.ping.remote())

    occurrences = 1 if enable_v2 else 0
    assert (
        subprocess.check_output("ray status --verbose", shell=True)
        .decode()
        .count("Resource: CPU currently in use.")
        == occurrences
    )

    from ray.util.placement_group import placement_group

    pg = placement_group([{"CPU": 2}], strategy="STRICT_SPREAD")
    ray.get(pg.ready())

    occurrences = 2 if enable_v2 else 0
    assert (
        subprocess.check_output("ray status --verbose", shell=True)
        .decode()
        .count("Resource: CPU currently in use.")
        == occurrences
    )

    assert (
        subprocess.check_output("ray status --verbose", shell=True)
        .decode()
        .count("Resource: bundle_group_")
        == 0
    )


@pytest.mark.parametrize(
    "local_autoscaling_cluster",
    [
        (
            {"CPU": 0},
            {
                "type-i": {
                    "resources": {"CPU": 1, "fun": 1},
                    "node_config": {},
                    "min_workers": 1,
                    "max_workers": 1,
                },
                "type-ii": {
                    "resources": {"CPU": 1, "fun": 100},
                    "node_config": {},
                    "min_workers": 1,
                    "max_workers": 1,
                },
            },
            None,
        )
    ],
    indirect=["local_autoscaling_cluster"],
)
@pytest.mark.parametrize("enable_v2", [True, False], ids=["v2", "v1"])
def test_ray_status_e2e(local_autoscaling_cluster, shutdown_only):

    ray.init(address="auto")

    @ray.remote(num_cpus=0, resources={"fun": 2})
    class Actor:
        def ping(self):
            return None

    actor = Actor.remote()
    ray.get(actor.ping.remote())

    assert (
        "Pending Demands" in subprocess.check_output("ray status", shell=True).decode()
    )
    assert (
        "Pending Demands"
        in subprocess.check_output("ray status -v", shell=True).decode()
    )
    assert (
        "Pending Demands"
        in subprocess.check_output("ray status --verbose", shell=True).decode()
    )


@pytest.mark.parametrize(
    "local_autoscaling_cluster",
    [
        (
            {"CPU": 0},
            {
                "type-i": {
                    "resources": {"CPU": 1},
                    "node_config": {},
                    "min_workers": 0,
                    "max_workers": 1,
                },
                "type-ii": {
                    "resources": {"CPU": 1},
                    "node_config": {},
                    "min_workers": 0,
                    "max_workers": 1,
                },
            },
            None,
        )
    ],
    indirect=["local_autoscaling_cluster"],
)
@pytest.mark.parametrize("enable_v2", [False, True], ids=["v2", "v1"])
def test_metrics(local_autoscaling_cluster, shutdown_only):

    info = ray.init(address="auto")
    autoscaler_export_addr = build_address(
        info.address_info["node_ip_address"], AUTOSCALER_METRIC_PORT
    )

    @ray.remote(num_cpus=1)
    class Foo:
        def ping(self):
            return True

    timeseries = PrometheusTimeseries()
    zero_reported_condition = get_metric_check_condition(
        [
            MetricSamplePattern(
                name="autoscaler_cluster_resources",
                value=0,
                partial_label_match={"resource": "CPU"},
            ),
            MetricSamplePattern(name="autoscaler_pending_resources", value=0),
            MetricSamplePattern(name="autoscaler_pending_nodes", value=0),
            MetricSamplePattern(
                name="autoscaler_active_nodes",
                value=0,
                partial_label_match={"NodeType": "type-i"},
            ),
            MetricSamplePattern(
                name="autoscaler_active_nodes",
                value=0,
                partial_label_match={"NodeType": "type-ii"},
            ),
            MetricSamplePattern(
                name="autoscaler_active_nodes",
                value=1,
                partial_label_match={"NodeType": "ray.head.default"},
            ),
        ],
        timeseries,
        export_addr=autoscaler_export_addr,
    )
    wait_for_condition(zero_reported_condition)

    actors = [Foo.remote() for _ in range(2)]
    ray.get([actor.ping.remote() for actor in actors])

    two_cpu_no_pending_condition = get_metric_check_condition(
        [
            MetricSamplePattern(
                name="autoscaler_cluster_resources",
                value=2,
                partial_label_match={"resource": "CPU"},
            ),
            MetricSamplePattern(
                name="autoscaler_pending_nodes",
                value=0,
                partial_label_match={"NodeType": "type-i"},
            ),
            MetricSamplePattern(
                name="autoscaler_pending_nodes",
                value=0,
                partial_label_match={"NodeType": "type-ii"},
            ),
            MetricSamplePattern(
                name="autoscaler_active_nodes",
                value=1,
                partial_label_match={"NodeType": "type-i"},
            ),
            MetricSamplePattern(
                name="autoscaler_active_nodes",
                value=1,
                partial_label_match={"NodeType": "type-ii"},
            ),
            MetricSamplePattern(
                name="autoscaler_active_nodes",
                value=1,
                partial_label_match={"NodeType": "ray.head.default"},
            ),
        ],
        timeseries,
        export_addr=autoscaler_export_addr,
    )
    wait_for_condition(two_cpu_no_pending_condition)
    # TODO (Alex): Ideally we'd also assert that pending increases
    # eventually became 1 or 2, but it's difficult to do that in a
    # non-racey way. (Perhaps we would need to artificially delay the fake
    # autoscaler node launch?).


def test_node_launch_exception_serialization(shutdown_only):
    ray.init(num_cpus=1)

    exc_info = None
    try:
        raise Exception("Test exception.")
    except Exception:
        exc_info = sys.exc_info()
    assert exc_info is not None

    exc = NodeLaunchException("cat", "desc", exc_info)

    after_serialization = ray.get(ray.put(exc))

    assert after_serialization.category == exc.category
    assert after_serialization.description == exc.description
    assert after_serialization.src_exc_info is None


@pytest.mark.parametrize(
    "local_autoscaling_cluster",
    [
        (
            {"CPU": 0},
            {
                "type-i": {
                    "resources": {"CPU": 1},
                    "node_config": {},
                    "min_workers": 1,
                    "max_workers": 1,
                },
            },
            {"enable_infeasible_task_early_exit": True},
        )
    ],
    indirect=["local_autoscaling_cluster"],
)
@pytest.mark.parametrize("enable_v2", [True], ids=["v2"])
def test_infeasible_task_early_cancellation_normal_tasks(
    local_autoscaling_cluster, shutdown_only
):

    ray.init(address="auto")

    signal = SignalActor.remote()

    @ray.remote(num_cpus=1)
    def feasible_task():
        signal.wait.remote()
        return 1

    @ray.remote(num_cpus=10)
    def infeasible_task():
        return 2

    obj_feasible = feasible_task.remote()
    obj_infeasible = infeasible_task.remote()

    # The infeasible task should be cancelled with TaskUnschedulableError
    with pytest.raises(
        ray.exceptions.TaskUnschedulableError,
        match=r"The task is not schedulable: Tasks or actors with resource shapes \[{CPU: 10}] failed to schedule because there are not enough resources for the tasks or actors on the whole cluster.",
    ):
        ray.get(obj_infeasible, timeout=10)

    # The feasible task should continue to run successfully
    signal.send.remote()
    assert ray.get(obj_feasible, timeout=5) == 1


@pytest.mark.parametrize(
    "local_autoscaling_cluster",
    [
        (
            {"CPU": 0},
            {
                "type-i": {
                    "resources": {"CPU": 1},
                    "node_config": {},
                    "min_workers": 1,
                    "max_workers": 1,
                },
            },
            {"enable_infeasible_task_early_exit": True},
        )
    ],
    indirect=["local_autoscaling_cluster"],
)
@pytest.mark.parametrize("enable_v2", [True], ids=["v2"])
def test_infeasible_task_early_cancellation_actor_creation(
    local_autoscaling_cluster, shutdown_only
):

    ray.init(address="auto")

    signal = SignalActor.remote()

    @ray.remote(num_cpus=1)
    class FeasibleActor:
        def f(self):
            signal.wait.remote()
            return 1

    @ray.remote(num_cpus=10)
    class InfeasibleActor:
        def f(self):
            return 2

    feasible_actor = FeasibleActor.remote()
    infeasible_actor = InfeasibleActor.remote()

    # The infeasible actor should be cancelled with ActorUnschedulableError
    with pytest.raises(
        ray.exceptions.ActorUnschedulableError,
        match=r"The actor is not schedulable: Tasks or actors with resource shapes \[{CPU: 10}] failed to schedule because there are not enough resources for the tasks or actors on the whole cluster.",
    ):
        ray.get(infeasible_actor.f.remote(), timeout=5)

    # The feasible actor should continue to run successfully
    signal.send.remote()
    assert ray.get(feasible_actor.f.remote(), timeout=5) == 1


if __name__ == "__main__":
    sys.exit(pytest.main(["-sv", __file__]))