chore: import upstream snapshot with attribution

python/ray/tests/test_autoscaler_e2e.py

@@ -0,0 +1,382 @@
+import subprocess
+import sys
+
+import pytest
+
+import ray
+from ray._common.network_utils import build_address
+from ray._common.test_utils import (
+    MetricSamplePattern,
+    PrometheusTimeseries,
+    SignalActor,
+    wait_for_condition,
+)
+from ray._private.test_utils import get_metric_check_condition
+from ray.autoscaler._private.constants import AUTOSCALER_METRIC_PORT
+from ray.autoscaler.node_launch_exception import NodeLaunchException
+
+
+@pytest.mark.parametrize(
+    "local_autoscaling_cluster",
+    [
+        (
+            {"CPU": 0},
+            {
+                "type-i": {
+                    "resources": {"CPU": 4, "fun": 1},
+                    "node_config": {},
+                    "min_workers": 1,
+                    "max_workers": 1,
+                },
+                "type-ii": {
+                    "resources": {"CPU": 3, "fun": 100},
+                    "node_config": {},
+                    "min_workers": 1,
+                    "max_workers": 1,
+                },
+            },
+            None,
+        )
+    ],
+    indirect=["local_autoscaling_cluster"],
+)
+@pytest.mark.parametrize("enable_v2", [True, False], ids=["v2", "v1"])
+def test_ray_status_activity(local_autoscaling_cluster, shutdown_only, enable_v2):
+
+    ray.init(address="auto")
+    if enable_v2:
+        wait_for_condition(
+            lambda: subprocess.run(
+                "ray status --verbose", shell=True, capture_output=True
+            )
+            .stdout.decode()
+            .count("Idle: ")
+            > 0
+        )
+
+    @ray.remote(num_cpus=2, resources={"fun": 2})
+    class Actor:
+        def ping(self):
+            return None
+
+    actor = Actor.remote()
+    ray.get(actor.ping.remote())
+
+    occurrences = 1 if enable_v2 else 0
+    assert (
+        subprocess.check_output("ray status --verbose", shell=True)
+        .decode()
+        .count("Resource: CPU currently in use.")
+        == occurrences
+    )
+
+    from ray.util.placement_group import placement_group
+
+    pg = placement_group([{"CPU": 2}], strategy="STRICT_SPREAD")
+    ray.get(pg.ready())
+
+    occurrences = 2 if enable_v2 else 0
+    assert (
+        subprocess.check_output("ray status --verbose", shell=True)
+        .decode()
+        .count("Resource: CPU currently in use.")
+        == occurrences
+    )
+
+    assert (
+        subprocess.check_output("ray status --verbose", shell=True)
+        .decode()
+        .count("Resource: bundle_group_")
+        == 0
+    )
+
+
+@pytest.mark.parametrize(
+    "local_autoscaling_cluster",
+    [
+        (
+            {"CPU": 0},
+            {
+                "type-i": {
+                    "resources": {"CPU": 1, "fun": 1},
+                    "node_config": {},
+                    "min_workers": 1,
+                    "max_workers": 1,
+                },
+                "type-ii": {
+                    "resources": {"CPU": 1, "fun": 100},
+                    "node_config": {},
+                    "min_workers": 1,
+                    "max_workers": 1,
+                },
+            },
+            None,
+        )
+    ],
+    indirect=["local_autoscaling_cluster"],
+)
+@pytest.mark.parametrize("enable_v2", [True, False], ids=["v2", "v1"])
+def test_ray_status_e2e(local_autoscaling_cluster, shutdown_only):
+
+    ray.init(address="auto")
+
+    @ray.remote(num_cpus=0, resources={"fun": 2})
+    class Actor:
+        def ping(self):
+            return None
+
+    actor = Actor.remote()
+    ray.get(actor.ping.remote())
+
+    assert (
+        "Pending Demands" in subprocess.check_output("ray status", shell=True).decode()
+    )
+    assert (
+        "Pending Demands"
+        in subprocess.check_output("ray status -v", shell=True).decode()
+    )
+    assert (
+        "Pending Demands"
+        in subprocess.check_output("ray status --verbose", shell=True).decode()
+    )
+
+
+@pytest.mark.parametrize(
+    "local_autoscaling_cluster",
+    [
+        (
+            {"CPU": 0},
+            {
+                "type-i": {
+                    "resources": {"CPU": 1},
+                    "node_config": {},
+                    "min_workers": 0,
+                    "max_workers": 1,
+                },
+                "type-ii": {
+                    "resources": {"CPU": 1},
+                    "node_config": {},
+                    "min_workers": 0,
+                    "max_workers": 1,
+                },
+            },
+            None,
+        )
+    ],
+    indirect=["local_autoscaling_cluster"],
+)
+@pytest.mark.parametrize("enable_v2", [False, True], ids=["v2", "v1"])
+def test_metrics(local_autoscaling_cluster, shutdown_only):
+
+    info = ray.init(address="auto")
+    autoscaler_export_addr = build_address(
+        info.address_info["node_ip_address"], AUTOSCALER_METRIC_PORT
+    )
+
+    @ray.remote(num_cpus=1)
+    class Foo:
+        def ping(self):
+            return True
+
+    timeseries = PrometheusTimeseries()
+    zero_reported_condition = get_metric_check_condition(
+        [
+            MetricSamplePattern(
+                name="autoscaler_cluster_resources",
+                value=0,
+                partial_label_match={"resource": "CPU"},
+            ),
+            MetricSamplePattern(name="autoscaler_pending_resources", value=0),
+            MetricSamplePattern(name="autoscaler_pending_nodes", value=0),
+            MetricSamplePattern(
+                name="autoscaler_active_nodes",
+                value=0,
+                partial_label_match={"NodeType": "type-i"},
+            ),
+            MetricSamplePattern(
+                name="autoscaler_active_nodes",
+                value=0,
+                partial_label_match={"NodeType": "type-ii"},
+            ),
+            MetricSamplePattern(
+                name="autoscaler_active_nodes",
+                value=1,
+                partial_label_match={"NodeType": "ray.head.default"},
+            ),
+        ],
+        timeseries,
+        export_addr=autoscaler_export_addr,
+    )
+    wait_for_condition(zero_reported_condition)
+
+    actors = [Foo.remote() for _ in range(2)]
+    ray.get([actor.ping.remote() for actor in actors])
+
+    two_cpu_no_pending_condition = get_metric_check_condition(
+        [
+            MetricSamplePattern(
+                name="autoscaler_cluster_resources",
+                value=2,
+                partial_label_match={"resource": "CPU"},
+            ),
+            MetricSamplePattern(
+                name="autoscaler_pending_nodes",
+                value=0,
+                partial_label_match={"NodeType": "type-i"},
+            ),
+            MetricSamplePattern(
+                name="autoscaler_pending_nodes",
+                value=0,
+                partial_label_match={"NodeType": "type-ii"},
+            ),
+            MetricSamplePattern(
+                name="autoscaler_active_nodes",
+                value=1,
+                partial_label_match={"NodeType": "type-i"},
+            ),
+            MetricSamplePattern(
+                name="autoscaler_active_nodes",
+                value=1,
+                partial_label_match={"NodeType": "type-ii"},
+            ),
+            MetricSamplePattern(
+                name="autoscaler_active_nodes",
+                value=1,
+                partial_label_match={"NodeType": "ray.head.default"},
+            ),
+        ],
+        timeseries,
+        export_addr=autoscaler_export_addr,
+    )
+    wait_for_condition(two_cpu_no_pending_condition)
+    # TODO (Alex): Ideally we'd also assert that pending increases
+    # eventually became 1 or 2, but it's difficult to do that in a
+    # non-racey way. (Perhaps we would need to artificially delay the fake
+    # autoscaler node launch?).
+
+
+def test_node_launch_exception_serialization(shutdown_only):
+    ray.init(num_cpus=1)
+
+    exc_info = None
+    try:
+        raise Exception("Test exception.")
+    except Exception:
+        exc_info = sys.exc_info()
+    assert exc_info is not None
+
+    exc = NodeLaunchException("cat", "desc", exc_info)
+
+    after_serialization = ray.get(ray.put(exc))
+
+    assert after_serialization.category == exc.category
+    assert after_serialization.description == exc.description
+    assert after_serialization.src_exc_info is None
+
+
+@pytest.mark.parametrize(
+    "local_autoscaling_cluster",
+    [
+        (
+            {"CPU": 0},
+            {
+                "type-i": {
+                    "resources": {"CPU": 1},
+                    "node_config": {},
+                    "min_workers": 1,
+                    "max_workers": 1,
+                },
+            },
+            {"enable_infeasible_task_early_exit": True},
+        )
+    ],
+    indirect=["local_autoscaling_cluster"],
+)
+@pytest.mark.parametrize("enable_v2", [True], ids=["v2"])
+def test_infeasible_task_early_cancellation_normal_tasks(
+    local_autoscaling_cluster, shutdown_only
+):
+
+    ray.init(address="auto")
+
+    signal = SignalActor.remote()
+
+    @ray.remote(num_cpus=1)
+    def feasible_task():
+        signal.wait.remote()
+        return 1
+
+    @ray.remote(num_cpus=10)
+    def infeasible_task():
+        return 2
+
+    obj_feasible = feasible_task.remote()
+    obj_infeasible = infeasible_task.remote()
+
+    # The infeasible task should be cancelled with TaskUnschedulableError
+    with pytest.raises(
+        ray.exceptions.TaskUnschedulableError,
+        match=r"The task is not schedulable: Tasks or actors with resource shapes \[{CPU: 10}] failed to schedule because there are not enough resources for the tasks or actors on the whole cluster.",
+    ):
+        ray.get(obj_infeasible, timeout=10)
+
+    # The feasible task should continue to run successfully
+    signal.send.remote()
+    assert ray.get(obj_feasible, timeout=5) == 1
+
+
+@pytest.mark.parametrize(
+    "local_autoscaling_cluster",
+    [
+        (
+            {"CPU": 0},
+            {
+                "type-i": {
+                    "resources": {"CPU": 1},
+                    "node_config": {},
+                    "min_workers": 1,
+                    "max_workers": 1,
+                },
+            },
+            {"enable_infeasible_task_early_exit": True},
+        )
+    ],
+    indirect=["local_autoscaling_cluster"],
+)
+@pytest.mark.parametrize("enable_v2", [True], ids=["v2"])
+def test_infeasible_task_early_cancellation_actor_creation(
+    local_autoscaling_cluster, shutdown_only
+):
+
+    ray.init(address="auto")
+
+    signal = SignalActor.remote()
+
+    @ray.remote(num_cpus=1)
+    class FeasibleActor:
+        def f(self):
+            signal.wait.remote()
+            return 1
+
+    @ray.remote(num_cpus=10)
+    class InfeasibleActor:
+        def f(self):
+            return 2
+
+    feasible_actor = FeasibleActor.remote()
+    infeasible_actor = InfeasibleActor.remote()
+
+    # The infeasible actor should be cancelled with ActorUnschedulableError
+    with pytest.raises(
+        ray.exceptions.ActorUnschedulableError,
+        match=r"The actor is not schedulable: Tasks or actors with resource shapes \[{CPU: 10}] failed to schedule because there are not enough resources for the tasks or actors on the whole cluster.",
+    ):
+        ray.get(infeasible_actor.f.remote(), timeout=5)
+
+    # The feasible actor should continue to run successfully
+    signal.send.remote()
+    assert ray.get(feasible_actor.f.remote(), timeout=5) == 1
+
+
+if __name__ == "__main__":
+    sys.exit(pytest.main(["-sv", __file__]))