chore: import upstream snapshot with attribution

python/ray/dashboard/modules/node/tests/test_actor.py

@@ -0,0 +1,378 @@
+import logging
+import sys
+import time
+import traceback
+
+import pytest
+import requests
+
+import ray
+from ray._private.test_utils import format_web_url, wait_until_server_available
+from ray.dashboard.modules.node import actor_consts
+from ray.dashboard.tests.conftest import *  # noqa
+from ray.util.placement_group import placement_group
+from ray.util.scheduling_strategies import PlacementGroupSchedulingStrategy
+
+logger = logging.getLogger(__name__)
+
+
+def test_actors(disable_aiohttp_cache, ray_start_with_dashboard):
+    """
+    Tests the REST API dashboard calls on:
+    - alive actors
+    - infeasible actors
+    - dead actors
+    - pg actors (with pg_id set and required_resources formatted)
+    """
+
+    @ray.remote
+    class Foo:
+        def __init__(self, num):
+            self.num = num
+
+        def do_task(self):
+            return self.num
+
+        def get_node_id(self):
+            return ray.get_runtime_context().get_node_id()
+
+        def get_pid(self):
+            import os
+
+            return os.getpid()
+
+        def __repr__(self) -> str:
+            return "Foo1"
+
+    @ray.remote(num_cpus=0, resources={"infeasible_actor": 1})
+    class InfeasibleActor:
+        pass
+
+    pg = placement_group([{"CPU": 1}])
+
+    @ray.remote(num_cpus=1)
+    class PgActor:
+        def __init__(self):
+            pass
+
+        def do_task(self):
+            return 1
+
+        def get_placement_group_id(self):
+            return ray.get_runtime_context().get_placement_group_id()
+
+    foo_actors = [Foo.options(name="first").remote(4), Foo.remote(5)]
+    infeasible_actor = InfeasibleActor.options(name="infeasible").remote()  # noqa
+    dead_actor = Foo.options(name="dead").remote(1)
+    pg_actor = PgActor.options(
+        name="pg",
+        scheduling_strategy=PlacementGroupSchedulingStrategy(
+            placement_group=pg,
+        ),
+    ).remote()
+
+    ray.kill(dead_actor)
+    [actor.do_task.remote() for actor in foo_actors]
+    pg_actor.do_task.remote()
+    webui_url = ray_start_with_dashboard["webui_url"]
+    assert wait_until_server_available(webui_url)
+    webui_url = format_web_url(webui_url)
+    job_id = ray.get_runtime_context().get_job_id()
+    node_id = ray.get(foo_actors[0].get_node_id.remote())
+    pid = ray.get(foo_actors[0].get_pid.remote())
+    placement_group_id = ray.get(pg_actor.get_placement_group_id.remote())
+
+    timeout_seconds = 5
+    start_time = time.time()
+    last_ex = None
+    while True:
+        time.sleep(1)
+        try:
+            resp = requests.get(f"{webui_url}/logical/actors")
+            resp_json = resp.json()
+            resp_data = resp_json["data"]
+            actors = resp_data["actors"]
+            assert len(actors) == 5
+
+            for a in actors.values():
+                if a["name"] == "first":
+                    actor_response = a
+            assert actor_response["jobId"] == job_id
+            assert "Foo" in actor_response["className"]
+            assert "address" in actor_response
+            assert type(actor_response["address"]) is dict
+            assert actor_response["address"]["nodeId"] == node_id
+            assert actor_response["state"] == "ALIVE"
+            assert actor_response["name"] == "first"
+            assert actor_response["numRestarts"] == "0"
+            assert actor_response["pid"] == pid
+            assert actor_response["startTime"] > 0
+            assert actor_response["requiredResources"] == {}
+            assert actor_response["endTime"] == 0
+            assert actor_response["exitDetail"] == "-"
+            assert actor_response["reprName"] == "Foo1"
+            for a in actors.values():
+                # "exitDetail always exits from the response"
+                assert "exitDetail" in a
+
+            # Check the dead actor metadata.
+            for a in actors.values():
+                if a["name"] == "dead":
+                    dead_actor_response = a
+            assert dead_actor_response["endTime"] > 0
+            assert "ray.kill" in dead_actor_response["exitDetail"]
+            assert dead_actor_response["state"] == "DEAD"
+            assert dead_actor_response["name"] == "dead"
+
+            # Check the infeasible actor metadata.
+            for a in actors.values():
+                if a["name"] == "infeasible":
+                    infeasible_actor_response = a
+            # Make sure the infeasible actor's resource name is correct.
+            assert infeasible_actor_response["requiredResources"] == {
+                "infeasible_actor": 1
+            }
+            assert infeasible_actor_response["pid"] == 0
+            all_pids = {entry["pid"] for entry in actors.values()}
+            assert 0 in all_pids  # The infeasible actor
+            assert len(all_pids) > 1
+
+            # Check the pg actor metadata.
+            for a in actors.values():
+                if a["name"] == "pg":
+                    pg_actor_response = a
+            assert pg_actor_response["placementGroupId"] == placement_group_id
+            assert pg_actor_response["requiredResources"] == {"CPU": 1.0}
+
+            break
+        except Exception as ex:
+            last_ex = ex
+        finally:
+            if time.time() > start_time + timeout_seconds:
+                ex_stack = (
+                    traceback.format_exception(
+                        type(last_ex), last_ex, last_ex.__traceback__
+                    )
+                    if last_ex
+                    else []
+                )
+                ex_stack = "".join(ex_stack)
+                raise Exception(f"Timed out while testing, {ex_stack}")
+
+
+def test_actor_with_ids(disable_aiohttp_cache, ray_start_with_dashboard):
+    """
+    Tests the REST API dashboard calls with actor ids in the URL
+    """
+
+    @ray.remote
+    class Actor:
+        def __init__(self, num):
+            self.num = num
+
+        def do_task(self):
+            return self.num
+
+        def get_actor_id(self):
+            return ray.get_runtime_context().get_actor_id()
+
+    webui_url = ray_start_with_dashboard["webui_url"]
+    assert wait_until_server_available(webui_url)
+    webui_url = format_web_url(webui_url)
+    actors = [Actor.options(name=f"Actor{i}").remote(i) for i in range(5)]
+    for actor in actors:
+        actor.do_task.remote()
+
+    actor_ids = [ray.get(actor.get_actor_id.remote()) for actor in actors]
+
+    timeout_seconds = 5
+    start_time = time.time()
+    last_ex = None
+    while True:
+        time.sleep(1)
+        try:
+            actor_idx = 2
+            resp = requests.get(f"{webui_url}/logical/actors/{actor_ids[actor_idx]}")
+            resp_json = resp.json()
+            resp_data = resp_json["data"]
+            actor_detail = resp_data["detail"]
+            assert actor_detail["state"] == "ALIVE"
+            assert actor_detail["name"] == f"Actor{actor_idx}"
+            assert actor_detail["numRestarts"] == "0"
+            assert actor_detail["startTime"] > 0
+            assert actor_detail["requiredResources"] == {}
+            assert actor_detail["endTime"] == 0
+            assert actor_detail["exitDetail"] == "-"
+
+            actor_idxs = [0, 1, 4]
+            actor_idxs_to_id_str = ",".join([str(actor_ids[i]) for i in actor_idxs])
+            resp = requests.get(
+                f"{webui_url}/logical/actors?ids={actor_idxs_to_id_str}"
+            )
+            resp_json = resp.json()
+            resp_actors = resp_json["data"]["actors"]
+            assert len(resp_actors) == len(actor_idxs)
+            for actor_idx in actor_idxs:
+                actor_detail = resp_actors[str(actor_ids[actor_idx])]
+                assert actor_detail["state"] == "ALIVE"
+                assert actor_detail["name"] == f"Actor{actor_idx}"
+                assert actor_detail["numRestarts"] == "0"
+                assert actor_detail["startTime"] > 0
+                assert actor_detail["requiredResources"] == {}
+                assert actor_detail["endTime"] == 0
+                assert actor_detail["exitDetail"] == "-"
+
+            break
+        except Exception as ex:
+            last_ex = ex
+        finally:
+            if time.time() > start_time + timeout_seconds:
+                ex_stack = (
+                    traceback.format_exception(
+                        type(last_ex), last_ex, last_ex.__traceback__
+                    )
+                    if last_ex
+                    else []
+                )
+                ex_stack = "".join(ex_stack)
+                raise Exception(f"Timed out while testing, {ex_stack}")
+
+
+def test_nil_node(enable_test_module, disable_aiohttp_cache, ray_start_with_dashboard):
+    assert wait_until_server_available(ray_start_with_dashboard["webui_url"]) is True
+    webui_url = ray_start_with_dashboard["webui_url"]
+    assert wait_until_server_available(webui_url)
+    webui_url = format_web_url(webui_url)
+
+    @ray.remote(num_gpus=1)
+    class InfeasibleActor:
+        pass
+
+    infeasible_actor = InfeasibleActor.remote()  # noqa
+
+    timeout_seconds = 5
+    start_time = time.time()
+    last_ex = None
+    while True:
+        time.sleep(1)
+        try:
+            resp = requests.get(f"{webui_url}/logical/actors")
+            resp_json = resp.json()
+            resp_data = resp_json["data"]
+            actors = resp_data["actors"]
+            assert len(actors) == 1
+            response = requests.get(webui_url + "/test/dump?key=node_actors")
+            response.raise_for_status()
+            result = response.json()
+            assert actor_consts.NIL_NODE_ID not in result["data"]["nodeActors"]
+            break
+        except Exception as ex:
+            last_ex = ex
+        finally:
+            if time.time() > start_time + timeout_seconds:
+                ex_stack = (
+                    traceback.format_exception(
+                        type(last_ex), last_ex, last_ex.__traceback__
+                    )
+                    if last_ex
+                    else []
+                )
+                ex_stack = "".join(ex_stack)
+                raise Exception(f"Timed out while testing, {ex_stack}")
+
+
+def test_actor_cleanup(
+    disable_aiohttp_cache, reduce_actor_cache, ray_start_with_dashboard
+):
+    @ray.remote
+    class Foo:
+        def __init__(self, num):
+            self.num = num
+
+        def do_task(self):
+            return self.num
+
+    @ray.remote(num_gpus=1)
+    class InfeasibleActor:
+        pass
+
+    infeasible_actor = InfeasibleActor.remote()  # noqa
+
+    foo_actors = [
+        Foo.remote(1),
+        Foo.remote(2),
+        Foo.remote(3),
+        Foo.remote(4),
+        Foo.remote(5),
+        Foo.remote(6),
+    ]
+    results = [actor.do_task.remote() for actor in foo_actors]  # noqa
+    webui_url = ray_start_with_dashboard["webui_url"]
+    assert wait_until_server_available(webui_url)
+    webui_url = format_web_url(webui_url)
+
+    timeout_seconds = 8
+    start_time = time.time()
+    last_ex = None
+    while True:
+        time.sleep(1)
+        try:
+            resp = requests.get(f"{webui_url}/logical/actors")
+            resp_json = resp.json()
+            resp_data = resp_json["data"]
+            actors = resp_data["actors"]
+            # Although max cache is 3, there should be 7 actors
+            # because they are all still alive.
+            assert len(actors) == 7
+
+            break
+        except Exception as ex:
+            last_ex = ex
+        finally:
+            if time.time() > start_time + timeout_seconds:
+                ex_stack = (
+                    traceback.format_exception(
+                        type(last_ex), last_ex, last_ex.__traceback__
+                    )
+                    if last_ex
+                    else []
+                )
+                ex_stack = "".join(ex_stack)
+                raise Exception(f"Timed out while testing, {ex_stack}")
+
+    # kill
+    ray.kill(infeasible_actor)
+    [ray.kill(foo_actor) for foo_actor in foo_actors]
+    # Wait 5 seconds for cleanup to finish
+    time.sleep(5)
+
+    # Check only three remaining in cache
+    start_time = time.time()
+    while True:
+        time.sleep(1)
+        try:
+            resp = requests.get(f"{webui_url}/logical/actors")
+            resp_json = resp.json()
+            resp_data = resp_json["data"]
+            actors = resp_data["actors"]
+            # Max cache is 3 so only 3 actors should be left.
+            assert len(actors) == 3
+
+            break
+        except Exception as ex:
+            last_ex = ex
+        finally:
+            if time.time() > start_time + timeout_seconds:
+                ex_stack = (
+                    traceback.format_exception(
+                        type(last_ex), last_ex, last_ex.__traceback__
+                    )
+                    if last_ex
+                    else []
+                )
+                ex_stack = "".join(ex_stack)
+                raise Exception(f"Timed out while testing, {ex_stack}")
+
+
+if __name__ == "__main__":
+    sys.exit(pytest.main(["-v", __file__]))