chore: import upstream snapshot with attribution

python/ray/dashboard/modules/node/tests/test_actor.py

@@ -0,0 +1,378 @@
+import logging
+import sys
+import time
+import traceback
+
+import pytest
+import requests
+
+import ray
+from ray._private.test_utils import format_web_url, wait_until_server_available
+from ray.dashboard.modules.node import actor_consts
+from ray.dashboard.tests.conftest import *  # noqa
+from ray.util.placement_group import placement_group
+from ray.util.scheduling_strategies import PlacementGroupSchedulingStrategy
+
+logger = logging.getLogger(__name__)
+
+
+def test_actors(disable_aiohttp_cache, ray_start_with_dashboard):
+    """
+    Tests the REST API dashboard calls on:
+    - alive actors
+    - infeasible actors
+    - dead actors
+    - pg actors (with pg_id set and required_resources formatted)
+    """
+
+    @ray.remote
+    class Foo:
+        def __init__(self, num):
+            self.num = num
+
+        def do_task(self):
+            return self.num
+
+        def get_node_id(self):
+            return ray.get_runtime_context().get_node_id()
+
+        def get_pid(self):
+            import os
+
+            return os.getpid()
+
+        def __repr__(self) -> str:
+            return "Foo1"
+
+    @ray.remote(num_cpus=0, resources={"infeasible_actor": 1})
+    class InfeasibleActor:
+        pass
+
+    pg = placement_group([{"CPU": 1}])
+
+    @ray.remote(num_cpus=1)
+    class PgActor:
+        def __init__(self):
+            pass
+
+        def do_task(self):
+            return 1
+
+        def get_placement_group_id(self):
+            return ray.get_runtime_context().get_placement_group_id()
+
+    foo_actors = [Foo.options(name="first").remote(4), Foo.remote(5)]
+    infeasible_actor = InfeasibleActor.options(name="infeasible").remote()  # noqa
+    dead_actor = Foo.options(name="dead").remote(1)
+    pg_actor = PgActor.options(
+        name="pg",
+        scheduling_strategy=PlacementGroupSchedulingStrategy(
+            placement_group=pg,
+        ),
+    ).remote()
+
+    ray.kill(dead_actor)
+    [actor.do_task.remote() for actor in foo_actors]
+    pg_actor.do_task.remote()
+    webui_url = ray_start_with_dashboard["webui_url"]
+    assert wait_until_server_available(webui_url)
+    webui_url = format_web_url(webui_url)
+    job_id = ray.get_runtime_context().get_job_id()
+    node_id = ray.get(foo_actors[0].get_node_id.remote())
+    pid = ray.get(foo_actors[0].get_pid.remote())
+    placement_group_id = ray.get(pg_actor.get_placement_group_id.remote())
+
+    timeout_seconds = 5
+    start_time = time.time()
+    last_ex = None
+    while True:
+        time.sleep(1)
+        try:
+            resp = requests.get(f"{webui_url}/logical/actors")
+            resp_json = resp.json()
+            resp_data = resp_json["data"]
+            actors = resp_data["actors"]
+            assert len(actors) == 5
+
+            for a in actors.values():
+                if a["name"] == "first":
+                    actor_response = a
+            assert actor_response["jobId"] == job_id
+            assert "Foo" in actor_response["className"]
+            assert "address" in actor_response
+            assert type(actor_response["address"]) is dict
+            assert actor_response["address"]["nodeId"] == node_id
+            assert actor_response["state"] == "ALIVE"
+            assert actor_response["name"] == "first"
+            assert actor_response["numRestarts"] == "0"
+            assert actor_response["pid"] == pid
+            assert actor_response["startTime"] > 0
+            assert actor_response["requiredResources"] == {}
+            assert actor_response["endTime"] == 0
+            assert actor_response["exitDetail"] == "-"
+            assert actor_response["reprName"] == "Foo1"
+            for a in actors.values():
+                # "exitDetail always exits from the response"
+                assert "exitDetail" in a
+
+            # Check the dead actor metadata.
+            for a in actors.values():
+                if a["name"] == "dead":
+                    dead_actor_response = a
+            assert dead_actor_response["endTime"] > 0
+            assert "ray.kill" in dead_actor_response["exitDetail"]
+            assert dead_actor_response["state"] == "DEAD"
+            assert dead_actor_response["name"] == "dead"
+
+            # Check the infeasible actor metadata.
+            for a in actors.values():
+                if a["name"] == "infeasible":
+                    infeasible_actor_response = a
+            # Make sure the infeasible actor's resource name is correct.
+            assert infeasible_actor_response["requiredResources"] == {
+                "infeasible_actor": 1
+            }
+            assert infeasible_actor_response["pid"] == 0
+            all_pids = {entry["pid"] for entry in actors.values()}
+            assert 0 in all_pids  # The infeasible actor
+            assert len(all_pids) > 1
+
+            # Check the pg actor metadata.
+            for a in actors.values():
+                if a["name"] == "pg":
+                    pg_actor_response = a
+            assert pg_actor_response["placementGroupId"] == placement_group_id
+            assert pg_actor_response["requiredResources"] == {"CPU": 1.0}
+
+            break
+        except Exception as ex:
+            last_ex = ex
+        finally:
+            if time.time() > start_time + timeout_seconds:
+                ex_stack = (
+                    traceback.format_exception(
+                        type(last_ex), last_ex, last_ex.__traceback__
+                    )
+                    if last_ex
+                    else []
+                )
+                ex_stack = "".join(ex_stack)
+                raise Exception(f"Timed out while testing, {ex_stack}")
+
+
+def test_actor_with_ids(disable_aiohttp_cache, ray_start_with_dashboard):
+    """
+    Tests the REST API dashboard calls with actor ids in the URL
+    """
+
+    @ray.remote
+    class Actor:
+        def __init__(self, num):
+            self.num = num
+
+        def do_task(self):
+            return self.num
+
+        def get_actor_id(self):
+            return ray.get_runtime_context().get_actor_id()
+
+    webui_url = ray_start_with_dashboard["webui_url"]
+    assert wait_until_server_available(webui_url)
+    webui_url = format_web_url(webui_url)
+    actors = [Actor.options(name=f"Actor{i}").remote(i) for i in range(5)]
+    for actor in actors:
+        actor.do_task.remote()
+
+    actor_ids = [ray.get(actor.get_actor_id.remote()) for actor in actors]
+
+    timeout_seconds = 5
+    start_time = time.time()
+    last_ex = None
+    while True:
+        time.sleep(1)
+        try:
+            actor_idx = 2
+            resp = requests.get(f"{webui_url}/logical/actors/{actor_ids[actor_idx]}")
+            resp_json = resp.json()
+            resp_data = resp_json["data"]
+            actor_detail = resp_data["detail"]
+            assert actor_detail["state"] == "ALIVE"
+            assert actor_detail["name"] == f"Actor{actor_idx}"
+            assert actor_detail["numRestarts"] == "0"
+            assert actor_detail["startTime"] > 0
+            assert actor_detail["requiredResources"] == {}
+            assert actor_detail["endTime"] == 0
+            assert actor_detail["exitDetail"] == "-"
+
+            actor_idxs = [0, 1, 4]
+            actor_idxs_to_id_str = ",".join([str(actor_ids[i]) for i in actor_idxs])
+            resp = requests.get(
+                f"{webui_url}/logical/actors?ids={actor_idxs_to_id_str}"
+            )
+            resp_json = resp.json()
+            resp_actors = resp_json["data"]["actors"]
+            assert len(resp_actors) == len(actor_idxs)
+            for actor_idx in actor_idxs:
+                actor_detail = resp_actors[str(actor_ids[actor_idx])]
+                assert actor_detail["state"] == "ALIVE"
+                assert actor_detail["name"] == f"Actor{actor_idx}"
+                assert actor_detail["numRestarts"] == "0"
+                assert actor_detail["startTime"] > 0
+                assert actor_detail["requiredResources"] == {}
+                assert actor_detail["endTime"] == 0
+                assert actor_detail["exitDetail"] == "-"
+
+            break
+        except Exception as ex:
+            last_ex = ex
+        finally:
+            if time.time() > start_time + timeout_seconds:
+                ex_stack = (
+                    traceback.format_exception(
+                        type(last_ex), last_ex, last_ex.__traceback__
+                    )
+                    if last_ex
+                    else []
+                )
+                ex_stack = "".join(ex_stack)
+                raise Exception(f"Timed out while testing, {ex_stack}")
+
+
+def test_nil_node(enable_test_module, disable_aiohttp_cache, ray_start_with_dashboard):
+    assert wait_until_server_available(ray_start_with_dashboard["webui_url"]) is True
+    webui_url = ray_start_with_dashboard["webui_url"]
+    assert wait_until_server_available(webui_url)
+    webui_url = format_web_url(webui_url)
+
+    @ray.remote(num_gpus=1)
+    class InfeasibleActor:
+        pass
+
+    infeasible_actor = InfeasibleActor.remote()  # noqa
+
+    timeout_seconds = 5
+    start_time = time.time()
+    last_ex = None
+    while True:
+        time.sleep(1)
+        try:
+            resp = requests.get(f"{webui_url}/logical/actors")
+            resp_json = resp.json()
+            resp_data = resp_json["data"]
+            actors = resp_data["actors"]
+            assert len(actors) == 1
+            response = requests.get(webui_url + "/test/dump?key=node_actors")
+            response.raise_for_status()
+            result = response.json()
+            assert actor_consts.NIL_NODE_ID not in result["data"]["nodeActors"]
+            break
+        except Exception as ex:
+            last_ex = ex
+        finally:
+            if time.time() > start_time + timeout_seconds:
+                ex_stack = (
+                    traceback.format_exception(
+                        type(last_ex), last_ex, last_ex.__traceback__
+                    )
+                    if last_ex
+                    else []
+                )
+                ex_stack = "".join(ex_stack)
+                raise Exception(f"Timed out while testing, {ex_stack}")
+
+
+def test_actor_cleanup(
+    disable_aiohttp_cache, reduce_actor_cache, ray_start_with_dashboard
+):
+    @ray.remote
+    class Foo:
+        def __init__(self, num):
+            self.num = num
+
+        def do_task(self):
+            return self.num
+
+    @ray.remote(num_gpus=1)
+    class InfeasibleActor:
+        pass
+
+    infeasible_actor = InfeasibleActor.remote()  # noqa
+
+    foo_actors = [
+        Foo.remote(1),
+        Foo.remote(2),
+        Foo.remote(3),
+        Foo.remote(4),
+        Foo.remote(5),
+        Foo.remote(6),
+    ]
+    results = [actor.do_task.remote() for actor in foo_actors]  # noqa
+    webui_url = ray_start_with_dashboard["webui_url"]
+    assert wait_until_server_available(webui_url)
+    webui_url = format_web_url(webui_url)
+
+    timeout_seconds = 8
+    start_time = time.time()
+    last_ex = None
+    while True:
+        time.sleep(1)
+        try:
+            resp = requests.get(f"{webui_url}/logical/actors")
+            resp_json = resp.json()
+            resp_data = resp_json["data"]
+            actors = resp_data["actors"]
+            # Although max cache is 3, there should be 7 actors
+            # because they are all still alive.
+            assert len(actors) == 7
+
+            break
+        except Exception as ex:
+            last_ex = ex
+        finally:
+            if time.time() > start_time + timeout_seconds:
+                ex_stack = (
+                    traceback.format_exception(
+                        type(last_ex), last_ex, last_ex.__traceback__
+                    )
+                    if last_ex
+                    else []
+                )
+                ex_stack = "".join(ex_stack)
+                raise Exception(f"Timed out while testing, {ex_stack}")
+
+    # kill
+    ray.kill(infeasible_actor)
+    [ray.kill(foo_actor) for foo_actor in foo_actors]
+    # Wait 5 seconds for cleanup to finish
+    time.sleep(5)
+
+    # Check only three remaining in cache
+    start_time = time.time()
+    while True:
+        time.sleep(1)
+        try:
+            resp = requests.get(f"{webui_url}/logical/actors")
+            resp_json = resp.json()
+            resp_data = resp_json["data"]
+            actors = resp_data["actors"]
+            # Max cache is 3 so only 3 actors should be left.
+            assert len(actors) == 3
+
+            break
+        except Exception as ex:
+            last_ex = ex
+        finally:
+            if time.time() > start_time + timeout_seconds:
+                ex_stack = (
+                    traceback.format_exception(
+                        type(last_ex), last_ex, last_ex.__traceback__
+                    )
+                    if last_ex
+                    else []
+                )
+                ex_stack = "".join(ex_stack)
+                raise Exception(f"Timed out while testing, {ex_stack}")
+
+
+if __name__ == "__main__":
+    sys.exit(pytest.main(["-v", __file__]))

python/ray/dashboard/modules/node/tests/test_node.py

@@ -0,0 +1,416 @@
+import logging
+import os
+import random
+import sys
+import threading
+import time
+import traceback
+from datetime import datetime, timedelta
+
+import pytest
+import requests
+
+import ray
+from ray._common.test_utils import wait_for_condition
+from ray._private.test_utils import (
+    format_web_url,
+    wait_until_server_available,
+)
+from ray.cluster_utils import Cluster
+from ray.dashboard.consts import RAY_DASHBOARD_STATS_UPDATING_INTERVAL
+from ray.dashboard.tests.conftest import *  # noqa
+
+logger = logging.getLogger(__name__)
+
+
+def test_nodes_update(enable_test_module, ray_start_with_dashboard):
+    assert wait_until_server_available(ray_start_with_dashboard["webui_url"]) is True
+    webui_url = ray_start_with_dashboard["webui_url"]
+    webui_url = format_web_url(webui_url)
+
+    timeout_seconds = 10
+    start_time = time.time()
+    while True:
+        time.sleep(1)
+        try:
+            response = requests.get(webui_url + "/test/dump")
+            response.raise_for_status()
+            try:
+                dump_info = response.json()
+            except Exception as ex:
+                logger.info("failed response: %s", response.text)
+                raise ex
+            assert dump_info["result"] is True
+            dump_data = dump_info["data"]
+            assert len(dump_data["nodes"]) == 1
+            break
+
+        except (AssertionError, requests.exceptions.ConnectionError):
+            logger.exception("Retry")
+        finally:
+            if time.time() > start_time + timeout_seconds:
+                raise Exception("Timed out while testing.")
+
+
+def test_node_info(disable_aiohttp_cache, ray_start_with_dashboard):
+    @ray.remote
+    class Actor:
+        def getpid(self):
+            print(f"actor pid={os.getpid()}")
+            return os.getpid()
+
+    actors = [Actor.remote(), Actor.remote()]
+    actor_pids = [actor.getpid.remote() for actor in actors]
+    actor_pids = set(ray.get(actor_pids))
+
+    assert wait_until_server_available(ray_start_with_dashboard["webui_url"]) is True
+    webui_url = ray_start_with_dashboard["webui_url"]
+    webui_url = format_web_url(webui_url)
+    node_id = ray_start_with_dashboard["node_id"]
+
+    # NOTE: Leaving sum buffer time for data to get refreshed
+    timeout_seconds = RAY_DASHBOARD_STATS_UPDATING_INTERVAL * 1.5
+
+    start_time = time.time()
+    last_ex = None
+    while True:
+        time.sleep(1)
+        try:
+            response = requests.get(webui_url + "/nodes?view=hostnamelist")
+            response.raise_for_status()
+            hostname_list = response.json()
+            assert hostname_list["result"] is True, hostname_list["msg"]
+            hostname_list = hostname_list["data"]["hostNameList"]
+            assert len(hostname_list) == 1
+
+            hostname = hostname_list[0]
+            response = requests.get(webui_url + f"/nodes/{node_id}")
+            response.raise_for_status()
+            detail = response.json()
+            assert detail["result"] is True, detail["msg"]
+            detail = detail["data"]["detail"]
+            assert detail["hostname"] == hostname
+            assert detail["raylet"]["state"] == "ALIVE"
+            assert detail["raylet"]["isHeadNode"] is True
+            assert "raylet" in detail["cmdline"][0]
+            assert len(detail["workers"]) >= 2
+            assert len(detail["actors"]) == 2, detail["actors"]
+
+            actor_worker_pids = set()
+            for worker in detail["workers"]:
+                if "ray::Actor" in worker["cmdline"][0]:
+                    actor_worker_pids.add(worker["pid"])
+            assert actor_worker_pids == actor_pids
+
+            response = requests.get(webui_url + "/nodes?view=summary")
+            response.raise_for_status()
+            summary = response.json()
+            assert summary["result"] is True, summary["msg"]
+            assert len(summary["data"]["summary"]) == 1
+            summary = summary["data"]["summary"][0]
+            assert summary["hostname"] == hostname
+            assert summary["raylet"]["state"] == "ALIVE"
+            assert "raylet" in summary["cmdline"][0]
+            assert "workers" not in summary
+            assert "actors" not in summary
+            assert "objectStoreAvailableMemory" in summary["raylet"]
+            assert "objectStoreUsedMemory" in summary["raylet"]
+            break
+        except Exception as ex:
+            last_ex = ex
+        finally:
+            if time.time() > start_time + timeout_seconds:
+                ex_stack = (
+                    traceback.format_exception(
+                        type(last_ex), last_ex, last_ex.__traceback__
+                    )
+                    if last_ex
+                    else []
+                )
+                ex_stack = "".join(ex_stack)
+                raise Exception(f"Timed out while testing, {ex_stack}")
+
+
+@pytest.mark.parametrize(
+    "ray_start_cluster_head_with_env_vars",
+    [
+        {
+            "include_dashboard": True,
+            "env_vars": {
+                "RAY_maximum_gcs_dead_node_cached_count": "1",
+            },
+            "_system_config": {
+                "health_check_initial_delay_ms": 0,
+                "health_check_timeout_ms": 100,
+                "health_check_failure_threshold": 3,
+                "health_check_period_ms": 100,
+            },
+        }
+    ],
+    indirect=True,
+)
+def test_dead_node_cache_contains_latest_dead_node_if_cache_overflows(
+    enable_test_module, disable_aiohttp_cache, ray_start_cluster_head_with_env_vars
+):
+    cluster: Cluster = ray_start_cluster_head_with_env_vars
+    assert wait_until_server_available(
+        cluster.webui_url
+    ), "Failed to connect to the Dashboard Server"
+    webui_url = format_web_url(cluster.webui_url)
+
+    def _compare_dead_node_set(expected_alive_nodes, expected_dead_nodes):
+        try:
+            response = requests.get(webui_url + "/nodes?view=summary")
+            response.raise_for_status()
+            summary = response.json()
+            assert summary["result"] is True, summary["msg"]
+            summary = summary["data"]["summary"]
+            dead_nodes = set()
+            alive_nodes = set()
+            for node_info in summary:
+                node_id = node_info["raylet"]["nodeId"]
+                response = requests.get(webui_url + f"/nodes/{node_id}")
+                response.raise_for_status()
+                if node_info["raylet"]["state"] == "DEAD":
+                    dead_nodes.add(node_id)
+                if node_info["raylet"]["state"] == "ALIVE":
+                    alive_nodes.add(node_id)
+            assert alive_nodes == expected_alive_nodes
+            assert dead_nodes == expected_dead_nodes
+            return True
+        except Exception as ex:
+            logger.info(ex)
+            return False
+
+    node_1 = cluster.add_node()
+    head_node_id = ray.get_runtime_context().get_node_id()
+    node_1_id = node_1.node_id
+    curr_alive_nodes = {head_node_id, node_1_id}
+    curr_dead_nodes = set()
+    wait_for_condition(
+        _compare_dead_node_set,
+        10,
+        expected_alive_nodes=curr_alive_nodes,
+        expected_dead_nodes=curr_dead_nodes,
+    )
+
+    node_2 = cluster.add_node()
+    node_2_id = node_2.node_id
+    curr_alive_nodes.add(node_2_id)
+    wait_for_condition(
+        _compare_dead_node_set,
+        10,
+        expected_alive_nodes=curr_alive_nodes,
+        expected_dead_nodes=curr_dead_nodes,
+    )
+
+    cluster.remove_node(node_1, allow_graceful=False)
+    curr_alive_nodes.remove(node_1_id)
+    curr_dead_nodes.add(node_1_id)
+    wait_for_condition(
+        _compare_dead_node_set,
+        10,
+        expected_alive_nodes=curr_alive_nodes,
+        expected_dead_nodes=curr_dead_nodes,
+    )
+
+    cluster.remove_node(node_2, allow_graceful=False)
+    curr_alive_nodes.remove(node_2_id)
+    curr_dead_nodes.remove(node_1_id)
+    curr_dead_nodes.add(node_2_id)
+    wait_for_condition(
+        _compare_dead_node_set,
+        10,
+        expected_alive_nodes=curr_alive_nodes,
+        expected_dead_nodes=curr_dead_nodes,
+    )
+
+
+@pytest.mark.parametrize(
+    "ray_start_cluster_head",
+    [
+        {
+            "include_dashboard": True,
+            "_system_config": {
+                "health_check_initial_delay_ms": 0,
+                "health_check_timeout_ms": 100,
+                "health_check_failure_threshold": 3,
+                "health_check_period_ms": 100,
+            },
+        }
+    ],
+    indirect=True,
+)
+def test_multi_nodes_info(
+    enable_test_module, disable_aiohttp_cache, ray_start_cluster_head
+):
+    cluster: Cluster = ray_start_cluster_head
+    assert wait_until_server_available(cluster.webui_url) is True
+    webui_url = cluster.webui_url
+    webui_url = format_web_url(webui_url)
+    cluster.add_node()
+    cluster.add_node()
+    dead_node = cluster.add_node()
+    cluster.remove_node(dead_node, allow_graceful=False)
+
+    def _check_nodes():
+        try:
+            response = requests.get(webui_url + "/nodes?view=summary")
+            response.raise_for_status()
+            summary = response.json()
+            assert summary["result"] is True, summary["msg"]
+            summary = summary["data"]["summary"]
+            assert len(summary) == 4
+            for node_info in summary:
+                node_id = node_info["raylet"]["nodeId"]
+                response = requests.get(webui_url + f"/nodes/{node_id}")
+                response.raise_for_status()
+                detail = response.json()
+                assert detail["result"] is True, detail["msg"]
+                detail = detail["data"]["detail"]
+                if node_id != dead_node.node_id:
+                    assert detail["raylet"]["state"] == "ALIVE"
+                else:
+                    assert detail["raylet"]["state"] == "DEAD"
+                    assert detail["raylet"].get("objectStoreAvailableMemory", 0) == 0
+            return True
+        except Exception as ex:
+            logger.info(ex)
+            return False
+
+    wait_for_condition(_check_nodes, timeout=15)
+
+
+@pytest.mark.parametrize(
+    "ray_start_cluster_head", [{"include_dashboard": True}], indirect=True
+)
+def test_multi_node_churn(
+    enable_test_module, disable_aiohttp_cache, ray_start_cluster_head
+):
+    cluster: Cluster = ray_start_cluster_head
+    assert wait_until_server_available(cluster.webui_url) is True
+    webui_url = format_web_url(cluster.webui_url)
+
+    success = True
+
+    def verify():
+        nonlocal success
+        while True:
+            try:
+                resp = requests.get(webui_url)
+                resp.raise_for_status()
+                resp = requests.get(webui_url + "/nodes?view=summary")
+                resp.raise_for_status()
+                summary = resp.json()
+                assert summary["result"] is True, summary["msg"]
+                assert summary["data"]["summary"]
+                time.sleep(1)
+            except Exception:
+                success = False
+                break
+
+    t = threading.Thread(target=verify, daemon=True)
+    t.start()
+
+    t_st = datetime.now()
+    duration = timedelta(seconds=60)
+    worker_nodes = []
+    while datetime.now() < t_st + duration:
+        time.sleep(5)
+        if len(worker_nodes) < 2:
+            worker_nodes.append(cluster.add_node())
+            continue
+        should_add_node = random.randint(0, 1)
+        if should_add_node:
+            worker_nodes.append(cluster.add_node())
+        else:
+            node_index = random.randrange(0, len(worker_nodes))
+            node_to_remove = worker_nodes.pop(node_index)
+            cluster.remove_node(node_to_remove)
+
+    assert success
+
+
+@pytest.mark.skipif(
+    sys.platform == "win32", reason="setproctitle does not change psutil.cmdline"
+)
+def test_node_physical_stats(enable_test_module, shutdown_only):
+    """
+    Tests NodeHead._update_node_physical_stats.
+    """
+    addresses = ray.init(include_dashboard=True, num_cpus=6)
+
+    @ray.remote(num_cpus=1)
+    class Actor:
+        def getpid(self):
+            return os.getpid()
+
+    actors = [Actor.remote() for _ in range(6)]
+    actor_pids = ray.get([actor.getpid.remote() for actor in actors])
+    actor_pids = set(actor_pids)
+
+    webui_url = addresses["webui_url"]
+    assert wait_until_server_available(webui_url) is True
+    webui_url = format_web_url(webui_url)
+
+    def _check_workers():
+        try:
+            resp = requests.get(webui_url + "/test/dump?key=node_physical_stats")
+            resp.raise_for_status()
+            result = resp.json()
+            assert result["result"] is True
+            node_physical_stats = result["data"]["nodePhysicalStats"]
+            assert len(node_physical_stats) == 1
+            current_stats = node_physical_stats[addresses["node_id"]]
+            # Check Actor workers
+            current_actor_pids = set()
+            for worker in current_stats["workers"]:
+                if "ray::Actor" in worker["cmdline"][0]:
+                    current_actor_pids.add(worker["pid"])
+            assert current_actor_pids == actor_pids
+            # Check raylet cmdline
+            assert "raylet" in current_stats["cmdline"][0]
+            return True
+        except Exception as ex:
+            logger.info(ex)
+            return False
+
+    wait_for_condition(_check_workers, timeout=10)
+
+
+def test_worker_pids_reported(enable_test_module, ray_start_with_dashboard):
+    assert wait_until_server_available(ray_start_with_dashboard["webui_url"]) is True
+    webui_url = ray_start_with_dashboard["webui_url"]
+    webui_url = format_web_url(webui_url)
+    node_id = ray_start_with_dashboard["node_id"]
+
+    @ray.remote(runtime_env={"uv": {"packages": ["requests==2.32.5"]}})
+    class UvActor:
+        def get_pid(self):
+            return os.getpid()
+
+    uv_actor = UvActor.remote()
+    uv_actor_pid = ray.get(uv_actor.get_pid.remote())
+    driver_pid = os.getpid()
+
+    def _check_worker_pids():
+        try:
+            response = requests.get(webui_url + f"/nodes/{node_id}")
+            response.raise_for_status()
+            dump_info = response.json()
+            assert dump_info["result"] is True
+            detail = dump_info["data"]["detail"]
+            pids = [worker["pid"] for worker in detail["workers"]]
+            assert len(pids) >= 2  # might include idle worker
+            assert uv_actor_pid in pids
+            assert driver_pid in pids
+            return True
+        except Exception as ex:
+            logger.info(ex)
+            return False
+
+    wait_for_condition(_check_worker_pids, timeout=20)
+
+
+if __name__ == "__main__":
+    sys.exit(pytest.main(["-v", __file__]))