chore: import upstream snapshot with attribution

python/ray/dashboard/modules/node/actor_consts.py

@@ -0,0 +1,4 @@
+import ray
+
+NIL_NODE_ID = ray.NodeID.nil().hex()
+RETRY_GET_ALL_ACTOR_INFO_INTERVAL_SECONDS = 1

python/ray/dashboard/modules/node/datacenter.py

@@ -0,0 +1,248 @@
+import copy
+from typing import List, Optional
+
+import ray.dashboard.consts as dashboard_consts
+from ray._common.utils import (
+    get_or_create_event_loop,
+)
+from ray._private.utils import (
+    parse_pg_formatted_resources_to_original,
+)
+from ray.dashboard.utils import (
+    async_loop_forever,
+    compose_state_message,
+)
+
+
+class DataSource:
+    # {node id hex(str): node stats(dict of GetNodeStatsReply
+    # in node_manager.proto)}
+    node_stats = {}
+    # {node id hex(str): node physical stats(dict from reporter_agent.py)}
+    node_physical_stats = {}
+    # {actor id hex(str): actor table data(dict of ActorTableData
+    # in gcs.proto)}
+    actors = {}
+    # {node id hex(str): gcs node info(dict of GcsNodeInfo in gcs.proto)}
+    nodes = {}
+    # {node id hex(str): worker list}
+    node_workers = {}
+    # {node id hex(str): {actor id hex(str): actor table data}}
+    node_actors = {}
+    # {worker id(str): core worker stats}
+    core_worker_stats = {}
+
+
+class DataOrganizer:
+    @staticmethod
+    @async_loop_forever(dashboard_consts.RAY_DASHBOARD_STATS_PURGING_INTERVAL)
+    async def purge():
+        # Purge data that is out of date.
+        # These data sources are maintained by DashboardHead,
+        # we do not needs to purge them:
+        #   * agents
+        #   * nodes
+        alive_nodes = {
+            node_id
+            for node_id, node_info in DataSource.nodes.items()
+            if node_info["state"] == "ALIVE"
+        }
+        for key in DataSource.node_stats.keys() - alive_nodes:
+            DataSource.node_stats.pop(key)
+
+        for key in DataSource.node_physical_stats.keys() - alive_nodes:
+            DataSource.node_physical_stats.pop(key)
+
+    @classmethod
+    @async_loop_forever(dashboard_consts.RAY_DASHBOARD_STATS_UPDATING_INTERVAL)
+    async def organize(cls, thread_pool_executor):
+        """
+        Organizes data: read from (node_physical_stats, node_stats) and updates
+        (node_workers, node_worker_stats).
+
+        This methods is not really async, but DataSource is not thread safe so we need
+        to make sure it's on the main event loop thread. To avoid blocking the main
+        event loop, we yield after each node processed.
+        """
+        loop = get_or_create_event_loop()
+
+        node_workers = {}
+        core_worker_stats = {}
+
+        # NOTE: We copy keys of the `DataSource.nodes` to make sure
+        #       it doesn't change during the iteration (since its being updated
+        #       from another async task)
+        for node_id in list(DataSource.nodes.keys()):
+            node_physical_stats = DataSource.node_physical_stats.get(node_id, {})
+            node_stats = DataSource.node_stats.get(node_id, {})
+            # Offloads the blocking operation to a thread pool executor. This also
+            # yields to the event loop.
+            workers = await loop.run_in_executor(
+                thread_pool_executor,
+                cls._extract_workers_for_node,
+                node_physical_stats,
+                node_stats,
+            )
+
+            for worker in workers:
+                for stats in worker.get("coreWorkerStats", []):
+                    worker_id = stats["workerId"]
+                    core_worker_stats[worker_id] = stats
+
+            node_workers[node_id] = workers
+
+        DataSource.node_workers = node_workers
+        DataSource.core_worker_stats = core_worker_stats
+
+    @classmethod
+    def _extract_workers_for_node(cls, node_physical_stats, node_stats):
+        workers = []
+        # Merge coreWorkerStats (node stats) to workers (node physical stats)
+        pid_to_worker_stats = {}
+        pid_to_language = {}
+        pid_to_job_id = {}
+
+        for core_worker_stats in node_stats.get("coreWorkersStats", []):
+            pid = core_worker_stats["pid"]
+
+            pid_to_worker_stats[pid] = core_worker_stats
+            pid_to_language[pid] = core_worker_stats["language"]
+            pid_to_job_id[pid] = core_worker_stats["jobId"]
+
+        for worker in node_physical_stats.get("workers", []):
+            worker = dict(worker)
+            pid = worker["pid"]
+
+            core_worker_stats = pid_to_worker_stats.get(pid)
+            # Empty list means core worker stats is not available.
+            worker["coreWorkerStats"] = [core_worker_stats] if core_worker_stats else []
+            worker["language"] = pid_to_language.get(
+                pid, dashboard_consts.DEFAULT_LANGUAGE
+            )
+            worker["jobId"] = pid_to_job_id.get(pid, dashboard_consts.DEFAULT_JOB_ID)
+
+            workers.append(worker)
+
+        return workers
+
+    @classmethod
+    async def get_node_info(cls, node_id, get_summary=False):
+        node_physical_stats = dict(DataSource.node_physical_stats.get(node_id, {}))
+        node_stats = dict(DataSource.node_stats.get(node_id, {}))
+        node = DataSource.nodes.get(node_id, {})
+
+        if get_summary:
+            node_physical_stats.pop("workers", None)
+            node_stats.pop("workersStats", None)
+        else:
+            node_stats.pop("coreWorkersStats", None)
+        store_stats = node_stats.get("storeStats", {})
+        used = int(store_stats.get("objectStoreBytesUsed", 0))
+        # objectStoreBytesAvail == total in the object_manager.cc definition.
+        total = int(store_stats.get("objectStoreBytesAvail", 0))
+        ray_stats = {
+            "object_store_used_memory": used,
+            "object_store_available_memory": total - used,
+        }
+
+        node_info = node_physical_stats
+
+        # Merge node stats to node physical stats under raylet
+        node_info["raylet"] = node_stats
+        node_info["raylet"].update(ray_stats)
+
+        # Merge GcsNodeInfo to node physical stats
+        node_info["raylet"].update(node)
+        death_info = node.get("deathInfo", {})
+        node_info["raylet"]["stateMessage"] = compose_state_message(
+            death_info.get("reason", None), death_info.get("reasonMessage", None)
+        )
+
+        # TODO(spencer-p): TPU process linking is currently prone to over-counting
+        # as it attaches all TPU workers to every chip. This will be addressed
+        # with a more robust mapping in a future update.
+        node_info["tpus"] = copy.deepcopy(node_info.get("tpus", []))
+
+        if not get_summary:
+            actor_table_entries = DataSource.node_actors.get(node_id, {})
+
+            # Merge actors to node physical stats
+            node_info["actors"] = {
+                actor_id: await DataOrganizer._get_actor_info(actor_table_entry)
+                for actor_id, actor_table_entry in actor_table_entries.items()
+            }
+
+            # Update workers to node physical stats
+            node_info["workers"] = DataSource.node_workers.get(node_id, [])
+
+        return node_info
+
+    @classmethod
+    async def get_all_node_summary(cls):
+        return [
+            # NOTE: We're intentionally awaiting in a loop to avoid excessive
+            #       concurrency spinning up excessive # of tasks for large clusters
+            await DataOrganizer.get_node_info(node_id, get_summary=True)
+            for node_id in DataSource.nodes.keys()
+        ]
+
+    @classmethod
+    async def get_actor_infos(cls, actor_ids: Optional[List[str]] = None):
+        target_actor_table_entries: dict[str, Optional[dict]]
+        if actor_ids is not None:
+            target_actor_table_entries = {
+                actor_id: DataSource.actors.get(actor_id) for actor_id in actor_ids
+            }
+        else:
+            target_actor_table_entries = DataSource.actors
+
+        return {
+            actor_id: await DataOrganizer._get_actor_info(actor_table_entry)
+            for actor_id, actor_table_entry in target_actor_table_entries.items()
+        }
+
+    @staticmethod
+    async def _get_actor_info(actor: Optional[dict]) -> Optional[dict]:
+        if actor is None:
+            return None
+
+        actor = actor.copy()
+        worker_id = actor["address"]["workerId"]
+        core_worker_stats = DataSource.core_worker_stats.get(worker_id, {})
+        actor.update(core_worker_stats)
+
+        # TODO(fyrestone): remove this, give a link from actor
+        # info to worker info in front-end.
+        node_id = actor["address"]["nodeId"]
+        pid = core_worker_stats.get("pid")
+        node_physical_stats = DataSource.node_physical_stats.get(node_id, {})
+        actor_process_stats = None
+        actor_process_gpu_stats = []
+        if pid:
+            for process_stats in node_physical_stats.get("workers", []):
+                if process_stats["pid"] == pid:
+                    actor_process_stats = process_stats
+                    break
+
+            for gpu_stats in node_physical_stats.get("gpus", []):
+                # gpu_stats.get("processesPids") can be None, an empty list or a
+                # list of dictionaries.
+                for process in gpu_stats.get("processesPids") or []:
+                    if process["pid"] == pid:
+                        actor_process_gpu_stats.append(gpu_stats)
+                        break
+
+        actor["gpus"] = actor_process_gpu_stats
+        actor["processStats"] = actor_process_stats
+        actor["mem"] = node_physical_stats.get("mem", [])
+
+        required_resources = parse_pg_formatted_resources_to_original(
+            actor["requiredResources"]
+        )
+        actor["requiredResources"] = required_resources
+
+        # TODO(spencer-p): TPU process linking is currently prone to over-counting.
+        # This will be addressed with a more robust mapping in a future update.
+        actor["tpus"] = []
+
+        return actor

python/ray/dashboard/modules/node/node_consts.py

@@ -0,0 +1,17 @@
+from ray._private.ray_constants import env_integer
+
+NODE_STATS_UPDATE_INTERVAL_SECONDS = env_integer(
+    "NODE_STATS_UPDATE_INTERVAL_SECONDS", 15
+)
+RAY_DASHBOARD_HEAD_NODE_REGISTRATION_TIMEOUT = env_integer(
+    "RAY_DASHBOARD_HEAD_NODE_REGISTRATION_TIMEOUT", 10
+)
+MAX_COUNT_OF_GCS_RPC_ERROR = 10
+# This is consistent with gcs_node_manager.cc
+MAX_DEAD_NODES_TO_CACHE = env_integer("RAY_maximum_gcs_dead_node_cached_count", 1000)
+RAY_DASHBOARD_NODE_SUBSCRIBER_POLL_SIZE = env_integer(
+    "RAY_DASHBOARD_NODE_SUBSCRIBER_POLL_SIZE", 200
+)
+RAY_DASHBOARD_AGENT_POLL_INTERVAL_S = env_integer(
+    "RAY_DASHBOARD_AGENT_POLL_INTERVAL_S", 1
+)

python/ray/dashboard/modules/node/node_head.py

@@ -0,0 +1,776 @@
+import asyncio
+import json
+import logging
+import time
+from collections import defaultdict, deque
+from concurrent.futures import ThreadPoolExecutor
+from itertools import chain
+from typing import Any, AsyncGenerator, Dict, Iterable, List, Optional, Set
+
+import aiohttp.web
+import grpc
+
+import ray._private.utils
+import ray.dashboard.optional_utils as dashboard_optional_utils
+import ray.dashboard.utils as dashboard_utils
+from ray._common.utils import get_or_create_event_loop
+from ray._private import ray_constants
+from ray._private.collections_utils import split
+from ray._private.gcs_pubsub import (
+    GcsAioActorSubscriber,
+    GcsAioNodeInfoSubscriber,
+    GcsAioResourceUsageSubscriber,
+)
+from ray._private.grpc_utils import init_grpc_channel
+from ray._private.ray_constants import (
+    DEBUG_AUTOSCALING_ERROR,
+    DEBUG_AUTOSCALING_STATUS,
+    env_integer,
+)
+from ray.autoscaler._private.util import (
+    LoadMetricsSummary,
+    get_per_node_breakdown_as_dict,
+    parse_usage,
+)
+from ray.core.generated import gcs_pb2, node_manager_pb2, node_manager_pb2_grpc
+from ray.dashboard.consts import (
+    DASHBOARD_AGENT_ADDR_IP_PREFIX,
+    DASHBOARD_AGENT_ADDR_NODE_ID_PREFIX,
+    GCS_RPC_TIMEOUT_SECONDS,
+)
+from ray.dashboard.modules.node import actor_consts, node_consts
+from ray.dashboard.modules.node.datacenter import DataOrganizer, DataSource
+from ray.dashboard.modules.reporter.reporter_models import StatsPayload
+from ray.dashboard.subprocesses.module import SubprocessModule
+from ray.dashboard.subprocesses.routes import SubprocessRouteTable as routes
+from ray.dashboard.utils import async_loop_forever
+
+logger = logging.getLogger(__name__)
+
+
+# NOTE: Executor in this head is intentionally constrained to just 1 thread by
+#       default to limit its concurrency, therefore reducing potential for
+#       GIL contention
+RAY_DASHBOARD_NODE_HEAD_TPE_MAX_WORKERS = env_integer(
+    "RAY_DASHBOARD_NODE_HEAD_TPE_MAX_WORKERS", 1
+)
+
+MAX_DESTROYED_ACTORS_TO_CACHE = max(
+    0, ray._config.maximum_gcs_destroyed_actor_cached_count()
+)
+
+ACTOR_CLEANUP_FREQUENCY = 1  # seconds
+
+
+ACTOR_TABLE_STATE_COLUMNS = (
+    "state",
+    "address",
+    "numRestarts",
+    "timestamp",
+    "pid",
+    "exitDetail",
+    "startTime",
+    "endTime",
+    "reprName",
+)
+
+
+def _gcs_node_info_to_dict(message: gcs_pb2.GcsNodeInfo) -> dict:
+    return dashboard_utils.message_to_dict(
+        message, {"nodeId"}, always_print_fields_with_no_presence=True
+    )
+
+
+def _actor_table_data_to_dict(message):
+    orig_message = dashboard_utils.message_to_dict(
+        message,
+        {
+            "actorId",
+            "parentId",
+            "jobId",
+            "workerId",
+            "nodeId",
+            "callerId",
+            "taskId",
+            "parentTaskId",
+            "sourceActorId",
+            "placementGroupId",
+        },
+        always_print_fields_with_no_presence=True,
+    )
+    # The complete schema for actor table is here:
+    #     src/ray/protobuf/gcs.proto
+    # It is super big and for dashboard, we don't need that much information.
+    # Only preserve the necessary ones here for memory usage.
+    fields = {
+        "actorId",
+        "jobId",
+        "pid",
+        "address",
+        "state",
+        "name",
+        "numRestarts",
+        "timestamp",
+        "className",
+        "startTime",
+        "endTime",
+        "reprName",
+        "placementGroupId",
+        "callSite",
+        "labelSelector",
+        "fallbackStrategy",
+    }
+    light_message = {k: v for (k, v) in orig_message.items() if k in fields}
+    light_message["actorClass"] = orig_message["className"]
+    exit_detail = "-"
+    if "deathCause" in orig_message:
+        context = orig_message["deathCause"]
+        if "actorDiedErrorContext" in context:
+            exit_detail = context["actorDiedErrorContext"]["errorMessage"]  # noqa
+        elif "runtimeEnvFailedContext" in context:
+            exit_detail = context["runtimeEnvFailedContext"]["errorMessage"]  # noqa
+        elif "actorUnschedulableContext" in context:
+            exit_detail = context["actorUnschedulableContext"]["errorMessage"]  # noqa
+        elif "creationTaskFailureContext" in context:
+            exit_detail = context["creationTaskFailureContext"][
+                "formattedExceptionString"
+            ]  # noqa
+    light_message["exitDetail"] = exit_detail
+    light_message["startTime"] = int(light_message["startTime"])
+    light_message["endTime"] = int(light_message["endTime"])
+    light_message["requiredResources"] = dict(message.required_resources)
+    light_message["labelSelector"] = dict(message.label_selector)
+    return light_message
+
+
+class NodeHead(SubprocessModule):
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+
+        self._stubs = {}
+        self._collect_memory_info = False
+
+        # The time where the module is started.
+        self._module_start_time = time.time()
+        # The time it takes until the head node is registered. None means
+        # head node hasn't been registered.
+        self._head_node_registration_time_s = None
+        # The node ID of the current head node
+        self._registered_head_node_id = None
+        # Queue of dead nodes to be removed, up to MAX_DEAD_NODES_TO_CACHE
+        self._dead_node_queue = deque()
+
+        self._node_executor = ThreadPoolExecutor(
+            max_workers=RAY_DASHBOARD_NODE_HEAD_TPE_MAX_WORKERS,
+            thread_name_prefix="node_head_node_executor",
+        )
+
+        self._gcs_actor_channel_subscriber = None
+        # A queue of dead actors in order of when they died
+        self._destroyed_actors_queue = deque()
+
+        # -- Internal state --
+        self._loop = get_or_create_event_loop()
+        # NOTE: This executor is intentionally constrained to just 1 thread to
+        #       limit its concurrency, therefore reducing potential for GIL contention
+        self._actor_executor = ThreadPoolExecutor(
+            max_workers=1, thread_name_prefix="node_head_actor_executor"
+        )
+
+        self._background_tasks: Set[asyncio.Task] = set()
+
+    def get_internal_states(self):
+        return {
+            "head_node_registration_time_s": self._head_node_registration_time_s,
+            "registered_nodes": len(DataSource.nodes),
+            "module_lifetime_s": time.time() - self._module_start_time,
+        }
+
+    async def _subscribe_for_node_updates(self) -> AsyncGenerator[dict, None]:
+        """
+        Yields the initial state of all nodes, then yields the updated state of nodes.
+
+        It makes GetAllNodeInfo call only once after the subscription is done, to get
+        the initial state of the nodes.
+        """
+        subscriber = GcsAioNodeInfoSubscriber(address=self.gcs_address)
+        await subscriber.subscribe()
+
+        # Get all node info from GCS. To prevent Time-of-check to time-of-use issue [1],
+        # it happens after the subscription. That is, an update between
+        # get-all-node-info and the subscription is not missed.
+        # [1] https://en.wikipedia.org/wiki/Time-of-check_to_time-of-use
+        node_infos, _ = await self.gcs_client.async_get_all_node_info(timeout=None)
+
+        def _convert_to_dict(messages: Iterable[gcs_pb2.GcsNodeInfo]) -> List[dict]:
+            return [_gcs_node_info_to_dict(m) for m in messages]
+
+        all_node_infos = await self._loop.run_in_executor(
+            self._node_executor,
+            _convert_to_dict,
+            node_infos.values(),
+        )
+
+        for node in all_node_infos:
+            yield node
+
+        while True:
+            try:
+                node_id_updated_info_tuples = await subscriber.poll(
+                    batch_size=node_consts.RAY_DASHBOARD_NODE_SUBSCRIBER_POLL_SIZE
+                )
+
+                if node_id_updated_info_tuples:
+                    _, updated_infos_proto = zip(*node_id_updated_info_tuples)
+                else:
+                    updated_infos_proto = []
+
+                updated_infos = await self._loop.run_in_executor(
+                    self._node_executor,
+                    _convert_to_dict,
+                    updated_infos_proto,
+                )
+
+                for node in updated_infos:
+                    yield node
+            except Exception:
+                logger.exception("Failed handling updated nodes.")
+
+    async def _update_node(self, node: dict):
+        node_id = node["nodeId"]
+        if (
+            node["isHeadNode"]
+            and node["state"] == "ALIVE"
+            and self._registered_head_node_id != node_id
+        ):
+            if self._registered_head_node_id is not None:
+                logger.warning(
+                    "A new head node has become ALIVE. New head node ID: %s, old head node ID: %s, internal states: %s",
+                    node_id,
+                    self._registered_head_node_id,
+                    self.get_internal_states(),
+                )
+            self._registered_head_node_id = node_id
+            self._head_node_registration_time_s = time.time() - self._module_start_time
+            # Put head node ID in the internal KV to be read by JobAgent.
+            # TODO(architkulkarni): Remove once State API exposes which
+            # node is the head node.
+            await self.gcs_client.async_internal_kv_put(
+                ray_constants.KV_HEAD_NODE_ID_KEY,
+                node_id.encode(),
+                overwrite=True,
+                namespace=ray_constants.KV_NAMESPACE_JOB,
+                timeout=GCS_RPC_TIMEOUT_SECONDS,
+            )
+        assert node["state"] in ["ALIVE", "DEAD"]
+        is_alive = node["state"] == "ALIVE"
+        if not is_alive:
+            # Remove the agent address from the internal KV.
+            keys = [
+                f"{DASHBOARD_AGENT_ADDR_NODE_ID_PREFIX}{node_id}",
+                f"{DASHBOARD_AGENT_ADDR_IP_PREFIX}{node['nodeManagerAddress']}",
+            ]
+            tasks = [
+                self.gcs_client.async_internal_kv_del(
+                    key,
+                    del_by_prefix=False,
+                    namespace=ray_constants.KV_NAMESPACE_DASHBOARD,
+                    timeout=GCS_RPC_TIMEOUT_SECONDS,
+                )
+                for key in keys
+            ]
+            await asyncio.gather(*tasks)
+
+            self._dead_node_queue.append(node_id)
+            if len(self._dead_node_queue) > node_consts.MAX_DEAD_NODES_TO_CACHE:
+                evicted_node_id = self._dead_node_queue.popleft()
+                DataSource.nodes.pop(evicted_node_id, None)
+                self._stubs.pop(evicted_node_id, None)
+        DataSource.nodes[node_id] = node
+        # TODO(fyrestone): Handle exceptions.
+        address = "{}:{}".format(
+            node["nodeManagerAddress"], int(node["nodeManagerPort"])
+        )
+        options = ray_constants.GLOBAL_GRPC_OPTIONS
+        channel = init_grpc_channel(address, options, asynchronous=True)
+        stub = node_manager_pb2_grpc.NodeManagerServiceStub(channel)
+        self._stubs[node_id] = stub
+
+    async def _update_nodes(self):
+        """
+        Subscribe to node updates and update the internal states. If the head node is
+        not registered after RAY_DASHBOARD_HEAD_NODE_REGISTRATION_TIMEOUT, it logs a
+        warning only once.
+        """
+        warning_shown = False
+        async for node in self._subscribe_for_node_updates():
+            await self._update_node(node)
+            if not self._head_node_registration_time_s:
+                # head node is not registered yet
+                if (
+                    not warning_shown
+                    and (time.time() - self._module_start_time)
+                    > node_consts.RAY_DASHBOARD_HEAD_NODE_REGISTRATION_TIMEOUT
+                ):
+                    logger.warning(
+                        "Head node is not registered even after "
+                        f"{node_consts.RAY_DASHBOARD_HEAD_NODE_REGISTRATION_TIMEOUT} seconds. "
+                        "The API server might not work correctly. Please "
+                        "report a Github issue. Internal states :"
+                        f"{self.get_internal_states()}"
+                    )
+                    warning_shown = True
+
+    async def get_nodes_logical_resources(self) -> dict:
+
+        from ray.autoscaler.v2.utils import is_autoscaler_v2
+
+        if is_autoscaler_v2():
+            from ray.autoscaler.v2.schema import Stats
+            from ray.autoscaler.v2.sdk import ClusterStatusParser
+
+            try:
+                # here we have a sync request
+                req_time = time.time()
+                cluster_status = await self.gcs_client.async_get_cluster_status()
+                reply_time = time.time()
+                cluster_status = ClusterStatusParser.from_get_cluster_status_reply(
+                    cluster_status,
+                    stats=Stats(
+                        gcs_request_time_s=reply_time - req_time, request_ts_s=req_time
+                    ),
+                )
+            except Exception:
+                logger.exception("Error getting cluster status")
+                return {}
+
+            per_node_resources = {}
+            # TODO(rickyx): we should just return structure data rather than strings.
+            for node in chain(cluster_status.active_nodes, cluster_status.idle_nodes):
+                if not node.resource_usage:
+                    continue
+
+                usage_dict = {
+                    r.resource_name: (r.used, r.total)
+                    for r in node.resource_usage.usage
+                }
+                per_node_resources[node.node_id] = "\n".join(
+                    parse_usage(usage_dict, verbose=True)
+                )
+
+            return per_node_resources
+
+        # Legacy autoscaler status code.
+        (status_string, error) = await asyncio.gather(
+            *[
+                self.gcs_client.async_internal_kv_get(
+                    key.encode(), namespace=None, timeout=GCS_RPC_TIMEOUT_SECONDS
+                )
+                for key in [
+                    DEBUG_AUTOSCALING_STATUS,
+                    DEBUG_AUTOSCALING_ERROR,
+                ]
+            ]
+        )
+        if not status_string:
+            return {}
+        status_dict = json.loads(status_string)
+
+        lm_summary_dict = status_dict.get("load_metrics_report")
+        if lm_summary_dict:
+            lm_summary = LoadMetricsSummary(**lm_summary_dict)
+
+        node_logical_resources = get_per_node_breakdown_as_dict(lm_summary)
+        return node_logical_resources if error is None else {}
+
+    @routes.get("/nodes")
+    @dashboard_optional_utils.aiohttp_cache
+    async def get_all_nodes(self, req) -> aiohttp.web.Response:
+        view = req.query.get("view")
+        if view == "summary":
+            all_node_summary_task = DataOrganizer.get_all_node_summary()
+            nodes_logical_resource_task = self.get_nodes_logical_resources()
+
+            all_node_summary, nodes_logical_resources = await asyncio.gather(
+                all_node_summary_task, nodes_logical_resource_task
+            )
+
+            return dashboard_optional_utils.rest_response(
+                status_code=dashboard_utils.HTTPStatusCode.OK,
+                message="Node summary fetched.",
+                summary=all_node_summary,
+                node_logical_resources=nodes_logical_resources,
+            )
+        elif view is not None and view.lower() == "hostNameList".lower():
+            alive_hostnames = set()
+            for node in DataSource.nodes.values():
+                if node["state"] == "ALIVE":
+                    alive_hostnames.add(node["nodeManagerHostname"])
+            return dashboard_optional_utils.rest_response(
+                status_code=dashboard_utils.HTTPStatusCode.OK,
+                message="Node hostname list fetched.",
+                host_name_list=list(alive_hostnames),
+            )
+        else:
+            return dashboard_optional_utils.rest_response(
+                status_code=dashboard_utils.HTTPStatusCode.INTERNAL_ERROR,
+                message=f"Unknown view {view}",
+            )
+
+    @routes.get("/nodes/{node_id}")
+    @dashboard_optional_utils.aiohttp_cache
+    async def get_node(self, req) -> aiohttp.web.Response:
+        node_id = req.match_info.get("node_id")
+        node_info = await DataOrganizer.get_node_info(node_id)
+        return dashboard_optional_utils.rest_response(
+            status_code=dashboard_utils.HTTPStatusCode.OK,
+            message="Node details fetched.",
+            detail=node_info,
+        )
+
+    @async_loop_forever(node_consts.NODE_STATS_UPDATE_INTERVAL_SECONDS)
+    async def _update_node_stats(self):
+        timeout = max(2, node_consts.NODE_STATS_UPDATE_INTERVAL_SECONDS - 1)
+
+        # NOTE: We copy stubs to make sure
+        #       it doesn't change during the iteration (since its being updated
+        #       from another async task)
+        current_stub_node_id_tuples = list(self._stubs.items())
+
+        node_ids = []
+        get_node_stats_tasks = []
+
+        for _, (node_id, stub) in enumerate(current_stub_node_id_tuples):
+            node_info = DataSource.nodes.get(node_id)
+            if node_info["state"] != "ALIVE":
+                continue
+
+            node_ids.append(node_id)
+            get_node_stats_tasks.append(
+                stub.GetNodeStats(
+                    node_manager_pb2.GetNodeStatsRequest(
+                        include_memory_info=self._collect_memory_info
+                    ),
+                    timeout=timeout,
+                )
+            )
+
+        responses = []
+
+        # NOTE: We're chunking up fetching of the stats to run in batches of no more
+        #       than 100 nodes at a time to avoid flooding the event-loop's queue
+        #       with potentially a large, uninterrupted sequence of tasks updating
+        #       the node stats for very large clusters.
+        for get_node_stats_tasks_chunk in split(get_node_stats_tasks, 100):
+            current_chunk_responses = await asyncio.gather(
+                *get_node_stats_tasks_chunk,
+                return_exceptions=True,
+            )
+
+            responses.extend(current_chunk_responses)
+
+            # We're doing short (25ms) yield after every chunk to make sure
+            #   - We're not overloading the event-loop with excessive # of tasks
+            #   - Allowing 10k nodes stats fetches be sent out performed in 2.5s
+            await asyncio.sleep(0.025)
+
+        def postprocess(node_id_response_tuples):
+            """Pure function reorganizing the data into {node_id: stats}."""
+            new_node_stats = {}
+
+            for node_id, response in node_id_response_tuples:
+                if isinstance(response, asyncio.CancelledError):
+                    pass
+                elif isinstance(response, grpc.RpcError):
+                    if response.code() == grpc.StatusCode.DEADLINE_EXCEEDED:
+                        message = (
+                            f"Cannot reach the node, {node_id}, after timeout "
+                            f" {timeout}. This node may have been overloaded, "
+                            "terminated, or the network is slow."
+                        )
+                    elif response.code() == grpc.StatusCode.UNAVAILABLE:
+                        message = (
+                            f"Cannot reach the node, {node_id}. "
+                            "The node may have been terminated."
+                        )
+                    else:
+                        message = f"Error updating node stats of {node_id}."
+
+                    logger.error(message, exc_info=response)
+                elif isinstance(response, Exception):
+                    logger.error(
+                        f"Error updating node stats of {node_id}.", exc_info=response
+                    )
+                else:
+                    new_node_stats[node_id] = dashboard_utils.node_stats_to_dict(
+                        response
+                    )
+
+            return new_node_stats
+
+        # NOTE: Zip will silently truncate to shorter argument that potentially
+        #       could lead to subtle hard to catch issues, hence the assertion
+        assert len(node_ids) == len(
+            responses
+        ), f"node_ids({len(node_ids)}): {node_ids}, responses({len(responses)}): {responses}"
+
+        new_node_stats = await self._loop.run_in_executor(
+            self._node_executor, postprocess, zip(node_ids, responses)
+        )
+
+        for node_id, new_stat in new_node_stats.items():
+            DataSource.node_stats[node_id] = new_stat
+
+    async def _update_node_physical_stats(self):
+        """
+        Update DataSource.node_physical_stats by subscribing to the GCS resource usage.
+        """
+        subscriber = GcsAioResourceUsageSubscriber(address=self.gcs_address)
+        await subscriber.subscribe()
+
+        while True:
+            try:
+                # The key is b'RAY_REPORTER:{node id hex}',
+                # e.g. b'RAY_REPORTER:2b4fbd...'
+                key, data = await subscriber.poll()
+                if key is None:
+                    continue
+
+                # NOTE: Every iteration is executed inside the thread-pool executor
+                #       (TPE) to avoid blocking the Dashboard's event-loop
+                parsed_data = await self._loop.run_in_executor(
+                    self._node_executor, _parse_node_stats, data
+                )
+
+                node_id = key.split(":")[-1]
+                DataSource.node_physical_stats[node_id] = parsed_data
+            except Exception:
+                logger.exception(
+                    "Error receiving node physical stats from _update_node_physical_stats."
+                )
+
+    async def _update_actors(self):
+        """
+        Processes actor info. First gets all actors from GCS, then subscribes to
+        actor updates. For each actor update, updates DataSource.node_actors and
+        DataSource.actors.
+        """
+
+        # To prevent Time-of-check to time-of-use issue [1], the get-all-actor-info
+        # happens after the subscription. That is, an update between get-all-actor-info
+        # and the subscription is not missed.
+        #
+        # [1] https://en.wikipedia.org/wiki/Time-of-check_to_time-of-use
+        gcs_addr = self.gcs_address
+        actor_channel_subscriber = GcsAioActorSubscriber(address=gcs_addr)
+        await actor_channel_subscriber.subscribe()
+
+        # Get all actor info.
+        while True:
+            try:
+                logger.info("Getting all actor info from GCS.")
+
+                actor_dicts = await self._get_all_actors()
+                # Update actors
+                DataSource.actors = actor_dicts
+
+                # Update node actors and job actors.
+                node_actors = defaultdict(dict)
+                for actor_id_bytes, updated_actor_table in actor_dicts.items():
+                    node_id = updated_actor_table["address"]["nodeId"]
+                    # Update only when node_id is not Nil.
+                    if node_id != actor_consts.NIL_NODE_ID:
+                        node_actors[node_id][actor_id_bytes] = updated_actor_table
+
+                # Update node's actor info
+                DataSource.node_actors = node_actors
+
+                logger.info("Received %d actor info from GCS.", len(actor_dicts))
+
+                # Break, once all initial actors are successfully fetched
+                break
+            except Exception as e:
+                logger.exception("Error Getting all actor info from GCS", exc_info=e)
+                await asyncio.sleep(
+                    actor_consts.RETRY_GET_ALL_ACTOR_INFO_INTERVAL_SECONDS
+                )
+
+        # Pull incremental updates from the GCS channel
+        while True:
+            try:
+                updated_actor_table_entries = await self._poll_updated_actor_table_data(
+                    actor_channel_subscriber
+                )
+
+                for (
+                    actor_id,
+                    updated_actor_table,
+                ) in updated_actor_table_entries.items():
+                    self._process_updated_actor_table(actor_id, updated_actor_table)
+
+                # TODO emit metrics
+                logger.debug(
+                    f"Total events processed: {len(updated_actor_table_entries)}, "
+                    f"queue size: {actor_channel_subscriber.queue_size}"
+                )
+
+            except Exception as e:
+                logger.exception("Error processing actor info from GCS.", exc_info=e)
+
+    async def _poll_updated_actor_table_data(
+        self, actor_channel_subscriber: GcsAioActorSubscriber
+    ) -> Dict[str, Dict[str, Any]]:
+        # TODO make batch size configurable
+        batch = await actor_channel_subscriber.poll(batch_size=200)
+
+        # NOTE: We're offloading conversion to a TPE to make sure we're not
+        #       blocking the event-loop for prolonged period of time irrespective
+        #       of the batch size
+        def _convert_to_dict():
+            return {
+                actor_id_bytes.hex(): _actor_table_data_to_dict(
+                    actor_table_data_message
+                )
+                for actor_id_bytes, actor_table_data_message in batch
+                if actor_id_bytes is not None
+            }
+
+        return await self._loop.run_in_executor(self._actor_executor, _convert_to_dict)
+
+    def _process_updated_actor_table(
+        self, actor_id: str, actor_table_data: Dict[str, Any]
+    ):
+        """NOTE: This method has to be executed on the event-loop, provided that it
+        accesses DataSource data structures (to follow its thread-safety model)"""
+
+        # If actor is not new registered but updated, we only update
+        # states related fields.
+        actor = DataSource.actors.get(actor_id)
+
+        if actor and actor_table_data["state"] != "DEPENDENCIES_UNREADY":
+            for k in ACTOR_TABLE_STATE_COLUMNS:
+                if k in actor_table_data:
+                    actor[k] = actor_table_data[k]
+            actor_table_data = actor
+
+        actor_id = actor_table_data["actorId"]
+        node_id = actor_table_data["address"]["nodeId"]
+
+        if actor_table_data["state"] == "DEAD":
+            self._destroyed_actors_queue.append(actor_id)
+
+        # Update actors.
+        DataSource.actors[actor_id] = actor_table_data
+        # Update node actors (only when node_id is not Nil).
+        if node_id != actor_consts.NIL_NODE_ID:
+            node_actors = DataSource.node_actors.get(node_id, {})
+            node_actors[actor_id] = actor_table_data
+            DataSource.node_actors[node_id] = node_actors
+
+    async def _get_all_actors(self) -> Dict[str, dict]:
+        actors = await self.gcs_client.async_get_all_actor_info(
+            timeout=GCS_RPC_TIMEOUT_SECONDS
+        )
+
+        # NOTE: We're offloading conversion to a TPE to make sure we're not
+        #       blocking the event-loop for prolonged period of time for large clusters
+        def _convert_to_dict():
+            return {
+                actor_id.hex(): _actor_table_data_to_dict(actor_table_data)
+                for actor_id, actor_table_data in actors.items()
+            }
+
+        return await self._loop.run_in_executor(self._actor_executor, _convert_to_dict)
+
+    async def _cleanup_actors(self):
+        while True:
+            try:
+                while len(self._destroyed_actors_queue) > MAX_DESTROYED_ACTORS_TO_CACHE:
+                    actor_id = self._destroyed_actors_queue.popleft()
+                    if actor_id in DataSource.actors:
+                        actor = DataSource.actors.pop(actor_id)
+                        node_id = actor["address"].get("nodeId")
+                        if node_id and node_id != actor_consts.NIL_NODE_ID:
+                            del DataSource.node_actors[node_id][actor_id]
+                await asyncio.sleep(ACTOR_CLEANUP_FREQUENCY)
+            except Exception:
+                logger.exception("Error cleaning up actor info from GCS.")
+
+    @routes.get("/logical/actors")
+    @dashboard_optional_utils.aiohttp_cache
+    async def get_all_actors(self, req) -> aiohttp.web.Response:
+        actor_ids: Optional[List[str]] = None
+        if "ids" in req.query:
+            actor_ids = req.query["ids"].split(",")
+        actors = await DataOrganizer.get_actor_infos(actor_ids=actor_ids)
+        return dashboard_optional_utils.rest_response(
+            status_code=dashboard_utils.HTTPStatusCode.OK,
+            message="All actors fetched.",
+            actors=actors,
+            # False to avoid converting Ray resource name to google style.
+            # It's not necessary here because the fields are already
+            # google formatted when protobuf was converted into dict.
+            convert_google_style=False,
+        )
+
+    @routes.get("/logical/actors/{actor_id}")
+    @dashboard_optional_utils.aiohttp_cache
+    async def get_actor(self, req) -> aiohttp.web.Response:
+        actor_id = req.match_info.get("actor_id")
+        actors = await DataOrganizer.get_actor_infos(actor_ids=[actor_id])
+        return dashboard_optional_utils.rest_response(
+            status_code=dashboard_utils.HTTPStatusCode.OK,
+            message="Actor details fetched.",
+            detail=actors[actor_id],
+        )
+
+    @routes.get("/test/dump")
+    async def dump(self, req) -> aiohttp.web.Response:
+        """
+        Dump all data from datacenter. This is used for testing purpose only.
+        """
+        key = req.query.get("key")
+        if key is None:
+            all_data = {
+                k: dict(v)
+                for k, v in DataSource.__dict__.items()
+                if not k.startswith("_")
+            }
+            return dashboard_optional_utils.rest_response(
+                status_code=dashboard_utils.HTTPStatusCode.OK,
+                message="Fetch all data from datacenter success.",
+                **all_data,
+            )
+        else:
+            data = dict(DataSource.__dict__.get(key))
+            return dashboard_optional_utils.rest_response(
+                status_code=dashboard_utils.HTTPStatusCode.OK,
+                message=f"Fetch {key} from datacenter success.",
+                **{key: data},
+            )
+
+    async def run(self):
+        await super().run()
+        coros = [
+            self._update_nodes(),
+            self._update_node_stats(),
+            self._update_node_physical_stats(),
+            self._update_actors(),
+            self._cleanup_actors(),
+            DataOrganizer.purge(),
+            DataOrganizer.organize(self._node_executor),
+        ]
+        for coro in coros:
+            task = self._loop.create_task(coro)
+            self._background_tasks.add(task)
+            task.add_done_callback(self._background_tasks.discard)
+
+
+def _parse_node_stats(node_stats_str: str) -> dict:
+    stats_dict = json.loads(node_stats_str)
+    if StatsPayload is not None:
+        # Validate the response by parsing the stats_dict.
+        StatsPayload.parse_obj(stats_dict)
+        return stats_dict
+    else:
+        return stats_dict

python/ray/dashboard/modules/node/tests/test_actor.py

@@ -0,0 +1,378 @@
+import logging
+import sys
+import time
+import traceback
+
+import pytest
+import requests
+
+import ray
+from ray._private.test_utils import format_web_url, wait_until_server_available
+from ray.dashboard.modules.node import actor_consts
+from ray.dashboard.tests.conftest import *  # noqa
+from ray.util.placement_group import placement_group
+from ray.util.scheduling_strategies import PlacementGroupSchedulingStrategy
+
+logger = logging.getLogger(__name__)
+
+
+def test_actors(disable_aiohttp_cache, ray_start_with_dashboard):
+    """
+    Tests the REST API dashboard calls on:
+    - alive actors
+    - infeasible actors
+    - dead actors
+    - pg actors (with pg_id set and required_resources formatted)
+    """
+
+    @ray.remote
+    class Foo:
+        def __init__(self, num):
+            self.num = num
+
+        def do_task(self):
+            return self.num
+
+        def get_node_id(self):
+            return ray.get_runtime_context().get_node_id()
+
+        def get_pid(self):
+            import os
+
+            return os.getpid()
+
+        def __repr__(self) -> str:
+            return "Foo1"
+
+    @ray.remote(num_cpus=0, resources={"infeasible_actor": 1})
+    class InfeasibleActor:
+        pass
+
+    pg = placement_group([{"CPU": 1}])
+
+    @ray.remote(num_cpus=1)
+    class PgActor:
+        def __init__(self):
+            pass
+
+        def do_task(self):
+            return 1
+
+        def get_placement_group_id(self):
+            return ray.get_runtime_context().get_placement_group_id()
+
+    foo_actors = [Foo.options(name="first").remote(4), Foo.remote(5)]
+    infeasible_actor = InfeasibleActor.options(name="infeasible").remote()  # noqa
+    dead_actor = Foo.options(name="dead").remote(1)
+    pg_actor = PgActor.options(
+        name="pg",
+        scheduling_strategy=PlacementGroupSchedulingStrategy(
+            placement_group=pg,
+        ),
+    ).remote()
+
+    ray.kill(dead_actor)
+    [actor.do_task.remote() for actor in foo_actors]
+    pg_actor.do_task.remote()
+    webui_url = ray_start_with_dashboard["webui_url"]
+    assert wait_until_server_available(webui_url)
+    webui_url = format_web_url(webui_url)
+    job_id = ray.get_runtime_context().get_job_id()
+    node_id = ray.get(foo_actors[0].get_node_id.remote())
+    pid = ray.get(foo_actors[0].get_pid.remote())
+    placement_group_id = ray.get(pg_actor.get_placement_group_id.remote())
+
+    timeout_seconds = 5
+    start_time = time.time()
+    last_ex = None
+    while True:
+        time.sleep(1)
+        try:
+            resp = requests.get(f"{webui_url}/logical/actors")
+            resp_json = resp.json()
+            resp_data = resp_json["data"]
+            actors = resp_data["actors"]
+            assert len(actors) == 5
+
+            for a in actors.values():
+                if a["name"] == "first":
+                    actor_response = a
+            assert actor_response["jobId"] == job_id
+            assert "Foo" in actor_response["className"]
+            assert "address" in actor_response
+            assert type(actor_response["address"]) is dict
+            assert actor_response["address"]["nodeId"] == node_id
+            assert actor_response["state"] == "ALIVE"
+            assert actor_response["name"] == "first"
+            assert actor_response["numRestarts"] == "0"
+            assert actor_response["pid"] == pid
+            assert actor_response["startTime"] > 0
+            assert actor_response["requiredResources"] == {}
+            assert actor_response["endTime"] == 0
+            assert actor_response["exitDetail"] == "-"
+            assert actor_response["reprName"] == "Foo1"
+            for a in actors.values():
+                # "exitDetail always exits from the response"
+                assert "exitDetail" in a
+
+            # Check the dead actor metadata.
+            for a in actors.values():
+                if a["name"] == "dead":
+                    dead_actor_response = a
+            assert dead_actor_response["endTime"] > 0
+            assert "ray.kill" in dead_actor_response["exitDetail"]
+            assert dead_actor_response["state"] == "DEAD"
+            assert dead_actor_response["name"] == "dead"
+
+            # Check the infeasible actor metadata.
+            for a in actors.values():
+                if a["name"] == "infeasible":
+                    infeasible_actor_response = a
+            # Make sure the infeasible actor's resource name is correct.
+            assert infeasible_actor_response["requiredResources"] == {
+                "infeasible_actor": 1
+            }
+            assert infeasible_actor_response["pid"] == 0
+            all_pids = {entry["pid"] for entry in actors.values()}
+            assert 0 in all_pids  # The infeasible actor
+            assert len(all_pids) > 1
+
+            # Check the pg actor metadata.
+            for a in actors.values():
+                if a["name"] == "pg":
+                    pg_actor_response = a
+            assert pg_actor_response["placementGroupId"] == placement_group_id
+            assert pg_actor_response["requiredResources"] == {"CPU": 1.0}
+
+            break
+        except Exception as ex:
+            last_ex = ex
+        finally:
+            if time.time() > start_time + timeout_seconds:
+                ex_stack = (
+                    traceback.format_exception(
+                        type(last_ex), last_ex, last_ex.__traceback__
+                    )
+                    if last_ex
+                    else []
+                )
+                ex_stack = "".join(ex_stack)
+                raise Exception(f"Timed out while testing, {ex_stack}")
+
+
+def test_actor_with_ids(disable_aiohttp_cache, ray_start_with_dashboard):
+    """
+    Tests the REST API dashboard calls with actor ids in the URL
+    """
+
+    @ray.remote
+    class Actor:
+        def __init__(self, num):
+            self.num = num
+
+        def do_task(self):
+            return self.num
+
+        def get_actor_id(self):
+            return ray.get_runtime_context().get_actor_id()
+
+    webui_url = ray_start_with_dashboard["webui_url"]
+    assert wait_until_server_available(webui_url)
+    webui_url = format_web_url(webui_url)
+    actors = [Actor.options(name=f"Actor{i}").remote(i) for i in range(5)]
+    for actor in actors:
+        actor.do_task.remote()
+
+    actor_ids = [ray.get(actor.get_actor_id.remote()) for actor in actors]
+
+    timeout_seconds = 5
+    start_time = time.time()
+    last_ex = None
+    while True:
+        time.sleep(1)
+        try:
+            actor_idx = 2
+            resp = requests.get(f"{webui_url}/logical/actors/{actor_ids[actor_idx]}")
+            resp_json = resp.json()
+            resp_data = resp_json["data"]
+            actor_detail = resp_data["detail"]
+            assert actor_detail["state"] == "ALIVE"
+            assert actor_detail["name"] == f"Actor{actor_idx}"
+            assert actor_detail["numRestarts"] == "0"
+            assert actor_detail["startTime"] > 0
+            assert actor_detail["requiredResources"] == {}
+            assert actor_detail["endTime"] == 0
+            assert actor_detail["exitDetail"] == "-"
+
+            actor_idxs = [0, 1, 4]
+            actor_idxs_to_id_str = ",".join([str(actor_ids[i]) for i in actor_idxs])
+            resp = requests.get(
+                f"{webui_url}/logical/actors?ids={actor_idxs_to_id_str}"
+            )
+            resp_json = resp.json()
+            resp_actors = resp_json["data"]["actors"]
+            assert len(resp_actors) == len(actor_idxs)
+            for actor_idx in actor_idxs:
+                actor_detail = resp_actors[str(actor_ids[actor_idx])]
+                assert actor_detail["state"] == "ALIVE"
+                assert actor_detail["name"] == f"Actor{actor_idx}"
+                assert actor_detail["numRestarts"] == "0"
+                assert actor_detail["startTime"] > 0
+                assert actor_detail["requiredResources"] == {}
+                assert actor_detail["endTime"] == 0
+                assert actor_detail["exitDetail"] == "-"
+
+            break
+        except Exception as ex:
+            last_ex = ex
+        finally:
+            if time.time() > start_time + timeout_seconds:
+                ex_stack = (
+                    traceback.format_exception(
+                        type(last_ex), last_ex, last_ex.__traceback__
+                    )
+                    if last_ex
+                    else []
+                )
+                ex_stack = "".join(ex_stack)
+                raise Exception(f"Timed out while testing, {ex_stack}")
+
+
+def test_nil_node(enable_test_module, disable_aiohttp_cache, ray_start_with_dashboard):
+    assert wait_until_server_available(ray_start_with_dashboard["webui_url"]) is True
+    webui_url = ray_start_with_dashboard["webui_url"]
+    assert wait_until_server_available(webui_url)
+    webui_url = format_web_url(webui_url)
+
+    @ray.remote(num_gpus=1)
+    class InfeasibleActor:
+        pass
+
+    infeasible_actor = InfeasibleActor.remote()  # noqa
+
+    timeout_seconds = 5
+    start_time = time.time()
+    last_ex = None
+    while True:
+        time.sleep(1)
+        try:
+            resp = requests.get(f"{webui_url}/logical/actors")
+            resp_json = resp.json()
+            resp_data = resp_json["data"]
+            actors = resp_data["actors"]
+            assert len(actors) == 1
+            response = requests.get(webui_url + "/test/dump?key=node_actors")
+            response.raise_for_status()
+            result = response.json()
+            assert actor_consts.NIL_NODE_ID not in result["data"]["nodeActors"]
+            break
+        except Exception as ex:
+            last_ex = ex
+        finally:
+            if time.time() > start_time + timeout_seconds:
+                ex_stack = (
+                    traceback.format_exception(
+                        type(last_ex), last_ex, last_ex.__traceback__
+                    )
+                    if last_ex
+                    else []
+                )
+                ex_stack = "".join(ex_stack)
+                raise Exception(f"Timed out while testing, {ex_stack}")
+
+
+def test_actor_cleanup(
+    disable_aiohttp_cache, reduce_actor_cache, ray_start_with_dashboard
+):
+    @ray.remote
+    class Foo:
+        def __init__(self, num):
+            self.num = num
+
+        def do_task(self):
+            return self.num
+
+    @ray.remote(num_gpus=1)
+    class InfeasibleActor:
+        pass
+
+    infeasible_actor = InfeasibleActor.remote()  # noqa
+
+    foo_actors = [
+        Foo.remote(1),
+        Foo.remote(2),
+        Foo.remote(3),
+        Foo.remote(4),
+        Foo.remote(5),
+        Foo.remote(6),
+    ]
+    results = [actor.do_task.remote() for actor in foo_actors]  # noqa
+    webui_url = ray_start_with_dashboard["webui_url"]
+    assert wait_until_server_available(webui_url)
+    webui_url = format_web_url(webui_url)
+
+    timeout_seconds = 8
+    start_time = time.time()
+    last_ex = None
+    while True:
+        time.sleep(1)
+        try:
+            resp = requests.get(f"{webui_url}/logical/actors")
+            resp_json = resp.json()
+            resp_data = resp_json["data"]
+            actors = resp_data["actors"]
+            # Although max cache is 3, there should be 7 actors
+            # because they are all still alive.
+            assert len(actors) == 7
+
+            break
+        except Exception as ex:
+            last_ex = ex
+        finally:
+            if time.time() > start_time + timeout_seconds:
+                ex_stack = (
+                    traceback.format_exception(
+                        type(last_ex), last_ex, last_ex.__traceback__
+                    )
+                    if last_ex
+                    else []
+                )
+                ex_stack = "".join(ex_stack)
+                raise Exception(f"Timed out while testing, {ex_stack}")
+
+    # kill
+    ray.kill(infeasible_actor)
+    [ray.kill(foo_actor) for foo_actor in foo_actors]
+    # Wait 5 seconds for cleanup to finish
+    time.sleep(5)
+
+    # Check only three remaining in cache
+    start_time = time.time()
+    while True:
+        time.sleep(1)
+        try:
+            resp = requests.get(f"{webui_url}/logical/actors")
+            resp_json = resp.json()
+            resp_data = resp_json["data"]
+            actors = resp_data["actors"]
+            # Max cache is 3 so only 3 actors should be left.
+            assert len(actors) == 3
+
+            break
+        except Exception as ex:
+            last_ex = ex
+        finally:
+            if time.time() > start_time + timeout_seconds:
+                ex_stack = (
+                    traceback.format_exception(
+                        type(last_ex), last_ex, last_ex.__traceback__
+                    )
+                    if last_ex
+                    else []
+                )
+                ex_stack = "".join(ex_stack)
+                raise Exception(f"Timed out while testing, {ex_stack}")
+
+
+if __name__ == "__main__":
+    sys.exit(pytest.main(["-v", __file__]))

python/ray/dashboard/modules/node/tests/test_node.py

@@ -0,0 +1,416 @@
+import logging
+import os
+import random
+import sys
+import threading
+import time
+import traceback
+from datetime import datetime, timedelta
+
+import pytest
+import requests
+
+import ray
+from ray._common.test_utils import wait_for_condition
+from ray._private.test_utils import (
+    format_web_url,
+    wait_until_server_available,
+)
+from ray.cluster_utils import Cluster
+from ray.dashboard.consts import RAY_DASHBOARD_STATS_UPDATING_INTERVAL
+from ray.dashboard.tests.conftest import *  # noqa
+
+logger = logging.getLogger(__name__)
+
+
+def test_nodes_update(enable_test_module, ray_start_with_dashboard):
+    assert wait_until_server_available(ray_start_with_dashboard["webui_url"]) is True
+    webui_url = ray_start_with_dashboard["webui_url"]
+    webui_url = format_web_url(webui_url)
+
+    timeout_seconds = 10
+    start_time = time.time()
+    while True:
+        time.sleep(1)
+        try:
+            response = requests.get(webui_url + "/test/dump")
+            response.raise_for_status()
+            try:
+                dump_info = response.json()
+            except Exception as ex:
+                logger.info("failed response: %s", response.text)
+                raise ex
+            assert dump_info["result"] is True
+            dump_data = dump_info["data"]
+            assert len(dump_data["nodes"]) == 1
+            break
+
+        except (AssertionError, requests.exceptions.ConnectionError):
+            logger.exception("Retry")
+        finally:
+            if time.time() > start_time + timeout_seconds:
+                raise Exception("Timed out while testing.")
+
+
+def test_node_info(disable_aiohttp_cache, ray_start_with_dashboard):
+    @ray.remote
+    class Actor:
+        def getpid(self):
+            print(f"actor pid={os.getpid()}")
+            return os.getpid()
+
+    actors = [Actor.remote(), Actor.remote()]
+    actor_pids = [actor.getpid.remote() for actor in actors]
+    actor_pids = set(ray.get(actor_pids))
+
+    assert wait_until_server_available(ray_start_with_dashboard["webui_url"]) is True
+    webui_url = ray_start_with_dashboard["webui_url"]
+    webui_url = format_web_url(webui_url)
+    node_id = ray_start_with_dashboard["node_id"]
+
+    # NOTE: Leaving sum buffer time for data to get refreshed
+    timeout_seconds = RAY_DASHBOARD_STATS_UPDATING_INTERVAL * 1.5
+
+    start_time = time.time()
+    last_ex = None
+    while True:
+        time.sleep(1)
+        try:
+            response = requests.get(webui_url + "/nodes?view=hostnamelist")
+            response.raise_for_status()
+            hostname_list = response.json()
+            assert hostname_list["result"] is True, hostname_list["msg"]
+            hostname_list = hostname_list["data"]["hostNameList"]
+            assert len(hostname_list) == 1
+
+            hostname = hostname_list[0]
+            response = requests.get(webui_url + f"/nodes/{node_id}")
+            response.raise_for_status()
+            detail = response.json()
+            assert detail["result"] is True, detail["msg"]
+            detail = detail["data"]["detail"]
+            assert detail["hostname"] == hostname
+            assert detail["raylet"]["state"] == "ALIVE"
+            assert detail["raylet"]["isHeadNode"] is True
+            assert "raylet" in detail["cmdline"][0]
+            assert len(detail["workers"]) >= 2
+            assert len(detail["actors"]) == 2, detail["actors"]
+
+            actor_worker_pids = set()
+            for worker in detail["workers"]:
+                if "ray::Actor" in worker["cmdline"][0]:
+                    actor_worker_pids.add(worker["pid"])
+            assert actor_worker_pids == actor_pids
+
+            response = requests.get(webui_url + "/nodes?view=summary")
+            response.raise_for_status()
+            summary = response.json()
+            assert summary["result"] is True, summary["msg"]
+            assert len(summary["data"]["summary"]) == 1
+            summary = summary["data"]["summary"][0]
+            assert summary["hostname"] == hostname
+            assert summary["raylet"]["state"] == "ALIVE"
+            assert "raylet" in summary["cmdline"][0]
+            assert "workers" not in summary
+            assert "actors" not in summary
+            assert "objectStoreAvailableMemory" in summary["raylet"]
+            assert "objectStoreUsedMemory" in summary["raylet"]
+            break
+        except Exception as ex:
+            last_ex = ex
+        finally:
+            if time.time() > start_time + timeout_seconds:
+                ex_stack = (
+                    traceback.format_exception(
+                        type(last_ex), last_ex, last_ex.__traceback__
+                    )
+                    if last_ex
+                    else []
+                )
+                ex_stack = "".join(ex_stack)
+                raise Exception(f"Timed out while testing, {ex_stack}")
+
+
+@pytest.mark.parametrize(
+    "ray_start_cluster_head_with_env_vars",
+    [
+        {
+            "include_dashboard": True,
+            "env_vars": {
+                "RAY_maximum_gcs_dead_node_cached_count": "1",
+            },
+            "_system_config": {
+                "health_check_initial_delay_ms": 0,
+                "health_check_timeout_ms": 100,
+                "health_check_failure_threshold": 3,
+                "health_check_period_ms": 100,
+            },
+        }
+    ],
+    indirect=True,
+)
+def test_dead_node_cache_contains_latest_dead_node_if_cache_overflows(
+    enable_test_module, disable_aiohttp_cache, ray_start_cluster_head_with_env_vars
+):
+    cluster: Cluster = ray_start_cluster_head_with_env_vars
+    assert wait_until_server_available(
+        cluster.webui_url
+    ), "Failed to connect to the Dashboard Server"
+    webui_url = format_web_url(cluster.webui_url)
+
+    def _compare_dead_node_set(expected_alive_nodes, expected_dead_nodes):
+        try:
+            response = requests.get(webui_url + "/nodes?view=summary")
+            response.raise_for_status()
+            summary = response.json()
+            assert summary["result"] is True, summary["msg"]
+            summary = summary["data"]["summary"]
+            dead_nodes = set()
+            alive_nodes = set()
+            for node_info in summary:
+                node_id = node_info["raylet"]["nodeId"]
+                response = requests.get(webui_url + f"/nodes/{node_id}")
+                response.raise_for_status()
+                if node_info["raylet"]["state"] == "DEAD":
+                    dead_nodes.add(node_id)
+                if node_info["raylet"]["state"] == "ALIVE":
+                    alive_nodes.add(node_id)
+            assert alive_nodes == expected_alive_nodes
+            assert dead_nodes == expected_dead_nodes
+            return True
+        except Exception as ex:
+            logger.info(ex)
+            return False
+
+    node_1 = cluster.add_node()
+    head_node_id = ray.get_runtime_context().get_node_id()
+    node_1_id = node_1.node_id
+    curr_alive_nodes = {head_node_id, node_1_id}
+    curr_dead_nodes = set()
+    wait_for_condition(
+        _compare_dead_node_set,
+        10,
+        expected_alive_nodes=curr_alive_nodes,
+        expected_dead_nodes=curr_dead_nodes,
+    )
+
+    node_2 = cluster.add_node()
+    node_2_id = node_2.node_id
+    curr_alive_nodes.add(node_2_id)
+    wait_for_condition(
+        _compare_dead_node_set,
+        10,
+        expected_alive_nodes=curr_alive_nodes,
+        expected_dead_nodes=curr_dead_nodes,
+    )
+
+    cluster.remove_node(node_1, allow_graceful=False)
+    curr_alive_nodes.remove(node_1_id)
+    curr_dead_nodes.add(node_1_id)
+    wait_for_condition(
+        _compare_dead_node_set,
+        10,
+        expected_alive_nodes=curr_alive_nodes,
+        expected_dead_nodes=curr_dead_nodes,
+    )
+
+    cluster.remove_node(node_2, allow_graceful=False)
+    curr_alive_nodes.remove(node_2_id)
+    curr_dead_nodes.remove(node_1_id)
+    curr_dead_nodes.add(node_2_id)
+    wait_for_condition(
+        _compare_dead_node_set,
+        10,
+        expected_alive_nodes=curr_alive_nodes,
+        expected_dead_nodes=curr_dead_nodes,
+    )
+
+
+@pytest.mark.parametrize(
+    "ray_start_cluster_head",
+    [
+        {
+            "include_dashboard": True,
+            "_system_config": {
+                "health_check_initial_delay_ms": 0,
+                "health_check_timeout_ms": 100,
+                "health_check_failure_threshold": 3,
+                "health_check_period_ms": 100,
+            },
+        }
+    ],
+    indirect=True,
+)
+def test_multi_nodes_info(
+    enable_test_module, disable_aiohttp_cache, ray_start_cluster_head
+):
+    cluster: Cluster = ray_start_cluster_head
+    assert wait_until_server_available(cluster.webui_url) is True
+    webui_url = cluster.webui_url
+    webui_url = format_web_url(webui_url)
+    cluster.add_node()
+    cluster.add_node()
+    dead_node = cluster.add_node()
+    cluster.remove_node(dead_node, allow_graceful=False)
+
+    def _check_nodes():
+        try:
+            response = requests.get(webui_url + "/nodes?view=summary")
+            response.raise_for_status()
+            summary = response.json()
+            assert summary["result"] is True, summary["msg"]
+            summary = summary["data"]["summary"]
+            assert len(summary) == 4
+            for node_info in summary:
+                node_id = node_info["raylet"]["nodeId"]
+                response = requests.get(webui_url + f"/nodes/{node_id}")
+                response.raise_for_status()
+                detail = response.json()
+                assert detail["result"] is True, detail["msg"]
+                detail = detail["data"]["detail"]
+                if node_id != dead_node.node_id:
+                    assert detail["raylet"]["state"] == "ALIVE"
+                else:
+                    assert detail["raylet"]["state"] == "DEAD"
+                    assert detail["raylet"].get("objectStoreAvailableMemory", 0) == 0
+            return True
+        except Exception as ex:
+            logger.info(ex)
+            return False
+
+    wait_for_condition(_check_nodes, timeout=15)
+
+
+@pytest.mark.parametrize(
+    "ray_start_cluster_head", [{"include_dashboard": True}], indirect=True
+)
+def test_multi_node_churn(
+    enable_test_module, disable_aiohttp_cache, ray_start_cluster_head
+):
+    cluster: Cluster = ray_start_cluster_head
+    assert wait_until_server_available(cluster.webui_url) is True
+    webui_url = format_web_url(cluster.webui_url)
+
+    success = True
+
+    def verify():
+        nonlocal success
+        while True:
+            try:
+                resp = requests.get(webui_url)
+                resp.raise_for_status()
+                resp = requests.get(webui_url + "/nodes?view=summary")
+                resp.raise_for_status()
+                summary = resp.json()
+                assert summary["result"] is True, summary["msg"]
+                assert summary["data"]["summary"]
+                time.sleep(1)
+            except Exception:
+                success = False
+                break
+
+    t = threading.Thread(target=verify, daemon=True)
+    t.start()
+
+    t_st = datetime.now()
+    duration = timedelta(seconds=60)
+    worker_nodes = []
+    while datetime.now() < t_st + duration:
+        time.sleep(5)
+        if len(worker_nodes) < 2:
+            worker_nodes.append(cluster.add_node())
+            continue
+        should_add_node = random.randint(0, 1)
+        if should_add_node:
+            worker_nodes.append(cluster.add_node())
+        else:
+            node_index = random.randrange(0, len(worker_nodes))
+            node_to_remove = worker_nodes.pop(node_index)
+            cluster.remove_node(node_to_remove)
+
+    assert success
+
+
+@pytest.mark.skipif(
+    sys.platform == "win32", reason="setproctitle does not change psutil.cmdline"
+)
+def test_node_physical_stats(enable_test_module, shutdown_only):
+    """
+    Tests NodeHead._update_node_physical_stats.
+    """
+    addresses = ray.init(include_dashboard=True, num_cpus=6)
+
+    @ray.remote(num_cpus=1)
+    class Actor:
+        def getpid(self):
+            return os.getpid()
+
+    actors = [Actor.remote() for _ in range(6)]
+    actor_pids = ray.get([actor.getpid.remote() for actor in actors])
+    actor_pids = set(actor_pids)
+
+    webui_url = addresses["webui_url"]
+    assert wait_until_server_available(webui_url) is True
+    webui_url = format_web_url(webui_url)
+
+    def _check_workers():
+        try:
+            resp = requests.get(webui_url + "/test/dump?key=node_physical_stats")
+            resp.raise_for_status()
+            result = resp.json()
+            assert result["result"] is True
+            node_physical_stats = result["data"]["nodePhysicalStats"]
+            assert len(node_physical_stats) == 1
+            current_stats = node_physical_stats[addresses["node_id"]]
+            # Check Actor workers
+            current_actor_pids = set()
+            for worker in current_stats["workers"]:
+                if "ray::Actor" in worker["cmdline"][0]:
+                    current_actor_pids.add(worker["pid"])
+            assert current_actor_pids == actor_pids
+            # Check raylet cmdline
+            assert "raylet" in current_stats["cmdline"][0]
+            return True
+        except Exception as ex:
+            logger.info(ex)
+            return False
+
+    wait_for_condition(_check_workers, timeout=10)
+
+
+def test_worker_pids_reported(enable_test_module, ray_start_with_dashboard):
+    assert wait_until_server_available(ray_start_with_dashboard["webui_url"]) is True
+    webui_url = ray_start_with_dashboard["webui_url"]
+    webui_url = format_web_url(webui_url)
+    node_id = ray_start_with_dashboard["node_id"]
+
+    @ray.remote(runtime_env={"uv": {"packages": ["requests==2.32.5"]}})
+    class UvActor:
+        def get_pid(self):
+            return os.getpid()
+
+    uv_actor = UvActor.remote()
+    uv_actor_pid = ray.get(uv_actor.get_pid.remote())
+    driver_pid = os.getpid()
+
+    def _check_worker_pids():
+        try:
+            response = requests.get(webui_url + f"/nodes/{node_id}")
+            response.raise_for_status()
+            dump_info = response.json()
+            assert dump_info["result"] is True
+            detail = dump_info["data"]["detail"]
+            pids = [worker["pid"] for worker in detail["workers"]]
+            assert len(pids) >= 2  # might include idle worker
+            assert uv_actor_pid in pids
+            assert driver_pid in pids
+            return True
+        except Exception as ex:
+            logger.info(ex)
+            return False
+
+    wait_for_condition(_check_worker_pids, timeout=20)
+
+
+if __name__ == "__main__":
+    sys.exit(pytest.main(["-v", __file__]))