chore: import upstream snapshot with attribution

python/ray/dashboard/modules/node/tests/test_node.py

@@ -0,0 +1,416 @@
+import logging
+import os
+import random
+import sys
+import threading
+import time
+import traceback
+from datetime import datetime, timedelta
+
+import pytest
+import requests
+
+import ray
+from ray._common.test_utils import wait_for_condition
+from ray._private.test_utils import (
+    format_web_url,
+    wait_until_server_available,
+)
+from ray.cluster_utils import Cluster
+from ray.dashboard.consts import RAY_DASHBOARD_STATS_UPDATING_INTERVAL
+from ray.dashboard.tests.conftest import *  # noqa
+
+logger = logging.getLogger(__name__)
+
+
+def test_nodes_update(enable_test_module, ray_start_with_dashboard):
+    assert wait_until_server_available(ray_start_with_dashboard["webui_url"]) is True
+    webui_url = ray_start_with_dashboard["webui_url"]
+    webui_url = format_web_url(webui_url)
+
+    timeout_seconds = 10
+    start_time = time.time()
+    while True:
+        time.sleep(1)
+        try:
+            response = requests.get(webui_url + "/test/dump")
+            response.raise_for_status()
+            try:
+                dump_info = response.json()
+            except Exception as ex:
+                logger.info("failed response: %s", response.text)
+                raise ex
+            assert dump_info["result"] is True
+            dump_data = dump_info["data"]
+            assert len(dump_data["nodes"]) == 1
+            break
+
+        except (AssertionError, requests.exceptions.ConnectionError):
+            logger.exception("Retry")
+        finally:
+            if time.time() > start_time + timeout_seconds:
+                raise Exception("Timed out while testing.")
+
+
+def test_node_info(disable_aiohttp_cache, ray_start_with_dashboard):
+    @ray.remote
+    class Actor:
+        def getpid(self):
+            print(f"actor pid={os.getpid()}")
+            return os.getpid()
+
+    actors = [Actor.remote(), Actor.remote()]
+    actor_pids = [actor.getpid.remote() for actor in actors]
+    actor_pids = set(ray.get(actor_pids))
+
+    assert wait_until_server_available(ray_start_with_dashboard["webui_url"]) is True
+    webui_url = ray_start_with_dashboard["webui_url"]
+    webui_url = format_web_url(webui_url)
+    node_id = ray_start_with_dashboard["node_id"]
+
+    # NOTE: Leaving sum buffer time for data to get refreshed
+    timeout_seconds = RAY_DASHBOARD_STATS_UPDATING_INTERVAL * 1.5
+
+    start_time = time.time()
+    last_ex = None
+    while True:
+        time.sleep(1)
+        try:
+            response = requests.get(webui_url + "/nodes?view=hostnamelist")
+            response.raise_for_status()
+            hostname_list = response.json()
+            assert hostname_list["result"] is True, hostname_list["msg"]
+            hostname_list = hostname_list["data"]["hostNameList"]
+            assert len(hostname_list) == 1
+
+            hostname = hostname_list[0]
+            response = requests.get(webui_url + f"/nodes/{node_id}")
+            response.raise_for_status()
+            detail = response.json()
+            assert detail["result"] is True, detail["msg"]
+            detail = detail["data"]["detail"]
+            assert detail["hostname"] == hostname
+            assert detail["raylet"]["state"] == "ALIVE"
+            assert detail["raylet"]["isHeadNode"] is True
+            assert "raylet" in detail["cmdline"][0]
+            assert len(detail["workers"]) >= 2
+            assert len(detail["actors"]) == 2, detail["actors"]
+
+            actor_worker_pids = set()
+            for worker in detail["workers"]:
+                if "ray::Actor" in worker["cmdline"][0]:
+                    actor_worker_pids.add(worker["pid"])
+            assert actor_worker_pids == actor_pids
+
+            response = requests.get(webui_url + "/nodes?view=summary")
+            response.raise_for_status()
+            summary = response.json()
+            assert summary["result"] is True, summary["msg"]
+            assert len(summary["data"]["summary"]) == 1
+            summary = summary["data"]["summary"][0]
+            assert summary["hostname"] == hostname
+            assert summary["raylet"]["state"] == "ALIVE"
+            assert "raylet" in summary["cmdline"][0]
+            assert "workers" not in summary
+            assert "actors" not in summary
+            assert "objectStoreAvailableMemory" in summary["raylet"]
+            assert "objectStoreUsedMemory" in summary["raylet"]
+            break
+        except Exception as ex:
+            last_ex = ex
+        finally:
+            if time.time() > start_time + timeout_seconds:
+                ex_stack = (
+                    traceback.format_exception(
+                        type(last_ex), last_ex, last_ex.__traceback__
+                    )
+                    if last_ex
+                    else []
+                )
+                ex_stack = "".join(ex_stack)
+                raise Exception(f"Timed out while testing, {ex_stack}")
+
+
+@pytest.mark.parametrize(
+    "ray_start_cluster_head_with_env_vars",
+    [
+        {
+            "include_dashboard": True,
+            "env_vars": {
+                "RAY_maximum_gcs_dead_node_cached_count": "1",
+            },
+            "_system_config": {
+                "health_check_initial_delay_ms": 0,
+                "health_check_timeout_ms": 100,
+                "health_check_failure_threshold": 3,
+                "health_check_period_ms": 100,
+            },
+        }
+    ],
+    indirect=True,
+)
+def test_dead_node_cache_contains_latest_dead_node_if_cache_overflows(
+    enable_test_module, disable_aiohttp_cache, ray_start_cluster_head_with_env_vars
+):
+    cluster: Cluster = ray_start_cluster_head_with_env_vars
+    assert wait_until_server_available(
+        cluster.webui_url
+    ), "Failed to connect to the Dashboard Server"
+    webui_url = format_web_url(cluster.webui_url)
+
+    def _compare_dead_node_set(expected_alive_nodes, expected_dead_nodes):
+        try:
+            response = requests.get(webui_url + "/nodes?view=summary")
+            response.raise_for_status()
+            summary = response.json()
+            assert summary["result"] is True, summary["msg"]
+            summary = summary["data"]["summary"]
+            dead_nodes = set()
+            alive_nodes = set()
+            for node_info in summary:
+                node_id = node_info["raylet"]["nodeId"]
+                response = requests.get(webui_url + f"/nodes/{node_id}")
+                response.raise_for_status()
+                if node_info["raylet"]["state"] == "DEAD":
+                    dead_nodes.add(node_id)
+                if node_info["raylet"]["state"] == "ALIVE":
+                    alive_nodes.add(node_id)
+            assert alive_nodes == expected_alive_nodes
+            assert dead_nodes == expected_dead_nodes
+            return True
+        except Exception as ex:
+            logger.info(ex)
+            return False
+
+    node_1 = cluster.add_node()
+    head_node_id = ray.get_runtime_context().get_node_id()
+    node_1_id = node_1.node_id
+    curr_alive_nodes = {head_node_id, node_1_id}
+    curr_dead_nodes = set()
+    wait_for_condition(
+        _compare_dead_node_set,
+        10,
+        expected_alive_nodes=curr_alive_nodes,
+        expected_dead_nodes=curr_dead_nodes,
+    )
+
+    node_2 = cluster.add_node()
+    node_2_id = node_2.node_id
+    curr_alive_nodes.add(node_2_id)
+    wait_for_condition(
+        _compare_dead_node_set,
+        10,
+        expected_alive_nodes=curr_alive_nodes,
+        expected_dead_nodes=curr_dead_nodes,
+    )
+
+    cluster.remove_node(node_1, allow_graceful=False)
+    curr_alive_nodes.remove(node_1_id)
+    curr_dead_nodes.add(node_1_id)
+    wait_for_condition(
+        _compare_dead_node_set,
+        10,
+        expected_alive_nodes=curr_alive_nodes,
+        expected_dead_nodes=curr_dead_nodes,
+    )
+
+    cluster.remove_node(node_2, allow_graceful=False)
+    curr_alive_nodes.remove(node_2_id)
+    curr_dead_nodes.remove(node_1_id)
+    curr_dead_nodes.add(node_2_id)
+    wait_for_condition(
+        _compare_dead_node_set,
+        10,
+        expected_alive_nodes=curr_alive_nodes,
+        expected_dead_nodes=curr_dead_nodes,
+    )
+
+
+@pytest.mark.parametrize(
+    "ray_start_cluster_head",
+    [
+        {
+            "include_dashboard": True,
+            "_system_config": {
+                "health_check_initial_delay_ms": 0,
+                "health_check_timeout_ms": 100,
+                "health_check_failure_threshold": 3,
+                "health_check_period_ms": 100,
+            },
+        }
+    ],
+    indirect=True,
+)
+def test_multi_nodes_info(
+    enable_test_module, disable_aiohttp_cache, ray_start_cluster_head
+):
+    cluster: Cluster = ray_start_cluster_head
+    assert wait_until_server_available(cluster.webui_url) is True
+    webui_url = cluster.webui_url
+    webui_url = format_web_url(webui_url)
+    cluster.add_node()
+    cluster.add_node()
+    dead_node = cluster.add_node()
+    cluster.remove_node(dead_node, allow_graceful=False)
+
+    def _check_nodes():
+        try:
+            response = requests.get(webui_url + "/nodes?view=summary")
+            response.raise_for_status()
+            summary = response.json()
+            assert summary["result"] is True, summary["msg"]
+            summary = summary["data"]["summary"]
+            assert len(summary) == 4
+            for node_info in summary:
+                node_id = node_info["raylet"]["nodeId"]
+                response = requests.get(webui_url + f"/nodes/{node_id}")
+                response.raise_for_status()
+                detail = response.json()
+                assert detail["result"] is True, detail["msg"]
+                detail = detail["data"]["detail"]
+                if node_id != dead_node.node_id:
+                    assert detail["raylet"]["state"] == "ALIVE"
+                else:
+                    assert detail["raylet"]["state"] == "DEAD"
+                    assert detail["raylet"].get("objectStoreAvailableMemory", 0) == 0
+            return True
+        except Exception as ex:
+            logger.info(ex)
+            return False
+
+    wait_for_condition(_check_nodes, timeout=15)
+
+
+@pytest.mark.parametrize(
+    "ray_start_cluster_head", [{"include_dashboard": True}], indirect=True
+)
+def test_multi_node_churn(
+    enable_test_module, disable_aiohttp_cache, ray_start_cluster_head
+):
+    cluster: Cluster = ray_start_cluster_head
+    assert wait_until_server_available(cluster.webui_url) is True
+    webui_url = format_web_url(cluster.webui_url)
+
+    success = True
+
+    def verify():
+        nonlocal success
+        while True:
+            try:
+                resp = requests.get(webui_url)
+                resp.raise_for_status()
+                resp = requests.get(webui_url + "/nodes?view=summary")
+                resp.raise_for_status()
+                summary = resp.json()
+                assert summary["result"] is True, summary["msg"]
+                assert summary["data"]["summary"]
+                time.sleep(1)
+            except Exception:
+                success = False
+                break
+
+    t = threading.Thread(target=verify, daemon=True)
+    t.start()
+
+    t_st = datetime.now()
+    duration = timedelta(seconds=60)
+    worker_nodes = []
+    while datetime.now() < t_st + duration:
+        time.sleep(5)
+        if len(worker_nodes) < 2:
+            worker_nodes.append(cluster.add_node())
+            continue
+        should_add_node = random.randint(0, 1)
+        if should_add_node:
+            worker_nodes.append(cluster.add_node())
+        else:
+            node_index = random.randrange(0, len(worker_nodes))
+            node_to_remove = worker_nodes.pop(node_index)
+            cluster.remove_node(node_to_remove)
+
+    assert success
+
+
+@pytest.mark.skipif(
+    sys.platform == "win32", reason="setproctitle does not change psutil.cmdline"
+)
+def test_node_physical_stats(enable_test_module, shutdown_only):
+    """
+    Tests NodeHead._update_node_physical_stats.
+    """
+    addresses = ray.init(include_dashboard=True, num_cpus=6)
+
+    @ray.remote(num_cpus=1)
+    class Actor:
+        def getpid(self):
+            return os.getpid()
+
+    actors = [Actor.remote() for _ in range(6)]
+    actor_pids = ray.get([actor.getpid.remote() for actor in actors])
+    actor_pids = set(actor_pids)
+
+    webui_url = addresses["webui_url"]
+    assert wait_until_server_available(webui_url) is True
+    webui_url = format_web_url(webui_url)
+
+    def _check_workers():
+        try:
+            resp = requests.get(webui_url + "/test/dump?key=node_physical_stats")
+            resp.raise_for_status()
+            result = resp.json()
+            assert result["result"] is True
+            node_physical_stats = result["data"]["nodePhysicalStats"]
+            assert len(node_physical_stats) == 1
+            current_stats = node_physical_stats[addresses["node_id"]]
+            # Check Actor workers
+            current_actor_pids = set()
+            for worker in current_stats["workers"]:
+                if "ray::Actor" in worker["cmdline"][0]:
+                    current_actor_pids.add(worker["pid"])
+            assert current_actor_pids == actor_pids
+            # Check raylet cmdline
+            assert "raylet" in current_stats["cmdline"][0]
+            return True
+        except Exception as ex:
+            logger.info(ex)
+            return False
+
+    wait_for_condition(_check_workers, timeout=10)
+
+
+def test_worker_pids_reported(enable_test_module, ray_start_with_dashboard):
+    assert wait_until_server_available(ray_start_with_dashboard["webui_url"]) is True
+    webui_url = ray_start_with_dashboard["webui_url"]
+    webui_url = format_web_url(webui_url)
+    node_id = ray_start_with_dashboard["node_id"]
+
+    @ray.remote(runtime_env={"uv": {"packages": ["requests==2.32.5"]}})
+    class UvActor:
+        def get_pid(self):
+            return os.getpid()
+
+    uv_actor = UvActor.remote()
+    uv_actor_pid = ray.get(uv_actor.get_pid.remote())
+    driver_pid = os.getpid()
+
+    def _check_worker_pids():
+        try:
+            response = requests.get(webui_url + f"/nodes/{node_id}")
+            response.raise_for_status()
+            dump_info = response.json()
+            assert dump_info["result"] is True
+            detail = dump_info["data"]["detail"]
+            pids = [worker["pid"] for worker in detail["workers"]]
+            assert len(pids) >= 2  # might include idle worker
+            assert uv_actor_pid in pids
+            assert driver_pid in pids
+            return True
+        except Exception as ex:
+            logger.info(ex)
+            return False
+
+    wait_for_condition(_check_worker_pids, timeout=20)
+
+
+if __name__ == "__main__":
+    sys.exit(pytest.main(["-v", __file__]))