wehub-resource-sync
417 lines
14 KiB
Python
417 lines
14 KiB
Python
import logging
|
|
import os
|
|
import random
|
|
import sys
|
|
import threading
|
|
import time
|
|
import traceback
|
|
from datetime import datetime, timedelta
|
|
|
|
import pytest
|
|
import requests
|
|
|
|
import ray
|
|
from ray._common.test_utils import wait_for_condition
|
|
from ray._private.test_utils import (
|
|
format_web_url,
|
|
wait_until_server_available,
|
|
)
|
|
from ray.cluster_utils import Cluster
|
|
from ray.dashboard.consts import RAY_DASHBOARD_STATS_UPDATING_INTERVAL
|
|
from ray.dashboard.tests.conftest import * # noqa
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
|
|
def test_nodes_update(enable_test_module, ray_start_with_dashboard):
|
|
assert wait_until_server_available(ray_start_with_dashboard["webui_url"]) is True
|
|
webui_url = ray_start_with_dashboard["webui_url"]
|
|
webui_url = format_web_url(webui_url)
|
|
|
|
timeout_seconds = 10
|
|
start_time = time.time()
|
|
while True:
|
|
time.sleep(1)
|
|
try:
|
|
response = requests.get(webui_url + "/test/dump")
|
|
response.raise_for_status()
|
|
try:
|
|
dump_info = response.json()
|
|
except Exception as ex:
|
|
logger.info("failed response: %s", response.text)
|
|
raise ex
|
|
assert dump_info["result"] is True
|
|
dump_data = dump_info["data"]
|
|
assert len(dump_data["nodes"]) == 1
|
|
break
|
|
|
|
except (AssertionError, requests.exceptions.ConnectionError):
|
|
logger.exception("Retry")
|
|
finally:
|
|
if time.time() > start_time + timeout_seconds:
|
|
raise Exception("Timed out while testing.")
|
|
|
|
|
|
def test_node_info(disable_aiohttp_cache, ray_start_with_dashboard):
|
|
@ray.remote
|
|
class Actor:
|
|
def getpid(self):
|
|
print(f"actor pid={os.getpid()}")
|
|
return os.getpid()
|
|
|
|
actors = [Actor.remote(), Actor.remote()]
|
|
actor_pids = [actor.getpid.remote() for actor in actors]
|
|
actor_pids = set(ray.get(actor_pids))
|
|
|
|
assert wait_until_server_available(ray_start_with_dashboard["webui_url"]) is True
|
|
webui_url = ray_start_with_dashboard["webui_url"]
|
|
webui_url = format_web_url(webui_url)
|
|
node_id = ray_start_with_dashboard["node_id"]
|
|
|
|
# NOTE: Leaving sum buffer time for data to get refreshed
|
|
timeout_seconds = RAY_DASHBOARD_STATS_UPDATING_INTERVAL * 1.5
|
|
|
|
start_time = time.time()
|
|
last_ex = None
|
|
while True:
|
|
time.sleep(1)
|
|
try:
|
|
response = requests.get(webui_url + "/nodes?view=hostnamelist")
|
|
response.raise_for_status()
|
|
hostname_list = response.json()
|
|
assert hostname_list["result"] is True, hostname_list["msg"]
|
|
hostname_list = hostname_list["data"]["hostNameList"]
|
|
assert len(hostname_list) == 1
|
|
|
|
hostname = hostname_list[0]
|
|
response = requests.get(webui_url + f"/nodes/{node_id}")
|
|
response.raise_for_status()
|
|
detail = response.json()
|
|
assert detail["result"] is True, detail["msg"]
|
|
detail = detail["data"]["detail"]
|
|
assert detail["hostname"] == hostname
|
|
assert detail["raylet"]["state"] == "ALIVE"
|
|
assert detail["raylet"]["isHeadNode"] is True
|
|
assert "raylet" in detail["cmdline"][0]
|
|
assert len(detail["workers"]) >= 2
|
|
assert len(detail["actors"]) == 2, detail["actors"]
|
|
|
|
actor_worker_pids = set()
|
|
for worker in detail["workers"]:
|
|
if "ray::Actor" in worker["cmdline"][0]:
|
|
actor_worker_pids.add(worker["pid"])
|
|
assert actor_worker_pids == actor_pids
|
|
|
|
response = requests.get(webui_url + "/nodes?view=summary")
|
|
response.raise_for_status()
|
|
summary = response.json()
|
|
assert summary["result"] is True, summary["msg"]
|
|
assert len(summary["data"]["summary"]) == 1
|
|
summary = summary["data"]["summary"][0]
|
|
assert summary["hostname"] == hostname
|
|
assert summary["raylet"]["state"] == "ALIVE"
|
|
assert "raylet" in summary["cmdline"][0]
|
|
assert "workers" not in summary
|
|
assert "actors" not in summary
|
|
assert "objectStoreAvailableMemory" in summary["raylet"]
|
|
assert "objectStoreUsedMemory" in summary["raylet"]
|
|
break
|
|
except Exception as ex:
|
|
last_ex = ex
|
|
finally:
|
|
if time.time() > start_time + timeout_seconds:
|
|
ex_stack = (
|
|
traceback.format_exception(
|
|
type(last_ex), last_ex, last_ex.__traceback__
|
|
)
|
|
if last_ex
|
|
else []
|
|
)
|
|
ex_stack = "".join(ex_stack)
|
|
raise Exception(f"Timed out while testing, {ex_stack}")
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
"ray_start_cluster_head_with_env_vars",
|
|
[
|
|
{
|
|
"include_dashboard": True,
|
|
"env_vars": {
|
|
"RAY_maximum_gcs_dead_node_cached_count": "1",
|
|
},
|
|
"_system_config": {
|
|
"health_check_initial_delay_ms": 0,
|
|
"health_check_timeout_ms": 100,
|
|
"health_check_failure_threshold": 3,
|
|
"health_check_period_ms": 100,
|
|
},
|
|
}
|
|
],
|
|
indirect=True,
|
|
)
|
|
def test_dead_node_cache_contains_latest_dead_node_if_cache_overflows(
|
|
enable_test_module, disable_aiohttp_cache, ray_start_cluster_head_with_env_vars
|
|
):
|
|
cluster: Cluster = ray_start_cluster_head_with_env_vars
|
|
assert wait_until_server_available(
|
|
cluster.webui_url
|
|
), "Failed to connect to the Dashboard Server"
|
|
webui_url = format_web_url(cluster.webui_url)
|
|
|
|
def _compare_dead_node_set(expected_alive_nodes, expected_dead_nodes):
|
|
try:
|
|
response = requests.get(webui_url + "/nodes?view=summary")
|
|
response.raise_for_status()
|
|
summary = response.json()
|
|
assert summary["result"] is True, summary["msg"]
|
|
summary = summary["data"]["summary"]
|
|
dead_nodes = set()
|
|
alive_nodes = set()
|
|
for node_info in summary:
|
|
node_id = node_info["raylet"]["nodeId"]
|
|
response = requests.get(webui_url + f"/nodes/{node_id}")
|
|
response.raise_for_status()
|
|
if node_info["raylet"]["state"] == "DEAD":
|
|
dead_nodes.add(node_id)
|
|
if node_info["raylet"]["state"] == "ALIVE":
|
|
alive_nodes.add(node_id)
|
|
assert alive_nodes == expected_alive_nodes
|
|
assert dead_nodes == expected_dead_nodes
|
|
return True
|
|
except Exception as ex:
|
|
logger.info(ex)
|
|
return False
|
|
|
|
node_1 = cluster.add_node()
|
|
head_node_id = ray.get_runtime_context().get_node_id()
|
|
node_1_id = node_1.node_id
|
|
curr_alive_nodes = {head_node_id, node_1_id}
|
|
curr_dead_nodes = set()
|
|
wait_for_condition(
|
|
_compare_dead_node_set,
|
|
10,
|
|
expected_alive_nodes=curr_alive_nodes,
|
|
expected_dead_nodes=curr_dead_nodes,
|
|
)
|
|
|
|
node_2 = cluster.add_node()
|
|
node_2_id = node_2.node_id
|
|
curr_alive_nodes.add(node_2_id)
|
|
wait_for_condition(
|
|
_compare_dead_node_set,
|
|
10,
|
|
expected_alive_nodes=curr_alive_nodes,
|
|
expected_dead_nodes=curr_dead_nodes,
|
|
)
|
|
|
|
cluster.remove_node(node_1, allow_graceful=False)
|
|
curr_alive_nodes.remove(node_1_id)
|
|
curr_dead_nodes.add(node_1_id)
|
|
wait_for_condition(
|
|
_compare_dead_node_set,
|
|
10,
|
|
expected_alive_nodes=curr_alive_nodes,
|
|
expected_dead_nodes=curr_dead_nodes,
|
|
)
|
|
|
|
cluster.remove_node(node_2, allow_graceful=False)
|
|
curr_alive_nodes.remove(node_2_id)
|
|
curr_dead_nodes.remove(node_1_id)
|
|
curr_dead_nodes.add(node_2_id)
|
|
wait_for_condition(
|
|
_compare_dead_node_set,
|
|
10,
|
|
expected_alive_nodes=curr_alive_nodes,
|
|
expected_dead_nodes=curr_dead_nodes,
|
|
)
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
"ray_start_cluster_head",
|
|
[
|
|
{
|
|
"include_dashboard": True,
|
|
"_system_config": {
|
|
"health_check_initial_delay_ms": 0,
|
|
"health_check_timeout_ms": 100,
|
|
"health_check_failure_threshold": 3,
|
|
"health_check_period_ms": 100,
|
|
},
|
|
}
|
|
],
|
|
indirect=True,
|
|
)
|
|
def test_multi_nodes_info(
|
|
enable_test_module, disable_aiohttp_cache, ray_start_cluster_head
|
|
):
|
|
cluster: Cluster = ray_start_cluster_head
|
|
assert wait_until_server_available(cluster.webui_url) is True
|
|
webui_url = cluster.webui_url
|
|
webui_url = format_web_url(webui_url)
|
|
cluster.add_node()
|
|
cluster.add_node()
|
|
dead_node = cluster.add_node()
|
|
cluster.remove_node(dead_node, allow_graceful=False)
|
|
|
|
def _check_nodes():
|
|
try:
|
|
response = requests.get(webui_url + "/nodes?view=summary")
|
|
response.raise_for_status()
|
|
summary = response.json()
|
|
assert summary["result"] is True, summary["msg"]
|
|
summary = summary["data"]["summary"]
|
|
assert len(summary) == 4
|
|
for node_info in summary:
|
|
node_id = node_info["raylet"]["nodeId"]
|
|
response = requests.get(webui_url + f"/nodes/{node_id}")
|
|
response.raise_for_status()
|
|
detail = response.json()
|
|
assert detail["result"] is True, detail["msg"]
|
|
detail = detail["data"]["detail"]
|
|
if node_id != dead_node.node_id:
|
|
assert detail["raylet"]["state"] == "ALIVE"
|
|
else:
|
|
assert detail["raylet"]["state"] == "DEAD"
|
|
assert detail["raylet"].get("objectStoreAvailableMemory", 0) == 0
|
|
return True
|
|
except Exception as ex:
|
|
logger.info(ex)
|
|
return False
|
|
|
|
wait_for_condition(_check_nodes, timeout=15)
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
"ray_start_cluster_head", [{"include_dashboard": True}], indirect=True
|
|
)
|
|
def test_multi_node_churn(
|
|
enable_test_module, disable_aiohttp_cache, ray_start_cluster_head
|
|
):
|
|
cluster: Cluster = ray_start_cluster_head
|
|
assert wait_until_server_available(cluster.webui_url) is True
|
|
webui_url = format_web_url(cluster.webui_url)
|
|
|
|
success = True
|
|
|
|
def verify():
|
|
nonlocal success
|
|
while True:
|
|
try:
|
|
resp = requests.get(webui_url)
|
|
resp.raise_for_status()
|
|
resp = requests.get(webui_url + "/nodes?view=summary")
|
|
resp.raise_for_status()
|
|
summary = resp.json()
|
|
assert summary["result"] is True, summary["msg"]
|
|
assert summary["data"]["summary"]
|
|
time.sleep(1)
|
|
except Exception:
|
|
success = False
|
|
break
|
|
|
|
t = threading.Thread(target=verify, daemon=True)
|
|
t.start()
|
|
|
|
t_st = datetime.now()
|
|
duration = timedelta(seconds=60)
|
|
worker_nodes = []
|
|
while datetime.now() < t_st + duration:
|
|
time.sleep(5)
|
|
if len(worker_nodes) < 2:
|
|
worker_nodes.append(cluster.add_node())
|
|
continue
|
|
should_add_node = random.randint(0, 1)
|
|
if should_add_node:
|
|
worker_nodes.append(cluster.add_node())
|
|
else:
|
|
node_index = random.randrange(0, len(worker_nodes))
|
|
node_to_remove = worker_nodes.pop(node_index)
|
|
cluster.remove_node(node_to_remove)
|
|
|
|
assert success
|
|
|
|
|
|
@pytest.mark.skipif(
|
|
sys.platform == "win32", reason="setproctitle does not change psutil.cmdline"
|
|
)
|
|
def test_node_physical_stats(enable_test_module, shutdown_only):
|
|
"""
|
|
Tests NodeHead._update_node_physical_stats.
|
|
"""
|
|
addresses = ray.init(include_dashboard=True, num_cpus=6)
|
|
|
|
@ray.remote(num_cpus=1)
|
|
class Actor:
|
|
def getpid(self):
|
|
return os.getpid()
|
|
|
|
actors = [Actor.remote() for _ in range(6)]
|
|
actor_pids = ray.get([actor.getpid.remote() for actor in actors])
|
|
actor_pids = set(actor_pids)
|
|
|
|
webui_url = addresses["webui_url"]
|
|
assert wait_until_server_available(webui_url) is True
|
|
webui_url = format_web_url(webui_url)
|
|
|
|
def _check_workers():
|
|
try:
|
|
resp = requests.get(webui_url + "/test/dump?key=node_physical_stats")
|
|
resp.raise_for_status()
|
|
result = resp.json()
|
|
assert result["result"] is True
|
|
node_physical_stats = result["data"]["nodePhysicalStats"]
|
|
assert len(node_physical_stats) == 1
|
|
current_stats = node_physical_stats[addresses["node_id"]]
|
|
# Check Actor workers
|
|
current_actor_pids = set()
|
|
for worker in current_stats["workers"]:
|
|
if "ray::Actor" in worker["cmdline"][0]:
|
|
current_actor_pids.add(worker["pid"])
|
|
assert current_actor_pids == actor_pids
|
|
# Check raylet cmdline
|
|
assert "raylet" in current_stats["cmdline"][0]
|
|
return True
|
|
except Exception as ex:
|
|
logger.info(ex)
|
|
return False
|
|
|
|
wait_for_condition(_check_workers, timeout=10)
|
|
|
|
|
|
def test_worker_pids_reported(enable_test_module, ray_start_with_dashboard):
|
|
assert wait_until_server_available(ray_start_with_dashboard["webui_url"]) is True
|
|
webui_url = ray_start_with_dashboard["webui_url"]
|
|
webui_url = format_web_url(webui_url)
|
|
node_id = ray_start_with_dashboard["node_id"]
|
|
|
|
@ray.remote(runtime_env={"uv": {"packages": ["requests==2.32.5"]}})
|
|
class UvActor:
|
|
def get_pid(self):
|
|
return os.getpid()
|
|
|
|
uv_actor = UvActor.remote()
|
|
uv_actor_pid = ray.get(uv_actor.get_pid.remote())
|
|
driver_pid = os.getpid()
|
|
|
|
def _check_worker_pids():
|
|
try:
|
|
response = requests.get(webui_url + f"/nodes/{node_id}")
|
|
response.raise_for_status()
|
|
dump_info = response.json()
|
|
assert dump_info["result"] is True
|
|
detail = dump_info["data"]["detail"]
|
|
pids = [worker["pid"] for worker in detail["workers"]]
|
|
assert len(pids) >= 2 # might include idle worker
|
|
assert uv_actor_pid in pids
|
|
assert driver_pid in pids
|
|
return True
|
|
except Exception as ex:
|
|
logger.info(ex)
|
|
return False
|
|
|
|
wait_for_condition(_check_worker_pids, timeout=20)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(pytest.main(["-v", __file__]))
|