wehub-resource-sync
74 lines
2.1 KiB
Python
74 lines
2.1 KiB
Python
# coding: utf-8
|
|
import logging
|
|
import signal
|
|
import subprocess
|
|
import sys
|
|
import time
|
|
|
|
import pytest
|
|
|
|
import ray
|
|
from ray._private import ray_constants
|
|
|
|
import psutil
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
|
|
def test_healthcheck():
|
|
res = subprocess.run(["ray", "health-check"])
|
|
assert res.returncode != 0
|
|
|
|
ray.init()
|
|
res = subprocess.run(["ray", "health-check"])
|
|
assert res.returncode == 0, res.stdout
|
|
|
|
# Kill GCS to test ray health-check.
|
|
all_processes = ray._private.worker._global_node.all_processes
|
|
assert ray_constants.PROCESS_TYPE_GCS_SERVER in all_processes
|
|
gcs_proc_info = all_processes[ray_constants.PROCESS_TYPE_GCS_SERVER][0]
|
|
gcs_proc = psutil.Process(gcs_proc_info.process.pid)
|
|
gcs_proc.kill()
|
|
gcs_proc.wait(10)
|
|
|
|
res = subprocess.run(["ray", "health-check"])
|
|
assert res.returncode != 0
|
|
|
|
ray.shutdown()
|
|
|
|
res = subprocess.run(["ray", "health-check"])
|
|
assert res.returncode != 0
|
|
|
|
|
|
@pytest.mark.skipif(sys.platform == "win32", reason="Uses unix SIGKILL")
|
|
def test_healthcheck_ray_client_server():
|
|
res = subprocess.run(["ray", "health-check", "--component", "ray_client_server"])
|
|
assert res.returncode != 0
|
|
|
|
ray.init()
|
|
res = subprocess.run(["ray", "health-check", "--component", "ray_client_server"])
|
|
assert res.returncode != 0, res.stdout
|
|
|
|
client_server_handle = subprocess.Popen(
|
|
[sys.executable, "-m", "ray.util.client.server"]
|
|
)
|
|
# Gotta give the server time to initialize.
|
|
time.sleep(5)
|
|
|
|
res = subprocess.run(["ray", "health-check", "--component", "ray_client_server"])
|
|
assert res.returncode == 0, res.stdout
|
|
|
|
client_server_handle.send_signal(signal.SIGKILL)
|
|
time.sleep(ray._private.ray_constants.HEALTHCHECK_EXPIRATION_S)
|
|
res = subprocess.run(["ray", "health-check", "--component", "ray_client_server"])
|
|
assert res.returncode != 0, res.stdout
|
|
|
|
ray.shutdown()
|
|
res = subprocess.run(["ray", "health-check", "--component", "ray_client_server"])
|
|
assert res.returncode != 0, res.stdout
|
|
|
|
|
|
if __name__ == "__main__":
|
|
|
|
sys.exit(pytest.main(["-sv", __file__]))
|