chore: import upstream snapshot with attribution
This commit is contained in:
wehub-resource-sync
@@ -0,0 +1,780 @@
|
||||
import asyncio
|
||||
import copy
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import random
|
||||
import socket
|
||||
import sys
|
||||
import tempfile
|
||||
import time
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from datetime import datetime
|
||||
from pprint import pprint
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
import requests
|
||||
|
||||
import ray
|
||||
from ray._common.test_utils import wait_for_condition
|
||||
from ray._common.utils import binary_to_hex
|
||||
from ray._private.event.event_logger import (
|
||||
EventLoggerAdapter,
|
||||
filter_event_by_level,
|
||||
get_event_id,
|
||||
get_event_logger,
|
||||
)
|
||||
from ray._private.event.export_event_logger import (
|
||||
EventLogType,
|
||||
ExportEventLoggerAdapter,
|
||||
get_export_event_logger,
|
||||
)
|
||||
from ray._private.protobuf_compat import message_to_dict
|
||||
from ray._private.state_api_test_utils import create_api_options, verify_schema
|
||||
from ray._private.test_utils import (
|
||||
format_web_url,
|
||||
wait_until_server_available,
|
||||
)
|
||||
from ray.cluster_utils import AutoscalingCluster
|
||||
from ray.core.generated import (
|
||||
event_pb2,
|
||||
export_submission_job_event_pb2,
|
||||
)
|
||||
from ray.dashboard.modules.event import event_consts
|
||||
from ray.dashboard.modules.event.event_head import _list_cluster_events_impl
|
||||
from ray.dashboard.modules.event.event_utils import monitor_events
|
||||
from ray.dashboard.tests.conftest import * # noqa
|
||||
from ray.job_submission import JobSubmissionClient
|
||||
from ray.util.state import list_cluster_events
|
||||
from ray.util.state.common import ClusterEventState
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _get_event(msg="empty message", job_id=None, source_type=None):
|
||||
return {
|
||||
"event_id": binary_to_hex(np.random.bytes(18)),
|
||||
"source_type": (
|
||||
random.choice(event_pb2.Event.SourceType.keys())
|
||||
if source_type is None
|
||||
else source_type
|
||||
),
|
||||
"host_name": "po-dev.inc.alipay.net",
|
||||
"pid": random.randint(1, 65536),
|
||||
"label": "",
|
||||
"message": msg,
|
||||
"timestamp": time.time(),
|
||||
"severity": "INFO",
|
||||
"custom_fields": {
|
||||
"job_id": (
|
||||
ray.JobID.from_int(random.randint(1, 100)).hex()
|
||||
if job_id is None
|
||||
else job_id
|
||||
),
|
||||
"node_id": "",
|
||||
"task_id": "",
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def _test_logger(name, log_file, max_bytes, backup_count):
|
||||
handler = logging.handlers.RotatingFileHandler(
|
||||
log_file, maxBytes=max_bytes, backupCount=backup_count
|
||||
)
|
||||
formatter = logging.Formatter("%(message)s")
|
||||
handler.setFormatter(formatter)
|
||||
|
||||
logger = logging.getLogger(name)
|
||||
logger.propagate = False
|
||||
logger.setLevel(logging.INFO)
|
||||
logger.addHandler(handler)
|
||||
|
||||
return logger
|
||||
|
||||
|
||||
def test_python_global_event_logger(tmp_path):
|
||||
logger = get_event_logger(event_pb2.Event.SourceType.GCS, str(tmp_path))
|
||||
logger.set_global_context({"test_meta": "1"})
|
||||
logger.info("message", a="a", b="b")
|
||||
logger.error("message", a="a", b="b")
|
||||
logger.warning("message", a="a", b="b")
|
||||
logger.fatal("message", a="a", b="b")
|
||||
event_dir = tmp_path / "events"
|
||||
assert event_dir.exists()
|
||||
event_file = event_dir / "event_GCS.log"
|
||||
assert event_file.exists()
|
||||
|
||||
line_severities = ["INFO", "ERROR", "WARNING", "FATAL"]
|
||||
|
||||
with event_file.open() as f:
|
||||
for line, severity in zip(f.readlines(), line_severities):
|
||||
data = json.loads(line)
|
||||
assert data["severity"] == severity
|
||||
assert data["label"] == ""
|
||||
assert "timestamp" in data
|
||||
assert len(data["event_id"]) == 36
|
||||
assert data["message"] == "message"
|
||||
assert data["source_type"] == "GCS"
|
||||
assert data["source_hostname"] == socket.gethostname()
|
||||
assert data["source_pid"] == os.getpid()
|
||||
assert data["custom_fields"]["a"] == "a"
|
||||
assert data["custom_fields"]["b"] == "b"
|
||||
|
||||
|
||||
def test_event_basic(disable_aiohttp_cache, ray_start_with_dashboard):
|
||||
assert wait_until_server_available(ray_start_with_dashboard["webui_url"])
|
||||
webui_url = format_web_url(ray_start_with_dashboard["webui_url"])
|
||||
session_dir = ray_start_with_dashboard["session_dir"]
|
||||
event_dir = os.path.join(session_dir, "logs", "events")
|
||||
job_id = ray.JobID.from_int(100).hex()
|
||||
|
||||
source_type_gcs = event_pb2.Event.SourceType.Name(event_pb2.Event.GCS)
|
||||
source_type_raylet = event_pb2.Event.SourceType.Name(event_pb2.Event.RAYLET)
|
||||
test_count = 20
|
||||
|
||||
for source_type in [source_type_gcs, source_type_raylet]:
|
||||
test_log_file = os.path.join(event_dir, f"event_{source_type}.log")
|
||||
test_logger = _test_logger(
|
||||
__name__ + str(random.random()),
|
||||
test_log_file,
|
||||
max_bytes=2000,
|
||||
backup_count=0,
|
||||
)
|
||||
for i in range(test_count):
|
||||
sample_event = _get_event(str(i), job_id=job_id, source_type=source_type)
|
||||
test_logger.info("%s", json.dumps(sample_event))
|
||||
|
||||
def _check_events():
|
||||
try:
|
||||
resp = requests.get(f"{webui_url}/events")
|
||||
resp.raise_for_status()
|
||||
result = resp.json()
|
||||
all_events = result["data"]["events"]
|
||||
job_events = all_events[job_id]
|
||||
assert len(job_events) >= test_count * 2
|
||||
source_messages = {}
|
||||
for e in job_events:
|
||||
source_type = e["sourceType"]
|
||||
message = e["message"]
|
||||
source_messages.setdefault(source_type, set()).add(message)
|
||||
assert len(source_messages[source_type_gcs]) >= test_count
|
||||
assert len(source_messages[source_type_raylet]) >= test_count
|
||||
data = {str(i) for i in range(test_count)}
|
||||
assert data & source_messages[source_type_gcs] == data
|
||||
assert data & source_messages[source_type_raylet] == data
|
||||
return True
|
||||
except Exception as ex:
|
||||
logger.exception(ex)
|
||||
return False
|
||||
|
||||
wait_for_condition(_check_events, timeout=15)
|
||||
|
||||
|
||||
def test_event_message_limit(
|
||||
small_event_line_limit, disable_aiohttp_cache, ray_start_with_dashboard
|
||||
):
|
||||
event_read_line_length_limit = small_event_line_limit
|
||||
assert wait_until_server_available(ray_start_with_dashboard["webui_url"])
|
||||
webui_url = format_web_url(ray_start_with_dashboard["webui_url"])
|
||||
session_dir = ray_start_with_dashboard["session_dir"]
|
||||
event_dir = os.path.join(session_dir, "logs", "events")
|
||||
job_id = ray.JobID.from_int(100).hex()
|
||||
events = []
|
||||
# Sample event equals with limit.
|
||||
sample_event = _get_event("", job_id=job_id)
|
||||
message_len = event_read_line_length_limit - len(json.dumps(sample_event))
|
||||
for i in range(10):
|
||||
sample_event = copy.deepcopy(sample_event)
|
||||
sample_event["event_id"] = binary_to_hex(np.random.bytes(18))
|
||||
sample_event["message"] = str(i) * message_len
|
||||
assert len(json.dumps(sample_event)) == event_read_line_length_limit
|
||||
events.append(sample_event)
|
||||
# Sample event longer than limit.
|
||||
sample_event = copy.deepcopy(sample_event)
|
||||
sample_event["event_id"] = binary_to_hex(np.random.bytes(18))
|
||||
sample_event["message"] = "2" * (message_len + 1)
|
||||
assert len(json.dumps(sample_event)) > event_read_line_length_limit
|
||||
events.append(sample_event)
|
||||
|
||||
for i in range(event_consts.EVENT_READ_LINE_COUNT_LIMIT):
|
||||
events.append(_get_event(str(i), job_id=job_id))
|
||||
|
||||
with open(os.path.join(event_dir, "tmp.log"), "w") as f:
|
||||
f.writelines([(json.dumps(e) + "\n") for e in events])
|
||||
|
||||
try:
|
||||
os.remove(os.path.join(event_dir, "event_GCS.log"))
|
||||
except Exception:
|
||||
pass
|
||||
os.rename(
|
||||
os.path.join(event_dir, "tmp.log"), os.path.join(event_dir, "event_GCS.log")
|
||||
)
|
||||
|
||||
def _check_events():
|
||||
try:
|
||||
resp = requests.get(f"{webui_url}/events")
|
||||
resp.raise_for_status()
|
||||
result = resp.json()
|
||||
all_events = result["data"]["events"]
|
||||
assert (
|
||||
len(all_events[job_id]) >= event_consts.EVENT_READ_LINE_COUNT_LIMIT + 10
|
||||
)
|
||||
messages = [e["message"] for e in all_events[job_id]]
|
||||
for i in range(10):
|
||||
assert str(i) * message_len in messages
|
||||
assert "2" * (message_len + 1) not in messages
|
||||
assert str(event_consts.EVENT_READ_LINE_COUNT_LIMIT - 1) in messages
|
||||
return True
|
||||
except Exception as ex:
|
||||
logger.exception(ex)
|
||||
return False
|
||||
|
||||
wait_for_condition(_check_events, timeout=15)
|
||||
|
||||
|
||||
def test_report_events(ray_start_with_dashboard):
|
||||
assert wait_until_server_available(ray_start_with_dashboard["webui_url"])
|
||||
webui_url = format_web_url(ray_start_with_dashboard["webui_url"])
|
||||
url = f"{webui_url}/report_events"
|
||||
|
||||
resp = requests.post(url)
|
||||
assert resp.status_code == 400
|
||||
resp = requests.post(url, json={"Hello": "World"})
|
||||
assert resp.status_code == 400
|
||||
|
||||
job_id = ray.JobID.from_int(100).hex()
|
||||
sample_event = _get_event("Hello", job_id=job_id)
|
||||
resp = requests.post(url, json=[json.dumps(sample_event)])
|
||||
assert resp.status_code == 200
|
||||
|
||||
resp = requests.get(f"{webui_url}/events")
|
||||
assert resp.status_code == 200
|
||||
result = resp.json()
|
||||
all_events = result["data"]["events"]
|
||||
assert len(all_events) == 1
|
||||
assert job_id in all_events
|
||||
assert len(all_events[job_id]) == 1
|
||||
assert all_events[job_id][0]["message"] == "Hello"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_monitor_events():
|
||||
with tempfile.TemporaryDirectory() as temp_dir:
|
||||
common = event_pb2.Event.SourceType.Name(event_pb2.Event.COMMON)
|
||||
common_log = os.path.join(temp_dir, f"event_{common}.log")
|
||||
test_logger = _test_logger(
|
||||
__name__ + str(random.random()), common_log, max_bytes=10, backup_count=0
|
||||
)
|
||||
test_events1 = []
|
||||
monitor_task = monitor_events(
|
||||
temp_dir, lambda x: test_events1.extend(x), None, scan_interval_seconds=0.01
|
||||
)
|
||||
assert not monitor_task.done()
|
||||
count = 10
|
||||
|
||||
async def _writer(*args, read_events, spin=True):
|
||||
for x in range(*args):
|
||||
test_logger.info("%s", x)
|
||||
if spin:
|
||||
while str(x) not in read_events:
|
||||
await asyncio.sleep(0.01)
|
||||
|
||||
async def _check_events(expect_events, read_events, timeout=10):
|
||||
start_time = time.time()
|
||||
while True:
|
||||
sorted_events = sorted(int(i) for i in read_events)
|
||||
sorted_events = [str(i) for i in sorted_events]
|
||||
if time.time() - start_time > timeout:
|
||||
raise TimeoutError(
|
||||
f"Timeout, read events: {sorted_events}, "
|
||||
f"expect events: {expect_events}"
|
||||
)
|
||||
if len(sorted_events) == len(expect_events):
|
||||
if sorted_events == expect_events:
|
||||
break
|
||||
await asyncio.sleep(1)
|
||||
|
||||
await asyncio.gather(
|
||||
_writer(count, read_events=test_events1),
|
||||
_check_events([str(i) for i in range(count)], read_events=test_events1),
|
||||
)
|
||||
|
||||
monitor_task.cancel()
|
||||
test_events2 = []
|
||||
monitor_task = monitor_events(
|
||||
temp_dir, lambda x: test_events2.extend(x), None, scan_interval_seconds=0.1
|
||||
)
|
||||
|
||||
await _check_events([str(i) for i in range(count)], read_events=test_events2)
|
||||
|
||||
await _writer(count, count * 2, read_events=test_events2)
|
||||
await _check_events(
|
||||
[str(i) for i in range(count * 2)], read_events=test_events2
|
||||
)
|
||||
|
||||
log_file_count = len(os.listdir(temp_dir))
|
||||
|
||||
test_logger = _test_logger(
|
||||
__name__ + str(random.random()), common_log, max_bytes=1000, backup_count=0
|
||||
)
|
||||
assert len(os.listdir(temp_dir)) == log_file_count
|
||||
|
||||
await _writer(count * 2, count * 3, spin=False, read_events=test_events2)
|
||||
await _check_events(
|
||||
[str(i) for i in range(count * 3)], read_events=test_events2
|
||||
)
|
||||
await _writer(count * 3, count * 4, spin=False, read_events=test_events2)
|
||||
await _check_events(
|
||||
[str(i) for i in range(count * 4)], read_events=test_events2
|
||||
)
|
||||
|
||||
# Test cancel monitor task.
|
||||
monitor_task.cancel()
|
||||
with pytest.raises(asyncio.CancelledError):
|
||||
await monitor_task
|
||||
assert monitor_task.done()
|
||||
|
||||
assert len(os.listdir(temp_dir)) == 1, "There should just be 1 event log"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("autoscaler_v2", [False, True], ids=["v1", "v2"])
|
||||
def test_autoscaler_cluster_events(autoscaler_v2, shutdown_only):
|
||||
cluster = AutoscalingCluster(
|
||||
head_resources={"CPU": 2},
|
||||
worker_node_types={
|
||||
"cpu_node": {
|
||||
"resources": {
|
||||
"CPU": 4,
|
||||
},
|
||||
"node_config": {},
|
||||
"min_workers": 0,
|
||||
"max_workers": 1,
|
||||
},
|
||||
"gpu_node": {
|
||||
"resources": {
|
||||
"CPU": 2,
|
||||
"GPU": 1,
|
||||
},
|
||||
"node_config": {},
|
||||
"min_workers": 0,
|
||||
"max_workers": 1,
|
||||
},
|
||||
},
|
||||
autoscaler_v2=autoscaler_v2,
|
||||
idle_timeout_minutes=1,
|
||||
)
|
||||
|
||||
try:
|
||||
cluster.start()
|
||||
ray.init("auto")
|
||||
|
||||
# Triggers the addition of a GPU node.
|
||||
@ray.remote(num_gpus=1)
|
||||
def f():
|
||||
print("gpu ok")
|
||||
|
||||
# Triggers the addition of a CPU node.
|
||||
@ray.remote(num_cpus=3)
|
||||
def g():
|
||||
print("cpu ok")
|
||||
|
||||
wait_for_condition(lambda: ray.cluster_resources()["CPU"] == 2)
|
||||
ray.get(f.remote())
|
||||
wait_for_condition(lambda: ray.cluster_resources()["CPU"] == 4)
|
||||
wait_for_condition(lambda: ray.cluster_resources()["GPU"] == 1)
|
||||
ray.get(g.remote())
|
||||
wait_for_condition(lambda: ray.cluster_resources()["CPU"] == 8)
|
||||
wait_for_condition(lambda: ray.cluster_resources()["GPU"] == 1)
|
||||
|
||||
# Trigger an infeasible task
|
||||
g.options(num_cpus=0, num_gpus=5).remote()
|
||||
|
||||
def verify():
|
||||
cluster_events = list_cluster_events()
|
||||
print(cluster_events)
|
||||
messages = {(e["message"], e["source_type"]) for e in cluster_events}
|
||||
if not autoscaler_v2:
|
||||
# With head node resources, we don't actually resized. So this event is
|
||||
# not really accurate.
|
||||
assert ("Resized to 2 CPUs.", "AUTOSCALER") in messages, cluster_events
|
||||
assert (
|
||||
"Adding 1 node(s) of type gpu_node.",
|
||||
"AUTOSCALER",
|
||||
) in messages, cluster_events
|
||||
assert (
|
||||
"Resized to 4 CPUs, 1 GPUs.",
|
||||
"AUTOSCALER",
|
||||
) in messages, cluster_events
|
||||
assert (
|
||||
"Adding 1 node(s) of type cpu_node.",
|
||||
"AUTOSCALER",
|
||||
) in messages, cluster_events
|
||||
assert (
|
||||
"Resized to 8 CPUs, 1 GPUs.",
|
||||
"AUTOSCALER",
|
||||
) in messages, cluster_events
|
||||
assert "No available node types can fulfill resource request" in "".join(
|
||||
[t[0] for t in messages]
|
||||
)
|
||||
|
||||
return True
|
||||
|
||||
wait_for_condition(verify, timeout=30)
|
||||
pprint(list_cluster_events())
|
||||
finally:
|
||||
ray.shutdown()
|
||||
cluster.shutdown()
|
||||
|
||||
|
||||
def test_filter_event_by_level(monkeypatch):
|
||||
def gen_event(level: str):
|
||||
return event_pb2.Event(
|
||||
source_type=event_pb2.Event.AUTOSCALER,
|
||||
severity=event_pb2.Event.Severity.Value(level),
|
||||
message=level,
|
||||
)
|
||||
|
||||
trace = gen_event("TRACE")
|
||||
debug = gen_event("DEBUG")
|
||||
info = gen_event("INFO")
|
||||
warning = gen_event("WARNING")
|
||||
error = gen_event("ERROR")
|
||||
fatal = gen_event("FATAL")
|
||||
|
||||
def assert_events_filtered(events, expected, filter_level):
|
||||
filtered = [e for e in events if filter_event_by_level(e, filter_level)]
|
||||
print(filtered)
|
||||
assert len(filtered) == len(expected)
|
||||
assert {e.message for e in filtered} == {e.message for e in expected}
|
||||
|
||||
events = [trace, debug, info, warning, error, fatal]
|
||||
assert_events_filtered(events, [], "TRACE")
|
||||
assert_events_filtered(events, [trace], "DEBUG")
|
||||
assert_events_filtered(events, [trace, debug], "INFO")
|
||||
assert_events_filtered(events, [trace, debug, info], "WARNING")
|
||||
assert_events_filtered(events, [trace, debug, info, warning], "ERROR")
|
||||
assert_events_filtered(events, [trace, debug, info, warning, error], "FATAL")
|
||||
|
||||
|
||||
def test_jobs_cluster_events(shutdown_only):
|
||||
ray.init()
|
||||
address = ray._private.worker._global_node.webui_url
|
||||
address = format_web_url(address)
|
||||
client = JobSubmissionClient(address)
|
||||
submission_id = client.submit_job(entrypoint="ls")
|
||||
|
||||
def verify():
|
||||
events = list_cluster_events()
|
||||
assert len(list_cluster_events()) == 2
|
||||
start_event = events[0]
|
||||
completed_event = events[1]
|
||||
|
||||
assert start_event["source_type"] == "JOBS"
|
||||
assert f"Started a ray job {submission_id}" in start_event["message"]
|
||||
assert start_event["severity"] == "INFO"
|
||||
assert completed_event["source_type"] == "JOBS"
|
||||
assert (
|
||||
f"Completed a ray job {submission_id} with a status SUCCEEDED."
|
||||
== completed_event["message"]
|
||||
)
|
||||
assert completed_event["severity"] == "INFO"
|
||||
return True
|
||||
|
||||
print("Test successful job run.")
|
||||
wait_for_condition(verify)
|
||||
pprint(list_cluster_events())
|
||||
|
||||
# Test the failure case. In this part, job fails because the runtime env
|
||||
# creation fails.
|
||||
submission_id = client.submit_job(
|
||||
entrypoint="ls",
|
||||
runtime_env={"pip": ["nonexistent_dep"]},
|
||||
)
|
||||
|
||||
def verify():
|
||||
events = list_cluster_events(detail=True)
|
||||
failed_events = []
|
||||
|
||||
for e in events:
|
||||
if (
|
||||
"submission_id" in e["custom_fields"]
|
||||
and e["custom_fields"]["submission_id"] == submission_id
|
||||
):
|
||||
failed_events.append(e)
|
||||
|
||||
assert len(failed_events) == 2
|
||||
failed_start = failed_events[0]
|
||||
failed_completed = failed_events[1]
|
||||
|
||||
assert failed_start["source_type"] == "JOBS"
|
||||
assert f"Started a ray job {submission_id}" in failed_start["message"]
|
||||
assert failed_completed["source_type"] == "JOBS"
|
||||
assert failed_completed["severity"] == "ERROR"
|
||||
assert (
|
||||
f"Completed a ray job {submission_id} with a status FAILED."
|
||||
in failed_completed["message"]
|
||||
)
|
||||
|
||||
# Make sure the error message is included.
|
||||
assert "ERROR: No matching distribution found" in failed_completed["message"]
|
||||
return True
|
||||
|
||||
print("Test failed (runtime_env failure) job run.")
|
||||
wait_for_condition(verify, timeout=30)
|
||||
pprint(list_cluster_events())
|
||||
|
||||
|
||||
def test_core_events(shutdown_only):
|
||||
# Test events recorded from core RAY_EVENT APIs.
|
||||
ray.init()
|
||||
|
||||
@ray.remote
|
||||
class Actor:
|
||||
def getpid(self):
|
||||
return os.getpid()
|
||||
|
||||
a = Actor.remote()
|
||||
pid = ray.get(a.getpid.remote())
|
||||
os.kill(pid, 9)
|
||||
s = time.time()
|
||||
|
||||
def verify():
|
||||
events = list_cluster_events(filters=[("source_type", "=", "RAYLET")])
|
||||
print(events)
|
||||
assert len(list_cluster_events()) == 1
|
||||
event = events[0]
|
||||
assert event["severity"] == "ERROR"
|
||||
datetime_str = event["time"]
|
||||
datetime_obj = datetime.strptime(datetime_str, "%Y-%m-%d %H:%M:%S")
|
||||
timestamp = time.mktime(datetime_obj.timetuple())
|
||||
|
||||
# Make sure timestamp is not incorrect. Add sufficient buffer (60 seconds)
|
||||
assert abs(timestamp - s) < 60
|
||||
assert (
|
||||
"A worker died or was killed while executing "
|
||||
"a task by an unexpected system error" in event["message"]
|
||||
)
|
||||
return True
|
||||
|
||||
wait_for_condition(verify)
|
||||
pprint(list_cluster_events())
|
||||
|
||||
|
||||
def test_cluster_events_retention(monkeypatch, shutdown_only):
|
||||
with monkeypatch.context() as m:
|
||||
# defer for 5s for the second node.
|
||||
# This will help the API not return until the node is killed.
|
||||
m.setenv("RAY_DASHBOARD_MAX_EVENTS_TO_CACHE", "10")
|
||||
ray.init()
|
||||
address = ray._private.worker._global_node.webui_url
|
||||
address = format_web_url(address)
|
||||
client = JobSubmissionClient(address)
|
||||
|
||||
submission_ids = []
|
||||
for _ in range(12):
|
||||
submission_ids.append(client.submit_job(entrypoint="ls"))
|
||||
print(submission_ids)
|
||||
|
||||
def verify():
|
||||
events = list_cluster_events()
|
||||
assert len(list_cluster_events()) == 10
|
||||
|
||||
messages = [event["message"] for event in events]
|
||||
|
||||
# Make sure the first two has been GC'ed.
|
||||
for m in messages:
|
||||
assert submission_ids[0] not in m
|
||||
assert submission_ids[1] not in m
|
||||
return True
|
||||
|
||||
wait_for_condition(verify)
|
||||
pprint(list_cluster_events())
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_list_cluster_events_impl():
|
||||
executor = ThreadPoolExecutor(
|
||||
max_workers=1,
|
||||
thread_name_prefix="event_head_executor",
|
||||
)
|
||||
|
||||
event_id_1 = get_event_id()
|
||||
event_id_2 = get_event_id()
|
||||
events = {
|
||||
"job_1": {
|
||||
event_id_1: {
|
||||
"timestamp": 10,
|
||||
"severity": "DEBUG",
|
||||
"message": "a",
|
||||
"event_id": event_id_1,
|
||||
"source_type": "GCS",
|
||||
},
|
||||
event_id_2: {
|
||||
"timestamp": 10,
|
||||
"severity": "INFO",
|
||||
"message": "b",
|
||||
"event_id": event_id_2,
|
||||
"source_type": "GCS",
|
||||
},
|
||||
}
|
||||
}
|
||||
result = await _list_cluster_events_impl(
|
||||
all_events=events, executor=executor, option=create_api_options()
|
||||
)
|
||||
data = result.result
|
||||
data = data[0]
|
||||
verify_schema(ClusterEventState, data)
|
||||
assert result.total == 2
|
||||
|
||||
"""
|
||||
Test detail
|
||||
"""
|
||||
# TODO(sang)
|
||||
|
||||
"""
|
||||
Test limit
|
||||
"""
|
||||
assert len(result.result) == 2
|
||||
result = await _list_cluster_events_impl(
|
||||
all_events=events, executor=executor, option=create_api_options(limit=1)
|
||||
)
|
||||
data = result.result
|
||||
assert len(data) == 1
|
||||
assert result.total == 2
|
||||
|
||||
"""
|
||||
Test filters
|
||||
"""
|
||||
# If the column is not supported for filtering, it should raise an exception.
|
||||
with pytest.raises(ValueError):
|
||||
result = await _list_cluster_events_impl(
|
||||
all_events=events,
|
||||
executor=executor,
|
||||
option=create_api_options(filters=[("time", "=", "20")]),
|
||||
)
|
||||
result = await _list_cluster_events_impl(
|
||||
all_events=events,
|
||||
executor=executor,
|
||||
option=create_api_options(filters=[("severity", "=", "INFO")]),
|
||||
)
|
||||
assert len(result.result) == 1
|
||||
|
||||
|
||||
def test_export_event_logger(tmp_path):
|
||||
"""
|
||||
Unit test a mock export event of type ExportSubmissionJobEventData
|
||||
is correctly written to file. This doesn't events are correctly generated.
|
||||
"""
|
||||
logger = get_export_event_logger(EventLogType.SUBMISSION_JOB, str(tmp_path))
|
||||
ExportSubmissionJobEventData = (
|
||||
export_submission_job_event_pb2.ExportSubmissionJobEventData
|
||||
)
|
||||
event_data = ExportSubmissionJobEventData(
|
||||
submission_job_id="submission_job_id0",
|
||||
status=ExportSubmissionJobEventData.JobStatus.RUNNING,
|
||||
entrypoint="ls",
|
||||
metadata={},
|
||||
)
|
||||
logger.send_event(event_data)
|
||||
|
||||
event_dir = tmp_path / "export_events"
|
||||
assert event_dir.exists()
|
||||
event_file = event_dir / "event_EXPORT_SUBMISSION_JOB.log"
|
||||
assert event_file.exists()
|
||||
|
||||
with event_file.open() as f:
|
||||
lines = f.readlines()
|
||||
assert len(lines) == 1
|
||||
|
||||
line = lines[0]
|
||||
data = json.loads(line)
|
||||
assert data["source_type"] == "EXPORT_SUBMISSION_JOB"
|
||||
assert data["event_data"] == message_to_dict(
|
||||
event_data,
|
||||
always_print_fields_with_no_presence=True,
|
||||
preserving_proto_field_name=True,
|
||||
use_integers_for_enums=False,
|
||||
)
|
||||
|
||||
|
||||
def test_event_logger_flushes_all_handlers():
|
||||
mock_logger = MagicMock()
|
||||
handlers = [MagicMock() for _ in range(3)]
|
||||
mock_logger.handlers = handlers
|
||||
|
||||
adapter = EventLoggerAdapter(event_pb2.Event.GCS, mock_logger)
|
||||
adapter.info("message")
|
||||
|
||||
for handler in handlers:
|
||||
handler.flush.assert_called_once()
|
||||
|
||||
|
||||
def test_event_logger_allows_empty_handlers():
|
||||
mock_logger = MagicMock()
|
||||
mock_logger.handlers = []
|
||||
|
||||
adapter = EventLoggerAdapter(event_pb2.Event.GCS, mock_logger)
|
||||
adapter.info("message")
|
||||
|
||||
|
||||
def test_export_event_logger_flushes_all_handlers():
|
||||
mock_logger = MagicMock()
|
||||
handlers = [MagicMock() for _ in range(3)]
|
||||
mock_logger.handlers = handlers
|
||||
|
||||
adapter = ExportEventLoggerAdapter(EventLogType.SUBMISSION_JOB, mock_logger)
|
||||
event_data = export_submission_job_event_pb2.ExportSubmissionJobEventData(
|
||||
submission_job_id="submission_job_id0",
|
||||
status=(
|
||||
export_submission_job_event_pb2.ExportSubmissionJobEventData.JobStatus.RUNNING
|
||||
),
|
||||
entrypoint="ls",
|
||||
metadata={},
|
||||
)
|
||||
adapter.send_event(event_data)
|
||||
|
||||
for handler in handlers:
|
||||
handler.flush.assert_called_once()
|
||||
|
||||
|
||||
def test_export_event_logger_allows_empty_handlers():
|
||||
mock_logger = MagicMock()
|
||||
mock_logger.handlers = []
|
||||
|
||||
adapter = ExportEventLoggerAdapter(EventLogType.SUBMISSION_JOB, mock_logger)
|
||||
event_data = export_submission_job_event_pb2.ExportSubmissionJobEventData(
|
||||
submission_job_id="submission_job_id0",
|
||||
status=(
|
||||
export_submission_job_event_pb2.ExportSubmissionJobEventData.JobStatus.RUNNING
|
||||
),
|
||||
entrypoint="ls",
|
||||
metadata={},
|
||||
)
|
||||
adapter.send_event(event_data)
|
||||
|
||||
|
||||
def test_export_event_logger_continues_flushing_after_handler_error():
|
||||
mock_logger = MagicMock()
|
||||
handler1 = MagicMock()
|
||||
handler1.flush.side_effect = RuntimeError("flush failed")
|
||||
handler2 = MagicMock()
|
||||
mock_logger.handlers = [handler1, handler2]
|
||||
|
||||
adapter = ExportEventLoggerAdapter(EventLogType.SUBMISSION_JOB, mock_logger)
|
||||
event_data = export_submission_job_event_pb2.ExportSubmissionJobEventData(
|
||||
submission_job_id="submission_job_id0",
|
||||
status=(
|
||||
export_submission_job_event_pb2.ExportSubmissionJobEventData.JobStatus.RUNNING
|
||||
),
|
||||
entrypoint="ls",
|
||||
metadata={},
|
||||
)
|
||||
adapter.send_event(event_data)
|
||||
|
||||
handler2.flush.assert_called_once()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(pytest.main(["-v", __file__]))
|
||||
@@ -0,0 +1,66 @@
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
|
||||
import pytest
|
||||
|
||||
import ray
|
||||
from ray._common.test_utils import wait_for_condition
|
||||
from ray._private.test_utils import wait_until_server_available
|
||||
from ray.dashboard.tests.conftest import * # noqa
|
||||
|
||||
os.environ["RAY_enable_export_api_write"] = "1"
|
||||
os.environ["RAY_enable_core_worker_ray_event_to_aggregator"] = "0"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_task_labels(disable_aiohttp_cache, ray_start_with_dashboard):
|
||||
"""
|
||||
Test task events are correctly generated and written to file
|
||||
"""
|
||||
assert wait_until_server_available(ray_start_with_dashboard["webui_url"])
|
||||
export_event_path = os.path.join(
|
||||
ray_start_with_dashboard["session_dir"], "logs", "export_events"
|
||||
)
|
||||
|
||||
# A simple task to trigger the export event
|
||||
@ray.remote
|
||||
def hi_w00t_task():
|
||||
return 1
|
||||
|
||||
ray.get(hi_w00t_task.options(_labels={"hi": "w00t"}).remote())
|
||||
|
||||
def _verify():
|
||||
# Verify export events are written
|
||||
events = []
|
||||
for filename in os.listdir(export_event_path):
|
||||
if not filename.startswith("event_EXPORT_TASK"):
|
||||
continue
|
||||
with open(f"{export_event_path}/{filename}", "r") as f:
|
||||
for line in f.readlines():
|
||||
events.append(json.loads(line))
|
||||
|
||||
hi_w00t_event = next(
|
||||
(
|
||||
event
|
||||
for event in events
|
||||
if event["source_type"] == "EXPORT_TASK"
|
||||
and event["event_data"].get("task_info", {}).get("func_or_class_name")
|
||||
== "hi_w00t_task"
|
||||
),
|
||||
None,
|
||||
)
|
||||
return (
|
||||
hi_w00t_event is not None
|
||||
and hi_w00t_event["event_data"]
|
||||
.get("task_info", {})
|
||||
.get("labels", {})
|
||||
.get("hi")
|
||||
== "w00t"
|
||||
)
|
||||
|
||||
wait_for_condition(_verify, timeout=30)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(pytest.main(["-v", __file__]))
|
||||
@@ -0,0 +1,197 @@
|
||||
# isort: skip_file
|
||||
# ruff: noqa: E402
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
|
||||
import pytest
|
||||
|
||||
# RAY_enable_export_api_write_config env var must be set before importing
|
||||
# `ray` so the correct value is set for RAY_ENABLE_EXPORT_API_WRITE_CONFIG
|
||||
# even outside a Ray driver.
|
||||
os.environ["RAY_enable_export_api_write_config"] = "EXPORT_SUBMISSION_JOB"
|
||||
|
||||
import ray
|
||||
from ray._common.test_utils import async_wait_for_condition
|
||||
from ray.dashboard.modules.job.job_manager import JobManager
|
||||
from ray.job_submission import JobStatus
|
||||
from ray.tests.conftest import call_ray_start # noqa: F401
|
||||
|
||||
|
||||
async def check_job_succeeded(job_manager, job_id):
|
||||
data = await job_manager.get_job_info(job_id)
|
||||
status = data.status
|
||||
if status == JobStatus.FAILED:
|
||||
raise RuntimeError(f"Job failed! {data.message}")
|
||||
assert status in {JobStatus.PENDING, JobStatus.RUNNING, JobStatus.SUCCEEDED}
|
||||
if status == JobStatus.SUCCEEDED:
|
||||
assert data.driver_exit_code == 0
|
||||
else:
|
||||
assert data.driver_exit_code is None
|
||||
return status == JobStatus.SUCCEEDED
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
"call_ray_start",
|
||||
[
|
||||
{
|
||||
"env": {
|
||||
"RAY_enable_export_api_write_config": "EXPORT_SUBMISSION_JOB,EXPORT_TASK",
|
||||
},
|
||||
"cmd": "ray start --head",
|
||||
}
|
||||
],
|
||||
indirect=True,
|
||||
)
|
||||
async def test_check_export_api_enabled(call_ray_start, tmp_path): # noqa: F811
|
||||
"""
|
||||
Test check_export_api_enabled is True for EXPORT_SUBMISSION_JOB and EXPORT_TASK but
|
||||
not for EXPORT_ACTOR because of the value of RAY_enable_export_api_write_config.
|
||||
"""
|
||||
|
||||
@ray.remote
|
||||
def test_check_export_api_enabled_remote():
|
||||
from ray._private.event.export_event_logger import check_export_api_enabled
|
||||
from ray.core.generated.export_event_pb2 import ExportEvent
|
||||
|
||||
success = True
|
||||
success = success and check_export_api_enabled(
|
||||
ExportEvent.SourceType.EXPORT_SUBMISSION_JOB
|
||||
)
|
||||
success = success and check_export_api_enabled(
|
||||
ExportEvent.SourceType.EXPORT_TASK
|
||||
)
|
||||
success = success and (
|
||||
not check_export_api_enabled(ExportEvent.SourceType.EXPORT_ACTOR)
|
||||
)
|
||||
return success
|
||||
|
||||
assert ray.get(test_check_export_api_enabled_remote.remote())
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
"call_ray_start",
|
||||
[
|
||||
{
|
||||
"env": {
|
||||
"RAY_enable_export_api_write": "true",
|
||||
},
|
||||
"cmd": "ray start --head",
|
||||
}
|
||||
],
|
||||
indirect=True,
|
||||
)
|
||||
async def test_check_export_api_enabled_global(call_ray_start, tmp_path): # noqa: F811
|
||||
"""
|
||||
Test check_export_api_enabled always returns True because RAY_enable_export_api_write
|
||||
is set to True.
|
||||
"""
|
||||
|
||||
@ray.remote
|
||||
def test_check_export_api_enabled_remote():
|
||||
from ray._private.event.export_event_logger import check_export_api_enabled
|
||||
from ray.core.generated.export_event_pb2 import ExportEvent
|
||||
|
||||
success = True
|
||||
success = success and check_export_api_enabled(
|
||||
ExportEvent.SourceType.EXPORT_SUBMISSION_JOB
|
||||
)
|
||||
success = success and check_export_api_enabled(
|
||||
ExportEvent.SourceType.EXPORT_ACTOR
|
||||
)
|
||||
return success
|
||||
|
||||
assert ray.get(test_check_export_api_enabled_remote.remote())
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
"call_ray_start",
|
||||
[
|
||||
{
|
||||
"env": {
|
||||
"RAY_enable_export_api_write_config": "invalid source type",
|
||||
},
|
||||
"cmd": "ray start --head",
|
||||
}
|
||||
],
|
||||
indirect=True,
|
||||
)
|
||||
async def test_check_export_api_empty_config(call_ray_start, tmp_path): # noqa: F811
|
||||
"""
|
||||
Test check_export_api_enabled is False for all sources because
|
||||
RAY_enable_export_api_write_config is not a vaild source type.
|
||||
"""
|
||||
|
||||
@ray.remote
|
||||
def test_check_export_api_enabled_remote():
|
||||
from ray._private.event.export_event_logger import check_export_api_enabled
|
||||
from ray.core.generated.export_event_pb2 import ExportEvent
|
||||
|
||||
success = True
|
||||
success = success and not (
|
||||
check_export_api_enabled(ExportEvent.SourceType.EXPORT_SUBMISSION_JOB)
|
||||
)
|
||||
success = success and (
|
||||
not check_export_api_enabled(ExportEvent.SourceType.EXPORT_ACTOR)
|
||||
)
|
||||
return success
|
||||
|
||||
assert ray.get(test_check_export_api_enabled_remote.remote())
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
"call_ray_start",
|
||||
[
|
||||
{
|
||||
"env": {
|
||||
"RAY_enable_export_api_write_config": "EXPORT_SUBMISSION_JOB",
|
||||
},
|
||||
"cmd": "ray start --head",
|
||||
}
|
||||
],
|
||||
indirect=True,
|
||||
)
|
||||
async def test_submission_job_export_events(call_ray_start, tmp_path): # noqa: F811
|
||||
"""
|
||||
Test submission job events are correctly generated and written to file
|
||||
as the job goes through various state changes in its lifecycle.
|
||||
"""
|
||||
|
||||
ray.init(address=call_ray_start)
|
||||
gcs_client = ray._private.worker.global_worker.gcs_client
|
||||
job_manager = JobManager(gcs_client, tmp_path)
|
||||
|
||||
# Submit a job.
|
||||
submission_id = await job_manager.submit_job(
|
||||
entrypoint="ls",
|
||||
)
|
||||
|
||||
# Wait for the job to be finished.
|
||||
await async_wait_for_condition(
|
||||
check_job_succeeded, job_manager=job_manager, job_id=submission_id
|
||||
)
|
||||
|
||||
# Verify export events are written
|
||||
event_dir = f"{tmp_path}/export_events"
|
||||
assert os.path.isdir(event_dir)
|
||||
event_file = f"{event_dir}/event_EXPORT_SUBMISSION_JOB.log"
|
||||
assert os.path.isfile(event_file)
|
||||
|
||||
with open(event_file, "r") as f:
|
||||
lines = f.readlines()
|
||||
assert len(lines) == 3
|
||||
expected_status_values = ["PENDING", "RUNNING", "SUCCEEDED"]
|
||||
|
||||
for line, expected_status in zip(lines, expected_status_values):
|
||||
data = json.loads(line)
|
||||
assert data["source_type"] == "EXPORT_SUBMISSION_JOB"
|
||||
assert data["event_data"]["submission_job_id"] == submission_id
|
||||
assert data["event_data"]["status"] == expected_status
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(pytest.main(["-v", __file__]))
|
||||
Reference in New Issue
Block a user