wehub-resource-sync
781 lines
26 KiB
Python
781 lines
26 KiB
Python
import asyncio
|
|
import copy
|
|
import json
|
|
import logging
|
|
import os
|
|
import random
|
|
import socket
|
|
import sys
|
|
import tempfile
|
|
import time
|
|
from concurrent.futures import ThreadPoolExecutor
|
|
from datetime import datetime
|
|
from pprint import pprint
|
|
from unittest.mock import MagicMock
|
|
|
|
import numpy as np
|
|
import pytest
|
|
import requests
|
|
|
|
import ray
|
|
from ray._common.test_utils import wait_for_condition
|
|
from ray._common.utils import binary_to_hex
|
|
from ray._private.event.event_logger import (
|
|
EventLoggerAdapter,
|
|
filter_event_by_level,
|
|
get_event_id,
|
|
get_event_logger,
|
|
)
|
|
from ray._private.event.export_event_logger import (
|
|
EventLogType,
|
|
ExportEventLoggerAdapter,
|
|
get_export_event_logger,
|
|
)
|
|
from ray._private.protobuf_compat import message_to_dict
|
|
from ray._private.state_api_test_utils import create_api_options, verify_schema
|
|
from ray._private.test_utils import (
|
|
format_web_url,
|
|
wait_until_server_available,
|
|
)
|
|
from ray.cluster_utils import AutoscalingCluster
|
|
from ray.core.generated import (
|
|
event_pb2,
|
|
export_submission_job_event_pb2,
|
|
)
|
|
from ray.dashboard.modules.event import event_consts
|
|
from ray.dashboard.modules.event.event_head import _list_cluster_events_impl
|
|
from ray.dashboard.modules.event.event_utils import monitor_events
|
|
from ray.dashboard.tests.conftest import * # noqa
|
|
from ray.job_submission import JobSubmissionClient
|
|
from ray.util.state import list_cluster_events
|
|
from ray.util.state.common import ClusterEventState
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
|
|
def _get_event(msg="empty message", job_id=None, source_type=None):
|
|
return {
|
|
"event_id": binary_to_hex(np.random.bytes(18)),
|
|
"source_type": (
|
|
random.choice(event_pb2.Event.SourceType.keys())
|
|
if source_type is None
|
|
else source_type
|
|
),
|
|
"host_name": "po-dev.inc.alipay.net",
|
|
"pid": random.randint(1, 65536),
|
|
"label": "",
|
|
"message": msg,
|
|
"timestamp": time.time(),
|
|
"severity": "INFO",
|
|
"custom_fields": {
|
|
"job_id": (
|
|
ray.JobID.from_int(random.randint(1, 100)).hex()
|
|
if job_id is None
|
|
else job_id
|
|
),
|
|
"node_id": "",
|
|
"task_id": "",
|
|
},
|
|
}
|
|
|
|
|
|
def _test_logger(name, log_file, max_bytes, backup_count):
|
|
handler = logging.handlers.RotatingFileHandler(
|
|
log_file, maxBytes=max_bytes, backupCount=backup_count
|
|
)
|
|
formatter = logging.Formatter("%(message)s")
|
|
handler.setFormatter(formatter)
|
|
|
|
logger = logging.getLogger(name)
|
|
logger.propagate = False
|
|
logger.setLevel(logging.INFO)
|
|
logger.addHandler(handler)
|
|
|
|
return logger
|
|
|
|
|
|
def test_python_global_event_logger(tmp_path):
|
|
logger = get_event_logger(event_pb2.Event.SourceType.GCS, str(tmp_path))
|
|
logger.set_global_context({"test_meta": "1"})
|
|
logger.info("message", a="a", b="b")
|
|
logger.error("message", a="a", b="b")
|
|
logger.warning("message", a="a", b="b")
|
|
logger.fatal("message", a="a", b="b")
|
|
event_dir = tmp_path / "events"
|
|
assert event_dir.exists()
|
|
event_file = event_dir / "event_GCS.log"
|
|
assert event_file.exists()
|
|
|
|
line_severities = ["INFO", "ERROR", "WARNING", "FATAL"]
|
|
|
|
with event_file.open() as f:
|
|
for line, severity in zip(f.readlines(), line_severities):
|
|
data = json.loads(line)
|
|
assert data["severity"] == severity
|
|
assert data["label"] == ""
|
|
assert "timestamp" in data
|
|
assert len(data["event_id"]) == 36
|
|
assert data["message"] == "message"
|
|
assert data["source_type"] == "GCS"
|
|
assert data["source_hostname"] == socket.gethostname()
|
|
assert data["source_pid"] == os.getpid()
|
|
assert data["custom_fields"]["a"] == "a"
|
|
assert data["custom_fields"]["b"] == "b"
|
|
|
|
|
|
def test_event_basic(disable_aiohttp_cache, ray_start_with_dashboard):
|
|
assert wait_until_server_available(ray_start_with_dashboard["webui_url"])
|
|
webui_url = format_web_url(ray_start_with_dashboard["webui_url"])
|
|
session_dir = ray_start_with_dashboard["session_dir"]
|
|
event_dir = os.path.join(session_dir, "logs", "events")
|
|
job_id = ray.JobID.from_int(100).hex()
|
|
|
|
source_type_gcs = event_pb2.Event.SourceType.Name(event_pb2.Event.GCS)
|
|
source_type_raylet = event_pb2.Event.SourceType.Name(event_pb2.Event.RAYLET)
|
|
test_count = 20
|
|
|
|
for source_type in [source_type_gcs, source_type_raylet]:
|
|
test_log_file = os.path.join(event_dir, f"event_{source_type}.log")
|
|
test_logger = _test_logger(
|
|
__name__ + str(random.random()),
|
|
test_log_file,
|
|
max_bytes=2000,
|
|
backup_count=0,
|
|
)
|
|
for i in range(test_count):
|
|
sample_event = _get_event(str(i), job_id=job_id, source_type=source_type)
|
|
test_logger.info("%s", json.dumps(sample_event))
|
|
|
|
def _check_events():
|
|
try:
|
|
resp = requests.get(f"{webui_url}/events")
|
|
resp.raise_for_status()
|
|
result = resp.json()
|
|
all_events = result["data"]["events"]
|
|
job_events = all_events[job_id]
|
|
assert len(job_events) >= test_count * 2
|
|
source_messages = {}
|
|
for e in job_events:
|
|
source_type = e["sourceType"]
|
|
message = e["message"]
|
|
source_messages.setdefault(source_type, set()).add(message)
|
|
assert len(source_messages[source_type_gcs]) >= test_count
|
|
assert len(source_messages[source_type_raylet]) >= test_count
|
|
data = {str(i) for i in range(test_count)}
|
|
assert data & source_messages[source_type_gcs] == data
|
|
assert data & source_messages[source_type_raylet] == data
|
|
return True
|
|
except Exception as ex:
|
|
logger.exception(ex)
|
|
return False
|
|
|
|
wait_for_condition(_check_events, timeout=15)
|
|
|
|
|
|
def test_event_message_limit(
|
|
small_event_line_limit, disable_aiohttp_cache, ray_start_with_dashboard
|
|
):
|
|
event_read_line_length_limit = small_event_line_limit
|
|
assert wait_until_server_available(ray_start_with_dashboard["webui_url"])
|
|
webui_url = format_web_url(ray_start_with_dashboard["webui_url"])
|
|
session_dir = ray_start_with_dashboard["session_dir"]
|
|
event_dir = os.path.join(session_dir, "logs", "events")
|
|
job_id = ray.JobID.from_int(100).hex()
|
|
events = []
|
|
# Sample event equals with limit.
|
|
sample_event = _get_event("", job_id=job_id)
|
|
message_len = event_read_line_length_limit - len(json.dumps(sample_event))
|
|
for i in range(10):
|
|
sample_event = copy.deepcopy(sample_event)
|
|
sample_event["event_id"] = binary_to_hex(np.random.bytes(18))
|
|
sample_event["message"] = str(i) * message_len
|
|
assert len(json.dumps(sample_event)) == event_read_line_length_limit
|
|
events.append(sample_event)
|
|
# Sample event longer than limit.
|
|
sample_event = copy.deepcopy(sample_event)
|
|
sample_event["event_id"] = binary_to_hex(np.random.bytes(18))
|
|
sample_event["message"] = "2" * (message_len + 1)
|
|
assert len(json.dumps(sample_event)) > event_read_line_length_limit
|
|
events.append(sample_event)
|
|
|
|
for i in range(event_consts.EVENT_READ_LINE_COUNT_LIMIT):
|
|
events.append(_get_event(str(i), job_id=job_id))
|
|
|
|
with open(os.path.join(event_dir, "tmp.log"), "w") as f:
|
|
f.writelines([(json.dumps(e) + "\n") for e in events])
|
|
|
|
try:
|
|
os.remove(os.path.join(event_dir, "event_GCS.log"))
|
|
except Exception:
|
|
pass
|
|
os.rename(
|
|
os.path.join(event_dir, "tmp.log"), os.path.join(event_dir, "event_GCS.log")
|
|
)
|
|
|
|
def _check_events():
|
|
try:
|
|
resp = requests.get(f"{webui_url}/events")
|
|
resp.raise_for_status()
|
|
result = resp.json()
|
|
all_events = result["data"]["events"]
|
|
assert (
|
|
len(all_events[job_id]) >= event_consts.EVENT_READ_LINE_COUNT_LIMIT + 10
|
|
)
|
|
messages = [e["message"] for e in all_events[job_id]]
|
|
for i in range(10):
|
|
assert str(i) * message_len in messages
|
|
assert "2" * (message_len + 1) not in messages
|
|
assert str(event_consts.EVENT_READ_LINE_COUNT_LIMIT - 1) in messages
|
|
return True
|
|
except Exception as ex:
|
|
logger.exception(ex)
|
|
return False
|
|
|
|
wait_for_condition(_check_events, timeout=15)
|
|
|
|
|
|
def test_report_events(ray_start_with_dashboard):
|
|
assert wait_until_server_available(ray_start_with_dashboard["webui_url"])
|
|
webui_url = format_web_url(ray_start_with_dashboard["webui_url"])
|
|
url = f"{webui_url}/report_events"
|
|
|
|
resp = requests.post(url)
|
|
assert resp.status_code == 400
|
|
resp = requests.post(url, json={"Hello": "World"})
|
|
assert resp.status_code == 400
|
|
|
|
job_id = ray.JobID.from_int(100).hex()
|
|
sample_event = _get_event("Hello", job_id=job_id)
|
|
resp = requests.post(url, json=[json.dumps(sample_event)])
|
|
assert resp.status_code == 200
|
|
|
|
resp = requests.get(f"{webui_url}/events")
|
|
assert resp.status_code == 200
|
|
result = resp.json()
|
|
all_events = result["data"]["events"]
|
|
assert len(all_events) == 1
|
|
assert job_id in all_events
|
|
assert len(all_events[job_id]) == 1
|
|
assert all_events[job_id][0]["message"] == "Hello"
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_monitor_events():
|
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
common = event_pb2.Event.SourceType.Name(event_pb2.Event.COMMON)
|
|
common_log = os.path.join(temp_dir, f"event_{common}.log")
|
|
test_logger = _test_logger(
|
|
__name__ + str(random.random()), common_log, max_bytes=10, backup_count=0
|
|
)
|
|
test_events1 = []
|
|
monitor_task = monitor_events(
|
|
temp_dir, lambda x: test_events1.extend(x), None, scan_interval_seconds=0.01
|
|
)
|
|
assert not monitor_task.done()
|
|
count = 10
|
|
|
|
async def _writer(*args, read_events, spin=True):
|
|
for x in range(*args):
|
|
test_logger.info("%s", x)
|
|
if spin:
|
|
while str(x) not in read_events:
|
|
await asyncio.sleep(0.01)
|
|
|
|
async def _check_events(expect_events, read_events, timeout=10):
|
|
start_time = time.time()
|
|
while True:
|
|
sorted_events = sorted(int(i) for i in read_events)
|
|
sorted_events = [str(i) for i in sorted_events]
|
|
if time.time() - start_time > timeout:
|
|
raise TimeoutError(
|
|
f"Timeout, read events: {sorted_events}, "
|
|
f"expect events: {expect_events}"
|
|
)
|
|
if len(sorted_events) == len(expect_events):
|
|
if sorted_events == expect_events:
|
|
break
|
|
await asyncio.sleep(1)
|
|
|
|
await asyncio.gather(
|
|
_writer(count, read_events=test_events1),
|
|
_check_events([str(i) for i in range(count)], read_events=test_events1),
|
|
)
|
|
|
|
monitor_task.cancel()
|
|
test_events2 = []
|
|
monitor_task = monitor_events(
|
|
temp_dir, lambda x: test_events2.extend(x), None, scan_interval_seconds=0.1
|
|
)
|
|
|
|
await _check_events([str(i) for i in range(count)], read_events=test_events2)
|
|
|
|
await _writer(count, count * 2, read_events=test_events2)
|
|
await _check_events(
|
|
[str(i) for i in range(count * 2)], read_events=test_events2
|
|
)
|
|
|
|
log_file_count = len(os.listdir(temp_dir))
|
|
|
|
test_logger = _test_logger(
|
|
__name__ + str(random.random()), common_log, max_bytes=1000, backup_count=0
|
|
)
|
|
assert len(os.listdir(temp_dir)) == log_file_count
|
|
|
|
await _writer(count * 2, count * 3, spin=False, read_events=test_events2)
|
|
await _check_events(
|
|
[str(i) for i in range(count * 3)], read_events=test_events2
|
|
)
|
|
await _writer(count * 3, count * 4, spin=False, read_events=test_events2)
|
|
await _check_events(
|
|
[str(i) for i in range(count * 4)], read_events=test_events2
|
|
)
|
|
|
|
# Test cancel monitor task.
|
|
monitor_task.cancel()
|
|
with pytest.raises(asyncio.CancelledError):
|
|
await monitor_task
|
|
assert monitor_task.done()
|
|
|
|
assert len(os.listdir(temp_dir)) == 1, "There should just be 1 event log"
|
|
|
|
|
|
@pytest.mark.parametrize("autoscaler_v2", [False, True], ids=["v1", "v2"])
|
|
def test_autoscaler_cluster_events(autoscaler_v2, shutdown_only):
|
|
cluster = AutoscalingCluster(
|
|
head_resources={"CPU": 2},
|
|
worker_node_types={
|
|
"cpu_node": {
|
|
"resources": {
|
|
"CPU": 4,
|
|
},
|
|
"node_config": {},
|
|
"min_workers": 0,
|
|
"max_workers": 1,
|
|
},
|
|
"gpu_node": {
|
|
"resources": {
|
|
"CPU": 2,
|
|
"GPU": 1,
|
|
},
|
|
"node_config": {},
|
|
"min_workers": 0,
|
|
"max_workers": 1,
|
|
},
|
|
},
|
|
autoscaler_v2=autoscaler_v2,
|
|
idle_timeout_minutes=1,
|
|
)
|
|
|
|
try:
|
|
cluster.start()
|
|
ray.init("auto")
|
|
|
|
# Triggers the addition of a GPU node.
|
|
@ray.remote(num_gpus=1)
|
|
def f():
|
|
print("gpu ok")
|
|
|
|
# Triggers the addition of a CPU node.
|
|
@ray.remote(num_cpus=3)
|
|
def g():
|
|
print("cpu ok")
|
|
|
|
wait_for_condition(lambda: ray.cluster_resources()["CPU"] == 2)
|
|
ray.get(f.remote())
|
|
wait_for_condition(lambda: ray.cluster_resources()["CPU"] == 4)
|
|
wait_for_condition(lambda: ray.cluster_resources()["GPU"] == 1)
|
|
ray.get(g.remote())
|
|
wait_for_condition(lambda: ray.cluster_resources()["CPU"] == 8)
|
|
wait_for_condition(lambda: ray.cluster_resources()["GPU"] == 1)
|
|
|
|
# Trigger an infeasible task
|
|
g.options(num_cpus=0, num_gpus=5).remote()
|
|
|
|
def verify():
|
|
cluster_events = list_cluster_events()
|
|
print(cluster_events)
|
|
messages = {(e["message"], e["source_type"]) for e in cluster_events}
|
|
if not autoscaler_v2:
|
|
# With head node resources, we don't actually resized. So this event is
|
|
# not really accurate.
|
|
assert ("Resized to 2 CPUs.", "AUTOSCALER") in messages, cluster_events
|
|
assert (
|
|
"Adding 1 node(s) of type gpu_node.",
|
|
"AUTOSCALER",
|
|
) in messages, cluster_events
|
|
assert (
|
|
"Resized to 4 CPUs, 1 GPUs.",
|
|
"AUTOSCALER",
|
|
) in messages, cluster_events
|
|
assert (
|
|
"Adding 1 node(s) of type cpu_node.",
|
|
"AUTOSCALER",
|
|
) in messages, cluster_events
|
|
assert (
|
|
"Resized to 8 CPUs, 1 GPUs.",
|
|
"AUTOSCALER",
|
|
) in messages, cluster_events
|
|
assert "No available node types can fulfill resource request" in "".join(
|
|
[t[0] for t in messages]
|
|
)
|
|
|
|
return True
|
|
|
|
wait_for_condition(verify, timeout=30)
|
|
pprint(list_cluster_events())
|
|
finally:
|
|
ray.shutdown()
|
|
cluster.shutdown()
|
|
|
|
|
|
def test_filter_event_by_level(monkeypatch):
|
|
def gen_event(level: str):
|
|
return event_pb2.Event(
|
|
source_type=event_pb2.Event.AUTOSCALER,
|
|
severity=event_pb2.Event.Severity.Value(level),
|
|
message=level,
|
|
)
|
|
|
|
trace = gen_event("TRACE")
|
|
debug = gen_event("DEBUG")
|
|
info = gen_event("INFO")
|
|
warning = gen_event("WARNING")
|
|
error = gen_event("ERROR")
|
|
fatal = gen_event("FATAL")
|
|
|
|
def assert_events_filtered(events, expected, filter_level):
|
|
filtered = [e for e in events if filter_event_by_level(e, filter_level)]
|
|
print(filtered)
|
|
assert len(filtered) == len(expected)
|
|
assert {e.message for e in filtered} == {e.message for e in expected}
|
|
|
|
events = [trace, debug, info, warning, error, fatal]
|
|
assert_events_filtered(events, [], "TRACE")
|
|
assert_events_filtered(events, [trace], "DEBUG")
|
|
assert_events_filtered(events, [trace, debug], "INFO")
|
|
assert_events_filtered(events, [trace, debug, info], "WARNING")
|
|
assert_events_filtered(events, [trace, debug, info, warning], "ERROR")
|
|
assert_events_filtered(events, [trace, debug, info, warning, error], "FATAL")
|
|
|
|
|
|
def test_jobs_cluster_events(shutdown_only):
|
|
ray.init()
|
|
address = ray._private.worker._global_node.webui_url
|
|
address = format_web_url(address)
|
|
client = JobSubmissionClient(address)
|
|
submission_id = client.submit_job(entrypoint="ls")
|
|
|
|
def verify():
|
|
events = list_cluster_events()
|
|
assert len(list_cluster_events()) == 2
|
|
start_event = events[0]
|
|
completed_event = events[1]
|
|
|
|
assert start_event["source_type"] == "JOBS"
|
|
assert f"Started a ray job {submission_id}" in start_event["message"]
|
|
assert start_event["severity"] == "INFO"
|
|
assert completed_event["source_type"] == "JOBS"
|
|
assert (
|
|
f"Completed a ray job {submission_id} with a status SUCCEEDED."
|
|
== completed_event["message"]
|
|
)
|
|
assert completed_event["severity"] == "INFO"
|
|
return True
|
|
|
|
print("Test successful job run.")
|
|
wait_for_condition(verify)
|
|
pprint(list_cluster_events())
|
|
|
|
# Test the failure case. In this part, job fails because the runtime env
|
|
# creation fails.
|
|
submission_id = client.submit_job(
|
|
entrypoint="ls",
|
|
runtime_env={"pip": ["nonexistent_dep"]},
|
|
)
|
|
|
|
def verify():
|
|
events = list_cluster_events(detail=True)
|
|
failed_events = []
|
|
|
|
for e in events:
|
|
if (
|
|
"submission_id" in e["custom_fields"]
|
|
and e["custom_fields"]["submission_id"] == submission_id
|
|
):
|
|
failed_events.append(e)
|
|
|
|
assert len(failed_events) == 2
|
|
failed_start = failed_events[0]
|
|
failed_completed = failed_events[1]
|
|
|
|
assert failed_start["source_type"] == "JOBS"
|
|
assert f"Started a ray job {submission_id}" in failed_start["message"]
|
|
assert failed_completed["source_type"] == "JOBS"
|
|
assert failed_completed["severity"] == "ERROR"
|
|
assert (
|
|
f"Completed a ray job {submission_id} with a status FAILED."
|
|
in failed_completed["message"]
|
|
)
|
|
|
|
# Make sure the error message is included.
|
|
assert "ERROR: No matching distribution found" in failed_completed["message"]
|
|
return True
|
|
|
|
print("Test failed (runtime_env failure) job run.")
|
|
wait_for_condition(verify, timeout=30)
|
|
pprint(list_cluster_events())
|
|
|
|
|
|
def test_core_events(shutdown_only):
|
|
# Test events recorded from core RAY_EVENT APIs.
|
|
ray.init()
|
|
|
|
@ray.remote
|
|
class Actor:
|
|
def getpid(self):
|
|
return os.getpid()
|
|
|
|
a = Actor.remote()
|
|
pid = ray.get(a.getpid.remote())
|
|
os.kill(pid, 9)
|
|
s = time.time()
|
|
|
|
def verify():
|
|
events = list_cluster_events(filters=[("source_type", "=", "RAYLET")])
|
|
print(events)
|
|
assert len(list_cluster_events()) == 1
|
|
event = events[0]
|
|
assert event["severity"] == "ERROR"
|
|
datetime_str = event["time"]
|
|
datetime_obj = datetime.strptime(datetime_str, "%Y-%m-%d %H:%M:%S")
|
|
timestamp = time.mktime(datetime_obj.timetuple())
|
|
|
|
# Make sure timestamp is not incorrect. Add sufficient buffer (60 seconds)
|
|
assert abs(timestamp - s) < 60
|
|
assert (
|
|
"A worker died or was killed while executing "
|
|
"a task by an unexpected system error" in event["message"]
|
|
)
|
|
return True
|
|
|
|
wait_for_condition(verify)
|
|
pprint(list_cluster_events())
|
|
|
|
|
|
def test_cluster_events_retention(monkeypatch, shutdown_only):
|
|
with monkeypatch.context() as m:
|
|
# defer for 5s for the second node.
|
|
# This will help the API not return until the node is killed.
|
|
m.setenv("RAY_DASHBOARD_MAX_EVENTS_TO_CACHE", "10")
|
|
ray.init()
|
|
address = ray._private.worker._global_node.webui_url
|
|
address = format_web_url(address)
|
|
client = JobSubmissionClient(address)
|
|
|
|
submission_ids = []
|
|
for _ in range(12):
|
|
submission_ids.append(client.submit_job(entrypoint="ls"))
|
|
print(submission_ids)
|
|
|
|
def verify():
|
|
events = list_cluster_events()
|
|
assert len(list_cluster_events()) == 10
|
|
|
|
messages = [event["message"] for event in events]
|
|
|
|
# Make sure the first two has been GC'ed.
|
|
for m in messages:
|
|
assert submission_ids[0] not in m
|
|
assert submission_ids[1] not in m
|
|
return True
|
|
|
|
wait_for_condition(verify)
|
|
pprint(list_cluster_events())
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_list_cluster_events_impl():
|
|
executor = ThreadPoolExecutor(
|
|
max_workers=1,
|
|
thread_name_prefix="event_head_executor",
|
|
)
|
|
|
|
event_id_1 = get_event_id()
|
|
event_id_2 = get_event_id()
|
|
events = {
|
|
"job_1": {
|
|
event_id_1: {
|
|
"timestamp": 10,
|
|
"severity": "DEBUG",
|
|
"message": "a",
|
|
"event_id": event_id_1,
|
|
"source_type": "GCS",
|
|
},
|
|
event_id_2: {
|
|
"timestamp": 10,
|
|
"severity": "INFO",
|
|
"message": "b",
|
|
"event_id": event_id_2,
|
|
"source_type": "GCS",
|
|
},
|
|
}
|
|
}
|
|
result = await _list_cluster_events_impl(
|
|
all_events=events, executor=executor, option=create_api_options()
|
|
)
|
|
data = result.result
|
|
data = data[0]
|
|
verify_schema(ClusterEventState, data)
|
|
assert result.total == 2
|
|
|
|
"""
|
|
Test detail
|
|
"""
|
|
# TODO(sang)
|
|
|
|
"""
|
|
Test limit
|
|
"""
|
|
assert len(result.result) == 2
|
|
result = await _list_cluster_events_impl(
|
|
all_events=events, executor=executor, option=create_api_options(limit=1)
|
|
)
|
|
data = result.result
|
|
assert len(data) == 1
|
|
assert result.total == 2
|
|
|
|
"""
|
|
Test filters
|
|
"""
|
|
# If the column is not supported for filtering, it should raise an exception.
|
|
with pytest.raises(ValueError):
|
|
result = await _list_cluster_events_impl(
|
|
all_events=events,
|
|
executor=executor,
|
|
option=create_api_options(filters=[("time", "=", "20")]),
|
|
)
|
|
result = await _list_cluster_events_impl(
|
|
all_events=events,
|
|
executor=executor,
|
|
option=create_api_options(filters=[("severity", "=", "INFO")]),
|
|
)
|
|
assert len(result.result) == 1
|
|
|
|
|
|
def test_export_event_logger(tmp_path):
|
|
"""
|
|
Unit test a mock export event of type ExportSubmissionJobEventData
|
|
is correctly written to file. This doesn't events are correctly generated.
|
|
"""
|
|
logger = get_export_event_logger(EventLogType.SUBMISSION_JOB, str(tmp_path))
|
|
ExportSubmissionJobEventData = (
|
|
export_submission_job_event_pb2.ExportSubmissionJobEventData
|
|
)
|
|
event_data = ExportSubmissionJobEventData(
|
|
submission_job_id="submission_job_id0",
|
|
status=ExportSubmissionJobEventData.JobStatus.RUNNING,
|
|
entrypoint="ls",
|
|
metadata={},
|
|
)
|
|
logger.send_event(event_data)
|
|
|
|
event_dir = tmp_path / "export_events"
|
|
assert event_dir.exists()
|
|
event_file = event_dir / "event_EXPORT_SUBMISSION_JOB.log"
|
|
assert event_file.exists()
|
|
|
|
with event_file.open() as f:
|
|
lines = f.readlines()
|
|
assert len(lines) == 1
|
|
|
|
line = lines[0]
|
|
data = json.loads(line)
|
|
assert data["source_type"] == "EXPORT_SUBMISSION_JOB"
|
|
assert data["event_data"] == message_to_dict(
|
|
event_data,
|
|
always_print_fields_with_no_presence=True,
|
|
preserving_proto_field_name=True,
|
|
use_integers_for_enums=False,
|
|
)
|
|
|
|
|
|
def test_event_logger_flushes_all_handlers():
|
|
mock_logger = MagicMock()
|
|
handlers = [MagicMock() for _ in range(3)]
|
|
mock_logger.handlers = handlers
|
|
|
|
adapter = EventLoggerAdapter(event_pb2.Event.GCS, mock_logger)
|
|
adapter.info("message")
|
|
|
|
for handler in handlers:
|
|
handler.flush.assert_called_once()
|
|
|
|
|
|
def test_event_logger_allows_empty_handlers():
|
|
mock_logger = MagicMock()
|
|
mock_logger.handlers = []
|
|
|
|
adapter = EventLoggerAdapter(event_pb2.Event.GCS, mock_logger)
|
|
adapter.info("message")
|
|
|
|
|
|
def test_export_event_logger_flushes_all_handlers():
|
|
mock_logger = MagicMock()
|
|
handlers = [MagicMock() for _ in range(3)]
|
|
mock_logger.handlers = handlers
|
|
|
|
adapter = ExportEventLoggerAdapter(EventLogType.SUBMISSION_JOB, mock_logger)
|
|
event_data = export_submission_job_event_pb2.ExportSubmissionJobEventData(
|
|
submission_job_id="submission_job_id0",
|
|
status=(
|
|
export_submission_job_event_pb2.ExportSubmissionJobEventData.JobStatus.RUNNING
|
|
),
|
|
entrypoint="ls",
|
|
metadata={},
|
|
)
|
|
adapter.send_event(event_data)
|
|
|
|
for handler in handlers:
|
|
handler.flush.assert_called_once()
|
|
|
|
|
|
def test_export_event_logger_allows_empty_handlers():
|
|
mock_logger = MagicMock()
|
|
mock_logger.handlers = []
|
|
|
|
adapter = ExportEventLoggerAdapter(EventLogType.SUBMISSION_JOB, mock_logger)
|
|
event_data = export_submission_job_event_pb2.ExportSubmissionJobEventData(
|
|
submission_job_id="submission_job_id0",
|
|
status=(
|
|
export_submission_job_event_pb2.ExportSubmissionJobEventData.JobStatus.RUNNING
|
|
),
|
|
entrypoint="ls",
|
|
metadata={},
|
|
)
|
|
adapter.send_event(event_data)
|
|
|
|
|
|
def test_export_event_logger_continues_flushing_after_handler_error():
|
|
mock_logger = MagicMock()
|
|
handler1 = MagicMock()
|
|
handler1.flush.side_effect = RuntimeError("flush failed")
|
|
handler2 = MagicMock()
|
|
mock_logger.handlers = [handler1, handler2]
|
|
|
|
adapter = ExportEventLoggerAdapter(EventLogType.SUBMISSION_JOB, mock_logger)
|
|
event_data = export_submission_job_event_pb2.ExportSubmissionJobEventData(
|
|
submission_job_id="submission_job_id0",
|
|
status=(
|
|
export_submission_job_event_pb2.ExportSubmissionJobEventData.JobStatus.RUNNING
|
|
),
|
|
entrypoint="ls",
|
|
metadata={},
|
|
)
|
|
adapter.send_event(event_data)
|
|
|
|
handler2.flush.assert_called_once()
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(pytest.main(["-v", __file__]))
|