chore: import upstream snapshot with attribution
This commit is contained in:
wehub-resource-sync
@@ -0,0 +1,138 @@
|
||||
import asyncio
|
||||
import logging
|
||||
import os
|
||||
import time
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from typing import Union
|
||||
|
||||
import ray._private.ray_constants as ray_constants
|
||||
import ray.dashboard.consts as dashboard_consts
|
||||
import ray.dashboard.utils as dashboard_utils
|
||||
from ray._private.authentication.http_token_authentication import (
|
||||
get_auth_headers_if_auth_enabled,
|
||||
)
|
||||
from ray.dashboard.modules.event import event_consts
|
||||
from ray.dashboard.modules.event.event_utils import monitor_events
|
||||
from ray.dashboard.utils import async_loop_forever, create_task
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
# NOTE: Executor in this head is intentionally constrained to just 1 thread by
|
||||
# default to limit its concurrency, therefore reducing potential for
|
||||
# GIL contention
|
||||
RAY_DASHBOARD_EVENT_AGENT_TPE_MAX_WORKERS = ray_constants.env_integer(
|
||||
"RAY_DASHBOARD_EVENT_AGENT_TPE_MAX_WORKERS", 1
|
||||
)
|
||||
|
||||
|
||||
class EventAgent(dashboard_utils.DashboardAgentModule):
|
||||
def __init__(self, dashboard_agent):
|
||||
super().__init__(dashboard_agent)
|
||||
self._event_dir = os.path.join(self._dashboard_agent.log_dir, "events")
|
||||
os.makedirs(self._event_dir, exist_ok=True)
|
||||
self._monitor: Union[asyncio.Task, None] = None
|
||||
# Lazy initialized on first use. Once initialized, it will not be
|
||||
# changed.
|
||||
self._dashboard_http_address = None
|
||||
self._cached_events = asyncio.Queue(event_consts.EVENT_AGENT_CACHE_SIZE)
|
||||
self._gcs_client = dashboard_agent.gcs_client
|
||||
# Total number of event created from this agent.
|
||||
self.total_event_reported = 0
|
||||
# Total number of event report request sent.
|
||||
self.total_request_sent = 0
|
||||
self.module_started = time.monotonic()
|
||||
|
||||
self._executor = ThreadPoolExecutor(
|
||||
max_workers=RAY_DASHBOARD_EVENT_AGENT_TPE_MAX_WORKERS,
|
||||
thread_name_prefix="event_agent_executor",
|
||||
)
|
||||
|
||||
logger.info("Event agent cache buffer size: %s", self._cached_events.maxsize)
|
||||
|
||||
async def _get_dashboard_http_address(self):
|
||||
"""
|
||||
Lazily get the dashboard http address from InternalKV. If it's not set, sleep
|
||||
and retry forever.
|
||||
"""
|
||||
while True:
|
||||
if self._dashboard_http_address:
|
||||
return self._dashboard_http_address
|
||||
try:
|
||||
dashboard_http_address = await self._gcs_client.async_internal_kv_get(
|
||||
ray_constants.DASHBOARD_ADDRESS.encode(),
|
||||
namespace=ray_constants.KV_NAMESPACE_DASHBOARD,
|
||||
timeout=dashboard_consts.GCS_RPC_TIMEOUT_SECONDS,
|
||||
)
|
||||
if not dashboard_http_address:
|
||||
raise ValueError("Dashboard http address not found in InternalKV.")
|
||||
address = dashboard_http_address.decode()
|
||||
if not address.startswith(("http://", "https://")):
|
||||
address = f"http://{address}"
|
||||
self._dashboard_http_address = address
|
||||
return self._dashboard_http_address
|
||||
except Exception:
|
||||
logger.exception("Get dashboard http address failed.")
|
||||
await asyncio.sleep(1)
|
||||
|
||||
@async_loop_forever(event_consts.EVENT_AGENT_REPORT_INTERVAL_SECONDS)
|
||||
async def report_events(self):
|
||||
"""Report events from cached events queue. Reconnect to dashboard if
|
||||
report failed. Log error after retry EVENT_AGENT_RETRY_TIMES.
|
||||
|
||||
This method will never returns.
|
||||
"""
|
||||
dashboard_http_address = await self._get_dashboard_http_address()
|
||||
data = await self._cached_events.get()
|
||||
self.total_event_reported += len(data)
|
||||
last_exception = None
|
||||
for _ in range(event_consts.EVENT_AGENT_RETRY_TIMES):
|
||||
try:
|
||||
logger.debug("Report %s events.", len(data))
|
||||
async with self._dashboard_agent.http_session.post(
|
||||
f"{dashboard_http_address}/report_events",
|
||||
json=data,
|
||||
headers=get_auth_headers_if_auth_enabled({}),
|
||||
) as response:
|
||||
response.raise_for_status()
|
||||
self.total_request_sent += 1
|
||||
break
|
||||
except Exception as e:
|
||||
logger.warning(f"Report event failed, retrying... {e}")
|
||||
last_exception = e
|
||||
else:
|
||||
data_str = str(data)
|
||||
limit = event_consts.LOG_ERROR_EVENT_STRING_LENGTH_LIMIT
|
||||
logger.error(
|
||||
"Report event failed: %s",
|
||||
data_str[:limit] + (data_str[limit:] and "..."),
|
||||
exc_info=last_exception,
|
||||
)
|
||||
|
||||
async def get_internal_states(self):
|
||||
if self.total_event_reported <= 0 or self.total_request_sent <= 0:
|
||||
return
|
||||
|
||||
elapsed = time.monotonic() - self.module_started
|
||||
return {
|
||||
"total_events_reported": self.total_event_reported,
|
||||
"Total_report_request": self.total_request_sent,
|
||||
"queue_size": self._cached_events.qsize(),
|
||||
"total_uptime": elapsed,
|
||||
}
|
||||
|
||||
async def run(self, server):
|
||||
# Start monitor task.
|
||||
self._monitor = monitor_events(
|
||||
self._event_dir,
|
||||
lambda data: create_task(self._cached_events.put(data)),
|
||||
self._executor,
|
||||
)
|
||||
|
||||
await asyncio.gather(
|
||||
self.report_events(),
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def is_minimal_module():
|
||||
return False
|
||||
@@ -0,0 +1,20 @@
|
||||
from ray._private.ray_constants import env_float, env_integer
|
||||
from ray.core.generated import event_pb2
|
||||
|
||||
LOG_ERROR_EVENT_STRING_LENGTH_LIMIT = 1000
|
||||
# Monitor events
|
||||
SCAN_EVENT_DIR_INTERVAL_SECONDS = env_integer("SCAN_EVENT_DIR_INTERVAL_SECONDS", 2)
|
||||
SCAN_EVENT_START_OFFSET_SECONDS = -30 * 60
|
||||
CONCURRENT_READ_LIMIT = 50
|
||||
EVENT_READ_LINE_COUNT_LIMIT = 200
|
||||
EVENT_READ_LINE_LENGTH_LIMIT = env_integer(
|
||||
"EVENT_READ_LINE_LENGTH_LIMIT", 2 * 1024 * 1024
|
||||
) # 2MB
|
||||
# Report events
|
||||
EVENT_AGENT_REPORT_INTERVAL_SECONDS = env_float(
|
||||
"EVENT_AGENT_REPORT_INTERVAL_SECONDS", 0.1
|
||||
)
|
||||
EVENT_AGENT_RETRY_TIMES = 10
|
||||
EVENT_AGENT_CACHE_SIZE = 10240
|
||||
# Event sources
|
||||
EVENT_SOURCE_ALL = event_pb2.Event.SourceType.keys()
|
||||
@@ -0,0 +1,237 @@
|
||||
import asyncio
|
||||
import logging
|
||||
import os
|
||||
import time
|
||||
from collections import OrderedDict, defaultdict
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from datetime import datetime
|
||||
from itertools import islice
|
||||
from typing import Dict, List, Union
|
||||
|
||||
import aiohttp.web
|
||||
|
||||
import ray
|
||||
import ray.dashboard.optional_utils as dashboard_optional_utils
|
||||
import ray.dashboard.utils as dashboard_utils
|
||||
from ray._common.usage.usage_lib import TagKey, record_extra_usage_tag
|
||||
from ray._common.utils import get_or_create_event_loop
|
||||
from ray._private.ray_constants import env_integer
|
||||
from ray.dashboard.consts import (
|
||||
RAY_STATE_SERVER_MAX_HTTP_REQUEST,
|
||||
RAY_STATE_SERVER_MAX_HTTP_REQUEST_ALLOWED,
|
||||
RAY_STATE_SERVER_MAX_HTTP_REQUEST_ENV_NAME,
|
||||
)
|
||||
from ray.dashboard.modules.event.event_utils import monitor_events, parse_event_strings
|
||||
from ray.dashboard.state_api_utils import do_filter, handle_list_api
|
||||
from ray.dashboard.subprocesses.module import SubprocessModule
|
||||
from ray.dashboard.subprocesses.routes import SubprocessRouteTable as routes
|
||||
from ray.util.state.common import ClusterEventState, ListApiOptions, ListApiResponse
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
JobEvents = OrderedDict
|
||||
dashboard_utils._json_compatible_types.add(JobEvents)
|
||||
|
||||
MAX_EVENTS_TO_CACHE = int(os.environ.get("RAY_DASHBOARD_MAX_EVENTS_TO_CACHE", 10000))
|
||||
|
||||
# NOTE: Executor in this head is intentionally constrained to just 1 thread by
|
||||
# default to limit its concurrency, therefore reducing potential for
|
||||
# GIL contention
|
||||
RAY_DASHBOARD_EVENT_HEAD_TPE_MAX_WORKERS = env_integer(
|
||||
"RAY_DASHBOARD_EVENT_HEAD_TPE_MAX_WORKERS", 1
|
||||
)
|
||||
|
||||
|
||||
async def _list_cluster_events_impl(
|
||||
*,
|
||||
all_events: Dict[str, JobEvents],
|
||||
executor: ThreadPoolExecutor,
|
||||
option: ListApiOptions,
|
||||
) -> ListApiResponse:
|
||||
"""List all cluster events from the cluster. Made a free function to allow unit tests.
|
||||
|
||||
Args:
|
||||
all_events: Mapping of ``job_id`` to per-job event dictionaries.
|
||||
executor: Executor used to run the (CPU-bound) transform off the event loop.
|
||||
option: Query options (filters, limit, detail flag).
|
||||
|
||||
Returns:
|
||||
A list of cluster events in the cluster.
|
||||
The schema of returned "dict" is equivalent to the
|
||||
`ClusterEventState` protobuf message.
|
||||
"""
|
||||
|
||||
def transform(all_events) -> ListApiResponse:
|
||||
result = []
|
||||
for _, events in all_events.items():
|
||||
for _, event in events.items():
|
||||
event["time"] = str(datetime.fromtimestamp(int(event["timestamp"])))
|
||||
result.append(event)
|
||||
|
||||
num_after_truncation = len(result)
|
||||
result.sort(key=lambda entry: entry["timestamp"])
|
||||
total = len(result)
|
||||
result = do_filter(result, option.filters, ClusterEventState, option.detail)
|
||||
num_filtered = len(result)
|
||||
# Sort to make the output deterministic.
|
||||
result = list(islice(result, option.limit))
|
||||
return ListApiResponse(
|
||||
result=result,
|
||||
total=total,
|
||||
num_after_truncation=num_after_truncation,
|
||||
num_filtered=num_filtered,
|
||||
)
|
||||
|
||||
return await get_or_create_event_loop().run_in_executor(
|
||||
executor, transform, all_events
|
||||
)
|
||||
|
||||
|
||||
class EventHead(
|
||||
SubprocessModule,
|
||||
dashboard_utils.RateLimitedModule,
|
||||
):
|
||||
def __init__(self, *args, **kwargs):
|
||||
SubprocessModule.__init__(self, *args, **kwargs)
|
||||
dashboard_utils.RateLimitedModule.__init__(
|
||||
self,
|
||||
min(
|
||||
RAY_STATE_SERVER_MAX_HTTP_REQUEST,
|
||||
RAY_STATE_SERVER_MAX_HTTP_REQUEST_ALLOWED,
|
||||
),
|
||||
)
|
||||
self._event_dir = os.path.join(self.log_dir, "events")
|
||||
os.makedirs(self._event_dir, exist_ok=True)
|
||||
self._monitor: Union[asyncio.Task, None] = None
|
||||
self.total_report_events_count = 0
|
||||
self.total_events_received = 0
|
||||
self.module_started = time.monotonic()
|
||||
# {job_id hex(str): {event_id (str): event (dict)}}
|
||||
self.events: Dict[str, JobEvents] = defaultdict(JobEvents)
|
||||
|
||||
self._executor = ThreadPoolExecutor(
|
||||
max_workers=RAY_DASHBOARD_EVENT_HEAD_TPE_MAX_WORKERS,
|
||||
thread_name_prefix="event_head_executor",
|
||||
)
|
||||
|
||||
# To init gcs_client in internal_kv for record_extra_usage_tag.
|
||||
assert self.gcs_client is not None
|
||||
assert ray.experimental.internal_kv._internal_kv_initialized()
|
||||
|
||||
async def limit_handler_(self):
|
||||
return dashboard_optional_utils.rest_response(
|
||||
status_code=dashboard_utils.HTTPStatusCode.INTERNAL_ERROR,
|
||||
error_message=(
|
||||
"Max number of in-progress requests="
|
||||
f"{self.max_num_call_} reached. "
|
||||
"To set a higher limit, set environment variable: "
|
||||
f"export {RAY_STATE_SERVER_MAX_HTTP_REQUEST_ENV_NAME}='xxx'. "
|
||||
f"Max allowed = {RAY_STATE_SERVER_MAX_HTTP_REQUEST_ALLOWED}"
|
||||
),
|
||||
result=None,
|
||||
)
|
||||
|
||||
def _update_events(self, event_list):
|
||||
# {job_id: {event_id: event}}
|
||||
all_job_events = defaultdict(JobEvents)
|
||||
for event in event_list:
|
||||
event_id = event["event_id"]
|
||||
custom_fields = event.get("custom_fields")
|
||||
system_event = False
|
||||
if custom_fields:
|
||||
job_id = custom_fields.get("job_id", "global") or "global"
|
||||
else:
|
||||
job_id = "global"
|
||||
if system_event is False:
|
||||
all_job_events[job_id][event_id] = event
|
||||
|
||||
for job_id, new_job_events in all_job_events.items():
|
||||
job_events = self.events[job_id]
|
||||
job_events.update(new_job_events)
|
||||
|
||||
# Limit the # of events cached if it exceeds the threshold.
|
||||
if len(job_events) > MAX_EVENTS_TO_CACHE * 1.1:
|
||||
while len(job_events) > MAX_EVENTS_TO_CACHE:
|
||||
job_events.popitem(last=False)
|
||||
|
||||
@routes.post("/report_events")
|
||||
async def report_events(self, request):
|
||||
"""
|
||||
Report events to the dashboard.
|
||||
The request body is a JSON array of event strings in type string.
|
||||
Response should contain {"success": true}.
|
||||
"""
|
||||
try:
|
||||
request_body: List[str] = await request.json()
|
||||
except Exception as e:
|
||||
logger.warning(f"Failed to parse request body: {request=}, {e=}")
|
||||
raise aiohttp.web.HTTPBadRequest()
|
||||
if not isinstance(request_body, list):
|
||||
logger.warning(f"Request body is not a list, {request_body=}")
|
||||
raise aiohttp.web.HTTPBadRequest()
|
||||
events = parse_event_strings(request_body)
|
||||
logger.debug("Received %d events", len(events))
|
||||
self._update_events(events)
|
||||
self.total_report_events_count += 1
|
||||
self.total_events_received += len(events)
|
||||
return dashboard_optional_utils.rest_response(
|
||||
success=True,
|
||||
message="",
|
||||
status_code=dashboard_utils.HTTPStatusCode.OK,
|
||||
)
|
||||
|
||||
async def _periodic_state_print(self):
|
||||
if self.total_events_received <= 0 or self.total_report_events_count <= 0:
|
||||
return
|
||||
|
||||
elapsed = time.monotonic() - self.module_started
|
||||
return {
|
||||
"total_events_received": self.total_events_received,
|
||||
"Total_requests_received": self.total_report_events_count,
|
||||
"total_uptime": elapsed,
|
||||
}
|
||||
|
||||
@routes.get("/events")
|
||||
@dashboard_optional_utils.aiohttp_cache
|
||||
async def get_event(self, req) -> aiohttp.web.Response:
|
||||
job_id = req.query.get("job_id")
|
||||
if job_id is None:
|
||||
all_events = {
|
||||
job_id: list(job_events.values())
|
||||
for job_id, job_events in self.events.items()
|
||||
}
|
||||
return dashboard_optional_utils.rest_response(
|
||||
status_code=dashboard_utils.HTTPStatusCode.OK,
|
||||
message="All events fetched.",
|
||||
events=all_events,
|
||||
)
|
||||
|
||||
job_events = self.events[job_id]
|
||||
return dashboard_optional_utils.rest_response(
|
||||
status_code=dashboard_utils.HTTPStatusCode.OK,
|
||||
message="Job events fetched.",
|
||||
job_id=job_id,
|
||||
events=list(job_events.values()),
|
||||
)
|
||||
|
||||
@routes.get("/api/v0/cluster_events")
|
||||
@dashboard_utils.RateLimitedModule.enforce_max_concurrent_calls
|
||||
async def list_cluster_events(
|
||||
self, req: aiohttp.web.Request
|
||||
) -> aiohttp.web.Response:
|
||||
record_extra_usage_tag(TagKey.CORE_STATE_API_LIST_CLUSTER_EVENTS, "1")
|
||||
|
||||
async def list_api_fn(option: ListApiOptions):
|
||||
return await _list_cluster_events_impl(
|
||||
all_events=self.events, executor=self._executor, option=option
|
||||
)
|
||||
|
||||
return await handle_list_api(list_api_fn, req)
|
||||
|
||||
async def run(self):
|
||||
await super().run()
|
||||
self._monitor = monitor_events(
|
||||
self._event_dir,
|
||||
lambda data: self._update_events(parse_event_strings(data)),
|
||||
self._executor,
|
||||
)
|
||||
@@ -0,0 +1,210 @@
|
||||
import asyncio
|
||||
import collections
|
||||
import fnmatch
|
||||
import itertools
|
||||
import json
|
||||
import logging.handlers
|
||||
import mmap
|
||||
import os
|
||||
import time
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from typing import Callable, Dict, List, Optional
|
||||
|
||||
from ray._common.utils import get_or_create_event_loop, run_background_task
|
||||
from ray.dashboard.modules.event import event_consts
|
||||
from ray.dashboard.utils import async_loop_forever
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _get_source_files(event_dir, source_types=None, event_file_filter=None):
|
||||
event_log_names = os.listdir(event_dir)
|
||||
source_files = {}
|
||||
all_source_types = set(event_consts.EVENT_SOURCE_ALL)
|
||||
for source_type in source_types or event_consts.EVENT_SOURCE_ALL:
|
||||
assert source_type in all_source_types, f"Invalid source type: {source_type}"
|
||||
files = []
|
||||
for n in event_log_names:
|
||||
if fnmatch.fnmatch(n, f"*{source_type}*.log"):
|
||||
f = os.path.join(event_dir, n)
|
||||
if event_file_filter is not None and not event_file_filter(f):
|
||||
continue
|
||||
files.append(f)
|
||||
if files:
|
||||
source_files[source_type] = files
|
||||
return source_files
|
||||
|
||||
|
||||
def _restore_newline(event_dict):
|
||||
try:
|
||||
event_dict["message"] = (
|
||||
event_dict["message"].replace("\\n", "\n").replace("\\r", "\n")
|
||||
)
|
||||
except Exception:
|
||||
logger.exception("Restore newline for event failed: %s", event_dict)
|
||||
return event_dict
|
||||
|
||||
|
||||
def _parse_line(event_str):
|
||||
return _restore_newline(json.loads(event_str))
|
||||
|
||||
|
||||
def parse_event_strings(event_string_list):
|
||||
events = []
|
||||
for data in event_string_list:
|
||||
if not data:
|
||||
continue
|
||||
try:
|
||||
event = _parse_line(data)
|
||||
events.append(event)
|
||||
except Exception:
|
||||
logger.exception("Parse event line failed: %s", repr(data))
|
||||
return events
|
||||
|
||||
|
||||
ReadFileResult = collections.namedtuple(
|
||||
"ReadFileResult", ["fid", "size", "mtime", "position", "lines"]
|
||||
)
|
||||
|
||||
|
||||
def _read_file(
|
||||
file, pos, n_lines=event_consts.EVENT_READ_LINE_COUNT_LIMIT, closefd=True
|
||||
):
|
||||
with open(file, "rb", closefd=closefd) as f:
|
||||
# The ino may be 0 on Windows.
|
||||
stat = os.stat(f.fileno())
|
||||
fid = stat.st_ino or file
|
||||
lines = []
|
||||
with mmap.mmap(f.fileno(), 0, access=mmap.ACCESS_READ) as mm:
|
||||
start = pos
|
||||
for _ in range(n_lines):
|
||||
sep = mm.find(b"\n", start)
|
||||
if sep == -1:
|
||||
break
|
||||
if sep - start <= event_consts.EVENT_READ_LINE_LENGTH_LIMIT:
|
||||
lines.append(mm[start:sep].decode("utf-8"))
|
||||
else:
|
||||
truncated_size = min(100, event_consts.EVENT_READ_LINE_LENGTH_LIMIT)
|
||||
logger.warning(
|
||||
"Ignored long string: %s...(%s chars)",
|
||||
mm[start : start + truncated_size].decode("utf-8"),
|
||||
sep - start,
|
||||
)
|
||||
start = sep + 1
|
||||
return ReadFileResult(fid, stat.st_size, stat.st_mtime, start, lines)
|
||||
|
||||
|
||||
def monitor_events(
|
||||
event_dir: str,
|
||||
callback: Callable[[List[str]], None],
|
||||
monitor_thread_pool_executor: ThreadPoolExecutor,
|
||||
scan_interval_seconds: float = event_consts.SCAN_EVENT_DIR_INTERVAL_SECONDS,
|
||||
start_mtime: float = time.time() + event_consts.SCAN_EVENT_START_OFFSET_SECONDS,
|
||||
monitor_files: Optional[Dict[int, tuple]] = None,
|
||||
source_types: Optional[List[str]] = None,
|
||||
) -> asyncio.Task:
|
||||
"""Monitor events in directory. New events will be read and passed to the
|
||||
callback.
|
||||
|
||||
Args:
|
||||
event_dir: The event log directory.
|
||||
callback: A callback that accepts a list of event strings.
|
||||
monitor_thread_pool_executor: A thread pool exector to monitor/update
|
||||
events. None means it will use the default execturo which uses
|
||||
num_cpus of the machine * 5 threads (before python 3.8) or
|
||||
min(32, num_cpus + 5) (from Python 3.8).
|
||||
scan_interval_seconds: An interval seconds between two scans.
|
||||
start_mtime: Only the event log files whose last modification
|
||||
time is greater than start_mtime are monitored.
|
||||
monitor_files: The map from event log file id to MonitorFile object.
|
||||
Monitor all files start from the beginning if the value is None.
|
||||
source_types: A list of source type name from
|
||||
event_pb2.Event.SourceType.keys(). Monitor all source types if the
|
||||
value is None.
|
||||
|
||||
Returns:
|
||||
The background ``asyncio.Task`` driving the periodic directory scan.
|
||||
"""
|
||||
loop = get_or_create_event_loop()
|
||||
if monitor_files is None:
|
||||
monitor_files = {}
|
||||
|
||||
logger.info(
|
||||
"Monitor events logs modified after %s on %s, the source types are %s.",
|
||||
start_mtime,
|
||||
event_dir,
|
||||
"all" if source_types is None else source_types,
|
||||
)
|
||||
|
||||
MonitorFile = collections.namedtuple("MonitorFile", ["size", "mtime", "position"])
|
||||
|
||||
def _source_file_filter(source_file):
|
||||
stat = os.stat(source_file)
|
||||
return stat.st_mtime > start_mtime
|
||||
|
||||
def _read_monitor_file(file, pos):
|
||||
assert isinstance(
|
||||
file, str
|
||||
), f"File should be a str, but a {type(file)}({file}) found"
|
||||
fd = os.open(file, os.O_RDONLY)
|
||||
try:
|
||||
stat = os.stat(fd)
|
||||
# Check the file size to avoid raising the exception
|
||||
# ValueError: cannot mmap an empty file
|
||||
if stat.st_size <= 0:
|
||||
return []
|
||||
fid = stat.st_ino or file
|
||||
monitor_file = monitor_files.get(fid)
|
||||
if monitor_file:
|
||||
if (
|
||||
monitor_file.position == monitor_file.size
|
||||
and monitor_file.size == stat.st_size
|
||||
and monitor_file.mtime == stat.st_mtime
|
||||
):
|
||||
logger.debug(
|
||||
"Skip reading the file because there is no change: %s", file
|
||||
)
|
||||
return []
|
||||
position = monitor_file.position
|
||||
else:
|
||||
logger.info("Found new event log file: %s", file)
|
||||
position = pos
|
||||
# Close the fd in finally.
|
||||
r = _read_file(fd, position, closefd=False)
|
||||
# It should be fine to update the dict in executor thread.
|
||||
monitor_files[r.fid] = MonitorFile(r.size, r.mtime, r.position)
|
||||
loop.call_soon_threadsafe(callback, r.lines)
|
||||
except Exception as e:
|
||||
raise Exception(f"Read event file failed: {file}") from e
|
||||
finally:
|
||||
os.close(fd)
|
||||
|
||||
@async_loop_forever(scan_interval_seconds, cancellable=True)
|
||||
async def _scan_event_log_files():
|
||||
# Scan event files.
|
||||
source_files = await loop.run_in_executor(
|
||||
monitor_thread_pool_executor,
|
||||
_get_source_files,
|
||||
event_dir,
|
||||
source_types,
|
||||
_source_file_filter,
|
||||
)
|
||||
|
||||
# Limit concurrent read to avoid fd exhaustion.
|
||||
semaphore = asyncio.Semaphore(event_consts.CONCURRENT_READ_LIMIT)
|
||||
|
||||
async def _concurrent_coro(filename):
|
||||
async with semaphore:
|
||||
return await loop.run_in_executor(
|
||||
monitor_thread_pool_executor, _read_monitor_file, filename, 0
|
||||
)
|
||||
|
||||
# Read files.
|
||||
await asyncio.gather(
|
||||
*[
|
||||
_concurrent_coro(filename)
|
||||
for filename in list(itertools.chain(*source_files.values()))
|
||||
]
|
||||
)
|
||||
|
||||
return run_background_task(_scan_event_log_files())
|
||||
@@ -0,0 +1,780 @@
|
||||
import asyncio
|
||||
import copy
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import random
|
||||
import socket
|
||||
import sys
|
||||
import tempfile
|
||||
import time
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from datetime import datetime
|
||||
from pprint import pprint
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
import requests
|
||||
|
||||
import ray
|
||||
from ray._common.test_utils import wait_for_condition
|
||||
from ray._common.utils import binary_to_hex
|
||||
from ray._private.event.event_logger import (
|
||||
EventLoggerAdapter,
|
||||
filter_event_by_level,
|
||||
get_event_id,
|
||||
get_event_logger,
|
||||
)
|
||||
from ray._private.event.export_event_logger import (
|
||||
EventLogType,
|
||||
ExportEventLoggerAdapter,
|
||||
get_export_event_logger,
|
||||
)
|
||||
from ray._private.protobuf_compat import message_to_dict
|
||||
from ray._private.state_api_test_utils import create_api_options, verify_schema
|
||||
from ray._private.test_utils import (
|
||||
format_web_url,
|
||||
wait_until_server_available,
|
||||
)
|
||||
from ray.cluster_utils import AutoscalingCluster
|
||||
from ray.core.generated import (
|
||||
event_pb2,
|
||||
export_submission_job_event_pb2,
|
||||
)
|
||||
from ray.dashboard.modules.event import event_consts
|
||||
from ray.dashboard.modules.event.event_head import _list_cluster_events_impl
|
||||
from ray.dashboard.modules.event.event_utils import monitor_events
|
||||
from ray.dashboard.tests.conftest import * # noqa
|
||||
from ray.job_submission import JobSubmissionClient
|
||||
from ray.util.state import list_cluster_events
|
||||
from ray.util.state.common import ClusterEventState
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _get_event(msg="empty message", job_id=None, source_type=None):
|
||||
return {
|
||||
"event_id": binary_to_hex(np.random.bytes(18)),
|
||||
"source_type": (
|
||||
random.choice(event_pb2.Event.SourceType.keys())
|
||||
if source_type is None
|
||||
else source_type
|
||||
),
|
||||
"host_name": "po-dev.inc.alipay.net",
|
||||
"pid": random.randint(1, 65536),
|
||||
"label": "",
|
||||
"message": msg,
|
||||
"timestamp": time.time(),
|
||||
"severity": "INFO",
|
||||
"custom_fields": {
|
||||
"job_id": (
|
||||
ray.JobID.from_int(random.randint(1, 100)).hex()
|
||||
if job_id is None
|
||||
else job_id
|
||||
),
|
||||
"node_id": "",
|
||||
"task_id": "",
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def _test_logger(name, log_file, max_bytes, backup_count):
|
||||
handler = logging.handlers.RotatingFileHandler(
|
||||
log_file, maxBytes=max_bytes, backupCount=backup_count
|
||||
)
|
||||
formatter = logging.Formatter("%(message)s")
|
||||
handler.setFormatter(formatter)
|
||||
|
||||
logger = logging.getLogger(name)
|
||||
logger.propagate = False
|
||||
logger.setLevel(logging.INFO)
|
||||
logger.addHandler(handler)
|
||||
|
||||
return logger
|
||||
|
||||
|
||||
def test_python_global_event_logger(tmp_path):
|
||||
logger = get_event_logger(event_pb2.Event.SourceType.GCS, str(tmp_path))
|
||||
logger.set_global_context({"test_meta": "1"})
|
||||
logger.info("message", a="a", b="b")
|
||||
logger.error("message", a="a", b="b")
|
||||
logger.warning("message", a="a", b="b")
|
||||
logger.fatal("message", a="a", b="b")
|
||||
event_dir = tmp_path / "events"
|
||||
assert event_dir.exists()
|
||||
event_file = event_dir / "event_GCS.log"
|
||||
assert event_file.exists()
|
||||
|
||||
line_severities = ["INFO", "ERROR", "WARNING", "FATAL"]
|
||||
|
||||
with event_file.open() as f:
|
||||
for line, severity in zip(f.readlines(), line_severities):
|
||||
data = json.loads(line)
|
||||
assert data["severity"] == severity
|
||||
assert data["label"] == ""
|
||||
assert "timestamp" in data
|
||||
assert len(data["event_id"]) == 36
|
||||
assert data["message"] == "message"
|
||||
assert data["source_type"] == "GCS"
|
||||
assert data["source_hostname"] == socket.gethostname()
|
||||
assert data["source_pid"] == os.getpid()
|
||||
assert data["custom_fields"]["a"] == "a"
|
||||
assert data["custom_fields"]["b"] == "b"
|
||||
|
||||
|
||||
def test_event_basic(disable_aiohttp_cache, ray_start_with_dashboard):
|
||||
assert wait_until_server_available(ray_start_with_dashboard["webui_url"])
|
||||
webui_url = format_web_url(ray_start_with_dashboard["webui_url"])
|
||||
session_dir = ray_start_with_dashboard["session_dir"]
|
||||
event_dir = os.path.join(session_dir, "logs", "events")
|
||||
job_id = ray.JobID.from_int(100).hex()
|
||||
|
||||
source_type_gcs = event_pb2.Event.SourceType.Name(event_pb2.Event.GCS)
|
||||
source_type_raylet = event_pb2.Event.SourceType.Name(event_pb2.Event.RAYLET)
|
||||
test_count = 20
|
||||
|
||||
for source_type in [source_type_gcs, source_type_raylet]:
|
||||
test_log_file = os.path.join(event_dir, f"event_{source_type}.log")
|
||||
test_logger = _test_logger(
|
||||
__name__ + str(random.random()),
|
||||
test_log_file,
|
||||
max_bytes=2000,
|
||||
backup_count=0,
|
||||
)
|
||||
for i in range(test_count):
|
||||
sample_event = _get_event(str(i), job_id=job_id, source_type=source_type)
|
||||
test_logger.info("%s", json.dumps(sample_event))
|
||||
|
||||
def _check_events():
|
||||
try:
|
||||
resp = requests.get(f"{webui_url}/events")
|
||||
resp.raise_for_status()
|
||||
result = resp.json()
|
||||
all_events = result["data"]["events"]
|
||||
job_events = all_events[job_id]
|
||||
assert len(job_events) >= test_count * 2
|
||||
source_messages = {}
|
||||
for e in job_events:
|
||||
source_type = e["sourceType"]
|
||||
message = e["message"]
|
||||
source_messages.setdefault(source_type, set()).add(message)
|
||||
assert len(source_messages[source_type_gcs]) >= test_count
|
||||
assert len(source_messages[source_type_raylet]) >= test_count
|
||||
data = {str(i) for i in range(test_count)}
|
||||
assert data & source_messages[source_type_gcs] == data
|
||||
assert data & source_messages[source_type_raylet] == data
|
||||
return True
|
||||
except Exception as ex:
|
||||
logger.exception(ex)
|
||||
return False
|
||||
|
||||
wait_for_condition(_check_events, timeout=15)
|
||||
|
||||
|
||||
def test_event_message_limit(
|
||||
small_event_line_limit, disable_aiohttp_cache, ray_start_with_dashboard
|
||||
):
|
||||
event_read_line_length_limit = small_event_line_limit
|
||||
assert wait_until_server_available(ray_start_with_dashboard["webui_url"])
|
||||
webui_url = format_web_url(ray_start_with_dashboard["webui_url"])
|
||||
session_dir = ray_start_with_dashboard["session_dir"]
|
||||
event_dir = os.path.join(session_dir, "logs", "events")
|
||||
job_id = ray.JobID.from_int(100).hex()
|
||||
events = []
|
||||
# Sample event equals with limit.
|
||||
sample_event = _get_event("", job_id=job_id)
|
||||
message_len = event_read_line_length_limit - len(json.dumps(sample_event))
|
||||
for i in range(10):
|
||||
sample_event = copy.deepcopy(sample_event)
|
||||
sample_event["event_id"] = binary_to_hex(np.random.bytes(18))
|
||||
sample_event["message"] = str(i) * message_len
|
||||
assert len(json.dumps(sample_event)) == event_read_line_length_limit
|
||||
events.append(sample_event)
|
||||
# Sample event longer than limit.
|
||||
sample_event = copy.deepcopy(sample_event)
|
||||
sample_event["event_id"] = binary_to_hex(np.random.bytes(18))
|
||||
sample_event["message"] = "2" * (message_len + 1)
|
||||
assert len(json.dumps(sample_event)) > event_read_line_length_limit
|
||||
events.append(sample_event)
|
||||
|
||||
for i in range(event_consts.EVENT_READ_LINE_COUNT_LIMIT):
|
||||
events.append(_get_event(str(i), job_id=job_id))
|
||||
|
||||
with open(os.path.join(event_dir, "tmp.log"), "w") as f:
|
||||
f.writelines([(json.dumps(e) + "\n") for e in events])
|
||||
|
||||
try:
|
||||
os.remove(os.path.join(event_dir, "event_GCS.log"))
|
||||
except Exception:
|
||||
pass
|
||||
os.rename(
|
||||
os.path.join(event_dir, "tmp.log"), os.path.join(event_dir, "event_GCS.log")
|
||||
)
|
||||
|
||||
def _check_events():
|
||||
try:
|
||||
resp = requests.get(f"{webui_url}/events")
|
||||
resp.raise_for_status()
|
||||
result = resp.json()
|
||||
all_events = result["data"]["events"]
|
||||
assert (
|
||||
len(all_events[job_id]) >= event_consts.EVENT_READ_LINE_COUNT_LIMIT + 10
|
||||
)
|
||||
messages = [e["message"] for e in all_events[job_id]]
|
||||
for i in range(10):
|
||||
assert str(i) * message_len in messages
|
||||
assert "2" * (message_len + 1) not in messages
|
||||
assert str(event_consts.EVENT_READ_LINE_COUNT_LIMIT - 1) in messages
|
||||
return True
|
||||
except Exception as ex:
|
||||
logger.exception(ex)
|
||||
return False
|
||||
|
||||
wait_for_condition(_check_events, timeout=15)
|
||||
|
||||
|
||||
def test_report_events(ray_start_with_dashboard):
|
||||
assert wait_until_server_available(ray_start_with_dashboard["webui_url"])
|
||||
webui_url = format_web_url(ray_start_with_dashboard["webui_url"])
|
||||
url = f"{webui_url}/report_events"
|
||||
|
||||
resp = requests.post(url)
|
||||
assert resp.status_code == 400
|
||||
resp = requests.post(url, json={"Hello": "World"})
|
||||
assert resp.status_code == 400
|
||||
|
||||
job_id = ray.JobID.from_int(100).hex()
|
||||
sample_event = _get_event("Hello", job_id=job_id)
|
||||
resp = requests.post(url, json=[json.dumps(sample_event)])
|
||||
assert resp.status_code == 200
|
||||
|
||||
resp = requests.get(f"{webui_url}/events")
|
||||
assert resp.status_code == 200
|
||||
result = resp.json()
|
||||
all_events = result["data"]["events"]
|
||||
assert len(all_events) == 1
|
||||
assert job_id in all_events
|
||||
assert len(all_events[job_id]) == 1
|
||||
assert all_events[job_id][0]["message"] == "Hello"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_monitor_events():
|
||||
with tempfile.TemporaryDirectory() as temp_dir:
|
||||
common = event_pb2.Event.SourceType.Name(event_pb2.Event.COMMON)
|
||||
common_log = os.path.join(temp_dir, f"event_{common}.log")
|
||||
test_logger = _test_logger(
|
||||
__name__ + str(random.random()), common_log, max_bytes=10, backup_count=0
|
||||
)
|
||||
test_events1 = []
|
||||
monitor_task = monitor_events(
|
||||
temp_dir, lambda x: test_events1.extend(x), None, scan_interval_seconds=0.01
|
||||
)
|
||||
assert not monitor_task.done()
|
||||
count = 10
|
||||
|
||||
async def _writer(*args, read_events, spin=True):
|
||||
for x in range(*args):
|
||||
test_logger.info("%s", x)
|
||||
if spin:
|
||||
while str(x) not in read_events:
|
||||
await asyncio.sleep(0.01)
|
||||
|
||||
async def _check_events(expect_events, read_events, timeout=10):
|
||||
start_time = time.time()
|
||||
while True:
|
||||
sorted_events = sorted(int(i) for i in read_events)
|
||||
sorted_events = [str(i) for i in sorted_events]
|
||||
if time.time() - start_time > timeout:
|
||||
raise TimeoutError(
|
||||
f"Timeout, read events: {sorted_events}, "
|
||||
f"expect events: {expect_events}"
|
||||
)
|
||||
if len(sorted_events) == len(expect_events):
|
||||
if sorted_events == expect_events:
|
||||
break
|
||||
await asyncio.sleep(1)
|
||||
|
||||
await asyncio.gather(
|
||||
_writer(count, read_events=test_events1),
|
||||
_check_events([str(i) for i in range(count)], read_events=test_events1),
|
||||
)
|
||||
|
||||
monitor_task.cancel()
|
||||
test_events2 = []
|
||||
monitor_task = monitor_events(
|
||||
temp_dir, lambda x: test_events2.extend(x), None, scan_interval_seconds=0.1
|
||||
)
|
||||
|
||||
await _check_events([str(i) for i in range(count)], read_events=test_events2)
|
||||
|
||||
await _writer(count, count * 2, read_events=test_events2)
|
||||
await _check_events(
|
||||
[str(i) for i in range(count * 2)], read_events=test_events2
|
||||
)
|
||||
|
||||
log_file_count = len(os.listdir(temp_dir))
|
||||
|
||||
test_logger = _test_logger(
|
||||
__name__ + str(random.random()), common_log, max_bytes=1000, backup_count=0
|
||||
)
|
||||
assert len(os.listdir(temp_dir)) == log_file_count
|
||||
|
||||
await _writer(count * 2, count * 3, spin=False, read_events=test_events2)
|
||||
await _check_events(
|
||||
[str(i) for i in range(count * 3)], read_events=test_events2
|
||||
)
|
||||
await _writer(count * 3, count * 4, spin=False, read_events=test_events2)
|
||||
await _check_events(
|
||||
[str(i) for i in range(count * 4)], read_events=test_events2
|
||||
)
|
||||
|
||||
# Test cancel monitor task.
|
||||
monitor_task.cancel()
|
||||
with pytest.raises(asyncio.CancelledError):
|
||||
await monitor_task
|
||||
assert monitor_task.done()
|
||||
|
||||
assert len(os.listdir(temp_dir)) == 1, "There should just be 1 event log"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("autoscaler_v2", [False, True], ids=["v1", "v2"])
|
||||
def test_autoscaler_cluster_events(autoscaler_v2, shutdown_only):
|
||||
cluster = AutoscalingCluster(
|
||||
head_resources={"CPU": 2},
|
||||
worker_node_types={
|
||||
"cpu_node": {
|
||||
"resources": {
|
||||
"CPU": 4,
|
||||
},
|
||||
"node_config": {},
|
||||
"min_workers": 0,
|
||||
"max_workers": 1,
|
||||
},
|
||||
"gpu_node": {
|
||||
"resources": {
|
||||
"CPU": 2,
|
||||
"GPU": 1,
|
||||
},
|
||||
"node_config": {},
|
||||
"min_workers": 0,
|
||||
"max_workers": 1,
|
||||
},
|
||||
},
|
||||
autoscaler_v2=autoscaler_v2,
|
||||
idle_timeout_minutes=1,
|
||||
)
|
||||
|
||||
try:
|
||||
cluster.start()
|
||||
ray.init("auto")
|
||||
|
||||
# Triggers the addition of a GPU node.
|
||||
@ray.remote(num_gpus=1)
|
||||
def f():
|
||||
print("gpu ok")
|
||||
|
||||
# Triggers the addition of a CPU node.
|
||||
@ray.remote(num_cpus=3)
|
||||
def g():
|
||||
print("cpu ok")
|
||||
|
||||
wait_for_condition(lambda: ray.cluster_resources()["CPU"] == 2)
|
||||
ray.get(f.remote())
|
||||
wait_for_condition(lambda: ray.cluster_resources()["CPU"] == 4)
|
||||
wait_for_condition(lambda: ray.cluster_resources()["GPU"] == 1)
|
||||
ray.get(g.remote())
|
||||
wait_for_condition(lambda: ray.cluster_resources()["CPU"] == 8)
|
||||
wait_for_condition(lambda: ray.cluster_resources()["GPU"] == 1)
|
||||
|
||||
# Trigger an infeasible task
|
||||
g.options(num_cpus=0, num_gpus=5).remote()
|
||||
|
||||
def verify():
|
||||
cluster_events = list_cluster_events()
|
||||
print(cluster_events)
|
||||
messages = {(e["message"], e["source_type"]) for e in cluster_events}
|
||||
if not autoscaler_v2:
|
||||
# With head node resources, we don't actually resized. So this event is
|
||||
# not really accurate.
|
||||
assert ("Resized to 2 CPUs.", "AUTOSCALER") in messages, cluster_events
|
||||
assert (
|
||||
"Adding 1 node(s) of type gpu_node.",
|
||||
"AUTOSCALER",
|
||||
) in messages, cluster_events
|
||||
assert (
|
||||
"Resized to 4 CPUs, 1 GPUs.",
|
||||
"AUTOSCALER",
|
||||
) in messages, cluster_events
|
||||
assert (
|
||||
"Adding 1 node(s) of type cpu_node.",
|
||||
"AUTOSCALER",
|
||||
) in messages, cluster_events
|
||||
assert (
|
||||
"Resized to 8 CPUs, 1 GPUs.",
|
||||
"AUTOSCALER",
|
||||
) in messages, cluster_events
|
||||
assert "No available node types can fulfill resource request" in "".join(
|
||||
[t[0] for t in messages]
|
||||
)
|
||||
|
||||
return True
|
||||
|
||||
wait_for_condition(verify, timeout=30)
|
||||
pprint(list_cluster_events())
|
||||
finally:
|
||||
ray.shutdown()
|
||||
cluster.shutdown()
|
||||
|
||||
|
||||
def test_filter_event_by_level(monkeypatch):
|
||||
def gen_event(level: str):
|
||||
return event_pb2.Event(
|
||||
source_type=event_pb2.Event.AUTOSCALER,
|
||||
severity=event_pb2.Event.Severity.Value(level),
|
||||
message=level,
|
||||
)
|
||||
|
||||
trace = gen_event("TRACE")
|
||||
debug = gen_event("DEBUG")
|
||||
info = gen_event("INFO")
|
||||
warning = gen_event("WARNING")
|
||||
error = gen_event("ERROR")
|
||||
fatal = gen_event("FATAL")
|
||||
|
||||
def assert_events_filtered(events, expected, filter_level):
|
||||
filtered = [e for e in events if filter_event_by_level(e, filter_level)]
|
||||
print(filtered)
|
||||
assert len(filtered) == len(expected)
|
||||
assert {e.message for e in filtered} == {e.message for e in expected}
|
||||
|
||||
events = [trace, debug, info, warning, error, fatal]
|
||||
assert_events_filtered(events, [], "TRACE")
|
||||
assert_events_filtered(events, [trace], "DEBUG")
|
||||
assert_events_filtered(events, [trace, debug], "INFO")
|
||||
assert_events_filtered(events, [trace, debug, info], "WARNING")
|
||||
assert_events_filtered(events, [trace, debug, info, warning], "ERROR")
|
||||
assert_events_filtered(events, [trace, debug, info, warning, error], "FATAL")
|
||||
|
||||
|
||||
def test_jobs_cluster_events(shutdown_only):
|
||||
ray.init()
|
||||
address = ray._private.worker._global_node.webui_url
|
||||
address = format_web_url(address)
|
||||
client = JobSubmissionClient(address)
|
||||
submission_id = client.submit_job(entrypoint="ls")
|
||||
|
||||
def verify():
|
||||
events = list_cluster_events()
|
||||
assert len(list_cluster_events()) == 2
|
||||
start_event = events[0]
|
||||
completed_event = events[1]
|
||||
|
||||
assert start_event["source_type"] == "JOBS"
|
||||
assert f"Started a ray job {submission_id}" in start_event["message"]
|
||||
assert start_event["severity"] == "INFO"
|
||||
assert completed_event["source_type"] == "JOBS"
|
||||
assert (
|
||||
f"Completed a ray job {submission_id} with a status SUCCEEDED."
|
||||
== completed_event["message"]
|
||||
)
|
||||
assert completed_event["severity"] == "INFO"
|
||||
return True
|
||||
|
||||
print("Test successful job run.")
|
||||
wait_for_condition(verify)
|
||||
pprint(list_cluster_events())
|
||||
|
||||
# Test the failure case. In this part, job fails because the runtime env
|
||||
# creation fails.
|
||||
submission_id = client.submit_job(
|
||||
entrypoint="ls",
|
||||
runtime_env={"pip": ["nonexistent_dep"]},
|
||||
)
|
||||
|
||||
def verify():
|
||||
events = list_cluster_events(detail=True)
|
||||
failed_events = []
|
||||
|
||||
for e in events:
|
||||
if (
|
||||
"submission_id" in e["custom_fields"]
|
||||
and e["custom_fields"]["submission_id"] == submission_id
|
||||
):
|
||||
failed_events.append(e)
|
||||
|
||||
assert len(failed_events) == 2
|
||||
failed_start = failed_events[0]
|
||||
failed_completed = failed_events[1]
|
||||
|
||||
assert failed_start["source_type"] == "JOBS"
|
||||
assert f"Started a ray job {submission_id}" in failed_start["message"]
|
||||
assert failed_completed["source_type"] == "JOBS"
|
||||
assert failed_completed["severity"] == "ERROR"
|
||||
assert (
|
||||
f"Completed a ray job {submission_id} with a status FAILED."
|
||||
in failed_completed["message"]
|
||||
)
|
||||
|
||||
# Make sure the error message is included.
|
||||
assert "ERROR: No matching distribution found" in failed_completed["message"]
|
||||
return True
|
||||
|
||||
print("Test failed (runtime_env failure) job run.")
|
||||
wait_for_condition(verify, timeout=30)
|
||||
pprint(list_cluster_events())
|
||||
|
||||
|
||||
def test_core_events(shutdown_only):
|
||||
# Test events recorded from core RAY_EVENT APIs.
|
||||
ray.init()
|
||||
|
||||
@ray.remote
|
||||
class Actor:
|
||||
def getpid(self):
|
||||
return os.getpid()
|
||||
|
||||
a = Actor.remote()
|
||||
pid = ray.get(a.getpid.remote())
|
||||
os.kill(pid, 9)
|
||||
s = time.time()
|
||||
|
||||
def verify():
|
||||
events = list_cluster_events(filters=[("source_type", "=", "RAYLET")])
|
||||
print(events)
|
||||
assert len(list_cluster_events()) == 1
|
||||
event = events[0]
|
||||
assert event["severity"] == "ERROR"
|
||||
datetime_str = event["time"]
|
||||
datetime_obj = datetime.strptime(datetime_str, "%Y-%m-%d %H:%M:%S")
|
||||
timestamp = time.mktime(datetime_obj.timetuple())
|
||||
|
||||
# Make sure timestamp is not incorrect. Add sufficient buffer (60 seconds)
|
||||
assert abs(timestamp - s) < 60
|
||||
assert (
|
||||
"A worker died or was killed while executing "
|
||||
"a task by an unexpected system error" in event["message"]
|
||||
)
|
||||
return True
|
||||
|
||||
wait_for_condition(verify)
|
||||
pprint(list_cluster_events())
|
||||
|
||||
|
||||
def test_cluster_events_retention(monkeypatch, shutdown_only):
|
||||
with monkeypatch.context() as m:
|
||||
# defer for 5s for the second node.
|
||||
# This will help the API not return until the node is killed.
|
||||
m.setenv("RAY_DASHBOARD_MAX_EVENTS_TO_CACHE", "10")
|
||||
ray.init()
|
||||
address = ray._private.worker._global_node.webui_url
|
||||
address = format_web_url(address)
|
||||
client = JobSubmissionClient(address)
|
||||
|
||||
submission_ids = []
|
||||
for _ in range(12):
|
||||
submission_ids.append(client.submit_job(entrypoint="ls"))
|
||||
print(submission_ids)
|
||||
|
||||
def verify():
|
||||
events = list_cluster_events()
|
||||
assert len(list_cluster_events()) == 10
|
||||
|
||||
messages = [event["message"] for event in events]
|
||||
|
||||
# Make sure the first two has been GC'ed.
|
||||
for m in messages:
|
||||
assert submission_ids[0] not in m
|
||||
assert submission_ids[1] not in m
|
||||
return True
|
||||
|
||||
wait_for_condition(verify)
|
||||
pprint(list_cluster_events())
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_list_cluster_events_impl():
|
||||
executor = ThreadPoolExecutor(
|
||||
max_workers=1,
|
||||
thread_name_prefix="event_head_executor",
|
||||
)
|
||||
|
||||
event_id_1 = get_event_id()
|
||||
event_id_2 = get_event_id()
|
||||
events = {
|
||||
"job_1": {
|
||||
event_id_1: {
|
||||
"timestamp": 10,
|
||||
"severity": "DEBUG",
|
||||
"message": "a",
|
||||
"event_id": event_id_1,
|
||||
"source_type": "GCS",
|
||||
},
|
||||
event_id_2: {
|
||||
"timestamp": 10,
|
||||
"severity": "INFO",
|
||||
"message": "b",
|
||||
"event_id": event_id_2,
|
||||
"source_type": "GCS",
|
||||
},
|
||||
}
|
||||
}
|
||||
result = await _list_cluster_events_impl(
|
||||
all_events=events, executor=executor, option=create_api_options()
|
||||
)
|
||||
data = result.result
|
||||
data = data[0]
|
||||
verify_schema(ClusterEventState, data)
|
||||
assert result.total == 2
|
||||
|
||||
"""
|
||||
Test detail
|
||||
"""
|
||||
# TODO(sang)
|
||||
|
||||
"""
|
||||
Test limit
|
||||
"""
|
||||
assert len(result.result) == 2
|
||||
result = await _list_cluster_events_impl(
|
||||
all_events=events, executor=executor, option=create_api_options(limit=1)
|
||||
)
|
||||
data = result.result
|
||||
assert len(data) == 1
|
||||
assert result.total == 2
|
||||
|
||||
"""
|
||||
Test filters
|
||||
"""
|
||||
# If the column is not supported for filtering, it should raise an exception.
|
||||
with pytest.raises(ValueError):
|
||||
result = await _list_cluster_events_impl(
|
||||
all_events=events,
|
||||
executor=executor,
|
||||
option=create_api_options(filters=[("time", "=", "20")]),
|
||||
)
|
||||
result = await _list_cluster_events_impl(
|
||||
all_events=events,
|
||||
executor=executor,
|
||||
option=create_api_options(filters=[("severity", "=", "INFO")]),
|
||||
)
|
||||
assert len(result.result) == 1
|
||||
|
||||
|
||||
def test_export_event_logger(tmp_path):
|
||||
"""
|
||||
Unit test a mock export event of type ExportSubmissionJobEventData
|
||||
is correctly written to file. This doesn't events are correctly generated.
|
||||
"""
|
||||
logger = get_export_event_logger(EventLogType.SUBMISSION_JOB, str(tmp_path))
|
||||
ExportSubmissionJobEventData = (
|
||||
export_submission_job_event_pb2.ExportSubmissionJobEventData
|
||||
)
|
||||
event_data = ExportSubmissionJobEventData(
|
||||
submission_job_id="submission_job_id0",
|
||||
status=ExportSubmissionJobEventData.JobStatus.RUNNING,
|
||||
entrypoint="ls",
|
||||
metadata={},
|
||||
)
|
||||
logger.send_event(event_data)
|
||||
|
||||
event_dir = tmp_path / "export_events"
|
||||
assert event_dir.exists()
|
||||
event_file = event_dir / "event_EXPORT_SUBMISSION_JOB.log"
|
||||
assert event_file.exists()
|
||||
|
||||
with event_file.open() as f:
|
||||
lines = f.readlines()
|
||||
assert len(lines) == 1
|
||||
|
||||
line = lines[0]
|
||||
data = json.loads(line)
|
||||
assert data["source_type"] == "EXPORT_SUBMISSION_JOB"
|
||||
assert data["event_data"] == message_to_dict(
|
||||
event_data,
|
||||
always_print_fields_with_no_presence=True,
|
||||
preserving_proto_field_name=True,
|
||||
use_integers_for_enums=False,
|
||||
)
|
||||
|
||||
|
||||
def test_event_logger_flushes_all_handlers():
|
||||
mock_logger = MagicMock()
|
||||
handlers = [MagicMock() for _ in range(3)]
|
||||
mock_logger.handlers = handlers
|
||||
|
||||
adapter = EventLoggerAdapter(event_pb2.Event.GCS, mock_logger)
|
||||
adapter.info("message")
|
||||
|
||||
for handler in handlers:
|
||||
handler.flush.assert_called_once()
|
||||
|
||||
|
||||
def test_event_logger_allows_empty_handlers():
|
||||
mock_logger = MagicMock()
|
||||
mock_logger.handlers = []
|
||||
|
||||
adapter = EventLoggerAdapter(event_pb2.Event.GCS, mock_logger)
|
||||
adapter.info("message")
|
||||
|
||||
|
||||
def test_export_event_logger_flushes_all_handlers():
|
||||
mock_logger = MagicMock()
|
||||
handlers = [MagicMock() for _ in range(3)]
|
||||
mock_logger.handlers = handlers
|
||||
|
||||
adapter = ExportEventLoggerAdapter(EventLogType.SUBMISSION_JOB, mock_logger)
|
||||
event_data = export_submission_job_event_pb2.ExportSubmissionJobEventData(
|
||||
submission_job_id="submission_job_id0",
|
||||
status=(
|
||||
export_submission_job_event_pb2.ExportSubmissionJobEventData.JobStatus.RUNNING
|
||||
),
|
||||
entrypoint="ls",
|
||||
metadata={},
|
||||
)
|
||||
adapter.send_event(event_data)
|
||||
|
||||
for handler in handlers:
|
||||
handler.flush.assert_called_once()
|
||||
|
||||
|
||||
def test_export_event_logger_allows_empty_handlers():
|
||||
mock_logger = MagicMock()
|
||||
mock_logger.handlers = []
|
||||
|
||||
adapter = ExportEventLoggerAdapter(EventLogType.SUBMISSION_JOB, mock_logger)
|
||||
event_data = export_submission_job_event_pb2.ExportSubmissionJobEventData(
|
||||
submission_job_id="submission_job_id0",
|
||||
status=(
|
||||
export_submission_job_event_pb2.ExportSubmissionJobEventData.JobStatus.RUNNING
|
||||
),
|
||||
entrypoint="ls",
|
||||
metadata={},
|
||||
)
|
||||
adapter.send_event(event_data)
|
||||
|
||||
|
||||
def test_export_event_logger_continues_flushing_after_handler_error():
|
||||
mock_logger = MagicMock()
|
||||
handler1 = MagicMock()
|
||||
handler1.flush.side_effect = RuntimeError("flush failed")
|
||||
handler2 = MagicMock()
|
||||
mock_logger.handlers = [handler1, handler2]
|
||||
|
||||
adapter = ExportEventLoggerAdapter(EventLogType.SUBMISSION_JOB, mock_logger)
|
||||
event_data = export_submission_job_event_pb2.ExportSubmissionJobEventData(
|
||||
submission_job_id="submission_job_id0",
|
||||
status=(
|
||||
export_submission_job_event_pb2.ExportSubmissionJobEventData.JobStatus.RUNNING
|
||||
),
|
||||
entrypoint="ls",
|
||||
metadata={},
|
||||
)
|
||||
adapter.send_event(event_data)
|
||||
|
||||
handler2.flush.assert_called_once()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(pytest.main(["-v", __file__]))
|
||||
@@ -0,0 +1,66 @@
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
|
||||
import pytest
|
||||
|
||||
import ray
|
||||
from ray._common.test_utils import wait_for_condition
|
||||
from ray._private.test_utils import wait_until_server_available
|
||||
from ray.dashboard.tests.conftest import * # noqa
|
||||
|
||||
os.environ["RAY_enable_export_api_write"] = "1"
|
||||
os.environ["RAY_enable_core_worker_ray_event_to_aggregator"] = "0"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_task_labels(disable_aiohttp_cache, ray_start_with_dashboard):
|
||||
"""
|
||||
Test task events are correctly generated and written to file
|
||||
"""
|
||||
assert wait_until_server_available(ray_start_with_dashboard["webui_url"])
|
||||
export_event_path = os.path.join(
|
||||
ray_start_with_dashboard["session_dir"], "logs", "export_events"
|
||||
)
|
||||
|
||||
# A simple task to trigger the export event
|
||||
@ray.remote
|
||||
def hi_w00t_task():
|
||||
return 1
|
||||
|
||||
ray.get(hi_w00t_task.options(_labels={"hi": "w00t"}).remote())
|
||||
|
||||
def _verify():
|
||||
# Verify export events are written
|
||||
events = []
|
||||
for filename in os.listdir(export_event_path):
|
||||
if not filename.startswith("event_EXPORT_TASK"):
|
||||
continue
|
||||
with open(f"{export_event_path}/{filename}", "r") as f:
|
||||
for line in f.readlines():
|
||||
events.append(json.loads(line))
|
||||
|
||||
hi_w00t_event = next(
|
||||
(
|
||||
event
|
||||
for event in events
|
||||
if event["source_type"] == "EXPORT_TASK"
|
||||
and event["event_data"].get("task_info", {}).get("func_or_class_name")
|
||||
== "hi_w00t_task"
|
||||
),
|
||||
None,
|
||||
)
|
||||
return (
|
||||
hi_w00t_event is not None
|
||||
and hi_w00t_event["event_data"]
|
||||
.get("task_info", {})
|
||||
.get("labels", {})
|
||||
.get("hi")
|
||||
== "w00t"
|
||||
)
|
||||
|
||||
wait_for_condition(_verify, timeout=30)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(pytest.main(["-v", __file__]))
|
||||
@@ -0,0 +1,197 @@
|
||||
# isort: skip_file
|
||||
# ruff: noqa: E402
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
|
||||
import pytest
|
||||
|
||||
# RAY_enable_export_api_write_config env var must be set before importing
|
||||
# `ray` so the correct value is set for RAY_ENABLE_EXPORT_API_WRITE_CONFIG
|
||||
# even outside a Ray driver.
|
||||
os.environ["RAY_enable_export_api_write_config"] = "EXPORT_SUBMISSION_JOB"
|
||||
|
||||
import ray
|
||||
from ray._common.test_utils import async_wait_for_condition
|
||||
from ray.dashboard.modules.job.job_manager import JobManager
|
||||
from ray.job_submission import JobStatus
|
||||
from ray.tests.conftest import call_ray_start # noqa: F401
|
||||
|
||||
|
||||
async def check_job_succeeded(job_manager, job_id):
|
||||
data = await job_manager.get_job_info(job_id)
|
||||
status = data.status
|
||||
if status == JobStatus.FAILED:
|
||||
raise RuntimeError(f"Job failed! {data.message}")
|
||||
assert status in {JobStatus.PENDING, JobStatus.RUNNING, JobStatus.SUCCEEDED}
|
||||
if status == JobStatus.SUCCEEDED:
|
||||
assert data.driver_exit_code == 0
|
||||
else:
|
||||
assert data.driver_exit_code is None
|
||||
return status == JobStatus.SUCCEEDED
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
"call_ray_start",
|
||||
[
|
||||
{
|
||||
"env": {
|
||||
"RAY_enable_export_api_write_config": "EXPORT_SUBMISSION_JOB,EXPORT_TASK",
|
||||
},
|
||||
"cmd": "ray start --head",
|
||||
}
|
||||
],
|
||||
indirect=True,
|
||||
)
|
||||
async def test_check_export_api_enabled(call_ray_start, tmp_path): # noqa: F811
|
||||
"""
|
||||
Test check_export_api_enabled is True for EXPORT_SUBMISSION_JOB and EXPORT_TASK but
|
||||
not for EXPORT_ACTOR because of the value of RAY_enable_export_api_write_config.
|
||||
"""
|
||||
|
||||
@ray.remote
|
||||
def test_check_export_api_enabled_remote():
|
||||
from ray._private.event.export_event_logger import check_export_api_enabled
|
||||
from ray.core.generated.export_event_pb2 import ExportEvent
|
||||
|
||||
success = True
|
||||
success = success and check_export_api_enabled(
|
||||
ExportEvent.SourceType.EXPORT_SUBMISSION_JOB
|
||||
)
|
||||
success = success and check_export_api_enabled(
|
||||
ExportEvent.SourceType.EXPORT_TASK
|
||||
)
|
||||
success = success and (
|
||||
not check_export_api_enabled(ExportEvent.SourceType.EXPORT_ACTOR)
|
||||
)
|
||||
return success
|
||||
|
||||
assert ray.get(test_check_export_api_enabled_remote.remote())
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
"call_ray_start",
|
||||
[
|
||||
{
|
||||
"env": {
|
||||
"RAY_enable_export_api_write": "true",
|
||||
},
|
||||
"cmd": "ray start --head",
|
||||
}
|
||||
],
|
||||
indirect=True,
|
||||
)
|
||||
async def test_check_export_api_enabled_global(call_ray_start, tmp_path): # noqa: F811
|
||||
"""
|
||||
Test check_export_api_enabled always returns True because RAY_enable_export_api_write
|
||||
is set to True.
|
||||
"""
|
||||
|
||||
@ray.remote
|
||||
def test_check_export_api_enabled_remote():
|
||||
from ray._private.event.export_event_logger import check_export_api_enabled
|
||||
from ray.core.generated.export_event_pb2 import ExportEvent
|
||||
|
||||
success = True
|
||||
success = success and check_export_api_enabled(
|
||||
ExportEvent.SourceType.EXPORT_SUBMISSION_JOB
|
||||
)
|
||||
success = success and check_export_api_enabled(
|
||||
ExportEvent.SourceType.EXPORT_ACTOR
|
||||
)
|
||||
return success
|
||||
|
||||
assert ray.get(test_check_export_api_enabled_remote.remote())
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
"call_ray_start",
|
||||
[
|
||||
{
|
||||
"env": {
|
||||
"RAY_enable_export_api_write_config": "invalid source type",
|
||||
},
|
||||
"cmd": "ray start --head",
|
||||
}
|
||||
],
|
||||
indirect=True,
|
||||
)
|
||||
async def test_check_export_api_empty_config(call_ray_start, tmp_path): # noqa: F811
|
||||
"""
|
||||
Test check_export_api_enabled is False for all sources because
|
||||
RAY_enable_export_api_write_config is not a vaild source type.
|
||||
"""
|
||||
|
||||
@ray.remote
|
||||
def test_check_export_api_enabled_remote():
|
||||
from ray._private.event.export_event_logger import check_export_api_enabled
|
||||
from ray.core.generated.export_event_pb2 import ExportEvent
|
||||
|
||||
success = True
|
||||
success = success and not (
|
||||
check_export_api_enabled(ExportEvent.SourceType.EXPORT_SUBMISSION_JOB)
|
||||
)
|
||||
success = success and (
|
||||
not check_export_api_enabled(ExportEvent.SourceType.EXPORT_ACTOR)
|
||||
)
|
||||
return success
|
||||
|
||||
assert ray.get(test_check_export_api_enabled_remote.remote())
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
"call_ray_start",
|
||||
[
|
||||
{
|
||||
"env": {
|
||||
"RAY_enable_export_api_write_config": "EXPORT_SUBMISSION_JOB",
|
||||
},
|
||||
"cmd": "ray start --head",
|
||||
}
|
||||
],
|
||||
indirect=True,
|
||||
)
|
||||
async def test_submission_job_export_events(call_ray_start, tmp_path): # noqa: F811
|
||||
"""
|
||||
Test submission job events are correctly generated and written to file
|
||||
as the job goes through various state changes in its lifecycle.
|
||||
"""
|
||||
|
||||
ray.init(address=call_ray_start)
|
||||
gcs_client = ray._private.worker.global_worker.gcs_client
|
||||
job_manager = JobManager(gcs_client, tmp_path)
|
||||
|
||||
# Submit a job.
|
||||
submission_id = await job_manager.submit_job(
|
||||
entrypoint="ls",
|
||||
)
|
||||
|
||||
# Wait for the job to be finished.
|
||||
await async_wait_for_condition(
|
||||
check_job_succeeded, job_manager=job_manager, job_id=submission_id
|
||||
)
|
||||
|
||||
# Verify export events are written
|
||||
event_dir = f"{tmp_path}/export_events"
|
||||
assert os.path.isdir(event_dir)
|
||||
event_file = f"{event_dir}/event_EXPORT_SUBMISSION_JOB.log"
|
||||
assert os.path.isfile(event_file)
|
||||
|
||||
with open(event_file, "r") as f:
|
||||
lines = f.readlines()
|
||||
assert len(lines) == 3
|
||||
expected_status_values = ["PENDING", "RUNNING", "SUCCEEDED"]
|
||||
|
||||
for line, expected_status in zip(lines, expected_status_values):
|
||||
data = json.loads(line)
|
||||
assert data["source_type"] == "EXPORT_SUBMISSION_JOB"
|
||||
assert data["event_data"]["submission_job_id"] == submission_id
|
||||
assert data["event_data"]["status"] == expected_status
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(pytest.main(["-v", __file__]))
|
||||
Reference in New Issue
Block a user