chore: import upstream snapshot with attribution
This commit is contained in:
wehub-resource-sync
@@ -0,0 +1,405 @@
|
||||
import asyncio
|
||||
import concurrent.futures
|
||||
import io
|
||||
import logging
|
||||
import os
|
||||
from pathlib import Path
|
||||
from typing import AsyncIterator, Optional
|
||||
|
||||
import grpc
|
||||
|
||||
import ray.dashboard.modules.log.log_consts as log_consts
|
||||
import ray.dashboard.modules.log.log_utils as log_utils
|
||||
import ray.dashboard.optional_utils as dashboard_optional_utils
|
||||
import ray.dashboard.utils as dashboard_utils
|
||||
from ray._private.ray_constants import env_integer
|
||||
from ray.core.generated import reporter_pb2, reporter_pb2_grpc
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
routes = dashboard_optional_utils.DashboardAgentRouteTable
|
||||
|
||||
# 64 KB
|
||||
BLOCK_SIZE = 1 << 16
|
||||
|
||||
# Keep-alive interval for reading the file
|
||||
DEFAULT_KEEP_ALIVE_INTERVAL_SEC = 1
|
||||
|
||||
RAY_DASHBOARD_LOG_TASK_LOG_SEARCH_MAX_WORKER_COUNT = env_integer(
|
||||
"RAY_DASHBOARD_LOG_TASK_LOG_SEARCH_MAX_WORKER_COUNT", default=2
|
||||
)
|
||||
|
||||
|
||||
def find_offset_of_content_in_file(
|
||||
file: io.BufferedIOBase, content: bytes, start_offset: int = 0
|
||||
) -> int:
|
||||
"""Find the offset of the first occurrence of content in a file.
|
||||
|
||||
Args:
|
||||
file: File object
|
||||
content: Content to find
|
||||
start_offset: Start offset to read from, inclusive.
|
||||
|
||||
Returns:
|
||||
Offset of the first occurrence of content in a file.
|
||||
"""
|
||||
logger.debug(f"Finding offset of content {content} in file")
|
||||
file.seek(start_offset, io.SEEK_SET) # move file pointer to start of file
|
||||
offset = start_offset
|
||||
while True:
|
||||
# Read in block
|
||||
block_data = file.read(BLOCK_SIZE)
|
||||
if block_data == b"":
|
||||
# Stop reading
|
||||
return -1
|
||||
# Find the offset of the first occurrence of content in the block
|
||||
block_offset = block_data.find(content)
|
||||
if block_offset != -1:
|
||||
# Found the offset in the block
|
||||
return offset + block_offset
|
||||
# Continue reading
|
||||
offset += len(block_data)
|
||||
|
||||
|
||||
def find_end_offset_file(file: io.BufferedIOBase) -> int:
|
||||
"""
|
||||
Find the offset of the end of a file without changing the file pointer.
|
||||
|
||||
Args:
|
||||
file: File object
|
||||
|
||||
Returns:
|
||||
Offset of the end of a file.
|
||||
"""
|
||||
old_pos = file.tell() # store old position
|
||||
file.seek(0, io.SEEK_END) # move file pointer to end of file
|
||||
end = file.tell() # return end of file offset
|
||||
file.seek(old_pos, io.SEEK_SET)
|
||||
return end
|
||||
|
||||
|
||||
def find_end_offset_next_n_lines_from_offset(
|
||||
file: io.BufferedIOBase, start_offset: int, n: int
|
||||
) -> int:
|
||||
"""
|
||||
Find the offsets of next n lines from a start offset.
|
||||
|
||||
Args:
|
||||
file: File object
|
||||
start_offset: Start offset to read from, inclusive.
|
||||
n: Number of lines to find.
|
||||
|
||||
Returns:
|
||||
Offset of the end of the next n line (exclusive)
|
||||
"""
|
||||
file.seek(start_offset) # move file pointer to start offset
|
||||
end_offset = None
|
||||
for _ in range(n): # loop until we find n lines or reach end of file
|
||||
line = file.readline() # read a line and consume new line character
|
||||
if not line: # end of file
|
||||
break
|
||||
end_offset = file.tell() # end offset.
|
||||
|
||||
logger.debug(f"Found next {n} lines from {start_offset} offset")
|
||||
return (
|
||||
end_offset if end_offset is not None else file.seek(0, io.SEEK_END)
|
||||
) # return last line offset or end of file offset if no lines found
|
||||
|
||||
|
||||
def find_start_offset_last_n_lines_from_offset(
|
||||
file: io.BufferedIOBase, offset: int, n: int, block_size: int = BLOCK_SIZE
|
||||
) -> int:
|
||||
"""
|
||||
Find the offset of the beginning of the line of the last X lines from an offset.
|
||||
|
||||
Args:
|
||||
file: File object
|
||||
offset: Start offset from which to find last X lines, -1 means end of file.
|
||||
The offset is exclusive, i.e. data at the offset is not included
|
||||
in the result.
|
||||
n: Number of lines to find
|
||||
block_size: Block size to read from file
|
||||
|
||||
Returns:
|
||||
Offset of the beginning of the line of the last X lines from a start offset.
|
||||
"""
|
||||
logger.debug(f"Finding last {n} lines from {offset} offset")
|
||||
if offset == -1:
|
||||
offset = file.seek(0, io.SEEK_END) # move file pointer to end of file
|
||||
else:
|
||||
file.seek(offset, io.SEEK_SET) # move file pointer to start offset
|
||||
|
||||
if n == 0:
|
||||
return offset
|
||||
nbytes_from_end = (
|
||||
0 # Number of bytes that should be tailed from the end of the file
|
||||
)
|
||||
# Non new line terminating offset, adjust the line count and treat the non-newline
|
||||
# terminated line as the last line. e.g. line 1\nline 2
|
||||
file.seek(max(0, offset - 1), os.SEEK_SET)
|
||||
if file.read(1) != b"\n":
|
||||
n -= 1
|
||||
|
||||
# Remaining number of lines to tail
|
||||
lines_more = n
|
||||
read_offset = max(0, offset - block_size)
|
||||
# So that we know how much to read on the last block (the block 0)
|
||||
prev_offset = offset
|
||||
|
||||
while lines_more >= 0 and read_offset >= 0:
|
||||
# Seek to the current block start
|
||||
file.seek(read_offset, 0)
|
||||
# Read the current block (or less than block) data
|
||||
block_data = file.read(min(block_size, prev_offset - read_offset))
|
||||
num_lines = block_data.count(b"\n")
|
||||
if num_lines > lines_more:
|
||||
# This is the last block to read.
|
||||
# Need to find the offset of exact number of lines to tail
|
||||
# in the block.
|
||||
# Use `split` here to split away the extra lines, i.e.
|
||||
# first `num_lines - lines_more` lines.
|
||||
lines = block_data.split(b"\n", num_lines - lines_more)
|
||||
# Added the len of those lines that at the end of the block.
|
||||
nbytes_from_end += len(lines[-1])
|
||||
break
|
||||
|
||||
# Need to read more blocks.
|
||||
lines_more -= num_lines
|
||||
nbytes_from_end += len(block_data)
|
||||
|
||||
if read_offset == 0:
|
||||
# We have read all blocks (since the start)
|
||||
break
|
||||
# Continuing with the previous block
|
||||
prev_offset = read_offset
|
||||
read_offset = max(0, read_offset - block_size)
|
||||
|
||||
offset_read_start = offset - nbytes_from_end
|
||||
assert (
|
||||
offset_read_start >= 0
|
||||
), f"Read start offset({offset_read_start}) should be non-negative"
|
||||
return offset_read_start
|
||||
|
||||
|
||||
async def _stream_log_in_chunk(
|
||||
context: grpc.aio.ServicerContext,
|
||||
file: io.BufferedIOBase,
|
||||
start_offset: int,
|
||||
end_offset: int = -1,
|
||||
keep_alive_interval_sec: int = -1,
|
||||
block_size: int = BLOCK_SIZE,
|
||||
) -> AsyncIterator[reporter_pb2.StreamLogReply]:
|
||||
"""Streaming log in chunk from start to end offset.
|
||||
|
||||
Stream binary file content in chunks from start offset to an end
|
||||
offset if provided, else to the end of the file.
|
||||
|
||||
Args:
|
||||
context: gRPC server side context
|
||||
file: Binary file to stream
|
||||
start_offset: File offset where streaming starts
|
||||
end_offset: If -1, implying streaming til the EOF.
|
||||
keep_alive_interval_sec: Duration for which streaming will be
|
||||
retried when reaching the file end, -1 means no retry.
|
||||
block_size: Number of bytes per chunk, exposed for testing
|
||||
|
||||
Yields:
|
||||
reporter_pb2.StreamLogReply: Successive chunks of the file contents,
|
||||
one per block.
|
||||
"""
|
||||
assert "b" in file.mode, "Only binary file is supported."
|
||||
assert not (
|
||||
keep_alive_interval_sec >= 0 and end_offset != -1
|
||||
), "Keep-alive is not allowed when specifying an end offset"
|
||||
|
||||
file.seek(start_offset, 0)
|
||||
cur_offset = start_offset
|
||||
|
||||
# Until gRPC is done
|
||||
while not context.done():
|
||||
# Read in block
|
||||
if end_offset != -1:
|
||||
to_read = min(end_offset - cur_offset, block_size)
|
||||
else:
|
||||
to_read = block_size
|
||||
|
||||
bytes = file.read(to_read)
|
||||
|
||||
if bytes == b"":
|
||||
# Stop reading
|
||||
if keep_alive_interval_sec >= 0:
|
||||
await asyncio.sleep(keep_alive_interval_sec)
|
||||
# Try reading again
|
||||
continue
|
||||
|
||||
# Have read the entire file, done
|
||||
break
|
||||
logger.debug(f"Sending {len(bytes)} bytes at {cur_offset}")
|
||||
yield reporter_pb2.StreamLogReply(data=bytes)
|
||||
|
||||
# Have read the requested section [start_offset, end_offset), done
|
||||
cur_offset += len(bytes)
|
||||
if end_offset != -1 and cur_offset >= end_offset:
|
||||
break
|
||||
|
||||
|
||||
class LogAgent(dashboard_utils.DashboardAgentModule):
|
||||
def __init__(self, dashboard_agent):
|
||||
super().__init__(dashboard_agent)
|
||||
log_utils.register_mimetypes()
|
||||
routes.static("/logs", self._dashboard_agent.log_dir, show_index=True)
|
||||
|
||||
async def run(self, server):
|
||||
pass
|
||||
|
||||
@staticmethod
|
||||
def is_minimal_module():
|
||||
return False
|
||||
|
||||
|
||||
_task_log_search_worker_pool = concurrent.futures.ThreadPoolExecutor(
|
||||
max_workers=RAY_DASHBOARD_LOG_TASK_LOG_SEARCH_MAX_WORKER_COUNT
|
||||
)
|
||||
|
||||
|
||||
class LogAgentV1Grpc(dashboard_utils.DashboardAgentModule):
|
||||
def __init__(self, dashboard_agent):
|
||||
super().__init__(dashboard_agent)
|
||||
|
||||
async def run(self, server):
|
||||
if server:
|
||||
reporter_pb2_grpc.add_LogServiceServicer_to_server(self, server)
|
||||
|
||||
@property
|
||||
def node_id(self) -> Optional[str]:
|
||||
return self._dashboard_agent.get_node_id()
|
||||
|
||||
@staticmethod
|
||||
def is_minimal_module():
|
||||
# Dashboard is only available with non-minimal install now.
|
||||
return False
|
||||
|
||||
async def ListLogs(self, request, context):
|
||||
"""
|
||||
Lists all files in the active Ray logs directory.
|
||||
|
||||
Part of `LogService` gRPC.
|
||||
|
||||
NOTE: These RPCs are used by state_head.py, not log_head.py
|
||||
"""
|
||||
path = Path(self._dashboard_agent.log_dir)
|
||||
if not path.exists():
|
||||
raise FileNotFoundError(
|
||||
f"Could not find log dir at path: {self._dashboard_agent.log_dir}"
|
||||
"It is unexpected. Please report an issue to Ray Github."
|
||||
)
|
||||
log_files = []
|
||||
for p in path.glob(request.glob_filter):
|
||||
log_files.append(str(p.relative_to(path)) + ("/" if p.is_dir() else ""))
|
||||
return reporter_pb2.ListLogsReply(log_files=log_files)
|
||||
|
||||
@classmethod
|
||||
def _resolve_filename(cls, root_log_dir: Path, filename: str) -> Path:
|
||||
"""
|
||||
Resolves the file path relative to the root log directory.
|
||||
|
||||
Args:
|
||||
root_log_dir: Root log directory.
|
||||
filename: File path relative to the root log directory.
|
||||
|
||||
Raises:
|
||||
FileNotFoundError: If the file path is invalid.
|
||||
|
||||
Returns:
|
||||
The absolute file path resolved from the root log directory.
|
||||
"""
|
||||
if not Path(filename).is_absolute():
|
||||
filepath = root_log_dir / filename
|
||||
else:
|
||||
filepath = Path(filename)
|
||||
|
||||
# We want to allow relative paths that include symlinks pointing outside of the
|
||||
# `root_log_dir`, so use `os.path.abspath` instead of `Path.resolve()` because
|
||||
# `os.path.abspath` does not resolve symlinks.
|
||||
filepath = Path(os.path.abspath(filepath))
|
||||
|
||||
if not filepath.is_file():
|
||||
raise FileNotFoundError(f"A file is not found at: {filepath}")
|
||||
|
||||
try:
|
||||
filepath.relative_to(root_log_dir)
|
||||
except ValueError as e:
|
||||
raise FileNotFoundError(f"{filepath} not in {root_log_dir}: {e}")
|
||||
|
||||
# Fully resolve the path before returning (including following symlinks).
|
||||
return filepath.resolve()
|
||||
|
||||
async def StreamLog(self, request, context):
|
||||
"""
|
||||
Streams the log in real time starting from `request.lines` number of lines from
|
||||
the end of the file if `request.keep_alive == True`. Else, it terminates the
|
||||
stream once there are no more bytes to read from the log file.
|
||||
|
||||
Part of `LogService` gRPC.
|
||||
|
||||
NOTE: These RPCs are used by state_head.py, not log_head.py
|
||||
"""
|
||||
# NOTE: If the client side connection is closed, this handler will
|
||||
# be automatically terminated.
|
||||
lines = request.lines if request.lines else 1000
|
||||
|
||||
try:
|
||||
filepath = self._resolve_filename(
|
||||
Path(self._dashboard_agent.log_dir), request.log_file_name
|
||||
)
|
||||
except FileNotFoundError as e:
|
||||
await context.send_initial_metadata([[log_consts.LOG_GRPC_ERROR, str(e)]])
|
||||
else:
|
||||
with open(filepath, "rb") as f:
|
||||
await context.send_initial_metadata([])
|
||||
|
||||
# Default stream entire file
|
||||
start_offset = (
|
||||
request.start_offset if request.HasField("start_offset") else 0
|
||||
)
|
||||
end_offset = (
|
||||
request.end_offset
|
||||
if request.HasField("end_offset")
|
||||
else find_end_offset_file(f)
|
||||
)
|
||||
|
||||
if lines != -1:
|
||||
# If specified tail line number, cap the start offset
|
||||
# with lines from the current end offset
|
||||
start_offset = max(
|
||||
find_start_offset_last_n_lines_from_offset(
|
||||
f, offset=end_offset, n=lines
|
||||
),
|
||||
start_offset,
|
||||
)
|
||||
|
||||
# If keep alive: following the log every 'interval'
|
||||
keep_alive_interval_sec = -1
|
||||
if request.keep_alive:
|
||||
keep_alive_interval_sec = (
|
||||
request.interval
|
||||
if request.interval
|
||||
else DEFAULT_KEEP_ALIVE_INTERVAL_SEC
|
||||
)
|
||||
|
||||
# When following (keep_alive), it will read beyond the end
|
||||
end_offset = -1
|
||||
|
||||
logger.info(
|
||||
f"Tailing logs from {start_offset} to {end_offset} for "
|
||||
f"lines={lines}, with keep_alive={keep_alive_interval_sec}"
|
||||
)
|
||||
|
||||
# Read and send the file data in chunk
|
||||
async for chunk_res in _stream_log_in_chunk(
|
||||
context=context,
|
||||
file=f,
|
||||
start_offset=start_offset,
|
||||
end_offset=end_offset,
|
||||
keep_alive_interval_sec=keep_alive_interval_sec,
|
||||
):
|
||||
yield chunk_res
|
||||
@@ -0,0 +1,8 @@
|
||||
MIME_TYPES = {
|
||||
"text/plain": [".err", ".out", ".log"],
|
||||
}
|
||||
|
||||
LOG_GRPC_ERROR = "log_grpc_status"
|
||||
|
||||
# 10 seconds
|
||||
GRPC_TIMEOUT = 10
|
||||
@@ -0,0 +1,478 @@
|
||||
import logging
|
||||
import re
|
||||
from collections import defaultdict
|
||||
from typing import AsyncIterable, Awaitable, Callable, Dict, List, Optional, Tuple
|
||||
|
||||
from ray import ActorID, NodeID, WorkerID
|
||||
from ray._common.pydantic_compat import BaseModel
|
||||
from ray.core.generated.gcs_pb2 import ActorTableData
|
||||
from ray.dashboard.modules.job.common import JOB_LOGS_PATH_TEMPLATE
|
||||
from ray.util.state.common import (
|
||||
DEFAULT_RPC_TIMEOUT,
|
||||
GetLogOptions,
|
||||
protobuf_to_task_state_dict,
|
||||
)
|
||||
from ray.util.state.state_manager import StateDataSourceClient
|
||||
|
||||
if BaseModel is None:
|
||||
raise ModuleNotFoundError("Please install pydantic via `pip install pydantic`.")
|
||||
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
WORKER_LOG_PATTERN = re.compile(r".*worker-([0-9a-f]+)-([0-9a-f]+)-(\d+).(out|err)")
|
||||
|
||||
|
||||
class ResolvedStreamFileInfo(BaseModel):
|
||||
# The node id where the log file is located.
|
||||
node_id: str
|
||||
|
||||
# The log file path name. Could be a relative path relative to ray's logging folder,
|
||||
# or an absolute path.
|
||||
filename: str
|
||||
|
||||
# Start offset in the log file to stream from. None to indicate beginning of
|
||||
# the file, or determined by last tail lines.
|
||||
start_offset: Optional[int] = None
|
||||
|
||||
# End offset in the log file to stream from. None to indicate the end of the file.
|
||||
end_offset: Optional[int] = None
|
||||
|
||||
|
||||
class LogsManager:
|
||||
def __init__(self, data_source_client: StateDataSourceClient):
|
||||
self.client = data_source_client
|
||||
|
||||
@property
|
||||
def data_source_client(self) -> StateDataSourceClient:
|
||||
return self.client
|
||||
|
||||
async def ip_to_node_id(self, node_ip: Optional[str]) -> Optional[str]:
|
||||
"""Resolve the node id in hex from a given node ip.
|
||||
|
||||
Args:
|
||||
node_ip: The node ip.
|
||||
|
||||
Returns:
|
||||
node_id if there's a node id that matches the given node ip and is alive.
|
||||
None otherwise.
|
||||
"""
|
||||
return await self.client.ip_to_node_id(node_ip)
|
||||
|
||||
async def list_logs(
|
||||
self, node_id: str, timeout: int, glob_filter: str = "*"
|
||||
) -> Dict[str, List[str]]:
|
||||
"""Return a list of log files on a given node id filtered by the glob.
|
||||
|
||||
Args:
|
||||
node_id: The node id where log files present.
|
||||
timeout: The timeout of the API.
|
||||
glob_filter: The glob filter to filter out log files.
|
||||
|
||||
Returns:
|
||||
Dictionary of {component_name -> list of log files}
|
||||
|
||||
Raises:
|
||||
ValueError: If a source is unresponsive.
|
||||
"""
|
||||
reply = await self.client.list_logs(node_id, glob_filter, timeout=timeout)
|
||||
return self._categorize_log_files(reply.log_files)
|
||||
|
||||
async def stream_logs(
|
||||
self,
|
||||
options: GetLogOptions,
|
||||
get_actor_fn: Callable[[ActorID], Awaitable[Optional[ActorTableData]]],
|
||||
) -> AsyncIterable[bytes]:
|
||||
"""Generate a stream of logs in bytes.
|
||||
|
||||
Args:
|
||||
options: The option for streaming logs.
|
||||
get_actor_fn: Callable used to resolve actor metadata when the
|
||||
request targets an actor's logs.
|
||||
|
||||
Yields:
|
||||
bytes: Successive chunks of log content streamed from the agent.
|
||||
"""
|
||||
node_id = options.node_id
|
||||
if node_id is None:
|
||||
node_id = await self.ip_to_node_id(options.node_ip)
|
||||
|
||||
res = await self.resolve_filename(
|
||||
node_id=node_id,
|
||||
log_filename=options.filename,
|
||||
actor_id=options.actor_id,
|
||||
task_id=options.task_id,
|
||||
attempt_number=options.attempt_number,
|
||||
pid=options.pid,
|
||||
get_actor_fn=get_actor_fn,
|
||||
timeout=options.timeout,
|
||||
suffix=options.suffix,
|
||||
submission_id=options.submission_id,
|
||||
)
|
||||
|
||||
keep_alive = options.media_type == "stream"
|
||||
stream = await self.client.stream_log(
|
||||
node_id=res.node_id,
|
||||
log_file_name=res.filename,
|
||||
keep_alive=keep_alive,
|
||||
lines=options.lines,
|
||||
interval=options.interval,
|
||||
# If we keepalive logs connection, we shouldn't have timeout
|
||||
# otherwise the stream will be terminated forcefully
|
||||
# after the deadline is expired.
|
||||
timeout=options.timeout if not keep_alive else None,
|
||||
start_offset=res.start_offset,
|
||||
end_offset=res.end_offset,
|
||||
)
|
||||
|
||||
async for streamed_log in stream:
|
||||
yield streamed_log.data
|
||||
|
||||
async def _resolve_job_filename(self, sub_job_id: str) -> Tuple[str, str]:
|
||||
"""Return the log file name and node id for a given job submission id.
|
||||
|
||||
Args:
|
||||
sub_job_id: The job submission id.
|
||||
|
||||
Returns:
|
||||
The log file name and node id.
|
||||
"""
|
||||
job_infos = await self.client.get_job_info(timeout=DEFAULT_RPC_TIMEOUT)
|
||||
target_job = None
|
||||
for job_info in job_infos:
|
||||
if job_info.submission_id == sub_job_id:
|
||||
target_job = job_info
|
||||
break
|
||||
if target_job is None:
|
||||
logger.info(f"Submission job ID {sub_job_id} not found.")
|
||||
return None, None
|
||||
|
||||
node_id = job_info.driver_node_id
|
||||
if node_id is None:
|
||||
raise ValueError(
|
||||
f"Job {sub_job_id} has no driver node id info. "
|
||||
"This is likely a bug. Please file an issue."
|
||||
)
|
||||
|
||||
log_filename = JOB_LOGS_PATH_TEMPLATE.format(submission_id=sub_job_id)
|
||||
return node_id, log_filename
|
||||
|
||||
async def _resolve_worker_file(
|
||||
self,
|
||||
node_id_hex: str,
|
||||
worker_id_hex: Optional[str],
|
||||
pid: Optional[int],
|
||||
suffix: str,
|
||||
timeout: int,
|
||||
) -> Optional[str]:
|
||||
"""Resolve worker log file."""
|
||||
if worker_id_hex is not None and pid is not None:
|
||||
raise ValueError(
|
||||
f"Only one of worker id({worker_id_hex}) or pid({pid}) should be"
|
||||
"provided."
|
||||
)
|
||||
|
||||
if worker_id_hex is not None:
|
||||
log_files = await self.list_logs(
|
||||
node_id_hex, timeout, glob_filter=f"*{worker_id_hex}*{suffix}"
|
||||
)
|
||||
else:
|
||||
log_files = await self.list_logs(
|
||||
node_id_hex, timeout, glob_filter=f"*{pid}*{suffix}"
|
||||
)
|
||||
|
||||
# Find matching worker logs.
|
||||
for filename in [*log_files["worker_out"], *log_files["worker_err"]]:
|
||||
# Worker logs look like worker-[worker_id]-[job_id]-[pid].out
|
||||
if worker_id_hex is not None:
|
||||
worker_id_from_filename = WORKER_LOG_PATTERN.match(filename).group(1)
|
||||
if worker_id_from_filename == worker_id_hex:
|
||||
return filename
|
||||
else:
|
||||
worker_pid_from_filename = int(
|
||||
WORKER_LOG_PATTERN.match(filename).group(3)
|
||||
)
|
||||
if worker_pid_from_filename == pid:
|
||||
return filename
|
||||
return None
|
||||
|
||||
async def _resolve_actor_filename(
|
||||
self,
|
||||
actor_id: ActorID,
|
||||
get_actor_fn: Callable[[ActorID], Awaitable[Optional[ActorTableData]]],
|
||||
suffix: str,
|
||||
timeout: int,
|
||||
):
|
||||
"""Resolve actor log file.
|
||||
|
||||
Args:
|
||||
actor_id: The actor id.
|
||||
get_actor_fn: The function to get actor information.
|
||||
suffix: The suffix of the log file.
|
||||
timeout: Timeout in seconds.
|
||||
|
||||
Returns:
|
||||
The log file name and node id.
|
||||
|
||||
Raises:
|
||||
ValueError: If actor data is not found or get_actor_fn is not provided.
|
||||
"""
|
||||
if get_actor_fn is None:
|
||||
raise ValueError("get_actor_fn needs to be specified for actor_id")
|
||||
|
||||
actor_data = await get_actor_fn(actor_id)
|
||||
if actor_data is None:
|
||||
raise ValueError(f"Actor ID {actor_id} not found.")
|
||||
# TODO(sang): Only the latest worker id can be obtained from
|
||||
# actor information now. That means, if actors are restarted,
|
||||
# there's no way for us to get the past worker ids.
|
||||
worker_id_binary = actor_data.address.worker_id
|
||||
if not worker_id_binary:
|
||||
raise ValueError(
|
||||
f"Worker ID for Actor ID {actor_id} not found. "
|
||||
"Actor is not scheduled yet."
|
||||
)
|
||||
worker_id = WorkerID(worker_id_binary)
|
||||
node_id_binary = actor_data.address.node_id
|
||||
if not node_id_binary:
|
||||
raise ValueError(
|
||||
f"Node ID for Actor ID {actor_id} not found. "
|
||||
"Actor is not scheduled yet."
|
||||
)
|
||||
node_id = NodeID(node_id_binary)
|
||||
log_filename = await self._resolve_worker_file(
|
||||
node_id_hex=node_id.hex(),
|
||||
worker_id_hex=worker_id.hex(),
|
||||
pid=None,
|
||||
suffix=suffix,
|
||||
timeout=timeout,
|
||||
)
|
||||
return node_id.hex(), log_filename
|
||||
|
||||
async def _resolve_task_filename(
|
||||
self, task_id: str, attempt_number: int, suffix: str, timeout: int
|
||||
):
|
||||
"""Resolve log file for a task.
|
||||
|
||||
Args:
|
||||
task_id: The task id.
|
||||
attempt_number: The attempt number.
|
||||
suffix: The suffix of the log file, e.g. out or err.
|
||||
timeout: Timeout in seconds.
|
||||
|
||||
Returns:
|
||||
The log file name, node id, the start and end offsets of the
|
||||
corresponding task log in the file.
|
||||
|
||||
Raises:
|
||||
FileNotFoundError: If the log file is not found.
|
||||
ValueError: If the suffix is not out or err.
|
||||
"""
|
||||
log_filename = None
|
||||
node_id = None
|
||||
start_offset = None
|
||||
end_offset = None
|
||||
|
||||
if suffix not in ["out", "err"]:
|
||||
raise ValueError(f"Suffix {suffix} is not supported.")
|
||||
|
||||
reply = await self.client.get_all_task_info(
|
||||
filters=[("task_id", "=", task_id)], timeout=timeout
|
||||
)
|
||||
# Check if the task is found.
|
||||
if len(reply.events_by_task) == 0:
|
||||
raise FileNotFoundError(
|
||||
f"Could not find log file for task: {task_id}"
|
||||
f" (attempt {attempt_number}) with suffix: {suffix}"
|
||||
)
|
||||
task_event = None
|
||||
for t in reply.events_by_task:
|
||||
if t.attempt_number == attempt_number:
|
||||
task_event = t
|
||||
break
|
||||
|
||||
if task_event is None:
|
||||
raise FileNotFoundError(
|
||||
"Could not find log file for task attempt:"
|
||||
f"{task_id}({attempt_number})"
|
||||
)
|
||||
# Get the worker id and node id.
|
||||
task = protobuf_to_task_state_dict(task_event)
|
||||
|
||||
worker_id = task.get("worker_id", None)
|
||||
node_id = task.get("node_id", None)
|
||||
log_info = task.get("task_log_info", None)
|
||||
actor_id = task.get("actor_id", None)
|
||||
|
||||
if node_id is None:
|
||||
raise FileNotFoundError(
|
||||
"Could not find log file for task attempt."
|
||||
f"{task_id}({attempt_number}) due to missing node info."
|
||||
)
|
||||
|
||||
if log_info is None and actor_id is not None:
|
||||
# This is a concurrent actor task. The logs will be interleaved.
|
||||
# So we return the log file of the actor instead.
|
||||
raise FileNotFoundError(
|
||||
f"For actor task, please query actor log for "
|
||||
f"actor({actor_id}): e.g. ray logs actor --id {actor_id} . Or "
|
||||
"set RAY_ENABLE_RECORD_ACTOR_TASK_LOGGING=1 in actor's runtime env "
|
||||
"or when starting the cluster. Recording actor task's log could be "
|
||||
"expensive, so Ray turns it off by default."
|
||||
)
|
||||
elif log_info is None:
|
||||
raise FileNotFoundError(
|
||||
"Could not find log file for task attempt:"
|
||||
f"{task_id}({attempt_number})."
|
||||
f"Worker id = {worker_id}, node id = {node_id},"
|
||||
f"log_info = {log_info}"
|
||||
)
|
||||
|
||||
filename_key = "stdout_file" if suffix == "out" else "stderr_file"
|
||||
log_filename = log_info.get(filename_key, None)
|
||||
if log_filename is None:
|
||||
raise FileNotFoundError(
|
||||
f"Missing log filename info in {log_info} for task {task_id},"
|
||||
f"attempt {attempt_number}"
|
||||
)
|
||||
|
||||
start_offset = log_info.get(f"std{suffix}_start", None)
|
||||
end_offset = log_info.get(f"std{suffix}_end", None)
|
||||
|
||||
return node_id, log_filename, start_offset, end_offset
|
||||
|
||||
async def resolve_filename(
|
||||
self,
|
||||
*,
|
||||
node_id: Optional[str] = None,
|
||||
log_filename: Optional[str] = None,
|
||||
actor_id: Optional[str] = None,
|
||||
task_id: Optional[str] = None,
|
||||
attempt_number: Optional[int] = None,
|
||||
pid: Optional[str] = None,
|
||||
get_actor_fn: Optional[
|
||||
Callable[[ActorID], Awaitable[Optional[ActorTableData]]]
|
||||
] = None,
|
||||
timeout: int = DEFAULT_RPC_TIMEOUT,
|
||||
suffix: str = "out",
|
||||
submission_id: Optional[str] = None,
|
||||
) -> ResolvedStreamFileInfo:
|
||||
"""Return the file name given all options.
|
||||
|
||||
Args:
|
||||
node_id: The node's id from which logs are resolved.
|
||||
log_filename: Filename of the log file.
|
||||
actor_id: Id of the actor that generates the log file.
|
||||
task_id: Id of the task that generates the log file.
|
||||
attempt_number: The attempt number of the task. Used with
|
||||
``task_id`` to disambiguate retries.
|
||||
pid: Id of the worker process that generates the log file.
|
||||
get_actor_fn: Callback to get the actor's data by id.
|
||||
timeout: Timeout for the gRPC to listing logs on the node
|
||||
specified by `node_id`.
|
||||
suffix: Log suffix if no `log_filename` is provided, when
|
||||
resolving by other ids'. Default to "out".
|
||||
submission_id: The submission id for a submission job.
|
||||
|
||||
Returns:
|
||||
A ``ResolvedStreamFileInfo`` describing the resolved node id,
|
||||
filename, and (optional) byte offsets to stream.
|
||||
"""
|
||||
start_offset = None
|
||||
end_offset = None
|
||||
if suffix not in ["out", "err"]:
|
||||
raise ValueError(f"Suffix {suffix} is not supported. ")
|
||||
|
||||
# TODO(rickyx): We should make sure we do some sort of checking on the log
|
||||
# filename
|
||||
if actor_id:
|
||||
node_id, log_filename = await self._resolve_actor_filename(
|
||||
ActorID.from_hex(actor_id), get_actor_fn, suffix, timeout
|
||||
)
|
||||
|
||||
elif task_id:
|
||||
(
|
||||
node_id,
|
||||
log_filename,
|
||||
start_offset,
|
||||
end_offset,
|
||||
) = await self._resolve_task_filename(
|
||||
task_id, attempt_number, suffix, timeout
|
||||
)
|
||||
|
||||
elif submission_id:
|
||||
node_id, log_filename = await self._resolve_job_filename(submission_id)
|
||||
|
||||
elif pid:
|
||||
if node_id is None:
|
||||
raise ValueError(
|
||||
"Node id needs to be specified for resolving"
|
||||
f" filenames of pid {pid}"
|
||||
)
|
||||
log_filename = await self._resolve_worker_file(
|
||||
node_id_hex=node_id,
|
||||
worker_id_hex=None,
|
||||
pid=pid,
|
||||
suffix=suffix,
|
||||
timeout=timeout,
|
||||
)
|
||||
|
||||
if log_filename is None:
|
||||
raise FileNotFoundError(
|
||||
"Could not find a log file. Please make sure the given "
|
||||
"option exists in the cluster.\n"
|
||||
f"\tnode_id: {node_id}\n"
|
||||
f"\tfilename: {log_filename}\n"
|
||||
f"\tactor_id: {actor_id}\n"
|
||||
f"\ttask_id: {task_id}\n"
|
||||
f"\tpid: {pid}\n"
|
||||
f"\tsuffix: {suffix}\n"
|
||||
f"\tsubmission_id: {submission_id}\n"
|
||||
f"\tattempt_number: {attempt_number}\n"
|
||||
)
|
||||
|
||||
res = ResolvedStreamFileInfo(
|
||||
node_id=node_id,
|
||||
filename=log_filename,
|
||||
start_offset=start_offset,
|
||||
end_offset=end_offset,
|
||||
)
|
||||
logger.info(f"Resolved log file: {res}")
|
||||
return res
|
||||
|
||||
def _categorize_log_files(self, log_files: List[str]) -> Dict[str, List[str]]:
|
||||
"""Categorize the given log files after filterieng them out using a given glob.
|
||||
|
||||
Args:
|
||||
log_files: Filenames returned from a ``list_logs`` query, already
|
||||
filtered by the caller's glob.
|
||||
|
||||
Returns:
|
||||
Dictionary of {component_name -> list of log files}
|
||||
"""
|
||||
result = defaultdict(list)
|
||||
for log_file in log_files:
|
||||
if "worker" in log_file and (log_file.endswith(".out")):
|
||||
result["worker_out"].append(log_file)
|
||||
elif "worker" in log_file and (log_file.endswith(".err")):
|
||||
result["worker_err"].append(log_file)
|
||||
elif "core-worker" in log_file and log_file.endswith(".log"):
|
||||
result["core_worker"].append(log_file)
|
||||
elif "core-driver" in log_file and log_file.endswith(".log"):
|
||||
result["driver"].append(log_file)
|
||||
elif "raylet." in log_file:
|
||||
result["raylet"].append(log_file)
|
||||
elif "gcs_server." in log_file:
|
||||
result["gcs_server"].append(log_file)
|
||||
elif "log_monitor" in log_file:
|
||||
result["internal"].append(log_file)
|
||||
elif "monitor" in log_file:
|
||||
result["autoscaler"].append(log_file)
|
||||
elif "agent." in log_file:
|
||||
result["agent"].append(log_file)
|
||||
elif "dashboard." in log_file:
|
||||
result["dashboard"].append(log_file)
|
||||
else:
|
||||
result["internal"].append(log_file)
|
||||
|
||||
return result
|
||||
@@ -0,0 +1,9 @@
|
||||
import mimetypes
|
||||
|
||||
import ray.dashboard.modules.log.log_consts as log_consts
|
||||
|
||||
|
||||
def register_mimetypes():
|
||||
for _type, extensions in log_consts.MIME_TYPES.items():
|
||||
for ext in extensions:
|
||||
mimetypes.add_type(_type, ext)
|
||||
Reference in New Issue
Block a user