chore: import upstream snapshot with attribution
This commit is contained in:
wehub-resource-sync
@@ -0,0 +1,334 @@
|
||||
import argparse
|
||||
import logging
|
||||
import os
|
||||
import platform
|
||||
import signal
|
||||
import sys
|
||||
import traceback
|
||||
from typing import Optional, Set
|
||||
|
||||
import ray
|
||||
import ray._private.ray_constants as ray_constants
|
||||
import ray.dashboard.consts as dashboard_consts
|
||||
import ray.dashboard.head as dashboard_head
|
||||
import ray.dashboard.utils as dashboard_utils
|
||||
from ray._common.ray_constants import (
|
||||
LOGGING_ROTATE_BACKUP_COUNT,
|
||||
LOGGING_ROTATE_BYTES,
|
||||
)
|
||||
from ray._common.utils import get_or_create_event_loop
|
||||
from ray._private import logging_utils
|
||||
from ray._private.ray_logging import setup_component_logger
|
||||
from ray._private.utils import (
|
||||
format_error_message,
|
||||
publish_error_to_driver,
|
||||
)
|
||||
|
||||
# Logger for this module. It should be configured at the entry point
|
||||
# into the program using Ray. Ray provides a default configuration at
|
||||
# entry/init points.
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class Dashboard:
|
||||
"""A dashboard process for monitoring Ray nodes.
|
||||
|
||||
This dashboard is made up of a REST API which collates data published by
|
||||
Reporter processes on nodes into a json structure, and a webserver
|
||||
which polls said API for display purposes.
|
||||
|
||||
Args:
|
||||
host: Host address of dashboard aiohttp server.
|
||||
port: Port number of dashboard aiohttp server.
|
||||
port_retries: The retry times to select a valid port.
|
||||
gcs_address: GCS address of the cluster.
|
||||
cluster_id_hex: Cluster ID hex string.
|
||||
node_ip_address: The IP address of the dashboard.
|
||||
log_dir: Log directory of dashboard.
|
||||
logging_level: The logging level (e.g. logging.INFO, logging.DEBUG)
|
||||
logging_format: The format string for log messages
|
||||
logging_filename: The name of the log file
|
||||
logging_rotate_bytes: Max size in bytes before rotating log file
|
||||
logging_rotate_backup_count: Number of backup files to keep when rotating
|
||||
temp_dir: Specify the path of the temporary directory use by Ray process
|
||||
session_dir: Specify the path of the session directory of the cluster under the temp_dir
|
||||
minimal: Whether or not it will load the minimal modules.
|
||||
serve_frontend: Whether to serve the frontend HTML from the dashboard.
|
||||
modules_to_load: Specify the list of module names in [module_1],[module_2] format.
|
||||
E.g., JobHead,StateHead...
|
||||
If nothing is specified, all modules are loaded.
|
||||
proxy_server_url: The url to redirect api requests to
|
||||
Ex: proxy_server_url=http://historyserver:8080
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
host: str,
|
||||
port: int,
|
||||
port_retries: int,
|
||||
gcs_address: str,
|
||||
cluster_id_hex: str,
|
||||
node_ip_address: str,
|
||||
log_dir: str,
|
||||
logging_level: int,
|
||||
logging_format: str,
|
||||
logging_filename: str,
|
||||
logging_rotate_bytes: int,
|
||||
logging_rotate_backup_count: int,
|
||||
temp_dir: str = None,
|
||||
session_dir: str = None,
|
||||
minimal: bool = False,
|
||||
serve_frontend: bool = True,
|
||||
modules_to_load: Optional[Set[str]] = None,
|
||||
proxy_server_url: Optional[str] = None,
|
||||
):
|
||||
self.dashboard_head = dashboard_head.DashboardHead(
|
||||
http_host=host,
|
||||
http_port=port,
|
||||
http_port_retries=port_retries,
|
||||
gcs_address=gcs_address,
|
||||
cluster_id_hex=cluster_id_hex,
|
||||
node_ip_address=node_ip_address,
|
||||
log_dir=log_dir,
|
||||
logging_level=logging_level,
|
||||
logging_format=logging_format,
|
||||
logging_filename=logging_filename,
|
||||
logging_rotate_bytes=logging_rotate_bytes,
|
||||
logging_rotate_backup_count=logging_rotate_backup_count,
|
||||
temp_dir=temp_dir,
|
||||
session_dir=session_dir,
|
||||
minimal=minimal,
|
||||
serve_frontend=serve_frontend,
|
||||
modules_to_load=modules_to_load,
|
||||
proxy_server_url=proxy_server_url,
|
||||
)
|
||||
|
||||
async def run(self):
|
||||
await self.dashboard_head.run()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
parser = argparse.ArgumentParser(description="Ray dashboard.")
|
||||
parser.add_argument(
|
||||
"--host", required=True, type=str, help="The host to use for the HTTP server."
|
||||
)
|
||||
parser.add_argument(
|
||||
"--port", required=True, type=int, help="The port to use for the HTTP server."
|
||||
)
|
||||
parser.add_argument(
|
||||
"--port-retries",
|
||||
required=False,
|
||||
type=int,
|
||||
default=0,
|
||||
help="The retry times to select a valid port.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--gcs-address", required=True, type=str, help="The address (ip:port) of GCS."
|
||||
)
|
||||
parser.add_argument(
|
||||
"--cluster-id-hex", required=True, type=str, help="The cluster ID in hex."
|
||||
)
|
||||
parser.add_argument(
|
||||
"--node-ip-address",
|
||||
required=True,
|
||||
type=str,
|
||||
help="The IP address of the node where this is running.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--logging-level",
|
||||
required=False,
|
||||
type=lambda s: logging.getLevelName(s.upper()),
|
||||
default=ray_constants.LOGGER_LEVEL,
|
||||
choices=ray_constants.LOGGER_LEVEL_CHOICES,
|
||||
help=ray_constants.LOGGER_LEVEL_HELP,
|
||||
)
|
||||
parser.add_argument(
|
||||
"--logging-format",
|
||||
required=False,
|
||||
type=str,
|
||||
default=ray_constants.LOGGER_FORMAT,
|
||||
help=ray_constants.LOGGER_FORMAT_HELP,
|
||||
)
|
||||
parser.add_argument(
|
||||
"--logging-filename",
|
||||
required=False,
|
||||
type=str,
|
||||
default=dashboard_consts.DASHBOARD_LOG_FILENAME,
|
||||
help="Specify the name of log file, "
|
||||
'log to stdout if set empty, default is "{}"'.format(
|
||||
dashboard_consts.DASHBOARD_LOG_FILENAME
|
||||
),
|
||||
)
|
||||
parser.add_argument(
|
||||
"--logging-rotate-bytes",
|
||||
required=False,
|
||||
type=int,
|
||||
default=LOGGING_ROTATE_BYTES,
|
||||
help="Specify the max bytes for rotating "
|
||||
"log file, default is {} bytes.".format(LOGGING_ROTATE_BYTES),
|
||||
)
|
||||
parser.add_argument(
|
||||
"--logging-rotate-backup-count",
|
||||
required=False,
|
||||
type=int,
|
||||
default=LOGGING_ROTATE_BACKUP_COUNT,
|
||||
help="Specify the backup count of rotated log file, default is {}.".format(
|
||||
LOGGING_ROTATE_BACKUP_COUNT
|
||||
),
|
||||
)
|
||||
parser.add_argument(
|
||||
"--log-dir",
|
||||
required=True,
|
||||
type=str,
|
||||
default=None,
|
||||
help="Specify the path of log directory.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--temp-dir",
|
||||
required=True,
|
||||
type=str,
|
||||
default=None,
|
||||
help="Specify the path of the temporary directory use by Ray process.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--session-dir",
|
||||
required=True,
|
||||
type=str,
|
||||
default=None,
|
||||
help="Specify the path of the session directory of the cluster.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--minimal",
|
||||
action="store_true",
|
||||
help=(
|
||||
"Minimal dashboard only contains a subset of features that don't "
|
||||
"require additional dependencies installed when ray is installed "
|
||||
"by `pip install ray[default]`."
|
||||
),
|
||||
)
|
||||
parser.add_argument(
|
||||
"--modules-to-load",
|
||||
required=False,
|
||||
default=None,
|
||||
help=(
|
||||
"Specify the list of module names in [module_1],[module_2] format."
|
||||
"E.g., JobHead,StateHead... "
|
||||
"If nothing is specified, all modules are loaded."
|
||||
),
|
||||
)
|
||||
parser.add_argument(
|
||||
"--disable-frontend",
|
||||
action="store_true",
|
||||
help=("If configured, frontend html is not served from the server."),
|
||||
)
|
||||
parser.add_argument(
|
||||
"--stdout-filepath",
|
||||
required=False,
|
||||
type=str,
|
||||
default="",
|
||||
help="The filepath to dump dashboard stdout.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--stderr-filepath",
|
||||
required=False,
|
||||
type=str,
|
||||
default="",
|
||||
help="The filepath to dump dashboard stderr.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--proxy-server-url",
|
||||
required=False,
|
||||
type=str,
|
||||
default="",
|
||||
help="The proxy server url to redirect requests to"
|
||||
"Ex: --proxy-server-url=http://historyserver:8080 ",
|
||||
)
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
try:
|
||||
# Disable log rotation for windows platform.
|
||||
logging_rotation_bytes = (
|
||||
args.logging_rotate_bytes if sys.platform != "win32" else 0
|
||||
)
|
||||
logging_rotation_backup_count = (
|
||||
args.logging_rotate_backup_count if sys.platform != "win32" else 1
|
||||
)
|
||||
setup_component_logger(
|
||||
logging_level=args.logging_level,
|
||||
logging_format=args.logging_format,
|
||||
log_dir=args.log_dir,
|
||||
filename=args.logging_filename,
|
||||
max_bytes=logging_rotation_bytes,
|
||||
backup_count=logging_rotation_backup_count,
|
||||
)
|
||||
|
||||
# Setup stdout/stderr redirect files if redirection enabled.
|
||||
logging_utils.redirect_stdout_stderr_if_needed(
|
||||
args.stdout_filepath,
|
||||
args.stderr_filepath,
|
||||
logging_rotation_bytes,
|
||||
logging_rotation_backup_count,
|
||||
)
|
||||
|
||||
if args.modules_to_load:
|
||||
modules_to_load = set(args.modules_to_load.strip(" ,").split(","))
|
||||
else:
|
||||
# None == default.
|
||||
modules_to_load = None
|
||||
|
||||
loop = get_or_create_event_loop()
|
||||
dashboard = Dashboard(
|
||||
host=args.host,
|
||||
port=args.port,
|
||||
port_retries=args.port_retries,
|
||||
gcs_address=args.gcs_address,
|
||||
cluster_id_hex=args.cluster_id_hex,
|
||||
node_ip_address=args.node_ip_address,
|
||||
log_dir=args.log_dir,
|
||||
logging_level=args.logging_level,
|
||||
logging_format=args.logging_format,
|
||||
logging_filename=args.logging_filename,
|
||||
logging_rotate_bytes=logging_rotation_bytes,
|
||||
logging_rotate_backup_count=logging_rotation_backup_count,
|
||||
temp_dir=args.temp_dir,
|
||||
session_dir=args.session_dir,
|
||||
minimal=args.minimal,
|
||||
serve_frontend=(not args.disable_frontend),
|
||||
modules_to_load=modules_to_load,
|
||||
proxy_server_url=args.proxy_server_url,
|
||||
)
|
||||
|
||||
def sigterm_handler():
|
||||
logger.warning("Exiting with SIGTERM immediately...")
|
||||
os._exit(signal.SIGTERM)
|
||||
|
||||
if sys.platform != "win32":
|
||||
# TODO(rickyyx): we currently do not have any logic for actual
|
||||
# graceful termination in the dashboard. Most of the underlying
|
||||
# async tasks run by the dashboard head doesn't handle CancelledError.
|
||||
# So a truly graceful shutdown is not trivial w/o much refactoring.
|
||||
# Re-open the issue: https://github.com/ray-project/ray/issues/25518
|
||||
# if a truly graceful shutdown is required.
|
||||
loop.add_signal_handler(signal.SIGTERM, sigterm_handler)
|
||||
|
||||
loop.run_until_complete(dashboard.run())
|
||||
except Exception as e:
|
||||
traceback_str = format_error_message(traceback.format_exc())
|
||||
message = (
|
||||
f"The dashboard on node {platform.uname()[1]} "
|
||||
f"failed with the following "
|
||||
f"error:\n{traceback_str}"
|
||||
)
|
||||
if isinstance(e, dashboard_utils.FrontendNotFoundError):
|
||||
logger.warning(message)
|
||||
else:
|
||||
logger.error(message)
|
||||
raise e
|
||||
|
||||
# Something went wrong, so push an error to all drivers.
|
||||
publish_error_to_driver(
|
||||
ray_constants.DASHBOARD_DIED_ERROR,
|
||||
message,
|
||||
gcs_client=ray._raylet.GcsClient(address=args.gcs_address),
|
||||
)
|
||||
Reference in New Issue
Block a user