chore: import upstream snapshot with attribution
This commit is contained in:
wehub-resource-sync
@@ -0,0 +1,131 @@
|
||||
import copy
|
||||
import os
|
||||
from typing import Any, Dict, Tuple
|
||||
|
||||
from ray._common.utils import get_default_ray_temp_dir
|
||||
from ray.autoscaler._private.cli_logger import cli_logger
|
||||
|
||||
unsupported_field_message = "The field {} is not supported for on-premise clusters."
|
||||
|
||||
LOCAL_CLUSTER_NODE_TYPE = "local.cluster.node"
|
||||
|
||||
|
||||
def prepare_local(config: Dict[str, Any]) -> Tuple[Dict[str, Any], bool]:
|
||||
"""
|
||||
Prepare local cluster config for ingestion by cluster launcher and
|
||||
autoscaler.
|
||||
"""
|
||||
config = copy.deepcopy(config)
|
||||
for field in "head_node", "worker_nodes", "available_node_types":
|
||||
if config.get(field):
|
||||
# If the config already contains the internal node type, it's been prepared via ray up already hence return as-is.
|
||||
if (
|
||||
field == "available_node_types"
|
||||
and LOCAL_CLUSTER_NODE_TYPE in config.get(field, {})
|
||||
):
|
||||
return config, False
|
||||
err_msg = unsupported_field_message.format(field)
|
||||
cli_logger.abort(err_msg)
|
||||
# We use a config with a single node type for on-prem clusters.
|
||||
# Resources internally detected by Ray are not overridden by the autoscaler
|
||||
# (see NodeProvider.do_update)
|
||||
config["available_node_types"] = {
|
||||
LOCAL_CLUSTER_NODE_TYPE: {"node_config": {}, "resources": {}}
|
||||
}
|
||||
config["head_node_type"] = LOCAL_CLUSTER_NODE_TYPE
|
||||
if "coordinator_address" in config["provider"]:
|
||||
config = prepare_coordinator(config)
|
||||
else:
|
||||
config = prepare_manual(config)
|
||||
return config, True
|
||||
|
||||
|
||||
def prepare_coordinator(config: Dict[str, Any]) -> Dict[str, Any]:
|
||||
config = copy.deepcopy(config)
|
||||
# User should explicitly set the max number of workers for the coordinator
|
||||
# to allocate.
|
||||
if "max_workers" not in config:
|
||||
cli_logger.abort(
|
||||
"The field `max_workers` is required when using an "
|
||||
"automatically managed on-premise cluster."
|
||||
)
|
||||
node_type = config["available_node_types"][LOCAL_CLUSTER_NODE_TYPE]
|
||||
# The autoscaler no longer uses global `min_workers`.
|
||||
# Move `min_workers` to the node_type config.
|
||||
node_type["min_workers"] = config.pop("min_workers", 0)
|
||||
node_type["max_workers"] = config["max_workers"]
|
||||
return config
|
||||
|
||||
|
||||
def prepare_manual(config: Dict[str, Any]) -> Dict[str, Any]:
|
||||
"""Validates and sets defaults for configs of manually managed on-prem
|
||||
clusters.
|
||||
|
||||
- Checks for presence of required `worker_ips` and `head_ips` fields.
|
||||
- Defaults min and max workers to the number of `worker_ips`.
|
||||
- Caps min and max workers at the number of `worker_ips`.
|
||||
- Writes min and max worker info into the single worker node type.
|
||||
"""
|
||||
config = copy.deepcopy(config)
|
||||
if ("worker_ips" not in config["provider"]) or (
|
||||
"head_ip" not in config["provider"]
|
||||
):
|
||||
cli_logger.abort(
|
||||
"Please supply a `head_ip` and list of `worker_ips`. "
|
||||
"Alternatively, supply a `coordinator_address`."
|
||||
)
|
||||
num_ips = len(config["provider"]["worker_ips"])
|
||||
node_type = config["available_node_types"][LOCAL_CLUSTER_NODE_TYPE]
|
||||
# Default to keeping all provided ips in the cluster.
|
||||
config.setdefault("max_workers", num_ips)
|
||||
|
||||
# The autoscaler no longer uses global `min_workers`.
|
||||
# We will move `min_workers` to the node_type config.
|
||||
min_workers = config.pop("min_workers", num_ips)
|
||||
max_workers = config["max_workers"]
|
||||
|
||||
if min_workers > num_ips:
|
||||
cli_logger.warning(
|
||||
f"The value of `min_workers` supplied ({min_workers}) is greater"
|
||||
f" than the number of available worker ips ({num_ips})."
|
||||
f" Setting `min_workers={num_ips}`."
|
||||
)
|
||||
node_type["min_workers"] = num_ips
|
||||
else:
|
||||
node_type["min_workers"] = min_workers
|
||||
|
||||
if max_workers > num_ips:
|
||||
cli_logger.warning(
|
||||
f"The value of `max_workers` supplied ({max_workers}) is greater"
|
||||
f" than the number of available worker ips ({num_ips})."
|
||||
f" Setting `max_workers={num_ips}`."
|
||||
)
|
||||
node_type["max_workers"] = num_ips
|
||||
config["max_workers"] = num_ips
|
||||
else:
|
||||
node_type["max_workers"] = max_workers
|
||||
|
||||
if max_workers < num_ips:
|
||||
cli_logger.warning(
|
||||
f"The value of `max_workers` supplied ({max_workers}) is less"
|
||||
f" than the number of available worker ips ({num_ips})."
|
||||
f" At most {max_workers} Ray worker nodes will connect to the cluster."
|
||||
)
|
||||
|
||||
return config
|
||||
|
||||
|
||||
def get_lock_path(cluster_name: str) -> str:
|
||||
return os.path.join(
|
||||
get_default_ray_temp_dir(), "cluster-{}.lock".format(cluster_name)
|
||||
)
|
||||
|
||||
|
||||
def get_state_path(cluster_name: str) -> str:
|
||||
return os.path.join(
|
||||
get_default_ray_temp_dir(), "cluster-{}.state".format(cluster_name)
|
||||
)
|
||||
|
||||
|
||||
def bootstrap_local(config: Dict[str, Any]) -> Dict[str, Any]:
|
||||
return config
|
||||
@@ -0,0 +1,110 @@
|
||||
import json
|
||||
import logging
|
||||
from http.client import RemoteDisconnected
|
||||
|
||||
from ray.autoscaler.node_provider import NodeProvider
|
||||
from ray.autoscaler.tags import TAG_RAY_CLUSTER_NAME
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class CoordinatorSenderNodeProvider(NodeProvider):
|
||||
"""NodeProvider for automatically managed private/local clusters.
|
||||
|
||||
The cluster management is handled by a remote coordinating server.
|
||||
The server listens on <coordinator_address>, therefore, the address
|
||||
should be provided in the provider section in the cluster config.
|
||||
The server receieves HTTP requests from this class and uses
|
||||
LocalNodeProvider to get their responses.
|
||||
"""
|
||||
|
||||
def __init__(self, provider_config, cluster_name):
|
||||
NodeProvider.__init__(self, provider_config, cluster_name)
|
||||
self.coordinator_address = provider_config["coordinator_address"]
|
||||
|
||||
def _get_http_response(self, request):
|
||||
headers = {
|
||||
"Content-Type": "application/json",
|
||||
}
|
||||
request_message = json.dumps(request).encode()
|
||||
http_coordinator_address = "http://" + self.coordinator_address
|
||||
|
||||
try:
|
||||
import requests # `requests` is not part of stdlib.
|
||||
from requests.exceptions import ConnectionError
|
||||
|
||||
r = requests.get(
|
||||
http_coordinator_address,
|
||||
data=request_message,
|
||||
headers=headers,
|
||||
timeout=None,
|
||||
)
|
||||
except (RemoteDisconnected, ConnectionError):
|
||||
logger.exception(
|
||||
"Could not connect to: "
|
||||
+ http_coordinator_address
|
||||
+ ". Did you run python coordinator_server.py"
|
||||
+ " --ips <list_of_node_ips> --host <HOST> --port <PORT>?"
|
||||
)
|
||||
raise
|
||||
except ImportError:
|
||||
logger.exception(
|
||||
"Not all Ray Autoscaler dependencies were found. "
|
||||
"In Ray 1.4+, the Ray CLI, autoscaler, and dashboard will "
|
||||
'only be usable via `pip install "ray[default]"`. Please '
|
||||
"update your install command."
|
||||
)
|
||||
raise
|
||||
|
||||
response = r.json()
|
||||
return response
|
||||
|
||||
def non_terminated_nodes(self, tag_filters):
|
||||
# Only get the non terminated nodes associated with this cluster name.
|
||||
tag_filters[TAG_RAY_CLUSTER_NAME] = self.cluster_name
|
||||
request = {"type": "non_terminated_nodes", "args": (tag_filters,)}
|
||||
return self._get_http_response(request)
|
||||
|
||||
def is_running(self, node_id):
|
||||
request = {"type": "is_running", "args": (node_id,)}
|
||||
return self._get_http_response(request)
|
||||
|
||||
def is_terminated(self, node_id):
|
||||
request = {"type": "is_terminated", "args": (node_id,)}
|
||||
return self._get_http_response(request)
|
||||
|
||||
def node_tags(self, node_id):
|
||||
request = {"type": "node_tags", "args": (node_id,)}
|
||||
return self._get_http_response(request)
|
||||
|
||||
def external_ip(self, node_id):
|
||||
request = {"type": "external_ip", "args": (node_id,)}
|
||||
response = self._get_http_response(request)
|
||||
return response
|
||||
|
||||
def internal_ip(self, node_id):
|
||||
request = {"type": "internal_ip", "args": (node_id,)}
|
||||
response = self._get_http_response(request)
|
||||
return response
|
||||
|
||||
def create_node(self, node_config, tags, count):
|
||||
# Tag the newly created node with this cluster name. Helps to get
|
||||
# the right nodes when calling non_terminated_nodes.
|
||||
tags[TAG_RAY_CLUSTER_NAME] = self.cluster_name
|
||||
request = {
|
||||
"type": "create_node",
|
||||
"args": (node_config, tags, count),
|
||||
}
|
||||
self._get_http_response(request)
|
||||
|
||||
def set_node_tags(self, node_id, tags):
|
||||
request = {"type": "set_node_tags", "args": (node_id, tags)}
|
||||
self._get_http_response(request)
|
||||
|
||||
def terminate_node(self, node_id):
|
||||
request = {"type": "terminate_node", "args": (node_id,)}
|
||||
self._get_http_response(request)
|
||||
|
||||
def terminate_nodes(self, node_ids):
|
||||
request = {"type": "terminate_nodes", "args": (node_ids,)}
|
||||
self._get_http_response(request)
|
||||
@@ -0,0 +1,329 @@
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import socket
|
||||
from threading import RLock
|
||||
|
||||
from filelock import FileLock
|
||||
|
||||
from ray.autoscaler._private.local.config import (
|
||||
LOCAL_CLUSTER_NODE_TYPE,
|
||||
bootstrap_local,
|
||||
get_lock_path,
|
||||
get_state_path,
|
||||
)
|
||||
from ray.autoscaler.node_provider import NodeProvider
|
||||
from ray.autoscaler.tags import (
|
||||
NODE_KIND_HEAD,
|
||||
NODE_KIND_WORKER,
|
||||
STATUS_UP_TO_DATE,
|
||||
TAG_RAY_NODE_KIND,
|
||||
TAG_RAY_NODE_NAME,
|
||||
TAG_RAY_NODE_STATUS,
|
||||
TAG_RAY_USER_NODE_TYPE,
|
||||
)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
filelock_logger = logging.getLogger("filelock")
|
||||
filelock_logger.setLevel(logging.WARNING)
|
||||
|
||||
|
||||
class ClusterState:
|
||||
def __init__(self, lock_path, save_path, provider_config):
|
||||
self.lock = RLock()
|
||||
os.makedirs(os.path.dirname(lock_path), exist_ok=True)
|
||||
self.file_lock = FileLock(lock_path)
|
||||
self.save_path = save_path
|
||||
|
||||
with self.lock:
|
||||
with self.file_lock:
|
||||
if os.path.exists(self.save_path):
|
||||
workers = json.loads(open(self.save_path).read())
|
||||
head_config = workers.get(provider_config["head_ip"])
|
||||
if (
|
||||
not head_config
|
||||
or head_config.get("tags", {}).get(TAG_RAY_NODE_KIND)
|
||||
!= NODE_KIND_HEAD
|
||||
):
|
||||
workers = {}
|
||||
logger.info("Head IP changed - recreating cluster.")
|
||||
else:
|
||||
workers = {}
|
||||
logger.info(
|
||||
"ClusterState: Loaded cluster state: {}".format(list(workers))
|
||||
)
|
||||
for worker_ip in provider_config["worker_ips"]:
|
||||
if worker_ip not in workers:
|
||||
workers[worker_ip] = {
|
||||
"tags": {TAG_RAY_NODE_KIND: NODE_KIND_WORKER},
|
||||
"state": "terminated",
|
||||
}
|
||||
else:
|
||||
assert (
|
||||
workers[worker_ip]["tags"][TAG_RAY_NODE_KIND]
|
||||
== NODE_KIND_WORKER
|
||||
)
|
||||
if provider_config["head_ip"] not in workers:
|
||||
workers[provider_config["head_ip"]] = {
|
||||
"tags": {TAG_RAY_NODE_KIND: NODE_KIND_HEAD},
|
||||
"state": "terminated",
|
||||
}
|
||||
else:
|
||||
assert (
|
||||
workers[provider_config["head_ip"]]["tags"][TAG_RAY_NODE_KIND]
|
||||
== NODE_KIND_HEAD
|
||||
)
|
||||
# Relevant when a user reduces the number of workers
|
||||
# without changing the headnode.
|
||||
list_of_node_ips = list(provider_config["worker_ips"])
|
||||
list_of_node_ips.append(provider_config["head_ip"])
|
||||
for worker_ip in list(workers):
|
||||
if worker_ip not in list_of_node_ips:
|
||||
del workers[worker_ip]
|
||||
|
||||
# Set external head ip, if provided by user.
|
||||
# Necessary if calling `ray up` from outside the network.
|
||||
# Refer to LocalNodeProvider.external_ip function.
|
||||
external_head_ip = provider_config.get("external_head_ip")
|
||||
if external_head_ip:
|
||||
head = workers[provider_config["head_ip"]]
|
||||
head["external_ip"] = external_head_ip
|
||||
|
||||
assert len(workers) == len(provider_config["worker_ips"]) + 1
|
||||
with open(self.save_path, "w") as f:
|
||||
logger.debug(
|
||||
"ClusterState: Writing cluster state: {}".format(workers)
|
||||
)
|
||||
f.write(json.dumps(workers))
|
||||
|
||||
def get(self):
|
||||
with self.lock:
|
||||
with self.file_lock:
|
||||
workers = json.loads(open(self.save_path).read())
|
||||
return workers
|
||||
|
||||
def put(self, worker_id, info):
|
||||
assert "tags" in info
|
||||
assert "state" in info
|
||||
with self.lock:
|
||||
with self.file_lock:
|
||||
workers = self.get()
|
||||
workers[worker_id] = info
|
||||
with open(self.save_path, "w") as f:
|
||||
logger.info(
|
||||
"ClusterState: "
|
||||
"Writing cluster state: {}".format(list(workers))
|
||||
)
|
||||
f.write(json.dumps(workers))
|
||||
|
||||
|
||||
class OnPremCoordinatorState(ClusterState):
|
||||
"""Generates & updates the state file of CoordinatorSenderNodeProvider.
|
||||
|
||||
Unlike ClusterState, which generates a cluster specific file with
|
||||
predefined head and worker ips, OnPremCoordinatorState overwrites
|
||||
ClusterState's __init__ function to generate and manage a unified
|
||||
file of the status of all the nodes for multiple clusters.
|
||||
"""
|
||||
|
||||
def __init__(self, lock_path, save_path, list_of_node_ips):
|
||||
self.lock = RLock()
|
||||
self.file_lock = FileLock(lock_path)
|
||||
self.save_path = save_path
|
||||
|
||||
with self.lock:
|
||||
with self.file_lock:
|
||||
if os.path.exists(self.save_path):
|
||||
nodes = json.loads(open(self.save_path).read())
|
||||
else:
|
||||
nodes = {}
|
||||
logger.info(
|
||||
"OnPremCoordinatorState: "
|
||||
"Loaded on prem coordinator state: {}".format(nodes)
|
||||
)
|
||||
|
||||
# Filter removed node ips.
|
||||
for node_ip in list(nodes):
|
||||
if node_ip not in list_of_node_ips:
|
||||
del nodes[node_ip]
|
||||
|
||||
for node_ip in list_of_node_ips:
|
||||
if node_ip not in nodes:
|
||||
nodes[node_ip] = {
|
||||
"tags": {},
|
||||
"state": "terminated",
|
||||
}
|
||||
assert len(nodes) == len(list_of_node_ips)
|
||||
with open(self.save_path, "w") as f:
|
||||
logger.info(
|
||||
"OnPremCoordinatorState: "
|
||||
"Writing on prem coordinator state: {}".format(nodes)
|
||||
)
|
||||
f.write(json.dumps(nodes))
|
||||
|
||||
|
||||
class LocalNodeProvider(NodeProvider):
|
||||
"""NodeProvider for private/local clusters.
|
||||
|
||||
`node_id` is overloaded to also be `node_ip` in this class.
|
||||
|
||||
When `cluster_name` is provided, it manages a single cluster in a cluster
|
||||
specific state file. But when `cluster_name` is None, it manages multiple
|
||||
clusters in a unified state file that requires each node to be tagged with
|
||||
TAG_RAY_CLUSTER_NAME in create and non_terminated_nodes function calls to
|
||||
associate each node with the right cluster.
|
||||
|
||||
The current use case of managing multiple clusters is by
|
||||
OnPremCoordinatorServer which receives node provider HTTP requests
|
||||
from CoordinatorSenderNodeProvider and uses LocalNodeProvider to get
|
||||
the responses.
|
||||
"""
|
||||
|
||||
def __init__(self, provider_config, cluster_name):
|
||||
NodeProvider.__init__(self, provider_config, cluster_name)
|
||||
|
||||
if cluster_name:
|
||||
lock_path = get_lock_path(cluster_name)
|
||||
state_path = get_state_path(cluster_name)
|
||||
self.state = ClusterState(
|
||||
lock_path,
|
||||
state_path,
|
||||
provider_config,
|
||||
)
|
||||
self.use_coordinator = False
|
||||
else:
|
||||
# LocalNodeProvider with a coordinator server.
|
||||
self.state = OnPremCoordinatorState(
|
||||
"/tmp/coordinator.lock",
|
||||
"/tmp/coordinator.state",
|
||||
provider_config["list_of_node_ips"],
|
||||
)
|
||||
self.use_coordinator = True
|
||||
|
||||
def non_terminated_nodes(self, tag_filters):
|
||||
workers = self.state.get()
|
||||
matching_ips = []
|
||||
for worker_ip, info in workers.items():
|
||||
if info["state"] == "terminated":
|
||||
continue
|
||||
ok = True
|
||||
for k, v in tag_filters.items():
|
||||
if info["tags"].get(k) != v:
|
||||
ok = False
|
||||
break
|
||||
if ok:
|
||||
matching_ips.append(worker_ip)
|
||||
return matching_ips
|
||||
|
||||
def nodes_for_teardown(self, tag_filters):
|
||||
"""Return all known node ids matching tag_filters regardless of state.
|
||||
|
||||
The local state file on the machine invoking ``ray down`` may show
|
||||
workers as terminated because only the head node's autoscaler updates
|
||||
them to running. During teardown we still need to reach these nodes
|
||||
to stop their Docker containers.
|
||||
|
||||
Nodes that have already been fully torn down (i.e. terminate_node was
|
||||
called, setting teardown_complete) are excluded so they are not
|
||||
targeted again on subsequent teardown passes.
|
||||
"""
|
||||
workers = self.state.get()
|
||||
return [
|
||||
worker_ip
|
||||
for worker_ip, info in workers.items()
|
||||
if all(info["tags"].get(k) == v for k, v in tag_filters.items())
|
||||
and not info.get("teardown_complete", False)
|
||||
]
|
||||
|
||||
def is_running(self, node_id):
|
||||
return self.state.get()[node_id]["state"] == "running"
|
||||
|
||||
def is_terminated(self, node_id):
|
||||
return not self.is_running(node_id)
|
||||
|
||||
def node_tags(self, node_id):
|
||||
return self.state.get()[node_id]["tags"]
|
||||
|
||||
def external_ip(self, node_id):
|
||||
"""Returns an external ip if the user has supplied one.
|
||||
Otherwise, use the same logic as internal_ip below.
|
||||
|
||||
This can be used to call ray up from outside the network, for example
|
||||
if the Ray cluster exists in an AWS VPC and we're interacting with
|
||||
the cluster from a laptop (where using an internal_ip will not work).
|
||||
|
||||
Useful for debugging the local node provider with cloud VMs."""
|
||||
|
||||
node_state = self.state.get()[node_id]
|
||||
ext_ip = node_state.get("external_ip")
|
||||
if ext_ip:
|
||||
return ext_ip
|
||||
else:
|
||||
return socket.gethostbyname(node_id)
|
||||
|
||||
def internal_ip(self, node_id):
|
||||
return socket.gethostbyname(node_id)
|
||||
|
||||
def set_node_tags(self, node_id, tags):
|
||||
with self.state.lock:
|
||||
with self.state.file_lock:
|
||||
info = self.state.get()[node_id]
|
||||
info["tags"].update(tags)
|
||||
self.state.put(node_id, info)
|
||||
|
||||
def create_node(self, node_config, tags, count):
|
||||
"""Creates min(count, currently available) nodes."""
|
||||
node_type = tags[TAG_RAY_NODE_KIND]
|
||||
with self.state.lock:
|
||||
with self.state.file_lock:
|
||||
workers = self.state.get()
|
||||
for node_id, info in workers.items():
|
||||
if info["state"] == "terminated" and (
|
||||
self.use_coordinator
|
||||
or info["tags"][TAG_RAY_NODE_KIND] == node_type
|
||||
):
|
||||
info["tags"] = tags
|
||||
info["state"] = "running"
|
||||
info.pop("teardown_complete", None)
|
||||
self.state.put(node_id, info)
|
||||
count = count - 1
|
||||
if count == 0:
|
||||
return
|
||||
|
||||
def terminate_node(self, node_id):
|
||||
workers = self.state.get()
|
||||
info = workers[node_id]
|
||||
info["state"] = "terminated"
|
||||
info["teardown_complete"] = True
|
||||
self.state.put(node_id, info)
|
||||
|
||||
@staticmethod
|
||||
def bootstrap_config(cluster_config):
|
||||
return bootstrap_local(cluster_config)
|
||||
|
||||
|
||||
def record_local_head_state_if_needed(local_provider: LocalNodeProvider) -> None:
|
||||
"""This function is called on the Ray head from StandardAutoscaler.reset
|
||||
to record the head node's own existence in the cluster state file.
|
||||
|
||||
This is necessary because `provider.create_node` in
|
||||
`commands.get_or_create_head_node` records the head state on the
|
||||
cluster-launching machine but not on the head.
|
||||
"""
|
||||
head_ip = local_provider.provider_config["head_ip"]
|
||||
cluster_name = local_provider.cluster_name
|
||||
# If the head node is not marked as created in the cluster state file,
|
||||
if head_ip not in local_provider.non_terminated_nodes({}):
|
||||
# These tags are based on the ones in commands.get_or_create_head_node;
|
||||
# keep in sync.
|
||||
head_tags = {
|
||||
TAG_RAY_NODE_KIND: NODE_KIND_HEAD,
|
||||
TAG_RAY_USER_NODE_TYPE: LOCAL_CLUSTER_NODE_TYPE,
|
||||
TAG_RAY_NODE_NAME: "ray-{}-head".format(cluster_name),
|
||||
TAG_RAY_NODE_STATUS: STATUS_UP_TO_DATE,
|
||||
}
|
||||
# Mark the head node as created in the cluster state file.
|
||||
local_provider.create_node(node_config={}, tags=head_tags, count=1)
|
||||
|
||||
assert head_ip in local_provider.non_terminated_nodes({})
|
||||
Reference in New Issue
Block a user