wehub/ray-project--ray
1
0
Fork 0
Code Issues Pull Requests Actions Deployments Models Agent Notes Packages Projects Releases Wiki Activity

chore: import upstream snapshot with attribution

Browse Source
This commit is contained in:
wehub-resource-sync
2026-07-13 13:17:40 +08:00
commit f1825c8ceb
10096 changed files with 2364182 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
python/ray/autoscaler/v2/instance_manager/cloud_providers/__init__.py
View File
python/ray/autoscaler/v2/instance_manager/cloud_providers/kuberay/__init__.py
View File
python/ray/autoscaler/v2/instance_manager/cloud_providers/kuberay/cloud_provider.py
+768
View File
@@ -0,0 +1,768 @@
import copy
import logging
import time
from collections import defaultdict
from dataclasses import dataclass, field
from typing import Any, Dict, List, Optional, Set, Tuple
import requests
from ray._raylet import RAY_INTERNAL_NAMESPACE_PREFIX, GcsClient
# TODO(rickyx): We should eventually remove these imports
# when we deprecate the v1 kuberay node provider.
from ray.autoscaler._private.kuberay.node_provider import (
KUBERAY_KIND_HEAD,
KUBERAY_KIND_WORKER,
KUBERAY_LABEL_KEY_KIND,
KUBERAY_LABEL_KEY_TYPE,
RAY_HEAD_POD_NAME,
IKubernetesHttpApiClient,
KubernetesHttpApiClient,
_worker_group_index,
_worker_group_max_replicas,
_worker_group_num_of_hosts,
_worker_group_replicas,
worker_delete_patch,
worker_replica_patch,
)
from ray.autoscaler.v2.instance_manager.cloud_providers.kuberay.ippr_provider import (
KubeRayIPPRProvider,
)
from ray.autoscaler.v2.instance_manager.node_provider import (
CloudInstance,
CloudInstanceId,
CloudInstanceProviderError,
ICloudInstanceProvider,
LaunchNodeError,
NodeKind,
TerminateNodeError,
)
from ray.autoscaler.v2.schema import IPPRSpecs, IPPRStatus, NodeType
logger = logging.getLogger(__name__)
# Annotation the KubeRay operator acts on to terminate the cluster.
NO_DRIVER_TTL_EXPIRED_ANNOTATION = "ray.io/no-driver-ttl-expired"
AUTOSCALER_OPTIONS_KEY = "autoscalerOptions"
NO_DRIVER_TIMEOUT_SECONDS_KEY = "noDriverTimeoutSeconds"
class KubeRayProvider(ICloudInstanceProvider):
"""
This class is a thin wrapper around the Kubernetes API client. It modifies
the RayCluster resource spec on the Kubernetes API server to scale the cluster:
It launches new instances/nodes by submitting patches to the Kubernetes API
to update the RayCluster CRD.
"""
def __init__(
self,
cluster_name: str,
provider_config: Dict[str, Any],
gcs_client: GcsClient,
k8s_api_client: Optional[IKubernetesHttpApiClient] = None,
):
"""
Initializes a new KubeRayProvider.
Args:
cluster_name: The name of the RayCluster resource.
provider_config: The configuration for the RayCluster.
gcs_client: The client to the GCS server. Will be used for resizing raylets.
k8s_api_client: The client to the Kubernetes
API server. This can be used to mock the Kubernetes API server for testing.
"""
self._cluster_name = cluster_name
self._namespace = provider_config["namespace"]
self._k8s_api_client = k8s_api_client or KubernetesHttpApiClient(
namespace=self._namespace
)
self._gcs_client = gcs_client
# Below are states that are cached locally.
self._requests = set()
self._launch_errors_queue = []
self._terminate_errors_queue = []
# Below are states for idle-cluster termination tracking.
# Monotonic timestamp when no driver was first observed; None resets it.
self._no_driver_observed_since: Optional[float] = None
# Latest GCS job end time seen; a newer one means a driver came and went.
self._last_seen_job_end_time = 0
# No-driver timeout (seconds) from the CR; None disables the feature.
self._no_driver_timeout_seconds: Optional[float] = None
# Below are states that are fetched from the Kubernetes API server.
self._ray_cluster = None
self._cached_instances: Dict[CloudInstanceId, CloudInstance]
self._ippr_provider = KubeRayIPPRProvider(
gcs_client=gcs_client, k8s_api_client=self._k8s_api_client
)
@dataclass
class ScaleRequest:
"""Represents a scale request that contains the current states and go-to states
for the ray cluster.
This class will be converted to patches to be submitted to the Kubernetes API
server:
- For launching new instances, it will adjust the `replicas` field in the
workerGroupSpecs.
- For terminating instances, it will adjust the `workersToDelete` field in the
workerGroupSpecs.
"""
# The desired number of workers for each node type.
desired_num_workers: Dict[NodeType, int] = field(default_factory=dict)
# The workers to delete for each node type.
workers_to_delete: Dict[NodeType, List[CloudInstanceId]] = field(
default_factory=dict
)
# The worker groups with empty workersToDelete field.
# This is needed since we will also need to clear the workersToDelete field
# for the worker groups that have finished deletes.
worker_groups_without_pending_deletes: Set[NodeType] = field(
default_factory=set
)
# The worker groups that still have workers to be deleted.
worker_groups_with_pending_deletes: Set[NodeType] = field(default_factory=set)
################################
# Interface for ICloudInstanceProvider
################################
def get_non_terminated(self) -> Dict[CloudInstanceId, CloudInstance]:
self._sync_with_api_server()
self._evaluate_no_driver_termination()
return copy.deepcopy(dict(self._cached_instances))
def terminate(self, ids: List[CloudInstanceId], request_id: str) -> None:
if request_id in self._requests:
# This request is already processed.
logger.warning(f"Request {request_id} is already processed for: {ids}")
return
logger.info("Terminating worker pods: {}".format(ids))
scale_request = None
try:
scale_request = self._initialize_scale_request(
to_launch={}, to_delete_instances=ids
)
if scale_request.worker_groups_with_pending_deletes:
errors_msg = (
"There are workers to be deleted from: "
f"{scale_request.worker_groups_with_pending_deletes}. "
"Waiting for them to be deleted before adding new workers "
" to be deleted"
)
logger.warning(errors_msg)
self._add_terminate_errors(
ids,
request_id,
details=errors_msg,
)
return
self._submit_scale_request(scale_request)
# Only add to processed requests if successful
self._requests.add(request_id)
except Exception as e:
logger.exception(f"Error terminating nodes: {scale_request or 'N/A'}")
self._add_terminate_errors(ids, request_id, details=str(e), e=e)
def launch(self, shape: Dict[NodeType, int], request_id: str) -> None:
if request_id in self._requests:
# This request is already processed.
return
scale_request = None
try:
scale_request = self._initialize_scale_request(
to_launch=shape, to_delete_instances=[]
)
if scale_request.worker_groups_with_pending_deletes:
error_msg = (
"There are workers to be deleted from: "
f"{scale_request.worker_groups_with_pending_deletes}. "
"Waiting for them to be deleted before creating new workers."
)
logger.warning(error_msg)
self._add_launch_errors(
shape,
request_id,
details=error_msg,
)
return
self._submit_scale_request(scale_request)
# Only add to processed requests if successful
self._requests.add(request_id)
except Exception as e:
logger.exception(f"Error launching nodes: {scale_request or 'N/A'}")
self._add_launch_errors(shape, request_id, details=str(e), e=e)
def poll_errors(self) -> List[CloudInstanceProviderError]:
errors = []
errors += self._launch_errors_queue
errors += self._terminate_errors_queue
self._launch_errors_queue = []
self._terminate_errors_queue = []
return errors
def get_ippr_specs(self) -> IPPRSpecs:
"""Return the cached, validated IPPR specs for the cluster.
The IPPR specs are refreshed during the provider's periodic sync with the
API server by reading the RayCluster annotation and validating it against
the IPPR schema.
"""
return self._ippr_provider.get_ippr_specs()
def get_ippr_statuses(self) -> Dict[str, IPPRStatus]:
"""Return the latest per-pod IPPR statuses keyed by pod name.
These statuses are refreshed from the current pod list during the provider's
periodic sync with the API server.
"""
return self._ippr_provider.get_ippr_statuses()
def do_ippr_requests(self, resizes: List[IPPRStatus]) -> None:
"""Execute IPPR resize requests via the underlying IPPR provider.
Args:
resizes: The list of per-pod IPPR actions produced by the scheduler.
"""
self._ippr_provider.do_ippr_requests(resizes)
############################
# Private
############################
def _initialize_scale_request(
self, to_launch: Dict[NodeType, int], to_delete_instances: List[CloudInstanceId]
) -> "KubeRayProvider.ScaleRequest":
"""
Initialize the scale request based on the current state of the cluster and
the desired state (to launch, to delete).
Args:
to_launch: The desired number of workers to launch for each node type.
to_delete_instances: The instances to delete.
Returns:
The scale request.
"""
# Update the cached states.
self._sync_with_api_server()
ray_cluster = self.ray_cluster
cur_instances = self.instances
# Get the worker groups that have pending deletes and the worker groups that
# have finished deletes, and the set of workers included in the workersToDelete
# field of any worker group.
(
worker_groups_with_pending_deletes,
worker_groups_without_pending_deletes,
worker_to_delete_set,
) = self._get_workers_delete_info(ray_cluster, set(cur_instances.keys()))
observed_workers_dict = defaultdict(int)
for instance in cur_instances.values():
if instance.node_kind != NodeKind.WORKER:
continue
if instance.cloud_instance_id in worker_to_delete_set:
continue
observed_workers_dict[instance.node_type] += 1
# Calculate the desired number of workers by type.
num_workers_dict = defaultdict(int)
worker_groups = ray_cluster["spec"].get("workerGroupSpecs", [])
for worker_group in worker_groups:
node_type = worker_group["groupName"]
# Handle the case where users manually increase `minReplicas`
# to scale up the number of worker Pods. In this scenario,
# `replicas` will be smaller than `minReplicas`.
# num_workers_dict should account for multi-host replicas when
# `numOfHosts`` is set.
num_of_hosts = worker_group.get("numOfHosts", 1)
replicas = (
max(worker_group["replicas"], worker_group["minReplicas"])
* num_of_hosts
)
# The `replicas` field in worker group specs can be updated by users at any time.
# However, users should only increase the field (manually upscaling the worker group), not decrease it,
# because downscaling the worker group requires specifying which workers to delete explicitly in the `workersToDelete` field.
# Since we don't have a way to enforce this, we need to fix unexpected decreases on the `replicas` field by using actual observations.
# For example, if the user manually decreases the `replicas` field to 0 without specifying which workers to delete,
# we should fix the `replicas` field back to the number of observed workers excluding the workers to be deleted,
# otherwise, we won't have a correct `replicas` matches the actual number of workers eventually.
num_workers_dict[node_type] = max(
replicas, observed_workers_dict[node_type]
)
# Add to launch nodes.
for node_type, count in to_launch.items():
num_workers_dict[node_type] += count
to_delete_instances_by_type = defaultdict(list)
# Update the number of workers with to_delete_instances
# and group them by type.
for to_delete_id in to_delete_instances:
to_delete_instance = cur_instances.get(to_delete_id, None)
if to_delete_instance is None:
# This instance has already been deleted.
continue
if to_delete_instance.node_kind == NodeKind.HEAD:
# Not possible to delete head node.
continue
if to_delete_instance.cloud_instance_id in worker_to_delete_set:
# If the instance is already in the workersToDelete field of
# any worker group, skip it.
continue
num_workers_dict[to_delete_instance.node_type] -= 1
assert num_workers_dict[to_delete_instance.node_type] >= 0
to_delete_instances_by_type[to_delete_instance.node_type].append(
to_delete_instance
)
scale_request = KubeRayProvider.ScaleRequest(
desired_num_workers=num_workers_dict,
workers_to_delete=to_delete_instances_by_type,
worker_groups_without_pending_deletes=worker_groups_without_pending_deletes,
worker_groups_with_pending_deletes=worker_groups_with_pending_deletes,
)
return scale_request
def _submit_scale_request(
self, scale_request: "KubeRayProvider.ScaleRequest"
) -> None:
"""Submits a scale request to the Kubernetes API server.
This method will convert the scale request to patches and submit the patches
to the Kubernetes API server.
Args:
scale_request: The scale request.
Raises:
Exception: An exception is raised if the Kubernetes API server returns an
error.
"""
# Get the current ray cluster spec.
patch_payload = []
raycluster = self.ray_cluster
# Collect patches for replica counts.
for node_type, num_workers in scale_request.desired_num_workers.items():
group_index = _worker_group_index(raycluster, node_type)
group_max_replicas = _worker_group_max_replicas(raycluster, group_index)
group_num_of_hosts = _worker_group_num_of_hosts(raycluster, group_index)
# the num_workers from the scale request is multiplied by numOfHosts, so we need to divide it back.
target_replicas = num_workers // group_num_of_hosts
# Cap the replica count to maxReplicas.
if group_max_replicas is not None and group_max_replicas < target_replicas:
logger.warning(
"Autoscaler attempted to create "
+ "more than maxReplicas pods of type {}.".format(node_type)
)
target_replicas = group_max_replicas
# Check if we need to change the target count.
if target_replicas == _worker_group_replicas(raycluster, group_index):
# No patch required.
continue
# Need to patch replica count. Format the patch and add it to the payload.
patch = worker_replica_patch(group_index, target_replicas)
patch_payload.append(patch)
# Maps node_type to nodes to delete for that group.
for (
node_type,
workers_to_delete_of_type,
) in scale_request.workers_to_delete.items():
group_index = _worker_group_index(raycluster, node_type)
worker_ids_to_delete = [
worker.cloud_instance_id for worker in workers_to_delete_of_type
]
patch = worker_delete_patch(group_index, worker_ids_to_delete)
patch_payload.append(patch)
# Clear the workersToDelete field for the worker groups that have been deleted.
for node_type in scale_request.worker_groups_without_pending_deletes:
if node_type in scale_request.workers_to_delete:
# This node type is still being deleted.
continue
group_index = _worker_group_index(raycluster, node_type)
patch = worker_delete_patch(group_index, [])
patch_payload.append(patch)
if len(patch_payload) == 0:
# No patch required.
return
logger.info(f"Submitting a scale request: {scale_request}")
self._patch(f"rayclusters/{self._cluster_name}", patch_payload)
def _add_launch_errors(
self,
shape: Dict[NodeType, int],
request_id: str,
details: str,
e: Optional[Exception] = None,
) -> None:
"""
Adds launch errors to the error queue.
Args:
shape: The shape of the nodes that failed to launch.
request_id: The request id of the launch request.
details: The details of the error.
e: The exception that caused the error.
"""
for node_type, count in shape.items():
self._launch_errors_queue.append(
LaunchNodeError(
node_type=node_type,
timestamp_ns=time.time_ns(),
count=count,
request_id=request_id,
details=details,
cause=e,
)
)
def _add_terminate_errors(
self,
ids: List[CloudInstanceId],
request_id: str,
details: str,
e: Optional[Exception] = None,
) -> None:
"""
Adds terminate errors to the error queue.
Args:
ids: The ids of the nodes that failed to terminate.
request_id: The request id of the terminate request.
details: The details of the error.
e: The exception that caused the error.
"""
for id in ids:
self._terminate_errors_queue.append(
TerminateNodeError(
cloud_instance_id=id,
timestamp_ns=time.time_ns(),
request_id=request_id,
details=details,
cause=e,
)
)
def _sync_with_api_server(self) -> None:
"""Fetches the RayCluster resource from the Kubernetes API server."""
self._ray_cluster = self._get(f"rayclusters/{self._cluster_name}")
self._refresh_no_driver_timeout_seconds()
self._ippr_provider.validate_and_set_ippr_specs(self._ray_cluster)
self._cached_instances = self._fetch_instances()
self._ippr_provider.sync_with_raylets()
def _refresh_no_driver_timeout_seconds(self) -> None:
"""Reads noDriverTimeoutSeconds from the RayCluster CR."""
opts = self._ray_cluster["spec"].get(AUTOSCALER_OPTIONS_KEY, {})
secs = opts.get(NO_DRIVER_TIMEOUT_SECONDS_KEY)
self._no_driver_timeout_seconds = float(secs) if secs is not None else None
@property
def ray_cluster(self) -> Dict[str, Any]:
return copy.deepcopy(self._ray_cluster)
@property
def instances(self) -> Dict[CloudInstanceId, CloudInstance]:
return copy.deepcopy(self._cached_instances)
@staticmethod
def _get_workers_delete_info(
ray_cluster_spec: Dict[str, Any], node_set: Set[CloudInstanceId]
) -> Tuple[Set[NodeType], Set[NodeType], Set[CloudInstanceId]]:
"""
Gets the worker groups that have pending deletes and the worker groups that
have finished deletes.
Args:
ray_cluster_spec: The RayCluster CR spec dict.
node_set: The set of currently known cloud instance IDs.
Returns:
A tuple of:
- worker_groups_with_pending_deletes: The worker groups that have pending
deletes.
- worker_groups_with_finished_deletes: The worker groups that have finished
deletes.
- worker_to_delete_set: A set of Pods that are included in the
workersToDelete field of any worker group.
"""
worker_groups_with_pending_deletes = set()
worker_groups_with_deletes = set()
worker_to_delete_set = set()
worker_groups = ray_cluster_spec["spec"].get("workerGroupSpecs", [])
for worker_group in worker_groups:
workersToDelete = worker_group.get("scaleStrategy", {}).get(
"workersToDelete", []
)
if not workersToDelete:
# No workers to delete in this group.
continue
node_type = worker_group["groupName"]
worker_groups_with_deletes.add(node_type)
for worker in workersToDelete:
worker_to_delete_set.add(worker)
if worker in node_set:
worker_groups_with_pending_deletes.add(node_type)
worker_groups_with_finished_deletes = (
worker_groups_with_deletes - worker_groups_with_pending_deletes
)
return (
worker_groups_with_pending_deletes,
worker_groups_with_finished_deletes,
worker_to_delete_set,
)
def _fetch_instances(self) -> Dict[CloudInstanceId, CloudInstance]:
"""
Fetches the pods from the Kubernetes API server and convert them to Ray
CloudInstance.
Returns:
A dict of CloudInstanceId to CloudInstance.
"""
# Get the pods resource version.
# Specifying a resource version in list requests is important for scalability:
# https://kubernetes.io/docs/reference/using-api/api-concepts/#semantics-for-get-and-list
resource_version = self._get_head_pod_resource_version()
if resource_version:
logger.info(
f"Listing pods for RayCluster {self._cluster_name}"
f" in namespace {self._namespace}"
f" at pods resource version >= {resource_version}."
)
# Filter pods by cluster_name.
label_selector = requests.utils.quote(f"ray.io/cluster={self._cluster_name}")
resource_path = f"pods?labelSelector={label_selector}"
if resource_version:
resource_path += (
f"&resourceVersion={resource_version}"
+ "&resourceVersionMatch=NotOlderThan"
)
pod_list = self._get(resource_path)
fetched_resource_version = pod_list["metadata"]["resourceVersion"]
logger.info(
f"Fetched pod data at resource version" f" {fetched_resource_version}."
)
# Extract node data from the pod list.
cloud_instances = {}
for pod in pod_list["items"]:
# Kubernetes sets metadata.deletionTimestamp immediately after admitting a
# request to delete an object. Full removal of the object may take some time
# after the deletion timestamp is set. See link for details:
# https://kubernetes.io/docs/reference/using-api/api-concepts/#resource-deletion
if "deletionTimestamp" in pod["metadata"]:
# Ignore pods marked for termination.
continue
pod_name = pod["metadata"]["name"]
cloud_instance = self._cloud_instance_from_pod(pod)
if cloud_instance:
cloud_instances[pod_name] = cloud_instance
self._ippr_provider.sync_ippr_status_from_pods(pod_list["items"])
return cloud_instances
@staticmethod
def _cloud_instance_from_pod(pod: Dict[str, Any]) -> Optional[CloudInstance]:
"""
Convert a pod to a Ray CloudInstance.
Args:
pod: The pod resource dict.
Returns:
The CloudInstance representing the pod, or None if the pod is not a
tracked Ray node (e.g. a redis-cleanup pod).
"""
labels = pod["metadata"]["labels"]
if labels[KUBERAY_LABEL_KEY_KIND] == KUBERAY_KIND_HEAD:
kind = NodeKind.HEAD
type = labels[KUBERAY_LABEL_KEY_TYPE]
elif labels[KUBERAY_LABEL_KEY_KIND] == KUBERAY_KIND_WORKER:
kind = NodeKind.WORKER
type = labels[KUBERAY_LABEL_KEY_TYPE]
else:
# Other ray nodes types defined by KubeRay.
# e.g. this could also be `redis-cleanup`
# We will not track these nodes.
return None
# TODO: we should prob get from the pod's env var (RAY_CLOUD_INSTANCE_ID)
# directly.
cloud_instance_id = pod["metadata"]["name"]
return CloudInstance(
cloud_instance_id=cloud_instance_id,
node_type=type,
node_kind=kind,
is_running=KubeRayProvider._is_running(pod),
)
@staticmethod
def _is_running(pod) -> bool:
"""Convert pod state to Ray NodeStatus
A cloud instance is considered running if the pod is in the running state,
else it could be pending/containers-terminated.
When it disappears from the list, it is considered terminated.
"""
if (
"containerStatuses" not in pod["status"]
or not pod["status"]["containerStatuses"]
):
return False
state = pod["status"]["containerStatuses"][0]["state"]
if "running" in state:
return True
return False
def _get(self, remote_path: str) -> Dict[str, Any]:
"""Get a resource from the Kubernetes API server."""
return self._k8s_api_client.get(remote_path)
def _patch(self, remote_path: str, payload: List[Dict[str, Any]]) -> Dict[str, Any]:
"""Patch a resource on the Kubernetes API server."""
return self._k8s_api_client.patch(remote_path, payload)
def _evaluate_no_driver_termination(self) -> None:
"""Patches the no-driver-TTL annotation once no driver held for the timeout.
Detached actors do not count as a driver.
"""
# Feature disabled or a driver is attached: reset the anchor.
if self._no_driver_timeout_seconds is None:
self._no_driver_observed_since = None
return
has_active_driver, latest_job_end_time = self._driver_status()
if has_active_driver:
self._no_driver_observed_since = None
return
# A driver finished since the last check: it was attached during the
# no-driver window, so restart the timer.
if latest_job_end_time > self._last_seen_job_end_time:
self._last_seen_job_end_time = latest_job_end_time
self._no_driver_observed_since = None
# Anchor on the first loop with no driver, then dispatch once the
# no-driver window reaches the timeout.
now = time.monotonic()
if self._no_driver_observed_since is None:
self._no_driver_observed_since = now
if now - self._no_driver_observed_since < self._no_driver_timeout_seconds:
return
self._set_no_driver_annotation()
def _driver_status(self) -> Tuple[bool, int]:
"""Returns whether a non-internal driver is alive and the latest job end time.
Fails closed: a failed GCS query reports a driver as present.
"""
try:
jobs = self._gcs_client.get_all_job_info(
skip_submission_job_info_field=True,
skip_is_running_tasks_field=True,
)
except Exception:
logger.exception(
"Failed to query GCS job table; treating as drivers attached."
)
return True, self._last_seen_job_end_time
has_active_driver = False
latest_job_end_time = 0
for job in jobs.values():
# Ray-internal drivers (e.g. the dashboard) are not user activity.
if job.config.ray_namespace.startswith(RAY_INTERNAL_NAMESPACE_PREFIX):
continue
if job.is_dead:
latest_job_end_time = max(latest_job_end_time, job.end_time)
else:
has_active_driver = True
return has_active_driver, latest_job_end_time
def _set_no_driver_annotation(self) -> None:
"""Sets `ray.io/no-driver-ttl-expired=true` on the RayCluster CR.
Idempotent via the CR cached this reconcile loop; PATCH errors are swallowed.
"""
annotations = self._ray_cluster.get("metadata", {}).get("annotations", {})
if annotations.get(NO_DRIVER_TTL_EXPIRED_ANNOTATION) == "true":
return
path = f"rayclusters/{self._cluster_name}"
# Merge patch covers missing and present annotations in one call.
payload = {
"metadata": {"annotations": {NO_DRIVER_TTL_EXPIRED_ANNOTATION: "true"}}
}
try:
self._k8s_api_client.patch(
path,
payload,
content_type="application/merge-patch+json",
)
except Exception:
logger.exception(
"Failed to PATCH %s=true on RayCluster %s",
NO_DRIVER_TTL_EXPIRED_ANNOTATION,
self._cluster_name,
)
return
logger.info(
"Set %s=true on RayCluster %s.",
NO_DRIVER_TTL_EXPIRED_ANNOTATION,
self._cluster_name,
)
def _get_head_pod_resource_version(self) -> str:
"""
Extract a recent pods resource version by reading the head pod's
metadata.resourceVersion of the response.
"""
if not RAY_HEAD_POD_NAME:
return None
pod_resp = self._get(f"pods/{RAY_HEAD_POD_NAME}")
return pod_resp["metadata"]["resourceVersion"]
python/ray/autoscaler/v2/instance_manager/cloud_providers/kuberay/ippr_provider.py
+736
View File
@@ -0,0 +1,736 @@
import json
import logging
import re
import time
from decimal import Decimal
from typing import Any, Dict, List, Optional, Tuple, Union
import jsonschema
from ray._raylet import GcsClient
from ray.autoscaler._private.kuberay.node_provider import (
KUBERAY_KIND_HEAD,
KUBERAY_KIND_WORKER,
KUBERAY_LABEL_KEY_KIND,
KUBERAY_LABEL_KEY_TYPE,
IKubernetesHttpApiClient,
replace_patch,
)
from ray.autoscaler._private.kuberay.utils import parse_quantity
from ray.autoscaler.v2.schema import (
IPPRGroupSpec,
IPPRSpecs,
IPPRSpecsSchema,
IPPRStatus,
)
logger = logging.getLogger(__name__)
class KubeRayIPPRProvider:
"""Implements in-place pod resize (IPPR) operations for KubeRay pods.
This provider is responsible for:
- Validating and materializing IPPR specs from the RayCluster annotation
(``ray.io/ippr``) into typed structures (``IPPRSpecs``/``IPPRGroupSpec``).
- Tracking per-pod resize status (``IPPRStatus``) from Kubernetes pods and
computing the desired resize actions.
- Issuing Kubernetes Pod Resize API requests and keeping a shadow annotation
(``ray.io/ippr-status``) to track progress and temporary caps.
- Synchronizing successful resource changes with the Raylet so Ray's local
resource view matches Kubernetes.
Attributes:
_gcs_client: Ray GCS client used to fetch Raylet node information.
_k8s_api_client: Kubernetes HTTP client for patching pods.
_ippr_specs: Validated per-group IPPR specs (limits and timeouts).
_ippr_statuses: Latest per-pod IPPR statuses indexed by pod name.
_container_resources: Snapshot of container resource requests/limits
from both pod spec and pod status, per pod name, used to compute
patch diffs.
"""
def __init__(
self,
gcs_client: GcsClient,
k8s_api_client: IKubernetesHttpApiClient,
):
"""Create a new IPPR provider.
Args:
gcs_client: Ray GCS client for resolving Raylet addresses.
k8s_api_client: Kubernetes HTTP client to issue patch requests.
"""
self._gcs_client = gcs_client
self._k8s_api_client = k8s_api_client
self._ippr_specs: IPPRSpecs = IPPRSpecs(groups={})
self._ippr_statuses: Dict[str, IPPRStatus] = {}
self._container_resources: Dict[str, Any] = {}
def validate_and_set_ippr_specs(
self, ray_cluster: Optional[Dict[str, Any]]
) -> None:
"""Validate and load IPPR specs from a RayCluster CR.
Reads the ``ray.io/ippr`` annotation, validates it against
``IPPRSpecsSchema``, and converts it to typed ``IPPRSpecs`` with per-group
``IPPRGroupSpec`` entries. Minimal resources are derived from the group's
pod template; maximums and timeout come from the annotation. If the
annotation is removed, clear any previously loaded IPPR specs.
Args:
ray_cluster: The RayCluster custom resource as a dict. If missing or
lacking the annotation, this method is a no-op.
Raises:
ValueError: If the Ray pod template is incompatible with IPPR (e.g.,
missing required requests, using unsupported resizePolicy restarts,
or conflicting ``rayStartParams``).
Example:
import json
ray_cluster = {
"metadata": {
"name": "example-raycluster",
"annotations": {
"ray.io/ippr": json.dumps(
{
"groups": {
"headgroup": {
"max-cpu": "4",
"max-memory": "8Gi",
"resize-timeout": 300,
},
"small-workers": {
"max-cpu": 2,
"max-memory": "4Gi",
"resize-timeout": 120,
},
}
}
),
},
},
"spec": {
"headGroupSpec": {
"rayStartParams": {},
"template": {
"spec": {
"containers": [
{
"name": "ray-head",
"resources": {
"requests": {
"cpu": "1",
"memory": "2Gi",
},
"limits": {
"cpu": "2",
"memory": "4Gi",
},
},
"resizePolicy": [
{
"resourceName": "cpu",
"restartPolicy": "NotRequired",
},
{
"resourceName": "memory",
"restartPolicy": "NotRequired",
},
],
}
],
}
},
},
"workerGroupSpecs": [
{
"groupName": "small-workers",
"rayStartParams": {},
"template": {
"spec": {
"containers": [
{
"name": "ray-worker",
"resources": {
"requests": {
"cpu": "500m",
"memory": "1Gi",
},
},
}
],
}
},
}
],
},
}
provider.validate_and_set_ippr_specs(ray_cluster)
"""
if not ray_cluster:
return
specs_str = ray_cluster["metadata"].get("annotations", {}).get("ray.io/ippr")
if not specs_str:
self._ippr_specs = IPPRSpecs(groups={})
return
ippr_specs_raw = json.loads(specs_str)
jsonschema.validate(instance=ippr_specs_raw, schema=IPPRSpecsSchema)
# Validate and build typed spec per group
worker_groups = {
worker_group_spec["groupName"]: worker_group_spec
for worker_group_spec in ray_cluster["spec"].get("workerGroupSpecs", [])
}
worker_groups["headgroup"] = ray_cluster["spec"]["headGroupSpec"]
groups = {
group_name: _build_ippr_group_spec(group_spec, worker_groups[group_name])
for group_name, group_spec in ippr_specs_raw.get("groups", {}).items()
if group_name in worker_groups
}
self._ippr_specs = IPPRSpecs(groups=groups)
def sync_with_raylets(self) -> None:
"""Propagate completed K8s resizes to Raylets via GCS.
For any pod whose K8s resize has completed, update the corresponding Raylet's local resource
instances via GCS gRPC and clear the pending timestamp on the pod's
``ray.io/ippr-status`` annotation.
Three situations we can have exceptions are:
1. K8s API is not available.
2. GCS is not available.
3. Raylet is not available.
If a raylet is truly dead, its pod will also be deleted eventually.
All of the above exceptions can only be resolved in the future reconcile loops.
"""
for ippr_status in self._ippr_statuses.values():
if not ippr_status.need_sync_with_raylet():
continue
try:
self._gcs_client.resize_raylet_resource_instances(
ippr_status.raylet_id,
{
"CPU": ippr_status.current_cpu,
"memory": ippr_status.current_memory,
},
)
self._patch_ippr_status(ippr_status, resizing_at=None)
logger.info(f"Pod {ippr_status.cloud_instance_id} resized successfully")
except Exception as e:
logger.error(
f"Failed to resize pod {ippr_status.cloud_instance_id}: {e}. "
"If this persists, check GCS (e.g. Head/GCS logs and Raylet reachability) "
"and Kubernetes (e.g. API errors, pod events, ray.io/ippr-status) for "
"related request failures."
)
def sync_ippr_status_from_pods(self, pods: List[Dict[str, Any]]) -> None:
"""Refresh internal IPPR statuses and container resources from pods.
Parses pods to produce up-to-date ``IPPRStatus`` objects and stores
relevant request/limit snapshots for later patch computations.
Args:
pods: List of Kubernetes Pod resources for the Ray cluster.
"""
self._ippr_statuses = {}
self._container_resources = {}
if not self._ippr_specs.groups:
return
for pod in pods:
if "deletionTimestamp" in pod["metadata"]:
continue
labels = pod["metadata"].get("labels", {})
kind = labels.get(KUBERAY_LABEL_KEY_KIND)
if kind not in (KUBERAY_KIND_HEAD, KUBERAY_KIND_WORKER):
continue
ippr_group_spec = self._ippr_specs.groups.get(
labels.get(KUBERAY_LABEL_KEY_TYPE)
)
ippr_status, container_resource = _get_ippr_status_from_pod(
ippr_group_spec, pod
)
if ippr_status:
self._ippr_statuses[ippr_status.cloud_instance_id] = ippr_status
if ippr_status and container_resource:
self._container_resources[
ippr_status.cloud_instance_id
] = container_resource
def do_ippr_requests(self, resizes: List[IPPRStatus]) -> None:
"""Issue Kubernetes Pod Resize requests for the given targets.
If any dimension downsizes, attempt to first adjust the Raylet's local
resources via gRPC to avoid temporary overcommit in Ray's scheduler.
The raylet request uses ``min(desired, current)`` per resource so mixed
resizes (e.g. CPU up and memory down) do not advertise increases before
Kubernetes applies them; the reply is merged so upsizing targets are kept
for the pod patch.
Args:
resizes: List of IPPRStatus with desired resources and metadata.
"""
for resize in resizes:
logger.info(
f"Resizing pod {resize.cloud_instance_id} to cpu={resize.desired_cpu} memory={resize.desired_memory} from cpu={resize.current_cpu} memory={resize.current_memory}"
)
if resize.raylet_id and (
resize.desired_cpu < resize.current_cpu
or resize.desired_memory < resize.current_memory
):
# For any downsize, update Raylet first. Cap each resource at
# current so we never tell the scheduler about an upsize until
# sync_with_raylets runs after K8s applies the pod resize.
try:
updated_total_resources = (
self._gcs_client.resize_raylet_resource_instances(
resize.raylet_id,
{
"CPU": min(resize.desired_cpu, resize.current_cpu),
"memory": min(
resize.desired_memory, resize.current_memory
),
},
)
)
if (
"CPU" not in updated_total_resources
or "memory" not in updated_total_resources
):
raise RuntimeError(
f"CPU or memory not found in the response of resizing raylet resources: {updated_total_resources}"
)
if resize.desired_cpu < resize.current_cpu:
resize.desired_cpu = float(updated_total_resources["CPU"])
if resize.desired_memory < resize.current_memory:
resize.desired_memory = int(updated_total_resources["memory"])
except Exception as e:
logger.error(
f"Skip failed downsizing on pod {resize.cloud_instance_id}: {e}"
)
continue
self._patch_ippr_resize(resize)
def get_ippr_specs(self) -> IPPRSpecs:
"""Return the current validated IPPR specs."""
return self._ippr_specs
def get_ippr_statuses(self) -> Dict[str, IPPRStatus]:
"""Return the latest per-pod IPPR statuses keyed by pod name."""
return self._ippr_statuses
def _patch_ippr_resize(self, resize: IPPRStatus) -> None:
patch = self._make_ippr_patch(resize)
self._k8s_api_client.patch(
"pods/{}/resize".format(resize.cloud_instance_id), patch
)
self._patch_ippr_status(resize, resizing_at=int(time.time()))
def _patch_ippr_status(
self, resize: IPPRStatus, resizing_at: Optional[int]
) -> None:
"""Save the IPPR status to the pod annotation ``ray.io/ippr-status``.
The annotation is used to track the IPPR status of the pod across reconcile loops.
Args:
resize: The IPPR status to save.
resizing_at: Timestamp while a resize is in progress; pass ``None``
to clear after the resize completes (e.g. from ``sync_with_raylets``).
"""
self._k8s_api_client.patch(
"pods/{}".format(resize.cloud_instance_id),
{
"metadata": {
"annotations": {
"ray.io/ippr-status": json.dumps(
{
"raylet-id": resize.raylet_id,
"resizing-at": resizing_at,
"suggested-max-cpu": resize.suggested_max_cpu,
"suggested-max-memory": resize.suggested_max_memory,
"last-failed-at": resize.last_failed_at,
"last-failed-reason": resize.last_failed_reason,
}
)
}
}
},
content_type="application/strategic-merge-patch+json",
)
def _make_ippr_patch(self, resize: IPPRStatus) -> List[Dict[str, Any]]:
patch = []
path_prefix = "/spec/containers/0/resources"
container_resource = self._container_resources[resize.cloud_instance_id]
# When limits are present, preserve the existing gap (limits - requests)
# by adjusting requests proportionally so QoS doesn't change.
for resource_name, desired in (
("cpu", resize.desired_cpu),
("memory", resize.desired_memory),
):
if container_resource["status"]["limits"].get(resource_name):
# Gap between status limits and requests for each resource.
diff = _resource_gap(
container_resource["status"]["limits"],
container_resource["status"]["requests"],
resource_name,
)
patch.append(
replace_patch(f"{path_prefix}/limits/{resource_name}", desired)
)
patch.append(
replace_patch(
f"{path_prefix}/requests/{resource_name}",
_request_from_desired(desired, diff),
)
)
else:
# No limits configured: adjust requests only.
patch.append(
replace_patch(f"{path_prefix}/requests/{resource_name}", desired)
)
return patch
def _build_ippr_group_spec(
group_spec: Dict[str, Any], worker_group_spec: Dict[str, Any]
) -> IPPRGroupSpec:
# Disallow per-pod overrides that conflict with IPPR's dynamic sizing.
ray_start_params = worker_group_spec.get("rayStartParams", {})
if "num-cpus" in ray_start_params or "memory" in ray_start_params:
raise ValueError(
"should not have 'num-cpus' or 'memory' in rayStartParams if IPPR is used"
)
container_spec = worker_group_spec["template"]["spec"]["containers"][0]
pod_spec_requests = container_spec.get("resources", {}).get("requests", {})
# Pod template must declare baseline CPU/memory requests for IPPR.
if "cpu" not in pod_spec_requests or "memory" not in pod_spec_requests:
raise ValueError(
"should have 'cpu' and 'memory' in resource requests as the resources lower bounds if IPPR is used"
)
for policy in container_spec.get("resizePolicy", []):
resource_name = policy.get("resourceName")
if resource_name != "cpu" and resource_name != "memory":
continue
restart = policy.get("restartPolicy")
# IPPR requires NotRequired so that K8s won't restart the container
# during in-place resource updates.
if restart is not None and restart != "NotRequired":
raise ValueError("IPPR only supports restartPolicy=NotRequired")
# pod_spec_limits are the initial resource limits specified for the pod.
# we use it together with pod_spec_requests to derive the lower bounds for IPPR.
pod_spec_limits = container_spec.get("resources", {}).get("limits", {})
return IPPRGroupSpec(
min_cpu=_resource_value(pod_spec_requests, pod_spec_limits, "cpu", float),
min_memory=_resource_value(pod_spec_requests, pod_spec_limits, "memory", int),
max_cpu=float(parse_quantity(group_spec.get("max-cpu"))),
max_memory=int(parse_quantity(group_spec.get("max-memory"))),
resize_timeout=int(group_spec.get("resize-timeout")),
)
def _get_ippr_status_from_pod(
ippr_group_spec: Optional[IPPRGroupSpec],
pod: Dict[str, Any],
) -> Tuple[Optional[IPPRStatus], Optional[Dict[str, Any]]]:
"""Build IPPRStatus and container resource snapshots from a Pod.
Returns a tuple of (ippr_status, container_resource_snapshot). The snapshot
contains both spec and status requests/limits used to construct resize
patches that preserve the current QoS gap.
IPPRStatus includes the updated desired resources based on the failure messages of the previous resize request carried on the pod.
This function doesn't have any external side effects.
"""
if not ippr_group_spec:
return (None, None)
container_status = {}
other_container_resources = []
for status in pod.get("status", {}).get("containerStatuses", []):
if status["name"] == pod["spec"]["containers"][0]["name"]:
container_status = status
else:
# We need to substract other containers' resources when adjusting the
# the new resource requests based on the capactity report from the Kubelet.
requests = status.get("resources", {}).get("requests")
if requests:
other_container_resources.append(requests)
pod_spec_requests = (
pod["spec"]["containers"][0].get("resources", {}).get("requests", {})
)
pod_spec_limits = (
pod["spec"]["containers"][0].get("resources", {}).get("limits", {})
)
pod_status_requests = container_status.get("resources", {}).get(
"requests", pod_spec_requests
)
pod_status_limits = container_status.get("resources", {}).get(
"limits", pod_spec_limits
)
ippr_status = IPPRStatus(
cloud_instance_id=pod["metadata"]["name"],
spec=ippr_group_spec,
desired_cpu=_resource_value(pod_spec_requests, pod_spec_limits, "cpu", float),
desired_memory=_resource_value(
pod_spec_requests, pod_spec_limits, "memory", int
),
current_cpu=_resource_value(
pod_status_requests, pod_status_limits, "cpu", float
),
current_memory=_resource_value(
pod_status_requests, pod_status_limits, "memory", int
),
)
ippr_status = _restore_ippr_status_from_annotation(ippr_status, pod)
ippr_status = _apply_resize_conditions(
ippr_status=ippr_status,
pod=pod,
pod_status_requests=pod_status_requests,
pod_status_limits=pod_status_limits,
other_container_resources=other_container_resources,
)
ippr_status = _handle_failed_or_timed_out_ippr(ippr_status)
return (
ippr_status,
{
"spec": {
"requests": pod_spec_requests,
"limits": pod_spec_limits,
},
"status": {
"requests": pod_status_requests,
"limits": pod_status_limits,
},
},
)
def _restore_ippr_status_from_annotation(
ippr_status: IPPRStatus, pod: Dict[str, Any]
) -> IPPRStatus:
"""Restore previously persisted IPPR status fields from pod annotations."""
pod_ippr_status_json = (
pod["metadata"].get("annotations", {}).get("ray.io/ippr-status")
)
if not pod_ippr_status_json:
return ippr_status
pod_ippr_status = json.loads(pod_ippr_status_json)
ippr_status.raylet_id = pod_ippr_status.get("raylet-id")
ippr_status.resizing_at = pod_ippr_status.get("resizing-at")
ippr_status.suggested_max_cpu = pod_ippr_status.get("suggested-max-cpu")
ippr_status.suggested_max_memory = pod_ippr_status.get("suggested-max-memory")
ippr_status.last_failed_at = pod_ippr_status.get("last-failed-at")
ippr_status.last_failed_reason = pod_ippr_status.get("last-failed-reason")
return ippr_status
def _apply_resize_conditions(
ippr_status: IPPRStatus,
pod: Dict[str, Any],
pod_status_requests: Dict[str, Any],
pod_status_limits: Dict[str, Any],
other_container_resources: List[Dict[str, Any]],
) -> IPPRStatus:
"""Parse pod resize conditions and queue any follow-up suggestions."""
for condition in pod.get("status", {}).get("conditions", []):
if condition["type"] == "PodResizePending" and condition["status"] == "True":
ippr_status.k8s_resize_message = condition.get("message")
ippr_status.k8s_resize_status = condition.get("reason", "").lower()
ippr_status = _apply_resize_suggestion(
ippr_status=ippr_status,
pod_status_requests=pod_status_requests,
pod_status_limits=pod_status_limits,
other_container_resources=other_container_resources,
)
break
if condition["type"] == "PodResizeInProgress" and condition["status"] == "True":
ippr_status.k8s_resize_message = condition.get("message")
ippr_status.k8s_resize_status = "inprogress"
if condition.get("reason") == "Error":
ippr_status.k8s_resize_status = "error"
break
return ippr_status
def _apply_resize_suggestion(
ippr_status: IPPRStatus,
pod_status_requests: Dict[str, Any],
pod_status_limits: Dict[str, Any],
other_container_resources: List[Dict[str, Any]],
) -> IPPRStatus:
"""Parse Kubelet capacity reports and queue a suggested follow-up resize.
A suggested follow-up resize is taken from the failure message of the previous resize request on the pod.
For example, a message, "Node didn't have enough resource: cpu, requested: 8000, used: 5000, capacity: 9000",
means the pod can only have CPU request up to 4. For our suggested follow-up resize, we will set the suggested_max_cpu to
4 + (orignal cpu limit - original cpu request) to keep the gap.
"""
report = None
if ippr_status.k8s_resize_message and ippr_status.k8s_resize_status == "deferred":
report = re.search(
r"Node didn't have enough resource: (cpu|memory), requested: (\d+), used: (\d+), capacity: (\d+)",
ippr_status.k8s_resize_message,
)
elif (
ippr_status.k8s_resize_message and ippr_status.k8s_resize_status == "infeasible"
):
report = re.search(
r"Node didn't have enough capacity: (cpu|memory), requested: (\d+), ()capacity: (\d+)",
ippr_status.k8s_resize_message,
)
if not report:
return ippr_status
# Example (resize to max-cpu = 8, max-memory = 20Gi):
# Initial pod status:
# - CPU limit is 4 cores
# - CPU request is 1 core (gap = 3 cores)
# - Mem limit is 8Gi
# - Mem request is 2Gi (gap = 6Gi)
# Initial resize request will be:
# - desired_cpu=8 cores
# - desired_memory=20Gi
# The actual resize patch will be:
# - CPU limit is 8 cores (upsize from 4)
# - CPU request is 8 - 3 = 5 cores (upsize from 1, keep the gap 3 cores)
# - Mem limit is 20Gi (upsize from 8Gi)
# - Mem request is 20 − 6 = 14Gi (upsize from 2Gi, keep the gap 6Gi)
# If Kubelet reports (the deferred case):
# - CPU: used=5, capacity=9 → remaining_cpu = 4 cores
# - Mem: used=6Gi, capacity=10Gi → remaining_mem = 4Gi
# The suggestions used in the next patch will be:
# - suggested_max_cpu = remaining_cpu + cpu_gap = 4 + 3 = 7 cores
# - suggested_max_memory = remaining_mem + mem_gap = 4Gi + 6Gi = 10Gi
# The actual resize patch will be:
# - CPU limit is 7 cores
# - CPU request is 7 - 3 = 4 cores (aligned with the kubelet's report, and keep the gap 3 cores)
# - Mem limit is 10Gi
# - Mem request is 10Gi - 6Gi = 4Gi (aligned with the kubelet's report, and keep the gap 6Gi)
used = int(
report.group(3) or "0"
) # this field is the used resource request capacity of the k8s node excluding the current pod.
capacity = int(
report.group(4)
) # this field is the total resource request capacity of the k8s node.
max_request = (
capacity - used
) # so this max_request is the remaining resource request capacity that this pod can still request.
resource_name = report.group(1)
suggested = _suggested_resize_limit(
resource_name,
max_request,
pod_status_requests,
pod_status_limits,
other_container_resources,
)
if resource_name == "cpu":
ippr_status.suggested_max_cpu = float(suggested)
if ippr_status.queue_resize_request(desired_cpu=ippr_status.suggested_max_cpu):
logger.info(
f"Apply resize suggestions for {ippr_status.cloud_instance_id} to cpu={ippr_status.suggested_max_cpu}"
)
else:
ippr_status.suggested_max_memory = int(suggested)
if ippr_status.queue_resize_request(
desired_memory=ippr_status.suggested_max_memory,
):
logger.info(
f"Apply resize suggestions for {ippr_status.cloud_instance_id} to memory={ippr_status.suggested_max_memory}"
)
return ippr_status
def _suggested_resize_limit(
resource_name: str,
max_request: int,
pod_status_requests: Dict[str, Any],
pod_status_limits: Dict[str, Any],
other_container_resources: List[Dict[str, Any]],
) -> Union[float, int]:
gap = _resource_gap(pod_status_limits, pod_status_requests, resource_name)
other_requests = sum(
parse_quantity(requests.get(resource_name, "0"))
for requests in other_container_resources
)
if resource_name == "cpu":
available = Decimal(str(max_request)) / 1000
return float(available + gap - other_requests)
else:
available = Decimal(str(max_request))
return int(available + gap - other_requests)
def _resource_value(
requests: Dict[str, Any],
limits: Dict[str, Any],
resource_name: str,
value_type: Union[type[float], type[int]],
) -> Union[float, int]:
return value_type(
parse_quantity(limits.get(resource_name) or requests.get(resource_name))
)
def _resource_gap(
limits: Dict[str, Any],
requests: Dict[str, Any],
resource_name: str,
) -> Decimal:
return parse_quantity(
limits.get(resource_name) or requests.get(resource_name)
) - parse_quantity(requests.get(resource_name))
def _request_from_desired(
desired: Union[float, int], gap: Decimal
) -> Union[float, int]:
requested = Decimal(str(desired)) - gap
return type(desired)(requested)
def _handle_failed_or_timed_out_ippr(ippr_status: IPPRStatus) -> IPPRStatus:
"""Record terminal IPPR failures and queue a revert to current resources."""
if not (ippr_status.is_errored() or ippr_status.is_timeout()):
return ippr_status
if ippr_status.last_failed_at is None:
if ippr_status.is_errored():
ippr_status.record_failure(
reason=ippr_status.k8s_resize_message or "Pod resize failed"
)
else:
ippr_status.record_failure(reason="Pod resize timed out")
if ippr_status.queue_resize_request(
desired_cpu=ippr_status.current_cpu,
desired_memory=ippr_status.current_memory,
):
logger.info(
f"Revert failed or stuck IPPR for {ippr_status.cloud_instance_id} to cpu={ippr_status.current_cpu} memory={ippr_status.current_memory}"
)
return ippr_status
python/ray/autoscaler/v2/instance_manager/cloud_providers/read_only/__init__.py
View File
python/ray/autoscaler/v2/instance_manager/cloud_providers/read_only/cloud_provider.py
+73
View File
@@ -0,0 +1,73 @@
from typing import Dict, List
from ray._common.utils import binary_to_hex
from ray._raylet import GcsClient
from ray.autoscaler._private.util import format_readonly_node_type
from ray.autoscaler.v2.instance_manager.node_provider import (
CloudInstance,
CloudInstanceId,
CloudInstanceProviderError,
ICloudInstanceProvider,
NodeKind,
)
from ray.autoscaler.v2.sdk import get_cluster_resource_state
from ray.autoscaler.v2.utils import is_head_node
from ray.core.generated.autoscaler_pb2 import NodeStatus
class ReadOnlyProvider(ICloudInstanceProvider):
"""
A read only provider that use the ray node states from the GCS as the
cloud instances.
This is used for laptop mode / manual cluster setup modes, in order to
provide status reporting in the same way for users.
"""
def __init__(self, provider_config: dict):
self._provider_config = provider_config
self._gcs_address = provider_config["gcs_address"]
self._gcs_client = GcsClient(address=self._gcs_address)
def get_non_terminated(self) -> Dict[str, CloudInstance]:
cluster_resource_state = get_cluster_resource_state(self._gcs_client)
cloud_instances = {}
for gcs_node_state in cluster_resource_state.node_states:
if gcs_node_state.status == NodeStatus.DEAD:
# Skip dead nodes.
continue
# Use node's node id if instance id is not available
cloud_instance_id = (
gcs_node_state.instance_id
if gcs_node_state.instance_id
else binary_to_hex(gcs_node_state.node_id)
)
# TODO: we should add a field to the proto to indicate if the node is head
# or not.
is_head = is_head_node(gcs_node_state)
cloud_instances[cloud_instance_id] = CloudInstance(
cloud_instance_id=cloud_instance_id,
node_kind=NodeKind.HEAD if is_head else NodeKind.WORKER,
node_type=format_readonly_node_type(
binary_to_hex(gcs_node_state.node_id) # Legacy behavior.
),
is_running=True,
request_id="",
)
return cloud_instances
def terminate(self, ids: List[CloudInstanceId], request_id: str) -> None:
raise NotImplementedError("Cannot terminate instances in read-only mode.")
def launch(
self, shape: Dict[CloudInstanceId, int], request_id: CloudInstanceId
) -> None:
raise NotImplementedError("Cannot launch instances in read-only mode.")
def poll_errors(self) -> List[CloudInstanceProviderError]:
return []