wehub-resource-sync
283 lines
12 KiB
Python
283 lines
12 KiB
Python
import logging
|
|
from typing import Optional
|
|
|
|
import ray._common.utils
|
|
import ray._private.ray_constants as ray_constants
|
|
import ray._private.utils as utils
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
# See https://docs.kernel.org/admin-guide/cgroup-v2.html#weights
|
|
# for information about cpu weights
|
|
_CGROUP_CPU_MAX_WEIGHT: int = 10000
|
|
|
|
|
|
class ResourceIsolationConfig:
|
|
"""Configuration for enabling resource isolation by reserving memory and cpu for ray system processes through cgroupv2.
|
|
|
|
Validates configuration for resource isolation by enforcing types, correct combinations of values, applying default values,
|
|
and sanity checking cpu and memory reservations. Also, converts system_reserved_cpu into cpu.weights for cgroupv2.
|
|
|
|
Attributes:
|
|
enable_resource_isolation: True if cgroupv2 based isolation of ray
|
|
system processes is enabled.
|
|
cgroup_path: The path for the cgroup the raylet should use to enforce
|
|
resource isolation.
|
|
system_reserved_cpu: The amount of cores reserved for ray system
|
|
processes. Must be >= ray_constants.MINIMUM_SYSTEM_RESERVED_CPU_CORES
|
|
and < the total number of cores available.
|
|
system_reserved_memory: The amount of memory in bytes reserved
|
|
for ray system processes. Must be >= ray_constants.MINIMUM_SYSTEM_RESERVED_MEMORY_BYTES
|
|
and < the total memory available.
|
|
|
|
TODO(54703): Link documentation when it's available.
|
|
"""
|
|
|
|
def __init__(
|
|
self,
|
|
enable_resource_isolation: bool = False,
|
|
cgroup_path: Optional[str] = None,
|
|
system_reserved_cpu: Optional[float] = None,
|
|
system_reserved_memory: Optional[int] = None,
|
|
):
|
|
"""
|
|
Raises:
|
|
ValueError: On invalid inputs.
|
|
|
|
Args:
|
|
enable_resource_isolation: True if cgroupv2 based isolation of ray
|
|
system processes is enabled.
|
|
cgroup_path: The path for the cgroup the raylet should use to enforce
|
|
resource isolation.
|
|
system_reserved_cpu: The amount of cores reserved for ray system
|
|
processes. Must be >= ray_constants.MINIMUM_SYSTEM_RESERVED_CPU_CORES
|
|
and < the total number of cores available.
|
|
system_reserved_memory: The amount of memory in bytes reserved
|
|
for ray system processes. Must be >= ray_constants.MINIMUM_SYSTEM_RESERVED_MEMORY_BYTES
|
|
and < the total memory available.
|
|
"""
|
|
self._resource_isolation_enabled = enable_resource_isolation
|
|
self.cgroup_path = cgroup_path
|
|
self.system_reserved_memory = system_reserved_memory
|
|
self.system_pids = ""
|
|
|
|
# cgroupv2 cpu.weight calculated from system_reserved_cpu assumes ray uses all available cores.
|
|
self.system_reserved_cpu_weight: int = None
|
|
|
|
if not enable_resource_isolation:
|
|
if self.cgroup_path:
|
|
raise ValueError(
|
|
"cgroup_path cannot be set when resource isolation is not enabled. "
|
|
"Set enable_resource_isolation to True if you're using ray.init or use the "
|
|
"--enable-resource-isolation flag if you're using the ray cli."
|
|
)
|
|
if system_reserved_cpu is not None:
|
|
raise ValueError(
|
|
"system_reserved_cpu cannot be set when resource isolation is not enabled. "
|
|
"Set enable_resource_isolation to True if you're using ray.init or use the "
|
|
"--enable-resource-isolation flag if you're using the ray cli."
|
|
)
|
|
|
|
if system_reserved_memory is not None:
|
|
raise ValueError(
|
|
"system_reserved_memory cannot be set when resource isolation is not enabled. "
|
|
"Set enable_resource_isolation to True if you're using ray.init or use the "
|
|
"--enable-resource-isolation flag if you're using the ray cli."
|
|
)
|
|
return
|
|
|
|
self.system_reserved_cpu_weight = self._validate_and_get_system_reserved_cpu(
|
|
system_reserved_cpu
|
|
)
|
|
|
|
self.system_reserved_memory = self._validate_and_get_system_reserved_memory(
|
|
system_reserved_memory
|
|
)
|
|
|
|
self.cgroup_path = self._validate_and_get_cgroup_path(cgroup_path)
|
|
|
|
def is_enabled(self) -> bool:
|
|
return self._resource_isolation_enabled
|
|
|
|
def add_system_pids(self, system_pids: str):
|
|
"""A comma-separated list of pids to move into the system cgroup."""
|
|
self.system_pids = system_pids
|
|
|
|
@staticmethod
|
|
def _validate_and_get_cgroup_path(cgroup_path: Optional[str]) -> str:
|
|
"""Returns the ray_constants.DEFAULT_CGROUP_PATH if cgroup_path is not specified.
|
|
|
|
Args:
|
|
cgroup_path: The path for the cgroup the raylet should use to enforce
|
|
resource isolation.
|
|
|
|
Returns:
|
|
str: The validated cgroup path.
|
|
|
|
Raises:
|
|
ValueError: If cgroup_path is not a string.
|
|
"""
|
|
if not cgroup_path:
|
|
cgroup_path = ray_constants.DEFAULT_CGROUP_PATH
|
|
|
|
if not isinstance(cgroup_path, str):
|
|
raise ValueError(
|
|
f"Invalid value={cgroup_path} for cgroup_path. "
|
|
"Use a string to represent the path for the cgroup that the raylet should use "
|
|
"to enable resource isolation."
|
|
)
|
|
|
|
return cgroup_path
|
|
|
|
@staticmethod
|
|
def _validate_and_get_system_reserved_cpu(
|
|
system_reserved_cpu: Optional[float],
|
|
) -> int:
|
|
"""If system_reserved_cpu is specified, validates it, otherwise returns the default value.
|
|
|
|
Validation entails checking the type, ensuring that the value is in range, and converts it
|
|
into cpu.weights for cgroupv2. See https://docs.kernel.org/admin-guide/cgroup-v2.html#weights
|
|
for more information.
|
|
|
|
If system_reserved_cpu is not specified, returns a default value between
|
|
[DEFAULT_MIN_SYSTEM_RESERVED_CPU_CORES, DEFAULT_MAX_SYSTEM_RESERVED_CPU_CORES].
|
|
|
|
# TODO(54703): The errors from this method are user-facing and thus need
|
|
to be linked the user-facing documentation once it's available.
|
|
|
|
Args:
|
|
system_reserved_cpu: The amount of cores reserved for ray system
|
|
processes. Must be >= ray_constants.MINIMUM_SYSTEM_RESERVED_CPU_CORES
|
|
and < the total number of cores available.
|
|
|
|
Raises:
|
|
ValueError: If system_reserved_cpu is specified, but invalid or if the system
|
|
does not have enough available cpus.
|
|
|
|
Returns:
|
|
The cgroup v2 cpu.weight value derived from the reserved cpu cores.
|
|
"""
|
|
available_system_cpus = utils.get_num_cpus(truncate=False)
|
|
|
|
if available_system_cpus < ray_constants.DEFAULT_MIN_SYSTEM_RESERVED_CPU_CORES:
|
|
raise ValueError(
|
|
f"The available number of cpu cores on this system {available_system_cpus} is less than "
|
|
f"the minimum amount that is required for ray's system processes. "
|
|
f"Pick a number of cpu cores greater than or equal to {ray_constants.DEFAULT_MIN_SYSTEM_RESERVED_CPU_CORES}"
|
|
)
|
|
|
|
if system_reserved_cpu is None:
|
|
system_reserved_cpu = float(
|
|
min(
|
|
max(
|
|
ray_constants.DEFAULT_MIN_SYSTEM_RESERVED_CPU_CORES,
|
|
ray_constants.DEFAULT_SYSTEM_RESERVED_CPU_PROPORTION
|
|
* available_system_cpus,
|
|
),
|
|
ray_constants.DEFAULT_MAX_SYSTEM_RESERVED_CPU_CORES,
|
|
)
|
|
)
|
|
|
|
if not (
|
|
isinstance(system_reserved_cpu, float)
|
|
or isinstance(system_reserved_cpu, int)
|
|
):
|
|
raise ValueError(
|
|
f"Invalid value={system_reserved_cpu} for system_reserved_cpu. "
|
|
"Use a float to represent the number of cores that need to be reserved for "
|
|
"ray system processes to enable resource isolation."
|
|
)
|
|
|
|
system_reserved_cpu = float(system_reserved_cpu)
|
|
|
|
if system_reserved_cpu < ray_constants.DEFAULT_MIN_SYSTEM_RESERVED_CPU_CORES:
|
|
raise ValueError(
|
|
f"The requested system_reserved_cpu={system_reserved_cpu} is less than "
|
|
f"the minimum number of cpus that can be used for resource isolation. "
|
|
"Pick a number of cpu cores to reserve for ray system processes "
|
|
f"greater than or equal to {ray_constants.DEFAULT_MIN_SYSTEM_RESERVED_CPU_CORES}"
|
|
)
|
|
|
|
if system_reserved_cpu >= available_system_cpus:
|
|
raise ValueError(
|
|
f"The requested system_reserved_cpu={system_reserved_cpu} is greater than or equal to "
|
|
f"the number of cpus available={available_system_cpus}. "
|
|
"Pick a smaller number of cpu cores to reserve for ray system processes."
|
|
)
|
|
|
|
# Converting the number of cores the user defined into cpu.weights
|
|
# This assumes that ray is allowed to use all available CPU
|
|
# cores and distribute them between system, worker and
|
|
# user processes
|
|
return int(
|
|
(system_reserved_cpu / float(available_system_cpus))
|
|
* _CGROUP_CPU_MAX_WEIGHT
|
|
)
|
|
|
|
@staticmethod
|
|
def _validate_and_get_system_reserved_memory(
|
|
system_reserved_memory: Optional[int],
|
|
) -> int:
|
|
"""If system_reserved_memory is not specified, returns the default value. Otherwise,
|
|
checks the type, makes sure that the value is in range.
|
|
|
|
Args:
|
|
system_reserved_memory: The amount of memory in bytes reserved
|
|
for ray system processes. Must be >= ray_constants.MINIMUM_SYSTEM_RESERVED_MEMORY_BYTES
|
|
and < the total memory available.
|
|
|
|
Returns:
|
|
int: The validated system reserved memory in bytes.
|
|
|
|
Raises:
|
|
ValueError: If system_reserved_memory is specified, but invalid.
|
|
"""
|
|
available_system_memory = ray._common.utils.get_system_memory()
|
|
|
|
if (
|
|
available_system_memory
|
|
< ray_constants.DEFAULT_MIN_SYSTEM_RESERVED_MEMORY_BYTES
|
|
):
|
|
raise ValueError(
|
|
f"The available memory on this system {available_system_memory} is less than "
|
|
f"the minimum amount that is required for ray's system processes. "
|
|
f"Pick a number of bytes greater than or equal to {ray_constants.DEFAULT_MIN_SYSTEM_RESERVED_MEMORY_BYTES}"
|
|
)
|
|
|
|
if system_reserved_memory is None:
|
|
system_reserved_memory = int(
|
|
min(
|
|
max(
|
|
ray_constants.DEFAULT_MIN_SYSTEM_RESERVED_MEMORY_BYTES,
|
|
ray_constants.DEFAULT_SYSTEM_RESERVED_MEMORY_PROPORTION
|
|
* available_system_memory,
|
|
),
|
|
ray_constants.DEFAULT_MAX_SYSTEM_RESERVED_MEMORY_BYTES,
|
|
)
|
|
)
|
|
|
|
if not isinstance(system_reserved_memory, int):
|
|
raise ValueError(
|
|
f"Invalid value {system_reserved_memory} for system_reserved_memory. "
|
|
"Use an integer to represent the number bytes that need to be reserved for "
|
|
"ray system processes to enable resource isolation."
|
|
)
|
|
|
|
if (
|
|
system_reserved_memory
|
|
< ray_constants.DEFAULT_MIN_SYSTEM_RESERVED_MEMORY_BYTES
|
|
):
|
|
raise ValueError(
|
|
f"The requested system_reserved_memory {system_reserved_memory} is less than "
|
|
f"the minimum number of bytes that can be used for resource isolation. "
|
|
"Pick a number of bytes to reserve for ray system processes "
|
|
f"greater than or equal to {ray_constants.DEFAULT_MIN_SYSTEM_RESERVED_MEMORY_BYTES}"
|
|
)
|
|
|
|
if system_reserved_memory > available_system_memory:
|
|
raise ValueError(
|
|
f"The total requested system_reserved_memory={system_reserved_memory} "
|
|
f"is greater than the amount of memory available={available_system_memory}."
|
|
)
|
|
return system_reserved_memory
|