wehub-resource-sync
893 lines
28 KiB
Python
893 lines
28 KiB
Python
import asyncio
|
|
import collections
|
|
import copy
|
|
import errno
|
|
import importlib
|
|
import inspect
|
|
import logging
|
|
import random
|
|
import re
|
|
import time
|
|
import uuid
|
|
import zlib
|
|
from decimal import ROUND_HALF_UP, Decimal
|
|
from enum import Enum
|
|
from functools import wraps
|
|
from typing import Any, Callable, Dict, List, Optional, Set, TypeVar, Union
|
|
|
|
import requests
|
|
|
|
import ray
|
|
import ray.util.serialization_addons
|
|
from ray import cloudpickle
|
|
from ray._common.constants import HEAD_NODE_RESOURCE_NAME
|
|
from ray._common.utils import get_random_alphanumeric_string, import_attr
|
|
from ray._raylet import MessagePackSerializer
|
|
from ray.actor import ActorHandle
|
|
from ray.serve._private.common import DeploymentID, RequestMetadata, ServeComponentType
|
|
from ray.serve._private.constants import (
|
|
HTTP_PROXY_TIMEOUT,
|
|
SERVE_DEPLOYMENT_ACTOR_PREFIX,
|
|
SERVE_LOGGER_NAME,
|
|
SERVE_NAMESPACE,
|
|
)
|
|
from ray.types import ObjectRef
|
|
from ray.util.serialization import StandaloneSerializationContext
|
|
|
|
try:
|
|
import pandas as pd
|
|
except ImportError:
|
|
pd = None
|
|
|
|
try:
|
|
import numpy as np
|
|
except ImportError:
|
|
np = None
|
|
|
|
FILE_NAME_REGEX = r"[^\x20-\x7E]|[<>:\"/\\|?*]"
|
|
|
|
MESSAGE_PACK_OFFSET = 9
|
|
|
|
# Attribute set on functions/methods decorated with `@serve.multiplexed`. The
|
|
# `__serve_multiplex_wrapper` is only created lazily on the first call, so this
|
|
# marker is used to detect multiplexing statically (e.g. at replica startup)
|
|
# without invoking user code.
|
|
MULTIPLEXED_FUNCTION_MARKER_ATTR = "_serve_multiplexed_function"
|
|
|
|
|
|
def _callable_uses_multiplexing(callable_obj: Any) -> bool:
|
|
"""Whether `callable_obj` is or defines an `@serve.multiplexed` function.
|
|
|
|
Accepts a standalone function, a class, or a class instance, so it can be used
|
|
both at build time (where the deployment's `func_or_class` is available) and at
|
|
runtime (where an initialized instance is available).
|
|
|
|
For an instance it also inspects instance attributes, so multiplexing that is
|
|
wired up dynamically at init time (e.g. ``self._load_model =
|
|
serve.multiplexed(...)(fn)``) is detected. This case can only be caught at
|
|
runtime, since it is not visible on the class statically.
|
|
"""
|
|
# NOTE: the marker is checked with `is True` rather than truthiness because some
|
|
# objects (e.g. `DeploymentHandle`, whose `__getattr__` returns a handle for any
|
|
# name) return a truthy value for an arbitrary attribute. The decorator always
|
|
# sets the marker to the literal `True`, so this stays exact without false
|
|
# positives.
|
|
def _has_marker(obj: Any) -> bool:
|
|
return getattr(obj, MULTIPLEXED_FUNCTION_MARKER_ATTR, False) is True
|
|
|
|
# Standalone function deployment decorated with `@serve.multiplexed`.
|
|
if _has_marker(callable_obj):
|
|
return True
|
|
|
|
# A class (or instance of one) with a method decorated with `@serve.multiplexed`.
|
|
klass = callable_obj if isinstance(callable_obj, type) else type(callable_obj)
|
|
for base in klass.__mro__:
|
|
for attr in base.__dict__.values():
|
|
if _has_marker(attr):
|
|
return True
|
|
|
|
# An instance that stored a multiplexed wrapper as an instance attribute.
|
|
if not isinstance(callable_obj, type):
|
|
for attr in getattr(callable_obj, "__dict__", {}).values():
|
|
if _has_marker(attr):
|
|
return True
|
|
|
|
return False
|
|
|
|
|
|
def asyncio_grpc_exception_handler(loop, context):
|
|
"""Exception handler to filter out false positive BlockingIOErrors from gRPC."""
|
|
exc = context.get("exception")
|
|
msg = context.get("message")
|
|
if (
|
|
exc
|
|
and isinstance(exc, BlockingIOError)
|
|
and exc.errno == errno.EAGAIN
|
|
and "PollerCompletionQueue._handle_events" in msg
|
|
):
|
|
return
|
|
|
|
loop.default_exception_handler(context)
|
|
|
|
|
|
def validate_ssl_config(
|
|
ssl_certfile: Optional[str], ssl_keyfile: Optional[str]
|
|
) -> None:
|
|
"""Validate SSL configuration for HTTPS support.
|
|
|
|
Args:
|
|
ssl_certfile: Path to SSL certificate file
|
|
ssl_keyfile: Path to SSL private key file
|
|
|
|
Raises:
|
|
ValueError: If only one of ssl_certfile or ssl_keyfile is provided
|
|
"""
|
|
if (ssl_certfile and not ssl_keyfile) or (ssl_keyfile and not ssl_certfile):
|
|
raise ValueError(
|
|
"Both ssl_keyfile and ssl_certfile must be provided together "
|
|
"to enable HTTPS."
|
|
)
|
|
|
|
|
|
def get_deployment_actor_name(
|
|
deployment_id: DeploymentID,
|
|
actor_name: str,
|
|
code_version: str,
|
|
) -> str:
|
|
"""Return the deterministic Ray actor name for a deployment-scoped actor.
|
|
|
|
The name is versioned by code_version to allow old and new replicas to
|
|
coexist during rollout (each uses its version's actors). Actors serve as
|
|
central state for replicas, so we version by code_version to ensure fresh
|
|
actors when a new code version is deployed.
|
|
"""
|
|
base = (
|
|
f"{SERVE_DEPLOYMENT_ACTOR_PREFIX}{deployment_id.app_name}"
|
|
f"::{deployment_id.name}"
|
|
)
|
|
return f"{base}::{code_version}::{actor_name}"
|
|
|
|
|
|
GENERATOR_COMPOSITION_NOT_SUPPORTED_ERROR = RuntimeError(
|
|
"Streaming deployment handle results cannot be passed to "
|
|
"downstream handle calls. If you have a use case requiring "
|
|
"this feature, please file a feature request on GitHub."
|
|
)
|
|
|
|
|
|
# Use a global singleton enum to emulate default options. We cannot use None
|
|
# for those option because None is a valid new value.
|
|
class DEFAULT(Enum):
|
|
VALUE = 1
|
|
|
|
|
|
class DeploymentOptionUpdateType(str, Enum):
|
|
# Nothing needs to be done other than setting the target state.
|
|
LightWeight = "LightWeight"
|
|
# Each DeploymentReplica instance (tracked in DeploymentState) uses certain options
|
|
# from the deployment config. These values need to be updated in DeploymentReplica.
|
|
NeedsReconfigure = "NeedsReconfigure"
|
|
# Options that are sent to the replica actor. If changed, reconfigure() on the actor
|
|
# needs to be called to update these values.
|
|
NeedsActorReconfigure = "NeedsActorReconfigure"
|
|
# If changed, restart all replicas.
|
|
HeavyWeight = "HeavyWeight"
|
|
|
|
|
|
# Type alias: objects that can be DEFAULT.VALUE have type Default[T]
|
|
T = TypeVar("T")
|
|
Default = Union[DEFAULT, T]
|
|
|
|
logger = logging.getLogger(SERVE_LOGGER_NAME)
|
|
|
|
# Format for component files
|
|
FILE_FMT = "{component_name}_{component_id}{suffix}"
|
|
|
|
|
|
class _ServeCustomEncoders:
|
|
"""Group of custom encoders for common types that's not handled by FastAPI."""
|
|
|
|
@staticmethod
|
|
def encode_np_array(obj):
|
|
assert isinstance(obj, np.ndarray)
|
|
if obj.dtype.kind == "f": # floats
|
|
obj = obj.astype(float)
|
|
if obj.dtype.kind in {"i", "u"}: # signed and unsigned integers.
|
|
obj = obj.astype(int)
|
|
return obj.tolist()
|
|
|
|
@staticmethod
|
|
def encode_np_scaler(obj):
|
|
assert isinstance(obj, np.generic)
|
|
return obj.item()
|
|
|
|
@staticmethod
|
|
def encode_exception(obj):
|
|
assert isinstance(obj, Exception)
|
|
return str(obj)
|
|
|
|
@staticmethod
|
|
def encode_pandas_dataframe(obj):
|
|
assert isinstance(obj, pd.DataFrame)
|
|
return obj.to_dict(orient="records")
|
|
|
|
|
|
serve_encoders = {Exception: _ServeCustomEncoders.encode_exception}
|
|
|
|
if np is not None:
|
|
serve_encoders[np.ndarray] = _ServeCustomEncoders.encode_np_array
|
|
serve_encoders[np.generic] = _ServeCustomEncoders.encode_np_scaler
|
|
|
|
if pd is not None:
|
|
serve_encoders[pd.DataFrame] = _ServeCustomEncoders.encode_pandas_dataframe
|
|
|
|
|
|
@ray.remote(num_cpus=0)
|
|
def block_until_http_ready(
|
|
http_endpoint,
|
|
backoff_time_s=1,
|
|
check_ready=None,
|
|
timeout=HTTP_PROXY_TIMEOUT,
|
|
):
|
|
http_is_ready = False
|
|
start_time = time.time()
|
|
|
|
while not http_is_ready:
|
|
try:
|
|
resp = requests.get(http_endpoint)
|
|
assert resp.status_code == 200
|
|
if check_ready is None:
|
|
http_is_ready = True
|
|
else:
|
|
http_is_ready = check_ready(resp)
|
|
except Exception:
|
|
pass
|
|
|
|
if 0 < timeout < time.time() - start_time:
|
|
raise TimeoutError("HTTP proxy not ready after {} seconds.".format(timeout))
|
|
|
|
time.sleep(backoff_time_s)
|
|
|
|
|
|
def get_random_string(length: int = 8):
|
|
return get_random_alphanumeric_string(length)
|
|
|
|
|
|
def format_actor_name(actor_name, *modifiers):
|
|
name = actor_name
|
|
for modifier in modifiers:
|
|
name += "-{}".format(modifier)
|
|
|
|
return name
|
|
|
|
|
|
CLASS_WRAPPER_METADATA_ATTRS = (
|
|
"__name__",
|
|
"__qualname__",
|
|
"__module__",
|
|
"__doc__",
|
|
"__annotations__",
|
|
)
|
|
|
|
|
|
def copy_class_metadata(wrapper_cls, target_cls) -> None:
|
|
"""Copy common class-level metadata onto a wrapper class."""
|
|
for attr in CLASS_WRAPPER_METADATA_ATTRS:
|
|
if attr == "__annotations__":
|
|
target_annotations = getattr(target_cls, "__annotations__", None)
|
|
if target_annotations:
|
|
merged_annotations = dict(
|
|
wrapper_cls.__dict__.get("__annotations__", {})
|
|
)
|
|
for key, value in target_annotations.items():
|
|
merged_annotations.setdefault(key, value)
|
|
wrapper_cls.__annotations__ = merged_annotations
|
|
continue
|
|
|
|
if hasattr(target_cls, attr):
|
|
setattr(wrapper_cls, attr, getattr(target_cls, attr))
|
|
wrapper_cls.__wrapped__ = target_cls
|
|
|
|
|
|
def ensure_serialization_context():
|
|
"""Ensure the serialization addons on registered, even when Ray has not
|
|
been started."""
|
|
ctx = StandaloneSerializationContext()
|
|
ray.util.serialization_addons.apply(ctx)
|
|
|
|
|
|
def msgpack_serialize(obj):
|
|
ctx = ray._private.worker.global_worker.get_serialization_context()
|
|
buffer = ctx.serialize(obj)
|
|
serialized = buffer.to_bytes()
|
|
return serialized
|
|
|
|
|
|
def msgpack_deserialize(data):
|
|
# todo: Ray does not provide a msgpack deserialization api.
|
|
try:
|
|
obj = MessagePackSerializer.loads(data[MESSAGE_PACK_OFFSET:], None)
|
|
except Exception:
|
|
raise
|
|
return obj
|
|
|
|
|
|
def merge_dict(dict1, dict2):
|
|
if dict1 is None and dict2 is None:
|
|
return None
|
|
if dict1 is None:
|
|
dict1 = dict()
|
|
if dict2 is None:
|
|
dict2 = dict()
|
|
result = dict()
|
|
for key in dict1.keys() | dict2.keys():
|
|
result[key] = sum([e.get(key, 0) for e in (dict1, dict2)])
|
|
return result
|
|
|
|
|
|
def parse_import_path(import_path: str):
|
|
"""
|
|
Takes in an import_path of form:
|
|
|
|
[subdirectory 1].[subdir 2]...[subdir n].[file name].[attribute name]
|
|
|
|
Parses this path and returns the module name (everything before the last
|
|
dot) and attribute name (everything after the last dot), such that the
|
|
attribute can be imported using "from module_name import attr_name".
|
|
"""
|
|
|
|
nodes = import_path.split(".")
|
|
if len(nodes) < 2:
|
|
raise ValueError(
|
|
f"Got {import_path} as import path. The import path "
|
|
f"should at least specify the file name and "
|
|
f"attribute name connected by a dot."
|
|
)
|
|
|
|
return ".".join(nodes[:-1]), nodes[-1]
|
|
|
|
|
|
def override_runtime_envs_except_env_vars(parent_env: Dict, child_env: Dict) -> Dict:
|
|
"""Creates a runtime_env dict by merging a parent and child environment.
|
|
|
|
This method is not destructive. It leaves the parent and child envs
|
|
the same.
|
|
|
|
The merge is a shallow update where the child environment inherits the
|
|
parent environment's settings. If the child environment specifies any
|
|
env settings, those settings take precdence over the parent.
|
|
- Note: env_vars are a special case. The child's env_vars are combined
|
|
with the parent.
|
|
|
|
Args:
|
|
parent_env: The environment to inherit settings from.
|
|
child_env: The environment with override settings.
|
|
|
|
Returns:
|
|
A new dictionary containing the merged runtime_env settings.
|
|
|
|
Raises:
|
|
TypeError: If a dictionary is not passed in for parent_env or child_env.
|
|
"""
|
|
|
|
if not isinstance(parent_env, Dict):
|
|
raise TypeError(
|
|
f'Got unexpected type "{type(parent_env)}" for parent_env. '
|
|
"parent_env must be a dictionary."
|
|
)
|
|
if not isinstance(child_env, Dict):
|
|
raise TypeError(
|
|
f'Got unexpected type "{type(child_env)}" for child_env. '
|
|
"child_env must be a dictionary."
|
|
)
|
|
|
|
defaults = copy.deepcopy(parent_env)
|
|
overrides = copy.deepcopy(child_env)
|
|
|
|
default_env_vars = defaults.get("env_vars", {})
|
|
override_env_vars = overrides.get("env_vars", {})
|
|
|
|
defaults.update(overrides)
|
|
default_env_vars.update(override_env_vars)
|
|
|
|
defaults["env_vars"] = default_env_vars
|
|
|
|
return defaults
|
|
|
|
|
|
class JavaActorHandleProxy:
|
|
"""Wraps actor handle and translate snake_case to camelCase."""
|
|
|
|
def __init__(self, handle: ActorHandle):
|
|
self.handle = handle
|
|
self._available_attrs = set(dir(self.handle))
|
|
|
|
def __getattr__(self, key: str):
|
|
if key in self._available_attrs:
|
|
camel_case_key = key
|
|
else:
|
|
components = key.split("_")
|
|
camel_case_key = components[0] + "".join(x.title() for x in components[1:])
|
|
return getattr(self.handle, camel_case_key)
|
|
|
|
|
|
def require_packages(packages: List[str]):
|
|
"""Decorator making sure function run in specified environments
|
|
|
|
Examples:
|
|
>>> from ray.serve._private.utils import require_packages
|
|
>>> @require_packages(["numpy", "package_a"]) # doctest: +SKIP
|
|
... def func(): # doctest: +SKIP
|
|
... import numpy as np # doctest: +SKIP
|
|
... ... # doctest: +SKIP
|
|
>>> func() # doctest: +SKIP
|
|
ImportError: func requires ["numpy", "package_a"] but
|
|
["package_a"] are not available, please pip install them.
|
|
|
|
Args:
|
|
packages: The list of package names that must be importable when the
|
|
decorated function is invoked.
|
|
|
|
Returns:
|
|
A decorator that wraps the target function with the package check.
|
|
"""
|
|
|
|
def decorator(func):
|
|
def check_import_once():
|
|
if not hasattr(func, "_require_packages_checked"):
|
|
missing_packages = []
|
|
for package in packages:
|
|
try:
|
|
importlib.import_module(package)
|
|
except ModuleNotFoundError:
|
|
missing_packages.append(package)
|
|
if len(missing_packages) > 0:
|
|
raise ImportError(
|
|
f"{func} requires packages {packages} to run but "
|
|
f"{missing_packages} are missing. Please "
|
|
"`pip install` them or add them to "
|
|
"`runtime_env`."
|
|
)
|
|
func._require_packages_checked = True
|
|
|
|
if inspect.iscoroutinefunction(func):
|
|
|
|
@wraps(func)
|
|
async def wrapped(*args, **kwargs):
|
|
check_import_once()
|
|
return await func(*args, **kwargs)
|
|
|
|
elif inspect.isroutine(func):
|
|
|
|
@wraps(func)
|
|
def wrapped(*args, **kwargs):
|
|
check_import_once()
|
|
return func(*args, **kwargs)
|
|
|
|
else:
|
|
raise ValueError("Decorator expect callable functions.")
|
|
|
|
return wrapped
|
|
|
|
return decorator
|
|
|
|
|
|
def in_interactive_shell():
|
|
# Taken from:
|
|
# https://stackoverflow.com/questions/15411967/how-can-i-check-if-code-is-executed-in-the-ipython-notebook
|
|
import __main__ as main
|
|
|
|
return not hasattr(main, "__file__")
|
|
|
|
|
|
def snake_to_camel_case(snake_str: str) -> str:
|
|
"""Convert a snake case string to camel case."""
|
|
|
|
words = snake_str.strip("_").split("_")
|
|
return words[0] + "".join(word[:1].upper() + word[1:] for word in words[1:])
|
|
|
|
|
|
def check_obj_ref_ready_nowait(obj_ref: ObjectRef) -> bool:
|
|
"""Check if ray object reference is ready without waiting for it."""
|
|
finished, _ = ray.wait([obj_ref], timeout=0)
|
|
return len(finished) == 1
|
|
|
|
|
|
def compress_metric_report(report: Any) -> bytes:
|
|
"""Compress a metric report (HandleMetricReport or ReplicaMetricReport) for RPC.
|
|
|
|
Uses zlib level 9 (stdlib, no extra deps). ~75KB uncompressed -> ~5KB for 1000 replicas.
|
|
"""
|
|
return zlib.compress(cloudpickle.dumps(report), level=9)
|
|
|
|
|
|
def decompress_metric_report(compressed: bytes) -> Any:
|
|
"""Decompress a metric report from RPC."""
|
|
return cloudpickle.loads(zlib.decompress(compressed))
|
|
|
|
|
|
def extract_self_if_method_call(args: List[Any], func: Callable) -> Optional[object]:
|
|
"""Check if this is a method rather than a function.
|
|
|
|
Does this by checking to see if `func` is the attribute of the first
|
|
(`self`) argument under `func.__name__`. Unfortunately, this is the most
|
|
robust solution to this I was able to find. It would also be preferable
|
|
to do this check when the decorator runs, rather than when the method is.
|
|
|
|
Arguments:
|
|
args: arguments to the function/method call.
|
|
func: the unbound function that was called.
|
|
|
|
Returns:
|
|
The ``self`` object if it's a method call, else ``None``.
|
|
"""
|
|
if len(args) > 0:
|
|
method = getattr(args[0], func.__name__, False)
|
|
if method:
|
|
wrapped = getattr(method, "__wrapped__", False)
|
|
if wrapped and wrapped == func:
|
|
return args[0]
|
|
|
|
return None
|
|
|
|
|
|
def call_function_from_import_path(import_path: str) -> Any:
|
|
"""Call the function given import path.
|
|
|
|
Args:
|
|
import_path: The import path of the function to call.
|
|
Raises:
|
|
ValueError: If the import path is invalid.
|
|
TypeError: If the import path is not callable.
|
|
RuntimeError: if the function raise exeception during execution.
|
|
Returns:
|
|
The result of the function call.
|
|
"""
|
|
try:
|
|
callback_func = import_attr(import_path)
|
|
except Exception as e:
|
|
raise ValueError(f"The import path {import_path} cannot be imported: {e}")
|
|
|
|
if not callable(callback_func):
|
|
raise TypeError(f"The import path {import_path} is not callable.")
|
|
|
|
try:
|
|
return callback_func()
|
|
except Exception as e:
|
|
raise RuntimeError(f"The function {import_path} raised an exception: {e}")
|
|
|
|
|
|
def get_head_node_id() -> str:
|
|
"""Get the head node id.
|
|
|
|
Iterate through all nodes in the ray cluster and return the node id of the first
|
|
alive node with head node resource.
|
|
"""
|
|
head_node_id = None
|
|
for node in ray.nodes():
|
|
if HEAD_NODE_RESOURCE_NAME in node["Resources"] and node["Alive"]:
|
|
head_node_id = node["NodeID"]
|
|
break
|
|
assert head_node_id is not None, "Cannot find alive head node."
|
|
|
|
return head_node_id
|
|
|
|
|
|
def calculate_remaining_timeout(
|
|
*,
|
|
timeout_s: Optional[float],
|
|
start_time_s: float,
|
|
curr_time_s: float,
|
|
) -> Optional[float]:
|
|
"""Get the timeout remaining given an overall timeout, start time, and curr time.
|
|
|
|
If the timeout passed in was `None` or negative, will always return that timeout
|
|
directly.
|
|
|
|
If the timeout is >= 0, the returned remaining timeout always be >= 0.
|
|
"""
|
|
if timeout_s is None or timeout_s < 0:
|
|
return timeout_s
|
|
|
|
time_since_start_s = curr_time_s - start_time_s
|
|
return max(0, timeout_s - time_since_start_s)
|
|
|
|
|
|
def get_all_live_placement_group_names() -> List[str]:
|
|
"""Fetch and parse the Ray placement group table for live placement group names.
|
|
|
|
Placement groups are filtered based on their `scheduling_state`; any placement
|
|
group not in the "REMOVED" state is considered live.
|
|
"""
|
|
placement_group_table = ray.util.placement_group_table()
|
|
|
|
live_pg_names = []
|
|
for entry in placement_group_table.values():
|
|
pg_name = entry.get("name", "")
|
|
if (
|
|
pg_name
|
|
and entry.get("stats", {}).get("scheduling_state", "UNKNOWN") != "REMOVED"
|
|
):
|
|
live_pg_names.append(pg_name)
|
|
|
|
return live_pg_names
|
|
|
|
|
|
def get_active_placement_group_ids() -> Set[str]:
|
|
"""
|
|
Retrieve the set of placement group IDs referenced by alive Serve actors.
|
|
|
|
Returns:
|
|
The set of placement group IDs referenced by alive Serve actors.
|
|
"""
|
|
# TODO (jeffreywang): Move the imports to the top of the file.
|
|
# https://github.com/ray-project/ray/issues/61330
|
|
from ray.util.state import list_actors
|
|
from ray.util.state.common import RAY_MAX_LIMIT_FROM_API_SERVER
|
|
|
|
actors = list_actors(
|
|
filters=[
|
|
("ray_namespace", "=", SERVE_NAMESPACE),
|
|
("state", "=", "ALIVE"),
|
|
],
|
|
limit=RAY_MAX_LIMIT_FROM_API_SERVER,
|
|
detail=True,
|
|
raise_on_missing_output=False,
|
|
)
|
|
|
|
return {
|
|
actor.placement_group_id
|
|
for actor in actors
|
|
if actor.placement_group_id is not None
|
|
}
|
|
|
|
|
|
def get_current_actor_id() -> str:
|
|
"""Gets the ID of the calling actor.
|
|
|
|
If this is called in a driver, returns "DRIVER."
|
|
|
|
If otherwise called outside of an actor, returns an empty string.
|
|
|
|
This function hangs when GCS is down due to the `ray.get_runtime_context()`
|
|
call.
|
|
"""
|
|
|
|
worker_mode = ray.get_runtime_context().worker.mode
|
|
if worker_mode == ray.SCRIPT_MODE:
|
|
return "DRIVER"
|
|
else:
|
|
try:
|
|
actor_id = ray.get_runtime_context().get_actor_id()
|
|
if actor_id is None:
|
|
return ""
|
|
else:
|
|
return actor_id
|
|
except Exception:
|
|
return ""
|
|
|
|
|
|
def is_running_in_asyncio_loop() -> bool:
|
|
try:
|
|
asyncio.get_running_loop()
|
|
return True
|
|
except RuntimeError:
|
|
return False
|
|
|
|
|
|
def get_capacity_adjusted_num_replicas(
|
|
num_replicas: int, target_capacity: Optional[float]
|
|
) -> int:
|
|
"""Return the `num_replicas` adjusted by the `target_capacity`.
|
|
|
|
The output will only ever be 0 if `target_capacity` is 0 or `num_replicas` is
|
|
0 (to support autoscaling deployments using scale-to-zero).
|
|
|
|
Rather than using the default `round` behavior in Python, which rounds half to
|
|
even, uses the `decimal` module to round half up (standard rounding behavior).
|
|
"""
|
|
if target_capacity is None or target_capacity == 100:
|
|
return num_replicas
|
|
|
|
if target_capacity == 0 or num_replicas == 0:
|
|
return 0
|
|
|
|
adjusted_num_replicas = Decimal(num_replicas * target_capacity) / Decimal(100.0)
|
|
rounded_adjusted_num_replicas = adjusted_num_replicas.to_integral_value(
|
|
rounding=ROUND_HALF_UP
|
|
)
|
|
return max(1, int(rounded_adjusted_num_replicas))
|
|
|
|
|
|
def generate_request_id() -> str:
|
|
# NOTE(edoakes): we use random.getrandbits because it reduces CPU overhead
|
|
# significantly. This is less cryptographically secure but should be ok for
|
|
# request ID generation.
|
|
# See https://bugs.python.org/issue45556 for discussion.
|
|
return str(uuid.UUID(int=random.getrandbits(128), version=4))
|
|
|
|
|
|
def inside_ray_client_context() -> bool:
|
|
return ray.util.client.ray.is_connected()
|
|
|
|
|
|
def get_component_file_name(
|
|
component_name: str,
|
|
component_id: str,
|
|
component_type: Optional[ServeComponentType],
|
|
suffix: str = "",
|
|
) -> str:
|
|
"""Get the component's file name. Replaces special characters with underscores."""
|
|
component_name = re.sub(FILE_NAME_REGEX, "_", component_name)
|
|
|
|
# For DEPLOYMENT component type, we want to log the deployment name
|
|
# instead of adding the component type to the component name.
|
|
component_log_file_name = component_name
|
|
if component_type is not None:
|
|
component_log_file_name = f"{component_type.value}_{component_name}"
|
|
if component_type != ServeComponentType.REPLICA:
|
|
component_name = f"{component_type}_{component_name}"
|
|
file_name = FILE_FMT.format(
|
|
component_name=component_log_file_name,
|
|
component_id=component_id,
|
|
suffix=suffix,
|
|
)
|
|
return file_name
|
|
|
|
|
|
def validate_route_prefix(route_prefix: Union[DEFAULT, None, str]):
|
|
if route_prefix is DEFAULT.VALUE or route_prefix is None:
|
|
return
|
|
|
|
if not route_prefix.startswith("/"):
|
|
raise ValueError(
|
|
f"Invalid route_prefix '{route_prefix}', "
|
|
"must start with a forward slash ('/')."
|
|
)
|
|
|
|
if route_prefix != "/" and route_prefix.endswith("/"):
|
|
raise ValueError(
|
|
f"Invalid route_prefix '{route_prefix}', "
|
|
"may not end with a trailing '/'."
|
|
)
|
|
|
|
if "{" in route_prefix or "}" in route_prefix:
|
|
raise ValueError(
|
|
f"Invalid route_prefix '{route_prefix}', may not contain wildcards."
|
|
)
|
|
|
|
|
|
async def await_deployment_response(deployment_response):
|
|
return await deployment_response
|
|
|
|
|
|
async def resolve_deployment_response(obj: Any, request_metadata: RequestMetadata):
|
|
"""Resolve `DeploymentResponse` objects to underlying object references.
|
|
|
|
This enables composition without explicitly calling `_to_object_ref`.
|
|
"""
|
|
from ray.serve.handle import DeploymentResponse, DeploymentResponseGenerator
|
|
|
|
if isinstance(obj, DeploymentResponseGenerator):
|
|
raise GENERATOR_COMPOSITION_NOT_SUPPORTED_ERROR
|
|
elif isinstance(obj, DeploymentResponse):
|
|
if request_metadata._by_reference and obj.by_reference:
|
|
# If sending requests by reference, launch async task to
|
|
# convert DeploymentResponse to an object ref
|
|
return asyncio.create_task(obj._to_object_ref())
|
|
else:
|
|
# Otherwise, resolve DeploymentResponse directly to result
|
|
return asyncio.create_task(await_deployment_response(obj))
|
|
elif not request_metadata._by_reference and isinstance(obj, ray.ObjectRef):
|
|
# If the router is sending requests by value (i.e. using gRPC),
|
|
# resolve all Ray objects to mirror Ray behavior
|
|
return asyncio.wrap_future(obj.future())
|
|
|
|
|
|
def wait_for_interrupt() -> None:
|
|
try:
|
|
while True:
|
|
# Block, letting Ray print logs to the terminal.
|
|
time.sleep(10)
|
|
except KeyboardInterrupt:
|
|
logger.warning("Got KeyboardInterrupt, exiting...")
|
|
# We need to re-raise KeyboardInterrupt, so serve components can be shutdown
|
|
# from the main script.
|
|
raise
|
|
|
|
|
|
def is_grpc_enabled(grpc_config) -> bool:
|
|
return grpc_config.port > 0 and len(grpc_config.grpc_servicer_functions) > 0
|
|
|
|
|
|
class Semaphore:
|
|
"""Based on asyncio.Semaphore.
|
|
|
|
This is a semaphore that can be used to limit the number of concurrent requests.
|
|
Its maximum value is dynamic and is determined by the `get_value_fn` function.
|
|
"""
|
|
|
|
def __init__(self, get_value_fn: Callable[[], int]):
|
|
self._waiters = None
|
|
self._value = 0
|
|
self._get_value_fn = get_value_fn
|
|
|
|
def __repr__(self):
|
|
res = super().__repr__()
|
|
extra = "locked" if self.locked() else f"unlocked, value:{self._value}"
|
|
if self._waiters:
|
|
extra = f"{extra}, waiters:{len(self._waiters)}"
|
|
return f"<{res[1:-1]} [{extra}]>"
|
|
|
|
async def __aenter__(self):
|
|
await self.acquire()
|
|
# We have no use for the "as ..." clause in the with
|
|
# statement for locks.
|
|
return None
|
|
|
|
async def __aexit__(self, exc_type, exc, tb):
|
|
self.release()
|
|
|
|
def get_max_value(self):
|
|
return self._get_value_fn()
|
|
|
|
def locked(self):
|
|
"""Returns True if semaphore cannot be acquired immediately."""
|
|
return self._value >= self.get_max_value() or (
|
|
any(not w.cancelled() for w in (self._waiters or ()))
|
|
)
|
|
|
|
async def acquire(self):
|
|
"""Acquire a semaphore.
|
|
If the internal counter is larger than zero on entry,
|
|
decrement it by one and return True immediately. If it is
|
|
zero on entry, block, waiting until some other coroutine has
|
|
called release() to make it larger than 0, and then return
|
|
True.
|
|
"""
|
|
if not self.locked():
|
|
self._value += 1
|
|
return True
|
|
|
|
if self._waiters is None:
|
|
self._waiters = collections.deque()
|
|
fut = asyncio.Future()
|
|
self._waiters.append(fut)
|
|
|
|
# Finally block should be called before the CancelledError
|
|
# handling as we don't want CancelledError to call
|
|
# _wake_up_first() and attempt to wake up itself.
|
|
try:
|
|
try:
|
|
await fut
|
|
finally:
|
|
self._waiters.remove(fut)
|
|
except asyncio.CancelledError:
|
|
if not fut.cancelled():
|
|
self._value -= 1
|
|
self._wake_up_next()
|
|
raise
|
|
|
|
if self._value < self.get_max_value():
|
|
self._wake_up_next()
|
|
return True
|
|
|
|
def release(self):
|
|
"""Release a semaphore, incrementing the internal counter by one.
|
|
When it was zero on entry and another coroutine is waiting for it to
|
|
become larger than zero again, wake up that coroutine.
|
|
"""
|
|
self._value -= 1
|
|
self._wake_up_next()
|
|
|
|
def _wake_up_next(self):
|
|
"""Wake up the first waiter that isn't done."""
|
|
if not self._waiters:
|
|
return
|
|
|
|
for fut in self._waiters:
|
|
if not fut.done():
|
|
self._value += 1
|
|
fut.set_result(True)
|
|
return
|