wehub-resource-sync
83 lines
2.4 KiB
Python
83 lines
2.4 KiB
Python
from abc import ABC, abstractmethod
|
|
from dataclasses import dataclass, fields
|
|
|
|
import ray
|
|
from ray.serve._private.common import DeploymentHandleSource
|
|
from ray.serve._private.constants import (
|
|
RAY_SERVE_RUN_ROUTER_IN_SEPARATE_LOOP,
|
|
RAY_SERVE_USE_GRPC_BY_DEFAULT,
|
|
)
|
|
from ray.serve._private.utils import DEFAULT
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class InitHandleOptionsBase(ABC):
|
|
"""Init options for each ServeHandle instance.
|
|
|
|
These fields can be set by calling `.init()` on a handle before
|
|
sending the first request.
|
|
"""
|
|
|
|
_prefer_local_routing: bool = False
|
|
_source: DeploymentHandleSource = DeploymentHandleSource.UNKNOWN
|
|
_run_router_in_separate_loop: bool = RAY_SERVE_RUN_ROUTER_IN_SEPARATE_LOOP
|
|
|
|
@classmethod
|
|
@abstractmethod
|
|
def create(cls, **kwargs) -> "InitHandleOptionsBase":
|
|
raise NotImplementedError
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class InitHandleOptions(InitHandleOptionsBase):
|
|
@classmethod
|
|
def create(cls, **kwargs) -> "InitHandleOptions":
|
|
for k in list(kwargs.keys()):
|
|
if kwargs[k] == DEFAULT.VALUE:
|
|
# Use default value
|
|
del kwargs[k]
|
|
|
|
# Detect replica source for handles
|
|
if (
|
|
"_source" not in kwargs
|
|
and ray.serve.context._get_internal_replica_context() is not None
|
|
):
|
|
kwargs["_source"] = DeploymentHandleSource.REPLICA
|
|
|
|
return cls(**kwargs)
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class DynamicHandleOptionsBase(ABC):
|
|
"""Dynamic options for each ServeHandle instance.
|
|
|
|
These fields can be changed by calling `.options()` on a handle.
|
|
"""
|
|
|
|
method_name: str = "__call__"
|
|
multiplexed_model_id: str = ""
|
|
session_id: str = ""
|
|
stream: bool = False
|
|
|
|
@abstractmethod
|
|
def copy_and_update(self, **kwargs) -> "DynamicHandleOptionsBase":
|
|
pass
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class DynamicHandleOptions(DynamicHandleOptionsBase):
|
|
_by_reference: bool = not RAY_SERVE_USE_GRPC_BY_DEFAULT
|
|
request_serialization: str = "cloudpickle"
|
|
response_serialization: str = "cloudpickle"
|
|
|
|
def copy_and_update(self, **kwargs) -> "DynamicHandleOptions":
|
|
new_kwargs = {}
|
|
|
|
for f in fields(self):
|
|
if f.name not in kwargs or kwargs[f.name] == DEFAULT.VALUE:
|
|
new_kwargs[f.name] = getattr(self, f.name)
|
|
else:
|
|
new_kwargs[f.name] = kwargs[f.name]
|
|
|
|
return DynamicHandleOptions(**new_kwargs)
|