wehub-resource-sync
858 lines
26 KiB
Python
858 lines
26 KiB
Python
import sys
|
|
from typing import Any, Dict, List, Optional
|
|
|
|
import pytest
|
|
from fastapi import FastAPI
|
|
|
|
from ray import serve
|
|
from ray.serve._private.build_app import (
|
|
CUSTOM_INGRESS_REQUEST_ROUTER_UNSUPPORTED_ERROR,
|
|
BuiltApplication,
|
|
build_app,
|
|
)
|
|
from ray.serve._private.client import ServeControllerClient
|
|
from ray.serve._private.common import DeploymentID
|
|
from ray.serve.config import RequestRouterConfig
|
|
from ray.serve.deployment import Application, Deployment
|
|
from ray.serve.exceptions import RayServeException
|
|
from ray.serve.experimental.round_robin_router import RoundRobinRouter
|
|
from ray.serve.handle import DeploymentHandle
|
|
|
|
|
|
class FakeDeploymentHandle:
|
|
def __init__(self, deployment_name: str, app_name: str):
|
|
self.deployment_id = DeploymentID(deployment_name, app_name)
|
|
|
|
@classmethod
|
|
def from_deployment(cls, deployment, app_name: str) -> "FakeDeploymentHandle":
|
|
return cls(deployment.name, app_name)
|
|
|
|
def __eq__(self, other: Any) -> bool:
|
|
return (
|
|
isinstance(other, FakeDeploymentHandle)
|
|
and self.deployment_id == other.deployment_id
|
|
)
|
|
|
|
|
|
def _build_and_check(
|
|
app: Application,
|
|
*,
|
|
expected_ingress_name: str,
|
|
expected_deployments: List[Deployment],
|
|
app_name: str = "default",
|
|
default_runtime_env: Optional[Dict[str, Any]] = None,
|
|
):
|
|
built_app: BuiltApplication = build_app(
|
|
app,
|
|
name=app_name,
|
|
# Each real DeploymentHandle has a unique ID (intentionally), so the below
|
|
# equality checks don't work. Use a fake implementation instead.
|
|
make_deployment_handle=FakeDeploymentHandle.from_deployment,
|
|
default_runtime_env=default_runtime_env,
|
|
)
|
|
assert built_app.name == app_name
|
|
assert built_app.ingress_deployment_name == expected_ingress_name
|
|
assert len(built_app.deployments) == len(expected_deployments)
|
|
|
|
# Check that the returned deployment_handles are populated properly.
|
|
assert len(built_app.deployment_handles) == len(expected_deployments)
|
|
for d in expected_deployments:
|
|
h = built_app.deployment_handles.get(d.name, None)
|
|
assert h is not None, f"No handle returned for deployment {d.name}."
|
|
assert isinstance(h, FakeDeploymentHandle)
|
|
assert h.deployment_id == DeploymentID(d.name, app_name=app_name)
|
|
|
|
for expected_deployment in expected_deployments:
|
|
generated_deployment = None
|
|
for d in built_app.deployments:
|
|
if d.name == expected_deployment.name:
|
|
generated_deployment = d
|
|
|
|
assert generated_deployment is not None, (
|
|
f"Expected a deployment with name '{expected_deployment.name}' "
|
|
"to be generated but none was found. All generated names: "
|
|
+ str([d.name for d in built_app.deployments])
|
|
)
|
|
|
|
assert expected_deployment == generated_deployment
|
|
|
|
|
|
def test_real_deployment_handle_default():
|
|
"""Other tests inject a FakeDeploymentHandle, so check the default behavior."""
|
|
|
|
@serve.deployment
|
|
class D:
|
|
pass
|
|
|
|
built_app: BuiltApplication = build_app(
|
|
D.bind(D.options(name="Inner").bind()),
|
|
name="app-name",
|
|
)
|
|
assert len(built_app.deployments) == 2
|
|
assert len(built_app.deployments[1].init_args) == 1
|
|
assert isinstance(built_app.deployments[1].init_args[0], DeploymentHandle)
|
|
assert built_app.deployments[1].init_args[0].deployment_id == DeploymentID(
|
|
"Inner", app_name="app-name"
|
|
)
|
|
|
|
|
|
def test_single_deployment_basic():
|
|
@serve.deployment(
|
|
num_replicas=123,
|
|
max_ongoing_requests=10,
|
|
max_queued_requests=10,
|
|
)
|
|
class D:
|
|
pass
|
|
|
|
app = D.bind("hello world!", hello="world")
|
|
_build_and_check(
|
|
app,
|
|
expected_ingress_name="D",
|
|
expected_deployments=[
|
|
D.options(
|
|
name="D", _init_args=("hello world!",), _init_kwargs={"hello": "world"}
|
|
)
|
|
],
|
|
)
|
|
|
|
|
|
def test_single_deployment_custom_name():
|
|
@serve.deployment(
|
|
num_replicas=123,
|
|
max_ongoing_requests=10,
|
|
max_queued_requests=10,
|
|
)
|
|
class D:
|
|
pass
|
|
|
|
app = D.options(name="foobar123").bind("hello world!", hello="world")
|
|
_build_and_check(
|
|
app,
|
|
expected_ingress_name="foobar123",
|
|
expected_deployments=[
|
|
D.options(
|
|
name="foobar123",
|
|
_init_args=("hello world!",),
|
|
_init_kwargs={"hello": "world"},
|
|
)
|
|
],
|
|
)
|
|
|
|
# Change to `with pytest.raises(ValueError)` when the warning is removed.
|
|
with pytest.warns(UserWarning):
|
|
|
|
@serve.deployment(name="test#deployment")
|
|
def my_deployment():
|
|
return "Hello!"
|
|
|
|
with pytest.warns(UserWarning):
|
|
|
|
@serve.deployment()
|
|
def my_deployment():
|
|
return "Hello!"
|
|
|
|
my_deployment.options(name="test#deployment")
|
|
|
|
|
|
def test_multi_deployment_basic():
|
|
@serve.deployment(num_replicas=3)
|
|
class Inner:
|
|
pass
|
|
|
|
@serve.deployment(num_replicas=1)
|
|
class Outer:
|
|
pass
|
|
|
|
app = Outer.bind(Inner.bind(), other=Inner.options(name="Other").bind())
|
|
_build_and_check(
|
|
app,
|
|
expected_ingress_name="Outer",
|
|
expected_deployments=[
|
|
Inner.options(name="Inner", _init_args=tuple(), _init_kwargs={}),
|
|
Inner.options(name="Other", _init_args=tuple(), _init_kwargs={}),
|
|
Outer.options(
|
|
name="Outer",
|
|
_init_args=(
|
|
FakeDeploymentHandle(
|
|
"Inner",
|
|
app_name="default",
|
|
),
|
|
),
|
|
_init_kwargs={
|
|
"other": FakeDeploymentHandle(
|
|
"Other",
|
|
app_name="default",
|
|
),
|
|
},
|
|
),
|
|
],
|
|
)
|
|
|
|
|
|
def test_multi_deployment_handle_in_nested_obj():
|
|
@serve.deployment(num_replicas=3)
|
|
class Inner:
|
|
pass
|
|
|
|
@serve.deployment(num_replicas=1)
|
|
class Outer:
|
|
pass
|
|
|
|
app = Outer.bind([Inner.bind()])
|
|
_build_and_check(
|
|
app,
|
|
expected_ingress_name="Outer",
|
|
expected_deployments=[
|
|
Inner.options(name="Inner", _init_args=tuple(), _init_kwargs={}),
|
|
Outer.options(
|
|
name="Outer",
|
|
_init_args=(
|
|
[
|
|
FakeDeploymentHandle(
|
|
"Inner",
|
|
app_name="default",
|
|
),
|
|
],
|
|
),
|
|
_init_kwargs={},
|
|
),
|
|
],
|
|
)
|
|
|
|
|
|
def test_multi_deployment_custom_app_name():
|
|
@serve.deployment(num_replicas=3)
|
|
class Inner:
|
|
pass
|
|
|
|
@serve.deployment(num_replicas=1)
|
|
class Outer:
|
|
pass
|
|
|
|
app = Outer.bind(Inner.bind())
|
|
_build_and_check(
|
|
app,
|
|
app_name="custom",
|
|
expected_ingress_name="Outer",
|
|
expected_deployments=[
|
|
Inner.options(name="Inner", _init_args=tuple(), _init_kwargs={}),
|
|
Outer.options(
|
|
name="Outer",
|
|
_init_args=(
|
|
FakeDeploymentHandle(
|
|
"Inner",
|
|
app_name="custom",
|
|
),
|
|
),
|
|
_init_kwargs={},
|
|
),
|
|
],
|
|
)
|
|
|
|
|
|
def test_multi_deployment_name_collision():
|
|
@serve.deployment
|
|
class Inner:
|
|
pass
|
|
|
|
@serve.deployment
|
|
class Outer:
|
|
pass
|
|
|
|
app = Outer.bind(
|
|
Inner.bind("arg1"),
|
|
Inner.bind("arg2"),
|
|
)
|
|
_build_and_check(
|
|
app,
|
|
expected_ingress_name="Outer",
|
|
expected_deployments=[
|
|
Inner.options(name="Inner", _init_args=("arg1",), _init_kwargs={}),
|
|
Inner.options(name="Inner_1", _init_args=("arg2",), _init_kwargs={}),
|
|
Outer.options(
|
|
name="Outer",
|
|
_init_args=(
|
|
FakeDeploymentHandle(
|
|
"Inner",
|
|
app_name="default",
|
|
),
|
|
FakeDeploymentHandle(
|
|
"Inner_1",
|
|
app_name="default",
|
|
),
|
|
),
|
|
_init_kwargs={},
|
|
),
|
|
],
|
|
)
|
|
|
|
|
|
def test_multi_deployment_same_app_passed_twice():
|
|
@serve.deployment
|
|
class Shared:
|
|
pass
|
|
|
|
@serve.deployment(num_replicas=3)
|
|
class Inner:
|
|
pass
|
|
|
|
@serve.deployment(num_replicas=1)
|
|
class Outer:
|
|
pass
|
|
|
|
shared = Shared.bind()
|
|
app = Outer.bind(Inner.bind(shared), shared)
|
|
shared_handle = FakeDeploymentHandle(
|
|
"Shared",
|
|
app_name="default",
|
|
)
|
|
_build_and_check(
|
|
app,
|
|
expected_ingress_name="Outer",
|
|
expected_deployments=[
|
|
Shared.options(
|
|
name="Shared",
|
|
_init_args=tuple(),
|
|
_init_kwargs={},
|
|
),
|
|
Inner.options(name="Inner", _init_args=(shared_handle,), _init_kwargs={}),
|
|
Outer.options(
|
|
name="Outer",
|
|
_init_args=(
|
|
FakeDeploymentHandle(
|
|
"Inner",
|
|
app_name="default",
|
|
),
|
|
shared_handle,
|
|
),
|
|
_init_kwargs={},
|
|
),
|
|
],
|
|
)
|
|
|
|
|
|
def test_default_runtime_env():
|
|
@serve.deployment
|
|
class NoRayActorOptions:
|
|
pass
|
|
|
|
@serve.deployment(ray_actor_options={"num_cpus": 0, "num_gpus": 1})
|
|
class NoRuntimeEnv:
|
|
pass
|
|
|
|
@serve.deployment(ray_actor_options={"runtime_env": {"env_vars": {"ENV": "VAR"}}})
|
|
class RuntimeEnvNoWorkingDir:
|
|
pass
|
|
|
|
@serve.deployment(
|
|
ray_actor_options={"runtime_env": {"working_dir": "s3://test.zip"}}
|
|
)
|
|
class RuntimeEnvWithWorkingDir:
|
|
pass
|
|
|
|
@serve.deployment
|
|
class Outer:
|
|
pass
|
|
|
|
app = Outer.bind(
|
|
NoRayActorOptions.bind(),
|
|
NoRuntimeEnv.bind(),
|
|
RuntimeEnvNoWorkingDir.bind(),
|
|
RuntimeEnvWithWorkingDir.bind(),
|
|
)
|
|
|
|
handles = tuple(
|
|
FakeDeploymentHandle(name, app_name="default")
|
|
for name in [
|
|
"NoRayActorOptions",
|
|
"NoRuntimeEnv",
|
|
"RuntimeEnvNoWorkingDir",
|
|
"RuntimeEnvWithWorkingDir",
|
|
]
|
|
)
|
|
|
|
# 1) Test behavior when no default_runtime_env is passed.
|
|
_build_and_check(
|
|
app,
|
|
expected_ingress_name="Outer",
|
|
expected_deployments=[
|
|
Outer.options(name="Outer", _init_args=handles, _init_kwargs={}),
|
|
NoRayActorOptions.options(
|
|
name="NoRayActorOptions", _init_args=tuple(), _init_kwargs={}
|
|
),
|
|
NoRuntimeEnv.options(
|
|
name="NoRuntimeEnv", _init_args=tuple(), _init_kwargs={}
|
|
),
|
|
RuntimeEnvNoWorkingDir.options(
|
|
name="RuntimeEnvNoWorkingDir", _init_args=tuple(), _init_kwargs={}
|
|
),
|
|
RuntimeEnvWithWorkingDir.options(
|
|
name="RuntimeEnvWithWorkingDir", _init_args=tuple(), _init_kwargs={}
|
|
),
|
|
],
|
|
)
|
|
|
|
# 2) Test behavior when a default_runtime_env is passed without a working_dir.
|
|
default_runtime_env = {"env_vars": {"DEFAULT": "ENV"}}
|
|
_build_and_check(
|
|
app,
|
|
expected_ingress_name="Outer",
|
|
expected_deployments=[
|
|
Outer.options(
|
|
name="Outer",
|
|
_init_args=handles,
|
|
_init_kwargs={},
|
|
ray_actor_options={"num_cpus": 1, "runtime_env": default_runtime_env},
|
|
),
|
|
NoRayActorOptions.options(
|
|
name="NoRayActorOptions",
|
|
_init_args=tuple(),
|
|
_init_kwargs={},
|
|
ray_actor_options={"num_cpus": 1, "runtime_env": default_runtime_env},
|
|
),
|
|
NoRuntimeEnv.options(
|
|
name="NoRuntimeEnv",
|
|
_init_args=tuple(),
|
|
_init_kwargs={},
|
|
ray_actor_options={
|
|
"num_cpus": 0,
|
|
"num_gpus": 1,
|
|
"runtime_env": default_runtime_env,
|
|
},
|
|
),
|
|
# ray_actor_options shouldn't be affected.
|
|
RuntimeEnvNoWorkingDir.options(
|
|
name="RuntimeEnvNoWorkingDir", _init_args=tuple(), _init_kwargs={}
|
|
),
|
|
# ray_actor_options shouldn't be affected.
|
|
RuntimeEnvWithWorkingDir.options(
|
|
name="RuntimeEnvWithWorkingDir", _init_args=tuple(), _init_kwargs={}
|
|
),
|
|
],
|
|
default_runtime_env=default_runtime_env,
|
|
)
|
|
|
|
# 3) Test behavior when a default_runtime_env is passed with a working_dir.
|
|
default_runtime_env = {
|
|
"working_dir": "s3://default.zip",
|
|
"env_vars": {"DEFAULT": "ENV"},
|
|
}
|
|
_build_and_check(
|
|
app,
|
|
expected_ingress_name="Outer",
|
|
expected_deployments=[
|
|
Outer.options(
|
|
name="Outer",
|
|
_init_args=handles,
|
|
_init_kwargs={},
|
|
ray_actor_options={"num_cpus": 1, "runtime_env": default_runtime_env},
|
|
),
|
|
NoRayActorOptions.options(
|
|
name="NoRayActorOptions",
|
|
_init_args=tuple(),
|
|
_init_kwargs={},
|
|
ray_actor_options={
|
|
"num_cpus": 1,
|
|
"runtime_env": default_runtime_env,
|
|
},
|
|
),
|
|
NoRuntimeEnv.options(
|
|
name="NoRuntimeEnv",
|
|
_init_args=tuple(),
|
|
_init_kwargs={},
|
|
ray_actor_options={
|
|
"num_cpus": 0,
|
|
"num_gpus": 1,
|
|
"runtime_env": default_runtime_env,
|
|
},
|
|
),
|
|
# Only the working_dir field should be overridden.
|
|
RuntimeEnvNoWorkingDir.options(
|
|
name="RuntimeEnvNoWorkingDir",
|
|
_init_args=tuple(),
|
|
_init_kwargs={},
|
|
ray_actor_options={
|
|
"num_cpus": 1,
|
|
"runtime_env": {
|
|
"working_dir": "s3://default.zip",
|
|
**RuntimeEnvNoWorkingDir.ray_actor_options["runtime_env"],
|
|
},
|
|
},
|
|
),
|
|
# ray_actor_options shouldn't be affected.
|
|
RuntimeEnvWithWorkingDir.options(
|
|
name="RuntimeEnvWithWorkingDir", _init_args=tuple(), _init_kwargs={}
|
|
),
|
|
],
|
|
default_runtime_env=default_runtime_env,
|
|
)
|
|
|
|
|
|
def test_external_scaler_enabled():
|
|
"""Test that external_scaler_enabled is correctly set in BuiltApplication.
|
|
|
|
This test verifies that when build_app is called with external_scaler_enabled=True
|
|
or external_scaler_enabled=False, the resulting BuiltApplication object correctly
|
|
stores the external_scaler_enabled value.
|
|
"""
|
|
|
|
@serve.deployment
|
|
class D:
|
|
pass
|
|
|
|
app = D.bind()
|
|
|
|
# Test with external_scaler_enabled=True
|
|
built_app_with_scaler: BuiltApplication = build_app(
|
|
app,
|
|
name="app-with-scaler",
|
|
external_scaler_enabled=True,
|
|
)
|
|
assert built_app_with_scaler.external_scaler_enabled is True
|
|
assert built_app_with_scaler.name == "app-with-scaler"
|
|
|
|
# Test with external_scaler_enabled=False (explicit)
|
|
built_app_without_scaler: BuiltApplication = build_app(
|
|
app,
|
|
name="app-without-scaler",
|
|
external_scaler_enabled=False,
|
|
)
|
|
assert built_app_without_scaler.external_scaler_enabled is False
|
|
assert built_app_without_scaler.name == "app-without-scaler"
|
|
|
|
# Test with default value (should be False)
|
|
built_app_default: BuiltApplication = build_app(
|
|
app,
|
|
name="app-default",
|
|
)
|
|
assert built_app_default.external_scaler_enabled is False
|
|
assert built_app_default.name == "app-default"
|
|
|
|
|
|
def test_ingress_name_got_modified():
|
|
"""Test that the ingress deployment name is modified when there are name
|
|
collisions.
|
|
"""
|
|
|
|
@serve.deployment
|
|
class D:
|
|
pass
|
|
|
|
app = D.bind(D.bind(D.bind()))
|
|
built_app: BuiltApplication = build_app(
|
|
app,
|
|
name="default",
|
|
make_deployment_handle=FakeDeploymentHandle.from_deployment,
|
|
)
|
|
|
|
# The ingress deployment name should be modified to avoid collision.
|
|
assert built_app.ingress_deployment_name == "D_2"
|
|
|
|
# The ingress deployment should be the last one.
|
|
assert len(built_app.deployments) == 3
|
|
assert built_app.deployments[-1].name == "D_2"
|
|
|
|
|
|
def test_build_app_keeps_ingress_request_router_separate_from_app_deployments(
|
|
monkeypatch,
|
|
):
|
|
monkeypatch.setattr("ray.serve._private.build_app.RAY_SERVE_ENABLE_HA_PROXY", True)
|
|
|
|
@serve.deployment
|
|
class Ingress:
|
|
pass
|
|
|
|
@serve.deployment
|
|
class IngressRequestRouter:
|
|
pass
|
|
|
|
ingress_app = Ingress.bind()
|
|
app = ingress_app._with_ingress_request_router(
|
|
IngressRequestRouter.bind(llm_deployment=ingress_app)
|
|
)
|
|
|
|
built_app: BuiltApplication = build_app(
|
|
app,
|
|
name="default",
|
|
make_deployment_handle=FakeDeploymentHandle.from_deployment,
|
|
)
|
|
|
|
assert [deployment.name for deployment in built_app.deployments] == ["Ingress"]
|
|
assert set(built_app.deployment_handles) == {"Ingress"}
|
|
assert built_app.ingress_request_router_deployment is not None
|
|
assert built_app.ingress_request_router_deployment.name == "IngressRequestRouter"
|
|
|
|
|
|
def test_build_app_requires_ingress_request_router_to_be_single_deployment(
|
|
monkeypatch,
|
|
):
|
|
monkeypatch.setattr("ray.serve._private.build_app.RAY_SERVE_ENABLE_HA_PROXY", True)
|
|
|
|
@serve.deployment
|
|
class Ingress:
|
|
pass
|
|
|
|
@serve.deployment
|
|
class RouterChild:
|
|
pass
|
|
|
|
@serve.deployment
|
|
class IngressRequestRouter:
|
|
def __init__(self, child):
|
|
self._child = child
|
|
|
|
with pytest.raises(
|
|
ValueError,
|
|
match=(
|
|
"`ingress_request_router` to build into exactly one standalone "
|
|
"deployment"
|
|
),
|
|
):
|
|
ingress_app = Ingress.bind()
|
|
app = ingress_app._with_ingress_request_router(
|
|
IngressRequestRouter.bind(RouterChild.bind())
|
|
)
|
|
build_app(
|
|
app,
|
|
name="default",
|
|
make_deployment_handle=FakeDeploymentHandle.from_deployment,
|
|
)
|
|
|
|
|
|
def test_build_app_rejects_ingress_request_router_in_main_app_graph(monkeypatch):
|
|
monkeypatch.setattr("ray.serve._private.build_app.RAY_SERVE_ENABLE_HA_PROXY", True)
|
|
|
|
@serve.deployment
|
|
class IngressRequestRouter:
|
|
pass
|
|
|
|
@serve.deployment
|
|
class Ingress:
|
|
def __init__(self, router):
|
|
self._router = router
|
|
|
|
ingress_request_router = IngressRequestRouter.bind()
|
|
ingress_app = Ingress.bind(ingress_request_router)
|
|
app = ingress_app._with_ingress_request_router(ingress_request_router)
|
|
|
|
with pytest.raises(
|
|
ValueError,
|
|
match="did not produce any new deployments",
|
|
):
|
|
build_app(
|
|
app,
|
|
name="default",
|
|
make_deployment_handle=FakeDeploymentHandle.from_deployment,
|
|
)
|
|
|
|
|
|
def test_ingress_validation_excludes_ingress_request_router_fastapi_app(monkeypatch):
|
|
monkeypatch.setattr("ray.serve._private.build_app.RAY_SERVE_ENABLE_HA_PROXY", True)
|
|
|
|
ingress_api = FastAPI()
|
|
router_api = FastAPI()
|
|
|
|
@serve.deployment
|
|
@serve.ingress(ingress_api)
|
|
class Ingress:
|
|
pass
|
|
|
|
@serve.deployment
|
|
@serve.ingress(router_api)
|
|
class IngressRequestRouter:
|
|
pass
|
|
|
|
ingress_app = Ingress.bind()
|
|
app = ingress_app._with_ingress_request_router(
|
|
IngressRequestRouter.bind(llm_deployment=ingress_app)
|
|
)
|
|
|
|
built_app: BuiltApplication = build_app(
|
|
app,
|
|
name="default",
|
|
make_deployment_handle=FakeDeploymentHandle.from_deployment,
|
|
)
|
|
|
|
client = object.__new__(ServeControllerClient)
|
|
client._check_ingress_deployments([built_app])
|
|
|
|
|
|
def test_callable_uses_multiplexing_static():
|
|
"""Detects `@serve.multiplexed` on a method/function statically (class input)."""
|
|
from ray.serve._private.utils import _callable_uses_multiplexing
|
|
|
|
class MultiplexedClass:
|
|
@serve.multiplexed(max_num_models_per_replica=2)
|
|
async def load_model(self, model_id: str) -> str:
|
|
return model_id
|
|
|
|
@serve.multiplexed
|
|
async def standalone(model_id: str) -> str:
|
|
return model_id
|
|
|
|
class Plain:
|
|
async def __call__(self, request) -> str:
|
|
return "ok"
|
|
|
|
assert _callable_uses_multiplexing(MultiplexedClass)
|
|
assert _callable_uses_multiplexing(standalone)
|
|
assert not _callable_uses_multiplexing(Plain)
|
|
|
|
|
|
def test_callable_uses_multiplexing_dynamic_instance():
|
|
"""Detects multiplexing wired up dynamically at init (e.g. LLMServer LoRA).
|
|
|
|
This is only visible on a constructed instance, not on the class statically.
|
|
"""
|
|
from ray.serve._private.utils import _callable_uses_multiplexing
|
|
|
|
class DynamicMultiplexed:
|
|
def __init__(self):
|
|
async def _load(model_id: str) -> str:
|
|
return model_id
|
|
|
|
self._load_model = serve.multiplexed(max_num_models_per_replica=2)(_load)
|
|
|
|
# Not detectable on the class (the wrapper only exists after construction)...
|
|
assert not _callable_uses_multiplexing(DynamicMultiplexed)
|
|
# ...but detectable on the constructed instance.
|
|
assert _callable_uses_multiplexing(DynamicMultiplexed())
|
|
|
|
|
|
def test_callable_uses_multiplexing_ignores_handle_attrs():
|
|
"""A composed ingress holding DeploymentHandles must not be flagged.
|
|
|
|
`DeploymentHandle.__getattr__` returns a (truthy) handle for any attribute name,
|
|
so the instance-attribute scan must check the marker by identity (`is True`) to
|
|
avoid false positives that would break every composed app under direct ingress.
|
|
"""
|
|
from ray.serve._private.utils import _callable_uses_multiplexing
|
|
|
|
class FakeHandle:
|
|
# Mirrors DeploymentHandle: returns a truthy object for any attribute.
|
|
def __getattr__(self, name):
|
|
return self
|
|
|
|
class Ingress:
|
|
def __init__(self):
|
|
self._backend = FakeHandle()
|
|
|
|
async def __call__(self, request):
|
|
return await self._backend.remote()
|
|
|
|
assert not _callable_uses_multiplexing(Ingress())
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
"haproxy_enabled, request_router_class, rejected",
|
|
[
|
|
# A custom router on the ingress is rejected only under HAProxy, which
|
|
# load-balances ingress traffic and bypasses the Serve request router.
|
|
(True, RoundRobinRouter, True),
|
|
(False, RoundRobinRouter, False),
|
|
# The default router (left unset) is not custom.
|
|
(True, None, False),
|
|
],
|
|
)
|
|
def test_build_app_rejects_only_custom_ingress_request_router_under_haproxy(
|
|
monkeypatch, haproxy_enabled, request_router_class, rejected
|
|
):
|
|
"""A custom ingress router is rejected only under HAProxy. The default
|
|
router and the no-HAProxy case build."""
|
|
monkeypatch.setattr(
|
|
"ray.serve._private.build_app.RAY_SERVE_ENABLE_HA_PROXY", haproxy_enabled
|
|
)
|
|
|
|
options = {}
|
|
if request_router_class is not None:
|
|
options["request_router_config"] = RequestRouterConfig(
|
|
request_router_class=request_router_class
|
|
)
|
|
|
|
@serve.deployment
|
|
class Ingress:
|
|
pass
|
|
|
|
app = Ingress.options(**options).bind()
|
|
|
|
def build():
|
|
return build_app(
|
|
app,
|
|
name="default",
|
|
make_deployment_handle=FakeDeploymentHandle.from_deployment,
|
|
)
|
|
|
|
if rejected:
|
|
with pytest.raises(
|
|
RayServeException, match=CUSTOM_INGRESS_REQUEST_ROUTER_UNSUPPORTED_ERROR
|
|
):
|
|
build()
|
|
else:
|
|
assert [deployment.name for deployment in build().deployments] == ["Ingress"]
|
|
|
|
|
|
def test_build_app_allows_custom_ingress_request_router_in_direct_streaming(
|
|
monkeypatch,
|
|
):
|
|
"""Direct streaming attaches an ingress_request_router that delegates replica
|
|
selection back to the ingress deployment's router, so a custom router is
|
|
allowed even under HAProxy."""
|
|
monkeypatch.setattr("ray.serve._private.build_app.RAY_SERVE_ENABLE_HA_PROXY", True)
|
|
|
|
@serve.deployment(
|
|
request_router_config=RequestRouterConfig(request_router_class=RoundRobinRouter)
|
|
)
|
|
class Ingress:
|
|
pass
|
|
|
|
@serve.deployment
|
|
class IngressRequestRouter:
|
|
pass
|
|
|
|
ingress_app = Ingress.bind()
|
|
app = ingress_app._with_ingress_request_router(
|
|
IngressRequestRouter.bind(server=ingress_app)
|
|
)
|
|
|
|
built_app: BuiltApplication = build_app(
|
|
app,
|
|
name="default",
|
|
make_deployment_handle=FakeDeploymentHandle.from_deployment,
|
|
)
|
|
|
|
assert [deployment.name for deployment in built_app.deployments] == ["Ingress"]
|
|
assert built_app.ingress_request_router_deployment is not None
|
|
|
|
|
|
def test_build_app_haproxy_allows_custom_router_on_non_ingress_deployment(monkeypatch):
|
|
"""The guard targets only the ingress, so a custom router on a downstream
|
|
deployment is honored under HAProxy."""
|
|
monkeypatch.setattr("ray.serve._private.build_app.RAY_SERVE_ENABLE_HA_PROXY", True)
|
|
|
|
@serve.deployment(
|
|
request_router_config=RequestRouterConfig(request_router_class=RoundRobinRouter)
|
|
)
|
|
class Downstream:
|
|
pass
|
|
|
|
@serve.deployment
|
|
class Ingress:
|
|
def __init__(self, child):
|
|
pass
|
|
|
|
built_app: BuiltApplication = build_app(
|
|
Ingress.bind(Downstream.bind()),
|
|
name="default",
|
|
make_deployment_handle=FakeDeploymentHandle.from_deployment,
|
|
)
|
|
|
|
assert {deployment.name for deployment in built_app.deployments} == {
|
|
"Ingress",
|
|
"Downstream",
|
|
}
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(pytest.main(["-v", "-s", __file__]))
|