wehub-resource-sync
421 lines
14 KiB
Python
421 lines
14 KiB
Python
import functools
|
||
import os
|
||
import sys
|
||
import threading
|
||
import time
|
||
from concurrent.futures.thread import ThreadPoolExecutor
|
||
from typing import Dict
|
||
|
||
import httpx
|
||
import pytest
|
||
|
||
import ray
|
||
from ray import serve
|
||
from ray._common.test_utils import SignalActor, wait_for_condition
|
||
from ray.serve._private.common import DeploymentStatus
|
||
from ray.serve._private.logging_utils import get_serve_logs_dir
|
||
from ray.serve._private.test_utils import (
|
||
SharedFlag,
|
||
check_deployment_status,
|
||
check_num_replicas_eq,
|
||
get_application_url,
|
||
)
|
||
from ray.serve._private.utils import get_component_file_name
|
||
from ray.serve.schema import ApplicationStatus
|
||
from ray.util.state import list_actors
|
||
|
||
|
||
def test_deploy_zero_cpus(serve_instance):
|
||
@serve.deployment(ray_actor_options={"num_cpus": 0})
|
||
class D:
|
||
def hello(self):
|
||
return "hello"
|
||
|
||
h = serve.run(D.bind())
|
||
assert h.hello.remote().result() == "hello"
|
||
|
||
|
||
def test_deployment_error_handling(serve_instance):
|
||
@serve.deployment
|
||
def f():
|
||
pass
|
||
|
||
with pytest.raises(RuntimeError):
|
||
# This is an invalid configuration since dynamic upload of working
|
||
# directories is not supported. The error this causes in the controller
|
||
# code should be caught and reported back to the `deploy` caller.
|
||
|
||
serve.run(
|
||
f.options(ray_actor_options={"runtime_env": {"working_dir": "."}}).bind()
|
||
)
|
||
|
||
|
||
def test_json_serialization_user_config(serve_instance):
|
||
"""See https://github.com/ray-project/ray/issues/25345.
|
||
|
||
See https://github.com/ray-project/ray/pull/26235 for additional context
|
||
about this test.
|
||
"""
|
||
|
||
@serve.deployment(name="simple-deployment")
|
||
class SimpleDeployment:
|
||
value: str
|
||
nested_value: str
|
||
|
||
def reconfigure(self, config: Dict) -> None:
|
||
self.value = config["value"]
|
||
self.nested_value = config["nested"]["value"]
|
||
|
||
def get_value(self) -> None:
|
||
return self.value
|
||
|
||
def get_nested_value(self) -> None:
|
||
return self.nested_value
|
||
|
||
app = SimpleDeployment.options(
|
||
user_config={
|
||
"value": "Success!",
|
||
"nested": {"value": "Success!"},
|
||
}
|
||
).bind()
|
||
handle = serve.run(app)
|
||
|
||
assert handle.get_value.remote().result() == "Success!"
|
||
assert handle.get_nested_value.remote().result() == "Success!"
|
||
|
||
handle = serve.run(
|
||
SimpleDeployment.options(
|
||
user_config={
|
||
"value": "Failure!",
|
||
"another-value": "Failure!",
|
||
"nested": {"value": "Success!"},
|
||
}
|
||
).bind()
|
||
)
|
||
|
||
assert handle.get_value.remote().result() == "Failure!"
|
||
assert handle.get_nested_value.remote().result() == "Success!"
|
||
|
||
|
||
def test_http_proxy_request_cancellation(serve_instance):
|
||
# https://github.com/ray-project/ray/issues/21425
|
||
s = SignalActor.remote()
|
||
|
||
@serve.deployment(max_ongoing_requests=1)
|
||
class A:
|
||
def __init__(self) -> None:
|
||
self.counter = 0
|
||
|
||
async def __call__(self):
|
||
self.counter += 1
|
||
ret_val = self.counter
|
||
await s.wait.remote()
|
||
return ret_val
|
||
|
||
serve.run(A.bind())
|
||
url = get_application_url("HTTP")
|
||
# Windows usually resolves "localhost" to the IPv6 loopback ::1 first, but the
|
||
# Serve proxy is listening only on IPv4. The initial TCP connect then hangs,
|
||
# breaking the short‑timeout logic in this test. Using the literal IPv4 address
|
||
# 127.0.0.1 skips the IPv6 attempt and makes the test deterministic on Windows.
|
||
if sys.platform == "win32":
|
||
url = url.replace("localhost", "127.0.0.1")
|
||
|
||
with ThreadPoolExecutor() as pool:
|
||
# Send the first request, it should block for the result
|
||
first_blocking_fut = pool.submit(functools.partial(httpx.get, url, timeout=100))
|
||
wait_for_condition(lambda: ray.get(s.cur_num_waiters.remote()) == 1)
|
||
assert not first_blocking_fut.done()
|
||
|
||
# Send more requests, these should be queued in handle.
|
||
# But because first request is hanging and these have low timeout.
|
||
# They should all disconnect from http connection.
|
||
# These requests should never reach the replica.
|
||
rest_blocking_futs = [
|
||
pool.submit(functools.partial(httpx.get, url, timeout=0.5))
|
||
for _ in range(3)
|
||
]
|
||
wait_for_condition(lambda: all(f.done() for f in rest_blocking_futs))
|
||
|
||
# Now unblock the first request.
|
||
ray.get(s.send.remote())
|
||
assert first_blocking_fut.result().text == "1"
|
||
|
||
# Sending another request to verify that only one request has been
|
||
# processed so far.
|
||
assert httpx.get(url).text == "2"
|
||
|
||
|
||
def test_nonserializable_deployment(serve_instance):
|
||
lock = threading.Lock()
|
||
|
||
class D:
|
||
def hello(self, _):
|
||
return lock
|
||
|
||
# Check that the `inspect_serializability` trace was printed
|
||
with pytest.raises(
|
||
TypeError,
|
||
match=r"Could not serialize the deployment[\s\S]*was found to be non-serializable.*", # noqa
|
||
):
|
||
serve.deployment(D)
|
||
|
||
@serve.deployment
|
||
class E:
|
||
def __init__(self, arg):
|
||
self.arg = arg
|
||
|
||
with pytest.raises(TypeError, match="pickle"):
|
||
serve.run(E.bind(lock))
|
||
|
||
with pytest.raises(TypeError, match="pickle"):
|
||
serve.run(E.bind(arg=lock))
|
||
|
||
|
||
def test_deploy_application_unhealthy(serve_instance):
|
||
"""Test deploying an application that becomes unhealthy."""
|
||
|
||
event = SharedFlag.remote()
|
||
|
||
@serve.deployment(health_check_period_s=1, health_check_timeout_s=3)
|
||
class Model:
|
||
def __call__(self):
|
||
return "hello world"
|
||
|
||
def check_health(self):
|
||
if ray.get(event.is_set.remote()):
|
||
raise RuntimeError("Intentionally failing.")
|
||
|
||
handle = serve.run(Model.bind(), name="app")
|
||
assert handle.remote().result() == "hello world"
|
||
assert serve.status().applications["app"].status == ApplicationStatus.RUNNING
|
||
|
||
# When a deployment becomes unhealthy, application should transition -> UNHEALTHY
|
||
event.set.remote()
|
||
wait_for_condition(
|
||
lambda: serve.status().applications["app"].status == ApplicationStatus.UNHEALTHY
|
||
)
|
||
|
||
# Check that application stays unhealthy
|
||
for _ in range(10):
|
||
assert serve.status().applications["app"].status == ApplicationStatus.UNHEALTHY
|
||
time.sleep(0.1)
|
||
|
||
# At least 10 control loop iterations should have passed. Check that
|
||
# the logs from application state manager notifying about unhealthy
|
||
# deployments doesn't spam, they should get printed only once.
|
||
controller_pid = [
|
||
actor["pid"]
|
||
for actor in list_actors()
|
||
if actor["name"] == "SERVE_CONTROLLER_ACTOR"
|
||
][0]
|
||
controller_log_file_name = get_component_file_name(
|
||
"controller", controller_pid, component_type=None, suffix=".log"
|
||
)
|
||
controller_log_path = os.path.join(get_serve_logs_dir(), controller_log_file_name)
|
||
with open(controller_log_path, "r") as f:
|
||
s = f.read()
|
||
assert s.count("The deployments ['Model'] are UNHEALTHY.") <= 1
|
||
|
||
|
||
@pytest.mark.skipif(
|
||
sys.platform == "win32", reason="Runtime env support experimental on windows"
|
||
)
|
||
def test_deploy_bad_pip_package_deployment(serve_instance):
|
||
"""Test deploying with a bad runtime env at deployment level."""
|
||
|
||
@serve.deployment(ray_actor_options={"runtime_env": {"pip": ["does_not_exist"]}})
|
||
class Model:
|
||
def __call__(self):
|
||
return "hello world"
|
||
|
||
serve._run(Model.bind(), _blocking=False)
|
||
|
||
def check_fail():
|
||
app_status = serve.status().applications["default"]
|
||
assert app_status.status == ApplicationStatus.DEPLOY_FAILED
|
||
deployment_message = app_status.deployments["Model"].message
|
||
assert "No matching distribution found for does_not_exist" in deployment_message
|
||
return True
|
||
|
||
wait_for_condition(check_fail, timeout=60)
|
||
|
||
|
||
def test_deploy_same_deployment_name_different_app(serve_instance):
|
||
@serve.deployment
|
||
class Model:
|
||
def __init__(self, name):
|
||
self.name = name
|
||
|
||
def __call__(self):
|
||
return f"hello {self.name}"
|
||
|
||
serve.run(Model.bind("alice"), name="app1", route_prefix="/app1")
|
||
serve.run(Model.bind("bob"), name="app2", route_prefix="/app2")
|
||
|
||
url = get_application_url("HTTP", app_name="app1")
|
||
assert httpx.get(f"{url}").text == "hello alice"
|
||
url_without_route_prefix = get_application_url(
|
||
"HTTP", app_name="app1", exclude_route_prefix=True
|
||
)
|
||
routes_url = f"{url_without_route_prefix}/-/routes"
|
||
routes = httpx.get(routes_url).json()
|
||
assert routes["/app1"] == "app1"
|
||
|
||
url = get_application_url("HTTP", app_name="app2")
|
||
assert httpx.get(f"{url}").text == "hello bob"
|
||
url_without_route_prefix = get_application_url(
|
||
"HTTP", app_name="app2", exclude_route_prefix=True
|
||
)
|
||
routes_url = f"{url_without_route_prefix}/-/routes"
|
||
routes = httpx.get(routes_url).json()
|
||
assert routes["/app2"] == "app2"
|
||
|
||
app1_status = serve.status().applications["app1"]
|
||
app2_status = serve.status().applications["app2"]
|
||
assert app1_status.status == "RUNNING"
|
||
assert app1_status.deployments["Model"].status == "HEALTHY"
|
||
assert app2_status.status == "RUNNING"
|
||
assert app2_status.deployments["Model"].status == "HEALTHY"
|
||
|
||
|
||
@pytest.mark.parametrize("use_options", [True, False])
|
||
def test_num_replicas_auto_api(serve_instance, use_options):
|
||
"""Test setting only `num_replicas="auto"`."""
|
||
|
||
signal = SignalActor.remote()
|
||
|
||
class A:
|
||
async def __call__(self):
|
||
await signal.wait.remote()
|
||
|
||
if use_options:
|
||
A = serve.deployment(A).options(num_replicas="auto")
|
||
else:
|
||
A = serve.deployment(num_replicas="auto")(A)
|
||
|
||
serve.run(A.bind(), name="default")
|
||
wait_for_condition(
|
||
check_deployment_status, name="A", expected_status=DeploymentStatus.HEALTHY
|
||
)
|
||
check_num_replicas_eq("A", 1)
|
||
|
||
app_details = serve_instance.get_serve_details()["applications"]["default"]
|
||
deployment_config = app_details["deployments"]["A"]["deployment_config"]
|
||
assert "num_replicas" not in deployment_config
|
||
assert deployment_config["max_ongoing_requests"] == 5
|
||
assert deployment_config["autoscaling_config"] == {
|
||
# Set by `num_replicas="auto"`
|
||
"target_ongoing_requests": 2.0,
|
||
"min_replicas": 1,
|
||
"max_replicas": 100,
|
||
# Untouched defaults
|
||
"metrics_interval_s": 10.0,
|
||
"upscale_delay_s": 30.0,
|
||
"look_back_period_s": 30.0,
|
||
"downscale_delay_s": 600.0,
|
||
"downscale_to_zero_delay_s": None,
|
||
"upscale_smoothing_factor": None,
|
||
"downscale_smoothing_factor": None,
|
||
"upscaling_factor": None,
|
||
"downscaling_factor": None,
|
||
"smoothing_factor": 1.0,
|
||
"initial_replicas": None,
|
||
"aggregation_function": "mean",
|
||
"policy": {
|
||
"policy_function": "ray.serve.autoscaling_policy:default_autoscaling_policy",
|
||
"policy_kwargs": {},
|
||
},
|
||
}
|
||
|
||
|
||
@pytest.mark.parametrize("use_options", [True, False])
|
||
def test_num_replicas_auto_basic(serve_instance, use_options):
|
||
"""Test `num_replicas="auto"` and the defaults are used by autoscaling."""
|
||
|
||
signal = SignalActor.remote()
|
||
|
||
class A:
|
||
async def __call__(self):
|
||
await signal.wait.remote()
|
||
|
||
if use_options:
|
||
A = serve.deployment(A).options(
|
||
num_replicas="auto",
|
||
autoscaling_config={
|
||
"metrics_interval_s": 1,
|
||
"upscale_delay_s": 1,
|
||
"look_back_period_s": 2,
|
||
# Shrink from the 600s default so the transient extra replica
|
||
# (metric reading fractionally over target) downscales back
|
||
# within the test's wait window.
|
||
"downscale_delay_s": 4,
|
||
},
|
||
graceful_shutdown_timeout_s=1,
|
||
)
|
||
else:
|
||
A = serve.deployment(
|
||
num_replicas="auto",
|
||
autoscaling_config={
|
||
"metrics_interval_s": 1,
|
||
"upscale_delay_s": 1,
|
||
"look_back_period_s": 2,
|
||
# Shrink from the 600s default so the transient extra replica
|
||
# (metric reading fractionally over target) downscales back
|
||
# within the test's wait window.
|
||
"downscale_delay_s": 4,
|
||
},
|
||
graceful_shutdown_timeout_s=1,
|
||
)(A)
|
||
|
||
h = serve.run(A.bind(), name="default")
|
||
wait_for_condition(
|
||
check_deployment_status, name="A", expected_status=DeploymentStatus.HEALTHY
|
||
)
|
||
check_num_replicas_eq("A", 1)
|
||
|
||
app_details = serve_instance.get_serve_details()["applications"]["default"]
|
||
deployment_config = app_details["deployments"]["A"]["deployment_config"]
|
||
assert "num_replicas" not in deployment_config
|
||
assert deployment_config["max_ongoing_requests"] == 5
|
||
assert deployment_config["autoscaling_config"] == {
|
||
# Set by `num_replicas="auto"`
|
||
"target_ongoing_requests": 2.0,
|
||
"min_replicas": 1,
|
||
"max_replicas": 100,
|
||
# Overrided by `autoscaling_config`
|
||
"metrics_interval_s": 1.0,
|
||
"upscale_delay_s": 1.0,
|
||
"downscale_delay_s": 4.0,
|
||
# Untouched defaults
|
||
"look_back_period_s": 2.0,
|
||
"downscale_to_zero_delay_s": None,
|
||
"upscale_smoothing_factor": None,
|
||
"downscale_smoothing_factor": None,
|
||
"upscaling_factor": None,
|
||
"downscaling_factor": None,
|
||
"smoothing_factor": 1.0,
|
||
"initial_replicas": None,
|
||
"aggregation_function": "mean",
|
||
"policy": {
|
||
"policy_function": "ray.serve.autoscaling_policy:default_autoscaling_policy",
|
||
"policy_kwargs": {},
|
||
},
|
||
}
|
||
|
||
for i in range(3):
|
||
[h.remote() for _ in range(2)]
|
||
|
||
def check_num_waiters(target: int):
|
||
assert ray.get(signal.cur_num_waiters.remote()) == target
|
||
return True
|
||
|
||
wait_for_condition(check_num_waiters, target=2 * (i + 1), timeout=30)
|
||
print(time.time(), f"Number of waiters on signal reached {2*(i+1)}.")
|
||
wait_for_condition(check_num_replicas_eq, name="A", target=i + 1, timeout=30)
|
||
print(time.time(), f"Confirmed number of replicas are at {i+1}.")
|
||
|
||
|
||
if __name__ == "__main__":
|
||
sys.exit(pytest.main(["-v", "-s", __file__]))
|