wehub-resource-sync
852 lines
31 KiB
Python
852 lines
31 KiB
Python
import logging
|
|
import re
|
|
import sys
|
|
import time
|
|
from copy import copy
|
|
from functools import partial
|
|
from typing import List
|
|
|
|
import httpx
|
|
import pytest
|
|
|
|
import ray
|
|
import ray.actor
|
|
from ray import serve
|
|
from ray._common.test_utils import SignalActor, wait_for_condition
|
|
from ray.serve._private.common import DeploymentID, ReplicaID
|
|
from ray.serve._private.constants import (
|
|
RAY_SERVE_AGGREGATE_METRICS_AT_CONTROLLER,
|
|
RAY_SERVE_COLLECT_AUTOSCALING_METRICS_ON_HANDLE,
|
|
SERVE_DEFAULT_APP_NAME,
|
|
SERVE_NAMESPACE,
|
|
)
|
|
from ray.serve._private.test_utils import (
|
|
check_num_replicas_eq,
|
|
check_num_replicas_gte,
|
|
check_running,
|
|
check_target_groups_ready,
|
|
get_application_url,
|
|
)
|
|
from ray.serve.schema import (
|
|
ApplicationStatus,
|
|
ServeApplicationSchema,
|
|
ServeDeploySchema,
|
|
ServeInstanceDetails,
|
|
)
|
|
from ray.tests.conftest import call_ray_stop_only # noqa: F401
|
|
from ray.util.state import list_actors
|
|
|
|
|
|
def check_log_file(log_file: str, expected_regex: list):
|
|
with open(log_file, "r") as f:
|
|
s = f.read()
|
|
print(s)
|
|
for regex in expected_regex:
|
|
assert re.findall(regex, s) != [], f"Did not find pattern '{regex}' in {s}"
|
|
return True
|
|
|
|
|
|
def check_deployments_dead(deployment_ids: List[DeploymentID]):
|
|
prefixes = [f"{id.app_name}#{id.name}" for id in deployment_ids]
|
|
actor_names = [
|
|
actor["name"] for actor in list_actors(filters=[("state", "=", "ALIVE")])
|
|
]
|
|
return all(f"ServeReplica::{p}" not in actor_names for p in prefixes)
|
|
|
|
|
|
class TestDeploywithLoggingConfig:
|
|
def get_deploy_config(self, model_within_logging_config: bool = False):
|
|
if model_within_logging_config:
|
|
path = "ray.serve.tests.test_config_files.logging_config_test.model2"
|
|
else:
|
|
path = "ray.serve.tests.test_config_files.logging_config_test.model"
|
|
return {
|
|
"applications": [
|
|
{
|
|
"name": "app1",
|
|
"route_prefix": "/app1",
|
|
"import_path": path,
|
|
},
|
|
],
|
|
}
|
|
|
|
@pytest.mark.parametrize("encoding_type", ["TEXT", "JSON"])
|
|
def test_deploy_app_with_application_logging_config(
|
|
self, serve_instance, encoding_type: str
|
|
):
|
|
"""Deploy application with application logging config"""
|
|
client = serve_instance
|
|
config_dict = self.get_deploy_config()
|
|
|
|
config_dict["applications"][0]["logging_config"] = {
|
|
"encoding": encoding_type,
|
|
}
|
|
config = ServeDeploySchema.model_validate(config_dict)
|
|
client.deploy_apps(config)
|
|
wait_for_condition(
|
|
lambda: httpx.post("http://localhost:8000/app1").status_code == 200
|
|
)
|
|
|
|
resp = httpx.post("http://localhost:8000/app1").json()
|
|
|
|
replica_id = resp["replica"].split("#")[-1]
|
|
if encoding_type == "JSON":
|
|
expected_log_regex = [f'"replica": "{replica_id}", ']
|
|
else:
|
|
expected_log_regex = [f".*{replica_id}.*"]
|
|
check_log_file(resp["log_file"], expected_log_regex)
|
|
|
|
@pytest.mark.parametrize("encoding_type", ["TEXT", "JSON"])
|
|
def test_deploy_app_with_deployment_logging_config(
|
|
self, serve_instance, encoding_type: str
|
|
):
|
|
client = serve_instance
|
|
"""Deploy application with deployment logging config inside the yaml"""
|
|
config_dict = self.get_deploy_config()
|
|
|
|
config_dict["applications"][0]["deployments"] = [
|
|
{
|
|
"name": "Model",
|
|
"logging_config": {
|
|
"encoding": encoding_type,
|
|
},
|
|
},
|
|
]
|
|
config = ServeDeploySchema.model_validate(config_dict)
|
|
client.deploy_apps(config)
|
|
wait_for_condition(
|
|
lambda: httpx.post("http://localhost:8000/app1").status_code == 200
|
|
)
|
|
|
|
resp = httpx.post("http://localhost:8000/app1").json()
|
|
|
|
replica_id = resp["replica"].split("#")[-1]
|
|
if encoding_type == "JSON":
|
|
expected_log_regex = [f'"replica": "{replica_id}", ']
|
|
else:
|
|
expected_log_regex = [f".*{replica_id}.*"]
|
|
check_log_file(resp["log_file"], expected_log_regex)
|
|
|
|
def test_deployment_logging_config_in_code(self, serve_instance):
|
|
"""Deploy application with deployment logging config inside the code"""
|
|
client = serve_instance
|
|
config_dict = self.get_deploy_config(model_within_logging_config=True)
|
|
config = ServeDeploySchema.model_validate(config_dict)
|
|
client.deploy_apps(config)
|
|
wait_for_condition(
|
|
lambda: httpx.post("http://localhost:8000/app1").status_code == 200
|
|
)
|
|
resp = httpx.post("http://localhost:8000/app1").json()
|
|
check_log_file(resp["log_file"], [".*this_is_debug_info.*"])
|
|
|
|
def test_overwritting_logging_config(self, serve_instance):
|
|
"""Overwrite the default logging config with application logging config"""
|
|
client = serve_instance
|
|
config_dict = self.get_deploy_config()
|
|
config = ServeDeploySchema.model_validate(config_dict)
|
|
client.deploy_apps(config)
|
|
|
|
wait_for_condition(
|
|
lambda: httpx.post("http://localhost:8000/app1").status_code == 200
|
|
)
|
|
|
|
def get_replica_info_format(replica_id: ReplicaID) -> str:
|
|
app_name = replica_id.deployment_id.app_name
|
|
deployment_name = replica_id.deployment_id.name
|
|
return f"{app_name}_{deployment_name} {replica_id.unique_id}"
|
|
|
|
# By default, log level is "INFO"
|
|
r = httpx.post("http://localhost:8000/app1")
|
|
r.raise_for_status()
|
|
request_id = r.headers["X-Request-Id"]
|
|
replica_id = ReplicaID.from_full_id_str(r.json()["replica"])
|
|
|
|
# Make sure 'model_debug_level' log content does not exist.
|
|
with pytest.raises(AssertionError):
|
|
check_log_file(r.json()["log_file"], [".*this_is_debug_info.*"])
|
|
|
|
# Check the log formatting.
|
|
check_log_file(
|
|
r.json()["log_file"],
|
|
f" {get_replica_info_format(replica_id)} {request_id} ",
|
|
)
|
|
|
|
# Set log level to "DEBUG"
|
|
config_dict["applications"][0]["logging_config"] = {
|
|
"log_level": "DEBUG",
|
|
}
|
|
config = ServeDeploySchema.model_validate(config_dict)
|
|
client.deploy_apps(config)
|
|
|
|
wait_for_condition(
|
|
lambda: httpx.post("http://localhost:8000/app1").status_code == 200
|
|
and httpx.post("http://localhost:8000/app1").json()["log_level"]
|
|
== logging.DEBUG,
|
|
)
|
|
r = httpx.post("http://localhost:8000/app1")
|
|
r.raise_for_status()
|
|
request_id = r.headers["X-Request-Id"]
|
|
replica_id = ReplicaID.from_full_id_str(r.json()["replica"])
|
|
check_log_file(
|
|
r.json()["log_file"],
|
|
[
|
|
# Check for DEBUG-level log statement.
|
|
".*this_is_debug_info.*",
|
|
# Check that the log formatting has remained the same.
|
|
f" {get_replica_info_format(replica_id)} {request_id} ",
|
|
],
|
|
)
|
|
|
|
def test_not_overwritting_logging_config_in_yaml(self, serve_instance):
|
|
"""Deployment logging config in yaml should not be overwritten
|
|
by application logging config.
|
|
"""
|
|
client = serve_instance
|
|
config_dict = self.get_deploy_config()
|
|
config_dict["applications"][0]["deployments"] = [
|
|
{
|
|
"name": "Model",
|
|
"logging_config": {
|
|
"log_level": "DEBUG",
|
|
},
|
|
},
|
|
]
|
|
config_dict["applications"][0]["logging_config"] = {
|
|
"log_level": "INFO",
|
|
}
|
|
|
|
config = ServeDeploySchema.model_validate(config_dict)
|
|
client.deploy_apps(config)
|
|
wait_for_condition(
|
|
lambda: httpx.post("http://localhost:8000/app1").status_code == 200
|
|
)
|
|
resp = httpx.post("http://localhost:8000/app1").json()
|
|
check_log_file(resp["log_file"], [".*this_is_debug_info.*"])
|
|
|
|
def test_not_overwritting_logging_config_in_code(self, serve_instance):
|
|
"""Deployment logging config in code should not be overwritten
|
|
by application logging config.
|
|
"""
|
|
client = serve_instance
|
|
config_dict = self.get_deploy_config(model_within_logging_config=True)
|
|
config_dict["applications"][0]["logging_config"] = {
|
|
"log_level": "INFO",
|
|
}
|
|
|
|
config = ServeDeploySchema.model_validate(config_dict)
|
|
client.deploy_apps(config)
|
|
wait_for_condition(
|
|
lambda: httpx.post("http://localhost:8000/app1").status_code == 200
|
|
)
|
|
resp = httpx.post("http://localhost:8000/app1").json()
|
|
check_log_file(resp["log_file"], [".*this_is_debug_info.*"])
|
|
|
|
def test_logs_dir(self, serve_instance):
|
|
client = serve_instance
|
|
config_dict = self.get_deploy_config()
|
|
config_dict["applications"][0]["logging_config"] = {
|
|
"log_level": "DEBUG",
|
|
}
|
|
config = ServeDeploySchema.model_validate(config_dict)
|
|
client.deploy_apps(config)
|
|
wait_for_condition(
|
|
lambda: httpx.post("http://localhost:8000/app1").status_code == 200
|
|
)
|
|
resp = httpx.get("http://127.0.0.1:8000/app1").json()
|
|
|
|
# Construct a new path
|
|
# "/tmp/ray/session_xxx/logs/serve/new_dir"
|
|
paths = resp["log_file"].split("/")
|
|
paths[-1] = "new_dir"
|
|
new_log_dir = "/".join(paths)
|
|
|
|
config_dict["applications"][0]["logging_config"] = {
|
|
"log_level": "DEBUG",
|
|
"logs_dir": new_log_dir,
|
|
}
|
|
config = ServeDeploySchema.model_validate(config_dict)
|
|
client.deploy_apps(config)
|
|
wait_for_condition(
|
|
lambda: httpx.post("http://localhost:8000/app1").status_code == 200
|
|
and "new_dir" in httpx.get("http://127.0.0.1:8000/app1").json()["log_file"]
|
|
)
|
|
resp = httpx.get("http://127.0.0.1:8000/app1").json()
|
|
# log content should be redirected to new file
|
|
check_log_file(resp["log_file"], [".*this_is_debug_info.*"])
|
|
|
|
@pytest.mark.parametrize("enable_access_log", [True, False])
|
|
def test_access_log(self, serve_instance, enable_access_log: bool):
|
|
client = serve_instance
|
|
config_dict = self.get_deploy_config()
|
|
config_dict["applications"][0]["logging_config"] = {
|
|
"enable_access_log": enable_access_log,
|
|
}
|
|
config = ServeDeploySchema.model_validate(config_dict)
|
|
client.deploy_apps(config)
|
|
wait_for_condition(
|
|
lambda: httpx.post("http://localhost:8000/app1").status_code == 200
|
|
)
|
|
resp = httpx.get("http://127.0.0.1:8000/app1")
|
|
assert resp.status_code == 200
|
|
resp = resp.json()
|
|
if enable_access_log:
|
|
check_log_file(resp["log_file"], [".*this_is_access_log.*"])
|
|
else:
|
|
with pytest.raises(AssertionError):
|
|
check_log_file(resp["log_file"], [".*this_is_access_log.*"])
|
|
|
|
|
|
def test_deploy_with_no_applications(serve_instance):
|
|
"""Deploy an empty list of applications, serve should just be started."""
|
|
client = serve_instance
|
|
config = ServeDeploySchema.model_validate({"applications": []})
|
|
client.deploy_apps(config)
|
|
|
|
def serve_running():
|
|
ServeInstanceDetails.model_validate(
|
|
ray.get(client._controller.get_serve_instance_details.remote())
|
|
)
|
|
actors = list_actors(
|
|
filters=[
|
|
("ray_namespace", "=", SERVE_NAMESPACE),
|
|
("state", "=", "ALIVE"),
|
|
]
|
|
)
|
|
actor_names = [actor["class_name"] for actor in actors]
|
|
has_proxy = any("Proxy" in name for name in actor_names)
|
|
return "ServeController" in actor_names and has_proxy
|
|
|
|
wait_for_condition(serve_running)
|
|
|
|
|
|
def test_deployments_not_listed_in_config(serve_instance):
|
|
"""Apply a config without the app's deployments listed. The deployments should
|
|
not redeploy.
|
|
"""
|
|
client = serve_instance
|
|
config = {
|
|
"applications": [{"import_path": "ray.serve.tests.test_config_files.pid.node"}]
|
|
}
|
|
client.deploy_apps(ServeDeploySchema(**config), _blocking=True)
|
|
check_running()
|
|
pid1, _ = httpx.get("http://localhost:8000/").json()
|
|
|
|
# Redeploy the same config (with no deployments listed)
|
|
client.deploy_apps(ServeDeploySchema(**config))
|
|
wait_for_condition(check_running, timeout=15)
|
|
|
|
# It should be the same replica actor
|
|
pids = []
|
|
for _ in range(4):
|
|
pids.append(httpx.get("http://localhost:8000/").json()[0])
|
|
assert all(pid == pid1 for pid in pids)
|
|
|
|
|
|
@pytest.mark.parametrize("rebuild", [True, False])
|
|
def test_redeploy_old_config_after_failed_deployment(serve_instance, rebuild):
|
|
"""
|
|
1. Deploy application which succeeds.
|
|
2. Redeploy application with an import path that fails.
|
|
3. Redeploy the exact same config from step 1.
|
|
|
|
Verify that step 3 succeeds and the application returns to running state.
|
|
"""
|
|
client = serve_instance
|
|
app_config = {
|
|
"name": "default",
|
|
"import_path": "ray.serve.tests.test_config_files.world.DagNode",
|
|
}
|
|
client.deploy_apps(ServeDeploySchema(**{"applications": [app_config]}))
|
|
|
|
def check_application_running():
|
|
status = serve.status().applications["default"]
|
|
assert status.status == "RUNNING"
|
|
assert httpx.post("http://localhost:8000/").text == "wonderful world"
|
|
return True
|
|
|
|
wait_for_condition(check_application_running)
|
|
|
|
# Change config so that redeploy will error
|
|
new_app_config = copy(app_config)
|
|
if rebuild:
|
|
# New import path will cause an error upon importing app
|
|
new_app_config[
|
|
"import_path"
|
|
] = "ray.serve.tests.test_config_files.import_error.app"
|
|
err_msg = "ZeroDivisionError"
|
|
else:
|
|
# Set config for a nonexistent deployment
|
|
new_app_config["deployments"] = [{"name": "nonexistent", "num_replicas": 1}]
|
|
err_msg = "Deployment 'nonexistent' does not exist."
|
|
client.deploy_apps(ServeDeploySchema(**{"applications": [new_app_config]}))
|
|
|
|
def check_deploy_failed(message):
|
|
status = serve.status().applications["default"]
|
|
assert status.status == "DEPLOY_FAILED"
|
|
assert message in status.message
|
|
return True
|
|
|
|
wait_for_condition(check_deploy_failed, message=err_msg)
|
|
|
|
# Redeploy old config
|
|
client.deploy_apps(ServeDeploySchema(**{"applications": [app_config]}))
|
|
|
|
wait_for_condition(check_application_running)
|
|
|
|
|
|
def test_deploy_does_not_affect_dynamic_apps(serve_instance):
|
|
"""
|
|
Deploy a set of apps via the declarative API (REST API) and then a dynamic
|
|
app via the imperative API (`serve.run`).
|
|
|
|
Check that applying a new config via the declarative API does not affect
|
|
the app deployed using the imperative API.
|
|
"""
|
|
client = serve_instance
|
|
config = ServeDeploySchema(
|
|
applications=[
|
|
ServeApplicationSchema(
|
|
name="declarative-app-1",
|
|
route_prefix="/app-1",
|
|
import_path="ray.serve.tests.test_config_files.world.DagNode",
|
|
),
|
|
],
|
|
)
|
|
client.deploy_apps(config, _blocking=True)
|
|
check_running(app_name="declarative-app-1")
|
|
url = get_application_url(app_name="declarative-app-1")
|
|
assert httpx.post(url).text == "wonderful world"
|
|
|
|
# Now `serve.run` a dynamic app.
|
|
@serve.deployment
|
|
class D:
|
|
def __call__(self, *args) -> str:
|
|
return "Hello!"
|
|
|
|
serve.run(D.bind(), name="dynamic-app", route_prefix="/dynamic")
|
|
wait_for_condition(check_running, app_name="dynamic-app")
|
|
url = get_application_url(app_name="dynamic-app")
|
|
assert httpx.post(url).text == "Hello!"
|
|
|
|
# Add a new app via declarative API.
|
|
# Existing declarative app and dynamic app should not be affected.
|
|
config.applications.append(
|
|
ServeApplicationSchema(
|
|
name="declarative-app-2",
|
|
route_prefix="/app-2",
|
|
import_path="ray.serve.tests.test_config_files.world.DagNode",
|
|
),
|
|
)
|
|
client.deploy_apps(config, _blocking=True)
|
|
check_running(app_name="declarative-app-2")
|
|
url = get_application_url(app_name="declarative-app-2")
|
|
assert httpx.post(url).text == "wonderful world"
|
|
|
|
url = get_application_url(app_name="declarative-app-1")
|
|
assert httpx.post(url).text == "wonderful world"
|
|
|
|
url = get_application_url(app_name="dynamic-app")
|
|
assert httpx.post(url).text == "Hello!"
|
|
|
|
# Delete one of the apps via declarative API.
|
|
# Other declarative app and dynamic app should not be affected.
|
|
config.applications.pop(0)
|
|
client.deploy_apps(config)
|
|
wait_for_condition(check_running, app_name="declarative-app-2")
|
|
url = get_application_url(app_name="declarative-app-2")
|
|
assert httpx.post(url).text == "wonderful world"
|
|
|
|
url = get_application_url(app_name="dynamic-app")
|
|
assert httpx.post(url).text == "Hello!"
|
|
|
|
wait_for_condition(lambda: "declarative-app-1" not in serve.status().applications)
|
|
|
|
# Now overwrite the declarative app with a dynamic app with the same name.
|
|
# On subsequent declarative apply, that app should not be affected.
|
|
serve.run(D.bind(), name="declarative-app-2", route_prefix="/app-2")
|
|
wait_for_condition(check_running, app_name="declarative-app-2")
|
|
url = get_application_url(app_name="declarative-app-2")
|
|
assert httpx.post(url).text == "Hello!"
|
|
|
|
config.applications = [
|
|
ServeApplicationSchema(
|
|
name="declarative-app-1",
|
|
route_prefix="/app-1",
|
|
import_path="ray.serve.tests.test_config_files.world.DagNode",
|
|
),
|
|
]
|
|
client.deploy_apps(config, _blocking=True)
|
|
check_running(app_name="declarative-app-1")
|
|
url = get_application_url(app_name="declarative-app-1")
|
|
assert httpx.post(url).text == "wonderful world"
|
|
|
|
wait_for_condition(check_running, app_name="dynamic-app")
|
|
url = get_application_url(app_name="dynamic-app")
|
|
assert httpx.post(url).text == "Hello!"
|
|
|
|
wait_for_condition(check_running, app_name="declarative-app-2")
|
|
url = get_application_url(app_name="declarative-app-2")
|
|
assert httpx.post(url).text == "Hello!"
|
|
|
|
# Verify that the controller does not delete the dynamic apps on recovery.
|
|
ray.kill(client._controller, no_restart=False)
|
|
|
|
wait_for_condition(check_running, app_name="declarative-app-1")
|
|
# It takes some time for the target groups to be ready after controller recovery.
|
|
# So we make sure the target groups are ready before obtaining the URL.
|
|
wait_for_condition(
|
|
check_target_groups_ready, client=client, app_name="declarative-app-1"
|
|
)
|
|
url = get_application_url(app_name="declarative-app-1")
|
|
assert httpx.post(url).text == "wonderful world"
|
|
|
|
wait_for_condition(check_running, app_name="dynamic-app")
|
|
wait_for_condition(check_target_groups_ready, client=client, app_name="dynamic-app")
|
|
url = get_application_url(app_name="dynamic-app")
|
|
assert httpx.post(url).text == "Hello!"
|
|
|
|
wait_for_condition(check_running, app_name="declarative-app-2")
|
|
wait_for_condition(
|
|
check_target_groups_ready, client=client, app_name="declarative-app-2"
|
|
)
|
|
url = get_application_url(app_name="declarative-app-2")
|
|
assert httpx.post(url).text == "Hello!"
|
|
|
|
# Now overwrite the dynamic app with a declarative one and check that it gets
|
|
# deleted upon another apply that doesn't include it.
|
|
config.applications = [
|
|
ServeApplicationSchema(
|
|
name="declarative-app-2",
|
|
route_prefix="/app-2",
|
|
import_path="ray.serve.tests.test_config_files.world.DagNode",
|
|
),
|
|
]
|
|
client.deploy_apps(config, _blocking=True)
|
|
check_running(app_name="declarative-app-2")
|
|
url = get_application_url(app_name="declarative-app-2")
|
|
assert httpx.post(url).text == "wonderful world"
|
|
|
|
config.applications = []
|
|
client.deploy_apps(config)
|
|
|
|
wait_for_condition(lambda: "declarative-app-2" not in serve.status().applications)
|
|
|
|
|
|
def test_change_route_prefix(serve_instance):
|
|
# Deploy application with route prefix /old
|
|
client = serve_instance
|
|
app_config = {
|
|
"name": "default",
|
|
"route_prefix": "/old",
|
|
"import_path": "ray.serve.tests.test_config_files.pid.node",
|
|
}
|
|
client.deploy_apps(
|
|
ServeDeploySchema(**{"applications": [app_config]}), _blocking=True
|
|
)
|
|
check_running()
|
|
url = get_application_url()
|
|
pid1 = httpx.get(url).json()[0]
|
|
# Redeploy application with route prefix /new.
|
|
app_config["route_prefix"] = "/new"
|
|
client.deploy_apps(ServeDeploySchema(**{"applications": [app_config]}))
|
|
wait_for_condition(check_running)
|
|
# Check that the old route is gone and the response from the new route
|
|
# has the same PID (replica wasn't restarted).
|
|
def check_switched():
|
|
# Old route should be gone
|
|
url = get_application_url(exclude_route_prefix=True)
|
|
resp = httpx.get(f"{url}/old")
|
|
assert "Path '/old' not found." in resp.text
|
|
|
|
# Response from new route should be same PID
|
|
url = get_application_url(exclude_route_prefix=True)
|
|
pid2 = httpx.get(f"{url}/new").json()[0]
|
|
assert pid2 == pid1
|
|
return True
|
|
|
|
wait_for_condition(check_switched)
|
|
|
|
|
|
def test_num_replicas_auto_api(serve_instance):
|
|
"""Test setting only `num_replicas="auto"`."""
|
|
client = serve_instance
|
|
config_template = {
|
|
"import_path": "ray.serve.tests.test_config_files.pid.node",
|
|
"deployments": [{"name": "f", "num_replicas": "auto"}],
|
|
}
|
|
|
|
client.deploy_apps(
|
|
ServeDeploySchema.model_validate({"applications": [config_template]})
|
|
)
|
|
wait_for_condition(check_running, timeout=15)
|
|
print("Application is RUNNING.")
|
|
check_num_replicas_eq("f", 1)
|
|
|
|
app_details = client.get_serve_details()["applications"][SERVE_DEFAULT_APP_NAME]
|
|
deployment_config = app_details["deployments"]["f"]["deployment_config"]
|
|
assert "num_replicas" not in deployment_config
|
|
assert deployment_config["max_ongoing_requests"] == 5
|
|
assert deployment_config["autoscaling_config"] == {
|
|
# Set by `num_replicas="auto"`
|
|
"target_ongoing_requests": 2.0,
|
|
"min_replicas": 1,
|
|
"max_replicas": 100,
|
|
# Untouched defaults
|
|
"look_back_period_s": 30.0,
|
|
"metrics_interval_s": 10.0,
|
|
"upscale_delay_s": 30.0,
|
|
"downscale_delay_s": 600.0,
|
|
"downscale_to_zero_delay_s": None,
|
|
"upscale_smoothing_factor": None,
|
|
"downscale_smoothing_factor": None,
|
|
"upscaling_factor": None,
|
|
"downscaling_factor": None,
|
|
"smoothing_factor": 1.0,
|
|
"initial_replicas": None,
|
|
"aggregation_function": "mean",
|
|
"policy": {
|
|
"policy_function": "ray.serve.autoscaling_policy:default_autoscaling_policy",
|
|
"policy_kwargs": {},
|
|
},
|
|
}
|
|
|
|
|
|
def test_num_replicas_auto_basic(serve_instance):
|
|
"""Test `num_replicas="auto"` and the default values are used in autoscaling."""
|
|
client = serve_instance
|
|
signal = SignalActor.options(name="signal123").remote()
|
|
|
|
config_template = {
|
|
"import_path": "ray.serve.tests.test_config_files.get_signal.app",
|
|
"deployments": [
|
|
{
|
|
"name": "A",
|
|
"num_replicas": "auto",
|
|
"autoscaling_config": {
|
|
"look_back_period_s": 2.0,
|
|
"metrics_interval_s": 1.0,
|
|
"upscale_delay_s": 1.0,
|
|
},
|
|
"graceful_shutdown_timeout_s": 1,
|
|
}
|
|
],
|
|
}
|
|
|
|
print(time.ctime(), "Deploying pid application.")
|
|
client.deploy_apps(
|
|
ServeDeploySchema.model_validate({"applications": [config_template]})
|
|
)
|
|
wait_for_condition(check_running, timeout=15)
|
|
print(time.ctime(), "Application is RUNNING.")
|
|
check_num_replicas_eq("A", 1)
|
|
|
|
app_details = client.get_serve_details()["applications"][SERVE_DEFAULT_APP_NAME]
|
|
deployment_config = app_details["deployments"]["A"]["deployment_config"]
|
|
# Set by `num_replicas="auto"`
|
|
assert "num_replicas" not in deployment_config
|
|
assert deployment_config["max_ongoing_requests"] == 5
|
|
assert deployment_config["autoscaling_config"] == {
|
|
# Set by `num_replicas="auto"`
|
|
"target_ongoing_requests": 2.0,
|
|
"min_replicas": 1,
|
|
"max_replicas": 100,
|
|
# Overrided by `autoscaling_config`
|
|
"look_back_period_s": 2.0,
|
|
"metrics_interval_s": 1.0,
|
|
"upscale_delay_s": 1.0,
|
|
# Untouched defaults
|
|
"downscale_delay_s": 600.0,
|
|
"downscale_to_zero_delay_s": None,
|
|
"upscale_smoothing_factor": None,
|
|
"downscale_smoothing_factor": None,
|
|
"upscaling_factor": None,
|
|
"downscaling_factor": None,
|
|
"smoothing_factor": 1.0,
|
|
"initial_replicas": None,
|
|
"aggregation_function": "mean",
|
|
"policy": {
|
|
"policy_function": "ray.serve.autoscaling_policy:default_autoscaling_policy",
|
|
"policy_kwargs": {},
|
|
},
|
|
}
|
|
|
|
h = serve.get_app_handle(SERVE_DEFAULT_APP_NAME)
|
|
for i in range(3):
|
|
[h.remote() for _ in range(2)]
|
|
|
|
def check_num_waiters(target: int):
|
|
assert ray.get(signal.cur_num_waiters.remote()) == target
|
|
return True
|
|
|
|
wait_for_condition(check_num_waiters, target=2 * (i + 1), timeout=30)
|
|
print(time.time(), f"Number of waiters on signal reached {2*(i+1)}.")
|
|
if (
|
|
RAY_SERVE_COLLECT_AUTOSCALING_METRICS_ON_HANDLE is False
|
|
and RAY_SERVE_AGGREGATE_METRICS_AT_CONTROLLER is True
|
|
):
|
|
# When merging timeseries from replicas and handles with LOCF, the same request can appear in
|
|
# both because they report at different times (e.g. replica: 4 running, handle: 2 queued).
|
|
# That double-counts requests and inflates the total, biasing aggregations
|
|
# (especially mean) upward and causing over-scaling.
|
|
wait_for_condition(
|
|
check_num_replicas_gte, name="A", target=i + 1, timeout=30
|
|
)
|
|
else:
|
|
wait_for_condition(
|
|
check_num_replicas_eq, name="A", target=i + 1, timeout=30
|
|
)
|
|
print(time.time(), f"Confirmed number of replicas are at {i+1}.")
|
|
|
|
ray.get(signal.send.remote())
|
|
|
|
|
|
def test_deploy_one_app_failed(serve_instance):
|
|
"""Deploy two applications with separate runtime envs."""
|
|
client = serve_instance
|
|
world_import_path = "ray.serve.tests.test_config_files.world.DagNode"
|
|
fail_import_path = "ray.serve.tests.test_config_files.fail.node"
|
|
config_template = {
|
|
"applications": [
|
|
{
|
|
"name": "app1",
|
|
"route_prefix": "/app1",
|
|
"import_path": world_import_path,
|
|
},
|
|
{
|
|
"name": "app2",
|
|
"route_prefix": "/app2",
|
|
"import_path": fail_import_path,
|
|
},
|
|
],
|
|
}
|
|
|
|
client.deploy_apps(ServeDeploySchema(**config_template))
|
|
|
|
wait_for_condition(
|
|
lambda: httpx.post("http://localhost:8000/app1").text == "wonderful world"
|
|
)
|
|
|
|
wait_for_condition(
|
|
lambda: serve.status().applications["app1"].status == ApplicationStatus.RUNNING
|
|
and serve.status().applications["app2"].status
|
|
== ApplicationStatus.DEPLOY_FAILED
|
|
)
|
|
|
|
# Ensure the request doesn't hang and actually returns a 503 error.
|
|
# The timeout is there to prevent the test from hanging and blocking
|
|
# the test suite if it does fail.
|
|
r = httpx.post("http://localhost:8000/app2", timeout=10)
|
|
assert r.status_code == 503 and "unavailable" in r.text.lower()
|
|
|
|
|
|
def test_deploy_with_route_prefix_conflict(serve_instance):
|
|
world_import_path = "ray.serve.tests.test_config_files.world.DagNode"
|
|
pizza_import_path = "ray.serve.tests.test_config_files.pizza.serve_dag"
|
|
client = serve_instance
|
|
test_config = {
|
|
"applications": [
|
|
{
|
|
"name": "app1",
|
|
"route_prefix": "/app1",
|
|
"import_path": world_import_path,
|
|
},
|
|
{
|
|
"name": "app2",
|
|
"route_prefix": "/app2",
|
|
"import_path": pizza_import_path,
|
|
},
|
|
],
|
|
}
|
|
|
|
client.deploy_apps(ServeDeploySchema(**test_config))
|
|
|
|
wait_for_condition(
|
|
lambda: httpx.get("http://localhost:8000/app1").text == "wonderful world"
|
|
)
|
|
wait_for_condition(
|
|
lambda: httpx.post("http://localhost:8000/app2", json=["ADD", 2]).text
|
|
== "4 pizzas please!"
|
|
)
|
|
|
|
# Buffer time
|
|
time.sleep(1)
|
|
|
|
test_config["applications"][1] = {
|
|
"name": "app3",
|
|
"route_prefix": "/app2",
|
|
"import_path": world_import_path,
|
|
}
|
|
|
|
client.deploy_apps(ServeDeploySchema(**test_config))
|
|
|
|
def check():
|
|
serve_details = ServeInstanceDetails(
|
|
**ray.get(client._controller.get_serve_instance_details.remote())
|
|
)
|
|
app1_running = (
|
|
"app1" in serve_details.applications
|
|
and serve_details.applications["app1"].status == "RUNNING"
|
|
)
|
|
app3_running = (
|
|
"app3" in serve_details.applications
|
|
and serve_details.applications["app3"].status == "RUNNING"
|
|
)
|
|
app2_gone = "app2" not in serve_details.applications
|
|
return app1_running and app3_running and app2_gone
|
|
|
|
wait_for_condition(check)
|
|
|
|
# app1 and app3 should be up and running
|
|
wait_for_condition(
|
|
lambda: httpx.get("http://localhost:8000/app1").text == "wonderful world"
|
|
)
|
|
wait_for_condition(
|
|
lambda: httpx.get("http://localhost:8000/app2").text == "wonderful world"
|
|
)
|
|
|
|
|
|
def test_update_config_graceful_shutdown_timeout(serve_instance):
|
|
"""Check that replicas stay alive when graceful_shutdown_timeout_s is updated"""
|
|
client = serve_instance
|
|
|
|
config_template = {
|
|
"import_path": "ray.serve.tests.test_config_files.pid.node",
|
|
"deployments": [{"name": "f", "graceful_shutdown_timeout_s": 1000}],
|
|
}
|
|
|
|
# Deploy first time
|
|
client.deploy_apps(
|
|
ServeDeploySchema.model_validate({"applications": [config_template]})
|
|
)
|
|
wait_for_condition(check_running, timeout=15)
|
|
handle = serve.get_app_handle(SERVE_DEFAULT_APP_NAME)
|
|
|
|
# Start off with signal ready, and send query
|
|
handle.send.remote().result()
|
|
pid1 = handle.remote().result()[0]
|
|
print("PID of replica after first deployment:", pid1)
|
|
|
|
# Redeploy with shutdown timeout set to 5 seconds
|
|
config_template["deployments"][0]["graceful_shutdown_timeout_s"] = 5
|
|
client.deploy_apps(
|
|
ServeDeploySchema.model_validate({"applications": [config_template]})
|
|
)
|
|
wait_for_condition(check_running, timeout=15)
|
|
|
|
pid2 = handle.remote().result()[0]
|
|
assert pid1 == pid2
|
|
print("PID of replica after redeployment:", pid2)
|
|
|
|
# Send blocking query
|
|
handle.send.remote(clear=True)
|
|
handle.remote()
|
|
# Try to delete deployment, should be blocked until the timeout at 5 seconds
|
|
client.delete_apps([SERVE_DEFAULT_APP_NAME], blocking=False)
|
|
# Replica should be dead within 10 second timeout, which means
|
|
# graceful_shutdown_timeout_s was successfully updated lightweightly
|
|
wait_for_condition(partial(check_deployments_dead, [DeploymentID(name="f")]))
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(pytest.main(["-v", "-s", __file__]))
|