import logging
import re
import sys
import time
from copy import copy
from functools import partial
from typing import List

import httpx
import pytest

import ray
import ray.actor
from ray import serve
from ray._common.test_utils import SignalActor, wait_for_condition
from ray.serve._private.common import DeploymentID, ReplicaID
from ray.serve._private.constants import (
    RAY_SERVE_AGGREGATE_METRICS_AT_CONTROLLER,
    RAY_SERVE_COLLECT_AUTOSCALING_METRICS_ON_HANDLE,
    SERVE_DEFAULT_APP_NAME,
    SERVE_NAMESPACE,
)
from ray.serve._private.test_utils import (
    check_num_replicas_eq,
    check_num_replicas_gte,
    check_running,
    check_target_groups_ready,
    get_application_url,
)
from ray.serve.schema import (
    ApplicationStatus,
    ServeApplicationSchema,
    ServeDeploySchema,
    ServeInstanceDetails,
)
from ray.tests.conftest import call_ray_stop_only  # noqa: F401
from ray.util.state import list_actors


def check_log_file(log_file: str, expected_regex: list):
    with open(log_file, "r") as f:
        s = f.read()
        print(s)
        for regex in expected_regex:
            assert re.findall(regex, s) != [], f"Did not find pattern '{regex}' in {s}"
    return True


def check_deployments_dead(deployment_ids: List[DeploymentID]):
    prefixes = [f"{id.app_name}#{id.name}" for id in deployment_ids]
    actor_names = [
        actor["name"] for actor in list_actors(filters=[("state", "=", "ALIVE")])
    ]
    return all(f"ServeReplica::{p}" not in actor_names for p in prefixes)


class TestDeploywithLoggingConfig:
    def get_deploy_config(self, model_within_logging_config: bool = False):
        if model_within_logging_config:
            path = "ray.serve.tests.test_config_files.logging_config_test.model2"
        else:
            path = "ray.serve.tests.test_config_files.logging_config_test.model"
        return {
            "applications": [
                {
                    "name": "app1",
                    "route_prefix": "/app1",
                    "import_path": path,
                },
            ],
        }

    @pytest.mark.parametrize("encoding_type", ["TEXT", "JSON"])
    def test_deploy_app_with_application_logging_config(
        self, serve_instance, encoding_type: str
    ):
        """Deploy application with application logging config"""
        client = serve_instance
        config_dict = self.get_deploy_config()

        config_dict["applications"][0]["logging_config"] = {
            "encoding": encoding_type,
        }
        config = ServeDeploySchema.model_validate(config_dict)
        client.deploy_apps(config)
        wait_for_condition(
            lambda: httpx.post("http://localhost:8000/app1").status_code == 200
        )

        resp = httpx.post("http://localhost:8000/app1").json()

        replica_id = resp["replica"].split("#")[-1]
        if encoding_type == "JSON":
            expected_log_regex = [f'"replica": "{replica_id}", ']
        else:
            expected_log_regex = [f".*{replica_id}.*"]
        check_log_file(resp["log_file"], expected_log_regex)

    @pytest.mark.parametrize("encoding_type", ["TEXT", "JSON"])
    def test_deploy_app_with_deployment_logging_config(
        self, serve_instance, encoding_type: str
    ):
        client = serve_instance
        """Deploy application with deployment logging config inside the yaml"""
        config_dict = self.get_deploy_config()

        config_dict["applications"][0]["deployments"] = [
            {
                "name": "Model",
                "logging_config": {
                    "encoding": encoding_type,
                },
            },
        ]
        config = ServeDeploySchema.model_validate(config_dict)
        client.deploy_apps(config)
        wait_for_condition(
            lambda: httpx.post("http://localhost:8000/app1").status_code == 200
        )

        resp = httpx.post("http://localhost:8000/app1").json()

        replica_id = resp["replica"].split("#")[-1]
        if encoding_type == "JSON":
            expected_log_regex = [f'"replica": "{replica_id}", ']
        else:
            expected_log_regex = [f".*{replica_id}.*"]
        check_log_file(resp["log_file"], expected_log_regex)

    def test_deployment_logging_config_in_code(self, serve_instance):
        """Deploy application with deployment logging config inside the code"""
        client = serve_instance
        config_dict = self.get_deploy_config(model_within_logging_config=True)
        config = ServeDeploySchema.model_validate(config_dict)
        client.deploy_apps(config)
        wait_for_condition(
            lambda: httpx.post("http://localhost:8000/app1").status_code == 200
        )
        resp = httpx.post("http://localhost:8000/app1").json()
        check_log_file(resp["log_file"], [".*this_is_debug_info.*"])

    def test_overwritting_logging_config(self, serve_instance):
        """Overwrite the default logging config with application logging config"""
        client = serve_instance
        config_dict = self.get_deploy_config()
        config = ServeDeploySchema.model_validate(config_dict)
        client.deploy_apps(config)

        wait_for_condition(
            lambda: httpx.post("http://localhost:8000/app1").status_code == 200
        )

        def get_replica_info_format(replica_id: ReplicaID) -> str:
            app_name = replica_id.deployment_id.app_name
            deployment_name = replica_id.deployment_id.name
            return f"{app_name}_{deployment_name} {replica_id.unique_id}"

        # By default, log level is "INFO"
        r = httpx.post("http://localhost:8000/app1")
        r.raise_for_status()
        request_id = r.headers["X-Request-Id"]
        replica_id = ReplicaID.from_full_id_str(r.json()["replica"])

        # Make sure 'model_debug_level' log content does not exist.
        with pytest.raises(AssertionError):
            check_log_file(r.json()["log_file"], [".*this_is_debug_info.*"])

        # Check the log formatting.
        check_log_file(
            r.json()["log_file"],
            f" {get_replica_info_format(replica_id)} {request_id} ",
        )

        # Set log level to "DEBUG"
        config_dict["applications"][0]["logging_config"] = {
            "log_level": "DEBUG",
        }
        config = ServeDeploySchema.model_validate(config_dict)
        client.deploy_apps(config)

        wait_for_condition(
            lambda: httpx.post("http://localhost:8000/app1").status_code == 200
            and httpx.post("http://localhost:8000/app1").json()["log_level"]
            == logging.DEBUG,
        )
        r = httpx.post("http://localhost:8000/app1")
        r.raise_for_status()
        request_id = r.headers["X-Request-Id"]
        replica_id = ReplicaID.from_full_id_str(r.json()["replica"])
        check_log_file(
            r.json()["log_file"],
            [
                # Check for DEBUG-level log statement.
                ".*this_is_debug_info.*",
                # Check that the log formatting has remained the same.
                f" {get_replica_info_format(replica_id)} {request_id} ",
            ],
        )

    def test_not_overwritting_logging_config_in_yaml(self, serve_instance):
        """Deployment logging config in yaml should not be overwritten
        by application logging config.
        """
        client = serve_instance
        config_dict = self.get_deploy_config()
        config_dict["applications"][0]["deployments"] = [
            {
                "name": "Model",
                "logging_config": {
                    "log_level": "DEBUG",
                },
            },
        ]
        config_dict["applications"][0]["logging_config"] = {
            "log_level": "INFO",
        }

        config = ServeDeploySchema.model_validate(config_dict)
        client.deploy_apps(config)
        wait_for_condition(
            lambda: httpx.post("http://localhost:8000/app1").status_code == 200
        )
        resp = httpx.post("http://localhost:8000/app1").json()
        check_log_file(resp["log_file"], [".*this_is_debug_info.*"])

    def test_not_overwritting_logging_config_in_code(self, serve_instance):
        """Deployment logging config in code should not be overwritten
        by application logging config.
        """
        client = serve_instance
        config_dict = self.get_deploy_config(model_within_logging_config=True)
        config_dict["applications"][0]["logging_config"] = {
            "log_level": "INFO",
        }

        config = ServeDeploySchema.model_validate(config_dict)
        client.deploy_apps(config)
        wait_for_condition(
            lambda: httpx.post("http://localhost:8000/app1").status_code == 200
        )
        resp = httpx.post("http://localhost:8000/app1").json()
        check_log_file(resp["log_file"], [".*this_is_debug_info.*"])

    def test_logs_dir(self, serve_instance):
        client = serve_instance
        config_dict = self.get_deploy_config()
        config_dict["applications"][0]["logging_config"] = {
            "log_level": "DEBUG",
        }
        config = ServeDeploySchema.model_validate(config_dict)
        client.deploy_apps(config)
        wait_for_condition(
            lambda: httpx.post("http://localhost:8000/app1").status_code == 200
        )
        resp = httpx.get("http://127.0.0.1:8000/app1").json()

        # Construct a new path
        # "/tmp/ray/session_xxx/logs/serve/new_dir"
        paths = resp["log_file"].split("/")
        paths[-1] = "new_dir"
        new_log_dir = "/".join(paths)

        config_dict["applications"][0]["logging_config"] = {
            "log_level": "DEBUG",
            "logs_dir": new_log_dir,
        }
        config = ServeDeploySchema.model_validate(config_dict)
        client.deploy_apps(config)
        wait_for_condition(
            lambda: httpx.post("http://localhost:8000/app1").status_code == 200
            and "new_dir" in httpx.get("http://127.0.0.1:8000/app1").json()["log_file"]
        )
        resp = httpx.get("http://127.0.0.1:8000/app1").json()
        # log content should be redirected to new file
        check_log_file(resp["log_file"], [".*this_is_debug_info.*"])

    @pytest.mark.parametrize("enable_access_log", [True, False])
    def test_access_log(self, serve_instance, enable_access_log: bool):
        client = serve_instance
        config_dict = self.get_deploy_config()
        config_dict["applications"][0]["logging_config"] = {
            "enable_access_log": enable_access_log,
        }
        config = ServeDeploySchema.model_validate(config_dict)
        client.deploy_apps(config)
        wait_for_condition(
            lambda: httpx.post("http://localhost:8000/app1").status_code == 200
        )
        resp = httpx.get("http://127.0.0.1:8000/app1")
        assert resp.status_code == 200
        resp = resp.json()
        if enable_access_log:
            check_log_file(resp["log_file"], [".*this_is_access_log.*"])
        else:
            with pytest.raises(AssertionError):
                check_log_file(resp["log_file"], [".*this_is_access_log.*"])


def test_deploy_with_no_applications(serve_instance):
    """Deploy an empty list of applications, serve should just be started."""
    client = serve_instance
    config = ServeDeploySchema.model_validate({"applications": []})
    client.deploy_apps(config)

    def serve_running():
        ServeInstanceDetails.model_validate(
            ray.get(client._controller.get_serve_instance_details.remote())
        )
        actors = list_actors(
            filters=[
                ("ray_namespace", "=", SERVE_NAMESPACE),
                ("state", "=", "ALIVE"),
            ]
        )
        actor_names = [actor["class_name"] for actor in actors]
        has_proxy = any("Proxy" in name for name in actor_names)
        return "ServeController" in actor_names and has_proxy

    wait_for_condition(serve_running)


def test_deployments_not_listed_in_config(serve_instance):
    """Apply a config without the app's deployments listed. The deployments should
    not redeploy.
    """
    client = serve_instance
    config = {
        "applications": [{"import_path": "ray.serve.tests.test_config_files.pid.node"}]
    }
    client.deploy_apps(ServeDeploySchema(**config), _blocking=True)
    check_running()
    pid1, _ = httpx.get("http://localhost:8000/").json()

    # Redeploy the same config (with no deployments listed)
    client.deploy_apps(ServeDeploySchema(**config))
    wait_for_condition(check_running, timeout=15)

    # It should be the same replica actor
    pids = []
    for _ in range(4):
        pids.append(httpx.get("http://localhost:8000/").json()[0])
    assert all(pid == pid1 for pid in pids)


@pytest.mark.parametrize("rebuild", [True, False])
def test_redeploy_old_config_after_failed_deployment(serve_instance, rebuild):
    """
    1. Deploy application which succeeds.
    2. Redeploy application with an import path that fails.
    3. Redeploy the exact same config from step 1.

    Verify that step 3 succeeds and the application returns to running state.
    """
    client = serve_instance
    app_config = {
        "name": "default",
        "import_path": "ray.serve.tests.test_config_files.world.DagNode",
    }
    client.deploy_apps(ServeDeploySchema(**{"applications": [app_config]}))

    def check_application_running():
        status = serve.status().applications["default"]
        assert status.status == "RUNNING"
        assert httpx.post("http://localhost:8000/").text == "wonderful world"
        return True

    wait_for_condition(check_application_running)

    # Change config so that redeploy will error
    new_app_config = copy(app_config)
    if rebuild:
        # New import path will cause an error upon importing app
        new_app_config[
            "import_path"
        ] = "ray.serve.tests.test_config_files.import_error.app"
        err_msg = "ZeroDivisionError"
    else:
        # Set config for a nonexistent deployment
        new_app_config["deployments"] = [{"name": "nonexistent", "num_replicas": 1}]
        err_msg = "Deployment 'nonexistent' does not exist."
    client.deploy_apps(ServeDeploySchema(**{"applications": [new_app_config]}))

    def check_deploy_failed(message):
        status = serve.status().applications["default"]
        assert status.status == "DEPLOY_FAILED"
        assert message in status.message
        return True

    wait_for_condition(check_deploy_failed, message=err_msg)

    # Redeploy old config
    client.deploy_apps(ServeDeploySchema(**{"applications": [app_config]}))

    wait_for_condition(check_application_running)


def test_deploy_does_not_affect_dynamic_apps(serve_instance):
    """
    Deploy a set of apps via the declarative API (REST API) and then a dynamic
    app via the imperative API (`serve.run`).

    Check that applying a new config via the declarative API does not affect
    the app deployed using the imperative API.
    """
    client = serve_instance
    config = ServeDeploySchema(
        applications=[
            ServeApplicationSchema(
                name="declarative-app-1",
                route_prefix="/app-1",
                import_path="ray.serve.tests.test_config_files.world.DagNode",
            ),
        ],
    )
    client.deploy_apps(config, _blocking=True)
    check_running(app_name="declarative-app-1")
    url = get_application_url(app_name="declarative-app-1")
    assert httpx.post(url).text == "wonderful world"

    # Now `serve.run` a dynamic app.
    @serve.deployment
    class D:
        def __call__(self, *args) -> str:
            return "Hello!"

    serve.run(D.bind(), name="dynamic-app", route_prefix="/dynamic")
    wait_for_condition(check_running, app_name="dynamic-app")
    url = get_application_url(app_name="dynamic-app")
    assert httpx.post(url).text == "Hello!"

    # Add a new app via declarative API.
    # Existing declarative app and dynamic app should not be affected.
    config.applications.append(
        ServeApplicationSchema(
            name="declarative-app-2",
            route_prefix="/app-2",
            import_path="ray.serve.tests.test_config_files.world.DagNode",
        ),
    )
    client.deploy_apps(config, _blocking=True)
    check_running(app_name="declarative-app-2")
    url = get_application_url(app_name="declarative-app-2")
    assert httpx.post(url).text == "wonderful world"

    url = get_application_url(app_name="declarative-app-1")
    assert httpx.post(url).text == "wonderful world"

    url = get_application_url(app_name="dynamic-app")
    assert httpx.post(url).text == "Hello!"

    # Delete one of the apps via declarative API.
    # Other declarative app and dynamic app should not be affected.
    config.applications.pop(0)
    client.deploy_apps(config)
    wait_for_condition(check_running, app_name="declarative-app-2")
    url = get_application_url(app_name="declarative-app-2")
    assert httpx.post(url).text == "wonderful world"

    url = get_application_url(app_name="dynamic-app")
    assert httpx.post(url).text == "Hello!"

    wait_for_condition(lambda: "declarative-app-1" not in serve.status().applications)

    # Now overwrite the declarative app with a dynamic app with the same name.
    # On subsequent declarative apply, that app should not be affected.
    serve.run(D.bind(), name="declarative-app-2", route_prefix="/app-2")
    wait_for_condition(check_running, app_name="declarative-app-2")
    url = get_application_url(app_name="declarative-app-2")
    assert httpx.post(url).text == "Hello!"

    config.applications = [
        ServeApplicationSchema(
            name="declarative-app-1",
            route_prefix="/app-1",
            import_path="ray.serve.tests.test_config_files.world.DagNode",
        ),
    ]
    client.deploy_apps(config, _blocking=True)
    check_running(app_name="declarative-app-1")
    url = get_application_url(app_name="declarative-app-1")
    assert httpx.post(url).text == "wonderful world"

    wait_for_condition(check_running, app_name="dynamic-app")
    url = get_application_url(app_name="dynamic-app")
    assert httpx.post(url).text == "Hello!"

    wait_for_condition(check_running, app_name="declarative-app-2")
    url = get_application_url(app_name="declarative-app-2")
    assert httpx.post(url).text == "Hello!"

    # Verify that the controller does not delete the dynamic apps on recovery.
    ray.kill(client._controller, no_restart=False)

    wait_for_condition(check_running, app_name="declarative-app-1")
    # It takes some time for the target groups to be ready after controller recovery.
    # So we make sure the target groups are ready before obtaining the URL.
    wait_for_condition(
        check_target_groups_ready, client=client, app_name="declarative-app-1"
    )
    url = get_application_url(app_name="declarative-app-1")
    assert httpx.post(url).text == "wonderful world"

    wait_for_condition(check_running, app_name="dynamic-app")
    wait_for_condition(check_target_groups_ready, client=client, app_name="dynamic-app")
    url = get_application_url(app_name="dynamic-app")
    assert httpx.post(url).text == "Hello!"

    wait_for_condition(check_running, app_name="declarative-app-2")
    wait_for_condition(
        check_target_groups_ready, client=client, app_name="declarative-app-2"
    )
    url = get_application_url(app_name="declarative-app-2")
    assert httpx.post(url).text == "Hello!"

    # Now overwrite the dynamic app with a declarative one and check that it gets
    # deleted upon another apply that doesn't include it.
    config.applications = [
        ServeApplicationSchema(
            name="declarative-app-2",
            route_prefix="/app-2",
            import_path="ray.serve.tests.test_config_files.world.DagNode",
        ),
    ]
    client.deploy_apps(config, _blocking=True)
    check_running(app_name="declarative-app-2")
    url = get_application_url(app_name="declarative-app-2")
    assert httpx.post(url).text == "wonderful world"

    config.applications = []
    client.deploy_apps(config)

    wait_for_condition(lambda: "declarative-app-2" not in serve.status().applications)


def test_change_route_prefix(serve_instance):
    # Deploy application with route prefix /old
    client = serve_instance
    app_config = {
        "name": "default",
        "route_prefix": "/old",
        "import_path": "ray.serve.tests.test_config_files.pid.node",
    }
    client.deploy_apps(
        ServeDeploySchema(**{"applications": [app_config]}), _blocking=True
    )
    check_running()
    url = get_application_url()
    pid1 = httpx.get(url).json()[0]
    # Redeploy application with route prefix /new.
    app_config["route_prefix"] = "/new"
    client.deploy_apps(ServeDeploySchema(**{"applications": [app_config]}))
    wait_for_condition(check_running)
    # Check that the old route is gone and the response from the new route
    # has the same PID (replica wasn't restarted).
    def check_switched():
        # Old route should be gone
        url = get_application_url(exclude_route_prefix=True)
        resp = httpx.get(f"{url}/old")
        assert "Path '/old' not found." in resp.text

        # Response from new route should be same PID
        url = get_application_url(exclude_route_prefix=True)
        pid2 = httpx.get(f"{url}/new").json()[0]
        assert pid2 == pid1
        return True

    wait_for_condition(check_switched)


def test_num_replicas_auto_api(serve_instance):
    """Test setting only `num_replicas="auto"`."""
    client = serve_instance
    config_template = {
        "import_path": "ray.serve.tests.test_config_files.pid.node",
        "deployments": [{"name": "f", "num_replicas": "auto"}],
    }

    client.deploy_apps(
        ServeDeploySchema.model_validate({"applications": [config_template]})
    )
    wait_for_condition(check_running, timeout=15)
    print("Application is RUNNING.")
    check_num_replicas_eq("f", 1)

    app_details = client.get_serve_details()["applications"][SERVE_DEFAULT_APP_NAME]
    deployment_config = app_details["deployments"]["f"]["deployment_config"]
    assert "num_replicas" not in deployment_config
    assert deployment_config["max_ongoing_requests"] == 5
    assert deployment_config["autoscaling_config"] == {
        # Set by `num_replicas="auto"`
        "target_ongoing_requests": 2.0,
        "min_replicas": 1,
        "max_replicas": 100,
        # Untouched defaults
        "look_back_period_s": 30.0,
        "metrics_interval_s": 10.0,
        "upscale_delay_s": 30.0,
        "downscale_delay_s": 600.0,
        "downscale_to_zero_delay_s": None,
        "upscale_smoothing_factor": None,
        "downscale_smoothing_factor": None,
        "upscaling_factor": None,
        "downscaling_factor": None,
        "smoothing_factor": 1.0,
        "initial_replicas": None,
        "aggregation_function": "mean",
        "policy": {
            "policy_function": "ray.serve.autoscaling_policy:default_autoscaling_policy",
            "policy_kwargs": {},
        },
    }


def test_num_replicas_auto_basic(serve_instance):
    """Test `num_replicas="auto"` and the default values are used in autoscaling."""
    client = serve_instance
    signal = SignalActor.options(name="signal123").remote()

    config_template = {
        "import_path": "ray.serve.tests.test_config_files.get_signal.app",
        "deployments": [
            {
                "name": "A",
                "num_replicas": "auto",
                "autoscaling_config": {
                    "look_back_period_s": 2.0,
                    "metrics_interval_s": 1.0,
                    "upscale_delay_s": 1.0,
                },
                "graceful_shutdown_timeout_s": 1,
            }
        ],
    }

    print(time.ctime(), "Deploying pid application.")
    client.deploy_apps(
        ServeDeploySchema.model_validate({"applications": [config_template]})
    )
    wait_for_condition(check_running, timeout=15)
    print(time.ctime(), "Application is RUNNING.")
    check_num_replicas_eq("A", 1)

    app_details = client.get_serve_details()["applications"][SERVE_DEFAULT_APP_NAME]
    deployment_config = app_details["deployments"]["A"]["deployment_config"]
    # Set by `num_replicas="auto"`
    assert "num_replicas" not in deployment_config
    assert deployment_config["max_ongoing_requests"] == 5
    assert deployment_config["autoscaling_config"] == {
        # Set by `num_replicas="auto"`
        "target_ongoing_requests": 2.0,
        "min_replicas": 1,
        "max_replicas": 100,
        # Overrided by `autoscaling_config`
        "look_back_period_s": 2.0,
        "metrics_interval_s": 1.0,
        "upscale_delay_s": 1.0,
        # Untouched defaults
        "downscale_delay_s": 600.0,
        "downscale_to_zero_delay_s": None,
        "upscale_smoothing_factor": None,
        "downscale_smoothing_factor": None,
        "upscaling_factor": None,
        "downscaling_factor": None,
        "smoothing_factor": 1.0,
        "initial_replicas": None,
        "aggregation_function": "mean",
        "policy": {
            "policy_function": "ray.serve.autoscaling_policy:default_autoscaling_policy",
            "policy_kwargs": {},
        },
    }

    h = serve.get_app_handle(SERVE_DEFAULT_APP_NAME)
    for i in range(3):
        [h.remote() for _ in range(2)]

        def check_num_waiters(target: int):
            assert ray.get(signal.cur_num_waiters.remote()) == target
            return True

        wait_for_condition(check_num_waiters, target=2 * (i + 1), timeout=30)
        print(time.time(), f"Number of waiters on signal reached {2*(i+1)}.")
        if (
            RAY_SERVE_COLLECT_AUTOSCALING_METRICS_ON_HANDLE is False
            and RAY_SERVE_AGGREGATE_METRICS_AT_CONTROLLER is True
        ):
            # When merging timeseries from replicas and handles with LOCF, the same request can appear in
            # both because they report at different times (e.g. replica: 4 running, handle: 2 queued).
            # That double-counts requests and inflates the total, biasing aggregations
            # (especially mean) upward and causing over-scaling.
            wait_for_condition(
                check_num_replicas_gte, name="A", target=i + 1, timeout=30
            )
        else:
            wait_for_condition(
                check_num_replicas_eq, name="A", target=i + 1, timeout=30
            )
        print(time.time(), f"Confirmed number of replicas are at {i+1}.")

    ray.get(signal.send.remote())


def test_deploy_one_app_failed(serve_instance):
    """Deploy two applications with separate runtime envs."""
    client = serve_instance
    world_import_path = "ray.serve.tests.test_config_files.world.DagNode"
    fail_import_path = "ray.serve.tests.test_config_files.fail.node"
    config_template = {
        "applications": [
            {
                "name": "app1",
                "route_prefix": "/app1",
                "import_path": world_import_path,
            },
            {
                "name": "app2",
                "route_prefix": "/app2",
                "import_path": fail_import_path,
            },
        ],
    }

    client.deploy_apps(ServeDeploySchema(**config_template))

    wait_for_condition(
        lambda: httpx.post("http://localhost:8000/app1").text == "wonderful world"
    )

    wait_for_condition(
        lambda: serve.status().applications["app1"].status == ApplicationStatus.RUNNING
        and serve.status().applications["app2"].status
        == ApplicationStatus.DEPLOY_FAILED
    )

    # Ensure the request doesn't hang and actually returns a 503 error.
    # The timeout is there to prevent the test from hanging and blocking
    # the test suite if it does fail.
    r = httpx.post("http://localhost:8000/app2", timeout=10)
    assert r.status_code == 503 and "unavailable" in r.text.lower()


def test_deploy_with_route_prefix_conflict(serve_instance):
    world_import_path = "ray.serve.tests.test_config_files.world.DagNode"
    pizza_import_path = "ray.serve.tests.test_config_files.pizza.serve_dag"
    client = serve_instance
    test_config = {
        "applications": [
            {
                "name": "app1",
                "route_prefix": "/app1",
                "import_path": world_import_path,
            },
            {
                "name": "app2",
                "route_prefix": "/app2",
                "import_path": pizza_import_path,
            },
        ],
    }

    client.deploy_apps(ServeDeploySchema(**test_config))

    wait_for_condition(
        lambda: httpx.get("http://localhost:8000/app1").text == "wonderful world"
    )
    wait_for_condition(
        lambda: httpx.post("http://localhost:8000/app2", json=["ADD", 2]).text
        == "4 pizzas please!"
    )

    # Buffer time
    time.sleep(1)

    test_config["applications"][1] = {
        "name": "app3",
        "route_prefix": "/app2",
        "import_path": world_import_path,
    }

    client.deploy_apps(ServeDeploySchema(**test_config))

    def check():
        serve_details = ServeInstanceDetails(
            **ray.get(client._controller.get_serve_instance_details.remote())
        )
        app1_running = (
            "app1" in serve_details.applications
            and serve_details.applications["app1"].status == "RUNNING"
        )
        app3_running = (
            "app3" in serve_details.applications
            and serve_details.applications["app3"].status == "RUNNING"
        )
        app2_gone = "app2" not in serve_details.applications
        return app1_running and app3_running and app2_gone

    wait_for_condition(check)

    # app1 and app3 should be up and running
    wait_for_condition(
        lambda: httpx.get("http://localhost:8000/app1").text == "wonderful world"
    )
    wait_for_condition(
        lambda: httpx.get("http://localhost:8000/app2").text == "wonderful world"
    )


def test_update_config_graceful_shutdown_timeout(serve_instance):
    """Check that replicas stay alive when graceful_shutdown_timeout_s is updated"""
    client = serve_instance

    config_template = {
        "import_path": "ray.serve.tests.test_config_files.pid.node",
        "deployments": [{"name": "f", "graceful_shutdown_timeout_s": 1000}],
    }

    # Deploy first time
    client.deploy_apps(
        ServeDeploySchema.model_validate({"applications": [config_template]})
    )
    wait_for_condition(check_running, timeout=15)
    handle = serve.get_app_handle(SERVE_DEFAULT_APP_NAME)

    # Start off with signal ready, and send query
    handle.send.remote().result()
    pid1 = handle.remote().result()[0]
    print("PID of replica after first deployment:", pid1)

    # Redeploy with shutdown timeout set to 5 seconds
    config_template["deployments"][0]["graceful_shutdown_timeout_s"] = 5
    client.deploy_apps(
        ServeDeploySchema.model_validate({"applications": [config_template]})
    )
    wait_for_condition(check_running, timeout=15)

    pid2 = handle.remote().result()[0]
    assert pid1 == pid2
    print("PID of replica after redeployment:", pid2)

    # Send blocking query
    handle.send.remote(clear=True)
    handle.remote()
    # Try to delete deployment, should be blocked until the timeout at 5 seconds
    client.delete_apps([SERVE_DEFAULT_APP_NAME], blocking=False)
    # Replica should be dead within 10 second timeout, which means
    # graceful_shutdown_timeout_s was successfully updated lightweightly
    wait_for_condition(partial(check_deployments_dead, [DeploymentID(name="f")]))


if __name__ == "__main__":
    sys.exit(pytest.main(["-v", "-s", __file__]))