chore: import upstream snapshot with attribution

python/ray/serve/tests/test_config_files/apps_with_args.yaml

@@ -0,0 +1,17 @@
+applications:
+  - name: untyped_default
+    route_prefix: /untyped_default
+    import_path: ray.serve.tests.test_config_files.arg_builders.build_echo_app
+  - name: untyped_hello
+    route_prefix: /untyped_hello
+    import_path: ray.serve.tests.test_config_files.arg_builders.build_echo_app
+    args:
+      message: hello
+  - name: typed_default
+    route_prefix: /typed_default
+    import_path: ray.serve.tests.test_config_files.arg_builders.build_echo_app_typed
+  - name: typed_hello
+    route_prefix: /typed_hello
+    import_path: ray.serve.tests.test_config_files.arg_builders.build_echo_app_typed
+    args:
+      message: hello

python/ray/serve/tests/test_config_files/arg_builders.py

@@ -0,0 +1,25 @@
+from pydantic import BaseModel
+
+from ray import serve
+
+
+class TypedArgs(BaseModel):
+    message: str = "DEFAULT"
+
+
+@serve.deployment(ray_actor_options={"num_cpus": 0})
+class Echo:
+    def __init__(self, message: str):
+        print("Echo message:", message)
+        self._message = message
+
+    def __call__(self, *args):
+        return self._message
+
+
+def build_echo_app(args):
+    return Echo.bind(args.get("message", "DEFAULT"))
+
+
+def build_echo_app_typed(args: TypedArgs):
+    return Echo.bind(args.message)

python/ray/serve/tests/test_config_files/arithmetic.yaml

@@ -0,0 +1,20 @@
+applications:
+  - name: default
+    import_path: "dir.subdir.a.add_and_sub.serve_dag"
+
+    runtime_env:
+      # Keep these pinned remote URIs in sync with tests/common/remote_uris.py.
+      working_dir: "https://github.com/ray-project/test_dag/archive/203140040e4ab50b9d35b4773ec5c22615c034b3.zip"
+
+    deployments:
+      - name: "Router"
+        graceful_shutdown_timeout_s: 0.0001
+      - name: "Add"
+        graceful_shutdown_timeout_s: 0.0001
+      - name: "Subtract"
+        graceful_shutdown_timeout_s: 0.0001
+        ray_actor_options:
+          runtime_env:
+            # Keep these pinned remote URIs in sync with tests/common/remote_uris.py.
+            py_modules:
+              - "https://github.com/ray-project/test_module/archive/aa6f366f7daa78c98408c27d917a983caa9f888b.zip"

python/ray/serve/tests/test_config_files/bad_multi_config.yaml

@@ -0,0 +1,25 @@
+applications:
+  - name: "app1"
+    route_prefix: "/app1"
+    import_path: ray.serve.tests.test_config_files.pizza.serve_dag
+    deployments:
+      - name: Multiplier
+        user_config:
+          factor: 1
+
+      - name: Adder
+        user_config:
+          increment: 1
+
+  - name: "app2"
+    # Route prefixes should be unique across all apps!
+    route_prefix: "/app1"
+    import_path: ray.serve.tests.test_config_files.pizza.serve_dag
+    deployments:
+      - name: Multiplier
+        user_config:
+          factor: 2
+
+      - name: Adder
+        user_config:
+          increment: 3

python/ray/serve/tests/test_config_files/bad_runtime_env.yaml

@@ -0,0 +1,4 @@
+applications:
+  - name: default
+    import_path: ray.serve.tests.test_config_files.test_dag.basic_dag.DagNode
+    runtime_env: {"working_dir": "s3://does_not_exist.zip"}

python/ray/serve/tests/test_config_files/basic_graph.yaml

@@ -0,0 +1,4 @@
+applications:
+  - name: default
+    route_prefix: /
+    import_path: ray.serve.tests.test_config_files.test_dag.basic_dag.DagNode

python/ray/serve/tests/test_config_files/basic_graph_http.yaml

@@ -0,0 +1,7 @@
+http_options:
+  host: 127.0.0.1
+  port: 8005
+
+applications:
+  - name: default
+    import_path: ray.serve.tests.test_config_files.test_dag.basic_dag.DagNode

python/ray/serve/tests/test_config_files/basic_multi.yaml

@@ -0,0 +1,3 @@
+applications:
+  - name: "app1"
+    import_path: ray.serve.tests.test_config_files.test_dag.basic_dag.DagNode

python/ray/serve/tests/test_config_files/basic_multi_http.yaml

@@ -0,0 +1,6 @@
+http_options:
+  port: 8005
+
+applications:
+  - name: "app1"
+    import_path: ray.serve.tests.test_config_files.test_dag.basic_dag.DagNode

python/ray/serve/tests/test_config_files/broken_app.py

@@ -0,0 +1,26 @@
+import ray.cloudpickle as pickle
+from ray import serve
+
+
+class NonserializableException(Exception):
+    """This exception cannot be serialized."""
+
+    def __reduce__(self):
+        raise RuntimeError("This exception cannot be serialized!")
+
+
+# Confirm that NonserializableException cannot be serialized.
+try:
+    pickle.dumps(NonserializableException())
+except RuntimeError as e:
+    assert "This exception cannot be serialized!" in repr(e)
+
+raise NonserializableException("custom exception info")
+
+
+@serve.deployment
+def f():
+    pass
+
+
+app = f.bind()

python/ray/serve/tests/test_config_files/config_only_deployment_actor.py

@@ -0,0 +1,20 @@
+"""Deployment that uses deployment actors but defines them only via config override.
+
+The deployment has no deployment_actors in the @serve.deployment decorator;
+they are added purely through the declarative config override.
+"""
+
+import ray
+from ray import serve
+
+
+@serve.deployment(ray_actor_options={"num_cpus": 0.1})
+class ConfigOnlyDriver:
+    """Uses get_deployment_actor; deployment_actors come from config only."""
+
+    def __call__(self):
+        counter = serve.get_deployment_actor("counter")
+        return str(ray.get(counter.get.remote()))
+
+
+app = ConfigOnlyDriver.bind()

python/ray/serve/tests/test_config_files/delete_blocked.py

@@ -0,0 +1,13 @@
+import asyncio
+
+from ray import serve
+
+
+@serve.deployment
+class A:
+    async def __del__(self):
+        while True:
+            await asyncio.sleep(0.1)
+
+
+app = A.bind()

python/ray/serve/tests/test_config_files/deploy_grpc_app.yaml

@@ -0,0 +1,8 @@
+grpc_options:
+  port: 9000
+  grpc_servicer_functions:
+    - ray.serve.generated.serve_pb2_grpc.add_UserDefinedServiceServicer_to_server
+
+applications:
+  - name: app1
+    import_path: ray.serve.tests.test_config_files.grpc_deployment:g

python/ray/serve/tests/test_config_files/deploy_grpc_model_composition.yaml

@@ -0,0 +1,8 @@
+grpc_options:
+  port: 9000
+  grpc_servicer_functions:
+    - ray.serve.generated.serve_pb2_grpc.add_FruitServiceServicer_to_server
+
+applications:
+  - name: app1
+    import_path: ray.serve.tests.test_config_files.grpc_deployment:g2

python/ray/serve/tests/test_config_files/deploy_grpc_multiplexed_app.yaml

@@ -0,0 +1,8 @@
+grpc_options:
+  port: 9000
+  grpc_servicer_functions:
+    - ray.serve.generated.serve_pb2_grpc.add_UserDefinedServiceServicer_to_server
+
+applications:
+  - name: app1
+    import_path: ray.serve.tests.test_config_files.grpc_deployment:multiplexed_g

python/ray/serve/tests/test_config_files/deployment_actors.yaml

@@ -0,0 +1,11 @@
+applications:
+  - name: default
+    route_prefix: /
+    import_path: ray.serve.tests.test_config_files.config_only_deployment_actor:app
+    deployments:
+      - name: ConfigOnlyDriver
+        deployment_actors:
+          - name: counter
+            actor_class: ray.serve.tests.test_deployment_actors:SharedCounter
+            init_kwargs:
+              start: 88

python/ray/serve/tests/test_config_files/deployment_fail.yaml

@@ -0,0 +1,3 @@
+applications:
+  - name: default
+    import_path: ray.serve.tests.test_config_files.fail.node

python/ray/serve/tests/test_config_files/deployment_fail_2.yaml

@@ -0,0 +1,3 @@
+applications:
+  - name: default
+    import_path: ray.serve.tests.test_config_files.fail_2.node

python/ray/serve/tests/test_config_files/deployment_uses_utils_module.yaml

@@ -0,0 +1,4 @@
+applications:
+  - name: default
+    route_prefix: /
+    import_path: ray.serve.tests.test_config_files.test_dag.hello_serve:model

python/ray/serve/tests/test_config_files/duplicate_app_names.yaml

@@ -0,0 +1,8 @@
+applications:
+  - name: app1
+    route_prefix: /a
+    import_path: ray.serve.tests.test_config_files.test_dag.basic_dag.DagNode
+
+  - name: app1
+    route_prefix: /b
+    import_path: ray.serve.tests.test_config_files.test_dag.basic_dag.DagNode

python/ray/serve/tests/test_config_files/duplicate_app_routes.yaml

@@ -0,0 +1,8 @@
+applications:
+  - name: app1
+    route_prefix: /alice
+    import_path: ray.serve.tests.test_config_files.test_dag.basic_dag.DagNode
+
+  - name: app2
+    route_prefix: /alice
+    import_path: ray.serve.tests.test_config_files.test_dag.basic_dag.DagNode

python/ray/serve/tests/test_config_files/fail.py

@@ -0,0 +1,10 @@
+from ray import serve
+
+
+@serve.deployment
+class A:
+    def __init__(self):
+        _ = 1 / 0
+
+
+node = A.bind()

python/ray/serve/tests/test_config_files/fail_2.py

@@ -0,0 +1,13 @@
+import time
+
+from ray import serve
+
+
+@serve.deployment
+class A:
+    def __init__(self):
+        time.sleep(5)
+        _ = 1 / 0
+
+
+node = A.bind()

python/ray/serve/tests/test_config_files/fastapi_deployment.py

@@ -0,0 +1,16 @@
+from fastapi import FastAPI
+
+from ray import serve
+
+app = FastAPI(docs_url="/my_docs")
+
+
+@serve.deployment
+@serve.ingress(app)
+class FastAPIDeployment:
+    @app.get("/hello")
+    def incr(self):
+        return "Hello world!"
+
+
+node = FastAPIDeployment.bind()

python/ray/serve/tests/test_config_files/flaky_build.py

@@ -0,0 +1,29 @@
+"""Test fixture: raises on first FLAKY_BUILD_FAIL_COUNT imports, then succeeds.
+
+A counter file persists state across the fresh worker processes that
+``build_serve_application``'s ``max_calls=1`` decorator spawns on each retry.
+"""
+
+import os
+
+from ray import serve
+
+_COUNTER_FILE = os.environ["FLAKY_BUILD_COUNTER_FILE"]
+_FAIL_COUNT = int(os.environ.get("FLAKY_BUILD_FAIL_COUNT", "3"))
+
+with open(_COUNTER_FILE, "r") as f:
+    _attempts = int(f.read().strip() or "0")
+with open(_COUNTER_FILE, "w") as f:
+    f.write(str(_attempts + 1))
+
+if _attempts < _FAIL_COUNT:
+    raise RuntimeError(f"flaky build failure on attempt {_attempts + 1}")
+
+
+@serve.deployment
+class FlakyApp:
+    def __call__(self):
+        return "ok"
+
+
+node = FlakyApp.bind()

python/ray/serve/tests/test_config_files/flex_deployment_actor.py

@@ -0,0 +1,21 @@
+"""Flexible driver that works with or without deployment actors.
+
+Used by declarative redeployment tests that need to transition from
+no-deployment-actors to having them.
+"""
+
+import ray
+from ray import serve
+
+
+@serve.deployment(ray_actor_options={"num_cpus": 0.1})
+class FlexDriver:
+    def __call__(self):
+        try:
+            actor = serve.get_deployment_actor("counter")
+            return str(ray.get(actor.get.remote()))
+        except Exception:
+            return "no_actor"
+
+
+app = FlexDriver.bind()

python/ray/serve/tests/test_config_files/gang_scheduling.py

@@ -0,0 +1,15 @@
+import json
+import os
+
+from ray import serve
+
+
+@serve.deployment
+class GangApp:
+    def __call__(self, *args):
+        ctx = serve.context._get_internal_replica_context()
+        gc = ctx.gang_context
+        return json.dumps({"pid": os.getpid(), "gang_id": gc.gang_id if gc else None})
+
+
+app = GangApp.bind()

python/ray/serve/tests/test_config_files/gang_scheduling.yaml

@@ -0,0 +1,12 @@
+applications:
+- name: gang_app
+  route_prefix: /
+  import_path: ray.serve.tests.test_config_files.gang_scheduling:app
+  deployments:
+  - name: GangApp
+    num_replicas: 4
+    ray_actor_options:
+      num_cpus: 0.25
+    gang_scheduling_config:
+      gang_size: 2
+      gang_placement_strategy: PACK

python/ray/serve/tests/test_config_files/gang_scheduling_2_replicas.yaml

@@ -0,0 +1,12 @@
+applications:
+- name: gang_app
+  route_prefix: /
+  import_path: ray.serve.tests.test_config_files.gang_scheduling:app
+  deployments:
+  - name: GangApp
+    num_replicas: 2
+    ray_actor_options:
+      num_cpus: 0.25
+    gang_scheduling_config:
+      gang_size: 2
+      gang_placement_strategy: PACK

python/ray/serve/tests/test_config_files/gang_scheduling_4_replicas.yaml

@@ -0,0 +1,12 @@
+applications:
+- name: gang_app
+  route_prefix: /
+  import_path: ray.serve.tests.test_config_files.gang_scheduling:app
+  deployments:
+  - name: GangApp
+    num_replicas: 4
+    ray_actor_options:
+      num_cpus: 0.25
+    gang_scheduling_config:
+      gang_size: 2
+      gang_placement_strategy: PACK

python/ray/serve/tests/test_config_files/get_multi_deployment_signal_app.py

@@ -0,0 +1,29 @@
+import os
+
+import ray
+from ray import serve
+from ray.serve.handle import DeploymentHandle
+
+
+@serve.deployment
+class A:
+    def __init__(self, b: DeploymentHandle):
+        self.b = b
+        self.signal = ray.get_actor("signal_A", namespace="default_test_namespace")
+
+    async def __call__(self):
+        await self.signal.wait.remote()
+        return os.getpid()
+
+
+@serve.deployment
+class B:
+    def __init__(self):
+        self.signal = ray.get_actor("signal_B", namespace="default_test_namespace")
+
+    async def __call__(self):
+        await self.signal.wait.remote()
+        return os.getpid()
+
+
+app = A.bind(B.bind())

python/ray/serve/tests/test_config_files/get_signal.py

@@ -0,0 +1,15 @@
+import os
+
+import ray
+from ray import serve
+
+
+@serve.deployment
+class A:
+    async def __call__(self):
+        signal = ray.get_actor("signal123")
+        await signal.wait.remote()
+        return os.getpid()
+
+
+app = A.bind()

python/ray/serve/tests/test_config_files/grpc_deployment.py

@@ -0,0 +1,125 @@
+from typing import Dict
+
+# Users need to include their custom message type which will be embedded in the request.
+from ray import serve
+from ray.serve.generated import serve_pb2
+from ray.serve.handle import DeploymentHandle
+
+
+@serve.deployment
+class GrpcDeployment:
+    def __call__(self, user_message):
+        greeting = f"Hello {user_message.name} from {user_message.foo}"
+        num_x2 = user_message.num * 2
+        user_response = serve_pb2.UserDefinedResponse(
+            greeting=greeting,
+            num_x2=num_x2,
+        )
+        return user_response
+
+    def Method1(self, user_message):
+        greeting = f"Hello {user_message.name} from method1"
+        num_x2 = user_message.num * 3
+        user_response = serve_pb2.UserDefinedResponse(
+            greeting=greeting,
+            num_x2=num_x2,
+        )
+        return user_response
+
+    def Streaming(self, user_message):
+        for i in range(10):
+            greeting = f"{i}: Hello {user_message.name} from {user_message.foo}"
+            num_x2 = user_message.num * 2 + i
+            user_response = serve_pb2.UserDefinedResponse(
+                greeting=greeting,
+                num_x2=num_x2,
+            )
+            yield user_response
+
+
+g = GrpcDeployment.options(name="grpc-deployment").bind()
+
+
+# NOTE: model multiplexing is kept on a separate deployment (not the shared `g`)
+# because it is not supported on the ingress deployment when direct ingress /
+# HAProxy is enabled (the multiplexed model ID is not propagated to the replica).
+# Tests that exercise it must be skipped under those modes.
+@serve.deployment
+class MultiplexedGrpcDeployment:
+    def __call__(self, user_message):
+        greeting = f"Hello {user_message.name} from {user_message.foo}"
+        return serve_pb2.UserDefinedResponse(greeting=greeting)
+
+    @serve.multiplexed(max_num_models_per_replica=1)
+    async def get_model(self, model_id: str) -> str:
+        return f"loading model: {model_id}"
+
+    async def Method2(self, user_message):
+        model_id = serve.get_multiplexed_model_id()
+        model = await self.get_model(model_id)
+        user_response = serve_pb2.UserDefinedResponse(
+            greeting=f"Method2 called model, {model}",
+        )
+        return user_response
+
+
+multiplexed_g = MultiplexedGrpcDeployment.options(name="grpc-deployment").bind()
+
+
+@serve.deployment(ray_actor_options={"num_cpus": 0})
+class FruitMarket:
+    def __init__(
+        self,
+        _orange_stand: DeploymentHandle,
+        _apple_stand: DeploymentHandle,
+    ):
+        self.directory = {
+            "ORANGE": _orange_stand,
+            "APPLE": _apple_stand,
+        }
+
+    async def FruitStand(self, fruit_amounts_proto):
+        fruit_amounts = {}
+        if fruit_amounts_proto.orange:
+            fruit_amounts["ORANGE"] = fruit_amounts_proto.orange
+        if fruit_amounts_proto.apple:
+            fruit_amounts["APPLE"] = fruit_amounts_proto.apple
+        if fruit_amounts_proto.banana:
+            fruit_amounts["BANANA"] = fruit_amounts_proto.banana
+
+        costs = await self.check_price(fruit_amounts)
+        return serve_pb2.FruitCosts(costs=costs)
+
+    async def check_price(self, inputs: Dict[str, int]) -> float:
+        costs = 0
+        for fruit, amount in inputs.items():
+            if fruit not in self.directory:
+                return
+            fruit_stand = self.directory[fruit]
+            costs += await fruit_stand.remote(int(amount))
+        return costs
+
+
+@serve.deployment(ray_actor_options={"num_cpus": 0})
+class OrangeStand:
+    def __init__(self):
+        self.price = 2.0
+
+    def __call__(self, num_oranges: int):
+        return num_oranges * self.price
+
+
+@serve.deployment(ray_actor_options={"num_cpus": 0})
+class AppleStand:
+    def __init__(self):
+        self.price = 3.0
+
+    def __call__(self, num_oranges: int):
+        return num_oranges * self.price
+
+
+orange_stand = OrangeStand.bind()
+apple_stand = AppleStand.bind()
+g2 = FruitMarket.options(name="grpc-deployment-model-composition").bind(
+    orange_stand, apple_stand
+)

python/ray/serve/tests/test_config_files/hangs.py

@@ -0,0 +1,13 @@
+import ray
+from ray import serve
+
+signal = ray.get_actor("signal123")
+ray.get(signal.wait.remote())
+
+
+@serve.deployment(ray_actor_options={"num_cpus": 0.1})
+def f():
+    return "hello world"
+
+
+app = f.bind()

python/ray/serve/tests/test_config_files/http_option_app_sleeps.py

@@ -0,0 +1,14 @@
+import time
+
+from ray import serve
+
+
+@serve.deployment
+def sleep(request):
+    sleep_s = float(request.query_params.get("sleep_s", 0))
+    print(f"sleep_s: {sleep_s}")
+    time.sleep(sleep_s)
+    return "Task Succeeded!"
+
+
+sleep_node = sleep.bind()

python/ray/serve/tests/test_config_files/http_option_request_timeout_s.yaml

@@ -0,0 +1,7 @@
+http_options:
+  request_timeout_s: 0.1
+
+applications:
+  - name: "app1"
+    import_path: ray.serve.tests.test_config_files.http_option_app_sleeps.sleep_node
+    route_prefix: /app1

python/ray/serve/tests/test_config_files/import_error.py

@@ -0,0 +1,11 @@
+from ray import serve
+
+_ = 1 / 0
+
+
+@serve.deployment(ray_actor_options={"num_cpus": 0.1})
+def f(*args):
+    return "hello world"
+
+
+app = f.bind()

python/ray/serve/tests/test_config_files/logging_config_test.py

@@ -0,0 +1,62 @@
+import logging
+
+import ray
+from ray import serve
+from ray.exceptions import RayActorError
+from ray.serve.context import _get_global_client
+
+logger = logging.getLogger("ray.serve")
+
+
+@serve.deployment
+class Model:
+    def __call__(self):
+        logger.debug("this_is_debug_info")
+        logger.info("this_is_access_log", extra={"serve_access_log": True})
+
+        log_file = logger.handlers[1].target.baseFilename
+
+        return {
+            "log_file": log_file,
+            "replica": serve.get_replica_context().replica_id.to_full_id_str(),
+            "log_level": logger.level,
+            "num_handlers": len(logger.handlers),
+        }
+
+
+@serve.deployment
+class Router:
+    def __init__(self, handle):
+        self.handle = handle
+
+    async def __call__(self):
+        logger.debug("this_is_debug_info_from_router")
+        log_info = await self.handle.remote()
+        if len(logger.handlers) == 2:
+            log_info["router_log_file"] = logger.handlers[1].target.baseFilename
+        else:
+            log_info["router_log_file"] = None
+        log_info["router_log_level"] = logger.level
+
+        try:
+            # Add controller log file path
+            client = _get_global_client()
+            _, log_file_path = ray.get(client._controller._get_logging_config.remote())
+        except RayActorError:
+            log_file_path = None
+        log_info["controller_log_file"] = log_file_path
+        return log_info
+
+
+model = Router.bind(Model.bind())
+
+
+@serve.deployment(logging_config={"log_level": "DEBUG"})
+class ModelWithConfig:
+    def __call__(self):
+        logger.debug("this_is_debug_info")
+        log_file = logger.handlers[1].target.baseFilename
+        return {"log_file": log_file}
+
+
+model2 = ModelWithConfig.bind()

python/ray/serve/tests/test_config_files/max_replicas_per_node.py

@@ -0,0 +1,10 @@
+from ray import serve
+
+
+@serve.deployment
+class D:
+    def __call__(self, *args):
+        return "hi"
+
+
+app = D.bind()

python/ray/serve/tests/test_config_files/max_replicas_per_node.yaml

@@ -0,0 +1,15 @@
+applications:
+  - name: valid
+    route_prefix: /valid
+    import_path: ray.serve.tests.test_config_files.max_replicas_per_node.app
+    deployments:
+      - name: D
+        max_replicas_per_node: 2
+
+  - name: invalid
+    route_prefix: /invalid
+    import_path: ray.serve.tests.test_config_files.max_replicas_per_node.app
+    deployments:
+      - name: D
+        # Non-positive max_replicas_per_node.
+        max_replicas_per_node: 0

python/ray/serve/tests/test_config_files/missing_runtime_env.yaml

@@ -0,0 +1,4 @@
+applications:
+  - name: default
+    import_path: "basic_dag.DagNode"
+    route_prefix: /

python/ray/serve/tests/test_config_files/multi_fastapi.py

@@ -0,0 +1,28 @@
+from fastapi import FastAPI
+
+from ray import serve
+from ray.serve.handle import DeploymentHandle
+
+app1 = FastAPI()
+app2 = FastAPI()
+
+
+@serve.deployment
+@serve.ingress(app2)
+class SubModel:
+    def add(self, a: int):
+        return a + 1
+
+
+@serve.deployment
+@serve.ingress(app1)
+class Model:
+    def __init__(self, submodel: DeploymentHandle):
+        self.submodel = submodel
+
+    @app1.get("/{a}")
+    async def func(self, a: int):
+        return await self.submodel.add.remote(a)
+
+
+invalid_model = Model.bind(SubModel.bind())

python/ray/serve/tests/test_config_files/pid.py

@@ -0,0 +1,78 @@
+import asyncio
+import os
+import time
+
+from ray import serve
+
+
+@serve.deployment
+class f:
+    def __init__(self, async_wait: bool = False):
+        # whether to use wait() (busy spin) or async_wait() (busy spin
+        # while yielding event loop)
+        self._async = async_wait
+
+        # to be updated through reconfigure()
+        self.name = "default_name"
+
+        # used to block calls to __call__()
+        self.ready = True
+        # used to check how many times __call__() has been called
+        self.counter = 0
+
+        # used to block calls to health_check()
+        self.health_check_ready = True
+        # used to check how many times health_check() has been called
+        self.health_check_counter = 0
+
+    async def get_counter(self, health_check=False) -> int:
+        if health_check:
+            return self.health_check_counter
+        else:
+            return self.counter
+
+    def send(self, clear=False, health_check=False):
+        if health_check:
+            self.health_check_ready = not clear
+        else:
+            self.ready = not clear
+
+    def wait(self, _health_check=False):
+        if _health_check:
+            while not self.health_check_ready:
+                time.sleep(0.1)
+        else:
+            while not self.ready:
+                time.sleep(0.1)
+
+    async def async_wait(self, _health_check=False):
+        if _health_check:
+            while not self.health_check_ready:
+                await asyncio.sleep(0.1)
+        else:
+            while not self.ready:
+                await asyncio.sleep(0.1)
+
+    def reconfigure(self, config: dict):
+        self.name = config.get("name", "default_name")
+
+    async def __call__(self):
+        self.counter += 1
+        if self._async:
+            await self.async_wait()
+        else:
+            self.wait()
+
+        return os.getpid(), self.name
+
+    async def check_health(self):
+        self.health_check_counter += 1
+        if self._async:
+            await self.async_wait(_health_check=True)
+        else:
+            self.wait(_health_check=True)
+
+
+node = f.bind()
+dup_node = f.bind()
+async_node = f.bind(async_wait=True)

python/ray/serve/tests/test_config_files/pizza.py

@@ -0,0 +1,79 @@
+from enum import Enum
+from typing import Dict, List
+
+import starlette.requests
+
+from ray import serve
+from ray.serve.handle import DeploymentHandle
+
+
+class Operation(str, Enum):
+    ADDITION = "ADD"
+    MULTIPLICATION = "MUL"
+
+
+@serve.deployment(ray_actor_options={"num_cpus": 0.15})
+class Router:
+    def __init__(self, multiplier: DeploymentHandle, adder: DeploymentHandle):
+        self.adder = adder
+        self.multiplier = multiplier
+
+    async def route(self, op: Operation, input: int) -> str:
+        if op == Operation.ADDITION:
+            amount = await self.adder.add.remote(input)
+        elif op == Operation.MULTIPLICATION:
+            amount = await self.multiplier.multiply.remote(input)
+
+        return f"{amount} pizzas please!"
+
+    async def __call__(self, request: starlette.requests.Request) -> str:
+        op, input = await request.json()
+        return await self.route(op, input)
+
+
+@serve.deployment(
+    user_config={
+        "factor": 3,
+    },
+    ray_actor_options={"num_cpus": 0.15},
+)
+class Multiplier:
+    def __init__(self, factor: int):
+        self.factor = factor
+
+    def reconfigure(self, config: Dict):
+        self.factor = config.get("factor", -1)
+
+    def multiply(self, input_factor: int) -> int:
+        return input_factor * self.factor
+
+
+@serve.deployment(
+    user_config={
+        "increment": 2,
+    },
+    ray_actor_options={"num_cpus": 0.15},
+)
+class Adder:
+    def __init__(self, increment: int):
+        self.increment = increment
+
+    def reconfigure(self, config: Dict):
+        self.increment = config.get("increment", -1)
+
+    def add(self, input: int) -> int:
+        return input + self.increment
+
+
+async def json_resolver(request: starlette.requests.Request) -> List:
+    return await request.json()
+
+
+# Overwritten by user_config
+ORIGINAL_INCREMENT = 1
+ORIGINAL_FACTOR = 1
+
+
+multiplier = Multiplier.bind(ORIGINAL_FACTOR)
+adder = Adder.bind(ORIGINAL_INCREMENT)
+serve_dag = Router.bind(multiplier, adder)

python/ray/serve/tests/test_config_files/pizza.yaml

@@ -0,0 +1,20 @@
+applications:
+  - name: default
+    route_prefix: /
+    import_path: ray.serve.tests.test_config_files.test_dag.conditional_dag.serve_dag
+
+    deployments:
+      - name: Router
+        graceful_shutdown_timeout_s: 0.0001
+
+      - name: Multiplier
+        graceful_shutdown_timeout_s: 0.0001
+        user_config:
+          factor: 1
+
+      - name: Adder
+        graceful_shutdown_timeout_s: 0.0001
+        ray_actor_options:
+          runtime_env:
+            env_vars:
+              override_increment: '1'

python/ray/serve/tests/test_config_files/pizza_world.yaml

@@ -0,0 +1,16 @@
+applications:
+  - name: "app1"
+    route_prefix: "/app1"
+    import_path: ray.serve.tests.test_config_files.world.DagNode
+
+  - name: "app2"
+    route_prefix: "/app2"
+    import_path: ray.serve.tests.test_config_files.pizza.serve_dag
+    deployments:
+      - name: Multiplier
+        user_config:
+          factor: 10
+
+      - name: Adder
+        user_config:
+          increment: 10

python/ray/serve/tests/test_config_files/ray_already_initialized.py

@@ -0,0 +1,12 @@
+import ray
+from ray import serve
+
+ray.init(address="auto")
+
+
+@serve.deployment
+def f():
+    return "foobar"
+
+
+app = f.bind()

python/ray/serve/tests/test_config_files/replica_placement_groups.py

@@ -0,0 +1,10 @@
+from ray import serve
+
+
+@serve.deployment
+class D:
+    def __call__(self, *args):
+        return "hi"
+
+
+app = D.bind()

python/ray/serve/tests/test_config_files/replica_placement_groups.yaml

@@ -0,0 +1,27 @@
+applications:
+  - name: valid
+    route_prefix: /valid
+    import_path: ray.serve.tests.test_config_files.replica_placement_groups.app
+    deployments:
+      - name: D
+        placement_group_bundles:
+          - {"CPU": 1}
+        placement_group_strategy: STRICT_PACK
+
+  - name: invalid_bundles
+    route_prefix: /invalid_bundles
+    import_path: ray.serve.tests.test_config_files.replica_placement_groups.app
+    deployments:
+      - name: D
+        # Insufficient resources for the replica actor.
+        placement_group_bundles:
+          - {"CPU": 0.1}
+
+  - name: invalid_strategy
+    route_prefix: /invalid_strategy
+    import_path: ray.serve.tests.test_config_files.replica_placement_groups.app
+    deployments:
+      - name: D
+        placement_group_bundles:
+          - {"CPU": 1}
+        placement_group_strategy: FAKE_NEWS

python/ray/serve/tests/test_config_files/sqlalchemy.py

@@ -0,0 +1,15 @@
+from ray import serve
+
+
+@serve.deployment
+class TestDeployment:
+    def __init__(self):
+        import pymysql
+        from sqlalchemy import create_engine
+
+        pymysql.install_as_MySQLdb()
+
+        create_engine("mysql://some_wrong_url:3306").connect()
+
+
+app = TestDeployment.bind()

python/ray/serve/tests/test_config_files/sqlalchemy.yaml

@@ -0,0 +1,11 @@
+applications:
+  - name: default
+    import_path: ray.serve.tests.test_config_files.sqlalchemy.app
+    deployments:
+      - name: TestDeployment
+        num_replicas: 1
+        ray_actor_options:
+          runtime_env:
+            pip:
+              - PyMySQL
+              - sqlalchemy==1.3.19

python/ray/serve/tests/test_config_files/syntax_error.py

@@ -0,0 +1 @@
+x = (1 + 2

python/ray/serve/tests/test_config_files/syntax_error.yaml

@@ -0,0 +1,3 @@
+applications:
+  - name: default
+    import_path: ray.serve.tests.test_config_files.syntax_error.app

python/ray/serve/tests/test_config_files/test_dag/basic_dag.py

@@ -0,0 +1,28 @@
+from ray import serve
+from ray.serve.handle import DeploymentHandle
+
+
+@serve.deployment(
+    ray_actor_options={
+        "num_cpus": 0.1,
+    }
+)
+def f(*args):
+    return "wonderful world"
+
+
+@serve.deployment(
+    ray_actor_options={
+        "num_cpus": 0.1,
+    }
+)
+class BasicDriver:
+    def __init__(self, h: DeploymentHandle):
+        self._h = h
+
+    async def __call__(self):
+        return await self._h.remote()
+
+
+FNode = f.bind()
+DagNode = BasicDriver.bind(FNode)

python/ray/serve/tests/test_config_files/test_dag/conditional_dag.py

@@ -0,0 +1,97 @@
+import os
+from enum import Enum
+from typing import Dict
+
+import starlette.requests
+
+from ray import serve
+from ray.serve.handle import DeploymentHandle
+
+
+class Operation(str, Enum):
+    ADDITION = "ADD"
+    MULTIPLICATION = "MUL"
+
+
+@serve.deployment(
+    ray_actor_options={
+        "num_cpus": 0.1,
+    }
+)
+class Router:
+    def __init__(self, multiplier: DeploymentHandle, adder: DeploymentHandle):
+        self.adder = adder
+        self.multiplier = multiplier
+
+    async def route(self, op: Operation, input: int) -> int:
+        if op == Operation.ADDITION:
+            amount = await self.adder.add.remote(input)
+        elif op == Operation.MULTIPLICATION:
+            amount = await self.multiplier.multiply.remote(input)
+
+        return f"{amount} pizzas please!"
+
+    async def __call__(self, request: starlette.requests.Request) -> str:
+        op, input = await request.json()
+        return await self.route(op, input)
+
+
+@serve.deployment(
+    user_config={
+        "factor": 3,
+    },
+    ray_actor_options={
+        "num_cpus": 0.1,
+        "runtime_env": {
+            "env_vars": {
+                "override_factor": "-2",
+            }
+        },
+    },
+)
+class Multiplier:
+    def __init__(self, factor: int):
+        self.factor = factor
+
+    def reconfigure(self, config: Dict):
+        self.factor = config.get("factor", -1)
+
+    def multiply(self, input_factor: int) -> int:
+        if os.getenv("override_factor") is not None:
+            return input_factor * int(os.getenv("override_factor"))
+        return input_factor * self.factor
+
+
+@serve.deployment(
+    user_config={
+        "increment": 2,
+    },
+    ray_actor_options={
+        "num_cpus": 0.1,
+        "runtime_env": {
+            "env_vars": {
+                "override_increment": "-2",
+            }
+        },
+    },
+)
+class Adder:
+    def __init__(self, increment: int):
+        self.increment = increment
+
+    def reconfigure(self, config: Dict):
+        self.increment = config.get("increment", -1)
+
+    def add(self, input: int) -> int:
+        if os.getenv("override_increment") is not None:
+            return input + int(os.getenv("override_increment"))
+        return input + self.increment
+
+
+# Overwritten by user_config
+ORIGINAL_INCREMENT = 1
+ORIGINAL_FACTOR = 1
+
+multiplier = Multiplier.bind(ORIGINAL_FACTOR)
+adder = Adder.bind(ORIGINAL_INCREMENT)
+serve_dag = Router.bind(multiplier, adder)

python/ray/serve/tests/test_config_files/test_dag/dir/subdir/a/add_and_sub.py

@@ -0,0 +1,65 @@
+from enum import Enum
+
+import starlette.requests
+
+from ray import serve
+from ray.serve.handle import DeploymentHandle
+
+
+class Operation(str, Enum):
+    ADD = "ADD"
+    SUBTRACT = "SUB"
+
+
+@serve.deployment(
+    ray_actor_options={
+        "num_cpus": 0.1,
+    }
+)
+class Add:
+    # Requires the test_dag repo as a py_module:
+    # https://github.com/ray-project/test_dag
+
+    def add(self, input: int) -> int:
+        from dir2.library import add_one
+
+        return add_one(input)
+
+
+@serve.deployment(
+    ray_actor_options={
+        "num_cpus": 0.1,
+    }
+)
+class Subtract:
+    # Requires the test_module repo as a py_module:
+    # https://github.com/ray-project/test_module
+
+    def subtract(self, input: int) -> int:
+        from test_module.test import one
+
+        return input - one()  # Returns input - 2
+
+
+@serve.deployment(
+    ray_actor_options={
+        "num_cpus": 0.1,
+    }
+)
+class Router:
+    def __init__(self, adder: DeploymentHandle, subtractor: DeploymentHandle):
+        self.adder = adder
+        self.subtractor = subtractor
+
+    async def __call__(self, request: starlette.requests.Request) -> int:
+        op, input = await request.json()
+
+        if op == Operation.ADD:
+            return await self.adder.add.remote(input)
+        elif op == Operation.SUBTRACT:
+            return await self.subtractor.subtract.remote(input)
+
+
+adder = Add.bind()
+subtractor = Subtract.bind()
+serve_dag = Router.bind(adder, subtractor)

python/ray/serve/tests/test_config_files/test_dag/dir2/library.py

@@ -0,0 +1,2 @@
+def add_one(inp):
+    return inp + 1

python/ray/serve/tests/test_config_files/test_dag/hello_serve.py

@@ -0,0 +1,13 @@
+from starlette.requests import Request
+
+from ray import serve
+from ray.serve.tests.test_config_files.test_dag.utils.test import hello
+
+
+@serve.deployment
+class HelloModel:
+    async def __call__(self, starlette_request: Request) -> None:
+        return hello()
+
+
+model = HelloModel.bind()

python/ray/serve/tests/test_config_files/test_dag/utils/test.py

@@ -0,0 +1,2 @@
+def hello():
+    return "hello_from_utils"

python/ray/serve/tests/test_config_files/two_pizzas.yaml

@@ -0,0 +1,24 @@
+applications:
+  - name: "app1"
+    route_prefix: "/app1"
+    import_path: ray.serve.tests.test_config_files.pizza.serve_dag
+    deployments:
+      - name: Multiplier
+        user_config:
+          factor: 1
+
+      - name: Adder
+        user_config:
+          increment: 1
+
+  - name: "app2"
+    route_prefix: "/app2"
+    import_path: ray.serve.tests.test_config_files.pizza.serve_dag
+    deployments:
+      - name: Multiplier
+        user_config:
+          factor: 2
+
+      - name: Adder
+        user_config:
+          increment: 3

python/ray/serve/tests/test_config_files/use_current_working_directory.py

@@ -0,0 +1,9 @@
+from ray import serve
+
+
+@serve.deployment
+def f():
+    return "hi"
+
+
+app = f.bind()

python/ray/serve/tests/test_config_files/use_current_working_directory.yaml

@@ -0,0 +1,6 @@
+applications:
+  - name: app1
+    route_prefix: /
+    import_path: use_current_working_directory:app
+    deployments:
+      - name: f

python/ray/serve/tests/test_config_files/use_custom_autoscaling.yaml

@@ -0,0 +1,16 @@
+applications:
+- name: app1
+  route_prefix: /
+  import_path: ray.serve.tests.test_config_files.use_custom_autoscaling_policy:app
+  deployments:
+  - name: CustomAutoscalingPolicy
+    num_replicas: auto
+    ray_actor_options:
+      num_cpus: 0.0
+    autoscaling_config:
+      min_replicas: 1
+      max_replicas: 2
+      upscale_delay_s: 1
+      downscale_delay_s: 2
+      policy:
+        policy_function: ray.serve.tests.test_config_files.use_custom_autoscaling_policy.custom_autoscaling_policy

python/ray/serve/tests/test_config_files/use_custom_autoscaling_policy.py

@@ -0,0 +1,16 @@
+from ray import serve
+from ray.serve.config import AutoscalingContext
+
+
+def custom_autoscaling_policy(ctx: AutoscalingContext):
+    print("custom_autoscaling_policy")
+    return 2, {}
+
+
+@serve.deployment
+class CustomAutoscalingPolicy:
+    def __call__(self):
+        return "hello_from_custom_autoscaling_policy"
+
+
+app = CustomAutoscalingPolicy.bind()

python/ray/serve/tests/test_config_files/use_custom_request_router.py

@@ -0,0 +1,47 @@
+import random
+from typing import (
+    List,
+    Optional,
+)
+
+from ray import serve
+from ray.serve.context import _get_internal_replica_context
+from ray.serve.request_router import (
+    PendingRequest,
+    ReplicaID,
+    ReplicaResult,
+    RequestRouter,
+    RunningReplica,
+)
+
+
+class UniformRequestRouter(RequestRouter):
+    async def choose_replicas(
+        self,
+        candidate_replicas: List[RunningReplica],
+        pending_request: Optional[PendingRequest] = None,
+    ) -> List[List[RunningReplica]]:
+        print("UniformRequestRouter routing request")
+        index = random.randint(0, len(candidate_replicas) - 1)
+        return [[candidate_replicas[index]]]
+
+    def on_request_routed(
+        self,
+        pending_request: PendingRequest,
+        replica_id: ReplicaID,
+        result: ReplicaResult,
+    ):
+        print("on_request_routed callback is called!!")
+
+
+@serve.deployment
+class UniformRequestRouterApp:
+    def __init__(self):
+        context = _get_internal_replica_context()
+        self.replica_id: ReplicaID = context.replica_id
+
+    async def __call__(self):
+        return "hello_from_custom_request_router"
+
+
+app = UniformRequestRouterApp.bind()

python/ray/serve/tests/test_config_files/use_custom_request_router.yaml

@@ -0,0 +1,14 @@
+applications:
+- name: app1
+  route_prefix: /
+  import_path: ray.serve.tests.test_config_files.use_custom_request_router:app
+  deployments:
+  - name: UniformRequestRouterApp
+    num_replicas: 2
+    ray_actor_options:
+      num_cpus: 0.0
+    request_router_config:
+      request_router_class: ray.serve.tests.test_config_files.use_custom_request_router.UniformRequestRouter
+      request_router_kwargs: {}
+      request_routing_stats_period_s: 10
+      request_routing_stats_timeout_s: 30

python/ray/serve/tests/test_config_files/world.py

@@ -0,0 +1,20 @@
+from ray import serve
+from ray.serve.handle import DeploymentHandle
+
+
+@serve.deployment(ray_actor_options={"num_cpus": 0.1})
+def f(*args):
+    return "wonderful world"
+
+
+@serve.deployment(ray_actor_options={"num_cpus": 0.1})
+class BasicDriver:
+    def __init__(self, h: DeploymentHandle):
+        self._h = h
+
+    async def __call__(self):
+        return await self._h.remote()
+
+
+FNode = f.bind()
+DagNode = BasicDriver.bind(FNode)