chore: import upstream snapshot with attribution

doc/source/serve/doc_code/app_builder.py

@@ -0,0 +1,85 @@
+# flake8: noqa
+
+# __begin_untyped_builder__
+# hello.py
+
+from typing import Dict
+
+from ray import serve
+from ray.serve import Application
+
+
+@serve.deployment
+class HelloWorld:
+    def __init__(self, message: str):
+        self._message = message
+        print("Message:", self._message)
+
+    def __call__(self, request):
+        return self._message
+
+
+def app_builder(args: Dict[str, str]) -> Application:
+    return HelloWorld.bind(args["message"])
+
+
+# __end_untyped_builder__
+
+import requests
+
+serve.run(app_builder({"message": "Hello bar"}))
+resp = requests.get("http://localhost:8000")
+assert resp.text == "Hello bar"
+
+# __begin_typed_builder__
+# hello.py
+
+from pydantic import BaseModel
+
+from ray import serve
+from ray.serve import Application
+
+
+class HelloWorldArgs(BaseModel):
+    message: str
+
+
+@serve.deployment
+class HelloWorld:
+    def __init__(self, message: str):
+        self._message = message
+        print("Message:", self._message)
+
+    def __call__(self, request):
+        return self._message
+
+
+def typed_app_builder(args: HelloWorldArgs) -> Application:
+    return HelloWorld.bind(args.message)
+
+
+# __end_typed_builder__
+
+serve.run(typed_app_builder(HelloWorldArgs(message="Hello baz")))
+resp = requests.get("http://localhost:8000")
+assert resp.text == "Hello baz"
+
+# __begin_composed_builder__
+from pydantic import BaseModel
+
+from ray.serve import Application
+
+
+class ComposedArgs(BaseModel):
+    model1_uri: str
+    model2_uri: str
+
+
+def composed_app_builder(args: ComposedArgs) -> Application:
+    return IngressDeployment.bind(
+        Model1.bind(args.model1_uri),
+        Model2.bind(args.model2_uri),
+    )
+
+
+# __end_composed_builder__

doc/source/serve/doc_code/application_level_autoscaling.py

@@ -0,0 +1,36 @@
+# __serve_example_begin__
+import time
+from ray import serve
+
+
+@serve.deployment
+class Preprocessor:
+    def __call__(self, input_data: str) -> str:
+        # Simulate preprocessing work
+        time.sleep(0.05)
+        return f"preprocessed_{input_data}"
+
+
+@serve.deployment
+class Model:
+    def __call__(self, preprocessed_data: str) -> str:
+        # Simulate model inference (takes longer than preprocessing)
+        time.sleep(0.1)
+        return f"result_{preprocessed_data}"
+
+
+@serve.deployment
+class Driver:
+    def __init__(self, preprocessor, model):
+        self._preprocessor = preprocessor
+        self._model = model
+
+    async def __call__(self, input_data: str) -> str:
+        # Coordinate preprocessing and model inference
+        preprocessed = await self._preprocessor.remote(input_data)
+        result = await self._model.remote(preprocessed)
+        return result
+
+
+app = Driver.bind(Preprocessor.bind(), Model.bind())
+# __serve_example_end__

doc/source/serve/doc_code/application_level_autoscaling.yaml

@@ -0,0 +1,14 @@
+applications:
+  - name: MyApp
+    import_path: application_level_autoscaling:app
+    autoscaling_policy:
+      policy_function: autoscaling_policy:coordinated_scaling_policy
+    deployments:
+      - name: Preprocessor
+        autoscaling_config:
+          min_replicas: 1
+          max_replicas: 10
+      - name: Model
+        autoscaling_config:
+          min_replicas: 2
+          max_replicas: 20

doc/source/serve/doc_code/application_level_autoscaling_with_defaults.yaml

@@ -0,0 +1,20 @@
+applications:
+  - name: MyApp
+    import_path: application_level_autoscaling:app
+    autoscaling_policy:
+      policy_function: autoscaling_policy:coordinated_scaling_policy_with_defaults
+    deployments:
+      - name: Preprocessor
+        autoscaling_config:
+          min_replicas: 1
+          max_replicas: 10
+          target_ongoing_requests: 1
+          upscale_delay_s: 2
+          downscale_delay_s: 5
+      - name: Model
+        autoscaling_config:
+          min_replicas: 2
+          max_replicas: 20
+          target_ongoing_requests: 1
+          upscale_delay_s: 3
+          downscale_delay_s: 5

doc/source/serve/doc_code/async_inference_autoscaling.py

@@ -0,0 +1,40 @@
+# __basic_example_begin__
+from ray import serve
+from ray.serve.config import AutoscalingConfig, AutoscalingPolicy
+from ray.serve.schema import CeleryAdapterConfig, TaskProcessorConfig
+from ray.serve.task_consumer import task_consumer, task_handler
+
+processor_config = TaskProcessorConfig(
+    queue_name="my_queue",
+    adapter_config=CeleryAdapterConfig(
+        broker_url="redis://localhost:6379/0",
+        backend_url="redis://localhost:6379/1",
+    ),
+)
+
+
+@serve.deployment(
+    max_ongoing_requests=5,
+    autoscaling_config=AutoscalingConfig(
+        min_replicas=1,
+        max_replicas=10,
+        target_ongoing_requests=2,
+        policy=AutoscalingPolicy(
+            policy_function="ray.serve.async_inference_autoscaling_policy:AsyncInferenceAutoscalingPolicy",
+            policy_kwargs={
+                "broker_url": "redis://localhost:6379/0",
+                "queue_name": "my_queue",
+            },
+        ),
+    ),
+)
+@task_consumer(task_processor_config=processor_config)
+class MyConsumer:
+    @task_handler(name="process")
+    def process(self, data):
+        return f"processed: {data}"
+
+
+app = MyConsumer.bind()
+serve.run(app)
+# __basic_example_end__

doc/source/serve/doc_code/async_inference_autoscaling.yaml

@@ -0,0 +1,17 @@
+# __basic_example_begin__
+applications:
+  - name: default
+    import_path: my_module:app
+    deployments:
+      - name: MyConsumer
+        max_ongoing_requests: 5
+        autoscaling_config:
+          min_replicas: 1
+          max_replicas: 10
+          target_ongoing_requests: 2
+          policy:
+            policy_function: "ray.serve.async_inference_autoscaling_policy:AsyncInferenceAutoscalingPolicy"
+            policy_kwargs:
+              broker_url: "redis://localhost:6379/0"
+              queue_name: "my_queue"
+# __basic_example_end__

doc/source/serve/doc_code/asyncio_best_practices.py

@@ -0,0 +1,435 @@
+# flake8: noqa
+"""
+Code examples for the asyncio best practices guide.
+All examples are structured to be runnable and demonstrate key concepts.
+"""
+
+# __imports_begin__
+from ray import serve
+import asyncio
+# __imports_end__
+
+
+# __echo_async_begin__
+@serve.deployment
+class Echo:
+    async def __call__(self, request):
+        await asyncio.sleep(0.1)
+        return "ok"
+# __echo_async_end__
+
+
+# __blocking_echo_begin__
+@serve.deployment
+class BlockingEcho:
+    def __call__(self, request):
+        # Blocking.
+        import time
+        time.sleep(1)
+        return "ok"
+# __blocking_echo_end__
+
+
+# __fastapi_deployment_begin__
+from fastapi import FastAPI
+
+app = FastAPI()
+
+@serve.deployment
+@serve.ingress(app)
+class FastAPIDeployment:
+    @app.get("/sync")
+    def sync_endpoint(self):
+        # FastAPI runs this in a threadpool.
+        import time
+        time.sleep(1)
+        return "ok"
+
+    @app.get("/async")
+    async def async_endpoint(self):
+        # Runs directly on FastAPI's asyncio loop.
+        await asyncio.sleep(1)
+        return "ok"
+# __fastapi_deployment_end__
+
+
+# __blocking_http_begin__
+@serve.deployment
+class BlockingHTTP:
+    async def __call__(self, request):
+        # ❌ This blocks the event loop until the HTTP call finishes.
+        import requests
+        resp = requests.get("https://example.com/")
+        return resp.text
+# __blocking_http_end__
+
+
+# __async_http_begin__
+@serve.deployment
+class AsyncHTTP:
+    async def __call__(self, request):
+        import httpx
+
+        async with httpx.AsyncClient() as client:
+            resp = await client.get("https://example.com/")
+        return resp.text
+# __async_http_end__
+
+
+# __threaded_http_begin__
+@serve.deployment
+class ThreadedHTTP:
+    async def __call__(self, request):
+        import requests
+
+        def fetch():
+            return requests.get("https://example.com/").text
+
+        # ✅ Offload blocking I/O to a worker thread.
+        return await asyncio.to_thread(fetch)
+# __threaded_http_end__
+
+# __threadpool_override_begin__
+from concurrent.futures import ThreadPoolExecutor
+
+@serve.deployment
+class CustomThreadPool:
+    def __init__(self):
+        loop = asyncio.get_running_loop()
+        loop.set_default_executor(ThreadPoolExecutor(max_workers=16))
+
+    async def __call__(self, request):
+        return await asyncio.to_thread(lambda: "ok")
+# __threadpool_override_end__
+
+
+# __numpy_deployment_begin__
+@serve.deployment
+class NumpyDeployment:
+    def _heavy_numpy(self, array):
+        import numpy as np
+        # Many NumPy ops release the GIL while executing C/Fortran code.
+        return np.linalg.svd(array)[0]
+
+    async def __call__(self, request):
+        import numpy as np
+        # Create a sample array from request data
+        array = np.random.rand(100, 100)
+        # ✅ Multiple threads can run _heavy_numpy in parallel if
+        # the underlying implementation releases the GIL.
+        return await asyncio.to_thread(self._heavy_numpy, array)
+# __numpy_deployment_end__
+
+
+# __max_ongoing_requests_begin__
+@serve.deployment(max_ongoing_requests=32)
+class MyService:
+    async def __call__(self, request):
+        await asyncio.sleep(1)
+        return "ok"
+# __max_ongoing_requests_end__
+
+
+# __async_io_bound_begin__
+@serve.deployment(max_ongoing_requests=100)
+class AsyncIOBound:
+    async def __call__(self, request):
+        # Mostly waiting on an external system.
+        await asyncio.sleep(0.1)
+        return "ok"
+# __async_io_bound_end__
+
+
+# __blocking_cpu_begin__
+@serve.deployment(max_ongoing_requests=100)
+class BlockingCPU:
+    def __call__(self, request):
+        # ❌ Blocks the user event loop.
+        import time
+        time.sleep(1)
+        return "ok"
+# __blocking_cpu_end__
+
+
+# __cpu_with_threadpool_begin__
+@serve.deployment(max_ongoing_requests=100)
+class CPUWithThreadpool:
+    def __call__(self, request):
+        # With RAY_SERVE_RUN_SYNC_IN_THREADPOOL=1, each call runs in a thread.
+        import time
+        time.sleep(1)
+        return "ok"
+# __cpu_with_threadpool_end__
+
+
+# __batched_model_begin__
+
+@serve.deployment(max_ongoing_requests=64)
+class BatchedModel:
+    @serve.batch(max_batch_size=32)
+    async def __call__(self, requests):
+        # requests is a list of request objects.
+        inputs = [r for r in requests]
+        outputs = await self._run_model(inputs)
+        return outputs
+    
+    async def _run_model(self, inputs):
+        # Placeholder model function
+        return [f"result_{i}" for i in inputs]
+# __batched_model_end__
+
+
+# __batched_model_offload_begin__
+@serve.deployment(max_ongoing_requests=64)
+class BatchedModelOffload:
+    @serve.batch(max_batch_size=32)
+    async def __call__(self, requests):
+        # requests is a list of request objects.
+        inputs = [r for r in requests]
+        outputs = await self._run_model(inputs)
+        return outputs
+
+    async def _run_model(self, inputs):
+        def run_sync():
+            # Heavy CPU or GIL-releasing native code here.
+            # Placeholder model function
+            return [f"result_{i}" for i in inputs]
+
+        loop = asyncio.get_running_loop()
+        return await loop.run_in_executor(None, run_sync)
+# __batched_model_offload_end__
+
+
+# __blocking_stream_begin__
+@serve.deployment
+class BlockingStream:
+    def __call__(self, request):
+        # ❌ Blocks the event loop between yields.
+        import time
+        for i in range(10):
+            time.sleep(1)
+            yield f"{i}\n"
+# __blocking_stream_end__
+
+
+# __async_stream_begin__
+@serve.deployment
+class AsyncStream:
+    async def __call__(self, request):
+        # ✅ Yields items without blocking the loop.
+        async def generator():
+            for i in range(10):
+                await asyncio.sleep(1)
+                yield f"{i}\n"
+
+        return generator()
+# __async_stream_end__
+
+
+# __offload_io_begin__
+@serve.deployment
+class OffloadIO:
+    async def __call__(self, request):
+        import requests
+
+        def fetch():
+            return requests.get("https://example.com/").text
+
+        # Offload to a thread, free the event loop.
+        body = await asyncio.to_thread(fetch)
+        return body
+# __offload_io_end__
+
+
+# __offload_cpu_begin__
+@serve.deployment
+class OffloadCPU:
+    def _compute(self, x):
+        # CPU-intensive work.
+        total = 0
+        for i in range(10_000_000):
+            total += (i * x) % 7
+        return total
+
+    async def __call__(self, request):
+        x = 123
+        loop = asyncio.get_running_loop()
+        result = await loop.run_in_executor(None, self._compute, x)
+        return str(result)
+# __offload_cpu_end__
+
+
+# __ray_parallel_begin__
+import ray
+
+@ray.remote
+def heavy_task(x):
+    # Heavy compute runs in its own worker process.
+    return x * x
+
+
+@serve.deployment
+class RayParallel:
+    async def __call__(self, request):
+        values = [1, 2, 3, 4]
+        refs = [heavy_task.remote(v) for v in values]
+        results = await asyncio.gather(*[r for r in refs])
+        return {"results": results}
+# __ray_parallel_end__
+
+
+if __name__ == "__main__":
+    import ray
+    
+    # Initialize Ray if not already running
+    if not ray.is_initialized():
+        ray.init()
+    
+    print("Testing Echo deployment...")
+    # Test Echo
+    echo_handle = serve.run(Echo.bind())
+    result = echo_handle.remote(None).result()
+    print(f"Echo result: {result}")
+    assert result == "ok"
+    
+    print("\nTesting BlockingEcho deployment...")
+    # Test BlockingEcho
+    blocking_handle = serve.run(BlockingEcho.bind())
+    result = blocking_handle.remote(None).result()
+    print(f"BlockingEcho result: {result}")
+    assert result == "ok"
+    
+    print("\nTesting MyService deployment...")
+    # Test MyService
+    service_handle = serve.run(MyService.bind())
+    result = service_handle.remote(None).result()
+    print(f"MyService result: {result}")
+    assert result == "ok"
+    
+    print("\nTesting AsyncIOBound deployment...")
+    # Test AsyncIOBound
+    io_bound_handle = serve.run(AsyncIOBound.bind())
+    result = io_bound_handle.remote(None).result()
+    print(f"AsyncIOBound result: {result}")
+    assert result == "ok"
+    
+    print("\nTesting AsyncStream deployment...")
+    # Test AsyncStream (just create it, don't fully consume)
+    stream_handle = serve.run(AsyncStream.bind())
+    print("AsyncStream deployment created successfully")
+    
+    print("\nTesting OffloadCPU deployment...")
+    # Test OffloadCPU
+    cpu_handle = serve.run(OffloadCPU.bind())
+    result = cpu_handle.remote(None).result()
+    print(f"OffloadCPU result: {result}")
+    
+    print("\nTesting NumpyDeployment...")
+    # Test NumpyDeployment
+    numpy_handle = serve.run(NumpyDeployment.bind())
+    result = numpy_handle.remote(None).result()
+    print(f"NumpyDeployment result shape: {result.shape}")
+    assert result.shape == (100, 100)
+    
+    print("\nTesting BlockingCPU deployment...")
+    # Test BlockingCPU
+    blocking_cpu_handle = serve.run(BlockingCPU.bind())
+    result = blocking_cpu_handle.remote(None).result()
+    print(f"BlockingCPU result: {result}")
+    assert result == "ok"
+    
+    print("\nTesting CPUWithThreadpool deployment...")
+    # Test CPUWithThreadpool
+    cpu_threadpool_handle = serve.run(CPUWithThreadpool.bind())
+    result = cpu_threadpool_handle.remote(None).result()
+    print(f"CPUWithThreadpool result: {result}")
+    assert result == "ok"
+
+    print("\nTesting CustomThreadPool deployment...")
+    custom_threadpool_handle = serve.run(CustomThreadPool.bind())
+    result = custom_threadpool_handle.remote(None).result()
+    print(f"CustomThreadPool result: {result}")
+    assert result == "ok"
+
+    print("\nTesting BlockingStream deployment...")
+    # Test BlockingStream - just verify it can be created and called
+    blocking_stream_handle = serve.run(BlockingStream.bind())
+    # For generator responses, we need to handle them differently
+    # Just verify deployment works
+    print("BlockingStream deployment created successfully")
+
+    print("\nTesting RayParallel deployment...")
+    # Test RayParallel
+    ray_parallel_handle = serve.run(RayParallel.bind())
+    result = ray_parallel_handle.remote(None).result()
+    print(f"RayParallel result: {result}")
+    assert result == {"results": [1, 4, 9, 16]}
+
+    print("\nTesting BatchedModel deployment...")
+    # Test BatchedModel
+    batched_model_handle = serve.run(BatchedModel.bind())
+    result = batched_model_handle.remote(1).result()
+    print(f"BatchedModel result: {result}")
+    assert result == "result_1"
+
+    print("\nTesting BatchedModelOffload deployment...")
+    # Test BatchedModelOffload
+    batched_model_offload_handle = serve.run(BatchedModelOffload.bind())
+    result = batched_model_offload_handle.remote(1).result()
+    print(f"BatchedModelOffload result: {result}")
+    assert result == "result_1"
+    
+    # Test HTTP-related deployments with try-except
+    print("\n--- Testing HTTP-related deployments (may fail due to network) ---")
+    
+    print("\nTesting BlockingHTTP deployment...")
+    try:
+        blocking_http_handle = serve.run(BlockingHTTP.bind())
+        result = blocking_http_handle.remote(None).result()
+        print(f"BlockingHTTP result (first 50 chars): {result[:50]}...")
+        print("✅ BlockingHTTP test passed")
+    except Exception as e:
+        print(f"⚠️  BlockingHTTP test failed (expected): {type(e).__name__}: {e}")
+    
+    print("\nTesting AsyncHTTP deployment...")
+    try:
+        async_http_handle = serve.run(AsyncHTTP.bind())
+        result = async_http_handle.remote(None).result()
+        print(f"AsyncHTTP result (first 50 chars): {result[:50]}...")
+        print("✅ AsyncHTTP test passed")
+    except Exception as e:
+        print(f"⚠️  AsyncHTTP test failed (expected): {type(e).__name__}: {e}")
+    
+    print("\nTesting ThreadedHTTP deployment...")
+    try:
+        threaded_http_handle = serve.run(ThreadedHTTP.bind())
+        result = threaded_http_handle.remote(None).result()
+        print(f"ThreadedHTTP result (first 50 chars): {result[:50]}...")
+        print("✅ ThreadedHTTP test passed")
+    except Exception as e:
+        print(f"⚠️  ThreadedHTTP test failed (expected): {type(e).__name__}: {e}")
+
+    print("\nTesting OffloadIO deployment...")
+    try:
+        offload_io_handle = serve.run(OffloadIO.bind())
+        result = offload_io_handle.remote(None).result()
+        print(f"OffloadIO result (first 50 chars): {result[:50]}...")
+        print("✅ OffloadIO test passed")
+    except Exception as e:
+        print(f"⚠️  OffloadIO test failed (expected): {type(e).__name__}: {e}")
+    
+    print("\nTesting FastAPIDeployment...")
+    fastapi_handle = serve.run(FastAPIDeployment.bind())
+    # Give it a moment to start
+    import time
+    import requests
+    time.sleep(2)
+    # Test the sync endpoint
+    response = requests.get("http://127.0.0.1:8000/sync", timeout=5)
+    print(f"FastAPIDeployment /sync result: {response.json()}")
+    # Test the async endpoint
+    response = requests.get("http://127.0.0.1:8000/async", timeout=5)
+    print(f"FastAPIDeployment /async result: {response.json()}")
+    print("✅ FastAPIDeployment test passed")
+
+    print("\n✅ All core tests passed!")

doc/source/serve/doc_code/autoscale_model_comp_example.py

@@ -0,0 +1,48 @@
+# __serve_example_begin__
+import time
+
+from ray import serve
+from ray.serve.handle import DeploymentHandle
+
+
+@serve.deployment
+class LightLoad:
+    async def __call__(self) -> str:
+        start = time.time()
+        while time.time() - start < 0.1:
+            pass
+
+        return "light"
+
+
+@serve.deployment
+class HeavyLoad:
+    async def __call__(self) -> str:
+        start = time.time()
+        while time.time() - start < 0.2:
+            pass
+
+        return "heavy"
+
+
+@serve.deployment
+class Driver:
+    def __init__(self, a_handle, b_handle):
+        self.a_handle: DeploymentHandle = a_handle
+        self.b_handle: DeploymentHandle = b_handle
+
+    async def __call__(self) -> str:
+        a_future = self.a_handle.remote()
+        b_future = self.b_handle.remote()
+
+        return (await a_future), (await b_future)
+
+
+app = Driver.bind(HeavyLoad.bind(), LightLoad.bind())
+# __serve_example_end__
+
+import requests  # noqa
+
+serve.run(app)
+resp = requests.post("http://localhost:8000")
+assert resp.json() == ["heavy", "light"]

doc/source/serve/doc_code/autoscaling_policy.py

@@ -0,0 +1,178 @@
+# __begin_scheduled_batch_processing_policy__
+from datetime import datetime
+from typing import Any, Dict
+from ray.serve.config import AutoscalingContext
+
+
+def scheduled_batch_processing_policy(
+    ctx: AutoscalingContext,
+) -> tuple[int, Dict[str, Any]]:
+    current_time = datetime.now()
+    current_hour = current_time.hour
+    # Scale up during business hours (9 AM - 5 PM)
+    if 9 <= current_hour < 17:
+        return 2, {"reason": "Business hours"}
+    # Scale up for evening batch processing (6 PM - 8 PM)
+    elif 18 <= current_hour < 20:
+        return 4, {"reason": "Evening batch processing"}
+    # Minimal scaling during off-peak hours
+    else:
+        return 1, {"reason": "Off-peak hours"}
+
+
+# __end_scheduled_batch_processing_policy__
+
+
+# __begin_custom_metrics_autoscaling_policy__
+from typing import Any, Dict
+from ray.serve.config import AutoscalingContext
+
+
+def custom_metrics_autoscaling_policy(
+    ctx: AutoscalingContext,
+) -> tuple[int, Dict[str, Any]]:
+    cpu_usage_metric = ctx.aggregated_metrics.get("cpu_usage", {})
+    memory_usage_metric = ctx.aggregated_metrics.get("memory_usage", {})
+    max_cpu_usage = list(cpu_usage_metric.values())[-1] if cpu_usage_metric else 0
+    max_memory_usage = (
+        list(memory_usage_metric.values())[-1] if memory_usage_metric else 0
+    )
+
+    if max_cpu_usage > 80 or max_memory_usage > 85:
+        return min(ctx.capacity_adjusted_max_replicas, ctx.current_num_replicas + 1), {}
+    elif max_cpu_usage < 30 and max_memory_usage < 40:
+        return max(ctx.capacity_adjusted_min_replicas, ctx.current_num_replicas - 1), {}
+    else:
+        return ctx.current_num_replicas, {}
+
+
+# __end_custom_metrics_autoscaling_policy__
+
+
+# __begin_application_level_autoscaling_policy__
+from typing import Dict, Tuple
+from ray.serve.config import AutoscalingContext
+
+from ray.serve._private.common import DeploymentID
+from ray.serve.config import AutoscalingContext
+
+
+def coordinated_scaling_policy(
+    contexts: Dict[DeploymentID, AutoscalingContext]
+) -> Tuple[Dict[DeploymentID, int], Dict]:
+    """Scale deployments based on coordinated load balancing."""
+    decisions = {}
+
+    # Example: Scale a preprocessing deployment
+    preprocessing_id = [d for d in contexts if d.name == "Preprocessor"][0]
+    preprocessing_ctx = contexts[preprocessing_id]
+
+    # Scale based on queue depth
+    preprocessing_replicas = max(
+        preprocessing_ctx.capacity_adjusted_min_replicas,
+        min(
+            preprocessing_ctx.capacity_adjusted_max_replicas,
+            preprocessing_ctx.total_num_requests // 10,
+        ),
+    )
+    decisions[preprocessing_id] = preprocessing_replicas
+
+    # Example: Scale a model deployment proportionally
+    model_id = [d for d in contexts if d.name == "Model"][0]
+    model_ctx = contexts[model_id]
+
+    # Scale model to handle preprocessing output
+    # Assuming model takes 2x longer than preprocessing
+    model_replicas = max(
+        model_ctx.capacity_adjusted_min_replicas,
+        min(model_ctx.capacity_adjusted_max_replicas, preprocessing_replicas * 2),
+    )
+    decisions[model_id] = model_replicas
+
+    return decisions, {}
+
+# __end_application_level_autoscaling_policy__
+
+# __begin_stateful_application_level_policy__
+from typing import Dict, Tuple, Any
+from ray.serve.config import AutoscalingContext
+from ray.serve._private.common import DeploymentID
+
+def stateful_application_level_policy(
+    contexts: Dict[DeploymentID, AutoscalingContext]
+) -> Tuple[Dict[DeploymentID, int], Dict[DeploymentID, Dict[str, Any]]]:
+    """Example policy demonstrating per-deployment state persistence."""
+    decisions = {}
+    policy_state = {}
+
+    for deployment_id, ctx in contexts.items():
+        # Read previous state for this deployment (persisted from last iteration)
+        prev_state = ctx.policy_state or {}
+        scale_count = prev_state.get("scale_count", 0)
+        last_replicas = prev_state.get("last_replicas", ctx.current_num_replicas)
+
+        # Simple scaling logic: scale based on queue depth
+        desired_replicas = max(
+            ctx.capacity_adjusted_min_replicas,
+            min(
+                ctx.capacity_adjusted_max_replicas,
+                ctx.total_num_requests // 10,
+            ),
+        )
+        decisions[deployment_id] = desired_replicas
+
+        # Store per-deployment state that persists across iterations
+        policy_state[deployment_id] = {
+            "scale_count": scale_count + 1,
+            "last_replicas": desired_replicas,
+        }
+
+    return decisions, policy_state
+
+
+# __end_stateful_application_level_policy__
+# __begin_apply_autoscaling_config_example__
+from typing import Any, Dict
+from ray.serve.config import AutoscalingContext
+
+
+def queue_length_based_autoscaling_policy(
+    ctx: AutoscalingContext,
+) -> tuple[int, Dict[str, Any]]:
+    # This policy calculates the "raw" desired replicas based on queue length.
+    # Ray Serve automatically applies scaling factors, delays, and bounds from
+    # the deployment's autoscaling_config on top of this decision.
+
+    queue_length = ctx.total_num_requests
+
+    if queue_length > 50:
+        return 10, {}
+    elif queue_length > 10:
+        return 5, {}
+    else:
+        return 0, {}
+# __end_apply_autoscaling_config_example__
+
+# __begin_apply_autoscaling_config_usage__
+from ray import serve
+from ray.serve.config import AutoscalingConfig, AutoscalingPolicy
+
+@serve.deployment(
+    autoscaling_config=AutoscalingConfig(
+        min_replicas=1,
+        max_replicas=10,
+        metrics_interval_s=0.1,
+        upscale_delay_s=1.0,
+        downscale_delay_s=1.0,
+        policy=AutoscalingPolicy(
+            policy_function=queue_length_based_autoscaling_policy
+        )
+    ),
+    max_ongoing_requests=5,
+)
+class MyDeployment:
+    def __call__(self) -> str:
+        return "Hello, world!"
+
+app = MyDeployment.bind()
+# __end_apply_autoscaling_config_usage__

doc/source/serve/doc_code/aws_neuron_core_inference_serve.py

@@ -0,0 +1,102 @@
+# flake8: noqa
+# __compile_neuron_code_start__
+from transformers import AutoModelForSequenceClassification, AutoTokenizer
+import torch, torch_neuronx
+
+hf_model = "j-hartmann/emotion-english-distilroberta-base"
+neuron_model = "./sentiment_neuron.pt"
+
+model = AutoModelForSequenceClassification.from_pretrained(hf_model)
+tokenizer = AutoTokenizer.from_pretrained(hf_model)
+sequence_0 = "The company HuggingFace is based in New York City"
+sequence_1 = "HuggingFace's headquarters are situated in Manhattan"
+example_inputs = tokenizer.encode_plus(
+    sequence_0,
+    sequence_1,
+    return_tensors="pt",
+    padding="max_length",
+    truncation=True,
+    max_length=128,
+)
+neuron_inputs = example_inputs["input_ids"], example_inputs["attention_mask"]
+n_model = torch_neuronx.trace(model, neuron_inputs)
+n_model.save(neuron_model)
+print(f"Saved Neuron-compiled model {neuron_model}")
+# __compile_neuron_code_end__
+
+
+# __neuron_serve_code_start__
+from fastapi import FastAPI
+import torch
+
+from ray import serve
+from ray.serve.handle import DeploymentHandle
+
+app = FastAPI()
+
+hf_model = "j-hartmann/emotion-english-distilroberta-base"
+neuron_model = "./sentiment_neuron.pt"
+
+
+@serve.deployment(num_replicas=1)
+@serve.ingress(app)
+class APIIngress:
+    def __init__(self, bert_base_model_handle: DeploymentHandle) -> None:
+        self.handle = bert_base_model_handle
+
+    @app.get("/infer")
+    async def infer(self, sentence: str):
+        return await self.handle.infer.remote(sentence)
+
+
+@serve.deployment(
+    ray_actor_options={"resources": {"neuron_cores": 1}},
+    autoscaling_config={"min_replicas": 1, "max_replicas": 2},
+)
+class BertBaseModel:
+    def __init__(self):
+        import torch, torch_neuronx  # noqa
+        from transformers import AutoTokenizer
+
+        self.model = torch.jit.load(neuron_model)
+        self.tokenizer = AutoTokenizer.from_pretrained(hf_model)
+        self.classmap = {
+            0: "anger",
+            1: "disgust",
+            2: "fear",
+            3: "joy",
+            4: "neutral",
+            5: "sadness",
+            6: "surprise",
+        }
+
+    def infer(self, sentence: str):
+        inputs = self.tokenizer.encode_plus(
+            sentence,
+            return_tensors="pt",
+            padding="max_length",
+            truncation=True,
+            max_length=128,
+        )
+        output = self.model(*(inputs["input_ids"], inputs["attention_mask"]))
+        class_id = torch.argmax(output["logits"], dim=1).item()
+        return self.classmap[class_id]
+
+
+entrypoint = APIIngress.bind(BertBaseModel.bind())
+
+
+# __neuron_serve_code_end__
+if __name__ == "__main__":
+    import requests
+    import ray
+
+    # On inf2.8xlarge instance, there will be 2 neuron cores.
+    ray.init(resources={"neuron_cores": 2})
+
+    serve.run(entrypoint)
+    prompt = "Ray is super cool."
+    resp = requests.get(f"http://127.0.0.1:8000/infer?sentence={prompt}")
+    print(resp.status_code, resp.json())
+
+    assert resp.status_code == 200

doc/source/serve/doc_code/aws_neuron_core_inference_serve_stable_diffusion.py

@@ -0,0 +1,72 @@
+# __neuron_serve_code_start__
+from io import BytesIO
+from fastapi import FastAPI
+from fastapi.responses import Response
+from ray import serve
+
+
+app = FastAPI()
+
+neuron_cores = 2
+
+
+@serve.deployment(num_replicas=1)
+@serve.ingress(app)
+class APIIngress:
+    def __init__(self, diffusion_model_handle) -> None:
+        self.handle = diffusion_model_handle
+
+    @app.get(
+        "/imagine",
+        responses={200: {"content": {"image/png": {}}}},
+        response_class=Response,
+    )
+    async def generate(self, prompt: str):
+
+        image_ref = await self.handle.generate.remote(prompt)
+        image = image_ref
+        file_stream = BytesIO()
+        image.save(file_stream, "PNG")
+        return Response(content=file_stream.getvalue(), media_type="image/png")
+
+
+@serve.deployment(
+    ray_actor_options={"resources": {"neuron_cores": neuron_cores}},
+    autoscaling_config={"min_replicas": 1, "max_replicas": 2},
+)
+class StableDiffusionV2:
+    def __init__(self):
+        from optimum.neuron import NeuronStableDiffusionXLPipeline
+
+        compiled_model_id = "aws-neuron/stable-diffusion-xl-base-1-0-1024x1024"
+        self.pipe = NeuronStableDiffusionXLPipeline.from_pretrained(
+            compiled_model_id, device_ids=[0, 1]
+        )
+
+    async def generate(self, prompt: str):
+
+        assert len(prompt), "prompt parameter cannot be empty"
+        image = self.pipe(prompt).images[0]
+        return image
+
+
+entrypoint = APIIngress.bind(StableDiffusionV2.bind())
+
+# __neuron_serve_code_end__
+if __name__ == "__main__":
+    import requests
+    import ray
+
+    # On inf2.8xlarge instance, there are 2 Neuron cores.
+    ray.init(resources={"neuron_cores": 2})
+
+    serve.run(entrypoint)
+    prompt = "a zebra is dancing in the grass, river, sunlit"
+    input = "%20".join(prompt.split(" "))
+    resp = requests.get(f"http://127.0.0.1:8000/imagine?prompt={input}")
+
+    print("Write the response to `output.png`.")
+    with open("output.png", "wb") as f:
+        f.write(resp.content)
+
+    assert resp.status_code == 200

doc/source/serve/doc_code/batching_guide.py

@@ -0,0 +1,245 @@
+# flake8: noqa
+# __single_sample_begin__
+from ray import serve
+from ray.serve.handle import DeploymentHandle
+
+
+@serve.deployment
+class Model:
+    def __call__(self, single_sample: int) -> int:
+        return single_sample * 2
+
+
+handle: DeploymentHandle = serve.run(Model.bind())
+assert handle.remote(1).result() == 2
+# __single_sample_end__
+
+
+# __batch_begin__
+from typing import List
+
+import numpy as np
+
+from ray import serve
+from ray.serve.handle import DeploymentHandle
+
+
+@serve.deployment
+class Model:
+    @serve.batch(max_batch_size=8, batch_wait_timeout_s=0.1)
+    async def __call__(self, multiple_samples: List[int]) -> List[int]:
+        # Use numpy's vectorized computation to efficiently process a batch.
+        return np.array(multiple_samples) * 2
+
+
+handle: DeploymentHandle = serve.run(Model.bind())
+responses = [handle.remote(i) for i in range(8)]
+assert list(r.result() for r in responses) == [i * 2 for i in range(8)]
+# __batch_end__
+
+
+# __batch_params_update_begin__
+from typing import Dict
+
+
+@serve.deployment(
+    # These values can be overridden in the Serve config.
+    user_config={
+        "max_batch_size": 10,
+        "batch_wait_timeout_s": 0.5,
+    }
+)
+class Model:
+    @serve.batch(max_batch_size=8, batch_wait_timeout_s=0.1)
+    async def __call__(self, multiple_samples: List[int]) -> List[int]:
+        # Use numpy's vectorized computation to efficiently process a batch.
+        return np.array(multiple_samples) * 2
+
+    def reconfigure(self, user_config: Dict):
+        self.__call__.set_max_batch_size(user_config["max_batch_size"])
+        self.__call__.set_batch_wait_timeout_s(user_config["batch_wait_timeout_s"])
+
+
+# __batch_params_update_end__
+
+
+# __single_stream_begin__
+import asyncio
+from typing import AsyncGenerator
+from starlette.requests import Request
+from starlette.responses import StreamingResponse
+
+from ray import serve
+
+
+@serve.deployment
+class StreamingResponder:
+    async def generate_numbers(self, max: str) -> AsyncGenerator[str, None]:
+        for i in range(max):
+            yield str(i)
+            await asyncio.sleep(0.1)
+
+    def __call__(self, request: Request) -> StreamingResponse:
+        max = int(request.query_params.get("max", "25"))
+        gen = self.generate_numbers(max)
+        return StreamingResponse(gen, status_code=200, media_type="text/plain")
+
+
+# __single_stream_end__
+
+import requests
+
+serve.run(StreamingResponder.bind())
+r = requests.get("http://localhost:8000/", stream=True)
+chunks = []
+for chunk in r.iter_content(chunk_size=None, decode_unicode=True):
+    chunks.append(chunk)
+
+assert ",".join(list(map(str, range(25)))) == ",".join(chunks)
+
+# __batch_stream_begin__
+import asyncio
+from typing import List, AsyncGenerator, Union
+from starlette.requests import Request
+from starlette.responses import StreamingResponse
+
+from ray import serve
+
+
+@serve.deployment
+class StreamingResponder:
+    @serve.batch(max_batch_size=5, batch_wait_timeout_s=0.1)
+    async def generate_numbers(
+        self, max_list: List[str]
+    ) -> AsyncGenerator[List[Union[int, StopIteration]], None]:
+        for i in range(max(max_list)):
+            next_numbers = []
+            for requested_max in max_list:
+                if requested_max > i:
+                    next_numbers.append(str(i))
+                else:
+                    next_numbers.append(StopIteration)
+            yield next_numbers
+            await asyncio.sleep(0.1)
+
+    async def __call__(self, request: Request) -> StreamingResponse:
+        max = int(request.query_params.get("max", "25"))
+        gen = self.generate_numbers(max)
+        return StreamingResponse(gen, status_code=200, media_type="text/plain")
+
+
+# __batch_stream_end__
+
+import requests
+from functools import partial
+from concurrent.futures.thread import ThreadPoolExecutor
+
+serve.run(StreamingResponder.bind())
+
+
+def issue_request(max) -> List[str]:
+    url = "http://localhost:8000/?max="
+    response = requests.get(url + str(max), stream=True)
+    chunks = []
+    for chunk in response.iter_content(chunk_size=None, decode_unicode=True):
+        chunks.append(chunk)
+    return chunks
+
+
+requested_maxes = [1, 2, 5, 6, 9]
+with ThreadPoolExecutor(max_workers=5) as pool:
+    futs = [pool.submit(partial(issue_request, max)) for max in requested_maxes]
+    chunks_list = [fut.result() for fut in futs]
+    for max, chunks in zip(requested_maxes, chunks_list):
+        assert chunks == [str(i) for i in range(max)]
+
+# __batch_size_fn_begin__
+from typing import List
+
+from ray import serve
+from ray.serve.handle import DeploymentHandle
+
+
+class Graph:
+    """Simple graph data structure for GNN workloads."""
+
+    def __init__(self, num_nodes: int, node_features: list):
+        self.num_nodes = num_nodes
+        self.node_features = node_features
+
+
+@serve.deployment
+class GraphNeuralNetwork:
+    @serve.batch(
+        max_batch_size=10000,  # Maximum total nodes per batch
+        batch_wait_timeout_s=0.1,
+        batch_size_fn=lambda graphs: sum(g.num_nodes for g in graphs),
+    )
+    async def predict(self, graphs: List[Graph]) -> List[float]:
+        """Process a batch of graphs, batching by total node count."""
+        # The batch_size_fn ensures that the total number of nodes
+        # across all graphs in the batch doesn't exceed max_batch_size.
+        # This prevents GPU memory overflow.
+        results = []
+        for graph in graphs:
+            # Your GNN model inference logic here
+            # For this example, just return a simple score
+            score = float(graph.num_nodes * 0.1)
+            results.append(score)
+        return results
+
+    async def __call__(self, graph: Graph) -> float:
+        return await self.predict(graph)
+
+
+handle: DeploymentHandle = serve.run(GraphNeuralNetwork.bind())
+
+# Create test graphs with varying node counts
+graphs = [
+    Graph(num_nodes=100, node_features=[1.0] * 100),
+    Graph(num_nodes=5000, node_features=[2.0] * 5000),
+    Graph(num_nodes=3000, node_features=[3.0] * 3000),
+]
+
+# Send requests - they'll be batched by total node count
+results = [handle.remote(g).result() for g in graphs]
+print(f"Results: {results}")
+# __batch_size_fn_end__
+
+# __batch_size_fn_nlp_begin__
+from typing import List
+
+from ray import serve
+from ray.serve.handle import DeploymentHandle
+
+
+@serve.deployment
+class TokenBatcher:
+    @serve.batch(
+        max_batch_size=512,  # Maximum total tokens per batch
+        batch_wait_timeout_s=0.1,
+        batch_size_fn=lambda sequences: sum(len(s.split()) for s in sequences),
+    )
+    async def process(self, sequences: List[str]) -> List[int]:
+        """Process text sequences, batching by total token count."""
+        # The batch_size_fn ensures total tokens don't exceed max_batch_size.
+        # This is useful for transformer models with fixed context windows.
+        return [len(seq.split()) for seq in sequences]
+
+    async def __call__(self, sequence: str) -> int:
+        return await self.process(sequence)
+
+
+handle: DeploymentHandle = serve.run(TokenBatcher.bind())
+
+# Create sequences with different lengths
+sequences = [
+    "This is a short sentence",
+    "This is a much longer sentence with many more words to process",
+    "Short",
+]
+
+# Send requests - they'll be batched by total token count
+results = [handle.remote(seq).result() for seq in sequences]
+print(f"Token counts: {results}")
+# __batch_size_fn_nlp_end__

doc/source/serve/doc_code/capacity_queue_request_router_app.py

@@ -0,0 +1,56 @@
+# flake8: noqa
+
+# __begin_deploy_app_with_capacity_queue_router__
+import ray
+from ray import serve
+from ray.serve.config import DeploymentActorConfig, RequestRouterConfig
+from ray.serve.context import _get_internal_replica_context
+
+from ray.serve.experimental.capacity_queue import (
+    CapacityQueue,
+)
+
+
+@serve.deployment(
+    deployment_actors=[
+        DeploymentActorConfig(
+            name="capacity_queue",
+            actor_class=CapacityQueue,
+            init_kwargs={
+                "acquire_timeout_s": 0.5,
+                "token_ttl_s": 5,
+            },
+            actor_options={"num_cpus": 0},
+        ),
+    ],
+    request_router_config=RequestRouterConfig(
+        request_router_class=(
+            "ray.serve.experimental.capacity_queue_router:CapacityQueueRouter"
+        ),
+        request_router_kwargs={
+            "capacity_queue_actor_name": "capacity_queue",
+            # Fall back to Pow2 after this many consecutive CapacityQueue faults.
+            "max_fault_retries": 3,
+        },
+        # Backoff between retries when the CapacityQueue is unavailable or capacity is exhausted.
+        initial_backoff_s=0.05,
+        backoff_multiplier=2.0,
+        max_backoff_s=1.0,
+    ),
+    num_replicas=3,
+    max_ongoing_requests=5,
+    ray_actor_options={"num_cpus": 0},
+)
+class CapacityQueueApp:
+    def __init__(self):
+        context = _get_internal_replica_context()
+        self.replica_id = context.replica_id
+
+    async def __call__(self):
+        return self.replica_id
+
+
+handle = serve.run(CapacityQueueApp.bind())
+response = handle.remote().result()
+print(f"Response from CapacityQueueApp: {response}")
+# __end_deploy_app_with_capacity_queue_router__

doc/source/serve/doc_code/class_based_autoscaling.py

@@ -0,0 +1,46 @@
+# __serve_example_begin__
+import json
+import tempfile
+
+from ray import serve
+from ray.serve.config import AutoscalingConfig, AutoscalingPolicy
+
+# Create a JSON file with the initial target replica count.
+# In production this file would be written by an external system.
+scaling_file = tempfile.NamedTemporaryFile(
+    mode="w", suffix=".json", delete=False
+)
+json.dump({"replicas": 2}, scaling_file)
+scaling_file.close()
+
+
+@serve.deployment(
+    autoscaling_config=AutoscalingConfig(
+        min_replicas=1,
+        max_replicas=10,
+        upscale_delay_s=3,
+        downscale_delay_s=10,
+        policy=AutoscalingPolicy(
+            policy_function="class_based_autoscaling_policy:FileBasedAutoscalingPolicy",
+            policy_kwargs={
+                "file_path": scaling_file.name,
+                "poll_interval_s": 2.0,
+            },
+        ),
+    ),
+    max_ongoing_requests=100,
+)
+class MyDeployment:
+    async def __call__(self) -> str:
+        return "Hello, world!"
+
+
+app = MyDeployment.bind()
+# __serve_example_end__
+
+if __name__ == "__main__":
+    import requests  # noqa
+
+    serve.run(app)
+    resp = requests.get("http://localhost:8000/")
+    assert resp.text == "Hello, world!"

doc/source/serve/doc_code/class_based_autoscaling_policy.py

@@ -0,0 +1,57 @@
+# __begin_class_based_autoscaling_policy__
+import asyncio
+import json
+import logging
+from pathlib import Path
+from typing import Any, Dict, Tuple
+
+from ray.serve.config import AutoscalingContext
+
+logger = logging.getLogger("ray.serve")
+
+
+class FileBasedAutoscalingPolicy:
+    """Scale replicas based on a target written to a JSON file.
+
+    A background asyncio task re-reads the file every ``poll_interval_s``
+    seconds.  ``__call__`` returns the latest value on every autoscaling
+    tick.  In production you could replace the file read with an HTTP
+    call, a message-queue consumer, or any other async IO operation.
+    """
+
+    def __init__(self, file_path: str, poll_interval_s: float = 5.0):
+        self._file_path = Path(file_path)
+        self._poll_interval_s = poll_interval_s
+        self._desired_replicas: int = 1
+        self._task: asyncio.Task = None
+        self._started: bool = False
+
+    def _ensure_started(self) -> None:
+        """Lazily start the background poll on the controller event loop."""
+        if self._started:
+            return
+        self._started = True
+        loop = asyncio.get_running_loop()
+        self._task = loop.create_task(self._poll_file())
+
+    async def _poll_file(self) -> None:
+        """Read the target replica count from the JSON file in a loop."""
+        while True:
+            try:
+                text = self._file_path.read_text()
+                data = json.loads(text)
+                self._desired_replicas = int(data["replicas"])
+            except Exception:
+                pass  # Keep the last known value on failure.
+            await asyncio.sleep(self._poll_interval_s)
+
+    def __call__(
+        self, ctx: AutoscalingContext
+    ) -> Tuple[int, Dict[str, Any]]:
+        self._ensure_started()
+
+        desired = self._desired_replicas
+        return desired, {"last_polled_value": self._desired_replicas}
+
+
+# __end_class_based_autoscaling_policy__

doc/source/serve/doc_code/configure_serve_deployment/model_deployment.py

@@ -0,0 +1,36 @@
+# flake8: noqa
+
+import ray
+
+# __deployment_start__
+# File name: configure_serve.py
+
+from ray import serve
+
+
+@serve.deployment(
+    name="Translator",
+    num_replicas=2,
+    ray_actor_options={"num_cpus": 0.2, "num_gpus": 0},
+    max_ongoing_requests=100,
+    health_check_period_s=10,
+    health_check_timeout_s=30,
+    graceful_shutdown_timeout_s=20,
+    graceful_shutdown_wait_loop_s=2,
+)
+class Example:
+    ...
+
+
+example_app = Example.bind()
+# __deployment_end__
+
+example_app = Example.options(
+    ray_actor_options={"num_cpus": 0.2, "num_gpus": 0.0}
+).bind()
+
+# __options_end__
+serve.run(example_app)
+
+serve.shutdown()
+ray.shutdown()

doc/source/serve/doc_code/cross_node_parallelism_example.py

@@ -0,0 +1,197 @@
+# flake8: noqa
+"""
+Cross-node parallelism examples for Ray Serve LLM.
+
+TP / PP / custom placement group strategies
+for multi-node LLM deployments.
+"""
+
+# __cross_node_tp_example_start__
+import vllm
+from ray import serve
+from ray.serve.llm import LLMConfig, build_openai_app
+
+# Configure a model with tensor parallelism across 2 GPUs
+# Tensor parallelism splits model weights across GPUs
+llm_config = LLMConfig(
+    model_loading_config=dict(
+        model_id="llama-3.1-8b",
+        model_source="meta-llama/Llama-3.1-8B-Instruct",
+    ),
+    deployment_config=dict(
+        autoscaling_config=dict(
+            min_replicas=1,
+            max_replicas=2,
+        )
+    ),
+    accelerator_type="L4",
+    engine_kwargs=dict(
+        tensor_parallel_size=2,
+        max_model_len=8192,
+    ),
+)
+
+# Deploy the application
+app = build_openai_app({"llm_configs": [llm_config]})
+serve.run(app, blocking=True)
+# __cross_node_tp_example_end__
+
+# __cross_node_pp_example_start__
+from ray import serve
+from ray.serve.llm import LLMConfig, build_openai_app
+
+# Configure a model with pipeline parallelism across 2 GPUs
+# Pipeline parallelism splits model layers across GPUs
+llm_config = LLMConfig(
+    model_loading_config=dict(
+        model_id="llama-3.1-8b",
+        model_source="meta-llama/Llama-3.1-8B-Instruct",
+    ),
+    deployment_config=dict(
+        autoscaling_config=dict(
+            min_replicas=1,
+            max_replicas=1,
+        )
+    ),
+    accelerator_type="L4",
+    engine_kwargs=dict(
+        pipeline_parallel_size=2,
+        max_model_len=8192,
+    ),
+)
+
+# Deploy the application
+app = build_openai_app({"llm_configs": [llm_config]})
+serve.run(app, blocking=True)
+# __cross_node_pp_example_end__
+
+# __cross_node_tp_pp_example_start__
+from ray import serve
+from ray.serve.llm import LLMConfig, build_openai_app
+
+# Configure a model with both tensor and pipeline parallelism
+# This example uses 4 GPUs total (2 TP * 2 PP)
+llm_config = LLMConfig(
+    model_loading_config=dict(
+        model_id="llama-3.1-8b",
+        model_source="meta-llama/Llama-3.1-8B-Instruct",
+    ),
+    deployment_config=dict(
+        autoscaling_config=dict(
+            min_replicas=1,
+            max_replicas=1,
+        )
+    ),
+    accelerator_type="L4",
+    engine_kwargs=dict(
+        tensor_parallel_size=2,
+        pipeline_parallel_size=2,
+        max_model_len=8192,
+        enable_chunked_prefill=True,
+        max_num_batched_tokens=4096,
+    ),
+)
+
+# Deploy the application
+app = build_openai_app({"llm_configs": [llm_config]})
+serve.run(app, blocking=True)
+# __cross_node_tp_pp_example_end__
+
+# __custom_placement_group_pack_example_start__
+from ray import serve
+from ray.serve.llm import LLMConfig, build_openai_app
+
+# Configure a model with custom placement group using PACK strategy
+# PACK tries to place workers on as few nodes as possible for locality
+llm_config = LLMConfig(
+    model_loading_config=dict(
+        model_id="llama-3.1-8b",
+        model_source="meta-llama/Llama-3.1-8B-Instruct",
+    ),
+    deployment_config=dict(
+        autoscaling_config=dict(
+            min_replicas=1,
+            max_replicas=1,
+        )
+    ),
+    accelerator_type="L4",
+    engine_kwargs=dict(
+        tensor_parallel_size=2,
+        max_model_len=8192,
+    ),
+    placement_group_config=dict(
+        bundles=[{"GPU": 1}] * 2,
+        strategy="PACK",
+    ),
+)
+
+# Deploy the application
+app = build_openai_app({"llm_configs": [llm_config]})
+serve.run(app, blocking=True)
+# __custom_placement_group_pack_example_end__
+
+# __custom_placement_group_spread_example_start__
+from ray import serve
+from ray.serve.llm import LLMConfig, build_openai_app
+
+# Configure a model with custom placement group using SPREAD strategy
+# SPREAD distributes workers across nodes for fault tolerance
+llm_config = LLMConfig(
+    model_loading_config=dict(
+        model_id="llama-3.1-8b",
+        model_source="meta-llama/Llama-3.1-8B-Instruct",
+    ),
+    deployment_config=dict(
+        autoscaling_config=dict(
+            min_replicas=1,
+            max_replicas=1,
+        )
+    ),
+    accelerator_type="L4",
+    engine_kwargs=dict(
+        tensor_parallel_size=4,
+        max_model_len=8192,
+    ),
+    placement_group_config=dict(
+        bundles=[{"GPU": 1}] * 4,
+        strategy="SPREAD",
+    ),
+)
+
+# Deploy the application
+app = build_openai_app({"llm_configs": [llm_config]})
+serve.run(app, blocking=True)
+# __custom_placement_group_spread_example_end__
+
+# __custom_placement_group_strict_pack_example_start__
+from ray import serve
+from ray.serve.llm import LLMConfig, build_openai_app
+
+# Configure a model with custom placement group using STRICT_PACK strategy
+# STRICT_PACK ensures all workers are placed on the same node
+llm_config = LLMConfig(
+    model_loading_config=dict(
+        model_id="llama-3.1-8b",
+        model_source="meta-llama/Llama-3.1-8B-Instruct",
+    ),
+    deployment_config=dict(
+        autoscaling_config=dict(
+            min_replicas=1,
+            max_replicas=2,
+        )
+    ),
+    accelerator_type="A100",
+    engine_kwargs=dict(
+        tensor_parallel_size=2,
+        max_model_len=8192,
+    ),
+    placement_group_config=dict(
+        bundles=[{"GPU": 1}] * 2,
+        strategy="STRICT_PACK",
+    ),
+)
+
+# Deploy the application
+app = build_openai_app({"llm_configs": [llm_config]})
+serve.run(app, blocking=True)
+# __custom_placement_group_strict_pack_example_end__

doc/source/serve/doc_code/custom_metrics_autoscaling.py

@@ -0,0 +1,54 @@
+# __serve_example_begin__
+import time
+from typing import Dict
+
+import psutil
+from ray import serve
+
+
+@serve.deployment(
+    autoscaling_config={
+        "min_replicas": 1,
+        "max_replicas": 5,
+        "metrics_interval_s": 0.1,
+        "policy": {
+            "policy_function": "autoscaling_policy:custom_metrics_autoscaling_policy"
+        },
+    },
+    max_ongoing_requests=5,
+)
+class CustomMetricsDeployment:
+    def __init__(self):
+        self.process = psutil.Process()
+
+    def __call__(self) -> str:
+        # Simulate some work
+        time.sleep(0.5)
+        return "Hello, world!"
+
+    def record_autoscaling_stats(self) -> Dict[str, float]:
+        # Get CPU usage as a percentage
+        cpu_usage = self.process.cpu_percent(interval=0.1)
+
+        # Get memory usage as a percentage of system memory
+        memory_info = self.process.memory_full_info()
+        system_memory = psutil.virtual_memory().total
+        memory_usage = (memory_info.uss / system_memory) * 100
+
+        return {
+            "cpu_usage": cpu_usage,
+            "memory_usage": memory_usage,
+        }
+
+
+# Create the app
+app = CustomMetricsDeployment.bind()
+# __serve_example_end__
+
+if __name__ == "__main__":
+    import requests  # noqa
+
+    serve.run(app)
+    for _ in range(10):
+        resp = requests.get("http://localhost:8000/")
+        assert resp.text == "Hello, world!"

doc/source/serve/doc_code/custom_request_router.py

@@ -0,0 +1,123 @@
+# flake8: noqa
+# __begin_define_uniform_request_router__
+import random
+from ray.serve.request_router import (
+    PendingRequest,
+    RequestRouter,
+    ReplicaID,
+    ReplicaResult,
+    RunningReplica,
+)
+from typing import (
+    List,
+    Optional,
+)
+
+
+class UniformRequestRouter(RequestRouter):
+    async def choose_replicas(
+        self,
+        candidate_replicas: List[RunningReplica],
+        pending_request: Optional[PendingRequest] = None,
+    ) -> List[List[RunningReplica]]:
+        print("UniformRequestRouter routing request")
+        index = random.randint(0, len(candidate_replicas) - 1)
+        return [[candidate_replicas[index]]]
+
+    def on_request_routed(
+        self,
+        pending_request: PendingRequest,
+        replica_id: ReplicaID,
+        result: ReplicaResult,
+    ):
+        print("on_request_routed callback is called!!")
+
+
+# __end_define_uniform_request_router__
+
+
+# __begin_define_throughput_aware_request_router__
+from ray.serve.request_router import (
+    FIFOMixin,
+    LocalityMixin,
+    MultiplexMixin,
+    PendingRequest,
+    RequestRouter,
+    ReplicaID,
+    ReplicaResult,
+    RunningReplica,
+)
+from typing import (
+    Dict,
+    List,
+    Optional,
+)
+
+
+class ThroughputAwareRequestRouter(
+    FIFOMixin, MultiplexMixin, LocalityMixin, RequestRouter
+):
+    async def choose_replicas(
+        self,
+        candidate_replicas: List[RunningReplica],
+        pending_request: Optional[PendingRequest] = None,
+    ) -> List[List[RunningReplica]]:
+        """
+        This method chooses the best replica for the request based on
+        multiplexed, locality, and custom throughput stats. The algorithm
+        works as follows:
+
+        1. Populate top_ranked_replicas based on available replicas based on
+          multiplex_id
+        2. Populate and override top_ranked_replicas info based on locality
+         information of replicas (we want to prefer replicas that are in the
+          same vicinity to this deployment)
+        3. Select the replica with minimum throughput.
+        """
+
+        # Dictionary to hold the top-ranked replicas
+        top_ranked_replicas: Dict[ReplicaID, RunningReplica] = {}
+        # Take the best set of replicas for the multiplexed model
+        if (
+            pending_request is not None
+            and pending_request.metadata.multiplexed_model_id
+        ):
+            ranked_replicas_multiplex: List[RunningReplica] = (
+                self.rank_replicas_via_multiplex(
+                    replicas=candidate_replicas,
+                    multiplexed_model_id=pending_request.metadata.multiplexed_model_id,
+                )
+            )[0]
+
+            # Filter out replicas that are not available (queue length exceed max ongoing request)
+            ranked_replicas_multiplex = self.select_available_replicas(
+                candidates=ranked_replicas_multiplex
+            )
+
+            for replica in ranked_replicas_multiplex:
+                top_ranked_replicas[replica.replica_id] = replica
+
+        # Take the best set of replicas in terms of locality
+        ranked_replicas_locality: List[
+            RunningReplica
+        ] = self.rank_replicas_via_locality(replicas=candidate_replicas)[0]
+
+        # Filter out replicas that are not available (queue length exceed max ongoing request)
+        ranked_replicas_locality = self.select_available_replicas(
+            candidates=ranked_replicas_locality
+        )
+
+        for replica in ranked_replicas_locality:
+            top_ranked_replicas[replica.replica_id] = replica
+
+        print("ThroughputAwareRequestRouter routing request")
+
+        # Take the replica with minimum throughput.
+        min_throughput_replicas = min(
+            [replica for replica in top_ranked_replicas.values()],
+            key=lambda r: r.routing_stats.get("throughput", 0),
+        )
+        return [[min_throughput_replicas]]
+
+
+# __end_define_throughput_aware_request_router__

doc/source/serve/doc_code/custom_request_router_app.py

@@ -0,0 +1,164 @@
+# flake8: noqa
+
+# __begin_deploy_app_with_uniform_request_router__
+from ray import serve
+from ray.serve.request_router import ReplicaID
+import time
+from collections import defaultdict
+from ray.serve.context import _get_internal_replica_context
+from typing import Any, Dict
+from ray.serve.config import RequestRouterConfig
+
+
+@serve.deployment(
+    request_router_config=RequestRouterConfig(
+        request_router_class="custom_request_router:UniformRequestRouter",
+    ),
+    num_replicas=10,
+    ray_actor_options={"num_cpus": 0},
+)
+class UniformRequestRouterApp:
+    def __init__(self):
+        context = _get_internal_replica_context()
+        self.replica_id: ReplicaID = context.replica_id
+
+    async def __call__(self):
+        return self.replica_id
+
+
+handle = serve.run(UniformRequestRouterApp.bind())
+response = handle.remote().result()
+print(f"Response from UniformRequestRouterApp: {response}")
+# Example output:
+#   Response from UniformRequestRouterApp:
+#   Replica(id='67vc4ts5', deployment='UniformRequestRouterApp', app='default')
+# __end_deploy_app_with_uniform_request_router__
+
+
+# __begin_deploy_app_with_throughput_aware_request_router__
+def _time_ms() -> int:
+    return int(time.time() * 1000)
+
+
+@serve.deployment(
+    request_router_config=RequestRouterConfig(
+        request_router_class="custom_request_router:ThroughputAwareRequestRouter",
+        request_routing_stats_period_s=1,
+        request_routing_stats_timeout_s=1,
+    ),
+    num_replicas=3,
+    ray_actor_options={"num_cpus": 0},
+)
+class ThroughputAwareRequestRouterApp:
+    def __init__(self):
+        self.throughput_buckets: Dict[int, int] = defaultdict(int)
+        self.last_throughput_buckets = _time_ms()
+        context = _get_internal_replica_context()
+        self.replica_id: ReplicaID = context.replica_id
+
+    def __call__(self):
+        self.update_throughput()
+        return self.replica_id
+
+    def update_throughput(self):
+        current_timestamp_ms = _time_ms()
+
+        # Under high concurrency, requests can come in at different times. This
+        # early return helps to skip if the current_timestamp_ms is more than a
+        # second older than the last throughput bucket.
+        if current_timestamp_ms < self.last_throughput_buckets - 1000:
+            return
+
+        # Record the request to the bucket
+        self.throughput_buckets[current_timestamp_ms] += 1
+        self.last_throughput_buckets = current_timestamp_ms
+
+    def record_routing_stats(self) -> Dict[str, Any]:
+        current_timestamp_ms = _time_ms()
+        throughput = 0
+
+        for t, c in list(self.throughput_buckets.items()):
+            if t < current_timestamp_ms - 1000:
+                # Remove the bucket if it is older than 1 second
+                self.throughput_buckets.pop(t)
+            else:
+                throughput += c
+
+        return {
+            "throughput": throughput,
+        }
+
+
+handle = serve.run(ThroughputAwareRequestRouterApp.bind())
+response = handle.remote().result()
+print(f"Response from ThroughputAwareRequestRouterApp: {response}")
+# Example output:
+#   Response from ThroughputAwareRequestRouterApp:
+#   Replica(id='tkywafya', deployment='ThroughputAwareRequestRouterApp', app='default')
+# __end_deploy_app_with_throughput_aware_request_router__
+
+
+# __begin_deploy_app_with_round_robin_router__
+@serve.deployment(
+    request_router_config=RequestRouterConfig(
+        request_router_class=(
+            "ray.serve.experimental.round_robin_router:RoundRobinRouter"
+        ),
+    ),
+    num_replicas=4,
+    ray_actor_options={"num_cpus": 0},
+)
+class RoundRobinRouterApp:
+    def __init__(self):
+        context = _get_internal_replica_context()
+        self.replica_id: ReplicaID = context.replica_id
+
+    async def __call__(self):
+        return self.replica_id
+
+
+handle = serve.run(RoundRobinRouterApp.bind())
+response = handle.remote().result()
+print(f"Response from RoundRobinRouterApp: {response}")
+# __end_deploy_app_with_round_robin_router__
+
+
+# __begin_deploy_app_with_consistent_hash_router__
+import requests
+from starlette.requests import Request
+
+@serve.deployment(
+    request_router_config=RequestRouterConfig(
+        request_router_class=(
+            "ray.serve.experimental.consistent_hash_router:ConsistentHashRouter"
+        ),
+        request_router_kwargs={
+            "num_virtual_nodes": 100,
+            "num_fallback_replicas": 2,
+        },
+    ),
+    num_replicas=4,
+    ray_actor_options={"num_cpus": 0},
+)
+class ConsistentHashRouterApp:
+    def __init__(self):
+        context = _get_internal_replica_context()
+        self.replica_id: ReplicaID = context.replica_id
+
+    async def __call__(self, request: Request) -> str:
+        return str(self.replica_id)
+
+
+serve.run(ConsistentHashRouterApp.bind())
+
+# Clients pin a session to a replica by sending the same `x-session-id`
+# on every request.
+response = requests.get(
+    "http://localhost:8000/",
+    headers={"x-session-id": "example-session-id"},
+)
+print(f"Response from ConsistentHashRouterApp: {response.text}")
+# Example output:
+#   Response from ConsistentHashRouterApp:
+#   Replica(id='...', deployment='ConsistentHashRouterApp', app='default')
+# __end_deploy_app_with_consistent_hash_router__

doc/source/serve/doc_code/delayed_import.py

@@ -0,0 +1,9 @@
+from ray import serve
+
+
+@serve.deployment
+class MyDeployment:
+    def __call__(self, model_path):
+        from my_module import my_model
+
+        self.model = my_model.load(model_path)

doc/source/serve/doc_code/deployment_scoped_actors.py

@@ -0,0 +1,88 @@
+import asyncio
+import time
+
+import ray
+from ray import serve
+from ray.exceptions import RayActorError
+from ray.serve.config import DeploymentActorConfig
+
+
+# __begin_define_deployment_scoped_actor__
+@ray.remote
+class SharedCounter:
+    def __init__(self, start: int = 0):
+        self._value = start
+
+    def increment(self) -> int:
+        self._value += 1
+        return self._value
+
+
+@serve.deployment(
+    deployment_actors=[
+        DeploymentActorConfig(
+            name="counter",
+            actor_class=SharedCounter,
+            init_kwargs={"start": 10},
+            actor_options={"num_cpus": 0},
+        ),
+    ],
+)
+class SharedCounterDeployment:
+    async def __call__(self) -> int:
+        counter = serve.get_deployment_actor("counter")
+        return await counter.increment.remote()
+
+
+# __end_define_deployment_scoped_actor__
+
+
+# __begin_cached_handle_refresh__
+@serve.deployment(
+    deployment_actors=[
+        DeploymentActorConfig(
+            name="counter",
+            actor_class=SharedCounter,
+            init_kwargs={"start": 0},
+            actor_options={"num_cpus": 0},
+        ),
+    ],
+)
+class CachedHandleDeployment:
+    def __init__(self):
+        self._counter = serve.get_deployment_actor("counter")
+
+    async def _refresh_counter(self) -> None:
+        deadline = time.monotonic() + 30
+        while time.monotonic() < deadline:
+            try:
+                self._counter = serve.get_deployment_actor("counter")
+                return
+            except ValueError:
+                # The replacement actor might not be registered yet.
+                await asyncio.sleep(0.05)
+
+        raise TimeoutError("Timed out waiting for the deployment-scoped actor.")
+
+    async def __call__(self) -> int:
+        try:
+            return await self._counter.increment.remote()
+        except RayActorError:
+            await self._refresh_counter()
+            return await self._counter.increment.remote()
+
+
+# __end_cached_handle_refresh__
+
+
+if __name__ == "__main__":
+    ray.init()
+    try:
+        # __begin_run_deployment_scoped_actor_example__
+        handle = serve.run(SharedCounterDeployment.bind())
+        print(handle.remote().result())
+        print(handle.remote().result())
+        # __end_run_deployment_scoped_actor_example__
+    finally:
+        serve.shutdown()
+        ray.shutdown()

doc/source/serve/doc_code/develop_and_deploy.py

@@ -0,0 +1,58 @@
+# flake8: noqa
+
+# __deployment_start__
+import ray
+from ray import serve
+from fastapi import FastAPI
+
+from transformers import AutoModelForSeq2SeqLM, AutoTokenizer
+
+app = FastAPI()
+
+
+@serve.deployment(num_replicas=2, ray_actor_options={"num_cpus": 0.2, "num_gpus": 0})
+@serve.ingress(app)
+class Translator:
+    def __init__(self):
+        # Load model
+        self.tokenizer = AutoTokenizer.from_pretrained("t5-small")
+        self.model = AutoModelForSeq2SeqLM.from_pretrained("t5-small")
+
+    @app.post("/")
+    def translate(self, text: str) -> str:
+        # Run inference
+        input_ids = self.tokenizer(
+            f"translate English to French: {text}", return_tensors="pt"
+        ).input_ids
+        output_ids = self.model.generate(
+            input_ids, num_beams=4, early_stopping=True, max_length=300
+        )
+
+        # Post-process output to return only the translation text
+        translation = self.tokenizer.decode(
+            output_ids[0], skip_special_tokens=True, clean_up_tokenization_spaces=False
+        )
+
+        return translation
+
+
+translator_app = Translator.bind()
+# __deployment_end__
+
+translator_app = Translator.options(ray_actor_options={}).bind()
+serve.run(translator_app)
+
+# __client_function_start__
+# File name: model_client.py
+import requests
+
+response = requests.post("http://127.0.0.1:8000/", params={"text": "Hello world!"})
+french_text = response.json()
+
+print(french_text)
+# __client_function_end__
+
+assert french_text == "Bonjour monde!"
+
+serve.shutdown()
+ray.shutdown()

doc/source/serve/doc_code/distilbert.py

@@ -0,0 +1,61 @@
+# __example_code_start__
+from transformers import pipeline
+from fastapi import FastAPI
+import torch
+
+from ray import serve
+from ray.serve.handle import DeploymentHandle
+
+
+app = FastAPI()
+
+
+@serve.deployment(num_replicas=1)
+@serve.ingress(app)
+class APIIngress:
+    def __init__(self, distilbert_model_handle: DeploymentHandle) -> None:
+        self.handle = distilbert_model_handle
+
+    @app.get("/classify")
+    async def classify(self, sentence: str):
+        return await self.handle.classify.remote(sentence)
+
+
+@serve.deployment(
+    ray_actor_options={"num_gpus": 1},
+    autoscaling_config={"min_replicas": 0, "max_replicas": 2},
+)
+class DistilBertModel:
+    def __init__(self):
+        self.classifier = pipeline(
+            "sentiment-analysis",
+            model="distilbert-base-uncased",
+            framework="pt",
+            # Transformers requires you to pass device with index
+            device=torch.device("cuda:0"),
+        )
+
+    def classify(self, sentence: str):
+        return self.classifier(sentence)
+
+
+entrypoint = APIIngress.bind(DistilBertModel.bind())
+
+# __example_code_end__
+
+if __name__ == "__main__":
+    import requests
+    import ray
+
+    ray.init(runtime_env={"pip": ["transformers==4.36.2", "accelerate==0.28.0"]})
+    serve.run(entrypoint)
+
+    prompt = (
+        "This was a masterpiece. Not completely faithful to the books, but "
+        "enthralling  from beginning to end. Might be my favorite of the three."
+    )
+    input = "%20".join(prompt.split(" "))
+    resp = requests.get(f"http://127.0.0.1:8000/classify?sentence={prompt}")
+    print(resp.status_code, resp.json())
+
+    assert resp.status_code == 200

doc/source/serve/doc_code/external_scaler_config.yaml

@@ -0,0 +1,10 @@
+# __external_scaler_config_begin__
+applications:
+  - name: my-app
+    import_path: external_scaler_predictive:app
+    external_scaler_enabled: true
+    deployments:
+      - name: TextProcessor
+        num_replicas: 1
+# __external_scaler_config_end__
+

doc/source/serve/doc_code/external_scaler_predictive.py

@@ -0,0 +1,35 @@
+# __serve_example_begin__
+import time
+from ray import serve
+from typing import Any
+
+@serve.deployment(num_replicas=3)
+class TextProcessor:
+    """A simple text processing deployment that can be scaled externally."""
+    def __init__(self):
+        self.request_count = 0
+
+    def __call__(self, text: Any) -> dict:
+        # Simulate text processing work
+        time.sleep(0.1)
+        self.request_count += 1
+        return {
+            "request_count": self.request_count,
+        }
+
+
+app = TextProcessor.bind()
+# __serve_example_end__
+
+def main():
+    import requests
+
+    serve.run(app)
+    
+    # Test the deployment
+    resp = requests.post(
+        "http://localhost:8000/",
+        json="hello world"
+    )
+    print(f"Response: {resp.json()}")
+

doc/source/serve/doc_code/external_scaler_predictive_client.py

@@ -0,0 +1,82 @@
+# __client_script_begin__
+import logging
+import time
+from datetime import datetime
+import requests
+
+APPLICATION_NAME = "my-app"
+DEPLOYMENT_NAME = "TextProcessor"
+SERVE_ENDPOINT = "http://localhost:8265"
+SCALING_INTERVAL = 300  # Check every 5 minutes
+
+logger = logging.getLogger(__name__)
+
+
+def get_current_replicas(app_name: str, deployment_name: str) -> int:
+    """Get current replica count. Returns -1 on error."""
+    try:
+        resp = requests.get(
+            f"{SERVE_ENDPOINT}/api/serve/applications/",
+            timeout=10
+        )
+        if resp.status_code != 200:
+            logger.error(f"Failed to get applications: {resp.status_code}")
+            return -1
+            
+        apps = resp.json().get("applications", {})
+        if app_name not in apps:
+            logger.error(f"Application {app_name} not found")
+            return -1
+
+        deployments = apps[app_name].get("deployments", {})
+        if deployment_name in deployments:
+            return deployments[deployment_name]["target_num_replicas"]
+                
+        logger.error(f"Deployment {deployment_name} not found")
+        return -1
+    except requests.exceptions.RequestException as e:
+        logger.error(f"Request failed: {e}")
+        return -1
+
+
+def scale_deployment(app_name: str, deployment_name: str):
+    """Scale deployment based on time of day."""
+    hour = datetime.now().hour
+    current = get_current_replicas(app_name, deployment_name)
+    
+    # Check if we successfully retrieved the current replica count
+    if current == -1:
+        logger.error("Failed to get current replicas, skipping scaling decision")
+        return
+    
+    target = 10 if 9 <= hour < 17 else 3  # Peak hours: 9am-5pm
+    
+    delta = target - current
+    if delta == 0:
+        logger.info(f"Already at target ({current} replicas)")
+        return
+    
+    action = "Adding" if delta > 0 else "Removing"
+    logger.info(f"{action} {abs(delta)} replicas ({current} -> {target})")
+    
+    try:
+        resp = requests.post(
+            f"{SERVE_ENDPOINT}/api/v1/applications/{app_name}/deployments/{deployment_name}/scale",
+            headers={"Content-Type": "application/json"},
+            json={"target_num_replicas": target},
+            timeout=10
+        )
+        if resp.status_code == 200:
+            logger.info("Successfully scaled deployment")
+        else:
+            logger.error(f"Scale failed: {resp.status_code} - {resp.text}")
+    except requests.exceptions.RequestException as e:
+        logger.error(f"Request failed: {e}")
+
+
+def main():
+    logger.info(f"Starting predictive scaling for {APPLICATION_NAME}/{DEPLOYMENT_NAME}")
+    while True:
+        scale_deployment(APPLICATION_NAME, DEPLOYMENT_NAME)
+        time.sleep(SCALING_INTERVAL)
+# __client_script_end__

doc/source/serve/doc_code/fake_email_creator.py

@@ -0,0 +1,16 @@
+# __fake_start__
+from faker import Faker
+
+from ray import serve
+
+
+@serve.deployment
+def create_fake_email():
+    return Faker().email()
+
+
+app = create_fake_email.bind()
+# __fake_end__
+
+handle = serve.run(app)
+assert handle.remote().result() == "fake@fake.com"

doc/source/serve/doc_code/fake_email_creator.yaml

@@ -0,0 +1,61 @@
+# __fake_config_start__
+apiVersion: ray.io/v1alpha1
+kind: RayService
+metadata:
+  name: rayservice-fake-emails
+spec:
+  serviceUnhealthySecondThreshold: 300
+  deploymentUnhealthySecondThreshold: 300
+  serveConfigV2: |
+    applications:
+      - name: fake
+        import_path: fake:app
+        route_prefix: /
+  rayClusterConfig:
+    rayVersion: '2.5.0' # Should match Ray version in the containers
+    headGroupSpec:
+      rayStartParams:
+        dashboard-host: '0.0.0.0'
+      template:
+        spec:
+          containers:
+            - name: ray-head
+              image: shrekrisanyscale/serve-fake-email-example:example
+              resources:
+                limits:
+                  cpu: 2
+                  memory: 2Gi
+                requests:
+                  cpu: 2
+                  memory: 2Gi
+              ports:
+                - containerPort: 6379
+                  name: gcs-server
+                - containerPort: 8265 # Ray dashboard
+                  name: dashboard
+                - containerPort: 10001
+                  name: client
+                - containerPort: 8000
+                  name: serve
+    workerGroupSpecs:
+      - replicas: 1
+        minReplicas: 1
+        maxReplicas: 1
+        groupName: small-group
+        template:
+          spec:
+            containers:
+              - name: ray-worker
+                image: shrekrisanyscale/serve-fake-email-example:example
+                lifecycle:
+                  preStop:
+                    exec:
+                      command: ["/bin/sh","-c","ray stop"]
+                resources:
+                  limits:
+                    cpu: "1"
+                    memory: "2Gi"
+                  requests:
+                    cpu: "500m"
+                    memory: "2Gi"
+# __fake_config_end__

doc/source/serve/doc_code/faker.py

@@ -0,0 +1,8 @@
+class Faker:
+    """Mock Faker class to test fake_email_creator.py.
+
+    Meant to mock https://github.com/joke2k/faker package.
+    """
+
+    def email(self) -> str:
+        return "fake@fake.com"

doc/source/serve/doc_code/fastapi_example.py

@@ -0,0 +1,23 @@
+import requests
+from fastapi import FastAPI
+from ray import serve
+
+# 1: Define a FastAPI app and wrap it in a deployment with a route handler.
+app = FastAPI()
+
+
+@serve.deployment
+@serve.ingress(app)
+class FastAPIDeployment:
+    # FastAPI will automatically parse the HTTP request for us.
+    @app.get("/hello")
+    def say_hello(self, name: str) -> str:
+        return f"Hello {name}!"
+
+
+# 2: Deploy the deployment.
+serve.run(FastAPIDeployment.bind(), route_prefix="/")
+
+# 3: Query the deployment and print the result.
+print(requests.get("http://localhost:8000/hello", params={"name": "Theodore"}).json())
+# "Hello Theodore!"

doc/source/serve/doc_code/fault_tolerance/k8s_config.yaml

@@ -0,0 +1,178 @@
+# File name: config.yaml
+
+kind: ConfigMap
+apiVersion: v1
+metadata:
+  name: redis-config
+  labels:
+    app: redis
+data:
+  redis.conf: |-
+    port 6379
+    bind 0.0.0.0
+    protected-mode no
+    requirepass 5241590000000000
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: redis
+  labels:
+    app: redis
+spec:
+  type: ClusterIP
+  ports:
+    - name: redis
+      port: 6379
+  selector:
+    app: redis
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: redis
+  labels:
+    app: redis
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: redis
+  template:
+    metadata:
+      labels:
+        app: redis
+    spec:
+      containers:
+        - name: redis
+          image: redis:5.0.8
+          command:
+            - "sh"
+            - "-c"
+            - "redis-server /usr/local/etc/redis/redis.conf"
+          ports:
+            - containerPort: 6379
+          volumeMounts:
+            - name: config
+              mountPath: /usr/local/etc/redis/redis.conf
+              subPath: redis.conf
+      volumes:
+        - name: config
+          configMap:
+            name: redis-config
+---
+apiVersion: ray.io/v1alpha1
+kind: RayService
+metadata:
+  name: rayservice-sample
+  annotations:
+    ray.io/ft-enabled: "true"
+spec:
+  serviceUnhealthySecondThreshold: 300
+  deploymentUnhealthySecondThreshold: 300
+  serveConfig:
+    importPath: "sleepy_pid:app"
+    runtimeEnv: |
+      working_dir: "https://github.com/ray-project/serve_config_examples/archive/42d10bab77741b40d11304ad66d39a4ec2345247.zip"
+    deployments:
+      - name: SleepyPid
+        numReplicas: 6
+        rayActorOptions:
+          numCpus: 0
+  rayClusterConfig:
+    rayVersion: '2.3.0'
+    headGroupSpec:
+      replicas: 1
+      rayStartParams:
+        num-cpus: '2'
+        dashboard-host: '0.0.0.0'
+        redis-password: "5241590000000000"
+      template:
+        spec:
+          containers:
+            - name: ray-head
+              image: rayproject/ray:2.3.0
+              imagePullPolicy: Always
+              env:
+                - name: MY_POD_IP
+                  valueFrom:
+                    fieldRef:
+                      fieldPath: status.podIP
+                - name: RAY_REDIS_ADDRESS
+                  value: redis:6379
+              resources:
+                limits:
+                  cpu: 2
+                  memory: 2Gi
+                requests:
+                  cpu: 2
+                  memory: 2Gi
+              ports:
+                - containerPort: 6379
+                  name: redis
+                - containerPort: 8265
+                  name: dashboard
+                - containerPort: 10001
+                  name: client
+                - containerPort: 8000
+                  name: serve
+    workerGroupSpecs:
+      - replicas: 2
+        minReplicas: 2
+        maxReplicas: 2
+        groupName: small-group
+        rayStartParams:
+          node-ip-address: $MY_POD_IP
+        template:
+          spec:
+            containers:
+              - name: machine-learning
+                image: rayproject/ray:2.3.0
+                imagePullPolicy: Always
+                env:
+                  - name:  RAY_DISABLE_DOCKER_CPU_WARNING
+                    value: "1"
+                  - name: TYPE
+                    value: "worker"
+                  - name: CPU_REQUEST
+                    valueFrom:
+                      resourceFieldRef:
+                        containerName: machine-learning
+                        resource: requests.cpu
+                  - name: CPU_LIMITS
+                    valueFrom:
+                      resourceFieldRef:
+                        containerName: machine-learning
+                        resource: limits.cpu
+                  - name: MEMORY_LIMITS
+                    valueFrom:
+                      resourceFieldRef:
+                        containerName: machine-learning
+                        resource: limits.memory
+                  - name: MEMORY_REQUESTS
+                    valueFrom:
+                      resourceFieldRef:
+                        containerName: machine-learning
+                        resource: requests.memory
+                  - name: MY_POD_NAME
+                    valueFrom:
+                      fieldRef:
+                        fieldPath: metadata.name
+                  - name: MY_POD_IP
+                    valueFrom:
+                      fieldRef:
+                        fieldPath: status.podIP
+                ports:
+                  - containerPort: 80
+                    name: client
+                lifecycle:
+                  preStop:
+                    exec:
+                      command: ["/bin/sh","-c","ray stop"]
+                resources:
+                  limits:
+                    cpu: "1"
+                    memory: "2Gi"
+                  requests:
+                    cpu: "500m"
+                    memory: "2Gi"

doc/source/serve/doc_code/fault_tolerance/replica_health_check.py

@@ -0,0 +1,24 @@
+# flake8: noqa
+
+from ray import serve
+
+# Stubs
+def connect_to_db(*args, **kwargs):
+    pass
+
+
+# __health_check_start__
+@serve.deployment(health_check_period_s=10, health_check_timeout_s=30)
+class MyDeployment:
+    def __init__(self, db_addr: str):
+        self._my_db_connection = connect_to_db(db_addr)
+
+    def __call__(self, request):
+        return self._do_something_cool()
+
+    # Called by Serve to check the replica's health.
+    def check_health(self):
+        if not self._my_db_connection.is_connected():
+            # The specific type of exception is not important.
+            raise RuntimeError("uh-oh, DB connection is broken.")
+            # __health_check_end__

doc/source/serve/doc_code/fault_tolerance/sleepy_pid.py

@@ -0,0 +1,23 @@
+# flake8: noqa
+
+# __start__
+# File name: sleepy_pid.py
+
+from ray import serve
+
+
+@serve.deployment
+class SleepyPid:
+    def __init__(self):
+        import time
+
+        time.sleep(10)
+
+    def __call__(self) -> int:
+        import os
+
+        return os.getpid()
+
+
+app = SleepyPid.bind()
+# __end__

doc/source/serve/doc_code/gang_scheduling.py

@@ -0,0 +1,183 @@
+from ray import serve
+
+# __basic_gang_start__
+from ray import serve
+from ray.serve.config import GangSchedulingConfig
+
+
+@serve.deployment(
+    num_replicas=8,
+    ray_actor_options={"num_cpus": 0.25},
+    gang_scheduling_config=GangSchedulingConfig(gang_size=4),
+)
+class Gang:
+    def __call__(self, request):
+        return "Hello!"
+
+
+app = Gang.bind()
+# __basic_gang_end__
+
+# __gang_context_start__
+@serve.deployment(
+    num_replicas=4,
+    ray_actor_options={"num_cpus": 0.25},
+    gang_scheduling_config=GangSchedulingConfig(gang_size=2),
+)
+class GangWithContext:
+    def __init__(self):
+        ctx = serve.get_replica_context()
+        gc = ctx.gang_context
+        self.rank = gc.rank
+        self.world_size = gc.world_size
+        self.gang_id = gc.gang_id
+        self.member_ids = gc.member_replica_ids
+
+    def __call__(self, request):
+        return {
+            "gang_id": self.gang_id,
+            "rank": self.rank,
+            "world_size": self.world_size,
+        }
+
+
+gang_context_app = GangWithContext.bind()
+# __gang_context_end__
+
+# __pack_strategy_start__
+from ray import serve
+from ray.serve.config import GangPlacementStrategy, GangSchedulingConfig
+
+
+@serve.deployment(
+    num_replicas=4,
+    ray_actor_options={"num_cpus": 0.25},
+    gang_scheduling_config=GangSchedulingConfig(
+        gang_size=4,
+        gang_placement_strategy=GangPlacementStrategy.PACK,
+    ),
+)
+class PackedGang:
+    def __call__(self, request):
+        return "Packed on same node"
+
+
+packed_app = PackedGang.bind()
+# __pack_strategy_end__
+
+# __spread_strategy_start__
+@serve.deployment(
+    num_replicas=4,
+    ray_actor_options={"num_cpus": 0.25},
+    gang_scheduling_config=GangSchedulingConfig(
+        gang_size=2,
+        gang_placement_strategy=GangPlacementStrategy.SPREAD,
+    ),
+)
+class SpreadGang:
+    def __call__(self, request):
+        return "Spread across nodes"
+
+
+spread_app = SpreadGang.bind()
+# __spread_strategy_end__
+
+# __options_start__
+@serve.deployment
+class BaseGang:
+    def __call__(self, request):
+        return "Hello!"
+
+
+app_with_gang = BaseGang.options(
+    num_replicas=8,
+    ray_actor_options={"num_cpus": 0.25},
+    gang_scheduling_config=GangSchedulingConfig(gang_size=4),
+).bind()
+# __options_end__
+
+# __autoscaling_start__
+@serve.deployment(
+    autoscaling_config={
+        "min_replicas": 4,
+        "max_replicas": 16,
+        "initial_replicas": 8,
+        "target_ongoing_requests": 5,
+    },
+    ray_actor_options={"num_cpus": 0.25},
+    gang_scheduling_config=GangSchedulingConfig(gang_size=4),
+)
+class AutoscaledGang:
+    def __call__(self, request):
+        return "Hello!"
+
+
+autoscaled_app = AutoscaledGang.bind()
+# __autoscaling_end__
+
+# __fault_tolerance_start__
+from ray import serve
+from ray.serve.config import GangRuntimeFailurePolicy, GangSchedulingConfig
+
+
+@serve.deployment(
+    num_replicas=8,
+    ray_actor_options={"num_cpus": 0.25},
+    gang_scheduling_config=GangSchedulingConfig(
+        gang_size=4,
+        runtime_failure_policy=GangRuntimeFailurePolicy.RESTART_GANG,
+    ),
+)
+class FaultTolerantGang:
+    def __call__(self, request):
+        return "Hello!"
+
+
+fault_tolerant_app = FaultTolerantGang.bind()
+# __fault_tolerance_end__
+
+# __placement_group_bundles_start__
+@serve.deployment(
+    num_replicas=4,
+    ray_actor_options={"num_cpus": 0},
+    placement_group_bundles=[{"CPU": 1, "GPU": 1}],
+    gang_scheduling_config=GangSchedulingConfig(gang_size=2),
+)
+class GangWithSingleBundleReplica:
+    def __call__(self, request):
+        return "Running on reserved GPUs"
+
+
+gang_single_bundle_replica_app = GangWithSingleBundleReplica.bind()
+# __placement_group_bundles_end__
+
+# __multi_placement_group_bundles_start__
+@serve.deployment(
+    num_replicas=4,
+    ray_actor_options={"num_cpus": 1},
+    placement_group_bundles=[{"CPU": 1, "GPU": 1}, {"GPU": 1}],
+    gang_scheduling_config=GangSchedulingConfig(gang_size=2),
+)
+class GangWithMultiBundlesReplica:
+    def __call__(self, request):
+        return "Running on reserved GPUs"
+
+
+gang_multi_bundles_replica_app = GangWithMultiBundlesReplica.bind()
+# __multi_placement_group_bundles_end__
+
+# __label_selector_start__
+@serve.deployment(
+    num_replicas=4,
+    ray_actor_options={"num_cpus": 0},
+    placement_group_bundles=[{"CPU": 1, "GPU": 1}],
+    placement_group_bundle_label_selector=[{"ray.io/accelerator-type": "A100"}],
+    gang_scheduling_config=GangSchedulingConfig(gang_size=2),
+)
+class GangOnA100:
+    def __call__(self, request):
+        return "Running on A100"
+
+
+gang_a100_app = GangOnA100.bind()
+# __label_selector_end__

doc/source/serve/doc_code/getting_started/model_deployment.py

@@ -0,0 +1,67 @@
+# flake8: noqa
+
+# __import_start__
+from starlette.requests import Request
+
+import ray
+from ray import serve
+
+# __import_end__
+
+# __model_start__
+from transformers import AutoModelForSeq2SeqLM, AutoTokenizer
+
+
+@serve.deployment(num_replicas=2, ray_actor_options={"num_cpus": 0.2, "num_gpus": 0})
+class Translator:
+    def __init__(self):
+        # Load model
+        self.tokenizer = AutoTokenizer.from_pretrained("t5-small")
+        self.model = AutoModelForSeq2SeqLM.from_pretrained("t5-small")
+
+    def translate(self, text: str) -> str:
+        # Run inference
+        input_ids = self.tokenizer(
+            f"translate English to French: {text}", return_tensors="pt"
+        ).input_ids
+        output_ids = self.model.generate(
+            input_ids, num_beams=4, early_stopping=True, max_length=300
+        )
+
+        # Post-process output to return only the translation text
+        translation = self.tokenizer.decode(
+            output_ids[0], skip_special_tokens=True, clean_up_tokenization_spaces=False
+        )
+
+        return translation
+
+    async def __call__(self, http_request: Request) -> str:
+        english_text: str = await http_request.json()
+        return self.translate(english_text)
+
+
+# __model_end__
+
+# __model_deploy_start__
+translator_app = Translator.bind()
+# __model_deploy_end__
+
+translator_app = Translator.options(ray_actor_options={}).bind()
+serve.run(translator_app)
+
+# __client_function_start__
+# File name: model_client.py
+import requests
+
+english_text = "Hello world!"
+
+response = requests.post("http://127.0.0.1:8000/", json=english_text)
+french_text = response.text
+
+print(french_text)
+# __client_function_end__
+
+assert french_text == "Bonjour monde!"
+
+serve.shutdown()
+ray.shutdown()

doc/source/serve/doc_code/getting_started/model_deployment_full.py

@@ -0,0 +1,53 @@
+# flake8: noqa
+
+# __deployment_full_start__
+# File name: serve_quickstart.py
+from starlette.requests import Request
+
+import ray
+from ray import serve
+
+from transformers import AutoModelForSeq2SeqLM, AutoTokenizer
+
+
+@serve.deployment(num_replicas=2, ray_actor_options={"num_cpus": 0.2, "num_gpus": 0})
+class Translator:
+    def __init__(self):
+        # Load model
+        self.tokenizer = AutoTokenizer.from_pretrained("t5-small")
+        self.model = AutoModelForSeq2SeqLM.from_pretrained("t5-small")
+
+    def translate(self, text: str) -> str:
+        # Run inference
+        input_ids = self.tokenizer(
+            f"translate English to French: {text}", return_tensors="pt"
+        ).input_ids
+        output_ids = self.model.generate(
+            input_ids, num_beams=4, early_stopping=True, max_length=300
+        )
+
+        # Post-process output to return only the translation text
+        translation = self.tokenizer.decode(
+            output_ids[0], skip_special_tokens=True, clean_up_tokenization_spaces=False
+        )
+
+        return translation
+
+    async def __call__(self, http_request: Request) -> str:
+        english_text: str = await http_request.json()
+        return self.translate(english_text)
+
+
+translator_app = Translator.bind()
+# __deployment_full_end__
+
+translator_app = Translator.options(ray_actor_options={}).bind()
+serve.run(translator_app)
+
+import requests
+
+response = requests.post("http://127.0.0.1:8000/", json="Hello world!").text
+assert response == "Bonjour monde!"
+
+serve.shutdown()
+ray.shutdown()

doc/source/serve/doc_code/getting_started/models.py

@@ -0,0 +1,83 @@
+# flake8: noqa
+
+# __start_translation_model__
+# File name: model.py
+from transformers import AutoModelForSeq2SeqLM, AutoTokenizer
+
+
+class Translator:
+    def __init__(self):
+        # Load model
+        self.tokenizer = AutoTokenizer.from_pretrained("t5-small")
+        self.model = AutoModelForSeq2SeqLM.from_pretrained("t5-small")
+
+    def translate(self, text: str) -> str:
+        # Run inference
+        input_ids = self.tokenizer(
+            f"translate English to French: {text}", return_tensors="pt"
+        ).input_ids
+        output_ids = self.model.generate(
+            input_ids, num_beams=4, early_stopping=True, max_length=300
+        )
+
+        # Post-process output to return only the translation text
+        translation = self.tokenizer.decode(
+            output_ids[0], skip_special_tokens=True, clean_up_tokenization_spaces=False
+        )
+
+        return translation
+
+
+translator = Translator()
+
+translation = translator.translate("Hello world!")
+print(translation)
+# __end_translation_model__
+
+# Test model behavior
+assert translation == "Bonjour monde!"
+
+
+# __start_summarization_model__
+# File name: summary_model.py
+from transformers import AutoModelForSeq2SeqLM, AutoTokenizer
+
+
+class Summarizer:
+    def __init__(self):
+        # Load model
+        self.tokenizer = AutoTokenizer.from_pretrained("t5-small")
+        self.model = AutoModelForSeq2SeqLM.from_pretrained("t5-small")
+
+    def summarize(self, text: str) -> str:
+        # Run inference
+        input_ids = self.tokenizer(f"summarize: {text}", return_tensors="pt").input_ids
+        output_ids = self.model.generate(
+            input_ids,
+            num_beams=4,
+            early_stopping=True,
+            length_penalty=2.0,
+            no_repeat_ngram_size=3,
+            min_length=5,
+            max_length=15,
+        )
+
+        # Post-process output to return only the summary text
+        summary = self.tokenizer.decode(
+            output_ids[0], skip_special_tokens=True, clean_up_tokenization_spaces=False
+        )
+
+        return summary
+
+
+summarizer = Summarizer()
+
+summary = summarizer.summarize(
+    "It was the best of times, it was the worst of times, it was the age "
+    "of wisdom, it was the age of foolishness, it was the epoch of belief"
+)
+print(summary)
+# __end_summarization_model__
+
+# Test model behavior
+assert summary == "it was the best of times, it was worst of times ."

doc/source/serve/doc_code/getting_started/translator.py

@@ -0,0 +1,91 @@
+# flake8: noqa
+
+# __start_graph__
+# File name: serve_quickstart_composed.py
+from starlette.requests import Request
+
+import ray
+from ray import serve
+from ray.serve.handle import DeploymentHandle
+
+from transformers import AutoModelForSeq2SeqLM, AutoTokenizer
+
+
+@serve.deployment
+class Translator:
+    def __init__(self):
+        # Load model
+        self.tokenizer = AutoTokenizer.from_pretrained("t5-small")
+        self.model = AutoModelForSeq2SeqLM.from_pretrained("t5-small")
+
+    def translate(self, text: str) -> str:
+        # Run inference
+        input_ids = self.tokenizer(
+            f"translate English to French: {text}", return_tensors="pt"
+        ).input_ids
+        output_ids = self.model.generate(
+            input_ids, num_beams=4, early_stopping=True, max_length=300
+        )
+
+        # Post-process output to return only the translation text
+        translation = self.tokenizer.decode(
+            output_ids[0], skip_special_tokens=True, clean_up_tokenization_spaces=False
+        )
+
+        return translation
+
+
+@serve.deployment
+class Summarizer:
+    def __init__(self, translator: DeploymentHandle):
+        self.translator = translator
+
+        # Load model.
+        self.tokenizer = AutoTokenizer.from_pretrained("t5-small")
+        self.model = AutoModelForSeq2SeqLM.from_pretrained("t5-small")
+
+    def summarize(self, text: str) -> str:
+        # Run inference
+        input_ids = self.tokenizer(f"summarize: {text}", return_tensors="pt").input_ids
+        output_ids = self.model.generate(
+            input_ids, num_beams=4, early_stopping=True, max_length=15
+        )
+
+        # Post-process output to return only the summary text
+        summary = self.tokenizer.decode(
+            output_ids[0], skip_special_tokens=True, clean_up_tokenization_spaces=False
+        )
+
+        return summary
+
+    async def __call__(self, http_request: Request) -> str:
+        english_text: str = await http_request.json()
+        summary = self.summarize(english_text)
+
+        translation = await self.translator.translate.remote(summary)
+        return translation
+
+
+app = Summarizer.bind(Translator.bind())
+# __end_graph__
+
+serve.run(app)
+
+# __start_client__
+# File name: composed_client.py
+import requests
+
+english_text = (
+    "It was the best of times, it was the worst of times, it was the age "
+    "of wisdom, it was the age of foolishness, it was the epoch of belief"
+)
+response = requests.post("http://127.0.0.1:8000/", json=english_text)
+french_text = response.text
+
+print(french_text)
+# __end_client__
+
+assert french_text == "C'était le meilleur des temps, c'était le pire des temps,"
+
+serve.shutdown()
+ray.shutdown()

doc/source/serve/doc_code/gradio-integration-parallel.py

@@ -0,0 +1,74 @@
+import requests
+
+# __doc_import_begin__
+from ray import serve
+from ray.serve.handle import DeploymentHandle
+from ray.serve.gradio_integrations import GradioIngress
+
+import gradio as gr
+
+import asyncio
+from transformers import pipeline
+
+# __doc_import_end__
+
+
+# __doc_models_begin__
+@serve.deployment
+class TextGenerationModel:
+    def __init__(self, model_name):
+        self.generator = pipeline("text-generation", model=model_name)
+
+    def __call__(self, text):
+        generated_list = self.generator(
+            text, do_sample=True, min_length=20, max_length=100
+        )
+        generated = generated_list[0]["generated_text"]
+        return generated
+
+
+app1 = TextGenerationModel.bind("gpt2")
+app2 = TextGenerationModel.bind("distilgpt2")
+# __doc_models_end__
+
+
+# __doc_gradio_server_begin__
+@serve.deployment
+class MyGradioServer(GradioIngress):
+    def __init__(
+        self, downstream_model_1: DeploymentHandle, downstream_model_2: DeploymentHandle
+    ):
+        self._d1 = downstream_model_1
+        self._d2 = downstream_model_2
+
+        super().__init__(
+            lambda: gr.Interface(
+                self.fanout, "textbox", "textbox", api_name="predict"
+            )
+        )
+
+    async def fanout(self, text):
+        [result1, result2] = await asyncio.gather(
+            self._d1.remote(text), self._d2.remote(text)
+        )
+        return (
+            f"[Generated text version 1]\n{result1}\n\n"
+            f"[Generated text version 2]\n{result2}"
+        )
+        # __doc_gradio_server_end__
+
+
+# __doc_app_begin__
+app = MyGradioServer.bind(app1, app2)
+# __doc_app_end__
+
+# Test example code
+serve.run(app)
+response = requests.post(
+    "http://127.0.0.1:8000/gradio_api/run/predict/", json={"data": ["My name is Lewis"]}
+)
+assert response.status_code == 200
+print(
+    "gradio-integration-parallel.py: Response from example code is",
+    response.json()["data"],
+)

doc/source/serve/doc_code/gradio-integration.py

@@ -0,0 +1,70 @@
+import requests
+from ray import serve
+
+# __doc_import_begin__
+from ray.serve.gradio_integrations import GradioServer
+
+import gradio as gr
+
+from transformers import AutoModelForSeq2SeqLM, AutoTokenizer
+
+# __doc_import_end__
+
+
+# __doc_gradio_app_begin__
+example_input = (
+    "HOUSTON -- Men have landed and walked on the moon. "
+    "Two Americans, astronauts of Apollo 11, steered their fragile "
+    "four-legged lunar module safely and smoothly to the historic landing "
+    "yesterday at 4:17:40 P.M., Eastern daylight time. Neil A. Armstrong, the "
+    "38-year-old commander, radioed to earth and the mission control room "
+    'here: "Houston, Tranquility Base here. The Eagle has landed." The '
+    "first men to reach the moon -- Armstrong and his co-pilot, Col. Edwin E. "
+    "Aldrin Jr. of the Air Force -- brought their ship to rest on a level, "
+    "rock-strewn plain near the southwestern shore of the arid Sea of "
+    "Tranquility. About six and a half hours later, Armstrong opened the "
+    "landing craft's hatch, stepped slowly down the ladder and declared as "
+    "he planted the first human footprint on the lunar crust: \"That's one "
+    'small step for man, one giant leap for mankind." His first step on the '
+    "moon came at 10:56:20 P.M., as a television camera outside the craft "
+    "transmitted his every move to an awed and excited audience of hundreds "
+    "of millions of people on earth."
+)
+
+
+def gradio_summarizer_builder():
+    tokenizer = AutoTokenizer.from_pretrained("t5-small")
+    summarizer_model = AutoModelForSeq2SeqLM.from_pretrained("t5-small")
+
+    def model(text):
+        input_ids = tokenizer(f"summarize: {text}", return_tensors="pt").input_ids
+        output_ids = summarizer_model.generate(
+            input_ids, num_beams=4, early_stopping=True, max_length=200
+        )
+        return tokenizer.decode(
+            output_ids[0], skip_special_tokens=True, clean_up_tokenization_spaces=False
+        )
+
+    return gr.Interface(
+        fn=model,
+        inputs=[gr.Textbox(value=example_input, label="Input prompt")],
+        outputs=[gr.Textbox(label="Model output")],
+        api_name="predict",
+    )
+    # __doc_gradio_app_end__
+
+
+# __doc_app_begin__
+app = GradioServer.options(ray_actor_options={"num_cpus": 4}).bind(
+    gradio_summarizer_builder
+)
+# __doc_app_end__
+
+# Test example code
+serve.run(app)
+response = requests.post(
+    "http://127.0.0.1:8000/gradio_api/run/predict/", json={"data": [example_input]}
+)
+assert response.status_code == 200
+print("gradio-integration.py: Response from example code is", response.json()["data"])
+serve.shutdown()

doc/source/serve/doc_code/gradio-original.py

@@ -0,0 +1,36 @@
+import gradio as gr
+from transformers import pipeline
+import requests
+
+# __doc_code_begin__
+generator1 = pipeline("text-generation", model="gpt2")
+generator2 = pipeline("text-generation", model="distilgpt2")
+
+
+def model1(text):
+    generated_list = generator1(text, do_sample=True, min_length=20, max_length=100)
+    generated = generated_list[0]["generated_text"]
+    return generated
+
+
+def model2(text):
+    generated_list = generator2(text, do_sample=True, min_length=20, max_length=100)
+    generated = generated_list[0]["generated_text"]
+    return generated
+
+
+demo = gr.Interface(
+    lambda text: f"{model1(text)}\n------------\n{model2(text)}",
+    "textbox",
+    "textbox",
+    api_name="predict",
+)
+# __doc_code_end__
+
+# Test example code
+demo.launch(prevent_thread_lock=True)
+response = requests.post(
+    "http://127.0.0.1:7860/gradio_api/run/predict/", json={"data": ["My name is Lewis"]}
+)
+assert response.status_code == 200
+print("gradio-original.py: Response from example code is", response.json()["data"])

doc/source/serve/doc_code/grpc_proxy/grpc_guide.py

@@ -0,0 +1,513 @@
+# flake8: noqa
+import ray
+
+
+ray.init()
+
+# __begin_start_grpc_proxy__
+from ray import serve
+from ray.serve.config import gRPCOptions
+
+
+grpc_port = 9000
+grpc_servicer_functions = [
+    "user_defined_protos_pb2_grpc.add_UserDefinedServiceServicer_to_server",
+    "user_defined_protos_pb2_grpc.add_ImageClassificationServiceServicer_to_server",
+]
+serve.start(
+    grpc_options=gRPCOptions(
+        port=grpc_port,
+        grpc_servicer_functions=grpc_servicer_functions,
+    ),
+)
+# __end_start_grpc_proxy__
+
+
+# __begin_grpc_deployment__
+import time
+
+from typing import Generator
+from user_defined_protos_pb2 import (
+    UserDefinedMessage,
+    UserDefinedMessage2,
+    UserDefinedResponse,
+    UserDefinedResponse2,
+)
+
+import ray
+from ray import serve
+
+
+@serve.deployment
+class GrpcDeployment:
+    def __call__(self, user_message: UserDefinedMessage) -> UserDefinedResponse:
+        greeting = f"Hello {user_message.name} from {user_message.origin}"
+        num = user_message.num * 2
+        user_response = UserDefinedResponse(
+            greeting=greeting,
+            num=num,
+        )
+        return user_response
+
+    @serve.multiplexed(max_num_models_per_replica=1)
+    async def get_model(self, model_id: str) -> str:
+        return f"loading model: {model_id}"
+
+    async def Multiplexing(
+        self, user_message: UserDefinedMessage2
+    ) -> UserDefinedResponse2:
+        model_id = serve.get_multiplexed_model_id()
+        model = await self.get_model(model_id)
+        user_response = UserDefinedResponse2(
+            greeting=f"Method2 called model, {model}",
+        )
+        return user_response
+
+    def Streaming(
+        self, user_message: UserDefinedMessage
+    ) -> Generator[UserDefinedResponse, None, None]:
+        for i in range(10):
+            greeting = f"{i}: Hello {user_message.name} from {user_message.origin}"
+            num = user_message.num * 2 + i
+            user_response = UserDefinedResponse(
+                greeting=greeting,
+                num=num,
+            )
+            yield user_response
+
+            time.sleep(0.1)
+
+
+g = GrpcDeployment.bind()
+# __end_grpc_deployment__
+
+
+# __begin_deploy_grpc_app__
+app1 = "app1"
+serve.run(target=g, name=app1, route_prefix=f"/{app1}")
+# __end_deploy_grpc_app__
+
+
+# __begin_send_grpc_requests__
+import grpc
+from user_defined_protos_pb2_grpc import UserDefinedServiceStub
+from user_defined_protos_pb2 import UserDefinedMessage
+
+
+channel = grpc.insecure_channel("localhost:9000")
+stub = UserDefinedServiceStub(channel)
+request = UserDefinedMessage(name="foo", num=30, origin="bar")
+
+response, call = stub.__call__.with_call(request=request)
+print(f"status code: {call.code()}")  # grpc.StatusCode.OK
+print(f"greeting: {response.greeting}")  # "Hello foo from bar"
+print(f"num: {response.num}")  # 60
+# __end_send_grpc_requests__
+
+
+# __begin_health_check__
+import grpc
+from ray.serve.generated.serve_pb2_grpc import RayServeAPIServiceStub
+from ray.serve.generated.serve_pb2 import HealthzRequest, ListApplicationsRequest
+
+
+channel = grpc.insecure_channel("localhost:9000")
+stub = RayServeAPIServiceStub(channel)
+request = ListApplicationsRequest()
+response = stub.ListApplications(request=request)
+print(f"Applications: {response.application_names}")  # ["app1"]
+
+request = HealthzRequest()
+response = stub.Healthz(request=request)
+print(f"Health: {response.message}")  # "success"
+# __end_health_check__
+
+
+# __begin_metadata__
+import grpc
+from user_defined_protos_pb2_grpc import UserDefinedServiceStub
+from user_defined_protos_pb2 import UserDefinedMessage2
+
+
+channel = grpc.insecure_channel("localhost:9000")
+stub = UserDefinedServiceStub(channel)
+request = UserDefinedMessage2()
+app_name = "app1"
+request_id = "123"
+multiplexed_model_id = "999"
+metadata = (
+    ("application", app_name),
+    ("request_id", request_id),
+    ("multiplexed_model_id", multiplexed_model_id),
+)
+
+response, call = stub.Multiplexing.with_call(request=request, metadata=metadata)
+print(f"greeting: {response.greeting}")  # "Method2 called model, loading model: 999"
+for key, value in call.trailing_metadata():
+    print(f"trailing metadata key: {key}, value {value}")  # "request_id: 123"
+# __end_metadata__
+
+
+# __begin_streaming__
+import grpc
+from user_defined_protos_pb2_grpc import UserDefinedServiceStub
+from user_defined_protos_pb2 import UserDefinedMessage
+
+
+channel = grpc.insecure_channel("localhost:9000")
+stub = UserDefinedServiceStub(channel)
+request = UserDefinedMessage(name="foo", num=30, origin="bar")
+metadata = (("application", "app1"),)
+
+responses = stub.Streaming(request=request, metadata=metadata)
+for response in responses:
+    print(f"greeting: {response.greeting}")  # greeting: n: Hello foo from bar
+    print(f"num: {response.num}")  # num: 60 + n
+# __end_streaming__
+
+
+# __begin_model_composition_deployment__
+import requests
+import torch
+from typing import List
+from PIL import Image
+from io import BytesIO
+from torchvision import transforms
+from torchvision.models import resnet18, ResNet18_Weights
+from user_defined_protos_pb2 import (
+    ImageClass,
+    ImageData,
+)
+
+from ray import serve
+from ray.serve.handle import DeploymentHandle
+
+
+@serve.deployment
+class ImageClassifier:
+    def __init__(
+        self,
+        _image_downloader: DeploymentHandle,
+        _data_preprocessor: DeploymentHandle,
+    ):
+        self._image_downloader = _image_downloader
+        self._data_preprocessor = _data_preprocessor
+        self.model = resnet18(weights=ResNet18_Weights.DEFAULT)
+        self.model.eval()
+        self.categories = self._image_labels()
+
+    def _image_labels(self) -> List[str]:
+        categories = []
+        url = (
+            "https://raw.githubusercontent.com/pytorch/hub/master/imagenet_classes.txt"
+        )
+        labels = requests.get(url).text
+        for label in labels.split("\n"):
+            categories.append(label.strip())
+        return categories
+
+    async def Predict(self, image_data: ImageData) -> ImageClass:
+        # Download image
+        image = await self._image_downloader.remote(image_data.url)
+
+        # Preprocess image
+        input_batch = await self._data_preprocessor.remote(image)
+        # Predict image
+        with torch.no_grad():
+            output = self.model(input_batch)
+
+        probabilities = torch.nn.functional.softmax(output[0], dim=0)
+        return self.process_model_outputs(probabilities)
+
+    def process_model_outputs(self, probabilities: torch.Tensor) -> ImageClass:
+        image_classes = []
+        image_probabilities = []
+        # Show top categories per image
+        top5_prob, top5_catid = torch.topk(probabilities, 5)
+        for i in range(top5_prob.size(0)):
+            image_classes.append(self.categories[top5_catid[i]])
+            image_probabilities.append(top5_prob[i].item())
+
+        return ImageClass(
+            classes=image_classes,
+            probabilities=image_probabilities,
+        )
+
+
+@serve.deployment
+class ImageDownloader:
+    def __call__(self, image_url: str):
+        image_bytes = requests.get(image_url).content
+        return Image.open(BytesIO(image_bytes)).convert("RGB")
+
+
+@serve.deployment
+class DataPreprocessor:
+    def __init__(self):
+        self.preprocess = transforms.Compose(
+            [
+                transforms.Resize(256),
+                transforms.CenterCrop(224),
+                transforms.ToTensor(),
+                transforms.Normalize(
+                    mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]
+                ),
+            ]
+        )
+
+    def __call__(self, image: Image):
+        input_tensor = self.preprocess(image)
+        return input_tensor.unsqueeze(0)  # create a mini-batch as expected by the model
+
+
+image_downloader = ImageDownloader.bind()
+data_preprocessor = DataPreprocessor.bind()
+g2 = ImageClassifier.options(name="grpc-image-classifier").bind(
+    image_downloader, data_preprocessor
+)
+# __end_model_composition_deployment__
+
+
+# __begin_model_composition_deploy__
+app2 = "app2"
+serve.run(target=g2, name=app2, route_prefix=f"/{app2}")
+# __end_model_composition_deploy__
+
+
+# __begin_model_composition_client__
+import grpc
+from user_defined_protos_pb2_grpc import ImageClassificationServiceStub
+from user_defined_protos_pb2 import ImageData
+
+
+channel = grpc.insecure_channel("localhost:9000")
+stub = ImageClassificationServiceStub(channel)
+request = ImageData(url="https://github.com/pytorch/hub/raw/master/images/dog.jpg")
+metadata = (("application", "app2"),)  # Make sure application metadata is passed.
+
+response, call = stub.Predict.with_call(request=request, metadata=metadata)
+print(f"status code: {call.code()}")  # grpc.StatusCode.OK
+print(f"Classes: {response.classes}")  # ['Samoyed', ...]
+print(f"Probabilities: {response.probabilities}")  # [0.8846230506896973, ...]
+# __end_model_composition_client__
+
+
+# __begin_error_handle__
+import grpc
+from user_defined_protos_pb2_grpc import UserDefinedServiceStub
+from user_defined_protos_pb2 import UserDefinedMessage
+
+
+channel = grpc.insecure_channel("localhost:9000")
+stub = UserDefinedServiceStub(channel)
+request = UserDefinedMessage(name="foo", num=30, origin="bar")
+
+try:
+    response = stub.__call__(request=request)
+except grpc.RpcError as rpc_error:
+    print(f"status code: {rpc_error.code()}")  # StatusCode.NOT_FOUND
+    print(f"details: {rpc_error.details()}")  # Application metadata not set...
+# __end_error_handle__
+
+
+# __begin_grpc_context_define_app__
+from user_defined_protos_pb2 import UserDefinedMessage, UserDefinedResponse
+
+from ray import serve
+from ray.serve.grpc_util import RayServegRPCContext
+
+import grpc
+from typing import Tuple
+
+
+@serve.deployment
+class GrpcDeployment:
+    def __init__(self):
+        self.nums = {}
+
+    def num_lookup(self, name: str) -> Tuple[int, grpc.StatusCode, str]:
+        if name not in self.nums:
+            self.nums[name] = len(self.nums)
+            code = grpc.StatusCode.INVALID_ARGUMENT
+            message = f"{name} not found, adding to nums."
+        else:
+            code = grpc.StatusCode.OK
+            message = f"{name} found."
+        return self.nums[name], code, message
+
+    def __call__(
+        self,
+        user_message: UserDefinedMessage,
+        grpc_context: RayServegRPCContext,  # to use grpc context, add this kwarg
+    ) -> UserDefinedResponse:
+        greeting = f"Hello {user_message.name} from {user_message.origin}"
+        num, code, message = self.num_lookup(user_message.name)
+
+        # Set custom code, details, and trailing metadata.
+        grpc_context.set_code(code)
+        grpc_context.set_details(message)
+        grpc_context.set_trailing_metadata([("num", str(num))])
+
+        # You can also set a status code before raising an exception.
+        # The status code will be preserved in the response.
+        if user_message.name == "error":
+            grpc_context.set_code(grpc.StatusCode.RESOURCE_EXHAUSTED)
+            grpc_context.set_details("Resource exhausted, please retry later.")
+            raise RuntimeError("Simulated error")
+
+        user_response = UserDefinedResponse(
+            greeting=greeting,
+            num=num,
+        )
+        return user_response
+
+
+g = GrpcDeployment.bind()
+app1 = "app1"
+serve.run(target=g, name=app1, route_prefix=f"/{app1}")
+# __end_grpc_context_define_app__
+
+
+# __begin_grpc_context_client__
+import grpc
+from user_defined_protos_pb2_grpc import UserDefinedServiceStub
+from user_defined_protos_pb2 import UserDefinedMessage
+
+
+channel = grpc.insecure_channel("localhost:9000")
+stub = UserDefinedServiceStub(channel)
+request = UserDefinedMessage(name="foo", num=30, origin="bar")
+metadata = (("application", "app1"),)
+
+# First call is going to page miss and return INVALID_ARGUMENT status code.
+try:
+    response, call = stub.__call__.with_call(request=request, metadata=metadata)
+except grpc.RpcError as rpc_error:
+    assert rpc_error.code() == grpc.StatusCode.INVALID_ARGUMENT
+    assert rpc_error.details() == "foo not found, adding to nums."
+    assert any(
+        [key == "num" and value == "0" for key, value in rpc_error.trailing_metadata()]
+    )
+    assert any([key == "request_id" for key, _ in rpc_error.trailing_metadata()])
+
+# Second call is going to page hit and return OK status code.
+response, call = stub.__call__.with_call(request=request, metadata=metadata)
+assert call.code() == grpc.StatusCode.OK
+assert call.details() == "foo found."
+assert any([key == "num" and value == "0" for key, value in call.trailing_metadata()])
+assert any([key == "request_id" for key, _ in call.trailing_metadata()])
+# __end_grpc_context_client__
+
+
+# __begin_client_streaming_deployment__
+from ray import serve
+from ray.serve.grpc_util import gRPCInputStream
+from user_defined_protos_pb2 import UserDefinedResponse
+
+
+@serve.deployment
+class ClientStreamingService:
+    async def ClientStreaming(self, request_stream: gRPCInputStream):
+        """Receives stream of requests, returns a single response."""
+        total = 0
+        count = 0
+        async for request in request_stream:
+            total += request.num
+            count += 1
+        return UserDefinedResponse(
+            greeting=f"Received {count} messages",
+            num=total * 2,
+        )
+
+
+serve.run(ClientStreamingService.bind())
+# __end_client_streaming_deployment__
+
+
+# __begin_client_streaming_client__
+import grpc
+from user_defined_protos_pb2_grpc import UserDefinedServiceStub
+from user_defined_protos_pb2 import UserDefinedMessage
+
+
+channel = grpc.insecure_channel("localhost:9000")
+stub = UserDefinedServiceStub(channel)
+metadata = (("application", "default"),)
+
+
+def request_generator():
+    for i in range(5):
+        yield UserDefinedMessage(name=f"msg_{i}", num=i + 1, origin="client")
+
+
+response = stub.ClientStreaming(request_generator(), metadata=metadata)
+print(f"greeting: {response.greeting}")  # greeting: Received 5 messages
+print(f"num: {response.num}")  # num: 30
+# __end_client_streaming_client__
+
+
+# __begin_bidi_streaming_deployment__
+from ray import serve
+from ray.serve.grpc_util import gRPCInputStream
+from user_defined_protos_pb2 import UserDefinedResponse
+
+
+@serve.deployment
+class BidiStreamingService:
+    async def BidiStreaming(self, request_stream: gRPCInputStream):
+        """Receives stream of requests, yields response for each."""
+        async for request in request_stream:
+            yield UserDefinedResponse(
+                greeting=f"Hello {request.name}",
+                num=request.num * 2,
+            )
+
+
+serve.run(BidiStreamingService.bind())
+# __end_bidi_streaming_deployment__
+
+
+# __begin_bidi_streaming_client__
+import grpc
+from user_defined_protos_pb2_grpc import UserDefinedServiceStub
+from user_defined_protos_pb2 import UserDefinedMessage
+
+
+channel = grpc.insecure_channel("localhost:9000")
+stub = UserDefinedServiceStub(channel)
+metadata = (("application", "default"),)
+
+
+def request_generator():
+    for i in range(3):
+        yield UserDefinedMessage(name=f"user_{i}", num=i * 10, origin="client")
+
+
+responses = stub.BidiStreaming(request_generator(), metadata=metadata)
+for response in responses:
+    print(f"greeting: {response.greeting}")
+    print(f"num: {response.num}")
+# __end_bidi_streaming_client__
+
+
+# __begin_streaming_with_context__
+from ray import serve
+from ray.serve.grpc_util import gRPCInputStream, RayServegRPCContext
+from user_defined_protos_pb2 import UserDefinedResponse
+
+
+@serve.deployment
+class StreamingWithContext:
+    async def ClientStreaming(
+        self,
+        request_stream: gRPCInputStream,
+        grpc_context: RayServegRPCContext,
+    ):
+        """Receives stream and can modify gRPC context."""
+        count = 0
+        async for request in request_stream:
+            count += 1
+
+        grpc_context.set_trailing_metadata([("processed-count", str(count))])
+        return UserDefinedResponse(greeting=f"Processed {count} messages")
+# __end_streaming_with_context__

doc/source/serve/doc_code/grpc_proxy/user_defined_protos.proto

@@ -0,0 +1,51 @@
+// __begin_proto__
+// user_defined_protos.proto
+
+syntax = "proto3";
+
+option java_multiple_files = true;
+option java_package = "io.ray.examples.user_defined_protos";
+option java_outer_classname = "UserDefinedProtos";
+
+package userdefinedprotos;
+
+message UserDefinedMessage {
+  string name = 1;
+  string origin = 2;
+  int64 num = 3;
+}
+
+message UserDefinedResponse {
+  string greeting = 1;
+  int64 num = 2;
+}
+
+message UserDefinedMessage2 {}
+
+message UserDefinedResponse2 {
+  string greeting = 1;
+}
+
+message ImageData {
+  string url = 1;
+  string filename = 2;
+}
+
+message ImageClass {
+  repeated string classes = 1;
+  repeated float probabilities = 2;
+}
+
+service UserDefinedService {
+  rpc __call__(UserDefinedMessage) returns (UserDefinedResponse);
+  rpc Multiplexing(UserDefinedMessage2) returns (UserDefinedResponse2);
+  rpc Streaming(UserDefinedMessage) returns (stream UserDefinedResponse);
+  rpc ClientStreaming(stream UserDefinedMessage) returns (UserDefinedResponse);
+  rpc BidiStreaming(stream UserDefinedMessage) returns (stream UserDefinedResponse);
+}
+
+service ImageClassificationService {
+  rpc Predict(ImageData) returns (ImageClass);
+}
+
+// __end_proto__

doc/source/serve/doc_code/grpc_proxy/user_defined_protos_pb2_grpc.py

@@ -0,0 +1,259 @@
+# Generated by the gRPC Python protocol compiler plugin. DO NOT EDIT!
+"""Client and server classes corresponding to protobuf-defined services."""
+import grpc
+
+import user_defined_protos_pb2 as user__defined__protos__pb2
+
+
+class UserDefinedServiceStub(object):
+    """Missing associated documentation comment in .proto file."""
+
+    def __init__(self, channel):
+        """Constructor.
+
+        Args:
+            channel: A grpc.Channel.
+        """
+        self.__call__ = channel.unary_unary(
+                '/userdefinedprotos.UserDefinedService/__call__',
+                request_serializer=user__defined__protos__pb2.UserDefinedMessage.SerializeToString,
+                response_deserializer=user__defined__protos__pb2.UserDefinedResponse.FromString,
+                )
+        self.Multiplexing = channel.unary_unary(
+                '/userdefinedprotos.UserDefinedService/Multiplexing',
+                request_serializer=user__defined__protos__pb2.UserDefinedMessage2.SerializeToString,
+                response_deserializer=user__defined__protos__pb2.UserDefinedResponse2.FromString,
+                )
+        self.Streaming = channel.unary_stream(
+                '/userdefinedprotos.UserDefinedService/Streaming',
+                request_serializer=user__defined__protos__pb2.UserDefinedMessage.SerializeToString,
+                response_deserializer=user__defined__protos__pb2.UserDefinedResponse.FromString,
+                )
+        self.ClientStreaming = channel.stream_unary(
+                '/userdefinedprotos.UserDefinedService/ClientStreaming',
+                request_serializer=user__defined__protos__pb2.UserDefinedMessage.SerializeToString,
+                response_deserializer=user__defined__protos__pb2.UserDefinedResponse.FromString,
+                )
+        self.BidiStreaming = channel.stream_stream(
+                '/userdefinedprotos.UserDefinedService/BidiStreaming',
+                request_serializer=user__defined__protos__pb2.UserDefinedMessage.SerializeToString,
+                response_deserializer=user__defined__protos__pb2.UserDefinedResponse.FromString,
+                )
+
+
+class UserDefinedServiceServicer(object):
+    """Missing associated documentation comment in .proto file."""
+
+    def __call__(self, request, context):
+        """Missing associated documentation comment in .proto file."""
+        context.set_code(grpc.StatusCode.UNIMPLEMENTED)
+        context.set_details('Method not implemented!')
+        raise NotImplementedError('Method not implemented!')
+
+    def Multiplexing(self, request, context):
+        """Missing associated documentation comment in .proto file."""
+        context.set_code(grpc.StatusCode.UNIMPLEMENTED)
+        context.set_details('Method not implemented!')
+        raise NotImplementedError('Method not implemented!')
+
+    def Streaming(self, request, context):
+        """Missing associated documentation comment in .proto file."""
+        context.set_code(grpc.StatusCode.UNIMPLEMENTED)
+        context.set_details('Method not implemented!')
+        raise NotImplementedError('Method not implemented!')
+
+    def ClientStreaming(self, request_iterator, context):
+        """Missing associated documentation comment in .proto file."""
+        context.set_code(grpc.StatusCode.UNIMPLEMENTED)
+        context.set_details('Method not implemented!')
+        raise NotImplementedError('Method not implemented!')
+
+    def BidiStreaming(self, request_iterator, context):
+        """Missing associated documentation comment in .proto file."""
+        context.set_code(grpc.StatusCode.UNIMPLEMENTED)
+        context.set_details('Method not implemented!')
+        raise NotImplementedError('Method not implemented!')
+
+
+def add_UserDefinedServiceServicer_to_server(servicer, server):
+    rpc_method_handlers = {
+            '__call__': grpc.unary_unary_rpc_method_handler(
+                    servicer.__call__,
+                    request_deserializer=user__defined__protos__pb2.UserDefinedMessage.FromString,
+                    response_serializer=user__defined__protos__pb2.UserDefinedResponse.SerializeToString,
+            ),
+            'Multiplexing': grpc.unary_unary_rpc_method_handler(
+                    servicer.Multiplexing,
+                    request_deserializer=user__defined__protos__pb2.UserDefinedMessage2.FromString,
+                    response_serializer=user__defined__protos__pb2.UserDefinedResponse2.SerializeToString,
+            ),
+            'Streaming': grpc.unary_stream_rpc_method_handler(
+                    servicer.Streaming,
+                    request_deserializer=user__defined__protos__pb2.UserDefinedMessage.FromString,
+                    response_serializer=user__defined__protos__pb2.UserDefinedResponse.SerializeToString,
+            ),
+            'ClientStreaming': grpc.stream_unary_rpc_method_handler(
+                    servicer.ClientStreaming,
+                    request_deserializer=user__defined__protos__pb2.UserDefinedMessage.FromString,
+                    response_serializer=user__defined__protos__pb2.UserDefinedResponse.SerializeToString,
+            ),
+            'BidiStreaming': grpc.stream_stream_rpc_method_handler(
+                    servicer.BidiStreaming,
+                    request_deserializer=user__defined__protos__pb2.UserDefinedMessage.FromString,
+                    response_serializer=user__defined__protos__pb2.UserDefinedResponse.SerializeToString,
+            ),
+    }
+    generic_handler = grpc.method_handlers_generic_handler(
+            'userdefinedprotos.UserDefinedService', rpc_method_handlers)
+    server.add_generic_rpc_handlers((generic_handler,))
+
+
+ # This class is part of an EXPERIMENTAL API.
+class UserDefinedService(object):
+    """Missing associated documentation comment in .proto file."""
+
+    @staticmethod
+    def __call__(request,
+            target,
+            options=(),
+            channel_credentials=None,
+            call_credentials=None,
+            insecure=False,
+            compression=None,
+            wait_for_ready=None,
+            timeout=None,
+            metadata=None):
+        return grpc.experimental.unary_unary(request, target, '/userdefinedprotos.UserDefinedService/__call__',
+            user__defined__protos__pb2.UserDefinedMessage.SerializeToString,
+            user__defined__protos__pb2.UserDefinedResponse.FromString,
+            options, channel_credentials,
+            insecure, call_credentials, compression, wait_for_ready, timeout, metadata)
+
+    @staticmethod
+    def Multiplexing(request,
+            target,
+            options=(),
+            channel_credentials=None,
+            call_credentials=None,
+            insecure=False,
+            compression=None,
+            wait_for_ready=None,
+            timeout=None,
+            metadata=None):
+        return grpc.experimental.unary_unary(request, target, '/userdefinedprotos.UserDefinedService/Multiplexing',
+            user__defined__protos__pb2.UserDefinedMessage2.SerializeToString,
+            user__defined__protos__pb2.UserDefinedResponse2.FromString,
+            options, channel_credentials,
+            insecure, call_credentials, compression, wait_for_ready, timeout, metadata)
+
+    @staticmethod
+    def Streaming(request,
+            target,
+            options=(),
+            channel_credentials=None,
+            call_credentials=None,
+            insecure=False,
+            compression=None,
+            wait_for_ready=None,
+            timeout=None,
+            metadata=None):
+        return grpc.experimental.unary_stream(request, target, '/userdefinedprotos.UserDefinedService/Streaming',
+            user__defined__protos__pb2.UserDefinedMessage.SerializeToString,
+            user__defined__protos__pb2.UserDefinedResponse.FromString,
+            options, channel_credentials,
+            insecure, call_credentials, compression, wait_for_ready, timeout, metadata)
+
+    @staticmethod
+    def ClientStreaming(request_iterator,
+            target,
+            options=(),
+            channel_credentials=None,
+            call_credentials=None,
+            insecure=False,
+            compression=None,
+            wait_for_ready=None,
+            timeout=None,
+            metadata=None):
+        return grpc.experimental.stream_unary(request_iterator, target, '/userdefinedprotos.UserDefinedService/ClientStreaming',
+            user__defined__protos__pb2.UserDefinedMessage.SerializeToString,
+            user__defined__protos__pb2.UserDefinedResponse.FromString,
+            options, channel_credentials,
+            insecure, call_credentials, compression, wait_for_ready, timeout, metadata)
+
+    @staticmethod
+    def BidiStreaming(request_iterator,
+            target,
+            options=(),
+            channel_credentials=None,
+            call_credentials=None,
+            insecure=False,
+            compression=None,
+            wait_for_ready=None,
+            timeout=None,
+            metadata=None):
+        return grpc.experimental.stream_stream(request_iterator, target, '/userdefinedprotos.UserDefinedService/BidiStreaming',
+            user__defined__protos__pb2.UserDefinedMessage.SerializeToString,
+            user__defined__protos__pb2.UserDefinedResponse.FromString,
+            options, channel_credentials,
+            insecure, call_credentials, compression, wait_for_ready, timeout, metadata)
+
+
+class ImageClassificationServiceStub(object):
+    """Missing associated documentation comment in .proto file."""
+
+    def __init__(self, channel):
+        """Constructor.
+
+        Args:
+            channel: A grpc.Channel.
+        """
+        self.Predict = channel.unary_unary(
+                '/userdefinedprotos.ImageClassificationService/Predict',
+                request_serializer=user__defined__protos__pb2.ImageData.SerializeToString,
+                response_deserializer=user__defined__protos__pb2.ImageClass.FromString,
+                )
+
+
+class ImageClassificationServiceServicer(object):
+    """Missing associated documentation comment in .proto file."""
+
+    def Predict(self, request, context):
+        """Missing associated documentation comment in .proto file."""
+        context.set_code(grpc.StatusCode.UNIMPLEMENTED)
+        context.set_details('Method not implemented!')
+        raise NotImplementedError('Method not implemented!')
+
+
+def add_ImageClassificationServiceServicer_to_server(servicer, server):
+    rpc_method_handlers = {
+            'Predict': grpc.unary_unary_rpc_method_handler(
+                    servicer.Predict,
+                    request_deserializer=user__defined__protos__pb2.ImageData.FromString,
+                    response_serializer=user__defined__protos__pb2.ImageClass.SerializeToString,
+            ),
+    }
+    generic_handler = grpc.method_handlers_generic_handler(
+            'userdefinedprotos.ImageClassificationService', rpc_method_handlers)
+    server.add_generic_rpc_handlers((generic_handler,))
+
+
+ # This class is part of an EXPERIMENTAL API.
+class ImageClassificationService(object):
+    """Missing associated documentation comment in .proto file."""
+
+    @staticmethod
+    def Predict(request,
+            target,
+            options=(),
+            channel_credentials=None,
+            call_credentials=None,
+            insecure=False,
+            compression=None,
+            wait_for_ready=None,
+            timeout=None,
+            metadata=None):
+        return grpc.experimental.unary_unary(request, target, '/userdefinedprotos.ImageClassificationService/Predict',
+            user__defined__protos__pb2.ImageData.SerializeToString,
+            user__defined__protos__pb2.ImageClass.FromString,
+            options, channel_credentials,
+            insecure, call_credentials, compression, wait_for_ready, timeout, metadata)

doc/source/serve/doc_code/http_guide/disconnects.py

@@ -0,0 +1,121 @@
+# flake8: noqa
+# fmt: off
+
+import ray
+import sys
+from typing import List
+
+from ray._common.test_utils import wait_for_condition
+
+
+# Overwrite print statement to make doc code testable
+@ray.remote
+class PrintStorage:
+    def __init__(self):
+        self.print_storage: List[str] = []
+
+    def add(self, s: str):
+        self.print_storage.append(s)
+
+    def clear(self):
+        self.print_storage.clear()
+
+    def get(self) -> List[str]:
+        return self.print_storage
+
+
+print_storage_handle = PrintStorage.remote()
+
+
+def print(string: str):
+    ray.get(print_storage_handle.add.remote(string))
+    sys.stdout.write(f"{string}\n")
+
+
+# __start_basic_disconnect__
+import asyncio
+from ray import serve
+
+
+@serve.deployment
+async def startled():
+    try:
+        print("Replica received request!")
+        await asyncio.sleep(10000)
+    except asyncio.CancelledError:
+        # Add custom behavior that should run
+        # upon cancellation here.
+        print("Request got cancelled!")
+# __end_basic_disconnect__
+
+serve.run(startled.bind())
+
+import requests
+from requests.exceptions import Timeout
+
+# Intentionally time out request to test cancellation behavior
+try:
+    requests.get("http://localhost:8000", timeout=0.5)
+except Timeout:
+    pass
+
+wait_for_condition(
+    lambda: {"Replica received request!", "Request got cancelled!"}
+    == set(ray.get(print_storage_handle.get.remote())),
+    timeout=5,
+)
+
+sys.stdout.write(f"{ray.get(print_storage_handle.get.remote())}\n")
+
+ray.get(print_storage_handle.clear.remote())
+
+
+# __start_shielded_disconnect__
+import asyncio
+from ray import serve
+
+
+@serve.deployment
+class SnoringSleeper:
+    async def snore(self):
+        await asyncio.sleep(1)
+        print("ZZZ")
+
+    async def __call__(self):
+        try:
+            print("SnoringSleeper received request!")
+
+            # Prevent the snore() method from being cancelled
+            await asyncio.shield(self.snore())
+
+        except asyncio.CancelledError:
+            print("SnoringSleeper's request was cancelled!")
+
+
+app = SnoringSleeper.bind()
+# __end_shielded_disconnect__
+
+serve.run(app)
+
+import requests
+from requests.exceptions import Timeout
+
+# Intentionally time out request to test cancellation behavior
+try:
+    requests.get("http://localhost:8000", timeout=0.5)
+except Timeout:
+    pass
+
+wait_for_condition(
+    lambda: {
+        "SnoringSleeper received request!",
+        "SnoringSleeper's request was cancelled!",
+        "ZZZ",
+    }
+    == set(ray.get(print_storage_handle.get.remote())),
+    timeout=5,
+)
+
+sys.stdout.write(f"{ray.get(print_storage_handle.get.remote())}\n")
+
+ray.get(print_storage_handle.clear.remote())

doc/source/serve/doc_code/http_guide/http_guide.py

@@ -0,0 +1,176 @@
+# flake8: noqa
+
+# __begin_starlette__
+import starlette.requests
+import requests
+from ray import serve
+
+
+@serve.deployment
+class Counter:
+    def __call__(self, request: starlette.requests.Request):
+        return request.query_params
+
+
+serve.run(Counter.bind())
+resp = requests.get("http://localhost:8000?a=b&c=d")
+assert resp.json() == {"a": "b", "c": "d"}
+# __end_starlette__
+
+# __begin_fastapi__
+import ray
+import requests
+from fastapi import FastAPI
+from ray import serve
+
+app = FastAPI()
+
+
+@serve.deployment
+@serve.ingress(app)
+class MyFastAPIDeployment:
+    @app.get("/")
+    def root(self):
+        return "Hello, world!"
+
+
+serve.run(MyFastAPIDeployment.bind(), route_prefix="/hello")
+resp = requests.get("http://localhost:8000/hello")
+assert resp.json() == "Hello, world!"
+# __end_fastapi__
+
+
+# __begin_fastapi_multi_routes__
+import ray
+import requests
+from fastapi import FastAPI
+from ray import serve
+
+app = FastAPI()
+
+
+@serve.deployment
+@serve.ingress(app)
+class MyFastAPIDeployment:
+    @app.get("/")
+    def root(self):
+        return "Hello, world!"
+
+    @app.post("/{subpath}")
+    def root(self, subpath: str):
+        return f"Hello from {subpath}!"
+
+
+serve.run(MyFastAPIDeployment.bind(), route_prefix="/hello")
+resp = requests.post("http://localhost:8000/hello/Serve")
+assert resp.json() == "Hello from Serve!"
+# __end_fastapi_multi_routes__
+
+# __begin_byo_fastapi__
+import ray
+import requests
+from fastapi import FastAPI
+from ray import serve
+
+app = FastAPI()
+
+
+@app.get("/")
+def f():
+    return "Hello from the root!"
+
+
+@serve.deployment
+@serve.ingress(app)
+class FastAPIWrapper:
+    pass
+
+
+serve.run(FastAPIWrapper.bind(), route_prefix="/")
+resp = requests.get("http://localhost:8000/")
+assert resp.json() == "Hello from the root!"
+# __end_byo_fastapi__
+
+# __begin_fastapi_middleware__
+import requests
+from fastapi import FastAPI
+from fastapi.middleware.cors import CORSMiddleware
+from ray import serve
+
+app = FastAPI()
+app.add_middleware(
+    CORSMiddleware,
+    allow_origins=["https://example.com"],
+    allow_methods=["GET", "POST"],
+)
+
+
+@serve.deployment
+@serve.ingress(app)
+class Ingress:
+    @app.get("/")
+    def root(self):
+        return "ok"
+
+
+serve.run(Ingress.bind())
+resp = requests.get(
+    "http://localhost:8000/",
+    headers={"Origin": "https://example.com"},
+)
+assert resp.json() == "ok"
+assert resp.headers["access-control-allow-origin"] == "https://example.com"
+# __end_fastapi_middleware__
+
+
+# __begin_fastapi_factory_pattern__
+import requests
+from fastapi import FastAPI
+from ray import serve
+from opentelemetry.instrumentation.fastapi import FastAPIInstrumentor
+
+
+@serve.deployment
+class ChildDeployment:
+    def __call__(self):
+        return "Hello from the child deployment!"
+
+
+def fastapi_factory():
+    """Factory-style FastAPI app used as Serve ingress.
+
+    We build the FastAPI app inside a factory and pass the callable to
+    @serve.ingress.
+    """
+    app = FastAPI()
+
+    # In an object-based ingress (where the FastAPI app is stored on the
+    # deployment instance), Ray would need to serialize the app and its
+    # instrumentation. Some instrumentors (like FastAPIInstrumentor) are not
+    # picklable, which can cause serialization failures. Creating and
+    # instrumenting the app here sidesteps that issue.
+    FastAPIInstrumentor.instrument_app(app)
+
+    @app.get("/")
+    async def root():
+        # Handlers defined inside this factory don't have access to the
+        # ParentDeployment instance (i.e., there's no `self` here), so we
+        # can't call `self.child`. Instead, fetch a handle by deployment name.
+        handle = serve.get_deployment_handle("ChildDeployment", app_name="default")
+        return {"message": await handle.remote()}
+
+    return app
+
+
+@serve.deployment
+@serve.ingress(fastapi_factory)
+class ParentDeployment:
+    def __init__(self, child):
+        self.child = child
+
+
+serve.run(ParentDeployment.bind(ChildDeployment.bind()))
+
+resp = requests.get("http://localhost:8000/")
+assert resp.json() == {"message": "Hello from the child deployment!"}
+# __end_fastapi_factory_pattern__

doc/source/serve/doc_code/http_guide/streaming_example.py

@@ -0,0 +1,82 @@
+# flake8: noqa
+
+# __begin_example__
+import time
+from typing import Generator
+
+import requests
+from starlette.responses import StreamingResponse
+from starlette.requests import Request
+
+from ray import serve
+
+
+@serve.deployment
+class StreamingResponder:
+    def generate_numbers(self, max: int) -> Generator[str, None, None]:
+        for i in range(max):
+            yield str(i)
+            time.sleep(0.1)
+
+    def __call__(self, request: Request) -> StreamingResponse:
+        max = request.query_params.get("max", "25")
+        gen = self.generate_numbers(int(max))
+        return StreamingResponse(gen, status_code=200, media_type="text/plain")
+
+
+serve.run(StreamingResponder.bind())
+
+r = requests.get("http://localhost:8000?max=10", stream=True)
+start = time.time()
+r.raise_for_status()
+for chunk in r.iter_content(chunk_size=None, decode_unicode=True):
+    print(f"Got result {round(time.time()-start, 1)}s after start: '{chunk}'")
+# __end_example__
+
+
+r = requests.get("http://localhost:8000?max=10", stream=True)
+r.raise_for_status()
+for i, chunk in enumerate(r.iter_content(chunk_size=None, decode_unicode=True)):
+    assert chunk == str(i)
+
+
+# __begin_cancellation__
+import asyncio
+import time
+from typing import AsyncGenerator
+
+import requests
+from starlette.responses import StreamingResponse
+from starlette.requests import Request
+
+from ray import serve
+
+
+@serve.deployment
+class StreamingResponder:
+    async def generate_forever(self) -> AsyncGenerator[str, None]:
+        try:
+            i = 0
+            while True:
+                yield str(i)
+                i += 1
+                await asyncio.sleep(0.1)
+        except asyncio.CancelledError:
+            print("Cancelled! Exiting.")
+
+    def __call__(self, request: Request) -> StreamingResponse:
+        gen = self.generate_forever()
+        return StreamingResponse(gen, status_code=200, media_type="text/plain")
+
+
+serve.run(StreamingResponder.bind())
+
+r = requests.get("http://localhost:8000?max=10", stream=True)
+start = time.time()
+r.raise_for_status()
+for i, chunk in enumerate(r.iter_content(chunk_size=None, decode_unicode=True)):
+    print(f"Got result {round(time.time()-start, 1)}s after start: '{chunk}'")
+    if i == 10:
+        print("Client disconnecting")
+        break
+# __end_cancellation__

doc/source/serve/doc_code/http_guide/websockets_example.py

@@ -0,0 +1,39 @@
+# flake8: noqa
+
+# __websocket_serve_app_start__
+from fastapi import FastAPI, WebSocket, WebSocketDisconnect
+
+from ray import serve
+
+
+app = FastAPI()
+
+
+@serve.deployment
+@serve.ingress(app)
+class EchoServer:
+    @app.websocket("/")
+    async def echo(self, ws: WebSocket):
+        await ws.accept()
+
+        try:
+            while True:
+                text = await ws.receive_text()
+                await ws.send_text(text)
+        except WebSocketDisconnect:
+            print("Client disconnected.")
+
+
+serve_app = serve.run(EchoServer.bind())
+# __websocket_serve_app_end__
+
+# __websocket_serve_client_start__
+from websockets.sync.client import connect
+
+with connect("ws://localhost:8000") as websocket:
+    websocket.send("Eureka!")
+    assert websocket.recv() == "Eureka!"
+
+    websocket.send("I've found it!")
+    assert websocket.recv() == "I've found it!"
+# __websocket_serve_client_end__

doc/source/serve/doc_code/image_classifier_example.py

@@ -0,0 +1,98 @@
+# __serve_example_begin__
+import requests
+import starlette
+
+from transformers import pipeline
+from io import BytesIO
+from PIL import Image
+
+from ray import serve
+from ray.serve.handle import DeploymentHandle
+
+
+@serve.deployment
+def downloader(image_url: str):
+    image_bytes = requests.get(image_url).content
+    image = Image.open(BytesIO(image_bytes)).convert("RGB")
+    return image
+
+
+@serve.deployment
+class ImageClassifier:
+    def __init__(self, downloader: DeploymentHandle):
+        self.downloader = downloader
+        self.model = pipeline(
+            "image-classification", model="google/vit-base-patch16-224"
+        )
+
+    async def classify(self, image_url: str) -> str:
+        image = await self.downloader.remote(image_url)
+        results = self.model(image)
+        return results[0]["label"]
+
+    async def __call__(self, req: starlette.requests.Request):
+        req = await req.json()
+        return await self.classify(req["image_url"])
+
+
+app = ImageClassifier.bind(downloader.bind())
+# __serve_example_end__
+
+
+@serve.deployment
+class ModifiedImageClassifier:
+    def __init__(self, downloader: DeploymentHandle):
+        self.downloader = downloader
+        self.model = pipeline(
+            "image-classification", model="google/vit-base-patch16-224"
+        )
+
+    async def classify(self, image_url: str) -> str:
+        image = await self.downloader.remote(image_url)
+        results = self.model(image)
+        return results[0]["label"]
+
+    # __serve_example_modified_begin__
+    async def __call__(self, req: starlette.requests.Request):
+        req = await req.json()
+        result = await self.classify(req["image_url"])
+
+        if req.get("should_translate") is True:
+            handle: DeploymentHandle = serve.get_app_handle("app2")
+            return await handle.translate.remote(result)
+
+        return result
+        # __serve_example_modified_end__
+
+
+serve.run(app, name="app1", route_prefix="/classify")
+# __request_begin__
+bear_url = "https://cdn.britannica.com/41/156441-050-A4424AEC/Grizzly-bear-Jasper-National-Park-Canada-Alberta.jpg"  # noqa
+resp = requests.post("http://localhost:8000/classify", json={"image_url": bear_url})
+
+print(resp.text)
+# 'brown bear, bruin, Ursus arctos'
+# __request_end__
+assert resp.text == "brown bear, bruin, Ursus arctos"
+
+from translator_example import app as translator_app  # noqa
+
+serve.run(
+    ModifiedImageClassifier.bind(downloader.bind()),
+    name="app1",
+    route_prefix="/classify",
+)
+serve.run(translator_app, name="app2")
+
+# __second_request_begin__
+bear_url = "https://cdn.britannica.com/41/156441-050-A4424AEC/Grizzly-bear-Jasper-National-Park-Canada-Alberta.jpg"  # noqa
+resp = requests.post(
+    "http://localhost:8000/classify",
+    json={"image_url": bear_url, "should_translate": True},
+)
+
+print(resp.text)
+# 'Braunbär, Bruin, Ursus arctos'
+# __second_request_end__
+
+assert resp.text == "Braunbär, Bruin, Ursus arctos"

doc/source/serve/doc_code/intel_gaudi_inference_client.py

@@ -0,0 +1,23 @@
+# __main_code_start__
+import requests
+
+# Prompt for the model
+prompt = "Once upon a time,"
+
+# Add generation config here
+config = {}
+
+# Non-streaming response
+sample_input = {"text": prompt, "config": config, "stream": False}
+outputs = requests.post("http://127.0.0.1:8000/", json=sample_input, stream=False)
+print(outputs.text, flush=True)
+
+# Streaming response
+sample_input["stream"] = True
+outputs = requests.post("http://127.0.0.1:8000/", json=sample_input, stream=True)
+outputs.raise_for_status()
+for output in outputs.iter_content(chunk_size=None, decode_unicode=True):
+    print(output, end="", flush=True)
+print()
+
+# __main_code_end__

doc/source/serve/doc_code/intel_gaudi_inference_serve.py

@@ -0,0 +1,137 @@
+# __model_def_start__
+import asyncio
+from functools import partial
+from queue import Empty
+from typing import Dict, Any
+
+from starlette.requests import Request
+from starlette.responses import StreamingResponse
+import torch
+
+from ray import serve
+
+
+# Define the Ray Serve deployment
+@serve.deployment(ray_actor_options={"num_cpus": 10, "resources": {"HPU": 1}})
+class LlamaModel:
+    def __init__(self, model_id_or_path: str):
+        from transformers import AutoTokenizer, AutoModelForCausalLM, AutoConfig
+        from optimum.habana.transformers.modeling_utils import (
+            adapt_transformers_to_gaudi,
+        )
+
+        # Tweak transformers to optimize performance
+        adapt_transformers_to_gaudi()
+
+        self.device = torch.device("hpu")
+
+        self.tokenizer = AutoTokenizer.from_pretrained(
+            model_id_or_path, use_fast=False, use_auth_token=""
+        )
+        hf_config = AutoConfig.from_pretrained(
+            model_id_or_path,
+            torchscript=True,
+            use_auth_token="",
+            trust_remote_code=False,
+        )
+        # Load the model in Gaudi
+        model = AutoModelForCausalLM.from_pretrained(
+            model_id_or_path,
+            config=hf_config,
+            torch_dtype=torch.float32,
+            low_cpu_mem_usage=True,
+            use_auth_token="",
+        )
+        model = model.eval().to(self.device)
+
+        from habana_frameworks.torch.hpu import wrap_in_hpu_graph
+
+        # Enable hpu graph runtime
+        self.model = wrap_in_hpu_graph(model)
+
+        # Set pad token, etc.
+        self.tokenizer.pad_token_id = self.model.generation_config.pad_token_id
+        self.tokenizer.padding_side = "left"
+
+        # Use async loop in streaming
+        self.loop = asyncio.get_running_loop()
+
+    def tokenize(self, prompt: str):
+        """Tokenize the input and move to HPU."""
+
+        input_tokens = self.tokenizer(prompt, return_tensors="pt", padding=True)
+        return input_tokens.input_ids.to(device=self.device)
+
+    def generate(self, prompt: str, **config: Dict[str, Any]):
+        """Take a prompt and generate a response."""
+
+        input_ids = self.tokenize(prompt)
+        gen_tokens = self.model.generate(input_ids, **config)
+        return self.tokenizer.batch_decode(gen_tokens, skip_special_tokens=True)[0]
+
+    async def consume_streamer_async(self, streamer):
+        """Consume the streamer asynchronously."""
+
+        while True:
+            try:
+                for token in streamer:
+                    yield token
+                break
+            except Empty:
+                await asyncio.sleep(0.001)
+
+    def streaming_generate(self, prompt: str, streamer, **config: Dict[str, Any]):
+        """Generate a streamed response given an input."""
+
+        input_ids = self.tokenize(prompt)
+        self.model.generate(input_ids, streamer=streamer, **config)
+
+    async def __call__(self, http_request: Request):
+        """Handle HTTP requests."""
+
+        # Load fields from the request
+        json_request: str = await http_request.json()
+        text = json_request["text"]
+        # Config used in generation
+        config = json_request.get("config", {})
+        streaming_response = json_request["stream"]
+
+        # Prepare prompts
+        prompts = []
+        if isinstance(text, list):
+            prompts.extend(text)
+        else:
+            prompts.append(text)
+
+        # Process config
+        config.setdefault("max_new_tokens", 128)
+
+        # Enable HPU graph runtime
+        config["hpu_graphs"] = True
+        # Lazy mode should be True when using HPU graphs
+        config["lazy_mode"] = True
+
+        # Non-streaming case
+        if not streaming_response:
+            return self.generate(prompts, **config)
+
+        # Streaming case
+        from transformers import TextIteratorStreamer
+
+        streamer = TextIteratorStreamer(
+            self.tokenizer, skip_prompt=True, timeout=0, skip_special_tokens=True
+        )
+        # Convert the streamer into a generator
+        self.loop.run_in_executor(
+            None, partial(self.streaming_generate, prompts, streamer, **config)
+        )
+        return StreamingResponse(
+            self.consume_streamer_async(streamer),
+            status_code=200,
+            media_type="text/plain",
+        )
+
+
+# Replace the model ID with path if necessary
+entrypoint = LlamaModel.bind("meta-llama/Llama-2-7b-chat-hf")
+# __model_def_end__

doc/source/serve/doc_code/intel_gaudi_inference_serve_deepspeed.py

@@ -0,0 +1,273 @@
+# __worker_def_start__
+import tempfile
+from typing import Dict, Any
+from starlette.requests import Request
+from starlette.responses import StreamingResponse
+
+import torch
+from transformers import TextStreamer
+
+import ray
+from ray import serve
+from ray.util.queue import Queue
+from ray.runtime_env import RuntimeEnv
+
+
+@ray.remote(resources={"HPU": 1})
+class DeepSpeedInferenceWorker:
+    def __init__(self, model_id_or_path: str, world_size: int, local_rank: int):
+        """An actor that runs a DeepSpeed inference engine.
+
+        Arguments:
+            model_id_or_path: Either a Hugging Face model ID
+                or a path to a cached model.
+            world_size: Total number of worker processes.
+            local_rank: Rank of this worker process.
+                The rank 0 worker is the head worker.
+        """
+        from transformers import AutoTokenizer, AutoConfig
+        from optimum.habana.transformers.modeling_utils import (
+            adapt_transformers_to_gaudi,
+        )
+
+        # Tweak transformers for better performance on Gaudi.
+        adapt_transformers_to_gaudi()
+
+        self.model_id_or_path = model_id_or_path
+        self._world_size = world_size
+        self._local_rank = local_rank
+        self.device = torch.device("hpu")
+
+        self.model_config = AutoConfig.from_pretrained(
+            model_id_or_path,
+            torch_dtype=torch.bfloat16,
+            token="",
+            trust_remote_code=False,
+        )
+
+        # Load and configure the tokenizer.
+        self.tokenizer = AutoTokenizer.from_pretrained(
+            model_id_or_path, use_fast=False, token=""
+        )
+        self.tokenizer.padding_side = "left"
+        if self.tokenizer.pad_token is None:
+            self.tokenizer.pad_token = self.tokenizer.eos_token
+
+        import habana_frameworks.torch.distributed.hccl as hccl
+
+        # Initialize the distributed backend.
+        hccl.initialize_distributed_hpu(
+            world_size=world_size, rank=local_rank, local_rank=local_rank
+        )
+        torch.distributed.init_process_group(backend="hccl")
+
+    def load_model(self):
+        """Load the model to HPU and initialize the DeepSpeed inference engine."""
+
+        import deepspeed
+        from transformers import AutoModelForCausalLM
+        from optimum.habana.checkpoint_utils import (
+            get_ds_injection_policy,
+            write_checkpoints_json,
+        )
+
+        # Construct the model with fake meta Tensors.
+        # Loads the model weights from the checkpoint later.
+        with deepspeed.OnDevice(dtype=torch.bfloat16, device="meta"):
+            model = AutoModelForCausalLM.from_config(
+                self.model_config, torch_dtype=torch.bfloat16
+            )
+        model = model.eval()
+
+        # Create a file to indicate where the checkpoint is.
+        checkpoints_json = tempfile.NamedTemporaryFile(suffix=".json", mode="w+")
+        write_checkpoints_json(
+            self.model_id_or_path, self._local_rank, checkpoints_json, token=""
+        )
+
+        # Prepare the DeepSpeed inference configuration.
+        kwargs = {"dtype": torch.bfloat16}
+        kwargs["checkpoint"] = checkpoints_json.name
+        kwargs["tensor_parallel"] = {"tp_size": self._world_size}
+        # Enable the HPU graph, similar to the cuda graph.
+        kwargs["enable_cuda_graph"] = True
+        # Specify the injection policy, required by DeepSpeed Tensor parallelism.
+        kwargs["injection_policy"] = get_ds_injection_policy(self.model_config)
+
+        # Initialize the inference engine.
+        self.model = deepspeed.init_inference(model, **kwargs).module
+
+    def tokenize(self, prompt: str):
+        """Tokenize the input and move it to HPU."""
+
+        input_tokens = self.tokenizer(prompt, return_tensors="pt", padding=True)
+        return input_tokens.input_ids.to(device=self.device)
+
+    def generate(self, prompt: str, **config: Dict[str, Any]):
+        """Take in a prompt and generate a response."""
+
+        input_ids = self.tokenize(prompt)
+        gen_tokens = self.model.generate(input_ids, **config)
+        return self.tokenizer.batch_decode(gen_tokens, skip_special_tokens=True)[0]
+
+    def streaming_generate(self, prompt: str, streamer, **config: Dict[str, Any]):
+        """Generate a streamed response given an input."""
+
+        input_ids = self.tokenize(prompt)
+        self.model.generate(input_ids, streamer=streamer, **config)
+
+    def get_streamer(self):
+        """Return a streamer.
+
+        We only need the rank 0 worker's result.
+        Other workers return a fake streamer.
+        """
+
+        if self._local_rank == 0:
+            return RayTextIteratorStreamer(self.tokenizer, skip_special_tokens=True)
+        else:
+
+            class FakeStreamer:
+                def put(self, value):
+                    pass
+
+                def end(self):
+                    pass
+
+            return FakeStreamer()
+
+
+class RayTextIteratorStreamer(TextStreamer):
+    def __init__(
+        self,
+        tokenizer,
+        skip_prompt: bool = False,
+        timeout: int = None,
+        **decode_kwargs: Dict[str, Any],
+    ):
+        super().__init__(tokenizer, skip_prompt, **decode_kwargs)
+        self.text_queue = Queue()
+        self.stop_signal = None
+        self.timeout = timeout
+
+    def on_finalized_text(self, text: str, stream_end: bool = False):
+        self.text_queue.put(text, timeout=self.timeout)
+        if stream_end:
+            self.text_queue.put(self.stop_signal, timeout=self.timeout)
+
+    def __iter__(self):
+        return self
+
+    def __next__(self):
+        value = self.text_queue.get(timeout=self.timeout)
+        if value == self.stop_signal:
+            raise StopIteration()
+        else:
+            return value
+
+
+# __worker_def_end__
+
+# __deploy_def_start__
+# We need to set these variables for this example.
+HABANA_ENVS = {
+    "PT_HPU_LAZY_ACC_PAR_MODE": "0",
+    "PT_HPU_ENABLE_REFINE_DYNAMIC_SHAPES": "0",
+    "PT_HPU_ENABLE_WEIGHT_CPU_PERMUTE": "0",
+    "PT_HPU_ENABLE_LAZY_COLLECTIVES": "true",
+    "HABANA_VISIBLE_MODULES": "0,1,2,3,4,5,6,7",
+}
+
+
+# Define the Ray Serve deployment.
+@serve.deployment
+class DeepSpeedLlamaModel:
+    def __init__(self, world_size: int, model_id_or_path: str):
+        self._world_size = world_size
+
+        # Create the DeepSpeed workers
+        self.deepspeed_workers = []
+        for i in range(world_size):
+            self.deepspeed_workers.append(
+                DeepSpeedInferenceWorker.options(
+                    runtime_env=RuntimeEnv(env_vars=HABANA_ENVS)
+                ).remote(model_id_or_path, world_size, i)
+            )
+
+        # Load the model to all workers.
+        for worker in self.deepspeed_workers:
+            worker.load_model.remote()
+
+        # Get the workers' streamers.
+        self.streamers = ray.get(
+            [worker.get_streamer.remote() for worker in self.deepspeed_workers]
+        )
+
+    def generate(self, prompt: str, **config: Dict[str, Any]):
+        """Send the prompt to workers for generation.
+
+        Return after all workers finish the generation.
+        Only return the rank 0 worker's result.
+        """
+
+        futures = [
+            worker.generate.remote(prompt, **config)
+            for worker in self.deepspeed_workers
+        ]
+        return ray.get(futures)[0]
+
+    def streaming_generate(self, prompt: str, **config: Dict[str, Any]):
+        """Send the prompt to workers for streaming generation.
+
+        Only use the rank 0 worker's result.
+        """
+
+        for worker, streamer in zip(self.deepspeed_workers, self.streamers):
+            worker.streaming_generate.remote(prompt, streamer, **config)
+
+    def consume_streamer(self, streamer):
+        """Consume the streamer and return a generator."""
+        for token in streamer:
+            yield token
+
+    async def __call__(self, http_request: Request):
+        """Handle received HTTP requests."""
+
+        # Load fields from the request
+        json_request: str = await http_request.json()
+        text = json_request["text"]
+        # Config used in generation
+        config = json_request.get("config", {})
+        streaming_response = json_request["stream"]
+
+        # Prepare prompts
+        prompts = []
+        if isinstance(text, list):
+            prompts.extend(text)
+        else:
+            prompts.append(text)
+
+        # Process the configuration.
+        config.setdefault("max_new_tokens", 128)
+
+        # Enable HPU graph runtime.
+        config["hpu_graphs"] = True
+        # Lazy mode should be True when using HPU graphs.
+        config["lazy_mode"] = True
+
+        # Non-streaming case
+        if not streaming_response:
+            return self.generate(prompts, **config)
+
+        # Streaming case
+        self.streaming_generate(prompts, **config)
+        return StreamingResponse(
+            self.consume_streamer(self.streamers[0]),
+            status_code=200,
+            media_type="text/plain",
+        )
+
+
+# Replace the model ID with a path if necessary.
+entrypoint = DeepSpeedLlamaModel.bind(8, "meta-llama/Llama-2-70b-chat-hf")
+# __deploy_def_end__

doc/source/serve/doc_code/interdeployment_grpc.py

@@ -0,0 +1,32 @@
+# flake8: noqa
+
+from ray import serve
+from ray.serve.handle import DeploymentHandle
+
+# __start_grpc_override__
+@serve.deployment
+class Caller:
+    def __init__(self, target: DeploymentHandle):
+        # Override this specific handle to use actor RPC instead of gRPC.
+        # This is useful for large payloads (over ~1 MB) where passing
+        # objects by reference through Ray's object store is more efficient.
+        self._target = target.options(_by_reference=True)
+
+    async def __call__(self, data: bytes) -> str:
+        return await self._target.remote(data)
+
+
+@serve.deployment
+class LargePayloadProcessor:
+    def __call__(self, data: bytes) -> str:
+        return f"processed {len(data)} bytes"
+
+
+processor = LargePayloadProcessor.bind()
+app = Caller.bind(processor)
+
+handle: DeploymentHandle = serve.run(app)
+assert handle.remote(b"x" * 1024).result() == "processed 1024 bytes"
+# __end_grpc_override__
+
+serve.shutdown()

doc/source/serve/doc_code/key_concepts.py

@@ -0,0 +1,108 @@
+# flake8: noqa
+
+# __start_my_first_deployment__
+from ray import serve
+from ray.serve.handle import DeploymentHandle
+
+
+@serve.deployment
+class MyFirstDeployment:
+    # Take the message to return as an argument to the constructor.
+    def __init__(self, msg):
+        self.msg = msg
+
+    def __call__(self):
+        return self.msg
+
+
+my_first_deployment = MyFirstDeployment.bind("Hello world!")
+handle: DeploymentHandle = serve.run(my_first_deployment)
+assert handle.remote().result() == "Hello world!"
+# __end_my_first_deployment__
+
+# __start_deployment_handle__
+from ray import serve
+from ray.serve.handle import DeploymentHandle
+
+
+@serve.deployment
+class Hello:
+    def __call__(self) -> str:
+        return "Hello"
+
+
+@serve.deployment
+class World:
+    def __call__(self) -> str:
+        return " world!"
+
+
+@serve.deployment
+class Ingress:
+    def __init__(self, hello_handle: DeploymentHandle, world_handle: DeploymentHandle):
+        self._hello_handle = hello_handle
+        self._world_handle = world_handle
+
+    async def __call__(self) -> str:
+        hello_response = self._hello_handle.remote()
+        world_response = self._world_handle.remote()
+        return (await hello_response) + (await world_response)
+
+
+hello = Hello.bind()
+world = World.bind()
+
+# The deployments passed to the Ingress constructor are replaced with handles.
+app = Ingress.bind(hello, world)
+
+# Deploys Hello, World, and Ingress.
+handle: DeploymentHandle = serve.run(app)
+
+# `DeploymentHandle`s can also be used to call the ingress deployment of an application.
+assert handle.remote().result() == "Hello world!"
+# __end_deployment_handle__
+
+# __start_basic_ingress__
+import requests
+from starlette.requests import Request
+
+from ray import serve
+
+
+@serve.deployment
+class MostBasicIngress:
+    async def __call__(self, request: Request) -> str:
+        name = (await request.json())["name"]
+        return f"Hello {name}!"
+
+
+app = MostBasicIngress.bind()
+serve.run(app)
+assert (
+    requests.get("http://127.0.0.1:8000/", json={"name": "Corey"}).text
+    == "Hello Corey!"
+)
+# __end_basic_ingress__
+
+# __start_fastapi_ingress__
+import requests
+from fastapi import FastAPI
+from fastapi.responses import PlainTextResponse
+
+from ray import serve
+
+fastapi_app = FastAPI()
+
+
+@serve.deployment
+@serve.ingress(fastapi_app)
+class FastAPIIngress:
+    @fastapi_app.get("/{name}")
+    async def say_hi(self, name: str) -> str:
+        return PlainTextResponse(f"Hello {name}!")
+
+
+app = FastAPIIngress.bind()
+serve.run(app)
+assert requests.get("http://127.0.0.1:8000/Corey").text == "Hello Corey!"
+# __end_fastapi_ingress__

doc/source/serve/doc_code/load_shedding.py

@@ -0,0 +1,55 @@
+# flake8: noqa
+# fmt: off
+
+# __example_deployment_start__
+import time
+from ray import serve
+from starlette.requests import Request
+
+@serve.deployment(
+    # Each replica will be sent 2 requests at a time.
+    max_ongoing_requests=2,
+    # Each caller queues up to 2 requests at a time.
+    # (beyond those that are sent to replicas).
+    max_queued_requests=2,
+)
+class SlowDeployment:
+    def __call__(self, request: Request) -> str:
+        # Emulate a long-running request, such as ML inference.
+        time.sleep(2)
+        return "Hello!"
+# __example_deployment_end__
+
+# __client_test_start__
+import ray
+import aiohttp
+
+@ray.remote
+class Requester:
+    async def do_request(self) -> int:
+        async with aiohttp.ClientSession("http://localhost:8000/") as session:
+            return (await session.get("/")).status
+
+r = Requester.remote()
+serve.run(SlowDeployment.bind())
+
+# Send 4 requests first.
+# 2 of these will be sent to the replica. These requests take a few seconds to execute.
+first_refs = [r.do_request.remote() for _ in range(2)]
+_, pending = ray.wait(first_refs, timeout=1)
+assert len(pending) == 2
+# 2 will be queued in the proxy.
+queued_refs = [r.do_request.remote() for _ in range(2)]
+_, pending = ray.wait(queued_refs, timeout=0.1)
+assert len(pending) == 2
+
+# Send an additional 5 requests. These will be rejected immediately because
+# the replica and the proxy queue are already full.
+for status_code in ray.get([r.do_request.remote() for _ in range(5)]):
+    assert status_code == 503
+
+# The initial requests will finish successfully.
+for ref in first_refs:
+    print(f"Request finished with status code {ray.get(ref)}.")
+
+# __client_test_end__

doc/source/serve/doc_code/local_dev.py

@@ -0,0 +1,38 @@
+# __local_dev_start__
+# Filename: local_dev.py
+from starlette.requests import Request
+
+from ray import serve
+from ray.serve.handle import DeploymentHandle, DeploymentResponse
+
+
+@serve.deployment
+class Doubler:
+    def double(self, s: str):
+        return s + " " + s
+
+
+@serve.deployment
+class HelloDeployment:
+    def __init__(self, doubler: DeploymentHandle):
+        self.doubler = doubler
+
+    async def say_hello_twice(self, name: str):
+        return await self.doubler.double.remote(f"Hello, {name}!")
+
+    async def __call__(self, request: Request):
+        return await self.say_hello_twice(request.query_params["name"])
+
+
+app = HelloDeployment.bind(Doubler.bind())
+# __local_dev_end__
+
+# __local_dev_handle_start__
+handle: DeploymentHandle = serve.run(app)
+response: DeploymentResponse = handle.say_hello_twice.remote(name="Ray")
+assert response.result() == "Hello, Ray! Hello, Ray!"
+# __local_dev_handle_end__
+
+# __local_dev_testing_start__
+serve.run(app, _local_testing_mode=True)
+# __local_dev_testing_end__

doc/source/serve/doc_code/managing_deployments.py

@@ -0,0 +1,80 @@
+from ray import serve
+import time
+import os
+
+
+# __updating_a_deployment_start__
+@serve.deployment(name="my_deployment", num_replicas=1)
+class SimpleDeployment:
+    pass
+
+
+# Creates one initial replica.
+serve.run(SimpleDeployment.bind())
+
+
+# Re-deploys, creating an additional replica.
+# This could be the SAME Python script, modified and re-run.
+@serve.deployment(name="my_deployment", num_replicas=2)
+class SimpleDeployment:
+    pass
+
+
+serve.run(SimpleDeployment.bind())
+
+# You can also use Deployment.options() to change options without redefining
+# the class. This is useful for programmatically updating deployments.
+serve.run(SimpleDeployment.options(num_replicas=2).bind())
+# __updating_a_deployment_end__
+
+
+# __scaling_out_start__
+# Create with a single replica.
+@serve.deployment(num_replicas=1)
+def func(*args):
+    pass
+
+
+serve.run(func.bind())
+
+# Scale up to 3 replicas.
+serve.run(func.options(num_replicas=3).bind())
+
+# Scale back down to 1 replica.
+serve.run(func.options(num_replicas=1).bind())
+# __scaling_out_end__
+
+
+# __autoscaling_start__
+@serve.deployment(
+    autoscaling_config={
+        "min_replicas": 1,
+        "initial_replicas": 2,
+        "max_replicas": 5,
+        "target_ongoing_requests": 10,
+    }
+)
+def func(_):
+    time.sleep(1)
+    return ""
+
+
+serve.run(
+    func.bind()
+)  # The func deployment will now autoscale based on requests demand.
+# __autoscaling_end__
+
+
+# __configure_parallism_start__
+@serve.deployment
+class MyDeployment:
+    def __init__(self, parallelism: str):
+        os.environ["OMP_NUM_THREADS"] = parallelism
+        # Download model weights, initialize model, etc.
+
+    def __call__(self):
+        pass
+
+
+serve.run(MyDeployment.bind("12"))
+# __configure_parallism_end__

doc/source/serve/doc_code/mlflow_model_registry_integration.py

@@ -0,0 +1,139 @@
+# __train_model_start__
+from sklearn.datasets import make_regression
+from sklearn.ensemble import RandomForestRegressor
+from sklearn.metrics import mean_squared_error
+from sklearn.model_selection import train_test_split
+from sklearn.pipeline import Pipeline
+from sklearn.preprocessing import StandardScaler
+import mlflow
+import mlflow.sklearn
+import mlflow.pyfunc
+from mlflow.entities import LoggedModelStatus
+from mlflow.models import infer_signature
+import numpy as np
+
+
+def train_and_register_model():
+    # Initialize model in PENDING state
+    logged_model = mlflow.initialize_logged_model(
+        name="sk-learn-random-forest-reg-model",
+        model_type="sklearn",
+        tags={"model_type": "random_forest"},
+    )
+
+    try:
+        with mlflow.start_run() as run:
+            X, y = make_regression(n_features=4, n_informative=2, random_state=0, shuffle=False)
+            X_train, X_test, y_train, y_test = train_test_split(
+                X, y, test_size=0.2, random_state=42
+            )
+
+            params = {"max_depth": 2, "random_state": 42}
+            
+            # Best Practice: Use sklearn Pipeline to persist preprocessing
+            # This ensures training and serving transformations stay aligned
+            pipeline = Pipeline([
+                ("scaler", StandardScaler()),
+                ("regressor", RandomForestRegressor(**params))
+            ])
+            pipeline.fit(X_train, y_train)
+
+            # Log parameters and metrics
+            mlflow.log_params(params)
+
+            y_pred = pipeline.predict(X_test)
+            mlflow.log_metrics({"mse": mean_squared_error(y_test, y_pred)})
+
+            # Best Practice: Infer model signature for input validation
+            # Prevents silent failures from mismatched feature order or missing columns
+            signature = infer_signature(X_train, y_pred)
+
+            # Best Practice: Pin dependency versions explicitly
+            # Ensures identical behavior across training, evaluation, and serving
+            pip_requirements = [
+                f"scikit-learn=={__import__('sklearn').__version__}",
+                f"numpy=={np.__version__}",
+            ]
+
+            # Log the sklearn pipeline with signature and dependencies
+            mlflow.sklearn.log_model(
+                sk_model=pipeline,
+                name="sklearn-model",
+                input_example=X_train[:1],
+                signature=signature,
+                pip_requirements=pip_requirements,
+                registered_model_name="sk-learn-random-forest-reg-model",
+                model_id=logged_model.model_id,
+            )
+
+        # Finalize model as READY
+        mlflow.finalize_logged_model(logged_model.model_id, LoggedModelStatus.READY)
+
+        mlflow.set_logged_model_tags(
+            logged_model.model_id,
+            tags={"production": "true"},
+        )
+
+    except Exception as e:
+        # Mark model as FAILED if issues occur
+        mlflow.finalize_logged_model(logged_model.model_id, LoggedModelStatus.FAILED)
+        raise
+
+    # Retrieve and work with the logged model
+    final_model = mlflow.get_logged_model(logged_model.model_id)
+    print(f"Model {final_model.name} is {final_model.status}")
+# __train_model_end__
+
+
+# __deployment_start__
+from ray import serve
+import mlflow.pyfunc
+import numpy as np
+
+
+@serve.deployment
+class MLflowModelDeployment:
+    def __init__(self):
+        # Search for models with production tag
+        models = mlflow.search_logged_models(
+            filter_string="tags.production='true' AND name='sk-learn-random-forest-reg-model'",
+            order_by=[{"field_name": "creation_time", "ascending": False}],
+        )
+        if models.empty:
+            raise ValueError("No model with production tag found")
+        
+        # Get the most recent production model
+        model_row = models.iloc[0]
+        artifact_location = model_row["artifact_location"]
+        
+        # Best Practice: Load model once during initialization (warm-start)
+        # This eliminates first-request latency spikes
+        self.model = mlflow.pyfunc.load_model(artifact_location)
+        
+        # Pre-warm the model with a dummy prediction
+        dummy_input = np.zeros((1, 4))
+        _ = self.model.predict(dummy_input)
+
+    async def __call__(self, request):
+        data = await request.json()
+        features = np.array(data["features"])
+        
+        # MLflow validates input against the logged signature automatically
+        prediction = self.model.predict(features)
+        return {"prediction": prediction.tolist()}
+
+
+app = MLflowModelDeployment.bind()
+# __deployment_end__
+
+
+if __name__ == "__main__":
+    import requests
+    from ray import serve
+
+    train_and_register_model()
+    serve.run(app)
+
+    # Test prediction
+    response = requests.post("http://localhost:8000/", json={"features": [[0.1, 0.2, 0.3, 0.4]]})
+    print(response.json())

doc/source/serve/doc_code/model_composition/chaining_example.py

@@ -0,0 +1,50 @@
+# flake8: noqa
+# __chaining_example_start__
+# File name: chain.py
+from ray import serve
+from ray.serve.handle import DeploymentHandle, DeploymentResponse
+
+
+@serve.deployment
+class Adder:
+    def __init__(self, increment: int):
+        self._increment = increment
+
+    def __call__(self, val: int) -> int:
+        return val + self._increment
+
+
+@serve.deployment
+class Multiplier:
+    def __init__(self, multiple: int):
+        self._multiple = multiple
+
+    def __call__(self, val: int) -> int:
+        return val * self._multiple
+
+
+@serve.deployment
+class Ingress:
+    def __init__(self, adder: DeploymentHandle, multiplier: DeploymentHandle):
+        self._adder = adder
+        self._multiplier = multiplier
+
+    async def __call__(self, input: int) -> int:
+        adder_response: DeploymentResponse = self._adder.remote(input)
+        # Pass the adder response directly into the multiplier (no `await` needed).
+        multiplier_response: DeploymentResponse = self._multiplier.remote(
+            adder_response
+        )
+        # `await` the final chained response.
+        return await multiplier_response
+
+
+app = Ingress.bind(
+    Adder.bind(increment=1),
+    Multiplier.bind(multiple=2),
+)
+
+handle: DeploymentHandle = serve.run(app)
+response = handle.remote(5)
+assert response.result() == 12, "(5 + 1) * 2 = 12"
+# __chaining_example_end__

doc/source/serve/doc_code/model_composition/language_example.py

@@ -0,0 +1,98 @@
+# flake8: noqa
+
+import requests
+
+# __echo_class_start__
+# File name: echo.py
+from starlette.requests import Request
+
+from ray import serve
+
+
+@serve.deployment
+class EchoClass:
+    def __init__(self, echo_str: str):
+        self.echo_str = echo_str
+
+    def __call__(self, request: Request) -> str:
+        return self.echo_str
+
+
+# You can create ClassNodes from the EchoClass deployment
+foo_node = EchoClass.bind("foo")
+bar_node = EchoClass.bind("bar")
+baz_node = EchoClass.bind("baz")
+# __echo_class_end__
+
+for node, echo in [(foo_node, "foo"), (bar_node, "bar"), (baz_node, "baz")]:
+    serve.run(node)
+    assert requests.get("http://localhost:8000/").text == echo
+
+# __echo_client_start__
+# File name: echo_client.py
+import requests
+
+response = requests.get("http://localhost:8000/")
+echo = response.text
+print(echo)
+# __echo_client_end__
+
+# __hello_start__
+# File name: hello.py
+from ray import serve
+from ray.serve.handle import DeploymentHandle
+
+
+@serve.deployment
+class LanguageClassifer:
+    def __init__(
+        self, spanish_responder: DeploymentHandle, french_responder: DeploymentHandle
+    ):
+        self.spanish_responder = spanish_responder
+        self.french_responder = french_responder
+
+    async def __call__(self, http_request):
+        request = await http_request.json()
+        language, name = request["language"], request["name"]
+
+        if language == "spanish":
+            response = self.spanish_responder.say_hello.remote(name)
+        elif language == "french":
+            response = self.french_responder.say_hello.remote(name)
+        else:
+            return "Please try again."
+
+        return await response
+
+
+@serve.deployment
+class SpanishResponder:
+    def say_hello(self, name: str):
+        return f"Hola {name}"
+
+
+@serve.deployment
+class FrenchResponder:
+    def say_hello(self, name: str):
+        return f"Bonjour {name}"
+
+
+spanish_responder = SpanishResponder.bind()
+french_responder = FrenchResponder.bind()
+language_classifier = LanguageClassifer.bind(spanish_responder, french_responder)
+# __hello_end__
+
+serve.run(language_classifier)
+
+# __hello_client_start__
+# File name: hello_client.py
+import requests
+
+response = requests.post(
+    "http://localhost:8000", json={"language": "spanish", "name": "Dora"}
+)
+greeting = response.text
+print(greeting)
+# __hello_client_end__
+
+assert greeting == "Hola Dora"

doc/source/serve/doc_code/model_composition/response_to_object_ref_example.py

@@ -0,0 +1,37 @@
+# flake8: noqa
+# __response_to_object_ref_example_start__
+# File name: response_to_object_ref.py
+import ray
+from ray import serve
+from ray.serve.handle import DeploymentHandle, DeploymentResponse
+
+
+@ray.remote
+def say_hi_task(inp: str):
+    return f"Ray task got message: '{inp}'"
+
+
+@serve.deployment
+class SayHi:
+    def __call__(self) -> str:
+        return "Hi from Serve deployment"
+
+
+@serve.deployment
+class Ingress:
+    def __init__(self, say_hi: DeploymentHandle):
+        self._say_hi = say_hi
+
+    async def __call__(self):
+        # Make a call to the SayHi deployment and pass the result ref to
+        # a downstream Ray task.
+        response: DeploymentResponse = self._say_hi.remote()
+        response_obj_ref: ray.ObjectRef = await response._to_object_ref()
+        final_obj_ref: ray.ObjectRef = say_hi_task.remote(response_obj_ref)
+        return await final_obj_ref
+
+
+app = Ingress.bind(SayHi.bind())
+handle: DeploymentHandle = serve.run(app)
+assert handle.remote().result() == "Ray task got message: 'Hi from Serve deployment'"
+# __response_to_object_ref_example_end__

doc/source/serve/doc_code/model_composition/streaming_example.py

@@ -0,0 +1,42 @@
+# flake8: noqa
+# __streaming_example_start__
+# File name: stream.py
+from typing import AsyncGenerator, Generator
+
+from ray import serve
+from ray.serve.handle import DeploymentHandle, DeploymentResponseGenerator
+
+
+@serve.deployment
+class Streamer:
+    def __call__(self, limit: int) -> Generator[int, None, None]:
+        for i in range(limit):
+            yield i
+
+
+@serve.deployment
+class Caller:
+    def __init__(self, streamer: DeploymentHandle):
+        self._streamer = streamer.options(
+            # Must set `stream=True` on the handle, then the output will be a
+            # response generator.
+            stream=True,
+        )
+
+    async def __call__(self, limit: int) -> AsyncGenerator[int, None]:
+        # Response generator can be used in an `async for` block.
+        r: DeploymentResponseGenerator = self._streamer.remote(limit)
+        async for i in r:
+            yield i
+
+
+app = Caller.bind(Streamer.bind())
+
+handle: DeploymentHandle = serve.run(app).options(
+    stream=True,
+)
+
+# Response generator can also be used as a regular generator in a sync context.
+r: DeploymentResponseGenerator = handle.remote(10)
+assert list(r) == list(range(10))
+# __streaming_example_end__

doc/source/serve/doc_code/monitoring/custom_metric_snippet.py

@@ -0,0 +1,36 @@
+# __start__
+from ray import serve
+from ray.serve import metrics
+
+import time
+import requests
+
+
+@serve.deployment
+class MyDeployment:
+    def __init__(self):
+        self.num_requests = 0
+        self.my_counter = metrics.Counter(
+            "my_counter",
+            description=("The number of odd-numbered requests to this deployment."),
+            tag_keys=("model",),
+        )
+        self.my_counter.set_default_tags({"model": "123"})
+
+    def __call__(self):
+        self.num_requests += 1
+        if self.num_requests % 2 == 1:
+            self.my_counter.inc()
+
+
+my_deployment = MyDeployment.bind()
+serve.run(my_deployment)
+
+while True:
+    requests.get("http://localhost:8000/")
+    time.sleep(1)
+
+    # __end__
+    break
+response = requests.get("http://localhost:8000/")
+assert response.status_code == 200

doc/source/serve/doc_code/monitoring/deployment_logger.py

@@ -0,0 +1,29 @@
+# __start__
+from ray import serve
+
+import logging
+import requests
+
+logger = logging.getLogger("ray.serve")
+
+
+@serve.deployment
+class Counter:
+    def __init__(self):
+        self.count = 0
+
+    def __call__(self, request):
+        self.count += 1
+        logger.info(f"count: {self.count}")
+        return {"count": self.count}
+
+
+counter = Counter.bind()
+serve.run(counter)
+
+for i in range(10):
+    requests.get("http://127.0.0.1:8000/")
+# __end__
+
+response = requests.get("http://127.0.0.1:8000/")
+assert response.json() == {"count": 11}

doc/source/serve/doc_code/monitoring/logging_config.py

@@ -0,0 +1,129 @@
+# flake8: noqa
+#  __deployment_json_start__
+import requests
+from ray import serve
+from ray.serve.schema import LoggingConfig
+
+
+@serve.deployment(logging_config=LoggingConfig(encoding="JSON"))
+class Model:
+    def __call__(self) -> int:
+        return "hello world"
+
+
+serve.run(Model.bind())
+
+resp = requests.get("http://localhost:8000/")
+
+# __deployment_json_end__
+
+
+#  __serve_run_json_start__
+import requests
+from ray import serve
+from ray.serve.schema import LoggingConfig
+
+
+@serve.deployment
+class Model:
+    def __call__(self) -> int:
+        return "hello world"
+
+
+serve.run(Model.bind(), logging_config=LoggingConfig(encoding="JSON"))
+
+resp = requests.get("http://localhost:8000/")
+
+# __serve_run_json_end__
+
+
+#  __level_start__
+
+
+@serve.deployment(logging_config=LoggingConfig(log_level="DEBUG"))
+class Model:
+    def __call__(self) -> int:
+        logger = logging.getLogger("ray.serve")
+        logger.debug("This debug message is from the router.")
+        return "hello world"
+
+
+# __level_end__
+
+serve.run(Model.bind())
+
+resp = requests.get("http://localhost:8000/")
+
+
+# __logs_dir_start__
+@serve.deployment(logging_config=LoggingConfig(logs_dir="/my_dirs"))
+class Model:
+    def __call__(self) -> int:
+        return "hello world"
+
+
+# __logs_dir_end__
+
+
+# __enable_access_log_start__
+import requests
+import logging
+from ray import serve
+
+
+@serve.deployment(logging_config={"enable_access_log": False})
+class Model:
+    def __call__(self):
+        logger = logging.getLogger("ray.serve")
+        logger.info("hello world")
+
+
+serve.run(Model.bind())
+
+resp = requests.get("http://localhost:8000/")
+
+# __enable_access_log_end__
+
+
+# __application_and_deployment_start__
+import requests
+import logging
+from ray import serve
+
+
+@serve.deployment
+class Router:
+    def __init__(self, handle):
+        self.handle = handle
+
+    async def __call__(self):
+        logger = logging.getLogger("ray.serve")
+        logger.debug("This debug message is from the router.")
+        return await self.handle.remote()
+
+
+@serve.deployment(logging_config={"log_level": "INFO"})
+class Model:
+    def __call__(self) -> int:
+        logger = logging.getLogger("ray.serve")
+        logger.debug("This debug message is from the model.")
+        return "hello world"
+
+
+serve.run(Router.bind(Model.bind()), logging_config={"log_level": "DEBUG"})
+resp = requests.get("http://localhost:8000/")
+# __application_and_deployment_end__
+
+# __configure_serve_component_start__
+from ray import serve
+
+
+serve.start(
+    logging_config={
+        "encoding": "JSON",
+        "log_level": "DEBUG",
+        "enable_access_log": False,
+    }
+)
+
+# __configure_serve_component_end__

doc/source/serve/doc_code/monitoring/metrics_snippet.py

@@ -0,0 +1,23 @@
+# __start__
+from ray import serve
+
+import time
+import requests
+
+
+@serve.deployment
+def sleeper():
+    time.sleep(1)
+
+
+s = sleeper.bind()
+
+serve.run(s)
+
+while True:
+    requests.get("http://localhost:8000/")
+    # __end__
+    break
+
+response = requests.get("http://localhost:8000/")
+assert response.status_code == 200

doc/source/serve/doc_code/monitoring/monitor_deployment.py

@@ -0,0 +1,33 @@
+# flake8: noqa
+# fmt: off
+
+# __monitor_start__
+from typing import List, Dict
+
+from ray import serve
+from ray.serve.schema import ServeStatus, ApplicationStatusOverview
+
+
+@serve.deployment
+def get_healthy_apps() -> List[str]:
+    serve_status: ServeStatus = serve.status()
+    app_statuses: Dict[str, ApplicationStatusOverview] = serve_status.applications
+
+    running_apps = []
+    for app_name, app_status in app_statuses.items():
+        if app_status.status == "RUNNING":
+            running_apps.append(app_name)
+
+    return running_apps
+
+
+monitoring_app = get_healthy_apps.bind()
+# __monitor_end__
+
+serve.run(monitoring_app, name="monitor")
+
+import requests
+
+resp = requests.get("http://localhost:8000/")
+
+assert requests.get("http://localhost:8000/").json() == ["monitor"]

doc/source/serve/doc_code/monitoring/monitoring.py

@@ -0,0 +1,27 @@
+# flake8: noqa
+
+# __start__
+# File name: monitoring.py
+
+from ray import serve
+import logging
+from starlette.requests import Request
+
+logger = logging.getLogger("ray.serve")
+
+
+@serve.deployment
+class SayHello:
+    async def __call__(self, request: Request) -> str:
+        logger.info("Hello world!")
+        return "hi"
+
+
+say_hello = SayHello.bind()
+# __end__
+
+# serve.run(say_hello)
+
+# import requests
+# response = requests.get("http://localhost:8000/")
+# assert response.text == "hi"

doc/source/serve/doc_code/monitoring/request_id.py

@@ -0,0 +1,15 @@
+from ray import serve
+
+import requests
+
+
+@serve.deployment
+class Model:
+    def __call__(self) -> int:
+        return 1
+
+
+serve.run(Model.bind())
+resp = requests.get("http://localhost:8000", headers={"X-Request-ID": "123-234"})
+
+print(resp.headers["X-Request-ID"])

doc/source/serve/doc_code/multiplexed.py

@@ -0,0 +1,99 @@
+# __serve_deployment_example_begin__
+
+from ray import serve
+import aioboto3
+import torch
+import starlette
+
+
+@serve.deployment
+class ModelInferencer:
+    def __init__(self):
+        self.bucket_name = "my_bucket"
+
+    @serve.multiplexed(max_num_models_per_replica=3)
+    async def get_model(self, model_id: str):
+        session = aioboto3.Session()
+        async with session.resource("s3") as s3:
+            obj = await s3.Bucket(self.bucket_name)
+            await obj.download_file(f"{model_id}/model.pt", f"model_{model_id}.pt")
+            return torch.load(f"model_{model_id}.pt", weights_only=False)
+
+    async def __call__(self, request: starlette.requests.Request):
+        model_id = serve.get_multiplexed_model_id()
+        model = await self.get_model(model_id)
+        return model.forward(torch.rand(64, 3, 512, 512))
+
+
+entry = ModelInferencer.bind()
+
+# __serve_deployment_example_end__
+
+handle = serve.run(entry)
+
+# __serve_request_send_example_begin__
+import requests  # noqa: E402
+
+resp = requests.get(
+    "http://localhost:8000", headers={"serve_multiplexed_model_id": str("1")}
+)
+# __serve_request_send_example_end__
+
+# __serve_handle_send_example_begin__
+obj_ref = handle.options(multiplexed_model_id="1").remote("<your param>")
+# __serve_handle_send_example_end__
+
+
+from ray.serve.handle import DeploymentHandle  # noqa: E402
+
+
+# __serve_model_composition_example_begin__
+@serve.deployment
+class Downstream:
+    def __call__(self):
+        return serve.get_multiplexed_model_id()
+
+
+@serve.deployment
+class Upstream:
+    def __init__(self, downstream: DeploymentHandle):
+        self._h = downstream
+
+    async def __call__(self, request: starlette.requests.Request):
+        return await self._h.options(multiplexed_model_id="bar").remote()
+
+
+serve.run(Upstream.bind(Downstream.bind()))
+resp = requests.get("http://localhost:8000")
+# __serve_model_composition_example_end__
+
+
+# __serve_multiplexed_batching_example_begin__
+from typing import List  # noqa: E402
+from starlette.requests import Request
+
+
+@serve.deployment(max_ongoing_requests=15)
+class BatchedMultiplexModel:
+    @serve.multiplexed(max_num_models_per_replica=3)
+    async def get_model(self, model_id: str):
+        # Load and return your model here
+        return model_id
+
+    @serve.batch(max_batch_size=10, batch_wait_timeout_s=0.1)
+    async def batched_predict(self, inputs: List[str]) -> List[str]:
+        # Get the model ID - this works correctly inside batched functions
+        # because all requests in the batch target the same model
+        model_id = serve.get_multiplexed_model_id()
+        model = await self.get_model(model_id)
+
+        # Process the batch with the loaded model
+        return [f"{model}:{inp}" for inp in inputs]
+
+    async def __call__(self, request: Request):
+        # Extract input from the request body
+        input_text = await request.body()
+        return await self.batched_predict(input_text.decode())
+
+
+# __serve_multiplexed_batching_example_end__

doc/source/serve/doc_code/object_detection.py

@@ -0,0 +1,69 @@
+# __example_code_start__
+import torch
+from PIL import Image
+import numpy as np
+from io import BytesIO
+from fastapi.responses import Response
+from fastapi import FastAPI
+
+from ray import serve
+from ray.serve.handle import DeploymentHandle
+
+
+app = FastAPI()
+
+
+@serve.deployment(num_replicas=1)
+@serve.ingress(app)
+class APIIngress:
+    def __init__(self, object_detection_handle: DeploymentHandle):
+        self.handle = object_detection_handle
+
+    @app.get(
+        "/detect",
+        responses={200: {"content": {"image/jpeg": {}}}},
+        response_class=Response,
+    )
+    async def detect(self, image_url: str):
+        image = await self.handle.detect.remote(image_url)
+        file_stream = BytesIO()
+        image.save(file_stream, "jpeg")
+        return Response(content=file_stream.getvalue(), media_type="image/jpeg")
+
+
+@serve.deployment(
+    ray_actor_options={"num_gpus": 1},
+    autoscaling_config={"min_replicas": 1, "max_replicas": 2},
+)
+class ObjectDetection:
+    def __init__(self):
+        self.model = torch.hub.load("ultralytics/yolov5", "yolov5s")
+        self.model.cuda()
+        self.model.to(torch.device(0))
+
+    def detect(self, image_url: str):
+        result_im = self.model(image_url)
+        return Image.fromarray(result_im.render()[0].astype(np.uint8))
+
+
+entrypoint = APIIngress.bind(ObjectDetection.bind())
+
+
+# __example_code_end__
+
+if __name__ == "__main__":
+    import ray
+    import requests
+    import os
+
+    ray.init(runtime_env={"pip": ["seaborn", "ultralytics"]})
+
+    serve.run(entrypoint)
+    image_url = "https://ultralytics.com/images/zidane.jpg"
+    resp = requests.get(f"http://127.0.0.1:8000/detect?image_url={image_url}")
+
+    with open("output.jpeg", "wb") as f:
+        f.write(resp.content)
+
+    assert os.path.exists("output.jpeg")
+    os.remove("output.jpeg")

doc/source/serve/doc_code/pd_dissagregation/lmcache/mooncake.yaml

@@ -0,0 +1,17 @@
+# LMCache configuration for Mooncake store backend
+chunk_size: 256
+local_device: "cpu"
+remote_url: "mooncakestore://storage-server:49999/"
+remote_serde: "naive"
+pipelined_backend: false
+local_cpu: false
+max_local_cpu_size: 5
+extra_config:
+  local_hostname: "compute-node-001"
+  metadata_server: "etcd://metadata-server:2379"
+  protocol: "rdma"
+  device_name: "rdma0"
+  master_server_address: "storage-server:49999"
+  global_segment_size: 3355443200  # 3.125 GB
+  local_buffer_size: 1073741824    # 1 GB
+  transfer_timeout: 1

doc/source/serve/doc_code/pd_dissagregation/lmcache/nixl/decoder.yaml

@@ -0,0 +1,12 @@
+local_cpu: False
+max_local_cpu_size: 0
+max_local_disk_size: 0
+remote_serde: NULL
+
+enable_nixl: True
+nixl_role: "receiver"
+nixl_receiver_host: "localhost"
+nixl_receiver_port: 55555
+nixl_buffer_size: 1073741824 # 1GB
+nixl_buffer_device: "cuda"
+nixl_enable_gc: True

doc/source/serve/doc_code/pd_dissagregation/lmcache/nixl/prefiller.yaml

@@ -0,0 +1,12 @@
+local_cpu: False
+max_local_cpu_size: 0
+max_local_disk_size: 0
+remote_serde: NULL
+
+enable_nixl: True
+nixl_role: "sender"
+nixl_receiver_host: "localhost"
+nixl_receiver_port: 55555
+nixl_buffer_size: 1073741824 # 1GB
+nixl_buffer_device: "cuda"
+nixl_enable_gc: True

doc/source/serve/doc_code/pd_dissagregation/lmcache_mooncake_example.yaml

@@ -0,0 +1,34 @@
+# Example: LMCacheConnectorV1 with Mooncake store configuration
+
+applications:
+  - args:
+      prefill_config:
+        model_loading_config:
+          model_id: meta-llama/Llama-3.1-8B-Instruct
+        engine_kwargs:
+          kv_transfer_config: &kv_transfer_config
+            kv_connector: LMCacheConnectorV1
+            kv_role: kv_both
+        deployment_config:
+          autoscaling_config:
+            min_replicas: 2
+            max_replicas: 2
+        runtime_env: &runtime_env
+          env_vars:
+            LMCACHE_CONFIG_FILE: lmcache_mooncake.yaml
+            LMCACHE_USE_EXPERIMENTAL: "True"
+
+      decode_config:
+        model_loading_config:
+          model_id: meta-llama/Llama-3.1-8B-Instruct
+        engine_kwargs:
+          kv_transfer_config: *kv_transfer_config
+        deployment_config:
+          autoscaling_config:
+            min_replicas: 1
+            max_replicas: 1
+        runtime_env: *runtime_env
+
+    import_path: ray.serve.llm:build_pd_openai_app
+    name: pd-disaggregation-lmcache-mooncake
+    route_prefix: "/"

doc/source/serve/doc_code/pd_dissagregation/lmcache_nixl_example.yaml

@@ -0,0 +1,45 @@
+# Example: LMCacheConnectorV1 with NIXL backend configuration
+
+applications:
+  - args:
+      prefill_config:
+        model_loading_config:
+          model_id: meta-llama/Llama-3.1-8B-Instruct
+        engine_kwargs:
+          kv_transfer_config:
+            kv_connector: LMCacheConnectorV1
+            kv_role: kv_producer
+            kv_connector_extra_config:
+              discard_partial_chunks: false
+              lmcache_rpc_port: producer1
+        deployment_config:
+          autoscaling_config:
+            min_replicas: 2
+            max_replicas: 2
+        runtime_env:
+          env_vars:
+            LMCACHE_CONFIG_FILE: lmcache_prefiller.yaml
+            LMCACHE_USE_EXPERIMENTAL: "True"
+
+      decode_config:
+        model_loading_config:
+          model_id: meta-llama/Llama-3.1-8B-Instruct
+        engine_kwargs:
+          kv_transfer_config:
+            kv_connector: LMCacheConnectorV1
+            kv_role: kv_consumer
+            kv_connector_extra_config:
+              discard_partial_chunks: false
+              lmcache_rpc_port: consumer1
+        deployment_config:
+          autoscaling_config:
+            min_replicas: 6
+            max_replicas: 6
+        runtime_env:
+          env_vars:
+            LMCACHE_CONFIG_FILE: lmcache_decoder.yaml
+            LMCACHE_USE_EXPERIMENTAL: "True"
+
+    import_path: ray.serve.llm:build_pd_openai_app
+    name: pd-disaggregation-lmcache-nixl
+    route_prefix: "/"

doc/source/serve/doc_code/pd_dissagregation/nixl_example.yaml

@@ -0,0 +1,34 @@
+# Example: Basic NIXLConnector configuration for prefill/decode disaggregation
+# nixl_config.yaml
+
+applications:
+  - args:
+      prefill_config:
+        model_loading_config:
+          model_id: meta-llama/Llama-3.1-8B-Instruct
+        engine_kwargs:
+          kv_transfer_config:
+            kv_connector: NixlConnector
+            kv_role: kv_producer
+            engine_id: engine1
+        deployment_config:
+          autoscaling_config:
+            min_replicas: 2
+            max_replicas: 4
+      
+      decode_config:
+        model_loading_config:
+          model_id: meta-llama/Llama-3.1-8B-Instruct
+        engine_kwargs:
+          kv_transfer_config:
+            kv_connector: NixlConnector
+            kv_role: kv_consumer
+            engine_id: engine2
+        deployment_config:
+          autoscaling_config:
+            min_replicas: 6
+            max_replicas: 10
+
+    import_path: ray.serve.llm:build_pd_openai_app
+    name: pd-disaggregation-nixl
+    route_prefix: "/"

doc/source/serve/doc_code/production_guide/text_ml.py

@@ -0,0 +1,131 @@
+# flake8: noqa
+
+# __example_start__
+from starlette.requests import Request
+from typing import Dict
+
+import ray
+from ray import serve
+from ray.serve.handle import DeploymentHandle
+
+from transformers import AutoModelForSeq2SeqLM, AutoTokenizer
+
+
+@serve.deployment
+class Translator:
+    def __init__(self):
+        self.language = "french"
+        self.prefix = "translate English to French: "
+        self.tokenizer = AutoTokenizer.from_pretrained("t5-small")
+        self.model = AutoModelForSeq2SeqLM.from_pretrained("t5-small")
+
+    def translate(self, text: str) -> str:
+        input_ids = self.tokenizer(
+            f"{self.prefix}{text}", return_tensors="pt"
+        ).input_ids
+        output_ids = self.model.generate(
+            input_ids, num_beams=4, early_stopping=True, max_length=300
+        )
+
+        translation = self.tokenizer.decode(
+            output_ids[0], skip_special_tokens=True, clean_up_tokenization_spaces=False
+        )
+
+        return translation
+
+    def reconfigure(self, config: Dict):
+        self.language = config.get("language", "french")
+
+        if self.language.lower() == "french":
+            self.prefix = "translate English to French: "
+        elif self.language.lower() == "german":
+            self.prefix = "translate English to German: "
+        elif self.language.lower() == "romanian":
+            self.prefix = "translate English to Romanian: "
+        else:
+            pass
+
+
+@serve.deployment
+class Summarizer:
+    def __init__(self, translator: DeploymentHandle):
+        # Load model
+        self.tokenizer = AutoTokenizer.from_pretrained("t5-small")
+        self.model = AutoModelForSeq2SeqLM.from_pretrained("t5-small")
+        self.translator = translator
+        self.min_length = 5
+        self.max_length = 15
+
+    def summarize(self, text: str) -> str:
+        # Run inference
+        input_ids = self.tokenizer(f"summarize: {text}", return_tensors="pt").input_ids
+        output_ids = self.model.generate(
+            input_ids,
+            num_beams=4,
+            early_stopping=True,
+            min_length=self.min_length,
+            max_length=self.max_length,
+        )
+
+        # Post-process output to return only the summary text
+        summary = self.tokenizer.decode(
+            output_ids[0], skip_special_tokens=True, clean_up_tokenization_spaces=False
+        )
+
+        return summary
+
+    async def __call__(self, http_request: Request) -> str:
+        english_text: str = await http_request.json()
+        summary = self.summarize(english_text)
+
+        return await self.translator.translate.remote(summary)
+
+    def reconfigure(self, config: Dict):
+        self.min_length = config.get("min_length", 5)
+        self.max_length = config.get("max_length", 15)
+
+
+app = Summarizer.bind(Translator.bind())
+# __example_end__
+
+serve.run(app)
+
+# __start_client__
+import requests
+
+english_text = (
+    "It was the best of times, it was the worst of times, it was the age "
+    "of wisdom, it was the age of foolishness, it was the epoch of belief"
+)
+response = requests.post("http://127.0.0.1:8000/", json=english_text)
+french_text = response.text
+
+print(french_text)
+# 'C'était le meilleur des temps, c'était le pire des temps,'
+# __end_client__
+
+assert french_text == "C'était le meilleur des temps, c'était le pire des temps,"
+
+serve.run(
+    Summarizer.bind(Translator.options(user_config={"language": "german"}).bind())
+)
+
+
+# __start_second_client__
+import requests
+
+english_text = (
+    "It was the best of times, it was the worst of times, it was the age "
+    "of wisdom, it was the age of foolishness, it was the epoch of belief"
+)
+response = requests.post("http://127.0.0.1:8000/", json=english_text)
+german_text = response.text
+
+print(german_text)
+# 'es war die beste Zeit, es war die schlimmste Zeit,'
+# __end_second_client__
+
+assert german_text == "es war die beste Zeit, es war die schlimmste Zeit,"
+
+serve.shutdown()
+ray.shutdown()

doc/source/serve/doc_code/quickstart.py

@@ -0,0 +1,26 @@
+import requests
+from starlette.requests import Request
+from typing import Dict
+
+from ray import serve
+
+
+# 1: Define a Ray Serve application.
+@serve.deployment
+class MyModelDeployment:
+    def __init__(self, msg: str):
+        # Initialize model state: could be very large neural net weights.
+        self._msg = msg
+
+    def __call__(self, request: Request) -> Dict:
+        return {"result": self._msg}
+
+
+app = MyModelDeployment.bind(msg="Hello world!")
+
+# 2: Deploy the application locally.
+serve.run(app, route_prefix="/")
+
+# 3: Query the application and print the result.
+print(requests.get("http://localhost:8000/").json())
+# {'result': 'Hello world!'}

doc/source/serve/doc_code/quickstart_composed.py

@@ -0,0 +1,51 @@
+import requests
+import starlette
+from typing import Dict
+from ray import serve
+from ray.serve.handle import DeploymentHandle
+
+
+# 1. Define the models in our composition graph and an ingress that calls them.
+@serve.deployment
+class Adder:
+    def __init__(self, increment: int):
+        self.increment = increment
+
+    def add(self, inp: int):
+        return self.increment + inp
+
+
+@serve.deployment
+class Combiner:
+    def average(self, *inputs) -> float:
+        return sum(inputs) / len(inputs)
+
+
+@serve.deployment
+class Ingress:
+    def __init__(
+        self,
+        adder1: DeploymentHandle,
+        adder2: DeploymentHandle,
+        combiner: DeploymentHandle,
+    ):
+        self._adder1 = adder1
+        self._adder2 = adder2
+        self._combiner = combiner
+
+    async def __call__(self, request: starlette.requests.Request) -> Dict[str, float]:
+        input_json = await request.json()
+        final_result = await self._combiner.average.remote(
+            self._adder1.add.remote(input_json["val"]),
+            self._adder2.add.remote(input_json["val"]),
+        )
+        return {"result": final_result}
+
+
+# 2. Build the application consisting of the models and ingress.
+app = Ingress.bind(Adder.bind(increment=1), Adder.bind(increment=2), Combiner.bind())
+serve.run(app)
+
+# 3: Query the application and print the result.
+print(requests.post("http://localhost:8000/", json={"val": 100.0}).json())
+# {"result": 101.5}

doc/source/serve/doc_code/replica_rank.py

@@ -0,0 +1,102 @@
+# __replica_rank_start__
+from ray import serve
+
+
+@serve.deployment(num_replicas=4)
+class ModelShard:
+    def __call__(self):
+        context = serve.get_replica_context()
+        return {
+            "rank": context.rank.rank,  # Access the integer rank value
+            "world_size": context.world_size,
+        }
+
+
+app = ModelShard.bind()
+# __replica_rank_end__
+
+# __reconfigure_rank_start__
+from typing import Any
+from ray import serve
+from ray.serve.schema import ReplicaRank
+
+
+@serve.deployment(num_replicas=4, user_config={"name": "model_v1"})
+class RankAwareModel:
+    def __init__(self):
+        context = serve.get_replica_context()
+        self.rank = context.rank.rank  # Extract integer rank value
+        self.world_size = context.world_size
+        self.model_name = None
+        print(f"Replica rank: {self.rank}/{self.world_size}")
+
+    async def reconfigure(self, user_config: Any, rank: ReplicaRank):
+        """Called when user_config or rank changes."""
+        self.rank = rank.rank  # Extract integer rank value from ReplicaRank object
+        self.world_size = serve.get_replica_context().world_size
+        self.model_name = user_config.get("name")
+        print(f"Reconfigured: rank={self.rank}, model={self.model_name}")
+
+    def __call__(self):
+        return {"rank": self.rank, "model_name": self.model_name}
+
+
+app2 = RankAwareModel.bind()
+# __reconfigure_rank_end__
+
+if __name__ == "__main__":
+    # __replica_rank_start_run_main__
+    h = serve.run(app)
+    # Test that we can get rank information from replicas
+    seen_ranks = set()
+    for _ in range(20):
+        res = h.remote().result()
+        print(f"Output from __call__: {res}")
+        assert res["rank"] in [0, 1, 2, 3]
+        assert res["world_size"] == 4
+        seen_ranks.add(res["rank"])
+
+    # Verify we hit all replicas
+    print(f"Saw ranks: {sorted(seen_ranks)}")
+
+    # Output from __call__: {'rank': 2, 'world_size': 4}
+    # Output from __call__: {'rank': 1, 'world_size': 4}
+    # Output from __call__: {'rank': 3, 'world_size': 4}
+    # Output from __call__: {'rank': 0, 'world_size': 4}
+    # Output from __call__: {'rank': 0, 'world_size': 4}
+    # Output from __call__: {'rank': 0, 'world_size': 4}
+    # Output from __call__: {'rank': 0, 'world_size': 4}
+    # Output from __call__: {'rank': 3, 'world_size': 4}
+    # Output from __call__: {'rank': 1, 'world_size': 4}
+    # Output from __call__: {'rank': 1, 'world_size': 4}
+    # Output from __call__: {'rank': 0, 'world_size': 4}
+    # Output from __call__: {'rank': 1, 'world_size': 4}
+    # Output from __call__: {'rank': 3, 'world_size': 4}
+    # Output from __call__: {'rank': 2, 'world_size': 4}
+    # Output from __call__: {'rank': 0, 'world_size': 4}
+    # Output from __call__: {'rank': 0, 'world_size': 4}
+    # Output from __call__: {'rank': 2, 'world_size': 4}
+    # Output from __call__: {'rank': 1, 'world_size': 4}
+    # Output from __call__: {'rank': 3, 'world_size': 4}
+    # Output from __call__: {'rank': 0, 'world_size': 4}
+    # Saw ranks: [0, 1, 2, 3]
+
+    # __replica_rank_end_run_main__
+
+    # __reconfigure_rank_start_run_main__
+    h = serve.run(app2)
+    for _ in range(20):
+        res = h.remote().result()
+        assert res["rank"] in [0, 1, 2, 3]
+        assert res["model_name"] == "model_v1"
+        seen_ranks.add(res["rank"])
+
+    # (ServeReplica:default:RankAwareModel pid=1231505) Replica rank: 0/4
+    # (ServeReplica:default:RankAwareModel pid=1231505) Reconfigured: rank=0, model=model_v1
+    # (ServeReplica:default:RankAwareModel pid=1231504) Replica rank: 1/4
+    # (ServeReplica:default:RankAwareModel pid=1231504) Reconfigured: rank=1, model=model_v1
+    # (ServeReplica:default:RankAwareModel pid=1231502) Replica rank: 3/4
+    # (ServeReplica:default:RankAwareModel pid=1231502) Reconfigured: rank=3, model=model_v1
+    # (ServeReplica:default:RankAwareModel pid=1231503) Replica rank: 2/4
+    # (ServeReplica:default:RankAwareModel pid=1231503) Reconfigured: rank=2, model=model_v1
+    # __reconfigure_rank_end_run_main__

doc/source/serve/doc_code/replica_scheduling.py

@@ -0,0 +1,109 @@
+import ray
+
+# __max_replicas_per_node_start__
+from ray import serve
+
+
+@serve.deployment(num_replicas=6, max_replicas_per_node=2, ray_actor_options={"num_cpus": 0.1})
+class MyDeployment:
+    def __call__(self, request):
+        return "Hello!"
+
+
+app = MyDeployment.bind()
+# __max_replicas_per_node_end__
+
+# __placement_group_start__
+from ray import serve
+
+
+@serve.deployment(
+    ray_actor_options={"num_cpus": 0.1},
+    placement_group_bundles=[{"CPU": 0.1}, {"CPU": 0.1}],
+    placement_group_strategy="STRICT_PACK",
+)
+class MultiCPUModel:
+    def __call__(self, request):
+        return "Processed with 2 CPUs"
+
+
+multi_cpu_app = MultiCPUModel.bind()
+# __placement_group_end__
+
+# __placement_group_labels_start__
+@serve.deployment(
+    ray_actor_options={"num_cpus": 0.1},
+    placement_group_bundles=[{"CPU": 0.1, "GPU": 1}],
+    placement_group_bundle_label_selector=[
+        {"ray.io/accelerator-type": "A100"}
+    ]
+)
+def PlacementGroupBundleLabelSelector(request):
+    return "Running in PG on A100"
+
+pg_label_app = PlacementGroupBundleLabelSelector.bind()
+# __placement_group_labels_end__
+
+# __label_selectors_start__
+from ray import serve
+
+
+# Schedule only on nodes with A100 GPUs
+@serve.deployment(ray_actor_options={"label_selector": {"ray.io/accelerator-type": "A100"}})
+class A100Model:
+    def __call__(self, request):
+        return "Running on A100"
+
+
+# Schedule only on nodes with T4 GPUs
+@serve.deployment(ray_actor_options={"label_selector": {"ray.io/accelerator-type": "T4"}})
+class T4Model:
+    def __call__(self, request):
+        return "Running on T4"
+
+
+a100_app = A100Model.bind()
+t4_app = T4Model.bind()
+# __label_selectors_end__
+
+# __fallback_strategy_start__
+@serve.deployment(
+    ray_actor_options={
+        "label_selector": {"zone": "us-west-2a"},
+        "fallback_strategy": [{"label_selector": {"zone": "us-west-2b"}}]
+    }
+)
+class SoftAffinityDeployment:
+    def __call__(self, request):
+        return "Scheduling to a zone with soft constraints!"
+
+soft_affinity_app = SoftAffinityDeployment.bind()
+# __fallback_strategy_end__
+
+# __label_selector_main_start__
+if __name__ == "__main__":
+    # RayCluster with resources to run example tests.
+    ray.init(
+        labels={
+            "ray.io/accelerator-type": "A100",
+            "zone": "us-west-2b",
+        },
+        num_cpus=16,
+        num_gpus=1,
+        resources={"my_custom_resource": 10},
+    )
+
+    serve.run(a100_app, name="a100", route_prefix="/a100")
+# __label_selector_main_end__
+
+    # Run remaining doc code.
+    serve.run(MyDeployment.options(max_replicas_per_node=6).bind(), name="max_replicas", route_prefix="/max_replicas")
+
+    serve.run(multi_cpu_app, name="multi_cpu", route_prefix="/multi_cpu")
+
+    serve.run(pg_label_app, name="pg_label", route_prefix="/pg_label")
+
+    serve.run(soft_affinity_app, name="soft_affinity", route_prefix="/soft_affinity")
+
+    serve.shutdown()
+    ray.shutdown()

doc/source/serve/doc_code/requests_best_practices.py

@@ -0,0 +1,48 @@
+# flake8: noqa
+# fmt: off
+
+
+from ray import serve
+
+
+@serve.deployment
+def f(*args):
+    return "Hi there!"
+
+
+serve.run(f.bind())
+
+# __prototype_code_start__
+import requests
+
+response = requests.get("http://localhost:8000/")
+result = response.text
+# __prototype_code_end__
+
+assert result == "Hi there!"
+
+# __production_code_start__
+import requests
+from requests.adapters import HTTPAdapter, Retry
+
+session = requests.Session()
+
+retries = Retry(
+    total=5,  # 5 retries total
+    backoff_factor=1,  # Exponential backoff
+    status_forcelist=[  # Retry on server errors
+        500,
+        501,
+        502,
+        503,
+        504,
+    ],
+)
+
+session.mount("http://", HTTPAdapter(max_retries=retries))
+
+response = session.get("http://localhost:8000/", timeout=10)  # Add timeout
+result = response.text
+# __production_code_end__
+
+assert result == "Hi there!"

doc/source/serve/doc_code/resnet50_example.py

@@ -0,0 +1,66 @@
+# __serve_example_begin__
+import requests
+from io import BytesIO
+
+from PIL import Image
+import starlette.requests
+import torch
+from torchvision import transforms
+import torchvision.models as models
+from torchvision.models import ResNet50_Weights
+
+from ray import serve
+
+
+@serve.deployment(
+    ray_actor_options={"num_cpus": 1},
+    num_replicas="auto",
+)
+class Model:
+    def __init__(self):
+        self.resnet50 = (
+            models.resnet50(weights=ResNet50_Weights.DEFAULT).eval().to("cpu")
+        )
+        self.preprocess = transforms.Compose(
+            [
+                transforms.Resize(256),
+                transforms.CenterCrop(224),
+                transforms.ToTensor(),
+                transforms.Normalize(
+                    mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]
+                ),
+            ]
+        )
+        resp = requests.get(
+            "https://raw.githubusercontent.com/pytorch/hub/master/imagenet_classes.txt"
+        )
+        self.categories = resp.content.decode("utf-8").split("\n")
+
+    async def __call__(self, request: starlette.requests.Request) -> str:
+        uri = (await request.json())["uri"]
+        image_bytes = requests.get(uri).content
+        image = Image.open(BytesIO(image_bytes)).convert("RGB")
+
+        # Batch size is 1
+        input_tensor = torch.cat([self.preprocess(image).unsqueeze(0)]).to("cpu")
+        with torch.no_grad():
+            output = self.resnet50(input_tensor)
+            sm_output = torch.nn.functional.softmax(output[0], dim=0)
+        ind = torch.argmax(sm_output)
+        return self.categories[ind]
+
+
+app = Model.bind()
+# __serve_example_end__
+
+if __name__ == "__main__":
+    import requests  # noqa
+
+    serve.run(app)
+    resp = requests.post(
+        "http://localhost:8000/",
+        json={
+            "uri": "https://serve-resnet-benchmark-data.s3.us-west-1.amazonaws.com/000000000019.jpeg"  # noqa
+        },
+    )  # noqa
+    assert resp.text == "ox"

doc/source/serve/doc_code/scheduled_batch_processing.py

@@ -0,0 +1,32 @@
+# __serve_example_begin__
+import asyncio
+
+from ray import serve
+from ray.serve.config import AutoscalingConfig, AutoscalingPolicy
+
+
+@serve.deployment(
+    autoscaling_config=AutoscalingConfig(
+        min_replicas=1,
+        max_replicas=12,
+        policy=AutoscalingPolicy(
+            policy_function="autoscaling_policy:scheduled_batch_processing_policy"
+        ),
+    ),
+)
+class BatchProcessingDeployment:
+    async def __call__(self) -> str:
+        # Simulate batch processing work
+        await asyncio.sleep(0.5)
+        return "Hello, world!"
+
+
+app = BatchProcessingDeployment.bind()
+# __serve_example_end__
+
+if __name__ == "__main__":
+    import requests  # noqa
+
+    serve.run(app)
+    resp = requests.get("http://localhost:8000/")
+    assert resp.text == "Hello, world!"

doc/source/serve/doc_code/sklearn_quickstart.py

@@ -0,0 +1,44 @@
+# flake8: noqa
+# fmt: off
+
+# __serve_example_begin__
+import requests
+from starlette.requests import Request
+from typing import Dict
+
+from sklearn.datasets import load_iris
+from sklearn.ensemble import GradientBoostingClassifier
+
+from ray import serve
+
+
+# Train model.
+iris_dataset = load_iris()
+model = GradientBoostingClassifier()
+model.fit(iris_dataset["data"], iris_dataset["target"])
+
+
+@serve.deployment
+class BoostingModel:
+    def __init__(self, model):
+        self.model = model
+        self.label_list = iris_dataset["target_names"].tolist()
+
+    async def __call__(self, request: Request) -> Dict:
+        payload = (await request.json())["vector"]
+        print(f"Received http request with data {payload}")
+
+        prediction = self.model.predict([payload])[0]
+        human_name = self.label_list[prediction]
+        return {"result": human_name}
+
+
+# Deploy model.
+serve.run(BoostingModel.bind(model), route_prefix="/iris")
+
+# Query it!
+sample_request_input = {"vector": [1.2, 1.0, 1.1, 0.9]}
+response = requests.get(
+    "http://localhost:8000/iris", json=sample_request_input)
+print(response.text)
+# __serve_example_end__

doc/source/serve/doc_code/stable_diffusion.py

@@ -0,0 +1,88 @@
+# __example_code_start__
+
+from io import BytesIO
+from fastapi import FastAPI
+from fastapi.responses import Response
+import torch
+
+from ray import serve
+from ray.serve.handle import DeploymentHandle
+
+
+app = FastAPI()
+
+
+@serve.deployment(num_replicas=1)
+@serve.ingress(app)
+class APIIngress:
+    def __init__(self, diffusion_model_handle: DeploymentHandle) -> None:
+        self.handle = diffusion_model_handle
+
+    @app.get(
+        "/imagine",
+        responses={200: {"content": {"image/png": {}}}},
+        response_class=Response,
+    )
+    async def generate(self, prompt: str, img_size: int = 512):
+        assert len(prompt), "prompt parameter cannot be empty"
+
+        image = await self.handle.generate.remote(prompt, img_size=img_size)
+        file_stream = BytesIO()
+        image.save(file_stream, "PNG")
+        return Response(content=file_stream.getvalue(), media_type="image/png")
+
+
+@serve.deployment(
+    ray_actor_options={"num_gpus": 1},
+    autoscaling_config={"min_replicas": 0, "max_replicas": 2},
+)
+class StableDiffusionXL:
+    def __init__(self):
+        from diffusers import DiffusionPipeline
+
+        model_id = "stabilityai/stable-diffusion-xl-base-1.0"
+
+        self.pipe = DiffusionPipeline.from_pretrained(
+            model_id, torch_dtype=torch.float16, variant="fp16", use_safetensors=True
+        )
+        self.pipe = self.pipe.to("cuda")
+
+    def generate(self, prompt: str, img_size: int = 512):
+        assert len(prompt), "prompt parameter cannot be empty"
+
+        with torch.autocast("cuda"):
+            image = self.pipe(prompt, height=img_size, width=img_size).images[0]
+            return image
+
+
+entrypoint = APIIngress.bind(StableDiffusionXL.bind())
+
+# __example_code_end__
+
+
+if __name__ == "__main__":
+    import ray
+    import os
+    import requests
+
+    ray.init(
+        runtime_env={
+            "pip": [
+                "diffusers==0.33.1",
+                "transformers==4.51.3",
+            ]
+        }
+    )
+
+    handle = serve.run(entrypoint)
+    handle.generate.remote("hi").result()
+
+    prompt = "a cute cat is dancing on the grass."
+    prompt_query = "%20".join(prompt.split(" "))
+    resp = requests.get(f"http://127.0.0.1:8000/imagine?prompt={prompt_query}")
+
+    with open("output.png", "wb") as f:
+        f.write(resp.content)
+
+    assert os.path.exists("output.png")
+    os.remove("output.png")

doc/source/serve/doc_code/streaming_tutorial.py

@@ -0,0 +1,335 @@
+# flake8: noqa
+# fmt: off
+
+from typing import List
+
+# __textbot_setup_start__
+import asyncio
+import logging
+from queue import Empty
+
+from fastapi import FastAPI
+from starlette.responses import StreamingResponse
+from transformers import AutoModelForCausalLM, AutoTokenizer, TextIteratorStreamer
+
+from ray import serve
+
+logger = logging.getLogger("ray.serve")
+# __textbot_setup_end__
+
+
+# __textbot_constructor_start__
+fastapi_app = FastAPI()
+
+
+@serve.deployment
+@serve.ingress(fastapi_app)
+class Textbot:
+    def __init__(self, model_id: str):
+        self.loop = asyncio.get_running_loop()
+
+        self.model_id = model_id
+        self.model = AutoModelForCausalLM.from_pretrained(self.model_id)
+        self.tokenizer = AutoTokenizer.from_pretrained(self.model_id)
+
+    # __textbot_constructor_end__
+
+    # __textbot_logic_start__
+    @fastapi_app.post("/")
+    def handle_request(self, prompt: str) -> StreamingResponse:
+        logger.info(f'Got prompt: "{prompt}"')
+        streamer = TextIteratorStreamer(
+            self.tokenizer, timeout=0, skip_prompt=True, skip_special_tokens=True
+        )
+        self.loop.run_in_executor(None, self.generate_text, prompt, streamer)
+        return StreamingResponse(
+            self.consume_streamer(streamer), media_type="text/plain"
+        )
+
+    def generate_text(self, prompt: str, streamer: TextIteratorStreamer):
+        input_ids = self.tokenizer([prompt], return_tensors="pt").input_ids
+        self.model.generate(input_ids, streamer=streamer, max_length=10000)
+
+    async def consume_streamer(self, streamer: TextIteratorStreamer):
+        while True:
+            try:
+                for token in streamer:
+                    logger.info(f'Yielding token: "{token}"')
+                    yield token
+                break
+            except Empty:
+                # The streamer raises an Empty exception if the next token
+                # hasn't been generated yet. `await` here to yield control
+                # back to the event loop so other coroutines can run.
+                await asyncio.sleep(0.001)
+
+    # __textbot_logic_end__
+
+
+# __textbot_bind_start__
+app = Textbot.bind("microsoft/DialoGPT-small")
+# __textbot_bind_end__
+
+
+serve.run(app)
+
+chunks = []
+# __stream_client_start__
+import requests
+
+prompt = "Tell me a story about dogs."
+
+response = requests.post(f"http://localhost:8000/?prompt={prompt}", stream=True)
+response.raise_for_status()
+for chunk in response.iter_content(chunk_size=None, decode_unicode=True):
+    print(chunk, end="")
+
+    # Dogs are the best.
+    # __stream_client_end__
+    chunks.append(chunk)
+
+assert [c for c in chunks if c] == ["Dogs ", "are ", "the ", "best ", "."]
+
+
+# __chatbot_setup_start__
+import asyncio
+import logging
+from queue import Empty
+
+from fastapi import FastAPI, WebSocket, WebSocketDisconnect
+from transformers import AutoModelForCausalLM, AutoTokenizer, TextIteratorStreamer
+
+from ray import serve
+
+logger = logging.getLogger("ray.serve")
+# __chatbot_setup_end__
+
+
+# __chatbot_constructor_start__
+fastapi_app = FastAPI()
+
+
+@serve.deployment
+@serve.ingress(fastapi_app)
+class Chatbot:
+    def __init__(self, model_id: str):
+        self.loop = asyncio.get_running_loop()
+
+        self.model_id = model_id
+        self.model = AutoModelForCausalLM.from_pretrained(self.model_id)
+        self.tokenizer = AutoTokenizer.from_pretrained(self.model_id)
+
+    # __chatbot_constructor_end__
+
+    # __chatbot_logic_start__
+    @fastapi_app.websocket("/")
+    async def handle_request(self, ws: WebSocket) -> None:
+        await ws.accept()
+
+        conversation = ""
+        try:
+            while True:
+                prompt = await ws.receive_text()
+                logger.info(f'Got prompt: "{prompt}"')
+                conversation += prompt
+                streamer = TextIteratorStreamer(
+                    self.tokenizer,
+                    timeout=0,
+                    skip_prompt=True,
+                    skip_special_tokens=True,
+                )
+                self.loop.run_in_executor(
+                    None, self.generate_text, conversation, streamer
+                )
+                response = ""
+                async for text in self.consume_streamer(streamer):
+                    await ws.send_text(text)
+                    response += text
+                await ws.send_text("<<Response Finished>>")
+                conversation += response
+        except WebSocketDisconnect:
+            print("Client disconnected.")
+
+    def generate_text(self, prompt: str, streamer: TextIteratorStreamer):
+        input_ids = self.tokenizer([prompt], return_tensors="pt").input_ids
+        self.model.generate(input_ids, streamer=streamer, max_length=10000)
+
+    async def consume_streamer(self, streamer: TextIteratorStreamer):
+        while True:
+            try:
+                for token in streamer:
+                    logger.info(f'Yielding token: "{token}"')
+                    yield token
+                break
+            except Empty:
+                await asyncio.sleep(0.001)
+
+
+# __chatbot_logic_end__
+
+
+# __chatbot_bind_start__
+app = Chatbot.bind("microsoft/DialoGPT-small")
+# __chatbot_bind_end__
+
+serve.run(app)
+
+chunks = []
+# Monkeypatch `print` for testing
+original_print, print = print, (lambda chunk, end=None: chunks.append(chunk))
+
+# __ws_client_start__
+from websockets.sync.client import connect
+
+with connect("ws://localhost:8000") as websocket:
+    websocket.send("Space the final")
+    while True:
+        received = websocket.recv()
+        if received == "<<Response Finished>>":
+            break
+        print(received, end="")
+    print("\n")
+
+    websocket.send(" These are the voyages")
+    while True:
+        received = websocket.recv()
+        if received == "<<Response Finished>>":
+            break
+        print(received, end="")
+    print("\n")
+# __ws_client_end__
+
+assert [c for c in chunks if c] == [
+    " ", "frontier ", ".", "\n",
+    " ", "of ", "the ", "starship ", "Enterprise ", ".", "\n",
+]
+
+print = original_print
+
+# __batchbot_setup_start__
+import asyncio
+import logging
+from queue import Empty, Queue
+
+from fastapi import FastAPI
+from transformers import AutoModelForCausalLM, AutoTokenizer
+
+from ray import serve
+
+logger = logging.getLogger("ray.serve")
+# __batchbot_setup_end__
+
+# __raw_streamer_start__
+class RawStreamer:
+    def __init__(self, timeout: float = None):
+        self.q = Queue()
+        self.stop_signal = None
+        self.timeout = timeout
+
+    def put(self, values):
+        self.q.put(values)
+
+    def end(self):
+        self.q.put(self.stop_signal)
+
+    def __iter__(self):
+        return self
+
+    def __next__(self):
+        result = self.q.get(timeout=self.timeout)
+        if result == self.stop_signal:
+            raise StopIteration()
+        else:
+            return result
+
+
+# __raw_streamer_end__
+
+# __batchbot_constructor_start__
+fastapi_app = FastAPI()
+
+
+@serve.deployment
+@serve.ingress(fastapi_app)
+class Batchbot:
+    def __init__(self, model_id: str):
+        self.loop = asyncio.get_running_loop()
+
+        self.model_id = model_id
+        self.model = AutoModelForCausalLM.from_pretrained(self.model_id)
+        self.tokenizer = AutoTokenizer.from_pretrained(self.model_id)
+        self.tokenizer.pad_token = self.tokenizer.eos_token
+
+    # __batchbot_constructor_end__
+
+    # __batchbot_logic_start__
+    @fastapi_app.post("/")
+    async def handle_request(self, prompt: str) -> StreamingResponse:
+        logger.info(f'Got prompt: "{prompt}"')
+        return StreamingResponse(self.run_model(prompt), media_type="text/plain")
+
+    @serve.batch(max_batch_size=2, batch_wait_timeout_s=15)
+    async def run_model(self, prompts: List[str]):
+        streamer = RawStreamer()
+        self.loop.run_in_executor(None, self.generate_text, prompts, streamer)
+        on_prompt_tokens = True
+        async for decoded_token_batch in self.consume_streamer(streamer):
+            # The first batch of tokens contains the prompts, so we skip it.
+            if not on_prompt_tokens:
+                logger.info(f"Yielding decoded_token_batch: {decoded_token_batch}")
+                yield decoded_token_batch
+            else:
+                logger.info(f"Skipped prompts: {decoded_token_batch}")
+                on_prompt_tokens = False
+
+    def generate_text(self, prompts: str, streamer: RawStreamer):
+        input_ids = self.tokenizer(prompts, return_tensors="pt", padding=True).input_ids
+        self.model.generate(input_ids, streamer=streamer, max_length=10000)
+
+    async def consume_streamer(self, streamer: RawStreamer):
+        while True:
+            try:
+                for token_batch in streamer:
+                    decoded_tokens = []
+                    for token in token_batch:
+                        decoded_tokens.append(
+                            self.tokenizer.decode(token, skip_special_tokens=True)
+                        )
+                    logger.info(f"Yielding decoded tokens: {decoded_tokens}")
+                    yield decoded_tokens
+                break
+            except Empty:
+                await asyncio.sleep(0.001)
+
+
+# __batchbot_logic_end__
+
+
+# __batchbot_bind_start__
+app = Batchbot.bind("microsoft/DialoGPT-small")
+# __batchbot_bind_end__
+
+serve.run(app)
+
+# Test batching code
+from functools import partial
+from concurrent.futures.thread import ThreadPoolExecutor
+
+
+def get_buffered_response(prompt) -> List[str]:
+    response = requests.post(f"http://localhost:8000/?prompt={prompt}", stream=True)
+    chunks = []
+    for chunk in response.iter_content(chunk_size=None, decode_unicode=True):
+        chunks.append(chunk)
+    return chunks
+
+
+with ThreadPoolExecutor() as pool:
+    futs = [
+        pool.submit(partial(get_buffered_response, prompt))
+        for prompt in ["Introduce yourself to me!", "Tell me a story about dogs."]
+    ]
+    responses = [fut.result() for fut in futs]
+    assert len(responses) == 2 and all(
+        len(chunks) > 1 and "".join(chunks).strip() for chunks in responses
+    )

doc/source/serve/doc_code/test_service_pb2_grpc.py

@@ -0,0 +1,126 @@
+# flake8: noqa
+
+# Generated by the gRPC Python protocol compiler plugin. DO NOT EDIT!
+"""Client and server classes corresponding to protobuf-defined services."""
+import grpc
+
+import test_service_pb2 as src_dot_ray_dot_protobuf_dot_test__service__pb2
+
+
+class TestServiceStub(object):
+    """Missing associated documentation comment in .proto file."""
+
+    def __init__(self, channel):
+        """Constructor.
+
+        Args:
+            channel: A grpc.Channel.
+        """
+        self.Ping = channel.unary_unary(
+            "/ray.rpc.TestService/Ping",
+            request_serializer=src_dot_ray_dot_protobuf_dot_test__service__pb2.PingRequest.SerializeToString,
+            response_deserializer=src_dot_ray_dot_protobuf_dot_test__service__pb2.PingReply.FromString,
+        )
+        self.PingTimeout = channel.unary_unary(
+            "/ray.rpc.TestService/PingTimeout",
+            request_serializer=src_dot_ray_dot_protobuf_dot_test__service__pb2.PingTimeoutRequest.SerializeToString,
+            response_deserializer=src_dot_ray_dot_protobuf_dot_test__service__pb2.PingTimeoutReply.FromString,
+        )
+
+
+class TestServiceServicer(object):
+    """Missing associated documentation comment in .proto file."""
+
+    def Ping(self, request, context):
+        """Missing associated documentation comment in .proto file."""
+        context.set_code(grpc.StatusCode.UNIMPLEMENTED)
+        context.set_details("Method not implemented!")
+        raise NotImplementedError("Method not implemented!")
+
+    def PingTimeout(self, request, context):
+        """Missing associated documentation comment in .proto file."""
+        context.set_code(grpc.StatusCode.UNIMPLEMENTED)
+        context.set_details("Method not implemented!")
+        raise NotImplementedError("Method not implemented!")
+
+
+def add_TestServiceServicer_to_server(servicer, server):
+    rpc_method_handlers = {
+        "Ping": grpc.unary_unary_rpc_method_handler(
+            servicer.Ping,
+            request_deserializer=src_dot_ray_dot_protobuf_dot_test__service__pb2.PingRequest.FromString,
+            response_serializer=src_dot_ray_dot_protobuf_dot_test__service__pb2.PingReply.SerializeToString,
+        ),
+        "PingTimeout": grpc.unary_unary_rpc_method_handler(
+            servicer.PingTimeout,
+            request_deserializer=src_dot_ray_dot_protobuf_dot_test__service__pb2.PingTimeoutRequest.FromString,
+            response_serializer=src_dot_ray_dot_protobuf_dot_test__service__pb2.PingTimeoutReply.SerializeToString,
+        ),
+    }
+    generic_handler = grpc.method_handlers_generic_handler(
+        "ray.rpc.TestService", rpc_method_handlers
+    )
+    server.add_generic_rpc_handlers((generic_handler,))
+
+
+# This class is part of an EXPERIMENTAL API.
+class TestService(object):
+    """Missing associated documentation comment in .proto file."""
+
+    @staticmethod
+    def Ping(
+        request,
+        target,
+        options=(),
+        channel_credentials=None,
+        call_credentials=None,
+        insecure=False,
+        compression=None,
+        wait_for_ready=None,
+        timeout=None,
+        metadata=None,
+    ):
+        return grpc.experimental.unary_unary(
+            request,
+            target,
+            "/ray.rpc.TestService/Ping",
+            src_dot_ray_dot_protobuf_dot_test__service__pb2.PingRequest.SerializeToString,
+            src_dot_ray_dot_protobuf_dot_test__service__pb2.PingReply.FromString,
+            options,
+            channel_credentials,
+            insecure,
+            call_credentials,
+            compression,
+            wait_for_ready,
+            timeout,
+            metadata,
+        )
+
+    @staticmethod
+    def PingTimeout(
+        request,
+        target,
+        options=(),
+        channel_credentials=None,
+        call_credentials=None,
+        insecure=False,
+        compression=None,
+        wait_for_ready=None,
+        timeout=None,
+        metadata=None,
+    ):
+        return grpc.experimental.unary_unary(
+            request,
+            target,
+            "/ray.rpc.TestService/PingTimeout",
+            src_dot_ray_dot_protobuf_dot_test__service__pb2.PingTimeoutRequest.SerializeToString,
+            src_dot_ray_dot_protobuf_dot_test__service__pb2.PingTimeoutReply.FromString,
+            options,
+            channel_credentials,
+            insecure,
+            call_credentials,
+            compression,
+            wait_for_ready,
+            timeout,
+            metadata,
+        )

doc/source/serve/doc_code/transformers_example.py

@@ -0,0 +1,29 @@
+import requests
+from starlette.requests import Request
+from typing import Dict
+
+from transformers import pipeline
+
+from ray import serve
+
+
+# 1: Wrap the pretrained sentiment analysis model in a Serve deployment.
+@serve.deployment
+class SentimentAnalysisDeployment:
+    def __init__(self):
+        self._model = pipeline("sentiment-analysis")
+
+    def __call__(self, request: Request) -> Dict:
+        return self._model(request.query_params["text"])[0]
+
+
+# 2: Deploy the deployment.
+serve.run(SentimentAnalysisDeployment.bind(), route_prefix="/")
+
+# 3: Query the deployment and print the result.
+print(
+    requests.get(
+        "http://localhost:8000/", params={"text": "Ray Serve is great!"}
+    ).json()
+)
+# {'label': 'POSITIVE', 'score': 0.9998476505279541}

doc/source/serve/doc_code/translator_example.py

@@ -0,0 +1,47 @@
+import requests
+
+# __serve_example_begin__
+import starlette
+
+from transformers import AutoModelForSeq2SeqLM, AutoTokenizer
+
+from ray import serve
+
+
+@serve.deployment
+class Translator:
+    def __init__(self):
+        self.tokenizer = AutoTokenizer.from_pretrained("t5-small")
+        self.model = AutoModelForSeq2SeqLM.from_pretrained("t5-small")
+
+    def translate(self, text: str) -> str:
+        input_ids = self.tokenizer(
+            f"translate English to German: {text}", return_tensors="pt"
+        ).input_ids
+        output_ids = self.model.generate(
+            input_ids, num_beams=4, early_stopping=True, max_length=300
+        )
+        return self.tokenizer.decode(
+            output_ids[0], skip_special_tokens=True, clean_up_tokenization_spaces=False
+        )
+
+    async def __call__(self, req: starlette.requests.Request):
+        req = await req.json()
+        return self.translate(req["text"])
+
+
+app = Translator.bind()
+# __serve_example_end__
+
+
+serve.run(app, name="app2", route_prefix="/translate")
+
+# __request_begin__
+text = "Hello, the weather is quite fine today!"
+resp = requests.post("http://localhost:8000/translate", json={"text": text})
+
+print(resp.text)
+# 'Hallo, das Wetter ist heute ziemlich gut!'
+# __request_end__
+
+assert resp.text == "Hallo, das Wetter ist heute ziemlich gut!"

doc/source/serve/doc_code/tutorial_batch.py

@@ -0,0 +1,33 @@
+# fmt: off
+# __doc_import_begin__
+from typing import List
+
+from starlette.requests import Request
+from transformers import pipeline
+
+from ray import serve
+# __doc_import_end__
+# fmt: on
+
+
+# __doc_define_servable_begin__
+@serve.deployment
+class BatchTextGenerator:
+    def __init__(self, pipeline_key: str, model_key: str):
+        self.model = pipeline(pipeline_key, model_key)
+
+    @serve.batch(max_batch_size=4)
+    async def handle_batch(self, inputs: List[str]) -> List[str]:
+        print("Our input array has length:", len(inputs))
+
+        results = self.model(inputs)
+        return [result[0]["generated_text"] for result in results]
+
+    async def __call__(self, request: Request) -> List[str]:
+        return await self.handle_batch(request.query_params["text"])
+        # __doc_define_servable_end__
+
+
+# __doc_deploy_begin__
+generator = BatchTextGenerator.bind("text-generation", "gpt2")
+# __doc_deploy_end__

doc/source/serve/doc_code/tutorial_pytorch.py

@@ -0,0 +1,54 @@
+# fmt: off
+# __doc_import_begin__
+from ray import serve
+
+from io import BytesIO
+from PIL import Image
+from starlette.requests import Request
+from typing import Dict
+
+import torch
+from torchvision import transforms
+from torchvision.models import resnet18
+# __doc_import_end__
+# fmt: on
+
+
+# __doc_define_servable_begin__
+@serve.deployment
+class ImageModel:
+    def __init__(self):
+        self.model = resnet18(pretrained=True).eval()
+        self.preprocessor = transforms.Compose(
+            [
+                transforms.Resize(224),
+                transforms.CenterCrop(224),
+                transforms.ToTensor(),
+                transforms.Lambda(lambda t: t[:3, ...]),  # remove alpha channel
+                transforms.Normalize(
+                    mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]
+                ),
+            ]
+        )
+
+    async def __call__(self, starlette_request: Request) -> Dict:
+        image_payload_bytes = await starlette_request.body()
+        pil_image = Image.open(BytesIO(image_payload_bytes))
+        print("[1/3] Parsed image data: {}".format(pil_image))
+
+        pil_images = [pil_image]  # Our current batch size is one
+        input_tensor = torch.cat(
+            [self.preprocessor(i).unsqueeze(0) for i in pil_images]
+        )
+        print("[2/3] Images transformed, tensor shape {}".format(input_tensor.shape))
+
+        with torch.no_grad():
+            output_tensor = self.model(input_tensor)
+        print("[3/3] Inference done!")
+        return {"class_index": int(torch.argmax(output_tensor[0]))}
+        # __doc_define_servable_end__
+
+
+# __doc_deploy_begin__
+image_model = ImageModel.bind()
+# __doc_deploy_end__

doc/source/serve/doc_code/tutorial_sklearn.py

@@ -0,0 +1,81 @@
+# fmt: off
+# __doc_import_begin__
+from ray import serve
+
+import pickle
+import json
+import numpy as np
+import os
+import tempfile
+from starlette.requests import Request
+from typing import Dict
+
+from sklearn.datasets import load_iris
+from sklearn.ensemble import GradientBoostingClassifier
+from sklearn.metrics import mean_squared_error
+# __doc_import_end__
+# fmt: on
+
+# __doc_instantiate_model_begin__
+model = GradientBoostingClassifier()
+# __doc_instantiate_model_end__
+
+# __doc_data_begin__
+iris_dataset = load_iris()
+data, target, target_names = (
+    iris_dataset["data"],
+    iris_dataset["target"],
+    iris_dataset["target_names"],
+)
+
+np.random.shuffle(data)
+np.random.shuffle(target)
+train_x, train_y = data[:100], target[:100]
+val_x, val_y = data[100:], target[100:]
+# __doc_data_end__
+
+# __doc_train_model_begin__
+model.fit(train_x, train_y)
+print("MSE:", mean_squared_error(model.predict(val_x), val_y))
+
+# Save the model and label to file
+MODEL_PATH = os.path.join(
+    tempfile.gettempdir(), "iris_model_gradient_boosting_classifier.pkl"
+)
+LABEL_PATH = os.path.join(tempfile.gettempdir(), "iris_labels.json")
+
+with open(MODEL_PATH, "wb") as f:
+    pickle.dump(model, f)
+with open(LABEL_PATH, "w") as f:
+    json.dump(target_names.tolist(), f)
+# __doc_train_model_end__
+
+
+# __doc_define_servable_begin__
+@serve.deployment
+class BoostingModel:
+    def __init__(self, model_path: str, label_path: str):
+        with open(model_path, "rb") as f:
+            self.model = pickle.load(f)
+        with open(label_path) as f:
+            self.label_list = json.load(f)
+
+    async def __call__(self, starlette_request: Request) -> Dict:
+        payload = await starlette_request.json()
+        print("Worker: received starlette request with data", payload)
+
+        input_vector = [
+            payload["sepal length"],
+            payload["sepal width"],
+            payload["petal length"],
+            payload["petal width"],
+        ]
+        prediction = self.model.predict([input_vector])[0]
+        human_name = self.label_list[prediction]
+        return {"result": human_name}
+        # __doc_define_servable_end__
+
+
+# __doc_deploy_begin__
+boosting_model = BoostingModel.bind(MODEL_PATH, LABEL_PATH)
+# __doc_deploy_end__

doc/source/serve/doc_code/tutorial_tensorflow.py

@@ -0,0 +1,75 @@
+# fmt: off
+# __doc_import_begin__
+from ray import serve
+
+import os
+import tempfile
+import numpy as np
+from starlette.requests import Request
+from typing import Dict
+
+import tensorflow as tf
+# __doc_import_end__
+# fmt: on
+
+# __doc_train_model_begin__
+TRAINED_MODEL_PATH = os.path.join(tempfile.gettempdir(), "mnist_model.h5")
+
+
+def train_and_save_model():
+    # Load mnist dataset
+    mnist = tf.keras.datasets.mnist
+    (x_train, y_train), (x_test, y_test) = mnist.load_data()
+    x_train, x_test = x_train / 255.0, x_test / 255.0
+
+    # Train a simple neural net model
+    model = tf.keras.models.Sequential(
+        [
+            tf.keras.layers.Flatten(input_shape=(28, 28)),
+            tf.keras.layers.Dense(128, activation="relu"),
+            tf.keras.layers.Dropout(0.2),
+            tf.keras.layers.Dense(10),
+        ]
+    )
+    loss_fn = tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True)
+    model.compile(optimizer="adam", loss=loss_fn, metrics=["accuracy"])
+    model.fit(x_train, y_train, epochs=1)
+
+    model.evaluate(x_test, y_test, verbose=2)
+    model.summary()
+
+    # Save the model in h5 format in local file system
+    model.save(TRAINED_MODEL_PATH)
+
+
+if not os.path.exists(TRAINED_MODEL_PATH):
+    train_and_save_model()
+# __doc_train_model_end__
+
+
+# __doc_define_servable_begin__
+@serve.deployment
+class TFMnistModel:
+    def __init__(self, model_path: str):
+        import tensorflow as tf
+
+        self.model_path = model_path
+        self.model = tf.keras.models.load_model(model_path)
+
+    async def __call__(self, starlette_request: Request) -> Dict:
+        # Step 1: transform HTTP request -> tensorflow input
+        # Here we define the request schema to be a json array.
+        input_array = np.array((await starlette_request.json())["array"])
+        reshaped_array = input_array.reshape((1, 28, 28))
+
+        # Step 2: tensorflow input -> tensorflow output
+        prediction = self.model(reshaped_array)
+
+        # Step 3: tensorflow output -> web output
+        return {"prediction": prediction.numpy().tolist(), "file": self.model_path}
+        # __doc_define_servable_end__
+
+
+# __doc_deploy_begin__
+mnist_model = TFMnistModel.bind(TRAINED_MODEL_PATH)
+# __doc_deploy_end__

doc/source/serve/doc_code/varying_deps.py

@@ -0,0 +1,41 @@
+import requests
+from starlette.requests import Request
+
+from ray import serve
+from ray.serve.handle import DeploymentHandle
+
+
+@serve.deployment
+class Ingress:
+    def __init__(
+        self, ver_25_handle: DeploymentHandle, ver_26_handle: DeploymentHandle
+    ):
+        self.ver_25_handle = ver_25_handle
+        self.ver_26_handle = ver_26_handle
+
+    async def __call__(self, request: Request):
+        if request.query_params["version"] == "25":
+            return await self.ver_25_handle.remote()
+        else:
+            return await self.ver_26_handle.remote()
+
+
+@serve.deployment
+def requests_version():
+    return requests.__version__
+
+
+ver_25 = requests_version.options(
+    name="25",
+    ray_actor_options={"runtime_env": {"pip": ["requests==2.25.1"]}},
+).bind()
+ver_26 = requests_version.options(
+    name="26",
+    ray_actor_options={"runtime_env": {"pip": ["requests==2.26.0"]}},
+).bind()
+
+app = Ingress.bind(ver_25, ver_26)
+serve.run(app)
+
+assert requests.get("http://127.0.0.1:8000/?version=25").text == "2.25.1"
+assert requests.get("http://127.0.0.1:8000/?version=26").text == "2.26.0"

doc/source/serve/doc_code/vllm_example.py

@@ -0,0 +1,126 @@
+import json
+from typing import AsyncGenerator
+
+from fastapi import BackgroundTasks
+from starlette.requests import Request
+from starlette.responses import StreamingResponse, Response
+from vllm.engine.arg_utils import AsyncEngineArgs
+from vllm.engine.async_llm_engine import AsyncLLMEngine
+from vllm.sampling_params import SamplingParams
+from vllm.utils import random_uuid
+
+from ray import serve
+
+
+@serve.deployment(ray_actor_options={"num_gpus": 1})
+class VLLMPredictDeployment:
+    def __init__(self, **kwargs):
+        """
+        Construct a vLLM deployment.
+
+        Refer to https://github.com/vllm-project/vllm/blob/main/vllm/engine/arg_utils.py
+        for the full list of arguments.
+
+        Args:
+            model: name or path of the huggingface model to use
+            download_dir: directory to download and load the weights,
+                default to the default cache dir of huggingface.
+            use_np_weights: save a numpy copy of model weights for
+                faster loading. This can increase the disk usage by up to 2x.
+            use_dummy_weights: use dummy values for model weights.
+            dtype: data type for model weights and activations.
+                The "auto" option will use FP16 precision
+                for FP32 and FP16 models, and BF16 precision.
+                for BF16 models.
+            seed: random seed.
+            worker_use_ray: use Ray for distributed serving, will be
+                automatically set when using more than 1 GPU
+            pipeline_parallel_size: number of pipeline stages.
+            tensor_parallel_size: number of tensor parallel replicas.
+            block_size: token block size.
+            swap_space: CPU swap space size (GiB) per GPU.
+            gpu_memory_utilization: the percentage of GPU memory to be used for
+                the model executor
+            max_num_batched_tokens: maximum number of batched tokens per iteration
+            max_num_seqs: maximum number of sequences per iteration.
+            disable_log_stats: disable logging statistics.
+            engine_use_ray: use Ray to start the LLM engine in a separate
+                process as the server process.
+            disable_log_requests: disable logging requests.
+        """
+        args = AsyncEngineArgs(**kwargs)
+        self.engine = AsyncLLMEngine.from_engine_args(args)
+
+    async def stream_results(self, results_generator) -> AsyncGenerator[bytes, None]:
+        num_returned = 0
+        async for request_output in results_generator:
+            text_outputs = [output.text for output in request_output.outputs]
+            assert len(text_outputs) == 1
+            text_output = text_outputs[0][num_returned:]
+            ret = {"text": text_output}
+            yield (json.dumps(ret) + "\n").encode("utf-8")
+            num_returned += len(text_output)
+
+    async def may_abort_request(self, request_id) -> None:
+        await self.engine.abort(request_id)
+
+    async def __call__(self, request: Request) -> Response:
+        """Generate completion for the request.
+
+        The request should be a JSON object with the following fields:
+        - prompt: the prompt to use for the generation.
+        - stream: whether to stream the results or not.
+        - other fields: the sampling parameters (See `SamplingParams` for details).
+        """
+        request_dict = await request.json()
+        prompt = request_dict.pop("prompt")
+        stream = request_dict.pop("stream", False)
+        sampling_params = SamplingParams(**request_dict)
+        request_id = random_uuid()
+        results_generator = self.engine.generate(prompt, sampling_params, request_id)
+        if stream:
+            background_tasks = BackgroundTasks()
+            # Using background_taks to abort the request
+            # if the client disconnects.
+            background_tasks.add_task(self.may_abort_request, request_id)
+            return StreamingResponse(
+                self.stream_results(results_generator), background=background_tasks
+            )
+
+        # Non-streaming case
+        final_output = None
+        async for request_output in results_generator:
+            if await request.is_disconnected():
+                # Abort the request if the client disconnects.
+                await self.engine.abort(request_id)
+                return Response(status_code=499)
+            final_output = request_output
+
+        assert final_output is not None
+        prompt = final_output.prompt
+        text_outputs = [prompt + output.text for output in final_output.outputs]
+        ret = {"text": text_outputs}
+        return Response(content=json.dumps(ret))
+
+
+def send_sample_request():
+    import requests
+
+    prompt = "How do I cook fried rice?"
+    sample_input = {"prompt": prompt, "stream": True}
+    output = requests.post("http://localhost:8000/", json=sample_input)
+    for line in output.iter_lines():
+        print(line.decode("utf-8"))
+
+
+if __name__ == "__main__":
+    # To run this example, you need to install vllm which requires
+    # OS: Linux
+    # Python: 3.8 or higher
+    # CUDA: 11.0 – 11.8
+    # GPU: compute capability 7.0 or higher (e.g., V100, T4, RTX20xx, A100, L4, etc.)
+    # see https://vllm.readthedocs.io/en/latest/getting_started/installation.html
+    # for more details.
+    deployment = VLLMPredictDeployment.bind(model="facebook/opt-125m")
+    serve.run(deployment)
+    send_sample_request()

doc/source/serve/doc_code/whisper_example.py

@@ -0,0 +1,82 @@
+from ray import serve
+import re
+import subprocess
+from typing import List
+import starlette.requests
+
+
+def hard_normalize(word):
+    """Lower case the word and remove all non-alpha-numeric characters
+    from the entire word.
+    """
+
+    non_alpha_numeric = re.compile(r"[\W]+")
+    return non_alpha_numeric.sub("", word.lower())
+
+
+def clean_whisper_alignments(whisper_word_alignments: List[dict]) -> List[dict]:
+    """Change required to match gentle's tokenization with Whisper's word alignments"""
+
+    processed_words = []
+    for word_alignment in whisper_word_alignments:
+        if word_alignment.word == "%":
+            processed_words.append(word_alignment._replace(word=" percent"))
+        elif word_alignment.word[0] == "'" and len(processed_words) > 0:
+            # eg: "'Or" from ["d", "'Or"]
+            processed_words[-1]._replace(
+                word=processed_words[-1].word + word_alignment.word,
+                end=word_alignment.end,
+            )
+        elif hard_normalize(word_alignment.word) == "":
+            # eg: " -"
+            continue
+        else:
+            processed_words.append(word_alignment)
+
+    return processed_words
+
+
+@serve.deployment(ray_actor_options={"num_cpus": 1.0, "num_gpus": 1})
+class WhisperModel:
+    def __init__(self, model_size="large-v2"):
+        # Load model
+        from faster_whisper import WhisperModel
+
+        # Run on GPU with FP16
+        self.model = WhisperModel(model_size, device="cuda", compute_type="float16")
+
+    async def transcribe(self, file_path: str):
+        subprocess.check_call(["curl", "-o", "audio.mp3", "-sSfLO", file_path])
+
+        segments, info = self.model.transcribe(
+            "audio.mp3",
+            language="en",
+            initial_prompt="Here is the um, uh, Um, Uh, transcript.",
+            best_of=5,
+            beam_size=5,
+            word_timestamps=True,
+        )
+
+        whisper_alignments = []
+        transcript_text = ""
+        for seg in segments:
+            transcript_text += seg.text
+            whisper_alignments += clean_whisper_alignments(seg.words)
+
+        # Transcript change required to match gentle's tokenization with
+        # Whisper's word alignments
+        transcript_text = transcript_text.replace("% ", " percent ")
+        return {
+            "language": info.language,
+            "language_probability": info.language_probability,
+            "duration": info.duration,
+            "transcript_text": transcript_text,
+            "whisper_alignments": whisper_alignments,
+        }
+
+    async def __call__(self, req: starlette.requests.Request):
+        request = await req.json()
+        return await self.transcribe(file_path=request["filepath"])
+
+
+entrypoint = WhisperModel.bind()