chore: import upstream snapshot with attribution

doc/source/llm/doc_code/serve/custom_vllm/.gitignore

@@ -0,0 +1,3 @@
+build/
+*.egg-info/
+__pycache__/

doc/source/llm/doc_code/serve/custom_vllm/custom_vllm_config.yaml

@@ -0,0 +1,20 @@
+# config.yaml
+applications:
+- args:
+    llm_configs:
+        - model_loading_config:
+            model_id: qwen3-reward
+            model_source: Qwen/Qwen3-0.6B
+          # Register the plugin architecture in the Ray Serve LLM processes.
+          server_cls: qwen3_reward_plugin.serve_hook:RewardModelServer
+          engine_kwargs:
+            runner: pooling
+            hf_overrides:
+                architectures:
+                    - Qwen3CustomRewardModel
+                num_labels: 1
+                problem_type: regression
+            max_model_len: 4096
+  import_path: ray.serve.llm:build_openai_app
+  name: custom_vllm_app
+  route_prefix: "/"

doc/source/llm/doc_code/serve/custom_vllm/custom_vllm_example.py

@@ -0,0 +1,91 @@
+"""Documentation example and CI test: custom vLLM model with Ray Serve LLM.
+
+Structure:
+1. Test-only setup: force serve.run non-blocking. The plugin is baked into the
+   cluster image, and direct streaming is enabled through the cluster
+   environment.
+2. Docs example (between __custom_vllm_example_start/end__): embedded in the
+   guide via literalinclude.
+3. Test validation (deployment status polling + reward-head assertion + cleanup).
+"""
+
+import json
+import time
+import urllib.request
+
+from ray import serve
+from ray.serve._private.constants import SERVE_DEFAULT_APP_NAME
+from ray.serve.schema import ApplicationStatus
+
+_original_serve_run = serve.run
+
+def _non_blocking_serve_run(app, **kwargs):
+    """Forces blocking=False for testing."""
+    kwargs["blocking"] = False
+    return _original_serve_run(app, **kwargs)
+
+serve.run = _non_blocking_serve_run
+
+# __custom_vllm_example_start__
+from ray import serve
+from ray.serve.llm import LLMConfig, build_openai_app
+
+# Serve Qwen3-0.6B with a scalar reward head from the `qwen3_reward_plugin`
+# vLLM plugin (pip install it into your cluster image).
+llm_config = LLMConfig(
+    model_loading_config=dict(
+        model_id="qwen3-reward",
+        model_source="Qwen/Qwen3-0.6B",
+    ),
+    # vLLM's entry point registers the architecture in the engine and worker
+    # processes. Ray Serve LLM also resolves it while building the engine config.
+    server_cls="qwen3_reward_plugin.serve_hook:RewardModelServer",
+    engine_kwargs=dict(
+        runner="pooling",
+        hf_overrides=dict(
+            architectures=["Qwen3CustomRewardModel"],
+            num_labels=1,
+            problem_type="regression",
+        ),
+        max_model_len=4096,
+    ),
+)
+
+# Requires direct streaming so vLLM's native /classify route is exposed.
+app = build_openai_app({"llm_configs": [llm_config]})
+serve.run(app, blocking=True)
+# __custom_vllm_example_end__
+
+status = ApplicationStatus.NOT_STARTED
+timeout_seconds = 300
+start_time = time.time()
+
+while (
+    status != ApplicationStatus.RUNNING and time.time() - start_time < timeout_seconds
+):
+    status = serve.status().applications[SERVE_DEFAULT_APP_NAME].status
+    if status in [ApplicationStatus.DEPLOY_FAILED, ApplicationStatus.UNHEALTHY]:
+        raise AssertionError(f"Deployment failed with status: {status}")
+    time.sleep(1)
+
+if status != ApplicationStatus.RUNNING:
+    raise AssertionError(
+        f"Deployment failed to reach RUNNING status within {timeout_seconds}s. "
+        f"Current status: {status}"
+    )
+
+# Verify the reward head ran end to end: /classify returns a single scalar score
+# per input (num_labels=1), read from data[0].probs.
+body = json.dumps(
+    {"model": "qwen3-reward", "input": "The capital of France is Paris."}
+).encode()
+request = urllib.request.Request(
+    "http://localhost:8000/classify",
+    data=body,
+    headers={"Content-Type": "application/json"},
+)
+response = json.load(urllib.request.urlopen(request, timeout=60))
+reward = response["data"][0]["probs"]
+assert len(reward) == 1, f"Expected a scalar reward, got {len(reward)} values"
+
+serve.shutdown()

doc/source/llm/doc_code/serve/custom_vllm/custom_vllm_yaml_config_example.py

@@ -0,0 +1,63 @@
+"""Documentation example and CI test: custom vLLM model via a YAML config.
+
+Structure:
+1. Test-only setup: force serve.run non-blocking and strip accelerator
+   requirements for CI. The plugin is baked into the cluster image, and direct
+   streaming is enabled through the cluster environment.
+2. Load the YAML config and deploy it with build_openai_app.
+3. Test validation (deployment status polling + reward-head assertion + cleanup).
+"""
+
+import json
+import os
+import time
+import urllib.request
+
+import yaml
+
+from ray import serve
+from ray.serve import llm
+from ray.serve._private.constants import SERVE_DEFAULT_APP_NAME
+from ray.serve.schema import ApplicationStatus
+
+config_path = os.path.join(os.path.dirname(__file__), "custom_vllm_config.yaml")
+with open(config_path, "r") as f:
+    config_dict = yaml.safe_load(f)
+
+llm_configs = config_dict["applications"][0]["args"]["llm_configs"]
+app = llm.build_openai_app({"llm_configs": llm_configs})
+serve.run(app, blocking=False)
+
+status = ApplicationStatus.NOT_STARTED
+timeout_seconds = 300
+start_time = time.time()
+
+while (
+    status != ApplicationStatus.RUNNING and time.time() - start_time < timeout_seconds
+):
+    status = serve.status().applications[SERVE_DEFAULT_APP_NAME].status
+    if status in [ApplicationStatus.DEPLOY_FAILED, ApplicationStatus.UNHEALTHY]:
+        raise AssertionError(f"Deployment failed with status: {status}")
+    time.sleep(1)
+
+if status != ApplicationStatus.RUNNING:
+    raise AssertionError(
+        f"Deployment failed to reach RUNNING status within {timeout_seconds}s. "
+        f"Current status: {status}"
+    )
+
+# Verify the reward head ran end to end: /classify returns a single scalar score
+# per input (num_labels=1), read from data[0].probs.
+body = json.dumps(
+    {"model": "qwen3-reward", "input": "The capital of France is Paris."}
+).encode()
+request = urllib.request.Request(
+    "http://localhost:8000/classify",
+    data=body,
+    headers={"Content-Type": "application/json"},
+)
+response = json.load(urllib.request.urlopen(request, timeout=60))
+reward = response["data"][0]["probs"]
+assert len(reward) == 1, f"Expected a scalar reward, got {len(reward)} values"
+
+serve.shutdown()

doc/source/llm/doc_code/serve/custom_vllm/qwen3_reward_plugin/__init__.py

@@ -0,0 +1,11 @@
+# __register_start__
+"""vLLM plugin entry point: register the custom architecture."""
+def register() -> None:
+    from vllm import ModelRegistry
+
+    if "Qwen3CustomRewardModel" not in ModelRegistry.get_supported_archs():
+        ModelRegistry.register_model(
+            "Qwen3CustomRewardModel",
+            "qwen3_reward_plugin.qwen3_rm:Qwen3CustomRewardModel",
+        )
+# __register_end__

doc/source/llm/doc_code/serve/custom_vllm/qwen3_reward_plugin/qwen3_rm.py

@@ -0,0 +1,91 @@
+"""Qwen3 with the LM head replaced by a scalar reward head, as a vLLM plugin."""
+
+import os
+from collections.abc import Iterable
+
+import torch
+
+from vllm.config import VllmConfig
+from vllm.logger import init_logger
+from vllm.model_executor.layers.linear import ReplicatedLinear
+from vllm.model_executor.layers.pooler import DispatchPooler
+from vllm.model_executor.model_loader.weight_utils import default_weight_loader
+from vllm.model_executor.models.interfaces_base import default_pooling_type
+from vllm.model_executor.models.qwen3 import Qwen3ForCausalLM
+from vllm.model_executor.models.utils import AutoWeightsLoader, maybe_prefix
+
+logger = init_logger(__name__)
+
+REWARD_HEAD_FILE = "reward_head.pt"
+REWARD_HEAD_KEYS = ("linear.weight", "weight", "score.weight")
+
+
+@default_pooling_type(seq_pooling_type="LAST")
+class Qwen3CustomRewardModel(Qwen3ForCausalLM):
+    """Qwen3 backbone + scalar reward head, served as a pooling model."""
+
+    is_pooling_model = True
+
+    def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
+        # A single scalar output (set problem_type="regression" in the config
+        # for an identity activation, so /classify returns the raw reward).
+        vllm_config.model_config.hf_config.num_labels = 1
+        super().__init__(vllm_config=vllm_config, prefix=prefix)
+
+        model_config = vllm_config.model_config
+
+        # Scalar reward head. Replicated because the output dim is 1.
+        self.score = ReplicatedLinear(
+            model_config.hf_config.hidden_size,
+            1,
+            bias=False,
+            params_dtype=model_config.head_dtype,
+            return_bias=False,
+            prefix=maybe_prefix(prefix, "score"),
+        )
+
+        # LAST-token pooling -> reward head -> raw score. Advertising the
+        # "classify" task mounts vLLM's /classify route.
+        pooler_config = model_config.pooler_config
+        assert pooler_config is not None
+        self.pooler = DispatchPooler.for_seq_cls(pooler_config, classifier=self.score)
+
+    def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
+        skip_prefixes = ["score."]  # reward head loads separately, below
+        if self.config.tie_word_embeddings:
+            skip_prefixes.append("lm_head.")  # tied: no lm_head weight to load
+        loader = AutoWeightsLoader(self, skip_prefixes=skip_prefixes)
+        loaded = loader.load_weights(weights)
+        if self._load_reward_head():
+            loaded.add("score.weight")
+        return loaded
+
+    def _load_reward_head(self) -> bool:
+        """Load ``reward_head.pt``; return False (keep init values) if absent."""
+        env_path = os.environ.get("RM_REWARD_HEAD_PATH")
+        model_dir = self.vllm_config.model_config.model
+        head_path = (
+            env_path
+            if env_path and os.path.isfile(env_path)
+            else os.path.join(model_dir, REWARD_HEAD_FILE)
+        )
+        if not os.path.isfile(head_path):
+            logger.warning(
+                "Reward head %r not found; scores are not meaningful.", head_path
+            )
+            return False
+
+        state = torch.load(head_path, map_location="cpu", weights_only=True)
+        weight = (
+            state
+            if isinstance(state, torch.Tensor)
+            else next((state[k] for k in REWARD_HEAD_KEYS if k in state), None)
+        )
+        if weight is None:
+            raise KeyError(f"{REWARD_HEAD_FILE} has none of {REWARD_HEAD_KEYS}")
+
+        param = self.score.weight
+        weight_loader = getattr(param, "weight_loader", default_weight_loader)
+        weight_loader(param, weight.reshape(1, -1).to(param.dtype))
+        logger.info("Loaded reward head from %s.", head_path)
+        return True

doc/source/llm/doc_code/serve/custom_vllm/qwen3_reward_plugin/serve_hook.py

@@ -0,0 +1,13 @@
+# __serve_hook_start__
+"""LLMServer subclass for ``LLMConfig.server_cls``: importing it registers the plugin."""
+
+from ray.llm._internal.serve.core.server.llm_server import LLMServer
+
+from qwen3_reward_plugin import register
+
+register()
+
+
+class RewardModelServer(LLMServer):
+    pass
+# __serve_hook_end__

doc/source/llm/doc_code/serve/custom_vllm/setup.py

@@ -0,0 +1,16 @@
+# __setup_start__
+from setuptools import setup
+
+setup(
+    name="qwen3-reward-plugin",
+    version="0.1.0",
+    packages=["qwen3_reward_plugin"],
+    # vLLM discovers and runs this entry point in every process (the engine core
+    # and each rank worker) via load_general_plugins().
+    entry_points={
+        "vllm.general_plugins": [
+            "qwen3_reward = qwen3_reward_plugin:register",
+        ],
+    },
+)
+# __setup_end__

doc/source/llm/doc_code/serve/direct_streaming/direct_streaming_config.yaml

@@ -0,0 +1,20 @@
+# config.yaml
+applications:
+  - name: llm-direct-streaming
+    route_prefix: /
+    import_path: ray.serve.llm:build_openai_app
+    args:
+      llm_configs:
+        - model_loading_config:
+            model_id: qwen3.5-0.8b
+            model_source: Qwen/Qwen3.5-0.8B
+          deployment_config:
+            autoscaling_config:
+              min_replicas: 1
+              max_replicas: 4
+          engine_kwargs:
+            trust_remote_code: true
+            tensor_parallel_size: 1
+            enable_auto_tool_choice: true
+            tool_call_parser: qwen3_coder
+            reasoning_parser: qwen3

doc/source/llm/doc_code/serve/direct_streaming/direct_streaming_custom_router_example.py

@@ -0,0 +1,88 @@
+"""
+This file serves as a documentation example and CI test.
+
+Structure:
+1. Monkeypatch setup: Ensures serve.run is non-blocking and removes accelerator requirements for CI testing.
+2. Docs example (between __direct_streaming_custom_router_example_start/end__): Embedded in Sphinx docs via literalinclude.
+3. Test validation (deployment status polling + cleanup)
+
+Scope: validates the documented request_router_config snippet (config and the
+build_openai_app call signature) for direct streaming.
+"""
+
+import time
+from ray import serve
+from ray.serve.schema import ApplicationStatus
+from ray.serve._private.constants import SERVE_DEFAULT_APP_NAME
+from ray.serve import llm
+
+_original_serve_run = serve.run
+_original_build_openai_app = llm.build_openai_app
+
+
+def _non_blocking_serve_run(app, **kwargs):
+    """Forces blocking=False for testing"""
+    kwargs["blocking"] = False
+    return _original_serve_run(app, **kwargs)
+
+
+def _testing_build_openai_app(llm_serving_args):
+    """Removes accelerator requirements for testing"""
+    for config in llm_serving_args["llm_configs"]:
+        config.accelerator_type = None
+
+    return _original_build_openai_app(llm_serving_args)
+
+
+serve.run = _non_blocking_serve_run
+llm.build_openai_app = _testing_build_openai_app
+
+# __direct_streaming_custom_router_example_start__
+from ray.serve.config import RequestRouterConfig
+from ray.serve.llm import LLMConfig
+
+llm_config = LLMConfig(
+    model_loading_config={
+        "model_id": "qwen3.5-0.8b",
+        "model_source": "Qwen/Qwen3.5-0.8B",
+    },
+    deployment_config={
+        "request_router_config": RequestRouterConfig(
+            request_router_class="ray.serve.experimental.consistent_hash_router.ConsistentHashRouter",
+        ),
+    },
+    engine_kwargs={
+        "trust_remote_code": True,
+        "tensor_parallel_size": 1,
+        "enable_auto_tool_choice": True,
+        "tool_call_parser": "qwen3_coder",
+        "reasoning_parser": "qwen3",
+    },
+)
+# __direct_streaming_custom_router_example_end__
+
+from ray.serve.llm import build_openai_app
+
+app = build_openai_app({"llm_configs": [llm_config]})
+serve.run(app)
+
+status = ApplicationStatus.NOT_STARTED
+timeout_seconds = 180
+start_time = time.time()
+
+while (
+    status != ApplicationStatus.RUNNING and time.time() - start_time < timeout_seconds
+):
+    status = serve.status().applications[SERVE_DEFAULT_APP_NAME].status
+
+    if status in [ApplicationStatus.DEPLOY_FAILED, ApplicationStatus.UNHEALTHY]:
+        raise AssertionError(f"Deployment failed with status: {status}")
+
+    time.sleep(1)
+
+if status != ApplicationStatus.RUNNING:
+    raise AssertionError(
+        f"Deployment failed to reach RUNNING status within {timeout_seconds}s. Current status: {status}"
+    )
+
+serve.shutdown()

doc/source/llm/doc_code/serve/direct_streaming/direct_streaming_example.py

@@ -0,0 +1,88 @@
+"""
+This file serves as a documentation example and CI test.
+
+Structure:
+1. Monkeypatch setup: Ensures serve.run is non-blocking and removes accelerator requirements for CI testing.
+2. Docs example (between __direct_streaming_example_start/end__): Embedded in Sphinx docs via literalinclude.
+3. Test validation (deployment status polling + cleanup)
+
+Scope: builds and deploys the documented snippet with direct streaming enabled.
+The bazel target sets ``RAY_SERVE_ENABLE_HA_PROXY`` and
+``RAY_SERVE_LLM_ENABLE_DIRECT_STREAMING``, so build_openai_app returns the
+direct streaming deployment and the app reaches RUNNING on the HAProxy ingress.
+The end-to-end streaming data path is covered by the openai-compatibility
+direct-streaming test suite.
+"""
+
+import time
+from ray import serve
+from ray.serve.schema import ApplicationStatus
+from ray.serve._private.constants import SERVE_DEFAULT_APP_NAME
+from ray.serve import llm
+
+_original_serve_run = serve.run
+_original_build_openai_app = llm.build_openai_app
+
+
+def _non_blocking_serve_run(app, **kwargs):
+    """Forces blocking=False for testing"""
+    kwargs["blocking"] = False
+    return _original_serve_run(app, **kwargs)
+
+
+def _testing_build_openai_app(llm_serving_args):
+    """Removes accelerator requirements for testing"""
+    for config in llm_serving_args["llm_configs"]:
+        config.accelerator_type = None
+
+    return _original_build_openai_app(llm_serving_args)
+
+
+serve.run = _non_blocking_serve_run
+llm.build_openai_app = _testing_build_openai_app
+
+# __direct_streaming_example_start__
+from ray import serve
+from ray.serve.llm import LLMConfig, build_openai_app
+
+llm_config = LLMConfig(
+    model_loading_config={
+        "model_id": "qwen3.5-0.8b",
+        "model_source": "Qwen/Qwen3.5-0.8B",
+    },
+    deployment_config={
+        "autoscaling_config": {"min_replicas": 1, "max_replicas": 4},
+    },
+    engine_kwargs={
+        "trust_remote_code": True,
+        "tensor_parallel_size": 1,
+        "enable_auto_tool_choice": True,
+        "tool_call_parser": "qwen3_coder",
+        "reasoning_parser": "qwen3",
+    },
+)
+
+app = build_openai_app({"llm_configs": [llm_config]})
+serve.run(app)
+# __direct_streaming_example_end__
+
+status = ApplicationStatus.NOT_STARTED
+timeout_seconds = 180
+start_time = time.time()
+
+while (
+    status != ApplicationStatus.RUNNING and time.time() - start_time < timeout_seconds
+):
+    status = serve.status().applications[SERVE_DEFAULT_APP_NAME].status
+
+    if status in [ApplicationStatus.DEPLOY_FAILED, ApplicationStatus.UNHEALTHY]:
+        raise AssertionError(f"Deployment failed with status: {status}")
+
+    time.sleep(1)
+
+if status != ApplicationStatus.RUNNING:
+    raise AssertionError(
+        f"Deployment failed to reach RUNNING status within {timeout_seconds}s. Current status: {status}"
+    )
+
+serve.shutdown()

doc/source/llm/doc_code/serve/direct_streaming/direct_streaming_yaml_config_example.py

@@ -0,0 +1,62 @@
+"""
+This file serves as a documentation example and CI test for the direct
+streaming YAML config.
+
+Structure:
+1. Load the YAML config shown in the guide and convert it to an app with
+   build_openai_app, removing accelerator requirements for CI testing.
+2. Test validation (deployment status polling + cleanup).
+
+Scope: validates that the documented config.yaml parses into a valid
+LLMServingArgs and deploys with direct streaming enabled. The bazel target sets
+RAY_SERVE_ENABLE_HA_PROXY and RAY_SERVE_LLM_ENABLE_DIRECT_STREAMING, so the app
+reaches RUNNING on the HAProxy ingress. The end-to-end streaming data path is
+covered by the openai-compatibility direct-streaming test suite.
+"""
+
+import time
+import os
+import yaml
+from ray import serve
+from ray.serve.schema import ApplicationStatus
+from ray.serve._private.constants import SERVE_DEFAULT_APP_NAME
+from ray.serve import llm
+
+
+config_path = os.path.join(os.path.dirname(__file__), "direct_streaming_config.yaml")
+with open(config_path, "r") as f:
+    config_dict = yaml.safe_load(f)
+
+llm_configs = config_dict["applications"][0]["args"]["llm_configs"]
+for config in llm_configs:
+    config.pop("accelerator_type", None)
+    # Disable compile cache to avoid cache corruption in CI
+    if "runtime_env" not in config:
+        config["runtime_env"] = {}
+    if "env_vars" not in config["runtime_env"]:
+        config["runtime_env"]["env_vars"] = {}
+    config["runtime_env"]["env_vars"]["VLLM_DISABLE_COMPILE_CACHE"] = "1"
+
+app = llm.build_openai_app({"llm_configs": llm_configs})
+serve.run(app, blocking=False)
+
+status = ApplicationStatus.NOT_STARTED
+timeout_seconds = 180
+start_time = time.time()
+
+while (
+    status != ApplicationStatus.RUNNING and time.time() - start_time < timeout_seconds
+):
+    status = serve.status().applications[SERVE_DEFAULT_APP_NAME].status
+
+    if status in [ApplicationStatus.DEPLOY_FAILED, ApplicationStatus.UNHEALTHY]:
+        raise AssertionError(f"Deployment failed with status: {status}")
+
+    time.sleep(1)
+
+if status != ApplicationStatus.RUNNING:
+    raise AssertionError(
+        f"Deployment failed to reach RUNNING status within {timeout_seconds}s. Current status: {status}"
+    )
+
+serve.shutdown()

doc/source/llm/doc_code/serve/multi_gpu/bundle_per_worker_example.py

@@ -0,0 +1,91 @@
+# flake8: noqa
+"""
+This file serves as a documentation example and CI test for bundle_per_worker placement group config.
+
+Structure:
+1. Monkeypatch setup: Ensures serve.run is non-blocking and removes accelerator requirements for CI testing.
+2. Docs example (between __bundle_per_worker_example_start/end__): Embedded in Sphinx docs via literalinclude.
+3. Test validation (deployment status polling + cleanup)
+"""
+
+import time
+from ray import serve
+from ray.serve.schema import ApplicationStatus
+from ray.serve._private.constants import SERVE_DEFAULT_APP_NAME
+from ray.serve import llm
+
+_original_serve_run = serve.run
+_original_build_openai_app = llm.build_openai_app
+
+
+def _non_blocking_serve_run(app, **kwargs):
+    """Forces blocking=False for testing"""
+    kwargs["blocking"] = False
+    return _original_serve_run(app, **kwargs)
+
+
+def _testing_build_openai_app(config, **kwargs):
+    """Removes accelerator requirements for testing"""
+    llm_configs = config.get("llm_configs", [])
+    for llm_config in llm_configs:
+        if hasattr(llm_config, "accelerator_type") and llm_config.accelerator_type is not None:
+            llm_config.accelerator_type = None
+    return _original_build_openai_app(config, **kwargs)
+
+
+serve.run = _non_blocking_serve_run
+llm.build_openai_app = _testing_build_openai_app
+
+# __bundle_per_worker_example_start__
+from ray import serve
+from ray.serve.llm import LLMConfig, build_openai_app
+
+# Configure a model with bundle_per_worker for simple resource specification
+# Ray automatically replicates this bundle based on tp * pp (2 bundles total)
+llm_config = LLMConfig(
+    model_loading_config=dict(
+        model_id="qwen-0.5b",
+        model_source="Qwen/Qwen2.5-0.5B-Instruct",
+    ),
+    deployment_config=dict(
+        autoscaling_config=dict(
+            min_replicas=1,
+            max_replicas=1,
+        )
+    ),
+    engine_kwargs=dict(
+        tensor_parallel_size=2,
+        max_model_len=1024,
+        max_num_seqs=32,
+    ),
+    # Simple: specify resources per worker, auto-replicated by tp*pp
+    placement_group_config=dict(
+        bundle_per_worker={"GPU": 1, "CPU": 1},
+    ),
+)
+
+# Deploy the application
+app = build_openai_app({"llm_configs": [llm_config]})
+serve.run(app, blocking=True)
+# __bundle_per_worker_example_end__
+
+status = ApplicationStatus.NOT_STARTED
+timeout_seconds = 300
+start_time = time.time()
+
+while (
+    status != ApplicationStatus.RUNNING and time.time() - start_time < timeout_seconds
+):
+    status = serve.status().applications[SERVE_DEFAULT_APP_NAME].status
+
+    if status in [ApplicationStatus.DEPLOY_FAILED, ApplicationStatus.UNHEALTHY]:
+        raise AssertionError(f"Deployment failed with status: {status}")
+
+    time.sleep(1)
+
+if status != ApplicationStatus.RUNNING:
+    raise AssertionError(
+        f"Deployment failed to reach RUNNING status within {timeout_seconds}s. Current status: {status}"
+    )
+
+serve.shutdown()

doc/source/llm/doc_code/serve/multi_gpu/dp_autoscaling_example.py

@@ -0,0 +1,93 @@
+"""
+This file serves as a documentation example and CI test for autoscaling data parallel attention deployment.
+
+Structure:
+1. Monkeypatch setup: Ensures serve.run is non-blocking and removes accelerator requirements for CI testing.
+2. Docs example (between __dp_autoscaling_example_start/end__): Embedded in Sphinx docs via literalinclude.
+3. Test validation (deployment status polling + cleanup)
+"""
+
+import time
+from ray import serve
+from ray.serve.schema import ApplicationStatus
+from ray.serve._private.constants import SERVE_DEFAULT_APP_NAME
+from ray.serve import llm
+
+_original_serve_run = serve.run
+_original_build_dp_openai_app = llm.build_dp_openai_app
+
+
+def _non_blocking_serve_run(app, **kwargs):
+    """Forces blocking=False for testing"""
+    kwargs["blocking"] = False
+    return _original_serve_run(app, **kwargs)
+
+
+def _testing_build_dp_openai_app(builder_config, **kwargs):
+    """Removes accelerator requirements for testing"""
+    if "llm_config" in builder_config:
+        config = builder_config["llm_config"]
+        if hasattr(config, "accelerator_type") and config.accelerator_type is not None:
+            config.accelerator_type = None
+    return _original_build_dp_openai_app(builder_config, **kwargs)
+
+
+serve.run = _non_blocking_serve_run
+llm.build_dp_openai_app = _testing_build_dp_openai_app
+
+
+from ray import serve
+from ray.serve.llm import LLMConfig, build_dp_openai_app
+
+# __dp_autoscaling_example_start__
+config = LLMConfig(
+    model_loading_config={
+        "model_id": "microsoft/Phi-tiny-MoE-instruct"
+    },
+    deployment_config={
+        "num_replicas": "auto",
+        "autoscaling_config": {
+            "min_replicas": 1,  # Min number of DP groups
+            "max_replicas": 2,  # Max number of DP groups
+            "initial_replicas": 1,  # Initial number of DP groups
+            # Other Ray Serve autoscaling knobs still apply
+            "upscale_delay_s": 0.1,
+            "downscale_delay_s": 2,
+        },
+    },
+    engine_kwargs={
+        "data_parallel_size": 2,  # Number of DP replicas
+        "tensor_parallel_size": 1,  # TP size per replica
+        "max_model_len": 1024,
+        "max_num_seqs": 32,
+    },
+)
+
+app = build_dp_openai_app({
+    "llm_config": config
+})
+# __dp_autoscaling_example_end__
+
+serve.run(app, blocking=True)
+
+status = ApplicationStatus.NOT_STARTED
+timeout_seconds = 300
+start_time = time.time()
+
+while (
+    status != ApplicationStatus.RUNNING and time.time() - start_time < timeout_seconds
+):
+    status = serve.status().applications[SERVE_DEFAULT_APP_NAME].status
+
+    if status in [ApplicationStatus.DEPLOY_FAILED, ApplicationStatus.UNHEALTHY]:
+        raise AssertionError(f"Deployment failed with status: {status}")
+
+    time.sleep(1)
+
+if status != ApplicationStatus.RUNNING:
+    raise AssertionError(
+        f"Deployment failed to reach RUNNING status within {timeout_seconds}s. Current status: {status}"
+    )
+
+serve.shutdown()
+

doc/source/llm/doc_code/serve/multi_gpu/dp_basic_example.py

@@ -0,0 +1,86 @@
+"""
+This file serves as a documentation example and CI test for basic data parallel attention deployment.
+
+Structure:
+1. Monkeypatch setup: Ensures serve.run is non-blocking and removes accelerator requirements for CI testing.
+2. Docs example (between __dp_basic_example_start/end__): Embedded in Sphinx docs via literalinclude.
+3. Test validation (deployment status polling + cleanup)
+"""
+
+import time
+from ray import serve
+from ray.serve.schema import ApplicationStatus
+from ray.serve._private.constants import SERVE_DEFAULT_APP_NAME
+from ray.serve import llm
+
+_original_serve_run = serve.run
+_original_build_dp_openai_app = llm.build_dp_openai_app
+
+
+def _non_blocking_serve_run(app, **kwargs):
+    """Forces blocking=False for testing"""
+    kwargs["blocking"] = False
+    return _original_serve_run(app, **kwargs)
+
+
+def _testing_build_dp_openai_app(builder_config, **kwargs):
+    """Removes accelerator requirements for testing"""
+    if "llm_config" in builder_config:
+        config = builder_config["llm_config"]
+        if hasattr(config, "accelerator_type") and config.accelerator_type is not None:
+            config.accelerator_type = None
+    return _original_build_dp_openai_app(builder_config, **kwargs)
+
+
+serve.run = _non_blocking_serve_run
+llm.build_dp_openai_app = _testing_build_dp_openai_app
+
+# __dp_basic_example_start__
+from ray import serve
+from ray.serve.llm import LLMConfig, build_dp_openai_app
+
+# Configure the model with data parallel settings
+config = LLMConfig(
+    model_loading_config={
+        "model_id": "microsoft/Phi-tiny-MoE-instruct"
+    },
+    deployment_config={
+        "num_replicas": 2
+    },
+    engine_kwargs={
+        "data_parallel_size": 2,  # Number of DP replicas
+        "tensor_parallel_size": 1,  # TP size per replica
+        # Reduced for CI compatibility
+        "max_model_len": 1024,
+        "max_num_seqs": 32,
+    },
+)
+
+app = build_dp_openai_app({
+    "llm_config": config
+})
+
+serve.run(app, blocking=True)
+# __dp_basic_example_end__
+
+status = ApplicationStatus.NOT_STARTED
+timeout_seconds = 300
+start_time = time.time()
+
+while (
+    status != ApplicationStatus.RUNNING and time.time() - start_time < timeout_seconds
+):
+    status = serve.status().applications[SERVE_DEFAULT_APP_NAME].status
+
+    if status in [ApplicationStatus.DEPLOY_FAILED, ApplicationStatus.UNHEALTHY]:
+        raise AssertionError(f"Deployment failed with status: {status}")
+
+    time.sleep(1)
+
+if status != ApplicationStatus.RUNNING:
+    raise AssertionError(
+        f"Deployment failed to reach RUNNING status within {timeout_seconds}s. Current status: {status}"
+    )
+
+serve.shutdown()
+

doc/source/llm/doc_code/serve/multi_gpu/dp_pd_example.py

@@ -0,0 +1,119 @@
+"""
+This file serves as a documentation example and CI test for data parallel + prefill-decode disaggregation.
+
+Structure:
+1. Monkeypatch setup: Ensures serve.run is non-blocking and removes accelerator requirements for CI testing.
+2. Docs example (between __dp_pd_example_start/end__): Embedded in Sphinx docs via literalinclude.
+3. Test validation (deployment status polling + cleanup)
+"""
+
+import time
+from ray import serve
+from ray.serve.schema import ApplicationStatus
+from ray.serve._private.constants import SERVE_DEFAULT_APP_NAME
+from ray.serve import llm
+
+# Check if NIXL is available (required for NixlConnector)
+try:
+    import nixl  # noqa: F401
+    NIXL_AVAILABLE = True
+except ImportError:
+    NIXL_AVAILABLE = False
+
+if not NIXL_AVAILABLE:
+    raise ImportError(
+        "NIXL is required for this example but is not installed. "
+        "Install it with: pip install nixl or uv pip install nixl"
+    )
+
+_original_serve_run = serve.run
+_original_build_dp_deployment = llm.build_dp_deployment
+
+
+def _non_blocking_serve_run(app, **kwargs):
+    """Forces blocking=False for testing"""
+    kwargs["blocking"] = False
+    return _original_serve_run(app, **kwargs)
+
+
+def _testing_build_dp_deployment(llm_config, **kwargs):
+    """Removes accelerator requirements for testing"""
+    if llm_config.accelerator_type is not None:
+        llm_config.accelerator_type = None
+    return _original_build_dp_deployment(llm_config, **kwargs)
+
+
+serve.run = _non_blocking_serve_run
+llm.build_dp_deployment = _testing_build_dp_deployment
+
+# __dp_pd_example_start__
+from ray import serve
+from ray.serve.llm import LLMConfig, build_pd_openai_app
+
+# Configure prefill with data parallel attention
+prefill_config = LLMConfig(
+    model_loading_config={
+        "model_id": "microsoft/Phi-tiny-MoE-instruct"
+    },
+    engine_kwargs={
+        "data_parallel_size": 2,  # 2 DP replicas for prefill
+        "tensor_parallel_size": 1,
+        "kv_transfer_config": {
+            "kv_connector": "NixlConnector",
+            "kv_role": "kv_both",
+        },
+        # Reduced for CI compatibility
+        "max_model_len": 1024,
+        "max_num_seqs": 32,
+    },
+)
+
+# Configure decode with data parallel attention
+decode_config = LLMConfig(
+    model_loading_config={
+        "model_id": "microsoft/Phi-tiny-MoE-instruct"
+    },
+    engine_kwargs={
+        "data_parallel_size": 2,  # 2 DP replicas for decode
+        "tensor_parallel_size": 1,
+        "kv_transfer_config": {
+            "kv_connector": "NixlConnector",
+            "kv_role": "kv_both",
+        },
+        # Reduced for CI compatibility
+        "max_model_len": 1024,
+        "max_num_seqs": 32,
+    },
+)
+
+# Build and deploy the PD application (3-tier: ingress -> decode -> prefill)
+# PDDecodeServer orchestrates remote prefill then runs local decode.
+app = build_pd_openai_app(
+    {
+        "prefill_config": prefill_config,
+        "decode_config": decode_config,
+    }
+)
+serve.run(app, blocking=True)
+# __dp_pd_example_end__
+
+status = ApplicationStatus.NOT_STARTED
+timeout_seconds = 300  # Longer timeout for DP+PD setup
+start_time = time.time()
+
+while (
+    status != ApplicationStatus.RUNNING and time.time() - start_time < timeout_seconds
+):
+    status = serve.status().applications[SERVE_DEFAULT_APP_NAME].status
+
+    if status in [ApplicationStatus.DEPLOY_FAILED, ApplicationStatus.UNHEALTHY]:
+        raise AssertionError(f"Deployment failed with status: {status}")
+
+    time.sleep(1)
+
+if status != ApplicationStatus.RUNNING:
+    raise AssertionError(
+        f"Deployment failed to reach RUNNING status within {timeout_seconds}s. Current status: {status}"
+    )
+
+serve.shutdown()

doc/source/llm/doc_code/serve/prefix_aware_router/prefix_aware_example.py

@@ -0,0 +1,89 @@
+"""
+This file serves as a documentation example and CI test.
+
+Structure:
+1. Monkeypatch setup: Ensures serve.run is non-blocking and removes accelerator requirements for CI testing.
+2. Docs example (between __prefix_aware_example_start/end__): Embedded in Sphinx docs via literalinclude.
+3. Test validation (deployment status polling + cleanup)
+"""
+
+import time
+from ray import serve
+from ray.serve.schema import ApplicationStatus
+from ray.serve._private.constants import SERVE_DEFAULT_APP_NAME
+from ray.serve import llm
+
+_original_serve_run = serve.run
+_original_build_openai_app = llm.build_openai_app
+
+
+def _non_blocking_serve_run(app, **kwargs):
+    """Forces blocking=False for testing"""
+    kwargs["blocking"] = False
+    return _original_serve_run(app, **kwargs)
+
+
+def _testing_build_openai_app(llm_serving_args):
+    """Removes accelerator requirements for testing"""
+    for config in llm_serving_args["llm_configs"]:
+        config.accelerator_type = None
+
+    return _original_build_openai_app(llm_serving_args)
+
+
+serve.run = _non_blocking_serve_run
+llm.build_openai_app = _testing_build_openai_app
+
+# __prefix_aware_example_start__
+from ray import serve
+from ray.serve.llm import LLMConfig, build_openai_app
+from ray.serve.llm.request_router import PrefixCacheAffinityRouter
+
+llm_config = LLMConfig(
+    model_loading_config={
+        "model_id": "qwen-0.5b",
+        "model_source": "Qwen/Qwen2.5-0.5B-Instruct",
+    },
+    deployment_config={
+        "autoscaling_config": {
+            "min_replicas": 4,
+            "max_replicas": 4,
+        },
+        "request_router_config": {
+            "request_router_class": PrefixCacheAffinityRouter,
+            "request_router_kwargs": {
+                "imbalanced_threshold": 5,  # More aggressive load balancing
+                "match_rate_threshold": 0.15,  # Require 15% match rate
+                "do_eviction": True,  # Enable memory management
+                "eviction_threshold_chars": 500_000,
+                "eviction_target_chars": 400_000,
+                "eviction_interval_secs": 30,
+            },
+        },
+    },
+)
+
+app = build_openai_app({"llm_configs": [llm_config]})
+serve.run(app, blocking=True)
+# __prefix_aware_example_end__
+
+status = ApplicationStatus.NOT_STARTED
+timeout_seconds = 180
+start_time = time.time()
+
+while (
+    status != ApplicationStatus.RUNNING and time.time() - start_time < timeout_seconds
+):
+    status = serve.status().applications[SERVE_DEFAULT_APP_NAME].status
+
+    if status in [ApplicationStatus.DEPLOY_FAILED, ApplicationStatus.UNHEALTHY]:
+        raise AssertionError(f"Deployment failed with status: {status}")
+
+    time.sleep(1)
+
+if status != ApplicationStatus.RUNNING:
+    raise AssertionError(
+        f"Deployment failed to reach RUNNING status within {timeout_seconds}s. Current status: {status}"
+    )
+
+serve.shutdown()

doc/source/llm/doc_code/serve/qwen/llm_config_example.yaml

@@ -0,0 +1,23 @@
+# config.yaml
+applications:
+- args:
+    llm_configs:
+        - model_loading_config:
+            model_id: qwen-0.5b
+            model_source: Qwen/Qwen2.5-0.5B-Instruct
+          accelerator_type: A10G
+          deployment_config:
+            autoscaling_config:
+                min_replicas: 1
+                max_replicas: 2
+        - model_loading_config:
+            model_id: qwen-1.5b
+            model_source: Qwen/Qwen2.5-1.5B-Instruct
+          accelerator_type: A10G
+          deployment_config:
+            autoscaling_config:
+                min_replicas: 1
+                max_replicas: 2
+  import_path: ray.serve.llm:build_openai_app
+  name: llm_app
+  route_prefix: "/" 

doc/source/llm/doc_code/serve/qwen/llm_yaml_config_example.py

@@ -0,0 +1,53 @@
+"""
+This file serves as a documentation example and CI test for YAML config deployment.
+
+Structure:
+1. Monkeypatch setup: Ensures serve.run is non-blocking and removes accelerator requirements for CI testing.
+2. Load YAML config and convert to Python using build_openai_app
+3. Test validation (deployment status polling + cleanup)
+"""
+
+import time
+import os
+import yaml
+from ray import serve
+from ray.serve.schema import ApplicationStatus
+from ray.serve._private.constants import SERVE_DEFAULT_APP_NAME
+from ray.serve import llm
+
+
+config_path = os.path.join(os.path.dirname(__file__), "llm_config_example.yaml")
+with open(config_path, "r") as f:
+    config_dict = yaml.safe_load(f)
+
+llm_configs = config_dict["applications"][0]["args"]["llm_configs"]
+for config in llm_configs:
+    config.pop("accelerator_type", None)
+    # Disable compile cache to avoid cache corruption in CI
+    if "runtime_env" not in config:
+        config["runtime_env"] = {}
+    if "env_vars" not in config["runtime_env"]:
+        config["runtime_env"]["env_vars"] = {}
+    config["runtime_env"]["env_vars"]["VLLM_DISABLE_COMPILE_CACHE"] = "1"
+
+app = llm.build_openai_app({"llm_configs": llm_configs})
+serve.run(app, blocking=False)
+
+status = ApplicationStatus.NOT_STARTED
+timeout_seconds = 180
+start_time = time.time()
+
+while (
+    status != ApplicationStatus.RUNNING and time.time() - start_time < timeout_seconds
+):
+    status = serve.status().applications[SERVE_DEFAULT_APP_NAME].status
+
+    if status in [ApplicationStatus.DEPLOY_FAILED, ApplicationStatus.UNHEALTHY]:
+        raise AssertionError(f"Deployment failed with status: {status}")
+
+    time.sleep(1)
+
+if status != ApplicationStatus.RUNNING:
+    raise AssertionError(
+        f"Deployment failed to reach RUNNING status within {timeout_seconds}s. Current status: {status}"
+    )

doc/source/llm/doc_code/serve/qwen/qwen_example.py

@@ -0,0 +1,89 @@
+"""
+This file serves as a documentation example and CI test.
+
+Structure:
+1. Monkeypatch setup: Ensures serve.run is non-blocking and removes accelerator requirements for CI testing.
+2. Docs example (between __qwen_example_start/end__): Embedded in Sphinx docs via literalinclude.
+3. Test validation (deployment status polling + cleanup)
+"""
+
+import time
+from ray import serve
+from ray.serve.schema import ApplicationStatus
+from ray.serve._private.constants import SERVE_DEFAULT_APP_NAME
+from ray.serve import llm
+
+_original_serve_run = serve.run
+_original_build_openai_app = llm.build_openai_app
+
+
+def _non_blocking_serve_run(app, **kwargs):
+    """Forces blocking=False for testing"""
+    kwargs["blocking"] = False
+    return _original_serve_run(app, **kwargs)
+
+
+def _testing_build_openai_app(llm_serving_args):
+    """Removes accelerator requirements for testing"""
+    for config in llm_serving_args["llm_configs"]:
+        config.accelerator_type = None
+        # Disable compile cache to avoid cache corruption in CI
+        if not config.runtime_env:
+            config.runtime_env = {}
+        if "env_vars" not in config.runtime_env:
+            config.runtime_env["env_vars"] = {}
+        config.runtime_env["env_vars"]["VLLM_DISABLE_COMPILE_CACHE"] = "1"
+
+    return _original_build_openai_app(llm_serving_args)
+
+
+serve.run = _non_blocking_serve_run
+llm.build_openai_app = _testing_build_openai_app
+
+# __qwen_example_start__
+from ray import serve
+from ray.serve.llm import LLMConfig, build_openai_app
+
+llm_config = LLMConfig(
+    model_loading_config={
+        "model_id": "qwen-0.5b",
+        "model_source": "Qwen/Qwen2.5-0.5B-Instruct",
+    },
+    deployment_config={
+        "autoscaling_config": {
+            "min_replicas": 1,
+            "max_replicas": 2,
+        }
+    },
+    # Pass the desired accelerator type (e.g. A10G, L4, etc.)
+    accelerator_type="A10G",
+    # You can customize the engine arguments (e.g. vLLM engine kwargs)
+    engine_kwargs={
+        "tensor_parallel_size": 2,
+    },
+)
+
+app = build_openai_app({"llm_configs": [llm_config]})
+serve.run(app, blocking=True)
+# __qwen_example_end__
+
+status = ApplicationStatus.NOT_STARTED
+timeout_seconds = 180
+start_time = time.time()
+
+while (
+    status != ApplicationStatus.RUNNING and time.time() - start_time < timeout_seconds
+):
+    status = serve.status().applications[SERVE_DEFAULT_APP_NAME].status
+
+    if status in [ApplicationStatus.DEPLOY_FAILED, ApplicationStatus.UNHEALTHY]:
+        raise AssertionError(f"Deployment failed with status: {status}")
+
+    time.sleep(1)
+
+if status != ApplicationStatus.RUNNING:
+    raise AssertionError(
+        f"Deployment failed to reach RUNNING status within {timeout_seconds}s. Current status: {status}"
+    )
+
+serve.shutdown()

doc/source/llm/doc_code/serve/sglang/sglang_batch_example.py

@@ -0,0 +1,45 @@
+# __sglang_batch_start__
+import ray
+from ray.data.llm import SGLangEngineProcessorConfig, build_processor
+
+config = SGLangEngineProcessorConfig(
+    model_source="unsloth/Llama-3.1-8B-Instruct",
+    engine_kwargs=dict(
+        dtype="half",
+        mem_fraction_static=0.8,
+    ),
+    batch_size=32,
+    concurrency=1,
+)
+
+processor = build_processor(
+    config,
+    preprocess=lambda row: dict(
+        messages=[
+            {"role": "system", "content": "You are a helpful assistant."},
+            {"role": "user", "content": row["prompt"]},
+        ],
+        sampling_params=dict(
+            temperature=0.7,
+            max_new_tokens=256,
+        ),
+    ),
+    postprocess=lambda row: dict(
+        prompt=row["prompt"],
+        response=row["generated_text"],
+    ),
+)
+
+ds = ray.data.from_items(
+    [
+        {"prompt": "What is the capital of France?"},
+        {"prompt": "Explain photosynthesis in one sentence."},
+        {"prompt": "Write a haiku about programming."},
+    ]
+)
+ds = processor(ds)
+
+for row in ds.take_all():
+    print(f"Prompt: {row['prompt']}")
+    print(f"Response: {row['response']}\n")
+# __sglang_batch_end__

doc/source/llm/doc_code/serve/sglang/sglang_multinode_example.py

@@ -0,0 +1,36 @@
+# __sglang_multinode_start__
+from ray.llm._internal.serve.engines.sglang import SGLangServer
+
+from ray import serve
+from ray.serve.llm import LLMConfig, build_openai_app
+
+llm_config = LLMConfig(
+    model_loading_config={
+        "model_id": "Llama-3.1-70B-Instruct",
+        "model_source": "meta-llama/Llama-3.1-70B-Instruct",
+    },
+    deployment_config={
+        "autoscaling_config": {
+            "min_replicas": 1,
+            "max_replicas": 2,
+            "target_ongoing_requests": 4,
+        }
+    },
+    # PACK fills GPUs on each node before moving to the next.
+    # With 8 bundles across 2 nodes (4 GPUs each), each node gets 4 bundles.
+    placement_group_config={
+        "placement_group_bundles": [{"CPU": 1, "GPU": 1}] + [{"GPU": 1}] * 7,
+        "placement_group_strategy": "PACK",
+    },
+    server_cls=SGLangServer,
+    engine_kwargs={
+        "model_path": "meta-llama/Llama-3.1-70B-Instruct",
+        "tp_size": 4,
+        "pp_size": 2,
+        "mem_fraction_static": 0.8,
+    },
+)
+
+app = build_openai_app({"llm_configs": [llm_config]})
+serve.run(app, blocking=True)
+# __sglang_multinode_end__

doc/source/llm/doc_code/serve/sglang/sglang_query_example.py

@@ -0,0 +1,27 @@
+# __sglang_query_start__
+from openai import OpenAI
+
+client = OpenAI(base_url="http://localhost:8000/v1", api_key="fake-key")
+
+# Chat completions
+print("=== Chat Completions ===")
+chat_response = client.chat.completions.create(
+    model="Llama-3.1-8B-Instruct",
+    messages=[
+        {"role": "user", "content": "List 3 countries and their capitals."},
+    ],
+    temperature=0,
+    max_tokens=64,
+)
+print(chat_response.choices[0].message.content)
+
+# Text completions
+print("\n=== Text Completions ===")
+completion_response = client.completions.create(
+    model="Llama-3.1-8B-Instruct",
+    prompt="San Francisco is a",
+    temperature=0,
+    max_tokens=30,
+)
+print(completion_response.choices[0].text)
+# __sglang_query_end__

doc/source/llm/doc_code/serve/sglang/sglang_serving_example.py

@@ -0,0 +1,29 @@
+# __sglang_single_node_start__
+from ray.llm._internal.serve.engines.sglang import SGLangServer
+
+from ray import serve
+from ray.serve.llm import LLMConfig, build_openai_app
+
+llm_config = LLMConfig(
+    model_loading_config={
+        "model_id": "Llama-3.1-8B-Instruct",
+        "model_source": "unsloth/Llama-3.1-8B-Instruct",
+    },
+    deployment_config={
+        "autoscaling_config": {
+            "min_replicas": 1,
+            "max_replicas": 2,
+        }
+    },
+    server_cls=SGLangServer,
+    engine_kwargs={
+        "trust_remote_code": True,
+        "model_path": "unsloth/Llama-3.1-8B-Instruct",
+        "tp_size": 1,
+        "mem_fraction_static": 0.8,
+    },
+)
+
+app = build_openai_app({"llm_configs": [llm_config]})
+serve.run(app, blocking=True)
+# __sglang_single_node_end__

doc/source/llm/doc_code/serve/transcription/transcription_example.py

@@ -0,0 +1,100 @@
+"""
+This file serves as a documentation example and CI test.
+
+Structure:
+1. Monkeypatch setup: Ensures serve.run is non-blocking and removes accelerator requirements for CI testing.
+2. Docs example (between __transcription_example_start/end__): Embedded in Sphinx docs via literalinclude.
+3. Test validation (deployment status polling + cleanup)
+"""
+
+import time
+import openai
+import requests
+from ray import serve
+from ray.serve.schema import ApplicationStatus
+from ray.serve._private.constants import SERVE_DEFAULT_APP_NAME
+from ray.serve import llm
+
+_original_serve_run = serve.run
+_original_build_openai_app = llm.build_openai_app
+
+
+def _non_blocking_serve_run(app, **kwargs):
+    """Forces blocking=False for testing"""
+    kwargs["blocking"] = False
+    return _original_serve_run(app, **kwargs)
+
+
+def _testing_build_openai_app(llm_serving_args):
+    """Removes accelerator requirements for testing"""
+    for config in llm_serving_args["llm_configs"]:
+        config.accelerator_type = None
+
+    return _original_build_openai_app(llm_serving_args)
+
+
+serve.run = _non_blocking_serve_run
+llm.build_openai_app = _testing_build_openai_app
+
+# __transcription_example_start__
+from ray import serve
+from ray.serve.llm import LLMConfig, build_openai_app
+
+llm_config = LLMConfig(
+    model_loading_config={
+        "model_id": "whisper-small",
+        "model_source": "openai/whisper-small",
+    },
+    deployment_config={
+        "autoscaling_config": {
+            "min_replicas": 1,
+            "max_replicas": 4,
+        }
+    },
+    accelerator_type="A10G",
+    log_engine_metrics=True,
+)
+
+app = build_openai_app({"llm_configs": [llm_config]})
+serve.run(app, blocking=True)
+# __transcription_example_end__
+
+status = ApplicationStatus.NOT_STARTED
+timeout_seconds = 300
+start_time = time.time()
+
+while (
+    status != ApplicationStatus.RUNNING and time.time() - start_time < timeout_seconds
+):
+    status = serve.status().applications[SERVE_DEFAULT_APP_NAME].status
+
+    if status in [ApplicationStatus.DEPLOY_FAILED, ApplicationStatus.UNHEALTHY]:
+        raise AssertionError(f"Deployment failed with status: {status}")
+
+    time.sleep(1)
+
+if status != ApplicationStatus.RUNNING:
+    raise AssertionError(
+        f"Deployment failed to reach RUNNING status within {timeout_seconds}s. Current status: {status}"
+    )
+
+response = requests.get("https://voiceage.com/wbsamples/in_stereo/Sports.wav")
+with open("audio.wav", "wb") as f:
+    f.write(response.content)
+
+client = openai.OpenAI(base_url="http://localhost:8000/v1", api_key="fake-key")
+
+with open("audio.wav", "rb") as f:
+    try:
+        response = client.audio.transcriptions.create(
+            model="whisper-small",
+            file=f,
+            temperature=0.0,
+            language="en",
+        )
+    except Exception as e:
+        raise AssertionError(
+            f"Error while querying models: {e}. Check the logs for more details."
+        )
+
+serve.shutdown()