chore: import upstream snapshot with attribution

doc/source/llm/doc_code/serve/multi_gpu/bundle_per_worker_example.py

@@ -0,0 +1,91 @@
+# flake8: noqa
+"""
+This file serves as a documentation example and CI test for bundle_per_worker placement group config.
+
+Structure:
+1. Monkeypatch setup: Ensures serve.run is non-blocking and removes accelerator requirements for CI testing.
+2. Docs example (between __bundle_per_worker_example_start/end__): Embedded in Sphinx docs via literalinclude.
+3. Test validation (deployment status polling + cleanup)
+"""
+
+import time
+from ray import serve
+from ray.serve.schema import ApplicationStatus
+from ray.serve._private.constants import SERVE_DEFAULT_APP_NAME
+from ray.serve import llm
+
+_original_serve_run = serve.run
+_original_build_openai_app = llm.build_openai_app
+
+
+def _non_blocking_serve_run(app, **kwargs):
+    """Forces blocking=False for testing"""
+    kwargs["blocking"] = False
+    return _original_serve_run(app, **kwargs)
+
+
+def _testing_build_openai_app(config, **kwargs):
+    """Removes accelerator requirements for testing"""
+    llm_configs = config.get("llm_configs", [])
+    for llm_config in llm_configs:
+        if hasattr(llm_config, "accelerator_type") and llm_config.accelerator_type is not None:
+            llm_config.accelerator_type = None
+    return _original_build_openai_app(config, **kwargs)
+
+
+serve.run = _non_blocking_serve_run
+llm.build_openai_app = _testing_build_openai_app
+
+# __bundle_per_worker_example_start__
+from ray import serve
+from ray.serve.llm import LLMConfig, build_openai_app
+
+# Configure a model with bundle_per_worker for simple resource specification
+# Ray automatically replicates this bundle based on tp * pp (2 bundles total)
+llm_config = LLMConfig(
+    model_loading_config=dict(
+        model_id="qwen-0.5b",
+        model_source="Qwen/Qwen2.5-0.5B-Instruct",
+    ),
+    deployment_config=dict(
+        autoscaling_config=dict(
+            min_replicas=1,
+            max_replicas=1,
+        )
+    ),
+    engine_kwargs=dict(
+        tensor_parallel_size=2,
+        max_model_len=1024,
+        max_num_seqs=32,
+    ),
+    # Simple: specify resources per worker, auto-replicated by tp*pp
+    placement_group_config=dict(
+        bundle_per_worker={"GPU": 1, "CPU": 1},
+    ),
+)
+
+# Deploy the application
+app = build_openai_app({"llm_configs": [llm_config]})
+serve.run(app, blocking=True)
+# __bundle_per_worker_example_end__
+
+status = ApplicationStatus.NOT_STARTED
+timeout_seconds = 300
+start_time = time.time()
+
+while (
+    status != ApplicationStatus.RUNNING and time.time() - start_time < timeout_seconds
+):
+    status = serve.status().applications[SERVE_DEFAULT_APP_NAME].status
+
+    if status in [ApplicationStatus.DEPLOY_FAILED, ApplicationStatus.UNHEALTHY]:
+        raise AssertionError(f"Deployment failed with status: {status}")
+
+    time.sleep(1)
+
+if status != ApplicationStatus.RUNNING:
+    raise AssertionError(
+        f"Deployment failed to reach RUNNING status within {timeout_seconds}s. Current status: {status}"
+    )
+
+serve.shutdown()

doc/source/llm/doc_code/serve/multi_gpu/dp_autoscaling_example.py

@@ -0,0 +1,93 @@
+"""
+This file serves as a documentation example and CI test for autoscaling data parallel attention deployment.
+
+Structure:
+1. Monkeypatch setup: Ensures serve.run is non-blocking and removes accelerator requirements for CI testing.
+2. Docs example (between __dp_autoscaling_example_start/end__): Embedded in Sphinx docs via literalinclude.
+3. Test validation (deployment status polling + cleanup)
+"""
+
+import time
+from ray import serve
+from ray.serve.schema import ApplicationStatus
+from ray.serve._private.constants import SERVE_DEFAULT_APP_NAME
+from ray.serve import llm
+
+_original_serve_run = serve.run
+_original_build_dp_openai_app = llm.build_dp_openai_app
+
+
+def _non_blocking_serve_run(app, **kwargs):
+    """Forces blocking=False for testing"""
+    kwargs["blocking"] = False
+    return _original_serve_run(app, **kwargs)
+
+
+def _testing_build_dp_openai_app(builder_config, **kwargs):
+    """Removes accelerator requirements for testing"""
+    if "llm_config" in builder_config:
+        config = builder_config["llm_config"]
+        if hasattr(config, "accelerator_type") and config.accelerator_type is not None:
+            config.accelerator_type = None
+    return _original_build_dp_openai_app(builder_config, **kwargs)
+
+
+serve.run = _non_blocking_serve_run
+llm.build_dp_openai_app = _testing_build_dp_openai_app
+
+
+from ray import serve
+from ray.serve.llm import LLMConfig, build_dp_openai_app
+
+# __dp_autoscaling_example_start__
+config = LLMConfig(
+    model_loading_config={
+        "model_id": "microsoft/Phi-tiny-MoE-instruct"
+    },
+    deployment_config={
+        "num_replicas": "auto",
+        "autoscaling_config": {
+            "min_replicas": 1,  # Min number of DP groups
+            "max_replicas": 2,  # Max number of DP groups
+            "initial_replicas": 1,  # Initial number of DP groups
+            # Other Ray Serve autoscaling knobs still apply
+            "upscale_delay_s": 0.1,
+            "downscale_delay_s": 2,
+        },
+    },
+    engine_kwargs={
+        "data_parallel_size": 2,  # Number of DP replicas
+        "tensor_parallel_size": 1,  # TP size per replica
+        "max_model_len": 1024,
+        "max_num_seqs": 32,
+    },
+)
+
+app = build_dp_openai_app({
+    "llm_config": config
+})
+# __dp_autoscaling_example_end__
+
+serve.run(app, blocking=True)
+
+status = ApplicationStatus.NOT_STARTED
+timeout_seconds = 300
+start_time = time.time()
+
+while (
+    status != ApplicationStatus.RUNNING and time.time() - start_time < timeout_seconds
+):
+    status = serve.status().applications[SERVE_DEFAULT_APP_NAME].status
+
+    if status in [ApplicationStatus.DEPLOY_FAILED, ApplicationStatus.UNHEALTHY]:
+        raise AssertionError(f"Deployment failed with status: {status}")
+
+    time.sleep(1)
+
+if status != ApplicationStatus.RUNNING:
+    raise AssertionError(
+        f"Deployment failed to reach RUNNING status within {timeout_seconds}s. Current status: {status}"
+    )
+
+serve.shutdown()
+

doc/source/llm/doc_code/serve/multi_gpu/dp_basic_example.py

@@ -0,0 +1,86 @@
+"""
+This file serves as a documentation example and CI test for basic data parallel attention deployment.
+
+Structure:
+1. Monkeypatch setup: Ensures serve.run is non-blocking and removes accelerator requirements for CI testing.
+2. Docs example (between __dp_basic_example_start/end__): Embedded in Sphinx docs via literalinclude.
+3. Test validation (deployment status polling + cleanup)
+"""
+
+import time
+from ray import serve
+from ray.serve.schema import ApplicationStatus
+from ray.serve._private.constants import SERVE_DEFAULT_APP_NAME
+from ray.serve import llm
+
+_original_serve_run = serve.run
+_original_build_dp_openai_app = llm.build_dp_openai_app
+
+
+def _non_blocking_serve_run(app, **kwargs):
+    """Forces blocking=False for testing"""
+    kwargs["blocking"] = False
+    return _original_serve_run(app, **kwargs)
+
+
+def _testing_build_dp_openai_app(builder_config, **kwargs):
+    """Removes accelerator requirements for testing"""
+    if "llm_config" in builder_config:
+        config = builder_config["llm_config"]
+        if hasattr(config, "accelerator_type") and config.accelerator_type is not None:
+            config.accelerator_type = None
+    return _original_build_dp_openai_app(builder_config, **kwargs)
+
+
+serve.run = _non_blocking_serve_run
+llm.build_dp_openai_app = _testing_build_dp_openai_app
+
+# __dp_basic_example_start__
+from ray import serve
+from ray.serve.llm import LLMConfig, build_dp_openai_app
+
+# Configure the model with data parallel settings
+config = LLMConfig(
+    model_loading_config={
+        "model_id": "microsoft/Phi-tiny-MoE-instruct"
+    },
+    deployment_config={
+        "num_replicas": 2
+    },
+    engine_kwargs={
+        "data_parallel_size": 2,  # Number of DP replicas
+        "tensor_parallel_size": 1,  # TP size per replica
+        # Reduced for CI compatibility
+        "max_model_len": 1024,
+        "max_num_seqs": 32,
+    },
+)
+
+app = build_dp_openai_app({
+    "llm_config": config
+})
+
+serve.run(app, blocking=True)
+# __dp_basic_example_end__
+
+status = ApplicationStatus.NOT_STARTED
+timeout_seconds = 300
+start_time = time.time()
+
+while (
+    status != ApplicationStatus.RUNNING and time.time() - start_time < timeout_seconds
+):
+    status = serve.status().applications[SERVE_DEFAULT_APP_NAME].status
+
+    if status in [ApplicationStatus.DEPLOY_FAILED, ApplicationStatus.UNHEALTHY]:
+        raise AssertionError(f"Deployment failed with status: {status}")
+
+    time.sleep(1)
+
+if status != ApplicationStatus.RUNNING:
+    raise AssertionError(
+        f"Deployment failed to reach RUNNING status within {timeout_seconds}s. Current status: {status}"
+    )
+
+serve.shutdown()
+

doc/source/llm/doc_code/serve/multi_gpu/dp_pd_example.py

@@ -0,0 +1,119 @@
+"""
+This file serves as a documentation example and CI test for data parallel + prefill-decode disaggregation.
+
+Structure:
+1. Monkeypatch setup: Ensures serve.run is non-blocking and removes accelerator requirements for CI testing.
+2. Docs example (between __dp_pd_example_start/end__): Embedded in Sphinx docs via literalinclude.
+3. Test validation (deployment status polling + cleanup)
+"""
+
+import time
+from ray import serve
+from ray.serve.schema import ApplicationStatus
+from ray.serve._private.constants import SERVE_DEFAULT_APP_NAME
+from ray.serve import llm
+
+# Check if NIXL is available (required for NixlConnector)
+try:
+    import nixl  # noqa: F401
+    NIXL_AVAILABLE = True
+except ImportError:
+    NIXL_AVAILABLE = False
+
+if not NIXL_AVAILABLE:
+    raise ImportError(
+        "NIXL is required for this example but is not installed. "
+        "Install it with: pip install nixl or uv pip install nixl"
+    )
+
+_original_serve_run = serve.run
+_original_build_dp_deployment = llm.build_dp_deployment
+
+
+def _non_blocking_serve_run(app, **kwargs):
+    """Forces blocking=False for testing"""
+    kwargs["blocking"] = False
+    return _original_serve_run(app, **kwargs)
+
+
+def _testing_build_dp_deployment(llm_config, **kwargs):
+    """Removes accelerator requirements for testing"""
+    if llm_config.accelerator_type is not None:
+        llm_config.accelerator_type = None
+    return _original_build_dp_deployment(llm_config, **kwargs)
+
+
+serve.run = _non_blocking_serve_run
+llm.build_dp_deployment = _testing_build_dp_deployment
+
+# __dp_pd_example_start__
+from ray import serve
+from ray.serve.llm import LLMConfig, build_pd_openai_app
+
+# Configure prefill with data parallel attention
+prefill_config = LLMConfig(
+    model_loading_config={
+        "model_id": "microsoft/Phi-tiny-MoE-instruct"
+    },
+    engine_kwargs={
+        "data_parallel_size": 2,  # 2 DP replicas for prefill
+        "tensor_parallel_size": 1,
+        "kv_transfer_config": {
+            "kv_connector": "NixlConnector",
+            "kv_role": "kv_both",
+        },
+        # Reduced for CI compatibility
+        "max_model_len": 1024,
+        "max_num_seqs": 32,
+    },
+)
+
+# Configure decode with data parallel attention
+decode_config = LLMConfig(
+    model_loading_config={
+        "model_id": "microsoft/Phi-tiny-MoE-instruct"
+    },
+    engine_kwargs={
+        "data_parallel_size": 2,  # 2 DP replicas for decode
+        "tensor_parallel_size": 1,
+        "kv_transfer_config": {
+            "kv_connector": "NixlConnector",
+            "kv_role": "kv_both",
+        },
+        # Reduced for CI compatibility
+        "max_model_len": 1024,
+        "max_num_seqs": 32,
+    },
+)
+
+# Build and deploy the PD application (3-tier: ingress -> decode -> prefill)
+# PDDecodeServer orchestrates remote prefill then runs local decode.
+app = build_pd_openai_app(
+    {
+        "prefill_config": prefill_config,
+        "decode_config": decode_config,
+    }
+)
+serve.run(app, blocking=True)
+# __dp_pd_example_end__
+
+status = ApplicationStatus.NOT_STARTED
+timeout_seconds = 300  # Longer timeout for DP+PD setup
+start_time = time.time()
+
+while (
+    status != ApplicationStatus.RUNNING and time.time() - start_time < timeout_seconds
+):
+    status = serve.status().applications[SERVE_DEFAULT_APP_NAME].status
+
+    if status in [ApplicationStatus.DEPLOY_FAILED, ApplicationStatus.UNHEALTHY]:
+        raise AssertionError(f"Deployment failed with status: {status}")
+
+    time.sleep(1)
+
+if status != ApplicationStatus.RUNNING:
+    raise AssertionError(
+        f"Deployment failed to reach RUNNING status within {timeout_seconds}s. Current status: {status}"
+    )
+
+serve.shutdown()