chore: import upstream snapshot with attribution

python/ray/llm/tests/serve/gpu/deployments/llm/prefill_decode_disagg/test_prefill_decode_disagg_gpu.py

@@ -0,0 +1,54 @@
+import sys
+from types import SimpleNamespace
+from unittest.mock import MagicMock, patch
+
+import pytest
+
+from ray.llm._internal.serve.core.configs.llm_config import (
+    LLMConfig,
+)
+from ray.llm._internal.serve.engines.vllm.vllm_engine import (
+    VLLMEngine,
+)
+from ray.serve.schema import ReplicaRank
+
+
+class TestPDDisaggVLLMEngine:
+    """Test vLLM engine under PD disagg."""
+
+    @pytest.mark.asyncio
+    @pytest.mark.parametrize("kv_connector", ["NixlConnector", "LMCacheConnectorV1"])
+    async def test_pd_disagg_vllm_engine(
+        self,
+        # llm_config is a fixture defined in serve.tests.conftest.py
+        llm_config: LLMConfig,
+        kv_connector: str,
+        monkeypatch,
+    ):
+        """Test vLLM engine under PD disagg."""
+        if kv_connector == "LMCacheConnectorV1":
+            lmcache_mock = MagicMock()
+            monkeypatch.setitem(sys.modules, "lmcache", lmcache_mock)
+        llm_config = llm_config.model_copy(deep=True)
+        llm_config.engine_kwargs.update(
+            {
+                "kv_transfer_config": dict(
+                    kv_connector=kv_connector,
+                    kv_role="kv_both",
+                ),
+            }
+        )
+        # In production VLLMEngine is constructed inside a Serve replica, where
+        # the NIXL connector backend reads serve.get_replica_context() to derive
+        # a unique side-channel port offset. Outside a replica that call raises,
+        # so mock the replica context.
+        replica_context = SimpleNamespace(
+            rank=ReplicaRank(rank=0, node_rank=0, local_rank=0)
+        )
+        with patch("ray.serve.get_replica_context", return_value=replica_context):
+            vllm_engine = VLLMEngine(llm_config)
+        assert vllm_engine is not None
+
+
+if __name__ == "__main__":
+    sys.exit(pytest.main(["-v", __file__]))

python/ray/llm/tests/serve/gpu/deployments/llm/vllm/test_config_congruence.py

@@ -0,0 +1,219 @@
+"""Test VllmConfig consistency between Ray Serve LLM and vllm serve CLI.
+
+    This test verifies that Ray Serve LLM and vllm serve CLI generate identical
+    VllmConfig objects for the same model parameters across different GPU architectures.
+
+    1. Ray Serve LLM: VLLMEngine.start() -> AsyncLLM(vllm_config=...)
+    2. vllm serve CLI: build_async_engine_client() -> AsyncLLM.from_vllm_config(vllm_config=...)
+
+    Args:
+        gpu_type: GPU model name (L4, H100, B200)
+        capability: DeviceCapability object with compute capability version
+"""
+
+from typing import Any, Dict, Tuple
+from unittest.mock import MagicMock, patch
+
+import pytest
+from vllm.config import VllmConfig
+from vllm.entrypoints.openai.api_server import build_async_engine_client
+from vllm.platforms.interface import DeviceCapability
+
+from ray.llm._internal.serve.engines.vllm.vllm_engine import VLLMEngine
+from ray.serve.llm import LLMConfig, ModelLoadingConfig
+from ray.util import remove_placement_group
+from ray.util.placement_group import placement_group_table
+
+TEST_MODEL = "meta-llama/Llama-3.1-8B-Instruct"
+TEST_MAX_MODEL_LEN = 10500
+TEST_TENSOR_PARALLEL_SIZE = 1
+TEST_GPU_MEMORY_UTILIZATION = 0.95
+
+GPU_CONFIGS = [
+    ("L4", DeviceCapability(major=8, minor=9)),  # Ada Lovelace architecture
+    ("H100", DeviceCapability(major=9, minor=0)),  # Hopper architecture
+    ("B200", DeviceCapability(major=10, minor=0)),  # Blackwell architecture
+]
+
+EXPECTED_DIFF_FIELDS = {
+    "instance_id",
+}
+
+LLM_CONFIG = LLMConfig(
+    model_loading_config=ModelLoadingConfig(
+        model_id=TEST_MODEL,
+        model_source=TEST_MODEL,
+    ),
+    deployment_config={
+        "autoscaling_config": {
+            "min_replicas": 1,
+            "max_replicas": 1,
+        },
+        "max_ongoing_requests": 8192,
+    },
+    engine_kwargs={
+        "enable_chunked_prefill": True,
+        "max_model_len": TEST_MAX_MODEL_LEN,
+        "tensor_parallel_size": TEST_TENSOR_PARALLEL_SIZE,
+        "gpu_memory_utilization": TEST_GPU_MEMORY_UTILIZATION,
+    },
+)
+
+
+@pytest.fixture(autouse=True)
+def setup_placement_group_cleanup():
+    """Automatically clean up placement groups before each test."""
+    pg_table = placement_group_table()
+    for pg_info in pg_table.values():
+        if pg_info["state"] in ["CREATED", "CREATING"]:
+            try:
+                remove_placement_group(pg_info["placement_group_id"])
+            except Exception:
+                # Placement group may have already been removed
+                pass
+
+
+def deep_compare(dict1: Any, dict2: Any) -> bool:
+    if type(dict1) is not type(dict2):
+        return False
+    if isinstance(dict1, dict):
+        if dict1.keys() != dict2.keys():
+            return False
+        return all(deep_compare(dict1[k], dict2[k]) for k in dict1)
+    elif isinstance(dict1, list):
+        return set(dict1) == set(dict2)
+    else:
+        return dict1 == dict2
+
+
+async def normalize_parallel_config(config_dict: Dict[str, Any]) -> None:
+    """Placement groups may differ, that's okay."""
+    if "parallel_config" in config_dict:
+        pc_dict = vars(config_dict["parallel_config"]).copy()
+        pc_dict.pop("placement_group", None)
+        config_dict["parallel_config"] = pc_dict
+
+
+def get_config_differences(dict1: Dict[str, Any], dict2: Dict[str, Any]) -> list[str]:
+    differences = []
+    for key in dict1.keys() | dict2.keys():
+        if not deep_compare(dict1.get(key), dict2.get(key)):
+            differences.append(f"{key}: Ray={dict1.get(key)} vs CLI={dict2.get(key)}")
+    return differences
+
+
+async def get_ray_serve_llm_vllm_config() -> Tuple[Any, str]:
+    """Get VllmConfig by hooking into Ray Serve LLM's AsyncLLM instantiation."""
+    captured_configs = []
+
+    def mock_async_llm_class(vllm_config: VllmConfig = None, **kwargs):
+        captured_configs.append(vllm_config)
+        mock_obj = MagicMock()
+        mock_obj._dummy_engine = True
+        return mock_obj
+
+    with patch("vllm.v1.engine.async_llm.AsyncLLM", side_effect=mock_async_llm_class):
+        try:
+            engine = VLLMEngine(LLM_CONFIG)
+            await engine.start()
+        except Exception:
+            # Expected since we're mocking the constructor
+            pass
+
+    if not captured_configs:
+        raise RuntimeError("Failed to capture VllmConfig from Ray Serve LLM path")
+
+    return captured_configs[-1]
+
+
+async def get_vllm_standalone_config() -> Tuple[Any, str]:
+    """Get VllmConfig by hooking into vllm serve CLI's AsyncLLM instantiation."""
+    captured_configs = []
+
+    def mock_from_vllm_config(vllm_config=None, **kwargs):
+        captured_configs.append(vllm_config)
+        mock_engine = MagicMock()
+
+        async def dummy_reset():
+            pass
+
+        mock_engine.reset_mm_cache = MagicMock(return_value=dummy_reset())
+        mock_engine.shutdown = MagicMock()
+        return mock_engine
+
+    # Create CLI args using vLLM's argument parser
+    from vllm.entrypoints.openai.cli_args import make_arg_parser
+    from vllm.utils.argparse_utils import FlexibleArgumentParser
+
+    parser = make_arg_parser(FlexibleArgumentParser())
+    cli_args = parser.parse_args(
+        [
+            "--model",
+            TEST_MODEL,
+            "--enable-chunked-prefill",
+            "--max-model-len",
+            str(TEST_MAX_MODEL_LEN),
+            "--tensor-parallel-size",
+            str(TEST_TENSOR_PARALLEL_SIZE),
+            "--gpu-memory-utilization",
+            str(TEST_GPU_MEMORY_UTILIZATION),
+            "--distributed-executor-backend",
+            "ray",
+            "--disable-log-requests",
+        ]
+    )
+
+    with patch(
+        "vllm.v1.engine.async_llm.AsyncLLM.from_vllm_config",
+        side_effect=mock_from_vllm_config,
+    ):
+        try:
+            async with build_async_engine_client(cli_args):
+                pass
+        except Exception:
+            # Expected since we're mocking the constructor
+            pass
+
+    if not captured_configs:
+        raise RuntimeError("No valid VllmConfig found in captured configurations")
+
+    return captured_configs[-1]
+
+
+@pytest.mark.parametrize("gpu_type,capability", GPU_CONFIGS)
+@pytest.mark.asyncio
+async def test_vllm_config_ray_serve_vs_cli_comparison(
+    gpu_type: str, capability: DeviceCapability
+):
+    with patch(
+        "vllm.platforms.cuda.NvmlCudaPlatform.get_device_capability",
+        return_value=capability,
+    ):
+        ray_vllm_config = await get_ray_serve_llm_vllm_config()
+        cli_vllm_config = await get_vllm_standalone_config()
+
+        ray_config_dict = {
+            k: v
+            for k, v in vars(ray_vllm_config).items()
+            if k not in EXPECTED_DIFF_FIELDS
+        }
+        cli_config_dict = {
+            k: v
+            for k, v in vars(cli_vllm_config).items()
+            if k not in EXPECTED_DIFF_FIELDS
+        }
+
+        await normalize_parallel_config(ray_config_dict)
+        await normalize_parallel_config(cli_config_dict)
+
+        if not deep_compare(ray_config_dict, cli_config_dict):
+            differences = get_config_differences(ray_config_dict, cli_config_dict)
+            diff_msg = "\n".join(differences)
+            pytest.fail(
+                f"VllmConfig objects differ for {gpu_type} GPUs "
+                f"(compute capability {capability.major}.{capability.minor}):\n{diff_msg}"
+            )
+
+
+if __name__ == "__main__":
+    pytest.main(["-vs", __file__])

python/ray/llm/tests/serve/gpu/deployments/llm/vllm/test_vllm_engine_gpu.py

@@ -0,0 +1,67 @@
+import sys
+
+import pytest
+
+import ray
+from ray.llm._internal.serve.engines.vllm.vllm_engine import VLLMEngine
+from ray.serve.llm import LLMConfig, ModelLoadingConfig
+from ray.util.placement_group import placement_group
+from ray.util.scheduling_strategies import PlacementGroupSchedulingStrategy
+
+
+@pytest.mark.asyncio
+async def test_vllm_engine_start_with_custom_resource_bundle(
+    # defined in conftest.py
+    model_smolvlm_256m,
+):
+    """vLLM engine starts with custom resource bundle."""
+    llm_config = LLMConfig(
+        model_loading_config=ModelLoadingConfig(
+            model_id="smolvlm-256m",
+            model_source=model_smolvlm_256m,
+        ),
+        engine_kwargs=dict(
+            gpu_memory_utilization=0.4,
+            use_tqdm_on_load=False,
+            enforce_eager=True,
+            max_model_len=2048,
+        ),
+        placement_group_config={"bundles": [{"GPU": 0.49}]},
+        runtime_env=dict(
+            env_vars={
+                "VLLM_DISABLE_COMPILE_CACHE": "1",
+            },
+        ),
+    )
+
+    pg = placement_group(
+        bundles=[{"GPU": 1, "CPU": 1}],
+    )
+
+    strategy = PlacementGroupSchedulingStrategy(
+        pg, placement_group_capture_child_tasks=True, placement_group_bundle_index=0
+    )
+
+    @ray.remote(num_cpus=1, scheduling_strategy=strategy)
+    class Actor:
+        def __init__(self):
+            self.engine = VLLMEngine(llm_config)
+
+        async def start(self):
+            await self.engine.start()
+
+        async def check_health(self):
+            await self.engine.check_health()
+
+        async def shutdown(self):
+            self.engine.shutdown()
+
+    actor = Actor.remote()
+    await actor.start.remote()
+    await actor.check_health.remote()
+    await actor.shutdown.remote()
+    del pg
+
+
+if __name__ == "__main__":
+    sys.exit(pytest.main(["-v", __file__]))

python/ray/llm/tests/serve/gpu/integration/test_openai_compatibility.py

@@ -0,0 +1,175 @@
+import os
+import sys
+
+import openai
+import pytest
+import requests
+
+
+class TestOpenAICompatibility:
+    """Test that the rayllm are compatible with the OpenAI API"""
+
+    def test_models(self, testing_model):  # noqa: F811
+        client, model = testing_model
+        models = client.models.list()
+        assert len(models.data) == 1, "Only the test model should be returned"
+        assert models.data[0].id == model, "The test model id should match"
+        assert models.data[0].metadata["input_modality"] == "text"
+
+    def test_completions(self, testing_model):  # noqa: F811
+        client, model = testing_model
+        completion = client.completions.create(
+            model=model,
+            prompt="Hello world",
+            max_tokens=2,
+        )
+        assert completion.model == model
+        assert completion.model
+        assert completion.choices[0].text == "test_0 test_1"
+
+    def test_chat(self, testing_model):  # noqa: F811
+        client, model = testing_model
+        # create a chat completion
+        chat_completion = client.chat.completions.create(
+            model=model,
+            messages=[{"role": "user", "content": "Hello world"}],
+        )
+        assert chat_completion
+        assert chat_completion.usage
+        assert chat_completion.id
+        assert isinstance(chat_completion.choices, list)
+        assert chat_completion.choices[0].message.content
+
+    def test_completions_missing_model(self, testing_model):  # noqa: F811
+        client, _ = testing_model
+        with pytest.raises(openai.NotFoundError) as exc_info:
+            client.completions.create(
+                model="notarealmodel",
+                prompt="Hello world",
+            )
+        assert "Could not find" in str(exc_info.value)
+
+    def test_chat_missing_model(self, testing_model):  # noqa: F811
+        client, _ = testing_model
+        with pytest.raises(openai.NotFoundError) as exc_info:
+            client.chat.completions.create(
+                model="notarealmodel",
+                messages=[{"role": "user", "content": "Hello world"}],
+            )
+        assert "Could not find" in str(exc_info.value)
+
+    def test_completions_stream(self, testing_model):  # noqa: F811
+        client, model = testing_model
+        i = 0
+        for completion in client.completions.create(
+            model=model,
+            prompt="Hello world",
+            stream=True,
+        ):
+            i += 1
+            assert completion
+            assert completion.id
+            assert isinstance(completion.choices, list)
+            assert isinstance(completion.choices[0].text, str)
+        assert i > 4
+
+    def test_chat_stream(self, testing_model):  # noqa: F811
+        client, model = testing_model
+        i = 0
+        for chat_completion in client.chat.completions.create(
+            model=model,
+            messages=[{"role": "user", "content": "Hello world"}],
+            stream=True,
+            stream_options=dict(
+                include_usage=True,
+            ),
+            temperature=0.4,
+            frequency_penalty=0.02,
+            max_tokens=5,
+        ):
+            if i == 0:
+                assert chat_completion
+                assert chat_completion.id
+                assert isinstance(chat_completion.choices, list)
+                assert chat_completion.choices[0].delta.role
+            else:
+                assert chat_completion
+                assert chat_completion.id
+                assert isinstance(chat_completion.choices, list)
+                assert chat_completion.choices[0].delta == {} or hasattr(
+                    chat_completion.choices[0].delta, "content"
+                )
+            i += 1
+
+    def test_completions_stream_missing_model(self, testing_model):  # noqa: F811
+        client, _ = testing_model
+        with pytest.raises(openai.NotFoundError) as exc_info:
+            for _chat_completion in client.completions.create(
+                model="notarealmodel",
+                prompt="Hello world",
+                stream=True,
+            ):
+                pass
+        assert "Could not find" in str(exc_info.value)
+
+    def test_chat_stream_missing_model(self, testing_model):  # noqa: F811
+        client, _ = testing_model
+        with pytest.raises(openai.NotFoundError) as exc_info:
+            for _chat_completion in client.chat.completions.create(
+                model="notarealmodel",
+                messages=[{"role": "user", "content": "Hello world"}],
+                stream=True,
+            ):
+                pass
+        assert "Could not find" in str(exc_info.value)
+
+    def test_chat_without_model_parameter(self, testing_model):  # noqa: F811
+        """Test that chat completions work without model parameter when single model configured.
+
+        This follows vLLM's behavior from PR https://github.com/vllm-project/vllm/pull/13568
+        """
+        client, expected_model = testing_model
+        # Use requests directly since OpenAI client requires model parameter
+        response = requests.post(
+            f"{client.base_url}chat/completions",
+            json={
+                "messages": [{"role": "user", "content": "Hello world"}],
+            },
+            headers={"Authorization": f"Bearer {client.api_key}"},
+        )
+        assert (
+            response.status_code == 200
+        ), f"Expected 200, got {response.status_code}: {response.text}"
+        data = response.json()
+        assert data["model"] == expected_model
+        assert data["choices"][0]["message"]["content"]
+
+    @pytest.mark.skipif(
+        os.environ.get("RAY_SERVE_LLM_ENABLE_DIRECT_STREAMING") == "1",
+        reason="Direct streaming currently supports one LLM config.",
+    )
+    def test_chat_without_model_parameter_multiple_models(
+        self, testing_multiple_models
+    ):  # noqa: F811
+        """Test that chat completions return 400 when model not specified with multiple models.
+
+        When multiple models are configured and the model parameter is not specified,
+        an HTTP 400 Bad Request should be returned.
+        """
+        client, model_ids = testing_multiple_models
+        assert len(model_ids) > 1, "This test requires multiple models"
+        # Use requests directly since OpenAI client requires model parameter
+        response = requests.post(
+            f"{client.base_url}chat/completions",
+            json={
+                "messages": [{"role": "user", "content": "Hello world"}],
+            },
+            headers={"Authorization": f"Bearer {client.api_key}"},
+        )
+        assert (
+            response.status_code == 400
+        ), f"Expected 400, got {response.status_code}: {response.text}"
+
+
+if __name__ == "__main__":
+    sys.exit(pytest.main(["-v", __file__]))

python/ray/llm/tests/serve/gpu/integration/test_openai_compatibility_no_accelerator_type.py

@@ -0,0 +1,47 @@
+import sys
+
+import pytest
+
+
+class TestOpenAICompatibilityNoAcceleratorType:
+    """Test that rayllm is compatible with OpenAI API without specifying accelerator_type"""
+
+    def test_models_no_accelerator_type(
+        self, testing_model_no_accelerator
+    ):  # noqa: F811
+        """Check model listing without accelerator_type"""
+        client, model = testing_model_no_accelerator
+        models = client.models.list()
+        assert len(models.data) == 1, "Only the test model should be returned"
+        assert models.data[0].id == model, "The test model id should match"
+
+    def test_completions_no_accelerator_type(
+        self, testing_model_no_accelerator
+    ):  # noqa: F811
+        """Check completions without accelerator_type"""
+        client, model = testing_model_no_accelerator
+        completion = client.completions.create(
+            model=model,
+            prompt="Hello world",
+            max_tokens=2,
+        )
+        assert completion.model == model
+        assert completion.model
+        assert completion.choices[0].text == "test_0 test_1"
+
+    def test_chat_no_accelerator_type(self, testing_model_no_accelerator):  # noqa: F811
+        """Check chat completions without accelerator_type"""
+        client, model = testing_model_no_accelerator
+        chat_completion = client.chat.completions.create(
+            model=model,
+            messages=[{"role": "user", "content": "Hello world"}],
+        )
+        assert chat_completion
+        assert chat_completion.usage
+        assert chat_completion.id
+        assert isinstance(chat_completion.choices, list)
+        assert chat_completion.choices[0].message.content
+
+
+if __name__ == "__main__":
+    sys.exit(pytest.main(["-v", __file__]))