chore: import upstream snapshot with attribution
This commit is contained in:
wehub-resource-sync
@@ -0,0 +1,560 @@
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import pydantic
|
||||
import pytest
|
||||
|
||||
from ray.llm._internal.common.utils.download_utils import NodeModelDownloadable
|
||||
from ray.llm._internal.serve.core.configs.accelerators import (
|
||||
CPUAccelerator,
|
||||
CPUConfig,
|
||||
GPUAccelerator,
|
||||
GPUConfig,
|
||||
TPUAccelerator,
|
||||
TPUConfig,
|
||||
)
|
||||
from ray.llm._internal.serve.core.configs.llm_config import (
|
||||
LLMConfig,
|
||||
LoraConfig,
|
||||
ModelLoadingConfig,
|
||||
)
|
||||
from ray.llm._internal.serve.engines.vllm.vllm_models import VLLMEngineConfig
|
||||
|
||||
CONFIG_DIRS_PATH = str(Path(__file__).parent / "configs")
|
||||
|
||||
|
||||
class TestModelConfig:
|
||||
def test_construction(self):
|
||||
"""Test construct an LLMConfig doesn't error out and has correct attributes."""
|
||||
llm_config = LLMConfig(
|
||||
model_loading_config=ModelLoadingConfig(
|
||||
model_id="llm_model_id",
|
||||
),
|
||||
accelerator_type="A100-40G", # Dash instead of underscore when specifying accelerator type
|
||||
deployment_config={
|
||||
"autoscaling_config": {
|
||||
"min_replicas": 3,
|
||||
"max_replicas": 7,
|
||||
}
|
||||
},
|
||||
)
|
||||
assert llm_config.deployment_config["autoscaling_config"]["min_replicas"] == 3
|
||||
assert llm_config.deployment_config["autoscaling_config"]["max_replicas"] == 7
|
||||
assert llm_config.model_loading_config.model_id == "llm_model_id"
|
||||
assert llm_config.accelerator_type == "A100-40G"
|
||||
|
||||
def test_construction_requires_model_loading_config(self):
|
||||
"""Test that constructing an LLMConfig without model_loading_config errors out"""
|
||||
with pytest.raises(
|
||||
pydantic.ValidationError,
|
||||
):
|
||||
LLMConfig(
|
||||
accelerator_type="L4",
|
||||
)
|
||||
|
||||
def test_accelerator_type_optional(self):
|
||||
"""Test that accelerator_type is optional when initializing LLMConfig."""
|
||||
llm_config = LLMConfig(
|
||||
model_loading_config=ModelLoadingConfig(model_id="test_model")
|
||||
)
|
||||
assert llm_config.model_loading_config.model_id == "test_model"
|
||||
assert llm_config.accelerator_type is None
|
||||
|
||||
def test_invalid_accelerator_type(self):
|
||||
"""Test that invalid accelerator types raise validation errors."""
|
||||
with pytest.raises(pydantic.ValidationError):
|
||||
LLMConfig(
|
||||
model_loading_config=ModelLoadingConfig(model_id="test_model"),
|
||||
accelerator_type="INVALID_GPU", # Invalid string value
|
||||
)
|
||||
|
||||
# Test invalid numeric value
|
||||
with pytest.raises(pydantic.ValidationError):
|
||||
LLMConfig(
|
||||
model_loading_config=ModelLoadingConfig(model_id="test_model"),
|
||||
accelerator_type=123, # Must be a string
|
||||
)
|
||||
|
||||
# Test that underscore is not supported in accelerator type
|
||||
with pytest.raises(pydantic.ValidationError):
|
||||
LLMConfig(
|
||||
model_loading_config=ModelLoadingConfig(model_id="test_model"),
|
||||
accelerator_type="A100_40G", # Should use A100-40G instead
|
||||
)
|
||||
|
||||
def test_model_loading_config_forbids_extra_fields(self):
|
||||
"""Test that ModelLoadingConfig rejects extra fields."""
|
||||
|
||||
with pytest.raises(pydantic.ValidationError, match="engine_kwargs"):
|
||||
ModelLoadingConfig(
|
||||
model_id="test_model",
|
||||
model_source="test_source",
|
||||
engine_kwargs={"max_model_len": 8000}, # This should be rejected
|
||||
)
|
||||
|
||||
valid_config = ModelLoadingConfig(
|
||||
model_id="test_model", model_source="test_source"
|
||||
)
|
||||
assert valid_config.model_id == "test_model"
|
||||
assert valid_config.model_source == "test_source"
|
||||
|
||||
def test_invalid_generation_config(self, disable_placement_bundles):
|
||||
"""Test that passing an invalid generation_config raises an error."""
|
||||
with pytest.raises(
|
||||
pydantic.ValidationError,
|
||||
):
|
||||
LLMConfig(
|
||||
model_loading_config=ModelLoadingConfig(model_id="test_model"),
|
||||
accelerator_type="L4",
|
||||
generation_config="invalid_config", # Should be a dictionary, not a string
|
||||
)
|
||||
|
||||
def test_deployment_type_checking(self, disable_placement_bundles):
|
||||
"""Test that deployment config type checking works."""
|
||||
with pytest.raises(
|
||||
pydantic.ValidationError,
|
||||
):
|
||||
LLMConfig(
|
||||
model_loading_config=ModelLoadingConfig(model_id="test_model"),
|
||||
deployment_config={
|
||||
"max_ongoing_requests": -1,
|
||||
},
|
||||
accelerator_type="L4",
|
||||
)
|
||||
|
||||
def test_autoscaling_type_checking(self, disable_placement_bundles):
|
||||
"""Test that autoscaling config type checking works."""
|
||||
with pytest.raises(
|
||||
pydantic.ValidationError,
|
||||
):
|
||||
LLMConfig(
|
||||
model_loading_config=ModelLoadingConfig(model_id="test_model"),
|
||||
deployment_config={
|
||||
"autoscaling_config": {
|
||||
"min_replicas": -1,
|
||||
},
|
||||
},
|
||||
accelerator_type="L4",
|
||||
)
|
||||
|
||||
def test_deployment_unset_fields_are_not_included(self, disable_placement_bundles):
|
||||
"""Test that unset fields are not included in the deployment config."""
|
||||
llm_config = LLMConfig(
|
||||
model_loading_config=ModelLoadingConfig(model_id="test_model"),
|
||||
accelerator_type="L4",
|
||||
)
|
||||
assert "max_ongoing_requests" not in llm_config.deployment_config
|
||||
assert "graceful_shutdown_timeout_s" not in llm_config.deployment_config
|
||||
|
||||
def test_autoscaling_unset_fields_are_not_included(self, disable_placement_bundles):
|
||||
"""Test that unset fields are not included in the autoscaling config."""
|
||||
llm_config = LLMConfig(
|
||||
model_loading_config=ModelLoadingConfig(model_id="test_model"),
|
||||
deployment_config={
|
||||
"autoscaling_config": {
|
||||
"min_replicas": 3,
|
||||
"max_replicas": 7,
|
||||
},
|
||||
},
|
||||
accelerator_type="L4",
|
||||
)
|
||||
assert (
|
||||
"metrics_interval_s"
|
||||
not in llm_config.deployment_config["autoscaling_config"]
|
||||
)
|
||||
assert (
|
||||
"upscaling_factor" not in llm_config.deployment_config["autoscaling_config"]
|
||||
)
|
||||
|
||||
def test_engine_config_cached(self):
|
||||
"""Test that the engine config is cached and not recreated when calling
|
||||
get_engine_config so the attributes on the engine will be persisted."""
|
||||
|
||||
llm_config = LLMConfig(
|
||||
model_loading_config=ModelLoadingConfig(
|
||||
model_id="llm_model_id",
|
||||
),
|
||||
)
|
||||
old_engine_config = llm_config.get_engine_config()
|
||||
old_engine_config.hf_model_id = "fake_hf_model_id"
|
||||
new_engine_config = llm_config.get_engine_config()
|
||||
assert new_engine_config is old_engine_config
|
||||
|
||||
def test_remote_model_source_uses_model_id_as_hf_model_id(self):
|
||||
"""A remote model_source must not leak its URI into hf_model_id.
|
||||
|
||||
Using the URI verbatim propagates the scheme and slashes into the HF
|
||||
cache directory name (e.g. ``models--s3:----bucket--...``). The URI
|
||||
should instead be carried by mirror_config while hf_model_id falls back
|
||||
to the user-supplied model_id.
|
||||
"""
|
||||
bucket_uri = "s3://my-bucket/my-model"
|
||||
llm_config = LLMConfig(
|
||||
model_loading_config=ModelLoadingConfig(
|
||||
model_id="llm_model_id",
|
||||
model_source=bucket_uri,
|
||||
),
|
||||
)
|
||||
engine_config = llm_config.get_engine_config()
|
||||
assert engine_config.hf_model_id == "llm_model_id"
|
||||
assert engine_config.mirror_config is not None
|
||||
assert engine_config.mirror_config.bucket_uri == bucket_uri
|
||||
|
||||
def test_hf_model_source_used_as_hf_model_id(self):
|
||||
"""A plain HuggingFace model_source is used directly as hf_model_id."""
|
||||
llm_config = LLMConfig(
|
||||
model_loading_config=ModelLoadingConfig(
|
||||
model_id="llm_model_id",
|
||||
model_source="facebook/opt-1.3b",
|
||||
),
|
||||
)
|
||||
engine_config = llm_config.get_engine_config()
|
||||
assert engine_config.hf_model_id == "facebook/opt-1.3b"
|
||||
assert engine_config.mirror_config is None
|
||||
|
||||
def test_no_model_source_falls_back_to_model_id(self):
|
||||
"""With no model_source, hf_model_id falls back to model_id."""
|
||||
llm_config = LLMConfig(
|
||||
model_loading_config=ModelLoadingConfig(
|
||||
model_id="llm_model_id",
|
||||
),
|
||||
)
|
||||
engine_config = llm_config.get_engine_config()
|
||||
assert engine_config.hf_model_id == "llm_model_id"
|
||||
assert engine_config.mirror_config is None
|
||||
|
||||
def test_experimental_configs(self):
|
||||
"""Test that `experimental_configs` can be used."""
|
||||
# Test with a valid dictionary can be used.
|
||||
experimental_configs = {
|
||||
"experimental_feature1": "value1",
|
||||
"experimental_feature2": "value2",
|
||||
}
|
||||
llm_config = LLMConfig(
|
||||
model_loading_config=ModelLoadingConfig(
|
||||
model_id="llm_model_id",
|
||||
),
|
||||
experimental_configs=experimental_configs,
|
||||
)
|
||||
assert llm_config.experimental_configs == experimental_configs
|
||||
|
||||
# test with invalid dictionary will raise a validation error.
|
||||
with pytest.raises(
|
||||
pydantic.ValidationError,
|
||||
):
|
||||
LLMConfig(
|
||||
model_loading_config=ModelLoadingConfig(
|
||||
model_id="llm_model_id",
|
||||
),
|
||||
experimental_configs={123: "value1"},
|
||||
)
|
||||
|
||||
def test_log_engine_metrics_disable_log_stats_validation(self):
|
||||
"""Test that log_engine_metrics=True prevents disable_log_stats=True."""
|
||||
with pytest.raises(
|
||||
pydantic.ValidationError,
|
||||
match="disable_log_stats cannot be set to True when log_engine_metrics is enabled",
|
||||
):
|
||||
LLMConfig(
|
||||
model_loading_config=ModelLoadingConfig(model_id="test_model"),
|
||||
log_engine_metrics=True,
|
||||
engine_kwargs={"disable_log_stats": True},
|
||||
)
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"load_format,expected_download_model",
|
||||
[
|
||||
("runai_streamer", NodeModelDownloadable.NONE),
|
||||
("runai_streamer_sharded", NodeModelDownloadable.NONE),
|
||||
("tensorizer", NodeModelDownloadable.NONE),
|
||||
(None, NodeModelDownloadable.MODEL_AND_TOKENIZER),
|
||||
],
|
||||
)
|
||||
def test_load_format_callback_context(self, load_format, expected_download_model):
|
||||
"""Test that different load_format values set correct worker_node_download_model in callback context."""
|
||||
engine_kwargs = {"load_format": load_format} if load_format is not None else {}
|
||||
|
||||
llm_config = LLMConfig(
|
||||
model_loading_config=ModelLoadingConfig(model_id="test_model"),
|
||||
engine_kwargs=engine_kwargs,
|
||||
)
|
||||
|
||||
# Get the callback instance which should trigger the context setup
|
||||
callback = llm_config.get_or_create_callback()
|
||||
|
||||
# Check that the callback context has the correct worker_node_download_model value
|
||||
assert hasattr(callback, "ctx"), "Callback should have ctx attribute"
|
||||
assert callback.ctx.worker_node_download_model == expected_download_model
|
||||
|
||||
|
||||
class TestFieldValidators:
|
||||
"""Test the field validators for dict validation."""
|
||||
|
||||
def test_model_loading_config_dict_validation(self):
|
||||
"""Test that model_loading_config accepts and validates dict input."""
|
||||
config_dict = {"model_id": "microsoft/DialoGPT-medium"}
|
||||
|
||||
llm_config = LLMConfig(model_loading_config=config_dict, llm_engine="vLLM")
|
||||
|
||||
assert isinstance(llm_config.model_loading_config, ModelLoadingConfig)
|
||||
assert llm_config.model_loading_config.model_id == "microsoft/DialoGPT-medium"
|
||||
|
||||
def test_model_loading_config_validation_error(self):
|
||||
"""Test that invalid dict raises proper validation error."""
|
||||
with pytest.raises(pydantic.ValidationError) as exc_info:
|
||||
LLMConfig(
|
||||
model_loading_config={"invalid_field": "value"}, llm_engine="vLLM"
|
||||
)
|
||||
|
||||
assert "Invalid model_loading_config" in str(exc_info.value)
|
||||
|
||||
def test_lora_config_dict_validation(self):
|
||||
"""Test that lora_config accepts and validates dict input."""
|
||||
llm_config = LLMConfig(
|
||||
model_loading_config={"model_id": "test"},
|
||||
lora_config=None,
|
||||
llm_engine="vLLM",
|
||||
)
|
||||
|
||||
assert llm_config.lora_config is None
|
||||
|
||||
lora_dict = {
|
||||
"dynamic_lora_loading_path": "s3://bucket/lora",
|
||||
"max_num_adapters_per_replica": 8,
|
||||
}
|
||||
|
||||
llm_config2 = LLMConfig(
|
||||
model_loading_config={"model_id": "test"},
|
||||
lora_config=lora_dict,
|
||||
llm_engine="vLLM",
|
||||
)
|
||||
|
||||
assert isinstance(llm_config2.lora_config, LoraConfig)
|
||||
assert llm_config2.lora_config.max_num_adapters_per_replica == 8
|
||||
assert llm_config2.lora_config.dynamic_lora_loading_path == "s3://bucket/lora"
|
||||
|
||||
def test_lora_config_validation_error(self):
|
||||
"""Test that invalid lora config dict raises proper validation error."""
|
||||
with pytest.raises(pydantic.ValidationError) as exc_info:
|
||||
LLMConfig(
|
||||
model_loading_config={"model_id": "test"},
|
||||
lora_config={"max_num_adapters_per_replica": "invalid_string"},
|
||||
llm_engine="vLLM",
|
||||
)
|
||||
|
||||
assert "Invalid lora_config" in str(exc_info.value)
|
||||
|
||||
|
||||
class TestAcceleratorConfigLogic:
|
||||
"""Test the accelerator_config logic and its interaction with accelerator_type."""
|
||||
|
||||
def test_accelerator_config_field_basic(self):
|
||||
"""Test that accelerator_config field works with basic values."""
|
||||
# Test CPU config
|
||||
llm_config_cpu = LLMConfig(
|
||||
model_loading_config=ModelLoadingConfig(model_id="test_model"),
|
||||
accelerator_config={"kind": "cpu"},
|
||||
)
|
||||
assert llm_config_cpu.accelerator_config.kind == "cpu"
|
||||
engine_config = llm_config_cpu.get_engine_config()
|
||||
assert engine_config.accelerator_config.kind == "cpu"
|
||||
|
||||
# Test GPU config
|
||||
llm_config_gpu = LLMConfig(
|
||||
model_loading_config=ModelLoadingConfig(model_id="test_model"),
|
||||
accelerator_config={"kind": "gpu"},
|
||||
)
|
||||
assert llm_config_gpu.accelerator_config.kind == "gpu"
|
||||
engine_config_gpu = llm_config_gpu.get_engine_config()
|
||||
assert engine_config_gpu.accelerator_config.kind == "gpu"
|
||||
|
||||
def test_accelerator_type_with_cpu_config_raises_error(self):
|
||||
"""Test that accelerator_type with CPU config raises a validation error."""
|
||||
with pytest.raises(
|
||||
pydantic.ValidationError,
|
||||
match="accelerator_type='L4' cannot be used with CPU-only configurations",
|
||||
):
|
||||
LLMConfig(
|
||||
model_loading_config=ModelLoadingConfig(model_id="test_model"),
|
||||
accelerator_config={"kind": "cpu"},
|
||||
accelerator_type="L4",
|
||||
)
|
||||
|
||||
def test_accelerator_type_with_cpu_only_placement_group_raises_error(self):
|
||||
"""Test that accelerator_type with CPU-only placement_group_config raises error."""
|
||||
with pytest.raises(
|
||||
pydantic.ValidationError,
|
||||
match="accelerator_type='L4' cannot be used with CPU-only configurations",
|
||||
):
|
||||
LLMConfig(
|
||||
model_loading_config=ModelLoadingConfig(model_id="test_model"),
|
||||
accelerator_type="L4",
|
||||
placement_group_config={"bundles": [{"CPU": 4}]},
|
||||
)
|
||||
|
||||
def test_accelerator_type_with_empty_bundles_raises_error(self):
|
||||
"""Test that accelerator_type with empty bundles list raises error."""
|
||||
with pytest.raises(
|
||||
pydantic.ValidationError,
|
||||
match="accelerator_type='L4' cannot be used with CPU-only configurations",
|
||||
):
|
||||
LLMConfig(
|
||||
model_loading_config=ModelLoadingConfig(model_id="test_model"),
|
||||
accelerator_type="L4",
|
||||
placement_group_config={"bundles": []},
|
||||
)
|
||||
|
||||
def test_accelerator_type_with_gpu_placement_group_succeeds(self):
|
||||
"""Test that accelerator_type with GPU-containing placement_group_config succeeds."""
|
||||
llm_config = LLMConfig(
|
||||
model_loading_config=ModelLoadingConfig(model_id="test_model"),
|
||||
accelerator_type="L4",
|
||||
placement_group_config={"bundles": [{"GPU": 1, "CPU": 4}]},
|
||||
)
|
||||
assert llm_config.accelerator_type == "L4"
|
||||
|
||||
def test_accelerator_type_with_gpu_config_succeeds(self):
|
||||
"""Test that accelerator_type with GPU config succeeds."""
|
||||
llm_config = LLMConfig(
|
||||
model_loading_config=ModelLoadingConfig(model_id="test_model"),
|
||||
accelerator_type="L4",
|
||||
accelerator_config={"kind": "gpu"},
|
||||
)
|
||||
assert llm_config.accelerator_type == "L4"
|
||||
engine_config = llm_config.get_engine_config()
|
||||
assert engine_config.accelerator_type == "L4"
|
||||
|
||||
def test_vllm_engine_config_accelerator_type_with_cpu_config_raises_error(self):
|
||||
"""Test that VLLMEngineConfig rejects accelerator_type with CPU config."""
|
||||
with pytest.raises(
|
||||
pydantic.ValidationError,
|
||||
match="accelerator_type='L4' cannot be used with CPU-only configurations",
|
||||
):
|
||||
VLLMEngineConfig(
|
||||
model_id="test-model",
|
||||
accelerator_type="L4",
|
||||
accelerator_config=CPUConfig(kind="cpu"),
|
||||
)
|
||||
|
||||
def test_vllm_engine_config_accelerator_type_with_gpu_config_succeeds(self):
|
||||
"""Test that VLLMEngineConfig accepts accelerator_type with GPU config."""
|
||||
engine_config = VLLMEngineConfig(
|
||||
model_id="test-model",
|
||||
accelerator_type="L4",
|
||||
accelerator_config=GPUConfig(kind="gpu"),
|
||||
)
|
||||
|
||||
assert engine_config.accelerator_type == "L4"
|
||||
|
||||
def test_llm_config_accelerator_type_hardware_mismatch(self):
|
||||
"""Test that passing a GPU accelerator_type with a TPU config raises an error."""
|
||||
with pytest.raises(
|
||||
pydantic.ValidationError,
|
||||
match="Hardware mismatch",
|
||||
):
|
||||
LLMConfig(
|
||||
model_loading_config={"model_id": "test_model"},
|
||||
accelerator_type="L4",
|
||||
accelerator_config={"kind": "tpu", "topology": "4x4"},
|
||||
)
|
||||
|
||||
def test_engine_config_infers_tpu_from_accelerator_type_string(self):
|
||||
"""Test that the engine config infers a TPU backend directly from the accelerator_type string."""
|
||||
llm_config = LLMConfig(
|
||||
model_loading_config=ModelLoadingConfig(model_id="test_model"),
|
||||
accelerator_type="TPU-V6E",
|
||||
)
|
||||
|
||||
# Validate engine correctly inferred the TPU backend
|
||||
engine_config = llm_config.get_engine_config()
|
||||
|
||||
assert isinstance(engine_config.accelerator, TPUAccelerator)
|
||||
assert engine_config.accelerator_type == "TPU-V6E"
|
||||
|
||||
def test_requires_deferred_placement_group(self):
|
||||
"""Test that requires_deferred_placement_group correctly identifies deferred PG requirements."""
|
||||
cpu_accel = CPUAccelerator()
|
||||
assert cpu_accel.requires_deferred_placement_group is False
|
||||
|
||||
gpu_accel = GPUAccelerator()
|
||||
assert gpu_accel.requires_deferred_placement_group is False
|
||||
|
||||
tpu_accel_no_topo = TPUAccelerator(TPUConfig(kind="tpu"))
|
||||
assert tpu_accel_no_topo.requires_deferred_placement_group is False
|
||||
|
||||
tpu_accel_with_topo = TPUAccelerator(TPUConfig(kind="tpu", topology="4x4"))
|
||||
assert tpu_accel_with_topo.requires_deferred_placement_group is True
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"topology,num_devices,accelerator_type_str,expected_bundles_count,expected_chips_per_host",
|
||||
[
|
||||
("1x1", 1, "TPU-V6E", 1, 1),
|
||||
("1x1", 1, "TPU-V7X", 1, 1),
|
||||
("4x4", 16, "TPU-V6E", 4, 4),
|
||||
("2x2x2", 8, "TPU-V5P", 2, 4),
|
||||
("2x2", 4, "TPU-V5LITEPOD", 1, 4),
|
||||
("2x2x1", 4, "TPU-V4", 1, 4),
|
||||
("2x4", 8, "TPU-V6E", 1, 8),
|
||||
],
|
||||
)
|
||||
def test_default_bundles_topology(
|
||||
self,
|
||||
topology,
|
||||
num_devices,
|
||||
accelerator_type_str,
|
||||
expected_bundles_count,
|
||||
expected_chips_per_host,
|
||||
):
|
||||
"""Test that different topologies return correct per-host bundles."""
|
||||
tpu_accel = TPUAccelerator(TPUConfig(kind="tpu", topology=topology))
|
||||
bundles = tpu_accel.default_bundles(
|
||||
num_devices=num_devices, accelerator_type_str=accelerator_type_str
|
||||
)
|
||||
|
||||
assert len(bundles) == expected_bundles_count
|
||||
for bundle in bundles:
|
||||
assert bundle["TPU"] == expected_chips_per_host
|
||||
assert f"accelerator_type:{accelerator_type_str}" in bundle
|
||||
|
||||
def test_default_bundles_topology_missing_accelerator_type_raises(self):
|
||||
"""Test that ValueError is raised when topology is present but accelerator type is missing."""
|
||||
tpu_accel = TPUAccelerator(TPUConfig(kind="tpu", topology="4x4"))
|
||||
with pytest.raises(
|
||||
ValueError,
|
||||
match="`accelerator_type` must be specified when `topology` is present",
|
||||
):
|
||||
tpu_accel.default_bundles(num_devices=16, accelerator_type_str=None)
|
||||
|
||||
def test_default_bundles_topology_non_multiple_num_devices_raises(self):
|
||||
"""Test that ValueError is raised when num_devices is not a multiple of chips_per_host."""
|
||||
tpu_accel = TPUAccelerator(TPUConfig(kind="tpu", topology="4x4"))
|
||||
with pytest.raises(ValueError, match="must be a multiple of chips_per_host"):
|
||||
tpu_accel.default_bundles(num_devices=6, accelerator_type_str="TPU-V6E")
|
||||
|
||||
|
||||
class TestCheckpointInfo:
|
||||
def test_apply_checkpoint_info_uses_autoconfig_and_threads_trust_remote_code(self):
|
||||
"""apply_checkpoint_info uses AutoConfig (not PretrainedConfig) and forwards
|
||||
trust_remote_code to every HF config load call."""
|
||||
llm_config = LLMConfig(
|
||||
model_loading_config=ModelLoadingConfig(model_id="test_model")
|
||||
)
|
||||
mock_hf_config = MagicMock(spec=["architectures", "vision_config"])
|
||||
mock_hf_config.architectures = ["LlavaForCausalLM"]
|
||||
|
||||
with patch(
|
||||
"transformers.AutoConfig.from_pretrained", return_value=mock_hf_config
|
||||
) as mock_auto:
|
||||
llm_config.apply_checkpoint_info("vision/model", trust_remote_code=True)
|
||||
|
||||
assert all(
|
||||
call.kwargs["trust_remote_code"] is True
|
||||
for call in mock_auto.call_args_list
|
||||
)
|
||||
assert llm_config._supports_vision is True
|
||||
assert llm_config._model_architecture == "LlavaForCausalLM"
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(pytest.main(["-v", __file__]))
|
||||
@@ -0,0 +1,486 @@
|
||||
from typing import Any, Dict
|
||||
|
||||
import pydantic
|
||||
import pytest
|
||||
|
||||
from ray.llm._internal.serve.core.server.llm_server import LLMServer
|
||||
from ray.llm._internal.serve.engines.vllm.vllm_models import VLLMEngineConfig
|
||||
from ray.llm._internal.serve.serving_patterns.data_parallel.dp_server import DPServer
|
||||
from ray.serve.llm import LLMConfig, ModelLoadingConfig
|
||||
|
||||
|
||||
def get_llm_config_with_placement_group(
|
||||
tensor_parallel_size: int = 1,
|
||||
pipeline_parallel_size: int = 1,
|
||||
placement_group_config: Dict[str, Any] = None,
|
||||
) -> LLMConfig:
|
||||
"""Create LLMConfig with specified parallelism parameters and placement group config."""
|
||||
return LLMConfig(
|
||||
model_loading_config=ModelLoadingConfig(
|
||||
model_id="test_model",
|
||||
model_source="facebook/opt-1.3b",
|
||||
),
|
||||
deployment_config=dict(
|
||||
autoscaling_config=dict(
|
||||
min_replicas=1,
|
||||
max_replicas=1,
|
||||
),
|
||||
),
|
||||
engine_kwargs=dict(
|
||||
tensor_parallel_size=tensor_parallel_size,
|
||||
pipeline_parallel_size=pipeline_parallel_size,
|
||||
distributed_executor_backend="ray",
|
||||
),
|
||||
placement_group_config=placement_group_config,
|
||||
runtime_env=None,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"tp_size,pp_size,placement_strategy",
|
||||
[
|
||||
(2, 4, "PACK"), # Multi-node PP+TP with PACK
|
||||
(4, 2, "PACK"), # Multi-node PP+TP with PACK
|
||||
(8, 1, "SPREAD"), # Multi-node TP with SPREAD
|
||||
(1, 8, "SPREAD"), # Multi-node PP with SPREAD
|
||||
],
|
||||
)
|
||||
def test_llm_serve_custom_placement_group(tp_size, pp_size, placement_strategy):
|
||||
"""Test Ray Serve LLM with custom placement group configurations."""
|
||||
total_gpus = tp_size * pp_size
|
||||
|
||||
# Create custom placement group configuration
|
||||
placement_group_config = {
|
||||
"bundles": [{"GPU": 1, "CPU": 1}] * total_gpus,
|
||||
"strategy": placement_strategy,
|
||||
}
|
||||
|
||||
llm_config = get_llm_config_with_placement_group(
|
||||
tensor_parallel_size=tp_size,
|
||||
pipeline_parallel_size=pp_size,
|
||||
placement_group_config=placement_group_config,
|
||||
)
|
||||
|
||||
# Verify the configuration is properly set
|
||||
assert llm_config.placement_group_config == placement_group_config
|
||||
assert llm_config.engine_kwargs["tensor_parallel_size"] == tp_size
|
||||
assert llm_config.engine_kwargs["pipeline_parallel_size"] == pp_size
|
||||
|
||||
# Test that serve options are generated correctly
|
||||
serve_options = LLMServer.get_deployment_options(llm_config)
|
||||
assert "placement_group_bundles" in serve_options
|
||||
assert "placement_group_strategy" in serve_options
|
||||
assert serve_options["placement_group_strategy"] == placement_strategy
|
||||
assert len(serve_options["placement_group_bundles"]) == total_gpus
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"tp_size,pp_size",
|
||||
[
|
||||
(2, 1), # TP-only should use PACK by default
|
||||
(1, 2), # PP-only should use PACK by default
|
||||
(2, 2), # TP+PP should use PACK by default
|
||||
],
|
||||
)
|
||||
def test_llm_serve_default_placement_strategy(tp_size, pp_size):
|
||||
"""Test that Ray Serve LLM uses PACK strategy by default for all configurations."""
|
||||
llm_config = get_llm_config_with_placement_group(
|
||||
tensor_parallel_size=tp_size,
|
||||
pipeline_parallel_size=pp_size,
|
||||
placement_group_config=None, # Use defaults
|
||||
)
|
||||
|
||||
serve_options = LLMServer.get_deployment_options(llm_config)
|
||||
# All configurations should default to PACK strategy
|
||||
assert serve_options["placement_group_strategy"] == "PACK"
|
||||
assert len(serve_options["placement_group_bundles"]) == tp_size * pp_size
|
||||
|
||||
|
||||
def test_llm_serve_placement_group_validation():
|
||||
"""Test validation of placement group configurations."""
|
||||
|
||||
# Test missing both bundle_per_worker and bundles
|
||||
with pytest.raises(
|
||||
ValueError, match="must specify either 'bundle_per_worker' or 'bundles'"
|
||||
):
|
||||
llm_config = get_llm_config_with_placement_group(
|
||||
placement_group_config={"strategy": "PACK"}
|
||||
)
|
||||
LLMServer.get_deployment_options(llm_config)
|
||||
|
||||
# Test missing strategy (should default to PACK, not fail)
|
||||
llm_config = get_llm_config_with_placement_group(
|
||||
placement_group_config={"bundles": [{"GPU": 1}]}
|
||||
)
|
||||
serve_options = LLMServer.get_deployment_options(llm_config)
|
||||
assert serve_options["placement_group_strategy"] == "PACK"
|
||||
|
||||
|
||||
def test_llm_serve_multi_gpu_per_bundle_passes_through():
|
||||
"""Test multiple GPUs per bundle pass through Serve validation.
|
||||
|
||||
Serve allows GPU>1 per bundle in placement_group_config. vLLM will enforce
|
||||
its own GPU<=1 restriction during engine creation (not tested here).
|
||||
This confirms Serve doesn't block it, allowing vLLM to manage its constraints.
|
||||
"""
|
||||
llm_config = get_llm_config_with_placement_group(
|
||||
tensor_parallel_size=1,
|
||||
pipeline_parallel_size=1,
|
||||
placement_group_config={
|
||||
"bundles": [{"GPU": 2, "CPU": 4}],
|
||||
"strategy": "PACK",
|
||||
},
|
||||
)
|
||||
|
||||
# Serve should accept and pass through GPU=2 to placement group
|
||||
# First bundle gets CPU: 4 (from config) + 1 (replica actor) = 5
|
||||
serve_options = LLMServer.get_deployment_options(llm_config)
|
||||
assert serve_options["placement_group_bundles"][0]["GPU"] == 2
|
||||
assert serve_options["placement_group_bundles"][0]["CPU"] == 5
|
||||
|
||||
# vLLM will reject this during actual engine creation with a validation error
|
||||
# (not tested here since this is a config-only CPU test)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"tp_size,pp_size,expected_bundles",
|
||||
[
|
||||
(1, 1, 1),
|
||||
(2, 1, 2),
|
||||
(1, 2, 2),
|
||||
(2, 2, 4),
|
||||
(4, 2, 8),
|
||||
(2, 4, 8),
|
||||
],
|
||||
)
|
||||
def test_llm_serve_bundle_count(tp_size, pp_size, expected_bundles):
|
||||
"""Test that correct number of bundles are created for different TP/PP configs."""
|
||||
llm_config = get_llm_config_with_placement_group(
|
||||
tensor_parallel_size=tp_size,
|
||||
pipeline_parallel_size=pp_size,
|
||||
)
|
||||
|
||||
serve_options = LLMServer.get_deployment_options(llm_config)
|
||||
assert len(serve_options["placement_group_bundles"]) == expected_bundles
|
||||
|
||||
|
||||
def test_llm_serve_accelerator_and_resource_merging():
|
||||
"""Test accelerator type injection and replica actor resource merging."""
|
||||
placement_group_config = {
|
||||
"bundles": [{"GPU": 1, "CPU": 1}] * 2,
|
||||
"strategy": "PACK",
|
||||
}
|
||||
|
||||
llm_config = LLMConfig(
|
||||
model_loading_config=ModelLoadingConfig(
|
||||
model_id="test_model",
|
||||
model_source="facebook/opt-1.3b",
|
||||
),
|
||||
deployment_config=dict(
|
||||
autoscaling_config=dict(min_replicas=1, max_replicas=1),
|
||||
ray_actor_options=dict(
|
||||
num_cpus=2,
|
||||
num_gpus=1,
|
||||
memory=1000000000, # 1GB
|
||||
),
|
||||
),
|
||||
engine_kwargs=dict(
|
||||
tensor_parallel_size=2,
|
||||
pipeline_parallel_size=1,
|
||||
distributed_executor_backend="ray",
|
||||
),
|
||||
accelerator_type="L4",
|
||||
placement_group_config=placement_group_config,
|
||||
)
|
||||
|
||||
serve_options = LLMServer.get_deployment_options(llm_config)
|
||||
|
||||
# First bundle: merged replica actor resources
|
||||
# CPU: 1 (from bundle) + 2 (from replica actor) = 3
|
||||
# GPU: Already 1 in both
|
||||
first_bundle = serve_options["placement_group_bundles"][0]
|
||||
assert first_bundle["CPU"] == 3
|
||||
assert first_bundle["GPU"] == 2 # 1 from bundle + 1 from replica actor
|
||||
assert "memory" in first_bundle
|
||||
assert "accelerator_type:L4" in first_bundle
|
||||
|
||||
# Tail bundles: original config + accelerator type
|
||||
for bundle in serve_options["placement_group_bundles"][1:]:
|
||||
assert bundle["CPU"] == 1
|
||||
assert bundle["GPU"] == 1
|
||||
assert "accelerator_type:L4" in bundle
|
||||
assert bundle["accelerator_type:L4"] == 0.001
|
||||
|
||||
|
||||
def test_llm_serve_data_parallel_placement_override():
|
||||
"""Test that data parallel deployments override placement group strategy to STRICT_PACK."""
|
||||
placement_group_config = {
|
||||
"bundles": [{"GPU": 1, "CPU": 1}] * 2,
|
||||
"strategy": "SPREAD", # This should be overridden
|
||||
}
|
||||
|
||||
llm_config = LLMConfig(
|
||||
model_loading_config=ModelLoadingConfig(
|
||||
model_id="test_model",
|
||||
model_source="facebook/opt-1.3b",
|
||||
),
|
||||
# For DP correctness, do not set autoscaling_config; DP size fixes replicas
|
||||
deployment_config=dict(),
|
||||
engine_kwargs=dict(
|
||||
tensor_parallel_size=2,
|
||||
pipeline_parallel_size=1,
|
||||
data_parallel_size=2, # Enable data parallelism
|
||||
distributed_executor_backend="ray",
|
||||
),
|
||||
placement_group_config=placement_group_config,
|
||||
)
|
||||
|
||||
serve_options = DPServer.get_deployment_options(llm_config)
|
||||
|
||||
# Data parallel should override to STRICT_PACK regardless of user-specified strategy
|
||||
assert serve_options["placement_group_strategy"] == "STRICT_PACK"
|
||||
# Note: num_replicas is set by build_dp_deployment, not by get_deployment_options
|
||||
|
||||
|
||||
def test_fractional_gpu_env_inferred_from_pg():
|
||||
"""A fractional placement group should inject VLLM_RAY_PER_WORKER_GPUS."""
|
||||
placement_group_config = {"bundles": [{"GPU": 0.49}]}
|
||||
llm_config = LLMConfig(
|
||||
model_loading_config=ModelLoadingConfig(
|
||||
model_id="test_model",
|
||||
model_source="facebook/opt-125m",
|
||||
),
|
||||
placement_group_config=placement_group_config,
|
||||
runtime_env=dict(env_vars={"EXTRA_VAR": "1"}),
|
||||
)
|
||||
|
||||
engine_config = VLLMEngineConfig.from_llm_config(llm_config)
|
||||
runtime_env = engine_config.get_runtime_env_with_local_env_vars()
|
||||
|
||||
assert runtime_env["env_vars"]["EXTRA_VAR"] == "1"
|
||||
assert runtime_env["env_vars"]["VLLM_RAY_PER_WORKER_GPUS"] == "0.49"
|
||||
|
||||
|
||||
def test_fractional_gpu_env_var_override_preserved():
|
||||
"""User-provided env var should be preserved when set."""
|
||||
llm_config = LLMConfig(
|
||||
model_loading_config=ModelLoadingConfig(
|
||||
model_id="test_model",
|
||||
model_source="facebook/opt-125m",
|
||||
),
|
||||
placement_group_config={"bundles": [{"GPU": 0.4}]},
|
||||
runtime_env=dict(
|
||||
env_vars={
|
||||
"VLLM_RAY_PER_WORKER_GPUS": "0.6",
|
||||
}
|
||||
),
|
||||
)
|
||||
|
||||
engine_config = VLLMEngineConfig.from_llm_config(llm_config)
|
||||
runtime_env = engine_config.get_runtime_env_with_local_env_vars()
|
||||
|
||||
assert runtime_env["env_vars"]["VLLM_RAY_PER_WORKER_GPUS"] == "0.6"
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"tp_size,pp_size",
|
||||
[
|
||||
(2, 1),
|
||||
(1, 2),
|
||||
(2, 2),
|
||||
(4, 2),
|
||||
],
|
||||
)
|
||||
def test_bundle_per_worker_expands_correctly(tp_size, pp_size):
|
||||
"""Test that bundle_per_worker is auto-replicated based on tp*pp."""
|
||||
expected_bundles = tp_size * pp_size
|
||||
|
||||
llm_config = get_llm_config_with_placement_group(
|
||||
tensor_parallel_size=tp_size,
|
||||
pipeline_parallel_size=pp_size,
|
||||
placement_group_config={"bundle_per_worker": {"GPU": 1, "CPU": 2}},
|
||||
)
|
||||
|
||||
# Verify the configuration is properly set
|
||||
assert llm_config.placement_group_config == {
|
||||
"bundle_per_worker": {"GPU": 1, "CPU": 2}
|
||||
}
|
||||
|
||||
# Test that serve options are generated correctly
|
||||
serve_options = LLMServer.get_deployment_options(llm_config)
|
||||
assert "placement_group_bundles" in serve_options
|
||||
assert len(serve_options["placement_group_bundles"]) == expected_bundles
|
||||
|
||||
# Each bundle should have the specified resources (plus accelerator hint injection)
|
||||
for i, bundle in enumerate(serve_options["placement_group_bundles"]):
|
||||
# First bundle gets merged with replica actor resources
|
||||
if i == 0:
|
||||
assert bundle["GPU"] >= 1
|
||||
assert bundle["CPU"] >= 2
|
||||
else:
|
||||
assert bundle["GPU"] == 1
|
||||
assert bundle["CPU"] == 2
|
||||
|
||||
|
||||
def test_bundle_per_worker_conflict_with_bundles():
|
||||
"""Test that specifying both bundle_per_worker and bundles raises error."""
|
||||
with pytest.raises(ValueError, match="Cannot specify both"):
|
||||
llm_config = LLMConfig(
|
||||
model_loading_config=ModelLoadingConfig(
|
||||
model_id="test_model",
|
||||
model_source="facebook/opt-125m",
|
||||
),
|
||||
placement_group_config={
|
||||
"bundle_per_worker": {"GPU": 1, "CPU": 1},
|
||||
"bundles": [{"GPU": 1}],
|
||||
},
|
||||
)
|
||||
# Validation happens when VLLMEngineConfig is created
|
||||
VLLMEngineConfig.from_llm_config(llm_config)
|
||||
|
||||
|
||||
def test_bundle_per_worker_uses_pack_strategy():
|
||||
"""Test that bundle_per_worker defaults to PACK strategy."""
|
||||
llm_config = get_llm_config_with_placement_group(
|
||||
tensor_parallel_size=2,
|
||||
pipeline_parallel_size=1,
|
||||
placement_group_config={"bundle_per_worker": {"GPU": 1}},
|
||||
)
|
||||
|
||||
serve_options = LLMServer.get_deployment_options(llm_config)
|
||||
assert serve_options["placement_group_strategy"] == "PACK"
|
||||
|
||||
|
||||
def test_bundle_per_worker_injects_accelerator_type():
|
||||
"""Test that bundle_per_worker bundles get accelerator type hint injected."""
|
||||
llm_config = LLMConfig(
|
||||
model_loading_config=ModelLoadingConfig(
|
||||
model_id="test_model",
|
||||
model_source="facebook/opt-125m",
|
||||
),
|
||||
engine_kwargs=dict(
|
||||
tensor_parallel_size=2,
|
||||
pipeline_parallel_size=1,
|
||||
distributed_executor_backend="ray",
|
||||
),
|
||||
accelerator_type="L4",
|
||||
placement_group_config={"bundle_per_worker": {"GPU": 1, "CPU": 2}},
|
||||
)
|
||||
|
||||
serve_options = LLMServer.get_deployment_options(llm_config)
|
||||
|
||||
# All bundles should have accelerator type hint injected
|
||||
for bundle in serve_options["placement_group_bundles"]:
|
||||
assert "accelerator_type:L4" in bundle
|
||||
assert bundle["accelerator_type:L4"] == 0.001
|
||||
|
||||
|
||||
def test_bundle_per_worker_fractional_gpu_env_var():
|
||||
"""Test that fractional GPU in bundle_per_worker injects VLLM_RAY_PER_WORKER_GPUS."""
|
||||
llm_config = LLMConfig(
|
||||
model_loading_config=ModelLoadingConfig(
|
||||
model_id="test_model",
|
||||
model_source="facebook/opt-125m",
|
||||
),
|
||||
placement_group_config={"bundle_per_worker": {"GPU": 0.5, "CPU": 1}},
|
||||
)
|
||||
|
||||
engine_config = VLLMEngineConfig.from_llm_config(llm_config)
|
||||
runtime_env = engine_config.get_runtime_env_with_local_env_vars()
|
||||
|
||||
assert runtime_env["env_vars"]["VLLM_RAY_PER_WORKER_GPUS"] == "0.5"
|
||||
|
||||
|
||||
def test_bundle_per_worker_non_fractional_gpu_no_env_var():
|
||||
"""Test that non-fractional GPU in bundle_per_worker does not inject env var."""
|
||||
llm_config = LLMConfig(
|
||||
model_loading_config=ModelLoadingConfig(
|
||||
model_id="test_model",
|
||||
model_source="facebook/opt-125m",
|
||||
),
|
||||
placement_group_config={"bundle_per_worker": {"GPU": 1, "CPU": 1}},
|
||||
)
|
||||
|
||||
engine_config = VLLMEngineConfig.from_llm_config(llm_config)
|
||||
runtime_env = engine_config.get_runtime_env_with_local_env_vars()
|
||||
|
||||
assert "VLLM_RAY_PER_WORKER_GPUS" not in runtime_env.get("env_vars", {})
|
||||
|
||||
|
||||
def test_llm_serve_placement_group_explicit_none():
|
||||
"""Test that explicitly setting bundle_per_worker key to None does not crash."""
|
||||
llm_config = LLMConfig(
|
||||
model_loading_config=ModelLoadingConfig(
|
||||
model_id="test_model",
|
||||
model_source="facebook/opt-1.3b",
|
||||
),
|
||||
placement_group_config={
|
||||
"bundle_per_worker": None,
|
||||
"bundles": [{"GPU": 1}],
|
||||
},
|
||||
)
|
||||
|
||||
# This should succeed fall back to the GPU bundles
|
||||
serve_options = LLMServer.get_deployment_options(llm_config)
|
||||
assert len(serve_options["placement_group_bundles"]) > 0
|
||||
|
||||
|
||||
class TestAcceleratorTypeValidation:
|
||||
"""Test accelerator_type validation with CPU-only configurations."""
|
||||
|
||||
def test_llm_config_accelerator_type_with_cpu_config_raises_error(self):
|
||||
"""LLMConfig raises error with accelerator_type and CPU config."""
|
||||
with pytest.raises(
|
||||
pydantic.ValidationError,
|
||||
match="cannot be used with CPU-only configurations",
|
||||
):
|
||||
LLMConfig(
|
||||
model_loading_config={"model_id": "test_model"},
|
||||
accelerator_type="L4",
|
||||
accelerator_config={"kind": "cpu"},
|
||||
)
|
||||
|
||||
def test_llm_config_accelerator_type_with_cpu_only_bundles_raises_error(self):
|
||||
"""LLMConfig raises error with accelerator_type and CPU-only bundles."""
|
||||
with pytest.raises(
|
||||
pydantic.ValidationError,
|
||||
match="cannot be used with CPU-only configurations",
|
||||
):
|
||||
LLMConfig(
|
||||
model_loading_config={"model_id": "test_model"},
|
||||
accelerator_type="L4",
|
||||
placement_group_config={"bundles": [{"CPU": 4}]},
|
||||
)
|
||||
|
||||
def test_llm_config_accelerator_type_with_empty_bundles_raises_error(self):
|
||||
"""LLMConfig raises error with accelerator_type and empty bundles."""
|
||||
with pytest.raises(
|
||||
pydantic.ValidationError,
|
||||
match="cannot be used with CPU-only configurations",
|
||||
):
|
||||
LLMConfig(
|
||||
model_loading_config={"model_id": "test_model"},
|
||||
accelerator_type="L4",
|
||||
placement_group_config={"bundles": []},
|
||||
)
|
||||
|
||||
def test_llm_config_accelerator_type_with_gpu_bundles_succeeds(self):
|
||||
"""Test that LLMConfig succeeds when accelerator_type is set with GPU bundles."""
|
||||
config = LLMConfig(
|
||||
model_loading_config={"model_id": "test_model"},
|
||||
accelerator_type="L4",
|
||||
placement_group_config={"bundles": [{"GPU": 1, "CPU": 4}]},
|
||||
)
|
||||
assert config.accelerator_type == "L4"
|
||||
|
||||
def test_llm_config_accelerator_type_default_uses_gpu(self):
|
||||
"""Test that LLMConfig with accelerator_type defaults to GPU."""
|
||||
config = LLMConfig(
|
||||
model_loading_config={"model_id": "test_model"},
|
||||
accelerator_type="L4",
|
||||
)
|
||||
assert config.accelerator_type == "L4"
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
pytest.main(["-v", __file__])
|
||||
@@ -0,0 +1,232 @@
|
||||
"""Tests for openai_api_models.py engine-agnostic import behaviour.
|
||||
|
||||
SGLang fallback tests live in the llm_serve_sglang_e2e release test.
|
||||
"""
|
||||
|
||||
import importlib
|
||||
import sys
|
||||
|
||||
import pytest
|
||||
|
||||
_OAI_MODELS_MOD = "ray.llm._internal.serve.core.configs.openai_api_models"
|
||||
|
||||
|
||||
class _VLLMImportBlocker:
|
||||
"""Meta-path finder that simulates vLLM not being installed.
|
||||
|
||||
Raises ModuleNotFoundError with .name set to mirror what Python raises
|
||||
when a package is genuinely absent, so raise_llm_engine_import_error
|
||||
can distinguish "not installed" from "installed but broken".
|
||||
"""
|
||||
|
||||
def find_spec(self, fullname, path=None, target=None):
|
||||
if fullname == "vllm" or fullname.startswith("vllm."):
|
||||
err = ModuleNotFoundError(f"Mocked: {fullname} is not installed")
|
||||
err.name = fullname
|
||||
raise err
|
||||
return None
|
||||
|
||||
|
||||
class _VLLMBrokenInstallBlocker:
|
||||
"""Meta-path finder that simulates vLLM installed but broken at runtime
|
||||
(e.g. missing libcudart.so or a missing transitive dependency).
|
||||
"""
|
||||
|
||||
def __init__(self, error: ImportError):
|
||||
self._error = error
|
||||
|
||||
def find_spec(self, fullname, path=None, target=None):
|
||||
if fullname == "vllm" or fullname.startswith("vllm."):
|
||||
raise self._error
|
||||
return None
|
||||
|
||||
|
||||
class TestVLLMBackend:
|
||||
def test_wrapper_classes_inherit_from_vllm(self):
|
||||
from ray.llm._internal.serve.core.configs.openai_api_models import (
|
||||
ChatCompletionRequest,
|
||||
CompletionRequest,
|
||||
ErrorInfo,
|
||||
ErrorResponse,
|
||||
)
|
||||
|
||||
assert "vllm" in ChatCompletionRequest.__mro__[1].__module__
|
||||
assert "vllm" in CompletionRequest.__mro__[1].__module__
|
||||
assert "vllm" in ErrorInfo.__mro__[1].__module__
|
||||
assert "vllm" in ErrorResponse.__mro__[1].__module__
|
||||
|
||||
def test_error_response_round_trip(self):
|
||||
from ray.llm._internal.serve.core.configs.openai_api_models import (
|
||||
ErrorInfo,
|
||||
ErrorResponse,
|
||||
)
|
||||
|
||||
info = ErrorInfo(message="something broke", code=500, type="InternalError")
|
||||
resp = ErrorResponse(error=info)
|
||||
assert resp.error.message == "something broke"
|
||||
assert resp.error.code == 500
|
||||
assert resp.error.type == "InternalError"
|
||||
|
||||
dumped = resp.model_dump()
|
||||
assert dumped["error"]["message"] == "something broke"
|
||||
|
||||
def test_transcription_request_has_request_id(self):
|
||||
from ray.llm._internal.serve.core.configs.openai_api_models import (
|
||||
TranscriptionRequest,
|
||||
)
|
||||
|
||||
assert "request_id" in TranscriptionRequest.model_fields
|
||||
|
||||
def _reload_oai_models_with_blocker(self, blocker):
|
||||
"""Helper: evict vllm + the target module, install blocker, reimport."""
|
||||
saved = {
|
||||
k: sys.modules.pop(k)
|
||||
for k in list(sys.modules)
|
||||
if k == "vllm" or k.startswith("vllm.")
|
||||
}
|
||||
sys.modules.pop(_OAI_MODELS_MOD, None)
|
||||
sys.meta_path.insert(0, blocker)
|
||||
try:
|
||||
importlib.import_module(_OAI_MODELS_MOD)
|
||||
finally:
|
||||
sys.meta_path.remove(blocker)
|
||||
sys.modules.pop(_OAI_MODELS_MOD, None)
|
||||
sys.modules.update(saved)
|
||||
|
||||
def test_import_error_when_vllm_blocked(self):
|
||||
"""SGLang is not installed here either, so blocking vLLM means neither
|
||||
backend is available."""
|
||||
with pytest.raises(ImportError, match="Neither vLLM nor SGLang"):
|
||||
self._reload_oai_models_with_blocker(_VLLMImportBlocker())
|
||||
|
||||
def test_vllm_installed_but_broken_cuda(self):
|
||||
"""Plain ImportError (e.g. missing libcudart.so) → clear message that
|
||||
vLLM is installed but failed to load, not 'not installed'."""
|
||||
cuda_err = ImportError(
|
||||
"libcudart.so.12: cannot open shared object file: No such file or directory"
|
||||
)
|
||||
blocker = _VLLMBrokenInstallBlocker(cuda_err)
|
||||
with pytest.raises(ImportError, match="vLLM is installed but failed to import"):
|
||||
self._reload_oai_models_with_blocker(blocker)
|
||||
|
||||
def test_vllm_installed_but_missing_transitive_dep(self):
|
||||
"""ModuleNotFoundError for a *dependency* of vLLM (not vllm itself)
|
||||
must also be reported as 'installed but broken', not 'not installed'."""
|
||||
dep_err = ModuleNotFoundError("No module named 'msgpack'")
|
||||
dep_err.name = "msgpack"
|
||||
blocker = _VLLMBrokenInstallBlocker(dep_err)
|
||||
with pytest.raises(ImportError, match="vLLM is installed but failed to import"):
|
||||
self._reload_oai_models_with_blocker(blocker)
|
||||
|
||||
|
||||
class TestSanitizeChatCompletionRequest:
|
||||
def test_serializes_tool_calls_iterator(self):
|
||||
from ray.llm._internal.serve.core.configs.openai_api_models import (
|
||||
ChatCompletionRequest,
|
||||
)
|
||||
from ray.llm._internal.serve.core.ingress.ingress import (
|
||||
_sanitize_chat_completion_request,
|
||||
)
|
||||
|
||||
tool_calls = [
|
||||
{
|
||||
"id": "call_1",
|
||||
"type": "function",
|
||||
"function": {"name": "f", "arguments": "{}"},
|
||||
},
|
||||
]
|
||||
|
||||
request = ChatCompletionRequest(
|
||||
model="test-model",
|
||||
messages=[
|
||||
{"role": "user", "content": "hello"},
|
||||
{"role": "assistant", "content": None, "tool_calls": iter(tool_calls)},
|
||||
],
|
||||
)
|
||||
|
||||
result = _sanitize_chat_completion_request(request)
|
||||
assistant_msg = result.messages[1]
|
||||
assert isinstance(assistant_msg["tool_calls"], list)
|
||||
assert len(assistant_msg["tool_calls"]) == 1
|
||||
|
||||
def test_handles_no_tool_calls(self):
|
||||
from ray.llm._internal.serve.core.configs.openai_api_models import (
|
||||
ChatCompletionRequest,
|
||||
)
|
||||
from ray.llm._internal.serve.core.ingress.ingress import (
|
||||
_sanitize_chat_completion_request,
|
||||
)
|
||||
|
||||
request = ChatCompletionRequest(
|
||||
model="test-model",
|
||||
messages=[
|
||||
{"role": "user", "content": "hello"},
|
||||
{"role": "assistant", "content": "world"},
|
||||
],
|
||||
)
|
||||
|
||||
result = _sanitize_chat_completion_request(request)
|
||||
assert result is request
|
||||
# Sanitizer must not inject tool_calls onto messages that never had them.
|
||||
assistant_msg = result.messages[1]
|
||||
if isinstance(assistant_msg, dict):
|
||||
assert assistant_msg.get("tool_calls") is None
|
||||
else:
|
||||
assert getattr(assistant_msg, "tool_calls", None) is None
|
||||
|
||||
def test_serializes_content_iterator(self):
|
||||
"""When `content` is sent as a list of content parts on any message,
|
||||
Pydantic stores it as a ValidatorIterator against the `Iterable[...]`
|
||||
arm and the request becomes unpicklable until sanitized."""
|
||||
import pickle
|
||||
|
||||
from ray.llm._internal.serve.core.configs.openai_api_models import (
|
||||
ChatCompletionRequest,
|
||||
)
|
||||
from ray.llm._internal.serve.core.ingress.ingress import (
|
||||
_sanitize_chat_completion_request,
|
||||
)
|
||||
|
||||
request = ChatCompletionRequest(
|
||||
model="test-model",
|
||||
messages=[
|
||||
{"role": "system", "content": [{"text": "sys", "type": "text"}]},
|
||||
{"role": "user", "content": [{"text": "hi", "type": "text"}]},
|
||||
{
|
||||
"role": "assistant",
|
||||
"content": [{"text": "step", "type": "text"}],
|
||||
},
|
||||
{
|
||||
"role": "tool",
|
||||
"content": [{"text": "r", "type": "text"}],
|
||||
"tool_call_id": "c0",
|
||||
},
|
||||
],
|
||||
)
|
||||
|
||||
result = _sanitize_chat_completion_request(request)
|
||||
for msg in result.messages:
|
||||
assert isinstance(msg, dict)
|
||||
assert isinstance(msg["content"], list)
|
||||
assert type(msg["content"]).__name__ != "ValidatorIterator"
|
||||
|
||||
pickle.dumps(result)
|
||||
|
||||
|
||||
class TestLLMServerLazyImport:
|
||||
def test_default_engine_cls_is_none(self):
|
||||
mod_name = "ray.llm._internal.serve.core.server.llm_server"
|
||||
old_mod = sys.modules.pop(mod_name, None)
|
||||
try:
|
||||
mod = importlib.import_module(mod_name)
|
||||
assert mod.LLMServer._default_engine_cls is None
|
||||
finally:
|
||||
if old_mod is not None:
|
||||
sys.modules[mod_name] = old_mod
|
||||
|
||||
def test_ray_serve_llm_importable(self):
|
||||
import ray.serve.llm # noqa: F401
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(pytest.main(["-v", __file__]))
|
||||
Reference in New Issue
Block a user