# SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project import argparse import copy import dataclasses import functools import json import os import sys from collections.abc import Callable from dataclasses import MISSING, asdict, dataclass, fields, is_dataclass from itertools import permutations from types import UnionType from typing import ( TYPE_CHECKING, Annotated, Any, Literal, TypeAlias, TypeVar, Union, cast, get_args, get_origin, ) import huggingface_hub import regex as re import torch from pydantic import TypeAdapter, ValidationError from pydantic.fields import FieldInfo from typing_extensions import TypeIs import vllm.envs as envs from vllm.config import ( AttentionConfig, CacheConfig, CompilationConfig, ConfigType, DeviceConfig, DiffusionConfig, ECTransferConfig, EPLBConfig, KernelConfig, KVEventsConfig, KVTransferConfig, LoadConfig, LoRAConfig, MambaConfig, ModelConfig, MultiModalConfig, ObservabilityConfig, OffloadConfig, ParallelConfig, PoolerConfig, PrefetchOffloadConfig, ProfilerConfig, ReasoningConfig, SchedulerConfig, SpeculativeConfig, StructuredOutputsConfig, UVAOffloadConfig, VllmConfig, WeightTransferConfig, get_attr_docs, ) from vllm.config.cache import ( CacheDType, KVOffloadingBackend, MambaCacheMode, MambaDType, PrefixCachingHashAlgo, ) from vllm.config.device import Device from vllm.config.kernel import IrOpPriorityConfig, LinearBackend, MoEBackend from vllm.config.load import SafetensorsLoadStrategy from vllm.config.lora import MaxLoRARanks from vllm.config.mamba import MambaBackendEnum from vllm.config.model import ( ConvertOption, HfOverrides, LogprobsMode, ModelDType, RunnerOption, TokenizerMode, ) from vllm.config.multimodal import MMCacheType, MMEncoderTPMode, MMTensorIPC from vllm.config.observability import DetailedTraceModules from vllm.config.parallel import ( All2AllBackend, DataParallelBackend, DCPCommBackend, DistributedExecutorBackend, ExpertPlacementStrategy, ) from vllm.config.scheduler import SchedulerPolicy from vllm.config.utils import get_field from vllm.config.vllm import OptimizationLevel, PerformanceMode from vllm.logger import init_logger, suppress_logging from vllm.platforms import CpuArchEnum, current_platform from vllm.plugins import load_general_plugins from vllm.ray.lazy_utils import is_in_ray_actor, is_ray_initialized from vllm.transformers_utils.config import ( is_interleaved, maybe_override_with_speculators, ) from vllm.transformers_utils.repo_utils import get_model_path from vllm.transformers_utils.utils import is_cloud_storage from vllm.utils.argparse_utils import ( FlexibleArgumentParser, human_readable_int, human_readable_int_or_auto, ) from vllm.utils.mem_constants import GiB_bytes from vllm.utils.network_utils import get_ip from vllm.utils.torch_utils import resolve_kv_cache_dtype_string from vllm.v1.attention.backends.registry import AttentionBackendEnum from vllm.v1.sample.logits_processor import LogitsProcessor from vllm.version import __version__ as VLLM_VERSION if TYPE_CHECKING: from vllm.config.quantization import QuantizationConfigArgs from vllm.model_executor.layers.quantization import QuantizationMethods from vllm.model_executor.model_loader import LoadFormats from vllm.usage.usage_lib import UsageContext from vllm.v1.executor import Executor else: Executor = Any QuantizationMethods = str LoadFormats = str UsageContext = Any logger = init_logger(__name__) # object is used to allow for special typing forms T = TypeVar("T") TypeHint: TypeAlias = type[Any] | object TypeHintT: TypeAlias = type[T] | object def parse_type(return_type: Callable[[str], T]) -> Callable[[str], T]: def _parse_type(val: str) -> T: try: return return_type(val) except ValueError as e: raise argparse.ArgumentTypeError( f"Value {val} cannot be converted to {return_type}." ) from e return _parse_type def optional_type(return_type: Callable[[str], T]) -> Callable[[str], T | None]: def _optional_type(val: str) -> T | None: if val == "" or val == "None": return None return parse_type(return_type)(val) return _optional_type def union_dict_and_str(val: str) -> str | dict[str, str] | None: if not re.match(r"(?s)^\s*{.*}\s*$", val): return str(val) return optional_type(json.loads)(val) def is_type(type_hint: TypeHint, type: TypeHintT) -> TypeIs[TypeHintT]: """Check if the type hint is a specific type.""" return type_hint is type or get_origin(type_hint) is type def contains_type(type_hints: set[TypeHint], type: TypeHintT) -> bool: """Check if the type hints contain a specific type.""" return any(is_type(type_hint, type) for type_hint in type_hints) def get_type(type_hints: set[TypeHint], type: TypeHintT) -> TypeHintT: """Get the specific type from the type hints.""" return next((th for th in type_hints if is_type(th, type)), None) def literal_to_kwargs(type_hints: set[TypeHint]) -> dict[str, Any]: """Get the `type` and `choices` from a `Literal` type hint in `type_hints`. If `type_hints` also contains `str`, we use `metavar` instead of `choices`. """ type_hint = get_type(type_hints, Literal) options = get_args(type_hint) option_type = type(options[0]) if not all(isinstance(option, option_type) for option in options): raise ValueError( "All options must be of the same type. " f"Got {options} with types {[type(c) for c in options]}" ) kwarg = "metavar" if contains_type(type_hints, str) else "choices" return {"type": option_type, kwarg: sorted(options)} def collection_to_kwargs(type_hints: set[TypeHint], type: TypeHint) -> dict[str, Any]: type_hint = get_type(type_hints, type) types = get_args(type_hint) elem_type = types[0] # Handle Ellipsis assert all(t is elem_type for t in types if t is not Ellipsis), ( f"All non-Ellipsis elements must be of the same type. Got {types}." ) # Handle Union types if get_origin(elem_type) in {Union, UnionType}: # Union for Union[X, Y] and UnionType for X | Y assert str in get_args(elem_type), ( "If element can have multiple types, one must be 'str' " f"(i.e. 'list[int | str]'). Got {elem_type}." ) elem_type = str return { "type": elem_type, "nargs": "+" if type is not tuple or Ellipsis in types else len(types), } def is_not_builtin(type_hint: TypeHint) -> bool: """Check if the class is not a built-in type.""" return type_hint.__module__ != "builtins" def get_type_hints(type_hint: TypeHint) -> set[TypeHint]: """Extract type hints from Annotated or Union type hints.""" type_hints: set[TypeHint] = set() origin = get_origin(type_hint) args = get_args(type_hint) if origin is Annotated: type_hints.update(get_type_hints(args[0])) elif origin in {Union, UnionType}: # Union for Union[X, Y] and UnionType for X | Y for arg in args: type_hints.update(get_type_hints(arg)) else: type_hints.add(type_hint) return type_hints NEEDS_HELP = ( any("--help" in arg for arg in sys.argv) # vllm SUBCOMMAND --help or (argv0 := sys.argv[0]).endswith("mkdocs") # mkdocs SUBCOMMAND or argv0.endswith("mkdocs/__main__.py") # python -m mkdocs SUBCOMMAND ) def _maybe_add_docs_url(cls: Any) -> str: """Generate API docs URL for a vllm config class.""" if not cls.__module__.startswith("vllm.config"): return "" version = f"v{VLLM_VERSION}" if "dev" not in VLLM_VERSION else "latest" return f"\n\nAPI docs: https://docs.vllm.ai/en/{version}/api/vllm/config/#vllm.config.{cls.__name__}" def _expand_json_human_readable_numbers(val: str) -> str: """Expand human-readable number suffixes in a JSON string. Based on :func:`human_readable_int` so that the ``k/m/g/t`` (decimal) and ``K/M/G/T`` (binary) conventions work out the box. Also works inside JSON config arguments such as ``--kv-transfer-config '{"cpu_bytes_to_use": 80m}'``. Only bare (unquoted) tokens are replaced so that JSON string values like ``"model_name"`` are never modified. """ # Split on quoted strings so we only touch non-string regions. parts = re.split(r'("(?:[^"\\]|\\.)*")', val) for i in range(0, len(parts), 2): # even indices = outside strings parts[i] = re.sub( r"\b\d+(?:\.\d+)?[kKmMgGtT]\b", lambda m: str(human_readable_int(m.group())), parts[i], ) return "".join(parts) @functools.lru_cache(maxsize=30) def _compute_kwargs(cls: ConfigType) -> dict[str, dict[str, Any]]: # Save time only getting attr docs if we're generating help text cls_docs = get_attr_docs(cls) if NEEDS_HELP else {} kwargs = {} for field in fields(cls): # Get the set of possible types for the field type_hints: set[TypeHint] = get_type_hints(field.type) # If the field is a dataclass, we can use the model_validate_json generator = (th for th in type_hints if is_dataclass(th)) dataclass_cls = next(generator, None) # Get the default value of the field if field.default is not MISSING: default = field.default # Handle pydantic.Field defaults if isinstance(default, FieldInfo): if default.default_factory is None: default = default.default else: # VllmConfig's Fields have default_factory set to config classes. # These could emit logs on init, which would be confusing. with suppress_logging(): default = default.default_factory() # type: ignore[call-arg] elif field.default_factory is not MISSING: default = field.default_factory() # Get the help text for the field name = field.name help = cls_docs.get(name, "").strip() # Escape % for argparse help = help.replace("%", "%%") # Initialise the kwargs dictionary for the field kwargs[name] = {"default": default, "help": help} # Set other kwargs based on the type hints json_tip = ( "Should either be a valid JSON string or JSON keys passed individually." ) if dataclass_cls is not None: def parse_dataclass(val: str, cls=dataclass_cls) -> Any: try: val = _expand_json_human_readable_numbers(val) return TypeAdapter(cls).validate_json(val) except ValidationError as e: raise argparse.ArgumentTypeError(repr(e)) from e kwargs[name]["type"] = parse_dataclass kwargs[name]["help"] += _maybe_add_docs_url(dataclass_cls) kwargs[name]["help"] += f"\n\n{json_tip}" elif type_hints == {bool, str, type(None)}: # Optional-valued flag: bare flag -> True, value -> str. kwargs[name]["type"] = str kwargs[name]["nargs"] = "?" kwargs[name]["const"] = True elif contains_type(type_hints, bool): # Creates --no- and -- flags kwargs[name]["action"] = argparse.BooleanOptionalAction elif contains_type(type_hints, Literal): kwargs[name].update(literal_to_kwargs(type_hints)) elif contains_type(type_hints, tuple): kwargs[name].update(collection_to_kwargs(type_hints, tuple)) elif contains_type(type_hints, list): kwargs[name].update(collection_to_kwargs(type_hints, list)) elif contains_type(type_hints, set): kwargs[name].update(collection_to_kwargs(type_hints, set)) elif contains_type(type_hints, int): # Arguments that accept human-readable integer strings (e.g., 1K, 2M, 1G) human_readable_int_args = { "max_num_batched_tokens", "max_num_scheduled_tokens", "kv_cache_memory_bytes", "safetensors_prefetch_block_size", } if name == "max_model_len": kwargs[name]["type"] = human_readable_int_or_auto kwargs[name]["help"] += f"\n\n{human_readable_int_or_auto.__doc__}" elif name in human_readable_int_args: kwargs[name]["type"] = human_readable_int kwargs[name]["help"] += f"\n\n{human_readable_int.__doc__}" else: kwargs[name]["type"] = int elif contains_type(type_hints, float): kwargs[name]["type"] = float elif contains_type(type_hints, dict) and ( contains_type(type_hints, str) or any(is_not_builtin(th) for th in type_hints) ): kwargs[name]["type"] = union_dict_and_str elif contains_type(type_hints, dict): kwargs[name]["type"] = parse_type(json.loads) kwargs[name]["help"] += f"\n\n{json_tip}" elif contains_type(type_hints, str) or any( is_not_builtin(th) for th in type_hints ): kwargs[name]["type"] = str else: raise ValueError(f"Unsupported type {type_hints} for argument {name}.") # If the type hint was a sequence of literals, use the helper function # to update the type and choices if get_origin(kwargs[name].get("type")) is Literal: kwargs[name].update(literal_to_kwargs({kwargs[name]["type"]})) # If None is in type_hints, make the argument optional. # But not if it's a bool, argparse will handle this better. if type(None) in type_hints and not contains_type(type_hints, bool): kwargs[name]["type"] = optional_type(kwargs[name]["type"]) if kwargs[name].get("choices"): kwargs[name]["choices"].append("None") return kwargs def get_kwargs(cls: ConfigType) -> dict[str, dict[str, Any]]: """Return argparse kwargs for the given Config dataclass. If `--help` or `mkdocs` are not present in the command line command, the attribute documentation will not be included in the help output. The heavy computation is cached via functools.lru_cache, and a deep copy is returned so callers can mutate the dictionary without affecting the cached version. """ return copy.deepcopy(_compute_kwargs(cls)) @dataclass class EngineArgs: """Arguments for vLLM engine.""" model: str = ModelConfig.model enable_return_routed_experts: bool = ModelConfig.enable_return_routed_experts model_weights: str = ModelConfig.model_weights served_model_name: str | list[str] | None = ModelConfig.served_model_name tokenizer: str | None = ModelConfig.tokenizer hf_config_path: str | None = ModelConfig.hf_config_path runner: RunnerOption = ModelConfig.runner convert: ConvertOption = ModelConfig.convert skip_tokenizer_init: bool = ModelConfig.skip_tokenizer_init enable_prompt_embeds: bool = ModelConfig.enable_prompt_embeds tokenizer_mode: TokenizerMode | str = ModelConfig.tokenizer_mode trust_remote_code: bool = ModelConfig.trust_remote_code allowed_local_media_path: str = ModelConfig.allowed_local_media_path allowed_media_domains: list[str] | None = ModelConfig.allowed_media_domains download_dir: str | None = LoadConfig.download_dir safetensors_load_strategy: SafetensorsLoadStrategy | None = ( LoadConfig.safetensors_load_strategy ) safetensors_prefetch_num_threads: int = LoadConfig.safetensors_prefetch_num_threads safetensors_prefetch_block_size: int = LoadConfig.safetensors_prefetch_block_size load_format: str | LoadFormats = LoadConfig.load_format config_format: str = ModelConfig.config_format dtype: ModelDType = ModelConfig.dtype kv_cache_dtype: CacheDType = CacheConfig.cache_dtype seed: int = ModelConfig.seed max_model_len: int = ModelConfig.max_model_len cudagraph_capture_sizes: list[int] | None = ( CompilationConfig.cudagraph_capture_sizes ) max_cudagraph_capture_size: int | None = get_field( CompilationConfig, "max_cudagraph_capture_size" ) ir_op_priority: IrOpPriorityConfig = get_field(KernelConfig, "ir_op_priority") # Note: Specifying a custom executor backend by passing a class # is intended for expert use only. The API may change without # notice. distributed_executor_backend: ( str | DistributedExecutorBackend | type[Executor] | None ) = ParallelConfig.distributed_executor_backend # number of P/D disaggregation (or other disaggregation) workers pipeline_parallel_size: int = ParallelConfig.pipeline_parallel_size master_addr: str = ParallelConfig.master_addr master_port: int = ParallelConfig.master_port nnodes: int = ParallelConfig.nnodes node_rank: int = ParallelConfig.node_rank distributed_timeout_seconds: int | None = ParallelConfig.distributed_timeout_seconds cpu_distributed_timeout_seconds: int | None = ( ParallelConfig.cpu_distributed_timeout_seconds ) numa_bind: bool = ParallelConfig.numa_bind numa_bind_nodes: list[int] | None = ParallelConfig.numa_bind_nodes numa_bind_cpus: list[str] | None = ParallelConfig.numa_bind_cpus device_ids: list[int | str] | None = None tensor_parallel_size: int = ParallelConfig.tensor_parallel_size prefill_context_parallel_size: int = ParallelConfig.prefill_context_parallel_size decode_context_parallel_size: int = ParallelConfig.decode_context_parallel_size dcp_comm_backend: DCPCommBackend = ParallelConfig.dcp_comm_backend dcp_kv_cache_interleave_size: int = ParallelConfig.dcp_kv_cache_interleave_size cp_kv_cache_interleave_size: int = ParallelConfig.cp_kv_cache_interleave_size data_parallel_size: int = ParallelConfig.data_parallel_size data_parallel_rank: int | None = None data_parallel_start_rank: int | None = None data_parallel_size_local: int | None = None data_parallel_address: str | None = None data_parallel_rpc_port: int | None = None data_parallel_hybrid_lb: bool = False data_parallel_external_lb: bool = False data_parallel_multi_port_external_lb: bool = False data_parallel_backend: DataParallelBackend = ParallelConfig.data_parallel_backend enable_expert_parallel: bool = ParallelConfig.enable_expert_parallel enable_ep_weight_filter: bool = ParallelConfig.enable_ep_weight_filter moe_backend: MoEBackend = KernelConfig.moe_backend linear_backend: LinearBackend = KernelConfig.linear_backend all2all_backend: All2AllBackend = ParallelConfig.all2all_backend enable_elastic_ep: bool = ParallelConfig.enable_elastic_ep enable_dbo: bool = ParallelConfig.enable_dbo ubatch_size: int = ParallelConfig.ubatch_size dbo_decode_token_threshold: int = ParallelConfig.dbo_decode_token_threshold dbo_prefill_token_threshold: int = ParallelConfig.dbo_prefill_token_threshold disable_nccl_for_dp_synchronization: bool | None = ( ParallelConfig.disable_nccl_for_dp_synchronization ) eplb_config: EPLBConfig = get_field(ParallelConfig, "eplb_config") enable_eplb: bool = ParallelConfig.enable_eplb expert_placement_strategy: ExpertPlacementStrategy = ( ParallelConfig.expert_placement_strategy ) _api_process_count: int = ParallelConfig._api_process_count _api_process_rank: int = ParallelConfig._api_process_rank max_parallel_loading_workers: int | None = ( ParallelConfig.max_parallel_loading_workers ) block_size: int | None = None enable_prefix_caching: bool | None = None prefix_caching_hash_algo: PrefixCachingHashAlgo = ( CacheConfig.prefix_caching_hash_algo ) disable_sliding_window: bool = ModelConfig.disable_sliding_window disable_cascade_attn: bool = ModelConfig.disable_cascade_attn offload_backend: str = OffloadConfig.offload_backend cpu_offload_gb: float = UVAOffloadConfig.cpu_offload_gb cpu_offload_params: set[str] = get_field(UVAOffloadConfig, "cpu_offload_params") offload_group_size: int = PrefetchOffloadConfig.offload_group_size offload_num_in_group: int = PrefetchOffloadConfig.offload_num_in_group offload_prefetch_step: int = PrefetchOffloadConfig.offload_prefetch_step offload_params: set[str] = get_field(PrefetchOffloadConfig, "offload_params") gpu_memory_utilization: float = CacheConfig.gpu_memory_utilization kv_cache_memory_bytes: int | None = CacheConfig.kv_cache_memory_bytes max_num_batched_tokens: int | None = None max_num_scheduled_tokens: int | None = None max_num_partial_prefills: int = SchedulerConfig.max_num_partial_prefills max_long_partial_prefills: int = SchedulerConfig.max_long_partial_prefills long_prefill_token_threshold: int = SchedulerConfig.long_prefill_token_threshold max_num_seqs: int | None = None max_logprobs: int = ModelConfig.max_logprobs logprobs_mode: LogprobsMode = ModelConfig.logprobs_mode use_fp64_gumbel: bool = ModelConfig.use_fp64_gumbel disable_log_stats: bool = False aggregate_engine_logging: bool = False revision: str | None = ModelConfig.revision code_revision: str | None = ModelConfig.code_revision hf_token: bool | str | None = ModelConfig.hf_token hf_overrides: HfOverrides = get_field(ModelConfig, "hf_overrides") model_class_overrides: dict[str, str] = get_field( ModelConfig, "model_class_overrides" ) tokenizer_revision: str | None = ModelConfig.tokenizer_revision quantization: QuantizationMethods | str | None = ModelConfig.quantization quantization_config: "dict[str, Any] | QuantizationConfigArgs | None" = None """User-facing quantization configuration. Carries per-layer-kind QuantSpecs (linear, moe) and ignore patterns; see :class:`QuantizationConfigArgs`. Auto-populated from the matching online shorthand when `quantization` is one of the values in `ONLINE_QUANT_SHORTHAND_NAMES`.""" allow_deprecated_quantization: bool = ModelConfig.allow_deprecated_quantization enforce_eager: bool = ModelConfig.enforce_eager disable_custom_all_reduce: bool = ParallelConfig.disable_custom_all_reduce language_model_only: bool = MultiModalConfig.language_model_only limit_mm_per_prompt: dict[str, int | dict[str, int]] = get_field( MultiModalConfig, "limit_per_prompt" ) enable_mm_embeds: bool = MultiModalConfig.enable_mm_embeds interleave_mm_strings: bool = MultiModalConfig.interleave_mm_strings media_io_kwargs: dict[str, dict[str, Any]] = get_field( MultiModalConfig, "media_io_kwargs" ) mm_processor_kwargs: dict[str, Any] | None = MultiModalConfig.mm_processor_kwargs mm_processor_cache_gb: float = MultiModalConfig.mm_processor_cache_gb mm_processor_cache_type: MMCacheType | None = ( MultiModalConfig.mm_processor_cache_type ) mm_shm_cache_max_object_size_mb: int = ( MultiModalConfig.mm_shm_cache_max_object_size_mb ) mm_encoder_only: bool = MultiModalConfig.mm_encoder_only mm_encoder_tp_mode: MMEncoderTPMode = MultiModalConfig.mm_encoder_tp_mode mm_encoder_attn_backend: AttentionBackendEnum | str | None = ( MultiModalConfig.mm_encoder_attn_backend ) mm_encoder_attn_dtype: str | None = MultiModalConfig.mm_encoder_attn_dtype mm_encoder_fp8_scale_path: str | None = MultiModalConfig.mm_encoder_fp8_scale_path mm_encoder_fp8_scale_save_path: str | None = ( MultiModalConfig.mm_encoder_fp8_scale_save_path ) mm_encoder_fp8_scale_save_margin: float = ( MultiModalConfig.mm_encoder_fp8_scale_save_margin ) io_processor_plugin: str | None = None renderer_num_workers: int = 1 skip_mm_profiling: bool = MultiModalConfig.skip_mm_profiling video_pruning_rate: float | None = MultiModalConfig.video_pruning_rate mm_tensor_ipc: MMTensorIPC = MultiModalConfig.mm_tensor_ipc mm_ipc_gpu_memory_gb: float = MultiModalConfig.mm_ipc_gpu_memory_gb # LoRA fields enable_lora: bool = False max_loras: int = LoRAConfig.max_loras max_lora_rank: MaxLoRARanks = LoRAConfig.max_lora_rank default_mm_loras: dict[str, str] | None = LoRAConfig.default_mm_loras fully_sharded_loras: bool = LoRAConfig.fully_sharded_loras max_cpu_loras: int | None = LoRAConfig.max_cpu_loras lora_dtype: str | torch.dtype | None = LoRAConfig.lora_dtype lora_target_modules: list[str] | None = LoRAConfig.target_modules enable_tower_connector_lora: bool = LoRAConfig.enable_tower_connector_lora specialize_active_lora: bool = LoRAConfig.specialize_active_lora enable_mixed_moe_lora_format: bool = LoRAConfig.enable_mixed_moe_lora_format ray_workers_use_nsight: bool = ParallelConfig.ray_workers_use_nsight num_gpu_blocks_override: int | None = CacheConfig.num_gpu_blocks_override model_loader_extra_config: dict = get_field(LoadConfig, "model_loader_extra_config") ignore_patterns: str | list[str] = get_field(LoadConfig, "ignore_patterns") enable_chunked_prefill: bool | None = None disable_chunked_mm_input: bool = SchedulerConfig.disable_chunked_mm_input scheduler_reserve_full_isl: bool = SchedulerConfig.scheduler_reserve_full_isl prefill_schedule_interval: int = SchedulerConfig.prefill_schedule_interval watermark: float = SchedulerConfig.watermark disable_hybrid_kv_cache_manager: bool | None = ( SchedulerConfig.disable_hybrid_kv_cache_manager ) structured_outputs_config: StructuredOutputsConfig = get_field( VllmConfig, "structured_outputs_config" ) reasoning_parser: str = StructuredOutputsConfig.reasoning_parser reasoning_parser_plugin: str | None = None speculative_config: dict[str, Any] | None = None spec_method: str | None = None spec_model: str | None = None spec_tokens: int | None = None diffusion_config: dict[str, Any] | None = None show_hidden_metrics_for_version: str | None = ( ObservabilityConfig.show_hidden_metrics_for_version ) otlp_traces_endpoint: str | None = ObservabilityConfig.otlp_traces_endpoint collect_detailed_traces: list[DetailedTraceModules] | None = ( ObservabilityConfig.collect_detailed_traces ) kv_cache_metrics: bool = ObservabilityConfig.kv_cache_metrics kv_cache_metrics_sample: float = get_field( ObservabilityConfig, "kv_cache_metrics_sample" ) cudagraph_metrics: bool = ObservabilityConfig.cudagraph_metrics enable_layerwise_nvtx_tracing: bool = ( ObservabilityConfig.enable_layerwise_nvtx_tracing ) enable_mfu_metrics: bool = ObservabilityConfig.enable_mfu_metrics enable_logging_iteration_details: bool = ( ObservabilityConfig.enable_logging_iteration_details ) jit_monitor_mode: Literal["warn", "error"] = ObservabilityConfig.jit_monitor_mode jit_monitor_verbose: bool = ObservabilityConfig.jit_monitor_verbose enable_mm_processor_stats: bool = ObservabilityConfig.enable_mm_processor_stats scheduling_policy: SchedulerPolicy = SchedulerConfig.policy scheduler_cls: str | type[object] | None = SchedulerConfig.scheduler_cls pooler_config: PoolerConfig | None = ModelConfig.pooler_config compilation_config: CompilationConfig = get_field(VllmConfig, "compilation_config") attention_config: AttentionConfig = get_field(VllmConfig, "attention_config") mamba_config: MambaConfig = get_field(VllmConfig, "mamba_config") kernel_config: KernelConfig = get_field(VllmConfig, "kernel_config") enable_flashinfer_autotune: bool = get_field( KernelConfig, "enable_flashinfer_autotune" ) worker_cls: str = ParallelConfig.worker_cls worker_extension_cls: str = ParallelConfig.worker_extension_cls profiler_config: ProfilerConfig = get_field(VllmConfig, "profiler_config") kv_transfer_config: KVTransferConfig | None = None kv_events_config: KVEventsConfig | None = None ec_transfer_config: ECTransferConfig | None = None reasoning_config: ReasoningConfig = get_field(VllmConfig, "reasoning_config") generation_config: str = ModelConfig.generation_config enable_sleep_mode: bool = ModelConfig.enable_sleep_mode enable_cumem_allocator: bool = ModelConfig.enable_cumem_allocator override_generation_config: dict[str, Any] = get_field( ModelConfig, "override_generation_config" ) model_impl: str = ModelConfig.model_impl override_attention_dtype: str | None = ModelConfig.override_attention_dtype attention_backend: AttentionBackendEnum | None = AttentionConfig.backend calculate_kv_scales: bool = CacheConfig.calculate_kv_scales kv_cache_dtype_skip_layers: list[str] = get_field( CacheConfig, "kv_cache_dtype_skip_layers" ) mamba_cache_dtype: MambaDType = CacheConfig.mamba_cache_dtype mamba_ssm_cache_dtype: MambaDType = CacheConfig.mamba_ssm_cache_dtype mamba_block_size: int | None = get_field(CacheConfig, "mamba_block_size") prefix_match_unit: int | None = get_field(CacheConfig, "prefix_match_unit") mamba_cache_mode: MambaCacheMode = CacheConfig.mamba_cache_mode mamba_backend: MambaBackendEnum = MambaBackendEnum.TRITON enable_mamba_cache_stochastic_rounding: bool = ( MambaConfig.enable_stochastic_rounding ) mamba_cache_philox_rounds: int = MambaConfig.stochastic_rounding_philox_rounds additional_config: dict[str, Any] = get_field(VllmConfig, "additional_config") use_tqdm_on_load: bool = LoadConfig.use_tqdm_on_load pt_load_map_location: str | dict[str, str] = LoadConfig.pt_load_map_location logits_processors: list[str | type[LogitsProcessor]] | None = ( ModelConfig.logits_processors ) """Custom logitproc types""" async_scheduling: bool | None = SchedulerConfig.async_scheduling stream_interval: int = SchedulerConfig.stream_interval kv_sharing_fast_prefill: bool = CacheConfig.kv_sharing_fast_prefill optimization_level: OptimizationLevel = VllmConfig.optimization_level performance_mode: PerformanceMode = VllmConfig.performance_mode kv_offloading_size: float | None = CacheConfig.kv_offloading_size kv_offloading_backend: KVOffloadingBackend = CacheConfig.kv_offloading_backend tokens_only: bool = False shutdown_timeout: int = 0 weight_transfer_config: WeightTransferConfig | None = get_field( VllmConfig, "weight_transfer_config", ) fail_on_environ_validation: bool = False gdn_prefill_backend: Literal["flashinfer", "triton", "cutedsl"] | None = None def __post_init__(self): # support `EngineArgs(compilation_config={...})` # without having to manually construct a # CompilationConfig object if isinstance(self.compilation_config, dict): self.compilation_config = CompilationConfig(**self.compilation_config) if isinstance(self.attention_config, dict): self.attention_config = AttentionConfig(**self.attention_config) if isinstance(self.mamba_config, dict): self.mamba_config = MambaConfig(**self.mamba_config) if isinstance(self.kernel_config, dict): self.kernel_config = KernelConfig(**self.kernel_config) if isinstance(self.eplb_config, dict): self.eplb_config = EPLBConfig(**self.eplb_config) if isinstance(self.weight_transfer_config, dict): self.weight_transfer_config = WeightTransferConfig( **self.weight_transfer_config ) if isinstance(self.ir_op_priority, dict): self.ir_op_priority = IrOpPriorityConfig(**self.ir_op_priority) from vllm.config.quantization import resolve_quantization_config self.quantization_config = resolve_quantization_config( self.quantization, self.quantization_config ) # Setup plugins from vllm.plugins import load_general_plugins load_general_plugins() # when use hf offline,replace model and tokenizer id to local model path if huggingface_hub.constants.HF_HUB_OFFLINE: # Skip cloud storage URIs (s3://, gs://, az://) — they are not # HF repo IDs and will be resolved later by # ModelConfig.maybe_pull_model_tokenizer_for_runai(). if not is_cloud_storage(self.model): model_id = self.model self.model = get_model_path(self.model, self.revision) if model_id is not self.model: logger.info( "HF_HUB_OFFLINE is True, replace model_id " "[%s] to model_path [%s]", model_id, self.model, ) if self.tokenizer is not None and not is_cloud_storage(self.tokenizer): tokenizer_id = self.tokenizer self.tokenizer = get_model_path(self.tokenizer, self.tokenizer_revision) if tokenizer_id is not self.tokenizer: logger.info( "HF_HUB_OFFLINE is True, replace tokenizer_id [%s] " "to tokenizer_path [%s]", tokenizer_id, self.tokenizer, ) @staticmethod def add_cli_args(parser: FlexibleArgumentParser) -> FlexibleArgumentParser: """Shared CLI arguments for vLLM engine.""" # Model arguments model_kwargs = get_kwargs(ModelConfig) model_group = parser.add_argument_group( title="ModelConfig", description=ModelConfig.__doc__, ) if not ("serve" in sys.argv[1:] and "--help" in sys.argv[1:]): model_group.add_argument("--model", **model_kwargs["model"]) model_group.add_argument("--runner", **model_kwargs["runner"]) model_group.add_argument("--convert", **model_kwargs["convert"]) model_group.add_argument("--tokenizer", **model_kwargs["tokenizer"]) model_group.add_argument("--tokenizer-mode", **model_kwargs["tokenizer_mode"]) model_group.add_argument( "--trust-remote-code", **model_kwargs["trust_remote_code"] ) model_group.add_argument("--dtype", **model_kwargs["dtype"]) model_group.add_argument("--seed", **model_kwargs["seed"]) model_group.add_argument("--hf-config-path", **model_kwargs["hf_config_path"]) model_group.add_argument( "--allowed-local-media-path", **model_kwargs["allowed_local_media_path"] ) model_group.add_argument( "--allowed-media-domains", **model_kwargs["allowed_media_domains"] ) model_group.add_argument("--revision", **model_kwargs["revision"]) model_group.add_argument("--code-revision", **model_kwargs["code_revision"]) model_group.add_argument( "--tokenizer-revision", **model_kwargs["tokenizer_revision"] ) model_group.add_argument("--max-model-len", **model_kwargs["max_model_len"]) model_group.add_argument("--quantization", "-q", **model_kwargs["quantization"]) model_group.add_argument( "--quantization-config", **model_kwargs["quantization_config"] ) model_group.add_argument( "--allow-deprecated-quantization", **model_kwargs["allow_deprecated_quantization"], ) model_group.add_argument("--enforce-eager", **model_kwargs["enforce_eager"]) model_group.add_argument( "--enable-return-routed-experts", **model_kwargs["enable_return_routed_experts"], ) model_group.add_argument("--max-logprobs", **model_kwargs["max_logprobs"]) model_group.add_argument("--logprobs-mode", **model_kwargs["logprobs_mode"]) model_group.add_argument("--use-fp64-gumbel", **model_kwargs["use_fp64_gumbel"]) model_group.add_argument( "--disable-sliding-window", **model_kwargs["disable_sliding_window"] ) model_group.add_argument( "--disable-cascade-attn", **model_kwargs["disable_cascade_attn"] ) model_group.add_argument( "--skip-tokenizer-init", **model_kwargs["skip_tokenizer_init"] ) model_group.add_argument( "--enable-prompt-embeds", **model_kwargs["enable_prompt_embeds"] ) model_group.add_argument( "--served-model-name", **model_kwargs["served_model_name"] ) model_group.add_argument("--config-format", **model_kwargs["config_format"]) model_group.add_argument("--hf-token", **model_kwargs["hf_token"]) model_group.add_argument("--hf-overrides", **model_kwargs["hf_overrides"]) model_group.add_argument( "--model-class-overrides", **model_kwargs["model_class_overrides"] ) model_group.add_argument("--pooler-config", **model_kwargs["pooler_config"]) model_group.add_argument( "--generation-config", **model_kwargs["generation_config"] ) model_group.add_argument( "--override-generation-config", **model_kwargs["override_generation_config"] ) model_group.add_argument( "--enable-sleep-mode", **model_kwargs["enable_sleep_mode"] ) model_group.add_argument( "--enable-cumem-allocator", **model_kwargs["enable_cumem_allocator"] ) model_group.add_argument("--model-impl", **model_kwargs["model_impl"]) model_group.add_argument( "--override-attention-dtype", **model_kwargs["override_attention_dtype"] ) model_group.add_argument( "--logits-processors", **model_kwargs["logits_processors"] ) model_group.add_argument( "--io-processor-plugin", **model_kwargs["io_processor_plugin"] ) model_group.add_argument( "--renderer-num-workers", **model_kwargs["renderer_num_workers"], ) # Model loading arguments load_kwargs = get_kwargs(LoadConfig) load_group = parser.add_argument_group( title="LoadConfig", description=LoadConfig.__doc__, ) load_group.add_argument("--load-format", **load_kwargs["load_format"]) load_group.add_argument("--download-dir", **load_kwargs["download_dir"]) load_group.add_argument( "--safetensors-load-strategy", **load_kwargs["safetensors_load_strategy"] ) load_group.add_argument( "--safetensors-prefetch-num-threads", **load_kwargs["safetensors_prefetch_num_threads"], ) load_group.add_argument( "--safetensors-prefetch-block-size", **load_kwargs["safetensors_prefetch_block_size"], ) load_group.add_argument( "--model-loader-extra-config", **load_kwargs["model_loader_extra_config"] ) load_group.add_argument("--ignore-patterns", **load_kwargs["ignore_patterns"]) load_group.add_argument("--use-tqdm-on-load", **load_kwargs["use_tqdm_on_load"]) load_group.add_argument( "--pt-load-map-location", **load_kwargs["pt_load_map_location"] ) # Attention arguments attention_kwargs = get_kwargs(AttentionConfig) attention_group = parser.add_argument_group( title="AttentionConfig", description=AttentionConfig.__doc__, ) attention_group.add_argument( "--attention-backend", **attention_kwargs["backend"] ) # Mamba arguments mamba_kwargs = get_kwargs(MambaConfig) mamba_group = parser.add_argument_group( title="MambaConfig", description=MambaConfig.__doc__, ) mamba_group.add_argument("--mamba-backend", **mamba_kwargs["backend"]) mamba_group.add_argument( "--enable-mamba-cache-stochastic-rounding", **mamba_kwargs["enable_stochastic_rounding"], ) mamba_group.add_argument( "--mamba-cache-philox-rounds", **mamba_kwargs["stochastic_rounding_philox_rounds"], ) # Structured outputs arguments structured_outputs_kwargs = get_kwargs(StructuredOutputsConfig) structured_outputs_group = parser.add_argument_group( title="StructuredOutputsConfig", description=StructuredOutputsConfig.__doc__, ) structured_outputs_group.add_argument( "--reasoning-parser", # Choices need to be validated after parsing to include plugins **structured_outputs_kwargs["reasoning_parser"], ) structured_outputs_group.add_argument( "--reasoning-parser-plugin", **structured_outputs_kwargs["reasoning_parser_plugin"], ) # Parallel arguments parallel_kwargs = get_kwargs(ParallelConfig) parallel_group = parser.add_argument_group( title="ParallelConfig", description=ParallelConfig.__doc__, ) parallel_group.add_argument( "--distributed-executor-backend", **parallel_kwargs["distributed_executor_backend"], ) parallel_group.add_argument( "--pipeline-parallel-size", "-pp", **parallel_kwargs["pipeline_parallel_size"], ) parallel_group.add_argument("--master-addr", **parallel_kwargs["master_addr"]) parallel_group.add_argument("--master-port", **parallel_kwargs["master_port"]) parallel_group.add_argument("--nnodes", "-n", **parallel_kwargs["nnodes"]) parallel_group.add_argument("--node-rank", "-r", **parallel_kwargs["node_rank"]) parallel_group.add_argument( "--distributed-timeout-seconds", **parallel_kwargs["distributed_timeout_seconds"], ) parallel_group.add_argument( "--cpu-distributed-timeout-seconds", **parallel_kwargs["cpu_distributed_timeout_seconds"], ) parallel_group.add_argument("--numa-bind", **parallel_kwargs["numa_bind"]) parallel_group.add_argument( "--numa-bind-nodes", **parallel_kwargs["numa_bind_nodes"] ) parallel_group.add_argument( "--numa-bind-cpus", **parallel_kwargs["numa_bind_cpus"] ) parallel_group.add_argument( "--device-ids", type=lambda s: [ int(device_id) if device_id.isdigit() else device_id for device_id in (part.strip() for part in s.split(",")) ], default=None, help="Comma-separated physical GPU device IDs or UUIDs to use " '(e.g. --device-ids "2,3,5,7"). Avoids setting ' "CUDA_VISIBLE_DEVICES, preserving full GPU topology " "visibility for GPU-NIC affinity and DeepGEMM. " "Note: has no effect with Ray executors; use Ray " "placement groups for GPU selection instead.", ) parallel_group.add_argument( "--tensor-parallel-size", "-tp", **parallel_kwargs["tensor_parallel_size"] ) parallel_group.add_argument( "--decode-context-parallel-size", "-dcp", **parallel_kwargs["decode_context_parallel_size"], ) parallel_group.add_argument( "--dcp-comm-backend", **parallel_kwargs["dcp_comm_backend"], ) parallel_group.add_argument( "--dcp-kv-cache-interleave-size", **parallel_kwargs["dcp_kv_cache_interleave_size"], ) parallel_group.add_argument( "--cp-kv-cache-interleave-size", **parallel_kwargs["cp_kv_cache_interleave_size"], ) parallel_group.add_argument( "--prefill-context-parallel-size", "-pcp", **parallel_kwargs["prefill_context_parallel_size"], ) parallel_group.add_argument( "--data-parallel-size", "-dp", **parallel_kwargs["data_parallel_size"] ) parallel_group.add_argument( "--data-parallel-rank", "-dpn", type=int, help="Data parallel rank of this instance. " "When set, enables external load balancer mode for MoE " "data-parallel deployments. Unsupported for non-MoE models; " "launch independent vLLM instances instead.", ) parallel_group.add_argument( "--data-parallel-start-rank", "-dpr", type=int, help="Starting data parallel rank for secondary nodes.", ) parallel_group.add_argument( "--data-parallel-size-local", "-dpl", type=int, help="Number of data parallel replicas to run on this node.", ) parallel_group.add_argument( "--data-parallel-address", "-dpa", type=str, help="Address of data parallel cluster head-node.", ) parallel_group.add_argument( "--data-parallel-rpc-port", "-dpp", type=int, help="Port for data parallel RPC communication.", ) parallel_group.add_argument( "--data-parallel-backend", "-dpb", type=str, default="mp", help='Backend for data parallel, either "mp" or "ray".', ) parallel_group.add_argument( "--data-parallel-hybrid-lb", "-dph", **parallel_kwargs["data_parallel_hybrid_lb"], ) parallel_group.add_argument( "--data-parallel-external-lb", "-dpe", **parallel_kwargs["data_parallel_external_lb"], ) parallel_group.add_argument( "--data-parallel-multi-port-external-lb", "-dpm", action="store_true", default=False, help="Run a node-local supervisor that launches one external-LB API " "server per local data parallel rank and exposes aggregated health on " "a supervisor port.", ) parallel_group.add_argument( "--enable-expert-parallel", "-ep", **parallel_kwargs["enable_expert_parallel"], ) parallel_group.add_argument( "--enable-ep-weight-filter", **parallel_kwargs["enable_ep_weight_filter"], ) parallel_group.add_argument( "--all2all-backend", **parallel_kwargs["all2all_backend"] ) parallel_group.add_argument("--enable-dbo", **parallel_kwargs["enable_dbo"]) parallel_group.add_argument( "--ubatch-size", **parallel_kwargs["ubatch_size"], ) parallel_group.add_argument( "--enable-elastic-ep", **parallel_kwargs["enable_elastic_ep"] ) parallel_group.add_argument( "--dbo-decode-token-threshold", **parallel_kwargs["dbo_decode_token_threshold"], ) parallel_group.add_argument( "--dbo-prefill-token-threshold", **parallel_kwargs["dbo_prefill_token_threshold"], ) parallel_group.add_argument( "--disable-nccl-for-dp-synchronization", **parallel_kwargs["disable_nccl_for_dp_synchronization"], ) parallel_group.add_argument("--enable-eplb", **parallel_kwargs["enable_eplb"]) parallel_group.add_argument("--eplb-config", **parallel_kwargs["eplb_config"]) parallel_group.add_argument( "--expert-placement-strategy", **parallel_kwargs["expert_placement_strategy"], ) parallel_group.add_argument( "--max-parallel-loading-workers", **parallel_kwargs["max_parallel_loading_workers"], ) parallel_group.add_argument( "--ray-workers-use-nsight", **parallel_kwargs["ray_workers_use_nsight"] ) parallel_group.add_argument( "--disable-custom-all-reduce", **parallel_kwargs["disable_custom_all_reduce"], ) parallel_group.add_argument("--worker-cls", **parallel_kwargs["worker_cls"]) parallel_group.add_argument( "--worker-extension-cls", **parallel_kwargs["worker_extension_cls"] ) # KV cache arguments cache_kwargs = get_kwargs(CacheConfig) cache_group = parser.add_argument_group( title="CacheConfig", description=CacheConfig.__doc__, ) cache_group.add_argument("--block-size", **cache_kwargs["block_size"]) cache_group.add_argument( "--gpu-memory-utilization", **cache_kwargs["gpu_memory_utilization"] ) cache_group.add_argument( "--kv-cache-memory-bytes", **cache_kwargs["kv_cache_memory_bytes"] ) cache_group.add_argument("--kv-cache-dtype", **cache_kwargs["cache_dtype"]) cache_group.add_argument( "--num-gpu-blocks-override", **cache_kwargs["num_gpu_blocks_override"] ) cache_group.add_argument( "--enable-prefix-caching", **{ **cache_kwargs["enable_prefix_caching"], "default": None, }, ) cache_group.add_argument( "--prefix-caching-hash-algo", **cache_kwargs["prefix_caching_hash_algo"] ) cache_group.add_argument( "--calculate-kv-scales", **cache_kwargs["calculate_kv_scales"] ) cache_group.add_argument( "--kv-cache-dtype-skip-layers", **cache_kwargs["kv_cache_dtype_skip_layers"] ) cache_group.add_argument( "--kv-sharing-fast-prefill", **cache_kwargs["kv_sharing_fast_prefill"] ) cache_group.add_argument( "--mamba-cache-dtype", **cache_kwargs["mamba_cache_dtype"] ) cache_group.add_argument( "--mamba-ssm-cache-dtype", **cache_kwargs["mamba_ssm_cache_dtype"] ) cache_group.add_argument( "--mamba-block-size", **cache_kwargs["mamba_block_size"] ) cache_group.add_argument( "--prefix-match-unit", **cache_kwargs["prefix_match_unit"] ) cache_group.add_argument( "--mamba-cache-mode", **cache_kwargs["mamba_cache_mode"] ) cache_group.add_argument( "--kv-offloading-size", **cache_kwargs["kv_offloading_size"] ) cache_group.add_argument( "--kv-offloading-backend", **cache_kwargs["kv_offloading_backend"] ) # Model weight offload related configs offload_kwargs = get_kwargs(OffloadConfig) uva_kwargs = get_kwargs(UVAOffloadConfig) prefetch_kwargs = get_kwargs(PrefetchOffloadConfig) offload_group = parser.add_argument_group( title="OffloadConfig", description=OffloadConfig.__doc__, ) offload_group.add_argument( "--offload-backend", **offload_kwargs["offload_backend"] ) offload_group.add_argument("--cpu-offload-gb", **uva_kwargs["cpu_offload_gb"]) offload_group.add_argument( "--cpu-offload-params", **uva_kwargs["cpu_offload_params"] ) offload_group.add_argument( "--offload-group-size", **prefetch_kwargs["offload_group_size"], ) offload_group.add_argument( "--offload-num-in-group", **prefetch_kwargs["offload_num_in_group"], ) offload_group.add_argument( "--offload-prefetch-step", **prefetch_kwargs["offload_prefetch_step"], ) offload_group.add_argument( "--offload-params", **prefetch_kwargs["offload_params"] ) # Multimodal related configs multimodal_kwargs = get_kwargs(MultiModalConfig) multimodal_group = parser.add_argument_group( title="MultiModalConfig", description=MultiModalConfig.__doc__, ) multimodal_group.add_argument( "--language-model-only", **multimodal_kwargs["language_model_only"] ) multimodal_group.add_argument( "--limit-mm-per-prompt", **multimodal_kwargs["limit_per_prompt"] ) multimodal_group.add_argument( "--enable-mm-embeds", **multimodal_kwargs["enable_mm_embeds"] ) multimodal_group.add_argument( "--media-io-kwargs", **multimodal_kwargs["media_io_kwargs"] ) multimodal_group.add_argument( "--mm-processor-kwargs", **multimodal_kwargs["mm_processor_kwargs"] ) multimodal_group.add_argument( "--mm-processor-cache-gb", **multimodal_kwargs["mm_processor_cache_gb"] ) multimodal_group.add_argument( "--mm-processor-cache-type", **multimodal_kwargs["mm_processor_cache_type"] ) multimodal_group.add_argument( "--mm-shm-cache-max-object-size-mb", **multimodal_kwargs["mm_shm_cache_max_object_size_mb"], ) multimodal_group.add_argument( "--mm-encoder-only", **multimodal_kwargs["mm_encoder_only"] ) multimodal_group.add_argument( "--mm-encoder-tp-mode", **multimodal_kwargs["mm_encoder_tp_mode"] ) multimodal_group.add_argument( "--mm-encoder-attn-backend", **multimodal_kwargs["mm_encoder_attn_backend"], ) multimodal_group.add_argument( "--mm-encoder-attn-dtype", **multimodal_kwargs["mm_encoder_attn_dtype"], ) multimodal_group.add_argument( "--mm-encoder-fp8-scale-path", **multimodal_kwargs["mm_encoder_fp8_scale_path"], ) multimodal_group.add_argument( "--mm-encoder-fp8-scale-save-path", **multimodal_kwargs["mm_encoder_fp8_scale_save_path"], ) multimodal_group.add_argument( "--mm-encoder-fp8-scale-save-margin", **multimodal_kwargs["mm_encoder_fp8_scale_save_margin"], ) multimodal_group.add_argument( "--interleave-mm-strings", **multimodal_kwargs["interleave_mm_strings"] ) multimodal_group.add_argument( "--skip-mm-profiling", **multimodal_kwargs["skip_mm_profiling"] ) multimodal_group.add_argument( "--video-pruning-rate", **multimodal_kwargs["video_pruning_rate"] ) multimodal_group.add_argument( "--mm-tensor-ipc", **multimodal_kwargs["mm_tensor_ipc"] ) multimodal_group.add_argument( "--mm-ipc-gpu-memory-gb", **multimodal_kwargs["mm_ipc_gpu_memory_gb"], ) # LoRA related configs lora_kwargs = get_kwargs(LoRAConfig) lora_group = parser.add_argument_group( title="LoRAConfig", description=LoRAConfig.__doc__, ) lora_group.add_argument( "--enable-lora", action=argparse.BooleanOptionalAction, help="If True, enable handling of LoRA adapters.", ) lora_group.add_argument("--max-loras", **lora_kwargs["max_loras"]) lora_group.add_argument("--max-lora-rank", **lora_kwargs["max_lora_rank"]) lora_group.add_argument( "--lora-dtype", **lora_kwargs["lora_dtype"], ) lora_group.add_argument( "--enable-tower-connector-lora", **lora_kwargs["enable_tower_connector_lora"], ) lora_group.add_argument("--max-cpu-loras", **lora_kwargs["max_cpu_loras"]) lora_group.add_argument( "--fully-sharded-loras", **lora_kwargs["fully_sharded_loras"] ) lora_group.add_argument( "--lora-target-modules", **lora_kwargs["target_modules"] ) lora_group.add_argument("--default-mm-loras", **lora_kwargs["default_mm_loras"]) lora_group.add_argument( "--specialize-active-lora", **lora_kwargs["specialize_active_lora"] ) lora_group.add_argument( "--enable-mixed-moe-lora-format", **lora_kwargs["enable_mixed_moe_lora_format"], ) # Observability arguments observability_kwargs = get_kwargs(ObservabilityConfig) observability_group = parser.add_argument_group( title="ObservabilityConfig", description=ObservabilityConfig.__doc__, ) observability_group.add_argument( "--show-hidden-metrics-for-version", **observability_kwargs["show_hidden_metrics_for_version"], ) observability_group.add_argument( "--otlp-traces-endpoint", **observability_kwargs["otlp_traces_endpoint"] ) # TODO: generalise this special case choices = observability_kwargs["collect_detailed_traces"]["choices"] metavar = f"{{{','.join(choices)}}}" observability_kwargs["collect_detailed_traces"]["metavar"] = metavar observability_kwargs["collect_detailed_traces"]["choices"] += [ ",".join(p) for p in permutations(get_args(DetailedTraceModules), r=2) ] observability_group.add_argument( "--collect-detailed-traces", **observability_kwargs["collect_detailed_traces"], ) observability_group.add_argument( "--kv-cache-metrics", **observability_kwargs["kv_cache_metrics"] ) observability_group.add_argument( "--kv-cache-metrics-sample", **observability_kwargs["kv_cache_metrics_sample"], ) observability_group.add_argument( "--cudagraph-metrics", **observability_kwargs["cudagraph_metrics"], ) observability_group.add_argument( "--enable-layerwise-nvtx-tracing", **observability_kwargs["enable_layerwise_nvtx_tracing"], ) observability_group.add_argument( "--enable-mfu-metrics", **observability_kwargs["enable_mfu_metrics"], ) observability_group.add_argument( "--enable-logging-iteration-details", **observability_kwargs["enable_logging_iteration_details"], ) observability_group.add_argument( "--jit-monitor-mode", **observability_kwargs["jit_monitor_mode"], ) observability_group.add_argument( "--jit-monitor-verbose", **observability_kwargs["jit_monitor_verbose"], ) # Scheduler arguments scheduler_kwargs = get_kwargs(SchedulerConfig) scheduler_group = parser.add_argument_group( title="SchedulerConfig", description=SchedulerConfig.__doc__, ) scheduler_group.add_argument( "--max-num-batched-tokens", **{ **scheduler_kwargs["max_num_batched_tokens"], "default": None, }, ) scheduler_group.add_argument( "--max-num-scheduled-tokens", **{ **scheduler_kwargs["max_num_scheduled_tokens"], "default": None, }, ) scheduler_group.add_argument( "--max-num-seqs", **{ **scheduler_kwargs["max_num_seqs"], "default": None, }, ) scheduler_group.add_argument( "--max-num-partial-prefills", **scheduler_kwargs["max_num_partial_prefills"] ) scheduler_group.add_argument( "--max-long-partial-prefills", **scheduler_kwargs["max_long_partial_prefills"], ) scheduler_group.add_argument( "--long-prefill-token-threshold", **scheduler_kwargs["long_prefill_token_threshold"], ) # multi-step scheduling has been removed; corresponding arguments # are no longer supported. scheduler_group.add_argument( "--scheduling-policy", **scheduler_kwargs["policy"] ) scheduler_group.add_argument( "--enable-chunked-prefill", **{ **scheduler_kwargs["enable_chunked_prefill"], "default": None, }, ) scheduler_group.add_argument( "--disable-chunked-mm-input", **scheduler_kwargs["disable_chunked_mm_input"] ) scheduler_group.add_argument( "--scheduler-cls", **scheduler_kwargs["scheduler_cls"] ) scheduler_group.add_argument( "--scheduler-reserve-full-isl", **scheduler_kwargs["scheduler_reserve_full_isl"], ) scheduler_group.add_argument("--watermark", **scheduler_kwargs["watermark"]) scheduler_group.add_argument( "--prefill-schedule-interval", **scheduler_kwargs["prefill_schedule_interval"], ) scheduler_group.add_argument( "--disable-hybrid-kv-cache-manager", **scheduler_kwargs["disable_hybrid_kv_cache_manager"], ) scheduler_group.add_argument( "--async-scheduling", **scheduler_kwargs["async_scheduling"] ) scheduler_group.add_argument( "--stream-interval", **scheduler_kwargs["stream_interval"] ) # Compilation arguments compilation_kwargs = get_kwargs(CompilationConfig) compilation_group = parser.add_argument_group( title="CompilationConfig", description=CompilationConfig.__doc__, ) compilation_group.add_argument( "--cudagraph-capture-sizes", **compilation_kwargs["cudagraph_capture_sizes"] ) compilation_group.add_argument( "--max-cudagraph-capture-size", **compilation_kwargs["max_cudagraph_capture_size"], ) # Kernel arguments kernel_kwargs = get_kwargs(KernelConfig) kernel_group = parser.add_argument_group( title="KernelConfig", description=KernelConfig.__doc__, ) kernel_group.add_argument("--ir-op-priority", **kernel_kwargs["ir_op_priority"]) kernel_group.add_argument( "--enable-flashinfer-autotune", **kernel_kwargs["enable_flashinfer_autotune"], ) moe_backend_kwargs = kernel_kwargs["moe_backend"] moe_backend_kwargs["type"] = lambda s: s.lower().replace("-", "_") kernel_group.add_argument("--moe-backend", **moe_backend_kwargs) linear_backend_kwargs = kernel_kwargs["linear_backend"] linear_backend_kwargs["type"] = lambda s: s.lower().replace("-", "_") kernel_group.add_argument("--linear-backend", **linear_backend_kwargs) # vLLM arguments vllm_kwargs = get_kwargs(VllmConfig) vllm_group = parser.add_argument_group( title="VllmConfig", description=VllmConfig.__doc__, ) # We construct SpeculativeConfig using fields from other configs in # create_engine_config. So we set the type to a JSON string here to # delay the Pydantic validation that comes with SpeculativeConfig. vllm_kwargs["speculative_config"]["type"] = optional_type(json.loads) vllm_group.add_argument( "--speculative-config", "-sc", **vllm_kwargs["speculative_config"] ) speculative_kwargs = get_kwargs(SpeculativeConfig) vllm_group.add_argument("--spec-method", **speculative_kwargs["method"]) vllm_group.add_argument("--spec-model", **speculative_kwargs["model"]) vllm_group.add_argument( "--spec-tokens", **speculative_kwargs["num_speculative_tokens"] ) vllm_kwargs["diffusion_config"]["type"] = optional_type(json.loads) vllm_group.add_argument( "--diffusion-config", "-dc", **vllm_kwargs["diffusion_config"] ) vllm_group.add_argument( "--kv-transfer-config", **vllm_kwargs["kv_transfer_config"] ) vllm_group.add_argument("--kv-events-config", **vllm_kwargs["kv_events_config"]) vllm_group.add_argument( "--ec-transfer-config", **vllm_kwargs["ec_transfer_config"] ) vllm_group.add_argument( "--compilation-config", "-cc", **vllm_kwargs["compilation_config"] ) vllm_group.add_argument( "--attention-config", "-ac", **vllm_kwargs["attention_config"] ) vllm_group.add_argument("--reasoning-config", **vllm_kwargs["reasoning_config"]) vllm_group.add_argument("--kernel-config", **vllm_kwargs["kernel_config"]) vllm_group.add_argument( "--additional-config", **vllm_kwargs["additional_config"] ) vllm_group.add_argument( "--structured-outputs-config", **vllm_kwargs["structured_outputs_config"] ) vllm_group.add_argument("--profiler-config", **vllm_kwargs["profiler_config"]) vllm_group.add_argument( "--optimization-level", **vllm_kwargs["optimization_level"] ) vllm_group.add_argument("--performance-mode", **vllm_kwargs["performance_mode"]) vllm_group.add_argument( "--weight-transfer-config", **vllm_kwargs["weight_transfer_config"] ) # Other arguments parser.add_argument( "--disable-log-stats", action="store_true", help="Disable logging statistics.", ) parser.add_argument( "--aggregate-engine-logging", action="store_true", help="Log aggregate rather than per-engine statistics " "when using data parallelism.", ) parser.add_argument( "--fail-on-environ-validation", help="If set, the engine will raise an error if " "environment validation fails.", default=False, action=argparse.BooleanOptionalAction, ) parser.add_argument( "--shutdown-timeout", type=int, default=0, help="Shutdown timeout in seconds. 0 = abort, >0 = wait.", ) parser.add_argument( "--gdn-prefill-backend", dest="gdn_prefill_backend", choices=["flashinfer", "triton", "cutedsl"], default=None, help="Select GDN prefill backend.", ) return parser @classmethod def from_cli_args(cls, args: argparse.Namespace): # Get the list of attributes of this dataclass. attrs = [attr.name for attr in dataclasses.fields(cls)] # Set the attributes from the parsed arguments. engine_args = cls( **{attr: getattr(args, attr) for attr in attrs if hasattr(args, attr)} ) return engine_args def create_model_config(self) -> ModelConfig: if not envs.VLLM_ENABLE_V1_MULTIPROCESSING: logger.warning( "The global random seed is set to %d. Since " "VLLM_ENABLE_V1_MULTIPROCESSING is set to False, this may " "affect the random state of the Python process that " "launched vLLM.", self.seed, ) return ModelConfig( model=self.model, model_weights=self.model_weights, hf_config_path=self.hf_config_path, runner=self.runner, convert=self.convert, tokenizer=self.tokenizer, # type: ignore[arg-type] tokenizer_mode=self.tokenizer_mode, trust_remote_code=self.trust_remote_code, allowed_local_media_path=self.allowed_local_media_path, allowed_media_domains=self.allowed_media_domains, dtype=self.dtype, seed=self.seed, revision=self.revision, code_revision=self.code_revision, hf_token=self.hf_token, hf_overrides=self.hf_overrides, model_class_overrides=self.model_class_overrides, tokenizer_revision=self.tokenizer_revision, max_model_len=self.max_model_len, quantization=self.quantization, quantization_config=self.quantization_config, allow_deprecated_quantization=self.allow_deprecated_quantization, enforce_eager=self.enforce_eager, enable_return_routed_experts=self.enable_return_routed_experts, max_logprobs=self.max_logprobs, logprobs_mode=self.logprobs_mode, use_fp64_gumbel=self.use_fp64_gumbel, disable_sliding_window=self.disable_sliding_window, disable_cascade_attn=self.disable_cascade_attn, skip_tokenizer_init=self.skip_tokenizer_init, enable_prompt_embeds=self.enable_prompt_embeds, served_model_name=self.served_model_name, language_model_only=self.language_model_only, limit_mm_per_prompt=self.limit_mm_per_prompt, enable_mm_embeds=self.enable_mm_embeds, interleave_mm_strings=self.interleave_mm_strings, media_io_kwargs=self.media_io_kwargs, skip_mm_profiling=self.skip_mm_profiling, config_format=self.config_format, mm_processor_kwargs=self.mm_processor_kwargs, mm_processor_cache_gb=self.mm_processor_cache_gb, mm_processor_cache_type=self.mm_processor_cache_type, mm_shm_cache_max_object_size_mb=self.mm_shm_cache_max_object_size_mb, mm_encoder_only=self.mm_encoder_only, mm_encoder_tp_mode=self.mm_encoder_tp_mode, mm_encoder_attn_backend=self.mm_encoder_attn_backend, mm_encoder_attn_dtype=self.mm_encoder_attn_dtype, mm_encoder_fp8_scale_path=self.mm_encoder_fp8_scale_path, mm_encoder_fp8_scale_save_path=self.mm_encoder_fp8_scale_save_path, mm_encoder_fp8_scale_save_margin=self.mm_encoder_fp8_scale_save_margin, pooler_config=self.pooler_config, generation_config=self.generation_config, override_generation_config=self.override_generation_config, enable_sleep_mode=self.enable_sleep_mode, enable_cumem_allocator=self.enable_cumem_allocator, model_impl=self.model_impl, override_attention_dtype=self.override_attention_dtype, logits_processors=self.logits_processors, video_pruning_rate=self.video_pruning_rate, mm_tensor_ipc=self.mm_tensor_ipc, mm_ipc_gpu_memory_gb=self.mm_ipc_gpu_memory_gb, io_processor_plugin=self.io_processor_plugin, renderer_num_workers=self.renderer_num_workers, ) def validate_tensorizer_args(self): from vllm.model_executor.model_loader.tensorizer import TensorizerConfig for key in self.model_loader_extra_config: if key in TensorizerConfig._fields: self.model_loader_extra_config["tensorizer_config"][key] = ( self.model_loader_extra_config[key] ) def create_load_config(self) -> LoadConfig: if self.quantization == "bitsandbytes": self.load_format = "bitsandbytes" if self.load_format == "tensorizer": if hasattr(self.model_loader_extra_config, "to_serializable"): self.model_loader_extra_config = ( self.model_loader_extra_config.to_serializable() ) self.model_loader_extra_config["tensorizer_config"] = {} self.model_loader_extra_config["tensorizer_config"]["tensorizer_dir"] = ( self.model ) self.validate_tensorizer_args() return LoadConfig( load_format=self.load_format, download_dir=self.download_dir, safetensors_load_strategy=self.safetensors_load_strategy, safetensors_prefetch_num_threads=self.safetensors_prefetch_num_threads, safetensors_prefetch_block_size=self.safetensors_prefetch_block_size, model_loader_extra_config=self.model_loader_extra_config, ignore_patterns=self.ignore_patterns, use_tqdm_on_load=self.use_tqdm_on_load, pt_load_map_location=self.pt_load_map_location, ) def create_speculative_config( self, target_model_config: ModelConfig, target_parallel_config: ParallelConfig, ) -> SpeculativeConfig | None: """Initializes and returns a SpeculativeConfig object based on `speculative_config`. """ for flag, key, value in ( ("--spec-method", "method", self.spec_method), ("--spec-model", "model", self.spec_model), ("--spec-tokens", "num_speculative_tokens", self.spec_tokens), ): if value is None: continue if self.speculative_config is None: self.speculative_config = {} if key in self.speculative_config: raise ValueError( f"{flag} and --speculative-config['{key}'] are mutually exclusive" ) self.speculative_config[key] = value if self.speculative_config is None: return None # Note(Shangming): These parameters are not obtained from the cli arg # '--speculative-config' and must be passed in when creating the engine # config. self.speculative_config.update( { "target_model_config": target_model_config, "target_parallel_config": target_parallel_config, } ) return SpeculativeConfig(**self.speculative_config) def _resolve_device_ids(self) -> list[int] | None: if not self.device_ids: return None if self.distributed_executor_backend == "ray": logger.warning( "--device-ids has no effect when using the Ray executor. " "Use Ray placement groups for GPU selection instead." ) ids = self.device_ids if len(set(ids)) != len(ids): raise ValueError(f"--device-ids must not contain duplicates: {ids}") if all(isinstance(i, str) for i in ids): return [ current_platform.device_control_id_to_physical_device_id(i) for i in cast(list[str], ids) ] if any(isinstance(i, str) for i in ids): raise ValueError("--device-ids must not mix integer IDs and UUIDs") int_ids = cast(list[int], ids) # Compose with CUDA_VISIBLE_DEVICES: if CVD is set, treat # --device-ids values as indices into the CVD-visible set. cvd = getattr( envs, current_platform.device_control_env_var, os.environ.get(current_platform.device_control_env_var), ) if cvd: cvd_ids = [ current_platform.device_control_id_to_physical_device_id(x) for x in cvd.split(",") ] for i in int_ids: if i >= len(cvd_ids): raise ValueError( f"--device-ids index {i} is out of range for " f"{current_platform.device_control_env_var}" f"={cvd} ({len(cvd_ids)} devices visible)" ) return [cvd_ids[i] for i in int_ids] return int_ids def create_diffusion_config(self) -> DiffusionConfig | None: if self.diffusion_config is None: return None cfg = self.diffusion_config if isinstance(cfg, str): cfg = json.loads(cfg) return DiffusionConfig(**cfg) def create_observability_config(self) -> ObservabilityConfig: return ObservabilityConfig( show_hidden_metrics_for_version=self.show_hidden_metrics_for_version, otlp_traces_endpoint=self.otlp_traces_endpoint, collect_detailed_traces=self.collect_detailed_traces, kv_cache_metrics=self.kv_cache_metrics, kv_cache_metrics_sample=self.kv_cache_metrics_sample, cudagraph_metrics=self.cudagraph_metrics, enable_layerwise_nvtx_tracing=self.enable_layerwise_nvtx_tracing, enable_mfu_metrics=self.enable_mfu_metrics, enable_mm_processor_stats=self.enable_mm_processor_stats, enable_logging_iteration_details=self.enable_logging_iteration_details, jit_monitor_mode=self.jit_monitor_mode, jit_monitor_verbose=self.jit_monitor_verbose, ) def create_engine_config( self, usage_context: UsageContext | None = None, headless: bool = False, ) -> VllmConfig: """ Create the VllmConfig. NOTE: If VllmConfig is incompatible, we raise an error. """ current_platform.pre_register_and_update() device_config = DeviceConfig(device=cast(Device, current_platform.device_type)) envs.validate_environ(self.fail_on_environ_validation) # Check if the model is a speculator and override model/tokenizer/config # BEFORE creating ModelConfig, so the config is created with the target model # Skip speculator detection for cloud storage models (eg: S3, GCS) since # HuggingFace cannot load configs directly from S3 URLs. S3 models can still # use speculators with explicit --speculative-config. if not is_cloud_storage(self.model): (self.model, self.tokenizer, self.speculative_config) = ( maybe_override_with_speculators( model=self.model, tokenizer=self.tokenizer, revision=self.revision, trust_remote_code=self.trust_remote_code, vllm_speculative_config=self.speculative_config, hf_token=self.hf_token, ) ) model_config = self.create_model_config() self.model = model_config.model self.model_weights = model_config.model_weights self.tokenizer = model_config.tokenizer self._check_feature_supported() self._set_default_chunked_prefill_and_prefix_caching_args(model_config) self._set_default_reasoning_config_args() sliding_window: int | None = None if not is_interleaved(model_config.hf_text_config): # Only set CacheConfig.sliding_window if the model is all sliding # window. Otherwise CacheConfig.sliding_window will override the # global layers in interleaved sliding window models. sliding_window = model_config.get_sliding_window() # Resolve "auto" kv_cache_dtype to actual value from model config resolved_cache_dtype = resolve_kv_cache_dtype_string( self.kv_cache_dtype, model_config ) assert self.enable_prefix_caching is not None, ( "enable_prefix_caching must be set by this point" ) cache_config = CacheConfig( block_size=self.block_size, # type: ignore[arg-type] gpu_memory_utilization=self.gpu_memory_utilization, kv_cache_memory_bytes=self.kv_cache_memory_bytes, cache_dtype=resolved_cache_dtype, # type: ignore[arg-type] is_attention_free=model_config.is_attention_free, num_gpu_blocks_override=self.num_gpu_blocks_override, sliding_window=sliding_window, enable_prefix_caching=self.enable_prefix_caching, prefix_caching_hash_algo=self.prefix_caching_hash_algo, calculate_kv_scales=self.calculate_kv_scales, kv_cache_dtype_skip_layers=self.kv_cache_dtype_skip_layers, kv_sharing_fast_prefill=self.kv_sharing_fast_prefill, mamba_cache_dtype=self.mamba_cache_dtype, mamba_ssm_cache_dtype=self.mamba_ssm_cache_dtype, mamba_block_size=self.mamba_block_size, prefix_match_unit=self.prefix_match_unit, mamba_cache_mode=self.mamba_cache_mode, kv_offloading_size=self.kv_offloading_size, kv_offloading_backend=self.kv_offloading_backend, ) if resolved_cache_dtype.startswith("turboquant_"): from vllm.model_executor.layers.quantization.turboquant.config import ( TurboQuantConfig, ) boundary = TurboQuantConfig.get_boundary_skip_layers(model_config) existing = set(cache_config.kv_cache_dtype_skip_layers) cache_config.kv_cache_dtype_skip_layers = sorted( existing | set(boundary), key=int ) ray_runtime_env = None if is_ray_initialized(): # Ray Serve LLM calls `create_engine_config` in the context # of a Ray task, therefore we check is_ray_initialized() # as opposed to is_in_ray_actor(). import ray ray_runtime_env = ray.get_runtime_context().runtime_env # Avoid logging sensitive environment variables sanitized_env = ray_runtime_env.to_dict() if ray_runtime_env else {} if "env_vars" in sanitized_env: sanitized_env["env_vars"] = { k: "***" for k in sanitized_env["env_vars"] } logger.info("Using ray runtime env (env vars redacted): %s", sanitized_env) # Get the current placement group if Ray is initialized and # we are in a Ray actor. If so, then the placement group will be # passed to spawned processes. placement_group = None if is_in_ray_actor(): import ray # This call initializes Ray automatically if it is not initialized, # but we should not do this here. placement_group = ray.util.get_current_placement_group() assert not headless or not self.data_parallel_hybrid_lb, ( "data_parallel_hybrid_lb is not applicable in headless mode" ) assert not (self.data_parallel_hybrid_lb and self.data_parallel_external_lb), ( "data_parallel_hybrid_lb and data_parallel_external_lb cannot both be True." ) assert self.data_parallel_backend == "mp" or self.nnodes == 1, ( "nnodes > 1 is only supported with data_parallel_backend=mp" ) inferred_data_parallel_rank = 0 if self.nnodes > 1: world_size = ( self.data_parallel_size * self.pipeline_parallel_size * self.tensor_parallel_size ) world_size_within_dp = ( self.pipeline_parallel_size * self.tensor_parallel_size ) local_world_size = world_size // self.nnodes assert world_size % self.nnodes == 0, ( f"world_size={world_size} must be divisible by nnodes={self.nnodes}." ) assert self.node_rank < self.nnodes, ( f"node_rank={self.node_rank} must be less than nnodes={self.nnodes}." ) inferred_data_parallel_rank = ( self.node_rank * local_world_size ) // world_size_within_dp if self.data_parallel_size > 1 and self.data_parallel_external_lb: self.data_parallel_rank = inferred_data_parallel_rank logger.info( "Inferred data_parallel_rank %d from node_rank %d for external lb", self.data_parallel_rank, self.node_rank, ) elif self.data_parallel_size_local is None: # Infer data parallel size local for internal dplb: self.data_parallel_size_local = max( local_world_size // world_size_within_dp, 1 ) data_parallel_external_lb = ( self.data_parallel_external_lb or self.data_parallel_rank is not None ) if ( self.data_parallel_size > 1 and data_parallel_external_lb and not model_config.is_moe ): raise ValueError( "Non-MoE models do not support external data parallel mode. " "For external load balancing, launch independent vLLM " "instances without --data-parallel-* arguments." ) # Local DP rank = 1, use pure-external LB. if data_parallel_external_lb: assert self.data_parallel_rank is not None, ( "data_parallel_rank or node_rank must be specified if " "data_parallel_external_lb is enable." ) assert self.data_parallel_size_local in (1, None), ( "data_parallel_size_local must be 1 or None when data_parallel_rank " "is set" ) data_parallel_size_local = 1 # Use full external lb if we have local_size of 1. self.data_parallel_hybrid_lb = False elif self.data_parallel_size_local is not None: data_parallel_size_local = self.data_parallel_size_local if self.data_parallel_start_rank and not headless: # Infer hybrid LB mode. self.data_parallel_hybrid_lb = True if self.data_parallel_hybrid_lb and data_parallel_size_local == 1: # Use full external lb if we have local_size of 1. logger.warning( "data_parallel_hybrid_lb is not eligible when " "data_parallel_size_local = 1, autoswitch to " "data_parallel_external_lb." ) data_parallel_external_lb = True self.data_parallel_hybrid_lb = False if data_parallel_size_local == self.data_parallel_size: # Disable hybrid LB mode if set for a single node self.data_parallel_hybrid_lb = False self.data_parallel_rank = ( self.data_parallel_start_rank or inferred_data_parallel_rank ) if self.nnodes > 1: logger.info( "Inferred data_parallel_rank %d from node_rank %d", self.data_parallel_rank, self.node_rank, ) else: assert not self.data_parallel_hybrid_lb, ( "data_parallel_size_local must be set to use data_parallel_hybrid_lb." ) if self.data_parallel_backend == "ray" and ( envs.VLLM_RAY_DP_PACK_STRATEGY == "span" ): # Data parallel size defaults to 1 if DP ranks are spanning # multiple nodes data_parallel_size_local = 1 else: # Otherwise local DP size defaults to global DP size if not set data_parallel_size_local = self.data_parallel_size # DP address, used in multi-node case for torch distributed group # and ZMQ sockets. if self.data_parallel_address is None: if self.data_parallel_backend == "ray": host_ip = get_ip() logger.info( "Using host IP %s as ray-based data parallel address", host_ip ) data_parallel_address = host_ip else: assert self.data_parallel_backend == "mp", ( "data_parallel_backend can only be ray or mp, got %s", self.data_parallel_backend, ) data_parallel_address = ( self.master_addr or ParallelConfig.data_parallel_master_ip ) else: data_parallel_address = self.data_parallel_address # This port is only used when there are remote data parallel engines, # otherwise the local IPC transport is used. data_parallel_rpc_port = ( self.data_parallel_rpc_port if (self.data_parallel_rpc_port is not None) else ParallelConfig.data_parallel_rpc_port ) if self.tokens_only and not model_config.skip_tokenizer_init: model_config.skip_tokenizer_init = True logger.info("Skipping tokenizer initialization for tokens-only mode.") parallel_config = ParallelConfig( pipeline_parallel_size=self.pipeline_parallel_size, tensor_parallel_size=self.tensor_parallel_size, prefill_context_parallel_size=self.prefill_context_parallel_size, data_parallel_size=self.data_parallel_size, data_parallel_rank=self.data_parallel_rank or 0, data_parallel_external_lb=data_parallel_external_lb, data_parallel_size_local=data_parallel_size_local, master_addr=self.master_addr, master_port=self.master_port, nnodes=self.nnodes, node_rank=self.node_rank, distributed_timeout_seconds=self.distributed_timeout_seconds, cpu_distributed_timeout_seconds=self.cpu_distributed_timeout_seconds, data_parallel_master_ip=data_parallel_address, data_parallel_rpc_port=data_parallel_rpc_port, data_parallel_backend=self.data_parallel_backend, data_parallel_hybrid_lb=self.data_parallel_hybrid_lb, is_moe_model=model_config.is_moe, enable_expert_parallel=self.enable_expert_parallel, enable_ep_weight_filter=self.enable_ep_weight_filter, all2all_backend=self.all2all_backend, enable_elastic_ep=self.enable_elastic_ep, enable_dbo=self.enable_dbo, ubatch_size=self.ubatch_size, dbo_decode_token_threshold=self.dbo_decode_token_threshold, dbo_prefill_token_threshold=self.dbo_prefill_token_threshold, disable_nccl_for_dp_synchronization=self.disable_nccl_for_dp_synchronization, enable_eplb=self.enable_eplb, eplb_config=self.eplb_config, expert_placement_strategy=self.expert_placement_strategy, max_parallel_loading_workers=self.max_parallel_loading_workers, disable_custom_all_reduce=self.disable_custom_all_reduce, ray_workers_use_nsight=self.ray_workers_use_nsight, ray_runtime_env=ray_runtime_env, placement_group=placement_group, distributed_executor_backend=self.distributed_executor_backend, worker_cls=self.worker_cls, worker_extension_cls=self.worker_extension_cls, decode_context_parallel_size=self.decode_context_parallel_size, dcp_comm_backend=self.dcp_comm_backend, dcp_kv_cache_interleave_size=self.dcp_kv_cache_interleave_size, cp_kv_cache_interleave_size=self.cp_kv_cache_interleave_size, _api_process_count=self._api_process_count, _api_process_rank=self._api_process_rank, assigned_physical_gpu_ids=self._resolve_device_ids(), numa_bind=self.numa_bind, numa_bind_nodes=self.numa_bind_nodes, numa_bind_cpus=self.numa_bind_cpus, ) speculative_config = self.create_speculative_config( target_model_config=model_config, target_parallel_config=parallel_config, ) diffusion_config = self.create_diffusion_config() self._set_default_max_num_seqs_and_batched_tokens_args( usage_context, model_config, parallel_config, ) assert self.max_num_batched_tokens is not None, ( "max_num_batched_tokens must be set by this point" ) assert self.max_num_seqs is not None, "max_num_seqs must be set by this point" assert self.enable_chunked_prefill is not None, ( "enable_chunked_prefill must be set by this point" ) assert model_config.max_model_len is not None, ( "max_model_len must be set by this point" ) scheduler_config = SchedulerConfig( runner_type=model_config.runner_type, max_num_batched_tokens=self.max_num_batched_tokens, max_num_scheduled_tokens=self.max_num_scheduled_tokens, max_num_seqs=self.max_num_seqs, max_model_len=model_config.max_model_len, enable_chunked_prefill=self.enable_chunked_prefill, disable_chunked_mm_input=self.disable_chunked_mm_input, is_multimodal_model=model_config.is_multimodal_model, is_encoder_decoder=model_config.is_encoder_decoder, policy=self.scheduling_policy, scheduler_cls=self.scheduler_cls, max_num_partial_prefills=self.max_num_partial_prefills, max_long_partial_prefills=self.max_long_partial_prefills, long_prefill_token_threshold=self.long_prefill_token_threshold, scheduler_reserve_full_isl=self.scheduler_reserve_full_isl, watermark=self.watermark, prefill_schedule_interval=self.prefill_schedule_interval, disable_hybrid_kv_cache_manager=self.disable_hybrid_kv_cache_manager, async_scheduling=self.async_scheduling, stream_interval=self.stream_interval, ) if not model_config.is_multimodal_model and self.default_mm_loras: raise ValueError( "Default modality-specific LoRA(s) were provided for a " "non multimodal model" ) lora_config = ( LoRAConfig( max_lora_rank=self.max_lora_rank, max_loras=self.max_loras, default_mm_loras=self.default_mm_loras, fully_sharded_loras=self.fully_sharded_loras, lora_dtype=self.lora_dtype, target_modules=self.lora_target_modules, enable_tower_connector_lora=self.enable_tower_connector_lora, specialize_active_lora=self.specialize_active_lora, enable_mixed_moe_lora_format=self.enable_mixed_moe_lora_format, max_cpu_loras=self.max_cpu_loras if self.max_cpu_loras and self.max_cpu_loras > 0 else None, ) if self.enable_lora else None ) if ( lora_config is not None and speculative_config is not None and scheduler_config.max_num_batched_tokens < ( scheduler_config.max_num_seqs * (speculative_config.num_speculative_tokens + 1) ) ): raise ValueError( "Consider increasing max_num_batched_tokens or " "decreasing num_speculative_tokens" ) # bitsandbytes pre-quantized model need a specific model loader if model_config.quantization == "bitsandbytes": self.quantization = self.load_format = "bitsandbytes" # Attention config overrides attention_config = copy.deepcopy(self.attention_config) if self.attention_backend is not None: if attention_config.backend is not None: raise ValueError( "attention_backend and attention_config.backend " "are mutually exclusive" ) # Reuse the validator to handle "auto" and string-to-enum conversion attention_config.backend = AttentionConfig.validate_backend_before( self.attention_backend ) # TurboQuant requires FlashAttention 2 — FA3 boundary layers assert # FlashAttentionImpl which fails with TurboQuantAttentionImpl. if resolved_cache_dtype.startswith("turboquant_") and ( attention_config.flash_attn_version is None or attention_config.flash_attn_version >= 3 ): logger.warning( "TurboQuant is not yet compatible with FlashAttention >= 3. " "Overriding flash_attn_version to 2. To silence this " "warning, pass --attention-config.flash_attn_version=2" ) attention_config.flash_attn_version = 2 # Mamba config overrides mamba_config = copy.deepcopy(self.mamba_config) # Convert string to enum if needed (CLI parsing returns a string) if isinstance(self.mamba_backend, str): mamba_config.backend = MambaBackendEnum[self.mamba_backend.upper()] else: mamba_config.backend = self.mamba_backend if self.enable_mamba_cache_stochastic_rounding: mamba_config.enable_stochastic_rounding = ( self.enable_mamba_cache_stochastic_rounding ) if self.mamba_cache_philox_rounds: mamba_config.stochastic_rounding_philox_rounds = ( self.mamba_cache_philox_rounds ) # Kernel config overrides kernel_config = copy.deepcopy(self.kernel_config) if self.enable_flashinfer_autotune is not None: if kernel_config.enable_flashinfer_autotune is not None: raise ValueError( "enable_flashinfer_autotune and " "kernel_config.enable_flashinfer_autotune " "are mutually exclusive" ) kernel_config.enable_flashinfer_autotune = self.enable_flashinfer_autotune if self.moe_backend != "auto": kernel_config.moe_backend = self.moe_backend if self.linear_backend != "auto": kernel_config.linear_backend = self.linear_backend # Transfer top-level ir_op_priority into KernelConfig.ir_op_priority for op_name, op_priority in asdict(self.ir_op_priority).items(): # Empty means unset if not op_priority: continue # Priority cannot be set 2x for the same op if getattr(kernel_config.ir_op_priority, op_name): raise ValueError( f"Op priority for {op_name} specified via both ir_op_priority " f"and KernelConfig.ir_op_priority, only one allowed at a time." ) # Set the attribute setattr(kernel_config.ir_op_priority, op_name, op_priority) load_config = self.create_load_config() # Pass reasoning_parser into StructuredOutputsConfig if self.reasoning_parser: self.structured_outputs_config.reasoning_parser = self.reasoning_parser if self.reasoning_parser_plugin: self.structured_outputs_config.reasoning_parser_plugin = ( self.reasoning_parser_plugin ) observability_config = self.create_observability_config() # Compilation config overrides compilation_config = copy.deepcopy(self.compilation_config) if self.cudagraph_capture_sizes is not None: if compilation_config.cudagraph_capture_sizes is not None: raise ValueError( "cudagraph_capture_sizes and compilation_config." "cudagraph_capture_sizes are mutually exclusive" ) compilation_config.cudagraph_capture_sizes = self.cudagraph_capture_sizes if self.max_cudagraph_capture_size is not None: if compilation_config.max_cudagraph_capture_size is not None: raise ValueError( "max_cudagraph_capture_size and compilation_config." "max_cudagraph_capture_size are mutually exclusive" ) compilation_config.max_cudagraph_capture_size = ( self.max_cudagraph_capture_size ) offload_config = OffloadConfig( offload_backend=self.offload_backend, uva=UVAOffloadConfig( cpu_offload_gb=self.cpu_offload_gb, cpu_offload_params=self.cpu_offload_params, ), prefetch=PrefetchOffloadConfig( offload_group_size=self.offload_group_size, offload_num_in_group=self.offload_num_in_group, offload_prefetch_step=self.offload_prefetch_step, offload_params=self.offload_params, ), ) if self.gdn_prefill_backend is not None: self.additional_config["gdn_prefill_backend"] = self.gdn_prefill_backend config = VllmConfig( model_config=model_config, cache_config=cache_config, parallel_config=parallel_config, scheduler_config=scheduler_config, device_config=device_config, load_config=load_config, offload_config=offload_config, attention_config=attention_config, mamba_config=mamba_config, kernel_config=kernel_config, lora_config=lora_config, speculative_config=speculative_config, diffusion_config=diffusion_config, structured_outputs_config=self.structured_outputs_config, observability_config=observability_config, compilation_config=compilation_config, kv_transfer_config=self.kv_transfer_config, kv_events_config=self.kv_events_config, ec_transfer_config=self.ec_transfer_config, reasoning_config=self.reasoning_config, profiler_config=self.profiler_config, additional_config=self.additional_config, optimization_level=self.optimization_level, performance_mode=self.performance_mode, weight_transfer_config=self.weight_transfer_config, shutdown_timeout=self.shutdown_timeout, ) return config def _check_feature_supported(self): """Raise an error if the feature is not supported.""" # No Concurrent Partial Prefills so far. if ( self.max_num_partial_prefills != SchedulerConfig.max_num_partial_prefills or self.max_long_partial_prefills != SchedulerConfig.max_long_partial_prefills ): _raise_unsupported_error(feature_name="Concurrent Partial Prefill") if self.pipeline_parallel_size > 1: supports_pp = getattr( self.distributed_executor_backend, "supports_pp", False ) if not supports_pp and self.distributed_executor_backend not in ( ParallelConfig.distributed_executor_backend, "ray", "mp", "external_launcher", ): name = ( "Pipeline Parallelism without Ray distributed " "executor or multiprocessing executor or external " "launcher" ) _raise_unsupported_error(feature_name=name) @classmethod def get_batch_defaults( cls, world_size: int, ) -> tuple[dict[UsageContext | None, int], dict[UsageContext | None, int]]: from vllm.usage.usage_lib import UsageContext default_max_num_batched_tokens: dict[UsageContext | None, int] default_max_num_seqs: dict[UsageContext | None, int] # When no user override, set the default values based on the usage # context. # Use different default values for different hardware. # Try to query the device name on the current platform. If it fails, # it may be because the platform that imports vLLM is not the same # as the platform that vLLM is running on (e.g. the case of scaling # vLLM with Ray) and has no GPUs. In this case we use the default # values for non-H100/H200 GPUs. try: device_memory = current_platform.get_device_total_memory() device_name = current_platform.get_device_name().lower() except Exception: # This is only used to set default_max_num_batched_tokens device_memory = 0 device_name = "" # NOTE(Kuntai): Setting large `max_num_batched_tokens` for A100 reduces # throughput, see PR #17885 for more details. # So here we do an extra device name check to prevent such regression. if device_memory >= 70 * GiB_bytes and "a100" not in device_name: # For GPUs like H100 and MI300x, use larger default values. default_max_num_batched_tokens = { UsageContext.LLM_CLASS: 16384, UsageContext.OPENAI_API_SERVER: 8192, } default_max_num_seqs = { UsageContext.LLM_CLASS: 1024, UsageContext.OPENAI_API_SERVER: 1024, } else: # TODO(woosuk): Tune the default values for other hardware. default_max_num_batched_tokens = { UsageContext.LLM_CLASS: 8192, UsageContext.OPENAI_API_SERVER: 2048, } default_max_num_seqs = { UsageContext.LLM_CLASS: 256, UsageContext.OPENAI_API_SERVER: 256, } # tpu specific default values. if current_platform.is_tpu(): chip_name = current_platform.get_device_name() if chip_name == "V6E": default_max_num_batched_tokens = { UsageContext.LLM_CLASS: 2048, UsageContext.OPENAI_API_SERVER: 1024, } elif chip_name == "V5E": default_max_num_batched_tokens = { UsageContext.LLM_CLASS: 1024, UsageContext.OPENAI_API_SERVER: 512, } elif chip_name == "V5P": default_max_num_batched_tokens = { UsageContext.LLM_CLASS: 512, UsageContext.OPENAI_API_SERVER: 256, } # cpu specific default values. if current_platform.is_cpu(): default_max_num_batched_tokens = { UsageContext.LLM_CLASS: 4096 * world_size, UsageContext.OPENAI_API_SERVER: 2048 * world_size, } default_max_num_seqs = { UsageContext.LLM_CLASS: 256 * world_size, UsageContext.OPENAI_API_SERVER: 128 * world_size, } return default_max_num_batched_tokens, default_max_num_seqs def _set_default_chunked_prefill_and_prefix_caching_args( self, model_config: ModelConfig ) -> None: default_chunked_prefill = model_config.is_chunked_prefill_supported # Hybrid models support prefix caching but keep it opt-in for now # while the feature matures. default_prefix_caching = ( model_config.is_prefix_caching_supported and not model_config.is_hybrid ) if self.enable_chunked_prefill is None: self.enable_chunked_prefill = default_chunked_prefill logger.debug( "%s chunked prefill by default", "Enabling" if default_chunked_prefill else "Disabling", ) elif ( model_config.runner_type == "generate" and not self.enable_chunked_prefill and default_chunked_prefill ): logger.warning_once( "This model does not officially support disabling chunked prefill. " "Disabling this manually may cause the engine to crash " "or produce incorrect outputs.", ) elif ( model_config.runner_type == "pooling" and self.enable_chunked_prefill and not default_chunked_prefill ): logger.warning_once( "This model does not officially support chunked prefill. " "Enabling this manually may cause the engine to crash " "or produce incorrect outputs.", ) if self.enable_prefix_caching is None: self.enable_prefix_caching = default_prefix_caching logger.debug( "%s prefix caching by default", "Enabling" if default_prefix_caching else "Disabling", ) elif ( model_config.runner_type == "pooling" and self.enable_prefix_caching and not default_prefix_caching ): logger.warning_once( "This model does not officially support prefix caching. " "Enabling this manually may cause the engine to crash " "or produce incorrect outputs.", ) # Disable chunked prefill and prefix caching for: # RISCV CPUs in V1 if current_platform.is_cpu() and current_platform.get_cpu_architecture() in ( CpuArchEnum.RISCV, ): logger.info( "Chunked prefill is not supported for" "RISC-V CPUs; " "disabling it for V1 backend." ) self.enable_chunked_prefill = False logger.info( "Prefix caching is not supported for " "RISC-V CPUs; " "disabling it for V1 backend." ) self.enable_prefix_caching = False def _set_default_reasoning_config_args(self): if not self.reasoning_parser: return if self.reasoning_config is None: self.reasoning_config = ReasoningConfig() self.reasoning_config.reasoning_parser = self.reasoning_parser @staticmethod def _get_min_mm_batched_tokens( model_config: ModelConfig, ) -> tuple[int, str] | None: """Get the minimum max_num_batched_tokens needed for a multimodal prefix-LM model to process at least one item of any supported modality. Returns (token_count, modality_name) for the most expensive modality, or None if the value cannot be determined at this stage. """ try: from vllm.multimodal import MULTIMODAL_REGISTRY # get_processing_info returns the model's multimodal processing # metadata (supported modalities, token limits) without loading # model weights or generating dummy data. info = MULTIMODAL_REGISTRY.get_processing_info(model_config) mm_counts = {modality: 1 for modality in info.supported_mm_limits} # get_mm_max_tokens_per_item returns pre-computed per-item token # ceilings for models that override it (e.g., Gemma4), or None # for models that rely on dummy-input profiling. When None is # returned we bail out — no dummy generation is triggered here. max_tokens = info.get_mm_max_tokens_per_item( seq_len=model_config.max_model_len, mm_counts=mm_counts, ) if max_tokens is not None: modality = max(max_tokens, key=max_tokens.__getitem__) return (max_tokens[modality], modality) except Exception as e: logger.warning("Failed to determine min multimodal batched tokens: %s", e) return None def _set_default_max_num_seqs_and_batched_tokens_args( self, usage_context: UsageContext | None, model_config: ModelConfig, parallel_config: ParallelConfig, ): world_size = self.pipeline_parallel_size * self.tensor_parallel_size ( default_max_num_batched_tokens, default_max_num_seqs, ) = self.get_batch_defaults(world_size) orig_max_num_batched_tokens = self.max_num_batched_tokens orig_max_num_seqs = self.max_num_seqs if self.max_num_batched_tokens is None: if parallel_config.use_batched_dp_moe: self.max_num_batched_tokens = ( SchedulerConfig.DEFAULT_MAX_NUM_BATCHED_TOKENS_FOR_BATCHED_DP ) else: self.max_num_batched_tokens = default_max_num_batched_tokens.get( usage_context, SchedulerConfig.DEFAULT_MAX_NUM_BATCHED_TOKENS, ) if self.max_num_seqs is None: self.max_num_seqs = default_max_num_seqs.get( usage_context, SchedulerConfig.DEFAULT_MAX_NUM_SEQS, ) # If throughput mode is set, double max_num_batched_tokens and max_num_seqs. if self.performance_mode == "throughput": if orig_max_num_batched_tokens is None: self.max_num_batched_tokens *= 2 if orig_max_num_seqs is None: self.max_num_seqs *= 2 if orig_max_num_batched_tokens is None: assert model_config.max_model_len is not None, ( "max_model_len must be set by this point" ) if not self.enable_chunked_prefill: # If max_model_len is too short, use the default for higher throughput. self.max_num_batched_tokens = max( model_config.max_model_len, self.max_num_batched_tokens, ) # For multimodal prefix-LM models (e.g., Gemma 4) that disable # chunked MM input, a single multimodal item must fit in one batch. # Raise the floor to accommodate the largest per-item token count. if model_config.is_multimodal_model and model_config.is_mm_prefix_lm: result = self._get_min_mm_batched_tokens(model_config) if result is not None and result[0] > self.max_num_batched_tokens: mm_min, modality = result logger.info( "Raising max_num_batched_tokens from %d to %d to " "accommodate '%s' input for prefix-LM model %s.", self.max_num_batched_tokens, mm_min, modality, model_config.model, ) self.max_num_batched_tokens = mm_min # When using default settings, # Ensure max_num_batched_tokens does not exceed model limit. # Some models (e.g., Whisper) have embeddings tied to max length. self.max_num_batched_tokens = min( self.max_num_seqs * model_config.max_model_len, self.max_num_batched_tokens, ) logger.debug( "Defaulting max_num_batched_tokens to %d for %s usage context.", self.max_num_batched_tokens, usage_context.value if usage_context else None, ) if orig_max_num_seqs is None: assert self.max_num_batched_tokens is not None # For type checking self.max_num_seqs = min(self.max_num_seqs, self.max_num_batched_tokens) logger.debug( "Defaulting max_num_seqs to %d for %s usage context.", self.max_num_seqs, usage_context.value if usage_context else None, ) @dataclass class AsyncEngineArgs(EngineArgs): """Arguments for asynchronous vLLM engine.""" enable_log_requests: bool = False @staticmethod def add_cli_args( parser: FlexibleArgumentParser, async_args_only: bool = False ) -> FlexibleArgumentParser: # Initialize plugin to update the parser, for example, The plugin may # add a new kind of quantization method to --quantization argument or # a new device to --device argument. load_general_plugins() if not async_args_only: parser = EngineArgs.add_cli_args(parser) parser.add_argument( "--enable-log-requests", action=argparse.BooleanOptionalAction, default=AsyncEngineArgs.enable_log_requests, help="Enable logging request information, dependent on log level:\n" "- INFO: Request ID, parameters and LoRA request.\n" "- DEBUG: Prompt inputs (e.g: text, token IDs).\n" "You can set the minimum log level via `VLLM_LOGGING_LEVEL`.", ) current_platform.pre_register_and_update(parser) return parser def _raise_unsupported_error(feature_name: str): msg = ( f"{feature_name} is not supported. We recommend to " f"remove {feature_name} from your config." ) raise NotImplementedError(msg)