# SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project import functools import json import logging import os import sys import tempfile import uuid from collections.abc import Callable from typing import TYPE_CHECKING, Any, Literal if TYPE_CHECKING: VLLM_HOST_IP: str = "" VLLM_PORT: int | None = None VLLM_RPC_BASE_PATH: str = tempfile.gettempdir() VLLM_USE_MODELSCOPE: bool = False VLLM_USE_FASTOKENS: bool = False VLLM_RINGBUFFER_WARNING_INTERVAL: int = 60 VLLM_NCCL_SO_PATH: str | None = None LD_LIBRARY_PATH: str | None = None VLLM_ROCM_SLEEP_MEM_CHUNK_SIZE: int = 256 LOCAL_RANK: int = 0 CUDA_VISIBLE_DEVICES: str | None = None VLLM_ENGINE_ITERATION_TIMEOUT_S: int = 60 VLLM_ENGINE_READY_TIMEOUT_S: int = 600 VLLM_API_KEY: str | None = None VLLM_DEBUG_LOG_API_SERVER_RESPONSE: bool = False S3_ACCESS_KEY_ID: str | None = None S3_SECRET_ACCESS_KEY: str | None = None S3_ENDPOINT_URL: str | None = None VLLM_MODEL_REDIRECT_PATH: str | None = None VLLM_CACHE_ROOT: str = os.path.expanduser("~/.cache/vllm") VLLM_CONFIG_ROOT: str = os.path.expanduser("~/.config/vllm") VLLM_USAGE_STATS_SERVER: str = "https://stats.vllm.ai" VLLM_NO_USAGE_STATS: bool = False VLLM_DO_NOT_TRACK: bool = False VLLM_USAGE_SOURCE: str = "production" VLLM_CONFIGURE_LOGGING: bool = True VLLM_LOGGING_LEVEL: str = "INFO" VLLM_LOGGING_PREFIX: str = "" VLLM_LOGGING_STREAM: str = "ext://sys.stdout" VLLM_LOGGING_CONFIG_PATH: str | None = None VLLM_LOGGING_COLOR: str = "auto" NO_COLOR: bool = False VLLM_LOG_STATS_INTERVAL: float = 10.0 VLLM_TRACE_FUNCTION: int = 0 VLLM_USE_FLASHINFER_SAMPLER: bool = True VLLM_PP_LAYER_PARTITION: str | None = None VLLM_CPU_KVCACHE_SPACE: int | None = 0 VLLM_CPU_OMP_THREADS_BIND: str = "auto" VLLM_CPU_NUM_OF_RESERVED_CPU: int | None = None VLLM_CPU_SGL_KERNEL: bool = False VLLM_CPU_ATTN_SPLIT_KV: bool = True VLLM_ZENTORCH_WEIGHT_PREPACK: bool = True VLLM_CPU_INT4_W4A8: bool = True VLLM_XLA_CACHE_PATH: str = os.path.join(VLLM_CACHE_ROOT, "xla_cache") VLLM_XLA_CHECK_RECOMPILATION: bool = False VLLM_SPARSE_INDEXER_MAX_LOGITS_MB: int = 512 VLLM_USE_RAY_COMPILED_DAG_CHANNEL_TYPE: Literal["auto", "nccl", "shm"] = "auto" VLLM_USE_RAY_COMPILED_DAG_OVERLAP_COMM: bool = False VLLM_USE_RAY_WRAPPED_PP_COMM: bool = True VLLM_USE_RAY_V2_EXECUTOR_BACKEND: bool = False VLLM_DISTRIBUTED_USE_SPLIT_GROUP: bool = False VLLM_XLA_USE_SPMD: bool = False VLLM_WORKER_MULTIPROC_METHOD: Literal["fork", "spawn"] = "fork" VLLM_ASSETS_CACHE: str = os.path.join(VLLM_CACHE_ROOT, "assets") VLLM_ASSETS_CACHE_MODEL_CLEAN: bool = False VLLM_IMAGE_FETCH_TIMEOUT: int = 5 VLLM_VIDEO_FETCH_TIMEOUT: int = 30 VLLM_AUDIO_FETCH_TIMEOUT: int = 10 VLLM_MEDIA_CACHE: str = "" VLLM_MEDIA_CACHE_MAX_SIZE_MB: int = 5120 VLLM_MEDIA_CACHE_TTL_HOURS: float = 24 VLLM_MEDIA_FETCH_MAX_RETRIES: int = 3 VLLM_MEDIA_URL_ALLOW_REDIRECTS: bool = True VLLM_MEDIA_LOADING_THREAD_COUNT: int = 8 VLLM_MAX_AUDIO_CLIP_FILESIZE_MB: int = 25 VLLM_MAX_AUDIO_DECODE_DURATION_S: int = 600 VLLM_MAX_AUDIO_PREPROCESS_WORKERS: int = max(1, min(os.cpu_count() or 1, 2)) VLLM_MAX_IMAGE_PIXELS: int = 178_956_970 VLLM_VIDEO_LOADER_BACKEND: str = "opencv" VLLM_MEDIA_CONNECTOR: str = "http" VLLM_MM_HASHER_ALGORITHM: str = "blake3" VLLM_TARGET_DEVICE: str = "cuda" VLLM_MAIN_CUDA_VERSION: str = "13.0" VLLM_FLOAT32_MATMUL_PRECISION: Literal["highest", "high", "medium"] = "highest" VLLM_BATCH_INVARIANT: bool = False VLLM_TRITON_ATTN_USE_TD: bool | None = None VLLM_GPU_SYNC_CHECK: Literal["warn", "error"] | None = None MAX_JOBS: str | None = None NVCC_THREADS: str | None = None VLLM_USE_PRECOMPILED: bool = False VLLM_USE_PRECOMPILED_RUST: bool = False VLLM_SKIP_PRECOMPILED_VERSION_SUFFIX: bool = False VLLM_DOCKER_BUILD_CONTEXT: bool = False VLLM_BUILD_COMMIT: str = "unknown" VLLM_BUILD_PIPELINE: str = "local" VLLM_BUILD_URL: str = "" VLLM_IMAGE_TAG: str = "" VLLM_KEEP_ALIVE_ON_ENGINE_DEATH: bool = False CMAKE_BUILD_TYPE: Literal["Debug", "Release", "RelWithDebInfo"] | None = None VERBOSE: bool = False VLLM_ALLOW_LONG_MAX_MODEL_LEN: bool = False VLLM_HTTP_TIMEOUT_KEEP_ALIVE: int = 5 # seconds VLLM_MAX_N_SEQUENCES: int = 16384 VLLM_MAX_COMPLETION_PROMPTS: int = 1024 VLLM_PLUGINS: list[str] | None = None VLLM_LORA_RESOLVER_CACHE_DIR: str | None = None VLLM_LORA_RESOLVER_HF_REPO_LIST: str | None = None VLLM_USE_AOT_COMPILE: bool = False VLLM_USE_BYTECODE_HOOK: bool = True VLLM_FORCE_AOT_LOAD: bool = False VLLM_USE_MEGA_AOT_ARTIFACT: bool = False VLLM_USE_TRITON_AWQ: bool = False VLLM_FASTSAFETENSORS_QUEUE_SIZE: int = 0 VLLM_TRITON_FORCE_FIRST_CONFIG: bool = False VLLM_ALLOW_RUNTIME_LORA_UPDATING: bool = False VLLM_SKIP_P2P_CHECK: bool = False VLLM_DISABLED_KERNELS: list[str] = [] VLLM_ENABLE_FLA_PACKED_RECURRENT_DECODE: bool = True VLLM_DISABLE_PYNCCL: bool = False VLLM_USE_OINK_OPS: bool = False VLLM_MXFP8_EMULATION_DEQUANT_AT_LOAD: bool = True VLLM_ROCM_USE_AITER: bool = False VLLM_ROCM_USE_AITER_CUSTOM_AR: bool = True VLLM_ROCM_USE_AITER_PAGED_ATTN: bool = False VLLM_ROCM_USE_AITER_LINEAR: bool = True VLLM_ROCM_USE_AITER_LINEAR_HIPBMM: bool = False VLLM_ROCM_USE_AITER_MOE: bool = True VLLM_ROCM_AITER_MOE_DISPATCH_POLICY: int = 0 VLLM_ROCM_USE_AITER_RMSNORM: bool = True VLLM_ROCM_USE_AITER_MLA: bool = True VLLM_ROCM_USE_AITER_MHA: bool = True VLLM_ROCM_USE_AITER_FP4_ASM_GEMM: bool = False VLLM_ROCM_USE_AITER_TRITON_ROPE: bool = False VLLM_ROCM_USE_AITER_FP8BMM: bool = True VLLM_ROCM_USE_AITER_FP4BMM: bool = True VLLM_ROCM_USE_AITER_UNIFIED_ATTENTION: bool = False VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS: bool = False VLLM_ROCM_USE_AITER_TRITON_GEMM: bool = True VLLM_ROCM_USE_SKINNY_GEMM: bool = True VLLM_ROCM_FP8_PADDING: bool = True VLLM_ROCM_MOE_PADDING: bool = True VLLM_ROCM_SHUFFLE_KV_CACHE_LAYOUT: bool = False VLLM_ENABLE_V1_MULTIPROCESSING: bool = True VLLM_LOG_BATCHSIZE_INTERVAL: float = -1 VLLM_DISABLE_COMPILE_CACHE: bool = False VLLM_USE_LAYERNAME: bool = True Q_SCALE_CONSTANT: int = 200 K_SCALE_CONSTANT: int = 200 V_SCALE_CONSTANT: int = 100 VLLM_USE_RUST_FRONTEND: bool = False VLLM_RUST_FRONTEND_PATH: str | None = "auto" VLLM_SERVER_DEV_MODE: bool = False VLLM_V1_OUTPUT_PROC_CHUNK_SIZE: int = 128 VLLM_MLA_DISABLE: bool = False VLLM_RAY_PER_WORKER_GPUS: float = 1.0 VLLM_RAY_BUNDLE_INDICES: str = "" VLLM_CUDART_SO_PATH: str | None = None VLLM_DP_RANK: int = 0 VLLM_DP_RANK_LOCAL: int = -1 VLLM_DP_SIZE: int = 1 VLLM_USE_STANDALONE_COMPILE: bool = True VLLM_ENABLE_PREGRAD_PASSES: bool = True VLLM_USE_BREAKABLE_CUDAGRAPH: bool = False VLLM_DP_MASTER_IP: str = "" VLLM_DP_MASTER_PORT: int = 0 VLLM_RANDOMIZE_DP_DUMMY_INPUTS: bool = False VLLM_RAY_DP_PACK_STRATEGY: Literal["strict", "fill", "span"] = "strict" VLLM_RAY_DP_PLACEMENT_NODE_IPS: str = "" VLLM_RAY_EXTRA_ENV_VAR_PREFIXES_TO_COPY: str = "" VLLM_RAY_EXTRA_ENV_VARS_TO_COPY: str = "" VLLM_MARLIN_USE_ATOMIC_ADD: bool = False VLLM_MARLIN_INPUT_DTYPE: Literal["int8", "fp8"] | None = None VLLM_HUMMING_ONLINE_QUANT_CONFIG: dict[str, Any] | None = None VLLM_HUMMING_INPUT_QUANT_CONFIG: dict[str, Any] | None = None VLLM_HUMMING_USE_F16_ACCUM: bool = False VLLM_HUMMING_MOE_GEMM_TYPE: Literal["indexed", "grouped", "auto"] | None = None VLLM_DEEPEPLL_NVFP4_DISPATCH: bool = False VLLM_V1_USE_OUTLINES_CACHE: bool = False VLLM_TPU_BUCKET_PADDING_GAP: int = 0 VLLM_TPU_MOST_MODEL_LEN: int | None = None VLLM_TPU_USING_PATHWAYS: bool = False VLLM_USE_DEEP_GEMM: bool = True VLLM_MOE_USE_DEEP_GEMM: bool = True VLLM_USE_DEEP_GEMM_E8M0: bool = True VLLM_USE_DEEP_GEMM_TMA_ALIGNED_SCALES: bool = True VLLM_DEEP_GEMM_WARMUP: Literal[ "skip", "full", "relax", ] = "relax" VLLM_USE_FUSED_MOE_GROUPED_TOPK: bool = True VLLM_MOE_SKIP_PADDING: bool = False VLLM_BLOCKSCALE_FP8_GEMM_FLASHINFER: bool = True VLLM_USE_FLASHINFER_MOE_INT4: bool = False VLLM_FLASHINFER_AUTOTUNE_CACHE_DIR: str | None = None VLLM_FLASHINFER_AUTOTUNE_SKIP_OPS: list[str] | None = None VLLM_FLASHINFER_ALLREDUCE_BACKEND: Literal["auto", "trtllm", "mnnvl"] = "auto" VLLM_FLASHINFER_WORKSPACE_BUFFER_SIZE: int = 394 * 1024 * 1024 VLLM_XGRAMMAR_CACHE_MB: int = 0 VLLM_REGEX_COMPILATION_TIMEOUT_S: int = 5 VLLM_MSGPACK_ZERO_COPY_THRESHOLD: int = 256 VLLM_ALLOW_INSECURE_SERIALIZATION: bool = False VLLM_DISABLE_REQUEST_ID_RANDOMIZATION: bool = False VLLM_NIXL_SIDE_CHANNEL_HOST: str = "localhost" VLLM_NIXL_SIDE_CHANNEL_PORT: int = 5600 VLLM_EC_SIDE_CHANNEL_HOST: str = "localhost" VLLM_EC_SIDE_CHANNEL_PORT: int = 5601 VLLM_MOONCAKE_BOOTSTRAP_PORT: int = 8998 VLLM_MOONCAKE_STORE_TIER_LOG: bool = False VLLM_MOONCAKE_LOAD_RECV_THREADS: int = 1 VLLM_MOONCAKE_DISK_STAGING_USABLE_RATIO: float = 0.9 MOONCAKE_PREFERRED_SEGMENT: str | None = None MOONCAKE_REQUESTER_LOCAL_HOSTNAME: str | None = None VLLM_MAX_TOKENS_PER_EXPERT_FP4_MOE: int = 163840 VLLM_TOOL_PARSE_REGEX_TIMEOUT_SECONDS: int = 1 VLLM_ENFORCE_STRICT_TOOL_CALLING: bool = True VLLM_MQ_MAX_CHUNK_BYTES_MB: int = 16 VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: int = 300 VLLM_WORKER_SHUTDOWN_TIMEOUT_SECONDS: int = 5 VLLM_KV_CACHE_LAYOUT: Literal["NHD", "HND"] | None = None VLLM_SSM_CONV_STATE_LAYOUT: Literal["SD", "DS"] | None = None VLLM_COMPUTE_NANS_IN_LOGITS: bool = False VLLM_ROCM_QUICK_REDUCE_QUANTIZATION: Literal[ "FP", "INT8", "INT6", "INT4", "INT3", "NONE" ] = "NONE" VLLM_ROCM_QUICK_REDUCE_CAST_BF16_TO_FP16: bool = True VLLM_ROCM_QUICK_REDUCE_MAX_SIZE_BYTES_MB: int | None = None VLLM_ROCM_QUICK_REDUCE_MIN_SIZE_BYTES_MB: int | None = None VLLM_ROCM_QUICK_REDUCE_QUANTIZATION_MIN_SIZE_KB: int | None = None VLLM_MOONCAKE_ABORT_REQUEST_TIMEOUT: int = 480 VLLM_ENABLE_CUDAGRAPH_GC: bool = False VLLM_LOOPBACK_IP: str = "" VLLM_ALLOW_CHUNKED_LOCAL_ATTN_WITH_HYBRID_KV_CACHE: bool = True VLLM_ENABLE_RESPONSES_API_STORE: bool = False VLLM_HAS_FLASHINFER_CUBIN: bool = False VLLM_ROCM_FP8_MFMA_PAGE_ATTN: bool = False VLLM_ALLREDUCE_USE_SYMM_MEM: bool = True VLLM_ALLREDUCE_USE_FLASHINFER: bool = False VLLM_TUNED_CONFIG_FOLDER: str | None = None VLLM_ENABLE_STARTUP_PLAN: bool = False VLLM_GPT_OSS_SYSTEM_TOOL_MCP_LABELS: set[str] = set() VLLM_USE_EXPERIMENTAL_PARSER_CONTEXT: bool = False VLLM_GPT_OSS_HARMONY_SYSTEM_INSTRUCTIONS: bool = False VLLM_SYSTEM_START_DATE: str | None = None VLLM_TOOL_JSON_ERROR_AUTOMATIC_RETRY: bool = False VLLM_CUSTOM_SCOPES_FOR_PROFILING: bool = False VLLM_NVTX_SCOPES_FOR_PROFILING: bool = False VLLM_KV_EVENTS_USE_INT_BLOCK_HASHES: bool = True VLLM_OBJECT_STORAGE_SHM_BUFFER_NAME: str = "VLLM_OBJECT_STORAGE_SHM_BUFFER" VLLM_DEEPEP_BUFFER_SIZE_MB: int = 1024 VLLM_DEEPEP_HIGH_THROUGHPUT_FORCE_INTRA_NODE: bool = False VLLM_DEEPEP_LOW_LATENCY_USE_MNNVL: bool = False VLLM_DEEPEP_V2_ALLOW_HYBRID_MODE: bool = True VLLM_DEEPEP_V2_PREFER_OVERLAP: bool = False VLLM_DEEPEP_V2_ALLOW_MULTIPLE_REDUCTION: bool = False VLLM_DBO_COMM_SMS: int = 20 VLLM_PATTERN_MATCH_DEBUG: str | None = None VLLM_DEBUG_DUMP_PATH: str | None = None VLLM_ENABLE_INDUCTOR_MAX_AUTOTUNE: bool = True VLLM_ENABLE_INDUCTOR_COORDINATE_DESCENT_TUNING: bool = True VLLM_USE_NCCL_SYMM_MEM: bool = False VLLM_NCCL_INCLUDE_PATH: str | None = None VLLM_GC_DEBUG: str = "" VLLM_DEBUG_WORKSPACE: bool = False VLLM_DISABLE_SHARED_EXPERTS_STREAM: bool = False VLLM_SHARED_EXPERTS_STREAM_TOKEN_THRESHOLD: int = 256 VLLM_MULTI_STREAM_GEMM_TOKEN_THRESHOLD: int = 1024 VLLM_COMPILE_CACHE_SAVE_FORMAT: Literal["binary", "unpacked"] = "binary" VLLM_USE_V2_MODEL_RUNNER: bool | None = None VLLM_LOG_MODEL_INSPECTION: bool = False VLLM_DEBUG_MFU_METRICS: bool = False VLLM_WEIGHT_OFFLOADING_DISABLE_PIN_MEMORY: bool = False VLLM_WEIGHT_OFFLOADING_DISABLE_UVA: bool = False VLLM_WSL2_ENABLE_PIN_MEMORY: bool = False VLLM_DISABLE_LOG_LOGO: bool = False VLLM_LORA_DISABLE_PDL: bool = False VLLM_ENABLE_CUDA_COMPATIBILITY: bool = False VLLM_CUDA_COMPATIBILITY_PATH: str | None = None VLLM_SKIP_MODEL_NAME_VALIDATION: bool = False """If set, vLLM will skip model name validation in API requests. This allows any model name to be accepted in the 'model' field of requests, making the server model-name agnostic. Useful for proxy/gateway scenarios.""" VLLM_ELASTIC_EP_SCALE_UP_LAUNCH: bool = False VLLM_ELASTIC_EP_DRAIN_REQUESTS: bool = False VLLM_MEMORY_PROFILER_ESTIMATE_CUDAGRAPHS: bool = True VLLM_NIXL_EP_MAX_NUM_RANKS: int = 32 VLLM_XPU_ENABLE_XPU_GRAPH: bool = False VLLM_XPU_USE_SAMPLER_KERNEL: bool = True VLLM_LORA_ENABLE_DUAL_STREAM: bool = False VLLM_GPU_NIC_PCIE_MAPPING: str = "" VLLM_NIC_SELECTION_VARS: str = "" VLLM_PREFIX_CACHE_RETENTION_INTERVAL: int | None = None def get_default_cache_root(): return os.getenv( "XDG_CACHE_HOME", os.path.join(os.path.expanduser("~"), ".cache"), ) def get_default_config_root(): return os.getenv( "XDG_CONFIG_HOME", os.path.join(os.path.expanduser("~"), ".config"), ) def maybe_convert_int(value: str | None) -> int | None: if value is None: return None return int(value) def maybe_convert_bool(value: str | None) -> bool | None: if value is None: return None return bool(int(value)) def maybe_convert_json_str_or_file(value: str | None) -> dict[str, Any] | None: if value is None: return None if os.path.exists(value): with open(value) as f: return json.load(f) return json.loads(value) def disable_compile_cache() -> bool: return bool(int(os.getenv("VLLM_DISABLE_COMPILE_CACHE", "0"))) def use_aot_compile() -> bool: from vllm.utils.torch_utils import is_torch_equal_or_newer default_value = ( "1" if is_torch_equal_or_newer("2.10.0") and not disable_compile_cache() else "0" ) return os.environ.get("VLLM_USE_AOT_COMPILE", default_value) == "1" def use_mega_aot_artifact(): from vllm.utils.torch_utils import is_torch_equal_or_newer default_value = ( "1" if is_torch_equal_or_newer("2.12.0.dev") and use_aot_compile() else "0" ) return os.environ.get("VLLM_USE_MEGA_AOT_ARTIFACT", default_value) == "1" def env_with_choices( env_name: str, default: str | None, choices: list[str] | Callable[[], list[str]], case_sensitive: bool = True, ) -> Callable[[], str | None]: """ Create a lambda that validates environment variable against allowed choices Args: env_name: Name of the environment variable default: Default value if not set (can be None) choices: List of valid string options or callable that returns list case_sensitive: Whether validation should be case sensitive Returns: Lambda function for environment_variables dict """ def _get_validated_env() -> str | None: value = os.getenv(env_name) if value is None: return default # Resolve choices if it's a callable (for lazy loading) actual_choices = choices() if callable(choices) else choices if not case_sensitive: check_value = value.lower() check_choices = [choice.lower() for choice in actual_choices] else: check_value = value check_choices = actual_choices if check_value not in check_choices: raise ValueError( f"Invalid value '{value}' for {env_name}. " f"Valid options: {actual_choices}." ) return value return _get_validated_env def env_list_with_choices( env_name: str, default: list[str], choices: list[str] | Callable[[], list[str]], case_sensitive: bool = True, ) -> Callable[[], list[str]]: """ Create a lambda that validates environment variable containing comma-separated values against allowed choices Args: env_name: Name of the environment variable default: Default list of values if not set choices: List of valid string options or callable that returns list case_sensitive: Whether validation should be case sensitive Returns: Lambda function for environment_variables dict that returns list of strings """ def _get_validated_env_list() -> list[str]: value = os.getenv(env_name) if value is None: return default # Split comma-separated values and strip whitespace values = [v.strip() for v in value.split(",") if v.strip()] if not values: return default # Resolve choices if it's a callable (for lazy loading) actual_choices = choices() if callable(choices) else choices # Validate each value for val in values: if not case_sensitive: check_value = val.lower() check_choices = [choice.lower() for choice in actual_choices] else: check_value = val check_choices = actual_choices if check_value not in check_choices: raise ValueError( f"Invalid value '{val}' in {env_name}. " f"Valid options: {actual_choices}." ) return values return _get_validated_env_list def env_set_with_choices( env_name: str, default: list[str], choices: list[str] | Callable[[], list[str]], case_sensitive: bool = True, ) -> Callable[[], set[str]]: """ Creates a lambda which that validates environment variable containing comma-separated values against allowed choices which returns choices as a set. """ def _get_validated_env_set() -> set[str]: return set(env_list_with_choices(env_name, default, choices, case_sensitive)()) return _get_validated_env_set def get_vllm_port() -> int | None: """Get the port from VLLM_PORT environment variable. Returns: The port number as an integer if VLLM_PORT is set, None otherwise. Raises: ValueError: If VLLM_PORT is a URI, suggest k8s service discovery issue. """ if "VLLM_PORT" not in os.environ: return None port = os.getenv("VLLM_PORT", "0") try: return int(port) except ValueError as err: from urllib3.util import parse_url parsed = parse_url(port) if parsed.scheme: raise ValueError( f"VLLM_PORT '{port}' appears to be a URI. " "This may be caused by a Kubernetes service discovery issue," "check the warning in: https://docs.vllm.ai/en/latest/configuration/env_vars.html" ) from None raise ValueError(f"VLLM_PORT '{port}' must be a valid integer") from err def get_env_or_set_default( env_name: str, default_factory: Callable[[], str], ) -> Callable[[], str]: """ Create a lambda that returns an environment variable value if set, or generates and sets a default value using the provided factory function. """ def _get_or_set_default() -> str: value = os.getenv(env_name) if value is not None: return value default_value = default_factory() os.environ[env_name] = default_value return default_value return _get_or_set_default # The start-* and end* here are used by the documentation generator # to extract the used env vars. # --8<-- [start:env-vars-definition] logger = logging.getLogger(__name__) def _resolve_rust_frontend_path() -> str | None: """Resolve the Rust frontend binary path. Returns None if VLLM_USE_RUST_FRONTEND is not enabled. When enabled, resolves VLLM_RUST_FRONTEND_PATH ("auto" by default) to the actual binary path. """ use_rust = bool(int(os.environ.get("VLLM_USE_RUST_FRONTEND", "0"))) raw = os.environ.get("VLLM_RUST_FRONTEND_PATH", "auto") if not use_rust: if os.environ.get("VLLM_RUST_FRONTEND_PATH") is not None: logger.warning( "VLLM_RUST_FRONTEND_PATH is set but VLLM_USE_RUST_FRONTEND " "is not enabled. The Rust frontend will not be used. " "Set VLLM_USE_RUST_FRONTEND=1 to enable it." ) return None if raw.lower() in ("auto", "1", "true"): pkg_dir = os.path.dirname(os.path.abspath(__file__)) candidate = os.path.join(pkg_dir, "vllm-rs") if os.path.isfile(candidate) and os.access(candidate, os.X_OK): return candidate raise FileNotFoundError( "VLLM_RUST_FRONTEND_PATH=auto but the vllm-rs binary was " f"not found at {candidate}. " "Build with setuptools-rust or set the path explicitly." ) return raw environment_variables: dict[str, Callable[[], Any]] = { # ================== Installation Time Env Vars ================== # Target device of vLLM, supporting [cuda (by default), # rocm, cpu] "VLLM_TARGET_DEVICE": lambda: os.getenv("VLLM_TARGET_DEVICE", "cuda").lower(), # Main CUDA version of vLLM. This follows PyTorch but can be overridden. "VLLM_MAIN_CUDA_VERSION": lambda: ( os.getenv("VLLM_MAIN_CUDA_VERSION", "").lower() or "13.0" ), # Controls PyTorch float32 matmul precision mode within vLLM workers. # Valid options mirror torch.set_float32_matmul_precision "VLLM_FLOAT32_MATMUL_PRECISION": env_with_choices( "VLLM_FLOAT32_MATMUL_PRECISION", "highest", ["highest", "high", "medium"], case_sensitive=False, ), # Enable batch-invariant mode: deterministic results regardless of # batch composition. Requires NVIDIA GPU with compute capability >= 9.0. "VLLM_BATCH_INVARIANT": lambda: bool(int(os.getenv("VLLM_BATCH_INVARIANT", "0"))), # Use tensor descriptors for Q/K/V loads and output stores in the # Triton unified-attention kernel. Enables HW 2D block reads on # Intel Xe2/Xe3; the non-TD branch is dead-code-eliminated at Triton # compile time so other platforms see no overhead. Tri-state override: # unset (default) lets the `triton_attn` backend auto-select per # platform (currently auto-enabled on XPU only); ``1`` forces TD on; # ``0`` forces TD off. Useful for A/B benchmarking the TD path. "VLLM_TRITON_ATTN_USE_TD": lambda: {"1": True, "0": False}.get( os.getenv("VLLM_TRITON_ATTN_USE_TD", "").strip() ), # If set, enable PyTorch's GPU<->CPU synchronization debug mode around # the worker's `execute_model` and `sample_tokens` calls. Valid values # are "warn" (print a warning on each sync) or "error" (raise on sync). # Unset disables the check. See `torch.cuda.set_sync_debug_mode`. "VLLM_GPU_SYNC_CHECK": env_with_choices( "VLLM_GPU_SYNC_CHECK", None, ["warn", "error"] ), # Maximum number of compilation jobs to run in parallel. # By default this is the number of CPUs "MAX_JOBS": lambda: os.getenv("MAX_JOBS", None), # Number of threads to use for nvcc # By default this is 1. # If set, `MAX_JOBS` will be reduced to avoid oversubscribing the CPU. "NVCC_THREADS": lambda: os.getenv("NVCC_THREADS", None), # If set, vllm will use precompiled native binaries (*.so and vllm-rs). "VLLM_USE_PRECOMPILED": lambda: ( os.environ.get("VLLM_USE_PRECOMPILED", "").strip().lower() in ("1", "true") or bool(os.environ.get("VLLM_PRECOMPILED_WHEEL_LOCATION")) ), # If set, vllm will use the precompiled Rust frontend binary (vllm-rs). "VLLM_USE_PRECOMPILED_RUST": lambda: ( os.environ.get("VLLM_USE_PRECOMPILED_RUST", "").strip().lower() in ("1", "true") ), # If set, skip adding +precompiled suffix to version string "VLLM_SKIP_PRECOMPILED_VERSION_SUFFIX": lambda: bool( int(os.environ.get("VLLM_SKIP_PRECOMPILED_VERSION_SUFFIX", "0")) ), # Used to mark that setup.py is running in a Docker build context, # in order to force the use of precompiled binaries. "VLLM_DOCKER_BUILD_CONTEXT": lambda: ( os.environ.get("VLLM_DOCKER_BUILD_CONTEXT", "").strip().lower() in ("1", "true") ), # Build provenance metadata embedded in official vllm-openai images. # Set via Docker ENV at image build time; informational only. "VLLM_BUILD_COMMIT": lambda: os.environ.get("VLLM_BUILD_COMMIT", "unknown"), "VLLM_BUILD_PIPELINE": lambda: os.environ.get("VLLM_BUILD_PIPELINE", "local"), "VLLM_BUILD_URL": lambda: os.environ.get("VLLM_BUILD_URL", ""), "VLLM_IMAGE_TAG": lambda: os.environ.get("VLLM_IMAGE_TAG", ""), # CMake build type # If not set, defaults to "Debug" or "RelWithDebInfo" # Available options: "Debug", "Release", "RelWithDebInfo" "CMAKE_BUILD_TYPE": env_with_choices( "CMAKE_BUILD_TYPE", None, ["Debug", "Release", "RelWithDebInfo"] ), # If set, vllm will print verbose logs during installation "VERBOSE": lambda: bool(int(os.getenv("VERBOSE", "0"))), # Root directory for vLLM configuration files # Defaults to `~/.config/vllm` unless `XDG_CONFIG_HOME` is set # Note that this not only affects how vllm finds its configuration files # during runtime, but also affects how vllm installs its configuration # files during **installation**. "VLLM_CONFIG_ROOT": lambda: os.path.expanduser( os.getenv( "VLLM_CONFIG_ROOT", os.path.join(get_default_config_root(), "vllm"), ) ), # ================== Runtime Env Vars ================== # Root directory for vLLM cache files # Defaults to `~/.cache/vllm` unless `XDG_CACHE_HOME` is set "VLLM_CACHE_ROOT": lambda: os.path.expanduser( os.getenv( "VLLM_CACHE_ROOT", os.path.join(get_default_cache_root(), "vllm"), ) ), # used in distributed environment to determine the ip address # of the current node, when the node has multiple network interfaces. # If you are using multi-node inference, you should set this differently # on each node. "VLLM_HOST_IP": lambda: os.getenv("VLLM_HOST_IP", ""), # used in distributed environment to manually set the communication port # Note: if VLLM_PORT is set, and some code asks for multiple ports, the # VLLM_PORT will be used as the first port, and the rest will be generated # by incrementing the VLLM_PORT value. "VLLM_PORT": get_vllm_port, # path used for ipc when the frontend api server is running in # multi-processing mode to communicate with the backend engine process. "VLLM_RPC_BASE_PATH": lambda: os.getenv( "VLLM_RPC_BASE_PATH", tempfile.gettempdir() ), # If true, will load models from ModelScope instead of Hugging Face Hub. # note that the value is true or false, not numbers "VLLM_USE_MODELSCOPE": lambda: ( os.environ.get("VLLM_USE_MODELSCOPE", "False").lower() == "true" ), # If true, replace the Rust BPE backend that powers HF fast tokenizers # with the `fastokens` (https://github.com/crusoecloud/fastokens) shim. # Applies to any tokenizer mode that loads an HF fast tokenizer # (`hf`, `deepseek_v32`, `deepseek_v4`, …). The `fastokens` # Python package must be installed. "VLLM_USE_FASTOKENS": lambda: bool(int(os.getenv("VLLM_USE_FASTOKENS", "0"))), # Interval in seconds to log a warning message when the ring buffer is full "VLLM_RINGBUFFER_WARNING_INTERVAL": lambda: int( os.environ.get("VLLM_RINGBUFFER_WARNING_INTERVAL", "60") ), # path to cudatoolkit home directory, under which should be bin, include, # and lib directories. "CUDA_HOME": lambda: os.environ.get("CUDA_HOME", None), # Path to the NCCL library file. It is needed because nccl>=2.19 brought # by PyTorch contains a bug: https://github.com/NVIDIA/nccl/issues/1234 "VLLM_NCCL_SO_PATH": lambda: os.environ.get("VLLM_NCCL_SO_PATH", None), # when `VLLM_NCCL_SO_PATH` is not set, vllm will try to find the nccl # library file in the locations specified by `LD_LIBRARY_PATH` "LD_LIBRARY_PATH": lambda: os.environ.get("LD_LIBRARY_PATH", None), # flag to control the chunk size (in MB) for sleeping memory allocations under ROCm "VLLM_ROCM_SLEEP_MEM_CHUNK_SIZE": lambda: int( os.environ.get("VLLM_ROCM_SLEEP_MEM_CHUNK_SIZE", "256") ), # Feature flag to enable/disable Inductor standalone compile. # In torch <= 2.7 we ignore this flag; in torch >= 2.9 this is # enabled by default. "VLLM_USE_STANDALONE_COMPILE": lambda: ( os.environ.get("VLLM_USE_STANDALONE_COMPILE", "1") == "1" ), # Inductor's pre-grad passes don't do anything for vLLM. # The pre-grad passes get run even on cache-hit and negatively impact # vllm cold compile times by O(1s) # Can remove this after the following issue gets fixed # TODO(luka): maybe_inplace requires this # https://github.com/pytorch/pytorch/issues/174502 "VLLM_ENABLE_PREGRAD_PASSES": lambda: ( os.environ.get("VLLM_ENABLE_PREGRAD_PASSES", "1") == "1" ), # Experimental: breakable cudagraph does not rely on torch.compile "VLLM_USE_BREAKABLE_CUDAGRAPH": lambda: ( os.environ.get("VLLM_USE_BREAKABLE_CUDAGRAPH", "0") == "1" ), # Debug pattern matching inside custom passes. # Should be set to the fx.Node name (e.g. 'getitem_34' or 'scaled_mm_3'). "VLLM_PATTERN_MATCH_DEBUG": lambda: os.environ.get( "VLLM_PATTERN_MATCH_DEBUG", None ), # Dump fx graphs to the given directory. # It will override CompilationConfig.debug_dump_path if set. "VLLM_DEBUG_DUMP_PATH": lambda: os.environ.get("VLLM_DEBUG_DUMP_PATH", None), # Feature flag to enable/disable AOT compilation. This will ensure # compilation is done in warmup phase and the compilation will be # reused in subsequent calls. "VLLM_USE_AOT_COMPILE": use_aot_compile, # Feature flag to enable/disable bytecode in # TorchCompileWithNoGuardsWrapper. "VLLM_USE_BYTECODE_HOOK": lambda: bool( int(os.environ.get("VLLM_USE_BYTECODE_HOOK", "1")) ), # Force vllm to always load AOT compiled models from disk. Failure # to load will result in a hard error when this is enabled. # Will be ignored when VLLM_USE_AOT_COMPILE is disabled. "VLLM_FORCE_AOT_LOAD": lambda: os.environ.get("VLLM_FORCE_AOT_LOAD", "0") == "1", # Enable loading compiled models directly from cached standalone compile artifacts # without re-splitting graph modules. This reduces overhead during model # loading by using reconstruct_serializable_fn_from_mega_artifact. "VLLM_USE_MEGA_AOT_ARTIFACT": use_mega_aot_artifact, # local rank of the process in the distributed setting, used to determine # the GPU device id "LOCAL_RANK": lambda: int(os.environ.get("LOCAL_RANK", "0")), # used to control the visible devices in the distributed setting "CUDA_VISIBLE_DEVICES": lambda: os.environ.get("CUDA_VISIBLE_DEVICES", None), # timeout for each iteration in the engine "VLLM_ENGINE_ITERATION_TIMEOUT_S": lambda: int( os.environ.get("VLLM_ENGINE_ITERATION_TIMEOUT_S", "60") ), # Timeout in seconds for waiting for engine cores to become ready # during startup. Default is 600 seconds (10 minutes). "VLLM_ENGINE_READY_TIMEOUT_S": lambda: int( os.environ.get("VLLM_ENGINE_READY_TIMEOUT_S", "600") ), # API key for vLLM API server "VLLM_API_KEY": lambda: os.environ.get("VLLM_API_KEY", None), # Whether to log responses from API Server for debugging "VLLM_DEBUG_LOG_API_SERVER_RESPONSE": lambda: ( os.environ.get("VLLM_DEBUG_LOG_API_SERVER_RESPONSE", "False").lower() == "true" ), # S3 access information, used for tensorizer to load model from S3 "S3_ACCESS_KEY_ID": lambda: os.environ.get("S3_ACCESS_KEY_ID", None), "S3_SECRET_ACCESS_KEY": lambda: os.environ.get("S3_SECRET_ACCESS_KEY", None), "S3_ENDPOINT_URL": lambda: os.environ.get("S3_ENDPOINT_URL", None), # Usage stats collection "VLLM_USAGE_STATS_SERVER": lambda: os.environ.get( "VLLM_USAGE_STATS_SERVER", "https://stats.vllm.ai" ), "VLLM_NO_USAGE_STATS": lambda: os.environ.get("VLLM_NO_USAGE_STATS", "0") == "1", "VLLM_DO_NOT_TRACK": lambda: ( ( os.environ.get("VLLM_DO_NOT_TRACK", None) or os.environ.get("DO_NOT_TRACK", None) or "0" ) == "1" ), "VLLM_USAGE_SOURCE": lambda: os.environ.get("VLLM_USAGE_SOURCE", "production"), # Logging configuration # If set to 0, vllm will not configure logging # If set to 1, vllm will configure logging using the default configuration # or the configuration file specified by VLLM_LOGGING_CONFIG_PATH "VLLM_CONFIGURE_LOGGING": lambda: bool( int(os.getenv("VLLM_CONFIGURE_LOGGING", "1")) ), "VLLM_LOGGING_CONFIG_PATH": lambda: os.getenv("VLLM_LOGGING_CONFIG_PATH"), # this is used for configuring the default logging level "VLLM_LOGGING_LEVEL": lambda: os.getenv("VLLM_LOGGING_LEVEL", "INFO").upper(), # this is used for configuring the default logging stream "VLLM_LOGGING_STREAM": lambda: os.getenv("VLLM_LOGGING_STREAM", "ext://sys.stdout"), # if set, VLLM_LOGGING_PREFIX will be prepended to all log messages "VLLM_LOGGING_PREFIX": lambda: os.getenv("VLLM_LOGGING_PREFIX", ""), # Controls colored logging output. Options: "auto" (default, colors when terminal), # "1" (always use colors), "0" (never use colors) "VLLM_LOGGING_COLOR": lambda: os.getenv("VLLM_LOGGING_COLOR", "auto"), # Standard unix flag for disabling ANSI color codes "NO_COLOR": lambda: os.getenv("NO_COLOR", "0") != "0", # If set, vllm will log stats at this interval in seconds # If not set, vllm will log stats every 10 seconds. "VLLM_LOG_STATS_INTERVAL": lambda: ( val if (val := float(os.getenv("VLLM_LOG_STATS_INTERVAL", "10."))) > 0.0 else 10.0 ), # Trace function calls # If set to 1, vllm will trace function calls # Useful for debugging "VLLM_TRACE_FUNCTION": lambda: int(os.getenv("VLLM_TRACE_FUNCTION", "0")), # Whether to use the FlashInfer top-k / top-p sampler on CUDA. Enabled # by default when the hardware supports it — set to 0 to opt out # explicitly, which forces the PyTorch-native (Triton for bs>=8) path. "VLLM_USE_FLASHINFER_SAMPLER": lambda: ( bool(int(os.environ["VLLM_USE_FLASHINFER_SAMPLER"])) if "VLLM_USE_FLASHINFER_SAMPLER" in os.environ else True ), # Pipeline stage partition strategy "VLLM_PP_LAYER_PARTITION": lambda: os.getenv("VLLM_PP_LAYER_PARTITION", None), # (CPU backend only) CPU key-value cache space. # default is None and will be set as 4 GB "VLLM_CPU_KVCACHE_SPACE": lambda: ( int(os.getenv("VLLM_CPU_KVCACHE_SPACE", "0")) if "VLLM_CPU_KVCACHE_SPACE" in os.environ else None ), # (CPU backend only) CPU core ids bound by OpenMP threads, e.g., "0-31", # "0,1,2", "0-31,33". CPU cores of different ranks are separated by '|'. "VLLM_CPU_OMP_THREADS_BIND": lambda: os.getenv("VLLM_CPU_OMP_THREADS_BIND", "auto"), # (CPU backend only) CPU cores not used by OMP threads . # Those CPU cores will not be used by OMP threads of a rank. "VLLM_CPU_NUM_OF_RESERVED_CPU": lambda: ( int(os.getenv("VLLM_CPU_NUM_OF_RESERVED_CPU", "0")) if "VLLM_CPU_NUM_OF_RESERVED_CPU" in os.environ else None ), # (CPU backend only) whether to use SGL kernels, optimized for small batch. "VLLM_CPU_SGL_KERNEL": lambda: bool(int(os.getenv("VLLM_CPU_SGL_KERNEL", "0"))), # (CPU backend only) whether to enable attention spilt KV. "VLLM_CPU_ATTN_SPLIT_KV": lambda: bool( int(os.getenv("VLLM_CPU_ATTN_SPLIT_KV", "1")) ), # (Zen CPU backend) eagerly prepack weights into ZenDNN blocked layout # at model load time. Eliminates per-inference layout conversion overhead. "VLLM_ZENTORCH_WEIGHT_PREPACK": lambda: bool( int(os.getenv("VLLM_ZENTORCH_WEIGHT_PREPACK", "1")) ), # (CPU backend only) whether to use SGLang INT4 W4A8 kernels for AWQ. "VLLM_CPU_INT4_W4A8": lambda: bool(int(os.getenv("VLLM_CPU_INT4_W4A8", "1"))), # If the env var is set, Ray Compiled Graph uses the specified # channel type to communicate between workers belonging to # different pipeline-parallel stages. # Available options: # - "auto": use the default channel type # - "nccl": use NCCL for communication # - "shm": use shared memory and gRPC for communication "VLLM_USE_RAY_COMPILED_DAG_CHANNEL_TYPE": env_with_choices( "VLLM_USE_RAY_COMPILED_DAG_CHANNEL_TYPE", "auto", ["auto", "nccl", "shm"] ), # If the env var is set, it enables GPU communication overlap # (experimental feature) in Ray's Compiled Graph. "VLLM_USE_RAY_COMPILED_DAG_OVERLAP_COMM": lambda: bool( int(os.getenv("VLLM_USE_RAY_COMPILED_DAG_OVERLAP_COMM", "0")) ), # If the env var is set, it uses a Ray Communicator wrapping # vLLM's pipeline parallelism communicator to interact with Ray's # Compiled Graph. Otherwise, it uses Ray's NCCL communicator. "VLLM_USE_RAY_WRAPPED_PP_COMM": lambda: bool( int(os.getenv("VLLM_USE_RAY_WRAPPED_PP_COMM", "1")) ), # When True and distributed_executor_backend="ray", use RayExecutorV2 # (MQ-based) instead of RayDistributedExecutor (compiled-graph backend). "VLLM_USE_RAY_V2_EXECUTOR_BACKEND": lambda: bool( int(os.getenv("VLLM_USE_RAY_V2_EXECUTOR_BACKEND", "1")) ), # When True, GroupCoordinator constructs its CPU/device subgroups via # ``torch.distributed.split_group(backend=...)`` # and ``init_distributed_environment`` initializes the default PG with # mixed ``cpu:gloo,cuda:nccl`` backend + eager ``device_id`` binding. "VLLM_DISTRIBUTED_USE_SPLIT_GROUP": lambda: bool( int(os.getenv("VLLM_DISTRIBUTED_USE_SPLIT_GROUP", "0")) ), # Use dedicated multiprocess context for workers. # Both spawn and fork work "VLLM_WORKER_MULTIPROC_METHOD": env_with_choices( "VLLM_WORKER_MULTIPROC_METHOD", "fork", ["spawn", "fork"] ), # Path to the cache for storing downloaded assets "VLLM_ASSETS_CACHE": lambda: os.path.expanduser( os.getenv( "VLLM_ASSETS_CACHE", os.path.join(get_default_cache_root(), "vllm", "assets"), ) ), # If the env var is set, we will clean model file in # this path $VLLM_ASSETS_CACHE/model_streamer/$model_name "VLLM_ASSETS_CACHE_MODEL_CLEAN": lambda: bool( int(os.getenv("VLLM_ASSETS_CACHE_MODEL_CLEAN", "0")) ), # Timeout for fetching images when serving multimodal models # Default is 5 seconds "VLLM_IMAGE_FETCH_TIMEOUT": lambda: int(os.getenv("VLLM_IMAGE_FETCH_TIMEOUT", "5")), # Timeout for fetching videos when serving multimodal models # Default is 30 seconds "VLLM_VIDEO_FETCH_TIMEOUT": lambda: int( os.getenv("VLLM_VIDEO_FETCH_TIMEOUT", "30") ), # Timeout for fetching audio when serving multimodal models # Default is 10 seconds "VLLM_AUDIO_FETCH_TIMEOUT": lambda: int( os.getenv("VLLM_AUDIO_FETCH_TIMEOUT", "10") ), # Directory for caching media downloads (images, video, audio fetched # from URLs during inference). Empty string disables caching. "VLLM_MEDIA_CACHE": lambda: os.getenv("VLLM_MEDIA_CACHE", ""), # Maximum cache size in MB. When exceeded, least-recently-used entries # are evicted. Default is 5120 (5 GB). "VLLM_MEDIA_CACHE_MAX_SIZE_MB": lambda: int( os.getenv("VLLM_MEDIA_CACHE_MAX_SIZE_MB", "5120") ), # Time-to-live in hours for cached media files. Entries older than this # are evicted regardless of cache size. Default is 24 hours. "VLLM_MEDIA_CACHE_TTL_HOURS": lambda: float( os.getenv("VLLM_MEDIA_CACHE_TTL_HOURS", "24") ), # Maximum number of retries for fetching media (images, audio, video) # from URLs. Each retry quadruples the timeout. Default is 3. "VLLM_MEDIA_FETCH_MAX_RETRIES": lambda: int( os.getenv("VLLM_MEDIA_FETCH_MAX_RETRIES", "3") ), # Whether to allow HTTP redirects when fetching from media URLs. # Default to True "VLLM_MEDIA_URL_ALLOW_REDIRECTS": lambda: bool( int(os.getenv("VLLM_MEDIA_URL_ALLOW_REDIRECTS", "1")) ), # Max number of workers for the thread pool handling # media bytes loading. Set to 1 to disable parallel processing. # Default is 8 "VLLM_MEDIA_LOADING_THREAD_COUNT": lambda: int( os.getenv("VLLM_MEDIA_LOADING_THREAD_COUNT", "8") ), # Maximum filesize in MB for a single audio file when processing # speech-to-text requests. Files larger than this will be rejected. # Default is 25 MB "VLLM_MAX_AUDIO_CLIP_FILESIZE_MB": lambda: int( os.getenv("VLLM_MAX_AUDIO_CLIP_FILESIZE_MB", "25") ), # Maximum decoded audio duration in seconds. Compressed audio files # (e.g. OPUS at very low bitrate) can expand into gigabytes of float32 # PCM. This limit is enforced *during* decoding so the memory is never # allocated. Default is 600s (10 minutes). "VLLM_MAX_AUDIO_DECODE_DURATION_S": lambda: int( os.getenv("VLLM_MAX_AUDIO_DECODE_DURATION_S", "600") ), # Maximum number of worker threads used for STT preprocessing. The default # intentionally caps at 2 because that performed best in profiling. # https://github.com/vllm-project/vllm/pull/44612#issuecomment-4662757781 "VLLM_MAX_AUDIO_PREPROCESS_WORKERS": lambda: int( os.getenv( "VLLM_MAX_AUDIO_PREPROCESS_WORKERS", str(max(1, min(os.cpu_count() or 1, 2))), ) ), # Maximum decoded image size in pixels. Small compressed images can # expand into gigabytes of raster memory. This limit is enforced before # decoding so the memory is never allocated. Default matches PIL's # built-in 2x decompression-bomb threshold (~179M pixels, ~680 MB RGB). "VLLM_MAX_IMAGE_PIXELS": lambda: int( os.getenv("VLLM_MAX_IMAGE_PIXELS", "178956970") ), # Backend for Video IO — selects the frame-sampling algorithm. # - "opencv": uniform sampling. # - "opencv_dynamic": duration-aware dynamic sampling. # - "identity": returns raw video bytes for model processor to handle. # # Custom backend implementations can be registered # via `@VIDEO_LOADER_REGISTRY.register("my_custom_video_loader")` and # imported at runtime. # If a non-existing backend is used, an AssertionError will be thrown. "VLLM_VIDEO_LOADER_BACKEND": lambda: os.getenv( "VLLM_VIDEO_LOADER_BACKEND", "opencv" ), # Media connector implementation. # - "http": Default connector that supports fetching media via HTTP. # # Custom implementations can be registered # via `@MEDIA_CONNECTOR_REGISTRY.register("my_custom_media_connector")` and # imported at runtime. # If a non-existing backend is used, an AssertionError will be thrown. "VLLM_MEDIA_CONNECTOR": lambda: os.getenv("VLLM_MEDIA_CONNECTOR", "http"), # Hash algorithm for multimodal content hashing. # - "blake3": Default, fast cryptographic hash (not FIPS 140-3 compliant) # - "sha256": FIPS 140-3 compliant, widely supported # - "sha512": FIPS 140-3 compliant, faster on 64-bit systems # Use sha256 or sha512 for FIPS compliance in government/enterprise deployments "VLLM_MM_HASHER_ALGORITHM": env_with_choices( "VLLM_MM_HASHER_ALGORITHM", "blake3", ["blake3", "sha256", "sha512"], case_sensitive=False, ), # Path to the XLA persistent cache directory. # Only used for XLA devices such as TPUs. "VLLM_XLA_CACHE_PATH": lambda: os.path.expanduser( os.getenv( "VLLM_XLA_CACHE_PATH", os.path.join(get_default_cache_root(), "vllm", "xla_cache"), ) ), # If set, assert on XLA recompilation after each execution step. "VLLM_XLA_CHECK_RECOMPILATION": lambda: bool( int(os.getenv("VLLM_XLA_CHECK_RECOMPILATION", "0")) ), # Enable SPMD mode for TPU backend. "VLLM_XLA_USE_SPMD": lambda: bool(int(os.getenv("VLLM_XLA_USE_SPMD", "0"))), # Maximum size (in MB) for logits tensor in sparse MLA indexer prefill chunks. # Bounds the [M, N] float32 logits tensor to prevent CUDA OOM. # Default: 512 MB "VLLM_SPARSE_INDEXER_MAX_LOGITS_MB": lambda: int( os.getenv("VLLM_SPARSE_INDEXER_MAX_LOGITS_MB", "512") ), # If set, the OpenAI API server will stay alive even after the underlying # AsyncLLMEngine errors and stops serving requests "VLLM_KEEP_ALIVE_ON_ENGINE_DEATH": lambda: bool( int(os.getenv("VLLM_KEEP_ALIVE_ON_ENGINE_DEATH", "0")) ), # If the env var VLLM_ALLOW_LONG_MAX_MODEL_LEN is set, it allows # the user to specify a max sequence length greater than # the max length derived from the model's config.json. # To enable this, set VLLM_ALLOW_LONG_MAX_MODEL_LEN=1. "VLLM_ALLOW_LONG_MAX_MODEL_LEN": lambda: ( os.environ.get("VLLM_ALLOW_LONG_MAX_MODEL_LEN", "0").strip().lower() in ("1", "true") ), # If set, forces FP8 Marlin to be used for FP8 quantization regardless # of the hardware support for FP8 compute. "VLLM_TEST_FORCE_FP8_MARLIN": lambda: ( os.environ.get("VLLM_TEST_FORCE_FP8_MARLIN", "0").strip().lower() in ("1", "true") ), "VLLM_TEST_FORCE_LOAD_FORMAT": lambda: os.getenv( "VLLM_TEST_FORCE_LOAD_FORMAT", "dummy" ), # Queue size for fastsafetensors ParallelLoader pipelined weight # loading. Peak load-time VRAM is roughly # model_weights + (1 + queue_size) * shard_size. # Default 0 preserves the non-pipelined memory footprint so this # change does not shrink the loadable-model envelope. Set to 1 # (or higher) to overlap producing the next shard's device buffer # with the consumer copying the current shard into model params, # at the cost of `queue_size` extra shard-sized buffers resident # at peak during loading. "VLLM_FASTSAFETENSORS_QUEUE_SIZE": lambda: int( os.getenv("VLLM_FASTSAFETENSORS_QUEUE_SIZE", "0") ), # Timeout in seconds for keeping HTTP connections alive in API server "VLLM_HTTP_TIMEOUT_KEEP_ALIVE": lambda: int( os.environ.get("VLLM_HTTP_TIMEOUT_KEEP_ALIVE", "5") ), # Maximum allowed value for the `n` sampling parameter (number of output # sequences per request). Limits resource consumption to prevent # denial-of-service via excessively large fan-out. Default: 16384. "VLLM_MAX_N_SEQUENCES": lambda: int( os.environ.get("VLLM_MAX_N_SEQUENCES", "16384") ), # Maximum number of prompts allowed in a single /v1/completions request # when the prompt field is a list. Prevents unbounded fan-out of engine # requests from a single API call. Default: 1024. "VLLM_MAX_COMPLETION_PROMPTS": lambda: int( os.environ.get("VLLM_MAX_COMPLETION_PROMPTS", "1024") ), # a list of plugin names to load, separated by commas. # if this is not set, it means all plugins will be loaded # if this is set to an empty string, no plugins will be loaded "VLLM_PLUGINS": lambda: ( None if "VLLM_PLUGINS" not in os.environ else os.environ["VLLM_PLUGINS"].split(",") ), # Retain local sliding-window KV checkpoints for prefix caching. # Unset (default) preserves the dense local checkpointing behavior. `0` # retains only the latest completed prompt boundary. Positive values retain # checkpoints at the specified interval boundaries (rounded up to the # prefix-cache alignment). # Applies to sliding-window attention for now but not yet Mamba/linear attention. "VLLM_PREFIX_CACHE_RETENTION_INTERVAL": lambda: ( int(os.environ["VLLM_PREFIX_CACHE_RETENTION_INTERVAL"]) if "VLLM_PREFIX_CACHE_RETENTION_INTERVAL" in os.environ else None ), # a local directory to look in for unrecognized LoRA adapters. # only works if plugins are enabled and # VLLM_ALLOW_RUNTIME_LORA_UPDATING is enabled. "VLLM_LORA_RESOLVER_CACHE_DIR": lambda: os.getenv( "VLLM_LORA_RESOLVER_CACHE_DIR", None ), # A remote HF repo(s) containing one or more LoRA adapters, which # may be downloaded and leveraged as needed. Only works if plugins # are enabled and VLLM_ALLOW_RUNTIME_LORA_UPDATING is enabled. # Values should be comma separated. "VLLM_LORA_RESOLVER_HF_REPO_LIST": lambda: os.getenv( "VLLM_LORA_RESOLVER_HF_REPO_LIST", None ), # If set, vLLM will use Triton implementations of AWQ. "VLLM_USE_TRITON_AWQ": lambda: bool(int(os.getenv("VLLM_USE_TRITON_AWQ", "0"))), # If set, monkey-patch triton.runtime.autotuner.Autotuner.run to skip # benchmarking and select the first valid config (walking past invalid # ones). Used to eliminate autotuning variability when measuring kernel # performance and applied before running any kernel. "VLLM_TRITON_FORCE_FIRST_CONFIG": lambda: ( os.environ.get("VLLM_TRITON_FORCE_FIRST_CONFIG", "0").strip().lower() in ("1", "true") ), # If set, allow loading or unloading lora adapters in runtime, "VLLM_ALLOW_RUNTIME_LORA_UPDATING": lambda: ( os.environ.get("VLLM_ALLOW_RUNTIME_LORA_UPDATING", "0").strip().lower() in ("1", "true") ), # We assume drivers can report p2p status correctly. # If the program hangs when using custom allreduce, # potantially caused by a bug in the driver (535 series), # if might be helpful to set VLLM_SKIP_P2P_CHECK=0 # so that vLLM can verify if p2p is actually working. # See https://github.com/vllm-project/vllm/blob/a9b15c606fea67a072416ea0ea115261a2756058/vllm/distributed/device_communicators/custom_all_reduce_utils.py#L101-L108 for details. # noqa "VLLM_SKIP_P2P_CHECK": lambda: os.getenv("VLLM_SKIP_P2P_CHECK", "1") == "1", # List of quantization kernels that should be disabled, used for testing # and performance comparisons. Currently only affects MPLinearKernel # selection # (kernels: MacheteLinearKernel, MarlinLinearKernel, ExllamaLinearKernel) "VLLM_DISABLED_KERNELS": lambda: ( [] if "VLLM_DISABLED_KERNELS" not in os.environ else os.environ["VLLM_DISABLED_KERNELS"].split(",") ), "VLLM_ENABLE_FLA_PACKED_RECURRENT_DECODE": lambda: bool( int(os.getenv("VLLM_ENABLE_FLA_PACKED_RECURRENT_DECODE", "1")) ), # Disable pynccl (using torch.distributed instead) "VLLM_DISABLE_PYNCCL": lambda: ( os.getenv("VLLM_DISABLE_PYNCCL", "False").lower() in ("true", "1") ), # Optional: enable external Oink custom ops (e.g., Blackwell RMSNorm). # Disabled by default. "VLLM_USE_OINK_OPS": lambda: ( os.getenv("VLLM_USE_OINK_OPS", "False").lower() in ("true", "1") ), # Disable aiter ops unless specifically enabled. # Acts as a parent switch to enable the rest of the other operations. # On hardware without a native MXFP8 kernel (e.g. ROCm gfx942 / MI300), the # MXFP8 emulation path dequantizes weights MXFP8->BF16 once at load time and # runs as a BF16 checkpoint (no per-step dequant). Set to 0 to fall back to # per-step dequant: keeps the 1-byte MXFP8 weights (~half the weight memory) # at the cost of dequantizing every forward step (much slower). Default on. "VLLM_MXFP8_EMULATION_DEQUANT_AT_LOAD": lambda: ( os.getenv("VLLM_MXFP8_EMULATION_DEQUANT_AT_LOAD", "True").lower() in ("true", "1") ), "VLLM_ROCM_USE_AITER": lambda: ( os.getenv("VLLM_ROCM_USE_AITER", "False").lower() in ("true", "1") ), # Use AITER's CustomAllreduce as the custom-allreduce backend inside vLLM's # CudaCommunicator on ROCm. "VLLM_ROCM_USE_AITER_CUSTOM_AR": lambda: ( os.getenv("VLLM_ROCM_USE_AITER_CUSTOM_AR", "True").lower() in ("true", "1") ), # Whether to use aiter paged attention. # By default is disabled. "VLLM_ROCM_USE_AITER_PAGED_ATTN": lambda: ( os.getenv("VLLM_ROCM_USE_AITER_PAGED_ATTN", "False").lower() in ("true", "1") ), # use aiter linear op if aiter ops are enabled # The following list of related ops # - scaled_mm (per-tensor / rowwise) # - use aiter tuned gemms for unquantized gemms "VLLM_ROCM_USE_AITER_LINEAR": lambda: ( os.getenv("VLLM_ROCM_USE_AITER_LINEAR", "True").lower() in ("true", "1") ), "VLLM_ROCM_USE_AITER_LINEAR_HIPBMM": lambda: ( os.getenv("VLLM_ROCM_USE_AITER_LINEAR_HIPBMM", "False").lower() in ("true", "1") ), # Whether to use aiter moe ops. # By default is enabled. "VLLM_ROCM_USE_AITER_MOE": lambda: ( os.getenv("VLLM_ROCM_USE_AITER_MOE", "True").lower() in ("true", "1") ), # MoE sorting dispatch policy for AITER fused MoE kernels. # 0 = auto (default): single-pass for small batches, multi-pass # for large batches # 1 = always single-pass: one kernel launch, no workspace, # may be preferred for low-concurrency decode workloads # 2 = always multi-pass: can be faster for MoE-heavy models # (e.g., +2-5% on Qwen3-Next, +1.5% on DeepSeek-V3 at TP4, # see PR #39177 for benchmarks) "VLLM_ROCM_AITER_MOE_DISPATCH_POLICY": lambda: int( os.getenv("VLLM_ROCM_AITER_MOE_DISPATCH_POLICY", "0") ), # use aiter rms norm op if aiter ops are enabled. "VLLM_ROCM_USE_AITER_RMSNORM": lambda: ( os.getenv("VLLM_ROCM_USE_AITER_RMSNORM", "True").lower() in ("true", "1") ), # Whether to use aiter mla ops. # By default is enabled. "VLLM_ROCM_USE_AITER_MLA": lambda: ( os.getenv("VLLM_ROCM_USE_AITER_MLA", "True").lower() in ("true", "1") ), # Whether to use aiter mha ops. # By default is enabled. "VLLM_ROCM_USE_AITER_MHA": lambda: ( os.getenv("VLLM_ROCM_USE_AITER_MHA", "True").lower() in ("true", "1") ), # Whether to use aiter fp4 gemm asm. # By default is disabled. "VLLM_ROCM_USE_AITER_FP4_ASM_GEMM": lambda: ( os.getenv("VLLM_ROCM_USE_AITER_FP4_ASM_GEMM", "False").lower() in ("true", "1") ), # Whether to use aiter rope. # By default is disabled. "VLLM_ROCM_USE_AITER_TRITON_ROPE": lambda: ( os.getenv("VLLM_ROCM_USE_AITER_TRITON_ROPE", "False").lower() in ("true", "1") ), # Whether to use aiter triton fp8 bmm kernel # By default is enabled. "VLLM_ROCM_USE_AITER_FP8BMM": lambda: ( os.getenv("VLLM_ROCM_USE_AITER_FP8BMM", "True").lower() in ("true", "1") ), # Whether to use aiter triton fp4 bmm kernel # By default is enabled. "VLLM_ROCM_USE_AITER_FP4BMM": lambda: ( os.getenv("VLLM_ROCM_USE_AITER_FP4BMM", "True").lower() in ("true", "1") ), # Use AITER triton unified attention for V1 attention "VLLM_ROCM_USE_AITER_UNIFIED_ATTENTION": lambda: ( os.getenv("VLLM_ROCM_USE_AITER_UNIFIED_ATTENTION", "False").lower() in ("true", "1") ), # Whether to use aiter fusion shared experts ops. # By default is disabled. "VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS": lambda: ( os.getenv("VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS", "False").lower() in ("true", "1") ), # Whether to use aiter triton kernels for gemm ops. # By default is enabled. "VLLM_ROCM_USE_AITER_TRITON_GEMM": lambda: ( os.getenv("VLLM_ROCM_USE_AITER_TRITON_GEMM", "True").lower() in ("true", "1") ), # use rocm skinny gemms "VLLM_ROCM_USE_SKINNY_GEMM": lambda: ( os.getenv("VLLM_ROCM_USE_SKINNY_GEMM", "True").lower() in ("true", "1") ), # Pad the fp8 weights to 256 bytes for ROCm "VLLM_ROCM_FP8_PADDING": lambda: bool(int(os.getenv("VLLM_ROCM_FP8_PADDING", "1"))), # Pad the weights for the moe kernel "VLLM_ROCM_MOE_PADDING": lambda: bool(int(os.getenv("VLLM_ROCM_MOE_PADDING", "1"))), # Whether to use the shuffled kv cache layout "VLLM_ROCM_SHUFFLE_KV_CACHE_LAYOUT": lambda: ( os.getenv("VLLM_ROCM_SHUFFLE_KV_CACHE_LAYOUT", "False").lower() in ("true", "1") ), # Custom quick allreduce kernel for MI3* cards # Choice of quantization level: FP, INT8, INT6, INT4, INT3 or NONE # Recommended for large models to get allreduce "VLLM_ROCM_QUICK_REDUCE_QUANTIZATION": env_with_choices( "VLLM_ROCM_QUICK_REDUCE_QUANTIZATION", "NONE", ["FP", "INT8", "INT6", "INT4", "INT3", "NONE"], ), # Custom quick allreduce kernel for MI3* cards # Due to the lack of the bfloat16 asm instruction, bfloat16 # kernels are slower than fp16, # If environment variable is set to 1, the input is converted to fp16 "VLLM_ROCM_QUICK_REDUCE_CAST_BF16_TO_FP16": lambda: ( os.getenv("VLLM_ROCM_QUICK_REDUCE_CAST_BF16_TO_FP16", "True").lower() in ("true", "1") ), # Custom quick allreduce kernel for MI3* cards. # Controls the maximum allowed number of data bytes(MB) for custom quick # allreduce communication. # Default: 2048 MB. # Data exceeding this size will use either custom allreduce or RCCL # communication. "VLLM_ROCM_QUICK_REDUCE_MAX_SIZE_BYTES_MB": lambda: maybe_convert_int( os.environ.get("VLLM_ROCM_QUICK_REDUCE_MAX_SIZE_BYTES_MB", None) ), # Custom quick allreduce kernel for MI3* cards. # Controls the minimum allowed number of data bytes(MB) required to use # custom quick allreduce communication. # If unset, use the built-in threshold table. "VLLM_ROCM_QUICK_REDUCE_MIN_SIZE_BYTES_MB": lambda: maybe_convert_int( os.environ.get("VLLM_ROCM_QUICK_REDUCE_MIN_SIZE_BYTES_MB", None) ), # Controls the minimum tensor size (KB, where 1 KB = 1024 bytes) required # to use the configured QuickReduce codec. Smaller tensors use FP # QuickReduce. This does not affect QuickReduce eligibility. "VLLM_ROCM_QUICK_REDUCE_QUANTIZATION_MIN_SIZE_KB": lambda: maybe_convert_int( os.environ.get("VLLM_ROCM_QUICK_REDUCE_QUANTIZATION_MIN_SIZE_KB", None) ), # Divisor for dynamic query scale factor calculation for FP8 KV Cache "Q_SCALE_CONSTANT": lambda: int(os.getenv("Q_SCALE_CONSTANT", "200")), # Divisor for dynamic key scale factor calculation for FP8 KV Cache "K_SCALE_CONSTANT": lambda: int(os.getenv("K_SCALE_CONSTANT", "200")), # Divisor for dynamic value scale factor calculation for FP8 KV Cache "V_SCALE_CONSTANT": lambda: int(os.getenv("V_SCALE_CONSTANT", "100")), # If set, enable multiprocessing in LLM for the V1 code path. "VLLM_ENABLE_V1_MULTIPROCESSING": lambda: bool( int(os.getenv("VLLM_ENABLE_V1_MULTIPROCESSING", "1")) ), "VLLM_LOG_BATCHSIZE_INTERVAL": lambda: float( os.getenv("VLLM_LOG_BATCHSIZE_INTERVAL", "-1") ), "VLLM_DISABLE_COMPILE_CACHE": disable_compile_cache, # If set to "0", disable LayerName opaque type for layer_name # parameters in custom ops. Defaults to enabled on torch >= 2.11. "VLLM_USE_LAYERNAME": lambda: bool(int(os.getenv("VLLM_USE_LAYERNAME", "1"))), # If set, use the Rust frontend binary instead of the Python API server # process(es). "VLLM_USE_RUST_FRONTEND": lambda: bool( int(os.getenv("VLLM_USE_RUST_FRONTEND", "0")) ), # Path to the Rust frontend binary. Defaults to "auto" which discovers # the binary installed with the vllm package. Only used when # VLLM_USE_RUST_FRONTEND=1. "VLLM_RUST_FRONTEND_PATH": lambda: _resolve_rust_frontend_path(), # If set, vllm will run in development mode, which will enable # some additional endpoints for developing and debugging, # e.g. `/reset_prefix_cache` "VLLM_SERVER_DEV_MODE": lambda: bool(int(os.getenv("VLLM_SERVER_DEV_MODE", "0"))), # Controls the maximum number of requests to handle in a # single asyncio task when processing per-token outputs in the # V1 AsyncLLM interface. It is applicable when handling a high # concurrency of streaming requests. # Setting this too high can result in a higher variance of # inter-message latencies. Setting it too low can negatively impact # TTFT and overall throughput. "VLLM_V1_OUTPUT_PROC_CHUNK_SIZE": lambda: int( os.getenv("VLLM_V1_OUTPUT_PROC_CHUNK_SIZE", "128") ), # If set, vLLM will disable the MLA attention optimizations. "VLLM_MLA_DISABLE": lambda: bool(int(os.getenv("VLLM_MLA_DISABLE", "0"))), # If set, vLLM will pick up the provided Flash Attention MLA # Number of GPUs per worker in Ray, if it is set to be a fraction, # it allows ray to schedule multiple actors on a single GPU, # so that users can colocate other actors on the same GPUs as vLLM. "VLLM_RAY_PER_WORKER_GPUS": lambda: float( os.getenv("VLLM_RAY_PER_WORKER_GPUS", "1.0") ), # Bundle indices for Ray, if it is set, it can control precisely # which indices are used for the Ray bundle, for every worker. # Format: comma-separated list of integers, e.g. "0,1,2,3" "VLLM_RAY_BUNDLE_INDICES": lambda: os.getenv("VLLM_RAY_BUNDLE_INDICES", ""), # In some system, find_loaded_library() may not work. So we allow users to # specify the path through environment variable VLLM_CUDART_SO_PATH. "VLLM_CUDART_SO_PATH": lambda: os.getenv("VLLM_CUDART_SO_PATH", None), # Rank of the process in the data parallel setting "VLLM_DP_RANK": lambda: int(os.getenv("VLLM_DP_RANK", "0")), # Rank of the process in the data parallel setting. # Defaults to VLLM_DP_RANK when not set. "VLLM_DP_RANK_LOCAL": lambda: int( os.getenv("VLLM_DP_RANK_LOCAL", sys.modules[__name__].VLLM_DP_RANK) ), # World size of the data parallel setting "VLLM_DP_SIZE": lambda: int(os.getenv("VLLM_DP_SIZE", "1")), # IP address of the master node in the data parallel setting "VLLM_DP_MASTER_IP": lambda: os.getenv("VLLM_DP_MASTER_IP", "127.0.0.1"), # Port of the master node in the data parallel setting "VLLM_DP_MASTER_PORT": lambda: int(os.getenv("VLLM_DP_MASTER_PORT", "0")), # Randomize inputs during dummy runs when using Data Parallel "VLLM_RANDOMIZE_DP_DUMMY_INPUTS": lambda: ( os.environ.get("VLLM_RANDOMIZE_DP_DUMMY_INPUTS", "0") == "1" ), # Strategy to pack the data parallel ranks for Ray. # Available options: # - "fill": # for DP master node, allocate exactly data-parallel-size-local DP ranks, # for non-master nodes, allocate as many DP ranks as can fit; # - "strict": # allocate exactly data-parallel-size-local DP ranks to each picked node; # - "span": # Should be used only when a single DP rank requires multiple nodes. # allocate one DP rank over as many nodes as required for set world_size; # This environment variable is ignored if data-parallel-backend is not Ray. "VLLM_RAY_DP_PACK_STRATEGY": lambda: os.getenv( "VLLM_RAY_DP_PACK_STRATEGY", "strict" ), # Optional comma-separated list of node IPs that Ray data-parallel # placement groups may use. When set, create_dp_placement_groups only # considers these nodes (the DP master node is always included). # This environment variable is ignored if data-parallel-backend is not Ray. "VLLM_RAY_DP_PLACEMENT_NODE_IPS": lambda: os.getenv( "VLLM_RAY_DP_PLACEMENT_NODE_IPS", "" ), # Comma-separated *additional* prefixes of env vars to copy from the # driver to Ray workers. These are merged with the built-in defaults # defined in ``vllm.ray.ray_env`` (VLLM_, etc.). Example: "MYLIB_,OTHER_" "VLLM_RAY_EXTRA_ENV_VAR_PREFIXES_TO_COPY": lambda: os.getenv( "VLLM_RAY_EXTRA_ENV_VAR_PREFIXES_TO_COPY", "" ), # Comma-separated *additional* individual env var names to copy from # the driver to Ray workers. Merged with the built-in defaults # defined in ``vllm.ray.ray_env`` (PYTHONHASHSEED). # Example: "MY_SECRET,MY_FLAG" "VLLM_RAY_EXTRA_ENV_VARS_TO_COPY": lambda: os.getenv( "VLLM_RAY_EXTRA_ENV_VARS_TO_COPY", "" ), # Whether to use S3 path for model loading in CI via RunAI Streamer "VLLM_CI_USE_S3": lambda: os.environ.get("VLLM_CI_USE_S3", "0") == "1", # Use model_redirect to redirect the model name to a local folder. # `model_redirect` can be a json file mapping the model between # repo_id and local folder: # {"meta-llama/Llama-3.2-1B": "/tmp/Llama-3.2-1B"} # or a space separated values table file: # meta-llama/Llama-3.2-1B /tmp/Llama-3.2-1B "VLLM_MODEL_REDIRECT_PATH": lambda: os.environ.get( "VLLM_MODEL_REDIRECT_PATH", None ), # Whether to use atomicAdd reduce in gptq/awq marlin kernel. "VLLM_MARLIN_USE_ATOMIC_ADD": lambda: ( os.environ.get("VLLM_MARLIN_USE_ATOMIC_ADD", "0") == "1" ), # The activation dtype for marlin kernel "VLLM_MARLIN_INPUT_DTYPE": env_with_choices( "VLLM_MARLIN_INPUT_DTYPE", None, ["int8", "fp8"] ), # The online quantization dtype for humming kernel "VLLM_HUMMING_ONLINE_QUANT_CONFIG": lambda: maybe_convert_json_str_or_file( os.environ.get("VLLM_HUMMING_ONLINE_QUANT_CONFIG", None) ), # The activation dtype config for humming kernel "VLLM_HUMMING_INPUT_QUANT_CONFIG": lambda: maybe_convert_json_str_or_file( os.environ.get("VLLM_HUMMING_INPUT_QUANT_CONFIG", None) ), # Whether to use fp16 accumulator mma "VLLM_HUMMING_USE_F16_ACCUM": lambda: maybe_convert_bool( os.environ.get("VLLM_HUMMING_USE_F16_ACCUM", "0") ), # Whether to use indexed gemm for humming moe # if 1, force use indexed gemm # if 0, force use grouped gemm # if None, choose better gemm type automatically "VLLM_HUMMING_MOE_GEMM_TYPE": lambda: os.environ.get( "VLLM_HUMMING_MOE_GEMM_TYPE", None ), # Whether to use DeepEPLL kernels for NVFP4 quantization and dispatch method # only supported on Blackwell GPUs and with # https://github.com/deepseek-ai/DeepEP/pull/341 "VLLM_DEEPEPLL_NVFP4_DISPATCH": lambda: bool( int(os.getenv("VLLM_DEEPEPLL_NVFP4_DISPATCH", "0")) ), # Whether to turn on the outlines cache for V1 # This cache is unbounded and on disk, so it's not safe to use in # an environment with potentially malicious users. "VLLM_V1_USE_OUTLINES_CACHE": lambda: ( os.environ.get("VLLM_V1_USE_OUTLINES_CACHE", "0") == "1" ), # Gap between padding buckets for the forward pass. So we have # 8, we will run forward pass with [16, 24, 32, ...]. "VLLM_TPU_BUCKET_PADDING_GAP": lambda: ( int(os.environ["VLLM_TPU_BUCKET_PADDING_GAP"]) if "VLLM_TPU_BUCKET_PADDING_GAP" in os.environ else 0 ), "VLLM_TPU_MOST_MODEL_LEN": lambda: maybe_convert_int( os.environ.get("VLLM_TPU_MOST_MODEL_LEN", None) ), # Whether using Pathways "VLLM_TPU_USING_PATHWAYS": lambda: bool( "proxy" in os.getenv("JAX_PLATFORMS", "").lower() ), # Allow use of DeepGemm kernels for fused moe ops. "VLLM_USE_DEEP_GEMM": lambda: bool(int(os.getenv("VLLM_USE_DEEP_GEMM", "1"))), # Allow use of DeepGemm specifically for MoE fused ops (overrides only MoE). "VLLM_MOE_USE_DEEP_GEMM": lambda: bool( int(os.getenv("VLLM_MOE_USE_DEEP_GEMM", "1")) ), # Whether to use E8M0 scaling when DeepGEMM is used on Blackwell GPUs. "VLLM_USE_DEEP_GEMM_E8M0": lambda: bool( int(os.getenv("VLLM_USE_DEEP_GEMM_E8M0", "1")) ), # Whether to create TMA-aligned scale tensor when DeepGEMM is used. "VLLM_USE_DEEP_GEMM_TMA_ALIGNED_SCALES": lambda: bool( int(os.getenv("VLLM_USE_DEEP_GEMM_TMA_ALIGNED_SCALES", "1")) ), # DeepGemm JITs the kernels on-demand. The warmup attempts to make DeepGemm # JIT all the required kernels before model execution so there is no # JIT'ing in the hot-path. However, this warmup increases the engine # startup time by a couple of minutes. # Available options: # - "skip" : Skip warmup. # - "full" : Warmup deepgemm by running all possible gemm shapes the # engine could encounter. # - "relax" : Select gemm shapes to run based on some heuristics. The # heuristic aims to have the same effect as running all possible gemm # shapes, but provides no guarantees. "VLLM_DEEP_GEMM_WARMUP": env_with_choices( "VLLM_DEEP_GEMM_WARMUP", "relax", [ "skip", "full", "relax", ], ), # Whether to use fused grouped_topk used for MoE expert selection. "VLLM_USE_FUSED_MOE_GROUPED_TOPK": lambda: bool( int(os.getenv("VLLM_USE_FUSED_MOE_GROUPED_TOPK", "1")) ), # Skip cudagraph/DP padding tokens in the MoE path by forcing their expert # ids to -1 so the dispatch and experts drop them. Requires a MoE kernel that # treats topk_id == -1 as a skip sentinel; off by default because not all # kernels support it yet. "VLLM_MOE_SKIP_PADDING": lambda: bool(int(os.getenv("VLLM_MOE_SKIP_PADDING", "0"))), # Allow use of FlashInfer FP8 block-scale GEMM for linear layers. # This uses TensorRT-LLM kernels and requires SM90+ (Hopper). "VLLM_BLOCKSCALE_FP8_GEMM_FLASHINFER": lambda: bool( int(os.getenv("VLLM_BLOCKSCALE_FP8_GEMM_FLASHINFER", "1")) ), # Allow use of FlashInfer MxInt4 MoE kernels for fused moe ops. "VLLM_USE_FLASHINFER_MOE_INT4": lambda: bool( int(os.getenv("VLLM_USE_FLASHINFER_MOE_INT4", "0")) ), # Control the cache sized used by the xgrammar compiler. The default # of 512 MB should be enough for roughly 1000 JSON schemas. # It can be changed with this variable if needed for some reason. "VLLM_XGRAMMAR_CACHE_MB": lambda: int(os.getenv("VLLM_XGRAMMAR_CACHE_MB", "512")), # Maximum time in seconds allowed for regex compilation in structured # output backends (xgrammar, outlines). Prevents ReDoS attacks where # adversarial patterns cause exponential DFA state-space explosion. # Set to 0 to disable the timeout (not recommended in production). "VLLM_REGEX_COMPILATION_TIMEOUT_S": lambda: int( os.getenv("VLLM_REGEX_COMPILATION_TIMEOUT_S", "5") ), # Control the threshold for msgspec to use 'zero copy' for # serialization/deserialization of tensors. Tensors below # this limit will be encoded into the msgpack buffer, and # tensors above will instead be sent via a separate message. # While the sending side still actually copies the tensor # in all cases, on the receiving side, tensors above this # limit will actually be zero-copy decoded. "VLLM_MSGPACK_ZERO_COPY_THRESHOLD": lambda: int( os.getenv("VLLM_MSGPACK_ZERO_COPY_THRESHOLD", "256") ), # If set, allow insecure serialization using pickle. # This is useful for environments where it is deemed safe to use the # insecure method and it is needed for some reason. "VLLM_ALLOW_INSECURE_SERIALIZATION": lambda: bool( int(os.getenv("VLLM_ALLOW_INSECURE_SERIALIZATION", "0")) ), # Temporary: skip adding random suffix to internal request IDs. May be # needed for KV connectors that match request IDs across instances. "VLLM_DISABLE_REQUEST_ID_RANDOMIZATION": lambda: bool( int(os.getenv("VLLM_DISABLE_REQUEST_ID_RANDOMIZATION", "0")) ), # IP address used for NIXL handshake between remote agents. "VLLM_NIXL_SIDE_CHANNEL_HOST": lambda: os.getenv( "VLLM_NIXL_SIDE_CHANNEL_HOST", "localhost" ), # Port used for NIXL handshake between remote agents. "VLLM_NIXL_SIDE_CHANNEL_PORT": lambda: int( os.getenv("VLLM_NIXL_SIDE_CHANNEL_PORT", "5600") ), # IP address used for the EC connector's ZMQ side channel # (producer ROUTER bind, consumer DEALER dial). "VLLM_EC_SIDE_CHANNEL_HOST": lambda: os.getenv( "VLLM_EC_SIDE_CHANNEL_HOST", "localhost" ), # Port for the EC connector's ZMQ side channel; advertised to peers # via `ec_transfer_params.peer_port` on the producer's response. "VLLM_EC_SIDE_CHANNEL_PORT": lambda: int( os.getenv("VLLM_EC_SIDE_CHANNEL_PORT", "5601") ), # Port used for Mooncake handshake between remote agents. "VLLM_MOONCAKE_BOOTSTRAP_PORT": lambda: int( os.getenv("VLLM_MOONCAKE_BOOTSTRAP_PORT", "8998") ), # Log per-batch memory/disk tier breakdown on external GETs. "VLLM_MOONCAKE_STORE_TIER_LOG": lambda: ( os.getenv("VLLM_MOONCAKE_STORE_TIER_LOG", "False").lower() in ("true", "1") ), # Number of parallel KV-load receive threads per worker rank. Lets the # per-request control overhead (Python prep + master key lookup) of one # request overlap with the RDMA transfer of another, keeping the transfer # engine's queue pairs busy. Helps when that overhead is significant or # per-request batches are too small to saturate the link on their own. "VLLM_MOONCAKE_LOAD_RECV_THREADS": lambda: int( os.getenv("VLLM_MOONCAKE_LOAD_RECV_THREADS", "1") ), # Fraction of the owner's DirectIO staging buffer to fill per GET batch. "VLLM_MOONCAKE_DISK_STAGING_USABLE_RATIO": lambda: float( os.getenv("VLLM_MOONCAKE_DISK_STAGING_USABLE_RATIO", "0.9") ), # Pin this rank to a specific owner segment ("host:port"). "MOONCAKE_PREFERRED_SEGMENT": lambda: os.getenv("MOONCAKE_PREFERRED_SEGMENT"), # Override the hostname the rank registers as a Mooncake requester. "MOONCAKE_REQUESTER_LOCAL_HOSTNAME": lambda: os.getenv( "MOONCAKE_REQUESTER_LOCAL_HOSTNAME" ), # Override the directory for the FlashInfer autotune config cache. "VLLM_FLASHINFER_AUTOTUNE_CACHE_DIR": lambda: os.getenv( "VLLM_FLASHINFER_AUTOTUNE_CACHE_DIR", None ), # Comma-separated FlashInfer op names to exclude from autotuning, using # the heuristic fallback tactic instead. Unset: skip "fp4_gemm" when the # CuTe-DSL NVFP4 linear kernel is selected. Empty: skip nothing. "VLLM_FLASHINFER_AUTOTUNE_SKIP_OPS": lambda: ( None if "VLLM_FLASHINFER_AUTOTUNE_SKIP_OPS" not in os.environ else [ v.strip() for v in os.environ["VLLM_FLASHINFER_AUTOTUNE_SKIP_OPS"].split(",") if v.strip() ] ), # Flashinfer fused allreduce backend. "VLLM_FLASHINFER_ALLREDUCE_BACKEND": env_with_choices( "VLLM_FLASHINFER_ALLREDUCE_BACKEND", "auto", ["auto", "trtllm", "mnnvl"], ), # Control the workspace buffer size for the FlashInfer backend. "VLLM_FLASHINFER_WORKSPACE_BUFFER_SIZE": lambda: int( os.getenv("VLLM_FLASHINFER_WORKSPACE_BUFFER_SIZE", str(394 * 1024 * 1024)) ), # Control the maximum number of tokens per expert supported by the # NVFP4 MoE CUTLASS Kernel. This value is used to create a buffer for # the blockscale tensor of activations NVFP4 Quantization. # This is used to prevent the kernel from running out of memory. "VLLM_MAX_TOKENS_PER_EXPERT_FP4_MOE": lambda: int( os.getenv("VLLM_MAX_TOKENS_PER_EXPERT_FP4_MOE", "163840") ), # Specifies the thresholds of the communicated tensor sizes under which # vllm should use flashinfer fused allreduce. The variable should be a # JSON with the following format: # { : } # Unspecified world sizes will fall back to # { 2: 64, 4: 1, : 0.5 } "VLLM_FLASHINFER_ALLREDUCE_FUSION_THRESHOLDS_MB": lambda: json.loads( os.getenv("VLLM_FLASHINFER_ALLREDUCE_FUSION_THRESHOLDS_MB", "{}") ), # MoE routing strategy selector. # See `RoutingSimulator.get_available_strategies()` # for available # strategies. # Custom routing strategies can be registered by # RoutingSimulator.register_strategy() # Note: custom strategies may not produce correct model outputs "VLLM_MOE_ROUTING_SIMULATION_STRATEGY": lambda: os.environ.get( "VLLM_MOE_ROUTING_SIMULATION_STRATEGY", "" ).lower(), # Regex timeout for use by the vLLM tool parsing plugins. "VLLM_TOOL_PARSE_REGEX_TIMEOUT_SECONDS": lambda: int( os.getenv("VLLM_TOOL_PARSE_REGEX_TIMEOUT_SECONDS", "1") ), # Enforce function parameter schemas in structural-tag based tool calling. "VLLM_ENFORCE_STRICT_TOOL_CALLING": lambda: ( os.getenv("VLLM_ENFORCE_STRICT_TOOL_CALLING", "True").lower() in ("true", "1") ), # Control the max chunk bytes (in MB) for the rpc message queue. # Object larger than this threshold will be broadcast to worker # processes via zmq. "VLLM_MQ_MAX_CHUNK_BYTES_MB": lambda: int( os.getenv("VLLM_MQ_MAX_CHUNK_BYTES_MB", "16") ), # Timeout in seconds for execute_model RPC calls in multiprocessing # executor (only applies when TP > 1). "VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS": lambda: int( os.getenv("VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS", "300") ), # Timeout in seconds for engine and worker process shutdown "VLLM_WORKER_SHUTDOWN_TIMEOUT_SECONDS": lambda: int( os.getenv("VLLM_WORKER_SHUTDOWN_TIMEOUT_SECONDS", "5") ), # KV Cache layout used throughout vllm. # Some common values are: # - NHD # - HND # Where N=num_blocks, H=num_heads and D=head_size. The default value will # leave the layout choice to the backend. Mind that backends may only # implement and support a subset of all possible layouts. "VLLM_KV_CACHE_LAYOUT": env_with_choices( "VLLM_KV_CACHE_LAYOUT", None, ["NHD", "HND"] ), # SSM conv state layout used for Mamba models. # - SD: (state_len, dim) — dim contiguous (default) # - DS: (dim, state_len) — TP-sharded dim on dim1, # consistent with SSM temporal state and HND KV cache layout. "VLLM_SSM_CONV_STATE_LAYOUT": env_with_choices( "VLLM_SSM_CONV_STATE_LAYOUT", None, ["SD", "DS"] ), # Enable checking whether the generated logits contain NaNs, # indicating corrupted output. Useful for debugging low level bugs # or bad hardware but it may add compute overhead. "VLLM_COMPUTE_NANS_IN_LOGITS": lambda: bool( int(os.getenv("VLLM_COMPUTE_NANS_IN_LOGITS", "0")) ), # Timeout (in seconds) for MooncakeConnector in PD disaggregated setup. "VLLM_MOONCAKE_ABORT_REQUEST_TIMEOUT": lambda: int( os.getenv("VLLM_MOONCAKE_ABORT_REQUEST_TIMEOUT", "480") ), # If set, it means we pre-downloaded cubin files and flashinfer will # read the cubin files directly. "VLLM_HAS_FLASHINFER_CUBIN": lambda: bool( int(os.getenv("VLLM_HAS_FLASHINFER_CUBIN", "0")) ), # Controls garbage collection during CUDA graph capture. # If set to 0 (default), enables GC freezing to speed up capture time. # If set to 1, allows GC to run during capture. "VLLM_ENABLE_CUDAGRAPH_GC": lambda: bool( int(os.getenv("VLLM_ENABLE_CUDAGRAPH_GC", "0")) ), # Used to force set up loopback IP "VLLM_LOOPBACK_IP": lambda: os.getenv("VLLM_LOOPBACK_IP", ""), # Used to set the process name prefix for vLLM processes. # This is useful for debugging and monitoring purposes. # The default value is "VLLM". "VLLM_PROCESS_NAME_PREFIX": lambda: os.getenv("VLLM_PROCESS_NAME_PREFIX", "VLLM"), # Allow chunked local attention with hybrid kv cache manager. # Currently using the Hybrid KV cache manager with chunked local attention # in the Llama4 models (the only models currently using chunked local attn) # causes a latency regression. For this reason, we disable it by default. # This flag is used to allow users to enable it if they want to (to save on # kv-cache memory usage and enable longer contexts) # TODO(lucas): Remove this flag once latency regression is resolved. "VLLM_ALLOW_CHUNKED_LOCAL_ATTN_WITH_HYBRID_KV_CACHE": lambda: bool( int(os.getenv("VLLM_ALLOW_CHUNKED_LOCAL_ATTN_WITH_HYBRID_KV_CACHE", "1")) ), # Enables support for the "store" option in the OpenAI Responses API. # When set to 1, vLLM's OpenAI server will retain the input and output # messages for those requests in memory. By default, this is disabled (0), # and the "store" option is ignored. # NOTE/WARNING: # 1. Messages are kept in memory only (not persisted to disk) and will be # lost when the vLLM server shuts down. # 2. Enabling this option will cause a memory leak, as stored messages are # never removed from memory until the server terminates. "VLLM_ENABLE_RESPONSES_API_STORE": lambda: bool( int(os.getenv("VLLM_ENABLE_RESPONSES_API_STORE", "0")) ), # If set, use the fp8 mfma in rocm paged attention. "VLLM_ROCM_FP8_MFMA_PAGE_ATTN": lambda: bool( int(os.getenv("VLLM_ROCM_FP8_MFMA_PAGE_ATTN", "0")) ), # Whether to use pytorch symmetric memory for allreduce "VLLM_ALLREDUCE_USE_SYMM_MEM": lambda: bool( int(os.getenv("VLLM_ALLREDUCE_USE_SYMM_MEM", "1")) ), # Whether to use FlashInfer allreduce "VLLM_ALLREDUCE_USE_FLASHINFER": lambda: bool( int(os.getenv("VLLM_ALLREDUCE_USE_FLASHINFER", "0")) ), # Experimental: use this to enable MCP tool calling for non harmony models "VLLM_USE_EXPERIMENTAL_PARSER_CONTEXT": lambda: bool( int(os.getenv("VLLM_USE_EXPERIMENTAL_PARSER_CONTEXT", "0")) ), # User override folder for tuned Triton-kernel configs. Shared by MoE, # Mamba SSU, and LoRA. Filenames are distinct so one folder can hold all. # Each component first checks this folder, then the configs shipped with # vLLM (if any). If no JSON matches, it uses a hard-coded heuristic. "VLLM_TUNED_CONFIG_FOLDER": lambda: os.getenv("VLLM_TUNED_CONFIG_FOLDER", None), # Opt-in persistence of the startup plan. When enabled, each worker # saves the memory-profiling result (the suggested --kv-cache-memory value # and the free-memory baseline) under VLLM_CACHE_ROOT/startup_plan/, # keyed by a hardware+config fingerprint, and later boots auto-apply it # -- skipping memory profiling -- when the fingerprint matches and # current free memory >= the recorded baseline. # See vllm/v1/worker/startup_plan.py. "VLLM_ENABLE_STARTUP_PLAN": lambda: bool( int(os.getenv("VLLM_ENABLE_STARTUP_PLAN", "0")) ), # Valid values are container,code_interpreter,web_search_preview # ex VLLM_GPT_OSS_SYSTEM_TOOL_MCP_LABELS=container,code_interpreter # If the server_label of your mcp tool is not in this list it will # be completely ignored. "VLLM_GPT_OSS_SYSTEM_TOOL_MCP_LABELS": env_set_with_choices( "VLLM_GPT_OSS_SYSTEM_TOOL_MCP_LABELS", default=[], choices=["container", "code_interpreter", "web_search_preview"], ), # Allows harmony instructions to be injected on system messages "VLLM_GPT_OSS_HARMONY_SYSTEM_INSTRUCTIONS": lambda: bool( int(os.getenv("VLLM_GPT_OSS_HARMONY_SYSTEM_INSTRUCTIONS", "0")) ), # Pin the conversation start date injected into the Harmony system # message. When unset the current date is used, which introduces # non-determinism (different tokens -> different model behaviour at # temperature=0). Set to an ISO date string, e.g. "2023-09-12", # for reproducible inference or testing. "VLLM_SYSTEM_START_DATE": lambda: os.getenv("VLLM_SYSTEM_START_DATE", None), # Enable automatic retry when tool call JSON parsing fails # If enabled, returns an error message to the model to retry # If disabled (default), raises an exception and fails the request "VLLM_TOOL_JSON_ERROR_AUTOMATIC_RETRY": lambda: bool( int(os.getenv("VLLM_TOOL_JSON_ERROR_AUTOMATIC_RETRY", "0")) ), # Add optional custom scopes for profiling, disable to avoid overheads "VLLM_CUSTOM_SCOPES_FOR_PROFILING": lambda: bool( int(os.getenv("VLLM_CUSTOM_SCOPES_FOR_PROFILING", "0")) ), # Add optional nvtx scopes for profiling, disable to avoid overheads "VLLM_NVTX_SCOPES_FOR_PROFILING": lambda: bool( int(os.getenv("VLLM_NVTX_SCOPES_FOR_PROFILING", "0")) ), # Represent block hashes in KV cache events as 64-bit integers instead of # raw bytes. Defaults to True for backward compatibility. "VLLM_KV_EVENTS_USE_INT_BLOCK_HASHES": lambda: bool( int(os.getenv("VLLM_KV_EVENTS_USE_INT_BLOCK_HASHES", "1")) ), # Name of the shared memory buffer used for object storage. # Only effective when mm_config.mm_processor_cache_type == "shm". # Automatically generates a unique UUID-based name per process tree # if not explicitly set. "VLLM_OBJECT_STORAGE_SHM_BUFFER_NAME": get_env_or_set_default( "VLLM_OBJECT_STORAGE_SHM_BUFFER_NAME", lambda: f"VLLM_OBJECT_STORAGE_SHM_BUFFER_{uuid.uuid4().hex}", ), # The size in MB of the buffers (NVL and RDMA) used by DeepEP "VLLM_DEEPEP_BUFFER_SIZE_MB": lambda: int( os.getenv("VLLM_DEEPEP_BUFFER_SIZE_MB", "1024") ), # Force DeepEP to use intranode kernel for inter-node communication in # high throughput mode. This is useful archive higher prefill throughput # on system supports multi-node nvlink (e.g GB200). "VLLM_DEEPEP_HIGH_THROUGHPUT_FORCE_INTRA_NODE": lambda: bool( int(os.getenv("VLLM_DEEPEP_HIGH_THROUGHPUT_FORCE_INTRA_NODE", "0")) ), # Allow DeepEP to use MNNVL (multi-node nvlink) for internode_ll kernel, # turn this for better latency on GB200 like system "VLLM_DEEPEP_LOW_LATENCY_USE_MNNVL": lambda: bool( int(os.getenv("VLLM_DEEPEP_LOW_LATENCY_USE_MNNVL", "0")) ), # DeepEP v2: enable two-tier NVLink+RDMA hybrid mode "VLLM_DEEPEP_V2_ALLOW_HYBRID_MODE": lambda: bool( int(os.getenv("VLLM_DEEPEP_V2_ALLOW_HYBRID_MODE", "0")) ), # DeepEP v2: use fewer SMs at slight throughput cost "VLLM_DEEPEP_V2_PREFER_OVERLAP": lambda: bool( int(os.getenv("VLLM_DEEPEP_V2_PREFER_OVERLAP", "0")) ), # DeepEP v2: trade precision for transfer size in combine "VLLM_DEEPEP_V2_ALLOW_MULTIPLE_REDUCTION": lambda: bool( int(os.getenv("VLLM_DEEPEP_V2_ALLOW_MULTIPLE_REDUCTION", "0")) ), # The number of SMs/CUs to allocate for communication kernels when # running DBO; the rest will be allocated to compute. # Default: 20 on CUDA (SMs), 64 on ROCm (CUs). "VLLM_DBO_COMM_SMS": lambda: int( os.getenv( "VLLM_DBO_COMM_SMS", "64" if hasattr(__import__("torch").version, "hip") and __import__("torch").version.hip is not None else "20", ) ), # Enable max_autotune & coordinate_descent_tuning in inductor_config # to compile static shapes passed from compile_sizes in compilation_config # If set to 1, enable max_autotune; By default, this is enabled (1) "VLLM_ENABLE_INDUCTOR_MAX_AUTOTUNE": lambda: bool( int(os.getenv("VLLM_ENABLE_INDUCTOR_MAX_AUTOTUNE", "1")) ), # If set to 1, enable coordinate_descent_tuning; # By default, this is enabled (1) "VLLM_ENABLE_INDUCTOR_COORDINATE_DESCENT_TUNING": lambda: bool( int(os.getenv("VLLM_ENABLE_INDUCTOR_COORDINATE_DESCENT_TUNING", "1")) ), # Flag to enable NCCL symmetric memory allocation and registration "VLLM_USE_NCCL_SYMM_MEM": lambda: bool( int(os.getenv("VLLM_USE_NCCL_SYMM_MEM", "0")) ), # NCCL header path "VLLM_NCCL_INCLUDE_PATH": lambda: os.environ.get("VLLM_NCCL_INCLUDE_PATH", None), # GC debug config # - VLLM_GC_DEBUG=0: disable GC debugger # - VLLM_GC_DEBUG=1: enable GC debugger with gc.collect elpased times # - VLLM_GC_DEBUG='{"top_objects":5}': enable GC debugger with # top 5 collected objects "VLLM_GC_DEBUG": lambda: os.getenv("VLLM_GC_DEBUG", ""), # Debug workspace allocations. # logging of workspace resize operations. "VLLM_DEBUG_WORKSPACE": lambda: bool(int(os.getenv("VLLM_DEBUG_WORKSPACE", "0"))), # Disables parallel execution of shared_experts via separate cuda stream "VLLM_DISABLE_SHARED_EXPERTS_STREAM": lambda: bool( int(os.getenv("VLLM_DISABLE_SHARED_EXPERTS_STREAM", "0")) ), # Limits when we run shared_experts in a separate stream. # We found out that for large batch sizes, the separate stream # execution is not beneficial (most likely because of the input clone) # TODO(alexm-redhat): Tune to be more dynamic based on GPU type "VLLM_SHARED_EXPERTS_STREAM_TOKEN_THRESHOLD": lambda: int( int(os.getenv("VLLM_SHARED_EXPERTS_STREAM_TOKEN_THRESHOLD", 256)) ), # Token-count cutoff for multi-stream overlap of the attention input # GEMM with auxiliary GEMMs (e.g. fused_wqa_wkv overlapped with indexer # weights / kv-score projections in DeepSeek-V4). At or below this many # tokens the FP8 main GEMM has idle SMs to share with the bf16 aux GEMMs # and overlap is a 5-45% win; above it the FP8 GEMM saturates the device # and the cross-stream sync becomes pure overhead. Set to 0 to disable # the multi-stream path entirely. See #PR 41526 for the empirical result # for the default value of 1024 tokens. "VLLM_MULTI_STREAM_GEMM_TOKEN_THRESHOLD": lambda: int( os.getenv("VLLM_MULTI_STREAM_GEMM_TOKEN_THRESHOLD", "1024") ), # Format for saving torch.compile cache artifacts # - "binary": saves as binary file # Safe for multiple vllm serve processes accessing the same torch compile cache. # - "unpacked": saves as directory structure (for inspection/debugging) # NOT multiprocess safe - race conditions may occur with multiple processes. # Allows viewing and setting breakpoints in Inductor's code output files. "VLLM_COMPILE_CACHE_SAVE_FORMAT": env_with_choices( "VLLM_COMPILE_CACHE_SAVE_FORMAT", "binary", ["binary", "unpacked"] ), # Flag to control the v2 model runner. If unset, use config defaults. "VLLM_USE_V2_MODEL_RUNNER": lambda: maybe_convert_bool( os.getenv("VLLM_USE_V2_MODEL_RUNNER", None) ), # Log model inspection after loading. # If enabled, logs a transformers-style hierarchical view of the model # with quantization methods and attention backends. "VLLM_LOG_MODEL_INSPECTION": lambda: bool( int(os.getenv("VLLM_LOG_MODEL_INSPECTION", "0")) ), # Debug logging for --enable-mfu-metrics "VLLM_DEBUG_MFU_METRICS": lambda: bool( int(os.getenv("VLLM_DEBUG_MFU_METRICS", "0")) ), # Disable using pytorch's pin memory for CPU offloading. "VLLM_WEIGHT_OFFLOADING_DISABLE_PIN_MEMORY": lambda: bool( int(os.getenv("VLLM_WEIGHT_OFFLOADING_DISABLE_PIN_MEMORY", "0")) ), # Disable using UVA (Unified Virtual Addressing) for CPU offloading. "VLLM_WEIGHT_OFFLOADING_DISABLE_UVA": lambda: bool( int(os.getenv("VLLM_WEIGHT_OFFLOADING_DISABLE_UVA", "0")) ), # On WSL2 with a compatible kernel (>= 4.19.121), pinned memory is # supported but disabled by default due to a small performance regression. # Set to 1 when pinned memory or UVA is required (e.g. CPU offloading # or v2 model runner). "VLLM_WSL2_ENABLE_PIN_MEMORY": lambda: bool( int(os.getenv("VLLM_WSL2_ENABLE_PIN_MEMORY", "0")) ), # Disable logging of vLLM logo at server startup time. "VLLM_DISABLE_LOG_LOGO": lambda: bool(int(os.getenv("VLLM_DISABLE_LOG_LOGO", "0"))), # Disable PDL for LoRA, as enabling PDL with LoRA on SM100 causes # Triton compilation to fail. "VLLM_LORA_DISABLE_PDL": lambda: bool(int(os.getenv("VLLM_LORA_DISABLE_PDL", "0"))), # Enable CUDA compatibility mode for datacenter GPUs with older # driver versions than the CUDA toolkit major version of vLLM. "VLLM_ENABLE_CUDA_COMPATIBILITY": lambda: ( os.environ.get("VLLM_ENABLE_CUDA_COMPATIBILITY", "0").strip().lower() in ("1", "true") ), # Path to the CUDA compatibility libraries when CUDA compatibility is enabled. "VLLM_CUDA_COMPATIBILITY_PATH": lambda: os.environ.get( "VLLM_CUDA_COMPATIBILITY_PATH", None ), # Skip model name validation in OpenAI API requests. # When set to 1, any model name will be accepted in the 'model' field # of API requests. This is useful for proxy/gateway scenarios where # the actual model is served but different names may be used in requests. "VLLM_SKIP_MODEL_NAME_VALIDATION": lambda: ( os.getenv("VLLM_SKIP_MODEL_NAME_VALIDATION", "0").strip().lower() in ("1", "true") ), # Whether it is a scale up launch engine for elastic EP, # Should only be set by EngineCoreClient. "VLLM_ELASTIC_EP_SCALE_UP_LAUNCH": lambda: bool( int(os.getenv("VLLM_ELASTIC_EP_SCALE_UP_LAUNCH", "0")) ), # Whether to wait for all requests to drain before sending the # scaling command in elastic EP. "VLLM_ELASTIC_EP_DRAIN_REQUESTS": lambda: bool( int(os.getenv("VLLM_ELASTIC_EP_DRAIN_REQUESTS", "0")) ), # If set to 1, enable CUDA graph memory estimation during memory profiling. # This profiles CUDA graph memory usage to provide more accurate KV cache # memory allocation. Enabled by default as of v0.21.0 "VLLM_MEMORY_PROFILER_ESTIMATE_CUDAGRAPHS": lambda: bool( int(os.getenv("VLLM_MEMORY_PROFILER_ESTIMATE_CUDAGRAPHS", "1")) ), # NIXL EP environment variables "VLLM_NIXL_EP_MAX_NUM_RANKS": lambda: int( os.getenv("VLLM_NIXL_EP_MAX_NUM_RANKS", "32") ), # Whether enable XPU graph on Intel GPU "VLLM_XPU_ENABLE_XPU_GRAPH": lambda: bool( int(os.getenv("VLLM_XPU_ENABLE_XPU_GRAPH", "0")) ), # whether use xpu specific sample kernel "VLLM_XPU_USE_SAMPLER_KERNEL": lambda: bool( int(os.getenv("VLLM_XPU_USE_SAMPLER_KERNEL", "1")) ), # Enable simple KV offload. "VLLM_USE_SIMPLE_KV_OFFLOAD": lambda: bool( int(os.getenv("VLLM_USE_SIMPLE_KV_OFFLOAD", "0")) ), # Whether to enable dual cuda streams for LoRA computation # (used by both BaseLinearLayerWithLoRA and FusedMoEWithLoRA to # overlap the base layer compute with the LoRA fast path). "VLLM_LORA_ENABLE_DUAL_STREAM": lambda: bool( int(os.getenv("VLLM_LORA_ENABLE_DUAL_STREAM", "0")) ), # If set to 1, use Python spinloop extension to poll in a more efficient # way when using the mp backend. "VLLM_USE_SPINLOOP_EXT": lambda: bool(int(os.getenv("VLLM_USE_SPINLOOP_EXT", "0"))), # Comma-separated GPU_BDF=NIC_BDF pairs for RDMA NIC selection. # Must be set together with VLLM_NIC_SELECTION_VARS. "VLLM_GPU_NIC_PCIE_MAPPING": lambda: os.getenv("VLLM_GPU_NIC_PCIE_MAPPING", ""), # Comma-separated list of env vars to set from the GPU-NIC mapping. # Each entry is VAR_NAME or VAR_NAME: (suffix appended to # RDMA device name). Must be set together with VLLM_GPU_NIC_PCIE_MAPPING. "VLLM_NIC_SELECTION_VARS": lambda: os.getenv("VLLM_NIC_SELECTION_VARS", ""), } # --8<-- [end:env-vars-definition] def __getattr__(name: str): """ Gets environment variables lazily. NOTE: After enable_envs_cache() invocation (which triggered after service initialization), all environment variables will be cached. """ if name in environment_variables: return environment_variables[name]() raise AttributeError(f"module {__name__!r} has no attribute {name!r}") def _is_envs_cache_enabled() -> bool: """Checked if __getattr__ is wrapped with functools.cache""" global __getattr__ return hasattr(__getattr__, "cache_clear") def enable_envs_cache() -> None: """ Enables caching of environment variables. This is useful for performance reasons, as it avoids the need to re-evaluate environment variables on every call. NOTE: Currently, it's invoked after service initialization to reduce runtime overhead. This also means that environment variables should NOT be updated after the service is initialized. """ if _is_envs_cache_enabled(): # Avoid wrapping functools.cache multiple times return # Tag __getattr__ with functools.cache global __getattr__ __getattr__ = functools.cache(__getattr__) # Cache all environment variables for key in environment_variables: __getattr__(key) def disable_envs_cache() -> None: """ Resets the environment variables cache. It could be used to isolate environments between unit tests. """ global __getattr__ # If __getattr__ is wrapped by functions.cache, unwrap the caching layer. if _is_envs_cache_enabled(): assert hasattr(__getattr__, "__wrapped__") __getattr__ = __getattr__.__wrapped__ def __dir__(): return list(environment_variables.keys()) def is_set(name: str): """Check if an environment variable is explicitly set.""" if name in environment_variables: return name in os.environ raise AttributeError(f"module {__name__!r} has no attribute {name!r}") def validate_environ(hard_fail: bool) -> None: for env in os.environ: if env.startswith("VLLM_") and env not in environment_variables: if hard_fail: raise ValueError(f"Unknown vLLM environment variable detected: {env}") else: logger.warning("Unknown vLLM environment variable detected: %s", env) def compile_factors() -> dict[str, object]: """Return env vars used for torch.compile cache keys. Start with every known vLLM env var; drop entries in `ignored_factors`; hash everything else. This keeps the cache key aligned across workers.""" ignored_factors: set[str] = { "MAX_JOBS", "VLLM_RPC_BASE_PATH", "VLLM_USE_MODELSCOPE", "VLLM_RINGBUFFER_WARNING_INTERVAL", "VLLM_DEBUG_DUMP_PATH", "VLLM_PORT", "VLLM_CACHE_ROOT", # Runtime memory-plan persistence; does not affect compiled graphs. "VLLM_ENABLE_STARTUP_PLAN", "LD_LIBRARY_PATH", "VLLM_SERVER_DEV_MODE", "VLLM_DP_MASTER_IP", "VLLM_DP_MASTER_PORT", "VLLM_NIXL_SIDE_CHANNEL_HOST", "VLLM_RANDOMIZE_DP_DUMMY_INPUTS", "VLLM_CI_USE_S3", "VLLM_MODEL_REDIRECT_PATH", "VLLM_HOST_IP", "VLLM_FORCE_AOT_LOAD", "S3_ACCESS_KEY_ID", "S3_SECRET_ACCESS_KEY", "S3_ENDPOINT_URL", "VLLM_USAGE_STATS_SERVER", "VLLM_NO_USAGE_STATS", "VLLM_DO_NOT_TRACK", "VLLM_LOGGING_LEVEL", "VLLM_LOGGING_PREFIX", "VLLM_LOGGING_STREAM", "VLLM_LOGGING_CONFIG_PATH", "VLLM_LOGGING_COLOR", "VLLM_LOG_STATS_INTERVAL", "VLLM_DEBUG_LOG_API_SERVER_RESPONSE", "VLLM_TUNED_CONFIG_FOLDER", "VLLM_FLASHINFER_AUTOTUNE_CACHE_DIR", "VLLM_FLASHINFER_AUTOTUNE_SKIP_OPS", "VLLM_ENGINE_ITERATION_TIMEOUT_S", "VLLM_HTTP_TIMEOUT_KEEP_ALIVE", "VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS", "VLLM_WORKER_SHUTDOWN_TIMEOUT_SECONDS", "VLLM_KEEP_ALIVE_ON_ENGINE_DEATH", "VLLM_IMAGE_FETCH_TIMEOUT", "VLLM_VIDEO_FETCH_TIMEOUT", "VLLM_AUDIO_FETCH_TIMEOUT", "VLLM_MEDIA_CACHE", "VLLM_MEDIA_CACHE_MAX_SIZE_MB", "VLLM_MEDIA_CACHE_TTL_HOURS", "VLLM_MEDIA_FETCH_MAX_RETRIES", "VLLM_MEDIA_URL_ALLOW_REDIRECTS", "VLLM_MEDIA_LOADING_THREAD_COUNT", "VLLM_MAX_AUDIO_CLIP_FILESIZE_MB", "VLLM_MAX_AUDIO_DECODE_DURATION_S", "VLLM_MAX_AUDIO_PREPROCESS_WORKERS", "VLLM_MAX_IMAGE_PIXELS", "VLLM_VIDEO_LOADER_BACKEND", "VLLM_MEDIA_CONNECTOR", "VLLM_OBJECT_STORAGE_SHM_BUFFER_NAME", "VLLM_ASSETS_CACHE", "VLLM_ASSETS_CACHE_MODEL_CLEAN", "VLLM_WORKER_MULTIPROC_METHOD", "VLLM_ENABLE_V1_MULTIPROCESSING", "VLLM_V1_OUTPUT_PROC_CHUNK_SIZE", "VLLM_CPU_KVCACHE_SPACE", "VLLM_CPU_MOE_PREPACK", "VLLM_ZENTORCH_WEIGHT_PREPACK", "VLLM_TEST_FORCE_LOAD_FORMAT", "VLLM_ENABLE_CUDA_COMPATIBILITY", "VLLM_CUDA_COMPATIBILITY_PATH", "VLLM_SKIP_MODEL_NAME_VALIDATION", "LOCAL_RANK", "CUDA_VISIBLE_DEVICES", "NO_COLOR", } from vllm.config.utils import normalize_value factors: dict[str, object] = {} for factor, getter in environment_variables.items(): if factor in ignored_factors: continue try: raw = getter() except Exception as exc: # pragma: no cover - defensive logging logger.warning( "Skipping environment variable %s while hashing compile factors: %s", factor, exc, ) continue factors[factor] = normalize_value(raw) ray_noset_env_vars = [ # Refer to # https://github.com/ray-project/ray/blob/c584b1ea97b00793d1def71eaf81537d70efba42/python/ray/_private/accelerators/nvidia_gpu.py#L11 # https://github.com/ray-project/ray/blob/c584b1ea97b00793d1def71eaf81537d70efba42/python/ray/_private/accelerators/amd_gpu.py#L11 # https://github.com/ray-project/ray/blob/b97d21dab233c2bd8ed7db749a82a1e594222b5c/python/ray/_private/accelerators/amd_gpu.py#L10 # https://github.com/ray-project/ray/blob/c584b1ea97b00793d1def71eaf81537d70efba42/python/ray/_private/accelerators/npu.py#L12 # https://github.com/ray-project/ray/blob/c584b1ea97b00793d1def71eaf81537d70efba42/python/ray/_private/accelerators/hpu.py#L12 # https://github.com/ray-project/ray/blob/c584b1ea97b00793d1def71eaf81537d70efba42/python/ray/_private/accelerators/neuron.py#L14 # https://github.com/ray-project/ray/blob/c584b1ea97b00793d1def71eaf81537d70efba42/python/ray/_private/accelerators/tpu.py#L38 # https://github.com/ray-project/ray/blob/c584b1ea97b00793d1def71eaf81537d70efba42/python/ray/_private/accelerators/intel_gpu.py#L10 # https://github.com/ray-project/ray/blob/c584b1ea97b00793d1def71eaf81537d70efba42/python/ray/_private/accelerators/rbln.py#L10 "RAY_EXPERIMENTAL_NOSET_CUDA_VISIBLE_DEVICES", "RAY_EXPERIMENTAL_NOSET_ROCR_VISIBLE_DEVICES", "RAY_EXPERIMENTAL_NOSET_HIP_VISIBLE_DEVICES", "RAY_EXPERIMENTAL_NOSET_ASCEND_RT_VISIBLE_DEVICES", "RAY_EXPERIMENTAL_NOSET_HABANA_VISIBLE_MODULES", "RAY_EXPERIMENTAL_NOSET_NEURON_RT_VISIBLE_CORES", "RAY_EXPERIMENTAL_NOSET_TPU_VISIBLE_CHIPS", "RAY_EXPERIMENTAL_NOSET_ONEAPI_DEVICE_SELECTOR", "RAY_EXPERIMENTAL_NOSET_RBLN_RT_VISIBLE_DEVICES", ] for var in ray_noset_env_vars: factors[var] = normalize_value(os.getenv(var)) return factors