chore: import upstream snapshot with attribution

python/sglang/srt/utils/hf_transformers/__init__.py

@@ -0,0 +1,65 @@
+# Copyright 2023-2024 SGLang Team
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Hugging Face Transformers utilities.
+
+This package provides HF Transformers helpers, split into submodules
+(common, config, tokenizer, processor, mistral_utils).  Compatibility
+monkey-patches live in the sibling ``sglang.srt.utils.hf_transformers_patches``
+module and are applied at sglang import time.
+All public symbols are re-exported here for convenience.  The old import
+path ``sglang.srt.utils.hf_transformers_utils`` is preserved by a
+separate shim module.
+"""
+
+from ..hf_transformers_patches import normalize_rope_scaling_compat
+from .common import (
+    CONTEXT_LENGTH_KEYS,
+    AutoConfig,
+    attach_additional_stop_token_ids,
+    check_gguf_file,
+    download_from_hf,
+    get_context_length,
+    get_generation_config,
+    get_hf_text_config,
+    get_rope_config,
+    get_sparse_attention_config,
+    get_tokenizer_from_processor,
+)
+from .config import get_config
+from .processor import get_processor
+from .tokenizer import (
+    _fix_added_tokens_encoding,
+    _fix_v5_add_bos_eos_token,
+    get_tokenizer,
+)
+
+__all__ = [
+    "AutoConfig",
+    "CONTEXT_LENGTH_KEYS",
+    "_fix_added_tokens_encoding",
+    "_fix_v5_add_bos_eos_token",
+    "attach_additional_stop_token_ids",
+    "check_gguf_file",
+    "download_from_hf",
+    "get_config",
+    "get_context_length",
+    "get_generation_config",
+    "get_hf_text_config",
+    "get_processor",
+    "get_rope_config",
+    "get_sparse_attention_config",
+    "get_tokenizer",
+    "get_tokenizer_from_processor",
+    "normalize_rope_scaling_compat",
+]

python/sglang/srt/utils/hf_transformers/common.py

@@ -0,0 +1,499 @@
+# Copyright 2023-2024 SGLang Team
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Shared helpers used by config, tokenizer, and processor modules."""
+
+import json
+import os
+from pathlib import Path
+from typing import Any, Dict, Optional, Type, Union
+
+import torch
+from huggingface_hub import snapshot_download
+
+from sglang.srt.configs import (
+    AfmoeConfig,
+    BailingHybridConfig,
+    ChatGLMConfig,
+    DbrxConfig,
+    DeepseekVL2Config,
+    DotsOCRConfig,
+    DotsVLMConfig,
+    ExaoneConfig,
+    FalconH1Config,
+    GraniteMoeHybridConfig,
+    InternS2PreviewConfig,
+    JetNemotronConfig,
+    JetVLMConfig,
+    KimiK25Config,
+    KimiLinearConfig,
+    KimiVLConfig,
+    LagunaConfig,
+    LocateAnythingConfig,
+    LongcatFlashConfig,
+    MiniCPMV4_6Config,
+    MiniCPMV4_6VisionConfig,
+    MiniMaxM3VLConfig,
+    MultiModalityConfig,
+    NemotronH_Nano_Omni_Reasoning_V3_Config,
+    NemotronH_Nano_VL_V2_Config,
+    NemotronHConfig,
+    NemotronHPuzzleConfig,
+    Olmo3Config,
+    Qwen3_5Config,
+    Qwen3_5MoeConfig,
+    Qwen3NextConfig,
+    Step3p5Config,
+    Step3p7Config,
+    Step3VLConfig,
+)
+from sglang.srt.configs.deepseek_ocr import DeepseekVLV2Config
+from sglang.srt.configs.internvl import InternVLChatConfig
+from sglang.srt.utils import get_bool_env_var, logger, lru_cache_frozenset
+from sglang.srt.utils.runai_utils import ObjectStorageModel, is_runai_obj_uri
+
+from ..hf_transformers_patches import normalize_rope_scaling_compat
+
+if get_bool_env_var("SGLANG_USE_MODELSCOPE"):
+    from modelscope import AutoConfig, GenerationConfig
+else:
+    from transformers import AutoConfig, GenerationConfig
+
+from transformers import PretrainedConfig
+
+# ---------------------------------------------------------------------------
+# Config registry
+# ---------------------------------------------------------------------------
+
+_CONFIG_REGISTRY: Dict[str, Type[PretrainedConfig]] = {
+    cls.model_type: cls
+    for cls in [
+        AfmoeConfig,
+        BailingHybridConfig,
+        ChatGLMConfig,
+        DbrxConfig,
+        ExaoneConfig,
+        DeepseekVL2Config,
+        MultiModalityConfig,
+        KimiVLConfig,
+        LocateAnythingConfig,
+        InternVLChatConfig,
+        LagunaConfig,
+        Step3VLConfig,
+        LongcatFlashConfig,
+        Olmo3Config,
+        KimiLinearConfig,
+        Qwen3NextConfig,
+        FalconH1Config,
+        GraniteMoeHybridConfig,
+        DotsVLMConfig,
+        DotsOCRConfig,
+        NemotronH_Nano_VL_V2_Config,
+        NemotronH_Nano_Omni_Reasoning_V3_Config,
+        NemotronHConfig,
+        NemotronHPuzzleConfig,
+        DeepseekVLV2Config,
+        Qwen3_5Config,
+        Qwen3_5MoeConfig,
+        InternS2PreviewConfig,
+        JetNemotronConfig,
+        JetVLMConfig,
+        KimiK25Config,
+        Step3p5Config,
+        Step3p7Config,
+        MiniCPMV4_6Config,
+        MiniCPMV4_6VisionConfig,
+        MiniMaxM3VLConfig,
+    ]
+}
+
+# DeepSeek V3.2 / V4 reuse the V3 config schema. Subclass the upstream
+# transformers class with each model_type so AutoConfig.register passes its
+# consistency check (which requires class.model_type == registered key).
+# Default-value divergences (e.g. V4's topk_group) are handled in
+# model_config.py post-load.
+try:
+    from transformers import DeepseekV3Config as _HFDeepseekV3Config
+
+    class _DeepseekV32ConfigAlias(_HFDeepseekV3Config):
+        model_type = "deepseek_v32"
+
+    class _DeepseekV4ConfigAlias(_HFDeepseekV3Config):
+        model_type = "deepseek_v4"
+
+    _CONFIG_REGISTRY["deepseek_v32"] = _DeepseekV32ConfigAlias
+    _CONFIG_REGISTRY["deepseek_v4"] = _DeepseekV4ConfigAlias
+
+    # For kimi_k25_eagle3
+    class _KimiK2ConfigAlias(_HFDeepseekV3Config):
+        model_type = "kimi_k2"
+
+    _CONFIG_REGISTRY["kimi_k2"] = _KimiK2ConfigAlias
+except ImportError:
+    pass
+
+try:
+    from transformers import Gemma4Config as _HFGemma4Config
+
+    class _Gemma4UnifiedConfigAlias(_HFGemma4Config):
+        model_type = "gemma4_unified"
+
+    _CONFIG_REGISTRY["gemma4_unified"] = _Gemma4UnifiedConfigAlias
+except ImportError:
+    pass
+
+for name, cls in _CONFIG_REGISTRY.items():
+    try:
+        AutoConfig.register(name, cls)
+    except ValueError as e:
+        err = str(e).lower()
+        if "already registered" not in err and "already used" not in err:
+            logger.warning("Failed to register config %s: %s", name, e)
+
+
+# ---------------------------------------------------------------------------
+# Download / path helpers
+# ---------------------------------------------------------------------------
+
+
+def download_from_hf(
+    model_path: str,
+    allow_patterns: Optional[Union[str, list]] = None,
+):
+    if os.path.exists(model_path):
+        return model_path
+
+    if not allow_patterns:
+        allow_patterns = ["*.json", "*.bin", "*.model"]
+
+    return snapshot_download(model_path, allow_patterns=allow_patterns)
+
+
+def resolve_runai_obj_uri(model_name_or_path: str) -> str:
+    if is_runai_obj_uri(model_name_or_path):
+        return ObjectStorageModel.get_path(model_name_or_path)
+    return model_name_or_path
+
+
+def _resolve_local_or_cached_file(model_name_or_path, filename, revision=None):
+    """Resolve a file from a local directory or HF hub cache (no network)."""
+    local_path = Path(model_name_or_path) / filename
+    if local_path.is_file():
+        return str(local_path)
+    from huggingface_hub import hf_hub_download
+
+    return hf_hub_download(
+        model_name_or_path, filename, revision=revision, local_files_only=True
+    )
+
+
+def _cached_file_exists(model_name_or_path, filename, revision=None) -> bool:
+    """Whether *filename* is available locally or in the HF cache (no network)."""
+    try:
+        _resolve_local_or_cached_file(model_name_or_path, filename, revision)
+        return True
+    except Exception:
+        return False
+
+
+def _remote_file_exists(repo_id, filename, revision=None) -> bool:
+    """Whether *filename* exists on the HF hub (HEAD request only, no download).
+
+    Returns False on any error (offline, gated, network, invalid id) so callers
+    fall back to their default path instead of crashing.
+    """
+    from huggingface_hub.constants import HF_HUB_OFFLINE
+
+    if HF_HUB_OFFLINE:
+        return False
+    try:
+        from huggingface_hub import HfApi
+
+        return HfApi().file_exists(repo_id, filename, revision=revision)
+    except Exception:
+        return False
+
+
+def check_gguf_file(model: Union[str, os.PathLike]) -> bool:
+    model = Path(model)
+    if not model.is_file():
+        return False
+    elif model.suffix == ".gguf":
+        return True
+
+    with open(model, "rb") as f:
+        header = f.read(4)
+    return header == b"GGUF"
+
+
+# ---------------------------------------------------------------------------
+# Rope / text config helpers
+# ---------------------------------------------------------------------------
+
+
+def get_rope_config(config):
+    """Get (rope_theta, rope_params) from config, supporting both v4 and v5.
+
+    Trust-remote-code configs or parent configs passed to sub-models may not
+    have the v5 ``rope_parameters`` property, so we fall back to the v4-style
+    ``config.rope_theta`` / ``config.rope_scaling`` attributes.
+
+    Returns:
+        (rope_theta, rope_params): In v5, rope_params is the full
+        rope_parameters dict (which subsumes rope_scaling and includes
+        rope_theta). In v4, rope_params is the rope_scaling dict or None.
+    """
+    rope_params = getattr(config, "rope_parameters", None)
+    if rope_params is not None:
+        return rope_params["rope_theta"], rope_params
+    return getattr(config, "rope_theta", 10000), getattr(config, "rope_scaling", None)
+
+
+def _patch_text_config(parent_config: PretrainedConfig, text_config):
+    """Synchronize standard attributes between parent config and text sub-config.
+
+    In transformers v5, the "untangle config" refactor removed automatic
+    inheritance of top-level PretrainedConfig attributes (pad_token_id,
+    tie_word_embeddings, etc.) from sub-configs. Downstream code expects
+    these attributes to be present on both configs (some models pass the
+    parent directly to the language model, others pass the text sub-config),
+    so we propagate in both directions when an attribute is missing.
+    (See https://github.com/huggingface/transformers/pull/41541)
+    """
+    _ATTRS_TO_PROPAGATE = [
+        "pad_token_id",
+        "bos_token_id",
+        "eos_token_id",
+        "tie_word_embeddings",
+    ]
+    for attr in _ATTRS_TO_PROPAGATE:
+        parent_has = hasattr(parent_config, attr)
+        text_has = hasattr(text_config, attr)
+        if parent_has and not text_has:
+            setattr(text_config, attr, getattr(parent_config, attr))
+        elif text_has and not parent_has:
+            setattr(parent_config, attr, getattr(text_config, attr))
+    return text_config
+
+
+def get_hf_text_config(config: PretrainedConfig):
+    """Get the "sub" config relevant to llm for multi modal models.
+    No op for pure text models.
+    """
+    if config.architectures is not None:
+        class_name = config.architectures[0]
+        if class_name.startswith("Llava") and class_name.endswith("ForCausalLM"):
+            # We support non-hf version of llava models, so we do not want to
+            # read the wrong values from the unused default text_config.
+            # NOTE(HandH1998): We set `torch_dtype` of config to `torch.float16` for the weights, as
+            # `torch.float16` is default used for image features in `python/sglang/srt/models/llava.py`.
+            setattr(config, "dtype", torch.float16)
+            return config
+
+    text_config = None
+
+    # Some models (e.g. DeepSeek-OCR) store sub-configs as plain dicts.
+    # Convert to PretrainedConfig early so hasattr() checks and asserts work.
+    parent_dtype = getattr(config, "dtype", None)
+    for _attr in ("text_config", "llm_config", "language_config", "thinker_config"):
+        _sub = getattr(config, _attr, None)
+        if isinstance(_sub, dict):
+            _converted = PretrainedConfig(**_sub)
+            if getattr(_converted, "dtype", None) is None and parent_dtype is not None:
+                _converted.dtype = parent_dtype
+            setattr(config, _attr, _converted)
+        elif _sub is not None and parent_dtype is not None:
+            # transformers v5 multimodal configs (e.g. Mistral3Config) carry
+            # `dtype` only on the top-level config, leaving the sub-configs at
+            # None. Without this, _get_and_verify_dtype falls back to float32
+            # and then "auto" downcasts to float16, which overflows the Pixtral
+            # vision tower on real images and produces NaN features.
+            if getattr(_sub, "dtype", None) is None:
+                _sub.dtype = parent_dtype
+
+    # Priority: thinker_config > llm_config > language_config > text_config
+    if hasattr(config, "thinker_config"):
+        # qwen2.5 omni
+        thinker_config = config.thinker_config
+        if hasattr(thinker_config, "text_config"):
+            setattr(
+                thinker_config.text_config,
+                "dtype",
+                getattr(thinker_config, "dtype", None),
+            )
+            text_config = thinker_config.text_config
+        else:
+            text_config = thinker_config
+    elif hasattr(config, "llm_config"):
+        # PointsV1.5 Chat Model
+        assert hasattr(config.llm_config, "num_attention_heads")
+        text_config = config.llm_config
+    elif hasattr(config, "language_config"):
+        text_config = config.language_config
+    elif hasattr(config, "text_config"):
+        # The code operates under the assumption that text_config should have
+        # `num_attention_heads` (among others). Assert here to fail early
+        # if transformers config doesn't align with this assumption.
+        assert hasattr(config.text_config, "num_attention_heads")
+        text_config = config.text_config
+
+    # Ensure rope_scaling dicts have "type" for remote-code compat (v5).
+    normalize_rope_scaling_compat(config)
+
+    if text_config is not None:
+        return _patch_text_config(config, text_config)
+    return config
+
+
+# ---------------------------------------------------------------------------
+# Model-specific helpers
+# ---------------------------------------------------------------------------
+
+
+def _ensure_sub_configs(config: PretrainedConfig, *attr_names: str) -> None:
+    """Convert dict-valued sub-configs to proper AutoConfig objects in-place."""
+    for attr in attr_names:
+        sub = getattr(config, attr, None)
+        if sub is not None and isinstance(sub, dict):
+            setattr(config, attr, AutoConfig.for_model(**sub))
+
+
+def _is_deepseek_ocr_model(config: PretrainedConfig) -> bool:
+    # TODO: Remove this workaround once AutoConfig correctly identifies deepseek-ocr.
+    # Hugging Face's AutoConfig currently misidentifies it as deepseekvl2.
+    auto_map = getattr(config, "auto_map", None) or {}
+    return auto_map.get("AutoModel") == "modeling_deepseekocr.DeepseekOCRForCausalLM"
+
+
+def _is_deepseek_ocr2_model(config: PretrainedConfig) -> bool:
+    auto_map = getattr(config, "auto_map", None) or {}
+    return auto_map.get("AutoModel") == "modeling_deepseekocr2.DeepseekOCR2ForCausalLM"
+
+
+def _override_v_head_dim_if_zero(config: PretrainedConfig, patch: int = 128) -> None:
+    patched = False
+    for attr in ("text_config", "language_config"):
+        sub = getattr(config, attr, None)
+        if sub is None:
+            continue
+        if isinstance(sub, dict):
+            if sub.get("v_head_dim") == 0:
+                sub["v_head_dim"] = patch
+                patched = True
+        elif getattr(sub, "v_head_dim", None) == 0:
+            sub.v_head_dim = patch
+            patched = True
+    if patched:
+        logger.warning(
+            f"Overriding v_head_dim from 0 to {patch} to avoid potential issues."
+        )
+
+
+# ---------------------------------------------------------------------------
+# Context length / generation config / sparse attention
+# ---------------------------------------------------------------------------
+
+# Models don't use the same configuration key for determining the maximum
+# context length.  Store them here so we can sanely check them.
+# NOTE: The ordering here is important. Some models have two of these and we
+# have a preference for which value gets used.
+CONTEXT_LENGTH_KEYS = [
+    "max_sequence_length",
+    "seq_length",
+    "max_seq_len",
+    "model_max_length",
+    "max_position_embeddings",
+]
+
+
+def get_context_length(config):
+    """Get the context length of a model from a huggingface model configs."""
+    text_config = config
+    rope_scaling = getattr(text_config, "rope_scaling", None)
+    if rope_scaling:
+        rope_scaling_factor = rope_scaling.get("factor", 1)
+        if "original_max_position_embeddings" in rope_scaling:
+            rope_scaling_factor = 1
+        if rope_scaling.get("rope_type", None) == "llama3":
+            rope_scaling_factor = 1
+    else:
+        rope_scaling_factor = 1
+
+    for key in CONTEXT_LENGTH_KEYS:
+        val = getattr(text_config, key, None)
+        if val is not None:
+            return int(rope_scaling_factor * val)
+    return 2048
+
+
+@lru_cache_frozenset(maxsize=32)
+def get_generation_config(
+    model: str,
+    trust_remote_code: bool,
+    revision: Optional[str] = None,
+    **kwargs,
+):
+    try:
+        return GenerationConfig.from_pretrained(
+            model, trust_remote_code=trust_remote_code, revision=revision, **kwargs
+        )
+    except FileNotFoundError:
+        return None
+    except OSError as e:
+        logger.warning(
+            "Failed to load generation config for %s: %s. "
+            "Proceeding without generation config.",
+            model,
+            e,
+        )
+        return None
+
+
+# Qwen-1M related
+def get_sparse_attention_config(
+    model: str,
+    sparse_attention_config_filename: str = "sparse_attention_config.json",
+) -> Dict[str, Any]:
+    is_local = os.path.isdir(model)
+    if not is_local:
+        model = download_from_hf(model, allow_patterns=["*.json"])
+
+    config_file = os.path.join(model, sparse_attention_config_filename)
+    if not os.path.exists(config_file):
+        return {}
+
+    with open(config_file) as f:
+        config = json.load(f)
+    return config
+
+
+# ---------------------------------------------------------------------------
+# Tokenizer / processor helpers
+# ---------------------------------------------------------------------------
+
+
+# Some models don't have an available processor, e.g.: InternVL
+def get_tokenizer_from_processor(processor):
+    from transformers import PreTrainedTokenizerBase
+
+    if isinstance(processor, PreTrainedTokenizerBase):
+        return processor
+    return processor.tokenizer
+
+
+def attach_additional_stop_token_ids(tokenizer):
+    added = tokenizer.get_added_vocab()
+    if "<|eom_id|>" in added:
+        tokenizer.additional_stop_token_ids = {added["<|eom_id|>"]}
+    else:
+        tokenizer.additional_stop_token_ids = None

python/sglang/srt/utils/hf_transformers/config.py

@@ -0,0 +1,264 @@
+# Copyright 2023-2024 SGLang Team
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Config loading utilities."""
+
+from pathlib import Path
+from typing import Optional
+
+from transformers import PretrainedConfig
+from transformers.models.auto.modeling_auto import MODEL_FOR_CAUSAL_LM_MAPPING_NAMES
+
+from sglang.srt.configs.model_config_parser_registry import (
+    ModelConfigParserBase,
+    get_model_config_parser,
+    register_model_config_parser,
+)
+from sglang.srt.connector import create_remote_connector
+from sglang.srt.utils import is_remote_url, lru_cache_frozenset
+
+from ..hf_transformers_patches import _ensure_gguf_version
+from .common import (
+    _CONFIG_REGISTRY,
+    AutoConfig,
+    DeepseekVLV2Config,
+    _is_deepseek_ocr2_model,
+    _is_deepseek_ocr_model,
+    _override_v_head_dim_if_zero,
+    check_gguf_file,
+    get_hf_text_config,
+    resolve_runai_obj_uri,
+)
+from .mistral_utils import is_mistral_model, load_mistral_config
+
+
+def _set_architectures(config, arch_name):
+    config.update({"architectures": [arch_name]})
+
+
+def _apply_deepseek_ocr_overrides(config, model):
+    _override_v_head_dim_if_zero(config)
+    _set_architectures(config, "DeepseekOCRForCausalLM")
+    config._name_or_path = model
+
+
+_LONGCAT_ARCHS = {
+    "LongcatCausalLM",
+    "LongcatFlashForCausalLM",
+    "LongcatFlashNgramForCausalLM",
+}
+
+
+def _try_load_longcat_config(model, revision: Optional[str], **kwargs):
+    config_dict, _ = PretrainedConfig.get_config_dict(
+        model, revision=revision, **kwargs
+    )
+    architectures = config_dict.get("architectures") or []
+    if not any(arch in _LONGCAT_ARCHS for arch in architectures):
+        return None
+
+    return _CONFIG_REGISTRY["longcat_flash"].from_pretrained(
+        model, revision=revision, **kwargs
+    )
+
+
+@register_model_config_parser("hf")
+class HfModelConfigParser(ModelConfigParserBase):
+    def parse(
+        self,
+        model,
+        trust_remote_code: bool,
+        revision: Optional[str] = None,
+        **kwargs,
+    ):
+        config = _try_load_longcat_config(model, revision, **kwargs)
+        if config is None:
+            config = AutoConfig.from_pretrained(
+                model,
+                trust_remote_code=trust_remote_code,
+                revision=revision,
+                **kwargs,
+            )
+
+        if (
+            config.architectures is not None
+            and config.architectures[0] == "Phi4MMForCausalLM"
+        ):
+            from transformers import SiglipVisionConfig
+
+            config.vision_config = SiglipVisionConfig(
+                hidden_size=1152,
+                image_size=448,
+                intermediate_size=4304,
+                model_type="siglip_vision_model",
+                num_attention_heads=16,
+                num_hidden_layers=26,
+                patch_size=14,
+            )
+
+        if config.architectures in [
+            ["LongcatCausalLM"],
+            ["LongcatFlashForCausalLM"],
+            ["LongcatFlashNgramForCausalLM"],
+        ]:
+            config.model_type = "longcat_flash"
+
+        text_config = get_hf_text_config(config=config)
+
+        if isinstance(model, str) and text_config is not None:
+            items = (
+                text_config.items()
+                if hasattr(text_config, "items")
+                else vars(text_config).items()
+            )
+            for key, val in items:
+                if not hasattr(config, key) and val is not None:
+                    setattr(config, key, val)
+
+        is_ocr = _is_deepseek_ocr_model(config)
+        is_ocr2 = _is_deepseek_ocr2_model(config)
+
+        if is_ocr2:
+            _override_v_head_dim_if_zero(config)
+            config.model_type = "deepseek-ocr"
+            _set_architectures(config, "DeepseekOCRForCausalLM")
+            config = DeepseekVLV2Config.from_pretrained(model, revision=revision)
+            _apply_deepseek_ocr_overrides(config, model)
+        elif config.model_type in _CONFIG_REGISTRY:
+            model_type = config.model_type
+            if model_type == "deepseek_vl_v2" and is_ocr:
+                model_type = "deepseek-ocr"
+            config = _CONFIG_REGISTRY[model_type].from_pretrained(
+                model, revision=revision
+            )
+
+            # Re-check after reloading config from registry
+            if _is_deepseek_ocr_model(config) or _is_deepseek_ocr2_model(config):
+                _apply_deepseek_ocr_overrides(config, model)
+            else:
+                config._name_or_path = model
+
+        if isinstance(model, str) and config.model_type == "internvl_chat":
+            for key, val in config.llm_config.__dict__.items():
+                if not hasattr(config, key):
+                    setattr(config, key, val)
+
+        if config.model_type == "multi_modality":
+            _set_architectures(config, "MultiModalityCausalLM")
+
+        if config.model_type in (
+            "gemma4",
+            "gemma4_assistant",
+            "gemma4_unified",
+            "gemma4_unified_assistant",
+        ):
+            # Gemma4 configs use base attributes for SWA layers and `global_*`
+            # variants for full-attention layers.  SGLang expects the opposite:
+            # base = full-attention, `swa_*` = sliding-window overrides.
+            text_config = config.text_config
+            global_head_dim = getattr(text_config, "global_head_dim", None)
+            global_kv_heads = getattr(text_config, "num_global_key_value_heads", None)
+
+            swa_head_dim = text_config.head_dim
+            swa_kv_heads = text_config.num_key_value_heads
+
+            text_config.swa_head_dim = swa_head_dim
+            text_config.swa_v_head_dim = swa_head_dim
+            text_config.swa_num_key_value_heads = swa_kv_heads
+
+            if global_head_dim is not None:
+                text_config.head_dim = global_head_dim
+            if global_kv_heads is not None:
+                text_config.num_key_value_heads = global_kv_heads
+
+            if not hasattr(text_config, "v_head_dim"):
+                text_config.v_head_dim = text_config.head_dim
+            if not hasattr(text_config, "swa_v_head_dim"):
+                text_config.swa_v_head_dim = text_config.swa_head_dim
+
+            # Unified Gemma4 names the end-of-audio token `eoa_token_index`,
+            # but the multimodal processor expects `eoa_token_id`.
+            if not hasattr(config, "eoa_token_id") and hasattr(
+                config, "eoa_token_index"
+            ):
+                config.eoa_token_id = config.eoa_token_index
+
+        if config.model_type == "longcat_flash":
+            _set_architectures(config, "LongcatFlashForCausalLM")
+
+        return config
+
+
+@register_model_config_parser("mistral")
+class MistralModelConfigParser(ModelConfigParserBase):
+    def parse(
+        self,
+        model,
+        trust_remote_code: bool,
+        revision: Optional[str] = None,
+        **kwargs,
+    ):
+        del kwargs
+        return load_mistral_config(
+            model, trust_remote_code=trust_remote_code, revision=revision
+        )
+
+
+@lru_cache_frozenset(maxsize=32)
+def get_config(
+    model: str,
+    trust_remote_code: bool,
+    revision: Optional[str] = None,
+    model_override_args: Optional[dict] = None,
+    model_config_parser: str = "auto",
+    **kwargs,
+):
+    is_gguf = check_gguf_file(model)
+    if is_gguf:
+        if model_config_parser not in ("auto", "hf"):
+            raise ValueError(
+                f"model_config_parser={model_config_parser!r} is incompatible "
+                "with GGUF inputs; only 'hf' (or 'auto') is supported."
+            )
+        _ensure_gguf_version()
+        kwargs["gguf_file"] = model
+        model = Path(model).parent
+        # Skip auto-resolution for GGUF: the name-based Mistral heuristic
+        # would misfire on the rewritten parent dir.
+        model_config_parser = "hf"
+
+    model = resolve_runai_obj_uri(model)
+
+    if is_remote_url(model):
+        client = create_remote_connector(model)
+        client.pull_files(ignore_pattern=["*.pt", "*.safetensors", "*.bin"])
+        model = client.get_local_dir()
+
+    if model_config_parser == "auto":
+        # `model` is post-rewrite (gguf parent / runai uri / remote pull).
+        model_config_parser = "mistral" if is_mistral_model(model) else "hf"
+
+    parser = get_model_config_parser(model_config_parser)
+    config = parser.parse(
+        model, trust_remote_code=trust_remote_code, revision=revision, **kwargs
+    )
+
+    if model_override_args:
+        config.update(model_override_args)
+
+    if is_gguf:
+        if config.model_type not in MODEL_FOR_CAUSAL_LM_MAPPING_NAMES:
+            raise RuntimeError(f"Can't get gguf config for {config.model_type}.")
+        _set_architectures(config, MODEL_FOR_CAUSAL_LM_MAPPING_NAMES[config.model_type])
+
+    return config

python/sglang/srt/utils/hf_transformers/mistral_utils.py

@@ -0,0 +1,637 @@
+# Adapted from https://github.com/vllm-project/vllm/blob/main/vllm/transformers_utils/configs/mistral.py
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+import json
+import tempfile
+from functools import lru_cache
+from pathlib import Path
+from typing import Any, Optional
+
+from transformers import AutoConfig, PretrainedConfig, WhisperConfig
+
+from sglang.srt.utils import logger
+
+from .common import (
+    _cached_file_exists,
+    _ensure_sub_configs,
+    _remote_file_exists,
+    download_from_hf,
+)
+
+
+def adapt_config_dict(
+    config_dict: dict[str, Any], model: str, **kwargs
+) -> tuple[dict, PretrainedConfig]:
+    config_dict.update(kwargs)
+    config_dict = _remap_general_mistral_args(config_dict)
+
+    if bool(config_dict.get("quantization")):
+        config_dict = _remap_mistral_quantization_args(config_dict)
+
+    is_moe = bool(config_dict.get("moe"))
+    is_mistral_large_3 = (
+        is_moe and (config_dict["moe"].get("num_shared_experts") or 0) > 0
+    )
+    is_eagle = "eagle" in model.lower()
+    is_mla_eagle = is_eagle and any(
+        config_dict.get(k) is not None
+        for k in ("kv_lora_rank", "q_lora_rank", "v_head_dim")
+    )
+    if is_eagle and not is_moe and is_mla_eagle:
+        # Dense MLA EAGLE draft model (e.g. Mistral Small 4 EAGLE).
+        # Uses MLA attention like MistralLarge3 but has no MoE layers.
+        # Set model_type to deepseek_v3 for MLA support, and override
+        # MoE fields so all layers are dense.
+        config_dict["model_type"] = "deepseek_v3"
+        config_dict["architectures"] = ["MistralLarge3ForCausalLMEagle"]
+        num_layers = config_dict.get("num_hidden_layers", 0)
+        config_dict["n_routed_experts"] = 1
+        config_dict["first_k_dense_replace"] = num_layers
+        config_dict["moe_layer_freq"] = 1
+        config_dict["n_shared_experts"] = 0
+        config_dict["n_group"] = 1
+        config_dict["topk_group"] = 1
+        config_dict["num_experts_per_tok"] = 1
+        config_dict["moe_intermediate_size"] = 1
+        config_dict["routed_scaling_factor"] = 1.0
+        config_dict["topk_method"] = None
+        config_dict["scoring_func"] = "softmax"
+        config_dict["routing_method_type"] = 1
+    elif is_eagle and not is_moe:
+        # Dense GQA EAGLE draft model (e.g. Mistral Medium 3.5 EAGLE).
+        # Routes to a Llama-backbone draft body — no MoE shimming required.
+        config_dict["architectures"] = ["MistralForCausalLMEagle"]
+        config_dict["model_type"] = "mistral"
+        config_dict["rope_is_neox_style"] = False
+        for mla_key in (
+            "q_lora_rank",
+            "qk_rope_head_dim",
+            "qk_nope_head_dim",
+            "kv_lora_rank",
+            "v_head_dim",
+        ):
+            if config_dict.get(mla_key) is None:
+                config_dict.pop(mla_key, None)
+    elif is_moe:
+        if is_mistral_large_3:
+            config_dict = _remap_moe_args(config_dict)
+            config_dict["model_type"] = "deepseek_v3"
+            if is_eagle:
+                config_dict["architectures"] = ["MistralLarge3ForCausalLMEagle"]
+            else:
+                config_dict["architectures"] = ["MistralLarge3ForCausalLM"]
+
+            assert (
+                "llama_4_scaling" in config_dict
+            ), "MistralLarge3 expect llama4 scaling config."
+            llama_4_scaling_config_keys = ["original_max_position_embeddings", "beta"]
+            assert all(
+                [
+                    key in config_dict["llama_4_scaling"]
+                    for key in llama_4_scaling_config_keys
+                ]
+            ), (
+                "llama_4_scaling config should define the keys: "
+                f"{','.join(llama_4_scaling_config_keys)}"
+            )
+        else:
+            config_dict["architectures"] = ["MixtralForCausalLM"]
+    else:
+        config_dict["architectures"] = ["MistralForCausalLM"]
+        config_dict["model_type"] = "mistral"
+        # Mistral models use non-interleaved RoPE (is_neox_style=False),
+        # unlike Llama which defaults to True.
+        config_dict["rope_is_neox_style"] = False
+        # Remove None-valued MLA fields that would shadow defaults in
+        # model_config._derive_model_shapes (getattr returns None instead
+        # of the fallback when the attribute exists but is None).
+        for mla_key in (
+            "q_lora_rank",
+            "qk_rope_head_dim",
+            "qk_nope_head_dim",
+            "kv_lora_rank",
+            "v_head_dim",
+        ):
+            if config_dict.get(mla_key) is None:
+                config_dict.pop(mla_key, None)
+
+    if bool(config_dict.get("yarn")):
+        config_dict = _remap_mistral_yarn_args(config_dict)
+
+    is_vision = bool(
+        (config_dict.get("multimodal") or {}).get("vision_encoder_args")
+        or config_dict.get("vision_encoder")
+    )
+    is_audio = bool(
+        ((config_dict.get("multimodal") or {}).get("whisper_model_args") or {}).get(
+            "encoder_args"
+        )
+    )
+
+    assert not (is_vision and is_audio), "Vision and audio are mutually exclusive"
+
+    if is_vision:
+        config_dict = _remap_mistral_vision_args(config_dict)
+    if is_audio:
+        config_dict = _remap_mistral_audio_args(config_dict)
+
+    config = PretrainedConfig.from_dict(config_dict)
+
+    logger.debug("Initialized config %s", config)
+
+    return config_dict, config
+
+
+def _remap_mistral_vision_args(config: dict) -> dict:
+    if config.get("multimodal"):
+        vision_config = config.pop("multimodal")
+    else:
+        vision_config = config.pop("vision_encoder")
+
+    quant_config = config.get("quantization_config")
+
+    config = {
+        "model_type": "pixtral",
+        "architectures": ["PixtralForConditionalGeneration"],
+        "text_config": config,
+        "vision_config": {"model_type": "pixtral", **vision_config},
+    }
+    if quant_config:
+        config["quantization_config"] = quant_config
+    return config
+
+
+def _remap_mistral_yarn_args(config: dict) -> dict:
+    yarn_config_map = {
+        "factor": "factor",
+        "original_max_position_embeddings": "original_max_position_embeddings",
+        "beta": "beta_fast",
+        "alpha": "beta_slow",
+        "apply_scale": "apply_yarn_scaling",
+    }
+    yarn_config = config.get("yarn") or {}
+    config["rope_scaling"] = {
+        "rope_type": "deepseek_yarn",
+        "mscale_all_dim": 1,
+    }
+    # Include rope_theta in rope_scaling if present at the top level,
+    # as transformers yarn validation requires it.
+    if "rope_theta" in config:
+        config["rope_scaling"]["rope_theta"] = config["rope_theta"]
+    for old_name, new_name in yarn_config_map.items():
+        if old_name in yarn_config:
+            value = yarn_config.pop(old_name)
+            if new_name is not None:
+                config["rope_scaling"][new_name] = value
+
+    assert len(yarn_config) == 0, f"Unparsed yarn config: {yarn_config}"
+
+    return config
+
+
+def _remap_general_mistral_args(config: dict) -> dict:
+    # Mistral key -> HF key
+    config_mapping = {
+        "dim": "hidden_size",
+        "norm_eps": "rms_norm_eps",
+        "n_kv_heads": "num_key_value_heads",
+        "n_layers": "num_hidden_layers",
+        "n_heads": "num_attention_heads",
+        "hidden_dim": "intermediate_size",
+    }
+    # HF key -> (Mistral key, default value)
+    top_level_mapping_with_default = {
+        "model_type": ("model_type", "transformer"),
+        "hidden_act": ("activation", "silu"),
+        "tie_word_embeddings": ("tied_embeddings", False),
+        "max_seq_len": ("max_seq_len", 128_000),
+        "max_position_embeddings": ("max_position_embeddings", 128_000),
+    }
+
+    for key, new_key in config_mapping.items():
+        if key in config:
+            config[new_key] = config.pop(key)
+
+    for new_key, (key, default_value) in top_level_mapping_with_default.items():
+        config[new_key] = config.pop(key, default_value)
+
+    return config
+
+
+def _remap_mistral_quantization_args(config: dict) -> dict:
+    if config.get("quantization"):
+        quantization = config.pop("quantization", {})
+        if quantization.get("qformat_weight") == "fp8_e4m3":
+            qscheme_act = quantization.get("qscheme_act")
+            assert qscheme_act in (
+                "NO_SCALES",
+                "TENSOR",
+                None,
+            ), "Only NO_SCALES and TENSOR (default) are supported for qscheme_act"
+            is_dynamic = qscheme_act == "NO_SCALES"
+            config["quantization_config"] = {
+                "quant_method": "fp8",
+                "activation_scheme": "dynamic" if is_dynamic else "static",
+            }
+        else:
+            raise ValueError(f"Found unknown quantization='{quantization}' in config")
+
+    return config
+
+
+def _remap_mistral_audio_args(config: dict) -> dict:
+    whisper_args = config["multimodal"].pop("whisper_model_args")
+    encoder_args = whisper_args["encoder_args"]
+    downsample_args = whisper_args["downsample_args"]
+
+    quant_config = config.get("quantization_config")
+    config = {
+        "model_type": "whixtral",
+        "architectures": ["VoxtralForConditionalGeneration"],
+        "text_config": PretrainedConfig.from_dict(config),
+        "audio_config": WhisperConfig(
+            num_mel_bins=encoder_args["audio_encoding_args"]["num_mel_bins"],
+            window_size=encoder_args["audio_encoding_args"]["window_size"],
+            sampling_rate=encoder_args["audio_encoding_args"]["sampling_rate"],
+            hop_length=encoder_args["audio_encoding_args"]["hop_length"],
+            downsample_factor=downsample_args["downsample_factor"],
+            d_model=encoder_args["dim"],
+            encoder_layers=encoder_args["n_layers"],
+            encoder_ffn_dim=encoder_args["hidden_dim"],
+            encoder_attention_heads=encoder_args["n_heads"],
+            vocab_size=encoder_args["vocab_size"],
+            max_source_positions=encoder_args["max_source_positions"],
+            is_encoder_decoder=False,  # Override WhisperConfig default
+        ),
+    }
+    if quant_config:
+        config["quantization_config"] = quant_config
+    return config
+
+
+def _remap_moe_args(config: dict) -> dict:
+    moe_config_map = {
+        "route_every_n": "moe_layer_freq",
+        "first_k_dense_replace": "first_k_dense_replace",
+        "num_experts_per_tok": "num_experts_per_tok",
+        "num_experts": "n_routed_experts",
+        "expert_hidden_dim": "moe_intermediate_size",
+        "routed_scale": "routed_scaling_factor",
+        "num_shared_experts": "n_shared_experts",
+        "num_expert_groups": "n_group",
+        "num_expert_groups_per_tok": "topk_group",
+    }
+    moe_config = config.get("moe", {})
+    for old_name, new_name in moe_config_map.items():
+        if old_name in moe_config:
+            value = moe_config.pop(old_name)
+            config[new_name] = value
+
+    config["topk_method"] = None
+    config["scoring_func"] = "softmax"
+    config["routing_method_type"] = 1  # RoutingMethodType.Renormalize
+
+    return config
+
+
+class MistralConfigParser:
+    def get_hf_file_to_dict(
+        self, file_name: str, model: str | Path, revision: str | None = "main"
+    ):
+        file_path = Path(model) / file_name
+        if not file_path.is_file():
+            raise FileNotFoundError(f"File not found {model}, {file_name}")
+
+        with open(file_path) as file:
+            return json.load(file)
+
+    def _download_mistral_config_file(self, model, revision) -> dict:
+        config_file_name = "params.json"
+        config_dict = self.get_hf_file_to_dict(config_file_name, model, revision)
+        if config_dict is None:
+            raise ValueError(
+                f"Failed to load mistral '{config_file_name}' config for model "
+                f"{model}. Please check if the model is a mistral-format model "
+                f"and if the config file exists."
+            )
+        assert isinstance(config_dict, dict)
+        return config_dict
+
+    def parse(
+        self,
+        model: str | Path,
+        revision: str | None = None,
+        **kwargs,
+    ) -> tuple[dict, PretrainedConfig]:
+        config_dict = self._download_mistral_config_file(model, revision)
+        if config_dict.get("max_position_embeddings") is None:
+            logger.warning(
+                "The params.json file is missing 'max_position_embeddings'"
+                " and could not get a value from the HF config."
+                " Defaulting to 128000"
+            )
+            config_dict["max_position_embeddings"] = 128_000
+
+        config_dict, config = adapt_config_dict(config_dict, model)
+
+        # Mistral configs may define sliding_window as list[int]. Convert it
+        # to int and add the layer_types list[str] to make it HF compatible
+        if (sliding_window := getattr(config, "sliding_window", None)) and isinstance(
+            sliding_window, list
+        ):
+            pattern_repeats = config.num_hidden_layers // len(sliding_window)
+            layer_types = sliding_window * pattern_repeats
+            config.layer_types = [
+                "full_attention" if layer_type is None else "sliding_attention"
+                for layer_type in layer_types
+            ]
+            config.sliding_window = next(filter(None, sliding_window), None)
+
+        return config_dict, config
+
+
+def is_mistral_model(name) -> bool:
+    """Return True if *name* refers to a Mistral model needing the custom parser."""
+    lower = str(name).lower()
+    if "mistral-large-3" in lower or "mistral-small-4" in lower or "leanstral" in lower:
+        return True
+    # EAGLE drafts for Mistral targets ship native-format only (params.json +
+    # consolidated.safetensors, no config.json), so route them through the
+    # custom parser regardless of the base model name.
+    if "eagle" in lower and "mistral" in lower:
+        return True
+    return False
+
+
+@lru_cache(maxsize=2)
+def load_mistral_config(
+    model_path: str,
+    trust_remote_code: bool = False,
+    revision: Optional[str] = None,
+):
+    """Load and parse a Mistral model config via the custom params.json format.
+
+    Returns a ``PretrainedConfig`` with dict sub-configs (text_config,
+    vision_config) converted to proper AutoConfig objects.
+    """
+    local_path = download_from_hf(model_path)
+    parser = MistralConfigParser()
+    config_dict, _ = parser.parse(local_path)
+
+    with tempfile.NamedTemporaryFile(mode="w+", suffix=".json") as f:
+        json.dump(config_dict, f)
+        f.flush()
+        loaded_config = AutoConfig.from_pretrained(
+            f.name, trust_remote_code=trust_remote_code, revision=revision
+        )
+    _ensure_sub_configs(loaded_config, "text_config", "vision_config")
+
+    return loaded_config
+
+
+def wrap_as_pixtral(processor, config):
+    """Wrap a tokenizer as a PixtralProcessor for Mistral vision models."""
+    from transformers.models.pixtral.image_processing_pixtral import (
+        PixtralImageProcessor,
+    )
+    from transformers.models.pixtral.processing_pixtral import (
+        PixtralProcessor as HFPixtralProcessor,
+    )
+
+    vision_config = config.vision_config
+    patch_size = vision_config.patch_size
+    image_size = vision_config.image_size
+    spatial_merge_size = getattr(vision_config, "spatial_merge_size", 1)
+
+    effective_patch = patch_size * spatial_merge_size
+    image_processor = PixtralImageProcessor(
+        do_resize=True,
+        size={"longest_edge": image_size},
+        patch_size={"height": effective_patch, "width": effective_patch},
+    )
+    return HFPixtralProcessor(
+        image_processor=image_processor,
+        tokenizer=processor,
+        patch_size=patch_size,
+        spatial_merge_size=spatial_merge_size,
+    )
+
+
+# kwargs that MistralCommon tokenizers reject.
+_MISTRAL_COMMON_REJECTED_KWARGS = frozenset(
+    {
+        "trust_remote_code",
+        "tokenizer_revision",
+        "use_fast",
+        "_from_auto",
+        "clean_up_tokenization_spaces",
+    }
+)
+
+# Models whose tokenizer should be loaded from a different checkpoint.
+_MISTRAL_TOKENIZER_REDIRECTS = {
+    # TODO(Xinyuan): Remove this once we have a proper tokenizer for Devstral
+    "mistralai/Devstral-Small-2505": "mistralai/Mistral-Small-3.1-24B-Instruct-2503",
+}
+
+
+def is_bare_tekken_checkpoint(tokenizer_name, revision=None) -> bool:
+    """True iff the checkpoint ships tekken.json but no tokenizer.json.
+
+    AutoTokenizer converts tekken.json on the fly, but the converter assigns
+    BPE ids from rank 0, dropping the 1000 special-token slots that precede
+    the BPE vocab in tekken's id space — every encoded id is shifted and
+    generation produces garbage. Such checkpoints must load through the
+    mistral-common backed tokenizer instead.
+    """
+
+    local_dir = Path(tokenizer_name)
+    if local_dir.is_dir():
+        return (local_dir / "tekken.json").is_file() and not (
+            local_dir / "tokenizer.json"
+        ).is_file()
+
+    if _cached_file_exists(tokenizer_name, "tokenizer.json", revision):
+        return False
+    if _cached_file_exists(tokenizer_name, "tekken.json", revision):
+        return True
+
+    # Cold cache: the tokenizer loads before weights, so tekken.json isn't
+    # cached yet on a first launch — HEAD-probe the hub to still detect it.
+    if not _remote_file_exists(tokenizer_name, "tekken.json", revision):
+        return False
+    return not _remote_file_exists(tokenizer_name, "tokenizer.json", revision)
+
+
+def retry_without_mistral_common_kwargs(tokenizer_name, *args, **common_kwargs):
+    """Retry ``AutoTokenizer.from_pretrained`` without kwargs that MistralCommon rejects.
+
+    Returns the loaded tokenizer, or *None* if the error is not a
+    MistralCommon kwargs rejection.
+    """
+    from transformers import AutoTokenizer
+
+    stripped = {
+        k: v
+        for k, v in common_kwargs.items()
+        if k not in _MISTRAL_COMMON_REJECTED_KWARGS
+    }
+    return AutoTokenizer.from_pretrained(tokenizer_name, *args, **stripped)
+
+
+def patch_mistral_common_tokenizer(tokenizer):
+    """Patch MistralCommonTokenizer/Backend to be compatible with HF tokenizer API.
+
+    MistralCommon tokenizers (used by Voxtral, Pixtral, etc.) reject several
+    standard kwargs and lack some attributes that sglang expects.  We wrap the
+    offending methods once at load time so that the rest of the codebase does
+    not need any special-casing.
+    """
+    cls_name = type(tokenizer).__name__
+    if "MistralCommon" not in cls_name:
+        return tokenizer
+    if getattr(tokenizer, "_mistral_common_patched", False):
+        return tokenizer
+    tokenizer._mistral_common_patched = True
+
+    if not hasattr(tokenizer, "get_added_vocab"):
+        tokenizer.get_added_vocab = lambda: {}
+
+    # Keep the old no-op pad add working on transformers 5.12 MistralCommon.
+    _orig_add_special_tokens = tokenizer.add_special_tokens
+
+    def _safe_add_special_tokens(special_tokens_dict, *args, **kwargs):
+        if set(special_tokens_dict) == {"pad_token"}:
+            tokenizer.pad_token = special_tokens_dict["pad_token"]
+            return 0
+        return _orig_add_special_tokens(special_tokens_dict, *args, **kwargs)
+
+    tokenizer.add_special_tokens = _safe_add_special_tokens
+
+    # Set a chat_template containing "audio" so that sglang's content format
+    # detector returns "openai" (which preserves audio_url extraction).
+    if not hasattr(tokenizer, "chat_template") or tokenizer.chat_template is None:
+        tokenizer.chat_template = "<!-- audio/image multimodal -->"
+
+    _orig_convert = tokenizer.convert_tokens_to_ids
+
+    def _safe_convert(val):
+        try:
+            return _orig_convert(val)
+        except AssertionError:
+            logger.debug(
+                "convert_tokens_to_ids failed for %r, returning unk_token_id", val
+            )
+            return getattr(tokenizer, "unk_token_id", None)
+
+    tokenizer.convert_tokens_to_ids = _safe_convert
+
+    def _drop_kwargs(fn, keys):
+        def wrapper(*args, **kwargs):
+            for k in keys:
+                kwargs.pop(k, None)
+            return fn(*args, **kwargs)
+
+        return wrapper
+
+    tokenizer.decode = _drop_kwargs(tokenizer.decode, ["spaces_between_special_tokens"])
+    tokenizer.batch_decode = _drop_kwargs(
+        tokenizer.batch_decode, ["spaces_between_special_tokens"]
+    )
+
+    if hasattr(tokenizer, "_text_to_ids"):
+        _orig_text_to_ids = tokenizer._text_to_ids
+        marker_to_id = {
+            "[IMG]": tokenizer.convert_tokens_to_ids("[IMG]"),
+            "[IMG_BREAK]": tokenizer.convert_tokens_to_ids("[IMG_BREAK]"),
+            "[IMG_END]": tokenizer.convert_tokens_to_ids("[IMG_END]"),
+        }
+
+        def _text_to_ids_with_pixtral_markers(text, add_special_tokens):
+            if not isinstance(text, str) or not any(
+                marker in text for marker in marker_to_id
+            ):
+                return _orig_text_to_ids(text, add_special_tokens)
+
+            ids = []
+            pos = 0
+            while pos < len(text):
+                next_marker = None
+                next_idx = len(text)
+                for marker in marker_to_id:
+                    marker_idx = text.find(marker, pos)
+                    if marker_idx != -1 and marker_idx < next_idx:
+                        next_marker = marker
+                        next_idx = marker_idx
+
+                if next_marker is None:
+                    ids.extend(_orig_text_to_ids(text[pos:], False))
+                    break
+                if next_idx > pos:
+                    ids.extend(_orig_text_to_ids(text[pos:next_idx], False))
+                ids.append(marker_to_id[next_marker])
+                pos = next_idx + len(next_marker)
+
+            if add_special_tokens:
+                return tokenizer.build_inputs_with_special_tokens(ids)
+            return ids
+
+        tokenizer._text_to_ids = _text_to_ids_with_pixtral_markers
+
+    tokenizer._orig_apply_chat_template = tokenizer.apply_chat_template
+
+    def _adapt_placeholder_content_for_mistral_common(content):
+        if not isinstance(content, list):
+            return content
+
+        rendered_parts = []
+        has_placeholder = False
+        for part in content:
+            if not isinstance(part, dict):
+                return content
+            part_type = part.get("type")
+            if part_type in ("text", "input_text"):
+                rendered_parts.append(part.get("text", ""))
+            elif part_type == "image" and not any(
+                key in part for key in ("url", "path", "base64")
+            ):
+                has_placeholder = True
+                rendered_parts.append("[IMG]")
+            elif part_type in ("audio", "video") and not any(
+                key in part for key in ("url", "path", "base64")
+            ):
+                has_placeholder = True
+                continue
+            else:
+                return content
+
+        return "".join(rendered_parts) if has_placeholder else content
+
+    def _adapt_placeholder_messages_for_mistral_common(messages):
+        if not isinstance(messages, (list, tuple)):
+            return messages
+
+        adapted = []
+        for msg in messages:
+            if isinstance(msg, (list, tuple)):
+                adapted.append(_adapt_placeholder_messages_for_mistral_common(msg))
+            elif isinstance(msg, dict):
+                adapted.append(
+                    {
+                        **msg,
+                        "content": _adapt_placeholder_content_for_mistral_common(
+                            msg.get("content", "")
+                        ),
+                    }
+                )
+            else:
+                adapted.append(msg)
+        return adapted
+
+    def _safe_apply_chat_template(messages, **kwargs):
+        kwargs.pop("add_generation_prompt", None)
+        messages = _adapt_placeholder_messages_for_mistral_common(messages)
+        return tokenizer._orig_apply_chat_template(messages, **kwargs)
+
+    tokenizer.apply_chat_template = _safe_apply_chat_template
+    return tokenizer

python/sglang/srt/utils/hf_transformers/processor.py

@@ -0,0 +1,306 @@
+# Copyright 2023-2024 SGLang Team
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Processor loading utilities."""
+
+import json
+from pathlib import Path
+from typing import Optional
+
+from transformers import (
+    AutoProcessor,
+    AutoTokenizer,
+    PreTrainedTokenizerBase,
+)
+
+from sglang.srt.multimodal.customized_mm_processor_utils import _CUSTOMIZED_MM_PROCESSOR
+from sglang.srt.utils import logger
+
+from .common import (
+    AutoConfig,
+    _is_deepseek_ocr2_model,
+    _is_deepseek_ocr_model,
+    _override_v_head_dim_if_zero,
+    _resolve_local_or_cached_file,
+    attach_additional_stop_token_ids,
+    download_from_hf,
+    get_tokenizer_from_processor,
+    resolve_runai_obj_uri,
+)
+from .mistral_utils import (
+    is_mistral_model,
+    load_mistral_config,
+    patch_mistral_common_tokenizer,
+    wrap_as_pixtral,
+)
+from .tokenizer import (
+    _TOKENIZERS_BACKEND,
+    _fix_added_tokens_encoding,
+    _fix_special_tokens_pattern,
+)
+
+
+def _build_processor_manually(
+    model_path, config, trust_remote_code, revision, **kwargs
+):
+    """Build processor when AutoProcessor fails to resolve feature_extractor_type.
+
+    In transformers v5, AutoProcessor.from_pretrained calls
+    AutoFeatureExtractor.from_pretrained which fails if
+    preprocessor_config.json lacks 'feature_extractor_type'. This resolves
+    the processor class via dynamic module resolution and constructs it with
+    individually-loaded components.
+    """
+    import transformers
+    from transformers import AutoImageProcessor, AutoTokenizer
+    from transformers.dynamic_module_utils import get_class_from_dynamic_module
+
+    # Resolve processor class from auto_map -- check both the model config
+    # and the preprocessor_config.json (some models like MiniCPM-o only
+    # declare AutoProcessor in the latter).
+    auto_map = getattr(config, "auto_map", None) or {}
+    proc_ref = auto_map.get("AutoProcessor")
+    if not proc_ref:
+        try:
+            pp_file = _resolve_local_or_cached_file(
+                model_path, "preprocessor_config.json", revision
+            )
+            with open(pp_file) as f:
+                pp_auto_map = json.load(f).get("auto_map", {})
+            proc_ref = pp_auto_map.get("AutoProcessor")
+        except (OSError, json.JSONDecodeError, ValueError) as e:
+            logger.warning(
+                "_build_processor_manually: could not read preprocessor_config.json "
+                "for %s: %s",
+                model_path,
+                e,
+            )
+    if not proc_ref:
+        raise ValueError(f"Cannot determine processor class for {model_path}")
+
+    proc_cls = get_class_from_dynamic_module(
+        proc_ref, model_path, code_revision=revision
+    )
+
+    # Load sub-components individually (these succeed)
+    tokenizer = AutoTokenizer.from_pretrained(
+        model_path, trust_remote_code=trust_remote_code, revision=revision
+    )
+    init_kwargs = {"tokenizer": tokenizer}
+
+    if "image_processor" in getattr(proc_cls, "attributes", []):
+        try:
+            init_kwargs["image_processor"] = AutoImageProcessor.from_pretrained(
+                model_path, trust_remote_code=trust_remote_code, revision=revision
+            )
+        except (ImportError, OSError, ValueError) as e:
+            raise RuntimeError(
+                f"Failed to load image_processor for {model_path}: {e}. "
+                f"This model requires an image processor for multimodal features. "
+                f"Check that the model files are complete and accessible."
+            ) from e
+
+    # Instantiate feature extractor from its declared class
+    fe_class_name = getattr(proc_cls, "feature_extractor_class", None)
+    if fe_class_name:
+        fe_class = getattr(transformers, fe_class_name, None)
+        if fe_class is not None:
+            try:
+                init_kwargs["feature_extractor"] = fe_class()
+            except TypeError as e:
+                logger.warning(
+                    "Cannot instantiate feature extractor %s with no arguments "
+                    "for %s: %s",
+                    fe_class_name,
+                    model_path,
+                    e,
+                )
+        else:
+            logger.warning(
+                "Feature extractor class %s not found in transformers for %s",
+                fe_class_name,
+                model_path,
+            )
+
+    return proc_cls(**init_kwargs)
+
+
+def get_processor(
+    tokenizer_name: str,
+    *args,
+    tokenizer_mode: str = "auto",
+    trust_remote_code: bool = False,
+    tokenizer_revision: Optional[str] = None,
+    use_fast: Optional[bool] = True,
+    tokenizer_backend: str = "huggingface",
+    model_name: Optional[str] = None,
+    **kwargs,
+):
+    if tokenizer_backend == "fastokens":
+        from .tokenizer import _ensure_fastokens_patched
+
+        _ensure_fastokens_patched()
+
+    revision = kwargs.pop("revision", tokenizer_revision)
+    tokenizer_name = resolve_runai_obj_uri(tokenizer_name)
+
+    if is_mistral_model(tokenizer_name):
+        config = load_mistral_config(
+            tokenizer_name,
+            trust_remote_code=trust_remote_code,
+            revision=revision,
+        )
+    elif model_name is not None:
+        config = AutoConfig.from_pretrained(
+            model_name,
+            trust_remote_code=trust_remote_code,
+            revision=revision,
+            **kwargs,
+        )
+    else:
+        config = AutoConfig.from_pretrained(
+            tokenizer_name,
+            trust_remote_code=trust_remote_code,
+            revision=revision,
+            **kwargs,
+        )
+    is_ocr2 = _is_deepseek_ocr2_model(config)
+    if _is_deepseek_ocr_model(config) or is_ocr2:
+        config.model_type = "deepseek-ocr"
+        config.update({"architectures": ["DeepseekOCRForCausalLM"]})
+        if is_ocr2:
+            _override_v_head_dim_if_zero(config)
+
+    if config.model_type in {"qwen2_vl", "sarashina2_vision"}:
+        if "size" not in kwargs:
+            kwargs["size"] = {"shortest_edge": 3136, "longest_edge": 1003520}
+
+    if config.model_type not in {"llava", "clip"}:
+        kwargs["use_fast"] = use_fast
+    try:
+        if "InternVL3_5" in tokenizer_name:
+            processor = AutoTokenizer.from_pretrained(
+                tokenizer_name,
+                *args,
+                trust_remote_code=trust_remote_code,
+                revision=revision,
+                **kwargs,
+            )
+        else:
+            if config.model_type in _CUSTOMIZED_MM_PROCESSOR:
+                processor = _CUSTOMIZED_MM_PROCESSOR[config.model_type].from_pretrained(
+                    tokenizer_name,
+                    *args,
+                    trust_remote_code=trust_remote_code,
+                    revision=revision,
+                    **kwargs,
+                )
+            else:
+                processor = AutoProcessor.from_pretrained(
+                    tokenizer_name,
+                    *args,
+                    trust_remote_code=trust_remote_code,
+                    revision=revision,
+                    **kwargs,
+                )
+
+    except ValueError as e:
+        error_message = str(e)
+        if "does not have a slow version" in error_message:
+            logger.info(
+                "Processor %s does not have a slow version. Automatically use fast version",
+                tokenizer_name,
+            )
+            kwargs["use_fast"] = True
+            processor = AutoProcessor.from_pretrained(
+                tokenizer_name,
+                *args,
+                trust_remote_code=trust_remote_code,
+                revision=revision,
+                **kwargs,
+            )
+        elif "Unrecognized feature extractor" in error_message:
+            logger.info(
+                "AutoProcessor failed on feature extractor for %s, "
+                "constructing processor manually",
+                tokenizer_name,
+            )
+            processor = _build_processor_manually(
+                tokenizer_name,
+                config,
+                trust_remote_code,
+                revision,
+                **kwargs,
+            )
+        elif (
+            "are not supported by" in error_message and "MistralCommon" in error_message
+        ):
+            logger.info(
+                "AutoProcessor for %s rejected standard kwargs, "
+                "retrying without trust_remote_code/use_fast",
+                tokenizer_name,
+            )
+            kwargs.pop("use_fast", None)
+            kwargs.pop("_from_auto", None)
+            processor = AutoProcessor.from_pretrained(
+                tokenizer_name,
+                *args,
+                revision=revision,
+                **kwargs,
+            )
+        else:
+            raise
+    if (
+        isinstance(processor, PreTrainedTokenizerBase)
+        and getattr(config, "model_type", None) == "pixtral"
+    ):
+        processor = wrap_as_pixtral(processor, config)
+
+    tokenizer = get_tokenizer_from_processor(processor)
+
+    # AutoProcessor may internally create a TokenizersBackend tokenizer
+    # (same issue as get_tokenizer). Replace it with a properly loaded one.
+    if type(tokenizer).__name__ == _TOKENIZERS_BACKEND:
+        from .tokenizer import get_tokenizer
+
+        logger.warning(
+            "Processor tokenizer for %s is TokenizersBackend, "
+            "reloading via get_tokenizer",
+            tokenizer_name,
+        )
+        tokenizer = get_tokenizer(
+            tokenizer_name,
+            tokenizer_mode=tokenizer_mode,
+            trust_remote_code=trust_remote_code,
+            tokenizer_revision=revision,
+            tokenizer_backend=tokenizer_backend,
+        )
+        if isinstance(processor, PreTrainedTokenizerBase):
+            processor = tokenizer
+        else:
+            processor.tokenizer = tokenizer
+
+    if tokenizer.chat_template is None:
+        local_path = download_from_hf(
+            tokenizer_name, allow_patterns=["*.json", "*.jinja", "*.model"]
+        )
+        jinja_path = Path(local_path) / "chat_template.jinja"
+        if jinja_path.is_file():
+            tokenizer.chat_template = jinja_path.read_text()
+            logger.info("Loaded chat_template from %s", jinja_path)
+
+    patch_mistral_common_tokenizer(tokenizer)
+    _fix_special_tokens_pattern(tokenizer)
+    _fix_added_tokens_encoding(tokenizer)
+    attach_additional_stop_token_ids(tokenizer)
+    return processor

python/sglang/srt/utils/hf_transformers/tokenizer.py

@@ -0,0 +1,613 @@
+# Copyright 2023-2024 SGLang Team
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Tokenizer loading utilities."""
+
+import json
+import logging
+import warnings
+from pathlib import Path
+from typing import Optional, Union
+
+from transformers import (
+    AutoTokenizer,
+    PreTrainedTokenizer,
+    PreTrainedTokenizerFast,
+)
+
+from sglang.srt.connector import create_remote_connector
+from sglang.srt.utils import is_remote_url, logger
+from sglang.srt.utils.patch_tokenizer import patch_tokenizer
+
+from ..hf_transformers_patches import _ensure_gguf_version
+from .common import (
+    _resolve_local_or_cached_file,
+    attach_additional_stop_token_ids,
+    check_gguf_file,
+    resolve_runai_obj_uri,
+)
+from .mistral_utils import (
+    _MISTRAL_TOKENIZER_REDIRECTS,
+    is_bare_tekken_checkpoint,
+    patch_mistral_common_tokenizer,
+    retry_without_mistral_common_kwargs,
+)
+
+# A fast LLaMA tokenizer with the pre-processed `tokenizer.json` file.
+_FAST_LLAMA_TOKENIZER = "hf-internal-testing/llama-tokenizer"
+
+# Class name used by transformers v5 when no tokenizer mapping exists for a model_type.
+_TOKENIZERS_BACKEND = "TokenizersBackend"
+
+
+def _load_tokenizer_by_declared_class(tokenizer_name, *args, **kwargs):
+    """Load tokenizer by the class declared in tokenizer_config.json.
+
+    AutoTokenizer resolves to TokenizersBackend when the model's config
+    model_type has no tokenizer class mapping (e.g. deepseek_vl_v2), even
+    though tokenizer_config.json declares a standard class like
+    LlamaTokenizerFast.  Returns None if it cannot improve on AutoTokenizer.
+    """
+    import transformers
+
+    try:
+        revision = kwargs.get("revision") or kwargs.get("tokenizer_revision")
+        config_file = _resolve_local_or_cached_file(
+            tokenizer_name, "tokenizer_config.json", revision
+        )
+        with open(config_file) as f:
+            tok_config = json.load(f)
+        tok_class_name = tok_config.get("tokenizer_class")
+    except FileNotFoundError:
+        return None
+    except (OSError, json.JSONDecodeError) as e:
+        logger.debug(
+            "Failed to read tokenizer_config.json for %s: %s", tokenizer_name, e
+        )
+        return None
+
+    if not tok_class_name:
+        return None
+
+    # Skip base classes that don't implement required methods (e.g. get_vocab)
+    if tok_class_name in ("PreTrainedTokenizer", "PreTrainedTokenizerBase"):
+        return None
+
+    tok_cls = getattr(transformers, tok_class_name, None)
+    if tok_cls is None and kwargs.get("trust_remote_code"):
+        # Class not in transformers — try loading via auto_map.
+        try:
+            auto_map = tok_config.get("auto_map", {})
+            auto_tok_ref = auto_map.get("AutoTokenizer")
+            if isinstance(auto_tok_ref, (list, tuple)):
+                auto_tok_ref = auto_tok_ref[0]
+            if auto_tok_ref:
+                from transformers.dynamic_module_utils import (
+                    get_class_from_dynamic_module,
+                )
+
+                tok_cls = get_class_from_dynamic_module(
+                    auto_tok_ref,
+                    tokenizer_name,
+                    code_revision=revision,
+                )
+        except (OSError, ImportError, ValueError, RuntimeError) as e:
+            logger.debug("Dynamic module lookup for %s failed: %s", tok_class_name, e)
+    if tok_cls is None:
+        return None
+
+    logger.debug(
+        "Loading tokenizer for %s directly as %s (bypassing AutoTokenizer)",
+        tokenizer_name,
+        tok_class_name,
+    )
+    try:
+        return tok_cls.from_pretrained(tokenizer_name, *args, **kwargs)
+    except (OSError, ValueError, TypeError, ImportError) as e:
+        logger.warning(
+            "Direct load as %s failed for %s: %s. "
+            "Falling back to AutoTokenizer result.",
+            tok_class_name,
+            tokenizer_name,
+            e,
+        )
+        return None
+
+
+# Filter warnings like: https://github.com/sgl-project/sglang/issues/8082
+class TokenizerWarningsFilter(logging.Filter):
+    def filter(self, record: logging.LogRecord) -> bool:
+        return "Calling super().encode with" not in record.getMessage()
+
+
+# ---------------------------------------------------------------------------
+# Helpers for get_tokenizer
+# ---------------------------------------------------------------------------
+
+
+def _resolve_tokenizer_name(tokenizer_name, kwargs):
+    """Resolve special name formats (GGUF, remote URLs, etc.) to a local path.
+
+    May mutate *kwargs* (e.g. to add ``gguf_file``).
+    """
+    tokenizer_name = _MISTRAL_TOKENIZER_REDIRECTS.get(tokenizer_name, tokenizer_name)
+
+    if check_gguf_file(tokenizer_name):
+        _ensure_gguf_version()
+        kwargs["gguf_file"] = tokenizer_name
+        tokenizer_name = Path(tokenizer_name).parent
+
+    tokenizer_name = resolve_runai_obj_uri(tokenizer_name)
+
+    if is_remote_url(tokenizer_name):
+        # BaseConnector implements __del__() to clean up the local dir.
+        # Since config files need to exist all the time, so we DO NOT use
+        # with statement to avoid closing the client.
+        client = create_remote_connector(tokenizer_name)
+        client.pull_files(ignore_pattern=["*.pt", "*.safetensors", "*.bin"])
+        tokenizer_name = client.get_local_dir()
+
+    return tokenizer_name
+
+
+def _auto_tokenizer_from_pretrained(tokenizer_name, *args, **common_kwargs):
+    """Call ``AutoTokenizer.from_pretrained`` with error handling."""
+    try:
+        tokenizer = AutoTokenizer.from_pretrained(
+            tokenizer_name, *args, **common_kwargs
+        )
+        logging.getLogger(tokenizer.__class__.__module__).addFilter(
+            TokenizerWarningsFilter()
+        )
+        return tokenizer
+    except TypeError as e:
+        err_msg = (
+            "Failed to load the tokenizer. If you are using a LLaMA V1 model "
+            f"consider using '{_FAST_LLAMA_TOKENIZER}' instead of the "
+            "original tokenizer."
+        )
+        raise RuntimeError(err_msg) from e
+    except ValueError as e:
+        # MistralCommon tokenizers reject standard HF kwargs like
+        # trust_remote_code, use_fast etc. Retry without them.
+        if "are not supported by" in str(e) and "MistralCommon" in str(e):
+            return retry_without_mistral_common_kwargs(
+                tokenizer_name, *args, **common_kwargs
+            )
+        # If the error pertains to the tokenizer class not existing or not
+        # currently being imported, suggest using the --trust-remote-code flag.
+        if not common_kwargs.get("trust_remote_code") and (
+            "does not exist or is not currently imported." in str(e)
+            or "requires you to execute the tokenizer file" in str(e)
+        ):
+            err_msg = (
+                "Failed to load the tokenizer. If the tokenizer is a custom "
+                "tokenizer not yet available in the HuggingFace transformers "
+                "library, consider setting `trust_remote_code=True` in LLM "
+                "or using the `--trust-remote-code` flag in the CLI."
+            )
+            raise RuntimeError(err_msg) from e
+        raise
+
+
+def _resolve_tokenizers_backend(tokenizer_name, *args, **common_kwargs):
+    """Resolve generic ``TokenizersBackend`` to a proper tokenizer class.
+
+    In transformers v5, ``AutoTokenizer`` falls back to ``TokenizersBackend``
+    when the model_type has no tokenizer mapping.  This retries with
+    ``use_fast=False``, then attempts loading by the class declared in
+    ``tokenizer_config.json``.  May still return a ``TokenizersBackend``
+    if all retries fail (with a warning).
+    """
+    logger.debug(
+        "Tokenizer loaded as generic TokenizersBackend for %s, "
+        "retrying with use_fast=False",
+        tokenizer_name,
+    )
+    common_kwargs = {**common_kwargs, "use_fast": False}
+    try:
+        tokenizer = AutoTokenizer.from_pretrained(
+            tokenizer_name, *args, **common_kwargs
+        )
+    except (ValueError, TypeError, OSError, ImportError, RuntimeError) as e:
+        raise RuntimeError(
+            f"Retry with use_fast=False for {tokenizer_name} also failed "
+            f"(initial load returned TokenizersBackend): {e}"
+        ) from e
+
+    if type(tokenizer).__name__ == _TOKENIZERS_BACKEND:
+        tokenizer = (
+            _load_tokenizer_by_declared_class(tokenizer_name, *args, **common_kwargs)
+            or tokenizer
+        )
+
+    if type(tokenizer).__name__ == _TOKENIZERS_BACKEND:
+        if common_kwargs.get("trust_remote_code"):
+            logger.warning(
+                "Tokenizer for %s is still TokenizersBackend after retries "
+                "with --trust-remote-code. Model-specific tokenizer attributes "
+                "may be missing.",
+                tokenizer_name,
+            )
+        else:
+            logger.debug(
+                "Tokenizer for %s loaded as generic TokenizersBackend. "
+                "Set --trust-remote-code to load the model-specific tokenizer.",
+                tokenizer_name,
+            )
+
+    return tokenizer
+
+
+# ---------------------------------------------------------------------------
+# Post-load fixups
+# ---------------------------------------------------------------------------
+
+
+def _fix_v5_tokenizer_components(tokenizer, model_name_or_path, revision=None):
+    """Fix pre_tokenizer/decoder when a v5 tokenizer class overwrites them.
+
+    In transformers v5, some tokenizer classes (e.g. LlamaTokenizer) have a
+    custom __init__ that rebuilds the pre_tokenizer and decoder from scratch
+    with class-specific components, discarding the originals from tokenizer.json.
+    This breaks models that specify LlamaTokenizerFast but actually use a
+    different tokenizer architecture (e.g. DeepSeek-V3.2 uses ByteLevel).
+
+    Detects the mismatch by comparing against the raw tokenizer.json and
+    restores the original components when they differ.
+    """
+    backend = getattr(tokenizer, "_tokenizer", None)
+    if backend is None:
+        return
+
+    try:
+        from tokenizers import Tokenizer as RawTokenizer
+
+        tok_file = _resolve_local_or_cached_file(
+            model_name_or_path, "tokenizer.json", revision
+        )
+        raw = RawTokenizer.from_file(tok_file)
+    except FileNotFoundError:
+        return
+    except (OSError, ValueError, RuntimeError) as e:
+        logger.warning(
+            "_fix_v5_tokenizer_components: unexpected error loading tokenizer.json "
+            "for %s, v5 component fix will not be applied: %s",
+            model_name_or_path,
+            e,
+        )
+        return
+
+    raw_pre = type(raw.pre_tokenizer).__name__ if raw.pre_tokenizer else None
+    loaded_pre = type(backend.pre_tokenizer).__name__ if backend.pre_tokenizer else None
+
+    if raw_pre and loaded_pre and raw_pre != loaded_pre:
+        logger.info(
+            "Fixing v5 tokenizer component mismatch for %s: "
+            "pre_tokenizer %s -> %s, decoder %s -> %s",
+            model_name_or_path,
+            loaded_pre,
+            raw_pre,
+            type(backend.decoder).__name__ if backend.decoder else None,
+            type(raw.decoder).__name__ if raw.decoder else None,
+        )
+        backend.pre_tokenizer = raw.pre_tokenizer
+        backend.decoder = raw.decoder
+
+
+def _fix_v5_add_bos_eos_token(tokenizer, model_name_or_path, revision=None):
+    """Restore add_bos_token/add_eos_token stripped by transformers v5.
+
+    In transformers v5, _from_pretrained() strips add_bos_token and
+    add_eos_token from init kwargs when a tokenizer.json file is present,
+    assuming the tokenizer.json post-processor handles BOS/EOS addition.
+    However, many models (e.g. DeepSeek-V3) have a tokenizer.json whose
+    post-processor does NOT add BOS/EOS, and rely on the add_bos_token flag
+    from tokenizer_config.json instead. This causes silent accuracy regressions.
+
+    This function reads the tokenizer_config.json and restores the values,
+    but only for tokenizer classes that actually supported these flags in v4.
+    Classes like Qwen2Tokenizer did not support add_bos_token/add_eos_token
+    in v4, so restoring them would change behavior.
+    """
+    # In transformers v4, only certain tokenizer classes supported
+    # add_bos_token / add_eos_token as init parameters.  Restoring these
+    # flags for classes that never supported them (e.g. Qwen2Tokenizer)
+    # would incorrectly change tokenization behavior.
+    _V4_CLASSES_WITH_BOS_EOS_FLAGS = frozenset(
+        {
+            "LlamaTokenizer",
+            "LlamaTokenizerFast",
+            "CodeLlamaTokenizer",
+            "CodeLlamaTokenizerFast",
+            "GemmaTokenizer",
+            "GemmaTokenizerFast",
+            "CohereTokenizerFast",
+        }
+    )
+
+    try:
+        config_file = _resolve_local_or_cached_file(
+            model_name_or_path, "tokenizer_config.json", revision
+        )
+        with open(config_file) as f:
+            config = json.load(f)
+    except FileNotFoundError:
+        return
+    except (OSError, json.JSONDecodeError, ValueError) as e:
+        logger.warning(
+            "_fix_v5_add_bos_eos_token: failed to read tokenizer_config.json "
+            "for %s, BOS/EOS token restoration will not be applied: %s",
+            model_name_or_path,
+            e,
+        )
+        return
+
+    tokenizer_class = config.get("tokenizer_class", "")
+    if tokenizer_class not in _V4_CLASSES_WITH_BOS_EOS_FLAGS:
+        logger.debug(
+            "_fix_v5_add_bos_eos_token: skipping %s (tokenizer_class=%s "
+            "did not support add_bos/eos_token in v4)",
+            model_name_or_path,
+            tokenizer_class,
+        )
+        return
+
+    # In v4, Llama/Gemma tokenizers defaulted add_bos_token=True.
+    # When the config omits the key or has null, use the v4 default so that
+    # update_post_processor() doesn't drop BOS/EOS that was there before.
+    _V4_DEFAULTS = {"add_bos_token": True, "add_eos_token": False}
+
+    changed = False
+    for attr in ("add_bos_token", "add_eos_token"):
+        config_val = config.get(attr)
+        if config_val is None:
+            # Key missing or null -> use v4 default for this tokenizer class
+            config_val = _V4_DEFAULTS.get(attr, False)
+        # Fast tokenizers in v4 used tokenizer.json post-processor for EOS —
+        # the add_eos_token Python attribute was set but the post-processor
+        # came from tokenizer.json, not from the attribute.  In v5, the flag is
+        # stripped and both sglang and HF reference end up with add_eos_token=False.
+        # Restoring add_eos_token for fast tokenizers makes sglang diverge from
+        # the HF reference, breaking embedding models like e5-mistral-7b-instruct.
+        if attr == "add_eos_token" and isinstance(tokenizer, PreTrainedTokenizerFast):
+            config_val = _V4_DEFAULTS["add_eos_token"]  # False
+        current_val = getattr(tokenizer, attr, None)
+        if current_val != config_val:
+            logger.info(
+                "Restoring %s=%s for %s (was %s after v5 loading)",
+                attr,
+                config_val,
+                model_name_or_path,
+                current_val,
+            )
+            # Set the private backing attribute (not the property) because
+            # transformers tokenizers expose add_bos/eos_token as properties
+            # that read from the underscore-prefixed attribute.
+            setattr(tokenizer, f"_{attr}", config_val)
+            changed = True
+
+    # Rebuild the post-processor so it respects the restored flags
+    if changed and hasattr(tokenizer, "update_post_processor"):
+        tokenizer.update_post_processor()
+
+
+def _fix_special_tokens_pattern(tokenizer):
+    """Fix https://github.com/huggingface/transformers/pull/42563 which defaults
+    special_tokens_pattern to "cls_sep", inserting None into token IDs when
+    cls_token/sep_token are undefined (e.g. Kimi-VL's TikTokenTokenizer).
+    """
+    pattern = getattr(tokenizer, "special_tokens_pattern", None)
+    if pattern == "cls_sep" and (
+        tokenizer.cls_token_id is None or tokenizer.sep_token_id is None
+    ):
+        tokenizer.special_tokens_pattern = "none"
+
+
+def _apply_post_load_fixes(tokenizer, tokenizer_name, revision):
+    """Apply all post-load patches and return the final tokenizer."""
+    _fix_v5_tokenizer_components(tokenizer, tokenizer_name, revision)
+    _fix_v5_add_bos_eos_token(tokenizer, tokenizer_name, revision)
+
+    if not isinstance(tokenizer, PreTrainedTokenizerFast):
+        warnings.warn(
+            "Using a slow tokenizer. This might cause a significant "
+            "slowdown. Consider using a fast tokenizer instead."
+        )
+
+    patch_mistral_common_tokenizer(tokenizer)
+    _fix_special_tokens_pattern(tokenizer)
+    attach_additional_stop_token_ids(tokenizer)
+    return patch_tokenizer(tokenizer)
+
+
+# ---------------------------------------------------------------------------
+# Public entry point
+# ---------------------------------------------------------------------------
+
+
+_fastokens_patched = False
+
+
+def _ensure_fastokens_patched():
+    """Monkey-patch transformers to use the fastokens backend (once)."""
+    global _fastokens_patched
+    if _fastokens_patched:
+        return
+    try:
+        import fastokens
+    except ImportError:
+        raise ImportError(
+            "The fastokens package is required when --tokenizer-backend=fastokens. "
+            "Install it with: pip install 'sglang[fastokens]'"
+        ) from None
+
+    fastokens.patch_transformers()
+    _fastokens_patched = True
+    logger.info("fastokens backend enabled - transformers patched successfully")
+
+
+def get_tokenizer(
+    tokenizer_name: str,
+    *args,
+    tokenizer_mode: str = "auto",
+    trust_remote_code: bool = False,
+    tokenizer_revision: Optional[str] = None,
+    tokenizer_backend: str = "huggingface",
+    **kwargs,
+) -> Union[PreTrainedTokenizer, PreTrainedTokenizerFast]:
+    """Gets a tokenizer for the given model name via Huggingface."""
+    # Tiktoken format has its own backend — no fastokens patching needed.
+    if tokenizer_name.endswith(".json"):
+        from sglang.srt.tokenizer.tiktoken_tokenizer import TiktokenTokenizer
+
+        return TiktokenTokenizer(tokenizer_name)
+
+    if tokenizer_backend == "fastokens":
+        _ensure_fastokens_patched()
+
+    if tokenizer_mode == "slow":
+        if kwargs.get("use_fast", False):
+            raise ValueError("Cannot use the fast tokenizer in slow tokenizer mode.")
+        kwargs["use_fast"] = False
+    elif tokenizer_mode == "auto":
+        # Transformers v5 AutoTokenizer ignores use_fast (always fast), but
+        # some code paths pass kwargs to non-AutoTokenizer loaders where
+        # use_fast still matters. Set explicitly for those fallback paths.
+        if "use_fast" not in kwargs:
+            kwargs["use_fast"] = True
+
+    tokenizer_name = _resolve_tokenizer_name(tokenizer_name, kwargs)
+
+    common_kwargs = dict(
+        trust_remote_code=trust_remote_code,
+        tokenizer_revision=tokenizer_revision,
+        clean_up_tokenization_spaces=False,
+        **kwargs,
+    )
+
+    try:
+        if is_bare_tekken_checkpoint(tokenizer_name, tokenizer_revision):
+            from transformers.tokenization_mistral_common import (
+                MistralCommonTokenizer,
+            )
+
+            logger.info(
+                "Detected bare-tekken checkpoint %s (tekken.json, no "
+                "tokenizer.json); loading via mistral-common MistralCommonTokenizer, "
+                "ignoring tokenizer_backend=%r.",
+                tokenizer_name,
+                tokenizer_backend,
+            )
+
+            tokenizer = MistralCommonTokenizer.from_pretrained(
+                tokenizer_name, revision=tokenizer_revision
+            )
+        else:
+            tokenizer = _auto_tokenizer_from_pretrained(
+                tokenizer_name, *args, **common_kwargs
+            )
+
+            # With fastokens, the patched TokenizersBackend.from_pretrained already
+            # returned a tokenizer whose backend is a fastokens shim. Re-resolving via
+            # the declared class (e.g. Qwen2Tokenizer) would discard that work.
+            if (
+                type(tokenizer).__name__ == _TOKENIZERS_BACKEND
+                and tokenizer_backend != "fastokens"
+            ):
+                tokenizer = _resolve_tokenizers_backend(
+                    tokenizer_name, *args, **common_kwargs
+                )
+
+        return _apply_post_load_fixes(tokenizer, tokenizer_name, tokenizer_revision)
+    except Exception as e:
+        if tokenizer_backend == "fastokens":
+            raise RuntimeError(
+                f"fastokens failed to load tokenizer for {tokenizer_name!r}. "
+                f"This model's tokenizer may not be supported by fastokens — "
+                f"see https://github.com/crusoecloud/fastokens. "
+                f"Re-run without --tokenizer-backend=fastokens to use the default backend."
+            ) from e
+        raise
+
+
+# ---------------------------------------------------------------------------
+# Exported helpers (used by processor.py, etc.)
+# ---------------------------------------------------------------------------
+
+
+def _fix_added_tokens_encoding(tokenizer):
+    """Ensure special tokens encode as single tokens in transformers v5.
+
+    Some model tokenizers (e.g. MiniCPM-V-4) define special tokens like <image>,
+    <slice> as attributes on the tokenizer class with corresponding IDs in the
+    vocabulary (via tokenizer.json's added_tokens). In transformers v5, these
+    tokens may not appear in get_added_vocab() and encode() splits them into
+    subwords, breaking multimodal pipelines that rely on finding them in input_ids.
+
+    This function discovers such tokens by scanning tokenizer attributes, checks
+    if they encode correctly, and re-registers any that don't.
+    """
+
+    # Discover special token strings from tokenizer attributes.
+    # Model tokenizers (e.g. MiniCPMVTokenizerFast) store them as attributes
+    # like im_start="<image>", slice_start="<slice>", etc.
+    def _is_special_token_attr(val):
+        return (
+            isinstance(val, str)
+            and val.startswith("<")
+            and val.endswith(">")
+            and len(val) <= 20
+        )
+
+    candidates = {}
+    for attr in dir(tokenizer):
+        if attr.startswith("_"):
+            continue
+        try:
+            val = getattr(tokenizer, attr)
+        except (AttributeError, TypeError, ValueError):
+            continue
+        if not _is_special_token_attr(val):
+            continue
+        token_id = tokenizer.convert_tokens_to_ids(val)
+        if token_id is not None and token_id != tokenizer.unk_token_id:
+            candidates[val] = token_id
+
+    if not candidates:
+        return
+
+    def _encodes_correctly(token_str, expected_id):
+        try:
+            ids = tokenizer.encode(token_str, add_special_tokens=False)
+            return len(ids) == 1 and ids[0] == expected_id
+        except (ValueError, OverflowError, RuntimeError) as e:
+            logger.debug("Token %s encode check failed: %s", token_str, e)
+            return False
+
+    broken = [
+        tok for tok, eid in candidates.items() if not _encodes_correctly(tok, eid)
+    ]
+
+    if not broken:
+        return
+
+    from transformers import AddedToken
+
+    tokens_to_add = [AddedToken(tok, special=True, normalized=False) for tok in broken]
+    tokenizer.add_tokens(tokens_to_add, special_tokens=True)
+    logger.info(
+        "Re-registered %d special tokens for correct v5 encoding: %s",
+        len(broken),
+        broken[:10],
+    )