chore: import upstream snapshot with attribution

python/sglang/srt/parser/code_completion_parser.py

@@ -0,0 +1,138 @@
+# Copyright 2023-2024 SGLang Team
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Completion templates."""
+
+import dataclasses
+import logging
+from enum import Enum, auto
+from typing import Optional
+
+from sglang.srt.entrypoints.openai.protocol import CompletionRequest
+
+logger = logging.getLogger(__name__)
+completion_template_name: Optional[str] = None
+
+
+class FimPosition(Enum):
+    """Position of fim middle token."""
+
+    MIDDLE = auto()
+    END = auto()
+
+
+@dataclasses.dataclass
+class CompletionTemplate:
+    """A class that manages completion prompt templates. only for code completion currently."""
+
+    # The name of this template
+    name: str
+
+    # the fim begin token
+    fim_begin_token: str
+
+    # The fim middle token
+    fim_middle_token: str
+
+    # The fim end token
+    fim_end_token: str
+
+    # The position of the fim middle token
+    fim_position: FimPosition
+
+
+# A global registry for all completion templates
+completion_templates: dict[str, CompletionTemplate] = {}
+
+
+def register_completion_template(template: CompletionTemplate, override: bool = False):
+    """Register a new completion template."""
+    if not override:
+        assert (
+            template.name not in completion_templates
+        ), f"{template.name} has been registered."
+
+    completion_templates[template.name] = template
+
+
+def completion_template_exists(template_name: str) -> bool:
+    return template_name in completion_templates
+
+
+def set_completion_template(template_name: str) -> None:
+    global completion_template_name
+    if completion_template_name is None:
+        completion_template_name = template_name
+
+
+def is_completion_template_defined() -> bool:
+    global completion_template_name
+    return completion_template_name is not None
+
+
+def generate_completion_prompt_from_request(request: CompletionRequest) -> str:
+    global completion_template_name
+    if request.suffix == "":
+        return request.prompt
+
+    return generate_completion_prompt(
+        request.prompt, request.suffix, completion_template_name
+    )
+
+
+def generate_completion_prompt(prompt: str, suffix: str, template_name: str) -> str:
+
+    completion_template = completion_templates[template_name]
+    fim_begin_token = completion_template.fim_begin_token
+    fim_middle_token = completion_template.fim_middle_token
+    fim_end_token = completion_template.fim_end_token
+    fim_position = completion_template.fim_position
+
+    if fim_position == FimPosition.MIDDLE:
+        prompt = f"{fim_begin_token}{prompt}{fim_middle_token}{suffix}{fim_end_token}"
+    elif fim_position == FimPosition.END:
+        prompt = f"{fim_begin_token}{prompt}{fim_end_token}{suffix}{fim_middle_token}"
+
+    return prompt
+
+
+register_completion_template(
+    CompletionTemplate(
+        name="deepseek_coder",
+        fim_begin_token="<｜fim▁begin｜>",
+        fim_middle_token="<｜fim▁hole｜>",
+        fim_end_token="<｜fim▁end｜>",
+        fim_position=FimPosition.MIDDLE,
+    )
+)
+
+
+register_completion_template(
+    CompletionTemplate(
+        name="star_coder",
+        fim_begin_token="<fim_prefix>",
+        fim_middle_token="<fim_middle>",
+        fim_end_token="<fim_suffix>",
+        fim_position=FimPosition.END,
+    )
+)
+
+register_completion_template(
+    CompletionTemplate(
+        name="qwen_coder",
+        fim_begin_token="<|fim_prefix|>",
+        fim_middle_token="<|fim_middle|>",
+        fim_end_token="<|fim_suffix|>",
+        fim_position=FimPosition.END,
+    )
+)

python/sglang/srt/parser/harmony_parser.py

@@ -0,0 +1,588 @@
+import re
+from dataclasses import dataclass
+from typing import Iterator, List, Optional, Tuple
+
+
+@dataclass
+class Event:
+    """Represents a parsed event from the Harmony stream."""
+
+    event_type: str
+    content: str
+    raw_text: str = None  # Original text including structural markers
+
+
+@dataclass
+class Token:
+    """A structural token in the Harmony format."""
+
+    type: str
+    start: int
+    end: int
+
+
+def prefix_hold(text: str, tokens: List[str]) -> Tuple[str, str]:
+    """
+    Holds back the longest suffix of `text` that could be a prefix of any token.
+    Returns (emit_now, keep_for_later).
+    """
+    if not text:
+        return "", ""
+    max_hold = 0
+    for tok in tokens:
+        if not tok:
+            continue
+        # Check for prefixes of tok in the suffix of text
+        L = min(len(tok) - 1, len(text))
+        for k in range(L, 0, -1):
+            if tok.startswith(text[-k:]):
+                max_hold = max(max_hold, k)
+                break
+    if max_hold == 0:
+        return text, ""
+    return text[:-max_hold], text[-max_hold:]
+
+
+def iter_tokens(text: str, start_pos: int = 0) -> Iterator[Token]:
+    """Iterate over structural tokens in left-to-right order."""
+    TOKENS = {
+        "<|start|>": "START",
+        "<|channel|>": "CHANNEL",
+        "<|message|>": "MESSAGE",
+        "<|constrain|>": "CONSTRAIN",
+        "<|end|>": "END",
+        "<|call|>": "CALL",
+        "<|return|>": "RETURN",
+    }
+
+    pos = start_pos
+    has_unknown_tokens = False
+    while pos < len(text):
+        # Find next "<|"
+        marker_pos = text.find("<|", pos)
+        if marker_pos == -1:
+            break
+
+        # Emit any text before the marker
+        if marker_pos > pos:
+            yield Token("TEXT", pos, marker_pos)
+
+        # Check which token it is
+        found_token = False
+
+        for literal, token_type in TOKENS.items():
+            if text.startswith(literal, marker_pos):
+                yield Token(token_type, marker_pos, marker_pos + len(literal))
+                pos = marker_pos + len(literal)
+                found_token = True
+                break
+        if not found_token:
+            tail = text[marker_pos:]
+            is_partial = any(lit.startswith(tail) for lit in TOKENS)
+            if is_partial:
+                # Hold whole tail (partial token)
+                yield Token("TEXT", marker_pos, len(text))
+                pos = len(text)
+                break
+            else:
+                # Unknown token like <|weird|> ...
+                has_unknown_tokens = True
+                # Emit the "<|" as a TEXT token first
+                yield Token("TEXT", marker_pos, marker_pos + 2)
+
+                # Try to find a closing "|>" for this unknown token
+                close_pos = text.find("|>", marker_pos + 2)
+                if close_pos != -1:
+                    # Look ahead to the next structural token after the unknown close
+                    next_marker = text.find("<|", close_pos + 2)
+                    if next_marker != -1:
+                        # Emit the unknown body + any following plain text up to next marker
+                        yield Token("TEXT", marker_pos + 2, next_marker)
+                        pos = next_marker
+                    else:
+                        # Emit until the end
+                        yield Token("TEXT", marker_pos + 2, len(text))
+                        pos = len(text)
+                        break
+                else:
+                    # No closing; advance past "<|" and continue scanning
+                    pos = marker_pos + 2
+
+    # Emit any remaining text
+    if pos < len(text):
+        yield Token("TEXT", pos, len(text))
+    elif pos == len(text) and has_unknown_tokens:
+        # Add an empty trailing TEXT token only when we encountered unknown tokens
+        # and the text ends with a known structural token. This matches expected tests.
+        for literal in TOKENS.keys():
+            if text.endswith(literal):
+                yield Token("TEXT", pos, pos)
+                break
+
+
+class CanonicalStrategy:
+    """Parses the canonical Harmony format with channel markers."""
+
+    def __init__(self):
+        self.guard_tokens = [
+            "<|start|>",
+            "<|channel|>",
+            "<|message|>",
+            "<|constrain|>",
+            "<|end|>",
+            "<|call|>",
+            "<|return|>",
+        ]
+
+    def parse(self, text: str) -> Tuple[List[Event], str]:
+        events = []
+        tokens = list(iter_tokens(text))
+
+        if not tokens:
+            return events, ""
+
+        pos = 0
+        while pos < len(tokens):
+            token = tokens[pos]
+
+            if token.type == "TEXT":
+                # Check if this might be incomplete
+                if pos == len(tokens) - 1:  # Last token
+                    emit, hold = prefix_hold(
+                        text[token.start : token.end], self.guard_tokens
+                    )
+                    if emit:
+                        events.append(Event("normal", emit))
+                    return events, hold
+                else:
+                    # Check if this might be commentary filler between blocks
+                    if self._is_commentary_filler_between_blocks(text, tokens, pos):
+                        # Skip this filler text - don't emit as normal content
+                        pos += 1
+                    else:
+                        content = text[token.start : token.end]
+                        # Skip standalone structural tokens that shouldn't be emitted as normal text
+                        if not self._is_standalone_structural_token(content):
+                            events.append(Event("normal", content))
+                        pos += 1
+
+            elif token.type in ("START", "CHANNEL"):
+                # Parse a channel block starting here
+                block_result = self._parse_block(text, tokens, pos)
+                if block_result is None:
+                    # Incomplete block - check if we can emit partial reasoning content
+                    partial_result = self._parse_partial_analysis(text, tokens, pos)
+                    if partial_result:
+                        event, remaining_text = partial_result
+                        events.append(event)
+                        return events, remaining_text
+                    # No partial content, hold entire remaining text
+                    remaining_start = tokens[pos].start
+                    return events, text[remaining_start:]
+                event, new_pos = block_result
+                if event:
+                    events.append(event)
+                pos = new_pos
+
+            else:
+                # Check if this might be commentary filler between blocks
+                if self._is_commentary_filler_between_blocks(text, tokens, pos):
+                    # Skip this filler text - don't emit as normal content
+                    pos += 1
+                else:
+                    # Unexpected token - only emit as text if it's not a standalone structural token
+                    content = text[token.start : token.end]
+                    if not self._is_standalone_structural_token(content):
+                        events.append(Event("normal", content))
+                    pos += 1
+
+        return events, ""
+
+    def _parse_partial_analysis(
+        self, text: str, tokens: List[Token], start_pos: int
+    ) -> Optional[Tuple[Event, str]]:
+        """Try to parse partial analysis content for incremental streaming."""
+        pos = start_pos
+
+        # Skip <|start|> if present
+        if pos < len(tokens) and tokens[pos].type == "START":
+            pos += 1
+
+        # Look for <|channel|> followed by analysis
+        channel_pos = None
+        message_pos = None
+
+        for i in range(pos, len(tokens)):
+            if tokens[i].type == "CHANNEL" and channel_pos is None:
+                channel_pos = i
+            elif tokens[i].type == "MESSAGE":
+                message_pos = i
+                break
+
+        if channel_pos is None or message_pos is None:
+            return None
+
+        # Extract channel type
+        channel_start = (
+            tokens[channel_pos + 1].start
+            if channel_pos + 1 < len(tokens)
+            else tokens[channel_pos].end
+        )
+        channel_end = tokens[message_pos].start
+        channel_header = text[channel_start:channel_end]
+
+        channel_type = self._extract_channel_type(channel_header)
+        if channel_type != "analysis":
+            return None  # Only stream analysis content - tool calls wait for completion
+
+        # Extract partial content after <|message|>
+        content_start = tokens[message_pos].end
+        content = text[content_start:]
+
+        # Return partial reasoning content and preserve the channel structure for next parse
+        remaining_text = text[tokens[start_pos].start : content_start]
+        return Event("reasoning", content), remaining_text
+
+    def _extract_channel_type(self, header_text: str) -> Optional[str]:
+        """Extract channel type from header, ignoring other attributes like to=... or <|constrain|>..."""
+        # Look for channel type at the start of the header (case insensitive)
+        header_clean = header_text.strip()
+
+        if header_clean.lower().startswith("analysis"):
+            return "analysis"
+        elif header_clean.lower().startswith("commentary"):
+            return "commentary"
+        elif header_clean.lower().startswith("final"):
+            return "final"
+        else:
+            return None  # Unknown channel type
+
+    def _parse_block(
+        self, text: str, tokens: List[Token], start_pos: int
+    ) -> Optional[Tuple[Optional[Event], int]]:
+        """Parse a channel block. Returns (event, next_pos) or None if incomplete."""
+        pos = start_pos
+
+        # Skip <|start|> if present
+        if pos < len(tokens) and tokens[pos].type == "START":
+            pos += 1
+
+        # Look for <|channel|> or <|message|> (tool responses go direct to message)
+        channel_pos = None
+        message_pos = None
+
+        for i in range(pos, len(tokens)):
+            if tokens[i].type == "CHANNEL" and channel_pos is None:
+                channel_pos = i
+            elif tokens[i].type == "MESSAGE":
+                message_pos = i
+                break
+
+        if message_pos is None:
+            return None  # No message token found
+
+        # If no channel found, this is a tool response - treat as normal text
+        if channel_pos is None:
+            content_start = tokens[message_pos].end
+            # Find end token after message
+            end_token_pos = None
+            for i in range(message_pos + 1, len(tokens)):
+                if tokens[i].type in ("END", "CALL", "RETURN"):
+                    end_token_pos = i
+                    break
+            if end_token_pos is None:
+                return None  # Incomplete
+            content = text[content_start : tokens[end_token_pos].start]
+            return Event("normal", content), end_token_pos + 1
+
+        # Standard channel block processing - message_pos is already found above
+        pos = channel_pos + 1  # Skip CHANNEL token
+
+        # Extract channel type from header (ignoring other attributes like to=... or <|constrain|>...)
+        channel_start = tokens[pos].start if pos < len(tokens) else tokens[pos - 1].end
+        channel_end = tokens[message_pos].start
+        channel_header = text[channel_start:channel_end]
+
+        channel_type = self._extract_channel_type(channel_header)
+        if not channel_type:
+            return None  # Unknown or malformed channel
+
+        pos = message_pos + 1  # Skip MESSAGE token
+
+        # Find content and end token
+        content_start = tokens[message_pos].end
+        end_pos = pos
+
+        # Each channel type has specific valid end tokens
+        if channel_type == "final":
+            while end_pos < len(tokens) and tokens[end_pos].type != "RETURN":
+                end_pos += 1
+        elif channel_type == "analysis":
+            while end_pos < len(tokens) and tokens[end_pos].type not in ("END", "CALL"):
+                end_pos += 1
+        else:  # commentary
+            while end_pos < len(tokens) and tokens[end_pos].type not in ("END", "CALL"):
+                end_pos += 1
+
+        if end_pos >= len(tokens):
+            # No end token found
+            if channel_type == "final":
+                # Final blocks can end at end of input without requiring <|return|>
+                content = text[content_start:]
+                return Event("normal", content), end_pos
+            return None  # Analysis and commentary need proper end tokens
+
+        end_token = tokens[end_pos]
+        content = text[content_start : end_token.start]
+
+        # Create event based on channel and end token
+        if channel_type == "analysis":
+            if end_token.type == "CALL":
+                # Built-in tools (browser, python) use analysis channel with <|call|>
+                raw_text = text[tokens[start_pos].start : end_token.end]
+                return Event("tool_call", content.strip(), raw_text), end_pos + 1
+            else:
+                return Event("reasoning", content), end_pos + 1
+        elif channel_type == "commentary":
+            if end_token.type == "CALL":
+                raw_text = text[tokens[start_pos].start : end_token.end]
+                return Event("tool_call", content.strip(), raw_text), end_pos + 1
+            else:
+                return Event("normal", content), end_pos + 1
+        elif channel_type == "final":
+            # For final blocks, include any trailing TEXT immediately after <|return|>
+            final_content = content
+            if end_token.type == "RETURN" and end_pos + 1 < len(tokens):
+                next_token = tokens[end_pos + 1]
+                if next_token.type == "TEXT":
+                    final_content += text[next_token.start : next_token.end]
+                    return Event("normal", final_content), end_pos + 2
+            return Event("normal", final_content), end_pos + 1
+
+        return None, end_pos + 1
+
+    def _is_commentary_filler_between_blocks(
+        self, text: str, tokens: List[Token], pos: int
+    ) -> bool:
+        """Check if this is commentary filler text or problematic structural tokens in malformed sequences."""
+        current_token = tokens[pos]
+        current_text = text[current_token.start : current_token.end].strip()
+
+        # Check for commentary filler between CALL and CHANNEL
+        if pos > 0 and pos + 1 < len(tokens):
+            prev_token = tokens[pos - 1]
+            next_token = tokens[pos + 1]
+
+            # Check if we have CALL -> TEXT("commentary") -> CHANNEL pattern
+            if (
+                prev_token.type == "CALL"
+                and next_token.type == "CHANNEL"
+                and current_text.lower() == "commentary"
+            ):
+                return True
+
+        # Check for problematic patterns after CALL tokens (malformed sequences)
+        if pos > 0:
+            prev_token = tokens[pos - 1]
+
+            # Only filter structural tokens that appear immediately after CALL in malformed sequences
+            # These patterns indicate the content is malformed and the structural tokens are noise
+            if prev_token.type == "CALL":
+                # Filter MESSAGE tokens after CALL (should not happen in well-formed content)
+                if current_token.type == "MESSAGE":
+                    return True
+
+                # Filter standalone "commentary" text after CALL
+                if (
+                    current_token.type == "TEXT"
+                    and current_text.lower() == "commentary"
+                ):
+                    return True
+
+        return False
+
+    def _is_standalone_structural_token(self, content: str) -> bool:
+        """Check if content is just a standalone structural token that should be filtered."""
+        content_stripped = content.strip()
+        structural_tokens = [
+            "<|start|>",
+            "<|channel|>",
+            "<|message|>",
+            "<|constrain|>",
+            "<|end|>",
+            "<|call|>",
+            "<|return|>",
+        ]
+        return content_stripped in structural_tokens
+
+
+class TextStrategy:
+    """Parses the text-based Harmony fallback format."""
+
+    def __init__(self):
+        self.buffer_context = ""
+        self.patterns = {
+            "analysis_then_final": re.compile(
+                r"^\s*(?:assistant)?\s*(analysis|commentary)(.*?)\s*assistantfinal\s*(.*)\s*$",
+                re.IGNORECASE | re.DOTALL,
+            ),
+            "final_only": re.compile(
+                r"^\s*assistantfinal\s*(.*)\s*$", re.IGNORECASE | re.DOTALL
+            ),
+            "analysis_only": re.compile(
+                r"^\s*(?:assistant)?\s*(analysis|commentary)(.*)\s*$",
+                re.IGNORECASE | re.DOTALL,
+            ),
+        }
+
+    def set_buffer_context(self, buffer: str):
+        self.buffer_context = buffer
+
+    def parse(self, text: str) -> Tuple[List[Event], str]:
+        events = []
+
+        m = self.patterns["analysis_then_final"].match(text)
+        if m:
+            channel, reasoning, final = m.groups()
+            if channel.lower() == "analysis" and reasoning.strip():
+                events.append(Event("reasoning", reasoning.strip()))
+            elif channel.lower() == "commentary" and reasoning.strip():
+                events.append(Event("normal", reasoning.strip()))
+            if final.strip():
+                events.append(Event("normal", final.strip()))
+            return events, ""
+
+        # If assistantfinal appears to be incomplete (e.g., 'assistantfin'), hold entire buffer
+        if re.search(
+            r"(?:^|\s)(?:assistant)?\s*(analysis|commentary)", text, re.IGNORECASE
+        ):
+            low = text.lower()
+            if "assistantfin" in low and "assistantfinal" not in low:
+                return events, text
+
+        m = self.patterns["final_only"].match(text)
+        if m:
+            final = m.group(1)
+            if final.strip():
+                events.append(Event("normal", final.strip()))
+            return events, ""
+
+        m = self.patterns["analysis_only"].match(text)
+        if m:
+            channel, content = m.groups()
+            emit, hold = prefix_hold(content, ["assistantfinal"])
+            if channel.lower() == "analysis" and emit:
+                # Stream reasoning content as-is based on structural markers only.
+                events.append(Event("reasoning", emit))
+                # Keep the channel header in the remaining buffer to continue parsing
+                # subsequent chunks in the text fallback format. Preserve any held
+                # prefix that may complete into "assistantfinal".
+                if hold:
+                    return events, text[: m.start(2)] + hold
+                else:
+                    return events, channel
+            elif channel.lower() == "commentary" and emit:
+                # For commentary, stream as normal text. Preserve spaces unless holding.
+                content_out = emit if hold else emit.strip()
+                events.append(Event("normal", content_out))
+                if hold:
+                    return events, text[: m.start(2)] + hold
+                else:
+                    return events, ""
+            # If no emit, just return the held content
+            return events, text[: m.start(2)] + hold
+
+        emit, hold = prefix_hold(text, ["analysis", "commentary", "assistantfinal"])
+        if emit:
+            events.append(Event("normal", emit))
+        return events, hold
+
+
+class HarmonyParser:
+    """Facade for parsing Harmony format, switching between strategies."""
+
+    def __init__(self):
+        self.strategy = None
+        self._buffer = ""
+        self._should_filter_commentary = (
+            False  # Track if we should filter commentary in next chunks
+        )
+        self._partial_commentary = (
+            ""  # Track partial commentary being built across chunks
+        )
+
+    def parse(self, chunk: str) -> List[Event]:
+        self._buffer += chunk
+
+        if self.strategy is None:
+            if "<|channel|>" in self._buffer or "<|start|>" in self._buffer:
+                self.strategy = CanonicalStrategy()
+            elif re.search(
+                r"(?:^|\s)(?:assistant)?\s*(analysis|commentary|assistantfinal)",
+                self._buffer,
+                re.IGNORECASE,
+            ):
+                self.strategy = TextStrategy()
+            else:
+                # Not yet determined, hold
+                return []
+
+        if hasattr(self.strategy, "set_buffer_context"):
+            # Provide full buffer context to strategy for smarter whitespace handling
+            self.strategy.set_buffer_context(self._buffer)
+
+        events, remaining = self.strategy.parse(self._buffer)
+
+        # Check if we should start filtering commentary (after <|call|> token or tool_call event)
+        buffer_has_call_token = self._buffer.rstrip().endswith("<|call|>")
+
+        self._buffer = remaining
+
+        # Filter events for streaming case
+        filtered_events = []
+        for event in events:
+            should_filter = False
+
+            if event.event_type == "normal":
+                # Check if we're in a commentary filtering state
+                if self._should_filter_commentary or self._partial_commentary:
+                    # Try to build partial commentary
+                    potential_commentary = (
+                        self._partial_commentary + event.content.strip().lower()
+                    )
+
+                    if potential_commentary == "commentary":
+                        # Complete commentary found - filter it
+                        should_filter = True
+                        self._partial_commentary = ""  # Reset
+                        self._should_filter_commentary = False  # Done filtering
+                    elif "commentary".startswith(potential_commentary):
+                        # Partial match - accumulate and filter this chunk
+                        should_filter = True
+                        self._partial_commentary = potential_commentary
+                    else:
+                        # Not commentary - reset and keep the event
+                        self._partial_commentary = ""
+                        self._should_filter_commentary = False
+                else:
+                    # Not in commentary filtering state - reset partial state
+                    self._partial_commentary = ""
+
+            if should_filter:
+                # Skip this commentary filler
+                continue
+
+            # Update filtering state based on events and buffer state
+            if event.event_type == "tool_call":
+                self._should_filter_commentary = (
+                    True  # Filter commentary after tool calls
+                )
+                self._partial_commentary = ""  # Reset on tool call
+            elif buffer_has_call_token:
+                self._should_filter_commentary = (
+                    True  # Filter commentary after <|call|> token
+                )
+
+            filtered_events.append(event)
+
+        return filtered_events

python/sglang/srt/parser/jinja_template_utils.py

@@ -0,0 +1,239 @@
+"""Template utilities for Jinja template processing.
+
+This module provides utilities for analyzing and processing Jinja chat templates,
+including content format detection and message processing.
+"""
+
+import logging
+
+import jinja2
+import transformers.utils.chat_template_utils as hf_chat_utils
+
+from sglang.srt.utils import ImageData
+
+logger = logging.getLogger(__name__)
+
+# ============================================================================
+# JINJA TEMPLATE CONTENT FORMAT DETECTION
+# ============================================================================
+#
+# This adapts vLLM's approach for detecting chat template content format:
+# https://github.com/vllm-project/vllm/blob/02f0c7b220422792f5e53de2a7d51d2d3ff2df28/vllm/entrypoints/chat_utils.py#L296-L313
+# - Analyzes Jinja template AST to detect content iteration patterns
+# - 'openai' format: templates with {%- for content in message['content'] -%} loops
+# - 'string' format: templates that expect simple string content
+# - Processes content accordingly to match template expectations
+
+
+def _is_var_access(node: jinja2.nodes.Node, varname: str) -> bool:
+    """Check if node is a variable access like {{ varname }}"""
+    if isinstance(node, jinja2.nodes.Name):
+        return node.ctx == "load" and node.name == varname
+    return False
+
+
+def _is_attr_access(node: jinja2.nodes.Node, varname: str, key: str) -> bool:
+    """Check if node is an attribute access like {{ varname['key'] }} or {{ varname.key }}"""
+    if isinstance(node, jinja2.nodes.Getitem):
+        return (
+            _is_var_access(node.node, varname)
+            and isinstance(node.arg, jinja2.nodes.Const)
+            and node.arg.value == key
+        )
+
+    if isinstance(node, jinja2.nodes.Getattr):
+        return _is_var_access(node.node, varname) and node.attr == key
+
+    return False
+
+
+def _is_var_or_elems_access(
+    node: jinja2.nodes.Node,
+    varname: str,
+    key: str = None,
+) -> bool:
+    """Check if node accesses varname or varname[key] with filters/tests"""
+    if isinstance(node, jinja2.nodes.Filter):
+        return node.node is not None and _is_var_or_elems_access(
+            node.node, varname, key
+        )
+    if isinstance(node, jinja2.nodes.Test):
+        return _is_var_or_elems_access(node.node, varname, key)
+
+    if isinstance(node, jinja2.nodes.Getitem) and isinstance(
+        node.arg, jinja2.nodes.Slice
+    ):
+        return _is_var_or_elems_access(node.node, varname, key)
+
+    return _is_attr_access(node, varname, key) if key else _is_var_access(node, varname)
+
+
+def _try_extract_ast(chat_template: str):
+    """Try to parse the Jinja template into an AST"""
+    try:
+        jinja_compiled = hf_chat_utils._compile_jinja_template(chat_template)
+        return jinja_compiled.environment.parse(chat_template)
+    except Exception as e:
+        logger.debug(f"Error when compiling Jinja template: {e}")
+        return None
+
+
+def detect_jinja_template_content_format(chat_template: str) -> str:
+    """
+    Detect whether a chat template expects 'string' or 'openai' content format.
+
+    - 'string': content is a simple string (like DeepSeek templates)
+    - 'openai': content is a list of structured dicts (like Llama4 templates)
+
+    Detection logic:
+    - If template has loops like {%- for content in message['content'] -%} → 'openai'
+    - Otherwise → 'string'
+    """
+    # Shortcut for multimodal templates
+    if any(
+        keyword in chat_template for keyword in ["image", "audio", "video", "vision"]
+    ):
+        return "openai"
+
+    jinja_ast = _try_extract_ast(chat_template)
+    if jinja_ast is None:
+        return "string"
+
+    try:
+        # Look for patterns like: {%- for content in message['content'] -%}
+        for loop_ast in jinja_ast.find_all(jinja2.nodes.For):
+            loop_iter = loop_ast.iter
+
+            # Check if iterating over message['content'] or similar
+            if _is_var_or_elems_access(loop_iter, "message", "content"):
+                return "openai"  # Found content iteration → openai format
+
+            # Also check for patterns like: {%- for item in msg.content -%} or {%- for item in m.content -%}
+            if _is_var_or_elems_access(
+                loop_iter, "msg", "content"
+            ) or _is_var_or_elems_access(loop_iter, "m", "content"):
+                return "openai"  # Found content iteration → openai format (glm4v)
+
+        return "string"  # No content loops found → string format
+    except Exception as e:
+        logger.debug(f"Error when parsing AST of Jinja template: {e}")
+        return "string"
+
+
+def process_content_for_template_format(
+    msg_dict: dict,
+    content_format: str,
+    image_data: list,
+    video_data: list,
+    audio_data: list,
+    modalities: list,
+    use_dpsk_v32_encoding: bool = False,
+) -> dict:
+    """
+    Process message content based on detected template format.
+
+    Args:
+        msg_dict: Message dictionary with content
+        content_format: 'string' or 'openai' (detected via AST analysis)
+        image_data: List to append extracted image URLs
+        video_data: List to append extracted video URLs
+        audio_data: List to append extracted audio URLs
+        modalities: List to append modalities
+        use_dpsk_v32_encoding: If True, extract multimodal data and convert content to string (for DeepSeek-V3.2 encoding)
+
+    Returns:
+        Processed message dictionary
+    """
+    if not isinstance(msg_dict.get("content"), list):
+        # Already a string or None, no processing needed
+        return {k: v for k, v in msg_dict.items() if v is not None}
+
+    if content_format == "openai" or use_dpsk_v32_encoding:
+        # OpenAI format: preserve structured content list, normalize types
+        # V32 encoding: extract multimodal data but convert content to string
+        processed_content_parts = []
+        text_parts = []
+        for chunk in msg_dict["content"]:
+            if isinstance(chunk, dict):
+                chunk_type = chunk.get("type")
+
+                if chunk_type in ("image_url", "input_image"):
+                    image_obj = chunk.get("image_url") or {}
+                    if isinstance(image_obj, str):
+                        image_obj = {"url": image_obj, "detail": chunk.get("detail")}
+                    mdp = image_obj.get("max_dynamic_patch", None)
+                    # Also allow flat style: chunk["max_dynamic_patch"]
+                    image_data.append(
+                        ImageData(
+                            url=image_obj["url"],
+                            detail=image_obj.get("detail") or "auto",
+                            max_dynamic_patch=mdp,
+                        )
+                    )
+
+                    if chunk.get("modalities"):
+                        modalities.append(chunk.get("modalities"))
+                    # Normalize to simple 'image' type for template compatibility
+                    processed_content_parts.append({"type": "image"})
+                elif chunk_type == "video_url":
+                    video_obj = chunk.get("video_url") or {}
+                    mdp = video_obj.get("max_dynamic_patch", None)
+                    if mdp is None:
+                        video_data.append(chunk["video_url"]["url"])
+                    else:
+                        # Keep structured info for backend, but template only sees {"type":"video"}
+                        video_data.append(
+                            {
+                                "url": video_obj["url"],
+                                "max_dynamic_patch": mdp,
+                            }
+                        )
+                    if chunk.get("modalities"):
+                        modalities.append(chunk.get("modalities"))
+                    # Normalize to simple 'video' type for template compatibility
+                    processed_content_parts.append({"type": "video"})
+                elif chunk_type == "audio_url":
+                    audio_data.append(chunk["audio_url"]["url"])
+                    # Normalize to simple 'audio' type
+                    processed_content_parts.append({"type": "audio"})
+                elif chunk_type in ("text", "input_text"):
+                    # For v32 encoding, collect text parts separately
+                    if use_dpsk_v32_encoding:
+                        text_parts.append(chunk["text"])
+                    else:
+                        # Keep text content as-is for openai format
+                        processed_content_parts.append(
+                            {"type": "text", "text": chunk["text"]}
+                        )
+                elif chunk_type == "tool_reference":
+                    # GLM-specific extension: pass through so the chat template
+                    # can match tool_reference.name against tools[*].function.name
+                    # and render the referenced tool schemas inline.
+                    processed_content_parts.append(chunk)
+
+        new_msg = {
+            k: v for k, v in msg_dict.items() if v is not None and k != "content"
+        }
+        if use_dpsk_v32_encoding:
+            new_msg["content"] = " ".join(text_parts) if text_parts else ""
+        else:
+            new_msg["content"] = processed_content_parts
+        return new_msg
+
+    elif content_format == "string":
+        # String format: flatten to text only (for templates like DeepSeek)
+        text_parts = []
+        for chunk in msg_dict["content"]:
+            if isinstance(chunk, dict) and chunk.get("type") in ("text", "input_text"):
+                text_parts.append(chunk["text"])
+            # Note: For string format, we ignore images/audio since the template
+            # doesn't expect structured content - multimodal placeholders would
+            # need to be inserted differently
+
+        new_msg = msg_dict.copy()
+        new_msg["content"] = " ".join(text_parts) if text_parts else ""
+        new_msg = {k: v for k, v in new_msg.items() if v is not None}
+        return new_msg
+
+    else:
+        raise ValueError(f"Invalid content format: {content_format}")

python/sglang/srt/parser/template_detection.py

@@ -0,0 +1,707 @@
+# Copyright 2026 SGLang Team
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""
+Template detection utilities for auto-detecting reasoning and tool-call parsers.
+
+Provides rule-based detection of reasoning mode, reasoning parser, and tool-call
+parser from chat templates and tokenizer vocabularies.
+"""
+
+import logging
+import os
+import re
+from dataclasses import dataclass
+from typing import Callable, Optional, Tuple
+
+import jinja2
+import jinja2.ext
+import jinja2.sandbox
+
+logger = logging.getLogger(__name__)
+
+
+@dataclass(frozen=True)
+class TemplateDetectionContext:
+    template: str
+    reasoning_config: Optional["ReasoningToggleConfig"]
+    force_reasoning: bool
+    vocab: set[str]
+
+    def has_text(self, needle: str) -> bool:
+        return needle in self.template
+
+    def has_vocab(self, token: str) -> bool:
+        return token in self.vocab
+
+    def has_pattern(self, pattern: str, flags: int = 0) -> bool:
+        return re.search(pattern, self.template, flags) is not None
+
+    def has_vocab_pattern(self, pattern: str) -> bool:
+        compiled = re.compile(pattern)
+        return any(isinstance(tok, str) and compiled.search(tok) for tok in self.vocab)
+
+
+@dataclass(frozen=True)
+class DetectionRule:
+    name: str
+    value: object
+    predicate: Callable[[TemplateDetectionContext], bool]
+
+
+@dataclass(frozen=True)
+class ReasoningToggleConfig:
+    toggle_param: Optional[str] = None
+    default_enabled: Optional[bool] = None
+    special_case: Optional[str] = None
+    effort_kwarg: Optional[str] = None
+
+    @property
+    def always_on(self) -> bool:
+        return self.special_case == "always"
+
+
+# ---------------------------------------------------------------------------
+# Reasoning mode rules (detect toggle config from template)
+# ---------------------------------------------------------------------------
+
+REASONING_MODE_RULES = (
+    DetectionRule(
+        name="gpt_oss_channel_markers",
+        value=ReasoningToggleConfig(special_case="always"),
+        predicate=lambda ctx: ctx.has_text("<|channel|>"),
+    ),
+    DetectionRule(
+        name="force_reasoning_pattern",
+        value=ReasoningToggleConfig(special_case="always"),
+        predicate=lambda ctx: ctx.has_pattern(r"<\|im_start\|>assistant\\n<think>\\n")
+        and not ctx.has_text("enable_thinking")
+        and not ctx.has_text("thinking"),
+    ),
+    DetectionRule(
+        name="mistral_reasoning_effort",
+        value=ReasoningToggleConfig(special_case="mistral"),
+        predicate=lambda ctx: ctx.has_text("reasoning_effort")
+        and ctx.has_text("[THINK]"),
+    ),
+    DetectionRule(
+        name="explicit_enable_thinking_default_false",
+        value=ReasoningToggleConfig(
+            toggle_param="enable_thinking", default_enabled=False
+        ),
+        predicate=lambda ctx: ctx.has_pattern(
+            r"{%\s*if\s+not\s+enable_thinking\s+is\s+defined\s*%}.*?"
+            r"{%\s*set\s+enable_thinking\s*=\s*(?:false|False)\s*%}",
+            re.DOTALL,
+        ),
+    ),
+    DetectionRule(
+        name="nemotron_3_super_low_effort",
+        value=ReasoningToggleConfig(
+            toggle_param="enable_thinking",
+            default_enabled=True,
+            effort_kwarg="low_effort",
+        ),
+        predicate=lambda ctx: ctx.has_text("low_effort")
+        and ctx.has_text("truncate_history_thinking"),
+    ),
+    DetectionRule(
+        name="enable_thinking_default_true",
+        value=ReasoningToggleConfig(
+            toggle_param="enable_thinking", default_enabled=True
+        ),
+        predicate=lambda ctx: ctx.has_pattern(
+            r"{%\s*if\s+not\s+enable_thinking\s+is\s+defined\s*%}.*?"
+            r"{%\s*set\s+enable_thinking\s*=\s*(?:true|True)\s*%}",
+            re.DOTALL,
+        )
+        or ctx.has_pattern(
+            r"set\s+enable_thinking\s*=\s*enable_thinking\s+if\s+enable_thinking\s+is\s+defined\s+else\s+(?:true|True)"
+        )
+        or ctx.has_pattern(
+            r"enable_thinking\s+is\s+defined\s+and\s+(?:enable_thinking\s+is\s+false|not\s+enable_thinking)"
+        )
+        or ctx.has_pattern(
+            r"enable_thinking\s+is\s+not\s+defined\s+or\s+enable_thinking"
+        )
+        or ctx.has_pattern(r"namespace\([^)]*enable_thinking\s*=\s*true"),
+    ),
+    DetectionRule(
+        name="explicit_thinking_default_false",
+        value=ReasoningToggleConfig(toggle_param="thinking", default_enabled=False),
+        predicate=lambda ctx: ctx.has_pattern(
+            r"{%\s*if\s+not\s+thinking\s+is\s+defined\s*%}.*?"
+            r"{%\s*set\s+thinking\s*=\s*(?:false|False)\s*%}",
+            re.DOTALL,
+        ),
+    ),
+    DetectionRule(
+        name="thinking_default_true",
+        value=ReasoningToggleConfig(toggle_param="thinking", default_enabled=True),
+        predicate=lambda ctx: ctx.has_pattern(
+            r"{%\s*if\s+not\s+thinking\s+is\s+defined\s*%}.*?"
+            r"{%\s*set\s+thinking\s*=\s*(?:true|True)\s*%}",
+            re.DOTALL,
+        )
+        or ctx.has_pattern(
+            r"set\s+thinking\s*=\s*thinking\s+if\s+thinking\s+is\s+defined\s+else\s+(?:true|True)"
+        )
+        or ctx.has_pattern(
+            r"thinking\s+is\s+defined\s+and\s+(?:thinking\s+is\s+false|not\s+thinking)"
+        )
+        or ctx.has_pattern(r"thinking\s+is\s+not\s+defined\s+or\s+thinking")
+        or ctx.has_pattern(r"namespace\([^)]*thinking\s*=\s*true"),
+    ),
+)
+
+
+# ---------------------------------------------------------------------------
+# Shared predicates for model-family detection
+# ---------------------------------------------------------------------------
+
+
+def _is_apertus2509(ctx):
+    return ctx.has_vocab("<|inner_prefix|>")
+
+
+def _is_gemma4(ctx):
+    return ctx.has_text("<|channel>")
+
+
+def _is_kimi(ctx):
+    return ctx.has_text("◁think▷")
+
+
+def _is_interns1(ctx):
+    return ctx.has_text("default_thinking_sys") and ctx.reasoning_config == (
+        ReasoningToggleConfig(toggle_param="enable_thinking", default_enabled=True)
+    )
+
+
+def _is_mistral(ctx):
+    return (
+        ctx.reasoning_config is not None
+        and ctx.reasoning_config.special_case == "mistral"
+    )
+
+
+def _is_gpt_oss(ctx):
+    return ctx.has_text("<|channel|>")
+
+
+def _is_kimi_k2(ctx):
+    return ctx.has_vocab("<|tool_calls_section_begin|>")
+
+
+def _is_nemotron_3(ctx):
+    return ctx.has_text("truncate_history_thinking") and (
+        ctx.reasoning_config is not None
+        and ctx.reasoning_config.toggle_param == "enable_thinking"
+        and ctx.reasoning_config.default_enabled is True
+    )
+
+
+def _is_glm45(ctx):
+    return (
+        (
+            ctx.has_text("[gMASK]<sop>")
+            or ctx.has_pattern(r"(?<!<)/nothink")
+            or ctx.has_pattern(r"(?<!<)/think")
+        )
+        and ctx.has_vocab("<tool_call>")
+        and ctx.reasoning_config
+        == ReasoningToggleConfig(toggle_param="enable_thinking", default_enabled=True)
+        and (ctx.has_vocab("<|user|>") or ctx.has_vocab("<|endoftext|>"))
+    )
+
+
+def _is_glm47(ctx):
+    return _is_glm45(ctx) and ctx.has_pattern(
+        r"\{\{[-\s]*['\"]<tool_call>['\"]\s*\+\s*tc\.name"
+    )
+
+
+def _is_xml_kv_tool_call(ctx):
+    # Structural signature for the GLM-4.5 / GLM-4.6 style tool-call format
+    # (`<tool_call>name<arg_key>k</arg_key>\n<arg_value>v</arg_value>...</tool_call>`).
+    # Matches any model whose tokenizer carries `<arg_key>` and `<arg_value>` as
+    # added tokens — e.g., inclusionAI/Ring-2.6, which borrows GLM's tool-call
+    # format but doesn't share the `[gMASK]<sop>` / `enable_thinking` family
+    # signature checked by `_is_glm45`.
+    return ctx.has_vocab("<arg_key>") and ctx.has_vocab("<arg_value>")
+
+
+def _is_deepseek_v31(ctx):
+    return ctx.has_text("<｜tool▁calls▁begin｜>") and ctx.has_text("<｜tool▁sep｜>")
+
+
+def _is_deepseek_v32(ctx):
+    return ctx.has_text("<｜DSML｜function_calls>")
+
+
+def _is_deepseek_v4(ctx):
+    return ctx.has_text("<｜DSML｜tool_calls>")
+
+
+def _is_hunyuan(ctx):
+    # The shipping Hy3 tokenizer appends a shared suffix to each special token
+    # (e.g. ``<tool_calls:opensource>``), so match the bare or suffixed form.
+    tc = ctx.has_text("<tool_calls>") or ctx.has_vocab_pattern(
+        r"^<tool_calls(?::[^>]+)?>$"
+    )
+    sep = ctx.has_text("<tool_sep>") or ctx.has_vocab_pattern(
+        r"^<tool_sep(?::[^>]+)?>$"
+    )
+    return (tc and sep) or (
+        ctx.has_text("reasoning_effort") and ctx.has_text("interleaved_thinking")
+    )
+
+
+def _is_poolside_v1(ctx):
+    has_poolside_tool_format = (
+        ctx.has_text("unescaped XML-like object")
+        and ctx.has_text("<tool_call>function-name")
+        and ctx.has_text("<arg_key>")
+        and ctx.has_text("<arg_value>")
+    )
+    return has_poolside_tool_format or (
+        ctx.reasoning_config
+        == ReasoningToggleConfig(toggle_param="enable_thinking", default_enabled=False)
+        and not _is_hunyuan(ctx)
+        and (ctx.has_text("<arg_key>") or ctx.has_vocab("<arg_key>"))
+        and (ctx.has_text("<arg_value>") or ctx.has_vocab("<arg_value>"))
+    )
+
+
+def _is_mimo(ctx):
+    return ctx.reasoning_config == ReasoningToggleConfig(
+        toggle_param="enable_thinking", default_enabled=False
+    )
+
+
+def _is_minimax(ctx):
+    return ctx.has_text("<minimax:tool_call>")
+
+
+def _is_minimax_m3(ctx):
+    return ctx.has_text("<mm:think>") or ctx.has_text("]<]minimax[>[")
+
+
+def _is_minicpm5(ctx):
+    if ctx.has_vocab("<function") and ctx.has_vocab("<param"):
+        return True
+    return ctx.has_pattern(r"<function\s+name=") and ctx.has_pattern(r"<param\s+name=")
+
+
+def _is_lfm2(ctx):
+    return (
+        ctx.has_text("<|tool_call_start|>") or ctx.has_vocab("<|tool_call_start|>")
+    ) and (ctx.has_text("<|tool_call_end|>") or ctx.has_vocab("<|tool_call_end|>"))
+
+
+def _is_step3p5(ctx):
+    return ctx.has_pattern(r"Step-?3(?:\.|p)?[57]", re.IGNORECASE) or (
+        ctx.has_text("reasoning_effort")
+        and ctx.has_text("Reasoning: ")
+        and _is_qwen3_coder(ctx)
+    )
+
+
+def _is_step3(ctx):
+    return ctx.has_text("<steptml:invoke") or (
+        ctx.has_text("<｜tool_calls_begin｜>") and ctx.has_text("<｜tool_sep｜>")
+    )
+
+
+def _is_qwen3_coder(ctx):
+    return ctx.has_text("<function=") and ctx.has_text("<parameter=")
+
+
+def _is_qwen3(ctx):
+    return ctx.reasoning_config == ReasoningToggleConfig(
+        toggle_param="enable_thinking", default_enabled=True
+    )
+
+
+def _is_deepseek_v3(ctx):
+    return ctx.reasoning_config == ReasoningToggleConfig(
+        toggle_param="thinking", default_enabled=False
+    )
+
+
+def _is_deepseek_r1(ctx):
+    return ctx.force_reasoning
+
+
+def _is_deepseek_r1_think_tags(ctx):
+    return not _is_lfm2(ctx) and (ctx.has_text("<think>") or ctx.has_text("</think>"))
+
+
+# ---------------------------------------------------------------------------
+# Reasoning parser rules
+# ---------------------------------------------------------------------------
+
+REASONING_PARSER_RULES = (
+    DetectionRule(name="apertus2509", value="apertus2509", predicate=_is_apertus2509),
+    DetectionRule(name="gemma4", value="gemma4", predicate=_is_gemma4),
+    DetectionRule(name="kimi", value="kimi", predicate=_is_kimi),
+    DetectionRule(name="interns1", value="interns1", predicate=_is_interns1),
+    DetectionRule(name="mistral", value="mistral", predicate=_is_mistral),
+    DetectionRule(name="gpt_oss", value="gpt-oss", predicate=_is_gpt_oss),
+    DetectionRule(name="kimi_k2", value="kimi_k2", predicate=_is_kimi_k2),
+    DetectionRule(name="nemotron_3", value="nemotron_3", predicate=_is_nemotron_3),
+    DetectionRule(name="glm45", value="glm45", predicate=_is_glm45),
+    DetectionRule(name="hunyuan", value="hunyuan", predicate=_is_hunyuan),
+    DetectionRule(name="poolside_v1", value="poolside_v1", predicate=_is_poolside_v1),
+    DetectionRule(name="mimo", value="mimo", predicate=_is_mimo),
+    DetectionRule(name="minimax_m3", value="minimax-m3", predicate=_is_minimax_m3),
+    DetectionRule(name="minimax", value="minimax", predicate=_is_minimax),
+    DetectionRule(name="step3p5", value="step3p5", predicate=_is_step3p5),
+    DetectionRule(name="step3", value="step3", predicate=_is_step3),
+    DetectionRule(name="qwen3", value="qwen3", predicate=_is_qwen3),
+    DetectionRule(name="deepseek_v4", value="deepseek-v4", predicate=_is_deepseek_v4),
+    DetectionRule(name="deepseek_v3", value="deepseek-v3", predicate=_is_deepseek_v3),
+    DetectionRule(
+        name="deepseek_r1_force", value="deepseek-r1", predicate=_is_deepseek_r1
+    ),
+    DetectionRule(
+        name="deepseek_r1_think_tags",
+        value="deepseek-r1",
+        predicate=_is_deepseek_r1_think_tags,
+    ),
+)
+
+# ---------------------------------------------------------------------------
+# Tool-call parser rules (reuse shared predicates, different values)
+# ---------------------------------------------------------------------------
+
+TOOL_CALL_PARSER_RULES = (
+    DetectionRule(name="apertus2509", value="apertus2509", predicate=_is_apertus2509),
+    DetectionRule(name="gemma4", value="gemma4", predicate=_is_gemma4),
+    DetectionRule(name="gpt_oss", value="gpt-oss", predicate=_is_gpt_oss),
+    DetectionRule(name="kimi_k2", value="kimi_k2", predicate=_is_kimi_k2),
+    DetectionRule(name="minimax_m3", value="minimax-m3", predicate=_is_minimax_m3),
+    DetectionRule(name="minimax", value="minimax-m2", predicate=_is_minimax),
+    DetectionRule(name="interns1", value="interns1", predicate=_is_interns1),
+    DetectionRule(name="mistral", value="mistral", predicate=_is_mistral),
+    DetectionRule(name="deepseek_v4", value="deepseekv4", predicate=_is_deepseek_v4),
+    DetectionRule(name="deepseek_v32", value="deepseekv32", predicate=_is_deepseek_v32),
+    DetectionRule(name="deepseek_v31", value="deepseekv31", predicate=_is_deepseek_v31),
+    DetectionRule(name="lfm2", value="lfm2", predicate=_is_lfm2),
+    DetectionRule(name="glm47", value="glm47", predicate=_is_glm47),
+    DetectionRule(name="glm45", value="glm45", predicate=_is_glm45),
+    DetectionRule(name="minicpm5", value="minicpm5", predicate=_is_minicpm5),
+    DetectionRule(name="hunyuan", value="hunyuan", predicate=_is_hunyuan),
+    DetectionRule(name="poolside_v1", value="poolside_v1", predicate=_is_poolside_v1),
+    DetectionRule(name="step3p5", value="step3p5", predicate=_is_step3p5),
+    DetectionRule(name="step3", value="step3", predicate=_is_step3),
+    DetectionRule(
+        name="xml_kv_tool_call", value="glm45", predicate=_is_xml_kv_tool_call
+    ),
+    DetectionRule(name="mimo", value="mimo", predicate=_is_mimo),
+    DetectionRule(name="qwen3_coder", value="qwen3_coder", predicate=_is_qwen3_coder),
+    DetectionRule(name="qwen", value="qwen", predicate=_is_qwen3),
+    DetectionRule(name="deepseek_v3", value="deepseekv3", predicate=_is_deepseek_v3),
+    DetectionRule(name="deepseek_r1", value="deepseekv3", predicate=_is_deepseek_r1),
+)
+
+
+# ---------------------------------------------------------------------------
+# Detection functions
+# ---------------------------------------------------------------------------
+
+
+def build_detection_context(
+    template: Optional[str],
+    tokenizer,
+    reasoning_config: Optional[ReasoningToggleConfig] = None,
+    force_reasoning: bool = False,
+) -> Optional[TemplateDetectionContext]:
+    if template is None:
+        return None
+    vocab = set()
+    if tokenizer is not None:
+        try:
+            vocab = set(tokenizer.get_vocab().keys())
+        except Exception as e:
+            logger.warning(
+                "Failed to load tokenizer vocab for template detection: %s. "
+                "Vocab-dependent detection rules will be skipped.",
+                e,
+            )
+    return TemplateDetectionContext(
+        template=template,
+        reasoning_config=reasoning_config,
+        force_reasoning=force_reasoning,
+        vocab=vocab,
+    )
+
+
+def match_rules(
+    ctx: TemplateDetectionContext,
+    rules: Tuple[DetectionRule, ...],
+    label: str,
+) -> Optional[str]:
+    for rule in rules:
+        try:
+            if rule.predicate(ctx):
+                return rule.value
+        except Exception as e:
+            logger.warning(
+                "Detection rule '%s' for %s raised an exception: %s. Skipping.",
+                rule.name,
+                label,
+                e,
+                exc_info=True,
+            )
+    return None
+
+
+def detect_reasoning_pattern(
+    template: Optional[str],
+) -> Tuple[bool, Optional[ReasoningToggleConfig]]:
+    """Detect if the chat template contains reasoning/thinking patterns."""
+    if template is None:
+        return False, None
+
+    ctx = TemplateDetectionContext(
+        template=template,
+        reasoning_config=None,
+        force_reasoning=False,
+        vocab=set(),
+    )
+    for rule in REASONING_MODE_RULES:
+        if rule.predicate(ctx):
+            return rule.value.always_on, rule.value
+
+    return False, None
+
+
+def detect_reasoning_parser(
+    template: Optional[str],
+    tokenizer,
+    reasoning_config: Optional[ReasoningToggleConfig] = None,
+    force_reasoning: bool = False,
+) -> Optional[str]:
+    """Auto-detect which reasoning parser to use from the chat template."""
+    ctx = build_detection_context(
+        template, tokenizer, reasoning_config, force_reasoning
+    )
+    if ctx is None:
+        return None
+    return match_rules(ctx, REASONING_PARSER_RULES, "reasoning parser")
+
+
+def detect_tool_call_parser(
+    template: Optional[str],
+    tokenizer,
+    reasoning_config: Optional[ReasoningToggleConfig] = None,
+    force_reasoning: bool = False,
+) -> Optional[str]:
+    """Auto-detect which tool-call parser to use from the chat template."""
+    ctx = build_detection_context(
+        template, tokenizer, reasoning_config, force_reasoning
+    )
+    if ctx is None:
+        return None
+    return match_rules(ctx, TOOL_CALL_PARSER_RULES, "tool-call parser")
+
+
+def detect_inline_system_support(chat_template: Optional[str]) -> bool:
+    """True if mid-conversation ``role: "system"`` renders inline; False if the
+    template raises or silently drops it (then merge into the leading block).
+
+    The probe requires the second system's sentinel to appear in the output —
+    not raising isn't enough, since some templates ignore non-leading system."""
+    if not chat_template:
+        return False
+    sentinel = "__sglang_inline_system_sentinel__"
+    try:
+        env = jinja2.sandbox.ImmutableSandboxedEnvironment(
+            trim_blocks=True,
+            lstrip_blocks=True,
+            extensions=[jinja2.ext.loopcontrols],
+        )
+        rendered = env.from_string(chat_template).render(
+            messages=[
+                {"role": "system", "content": "t"},
+                {"role": "user", "content": "t"},
+                {"role": "system", "content": sentinel},
+                {"role": "user", "content": "t"},
+            ],
+            add_generation_prompt=False,
+        )
+        return sentinel in rendered
+    except jinja2.TemplateError:
+        return False
+    except Exception:
+        return False
+
+
+def _resolve_auto_parser(
+    server_args,
+    attr: str,
+    ctx: TemplateDetectionContext,
+    rules: Tuple[DetectionRule, ...],
+    label: str,
+) -> None:
+    """Resolve a single auto parser, updating server_args in place."""
+    detected = match_rules(ctx, rules, label)
+    if detected:
+        server_args.override(source="template-detection", **{attr: detected})
+        logger.info(
+            f"Auto-detected --{attr.replace('_', '-')} as '{detected}' from chat template"
+        )
+    else:
+        logger.warning(
+            f"--{attr.replace('_', '-')}=auto specified but could not detect "
+            f"{label} from chat template. Disabling {label}."
+        )
+        server_args.override(source="template-detection", **{attr: None})
+
+
+def _load_explicit_jinja_template(chat_template_arg: Optional[str]) -> Optional[str]:
+    if not chat_template_arg or not isinstance(chat_template_arg, str):
+        return None
+    if not chat_template_arg.endswith(".jinja") or not os.path.exists(
+        chat_template_arg
+    ):
+        return None
+    with open(chat_template_arg, encoding="utf-8") as f:
+        return f.read().replace("\\n", "\n")
+
+
+def _disable_auto_parser(server_args, attr: str, label: str) -> None:
+    logger.warning(
+        f"--{attr.replace('_', '-')}=auto specified but could not detect "
+        f"{label} from chat template. Disabling {label}."
+    )
+    server_args.override(source="template-detection", **{attr: None})
+
+
+def _resolve_architecture_auto_parsers(server_args) -> None:
+    from sglang.srt.utils.hf_transformers_utils import get_config
+
+    config = get_config(
+        server_args.model_path,
+        trust_remote_code=server_args.trust_remote_code,
+        revision=getattr(server_args, "revision", None),
+        model_config_parser=getattr(server_args, "model_config_parser", "auto"),
+    )
+    architectures = getattr(config, "architectures", None) or []
+    arch = architectures[0] if architectures else ""
+
+    if "DeepseekV4" in arch:
+        reasoning_parser, tool_call_parser = "deepseek-v4", "deepseekv4"
+    elif "DeepseekV3" in arch:
+        reasoning_parser, tool_call_parser = "deepseek-v3", "deepseekv32"
+    else:
+        return
+
+    for attr, detected in (
+        ("reasoning_parser", reasoning_parser),
+        ("tool_call_parser", tool_call_parser),
+    ):
+        if getattr(server_args, attr) == "auto":
+            server_args.override(source="template-detection", **{attr: detected})
+            logger.info(
+                f"Auto-detected --{attr.replace('_', '-')} as '{detected}' "
+                f"from model architecture '{arch}'"
+            )
+
+
+def resolve_auto_parsers(server_args) -> None:
+    """Resolve --reasoning-parser=auto and --tool-call-parser=auto before scheduler.
+
+    This performs a lightweight tokenizer load to detect parsers from the chat
+    template. Called early in engine init before scheduler subprocesses are spawned.
+    """
+    needs_reasoning = server_args.reasoning_parser == "auto"
+    needs_tool_call = server_args.tool_call_parser == "auto"
+
+    if not needs_reasoning and not needs_tool_call:
+        return
+
+    from sglang.srt.utils.hf_transformers_utils import get_tokenizer
+
+    chat_template_arg = getattr(server_args, "chat_template", None)
+    try:
+        explicit_jinja_template = _load_explicit_jinja_template(chat_template_arg)
+    except Exception as e:
+        logger.warning("Failed to load explicit Jinja chat template: %s", e)
+        explicit_jinja_template = None
+    has_explicit_template_without_detection = (
+        chat_template_arg is not None and explicit_jinja_template is None
+    )
+
+    tokenizer = None
+    try:
+        tokenizer = get_tokenizer(
+            server_args.model_path,
+            trust_remote_code=server_args.trust_remote_code,
+        )
+    except Exception as e:
+        logger.warning(f"Failed to load tokenizer for auto-detection: {e}")
+
+    template = explicit_jinja_template
+    if template is None and tokenizer is not None:
+        template = getattr(tokenizer, "chat_template", None)
+
+    force_reasoning, reasoning_config = detect_reasoning_pattern(template)
+    ctx = build_detection_context(
+        template, tokenizer, reasoning_config, force_reasoning
+    )
+    if ctx is None:
+        if has_explicit_template_without_detection:
+            logger.warning(
+                "--chat-template=%s is explicit but is not a readable Jinja template, so "
+                "parser auto-detection from chat template is not available.",
+                chat_template_arg,
+            )
+        else:
+            try:
+                _resolve_architecture_auto_parsers(server_args)
+            except Exception as e:
+                logger.warning(
+                    "Failed to load model config for architecture-based auto-detection: %s",
+                    e,
+                )
+        if needs_reasoning:
+            if server_args.reasoning_parser == "auto":
+                _disable_auto_parser(
+                    server_args, "reasoning_parser", "reasoning parser"
+                )
+        if needs_tool_call:
+            if server_args.tool_call_parser == "auto":
+                _disable_auto_parser(
+                    server_args, "tool_call_parser", "tool-call parser"
+                )
+        return
+
+    if needs_reasoning:
+        _resolve_auto_parser(
+            server_args,
+            "reasoning_parser",
+            ctx,
+            REASONING_PARSER_RULES,
+            "reasoning parser",
+        )
+
+    if needs_tool_call:
+        _resolve_auto_parser(
+            server_args,
+            "tool_call_parser",
+            ctx,
+            TOOL_CALL_PARSER_RULES,
+            "tool-call parser",
+        )

python/sglang/srt/parser/template_manager.py

@@ -0,0 +1,388 @@
+# Copyright 2023-2024 SGLang Team
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""
+Centralized template management for chat templates and completion templates.
+
+This module provides a unified interface for managing both chat conversation templates
+and code completion templates, eliminating global state and improving modularity.
+"""
+
+import json
+import logging
+import os
+from typing import Dict, Optional
+
+from sglang.srt.managers.tokenizer_manager import TokenizerManager
+from sglang.srt.parser.code_completion_parser import (
+    CompletionTemplate,
+    FimPosition,
+    completion_template_exists,
+    register_completion_template,
+    set_completion_template,
+)
+from sglang.srt.parser.conversation import (
+    Conversation,
+    SeparatorStyle,
+    chat_template_exists,
+    get_conv_template_by_model_path,
+    register_conv_template,
+)
+from sglang.srt.parser.jinja_template_utils import detect_jinja_template_content_format
+from sglang.srt.parser.template_detection import (
+    REASONING_PARSER_RULES,
+    TOOL_CALL_PARSER_RULES,
+    ReasoningToggleConfig,
+    build_detection_context,
+    detect_reasoning_pattern,
+    match_rules,
+)
+
+logger = logging.getLogger(__name__)
+
+
+class TemplateManager:
+    """
+    Centralized manager for chat and completion templates.
+
+    This class encapsulates all template-related state and operations,
+    eliminating the need for global variables and providing a clean
+    interface for template management.
+    """
+
+    def __init__(self):
+        self._chat_template_name: Optional[str] = None
+        self._completion_template_name: Optional[str] = None
+        self._jinja_template_content_format: Optional[str] = "openai"
+        self._force_reasoning: bool = False
+        self._reasoning_config: Optional[ReasoningToggleConfig] = None
+        self._suggested_reasoning_parser: Optional[str] = None
+        self._suggested_tool_call_parser: Optional[str] = None
+
+    @property
+    def chat_template_name(self) -> Optional[str]:
+        """Get the current chat template name."""
+        return self._chat_template_name
+
+    @property
+    def completion_template_name(self) -> Optional[str]:
+        """Get the current completion template name."""
+        return self._completion_template_name
+
+    @property
+    def jinja_template_content_format(self) -> Optional[str]:
+        """Get the detected template content format ('string' or 'openai' or None)."""
+        return self._jinja_template_content_format
+
+    @property
+    def force_reasoning(self) -> bool:
+        """
+        Check if the current chat template enforces reasoning/thinking.
+
+        Returns:
+            True if the template contains reasoning patterns like <think> tags
+        """
+        return self._force_reasoning
+
+    @property
+    def reasoning_config(self) -> Optional[ReasoningToggleConfig]:
+        """Get the reasoning toggle config inferred from chat template."""
+        return self._reasoning_config
+
+    @property
+    def suggested_reasoning_parser(self) -> Optional[str]:
+        """Get the auto-detected reasoning parser name, or None."""
+        return self._suggested_reasoning_parser
+
+    @property
+    def suggested_tool_call_parser(self) -> Optional[str]:
+        """Get the auto-detected tool-call parser name, or None."""
+        return self._suggested_tool_call_parser
+
+    def _run_template_detection(self, template, tokenizer) -> None:
+        """Run reasoning pattern and parser detection on a template."""
+        self._force_reasoning, self._reasoning_config = detect_reasoning_pattern(
+            template
+        )
+        # Build context once, reuse for both parser detections (avoids
+        # duplicate tokenizer.get_vocab() calls).
+        ctx = build_detection_context(
+            template, tokenizer, self._reasoning_config, self._force_reasoning
+        )
+        if ctx is None:
+            return
+        self._suggested_reasoning_parser = match_rules(
+            ctx, REASONING_PARSER_RULES, "reasoning parser"
+        )
+        self._suggested_tool_call_parser = match_rules(
+            ctx, TOOL_CALL_PARSER_RULES, "tool-call parser"
+        )
+
+    def load_chat_template(
+        self,
+        tokenizer_manager: TokenizerManager,
+        chat_template_arg: Optional[str],
+        model_path: str,
+    ) -> None:
+        """
+        Load a chat template from various sources.
+
+        Args:
+            tokenizer_manager: The tokenizer manager instance
+            chat_template_arg: Template name, file path, or None to auto-detect
+            model_path: Path to the model
+        """
+        if chat_template_arg:
+            self._load_explicit_chat_template(tokenizer_manager, chat_template_arg)
+        else:
+            # Guess chat template from model path
+            self.guess_chat_template_from_model_path(model_path)
+
+            # If no pre-defined template was found, fallback to HuggingFace template
+            if self._chat_template_name is None:
+                # Try HuggingFace template first
+                hf_template = self._resolve_hf_chat_template(tokenizer_manager)
+                if hf_template:
+                    # override the chat template
+                    if tokenizer_manager.tokenizer:
+                        tokenizer_manager.tokenizer.chat_template = hf_template
+                    self._jinja_template_content_format = (
+                        detect_jinja_template_content_format(hf_template)
+                    )
+                    logger.info(
+                        f"Using default HuggingFace chat template with detected content format: {self._jinja_template_content_format}"
+                    )
+                else:
+                    # Default to string content format if no template was found
+                    self._jinja_template_content_format = "string"
+                    logger.info(
+                        "No chat template found, defaulting to 'string' content format"
+                    )
+
+        # Detect reasoning pattern and suggest parser from chat template
+        if tokenizer_manager.tokenizer:
+            template = tokenizer_manager.tokenizer.chat_template
+            self._run_template_detection(template, tokenizer_manager.tokenizer)
+            parts = []
+            if self._reasoning_config:
+                parts.append(f"reasoning_config={self._reasoning_config}")
+            if self._suggested_reasoning_parser:
+                parts.append(f"reasoning_parser={self._suggested_reasoning_parser}")
+            if self._suggested_tool_call_parser:
+                parts.append(f"tool_call_parser={self._suggested_tool_call_parser}")
+            if parts:
+                logger.info(f"Auto-detected template features: {', '.join(parts)}")
+
+    def _load_explicit_chat_template(
+        self, tokenizer_manager: TokenizerManager, chat_template_arg: str
+    ) -> None:
+        """Load explicitly specified chat template."""
+        logger.info(f"Loading chat template from argument: {chat_template_arg}")
+
+        if chat_template_exists(chat_template_arg):
+            self._chat_template_name = chat_template_arg
+            return
+
+        if not os.path.exists(chat_template_arg):
+            raise RuntimeError(
+                f"Chat template {chat_template_arg} is not a built-in template name "
+                "or a valid chat template file path."
+            )
+
+        if chat_template_arg.endswith(".jinja"):
+            self._load_jinja_template(tokenizer_manager, chat_template_arg)
+        else:
+            self._load_json_chat_template(chat_template_arg)
+
+    def guess_chat_template_from_model_path(self, model_path: str) -> None:
+        """
+        Infer chat template name from model path.
+
+        Args:
+            model_path: Path to the model
+        """
+        template_name = get_conv_template_by_model_path(model_path)
+        if template_name is not None:
+            logger.info(f"Inferred chat template from model path: {template_name}")
+            self._chat_template_name = template_name
+
+    def load_completion_template(self, completion_template_arg: str) -> None:
+        """
+        Load completion template for code completion.
+
+        Args:
+            completion_template_arg: Template name or file path
+        """
+        logger.info(f"Loading completion template: {completion_template_arg}")
+
+        if not completion_template_exists(completion_template_arg):
+            if not os.path.exists(completion_template_arg):
+                raise RuntimeError(
+                    f"Completion template {completion_template_arg} is not a built-in template name "
+                    "or a valid completion template file path."
+                )
+
+            self._load_json_completion_template(completion_template_arg)
+        else:
+            self._completion_template_name = completion_template_arg
+
+        set_completion_template(self._completion_template_name)
+
+    def initialize_templates(
+        self,
+        tokenizer_manager: TokenizerManager,
+        model_path: str,
+        chat_template: Optional[str] = None,
+        completion_template: Optional[str] = None,
+    ) -> None:
+        """
+        Initialize all templates based on provided configuration.
+
+        Args:
+            tokenizer_manager: The tokenizer manager instance
+            model_path: Path to the model
+            chat_template: Optional chat template name/path
+            completion_template: Optional completion template name/path
+        """
+        # Load chat template
+        self.load_chat_template(tokenizer_manager, chat_template, model_path)
+
+        # Load completion template
+        if completion_template:
+            self.load_completion_template(completion_template)
+
+    def _load_jinja_template(
+        self, tokenizer_manager: TokenizerManager, template_path: str
+    ) -> None:
+        """Load a Jinja template file."""
+        with open(template_path, "r") as f:
+            chat_template = "".join(f.readlines()).strip("\n")
+        tokenizer_manager.tokenizer.chat_template = chat_template.replace("\\n", "\n")
+        self._chat_template_name = None
+        # Detect content format from the loaded template
+        self._jinja_template_content_format = detect_jinja_template_content_format(
+            chat_template
+        )
+        logger.info(
+            f"Detected user specified Jinja chat template with content format: {self._jinja_template_content_format}"
+        )
+
+    def _load_json_chat_template(self, template_path: str) -> None:
+        """Load a JSON chat template file."""
+        assert template_path.endswith(
+            ".json"
+        ), "unrecognized format of chat template file"
+
+        with open(template_path, "r") as filep:
+            template = json.load(filep)
+            try:
+                sep_style = SeparatorStyle[template["sep_style"]]
+            except KeyError:
+                raise ValueError(
+                    f"Unknown separator style: {template['sep_style']}"
+                ) from None
+
+            register_conv_template(
+                Conversation(
+                    name=template["name"],
+                    system_template=template["system"] + "\n{system_message}",
+                    system_message=template.get("system_message", ""),
+                    roles=(template["user"], template["assistant"]),
+                    sep_style=sep_style,
+                    sep=template.get("sep", "\n"),
+                    stop_str=template["stop_str"],
+                ),
+                override=True,
+            )
+        self._chat_template_name = template["name"]
+
+    def _load_json_completion_template(self, template_path: str) -> None:
+        """Load a JSON completion template file."""
+        assert template_path.endswith(
+            ".json"
+        ), "unrecognized format of completion template file"
+
+        with open(template_path, "r") as filep:
+            template = json.load(filep)
+            try:
+                fim_position = FimPosition[template["fim_position"]]
+            except KeyError:
+                raise ValueError(
+                    f"Unknown fim position: {template['fim_position']}"
+                ) from None
+
+            register_completion_template(
+                CompletionTemplate(
+                    name=template["name"],
+                    fim_begin_token=template["fim_begin_token"],
+                    fim_middle_token=template["fim_middle_token"],
+                    fim_end_token=template["fim_end_token"],
+                    fim_position=fim_position,
+                ),
+                override=True,
+            )
+        self._completion_template_name = template["name"]
+
+    def _resolve_hf_chat_template(
+        self, tokenizer_manager: TokenizerManager
+    ) -> Optional[str]:
+        try:
+            # Try (mm-)processor first, then tokenizer
+            template = (
+                getattr(tokenizer_manager.processor, "chat_template", None)
+                if tokenizer_manager.processor
+                else None
+            ) or (
+                getattr(tokenizer_manager.tokenizer, "chat_template", None)
+                if tokenizer_manager.tokenizer
+                else None
+            )
+
+            if template is None:
+                logger.warning("No HuggingFace chat template found")
+                return None
+
+            # Handle dict templates (multiple named templates)
+            if isinstance(template, dict):
+                return self._select_named_template(template, tokenizer_manager)
+
+            # Single string template
+            return template
+
+        except Exception as e:
+            logger.warning(f"Error getting chat template: {e}")
+            return None
+
+    def _select_named_template(
+        self, templates: Dict[str, str], tokenizer_manager: TokenizerManager
+    ) -> str:
+        if not templates:
+            raise ValueError("Empty templates dict provided")
+
+        available_names = list(templates.keys())
+        logger.info(f"Multiple HuggingFace chat templates available: {available_names}")
+
+        # Use specified template if provided
+        if preferred_name := tokenizer_manager.server_args.hf_chat_template_name:
+            if preferred_name not in templates:
+                raise ValueError(
+                    f"Specified template '{preferred_name}' not found. "
+                    f"Available templates: {available_names}"
+                )
+            logger.info(f"Using specified chat template: '{preferred_name}'")
+            return templates[preferred_name]
+
+        # Fallback: Use first available template
+        first_name = available_names[0]
+        logger.info(f"Using first available template: '{first_name}'")
+        return templates[first_name]