chore: import upstream snapshot with attribution

vllm/parser/engine/__init__.py

@@ -0,0 +1,17 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+"""
+Streaming parser engine framework for tool call and reasoning extraction.
+
+Instead of hand-rolling a parser for every model's tool-call / reasoning
+format, each format is declared as a ParserEngineConfig (terminals,
+states, and transitions) and a shared incremental engine handles
+streaming, ambiguity buffering, token-ID mapping, and delta computation.
+"""
+
+from vllm.parser.engine.events import EventType, SemanticEvent
+
+__all__ = [
+    "EventType",
+    "SemanticEvent",
+]

vllm/parser/engine/adapters.py

@@ -0,0 +1,210 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+"""Adapters that expose :class:`ParserEngine` through the legacy
+:class:`ReasoningParser` and :class:`ToolParser` interfaces.
+
+This lets parser engines flow through the existing serving-layer code
+paths that expect separate reasoning and tool parser instances, without
+any changes to the serving layer itself.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Iterator, Sequence
+from contextlib import contextmanager
+from typing import TYPE_CHECKING
+
+from vllm.parser.engine.parser_engine_config import ParserState
+from vllm.reasoning.abs_reasoning_parsers import ReasoningParser
+from vllm.tool_parsers.abstract_tool_parser import ToolParser
+
+if TYPE_CHECKING:
+    from vllm.entrypoints.openai.chat_completion.protocol import (
+        ChatCompletionRequest,
+    )
+    from vllm.entrypoints.openai.engine.protocol import (
+        DeltaMessage,
+        ExtractedToolCallInformation,
+    )
+    from vllm.entrypoints.openai.responses.protocol import ResponsesRequest
+    from vllm.parser.engine.parser_engine import ParserEngine
+    from vllm.tokenizers import TokenizerLike
+    from vllm.tool_parsers.utils import Tool
+
+
+class ParserEngineReasoningAdapter(ReasoningParser):
+    """Adapts a :class:`ParserEngine` to the :class:`ReasoningParser`
+    interface so parser engines can be used as reasoning parsers in the
+    existing serving code.
+
+    Subclasses set :attr:`_parser_engine_cls` to the concrete
+    :class:`ParserEngine` class.
+    """
+
+    _parser_engine_cls: type[ParserEngine]
+    engine_based_streaming: bool = True
+
+    def __init__(self, tokenizer: TokenizerLike, *args, **kwargs) -> None:
+        super().__init__(tokenizer, *args, **kwargs)
+        self._parser_engine = self._parser_engine_cls(tokenizer, **kwargs)  # type: ignore[call-arg]
+
+    @contextmanager
+    def _skip_tool_parsing(self) -> Iterator[None]:
+        saved = self._parser_engine.skip_tool_parsing
+        self._parser_engine.skip_tool_parsing = True
+        try:
+            yield
+        finally:
+            self._parser_engine.skip_tool_parsing = saved
+
+    def is_reasoning_end(self, input_ids: Sequence[int]) -> bool:
+        return self._parser_engine.is_reasoning_end(list(input_ids))
+
+    def adjust_initial_state_from_prompt(self, prompt_token_ids: Sequence[int]) -> None:
+        self._parser_engine.adjust_initial_state_from_prompt(prompt_token_ids)
+
+    def extract_content_ids(self, input_ids: list[int]) -> list[int]:
+        return self._parser_engine.extract_content_ids(input_ids)
+
+    def extract_reasoning(
+        self,
+        model_output: str,
+        request: ChatCompletionRequest | ResponsesRequest,
+    ) -> tuple[str | None, str | None]:
+        with self._skip_tool_parsing():
+            return self._parser_engine.extract_reasoning(model_output, request)
+
+    def extract_reasoning_streaming(
+        self,
+        previous_text: str,
+        current_text: str,
+        delta_text: str,
+        previous_token_ids: Sequence[int],
+        current_token_ids: Sequence[int],
+        delta_token_ids: Sequence[int],
+    ) -> DeltaMessage | None:
+        with self._skip_tool_parsing():
+            return self._parser_engine.extract_reasoning_streaming(
+                previous_text,
+                current_text,
+                delta_text,
+                previous_token_ids,
+                current_token_ids,
+                delta_token_ids,
+            )
+
+    @property
+    def reasoning_start_str(self) -> str | None:
+        return self._parser_engine.reasoning_start_str
+
+    @property
+    def reasoning_end_str(self) -> str | None:
+        return self._parser_engine.reasoning_end_str
+
+    def adjust_request(
+        self,
+        request: ChatCompletionRequest | ResponsesRequest,
+    ) -> ChatCompletionRequest | ResponsesRequest:
+        return self._parser_engine.adjust_request(request)
+
+    def has_engine_confirmed_reasoning_end(self) -> bool:
+        return self._parser_engine.reasoning_ended
+
+    def finish_streaming(self) -> DeltaMessage | None:
+        with self._skip_tool_parsing():
+            return self._parser_engine.finish_streaming()
+
+    def get_streaming_fallback_content(
+        self,
+        text: str,
+        request: ChatCompletionRequest | ResponsesRequest,
+    ) -> str | None:
+        return self._parser_engine.get_streaming_fallback_content(text, request)
+
+    def count_reasoning_tokens(self, token_ids: Sequence[int]) -> int:
+        return self._parser_engine.count_reasoning_tokens(token_ids)
+
+
+class ParserEngineToolAdapter(ToolParser):
+    """Adapts a :class:`ParserEngine` to the :class:`ToolParser` interface.
+
+    :meth:`extract_tool_calls` starts the parser engine in ``CONTENT``
+    state so it can parse reasoning-stripped content (i.e. the output of
+    :meth:`ReasoningParser.extract_reasoning`).
+
+    Subclasses set :attr:`_parser_engine_cls` to the concrete
+    :class:`ParserEngine` class.
+    """
+
+    _parser_engine_cls: type[ParserEngine]
+    engine_based_streaming: bool = True
+
+    def __init__(
+        self,
+        tokenizer: TokenizerLike,
+        tools: list[Tool] | None = None,
+        **kwargs,
+    ) -> None:
+        super().__init__(tokenizer, tools)
+        self._parser_engine = self._parser_engine_cls(tokenizer, tools, **kwargs)  # type: ignore[call-arg]
+
+    def adjust_request(
+        self,
+        request: ChatCompletionRequest | ResponsesRequest,
+    ) -> ChatCompletionRequest | ResponsesRequest:
+        request = super().adjust_request(request)
+        return self._parser_engine.adjust_request(request)
+
+    def extract_tool_calls(
+        self,
+        model_output: str,
+        request: ChatCompletionRequest,
+    ) -> ExtractedToolCallInformation:
+        return self._parser_engine.extract_tool_calls_from_content(
+            model_output, request
+        )
+
+    def extract_tool_calls_streaming(
+        self,
+        previous_text: str,
+        current_text: str,
+        delta_text: str,
+        previous_token_ids: Sequence[int],
+        current_token_ids: Sequence[int],
+        delta_token_ids: Sequence[int],
+        request: ChatCompletionRequest,
+    ) -> DeltaMessage | None:
+        engine = self._parser_engine
+        engine.initialize_streaming(initial_state=ParserState.CONTENT)
+        return engine.extract_tool_calls_streaming(
+            previous_text,
+            current_text,
+            delta_text,
+            previous_token_ids,
+            current_token_ids,
+            delta_token_ids,
+            request,
+        )
+
+    def finish_streaming(self) -> DeltaMessage | None:
+        return self._parser_engine.finish_streaming()
+
+
+def make_adapters(
+    parser_engine_cls: type[ParserEngine],
+) -> tuple[type[ParserEngineReasoningAdapter], type[ParserEngineToolAdapter]]:
+    reasoning_adapter = type(
+        f"{parser_engine_cls.__name__}ReasoningAdapter",
+        (ParserEngineReasoningAdapter,),
+        {"_parser_engine_cls": parser_engine_cls},
+    )
+    tool_adapter = type(
+        f"{parser_engine_cls.__name__}ToolAdapter",
+        (ParserEngineToolAdapter,),
+        {"_parser_engine_cls": parser_engine_cls},
+    )
+    # Let the serving layer find the adapters and call adjust_request(),
+    # which sets skip_special_tokens=False for the detokenizer.
+    parser_engine_cls.reasoning_parser_cls = reasoning_adapter  # type: ignore[attr-defined]
+    parser_engine_cls.tool_parser_cls = tool_adapter  # type: ignore[attr-defined]
+    return reasoning_adapter, tool_adapter

vllm/parser/engine/events.py

@@ -0,0 +1,26 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+"""Semantic event types emitted by the streaming parser engine."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from enum import Enum, auto
+
+
+class EventType(Enum):
+    TEXT_CHUNK = auto()
+    REASONING_START = auto()
+    REASONING_CHUNK = auto()
+    REASONING_END = auto()
+    TOOL_CALL_START = auto()
+    TOOL_NAME = auto()
+    ARG_VALUE_CHUNK = auto()
+    TOOL_CALL_END = auto()
+
+
+@dataclass(slots=True)
+class SemanticEvent:
+    type: EventType
+    value: str = ""
+    tool_index: int = -1

vllm/parser/engine/incremental_lexer.py

@@ -0,0 +1,223 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+"""Incremental text lexer that converts text chunks into terminal
+tokens, with prefix-match buffering for ambiguous boundaries."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+import regex as re
+
+CONTENT_TERMINAL = "__CONTENT__"
+
+
+@dataclass(slots=True)
+class TerminalDef:
+    name: str
+    pattern: re.Pattern[str]
+    is_literal: bool = False
+    literal: str = ""
+
+
+@dataclass(slots=True)
+class LexToken:
+    terminal: str
+    value: str
+
+
+class LexerShape:
+    """Immutable pre-computed data derived from terminal definitions.
+
+    Created once per :class:`ParserEngineConfig` and shared across all
+    :class:`IncrementalLexer` instances that use the same config.
+    """
+
+    __slots__ = (
+        "terminals",
+        "literal_strings",
+        "max_literal_len",
+        "literal_first_chars",
+        "has_only_literals",
+        "prefix_set",
+        "literals_by_first",
+    )
+
+    def __init__(self, terminals: list[TerminalDef]) -> None:
+        self.terminals = sorted(
+            terminals,
+            key=lambda t: (not t.is_literal, -len(t.pattern.pattern)),
+        )
+        literal_strings: list[tuple[str, str]] = []
+        for t in self.terminals:
+            if t.is_literal:
+                literal_strings.append((t.literal, t.name))
+
+        self.literal_strings = literal_strings
+        max_len = 0
+        for lit, _ in literal_strings:
+            if len(lit) > max_len:
+                max_len = len(lit)
+        self.max_literal_len = max_len
+        self.literal_first_chars = frozenset(
+            lit[0] for lit, _ in literal_strings if lit
+        )
+        self.has_only_literals = all(t.is_literal for t in terminals)
+
+        prefix_set: set[str] = set()
+        for lit, _ in literal_strings:
+            for i in range(1, len(lit)):
+                prefix_set.add(lit[:i])
+        self.prefix_set = frozenset(prefix_set)
+
+        by_first: dict[str, list[tuple[str, str]]] = {}
+        for lit, name in literal_strings:
+            if lit:
+                by_first.setdefault(lit[0], []).append((lit, name))
+        self.literals_by_first = by_first
+
+
+class IncrementalLexer:
+    """Converts streaming text into terminal tokens.
+
+    The key feature is **prefix-match buffering**: when the text in the
+    buffer could be the start of a multi-character terminal (e.g.
+    ``"<tool_"`` that could become ``"<tool_call>"``), the lexer holds
+    the text rather than emitting it.  When the next chunk arrives, it
+    either completes the terminal or flushes the buffered text as
+    content.
+
+    Terminals are tried in priority order (literals first, then by
+    descending priority, then by pattern length).
+    """
+
+    def __init__(
+        self,
+        terminals: list[TerminalDef] | LexerShape,
+        content_terminal: str = CONTENT_TERMINAL,
+    ) -> None:
+        if isinstance(terminals, LexerShape):
+            shape = terminals
+        else:
+            shape = LexerShape(terminals)
+        self._shape = shape
+        self.terminals = shape.terminals
+        self.content_terminal = content_terminal
+        self.buffer = ""
+
+        self._literal_strings = shape.literal_strings
+        self._max_literal_len = shape.max_literal_len
+        self._literal_first_chars = shape.literal_first_chars
+        self._has_only_literals = shape.has_only_literals
+        self._prefix_set = shape.prefix_set
+        self._literals_by_first = shape.literals_by_first
+
+    def reset(self) -> None:
+        self.buffer = ""
+
+    def feed(self, text: str) -> list[LexToken]:
+        if not self.buffer and self._has_only_literals and self._literal_first_chars:
+            for ch in text:
+                if ch in self._literal_first_chars:
+                    break
+            else:
+                return [LexToken(self.content_terminal, text)]
+        self.buffer += text
+        return self._drain()
+
+    def flush(self) -> list[LexToken]:
+        tokens: list[LexToken] = []
+        if self.buffer:
+            tokens.extend(self._drain(final=True))
+        if self.buffer:
+            tokens.append(LexToken(self.content_terminal, self.buffer))
+            self.buffer = ""
+        return tokens
+
+    def _drain(self, *, final: bool = False) -> list[LexToken]:
+        tokens: list[LexToken] = []
+        first_chars = self._literal_first_chars
+        content_terminal = self.content_terminal
+        has_only_literals = self._has_only_literals
+        literals_by_first = self._literals_by_first
+        prefix_set = self._prefix_set
+
+        while self.buffer:
+            if has_only_literals and first_chars:
+                has_potential = False
+                for ch in self.buffer:
+                    if ch in first_chars:
+                        has_potential = True
+                        break
+                if not has_potential:
+                    tokens.append(LexToken(content_terminal, self.buffer))
+                    self.buffer = ""
+                    break
+
+            best_match: tuple[str, str, int] | None = None
+
+            first = self.buffer[0]
+            for lit, name in literals_by_first.get(first, ()):
+                if self.buffer.startswith(lit) and (
+                    best_match is None or len(lit) > best_match[2]
+                ):
+                    best_match = (name, lit, len(lit))
+
+            # If the current buffer is both a complete literal and the prefix
+            # of a longer literal, wait for the next chunk. For example,
+            # "<invoke name=" should not be emitted before the next chunk
+            # proves whether this is the quoted form '<invoke name="'.
+            if self.buffer in prefix_set and not final:
+                if best_match is not None:
+                    longer_match = False
+                    for lit, _ in literals_by_first.get(first, ()):
+                        if len(lit) > best_match[2] and lit.startswith(self.buffer):
+                            longer_match = True
+                            break
+                    if not longer_match:
+                        tokens.append(LexToken(best_match[0], best_match[1]))
+                        self.buffer = self.buffer[best_match[2] :]
+                        continue
+                    break
+                else:
+                    break
+
+            if best_match is not None:
+                tokens.append(LexToken(best_match[0], best_match[1]))
+                self.buffer = self.buffer[best_match[2] :]
+            else:
+                content_end = self._find_content_boundary()
+                if content_end > 0:
+                    tokens.append(LexToken(content_terminal, self.buffer[:content_end]))
+                    self.buffer = self.buffer[content_end:]
+                else:
+                    tokens.append(LexToken(content_terminal, self.buffer[0]))
+                    self.buffer = self.buffer[1:]
+
+        return tokens
+
+    def _find_content_boundary(self) -> int:
+        buf = self.buffer
+        n = len(buf)
+        first_chars = self._literal_first_chars
+        for i in range(1, n):
+            if buf[i] not in first_chars:
+                continue
+            remaining = n - i
+            for lit, _ in self._literal_strings:
+                check_len = min(remaining, len(lit))
+                if buf[i : i + check_len] == lit[:check_len]:
+                    return i
+        return n
+
+
+def terminals_from_literals(literals: dict[str, str]) -> list[TerminalDef]:
+    return [
+        TerminalDef(
+            name=name,
+            pattern=re.compile(re.escape(lit)),
+            is_literal=True,
+            literal=lit,
+        )
+        for name, lit in literals.items()
+    ]

vllm/parser/engine/parser_engine_config.py

@@ -0,0 +1,108 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+"""Declarative configuration for model tool-call and reasoning formats.
+
+Each model format is described by a :class:`ParserEngineConfig` that specifies:
+
+* **terminals** – literal strings or regex patterns that delimit the format
+  (e.g. ``<tool_call>``, ``</think>``).
+* **token_id_terminals** – terminals that should be matched by token ID
+  rather than (or in addition to) text.
+* **transitions** – a state machine mapping
+  ``(state, terminal) → (new_state, events_to_emit)`` that drives semantic
+  event generation during streaming.
+* **content_events** – what :class:`EventType` to emit for plain content
+  (non-terminal text) in each state.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Callable
+from dataclasses import dataclass, field
+from enum import Enum, auto
+from functools import cached_property
+
+from vllm.parser.engine.events import EventType
+
+
+class ParserState(Enum):
+    CONTENT = auto()
+    REASONING = auto()
+    TOOL_PREAMBLE = auto()
+    TOOL_NAME = auto()
+    TOOL_ARGS = auto()
+    TOOL_BETWEEN = auto()
+
+
+@dataclass(frozen=True, slots=True)
+class Transition:
+    next_state: ParserState
+    events: tuple[EventType, ...] = field(default_factory=tuple)
+    skip_in_token_id_mode: bool = False
+
+
+@dataclass(frozen=True)
+class ParserEngineConfig:
+    """Declarative description of a model's tool-call / reasoning format.
+
+    The engine feeds terminals from the incremental lexer into the
+    transition table and emits the corresponding semantic events.
+    Content tokens (text between terminals) are classified by the
+    current state via ``content_events``.
+    """
+
+    name: str
+
+    terminals: dict[str, str] = field(default_factory=dict)
+
+    token_id_terminals: dict[str, str] = field(default_factory=dict)
+
+    transitions: dict[tuple[ParserState, str], Transition] = field(
+        default_factory=dict,
+    )
+
+    content_events: dict[ParserState, EventType] = field(
+        default_factory=lambda: {
+            ParserState.CONTENT: EventType.TEXT_CHUNK,
+            ParserState.REASONING: EventType.REASONING_CHUNK,
+            ParserState.TOOL_NAME: EventType.TOOL_NAME,
+            ParserState.TOOL_ARGS: EventType.ARG_VALUE_CHUNK,
+        },
+    )
+
+    initial_state: ParserState = ParserState.CONTENT
+
+    arg_converter: Callable[[str, bool], str] | None = None
+
+    stream_arg_deltas: bool = True
+
+    tool_args_json: bool = True
+
+    arg_structural_chars: frozenset[str] | None = None
+
+    # Special tokens exempt from auto-drop but not state-machine terminals.
+    preserve_tokens: frozenset[str] = field(default_factory=frozenset)
+
+    # Prevents trailing-whitespace accumulation across multi-turn conversations.
+    strip_trailing_reasoning_whitespace: bool = True
+
+    # Drop content that is entirely whitespace when tool calls follow.
+    drop_whitespace_only_content_before_tools: bool = True
+
+    # .strip() content text when tool calls are present.
+    strip_content_whitespace_with_tools: bool = True
+
+    # Reject tool calls whose names are absent from the request tools.
+    validate_tool_names: bool = False
+
+    @cached_property
+    def terminal_defs(self):
+        from vllm.parser.engine.incremental_lexer import terminals_from_literals
+
+        return terminals_from_literals(self.terminals)
+
+    @cached_property
+    def lexer_shape(self):
+        from vllm.parser.engine.incremental_lexer import LexerShape
+
+        return LexerShape(self.terminal_defs)

vllm/parser/engine/registered_adapters.py

@@ -0,0 +1,64 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+"""Concrete adapter classes for each registered parser engine.
+
+These are created via :func:`make_adapters` and exposed as module-level
+names so that :class:`ReasoningParserManager` and
+:class:`ToolParserManager` can load them lazily.
+"""
+
+from vllm.parser.deepseek_v4 import DeepSeekV4Parser
+from vllm.parser.deepseek_v32 import DeepSeekV32Parser
+from vllm.parser.engine.adapters import make_adapters
+from vllm.parser.gemma4 import Gemma4Parser
+from vllm.parser.glm47_moe import Glm47MoeParser
+from vllm.parser.kimi_k2 import KimiK2Parser
+from vllm.parser.minimax_m2 import MinimaxM2Parser
+from vllm.parser.nemotron_v3 import NemotronV3Parser
+from vllm.parser.qwen3 import Qwen3Parser
+from vllm.parser.seed_oss import SeedOssParser
+
+(
+    DeepSeekV32ParserReasoningAdapter,
+    DeepSeekV32ParserToolAdapter,
+) = make_adapters(DeepSeekV32Parser)
+
+(
+    DeepSeekV4ParserReasoningAdapter,
+    DeepSeekV4ParserToolAdapter,
+) = make_adapters(DeepSeekV4Parser)
+
+(
+    MinimaxM2ParserReasoningAdapter,
+    MinimaxM2ParserToolAdapter,
+) = make_adapters(MinimaxM2Parser)
+
+(
+    Gemma4ParserReasoningAdapter,
+    Gemma4ParserToolAdapter,
+) = make_adapters(Gemma4Parser)
+
+(
+    NemotronV3ParserReasoningAdapter,
+    NemotronV3ParserToolAdapter,
+) = make_adapters(NemotronV3Parser)
+
+(
+    Qwen3ParserReasoningAdapter,
+    Qwen3ParserToolAdapter,
+) = make_adapters(Qwen3Parser)
+
+(
+    SeedOssParserReasoningAdapter,
+    SeedOssParserToolAdapter,
+) = make_adapters(SeedOssParser)
+
+(
+    Glm47MoeParserReasoningAdapter,
+    Glm47MoeParserToolAdapter,
+) = make_adapters(Glm47MoeParser)
+
+(
+    KimiK2ParserReasoningAdapter,
+    KimiK2ParserToolAdapter,
+) = make_adapters(KimiK2Parser)

vllm/parser/engine/streaming_parser_engine.py

@@ -0,0 +1,472 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+"""Streaming parser engine that orchestrates token ID scanning,
+incremental lexing, and state-machine-driven semantic event emission."""
+
+from __future__ import annotations
+
+from collections.abc import Sequence
+from dataclasses import dataclass
+
+from vllm.parser.engine.events import EventType, SemanticEvent
+from vllm.parser.engine.incremental_lexer import (
+    CONTENT_TERMINAL,
+    IncrementalLexer,
+    LexerShape,
+    LexToken,
+    TerminalDef,
+)
+from vllm.parser.engine.parser_engine_config import (
+    ParserEngineConfig,
+    ParserState,
+    Transition,
+)
+from vllm.parser.engine.token_id_scanner import (
+    DROP_TERMINAL,
+    LexerInput,
+    PreLexedTerminal,
+    TextChunk,
+    TokenIDScanner,
+)
+
+
+@dataclass(slots=True)
+class _DropInfo:
+    lexer_shape: LexerShape
+    extra_token_ids: dict[int, str]
+
+
+def _build_drop_info(
+    config: ParserEngineConfig,
+    tokenizer,
+) -> _DropInfo | None:
+    try:
+        special_tokens: list[str] = list(tokenizer.all_special_tokens)
+        special_ids: list[int] = list(tokenizer.all_special_ids)
+    except (AttributeError, NotImplementedError):
+        return None
+
+    if not special_tokens:
+        return None
+
+    configured_texts = (
+        set(config.token_id_terminals.values())
+        | set(config.terminals.values())
+        | config.preserve_tokens
+    )
+
+    extra_token_ids: dict[int, str] = {}
+    drop_texts: set[str] = set()
+    for text, tid in zip(special_tokens, special_ids):
+        if text not in configured_texts:
+            extra_token_ids[tid] = DROP_TERMINAL
+            drop_texts.add(text)
+
+    if not drop_texts:
+        return None
+
+    import regex as re
+
+    drop_terminal_defs = [
+        TerminalDef(
+            name=DROP_TERMINAL,
+            pattern=re.compile(re.escape(text)),
+            is_literal=True,
+            literal=text,
+        )
+        for text in drop_texts
+    ]
+
+    all_terminal_defs = list(config.terminal_defs) + drop_terminal_defs
+    lexer_shape = LexerShape(all_terminal_defs)
+
+    return _DropInfo(
+        lexer_shape=lexer_shape,
+        extra_token_ids=extra_token_ids,
+    )
+
+
+class StreamingParserEngine:
+    """Consumes ``(delta_text, delta_token_ids)`` pairs and produces a
+    stream of :class:`SemanticEvent` instances.
+
+    This is the main entry point for streaming parsing.
+    Create one per request (it is stateful).
+
+    The pipeline is::
+
+        delta_text + delta_token_ids
+            → TokenIDScanner  (special token pre-lexing)
+            → IncrementalLexer  (text → terminal tokens with prefix buffering)
+            → State Machine  (terminal → semantic events)
+            → list[SemanticEvent]
+
+    Usage::
+
+        engine = StreamingParserEngine(config, tokenizer)
+        for each streaming delta:
+            events = engine.feed(delta_text, delta_token_ids)
+            # convert events to DeltaMessage
+    """
+
+    def __init__(
+        self,
+        config: ParserEngineConfig,
+        tokenizer,
+        initial_state: ParserState | None = None,
+        vocab: dict[str, int] | None = None,
+    ) -> None:
+        self.config = config
+
+        resolved_token_ids: dict[int, str] = {}
+        if tokenizer is not None:
+            if vocab is None:
+                vocab = tokenizer.get_vocab()
+            if config.token_id_terminals:
+                for terminal_name, token_text in config.token_id_terminals.items():
+                    tid = vocab.get(token_text)
+                    if tid is not None:
+                        resolved_token_ids[tid] = terminal_name
+
+        drop_info: _DropInfo | None = None
+        if tokenizer is not None:
+            drop_info = _build_drop_info(config, tokenizer)
+
+        lexer_shape = config.lexer_shape
+        if drop_info is not None:
+            resolved_token_ids.update(drop_info.extra_token_ids)
+            lexer_shape = drop_info.lexer_shape
+
+        self._resolved_token_ids = resolved_token_ids
+        self._has_drops = drop_info is not None
+
+        self._scanner = TokenIDScanner(
+            resolved_token_ids,
+            tokenizer,
+        )
+
+        self._token_id_terminal_names: frozenset[str] = frozenset(
+            resolved_token_ids.values()
+        )
+
+        self._lexer = IncrementalLexer(lexer_shape, content_terminal=CONTENT_TERMINAL)
+
+        self._tool_terminals: frozenset[str] = frozenset(
+            terminal
+            for (state, terminal), tr in config.transitions.items()
+            if tr.next_state in self._TOOL_STATES or state in self._TOOL_STATES
+        )
+
+        self.skip_tool_parsing = False
+        self.reset(initial_state=initial_state)
+
+    def _reset_args_state(self) -> None:
+        self._args_buffer: str = ""
+        self._args_safe_end: int = 0
+        self._args_brace_depth: int = 0
+        self._args_in_string: bool = False
+        self._args_escape_next: bool = False
+
+    def reset(self, initial_state: ParserState | None = None) -> None:
+        """Reset mutable state for reuse across requests.
+
+        Preserves cached immutable structures (compiled terminals,
+        resolved token IDs, lexer shape, token text cache) to avoid
+        redundant initialization work.
+        """
+        self.state = (
+            initial_state if initial_state is not None else self.config.initial_state
+        )
+        self.tool_index = -1
+        self._ever_had_token_ids = False
+        # DO NOT reset skip_tool_parsing here — callers set it before
+        # calling methods that trigger reset() (e.g. extract_reasoning),
+        # and clearing it silently breaks non-streaming tool-call-as-
+        # implicit-reasoning-end (content returns None).
+        self._scanner.reset()
+        self._lexer.reset()
+        self._reset_args_state()
+
+    def feed(
+        self,
+        delta_text: str,
+        delta_token_ids: Sequence[int],
+    ) -> list[SemanticEvent]:
+        if delta_token_ids:
+            self._ever_had_token_ids = True
+
+        # Fast path: skip scanner and lexer when the delta is plain
+        # content with no special tokens and no terminal-starting chars.
+        if (
+            delta_text
+            and not self._lexer.buffer
+            and not self._scanner._deferred_terminals
+            and self._lexer._literal_first_chars.isdisjoint(delta_text)
+        ):
+            has_special = False
+            for tid in delta_token_ids:
+                if tid in self._resolved_token_ids:
+                    has_special = True
+                    break
+            if not has_special:
+                return self._emit_for_state(delta_text)
+
+        scanner_items = self._scanner.scan(delta_text, delta_token_ids)
+
+        if len(scanner_items) == 1 and isinstance(scanner_items[0], TextChunk):
+            lex_tokens = self._lexer.feed(scanner_items[0].text)
+            if len(lex_tokens) == 1 and lex_tokens[0].terminal == CONTENT_TERMINAL:
+                text = lex_tokens[0].value
+                return self._emit_for_state(text)
+            return self._process_lex_tokens(lex_tokens)
+
+        return self._process_scanner_items(scanner_items)
+
+    def _process_scanner_items(
+        self, items: Sequence[LexerInput]
+    ) -> list[SemanticEvent]:
+        events: list[SemanticEvent] = []
+        for item in items:
+            if isinstance(item, PreLexedTerminal):
+                events.extend(self._process_lex_tokens(self._lexer.flush()))
+                events.extend(self._on_terminal(item.terminal, item.text))
+            elif isinstance(item, TextChunk):
+                events.extend(self._process_lex_tokens(self._lexer.feed(item.text)))
+        return events
+
+    def finish(self) -> list[SemanticEvent]:
+        events = self._process_scanner_items(self._scanner.flush_pending())
+
+        events.extend(self._process_lex_tokens(self._lexer.flush()))
+
+        if self._args_buffer:
+            events.append(
+                SemanticEvent(
+                    EventType.ARG_VALUE_CHUNK,
+                    value=self._args_buffer,
+                    tool_index=self.tool_index,
+                )
+            )
+            self._args_buffer = ""
+            self._args_safe_end = 0
+
+        if self.state in (
+            ParserState.TOOL_PREAMBLE,
+            ParserState.TOOL_ARGS,
+            ParserState.TOOL_NAME,
+            ParserState.TOOL_BETWEEN,
+        ):
+            if self.tool_index >= 0:
+                events.append(
+                    SemanticEvent(
+                        EventType.TOOL_CALL_END,
+                        tool_index=self.tool_index,
+                    )
+                )
+            self.state = ParserState.CONTENT
+        elif self.state == ParserState.REASONING:
+            events.append(
+                SemanticEvent(EventType.REASONING_END, tool_index=self.tool_index)
+            )
+            self.state = ParserState.CONTENT
+
+        return events
+
+    def parse_complete(self, text: str) -> list[SemanticEvent]:
+        token_ids: list[int] = []
+        events = self.feed(text, token_ids)
+        events.extend(self.finish())
+        return events
+
+    def _process_lex_tokens(self, tokens: list[LexToken]) -> list[SemanticEvent]:
+        events: list[SemanticEvent] = []
+        strict = self._token_id_terminal_names if self._ever_had_token_ids else None
+        for tok in tokens:
+            if tok.terminal == CONTENT_TERMINAL or (strict and tok.terminal in strict):
+                events.extend(self._on_content(tok.value))
+            else:
+                events.extend(self._on_terminal(tok.terminal, tok.value))
+        return events
+
+    _TOOL_STATES = frozenset(
+        {
+            ParserState.TOOL_PREAMBLE,
+            ParserState.TOOL_NAME,
+            ParserState.TOOL_ARGS,
+            ParserState.TOOL_BETWEEN,
+        }
+    )
+
+    def _on_terminal(self, terminal: str, value: str) -> list[SemanticEvent]:
+        key = (self.state, terminal)
+        transition = self.config.transitions.get(key)
+
+        if transition is None:
+            if (
+                self._has_drops
+                and terminal == DROP_TERMINAL
+                # Preserve drop tokens when skip_tool_parsing is active so
+                # the reasoning pass doesn't silently remove tokens that a
+                # later tool-call pass might need to see.
+                and not self.skip_tool_parsing
+            ):
+                return []
+            return self._emit_for_state(value)
+
+        if self.skip_tool_parsing and terminal in self._tool_terminals:
+            if EventType.REASONING_END in transition.events:
+                self.state = ParserState.CONTENT
+                return [
+                    SemanticEvent(
+                        EventType.REASONING_END,
+                        value=value,
+                        tool_index=self.tool_index,
+                    ),
+                    SemanticEvent(
+                        EventType.TEXT_CHUNK,
+                        value=value,
+                        tool_index=self.tool_index,
+                    ),
+                ]
+            content_type = self.config.content_events.get(self.state)
+            if content_type is not None:
+                return [
+                    SemanticEvent(content_type, value=value, tool_index=self.tool_index)
+                ]
+            return []
+
+        if transition.skip_in_token_id_mode and self._ever_had_token_ids:
+            return self._emit_for_state(value)
+
+        return self._apply_transition(transition, value)
+
+    def _emit_for_state(self, text: str) -> list[SemanticEvent]:
+        if self.state == ParserState.TOOL_ARGS:
+            if self.config.tool_args_json:
+                return self._feed_args_text(text)
+            return [
+                SemanticEvent(
+                    EventType.ARG_VALUE_CHUNK,
+                    value=text,
+                    tool_index=self.tool_index,
+                )
+            ]
+        content_type = self.config.content_events.get(self.state)
+        if content_type is not None:
+            return [SemanticEvent(content_type, value=text, tool_index=self.tool_index)]
+        return []
+
+    def _on_content(self, text: str) -> list[SemanticEvent]:
+        if not text:
+            return []
+        return self._emit_for_state(text)
+
+    def _apply_transition(
+        self,
+        transition: Transition,
+        value: str,
+    ) -> list[SemanticEvent]:
+        events: list[SemanticEvent] = []
+
+        if (
+            self.state == ParserState.TOOL_ARGS
+            and transition.next_state != ParserState.TOOL_ARGS
+            and self._args_buffer
+        ):
+            events.append(
+                SemanticEvent(
+                    EventType.ARG_VALUE_CHUNK,
+                    value=self._args_buffer,
+                    tool_index=self.tool_index,
+                )
+            )
+            self._args_buffer = ""
+
+        self.state = transition.next_state
+
+        for event_type in transition.events:
+            if event_type == EventType.TOOL_CALL_START:
+                self.tool_index += 1
+            events.append(
+                SemanticEvent(
+                    event_type,
+                    value=value,
+                    tool_index=self.tool_index,
+                )
+            )
+
+        if self.state == ParserState.TOOL_ARGS:
+            self._args_brace_depth = 0
+            self._args_in_string = False
+            self._args_escape_next = False
+            self._args_safe_end = 0
+
+        return events
+
+    def _feed_args_text(self, text: str) -> list[SemanticEvent]:
+        """Feed text into the JSON argument streaming buffer.
+
+        Streams argument characters incrementally while holding back
+        closing braces/brackets that might change as more input arrives.
+        """
+        events: list[SemanticEvent] = []
+        for ch in text:
+            result = self._feed_args_char(ch)
+            events.extend(result)
+        return events
+
+    def _feed_args_char(self, ch: str) -> list[SemanticEvent]:
+        self._args_buffer += ch
+
+        if self._args_escape_next:
+            self._args_escape_next = False
+            self._args_safe_end = len(self._args_buffer)
+            return self._flush_safe_args()
+
+        if self._args_in_string:
+            if ch == "\\":
+                self._args_escape_next = True
+            elif ch == '"':
+                self._args_in_string = False
+            self._args_safe_end = len(self._args_buffer)
+            return self._flush_safe_args()
+
+        if ch == '"':
+            self._args_in_string = True
+            self._args_safe_end = len(self._args_buffer)
+            return self._flush_safe_args()
+
+        if ch in ("{", "["):
+            self._args_brace_depth += 1
+            self._args_safe_end = len(self._args_buffer)
+            return self._flush_safe_args()
+
+        if ch in ("}", "]"):
+            if self._args_brace_depth > 0:
+                self._args_brace_depth -= 1
+            if self._args_brace_depth == 0:
+                return []
+            self._args_safe_end = len(self._args_buffer)
+            return self._flush_safe_args()
+
+        self._args_safe_end = len(self._args_buffer)
+        return self._flush_safe_args()
+
+    def _flush_safe_args(self) -> list[SemanticEvent]:
+        """Emit buffered argument characters up to the safe-end watermark.
+
+        Top-level closing braces are held back (safe_end not advanced)
+        until confirmed safe by a subsequent character or finish().
+        """
+        if self._args_safe_end == 0:
+            return []
+        to_emit = self._args_buffer[: self._args_safe_end]
+        self._args_buffer = self._args_buffer[self._args_safe_end :]
+        self._args_safe_end = 0
+        return [
+            SemanticEvent(
+                EventType.ARG_VALUE_CHUNK,
+                value=to_emit,
+                tool_index=self.tool_index,
+            )
+        ]

vllm/parser/engine/token_id_scanner.py

@@ -0,0 +1,296 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+"""Scan delta token IDs for special tokens and split the stream into
+pre-lexed terminals and plain text chunks."""
+
+from __future__ import annotations
+
+from collections.abc import Sequence
+from dataclasses import dataclass
+
+DROP_TERMINAL = "__DROP__"
+
+
+@dataclass(slots=True)
+class TextChunk:
+    text: str
+
+
+@dataclass(slots=True)
+class PreLexedTerminal:
+    terminal: str
+    token_id: int
+    text: str
+
+
+LexerInput = TextChunk | PreLexedTerminal
+
+
+class TokenIDScanner:
+    """Maps special token IDs in the delta to terminals.
+
+    Before text-based lexing happens, the scanner checks each token ID
+    in the delta against a mapping of ``{token_id: terminal_name}``.
+    Matched tokens are emitted as :class:`PreLexedTerminal` items;
+    everything else is grouped into :class:`TextChunk` items for the
+    incremental lexer to process.
+
+    When a terminal's text is not yet in ``delta_text`` (held back by
+    the detokenizer), the terminal is deferred until the text arrives
+    in a subsequent delta.
+    """
+
+    def __init__(
+        self,
+        token_id_to_terminal: dict[int, str],
+        tokenizer,
+    ) -> None:
+        self.token_id_to_terminal = token_id_to_terminal
+        self.tokenizer = tokenizer
+        self._token_text_cache: dict[int, str] = {}
+        self._deferred_terminals: list[PreLexedTerminal] = []
+        self._deferred_post_text: str = ""
+
+    def reset(self) -> None:
+        """Clear mutable state for reuse. Preserves the token text cache."""
+        self._deferred_terminals.clear()
+        self._deferred_post_text = ""
+
+    def _decode_token(self, token_id: int) -> str:
+        if token_id not in self._token_text_cache:
+            self._token_text_cache[token_id] = self.tokenizer.decode([token_id])
+        return self._token_text_cache[token_id]
+
+    _EMPTY: tuple[LexerInput, ...] = ()
+
+    def scan(
+        self,
+        delta_text: str,
+        delta_token_ids: Sequence[int],
+    ) -> Sequence[LexerInput]:
+        prefix_items: list[LexerInput] = []
+        effective_text = delta_text
+
+        if self._deferred_terminals:
+            prefix_items, effective_text = self._resolve_deferred(delta_text)
+
+        if not self.token_id_to_terminal:
+            if effective_text:
+                prefix_items.append(TextChunk(effective_text))
+            return prefix_items
+
+        has_special = False
+        token_id_to_terminal = self.token_id_to_terminal
+        for tid in delta_token_ids:
+            if tid in token_id_to_terminal:
+                has_special = True
+                break
+
+        if not has_special:
+            if effective_text:
+                if not prefix_items:
+                    return [TextChunk(effective_text)]
+                prefix_items.append(TextChunk(effective_text))
+            return prefix_items or self._EMPTY
+
+        token_texts = [self._decode_token(tid) for tid in delta_token_ids]
+
+        results: list[LexerInput] = []
+        text_accum: list[str] = []
+
+        for idx, tid in enumerate(delta_token_ids):
+            terminal = self.token_id_to_terminal.get(tid)
+            if terminal is not None:
+                if text_accum:
+                    joined = "".join(text_accum)
+                    if joined:
+                        results.append(TextChunk(joined))
+                    text_accum.clear()
+                results.append(PreLexedTerminal(terminal, tid, token_texts[idx]))
+            else:
+                text_accum.append(token_texts[idx])
+
+        if text_accum:
+            joined = "".join(text_accum)
+            if joined:
+                results.append(TextChunk(joined))
+
+        if effective_text:
+            results = self._recover_holdback_text(effective_text, results)
+        else:
+            # No detokenizer text to validate against — individually-decoded
+            # TextChunks are unreliable (context-dependent decoding).
+            # Defer PreLexedTerminals so the state machine doesn't
+            # transition before the preceding text has arrived.  The
+            # deferred terminals will be resolved against the actual
+            # delta_text in a subsequent scan() or flushed by finish().
+            for r in results:
+                if isinstance(r, PreLexedTerminal):
+                    self._deferred_terminals.append(r)
+            results = []
+
+        return prefix_items + results
+
+    def flush_pending(self) -> list[LexerInput]:
+        if not self._deferred_terminals and not self._deferred_post_text:
+            return []
+        results: list[LexerInput] = []
+        if self._deferred_post_text:
+            results.append(TextChunk(self._deferred_post_text))
+            self._deferred_post_text = ""
+        results.extend(self._deferred_terminals)
+        self._deferred_terminals.clear()
+        return results
+
+    def _resolve_deferred(
+        self,
+        delta_text: str,
+    ) -> tuple[list[LexerInput], str]:
+        """Resolve deferred terminals against new delta_text.
+
+        When a previous ``scan()`` deferred a terminal (its text hadn't
+        arrived yet), the next delta's text should contain that terminal's
+        text.  Split delta_text at the terminal boundary: text before
+        belongs to the previous parser state, the terminal triggers the
+        state transition, and text after belongs to the new state.
+
+        Returns ``(prefix_items, remaining_text)`` where prefix_items
+        are the resolved deferred terminals (with any preceding text)
+        and remaining_text is the unconsumed portion of delta_text that
+        should be scanned with the current delta's token IDs.
+        """
+        deferred = self._deferred_terminals
+        self._deferred_terminals = []
+
+        results: list[LexerInput] = []
+        remaining = delta_text
+
+        if self._deferred_post_text:
+            remaining = self._deferred_post_text + remaining
+            self._deferred_post_text = ""
+
+        # Duplicate-text deferred terminals resolve left-to-right via
+        # find(); correct when each terminal text appears once in sequence.
+        for terminal in deferred:
+            pos = remaining.find(terminal.text)
+            if pos > 0:
+                results.append(TextChunk(remaining[:pos]))
+                results.append(terminal)
+                remaining = remaining[pos + len(terminal.text) :]
+            elif pos == 0:
+                results.append(terminal)
+                remaining = remaining[len(terminal.text) :]
+            else:
+                # Accumulate text until terminal text arrives —
+                # only the terminal provides a reliable split point.
+                if remaining:
+                    self._deferred_post_text += remaining
+                    remaining = ""
+                self._deferred_terminals.append(terminal)
+
+        return results, remaining
+
+    def _recover_holdback_text(
+        self,
+        delta_text: str,
+        results: list[LexerInput],
+    ) -> list[LexerInput]:
+        """Recover detokenizer hold-back text not in delta_token_ids.
+
+        The detokenizer may flush previously held-back text in
+        ``delta_text`` that has no corresponding token ID in
+        ``delta_token_ids``.  This hold-back text always appears as a
+        prefix of ``delta_text``.
+        """
+        if not results:
+            return [TextChunk(delta_text)]
+
+        reconstructed = self._join_decoded_text(results)
+
+        if not reconstructed:
+            return [TextChunk(delta_text)] + results
+
+        pos = delta_text.find(reconstructed)
+        if pos > 0:
+            return [TextChunk(delta_text[:pos])] + results
+        if pos == 0:
+            return results
+
+        # Fallback: SentencePiece context-dependent decoding mismatch.
+        # Rebuild from delta_text using PreLexedTerminals as split anchors.
+        return self._rebuild_from_anchors(delta_text, results)
+
+    def _join_decoded_text(self, results: list[LexerInput]) -> str:
+        """Join TextChunk and PreLexedTerminal text into one string."""
+        parts: list[str] = []
+        for item in results:
+            if isinstance(item, (TextChunk, PreLexedTerminal)):
+                parts.append(item.text)
+        return "".join(parts)
+
+    def _rebuild_from_anchors(
+        self,
+        delta_text: str,
+        results: list[LexerInput],
+    ) -> list[LexerInput]:
+        """Rebuild results from delta_text using terminals as anchors.
+
+        When context-dependent decoding creates a mismatch between
+        individually-decoded tokens and delta_text, use
+        PreLexedTerminals as split points and reallocate text from
+        delta_text.  If a terminal's text is not found in delta_text,
+        it is deferred to the next scan() call.
+
+        Anchors are resolved right-to-left with ``rfind`` so that each
+        anchor binds to the *rightmost* available occurrence of its
+        text.  This prevents earlier literal lookalikes (e.g. a user
+        mentioning ``<tool_call>`` in prose) from stealing the position
+        of a real special-token anchor that appears later.
+
+        If the same anchor text appears multiple times as real special
+        tokens (not prose), the rightmost-first binding could misalign.
+        In practice this doesn't happen: each special token ID maps to
+        a distinct PreLexedTerminal, and duplicates in prose are resolved
+        by the token-ID filtering layer above.
+        """
+        anchors = [item for item in results if isinstance(item, PreLexedTerminal)]
+        if not anchors:
+            return [TextChunk(delta_text)]
+
+        # Resolve positions right-to-left: each anchor gets the
+        # rightmost occurrence that is still before the next anchor.
+        positions: list[int] = [-1] * len(anchors)
+        search_end = len(delta_text)
+        for i in range(len(anchors) - 1, -1, -1):
+            pos = delta_text.rfind(anchors[i].text, 0, search_end)
+            if pos >= 0:
+                positions[i] = pos
+                search_end = pos
+
+        # Build results left-to-right using the resolved positions.
+        new_results: list[LexerInput] = []
+        consumed = 0
+        for i, anchor in enumerate(anchors):
+            pos = positions[i]
+            if pos >= consumed:
+                if pos > consumed:
+                    new_results.append(TextChunk(delta_text[consumed:pos]))
+                new_results.append(anchor)
+                consumed = pos + len(anchor.text)
+            else:
+                has_later_valid = any(p >= 0 for p in positions[i + 1 :])
+                # DROP anchors (EOS, etc.) may have text that never
+                # arrives in delta_text (stripped by detokenizer).
+                # Don't defer remaining content waiting for text
+                # that will never come.
+                if (
+                    not has_later_valid
+                    and consumed < len(delta_text)
+                    and anchor.terminal != DROP_TERMINAL
+                ):
+                    self._deferred_post_text += delta_text[consumed:]
+                    consumed = len(delta_text)
+                self._deferred_terminals.append(anchor)
+        if consumed < len(delta_text):
+            new_results.append(TextChunk(delta_text[consumed:]))
+        return new_results